gtsocial-umbx

lex.go (11583B)

---

 // Package html is an HTML5 lexer following the specifications at http://www.w3.org/TR/html5/syntax.html.
 package html
 
 import (
 	"strconv"
 
 	"github.com/tdewolff/parse/v2"
 )
 
 // TokenType determines the type of token, eg. a number or a semicolon.
 type TokenType uint32
 
 // TokenType values.
 const (
 	ErrorToken TokenType = iota // extra token when errors occur
 	CommentToken
 	DoctypeToken
 	StartTagToken
 	StartTagCloseToken
 	StartTagVoidToken
 	EndTagToken
 	AttributeToken
 	TextToken
 	SvgToken
 	MathToken
 )
 
 // String returns the string representation of a TokenType.
 func (tt TokenType) String() string {
 	switch tt {
 	case ErrorToken:
 		return "Error"
 	case CommentToken:
 		return "Comment"
 	case DoctypeToken:
 		return "Doctype"
 	case StartTagToken:
 		return "StartTag"
 	case StartTagCloseToken:
 		return "StartTagClose"
 	case StartTagVoidToken:
 		return "StartTagVoid"
 	case EndTagToken:
 		return "EndTag"
 	case AttributeToken:
 		return "Attribute"
 	case TextToken:
 		return "Text"
 	case SvgToken:
 		return "Svg"
 	case MathToken:
 		return "Math"
 	}
 	return "Invalid(" + strconv.Itoa(int(tt)) + ")"
 }
 
 ////////////////////////////////////////////////////////////////
 
 // Lexer is the state for the lexer.
 type Lexer struct {
 	r   *parse.Input
 	err error
 
 	rawTag Hash
 	inTag  bool
 
 	text    []byte
 	attrVal []byte
 }
 
 // NewLexer returns a new Lexer for a given io.Reader.
 func NewLexer(r *parse.Input) *Lexer {
 	return &Lexer{
 		r: r,
 	}
 }
 
 // Err returns the error encountered during lexing, this is often io.EOF but also other errors can be returned.
 func (l *Lexer) Err() error {
 	if l.err != nil {
 		return l.err
 	}
 	return l.r.Err()
 }
 
 // Text returns the textual representation of a token. This excludes delimiters and additional leading/trailing characters.
 func (l *Lexer) Text() []byte {
 	return l.text
 }
 
 // AttrVal returns the attribute value when an AttributeToken was returned from Next.
 func (l *Lexer) AttrVal() []byte {
 	return l.attrVal
 }
 
 // Next returns the next Token. It returns ErrorToken when an error was encountered. Using Err() one can retrieve the error message.
 func (l *Lexer) Next() (TokenType, []byte) {
 	l.text = nil
 	var c byte
 	if l.inTag {
 		l.attrVal = nil
 		for { // before attribute name state
 			if c = l.r.Peek(0); c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' {
 				l.r.Move(1)
 				continue
 			}
 			break
 		}
 		if c == 0 && l.r.Err() != nil {
 			return ErrorToken, nil
 		} else if c != '>' && (c != '/' || l.r.Peek(1) != '>') {
 			return AttributeToken, l.shiftAttribute()
 		}
 		l.r.Skip()
 		l.inTag = false
 		if c == '/' {
 			l.r.Move(2)
 			return StartTagVoidToken, l.r.Shift()
 		}
 		l.r.Move(1)
 		return StartTagCloseToken, l.r.Shift()
 	}
 
 	if l.rawTag != 0 {
 		if rawText := l.shiftRawText(); len(rawText) > 0 {
 			l.text = rawText
 			l.rawTag = 0
 			return TextToken, rawText
 		}
 		l.rawTag = 0
 	}
 
 	for {
 		c = l.r.Peek(0)
 		if c == '<' {
 			c = l.r.Peek(1)
 			isEndTag := c == '/' && l.r.Peek(2) != '>' && (l.r.Peek(2) != 0 || l.r.PeekErr(2) == nil)
 			if l.r.Pos() > 0 {
 				if isEndTag || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '!' || c == '?' {
 					// return currently buffered texttoken so that we can return tag next iteration
 					l.text = l.r.Shift()
 					return TextToken, l.text
 				}
 			} else if isEndTag {
 				l.r.Move(2)
 				// only endtags that are not followed by > or EOF arrive here
 				if c = l.r.Peek(0); !('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') {
 					return CommentToken, l.shiftBogusComment()
 				}
 				return EndTagToken, l.shiftEndTag()
 			} else if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' {
 				l.r.Move(1)
 				l.inTag = true
 				return l.shiftStartTag()
 			} else if c == '!' {
 				l.r.Move(2)
 				return l.readMarkup()
 			} else if c == '?' {
 				l.r.Move(1)
 				return CommentToken, l.shiftBogusComment()
 			}
 		} else if c == 0 && l.r.Err() != nil {
 			if l.r.Pos() > 0 {
 				l.text = l.r.Shift()
 				return TextToken, l.text
 			}
 			return ErrorToken, nil
 		}
 		l.r.Move(1)
 	}
 }
 
 ////////////////////////////////////////////////////////////////
 
 // The following functions follow the specifications at https://html.spec.whatwg.org/multipage/parsing.html
 
 func (l *Lexer) shiftRawText() []byte {
 	if l.rawTag == Plaintext {
 		for {
 			if l.r.Peek(0) == 0 && l.r.Err() != nil {
 				return l.r.Shift()
 			}
 			l.r.Move(1)
 		}
 	} else { // RCDATA, RAWTEXT and SCRIPT
 		for {
 			c := l.r.Peek(0)
 			if c == '<' {
 				if l.r.Peek(1) == '/' {
 					mark := l.r.Pos()
 					l.r.Move(2)
 					for {
 						if c = l.r.Peek(0); !('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') {
 							break
 						}
 						l.r.Move(1)
 					}
 					if h := ToHash(parse.ToLower(parse.Copy(l.r.Lexeme()[mark+2:]))); h == l.rawTag { // copy so that ToLower doesn't change the case of the underlying slice
 						l.r.Rewind(mark)
 						return l.r.Shift()
 					}
 				} else if l.rawTag == Script && l.r.Peek(1) == '!' && l.r.Peek(2) == '-' && l.r.Peek(3) == '-' {
 					l.r.Move(4)
 					inScript := false
 					for {
 						c := l.r.Peek(0)
 						if c == '-' && l.r.Peek(1) == '-' && l.r.Peek(2) == '>' {
 							l.r.Move(3)
 							break
 						} else if c == '<' {
 							isEnd := l.r.Peek(1) == '/'
 							if isEnd {
 								l.r.Move(2)
 							} else {
 								l.r.Move(1)
 							}
 							mark := l.r.Pos()
 							for {
 								if c = l.r.Peek(0); !('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') {
 									break
 								}
 								l.r.Move(1)
 							}
 							if h := ToHash(parse.ToLower(parse.Copy(l.r.Lexeme()[mark:]))); h == Script { // copy so that ToLower doesn't change the case of the underlying slice
 								if !isEnd {
 									inScript = true
 								} else {
 									if !inScript {
 										l.r.Rewind(mark - 2)
 										return l.r.Shift()
 									}
 									inScript = false
 								}
 							}
 						} else if c == 0 && l.r.Err() != nil {
 							return l.r.Shift()
 						} else {
 							l.r.Move(1)
 						}
 					}
 				} else {
 					l.r.Move(1)
 				}
 			} else if c == 0 && l.r.Err() != nil {
 				return l.r.Shift()
 			} else {
 				l.r.Move(1)
 			}
 		}
 	}
 }
 
 func (l *Lexer) readMarkup() (TokenType, []byte) {
 	if l.at('-', '-') {
 		l.r.Move(2)
 		for {
 			if l.r.Peek(0) == 0 && l.r.Err() != nil {
 				l.text = l.r.Lexeme()[4:]
 				return CommentToken, l.r.Shift()
 			} else if l.at('-', '-', '>') {
 				l.text = l.r.Lexeme()[4:]
 				l.r.Move(3)
 				return CommentToken, l.r.Shift()
 			} else if l.at('-', '-', '!', '>') {
 				l.text = l.r.Lexeme()[4:]
 				l.r.Move(4)
 				return CommentToken, l.r.Shift()
 			}
 			l.r.Move(1)
 		}
 	} else if l.at('[', 'C', 'D', 'A', 'T', 'A', '[') {
 		l.r.Move(7)
 		for {
 			if l.r.Peek(0) == 0 && l.r.Err() != nil {
 				l.text = l.r.Lexeme()[9:]
 				return TextToken, l.r.Shift()
 			} else if l.at(']', ']', '>') {
 				l.text = l.r.Lexeme()[9:]
 				l.r.Move(3)
 				return TextToken, l.r.Shift()
 			}
 			l.r.Move(1)
 		}
 	} else {
 		if l.atCaseInsensitive('d', 'o', 'c', 't', 'y', 'p', 'e') {
 			l.r.Move(7)
 			if l.r.Peek(0) == ' ' {
 				l.r.Move(1)
 			}
 			for {
 				if c := l.r.Peek(0); c == '>' || c == 0 && l.r.Err() != nil {
 					l.text = l.r.Lexeme()[9:]
 					if c == '>' {
 						l.r.Move(1)
 					}
 					return DoctypeToken, l.r.Shift()
 				}
 				l.r.Move(1)
 			}
 		}
 	}
 	return CommentToken, l.shiftBogusComment()
 }
 
 func (l *Lexer) shiftBogusComment() []byte {
 	for {
 		c := l.r.Peek(0)
 		if c == '>' {
 			l.text = l.r.Lexeme()[2:]
 			l.r.Move(1)
 			return l.r.Shift()
 		} else if c == 0 && l.r.Err() != nil {
 			l.text = l.r.Lexeme()[2:]
 			return l.r.Shift()
 		}
 		l.r.Move(1)
 	}
 }
 
 func (l *Lexer) shiftStartTag() (TokenType, []byte) {
 	for {
 		if c := l.r.Peek(0); c == ' ' || c == '>' || c == '/' && l.r.Peek(1) == '>' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == 0 && l.r.Err() != nil {
 			break
 		}
 		l.r.Move(1)
 	}
 	l.text = parse.ToLower(l.r.Lexeme()[1:])
 	if h := ToHash(l.text); h == Textarea || h == Title || h == Style || h == Xmp || h == Iframe || h == Script || h == Plaintext || h == Svg || h == Math {
 		if h == Svg || h == Math {
 			data := l.shiftXML(h)
 			if l.err != nil {
 				return ErrorToken, nil
 			}
 
 			l.inTag = false
 			if h == Svg {
 				return SvgToken, data
 			}
 			return MathToken, data
 		}
 		l.rawTag = h
 	}
 	return StartTagToken, l.r.Shift()
 }
 
 func (l *Lexer) shiftAttribute() []byte {
 	nameStart := l.r.Pos()
 	var c byte
 	for { // attribute name state
 		if c = l.r.Peek(0); c == ' ' || c == '=' || c == '>' || c == '/' && l.r.Peek(1) == '>' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == 0 && l.r.Err() != nil {
 			break
 		}
 		l.r.Move(1)
 	}
 	nameEnd := l.r.Pos()
 	for { // after attribute name state
 		if c = l.r.Peek(0); c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' {
 			l.r.Move(1)
 			continue
 		}
 		break
 	}
 	if c == '=' {
 		l.r.Move(1)
 		for { // before attribute value state
 			if c = l.r.Peek(0); c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' {
 				l.r.Move(1)
 				continue
 			}
 			break
 		}
 		attrPos := l.r.Pos()
 		delim := c
 		if delim == '"' || delim == '\'' { // attribute value single- and double-quoted state
 			l.r.Move(1)
 			for {
 				c := l.r.Peek(0)
 				if c == delim {
 					l.r.Move(1)
 					break
 				} else if c == 0 && l.r.Err() != nil {
 					break
 				}
 				l.r.Move(1)
 			}
 		} else { // attribute value unquoted state
 			for {
 				if c := l.r.Peek(0); c == ' ' || c == '>' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == 0 && l.r.Err() != nil {
 					break
 				}
 				l.r.Move(1)
 			}
 		}
 		l.attrVal = l.r.Lexeme()[attrPos:]
 	} else {
 		l.r.Rewind(nameEnd)
 		l.attrVal = nil
 	}
 	l.text = parse.ToLower(l.r.Lexeme()[nameStart:nameEnd])
 	return l.r.Shift()
 }
 
 func (l *Lexer) shiftEndTag() []byte {
 	for {
 		c := l.r.Peek(0)
 		if c == '>' {
 			l.text = l.r.Lexeme()[2:]
 			l.r.Move(1)
 			break
 		} else if c == 0 && l.r.Err() != nil {
 			l.text = l.r.Lexeme()[2:]
 			break
 		}
 		l.r.Move(1)
 	}
 
 	end := len(l.text)
 	for end > 0 {
 		if c := l.text[end-1]; c == ' ' || c == '\t' || c == '\n' || c == '\r' {
 			end--
 			continue
 		}
 		break
 	}
 	l.text = l.text[:end]
 	return parse.ToLower(l.r.Shift())
 }
 
 // shiftXML parses the content of a svg or math tag according to the XML 1.1 specifications, including the tag itself.
 // So far we have already parsed `<svg` or `<math`.
 func (l *Lexer) shiftXML(rawTag Hash) []byte {
 	inQuote := false
 	for {
 		c := l.r.Peek(0)
 		if c == '"' {
 			inQuote = !inQuote
 			l.r.Move(1)
 		} else if c == '<' && !inQuote && l.r.Peek(1) == '/' {
 			mark := l.r.Pos()
 			l.r.Move(2)
 			for {
 				if c = l.r.Peek(0); !('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') {
 					break
 				}
 				l.r.Move(1)
 			}
 			if h := ToHash(parse.ToLower(parse.Copy(l.r.Lexeme()[mark+2:]))); h == rawTag { // copy so that ToLower doesn't change the case of the underlying slice
 				break
 			}
 		} else if c == 0 {
 			if l.r.Err() == nil {
 				l.err = parse.NewErrorLexer(l.r, "HTML parse error: unexpected NULL character")
 			}
 			return l.r.Shift()
 		} else {
 			l.r.Move(1)
 		}
 	}
 
 	for {
 		c := l.r.Peek(0)
 		if c == '>' {
 			l.r.Move(1)
 			break
 		} else if c == 0 {
 			if l.r.Err() == nil {
 				l.err = parse.NewErrorLexer(l.r, "HTML parse error: unexpected NULL character")
 			}
 			return l.r.Shift()
 		}
 		l.r.Move(1)
 	}
 	return l.r.Shift()
 }
 
 ////////////////////////////////////////////////////////////////
 
 func (l *Lexer) at(b ...byte) bool {
 	for i, c := range b {
 		if l.r.Peek(i) != c {
 			return false
 		}
 	}
 	return true
 }
 
 func (l *Lexer) atCaseInsensitive(b ...byte) bool {
 	for i, c := range b {
 		if l.r.Peek(i) != c && (l.r.Peek(i)+('a'-'A')) != c {
 			return false
 		}
 	}
 	return true
 }