395 lines
9.2 KiB
Go
395 lines
9.2 KiB
Go
package lexer
|
|
|
|
import (
|
|
"strings"
|
|
"unicode"
|
|
|
|
"github.com/tomwright/dasel/v3/internal/ptr"
|
|
)
|
|
|
|
type Tokenizer struct {
|
|
i int
|
|
src string
|
|
srcLen int
|
|
}
|
|
|
|
func NewTokenizer(src string) *Tokenizer {
|
|
return &Tokenizer{
|
|
i: 0,
|
|
src: src,
|
|
srcLen: len([]rune(src)),
|
|
}
|
|
}
|
|
|
|
func (p *Tokenizer) Tokenize() (Tokens, error) {
|
|
var tokens Tokens
|
|
for {
|
|
tok, err := p.Next()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if tok.Kind == EOF {
|
|
break
|
|
}
|
|
tokens = append(tokens, tok)
|
|
}
|
|
return tokens, nil
|
|
}
|
|
|
|
func (p *Tokenizer) peekRuneEqual(i int, to rune) bool {
|
|
if i >= p.srcLen {
|
|
return false
|
|
}
|
|
return rune(p.src[i]) == to
|
|
}
|
|
|
|
func (p *Tokenizer) peekRuneMatches(i int, fn func(rune) bool) bool {
|
|
if i >= p.srcLen {
|
|
return false
|
|
}
|
|
return fn(rune(p.src[i]))
|
|
}
|
|
|
|
func (p *Tokenizer) parseCurRune() (Token, error) {
|
|
// Skip over whitespace
|
|
for p.i < p.srcLen && unicode.IsSpace(rune(p.src[p.i])) {
|
|
p.i++
|
|
}
|
|
|
|
// Skip over comments
|
|
if p.src[p.i] == '/' && p.i+1 < p.srcLen && p.src[p.i+1] == '/' {
|
|
p.i += 2
|
|
for p.i < p.srcLen && p.src[p.i] != '\n' {
|
|
p.i++
|
|
}
|
|
// After skipping the comment, skip any whitespace again.
|
|
for p.i < p.srcLen && unicode.IsSpace(rune(p.src[p.i])) {
|
|
p.i++
|
|
}
|
|
if p.i >= p.srcLen {
|
|
return NewToken(EOF, "", p.i, 0), nil
|
|
}
|
|
}
|
|
|
|
switch p.src[p.i] {
|
|
case '.':
|
|
if p.peekRuneEqual(p.i+1, '.') && p.peekRuneEqual(p.i+2, '.') {
|
|
return NewToken(Spread, "...", p.i, 3), nil
|
|
}
|
|
if p.peekRuneEqual(p.i+1, '.') {
|
|
return NewToken(RecursiveDescent, "..", p.i, 2), nil
|
|
}
|
|
return NewToken(Dot, ".", p.i, 1), nil
|
|
case ',':
|
|
return NewToken(Comma, ",", p.i, 1), nil
|
|
case ':':
|
|
return NewToken(Colon, ":", p.i, 1), nil
|
|
case ';':
|
|
return NewToken(Semicolon, ";", p.i, 1), nil
|
|
case '[':
|
|
return NewToken(OpenBracket, "[", p.i, 1), nil
|
|
case ']':
|
|
return NewToken(CloseBracket, "]", p.i, 1), nil
|
|
case '(':
|
|
return NewToken(OpenParen, "(", p.i, 1), nil
|
|
case ')':
|
|
return NewToken(CloseParen, ")", p.i, 1), nil
|
|
case '{':
|
|
return NewToken(OpenCurly, "{", p.i, 1), nil
|
|
case '}':
|
|
return NewToken(CloseCurly, "}", p.i, 1), nil
|
|
case '*':
|
|
return NewToken(Star, "*", p.i, 1), nil
|
|
case '/':
|
|
return NewToken(Slash, "/", p.i, 1), nil
|
|
case '%':
|
|
return NewToken(Percent, "%", p.i, 1), nil
|
|
case '$':
|
|
if p.peekRuneMatches(p.i+1, unicode.IsLetter) || p.peekRuneEqual(p.i+1, '_') {
|
|
pos := p.i + 1
|
|
for pos < p.srcLen && (unicode.IsLetter(rune(p.src[pos])) ||
|
|
unicode.IsDigit(rune(p.src[pos])) ||
|
|
p.src[pos] == '_') {
|
|
pos++
|
|
}
|
|
return NewToken(Variable, p.src[p.i+1:pos], p.i, pos-p.i), nil
|
|
}
|
|
return NewToken(Dollar, "$", p.i, 1), nil
|
|
case '=':
|
|
if p.peekRuneEqual(p.i+1, '=') {
|
|
return NewToken(Equal, "==", p.i, 2), nil
|
|
}
|
|
if p.peekRuneEqual(p.i+1, '~') {
|
|
return NewToken(Like, "=~", p.i, 2), nil
|
|
}
|
|
return NewToken(Equals, "=", p.i, 1), nil
|
|
case '+':
|
|
if p.peekRuneEqual(p.i+1, '=') {
|
|
return NewToken(IncrementBy, "+=", p.i, 2), nil
|
|
}
|
|
if p.peekRuneEqual(p.i+1, '+') {
|
|
return NewToken(Increment, "++", p.i, 2), nil
|
|
}
|
|
return NewToken(Plus, "+", p.i, 1), nil
|
|
case '-':
|
|
if p.peekRuneEqual(p.i+1, '=') {
|
|
return NewToken(DecrementBy, "-=", p.i, 2), nil
|
|
}
|
|
if p.peekRuneEqual(p.i+1, '-') {
|
|
return NewToken(Decrement, "--", p.i, 2), nil
|
|
}
|
|
return NewToken(Dash, "-", p.i, 1), nil
|
|
case '>':
|
|
if p.peekRuneEqual(p.i+1, '=') {
|
|
return NewToken(GreaterThanOrEqual, ">=", p.i, 2), nil
|
|
}
|
|
return NewToken(GreaterThan, ">", p.i, 1), nil
|
|
case '<':
|
|
if p.peekRuneEqual(p.i+1, '=') {
|
|
return NewToken(LessThanOrEqual, "<=", p.i, 2), nil
|
|
}
|
|
return NewToken(LessThan, "<", p.i, 1), nil
|
|
case '!':
|
|
if p.peekRuneEqual(p.i+1, '=') {
|
|
return NewToken(NotEqual, "!=", p.i, 2), nil
|
|
}
|
|
if p.peekRuneEqual(p.i+1, '~') {
|
|
return NewToken(NotLike, "!~", p.i, 2), nil
|
|
}
|
|
return NewToken(Exclamation, "!", p.i, 1), nil
|
|
case '&':
|
|
if p.peekRuneEqual(p.i+1, '&') {
|
|
return NewToken(And, "&&", p.i, 2), nil
|
|
}
|
|
return Token{}, &UnexpectedTokenError{
|
|
Pos: p.i,
|
|
Token: rune(p.src[p.i]),
|
|
}
|
|
case '|':
|
|
if p.peekRuneEqual(p.i+1, '|') {
|
|
return NewToken(Or, "||", p.i, 2), nil
|
|
}
|
|
return Token{}, &UnexpectedTokenError{
|
|
Pos: p.i,
|
|
Token: rune(p.src[p.i]),
|
|
}
|
|
case '?':
|
|
if p.peekRuneEqual(p.i+1, '?') {
|
|
return NewToken(DoubleQuestionMark, "??", p.i, 2), nil
|
|
}
|
|
return NewToken(QuestionMark, "?", p.i, 1), nil
|
|
case '"', '\'':
|
|
pos := p.i
|
|
buf := make([]rune, 0)
|
|
|
|
escapeCharacters := map[rune]bool{
|
|
// The closing quote character can be escaped.
|
|
rune(p.src[p.i]): true,
|
|
'\\': true,
|
|
't': true,
|
|
'r': true,
|
|
'n': true,
|
|
}
|
|
|
|
pos++
|
|
foundCloseRune := false
|
|
for pos < p.srcLen {
|
|
if p.src[pos] == p.src[p.i] {
|
|
foundCloseRune = true
|
|
break
|
|
}
|
|
if p.src[pos] == '\\' {
|
|
// Handle escape characters.
|
|
pos++
|
|
|
|
if pos >= p.srcLen {
|
|
return Token{}, &UnexpectedEOFError{Pos: pos}
|
|
}
|
|
|
|
if p.peekRuneMatches(pos, func(r rune) bool {
|
|
_, ok := escapeCharacters[r]
|
|
return ok
|
|
}) {
|
|
switch p.src[pos] {
|
|
case '\\':
|
|
buf = append(buf, '\\')
|
|
case 't':
|
|
buf = append(buf, '\t')
|
|
case 'r':
|
|
buf = append(buf, '\r')
|
|
case 'n':
|
|
buf = append(buf, '\n')
|
|
default:
|
|
// This must be the closing quote character.
|
|
buf = append(buf, rune(p.src[pos]))
|
|
}
|
|
pos++
|
|
} else {
|
|
buf = append(buf, rune(p.src[pos]))
|
|
pos++
|
|
}
|
|
|
|
continue
|
|
}
|
|
buf = append(buf, rune(p.src[pos]))
|
|
pos++
|
|
}
|
|
if !foundCloseRune {
|
|
// We didn't find a closing quote.
|
|
if pos < p.srcLen {
|
|
// This shouldn't be possible.
|
|
return Token{}, &UnexpectedTokenError{
|
|
Pos: p.i,
|
|
Token: rune(p.src[pos]),
|
|
}
|
|
}
|
|
// This can happen if the selector ends before the closing quote.
|
|
return Token{}, &UnexpectedEOFError{
|
|
Pos: pos,
|
|
}
|
|
}
|
|
res := NewToken(String, string(buf), p.i, pos+1-p.i)
|
|
return res, nil
|
|
default:
|
|
pos := p.i
|
|
|
|
matchStr := func(pos int, m string, caseInsensitive bool, kind TokenKind) *Token {
|
|
l := len(m)
|
|
if pos+(l-1) >= p.srcLen {
|
|
return nil
|
|
}
|
|
other := p.src[pos : pos+l]
|
|
if m != other && (!caseInsensitive || !strings.EqualFold(m, other)) {
|
|
return nil
|
|
}
|
|
|
|
if pos+(l) < p.srcLen && (unicode.IsLetter(rune(p.src[pos+l])) || unicode.IsDigit(rune(p.src[pos+l]))) {
|
|
// There is a follow letter or digit.
|
|
return nil
|
|
}
|
|
|
|
return ptr.To(NewToken(kind, other, pos, l))
|
|
}
|
|
|
|
matchRegexPattern := func(pos int) *Token {
|
|
if p.src[pos] != 'r' || !p.peekRuneEqual(pos+1, '/') {
|
|
return nil
|
|
}
|
|
start := pos
|
|
pos += 2
|
|
for !p.peekRuneEqual(pos, '/') {
|
|
pos++
|
|
}
|
|
pos++
|
|
return ptr.To(NewToken(RegexPattern, p.src[start+2:pos-1], start, pos-start))
|
|
}
|
|
|
|
if t := matchStr(pos, "null", true, Null); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "true", true, Bool); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "false", true, Bool); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "elseif", false, ElseIf); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "if", false, If); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "else", false, Else); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "branch", false, Branch); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "map", false, Map); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "each", false, Each); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "filter", false, Filter); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "search", false, Search); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "sortBy", false, SortBy); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "groupBy", false, GroupBy); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "reduce", false, Reduce); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "mapValues", false, MapValues); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "any", false, Any); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "all", false, All); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "count", false, Count); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "asc", false, Asc); t != nil {
|
|
return *t, nil
|
|
}
|
|
if t := matchStr(pos, "desc", false, Desc); t != nil {
|
|
return *t, nil
|
|
}
|
|
|
|
if t := matchRegexPattern(pos); t != nil {
|
|
return *t, nil
|
|
}
|
|
|
|
if unicode.IsDigit(rune(p.src[pos])) {
|
|
// Handle whole numbers
|
|
for pos < p.srcLen && unicode.IsDigit(rune(p.src[pos])) {
|
|
pos++
|
|
}
|
|
// Handle floats
|
|
if pos < p.srcLen && p.src[pos] == '.' && pos+1 < p.srcLen && unicode.IsDigit(rune(p.src[pos+1])) {
|
|
pos++
|
|
for pos < p.srcLen && unicode.IsDigit(rune(p.src[pos])) {
|
|
pos++
|
|
}
|
|
}
|
|
return NewToken(Number, p.src[p.i:pos], p.i, pos-p.i), nil
|
|
}
|
|
|
|
if unicode.IsLetter(rune(p.src[pos])) || p.src[pos] == '_' {
|
|
for pos < p.srcLen && (unicode.IsLetter(rune(p.src[pos])) ||
|
|
unicode.IsDigit(rune(p.src[pos])) ||
|
|
p.src[pos] == '_') {
|
|
pos++
|
|
}
|
|
return NewToken(Symbol, p.src[p.i:pos], p.i, pos-p.i), nil
|
|
}
|
|
|
|
return Token{}, &UnexpectedTokenError{
|
|
Pos: p.i,
|
|
Token: rune(p.src[p.i]),
|
|
}
|
|
}
|
|
}
|
|
|
|
func (p *Tokenizer) Next() (Token, error) {
|
|
if p.i >= len(p.src) {
|
|
return NewToken(EOF, "", p.i, 0), nil
|
|
}
|
|
|
|
t, err := p.parseCurRune()
|
|
if err != nil {
|
|
return Token{}, err
|
|
}
|
|
p.i += t.Len
|
|
return t, nil
|
|
}
|