135 lines
1.8 KiB
Go
135 lines
1.8 KiB
Go
package lexer
|
|
|
|
import (
|
|
"fmt"
|
|
"slices"
|
|
)
|
|
|
|
type TokenKind int
|
|
|
|
func TokenKinds(tk ...TokenKind) []TokenKind {
|
|
return tk
|
|
}
|
|
|
|
const (
|
|
EOF TokenKind = iota
|
|
Symbol
|
|
Comma
|
|
Colon
|
|
OpenBracket // [
|
|
CloseBracket // ]
|
|
OpenCurly
|
|
CloseCurly
|
|
OpenParen
|
|
CloseParen
|
|
Equal // ==
|
|
Equals // =
|
|
NotEqual // !=
|
|
And
|
|
Or
|
|
Like // =~
|
|
NotLike // !~
|
|
String
|
|
Number
|
|
Bool
|
|
Plus
|
|
Increment
|
|
IncrementBy
|
|
Dash
|
|
Decrement
|
|
DecrementBy
|
|
Star
|
|
Slash
|
|
Percent
|
|
Dot
|
|
Spread // ...
|
|
RecursiveDescent // ..
|
|
Dollar
|
|
Variable
|
|
GreaterThan
|
|
GreaterThanOrEqual
|
|
LessThan
|
|
LessThanOrEqual
|
|
Exclamation
|
|
Null
|
|
If
|
|
Else
|
|
ElseIf
|
|
Branch
|
|
Map
|
|
Each
|
|
Filter
|
|
Search
|
|
RegexPattern
|
|
SortBy
|
|
GroupBy
|
|
Reduce
|
|
MapValues
|
|
Any
|
|
All
|
|
Count
|
|
Asc
|
|
Desc
|
|
QuestionMark
|
|
DoubleQuestionMark
|
|
Semicolon
|
|
)
|
|
|
|
type Tokens []Token
|
|
|
|
func (tt Tokens) Split(kind TokenKind) []Tokens {
|
|
var res []Tokens
|
|
var cur Tokens
|
|
for _, t := range tt {
|
|
if t.Kind == kind {
|
|
if len(cur) > 0 {
|
|
res = append(res, cur)
|
|
}
|
|
cur = nil
|
|
continue
|
|
}
|
|
cur = append(cur, t)
|
|
}
|
|
if len(cur) > 0 {
|
|
res = append(res, cur)
|
|
}
|
|
return res
|
|
}
|
|
|
|
type Token struct {
|
|
Kind TokenKind
|
|
Value string
|
|
Pos int
|
|
Len int
|
|
}
|
|
|
|
func NewToken(kind TokenKind, value string, pos int, len int) Token {
|
|
return Token{
|
|
Kind: kind,
|
|
Value: value,
|
|
Pos: pos,
|
|
Len: len,
|
|
}
|
|
}
|
|
|
|
func (t Token) IsKind(kind ...TokenKind) bool {
|
|
return slices.Contains(kind, t.Kind)
|
|
}
|
|
|
|
type UnexpectedTokenError struct {
|
|
Pos int
|
|
Token rune
|
|
}
|
|
|
|
func (e *UnexpectedTokenError) Error() string {
|
|
return fmt.Sprintf("failed to tokenize: unexpected token: %s at position %d.", string(e.Token), e.Pos)
|
|
}
|
|
|
|
type UnexpectedEOFError struct {
|
|
Pos int
|
|
}
|
|
|
|
func (e *UnexpectedEOFError) Error() string {
|
|
return fmt.Sprintf("failed to tokenize: unexpected EOF at position %d.", e.Pos)
|
|
}
|