cli/q
Code Issues Pull Requests Activity

Improved tokenizer performance

Browse Source
This commit is contained in:
Eduard Urbach 2024-07-24 12:14:51 +02:00
parent b9dd5c1b59
commit 70792acb12
Signed by: akyoto
GPG Key ID: C874F672B1AF20C0
2 changed files with 80 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/build/token/Operators.go
View File

@ -1,39 +0,0 @@
package token
// Operators is a map of all operators used in the language.
var Operators = map[string]Kind{
".": Period,
"=": Assign,
":=": Define,
"+": Add,
"-": Sub,
"*": Mul,
"/": Div,
"%": Mod,
"&": And,
"|": Or,
"^": Xor,
"<<": Shl,
">>": Shr,
"&&": LogicalAnd,
"||": LogicalOr,
"!": Not,
"==": Equal,
"!=": NotEqual,
">": Greater,
"<": Less,
">=": GreaterEqual,
"<=": LessEqual,
"+=": AddAssign,
"-=": SubAssign,
"*=": MulAssign,
"/=": DivAssign,
"%=": ModAssign,
"&=": AndAssign,
"|=": OrAssign,
"^=": XorAssign,
"<<=": ShlAssign,
">>=": ShrAssign,
"λ": Call,
"@": Array,
}

82
src/build/token/Tokenize.go
View File

@ -35,7 +35,16 @@ func Tokenize(buffer []byte) List {
i++
}
tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
kind := Invalid
switch string(buffer[position:i]) {
case "/":
kind = Div
case "/=":
kind = DivAssign
}
tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(i - position)})
} else {
position := i
@ -120,7 +129,76 @@ func Tokenize(buffer []byte) List {
i++
}
tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
kind := Invalid
switch string(buffer[position:i]) {
case "!":
kind = Not
case "!=":
kind = NotEqual
case "%":
kind = Mod
case "%=":
kind = ModAssign
case "&":
kind = And
case "&&":
kind = LogicalAnd
case "&=":
kind = AndAssign
case "*":
kind = Mul
case "*=":
kind = MulAssign
case "+":
kind = Add
case "+=":
kind = AddAssign
case "-":
kind = Sub
case "-=":
kind = SubAssign
case ".":
kind = Period
case "/":
kind = Div
case "/=":
kind = DivAssign
case ":=":
kind = Define
case "<":
kind = Less
case "<<":
kind = Shl
case "<<=":
kind = ShlAssign
case "<=":
kind = LessEqual
case "=":
kind = Assign
case "==":
kind = Equal
case ">":
kind = Greater
case ">=":
kind = GreaterEqual
case ">>":
kind = Shr
case ">>=":
kind = ShrAssign
case "^":
kind = Xor
case "^=":
kind = XorAssign
case "|":
kind = Or
case "|=":
kind = OrAssign
case "||":
kind = LogicalOr
}
tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(i - position)})
continue
}