cli/q
Code Issues 1 Pull Requests Activity

Improved tokenizer performance

Browse Source
This commit is contained in:
Eduard Urbach
2024-07-24 12:14:51 +02:00
parent b9dd5c1b59
commit 70792acb12
2 changed files with 80 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
src/build/token/Tokenize.go
View File

@ -35,7 +35,16 @@ func Tokenize(buffer []byte) List {
i++
}
tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
kind := Invalid
switch string(buffer[position:i]) {
case "/":
kind = Div
case "/=":
kind = DivAssign
}
tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(i - position)})
} else {
position := i
@ -120,7 +129,76 @@ func Tokenize(buffer []byte) List {
i++
}
tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
kind := Invalid
switch string(buffer[position:i]) {
case "!":
kind = Not
case "!=":
kind = NotEqual
case "%":
kind = Mod
case "%=":
kind = ModAssign
case "&":
kind = And
case "&&":
kind = LogicalAnd
case "&=":
kind = AndAssign
case "*":
kind = Mul
case "*=":
kind = MulAssign
case "+":
kind = Add
case "+=":
kind = AddAssign
case "-":
kind = Sub
case "-=":
kind = SubAssign
case ".":
kind = Period
case "/":
kind = Div
case "/=":
kind = DivAssign
case ":=":
kind = Define
case "<":
kind = Less
case "<<":
kind = Shl
case "<<=":
kind = ShlAssign
case "<=":
kind = LessEqual
case "=":
kind = Assign
case "==":
kind = Equal
case ">":
kind = Greater
case ">=":
kind = GreaterEqual
case ">>":
kind = Shr
case ">>=":
kind = ShrAssign
case "^":
kind = Xor
case "^=":
kind = XorAssign
case "|":
kind = Or
case "|=":
kind = OrAssign
case "||":
kind = LogicalOr
}
tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(i - position)})
continue
}