Improved tokenizer performance
This commit is contained in:
parent
b9dd5c1b59
commit
70792acb12
@ -1,39 +0,0 @@
|
||||
package token
|
||||
|
||||
// Operators is a map of all operators used in the language.
|
||||
var Operators = map[string]Kind{
|
||||
".": Period,
|
||||
"=": Assign,
|
||||
":=": Define,
|
||||
"+": Add,
|
||||
"-": Sub,
|
||||
"*": Mul,
|
||||
"/": Div,
|
||||
"%": Mod,
|
||||
"&": And,
|
||||
"|": Or,
|
||||
"^": Xor,
|
||||
"<<": Shl,
|
||||
">>": Shr,
|
||||
"&&": LogicalAnd,
|
||||
"||": LogicalOr,
|
||||
"!": Not,
|
||||
"==": Equal,
|
||||
"!=": NotEqual,
|
||||
">": Greater,
|
||||
"<": Less,
|
||||
">=": GreaterEqual,
|
||||
"<=": LessEqual,
|
||||
"+=": AddAssign,
|
||||
"-=": SubAssign,
|
||||
"*=": MulAssign,
|
||||
"/=": DivAssign,
|
||||
"%=": ModAssign,
|
||||
"&=": AndAssign,
|
||||
"|=": OrAssign,
|
||||
"^=": XorAssign,
|
||||
"<<=": ShlAssign,
|
||||
">>=": ShrAssign,
|
||||
"λ": Call,
|
||||
"@": Array,
|
||||
}
|
@ -35,7 +35,16 @@ func Tokenize(buffer []byte) List {
|
||||
i++
|
||||
}
|
||||
|
||||
tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
|
||||
kind := Invalid
|
||||
|
||||
switch string(buffer[position:i]) {
|
||||
case "/":
|
||||
kind = Div
|
||||
case "/=":
|
||||
kind = DivAssign
|
||||
}
|
||||
|
||||
tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(i - position)})
|
||||
} else {
|
||||
position := i
|
||||
|
||||
@ -120,7 +129,76 @@ func Tokenize(buffer []byte) List {
|
||||
i++
|
||||
}
|
||||
|
||||
tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
|
||||
kind := Invalid
|
||||
|
||||
switch string(buffer[position:i]) {
|
||||
case "!":
|
||||
kind = Not
|
||||
case "!=":
|
||||
kind = NotEqual
|
||||
case "%":
|
||||
kind = Mod
|
||||
case "%=":
|
||||
kind = ModAssign
|
||||
case "&":
|
||||
kind = And
|
||||
case "&&":
|
||||
kind = LogicalAnd
|
||||
case "&=":
|
||||
kind = AndAssign
|
||||
case "*":
|
||||
kind = Mul
|
||||
case "*=":
|
||||
kind = MulAssign
|
||||
case "+":
|
||||
kind = Add
|
||||
case "+=":
|
||||
kind = AddAssign
|
||||
case "-":
|
||||
kind = Sub
|
||||
case "-=":
|
||||
kind = SubAssign
|
||||
case ".":
|
||||
kind = Period
|
||||
case "/":
|
||||
kind = Div
|
||||
case "/=":
|
||||
kind = DivAssign
|
||||
case ":=":
|
||||
kind = Define
|
||||
case "<":
|
||||
kind = Less
|
||||
case "<<":
|
||||
kind = Shl
|
||||
case "<<=":
|
||||
kind = ShlAssign
|
||||
case "<=":
|
||||
kind = LessEqual
|
||||
case "=":
|
||||
kind = Assign
|
||||
case "==":
|
||||
kind = Equal
|
||||
case ">":
|
||||
kind = Greater
|
||||
case ">=":
|
||||
kind = GreaterEqual
|
||||
case ">>":
|
||||
kind = Shr
|
||||
case ">>=":
|
||||
kind = ShrAssign
|
||||
case "^":
|
||||
kind = Xor
|
||||
case "^=":
|
||||
kind = XorAssign
|
||||
case "|":
|
||||
kind = Or
|
||||
case "|=":
|
||||
kind = OrAssign
|
||||
case "||":
|
||||
kind = LogicalOr
|
||||
}
|
||||
|
||||
tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(i - position)})
|
||||
continue
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user