From 70792acb12435a1a7d53daa058577cd387a4dd8a Mon Sep 17 00:00:00 2001
From: Eduard Urbach <admin@akyoto.dev>
Date: Wed, 24 Jul 2024 12:14:51 +0200
Subject: [PATCH] Improved tokenizer performance

---
 src/build/token/Operators.go | 39 -----------------
 src/build/token/Tokenize.go  | 82 +++++++++++++++++++++++++++++++++++-
 2 files changed, 80 insertions(+), 41 deletions(-)
 delete mode 100644 src/build/token/Operators.go

diff --git a/src/build/token/Operators.go b/src/build/token/Operators.go
deleted file mode 100644
index 842fb94..0000000
--- a/src/build/token/Operators.go
+++ /dev/null
@@ -1,39 +0,0 @@
-package token
-
-// Operators is a map of all operators used in the language.
-var Operators = map[string]Kind{
-	".":   Period,
-	"=":   Assign,
-	":=":  Define,
-	"+":   Add,
-	"-":   Sub,
-	"*":   Mul,
-	"/":   Div,
-	"%":   Mod,
-	"&":   And,
-	"|":   Or,
-	"^":   Xor,
-	"<<":  Shl,
-	">>":  Shr,
-	"&&":  LogicalAnd,
-	"||":  LogicalOr,
-	"!":   Not,
-	"==":  Equal,
-	"!=":  NotEqual,
-	">":   Greater,
-	"<":   Less,
-	">=":  GreaterEqual,
-	"<=":  LessEqual,
-	"+=":  AddAssign,
-	"-=":  SubAssign,
-	"*=":  MulAssign,
-	"/=":  DivAssign,
-	"%=":  ModAssign,
-	"&=":  AndAssign,
-	"|=":  OrAssign,
-	"^=":  XorAssign,
-	"<<=": ShlAssign,
-	">>=": ShrAssign,
-	"λ":   Call,
-	"@":   Array,
-}
diff --git a/src/build/token/Tokenize.go b/src/build/token/Tokenize.go
index 23bf106..a6a34fc 100644
--- a/src/build/token/Tokenize.go
+++ b/src/build/token/Tokenize.go
@@ -35,7 +35,16 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
+				kind := Invalid
+
+				switch string(buffer[position:i]) {
+				case "/":
+					kind = Div
+				case "/=":
+					kind = DivAssign
+				}
+
+				tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(i - position)})
 			} else {
 				position := i
 
@@ -120,7 +129,76 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
+				kind := Invalid
+
+				switch string(buffer[position:i]) {
+				case "!":
+					kind = Not
+				case "!=":
+					kind = NotEqual
+				case "%":
+					kind = Mod
+				case "%=":
+					kind = ModAssign
+				case "&":
+					kind = And
+				case "&&":
+					kind = LogicalAnd
+				case "&=":
+					kind = AndAssign
+				case "*":
+					kind = Mul
+				case "*=":
+					kind = MulAssign
+				case "+":
+					kind = Add
+				case "+=":
+					kind = AddAssign
+				case "-":
+					kind = Sub
+				case "-=":
+					kind = SubAssign
+				case ".":
+					kind = Period
+				case "/":
+					kind = Div
+				case "/=":
+					kind = DivAssign
+				case ":=":
+					kind = Define
+				case "<":
+					kind = Less
+				case "<<":
+					kind = Shl
+				case "<<=":
+					kind = ShlAssign
+				case "<=":
+					kind = LessEqual
+				case "=":
+					kind = Assign
+				case "==":
+					kind = Equal
+				case ">":
+					kind = Greater
+				case ">=":
+					kind = GreaterEqual
+				case ">>":
+					kind = Shr
+				case ">>=":
+					kind = ShrAssign
+				case "^":
+					kind = Xor
+				case "^=":
+					kind = XorAssign
+				case "|":
+					kind = Or
+				case "|=":
+					kind = OrAssign
+				case "||":
+					kind = LogicalOr
+				}
+
+				tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(i - position)})
 				continue
 			}