Reduced token size

src/build/token/Tokenize.go

@@ -1,20 +1,5 @@
 package token
 
-import "git.akyoto.dev/cli/q/src/build/keyword"
-
-// Pre-allocate these byte buffers so we can re-use them
-// instead of allocating a new buffer every time.
-var (
-	groupStartBytes = []byte{'('}
-	groupEndBytes   = []byte{')'}
-	blockStartBytes = []byte{'{'}
-	blockEndBytes   = []byte{'}'}
-	arrayStartBytes = []byte{'['}
-	arrayEndBytes   = []byte{']'}
-	separatorBytes  = []byte{','}
-	newLineBytes    = []byte{'\n'}
-)
-
 // Tokenize turns the file contents into a list of tokens.
 func Tokenize(buffer []byte) List {
 	var (
@@ -26,21 +11,21 @@ func Tokenize(buffer []byte) List {
 		switch buffer[i] {
 		case ' ', '\t':
 		case ',':
-			tokens = append(tokens, Token{Kind: Separator, Position: i, Bytes: separatorBytes})
+			tokens = append(tokens, Token{Kind: Separator, Position: i, Length: 1})
 		case '(':
-			tokens = append(tokens, Token{Kind: GroupStart, Position: i, Bytes: groupStartBytes})
+			tokens = append(tokens, Token{Kind: GroupStart, Position: i, Length: 1})
 		case ')':
-			tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Bytes: groupEndBytes})
+			tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Length: 1})
 		case '{':
-			tokens = append(tokens, Token{Kind: BlockStart, Position: i, Bytes: blockStartBytes})
+			tokens = append(tokens, Token{Kind: BlockStart, Position: i, Length: 1})
 		case '}':
-			tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Bytes: blockEndBytes})
+			tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Length: 1})
 		case '[':
-			tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Bytes: arrayStartBytes})
+			tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Length: 1})
 		case ']':
-			tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Bytes: arrayEndBytes})
+			tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Length: 1})
 		case '\n':
-			tokens = append(tokens, Token{Kind: NewLine, Position: i, Bytes: newLineBytes})
+			tokens = append(tokens, Token{Kind: NewLine, Position: i, Length: 1})
 		case '/':
 			if i+1 >= Position(len(buffer)) || buffer[i+1] != '/' {
 				position := i
@@ -50,7 +35,7 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]})
+				tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
 			} else {
 				position := i
 
@@ -58,7 +43,7 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				tokens = append(tokens, Token{Kind: Comment, Position: position, Bytes: buffer[position:i]})
+				tokens = append(tokens, Token{Kind: Comment, Position: position, Length: Length(i - position)})
 			}
 
 			continue
@@ -78,7 +63,7 @@ func Tokenize(buffer []byte) List {
 				i++
 			}
 
-			tokens = append(tokens, Token{Kind: String, Position: start, Bytes: buffer[start:end]})
+			tokens = append(tokens, Token{Kind: String, Position: start, Length: Length(end - start)})
 			continue
 
 		default:
@@ -91,14 +76,14 @@ func Tokenize(buffer []byte) List {
 				}
 
 				identifier := buffer[position:i]
-				keyword, isKeyword := keyword.Map[string(identifier)]
+				kind := Identifier
+				keyword, isKeyword := Keywords[string(identifier)]
 
 				if isKeyword {
-					tokens = append(tokens, Token{Kind: Keyword, Position: position, Bytes: keyword})
-				} else {
-					tokens = append(tokens, Token{Kind: Identifier, Position: position, Bytes: identifier})
+					kind = keyword
 				}
 
+				tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(len(identifier))})
 				continue
 			}
 
@@ -110,7 +95,7 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				tokens = append(tokens, Token{Kind: Number, Position: position, Bytes: buffer[position:i]})
+				tokens = append(tokens, Token{Kind: Number, Position: position, Length: Length(i - position)})
 				continue
 			}
 
@@ -122,17 +107,17 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]})
+				tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
 				continue
 			}
 
-			tokens = append(tokens, Token{Kind: Invalid, Position: i, Bytes: buffer[i : i+1]})
+			tokens = append(tokens, Token{Kind: Invalid, Position: i, Length: 1})
 		}
 
 		i++
 	}
 
-	tokens = append(tokens, Token{Kind: EOF, Position: i, Bytes: nil})
+	tokens = append(tokens, Token{Kind: EOF, Position: i, Length: 0})
 	return tokens
 }