Implemented an abstract syntax tree

src/build/token/Tokenize.go

@@ -1,5 +1,7 @@
 package token
 
+import "git.akyoto.dev/cli/q/src/build/keyword"
+
 // Pre-allocate these byte buffers so we can re-use them
 // instead of allocating a new buffer every time.
 var (
@@ -26,35 +28,35 @@ func Tokenize(buffer []byte) List {
 		case ' ', '\t':
 		// Separator
 		case ',':
-			tokens = append(tokens, Token{Separator, i, separatorBytes})
+			tokens = append(tokens, Token{Kind: Separator, Position: i, Bytes: separatorBytes})
 
 		// Parentheses start
 		case '(':
-			tokens = append(tokens, Token{GroupStart, i, groupStartBytes})
+			tokens = append(tokens, Token{Kind: GroupStart, Position: i, Bytes: groupStartBytes})
 
 		// Parentheses end
 		case ')':
-			tokens = append(tokens, Token{GroupEnd, i, groupEndBytes})
+			tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Bytes: groupEndBytes})
 
 		// Block start
 		case '{':
-			tokens = append(tokens, Token{BlockStart, i, blockStartBytes})
+			tokens = append(tokens, Token{Kind: BlockStart, Position: i, Bytes: blockStartBytes})
 
 		// Block end
 		case '}':
-			tokens = append(tokens, Token{BlockEnd, i, blockEndBytes})
+			tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Bytes: blockEndBytes})
 
 		// Array start
 		case '[':
-			tokens = append(tokens, Token{ArrayStart, i, arrayStartBytes})
+			tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Bytes: arrayStartBytes})
 
 		// Array end
 		case ']':
-			tokens = append(tokens, Token{ArrayEnd, i, arrayEndBytes})
+			tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Bytes: arrayEndBytes})
 
 		// New line
 		case '\n':
-			tokens = append(tokens, Token{NewLine, i, newLineBytes})
+			tokens = append(tokens, Token{Kind: NewLine, Position: i, Bytes: newLineBytes})
 
 		// Comment
 		case '/':
@@ -66,7 +68,7 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				tokens = append(tokens, Token{Operator, position, buffer[position:i]})
+				tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]})
 				continue
 			}
 
@@ -76,7 +78,7 @@ func Tokenize(buffer []byte) List {
 				i++
 			}
 
-			tokens = append(tokens, Token{Comment, position, buffer[position:i]})
+			tokens = append(tokens, Token{Kind: Comment, Position: position, Bytes: buffer[position:i]})
 			continue
 
 		// String
@@ -95,7 +97,7 @@ func Tokenize(buffer []byte) List {
 				i++
 			}
 
-			tokens = append(tokens, Token{String, start, buffer[start:end]})
+			tokens = append(tokens, Token{Kind: String, Position: start, Bytes: buffer[start:end]})
 			continue
 
 		default:
@@ -108,13 +110,15 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				token := Token{Identifier, position, buffer[position:i]}
+				identifier := buffer[position:i]
+				keyword, isKeyword := keyword.Map[string(identifier)]
 
-				if Keywords[string(token.Bytes)] {
-					token.Kind = Keyword
+				if isKeyword {
+					tokens = append(tokens, Token{Kind: Keyword, Position: position, Bytes: keyword})
+				} else {
+					tokens = append(tokens, Token{Kind: Identifier, Position: position, Bytes: identifier})
 				}
 
-				tokens = append(tokens, token)
 				continue
 			}
 
@@ -127,7 +131,7 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				tokens = append(tokens, Token{Number, position, buffer[position:i]})
+				tokens = append(tokens, Token{Kind: Number, Position: position, Bytes: buffer[position:i]})
 				continue
 			}
 
@@ -140,18 +144,18 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				tokens = append(tokens, Token{Operator, position, buffer[position:i]})
+				tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]})
 				continue
 			}
 
 			// Invalid character
-			tokens = append(tokens, Token{Invalid, i, buffer[i : i+1]})
+			tokens = append(tokens, Token{Kind: Invalid, Position: i, Bytes: buffer[i : i+1]})
 		}
 
 		i++
 	}
 
-	tokens = append(tokens, Token{EOF, i, nil})
+	tokens = append(tokens, Token{Kind: EOF, Position: i, Bytes: nil})
 	return tokens
 }