Added a tokenizer

src/token/Tokenize.go

@@ -0,0 +1,112 @@
+package token
+
+import "git.akyoto.dev/cli/q/src/keywords"
+
+// Pre-allocate these byte buffers so we can re-use them
+// instead of allocating a new buffer every time.
+var (
+	groupStartBytes = []byte{'('}
+	groupEndBytes   = []byte{')'}
+	blockStartBytes = []byte{'{'}
+	blockEndBytes   = []byte{'}'}
+	arrayStartBytes = []byte{'['}
+	arrayEndBytes   = []byte{']'}
+	separatorBytes  = []byte{','}
+	newLineBytes    = []byte{'\n'}
+)
+
+// Tokenize turns the file contents into a list of tokens.
+func Tokenize(buffer []byte) List {
+	var (
+		i      int
+		c      byte
+		tokens = make(List, 0, len(buffer)/2)
+	)
+
+	for i < len(buffer) {
+		c = buffer[i]
+
+		switch {
+		// Identifiers
+		case isIdentifierStart(c):
+			position := i
+			i++
+
+			for i < len(buffer) && isIdentifier(buffer[i]) {
+				i++
+			}
+
+			token := Token{
+				Identifier,
+				position,
+				buffer[position:i],
+			}
+
+			if keywords.All[string(token.Bytes)] {
+				token.Kind = Keyword
+			}
+
+			tokens = append(tokens, token)
+			i--
+
+		// Texts
+		case c == '"':
+			i++
+			position := i
+
+			for i < len(buffer) && buffer[i] != '"' {
+				i++
+			}
+
+			tokens = append(tokens, Token{
+				Text,
+				position,
+				buffer[position:i],
+			})
+
+		// Parentheses start
+		case c == '(':
+			tokens = append(tokens, Token{GroupStart, i, groupStartBytes})
+
+		// Parentheses end
+		case c == ')':
+			tokens = append(tokens, Token{GroupEnd, i, groupEndBytes})
+
+		// Block start
+		case c == '{':
+			tokens = append(tokens, Token{BlockStart, i, blockStartBytes})
+
+		// Block end
+		case c == '}':
+			tokens = append(tokens, Token{BlockEnd, i, blockEndBytes})
+
+		// Array start
+		case c == '[':
+			tokens = append(tokens, Token{ArrayStart, i, arrayStartBytes})
+
+		// Array end
+		case c == ']':
+			tokens = append(tokens, Token{ArrayEnd, i, arrayEndBytes})
+
+		// Separator
+		case c == ',':
+			tokens = append(tokens, Token{Separator, i, separatorBytes})
+
+		// New line
+		case c == '\n':
+			tokens = append(tokens, Token{NewLine, i, newLineBytes})
+		}
+
+		i++
+	}
+
+	return tokens
+}
+
+func isIdentifierStart(c byte) bool {
+	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
+}
+
+func isIdentifier(c byte) bool {
+	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')
+}