Added tokenizer benchmark

src/build/token/Token_test.go

@@ -137,6 +137,42 @@ func TestNumber(t *testing.T) {
 	})
 }
 
+func TestOperator(t *testing.T) {
+	tokens := token.Tokenize([]byte(`+ - * / ==`))
+	assert.DeepEqual(t, tokens, token.List{
+		{
+			Kind:     token.Operator,
+			Bytes:    []byte("+"),
+			Position: 0,
+		},
+		{
+			Kind:     token.Operator,
+			Bytes:    []byte("-"),
+			Position: 2,
+		},
+		{
+			Kind:     token.Operator,
+			Bytes:    []byte("*"),
+			Position: 4,
+		},
+		{
+			Kind:     token.Operator,
+			Bytes:    []byte("/"),
+			Position: 6,
+		},
+		{
+			Kind:     token.Operator,
+			Bytes:    []byte("=="),
+			Position: 8,
+		},
+		{
+			Kind:     token.EOF,
+			Bytes:    nil,
+			Position: 10,
+		},
+	})
+}
+
 func TestSeparator(t *testing.T) {
 	tokens := token.Tokenize([]byte("a,b,c"))
 	assert.DeepEqual(t, tokens, token.List{

src/build/token/Tokenize.go

@@ -144,5 +144,10 @@ func isNumber(c byte) bool {
 }
 
 func isOperator(c byte) bool {
-	return c == '=' || c == ':' || c == '+' || c == '-' || c == '*' || c == '/' || c == '<' || c == '>' || c == '!' || c == '&' || c == '|' || c == '^' || c == '%' || c == '.'
+	switch c {
+	case '=', ':', '.', '+', '-', '*', '/', '<', '>', '&', '|', '^', '%', '!':
+		return true
+	default:
+		return false
+	}
 }

src/build/token/bench_test.go

@@ -0,0 +1,15 @@
+package token_test
+
+import (
+	"testing"
+
+	"git.akyoto.dev/cli/q/src/build/token"
+)
+
+func BenchmarkTokenize(b *testing.B) {
+	input := []byte("hello := 123\nworld := 456")
+
+	for i := 0; i < b.N; i++ {
+		token.Tokenize(input)
+	}
+}