Improved tokenizer benchmarks

src/build/token/bench_test.go

@@ -1,15 +1,26 @@
 package token_test
 
 import (
+	"bytes"
 	"testing"
 
 	"git.akyoto.dev/cli/q/src/build/token"
 )
 
-func BenchmarkTokenize(b *testing.B) {
+func BenchmarkLines(b *testing.B) {
-	input := []byte("hello := 123\nworld := 456")
+	b.Run("__1", bench(1))
+	b.Run("_10", bench(10))
+	b.Run("100", bench(100))
+}
 
-	for i := 0; i < b.N; i++ {
+func bench(n int) func(b *testing.B) {
-		token.Tokenize(input)
+	line := []byte("hello := 123\n")
+
+	return func(b *testing.B) {
+		input := bytes.Repeat(line, n)
+
+		for i := 0; i < b.N; i++ {
+			token.Tokenize(input)
+		}
 	}
 }