Added tokenizer benchmark

This commit is contained in:
Eduard Urbach 2024-06-26 18:49:03 +02:00
parent 49b75dbda4
commit 3268f7a7ee
Signed by: akyoto
GPG Key ID: C874F672B1AF20C0
3 changed files with 57 additions and 1 deletions

View File

@ -137,6 +137,42 @@ func TestNumber(t *testing.T) {
})
}
func TestOperator(t *testing.T) {
tokens := token.Tokenize([]byte(`+ - * / ==`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Operator,
Bytes: []byte("+"),
Position: 0,
},
{
Kind: token.Operator,
Bytes: []byte("-"),
Position: 2,
},
{
Kind: token.Operator,
Bytes: []byte("*"),
Position: 4,
},
{
Kind: token.Operator,
Bytes: []byte("/"),
Position: 6,
},
{
Kind: token.Operator,
Bytes: []byte("=="),
Position: 8,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 10,
},
})
}
func TestSeparator(t *testing.T) {
tokens := token.Tokenize([]byte("a,b,c"))
assert.DeepEqual(t, tokens, token.List{

View File

@ -144,5 +144,10 @@ func isNumber(c byte) bool {
}
func isOperator(c byte) bool {
return c == '=' || c == ':' || c == '+' || c == '-' || c == '*' || c == '/' || c == '<' || c == '>' || c == '!' || c == '&' || c == '|' || c == '^' || c == '%' || c == '.'
switch c {
case '=', ':', '.', '+', '-', '*', '/', '<', '>', '&', '|', '^', '%', '!':
return true
default:
return false
}
}

View File

@ -0,0 +1,15 @@
package token_test
import (
"testing"
"git.akyoto.dev/cli/q/src/build/token"
)
func BenchmarkTokenize(b *testing.B) {
input := []byte("hello := 123\nworld := 456")
for i := 0; i < b.N; i++ {
token.Tokenize(input)
}
}