package token_test import ( "testing" "git.akyoto.dev/cli/q/src/token" "git.akyoto.dev/go/assert" ) func TestFunction(t *testing.T) { tokens := token.Tokenize([]byte("main(){}")) expected := []token.Kind{ token.Identifier, token.GroupStart, token.GroupEnd, token.BlockStart, token.BlockEnd, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestKeyword(t *testing.T) { tokens := token.Tokenize([]byte("assert if import else extern loop return struct switch")) expected := []token.Kind{ token.Assert, token.If, token.Import, token.Else, token.Extern, token.Loop, token.Return, token.Struct, token.Switch, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestArray(t *testing.T) { tokens := token.Tokenize([]byte("array[i]")) expected := []token.Kind{ token.Identifier, token.ArrayStart, token.Identifier, token.ArrayEnd, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestNewline(t *testing.T) { tokens := token.Tokenize([]byte("\n\n")) expected := []token.Kind{ token.NewLine, token.NewLine, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestNumber(t *testing.T) { tokens := token.Tokenize([]byte(`123 456`)) expected := []token.Kind{ token.Number, token.Number, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestOperator(t *testing.T) { tokens := token.Tokenize([]byte(`a + b - c * d / e % f << g >> h & i | j ^ k`)) expected := []token.Kind{ token.Identifier, token.Add, token.Identifier, token.Sub, token.Identifier, token.Mul, token.Identifier, token.Div, token.Identifier, token.Mod, token.Identifier, token.Shl, token.Identifier, token.Shr, token.Identifier, token.And, token.Identifier, token.Or, token.Identifier, token.Xor, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestOperatorAssign(t *testing.T) { tokens := token.Tokenize([]byte(`a = b += c -= d *= e /= f %= g &= h |= i ^= j <<= k >>= l`)) expected := []token.Kind{ token.Identifier, token.Assign, token.Identifier, token.AddAssign, token.Identifier, token.SubAssign, token.Identifier, token.MulAssign, token.Identifier, token.DivAssign, token.Identifier, token.ModAssign, token.Identifier, token.AndAssign, token.Identifier, token.OrAssign, token.Identifier, token.XorAssign, token.Identifier, token.ShlAssign, token.Identifier, token.ShrAssign, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestOperatorEquality(t *testing.T) { tokens := token.Tokenize([]byte(`a == b != c <= d >= e < f > g`)) expected := []token.Kind{ token.Identifier, token.Equal, token.Identifier, token.NotEqual, token.Identifier, token.LessEqual, token.Identifier, token.GreaterEqual, token.Identifier, token.Less, token.Identifier, token.Greater, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestOperatorLogical(t *testing.T) { tokens := token.Tokenize([]byte(`a && b || c`)) expected := []token.Kind{ token.Identifier, token.LogicalAnd, token.Identifier, token.LogicalOr, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestDefine(t *testing.T) { tokens := token.Tokenize([]byte(`a := b`)) expected := []token.Kind{ token.Identifier, token.Define, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestPeriod(t *testing.T) { tokens := token.Tokenize([]byte(`a.b.c`)) expected := []token.Kind{ token.Identifier, token.Period, token.Identifier, token.Period, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestNot(t *testing.T) { tokens := token.Tokenize([]byte(`!a`)) expected := []token.Kind{ token.Not, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestNegateFirstToken(t *testing.T) { tokens := token.Tokenize([]byte(`-a`)) expected := []token.Kind{ token.Negate, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestNegateAfterGroupStart(t *testing.T) { tokens := token.Tokenize([]byte(`(-a)`)) expected := []token.Kind{ token.GroupStart, token.Negate, token.Identifier, token.GroupEnd, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestNegateSub(t *testing.T) { tokens := token.Tokenize([]byte(`-a-b`)) expected := []token.Kind{ token.Negate, token.Identifier, token.Sub, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestNegateAfterOperator(t *testing.T) { tokens := token.Tokenize([]byte(`-a + -b`)) expected := []token.Kind{ token.Negate, token.Identifier, token.Add, token.Negate, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestNegateNumber(t *testing.T) { tokens := token.Tokenize([]byte(`-1`)) expected := []token.Kind{ token.Number, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestBinaryNumber(t *testing.T) { tokens := token.Tokenize([]byte(`0b1010`)) expected := []token.Kind{ token.Number, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestOctalNumber(t *testing.T) { tokens := token.Tokenize([]byte(`0o755`)) expected := []token.Kind{ token.Number, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestHexadecimalNumber(t *testing.T) { tokens := token.Tokenize([]byte(`0xCAFE`)) expected := []token.Kind{ token.Number, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestStandaloneZero(t *testing.T) { tokens := token.Tokenize([]byte(`0`)) expected := []token.Kind{ token.Number, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestLeadingZero(t *testing.T) { tokens := token.Tokenize([]byte(`0123`)) expected := []token.Kind{ token.Number, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestSeparator(t *testing.T) { tokens := token.Tokenize([]byte("a,b,c")) expected := []token.Kind{ token.Identifier, token.Separator, token.Identifier, token.Separator, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestComment(t *testing.T) { tokens := token.Tokenize([]byte("// Hello\n// World")) expected := []token.Kind{ token.Comment, token.NewLine, token.Comment, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } tokens = token.Tokenize([]byte("// Hello\n")) expected = []token.Kind{ token.Comment, token.NewLine, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } tokens = token.Tokenize([]byte(`// Hello`)) expected = []token.Kind{ token.Comment, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } tokens = token.Tokenize([]byte(`//`)) expected = []token.Kind{ token.Comment, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } tokens = token.Tokenize([]byte(`/`)) expected = []token.Kind{ token.Div, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestInvalid(t *testing.T) { tokens := token.Tokenize([]byte(`##`)) expected := []token.Kind{ token.Invalid, token.Invalid, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestString(t *testing.T) { tokens := token.Tokenize([]byte(`"Hello" "World"`)) expected := []token.Kind{ token.String, token.String, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestStringMultiline(t *testing.T) { tokens := token.Tokenize([]byte("\"Hello\nWorld\"")) expected := []token.Kind{ token.String, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestStringEOF(t *testing.T) { tokens := token.Tokenize([]byte(`"EOF`)) expected := []token.Kind{ token.String, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestReturnType(t *testing.T) { tokens := token.Tokenize([]byte("()->")) expected := []token.Kind{ token.GroupStart, token.GroupEnd, token.ReturnType, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestMinusAtEOF(t *testing.T) { tokens := token.Tokenize([]byte("1-")) expected := []token.Kind{ token.Number, token.Sub, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestRune(t *testing.T) { tokens := token.Tokenize([]byte("'a'")) expected := []token.Kind{ token.Rune, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } }