Added more tokenizer tests

src/token/List_test.go

@@ -30,8 +30,37 @@ func TestSplit(t *testing.T) {
 
 	assert.Nil(t, err)
 	assert.DeepEqual(t, parameters, []string{"1+2", "3*4", "5*6", "7+8"})
+}
 
-	err = tokens.Split(func(parameter token.List) error {
+func TestSplitGroups(t *testing.T) {
+	src := []byte("f(1,2),g(3,4)")
+	tokens := token.Tokenize(src)
+	parameters := []string{}
+
+	err := tokens.Split(func(parameter token.List) error {
+		parameters = append(parameters, parameter.Text(src))
+		return nil
+	})
+
+	assert.Nil(t, err)
+	assert.DeepEqual(t, parameters, []string{"f(1,2)", "g(3,4)"})
+}
+
+func TestSplitEmpty(t *testing.T) {
+	tokens := token.List{}
+
+	err := tokens.Split(func(parameter token.List) error {
+		return errors.New("error")
+	})
+
+	assert.Nil(t, err)
+}
+
+func TestSplitError(t *testing.T) {
+	src := []byte("1,2,3")
+	tokens := token.Tokenize(src)
+
+	err := tokens.Split(func(parameter token.List) error {
 		return errors.New("error")
 	})
 

src/token/Token_test.go

@@ -45,8 +45,14 @@ func TestTokenGroups(t *testing.T) {
 	assignment := token.Token{Kind: token.Assign}
 	operator := token.Token{Kind: token.Add}
 	keyword := token.Token{Kind: token.If}
+	unary := token.Token{Kind: token.Not}
+	number := token.Token{Kind: token.Number}
+	comparison := token.Token{Kind: token.Equal}
 
 	assert.True(t, assignment.IsAssignment())
 	assert.True(t, operator.IsOperator())
 	assert.True(t, keyword.IsKeyword())
+	assert.True(t, unary.IsUnaryOperator())
+	assert.True(t, number.IsNumeric())
+	assert.True(t, comparison.IsComparison())
 }

src/token/Tokenize_test.go

@@ -121,9 +121,11 @@ func TestOperator(t *testing.T) {
 }
 
 func TestOperatorAssign(t *testing.T) {
-	tokens := token.Tokenize([]byte(`a += b -= c *= d /= e %= f &= g |= h ^= i <<= j >>= k`))
+	tokens := token.Tokenize([]byte(`a = b += c -= d *= e /= f %= g &= h |= i ^= j <<= k >>= l`))
 
 	expected := []token.Kind{
+		token.Identifier,
+		token.Assign,
 		token.Identifier,
 		token.AddAssign,
 		token.Identifier,
@@ -178,6 +180,69 @@ func TestOperatorEquality(t *testing.T) {
 	}
 }
 
+func TestOperatorLogical(t *testing.T) {
+	tokens := token.Tokenize([]byte(`a && b || c`))
+
+	expected := []token.Kind{
+		token.Identifier,
+		token.LogicalAnd,
+		token.Identifier,
+		token.LogicalOr,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestDefine(t *testing.T) {
+	tokens := token.Tokenize([]byte(`a := b`))
+
+	expected := []token.Kind{
+		token.Identifier,
+		token.Define,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestPeriod(t *testing.T) {
+	tokens := token.Tokenize([]byte(`a.b.c`))
+
+	expected := []token.Kind{
+		token.Identifier,
+		token.Period,
+		token.Identifier,
+		token.Period,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestNot(t *testing.T) {
+	tokens := token.Tokenize([]byte(`!a`))
+
+	expected := []token.Kind{
+		token.Not,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
 func TestNegateFirstToken(t *testing.T) {
 	tokens := token.Tokenize([]byte(`-a`))