Added more tokenizer tests

src/token/Tokenize_test.go

@@ -121,9 +121,11 @@ func TestOperator(t *testing.T) {
 }
 
 func TestOperatorAssign(t *testing.T) {
-	tokens := token.Tokenize([]byte(`a += b -= c *= d /= e %= f &= g |= h ^= i <<= j >>= k`))
+	tokens := token.Tokenize([]byte(`a = b += c -= d *= e /= f %= g &= h |= i ^= j <<= k >>= l`))
 
 	expected := []token.Kind{
+		token.Identifier,
+		token.Assign,
 		token.Identifier,
 		token.AddAssign,
 		token.Identifier,
@@ -178,6 +180,69 @@ func TestOperatorEquality(t *testing.T) {
 	}
 }
 
+func TestOperatorLogical(t *testing.T) {
+	tokens := token.Tokenize([]byte(`a && b || c`))
+
+	expected := []token.Kind{
+		token.Identifier,
+		token.LogicalAnd,
+		token.Identifier,
+		token.LogicalOr,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestDefine(t *testing.T) {
+	tokens := token.Tokenize([]byte(`a := b`))
+
+	expected := []token.Kind{
+		token.Identifier,
+		token.Define,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestPeriod(t *testing.T) {
+	tokens := token.Tokenize([]byte(`a.b.c`))
+
+	expected := []token.Kind{
+		token.Identifier,
+		token.Period,
+		token.Identifier,
+		token.Period,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestNot(t *testing.T) {
+	tokens := token.Tokenize([]byte(`!a`))
+
+	expected := []token.Kind{
+		token.Not,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
 func TestNegateFirstToken(t *testing.T) {
 	tokens := token.Tokenize([]byte(`-a`))