Implemented invalid character checks

errors_test.go

@@ -22,6 +22,9 @@ func TestErrors(t *testing.T) {
 		{"InvalidInstructionNumber.q", &errors.InvalidInstruction{Instruction: "123"}},
 		{"InvalidExpression.q", errors.InvalidExpression},
 		{"InvalidOperator.q", &errors.InvalidOperator{Operator: "+++"}},
+		{"InvalidCharacter.q", &errors.InvalidCharacter{Character: "@"}},
+		{"InvalidCharacter2.q", &errors.InvalidCharacter{Character: "@"}},
+		{"InvalidCharacter3.q", &errors.InvalidCharacter{Character: "@"}},
 		{"MissingAssignValue.q", errors.MissingAssignValue},
 		{"MissingBlockEnd.q", errors.MissingBlockEnd},
 		{"MissingBlockStart.q", errors.MissingBlockStart},

src/build/scan.go

@@ -119,6 +119,10 @@ func scanFile(path string, functions chan<- *Function) error {
 				continue
 			}
 
+			if tokens[i].Kind == token.Invalid {
+				return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
+			}
+
 			if tokens[i].Kind == token.EOF {
 				return nil
 			}
@@ -156,6 +160,10 @@ func scanFile(path string, functions chan<- *Function) error {
 				continue
 			}
 
+			if tokens[i].Kind == token.Invalid {
+				return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
+			}
+
 			if tokens[i].Kind == token.EOF {
 				if groupLevel > 0 {
 					return errors.New(errors.MissingGroupEnd, file, tokens[i].Position)
@@ -204,6 +212,10 @@ func scanFile(path string, functions chan<- *Function) error {
 				continue
 			}
 
+			if tokens[i].Kind == token.Invalid {
+				return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
+			}
+
 			if tokens[i].Kind == token.EOF {
 				if blockLevel > 0 {
 					return errors.New(errors.MissingBlockEnd, file, tokens[i].Position)

src/build/token/Tokenize.go

@@ -22,6 +22,8 @@ func Tokenize(buffer []byte) List {
 
 	for i < len(buffer) {
 		switch buffer[i] {
+		// Whitespace
+		case ' ', '\t':
 		// Texts
 		case '"':
 			start := i
@@ -118,6 +120,9 @@ func Tokenize(buffer []byte) List {
 				tokens = append(tokens, Token{Operator, position, buffer[position:i]})
 				continue
 			}
+
+			// Invalid characters
+			tokens = append(tokens, Token{Invalid, i, buffer[i : i+1]})
 		}
 
 		i++

src/errors/InvalidCharacter.go

@@ -0,0 +1,13 @@
+package errors
+
+import "fmt"
+
+// InvalidCharacter error is created when an invalid character appears.
+type InvalidCharacter struct {
+	Character string
+}
+
+// Error generates the string representation.
+func (err *InvalidCharacter) Error() string {
+	return fmt.Sprintf("Invalid character '%s'", err.Character)
+}

tests/errors/InvalidCharacter.q

@@ -0,0 +1 @@
+@

tests/errors/InvalidCharacter2.q

@@ -0,0 +1 @@
+main@

tests/errors/InvalidCharacter3.q

@@ -0,0 +1 @@
+main()@