Implemented invalid character checks

src/build/scan.go

@@ -119,6 +119,10 @@ func scanFile(path string, functions chan<- *Function) error {
 				continue
 			}
 
+			if tokens[i].Kind == token.Invalid {
+				return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
+			}
+
 			if tokens[i].Kind == token.EOF {
 				return nil
 			}
@@ -156,6 +160,10 @@ func scanFile(path string, functions chan<- *Function) error {
 				continue
 			}
 
+			if tokens[i].Kind == token.Invalid {
+				return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
+			}
+
 			if tokens[i].Kind == token.EOF {
 				if groupLevel > 0 {
 					return errors.New(errors.MissingGroupEnd, file, tokens[i].Position)
@@ -204,6 +212,10 @@ func scanFile(path string, functions chan<- *Function) error {
 				continue
 			}
 
+			if tokens[i].Kind == token.Invalid {
+				return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
+			}
+
 			if tokens[i].Kind == token.EOF {
 				if blockLevel > 0 {
 					return errors.New(errors.MissingBlockEnd, file, tokens[i].Position)

src/build/token/Tokenize.go

@@ -22,6 +22,8 @@ func Tokenize(buffer []byte) List {
 
 	for i < len(buffer) {
 		switch buffer[i] {
+		// Whitespace
+		case ' ', '\t':
 		// Texts
 		case '"':
 			start := i
@@ -118,6 +120,9 @@ func Tokenize(buffer []byte) List {
 				tokens = append(tokens, Token{Operator, position, buffer[position:i]})
 				continue
 			}
+
+			// Invalid characters
+			tokens = append(tokens, Token{Invalid, i, buffer[i : i+1]})
 		}
 
 		i++

src/errors/InvalidCharacter.go

@@ -0,0 +1,13 @@
+package errors
+
+import "fmt"
+
+// InvalidCharacter error is created when an invalid character appears.
+type InvalidCharacter struct {
+	Character string
+}
+
+// Error generates the string representation.
+func (err *InvalidCharacter) Error() string {
+	return fmt.Sprintf("Invalid character '%s'", err.Character)
+}