From 3664e740744f9cf6b43499a3409227ec3bc2d514 Mon Sep 17 00:00:00 2001
From: Eduard Urbach <admin@akyoto.dev>
Date: Fri, 28 Jun 2024 19:54:26 +0200
Subject: [PATCH] Implemented invalid character checks

---
 errors_test.go                   |  3 +++
 src/build/scan.go                | 12 ++++++++++++
 src/build/token/Tokenize.go      |  5 +++++
 src/errors/InvalidCharacter.go   | 13 +++++++++++++
 tests/errors/InvalidCharacter.q  |  1 +
 tests/errors/InvalidCharacter2.q |  1 +
 tests/errors/InvalidCharacter3.q |  1 +
 7 files changed, 36 insertions(+)
 create mode 100644 src/errors/InvalidCharacter.go
 create mode 100644 tests/errors/InvalidCharacter.q
 create mode 100644 tests/errors/InvalidCharacter2.q
 create mode 100644 tests/errors/InvalidCharacter3.q

diff --git a/errors_test.go b/errors_test.go
index f386c66..c18344e 100644
--- a/errors_test.go
+++ b/errors_test.go
@@ -22,6 +22,9 @@ func TestErrors(t *testing.T) {
 		{"InvalidInstructionNumber.q", &errors.InvalidInstruction{Instruction: "123"}},
 		{"InvalidExpression.q", errors.InvalidExpression},
 		{"InvalidOperator.q", &errors.InvalidOperator{Operator: "+++"}},
+		{"InvalidCharacter.q", &errors.InvalidCharacter{Character: "@"}},
+		{"InvalidCharacter2.q", &errors.InvalidCharacter{Character: "@"}},
+		{"InvalidCharacter3.q", &errors.InvalidCharacter{Character: "@"}},
 		{"MissingAssignValue.q", errors.MissingAssignValue},
 		{"MissingBlockEnd.q", errors.MissingBlockEnd},
 		{"MissingBlockStart.q", errors.MissingBlockStart},
diff --git a/src/build/scan.go b/src/build/scan.go
index fd01a26..15fdfe9 100644
--- a/src/build/scan.go
+++ b/src/build/scan.go
@@ -119,6 +119,10 @@ func scanFile(path string, functions chan<- *Function) error {
 				continue
 			}
 
+			if tokens[i].Kind == token.Invalid {
+				return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
+			}
+
 			if tokens[i].Kind == token.EOF {
 				return nil
 			}
@@ -156,6 +160,10 @@ func scanFile(path string, functions chan<- *Function) error {
 				continue
 			}
 
+			if tokens[i].Kind == token.Invalid {
+				return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
+			}
+
 			if tokens[i].Kind == token.EOF {
 				if groupLevel > 0 {
 					return errors.New(errors.MissingGroupEnd, file, tokens[i].Position)
@@ -204,6 +212,10 @@ func scanFile(path string, functions chan<- *Function) error {
 				continue
 			}
 
+			if tokens[i].Kind == token.Invalid {
+				return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
+			}
+
 			if tokens[i].Kind == token.EOF {
 				if blockLevel > 0 {
 					return errors.New(errors.MissingBlockEnd, file, tokens[i].Position)
diff --git a/src/build/token/Tokenize.go b/src/build/token/Tokenize.go
index a8e4acd..4c4223f 100644
--- a/src/build/token/Tokenize.go
+++ b/src/build/token/Tokenize.go
@@ -22,6 +22,8 @@ func Tokenize(buffer []byte) List {
 
 	for i < len(buffer) {
 		switch buffer[i] {
+		// Whitespace
+		case ' ', '\t':
 		// Texts
 		case '"':
 			start := i
@@ -118,6 +120,9 @@ func Tokenize(buffer []byte) List {
 				tokens = append(tokens, Token{Operator, position, buffer[position:i]})
 				continue
 			}
+
+			// Invalid characters
+			tokens = append(tokens, Token{Invalid, i, buffer[i : i+1]})
 		}
 
 		i++
diff --git a/src/errors/InvalidCharacter.go b/src/errors/InvalidCharacter.go
new file mode 100644
index 0000000..9d03918
--- /dev/null
+++ b/src/errors/InvalidCharacter.go
@@ -0,0 +1,13 @@
+package errors
+
+import "fmt"
+
+// InvalidCharacter error is created when an invalid character appears.
+type InvalidCharacter struct {
+	Character string
+}
+
+// Error generates the string representation.
+func (err *InvalidCharacter) Error() string {
+	return fmt.Sprintf("Invalid character '%s'", err.Character)
+}
diff --git a/tests/errors/InvalidCharacter.q b/tests/errors/InvalidCharacter.q
new file mode 100644
index 0000000..b516b2c
--- /dev/null
+++ b/tests/errors/InvalidCharacter.q
@@ -0,0 +1 @@
+@
\ No newline at end of file
diff --git a/tests/errors/InvalidCharacter2.q b/tests/errors/InvalidCharacter2.q
new file mode 100644
index 0000000..f870343
--- /dev/null
+++ b/tests/errors/InvalidCharacter2.q
@@ -0,0 +1 @@
+main@
\ No newline at end of file
diff --git a/tests/errors/InvalidCharacter3.q b/tests/errors/InvalidCharacter3.q
new file mode 100644
index 0000000..5f777b8
--- /dev/null
+++ b/tests/errors/InvalidCharacter3.q
@@ -0,0 +1 @@
+main()@
\ No newline at end of file