Implemented invalid character checks

This commit is contained in:
2024-06-28 19:54:26 +02:00
parent 6852cbb24e
commit 3664e74074
7 changed files with 36 additions and 0 deletions

View File

@ -22,6 +22,8 @@ func Tokenize(buffer []byte) List {
for i < len(buffer) {
switch buffer[i] {
// Whitespace
case ' ', '\t':
// Texts
case '"':
start := i
@ -118,6 +120,9 @@ func Tokenize(buffer []byte) List {
tokens = append(tokens, Token{Operator, position, buffer[position:i]})
continue
}
// Invalid characters
tokens = append(tokens, Token{Invalid, i, buffer[i : i+1]})
}
i++