Implemented invalid character checks
This commit is contained in:
@ -22,6 +22,8 @@ func Tokenize(buffer []byte) List {
|
||||
|
||||
for i < len(buffer) {
|
||||
switch buffer[i] {
|
||||
// Whitespace
|
||||
case ' ', '\t':
|
||||
// Texts
|
||||
case '"':
|
||||
start := i
|
||||
@ -118,6 +120,9 @@ func Tokenize(buffer []byte) List {
|
||||
tokens = append(tokens, Token{Operator, position, buffer[position:i]})
|
||||
continue
|
||||
}
|
||||
|
||||
// Invalid characters
|
||||
tokens = append(tokens, Token{Invalid, i, buffer[i : i+1]})
|
||||
}
|
||||
|
||||
i++
|
||||
|
Reference in New Issue
Block a user