Reordered tokenizer cases

This commit is contained in:
Eduard Urbach 2024-06-28 21:11:05 +02:00
parent 2520681ad3
commit 445556b64d
Signed by: akyoto
GPG Key ID: C874F672B1AF20C0

View File

@ -24,24 +24,9 @@ func Tokenize(buffer []byte) List {
switch buffer[i] {
// Whitespace
case ' ', '\t':
// Texts
case '"':
start := i
end := len(buffer)
i++
for i < len(buffer) {
if buffer[i] == '"' {
end = i + 1
i++
break
}
i++
}
tokens = append(tokens, Token{String, start, buffer[start:end]})
continue
// Separator
case ',':
tokens = append(tokens, Token{Separator, i, separatorBytes})
// Parentheses start
case '(':
@ -67,10 +52,6 @@ func Tokenize(buffer []byte) List {
case ']':
tokens = append(tokens, Token{ArrayEnd, i, arrayEndBytes})
// Separator
case ',':
tokens = append(tokens, Token{Separator, i, separatorBytes})
// New line
case '\n':
tokens = append(tokens, Token{NewLine, i, newLineBytes})
@ -89,8 +70,27 @@ func Tokenize(buffer []byte) List {
continue
// String
case '"':
start := i
end := len(buffer)
i++
for i < len(buffer) {
if buffer[i] == '"' {
end = i + 1
i++
break
}
i++
}
tokens = append(tokens, Token{String, start, buffer[start:end]})
continue
default:
// Identifiers
// Identifier
if isIdentifierStart(buffer[i]) {
position := i
i++
@ -109,7 +109,7 @@ func Tokenize(buffer []byte) List {
continue
}
// Numbers
// Number
if isNumber(buffer[i]) {
position := i
i++
@ -122,7 +122,7 @@ func Tokenize(buffer []byte) List {
continue
}
// Operators
// Operator
if isOperator(buffer[i]) {
position := i
i++
@ -135,7 +135,7 @@ func Tokenize(buffer []byte) List {
continue
}
// Invalid characters
// Invalid character
tokens = append(tokens, Token{Invalid, i, buffer[i : i+1]})
}