Added more tests

This commit is contained in:
2024-06-28 23:13:55 +02:00
parent 445556b64d
commit 5f2ff5e74e
5 changed files with 73 additions and 30 deletions

View File

@ -59,15 +59,24 @@ func Tokenize(buffer []byte) List {
// Comment
case '/':
if i+1 >= len(buffer) || buffer[i+1] != '/' {
tokens = append(tokens, Token{Operator, i, buffer[i : i+1]})
position := i
i++
for i < len(buffer) && isOperator(buffer[i]) {
i++
}
tokens = append(tokens, Token{Operator, position, buffer[position:i]})
continue
}
position := i
for i < len(buffer) && buffer[i] != '\n' {
i++
}
tokens = append(tokens, Token{Comment, position, buffer[position:i]})
continue
// String