Added more tests
This commit is contained in:
@ -59,15 +59,24 @@ func Tokenize(buffer []byte) List {
|
||||
// Comment
|
||||
case '/':
|
||||
if i+1 >= len(buffer) || buffer[i+1] != '/' {
|
||||
tokens = append(tokens, Token{Operator, i, buffer[i : i+1]})
|
||||
position := i
|
||||
i++
|
||||
|
||||
for i < len(buffer) && isOperator(buffer[i]) {
|
||||
i++
|
||||
}
|
||||
|
||||
tokens = append(tokens, Token{Operator, position, buffer[position:i]})
|
||||
continue
|
||||
}
|
||||
|
||||
position := i
|
||||
|
||||
for i < len(buffer) && buffer[i] != '\n' {
|
||||
i++
|
||||
}
|
||||
|
||||
tokens = append(tokens, Token{Comment, position, buffer[position:i]})
|
||||
continue
|
||||
|
||||
// String
|
||||
|
Reference in New Issue
Block a user