Implemented runes
This commit is contained in:
@ -9,6 +9,7 @@ const (
|
||||
NewLine // NewLine is the newline character.
|
||||
Identifier // Identifier is a series of characters used to identify a variable or function.
|
||||
Number // Number is a series of numerical characters.
|
||||
Rune // Rune is a single unicode code point.
|
||||
String // String is an uninterpreted series of characters in the source code.
|
||||
Comment // Comment is a comment.
|
||||
Separator // ,
|
||||
|
@ -48,13 +48,14 @@ func Tokenize(buffer []byte) List {
|
||||
|
||||
continue
|
||||
|
||||
case '"':
|
||||
case '"', '\'':
|
||||
limiter := buffer[i]
|
||||
start := i
|
||||
end := Position(len(buffer))
|
||||
i++
|
||||
|
||||
for i < Position(len(buffer)) {
|
||||
if buffer[i] == '"' {
|
||||
if buffer[i] == limiter {
|
||||
end = i + 1
|
||||
i++
|
||||
break
|
||||
@ -63,7 +64,13 @@ func Tokenize(buffer []byte) List {
|
||||
i++
|
||||
}
|
||||
|
||||
tokens = append(tokens, Token{Kind: String, Position: start, Length: Length(end - start)})
|
||||
kind := String
|
||||
|
||||
if limiter == '\'' {
|
||||
kind = Rune
|
||||
}
|
||||
|
||||
tokens = append(tokens, Token{Kind: kind, Position: start, Length: Length(end - start)})
|
||||
continue
|
||||
|
||||
default:
|
||||
|
Reference in New Issue
Block a user