cli/q
Code Issues 1 Pull Requests Activity

Implemented runes

Browse Source
This commit is contained in:
Eduard Urbach
2024-07-22 15:32:16 +02:00
parent e91e894046
commit 21017e6378
11 changed files with 88 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/build/token/Tokenize.go
View File

@ -48,13 +48,14 @@ func Tokenize(buffer []byte) List {
continue
case '"':
case '"', '\'':
limiter := buffer[i]
start := i
end := Position(len(buffer))
i++
for i < Position(len(buffer)) {
if buffer[i] == '"' {
if buffer[i] == limiter {
end = i + 1
i++
break
@ -63,7 +64,13 @@ func Tokenize(buffer []byte) List {
i++
}
tokens = append(tokens, Token{Kind: String, Position: start, Length: Length(end - start)})
kind := String
if limiter == '\'' {
kind = Rune
}
tokens = append(tokens, Token{Kind: kind, Position: start, Length: Length(end - start)})
continue
default: