cli/q
Code Issues 1 Pull Requests Activity

Reduced token size

Browse Source
This commit is contained in:
Eduard Urbach
2024-07-21 14:35:06 +02:00
parent ca36d34cb9
commit 04ba68a075
47 changed files with 543 additions and 764 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/build/token/Count.go
View File

@ -1,11 +1,11 @@
package token
// Count counts how often the given token appears in the token list.
func Count(tokens List, kind Kind, name string) int {
func Count(tokens []Token, buffer []byte, kind Kind, name string) int {
count := 0
for _, t := range tokens {
if t.Kind == Identifier && t.Text() == name {
if t.Kind == Identifier && t.Text(buffer) == name {
count++
}
}

11
src/build/token/Count_test.go
View File

@ -8,9 +8,10 @@ import (
)
func TestCount(t *testing.T) {
tokens := token.Tokenize([]byte(`a b b c c c`))
assert.Equal(t, token.Count(tokens, token.Identifier, "a"), 1)
assert.Equal(t, token.Count(tokens, token.Identifier, "b"), 2)
assert.Equal(t, token.Count(tokens, token.Identifier, "c"), 3)
assert.Equal(t, token.Count(tokens, token.Identifier, "d"), 0)
buffer := []byte(`a b b c c c`)
tokens := token.Tokenize(buffer)
assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "a"), 1)
assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "b"), 2)
assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "c"), 3)
assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "d"), 0)
}

9
src/build/token/Keywords.go Normal file
View File

@ -0,0 +1,9 @@
package token
// Keywords is a map of all keywords used in the language.
var Keywords = map[string]Kind{
"if": If,
"import": Import,
"loop": Loop,
"return": Return,
}

127
src/build/token/Kind.go
View File

@ -4,73 +4,62 @@ package token
type Kind uint8
const (
// Invalid represents an invalid token.
Invalid Kind = iota
// EOF represents the end of file.
EOF
// NewLine represents the newline character.
NewLine
// Identifier represents a series of characters used to identify a variable or function.
Identifier
// Keyword represents a language keyword.
Keyword
// String represents an uninterpreted series of characters in the source code.
String
// Number represents a series of numerical characters.
Number
// Operator represents a mathematical operator.
Operator
// Separator represents a comma.
Separator
// Comment represents a comment.
Comment
// GroupStart represents '('.
GroupStart
// GroupEnd represents ')'.
GroupEnd
// BlockStart represents '{'.
BlockStart
// BlockEnd represents '}'.
BlockEnd
// ArrayStart represents '['.
ArrayStart
// ArrayEnd represents ']'.
ArrayEnd
Invalid Kind = iota // Invalid is an invalid token.
EOF // EOF is the end of file.
NewLine // NewLine is the newline character.
Identifier // Identifier is a series of characters used to identify a variable or function.
Number // Number is a series of numerical characters.
String // String is an uninterpreted series of characters in the source code.
Comment // Comment is a comment.
Separator // ,
GroupStart // (
GroupEnd // )
BlockStart // {
BlockEnd // }
ArrayStart // [
ArrayEnd // ]
_keywords // <keywords>
If // if
Import // import
Loop // loop
Return // return
_keywordsEnd // </keywords>
_operators // <operators>
Add // +
Sub // -
Mul // *
Div // /
Mod // %
And // &
Or // |
Xor // ^
Shl // <<
Shr // >>
LogicalAnd // &&
LogicalOr // ||
Equal // ==
Less // <
Greater // >
Not // !
NotEqual // !=
LessEqual // <=
GreaterEqual // >=
Define // :=
Period // .
Call // x()
Array // [x]
_assignments // <assignments>
Assign // =
AddAssign // +=
SubAssign // -=
MulAssign // *=
DivAssign // /=
ModAssign // %=
AndAssign // &=
OrAssign // |=
XorAssign // ^=
ShlAssign // <<=
ShrAssign // >>=
_assignmentsEnd // </assignments>
_operatorsEnd // </operators>
)
// String returns the text representation.
func (kind Kind) String() string {
return [...]string{
"Invalid",
"EOF",
"NewLine",
"Identifier",
"Keyword",
"String",
"Number",
"Operator",
"Separator",
"Comment",
"GroupStart",
"GroupEnd",
"BlockStart",
"BlockEnd",
"ArrayStart",
"ArrayEnd",
}[kind]
}

27
src/build/token/Kind_test.go
View File

@ -1,27 +0,0 @@
package token_test
import (
"testing"
"git.akyoto.dev/cli/q/src/build/token"
"git.akyoto.dev/go/assert"
)
func TestTokenKind(t *testing.T) {
assert.Equal(t, token.Invalid.String(), "Invalid")
assert.Equal(t, token.EOF.String(), "EOF")
assert.Equal(t, token.NewLine.String(), "NewLine")
assert.Equal(t, token.Identifier.String(), "Identifier")
assert.Equal(t, token.Keyword.String(), "Keyword")
assert.Equal(t, token.String.String(), "String")
assert.Equal(t, token.Number.String(), "Number")
assert.Equal(t, token.Operator.String(), "Operator")
assert.Equal(t, token.Separator.String(), "Separator")
assert.Equal(t, token.Comment.String(), "Comment")
assert.Equal(t, token.GroupStart.String(), "GroupStart")
assert.Equal(t, token.GroupEnd.String(), "GroupEnd")
assert.Equal(t, token.BlockStart.String(), "BlockStart")
assert.Equal(t, token.BlockEnd.String(), "BlockEnd")
assert.Equal(t, token.ArrayStart.String(), "ArrayStart")
assert.Equal(t, token.ArrayEnd.String(), "ArrayEnd")
}

4
src/build/token/Length.go Normal file
View File

@ -0,0 +1,4 @@
package token
// Length is the data type for storing token lengths.
type Length = uint16

21
src/build/token/List.go
View File

@ -1,9 +1,5 @@
package token
import (
"bytes"
)
// List is a slice of tokens.
type List []Token
@ -28,20 +24,3 @@ func (list List) LastIndexKind(kind Kind) int {
return -1
}
// String implements string serialization.
func (list List) String() string {
builder := bytes.Buffer{}
var last Token
for _, t := range list {
if last.Kind == Keyword || last.Kind == Separator || last.Kind == Operator || t.Kind == Operator {
builder.WriteByte(' ')
}
builder.Write(t.Bytes)
last = t
}
return builder.String()
}

39
src/build/token/Operators.go Normal file
View File

@ -0,0 +1,39 @@
package token
// Operators is a map of all operators used in the language.
var Operators = map[string]Kind{
".": Period,
"=": Assign,
":=": Define,
"+": Add,
"-": Sub,
"*": Mul,
"/": Div,
"%": Mod,
"&": And,
"|": Or,
"^": Xor,
"<<": Shl,
">>": Shr,
"&&": LogicalAnd,
"||": LogicalOr,
"!": Not,
"==": Equal,
"!=": NotEqual,
">": Greater,
"<": Less,
">=": GreaterEqual,
"<=": LessEqual,
"+=": AddAssign,
"-=": SubAssign,
"*=": MulAssign,
"/=": DivAssign,
"%=": ModAssign,
"&=": AndAssign,
"|=": OrAssign,
"^=": XorAssign,
"<<=": ShlAssign,
">>=": ShrAssign,
"λ": Call,
"@": Array,
}

38
src/build/token/Token.go
View File

@ -1,7 +1,6 @@
package token
import (
"fmt"
"unsafe"
)
@ -9,29 +8,44 @@ import (
// The characters that make up an identifier are grouped into a single token.
// This makes parsing easier and allows us to do better syntax checks.
type Token struct {
Bytes []byte
Position Position
Length Length
Kind Kind
}
// End returns the position after the token.
func (t *Token) End() Position {
return t.Position + Position(len(t.Bytes))
// Bytes returns the byte slice.
func (t Token) Bytes(buffer []byte) []byte {
return buffer[t.Position : t.Position+Position(t.Length)]
}
// String creates a human readable representation for debugging purposes.
func (t *Token) String() string {
return fmt.Sprintf("%s %s", t.Kind, t.Text())
// End returns the position after the token.
func (t Token) End() Position {
return t.Position + Position(t.Length)
}
// IsAssignment returns true if the token is an assignment operator.
func (t Token) IsAssignment() bool {
return t.Kind > _assignments && t.Kind < _assignmentsEnd
}
// IsKeyword returns true if the token is a keyword.
func (t Token) IsKeyword() bool {
return t.Kind > _keywords && t.Kind < _keywordsEnd
}
// IsOperator returns true if the token is an operator.
func (t Token) IsOperator() bool {
return t.Kind > _operators && t.Kind < _operatorsEnd
}
// Reset resets the token to default values.
func (t *Token) Reset() {
t.Kind = Invalid
t.Position = 0
t.Bytes = nil
t.Length = 0
t.Kind = Invalid
}
// Text returns the token text.
func (t *Token) Text() string {
return unsafe.String(unsafe.SliceData(t.Bytes), len(t.Bytes))
func (t Token) Text(buffer []byte) string {
return unsafe.String(unsafe.SliceData(t.Bytes(buffer)), t.Length)
}

41
src/build/token/Token_test.go
View File

@ -10,8 +10,8 @@ import (
func TestTokenEnd(t *testing.T) {
hello := token.Token{
Kind: token.Identifier,
Bytes: []byte("hello"),
Position: 0,
Length: 5,
}
assert.Equal(t, hello.End(), 5)
@ -20,34 +20,33 @@ func TestTokenEnd(t *testing.T) {
func TestTokenReset(t *testing.T) {
hello := token.Token{
Kind: token.Identifier,
Bytes: []byte("hello"),
Position: 1,
Length: 5,
}
hello.Reset()
assert.Nil(t, hello.Bytes)
assert.Equal(t, hello.Position, 0)
assert.Equal(t, hello.Length, 0)
assert.Equal(t, hello.Kind, token.Invalid)
}
func TestTokenString(t *testing.T) {
hello := token.Token{
Kind: token.Identifier,
Bytes: []byte("hello"),
Position: 0,
}
assert.Equal(t, hello.String(), "Identifier hello")
}
func TestTokenText(t *testing.T) {
hello := token.Token{Kind: token.Identifier, Bytes: []byte("hello"), Position: 0}
comma := token.Token{Kind: token.Separator, Bytes: []byte(","), Position: 5}
world := token.Token{Kind: token.Identifier, Bytes: []byte("world"), Position: 7}
buffer := []byte("hello, world")
hello := token.Token{Kind: token.Identifier, Position: 0, Length: 5}
comma := token.Token{Kind: token.Separator, Position: 5, Length: 1}
world := token.Token{Kind: token.Identifier, Position: 7, Length: 5}
assert.Equal(t, hello.Text(), "hello")
assert.Equal(t, world.Text(), "world")
list := token.List{hello, comma, world}
assert.Equal(t, list.String(), "hello, world")
assert.Equal(t, hello.Text(buffer), "hello")
assert.Equal(t, comma.Text(buffer), ",")
assert.Equal(t, world.Text(buffer), "world")
}
func TestTokenGroups(t *testing.T) {
assignment := token.Token{Kind: token.Assign}
operator := token.Token{Kind: token.Add}
keyword := token.Token{Kind: token.If}
assert.True(t, assignment.IsAssignment())
assert.True(t, operator.IsOperator())
assert.True(t, keyword.IsKeyword())
}

53
src/build/token/Tokenize.go
View File

@ -1,20 +1,5 @@
package token
import "git.akyoto.dev/cli/q/src/build/keyword"
// Pre-allocate these byte buffers so we can re-use them
// instead of allocating a new buffer every time.
var (
groupStartBytes = []byte{'('}
groupEndBytes = []byte{')'}
blockStartBytes = []byte{'{'}
blockEndBytes = []byte{'}'}
arrayStartBytes = []byte{'['}
arrayEndBytes = []byte{']'}
separatorBytes = []byte{','}
newLineBytes = []byte{'\n'}
)
// Tokenize turns the file contents into a list of tokens.
func Tokenize(buffer []byte) List {
var (
@ -26,21 +11,21 @@ func Tokenize(buffer []byte) List {
switch buffer[i] {
case ' ', '\t':
case ',':
tokens = append(tokens, Token{Kind: Separator, Position: i, Bytes: separatorBytes})
tokens = append(tokens, Token{Kind: Separator, Position: i, Length: 1})
case '(':
tokens = append(tokens, Token{Kind: GroupStart, Position: i, Bytes: groupStartBytes})
tokens = append(tokens, Token{Kind: GroupStart, Position: i, Length: 1})
case ')':
tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Bytes: groupEndBytes})
tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Length: 1})
case '{':
tokens = append(tokens, Token{Kind: BlockStart, Position: i, Bytes: blockStartBytes})
tokens = append(tokens, Token{Kind: BlockStart, Position: i, Length: 1})
case '}':
tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Bytes: blockEndBytes})
tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Length: 1})
case '[':
tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Bytes: arrayStartBytes})
tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Length: 1})
case ']':
tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Bytes: arrayEndBytes})
tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Length: 1})
case '\n':
tokens = append(tokens, Token{Kind: NewLine, Position: i, Bytes: newLineBytes})
tokens = append(tokens, Token{Kind: NewLine, Position: i, Length: 1})
case '/':
if i+1 >= Position(len(buffer)) || buffer[i+1] != '/' {
position := i
@ -50,7 +35,7 @@ func Tokenize(buffer []byte) List {
i++
}
tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]})
tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
} else {
position := i
@ -58,7 +43,7 @@ func Tokenize(buffer []byte) List {
i++
}
tokens = append(tokens, Token{Kind: Comment, Position: position, Bytes: buffer[position:i]})
tokens = append(tokens, Token{Kind: Comment, Position: position, Length: Length(i - position)})
}
continue
@ -78,7 +63,7 @@ func Tokenize(buffer []byte) List {
i++
}
tokens = append(tokens, Token{Kind: String, Position: start, Bytes: buffer[start:end]})
tokens = append(tokens, Token{Kind: String, Position: start, Length: Length(end - start)})
continue
default:
@ -91,14 +76,14 @@ func Tokenize(buffer []byte) List {
}
identifier := buffer[position:i]
keyword, isKeyword := keyword.Map[string(identifier)]
kind := Identifier
keyword, isKeyword := Keywords[string(identifier)]
if isKeyword {
tokens = append(tokens, Token{Kind: Keyword, Position: position, Bytes: keyword})
} else {
tokens = append(tokens, Token{Kind: Identifier, Position: position, Bytes: identifier})
kind = keyword
}
tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(len(identifier))})
continue
}
@ -110,7 +95,7 @@ func Tokenize(buffer []byte) List {
i++
}
tokens = append(tokens, Token{Kind: Number, Position: position, Bytes: buffer[position:i]})
tokens = append(tokens, Token{Kind: Number, Position: position, Length: Length(i - position)})
continue
}
@ -122,17 +107,17 @@ func Tokenize(buffer []byte) List {
i++
}
tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]})
tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
continue
}
tokens = append(tokens, Token{Kind: Invalid, Position: i, Bytes: buffer[i : i+1]})
tokens = append(tokens, Token{Kind: Invalid, Position: i, Length: 1})
}
i++
}
tokens = append(tokens, Token{Kind: EOF, Position: i, Bytes: nil})
tokens = append(tokens, Token{Kind: EOF, Position: i, Length: 0})
return tokens
}

521
src/build/token/Tokenize_test.go
View File

@ -9,394 +9,243 @@ import (
func TestFunction(t *testing.T) {
tokens := token.Tokenize([]byte("main(){}"))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Identifier,
Bytes: []byte("main"),
Position: 0,
},
{
Kind: token.GroupStart,
Bytes: []byte("("),
Position: 4,
},
{
Kind: token.GroupEnd,
Bytes: []byte(")"),
Position: 5,
},
{
Kind: token.BlockStart,
Bytes: []byte("{"),
Position: 6,
},
{
Kind: token.BlockEnd,
Bytes: []byte("}"),
Position: 7,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 8,
},
})
expected := []token.Kind{
token.Identifier,
token.GroupStart,
token.GroupEnd,
token.BlockStart,
token.BlockEnd,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
}
func TestKeyword(t *testing.T) {
tokens := token.Tokenize([]byte("return x"))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Keyword,
Bytes: []byte("return"),
Position: 0,
},
{
Kind: token.Identifier,
Bytes: []byte("x"),
Position: 7,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 8,
},
})
expected := []token.Kind{
token.Return,
token.Identifier,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
}
func TestArray(t *testing.T) {
tokens := token.Tokenize([]byte("array[i]"))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Identifier,
Bytes: []byte("array"),
Position: 0,
},
{
Kind: token.ArrayStart,
Bytes: []byte("["),
Position: 5,
},
{
Kind: token.Identifier,
Bytes: []byte("i"),
Position: 6,
},
{
Kind: token.ArrayEnd,
Bytes: []byte("]"),
Position: 7,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 8,
},
})
expected := []token.Kind{
token.Identifier,
token.ArrayStart,
token.Identifier,
token.ArrayEnd,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
}
func TestNewline(t *testing.T) {
tokens := token.Tokenize([]byte("\n\n"))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.NewLine,
Bytes: []byte("\n"),
Position: 0,
},
{
Kind: token.NewLine,
Bytes: []byte("\n"),
Position: 1,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 2,
},
})
expected := []token.Kind{
token.NewLine,
token.NewLine,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
}
func TestNumber(t *testing.T) {
tokens := token.Tokenize([]byte(`123 456`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Number,
Bytes: []byte("123"),
Position: 0,
},
{
Kind: token.Number,
Bytes: []byte("456"),
Position: 4,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 7,
},
})
expected := []token.Kind{
token.Number,
token.Number,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
}
func TestOperator(t *testing.T) {
tokens := token.Tokenize([]byte(`+ - * /`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Operator,
Bytes: []byte("+"),
Position: 0,
},
{
Kind: token.Operator,
Bytes: []byte("-"),
Position: 2,
},
{
Kind: token.Operator,
Bytes: []byte("*"),
Position: 4,
},
{
Kind: token.Operator,
Bytes: []byte("/"),
Position: 6,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 7,
},
})
expected := []token.Kind{
token.Add,
token.Sub,
token.Mul,
token.Div,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
}
func TestOperatorAssign(t *testing.T) {
tokens := token.Tokenize([]byte(`+= -= *= /= ==`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Operator,
Bytes: []byte("+="),
Position: 0,
},
{
Kind: token.Operator,
Bytes: []byte("-="),
Position: 3,
},
{
Kind: token.Operator,
Bytes: []byte("*="),
Position: 6,
},
{
Kind: token.Operator,
Bytes: []byte("/="),
Position: 9,
},
{
Kind: token.Operator,
Bytes: []byte("=="),
Position: 12,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 14,
},
})
tokens := token.Tokenize([]byte(`+= -= *= /= &= |= ^= <<= >>=`))
expected := []token.Kind{
token.AddAssign,
token.SubAssign,
token.MulAssign,
token.DivAssign,
token.AndAssign,
token.OrAssign,
token.XorAssign,
token.ShlAssign,
token.ShrAssign,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
}
func TestSeparator(t *testing.T) {
tokens := token.Tokenize([]byte("a,b,c"))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Identifier,
Bytes: []byte("a"),
Position: 0,
},
{
Kind: token.Separator,
Bytes: []byte(","),
Position: 1,
},
{
Kind: token.Identifier,
Bytes: []byte("b"),
Position: 2,
},
{
Kind: token.Separator,
Bytes: []byte(","),
Position: 3,
},
{
Kind: token.Identifier,
Bytes: []byte("c"),
Position: 4,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 5,
},
})
expected := []token.Kind{
token.Identifier,
token.Separator,
token.Identifier,
token.Separator,
token.Identifier,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
}
func TestComment(t *testing.T) {
tokens := token.Tokenize([]byte("// Hello\n// World"))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Comment,
Bytes: []byte(`// Hello`),
Position: 0,
},
{
Kind: token.NewLine,
Bytes: []byte("\n"),
Position: 8,
},
{
Kind: token.Comment,
Bytes: []byte(`// World`),
Position: 9,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 17,
},
})
expected := []token.Kind{
token.Comment,
token.NewLine,
token.Comment,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
tokens = token.Tokenize([]byte("// Hello\n"))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Comment,
Bytes: []byte(`// Hello`),
Position: 0,
},
{
Kind: token.NewLine,
Bytes: []byte("\n"),
Position: 8,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 9,
},
})
expected = []token.Kind{
token.Comment,
token.NewLine,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
tokens = token.Tokenize([]byte(`// Hello`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Comment,
Bytes: []byte(`// Hello`),
Position: 0,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 8,
},
})
expected = []token.Kind{
token.Comment,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
tokens = token.Tokenize([]byte(`//`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Comment,
Bytes: []byte(`//`),
Position: 0,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 2,
},
})
expected = []token.Kind{
token.Comment,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
tokens = token.Tokenize([]byte(`/`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Operator,
Bytes: []byte(`/`),
Position: 0,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 1,
},
})
expected = []token.Kind{
token.Div,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
}
func TestInvalid(t *testing.T) {
tokens := token.Tokenize([]byte(`@#`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Invalid,
Bytes: []byte(`@`),
Position: 0,
},
{
Kind: token.Invalid,
Bytes: []byte(`#`),
Position: 1,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 2,
},
})
tokens := token.Tokenize([]byte(`##`))
expected := []token.Kind{
token.Invalid,
token.Invalid,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
}
func TestString(t *testing.T) {
tokens := token.Tokenize([]byte(`"Hello" "World"`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.String,
Bytes: []byte(`"Hello"`),
Position: 0,
},
{
Kind: token.String,
Bytes: []byte(`"World"`),
Position: 8,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 15,
},
})
expected := []token.Kind{
token.String,
token.String,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
}
func TestStringMultiline(t *testing.T) {
tokens := token.Tokenize([]byte("\"Hello\nWorld\""))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.String,
Bytes: []byte("\"Hello\nWorld\""),
Position: 0,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 13,
},
})
expected := []token.Kind{
token.String,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
}
func TestStringEOF(t *testing.T) {
tokens := token.Tokenize([]byte(`"EOF`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.String,
Bytes: []byte(`"EOF`),
Position: 0,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 4,
},
})
expected := []token.Kind{
token.String,
token.EOF,
}
for i, kind := range expected {
assert.Equal(t, tokens[i].Kind, kind)
}
}