diff --git a/src/build/ast/AST.go b/src/build/ast/AST.go
index 1c8f6dd..c7c4c47 100644
--- a/src/build/ast/AST.go
+++ b/src/build/ast/AST.go
@@ -1,6 +1,4 @@
package ast
-import "fmt"
-
-type Node fmt.Stringer
+type Node any
type AST []Node
diff --git a/src/build/ast/Assign.go b/src/build/ast/Assign.go
index 32c6819..161f968 100644
--- a/src/build/ast/Assign.go
+++ b/src/build/ast/Assign.go
@@ -8,7 +8,3 @@ import (
type Assign struct {
Expression *expression.Expression
}
-
-func (node *Assign) String() string {
- return node.Expression.String()
-}
diff --git a/src/build/ast/Call.go b/src/build/ast/Call.go
index d7825b3..6bd7dec 100644
--- a/src/build/ast/Call.go
+++ b/src/build/ast/Call.go
@@ -6,7 +6,3 @@ import "git.akyoto.dev/cli/q/src/build/expression"
type Call struct {
Expression *expression.Expression
}
-
-func (node *Call) String() string {
- return node.Expression.String()
-}
diff --git a/src/build/ast/Count.go b/src/build/ast/Count.go
index 0cc6986..b14c652 100644
--- a/src/build/ast/Count.go
+++ b/src/build/ast/Count.go
@@ -3,31 +3,31 @@ package ast
import "git.akyoto.dev/cli/q/src/build/token"
// Count counts how often the given token appears in the AST.
-func Count(body AST, kind token.Kind, name string) int {
+func Count(body AST, buffer []byte, kind token.Kind, name string) int {
count := 0
for _, node := range body {
switch node := node.(type) {
case *Assign:
- count += node.Expression.Count(kind, name)
+ count += node.Expression.Count(buffer, kind, name)
case *Call:
- count += node.Expression.Count(kind, name)
+ count += node.Expression.Count(buffer, kind, name)
case *Define:
- count += node.Value.Count(kind, name)
+ count += node.Value.Count(buffer, kind, name)
case *Return:
if node.Value != nil {
- count += node.Value.Count(kind, name)
+ count += node.Value.Count(buffer, kind, name)
}
case *If:
- count += node.Condition.Count(kind, name)
- count += Count(node.Body, kind, name)
+ count += node.Condition.Count(buffer, kind, name)
+ count += Count(node.Body, buffer, kind, name)
case *Loop:
- count += Count(node.Body, kind, name)
+ count += Count(node.Body, buffer, kind, name)
default:
panic("unknown AST type")
diff --git a/src/build/ast/Define.go b/src/build/ast/Define.go
index 6751491..64f0c15 100644
--- a/src/build/ast/Define.go
+++ b/src/build/ast/Define.go
@@ -1,8 +1,6 @@
package ast
import (
- "fmt"
-
"git.akyoto.dev/cli/q/src/build/expression"
"git.akyoto.dev/cli/q/src/build/token"
)
@@ -12,7 +10,3 @@ type Define struct {
Value *expression.Expression
Name token.Token
}
-
-func (node *Define) String() string {
- return fmt.Sprintf("(= %s %s)", node.Name.Text(), node.Value)
-}
diff --git a/src/build/ast/If.go b/src/build/ast/If.go
index 47d98d4..4275f34 100644
--- a/src/build/ast/If.go
+++ b/src/build/ast/If.go
@@ -1,8 +1,6 @@
package ast
import (
- "fmt"
-
"git.akyoto.dev/cli/q/src/build/expression"
)
@@ -11,7 +9,3 @@ type If struct {
Condition *expression.Expression
Body AST
}
-
-func (node *If) String() string {
- return fmt.Sprintf("(if %s %s)", node.Condition, node.Body)
-}
diff --git a/src/build/ast/Loop.go b/src/build/ast/Loop.go
index 5b0dc33..e1ca210 100644
--- a/src/build/ast/Loop.go
+++ b/src/build/ast/Loop.go
@@ -1,12 +1,6 @@
package ast
-import "fmt"
-
// Loop represents a block of repeatable statements.
type Loop struct {
Body AST
}
-
-func (node *Loop) String() string {
- return fmt.Sprintf("(loop %s)", node.Body)
-}
diff --git a/src/build/ast/Parse.go b/src/build/ast/Parse.go
index eae5666..5610e1a 100644
--- a/src/build/ast/Parse.go
+++ b/src/build/ast/Parse.go
@@ -3,16 +3,15 @@ package ast
import (
"git.akyoto.dev/cli/q/src/build/errors"
"git.akyoto.dev/cli/q/src/build/expression"
- "git.akyoto.dev/cli/q/src/build/keyword"
"git.akyoto.dev/cli/q/src/build/token"
)
// Parse generates an AST from a list of tokens.
-func Parse(tokens token.List) (AST, error) {
+func Parse(tokens []token.Token, buffer []byte) (AST, error) {
tree := make(AST, 0, len(tokens)/64)
err := EachInstruction(tokens, func(instruction token.List) error {
- node, err := toASTNode(instruction)
+ node, err := toASTNode(instruction, buffer)
if err == nil && node != nil {
tree = append(tree, node)
@@ -25,11 +24,9 @@ func Parse(tokens token.List) (AST, error) {
}
// toASTNode generates an AST node from an instruction.
-func toASTNode(tokens token.List) (Node, error) {
- if tokens[0].Kind == token.Keyword {
- word := tokens[0].Text()
-
- if word == keyword.Return {
+func toASTNode(tokens token.List, buffer []byte) (Node, error) {
+ if tokens[0].IsKeyword() {
+ if tokens[0].Kind == token.Return {
if len(tokens) == 1 {
return &Return{}, nil
}
@@ -38,7 +35,7 @@ func toASTNode(tokens token.List) (Node, error) {
return &Return{Value: value}, nil
}
- if keywordHasBlock(word) {
+ if keywordHasBlock(tokens[0].Kind) {
blockStart := tokens.IndexKind(token.BlockStart)
blockEnd := tokens.LastIndexKind(token.BlockEnd)
@@ -50,19 +47,19 @@ func toASTNode(tokens token.List) (Node, error) {
return nil, errors.New(errors.MissingBlockEnd, nil, tokens[len(tokens)-1].End())
}
- body, err := Parse(tokens[blockStart+1 : blockEnd])
+ body, err := Parse(tokens[blockStart+1:blockEnd], buffer)
- switch word {
- case keyword.If:
+ switch tokens[0].Kind {
+ case token.If:
condition := expression.Parse(tokens[1:blockStart])
return &If{Condition: condition, Body: body}, err
- case keyword.Loop:
+ case token.Loop:
return &Loop{Body: body}, err
}
}
- return nil, errors.New(&errors.KeywordNotImplemented{Keyword: word}, nil, tokens[0].Position)
+ return nil, errors.New(&errors.KeywordNotImplemented{Keyword: tokens[0].Text(buffer)}, nil, tokens[0].Position)
}
expr := expression.Parse(tokens)
@@ -92,26 +89,26 @@ func toASTNode(tokens token.List) (Node, error) {
return &Call{Expression: expr}, nil
default:
- return nil, errors.New(&errors.InvalidInstruction{Instruction: expr.Token.Text()}, nil, expr.Token.Position)
+ return nil, errors.New(&errors.InvalidInstruction{Instruction: expr.Token.Text(buffer)}, nil, expr.Token.Position)
}
}
// IsAssignment returns true if the expression is an assignment.
func IsAssignment(expr *expression.Expression) bool {
- return expr.Token.Kind == token.Operator && expr.Token.Bytes[len(expr.Token.Bytes)-1] == '='
+ return expr.Token.IsAssignment()
}
// IsFunctionCall returns true if the expression is a function call.
func IsFunctionCall(expr *expression.Expression) bool {
- return expr.Token.Kind == token.Operator && expr.Token.Text() == "λ"
+ return expr.Token.Kind == token.Call
}
// IsVariableDefinition returns true if the expression is a variable definition.
func IsVariableDefinition(expr *expression.Expression) bool {
- return expr.Token.Kind == token.Operator && expr.Token.Text() == ":="
+ return expr.Token.Kind == token.Define
}
// keywordHasBlock returns true if the keyword requires a block.
-func keywordHasBlock(word string) bool {
- return word == keyword.If || word == keyword.Loop
+func keywordHasBlock(kind token.Kind) bool {
+ return kind == token.If || kind == token.Loop
}
diff --git a/src/build/ast/Return.go b/src/build/ast/Return.go
index a56520f..37574e2 100644
--- a/src/build/ast/Return.go
+++ b/src/build/ast/Return.go
@@ -1,8 +1,6 @@
package ast
import (
- "fmt"
-
"git.akyoto.dev/cli/q/src/build/expression"
)
@@ -10,7 +8,3 @@ import (
type Return struct {
Value *expression.Expression
}
-
-func (node *Return) String() string {
- return fmt.Sprintf("(return %s)", node.Value)
-}
diff --git a/src/build/core/Compare.go b/src/build/core/Compare.go
index 3be29ec..6cd0b5d 100644
--- a/src/build/core/Compare.go
+++ b/src/build/core/Compare.go
@@ -13,7 +13,7 @@ func (f *Function) Compare(comparison *expression.Expression) error {
right := comparison.Children[1]
if left.IsLeaf() && left.Token.Kind == token.Identifier {
- name := left.Token.Text()
+ name := left.Token.Text(f.File.Bytes)
variable := f.VariableByName(name)
if variable == nil {
diff --git a/src/build/core/CompileAssign.go b/src/build/core/CompileAssign.go
index 384630d..4d3e08b 100644
--- a/src/build/core/CompileAssign.go
+++ b/src/build/core/CompileAssign.go
@@ -16,7 +16,7 @@ func (f *Function) CompileAssign(node *ast.Assign) error {
right := node.Expression.Children[1]
if left.IsLeaf() {
- name := left.Token.Text()
+ name := left.Token.Text(f.File.Bytes)
variable := f.VariableByName(name)
if variable == nil {
@@ -27,8 +27,8 @@ func (f *Function) CompileAssign(node *ast.Assign) error {
return f.Execute(operator, variable.Register, right)
}
- if left.Token.Kind == token.Operator && left.Token.Text() == "@" {
- name := left.Children[0].Token.Text()
+ if left.Token.Kind == token.Array {
+ name := left.Children[0].Token.Text(f.File.Bytes)
variable := f.VariableByName(name)
if variable == nil {
@@ -38,13 +38,13 @@ func (f *Function) CompileAssign(node *ast.Assign) error {
defer f.useVariable(variable)
index := left.Children[1]
- offset, err := strconv.Atoi(index.Token.Text())
+ offset, err := strconv.Atoi(index.Token.Text(f.File.Bytes))
if err != nil {
return err
}
- num, err := strconv.Atoi(right.Token.Text())
+ num, err := strconv.Atoi(right.Token.Text(f.File.Bytes))
if err != nil {
return err
diff --git a/src/build/core/CompileCall.go b/src/build/core/CompileCall.go
index 4c2adc6..3270d01 100644
--- a/src/build/core/CompileCall.go
+++ b/src/build/core/CompileCall.go
@@ -15,9 +15,9 @@ func (f *Function) CompileCall(root *expression.Expression) error {
funcName := ""
if funcNameRoot.IsLeaf() {
- funcName = funcNameRoot.Token.Text()
+ funcName = funcNameRoot.Token.Text(f.File.Bytes)
} else {
- funcName = funcNameRoot.Children[0].Token.Text() + funcNameRoot.Token.Text() + funcNameRoot.Children[1].Token.Text()
+ funcName = funcNameRoot.Children[0].Token.Text(f.File.Bytes) + funcNameRoot.Token.Text(f.File.Bytes) + funcNameRoot.Children[1].Token.Text(f.File.Bytes)
}
isSyscall := funcName == "syscall"
diff --git a/src/build/core/CompileCondition.go b/src/build/core/CompileCondition.go
index 621a7e7..54e0ab0 100644
--- a/src/build/core/CompileCondition.go
+++ b/src/build/core/CompileCondition.go
@@ -4,12 +4,13 @@ import (
"fmt"
"git.akyoto.dev/cli/q/src/build/expression"
+ "git.akyoto.dev/cli/q/src/build/token"
)
// CompileCondition inserts code to jump to the start label or end label depending on the truth of the condition.
func (f *Function) CompileCondition(condition *expression.Expression, successLabel string, failLabel string) error {
- switch condition.Token.Text() {
- case "||":
+ switch condition.Token.Kind {
+ case token.LogicalOr:
f.count.subBranch++
leftFailLabel := fmt.Sprintf("%s_false_%d", f.Name, f.count.subBranch)
@@ -21,22 +22,22 @@ func (f *Function) CompileCondition(condition *expression.Expression, successLab
return err
}
- f.JumpIfTrue(left.Token.Text(), successLabel)
+ f.JumpIfTrue(left.Token.Kind, successLabel)
// Right
f.AddLabel(leftFailLabel)
right := condition.Children[1]
err = f.CompileCondition(right, successLabel, failLabel)
- if condition.Parent != nil && condition.Parent.Token.Text() == "||" && condition != condition.Parent.LastChild() {
- f.JumpIfTrue(right.Token.Text(), successLabel)
+ if condition.Parent != nil && condition.Parent.Token.Kind == token.LogicalOr && condition != condition.Parent.LastChild() {
+ f.JumpIfTrue(right.Token.Kind, successLabel)
} else {
- f.JumpIfFalse(right.Token.Text(), failLabel)
+ f.JumpIfFalse(right.Token.Kind, failLabel)
}
return err
- case "&&":
+ case token.LogicalAnd:
f.count.subBranch++
leftSuccessLabel := fmt.Sprintf("%s_true_%d", f.Name, f.count.subBranch)
@@ -48,17 +49,17 @@ func (f *Function) CompileCondition(condition *expression.Expression, successLab
return err
}
- f.JumpIfFalse(left.Token.Text(), failLabel)
+ f.JumpIfFalse(left.Token.Kind, failLabel)
// Right
f.AddLabel(leftSuccessLabel)
right := condition.Children[1]
err = f.CompileCondition(right, successLabel, failLabel)
- if condition.Parent != nil && condition.Parent.Token.Text() == "||" && condition != condition.Parent.LastChild() {
- f.JumpIfTrue(right.Token.Text(), successLabel)
+ if condition.Parent != nil && condition.Parent.Token.Kind == token.LogicalOr && condition != condition.Parent.LastChild() {
+ f.JumpIfTrue(right.Token.Kind, successLabel)
} else {
- f.JumpIfFalse(right.Token.Text(), failLabel)
+ f.JumpIfFalse(right.Token.Kind, failLabel)
}
return err
@@ -67,7 +68,7 @@ func (f *Function) CompileCondition(condition *expression.Expression, successLab
err := f.Compare(condition)
if condition.Parent == nil {
- f.JumpIfFalse(condition.Token.Text(), failLabel)
+ f.JumpIfFalse(condition.Token.Kind, failLabel)
}
return err
diff --git a/src/build/core/CompileDefinition.go b/src/build/core/CompileDefinition.go
index a8fa1eb..c5473b1 100644
--- a/src/build/core/CompileDefinition.go
+++ b/src/build/core/CompileDefinition.go
@@ -8,13 +8,13 @@ import (
// CompileDefinition compiles a variable definition.
func (f *Function) CompileDefinition(node *ast.Define) error {
- name := node.Name.Text()
+ name := node.Name.Text(f.File.Bytes)
if f.identifierExists(name) {
return errors.New(&errors.VariableAlreadyExists{Name: name}, f.File, node.Name.Position)
}
- uses := token.Count(f.Body, token.Identifier, name) - 1
+ uses := token.Count(f.Body, f.File.Bytes, token.Identifier, name) - 1
if uses == 0 {
return errors.New(&errors.UnusedVariable{Name: name}, f.File, node.Name.Position)
diff --git a/src/build/core/CompileIf.go b/src/build/core/CompileIf.go
index 1b61c3c..3eeb4ae 100644
--- a/src/build/core/CompileIf.go
+++ b/src/build/core/CompileIf.go
@@ -18,7 +18,7 @@ func (f *Function) CompileIf(branch *ast.If) error {
}
f.AddLabel(success)
- f.PushScope(branch.Body)
+ f.PushScope(branch.Body, f.File.Bytes)
err = f.CompileAST(branch.Body)
f.PopScope()
f.AddLabel(fail)
diff --git a/src/build/core/CompileLoop.go b/src/build/core/CompileLoop.go
index de9d8b6..a1b116a 100644
--- a/src/build/core/CompileLoop.go
+++ b/src/build/core/CompileLoop.go
@@ -12,7 +12,7 @@ func (f *Function) CompileLoop(loop *ast.Loop) error {
f.count.loop++
label := fmt.Sprintf("%s_loop_%d", f.Name, f.count.loop)
f.AddLabel(label)
- scope := f.PushScope(loop.Body)
+ scope := f.PushScope(loop.Body, f.File.Bytes)
scope.InLoop = true
err := f.CompileAST(loop.Body)
f.Jump(asm.JUMP, label)
diff --git a/src/build/core/CompileTokens.go b/src/build/core/CompileTokens.go
index 7b41804..634c5d7 100644
--- a/src/build/core/CompileTokens.go
+++ b/src/build/core/CompileTokens.go
@@ -7,8 +7,8 @@ import (
)
// CompileTokens compiles a token list.
-func (f *Function) CompileTokens(tokens token.List) error {
- body, err := ast.Parse(tokens)
+func (f *Function) CompileTokens(tokens []token.Token) error {
+ body, err := ast.Parse(tokens, f.File.Bytes)
if err != nil {
err.(*errors.Error).File = f.File
diff --git a/src/build/core/ExecuteLeaf.go b/src/build/core/ExecuteLeaf.go
index 947518d..d6cc124 100644
--- a/src/build/core/ExecuteLeaf.go
+++ b/src/build/core/ExecuteLeaf.go
@@ -12,7 +12,7 @@ import (
func (f *Function) ExecuteLeaf(operation token.Token, register cpu.Register, operand token.Token) error {
switch operand.Kind {
case token.Identifier:
- name := operand.Text()
+ name := operand.Text(f.File.Bytes)
variable := f.VariableByName(name)
if variable == nil {
@@ -23,7 +23,7 @@ func (f *Function) ExecuteLeaf(operation token.Token, register cpu.Register, ope
return f.ExecuteRegisterRegister(operation, register, variable.Register)
case token.Number:
- value := operand.Text()
+ value := operand.Text(f.File.Bytes)
number, err := strconv.Atoi(value)
if err != nil {
@@ -33,7 +33,7 @@ func (f *Function) ExecuteLeaf(operation token.Token, register cpu.Register, ope
return f.ExecuteRegisterNumber(operation, register, number)
case token.String:
- if operation.Text() == "=" {
+ if operation.Kind == token.Assign {
return f.TokenToRegister(operand, register)
}
}
diff --git a/src/build/core/ExecuteRegisterNumber.go b/src/build/core/ExecuteRegisterNumber.go
index 19b07ea..43675a2 100644
--- a/src/build/core/ExecuteRegisterNumber.go
+++ b/src/build/core/ExecuteRegisterNumber.go
@@ -9,27 +9,27 @@ import (
// ExecuteRegisterNumber performs an operation on a register and a number.
func (f *Function) ExecuteRegisterNumber(operation token.Token, register cpu.Register, number int) error {
- switch operation.Text() {
- case "+", "+=":
+ switch operation.Kind {
+ case token.Add, token.AddAssign:
f.RegisterNumber(asm.ADD, register, number)
- case "-", "-=":
+ case token.Sub, token.SubAssign:
f.RegisterNumber(asm.SUB, register, number)
- case "*", "*=":
+ case token.Mul, token.MulAssign:
f.RegisterNumber(asm.MUL, register, number)
- case "/", "/=":
+ case token.Div, token.DivAssign:
f.RegisterNumber(asm.DIV, register, number)
- case "==", "!=", "<", "<=", ">", ">=":
+ case token.Equal, token.NotEqual, token.Less, token.LessEqual, token.Greater, token.GreaterEqual:
f.RegisterNumber(asm.COMPARE, register, number)
- case "=":
+ case token.Assign:
f.RegisterNumber(asm.MOVE, register, number)
default:
- return errors.New(&errors.InvalidOperator{Operator: operation.Text()}, f.File, operation.Position)
+ return errors.New(&errors.InvalidOperator{Operator: operation.Text(f.File.Bytes)}, f.File, operation.Position)
}
return nil
diff --git a/src/build/core/ExecuteRegisterRegister.go b/src/build/core/ExecuteRegisterRegister.go
index c163e23..8206b4f 100644
--- a/src/build/core/ExecuteRegisterRegister.go
+++ b/src/build/core/ExecuteRegisterRegister.go
@@ -9,27 +9,27 @@ import (
// ExecuteRegisterRegister performs an operation on two registers.
func (f *Function) ExecuteRegisterRegister(operation token.Token, destination cpu.Register, source cpu.Register) error {
- switch operation.Text() {
- case "+", "+=":
+ switch operation.Kind {
+ case token.Add, token.AddAssign:
f.RegisterRegister(asm.ADD, destination, source)
- case "-", "-=":
+ case token.Sub, token.SubAssign:
f.RegisterRegister(asm.SUB, destination, source)
- case "*", "*=":
+ case token.Mul, token.MulAssign:
f.RegisterRegister(asm.MUL, destination, source)
- case "/", "/=":
+ case token.Div, token.DivAssign:
f.RegisterRegister(asm.DIV, destination, source)
- case "==", "!=", "<", "<=", ">", ">=":
+ case token.Equal, token.NotEqual, token.Less, token.LessEqual, token.Greater, token.GreaterEqual:
f.RegisterRegister(asm.COMPARE, destination, source)
- case "=":
+ case token.Assign:
f.RegisterRegister(asm.MOVE, destination, source)
default:
- return errors.New(&errors.InvalidOperator{Operator: operation.Text()}, f.File, operation.Position)
+ return errors.New(&errors.InvalidOperator{Operator: operation.Text(f.File.Bytes)}, f.File, operation.Position)
}
return nil
diff --git a/src/build/core/Function.go b/src/build/core/Function.go
index 79e1b31..36ac29d 100644
--- a/src/build/core/Function.go
+++ b/src/build/core/Function.go
@@ -13,7 +13,7 @@ type Function struct {
scope.Stack
Name string
File *fs.File
- Body token.List
+ Body []token.Token
Assembler asm.Assembler
Functions map[string]*Function
Err error
diff --git a/src/build/core/JumpIfFalse.go b/src/build/core/JumpIfFalse.go
index baedd51..042a676 100644
--- a/src/build/core/JumpIfFalse.go
+++ b/src/build/core/JumpIfFalse.go
@@ -2,22 +2,23 @@ package core
import (
"git.akyoto.dev/cli/q/src/build/asm"
+ "git.akyoto.dev/cli/q/src/build/token"
)
// JumpIfFalse jumps to the label if the previous comparison was false.
-func (f *Function) JumpIfFalse(operator string, label string) {
+func (f *Function) JumpIfFalse(operator token.Kind, label string) {
switch operator {
- case "==":
+ case token.Equal:
f.Jump(asm.JNE, label)
- case "!=":
+ case token.NotEqual:
f.Jump(asm.JE, label)
- case ">":
+ case token.Greater:
f.Jump(asm.JLE, label)
- case "<":
+ case token.Less:
f.Jump(asm.JGE, label)
- case ">=":
+ case token.GreaterEqual:
f.Jump(asm.JL, label)
- case "<=":
+ case token.LessEqual:
f.Jump(asm.JG, label)
}
}
diff --git a/src/build/core/JumpIfTrue.go b/src/build/core/JumpIfTrue.go
index 4c04a0b..059618e 100644
--- a/src/build/core/JumpIfTrue.go
+++ b/src/build/core/JumpIfTrue.go
@@ -2,22 +2,23 @@ package core
import (
"git.akyoto.dev/cli/q/src/build/asm"
+ "git.akyoto.dev/cli/q/src/build/token"
)
// JumpIfTrue jumps to the label if the previous comparison was true.
-func (f *Function) JumpIfTrue(operator string, label string) {
+func (f *Function) JumpIfTrue(operator token.Kind, label string) {
switch operator {
- case "==":
+ case token.Equal:
f.Jump(asm.JE, label)
- case "!=":
+ case token.NotEqual:
f.Jump(asm.JNE, label)
- case ">":
+ case token.Greater:
f.Jump(asm.JG, label)
- case "<":
+ case token.Less:
f.Jump(asm.JL, label)
- case ">=":
+ case token.GreaterEqual:
f.Jump(asm.JGE, label)
- case "<=":
+ case token.LessEqual:
f.Jump(asm.JLE, label)
}
}
diff --git a/src/build/core/NewFunction.go b/src/build/core/NewFunction.go
index 1394c4e..2dc683d 100644
--- a/src/build/core/NewFunction.go
+++ b/src/build/core/NewFunction.go
@@ -10,7 +10,7 @@ import (
)
// NewFunction creates a new function.
-func NewFunction(name string, file *fs.File, body token.List) *Function {
+func NewFunction(name string, file *fs.File, body []token.Token) *Function {
return &Function{
Name: name,
File: file,
diff --git a/src/build/core/TokenToRegister.go b/src/build/core/TokenToRegister.go
index a40f471..f048fac 100644
--- a/src/build/core/TokenToRegister.go
+++ b/src/build/core/TokenToRegister.go
@@ -15,7 +15,7 @@ import (
func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error {
switch t.Kind {
case token.Identifier:
- name := t.Text()
+ name := t.Text(f.File.Bytes)
variable := f.VariableByName(name)
if variable == nil {
@@ -27,7 +27,7 @@ func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error {
return nil
case token.Number:
- value := t.Text()
+ value := t.Text(f.File.Bytes)
n, err := strconv.Atoi(value)
if err != nil {
@@ -40,7 +40,7 @@ func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error {
case token.String:
f.count.data++
label := fmt.Sprintf("%s_data_%d", f.Name, f.count.data)
- value := t.Bytes[1 : len(t.Bytes)-1]
+ value := t.Bytes(f.File.Bytes)[1 : t.Length-1]
f.Assembler.SetData(label, value)
f.RegisterLabel(asm.MOVE, register, label)
return nil
diff --git a/src/build/expression/Expression.go b/src/build/expression/Expression.go
index 60b3794..5027b8e 100644
--- a/src/build/expression/Expression.go
+++ b/src/build/expression/Expression.go
@@ -37,11 +37,11 @@ func (expr *Expression) AddChild(child *Expression) {
}
// Count counts how often the given token appears in the expression.
-func (expr *Expression) Count(kind token.Kind, name string) int {
+func (expr *Expression) Count(buffer []byte, kind token.Kind, name string) int {
count := 0
expr.EachLeaf(func(leaf *Expression) error {
- if leaf.Token.Kind == kind && leaf.Token.Text() == name {
+ if leaf.Token.Kind == kind && leaf.Token.Text(buffer) == name {
count++
}
@@ -112,25 +112,33 @@ func (expr *Expression) LastChild() *Expression {
}
// String generates a textual representation of the expression.
-func (expr *Expression) String() string {
+func (expr *Expression) String(data []byte) string {
builder := strings.Builder{}
- expr.write(&builder)
+ expr.write(&builder, data)
return builder.String()
}
// write generates a textual representation of the expression.
-func (expr *Expression) write(builder *strings.Builder) {
+func (expr *Expression) write(builder *strings.Builder, data []byte) {
if expr.IsLeaf() {
- builder.WriteString(expr.Token.Text())
+ builder.WriteString(expr.Token.Text(data))
return
}
builder.WriteByte('(')
- builder.WriteString(expr.Token.Text())
+
+ switch expr.Token.Kind {
+ case token.Call:
+ builder.WriteString("λ")
+ case token.Array:
+ builder.WriteString("@")
+ default:
+ builder.WriteString(expr.Token.Text(data))
+ }
for _, child := range expr.Children {
builder.WriteByte(' ')
- child.write(builder)
+ child.write(builder, data)
}
builder.WriteByte(')')
diff --git a/src/build/expression/Expression_test.go b/src/build/expression/Expression_test.go
index 6794da8..a313cf3 100644
--- a/src/build/expression/Expression_test.go
+++ b/src/build/expression/Expression_test.go
@@ -95,7 +95,7 @@ func TestParse(t *testing.T) {
defer expr.Reset()
assert.NotNil(t, expr)
- assert.Equal(t, expr.String(), test.Result)
+ assert.Equal(t, expr.String(src), test.Result)
})
}
}
@@ -104,11 +104,11 @@ func TestCount(t *testing.T) {
src := []byte("(a+b-c*d)+(a*b-c+d)")
tokens := token.Tokenize(src)
expr := expression.Parse(tokens)
- assert.Equal(t, expr.Count(token.Identifier, "a"), 2)
- assert.Equal(t, expr.Count(token.Identifier, "b"), 2)
- assert.Equal(t, expr.Count(token.Identifier, "c"), 2)
- assert.Equal(t, expr.Count(token.Identifier, "d"), 2)
- assert.Equal(t, expr.Count(token.Identifier, "e"), 0)
+ assert.Equal(t, expr.Count(src, token.Identifier, "a"), 2)
+ assert.Equal(t, expr.Count(src, token.Identifier, "b"), 2)
+ assert.Equal(t, expr.Count(src, token.Identifier, "c"), 2)
+ assert.Equal(t, expr.Count(src, token.Identifier, "d"), 2)
+ assert.Equal(t, expr.Count(src, token.Identifier, "e"), 0)
}
func TestEachLeaf(t *testing.T) {
@@ -118,7 +118,7 @@ func TestEachLeaf(t *testing.T) {
leaves := []string{}
err := expr.EachLeaf(func(leaf *expression.Expression) error {
- leaves = append(leaves, leaf.Token.Text())
+ leaves = append(leaves, leaf.Token.Text(src))
return nil
})
@@ -140,7 +140,7 @@ func TestEachParameter(t *testing.T) {
err := expression.EachParameter(tokens, func(parameter token.List) error {
expr := expression.Parse(parameter)
- parameters = append(parameters, expr.String())
+ parameters = append(parameters, expr.String(src))
return nil
})
@@ -178,17 +178,3 @@ func TestNilGroup(t *testing.T) {
expr := expression.Parse(tokens)
assert.Nil(t, expr)
}
-
-func TestInvalidOperator(t *testing.T) {
- src := []byte("a +++ 2")
- tokens := token.Tokenize(src)
- expr := expression.Parse(tokens)
- assert.Equal(t, expr.String(), "(+++ a 2)")
-}
-
-func TestInvalidOperatorCall(t *testing.T) {
- src := []byte("+++()")
- tokens := token.Tokenize(src)
- expr := expression.Parse(tokens)
- assert.NotNil(t, expr)
-}
diff --git a/src/build/expression/Operator.go b/src/build/expression/Operator.go
index b2d15e1..1d9e18b 100644
--- a/src/build/expression/Operator.go
+++ b/src/build/expression/Operator.go
@@ -15,39 +15,39 @@ type Operator struct {
// Operators defines the operators used in the language.
// The number corresponds to the operator priority and can not be zero.
-var Operators = map[string]*Operator{
- ".": {".", 13, 2},
- "λ": {"λ", 12, 1},
- "@": {"@", 12, 2},
- "!": {"!", 11, 1},
- "*": {"*", 10, 2},
- "/": {"/", 10, 2},
- "%": {"%", 10, 2},
- "+": {"+", 9, 2},
- "-": {"-", 9, 2},
- ">>": {">>", 8, 2},
- "<<": {"<<", 8, 2},
- "&": {"&", 7, 2},
- "^": {"^", 6, 2},
- "|": {"|", 5, 2},
+var Operators = map[token.Kind]*Operator{
+ token.Period: {".", 13, 2},
+ token.Call: {"λ", 12, 1},
+ token.Array: {"@", 12, 2},
+ token.Not: {"!", 11, 1},
+ token.Mul: {"*", 10, 2},
+ token.Div: {"/", 10, 2},
+ token.Mod: {"%", 10, 2},
+ token.Add: {"+", 9, 2},
+ token.Sub: {"-", 9, 2},
+ token.Shr: {">>", 8, 2},
+ token.Shl: {"<<", 8, 2},
+ token.And: {"&", 7, 2},
+ token.Xor: {"^", 6, 2},
+ token.Or: {"|", 5, 2},
- ">": {">", 4, 2},
- "<": {"<", 4, 2},
- ">=": {">=", 4, 2},
- "<=": {"<=", 4, 2},
- "==": {"==", 3, 2},
- "!=": {"!=", 3, 2},
- "&&": {"&&", 2, 2},
- "||": {"||", 1, 2},
+ token.Greater: {">", 4, 2},
+ token.Less: {"<", 4, 2},
+ token.GreaterEqual: {">=", 4, 2},
+ token.LessEqual: {"<=", 4, 2},
+ token.Equal: {"==", 3, 2},
+ token.NotEqual: {"!=", 3, 2},
+ token.LogicalAnd: {"&&", 2, 2},
+ token.LogicalOr: {"||", 1, 2},
- "=": {"=", math.MinInt8, 2},
- ":=": {":=", math.MinInt8, 2},
- "+=": {"+=", math.MinInt8, 2},
- "-=": {"-=", math.MinInt8, 2},
- "*=": {"*=", math.MinInt8, 2},
- "/=": {"/=", math.MinInt8, 2},
- ">>=": {">>=", math.MinInt8, 2},
- "<<=": {"<<=", math.MinInt8, 2},
+ token.Assign: {"=", math.MinInt8, 2},
+ token.Define: {":=", math.MinInt8, 2},
+ token.AddAssign: {"+=", math.MinInt8, 2},
+ token.SubAssign: {"-=", math.MinInt8, 2},
+ token.MulAssign: {"*=", math.MinInt8, 2},
+ token.DivAssign: {"/=", math.MinInt8, 2},
+ token.ShrAssign: {">>=", math.MinInt8, 2},
+ token.ShlAssign: {"<<=", math.MinInt8, 2},
}
func isComplete(expr *Expression) bool {
@@ -59,14 +59,14 @@ func isComplete(expr *Expression) bool {
return true
}
- if expr.Token.Kind == token.Operator && len(expr.Children) == numOperands(expr.Token.Text()) {
+ if expr.Token.IsOperator() && len(expr.Children) == numOperands(expr.Token.Kind) {
return true
}
return false
}
-func numOperands(symbol string) int {
+func numOperands(symbol token.Kind) int {
operator, exists := Operators[symbol]
if !exists {
@@ -76,7 +76,7 @@ func numOperands(symbol string) int {
return operator.Operands
}
-func precedence(symbol string) int8 {
+func precedence(symbol token.Kind) int8 {
operator, exists := Operators[symbol]
if !exists {
diff --git a/src/build/expression/Parse.go b/src/build/expression/Parse.go
index 33dd062..f2e8b56 100644
--- a/src/build/expression/Parse.go
+++ b/src/build/expression/Parse.go
@@ -6,13 +6,8 @@ import (
"git.akyoto.dev/cli/q/src/build/token"
)
-var (
- call = []byte("λ")
- array = []byte("@")
-)
-
// Parse generates an expression tree from tokens.
-func Parse(tokens token.List) *Expression {
+func Parse(tokens []token.Token) *Expression {
var (
cursor *Expression
root *Expression
@@ -43,20 +38,18 @@ func Parse(tokens token.List) *Expression {
parameters := NewList(tokens[groupPosition:i])
node := New()
- node.Token.Kind = token.Operator
node.Token.Position = tokens[groupPosition].Position
switch t.Kind {
case token.GroupEnd:
- node.Token.Bytes = call
- node.Precedence = precedence("λ")
-
+ node.Token.Kind = token.Call
case token.ArrayEnd:
- node.Token.Bytes = array
- node.Precedence = precedence("@")
+ node.Token.Kind = token.Array
}
- if cursor.Token.Kind == token.Operator && node.Precedence > cursor.Precedence {
+ node.Precedence = precedence(node.Token.Kind)
+
+ if cursor.Token.IsOperator() && node.Precedence > cursor.Precedence {
cursor.LastChild().Replace(node)
} else {
if cursor == root {
@@ -108,18 +101,18 @@ func Parse(tokens token.List) *Expression {
continue
}
- if t.Kind == token.Operator {
+ if t.IsOperator() {
if cursor == nil {
cursor = NewLeaf(t)
- cursor.Precedence = precedence(t.Text())
+ cursor.Precedence = precedence(t.Kind)
root = cursor
continue
}
node := NewLeaf(t)
- node.Precedence = precedence(t.Text())
+ node.Precedence = precedence(t.Kind)
- if cursor.Token.Kind == token.Operator {
+ if cursor.Token.IsOperator() {
oldPrecedence := cursor.Precedence
newPrecedence := node.Precedence
diff --git a/src/build/fs/File.go b/src/build/fs/File.go
index d4fd1cf..1a10a1d 100644
--- a/src/build/fs/File.go
+++ b/src/build/fs/File.go
@@ -5,5 +5,6 @@ import "git.akyoto.dev/cli/q/src/build/token"
// File represents a single source file.
type File struct {
Path string
+ Bytes []byte
Tokens token.List
}
diff --git a/src/build/keyword/Keyword.go b/src/build/keyword/Keyword.go
deleted file mode 100644
index ef62575..0000000
--- a/src/build/keyword/Keyword.go
+++ /dev/null
@@ -1,16 +0,0 @@
-package keyword
-
-const (
- If = "if"
- Import = "import"
- Loop = "loop"
- Return = "return"
-)
-
-// Map is a map of all keywords used in the language.
-var Map = map[string][]byte{
- If: []byte(If),
- Import: []byte(Import),
- Loop: []byte(Loop),
- Return: []byte(Return),
-}
diff --git a/src/build/scanner/scanFile.go b/src/build/scanner/scanFile.go
index b075fc6..e724767 100644
--- a/src/build/scanner/scanFile.go
+++ b/src/build/scanner/scanFile.go
@@ -11,7 +11,6 @@ import (
"git.akyoto.dev/cli/q/src/build/errors"
"git.akyoto.dev/cli/q/src/build/expression"
"git.akyoto.dev/cli/q/src/build/fs"
- "git.akyoto.dev/cli/q/src/build/keyword"
"git.akyoto.dev/cli/q/src/build/scope"
"git.akyoto.dev/cli/q/src/build/token"
)
@@ -27,8 +26,9 @@ func (s *Scanner) scanFile(path string, pkg string) error {
tokens := token.Tokenize(contents)
file := &fs.File{
- Tokens: tokens,
Path: path,
+ Bytes: contents,
+ Tokens: tokens,
}
var (
@@ -42,14 +42,14 @@ func (s *Scanner) scanFile(path string, pkg string) error {
)
for {
- for i < len(tokens) && tokens[i].Kind == token.Keyword && tokens[i].Text() == keyword.Import {
+ for i < len(tokens) && tokens[i].Kind == token.Import {
i++
if tokens[i].Kind != token.Identifier {
panic("expected package name")
}
- packageName := tokens[i].Text()
+ packageName := tokens[i].Text(contents)
s.queueDirectory(filepath.Join(config.Library, packageName), packageName)
i++
@@ -75,7 +75,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
}
if tokens[i].Kind == token.Invalid {
- return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
+ return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text(contents)}, file, tokens[i].Position)
}
if tokens[i].Kind == token.EOF {
@@ -116,7 +116,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
}
if tokens[i].Kind == token.Invalid {
- return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
+ return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text(contents)}, file, tokens[i].Position)
}
if tokens[i].Kind == token.EOF {
@@ -168,7 +168,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
}
if tokens[i].Kind == token.Invalid {
- return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
+ return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text(contents)}, file, tokens[i].Position)
}
if tokens[i].Kind == token.EOF {
@@ -191,7 +191,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
return errors.New(errors.ExpectedFunctionDefinition, file, tokens[i].Position)
}
- name := tokens[nameStart].Text()
+ name := tokens[nameStart].Text(contents)
body := tokens[bodyStart:i]
if pkg != "" {
@@ -207,9 +207,9 @@ func (s *Scanner) scanFile(path string, pkg string) error {
return errors.New(errors.NotImplemented, file, tokens[0].Position)
}
- name := tokens[0].Text()
+ name := tokens[0].Text(contents)
register := x64.CallRegisters[count]
- uses := token.Count(function.Body, token.Identifier, name)
+ uses := token.Count(function.Body, contents, token.Identifier, name)
if uses == 0 {
return errors.New(&errors.UnusedVariable{Name: name}, file, tokens[0].Position)
diff --git a/src/build/scope/Stack.go b/src/build/scope/Stack.go
index 5f634ff..1760f6b 100644
--- a/src/build/scope/Stack.go
+++ b/src/build/scope/Stack.go
@@ -20,7 +20,7 @@ func (stack *Stack) PopScope() {
}
// PushScope pushes a new scope to the top of the stack.
-func (stack *Stack) PushScope(body ast.AST) *Scope {
+func (stack *Stack) PushScope(body ast.AST, buffer []byte) *Scope {
s := &Scope{}
if len(stack.Scopes) > 0 {
@@ -30,7 +30,7 @@ func (stack *Stack) PushScope(body ast.AST) *Scope {
s.InLoop = lastScope.InLoop
for k, v := range lastScope.Variables {
- count := ast.Count(body, token.Identifier, v.Name)
+ count := ast.Count(body, buffer, token.Identifier, v.Name)
if count == 0 {
continue
diff --git a/src/build/token/Count.go b/src/build/token/Count.go
index ae83143..e83fff6 100644
--- a/src/build/token/Count.go
+++ b/src/build/token/Count.go
@@ -1,11 +1,11 @@
package token
// Count counts how often the given token appears in the token list.
-func Count(tokens List, kind Kind, name string) int {
+func Count(tokens []Token, buffer []byte, kind Kind, name string) int {
count := 0
for _, t := range tokens {
- if t.Kind == Identifier && t.Text() == name {
+ if t.Kind == Identifier && t.Text(buffer) == name {
count++
}
}
diff --git a/src/build/token/Count_test.go b/src/build/token/Count_test.go
index 0128e62..8742d13 100644
--- a/src/build/token/Count_test.go
+++ b/src/build/token/Count_test.go
@@ -8,9 +8,10 @@ import (
)
func TestCount(t *testing.T) {
- tokens := token.Tokenize([]byte(`a b b c c c`))
- assert.Equal(t, token.Count(tokens, token.Identifier, "a"), 1)
- assert.Equal(t, token.Count(tokens, token.Identifier, "b"), 2)
- assert.Equal(t, token.Count(tokens, token.Identifier, "c"), 3)
- assert.Equal(t, token.Count(tokens, token.Identifier, "d"), 0)
+ buffer := []byte(`a b b c c c`)
+ tokens := token.Tokenize(buffer)
+ assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "a"), 1)
+ assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "b"), 2)
+ assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "c"), 3)
+ assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "d"), 0)
}
diff --git a/src/build/token/Keywords.go b/src/build/token/Keywords.go
new file mode 100644
index 0000000..c75fd75
--- /dev/null
+++ b/src/build/token/Keywords.go
@@ -0,0 +1,9 @@
+package token
+
+// Keywords is a map of all keywords used in the language.
+var Keywords = map[string]Kind{
+ "if": If,
+ "import": Import,
+ "loop": Loop,
+ "return": Return,
+}
diff --git a/src/build/token/Kind.go b/src/build/token/Kind.go
index b221442..1df7adf 100644
--- a/src/build/token/Kind.go
+++ b/src/build/token/Kind.go
@@ -4,73 +4,62 @@ package token
type Kind uint8
const (
- // Invalid represents an invalid token.
- Invalid Kind = iota
-
- // EOF represents the end of file.
- EOF
-
- // NewLine represents the newline character.
- NewLine
-
- // Identifier represents a series of characters used to identify a variable or function.
- Identifier
-
- // Keyword represents a language keyword.
- Keyword
-
- // String represents an uninterpreted series of characters in the source code.
- String
-
- // Number represents a series of numerical characters.
- Number
-
- // Operator represents a mathematical operator.
- Operator
-
- // Separator represents a comma.
- Separator
-
- // Comment represents a comment.
- Comment
-
- // GroupStart represents '('.
- GroupStart
-
- // GroupEnd represents ')'.
- GroupEnd
-
- // BlockStart represents '{'.
- BlockStart
-
- // BlockEnd represents '}'.
- BlockEnd
-
- // ArrayStart represents '['.
- ArrayStart
-
- // ArrayEnd represents ']'.
- ArrayEnd
+ Invalid Kind = iota // Invalid is an invalid token.
+ EOF // EOF is the end of file.
+ NewLine // NewLine is the newline character.
+ Identifier // Identifier is a series of characters used to identify a variable or function.
+ Number // Number is a series of numerical characters.
+ String // String is an uninterpreted series of characters in the source code.
+ Comment // Comment is a comment.
+ Separator // ,
+ GroupStart // (
+ GroupEnd // )
+ BlockStart // {
+ BlockEnd // }
+ ArrayStart // [
+ ArrayEnd // ]
+ _keywords //
+ If // if
+ Import // import
+ Loop // loop
+ Return // return
+ _keywordsEnd //
+ _operators //
+ Add // +
+ Sub // -
+ Mul // *
+ Div // /
+ Mod // %
+ And // &
+ Or // |
+ Xor // ^
+ Shl // <<
+ Shr // >>
+ LogicalAnd // &&
+ LogicalOr // ||
+ Equal // ==
+ Less // <
+ Greater // >
+ Not // !
+ NotEqual // !=
+ LessEqual // <=
+ GreaterEqual // >=
+ Define // :=
+ Period // .
+ Call // x()
+ Array // [x]
+ _assignments //
+ Assign // =
+ AddAssign // +=
+ SubAssign // -=
+ MulAssign // *=
+ DivAssign // /=
+ ModAssign // %=
+ AndAssign // &=
+ OrAssign // |=
+ XorAssign // ^=
+ ShlAssign // <<=
+ ShrAssign // >>=
+ _assignmentsEnd //
+ _operatorsEnd //
)
-
-// String returns the text representation.
-func (kind Kind) String() string {
- return [...]string{
- "Invalid",
- "EOF",
- "NewLine",
- "Identifier",
- "Keyword",
- "String",
- "Number",
- "Operator",
- "Separator",
- "Comment",
- "GroupStart",
- "GroupEnd",
- "BlockStart",
- "BlockEnd",
- "ArrayStart",
- "ArrayEnd",
- }[kind]
-}
diff --git a/src/build/token/Kind_test.go b/src/build/token/Kind_test.go
deleted file mode 100644
index 1db0382..0000000
--- a/src/build/token/Kind_test.go
+++ /dev/null
@@ -1,27 +0,0 @@
-package token_test
-
-import (
- "testing"
-
- "git.akyoto.dev/cli/q/src/build/token"
- "git.akyoto.dev/go/assert"
-)
-
-func TestTokenKind(t *testing.T) {
- assert.Equal(t, token.Invalid.String(), "Invalid")
- assert.Equal(t, token.EOF.String(), "EOF")
- assert.Equal(t, token.NewLine.String(), "NewLine")
- assert.Equal(t, token.Identifier.String(), "Identifier")
- assert.Equal(t, token.Keyword.String(), "Keyword")
- assert.Equal(t, token.String.String(), "String")
- assert.Equal(t, token.Number.String(), "Number")
- assert.Equal(t, token.Operator.String(), "Operator")
- assert.Equal(t, token.Separator.String(), "Separator")
- assert.Equal(t, token.Comment.String(), "Comment")
- assert.Equal(t, token.GroupStart.String(), "GroupStart")
- assert.Equal(t, token.GroupEnd.String(), "GroupEnd")
- assert.Equal(t, token.BlockStart.String(), "BlockStart")
- assert.Equal(t, token.BlockEnd.String(), "BlockEnd")
- assert.Equal(t, token.ArrayStart.String(), "ArrayStart")
- assert.Equal(t, token.ArrayEnd.String(), "ArrayEnd")
-}
diff --git a/src/build/token/Length.go b/src/build/token/Length.go
new file mode 100644
index 0000000..6711706
--- /dev/null
+++ b/src/build/token/Length.go
@@ -0,0 +1,4 @@
+package token
+
+// Length is the data type for storing token lengths.
+type Length = uint16
diff --git a/src/build/token/List.go b/src/build/token/List.go
index 1b725b6..7e57693 100644
--- a/src/build/token/List.go
+++ b/src/build/token/List.go
@@ -1,9 +1,5 @@
package token
-import (
- "bytes"
-)
-
// List is a slice of tokens.
type List []Token
@@ -28,20 +24,3 @@ func (list List) LastIndexKind(kind Kind) int {
return -1
}
-
-// String implements string serialization.
-func (list List) String() string {
- builder := bytes.Buffer{}
- var last Token
-
- for _, t := range list {
- if last.Kind == Keyword || last.Kind == Separator || last.Kind == Operator || t.Kind == Operator {
- builder.WriteByte(' ')
- }
-
- builder.Write(t.Bytes)
- last = t
- }
-
- return builder.String()
-}
diff --git a/src/build/token/Operators.go b/src/build/token/Operators.go
new file mode 100644
index 0000000..842fb94
--- /dev/null
+++ b/src/build/token/Operators.go
@@ -0,0 +1,39 @@
+package token
+
+// Operators is a map of all operators used in the language.
+var Operators = map[string]Kind{
+ ".": Period,
+ "=": Assign,
+ ":=": Define,
+ "+": Add,
+ "-": Sub,
+ "*": Mul,
+ "/": Div,
+ "%": Mod,
+ "&": And,
+ "|": Or,
+ "^": Xor,
+ "<<": Shl,
+ ">>": Shr,
+ "&&": LogicalAnd,
+ "||": LogicalOr,
+ "!": Not,
+ "==": Equal,
+ "!=": NotEqual,
+ ">": Greater,
+ "<": Less,
+ ">=": GreaterEqual,
+ "<=": LessEqual,
+ "+=": AddAssign,
+ "-=": SubAssign,
+ "*=": MulAssign,
+ "/=": DivAssign,
+ "%=": ModAssign,
+ "&=": AndAssign,
+ "|=": OrAssign,
+ "^=": XorAssign,
+ "<<=": ShlAssign,
+ ">>=": ShrAssign,
+ "λ": Call,
+ "@": Array,
+}
diff --git a/src/build/token/Token.go b/src/build/token/Token.go
index 71dd1e5..134f054 100644
--- a/src/build/token/Token.go
+++ b/src/build/token/Token.go
@@ -1,7 +1,6 @@
package token
import (
- "fmt"
"unsafe"
)
@@ -9,29 +8,44 @@ import (
// The characters that make up an identifier are grouped into a single token.
// This makes parsing easier and allows us to do better syntax checks.
type Token struct {
- Bytes []byte
Position Position
+ Length Length
Kind Kind
}
-// End returns the position after the token.
-func (t *Token) End() Position {
- return t.Position + Position(len(t.Bytes))
+// Bytes returns the byte slice.
+func (t Token) Bytes(buffer []byte) []byte {
+ return buffer[t.Position : t.Position+Position(t.Length)]
}
-// String creates a human readable representation for debugging purposes.
-func (t *Token) String() string {
- return fmt.Sprintf("%s %s", t.Kind, t.Text())
+// End returns the position after the token.
+func (t Token) End() Position {
+ return t.Position + Position(t.Length)
+}
+
+// IsAssignment returns true if the token is an assignment operator.
+func (t Token) IsAssignment() bool {
+ return t.Kind > _assignments && t.Kind < _assignmentsEnd
+}
+
+// IsKeyword returns true if the token is a keyword.
+func (t Token) IsKeyword() bool {
+ return t.Kind > _keywords && t.Kind < _keywordsEnd
+}
+
+// IsOperator returns true if the token is an operator.
+func (t Token) IsOperator() bool {
+ return t.Kind > _operators && t.Kind < _operatorsEnd
}
// Reset resets the token to default values.
func (t *Token) Reset() {
- t.Kind = Invalid
t.Position = 0
- t.Bytes = nil
+ t.Length = 0
+ t.Kind = Invalid
}
// Text returns the token text.
-func (t *Token) Text() string {
- return unsafe.String(unsafe.SliceData(t.Bytes), len(t.Bytes))
+func (t Token) Text(buffer []byte) string {
+ return unsafe.String(unsafe.SliceData(t.Bytes(buffer)), t.Length)
}
diff --git a/src/build/token/Token_test.go b/src/build/token/Token_test.go
index 47fc665..ba825cc 100644
--- a/src/build/token/Token_test.go
+++ b/src/build/token/Token_test.go
@@ -10,8 +10,8 @@ import (
func TestTokenEnd(t *testing.T) {
hello := token.Token{
Kind: token.Identifier,
- Bytes: []byte("hello"),
Position: 0,
+ Length: 5,
}
assert.Equal(t, hello.End(), 5)
@@ -20,34 +20,33 @@ func TestTokenEnd(t *testing.T) {
func TestTokenReset(t *testing.T) {
hello := token.Token{
Kind: token.Identifier,
- Bytes: []byte("hello"),
Position: 1,
+ Length: 5,
}
hello.Reset()
- assert.Nil(t, hello.Bytes)
assert.Equal(t, hello.Position, 0)
+ assert.Equal(t, hello.Length, 0)
assert.Equal(t, hello.Kind, token.Invalid)
}
-func TestTokenString(t *testing.T) {
- hello := token.Token{
- Kind: token.Identifier,
- Bytes: []byte("hello"),
- Position: 0,
- }
-
- assert.Equal(t, hello.String(), "Identifier hello")
-}
-
func TestTokenText(t *testing.T) {
- hello := token.Token{Kind: token.Identifier, Bytes: []byte("hello"), Position: 0}
- comma := token.Token{Kind: token.Separator, Bytes: []byte(","), Position: 5}
- world := token.Token{Kind: token.Identifier, Bytes: []byte("world"), Position: 7}
+ buffer := []byte("hello, world")
+ hello := token.Token{Kind: token.Identifier, Position: 0, Length: 5}
+ comma := token.Token{Kind: token.Separator, Position: 5, Length: 1}
+ world := token.Token{Kind: token.Identifier, Position: 7, Length: 5}
- assert.Equal(t, hello.Text(), "hello")
- assert.Equal(t, world.Text(), "world")
-
- list := token.List{hello, comma, world}
- assert.Equal(t, list.String(), "hello, world")
+ assert.Equal(t, hello.Text(buffer), "hello")
+ assert.Equal(t, comma.Text(buffer), ",")
+ assert.Equal(t, world.Text(buffer), "world")
+}
+
+func TestTokenGroups(t *testing.T) {
+ assignment := token.Token{Kind: token.Assign}
+ operator := token.Token{Kind: token.Add}
+ keyword := token.Token{Kind: token.If}
+
+ assert.True(t, assignment.IsAssignment())
+ assert.True(t, operator.IsOperator())
+ assert.True(t, keyword.IsKeyword())
}
diff --git a/src/build/token/Tokenize.go b/src/build/token/Tokenize.go
index 649f376..0060958 100644
--- a/src/build/token/Tokenize.go
+++ b/src/build/token/Tokenize.go
@@ -1,20 +1,5 @@
package token
-import "git.akyoto.dev/cli/q/src/build/keyword"
-
-// Pre-allocate these byte buffers so we can re-use them
-// instead of allocating a new buffer every time.
-var (
- groupStartBytes = []byte{'('}
- groupEndBytes = []byte{')'}
- blockStartBytes = []byte{'{'}
- blockEndBytes = []byte{'}'}
- arrayStartBytes = []byte{'['}
- arrayEndBytes = []byte{']'}
- separatorBytes = []byte{','}
- newLineBytes = []byte{'\n'}
-)
-
// Tokenize turns the file contents into a list of tokens.
func Tokenize(buffer []byte) List {
var (
@@ -26,21 +11,21 @@ func Tokenize(buffer []byte) List {
switch buffer[i] {
case ' ', '\t':
case ',':
- tokens = append(tokens, Token{Kind: Separator, Position: i, Bytes: separatorBytes})
+ tokens = append(tokens, Token{Kind: Separator, Position: i, Length: 1})
case '(':
- tokens = append(tokens, Token{Kind: GroupStart, Position: i, Bytes: groupStartBytes})
+ tokens = append(tokens, Token{Kind: GroupStart, Position: i, Length: 1})
case ')':
- tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Bytes: groupEndBytes})
+ tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Length: 1})
case '{':
- tokens = append(tokens, Token{Kind: BlockStart, Position: i, Bytes: blockStartBytes})
+ tokens = append(tokens, Token{Kind: BlockStart, Position: i, Length: 1})
case '}':
- tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Bytes: blockEndBytes})
+ tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Length: 1})
case '[':
- tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Bytes: arrayStartBytes})
+ tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Length: 1})
case ']':
- tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Bytes: arrayEndBytes})
+ tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Length: 1})
case '\n':
- tokens = append(tokens, Token{Kind: NewLine, Position: i, Bytes: newLineBytes})
+ tokens = append(tokens, Token{Kind: NewLine, Position: i, Length: 1})
case '/':
if i+1 >= Position(len(buffer)) || buffer[i+1] != '/' {
position := i
@@ -50,7 +35,7 @@ func Tokenize(buffer []byte) List {
i++
}
- tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]})
+ tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
} else {
position := i
@@ -58,7 +43,7 @@ func Tokenize(buffer []byte) List {
i++
}
- tokens = append(tokens, Token{Kind: Comment, Position: position, Bytes: buffer[position:i]})
+ tokens = append(tokens, Token{Kind: Comment, Position: position, Length: Length(i - position)})
}
continue
@@ -78,7 +63,7 @@ func Tokenize(buffer []byte) List {
i++
}
- tokens = append(tokens, Token{Kind: String, Position: start, Bytes: buffer[start:end]})
+ tokens = append(tokens, Token{Kind: String, Position: start, Length: Length(end - start)})
continue
default:
@@ -91,14 +76,14 @@ func Tokenize(buffer []byte) List {
}
identifier := buffer[position:i]
- keyword, isKeyword := keyword.Map[string(identifier)]
+ kind := Identifier
+ keyword, isKeyword := Keywords[string(identifier)]
if isKeyword {
- tokens = append(tokens, Token{Kind: Keyword, Position: position, Bytes: keyword})
- } else {
- tokens = append(tokens, Token{Kind: Identifier, Position: position, Bytes: identifier})
+ kind = keyword
}
+ tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(len(identifier))})
continue
}
@@ -110,7 +95,7 @@ func Tokenize(buffer []byte) List {
i++
}
- tokens = append(tokens, Token{Kind: Number, Position: position, Bytes: buffer[position:i]})
+ tokens = append(tokens, Token{Kind: Number, Position: position, Length: Length(i - position)})
continue
}
@@ -122,17 +107,17 @@ func Tokenize(buffer []byte) List {
i++
}
- tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]})
+ tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
continue
}
- tokens = append(tokens, Token{Kind: Invalid, Position: i, Bytes: buffer[i : i+1]})
+ tokens = append(tokens, Token{Kind: Invalid, Position: i, Length: 1})
}
i++
}
- tokens = append(tokens, Token{Kind: EOF, Position: i, Bytes: nil})
+ tokens = append(tokens, Token{Kind: EOF, Position: i, Length: 0})
return tokens
}
diff --git a/src/build/token/Tokenize_test.go b/src/build/token/Tokenize_test.go
index 63e689d..19f57e5 100644
--- a/src/build/token/Tokenize_test.go
+++ b/src/build/token/Tokenize_test.go
@@ -9,394 +9,243 @@ import (
func TestFunction(t *testing.T) {
tokens := token.Tokenize([]byte("main(){}"))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.Identifier,
- Bytes: []byte("main"),
- Position: 0,
- },
- {
- Kind: token.GroupStart,
- Bytes: []byte("("),
- Position: 4,
- },
- {
- Kind: token.GroupEnd,
- Bytes: []byte(")"),
- Position: 5,
- },
- {
- Kind: token.BlockStart,
- Bytes: []byte("{"),
- Position: 6,
- },
- {
- Kind: token.BlockEnd,
- Bytes: []byte("}"),
- Position: 7,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 8,
- },
- })
+
+ expected := []token.Kind{
+ token.Identifier,
+ token.GroupStart,
+ token.GroupEnd,
+ token.BlockStart,
+ token.BlockEnd,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
}
func TestKeyword(t *testing.T) {
tokens := token.Tokenize([]byte("return x"))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.Keyword,
- Bytes: []byte("return"),
- Position: 0,
- },
- {
- Kind: token.Identifier,
- Bytes: []byte("x"),
- Position: 7,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 8,
- },
- })
+
+ expected := []token.Kind{
+ token.Return,
+ token.Identifier,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
}
func TestArray(t *testing.T) {
tokens := token.Tokenize([]byte("array[i]"))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.Identifier,
- Bytes: []byte("array"),
- Position: 0,
- },
- {
- Kind: token.ArrayStart,
- Bytes: []byte("["),
- Position: 5,
- },
- {
- Kind: token.Identifier,
- Bytes: []byte("i"),
- Position: 6,
- },
- {
- Kind: token.ArrayEnd,
- Bytes: []byte("]"),
- Position: 7,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 8,
- },
- })
+
+ expected := []token.Kind{
+ token.Identifier,
+ token.ArrayStart,
+ token.Identifier,
+ token.ArrayEnd,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
}
func TestNewline(t *testing.T) {
tokens := token.Tokenize([]byte("\n\n"))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.NewLine,
- Bytes: []byte("\n"),
- Position: 0,
- },
- {
- Kind: token.NewLine,
- Bytes: []byte("\n"),
- Position: 1,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 2,
- },
- })
+
+ expected := []token.Kind{
+ token.NewLine,
+ token.NewLine,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
}
func TestNumber(t *testing.T) {
tokens := token.Tokenize([]byte(`123 456`))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.Number,
- Bytes: []byte("123"),
- Position: 0,
- },
- {
- Kind: token.Number,
- Bytes: []byte("456"),
- Position: 4,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 7,
- },
- })
+
+ expected := []token.Kind{
+ token.Number,
+ token.Number,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
}
func TestOperator(t *testing.T) {
tokens := token.Tokenize([]byte(`+ - * /`))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.Operator,
- Bytes: []byte("+"),
- Position: 0,
- },
- {
- Kind: token.Operator,
- Bytes: []byte("-"),
- Position: 2,
- },
- {
- Kind: token.Operator,
- Bytes: []byte("*"),
- Position: 4,
- },
- {
- Kind: token.Operator,
- Bytes: []byte("/"),
- Position: 6,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 7,
- },
- })
+
+ expected := []token.Kind{
+ token.Add,
+ token.Sub,
+ token.Mul,
+ token.Div,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
}
func TestOperatorAssign(t *testing.T) {
- tokens := token.Tokenize([]byte(`+= -= *= /= ==`))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.Operator,
- Bytes: []byte("+="),
- Position: 0,
- },
- {
- Kind: token.Operator,
- Bytes: []byte("-="),
- Position: 3,
- },
- {
- Kind: token.Operator,
- Bytes: []byte("*="),
- Position: 6,
- },
- {
- Kind: token.Operator,
- Bytes: []byte("/="),
- Position: 9,
- },
- {
- Kind: token.Operator,
- Bytes: []byte("=="),
- Position: 12,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 14,
- },
- })
+ tokens := token.Tokenize([]byte(`+= -= *= /= &= |= ^= <<= >>=`))
+
+ expected := []token.Kind{
+ token.AddAssign,
+ token.SubAssign,
+ token.MulAssign,
+ token.DivAssign,
+ token.AndAssign,
+ token.OrAssign,
+ token.XorAssign,
+ token.ShlAssign,
+ token.ShrAssign,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
}
func TestSeparator(t *testing.T) {
tokens := token.Tokenize([]byte("a,b,c"))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.Identifier,
- Bytes: []byte("a"),
- Position: 0,
- },
- {
- Kind: token.Separator,
- Bytes: []byte(","),
- Position: 1,
- },
- {
- Kind: token.Identifier,
- Bytes: []byte("b"),
- Position: 2,
- },
- {
- Kind: token.Separator,
- Bytes: []byte(","),
- Position: 3,
- },
- {
- Kind: token.Identifier,
- Bytes: []byte("c"),
- Position: 4,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 5,
- },
- })
+
+ expected := []token.Kind{
+ token.Identifier,
+ token.Separator,
+ token.Identifier,
+ token.Separator,
+ token.Identifier,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
}
func TestComment(t *testing.T) {
tokens := token.Tokenize([]byte("// Hello\n// World"))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.Comment,
- Bytes: []byte(`// Hello`),
- Position: 0,
- },
- {
- Kind: token.NewLine,
- Bytes: []byte("\n"),
- Position: 8,
- },
- {
- Kind: token.Comment,
- Bytes: []byte(`// World`),
- Position: 9,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 17,
- },
- })
+
+ expected := []token.Kind{
+ token.Comment,
+ token.NewLine,
+ token.Comment,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
tokens = token.Tokenize([]byte("// Hello\n"))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.Comment,
- Bytes: []byte(`// Hello`),
- Position: 0,
- },
- {
- Kind: token.NewLine,
- Bytes: []byte("\n"),
- Position: 8,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 9,
- },
- })
+
+ expected = []token.Kind{
+ token.Comment,
+ token.NewLine,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
tokens = token.Tokenize([]byte(`// Hello`))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.Comment,
- Bytes: []byte(`// Hello`),
- Position: 0,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 8,
- },
- })
+
+ expected = []token.Kind{
+ token.Comment,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
tokens = token.Tokenize([]byte(`//`))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.Comment,
- Bytes: []byte(`//`),
- Position: 0,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 2,
- },
- })
+
+ expected = []token.Kind{
+ token.Comment,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
tokens = token.Tokenize([]byte(`/`))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.Operator,
- Bytes: []byte(`/`),
- Position: 0,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 1,
- },
- })
+
+ expected = []token.Kind{
+ token.Div,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
}
func TestInvalid(t *testing.T) {
- tokens := token.Tokenize([]byte(`@#`))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.Invalid,
- Bytes: []byte(`@`),
- Position: 0,
- },
- {
- Kind: token.Invalid,
- Bytes: []byte(`#`),
- Position: 1,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 2,
- },
- })
+ tokens := token.Tokenize([]byte(`##`))
+
+ expected := []token.Kind{
+ token.Invalid,
+ token.Invalid,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
}
func TestString(t *testing.T) {
tokens := token.Tokenize([]byte(`"Hello" "World"`))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.String,
- Bytes: []byte(`"Hello"`),
- Position: 0,
- },
- {
- Kind: token.String,
- Bytes: []byte(`"World"`),
- Position: 8,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 15,
- },
- })
+
+ expected := []token.Kind{
+ token.String,
+ token.String,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
}
func TestStringMultiline(t *testing.T) {
tokens := token.Tokenize([]byte("\"Hello\nWorld\""))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.String,
- Bytes: []byte("\"Hello\nWorld\""),
- Position: 0,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 13,
- },
- })
+
+ expected := []token.Kind{
+ token.String,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
}
func TestStringEOF(t *testing.T) {
tokens := token.Tokenize([]byte(`"EOF`))
- assert.DeepEqual(t, tokens, token.List{
- {
- Kind: token.String,
- Bytes: []byte(`"EOF`),
- Position: 0,
- },
- {
- Kind: token.EOF,
- Bytes: nil,
- Position: 4,
- },
- })
+
+ expected := []token.Kind{
+ token.String,
+ token.EOF,
+ }
+
+ for i, kind := range expected {
+ assert.Equal(t, tokens[i].Kind, kind)
+ }
}
diff --git a/tests/errors/InvalidOperator.q b/tests/errors/InvalidCharacter4.q
similarity index 100%
rename from tests/errors/InvalidOperator.q
rename to tests/errors/InvalidCharacter4.q
diff --git a/tests/errors_test.go b/tests/errors_test.go
index 6c08acd..5b28ee2 100644
--- a/tests/errors_test.go
+++ b/tests/errors_test.go
@@ -21,10 +21,10 @@ var errs = []struct {
{"InvalidInstructionNumber.q", &errors.InvalidInstruction{Instruction: "123"}},
{"InvalidInstructionExpression.q", &errors.InvalidInstruction{Instruction: "+"}},
{"InvalidExpression.q", errors.InvalidExpression},
- {"InvalidOperator.q", &errors.InvalidOperator{Operator: "+++"}},
{"InvalidCharacter.q", &errors.InvalidCharacter{Character: "@"}},
{"InvalidCharacter2.q", &errors.InvalidCharacter{Character: "@"}},
{"InvalidCharacter3.q", &errors.InvalidCharacter{Character: "@"}},
+ {"InvalidCharacter4.q", &errors.InvalidCharacter{Character: "+++"}},
{"MissingBlockEnd.q", errors.MissingBlockEnd},
{"MissingBlockStart.q", errors.MissingBlockStart},
{"MissingGroupEnd.q", errors.MissingGroupEnd},