Reduced token size
This commit is contained in:
parent
ca36d34cb9
commit
04ba68a075
@ -1,6 +1,4 @@
|
|||||||
package ast
|
package ast
|
||||||
|
|
||||||
import "fmt"
|
type Node any
|
||||||
|
|
||||||
type Node fmt.Stringer
|
|
||||||
type AST []Node
|
type AST []Node
|
||||||
|
@ -8,7 +8,3 @@ import (
|
|||||||
type Assign struct {
|
type Assign struct {
|
||||||
Expression *expression.Expression
|
Expression *expression.Expression
|
||||||
}
|
}
|
||||||
|
|
||||||
func (node *Assign) String() string {
|
|
||||||
return node.Expression.String()
|
|
||||||
}
|
|
||||||
|
@ -6,7 +6,3 @@ import "git.akyoto.dev/cli/q/src/build/expression"
|
|||||||
type Call struct {
|
type Call struct {
|
||||||
Expression *expression.Expression
|
Expression *expression.Expression
|
||||||
}
|
}
|
||||||
|
|
||||||
func (node *Call) String() string {
|
|
||||||
return node.Expression.String()
|
|
||||||
}
|
|
||||||
|
@ -3,31 +3,31 @@ package ast
|
|||||||
import "git.akyoto.dev/cli/q/src/build/token"
|
import "git.akyoto.dev/cli/q/src/build/token"
|
||||||
|
|
||||||
// Count counts how often the given token appears in the AST.
|
// Count counts how often the given token appears in the AST.
|
||||||
func Count(body AST, kind token.Kind, name string) int {
|
func Count(body AST, buffer []byte, kind token.Kind, name string) int {
|
||||||
count := 0
|
count := 0
|
||||||
|
|
||||||
for _, node := range body {
|
for _, node := range body {
|
||||||
switch node := node.(type) {
|
switch node := node.(type) {
|
||||||
case *Assign:
|
case *Assign:
|
||||||
count += node.Expression.Count(kind, name)
|
count += node.Expression.Count(buffer, kind, name)
|
||||||
|
|
||||||
case *Call:
|
case *Call:
|
||||||
count += node.Expression.Count(kind, name)
|
count += node.Expression.Count(buffer, kind, name)
|
||||||
|
|
||||||
case *Define:
|
case *Define:
|
||||||
count += node.Value.Count(kind, name)
|
count += node.Value.Count(buffer, kind, name)
|
||||||
|
|
||||||
case *Return:
|
case *Return:
|
||||||
if node.Value != nil {
|
if node.Value != nil {
|
||||||
count += node.Value.Count(kind, name)
|
count += node.Value.Count(buffer, kind, name)
|
||||||
}
|
}
|
||||||
|
|
||||||
case *If:
|
case *If:
|
||||||
count += node.Condition.Count(kind, name)
|
count += node.Condition.Count(buffer, kind, name)
|
||||||
count += Count(node.Body, kind, name)
|
count += Count(node.Body, buffer, kind, name)
|
||||||
|
|
||||||
case *Loop:
|
case *Loop:
|
||||||
count += Count(node.Body, kind, name)
|
count += Count(node.Body, buffer, kind, name)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
panic("unknown AST type")
|
panic("unknown AST type")
|
||||||
|
@ -1,8 +1,6 @@
|
|||||||
package ast
|
package ast
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"git.akyoto.dev/cli/q/src/build/expression"
|
"git.akyoto.dev/cli/q/src/build/expression"
|
||||||
"git.akyoto.dev/cli/q/src/build/token"
|
"git.akyoto.dev/cli/q/src/build/token"
|
||||||
)
|
)
|
||||||
@ -12,7 +10,3 @@ type Define struct {
|
|||||||
Value *expression.Expression
|
Value *expression.Expression
|
||||||
Name token.Token
|
Name token.Token
|
||||||
}
|
}
|
||||||
|
|
||||||
func (node *Define) String() string {
|
|
||||||
return fmt.Sprintf("(= %s %s)", node.Name.Text(), node.Value)
|
|
||||||
}
|
|
||||||
|
@ -1,8 +1,6 @@
|
|||||||
package ast
|
package ast
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"git.akyoto.dev/cli/q/src/build/expression"
|
"git.akyoto.dev/cli/q/src/build/expression"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -11,7 +9,3 @@ type If struct {
|
|||||||
Condition *expression.Expression
|
Condition *expression.Expression
|
||||||
Body AST
|
Body AST
|
||||||
}
|
}
|
||||||
|
|
||||||
func (node *If) String() string {
|
|
||||||
return fmt.Sprintf("(if %s %s)", node.Condition, node.Body)
|
|
||||||
}
|
|
||||||
|
@ -1,12 +1,6 @@
|
|||||||
package ast
|
package ast
|
||||||
|
|
||||||
import "fmt"
|
|
||||||
|
|
||||||
// Loop represents a block of repeatable statements.
|
// Loop represents a block of repeatable statements.
|
||||||
type Loop struct {
|
type Loop struct {
|
||||||
Body AST
|
Body AST
|
||||||
}
|
}
|
||||||
|
|
||||||
func (node *Loop) String() string {
|
|
||||||
return fmt.Sprintf("(loop %s)", node.Body)
|
|
||||||
}
|
|
||||||
|
@ -3,16 +3,15 @@ package ast
|
|||||||
import (
|
import (
|
||||||
"git.akyoto.dev/cli/q/src/build/errors"
|
"git.akyoto.dev/cli/q/src/build/errors"
|
||||||
"git.akyoto.dev/cli/q/src/build/expression"
|
"git.akyoto.dev/cli/q/src/build/expression"
|
||||||
"git.akyoto.dev/cli/q/src/build/keyword"
|
|
||||||
"git.akyoto.dev/cli/q/src/build/token"
|
"git.akyoto.dev/cli/q/src/build/token"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Parse generates an AST from a list of tokens.
|
// Parse generates an AST from a list of tokens.
|
||||||
func Parse(tokens token.List) (AST, error) {
|
func Parse(tokens []token.Token, buffer []byte) (AST, error) {
|
||||||
tree := make(AST, 0, len(tokens)/64)
|
tree := make(AST, 0, len(tokens)/64)
|
||||||
|
|
||||||
err := EachInstruction(tokens, func(instruction token.List) error {
|
err := EachInstruction(tokens, func(instruction token.List) error {
|
||||||
node, err := toASTNode(instruction)
|
node, err := toASTNode(instruction, buffer)
|
||||||
|
|
||||||
if err == nil && node != nil {
|
if err == nil && node != nil {
|
||||||
tree = append(tree, node)
|
tree = append(tree, node)
|
||||||
@ -25,11 +24,9 @@ func Parse(tokens token.List) (AST, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// toASTNode generates an AST node from an instruction.
|
// toASTNode generates an AST node from an instruction.
|
||||||
func toASTNode(tokens token.List) (Node, error) {
|
func toASTNode(tokens token.List, buffer []byte) (Node, error) {
|
||||||
if tokens[0].Kind == token.Keyword {
|
if tokens[0].IsKeyword() {
|
||||||
word := tokens[0].Text()
|
if tokens[0].Kind == token.Return {
|
||||||
|
|
||||||
if word == keyword.Return {
|
|
||||||
if len(tokens) == 1 {
|
if len(tokens) == 1 {
|
||||||
return &Return{}, nil
|
return &Return{}, nil
|
||||||
}
|
}
|
||||||
@ -38,7 +35,7 @@ func toASTNode(tokens token.List) (Node, error) {
|
|||||||
return &Return{Value: value}, nil
|
return &Return{Value: value}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if keywordHasBlock(word) {
|
if keywordHasBlock(tokens[0].Kind) {
|
||||||
blockStart := tokens.IndexKind(token.BlockStart)
|
blockStart := tokens.IndexKind(token.BlockStart)
|
||||||
blockEnd := tokens.LastIndexKind(token.BlockEnd)
|
blockEnd := tokens.LastIndexKind(token.BlockEnd)
|
||||||
|
|
||||||
@ -50,19 +47,19 @@ func toASTNode(tokens token.List) (Node, error) {
|
|||||||
return nil, errors.New(errors.MissingBlockEnd, nil, tokens[len(tokens)-1].End())
|
return nil, errors.New(errors.MissingBlockEnd, nil, tokens[len(tokens)-1].End())
|
||||||
}
|
}
|
||||||
|
|
||||||
body, err := Parse(tokens[blockStart+1 : blockEnd])
|
body, err := Parse(tokens[blockStart+1:blockEnd], buffer)
|
||||||
|
|
||||||
switch word {
|
switch tokens[0].Kind {
|
||||||
case keyword.If:
|
case token.If:
|
||||||
condition := expression.Parse(tokens[1:blockStart])
|
condition := expression.Parse(tokens[1:blockStart])
|
||||||
return &If{Condition: condition, Body: body}, err
|
return &If{Condition: condition, Body: body}, err
|
||||||
|
|
||||||
case keyword.Loop:
|
case token.Loop:
|
||||||
return &Loop{Body: body}, err
|
return &Loop{Body: body}, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, errors.New(&errors.KeywordNotImplemented{Keyword: word}, nil, tokens[0].Position)
|
return nil, errors.New(&errors.KeywordNotImplemented{Keyword: tokens[0].Text(buffer)}, nil, tokens[0].Position)
|
||||||
}
|
}
|
||||||
|
|
||||||
expr := expression.Parse(tokens)
|
expr := expression.Parse(tokens)
|
||||||
@ -92,26 +89,26 @@ func toASTNode(tokens token.List) (Node, error) {
|
|||||||
return &Call{Expression: expr}, nil
|
return &Call{Expression: expr}, nil
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return nil, errors.New(&errors.InvalidInstruction{Instruction: expr.Token.Text()}, nil, expr.Token.Position)
|
return nil, errors.New(&errors.InvalidInstruction{Instruction: expr.Token.Text(buffer)}, nil, expr.Token.Position)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsAssignment returns true if the expression is an assignment.
|
// IsAssignment returns true if the expression is an assignment.
|
||||||
func IsAssignment(expr *expression.Expression) bool {
|
func IsAssignment(expr *expression.Expression) bool {
|
||||||
return expr.Token.Kind == token.Operator && expr.Token.Bytes[len(expr.Token.Bytes)-1] == '='
|
return expr.Token.IsAssignment()
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsFunctionCall returns true if the expression is a function call.
|
// IsFunctionCall returns true if the expression is a function call.
|
||||||
func IsFunctionCall(expr *expression.Expression) bool {
|
func IsFunctionCall(expr *expression.Expression) bool {
|
||||||
return expr.Token.Kind == token.Operator && expr.Token.Text() == "λ"
|
return expr.Token.Kind == token.Call
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsVariableDefinition returns true if the expression is a variable definition.
|
// IsVariableDefinition returns true if the expression is a variable definition.
|
||||||
func IsVariableDefinition(expr *expression.Expression) bool {
|
func IsVariableDefinition(expr *expression.Expression) bool {
|
||||||
return expr.Token.Kind == token.Operator && expr.Token.Text() == ":="
|
return expr.Token.Kind == token.Define
|
||||||
}
|
}
|
||||||
|
|
||||||
// keywordHasBlock returns true if the keyword requires a block.
|
// keywordHasBlock returns true if the keyword requires a block.
|
||||||
func keywordHasBlock(word string) bool {
|
func keywordHasBlock(kind token.Kind) bool {
|
||||||
return word == keyword.If || word == keyword.Loop
|
return kind == token.If || kind == token.Loop
|
||||||
}
|
}
|
||||||
|
@ -1,8 +1,6 @@
|
|||||||
package ast
|
package ast
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"git.akyoto.dev/cli/q/src/build/expression"
|
"git.akyoto.dev/cli/q/src/build/expression"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -10,7 +8,3 @@ import (
|
|||||||
type Return struct {
|
type Return struct {
|
||||||
Value *expression.Expression
|
Value *expression.Expression
|
||||||
}
|
}
|
||||||
|
|
||||||
func (node *Return) String() string {
|
|
||||||
return fmt.Sprintf("(return %s)", node.Value)
|
|
||||||
}
|
|
||||||
|
@ -13,7 +13,7 @@ func (f *Function) Compare(comparison *expression.Expression) error {
|
|||||||
right := comparison.Children[1]
|
right := comparison.Children[1]
|
||||||
|
|
||||||
if left.IsLeaf() && left.Token.Kind == token.Identifier {
|
if left.IsLeaf() && left.Token.Kind == token.Identifier {
|
||||||
name := left.Token.Text()
|
name := left.Token.Text(f.File.Bytes)
|
||||||
variable := f.VariableByName(name)
|
variable := f.VariableByName(name)
|
||||||
|
|
||||||
if variable == nil {
|
if variable == nil {
|
||||||
|
@ -16,7 +16,7 @@ func (f *Function) CompileAssign(node *ast.Assign) error {
|
|||||||
right := node.Expression.Children[1]
|
right := node.Expression.Children[1]
|
||||||
|
|
||||||
if left.IsLeaf() {
|
if left.IsLeaf() {
|
||||||
name := left.Token.Text()
|
name := left.Token.Text(f.File.Bytes)
|
||||||
variable := f.VariableByName(name)
|
variable := f.VariableByName(name)
|
||||||
|
|
||||||
if variable == nil {
|
if variable == nil {
|
||||||
@ -27,8 +27,8 @@ func (f *Function) CompileAssign(node *ast.Assign) error {
|
|||||||
return f.Execute(operator, variable.Register, right)
|
return f.Execute(operator, variable.Register, right)
|
||||||
}
|
}
|
||||||
|
|
||||||
if left.Token.Kind == token.Operator && left.Token.Text() == "@" {
|
if left.Token.Kind == token.Array {
|
||||||
name := left.Children[0].Token.Text()
|
name := left.Children[0].Token.Text(f.File.Bytes)
|
||||||
variable := f.VariableByName(name)
|
variable := f.VariableByName(name)
|
||||||
|
|
||||||
if variable == nil {
|
if variable == nil {
|
||||||
@ -38,13 +38,13 @@ func (f *Function) CompileAssign(node *ast.Assign) error {
|
|||||||
defer f.useVariable(variable)
|
defer f.useVariable(variable)
|
||||||
|
|
||||||
index := left.Children[1]
|
index := left.Children[1]
|
||||||
offset, err := strconv.Atoi(index.Token.Text())
|
offset, err := strconv.Atoi(index.Token.Text(f.File.Bytes))
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
num, err := strconv.Atoi(right.Token.Text())
|
num, err := strconv.Atoi(right.Token.Text(f.File.Bytes))
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -15,9 +15,9 @@ func (f *Function) CompileCall(root *expression.Expression) error {
|
|||||||
funcName := ""
|
funcName := ""
|
||||||
|
|
||||||
if funcNameRoot.IsLeaf() {
|
if funcNameRoot.IsLeaf() {
|
||||||
funcName = funcNameRoot.Token.Text()
|
funcName = funcNameRoot.Token.Text(f.File.Bytes)
|
||||||
} else {
|
} else {
|
||||||
funcName = funcNameRoot.Children[0].Token.Text() + funcNameRoot.Token.Text() + funcNameRoot.Children[1].Token.Text()
|
funcName = funcNameRoot.Children[0].Token.Text(f.File.Bytes) + funcNameRoot.Token.Text(f.File.Bytes) + funcNameRoot.Children[1].Token.Text(f.File.Bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
isSyscall := funcName == "syscall"
|
isSyscall := funcName == "syscall"
|
||||||
|
@ -4,12 +4,13 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"git.akyoto.dev/cli/q/src/build/expression"
|
"git.akyoto.dev/cli/q/src/build/expression"
|
||||||
|
"git.akyoto.dev/cli/q/src/build/token"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CompileCondition inserts code to jump to the start label or end label depending on the truth of the condition.
|
// CompileCondition inserts code to jump to the start label or end label depending on the truth of the condition.
|
||||||
func (f *Function) CompileCondition(condition *expression.Expression, successLabel string, failLabel string) error {
|
func (f *Function) CompileCondition(condition *expression.Expression, successLabel string, failLabel string) error {
|
||||||
switch condition.Token.Text() {
|
switch condition.Token.Kind {
|
||||||
case "||":
|
case token.LogicalOr:
|
||||||
f.count.subBranch++
|
f.count.subBranch++
|
||||||
leftFailLabel := fmt.Sprintf("%s_false_%d", f.Name, f.count.subBranch)
|
leftFailLabel := fmt.Sprintf("%s_false_%d", f.Name, f.count.subBranch)
|
||||||
|
|
||||||
@ -21,22 +22,22 @@ func (f *Function) CompileCondition(condition *expression.Expression, successLab
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
f.JumpIfTrue(left.Token.Text(), successLabel)
|
f.JumpIfTrue(left.Token.Kind, successLabel)
|
||||||
|
|
||||||
// Right
|
// Right
|
||||||
f.AddLabel(leftFailLabel)
|
f.AddLabel(leftFailLabel)
|
||||||
right := condition.Children[1]
|
right := condition.Children[1]
|
||||||
err = f.CompileCondition(right, successLabel, failLabel)
|
err = f.CompileCondition(right, successLabel, failLabel)
|
||||||
|
|
||||||
if condition.Parent != nil && condition.Parent.Token.Text() == "||" && condition != condition.Parent.LastChild() {
|
if condition.Parent != nil && condition.Parent.Token.Kind == token.LogicalOr && condition != condition.Parent.LastChild() {
|
||||||
f.JumpIfTrue(right.Token.Text(), successLabel)
|
f.JumpIfTrue(right.Token.Kind, successLabel)
|
||||||
} else {
|
} else {
|
||||||
f.JumpIfFalse(right.Token.Text(), failLabel)
|
f.JumpIfFalse(right.Token.Kind, failLabel)
|
||||||
}
|
}
|
||||||
|
|
||||||
return err
|
return err
|
||||||
|
|
||||||
case "&&":
|
case token.LogicalAnd:
|
||||||
f.count.subBranch++
|
f.count.subBranch++
|
||||||
leftSuccessLabel := fmt.Sprintf("%s_true_%d", f.Name, f.count.subBranch)
|
leftSuccessLabel := fmt.Sprintf("%s_true_%d", f.Name, f.count.subBranch)
|
||||||
|
|
||||||
@ -48,17 +49,17 @@ func (f *Function) CompileCondition(condition *expression.Expression, successLab
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
f.JumpIfFalse(left.Token.Text(), failLabel)
|
f.JumpIfFalse(left.Token.Kind, failLabel)
|
||||||
|
|
||||||
// Right
|
// Right
|
||||||
f.AddLabel(leftSuccessLabel)
|
f.AddLabel(leftSuccessLabel)
|
||||||
right := condition.Children[1]
|
right := condition.Children[1]
|
||||||
err = f.CompileCondition(right, successLabel, failLabel)
|
err = f.CompileCondition(right, successLabel, failLabel)
|
||||||
|
|
||||||
if condition.Parent != nil && condition.Parent.Token.Text() == "||" && condition != condition.Parent.LastChild() {
|
if condition.Parent != nil && condition.Parent.Token.Kind == token.LogicalOr && condition != condition.Parent.LastChild() {
|
||||||
f.JumpIfTrue(right.Token.Text(), successLabel)
|
f.JumpIfTrue(right.Token.Kind, successLabel)
|
||||||
} else {
|
} else {
|
||||||
f.JumpIfFalse(right.Token.Text(), failLabel)
|
f.JumpIfFalse(right.Token.Kind, failLabel)
|
||||||
}
|
}
|
||||||
|
|
||||||
return err
|
return err
|
||||||
@ -67,7 +68,7 @@ func (f *Function) CompileCondition(condition *expression.Expression, successLab
|
|||||||
err := f.Compare(condition)
|
err := f.Compare(condition)
|
||||||
|
|
||||||
if condition.Parent == nil {
|
if condition.Parent == nil {
|
||||||
f.JumpIfFalse(condition.Token.Text(), failLabel)
|
f.JumpIfFalse(condition.Token.Kind, failLabel)
|
||||||
}
|
}
|
||||||
|
|
||||||
return err
|
return err
|
||||||
|
@ -8,13 +8,13 @@ import (
|
|||||||
|
|
||||||
// CompileDefinition compiles a variable definition.
|
// CompileDefinition compiles a variable definition.
|
||||||
func (f *Function) CompileDefinition(node *ast.Define) error {
|
func (f *Function) CompileDefinition(node *ast.Define) error {
|
||||||
name := node.Name.Text()
|
name := node.Name.Text(f.File.Bytes)
|
||||||
|
|
||||||
if f.identifierExists(name) {
|
if f.identifierExists(name) {
|
||||||
return errors.New(&errors.VariableAlreadyExists{Name: name}, f.File, node.Name.Position)
|
return errors.New(&errors.VariableAlreadyExists{Name: name}, f.File, node.Name.Position)
|
||||||
}
|
}
|
||||||
|
|
||||||
uses := token.Count(f.Body, token.Identifier, name) - 1
|
uses := token.Count(f.Body, f.File.Bytes, token.Identifier, name) - 1
|
||||||
|
|
||||||
if uses == 0 {
|
if uses == 0 {
|
||||||
return errors.New(&errors.UnusedVariable{Name: name}, f.File, node.Name.Position)
|
return errors.New(&errors.UnusedVariable{Name: name}, f.File, node.Name.Position)
|
||||||
|
@ -18,7 +18,7 @@ func (f *Function) CompileIf(branch *ast.If) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
f.AddLabel(success)
|
f.AddLabel(success)
|
||||||
f.PushScope(branch.Body)
|
f.PushScope(branch.Body, f.File.Bytes)
|
||||||
err = f.CompileAST(branch.Body)
|
err = f.CompileAST(branch.Body)
|
||||||
f.PopScope()
|
f.PopScope()
|
||||||
f.AddLabel(fail)
|
f.AddLabel(fail)
|
||||||
|
@ -12,7 +12,7 @@ func (f *Function) CompileLoop(loop *ast.Loop) error {
|
|||||||
f.count.loop++
|
f.count.loop++
|
||||||
label := fmt.Sprintf("%s_loop_%d", f.Name, f.count.loop)
|
label := fmt.Sprintf("%s_loop_%d", f.Name, f.count.loop)
|
||||||
f.AddLabel(label)
|
f.AddLabel(label)
|
||||||
scope := f.PushScope(loop.Body)
|
scope := f.PushScope(loop.Body, f.File.Bytes)
|
||||||
scope.InLoop = true
|
scope.InLoop = true
|
||||||
err := f.CompileAST(loop.Body)
|
err := f.CompileAST(loop.Body)
|
||||||
f.Jump(asm.JUMP, label)
|
f.Jump(asm.JUMP, label)
|
||||||
|
@ -7,8 +7,8 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// CompileTokens compiles a token list.
|
// CompileTokens compiles a token list.
|
||||||
func (f *Function) CompileTokens(tokens token.List) error {
|
func (f *Function) CompileTokens(tokens []token.Token) error {
|
||||||
body, err := ast.Parse(tokens)
|
body, err := ast.Parse(tokens, f.File.Bytes)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err.(*errors.Error).File = f.File
|
err.(*errors.Error).File = f.File
|
||||||
|
@ -12,7 +12,7 @@ import (
|
|||||||
func (f *Function) ExecuteLeaf(operation token.Token, register cpu.Register, operand token.Token) error {
|
func (f *Function) ExecuteLeaf(operation token.Token, register cpu.Register, operand token.Token) error {
|
||||||
switch operand.Kind {
|
switch operand.Kind {
|
||||||
case token.Identifier:
|
case token.Identifier:
|
||||||
name := operand.Text()
|
name := operand.Text(f.File.Bytes)
|
||||||
variable := f.VariableByName(name)
|
variable := f.VariableByName(name)
|
||||||
|
|
||||||
if variable == nil {
|
if variable == nil {
|
||||||
@ -23,7 +23,7 @@ func (f *Function) ExecuteLeaf(operation token.Token, register cpu.Register, ope
|
|||||||
return f.ExecuteRegisterRegister(operation, register, variable.Register)
|
return f.ExecuteRegisterRegister(operation, register, variable.Register)
|
||||||
|
|
||||||
case token.Number:
|
case token.Number:
|
||||||
value := operand.Text()
|
value := operand.Text(f.File.Bytes)
|
||||||
number, err := strconv.Atoi(value)
|
number, err := strconv.Atoi(value)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -33,7 +33,7 @@ func (f *Function) ExecuteLeaf(operation token.Token, register cpu.Register, ope
|
|||||||
return f.ExecuteRegisterNumber(operation, register, number)
|
return f.ExecuteRegisterNumber(operation, register, number)
|
||||||
|
|
||||||
case token.String:
|
case token.String:
|
||||||
if operation.Text() == "=" {
|
if operation.Kind == token.Assign {
|
||||||
return f.TokenToRegister(operand, register)
|
return f.TokenToRegister(operand, register)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -9,27 +9,27 @@ import (
|
|||||||
|
|
||||||
// ExecuteRegisterNumber performs an operation on a register and a number.
|
// ExecuteRegisterNumber performs an operation on a register and a number.
|
||||||
func (f *Function) ExecuteRegisterNumber(operation token.Token, register cpu.Register, number int) error {
|
func (f *Function) ExecuteRegisterNumber(operation token.Token, register cpu.Register, number int) error {
|
||||||
switch operation.Text() {
|
switch operation.Kind {
|
||||||
case "+", "+=":
|
case token.Add, token.AddAssign:
|
||||||
f.RegisterNumber(asm.ADD, register, number)
|
f.RegisterNumber(asm.ADD, register, number)
|
||||||
|
|
||||||
case "-", "-=":
|
case token.Sub, token.SubAssign:
|
||||||
f.RegisterNumber(asm.SUB, register, number)
|
f.RegisterNumber(asm.SUB, register, number)
|
||||||
|
|
||||||
case "*", "*=":
|
case token.Mul, token.MulAssign:
|
||||||
f.RegisterNumber(asm.MUL, register, number)
|
f.RegisterNumber(asm.MUL, register, number)
|
||||||
|
|
||||||
case "/", "/=":
|
case token.Div, token.DivAssign:
|
||||||
f.RegisterNumber(asm.DIV, register, number)
|
f.RegisterNumber(asm.DIV, register, number)
|
||||||
|
|
||||||
case "==", "!=", "<", "<=", ">", ">=":
|
case token.Equal, token.NotEqual, token.Less, token.LessEqual, token.Greater, token.GreaterEqual:
|
||||||
f.RegisterNumber(asm.COMPARE, register, number)
|
f.RegisterNumber(asm.COMPARE, register, number)
|
||||||
|
|
||||||
case "=":
|
case token.Assign:
|
||||||
f.RegisterNumber(asm.MOVE, register, number)
|
f.RegisterNumber(asm.MOVE, register, number)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return errors.New(&errors.InvalidOperator{Operator: operation.Text()}, f.File, operation.Position)
|
return errors.New(&errors.InvalidOperator{Operator: operation.Text(f.File.Bytes)}, f.File, operation.Position)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
@ -9,27 +9,27 @@ import (
|
|||||||
|
|
||||||
// ExecuteRegisterRegister performs an operation on two registers.
|
// ExecuteRegisterRegister performs an operation on two registers.
|
||||||
func (f *Function) ExecuteRegisterRegister(operation token.Token, destination cpu.Register, source cpu.Register) error {
|
func (f *Function) ExecuteRegisterRegister(operation token.Token, destination cpu.Register, source cpu.Register) error {
|
||||||
switch operation.Text() {
|
switch operation.Kind {
|
||||||
case "+", "+=":
|
case token.Add, token.AddAssign:
|
||||||
f.RegisterRegister(asm.ADD, destination, source)
|
f.RegisterRegister(asm.ADD, destination, source)
|
||||||
|
|
||||||
case "-", "-=":
|
case token.Sub, token.SubAssign:
|
||||||
f.RegisterRegister(asm.SUB, destination, source)
|
f.RegisterRegister(asm.SUB, destination, source)
|
||||||
|
|
||||||
case "*", "*=":
|
case token.Mul, token.MulAssign:
|
||||||
f.RegisterRegister(asm.MUL, destination, source)
|
f.RegisterRegister(asm.MUL, destination, source)
|
||||||
|
|
||||||
case "/", "/=":
|
case token.Div, token.DivAssign:
|
||||||
f.RegisterRegister(asm.DIV, destination, source)
|
f.RegisterRegister(asm.DIV, destination, source)
|
||||||
|
|
||||||
case "==", "!=", "<", "<=", ">", ">=":
|
case token.Equal, token.NotEqual, token.Less, token.LessEqual, token.Greater, token.GreaterEqual:
|
||||||
f.RegisterRegister(asm.COMPARE, destination, source)
|
f.RegisterRegister(asm.COMPARE, destination, source)
|
||||||
|
|
||||||
case "=":
|
case token.Assign:
|
||||||
f.RegisterRegister(asm.MOVE, destination, source)
|
f.RegisterRegister(asm.MOVE, destination, source)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return errors.New(&errors.InvalidOperator{Operator: operation.Text()}, f.File, operation.Position)
|
return errors.New(&errors.InvalidOperator{Operator: operation.Text(f.File.Bytes)}, f.File, operation.Position)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
@ -13,7 +13,7 @@ type Function struct {
|
|||||||
scope.Stack
|
scope.Stack
|
||||||
Name string
|
Name string
|
||||||
File *fs.File
|
File *fs.File
|
||||||
Body token.List
|
Body []token.Token
|
||||||
Assembler asm.Assembler
|
Assembler asm.Assembler
|
||||||
Functions map[string]*Function
|
Functions map[string]*Function
|
||||||
Err error
|
Err error
|
||||||
|
@ -2,22 +2,23 @@ package core
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"git.akyoto.dev/cli/q/src/build/asm"
|
"git.akyoto.dev/cli/q/src/build/asm"
|
||||||
|
"git.akyoto.dev/cli/q/src/build/token"
|
||||||
)
|
)
|
||||||
|
|
||||||
// JumpIfFalse jumps to the label if the previous comparison was false.
|
// JumpIfFalse jumps to the label if the previous comparison was false.
|
||||||
func (f *Function) JumpIfFalse(operator string, label string) {
|
func (f *Function) JumpIfFalse(operator token.Kind, label string) {
|
||||||
switch operator {
|
switch operator {
|
||||||
case "==":
|
case token.Equal:
|
||||||
f.Jump(asm.JNE, label)
|
f.Jump(asm.JNE, label)
|
||||||
case "!=":
|
case token.NotEqual:
|
||||||
f.Jump(asm.JE, label)
|
f.Jump(asm.JE, label)
|
||||||
case ">":
|
case token.Greater:
|
||||||
f.Jump(asm.JLE, label)
|
f.Jump(asm.JLE, label)
|
||||||
case "<":
|
case token.Less:
|
||||||
f.Jump(asm.JGE, label)
|
f.Jump(asm.JGE, label)
|
||||||
case ">=":
|
case token.GreaterEqual:
|
||||||
f.Jump(asm.JL, label)
|
f.Jump(asm.JL, label)
|
||||||
case "<=":
|
case token.LessEqual:
|
||||||
f.Jump(asm.JG, label)
|
f.Jump(asm.JG, label)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -2,22 +2,23 @@ package core
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"git.akyoto.dev/cli/q/src/build/asm"
|
"git.akyoto.dev/cli/q/src/build/asm"
|
||||||
|
"git.akyoto.dev/cli/q/src/build/token"
|
||||||
)
|
)
|
||||||
|
|
||||||
// JumpIfTrue jumps to the label if the previous comparison was true.
|
// JumpIfTrue jumps to the label if the previous comparison was true.
|
||||||
func (f *Function) JumpIfTrue(operator string, label string) {
|
func (f *Function) JumpIfTrue(operator token.Kind, label string) {
|
||||||
switch operator {
|
switch operator {
|
||||||
case "==":
|
case token.Equal:
|
||||||
f.Jump(asm.JE, label)
|
f.Jump(asm.JE, label)
|
||||||
case "!=":
|
case token.NotEqual:
|
||||||
f.Jump(asm.JNE, label)
|
f.Jump(asm.JNE, label)
|
||||||
case ">":
|
case token.Greater:
|
||||||
f.Jump(asm.JG, label)
|
f.Jump(asm.JG, label)
|
||||||
case "<":
|
case token.Less:
|
||||||
f.Jump(asm.JL, label)
|
f.Jump(asm.JL, label)
|
||||||
case ">=":
|
case token.GreaterEqual:
|
||||||
f.Jump(asm.JGE, label)
|
f.Jump(asm.JGE, label)
|
||||||
case "<=":
|
case token.LessEqual:
|
||||||
f.Jump(asm.JLE, label)
|
f.Jump(asm.JLE, label)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -10,7 +10,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// NewFunction creates a new function.
|
// NewFunction creates a new function.
|
||||||
func NewFunction(name string, file *fs.File, body token.List) *Function {
|
func NewFunction(name string, file *fs.File, body []token.Token) *Function {
|
||||||
return &Function{
|
return &Function{
|
||||||
Name: name,
|
Name: name,
|
||||||
File: file,
|
File: file,
|
||||||
|
@ -15,7 +15,7 @@ import (
|
|||||||
func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error {
|
func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error {
|
||||||
switch t.Kind {
|
switch t.Kind {
|
||||||
case token.Identifier:
|
case token.Identifier:
|
||||||
name := t.Text()
|
name := t.Text(f.File.Bytes)
|
||||||
variable := f.VariableByName(name)
|
variable := f.VariableByName(name)
|
||||||
|
|
||||||
if variable == nil {
|
if variable == nil {
|
||||||
@ -27,7 +27,7 @@ func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error {
|
|||||||
return nil
|
return nil
|
||||||
|
|
||||||
case token.Number:
|
case token.Number:
|
||||||
value := t.Text()
|
value := t.Text(f.File.Bytes)
|
||||||
n, err := strconv.Atoi(value)
|
n, err := strconv.Atoi(value)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -40,7 +40,7 @@ func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error {
|
|||||||
case token.String:
|
case token.String:
|
||||||
f.count.data++
|
f.count.data++
|
||||||
label := fmt.Sprintf("%s_data_%d", f.Name, f.count.data)
|
label := fmt.Sprintf("%s_data_%d", f.Name, f.count.data)
|
||||||
value := t.Bytes[1 : len(t.Bytes)-1]
|
value := t.Bytes(f.File.Bytes)[1 : t.Length-1]
|
||||||
f.Assembler.SetData(label, value)
|
f.Assembler.SetData(label, value)
|
||||||
f.RegisterLabel(asm.MOVE, register, label)
|
f.RegisterLabel(asm.MOVE, register, label)
|
||||||
return nil
|
return nil
|
||||||
|
@ -37,11 +37,11 @@ func (expr *Expression) AddChild(child *Expression) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Count counts how often the given token appears in the expression.
|
// Count counts how often the given token appears in the expression.
|
||||||
func (expr *Expression) Count(kind token.Kind, name string) int {
|
func (expr *Expression) Count(buffer []byte, kind token.Kind, name string) int {
|
||||||
count := 0
|
count := 0
|
||||||
|
|
||||||
expr.EachLeaf(func(leaf *Expression) error {
|
expr.EachLeaf(func(leaf *Expression) error {
|
||||||
if leaf.Token.Kind == kind && leaf.Token.Text() == name {
|
if leaf.Token.Kind == kind && leaf.Token.Text(buffer) == name {
|
||||||
count++
|
count++
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -112,25 +112,33 @@ func (expr *Expression) LastChild() *Expression {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// String generates a textual representation of the expression.
|
// String generates a textual representation of the expression.
|
||||||
func (expr *Expression) String() string {
|
func (expr *Expression) String(data []byte) string {
|
||||||
builder := strings.Builder{}
|
builder := strings.Builder{}
|
||||||
expr.write(&builder)
|
expr.write(&builder, data)
|
||||||
return builder.String()
|
return builder.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
// write generates a textual representation of the expression.
|
// write generates a textual representation of the expression.
|
||||||
func (expr *Expression) write(builder *strings.Builder) {
|
func (expr *Expression) write(builder *strings.Builder, data []byte) {
|
||||||
if expr.IsLeaf() {
|
if expr.IsLeaf() {
|
||||||
builder.WriteString(expr.Token.Text())
|
builder.WriteString(expr.Token.Text(data))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
builder.WriteByte('(')
|
builder.WriteByte('(')
|
||||||
builder.WriteString(expr.Token.Text())
|
|
||||||
|
switch expr.Token.Kind {
|
||||||
|
case token.Call:
|
||||||
|
builder.WriteString("λ")
|
||||||
|
case token.Array:
|
||||||
|
builder.WriteString("@")
|
||||||
|
default:
|
||||||
|
builder.WriteString(expr.Token.Text(data))
|
||||||
|
}
|
||||||
|
|
||||||
for _, child := range expr.Children {
|
for _, child := range expr.Children {
|
||||||
builder.WriteByte(' ')
|
builder.WriteByte(' ')
|
||||||
child.write(builder)
|
child.write(builder, data)
|
||||||
}
|
}
|
||||||
|
|
||||||
builder.WriteByte(')')
|
builder.WriteByte(')')
|
||||||
|
@ -95,7 +95,7 @@ func TestParse(t *testing.T) {
|
|||||||
defer expr.Reset()
|
defer expr.Reset()
|
||||||
|
|
||||||
assert.NotNil(t, expr)
|
assert.NotNil(t, expr)
|
||||||
assert.Equal(t, expr.String(), test.Result)
|
assert.Equal(t, expr.String(src), test.Result)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -104,11 +104,11 @@ func TestCount(t *testing.T) {
|
|||||||
src := []byte("(a+b-c*d)+(a*b-c+d)")
|
src := []byte("(a+b-c*d)+(a*b-c+d)")
|
||||||
tokens := token.Tokenize(src)
|
tokens := token.Tokenize(src)
|
||||||
expr := expression.Parse(tokens)
|
expr := expression.Parse(tokens)
|
||||||
assert.Equal(t, expr.Count(token.Identifier, "a"), 2)
|
assert.Equal(t, expr.Count(src, token.Identifier, "a"), 2)
|
||||||
assert.Equal(t, expr.Count(token.Identifier, "b"), 2)
|
assert.Equal(t, expr.Count(src, token.Identifier, "b"), 2)
|
||||||
assert.Equal(t, expr.Count(token.Identifier, "c"), 2)
|
assert.Equal(t, expr.Count(src, token.Identifier, "c"), 2)
|
||||||
assert.Equal(t, expr.Count(token.Identifier, "d"), 2)
|
assert.Equal(t, expr.Count(src, token.Identifier, "d"), 2)
|
||||||
assert.Equal(t, expr.Count(token.Identifier, "e"), 0)
|
assert.Equal(t, expr.Count(src, token.Identifier, "e"), 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEachLeaf(t *testing.T) {
|
func TestEachLeaf(t *testing.T) {
|
||||||
@ -118,7 +118,7 @@ func TestEachLeaf(t *testing.T) {
|
|||||||
leaves := []string{}
|
leaves := []string{}
|
||||||
|
|
||||||
err := expr.EachLeaf(func(leaf *expression.Expression) error {
|
err := expr.EachLeaf(func(leaf *expression.Expression) error {
|
||||||
leaves = append(leaves, leaf.Token.Text())
|
leaves = append(leaves, leaf.Token.Text(src))
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -140,7 +140,7 @@ func TestEachParameter(t *testing.T) {
|
|||||||
|
|
||||||
err := expression.EachParameter(tokens, func(parameter token.List) error {
|
err := expression.EachParameter(tokens, func(parameter token.List) error {
|
||||||
expr := expression.Parse(parameter)
|
expr := expression.Parse(parameter)
|
||||||
parameters = append(parameters, expr.String())
|
parameters = append(parameters, expr.String(src))
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -178,17 +178,3 @@ func TestNilGroup(t *testing.T) {
|
|||||||
expr := expression.Parse(tokens)
|
expr := expression.Parse(tokens)
|
||||||
assert.Nil(t, expr)
|
assert.Nil(t, expr)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestInvalidOperator(t *testing.T) {
|
|
||||||
src := []byte("a +++ 2")
|
|
||||||
tokens := token.Tokenize(src)
|
|
||||||
expr := expression.Parse(tokens)
|
|
||||||
assert.Equal(t, expr.String(), "(+++ a 2)")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestInvalidOperatorCall(t *testing.T) {
|
|
||||||
src := []byte("+++()")
|
|
||||||
tokens := token.Tokenize(src)
|
|
||||||
expr := expression.Parse(tokens)
|
|
||||||
assert.NotNil(t, expr)
|
|
||||||
}
|
|
||||||
|
@ -15,39 +15,39 @@ type Operator struct {
|
|||||||
|
|
||||||
// Operators defines the operators used in the language.
|
// Operators defines the operators used in the language.
|
||||||
// The number corresponds to the operator priority and can not be zero.
|
// The number corresponds to the operator priority and can not be zero.
|
||||||
var Operators = map[string]*Operator{
|
var Operators = map[token.Kind]*Operator{
|
||||||
".": {".", 13, 2},
|
token.Period: {".", 13, 2},
|
||||||
"λ": {"λ", 12, 1},
|
token.Call: {"λ", 12, 1},
|
||||||
"@": {"@", 12, 2},
|
token.Array: {"@", 12, 2},
|
||||||
"!": {"!", 11, 1},
|
token.Not: {"!", 11, 1},
|
||||||
"*": {"*", 10, 2},
|
token.Mul: {"*", 10, 2},
|
||||||
"/": {"/", 10, 2},
|
token.Div: {"/", 10, 2},
|
||||||
"%": {"%", 10, 2},
|
token.Mod: {"%", 10, 2},
|
||||||
"+": {"+", 9, 2},
|
token.Add: {"+", 9, 2},
|
||||||
"-": {"-", 9, 2},
|
token.Sub: {"-", 9, 2},
|
||||||
">>": {">>", 8, 2},
|
token.Shr: {">>", 8, 2},
|
||||||
"<<": {"<<", 8, 2},
|
token.Shl: {"<<", 8, 2},
|
||||||
"&": {"&", 7, 2},
|
token.And: {"&", 7, 2},
|
||||||
"^": {"^", 6, 2},
|
token.Xor: {"^", 6, 2},
|
||||||
"|": {"|", 5, 2},
|
token.Or: {"|", 5, 2},
|
||||||
|
|
||||||
">": {">", 4, 2},
|
token.Greater: {">", 4, 2},
|
||||||
"<": {"<", 4, 2},
|
token.Less: {"<", 4, 2},
|
||||||
">=": {">=", 4, 2},
|
token.GreaterEqual: {">=", 4, 2},
|
||||||
"<=": {"<=", 4, 2},
|
token.LessEqual: {"<=", 4, 2},
|
||||||
"==": {"==", 3, 2},
|
token.Equal: {"==", 3, 2},
|
||||||
"!=": {"!=", 3, 2},
|
token.NotEqual: {"!=", 3, 2},
|
||||||
"&&": {"&&", 2, 2},
|
token.LogicalAnd: {"&&", 2, 2},
|
||||||
"||": {"||", 1, 2},
|
token.LogicalOr: {"||", 1, 2},
|
||||||
|
|
||||||
"=": {"=", math.MinInt8, 2},
|
token.Assign: {"=", math.MinInt8, 2},
|
||||||
":=": {":=", math.MinInt8, 2},
|
token.Define: {":=", math.MinInt8, 2},
|
||||||
"+=": {"+=", math.MinInt8, 2},
|
token.AddAssign: {"+=", math.MinInt8, 2},
|
||||||
"-=": {"-=", math.MinInt8, 2},
|
token.SubAssign: {"-=", math.MinInt8, 2},
|
||||||
"*=": {"*=", math.MinInt8, 2},
|
token.MulAssign: {"*=", math.MinInt8, 2},
|
||||||
"/=": {"/=", math.MinInt8, 2},
|
token.DivAssign: {"/=", math.MinInt8, 2},
|
||||||
">>=": {">>=", math.MinInt8, 2},
|
token.ShrAssign: {">>=", math.MinInt8, 2},
|
||||||
"<<=": {"<<=", math.MinInt8, 2},
|
token.ShlAssign: {"<<=", math.MinInt8, 2},
|
||||||
}
|
}
|
||||||
|
|
||||||
func isComplete(expr *Expression) bool {
|
func isComplete(expr *Expression) bool {
|
||||||
@ -59,14 +59,14 @@ func isComplete(expr *Expression) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
if expr.Token.Kind == token.Operator && len(expr.Children) == numOperands(expr.Token.Text()) {
|
if expr.Token.IsOperator() && len(expr.Children) == numOperands(expr.Token.Kind) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func numOperands(symbol string) int {
|
func numOperands(symbol token.Kind) int {
|
||||||
operator, exists := Operators[symbol]
|
operator, exists := Operators[symbol]
|
||||||
|
|
||||||
if !exists {
|
if !exists {
|
||||||
@ -76,7 +76,7 @@ func numOperands(symbol string) int {
|
|||||||
return operator.Operands
|
return operator.Operands
|
||||||
}
|
}
|
||||||
|
|
||||||
func precedence(symbol string) int8 {
|
func precedence(symbol token.Kind) int8 {
|
||||||
operator, exists := Operators[symbol]
|
operator, exists := Operators[symbol]
|
||||||
|
|
||||||
if !exists {
|
if !exists {
|
||||||
|
@ -6,13 +6,8 @@ import (
|
|||||||
"git.akyoto.dev/cli/q/src/build/token"
|
"git.akyoto.dev/cli/q/src/build/token"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
|
||||||
call = []byte("λ")
|
|
||||||
array = []byte("@")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Parse generates an expression tree from tokens.
|
// Parse generates an expression tree from tokens.
|
||||||
func Parse(tokens token.List) *Expression {
|
func Parse(tokens []token.Token) *Expression {
|
||||||
var (
|
var (
|
||||||
cursor *Expression
|
cursor *Expression
|
||||||
root *Expression
|
root *Expression
|
||||||
@ -43,20 +38,18 @@ func Parse(tokens token.List) *Expression {
|
|||||||
parameters := NewList(tokens[groupPosition:i])
|
parameters := NewList(tokens[groupPosition:i])
|
||||||
|
|
||||||
node := New()
|
node := New()
|
||||||
node.Token.Kind = token.Operator
|
|
||||||
node.Token.Position = tokens[groupPosition].Position
|
node.Token.Position = tokens[groupPosition].Position
|
||||||
|
|
||||||
switch t.Kind {
|
switch t.Kind {
|
||||||
case token.GroupEnd:
|
case token.GroupEnd:
|
||||||
node.Token.Bytes = call
|
node.Token.Kind = token.Call
|
||||||
node.Precedence = precedence("λ")
|
|
||||||
|
|
||||||
case token.ArrayEnd:
|
case token.ArrayEnd:
|
||||||
node.Token.Bytes = array
|
node.Token.Kind = token.Array
|
||||||
node.Precedence = precedence("@")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if cursor.Token.Kind == token.Operator && node.Precedence > cursor.Precedence {
|
node.Precedence = precedence(node.Token.Kind)
|
||||||
|
|
||||||
|
if cursor.Token.IsOperator() && node.Precedence > cursor.Precedence {
|
||||||
cursor.LastChild().Replace(node)
|
cursor.LastChild().Replace(node)
|
||||||
} else {
|
} else {
|
||||||
if cursor == root {
|
if cursor == root {
|
||||||
@ -108,18 +101,18 @@ func Parse(tokens token.List) *Expression {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if t.Kind == token.Operator {
|
if t.IsOperator() {
|
||||||
if cursor == nil {
|
if cursor == nil {
|
||||||
cursor = NewLeaf(t)
|
cursor = NewLeaf(t)
|
||||||
cursor.Precedence = precedence(t.Text())
|
cursor.Precedence = precedence(t.Kind)
|
||||||
root = cursor
|
root = cursor
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
node := NewLeaf(t)
|
node := NewLeaf(t)
|
||||||
node.Precedence = precedence(t.Text())
|
node.Precedence = precedence(t.Kind)
|
||||||
|
|
||||||
if cursor.Token.Kind == token.Operator {
|
if cursor.Token.IsOperator() {
|
||||||
oldPrecedence := cursor.Precedence
|
oldPrecedence := cursor.Precedence
|
||||||
newPrecedence := node.Precedence
|
newPrecedence := node.Precedence
|
||||||
|
|
||||||
|
@ -5,5 +5,6 @@ import "git.akyoto.dev/cli/q/src/build/token"
|
|||||||
// File represents a single source file.
|
// File represents a single source file.
|
||||||
type File struct {
|
type File struct {
|
||||||
Path string
|
Path string
|
||||||
|
Bytes []byte
|
||||||
Tokens token.List
|
Tokens token.List
|
||||||
}
|
}
|
||||||
|
@ -1,16 +0,0 @@
|
|||||||
package keyword
|
|
||||||
|
|
||||||
const (
|
|
||||||
If = "if"
|
|
||||||
Import = "import"
|
|
||||||
Loop = "loop"
|
|
||||||
Return = "return"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Map is a map of all keywords used in the language.
|
|
||||||
var Map = map[string][]byte{
|
|
||||||
If: []byte(If),
|
|
||||||
Import: []byte(Import),
|
|
||||||
Loop: []byte(Loop),
|
|
||||||
Return: []byte(Return),
|
|
||||||
}
|
|
@ -11,7 +11,6 @@ import (
|
|||||||
"git.akyoto.dev/cli/q/src/build/errors"
|
"git.akyoto.dev/cli/q/src/build/errors"
|
||||||
"git.akyoto.dev/cli/q/src/build/expression"
|
"git.akyoto.dev/cli/q/src/build/expression"
|
||||||
"git.akyoto.dev/cli/q/src/build/fs"
|
"git.akyoto.dev/cli/q/src/build/fs"
|
||||||
"git.akyoto.dev/cli/q/src/build/keyword"
|
|
||||||
"git.akyoto.dev/cli/q/src/build/scope"
|
"git.akyoto.dev/cli/q/src/build/scope"
|
||||||
"git.akyoto.dev/cli/q/src/build/token"
|
"git.akyoto.dev/cli/q/src/build/token"
|
||||||
)
|
)
|
||||||
@ -27,8 +26,9 @@ func (s *Scanner) scanFile(path string, pkg string) error {
|
|||||||
tokens := token.Tokenize(contents)
|
tokens := token.Tokenize(contents)
|
||||||
|
|
||||||
file := &fs.File{
|
file := &fs.File{
|
||||||
Tokens: tokens,
|
|
||||||
Path: path,
|
Path: path,
|
||||||
|
Bytes: contents,
|
||||||
|
Tokens: tokens,
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -42,14 +42,14 @@ func (s *Scanner) scanFile(path string, pkg string) error {
|
|||||||
)
|
)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
for i < len(tokens) && tokens[i].Kind == token.Keyword && tokens[i].Text() == keyword.Import {
|
for i < len(tokens) && tokens[i].Kind == token.Import {
|
||||||
i++
|
i++
|
||||||
|
|
||||||
if tokens[i].Kind != token.Identifier {
|
if tokens[i].Kind != token.Identifier {
|
||||||
panic("expected package name")
|
panic("expected package name")
|
||||||
}
|
}
|
||||||
|
|
||||||
packageName := tokens[i].Text()
|
packageName := tokens[i].Text(contents)
|
||||||
s.queueDirectory(filepath.Join(config.Library, packageName), packageName)
|
s.queueDirectory(filepath.Join(config.Library, packageName), packageName)
|
||||||
|
|
||||||
i++
|
i++
|
||||||
@ -75,7 +75,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if tokens[i].Kind == token.Invalid {
|
if tokens[i].Kind == token.Invalid {
|
||||||
return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
|
return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text(contents)}, file, tokens[i].Position)
|
||||||
}
|
}
|
||||||
|
|
||||||
if tokens[i].Kind == token.EOF {
|
if tokens[i].Kind == token.EOF {
|
||||||
@ -116,7 +116,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if tokens[i].Kind == token.Invalid {
|
if tokens[i].Kind == token.Invalid {
|
||||||
return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
|
return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text(contents)}, file, tokens[i].Position)
|
||||||
}
|
}
|
||||||
|
|
||||||
if tokens[i].Kind == token.EOF {
|
if tokens[i].Kind == token.EOF {
|
||||||
@ -168,7 +168,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if tokens[i].Kind == token.Invalid {
|
if tokens[i].Kind == token.Invalid {
|
||||||
return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
|
return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text(contents)}, file, tokens[i].Position)
|
||||||
}
|
}
|
||||||
|
|
||||||
if tokens[i].Kind == token.EOF {
|
if tokens[i].Kind == token.EOF {
|
||||||
@ -191,7 +191,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
|
|||||||
return errors.New(errors.ExpectedFunctionDefinition, file, tokens[i].Position)
|
return errors.New(errors.ExpectedFunctionDefinition, file, tokens[i].Position)
|
||||||
}
|
}
|
||||||
|
|
||||||
name := tokens[nameStart].Text()
|
name := tokens[nameStart].Text(contents)
|
||||||
body := tokens[bodyStart:i]
|
body := tokens[bodyStart:i]
|
||||||
|
|
||||||
if pkg != "" {
|
if pkg != "" {
|
||||||
@ -207,9 +207,9 @@ func (s *Scanner) scanFile(path string, pkg string) error {
|
|||||||
return errors.New(errors.NotImplemented, file, tokens[0].Position)
|
return errors.New(errors.NotImplemented, file, tokens[0].Position)
|
||||||
}
|
}
|
||||||
|
|
||||||
name := tokens[0].Text()
|
name := tokens[0].Text(contents)
|
||||||
register := x64.CallRegisters[count]
|
register := x64.CallRegisters[count]
|
||||||
uses := token.Count(function.Body, token.Identifier, name)
|
uses := token.Count(function.Body, contents, token.Identifier, name)
|
||||||
|
|
||||||
if uses == 0 {
|
if uses == 0 {
|
||||||
return errors.New(&errors.UnusedVariable{Name: name}, file, tokens[0].Position)
|
return errors.New(&errors.UnusedVariable{Name: name}, file, tokens[0].Position)
|
||||||
|
@ -20,7 +20,7 @@ func (stack *Stack) PopScope() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// PushScope pushes a new scope to the top of the stack.
|
// PushScope pushes a new scope to the top of the stack.
|
||||||
func (stack *Stack) PushScope(body ast.AST) *Scope {
|
func (stack *Stack) PushScope(body ast.AST, buffer []byte) *Scope {
|
||||||
s := &Scope{}
|
s := &Scope{}
|
||||||
|
|
||||||
if len(stack.Scopes) > 0 {
|
if len(stack.Scopes) > 0 {
|
||||||
@ -30,7 +30,7 @@ func (stack *Stack) PushScope(body ast.AST) *Scope {
|
|||||||
s.InLoop = lastScope.InLoop
|
s.InLoop = lastScope.InLoop
|
||||||
|
|
||||||
for k, v := range lastScope.Variables {
|
for k, v := range lastScope.Variables {
|
||||||
count := ast.Count(body, token.Identifier, v.Name)
|
count := ast.Count(body, buffer, token.Identifier, v.Name)
|
||||||
|
|
||||||
if count == 0 {
|
if count == 0 {
|
||||||
continue
|
continue
|
||||||
|
@ -1,11 +1,11 @@
|
|||||||
package token
|
package token
|
||||||
|
|
||||||
// Count counts how often the given token appears in the token list.
|
// Count counts how often the given token appears in the token list.
|
||||||
func Count(tokens List, kind Kind, name string) int {
|
func Count(tokens []Token, buffer []byte, kind Kind, name string) int {
|
||||||
count := 0
|
count := 0
|
||||||
|
|
||||||
for _, t := range tokens {
|
for _, t := range tokens {
|
||||||
if t.Kind == Identifier && t.Text() == name {
|
if t.Kind == Identifier && t.Text(buffer) == name {
|
||||||
count++
|
count++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -8,9 +8,10 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestCount(t *testing.T) {
|
func TestCount(t *testing.T) {
|
||||||
tokens := token.Tokenize([]byte(`a b b c c c`))
|
buffer := []byte(`a b b c c c`)
|
||||||
assert.Equal(t, token.Count(tokens, token.Identifier, "a"), 1)
|
tokens := token.Tokenize(buffer)
|
||||||
assert.Equal(t, token.Count(tokens, token.Identifier, "b"), 2)
|
assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "a"), 1)
|
||||||
assert.Equal(t, token.Count(tokens, token.Identifier, "c"), 3)
|
assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "b"), 2)
|
||||||
assert.Equal(t, token.Count(tokens, token.Identifier, "d"), 0)
|
assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "c"), 3)
|
||||||
|
assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "d"), 0)
|
||||||
}
|
}
|
||||||
|
9
src/build/token/Keywords.go
Normal file
9
src/build/token/Keywords.go
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
package token
|
||||||
|
|
||||||
|
// Keywords is a map of all keywords used in the language.
|
||||||
|
var Keywords = map[string]Kind{
|
||||||
|
"if": If,
|
||||||
|
"import": Import,
|
||||||
|
"loop": Loop,
|
||||||
|
"return": Return,
|
||||||
|
}
|
@ -4,73 +4,62 @@ package token
|
|||||||
type Kind uint8
|
type Kind uint8
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// Invalid represents an invalid token.
|
Invalid Kind = iota // Invalid is an invalid token.
|
||||||
Invalid Kind = iota
|
EOF // EOF is the end of file.
|
||||||
|
NewLine // NewLine is the newline character.
|
||||||
// EOF represents the end of file.
|
Identifier // Identifier is a series of characters used to identify a variable or function.
|
||||||
EOF
|
Number // Number is a series of numerical characters.
|
||||||
|
String // String is an uninterpreted series of characters in the source code.
|
||||||
// NewLine represents the newline character.
|
Comment // Comment is a comment.
|
||||||
NewLine
|
Separator // ,
|
||||||
|
GroupStart // (
|
||||||
// Identifier represents a series of characters used to identify a variable or function.
|
GroupEnd // )
|
||||||
Identifier
|
BlockStart // {
|
||||||
|
BlockEnd // }
|
||||||
// Keyword represents a language keyword.
|
ArrayStart // [
|
||||||
Keyword
|
ArrayEnd // ]
|
||||||
|
_keywords // <keywords>
|
||||||
// String represents an uninterpreted series of characters in the source code.
|
If // if
|
||||||
String
|
Import // import
|
||||||
|
Loop // loop
|
||||||
// Number represents a series of numerical characters.
|
Return // return
|
||||||
Number
|
_keywordsEnd // </keywords>
|
||||||
|
_operators // <operators>
|
||||||
// Operator represents a mathematical operator.
|
Add // +
|
||||||
Operator
|
Sub // -
|
||||||
|
Mul // *
|
||||||
// Separator represents a comma.
|
Div // /
|
||||||
Separator
|
Mod // %
|
||||||
|
And // &
|
||||||
// Comment represents a comment.
|
Or // |
|
||||||
Comment
|
Xor // ^
|
||||||
|
Shl // <<
|
||||||
// GroupStart represents '('.
|
Shr // >>
|
||||||
GroupStart
|
LogicalAnd // &&
|
||||||
|
LogicalOr // ||
|
||||||
// GroupEnd represents ')'.
|
Equal // ==
|
||||||
GroupEnd
|
Less // <
|
||||||
|
Greater // >
|
||||||
// BlockStart represents '{'.
|
Not // !
|
||||||
BlockStart
|
NotEqual // !=
|
||||||
|
LessEqual // <=
|
||||||
// BlockEnd represents '}'.
|
GreaterEqual // >=
|
||||||
BlockEnd
|
Define // :=
|
||||||
|
Period // .
|
||||||
// ArrayStart represents '['.
|
Call // x()
|
||||||
ArrayStart
|
Array // [x]
|
||||||
|
_assignments // <assignments>
|
||||||
// ArrayEnd represents ']'.
|
Assign // =
|
||||||
ArrayEnd
|
AddAssign // +=
|
||||||
|
SubAssign // -=
|
||||||
|
MulAssign // *=
|
||||||
|
DivAssign // /=
|
||||||
|
ModAssign // %=
|
||||||
|
AndAssign // &=
|
||||||
|
OrAssign // |=
|
||||||
|
XorAssign // ^=
|
||||||
|
ShlAssign // <<=
|
||||||
|
ShrAssign // >>=
|
||||||
|
_assignmentsEnd // </assignments>
|
||||||
|
_operatorsEnd // </operators>
|
||||||
)
|
)
|
||||||
|
|
||||||
// String returns the text representation.
|
|
||||||
func (kind Kind) String() string {
|
|
||||||
return [...]string{
|
|
||||||
"Invalid",
|
|
||||||
"EOF",
|
|
||||||
"NewLine",
|
|
||||||
"Identifier",
|
|
||||||
"Keyword",
|
|
||||||
"String",
|
|
||||||
"Number",
|
|
||||||
"Operator",
|
|
||||||
"Separator",
|
|
||||||
"Comment",
|
|
||||||
"GroupStart",
|
|
||||||
"GroupEnd",
|
|
||||||
"BlockStart",
|
|
||||||
"BlockEnd",
|
|
||||||
"ArrayStart",
|
|
||||||
"ArrayEnd",
|
|
||||||
}[kind]
|
|
||||||
}
|
|
||||||
|
@ -1,27 +0,0 @@
|
|||||||
package token_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.akyoto.dev/cli/q/src/build/token"
|
|
||||||
"git.akyoto.dev/go/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestTokenKind(t *testing.T) {
|
|
||||||
assert.Equal(t, token.Invalid.String(), "Invalid")
|
|
||||||
assert.Equal(t, token.EOF.String(), "EOF")
|
|
||||||
assert.Equal(t, token.NewLine.String(), "NewLine")
|
|
||||||
assert.Equal(t, token.Identifier.String(), "Identifier")
|
|
||||||
assert.Equal(t, token.Keyword.String(), "Keyword")
|
|
||||||
assert.Equal(t, token.String.String(), "String")
|
|
||||||
assert.Equal(t, token.Number.String(), "Number")
|
|
||||||
assert.Equal(t, token.Operator.String(), "Operator")
|
|
||||||
assert.Equal(t, token.Separator.String(), "Separator")
|
|
||||||
assert.Equal(t, token.Comment.String(), "Comment")
|
|
||||||
assert.Equal(t, token.GroupStart.String(), "GroupStart")
|
|
||||||
assert.Equal(t, token.GroupEnd.String(), "GroupEnd")
|
|
||||||
assert.Equal(t, token.BlockStart.String(), "BlockStart")
|
|
||||||
assert.Equal(t, token.BlockEnd.String(), "BlockEnd")
|
|
||||||
assert.Equal(t, token.ArrayStart.String(), "ArrayStart")
|
|
||||||
assert.Equal(t, token.ArrayEnd.String(), "ArrayEnd")
|
|
||||||
}
|
|
4
src/build/token/Length.go
Normal file
4
src/build/token/Length.go
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
package token
|
||||||
|
|
||||||
|
// Length is the data type for storing token lengths.
|
||||||
|
type Length = uint16
|
@ -1,9 +1,5 @@
|
|||||||
package token
|
package token
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
)
|
|
||||||
|
|
||||||
// List is a slice of tokens.
|
// List is a slice of tokens.
|
||||||
type List []Token
|
type List []Token
|
||||||
|
|
||||||
@ -28,20 +24,3 @@ func (list List) LastIndexKind(kind Kind) int {
|
|||||||
|
|
||||||
return -1
|
return -1
|
||||||
}
|
}
|
||||||
|
|
||||||
// String implements string serialization.
|
|
||||||
func (list List) String() string {
|
|
||||||
builder := bytes.Buffer{}
|
|
||||||
var last Token
|
|
||||||
|
|
||||||
for _, t := range list {
|
|
||||||
if last.Kind == Keyword || last.Kind == Separator || last.Kind == Operator || t.Kind == Operator {
|
|
||||||
builder.WriteByte(' ')
|
|
||||||
}
|
|
||||||
|
|
||||||
builder.Write(t.Bytes)
|
|
||||||
last = t
|
|
||||||
}
|
|
||||||
|
|
||||||
return builder.String()
|
|
||||||
}
|
|
||||||
|
39
src/build/token/Operators.go
Normal file
39
src/build/token/Operators.go
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
package token
|
||||||
|
|
||||||
|
// Operators is a map of all operators used in the language.
|
||||||
|
var Operators = map[string]Kind{
|
||||||
|
".": Period,
|
||||||
|
"=": Assign,
|
||||||
|
":=": Define,
|
||||||
|
"+": Add,
|
||||||
|
"-": Sub,
|
||||||
|
"*": Mul,
|
||||||
|
"/": Div,
|
||||||
|
"%": Mod,
|
||||||
|
"&": And,
|
||||||
|
"|": Or,
|
||||||
|
"^": Xor,
|
||||||
|
"<<": Shl,
|
||||||
|
">>": Shr,
|
||||||
|
"&&": LogicalAnd,
|
||||||
|
"||": LogicalOr,
|
||||||
|
"!": Not,
|
||||||
|
"==": Equal,
|
||||||
|
"!=": NotEqual,
|
||||||
|
">": Greater,
|
||||||
|
"<": Less,
|
||||||
|
">=": GreaterEqual,
|
||||||
|
"<=": LessEqual,
|
||||||
|
"+=": AddAssign,
|
||||||
|
"-=": SubAssign,
|
||||||
|
"*=": MulAssign,
|
||||||
|
"/=": DivAssign,
|
||||||
|
"%=": ModAssign,
|
||||||
|
"&=": AndAssign,
|
||||||
|
"|=": OrAssign,
|
||||||
|
"^=": XorAssign,
|
||||||
|
"<<=": ShlAssign,
|
||||||
|
">>=": ShrAssign,
|
||||||
|
"λ": Call,
|
||||||
|
"@": Array,
|
||||||
|
}
|
@ -1,7 +1,6 @@
|
|||||||
package token
|
package token
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"unsafe"
|
"unsafe"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -9,29 +8,44 @@ import (
|
|||||||
// The characters that make up an identifier are grouped into a single token.
|
// The characters that make up an identifier are grouped into a single token.
|
||||||
// This makes parsing easier and allows us to do better syntax checks.
|
// This makes parsing easier and allows us to do better syntax checks.
|
||||||
type Token struct {
|
type Token struct {
|
||||||
Bytes []byte
|
|
||||||
Position Position
|
Position Position
|
||||||
|
Length Length
|
||||||
Kind Kind
|
Kind Kind
|
||||||
}
|
}
|
||||||
|
|
||||||
// End returns the position after the token.
|
// Bytes returns the byte slice.
|
||||||
func (t *Token) End() Position {
|
func (t Token) Bytes(buffer []byte) []byte {
|
||||||
return t.Position + Position(len(t.Bytes))
|
return buffer[t.Position : t.Position+Position(t.Length)]
|
||||||
}
|
}
|
||||||
|
|
||||||
// String creates a human readable representation for debugging purposes.
|
// End returns the position after the token.
|
||||||
func (t *Token) String() string {
|
func (t Token) End() Position {
|
||||||
return fmt.Sprintf("%s %s", t.Kind, t.Text())
|
return t.Position + Position(t.Length)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsAssignment returns true if the token is an assignment operator.
|
||||||
|
func (t Token) IsAssignment() bool {
|
||||||
|
return t.Kind > _assignments && t.Kind < _assignmentsEnd
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsKeyword returns true if the token is a keyword.
|
||||||
|
func (t Token) IsKeyword() bool {
|
||||||
|
return t.Kind > _keywords && t.Kind < _keywordsEnd
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsOperator returns true if the token is an operator.
|
||||||
|
func (t Token) IsOperator() bool {
|
||||||
|
return t.Kind > _operators && t.Kind < _operatorsEnd
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reset resets the token to default values.
|
// Reset resets the token to default values.
|
||||||
func (t *Token) Reset() {
|
func (t *Token) Reset() {
|
||||||
t.Kind = Invalid
|
|
||||||
t.Position = 0
|
t.Position = 0
|
||||||
t.Bytes = nil
|
t.Length = 0
|
||||||
|
t.Kind = Invalid
|
||||||
}
|
}
|
||||||
|
|
||||||
// Text returns the token text.
|
// Text returns the token text.
|
||||||
func (t *Token) Text() string {
|
func (t Token) Text(buffer []byte) string {
|
||||||
return unsafe.String(unsafe.SliceData(t.Bytes), len(t.Bytes))
|
return unsafe.String(unsafe.SliceData(t.Bytes(buffer)), t.Length)
|
||||||
}
|
}
|
||||||
|
@ -10,8 +10,8 @@ import (
|
|||||||
func TestTokenEnd(t *testing.T) {
|
func TestTokenEnd(t *testing.T) {
|
||||||
hello := token.Token{
|
hello := token.Token{
|
||||||
Kind: token.Identifier,
|
Kind: token.Identifier,
|
||||||
Bytes: []byte("hello"),
|
|
||||||
Position: 0,
|
Position: 0,
|
||||||
|
Length: 5,
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.Equal(t, hello.End(), 5)
|
assert.Equal(t, hello.End(), 5)
|
||||||
@ -20,34 +20,33 @@ func TestTokenEnd(t *testing.T) {
|
|||||||
func TestTokenReset(t *testing.T) {
|
func TestTokenReset(t *testing.T) {
|
||||||
hello := token.Token{
|
hello := token.Token{
|
||||||
Kind: token.Identifier,
|
Kind: token.Identifier,
|
||||||
Bytes: []byte("hello"),
|
|
||||||
Position: 1,
|
Position: 1,
|
||||||
|
Length: 5,
|
||||||
}
|
}
|
||||||
|
|
||||||
hello.Reset()
|
hello.Reset()
|
||||||
assert.Nil(t, hello.Bytes)
|
|
||||||
assert.Equal(t, hello.Position, 0)
|
assert.Equal(t, hello.Position, 0)
|
||||||
|
assert.Equal(t, hello.Length, 0)
|
||||||
assert.Equal(t, hello.Kind, token.Invalid)
|
assert.Equal(t, hello.Kind, token.Invalid)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTokenString(t *testing.T) {
|
|
||||||
hello := token.Token{
|
|
||||||
Kind: token.Identifier,
|
|
||||||
Bytes: []byte("hello"),
|
|
||||||
Position: 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, hello.String(), "Identifier hello")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTokenText(t *testing.T) {
|
func TestTokenText(t *testing.T) {
|
||||||
hello := token.Token{Kind: token.Identifier, Bytes: []byte("hello"), Position: 0}
|
buffer := []byte("hello, world")
|
||||||
comma := token.Token{Kind: token.Separator, Bytes: []byte(","), Position: 5}
|
hello := token.Token{Kind: token.Identifier, Position: 0, Length: 5}
|
||||||
world := token.Token{Kind: token.Identifier, Bytes: []byte("world"), Position: 7}
|
comma := token.Token{Kind: token.Separator, Position: 5, Length: 1}
|
||||||
|
world := token.Token{Kind: token.Identifier, Position: 7, Length: 5}
|
||||||
|
|
||||||
assert.Equal(t, hello.Text(), "hello")
|
assert.Equal(t, hello.Text(buffer), "hello")
|
||||||
assert.Equal(t, world.Text(), "world")
|
assert.Equal(t, comma.Text(buffer), ",")
|
||||||
|
assert.Equal(t, world.Text(buffer), "world")
|
||||||
list := token.List{hello, comma, world}
|
}
|
||||||
assert.Equal(t, list.String(), "hello, world")
|
|
||||||
|
func TestTokenGroups(t *testing.T) {
|
||||||
|
assignment := token.Token{Kind: token.Assign}
|
||||||
|
operator := token.Token{Kind: token.Add}
|
||||||
|
keyword := token.Token{Kind: token.If}
|
||||||
|
|
||||||
|
assert.True(t, assignment.IsAssignment())
|
||||||
|
assert.True(t, operator.IsOperator())
|
||||||
|
assert.True(t, keyword.IsKeyword())
|
||||||
}
|
}
|
||||||
|
@ -1,20 +1,5 @@
|
|||||||
package token
|
package token
|
||||||
|
|
||||||
import "git.akyoto.dev/cli/q/src/build/keyword"
|
|
||||||
|
|
||||||
// Pre-allocate these byte buffers so we can re-use them
|
|
||||||
// instead of allocating a new buffer every time.
|
|
||||||
var (
|
|
||||||
groupStartBytes = []byte{'('}
|
|
||||||
groupEndBytes = []byte{')'}
|
|
||||||
blockStartBytes = []byte{'{'}
|
|
||||||
blockEndBytes = []byte{'}'}
|
|
||||||
arrayStartBytes = []byte{'['}
|
|
||||||
arrayEndBytes = []byte{']'}
|
|
||||||
separatorBytes = []byte{','}
|
|
||||||
newLineBytes = []byte{'\n'}
|
|
||||||
)
|
|
||||||
|
|
||||||
// Tokenize turns the file contents into a list of tokens.
|
// Tokenize turns the file contents into a list of tokens.
|
||||||
func Tokenize(buffer []byte) List {
|
func Tokenize(buffer []byte) List {
|
||||||
var (
|
var (
|
||||||
@ -26,21 +11,21 @@ func Tokenize(buffer []byte) List {
|
|||||||
switch buffer[i] {
|
switch buffer[i] {
|
||||||
case ' ', '\t':
|
case ' ', '\t':
|
||||||
case ',':
|
case ',':
|
||||||
tokens = append(tokens, Token{Kind: Separator, Position: i, Bytes: separatorBytes})
|
tokens = append(tokens, Token{Kind: Separator, Position: i, Length: 1})
|
||||||
case '(':
|
case '(':
|
||||||
tokens = append(tokens, Token{Kind: GroupStart, Position: i, Bytes: groupStartBytes})
|
tokens = append(tokens, Token{Kind: GroupStart, Position: i, Length: 1})
|
||||||
case ')':
|
case ')':
|
||||||
tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Bytes: groupEndBytes})
|
tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Length: 1})
|
||||||
case '{':
|
case '{':
|
||||||
tokens = append(tokens, Token{Kind: BlockStart, Position: i, Bytes: blockStartBytes})
|
tokens = append(tokens, Token{Kind: BlockStart, Position: i, Length: 1})
|
||||||
case '}':
|
case '}':
|
||||||
tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Bytes: blockEndBytes})
|
tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Length: 1})
|
||||||
case '[':
|
case '[':
|
||||||
tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Bytes: arrayStartBytes})
|
tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Length: 1})
|
||||||
case ']':
|
case ']':
|
||||||
tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Bytes: arrayEndBytes})
|
tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Length: 1})
|
||||||
case '\n':
|
case '\n':
|
||||||
tokens = append(tokens, Token{Kind: NewLine, Position: i, Bytes: newLineBytes})
|
tokens = append(tokens, Token{Kind: NewLine, Position: i, Length: 1})
|
||||||
case '/':
|
case '/':
|
||||||
if i+1 >= Position(len(buffer)) || buffer[i+1] != '/' {
|
if i+1 >= Position(len(buffer)) || buffer[i+1] != '/' {
|
||||||
position := i
|
position := i
|
||||||
@ -50,7 +35,7 @@ func Tokenize(buffer []byte) List {
|
|||||||
i++
|
i++
|
||||||
}
|
}
|
||||||
|
|
||||||
tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]})
|
tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
|
||||||
} else {
|
} else {
|
||||||
position := i
|
position := i
|
||||||
|
|
||||||
@ -58,7 +43,7 @@ func Tokenize(buffer []byte) List {
|
|||||||
i++
|
i++
|
||||||
}
|
}
|
||||||
|
|
||||||
tokens = append(tokens, Token{Kind: Comment, Position: position, Bytes: buffer[position:i]})
|
tokens = append(tokens, Token{Kind: Comment, Position: position, Length: Length(i - position)})
|
||||||
}
|
}
|
||||||
|
|
||||||
continue
|
continue
|
||||||
@ -78,7 +63,7 @@ func Tokenize(buffer []byte) List {
|
|||||||
i++
|
i++
|
||||||
}
|
}
|
||||||
|
|
||||||
tokens = append(tokens, Token{Kind: String, Position: start, Bytes: buffer[start:end]})
|
tokens = append(tokens, Token{Kind: String, Position: start, Length: Length(end - start)})
|
||||||
continue
|
continue
|
||||||
|
|
||||||
default:
|
default:
|
||||||
@ -91,14 +76,14 @@ func Tokenize(buffer []byte) List {
|
|||||||
}
|
}
|
||||||
|
|
||||||
identifier := buffer[position:i]
|
identifier := buffer[position:i]
|
||||||
keyword, isKeyword := keyword.Map[string(identifier)]
|
kind := Identifier
|
||||||
|
keyword, isKeyword := Keywords[string(identifier)]
|
||||||
|
|
||||||
if isKeyword {
|
if isKeyword {
|
||||||
tokens = append(tokens, Token{Kind: Keyword, Position: position, Bytes: keyword})
|
kind = keyword
|
||||||
} else {
|
|
||||||
tokens = append(tokens, Token{Kind: Identifier, Position: position, Bytes: identifier})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(len(identifier))})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -110,7 +95,7 @@ func Tokenize(buffer []byte) List {
|
|||||||
i++
|
i++
|
||||||
}
|
}
|
||||||
|
|
||||||
tokens = append(tokens, Token{Kind: Number, Position: position, Bytes: buffer[position:i]})
|
tokens = append(tokens, Token{Kind: Number, Position: position, Length: Length(i - position)})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -122,17 +107,17 @@ func Tokenize(buffer []byte) List {
|
|||||||
i++
|
i++
|
||||||
}
|
}
|
||||||
|
|
||||||
tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]})
|
tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
tokens = append(tokens, Token{Kind: Invalid, Position: i, Bytes: buffer[i : i+1]})
|
tokens = append(tokens, Token{Kind: Invalid, Position: i, Length: 1})
|
||||||
}
|
}
|
||||||
|
|
||||||
i++
|
i++
|
||||||
}
|
}
|
||||||
|
|
||||||
tokens = append(tokens, Token{Kind: EOF, Position: i, Bytes: nil})
|
tokens = append(tokens, Token{Kind: EOF, Position: i, Length: 0})
|
||||||
return tokens
|
return tokens
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -9,394 +9,243 @@ import (
|
|||||||
|
|
||||||
func TestFunction(t *testing.T) {
|
func TestFunction(t *testing.T) {
|
||||||
tokens := token.Tokenize([]byte("main(){}"))
|
tokens := token.Tokenize([]byte("main(){}"))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected := []token.Kind{
|
||||||
Kind: token.Identifier,
|
token.Identifier,
|
||||||
Bytes: []byte("main"),
|
token.GroupStart,
|
||||||
Position: 0,
|
token.GroupEnd,
|
||||||
},
|
token.BlockStart,
|
||||||
{
|
token.BlockEnd,
|
||||||
Kind: token.GroupStart,
|
token.EOF,
|
||||||
Bytes: []byte("("),
|
}
|
||||||
Position: 4,
|
|
||||||
},
|
for i, kind := range expected {
|
||||||
{
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
Kind: token.GroupEnd,
|
}
|
||||||
Bytes: []byte(")"),
|
|
||||||
Position: 5,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.BlockStart,
|
|
||||||
Bytes: []byte("{"),
|
|
||||||
Position: 6,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.BlockEnd,
|
|
||||||
Bytes: []byte("}"),
|
|
||||||
Position: 7,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.EOF,
|
|
||||||
Bytes: nil,
|
|
||||||
Position: 8,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestKeyword(t *testing.T) {
|
func TestKeyword(t *testing.T) {
|
||||||
tokens := token.Tokenize([]byte("return x"))
|
tokens := token.Tokenize([]byte("return x"))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected := []token.Kind{
|
||||||
Kind: token.Keyword,
|
token.Return,
|
||||||
Bytes: []byte("return"),
|
token.Identifier,
|
||||||
Position: 0,
|
token.EOF,
|
||||||
},
|
}
|
||||||
{
|
|
||||||
Kind: token.Identifier,
|
for i, kind := range expected {
|
||||||
Bytes: []byte("x"),
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
Position: 7,
|
}
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.EOF,
|
|
||||||
Bytes: nil,
|
|
||||||
Position: 8,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestArray(t *testing.T) {
|
func TestArray(t *testing.T) {
|
||||||
tokens := token.Tokenize([]byte("array[i]"))
|
tokens := token.Tokenize([]byte("array[i]"))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected := []token.Kind{
|
||||||
Kind: token.Identifier,
|
token.Identifier,
|
||||||
Bytes: []byte("array"),
|
token.ArrayStart,
|
||||||
Position: 0,
|
token.Identifier,
|
||||||
},
|
token.ArrayEnd,
|
||||||
{
|
token.EOF,
|
||||||
Kind: token.ArrayStart,
|
}
|
||||||
Bytes: []byte("["),
|
|
||||||
Position: 5,
|
for i, kind := range expected {
|
||||||
},
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
{
|
}
|
||||||
Kind: token.Identifier,
|
|
||||||
Bytes: []byte("i"),
|
|
||||||
Position: 6,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.ArrayEnd,
|
|
||||||
Bytes: []byte("]"),
|
|
||||||
Position: 7,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.EOF,
|
|
||||||
Bytes: nil,
|
|
||||||
Position: 8,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewline(t *testing.T) {
|
func TestNewline(t *testing.T) {
|
||||||
tokens := token.Tokenize([]byte("\n\n"))
|
tokens := token.Tokenize([]byte("\n\n"))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected := []token.Kind{
|
||||||
Kind: token.NewLine,
|
token.NewLine,
|
||||||
Bytes: []byte("\n"),
|
token.NewLine,
|
||||||
Position: 0,
|
token.EOF,
|
||||||
},
|
}
|
||||||
{
|
|
||||||
Kind: token.NewLine,
|
for i, kind := range expected {
|
||||||
Bytes: []byte("\n"),
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
Position: 1,
|
}
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.EOF,
|
|
||||||
Bytes: nil,
|
|
||||||
Position: 2,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNumber(t *testing.T) {
|
func TestNumber(t *testing.T) {
|
||||||
tokens := token.Tokenize([]byte(`123 456`))
|
tokens := token.Tokenize([]byte(`123 456`))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected := []token.Kind{
|
||||||
Kind: token.Number,
|
token.Number,
|
||||||
Bytes: []byte("123"),
|
token.Number,
|
||||||
Position: 0,
|
token.EOF,
|
||||||
},
|
}
|
||||||
{
|
|
||||||
Kind: token.Number,
|
for i, kind := range expected {
|
||||||
Bytes: []byte("456"),
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
Position: 4,
|
}
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.EOF,
|
|
||||||
Bytes: nil,
|
|
||||||
Position: 7,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOperator(t *testing.T) {
|
func TestOperator(t *testing.T) {
|
||||||
tokens := token.Tokenize([]byte(`+ - * /`))
|
tokens := token.Tokenize([]byte(`+ - * /`))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected := []token.Kind{
|
||||||
Kind: token.Operator,
|
token.Add,
|
||||||
Bytes: []byte("+"),
|
token.Sub,
|
||||||
Position: 0,
|
token.Mul,
|
||||||
},
|
token.Div,
|
||||||
{
|
token.EOF,
|
||||||
Kind: token.Operator,
|
}
|
||||||
Bytes: []byte("-"),
|
|
||||||
Position: 2,
|
for i, kind := range expected {
|
||||||
},
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
{
|
}
|
||||||
Kind: token.Operator,
|
|
||||||
Bytes: []byte("*"),
|
|
||||||
Position: 4,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.Operator,
|
|
||||||
Bytes: []byte("/"),
|
|
||||||
Position: 6,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.EOF,
|
|
||||||
Bytes: nil,
|
|
||||||
Position: 7,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOperatorAssign(t *testing.T) {
|
func TestOperatorAssign(t *testing.T) {
|
||||||
tokens := token.Tokenize([]byte(`+= -= *= /= ==`))
|
tokens := token.Tokenize([]byte(`+= -= *= /= &= |= ^= <<= >>=`))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected := []token.Kind{
|
||||||
Kind: token.Operator,
|
token.AddAssign,
|
||||||
Bytes: []byte("+="),
|
token.SubAssign,
|
||||||
Position: 0,
|
token.MulAssign,
|
||||||
},
|
token.DivAssign,
|
||||||
{
|
token.AndAssign,
|
||||||
Kind: token.Operator,
|
token.OrAssign,
|
||||||
Bytes: []byte("-="),
|
token.XorAssign,
|
||||||
Position: 3,
|
token.ShlAssign,
|
||||||
},
|
token.ShrAssign,
|
||||||
{
|
token.EOF,
|
||||||
Kind: token.Operator,
|
}
|
||||||
Bytes: []byte("*="),
|
|
||||||
Position: 6,
|
for i, kind := range expected {
|
||||||
},
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
{
|
}
|
||||||
Kind: token.Operator,
|
|
||||||
Bytes: []byte("/="),
|
|
||||||
Position: 9,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.Operator,
|
|
||||||
Bytes: []byte("=="),
|
|
||||||
Position: 12,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.EOF,
|
|
||||||
Bytes: nil,
|
|
||||||
Position: 14,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSeparator(t *testing.T) {
|
func TestSeparator(t *testing.T) {
|
||||||
tokens := token.Tokenize([]byte("a,b,c"))
|
tokens := token.Tokenize([]byte("a,b,c"))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected := []token.Kind{
|
||||||
Kind: token.Identifier,
|
token.Identifier,
|
||||||
Bytes: []byte("a"),
|
token.Separator,
|
||||||
Position: 0,
|
token.Identifier,
|
||||||
},
|
token.Separator,
|
||||||
{
|
token.Identifier,
|
||||||
Kind: token.Separator,
|
token.EOF,
|
||||||
Bytes: []byte(","),
|
}
|
||||||
Position: 1,
|
|
||||||
},
|
for i, kind := range expected {
|
||||||
{
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
Kind: token.Identifier,
|
}
|
||||||
Bytes: []byte("b"),
|
|
||||||
Position: 2,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.Separator,
|
|
||||||
Bytes: []byte(","),
|
|
||||||
Position: 3,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.Identifier,
|
|
||||||
Bytes: []byte("c"),
|
|
||||||
Position: 4,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.EOF,
|
|
||||||
Bytes: nil,
|
|
||||||
Position: 5,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestComment(t *testing.T) {
|
func TestComment(t *testing.T) {
|
||||||
tokens := token.Tokenize([]byte("// Hello\n// World"))
|
tokens := token.Tokenize([]byte("// Hello\n// World"))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected := []token.Kind{
|
||||||
Kind: token.Comment,
|
token.Comment,
|
||||||
Bytes: []byte(`// Hello`),
|
token.NewLine,
|
||||||
Position: 0,
|
token.Comment,
|
||||||
},
|
token.EOF,
|
||||||
{
|
}
|
||||||
Kind: token.NewLine,
|
|
||||||
Bytes: []byte("\n"),
|
for i, kind := range expected {
|
||||||
Position: 8,
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
},
|
}
|
||||||
{
|
|
||||||
Kind: token.Comment,
|
|
||||||
Bytes: []byte(`// World`),
|
|
||||||
Position: 9,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.EOF,
|
|
||||||
Bytes: nil,
|
|
||||||
Position: 17,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
tokens = token.Tokenize([]byte("// Hello\n"))
|
tokens = token.Tokenize([]byte("// Hello\n"))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected = []token.Kind{
|
||||||
Kind: token.Comment,
|
token.Comment,
|
||||||
Bytes: []byte(`// Hello`),
|
token.NewLine,
|
||||||
Position: 0,
|
token.EOF,
|
||||||
},
|
}
|
||||||
{
|
|
||||||
Kind: token.NewLine,
|
for i, kind := range expected {
|
||||||
Bytes: []byte("\n"),
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
Position: 8,
|
}
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.EOF,
|
|
||||||
Bytes: nil,
|
|
||||||
Position: 9,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
tokens = token.Tokenize([]byte(`// Hello`))
|
tokens = token.Tokenize([]byte(`// Hello`))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected = []token.Kind{
|
||||||
Kind: token.Comment,
|
token.Comment,
|
||||||
Bytes: []byte(`// Hello`),
|
token.EOF,
|
||||||
Position: 0,
|
}
|
||||||
},
|
|
||||||
{
|
for i, kind := range expected {
|
||||||
Kind: token.EOF,
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
Bytes: nil,
|
}
|
||||||
Position: 8,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
tokens = token.Tokenize([]byte(`//`))
|
tokens = token.Tokenize([]byte(`//`))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected = []token.Kind{
|
||||||
Kind: token.Comment,
|
token.Comment,
|
||||||
Bytes: []byte(`//`),
|
token.EOF,
|
||||||
Position: 0,
|
}
|
||||||
},
|
|
||||||
{
|
for i, kind := range expected {
|
||||||
Kind: token.EOF,
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
Bytes: nil,
|
}
|
||||||
Position: 2,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
tokens = token.Tokenize([]byte(`/`))
|
tokens = token.Tokenize([]byte(`/`))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected = []token.Kind{
|
||||||
Kind: token.Operator,
|
token.Div,
|
||||||
Bytes: []byte(`/`),
|
token.EOF,
|
||||||
Position: 0,
|
}
|
||||||
},
|
|
||||||
{
|
for i, kind := range expected {
|
||||||
Kind: token.EOF,
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
Bytes: nil,
|
}
|
||||||
Position: 1,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestInvalid(t *testing.T) {
|
func TestInvalid(t *testing.T) {
|
||||||
tokens := token.Tokenize([]byte(`@#`))
|
tokens := token.Tokenize([]byte(`##`))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected := []token.Kind{
|
||||||
Kind: token.Invalid,
|
token.Invalid,
|
||||||
Bytes: []byte(`@`),
|
token.Invalid,
|
||||||
Position: 0,
|
token.EOF,
|
||||||
},
|
}
|
||||||
{
|
|
||||||
Kind: token.Invalid,
|
for i, kind := range expected {
|
||||||
Bytes: []byte(`#`),
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
Position: 1,
|
}
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.EOF,
|
|
||||||
Bytes: nil,
|
|
||||||
Position: 2,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestString(t *testing.T) {
|
func TestString(t *testing.T) {
|
||||||
tokens := token.Tokenize([]byte(`"Hello" "World"`))
|
tokens := token.Tokenize([]byte(`"Hello" "World"`))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected := []token.Kind{
|
||||||
Kind: token.String,
|
token.String,
|
||||||
Bytes: []byte(`"Hello"`),
|
token.String,
|
||||||
Position: 0,
|
token.EOF,
|
||||||
},
|
}
|
||||||
{
|
|
||||||
Kind: token.String,
|
for i, kind := range expected {
|
||||||
Bytes: []byte(`"World"`),
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
Position: 8,
|
}
|
||||||
},
|
|
||||||
{
|
|
||||||
Kind: token.EOF,
|
|
||||||
Bytes: nil,
|
|
||||||
Position: 15,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStringMultiline(t *testing.T) {
|
func TestStringMultiline(t *testing.T) {
|
||||||
tokens := token.Tokenize([]byte("\"Hello\nWorld\""))
|
tokens := token.Tokenize([]byte("\"Hello\nWorld\""))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected := []token.Kind{
|
||||||
Kind: token.String,
|
token.String,
|
||||||
Bytes: []byte("\"Hello\nWorld\""),
|
token.EOF,
|
||||||
Position: 0,
|
}
|
||||||
},
|
|
||||||
{
|
for i, kind := range expected {
|
||||||
Kind: token.EOF,
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
Bytes: nil,
|
}
|
||||||
Position: 13,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStringEOF(t *testing.T) {
|
func TestStringEOF(t *testing.T) {
|
||||||
tokens := token.Tokenize([]byte(`"EOF`))
|
tokens := token.Tokenize([]byte(`"EOF`))
|
||||||
assert.DeepEqual(t, tokens, token.List{
|
|
||||||
{
|
expected := []token.Kind{
|
||||||
Kind: token.String,
|
token.String,
|
||||||
Bytes: []byte(`"EOF`),
|
token.EOF,
|
||||||
Position: 0,
|
}
|
||||||
},
|
|
||||||
{
|
for i, kind := range expected {
|
||||||
Kind: token.EOF,
|
assert.Equal(t, tokens[i].Kind, kind)
|
||||||
Bytes: nil,
|
}
|
||||||
Position: 4,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
@ -21,10 +21,10 @@ var errs = []struct {
|
|||||||
{"InvalidInstructionNumber.q", &errors.InvalidInstruction{Instruction: "123"}},
|
{"InvalidInstructionNumber.q", &errors.InvalidInstruction{Instruction: "123"}},
|
||||||
{"InvalidInstructionExpression.q", &errors.InvalidInstruction{Instruction: "+"}},
|
{"InvalidInstructionExpression.q", &errors.InvalidInstruction{Instruction: "+"}},
|
||||||
{"InvalidExpression.q", errors.InvalidExpression},
|
{"InvalidExpression.q", errors.InvalidExpression},
|
||||||
{"InvalidOperator.q", &errors.InvalidOperator{Operator: "+++"}},
|
|
||||||
{"InvalidCharacter.q", &errors.InvalidCharacter{Character: "@"}},
|
{"InvalidCharacter.q", &errors.InvalidCharacter{Character: "@"}},
|
||||||
{"InvalidCharacter2.q", &errors.InvalidCharacter{Character: "@"}},
|
{"InvalidCharacter2.q", &errors.InvalidCharacter{Character: "@"}},
|
||||||
{"InvalidCharacter3.q", &errors.InvalidCharacter{Character: "@"}},
|
{"InvalidCharacter3.q", &errors.InvalidCharacter{Character: "@"}},
|
||||||
|
{"InvalidCharacter4.q", &errors.InvalidCharacter{Character: "+++"}},
|
||||||
{"MissingBlockEnd.q", errors.MissingBlockEnd},
|
{"MissingBlockEnd.q", errors.MissingBlockEnd},
|
||||||
{"MissingBlockStart.q", errors.MissingBlockStart},
|
{"MissingBlockStart.q", errors.MissingBlockStart},
|
||||||
{"MissingGroupEnd.q", errors.MissingGroupEnd},
|
{"MissingGroupEnd.q", errors.MissingGroupEnd},
|
||||||
|
Loading…
Reference in New Issue
Block a user