From 04ba68a07501d811dcc7440a8b915e5eb92eccb6 Mon Sep 17 00:00:00 2001 From: Eduard Urbach Date: Sun, 21 Jul 2024 14:35:06 +0200 Subject: [PATCH] Reduced token size --- src/build/ast/AST.go | 4 +- src/build/ast/Assign.go | 4 - src/build/ast/Call.go | 4 - src/build/ast/Count.go | 16 +- src/build/ast/Define.go | 6 - src/build/ast/If.go | 6 - src/build/ast/Loop.go | 6 - src/build/ast/Parse.go | 37 +- src/build/ast/Return.go | 6 - src/build/core/Compare.go | 2 +- src/build/core/CompileAssign.go | 10 +- src/build/core/CompileCall.go | 4 +- src/build/core/CompileCondition.go | 25 +- src/build/core/CompileDefinition.go | 4 +- src/build/core/CompileIf.go | 2 +- src/build/core/CompileLoop.go | 2 +- src/build/core/CompileTokens.go | 4 +- src/build/core/ExecuteLeaf.go | 6 +- src/build/core/ExecuteRegisterNumber.go | 16 +- src/build/core/ExecuteRegisterRegister.go | 16 +- src/build/core/Function.go | 2 +- src/build/core/JumpIfFalse.go | 15 +- src/build/core/JumpIfTrue.go | 15 +- src/build/core/NewFunction.go | 2 +- src/build/core/TokenToRegister.go | 6 +- src/build/expression/Expression.go | 24 +- src/build/expression/Expression_test.go | 30 +- src/build/expression/Operator.go | 68 +-- src/build/expression/Parse.go | 27 +- src/build/fs/File.go | 1 + src/build/keyword/Keyword.go | 16 - src/build/scanner/scanFile.go | 20 +- src/build/scope/Stack.go | 4 +- src/build/token/Count.go | 4 +- src/build/token/Count_test.go | 11 +- src/build/token/Keywords.go | 9 + src/build/token/Kind.go | 127 ++--- src/build/token/Kind_test.go | 27 - src/build/token/Length.go | 4 + src/build/token/List.go | 21 - src/build/token/Operators.go | 39 ++ src/build/token/Token.go | 38 +- src/build/token/Token_test.go | 41 +- src/build/token/Tokenize.go | 53 +- src/build/token/Tokenize_test.go | 521 +++++++----------- ...{InvalidOperator.q => InvalidCharacter4.q} | 0 tests/errors_test.go | 2 +- 47 files changed, 543 insertions(+), 764 deletions(-) delete mode 100644 src/build/keyword/Keyword.go create mode 100644 src/build/token/Keywords.go delete mode 100644 src/build/token/Kind_test.go create mode 100644 src/build/token/Length.go create mode 100644 src/build/token/Operators.go rename tests/errors/{InvalidOperator.q => InvalidCharacter4.q} (100%) diff --git a/src/build/ast/AST.go b/src/build/ast/AST.go index 1c8f6dd..c7c4c47 100644 --- a/src/build/ast/AST.go +++ b/src/build/ast/AST.go @@ -1,6 +1,4 @@ package ast -import "fmt" - -type Node fmt.Stringer +type Node any type AST []Node diff --git a/src/build/ast/Assign.go b/src/build/ast/Assign.go index 32c6819..161f968 100644 --- a/src/build/ast/Assign.go +++ b/src/build/ast/Assign.go @@ -8,7 +8,3 @@ import ( type Assign struct { Expression *expression.Expression } - -func (node *Assign) String() string { - return node.Expression.String() -} diff --git a/src/build/ast/Call.go b/src/build/ast/Call.go index d7825b3..6bd7dec 100644 --- a/src/build/ast/Call.go +++ b/src/build/ast/Call.go @@ -6,7 +6,3 @@ import "git.akyoto.dev/cli/q/src/build/expression" type Call struct { Expression *expression.Expression } - -func (node *Call) String() string { - return node.Expression.String() -} diff --git a/src/build/ast/Count.go b/src/build/ast/Count.go index 0cc6986..b14c652 100644 --- a/src/build/ast/Count.go +++ b/src/build/ast/Count.go @@ -3,31 +3,31 @@ package ast import "git.akyoto.dev/cli/q/src/build/token" // Count counts how often the given token appears in the AST. -func Count(body AST, kind token.Kind, name string) int { +func Count(body AST, buffer []byte, kind token.Kind, name string) int { count := 0 for _, node := range body { switch node := node.(type) { case *Assign: - count += node.Expression.Count(kind, name) + count += node.Expression.Count(buffer, kind, name) case *Call: - count += node.Expression.Count(kind, name) + count += node.Expression.Count(buffer, kind, name) case *Define: - count += node.Value.Count(kind, name) + count += node.Value.Count(buffer, kind, name) case *Return: if node.Value != nil { - count += node.Value.Count(kind, name) + count += node.Value.Count(buffer, kind, name) } case *If: - count += node.Condition.Count(kind, name) - count += Count(node.Body, kind, name) + count += node.Condition.Count(buffer, kind, name) + count += Count(node.Body, buffer, kind, name) case *Loop: - count += Count(node.Body, kind, name) + count += Count(node.Body, buffer, kind, name) default: panic("unknown AST type") diff --git a/src/build/ast/Define.go b/src/build/ast/Define.go index 6751491..64f0c15 100644 --- a/src/build/ast/Define.go +++ b/src/build/ast/Define.go @@ -1,8 +1,6 @@ package ast import ( - "fmt" - "git.akyoto.dev/cli/q/src/build/expression" "git.akyoto.dev/cli/q/src/build/token" ) @@ -12,7 +10,3 @@ type Define struct { Value *expression.Expression Name token.Token } - -func (node *Define) String() string { - return fmt.Sprintf("(= %s %s)", node.Name.Text(), node.Value) -} diff --git a/src/build/ast/If.go b/src/build/ast/If.go index 47d98d4..4275f34 100644 --- a/src/build/ast/If.go +++ b/src/build/ast/If.go @@ -1,8 +1,6 @@ package ast import ( - "fmt" - "git.akyoto.dev/cli/q/src/build/expression" ) @@ -11,7 +9,3 @@ type If struct { Condition *expression.Expression Body AST } - -func (node *If) String() string { - return fmt.Sprintf("(if %s %s)", node.Condition, node.Body) -} diff --git a/src/build/ast/Loop.go b/src/build/ast/Loop.go index 5b0dc33..e1ca210 100644 --- a/src/build/ast/Loop.go +++ b/src/build/ast/Loop.go @@ -1,12 +1,6 @@ package ast -import "fmt" - // Loop represents a block of repeatable statements. type Loop struct { Body AST } - -func (node *Loop) String() string { - return fmt.Sprintf("(loop %s)", node.Body) -} diff --git a/src/build/ast/Parse.go b/src/build/ast/Parse.go index eae5666..5610e1a 100644 --- a/src/build/ast/Parse.go +++ b/src/build/ast/Parse.go @@ -3,16 +3,15 @@ package ast import ( "git.akyoto.dev/cli/q/src/build/errors" "git.akyoto.dev/cli/q/src/build/expression" - "git.akyoto.dev/cli/q/src/build/keyword" "git.akyoto.dev/cli/q/src/build/token" ) // Parse generates an AST from a list of tokens. -func Parse(tokens token.List) (AST, error) { +func Parse(tokens []token.Token, buffer []byte) (AST, error) { tree := make(AST, 0, len(tokens)/64) err := EachInstruction(tokens, func(instruction token.List) error { - node, err := toASTNode(instruction) + node, err := toASTNode(instruction, buffer) if err == nil && node != nil { tree = append(tree, node) @@ -25,11 +24,9 @@ func Parse(tokens token.List) (AST, error) { } // toASTNode generates an AST node from an instruction. -func toASTNode(tokens token.List) (Node, error) { - if tokens[0].Kind == token.Keyword { - word := tokens[0].Text() - - if word == keyword.Return { +func toASTNode(tokens token.List, buffer []byte) (Node, error) { + if tokens[0].IsKeyword() { + if tokens[0].Kind == token.Return { if len(tokens) == 1 { return &Return{}, nil } @@ -38,7 +35,7 @@ func toASTNode(tokens token.List) (Node, error) { return &Return{Value: value}, nil } - if keywordHasBlock(word) { + if keywordHasBlock(tokens[0].Kind) { blockStart := tokens.IndexKind(token.BlockStart) blockEnd := tokens.LastIndexKind(token.BlockEnd) @@ -50,19 +47,19 @@ func toASTNode(tokens token.List) (Node, error) { return nil, errors.New(errors.MissingBlockEnd, nil, tokens[len(tokens)-1].End()) } - body, err := Parse(tokens[blockStart+1 : blockEnd]) + body, err := Parse(tokens[blockStart+1:blockEnd], buffer) - switch word { - case keyword.If: + switch tokens[0].Kind { + case token.If: condition := expression.Parse(tokens[1:blockStart]) return &If{Condition: condition, Body: body}, err - case keyword.Loop: + case token.Loop: return &Loop{Body: body}, err } } - return nil, errors.New(&errors.KeywordNotImplemented{Keyword: word}, nil, tokens[0].Position) + return nil, errors.New(&errors.KeywordNotImplemented{Keyword: tokens[0].Text(buffer)}, nil, tokens[0].Position) } expr := expression.Parse(tokens) @@ -92,26 +89,26 @@ func toASTNode(tokens token.List) (Node, error) { return &Call{Expression: expr}, nil default: - return nil, errors.New(&errors.InvalidInstruction{Instruction: expr.Token.Text()}, nil, expr.Token.Position) + return nil, errors.New(&errors.InvalidInstruction{Instruction: expr.Token.Text(buffer)}, nil, expr.Token.Position) } } // IsAssignment returns true if the expression is an assignment. func IsAssignment(expr *expression.Expression) bool { - return expr.Token.Kind == token.Operator && expr.Token.Bytes[len(expr.Token.Bytes)-1] == '=' + return expr.Token.IsAssignment() } // IsFunctionCall returns true if the expression is a function call. func IsFunctionCall(expr *expression.Expression) bool { - return expr.Token.Kind == token.Operator && expr.Token.Text() == "λ" + return expr.Token.Kind == token.Call } // IsVariableDefinition returns true if the expression is a variable definition. func IsVariableDefinition(expr *expression.Expression) bool { - return expr.Token.Kind == token.Operator && expr.Token.Text() == ":=" + return expr.Token.Kind == token.Define } // keywordHasBlock returns true if the keyword requires a block. -func keywordHasBlock(word string) bool { - return word == keyword.If || word == keyword.Loop +func keywordHasBlock(kind token.Kind) bool { + return kind == token.If || kind == token.Loop } diff --git a/src/build/ast/Return.go b/src/build/ast/Return.go index a56520f..37574e2 100644 --- a/src/build/ast/Return.go +++ b/src/build/ast/Return.go @@ -1,8 +1,6 @@ package ast import ( - "fmt" - "git.akyoto.dev/cli/q/src/build/expression" ) @@ -10,7 +8,3 @@ import ( type Return struct { Value *expression.Expression } - -func (node *Return) String() string { - return fmt.Sprintf("(return %s)", node.Value) -} diff --git a/src/build/core/Compare.go b/src/build/core/Compare.go index 3be29ec..6cd0b5d 100644 --- a/src/build/core/Compare.go +++ b/src/build/core/Compare.go @@ -13,7 +13,7 @@ func (f *Function) Compare(comparison *expression.Expression) error { right := comparison.Children[1] if left.IsLeaf() && left.Token.Kind == token.Identifier { - name := left.Token.Text() + name := left.Token.Text(f.File.Bytes) variable := f.VariableByName(name) if variable == nil { diff --git a/src/build/core/CompileAssign.go b/src/build/core/CompileAssign.go index 384630d..4d3e08b 100644 --- a/src/build/core/CompileAssign.go +++ b/src/build/core/CompileAssign.go @@ -16,7 +16,7 @@ func (f *Function) CompileAssign(node *ast.Assign) error { right := node.Expression.Children[1] if left.IsLeaf() { - name := left.Token.Text() + name := left.Token.Text(f.File.Bytes) variable := f.VariableByName(name) if variable == nil { @@ -27,8 +27,8 @@ func (f *Function) CompileAssign(node *ast.Assign) error { return f.Execute(operator, variable.Register, right) } - if left.Token.Kind == token.Operator && left.Token.Text() == "@" { - name := left.Children[0].Token.Text() + if left.Token.Kind == token.Array { + name := left.Children[0].Token.Text(f.File.Bytes) variable := f.VariableByName(name) if variable == nil { @@ -38,13 +38,13 @@ func (f *Function) CompileAssign(node *ast.Assign) error { defer f.useVariable(variable) index := left.Children[1] - offset, err := strconv.Atoi(index.Token.Text()) + offset, err := strconv.Atoi(index.Token.Text(f.File.Bytes)) if err != nil { return err } - num, err := strconv.Atoi(right.Token.Text()) + num, err := strconv.Atoi(right.Token.Text(f.File.Bytes)) if err != nil { return err diff --git a/src/build/core/CompileCall.go b/src/build/core/CompileCall.go index 4c2adc6..3270d01 100644 --- a/src/build/core/CompileCall.go +++ b/src/build/core/CompileCall.go @@ -15,9 +15,9 @@ func (f *Function) CompileCall(root *expression.Expression) error { funcName := "" if funcNameRoot.IsLeaf() { - funcName = funcNameRoot.Token.Text() + funcName = funcNameRoot.Token.Text(f.File.Bytes) } else { - funcName = funcNameRoot.Children[0].Token.Text() + funcNameRoot.Token.Text() + funcNameRoot.Children[1].Token.Text() + funcName = funcNameRoot.Children[0].Token.Text(f.File.Bytes) + funcNameRoot.Token.Text(f.File.Bytes) + funcNameRoot.Children[1].Token.Text(f.File.Bytes) } isSyscall := funcName == "syscall" diff --git a/src/build/core/CompileCondition.go b/src/build/core/CompileCondition.go index 621a7e7..54e0ab0 100644 --- a/src/build/core/CompileCondition.go +++ b/src/build/core/CompileCondition.go @@ -4,12 +4,13 @@ import ( "fmt" "git.akyoto.dev/cli/q/src/build/expression" + "git.akyoto.dev/cli/q/src/build/token" ) // CompileCondition inserts code to jump to the start label or end label depending on the truth of the condition. func (f *Function) CompileCondition(condition *expression.Expression, successLabel string, failLabel string) error { - switch condition.Token.Text() { - case "||": + switch condition.Token.Kind { + case token.LogicalOr: f.count.subBranch++ leftFailLabel := fmt.Sprintf("%s_false_%d", f.Name, f.count.subBranch) @@ -21,22 +22,22 @@ func (f *Function) CompileCondition(condition *expression.Expression, successLab return err } - f.JumpIfTrue(left.Token.Text(), successLabel) + f.JumpIfTrue(left.Token.Kind, successLabel) // Right f.AddLabel(leftFailLabel) right := condition.Children[1] err = f.CompileCondition(right, successLabel, failLabel) - if condition.Parent != nil && condition.Parent.Token.Text() == "||" && condition != condition.Parent.LastChild() { - f.JumpIfTrue(right.Token.Text(), successLabel) + if condition.Parent != nil && condition.Parent.Token.Kind == token.LogicalOr && condition != condition.Parent.LastChild() { + f.JumpIfTrue(right.Token.Kind, successLabel) } else { - f.JumpIfFalse(right.Token.Text(), failLabel) + f.JumpIfFalse(right.Token.Kind, failLabel) } return err - case "&&": + case token.LogicalAnd: f.count.subBranch++ leftSuccessLabel := fmt.Sprintf("%s_true_%d", f.Name, f.count.subBranch) @@ -48,17 +49,17 @@ func (f *Function) CompileCondition(condition *expression.Expression, successLab return err } - f.JumpIfFalse(left.Token.Text(), failLabel) + f.JumpIfFalse(left.Token.Kind, failLabel) // Right f.AddLabel(leftSuccessLabel) right := condition.Children[1] err = f.CompileCondition(right, successLabel, failLabel) - if condition.Parent != nil && condition.Parent.Token.Text() == "||" && condition != condition.Parent.LastChild() { - f.JumpIfTrue(right.Token.Text(), successLabel) + if condition.Parent != nil && condition.Parent.Token.Kind == token.LogicalOr && condition != condition.Parent.LastChild() { + f.JumpIfTrue(right.Token.Kind, successLabel) } else { - f.JumpIfFalse(right.Token.Text(), failLabel) + f.JumpIfFalse(right.Token.Kind, failLabel) } return err @@ -67,7 +68,7 @@ func (f *Function) CompileCondition(condition *expression.Expression, successLab err := f.Compare(condition) if condition.Parent == nil { - f.JumpIfFalse(condition.Token.Text(), failLabel) + f.JumpIfFalse(condition.Token.Kind, failLabel) } return err diff --git a/src/build/core/CompileDefinition.go b/src/build/core/CompileDefinition.go index a8fa1eb..c5473b1 100644 --- a/src/build/core/CompileDefinition.go +++ b/src/build/core/CompileDefinition.go @@ -8,13 +8,13 @@ import ( // CompileDefinition compiles a variable definition. func (f *Function) CompileDefinition(node *ast.Define) error { - name := node.Name.Text() + name := node.Name.Text(f.File.Bytes) if f.identifierExists(name) { return errors.New(&errors.VariableAlreadyExists{Name: name}, f.File, node.Name.Position) } - uses := token.Count(f.Body, token.Identifier, name) - 1 + uses := token.Count(f.Body, f.File.Bytes, token.Identifier, name) - 1 if uses == 0 { return errors.New(&errors.UnusedVariable{Name: name}, f.File, node.Name.Position) diff --git a/src/build/core/CompileIf.go b/src/build/core/CompileIf.go index 1b61c3c..3eeb4ae 100644 --- a/src/build/core/CompileIf.go +++ b/src/build/core/CompileIf.go @@ -18,7 +18,7 @@ func (f *Function) CompileIf(branch *ast.If) error { } f.AddLabel(success) - f.PushScope(branch.Body) + f.PushScope(branch.Body, f.File.Bytes) err = f.CompileAST(branch.Body) f.PopScope() f.AddLabel(fail) diff --git a/src/build/core/CompileLoop.go b/src/build/core/CompileLoop.go index de9d8b6..a1b116a 100644 --- a/src/build/core/CompileLoop.go +++ b/src/build/core/CompileLoop.go @@ -12,7 +12,7 @@ func (f *Function) CompileLoop(loop *ast.Loop) error { f.count.loop++ label := fmt.Sprintf("%s_loop_%d", f.Name, f.count.loop) f.AddLabel(label) - scope := f.PushScope(loop.Body) + scope := f.PushScope(loop.Body, f.File.Bytes) scope.InLoop = true err := f.CompileAST(loop.Body) f.Jump(asm.JUMP, label) diff --git a/src/build/core/CompileTokens.go b/src/build/core/CompileTokens.go index 7b41804..634c5d7 100644 --- a/src/build/core/CompileTokens.go +++ b/src/build/core/CompileTokens.go @@ -7,8 +7,8 @@ import ( ) // CompileTokens compiles a token list. -func (f *Function) CompileTokens(tokens token.List) error { - body, err := ast.Parse(tokens) +func (f *Function) CompileTokens(tokens []token.Token) error { + body, err := ast.Parse(tokens, f.File.Bytes) if err != nil { err.(*errors.Error).File = f.File diff --git a/src/build/core/ExecuteLeaf.go b/src/build/core/ExecuteLeaf.go index 947518d..d6cc124 100644 --- a/src/build/core/ExecuteLeaf.go +++ b/src/build/core/ExecuteLeaf.go @@ -12,7 +12,7 @@ import ( func (f *Function) ExecuteLeaf(operation token.Token, register cpu.Register, operand token.Token) error { switch operand.Kind { case token.Identifier: - name := operand.Text() + name := operand.Text(f.File.Bytes) variable := f.VariableByName(name) if variable == nil { @@ -23,7 +23,7 @@ func (f *Function) ExecuteLeaf(operation token.Token, register cpu.Register, ope return f.ExecuteRegisterRegister(operation, register, variable.Register) case token.Number: - value := operand.Text() + value := operand.Text(f.File.Bytes) number, err := strconv.Atoi(value) if err != nil { @@ -33,7 +33,7 @@ func (f *Function) ExecuteLeaf(operation token.Token, register cpu.Register, ope return f.ExecuteRegisterNumber(operation, register, number) case token.String: - if operation.Text() == "=" { + if operation.Kind == token.Assign { return f.TokenToRegister(operand, register) } } diff --git a/src/build/core/ExecuteRegisterNumber.go b/src/build/core/ExecuteRegisterNumber.go index 19b07ea..43675a2 100644 --- a/src/build/core/ExecuteRegisterNumber.go +++ b/src/build/core/ExecuteRegisterNumber.go @@ -9,27 +9,27 @@ import ( // ExecuteRegisterNumber performs an operation on a register and a number. func (f *Function) ExecuteRegisterNumber(operation token.Token, register cpu.Register, number int) error { - switch operation.Text() { - case "+", "+=": + switch operation.Kind { + case token.Add, token.AddAssign: f.RegisterNumber(asm.ADD, register, number) - case "-", "-=": + case token.Sub, token.SubAssign: f.RegisterNumber(asm.SUB, register, number) - case "*", "*=": + case token.Mul, token.MulAssign: f.RegisterNumber(asm.MUL, register, number) - case "/", "/=": + case token.Div, token.DivAssign: f.RegisterNumber(asm.DIV, register, number) - case "==", "!=", "<", "<=", ">", ">=": + case token.Equal, token.NotEqual, token.Less, token.LessEqual, token.Greater, token.GreaterEqual: f.RegisterNumber(asm.COMPARE, register, number) - case "=": + case token.Assign: f.RegisterNumber(asm.MOVE, register, number) default: - return errors.New(&errors.InvalidOperator{Operator: operation.Text()}, f.File, operation.Position) + return errors.New(&errors.InvalidOperator{Operator: operation.Text(f.File.Bytes)}, f.File, operation.Position) } return nil diff --git a/src/build/core/ExecuteRegisterRegister.go b/src/build/core/ExecuteRegisterRegister.go index c163e23..8206b4f 100644 --- a/src/build/core/ExecuteRegisterRegister.go +++ b/src/build/core/ExecuteRegisterRegister.go @@ -9,27 +9,27 @@ import ( // ExecuteRegisterRegister performs an operation on two registers. func (f *Function) ExecuteRegisterRegister(operation token.Token, destination cpu.Register, source cpu.Register) error { - switch operation.Text() { - case "+", "+=": + switch operation.Kind { + case token.Add, token.AddAssign: f.RegisterRegister(asm.ADD, destination, source) - case "-", "-=": + case token.Sub, token.SubAssign: f.RegisterRegister(asm.SUB, destination, source) - case "*", "*=": + case token.Mul, token.MulAssign: f.RegisterRegister(asm.MUL, destination, source) - case "/", "/=": + case token.Div, token.DivAssign: f.RegisterRegister(asm.DIV, destination, source) - case "==", "!=", "<", "<=", ">", ">=": + case token.Equal, token.NotEqual, token.Less, token.LessEqual, token.Greater, token.GreaterEqual: f.RegisterRegister(asm.COMPARE, destination, source) - case "=": + case token.Assign: f.RegisterRegister(asm.MOVE, destination, source) default: - return errors.New(&errors.InvalidOperator{Operator: operation.Text()}, f.File, operation.Position) + return errors.New(&errors.InvalidOperator{Operator: operation.Text(f.File.Bytes)}, f.File, operation.Position) } return nil diff --git a/src/build/core/Function.go b/src/build/core/Function.go index 79e1b31..36ac29d 100644 --- a/src/build/core/Function.go +++ b/src/build/core/Function.go @@ -13,7 +13,7 @@ type Function struct { scope.Stack Name string File *fs.File - Body token.List + Body []token.Token Assembler asm.Assembler Functions map[string]*Function Err error diff --git a/src/build/core/JumpIfFalse.go b/src/build/core/JumpIfFalse.go index baedd51..042a676 100644 --- a/src/build/core/JumpIfFalse.go +++ b/src/build/core/JumpIfFalse.go @@ -2,22 +2,23 @@ package core import ( "git.akyoto.dev/cli/q/src/build/asm" + "git.akyoto.dev/cli/q/src/build/token" ) // JumpIfFalse jumps to the label if the previous comparison was false. -func (f *Function) JumpIfFalse(operator string, label string) { +func (f *Function) JumpIfFalse(operator token.Kind, label string) { switch operator { - case "==": + case token.Equal: f.Jump(asm.JNE, label) - case "!=": + case token.NotEqual: f.Jump(asm.JE, label) - case ">": + case token.Greater: f.Jump(asm.JLE, label) - case "<": + case token.Less: f.Jump(asm.JGE, label) - case ">=": + case token.GreaterEqual: f.Jump(asm.JL, label) - case "<=": + case token.LessEqual: f.Jump(asm.JG, label) } } diff --git a/src/build/core/JumpIfTrue.go b/src/build/core/JumpIfTrue.go index 4c04a0b..059618e 100644 --- a/src/build/core/JumpIfTrue.go +++ b/src/build/core/JumpIfTrue.go @@ -2,22 +2,23 @@ package core import ( "git.akyoto.dev/cli/q/src/build/asm" + "git.akyoto.dev/cli/q/src/build/token" ) // JumpIfTrue jumps to the label if the previous comparison was true. -func (f *Function) JumpIfTrue(operator string, label string) { +func (f *Function) JumpIfTrue(operator token.Kind, label string) { switch operator { - case "==": + case token.Equal: f.Jump(asm.JE, label) - case "!=": + case token.NotEqual: f.Jump(asm.JNE, label) - case ">": + case token.Greater: f.Jump(asm.JG, label) - case "<": + case token.Less: f.Jump(asm.JL, label) - case ">=": + case token.GreaterEqual: f.Jump(asm.JGE, label) - case "<=": + case token.LessEqual: f.Jump(asm.JLE, label) } } diff --git a/src/build/core/NewFunction.go b/src/build/core/NewFunction.go index 1394c4e..2dc683d 100644 --- a/src/build/core/NewFunction.go +++ b/src/build/core/NewFunction.go @@ -10,7 +10,7 @@ import ( ) // NewFunction creates a new function. -func NewFunction(name string, file *fs.File, body token.List) *Function { +func NewFunction(name string, file *fs.File, body []token.Token) *Function { return &Function{ Name: name, File: file, diff --git a/src/build/core/TokenToRegister.go b/src/build/core/TokenToRegister.go index a40f471..f048fac 100644 --- a/src/build/core/TokenToRegister.go +++ b/src/build/core/TokenToRegister.go @@ -15,7 +15,7 @@ import ( func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error { switch t.Kind { case token.Identifier: - name := t.Text() + name := t.Text(f.File.Bytes) variable := f.VariableByName(name) if variable == nil { @@ -27,7 +27,7 @@ func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error { return nil case token.Number: - value := t.Text() + value := t.Text(f.File.Bytes) n, err := strconv.Atoi(value) if err != nil { @@ -40,7 +40,7 @@ func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error { case token.String: f.count.data++ label := fmt.Sprintf("%s_data_%d", f.Name, f.count.data) - value := t.Bytes[1 : len(t.Bytes)-1] + value := t.Bytes(f.File.Bytes)[1 : t.Length-1] f.Assembler.SetData(label, value) f.RegisterLabel(asm.MOVE, register, label) return nil diff --git a/src/build/expression/Expression.go b/src/build/expression/Expression.go index 60b3794..5027b8e 100644 --- a/src/build/expression/Expression.go +++ b/src/build/expression/Expression.go @@ -37,11 +37,11 @@ func (expr *Expression) AddChild(child *Expression) { } // Count counts how often the given token appears in the expression. -func (expr *Expression) Count(kind token.Kind, name string) int { +func (expr *Expression) Count(buffer []byte, kind token.Kind, name string) int { count := 0 expr.EachLeaf(func(leaf *Expression) error { - if leaf.Token.Kind == kind && leaf.Token.Text() == name { + if leaf.Token.Kind == kind && leaf.Token.Text(buffer) == name { count++ } @@ -112,25 +112,33 @@ func (expr *Expression) LastChild() *Expression { } // String generates a textual representation of the expression. -func (expr *Expression) String() string { +func (expr *Expression) String(data []byte) string { builder := strings.Builder{} - expr.write(&builder) + expr.write(&builder, data) return builder.String() } // write generates a textual representation of the expression. -func (expr *Expression) write(builder *strings.Builder) { +func (expr *Expression) write(builder *strings.Builder, data []byte) { if expr.IsLeaf() { - builder.WriteString(expr.Token.Text()) + builder.WriteString(expr.Token.Text(data)) return } builder.WriteByte('(') - builder.WriteString(expr.Token.Text()) + + switch expr.Token.Kind { + case token.Call: + builder.WriteString("λ") + case token.Array: + builder.WriteString("@") + default: + builder.WriteString(expr.Token.Text(data)) + } for _, child := range expr.Children { builder.WriteByte(' ') - child.write(builder) + child.write(builder, data) } builder.WriteByte(')') diff --git a/src/build/expression/Expression_test.go b/src/build/expression/Expression_test.go index 6794da8..a313cf3 100644 --- a/src/build/expression/Expression_test.go +++ b/src/build/expression/Expression_test.go @@ -95,7 +95,7 @@ func TestParse(t *testing.T) { defer expr.Reset() assert.NotNil(t, expr) - assert.Equal(t, expr.String(), test.Result) + assert.Equal(t, expr.String(src), test.Result) }) } } @@ -104,11 +104,11 @@ func TestCount(t *testing.T) { src := []byte("(a+b-c*d)+(a*b-c+d)") tokens := token.Tokenize(src) expr := expression.Parse(tokens) - assert.Equal(t, expr.Count(token.Identifier, "a"), 2) - assert.Equal(t, expr.Count(token.Identifier, "b"), 2) - assert.Equal(t, expr.Count(token.Identifier, "c"), 2) - assert.Equal(t, expr.Count(token.Identifier, "d"), 2) - assert.Equal(t, expr.Count(token.Identifier, "e"), 0) + assert.Equal(t, expr.Count(src, token.Identifier, "a"), 2) + assert.Equal(t, expr.Count(src, token.Identifier, "b"), 2) + assert.Equal(t, expr.Count(src, token.Identifier, "c"), 2) + assert.Equal(t, expr.Count(src, token.Identifier, "d"), 2) + assert.Equal(t, expr.Count(src, token.Identifier, "e"), 0) } func TestEachLeaf(t *testing.T) { @@ -118,7 +118,7 @@ func TestEachLeaf(t *testing.T) { leaves := []string{} err := expr.EachLeaf(func(leaf *expression.Expression) error { - leaves = append(leaves, leaf.Token.Text()) + leaves = append(leaves, leaf.Token.Text(src)) return nil }) @@ -140,7 +140,7 @@ func TestEachParameter(t *testing.T) { err := expression.EachParameter(tokens, func(parameter token.List) error { expr := expression.Parse(parameter) - parameters = append(parameters, expr.String()) + parameters = append(parameters, expr.String(src)) return nil }) @@ -178,17 +178,3 @@ func TestNilGroup(t *testing.T) { expr := expression.Parse(tokens) assert.Nil(t, expr) } - -func TestInvalidOperator(t *testing.T) { - src := []byte("a +++ 2") - tokens := token.Tokenize(src) - expr := expression.Parse(tokens) - assert.Equal(t, expr.String(), "(+++ a 2)") -} - -func TestInvalidOperatorCall(t *testing.T) { - src := []byte("+++()") - tokens := token.Tokenize(src) - expr := expression.Parse(tokens) - assert.NotNil(t, expr) -} diff --git a/src/build/expression/Operator.go b/src/build/expression/Operator.go index b2d15e1..1d9e18b 100644 --- a/src/build/expression/Operator.go +++ b/src/build/expression/Operator.go @@ -15,39 +15,39 @@ type Operator struct { // Operators defines the operators used in the language. // The number corresponds to the operator priority and can not be zero. -var Operators = map[string]*Operator{ - ".": {".", 13, 2}, - "λ": {"λ", 12, 1}, - "@": {"@", 12, 2}, - "!": {"!", 11, 1}, - "*": {"*", 10, 2}, - "/": {"/", 10, 2}, - "%": {"%", 10, 2}, - "+": {"+", 9, 2}, - "-": {"-", 9, 2}, - ">>": {">>", 8, 2}, - "<<": {"<<", 8, 2}, - "&": {"&", 7, 2}, - "^": {"^", 6, 2}, - "|": {"|", 5, 2}, +var Operators = map[token.Kind]*Operator{ + token.Period: {".", 13, 2}, + token.Call: {"λ", 12, 1}, + token.Array: {"@", 12, 2}, + token.Not: {"!", 11, 1}, + token.Mul: {"*", 10, 2}, + token.Div: {"/", 10, 2}, + token.Mod: {"%", 10, 2}, + token.Add: {"+", 9, 2}, + token.Sub: {"-", 9, 2}, + token.Shr: {">>", 8, 2}, + token.Shl: {"<<", 8, 2}, + token.And: {"&", 7, 2}, + token.Xor: {"^", 6, 2}, + token.Or: {"|", 5, 2}, - ">": {">", 4, 2}, - "<": {"<", 4, 2}, - ">=": {">=", 4, 2}, - "<=": {"<=", 4, 2}, - "==": {"==", 3, 2}, - "!=": {"!=", 3, 2}, - "&&": {"&&", 2, 2}, - "||": {"||", 1, 2}, + token.Greater: {">", 4, 2}, + token.Less: {"<", 4, 2}, + token.GreaterEqual: {">=", 4, 2}, + token.LessEqual: {"<=", 4, 2}, + token.Equal: {"==", 3, 2}, + token.NotEqual: {"!=", 3, 2}, + token.LogicalAnd: {"&&", 2, 2}, + token.LogicalOr: {"||", 1, 2}, - "=": {"=", math.MinInt8, 2}, - ":=": {":=", math.MinInt8, 2}, - "+=": {"+=", math.MinInt8, 2}, - "-=": {"-=", math.MinInt8, 2}, - "*=": {"*=", math.MinInt8, 2}, - "/=": {"/=", math.MinInt8, 2}, - ">>=": {">>=", math.MinInt8, 2}, - "<<=": {"<<=", math.MinInt8, 2}, + token.Assign: {"=", math.MinInt8, 2}, + token.Define: {":=", math.MinInt8, 2}, + token.AddAssign: {"+=", math.MinInt8, 2}, + token.SubAssign: {"-=", math.MinInt8, 2}, + token.MulAssign: {"*=", math.MinInt8, 2}, + token.DivAssign: {"/=", math.MinInt8, 2}, + token.ShrAssign: {">>=", math.MinInt8, 2}, + token.ShlAssign: {"<<=", math.MinInt8, 2}, } func isComplete(expr *Expression) bool { @@ -59,14 +59,14 @@ func isComplete(expr *Expression) bool { return true } - if expr.Token.Kind == token.Operator && len(expr.Children) == numOperands(expr.Token.Text()) { + if expr.Token.IsOperator() && len(expr.Children) == numOperands(expr.Token.Kind) { return true } return false } -func numOperands(symbol string) int { +func numOperands(symbol token.Kind) int { operator, exists := Operators[symbol] if !exists { @@ -76,7 +76,7 @@ func numOperands(symbol string) int { return operator.Operands } -func precedence(symbol string) int8 { +func precedence(symbol token.Kind) int8 { operator, exists := Operators[symbol] if !exists { diff --git a/src/build/expression/Parse.go b/src/build/expression/Parse.go index 33dd062..f2e8b56 100644 --- a/src/build/expression/Parse.go +++ b/src/build/expression/Parse.go @@ -6,13 +6,8 @@ import ( "git.akyoto.dev/cli/q/src/build/token" ) -var ( - call = []byte("λ") - array = []byte("@") -) - // Parse generates an expression tree from tokens. -func Parse(tokens token.List) *Expression { +func Parse(tokens []token.Token) *Expression { var ( cursor *Expression root *Expression @@ -43,20 +38,18 @@ func Parse(tokens token.List) *Expression { parameters := NewList(tokens[groupPosition:i]) node := New() - node.Token.Kind = token.Operator node.Token.Position = tokens[groupPosition].Position switch t.Kind { case token.GroupEnd: - node.Token.Bytes = call - node.Precedence = precedence("λ") - + node.Token.Kind = token.Call case token.ArrayEnd: - node.Token.Bytes = array - node.Precedence = precedence("@") + node.Token.Kind = token.Array } - if cursor.Token.Kind == token.Operator && node.Precedence > cursor.Precedence { + node.Precedence = precedence(node.Token.Kind) + + if cursor.Token.IsOperator() && node.Precedence > cursor.Precedence { cursor.LastChild().Replace(node) } else { if cursor == root { @@ -108,18 +101,18 @@ func Parse(tokens token.List) *Expression { continue } - if t.Kind == token.Operator { + if t.IsOperator() { if cursor == nil { cursor = NewLeaf(t) - cursor.Precedence = precedence(t.Text()) + cursor.Precedence = precedence(t.Kind) root = cursor continue } node := NewLeaf(t) - node.Precedence = precedence(t.Text()) + node.Precedence = precedence(t.Kind) - if cursor.Token.Kind == token.Operator { + if cursor.Token.IsOperator() { oldPrecedence := cursor.Precedence newPrecedence := node.Precedence diff --git a/src/build/fs/File.go b/src/build/fs/File.go index d4fd1cf..1a10a1d 100644 --- a/src/build/fs/File.go +++ b/src/build/fs/File.go @@ -5,5 +5,6 @@ import "git.akyoto.dev/cli/q/src/build/token" // File represents a single source file. type File struct { Path string + Bytes []byte Tokens token.List } diff --git a/src/build/keyword/Keyword.go b/src/build/keyword/Keyword.go deleted file mode 100644 index ef62575..0000000 --- a/src/build/keyword/Keyword.go +++ /dev/null @@ -1,16 +0,0 @@ -package keyword - -const ( - If = "if" - Import = "import" - Loop = "loop" - Return = "return" -) - -// Map is a map of all keywords used in the language. -var Map = map[string][]byte{ - If: []byte(If), - Import: []byte(Import), - Loop: []byte(Loop), - Return: []byte(Return), -} diff --git a/src/build/scanner/scanFile.go b/src/build/scanner/scanFile.go index b075fc6..e724767 100644 --- a/src/build/scanner/scanFile.go +++ b/src/build/scanner/scanFile.go @@ -11,7 +11,6 @@ import ( "git.akyoto.dev/cli/q/src/build/errors" "git.akyoto.dev/cli/q/src/build/expression" "git.akyoto.dev/cli/q/src/build/fs" - "git.akyoto.dev/cli/q/src/build/keyword" "git.akyoto.dev/cli/q/src/build/scope" "git.akyoto.dev/cli/q/src/build/token" ) @@ -27,8 +26,9 @@ func (s *Scanner) scanFile(path string, pkg string) error { tokens := token.Tokenize(contents) file := &fs.File{ - Tokens: tokens, Path: path, + Bytes: contents, + Tokens: tokens, } var ( @@ -42,14 +42,14 @@ func (s *Scanner) scanFile(path string, pkg string) error { ) for { - for i < len(tokens) && tokens[i].Kind == token.Keyword && tokens[i].Text() == keyword.Import { + for i < len(tokens) && tokens[i].Kind == token.Import { i++ if tokens[i].Kind != token.Identifier { panic("expected package name") } - packageName := tokens[i].Text() + packageName := tokens[i].Text(contents) s.queueDirectory(filepath.Join(config.Library, packageName), packageName) i++ @@ -75,7 +75,7 @@ func (s *Scanner) scanFile(path string, pkg string) error { } if tokens[i].Kind == token.Invalid { - return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position) + return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text(contents)}, file, tokens[i].Position) } if tokens[i].Kind == token.EOF { @@ -116,7 +116,7 @@ func (s *Scanner) scanFile(path string, pkg string) error { } if tokens[i].Kind == token.Invalid { - return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position) + return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text(contents)}, file, tokens[i].Position) } if tokens[i].Kind == token.EOF { @@ -168,7 +168,7 @@ func (s *Scanner) scanFile(path string, pkg string) error { } if tokens[i].Kind == token.Invalid { - return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position) + return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text(contents)}, file, tokens[i].Position) } if tokens[i].Kind == token.EOF { @@ -191,7 +191,7 @@ func (s *Scanner) scanFile(path string, pkg string) error { return errors.New(errors.ExpectedFunctionDefinition, file, tokens[i].Position) } - name := tokens[nameStart].Text() + name := tokens[nameStart].Text(contents) body := tokens[bodyStart:i] if pkg != "" { @@ -207,9 +207,9 @@ func (s *Scanner) scanFile(path string, pkg string) error { return errors.New(errors.NotImplemented, file, tokens[0].Position) } - name := tokens[0].Text() + name := tokens[0].Text(contents) register := x64.CallRegisters[count] - uses := token.Count(function.Body, token.Identifier, name) + uses := token.Count(function.Body, contents, token.Identifier, name) if uses == 0 { return errors.New(&errors.UnusedVariable{Name: name}, file, tokens[0].Position) diff --git a/src/build/scope/Stack.go b/src/build/scope/Stack.go index 5f634ff..1760f6b 100644 --- a/src/build/scope/Stack.go +++ b/src/build/scope/Stack.go @@ -20,7 +20,7 @@ func (stack *Stack) PopScope() { } // PushScope pushes a new scope to the top of the stack. -func (stack *Stack) PushScope(body ast.AST) *Scope { +func (stack *Stack) PushScope(body ast.AST, buffer []byte) *Scope { s := &Scope{} if len(stack.Scopes) > 0 { @@ -30,7 +30,7 @@ func (stack *Stack) PushScope(body ast.AST) *Scope { s.InLoop = lastScope.InLoop for k, v := range lastScope.Variables { - count := ast.Count(body, token.Identifier, v.Name) + count := ast.Count(body, buffer, token.Identifier, v.Name) if count == 0 { continue diff --git a/src/build/token/Count.go b/src/build/token/Count.go index ae83143..e83fff6 100644 --- a/src/build/token/Count.go +++ b/src/build/token/Count.go @@ -1,11 +1,11 @@ package token // Count counts how often the given token appears in the token list. -func Count(tokens List, kind Kind, name string) int { +func Count(tokens []Token, buffer []byte, kind Kind, name string) int { count := 0 for _, t := range tokens { - if t.Kind == Identifier && t.Text() == name { + if t.Kind == Identifier && t.Text(buffer) == name { count++ } } diff --git a/src/build/token/Count_test.go b/src/build/token/Count_test.go index 0128e62..8742d13 100644 --- a/src/build/token/Count_test.go +++ b/src/build/token/Count_test.go @@ -8,9 +8,10 @@ import ( ) func TestCount(t *testing.T) { - tokens := token.Tokenize([]byte(`a b b c c c`)) - assert.Equal(t, token.Count(tokens, token.Identifier, "a"), 1) - assert.Equal(t, token.Count(tokens, token.Identifier, "b"), 2) - assert.Equal(t, token.Count(tokens, token.Identifier, "c"), 3) - assert.Equal(t, token.Count(tokens, token.Identifier, "d"), 0) + buffer := []byte(`a b b c c c`) + tokens := token.Tokenize(buffer) + assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "a"), 1) + assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "b"), 2) + assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "c"), 3) + assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "d"), 0) } diff --git a/src/build/token/Keywords.go b/src/build/token/Keywords.go new file mode 100644 index 0000000..c75fd75 --- /dev/null +++ b/src/build/token/Keywords.go @@ -0,0 +1,9 @@ +package token + +// Keywords is a map of all keywords used in the language. +var Keywords = map[string]Kind{ + "if": If, + "import": Import, + "loop": Loop, + "return": Return, +} diff --git a/src/build/token/Kind.go b/src/build/token/Kind.go index b221442..1df7adf 100644 --- a/src/build/token/Kind.go +++ b/src/build/token/Kind.go @@ -4,73 +4,62 @@ package token type Kind uint8 const ( - // Invalid represents an invalid token. - Invalid Kind = iota - - // EOF represents the end of file. - EOF - - // NewLine represents the newline character. - NewLine - - // Identifier represents a series of characters used to identify a variable or function. - Identifier - - // Keyword represents a language keyword. - Keyword - - // String represents an uninterpreted series of characters in the source code. - String - - // Number represents a series of numerical characters. - Number - - // Operator represents a mathematical operator. - Operator - - // Separator represents a comma. - Separator - - // Comment represents a comment. - Comment - - // GroupStart represents '('. - GroupStart - - // GroupEnd represents ')'. - GroupEnd - - // BlockStart represents '{'. - BlockStart - - // BlockEnd represents '}'. - BlockEnd - - // ArrayStart represents '['. - ArrayStart - - // ArrayEnd represents ']'. - ArrayEnd + Invalid Kind = iota // Invalid is an invalid token. + EOF // EOF is the end of file. + NewLine // NewLine is the newline character. + Identifier // Identifier is a series of characters used to identify a variable or function. + Number // Number is a series of numerical characters. + String // String is an uninterpreted series of characters in the source code. + Comment // Comment is a comment. + Separator // , + GroupStart // ( + GroupEnd // ) + BlockStart // { + BlockEnd // } + ArrayStart // [ + ArrayEnd // ] + _keywords // + If // if + Import // import + Loop // loop + Return // return + _keywordsEnd // + _operators // + Add // + + Sub // - + Mul // * + Div // / + Mod // % + And // & + Or // | + Xor // ^ + Shl // << + Shr // >> + LogicalAnd // && + LogicalOr // || + Equal // == + Less // < + Greater // > + Not // ! + NotEqual // != + LessEqual // <= + GreaterEqual // >= + Define // := + Period // . + Call // x() + Array // [x] + _assignments // + Assign // = + AddAssign // += + SubAssign // -= + MulAssign // *= + DivAssign // /= + ModAssign // %= + AndAssign // &= + OrAssign // |= + XorAssign // ^= + ShlAssign // <<= + ShrAssign // >>= + _assignmentsEnd // + _operatorsEnd // ) - -// String returns the text representation. -func (kind Kind) String() string { - return [...]string{ - "Invalid", - "EOF", - "NewLine", - "Identifier", - "Keyword", - "String", - "Number", - "Operator", - "Separator", - "Comment", - "GroupStart", - "GroupEnd", - "BlockStart", - "BlockEnd", - "ArrayStart", - "ArrayEnd", - }[kind] -} diff --git a/src/build/token/Kind_test.go b/src/build/token/Kind_test.go deleted file mode 100644 index 1db0382..0000000 --- a/src/build/token/Kind_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package token_test - -import ( - "testing" - - "git.akyoto.dev/cli/q/src/build/token" - "git.akyoto.dev/go/assert" -) - -func TestTokenKind(t *testing.T) { - assert.Equal(t, token.Invalid.String(), "Invalid") - assert.Equal(t, token.EOF.String(), "EOF") - assert.Equal(t, token.NewLine.String(), "NewLine") - assert.Equal(t, token.Identifier.String(), "Identifier") - assert.Equal(t, token.Keyword.String(), "Keyword") - assert.Equal(t, token.String.String(), "String") - assert.Equal(t, token.Number.String(), "Number") - assert.Equal(t, token.Operator.String(), "Operator") - assert.Equal(t, token.Separator.String(), "Separator") - assert.Equal(t, token.Comment.String(), "Comment") - assert.Equal(t, token.GroupStart.String(), "GroupStart") - assert.Equal(t, token.GroupEnd.String(), "GroupEnd") - assert.Equal(t, token.BlockStart.String(), "BlockStart") - assert.Equal(t, token.BlockEnd.String(), "BlockEnd") - assert.Equal(t, token.ArrayStart.String(), "ArrayStart") - assert.Equal(t, token.ArrayEnd.String(), "ArrayEnd") -} diff --git a/src/build/token/Length.go b/src/build/token/Length.go new file mode 100644 index 0000000..6711706 --- /dev/null +++ b/src/build/token/Length.go @@ -0,0 +1,4 @@ +package token + +// Length is the data type for storing token lengths. +type Length = uint16 diff --git a/src/build/token/List.go b/src/build/token/List.go index 1b725b6..7e57693 100644 --- a/src/build/token/List.go +++ b/src/build/token/List.go @@ -1,9 +1,5 @@ package token -import ( - "bytes" -) - // List is a slice of tokens. type List []Token @@ -28,20 +24,3 @@ func (list List) LastIndexKind(kind Kind) int { return -1 } - -// String implements string serialization. -func (list List) String() string { - builder := bytes.Buffer{} - var last Token - - for _, t := range list { - if last.Kind == Keyword || last.Kind == Separator || last.Kind == Operator || t.Kind == Operator { - builder.WriteByte(' ') - } - - builder.Write(t.Bytes) - last = t - } - - return builder.String() -} diff --git a/src/build/token/Operators.go b/src/build/token/Operators.go new file mode 100644 index 0000000..842fb94 --- /dev/null +++ b/src/build/token/Operators.go @@ -0,0 +1,39 @@ +package token + +// Operators is a map of all operators used in the language. +var Operators = map[string]Kind{ + ".": Period, + "=": Assign, + ":=": Define, + "+": Add, + "-": Sub, + "*": Mul, + "/": Div, + "%": Mod, + "&": And, + "|": Or, + "^": Xor, + "<<": Shl, + ">>": Shr, + "&&": LogicalAnd, + "||": LogicalOr, + "!": Not, + "==": Equal, + "!=": NotEqual, + ">": Greater, + "<": Less, + ">=": GreaterEqual, + "<=": LessEqual, + "+=": AddAssign, + "-=": SubAssign, + "*=": MulAssign, + "/=": DivAssign, + "%=": ModAssign, + "&=": AndAssign, + "|=": OrAssign, + "^=": XorAssign, + "<<=": ShlAssign, + ">>=": ShrAssign, + "λ": Call, + "@": Array, +} diff --git a/src/build/token/Token.go b/src/build/token/Token.go index 71dd1e5..134f054 100644 --- a/src/build/token/Token.go +++ b/src/build/token/Token.go @@ -1,7 +1,6 @@ package token import ( - "fmt" "unsafe" ) @@ -9,29 +8,44 @@ import ( // The characters that make up an identifier are grouped into a single token. // This makes parsing easier and allows us to do better syntax checks. type Token struct { - Bytes []byte Position Position + Length Length Kind Kind } -// End returns the position after the token. -func (t *Token) End() Position { - return t.Position + Position(len(t.Bytes)) +// Bytes returns the byte slice. +func (t Token) Bytes(buffer []byte) []byte { + return buffer[t.Position : t.Position+Position(t.Length)] } -// String creates a human readable representation for debugging purposes. -func (t *Token) String() string { - return fmt.Sprintf("%s %s", t.Kind, t.Text()) +// End returns the position after the token. +func (t Token) End() Position { + return t.Position + Position(t.Length) +} + +// IsAssignment returns true if the token is an assignment operator. +func (t Token) IsAssignment() bool { + return t.Kind > _assignments && t.Kind < _assignmentsEnd +} + +// IsKeyword returns true if the token is a keyword. +func (t Token) IsKeyword() bool { + return t.Kind > _keywords && t.Kind < _keywordsEnd +} + +// IsOperator returns true if the token is an operator. +func (t Token) IsOperator() bool { + return t.Kind > _operators && t.Kind < _operatorsEnd } // Reset resets the token to default values. func (t *Token) Reset() { - t.Kind = Invalid t.Position = 0 - t.Bytes = nil + t.Length = 0 + t.Kind = Invalid } // Text returns the token text. -func (t *Token) Text() string { - return unsafe.String(unsafe.SliceData(t.Bytes), len(t.Bytes)) +func (t Token) Text(buffer []byte) string { + return unsafe.String(unsafe.SliceData(t.Bytes(buffer)), t.Length) } diff --git a/src/build/token/Token_test.go b/src/build/token/Token_test.go index 47fc665..ba825cc 100644 --- a/src/build/token/Token_test.go +++ b/src/build/token/Token_test.go @@ -10,8 +10,8 @@ import ( func TestTokenEnd(t *testing.T) { hello := token.Token{ Kind: token.Identifier, - Bytes: []byte("hello"), Position: 0, + Length: 5, } assert.Equal(t, hello.End(), 5) @@ -20,34 +20,33 @@ func TestTokenEnd(t *testing.T) { func TestTokenReset(t *testing.T) { hello := token.Token{ Kind: token.Identifier, - Bytes: []byte("hello"), Position: 1, + Length: 5, } hello.Reset() - assert.Nil(t, hello.Bytes) assert.Equal(t, hello.Position, 0) + assert.Equal(t, hello.Length, 0) assert.Equal(t, hello.Kind, token.Invalid) } -func TestTokenString(t *testing.T) { - hello := token.Token{ - Kind: token.Identifier, - Bytes: []byte("hello"), - Position: 0, - } - - assert.Equal(t, hello.String(), "Identifier hello") -} - func TestTokenText(t *testing.T) { - hello := token.Token{Kind: token.Identifier, Bytes: []byte("hello"), Position: 0} - comma := token.Token{Kind: token.Separator, Bytes: []byte(","), Position: 5} - world := token.Token{Kind: token.Identifier, Bytes: []byte("world"), Position: 7} + buffer := []byte("hello, world") + hello := token.Token{Kind: token.Identifier, Position: 0, Length: 5} + comma := token.Token{Kind: token.Separator, Position: 5, Length: 1} + world := token.Token{Kind: token.Identifier, Position: 7, Length: 5} - assert.Equal(t, hello.Text(), "hello") - assert.Equal(t, world.Text(), "world") - - list := token.List{hello, comma, world} - assert.Equal(t, list.String(), "hello, world") + assert.Equal(t, hello.Text(buffer), "hello") + assert.Equal(t, comma.Text(buffer), ",") + assert.Equal(t, world.Text(buffer), "world") +} + +func TestTokenGroups(t *testing.T) { + assignment := token.Token{Kind: token.Assign} + operator := token.Token{Kind: token.Add} + keyword := token.Token{Kind: token.If} + + assert.True(t, assignment.IsAssignment()) + assert.True(t, operator.IsOperator()) + assert.True(t, keyword.IsKeyword()) } diff --git a/src/build/token/Tokenize.go b/src/build/token/Tokenize.go index 649f376..0060958 100644 --- a/src/build/token/Tokenize.go +++ b/src/build/token/Tokenize.go @@ -1,20 +1,5 @@ package token -import "git.akyoto.dev/cli/q/src/build/keyword" - -// Pre-allocate these byte buffers so we can re-use them -// instead of allocating a new buffer every time. -var ( - groupStartBytes = []byte{'('} - groupEndBytes = []byte{')'} - blockStartBytes = []byte{'{'} - blockEndBytes = []byte{'}'} - arrayStartBytes = []byte{'['} - arrayEndBytes = []byte{']'} - separatorBytes = []byte{','} - newLineBytes = []byte{'\n'} -) - // Tokenize turns the file contents into a list of tokens. func Tokenize(buffer []byte) List { var ( @@ -26,21 +11,21 @@ func Tokenize(buffer []byte) List { switch buffer[i] { case ' ', '\t': case ',': - tokens = append(tokens, Token{Kind: Separator, Position: i, Bytes: separatorBytes}) + tokens = append(tokens, Token{Kind: Separator, Position: i, Length: 1}) case '(': - tokens = append(tokens, Token{Kind: GroupStart, Position: i, Bytes: groupStartBytes}) + tokens = append(tokens, Token{Kind: GroupStart, Position: i, Length: 1}) case ')': - tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Bytes: groupEndBytes}) + tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Length: 1}) case '{': - tokens = append(tokens, Token{Kind: BlockStart, Position: i, Bytes: blockStartBytes}) + tokens = append(tokens, Token{Kind: BlockStart, Position: i, Length: 1}) case '}': - tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Bytes: blockEndBytes}) + tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Length: 1}) case '[': - tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Bytes: arrayStartBytes}) + tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Length: 1}) case ']': - tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Bytes: arrayEndBytes}) + tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Length: 1}) case '\n': - tokens = append(tokens, Token{Kind: NewLine, Position: i, Bytes: newLineBytes}) + tokens = append(tokens, Token{Kind: NewLine, Position: i, Length: 1}) case '/': if i+1 >= Position(len(buffer)) || buffer[i+1] != '/' { position := i @@ -50,7 +35,7 @@ func Tokenize(buffer []byte) List { i++ } - tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]}) + tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)}) } else { position := i @@ -58,7 +43,7 @@ func Tokenize(buffer []byte) List { i++ } - tokens = append(tokens, Token{Kind: Comment, Position: position, Bytes: buffer[position:i]}) + tokens = append(tokens, Token{Kind: Comment, Position: position, Length: Length(i - position)}) } continue @@ -78,7 +63,7 @@ func Tokenize(buffer []byte) List { i++ } - tokens = append(tokens, Token{Kind: String, Position: start, Bytes: buffer[start:end]}) + tokens = append(tokens, Token{Kind: String, Position: start, Length: Length(end - start)}) continue default: @@ -91,14 +76,14 @@ func Tokenize(buffer []byte) List { } identifier := buffer[position:i] - keyword, isKeyword := keyword.Map[string(identifier)] + kind := Identifier + keyword, isKeyword := Keywords[string(identifier)] if isKeyword { - tokens = append(tokens, Token{Kind: Keyword, Position: position, Bytes: keyword}) - } else { - tokens = append(tokens, Token{Kind: Identifier, Position: position, Bytes: identifier}) + kind = keyword } + tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(len(identifier))}) continue } @@ -110,7 +95,7 @@ func Tokenize(buffer []byte) List { i++ } - tokens = append(tokens, Token{Kind: Number, Position: position, Bytes: buffer[position:i]}) + tokens = append(tokens, Token{Kind: Number, Position: position, Length: Length(i - position)}) continue } @@ -122,17 +107,17 @@ func Tokenize(buffer []byte) List { i++ } - tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]}) + tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)}) continue } - tokens = append(tokens, Token{Kind: Invalid, Position: i, Bytes: buffer[i : i+1]}) + tokens = append(tokens, Token{Kind: Invalid, Position: i, Length: 1}) } i++ } - tokens = append(tokens, Token{Kind: EOF, Position: i, Bytes: nil}) + tokens = append(tokens, Token{Kind: EOF, Position: i, Length: 0}) return tokens } diff --git a/src/build/token/Tokenize_test.go b/src/build/token/Tokenize_test.go index 63e689d..19f57e5 100644 --- a/src/build/token/Tokenize_test.go +++ b/src/build/token/Tokenize_test.go @@ -9,394 +9,243 @@ import ( func TestFunction(t *testing.T) { tokens := token.Tokenize([]byte("main(){}")) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.Identifier, - Bytes: []byte("main"), - Position: 0, - }, - { - Kind: token.GroupStart, - Bytes: []byte("("), - Position: 4, - }, - { - Kind: token.GroupEnd, - Bytes: []byte(")"), - Position: 5, - }, - { - Kind: token.BlockStart, - Bytes: []byte("{"), - Position: 6, - }, - { - Kind: token.BlockEnd, - Bytes: []byte("}"), - Position: 7, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 8, - }, - }) + + expected := []token.Kind{ + token.Identifier, + token.GroupStart, + token.GroupEnd, + token.BlockStart, + token.BlockEnd, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } } func TestKeyword(t *testing.T) { tokens := token.Tokenize([]byte("return x")) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.Keyword, - Bytes: []byte("return"), - Position: 0, - }, - { - Kind: token.Identifier, - Bytes: []byte("x"), - Position: 7, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 8, - }, - }) + + expected := []token.Kind{ + token.Return, + token.Identifier, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } } func TestArray(t *testing.T) { tokens := token.Tokenize([]byte("array[i]")) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.Identifier, - Bytes: []byte("array"), - Position: 0, - }, - { - Kind: token.ArrayStart, - Bytes: []byte("["), - Position: 5, - }, - { - Kind: token.Identifier, - Bytes: []byte("i"), - Position: 6, - }, - { - Kind: token.ArrayEnd, - Bytes: []byte("]"), - Position: 7, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 8, - }, - }) + + expected := []token.Kind{ + token.Identifier, + token.ArrayStart, + token.Identifier, + token.ArrayEnd, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } } func TestNewline(t *testing.T) { tokens := token.Tokenize([]byte("\n\n")) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.NewLine, - Bytes: []byte("\n"), - Position: 0, - }, - { - Kind: token.NewLine, - Bytes: []byte("\n"), - Position: 1, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 2, - }, - }) + + expected := []token.Kind{ + token.NewLine, + token.NewLine, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } } func TestNumber(t *testing.T) { tokens := token.Tokenize([]byte(`123 456`)) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.Number, - Bytes: []byte("123"), - Position: 0, - }, - { - Kind: token.Number, - Bytes: []byte("456"), - Position: 4, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 7, - }, - }) + + expected := []token.Kind{ + token.Number, + token.Number, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } } func TestOperator(t *testing.T) { tokens := token.Tokenize([]byte(`+ - * /`)) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.Operator, - Bytes: []byte("+"), - Position: 0, - }, - { - Kind: token.Operator, - Bytes: []byte("-"), - Position: 2, - }, - { - Kind: token.Operator, - Bytes: []byte("*"), - Position: 4, - }, - { - Kind: token.Operator, - Bytes: []byte("/"), - Position: 6, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 7, - }, - }) + + expected := []token.Kind{ + token.Add, + token.Sub, + token.Mul, + token.Div, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } } func TestOperatorAssign(t *testing.T) { - tokens := token.Tokenize([]byte(`+= -= *= /= ==`)) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.Operator, - Bytes: []byte("+="), - Position: 0, - }, - { - Kind: token.Operator, - Bytes: []byte("-="), - Position: 3, - }, - { - Kind: token.Operator, - Bytes: []byte("*="), - Position: 6, - }, - { - Kind: token.Operator, - Bytes: []byte("/="), - Position: 9, - }, - { - Kind: token.Operator, - Bytes: []byte("=="), - Position: 12, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 14, - }, - }) + tokens := token.Tokenize([]byte(`+= -= *= /= &= |= ^= <<= >>=`)) + + expected := []token.Kind{ + token.AddAssign, + token.SubAssign, + token.MulAssign, + token.DivAssign, + token.AndAssign, + token.OrAssign, + token.XorAssign, + token.ShlAssign, + token.ShrAssign, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } } func TestSeparator(t *testing.T) { tokens := token.Tokenize([]byte("a,b,c")) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.Identifier, - Bytes: []byte("a"), - Position: 0, - }, - { - Kind: token.Separator, - Bytes: []byte(","), - Position: 1, - }, - { - Kind: token.Identifier, - Bytes: []byte("b"), - Position: 2, - }, - { - Kind: token.Separator, - Bytes: []byte(","), - Position: 3, - }, - { - Kind: token.Identifier, - Bytes: []byte("c"), - Position: 4, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 5, - }, - }) + + expected := []token.Kind{ + token.Identifier, + token.Separator, + token.Identifier, + token.Separator, + token.Identifier, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } } func TestComment(t *testing.T) { tokens := token.Tokenize([]byte("// Hello\n// World")) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.Comment, - Bytes: []byte(`// Hello`), - Position: 0, - }, - { - Kind: token.NewLine, - Bytes: []byte("\n"), - Position: 8, - }, - { - Kind: token.Comment, - Bytes: []byte(`// World`), - Position: 9, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 17, - }, - }) + + expected := []token.Kind{ + token.Comment, + token.NewLine, + token.Comment, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } tokens = token.Tokenize([]byte("// Hello\n")) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.Comment, - Bytes: []byte(`// Hello`), - Position: 0, - }, - { - Kind: token.NewLine, - Bytes: []byte("\n"), - Position: 8, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 9, - }, - }) + + expected = []token.Kind{ + token.Comment, + token.NewLine, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } tokens = token.Tokenize([]byte(`// Hello`)) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.Comment, - Bytes: []byte(`// Hello`), - Position: 0, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 8, - }, - }) + + expected = []token.Kind{ + token.Comment, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } tokens = token.Tokenize([]byte(`//`)) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.Comment, - Bytes: []byte(`//`), - Position: 0, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 2, - }, - }) + + expected = []token.Kind{ + token.Comment, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } tokens = token.Tokenize([]byte(`/`)) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.Operator, - Bytes: []byte(`/`), - Position: 0, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 1, - }, - }) + + expected = []token.Kind{ + token.Div, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } } func TestInvalid(t *testing.T) { - tokens := token.Tokenize([]byte(`@#`)) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.Invalid, - Bytes: []byte(`@`), - Position: 0, - }, - { - Kind: token.Invalid, - Bytes: []byte(`#`), - Position: 1, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 2, - }, - }) + tokens := token.Tokenize([]byte(`##`)) + + expected := []token.Kind{ + token.Invalid, + token.Invalid, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } } func TestString(t *testing.T) { tokens := token.Tokenize([]byte(`"Hello" "World"`)) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.String, - Bytes: []byte(`"Hello"`), - Position: 0, - }, - { - Kind: token.String, - Bytes: []byte(`"World"`), - Position: 8, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 15, - }, - }) + + expected := []token.Kind{ + token.String, + token.String, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } } func TestStringMultiline(t *testing.T) { tokens := token.Tokenize([]byte("\"Hello\nWorld\"")) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.String, - Bytes: []byte("\"Hello\nWorld\""), - Position: 0, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 13, - }, - }) + + expected := []token.Kind{ + token.String, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } } func TestStringEOF(t *testing.T) { tokens := token.Tokenize([]byte(`"EOF`)) - assert.DeepEqual(t, tokens, token.List{ - { - Kind: token.String, - Bytes: []byte(`"EOF`), - Position: 0, - }, - { - Kind: token.EOF, - Bytes: nil, - Position: 4, - }, - }) + + expected := []token.Kind{ + token.String, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } } diff --git a/tests/errors/InvalidOperator.q b/tests/errors/InvalidCharacter4.q similarity index 100% rename from tests/errors/InvalidOperator.q rename to tests/errors/InvalidCharacter4.q diff --git a/tests/errors_test.go b/tests/errors_test.go index 6c08acd..5b28ee2 100644 --- a/tests/errors_test.go +++ b/tests/errors_test.go @@ -21,10 +21,10 @@ var errs = []struct { {"InvalidInstructionNumber.q", &errors.InvalidInstruction{Instruction: "123"}}, {"InvalidInstructionExpression.q", &errors.InvalidInstruction{Instruction: "+"}}, {"InvalidExpression.q", errors.InvalidExpression}, - {"InvalidOperator.q", &errors.InvalidOperator{Operator: "+++"}}, {"InvalidCharacter.q", &errors.InvalidCharacter{Character: "@"}}, {"InvalidCharacter2.q", &errors.InvalidCharacter{Character: "@"}}, {"InvalidCharacter3.q", &errors.InvalidCharacter{Character: "@"}}, + {"InvalidCharacter4.q", &errors.InvalidCharacter{Character: "+++"}}, {"MissingBlockEnd.q", errors.MissingBlockEnd}, {"MissingBlockStart.q", errors.MissingBlockStart}, {"MissingGroupEnd.q", errors.MissingGroupEnd},