From f479b5a03ab698c7423cb996b245f154ebe625bd Mon Sep 17 00:00:00 2001 From: Eduard Urbach Date: Sun, 30 Jun 2024 22:54:59 +0200 Subject: [PATCH] Implemented an abstract syntax tree --- src/build/AST.go | 87 +++++++++++++++++++ src/build/Call.go | 21 +++++ src/build/Execute.go | 108 ------------------------ src/build/ExpressionToRegister.go | 47 +++++++++++ src/build/Function.go | 90 ++++++++++---------- src/build/Instruction.go | 66 +++++++++------ src/build/Keyword.go | 20 ----- src/build/Loop.go | 24 ++---- src/build/Return.go | 16 +--- src/build/TokenToRegister.go | 59 +++++++++++++ src/build/Variable.go | 2 +- src/build/VariableDefinition.go | 17 ++-- src/build/asm/Instruction.go | 2 +- src/build/asm/Pointer.go | 2 +- src/build/ast/AST.go | 36 ++++++++ src/build/compiler.go | 16 ++-- src/build/expression/Expression.go | 19 ++--- src/build/expression/Expression_test.go | 2 +- src/build/expression/Operator.go | 20 ++--- src/build/expression/Parse.go | 2 +- src/build/expression/bench_test.go | 3 +- src/build/expression/pool.go | 9 -- src/build/fs/File.go | 2 +- src/build/keyword/Keyword.go | 12 +++ src/build/token/Keywords.go | 7 -- src/build/token/Token.go | 4 +- src/build/token/Tokenize.go | 42 ++++----- src/errors/Error.go | 2 +- 28 files changed, 422 insertions(+), 315 deletions(-) create mode 100644 src/build/AST.go create mode 100644 src/build/ExpressionToRegister.go delete mode 100644 src/build/Keyword.go create mode 100644 src/build/TokenToRegister.go create mode 100644 src/build/ast/AST.go delete mode 100644 src/build/expression/pool.go create mode 100644 src/build/keyword/Keyword.go delete mode 100644 src/build/token/Keywords.go diff --git a/src/build/AST.go b/src/build/AST.go new file mode 100644 index 0000000..b12cc5e --- /dev/null +++ b/src/build/AST.go @@ -0,0 +1,87 @@ +package build + +import ( + "git.akyoto.dev/cli/q/src/build/ast" + "git.akyoto.dev/cli/q/src/build/expression" + "git.akyoto.dev/cli/q/src/build/keyword" + "git.akyoto.dev/cli/q/src/build/token" + "git.akyoto.dev/cli/q/src/errors" +) + +// toAST generates an AST from a list of tokens. +func (f *Function) toAST(tokens token.List) (ast.AST, error) { + tree := make(ast.AST, 0, len(tokens)/64) + + err := EachInstruction(tokens, func(instruction token.List) error { + node, err := f.toASTNode(instruction) + + if err == nil && node != nil { + tree = append(tree, node) + } + + return err + }) + + return tree, err +} + +// toASTNode generates an AST node from an instruction. +func (f *Function) toASTNode(tokens token.List) (ast.Node, error) { + if tokens[0].Kind == token.Keyword { + switch tokens[0].Text() { + case keyword.Return: + value := expression.Parse(tokens[1:]) + return &ast.Return{Value: value}, nil + + case keyword.Loop: + blockStart := tokens.IndexKind(token.BlockStart) + 1 + blockEnd := tokens.LastIndexKind(token.BlockEnd) + + if blockStart == -1 { + return nil, errors.New(errors.MissingBlockStart, f.File, tokens[0].End()) + } + + if blockEnd == -1 { + return nil, errors.New(errors.MissingBlockEnd, f.File, tokens[len(tokens)-1].End()) + } + + tree, err := f.toAST(tokens[blockStart:blockEnd]) + return &ast.Loop{Body: tree}, err + + default: + return nil, errors.New(&errors.KeywordNotImplemented{Keyword: tokens[0].Text()}, f.File, tokens[0].Position) + } + } + + expr := expression.Parse(tokens) + + if expr == nil { + return nil, nil + } + + switch { + case isVariableDefinition(expr): + if len(expr.Children) < 2 { + return nil, errors.New(errors.MissingAssignValue, f.File, expr.LastChild().Token.End()) + } + + name := expr.Children[0].Token + value := expr.Children[1] + return &ast.Define{Name: name, Value: value}, nil + + case isAssignment(expr): + if len(expr.Children) < 2 { + return nil, errors.New(errors.MissingAssignValue, f.File, expr.LastChild().Token.End()) + } + + name := expr.Children[0].Token + value := expr.Children[1] + return &ast.Assign{Name: name, Value: value}, nil + + case isFunctionCall(expr): + return &ast.Call{Expression: expr}, nil + + default: + return nil, errors.New(&errors.InvalidInstruction{Instruction: expr.Token.Text()}, f.File, expr.Token.Position) + } +} diff --git a/src/build/Call.go b/src/build/Call.go index ee813fc..44df7c9 100644 --- a/src/build/Call.go +++ b/src/build/Call.go @@ -2,6 +2,7 @@ package build import ( "git.akyoto.dev/cli/q/src/build/config" + "git.akyoto.dev/cli/q/src/build/cpu" "git.akyoto.dev/cli/q/src/build/expression" ) @@ -45,3 +46,23 @@ func (f *Function) CompileSyscall(expr *expression.Expression) error { f.sideEffects++ return err } + +// ExpressionsToRegisters moves multiple expressions into the specified registers. +func (f *Function) ExpressionsToRegisters(expressions []*expression.Expression, registers []cpu.Register) error { + for _, register := range registers { + f.SaveRegister(register) + } + + for i := len(expressions) - 1; i >= 0; i-- { + expression := expressions[i] + register := registers[i] + + err := f.ExpressionToRegister(expression, register) + + if err != nil { + return err + } + } + + return nil +} diff --git a/src/build/Execute.go b/src/build/Execute.go index 4903ef7..6a96ce8 100644 --- a/src/build/Execute.go +++ b/src/build/Execute.go @@ -3,7 +3,6 @@ package build import ( "strconv" - "git.akyoto.dev/cli/q/src/build/asm" "git.akyoto.dev/cli/q/src/build/config" "git.akyoto.dev/cli/q/src/build/cpu" "git.akyoto.dev/cli/q/src/build/expression" @@ -72,110 +71,3 @@ func (f *Function) ExecuteLeaf(operation token.Token, register cpu.Register, ope return errors.New(errors.NotImplemented, f.File, operation.Position) } - -// ExpressionToRegister moves the result of an expression into the given register. -func (f *Function) ExpressionToRegister(root *expression.Expression, register cpu.Register) error { - if config.Verbose { - f.Logf("%s to register %s", root, register) - } - - operation := root.Token - - if root.IsLeaf() { - return f.TokenToRegister(operation, register) - } - - if isFunctionCall(root) { - err := f.CompileFunctionCall(root) - - if err != nil { - return err - } - - if register != f.cpu.Return[0] { - f.assembler.RegisterRegister(asm.MOVE, register, f.cpu.Return[0]) - } - - return nil - } - - left := root.Children[0] - right := root.Children[1] - - err := f.ExpressionToRegister(left, register) - - if err != nil { - return err - } - - f.SaveRegister(register) - return f.Execute(operation, register, right) -} - -// ExpressionsToRegisters moves multiple expressions into the specified registers. -func (f *Function) ExpressionsToRegisters(expressions []*expression.Expression, registers []cpu.Register) error { - for _, register := range registers { - f.SaveRegister(register) - } - - for i := len(expressions) - 1; i >= 0; i-- { - expression := expressions[i] - register := registers[i] - - err := f.ExpressionToRegister(expression, register) - - if err != nil { - return err - } - } - - return nil -} - -// TokenToRegister moves a token into a register. -// It only works with identifiers, numbers and strings. -func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error { - switch t.Kind { - case token.Identifier: - name := t.Text() - constant, exists := f.definitions[name] - - if exists { - if config.Verbose { - f.Logf("constant %s = %s", constant.Name, constant.Value) - } - - return f.ExpressionToRegister(constant.Value, register) - } - - variable, exists := f.variables[name] - - if !exists { - return errors.New(&errors.UnknownIdentifier{Name: name}, f.File, t.Position) - } - - if register != variable.Register { - f.assembler.RegisterRegister(asm.MOVE, register, variable.Register) - } - - f.useVariable(variable) - return nil - - case token.Number: - value := t.Text() - n, err := strconv.Atoi(value) - - if err != nil { - return err - } - - f.assembler.RegisterNumber(asm.MOVE, register, n) - return nil - - case token.String: - return errors.New(errors.NotImplemented, f.File, t.Position) - - default: - return errors.New(errors.InvalidExpression, f.File, t.Position) - } -} diff --git a/src/build/ExpressionToRegister.go b/src/build/ExpressionToRegister.go new file mode 100644 index 0000000..9a6f09d --- /dev/null +++ b/src/build/ExpressionToRegister.go @@ -0,0 +1,47 @@ +package build + +import ( + "git.akyoto.dev/cli/q/src/build/asm" + "git.akyoto.dev/cli/q/src/build/config" + "git.akyoto.dev/cli/q/src/build/cpu" + "git.akyoto.dev/cli/q/src/build/expression" +) + +// ExpressionToRegister moves the result of an expression into the given register. +func (f *Function) ExpressionToRegister(root *expression.Expression, register cpu.Register) error { + if config.Verbose { + f.Logf("%s to register %s", root, register) + } + + operation := root.Token + + if root.IsLeaf() { + return f.TokenToRegister(operation, register) + } + + if isFunctionCall(root) { + err := f.CompileFunctionCall(root) + + if err != nil { + return err + } + + if register != f.cpu.Return[0] { + f.assembler.RegisterRegister(asm.MOVE, register, f.cpu.Return[0]) + } + + return nil + } + + left := root.Children[0] + right := root.Children[1] + + err := f.ExpressionToRegister(left, register) + + if err != nil { + return err + } + + f.SaveRegister(register) + return f.Execute(operation, register, right) +} diff --git a/src/build/Function.go b/src/build/Function.go index d615a90..39bf0c3 100644 --- a/src/build/Function.go +++ b/src/build/Function.go @@ -3,6 +3,7 @@ package build import ( "fmt" + "git.akyoto.dev/cli/q/src/build/ast" "git.akyoto.dev/cli/q/src/build/config" "git.akyoto.dev/cli/q/src/build/expression" "git.akyoto.dev/cli/q/src/build/fs" @@ -11,8 +12,8 @@ import ( // Function represents a function. type Function struct { - Name string File *fs.File + Name string Body token.List compiler } @@ -26,61 +27,60 @@ func (f *Function) Compile() { } // CompileTokens compiles a token list. -func (f *Function) CompileTokens(body token.List) error { - start := 0 - groupLevel := 0 - blockLevel := 0 +func (f *Function) CompileTokens(tokens token.List) error { + tree, err := f.toAST(tokens) - for i, t := range body { - if start == i && t.Kind == token.NewLine { - start = i + 1 - continue + if err != nil { + return err + } + + return f.CompileAST(tree) +} + +// CompileAST compiles an abstract syntax tree. +func (f *Function) CompileAST(tree ast.AST) error { + for _, node := range tree { + if config.Verbose { + f.Logf("%T %s", node, node) } - switch t.Kind { - case token.NewLine: - if groupLevel > 0 || blockLevel > 0 { - continue - } + if config.Assembler { + f.debug = append(f.debug, debug{ + position: len(f.assembler.Instructions), + source: node, + }) + } - instruction := body[start:i] + err := f.CompileNode(node) - if config.Verbose { - f.Logf("compiling: %s", instruction) - } - - if config.Assembler { - f.debug = append(f.debug, debug{ - position: len(f.assembler.Instructions), - source: instruction, - }) - } - - err := f.CompileInstruction(instruction) - - if err != nil { - return err - } - - start = i + 1 - - case token.GroupStart: - groupLevel++ - - case token.GroupEnd: - groupLevel-- - - case token.BlockStart: - blockLevel++ - - case token.BlockEnd: - blockLevel-- + if err != nil { + return err } } return nil } +// CompileNode compiles a node in the AST. +func (f *Function) CompileNode(node ast.Node) error { + switch node := node.(type) { + case *ast.Call: + return f.CompileFunctionCall(node.Expression) + + case *ast.Define: + return f.CompileVariableDefinition(node) + + case *ast.Return: + return f.CompileReturn(node) + + case *ast.Loop: + return f.CompileLoop(node) + + default: + panic("Unknown AST type") + } +} + // Logf formats a message for verbose output. func (f *Function) Logf(format string, data ...any) { fmt.Printf("[%s @ %d] %s\n", f, len(f.assembler.Instructions), fmt.Sprintf(format, data...)) diff --git a/src/build/Instruction.go b/src/build/Instruction.go index c668258..944ab92 100644 --- a/src/build/Instruction.go +++ b/src/build/Instruction.go @@ -1,36 +1,48 @@ package build import ( - "git.akyoto.dev/cli/q/src/build/expression" "git.akyoto.dev/cli/q/src/build/token" - "git.akyoto.dev/cli/q/src/errors" ) -// CompileInstruction compiles a single instruction. -func (f *Function) CompileInstruction(instruction token.List) error { - if instruction[0].Kind == token.Keyword { - return f.CompileKeyword(instruction) +// EachInstruction calls the function on each instruction. +func EachInstruction(body token.List, call func(token.List) error) error { + start := 0 + groupLevel := 0 + blockLevel := 0 + + for i, t := range body { + if start == i && t.Kind == token.NewLine { + start = i + 1 + continue + } + + switch t.Kind { + case token.NewLine: + if groupLevel > 0 || blockLevel > 0 { + continue + } + + err := call(body[start:i]) + + if err != nil { + return err + } + + start = i + 1 + + case token.GroupStart: + groupLevel++ + + case token.GroupEnd: + groupLevel-- + + case token.BlockStart: + blockLevel++ + + case token.BlockEnd: + blockLevel-- + } } - expr := expression.Parse(instruction) - - if expr == nil { - return nil - } - - defer expr.Close() - - switch true { - case isVariableDefinition(expr): - return f.CompileVariableDefinition(expr) - - case isAssignment(expr): - return f.CompileAssignment(expr) - - case isFunctionCall(expr): - return f.CompileFunctionCall(expr) - - default: - return errors.New(&errors.InvalidInstruction{Instruction: expr.Token.Text()}, f.File, expr.Token.Position) - } + return nil } diff --git a/src/build/Keyword.go b/src/build/Keyword.go deleted file mode 100644 index 80e5ee9..0000000 --- a/src/build/Keyword.go +++ /dev/null @@ -1,20 +0,0 @@ -package build - -import ( - "git.akyoto.dev/cli/q/src/build/token" - "git.akyoto.dev/cli/q/src/errors" -) - -// CompileKeyword compiles an instruction that starts with a keyword. -func (f *Function) CompileKeyword(tokens token.List) error { - switch tokens[0].Text() { - case "return": - return f.CompileReturn(tokens) - - case "loop": - return f.CompileLoop(tokens) - - default: - return errors.New(&errors.KeywordNotImplemented{Keyword: tokens[0].Text()}, f.File, tokens[0].Position) - } -} diff --git a/src/build/Loop.go b/src/build/Loop.go index 41a9b4f..2165eca 100644 --- a/src/build/Loop.go +++ b/src/build/Loop.go @@ -3,26 +3,14 @@ package build import ( "fmt" - "git.akyoto.dev/cli/q/src/build/token" - "git.akyoto.dev/cli/q/src/errors" + "git.akyoto.dev/cli/q/src/build/ast" ) // CompileLoop compiles a loop instruction. -func (f *Function) CompileLoop(tokens token.List) error { - blockStart := tokens.IndexKind(token.BlockStart) + 1 - blockEnd := tokens.LastIndexKind(token.BlockEnd) - - if blockStart == -1 { - return errors.New(errors.MissingBlockStart, f.File, tokens[0].End()) - } - - if blockEnd == -1 { - return errors.New(errors.MissingBlockEnd, f.File, tokens[len(tokens)-1].End()) - } - - loop := fmt.Sprintf("%s_loop_%d", f.Name, f.count.loop) - f.assembler.Label(loop) - defer f.assembler.Jump(loop) +func (f *Function) CompileLoop(loop *ast.Loop) error { + label := fmt.Sprintf("%s_loop_%d", f.Name, f.count.loop) + f.assembler.Label(label) + defer f.assembler.Jump(label) f.count.loop++ - return f.CompileTokens(tokens[blockStart:blockEnd]) + return f.CompileAST(loop.Body) } diff --git a/src/build/Return.go b/src/build/Return.go index 8ffd8fb..c661fc7 100644 --- a/src/build/Return.go +++ b/src/build/Return.go @@ -1,24 +1,16 @@ package build import ( - "git.akyoto.dev/cli/q/src/build/expression" - "git.akyoto.dev/cli/q/src/build/token" + "git.akyoto.dev/cli/q/src/build/ast" ) // CompileReturn compiles a return instruction. -func (f *Function) CompileReturn(tokens token.List) error { +func (f *Function) CompileReturn(node *ast.Return) error { defer f.assembler.Return() - if len(tokens) == 1 { + if node.Value == nil { return nil } - value := expression.Parse(tokens[1:]) - - if value == nil { - return nil - } - - defer value.Close() - return f.ExpressionToRegister(value, f.cpu.Return[0]) + return f.ExpressionToRegister(node.Value, f.cpu.Return[0]) } diff --git a/src/build/TokenToRegister.go b/src/build/TokenToRegister.go new file mode 100644 index 0000000..6bef4cb --- /dev/null +++ b/src/build/TokenToRegister.go @@ -0,0 +1,59 @@ +package build + +import ( + "strconv" + + "git.akyoto.dev/cli/q/src/build/asm" + "git.akyoto.dev/cli/q/src/build/config" + "git.akyoto.dev/cli/q/src/build/cpu" + "git.akyoto.dev/cli/q/src/build/token" + "git.akyoto.dev/cli/q/src/errors" +) + +// TokenToRegister moves a token into a register. +// It only works with identifiers, numbers and strings. +func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error { + switch t.Kind { + case token.Identifier: + name := t.Text() + constant, exists := f.definitions[name] + + if exists { + if config.Verbose { + f.Logf("constant %s = %s", constant.Name, constant.Value) + } + + return f.ExpressionToRegister(constant.Value, register) + } + + variable, exists := f.variables[name] + + if !exists { + return errors.New(&errors.UnknownIdentifier{Name: name}, f.File, t.Position) + } + + if register != variable.Register { + f.assembler.RegisterRegister(asm.MOVE, register, variable.Register) + } + + f.useVariable(variable) + return nil + + case token.Number: + value := t.Text() + n, err := strconv.Atoi(value) + + if err != nil { + return err + } + + f.assembler.RegisterNumber(asm.MOVE, register, n) + return nil + + case token.String: + return errors.New(errors.NotImplemented, f.File, t.Position) + + default: + return errors.New(errors.InvalidExpression, f.File, t.Position) + } +} diff --git a/src/build/Variable.go b/src/build/Variable.go index 291d2d4..c81b850 100644 --- a/src/build/Variable.go +++ b/src/build/Variable.go @@ -14,6 +14,6 @@ type Variable struct { // Definitions are single use expressions that don't reside in a register yet. type Definition struct { - Name string Value *expression.Expression + Name string } diff --git a/src/build/VariableDefinition.go b/src/build/VariableDefinition.go index 58a4844..917f12f 100644 --- a/src/build/VariableDefinition.go +++ b/src/build/VariableDefinition.go @@ -1,6 +1,7 @@ package build import ( + "git.akyoto.dev/cli/q/src/build/ast" "git.akyoto.dev/cli/q/src/build/config" "git.akyoto.dev/cli/q/src/build/expression" "git.akyoto.dev/cli/q/src/build/token" @@ -8,24 +9,20 @@ import ( ) // CompileVariableDefinition compiles a variable definition. -func (f *Function) CompileVariableDefinition(expr *expression.Expression) error { - if len(expr.Children) < 2 { - return errors.New(errors.MissingAssignValue, f.File, expr.LastChild().Token.End()) - } - - name := expr.Children[0].Token.Text() +func (f *Function) CompileVariableDefinition(node *ast.Define) error { + name := node.Name.Text() if f.identifierExists(name) { - return errors.New(&errors.VariableAlreadyExists{Name: name}, f.File, expr.Children[0].Token.Position) + return errors.New(&errors.VariableAlreadyExists{Name: name}, f.File, node.Name.Position) } uses := countIdentifier(f.Body, name) - 1 if uses == 0 { - return errors.New(&errors.UnusedVariable{Name: name}, f.File, expr.Children[0].Token.Position) + return errors.New(&errors.UnusedVariable{Name: name}, f.File, node.Name.Position) } - value := expr.Children[1] + value := node.Value err := value.EachLeaf(func(leaf *expression.Expression) error { if leaf.Token.Kind == token.Identifier && !f.identifierExists(leaf.Token.Text()) { @@ -40,8 +37,6 @@ func (f *Function) CompileVariableDefinition(expr *expression.Expression) error } if uses == 1 { - expr.RemoveChild(value) - f.definitions[name] = &Definition{ Name: name, Value: value, diff --git a/src/build/asm/Instruction.go b/src/build/asm/Instruction.go index 5b2bbce..2447bae 100644 --- a/src/build/asm/Instruction.go +++ b/src/build/asm/Instruction.go @@ -4,6 +4,6 @@ import "fmt" // Instruction represents a single instruction which can be converted to machine code. type Instruction struct { - Mnemonic Mnemonic Data fmt.Stringer + Mnemonic Mnemonic } diff --git a/src/build/asm/Pointer.go b/src/build/asm/Pointer.go index 9be746f..4e33a63 100644 --- a/src/build/asm/Pointer.go +++ b/src/build/asm/Pointer.go @@ -7,7 +7,7 @@ type Address = uint32 // Position: The machine code offset where the address was inserted. // Resolve: The function that will return the final address. type Pointer struct { + Resolve func() Address Position Address Size uint8 - Resolve func() Address } diff --git a/src/build/ast/AST.go b/src/build/ast/AST.go new file mode 100644 index 0000000..2d8314a --- /dev/null +++ b/src/build/ast/AST.go @@ -0,0 +1,36 @@ +package ast + +import ( + "git.akyoto.dev/cli/q/src/build/expression" + "git.akyoto.dev/cli/q/src/build/token" +) + +type Node interface{} +type AST []Node + +type Assign struct { + Value *expression.Expression + Name token.Token +} + +type Call struct { + Expression *expression.Expression +} + +type Define struct { + Value *expression.Expression + Name token.Token +} + +type If struct { + Condition *expression.Expression + Body AST +} + +type Loop struct { + Body AST +} + +type Return struct { + Value *expression.Expression +} diff --git a/src/build/compiler.go b/src/build/compiler.go index 60cfec0..ac28b37 100644 --- a/src/build/compiler.go +++ b/src/build/compiler.go @@ -4,23 +4,23 @@ import ( "fmt" "git.akyoto.dev/cli/q/src/build/asm" + "git.akyoto.dev/cli/q/src/build/ast" "git.akyoto.dev/cli/q/src/build/cpu" - "git.akyoto.dev/cli/q/src/build/token" "git.akyoto.dev/go/color/ansi" ) // compiler is the data structure we embed in each function to preserve compilation state. type compiler struct { - assembler asm.Assembler - count counter - cpu cpu.CPU - debug []debug err error definitions map[string]*Definition variables map[string]*Variable functions map[string]*Function - sideEffects int finished chan struct{} + assembler asm.Assembler + debug []debug + cpu cpu.CPU + count counter + sideEffects int } // counter stores how often a certain statement appeared so we can generate a unique label from it. @@ -30,8 +30,8 @@ type counter struct { // debug is used to look up the source code at the given position. type debug struct { + source ast.Node position int - source token.List } // PrintInstructions shows the assembly instructions. @@ -66,7 +66,7 @@ func (c *compiler) PrintInstructions() { } // sourceAt retrieves the source code at the given position or `nil`. -func (c *compiler) sourceAt(position int) token.List { +func (c *compiler) sourceAt(position int) ast.Node { for _, record := range c.debug { if record.position == position { return record.source diff --git a/src/build/expression/Expression.go b/src/build/expression/Expression.go index 7776c35..1d2efa8 100644 --- a/src/build/expression/Expression.go +++ b/src/build/expression/Expression.go @@ -8,22 +8,22 @@ import ( // Expression is a binary tree with an operator on each node. type Expression struct { - Token token.Token Parent *Expression Children []*Expression - Precedence int + Token token.Token + Precedence int8 } // New creates a new expression. func New() *Expression { - return pool.Get().(*Expression) + return &Expression{} } // NewLeaf creates a new leaf node. func NewLeaf(t token.Token) *Expression { - expr := New() - expr.Token = t - return expr + return &Expression{ + Token: t, + } } // AddChild adds a child to the expression. @@ -32,17 +32,16 @@ func (expr *Expression) AddChild(child *Expression) { child.Parent = expr } -// Close puts the expression back into the memory pool. -func (expr *Expression) Close() { +// Reset resets all values to the default. +func (expr *Expression) Reset() { for _, child := range expr.Children { - child.Close() + child.Reset() } expr.Token.Reset() expr.Parent = nil expr.Children = expr.Children[:0] expr.Precedence = 0 - pool.Put(expr) } // EachLeaf iterates through all leaves in the tree. diff --git a/src/build/expression/Expression_test.go b/src/build/expression/Expression_test.go index eb992c6..08c4e5d 100644 --- a/src/build/expression/Expression_test.go +++ b/src/build/expression/Expression_test.go @@ -88,7 +88,7 @@ func TestExpressionParse(t *testing.T) { src := []byte(test.Expression) tokens := token.Tokenize(src) expr := expression.Parse(tokens) - defer expr.Close() + defer expr.Reset() assert.NotNil(t, expr) assert.Equal(t, expr.String(), test.Result) diff --git a/src/build/expression/Operator.go b/src/build/expression/Operator.go index 5e50dc5..07fc5a1 100644 --- a/src/build/expression/Operator.go +++ b/src/build/expression/Operator.go @@ -9,7 +9,7 @@ import ( // Operator represents an operator for mathematical expressions. type Operator struct { Symbol string - Precedence int + Precedence int8 Operands int } @@ -39,14 +39,14 @@ var Operators = map[string]*Operator{ "&&": {"&&", 2, 2}, "||": {"||", 1, 2}, - "=": {"=", math.MinInt, 2}, - ":=": {":=", math.MinInt, 2}, - "+=": {"+=", math.MinInt, 2}, - "-=": {"-=", math.MinInt, 2}, - "*=": {"*=", math.MinInt, 2}, - "/=": {"/=", math.MinInt, 2}, - ">>=": {">>=", math.MinInt, 2}, - "<<=": {"<<=", math.MinInt, 2}, + "=": {"=", math.MinInt8, 2}, + ":=": {":=", math.MinInt8, 2}, + "+=": {"+=", math.MinInt8, 2}, + "-=": {"-=", math.MinInt8, 2}, + "*=": {"*=", math.MinInt8, 2}, + "/=": {"/=", math.MinInt8, 2}, + ">>=": {">>=", math.MinInt8, 2}, + "<<=": {"<<=", math.MinInt8, 2}, } func isComplete(expr *Expression) bool { @@ -75,7 +75,7 @@ func numOperands(symbol string) int { return operator.Operands } -func precedence(symbol string) int { +func precedence(symbol string) int8 { operator, exists := Operators[symbol] if !exists { diff --git a/src/build/expression/Parse.go b/src/build/expression/Parse.go index 072dfe4..ee90a0b 100644 --- a/src/build/expression/Parse.go +++ b/src/build/expression/Parse.go @@ -70,7 +70,7 @@ func Parse(tokens token.List) *Expression { continue } - group.Precedence = math.MaxInt + group.Precedence = math.MaxInt8 if cursor == nil { cursor = group diff --git a/src/build/expression/bench_test.go b/src/build/expression/bench_test.go index 8287416..c447910 100644 --- a/src/build/expression/bench_test.go +++ b/src/build/expression/bench_test.go @@ -12,7 +12,6 @@ func BenchmarkExpression(b *testing.B) { tokens := token.Tokenize(src) for i := 0; i < b.N; i++ { - expr := expression.Parse(tokens) - expr.Close() + expression.Parse(tokens) } } diff --git a/src/build/expression/pool.go b/src/build/expression/pool.go deleted file mode 100644 index 42b571d..0000000 --- a/src/build/expression/pool.go +++ /dev/null @@ -1,9 +0,0 @@ -package expression - -import "sync" - -var pool = sync.Pool{ - New: func() interface{} { - return &Expression{} - }, -} diff --git a/src/build/fs/File.go b/src/build/fs/File.go index 53d49c2..d4fd1cf 100644 --- a/src/build/fs/File.go +++ b/src/build/fs/File.go @@ -4,6 +4,6 @@ import "git.akyoto.dev/cli/q/src/build/token" // File represents a single source file. type File struct { - Tokens token.List Path string + Tokens token.List } diff --git a/src/build/keyword/Keyword.go b/src/build/keyword/Keyword.go new file mode 100644 index 0000000..d3d9b18 --- /dev/null +++ b/src/build/keyword/Keyword.go @@ -0,0 +1,12 @@ +package keyword + +const ( + Loop = "loop" + Return = "return" +) + +// Map is a map of all keywords used in the language. +var Map = map[string][]byte{ + Loop: []byte(Loop), + Return: []byte(Return), +} diff --git a/src/build/token/Keywords.go b/src/build/token/Keywords.go deleted file mode 100644 index d5f6231..0000000 --- a/src/build/token/Keywords.go +++ /dev/null @@ -1,7 +0,0 @@ -package token - -// Keywords defines the keywords used in the language. -var Keywords = map[string]bool{ - "return": true, - "loop": true, -} diff --git a/src/build/token/Token.go b/src/build/token/Token.go index fadf760..e10f8bc 100644 --- a/src/build/token/Token.go +++ b/src/build/token/Token.go @@ -6,9 +6,9 @@ import "fmt" // The characters that make up an identifier are grouped into a single token. // This makes parsing easier and allows us to do better syntax checks. type Token struct { - Kind Kind - Position int Bytes []byte + Position int + Kind Kind } // End returns the position after the token. diff --git a/src/build/token/Tokenize.go b/src/build/token/Tokenize.go index 5388b4b..7347252 100644 --- a/src/build/token/Tokenize.go +++ b/src/build/token/Tokenize.go @@ -1,5 +1,7 @@ package token +import "git.akyoto.dev/cli/q/src/build/keyword" + // Pre-allocate these byte buffers so we can re-use them // instead of allocating a new buffer every time. var ( @@ -26,35 +28,35 @@ func Tokenize(buffer []byte) List { case ' ', '\t': // Separator case ',': - tokens = append(tokens, Token{Separator, i, separatorBytes}) + tokens = append(tokens, Token{Kind: Separator, Position: i, Bytes: separatorBytes}) // Parentheses start case '(': - tokens = append(tokens, Token{GroupStart, i, groupStartBytes}) + tokens = append(tokens, Token{Kind: GroupStart, Position: i, Bytes: groupStartBytes}) // Parentheses end case ')': - tokens = append(tokens, Token{GroupEnd, i, groupEndBytes}) + tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Bytes: groupEndBytes}) // Block start case '{': - tokens = append(tokens, Token{BlockStart, i, blockStartBytes}) + tokens = append(tokens, Token{Kind: BlockStart, Position: i, Bytes: blockStartBytes}) // Block end case '}': - tokens = append(tokens, Token{BlockEnd, i, blockEndBytes}) + tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Bytes: blockEndBytes}) // Array start case '[': - tokens = append(tokens, Token{ArrayStart, i, arrayStartBytes}) + tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Bytes: arrayStartBytes}) // Array end case ']': - tokens = append(tokens, Token{ArrayEnd, i, arrayEndBytes}) + tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Bytes: arrayEndBytes}) // New line case '\n': - tokens = append(tokens, Token{NewLine, i, newLineBytes}) + tokens = append(tokens, Token{Kind: NewLine, Position: i, Bytes: newLineBytes}) // Comment case '/': @@ -66,7 +68,7 @@ func Tokenize(buffer []byte) List { i++ } - tokens = append(tokens, Token{Operator, position, buffer[position:i]}) + tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]}) continue } @@ -76,7 +78,7 @@ func Tokenize(buffer []byte) List { i++ } - tokens = append(tokens, Token{Comment, position, buffer[position:i]}) + tokens = append(tokens, Token{Kind: Comment, Position: position, Bytes: buffer[position:i]}) continue // String @@ -95,7 +97,7 @@ func Tokenize(buffer []byte) List { i++ } - tokens = append(tokens, Token{String, start, buffer[start:end]}) + tokens = append(tokens, Token{Kind: String, Position: start, Bytes: buffer[start:end]}) continue default: @@ -108,13 +110,15 @@ func Tokenize(buffer []byte) List { i++ } - token := Token{Identifier, position, buffer[position:i]} + identifier := buffer[position:i] + keyword, isKeyword := keyword.Map[string(identifier)] - if Keywords[string(token.Bytes)] { - token.Kind = Keyword + if isKeyword { + tokens = append(tokens, Token{Kind: Keyword, Position: position, Bytes: keyword}) + } else { + tokens = append(tokens, Token{Kind: Identifier, Position: position, Bytes: identifier}) } - tokens = append(tokens, token) continue } @@ -127,7 +131,7 @@ func Tokenize(buffer []byte) List { i++ } - tokens = append(tokens, Token{Number, position, buffer[position:i]}) + tokens = append(tokens, Token{Kind: Number, Position: position, Bytes: buffer[position:i]}) continue } @@ -140,18 +144,18 @@ func Tokenize(buffer []byte) List { i++ } - tokens = append(tokens, Token{Operator, position, buffer[position:i]}) + tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]}) continue } // Invalid character - tokens = append(tokens, Token{Invalid, i, buffer[i : i+1]}) + tokens = append(tokens, Token{Kind: Invalid, Position: i, Bytes: buffer[i : i+1]}) } i++ } - tokens = append(tokens, Token{EOF, i, nil}) + tokens = append(tokens, Token{Kind: EOF, Position: i, Bytes: nil}) return tokens } diff --git a/src/errors/Error.go b/src/errors/Error.go index ad434d3..0821e25 100644 --- a/src/errors/Error.go +++ b/src/errors/Error.go @@ -13,8 +13,8 @@ import ( type Error struct { Err error File *fs.File - Position int Stack string + Position int } // New generates an error message at the current token position.