cli/q
Code Issues Pull Requests Activity

Implemented an abstract syntax tree

Browse Source
This commit is contained in:
Eduard Urbach 2024-06-30 22:54:59 +02:00
parent 27c707b6ff
commit f479b5a03a
Signed by: akyoto
GPG Key ID: C874F672B1AF20C0
28 changed files with 422 additions and 315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
src/build/AST.go Normal file
View File

@ -0,0 +1,87 @@
package build
import (
"git.akyoto.dev/cli/q/src/build/ast"
"git.akyoto.dev/cli/q/src/build/expression"
"git.akyoto.dev/cli/q/src/build/keyword"
"git.akyoto.dev/cli/q/src/build/token"
"git.akyoto.dev/cli/q/src/errors"
)
// toAST generates an AST from a list of tokens.
func (f *Function) toAST(tokens token.List) (ast.AST, error) {
tree := make(ast.AST, 0, len(tokens)/64)
err := EachInstruction(tokens, func(instruction token.List) error {
node, err := f.toASTNode(instruction)
if err == nil && node != nil {
tree = append(tree, node)
}
return err
})
return tree, err
}
// toASTNode generates an AST node from an instruction.
func (f *Function) toASTNode(tokens token.List) (ast.Node, error) {
if tokens[0].Kind == token.Keyword {
switch tokens[0].Text() {
case keyword.Return:
value := expression.Parse(tokens[1:])
return &ast.Return{Value: value}, nil
case keyword.Loop:
blockStart := tokens.IndexKind(token.BlockStart) + 1
blockEnd := tokens.LastIndexKind(token.BlockEnd)
if blockStart == -1 {
return nil, errors.New(errors.MissingBlockStart, f.File, tokens[0].End())
}
if blockEnd == -1 {
return nil, errors.New(errors.MissingBlockEnd, f.File, tokens[len(tokens)-1].End())
}
tree, err := f.toAST(tokens[blockStart:blockEnd])
return &ast.Loop{Body: tree}, err
default:
return nil, errors.New(&errors.KeywordNotImplemented{Keyword: tokens[0].Text()}, f.File, tokens[0].Position)
}
}
expr := expression.Parse(tokens)
if expr == nil {
return nil, nil
}
switch {
case isVariableDefinition(expr):
if len(expr.Children) < 2 {
return nil, errors.New(errors.MissingAssignValue, f.File, expr.LastChild().Token.End())
}
name := expr.Children[0].Token
value := expr.Children[1]
return &ast.Define{Name: name, Value: value}, nil
case isAssignment(expr):
if len(expr.Children) < 2 {
return nil, errors.New(errors.MissingAssignValue, f.File, expr.LastChild().Token.End())
}
name := expr.Children[0].Token
value := expr.Children[1]
return &ast.Assign{Name: name, Value: value}, nil
case isFunctionCall(expr):
return &ast.Call{Expression: expr}, nil
default:
return nil, errors.New(&errors.InvalidInstruction{Instruction: expr.Token.Text()}, f.File, expr.Token.Position)
}
}

21
src/build/Call.go
View File

@ -2,6 +2,7 @@ package build
import (
"git.akyoto.dev/cli/q/src/build/config"
"git.akyoto.dev/cli/q/src/build/cpu"
"git.akyoto.dev/cli/q/src/build/expression"
)
@ -45,3 +46,23 @@ func (f *Function) CompileSyscall(expr *expression.Expression) error {
f.sideEffects++
return err
}
// ExpressionsToRegisters moves multiple expressions into the specified registers.
func (f *Function) ExpressionsToRegisters(expressions []*expression.Expression, registers []cpu.Register) error {
for _, register := range registers {
f.SaveRegister(register)
}
for i := len(expressions) - 1; i >= 0; i-- {
expression := expressions[i]
register := registers[i]
err := f.ExpressionToRegister(expression, register)
if err != nil {
return err
}
}
return nil
}

108
src/build/Execute.go
View File

@ -3,7 +3,6 @@ package build
import (
"strconv"
"git.akyoto.dev/cli/q/src/build/asm"
"git.akyoto.dev/cli/q/src/build/config"
"git.akyoto.dev/cli/q/src/build/cpu"
"git.akyoto.dev/cli/q/src/build/expression"
@ -72,110 +71,3 @@ func (f *Function) ExecuteLeaf(operation token.Token, register cpu.Register, ope
return errors.New(errors.NotImplemented, f.File, operation.Position)
}
// ExpressionToRegister moves the result of an expression into the given register.
func (f *Function) ExpressionToRegister(root *expression.Expression, register cpu.Register) error {
if config.Verbose {
f.Logf("%s to register %s", root, register)
}
operation := root.Token
if root.IsLeaf() {
return f.TokenToRegister(operation, register)
}
if isFunctionCall(root) {
err := f.CompileFunctionCall(root)
if err != nil {
return err
}
if register != f.cpu.Return[0] {
f.assembler.RegisterRegister(asm.MOVE, register, f.cpu.Return[0])
}
return nil
}
left := root.Children[0]
right := root.Children[1]
err := f.ExpressionToRegister(left, register)
if err != nil {
return err
}
f.SaveRegister(register)
return f.Execute(operation, register, right)
}
// ExpressionsToRegisters moves multiple expressions into the specified registers.
func (f *Function) ExpressionsToRegisters(expressions []*expression.Expression, registers []cpu.Register) error {
for _, register := range registers {
f.SaveRegister(register)
}
for i := len(expressions) - 1; i >= 0; i-- {
expression := expressions[i]
register := registers[i]
err := f.ExpressionToRegister(expression, register)
if err != nil {
return err
}
}
return nil
}
// TokenToRegister moves a token into a register.
// It only works with identifiers, numbers and strings.
func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error {
switch t.Kind {
case token.Identifier:
name := t.Text()
constant, exists := f.definitions[name]
if exists {
if config.Verbose {
f.Logf("constant %s = %s", constant.Name, constant.Value)
}
return f.ExpressionToRegister(constant.Value, register)
}
variable, exists := f.variables[name]
if !exists {
return errors.New(&errors.UnknownIdentifier{Name: name}, f.File, t.Position)
}
if register != variable.Register {
f.assembler.RegisterRegister(asm.MOVE, register, variable.Register)
}
f.useVariable(variable)
return nil
case token.Number:
value := t.Text()
n, err := strconv.Atoi(value)
if err != nil {
return err
}
f.assembler.RegisterNumber(asm.MOVE, register, n)
return nil
case token.String:
return errors.New(errors.NotImplemented, f.File, t.Position)
default:
return errors.New(errors.InvalidExpression, f.File, t.Position)
}
}

47
src/build/ExpressionToRegister.go Normal file
View File

@ -0,0 +1,47 @@
package build
import (
"git.akyoto.dev/cli/q/src/build/asm"
"git.akyoto.dev/cli/q/src/build/config"
"git.akyoto.dev/cli/q/src/build/cpu"
"git.akyoto.dev/cli/q/src/build/expression"
)
// ExpressionToRegister moves the result of an expression into the given register.
func (f *Function) ExpressionToRegister(root *expression.Expression, register cpu.Register) error {
if config.Verbose {
f.Logf("%s to register %s", root, register)
}
operation := root.Token
if root.IsLeaf() {
return f.TokenToRegister(operation, register)
}
if isFunctionCall(root) {
err := f.CompileFunctionCall(root)
if err != nil {
return err
}
if register != f.cpu.Return[0] {
f.assembler.RegisterRegister(asm.MOVE, register, f.cpu.Return[0])
}
return nil
}
left := root.Children[0]
right := root.Children[1]
err := f.ExpressionToRegister(left, register)
if err != nil {
return err
}
f.SaveRegister(register)
return f.Execute(operation, register, right)
}

90
src/build/Function.go
View File

@ -3,6 +3,7 @@ package build
import (
"fmt"
"git.akyoto.dev/cli/q/src/build/ast"
"git.akyoto.dev/cli/q/src/build/config"
"git.akyoto.dev/cli/q/src/build/expression"
"git.akyoto.dev/cli/q/src/build/fs"
@ -11,8 +12,8 @@ import (
// Function represents a function.
type Function struct {
Name string
File *fs.File
Name string
Body token.List
compiler
}
@ -26,61 +27,60 @@ func (f *Function) Compile() {
}
// CompileTokens compiles a token list.
func (f *Function) CompileTokens(body token.List) error {
start := 0
groupLevel := 0
blockLevel := 0
func (f *Function) CompileTokens(tokens token.List) error {
tree, err := f.toAST(tokens)
for i, t := range body {
if start == i && t.Kind == token.NewLine {
start = i + 1
continue
if err != nil {
return err
}
return f.CompileAST(tree)
}
// CompileAST compiles an abstract syntax tree.
func (f *Function) CompileAST(tree ast.AST) error {
for _, node := range tree {
if config.Verbose {
f.Logf("%T %s", node, node)
}
switch t.Kind {
case token.NewLine:
if groupLevel > 0 || blockLevel > 0 {
continue
}
if config.Assembler {
f.debug = append(f.debug, debug{
position: len(f.assembler.Instructions),
source: node,
})
}
instruction := body[start:i]
err := f.CompileNode(node)
if config.Verbose {
f.Logf("compiling: %s", instruction)
}
if config.Assembler {
f.debug = append(f.debug, debug{
position: len(f.assembler.Instructions),
source: instruction,
})
}
err := f.CompileInstruction(instruction)
if err != nil {
return err
}
start = i + 1
case token.GroupStart:
groupLevel++
case token.GroupEnd:
groupLevel--
case token.BlockStart:
blockLevel++
case token.BlockEnd:
blockLevel--
if err != nil {
return err
}
}
return nil
}
// CompileNode compiles a node in the AST.
func (f *Function) CompileNode(node ast.Node) error {
switch node := node.(type) {
case *ast.Call:
return f.CompileFunctionCall(node.Expression)
case *ast.Define:
return f.CompileVariableDefinition(node)
case *ast.Return:
return f.CompileReturn(node)
case *ast.Loop:
return f.CompileLoop(node)
default:
panic("Unknown AST type")
}
}
// Logf formats a message for verbose output.
func (f *Function) Logf(format string, data ...any) {
fmt.Printf("[%s @ %d] %s\n", f, len(f.assembler.Instructions), fmt.Sprintf(format, data...))

66
src/build/Instruction.go
View File

@ -1,36 +1,48 @@
package build
import (
"git.akyoto.dev/cli/q/src/build/expression"
"git.akyoto.dev/cli/q/src/build/token"
"git.akyoto.dev/cli/q/src/errors"
)
// CompileInstruction compiles a single instruction.
func (f *Function) CompileInstruction(instruction token.List) error {
if instruction[0].Kind == token.Keyword {
return f.CompileKeyword(instruction)
// EachInstruction calls the function on each instruction.
func EachInstruction(body token.List, call func(token.List) error) error {
start := 0
groupLevel := 0
blockLevel := 0
for i, t := range body {
if start == i && t.Kind == token.NewLine {
start = i + 1
continue
}
switch t.Kind {
case token.NewLine:
if groupLevel > 0 || blockLevel > 0 {
continue
}
err := call(body[start:i])
if err != nil {
return err
}
start = i + 1
case token.GroupStart:
groupLevel++
case token.GroupEnd:
groupLevel--
case token.BlockStart:
blockLevel++
case token.BlockEnd:
blockLevel--
}
}
expr := expression.Parse(instruction)
if expr == nil {
return nil
}
defer expr.Close()
switch true {
case isVariableDefinition(expr):
return f.CompileVariableDefinition(expr)
case isAssignment(expr):
return f.CompileAssignment(expr)
case isFunctionCall(expr):
return f.CompileFunctionCall(expr)
default:
return errors.New(&errors.InvalidInstruction{Instruction: expr.Token.Text()}, f.File, expr.Token.Position)
}
return nil
}

20
src/build/Keyword.go
View File

@ -1,20 +0,0 @@
package build
import (
"git.akyoto.dev/cli/q/src/build/token"
"git.akyoto.dev/cli/q/src/errors"
)
// CompileKeyword compiles an instruction that starts with a keyword.
func (f *Function) CompileKeyword(tokens token.List) error {
switch tokens[0].Text() {
case "return":
return f.CompileReturn(tokens)
case "loop":
return f.CompileLoop(tokens)
default:
return errors.New(&errors.KeywordNotImplemented{Keyword: tokens[0].Text()}, f.File, tokens[0].Position)
}
}

24
src/build/Loop.go
View File

@ -3,26 +3,14 @@ package build
import (
"fmt"
"git.akyoto.dev/cli/q/src/build/token"
"git.akyoto.dev/cli/q/src/errors"
"git.akyoto.dev/cli/q/src/build/ast"
)
// CompileLoop compiles a loop instruction.
func (f *Function) CompileLoop(tokens token.List) error {
blockStart := tokens.IndexKind(token.BlockStart) + 1
blockEnd := tokens.LastIndexKind(token.BlockEnd)
if blockStart == -1 {
return errors.New(errors.MissingBlockStart, f.File, tokens[0].End())
}
if blockEnd == -1 {
return errors.New(errors.MissingBlockEnd, f.File, tokens[len(tokens)-1].End())
}
loop := fmt.Sprintf("%s_loop_%d", f.Name, f.count.loop)
f.assembler.Label(loop)
defer f.assembler.Jump(loop)
func (f *Function) CompileLoop(loop *ast.Loop) error {
label := fmt.Sprintf("%s_loop_%d", f.Name, f.count.loop)
f.assembler.Label(label)
defer f.assembler.Jump(label)
f.count.loop++
return f.CompileTokens(tokens[blockStart:blockEnd])
return f.CompileAST(loop.Body)
}

16
src/build/Return.go
View File

@ -1,24 +1,16 @@
package build
import (
"git.akyoto.dev/cli/q/src/build/expression"
"git.akyoto.dev/cli/q/src/build/token"
"git.akyoto.dev/cli/q/src/build/ast"
)
// CompileReturn compiles a return instruction.
func (f *Function) CompileReturn(tokens token.List) error {
func (f *Function) CompileReturn(node *ast.Return) error {
defer f.assembler.Return()
if len(tokens) == 1 {
if node.Value == nil {
return nil
}
value := expression.Parse(tokens[1:])
if value == nil {
return nil
}
defer value.Close()
return f.ExpressionToRegister(value, f.cpu.Return[0])
return f.ExpressionToRegister(node.Value, f.cpu.Return[0])
}

59
src/build/TokenToRegister.go Normal file
View File

@ -0,0 +1,59 @@
package build
import (
"strconv"
"git.akyoto.dev/cli/q/src/build/asm"
"git.akyoto.dev/cli/q/src/build/config"
"git.akyoto.dev/cli/q/src/build/cpu"
"git.akyoto.dev/cli/q/src/build/token"
"git.akyoto.dev/cli/q/src/errors"
)
// TokenToRegister moves a token into a register.
// It only works with identifiers, numbers and strings.
func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error {
switch t.Kind {
case token.Identifier:
name := t.Text()
constant, exists := f.definitions[name]
if exists {
if config.Verbose {
f.Logf("constant %s = %s", constant.Name, constant.Value)
}
return f.ExpressionToRegister(constant.Value, register)
}
variable, exists := f.variables[name]
if !exists {
return errors.New(&errors.UnknownIdentifier{Name: name}, f.File, t.Position)
}
if register != variable.Register {
f.assembler.RegisterRegister(asm.MOVE, register, variable.Register)
}
f.useVariable(variable)
return nil
case token.Number:
value := t.Text()
n, err := strconv.Atoi(value)
if err != nil {
return err
}
f.assembler.RegisterNumber(asm.MOVE, register, n)
return nil
case token.String:
return errors.New(errors.NotImplemented, f.File, t.Position)
default:
return errors.New(errors.InvalidExpression, f.File, t.Position)
}
}

2
src/build/Variable.go
View File

@ -14,6 +14,6 @@ type Variable struct {
// Definitions are single use expressions that don't reside in a register yet.
type Definition struct {
Name string
Value *expression.Expression
Name string
}

17
src/build/VariableDefinition.go
View File

@ -1,6 +1,7 @@
package build
import (
"git.akyoto.dev/cli/q/src/build/ast"
"git.akyoto.dev/cli/q/src/build/config"
"git.akyoto.dev/cli/q/src/build/expression"
"git.akyoto.dev/cli/q/src/build/token"
@ -8,24 +9,20 @@ import (
)
// CompileVariableDefinition compiles a variable definition.
func (f *Function) CompileVariableDefinition(expr *expression.Expression) error {
if len(expr.Children) < 2 {
return errors.New(errors.MissingAssignValue, f.File, expr.LastChild().Token.End())
}
name := expr.Children[0].Token.Text()
func (f *Function) CompileVariableDefinition(node *ast.Define) error {
name := node.Name.Text()
if f.identifierExists(name) {
return errors.New(&errors.VariableAlreadyExists{Name: name}, f.File, expr.Children[0].Token.Position)
return errors.New(&errors.VariableAlreadyExists{Name: name}, f.File, node.Name.Position)
}
uses := countIdentifier(f.Body, name) - 1
if uses == 0 {
return errors.New(&errors.UnusedVariable{Name: name}, f.File, expr.Children[0].Token.Position)
return errors.New(&errors.UnusedVariable{Name: name}, f.File, node.Name.Position)
}
value := expr.Children[1]
value := node.Value
err := value.EachLeaf(func(leaf *expression.Expression) error {
if leaf.Token.Kind == token.Identifier && !f.identifierExists(leaf.Token.Text()) {
@ -40,8 +37,6 @@ func (f *Function) CompileVariableDefinition(expr *expression.Expression) error
}
if uses == 1 {
expr.RemoveChild(value)
f.definitions[name] = &Definition{
Name: name,
Value: value,

2
src/build/asm/Instruction.go
View File

@ -4,6 +4,6 @@ import "fmt"
// Instruction represents a single instruction which can be converted to machine code.
type Instruction struct {
Mnemonic Mnemonic
Data fmt.Stringer
Mnemonic Mnemonic
}

2
src/build/asm/Pointer.go
View File

@ -7,7 +7,7 @@ type Address = uint32
// Position: The machine code offset where the address was inserted.
// Resolve: The function that will return the final address.
type Pointer struct {
Resolve func() Address
Position Address
Size uint8
Resolve func() Address
}

36
src/build/ast/AST.go Normal file
View File

@ -0,0 +1,36 @@
package ast
import (
"git.akyoto.dev/cli/q/src/build/expression"
"git.akyoto.dev/cli/q/src/build/token"
)
type Node interface{}
type AST []Node
type Assign struct {
Value *expression.Expression
Name token.Token
}
type Call struct {
Expression *expression.Expression
}
type Define struct {
Value *expression.Expression
Name token.Token
}
type If struct {
Condition *expression.Expression
Body AST
}
type Loop struct {
Body AST
}
type Return struct {
Value *expression.Expression
}

16
src/build/compiler.go
View File

@ -4,23 +4,23 @@ import (
"fmt"
"git.akyoto.dev/cli/q/src/build/asm"
"git.akyoto.dev/cli/q/src/build/ast"
"git.akyoto.dev/cli/q/src/build/cpu"
"git.akyoto.dev/cli/q/src/build/token"
"git.akyoto.dev/go/color/ansi"
)
// compiler is the data structure we embed in each function to preserve compilation state.
type compiler struct {
assembler asm.Assembler
count counter
cpu cpu.CPU
debug []debug
err error
definitions map[string]*Definition
variables map[string]*Variable
functions map[string]*Function
sideEffects int
finished chan struct{}
assembler asm.Assembler
debug []debug
cpu cpu.CPU
count counter
sideEffects int
}
// counter stores how often a certain statement appeared so we can generate a unique label from it.
@ -30,8 +30,8 @@ type counter struct {
// debug is used to look up the source code at the given position.
type debug struct {
source ast.Node
position int
source token.List
}
// PrintInstructions shows the assembly instructions.
@ -66,7 +66,7 @@ func (c *compiler) PrintInstructions() {
}
// sourceAt retrieves the source code at the given position or `nil`.
func (c *compiler) sourceAt(position int) token.List {
func (c *compiler) sourceAt(position int) ast.Node {
for _, record := range c.debug {
if record.position == position {
return record.source

19
src/build/expression/Expression.go
View File

@ -8,22 +8,22 @@ import (
// Expression is a binary tree with an operator on each node.
type Expression struct {
Token token.Token
Parent *Expression
Children []*Expression
Precedence int
Token token.Token
Precedence int8
}
// New creates a new expression.
func New() *Expression {
return pool.Get().(*Expression)
return &Expression{}
}
// NewLeaf creates a new leaf node.
func NewLeaf(t token.Token) *Expression {
expr := New()
expr.Token = t
return expr
return &Expression{
Token: t,
}
}
// AddChild adds a child to the expression.
@ -32,17 +32,16 @@ func (expr *Expression) AddChild(child *Expression) {
child.Parent = expr
}
// Close puts the expression back into the memory pool.
func (expr *Expression) Close() {
// Reset resets all values to the default.
func (expr *Expression) Reset() {
for _, child := range expr.Children {
child.Close()
child.Reset()
}
expr.Token.Reset()
expr.Parent = nil
expr.Children = expr.Children[:0]
expr.Precedence = 0
pool.Put(expr)
}
// EachLeaf iterates through all leaves in the tree.

2
src/build/expression/Expression_test.go
View File

@ -88,7 +88,7 @@ func TestExpressionParse(t *testing.T) {
src := []byte(test.Expression)
tokens := token.Tokenize(src)
expr := expression.Parse(tokens)
defer expr.Close()
defer expr.Reset()
assert.NotNil(t, expr)
assert.Equal(t, expr.String(), test.Result)

20
src/build/expression/Operator.go
View File

@ -9,7 +9,7 @@ import (
// Operator represents an operator for mathematical expressions.
type Operator struct {
Symbol string
Precedence int
Precedence int8
Operands int
}
@ -39,14 +39,14 @@ var Operators = map[string]*Operator{
"&&": {"&&", 2, 2},
"||": {"||", 1, 2},
"=": {"=", math.MinInt, 2},
":=": {":=", math.MinInt, 2},
"+=": {"+=", math.MinInt, 2},
"-=": {"-=", math.MinInt, 2},
"*=": {"*=", math.MinInt, 2},
"/=": {"/=", math.MinInt, 2},
">>=": {">>=", math.MinInt, 2},
"<<=": {"<<=", math.MinInt, 2},
"=": {"=", math.MinInt8, 2},
":=": {":=", math.MinInt8, 2},
"+=": {"+=", math.MinInt8, 2},
"-=": {"-=", math.MinInt8, 2},
"*=": {"*=", math.MinInt8, 2},
"/=": {"/=", math.MinInt8, 2},
">>=": {">>=", math.MinInt8, 2},
"<<=": {"<<=", math.MinInt8, 2},
}
func isComplete(expr *Expression) bool {
@ -75,7 +75,7 @@ func numOperands(symbol string) int {
return operator.Operands
}
func precedence(symbol string) int {
func precedence(symbol string) int8 {
operator, exists := Operators[symbol]
if !exists {

2
src/build/expression/Parse.go
View File

@ -70,7 +70,7 @@ func Parse(tokens token.List) *Expression {
continue
}
group.Precedence = math.MaxInt
group.Precedence = math.MaxInt8
if cursor == nil {
cursor = group

3
src/build/expression/bench_test.go
View File

@ -12,7 +12,6 @@ func BenchmarkExpression(b *testing.B) {
tokens := token.Tokenize(src)
for i := 0; i < b.N; i++ {
expr := expression.Parse(tokens)
expr.Close()
expression.Parse(tokens)
}
}

9
src/build/expression/pool.go
View File

@ -1,9 +0,0 @@
package expression
import "sync"
var pool = sync.Pool{
New: func() interface{} {
return &Expression{}
},
}

2
src/build/fs/File.go
View File

@ -4,6 +4,6 @@ import "git.akyoto.dev/cli/q/src/build/token"
// File represents a single source file.
type File struct {
Tokens token.List
Path string
Tokens token.List
}

12
src/build/keyword/Keyword.go Normal file
View File

@ -0,0 +1,12 @@
package keyword
const (
Loop = "loop"
Return = "return"
)
// Map is a map of all keywords used in the language.
var Map = map[string][]byte{
Loop: []byte(Loop),
Return: []byte(Return),
}

7
src/build/token/Keywords.go
View File

@ -1,7 +0,0 @@
package token
// Keywords defines the keywords used in the language.
var Keywords = map[string]bool{
"return": true,
"loop": true,
}

4
src/build/token/Token.go
View File

@ -6,9 +6,9 @@ import "fmt"
// The characters that make up an identifier are grouped into a single token.
// This makes parsing easier and allows us to do better syntax checks.
type Token struct {
Kind Kind
Position int
Bytes []byte
Position int
Kind Kind
}
// End returns the position after the token.

42
src/build/token/Tokenize.go
View File

@ -1,5 +1,7 @@
package token
import "git.akyoto.dev/cli/q/src/build/keyword"
// Pre-allocate these byte buffers so we can re-use them
// instead of allocating a new buffer every time.
var (
@ -26,35 +28,35 @@ func Tokenize(buffer []byte) List {
case ' ', '\t':
// Separator
case ',':
tokens = append(tokens, Token{Separator, i, separatorBytes})
tokens = append(tokens, Token{Kind: Separator, Position: i, Bytes: separatorBytes})
// Parentheses start
case '(':
tokens = append(tokens, Token{GroupStart, i, groupStartBytes})
tokens = append(tokens, Token{Kind: GroupStart, Position: i, Bytes: groupStartBytes})
// Parentheses end
case ')':
tokens = append(tokens, Token{GroupEnd, i, groupEndBytes})
tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Bytes: groupEndBytes})
// Block start
case '{':
tokens = append(tokens, Token{BlockStart, i, blockStartBytes})
tokens = append(tokens, Token{Kind: BlockStart, Position: i, Bytes: blockStartBytes})
// Block end
case '}':
tokens = append(tokens, Token{BlockEnd, i, blockEndBytes})
tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Bytes: blockEndBytes})
// Array start
case '[':
tokens = append(tokens, Token{ArrayStart, i, arrayStartBytes})
tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Bytes: arrayStartBytes})
// Array end
case ']':
tokens = append(tokens, Token{ArrayEnd, i, arrayEndBytes})
tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Bytes: arrayEndBytes})
// New line
case '\n':
tokens = append(tokens, Token{NewLine, i, newLineBytes})
tokens = append(tokens, Token{Kind: NewLine, Position: i, Bytes: newLineBytes})
// Comment
case '/':
@ -66,7 +68,7 @@ func Tokenize(buffer []byte) List {
i++
}
tokens = append(tokens, Token{Operator, position, buffer[position:i]})
tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]})
continue
}
@ -76,7 +78,7 @@ func Tokenize(buffer []byte) List {
i++
}
tokens = append(tokens, Token{Comment, position, buffer[position:i]})
tokens = append(tokens, Token{Kind: Comment, Position: position, Bytes: buffer[position:i]})
continue
// String
@ -95,7 +97,7 @@ func Tokenize(buffer []byte) List {
i++
}
tokens = append(tokens, Token{String, start, buffer[start:end]})
tokens = append(tokens, Token{Kind: String, Position: start, Bytes: buffer[start:end]})
continue
default:
@ -108,13 +110,15 @@ func Tokenize(buffer []byte) List {
i++
}
token := Token{Identifier, position, buffer[position:i]}
identifier := buffer[position:i]
keyword, isKeyword := keyword.Map[string(identifier)]
if Keywords[string(token.Bytes)] {
token.Kind = Keyword
if isKeyword {
tokens = append(tokens, Token{Kind: Keyword, Position: position, Bytes: keyword})
} else {
tokens = append(tokens, Token{Kind: Identifier, Position: position, Bytes: identifier})
}
tokens = append(tokens, token)
continue
}
@ -127,7 +131,7 @@ func Tokenize(buffer []byte) List {
i++
}
tokens = append(tokens, Token{Number, position, buffer[position:i]})
tokens = append(tokens, Token{Kind: Number, Position: position, Bytes: buffer[position:i]})
continue
}
@ -140,18 +144,18 @@ func Tokenize(buffer []byte) List {
i++
}
tokens = append(tokens, Token{Operator, position, buffer[position:i]})
tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]})
continue
}
// Invalid character
tokens = append(tokens, Token{Invalid, i, buffer[i : i+1]})
tokens = append(tokens, Token{Kind: Invalid, Position: i, Bytes: buffer[i : i+1]})
}
i++
}
tokens = append(tokens, Token{EOF, i, nil})
tokens = append(tokens, Token{Kind: EOF, Position: i, Bytes: nil})
return tokens
}

2
src/errors/Error.go
View File

@ -13,8 +13,8 @@ import (
type Error struct {
Err error
File *fs.File
Position int
Stack string
Position int
}
// New generates an error message at the current token position.