Reduced token size

src/build/ast/AST.go

@@ -1,6 +1,4 @@
 package ast
 
-import "fmt"
-
-type Node fmt.Stringer
+type Node any
 type AST []Node

src/build/ast/Assign.go

@@ -8,7 +8,3 @@ import (
 type Assign struct {
 	Expression *expression.Expression
 }
-
-func (node *Assign) String() string {
-	return node.Expression.String()
-}

src/build/ast/Call.go

@@ -6,7 +6,3 @@ import "git.akyoto.dev/cli/q/src/build/expression"
 type Call struct {
 	Expression *expression.Expression
 }
-
-func (node *Call) String() string {
-	return node.Expression.String()
-}

src/build/ast/Count.go

@@ -3,31 +3,31 @@ package ast
 import "git.akyoto.dev/cli/q/src/build/token"
 
 // Count counts how often the given token appears in the AST.
-func Count(body AST, kind token.Kind, name string) int {
+func Count(body AST, buffer []byte, kind token.Kind, name string) int {
 	count := 0
 
 	for _, node := range body {
 		switch node := node.(type) {
 		case *Assign:
-			count += node.Expression.Count(kind, name)
+			count += node.Expression.Count(buffer, kind, name)
 
 		case *Call:
-			count += node.Expression.Count(kind, name)
+			count += node.Expression.Count(buffer, kind, name)
 
 		case *Define:
-			count += node.Value.Count(kind, name)
+			count += node.Value.Count(buffer, kind, name)
 
 		case *Return:
 			if node.Value != nil {
-				count += node.Value.Count(kind, name)
+				count += node.Value.Count(buffer, kind, name)
 			}
 
 		case *If:
-			count += node.Condition.Count(kind, name)
-			count += Count(node.Body, kind, name)
+			count += node.Condition.Count(buffer, kind, name)
+			count += Count(node.Body, buffer, kind, name)
 
 		case *Loop:
-			count += Count(node.Body, kind, name)
+			count += Count(node.Body, buffer, kind, name)
 
 		default:
 			panic("unknown AST type")

src/build/ast/Define.go

@@ -1,8 +1,6 @@
 package ast
 
 import (
-	"fmt"
-
 	"git.akyoto.dev/cli/q/src/build/expression"
 	"git.akyoto.dev/cli/q/src/build/token"
 )
@@ -12,7 +10,3 @@ type Define struct {
 	Value *expression.Expression
 	Name  token.Token
 }
-
-func (node *Define) String() string {
-	return fmt.Sprintf("(= %s %s)", node.Name.Text(), node.Value)
-}

src/build/ast/If.go

@@ -1,8 +1,6 @@
 package ast
 
 import (
-	"fmt"
-
 	"git.akyoto.dev/cli/q/src/build/expression"
 )
 
@@ -11,7 +9,3 @@ type If struct {
 	Condition *expression.Expression
 	Body      AST
 }
-
-func (node *If) String() string {
-	return fmt.Sprintf("(if %s %s)", node.Condition, node.Body)
-}

src/build/ast/Loop.go

@@ -1,12 +1,6 @@
 package ast
 
-import "fmt"
-
 // Loop represents a block of repeatable statements.
 type Loop struct {
 	Body AST
 }
-
-func (node *Loop) String() string {
-	return fmt.Sprintf("(loop %s)", node.Body)
-}

src/build/ast/Parse.go

@@ -3,16 +3,15 @@ package ast
 import (
 	"git.akyoto.dev/cli/q/src/build/errors"
 	"git.akyoto.dev/cli/q/src/build/expression"
-	"git.akyoto.dev/cli/q/src/build/keyword"
 	"git.akyoto.dev/cli/q/src/build/token"
 )
 
 // Parse generates an AST from a list of tokens.
-func Parse(tokens token.List) (AST, error) {
+func Parse(tokens []token.Token, buffer []byte) (AST, error) {
 	tree := make(AST, 0, len(tokens)/64)
 
 	err := EachInstruction(tokens, func(instruction token.List) error {
-		node, err := toASTNode(instruction)
+		node, err := toASTNode(instruction, buffer)
 
 		if err == nil && node != nil {
 			tree = append(tree, node)
@@ -25,11 +24,9 @@ func Parse(tokens token.List) (AST, error) {
 }
 
 // toASTNode generates an AST node from an instruction.
-func toASTNode(tokens token.List) (Node, error) {
-	if tokens[0].Kind == token.Keyword {
-		word := tokens[0].Text()
-
-		if word == keyword.Return {
+func toASTNode(tokens token.List, buffer []byte) (Node, error) {
+	if tokens[0].IsKeyword() {
+		if tokens[0].Kind == token.Return {
 			if len(tokens) == 1 {
 				return &Return{}, nil
 			}
@@ -38,7 +35,7 @@ func toASTNode(tokens token.List) (Node, error) {
 			return &Return{Value: value}, nil
 		}
 
-		if keywordHasBlock(word) {
+		if keywordHasBlock(tokens[0].Kind) {
 			blockStart := tokens.IndexKind(token.BlockStart)
 			blockEnd := tokens.LastIndexKind(token.BlockEnd)
 
@@ -50,19 +47,19 @@ func toASTNode(tokens token.List) (Node, error) {
 				return nil, errors.New(errors.MissingBlockEnd, nil, tokens[len(tokens)-1].End())
 			}
 
-			body, err := Parse(tokens[blockStart+1 : blockEnd])
+			body, err := Parse(tokens[blockStart+1:blockEnd], buffer)
 
-			switch word {
-			case keyword.If:
+			switch tokens[0].Kind {
+			case token.If:
 				condition := expression.Parse(tokens[1:blockStart])
 				return &If{Condition: condition, Body: body}, err
 
-			case keyword.Loop:
+			case token.Loop:
 				return &Loop{Body: body}, err
 			}
 		}
 
-		return nil, errors.New(&errors.KeywordNotImplemented{Keyword: word}, nil, tokens[0].Position)
+		return nil, errors.New(&errors.KeywordNotImplemented{Keyword: tokens[0].Text(buffer)}, nil, tokens[0].Position)
 	}
 
 	expr := expression.Parse(tokens)
@@ -92,26 +89,26 @@ func toASTNode(tokens token.List) (Node, error) {
 		return &Call{Expression: expr}, nil
 
 	default:
-		return nil, errors.New(&errors.InvalidInstruction{Instruction: expr.Token.Text()}, nil, expr.Token.Position)
+		return nil, errors.New(&errors.InvalidInstruction{Instruction: expr.Token.Text(buffer)}, nil, expr.Token.Position)
 	}
 }
 
 // IsAssignment returns true if the expression is an assignment.
 func IsAssignment(expr *expression.Expression) bool {
-	return expr.Token.Kind == token.Operator && expr.Token.Bytes[len(expr.Token.Bytes)-1] == '='
+	return expr.Token.IsAssignment()
 }
 
 // IsFunctionCall returns true if the expression is a function call.
 func IsFunctionCall(expr *expression.Expression) bool {
-	return expr.Token.Kind == token.Operator && expr.Token.Text() == "λ"
+	return expr.Token.Kind == token.Call
 }
 
 // IsVariableDefinition returns true if the expression is a variable definition.
 func IsVariableDefinition(expr *expression.Expression) bool {
-	return expr.Token.Kind == token.Operator && expr.Token.Text() == ":="
+	return expr.Token.Kind == token.Define
 }
 
 // keywordHasBlock returns true if the keyword requires a block.
-func keywordHasBlock(word string) bool {
-	return word == keyword.If || word == keyword.Loop
+func keywordHasBlock(kind token.Kind) bool {
+	return kind == token.If || kind == token.Loop
 }

src/build/ast/Return.go

@@ -1,8 +1,6 @@
 package ast
 
 import (
-	"fmt"
-
 	"git.akyoto.dev/cli/q/src/build/expression"
 )
 
@@ -10,7 +8,3 @@ import (
 type Return struct {
 	Value *expression.Expression
 }
-
-func (node *Return) String() string {
-	return fmt.Sprintf("(return %s)", node.Value)
-}

src/build/core/Compare.go

@@ -13,7 +13,7 @@ func (f *Function) Compare(comparison *expression.Expression) error {
 	right := comparison.Children[1]
 
 	if left.IsLeaf() && left.Token.Kind == token.Identifier {
-		name := left.Token.Text()
+		name := left.Token.Text(f.File.Bytes)
 		variable := f.VariableByName(name)
 
 		if variable == nil {

src/build/core/CompileAssign.go

@@ -16,7 +16,7 @@ func (f *Function) CompileAssign(node *ast.Assign) error {
 	right := node.Expression.Children[1]
 
 	if left.IsLeaf() {
-		name := left.Token.Text()
+		name := left.Token.Text(f.File.Bytes)
 		variable := f.VariableByName(name)
 
 		if variable == nil {
@@ -27,8 +27,8 @@ func (f *Function) CompileAssign(node *ast.Assign) error {
 		return f.Execute(operator, variable.Register, right)
 	}
 
-	if left.Token.Kind == token.Operator && left.Token.Text() == "@" {
-		name := left.Children[0].Token.Text()
+	if left.Token.Kind == token.Array {
+		name := left.Children[0].Token.Text(f.File.Bytes)
 		variable := f.VariableByName(name)
 
 		if variable == nil {
@@ -38,13 +38,13 @@ func (f *Function) CompileAssign(node *ast.Assign) error {
 		defer f.useVariable(variable)
 
 		index := left.Children[1]
-		offset, err := strconv.Atoi(index.Token.Text())
+		offset, err := strconv.Atoi(index.Token.Text(f.File.Bytes))
 
 		if err != nil {
 			return err
 		}
 
-		num, err := strconv.Atoi(right.Token.Text())
+		num, err := strconv.Atoi(right.Token.Text(f.File.Bytes))
 
 		if err != nil {
 			return err

src/build/core/CompileCall.go

@@ -15,9 +15,9 @@ func (f *Function) CompileCall(root *expression.Expression) error {
 	funcName := ""
 
 	if funcNameRoot.IsLeaf() {
-		funcName = funcNameRoot.Token.Text()
+		funcName = funcNameRoot.Token.Text(f.File.Bytes)
 	} else {
-		funcName = funcNameRoot.Children[0].Token.Text() + funcNameRoot.Token.Text() + funcNameRoot.Children[1].Token.Text()
+		funcName = funcNameRoot.Children[0].Token.Text(f.File.Bytes) + funcNameRoot.Token.Text(f.File.Bytes) + funcNameRoot.Children[1].Token.Text(f.File.Bytes)
 	}
 
 	isSyscall := funcName == "syscall"

src/build/core/CompileCondition.go

@@ -4,12 +4,13 @@ import (
 	"fmt"
 
 	"git.akyoto.dev/cli/q/src/build/expression"
+	"git.akyoto.dev/cli/q/src/build/token"
 )
 
 // CompileCondition inserts code to jump to the start label or end label depending on the truth of the condition.
 func (f *Function) CompileCondition(condition *expression.Expression, successLabel string, failLabel string) error {
-	switch condition.Token.Text() {
-	case "||":
+	switch condition.Token.Kind {
+	case token.LogicalOr:
 		f.count.subBranch++
 		leftFailLabel := fmt.Sprintf("%s_false_%d", f.Name, f.count.subBranch)
 
@@ -21,22 +22,22 @@ func (f *Function) CompileCondition(condition *expression.Expression, successLab
 			return err
 		}
 
-		f.JumpIfTrue(left.Token.Text(), successLabel)
+		f.JumpIfTrue(left.Token.Kind, successLabel)
 
 		// Right
 		f.AddLabel(leftFailLabel)
 		right := condition.Children[1]
 		err = f.CompileCondition(right, successLabel, failLabel)
 
-		if condition.Parent != nil && condition.Parent.Token.Text() == "||" && condition != condition.Parent.LastChild() {
-			f.JumpIfTrue(right.Token.Text(), successLabel)
+		if condition.Parent != nil && condition.Parent.Token.Kind == token.LogicalOr && condition != condition.Parent.LastChild() {
+			f.JumpIfTrue(right.Token.Kind, successLabel)
 		} else {
-			f.JumpIfFalse(right.Token.Text(), failLabel)
+			f.JumpIfFalse(right.Token.Kind, failLabel)
 		}
 
 		return err
 
-	case "&&":
+	case token.LogicalAnd:
 		f.count.subBranch++
 		leftSuccessLabel := fmt.Sprintf("%s_true_%d", f.Name, f.count.subBranch)
 
@@ -48,17 +49,17 @@ func (f *Function) CompileCondition(condition *expression.Expression, successLab
 			return err
 		}
 
-		f.JumpIfFalse(left.Token.Text(), failLabel)
+		f.JumpIfFalse(left.Token.Kind, failLabel)
 
 		// Right
 		f.AddLabel(leftSuccessLabel)
 		right := condition.Children[1]
 		err = f.CompileCondition(right, successLabel, failLabel)
 
-		if condition.Parent != nil && condition.Parent.Token.Text() == "||" && condition != condition.Parent.LastChild() {
-			f.JumpIfTrue(right.Token.Text(), successLabel)
+		if condition.Parent != nil && condition.Parent.Token.Kind == token.LogicalOr && condition != condition.Parent.LastChild() {
+			f.JumpIfTrue(right.Token.Kind, successLabel)
 		} else {
-			f.JumpIfFalse(right.Token.Text(), failLabel)
+			f.JumpIfFalse(right.Token.Kind, failLabel)
 		}
 
 		return err
@@ -67,7 +68,7 @@ func (f *Function) CompileCondition(condition *expression.Expression, successLab
 		err := f.Compare(condition)
 
 		if condition.Parent == nil {
-			f.JumpIfFalse(condition.Token.Text(), failLabel)
+			f.JumpIfFalse(condition.Token.Kind, failLabel)
 		}
 
 		return err

src/build/core/CompileDefinition.go

@@ -8,13 +8,13 @@ import (
 
 // CompileDefinition compiles a variable definition.
 func (f *Function) CompileDefinition(node *ast.Define) error {
-	name := node.Name.Text()
+	name := node.Name.Text(f.File.Bytes)
 
 	if f.identifierExists(name) {
 		return errors.New(&errors.VariableAlreadyExists{Name: name}, f.File, node.Name.Position)
 	}
 
-	uses := token.Count(f.Body, token.Identifier, name) - 1
+	uses := token.Count(f.Body, f.File.Bytes, token.Identifier, name) - 1
 
 	if uses == 0 {
 		return errors.New(&errors.UnusedVariable{Name: name}, f.File, node.Name.Position)

src/build/core/CompileIf.go

@@ -18,7 +18,7 @@ func (f *Function) CompileIf(branch *ast.If) error {
 	}
 
 	f.AddLabel(success)
-	f.PushScope(branch.Body)
+	f.PushScope(branch.Body, f.File.Bytes)
 	err = f.CompileAST(branch.Body)
 	f.PopScope()
 	f.AddLabel(fail)

src/build/core/CompileLoop.go

@@ -12,7 +12,7 @@ func (f *Function) CompileLoop(loop *ast.Loop) error {
 	f.count.loop++
 	label := fmt.Sprintf("%s_loop_%d", f.Name, f.count.loop)
 	f.AddLabel(label)
-	scope := f.PushScope(loop.Body)
+	scope := f.PushScope(loop.Body, f.File.Bytes)
 	scope.InLoop = true
 	err := f.CompileAST(loop.Body)
 	f.Jump(asm.JUMP, label)

src/build/core/CompileTokens.go

@@ -7,8 +7,8 @@ import (
 )
 
 // CompileTokens compiles a token list.
-func (f *Function) CompileTokens(tokens token.List) error {
-	body, err := ast.Parse(tokens)
+func (f *Function) CompileTokens(tokens []token.Token) error {
+	body, err := ast.Parse(tokens, f.File.Bytes)
 
 	if err != nil {
 		err.(*errors.Error).File = f.File

src/build/core/ExecuteLeaf.go

@@ -12,7 +12,7 @@ import (
 func (f *Function) ExecuteLeaf(operation token.Token, register cpu.Register, operand token.Token) error {
 	switch operand.Kind {
 	case token.Identifier:
-		name := operand.Text()
+		name := operand.Text(f.File.Bytes)
 		variable := f.VariableByName(name)
 
 		if variable == nil {
@@ -23,7 +23,7 @@ func (f *Function) ExecuteLeaf(operation token.Token, register cpu.Register, ope
 		return f.ExecuteRegisterRegister(operation, register, variable.Register)
 
 	case token.Number:
-		value := operand.Text()
+		value := operand.Text(f.File.Bytes)
 		number, err := strconv.Atoi(value)
 
 		if err != nil {
@@ -33,7 +33,7 @@ func (f *Function) ExecuteLeaf(operation token.Token, register cpu.Register, ope
 		return f.ExecuteRegisterNumber(operation, register, number)
 
 	case token.String:
-		if operation.Text() == "=" {
+		if operation.Kind == token.Assign {
 			return f.TokenToRegister(operand, register)
 		}
 	}

src/build/core/ExecuteRegisterNumber.go

@@ -9,27 +9,27 @@ import (
 
 // ExecuteRegisterNumber performs an operation on a register and a number.
 func (f *Function) ExecuteRegisterNumber(operation token.Token, register cpu.Register, number int) error {
-	switch operation.Text() {
-	case "+", "+=":
+	switch operation.Kind {
+	case token.Add, token.AddAssign:
 		f.RegisterNumber(asm.ADD, register, number)
 
-	case "-", "-=":
+	case token.Sub, token.SubAssign:
 		f.RegisterNumber(asm.SUB, register, number)
 
-	case "*", "*=":
+	case token.Mul, token.MulAssign:
 		f.RegisterNumber(asm.MUL, register, number)
 
-	case "/", "/=":
+	case token.Div, token.DivAssign:
 		f.RegisterNumber(asm.DIV, register, number)
 
-	case "==", "!=", "<", "<=", ">", ">=":
+	case token.Equal, token.NotEqual, token.Less, token.LessEqual, token.Greater, token.GreaterEqual:
 		f.RegisterNumber(asm.COMPARE, register, number)
 
-	case "=":
+	case token.Assign:
 		f.RegisterNumber(asm.MOVE, register, number)
 
 	default:
-		return errors.New(&errors.InvalidOperator{Operator: operation.Text()}, f.File, operation.Position)
+		return errors.New(&errors.InvalidOperator{Operator: operation.Text(f.File.Bytes)}, f.File, operation.Position)
 	}
 
 	return nil

src/build/core/ExecuteRegisterRegister.go

@@ -9,27 +9,27 @@ import (
 
 // ExecuteRegisterRegister performs an operation on two registers.
 func (f *Function) ExecuteRegisterRegister(operation token.Token, destination cpu.Register, source cpu.Register) error {
-	switch operation.Text() {
-	case "+", "+=":
+	switch operation.Kind {
+	case token.Add, token.AddAssign:
 		f.RegisterRegister(asm.ADD, destination, source)
 
-	case "-", "-=":
+	case token.Sub, token.SubAssign:
 		f.RegisterRegister(asm.SUB, destination, source)
 
-	case "*", "*=":
+	case token.Mul, token.MulAssign:
 		f.RegisterRegister(asm.MUL, destination, source)
 
-	case "/", "/=":
+	case token.Div, token.DivAssign:
 		f.RegisterRegister(asm.DIV, destination, source)
 
-	case "==", "!=", "<", "<=", ">", ">=":
+	case token.Equal, token.NotEqual, token.Less, token.LessEqual, token.Greater, token.GreaterEqual:
 		f.RegisterRegister(asm.COMPARE, destination, source)
 
-	case "=":
+	case token.Assign:
 		f.RegisterRegister(asm.MOVE, destination, source)
 
 	default:
-		return errors.New(&errors.InvalidOperator{Operator: operation.Text()}, f.File, operation.Position)
+		return errors.New(&errors.InvalidOperator{Operator: operation.Text(f.File.Bytes)}, f.File, operation.Position)
 	}
 
 	return nil

src/build/core/Function.go

@@ -13,7 +13,7 @@ type Function struct {
 	scope.Stack
 	Name            string
 	File            *fs.File
-	Body            token.List
+	Body            []token.Token
 	Assembler       asm.Assembler
 	Functions       map[string]*Function
 	Err             error

src/build/core/JumpIfFalse.go

@@ -2,22 +2,23 @@ package core
 
 import (
 	"git.akyoto.dev/cli/q/src/build/asm"
+	"git.akyoto.dev/cli/q/src/build/token"
 )
 
 // JumpIfFalse jumps to the label if the previous comparison was false.
-func (f *Function) JumpIfFalse(operator string, label string) {
+func (f *Function) JumpIfFalse(operator token.Kind, label string) {
 	switch operator {
-	case "==":
+	case token.Equal:
 		f.Jump(asm.JNE, label)
-	case "!=":
+	case token.NotEqual:
 		f.Jump(asm.JE, label)
-	case ">":
+	case token.Greater:
 		f.Jump(asm.JLE, label)
-	case "<":
+	case token.Less:
 		f.Jump(asm.JGE, label)
-	case ">=":
+	case token.GreaterEqual:
 		f.Jump(asm.JL, label)
-	case "<=":
+	case token.LessEqual:
 		f.Jump(asm.JG, label)
 	}
 }

src/build/core/JumpIfTrue.go

@@ -2,22 +2,23 @@ package core
 
 import (
 	"git.akyoto.dev/cli/q/src/build/asm"
+	"git.akyoto.dev/cli/q/src/build/token"
 )
 
 // JumpIfTrue jumps to the label if the previous comparison was true.
-func (f *Function) JumpIfTrue(operator string, label string) {
+func (f *Function) JumpIfTrue(operator token.Kind, label string) {
 	switch operator {
-	case "==":
+	case token.Equal:
 		f.Jump(asm.JE, label)
-	case "!=":
+	case token.NotEqual:
 		f.Jump(asm.JNE, label)
-	case ">":
+	case token.Greater:
 		f.Jump(asm.JG, label)
-	case "<":
+	case token.Less:
 		f.Jump(asm.JL, label)
-	case ">=":
+	case token.GreaterEqual:
 		f.Jump(asm.JGE, label)
-	case "<=":
+	case token.LessEqual:
 		f.Jump(asm.JLE, label)
 	}
 }

src/build/core/NewFunction.go

@@ -10,7 +10,7 @@ import (
 )
 
 // NewFunction creates a new function.
-func NewFunction(name string, file *fs.File, body token.List) *Function {
+func NewFunction(name string, file *fs.File, body []token.Token) *Function {
 	return &Function{
 		Name: name,
 		File: file,

src/build/core/TokenToRegister.go

@@ -15,7 +15,7 @@ import (
 func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error {
 	switch t.Kind {
 	case token.Identifier:
-		name := t.Text()
+		name := t.Text(f.File.Bytes)
 		variable := f.VariableByName(name)
 
 		if variable == nil {
@@ -27,7 +27,7 @@ func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error {
 		return nil
 
 	case token.Number:
-		value := t.Text()
+		value := t.Text(f.File.Bytes)
 		n, err := strconv.Atoi(value)
 
 		if err != nil {
@@ -40,7 +40,7 @@ func (f *Function) TokenToRegister(t token.Token, register cpu.Register) error {
 	case token.String:
 		f.count.data++
 		label := fmt.Sprintf("%s_data_%d", f.Name, f.count.data)
-		value := t.Bytes[1 : len(t.Bytes)-1]
+		value := t.Bytes(f.File.Bytes)[1 : t.Length-1]
 		f.Assembler.SetData(label, value)
 		f.RegisterLabel(asm.MOVE, register, label)
 		return nil

src/build/expression/Expression.go

@@ -37,11 +37,11 @@ func (expr *Expression) AddChild(child *Expression) {
 }
 
 // Count counts how often the given token appears in the expression.
-func (expr *Expression) Count(kind token.Kind, name string) int {
+func (expr *Expression) Count(buffer []byte, kind token.Kind, name string) int {
 	count := 0
 
 	expr.EachLeaf(func(leaf *Expression) error {
-		if leaf.Token.Kind == kind && leaf.Token.Text() == name {
+		if leaf.Token.Kind == kind && leaf.Token.Text(buffer) == name {
 			count++
 		}
 
@@ -112,25 +112,33 @@ func (expr *Expression) LastChild() *Expression {
 }
 
 // String generates a textual representation of the expression.
-func (expr *Expression) String() string {
+func (expr *Expression) String(data []byte) string {
 	builder := strings.Builder{}
-	expr.write(&builder)
+	expr.write(&builder, data)
 	return builder.String()
 }
 
 // write generates a textual representation of the expression.
-func (expr *Expression) write(builder *strings.Builder) {
+func (expr *Expression) write(builder *strings.Builder, data []byte) {
 	if expr.IsLeaf() {
-		builder.WriteString(expr.Token.Text())
+		builder.WriteString(expr.Token.Text(data))
 		return
 	}
 
 	builder.WriteByte('(')
-	builder.WriteString(expr.Token.Text())
+
+	switch expr.Token.Kind {
+	case token.Call:
+		builder.WriteString("λ")
+	case token.Array:
+		builder.WriteString("@")
+	default:
+		builder.WriteString(expr.Token.Text(data))
+	}
 
 	for _, child := range expr.Children {
 		builder.WriteByte(' ')
-		child.write(builder)
+		child.write(builder, data)
 	}
 
 	builder.WriteByte(')')

src/build/expression/Expression_test.go

@@ -95,7 +95,7 @@ func TestParse(t *testing.T) {
 			defer expr.Reset()
 
 			assert.NotNil(t, expr)
-			assert.Equal(t, expr.String(), test.Result)
+			assert.Equal(t, expr.String(src), test.Result)
 		})
 	}
 }
@@ -104,11 +104,11 @@ func TestCount(t *testing.T) {
 	src := []byte("(a+b-c*d)+(a*b-c+d)")
 	tokens := token.Tokenize(src)
 	expr := expression.Parse(tokens)
-	assert.Equal(t, expr.Count(token.Identifier, "a"), 2)
-	assert.Equal(t, expr.Count(token.Identifier, "b"), 2)
-	assert.Equal(t, expr.Count(token.Identifier, "c"), 2)
-	assert.Equal(t, expr.Count(token.Identifier, "d"), 2)
-	assert.Equal(t, expr.Count(token.Identifier, "e"), 0)
+	assert.Equal(t, expr.Count(src, token.Identifier, "a"), 2)
+	assert.Equal(t, expr.Count(src, token.Identifier, "b"), 2)
+	assert.Equal(t, expr.Count(src, token.Identifier, "c"), 2)
+	assert.Equal(t, expr.Count(src, token.Identifier, "d"), 2)
+	assert.Equal(t, expr.Count(src, token.Identifier, "e"), 0)
 }
 
 func TestEachLeaf(t *testing.T) {
@@ -118,7 +118,7 @@ func TestEachLeaf(t *testing.T) {
 	leaves := []string{}
 
 	err := expr.EachLeaf(func(leaf *expression.Expression) error {
-		leaves = append(leaves, leaf.Token.Text())
+		leaves = append(leaves, leaf.Token.Text(src))
 		return nil
 	})
 
@@ -140,7 +140,7 @@ func TestEachParameter(t *testing.T) {
 
 	err := expression.EachParameter(tokens, func(parameter token.List) error {
 		expr := expression.Parse(parameter)
-		parameters = append(parameters, expr.String())
+		parameters = append(parameters, expr.String(src))
 		return nil
 	})
 
@@ -178,17 +178,3 @@ func TestNilGroup(t *testing.T) {
 	expr := expression.Parse(tokens)
 	assert.Nil(t, expr)
 }
-
-func TestInvalidOperator(t *testing.T) {
-	src := []byte("a +++ 2")
-	tokens := token.Tokenize(src)
-	expr := expression.Parse(tokens)
-	assert.Equal(t, expr.String(), "(+++ a 2)")
-}
-
-func TestInvalidOperatorCall(t *testing.T) {
-	src := []byte("+++()")
-	tokens := token.Tokenize(src)
-	expr := expression.Parse(tokens)
-	assert.NotNil(t, expr)
-}

src/build/expression/Operator.go

@@ -15,39 +15,39 @@ type Operator struct {
 
 // Operators defines the operators used in the language.
 // The number corresponds to the operator priority and can not be zero.
-var Operators = map[string]*Operator{
-	".":  {".", 13, 2},
-	"λ":  {"λ", 12, 1},
-	"@":  {"@", 12, 2},
-	"!":  {"!", 11, 1},
-	"*":  {"*", 10, 2},
-	"/":  {"/", 10, 2},
-	"%":  {"%", 10, 2},
-	"+":  {"+", 9, 2},
-	"-":  {"-", 9, 2},
-	">>": {">>", 8, 2},
-	"<<": {"<<", 8, 2},
-	"&":  {"&", 7, 2},
-	"^":  {"^", 6, 2},
-	"|":  {"|", 5, 2},
+var Operators = map[token.Kind]*Operator{
+	token.Period: {".", 13, 2},
+	token.Call:   {"λ", 12, 1},
+	token.Array:  {"@", 12, 2},
+	token.Not:    {"!", 11, 1},
+	token.Mul:    {"*", 10, 2},
+	token.Div:    {"/", 10, 2},
+	token.Mod:    {"%", 10, 2},
+	token.Add:    {"+", 9, 2},
+	token.Sub:    {"-", 9, 2},
+	token.Shr:    {">>", 8, 2},
+	token.Shl:    {"<<", 8, 2},
+	token.And:    {"&", 7, 2},
+	token.Xor:    {"^", 6, 2},
+	token.Or:     {"|", 5, 2},
 
-	">":  {">", 4, 2},
-	"<":  {"<", 4, 2},
-	">=": {">=", 4, 2},
-	"<=": {"<=", 4, 2},
-	"==": {"==", 3, 2},
-	"!=": {"!=", 3, 2},
-	"&&": {"&&", 2, 2},
-	"||": {"||", 1, 2},
+	token.Greater:      {">", 4, 2},
+	token.Less:         {"<", 4, 2},
+	token.GreaterEqual: {">=", 4, 2},
+	token.LessEqual:    {"<=", 4, 2},
+	token.Equal:        {"==", 3, 2},
+	token.NotEqual:     {"!=", 3, 2},
+	token.LogicalAnd:   {"&&", 2, 2},
+	token.LogicalOr:    {"||", 1, 2},
 
-	"=":   {"=", math.MinInt8, 2},
-	":=":  {":=", math.MinInt8, 2},
-	"+=":  {"+=", math.MinInt8, 2},
-	"-=":  {"-=", math.MinInt8, 2},
-	"*=":  {"*=", math.MinInt8, 2},
-	"/=":  {"/=", math.MinInt8, 2},
-	">>=": {">>=", math.MinInt8, 2},
-	"<<=": {"<<=", math.MinInt8, 2},
+	token.Assign:    {"=", math.MinInt8, 2},
+	token.Define:    {":=", math.MinInt8, 2},
+	token.AddAssign: {"+=", math.MinInt8, 2},
+	token.SubAssign: {"-=", math.MinInt8, 2},
+	token.MulAssign: {"*=", math.MinInt8, 2},
+	token.DivAssign: {"/=", math.MinInt8, 2},
+	token.ShrAssign: {">>=", math.MinInt8, 2},
+	token.ShlAssign: {"<<=", math.MinInt8, 2},
 }
 
 func isComplete(expr *Expression) bool {
@@ -59,14 +59,14 @@ func isComplete(expr *Expression) bool {
 		return true
 	}
 
-	if expr.Token.Kind == token.Operator && len(expr.Children) == numOperands(expr.Token.Text()) {
+	if expr.Token.IsOperator() && len(expr.Children) == numOperands(expr.Token.Kind) {
 		return true
 	}
 
 	return false
 }
 
-func numOperands(symbol string) int {
+func numOperands(symbol token.Kind) int {
 	operator, exists := Operators[symbol]
 
 	if !exists {
@@ -76,7 +76,7 @@ func numOperands(symbol string) int {
 	return operator.Operands
 }
 
-func precedence(symbol string) int8 {
+func precedence(symbol token.Kind) int8 {
 	operator, exists := Operators[symbol]
 
 	if !exists {

src/build/expression/Parse.go

@@ -6,13 +6,8 @@ import (
 	"git.akyoto.dev/cli/q/src/build/token"
 )
 
-var (
-	call  = []byte("λ")
-	array = []byte("@")
-)
-
 // Parse generates an expression tree from tokens.
-func Parse(tokens token.List) *Expression {
+func Parse(tokens []token.Token) *Expression {
 	var (
 		cursor        *Expression
 		root          *Expression
@@ -43,20 +38,18 @@ func Parse(tokens token.List) *Expression {
 				parameters := NewList(tokens[groupPosition:i])
 
 				node := New()
-				node.Token.Kind = token.Operator
 				node.Token.Position = tokens[groupPosition].Position
 
 				switch t.Kind {
 				case token.GroupEnd:
-					node.Token.Bytes = call
-					node.Precedence = precedence("λ")
-
+					node.Token.Kind = token.Call
 				case token.ArrayEnd:
-					node.Token.Bytes = array
-					node.Precedence = precedence("@")
+					node.Token.Kind = token.Array
 				}
 
-				if cursor.Token.Kind == token.Operator && node.Precedence > cursor.Precedence {
+				node.Precedence = precedence(node.Token.Kind)
+
+				if cursor.Token.IsOperator() && node.Precedence > cursor.Precedence {
 					cursor.LastChild().Replace(node)
 				} else {
 					if cursor == root {
@@ -108,18 +101,18 @@ func Parse(tokens token.List) *Expression {
 			continue
 		}
 
-		if t.Kind == token.Operator {
+		if t.IsOperator() {
 			if cursor == nil {
 				cursor = NewLeaf(t)
-				cursor.Precedence = precedence(t.Text())
+				cursor.Precedence = precedence(t.Kind)
 				root = cursor
 				continue
 			}
 
 			node := NewLeaf(t)
-			node.Precedence = precedence(t.Text())
+			node.Precedence = precedence(t.Kind)
 
-			if cursor.Token.Kind == token.Operator {
+			if cursor.Token.IsOperator() {
 				oldPrecedence := cursor.Precedence
 				newPrecedence := node.Precedence
 

src/build/fs/File.go

@@ -5,5 +5,6 @@ import "git.akyoto.dev/cli/q/src/build/token"
 // File represents a single source file.
 type File struct {
 	Path   string
+	Bytes  []byte
 	Tokens token.List
 }

src/build/keyword/Keyword.go

@@ -1,16 +0,0 @@
-package keyword
-
-const (
-	If     = "if"
-	Import = "import"
-	Loop   = "loop"
-	Return = "return"
-)
-
-// Map is a map of all keywords used in the language.
-var Map = map[string][]byte{
-	If:     []byte(If),
-	Import: []byte(Import),
-	Loop:   []byte(Loop),
-	Return: []byte(Return),
-}

src/build/scanner/scanFile.go

@@ -11,7 +11,6 @@ import (
 	"git.akyoto.dev/cli/q/src/build/errors"
 	"git.akyoto.dev/cli/q/src/build/expression"
 	"git.akyoto.dev/cli/q/src/build/fs"
-	"git.akyoto.dev/cli/q/src/build/keyword"
 	"git.akyoto.dev/cli/q/src/build/scope"
 	"git.akyoto.dev/cli/q/src/build/token"
 )
@@ -27,8 +26,9 @@ func (s *Scanner) scanFile(path string, pkg string) error {
 	tokens := token.Tokenize(contents)
 
 	file := &fs.File{
-		Tokens: tokens,
 		Path:   path,
+		Bytes:  contents,
+		Tokens: tokens,
 	}
 
 	var (
@@ -42,14 +42,14 @@ func (s *Scanner) scanFile(path string, pkg string) error {
 	)
 
 	for {
-		for i < len(tokens) && tokens[i].Kind == token.Keyword && tokens[i].Text() == keyword.Import {
+		for i < len(tokens) && tokens[i].Kind == token.Import {
 			i++
 
 			if tokens[i].Kind != token.Identifier {
 				panic("expected package name")
 			}
 
-			packageName := tokens[i].Text()
+			packageName := tokens[i].Text(contents)
 			s.queueDirectory(filepath.Join(config.Library, packageName), packageName)
 
 			i++
@@ -75,7 +75,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
 			}
 
 			if tokens[i].Kind == token.Invalid {
-				return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
+				return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text(contents)}, file, tokens[i].Position)
 			}
 
 			if tokens[i].Kind == token.EOF {
@@ -116,7 +116,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
 			}
 
 			if tokens[i].Kind == token.Invalid {
-				return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
+				return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text(contents)}, file, tokens[i].Position)
 			}
 
 			if tokens[i].Kind == token.EOF {
@@ -168,7 +168,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
 			}
 
 			if tokens[i].Kind == token.Invalid {
-				return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
+				return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text(contents)}, file, tokens[i].Position)
 			}
 
 			if tokens[i].Kind == token.EOF {
@@ -191,7 +191,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
 			return errors.New(errors.ExpectedFunctionDefinition, file, tokens[i].Position)
 		}
 
-		name := tokens[nameStart].Text()
+		name := tokens[nameStart].Text(contents)
 		body := tokens[bodyStart:i]
 
 		if pkg != "" {
@@ -207,9 +207,9 @@ func (s *Scanner) scanFile(path string, pkg string) error {
 				return errors.New(errors.NotImplemented, file, tokens[0].Position)
 			}
 
-			name := tokens[0].Text()
+			name := tokens[0].Text(contents)
 			register := x64.CallRegisters[count]
-			uses := token.Count(function.Body, token.Identifier, name)
+			uses := token.Count(function.Body, contents, token.Identifier, name)
 
 			if uses == 0 {
 				return errors.New(&errors.UnusedVariable{Name: name}, file, tokens[0].Position)

src/build/scope/Stack.go

@@ -20,7 +20,7 @@ func (stack *Stack) PopScope() {
 }
 
 // PushScope pushes a new scope to the top of the stack.
-func (stack *Stack) PushScope(body ast.AST) *Scope {
+func (stack *Stack) PushScope(body ast.AST, buffer []byte) *Scope {
 	s := &Scope{}
 
 	if len(stack.Scopes) > 0 {
@@ -30,7 +30,7 @@ func (stack *Stack) PushScope(body ast.AST) *Scope {
 		s.InLoop = lastScope.InLoop
 
 		for k, v := range lastScope.Variables {
-			count := ast.Count(body, token.Identifier, v.Name)
+			count := ast.Count(body, buffer, token.Identifier, v.Name)
 
 			if count == 0 {
 				continue

src/build/token/Count.go

@@ -1,11 +1,11 @@
 package token
 
 // Count counts how often the given token appears in the token list.
-func Count(tokens List, kind Kind, name string) int {
+func Count(tokens []Token, buffer []byte, kind Kind, name string) int {
 	count := 0
 
 	for _, t := range tokens {
-		if t.Kind == Identifier && t.Text() == name {
+		if t.Kind == Identifier && t.Text(buffer) == name {
 			count++
 		}
 	}

src/build/token/Count_test.go

@@ -8,9 +8,10 @@ import (
 )
 
 func TestCount(t *testing.T) {
-	tokens := token.Tokenize([]byte(`a b b c c c`))
-	assert.Equal(t, token.Count(tokens, token.Identifier, "a"), 1)
-	assert.Equal(t, token.Count(tokens, token.Identifier, "b"), 2)
-	assert.Equal(t, token.Count(tokens, token.Identifier, "c"), 3)
-	assert.Equal(t, token.Count(tokens, token.Identifier, "d"), 0)
+	buffer := []byte(`a b b c c c`)
+	tokens := token.Tokenize(buffer)
+	assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "a"), 1)
+	assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "b"), 2)
+	assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "c"), 3)
+	assert.Equal(t, token.Count(tokens, buffer, token.Identifier, "d"), 0)
 }

src/build/token/Keywords.go

@@ -0,0 +1,9 @@
+package token
+
+// Keywords is a map of all keywords used in the language.
+var Keywords = map[string]Kind{
+	"if":     If,
+	"import": Import,
+	"loop":   Loop,
+	"return": Return,
+}

src/build/token/Kind.go

@@ -4,73 +4,62 @@ package token
 type Kind uint8
 
 const (
-	// Invalid represents an invalid token.
-	Invalid Kind = iota
-
-	// EOF represents the end of file.
-	EOF
-
-	// NewLine represents the newline character.
-	NewLine
-
-	// Identifier represents a series of characters used to identify a variable or function.
-	Identifier
-
-	// Keyword represents a language keyword.
-	Keyword
-
-	// String represents an uninterpreted series of characters in the source code.
-	String
-
-	// Number represents a series of numerical characters.
-	Number
-
-	// Operator represents a mathematical operator.
-	Operator
-
-	// Separator represents a comma.
-	Separator
-
-	// Comment represents a comment.
-	Comment
-
-	// GroupStart represents '('.
-	GroupStart
-
-	// GroupEnd represents ')'.
-	GroupEnd
-
-	// BlockStart represents '{'.
-	BlockStart
-
-	// BlockEnd represents '}'.
-	BlockEnd
-
-	// ArrayStart represents '['.
-	ArrayStart
-
-	// ArrayEnd represents ']'.
-	ArrayEnd
+	Invalid         Kind = iota // Invalid is an invalid token.
+	EOF                         // EOF is the end of file.
+	NewLine                     // NewLine is the newline character.
+	Identifier                  // Identifier is a series of characters used to identify a variable or function.
+	Number                      // Number is a series of numerical characters.
+	String                      // String is an uninterpreted series of characters in the source code.
+	Comment                     // Comment is a comment.
+	Separator                   // ,
+	GroupStart                  // (
+	GroupEnd                    // )
+	BlockStart                  // {
+	BlockEnd                    // }
+	ArrayStart                  // [
+	ArrayEnd                    // ]
+	_keywords                   // <keywords>
+	If                          // if
+	Import                      // import
+	Loop                        // loop
+	Return                      // return
+	_keywordsEnd                // </keywords>
+	_operators                  // <operators>
+	Add                         // +
+	Sub                         // -
+	Mul                         // *
+	Div                         // /
+	Mod                         // %
+	And                         // &
+	Or                          // |
+	Xor                         // ^
+	Shl                         // <<
+	Shr                         // >>
+	LogicalAnd                  // &&
+	LogicalOr                   // ||
+	Equal                       // ==
+	Less                        // <
+	Greater                     // >
+	Not                         // !
+	NotEqual                    // !=
+	LessEqual                   // <=
+	GreaterEqual                // >=
+	Define                      // :=
+	Period                      // .
+	Call                        // x()
+	Array                       // [x]
+	_assignments                // <assignments>
+	Assign                      // =
+	AddAssign                   // +=
+	SubAssign                   // -=
+	MulAssign                   // *=
+	DivAssign                   // /=
+	ModAssign                   // %=
+	AndAssign                   // &=
+	OrAssign                    // |=
+	XorAssign                   // ^=
+	ShlAssign                   // <<=
+	ShrAssign                   // >>=
+	_assignmentsEnd             // </assignments>
+	_operatorsEnd               // </operators>
 )
-
-// String returns the text representation.
-func (kind Kind) String() string {
-	return [...]string{
-		"Invalid",
-		"EOF",
-		"NewLine",
-		"Identifier",
-		"Keyword",
-		"String",
-		"Number",
-		"Operator",
-		"Separator",
-		"Comment",
-		"GroupStart",
-		"GroupEnd",
-		"BlockStart",
-		"BlockEnd",
-		"ArrayStart",
-		"ArrayEnd",
-	}[kind]
-}

src/build/token/Kind_test.go

@@ -1,27 +0,0 @@
-package token_test
-
-import (
-	"testing"
-
-	"git.akyoto.dev/cli/q/src/build/token"
-	"git.akyoto.dev/go/assert"
-)
-
-func TestTokenKind(t *testing.T) {
-	assert.Equal(t, token.Invalid.String(), "Invalid")
-	assert.Equal(t, token.EOF.String(), "EOF")
-	assert.Equal(t, token.NewLine.String(), "NewLine")
-	assert.Equal(t, token.Identifier.String(), "Identifier")
-	assert.Equal(t, token.Keyword.String(), "Keyword")
-	assert.Equal(t, token.String.String(), "String")
-	assert.Equal(t, token.Number.String(), "Number")
-	assert.Equal(t, token.Operator.String(), "Operator")
-	assert.Equal(t, token.Separator.String(), "Separator")
-	assert.Equal(t, token.Comment.String(), "Comment")
-	assert.Equal(t, token.GroupStart.String(), "GroupStart")
-	assert.Equal(t, token.GroupEnd.String(), "GroupEnd")
-	assert.Equal(t, token.BlockStart.String(), "BlockStart")
-	assert.Equal(t, token.BlockEnd.String(), "BlockEnd")
-	assert.Equal(t, token.ArrayStart.String(), "ArrayStart")
-	assert.Equal(t, token.ArrayEnd.String(), "ArrayEnd")
-}

src/build/token/Length.go

@@ -0,0 +1,4 @@
+package token
+
+// Length is the data type for storing token lengths.
+type Length = uint16

src/build/token/List.go

@@ -1,9 +1,5 @@
 package token
 
-import (
-	"bytes"
-)
-
 // List is a slice of tokens.
 type List []Token
 
@@ -28,20 +24,3 @@ func (list List) LastIndexKind(kind Kind) int {
 
 	return -1
 }
-
-// String implements string serialization.
-func (list List) String() string {
-	builder := bytes.Buffer{}
-	var last Token
-
-	for _, t := range list {
-		if last.Kind == Keyword || last.Kind == Separator || last.Kind == Operator || t.Kind == Operator {
-			builder.WriteByte(' ')
-		}
-
-		builder.Write(t.Bytes)
-		last = t
-	}
-
-	return builder.String()
-}

src/build/token/Operators.go

@@ -0,0 +1,39 @@
+package token
+
+// Operators is a map of all operators used in the language.
+var Operators = map[string]Kind{
+	".":   Period,
+	"=":   Assign,
+	":=":  Define,
+	"+":   Add,
+	"-":   Sub,
+	"*":   Mul,
+	"/":   Div,
+	"%":   Mod,
+	"&":   And,
+	"|":   Or,
+	"^":   Xor,
+	"<<":  Shl,
+	">>":  Shr,
+	"&&":  LogicalAnd,
+	"||":  LogicalOr,
+	"!":   Not,
+	"==":  Equal,
+	"!=":  NotEqual,
+	">":   Greater,
+	"<":   Less,
+	">=":  GreaterEqual,
+	"<=":  LessEqual,
+	"+=":  AddAssign,
+	"-=":  SubAssign,
+	"*=":  MulAssign,
+	"/=":  DivAssign,
+	"%=":  ModAssign,
+	"&=":  AndAssign,
+	"|=":  OrAssign,
+	"^=":  XorAssign,
+	"<<=": ShlAssign,
+	">>=": ShrAssign,
+	"λ":   Call,
+	"@":   Array,
+}

src/build/token/Token.go

@@ -1,7 +1,6 @@
 package token
 
 import (
-	"fmt"
 	"unsafe"
 )
 
@@ -9,29 +8,44 @@ import (
 // The characters that make up an identifier are grouped into a single token.
 // This makes parsing easier and allows us to do better syntax checks.
 type Token struct {
-	Bytes    []byte
 	Position Position
+	Length   Length
 	Kind     Kind
 }
 
-// End returns the position after the token.
-func (t *Token) End() Position {
-	return t.Position + Position(len(t.Bytes))
+// Bytes returns the byte slice.
+func (t Token) Bytes(buffer []byte) []byte {
+	return buffer[t.Position : t.Position+Position(t.Length)]
 }
 
-// String creates a human readable representation for debugging purposes.
-func (t *Token) String() string {
-	return fmt.Sprintf("%s %s", t.Kind, t.Text())
+// End returns the position after the token.
+func (t Token) End() Position {
+	return t.Position + Position(t.Length)
+}
+
+// IsAssignment returns true if the token is an assignment operator.
+func (t Token) IsAssignment() bool {
+	return t.Kind > _assignments && t.Kind < _assignmentsEnd
+}
+
+// IsKeyword returns true if the token is a keyword.
+func (t Token) IsKeyword() bool {
+	return t.Kind > _keywords && t.Kind < _keywordsEnd
+}
+
+// IsOperator returns true if the token is an operator.
+func (t Token) IsOperator() bool {
+	return t.Kind > _operators && t.Kind < _operatorsEnd
 }
 
 // Reset resets the token to default values.
 func (t *Token) Reset() {
-	t.Kind = Invalid
 	t.Position = 0
-	t.Bytes = nil
+	t.Length = 0
+	t.Kind = Invalid
 }
 
 // Text returns the token text.
-func (t *Token) Text() string {
-	return unsafe.String(unsafe.SliceData(t.Bytes), len(t.Bytes))
+func (t Token) Text(buffer []byte) string {
+	return unsafe.String(unsafe.SliceData(t.Bytes(buffer)), t.Length)
 }

src/build/token/Token_test.go

@@ -10,8 +10,8 @@ import (
 func TestTokenEnd(t *testing.T) {
 	hello := token.Token{
 		Kind:     token.Identifier,
-		Bytes:    []byte("hello"),
 		Position: 0,
+		Length:   5,
 	}
 
 	assert.Equal(t, hello.End(), 5)
@@ -20,34 +20,33 @@ func TestTokenEnd(t *testing.T) {
 func TestTokenReset(t *testing.T) {
 	hello := token.Token{
 		Kind:     token.Identifier,
-		Bytes:    []byte("hello"),
 		Position: 1,
+		Length:   5,
 	}
 
 	hello.Reset()
-	assert.Nil(t, hello.Bytes)
 	assert.Equal(t, hello.Position, 0)
+	assert.Equal(t, hello.Length, 0)
 	assert.Equal(t, hello.Kind, token.Invalid)
 }
 
-func TestTokenString(t *testing.T) {
-	hello := token.Token{
-		Kind:     token.Identifier,
-		Bytes:    []byte("hello"),
-		Position: 0,
-	}
-
-	assert.Equal(t, hello.String(), "Identifier hello")
-}
-
 func TestTokenText(t *testing.T) {
-	hello := token.Token{Kind: token.Identifier, Bytes: []byte("hello"), Position: 0}
-	comma := token.Token{Kind: token.Separator, Bytes: []byte(","), Position: 5}
-	world := token.Token{Kind: token.Identifier, Bytes: []byte("world"), Position: 7}
+	buffer := []byte("hello, world")
+	hello := token.Token{Kind: token.Identifier, Position: 0, Length: 5}
+	comma := token.Token{Kind: token.Separator, Position: 5, Length: 1}
+	world := token.Token{Kind: token.Identifier, Position: 7, Length: 5}
 
-	assert.Equal(t, hello.Text(), "hello")
-	assert.Equal(t, world.Text(), "world")
-
-	list := token.List{hello, comma, world}
-	assert.Equal(t, list.String(), "hello, world")
+	assert.Equal(t, hello.Text(buffer), "hello")
+	assert.Equal(t, comma.Text(buffer), ",")
+	assert.Equal(t, world.Text(buffer), "world")
+}
+
+func TestTokenGroups(t *testing.T) {
+	assignment := token.Token{Kind: token.Assign}
+	operator := token.Token{Kind: token.Add}
+	keyword := token.Token{Kind: token.If}
+
+	assert.True(t, assignment.IsAssignment())
+	assert.True(t, operator.IsOperator())
+	assert.True(t, keyword.IsKeyword())
 }

src/build/token/Tokenize.go

@@ -1,20 +1,5 @@
 package token
 
-import "git.akyoto.dev/cli/q/src/build/keyword"
-
-// Pre-allocate these byte buffers so we can re-use them
-// instead of allocating a new buffer every time.
-var (
-	groupStartBytes = []byte{'('}
-	groupEndBytes   = []byte{')'}
-	blockStartBytes = []byte{'{'}
-	blockEndBytes   = []byte{'}'}
-	arrayStartBytes = []byte{'['}
-	arrayEndBytes   = []byte{']'}
-	separatorBytes  = []byte{','}
-	newLineBytes    = []byte{'\n'}
-)
-
 // Tokenize turns the file contents into a list of tokens.
 func Tokenize(buffer []byte) List {
 	var (
@@ -26,21 +11,21 @@ func Tokenize(buffer []byte) List {
 		switch buffer[i] {
 		case ' ', '\t':
 		case ',':
-			tokens = append(tokens, Token{Kind: Separator, Position: i, Bytes: separatorBytes})
+			tokens = append(tokens, Token{Kind: Separator, Position: i, Length: 1})
 		case '(':
-			tokens = append(tokens, Token{Kind: GroupStart, Position: i, Bytes: groupStartBytes})
+			tokens = append(tokens, Token{Kind: GroupStart, Position: i, Length: 1})
 		case ')':
-			tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Bytes: groupEndBytes})
+			tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Length: 1})
 		case '{':
-			tokens = append(tokens, Token{Kind: BlockStart, Position: i, Bytes: blockStartBytes})
+			tokens = append(tokens, Token{Kind: BlockStart, Position: i, Length: 1})
 		case '}':
-			tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Bytes: blockEndBytes})
+			tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Length: 1})
 		case '[':
-			tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Bytes: arrayStartBytes})
+			tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Length: 1})
 		case ']':
-			tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Bytes: arrayEndBytes})
+			tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Length: 1})
 		case '\n':
-			tokens = append(tokens, Token{Kind: NewLine, Position: i, Bytes: newLineBytes})
+			tokens = append(tokens, Token{Kind: NewLine, Position: i, Length: 1})
 		case '/':
 			if i+1 >= Position(len(buffer)) || buffer[i+1] != '/' {
 				position := i
@@ -50,7 +35,7 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]})
+				tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
 			} else {
 				position := i
 
@@ -58,7 +43,7 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				tokens = append(tokens, Token{Kind: Comment, Position: position, Bytes: buffer[position:i]})
+				tokens = append(tokens, Token{Kind: Comment, Position: position, Length: Length(i - position)})
 			}
 
 			continue
@@ -78,7 +63,7 @@ func Tokenize(buffer []byte) List {
 				i++
 			}
 
-			tokens = append(tokens, Token{Kind: String, Position: start, Bytes: buffer[start:end]})
+			tokens = append(tokens, Token{Kind: String, Position: start, Length: Length(end - start)})
 			continue
 
 		default:
@@ -91,14 +76,14 @@ func Tokenize(buffer []byte) List {
 				}
 
 				identifier := buffer[position:i]
-				keyword, isKeyword := keyword.Map[string(identifier)]
+				kind := Identifier
+				keyword, isKeyword := Keywords[string(identifier)]
 
 				if isKeyword {
-					tokens = append(tokens, Token{Kind: Keyword, Position: position, Bytes: keyword})
-				} else {
-					tokens = append(tokens, Token{Kind: Identifier, Position: position, Bytes: identifier})
+					kind = keyword
 				}
 
+				tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(len(identifier))})
 				continue
 			}
 
@@ -110,7 +95,7 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				tokens = append(tokens, Token{Kind: Number, Position: position, Bytes: buffer[position:i]})
+				tokens = append(tokens, Token{Kind: Number, Position: position, Length: Length(i - position)})
 				continue
 			}
 
@@ -122,17 +107,17 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]})
+				tokens = append(tokens, Token{Kind: Operators[string(buffer[position:i])], Position: position, Length: Length(i - position)})
 				continue
 			}
 
-			tokens = append(tokens, Token{Kind: Invalid, Position: i, Bytes: buffer[i : i+1]})
+			tokens = append(tokens, Token{Kind: Invalid, Position: i, Length: 1})
 		}
 
 		i++
 	}
 
-	tokens = append(tokens, Token{Kind: EOF, Position: i, Bytes: nil})
+	tokens = append(tokens, Token{Kind: EOF, Position: i, Length: 0})
 	return tokens
 }
 

src/build/token/Tokenize_test.go

@@ -9,394 +9,243 @@ import (
 
 func TestFunction(t *testing.T) {
 	tokens := token.Tokenize([]byte("main(){}"))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.Identifier,
-			Bytes:    []byte("main"),
-			Position: 0,
-		},
-		{
-			Kind:     token.GroupStart,
-			Bytes:    []byte("("),
-			Position: 4,
-		},
-		{
-			Kind:     token.GroupEnd,
-			Bytes:    []byte(")"),
-			Position: 5,
-		},
-		{
-			Kind:     token.BlockStart,
-			Bytes:    []byte("{"),
-			Position: 6,
-		},
-		{
-			Kind:     token.BlockEnd,
-			Bytes:    []byte("}"),
-			Position: 7,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 8,
-		},
-	})
+
+	expected := []token.Kind{
+		token.Identifier,
+		token.GroupStart,
+		token.GroupEnd,
+		token.BlockStart,
+		token.BlockEnd,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 }
 
 func TestKeyword(t *testing.T) {
 	tokens := token.Tokenize([]byte("return x"))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.Keyword,
-			Bytes:    []byte("return"),
-			Position: 0,
-		},
-		{
-			Kind:     token.Identifier,
-			Bytes:    []byte("x"),
-			Position: 7,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 8,
-		},
-	})
+
+	expected := []token.Kind{
+		token.Return,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 }
 
 func TestArray(t *testing.T) {
 	tokens := token.Tokenize([]byte("array[i]"))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.Identifier,
-			Bytes:    []byte("array"),
-			Position: 0,
-		},
-		{
-			Kind:     token.ArrayStart,
-			Bytes:    []byte("["),
-			Position: 5,
-		},
-		{
-			Kind:     token.Identifier,
-			Bytes:    []byte("i"),
-			Position: 6,
-		},
-		{
-			Kind:     token.ArrayEnd,
-			Bytes:    []byte("]"),
-			Position: 7,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 8,
-		},
-	})
+
+	expected := []token.Kind{
+		token.Identifier,
+		token.ArrayStart,
+		token.Identifier,
+		token.ArrayEnd,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 }
 
 func TestNewline(t *testing.T) {
 	tokens := token.Tokenize([]byte("\n\n"))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.NewLine,
-			Bytes:    []byte("\n"),
-			Position: 0,
-		},
-		{
-			Kind:     token.NewLine,
-			Bytes:    []byte("\n"),
-			Position: 1,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 2,
-		},
-	})
+
+	expected := []token.Kind{
+		token.NewLine,
+		token.NewLine,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 }
 
 func TestNumber(t *testing.T) {
 	tokens := token.Tokenize([]byte(`123 456`))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.Number,
-			Bytes:    []byte("123"),
-			Position: 0,
-		},
-		{
-			Kind:     token.Number,
-			Bytes:    []byte("456"),
-			Position: 4,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 7,
-		},
-	})
+
+	expected := []token.Kind{
+		token.Number,
+		token.Number,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 }
 
 func TestOperator(t *testing.T) {
 	tokens := token.Tokenize([]byte(`+ - * /`))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.Operator,
-			Bytes:    []byte("+"),
-			Position: 0,
-		},
-		{
-			Kind:     token.Operator,
-			Bytes:    []byte("-"),
-			Position: 2,
-		},
-		{
-			Kind:     token.Operator,
-			Bytes:    []byte("*"),
-			Position: 4,
-		},
-		{
-			Kind:     token.Operator,
-			Bytes:    []byte("/"),
-			Position: 6,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 7,
-		},
-	})
+
+	expected := []token.Kind{
+		token.Add,
+		token.Sub,
+		token.Mul,
+		token.Div,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 }
 
 func TestOperatorAssign(t *testing.T) {
-	tokens := token.Tokenize([]byte(`+= -= *= /= ==`))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.Operator,
-			Bytes:    []byte("+="),
-			Position: 0,
-		},
-		{
-			Kind:     token.Operator,
-			Bytes:    []byte("-="),
-			Position: 3,
-		},
-		{
-			Kind:     token.Operator,
-			Bytes:    []byte("*="),
-			Position: 6,
-		},
-		{
-			Kind:     token.Operator,
-			Bytes:    []byte("/="),
-			Position: 9,
-		},
-		{
-			Kind:     token.Operator,
-			Bytes:    []byte("=="),
-			Position: 12,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 14,
-		},
-	})
+	tokens := token.Tokenize([]byte(`+= -= *= /= &= |= ^= <<= >>=`))
+
+	expected := []token.Kind{
+		token.AddAssign,
+		token.SubAssign,
+		token.MulAssign,
+		token.DivAssign,
+		token.AndAssign,
+		token.OrAssign,
+		token.XorAssign,
+		token.ShlAssign,
+		token.ShrAssign,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 }
 
 func TestSeparator(t *testing.T) {
 	tokens := token.Tokenize([]byte("a,b,c"))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.Identifier,
-			Bytes:    []byte("a"),
-			Position: 0,
-		},
-		{
-			Kind:     token.Separator,
-			Bytes:    []byte(","),
-			Position: 1,
-		},
-		{
-			Kind:     token.Identifier,
-			Bytes:    []byte("b"),
-			Position: 2,
-		},
-		{
-			Kind:     token.Separator,
-			Bytes:    []byte(","),
-			Position: 3,
-		},
-		{
-			Kind:     token.Identifier,
-			Bytes:    []byte("c"),
-			Position: 4,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 5,
-		},
-	})
+
+	expected := []token.Kind{
+		token.Identifier,
+		token.Separator,
+		token.Identifier,
+		token.Separator,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 }
 
 func TestComment(t *testing.T) {
 	tokens := token.Tokenize([]byte("// Hello\n// World"))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.Comment,
-			Bytes:    []byte(`// Hello`),
-			Position: 0,
-		},
-		{
-			Kind:     token.NewLine,
-			Bytes:    []byte("\n"),
-			Position: 8,
-		},
-		{
-			Kind:     token.Comment,
-			Bytes:    []byte(`// World`),
-			Position: 9,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 17,
-		},
-	})
+
+	expected := []token.Kind{
+		token.Comment,
+		token.NewLine,
+		token.Comment,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 
 	tokens = token.Tokenize([]byte("// Hello\n"))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.Comment,
-			Bytes:    []byte(`// Hello`),
-			Position: 0,
-		},
-		{
-			Kind:     token.NewLine,
-			Bytes:    []byte("\n"),
-			Position: 8,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 9,
-		},
-	})
+
+	expected = []token.Kind{
+		token.Comment,
+		token.NewLine,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 
 	tokens = token.Tokenize([]byte(`// Hello`))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.Comment,
-			Bytes:    []byte(`// Hello`),
-			Position: 0,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 8,
-		},
-	})
+
+	expected = []token.Kind{
+		token.Comment,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 
 	tokens = token.Tokenize([]byte(`//`))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.Comment,
-			Bytes:    []byte(`//`),
-			Position: 0,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 2,
-		},
-	})
+
+	expected = []token.Kind{
+		token.Comment,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 
 	tokens = token.Tokenize([]byte(`/`))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.Operator,
-			Bytes:    []byte(`/`),
-			Position: 0,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 1,
-		},
-	})
+
+	expected = []token.Kind{
+		token.Div,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 }
 
 func TestInvalid(t *testing.T) {
-	tokens := token.Tokenize([]byte(`@#`))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.Invalid,
-			Bytes:    []byte(`@`),
-			Position: 0,
-		},
-		{
-			Kind:     token.Invalid,
-			Bytes:    []byte(`#`),
-			Position: 1,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 2,
-		},
-	})
+	tokens := token.Tokenize([]byte(`##`))
+
+	expected := []token.Kind{
+		token.Invalid,
+		token.Invalid,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 }
 
 func TestString(t *testing.T) {
 	tokens := token.Tokenize([]byte(`"Hello" "World"`))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.String,
-			Bytes:    []byte(`"Hello"`),
-			Position: 0,
-		},
-		{
-			Kind:     token.String,
-			Bytes:    []byte(`"World"`),
-			Position: 8,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 15,
-		},
-	})
+
+	expected := []token.Kind{
+		token.String,
+		token.String,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 }
 
 func TestStringMultiline(t *testing.T) {
 	tokens := token.Tokenize([]byte("\"Hello\nWorld\""))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.String,
-			Bytes:    []byte("\"Hello\nWorld\""),
-			Position: 0,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 13,
-		},
-	})
+
+	expected := []token.Kind{
+		token.String,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 }
 
 func TestStringEOF(t *testing.T) {
 	tokens := token.Tokenize([]byte(`"EOF`))
-	assert.DeepEqual(t, tokens, token.List{
-		{
-			Kind:     token.String,
-			Bytes:    []byte(`"EOF`),
-			Position: 0,
-		},
-		{
-			Kind:     token.EOF,
-			Bytes:    nil,
-			Position: 4,
-		},
-	})
+
+	expected := []token.Kind{
+		token.String,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
 }

tests/errors_test.go

@@ -21,10 +21,10 @@ var errs = []struct {
 	{"InvalidInstructionNumber.q", &errors.InvalidInstruction{Instruction: "123"}},
 	{"InvalidInstructionExpression.q", &errors.InvalidInstruction{Instruction: "+"}},
 	{"InvalidExpression.q", errors.InvalidExpression},
-	{"InvalidOperator.q", &errors.InvalidOperator{Operator: "+++"}},
 	{"InvalidCharacter.q", &errors.InvalidCharacter{Character: "@"}},
 	{"InvalidCharacter2.q", &errors.InvalidCharacter{Character: "@"}},
 	{"InvalidCharacter3.q", &errors.InvalidCharacter{Character: "@"}},
+	{"InvalidCharacter4.q", &errors.InvalidCharacter{Character: "+++"}},
 	{"MissingBlockEnd.q", errors.MissingBlockEnd},
 	{"MissingBlockStart.q", errors.MissingBlockStart},
 	{"MissingGroupEnd.q", errors.MissingGroupEnd},