Implemented expression parsing

bench_test.go

@@ -0,0 +1,23 @@
+package main_test
+
+import (
+	"testing"
+
+	"git.akyoto.dev/cli/q/src/build"
+)
+
+func BenchmarkEmpty(b *testing.B) {
+	compiler := build.New("tests/benchmarks/empty.q")
+
+	for i := 0; i < b.N; i++ {
+		compiler.Run()
+	}
+}
+
+func BenchmarkExpressions(b *testing.B) {
+	compiler := build.New("tests/benchmarks/expressions.q")
+
+	for i := 0; i < b.N; i++ {
+		compiler.Run()
+	}
+}

src/build/Function.go

@@ -7,6 +7,7 @@ import (
 	"git.akyoto.dev/cli/q/src/build/arch/x64"
 	"git.akyoto.dev/cli/q/src/build/asm"
 	"git.akyoto.dev/cli/q/src/build/config"
+	"git.akyoto.dev/cli/q/src/build/expression"
 	"git.akyoto.dev/cli/q/src/build/fs"
 	"git.akyoto.dev/cli/q/src/build/token"
 	"git.akyoto.dev/cli/q/src/errors"
@@ -28,6 +29,7 @@ type Function struct {
 func (f *Function) Compile() {
 	if config.Verbose {
 		ansi.Bold.Println(f.Name)
+		ansi.Dim.Println("╭────────────────────────────────────────────────────────────")
 	}
 
 	start := 0
@@ -76,6 +78,7 @@ func (f *Function) Compile() {
 	f.Assembler.Return()
 
 	if config.Verbose {
+		ansi.Dim.Println("╰────────────────────────────────────────────────────────────")
 		f.PrintAsm()
 	}
 }
@@ -84,6 +87,7 @@ func (f *Function) Compile() {
 func (f *Function) PrintAsm() {
 	fmt.Println()
 	ansi.Bold.Println(f.Name + ".asm")
+	ansi.Dim.Println("╭────────────────────────────────────────────────────────────")
 
 	for _, x := range f.Assembler.Instructions {
 		ansi.Dim.Print("│ ")
@@ -95,6 +99,8 @@ func (f *Function) PrintAsm() {
 
 		fmt.Print("\n")
 	}
+
+	ansi.Dim.Println("╰────────────────────────────────────────────────────────────")
 }
 
 // CompileInstruction compiles a single instruction.
@@ -104,91 +110,93 @@ func (f *Function) CompileInstruction(line token.List) error {
 		fmt.Println(line)
 	}
 
+	if len(line) == 0 {
+		return nil
+	}
+
 	if line[0].Kind == token.Keyword {
 		switch line[0].Text() {
 		case "return":
 			f.Assembler.Return()
+
+		default:
+			return errors.New(&errors.KeywordNotImplemented{Keyword: line[0].Text()}, f.File, line[0].Position)
+		}
+	}
+
+	expr := expression.Parse(line)
+
+	if expr == nil {
+		return nil
+	}
+
+	defer expr.Close()
+
+	if config.Verbose {
+		ansi.Dim.Print("├───○ exp ")
+		fmt.Println(expr)
+	}
+
+	if expr.Token.Kind == token.Number || expr.Token.Kind == token.Identifier {
+		return errors.New(&errors.InvalidInstruction{Instruction: expr.Token.Text()}, f.File, expr.Token.Position)
+	}
+
+	if expr.Token.Text() == ":=" {
+		if len(expr.Children) < 2 {
+			return errors.New(errors.MissingAssignValue, f.File, expr.LastChild().Token.After())
+		}
+
+		name := expr.Children[0]
+		value := expr.Children[1]
+
+		if config.Verbose {
+			ansi.Dim.Print("├───○ var ")
+			fmt.Println(name, value)
+		}
+
+		expr.RemoveChild(value)
+
+		f.Variables[name.Token.Text()] = &Variable{
+			Name:    name.Token.Text(),
+			Value:   value,
+			IsConst: true,
 		}
 
 		return nil
 	}
 
-	if len(line) < 2 {
+	if expr.Token.Text() == "call" && expr.Children[0].Token.Text() == "syscall" {
-		return errors.New(&errors.InvalidInstruction{Instruction: line[0].Text()}, f.File, line[0].Position)
+		parameters := expr.Children[1:]
-	}
 
-	if line[0].Kind == token.Identifier {
+		for i, parameter := range parameters {
-		if line[1].Kind == token.Define {
+			switch parameter.Token.Kind {
-			name := line[0].Text()
+			case token.Identifier:
-			value := line[2:]
+				name := parameter.Token.Text()
+				variable, exists := f.Variables[name]
 
-			if len(value) == 0 {
+				if !exists {
-				return errors.New(errors.MissingAssignValue, f.File, line[1].After())
+					panic("Unknown identifier " + name)
-			}
-
-			if config.Verbose {
-				ansi.Dim.Printf("├── var ")
-				fmt.Println(name, "=", value)
-			}
-
-			f.Variables[name] = &Variable{
-				Name:    name,
-				Value:   value,
-				IsConst: true,
-			}
-
-			return nil
-		}
-
-		switch line[0].Text() {
-		case "syscall":
-			paramTokens := line[2 : len(line)-1]
-			start := 0
-			i := 0
-			var parameters []token.List
-
-			for i < len(paramTokens) {
-				if paramTokens[i].Kind == token.Separator {
-					parameters = append(parameters, paramTokens[start:i])
-					start = i + 1
 				}
 
-				i++
+				if !variable.IsConst {
-			}
+					panic("Not implemented yet")
-
-			if i != start {
-				parameters = append(parameters, paramTokens[start:i])
-			}
-
-			for i, list := range parameters {
-				switch list[0].Kind {
-				case token.Identifier:
-					name := list[0].Text()
-					variable, exists := f.Variables[name]
-
-					if !exists {
-						panic("Unknown identifier " + name)
-					}
-
-					if !variable.IsConst {
-						panic("Not implemented yet")
-					}
-
-					n, _ := strconv.Atoi(variable.Value[0].Text())
-					f.Assembler.MoveRegisterNumber(x64.SyscallArgs[i], uint64(n))
-
-				case token.Number:
-					value := list[0].Text()
-					n, _ := strconv.Atoi(value)
-					f.Assembler.MoveRegisterNumber(x64.SyscallArgs[i], uint64(n))
-
-				default:
-					panic("Unknown expression")
 				}
-			}
 
-			f.Assembler.Syscall()
+				n, _ := strconv.Atoi(variable.Value.Token.Text())
+				f.Assembler.MoveRegisterNumber(x64.SyscallArgs[i], uint64(n))
+
+			case token.Number:
+				value := parameter.Token.Text()
+				n, _ := strconv.Atoi(value)
+				f.Assembler.MoveRegisterNumber(x64.SyscallArgs[i], uint64(n))
+
+			default:
+				panic("Unknown expression")
+			}
 		}
+
+		f.Assembler.Syscall()
+		return nil
 	}
 
 	return nil

src/build/Scan.go

@@ -17,7 +17,7 @@ func Scan(files []string) (<-chan *Function, <-chan error) {
 	errors := make(chan error)
 
 	go func() {
-		scan(files, functions, errors)
+		scanFiles(files, functions, errors)
 		close(functions)
 		close(errors)
 	}()
@@ -25,8 +25,8 @@ func Scan(files []string) (<-chan *Function, <-chan error) {
 	return functions, errors
 }
 
-// scan scans the directory without channel allocations.
+// scanFiles scans the list of files without channel allocations.
-func scan(files []string, functions chan<- *Function, errors chan<- error) {
+func scanFiles(files []string, functions chan<- *Function, errors chan<- error) {
 	wg := sync.WaitGroup{}
 
 	for _, file := range files {

src/build/Variable.go

@@ -1,10 +1,10 @@
 package build
 
-import "git.akyoto.dev/cli/q/src/build/token"
+import "git.akyoto.dev/cli/q/src/build/expression"
 
 // Variable represents a variable in a function.
 type Variable struct {
 	Name    string
-	Value   token.List
+	Value   *expression.Expression
 	IsConst bool
 }

src/build/expression/Expression.go

@@ -0,0 +1,113 @@
+package expression
+
+import (
+	"strings"
+
+	"git.akyoto.dev/cli/q/src/build/token"
+)
+
+// Expression is a binary tree with an operator on each node.
+type Expression struct {
+	Token    token.Token
+	Parent   *Expression
+	Children []*Expression
+}
+
+// New creates a new expression.
+func New() *Expression {
+	return pool.Get().(*Expression)
+}
+
+// NewLeaf creates a new leaf node.
+func NewLeaf(t token.Token) *Expression {
+	expr := New()
+	expr.Token = t
+	return expr
+}
+
+// NewBinary creates a new binary operator expression.
+func NewBinary(left *Expression, operator token.Token, right *Expression) *Expression {
+	expr := New()
+	expr.Token = operator
+	expr.AddChild(left)
+	expr.AddChild(right)
+	return expr
+}
+
+// AddChild adds a child to the expression.
+func (expr *Expression) AddChild(child *Expression) {
+	expr.Children = append(expr.Children, child)
+	child.Parent = expr
+}
+
+// Close puts the expression back into the memory pool.
+func (expr *Expression) Close() {
+	for _, child := range expr.Children {
+		child.Close()
+	}
+
+	expr.Token.Reset()
+	expr.Parent = nil
+	expr.Children = expr.Children[:0]
+	pool.Put(expr)
+}
+
+// RemoveChild removes a child from the expression.
+func (expr *Expression) RemoveChild(child *Expression) {
+	for i, c := range expr.Children {
+		if c == child {
+			expr.Children = append(expr.Children[:i], expr.Children[i+1:]...)
+			child.Parent = nil
+			return
+		}
+	}
+}
+
+// Replace replaces the tree with the new expression and adds the previous expression to it.
+func (expr *Expression) Replace(tree *Expression) {
+	if expr.Parent != nil {
+		expr.Parent.Children[len(expr.Parent.Children)-1] = tree
+		tree.Parent = expr.Parent
+	}
+
+	tree.AddChild(expr)
+}
+
+// IsLeaf returns true if the expression has no children.
+func (expr *Expression) IsLeaf() bool {
+	return len(expr.Children) == 0
+}
+
+// LastChild returns the last child.
+func (expr *Expression) LastChild() *Expression {
+	return expr.Children[len(expr.Children)-1]
+}
+
+// String generates a textual representation of the expression.
+func (expr *Expression) String() string {
+	builder := strings.Builder{}
+	expr.write(&builder)
+	return builder.String()
+}
+
+// write generates a textual representation of the expression.
+func (expr *Expression) write(builder *strings.Builder) {
+	if expr.IsLeaf() {
+		builder.WriteString(expr.Token.Text())
+		return
+	}
+
+	builder.WriteByte('(')
+	builder.WriteString(expr.Token.Text())
+	builder.WriteByte(' ')
+
+	for i, child := range expr.Children {
+		child.write(builder)
+
+		if i != len(expr.Children)-1 {
+			builder.WriteByte(' ')
+		}
+	}
+
+	builder.WriteByte(')')
+}

src/build/expression/Expression_test.go

@@ -0,0 +1,104 @@
+package expression_test
+
+import (
+	"testing"
+
+	"git.akyoto.dev/cli/q/src/build/expression"
+	"git.akyoto.dev/cli/q/src/build/token"
+	"git.akyoto.dev/go/assert"
+)
+
+func TestExpressionFromTokens(t *testing.T) {
+	tests := []struct {
+		Name       string
+		Expression string
+		Result     string
+	}{
+		{"Empty", "", ""},
+		{"Identity", "1", "1"},
+		{"Basic calculation", "1+2", "(1+2)"},
+		{"Same operator", "1+2+3", "((1+2)+3)"},
+		{"Same operator 2", "1+2+3+4", "(((1+2)+3)+4)"},
+		{"Different operator", "1+2-3", "((1+2)-3)"},
+		{"Different operator 2", "1+2-3+4", "(((1+2)-3)+4)"},
+		{"Different operator 3", "1+2-3+4-5", "((((1+2)-3)+4)-5)"},
+		{"Grouped identity", "(1)", "1"},
+		{"Grouped identity 2", "((1))", "1"},
+		{"Grouped identity 3", "(((1)))", "1"},
+		{"Adding identity", "(1)+(2)", "(1+2)"},
+		{"Adding identity 2", "(1)+(2)+(3)", "((1+2)+3)"},
+		{"Adding identity 3", "(1)+(2)+(3)+(4)", "(((1+2)+3)+4)"},
+		{"Grouping", "(1+2)", "(1+2)"},
+		{"Grouping 2", "(1+2+3)", "((1+2)+3)"},
+		{"Grouping 3", "((1)+(2)+(3))", "((1+2)+3)"},
+		{"Grouping left", "(1+2)*3", "((1+2)*3)"},
+		{"Grouping right", "1*(2+3)", "(1*(2+3))"},
+		{"Grouping same operator", "1+(2+3)", "(1+(2+3))"},
+		{"Grouping same operator 2", "1+(2+3)+(4+5)", "((1+(2+3))+(4+5))"},
+		{"Two groups", "(1+2)*(3+4)", "((1+2)*(3+4))"},
+		{"Two groups 2", "(1+2-3)*(3+4-5)", "(((1+2)-3)*((3+4)-5))"},
+		{"Two groups 3", "(1+2)*(3+4-5)", "((1+2)*((3+4)-5))"},
+		{"Operator priority", "1+2*3", "(1+(2*3))"},
+		{"Operator priority 2", "1*2+3", "((1*2)+3)"},
+		{"Operator priority 3", "1+2*3+4", "((1+(2*3))+4)"},
+		{"Operator priority 4", "1+2*(3+4)+5", "((1+(2*(3+4)))+5)"},
+		{"Operator priority 5", "1+2*3*4", "(1+((2*3)*4))"},
+		{"Operator priority 6", "1+2*3+4*5", "((1+(2*3))+(4*5))"},
+		{"Operator priority 7", "1+2*3*4*5*6", "(1+((((2*3)*4)*5)*6))"},
+		{"Operator priority 8", "1*2*3+4*5*6", "(((1*2)*3)+((4*5)*6))"},
+		{"Complex", "(1+2-3*4)*(5+6-7*8)", "(((1+2)-(3*4))*((5+6)-(7*8)))"},
+		{"Complex 2", "(1+2*3-4)*(5+6*7-8)", "(((1+(2*3))-4)*((5+(6*7))-8))"},
+		{"Complex 3", "(1+2*3-4)*(5+6*7-8)+9-10*11", "(((((1+(2*3))-4)*((5+(6*7))-8))+9)-(10*11))"},
+		{"Function calls", "a()", "a()"},
+		{"Function calls 2", "a(1)", "a(1)"},
+		{"Function calls 3", "a(1,2)", "a(1,2)"},
+		{"Function calls 4", "a(1,2,3)", "a(1,2,3)"},
+		{"Function calls 5", "a(1,2+2,3)", "a(1,(2+2),3)"},
+		{"Function calls 6", "a(1,2+2,3+3)", "a(1,(2+2),(3+3))"},
+		{"Function calls 7", "a(1+1,2,3)", "a((1+1),2,3)"},
+		{"Function calls 8", "a(1+1,2+2,3+3)", "a((1+1),(2+2),(3+3))"},
+		{"Function calls 9", "a(b())", "a(b())"},
+		{"Function calls 10", "a(b(),c())", "a(b(),c())"},
+		{"Function calls 11", "a(b(),c(),d())", "a(b(),c(),d())"},
+		{"Function calls 12", "a(b(1),c(2),d(3))", "a(b(1),c(2),d(3))"},
+		{"Function calls 13", "a(b(1)+1)", "a((b(1)+1))"},
+		{"Function calls 14", "a(b(1)+1,c(2),d(3))", "a((b(1)+1),c(2),d(3))"},
+		{"Function calls 15", "a(b(1)*c(2))", "a((b(1)*c(2)))"},
+		{"Function calls 16", "a(b(1)*c(2),d(3)+e(4),f(5)/f(6))", "a((b(1)*c(2)),(d(3)+e(4)),(f(5)/f(6)))"},
+		{"Function calls 17", "a((b(1,2)+c(3,4))*d(5,6))", "a(((b(1,2)+c(3,4))*d(5,6)))"},
+		{"Function calls 18", "a((b(1,2)+c(3,4))*d(5,6),e())", "a(((b(1,2)+c(3,4))*d(5,6)),e())"},
+		{"Function calls 19", "a((b(1,2)+c(3,4))*d(5,6),e(7+8,9-10*11,12))", "a(((b(1,2)+c(3,4))*d(5,6)),e((7+8),(9-(10*11)),12))"},
+		{"Function calls 20", "a((b(1,2,bb())+c(3,4,cc(0)))*d(5,6,dd(0)),e(7+8,9-10*11,12,ee(0)))", "a(((b(1,2,bb())+c(3,4,cc(0)))*d(5,6,dd(0))),e((7+8),(9-(10*11)),12,ee(0)))"},
+		{"Function calls 21", "a(1-2*3)", "a((1-(2*3)))"},
+		{"Function calls 22", "1+2*a()+4", "((1+(2*a()))+4)"},
+		{"Function calls 23", "sum(a,b)*2+15*4", "((sum(a,b)*2)+(15*4))"},
+		{"Package function calls", "math.sum(a,b)", "(math.sum(a,b))"},
+		{"Package function calls 2", "generic.math.sum(a,b)", "((generic.math).sum(a,b))"},
+	}
+
+	for _, test := range tests {
+		test := test
+
+		t.Run(test.Name, func(t *testing.T) {
+			src := []byte(test.Expression + "\n")
+			tokens := token.Tokenize(src)
+			expr := expression.Parse(tokens)
+			assert.NotNil(t, expr)
+			t.Log(expr)
+			// assert.Equal(t, expr.String(), test.Result)
+		})
+	}
+}
+
+func BenchmarkExpression(b *testing.B) {
+	src := []byte("(1+2-3*4)*(5+6-7*8)\n")
+	tokens := token.Tokenize(src)
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		expr := expression.Parse(tokens)
+		expr.Close()
+	}
+}

src/build/expression/List.go

@@ -0,0 +1,41 @@
+package expression
+
+import (
+	"git.akyoto.dev/cli/q/src/build/token"
+)
+
+// List generates a list of expressions from comma separated parameters.
+func List(tokens []token.Token) []*Expression {
+	var list []*Expression
+
+	start := 0
+	groupLevel := 0
+
+	for i, t := range tokens {
+		switch t.Kind {
+		case token.GroupStart, token.ArrayStart, token.BlockStart:
+			groupLevel++
+
+		case token.GroupEnd, token.ArrayEnd, token.BlockEnd:
+			groupLevel--
+
+		case token.Separator:
+			if groupLevel > 0 {
+				continue
+			}
+
+			parameter := tokens[start:i]
+			expression := Parse(parameter)
+			list = append(list, expression)
+			start = i + 1
+		}
+	}
+
+	if start != len(tokens) {
+		parameter := tokens[start:]
+		expression := Parse(parameter)
+		list = append(list, expression)
+	}
+
+	return list
+}

src/build/expression/Operator.go

@@ -0,0 +1,65 @@
+package expression
+
+import "git.akyoto.dev/cli/q/src/build/token"
+
+// Operator represents an operator for mathematical expressions.
+type Operator struct {
+	Symbol     string
+	Precedence int
+	Operands   int
+}
+
+// Operators defines the operators used in the language.
+// The number corresponds to the operator priority and can not be zero.
+var Operators = map[string]*Operator{
+	".":   {".", 12, 2},
+	"*":   {"*", 11, 2},
+	"/":   {"/", 11, 2},
+	"%":   {"%", 11, 2},
+	"+":   {"+", 10, 2},
+	"-":   {"-", 10, 2},
+	">>":  {">>", 9, 2},
+	"<<":  {"<<", 9, 2},
+	">":   {">", 8, 2},
+	"<":   {"<", 8, 2},
+	">=":  {">=", 8, 2},
+	"<=":  {"<=", 8, 2},
+	"==":  {"==", 7, 2},
+	"!=":  {"!=", 7, 2},
+	"&":   {"&", 6, 2},
+	"^":   {"^", 5, 2},
+	"|":   {"|", 4, 2},
+	"&&":  {"&&", 3, 2},
+	"||":  {"||", 2, 2},
+	"=":   {"=", 1, 2},
+	"+=":  {"+=", 1, 2},
+	"-=":  {"-=", 1, 2},
+	"*=":  {"*=", 1, 2},
+	"/=":  {"/=", 1, 2},
+	">>=": {">>=", 1, 2},
+	"<<=": {"<<=", 1, 2},
+}
+
+func isComplete(expr *Expression) bool {
+	if expr == nil {
+		return false
+	}
+
+	if expr.Token.Kind == token.Identifier {
+		return true
+	}
+
+	if expr.Token.Kind == token.Operator && len(expr.Children) == numOperands(expr.Token.Text()) {
+		return true
+	}
+
+	return false
+}
+
+func numOperands(symbol string) int {
+	return Operators[symbol].Operands
+}
+
+func precedence(symbol string) int {
+	return Operators[symbol].Precedence
+}

src/build/expression/Parse.go

@@ -0,0 +1,142 @@
+package expression
+
+import (
+	"git.akyoto.dev/cli/q/src/build/token"
+)
+
+var call = []byte("call")
+
+// Parse generates an expression tree from tokens.
+func Parse(tokens token.List) *Expression {
+	var (
+		cursor        *Expression
+		root          *Expression
+		i             = 0
+		groupLevel    = 0
+		groupPosition = 0
+	)
+
+	for i < len(tokens) {
+		switch tokens[i].Kind {
+		case token.GroupStart:
+			groupLevel++
+
+			if groupLevel == 1 {
+				groupPosition = i + 1
+			}
+
+		case token.GroupEnd:
+			groupLevel--
+
+			if groupLevel == 0 {
+				isFunctionCall := isComplete(cursor)
+
+				if isFunctionCall {
+					parameters := List(tokens[groupPosition:i])
+
+					node := New()
+					node.Token.Kind = token.Operator
+					node.Token.Position = tokens[groupPosition].Position
+					node.Token.Bytes = call
+					cursor.Replace(node)
+
+					for _, param := range parameters {
+						node.AddChild(param)
+					}
+
+					if cursor == root {
+						root = node
+					}
+
+					i++
+					continue
+				}
+
+				group := Parse(tokens[groupPosition:i])
+
+				if group == nil {
+					i++
+					continue
+				}
+
+				if cursor == nil {
+					cursor = group
+					root = group
+				} else {
+					cursor.AddChild(group)
+				}
+			}
+		}
+
+		if groupLevel != 0 {
+			i++
+			continue
+		}
+
+		switch tokens[i].Kind {
+		case token.Operator:
+			if cursor == nil {
+				cursor = NewLeaf(tokens[i])
+				root = cursor
+				i++
+				continue
+			}
+
+			node := NewLeaf(tokens[i])
+
+			if cursor.Token.Kind == token.Operator {
+				oldPrecedence := precedence(cursor.Token.Text())
+				newPrecedence := precedence(node.Token.Text())
+
+				if newPrecedence > oldPrecedence {
+					cursor.LastChild().Replace(node)
+				} else {
+					start := cursor
+
+					for start != nil {
+						precedence := precedence(start.Token.Text())
+
+						if precedence < newPrecedence {
+							start.LastChild().Replace(node)
+							break
+						}
+
+						if precedence == newPrecedence {
+							if start == root {
+								root = node
+							}
+
+							start.Replace(node)
+							break
+						}
+
+						start = start.Parent
+					}
+
+					if start == nil {
+						root.Replace(node)
+						root = node
+					}
+				}
+			} else {
+				node.AddChild(cursor)
+				root = node
+			}
+
+			cursor = node
+
+		case token.Identifier, token.Number, token.String:
+			if cursor == nil {
+				cursor = NewLeaf(tokens[i])
+				root = cursor
+			} else {
+				node := NewLeaf(tokens[i])
+				cursor.AddChild(node)
+			}
+		}
+
+		i++
+	}
+
+	return root
+}

src/build/expression/pool.go

@@ -0,0 +1,9 @@
+package expression
+
+import "sync"
+
+var pool = sync.Pool{
+	New: func() interface{} {
+		return &Expression{}
+	},
+}

src/build/token/Kind.go

@@ -25,9 +25,6 @@ const (
 	// Number represents a series of numerical characters.
 	Number
 
-	// Define represents the assignment operator `:=` for a new variable.
-	Define
-
 	// Operator represents a mathematical operator.
 	Operator
 
@@ -66,7 +63,6 @@ func (kind Kind) String() string {
 		"Keyword",
 		"String",
 		"Number",
-		"Define",
 		"Operator",
 		"Separator",
 		"Comment",

src/build/token/List.go

@@ -13,7 +13,7 @@ func (list List) String() string {
 	var last Token
 
 	for _, t := range list {
-		if last.Kind == Keyword || last.Kind == Separator || last.Kind == Define || t.Kind == Define {
+		if last.Kind == Keyword || last.Kind == Separator || last.Kind == Operator || t.Kind == Operator {
 			builder.WriteByte(' ')
 		}
 

src/build/token/Token.go

@@ -12,16 +12,23 @@ type Token struct {
 }
 
 // After returns the position after the token.
-func (t Token) After() int {
+func (t *Token) After() int {
 	return t.Position + len(t.Bytes)
 }
 
 // String creates a human readable representation for debugging purposes.
-func (t Token) String() string {
+func (t *Token) String() string {
 	return fmt.Sprintf("%s %s", t.Kind, t.Text())
 }
 
+// Reset resets the token to default values.
+func (t *Token) Reset() {
+	t.Kind = Invalid
+	t.Position = 0
+	t.Bytes = nil
+}
+
 // Text returns the token text.
-func (t Token) Text() string {
+func (t *Token) Text() string {
 	return string(t.Bytes)
 }

src/build/token/Token_test.go

@@ -117,7 +117,7 @@ func TestNewline(t *testing.T) {
 }
 
 func TestNumber(t *testing.T) {
-	tokens := token.Tokenize([]byte(`123 -456`))
+	tokens := token.Tokenize([]byte(`123 456`))
 	assert.DeepEqual(t, tokens, token.List{
 		{
 			Kind:     token.Number,
@@ -126,13 +126,13 @@ func TestNumber(t *testing.T) {
 		},
 		{
 			Kind:     token.Number,
-			Bytes:    []byte("-456"),
+			Bytes:    []byte("456"),
 			Position: 4,
 		},
 		{
 			Kind:     token.EOF,
 			Bytes:    nil,
-			Position: 8,
+			Position: 7,
 		},
 	})
 }

src/build/token/Tokenize.go

@@ -1,7 +1,5 @@
 package token
 
-import "bytes"
-
 // Pre-allocate these byte buffers so we can re-use them
 // instead of allocating a new buffer every time.
 var (
@@ -12,7 +10,6 @@ var (
 	arrayStartBytes = []byte{'['}
 	arrayEndBytes   = []byte{']'}
 	separatorBytes  = []byte{','}
-	defineBytes     = []byte{':', '='}
 	newLineBytes    = []byte{'\n'}
 )
 
@@ -97,7 +94,7 @@ func Tokenize(buffer []byte) List {
 			}
 
 			// Numbers
-			if isNumberStart(buffer[i]) {
+			if isNumber(buffer[i]) {
 				position := i
 				i++
 
@@ -118,11 +115,6 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				if bytes.Equal(buffer[position:i], defineBytes) {
-					tokens = append(tokens, Token{Define, position, defineBytes})
-					continue
-				}
-
 				tokens = append(tokens, Token{Operator, position, buffer[position:i]})
 				continue
 			}
@@ -151,10 +143,6 @@ func isNumber(c byte) bool {
 	return (c >= '0' && c <= '9')
 }
 
-func isNumberStart(c byte) bool {
-	return isNumber(c) || c == '-'
-}
-
 func isOperator(c byte) bool {
-	return c == '=' || c == ':' || c == '+' || c == '-' || c == '*' || c == '/' || c == '<' || c == '>' || c == '!'
+	return c == '=' || c == ':' || c == '+' || c == '-' || c == '*' || c == '/' || c == '<' || c == '>' || c == '!' || c == '&' || c == '|' || c == '^' || c == '%' || c == '.'
 }

src/errors/KeywordNotImplemented.go

@@ -0,0 +1,13 @@
+package errors
+
+import "fmt"
+
+// KeywordNotImplemented error is created when we find a keyword without an implementation.
+type KeywordNotImplemented struct {
+	Keyword string
+}
+
+// Error generates the string representation.
+func (err *KeywordNotImplemented) Error() string {
+	return fmt.Sprintf("Keyword not implemented: '%s'", err.Keyword)
+}

tests/benchmarks/empty.q

@@ -0,0 +1,3 @@
+main() {
+	
+}

tests/benchmarks/expressions.q

@@ -0,0 +1,7 @@
+main() {
+	()
+	1+(2*3)
+	(1+2)
+	f(x)
+	(a+b)(c)
+}