Simplified file structure

src/expression/Expression.go

@@ -0,0 +1,149 @@
+package expression
+
+import (
+	"strings"
+
+	"git.akyoto.dev/cli/q/src/token"
+)
+
+// Expression is a binary tree with an operator on each node.
+type Expression struct {
+	Parent     *Expression
+	Children   []*Expression
+	Token      token.Token
+	Value      int
+	Precedence int8
+	IsFolded   bool
+}
+
+// New creates a new expression.
+func New() *Expression {
+	return &Expression{}
+}
+
+// NewLeaf creates a new leaf node.
+func NewLeaf(t token.Token) *Expression {
+	return &Expression{
+		Token: t,
+	}
+}
+
+// AddChild adds a child to the expression.
+func (expr *Expression) AddChild(child *Expression) {
+	if expr.Children == nil {
+		expr.Children = make([]*Expression, 0, 2)
+	}
+
+	expr.Children = append(expr.Children, child)
+	child.Parent = expr
+}
+
+// Count counts how often the given token appears in the expression.
+func (expr *Expression) Count(buffer []byte, kind token.Kind, name string) uint8 {
+	count := uint8(0)
+
+	expr.EachLeaf(func(leaf *Expression) error {
+		if leaf.Token.Kind == kind && leaf.Token.Text(buffer) == name {
+			count++
+		}
+
+		return nil
+	})
+
+	return count
+}
+
+// Reset resets all values to the default.
+func (expr *Expression) Reset() {
+	expr.Parent = nil
+
+	if expr.Children != nil {
+		expr.Children = expr.Children[:0]
+	}
+
+	expr.Token.Reset()
+	expr.Value = 0
+	expr.Precedence = 0
+	expr.IsFolded = false
+}
+
+// EachLeaf iterates through all leaves in the tree.
+func (expr *Expression) EachLeaf(call func(*Expression) error) error {
+	if expr.IsLeaf() {
+		return call(expr)
+	}
+
+	for _, child := range expr.Children {
+		err := child.EachLeaf(call)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// RemoveChild removes a child from the expression.
+func (expr *Expression) RemoveChild(child *Expression) {
+	for i, c := range expr.Children {
+		if c == child {
+			expr.Children = append(expr.Children[:i], expr.Children[i+1:]...)
+			child.Parent = nil
+			return
+		}
+	}
+}
+
+// Replace replaces the tree with the new expression and adds the previous expression to it.
+func (expr *Expression) Replace(tree *Expression) {
+	if expr.Parent != nil {
+		expr.Parent.Children[len(expr.Parent.Children)-1] = tree
+		tree.Parent = expr.Parent
+	}
+
+	tree.AddChild(expr)
+}
+
+// IsLeaf returns true if the expression has no children.
+func (expr *Expression) IsLeaf() bool {
+	return len(expr.Children) == 0
+}
+
+// LastChild returns the last child.
+func (expr *Expression) LastChild() *Expression {
+	return expr.Children[len(expr.Children)-1]
+}
+
+// String generates a textual representation of the expression.
+func (expr *Expression) String(source []byte) string {
+	builder := strings.Builder{}
+	expr.write(&builder, source)
+	return builder.String()
+}
+
+// write generates a textual representation of the expression.
+func (expr *Expression) write(builder *strings.Builder, source []byte) {
+	if expr.IsLeaf() {
+		builder.WriteString(expr.Token.Text(source))
+		return
+	}
+
+	builder.WriteByte('(')
+
+	switch expr.Token.Kind {
+	case token.Call:
+		builder.WriteString("λ")
+	case token.Array:
+		builder.WriteString("@")
+	default:
+		builder.WriteString(expr.Token.Text(source))
+	}
+
+	for _, child := range expr.Children {
+		builder.WriteByte(' ')
+		child.write(builder, source)
+	}
+
+	builder.WriteByte(')')
+}

src/expression/Expression_test.go

@@ -0,0 +1,182 @@
+package expression_test
+
+import (
+	"fmt"
+	"testing"
+
+	"git.akyoto.dev/cli/q/src/expression"
+	"git.akyoto.dev/cli/q/src/token"
+	"git.akyoto.dev/go/assert"
+)
+
+func TestParse(t *testing.T) {
+	tests := []struct {
+		Name       string
+		Expression string
+		Result     string
+	}{
+		{"Identity", "1", "1"},
+		{"Basic calculation", "1+2", "(+ 1 2)"},
+
+		{"Same operator", "1+2+3", "(+ (+ 1 2) 3)"},
+		{"Same operator 2", "1+2+3+4", "(+ (+ (+ 1 2) 3) 4)"},
+
+		{"Different operator", "1+2-3", "(- (+ 1 2) 3)"},
+		{"Different operator 2", "1+2-3+4", "(+ (- (+ 1 2) 3) 4)"},
+		{"Different operator 3", "1+2-3+4-5", "(- (+ (- (+ 1 2) 3) 4) 5)"},
+
+		{"Grouped identity", "(1)", "1"},
+		{"Grouped identity 2", "((1))", "1"},
+		{"Grouped identity 3", "(((1)))", "1"},
+
+		{"Adding identity", "(1)+(2)", "(+ 1 2)"},
+		{"Adding identity 2", "(1)+(2)+(3)", "(+ (+ 1 2) 3)"},
+		{"Adding identity 3", "(1)+(2)+(3)+(4)", "(+ (+ (+ 1 2) 3) 4)"},
+
+		{"Grouping", "(1+2)", "(+ 1 2)"},
+		{"Grouping 2", "(1+2+3)", "(+ (+ 1 2) 3)"},
+		{"Grouping 3", "((1)+(2)+(3))", "(+ (+ 1 2) 3)"},
+		{"Grouping left", "(1+2)*3", "(* (+ 1 2) 3)"},
+		{"Grouping right", "1*(2+3)", "(* 1 (+ 2 3))"},
+		{"Grouping same operator", "1+(2+3)", "(+ 1 (+ 2 3))"},
+		{"Grouping same operator 2", "1+(2+3)+(4+5)", "(+ (+ 1 (+ 2 3)) (+ 4 5))"},
+
+		{"Two groups", "(1+2)*(3+4)", "(* (+ 1 2) (+ 3 4))"},
+		{"Two groups 2", "(1+2-3)*(3+4-5)", "(* (- (+ 1 2) 3) (- (+ 3 4) 5))"},
+		{"Two groups 3", "(1+2)*(3+4-5)", "(* (+ 1 2) (- (+ 3 4) 5))"},
+
+		{"Operator priority", "1+2*3", "(+ 1 (* 2 3))"},
+		{"Operator priority 2", "1*2+3", "(+ (* 1 2) 3)"},
+		{"Operator priority 3", "1+2*3+4", "(+ (+ 1 (* 2 3)) 4)"},
+		{"Operator priority 4", "1+2*(3+4)+5", "(+ (+ 1 (* 2 (+ 3 4))) 5)"},
+		{"Operator priority 5", "1+2*3*4", "(+ 1 (* (* 2 3) 4))"},
+		{"Operator priority 6", "1+2*3+4*5", "(+ (+ 1 (* 2 3)) (* 4 5))"},
+		{"Operator priority 7", "1+2*3*4*5*6", "(+ 1 (* (* (* (* 2 3) 4) 5) 6))"},
+		{"Operator priority 8", "1*2*3+4*5*6", "(+ (* (* 1 2) 3) (* (* 4 5) 6))"},
+
+		{"Complex", "(1+2-3*4)*(5+6-7*8)", "(* (- (+ 1 2) (* 3 4)) (- (+ 5 6) (* 7 8)))"},
+		{"Complex 2", "(1+2*3-4)*(5+6*7-8)", "(* (- (+ 1 (* 2 3)) 4) (- (+ 5 (* 6 7)) 8))"},
+		{"Complex 3", "(1+2*3-4)*(5+6*7-8)+9-10*11", "(- (+ (* (- (+ 1 (* 2 3)) 4) (- (+ 5 (* 6 7)) 8)) 9) (* 10 11))"},
+
+		{"Unary not", "!", "!"},
+		{"Unary not 2", "!a", "(! a)"},
+		{"Unary not 3", "!(!a)", "(! (! a))"},
+		{"Unary not 4", "!(a||b)", "(! (|| a b))"},
+		{"Unary not 5", "a || !b", "(|| a (! b))"},
+
+		{"Unary minus", "-", "-"},
+		{"Unary minus 2", "-a", "(- a)"},
+		{"Unary minus 3", "-(-a)", "(- (- a))"},
+		{"Unary minus 4", "-a+b", "(+ (- a) b)"},
+		{"Unary minus 5", "-(a+b)", "(- (+ a b))"},
+		{"Unary minus 6", "a + -b", "(+ a (- b))"},
+		{"Unary minus 7", "-a + -b", "(+ (- a) (- b))"},
+
+		{"Function calls", "a()", "(λ a)"},
+		{"Function calls 2", "a(1)", "(λ a 1)"},
+		{"Function calls 3", "a(1)+1", "(+ (λ a 1) 1)"},
+		{"Function calls 4", "1+a(1)", "(+ 1 (λ a 1))"},
+		{"Function calls 5", "a(1,2)", "(λ a 1 2)"},
+		{"Function calls 6", "a(1,2,3)", "(λ a 1 2 3)"},
+		{"Function calls 7", "a(1,2+2,3)", "(λ a 1 (+ 2 2) 3)"},
+		{"Function calls 8", "a(1,2+2,3+3)", "(λ a 1 (+ 2 2) (+ 3 3))"},
+		{"Function calls 9", "a(1+1,2,3)", "(λ a (+ 1 1) 2 3)"},
+		{"Function calls 10", "a(1+1,2+2,3+3)", "(λ a (+ 1 1) (+ 2 2) (+ 3 3))"},
+		{"Function calls 11", "a(b())", "(λ a (λ b))"},
+		{"Function calls 12", "a(b(),c())", "(λ a (λ b) (λ c))"},
+		{"Function calls 13", "a(b(),c(),d())", "(λ a (λ b) (λ c) (λ d))"},
+		{"Function calls 14", "a(b(1))", "(λ a (λ b 1))"},
+		{"Function calls 15", "a(b(1),c(2),d(3))", "(λ a (λ b 1) (λ c 2) (λ d 3))"},
+		{"Function calls 16", "a(b(1)+1)", "(λ a (+ (λ b 1) 1))"},
+		{"Function calls 17", "a(b(1)+1,c(2),d(3))", "(λ a (+ (λ b 1) 1) (λ c 2) (λ d 3))"},
+		{"Function calls 18", "a(b(1)*c(2))", "(λ a (* (λ b 1) (λ c 2)))"},
+		{"Function calls 19", "a(b(1)*c(2),d(3)+e(4),f(5)/f(6))", "(λ a (* (λ b 1) (λ c 2)) (+ (λ d 3) (λ e 4)) (/ (λ f 5) (λ f 6)))"},
+		{"Function calls 20", "a(b(1,2)+c(3,4)*d(5,6))", "(λ a (+ (λ b 1 2) (* (λ c 3 4) (λ d 5 6))))"},
+		{"Function calls 21", "a((b(1,2)+c(3,4))*d(5,6))", "(λ a (* (+ (λ b 1 2) (λ c 3 4)) (λ d 5 6)))"},
+		{"Function calls 22", "a((b(1,2)+c(3,4))*d(5,6),e())", "(λ a (* (+ (λ b 1 2) (λ c 3 4)) (λ d 5 6)) (λ e))"},
+		{"Function calls 23", "a((b(1,2)+c(3,4))*d(5,6),e(7+8,9-10*11,12))", "(λ a (* (+ (λ b 1 2) (λ c 3 4)) (λ d 5 6)) (λ e (+ 7 8) (- 9 (* 10 11)) 12))"},
+		{"Function calls 24", "a((b(1,2,bb())+c(3,4,cc(0)))*d(5,6,dd(0)),e(7+8,9-10*11,12,ee(0)))", "(λ a (* (+ (λ b 1 2 (λ bb)) (λ c 3 4 (λ cc 0))) (λ d 5 6 (λ dd 0))) (λ e (+ 7 8) (- 9 (* 10 11)) 12 (λ ee 0)))"},
+		{"Function calls 25", "a(1-2*3)", "(λ a (- 1 (* 2 3)))"},
+		{"Function calls 26", "1+2*a()+4", "(+ (+ 1 (* 2 (λ a))) 4)"},
+		{"Function calls 27", "sum(a,b)*2+15*4", "(+ (* (λ sum a b) 2) (* 15 4))"},
+
+		{"Package function calls", "math.sum(a,b)", "(λ (. math sum) a b)"},
+		{"Package function calls 2", "generic.math.sum(a,b)", "(λ (. (. generic math) sum) a b)"},
+
+		{"Array access", "a[0]", "(@ a 0)"},
+		{"Array access 2", "a[b+c]", "(@ a (+ b c))"},
+		{"Array access 3", "a.b[c]", "(@ (. a b) c)"},
+		{"Array access 4", "a.b[c+d]", "(@ (. a b) (+ c d))"},
+	}
+
+	for _, test := range tests {
+		test := test
+
+		t.Run(test.Name, func(t *testing.T) {
+			src := []byte(test.Expression)
+			tokens := token.Tokenize(src)
+			expr := expression.Parse(tokens)
+			defer expr.Reset()
+
+			assert.NotNil(t, expr)
+			assert.Equal(t, expr.String(src), test.Result)
+		})
+	}
+}
+
+func TestCount(t *testing.T) {
+	src := []byte("(a+b-c*d)+(a*b-c+d)")
+	tokens := token.Tokenize(src)
+	expr := expression.Parse(tokens)
+	assert.Equal(t, expr.Count(src, token.Identifier, "a"), 2)
+	assert.Equal(t, expr.Count(src, token.Identifier, "b"), 2)
+	assert.Equal(t, expr.Count(src, token.Identifier, "c"), 2)
+	assert.Equal(t, expr.Count(src, token.Identifier, "d"), 2)
+	assert.Equal(t, expr.Count(src, token.Identifier, "e"), 0)
+}
+
+func TestEachLeaf(t *testing.T) {
+	src := []byte("(1+2-3*4)+(5*6-7+8)")
+	tokens := token.Tokenize(src)
+	expr := expression.Parse(tokens)
+	leaves := []string{}
+
+	err := expr.EachLeaf(func(leaf *expression.Expression) error {
+		leaves = append(leaves, leaf.Token.Text(src))
+		return nil
+	})
+
+	assert.Nil(t, err)
+	assert.DeepEqual(t, leaves, []string{"1", "2", "3", "4", "5", "6", "7", "8"})
+
+	err = expr.EachLeaf(func(leaf *expression.Expression) error {
+		return fmt.Errorf("error")
+	})
+
+	assert.NotNil(t, err)
+	assert.Equal(t, err.Error(), "error")
+}
+
+func TestRemoveChild(t *testing.T) {
+	src := []byte("(1+2-3*4)+(5*6-7+8)")
+	tokens := token.Tokenize(src)
+	expr := expression.Parse(tokens)
+	left := expr.Children[0]
+	right := expr.Children[1]
+	expr.RemoveChild(left)
+	assert.Equal(t, expr.Children[0], right)
+}
+
+func TestNilExpression(t *testing.T) {
+	src := []byte("")
+	tokens := token.Tokenize(src)
+	expr := expression.Parse(tokens)
+	assert.Nil(t, expr)
+}
+
+func TestNilGroup(t *testing.T) {
+	src := []byte("()")
+	tokens := token.Tokenize(src)
+	expr := expression.Parse(tokens)
+	assert.Nil(t, expr)
+}

src/expression/List.go

@@ -0,0 +1,18 @@
+package expression
+
+import (
+	"git.akyoto.dev/cli/q/src/token"
+)
+
+// NewList generates a list of expressions from comma separated parameters.
+func NewList(tokens token.List) []*Expression {
+	var list []*Expression
+
+	tokens.Split(func(parameter token.List) error {
+		expression := Parse(parameter)
+		list = append(list, expression)
+		return nil
+	})
+
+	return list
+}

src/expression/Operator.go

@@ -0,0 +1,79 @@
+package expression
+
+import (
+	"math"
+
+	"git.akyoto.dev/cli/q/src/token"
+)
+
+// Operator represents an operator for mathematical expressions.
+type Operator struct {
+	Symbol     string
+	Precedence int8
+	Operands   int8
+}
+
+// Operators defines the operators used in the language.
+// The number corresponds to the operator priority and can not be zero.
+var Operators = [64]Operator{
+	token.Period: {".", 13, 2},
+	token.Call:   {"λ", 12, 1},
+	token.Array:  {"@", 12, 2},
+	token.Negate: {"-", 11, 1},
+	token.Not:    {"!", 11, 1},
+	token.Mul:    {"*", 10, 2},
+	token.Div:    {"/", 10, 2},
+	token.Mod:    {"%", 10, 2},
+	token.Add:    {"+", 9, 2},
+	token.Sub:    {"-", 9, 2},
+	token.Shr:    {">>", 8, 2},
+	token.Shl:    {"<<", 8, 2},
+	token.And:    {"&", 7, 2},
+	token.Xor:    {"^", 6, 2},
+	token.Or:     {"|", 5, 2},
+
+	token.Greater:      {">", 4, 2},
+	token.Less:         {"<", 4, 2},
+	token.GreaterEqual: {">=", 4, 2},
+	token.LessEqual:    {"<=", 4, 2},
+	token.Equal:        {"==", 3, 2},
+	token.NotEqual:     {"!=", 3, 2},
+	token.LogicalAnd:   {"&&", 2, 2},
+	token.LogicalOr:    {"||", 1, 2},
+
+	token.Separator: {",", 0, 2},
+
+	token.Assign:    {"=", math.MinInt8, 2},
+	token.Define:    {":=", math.MinInt8, 2},
+	token.AddAssign: {"+=", math.MinInt8, 2},
+	token.SubAssign: {"-=", math.MinInt8, 2},
+	token.MulAssign: {"*=", math.MinInt8, 2},
+	token.DivAssign: {"/=", math.MinInt8, 2},
+	token.ModAssign: {"%=", math.MinInt8, 2},
+	token.ShrAssign: {">>=", math.MinInt8, 2},
+	token.ShlAssign: {"<<=", math.MinInt8, 2},
+}
+
+func isComplete(expr *Expression) bool {
+	if expr == nil {
+		return false
+	}
+
+	if expr.Token.Kind == token.Identifier || expr.Token.Kind == token.Number || expr.Token.Kind == token.String {
+		return true
+	}
+
+	if expr.Token.IsOperator() && len(expr.Children) == numOperands(expr.Token.Kind) {
+		return true
+	}
+
+	return false
+}
+
+func numOperands(symbol token.Kind) int {
+	return int(Operators[symbol].Operands)
+}
+
+func precedence(symbol token.Kind) int8 {
+	return Operators[symbol].Precedence
+}

src/expression/Parse.go

@@ -0,0 +1,165 @@
+package expression
+
+import (
+	"math"
+
+	"git.akyoto.dev/cli/q/src/token"
+)
+
+// Parse generates an expression tree from tokens.
+func Parse(tokens []token.Token) *Expression {
+	var (
+		cursor        *Expression
+		root          *Expression
+		groupLevel    = 0
+		groupPosition = 0
+	)
+
+	for i, t := range tokens {
+		if t.Kind == token.GroupStart || t.Kind == token.ArrayStart {
+			groupLevel++
+
+			if groupLevel == 1 {
+				groupPosition = i + 1
+			}
+
+			continue
+		}
+
+		if t.Kind == token.GroupEnd || t.Kind == token.ArrayEnd {
+			groupLevel--
+
+			if groupLevel != 0 {
+				continue
+			}
+
+			// Function call or array access
+			if isComplete(cursor) {
+				parameters := NewList(tokens[groupPosition:i])
+
+				node := New()
+				node.Token.Position = tokens[groupPosition].Position
+
+				switch t.Kind {
+				case token.GroupEnd:
+					node.Token.Kind = token.Call
+				case token.ArrayEnd:
+					node.Token.Kind = token.Array
+				}
+
+				node.Precedence = precedence(node.Token.Kind)
+
+				if cursor.Token.IsOperator() && node.Precedence > cursor.Precedence {
+					cursor.LastChild().Replace(node)
+				} else {
+					if cursor == root {
+						root = node
+					}
+
+					cursor.Replace(node)
+				}
+
+				for _, param := range parameters {
+					node.AddChild(param)
+				}
+
+				cursor = node
+				continue
+			}
+
+			group := Parse(tokens[groupPosition:i])
+
+			if group == nil {
+				continue
+			}
+
+			group.Precedence = math.MaxInt8
+
+			if cursor == nil {
+				cursor = group
+				root = group
+			} else {
+				cursor.AddChild(group)
+			}
+
+			continue
+		}
+
+		if groupLevel > 0 {
+			continue
+		}
+
+		if t.Kind == token.Identifier || t.Kind == token.Number || t.Kind == token.String || t.Kind == token.Rune {
+			if cursor != nil {
+				node := NewLeaf(t)
+				cursor.AddChild(node)
+			} else {
+				cursor = NewLeaf(t)
+				root = cursor
+			}
+
+			continue
+		}
+
+		if !t.IsOperator() {
+			continue
+		}
+
+		if cursor == nil {
+			cursor = NewLeaf(t)
+			cursor.Precedence = precedence(t.Kind)
+			root = cursor
+			continue
+		}
+
+		node := NewLeaf(t)
+		node.Precedence = precedence(t.Kind)
+
+		if cursor.Token.IsOperator() {
+			oldPrecedence := cursor.Precedence
+			newPrecedence := node.Precedence
+
+			if newPrecedence > oldPrecedence {
+				if len(cursor.Children) == numOperands(cursor.Token.Kind) {
+					cursor.LastChild().Replace(node)
+				} else {
+					cursor.AddChild(node)
+				}
+			} else {
+				start := cursor
+
+				for start != nil {
+					precedence := start.Precedence
+
+					if precedence < newPrecedence {
+						start.LastChild().Replace(node)
+						break
+					}
+
+					if precedence == newPrecedence {
+						if start == root {
+							root = node
+						}
+
+						start.Replace(node)
+						break
+					}
+
+					start = start.Parent
+				}
+
+				if start == nil {
+					root.Replace(node)
+					root = node
+				}
+			}
+		} else {
+			node.AddChild(cursor)
+			root = node
+		}
+
+		cursor = node
+	}
+
+	return root
+}

src/expression/bench_test.go

@@ -0,0 +1,17 @@
+package expression_test
+
+import (
+	"testing"
+
+	"git.akyoto.dev/cli/q/src/expression"
+	"git.akyoto.dev/cli/q/src/token"
+)
+
+func BenchmarkExpression(b *testing.B) {
+	src := []byte("(1+2-3*4)+(5*6-7+8)")
+	tokens := token.Tokenize(src)
+
+	for i := 0; i < b.N; i++ {
+		expression.Parse(tokens)
+	}
+}