diff --git a/README.md b/README.md index d636314..e79d420 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,7 @@ This is what generates expressions from tokens. - [x] `<<=`, `>>=` - [x] `==`, `!=`, `<`, `<=`, `>`, `>=` - [x] `&&`, `||` +- [ ] `!`, `-` ### Architecture diff --git a/src/build/expression/Expression_test.go b/src/build/expression/Expression_test.go index a313cf3..ebcc895 100644 --- a/src/build/expression/Expression_test.go +++ b/src/build/expression/Expression_test.go @@ -17,17 +17,22 @@ func TestParse(t *testing.T) { }{ {"Identity", "1", "1"}, {"Basic calculation", "1+2", "(+ 1 2)"}, + {"Same operator", "1+2+3", "(+ (+ 1 2) 3)"}, {"Same operator 2", "1+2+3+4", "(+ (+ (+ 1 2) 3) 4)"}, + {"Different operator", "1+2-3", "(- (+ 1 2) 3)"}, {"Different operator 2", "1+2-3+4", "(+ (- (+ 1 2) 3) 4)"}, {"Different operator 3", "1+2-3+4-5", "(- (+ (- (+ 1 2) 3) 4) 5)"}, + {"Grouped identity", "(1)", "1"}, {"Grouped identity 2", "((1))", "1"}, {"Grouped identity 3", "(((1)))", "1"}, + {"Adding identity", "(1)+(2)", "(+ 1 2)"}, {"Adding identity 2", "(1)+(2)+(3)", "(+ (+ 1 2) 3)"}, {"Adding identity 3", "(1)+(2)+(3)+(4)", "(+ (+ (+ 1 2) 3) 4)"}, + {"Grouping", "(1+2)", "(+ 1 2)"}, {"Grouping 2", "(1+2+3)", "(+ (+ 1 2) 3)"}, {"Grouping 3", "((1)+(2)+(3))", "(+ (+ 1 2) 3)"}, @@ -35,9 +40,11 @@ func TestParse(t *testing.T) { {"Grouping right", "1*(2+3)", "(* 1 (+ 2 3))"}, {"Grouping same operator", "1+(2+3)", "(+ 1 (+ 2 3))"}, {"Grouping same operator 2", "1+(2+3)+(4+5)", "(+ (+ 1 (+ 2 3)) (+ 4 5))"}, + {"Two groups", "(1+2)*(3+4)", "(* (+ 1 2) (+ 3 4))"}, {"Two groups 2", "(1+2-3)*(3+4-5)", "(* (- (+ 1 2) 3) (- (+ 3 4) 5))"}, {"Two groups 3", "(1+2)*(3+4-5)", "(* (+ 1 2) (- (+ 3 4) 5))"}, + {"Operator priority", "1+2*3", "(+ 1 (* 2 3))"}, {"Operator priority 2", "1*2+3", "(+ (* 1 2) 3)"}, {"Operator priority 3", "1+2*3+4", "(+ (+ 1 (* 2 3)) 4)"}, @@ -46,10 +53,25 @@ func TestParse(t *testing.T) { {"Operator priority 6", "1+2*3+4*5", "(+ (+ 1 (* 2 3)) (* 4 5))"}, {"Operator priority 7", "1+2*3*4*5*6", "(+ 1 (* (* (* (* 2 3) 4) 5) 6))"}, {"Operator priority 8", "1*2*3+4*5*6", "(+ (* (* 1 2) 3) (* (* 4 5) 6))"}, + {"Complex", "(1+2-3*4)*(5+6-7*8)", "(* (- (+ 1 2) (* 3 4)) (- (+ 5 6) (* 7 8)))"}, {"Complex 2", "(1+2*3-4)*(5+6*7-8)", "(* (- (+ 1 (* 2 3)) 4) (- (+ 5 (* 6 7)) 8))"}, {"Complex 3", "(1+2*3-4)*(5+6*7-8)+9-10*11", "(- (+ (* (- (+ 1 (* 2 3)) 4) (- (+ 5 (* 6 7)) 8)) 9) (* 10 11))"}, - {"Unary", "!", "!"}, + + {"Unary not", "!", "!"}, + {"Unary not 2", "!a", "(! a)"}, + {"Unary not 3", "!(!a)", "(! (! a))"}, + {"Unary not 4", "!(a||b)", "(! (|| a b))"}, + {"Unary not 5", "a || !b", "(|| a (! b))"}, + + {"Unary minus", "-", "-"}, + {"Unary minus 2", "-a", "(- a)"}, + {"Unary minus 3", "-(-a)", "(- (- a))"}, + {"Unary minus 4", "-a+b", "(+ (- a) b)"}, + {"Unary minus 5", "-(a+b)", "(- (+ a b))"}, + {"Unary minus 6", "a + -b", "(+ a (- b))"}, + {"Unary minus 7", "-a + -b", "(+ (- a) (- b))"}, + {"Function calls", "a()", "(λ a)"}, {"Function calls 2", "a(1)", "(λ a 1)"}, {"Function calls 3", "a(1)+1", "(+ (λ a 1) 1)"}, @@ -77,8 +99,10 @@ func TestParse(t *testing.T) { {"Function calls 25", "a(1-2*3)", "(λ a (- 1 (* 2 3)))"}, {"Function calls 26", "1+2*a()+4", "(+ (+ 1 (* 2 (λ a))) 4)"}, {"Function calls 27", "sum(a,b)*2+15*4", "(+ (* (λ sum a b) 2) (* 15 4))"}, + {"Package function calls", "math.sum(a,b)", "(λ (. math sum) a b)"}, {"Package function calls 2", "generic.math.sum(a,b)", "(λ (. (. generic math) sum) a b)"}, + {"Array access", "a[0]", "(@ a 0)"}, {"Array access 2", "a[b+c]", "(@ a (+ b c))"}, {"Array access 3", "a.b[c]", "(@ (. a b) c)"}, diff --git a/src/build/expression/Operator.go b/src/build/expression/Operator.go index 1d9e18b..2a4ecdf 100644 --- a/src/build/expression/Operator.go +++ b/src/build/expression/Operator.go @@ -10,7 +10,7 @@ import ( type Operator struct { Symbol string Precedence int8 - Operands int + Operands int8 } // Operators defines the operators used in the language. @@ -19,6 +19,7 @@ var Operators = map[token.Kind]*Operator{ token.Period: {".", 13, 2}, token.Call: {"λ", 12, 1}, token.Array: {"@", 12, 2}, + token.Negate: {"-", 11, 1}, token.Not: {"!", 11, 1}, token.Mul: {"*", 10, 2}, token.Div: {"/", 10, 2}, @@ -73,7 +74,7 @@ func numOperands(symbol token.Kind) int { return -1 } - return operator.Operands + return int(operator.Operands) } func precedence(symbol token.Kind) int8 { diff --git a/src/build/expression/Parse.go b/src/build/expression/Parse.go index 7fbd6ad..22dbd39 100644 --- a/src/build/expression/Parse.go +++ b/src/build/expression/Parse.go @@ -117,7 +117,11 @@ func Parse(tokens []token.Token) *Expression { newPrecedence := node.Precedence if newPrecedence > oldPrecedence { - cursor.LastChild().Replace(node) + if len(cursor.Children) == numOperands(cursor.Token.Kind) { + cursor.LastChild().Replace(node) + } else { + cursor.AddChild(node) + } } else { start := cursor diff --git a/src/build/token/Kind.go b/src/build/token/Kind.go index 33df902..871e49f 100644 --- a/src/build/token/Kind.go +++ b/src/build/token/Kind.go @@ -38,10 +38,11 @@ const ( Shr // >> LogicalAnd // && LogicalOr // || + Not // ! (unary) + Negate // - (unary) Equal // == Less // < Greater // > - Not // ! NotEqual // != LessEqual // <= GreaterEqual // >= diff --git a/src/build/token/Token.go b/src/build/token/Token.go index 134f054..99cbe97 100644 --- a/src/build/token/Token.go +++ b/src/build/token/Token.go @@ -28,6 +28,11 @@ func (t Token) IsAssignment() bool { return t.Kind > _assignments && t.Kind < _assignmentsEnd } +// IsExpressionStart returns true if the token starts an expression. +func (t Token) IsExpressionStart() bool { + return t.Kind == GroupStart || t.Kind == ArrayStart || t.Kind == BlockStart +} + // IsKeyword returns true if the token is a keyword. func (t Token) IsKeyword() bool { return t.Kind > _keywords && t.Kind < _keywordsEnd diff --git a/src/build/token/Tokenize.go b/src/build/token/Tokenize.go index 610a36f..63ad4f7 100644 --- a/src/build/token/Tokenize.go +++ b/src/build/token/Tokenize.go @@ -26,8 +26,28 @@ func Tokenize(buffer []byte) List { tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Length: 1}) case '\n': tokens = append(tokens, Token{Kind: NewLine, Position: i, Length: 1}) + case '-': + if len(tokens) == 0 || tokens[len(tokens)-1].IsOperator() || tokens[len(tokens)-1].IsExpressionStart() { + tokens = append(tokens, Token{Kind: Negate, Position: i, Length: 1}) + } else { + if i+1 < Position(len(buffer)) && buffer[i+1] == '=' { + tokens = append(tokens, Token{Kind: SubAssign, Position: i, Length: 2}) + i++ + } else { + tokens = append(tokens, Token{Kind: Sub, Position: i, Length: 1}) + } + } + case '/': - if i+1 >= Position(len(buffer)) || buffer[i+1] != '/' { + if i+1 < Position(len(buffer)) && buffer[i+1] == '/' { + position := i + + for i < Position(len(buffer)) && buffer[i] != '\n' { + i++ + } + + tokens = append(tokens, Token{Kind: Comment, Position: position, Length: Length(i - position)}) + } else { position := i i++ @@ -45,14 +65,6 @@ func Tokenize(buffer []byte) List { } tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(i - position)}) - } else { - position := i - - for i < Position(len(buffer)) && buffer[i] != '\n' { - i++ - } - - tokens = append(tokens, Token{Kind: Comment, Position: position, Length: Length(i - position)}) } continue diff --git a/src/build/token/Tokenize_test.go b/src/build/token/Tokenize_test.go index 19f57e5..de67fbd 100644 --- a/src/build/token/Tokenize_test.go +++ b/src/build/token/Tokenize_test.go @@ -83,13 +83,81 @@ func TestNumber(t *testing.T) { } func TestOperator(t *testing.T) { - tokens := token.Tokenize([]byte(`+ - * /`)) + tokens := token.Tokenize([]byte(`a + b - c * d / e`)) expected := []token.Kind{ + token.Identifier, token.Add, + token.Identifier, token.Sub, + token.Identifier, token.Mul, + token.Identifier, token.Div, + token.Identifier, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } +} + +func TestNegateFirstToken(t *testing.T) { + tokens := token.Tokenize([]byte(`-a`)) + + expected := []token.Kind{ + token.Negate, + token.Identifier, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } +} + +func TestNegateAfterGroupStart(t *testing.T) { + tokens := token.Tokenize([]byte(`(-a)`)) + + expected := []token.Kind{ + token.GroupStart, + token.Negate, + token.Identifier, + token.GroupEnd, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } +} + +func TestNegateSub(t *testing.T) { + tokens := token.Tokenize([]byte(`-a-b`)) + + expected := []token.Kind{ + token.Negate, + token.Identifier, + token.Sub, + token.Identifier, + token.EOF, + } + + for i, kind := range expected { + assert.Equal(t, tokens[i].Kind, kind) + } +} + +func TestNegateAfterOperator(t *testing.T) { + tokens := token.Tokenize([]byte(`-a + -b`)) + + expected := []token.Kind{ + token.Negate, + token.Identifier, + token.Add, + token.Negate, + token.Identifier, token.EOF, } @@ -99,18 +167,28 @@ func TestOperator(t *testing.T) { } func TestOperatorAssign(t *testing.T) { - tokens := token.Tokenize([]byte(`+= -= *= /= &= |= ^= <<= >>=`)) + tokens := token.Tokenize([]byte(`a += b -= c *= d /= e &= f |= g ^= h <<= i >>= j`)) expected := []token.Kind{ + token.Identifier, token.AddAssign, + token.Identifier, token.SubAssign, + token.Identifier, token.MulAssign, + token.Identifier, token.DivAssign, + token.Identifier, token.AndAssign, + token.Identifier, token.OrAssign, + token.Identifier, token.XorAssign, + token.Identifier, token.ShlAssign, + token.Identifier, token.ShrAssign, + token.Identifier, token.EOF, }