Implemented expression parsing

src/build/token/Kind.go

@@ -25,9 +25,6 @@ const (
 	// Number represents a series of numerical characters.
 	Number
 
-	// Define represents the assignment operator `:=` for a new variable.
-	Define
-
 	// Operator represents a mathematical operator.
 	Operator
 
@@ -66,7 +63,6 @@ func (kind Kind) String() string {
 		"Keyword",
 		"String",
 		"Number",
-		"Define",
 		"Operator",
 		"Separator",
 		"Comment",

src/build/token/List.go

@@ -13,7 +13,7 @@ func (list List) String() string {
 	var last Token
 
 	for _, t := range list {
-		if last.Kind == Keyword || last.Kind == Separator || last.Kind == Define || t.Kind == Define {
+		if last.Kind == Keyword || last.Kind == Separator || last.Kind == Operator || t.Kind == Operator {
 			builder.WriteByte(' ')
 		}
 

src/build/token/Token.go

@@ -12,16 +12,23 @@ type Token struct {
 }
 
 // After returns the position after the token.
-func (t Token) After() int {
+func (t *Token) After() int {
 	return t.Position + len(t.Bytes)
 }
 
 // String creates a human readable representation for debugging purposes.
-func (t Token) String() string {
+func (t *Token) String() string {
 	return fmt.Sprintf("%s %s", t.Kind, t.Text())
 }
 
+// Reset resets the token to default values.
+func (t *Token) Reset() {
+	t.Kind = Invalid
+	t.Position = 0
+	t.Bytes = nil
+}
+
 // Text returns the token text.
-func (t Token) Text() string {
+func (t *Token) Text() string {
 	return string(t.Bytes)
 }

src/build/token/Token_test.go

@@ -117,7 +117,7 @@ func TestNewline(t *testing.T) {
 }
 
 func TestNumber(t *testing.T) {
-	tokens := token.Tokenize([]byte(`123 -456`))
+	tokens := token.Tokenize([]byte(`123 456`))
 	assert.DeepEqual(t, tokens, token.List{
 		{
 			Kind:     token.Number,
@@ -126,13 +126,13 @@ func TestNumber(t *testing.T) {
 		},
 		{
 			Kind:     token.Number,
-			Bytes:    []byte("-456"),
+			Bytes:    []byte("456"),
 			Position: 4,
 		},
 		{
 			Kind:     token.EOF,
 			Bytes:    nil,
-			Position: 8,
+			Position: 7,
 		},
 	})
 }

src/build/token/Tokenize.go

@@ -1,7 +1,5 @@
 package token
 
-import "bytes"
-
 // Pre-allocate these byte buffers so we can re-use them
 // instead of allocating a new buffer every time.
 var (
@@ -12,7 +10,6 @@ var (
 	arrayStartBytes = []byte{'['}
 	arrayEndBytes   = []byte{']'}
 	separatorBytes  = []byte{','}
-	defineBytes     = []byte{':', '='}
 	newLineBytes    = []byte{'\n'}
 )
 
@@ -97,7 +94,7 @@ func Tokenize(buffer []byte) List {
 			}
 
 			// Numbers
-			if isNumberStart(buffer[i]) {
+			if isNumber(buffer[i]) {
 				position := i
 				i++
 
@@ -118,11 +115,6 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				if bytes.Equal(buffer[position:i], defineBytes) {
-					tokens = append(tokens, Token{Define, position, defineBytes})
-					continue
-				}
-
 				tokens = append(tokens, Token{Operator, position, buffer[position:i]})
 				continue
 			}
@@ -151,10 +143,6 @@ func isNumber(c byte) bool {
 	return (c >= '0' && c <= '9')
 }
 
-func isNumberStart(c byte) bool {
-	return isNumber(c) || c == '-'
-}
-
 func isOperator(c byte) bool {
-	return c == '=' || c == ':' || c == '+' || c == '-' || c == '*' || c == '/' || c == '<' || c == '>' || c == '!'
+	return c == '=' || c == ':' || c == '+' || c == '-' || c == '*' || c == '/' || c == '<' || c == '>' || c == '!' || c == '&' || c == '|' || c == '^' || c == '%' || c == '.'
 }