Implemented expression parsing

src/build/token/Tokenize.go

@@ -1,7 +1,5 @@
 package token
 
-import "bytes"
-
 // Pre-allocate these byte buffers so we can re-use them
 // instead of allocating a new buffer every time.
 var (
@@ -12,7 +10,6 @@ var (
 	arrayStartBytes = []byte{'['}
 	arrayEndBytes   = []byte{']'}
 	separatorBytes  = []byte{','}
-	defineBytes     = []byte{':', '='}
 	newLineBytes    = []byte{'\n'}
 )
 
@@ -97,7 +94,7 @@ func Tokenize(buffer []byte) List {
 			}
 
 			// Numbers
-			if isNumberStart(buffer[i]) {
+			if isNumber(buffer[i]) {
 				position := i
 				i++
 
@@ -118,11 +115,6 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				if bytes.Equal(buffer[position:i], defineBytes) {
-					tokens = append(tokens, Token{Define, position, defineBytes})
-					continue
-				}
-
 				tokens = append(tokens, Token{Operator, position, buffer[position:i]})
 				continue
 			}
@@ -151,10 +143,6 @@ func isNumber(c byte) bool {
 	return (c >= '0' && c <= '9')
 }
 
-func isNumberStart(c byte) bool {
-	return isNumber(c) || c == '-'
-}
-
 func isOperator(c byte) bool {
-	return c == '=' || c == ':' || c == '+' || c == '-' || c == '*' || c == '/' || c == '<' || c == '>' || c == '!'
+	return c == '=' || c == ':' || c == '+' || c == '-' || c == '*' || c == '/' || c == '<' || c == '>' || c == '!' || c == '&' || c == '|' || c == '^' || c == '%' || c == '.'
 }