Added define operator

src/build/token/Tokenize.go

@@ -1,5 +1,7 @@
 package token
 
+import "bytes"
+
 // Pre-allocate these byte buffers so we can re-use them
 // instead of allocating a new buffer every time.
 var (
@@ -10,6 +12,7 @@ var (
 	arrayStartBytes = []byte{'['}
 	arrayEndBytes   = []byte{']'}
 	separatorBytes  = []byte{','}
+	defineBytes     = []byte{':', '='}
 	newLineBytes    = []byte{'\n'}
 )
 
@@ -38,12 +41,7 @@ func Tokenize(buffer []byte) List {
 				i++
 			}
 
-			tokens = append(tokens, Token{
-				String,
-				start,
-				buffer[start:end],
-			})
-
+			tokens = append(tokens, Token{String, start, buffer[start:end]})
 			continue
 
 		// Parentheses start
@@ -88,11 +86,7 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				token := Token{
-					Identifier,
-					position,
-					buffer[position:i],
-				}
+				token := Token{Identifier, position, buffer[position:i]}
 
 				if Keywords[string(token.Bytes)] {
 					token.Kind = Keyword
@@ -111,12 +105,7 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				tokens = append(tokens, Token{
-					Number,
-					position,
-					buffer[position:i],
-				})
-
+				tokens = append(tokens, Token{Number, position, buffer[position:i]})
 				continue
 			}
 
@@ -129,12 +118,12 @@ func Tokenize(buffer []byte) List {
 					i++
 				}
 
-				tokens = append(tokens, Token{
-					Operator,
-					position,
-					buffer[position:i],
-				})
+				if bytes.Equal(buffer[position:i], defineBytes) {
+					tokens = append(tokens, Token{Define, position, defineBytes})
+					continue
+				}
 
+				tokens = append(tokens, Token{Operator, position, buffer[position:i]})
 				continue
 			}
 		}