Cleaned up tokenizer

src/build/token/Tokenize.go

@@ -19,46 +19,28 @@ var (
 func Tokenize(buffer []byte) List {
 	var (
 		i      int
-		tokens = make(List, 0, len(buffer)/2)
+		tokens = make(List, 0, 8+len(buffer)/2)
 	)
 
 	for i < len(buffer) {
 		switch buffer[i] {
-		// Whitespace
 		case ' ', '\t':
-		// Separator
 		case ',':
 			tokens = append(tokens, Token{Kind: Separator, Position: i, Bytes: separatorBytes})
-
-		// Parentheses start
 		case '(':
 			tokens = append(tokens, Token{Kind: GroupStart, Position: i, Bytes: groupStartBytes})
-
-		// Parentheses end
 		case ')':
 			tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Bytes: groupEndBytes})
-
-		// Block start
 		case '{':
 			tokens = append(tokens, Token{Kind: BlockStart, Position: i, Bytes: blockStartBytes})
-
-		// Block end
 		case '}':
 			tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Bytes: blockEndBytes})
-
-		// Array start
 		case '[':
 			tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Bytes: arrayStartBytes})
-
-		// Array end
 		case ']':
 			tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Bytes: arrayEndBytes})
-
-		// New line
 		case '\n':
 			tokens = append(tokens, Token{Kind: NewLine, Position: i, Bytes: newLineBytes})
-
-		// Comment
 		case '/':
 			if i+1 >= len(buffer) || buffer[i+1] != '/' {
 				position := i
@@ -69,19 +51,18 @@ func Tokenize(buffer []byte) List {
 				}
 
 				tokens = append(tokens, Token{Kind: Operator, Position: position, Bytes: buffer[position:i]})
-				continue
+			} else {
+				position := i
+
+				for i < len(buffer) && buffer[i] != '\n' {
+					i++
+				}
+
+				tokens = append(tokens, Token{Kind: Comment, Position: position, Bytes: buffer[position:i]})
 			}
 
-			position := i
-
-			for i < len(buffer) && buffer[i] != '\n' {
-				i++
-			}
-
-			tokens = append(tokens, Token{Kind: Comment, Position: position, Bytes: buffer[position:i]})
 			continue
 
-		// String
 		case '"':
 			start := i
 			end := len(buffer)
@@ -101,7 +82,6 @@ func Tokenize(buffer []byte) List {
 			continue
 
 		default:
-			// Identifier
 			if isIdentifierStart(buffer[i]) {
 				position := i
 				i++
@@ -122,7 +102,6 @@ func Tokenize(buffer []byte) List {
 				continue
 			}
 
-			// Number
 			if isNumber(buffer[i]) {
 				position := i
 				i++
@@ -135,7 +114,6 @@ func Tokenize(buffer []byte) List {
 				continue
 			}
 
-			// Operator
 			if isOperator(buffer[i]) {
 				position := i
 				i++
@@ -148,7 +126,6 @@ func Tokenize(buffer []byte) List {
 				continue
 			}
 
-			// Invalid character
 			tokens = append(tokens, Token{Kind: Invalid, Position: i, Bytes: buffer[i : i+1]})
 		}