Reduced token size

src/build/token/Position.go

@@ -0,0 +1,4 @@
+package token
+
+// Position is the data type for storing file offsets.
+type Position = uint32

src/build/token/Token.go

@@ -10,13 +10,13 @@ import (
 // This makes parsing easier and allows us to do better syntax checks.
 type Token struct {
 	Bytes    []byte
-	Position int
+	Position Position
 	Kind     Kind
 }
 
 // End returns the position after the token.
-func (t *Token) End() int {
-	return t.Position + len(t.Bytes)
+func (t *Token) End() Position {
+	return t.Position + Position(len(t.Bytes))
 }
 
 // String creates a human readable representation for debugging purposes.

src/build/token/Tokenize.go

@@ -18,11 +18,11 @@ var (
 // Tokenize turns the file contents into a list of tokens.
 func Tokenize(buffer []byte) List {
 	var (
-		i      int
+		i      Position
 		tokens = make(List, 0, 8+len(buffer)/2)
 	)
 
-	for i < len(buffer) {
+	for i < Position(len(buffer)) {
 		switch buffer[i] {
 		case ' ', '\t':
 		case ',':
@@ -42,11 +42,11 @@ func Tokenize(buffer []byte) List {
 		case '\n':
 			tokens = append(tokens, Token{Kind: NewLine, Position: i, Bytes: newLineBytes})
 		case '/':
-			if i+1 >= len(buffer) || buffer[i+1] != '/' {
+			if i+1 >= Position(len(buffer)) || buffer[i+1] != '/' {
 				position := i
 				i++
 
-				for i < len(buffer) && isOperator(buffer[i]) {
+				for i < Position(len(buffer)) && isOperator(buffer[i]) {
 					i++
 				}
 
@@ -54,7 +54,7 @@ func Tokenize(buffer []byte) List {
 			} else {
 				position := i
 
-				for i < len(buffer) && buffer[i] != '\n' {
+				for i < Position(len(buffer)) && buffer[i] != '\n' {
 					i++
 				}
 
@@ -65,10 +65,10 @@ func Tokenize(buffer []byte) List {
 
 		case '"':
 			start := i
-			end := len(buffer)
+			end := Position(len(buffer))
 			i++
 
-			for i < len(buffer) {
+			for i < Position(len(buffer)) {
 				if buffer[i] == '"' {
 					end = i + 1
 					i++
@@ -86,7 +86,7 @@ func Tokenize(buffer []byte) List {
 				position := i
 				i++
 
-				for i < len(buffer) && isIdentifier(buffer[i]) {
+				for i < Position(len(buffer)) && isIdentifier(buffer[i]) {
 					i++
 				}
 
@@ -106,7 +106,7 @@ func Tokenize(buffer []byte) List {
 				position := i
 				i++
 
-				for i < len(buffer) && isNumber(buffer[i]) {
+				for i < Position(len(buffer)) && isNumber(buffer[i]) {
 					i++
 				}
 
@@ -118,7 +118,7 @@ func Tokenize(buffer []byte) List {
 				position := i
 				i++
 
-				for i < len(buffer) && isOperator(buffer[i]) {
+				for i < Position(len(buffer)) && isOperator(buffer[i]) {
 					i++
 				}