Reduced token size

src/build/token/Token.go

@@ -1,7 +1,6 @@
 package token
 
 import (
-	"fmt"
 	"unsafe"
 )
 
@@ -9,29 +8,44 @@ import (
 // The characters that make up an identifier are grouped into a single token.
 // This makes parsing easier and allows us to do better syntax checks.
 type Token struct {
-	Bytes    []byte
 	Position Position
+	Length   Length
 	Kind     Kind
 }
 
-// End returns the position after the token.
-func (t *Token) End() Position {
-	return t.Position + Position(len(t.Bytes))
+// Bytes returns the byte slice.
+func (t Token) Bytes(buffer []byte) []byte {
+	return buffer[t.Position : t.Position+Position(t.Length)]
 }
 
-// String creates a human readable representation for debugging purposes.
-func (t *Token) String() string {
-	return fmt.Sprintf("%s %s", t.Kind, t.Text())
+// End returns the position after the token.
+func (t Token) End() Position {
+	return t.Position + Position(t.Length)
+}
+
+// IsAssignment returns true if the token is an assignment operator.
+func (t Token) IsAssignment() bool {
+	return t.Kind > _assignments && t.Kind < _assignmentsEnd
+}
+
+// IsKeyword returns true if the token is a keyword.
+func (t Token) IsKeyword() bool {
+	return t.Kind > _keywords && t.Kind < _keywordsEnd
+}
+
+// IsOperator returns true if the token is an operator.
+func (t Token) IsOperator() bool {
+	return t.Kind > _operators && t.Kind < _operatorsEnd
 }
 
 // Reset resets the token to default values.
 func (t *Token) Reset() {
-	t.Kind = Invalid
 	t.Position = 0
-	t.Bytes = nil
+	t.Length = 0
+	t.Kind = Invalid
 }
 
 // Text returns the token text.
-func (t *Token) Text() string {
-	return unsafe.String(unsafe.SliceData(t.Bytes), len(t.Bytes))
+func (t Token) Text(buffer []byte) string {
+	return unsafe.String(unsafe.SliceData(t.Bytes(buffer)), t.Length)
 }