Implemented numbers with different bases

src/build/token/Tokenize.go

@@ -94,6 +94,38 @@ func Tokenize(buffer []byte) List {
 			tokens = append(tokens, Token{Kind: kind, Position: start, Length: Length(end - start)})
 			continue
 
+		case '0':
+			position := i
+			i++
+
+			if i >= Position(len(buffer)) {
+				tokens = append(tokens, Token{Kind: Number, Position: position, Length: 1})
+				break
+			}
+
+			filter := isDigit
+
+			switch buffer[i] {
+			case 'x':
+				i++
+				filter = isHexDigit
+
+			case 'b':
+				i++
+				filter = isBinaryDigit
+
+			case 'o':
+				i++
+				filter = isOctalDigit
+			}
+
+			for i < Position(len(buffer)) && filter(buffer[i]) {
+				i++
+			}
+
+			tokens = append(tokens, Token{Kind: Number, Position: position, Length: Length(i - position)})
+			continue
+
 		default:
 			if isIdentifierStart(buffer[i]) {
 				position := i
@@ -123,11 +155,11 @@ func Tokenize(buffer []byte) List {
 				continue
 			}
 
-			if isNumber(buffer[i]) {
+			if isDigit(buffer[i]) {
 				position := i
 				i++
 
-				for i < Position(len(buffer)) && isNumber(buffer[i]) {
+				for i < Position(len(buffer)) && isDigit(buffer[i]) {
 					i++
 				}
 
@@ -235,7 +267,7 @@ func Tokenize(buffer []byte) List {
 }
 
 func isIdentifier(c byte) bool {
-	return isLetter(c) || isNumber(c) || c == '_'
+	return isLetter(c) || isDigit(c) || c == '_'
 }
 
 func isIdentifierStart(c byte) bool {
@@ -246,8 +278,20 @@ func isLetter(c byte) bool {
 	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
 }
 
-func isNumber(c byte) bool {
-	return (c >= '0' && c <= '9')
+func isDigit(c byte) bool {
+	return c >= '0' && c <= '9'
+}
+
+func isHexDigit(c byte) bool {
+	return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F')
+}
+
+func isBinaryDigit(c byte) bool {
+	return c == '0' || c == '1'
+}
+
+func isOctalDigit(c byte) bool {
+	return c >= '0' && c <= '7'
 }
 
 func isOperator(c byte) bool {