From 7f86dc674788527118fb9fc384e397eb94f90b3f Mon Sep 17 00:00:00 2001
From: Eduard Urbach <eduard@urbach.dev>
Date: Wed, 19 Mar 2025 22:29:45 +0100
Subject: [PATCH] Improved tokenizer

---
 src/token/Reader.go   | 8 ++++++--
 src/token/Tokenize.go | 2 --
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/token/Reader.go b/src/token/Reader.go
index 401107b..3a77cc0 100644
--- a/src/token/Reader.go
+++ b/src/token/Reader.go
@@ -33,6 +33,10 @@ func (t *Reader) Next() byte {
 }
 
 func (t *Reader) read() error {
+	if t.Position >= t.Size {
+		return nil
+	}
+
 	n, err := t.File.Read(t.Buffer[len(t.Buffer):cap(t.Buffer)])
 	t.Buffer = t.Buffer[:len(t.Buffer)+n]
 
@@ -66,6 +70,6 @@ func (t *Reader) Open(path string) (err error) {
 	}
 
 	t.Size = Position(info.Size())
-	t.Buffer = make([]byte, 0, t.Size+1)
-	return nil
+	t.Buffer = make([]byte, 0, 4096)
+	return t.read()
 }
diff --git a/src/token/Tokenize.go b/src/token/Tokenize.go
index ee536ec..b981711 100644
--- a/src/token/Tokenize.go
+++ b/src/token/Tokenize.go
@@ -6,8 +6,6 @@ func Tokenize(reader *Reader) List {
 		tokens = make(List, 0, 8+reader.Size/2)
 	)
 
-	reader.read()
-
 	for reader.Position < reader.Size {
 		switch reader.Current() {
 		case ' ', '\t':