Improved tokenizer

src/build/Build.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"git.akyoto.dev/cli/q/src/compiler"
 	"git.akyoto.dev/cli/q/src/elf"
 	"git.akyoto.dev/cli/q/src/errors"
 )
@@ -12,7 +13,6 @@ import (
 // Build describes a compiler build.
 type Build struct {
 	Directory       string
-	Verbose         bool
 	WriteExecutable bool
 }
 
@@ -36,7 +36,7 @@ func (build *Build) Run() error {
 		return &errors.InvalidDirectory{Path: build.Directory}
 	}
 
-	functions, err := build.Compile()
+	functions, err := compiler.Compile(build.Directory)
 
 	if err != nil {
 		return err
@@ -46,7 +46,7 @@ func (build *Build) Run() error {
 		return nil
 	}
 
-	code, data := build.Finalize(functions)
+	code, data := compiler.Finalize(functions)
 	return writeToDisk(build.Executable(), code, data)
 }
 

src/build/Compile.go

@@ -1,40 +0,0 @@
-package build
-
-import "sync"
-
-// Compile compiles all the functions.
-func (build *Build) Compile() (map[string]*Function, error) {
-	functions, errors := Scan(build.Directory)
-	wg := sync.WaitGroup{}
-	allFunctions := map[string]*Function{}
-
-	for functions != nil || errors != nil {
-		select {
-		case err, ok := <-errors:
-			if !ok {
-				errors = nil
-				continue
-			}
-
-			return nil, err
-
-		case function, ok := <-functions:
-			if !ok {
-				functions = nil
-				continue
-			}
-
-			wg.Add(1)
-
-			go func() {
-				defer wg.Done()
-				function.Compile()
-			}()
-
-			allFunctions[function.Name] = function
-		}
-	}
-
-	wg.Wait()
-	return allFunctions, nil
-}

src/build/Finalize.go

@@ -1,23 +0,0 @@
-package build
-
-import (
-	"git.akyoto.dev/cli/q/src/asm"
-	"git.akyoto.dev/cli/q/src/linux"
-	"git.akyoto.dev/cli/q/src/register"
-)
-
-// Finalize generates the final machine code.
-func (build *Build) Finalize(functions map[string]*Function) ([]byte, []byte) {
-	a := asm.New()
-
-	for _, f := range functions {
-		a.Merge(&f.Assembler)
-	}
-
-	a.MoveRegisterNumber(register.Syscall0, linux.Exit)
-	a.MoveRegisterNumber(register.Syscall1, 0)
-	a.Syscall()
-
-	code, data := a.Finalize(build.Verbose)
-	return code, data
-}

src/build/Function.go

@@ -1,35 +0,0 @@
-package build
-
-import (
-	"git.akyoto.dev/cli/q/src/asm"
-	"git.akyoto.dev/cli/q/src/linux"
-	"git.akyoto.dev/cli/q/src/register"
-	"git.akyoto.dev/cli/q/src/token"
-)
-
-// Function represents a function.
-type Function struct {
-	Name      string
-	Head      token.List
-	Body      token.List
-	Assembler asm.Assembler
-}
-
-// Compile turns a function into machine code.
-func (f *Function) Compile() {
-	for i, t := range f.Body {
-		if t.Kind == token.Identifier && t.String() == "print" {
-			message := f.Body[i+2].Bytes
-			f.Assembler.MoveRegisterNumber(register.Syscall0, linux.Write)
-			f.Assembler.MoveRegisterNumber(register.Syscall1, 1)
-			f.Assembler.MoveRegisterData(register.Syscall2, message)
-			f.Assembler.MoveRegisterNumber(register.Syscall3, uint64(len(message)))
-			f.Assembler.Syscall()
-		}
-	}
-}
-
-// String returns the function name.
-func (f *Function) String() string {
-	return f.Name
-}

src/build/Scan.go

@@ -1,105 +0,0 @@
-package build
-
-import (
-	"os"
-	"path/filepath"
-	"strings"
-	"sync"
-
-	"git.akyoto.dev/cli/q/src/directory"
-	"git.akyoto.dev/cli/q/src/token"
-)
-
-// Scan scans the directory.
-func Scan(path string) (<-chan *Function, <-chan error) {
-	functions := make(chan *Function, 16)
-	errors := make(chan error)
-
-	go func() {
-		scanDirectory(path, functions, errors)
-		close(functions)
-		close(errors)
-	}()
-
-	return functions, errors
-}
-
-// scanDirectory scans the directory without channel allocations.
-func scanDirectory(path string, functions chan<- *Function, errors chan<- error) {
-	wg := sync.WaitGroup{}
-
-	directory.Walk(path, func(name string) {
-		if !strings.HasSuffix(name, ".q") {
-			return
-		}
-
-		fullPath := filepath.Join(path, name)
-		wg.Add(1)
-
-		go func() {
-			defer wg.Done()
-			err := scanFile(fullPath, functions)
-
-			if err != nil {
-				errors <- err
-			}
-		}()
-	})
-
-	wg.Wait()
-}
-
-// scanFile scans a single file.
-func scanFile(path string, functions chan<- *Function) error {
-	contents, err := os.ReadFile(path)
-
-	if err != nil {
-		return err
-	}
-
-	tokens := token.Tokenize(contents)
-
-	var (
-		groupLevel  = 0
-		blockLevel  = 0
-		headerStart = -1
-		bodyStart   = -1
-	)
-
-	for i, t := range tokens {
-		switch t.Kind {
-		case token.Identifier:
-			if blockLevel == 0 && groupLevel == 0 {
-				headerStart = i
-			}
-
-		case token.GroupStart:
-			groupLevel++
-
-		case token.GroupEnd:
-			groupLevel--
-
-		case token.BlockStart:
-			blockLevel++
-
-			if blockLevel == 1 {
-				bodyStart = i
-			}
-
-		case token.BlockEnd:
-			blockLevel--
-
-			if blockLevel == 0 {
-				function := &Function{
-					Name: tokens[headerStart].String(),
-					Head: tokens[headerStart:bodyStart],
-					Body: tokens[bodyStart : i+1],
-				}
-
-				functions <- function
-			}
-		}
-	}
-
-	return nil
-}