cli/q
Code Issues 1 Pull Requests Activity

Added a tokenizer

Browse Source
This commit is contained in:
Eduard Urbach
2023-10-31 11:57:37 +01:00
parent 8b19989372
commit ac157e580c
7 changed files with 274 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/build/Scan.go
View File

@ -8,6 +8,7 @@ import (
"git.akyoto.dev/cli/q/src/directory"
"git.akyoto.dev/cli/q/src/log"
"git.akyoto.dev/cli/q/src/token"
)
// Scan scans the directory.
@ -51,13 +52,17 @@ func scanDirectory(path string, functions chan<- *Function, errors chan<- error)
// scanFile scans a single file.
func scanFile(path string, functions chan<- *Function) error {
log.Info.Println(path)
contents, err := os.ReadFile(path)
if err != nil {
return err
}
log.Info.Println(string(contents))
tokens := token.Tokenize(contents)
for _, t := range tokens {
log.Info.Println(t.Kind, t.Position, strings.TrimSpace(t.String()))
}
return nil
}