Implemented token splitting as a generator

src/expression/List.go

@@ -8,11 +8,10 @@ import (
 func NewList(tokens token.List) []*Expression {
 	var list []*Expression
 
-	tokens.Split(func(parameter token.List) error {
-		expression := Parse(parameter)
+	for param := range tokens.Split {
+		expression := Parse(param)
 		list = append(list, expression)
-		return nil
-	})
+	}
 
 	return list
 }

src/scanner/scanFunctionSignature.go

@@ -96,30 +96,24 @@ func scanFunctionSignature(file *fs.File, tokens token.List, i int, delimiter to
 
 		outputTokens := tokens[typeStart:typeEnd]
 
-		err := outputTokens.Split(func(tokens token.List) error {
-			function.Output = append(function.Output, core.NewParameter(tokens))
-			return nil
-		})
-
-		if err != nil {
-			return nil, i, err
+		for param := range outputTokens.Split {
+			function.Output = append(function.Output, core.NewParameter(param))
 		}
 	}
 
 	parameters := tokens[paramsStart:paramsEnd]
 
-	err := parameters.Split(func(tokens token.List) error {
-		if len(tokens) == 0 {
-			return errors.New(errors.MissingParameter, file, parameters[0].Position)
+	for param := range parameters.Split {
+		if len(param) == 0 {
+			return nil, i, errors.New(errors.MissingParameter, file, parameters[0].Position)
 		}
 
-		if len(tokens) == 1 {
-			return errors.New(errors.MissingType, file, tokens[0].End())
+		if len(param) == 1 {
+			return nil, i, errors.New(errors.MissingType, file, param[0].End())
 		}
 
-		function.Input = append(function.Input, core.NewParameter(tokens))
-		return nil
-	})
+		function.Input = append(function.Input, core.NewParameter(param))
+	}
 
-	return function, i, err
+	return function, i, nil
 }

src/token/List.go

@@ -30,9 +30,9 @@ func (list List) LastIndexKind(kind Kind) int {
 }
 
 // Split calls the callback function on each set of tokens in a comma separated list.
-func (list List) Split(call func(List) error) error {
+func (list List) Split(yield func(List) bool) {
 	if len(list) == 0 {
-		return nil
+		return
 	}
 
 	start := 0
@@ -52,18 +52,16 @@ func (list List) Split(call func(List) error) error {
 			}
 
 			parameter := list[start:i]
-			err := call(parameter)
 
-			if err != nil {
-				return err
+			if !yield(parameter) {
+				return
 			}
 
 			start = i + 1
 		}
 	}
 
-	parameter := list[start:]
-	return call(parameter)
+	yield(list[start:])
 }
 
 // Text returns the concatenated token text.

src/token/List_test.go

@@ -1,7 +1,6 @@
 package token_test
 
 import (
-	"errors"
 	"testing"
 
 	"git.urbach.dev/cli/q/src/token"
@@ -23,35 +22,28 @@ func TestSplit(t *testing.T) {
 	tokens := token.Tokenize(src)
 	parameters := []string{}
 
-	err := tokens.Split(func(parameter token.List) error {
-		parameters = append(parameters, parameter.Text(src))
-		return nil
-	})
+	for param := range tokens.Split {
+		parameters = append(parameters, param.Text(src))
+	}
 
-	assert.Nil(t, err)
 	assert.DeepEqual(t, parameters, []string{"1+2", "3*4", "5*6", "7+8"})
 }
 
+func TestSplitBreak(t *testing.T) {
+	src := []byte("1,2")
+	tokens := token.Tokenize(src)
+
+	for range tokens.Split {
+		break
+	}
+}
+
 func TestSplitEmpty(t *testing.T) {
 	tokens := token.List{}
 
-	err := tokens.Split(func(parameter token.List) error {
-		return errors.New("error")
-	})
-
-	assert.Nil(t, err)
-}
-
-func TestSplitError(t *testing.T) {
-	src := []byte("1,2,3")
-	tokens := token.Tokenize(src)
-
-	err := tokens.Split(func(parameter token.List) error {
-		return errors.New("error")
-	})
-
-	assert.NotNil(t, err)
-	assert.Equal(t, err.Error(), "error")
+	for range tokens.Split {
+		t.Fail()
+	}
 }
 
 func TestSplitGroups(t *testing.T) {
@@ -59,12 +51,10 @@ func TestSplitGroups(t *testing.T) {
 	tokens := token.Tokenize(src)
 	parameters := []string{}
 
-	err := tokens.Split(func(parameter token.List) error {
-		parameters = append(parameters, parameter.Text(src))
-		return nil
-	})
+	for param := range tokens.Split {
+		parameters = append(parameters, param.Text(src))
+	}
 
-	assert.Nil(t, err)
 	assert.DeepEqual(t, parameters, []string{"f(1,2)", "g(3,4)"})
 }
 
@@ -73,11 +63,9 @@ func TestSplitSingle(t *testing.T) {
 	tokens := token.Tokenize(src)
 	parameters := []string{}
 
-	err := tokens.Split(func(parameter token.List) error {
-		parameters = append(parameters, parameter.Text(src))
-		return nil
-	})
+	for param := range tokens.Split {
+		parameters = append(parameters, param.Text(src))
+	}
 
-	assert.Nil(t, err)
 	assert.DeepEqual(t, parameters, []string{"123"})
 }