52 lines
1.3 KiB
Go
52 lines
1.3 KiB
Go
package token
|
|
|
|
import (
|
|
"unsafe"
|
|
)
|
|
|
|
// Token represents a single element in a source file.
|
|
// The characters that make up an identifier are grouped into a single token.
|
|
// This makes parsing easier and allows us to do better syntax checks.
|
|
type Token struct {
|
|
Position Position
|
|
Length Length
|
|
Kind Kind
|
|
}
|
|
|
|
// Bytes returns the byte slice.
|
|
func (t Token) Bytes(buffer []byte) []byte {
|
|
return buffer[t.Position : t.Position+Position(t.Length)]
|
|
}
|
|
|
|
// End returns the position after the token.
|
|
func (t Token) End() Position {
|
|
return t.Position + Position(t.Length)
|
|
}
|
|
|
|
// IsAssignment returns true if the token is an assignment operator.
|
|
func (t Token) IsAssignment() bool {
|
|
return t.Kind > _assignments && t.Kind < _assignmentsEnd
|
|
}
|
|
|
|
// IsKeyword returns true if the token is a keyword.
|
|
func (t Token) IsKeyword() bool {
|
|
return t.Kind > _keywords && t.Kind < _keywordsEnd
|
|
}
|
|
|
|
// IsOperator returns true if the token is an operator.
|
|
func (t Token) IsOperator() bool {
|
|
return t.Kind > _operators && t.Kind < _operatorsEnd
|
|
}
|
|
|
|
// Reset resets the token to default values.
|
|
func (t *Token) Reset() {
|
|
t.Position = 0
|
|
t.Length = 0
|
|
t.Kind = Invalid
|
|
}
|
|
|
|
// Text returns the token text.
|
|
func (t Token) Text(buffer []byte) string {
|
|
return unsafe.String(unsafe.SliceData(t.Bytes(buffer)), t.Length)
|
|
}
|