lexer

package
v1.2.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 1, 2024 License: MIT Imports: 4 Imported by: 0

Documentation

Overview

*

  • The lexer for parsing fet template

Index

Constants

View Source 
const (
	BYTE_UNDERSCORE byte = '_'
	BYTE_SPACE      byte = ' '
	BYTE_EOF        byte = 0
)

Variables

View Source 
var KEYWORDS = [][]byte{
	[]byte("true"),
	[]byte("false"),
	[]byte("null"),
}

Functions

func IsBaseNumberByte 

func IsBaseNumberByte(bt byte, base NumberBase) bool

func IsBinaryByte 

func IsBinaryByte(bt byte) bool

func IsBinaryOrParenOpToken 

func IsBinaryOrParenOpToken(token IToken) bool

func IsBytesInBytesList 

func IsBytesInBytesList(bts []byte, btsList [][]byte) bool

* 

*

func IsDecimalByte 

func IsDecimalByte(bt byte) bool

*

  • number byte functions

func IsEqualOpToken 

func IsEqualOpToken(token IToken) bool

func IsHexByte 

func IsHexByte(bt byte) bool

func IsOctalByte 

func IsOctalByte(bt byte) bool

func IsOpToken 

func IsOpToken(token IToken, op *Operator) bool

func IsPipeOpToken 

func IsPipeOpToken(token IToken) bool

func IsSpaceByte 

func IsSpaceByte(bt byte) bool

*

  • space

func IsSpaceToken 

func IsSpaceToken(token IToken) bool

Types

type ArrayField 

type ArrayField struct {
	IsEnd         bool
	Array         IToken
	Field         *IdentifierToken
	ComputedField *Expression
}

* * Array Field * e.g $a.b $a["b"]

func (*ArrayField) Add 

func (arr *ArrayField) Add(bt byte, exp *Expression) (IToken, error)

func (*ArrayField) End 

func (arr *ArrayField) End() error

func (*ArrayField) RawBytes 

func (arr *ArrayField) RawBytes() []byte

func (*ArrayField) Type 

func (arr *ArrayField) Type() TokenType

type ArrayInState 

type ArrayInState uint8
const (
	MaybeArrayKey ArrayInState = iota
	InArrayValue
)

type ArrayItem 

type ArrayItem struct {
	Key   IToken
	Value IToken
}

*

  • Array Literal
  • e.g => [ "a" => 1, "c"]

type ArrayLiteral 

type ArrayLiteral struct {
	IsEnd  bool
	HasKey bool
	State  ArrayInState
	Index  int
	CurExp *Expression
	Items  []ArrayItem
	Raw    []byte
}

func (*ArrayLiteral) Add 

func (arr *ArrayLiteral) Add(bt byte, exp *Expression) (IToken, error)

func (*ArrayLiteral) End 

func (arr *ArrayLiteral) End() error

func (*ArrayLiteral) RawBytes 

func (arr *ArrayLiteral) RawBytes() []byte

func (*ArrayLiteral) Type 

func (arr *ArrayLiteral) Type() TokenType

type AstNode 

type AstNode struct {
	Op    *Operator
	Left  IToken
	Right IToken
}

* * AST Node

func (*AstNode) Add 

func (ast *AstNode) Add(bt byte, exp *Expression) (IToken, error)

func (*AstNode) End 

func (ast *AstNode) End() error

func (*AstNode) RawBytes 

func (ast *AstNode) RawBytes() []byte

func (*AstNode) Type 

func (ast *AstNode) Type() TokenType

type DoubleQuoteStringToken 

type DoubleQuoteStringToken struct {
	InTranslate bool
	IsEnd       bool
	CurRawLen   int
	CurVar      *Expression
	CurRaw      []byte
	RawString   [][]byte
	VarString   []StringVariable
}

func DoubleQuoteString 

func DoubleQuoteString() *DoubleQuoteStringToken

Double Quote String

func (*DoubleQuoteStringToken) Add 

func (ds *DoubleQuoteStringToken) Add(bt byte, exp *Expression) (IToken, error)

func (*DoubleQuoteStringToken) End 

func (ds *DoubleQuoteStringToken) End() error

func (*DoubleQuoteStringToken) RawBytes 

func (ds *DoubleQuoteStringToken) RawBytes() []byte

func (*DoubleQuoteStringToken) Type 

func (ds *DoubleQuoteStringToken) Type() TokenType

type ExpOpts 

type ExpOpts struct {
	DisAllowedOp []byte
	TypeKeywords [][]byte
}

*

  • Expression

type Expression 

type Expression struct {
	LazyPipe  bool
	PrevToken IToken
	CurToken  IToken
	Options   *ExpOpts
	OpStack   []*OperatorToken
	Output    []IToken
}

func New 

func New(args ...*ExpOpts) *Expression

func (*Expression) Add 

func (exp *Expression) Add(bt byte, _ *Expression) (IToken, error)

func (*Expression) End 

func (exp *Expression) End() error

func (*Expression) Eof 

func (exp *Expression) Eof() error

* * End of the expression

func (*Expression) IsEmpty 

func (exp *Expression) IsEmpty() bool

func (*Expression) IsKeyword 

func (exp *Expression) IsKeyword(bts []byte) bool

func (*Expression) Parse 

func (exp *Expression) Parse(str string) (IToken, error)

* * Parse string to AST

func (*Expression) RawBytes 

func (exp *Expression) RawBytes() []byte

func (*Expression) Token 

func (exp *Expression) Token() IToken

func (*Expression) Type 

func (exp *Expression) Type() TokenType

type FunctionCall 

type FunctionCall struct {
	IsEnd  bool
	Name   IToken
	Args   []IToken
	CurArg *Expression
	Raw    []byte
}

*

  • Function Call
  • e.g => abc(), abc(1+2, "def")

func (*FunctionCall) Add 

func (fn *FunctionCall) Add(bt byte, exp *Expression) (IToken, error)

func (*FunctionCall) End 

func (fn *FunctionCall) End() error

func (*FunctionCall) RawBytes 

func (fn *FunctionCall) RawBytes() []byte

func (*FunctionCall) Type 

func (fn *FunctionCall) Type() TokenType

type IToken 

type IToken interface {
	Add(bt byte, exp *Expression) (IToken, error)
	Type() TokenType
	End() error
	RawBytes() []byte
}

func AddSpaceOrOperatorByte 

func AddSpaceOrOperatorByte(bt byte, args ...[]byte) (IToken, error)

func AddUnkownTokenByte 

func AddUnkownTokenByte(bt byte, exp *Expression) (IToken, error)

type IdentifierToken 

type IdentifierToken struct {
	IsVar bool
	Raw   []byte
}

*

  • Identifier token, e.g $a abc a123

func (*IdentifierToken) Add 

func (id *IdentifierToken) Add(bt byte, exp *Expression) (IToken, error)

func (*IdentifierToken) End 

func (id *IdentifierToken) End() error

func (*IdentifierToken) RawBytes 

func (id *IdentifierToken) RawBytes() []byte

func (*IdentifierToken) Type 

func (id *IdentifierToken) Type() TokenType

type Integer 

type Integer struct {
	PrevByte *byte
	Raw      []byte
}

*

  • Number token

func (*Integer) CheckPrevByte 

func (it *Integer) CheckPrevByte() (IToken, error)

type NumberBase 

type NumberBase uint8
const (
	HexBase     NumberBase = 16
	OctalBase   NumberBase = 8
	BinaryBase  NumberBase = 2
	DecimalBase NumberBase = 10
)

type NumberToken 

type NumberToken struct {
	BeginWithZero bool
	Base          NumberBase
	Integer       Integer
	Decimal       *Integer
	Exponent      *Integer
}

func Number 

func Number() *NumberToken

*

  • method => Number()
  • creates an instance of a number token
  • keep the base as 10 instead of the default 0.

func (*NumberToken) Add 

func (num *NumberToken) Add(bt byte, exp *Expression) (IToken, error)

func (*NumberToken) End 

func (num *NumberToken) End() error

*

  • End

func (*NumberToken) RawBytes 

func (num *NumberToken) RawBytes() []byte

func (*NumberToken) Type 

func (num *NumberToken) Type() TokenType

type ObjectProperty 

type ObjectProperty struct {
	Object   IToken
	Property *IdentifierToken
	Raw      []byte
}

* Object Property

func (*ObjectProperty) Add 

func (obj *ObjectProperty) Add(bt byte, exp *Expression) (IToken, error)

func (*ObjectProperty) End 

func (obj *ObjectProperty) End() error

func (*ObjectProperty) RawBytes 

func (obj *ObjectProperty) RawBytes() []byte

func (*ObjectProperty) Type 

func (obj *ObjectProperty) Type() TokenType

type Operator 

type Operator struct {
	Unary       bool
	RightToLeft bool
	Priority    uint8
	Raw         []byte
	NextMaybe   *Operator
}

*

  • Operator

func (*Operator) FixIfUnary 

func (op *Operator) FixIfUnary(prevToken IToken) (*Operator, error)

func (*Operator) IsPipe 

func (op *Operator) IsPipe() bool

func (*Operator) IsSureSingleOperator 

func (op *Operator) IsSureSingleOperator() bool

type OperatorToken 

type OperatorToken struct {
	Index int
	Op    *Operator
}

func (*OperatorToken) Add 

func (token *OperatorToken) Add(bt byte, exp *Expression) (IToken, error)

func (*OperatorToken) End 

func (token *OperatorToken) End() error

func (*OperatorToken) RawBytes 

func (token *OperatorToken) RawBytes() []byte

func (*OperatorToken) Type 

func (token *OperatorToken) Type() TokenType

type PipeFunction 

type PipeFunction struct {
	IsEnd  bool
	CurArg *Expression
	Name   *IdentifierToken
	Args   []IToken
	Raw    []byte
}

* * Pipe Function

func (*PipeFunction) Add 

func (pipe *PipeFunction) Add(bt byte, exp *Expression) (IToken, error)

func (*PipeFunction) End 

func (pipe *PipeFunction) End() error

func (*PipeFunction) RawBytes 

func (pipe *PipeFunction) RawBytes() []byte

func (*PipeFunction) Type 

func (pipe *PipeFunction) Type() TokenType

type SingleQuoteStringToken 

type SingleQuoteStringToken struct {
	InTranslate bool
	IsEnd       bool
	Raw         []byte
}

*

  • string token
  • SingleQuote: 'abc'
  • DoubleQuote: "abc`$a`"

Single Quote String

func (*SingleQuoteStringToken) Add 

func (ss *SingleQuoteStringToken) Add(bt byte, exp *Expression) (IToken, error)

func (*SingleQuoteStringToken) End 

func (ss *SingleQuoteStringToken) End() error

func (*SingleQuoteStringToken) RawBytes 

func (ss *SingleQuoteStringToken) RawBytes() []byte

func (*SingleQuoteStringToken) Type 

func (ss *SingleQuoteStringToken) Type() TokenType

type SpaceToken 

type SpaceToken struct {
	Raw []byte
}

*

  • [Space Token] struct

func (*SpaceToken) Add 

func (sp *SpaceToken) Add(bt byte, exp *Expression) (IToken, error)

func (*SpaceToken) End 

func (sp *SpaceToken) End() error

func (*SpaceToken) RawBytes 

func (sp *SpaceToken) RawBytes() []byte

func (*SpaceToken) Type 

func (sp *SpaceToken) Type() TokenType

type StringVariable 

type StringVariable struct {
	Index int
	Var   *Expression
}

type TokenType 

type TokenType uint8
const (
	OpType TokenType = iota
	IdentType
	SpaceType
	NumType
	StrType
	ArrLitType
	FuncCallType
	PipeFuncType
	ObjPropType
	ArrFieldType
	ExpType
	AstType
)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.