lexer

type BackTracker struct {
	Src scanner.Scanner // The source scanner
	// contains filtered or unexported fields
}

type BaseState struct {
	Cls Classifier // The classifier for the lex
}

type ChanLexer struct {
	Chan chan *Token // The input channel
}

type Classifier interface {
	// Classify takes a lexer, a state, and a backtracking scanner
	// and determines one or more recognizers to extract a token
	// or a set of tokens from the lexer input.
	Classify(lexer *Lexer) []Recognizer

	// Error is called by the lexer if all recognizers returned by
	// Classify return without success.
	Error(lexer *Lexer)
}

type IBackTracker interface {
	scanner.Scanner

	// More is used to determine if there are any more characters
	// available for Next to return, given the current state of
	// the BackTracker.
	More() bool

	// SetMax allows updating the maximum number of characters to
	// allow backtracking over.  Setting a TrackAll value will
	// allow all newly returned characters to be backtracked over.
	// If the new value for max is less than the previous value,
	// characters at the front of the backtracking queue will be
	// discarded to bring the size down to max.
	SetMax(max int)

	// Accept accepts characters from the backtracking queue,
	// leaving only the specified number of characters on the
	// queue.
	Accept(leave int)

	// Len returns the number of characters saved so far on the
	// backtracking queue.
	Len() int

	// Pos returns the position of the most recently returned
	// character within the saved character list.
	Pos() int

	// BackTrack resets to the beginning of the backtracking
	// queue.
	BackTrack()
}

type ILexer interface {
	// Next returns the next token.  At the end of the lexer, a
	// nil should be returned.
	Next() *Token
}

type Lexer struct {
	Scanner IBackTracker // The character source, wrapped in a BackTracker
	State   State        // The state of the lexer
	// contains filtered or unexported fields
}

type ListLexer struct {
	// contains filtered or unexported fields
}

type Recognizer interface {
	// Recognize is called by the lexer on the objects returned by
	// the Classifier.  Each will be called in turn until one of
	// the methods returns a boolean true value.  If no recognizer
	// returns true, or if the Classifier returns an empty list,
	// then the Error recognizer will be called, if one is
	// declared, after which the character will be discarded.  The
	// Recognize method will be called with the lexer, the state,
	// and a backtracking scanner.
	Recognize(lexer *Lexer) bool
}

type State interface {
	// Classifier must return the classifier to use.  It is safe
	// for the application to return different Classifier
	// implementations depending on the lexer state.
	Classifier() Classifier
}

type Token struct {
	Type  string           // The type of token
	Loc   scanner.Location // The location of the token
	Value interface{}      // The semantic value of the token; optional
	Text  string           // The original text of the token; optional
}