Documentation ¶
Index ¶
- Constants
- Variables
- type QuoteInjectSettings
- type Stream
- func (s *Stream) Close()
- func (s *Stream) CurrentToken() *Token
- func (s *Stream) GetParsedLength() int
- func (s *Stream) GetSnippet(before, after int) []Token
- func (s *Stream) GetSnippetAsString(before, after, maxStringLength int) string
- func (s *Stream) GoNext() *Stream
- func (s *Stream) GoNextIfNextIs(key TokenKey, otherKeys ...TokenKey) bool
- func (s *Stream) GoPrev() *Stream
- func (s *Stream) GoTo(id int) *Stream
- func (s *Stream) HeadToken() *Token
- func (s *Stream) IsAnyNextSequence(keys ...[]TokenKey) bool
- func (s *Stream) IsNextSequence(keys ...TokenKey) bool
- func (s *Stream) IsValid() bool
- func (s *Stream) NextToken() *Token
- func (s *Stream) PrevToken() *Token
- func (s *Stream) SetHistorySize(size int) *Stream
- func (s *Stream) String() string
- type StringSettings
- func (q *StringSettings) AddInjection(startTokenKey, endTokenKey TokenKey) *StringSettings
- func (q *StringSettings) AddSpecialStrings(special []string) *StringSettings
- func (q *StringSettings) SetEscapeSymbol(symbol byte) *StringSettings
- func (q *StringSettings) SetSpecialSymbols(special map[byte]byte) *StringSettings
- type Token
- func (t *Token) ID() int
- func (t *Token) Indent() []byte
- func (t *Token) Is(key TokenKey, keys ...TokenKey) bool
- func (t *Token) IsFloat() bool
- func (t *Token) IsInteger() bool
- func (t *Token) IsKeyword() bool
- func (t *Token) IsNumber() bool
- func (t *Token) IsString() bool
- func (t *Token) IsValid() bool
- func (t *Token) Key() TokenKey
- func (t *Token) Line() int
- func (t *Token) Offset() int
- func (t *Token) String() string
- func (t *Token) StringKey() TokenKey
- func (t *Token) StringSettings() *StringSettings
- func (t *Token) Value() []byte
- func (t *Token) ValueFloat() float64deprecated
- func (t *Token) ValueFloat64() float64
- func (t *Token) ValueInt() int64deprecated
- func (t *Token) ValueInt64() int64
- func (t *Token) ValueString() string
- func (t *Token) ValueUnescaped() []byte
- func (t *Token) ValueUnescapedString() string
- type TokenKey
- type Tokenizer
- func (t *Tokenizer) AllowKeywordSymbols(majorSymbols []rune, minorSymbols []rune) *Tokenizer
- func (t *Tokenizer) AllowKeywordUnderscore() *Tokenizer
- func (t *Tokenizer) AllowNumberUnderscore() *Tokenizer
- func (t *Tokenizer) AllowNumbersInKeyword() *Tokenizer
- func (t *Tokenizer) DefineStringToken(key TokenKey, startToken, endToken string) *StringSettings
- func (t *Tokenizer) DefineTokens(key TokenKey, tokens []string) *Tokenizer
- func (t *Tokenizer) ParseBytes(str []byte) *Stream
- func (t *Tokenizer) ParseStream(r io.Reader, bufferSize uint) *Stream
- func (t *Tokenizer) ParseString(str string) *Stream
- func (t *Tokenizer) SetWhiteSpaces(ws []byte) *Tokenizer
- func (t *Tokenizer) StopOnUndefinedToken() *Tokenizer
Constants ¶
const BackSlash = '\\'
BackSlash just backslash byte
const DefaultChunkSize = 4096
DefaultChunkSize default chunk size for reader.
Variables ¶
var DefaultSpecialString = []string{
"\\",
"n",
"r",
"t",
}
DefaultSpecialString is default escaped symbols.
var DefaultStringEscapes = map[byte]byte{
'n': '\n',
'r': '\r',
't': '\t',
'\\': '\\',
}
DefaultStringEscapes is default escaped symbols. Those symbols are often used everywhere. Deprecated: use DefaultSpecialString and AddSpecialStrings
var DefaultWhiteSpaces = []byte{' ', '\t', '\n', '\r'}
var Numbers = []rune{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}
var Underscore = []rune{'_'}
Functions ¶
This section is empty.
Types ¶
type QuoteInjectSettings ¶
type QuoteInjectSettings struct { // Token type witch opens quoted string. StartKey TokenKey // Token type witch closes quoted string. EndKey TokenKey }
QuoteInjectSettings describes open injection token and close injection token.
type Stream ¶
type Stream struct {
// contains filtered or unexported fields
}
Stream iterator via parsed tokens. If data reads from an infinite buffer then the iterator will be read data from reader chunk-by-chunk.
func NewInfStream ¶
func NewInfStream(p *parsing) *Stream
NewInfStream creates new stream with active parser.
func (*Stream) CurrentToken ¶
CurrentToken always returns the token. If the pointer is not valid (see IsValid) CurrentToken will be returns TokenUndef token. Do not save result (Token) into variables — current token may be changed at any time.
func (*Stream) GetParsedLength ¶
GetParsedLength returns currently count parsed bytes.
func (*Stream) GetSnippet ¶
GetSnippet returns slice of tokens. Slice generated from current token position and include tokens before and after current token.
func (*Stream) GetSnippetAsString ¶
GetSnippetAsString returns tokens before and after current token as string. `maxStringLength` specify max length of each token string. Zero — unlimited token string length. If string greater than maxLength method removes some runes in the middle of the string.
func (*Stream) GoNext ¶
GoNext moves stream pointer to the next token. If there is no token, it initiates the parsing of the next chunk of data. If there is no data, the pointer will point to the TokenUndef token.
func (*Stream) GoNextIfNextIs ¶
GoNextIfNextIs moves stream pointer to the next token if the next token has specific token keys. If keys matched pointer will be updated and method returned true. Otherwise, returned false.
func (*Stream) GoPrev ¶
GoPrev moves pointer of stream to the next token. The number of possible calls is limited if you specified SetHistorySize. If the beginning of the stream or the end of the history is reached, the pointer will point to the TokenUndef token.
func (*Stream) GoTo ¶
GoTo moves pointer of stream to specific token. The search is done by token ID.
func (*Stream) HeadToken ¶
HeadToken returns pointer to head-token. Head-token may be changed by parser if history size is enabled.
func (*Stream) IsAnyNextSequence ¶ added in v1.3.0
IsAnyNextSequence checks that at least one token from each group is contained in a sequence of tokens
func (*Stream) IsNextSequence ¶ added in v1.3.0
IsNextSequence checks if these are next tokens in exactly the same sequence as specified.
func (*Stream) IsValid ¶
IsValid checks if stream is valid. This means that the pointer has not reached the end of the stream.
func (*Stream) NextToken ¶
NextToken returns next token from the stream. If next token doesn't exist method return TypeUndef token. Do not save result (Token) into variables — next token may be changed at any time.
func (*Stream) PrevToken ¶
PrevToken returns previous token from the stream. If previous token doesn't exist method return TypeUndef token. Do not save result (Token) into variables — previous token may be changed at any time.
func (*Stream) SetHistorySize ¶
SetHistorySize sets the number of tokens that should remain after the current token
type StringSettings ¶
type StringSettings struct { Key TokenKey StartToken []byte EndToken []byte EscapeSymbol byte SpecSymbols [][]byte Injects []QuoteInjectSettings }
StringSettings describes framed(quoted) string tokens like quoted strings.
func (*StringSettings) AddInjection ¶
func (q *StringSettings) AddInjection(startTokenKey, endTokenKey TokenKey) *StringSettings
AddInjection configure injection in to string. Injection - parsable fragment of framed(quoted) string. Often used for parsing of placeholders or template's expressions in the framed string.
func (*StringSettings) AddSpecialStrings ¶ added in v1.4.0
func (q *StringSettings) AddSpecialStrings(special []string) *StringSettings
AddSpecialStrings set mapping of all escapable string for escape symbol, like \n, \t, \r.
func (*StringSettings) SetEscapeSymbol ¶
func (q *StringSettings) SetEscapeSymbol(symbol byte) *StringSettings
SetEscapeSymbol set escape symbol for framed(quoted) string. Escape symbol allows ignoring close token of framed string. Also escape symbol allows using special symbols in the frame strings, like \n, \t.
func (*StringSettings) SetSpecialSymbols ¶
func (q *StringSettings) SetSpecialSymbols(special map[byte]byte) *StringSettings
SetSpecialSymbols set mapping of all escapable symbols for escape symbol, like \n, \t, \r. Deprecated: use AddSpecialStrings
type Token ¶
type Token struct {
// contains filtered or unexported fields
}
Token struct describe one token.
func (*Token) IsNumber ¶
IsNumber checks if this token is integer or float — the key is TokenInteger or TokenFloat.
func (*Token) IsString ¶
IsString checks if current token is a quoted string. Token key may be TokenString or TokenStringFragment.
func (*Token) Key ¶
Key returns the key of the token pointed to by the pointer. If pointer is not valid (see IsValid) TokenUndef will be returned.
func (*Token) StringKey ¶
StringKey returns key of string. If key not defined for string TokenString will be returned.
func (*Token) StringSettings ¶
func (t *Token) StringSettings() *StringSettings
StringSettings returns StringSettings structure if token is framed string.
func (*Token) Value ¶
Value returns value of current token as slice of bytes from source. If current token is invalid value returns nil.
Do not change bytes in the slice. Copy slice before change.
func (*Token) ValueFloat
deprecated
func (*Token) ValueFloat64 ¶ added in v1.4.0
ValueFloat64 returns value as float64. If the token is not TokenInteger or TokenFloat then method returns zero. Method doesn't use cache — each call starts a number parser.
func (*Token) ValueInt64 ¶ added in v1.4.0
ValueInt64 returns value as int64. If the token is float the result wild be round by math's rules. If the token is not TokenInteger or TokenFloat then method returns zero Method doesn't use cache — each call starts a number parser.
func (*Token) ValueString ¶
ValueString returns value of the token as string. If the token is TokenUndef method returns empty string.
func (*Token) ValueUnescaped ¶
ValueUnescaped returns clear (unquoted) string
- without edge-tokens (quotes)
- with character escaping handling
For example quoted string
"one \"two\"\t three"
transforms to
one "two" three
Method doesn't use cache. Each call starts a string parser.
func (*Token) ValueUnescapedString ¶
ValueUnescapedString like as ValueUnescaped but returns string.
type TokenKey ¶
type TokenKey int
TokenKey token type identifier
const ( // TokenUnknown means that this token not embedded token and not user defined. TokenUnknown TokenKey = -6 // TokenStringFragment means that this is only fragment of quoted string with injections // For example, "one {{ two }} three", where "one " and " three" — TokenStringFragment TokenStringFragment TokenKey = -5 // TokenString means than this token is quoted string. // For example, "one two" TokenString TokenKey = -4 // TokenFloat means that this token is float number with point and/or exponent. // For example, 1.2, 1e6, 1E-6 TokenFloat TokenKey = -3 // TokenInteger means that this token is integer number. // For example, 3, 49983 TokenInteger TokenKey = -2 // TokenKeyword means that this token is word. // For example, one, two, три TokenKeyword TokenKey = -1 // TokenUndef means that token doesn't exist. // Then stream out of range of token list any getter or checker will return TokenUndef token. TokenUndef TokenKey = 0 )
type Tokenizer ¶
type Tokenizer struct {
// contains filtered or unexported fields
}
Tokenizer stores all tokens configuration and behaviors.
func (*Tokenizer) AllowKeywordSymbols ¶ added in v1.4.0
AllowKeywordSymbols sets major and minor symbols for keywords. Major symbols (any quantity) might be in begin, in middle and in the end of keyword. Minor symbols (any quantity) might be in middle and in the end of the keyword.
parser.AllowKeywordSymbols(tokenizer.Underscore, tokenizer.Numbers) // allows: "_one23", "__one2__two3" parser.AllowKeywordSymbols([]rune{'_', '@'}, tokenizer.Numbers) // allows: "one@23", "@_one_two23", "_one23", "_one2_two3", "@@one___two@_9"
Beware, the tokenizer does not control consecutive duplicates of these runes.
func (*Tokenizer) AllowKeywordUnderscore ¶
AllowKeywordUnderscore allows underscore symbol in keywords, like `one_two` or `_three` Deprecated: use AllowKeywordSymbols
func (*Tokenizer) AllowNumberUnderscore ¶ added in v1.4.0
AllowNumberUnderscore allows underscore symbol in numbers, like `1_000`
func (*Tokenizer) AllowNumbersInKeyword ¶
AllowNumbersInKeyword allows numbers in keywords, like `one1` or `r2d2` The method allows numbers in keywords, but the keyword itself must not start with a number. There should be no spaces between letters and numbers. Deprecated: use AllowKeywordSymbols
func (*Tokenizer) DefineStringToken ¶
func (t *Tokenizer) DefineStringToken(key TokenKey, startToken, endToken string) *StringSettings
DefineStringToken defines a token string. For example, a piece of data surrounded by quotes: "string in quotes" or 'string on sigle quotes'. Arguments startToken and endToken defines open and close "quotes".
`t.DefineStringToken("`", "`")` - parse string "one `two three`" will be parsed as [{key: TokenKeyword, value: "one"}, {key: TokenString, value: "`two three`"}]
`t.DefineStringToken("//", "\n")` - parse string "parse // like comment\n" will be parsed as [{key: TokenKeyword, value: "parse"}, {key: TokenString, value: "// like comment"}]
func (*Tokenizer) DefineTokens ¶
DefineTokens add custom token. There `key` unique is identifier of `tokens`, `tokens` — slice of string of tokens. If key already exists tokens will be rewritten.
func (*Tokenizer) ParseBytes ¶
ParseBytes parse and convert slice of bytes into stream of tokens.
func (*Tokenizer) ParseStream ¶
ParseStream parse and convert infinite stream of bytes into infinite stream of tokens.
func (*Tokenizer) ParseString ¶
ParseString parse string into stream of tokens.
func (*Tokenizer) SetWhiteSpaces ¶
SetWhiteSpaces sets custom whitespace symbols between tokens. By default: `{' ', '\t', '\n', '\r'}`
func (*Tokenizer) StopOnUndefinedToken ¶
StopOnUndefinedToken stops parsing if unknown token detected.