tok

package
v83.0.0-...-38e0eed Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 12, 2017 License: AGPL-3.0, Apache-2.0 Imports: 16 Imported by: 0

Documentation

Index

Constants

View Source
const (
	FTSTokenizerName = "fulltext"
)

Variables

This section is empty.

Functions

func EncodeGeoTokens

func EncodeGeoTokens(tokens []string)

func EncodeRegexTokens

func EncodeRegexTokens(tokens []string)

func FtsTokenizerName

func FtsTokenizerName(lang string) string

func GetTextTokens

func GetTextTokens(funcArgs []string, lang string) ([]string, error)

func GetTokens

func GetTokens(funcArgs []string) ([]string, error)

func RegisterTokenizer

func RegisterTokenizer(t Tokenizer)

RegisterTokenizer adds your tokenizer to our list.

Types

type BoolTokenizer

type BoolTokenizer struct{}

func (BoolTokenizer) Identifier

func (t BoolTokenizer) Identifier() byte

func (BoolTokenizer) IsLossy

func (t BoolTokenizer) IsLossy() bool

func (BoolTokenizer) IsSortable

func (t BoolTokenizer) IsSortable() bool

func (BoolTokenizer) Name

func (t BoolTokenizer) Name() string

func (BoolTokenizer) Tokens

func (t BoolTokenizer) Tokens(v types.Val) ([]string, error)

func (BoolTokenizer) Type

func (t BoolTokenizer) Type() types.TypeID

type DayTokenizer

type DayTokenizer struct{}

func (DayTokenizer) Identifier

func (t DayTokenizer) Identifier() byte

func (DayTokenizer) IsLossy

func (t DayTokenizer) IsLossy() bool

func (DayTokenizer) IsSortable

func (t DayTokenizer) IsSortable() bool

func (DayTokenizer) Name

func (t DayTokenizer) Name() string

func (DayTokenizer) Tokens

func (t DayTokenizer) Tokens(sv types.Val) ([]string, error)

func (DayTokenizer) Type

func (t DayTokenizer) Type() types.TypeID

type ExactTokenizer

type ExactTokenizer struct{}

func (ExactTokenizer) Identifier

func (t ExactTokenizer) Identifier() byte

func (ExactTokenizer) IsLossy

func (t ExactTokenizer) IsLossy() bool

func (ExactTokenizer) IsSortable

func (t ExactTokenizer) IsSortable() bool

func (ExactTokenizer) Name

func (t ExactTokenizer) Name() string

func (ExactTokenizer) Tokens

func (t ExactTokenizer) Tokens(sv types.Val) ([]string, error)

func (ExactTokenizer) Type

func (t ExactTokenizer) Type() types.TypeID

type FloatTokenizer

type FloatTokenizer struct{}

func (FloatTokenizer) Identifier

func (t FloatTokenizer) Identifier() byte

func (FloatTokenizer) IsLossy

func (t FloatTokenizer) IsLossy() bool

func (FloatTokenizer) IsSortable

func (t FloatTokenizer) IsSortable() bool

func (FloatTokenizer) Name

func (t FloatTokenizer) Name() string

func (FloatTokenizer) Tokens

func (t FloatTokenizer) Tokens(sv types.Val) ([]string, error)

func (FloatTokenizer) Type

func (t FloatTokenizer) Type() types.TypeID

type FullTextTokenizer

type FullTextTokenizer struct {
	Lang string
}

Full text tokenizer, with language support

func (FullTextTokenizer) Identifier

func (t FullTextTokenizer) Identifier() byte

func (FullTextTokenizer) IsLossy

func (t FullTextTokenizer) IsLossy() bool

func (FullTextTokenizer) IsSortable

func (t FullTextTokenizer) IsSortable() bool

func (FullTextTokenizer) Name

func (t FullTextTokenizer) Name() string

func (FullTextTokenizer) Tokens

func (t FullTextTokenizer) Tokens(sv types.Val) ([]string, error)

func (FullTextTokenizer) Type

func (t FullTextTokenizer) Type() types.TypeID

type GeoTokenizer

type GeoTokenizer struct{}

func (GeoTokenizer) Identifier

func (t GeoTokenizer) Identifier() byte

func (GeoTokenizer) IsLossy

func (t GeoTokenizer) IsLossy() bool

func (GeoTokenizer) IsSortable

func (t GeoTokenizer) IsSortable() bool

func (GeoTokenizer) Name

func (t GeoTokenizer) Name() string

func (GeoTokenizer) Tokens

func (t GeoTokenizer) Tokens(sv types.Val) ([]string, error)

func (GeoTokenizer) Type

func (t GeoTokenizer) Type() types.TypeID

type HashTokenizer

type HashTokenizer struct{}

func (HashTokenizer) Identifier

func (t HashTokenizer) Identifier() byte

func (HashTokenizer) IsLossy

func (t HashTokenizer) IsLossy() bool

func (HashTokenizer) IsSortable

func (t HashTokenizer) IsSortable() bool

func (HashTokenizer) Name

func (t HashTokenizer) Name() string

func (HashTokenizer) Tokens

func (t HashTokenizer) Tokens(sv types.Val) ([]string, error)

func (HashTokenizer) Type

func (t HashTokenizer) Type() types.TypeID

type HourTokenizer

type HourTokenizer struct{}

func (HourTokenizer) Identifier

func (t HourTokenizer) Identifier() byte

func (HourTokenizer) IsLossy

func (t HourTokenizer) IsLossy() bool

func (HourTokenizer) IsSortable

func (t HourTokenizer) IsSortable() bool

func (HourTokenizer) Name

func (t HourTokenizer) Name() string

func (HourTokenizer) Tokens

func (t HourTokenizer) Tokens(sv types.Val) ([]string, error)

func (HourTokenizer) Type

func (t HourTokenizer) Type() types.TypeID

type IntTokenizer

type IntTokenizer struct{}

func (IntTokenizer) Identifier

func (t IntTokenizer) Identifier() byte

func (IntTokenizer) IsLossy

func (t IntTokenizer) IsLossy() bool

func (IntTokenizer) IsSortable

func (t IntTokenizer) IsSortable() bool

func (IntTokenizer) Name

func (t IntTokenizer) Name() string

func (IntTokenizer) Tokens

func (t IntTokenizer) Tokens(sv types.Val) ([]string, error)

func (IntTokenizer) Type

func (t IntTokenizer) Type() types.TypeID

type MonthTokenizer

type MonthTokenizer struct{}

func (MonthTokenizer) Identifier

func (t MonthTokenizer) Identifier() byte

func (MonthTokenizer) IsLossy

func (t MonthTokenizer) IsLossy() bool

func (MonthTokenizer) IsSortable

func (t MonthTokenizer) IsSortable() bool

func (MonthTokenizer) Name

func (t MonthTokenizer) Name() string

func (MonthTokenizer) Tokens

func (t MonthTokenizer) Tokens(sv types.Val) ([]string, error)

func (MonthTokenizer) Type

func (t MonthTokenizer) Type() types.TypeID

type TermTokenizer

type TermTokenizer struct{}

func (TermTokenizer) Identifier

func (t TermTokenizer) Identifier() byte

func (TermTokenizer) IsLossy

func (t TermTokenizer) IsLossy() bool

func (TermTokenizer) IsSortable

func (t TermTokenizer) IsSortable() bool

func (TermTokenizer) Name

func (t TermTokenizer) Name() string

func (TermTokenizer) Tokens

func (t TermTokenizer) Tokens(sv types.Val) ([]string, error)

func (TermTokenizer) Type

func (t TermTokenizer) Type() types.TypeID

type Tokenizer

type Tokenizer interface {

	// Name is name of tokenizer. This should be unique.
	Name() string

	// Type returns typeID that we care about.
	Type() types.TypeID

	// Tokens return tokens for a given value.
	Tokens(sv types.Val) ([]string, error)

	// Identifier returns the prefix byte for this token type.
	Identifier() byte

	// IsSortable returns true if the tokenizer can be used for sorting/ordering.
	IsSortable() bool

	// IsLossy() returns true if we don't store the values directly as index keys
	// during tokenization. If a predicate is tokenized using an IsLossy() tokenizer,
	// then we need to fetch the actual value and compare.
	IsLossy() bool
}

Tokenizer defines what a tokenizer must provide.

func GetTokenizer

func GetTokenizer(name string) (Tokenizer, bool)

GetTokenizer returns tokenizer given unique name.

type TrigramTokenizer

type TrigramTokenizer struct{}

func (TrigramTokenizer) Identifier

func (t TrigramTokenizer) Identifier() byte

func (TrigramTokenizer) IsLossy

func (t TrigramTokenizer) IsLossy() bool

func (TrigramTokenizer) IsSortable

func (t TrigramTokenizer) IsSortable() bool

func (TrigramTokenizer) Name

func (t TrigramTokenizer) Name() string

func (TrigramTokenizer) Tokens

func (t TrigramTokenizer) Tokens(sv types.Val) ([]string, error)

func (TrigramTokenizer) Type

func (t TrigramTokenizer) Type() types.TypeID

type YearTokenizer

type YearTokenizer struct{}

func (YearTokenizer) Identifier

func (t YearTokenizer) Identifier() byte

func (YearTokenizer) IsLossy

func (t YearTokenizer) IsLossy() bool

func (YearTokenizer) IsSortable

func (t YearTokenizer) IsSortable() bool

func (YearTokenizer) Name

func (t YearTokenizer) Name() string

func (YearTokenizer) Tokens

func (t YearTokenizer) Tokens(sv types.Val) ([]string, error)

func (YearTokenizer) Type

func (t YearTokenizer) Type() types.TypeID

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL