token

package

v0.4.0 Latest Latest Go to latest Published: Jan 20, 2022 License: Apache-2.0 Imports: 4 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/mosuka/phalanx

Links

Open Source Insights

Documentation ¶

Index ¶

func NewDictionaryCompoundFilterWithOptions(opts map[string]interface{}) (*token.DictionaryCompoundFilter, error)
func NewEdgeNgramFilterWithOptions(opts map[string]interface{}) (*token.EdgeNgramFilter, error)
func NewElisionFilterWithOptions(opts map[string]interface{}) (*token.ElisionFilter, error)
func NewKeyWordMarkerFilterWithOptions(opts map[string]interface{}) (*token.KeyWordMarkerFilter, error)
func NewLengthFilterWithOptions(opts map[string]interface{}) (*token.LengthFilter, error)
func NewNgramFilterWithOptions(opts map[string]interface{}) (*token.NgramFilter, error)
func NewShingleFilterWithOptions(opts map[string]interface{}) (*token.ShingleFilter, error)
func NewStopTokensFilterWithOptions(opts map[string]interface{}) (*token.StopTokensFilter, error)
func NewTruncateTokenFilterWithOptions(opts map[string]interface{}) (*token.TruncateTokenFilter, error)
func NewUnicodeNormalizeFilterWithOptions(opts map[string]interface{}) (*token.UnicodeNormalizeFilter, error)
type TokenFilter
type TokenFilterSetting

Constants ¶

This section is empty.

Variables ¶

This section is empty.

Functions ¶

func NewDictionaryCompoundFilterWithOptions ¶

func NewDictionaryCompoundFilterWithOptions(opts map[string]interface{}) (*token.DictionaryCompoundFilter, error)

Create new DictionaryCompoundFilter with given options. Options example:

{
  "words": [
    "soft",
    "softest",
    "ball"
  ],
  "min_word_size": 5,
  "min_sub_word_size": 2,
  "max_sub_word_size": 15,
  "only_longest_match": false
}

func NewEdgeNgramFilterWithOptions ¶

func NewEdgeNgramFilterWithOptions(opts map[string]interface{}) (*token.EdgeNgramFilter, error)

Create new EdgeNgramFilter with given options. Options example:

{
  "back": false,
  "min_length": 1,
  "max_length": 2
}

func NewElisionFilterWithOptions ¶

func NewElisionFilterWithOptions(opts map[string]interface{}) (*token.ElisionFilter, error)

Create new ElisionFilter with given options. Options example:

{
  "articles": [
    "ar"
  ]
}

func NewKeyWordMarkerFilterWithOptions ¶

func NewKeyWordMarkerFilterWithOptions(opts map[string]interface{}) (*token.KeyWordMarkerFilter, error)

Create new KeyWordMarkerFilter with given options. Options example:

{
  "keywords": [
    "walk",
    "park"
  ]
}

func NewLengthFilterWithOptions ¶

func NewLengthFilterWithOptions(opts map[string]interface{}) (*token.LengthFilter, error)

Create new LengthFilter with given options. Options example:

{
  "min_length": 3,
  "max_length": 4
}

func NewNgramFilterWithOptions ¶

func NewNgramFilterWithOptions(opts map[string]interface{}) (*token.NgramFilter, error)

Create new NgramFilter with given options. Options example:

{
  "min_length": 1,
  "max_length": 2
}

func NewShingleFilterWithOptions ¶

func NewShingleFilterWithOptions(opts map[string]interface{}) (*token.ShingleFilter, error)

Create new ShingleFilter with given options. Options example:

{
  "min_length": 2,
  "max_length": 2,
  "output_original": true,
  "token_separator": " ",
  "fill": "_"
}

func NewStopTokensFilterWithOptions ¶

func NewStopTokensFilterWithOptions(opts map[string]interface{}) (*token.StopTokensFilter, error)

Create new StopTokensFilter with given options. Options example:

{
  "stop_tokens": [
    "a",
    "an",
    "and",
    "are",
    "as",
    "at",
    "be",
    "but",
    "by",
    "for",
    "if",
    "in",
    "into",
    "is",
    "it",
    "no",
    "not",
    "of",
    "on",
    "or",
    "such",
    "that",
    "the",
    "their",
    "then",
    "there",
    "these",
    "they",
    "this",
    "to",
    "was",
    "will",
    "with"
  ]
}

func NewTruncateTokenFilterWithOptions ¶

func NewTruncateTokenFilterWithOptions(opts map[string]interface{}) (*token.TruncateTokenFilter, error)

Create new TruncateTokenFilter with given options. Options example:

{
  "length": 5
}

func NewUnicodeNormalizeFilterWithOptions ¶

func NewUnicodeNormalizeFilterWithOptions(opts map[string]interface{}) (*token.UnicodeNormalizeFilter, error)

Create new UnicodeNormalizeFilter with given options. Options example:

{
  "form": "NFKC"
}

Types ¶

type TokenFilter ¶

type TokenFilter string

const (
	ApostropheTokenFilter         TokenFilter = "apostrophe"
	CamelCaseTokenFilter          TokenFilter = "camel_case"
	DictionaryCompoundTokenFilter TokenFilter = "dictionary_compound"
	EdgeNgramTokenFilter          TokenFilter = "edge_ngram"
	ElisionTokenFilter            TokenFilter = "elision"
	KeywordMarkerTokenFilter      TokenFilter = "keyword_marker"
	LengthTokenFilter             TokenFilter = "length"
	LowerCaseTokenFilter          TokenFilter = "lower_case"
	NgramTokenFilter              TokenFilter = "ngram"
	PorterStemmerTokenFilter      TokenFilter = "porter_stemmer"
	ReverseTokenFilter            TokenFilter = "reverse"
	ShingleTokenFilter            TokenFilter = "shingle"
	StopTokensTokenFilter         TokenFilter = "stop_tokens"
	TruncateTokenFilter           TokenFilter = "truncate"
	UnicodeNormalizeTokenFilter   TokenFilter = "unicode_normalize"
	UniqueTermTokenFilter         TokenFilter = "unique_term"
)

type TokenFilterSetting ¶

type TokenFilterSetting struct {
	Name    TokenFilter            `json:"name"`
	Options map[string]interface{} `json:"options"`
}

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL