tokenize

type EnglishContractions struct {
	// contains filtered or unexported fields
}

type LangContractions interface {
	Expand(*Token) ([]*Token, bool)
}

type SplitTokenizer struct {
	// contains filtered or unexported fields
}

type TBWordTokenizer struct {
	LangContractions  LangContractions
	ExpandContrations bool
	Normalize         bool
	// contains filtered or unexported fields
}

type Token struct {
	Runes         []rune `json:"runes"`
	Word          string `json:"word"`
	Pos           int    `json:"pos"`
	PosTag        string `json:"pos_tag"`
	IsQuoteStart  bool   `json:"is_quote_start"`
	IsQuoteEnd    bool   `json:"is_quote_end"`
	IsEllipsis    bool   `json:"is_ellipsis"`
	HasApostrophe bool   `json:"has_apostrophe"`
}

type TokenExtractor func([]rune, int) (*Token, bool)

type Tokenizer interface {
	Tokenize(string) []*Token
}