tokenize

type EnglishContractions struct {
	// contains filtered or unexported fields
}

type LangContractions interface {
	Expand(*SentenceToken) ([]*SentenceToken, bool)
}

type SentenceToken struct {
	Text          []rune `json:"text"`
	PosStart      int    `json:"pos_start"`
	PosEnd        int    `json:"pos_end"`
	IsQuoteStart  bool   `json:"is_quote_start"`
	IsQuoteEnd    bool   `json:"is_quote_end"`
	IsEllipsis    bool   `json:"is_ellipsis"`
	HasApostrophe bool   `json:"has_apostrophe"`
}

type TBWordTokenizer struct {
	LangContractions  LangContractions
	ExpandContrations bool
	Normalize         bool
	// contains filtered or unexported fields
}

type TokenExtractor func([]rune, int) (*SentenceToken, bool)