analyzer

type Analyzer interface {
	Tokenize(input []byte) []Token
}

type ChineseAnalyzer struct {
	Analyzer
	// contains filtered or unexported fields
}

type Token struct {
	Value string `json:"value"`
	TF    uint   `json:"tf"`
	// contains filtered or unexported fields
}