Documentation ¶
Index ¶
- Constants
- Variables
- func MakeStringNormaliser() *transform.Transformer
- type Context
- func (c *Context) AreDigramsEnabled() bool
- func (c *Context) AreIdsAllowed(ids []int) bool
- func (c *Context) AreQuadgramsEnabled() bool
- func (c *Context) AreQuintgramsEnabled() bool
- func (c *Context) AreTrigramsEnabled() bool
- func (c *Context) BanSubstrings(substrings []string) error
- func (c *Context) EnumerateKeytokenIds(tokens []ParsedToken) ([]int, error)
- func (c *Context) GetDictionaryTokensById(ids map[int]bool) (map[int]DictionaryToken, error)
- func (c *Context) GetDigrams(specs map[DigramSpec]bool, forward bool) (map[DigramSpec]Digram, error)
- func (c *Context) GetIdsBannedStatus(ids []int) map[int]bool
- func (c *Context) GetLanguage() string
- func (c *Context) GetMaxTokenLength() int
- func (c *Context) GetProductionBaseRepresentationThreshold() float32
- func (c *Context) GetProductionCalculateSurpriseForward() bool
- func (c *Context) GetProductionCalculateSurpriseReverse() bool
- func (c *Context) GetProductionMaxLength() int
- func (c *Context) GetProductionMaxParallelSearches() int
- func (c *Context) GetProductionMinLength() int
- func (c *Context) GetProductionSearchBranchesChildren() int
- func (c *Context) GetProductionSearchBranchesFromBoundaryInitial() int
- func (c *Context) GetProductionSearchBranchesInitial() int
- func (c *Context) GetProductionStopProbability() float32
- func (c *Context) GetProductionTargetMaxLength() int
- func (c *Context) GetProductionTargetMinLength() int
- func (c *Context) GetProductionTargetStopProbability() float32
- func (c *Context) GetProductionTokensInitial() int
- func (c *Context) GetQuadgrams(specs map[QuadgramSpec]bool, forward bool) (map[QuadgramSpec]Quadgram, error)
- func (c *Context) GetQuadgramsFromBoundary(dictionaryIdSecond int, count int, forward bool) ([]Quadgram, error)
- func (c *Context) GetQuadgramsOrigin(dictionaryIdFirst int, count int, forward bool) ([]Quadgram, error)
- func (c *Context) GetQuintgrams(specs map[QuintgramSpec]bool, forward bool) (map[QuintgramSpec]Quintgram, error)
- func (c *Context) GetQuintgramsFromBoundary(dictionaryIdSecond int, count int, forward bool) ([]Quintgram, error)
- func (c *Context) GetQuintgramsOrigin(dictionaryIdFirst int, count int, forward bool) ([]Quintgram, error)
- func (c *Context) GetTrigrams(specs map[TrigramSpec]bool, forward bool) (map[TrigramSpec]Trigram, error)
- func (c *Context) GetTrigramsOrigin(dictionaryIdFirst int, count int, forward bool) ([]Trigram, error)
- func (c *Context) IsAllowed(s string) bool
- func (c *Context) LearnInput(tokens []ParsedToken) error
- func (c *Context) UnbanSubstrings(substrings []string) error
- type ContextManager
- type DictionaryToken
- type Digram
- func (g *Digram) CalculateSurprise(dictionaryId int) float32
- func (g *Digram) ChooseTransitionIds(desired map[int]bool, count int) []int
- func (g *Digram) GetDictionaryIdFirst() int
- func (g *Digram) IsTerminal() bool
- func (g *Digram) SelectTransitionIds(count int, banCheck func([]int) map[int]bool, excludeBoundaries bool) []int
- type DigramSpec
- type Ngram
- type ParsedToken
- type Quadgram
- func (g *Quadgram) CalculateSurprise(dictionaryId int) float32
- func (g *Quadgram) ChooseTransitionIds(desired map[int]bool, count int) []int
- func (g *Quadgram) GetDictionaryIdFirst() int
- func (g *Quadgram) GetDictionaryIdSecond() int
- func (g *Quadgram) GetDictionaryIdThird() int
- func (g *Quadgram) IsTerminal() bool
- func (g *Quadgram) SelectTransitionIds(count int, banCheck func([]int) map[int]bool, excludeBoundaries bool) []int
- type QuadgramSpec
- type Quintgram
- func (g *Quintgram) CalculateSurprise(dictionaryId int) float32
- func (g *Quintgram) ChooseTransitionIds(desired map[int]bool, count int) []int
- func (g *Quintgram) GetDictionaryIdFirst() int
- func (g *Quintgram) GetDictionaryIdFourth() int
- func (g *Quintgram) GetDictionaryIdSecond() int
- func (g *Quintgram) GetDictionaryIdThird() int
- func (g *Quintgram) IsTerminal() bool
- func (g *Quintgram) SelectTransitionIds(count int, banCheck func([]int) map[int]bool, excludeBoundaries bool) []int
- type QuintgramSpec
- type Trigram
- func (g *Trigram) CalculateSurprise(dictionaryId int) float32
- func (g *Trigram) ChooseTransitionIds(desired map[int]bool, count int) []int
- func (g *Trigram) GetDictionaryIdFirst() int
- func (g *Trigram) GetDictionaryIdSecond() int
- func (g *Trigram) IsTerminal() bool
- func (g *Trigram) SelectTransitionIds(count int, banCheck func([]int) map[int]bool, excludeBoundaries bool) []int
- type TrigramSpec
Constants ¶
View Source
const BoundaryId = -2147483648 //int32 minimum; should constrain database byte-sizing
used to denote the end of a sentence
View Source
const LanguageEnglish = "english"
Variables ¶
View Source
var Punctuation = []punctuationSpec{ punctuationSpec{ // contains filtered or unexported fields }, punctuationSpec{ // contains filtered or unexported fields }, punctuationSpec{ // contains filtered or unexported fields }, punctuationSpec{ // contains filtered or unexported fields }, punctuationSpec{ // contains filtered or unexported fields }, punctuationSpec{ // contains filtered or unexported fields }, punctuationSpec{ // contains filtered or unexported fields }, punctuationSpec{ // contains filtered or unexported fields }, punctuationSpec{ // contains filtered or unexported fields }, punctuationSpec{ // contains filtered or unexported fields }, punctuationSpec{ // contains filtered or unexported fields }, punctuationSpec{ // contains filtered or unexported fields }, } //there's an upper limit of `reservedIdsPunctuation` elements on this structure
CAUTION: do not alter any previously defined punctuation or their IDs; only ever add to this list
View Source
var PunctuationIdsByToken map[string]int = make(map[string]int, len(Punctuation))
CAUTION: do not write code that alters these structures at runtime
View Source
var PunctuationTokensById map[int]string = make(map[int]string, len(Punctuation))
View Source
var PunctuationTokensNonSentenceInitial map[string]void = make(map[string]void, len(Punctuation)-1)
View Source
var SymbolRunes = make(map[rune]void, len(symbolsToRepresentation))
emoticon and kaomoji bits
View Source
var Symbols = []symbolSpec{ symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, symbolSpec{ // contains filtered or unexported fields }, } //there's an upper limit of `reservedIdsSymbols` elements on this structure
CAUTION: do not alter any previously defined symbols or their IDs; only ever add to this list
CAUTION: do not write code that alters these structures at runtime
Functions ¶
func MakeStringNormaliser ¶
func MakeStringNormaliser() *transform.Transformer
Types ¶
type Context ¶
type Context struct { //users of this struct are expected to respect this lock //learning is a writing flow; everything else is reading Lock sync.RWMutex // contains filtered or unexported fields }
func (*Context) AreDigramsEnabled ¶
func (*Context) AreIdsAllowed ¶
func (*Context) AreQuadgramsEnabled ¶
func (*Context) AreQuintgramsEnabled ¶
func (*Context) AreTrigramsEnabled ¶
func (*Context) BanSubstrings ¶
func (*Context) EnumerateKeytokenIds ¶
func (c *Context) EnumerateKeytokenIds(tokens []ParsedToken) ([]int, error)
func (*Context) GetDictionaryTokensById ¶
func (*Context) GetDigrams ¶
func (c *Context) GetDigrams( specs map[DigramSpec]bool, forward bool, ) (map[DigramSpec]Digram, error)
func (*Context) GetLanguage ¶
func (*Context) GetMaxTokenLength ¶
func (*Context) GetProductionBaseRepresentationThreshold ¶
func (*Context) GetProductionCalculateSurpriseForward ¶
func (*Context) GetProductionCalculateSurpriseReverse ¶
func (*Context) GetProductionMaxLength ¶
func (*Context) GetProductionMaxParallelSearches ¶
func (*Context) GetProductionMinLength ¶
func (*Context) GetProductionSearchBranchesChildren ¶
func (*Context) GetProductionSearchBranchesFromBoundaryInitial ¶
func (*Context) GetProductionSearchBranchesInitial ¶
func (*Context) GetProductionStopProbability ¶
func (*Context) GetProductionTargetMaxLength ¶
func (*Context) GetProductionTargetMinLength ¶
func (*Context) GetProductionTargetStopProbability ¶
func (*Context) GetProductionTokensInitial ¶
func (*Context) GetQuadgrams ¶
func (c *Context) GetQuadgrams( specs map[QuadgramSpec]bool, forward bool, ) (map[QuadgramSpec]Quadgram, error)
func (*Context) GetQuadgramsFromBoundary ¶
func (*Context) GetQuadgramsOrigin ¶
func (*Context) GetQuintgrams ¶
func (c *Context) GetQuintgrams( specs map[QuintgramSpec]bool, forward bool, ) (map[QuintgramSpec]Quintgram, error)
func (*Context) GetQuintgramsFromBoundary ¶
func (*Context) GetQuintgramsOrigin ¶
func (*Context) GetTrigrams ¶
func (c *Context) GetTrigrams( specs map[TrigramSpec]bool, forward bool, ) (map[TrigramSpec]Trigram, error)
func (*Context) GetTrigramsOrigin ¶
func (*Context) LearnInput ¶
func (c *Context) LearnInput(tokens []ParsedToken) error
func (*Context) UnbanSubstrings ¶
type ContextManager ¶
type ContextManager struct {
// contains filtered or unexported fields
}
func PrepareContextManager ¶
func PrepareContextManager(dataPath string) (*ContextManager, error)
func (*ContextManager) Close ¶
func (cm *ContextManager) Close()
func (*ContextManager) GetContext ¶
func (cm *ContextManager) GetContext(contextId string) (*Context, error)
type DictionaryToken ¶
type DictionaryToken struct {
// contains filtered or unexported fields
}
func (*DictionaryToken) GetId ¶
func (dt *DictionaryToken) GetId() int
type Digram ¶
type Digram struct { Ngram // contains filtered or unexported fields }
func (*Digram) CalculateSurprise ¶
func (*Digram) ChooseTransitionIds ¶
func (*Digram) GetDictionaryIdFirst ¶
func (*Digram) IsTerminal ¶
type DigramSpec ¶
type DigramSpec struct {
DictionaryIdFirst int
}
type ParsedToken ¶
func ParseSymbol ¶
func ParseSymbol(token []rune) []ParsedToken
type Quadgram ¶
type Quadgram struct { Ngram // contains filtered or unexported fields }
func (*Quadgram) CalculateSurprise ¶
func (*Quadgram) ChooseTransitionIds ¶
func (*Quadgram) GetDictionaryIdFirst ¶
func (*Quadgram) GetDictionaryIdSecond ¶
func (*Quadgram) GetDictionaryIdThird ¶
func (*Quadgram) IsTerminal ¶
type QuadgramSpec ¶
type Quintgram ¶
type Quintgram struct { Ngram // contains filtered or unexported fields }
func (*Quintgram) CalculateSurprise ¶
func (*Quintgram) ChooseTransitionIds ¶
func (*Quintgram) GetDictionaryIdFirst ¶
func (*Quintgram) GetDictionaryIdFourth ¶
func (*Quintgram) GetDictionaryIdSecond ¶
func (*Quintgram) GetDictionaryIdThird ¶
func (*Quintgram) IsTerminal ¶
type QuintgramSpec ¶
type Trigram ¶
type Trigram struct { Ngram // contains filtered or unexported fields }
func (*Trigram) CalculateSurprise ¶
func (*Trigram) ChooseTransitionIds ¶
func (*Trigram) GetDictionaryIdFirst ¶
func (*Trigram) GetDictionaryIdSecond ¶
func (*Trigram) IsTerminal ¶
type TrigramSpec ¶
Click to show internal directories.
Click to hide internal directories.