context

package
v0.0.0-...-b0721b0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 3, 2022 License: GPL-3.0 Imports: 24 Imported by: 0

Documentation

Index

Constants

View Source 
const BoundaryId = -2147483648 //int32 minimum; should constrain database byte-sizing

used to denote the end of a sentence

View Source 
const LanguageEnglish = "english"

Variables

View Source 
var Punctuation = []punctuationSpec{
	punctuationSpec{
		// contains filtered or unexported fields
	},
	punctuationSpec{
		// contains filtered or unexported fields
	},
	punctuationSpec{
		// contains filtered or unexported fields
	},
	punctuationSpec{
		// contains filtered or unexported fields
	},
	punctuationSpec{
		// contains filtered or unexported fields
	},
	punctuationSpec{
		// contains filtered or unexported fields
	},
	punctuationSpec{
		// contains filtered or unexported fields
	},
	punctuationSpec{
		// contains filtered or unexported fields
	},
	punctuationSpec{
		// contains filtered or unexported fields
	},
	punctuationSpec{
		// contains filtered or unexported fields
	},
	punctuationSpec{
		// contains filtered or unexported fields
	},
	punctuationSpec{
		// contains filtered or unexported fields
	},

} //there's an upper limit of `reservedIdsPunctuation` elements on this structure

CAUTION: do not alter any previously defined punctuation or their IDs; only ever add to this list

View Source 
var PunctuationIdsByToken map[string]int = make(map[string]int, len(Punctuation))

CAUTION: do not write code that alters these structures at runtime

View Source 
var PunctuationTokensById map[int]string = make(map[int]string, len(Punctuation))
View Source 
var PunctuationTokensNonSentenceInitial map[string]void = make(map[string]void, len(Punctuation)-1)
View Source 
var SymbolRunes = make(map[rune]void, len(symbolsToRepresentation))

emoticon and kaomoji bits

View Source 
var Symbols = []symbolSpec{

	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},

	symbolSpec{
		// contains filtered or unexported fields
	},

	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},
	symbolSpec{
		// contains filtered or unexported fields
	},

} //there's an upper limit of `reservedIdsSymbols` elements on this structure

CAUTION: do not alter any previously defined symbols or their IDs; only ever add to this list

View Source 
var SymbolsIdsByToken map[string]int = make(map[string]int, len(Symbols))

CAUTION: do not write code that alters these structures at runtime

View Source 
var SymbolsTokensById map[int]string = make(map[int]string, len(Symbols))

Functions

func MakeStringNormaliser 

func MakeStringNormaliser() *transform.Transformer

Types

type Context 

type Context struct {

	//users of this struct are expected to respect this lock
	//learning is a writing flow; everything else is reading
	Lock sync.RWMutex
	// contains filtered or unexported fields
}

func (*Context) AreDigramsEnabled 

func (c *Context) AreDigramsEnabled() bool

func (*Context) AreIdsAllowed 

func (c *Context) AreIdsAllowed(ids []int) bool

func (*Context) AreQuadgramsEnabled 

func (c *Context) AreQuadgramsEnabled() bool

func (*Context) AreQuintgramsEnabled 

func (c *Context) AreQuintgramsEnabled() bool

func (*Context) AreTrigramsEnabled 

func (c *Context) AreTrigramsEnabled() bool

func (*Context) BanSubstrings 

func (c *Context) BanSubstrings(substrings []string) error

func (*Context) EnumerateKeytokenIds 

func (c *Context) EnumerateKeytokenIds(tokens []ParsedToken) ([]int, error)

func (*Context) GetDictionaryTokensById 

func (c *Context) GetDictionaryTokensById(ids map[int]bool) (map[int]DictionaryToken, error)

func (*Context) GetDigrams 

func (c *Context) GetDigrams(
	specs map[DigramSpec]bool,
	forward bool,
) (map[DigramSpec]Digram, error)

func (*Context) GetIdsBannedStatus 

func (c *Context) GetIdsBannedStatus(ids []int) map[int]bool

func (*Context) GetLanguage 

func (c *Context) GetLanguage() string

func (*Context) GetMaxTokenLength 

func (c *Context) GetMaxTokenLength() int

func (*Context) GetProductionBaseRepresentationThreshold 

func (c *Context) GetProductionBaseRepresentationThreshold() float32

func (*Context) GetProductionCalculateSurpriseForward 

func (c *Context) GetProductionCalculateSurpriseForward() bool

func (*Context) GetProductionCalculateSurpriseReverse 

func (c *Context) GetProductionCalculateSurpriseReverse() bool

func (*Context) GetProductionMaxLength 

func (c *Context) GetProductionMaxLength() int

func (*Context) GetProductionMaxParallelSearches 

func (c *Context) GetProductionMaxParallelSearches() int

func (*Context) GetProductionMinLength 

func (c *Context) GetProductionMinLength() int

func (*Context) GetProductionSearchBranchesChildren 

func (c *Context) GetProductionSearchBranchesChildren() int

func (*Context) GetProductionSearchBranchesFromBoundaryInitial 

func (c *Context) GetProductionSearchBranchesFromBoundaryInitial() int

func (*Context) GetProductionSearchBranchesInitial 

func (c *Context) GetProductionSearchBranchesInitial() int

func (*Context) GetProductionStopProbability 

func (c *Context) GetProductionStopProbability() float32

func (*Context) GetProductionTargetMaxLength 

func (c *Context) GetProductionTargetMaxLength() int

func (*Context) GetProductionTargetMinLength 

func (c *Context) GetProductionTargetMinLength() int

func (*Context) GetProductionTargetStopProbability 

func (c *Context) GetProductionTargetStopProbability() float32

func (*Context) GetProductionTokensInitial 

func (c *Context) GetProductionTokensInitial() int

func (*Context) GetQuadgrams 

func (c *Context) GetQuadgrams(
	specs map[QuadgramSpec]bool,
	forward bool,
) (map[QuadgramSpec]Quadgram, error)

func (*Context) GetQuadgramsFromBoundary 

func (c *Context) GetQuadgramsFromBoundary(
	dictionaryIdSecond int,
	count int,
	forward bool,
) ([]Quadgram, error)

func (*Context) GetQuadgramsOrigin 

func (c *Context) GetQuadgramsOrigin(
	dictionaryIdFirst int,
	count int,
	forward bool,
) ([]Quadgram, error)

func (*Context) GetQuintgrams 

func (c *Context) GetQuintgrams(
	specs map[QuintgramSpec]bool,
	forward bool,
) (map[QuintgramSpec]Quintgram, error)

func (*Context) GetQuintgramsFromBoundary 

func (c *Context) GetQuintgramsFromBoundary(
	dictionaryIdSecond int,
	count int,
	forward bool,
) ([]Quintgram, error)

func (*Context) GetQuintgramsOrigin 

func (c *Context) GetQuintgramsOrigin(
	dictionaryIdFirst int,
	count int,
	forward bool,
) ([]Quintgram, error)

func (*Context) GetTrigrams 

func (c *Context) GetTrigrams(
	specs map[TrigramSpec]bool,
	forward bool,
) (map[TrigramSpec]Trigram, error)

func (*Context) GetTrigramsOrigin 

func (c *Context) GetTrigramsOrigin(
	dictionaryIdFirst int,
	count int,
	forward bool,
) ([]Trigram, error)

func (*Context) IsAllowed 

func (c *Context) IsAllowed(s string) bool

func (*Context) LearnInput 

func (c *Context) LearnInput(tokens []ParsedToken) error

func (*Context) UnbanSubstrings 

func (c *Context) UnbanSubstrings(substrings []string) error

type ContextManager 

type ContextManager struct {
	// contains filtered or unexported fields
}

func PrepareContextManager 

func PrepareContextManager(dataPath string) (*ContextManager, error)

func (*ContextManager) Close 

func (cm *ContextManager) Close()

func (*ContextManager) GetContext 

func (cm *ContextManager) GetContext(contextId string) (*Context, error)

type DictionaryToken 

type DictionaryToken struct {
	// contains filtered or unexported fields
}

func (*DictionaryToken) GetId 

func (dt *DictionaryToken) GetId() int

func (*DictionaryToken) Represent 

func (dt *DictionaryToken) Represent(baseRepresentationThreshold float32) (string, bool)

output is the representation to use and a boolean indicating whether it's the base form or not

type Digram 

type Digram struct {
	Ngram
	// contains filtered or unexported fields
}

func (*Digram) CalculateSurprise 

func (g *Digram) CalculateSurprise(dictionaryId int) float32

func (*Digram) ChooseTransitionIds 

func (g *Digram) ChooseTransitionIds(
	desired map[int]bool,
	count int,
) []int

func (*Digram) GetDictionaryIdFirst 

func (g *Digram) GetDictionaryIdFirst() int

func (*Digram) IsTerminal 

func (g *Digram) IsTerminal() bool

func (*Digram) SelectTransitionIds 

func (g *Digram) SelectTransitionIds(
	count int,
	banCheck func([]int) map[int]bool,
	excludeBoundaries bool,
) []int

type DigramSpec 

type DigramSpec struct {
	DictionaryIdFirst int
}

type Ngram 

type Ngram interface {
	IsTerminal() bool
	SelectTransitionIds(int, func([]int) map[int]bool, bool) []int
	ChooseTransitionIds(map[int]bool, int) []int
	CalculateSurprise(int) float32
	// contains filtered or unexported methods
}

type ParsedToken 

type ParsedToken struct {
	Base    string
	Variant string
}

func ParseSymbol 

func ParseSymbol(token []rune) []ParsedToken

type Quadgram 

type Quadgram struct {
	Ngram
	// contains filtered or unexported fields
}

func (*Quadgram) CalculateSurprise 

func (g *Quadgram) CalculateSurprise(dictionaryId int) float32

func (*Quadgram) ChooseTransitionIds 

func (g *Quadgram) ChooseTransitionIds(
	desired map[int]bool,
	count int,
) []int

func (*Quadgram) GetDictionaryIdFirst 

func (g *Quadgram) GetDictionaryIdFirst() int

func (*Quadgram) GetDictionaryIdSecond 

func (g *Quadgram) GetDictionaryIdSecond() int

func (*Quadgram) GetDictionaryIdThird 

func (g *Quadgram) GetDictionaryIdThird() int

func (*Quadgram) IsTerminal 

func (g *Quadgram) IsTerminal() bool

func (*Quadgram) SelectTransitionIds 

func (g *Quadgram) SelectTransitionIds(
	count int,
	banCheck func([]int) map[int]bool,
	excludeBoundaries bool,
) []int

type QuadgramSpec 

type QuadgramSpec struct {
	DictionaryIdFirst  int
	DictionaryIdSecond int
	DictionaryIdThird  int
}

type Quintgram 

type Quintgram struct {
	Ngram
	// contains filtered or unexported fields
}

func (*Quintgram) CalculateSurprise 

func (g *Quintgram) CalculateSurprise(dictionaryId int) float32

func (*Quintgram) ChooseTransitionIds 

func (g *Quintgram) ChooseTransitionIds(
	desired map[int]bool,
	count int,
) []int

func (*Quintgram) GetDictionaryIdFirst 

func (g *Quintgram) GetDictionaryIdFirst() int

func (*Quintgram) GetDictionaryIdFourth 

func (g *Quintgram) GetDictionaryIdFourth() int

func (*Quintgram) GetDictionaryIdSecond 

func (g *Quintgram) GetDictionaryIdSecond() int

func (*Quintgram) GetDictionaryIdThird 

func (g *Quintgram) GetDictionaryIdThird() int

func (*Quintgram) IsTerminal 

func (g *Quintgram) IsTerminal() bool

func (*Quintgram) SelectTransitionIds 

func (g *Quintgram) SelectTransitionIds(
	count int,
	banCheck func([]int) map[int]bool,
	excludeBoundaries bool,
) []int

type QuintgramSpec 

type QuintgramSpec struct {
	DictionaryIdFirst  int
	DictionaryIdSecond int
	DictionaryIdThird  int
	DictionaryIdFourth int
}

type Trigram 

type Trigram struct {
	Ngram
	// contains filtered or unexported fields
}

func (*Trigram) CalculateSurprise 

func (g *Trigram) CalculateSurprise(dictionaryId int) float32

func (*Trigram) ChooseTransitionIds 

func (g *Trigram) ChooseTransitionIds(
	desired map[int]bool,
	count int,
) []int

func (*Trigram) GetDictionaryIdFirst 

func (g *Trigram) GetDictionaryIdFirst() int

func (*Trigram) GetDictionaryIdSecond 

func (g *Trigram) GetDictionaryIdSecond() int

func (*Trigram) IsTerminal 

func (g *Trigram) IsTerminal() bool

func (*Trigram) SelectTransitionIds 

func (g *Trigram) SelectTransitionIds(
	count int,
	banCheck func([]int) map[int]bool,
	excludeBoundaries bool,
) []int

type TrigramSpec 

type TrigramSpec struct {
	DictionaryIdFirst  int
	DictionaryIdSecond int
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.