Documentation ¶
Index ¶
- type GeneratedPart
- type GenerationState
- type InferenceEngine
- func (ie *InferenceEngine) CreateInferenceContext() *model.InferenceContext
- func (ie *InferenceEngine) GenerateString(promptTokens []model.TokenId) (<-chan GeneratedPart, <-chan error)
- func (ie *InferenceEngine) GenerateStringFromOutputTokens(outputTokens []model.TokenId) (<-chan GeneratedPart, <-chan error)
- func (ie *InferenceEngine) GenerateStringGeneric(promptTokens []model.TokenId, tokenGeneratorFn TokenGeneratorFn) (<-chan GeneratedPart, <-chan error)
- func (ie *InferenceEngine) GenerateTokensGeneric(promptTokens []model.TokenId, tokenGeneratorFn TokenGeneratorFn) (<-chan generationStepResult[model.TokenId], <-chan error)
- func (ie *InferenceEngine) TokenBatchToDebugString(tokenIdBatch []model.TokenId) string
- func (ie *InferenceEngine) TokenBatchToString(tokenIdBatch []model.TokenId) ([]sentencepiece.SentencePiece, string)
- func (ie *InferenceEngine) TokenToString(tokenId model.TokenId, decodingContext *generationDecodingContext) (token sentencepiece.SentencePiece, resultString string, addedToWaiting bool)
- func (ie *InferenceEngine) Tokenize(text string, addBeginOfSentence bool) ([]model.TokenId, error)
- func (ie *InferenceEngine) TokenizeBatch(texts []string, addBeginOfSentence bool) (result [][]model.TokenId, err error)
- type RuneData
- type TokenGeneratorFn
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type GeneratedPart ¶
type GeneratedPart struct { DecodedString string TokenId model.TokenId Token sentencepiece.SentencePiece AddedToWaiting bool WaitingRunesExtraStr string IsResendOfWaiting bool GenerationState GenerationState }
type GenerationState ¶
type GenerationState byte
const ( GSInProgress GenerationState = 1 GSFinishedByReachingEOS GenerationState = 2 GSFinishedByReachingSeqLen GenerationState = 3 )
type InferenceEngine ¶
type InferenceEngine struct {
// contains filtered or unexported fields
}
func NewInferenceEngine ¶
func NewInferenceEngine(model *model.Model, inferenceArgs common.InferenceArgs, logFn func(format string, v ...any)) *InferenceEngine
func (*InferenceEngine) CreateInferenceContext ¶
func (ie *InferenceEngine) CreateInferenceContext() *model.InferenceContext
func (*InferenceEngine) GenerateString ¶
func (ie *InferenceEngine) GenerateString(promptTokens []model.TokenId) (<-chan GeneratedPart, <-chan error)
func (*InferenceEngine) GenerateStringFromOutputTokens ¶
func (ie *InferenceEngine) GenerateStringFromOutputTokens(outputTokens []model.TokenId) (<-chan GeneratedPart, <-chan error)
func (*InferenceEngine) GenerateStringGeneric ¶
func (ie *InferenceEngine) GenerateStringGeneric(promptTokens []model.TokenId, tokenGeneratorFn TokenGeneratorFn) (<-chan GeneratedPart, <-chan error)
func (*InferenceEngine) GenerateTokensGeneric ¶
func (ie *InferenceEngine) GenerateTokensGeneric(promptTokens []model.TokenId, tokenGeneratorFn TokenGeneratorFn) (<-chan generationStepResult[model.TokenId], <-chan error)
func (*InferenceEngine) TokenBatchToDebugString ¶
func (ie *InferenceEngine) TokenBatchToDebugString(tokenIdBatch []model.TokenId) string
func (*InferenceEngine) TokenBatchToString ¶
func (ie *InferenceEngine) TokenBatchToString(tokenIdBatch []model.TokenId) ([]sentencepiece.SentencePiece, string)
func (*InferenceEngine) TokenToString ¶
func (ie *InferenceEngine) TokenToString(tokenId model.TokenId, decodingContext *generationDecodingContext) (token sentencepiece.SentencePiece, resultString string, addedToWaiting bool)
func (*InferenceEngine) TokenizeBatch ¶
Click to show internal directories.
Click to hide internal directories.