model

type ChatMessageTemplateData ¶ added in v1.22.0

type ChatMessageTemplateData struct {
	SystemPrompt string
	Role         string
	RoleName     string
	Content      string
	MessageIndex int
}

TODO: Ask mudler about FunctionCall stuff being useful at the message level?

type GRPCProcessFilter ¶ added in v1.25.0

type GRPCProcessFilter = func(id string, p *process.Process) bool

type ModelLoader ¶

type ModelLoader struct {
	ModelPath string
	// contains filtered or unexported fields
}

TODO: Split ModelLoader and TemplateLoader? Just to keep things more organized. Left together to share a mutex until I look into that. Would split if we seperate directories for .bin/.yaml and .tmpl

func NewModelLoader(modelPath string) *ModelLoader

func (ml *ModelLoader) BackendLoader(opts ...Option) (model *grpc.Client, err error)

func (ml *ModelLoader) CheckIsLoaded(s string) *grpc.Client

func (ml *ModelLoader) EvaluateTemplateForChatMessage(templateName string, messageData ChatMessageTemplateData) (string, error)

func (ml *ModelLoader) EvaluateTemplateForPrompt(templateType TemplateType, templateName string, in PromptTemplateData) (string, error)

func (ml *ModelLoader) ExistsInModelPath(s string) bool

func (ml *ModelLoader) GetGRPCPID(id string) (int, error)

func (ml *ModelLoader) GreedyLoader(opts ...Option) (*grpc.Client, error)

func (ml *ModelLoader) ListModels() ([]string, error)

func (ml *ModelLoader) LoadModel(modelName string, loader func(string, string) (*grpc.Client, error)) (*grpc.Client, error)

func (ml *ModelLoader) ShutdownModel(modelName string) error

func (ml *ModelLoader) StopAllExcept(s string)

func (ml *ModelLoader) StopAllGRPC()

func (ml *ModelLoader) StopGRPC(filter GRPCProcessFilter)

type Option ¶ added in v1.21.0

type Option func(*Options)

func WithAssetDir(assetDir string) Option

func WithBackendString(backend string) Option

func WithContext(ctx context.Context) Option

func WithExternalBackend(name string, uri string) Option

func WithGRPCAttempts(attempts int) Option

func WithGRPCAttemptsDelay(delay int) Option

func WithLoadGRPCLoadModelOpts(opts *pb.ModelOptions) Option

func WithModel(modelFile string) Option

func WithSingleActiveBackend() Option

func WithThreads(threads uint32) Option

type Options ¶ added in v1.21.0

type Options struct {
	// contains filtered or unexported fields
}

func NewOptions(opts ...Option) *Options

type PromptTemplateData ¶ added in v1.22.0

type PromptTemplateData struct {
	SystemPrompt         string
	SuppressSystemPrompt bool // used by chat specifically to indicate that SystemPrompt above should be _ignored_
	Input                string
	Instruction          string
	Functions            []grammar.Function
	MessageIndex         int
}

Rather than pass an interface{} to the prompt template: These are the definitions of all possible variables LocalAI will currently populate for use in a prompt template file Please note: Not all of these are populated on every endpoint - your template should either be tested for each endpoint you map it to, or tolerant of zero values.

type TemplateType ¶ added in v1.22.0

type TemplateType int

Keep this in sync with config.TemplateConfig. Is there a more idiomatic way to accomplish this in go? Technically, order doesn't _really_ matter, but the count must stay in sync, see tests/integration/reflect_test.go

const (
	ChatPromptTemplate TemplateType = iota
	ChatMessageTemplate
	CompletionPromptTemplate
	EditPromptTemplate
	FunctionsPromptTemplate

	// The following TemplateType is **NOT** a valid value and MUST be last. It exists to make the sanity integration tests simpler!
	IntegrationTestTemplate
)