document

type ChangeSet[T any] struct {
	Added, Removed slice.Slice[T]
}

type Decoder[WordID, VariantID any] interface {
	IDToWord(WordID) string
	IDToVariant(VariantID) Variant
}

type Document[WordID, VariantID comparable] struct {
	Start            string
	ByteLen, WordLen int
	// Words holds the root words present in the document. This slice can
	// be reordered without effecting the encoding.
	Words    []Locations[WordID]
	Variants *huffslice.Slice[VariantID]
}

type DocumentDecoder[WordID, VariantID comparable] struct {
	Decoder[WordID, VariantID]
	WordSingleToken WordID
	VarSingleToken  VariantID
}

type DocumentEncoder[WordID, VariantID comparable] struct {
	Encoder[WordID, VariantID]
	Splitter        func(string) (string, []string)
	RootVariant     func(string) (string, Variant)
	WordSingleToken WordID
	VarSingleToken  VariantID
}

type Encoder[WordID, VariantID any] interface {
	WordToID(string) WordID
	VariantToID(Variant) VariantID
}

type Locations[T comparable] struct {
	ID   T
	Idxs []uint32
}

type Variant []byte