objectio

package
v1.1.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 15, 2024 License: Apache-2.0 Imports: 31 Imported by: 0

Documentation

Overview

Copyright 2021 Matrix Origin

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

Constants

View Source 
const (
	IOET_Empty   = 0
	IOET_ObjMeta = 1
	IOET_ColData = 2
	IOET_BF      = 3
	IOET_ZM      = 4
)
View Source 
const (
	SegmentIdSize = types.UuidSize
	ObjectIDSize  = types.ObjectidSize
)
View Source 
const (
	ExtentOff   = ObjectNameLen
	ExtentLen   = ExtentSize
	RowsOff     = ExtentOff + ExtentLen
	RowsLen     = 4
	BlockIDOff  = RowsOff + RowsLen
	BlockIDLen  = 2
	LocationLen = BlockIDOff + BlockIDLen
)
View Source 
const (
	FileNumOff         = SegmentIdSize
	FileNumLen         = 2
	NameStringOff      = FileNumOff + FileNumLen
	NameStringLen      = 42 //uuid[36]+_[1]+filename[5]
	ObjectNameLen      = NameStringOff + NameStringLen
	ObjectNameShortLen = NameStringOff
)
View Source 
const (
	SEQNUM_UPPER    = math.MaxUint16 - 5 // reserved 5 column for special committs、committs etc.
	SEQNUM_ROWID    = math.MaxUint16
	SEQNUM_ABORT    = math.MaxUint16 - 1
	SEQNUM_COMMITTS = math.MaxUint16 - 2
)
View Source 
const (
	IOET_ObjectMeta_V1  = 1
	IOET_ObjectMeta_V2  = 2
	IOET_ObjectMeta_V3  = 3
	IOET_ColumnData_V1  = 1
	IOET_BloomFilter_V1 = 1
	IOET_BloomFilter_V2 = 2
	IOET_ZoneMap_V1     = 1

	IOET_ObjectMeta_CurrVer  = IOET_ObjectMeta_V3
	IOET_ColumnData_CurrVer  = IOET_ColumnData_V1
	IOET_BloomFilter_CurrVer = IOET_BloomFilter_V2
	IOET_ZoneMap_CurrVer     = IOET_ZoneMap_V1
)
View Source 
const (
	WriterNormal = iota
	WriterCheckpoint
	WriterQueryResult
	WriterGC
	WriterETL
)
View Source 
const (
	ExtentSize = extentOriginOff + extentOriginLen
)
View Source 
const FSName = "local"
View Source 
const FooterSize = 64
View Source 
const HeaderSize = 64
View Source 
const IOEntryHeaderSize = 4
View Source 
const Magic = 0xFFFFFFFF
View Source 
const (
	ObjectStatsLen = objectOriginSizeOffset + objectOriginSizeLen
)
View Source 
const Version = 1
View Source 
const ZoneMapSize = index.ZMSize

Variables

View Source 
var (
	BuildObjectMeta        = buildObjectDataMetaV1
	NewObjectWriterSpecial = newObjectWriterSpecialV1
	NewObjectWriter        = newObjectWriterV1
	NewObjectReaderWithStr = newObjectReaderWithStrV1
	NewObjectReader        = newObjectReaderV1
)
View Source 
var BlkReadStats = newBlockReadStats()
View Source 
var BuildZM = index.BuildZM
View Source 
var EmptyZm = [64]byte{}
View Source 
var NameCheckPoint [NameStringOff]byte
View Source 
var NameDiskCleaner [NameStringOff]byte
View Source 
var NameETL [NameStringOff]byte
View Source 
var NameNormal [NameStringOff]byte
View Source 
var NameQueryResult [NameStringOff]byte
View Source 
var NewZM = index.NewZM
View Source 
var (
	RowidType types.Type
)

Functions

func BuildMetaData added in v0.8.0 

func BuildMetaData(blkCount, colCount uint16) objectDataMetaV1

test used

func ConstructRowidColumn added in v0.8.0 

func ConstructRowidColumn(
	id *Blockid, start, length uint32, mp *mpool.MPool,
) (vec *vector.Vector, err error)

func ConstructRowidColumnTo added in v0.8.0 

func ConstructRowidColumnTo(
	vec *vector.Vector,
	id *Blockid, start, length uint32, mp *mpool.MPool,
) (err error)

func ConstructRowidColumnToWithSels added in v0.8.0 

func ConstructRowidColumnToWithSels(
	vec *vector.Vector,
	id *Blockid,
	sels []int32,
	mp *mpool.MPool,
) (err error)

func ConvertToCkpIdx added in v1.0.0 

func ConvertToCkpIdx(dataType uint16) uint16

func Decode added in v0.8.0 

func Decode(buf []byte) (any, error)

func DecodeColumnDataV1 added in v0.8.0 

func DecodeColumnDataV1(buf []byte) (ioe any, err error)

func DecodeObjectMetaV1 added in v1.0.0 

func DecodeObjectMetaV1(buf []byte) (ioe any, err error)

func DecodeObjectMetaV2 added in v1.0.0 

func DecodeObjectMetaV2(buf []byte) (ioe any, err error)

func DecodeObjectMetaV3 added in v1.0.0 

func DecodeObjectMetaV3(buf []byte) (ioe any, err error)

func EncodeColumnDataV1 added in v0.8.0 

func EncodeColumnDataV1(ioe any) (buf []byte, err error)

func EncodeIOEntryHeader added in v0.8.0 

func EncodeIOEntryHeader(h *IOEntryHeader) []byte

func ExportCacheStats added in v1.0.0 

func ExportCacheStats() string

func ForeachObjectStats added in v1.1.0 

func ForeachObjectStats(onStats func(stats *ObjectStats) bool, statsList ...ObjectStats)

ForeachObjectStats executes onStats on each object stats until onStats returns false or all object stats have been visited

func GetIOEntryCodec added in v0.8.0 

func GetIOEntryCodec(h IOEntryHeader) (codec ioEntryCodec)

func HackRowidToU64 added in v1.0.0 

func HackRowidToU64(id Rowid) uint64

used only in some special cases

func InitMetaCache added in v0.8.0 

func InitMetaCache(size int64)

func IsBlockInObject added in v0.8.0 

func IsBlockInObject(blkID *types.Blockid, objID *ObjectName) bool

func IsEmptyBlkid added in v0.8.0 

func IsEmptyBlkid(id *Blockid) bool

func IsEmptySegid added in v0.8.0 

func IsEmptySegid(id *Segmentid) bool

func IsSameObjectLocVsMeta added in v0.8.0 

func IsSameObjectLocVsMeta(location Location, meta ObjectDataMeta) bool

func IsSameObjectLocVsShort added in v0.8.0 

func IsSameObjectLocVsShort(location Location, short *ObjectNameShort) bool

func NewArrayVector added in v1.1.0 

func NewArrayVector[T types.RealNumbers](n int, typ types.Type, m *mpool.MPool, random bool, vs [][]T) *vector.Vector

func NewBatch added in v1.1.0 

func NewBatch(ts []types.Type, random bool, n int, m *mpool.MPool) *batch.Batch

func NewBlockidVector added in v1.1.0 

func NewBlockidVector(n int, typ types.Type, m *mpool.MPool, _ bool, vs []types.Blockid) *vector.Vector

func NewBoolVector added in v1.1.0 

func NewBoolVector(n int, typ types.Type, m *mpool.MPool, _ bool, vs []bool) *vector.Vector

func NewDateVector added in v1.1.0 

func NewDateVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []string) *vector.Vector

func NewDatetimeVector added in v1.1.0 

func NewDatetimeVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []string) *vector.Vector

func NewDecimal128Vector added in v1.1.0 

func NewDecimal128Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []types.Decimal128) *vector.Vector

func NewDecimal64Vector added in v1.1.0 

func NewDecimal64Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []types.Decimal64) *vector.Vector

func NewFloat32Vector added in v1.1.0 

func NewFloat32Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []float32) *vector.Vector

func NewFloat64Vector added in v1.1.0 

func NewFloat64Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []float64) *vector.Vector

func NewInt16Vector added in v1.1.0 

func NewInt16Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []int16) *vector.Vector

func NewInt32Vector added in v1.1.0 

func NewInt32Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []int32) *vector.Vector

func NewInt64Vector added in v1.1.0 

func NewInt64Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []int64) *vector.Vector

func NewInt8Vector added in v1.1.0 

func NewInt8Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []int8) *vector.Vector

func NewJsonVector added in v1.1.0 

func NewJsonVector(n int, typ types.Type, m *mpool.MPool, _ bool, vs []string) *vector.Vector

func NewRowid added in v0.8.0 

func NewRowid(blkid *Blockid, offset uint32) *types.Rowid

func NewRowidVector added in v1.1.0 

func NewRowidVector(n int, typ types.Type, m *mpool.MPool, _ bool, vs []types.Rowid) *vector.Vector

func NewStringVector added in v1.1.0 

func NewStringVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []string) *vector.Vector

func NewTimeVector added in v1.1.0 

func NewTimeVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []string) *vector.Vector

func NewTimestampVector added in v1.1.0 

func NewTimestampVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []string) *vector.Vector

func NewTsVector added in v1.1.0 

func NewTsVector(n int, typ types.Type, m *mpool.MPool, _ bool, vs []types.TS) *vector.Vector

func NewUInt16Vector added in v1.1.0 

func NewUInt16Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []uint16) *vector.Vector

func NewUInt32Vector added in v1.1.0 

func NewUInt32Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []uint32) *vector.Vector

func NewUInt64Vector added in v1.1.0 

func NewUInt64Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []uint64) *vector.Vector

func NewUInt8Vector added in v1.1.0 

func NewUInt8Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []uint8) *vector.Vector

func NewVector added in v1.1.0 

func NewVector(n int, typ types.Type, m *mpool.MPool, random bool, Values interface{}) *vector.Vector

func ReadAllBlocksWithMeta added in v0.8.0 

func ReadAllBlocksWithMeta(
	ctx context.Context,
	meta *ObjectDataMeta,
	name string,
	cols []uint16,
	policy fileservice.Policy,
	m *mpool.MPool,
	fs fileservice.FileService,
	factory CacheConstructorFactory,
) (ioVec *fileservice.IOVector, err error)

func ReadBytes added in v0.8.0 

func ReadBytes(r io.Reader) (buf []byte, n int64, err error)

func ReadExtent added in v0.8.0 

func ReadExtent(
	ctx context.Context,
	name string,
	extent *Extent,
	policy fileservice.Policy,
	fs fileservice.FileService,
	factory CacheConstructorFactory,
) (v []byte, err error)

func ReadMultiBlocksWithMeta added in v0.8.0 

func ReadMultiBlocksWithMeta(
	ctx context.Context,
	name string,
	meta ObjectMeta,
	options map[uint16]*ReadBlockOptions,
	fs fileservice.FileService,
	factory CacheConstructorFactory,
) (ioVec *fileservice.IOVector, err error)

func ReadOneBlock added in v0.8.0 

func ReadOneBlock(
	ctx context.Context,
	meta *ObjectDataMeta,
	name string,
	blk uint16,
	seqnums []uint16,
	typs []types.Type,
	m *mpool.MPool,
	fs fileservice.FileService,
) (ioVec *fileservice.IOVector, err error)

func ReadOneBlockAllColumns added in v1.1.0 

func ReadOneBlockAllColumns(
	ctx context.Context,
	meta *ObjectDataMeta,
	name string,
	id uint32,
	cols []uint16,
	cachePolicy fileservice.Policy,
	fs fileservice.FileService,
) (bat *batch.Batch, err error)

func ReadOneBlockWithMeta added in v0.8.0 

func ReadOneBlockWithMeta(
	ctx context.Context,
	meta *ObjectDataMeta,
	name string,
	blk uint16,
	seqnums []uint16,
	typs []types.Type,
	m *mpool.MPool,
	fs fileservice.FileService,
	factory CacheConstructorFactory,
	policies ...fileservice.Policy,
) (ioVec *fileservice.IOVector, err error)

func ReadString added in v0.8.0 

func ReadString(r io.Reader) (str string, n int64, err error)

func RegisterIOEnrtyCodec added in v0.8.0 

func RegisterIOEnrtyCodec(h IOEntryHeader, encFn IOEncodeFunc, decFn IODecodeFunc)

func SetObjectStatsBlkCnt added in v1.1.0 

func SetObjectStatsBlkCnt(stats *ObjectStats, cnt uint32) error

func SetObjectStatsExtent added in v1.1.0 

func SetObjectStatsExtent(stats *ObjectStats, extent Extent) error

func SetObjectStatsLocation added in v1.1.0 

func SetObjectStatsLocation(stats *ObjectStats, location Location) error

func SetObjectStatsObjectName added in v1.1.0 

func SetObjectStatsObjectName(stats *ObjectStats, name ObjectName) error

func SetObjectStatsOriginSize added in v1.1.0 

func SetObjectStatsOriginSize(stats *ObjectStats, size uint32) error

func SetObjectStatsRowCnt added in v1.1.0 

func SetObjectStatsRowCnt(stats *ObjectStats, cnt uint32) error

func SetObjectStatsShortName added in v1.1.0 

func SetObjectStatsShortName(stats *ObjectStats, name *ObjectNameShort) error

func SetObjectStatsSize added in v1.1.0 

func SetObjectStatsSize(stats *ObjectStats, size uint32) error

func SetObjectStatsSortKeyZoneMap added in v1.1.0 

func SetObjectStatsSortKeyZoneMap(stats *ObjectStats, zoneMap ZoneMap) error

func TmpNewFileservice 

func TmpNewFileservice(ctx context.Context, dir string) fileservice.FileService

func WriteBytes added in v0.8.0 

func WriteBytes(b []byte, w io.Writer) (n int64, err error)

func WriteString added in v0.8.0 

func WriteString(str string, w io.Writer) (n int64, err error)

Types

type BlockHeader 

type BlockHeader []byte

func BuildBlockHeader added in v0.8.0 

func BuildBlockHeader() BlockHeader

func (BlockHeader) Appendable added in v1.1.0 

func (bh BlockHeader) Appendable() bool

func (BlockHeader) BFExtent added in v0.8.0 

func (bh BlockHeader) BFExtent() Extent

func (BlockHeader) BlockID added in v0.8.0 

func (bh BlockHeader) BlockID() *Blockid

func (BlockHeader) ColumnCount added in v0.8.0 

func (bh BlockHeader) ColumnCount() uint16

func (BlockHeader) IsEmpty added in v0.8.0 

func (bh BlockHeader) IsEmpty() bool

func (BlockHeader) MaxSeqnum added in v0.8.0 

func (bh BlockHeader) MaxSeqnum() uint16

func (BlockHeader) MetaColumnCount added in v0.8.0 

func (bh BlockHeader) MetaColumnCount() uint16

func (BlockHeader) MetaLocation added in v0.8.0 

func (bh BlockHeader) MetaLocation() Extent

func (BlockHeader) Rows added in v0.8.0 

func (bh BlockHeader) Rows() uint32

func (BlockHeader) Sequence added in v0.8.0 

func (bh BlockHeader) Sequence() uint16

func (BlockHeader) SetAppendable added in v1.1.0 

func (bh BlockHeader) SetAppendable(appendable bool)

func (BlockHeader) SetBFExtent added in v0.8.0 

func (bh BlockHeader) SetBFExtent(location Extent)

func (BlockHeader) SetBlockID added in v0.8.0 

func (bh BlockHeader) SetBlockID(id *Blockid)

func (BlockHeader) SetColumnCount added in v0.8.0 

func (bh BlockHeader) SetColumnCount(count uint16)

func (BlockHeader) SetMaxSeqnum added in v0.8.0 

func (bh BlockHeader) SetMaxSeqnum(seqnum uint16)

func (BlockHeader) SetMetaColumnCount added in v0.8.0 

func (bh BlockHeader) SetMetaColumnCount(count uint16)

func (BlockHeader) SetMetaLocation added in v0.8.0 

func (bh BlockHeader) SetMetaLocation(location Extent)

func (BlockHeader) SetRows added in v0.8.0 

func (bh BlockHeader) SetRows(rows uint32)

func (BlockHeader) SetSequence added in v0.8.0 

func (bh BlockHeader) SetSequence(seq uint16)

func (BlockHeader) SetSortKey added in v1.1.0 

func (bh BlockHeader) SetSortKey(idx uint16)

func (BlockHeader) SetStartID added in v1.0.0 

func (bh BlockHeader) SetStartID(id uint16)

func (BlockHeader) SetTableID added in v0.8.0 

func (bh BlockHeader) SetTableID(id uint64)

func (BlockHeader) SetZoneMapArea added in v0.8.0 

func (bh BlockHeader) SetZoneMapArea(location Extent)

func (BlockHeader) ShortName added in v0.8.0 

func (bh BlockHeader) ShortName() *ObjectNameShort

func (BlockHeader) SortKey added in v1.1.0 

func (bh BlockHeader) SortKey() uint16

func (BlockHeader) StartID added in v1.0.0 

func (bh BlockHeader) StartID() uint16

func (BlockHeader) TableID added in v0.8.0 

func (bh BlockHeader) TableID() uint64

func (BlockHeader) ZoneMapArea added in v0.8.0 

func (bh BlockHeader) ZoneMapArea() Extent

type BlockIndex added in v0.8.0 

type BlockIndex []byte

func BuildBlockIndex added in v0.8.0 

func BuildBlockIndex(count uint32) BlockIndex

func (BlockIndex) BlockCount added in v0.8.0 

func (oh BlockIndex) BlockCount() uint32

func (BlockIndex) BlockMetaPos added in v0.8.0 

func (oh BlockIndex) BlockMetaPos(BlockID uint32) (uint32, uint32)

func (BlockIndex) Length added in v0.8.0 

func (oh BlockIndex) Length() uint32

func (BlockIndex) SetBlockCount added in v0.8.0 

func (oh BlockIndex) SetBlockCount(cnt uint32)

func (BlockIndex) SetBlockMetaPos added in v0.8.0 

func (oh BlockIndex) SetBlockMetaPos(BlockID uint32, offset, length uint32)

type BlockObject 

type BlockObject []byte

func BuildBlockMeta added in v0.8.0 

func BuildBlockMeta(count uint16) BlockObject

func NewBlock 

func NewBlock(seqnums *Seqnums) BlockObject

func (BlockObject) AddColumnMeta added in v0.8.0 

func (bm BlockObject) AddColumnMeta(idx uint16, col ColumnMeta)

func (BlockObject) BlockHeader added in v0.8.0 

func (bm BlockObject) BlockHeader() BlockHeader

func (BlockObject) ColumnMeta added in v0.8.0 

func (bm BlockObject) ColumnMeta(seqnum uint16) ColumnMeta

ColumnMeta is for internal use only, it didn't consider the block does not contain the seqnum

func (BlockObject) GetBlockID added in v1.0.0 

func (bm BlockObject) GetBlockID(name ObjectName) *Blockid

func (BlockObject) GetColumnCount added in v0.8.0 

func (bm BlockObject) GetColumnCount() uint16

func (BlockObject) GetExtent 

func (bm BlockObject) GetExtent() Extent

func (BlockObject) GetID 

func (bm BlockObject) GetID() uint16

func (BlockObject) GetMaxSeqnum added in v0.8.0 

func (bm BlockObject) GetMaxSeqnum() uint16

func (BlockObject) GetMeta 

func (bm BlockObject) GetMeta() BlockObject

func (BlockObject) GetMetaColumnCount added in v0.8.0 

func (bm BlockObject) GetMetaColumnCount() uint16

func (BlockObject) GetRows 

func (bm BlockObject) GetRows() uint32

func (BlockObject) IsEmpty added in v0.8.0 

func (bm BlockObject) IsEmpty() bool

func (BlockObject) MustGetColumn added in v0.8.0 

func (bm BlockObject) MustGetColumn(seqnum uint16) ColumnMeta

MustGetColumn is for general use. it return a empty ColumnMeta if the block does not contain the seqnum

func (BlockObject) SetBlockMetaHeader added in v0.8.0 

func (bm BlockObject) SetBlockMetaHeader(header BlockHeader)

func (BlockObject) ToColumnZoneMaps added in v0.8.0 

func (bm BlockObject) ToColumnZoneMaps(seqnums []uint16) []ZoneMap

type BlockReadStats added in v1.0.0 

type BlockReadStats struct {
	// using this we can collect the number of blks have read and hit among them
	BlkCacheHitStats hitStats
	// using this we can collect the number of entries have read and hit among them
	EntryCacheHitStats hitStats
	// using this we can collect the number of blks each reader will read
	BlksByReaderStats hitStats
	CounterSet        *perfcounter.CounterSet
}

BlockReadStats collect blk read related cache statistics, include mem and disk

type Blockid added in v0.8.0 

type Blockid = types.Blockid

func BuildObjectBlockid added in v0.8.0 

func BuildObjectBlockid(name ObjectName, sequence uint16) *Blockid

func NewBlockid added in v0.8.0 

func NewBlockid(segid *Segmentid, fnum, blknum uint16) *Blockid

func NewBlockidWithObjectID added in v1.1.0 

func NewBlockidWithObjectID(segid *ObjectId, blknum uint16) *Blockid

func Str2Blockid added in v0.8.0 

func Str2Blockid(id string) *Blockid

type BloomFilter 

type BloomFilter []byte

func FastLoadBF added in v1.1.0 

func FastLoadBF(
	ctx context.Context,
	location Location,
	isPrefetch bool,
	fs fileservice.FileService,
) (BloomFilter, error)

func LoadBFWithMeta added in v1.0.0 

func LoadBFWithMeta(
	ctx context.Context,
	meta ObjectDataMeta,
	location Location,
	fs fileservice.FileService,
) (BloomFilter, error)

func ReadBloomFilter added in v0.8.0 

func ReadBloomFilter(
	ctx context.Context,
	name string,
	extent *Extent,
	policy fileservice.Policy,
	fs fileservice.FileService,
) (filters BloomFilter, err error)

func (BloomFilter) BlockCount added in v0.8.0 

func (bf BloomFilter) BlockCount() uint32

func (BloomFilter) GetBloomFilter added in v0.8.0 

func (bf BloomFilter) GetBloomFilter(BlockID uint32) []byte

func (BloomFilter) GetObjectBloomFilter added in v0.8.0 

func (bf BloomFilter) GetObjectBloomFilter() []byte

func (BloomFilter) Size added in v0.8.0 

func (bf BloomFilter) Size() int

type CacheConfig added in v0.8.0 

type CacheConfig struct {
	MemoryCapacity toml.ByteSize `toml:"memory-capacity"`
}

type CacheConstructor added in v0.8.0 

type CacheConstructor = func(r io.Reader, buf []byte, allocator fileservice.CacheDataAllocator) (fileservice.CacheData, error)

type CacheConstructorFactory added in v0.8.0 

type CacheConstructorFactory = func(size int64, algo uint8) CacheConstructor

type ColumnMeta 

type ColumnMeta []byte

func BuildColumnMeta added in v0.8.0 

func BuildColumnMeta() ColumnMeta

func BuildObjectColumnMeta added in v0.8.0 

func BuildObjectColumnMeta() ColumnMeta

func GetColumnMeta added in v0.8.0 

func GetColumnMeta(idx uint16, data []byte) ColumnMeta

func GetObjectColumnMeta added in v0.8.0 

func GetObjectColumnMeta(seqnum uint16, data []byte) ColumnMeta

caller makes sure the data has column meta fot the given seqnum

func (ColumnMeta) Checksum added in v0.8.0 

func (cm ColumnMeta) Checksum() uint32

func (ColumnMeta) DataType added in v0.8.0 

func (cm ColumnMeta) DataType() uint8

func (ColumnMeta) Idx added in v0.8.0 

func (cm ColumnMeta) Idx() uint16

func (ColumnMeta) IsEmpty added in v0.8.0 

func (cm ColumnMeta) IsEmpty() bool

func (ColumnMeta) Location added in v0.8.0 

func (cm ColumnMeta) Location() Extent

func (ColumnMeta) Ndv added in v0.8.0 

func (cm ColumnMeta) Ndv() uint32

func (ColumnMeta) NullCnt added in v0.8.0 

func (cm ColumnMeta) NullCnt() uint32

func (ColumnMeta) SetNdv added in v0.8.0 

func (cm ColumnMeta) SetNdv(cnt uint32)

func (ColumnMeta) SetNullCnt added in v0.8.0 

func (cm ColumnMeta) SetNullCnt(cnt uint32)

func (ColumnMeta) SetZoneMap added in v0.8.0 

func (cm ColumnMeta) SetZoneMap(zm ZoneMap)

func (ColumnMeta) ZoneMap added in v0.8.0 

func (cm ColumnMeta) ZoneMap() ZoneMap

type ColumnMetaFetcher added in v0.8.0 

type ColumnMetaFetcher interface {
	MustGetColumn(seqnum uint16) ColumnMeta
}

type CreateBlockOpt added in v0.8.0 

type CreateBlockOpt struct {
	Loc *struct {
		Metaloc  Location
		Deltaloc Location
	}

	Id *struct {
		Filen uint16
		Blkn  uint16
	}
}

func (*CreateBlockOpt) WithBlkIdx added in v0.8.0 

func (o *CreateBlockOpt) WithBlkIdx(s uint16) *CreateBlockOpt

func (*CreateBlockOpt) WithDetaloc added in v0.8.0 

func (o *CreateBlockOpt) WithDetaloc(s Location) *CreateBlockOpt

func (*CreateBlockOpt) WithFileIdx added in v0.8.0 

func (o *CreateBlockOpt) WithFileIdx(s uint16) *CreateBlockOpt

func (*CreateBlockOpt) WithMetaloc added in v0.8.0 

func (o *CreateBlockOpt) WithMetaloc(s Location) *CreateBlockOpt

type CreateObjOpt added in v1.1.0 

type CreateObjOpt struct {
	Id *types.Objectid
}

func (*CreateObjOpt) WithId added in v1.1.0 

func (o *CreateObjOpt) WithId(id *types.Objectid) *CreateObjOpt

type DataMetaType added in v1.0.0 

type DataMetaType uint16
const (
	SchemaData      DataMetaType = 0
	SchemaTombstone DataMetaType = 1

	CkpMetaStart DataMetaType = 2

	// CkpMetaEnd = CkpMetaStart + `MaxIDX`
	CkpMetaEnd DataMetaType = 28 + 2
)

func ConvertToSchemaType added in v1.0.0 

func ConvertToSchemaType(ckpIdx uint16) DataMetaType

type Extent 

type Extent []byte

Alg | Offset | Length | OriginSize ----|--------|--------|------------ 1 | 4 | 4 | 4 Alg: Specifies the compression algorithm Offset: The offset of the compressed data in the file Length: The length of the compressed data OriginSize: The length of the original data

func NewExtent 

func NewExtent(alg uint8, offset, length, originSize uint32) Extent

func (Extent) Alg added in v0.8.0 

func (ex Extent) Alg() uint8

func (Extent) End 

func (ex Extent) End() uint32

func (Extent) Length 

func (ex Extent) Length() uint32

func (Extent) Offset 

func (ex Extent) Offset() uint32

func (Extent) OriginSize 

func (ex Extent) OriginSize() uint32

func (Extent) SetAlg added in v0.8.0 

func (ex Extent) SetAlg(alg uint8)

func (Extent) SetLength added in v0.8.0 

func (ex Extent) SetLength(length uint32)

func (Extent) SetOffset added in v0.8.0 

func (ex Extent) SetOffset(offset uint32)

func (Extent) SetOriginSize added in v0.8.0 

func (ex Extent) SetOriginSize(originSize uint32)

func (Extent) String added in v0.8.0 

func (ex Extent) String() string
type Footer struct {
	// contains filtered or unexported fields
}

func (Footer) Marshal added in v0.8.0 

func (f Footer) Marshal() []byte
type Header []byte

func BuildHeader added in v0.8.0 

func BuildHeader() Header

func (Header) Extent added in v0.8.0 

func (h Header) Extent() Extent

func (Header) SchemaVersion added in v0.8.0 

func (h Header) SchemaVersion(ver uint32)

func (Header) SetExtent added in v0.8.0 

func (h Header) SetExtent(location Extent)

func (Header) SetSchemaVersion added in v0.8.0 

func (h Header) SetSchemaVersion(ver uint32)

type IODecodeFunc added in v0.8.0 

type IODecodeFunc = func([]byte) (any, error)

type IOEncodeFunc added in v0.8.0 

type IOEncodeFunc = func(any) ([]byte, error)

type IOEntry added in v0.8.0 

type IOEntry interface {
	MarshalBinary() ([]byte, error)
	UnmarshalBinary([]byte) error
}

type IOEntryHeader added in v0.8.0 

type IOEntryHeader struct {
	Type, Version uint16
}

func DecodeIOEntryHeader added in v0.8.0 

func DecodeIOEntryHeader(buf []byte) *IOEntryHeader

func (IOEntryHeader) String added in v0.8.0 

func (h IOEntryHeader) String() string

type Location added in v0.8.0 

type Location []byte

Location is a fixed-length unmodifiable byte array. Layout: ObjectName | Extent | Rows(uint32) | ID(uint16)

func BuildLocation added in v0.8.0 

func BuildLocation(name ObjectName, extent Extent, rows uint32, id uint16) Location

func MockLocation added in v0.8.0 

func MockLocation(name ObjectName) Location

func (Location) Extent added in v0.8.0 

func (l Location) Extent() Extent

func (Location) ID added in v0.8.0 

func (l Location) ID() uint16

func (Location) IsEmpty added in v0.8.0 

func (l Location) IsEmpty() bool

func (Location) Name added in v0.8.0 

func (l Location) Name() ObjectName

func (Location) Rows added in v0.8.0 

func (l Location) Rows() uint32

func (Location) SetID added in v1.0.0 

func (l Location) SetID(id uint16)

func (Location) ShortName added in v0.8.0 

func (l Location) ShortName() *ObjectNameShort

func (Location) String added in v0.8.0 

func (l Location) String() string

type Object 

type Object struct {
	// contains filtered or unexported fields
}

func NewObject 

func NewObject(name string, fs fileservice.FileService) *Object

func (*Object) GetFs added in v1.0.0 

func (o *Object) GetFs() fileservice.FileService

type ObjectBuffer 

type ObjectBuffer struct {
	// contains filtered or unexported fields
}

ObjectBuffer is the buffer prepared before writing to the object file, all data written to the object needs to be filled in it, and then written to the object file at one time

func NewObjectBuffer 

func NewObjectBuffer(name string) *ObjectBuffer

func (*ObjectBuffer) GetData 

func (b *ObjectBuffer) GetData() fileservice.IOVector

func (*ObjectBuffer) Length 

func (b *ObjectBuffer) Length() int

func (*ObjectBuffer) SetDataOptions added in v0.7.0 

func (b *ObjectBuffer) SetDataOptions(items ...WriteOptions)

func (*ObjectBuffer) Write 

func (b *ObjectBuffer) Write(buf []byte, items ...WriteOptions) (int, int)

type ObjectDataMeta added in v1.0.0 

type ObjectDataMeta = objectDataMetaV1

type ObjectDescriber added in v1.1.0 

type ObjectDescriber interface {
	DescribeObject() ([]ObjectStats, error)
}

type ObjectFS 

type ObjectFS struct {
	Service fileservice.FileService
	Dir     string
}

func NewObjectFS 

func NewObjectFS(service fileservice.FileService, dir string) *ObjectFS

func (*ObjectFS) DelFiles added in v0.7.0 

func (o *ObjectFS) DelFiles(ctx context.Context, name []string) error

func (*ObjectFS) Delete added in v0.7.0 

func (o *ObjectFS) Delete(fileName string) error

func (*ObjectFS) ListDir 

func (o *ObjectFS) ListDir(dir string) ([]fileservice.DirEntry, error)

type ObjectId added in v1.1.0 

type ObjectId = types.Objectid

func NewObjectid added in v1.1.0 

func NewObjectid() *ObjectId

func NewObjectidWithSegmentIDAndNum added in v1.1.0 

func NewObjectidWithSegmentIDAndNum(sid *Segmentid, num uint16) *ObjectId

type ObjectMeta added in v0.8.0 

type ObjectMeta interface {
	MustGetMeta(metaType DataMetaType) ObjectDataMeta

	HeaderLength() uint32

	DataMetaCount() uint16
	TombstoneMetaCount() uint16

	DataMeta() (ObjectDataMeta, bool)

	MustDataMeta() ObjectDataMeta

	TombstoneMeta() (ObjectDataMeta, bool)

	MustTombstoneMeta() ObjectDataMeta

	SetDataMetaCount(count uint16)

	SetDataMetaOffset(offset uint32)

	SetTombstoneMetaCount(count uint16)

	SetTombstoneMetaOffset(offset uint32)

	SubMeta(pos uint16) (ObjectDataMeta, bool)

	SubMetaCount() uint16

	SubMetaIndex() SubMetaIndex

	SubMetaTypes() []uint16
}

func FastLoadObjectMeta added in v0.8.0 

func FastLoadObjectMeta(
	ctx context.Context,
	location *Location,
	prefetch bool,
	fs fileservice.FileService,
) (ObjectMeta, error)

func LoadObjectMetaByExtent added in v0.8.0 

func LoadObjectMetaByExtent(
	ctx context.Context,
	name *ObjectName,
	extent *Extent,
	prefetch bool,
	policy fileservice.Policy,
	fs fileservice.FileService,
) (meta ObjectMeta, err error)

func ReadObjectMeta added in v0.8.0 

func ReadObjectMeta(
	ctx context.Context,
	name string,
	extent *Extent,
	policy fileservice.Policy,
	fs fileservice.FileService,
) (meta ObjectMeta, err error)

type ObjectName added in v0.8.0 

type ObjectName []byte
 ObjectName is a fixed-length unmodifiable byte array.
 Layout: SegmentId     |   Num    |  NameStr
	     (types.Uuid)    (uint16)   (string)
 Size:	   16B         +   2B     +  42B

func BuildCheckpointName added in v0.8.0 

func BuildCheckpointName() ObjectName

func BuildDiskCleanerName added in v0.8.0 

func BuildDiskCleanerName() ObjectName

func BuildETLName added in v0.8.0 

func BuildETLName() ObjectName

func BuildNormalName added in v0.8.0 

func BuildNormalName() ObjectName

func BuildObjectName added in v0.8.0 

func BuildObjectName(segid *Segmentid, num uint16) ObjectName

func BuildObjectNameWithObjectID added in v1.1.0 

func BuildObjectNameWithObjectID(segid *ObjectId) ObjectName

func BuildQueryResultName added in v0.8.0 

func BuildQueryResultName() ObjectName

func MockObjectName added in v0.8.0 

func MockObjectName() ObjectName

func (ObjectName) Equal added in v0.8.0 

func (o ObjectName) Equal(a ObjectName) bool

func (ObjectName) Num added in v0.8.0 

func (o ObjectName) Num() uint16

func (ObjectName) ObjectId added in v1.1.0 

func (o ObjectName) ObjectId() *ObjectId

func (ObjectName) SegmentId added in v0.8.0 

func (o ObjectName) SegmentId() Segmentid

func (ObjectName) Short added in v0.8.0 

func (o ObjectName) Short() *ObjectNameShort

func (ObjectName) String added in v0.8.0 

func (o ObjectName) String() string

type ObjectNameShort added in v0.8.0 

type ObjectNameShort [ObjectNameShortLen]byte

func ShortName added in v1.0.0 

func ShortName(b *Blockid) *ObjectNameShort

func ToObjectNameShort added in v0.8.0 

func ToObjectNameShort(blkID *Blockid) *ObjectNameShort

func (*ObjectNameShort) Equal added in v0.8.0 

func (s *ObjectNameShort) Equal(o []byte) bool

func (*ObjectNameShort) Num added in v0.8.0 

func (s *ObjectNameShort) Num() uint16

func (*ObjectNameShort) Segmentid added in v0.8.0 

func (s *ObjectNameShort) Segmentid() *Segmentid

type ObjectReader 

type ObjectReader = objectReaderV1

type ObjectStats added in v1.1.0 

type ObjectStats [ObjectStatsLen]byte

ObjectStats has format: +------------------------------------------------------------------------------------------------+ |object_name(60B)|extent(13B)|row_cnt(4B)|block_cnt(4B)|zone_map(64B)|objectSize|objectOriginSize| +------------------------------------------------------------------------------------------------+ 

var ZeroObjectStats ObjectStats

func NewObjectStats added in v1.1.0 

func NewObjectStats() *ObjectStats

func (*ObjectStats) BlkCnt added in v1.1.0 

func (des *ObjectStats) BlkCnt() uint32

func (*ObjectStats) Clone added in v1.1.0 

func (des *ObjectStats) Clone() *ObjectStats

Clone deep copies the stats and returns its pointer

func (*ObjectStats) Extent added in v1.1.0 

func (des *ObjectStats) Extent() Extent

func (*ObjectStats) IsZero added in v1.1.0 

func (des *ObjectStats) IsZero() bool

func (*ObjectStats) Marshal added in v1.1.0 

func (des *ObjectStats) Marshal() []byte

func (*ObjectStats) ObjectLocation added in v1.1.0 

func (des *ObjectStats) ObjectLocation() Location

func (*ObjectStats) ObjectName added in v1.1.0 

func (des *ObjectStats) ObjectName() ObjectName

func (*ObjectStats) ObjectShortName added in v1.1.0 

func (des *ObjectStats) ObjectShortName() *ObjectNameShort

func (*ObjectStats) OriginSize added in v1.1.0 

func (des *ObjectStats) OriginSize() uint32

func (*ObjectStats) Rows added in v1.1.0 

func (des *ObjectStats) Rows() uint32

func (*ObjectStats) Size added in v1.1.0 

func (des *ObjectStats) Size() uint32

func (*ObjectStats) SortKeyZoneMap added in v1.1.0 

func (des *ObjectStats) SortKeyZoneMap() ZoneMap

func (*ObjectStats) String added in v1.1.0 

func (des *ObjectStats) String() string

func (*ObjectStats) UnMarshal added in v1.1.0 

func (des *ObjectStats) UnMarshal(data []byte)

type ObjectWriter 

type ObjectWriter = objectWriterV1

type ReadBlockOptions added in v0.8.0 

type ReadBlockOptions struct {
	Id       uint16
	DataType uint16
	Idxes    map[uint16]bool
}

type Reader 

type Reader interface {
	// Read is to read columns data of a block from fileservice at one time
	// extent is location of the block meta
	// idxs is the column serial number of the data to be read
	Read(ctx context.Context,
		extent *Extent, idxs []uint16,
		id uint32,
		m *mpool.MPool,
		readFunc CacheConstructorFactory) (*fileservice.IOVector, error)

	ReadAll(
		ctx context.Context,
		extent *Extent,
		idxs []uint16,
		m *mpool.MPool,
		readFunc CacheConstructorFactory,
	) (*fileservice.IOVector, error)

	ReadBlocks(ctx context.Context,
		extent *Extent,
		ids map[uint32]*ReadBlockOptions,
		m *mpool.MPool,
		readFunc CacheConstructorFactory) (*fileservice.IOVector, error)

	// ReadMeta is the meta that reads a block
	// extent is location of the block meta
	ReadMeta(ctx context.Context, extent *Extent, m *mpool.MPool) (ObjectDataMeta, error)

	// ReadAllMeta is read the meta of all blocks in an object
	ReadAllMeta(ctx context.Context, m *mpool.MPool) (ObjectDataMeta, error)

	GetObject() *Object
}

Reader is to read data from fileservice

type ReaderOptionFunc added in v0.8.0 

type ReaderOptionFunc func(opt *ReaderOptions)

func WithDataCachePolicyOption added in v1.0.0 

func WithDataCachePolicyOption(noLRUCache fileservice.Policy) ReaderOptionFunc

func WithMetaCachePolicyOption added in v1.0.0 

func WithMetaCachePolicyOption(noLRUCache fileservice.Policy) ReaderOptionFunc

type ReaderOptions added in v0.8.0 

type ReaderOptions struct {
	// contains filtered or unexported fields
}

type Rowid added in v0.8.0 

type Rowid = types.Rowid

func HackBlockid2Rowid added in v1.0.0 

func HackBlockid2Rowid(id *Blockid) Rowid

used only in some special cases

func HackBytes2Rowid added in v1.0.0 

func HackBytes2Rowid(bs []byte) Rowid

used only in some special cases

func HackObjid2Rowid added in v1.1.0 

func HackObjid2Rowid(id *ObjectId) Rowid

used only in some special cases

func HackSegid2Rowid added in v1.0.0 

func HackSegid2Rowid(id *Segmentid) Rowid

used only in some special cases

func HackU64ToRowid added in v1.0.0 

func HackU64ToRowid(v uint64) Rowid

used only in some special cases

type Segmentid added in v0.8.0 

type Segmentid = types.Segmentid

func NewSegmentid added in v0.8.0 

func NewSegmentid() *Segmentid

func ToSegmentId added in v0.8.0 

func ToSegmentId(blkID *Blockid) *Segmentid

type Seqnums added in v0.8.0 

type Seqnums struct {
	Seqs       []uint16
	MaxSeq     uint16 // do not consider special column like rowid and committs
	MetaColCnt uint16 // include special columns
}

func NewSeqnums added in v0.8.0 

func NewSeqnums(seqs []uint16) *Seqnums

func (*Seqnums) InitWithColCnt added in v0.8.0 

func (s *Seqnums) InitWithColCnt(colcnt int)

type StaticFilter added in v0.8.0 

type StaticFilter = index.StaticFilter

type Stats added in v0.8.0 

type Stats struct {
	// contains filtered or unexported fields
}

func NewStats added in v0.8.0 

func NewStats() *Stats

func (*Stats) ExportBlockSelectivity added in v0.8.0 

func (s *Stats) ExportBlockSelectivity() (
	whit, wtotal int64,
)

func (*Stats) ExportColumnSelctivity added in v0.8.0 

func (s *Stats) ExportColumnSelctivity() (
	hit, total int64,
)

func (*Stats) ExportReadDel added in v1.0.0 

func (s *Stats) ExportReadDel() (total, read, bisect time.Duration, cnt int64)

func (*Stats) ExportReadFilterSelectivity added in v0.8.0 

func (s *Stats) ExportReadFilterSelectivity() (
	whit, wtotal int64, hit, total int64,
)

func (*Stats) ExportString added in v0.8.0 

func (s *Stats) ExportString() string

func (*Stats) RecordBlockSelectivity added in v0.8.0 

func (s *Stats) RecordBlockSelectivity(hit, total int)

func (*Stats) RecordColumnSelectivity added in v0.8.0 

func (s *Stats) RecordColumnSelectivity(hit, total int)

func (*Stats) RecordReadDel added in v1.0.0 

func (s *Stats) RecordReadDel(total, read, bisect time.Duration)

func (*Stats) RecordReadFilterSelectivity added in v0.8.0 

func (s *Stats) RecordReadFilterSelectivity(hit, total int)

type SubMetaIndex added in v1.0.0 

type SubMetaIndex []byte

func BuildSubMetaIndex added in v1.0.0 

func BuildSubMetaIndex(count uint16) SubMetaIndex

func (SubMetaIndex) Length added in v1.0.0 

func (oh SubMetaIndex) Length() uint32

func (SubMetaIndex) SetSchemaMeta added in v1.0.0 

func (oh SubMetaIndex) SetSchemaMeta(pos uint16, st uint16, count uint16, offset uint32)

func (SubMetaIndex) SetSubMetaCount added in v1.0.0 

func (oh SubMetaIndex) SetSubMetaCount(cnt uint16)

func (SubMetaIndex) SubMetaCount added in v1.0.0 

func (oh SubMetaIndex) SubMetaCount() uint16

type WriteOptions added in v0.7.0 

type WriteOptions struct {
	Type WriteType
	Val  any
}

type WriteType added in v0.7.0 

type WriteType int8
const (
	WriteTS WriteType = iota
)

type Writer 

type Writer interface {
	// Write writes one batch to the Buffer at a time,
	// one batch corresponds to a virtual block,
	// and returns the handle of the block.
	Write(batch *batch.Batch) (BlockObject, error)

	// Write metadata for every column of all blocks
	WriteObjectMeta(ctx context.Context, totalRow uint32, metas []ColumnMeta)

	// WriteEnd is to write multiple batches written to
	// the buffer to the fileservice at one time
	WriteEnd(ctx context.Context, items ...WriteOptions) ([]BlockObject, error)
}

Writer is to virtualize batches into multiple blocks and write them into filefservice at one time

type WriterType added in v0.8.0 

type WriterType int8

type ZoneMap 

type ZoneMap = index.ZM

type ZoneMapArea added in v0.8.0 

type ZoneMapArea []byte

func (ZoneMapArea) BlockCount added in v0.8.0 

func (zma ZoneMapArea) BlockCount() uint32

func (ZoneMapArea) GetZoneMap added in v0.8.0 

func (zma ZoneMapArea) GetZoneMap(idx uint16, BlockID uint32) ZoneMap

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.