common

package
v1.1.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 15, 2024 License: Apache-2.0 Imports: 24 Imported by: 0

Documentation

Overview

A few allocators for TAE

Index

Constants

View Source 
const (
	DefaultNotLoadMoreThan  = 4096
	DefaultMinRowsQualified = 40960
	DefaultMaxMergeObjN     = 2

	Const1GBytes = 1 << 30
	Const1MBytes = 1 << 20
)
View Source 
const (
	CloseIntervalSize int64 = int64(unsafe.Sizeof(ClosedInterval{}))
)
View Source 
const DefaultMaxRowsToPrint = 3
View Source 
const (
	IDSize int64 = int64(unsafe.Sizeof(ID{}))
)

Variables

View Source 
var (
	RuntimeNotLoadMoreThan  atomic.Int32
	RuntimeMaxMergeObjN     atomic.Int32
	RuntimeMinRowsQualified atomic.Int32
)
View Source 
var AnyField = zap.Any
View Source 
var CheckpointAllocator *mpool.MPool
View Source 
var DebugAllocator *mpool.MPool
View Source 
var DefaultAllocator *mpool.MPool

A few allocators for TAE

View Source 
var (
	GlobalSeqNum atomic.Uint64
)
View Source 
var LogtailAllocator *mpool.MPool
View Source 
var MergeAllocator *mpool.MPool
View Source 
var MutMemAllocator *mpool.MPool
View Source 
var SmallAllocator *mpool.MPool
View Source 
var StringerField = zap.Stringer
View Source 
var WorkspaceAllocator *mpool.MPool

Functions

func ApiBatchToString added in v1.0.0 

func ApiBatchToString(apiBat *api.Batch, printN int) string

func BitmapEqual added in v0.8.0 

func BitmapEqual(v1, v2 *nulls.Bitmap) bool

func BlockIDArraryString added in v0.7.0 

func BlockIDArraryString(ids []ID) string

func ContextField 

func ContextField(format string, a ...any) zap.Field

func CountField 

func CountField(val int) zap.Field

func DebugMoBatch added in v0.8.0 

func DebugMoBatch(moBat *batch.Batch) string

func DoIfDebugEnabled added in v0.8.0 

func DoIfDebugEnabled(fn func())

func DoIfErrorEnabled added in v0.8.0 

func DoIfErrorEnabled(fn func())

func DoIfFatalEnabled added in v0.8.0 

func DoIfFatalEnabled(fn func())

func DoIfInfoEnabled added in v0.8.0 

func DoIfInfoEnabled(fn func())

func DoIfWarnEnabled added in v0.8.0 

func DoIfWarnEnabled(fn func())

func DurationField 

func DurationField(val time.Duration) zap.Field

func EncodeCloseInterval added in v0.8.0 

func EncodeCloseInterval(i *ClosedInterval) []byte

func EncodeID added in v0.8.0 

func EncodeID(id *ID) []byte

func EntityField 

func EntityField(val any) zap.Field

func ErrorField 

func ErrorField(val error) zap.Field

func ExceptionField 

func ExceptionField(val any) zap.Field

func FormatFiled 

func FormatFiled(key string, format string, a ...any) zap.Field

func GetGlobalSeqNum 

func GetGlobalSeqNum() uint64

func HumanReadableBytes added in v1.0.0 

func HumanReadableBytes(bytes int) string

func IDArraryString 

func IDArraryString(ids []ID) string

func IDField 

func IDField(val int) zap.Field

func InitTAEMPool added in v0.6.0 

func InitTAEMPool()

func LoopGenericSortedDList added in v0.6.0 

func LoopGenericSortedDList[T any](head *GenericDLNode[T],
	fn func(node *GenericDLNode[T]) bool,
	reverse bool)

Loop the list and apply fn on each node head is the head node of a list fn is operation applied to each node during iterating. if fn(node) returns false, stop iterating. reverse is true to loop in reversed way.

func MOOrRoaringBitmap added in v0.8.0 

func MOOrRoaringBitmap(bm *nulls.Bitmap, rbm *roaring.Bitmap)

func MoBatchToString added in v1.0.0 

func MoBatchToString(moBat *batch.Batch, printN int) string

func MoVectorToString added in v1.0.0 

func MoVectorToString(v *vector.Vector, printN int, opts ...TypePrintOpt) string

func NameSpaceField 

func NameSpaceField(val string) zap.Field

func NextGlobalSeqNum 

func NextGlobalSeqNum() uint64

func OperandField 

func OperandField(val any) zap.Field

func OperandNameSpace 

func OperandNameSpace() zap.Field

func OperationField 

func OperationField(val string) zap.Field

func ReasonField 

func ReasonField(val string) zap.Field

func RepeatStr 

func RepeatStr(str string, times int) string

func ReprerField 

func ReprerField(key string, val Reprer) zap.Field

func RetryWithIntervalAndTimeout added in v0.7.0 

func RetryWithIntervalAndTimeout(
	op WaitOp,
	timeout time.Duration,
	interval time.Duration, suppressTimout bool) (err error)

func RoaringToMOBitmap added in v0.8.0 

func RoaringToMOBitmap(bm *roaring.Bitmap) *nulls.Bitmap

func ShortObjId added in v1.1.0 

func ShortObjId(x types.Objectid) string

func ShortSegId added in v1.0.0 

func ShortSegId(x types.Uuid) string

func TimestampField 

func TimestampField(val any) zap.Field

func TypeStringValue added in v0.6.0 

func TypeStringValue(t types.Type, v any, isNull bool, opts ...TypePrintOpt) string

Types

type ClosedInterval 

type ClosedInterval struct {
	Start, End uint64
}

func (*ClosedInterval) Append 

func (i *ClosedInterval) Append(id uint64) error

func (*ClosedInterval) AtomicUpdateEnd 

func (i *ClosedInterval) AtomicUpdateEnd(v uint64)

func (*ClosedInterval) Contains 

func (i *ClosedInterval) Contains(o ClosedInterval) bool

func (*ClosedInterval) GT 

func (i *ClosedInterval) GT(o *ClosedInterval) bool

func (*ClosedInterval) IsCoveredByInt 

func (i *ClosedInterval) IsCoveredByInt(idx uint64) bool

func (*ClosedInterval) LT 

func (i *ClosedInterval) LT(o *ClosedInterval) bool

func (*ClosedInterval) String 

func (i *ClosedInterval) String() string

func (*ClosedInterval) TryMerge 

func (i *ClosedInterval) TryMerge(o ClosedInterval) bool

func (*ClosedInterval) Uint64Contains added in v1.0.0 

func (i *ClosedInterval) Uint64Contains(start, end uint64) bool

type ClosedIntervals 

type ClosedIntervals struct {
	Intervals []*ClosedInterval
}

func NewClosedIntervals 

func NewClosedIntervals() *ClosedIntervals

func NewClosedIntervalsByInt 

func NewClosedIntervalsByInt(i uint64) *ClosedIntervals

func NewClosedIntervalsByInterval 

func NewClosedIntervalsByInterval(i *ClosedInterval) *ClosedIntervals

func NewClosedIntervalsByIntervals 

func NewClosedIntervalsByIntervals(i *ClosedIntervals) *ClosedIntervals

func NewClosedIntervalsBySlice 

func NewClosedIntervalsBySlice(array []uint64) *ClosedIntervals

func (*ClosedIntervals) Contains 

func (intervals *ClosedIntervals) Contains(o ClosedIntervals) bool

func (*ClosedIntervals) ContainsInt added in v0.6.0 

func (intervals *ClosedIntervals) ContainsInt(n uint64) bool

func (*ClosedIntervals) ContainsInterval 

func (intervals *ClosedIntervals) ContainsInterval(oIntervals ClosedInterval) bool

func (*ClosedIntervals) Equal 

func (intervals *ClosedIntervals) Equal(o *ClosedIntervals) bool

Equal is for test

func (*ClosedIntervals) GetCardinality 

func (intervals *ClosedIntervals) GetCardinality() int

func (*ClosedIntervals) GetMax added in v0.6.0 

func (intervals *ClosedIntervals) GetMax() uint64

func (*ClosedIntervals) GetMin added in v0.6.0 

func (intervals *ClosedIntervals) GetMin() uint64

func (*ClosedIntervals) IsCoveredByInt 

func (intervals *ClosedIntervals) IsCoveredByInt(i uint64) bool

func (*ClosedIntervals) ReadFrom 

func (intervals *ClosedIntervals) ReadFrom(r io.Reader) (n int64, err error)

func (*ClosedIntervals) TryMerge 

func (intervals *ClosedIntervals) TryMerge(o ClosedIntervals)

func (*ClosedIntervals) WriteTo 

func (intervals *ClosedIntervals) WriteTo(w io.Writer) (n int64, err error)

type DurationRecords added in v1.0.0 

type DurationRecords struct {
	Duration time.Duration
}

type GenericDLNode added in v0.6.0 

type GenericDLNode[T any] struct {
	// contains filtered or unexported fields
}

Doubly sorted linked-list node

func InsertGenericDLNode added in v0.6.0 

func InsertGenericDLNode[T any](payload T,
	head *GenericDLNode[T],
	compare func(T, T) int) (node, nhead, ntail *GenericDLNode[T])

Insert a wrapped object into a list specified by a head node node is the inserted dlnode nhead is the new head node ntail is the new tail node. If ntail is not nil, tail is updated. Else tail is not updated

func (*GenericDLNode[T]) GetNext added in v0.6.0 

func (l *GenericDLNode[T]) GetNext() *GenericDLNode[T]

func (*GenericDLNode[T]) GetPayload added in v0.6.0 

func (l *GenericDLNode[T]) GetPayload() T

func (*GenericDLNode[T]) GetPrev added in v0.6.0 

func (l *GenericDLNode[T]) GetPrev() *GenericDLNode[T]

func (*GenericDLNode[T]) KeepSorted added in v0.6.0 

func (l *GenericDLNode[T]) KeepSorted(compare func(T, T) int) (head, tail *GenericDLNode[T])

Keep node be sorted in the list

type GenericSortedDList added in v0.6.0 

type GenericSortedDList[T any] struct {
	// contains filtered or unexported fields
}

Sorted doubly linked-list

func NewGenericSortedDList added in v0.6.0 

func NewGenericSortedDList[T any](compare func(T, T) int) *GenericSortedDList[T]

func (*GenericSortedDList[T]) Delete added in v0.6.0 

func (l *GenericSortedDList[T]) Delete(n *GenericDLNode[T])

Given a node and remove it from the list 

Delete [node]

[prev node] <-> [node] <-> [next node] =============> [prev node] <-> [next node]

func (*GenericSortedDList[T]) Depth added in v0.6.0 

func (l *GenericSortedDList[T]) Depth() int

Get the length of the list

func (*GenericSortedDList[T]) GetHead added in v0.6.0 

func (l *GenericSortedDList[T]) GetHead() *GenericDLNode[T]

Get the head node

func (*GenericSortedDList[T]) GetTail added in v0.6.0 

func (l *GenericSortedDList[T]) GetTail() *GenericDLNode[T]

Get the tail node

func (*GenericSortedDList[T]) Insert added in v0.6.0 

func (l *GenericSortedDList[T]) Insert(payload T) *GenericDLNode[T]

Insert a object and wrap it as GenericDLNode into the list The inserted object must be instance of interface NodePayload [List]: [1,x1] <-> [5,x5] <-> [10,x10] Insert a node [7,x7] [List]: [1,x1] <-> [5,x5] <-> [7,x7] <-> [10,x10]

func (*GenericSortedDList[T]) Loop added in v0.6.0 

func (l *GenericSortedDList[T]) Loop(fn func(n *GenericDLNode[T]) bool, reverse bool)

Loop the list and apply fn on each node

func (*GenericSortedDList[T]) Update added in v0.6.0 

func (l *GenericSortedDList[T]) Update(n *GenericDLNode[T])

Update the node to keep the list be sorted

[List] [1,x1] <-> [3,x3] <-> [10,x10] <-> [20,x20] 

|

[Node] [10,x10]

--------- UPDATE [10,x10] TO [2, x10]--------------

[List] [1,x1] <-> [2,x10] <-> [3,x3] <-> [20,x20]

type GenericSortedDListIt added in v0.6.0 

type GenericSortedDListIt[T any] struct {
	// contains filtered or unexported fields
}

Sorted doubly linked-list iterator

func NewGenericSortedDListIt added in v0.6.0 

func NewGenericSortedDListIt[T any](linkLocker *sync.RWMutex,
	dlist *GenericSortedDList[T],
	reverse bool) *GenericSortedDListIt[T]

linkLocker is the outer locker to guard dlist access

func (*GenericSortedDListIt[T]) Get added in v0.6.0 

func (it *GenericSortedDListIt[T]) Get() *GenericDLNode[T]

func (*GenericSortedDListIt[T]) Next added in v0.6.0 

func (it *GenericSortedDListIt[T]) Next()

func (*GenericSortedDListIt[T]) Valid added in v0.6.0 

func (it *GenericSortedDListIt[T]) Valid() bool

type ID 

type ID struct {
	// Internal db id
	DbID uint64
	// Internal table id
	TableID uint64
	// Internal block id
	BlockID types.Blockid
}

ID is the general identifier type shared by different types like table, object, block, etc.

We could wrap info from upper level via ID, for instance, get the table id, object id, and the block id for one block by ID.AsBlockID, which made the resource management easier.

func (*ID) AsBlockID 

func (id *ID) AsBlockID() *ID

func (*ID) BlockString 

func (id *ID) BlockString() string

func (*ID) DBString added in v0.8.0 

func (id *ID) DBString() string

func (*ID) ObjectID added in v1.1.0 

func (id *ID) ObjectID() *types.Objectid

func (*ID) ObjectString added in v1.1.0 

func (id *ID) ObjectString() string

func (*ID) SegmentID 

func (id *ID) SegmentID() *types.Segmentid

func (*ID) SetObjectID added in v1.1.0 

func (id *ID) SetObjectID(oid *types.Objectid)

func (*ID) SetSegmentID added in v0.8.0 

func (id *ID) SetSegmentID(sid *types.Segmentid)

func (*ID) String 

func (id *ID) String() string

func (*ID) TableString 

func (id *ID) TableString() string

type IRef 

type IRef interface {
	RefCount() int64
	// RefIfHasRef increment refcnt if existing cnt > 0 and return true,
	// return false if cnt is zero. Note: the udpate is atomic
	RefIfHasRef() bool
	Ref()
	Unref()
}

IRef is the general representation of the resources that should be managed with a reference count. Once the reference count reached 0, the OnZeroCB would be called.

type ISLLNode 

type ISLLNode interface {
	IRef
}

type ISSLLNode 

type ISSLLNode interface {
	Insert(ISSLLNode)
	GetNext() ISSLLNode
	SetNext(ISSLLNode)
	ReleaseNextNode() ISSLLNode
	ReleaseFollowing() ISSLLNode
}

type IdAlloctor 

type IdAlloctor struct {
	// contains filtered or unexported fields
}

func NewIdAlloctor 

func NewIdAlloctor(from uint64) *IdAlloctor

func (*IdAlloctor) Alloc 

func (alloc *IdAlloctor) Alloc() uint64

func (*IdAlloctor) Get 

func (alloc *IdAlloctor) Get() uint64

func (*IdAlloctor) SetStart 

func (alloc *IdAlloctor) SetStart(start uint64)

type MergeHistory added in v1.0.0 

type MergeHistory struct {
	LastTime time.Time
	OSize    int
	NObj     int
	NBlk     int
}

func (*MergeHistory) Add added in v1.0.0 

func (h *MergeHistory) Add(osize, nobj, nblk int)

func (*MergeHistory) IsLastBefore added in v1.0.0 

func (h *MergeHistory) IsLastBefore(d time.Duration) bool

func (*MergeHistory) String added in v1.0.1 

func (h *MergeHistory) String() string

type OnZeroCB 

type OnZeroCB func()

type PPLevel 

type PPLevel int8
const (
	PPL0 PPLevel = iota
	PPL1
	PPL2
	PPL3
)

type PinnedItem added in v0.6.0 

type PinnedItem[T IRef] struct {
	Val T
}

func (*PinnedItem[T]) Close added in v0.6.0 

func (item *PinnedItem[T]) Close()

func (*PinnedItem[T]) Item added in v0.6.0 

func (item *PinnedItem[T]) Item() T

type RecordCtxKeyType added in v1.0.0 

type RecordCtxKeyType string
var ActiveHandleCommit RecordCtxKeyType = "Active Handle Commit"
var ActiveHandleRequests RecordCtxKeyType = "Active Handle Requests"
var DequeuePrepared RecordCtxKeyType = "Dequeue Prepared"
var DequeuePreparing RecordCtxKeyType = "Dequeue Preparing"
var PrepareLogtail RecordCtxKeyType = "Prepare Logtail"
var PrepareWAL RecordCtxKeyType = "Prepare WAL"
var StoreApplyCommit RecordCtxKeyType = "Store Apply Commit"
var StorePreApplyCommit RecordCtxKeyType = "Store Pre Apply Commit"
var StorePrePrepare RecordCtxKeyType = "Store Pre Prepare"
var StorePrepareCommit RecordCtxKeyType = "Store Prepare Commit"

type RefHelper 

type RefHelper struct {
	Refs     atomic.Int64
	OnZeroCB OnZeroCB
}

func (*RefHelper) Ref 

func (helper *RefHelper) Ref()

func (*RefHelper) RefCount 

func (helper *RefHelper) RefCount() int64

func (*RefHelper) RefIfHasRef added in v0.7.0 

func (helper *RefHelper) RefIfHasRef() bool

func (*RefHelper) Unref 

func (helper *RefHelper) Unref()

type Reprer 

type Reprer interface {
	Repr() string
}

type RetryOp 

type RetryOp = func() error

type RowGen 

type RowGen interface {
	HasNext() bool
	Next() uint32
}

type SLLNode 

type SLLNode struct {
	RefHelper
	*sync.RWMutex
	Next ISLLNode
}

SLLNode represent a single node in linked list. It is thread-safe.

func NewSLLNode 

func NewSLLNode(mu *sync.RWMutex) *SLLNode

func (*SLLNode) GetNextNode 

func (l *SLLNode) GetNextNode() ISLLNode

func (*SLLNode) Insert 

func (l *SLLNode) Insert(n ISLLNode)

func (*SLLNode) ReleaseNextNode 

func (l *SLLNode) ReleaseNextNode()

func (*SLLNode) SetNextNode 

func (l *SLLNode) SetNextNode(next ISLLNode)

func (*SLLNode) SetNextNodeNoLock 

func (l *SLLNode) SetNextNodeNoLock(next ISLLNode)

type SSLLNode 

type SSLLNode struct {
	// contains filtered or unexported fields
}

func NewSSLLNode 

func NewSSLLNode() *SSLLNode

func (*SSLLNode) GetNext 

func (n *SSLLNode) GetNext() ISSLLNode

func (*SSLLNode) Insert 

func (n *SSLLNode) Insert(nn ISSLLNode)

func (*SSLLNode) ReleaseFollowing 

func (n *SSLLNode) ReleaseFollowing() ISSLLNode

func (*SSLLNode) ReleaseNextNode 

func (n *SSLLNode) ReleaseNextNode() ISSLLNode

func (*SSLLNode) SetNext 

func (n *SSLLNode) SetNext(next ISSLLNode)

type TableCompactStat added in v1.0.0 

type TableCompactStat struct {
	sync.RWMutex

	Inited bool

	// how often to flush table tail
	// this duration will be add some random value to avoid flush many tables at the same time
	FlushGapDuration time.Duration
	// if the size of table tail, in bytes, exceeds FlushMemCapacity, flush it immediately
	FlushMemCapacity int

	// dirty end range flushed by last flush txn. If we are waiting for a ckp [a, b], and all dirty tables' LastFlush are greater than b,
	// the checkpoint is ready to collect data and write all down.
	LastFlush types.TS
	// FlushDeadline is the deadline to flush table tail
	FlushDeadline time.Time

	MergeHist MergeHistory
}

func (*TableCompactStat) AddMerge added in v1.0.0 

func (s *TableCompactStat) AddMerge(osize, nobj, nblk int)

func (*TableCompactStat) GetLastFlush added in v1.0.1 

func (s *TableCompactStat) GetLastFlush() types.TS

func (*TableCompactStat) GetLastMerge added in v1.0.0 

func (s *TableCompactStat) GetLastMerge() *MergeHistory

func (*TableCompactStat) InitWithLock added in v1.0.0 

func (s *TableCompactStat) InitWithLock(durationHint time.Duration)

func (*TableCompactStat) ResetDeadlineWithLock added in v1.0.0 

func (s *TableCompactStat) ResetDeadlineWithLock()

type TxnIDAllocator added in v0.6.0 

type TxnIDAllocator struct {
}

func NewTxnIDAllocator added in v0.6.0 

func NewTxnIDAllocator() *TxnIDAllocator

func (*TxnIDAllocator) Alloc added in v0.6.0 

func (alloc *TxnIDAllocator) Alloc() []byte

type TypePrintOpt added in v0.6.0 

type TypePrintOpt interface {
	// contains filtered or unexported methods
}

type WaitOp added in v0.7.0 

type WaitOp = func() (ok bool, err error)

type WithDoNotPrintBin added in v0.6.0 

type WithDoNotPrintBin struct{}

type WithSpecialRowid added in v0.6.0 

type WithSpecialRowid struct{}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.