tidb: github.com/pingcap/tidb/statistics/handle Index | Files

package handle

import "github.com/pingcap/tidb/statistics/handle"


Package Files

bootstrap.go ddl.go dump.go gc.go handle.go update.go


const (
    // StatsStatusInited is the status for extended stats which are just registered but have not been analyzed yet.
    StatsStatusInited uint8 = iota
    // StatsStatusAnalyzed is the status for extended stats which have been collected in analyze.
    // StatsStatusDeleted is the status for extended stats which were dropped. These "deleted" records would be removed from storage by GCStats().
const (
    // DumpAll indicates dump all the delta info in to kv.
    DumpAll dumpMode = true
    // DumpDelta indicates dump part of the delta info in to kv.
    DumpDelta dumpMode = false
const (
    // StatsOwnerKey is the stats owner path that is saved to etcd.
    StatsOwnerKey = "/tidb/stats/owner"
    // StatsPrompt is the prompt for stats owner manager.
    StatsPrompt = "stats"


var (
    // MinLogScanCount is the minimum scan count for a feedback to be logged.
    MinLogScanCount = int64(1000)
    // MinLogErrorRate is the minimum error rate for a feedback to be logged.
    MinLogErrorRate = 0.5
var AutoAnalyzeMinCnt int64 = 1000

AutoAnalyzeMinCnt means if the count of table is less than this value, we needn't do auto analyze.

var (
    // DumpStatsDeltaRatio is the lower bound of `Modify Count / Table Count` for stats delta to be dumped.
    DumpStatsDeltaRatio = 1 / 10000.0

func DurationToTS Uses

func DurationToTS(d time.Duration) uint64

DurationToTS converts duration to timestamp.

func NeedAnalyzeTable Uses

func NeedAnalyzeTable(tbl *statistics.Table, limit time.Duration, autoAnalyzeRatio float64, start, end, now time.Time) (bool, string)

NeedAnalyzeTable checks if we need to analyze the table: 1. If the table has never been analyzed, we need to analyze it when it has

not been modified for a while.

2. If the table had been analyzed before, we need to analyze it when

"tbl.ModifyCount/tbl.Count > autoAnalyzeRatio" and the current time is
between `start` and `end`.

func TableAnalyzed Uses

func TableAnalyzed(tbl *statistics.Table) bool

TableAnalyzed checks if the table is analyzed.

func TableStatsFromJSON Uses

func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *JSONTable) (*statistics.Table, error)

TableStatsFromJSON loads statistic from JSONTable and return the Table of statistic.

type GlobalIndexID Uses

type GlobalIndexID struct {
    TableID int64
    IndexID int64

GlobalIndexID is the key type for indexUsageMap.

type Handle Uses

type Handle struct {
    // contains filtered or unexported fields

Handle can update stats info periodically.

func NewHandle Uses

func NewHandle(ctx sessionctx.Context, lease time.Duration) (*Handle, error)

NewHandle creates a Handle for update stats.

func (*Handle) BuildExtendedStats Uses

func (h *Handle) BuildExtendedStats(tableID int64, cols []*model.ColumnInfo, collectors []*statistics.SampleCollector) (*statistics.ExtendedStatsColl, error)

BuildExtendedStats build extended stats for column groups if needed based on the column samples.

func (*Handle) Clear Uses

func (h *Handle) Clear()

Clear the statsCache, only for test.

func (*Handle) CurrentPruneMode Uses

func (h *Handle) CurrentPruneMode() variable.PartitionPruneMode

CurrentPruneMode indicates whether tbl support runtime prune for table and first partition id.

func (*Handle) DDLEventCh Uses

func (h *Handle) DDLEventCh() chan *util.Event

DDLEventCh returns ddl events channel in handle.

func (*Handle) DeleteTableStatsFromKV Uses

func (h *Handle) DeleteTableStatsFromKV(physicalID int64) (err error)

DeleteTableStatsFromKV deletes table statistics from kv.

func (*Handle) DumpFeedbackForIndex Uses

func (h *Handle) DumpFeedbackForIndex(q *statistics.QueryFeedback, t *statistics.Table) error

DumpFeedbackForIndex dumps the feedback for index. For queries that contains both equality and range query, we will split them and Update accordingly.

func (*Handle) DumpFeedbackToKV Uses

func (h *Handle) DumpFeedbackToKV(fb *statistics.QueryFeedback) error

DumpFeedbackToKV dumps the given feedback to physical kv layer.

func (*Handle) DumpIndexUsageToKV Uses

func (h *Handle) DumpIndexUsageToKV() error

DumpIndexUsageToKV will dump in-memory index usage information to KV.

func (*Handle) DumpStatsDeltaToKV Uses

func (h *Handle) DumpStatsDeltaToKV(mode dumpMode) error

DumpStatsDeltaToKV sweeps the whole list and updates the global map, then we dumps every table that held in map to KV. If the mode is `DumpDelta`, it will only dump that delta info that `Modify Count / Table Count` greater than a ratio.

func (*Handle) DumpStatsFeedbackToKV Uses

func (h *Handle) DumpStatsFeedbackToKV() error

DumpStatsFeedbackToKV dumps the stats feedback to KV.

func (*Handle) DumpStatsToJSON Uses

func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo, historyStatsExec sqlexec.RestrictedSQLExecutor) (*JSONTable, error)

DumpStatsToJSON dumps statistic to json.

func (*Handle) FlushStats Uses

func (h *Handle) FlushStats()

FlushStats flushes the cached stats update into store.

func (*Handle) GCIndexUsage Uses

func (h *Handle) GCIndexUsage() error

GCIndexUsage will delete the usage information of those indexes that do not exist.

func (*Handle) GCStats Uses

func (h *Handle) GCStats(is infoschema.InfoSchema, ddlLease time.Duration) error

GCStats will garbage collect the useless stats info. For dropped tables, we will first update their version so that other tidb could know that table is deleted.

func (*Handle) GetMemConsumed Uses

func (h *Handle) GetMemConsumed() (size int64)

GetMemConsumed returns the mem size of statscache consumed

func (*Handle) GetPartitionStats Uses

func (h *Handle) GetPartitionStats(tblInfo *model.TableInfo, pid int64) *statistics.Table

GetPartitionStats retrieves the partition stats from cache.

func (*Handle) GetQueryFeedback Uses

func (h *Handle) GetQueryFeedback() *statistics.QueryFeedbackMap

GetQueryFeedback gets the query feedback. It is only used in test.

func (*Handle) GetTableStats Uses

func (h *Handle) GetTableStats(tblInfo *model.TableInfo) *statistics.Table

GetTableStats retrieves the statistics table from cache, and the cache will be updated by a goroutine.

func (*Handle) HandleAutoAnalyze Uses

func (h *Handle) HandleAutoAnalyze(is infoschema.InfoSchema)

HandleAutoAnalyze analyzes the newly created table or index.

func (*Handle) HandleDDLEvent Uses

func (h *Handle) HandleDDLEvent(t *util.Event) error

HandleDDLEvent begins to process a ddl task.

func (*Handle) HandleUpdateStats Uses

func (h *Handle) HandleUpdateStats(is infoschema.InfoSchema) error

HandleUpdateStats update the stats using feedback.

func (*Handle) InitStats Uses

func (h *Handle) InitStats(is infoschema.InfoSchema) (err error)

InitStats will init the stats cache using full load strategy.

func (*Handle) InsertExtendedStats Uses

func (h *Handle) InsertExtendedStats(statsName string, colIDs []int64, tp int, tableID int64, ifNotExists bool) (err error)

InsertExtendedStats inserts a record into mysql.stats_extended and update version in mysql.stats_meta.

func (*Handle) LastUpdateVersion Uses

func (h *Handle) LastUpdateVersion() uint64

LastUpdateVersion gets the last update version.

func (*Handle) Lease Uses

func (h *Handle) Lease() time.Duration

Lease returns the stats lease.

func (*Handle) LoadNeededHistograms Uses

func (h *Handle) LoadNeededHistograms() (err error)

LoadNeededHistograms will load histograms for those needed columns.

func (*Handle) LoadStatsFromJSON Uses

func (h *Handle) LoadStatsFromJSON(is infoschema.InfoSchema, jsonTbl *JSONTable) error

LoadStatsFromJSON will load statistic from JSONTable, and save it to the storage.

func (*Handle) MarkExtendedStatsDeleted Uses

func (h *Handle) MarkExtendedStatsDeleted(statsName string, tableID int64, ifExists bool) (err error)

MarkExtendedStatsDeleted update the status of mysql.stats_extended to be `deleted` and the version of mysql.stats_meta.

func (*Handle) NewSessionIndexUsageCollector Uses

func (h *Handle) NewSessionIndexUsageCollector() *SessionIndexUsageCollector

NewSessionIndexUsageCollector will add a new SessionIndexUsageCollector into linked list headed by idxUsageListHead. idxUsageListHead always points to an empty SessionIndexUsageCollector as a sentinel node. So we let idxUsageListHead.next points to new item. It's helpful to sweepIdxUsageList.

func (*Handle) NewSessionStatsCollector Uses

func (h *Handle) NewSessionStatsCollector() *SessionStatsCollector

NewSessionStatsCollector allocates a stats collector for a session.

func (*Handle) RecalculateExpectCount Uses

func (h *Handle) RecalculateExpectCount(q *statistics.QueryFeedback) error

RecalculateExpectCount recalculates the expect row count if the origin row count is estimated by pseudo.

func (*Handle) RefreshVars Uses

func (h *Handle) RefreshVars() error

RefreshVars uses to pull PartitionPruneMethod vars from kv storage.

func (*Handle) ReloadExtendedStatistics Uses

func (h *Handle) ReloadExtendedStatistics() error

ReloadExtendedStatistics drops the cache for extended statistics and reload data from mysql.stats_extended.

func (*Handle) SaveExtendedStatsToStorage Uses

func (h *Handle) SaveExtendedStatsToStorage(tableID int64, extStats *statistics.ExtendedStatsColl, isLoad bool) (err error)

SaveExtendedStatsToStorage writes extended stats of a table into mysql.stats_extended.

func (*Handle) SaveMetaToStorage Uses

func (h *Handle) SaveMetaToStorage(tableID, count, modifyCount int64) (err error)

SaveMetaToStorage will save stats_meta to storage.

func (*Handle) SaveStatsToStorage Uses

func (h *Handle) SaveStatsToStorage(tableID int64, count int64, isIndex int, hg *statistics.Histogram, cms *statistics.CMSketch, topN *statistics.TopN, statsVersion int, isAnalyzed int64) (err error)

SaveStatsToStorage saves the stats to storage.

func (*Handle) SetLastUpdateVersion Uses

func (h *Handle) SetLastUpdateVersion(version uint64)

SetLastUpdateVersion sets the last update version.

func (*Handle) SetLease Uses

func (h *Handle) SetLease(lease time.Duration)

SetLease sets the stats lease.

func (*Handle) Update Uses

func (h *Handle) Update(is infoschema.InfoSchema) error

Update reads stats meta from store and updates the stats map.

func (*Handle) UpdateErrorRate Uses

func (h *Handle) UpdateErrorRate(is infoschema.InfoSchema)

UpdateErrorRate updates the error rate of columns from h.rateMap to cache.

func (*Handle) UpdateStatsByLocalFeedback Uses

func (h *Handle) UpdateStatsByLocalFeedback(is infoschema.InfoSchema)

UpdateStatsByLocalFeedback will update statistics by the local feedback. Currently, we dump the feedback with the period of 10 minutes, which means it takes 10 minutes for a feedback to take effect. However, we can use the feedback locally on this tidb-server, so it could be used more timely.

type IndexUsageInformation Uses

type IndexUsageInformation struct {
    QueryCount   int64
    RowsSelected int64
    LastUsedAt   string

IndexUsageInformation is the data struct to store index usage information.

type JSONTable Uses

type JSONTable struct {
    DatabaseName string                 `json:"database_name"`
    TableName    string                 `json:"table_name"`
    Columns      map[string]*jsonColumn `json:"columns"`
    Indices      map[string]*jsonColumn `json:"indices"`
    ExtStats     []*jsonExtendedStats   `json:"ext_stats"`
    Count        int64                  `json:"count"`
    ModifyCount  int64                  `json:"modify_count"`
    Partitions   map[string]*JSONTable  `json:"partitions"`

JSONTable is used for dumping statistics.

type SessionIndexUsageCollector Uses

type SessionIndexUsageCollector struct {
    // contains filtered or unexported fields

SessionIndexUsageCollector is a list item that holds the index usage mapper. If you want to write or read mapper, you must lock it.

func (*SessionIndexUsageCollector) Delete Uses

func (s *SessionIndexUsageCollector) Delete()

Delete will set s.deleted to true which means it can be deleted from linked list.

func (*SessionIndexUsageCollector) Update Uses

func (s *SessionIndexUsageCollector) Update(tableID int64, indexID int64, value *IndexUsageInformation)

Update updates the mapper in SessionIndexUsageCollector.

type SessionStatsCollector Uses

type SessionStatsCollector struct {
    // contains filtered or unexported fields

SessionStatsCollector is a list item that holds the delta mapper. If you want to write or read mapper, you must lock it.

func (*SessionStatsCollector) Delete Uses

func (s *SessionStatsCollector) Delete()

Delete only sets the deleted flag true, it will be deleted from list when DumpStatsDeltaToKV is called.

func (*SessionStatsCollector) StoreQueryFeedback Uses

func (s *SessionStatsCollector) StoreQueryFeedback(feedback interface{}, h *Handle) error

StoreQueryFeedback merges the feedback into stats collector.

func (*SessionStatsCollector) Update Uses

func (s *SessionStatsCollector) Update(id int64, delta int64, count int64, colSize *map[int64]int64)

Update will updates the delta and count for one table id.

Package handle imports 39 packages (graph) and is imported by 29 packages. Updated 2021-01-25. Refresh now. Tools for package owners.