chbackup

package
v0.5.3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 7, 2020 License: MIT Imports: 40 Imported by: 0

Documentation

Overview

Package chbackup describes the objects and functions for clickhouse-backup app

Index

Constants

View Source
const (
	// MetaFileName - meta file name
	MetaFileName = "meta.json"
	// BufferSize - size of ring buffer between stream handlers
	BufferSize = 4 * 1024 * 1024
)
View Source
const (
	// BackupTimeFormat - default backup name format
	BackupTimeFormat = "2006-01-02T15-04-05"
)

Variables

View Source
var (
	// ErrNotFound is returned when file/object cannot be found
	ErrNotFound = errors.New("file not found")
)
View Source
var (
	// ErrUnknownClickhouseDataPath -
	ErrUnknownClickhouseDataPath = errors.New("clickhouse data path is unknown, you can set data_path in config file")
)

Functions

func Clean

func Clean(config Config) error

Clean - removed all data in shadow folder

func CreateBackup

func CreateBackup(config Config, backupName, tablePattern string) error

CreateBackup - create new backup of all tables matched by tablePattern If backupName is empty string will use default backup name

func Download

func Download(config Config, backupName string) error

func FormatBytes

func FormatBytes(i int64) (result string)

FormatBytes - Convert bytes to human readable string

func Freeze

func Freeze(config Config, tablePattern string) error

Freeze - freeze tables by tablePattern

func GetLocalBackup

func GetLocalBackup(config Config, backupName string) error

func NewBackupName

func NewBackupName() string

NewBackupName - return default backup name

func PrintDefaultConfig

func PrintDefaultConfig()

PrintDefaultConfig - print default config to stdout

func PrintLocalBackups

func PrintLocalBackups(config Config, format string) error

PrintLocalBackups - print all backups stored locally

func PrintRemoteBackups

func PrintRemoteBackups(config Config, format string) error

PrintRemoteBackups - print all backups stored on remote storage

func PrintTables

func PrintTables(config Config) error

PrintTables - print all tables suitable for backup

func RemoveBackupLocal

func RemoveBackupLocal(config Config, backupName string) error

func RemoveBackupRemote

func RemoveBackupRemote(config Config, backupName string) error

func RemoveOldBackupsLocal

func RemoveOldBackupsLocal(config Config) error

func Restore

func Restore(config Config, backupName string, tablePattern string, schemaOnly bool, dataOnly bool) error

Restore - restore tables matched by tablePattern from backupName

func RestoreData

func RestoreData(config Config, backupName string, tablePattern string) error

RestoreData - restore data for tables matched by tablePattern from backupName

func TablePathEncode

func TablePathEncode(str string) string

func Upload

func Upload(config Config, backupName string, diffFrom string) error

Types

type Backup

type Backup struct {
	Name string
	Size int64
	Date time.Time
}

func GetBackupsToDelete

func GetBackupsToDelete(backups []Backup, keep int) []Backup

func ListLocalBackups

func ListLocalBackups(config Config) ([]Backup, error)

ListLocalBackups - return slice of all backups stored locally

type BackupDestination

type BackupDestination struct {
	RemoteStorage
	// contains filtered or unexported fields
}

func NewBackupDestination

func NewBackupDestination(config Config) (*BackupDestination, error)

func (*BackupDestination) BackupList

func (bd *BackupDestination) BackupList() ([]Backup, error)

func (*BackupDestination) BackupsToKeep

func (bd *BackupDestination) BackupsToKeep() int

func (*BackupDestination) CompressedStreamDownload

func (bd *BackupDestination) CompressedStreamDownload(remotePath string, localPath string) error

func (*BackupDestination) CompressedStreamUpload

func (bd *BackupDestination) CompressedStreamUpload(localPath, remotePath, diffFromPath string) error

func (*BackupDestination) RemoveBackup

func (bd *BackupDestination) RemoveBackup(backupName string) error

func (*BackupDestination) RemoveOldBackups

func (bd *BackupDestination) RemoveOldBackups(keep int) error

type BackupPartition

type BackupPartition struct {
	Name string
	Path string
}

BackupPartition - struct representing Clickhouse partition

type BackupTable

type BackupTable struct {
	Database   string
	Name       string
	Partitions []BackupPartition
}

BackupTable - struct to store additional information on partitions

type BackupTables

type BackupTables []BackupTable

BackupTables - slice of BackupTable

func (BackupTables) Sort

func (bt BackupTables) Sort()

Sort - sorting BackupTables slice orderly by name

type Bar

type Bar struct {
	// contains filtered or unexported fields
}

func StartNewBar

func StartNewBar(show bool, total int) *Bar

func StartNewByteBar

func StartNewByteBar(show bool, total int64) *Bar

func (*Bar) Add64

func (b *Bar) Add64(add int64)

func (*Bar) Finish

func (b *Bar) Finish()

func (*Bar) Increment

func (b *Bar) Increment()

func (*Bar) NewProxyReader

func (b *Bar) NewProxyReader(r io.Reader) io.Reader

func (*Bar) Set

func (b *Bar) Set(current int)

type COS

type COS struct {
	Config *COSConfig
	// contains filtered or unexported fields
}

func (*COS) Connect

func (c *COS) Connect() error

Connect - connect to cos

func (*COS) DeleteFile

func (c *COS) DeleteFile(key string) error

func (*COS) GetFile

func (c *COS) GetFile(key string) (RemoteFile, error)

func (*COS) GetFileReader

func (c *COS) GetFileReader(key string) (io.ReadCloser, error)

func (*COS) Kind

func (c *COS) Kind() string

func (*COS) PutFile

func (c *COS) PutFile(key string, r io.ReadCloser) error

func (*COS) Walk

func (c *COS) Walk(path string, process func(RemoteFile)) error

type COSConfig

type COSConfig struct {
	RowURL            string `yaml:"url" envconfig:"COS_URL"`
	Timeout           string `yaml:"timeout" envconfig:"COS_TIMEOUT"`
	SecretID          string `yaml:"secret_id" envconfig:"COS_SECRET_ID"`
	SecretKey         string `yaml:"secret_key" envconfig:"COS_SECRET_KEY"`
	Path              string `yaml:"path" envconfig:"COS_PATH"`
	CompressionFormat string `yaml:"compression_format" envconfig:"COS_COMPRESSION_FORMAT"`
	CompressionLevel  int    `yaml:"compression_level" envconfig:"COS_COMPRESSION_LEVEL"`
	Debug             bool   `yaml:"debug" envconfig:"COS_DEBUG"`
}

COSConfig - cos settings section

type ClickHouse

type ClickHouse struct {
	Config *ClickHouseConfig
	// contains filtered or unexported fields
}

ClickHouse - provide

func (*ClickHouse) AttachPatritions

func (ch *ClickHouse) AttachPatritions(table BackupTable) error

AttachPatritions - execute ATTACH command for specific table

func (*ClickHouse) Chown

func (ch *ClickHouse) Chown(filename string) error

Chown - set permission on file to clickhouse user This is necessary that the ClickHouse will be able to read parts files on restore

func (*ClickHouse) Close

func (ch *ClickHouse) Close() error

Close - closing connection to ClickHouse

func (*ClickHouse) Connect

func (ch *ClickHouse) Connect() error

Connect - establish connection to ClickHouse

func (*ClickHouse) CopyData

func (ch *ClickHouse) CopyData(table BackupTable) error

CopyData - copy partitions for specific table to detached folder

func (*ClickHouse) CreateDatabase

func (ch *ClickHouse) CreateDatabase(database string) error

CreateDatabase - create ClickHouse database

func (*ClickHouse) CreateTable

func (ch *ClickHouse) CreateTable(table RestoreTable) error

CreateTable - create ClickHouse table

func (*ClickHouse) FreezeTable

func (ch *ClickHouse) FreezeTable(table Table) error

FreezeTable - freeze all partitions for table This way available for ClickHouse sience v19.1

func (*ClickHouse) FreezeTableOldWay

func (ch *ClickHouse) FreezeTableOldWay(table Table) error

FreezeTableOldWay - freeze all partitions in table one by one This way using for ClickHouse below v19.1

func (*ClickHouse) GetBackupTables

func (ch *ClickHouse) GetBackupTables(backupName string) (map[string]BackupTable, error)

GetBackupTables - return list of backups of tables that can be restored

func (*ClickHouse) GetConn

func (ch *ClickHouse) GetConn() *sqlx.DB

GetConn - return current connection

func (*ClickHouse) GetDataPath

func (ch *ClickHouse) GetDataPath() (string, error)

GetDataPath - return ClickHouse data_path

func (*ClickHouse) GetTables

func (ch *ClickHouse) GetTables() ([]Table, error)

GetTables - return slice of all tables suitable for backup

func (*ClickHouse) GetVersion

func (ch *ClickHouse) GetVersion() (int, error)

GetVersion - returned ClickHouse version in number format Example value: 19001005

type ClickHouseConfig

type ClickHouseConfig struct {
	Username     string   `yaml:"username" envconfig:"CLICKHOUSE_USERNAME"`
	Password     string   `yaml:"password" envconfig:"CLICKHOUSE_PASSWORD"`
	Host         string   `yaml:"host" envconfig:"CLICKHOUSE_HOST"`
	Port         uint     `yaml:"port" envconfig:"CLICKHOUSE_PORT"`
	DataPath     string   `yaml:"data_path" envconfig:"CLICKHOUSE_DATA_PATH"`
	SkipTables   []string `yaml:"skip_tables" envconfig:"CLICKHOUSE_SKIP_TABLES"`
	Timeout      string   `yaml:"timeout" envconfig:"CLICKHOUSE_TIMEOUT"`
	FreezeByPart bool     `yaml:"freeze_by_part" envconfig:"CLICKHOUSE_FREEZE_BY_PART"`
}

ClickHouseConfig - clickhouse settings section

type Config

type Config struct {
	General    GeneralConfig    `yaml:"general"`
	ClickHouse ClickHouseConfig `yaml:"clickhouse"`
	S3         S3Config         `yaml:"s3"`
	GCS        GCSConfig        `yaml:"gcs"`
	COS        COSConfig        `yaml:"cos"`
}

Config - config file format

func DefaultConfig

func DefaultConfig() *Config

func LoadConfig

func LoadConfig(configLocation string) (*Config, error)

LoadConfig - load config from file

type GCS

type GCS struct {
	Config *GCSConfig
	// contains filtered or unexported fields
}

GCS - presents methods for manipulate data on GCS

func (*GCS) Connect

func (gcs *GCS) Connect() error

Connect - connect to GCS

func (*GCS) DeleteFile

func (gcs *GCS) DeleteFile(key string) error

func (*GCS) GetFile

func (gcs *GCS) GetFile(key string) (RemoteFile, error)

func (*GCS) GetFileReader

func (gcs *GCS) GetFileReader(key string) (io.ReadCloser, error)

func (*GCS) GetFileWriter

func (gcs *GCS) GetFileWriter(key string) io.WriteCloser

func (*GCS) Kind

func (gcs *GCS) Kind() string

func (*GCS) PutFile

func (gcs *GCS) PutFile(key string, r io.ReadCloser) error

func (*GCS) Walk

func (gcs *GCS) Walk(gcsPath string, process func(r RemoteFile)) error

type GCSConfig

type GCSConfig struct {
	CredentialsFile   string `yaml:"credentials_file" envconfig:"GCS_CREDENTIALS_FILE"`
	CredentialsJSON   string `yaml:"credentials_json" envconfig:"GCS_CREDENTIALS_JSON"`
	Bucket            string `yaml:"bucket" envconfig:"GCS_BUCKET"`
	Path              string `yaml:"path" envconfig:"GCS_PATH"`
	CompressionLevel  int    `yaml:"compression_level" envconfig:"GCS_COMPRESSION_LEVEL"`
	CompressionFormat string `yaml:"compression_format" envconfig:"GCS_COMPRESSION_FORMAT"`
}

GCSConfig - GCS settings section

type GeneralConfig

type GeneralConfig struct {
	RemoteStorage       string `yaml:"remote_storage" envconfig:"REMOTE_STORAGE"`
	DisableProgressBar  bool   `yaml:"disable_progress_bar" envconfig:"DISABLE_PROGRESS_BAR"`
	BackupsToKeepLocal  int    `yaml:"backups_to_keep_local" envconfig:"BACKUPS_TO_KEEP_LOCAL"`
	BackupsToKeepRemote int    `yaml:"backups_to_keep_remote" envconfig:"BACKUPS_TO_KEEP_REMOTE"`
}

GeneralConfig - general setting section

type MetaFile

type MetaFile struct {
	RequiredBackup string   `json:"required_backup"`
	Hardlinks      []string `json:"hardlinks"`
}

MetaFile - structure describe meta file that will be added to incremental backups archive. Contains info of required files in backup and files

type RemoteFile

type RemoteFile interface {
	Size() int64
	Name() string
	LastModified() time.Time
}

RemoteFile - interface describe file on remote storage

type RemoteStorage

type RemoteStorage interface {
	Kind() string
	GetFile(string) (RemoteFile, error)
	DeleteFile(string) error
	Connect() error
	Walk(string, func(RemoteFile)) error
	GetFileReader(key string) (io.ReadCloser, error)
	PutFile(key string, r io.ReadCloser) error
}

RemoteStorage -

type RestoreTable

type RestoreTable struct {
	Database string
	Table    string
	Query    string
	Path     string
}

RestoreTable - struct to store information needed during restore

type RestoreTables

type RestoreTables []RestoreTable

RestoreTables - slice of RestoreTable

func (RestoreTables) Sort

func (rt RestoreTables) Sort()

Sort - sorting BackupTables slice orderly by name

type S3

type S3 struct {
	Config *S3Config
	// contains filtered or unexported fields
}

S3 - presents methods for manipulate data on s3

func (*S3) Connect

func (s *S3) Connect() error

Connect - connect to s3

func (*S3) DeleteFile

func (s *S3) DeleteFile(key string) error

func (*S3) GetFile

func (s *S3) GetFile(key string) (RemoteFile, error)

func (*S3) GetFileReader

func (s *S3) GetFileReader(key string) (io.ReadCloser, error)

func (*S3) Kind

func (s *S3) Kind() string

func (*S3) PutFile

func (s *S3) PutFile(key string, r io.ReadCloser) error

func (*S3) Walk

func (s *S3) Walk(s3Path string, process func(r RemoteFile)) error

type S3Config

type S3Config struct {
	AccessKey               string `yaml:"access_key" envconfig:"S3_ACCESS_KEY"`
	SecretKey               string `yaml:"secret_key" envconfig:"S3_SECRET_KEY"`
	Bucket                  string `yaml:"bucket" envconfig:"S3_BUCKET"`
	Endpoint                string `yaml:"endpoint" envconfig:"S3_ENDPOINT"`
	Region                  string `yaml:"region" envconfig:"S3_REGION"`
	ACL                     string `yaml:"acl" envconfig:"S3_ACL"`
	ForcePathStyle          bool   `yaml:"force_path_style" envconfig:"S3_FORCE_PATH_STYLE"`
	Path                    string `yaml:"path" envconfig:"S3_PATH"`
	DisableSSL              bool   `yaml:"disable_ssl" envconfig:"S3_DISABLE_SSL"`
	PartSize                int64  `yaml:"part_size" envconfig:"S3_PART_SIZE"`
	CompressionLevel        int    `yaml:"compression_level" envconfig:"S3_COMPRESSION_LEVEL"`
	CompressionFormat       string `yaml:"compression_format" envconfig:"S3_COMPRESSION_FORMAT"`
	SSE                     string `yaml:"sse" envconfig:"S3_SSE"`
	DisableCertVerification bool   `yaml:"disable_cert_verification" envconfig:"S3_DISABLE_CERT_VERIFICATION"`
	MaxRetry                int    `yaml:"max_retry" envconfig:"S3_MAX_RETRY"`
}

S3Config - s3 settings section

type Table

type Table struct {
	Database string `db:"database"`
	Name     string `db:"name"`
	Skip     bool
}

Table - ClickHouse table struct

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL