internal

package
v0.6.20 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 26, 2019 License: Apache-2.0, BSD-3-Clause, MIT Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	//None No status avaiable.
	None = iota
	//Started Indicates that the file was included in the transfer.
	Started
	//Completed Indicates that the file was transferred succesfully.
	Completed
	//Ignored Indicates that the file was ignored - e.g. empty files...
	Ignored
)
View Source 
const ProgramVersion = "0.6.20"

ProgramVersion blobporter version

Variables

View Source 
var EventSink = newEventSink()

EventSink singleton instance of the EventSink

View Source 
var HTTPClientTimeout = 90

HTTPClientTimeout HTTP client timeout when reading from HTTP sources and try timeout for blob storage operations.

View Source 
var UserAgentStr = fmt.Sprintf("%s/%s/%s", "BlobPorter", ProgramVersion, runtime.GOARCH)

UserAgentStr value included in the user-agent header when calling the blob API

Functions

This section is empty.

Types

type AzUtil 

type AzUtil struct {
	// contains filtered or unexported fields
}

AzUtil TODO

func NewAzUtil 

func NewAzUtil(accountName string, accountKey string, container string, baseBlobURL string) (*AzUtil, error)

NewAzUtil TODO

func (*AzUtil) CleanUncommittedBlocks 

func (p *AzUtil) CleanUncommittedBlocks(blobName string) error

CleanUncommittedBlocks TODO

func (*AzUtil) CreateContainerIfNotExists 

func (p *AzUtil) CreateContainerIfNotExists() (bool, error)

CreateContainerIfNotExists returs true if the container did not exist.

func (*AzUtil) CreatePageBlob 

func (p *AzUtil) CreatePageBlob(blobName string, size uint64) error

CreatePageBlob TODO

func (*AzUtil) DeleteContainerIfNotExists added in v0.6.20 

func (p *AzUtil) DeleteContainerIfNotExists() error

DeleteContainerIfNotExists TODO

func (*AzUtil) GetBlobURLWithReadOnlySASToken 

func (p *AzUtil) GetBlobURLWithReadOnlySASToken(blobName string, expTime time.Time) (*url.URL, error)

GetBlobURLWithReadOnlySASToken TODO

func (*AzUtil) IterateBlobList 

func (p *AzUtil) IterateBlobList(prefix string, callback BlobCallback) error

IterateBlobList TODO

func (*AzUtil) PutBlock 

func (p *AzUtil) PutBlock(container string, blobName string, id string, body io.ReadSeeker, md5 []byte) error

PutBlock TODO

func (*AzUtil) PutBlockBlob 

func (p *AzUtil) PutBlockBlob(blobName string, body io.ReadSeeker, md5 []byte) error

PutBlockBlob TODO

func (*AzUtil) PutBlockBlobFromURL added in v0.6.20 

func (p *AzUtil) PutBlockBlobFromURL(blobName string, id string, sourceURL string, offset int64, length int64) error

PutBlockBlobFromURL TODO

func (*AzUtil) PutBlockList 

func (p *AzUtil) PutBlockList(blobName string, blockIDs []string) error

PutBlockList TODO

func (*AzUtil) PutEmptyBlockBlob added in v0.6.14 

func (p *AzUtil) PutEmptyBlockBlob(blobName string) error

PutEmptyBlockBlob TODO

func (*AzUtil) PutPages 

func (p *AzUtil) PutPages(blobName string, start int64, end int64, body io.ReadSeeker, md5 []byte) error

PutPages TODO

type BlobCallback 

type BlobCallback func(*azblob.BlobItem, string) (bool, error)

BlobCallback TODO

type EventAction added in v0.6.20 

type EventAction int

EventAction used to determined how the event sink will handle the event 

const (
	//Discrete the event flows as distinct unit
	Discrete EventAction = iota
	//Sum the value of the event is aggregated overtime with a sum operation
	Sum
)

type EventData added in v0.6.20 

type EventData struct {
	Key   string
	Value interface{}
}

EventData key-value tuple with event data

type EventDelegate added in v0.6.20 

type EventDelegate func(e EventItem, a EventItemAggregate)

EventDelegate represents a delegate that handles the event

type EventItem added in v0.6.20 

type EventItem struct {
	Name     EventName
	Resource string
	Data     []EventData
	Source   EventSource
	Action   EventAction
}

EventItem represents an event created by either by a Source

type EventItemAggregate added in v0.6.20 

type EventItemAggregate struct {
	NumItems      int64
	LastEventItem EventItem
	Value         float64
}

EventItemAggregate represents an event that its value is aggregated overtime

type EventName added in v0.6.20 

type EventName string

EventName base type for the event name enuk

type EventSource added in v0.6.20 

type EventSource int

EventSource base type for the event source enum 

const (
	//Transfer the event is created by the Transfer instance
	Transfer EventSource = iota
	//Reader the event is created by a Reader
	Reader
	//Worker the event is created by a Worker
	Worker
	//CommitListHandler the event is created by the commit list handler
	CommitListHandler
)

type EventSubscription added in v0.6.20 

type EventSubscription struct {
	Delegate EventDelegate
	Source   EventSource
	Type     EventSubscriptionType
}

EventSubscription represents a subscription for events created by a given Source

type EventSubscriptionType added in v0.6.20 

type EventSubscriptionType int

EventSubscriptionType based type of the types of subscriptions 

const (
	//RealTime the delegate will be called every time the event is triggered
	RealTime EventSubscriptionType = iota
	//OnDone the delegate will be called only at the end of the transfer
	OnDone
)

type FileHandlePool 

type FileHandlePool struct {
	// contains filtered or unexported fields
}

FileHandlePool TODO

func NewFileHandlePool 

func NewFileHandlePool(maxHandlesPerFile int, mode HandleMode, overwrite bool) *FileHandlePool

NewFileHandlePool TODO

func (*FileHandlePool) CloseHandles 

func (f *FileHandlePool) CloseHandles(path string) error

CloseHandles TODO

func (*FileHandlePool) GetHandle 

func (f *FileHandlePool) GetHandle(path string) (*os.File, error)

GetHandle TODO

func (*FileHandlePool) ReturnHandle 

func (f *FileHandlePool) ReturnHandle(path string, handle *os.File) error

ReturnHandle TODO

type HandleMode 

type HandleMode int

HandleMode TODO 

const (
	//Read read only file handles
	Read HandleMode = iota
	//Write write and append file handles
	Write
)

type SourceFilter 

type SourceFilter interface {
	IsTransferredAndTrackIfNot(name string, size int64) (bool, error)
}

SourceFilter TODO

type TransferStatus 

type TransferStatus int

TransferStatus TODO

func (TransferStatus) String 

func (t TransferStatus) String() string

type TransferTracker 

type TransferTracker struct {
	// contains filtered or unexported fields
}

TransferTracker TODO

func NewTransferTracker 

func NewTransferTracker(logPath string) (*TransferTracker, error)

NewTransferTracker TODO

func (*TransferTracker) IsTransferredAndTrackIfNot 

func (t *TransferTracker) IsTransferredAndTrackIfNot(name string, size int64) (bool, error)

IsTransferredAndTrackIfNot returns true if the file was previously transferred. If not, track/log that the transferred was started.

func (*TransferTracker) TrackFileTransferComplete 

func (t *TransferTracker) TrackFileTransferComplete(name string) error

TrackFileTransferComplete TODO

func (*TransferTracker) TrackTransferComplete 

func (t *TransferTracker) TrackTransferComplete() error

TrackTransferComplete TODO

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.