file

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 24, 2023 License: MIT Imports: 19 Imported by: 0

Documentation

Index

Constants

View Source
const (
	//LocalFileStorage name of local file storage
	LocalFileStorage = "File"
	//FTPFileStorage name of ftp file storage
	FTPFileStorage = "FTP"
)
View Source
const (
	//TSV a file type that have tab-separated values as its content
	TSV = "tsv"
	//CSV a file type that have comma-separated values as its content
	CSV = "csv"
	//JSON a file type that have json string as its content
	JSON = "json"
)
View Source
const (
	OKFlag = "OK"
	MD5    = "MD5"
	SHA1   = "SHA1"
	SHA256 = "SHA256"
	SHA512 = "SHA512"
)

Variables

This section is empty.

Functions

func Count

func Count(fd FileObjectModel) (int64, error)

Count the default implementation to count number of items contained by the file

func Merge

func Merge(src []FileObjectModel, dest FileObjectModel) error

Merge the default implementation to merge multiple files into a simple one

func RegisterChecksumer

func RegisterChecksumer(key string, ch Checksumer)

func RegisterFileType

func RegisterFileType(ftype string, reader FileItemReader, writer FileItemWriter, mergeSplitter MergeSplitter)

RegisterFileType register file type

func Split

func Split(srcFd FileObjectModel, destFds []FileObjectModel, strategy FileSplitStrategy) error

Split the default implementation to split a file into multiple destinations

Types

type ChecksumFlusher

type ChecksumFlusher interface {
	//Checksum generate checksum and write a check file for the corresponding data file
	Checksum(fd FileObjectModel) error
}

ChecksumFlusher write a check file for the corresponding data file

type ChecksumVerifier

type ChecksumVerifier interface {
	//Verify verify a data file according to the corresponding check file
	Verify(fd FileObjectModel) (bool, error)
}

ChecksumVerifier verify a data file according to the corresponding check file

type Checksumer

type Checksumer interface {
	ChecksumVerifier
	ChecksumFlusher
}

Checksumer the combination of ChecksumVerifier and ChecksumFlusher

func GetChecksumer

func GetChecksumer(key string) Checksumer

GetChecksumer get Checksumer by type

type FTPFileInStream

type FTPFileInStream struct {
	// contains filtered or unexported fields
}

FTPFileInStream represents a ftp file input stream

func (*FTPFileInStream) Close

func (fis *FTPFileInStream) Close() (err error)

func (*FTPFileInStream) Read

func (fis *FTPFileInStream) Read(p []byte) (n int, err error)

type FTPFileOutStream

type FTPFileOutStream struct {
	// contains filtered or unexported fields
}

FTPFileOutStream represents a ftp file output stream

func (*FTPFileOutStream) Close

func (fis *FTPFileOutStream) Close() (err error)

func (*FTPFileOutStream) Write

func (fis *FTPFileOutStream) Write(p []byte) (n int, err error)

type FTPFileSystem

type FTPFileSystem struct {
	Hort        string
	Port        int
	User        string
	Password    string
	ConnTimeout time.Duration
}

FTPFileSystem a FileStorage implementation backed by ftp

func (*FTPFileSystem) Create

func (fs *FTPFileSystem) Create(fileName, encoding string) (writer io.WriteCloser, err error)

func (*FTPFileSystem) Exists

func (fs *FTPFileSystem) Exists(fileName string) (bool, error)

func (*FTPFileSystem) Name

func (fs *FTPFileSystem) Name() string

func (*FTPFileSystem) Open

func (fs *FTPFileSystem) Open(fileName, encoding string) (io.ReadCloser, error)

type FileItemReader

type FileItemReader interface {
	//Open open file of the FileObjectModel, returns a handle for later read
	Open(fd FileObjectModel) (handle interface{}, err error)
	//Close open the file handle
	Close(handle interface{}) error
	//ReadItem read an item from the file handle
	ReadItem(handle interface{}) (interface{}, error)
	//SkipTo skip to the pos'th record in the file
	SkipTo(handle interface{}, pos int64) error
	//Count return number of items contained by the file
	Count(fd FileObjectModel) (int64, error)
}

FileItemReader read object from file according a FileObjectModel

func GetFileItemReader

func GetFileItemReader(ftype string) FileItemReader

GetFileItemReader get FileItemReader by type

type FileItemWriter

type FileItemWriter interface {
	//Open open file of the FileObjectModel, returns a handle for later write
	Open(fd FileObjectModel) (handle interface{}, err error)
	//Close open the file handle
	Close(handle interface{}) error
	//WriteItem write an item to the file
	WriteItem(handle interface{}, data interface{}) error
}

FileItemWriter write object to file according to a FileObjectModel

func GetFileItemWriter

func GetFileItemWriter(ftype string) FileItemWriter

GetFileItemWriter get FileItemWriter by type

type FileMerger

type FileMerger interface {
	//Merge merge multiple files into a simple one
	Merge(src []FileObjectModel, dest FileObjectModel) (err error)
}

FileMerger merge multiple files into a simple one

type FileMove

type FileMove struct {
	FromFileName  string
	FromFileStore FileStorage
	ToFileName    string
	ToFileStore   FileStorage
}

FileMove represents source and destination of a file movement

type FileObjectModel

type FileObjectModel struct {
	FileStore      FileStorage
	FileName       string
	Type           string
	Encoding       string
	Header         bool
	FieldSeparator string
	Checksum       string
	ItemPrototype  interface{}
}

FileObjectModel represents mapping a file to/from a struct

func (FileObjectModel) ItemType

func (fd FileObjectModel) ItemType() (reflect.Type, error)

ItemType returns the reflect.Type of the item within a FileObjectModel

func (FileObjectModel) String

func (fd FileObjectModel) String() string

type FileSplitStrategy

type FileSplitStrategy interface {
	//DecideDestIndex decide the destination of every record, return the index of destination file in the dest array
	DecideDestIndex(line string, dest []FileObjectModel) int
}

FileSplitStrategy decide the destination of every record

type FileSplitter

type FileSplitter interface {
	//Split split a src file into multiple destinations, decide the destination of every record according to a FileSplitStrategy
	Split(src FileObjectModel, dest []FileObjectModel, strategy FileSplitStrategy) (err error)
}

FileSplitter split a src file into multiple dest ones

type FileStorage

type FileStorage interface {
	//Name return name of the file store
	Name() string
	//Exists check if a file with the specified fileName exists
	Exists(fileName string) (ok bool, err error)
	//Open return a reader of the specified fileName, transform encoding if necessary
	Open(fileName, encoding string) (reader io.ReadCloser, err error)
	//Create return a writer of the specified fileName, transform encoding if necessary
	Create(fileName, encoding string) (writer io.WriteCloser, err error)
}

FileStorage represents a file store media, from which we can read or write files

type LocalFileSystem

type LocalFileSystem struct {
}

LocalFileSystem a FileStorage implementation backed by local file system

func (*LocalFileSystem) Create

func (fs *LocalFileSystem) Create(fileName, encoding string) (io.WriteCloser, error)

func (*LocalFileSystem) Exists

func (fs *LocalFileSystem) Exists(fileName string) (bool, error)

func (*LocalFileSystem) Name

func (fs *LocalFileSystem) Name() string

func (*LocalFileSystem) Open

func (fs *LocalFileSystem) Open(fileName, encoding string) (io.ReadCloser, error)

type MD5Checksumer

type MD5Checksumer struct {
}

MD5Checksumer generate and verify a check file containing the md5 digest of the data file

func (*MD5Checksumer) Checksum

func (ch *MD5Checksumer) Checksum(fd FileObjectModel) error

func (*MD5Checksumer) Verify

func (ch *MD5Checksumer) Verify(fd FileObjectModel) (bool, error)

type MergeSplitter

type MergeSplitter interface {
	FileMerger
	FileSplitter
}

MergeSplitter the combination of FileMerger and FileSplitter

func GetFileMergeSplitter

func GetFileMergeSplitter(ftype string) MergeSplitter

GetFileMergeSplitter get MergeSplitter by type

type OKFlagChecksumer

type OKFlagChecksumer struct {
}

OKFlagChecksumer generate and verify an empty file with '.ok' suffix indicating the data file completed

func (*OKFlagChecksumer) Checksum

func (ch *OKFlagChecksumer) Checksum(fd FileObjectModel) error

func (*OKFlagChecksumer) Verify

func (ch *OKFlagChecksumer) Verify(fd FileObjectModel) (bool, error)

type SHA1Checksumer

type SHA1Checksumer struct {
}

SHA1Checksumer generate and verify a check file containing the sha-1 digest of the data file

func (*SHA1Checksumer) Checksum

func (ch *SHA1Checksumer) Checksum(fd FileObjectModel) error

func (*SHA1Checksumer) Verify

func (ch *SHA1Checksumer) Verify(fd FileObjectModel) (bool, error)

type SHA256Checksumer

type SHA256Checksumer struct {
}

SHA256Checksumer generate and verify a check file containing the sha-256 digest of the data file

func (*SHA256Checksumer) Checksum

func (ch *SHA256Checksumer) Checksum(fd FileObjectModel) error

func (*SHA256Checksumer) Verify

func (ch *SHA256Checksumer) Verify(fd FileObjectModel) (bool, error)

type SHA512Checksumer

type SHA512Checksumer struct {
}

SHA512Checksumer generate and verify a check file containing the sha-512 digest of the data file

func (*SHA512Checksumer) Checksum

func (ch *SHA512Checksumer) Checksum(fd FileObjectModel) error

func (*SHA512Checksumer) Verify

func (ch *SHA512Checksumer) Verify(fd FileObjectModel) (bool, error)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL