filestore

package module
v0.1.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 27, 2021 License: MIT Imports: 17 Imported by: 17

README

filestore

Overview

filestore is a generic implementation for working with local files or cloud-based objects through a single interface. Currently, filestore supports AWS S3, with intentions of supporting Azure Blob Storage in the future.

Installation

go get github.com/USACE/filestore

Usage

package main

import "github.com/USACE/filestore"

func main() {

	config := filestore.S3FSConfig{
		S3Id:     "************",  // AWS_ACCESS_KEY_ID
		S3Key:    "************",  // AWS_SECRET_ACCESS_KEY
		S3Region: "us-east-1",
		S3Bucket: "my-bucket",
	}

	fs, err := filestore.NewFileStore(config)
	if err != nil {
		log.Fatal(err)
	}

	err = fs.Walk("path-prefix", walkFunc)
	if err != nil {
		log.Fatal(err)
	}

}

func walkFunc(path string, file os.FileInfo) error {
	fmt.Println(path)
	return nil
}

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type BlockFS 

type BlockFS struct{}

func (*BlockFS) CompleteObjectUpload 

func (b *BlockFS) CompleteObjectUpload(u CompletedObjectUploadConfig) error

func (*BlockFS) DeleteObjects 

func (b *BlockFS) DeleteObjects(path ...string) error

func (*BlockFS) GetDir 

func (b *BlockFS) GetDir(path string, recursive bool) (*[]FileStoreResultObject, error)

func (*BlockFS) GetObject 

func (b *BlockFS) GetObject(path string) (io.ReadCloser, error)

func (*BlockFS) InitializeObjectUpload 

func (b *BlockFS) InitializeObjectUpload(u UploadConfig) (UploadResult, error)

func (*BlockFS) PutObject 

func (b *BlockFS) PutObject(path string, data []byte) (*FileOperationOutput, error)

func (*BlockFS) Walk 

func (b *BlockFS) Walk(path string, vistorFunction FileVisitFunction) error

func (*BlockFS) WriteChunk 

func (b *BlockFS) WriteChunk(u UploadConfig) (UploadResult, error)

type BlockFSConfig 

type BlockFSConfig struct{}

@TODO this is kind of clunky. BlockFSConfig is only used in NewFileStore as a type case so we know to create a Block File Store as of now I don't actually need any config properties

type CompletedObjectUploadConfig 

type CompletedObjectUploadConfig struct {
	UploadId string
	//PathInfo       models.ModelPathInfo
	//DirPath        string
	//FilePath       string
	ObjectPath string
	//ObjectName     string
	ChunkUploadIds []string
}

type FileOperationOutput 

type FileOperationOutput struct {
	Md5 string
}

type FileStore 

type FileStore interface {
	GetDir(string, bool) (*[]FileStoreResultObject, error)
	GetObject(string) (io.ReadCloser, error)
	PutObject(string, []byte) (*FileOperationOutput, error)
	DeleteObjects(path ...string) error
	//PutMultipartObject(u UploadConfig) (UploadResult, error)
	//InitializeMultipartWrite
	//PutPart(u UploadConfig) (UploadResult, error)
	Walk(string, FileVisitFunction) error

	/////depricate
	InitializeObjectUpload(UploadConfig) (UploadResult, error)
	WriteChunk(UploadConfig) (UploadResult, error)
	CompleteObjectUpload(CompletedObjectUploadConfig) error
}

func NewFileStore 

func NewFileStore(config interface{}) (FileStore, error)

type FileStoreResultObject 

type FileStoreResultObject struct {
	ID         int       `json:"id"`
	Name       string    `json:"fileName"`
	Size       string    `json:"size"`
	Path       string    `json:"filePath"`
	Type       string    `json:"type"`
	IsDir      bool      `json:"isdir"`
	Modified   time.Time `json:"modified"`
	ModifiedBy string    `json:"modifiedBy"`
}

type FileVisitFunction 

type FileVisitFunction func(path string, file os.FileInfo) error

type PATHTYPE 

type PATHTYPE int
const (
	FILE PATHTYPE = iota
	FOLDER
)

type PathParts 

type PathParts struct {
	Parts []string
}

func (PathParts) ToFilePath 

func (pp PathParts) ToFilePath(additionalParts ...string) string

func (PathParts) ToPath 

func (pp PathParts) ToPath(additionalParts ...string) string

type S3FS 

type S3FS struct {
	// contains filtered or unexported fields
}

S3FS satisfies the FileStore interface, allowing for generic file operations to be done on s3 blobs

func (*S3FS) CompleteObjectUpload 

func (s3fs *S3FS) CompleteObjectUpload(u CompletedObjectUploadConfig) error

func (*S3FS) DeleteObjects 

func (s3fs *S3FS) DeleteObjects(path ...string) error

DeleteObjects will take one or more paths, and delete them from the s3 file system

func (*S3FS) GetDir 

func (s3fs *S3FS) GetDir(path string, recursive bool) (*[]FileStoreResultObject, error)

GetDir is similar to an ls unix call. It lists the objects at an s3 prefix, with the option of being recursive

func (*S3FS) GetObject 

func (s3fs *S3FS) GetObject(path string) (io.ReadCloser, error)

GetObject will return the body of an s3 object as a ReadCloser, meaning it has the basic Read and Close methods

func (*S3FS) InitializeObjectUpload 

func (s3fs *S3FS) InitializeObjectUpload(u UploadConfig) (UploadResult, error)

func (*S3FS) Ping 

func (s3fs *S3FS) Ping() error

Ping makes a cheap call to the s3 bucket to ensure connection

func (*S3FS) PutObject 

func (s3fs *S3FS) PutObject(path string, data []byte) (*FileOperationOutput, error)

PutObject will take the data provided and put it on s3 at the path provided

func (*S3FS) SetObjectPublic 

func (s3fs *S3FS) SetObjectPublic(path string) (string, error)

SetObjectPublic will change the acl permissions on an s3 object and make it publically readable

func (*S3FS) SharedAccessURL 

func (s3fs *S3FS) SharedAccessURL(path string, expiration time.Duration) (string, error)

SharedAccessURL will create a presigned url that can be used to access/download an object from an s3 bucket. It will only be valid for the duration specified

func (*S3FS) Walk 

func (s3fs *S3FS) Walk(path string, vistorFunction FileVisitFunction) error

Walk will traverse an s3 file system recursively, starting at the provided prefix, and apply the visitorFunction to each s3 object

func (*S3FS) WriteChunk 

func (s3fs *S3FS) WriteChunk(u UploadConfig) (UploadResult, error)

type S3FSConfig 

type S3FSConfig struct {
	S3Id     string
	S3Key    string
	S3Region string
	S3Bucket string
}

S3FSConfig stores the configuration and credentials necessary to create an s3 instance of the filestore

type S3FileInfo 

type S3FileInfo struct {
	// contains filtered or unexported fields
}

S3FileInfo is a wrapper around the s3.Object struct that implements the os.FileInfo interface

func (*S3FileInfo) IsDir 

func (obj *S3FileInfo) IsDir() bool

IsDir returns a boolean determining whether an object is a directory or not

func (*S3FileInfo) ModTime 

func (obj *S3FileInfo) ModTime() time.Time

ModTime returns the time the s3 object was last modified

func (*S3FileInfo) Mode 

func (obj *S3FileInfo) Mode() os.FileMode

Mode defaults to Irregular

func (*S3FileInfo) Name 

func (obj *S3FileInfo) Name() string

Name returns the file name of the s3 object

func (*S3FileInfo) Size 

func (obj *S3FileInfo) Size() int64

Size returns the file size in bytes

func (*S3FileInfo) Sys 

func (obj *S3FileInfo) Sys() interface{}

Sys defaults to nil for objects of s3

type UploadConfig 

type UploadConfig struct {
	//PathInfo   models.ModelPathInfo
	//DirPath    string
	//FilePath   string
	ObjectPath string
	//ObjectName string
	ChunkId int64
	//FileId     uuid.UUID
	UploadId string
	Data     []byte
}

type UploadResult 

type UploadResult struct {
	ID         string `json:"id"`
	WriteSize  int    `json:"size"`
	IsComplete bool   `json:"isComplete"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.