storage

package
v1.4.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 30, 2019 License: Apache-2.0 Imports: 34 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	MAX_PART_SIZE   = 5 << 30 // 5GB
	MAX_PART_NUMBER = 10000
)
View Source 
const (
	CLUSTER_MAX_USED_SPACE_PERCENT = 85
	BIG_FILE_THRESHOLD             = 128 << 10 /* 128K */
)
View Source 
const (
	RECYCLE_QUEUE_SIZE = 100
	MAX_TRY_TIMES      = 3
)
View Source 
const (
	AES_BLOCK_SIZE               = 16
	ENCRYPTION_KEY_LENGTH        = 32 // key size for AES-"256"
	INITIALIZATION_VECTOR_LENGTH = 16 // block size of AES
)
View Source 
const (
	// only objects smaller than threshold are cached
	FILE_CACHE_THRESHOLD_SIZE = 4 << 20 // 4M
)

Variables

View Source 
var RecycleQueue chan objectToRecycle

Functions

This section is empty.

Types

type DataCache 

type DataCache interface {
	WriteFromCache(object *meta.Object, startOffset int64, length int64,
		out io.Writer, writeThrough func(io.Writer) error,
		onCacheMiss func(io.Writer) error) error
	GetAlignedReader(object *meta.Object, startOffset int64, length int64,
		readThrough func() (io.ReadCloser, error),
		onCacheMiss func(io.Writer) error) (io.ReadCloser, error)
	Remove(key string)
}

type ReadCloser 

type ReadCloser struct {
	// contains filtered or unexported fields
}

func (*ReadCloser) Close 

func (r *ReadCloser) Close() error

func (*ReadCloser) Read 

func (r *ReadCloser) Read(b []byte) (n int, err error)

type YigStorage 

type YigStorage struct {
	DataStorage map[string]backend.Cluster
	DataCache   DataCache
	MetaStorage *meta.Meta
	KMS         crypto.KMS
	Stopping    bool
	WaitGroup   *sync.WaitGroup
}

*YigStorage implements api.ObjectLayer

func New 

func New(metaCacheType int, enableDataCache bool) *YigStorage

func (*YigStorage) AbortMultipartUpload 

func (yig *YigStorage) AbortMultipartUpload(credential common.Credential,
	bucketName, objectName, uploadId string) error

func (*YigStorage) AppendObject 

func (yig *YigStorage) AppendObject(bucketName string, objectName string, credential common.Credential,
	offset uint64, size int64, data io.ReadCloser, metadata map[string]string, acl datatype.Acl,
	sseRequest datatype.SseRequest, storageClass types.StorageClass, objInfo *types.Object) (result datatype.AppendObjectResult, err error)

TODO: Append Support Encryption

func (*YigStorage) CompleteMultipartUpload 

func (yig *YigStorage) CompleteMultipartUpload(credential common.Credential, bucketName,
	objectName, uploadId string, uploadedParts []meta.CompletePart) (result datatype.CompleteMultipartResult,
	err error)

func (*YigStorage) CopyObject 

func (yig *YigStorage) CopyObject(targetObject *meta.Object, source io.Reader, credential common.Credential,
	sseRequest datatype.SseRequest) (result datatype.PutObjectResult, err error)

func (*YigStorage) CopyObjectPart 

func (yig *YigStorage) CopyObjectPart(bucketName, objectName, uploadId string, partId int,
	size int64, data io.Reader, credential common.Credential,
	sseRequest datatype.SseRequest) (result datatype.PutObjectResult, err error)

func (*YigStorage) DelBucketLifecycle 

func (yig *YigStorage) DelBucketLifecycle(bucketName string, credential common.Credential) error

func (*YigStorage) DeleteBucket 

func (yig *YigStorage) DeleteBucket(bucketName string, credential common.Credential) (err error)

func (*YigStorage) DeleteBucketCors 

func (yig *YigStorage) DeleteBucketCors(bucketName string, credential common.Credential) error

func (*YigStorage) DeleteBucketPolicy 

func (yig *YigStorage) DeleteBucketPolicy(credential common.Credential, bucketName string) error

func (*YigStorage) DeleteBucketWebsite 

func (yig *YigStorage) DeleteBucketWebsite(bucket *meta.Bucket) error

func (*YigStorage) DeleteObject 

func (yig *YigStorage) DeleteObject(bucketName string, objectName string, version string,
	credential common.Credential) (result datatype.DeleteObjectResult, err error)

When bucket versioning is Disabled/Enabled/Suspended, and request versionId is set/unset:

| | with versionId | without versionId | |-----------|------------------------------|--------------------------------------------------------| | Disabled | error | remove object | | Enabled | remove corresponding version | add a delete marker | | Suspended | remove corresponding version | remove null version object(if exists) and add a | | | | null version delete marker |

See http://docs.aws.amazon.com/AmazonS3/latest/dev/Versioning.html

func (*YigStorage) GetBucket 

func (yig *YigStorage) GetBucket(bucketName string) (*meta.Bucket, error)

For INTERNAL USE ONLY

func (*YigStorage) GetBucketAcl 

func (yig *YigStorage) GetBucketAcl(bucketName string, credential common.Credential) (
	policy datatype.AccessControlPolicyResponse, err error)

func (*YigStorage) GetBucketCors 

func (yig *YigStorage) GetBucketCors(bucketName string,
	credential common.Credential) (cors datatype.Cors, err error)

func (*YigStorage) GetBucketInfo 

func (yig *YigStorage) GetBucketInfo(bucketName string,
	credential common.Credential) (bucket *meta.Bucket, err error)

func (*YigStorage) GetBucketInfoByCtx 

func (yig *YigStorage) GetBucketInfoByCtx(ctx api.RequestContext,
	credential common.Credential) (bucket *meta.Bucket, err error)

func (*YigStorage) GetBucketLifecycle 

func (yig *YigStorage) GetBucketLifecycle(bucketName string, credential common.Credential) (lc datatype.Lifecycle,
	err error)

func (*YigStorage) GetBucketPolicy 

func (yig *YigStorage) GetBucketPolicy(credential common.Credential, bucketName string) (bucketPolicy policy.Policy, err error)

func (*YigStorage) GetBucketVersioning 

func (yig *YigStorage) GetBucketVersioning(bucketName string, credential common.Credential) (
	versioning datatype.Versioning, err error)

func (*YigStorage) GetBucketWebsite 

func (yig *YigStorage) GetBucketWebsite(bucketName string) (config datatype.WebsiteConfiguration, err error)

func (*YigStorage) GetClusterByFsName 

func (yig *YigStorage) GetClusterByFsName(fsName string) (cluster backend.Cluster, err error)

func (*YigStorage) GetObject 

func (yig *YigStorage) GetObject(object *meta.Object, startOffset int64,
	length int64, writer io.Writer, sseRequest datatype.SseRequest) (err error)

func (*YigStorage) GetObjectAcl 

func (yig *YigStorage) GetObjectAcl(bucketName string, objectName string,
	version string, credential common.Credential) (policy datatype.AccessControlPolicyResponse, err error)

func (*YigStorage) GetObjectInfo 

func (yig *YigStorage) GetObjectInfo(bucketName string, objectName string,
	version string, credential common.Credential) (object *meta.Object, err error)

func (*YigStorage) GetObjectInfoByCtx 

func (yig *YigStorage) GetObjectInfoByCtx(ctx api.RequestContext,
	version string, credential common.Credential) (object *meta.Object, err error)

func (*YigStorage) ListBuckets 

func (yig *YigStorage) ListBuckets(credential common.Credential) (buckets []meta.Bucket, err error)

func (*YigStorage) ListMultipartUploads 

func (yig *YigStorage) ListMultipartUploads(credential common.Credential, bucketName string,
	request datatype.ListUploadsRequest) (result datatype.ListMultipartUploadsResponse, err error)

func (*YigStorage) ListObjectParts 

func (yig *YigStorage) ListObjectParts(credential common.Credential, bucketName, objectName string,
	request datatype.ListPartsRequest) (result datatype.ListPartsResponse, err error)

func (*YigStorage) ListObjects 

func (yig *YigStorage) ListObjects(credential common.Credential, bucketName string,
	request datatype.ListObjectsRequest) (result meta.ListObjectsInfo, err error)

func (*YigStorage) ListObjectsInternal 

func (yig *YigStorage) ListObjectsInternal(bucketName string,
	request datatype.ListObjectsRequest) (retObjects []*meta.Object, prefixes []string, truncated bool,
	nextMarker, nextVerIdMarker string, err error)

func (*YigStorage) ListVersionedObjects 

func (yig *YigStorage) ListVersionedObjects(credential common.Credential, bucketName string,
	request datatype.ListObjectsRequest) (result meta.VersionedListObjectsInfo, err error)

TODO: refactor, similar to ListObjects or not?

func (*YigStorage) MakeBucket 

func (yig *YigStorage) MakeBucket(bucketName string, acl datatype.Acl,
	credential common.Credential) error

func (*YigStorage) NewMultipartUpload 

func (yig *YigStorage) NewMultipartUpload(credential common.Credential, bucketName, objectName string,
	metadata map[string]string, acl datatype.Acl,
	sseRequest datatype.SseRequest, storageClass meta.StorageClass) (uploadId string, err error)

func (*YigStorage) PingCache 

func (y *YigStorage) PingCache(interval time.Duration)

check cache health per one second if enable cache

func (*YigStorage) PutObject 

func (yig *YigStorage) PutObject(bucketName string, objectName string, credential common.Credential,
	size int64, data io.ReadCloser, metadata map[string]string, acl datatype.Acl,
	sseRequest datatype.SseRequest, storageClass meta.StorageClass) (result datatype.PutObjectResult, err error)

Write path: 

+-----------+

PUT object/part | | Ceph 

+---------+------------+----------+ Encryptor +----->
          |            |          |           |
          |            |          +-----------+
          v            v
         SHA256      MD5(ETag)

SHA256 is calculated only for v4 signed authentication Encryptor is enabled when user set SSE headers

func (*YigStorage) PutObjectMeta 

func (yig *YigStorage) PutObjectMeta(bucket *meta.Bucket, targetObject *meta.Object, credential common.Credential) (err error)

func (*YigStorage) PutObjectPart 

func (yig *YigStorage) PutObjectPart(bucketName, objectName string, credential common.Credential,
	uploadId string, partId int, size int64, data io.ReadCloser, md5Hex string,
	sseRequest datatype.SseRequest) (result datatype.PutObjectPartResult, err error)

func (*YigStorage) RenameObject 

func (yig *YigStorage) RenameObject(targetObject *meta.Object, sourceObject string, credential common.Credential) (result datatype.RenameObjectResult, err error)

func (*YigStorage) SetBucketAcl 

func (yig *YigStorage) SetBucketAcl(bucketName string, policy datatype.AccessControlPolicy, acl datatype.Acl,
	credential common.Credential) error

func (*YigStorage) SetBucketCors 

func (yig *YigStorage) SetBucketCors(bucketName string, cors datatype.Cors,
	credential common.Credential) error

func (*YigStorage) SetBucketLifecycle 

func (yig *YigStorage) SetBucketLifecycle(bucketName string, lc datatype.Lifecycle,
	credential common.Credential) error

func (*YigStorage) SetBucketPolicy 

func (yig *YigStorage) SetBucketPolicy(credential common.Credential, bucketName string, bucketPolicy policy.Policy) (err error)

func (*YigStorage) SetBucketVersioning 

func (yig *YigStorage) SetBucketVersioning(bucketName string, versioning datatype.Versioning,
	credential common.Credential) error

func (*YigStorage) SetBucketWebsite 

func (yig *YigStorage) SetBucketWebsite(bucket *meta.Bucket, config datatype.WebsiteConfiguration) (err error)

func (*YigStorage) SetObjectAcl 

func (yig *YigStorage) SetObjectAcl(bucketName string, objectName string, version string,
	policy datatype.AccessControlPolicy, acl datatype.Acl, credential common.Credential) error

func (*YigStorage) Stop 

func (y *YigStorage) Stop()

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.