s3

package
v1.3.22 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 25, 2024 License: MIT Imports: 21 Imported by: 0

Documentation

Overview

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.

Index

Constants

View Source
const (
	// AWS URL params
	QparamVersioning        = "versioning"
	QparamLifecycle         = "lifecycle"
	QparamCORS              = "cors"
	QparamPolicy            = "policy"
	QparamACL               = "acl"
	QparamMultiDelete       = "delete"
	QparamMaxKeys           = "max-keys"
	QparamPrefix            = "prefix"
	QparamContinuationToken = "continuation-token"
	QparamStartAfter        = "start-after"
	QparamDelimiter         = "delimiter"

	// multipart
	QparamMptUploads        = "uploads"
	QparamMptUploadID       = "uploadId"
	QparamMptPartNo         = "partNumber"
	QparamMptMaxUploads     = "max-uploads"
	QparamMptUploadIDMarker = "upload-id-marker"

	QparamAccessKeyID = "AWSAccessKeyId"
	QparamExpires     = "Expires"
	QparamSignature   = "Signature"
	QparamXID         = "x-id"

	HeaderAlgorithm     = "X-Amz-Algorithm"
	HeaderCredentials   = "X-Amz-Credential" //nolint:gosec // This is just a header name definition...
	HeaderDate          = "X-Amz-Date"
	HeaderExpires       = "X-Amz-Expires"
	HeaderSignedHeaders = "X-Amz-SignedHeaders"
	HeaderSignature     = "X-Amz-Signature"

	// Maximum number of parts per upload
	// https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
	MaxPartsPerUpload = 10000

	AISRegion = "ais"
	AISServer = "AIStore"
)

Variables

This section is empty.

Functions

func AddPart

func AddPart(id string, npart *MptPart) (err error)

Add part to an active upload. Some clients may omit size and md5. Only partNum is must-have. md5 and fqn is filled by a target after successful saving the data to a workfile.

func CleanupUpload added in v1.3.22

func CleanupUpload(id, fqn string, aborted bool) (exists bool)

remove all temp files and delete from the map if completed (i.e., not aborted): store xattr

func FillLsoMsg added in v1.3.22

func FillLsoMsg(query url.Values, msg *apc.LsoMsg)

func Init

func Init()

func InitUpload

func InitUpload(id, bckName, objName string)

Start miltipart upload

func ObjName

func ObjName(items []string) string

func ObjSize

func ObjSize(id string) (size int64, err error)

Return a sum of upload part sizes. Used on upload completion to calculate the final size of the object.

func OffsetSorted

func OffsetSorted(lom *core.LOM, partNum int32) (off, size int64, status int, err error)

func ParsePartNum

func ParsePartNum(s string) (int32, error)

func SetEtag added in v1.3.22

func SetEtag(hdr http.Header, lom *core.LOM)

func WriteErr

func WriteErr(w http.ResponseWriter, r *http.Request, err error, errCode int)

func WriteMptErr added in v1.3.22

func WriteMptErr(w http.ResponseWriter, r *http.Request, err error, errCode int, lom *core.LOM, uploadID string)

with user-friendly tip

Types

type BckOwner

type BckOwner struct {
	ID   string `xml:"ID"`
	Name string `xml:"DisplayName"`
}

type Bucket

type Bucket struct {
	Name    string `xml:"Name"`
	Created string `xml:"CreationDate"`
	String  string `xml:"String"`
}

type CommonPrefix added in v1.3.16

type CommonPrefix struct {
	Prefix string `xml:"Prefix"`
}

NOTE: do not rename structs that have `xml` tags. The names of those structs become a top level tag of resulting XML, and those tags S3-compatible clients require.

type CompleteMptUpload

type CompleteMptUpload struct {
	Parts []*PartInfo `xml:"Part"`
}

Multipart upload completion request

type CompleteMptUploadResult

type CompleteMptUploadResult struct {
	Bucket string `xml:"Bucket"`
	Key    string `xml:"Key"`
	ETag   string `xml:"ETag"`
}

Multipart upload completion response

func (*CompleteMptUploadResult) MustMarshal

func (r *CompleteMptUploadResult) MustMarshal(sgl *memsys.SGL)

type CopyObjectResult

type CopyObjectResult struct {
	LastModified string `xml:"LastModified"` // e.g. <LastModified>2009-10-12T17:50:30.000Z</LastModified>
	ETag         string `xml:"ETag"`
}

Response for object copy request

func (*CopyObjectResult) MustMarshal

func (r *CopyObjectResult) MustMarshal(sgl *memsys.SGL)

type Delete

type Delete struct {
	Object []*DeleteObjectInfo `xml:"Object"`
	Quiet  bool                `xml:"Quiet"`
}

Multiple object delete request

type DeleteObjectInfo

type DeleteObjectInfo struct {
	Key     string `xml:"Key"`
	Version string `xml:"Version"`
}

type DeleteResult added in v1.3.16

type DeleteResult struct {
	Objs []DeletedObjInfo `xml:"Deleted"`
}

NOTE: do not rename structs that have `xml` tags. The names of those structs become a top level tag of resulting XML, and those tags S3-compatible clients require.

func (*DeleteResult) MustMarshal added in v1.3.16

func (r *DeleteResult) MustMarshal(sgl *memsys.SGL)

type DeletedObjInfo added in v1.3.16

type DeletedObjInfo struct {
	Key string `xml:"Key"`
}

Deleted result: list of deleted objects and errors

type Error

type Error struct {
	Code      string
	Message   string
	Resource  string
	RequestID string `xml:"RequestId"`
}

type InitiateMptUploadResult

type InitiateMptUploadResult struct {
	Bucket   string `xml:"Bucket"`
	Key      string `xml:"Key"`
	UploadID string `xml:"UploadId"`
}

Multipart upload start response

func (*InitiateMptUploadResult) MustMarshal

func (r *InitiateMptUploadResult) MustMarshal(sgl *memsys.SGL)

type ListBucketResult

type ListBucketResult struct {
	Ns      string    `xml:"xmlns,attr"`
	Owner   BckOwner  `xml:"Owner"`
	Buckets []*Bucket `xml:"Buckets>Bucket"`
}

List bucket response

func NewListBucketResult

func NewListBucketResult() (r *ListBucketResult)

func (*ListBucketResult) Add

func (r *ListBucketResult) Add(bck *meta.Bck)

func (*ListBucketResult) MustMarshal

func (r *ListBucketResult) MustMarshal(sgl *memsys.SGL)

type ListMptUploadsResult

type ListMptUploadsResult struct {
	Bucket         string             `xml:"Bucket"`
	UploadIDMarker string             `xml:"UploadIdMarker"`
	Uploads        []UploadInfoResult `xml:"Upload"`
	MaxUploads     int
	IsTruncated    bool
}

List of active multipart uploads response

func ListUploads

func ListUploads(bckName, idMarker string, maxUploads int) (result *ListMptUploadsResult)

func (*ListMptUploadsResult) MustMarshal

func (r *ListMptUploadsResult) MustMarshal(sgl *memsys.SGL)

type ListObjectResult

type ListObjectResult struct {
	Name                  string          `xml:"Name"`
	Ns                    string          `xml:"xmlns,attr"`
	Prefix                string          `xml:"Prefix"`
	KeyCount              int             `xml:"KeyCount"`                 // number of object names in the response
	MaxKeys               int             `xml:"MaxKeys"`                  // "The maximum number of keys returned ..." (s3)
	IsTruncated           bool            `xml:"IsTruncated"`              // true if there are more pages to read
	ContinuationToken     string          `xml:"ContinuationToken"`        // original ContinuationToken
	NextContinuationToken string          `xml:"NextContinuationToken"`    // NextContinuationToken to read the next page
	Contents              []*ObjInfo      `xml:"Contents"`                 // list of objects
	CommonPrefixes        []*CommonPrefix `xml:"CommonPrefixes,omitempty"` // list of dirs (used with `apc.LsNoRecursion`)
}

List objects response

func NewListObjectResult

func NewListObjectResult(bucket string) *ListObjectResult

func (*ListObjectResult) Add

func (r *ListObjectResult) Add(entry *cmn.LsoEntry, lsmsg *apc.LsoMsg)

func (*ListObjectResult) FromLsoResult added in v1.3.22

func (r *ListObjectResult) FromLsoResult(lst *cmn.LsoResult, lsmsg *apc.LsoMsg)

func (*ListObjectResult) MustMarshal

func (r *ListObjectResult) MustMarshal(sgl *memsys.SGL)

type ListPartsResult

type ListPartsResult struct {
	Bucket   string      `xml:"Bucket"`
	Key      string      `xml:"Key"`
	UploadID string      `xml:"UploadId"`
	Parts    []*PartInfo `xml:"Part"`
}

Multipart uploaded parts response

func (*ListPartsResult) MustMarshal

func (r *ListPartsResult) MustMarshal(sgl *memsys.SGL)

type MptPart

type MptPart struct {
	MD5  string // MD5 of the part (*)
	FQN  string // FQN of the corresponding workfile
	Size int64  // part size in bytes (*)
	Num  int32  // part number (*)
}

NOTE: xattr stores only the (*) marked attributes

func CheckParts

func CheckParts(id string, parts []*PartInfo) ([]*MptPart, error)

TODO: compare non-zero sizes (note: s3cmd sends 0) and part.ETag as well, if specified

type ObjInfo

type ObjInfo struct {
	Key          string `xml:"Key"`
	LastModified string `xml:"LastModified"`
	ETag         string `xml:"ETag"`
	Size         int64  `xml:"Size"`
	Class        string `xml:"StorageClass"`
}

NOTE: do not rename structs that have `xml` tags. The names of those structs become a top level tag of resulting XML, and those tags S3-compatible clients require.

type PartInfo

type PartInfo struct {
	ETag       string `xml:"ETag"`
	PartNumber int32  `xml:"PartNumber"`
	Size       int64  `xml:"Size,omitempty"`
}

Multipart uploaded part

func ListParts

func ListParts(id string, lom *core.LOM) (parts []*PartInfo, errCode int, err error)

type PassThroughSignedReq added in v1.3.22

type PassThroughSignedReq struct {
	// contains filtered or unexported fields
}

func NewPassThroughSignedReq added in v1.3.22

func NewPassThroughSignedReq(c *http.Client, oreq *http.Request, lom *core.LOM, body io.ReadCloser, q url.Values) *PassThroughSignedReq

func (*PassThroughSignedReq) Do added in v1.3.22

type PassThroughSignedResp added in v1.3.22

type PassThroughSignedResp struct {
	Body       []byte
	Header     http.Header
	StatusCode int
}

func (*PassThroughSignedResp) ObjAttrs added in v1.3.22

func (resp *PassThroughSignedResp) ObjAttrs() (oa *cmn.ObjAttrs)

(compare w/ cmn/objattrs FromHeader)

type UploadInfoResult

type UploadInfoResult struct {
	Key       string    `xml:"Key"`
	UploadID  string    `xml:"UploadId"`
	Initiated time.Time `xml:"Initiated"`
}

Active upload info

type VersioningConfiguration

type VersioningConfiguration struct {
	Status string `xml:"Status"`
}

Bucket versioning

func NewVersioningConfiguration

func NewVersioningConfiguration(enabled bool) *VersioningConfiguration

func (*VersioningConfiguration) Enabled

func (r *VersioningConfiguration) Enabled() bool

func (*VersioningConfiguration) MustMarshal

func (r *VersioningConfiguration) MustMarshal(sgl *memsys.SGL)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL