s3

package
v1.3.19 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 6, 2023 License: MIT Imports: 19 Imported by: 0

Documentation

Overview

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.

Index

Constants

View Source 
const (
	// AWS URL params
	QparamVersioning        = "versioning"
	QparamLifecycle         = "lifecycle"
	QparamCORS              = "cors"
	QparamPolicy            = "policy"
	QparamACL               = "acl"
	QparamMultiDelete       = "delete"
	QparamMaxKeys           = "max-keys"
	QparamPrefix            = "prefix"
	QparamContinuationToken = "continuation-token"
	QparamStartAfter        = "start-after"
	QparamDelimiter         = "delimiter"

	// multipart
	QparamMptUploads        = "uploads"
	QparamMptUploadID       = "uploadId"
	QparamMptPartNo         = "partNumber"
	QparamMptMaxUploads     = "max-uploads"
	QparamMptUploadIDMarker = "upload-id-marker"

	// Maximum number of parts per upload
	// https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
	MaxPartsPerUpload = 10000

	AISRegion = "ais"
	AISServer = "AIStore"
)

Variables

This section is empty.

Functions

func AddPart 

func AddPart(id string, npart *MptPart) (err error)

Add part to an active upload. Some clients may omit size and md5. Only partNum is must-have. md5 and fqn is filled by a target after successful saving the data to a workfile.

func FillMsgFromS3Query 

func FillMsgFromS3Query(query url.Values, msg *apc.LsoMsg)

func FinishUpload 

func FinishUpload(id, fqn string, aborted bool) (exists bool)

remove all temp files and delete from the map if completed (i.e., not aborted): store xattr

func Init 

func Init()

func InitUpload 

func InitUpload(id, bckName, objName string)

Start miltipart upload

func ObjName 

func ObjName(items []string) string

func ObjSize 

func ObjSize(id string) (size int64, err error)

Return a sum of upload part sizes. Used on upload completion to calculate the final size of the object.

func OffsetSorted 

func OffsetSorted(lom *cluster.LOM, partNum int64) (off, size int64, status int, err error)

func ParsePartNum 

func ParsePartNum(s string) (partNum int64, err error)

func SetETag 

func SetETag(header http.Header, lom *cluster.LOM)

func WriteErr 

func WriteErr(w http.ResponseWriter, r *http.Request, err error, errCode int)

Types

type BckOwner 

type BckOwner struct {
	ID   string `xml:"ID"`
	Name string `xml:"DisplayName"`
}

type Bucket 

type Bucket struct {
	Name    string `xml:"Name"`
	Created string `xml:"CreationDate"`
	String  string `xml:"String"`
}

type CommonPrefix 

type CommonPrefix struct {
	Prefix string `xml:"Prefix"`
}

NOTE: do not rename structs that have `xml` tags. The names of those structs become a top level tag of resulting XML, and those tags S3-compatible clients require.

type CompleteMptUpload 

type CompleteMptUpload struct {
	Parts []*PartInfo `xml:"Part"`
}

Multipart upload completion request

type CompleteMptUploadResult 

type CompleteMptUploadResult struct {
	Bucket string `xml:"Bucket"`
	Key    string `xml:"Key"`
	ETag   string `xml:"ETag"`
}

Multipart upload completion response

func (*CompleteMptUploadResult) MustMarshal 

func (r *CompleteMptUploadResult) MustMarshal(sgl *memsys.SGL)

type CopyObjectResult 

type CopyObjectResult struct {
	LastModified string `xml:"LastModified"` // e.g. <LastModified>2009-10-12T17:50:30.000Z</LastModified>
	ETag         string `xml:"ETag"`
}

Response for object copy request

func (*CopyObjectResult) MustMarshal 

func (r *CopyObjectResult) MustMarshal(sgl *memsys.SGL)

type Delete 

type Delete struct {
	Object []*DeleteObjectInfo `xml:"Object"`
	Quiet  bool                `xml:"Quiet"`
}

Multiple object delete request

type DeleteObjectInfo 

type DeleteObjectInfo struct {
	Key     string `xml:"Key"`
	Version string `xml:"Version"`
}

type DeleteResult 

type DeleteResult struct {
	Objs []DeletedObjInfo `xml:"Deleted"`
}

NOTE: do not rename structs that have `xml` tags. The names of those structs become a top level tag of resulting XML, and those tags S3-compatible clients require.

func (*DeleteResult) MustMarshal 

func (r *DeleteResult) MustMarshal(sgl *memsys.SGL)

type DeletedObjInfo 

type DeletedObjInfo struct {
	Key string `xml:"Key"`
}

Deleted result: list of deleted objects and errors

type Error 

type Error struct {
	Code      string
	Message   string
	Resource  string
	RequestID string `xml:"RequestId"`
}

type InitiateMptUploadResult 

type InitiateMptUploadResult struct {
	Bucket   string `xml:"Bucket"`
	Key      string `xml:"Key"`
	UploadID string `xml:"UploadId"`
}

Multipart upload start response

func (*InitiateMptUploadResult) MustMarshal 

func (r *InitiateMptUploadResult) MustMarshal(sgl *memsys.SGL)

type ListBucketResult 

type ListBucketResult struct {
	Ns      string    `xml:"xmlns,attr"`
	Owner   BckOwner  `xml:"Owner"`
	Buckets []*Bucket `xml:"Buckets>Bucket"`
}

List bucket response

func NewListBucketResult 

func NewListBucketResult() (r *ListBucketResult)

func (*ListBucketResult) Add 

func (r *ListBucketResult) Add(bck *meta.Bck)

func (*ListBucketResult) MustMarshal 

func (r *ListBucketResult) MustMarshal(sgl *memsys.SGL)

type ListMptUploadsResult 

type ListMptUploadsResult struct {
	Bucket         string             `xml:"Bucket"`
	UploadIDMarker string             `xml:"UploadIdMarker"`
	Uploads        []UploadInfoResult `xml:"Upload"`
	MaxUploads     int
	IsTruncated    bool
}

List of active multipart uploads response

func ListUploads 

func ListUploads(bckName, idMarker string, maxUploads int) (result *ListMptUploadsResult)

func (*ListMptUploadsResult) MustMarshal 

func (r *ListMptUploadsResult) MustMarshal(sgl *memsys.SGL)

type ListObjectResult 

type ListObjectResult struct {
	Name                  string          `xml:"Name"`
	Ns                    string          `xml:"xmlns,attr"`
	Prefix                string          `xml:"Prefix"`
	KeyCount              int             `xml:"KeyCount"`                 // number of object names in the response
	MaxKeys               int             `xml:"MaxKeys"`                  // "The maximum number of keys returned ..." (s3)
	IsTruncated           bool            `xml:"IsTruncated"`              // true if there are more pages to read
	ContinuationToken     string          `xml:"ContinuationToken"`        // original ContinuationToken
	NextContinuationToken string          `xml:"NextContinuationToken"`    // NextContinuationToken to read the next page
	Contents              []*ObjInfo      `xml:"Contents"`                 // list of objects
	CommonPrefixes        []*CommonPrefix `xml:"CommonPrefixes,omitempty"` // list of dirs (used with `apc.LsNoRecursion`)
}

List objects response

func NewListObjectResult 

func NewListObjectResult(bucket string) *ListObjectResult

func (*ListObjectResult) Add 

func (r *ListObjectResult) Add(entry *cmn.LsoEntry, lsmsg *apc.LsoMsg)

func (*ListObjectResult) FillFromAisBckList 

func (r *ListObjectResult) FillFromAisBckList(bckList *cmn.LsoResult, lsmsg *apc.LsoMsg)

func (*ListObjectResult) MustMarshal 

func (r *ListObjectResult) MustMarshal(sgl *memsys.SGL)

type ListPartsResult 

type ListPartsResult struct {
	Bucket   string      `xml:"Bucket"`
	Key      string      `xml:"Key"`
	UploadID string      `xml:"UploadId"`
	Parts    []*PartInfo `xml:"Part"`
}

Multipart uploaded parts response

func (*ListPartsResult) MustMarshal 

func (r *ListPartsResult) MustMarshal(sgl *memsys.SGL)

type MptPart 

type MptPart struct {
	MD5  string // MD5 of the part (*)
	FQN  string // FQN of the corresponding workfile
	Size int64  // part size in bytes (*)
	Num  int64  // part number (*)
}

NOTE: xattr stores only the (*) marked attributes

func CheckParts 

func CheckParts(id string, parts []*PartInfo) ([]*MptPart, error)

TODO: compare non-zero sizes (note: s3cmd sends 0) and part.ETag as well, if specified

type ObjInfo 

type ObjInfo struct {
	Key          string `xml:"Key"`
	LastModified string `xml:"LastModified"`
	ETag         string `xml:"ETag"`
	Size         int64  `xml:"Size"`
	Class        string `xml:"StorageClass"`
}

NOTE: do not rename structs that have `xml` tags. The names of those structs become a top level tag of resulting XML, and those tags S3-compatible clients require.

type PartInfo 

type PartInfo struct {
	ETag       string `xml:"ETag"`
	PartNumber int64  `xml:"PartNumber"`
	Size       int64  `xml:"Size,omitempty"`
}

Multipart uploaded part

func ListParts 

func ListParts(id string, lom *cluster.LOM) (parts []*PartInfo, err error, errCode int)

type UploadInfoResult 

type UploadInfoResult struct {
	Key       string    `xml:"Key"`
	UploadID  string    `xml:"UploadId"`
	Initiated time.Time `xml:"Initiated"`
}

Active upload info

type VersioningConfiguration 

type VersioningConfiguration struct {
	Status string `xml:"Status"`
}

Bucket versioning

func NewVersioningConfiguration 

func NewVersioningConfiguration(enabled bool) *VersioningConfiguration

func (*VersioningConfiguration) Enabled 

func (r *VersioningConfiguration) Enabled() bool

func (*VersioningConfiguration) MustMarshal 

func (r *VersioningConfiguration) MustMarshal(sgl *memsys.SGL)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.