objectstore

package module
v0.0.0-...-ec50bc7 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 1, 2024 License: MIT Imports: 22 Imported by: 0

README

objectstore

A prototype object store written in go using the S3 protocol using the bbolt object database as the backend.

It is loosely based on gofakes3 by johannesboyne but converted to use both of my kernel & rest frameworks, making it not just standalone but embedable within other projects. It also stores objects differently to return the correct object sizes when listing them and to reduce the memory footprint when listing large objects.

Full details is in the These are in the wiki

Supported features

  • Create bucket
  • Delete bucket
  • List buckets
  • Create object
  • List objects
  • Retrieve object
  • Delete object
  • Docker container
  • Event notification, currently supports RabbitMQ

Supported clients

These are now listed in the wiki but currently common command line tools like aws cli, minio mc & s3cmd are supported as is s3fs fuse filesystem

Similar notable projects

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type AccessControlPolicy

type AccessControlPolicy struct {
	XMLName     xml.Name `xml:"AccessControlPolicy"`
	Id          string   `xml:"Owner>ID"`
	DisplayName string   `xml:"Owner>DisplayName"`
	Grant       []Grant  `xml:"AccessControlList"`
}

type Bucket

type Bucket struct {
	XMLName  xml.Name   `xml:"ListBucketResult"`
	Xmlns    string     `xml:"xmlns,attr"`
	Name     string     `xml:"Name"`
	Prefix   string     `xml:"Prefix"`
	Marker   string     `xml:"Marker"`
	Contents []*Content `xml:"Contents"`
}

type BucketInfo

type BucketInfo struct {
	Name         string `xml:"Bucket>Name"`
	CreationDate string `xml:"Bucket>CreationDate"`
}

type CompleteMultipartUpload

type CompleteMultipartUpload struct {
	XMLName xml.Name              `xml:"CompleteMultipartUpload"`
	Parts   []MultipartUploadPart `xml:"Part"`
}

type CompleteMultipartUploadResult

type CompleteMultipartUploadResult struct {
	XMLName  xml.Name `xml:"CompleteMultipartUploadResult"`
	Location string
	Bucket   string
	Key      string
	ETag     string
}

type Content

type Content struct {
	Key          string `xml:"Key"`
	LastModified string `xml:"LastModified"`
	ETag         string `xml:"ETag"`
	Size         int    `xml:"Size"`
	StorageClass string `xml:"StorageClass"`
}

type CopyObjectResult

type CopyObjectResult struct {
	XMLName      xml.Name `xml:"CopyObjectResult"`
	LastModified time.Time
	ETag         string
}

type Grant

type Grant struct {
	XMLName     xml.Name `xml:"Grant"`
	Id          string   `xml:"Grantee>ID"`
	DisplayName string   `xml:"Grantee>DisplayName"`
	Permission  string   `xml:"Permission"`
}

type InitiateMultipartUploadResult

type InitiateMultipartUploadResult struct {
	XMLName  xml.Name `xml:"InitiateMultipartUploadResult"`
	Xmlns    string   `xml:"xmlns,attr"`
	Bucket   string   `xml:"Bucket"`
	Key      string   `xml:"Key"`
	UploadId string   `xml:"UploadId"`
}

type MultipartUpload

type MultipartUpload struct {
	// Final object name for this upload
	ObjectName string
	// Generated uploadId
	UploadId string
	// Index of uploaded parts
	Parts map[string]string
	// Time of when upload was initiated.
	// TODO this is for future use, we'll use this to cleanup incomplete uploads
	Time time.Time
	// Metadata
	Meta map[string]string
}

type MultipartUploadPart

type MultipartUploadPart struct {
	XMLName    xml.Name `xml:"Part"`
	PartNumber string   `xml:"PartNumber"`
	ETag       string   `xml:"ETag"`
}

type Object

type Object struct {
	// The true object name
	Name string
	// Metadata
	Metadata map[string]string
	// When last modified
	LastModified time.Time
	// Length
	Length int
	// ETag
	ETag string
	// The parts
	Parts []ObjectPart
}

The metadata for each stored object

type ObjectPart

type ObjectPart struct {
	// The part number
	PartNumber int
	// The start position in the object
	Start int
	// The length of this part
	Length int
}

type ObjectReader

type ObjectReader struct {
	// contains filtered or unexported fields
}

Reader used to read bytes from an object. Initially this will treat the object as one large object but we will support reading just what's requried to reduce the memory footprint with large objects later.

func (*ObjectReader) Close

func (r *ObjectReader) Close() error

func (*ObjectReader) Read

func (r *ObjectReader) Read(p []byte) (int, error)

type ObjectStore

type ObjectStore struct {
	// contains filtered or unexported fields
}

func (*ObjectStore) CreateBucket

func (s *ObjectStore) CreateBucket(r *rest.Rest) error

CreateBucket creates a new S3 bucket in the BoltDB storage.

func (*ObjectStore) CreateObjectBrowserUpload

func (s *ObjectStore) CreateObjectBrowserUpload(r *rest.Rest) error

CreateObjectBrowserUpload creates a new S3 object using a MultipartForm

func (*ObjectStore) DeleteBucket

func (s *ObjectStore) DeleteBucket(r *rest.Rest) error

DeleteBucket deletes a S3 bucket in the BoltDB storage.

func (*ObjectStore) DeleteObject

func (s *ObjectStore) DeleteObject(r *rest.Rest) error

DeleteObject deletes a S3 object from the bucket.

func (*ObjectStore) GetBucket

func (s *ObjectStore) GetBucket(r *rest.Rest) error

GetBucket lists the contents of a bucket.

func (*ObjectStore) GetBuckets

func (s *ObjectStore) GetBuckets(r *rest.Rest) error

GetBuckets returns a list of all Buckets

func (*ObjectStore) GetObject

func (s *ObjectStore) GetObject(r *rest.Rest) error

GetObject retrievs a bucket object.

func (*ObjectStore) HeadBucket

func (s *ObjectStore) HeadBucket(r *rest.Rest) error

HeadBucket checks whether a bucket exists.

func (*ObjectStore) HeadObject

func (s *ObjectStore) HeadObject(r *rest.Rest) error

HeadObject retrieves only meta information of an object and not the whole.

func (*ObjectStore) Init

func (s *ObjectStore) Init(k *kernel.Kernel) error

func (*ObjectStore) Name

func (s *ObjectStore) Name() string

func (*ObjectStore) PostInit

func (s *ObjectStore) PostInit() error

type Storage

type Storage struct {
	XMLName     xml.Name     `xml:"ListAllMyBucketsResult"`
	Xmlns       string       `xml:"xmlns,attr"`
	Id          string       `xml:"Owner>ID"`
	DisplayName string       `xml:"Owner>DisplayName"`
	Buckets     []BucketInfo `xml:"Buckets"`
}

Directories

Path Synopsis
tools

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL