s3

package

v1.6.2 Latest Latest Go to latest Published: Jan 5, 2024 License: Apache-2.0 Imports: 17 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/kubemq-io/kubemq-sources

Links

Open Source Insights

README ¶

Kubemq AWS S3 Source Connector

Kubemq AWS S3 source connector allows services using kubemq server to sync aws s3 objects to remote location.

Prerequisites

The following are required to run the aws s3 source connector:

kubemq cluster
kubemq-sources deployment

Configuration

S3 source connector configuration properties:

Properties Key	Required	Description	Example
aws_key	yes	aws key	aws key supplied by aws
aws_secret_key	yes	aws secret key	aws secret key supplied by aws
region	yes	region	aws region
token	no	aws token ("default" empty string	aws token
folders	yes	set list of folders to watch	"/"
target_type	yes	set remote target sync type	"aws", "gcp", "minio", "filesystem","hdfs","azure","pass-through"
bucket_name	yes	set source bucket	"bucket"
concurrency	no	set sending concurrency	"1"
scan_interval	no	set bucket scan interval in sec	"5"

Example:

bindings:
- name: s3
  source:
    kind: aws.s3
    properties:
      aws_key: "id"
      aws_secret_key: 'json'
      region:  "region"
      token: ""
      folders: 'folder1,folder2/sub1/sub2'
      target_type: filesystem
      bucket_name: bucket
      concurrency: 1
      scan_interval: 5
  target:
    kind: kubemq.queue
    properties:
      address: localhost:50000
      channel: queue.s3
  properties: {}

Documentation ¶

Index ¶

func Connector() *common.Connector
type Client
- func New() *Client
type SourceFile
- func NewSourceFile(c *s3.S3, downloader *s3manager.Downloader, bucket string, obj *s3.Object) *SourceFile
type TargetsMetadata
- func NewTargetMetadata() TargetsMetadata
- func (m TargetsMetadata) Set(key, value string) TargetsMetadata
- func (m TargetsMetadata) String() string
type TargetsRequest
- func NewTargetsRequest() *TargetsRequest
- func ParseRequest(body []byte) (*TargetsRequest, error)

Constants ¶

This section is empty.

Variables ¶

This section is empty.

Functions ¶

func Connector ¶

func Connector() *common.Connector

Types ¶

type Client ¶

type Client struct {
	// contains filtered or unexported fields
}

func New ¶

func New() *Client

func (*Client) Connector ¶

func (c *Client) Connector() *common.Connector

func (*Client) Init ¶

func (c *Client) Init(ctx context.Context, cfg config.Spec, log *logger.Logger) error

func (*Client) Start ¶

func (c *Client) Start(ctx context.Context, target middleware.Middleware) error

func (*Client) Stop ¶

func (c *Client) Stop() error

type SourceFile ¶

type SourceFile struct {
	Object *s3.Object
	Bucket string
	// contains filtered or unexported fields
}

func NewSourceFile ¶

func NewSourceFile(c *s3.S3, downloader *s3manager.Downloader, bucket string, obj *s3.Object) *SourceFile

func (*SourceFile) Delete ¶

func (s *SourceFile) Delete(ctx context.Context) error

func (*SourceFile) Do ¶

func (s *SourceFile) Do(ctx context.Context) error

func (*SourceFile) FileDir ¶

func (s *SourceFile) FileDir() string

func (*SourceFile) FileName ¶

func (s *SourceFile) FileName() string

func (*SourceFile) FullPath ¶

func (s *SourceFile) FullPath() string

func (*SourceFile) Hash ¶

func (s *SourceFile) Hash() string

func (*SourceFile) Load ¶

func (s *SourceFile) Load(ctx context.Context) ([]byte, error)

func (*SourceFile) Metadata ¶

func (s *SourceFile) Metadata() string

func (*SourceFile) Request ¶

func (s *SourceFile) Request(ctx context.Context, bucketType string, bucketName string) (*types.Request, error)

func (*SourceFile) RootDir ¶

func (s *SourceFile) RootDir() string

type TargetsMetadata ¶

type TargetsMetadata map[string]string

func NewTargetMetadata ¶

func NewTargetMetadata() TargetsMetadata

func (TargetsMetadata) Set ¶

func (m TargetsMetadata) Set(key, value string) TargetsMetadata

func (TargetsMetadata) String ¶

func (m TargetsMetadata) String() string

type TargetsRequest ¶

type TargetsRequest struct {
	Metadata TargetsMetadata `json:"metadata,omitempty"`
	Data     []byte          `json:"data,omitempty"`
}

func NewTargetsRequest ¶

func NewTargetsRequest() *TargetsRequest

func ParseRequest ¶

func ParseRequest(body []byte) (*TargetsRequest, error)

func (*TargetsRequest) MarshalBinary ¶

func (r *TargetsRequest) MarshalBinary() []byte

func (*TargetsRequest) SetData ¶

func (r *TargetsRequest) SetData(value []byte) *TargetsRequest

func (*TargetsRequest) SetMetadata ¶

func (r *TargetsRequest) SetMetadata(value TargetsMetadata) *TargetsRequest

func (*TargetsRequest) SetMetadataKeyValue ¶

func (r *TargetsRequest) SetMetadataKeyValue(key, value string) *TargetsRequest

func (*TargetsRequest) Size ¶

func (r *TargetsRequest) Size() float64

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL