stores

type CheckpointStore struct {
	// contains filtered or unexported fields
}

type CheckpointStoreConfig struct {
	// PubSub is used:
	//   - store ingress checkpoints, and
	//   - distribute the checkpoints to all running instances.
	PubSub core.Factory `required:"true"`

	// Unique name that identifies the local region/data center/cloud.
	//
	// Field value is required.
	LocalRegion string `required:"true"`

	// Kafka topic name for ingress checkpoints.
	//
	// The topic cleanup policy needs to be set to 'compacted' to retain
	// only the last checkpoint for each <region, topic, partition> tuple.
	// This avoids unecessary disk space consumption and improved startup time.
	//
	// Field value is required.
	Topic string `required:"true"`
}

type ConsistentSegmentStore struct {
	// contains filtered or unexported fields
}

type ConsistentSegmentStoreConfig struct {
	// PubSub is used to:
	//   - distribute the incoming events to all running instances,
	//   - temporarily store events for group rebalance scenarios, and
	//   - to address problems that can arise due to AWS S3 eventual consistency model.
	PubSub core.Factory `required:"true"`

	// SegmentStore is the underlying segment store implementation.
	SegmentStore core.Factory `required:"true"`

	// SegmentEventSource is the source of segment events for underlying segment store.
	SegmentEventSource core.Factory `required:"true"`

	// Kafka topic name for storing segment events.
	//
	// The topic cleanup policy needs to be set to 'delete' with appropriate
	// retention time set to discard old segment events.
	//
	// Field value is required.
	Topic string `required:"true"`

	// The duration each segment event is tracked.
	//
	// Should match the configured Kafka topic retention time.
	//
	// Field value is required.
	EventsRetention time.Duration `min:"10m"`

	// The interval when background expiration task is running to remove old event from memory.
	//
	// Default value is set via DefaultEventsRetentionCheckInterval variable.
	EventsRetentionCheckInterval time.Duration `min:"1ms" max:"10m"`

	// Size of events buffered channel.
	//
	// Default value is set via DefaultSegmentEventsChanSize variable.
	EventsChanSize int `min:"1"`
}

type S3SegmentStore struct {
	// contains filtered or unexported fields
}

type S3SegmentStoreConfig struct {
	// The segment file format used to read and write segments.
	SegmentFormat core.Factory `required:"true"`

	// The AWS config object used to create the AWS S3 client.
	//
	// Field value is required.
	AWSConfig *aws.Config `required:"true"`

	// The AWS session object used to create the AWS S3 client.
	//
	// Field value is required.
	AWSSession *session.Session `required:"true"`

	// Bucket name to store segments.
	//
	// Field value is required.
	Bucket string `required:"true"`

	// Key prefix for written segment objects.
	//
	// Default value is set via DefaultS3KeyPrefix variable.
	KeyPrefix string

	// Key prefix for temporary segment objects.
	//
	// Default value is set via DefaultS3TempKeyPrefix variable.
	TempKeyPrefix string

	// Breaker enables tracking AWS S3 client error rate.
	//
	// Default value is set via DefaultS3Breaker variable.
	Breaker core.Breaker
}

type SQSSegmentEventSource struct {
	// contains filtered or unexported fields
}

type SQSSegmentEventSourceConfig struct {
	// The AWS config object used to create the AWS SQS client.
	//
	// Field value is required.
	AWSConfig *aws.Config `required:"true"`

	// The AWS session object used to create the AWS SQS client.
	//
	// Field value is required.
	AWSSession *session.Session `required:"true"`

	// AWS SQS queue name where AWS S3 notification events are published.
	//
	// The implementation expects that both Created and Removed event types to be enabled
	// for keys storing segments (i.e. the keys with DataKeyPrefix).
	//
	// Check the AWS S3 documentation for instructions on how to enable event notifications:
	// https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html
	//
	// Field value is required.
	QueueName string `required:"true"`

	// Key prefix for segment notification events.
	//
	// Default value is set via DefaultS3KeyPrefix variable.
	S3KeyPrefix string

	// The maximum number of messages to return in each receive messages call.
	//
	// Default value is set via DefaultSQSReceiveMaxNumberOfMessages variable.
	ReceiveMaxNumberOfMessages int `min:"1" max:"10"`

	// The duration for which the receive call waits for a message to arrive in the queue before returning.
	//
	// Minimum value must be greater than zero to enable the use of long polling.
	// Default value is set via DefaultSQSReceiveWaitTime variable.
	ReceiveWaitTime time.Duration `min:"5s"`

	// The duration that the received messages are hidden from subsequent retrieve requests.
	//
	// Default value is set via DefaultSQSMessageVisibilityTimeout variable.
	MessageVisibilityTimeout time.Duration `min:"1s"`

	// Maximum number of retries for a failed message before it is dropped.
	//
	// Default value is set via DefaultSQSMessageMaxRetryCount variable.
	MessageMaxRetryCount int `min:"1"`

	// Size of events buffered channel.
	//
	// Default value is set via DefaultSQSEventsChanSize variable.
	EventsChanSize int `min:"1"`

	// Breaker enables tracking AWS SQS client error rate.
	//
	// Default value is set via DefaultSQSBreaker variable.
	Breaker core.Breaker
}