airbyte

package
v0.2.0-alpha Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 20, 2023 License: MIT Imports: 27 Imported by: 2

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func Init

func Init(logger *zap.Logger, options ConnectorOptions) base.IConnector

func InitAirbyteCatalog

func InitAirbyteCatalog(logger *zap.Logger, vdpProtocolPath string)

InitAirbyteCatalog reads all task AirbyteCatalog files and stores the JSON content in the global TaskAirbyteCatalog variable

Types

type AirbyteCatalog

type AirbyteCatalog struct {
	Streams []AirbyteStream `json:"streams"`
}

AirbyteCatalog defines the AirbyteCatalog protocol as in: https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L212-L222

var TaskOutputAirbyteCatalog AirbyteCatalog

TaskOutputAirbyteCatalog stores the pre-defined task AirbyteCatalog

type AirbyteMessage

type AirbyteMessage struct {
	Type   string                `json:"type"`
	Record *AirbyteRecordMessage `json:"record"`
}

AirbyteMessage defines the AirbyteMessage protocol as in https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L13-L49

type AirbyteRecordMessage

type AirbyteRecordMessage struct {
	Stream    string          `json:"stream"`
	Data      json.RawMessage `json:"data"`
	EmittedAt int64           `json:"emitted_at"`
}

AirbyteRecordMessage defines the RECORD type of AirbyteMessage, AirbyteRecordMessage, protocol as in (without namespace field) https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L50-L70

type AirbyteStream

type AirbyteStream struct {
	Name                    string          `json:"name"`
	JSONSchema              json.RawMessage `json:"json_schema"`
	SupportedSyncModes      []string        `json:"supported_sync_modes"`
	SourceDefinedCursor     bool            `json:"source_defined_cursor"`
	DefaultCursorField      []string        `json:"default_cursor_field"`
	SourceDefinedPrimaryKey [][]string      `json:"source_defined_primary_key"`
}

AirbyteStream defines the AirbyteStream protocol as in (without namespace field): https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L223-L260

type ConfiguredAirbyteCatalog

type ConfiguredAirbyteCatalog struct {
	Streams []ConfiguredAirbyteStream `json:"streams"`
}

ConfiguredAirbyteCatalog defines the ConfiguredAirbyteCatalog protocol as in: https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L261-L271

type ConfiguredAirbyteStream

type ConfiguredAirbyteStream struct {
	Stream              *AirbyteStream `json:"stream"`
	SyncMode            string         `json:"sync_mode"`
	CursorField         []string       `json:"cursor_field"`
	DestinationSyncMode string         `json:"destination_sync_mode"`
	PrimaryKey          []string       `json:"primary_key"`
}

ConfiguredAirbyteStream defines the ConfiguredAirbyteStream protocol as in: https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L272-L299

type Connection

type Connection struct {
	base.BaseConnection
	// contains filtered or unexported fields
}

func (*Connection) Execute

func (con *Connection) Execute(inputs []*connectorPB.DataPayload) ([]*connectorPB.DataPayload, error)

func (*Connection) GetTask

func (con *Connection) GetTask() (connectorPB.Task, error)

func (*Connection) Test

type Connector

type Connector struct {
	base.BaseConnector
	// contains filtered or unexported fields
}

func (*Connector) CreateConnection

func (c *Connector) CreateConnection(defUid uuid.UUID, config *structpb.Struct, logger *zap.Logger) (base.IConnection, error)

func (*Connector) PreDownloadImage

func (c *Connector) PreDownloadImage(logger *zap.Logger, uids []uuid.UUID) error

type ConnectorOptions

type ConnectorOptions struct {
	MountSourceVDP        string
	MountTargetVDP        string
	MountSourceAirbyte    string
	MountTargetAirbyte    string
	VDPProtocolPath       string
	ExcludeLocalConnector bool
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL