airbyte

package

v0.0.0-...-8aeb8a1 Latest Latest Go to latest Published: Jun 29, 2023 License: MIT Imports: 26 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/jitsucom/jitsu

Documentation ¶

Index ¶

Constants
Variables
func Init(ctx context.Context, containerizedRun bool, configDir, workspaceVolume string, ...) error
type Bridge
type Catalog
type CatalogRow
type LogRow
type RecordRow
type Row
type Runner
- func NewRunner(sourceID, dockerImage, imageVersion, identifier string) *Runner
type Schema
type StateRow
type StatusRow
type Stream
type WrappedStream

Constants ¶

View Source

const (
	BridgeType                  = "airbyte_bridge"
	DockerImageRepositoryPrefix = "airbyte/"

	VolumeAlias     = "/tmp/airbyte/"
	DockerCommand   = "docker"
	LatestVersion   = "latest"
	MountVolumeType = "volume"
)

View Source

const (
	LogType              = "LOG"
	ConnectionStatusType = "CONNECTION_STATUS"
	StateType            = "STATE"
	RecordType           = "RECORD"
	CatalogType          = "CATALOG"
	SpecType             = "SPEC"
)

Variables ¶

View Source

var (
	Instance      *Bridge
	InstanceError error
)

Functions ¶

func Init ¶

func Init(ctx context.Context, containerizedRun bool, configDir, workspaceVolume string, batchSize int, logWriter io.Writer) error

Init initializes airbyte Bridge

Types ¶

type Bridge ¶

type Bridge struct {
	LogWriter       io.Writer
	ConfigDir       string
	WorkspaceVolume string
	// contains filtered or unexported fields
}

func (*Bridge) AddAirbytePrefix ¶

func (b *Bridge) AddAirbytePrefix(dockerImage string) string

AddAirbytePrefix adds airbyte/ prefix to dockerImage if doesn't exist

func (*Bridge) BuildMsg ¶

func (b *Bridge) BuildMsg(prefix string, outWriter, errWriter *logging.StringWriter, err error) string

BuildMsg returns formatted error

func (*Bridge) IsImagePulled ¶

func (b *Bridge) IsImagePulled(dockerRepoImage, version string) (bool, error)

IsImagePulled returns true if the image is pulled or start pulling the image asynchronously and returns false

type Catalog ¶

type Catalog struct {
	Streams []*WrappedStream `json:"streams,omitempty"`
}

Catalog is a dto for formatted airbyte catalog serialization

type CatalogRow ¶

type CatalogRow struct {
	Streams []*Stream `json:"streams,omitempty"`
}

CatalogRow is a dto for Airbyte discover output serialization

type LogRow ¶

type LogRow struct {
	Level   string `json:"level,omitempty"`
	Message string `json:"message,omitempty"`
}

LogRow is a dto for airbyte logs serialization

type RecordRow ¶

type RecordRow struct {
	Stream string                 `json:"stream,omitempty"`
	Data   map[string]interface{} `json:"data,omitempty"`
}

RecordRow is a dto for airbyte record serialization

type Row ¶

type Row struct {
	Type             string                 `json:"type"`
	Log              *LogRow                `json:"log,omitempty"`
	ConnectionStatus *StatusRow             `json:"connectionStatus,omitempty"`
	State            *StateRow              `json:"state,omitempty"`
	Record           *RecordRow             `json:"record,omitempty"`
	Catalog          *CatalogRow            `json:"catalog,omitempty"`
	Spec             map[string]interface{} `json:"spec,omitempty"`
}

Row is a dto for airbyte output row representation

type Runner ¶

type Runner struct {

	//DockerImage without 'airbyte/' prefix
	DockerImage string
	Version     string
	// contains filtered or unexported fields
}

Runner is an Airbyte Docker runner Can only be used once Self-closed (see run() func)

func NewRunner ¶

func NewRunner(sourceID, dockerImage, imageVersion, identifier string) *Runner

NewRunner returns configured Airbyte Runner

func (*Runner) Check ¶

func (r *Runner) Check(airbyteSourceConfig interface{}) error

func (*Runner) Close ¶

func (r *Runner) Close() error

func (*Runner) Discover ¶

func (r *Runner) Discover(airbyteSourceConfig interface{}, timeout time.Duration) (*CatalogRow, error)

Discover returns discovered raw catalog

func (*Runner) IsReady ¶

func (r *Runner) IsReady() (bool, error)

func (*Runner) Read ¶

func (r *Runner) Read(dataConsumer base.CLIDataConsumer, streamsRepresentation map[string]*base.StreamRepresentation, taskLogger logging.TaskLogger, taskCloser base.CLITaskCloser, sourceID, statePath string) error

func (*Runner) Spec ¶

func (r *Runner) Spec() (interface{}, error)

Spec runs airbyte docker spec command and returns spec and err if occurred

func (*Runner) String ¶

func (r *Runner) String() string

String returns exec command string

type Schema ¶

type Schema struct {
	Properties map[string]*base.Property `json:"properties,omitempty"`
}

Schema is a dto for Airbyte catalog Schema object serialization

type StateRow ¶

type StateRow struct {
	Data map[string]interface{} `json:"data,omitempty"`
}

StateRow is a dto for airbyte state serialization

type StatusRow ¶

type StatusRow struct {
	Status  string `json:"status,omitempty"`
	Message string `json:"message,omitempty"`
}

StatusRow is a dto for airbyte result status serialization

type Stream ¶

type Stream struct {
	Name                    string     `json:"name,omitempty"`
	JsonSchema              *Schema    `json:"json_schema,omitempty"`
	SupportedSyncModes      []string   `json:"supported_sync_modes,omitempty"`
	SourceDefinedPrimaryKey [][]string `json:"source_defined_primary_key,omitempty"`
	SourceDefinedCursor     bool       `json:"source_defined_cursor"`
	DefaultCursorField      []string   `json:"default_cursor_field,omitempty"`

	Namespace string `json:"namespace,omitempty"`

	SyncMode            string   `json:"-" yaml:"-"` //without serialization
	SelectedCursorField []string `json:"-" yaml:"-"` //without serialization
}

Stream is a dto for Airbyte catalog Stream object serialization

type WrappedStream ¶

type WrappedStream struct {
	SyncMode            string   `json:"sync_mode,omitempty"`
	DestinationSyncMode string   `json:"destination_sync_mode,omitempty"`
	CursorField         []string `json:"cursor_field,omitempty"`
	Stream              *Stream  `json:"stream,omitempty"`
}

WrappedStream is a dto for formatted stream

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL