Documentation ¶
Index ¶
- Variables
- func New(_ context.Context, logger zerolog.Logger, spec []byte, ...) (plugin.Client, error)
- type Client
- func (c *Client) Close(ctx context.Context) error
- func (c *Client) DeleteStale(ctx context.Context, msgs message.WriteDeleteStales) error
- func (c *Client) MigrateTables(ctx context.Context, msgs message.WriteMigrateTables) error
- func (c *Client) Read(ctx context.Context, table *schema.Table, res chan<- arrow.Record) error
- func (*Client) SchemaTypeToSnowflake(t arrow.DataType) string
- func (c *Client) Write(ctx context.Context, msgs <-chan message.WriteMessage) error
- func (c *Client) WriteTableBatch(ctx context.Context, name string, msgs message.WriteInserts) error
- type Spec
Constants ¶
This section is empty.
Variables ¶
View Source
var JSONSchema string
Functions ¶
Types ¶
type Client ¶
type Client struct { plugin.UnimplementedSource batchwriter.UnimplementedDeleteRecord // contains filtered or unexported fields }
func (*Client) DeleteStale ¶
func (*Client) MigrateTables ¶
func (*Client) SchemaTypeToSnowflake ¶
func (*Client) WriteTableBatch ¶
type Spec ¶
type Spec struct { // Snowflake `connection_string`. ConnectionString string `json:"connection_string" jsonschema:"required,minLength=1"` // A PEM-encoded private key for connecting to snowflake. Equivalent to adding // `authenticator=snowflake_jwt&privateKey=...` to the `connection_string` but // parses, validates, and correctly encodes the key for use with snowflake. PrivateKey string `json:"private_key,omitempty"` // Number of records to batch together before sending to the database. BatchSize int `json:"batch_size,omitempty" jsonschema:"minimum=1,default=1000"` // Number of bytes (as Arrow buffer size) to batch together before sending to the database. BatchSizeBytes int `json:"batch_size_bytes,omitempty" jsonschema:"minimum=1,default=4194304"` // By default, tables are migrated one at a time. // This option allows you to migrate multiple tables concurrently. // This can be useful if you have a lot of tables to migrate and want to speed up the process. MigrateConcurrency int `json:"migrate_concurrency,omitempty" jsonschema:"minimum=1,default=1"` // If set to true, intermediary files used to load data to the Snowflake stage are left in the temp directory. This can be useful for debugging purposes. LeaveStageFiles bool `json:"leave_stage_files,omitempty" jsonschema:"default=false"` }
func (*Spec) SetDefaults ¶
func (s *Spec) SetDefaults()
Source Files ¶
Click to show internal directories.
Click to hide internal directories.