materialize

type ApplyRequest struct {
	// Materialization to be applied.
	Materialization *flow.MaterializationSpec `protobuf:"bytes,1,opt,name=materialization,proto3" json:"materialization,omitempty"`
	// Version of the MaterializationSpec being applied.
	Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"`
	// Is this Apply a dry-run? If so, no action is undertaken and Apply will
	// report only what would have happened.
	DryRun               bool     `protobuf:"varint,3,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

type ApplyResponse struct {
	// Human-readable description of the action that the Driver took (or, if
	// dry_run, would have taken). If empty, this Apply is to be considered a
	// "no-op".
	ActionDescription    string   `protobuf:"bytes,1,opt,name=action_description,json=actionDescription,proto3" json:"action_description,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

type CommitOps struct {
	// DriverCommitted resolves on reading DriverCommitted of the prior transaction.
	// This operation is a depencency of a staged recovery log write, and its
	// resolution allows the recovery log commit to proceed.
	// Nil if there isn't an ongoing driver commit.
	DriverCommitted *client.AsyncOperation
	// LogCommitted resolves on the prior transactions's commit to the recovery log.
	// When resolved, the TxnClient notifies the driver by sending Acknowledge.
	// Nil if there isn't an ongoing recovery log commit.
	LogCommitted client.OpFuture
	// Acknowledged resolves on reading Acknowledged from the driver, and completes
	// the transaction lifecycle. Once resolved (and not before), a current and
	// concurrent may begin to commit (by sending Prepared).
	Acknowledged *client.AsyncOperation
}

type Constraint struct {
	Type Constraint_Type `protobuf:"varint,2,opt,name=type,proto3,enum=materialize.Constraint_Type" json:"type,omitempty"`
	// Optional human readable reason for the given constraint.
	// Implementations are strongly encouraged to supply a descriptive message.
	Reason               string   `protobuf:"bytes,3,opt,name=reason,proto3" json:"reason,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

type Constraint_Type int32

type DriverClient interface {
	// Spec returns the specification definition of this driver.
	// Notably this includes its endpoint and resource configuration JSON schema.
	Spec(ctx context.Context, in *SpecRequest, opts ...grpc.CallOption) (*SpecResponse, error)
	// Validate that store resources and proposed collection bindings are
	// compatible, and return constraints over the projections of each binding.
	Validate(ctx context.Context, in *ValidateRequest, opts ...grpc.CallOption) (*ValidateResponse, error)
	// ApplyUpsert applies a new or updated materialization to the store.
	ApplyUpsert(ctx context.Context, in *ApplyRequest, opts ...grpc.CallOption) (*ApplyResponse, error)
	// ApplyDelete deletes an existing materialization from the store.
	ApplyDelete(ctx context.Context, in *ApplyRequest, opts ...grpc.CallOption) (*ApplyResponse, error)
	// Transactions is a very long lived RPC through which the Flow runtime and a
	// Driver cooperatively execute an unbounded number of transactions.
	//
	// This RPC workflow maintains a materialized view of a Flow collection
	// in an external system. It has distinct load, prepare, store, and commit
	// phases. The Flow runtime and driver cooperatively maintain a fully-reduced
	// view of each document by loading current states from the store, reducing in
	// a number of updates, and then transactionally storing updated documents and
	// checkpoints.
	//
	// Push-only Endpoints & Delta Updates
	// ===================================
	//
	// Some systems, such as APIs, Webhooks, and Pub/Sub, are push-only in nature.
	// Flow materializations can run in a "delta updates" mode, where the load
	// phase is always skipped and Flow does not attempt to store fully-reduced
	// documents. Instead, during the store phase, the runtime sends delta
	// updates which reflect the combined roll-up of collection documents
	// processed only within this transaction.
	//
	// To illustrate the meaning of a delta update, consider documents which are
	// simple counters, having a collection schema that uses a `sum` reduction
	// strategy.
	//
	// Without delta updates, Flow would reduce documents -1, 3, and 2 by `sum`
	// to arrive at document 4, which is stored. The next transaction,
	// document 4 is loaded and reduced with 6, -7, and -1 to arrive at a new
	// stored document 2. This document, 2, represents the full reduction of the
	// collection documents materialized thus far.
	//
	// Compare to delta updates mode: collection documents -1, 3, and 2 are
	// combined to store a delta-update document of 4. The next transaction starts
	// anew, and 6, -7, and -1 combine to arrive at a delta-update document of -2.
	// These delta updates are a windowed combine over documents seen in the
	// current transaction only, and unlike before are not a full reduction of the
	// document. If delta updates were written to pub/sub, note that a subscriber
	// could further reduce over each delta update to recover the fully reduced
	// document of 2.
	//
	// Note that many use cases require only `lastWriteWins` reduction behavior,
	// and for these use cases delta updates does the "right thing" by trivially
	// re-writing each document with its most recent version. This matches the
	// behavior of Kafka Connect, for example.
	//
	// On Transactionality
	// ===================
	//
	// The beating heart of transactionality in materializations is this:
	// there is a consumption checkpoint, and there is a state of the view.
	// As the materialization progresses, both the checkpoint and the view state
	// will change. Updates to the checkpoint and to the view state MUST always
	// commit together, in the exact same transaction.
	//
	// Flow transaction tasks have a backing transactional recovery log,
	// which is capable of durable commits that update both the checkpoint
	// and also a (reasonably small) driver-defined state. More on driver
	// states later.
	//
	// Many interesting systems are also fully transactional in nature.
	//
	// When implementing a matherialization driver, the first question an
	// implementor must answer is: whose commit is authoritative?
	// Flow's recovery log, or the materialized system ?
	// This protocol supports either.
	//
	// Implementation Pattern: Remote Store is Authoritative
	// =====================================================
	//
	// In this pattern, the remote store persists view states and the Flow
	// consumption checkpoints which those views reflect (there are many such
	// checkpoints: one per task split). The Flow recovery log is not used.
	//
	// Typically this workflow runs in the context of a synchronous BEGIN/COMMIT
	// transaction, which updates table states and a Flow checkpoint together.
	// The transaction need be scoped only to the store phase of this workflow,
	// as the Flow runtime assumes only read-committed loads.
	//
	// Flow is a distributed system, and an important consideration is the effect
	// of a "zombie" assignment of a materialization task, which can race a
	// newly-promoted assignment of that same task.
	//
	// Fencing is a technique which uses the transactional capabilities of a store
	// to "fence off" an older zombie assignment, such that it's prevented from
	// committing further transactions. This avoids a failure mode where:
	//  - New assignment N recovers a checkpoint at Ti.
	//  - Zombie assignment Z commits another transaction at Ti+1.
	//  - N beings processing from Ti, inadvertently duplicating the effects of
	//  Ti+1.
	//
	// When authoritative, the remote store must implement fencing behavior.
	// As a sketch, the store can maintain a nonce value alongside the checkpoint
	// of each task split. The nonce is updated on each open of this RPC,
	// and each commit transaction then verifies that the nonce has not been
	// changed.
	//
	// In the future, if another RPC opens and updates the nonce, it fences off
	// this instance of the task split and prevents it from committing further
	// transactions.
	//
	// Implementation Pattern: Recovery Log with Non-Transactional Store
	// =================================================================
	//
	// In this pattern, the recovery log persists the Flow checkpoint and handles
	// fencing semantics. During the load and store phases, the driver
	// directly manipulates a non-transactional store or API.
	//
	// Note that this pattern is at-least-once. A transaction may fail part-way
	// through and be restarted, causing its effects to be partially or fully
	// replayed.
	//
	// Care must be taken if the collection's schema has reduction annotations
	// such as `sum`, as those reductions may be applied more than once due to
	// a partially completed, but ultimately failed transaction.
	//
	// If the collection's schema is last-write-wins, this mode still provides
	// effectively-once behavior. Collections which aren't last-write-wins
	// can be turned into last-write-wins through the use of derivation
	// registers.
	//
	// Implementation Pattern: Recovery Log with Idempotent Apply
	// ==========================================================
	//
	// In this pattern the recovery log is authoritative, but the driver uses
	// external stable storage to stage the effects of a transaction -- rather
	// than directly applying them to the store -- such that those effects can be
	// idempotently applied after the transaction commits.
	//
	// This allows stores which feature a weaker transactionality guarantee to
	// still be used in an exactly-once way, so long as they support an idempotent
	// apply operation.
	//
	// Driver checkpoints can facilitate this pattern. For example, a driver might
	// generate a unique filename in S3 and reference it in its prepared
	// checkpoint, which is committed to the recovery log. During the "store"
	// phase, it writes to this S3 file. After the transaction commits, it tells
	// the store of the new file to incorporate. The store must handle
	// idempotency, by applying the effects of the unique file just once, even if
	// told of the file multiple times.
	//
	// A related extension of this pattern is for the driver to embed a Flow
	// checkpoint into its driver checkpoint. Doing so allows the driver to
	// express an intention to restart from an older alternative checkpoint, as
	// compared to the most recent committed checkpoint of the recovery log.
	//
	// As mentioned above, it's crucial that store states and checkpoints commit
	// together. While seemingly bending that rule, this pattern is consistent
	// with it because, on commit, the semantic contents of the store include BOTH
	// its base state, as well as the staged idempotent update. The store just may
	// not know it yet, but eventually it must because of the retried idempotent
	// apply.
	//
	// Note the driver must therefore ensure that staged updates are fully applied
	// before returning an "load" responses, in order to provide the correct
	// read-committed semantics required by the Flow runtime.
	//
	// RPC Lifecycle
	// =============
	//
	// The RPC follows the following lifecycle:
	//
	// :TransactionRequest.Open:
	//    - The Flow runtime opens the stream.
	// :TransactionResponse.Opened:
	//    - If the remote store is authoritative, it must fence off other RPCs
	//      of this task split from committing further transactions,
	//      and it retrieves a Flow checkpoint which is returned to the runtime.
	//
	// TransactionRequest.Open and TransactionResponse.Opened are sent only
	// once, at the commencement of the stream. Thereafter the protocol loops:
	//
	// Load phase
	// ==========
	//
	// The Load phases is Load requests *intermixed* with one
	// Acknowledge/Acknowledged message flow. The driver must accomodate an
	// Acknowledge that occurs before, during, or after a sequence of Load
	// requests. It's guaranteed to see exactly one Acknowledge request during
	// this phase.
	//
	// :TransactionRequest.Acknowledge:
	//    - The runtime tells the driver that a commit to the recovery log has
	//      completed.
	//    - The driver applies a staged update to the base store, where
	//      applicable.
	//    - Note Acknowledge is sent in the very first iteration for consistency.
	//      Semantically, it's an acknowledgement of the recovered checkpoint.
	//      If a previous invocation failed after recovery log commit but before
	//      applying the staged change, this is an opportunity to ensure that
	//      apply occurs.
	// :TransactionResponse.Acknowledged:
	//    - The driver responds to the runtime only after applying a staged
	//      update, where applicable.
	//    - If there is no staged update, the driver immediately responds on
	//      seeing Acknowledge.
	//
	// :TransactionRequest.Load:
	//    - The runtime sends zero or more Load messages.
	//    - The driver may send any number of TransactionResponse.Loaded in
	//      response.
	//    - If the driver will apply a staged update, it must await Acknowledge
	//      and have applied the update to the store *before* evaluating any
	//      Loads, to ensure correct read-committed behavior.
	//    - The driver may defer responding with some or all loads until the
	//      prepare phase.
	// :TransactionResponse.Loaded:
	//    - The driver sends zero or more Loaded messages, once for each loaded
	//      document.
	//    - Document keys not found in the store are omitted and not sent as
	//      Loaded.
	//
	// Prepare phase
	// =============
	//
	// The prepare phase begins only after the prior transaction has both
	// committed and also been acknowledged. It marks the bounds of the present
	// transaction.
	//
	// Upon entering this phase, the driver must immediately evaluate any deferred
	// Load requests and send remaining Loaded responses.
	//
	// :TransactionRequest.Prepare:
	//    - The runtime sends a Prepare message with its Flow checkpoint.
	// :TransactionResponse.Prepared:
	//    - The driver sends Prepared after having flushed all Loaded responses.
	//    - The driver may include a driver checkpoint update which will be
	//      committed to the recovery log with this transaction.
	//
	// Store phase
	// ===========
	//
	// The store phase is when the runtime sends the driver materialized document
	// updates, as well as an indication of whether the document is an insert,
	// update, or delete (in other words, was it returned in a Loaded response?).
	//
	// :TransactionRequest.Store:
	//    - The runtime sends zero or more Store messages.
	//
	// Commit phase
	// ============
	//
	// The commit phase marks the end of the store phase, and tells the driver of
	// the runtime's intent to commit to its recovery log. If the remote store is
	// authoritative, the driver must commit its transaction at this time.
	//
	// :TransactionRequest.Commit:
	//    - The runtime sends a Commit message, denoting its intention to commit.
	//    - If the remote store is authoritative, the driver includes the Flow
	//      checkpoint into its transaction and commits it along with view state
	//      updates.
	//    - Otherwise, the driver immediately responds with DriverCommitted.
	// :TransactionResponse.DriverCommitted:
	//    - The driver sends a DriverCommitted message.
	//    - The runtime commits Flow and driver checkpoint to its recovery
	//      log. The completion of this commit will be marked by an
	//      Acknowledge during the next load phase.
	//    - Runtime and driver begin a new, pipelined transaction by looping to
	//      load while this transaction continues to commit.
	//
	// An error of any kind rolls back the transaction in progress and terminates
	// the stream.
	Transactions(ctx context.Context, opts ...grpc.CallOption) (Driver_TransactionsClient, error)
}

type DriverServer interface {
	// Spec returns the specification definition of this driver.
	// Notably this includes its endpoint and resource configuration JSON schema.
	Spec(context.Context, *SpecRequest) (*SpecResponse, error)
	// Validate that store resources and proposed collection bindings are
	// compatible, and return constraints over the projections of each binding.
	Validate(context.Context, *ValidateRequest) (*ValidateResponse, error)
	// ApplyUpsert applies a new or updated materialization to the store.
	ApplyUpsert(context.Context, *ApplyRequest) (*ApplyResponse, error)
	// ApplyDelete deletes an existing materialization from the store.
	ApplyDelete(context.Context, *ApplyRequest) (*ApplyResponse, error)
	// Transactions is a very long lived RPC through which the Flow runtime and a
	// Driver cooperatively execute an unbounded number of transactions.
	//
	// This RPC workflow maintains a materialized view of a Flow collection
	// in an external system. It has distinct load, prepare, store, and commit
	// phases. The Flow runtime and driver cooperatively maintain a fully-reduced
	// view of each document by loading current states from the store, reducing in
	// a number of updates, and then transactionally storing updated documents and
	// checkpoints.
	//
	// Push-only Endpoints & Delta Updates
	// ===================================
	//
	// Some systems, such as APIs, Webhooks, and Pub/Sub, are push-only in nature.
	// Flow materializations can run in a "delta updates" mode, where the load
	// phase is always skipped and Flow does not attempt to store fully-reduced
	// documents. Instead, during the store phase, the runtime sends delta
	// updates which reflect the combined roll-up of collection documents
	// processed only within this transaction.
	//
	// To illustrate the meaning of a delta update, consider documents which are
	// simple counters, having a collection schema that uses a `sum` reduction
	// strategy.
	//
	// Without delta updates, Flow would reduce documents -1, 3, and 2 by `sum`
	// to arrive at document 4, which is stored. The next transaction,
	// document 4 is loaded and reduced with 6, -7, and -1 to arrive at a new
	// stored document 2. This document, 2, represents the full reduction of the
	// collection documents materialized thus far.
	//
	// Compare to delta updates mode: collection documents -1, 3, and 2 are
	// combined to store a delta-update document of 4. The next transaction starts
	// anew, and 6, -7, and -1 combine to arrive at a delta-update document of -2.
	// These delta updates are a windowed combine over documents seen in the
	// current transaction only, and unlike before are not a full reduction of the
	// document. If delta updates were written to pub/sub, note that a subscriber
	// could further reduce over each delta update to recover the fully reduced
	// document of 2.
	//
	// Note that many use cases require only `lastWriteWins` reduction behavior,
	// and for these use cases delta updates does the "right thing" by trivially
	// re-writing each document with its most recent version. This matches the
	// behavior of Kafka Connect, for example.
	//
	// On Transactionality
	// ===================
	//
	// The beating heart of transactionality in materializations is this:
	// there is a consumption checkpoint, and there is a state of the view.
	// As the materialization progresses, both the checkpoint and the view state
	// will change. Updates to the checkpoint and to the view state MUST always
	// commit together, in the exact same transaction.
	//
	// Flow transaction tasks have a backing transactional recovery log,
	// which is capable of durable commits that update both the checkpoint
	// and also a (reasonably small) driver-defined state. More on driver
	// states later.
	//
	// Many interesting systems are also fully transactional in nature.
	//
	// When implementing a matherialization driver, the first question an
	// implementor must answer is: whose commit is authoritative?
	// Flow's recovery log, or the materialized system ?
	// This protocol supports either.
	//
	// Implementation Pattern: Remote Store is Authoritative
	// =====================================================
	//
	// In this pattern, the remote store persists view states and the Flow
	// consumption checkpoints which those views reflect (there are many such
	// checkpoints: one per task split). The Flow recovery log is not used.
	//
	// Typically this workflow runs in the context of a synchronous BEGIN/COMMIT
	// transaction, which updates table states and a Flow checkpoint together.
	// The transaction need be scoped only to the store phase of this workflow,
	// as the Flow runtime assumes only read-committed loads.
	//
	// Flow is a distributed system, and an important consideration is the effect
	// of a "zombie" assignment of a materialization task, which can race a
	// newly-promoted assignment of that same task.
	//
	// Fencing is a technique which uses the transactional capabilities of a store
	// to "fence off" an older zombie assignment, such that it's prevented from
	// committing further transactions. This avoids a failure mode where:
	//  - New assignment N recovers a checkpoint at Ti.
	//  - Zombie assignment Z commits another transaction at Ti+1.
	//  - N beings processing from Ti, inadvertently duplicating the effects of
	//  Ti+1.
	//
	// When authoritative, the remote store must implement fencing behavior.
	// As a sketch, the store can maintain a nonce value alongside the checkpoint
	// of each task split. The nonce is updated on each open of this RPC,
	// and each commit transaction then verifies that the nonce has not been
	// changed.
	//
	// In the future, if another RPC opens and updates the nonce, it fences off
	// this instance of the task split and prevents it from committing further
	// transactions.
	//
	// Implementation Pattern: Recovery Log with Non-Transactional Store
	// =================================================================
	//
	// In this pattern, the recovery log persists the Flow checkpoint and handles
	// fencing semantics. During the load and store phases, the driver
	// directly manipulates a non-transactional store or API.
	//
	// Note that this pattern is at-least-once. A transaction may fail part-way
	// through and be restarted, causing its effects to be partially or fully
	// replayed.
	//
	// Care must be taken if the collection's schema has reduction annotations
	// such as `sum`, as those reductions may be applied more than once due to
	// a partially completed, but ultimately failed transaction.
	//
	// If the collection's schema is last-write-wins, this mode still provides
	// effectively-once behavior. Collections which aren't last-write-wins
	// can be turned into last-write-wins through the use of derivation
	// registers.
	//
	// Implementation Pattern: Recovery Log with Idempotent Apply
	// ==========================================================
	//
	// In this pattern the recovery log is authoritative, but the driver uses
	// external stable storage to stage the effects of a transaction -- rather
	// than directly applying them to the store -- such that those effects can be
	// idempotently applied after the transaction commits.
	//
	// This allows stores which feature a weaker transactionality guarantee to
	// still be used in an exactly-once way, so long as they support an idempotent
	// apply operation.
	//
	// Driver checkpoints can facilitate this pattern. For example, a driver might
	// generate a unique filename in S3 and reference it in its prepared
	// checkpoint, which is committed to the recovery log. During the "store"
	// phase, it writes to this S3 file. After the transaction commits, it tells
	// the store of the new file to incorporate. The store must handle
	// idempotency, by applying the effects of the unique file just once, even if
	// told of the file multiple times.
	//
	// A related extension of this pattern is for the driver to embed a Flow
	// checkpoint into its driver checkpoint. Doing so allows the driver to
	// express an intention to restart from an older alternative checkpoint, as
	// compared to the most recent committed checkpoint of the recovery log.
	//
	// As mentioned above, it's crucial that store states and checkpoints commit
	// together. While seemingly bending that rule, this pattern is consistent
	// with it because, on commit, the semantic contents of the store include BOTH
	// its base state, as well as the staged idempotent update. The store just may
	// not know it yet, but eventually it must because of the retried idempotent
	// apply.
	//
	// Note the driver must therefore ensure that staged updates are fully applied
	// before returning an "load" responses, in order to provide the correct
	// read-committed semantics required by the Flow runtime.
	//
	// RPC Lifecycle
	// =============
	//
	// The RPC follows the following lifecycle:
	//
	// :TransactionRequest.Open:
	//    - The Flow runtime opens the stream.
	// :TransactionResponse.Opened:
	//    - If the remote store is authoritative, it must fence off other RPCs
	//      of this task split from committing further transactions,
	//      and it retrieves a Flow checkpoint which is returned to the runtime.
	//
	// TransactionRequest.Open and TransactionResponse.Opened are sent only
	// once, at the commencement of the stream. Thereafter the protocol loops:
	//
	// Load phase
	// ==========
	//
	// The Load phases is Load requests *intermixed* with one
	// Acknowledge/Acknowledged message flow. The driver must accomodate an
	// Acknowledge that occurs before, during, or after a sequence of Load
	// requests. It's guaranteed to see exactly one Acknowledge request during
	// this phase.
	//
	// :TransactionRequest.Acknowledge:
	//    - The runtime tells the driver that a commit to the recovery log has
	//      completed.
	//    - The driver applies a staged update to the base store, where
	//      applicable.
	//    - Note Acknowledge is sent in the very first iteration for consistency.
	//      Semantically, it's an acknowledgement of the recovered checkpoint.
	//      If a previous invocation failed after recovery log commit but before
	//      applying the staged change, this is an opportunity to ensure that
	//      apply occurs.
	// :TransactionResponse.Acknowledged:
	//    - The driver responds to the runtime only after applying a staged
	//      update, where applicable.
	//    - If there is no staged update, the driver immediately responds on
	//      seeing Acknowledge.
	//
	// :TransactionRequest.Load:
	//    - The runtime sends zero or more Load messages.
	//    - The driver may send any number of TransactionResponse.Loaded in
	//      response.
	//    - If the driver will apply a staged update, it must await Acknowledge
	//      and have applied the update to the store *before* evaluating any
	//      Loads, to ensure correct read-committed behavior.
	//    - The driver may defer responding with some or all loads until the
	//      prepare phase.
	// :TransactionResponse.Loaded:
	//    - The driver sends zero or more Loaded messages, once for each loaded
	//      document.
	//    - Document keys not found in the store are omitted and not sent as
	//      Loaded.
	//
	// Prepare phase
	// =============
	//
	// The prepare phase begins only after the prior transaction has both
	// committed and also been acknowledged. It marks the bounds of the present
	// transaction.
	//
	// Upon entering this phase, the driver must immediately evaluate any deferred
	// Load requests and send remaining Loaded responses.
	//
	// :TransactionRequest.Prepare:
	//    - The runtime sends a Prepare message with its Flow checkpoint.
	// :TransactionResponse.Prepared:
	//    - The driver sends Prepared after having flushed all Loaded responses.
	//    - The driver may include a driver checkpoint update which will be
	//      committed to the recovery log with this transaction.
	//
	// Store phase
	// ===========
	//
	// The store phase is when the runtime sends the driver materialized document
	// updates, as well as an indication of whether the document is an insert,
	// update, or delete (in other words, was it returned in a Loaded response?).
	//
	// :TransactionRequest.Store:
	//    - The runtime sends zero or more Store messages.
	//
	// Commit phase
	// ============
	//
	// The commit phase marks the end of the store phase, and tells the driver of
	// the runtime's intent to commit to its recovery log. If the remote store is
	// authoritative, the driver must commit its transaction at this time.
	//
	// :TransactionRequest.Commit:
	//    - The runtime sends a Commit message, denoting its intention to commit.
	//    - If the remote store is authoritative, the driver includes the Flow
	//      checkpoint into its transaction and commits it along with view state
	//      updates.
	//    - Otherwise, the driver immediately responds with DriverCommitted.
	// :TransactionResponse.DriverCommitted:
	//    - The driver sends a DriverCommitted message.
	//    - The runtime commits Flow and driver checkpoint to its recovery
	//      log. The completion of this commit will be marked by an
	//      Acknowledge during the next load phase.
	//    - Runtime and driver begin a new, pipelined transaction by looping to
	//      load while this transaction continues to commit.
	//
	// An error of any kind rolls back the transaction in progress and terminates
	// the stream.
	Transactions(Driver_TransactionsServer) error
}

type Driver_TransactionsClient interface {
	Send(*TransactionRequest) error
	Recv() (*TransactionResponse, error)
	grpc.ClientStream
}

type Driver_TransactionsServer interface {
	Send(*TransactionResponse) error
	Recv() (*TransactionRequest, error)
	grpc.ServerStream
}

type LoadIterator struct {
	Binding int         // Binding index of this document to load.
	Key     tuple.Tuple // Key of the next document to load.
	// contains filtered or unexported fields
}

type SpecRequest struct {
	// Endpoint type addressed by this request.
	EndpointType flow.EndpointType `protobuf:"varint,1,opt,name=endpoint_type,json=endpointType,proto3,enum=flow.EndpointType" json:"endpoint_type,omitempty"`
	// Driver specification, as an encoded JSON object.
	// This may be a partial specification (for example, a Docker image),
	// providing only enough information to fetch the remainder of the
	// specification schema.
	EndpointSpecJson     encoding_json.RawMessage `` /* 141-byte string literal not displayed */
	XXX_NoUnkeyedLiteral struct{}                 `json:"-"`
	XXX_unrecognized     []byte                   `json:"-"`
	XXX_sizecache        int32                    `json:"-"`
}

type SpecResponse struct {
	// JSON schema of an endpoint specification.
	EndpointSpecSchemaJson encoding_json.RawMessage `` /* 161-byte string literal not displayed */
	// JSON schema of a resource specification.
	ResourceSpecSchemaJson encoding_json.RawMessage `` /* 161-byte string literal not displayed */
	// URL for connector's documention.
	DocumentationUrl     string   `protobuf:"bytes,3,opt,name=documentation_url,json=documentationUrl,proto3" json:"documentation_url,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

type StoreIterator struct {
	Binding int             // Binding index of this stored document.
	Key     tuple.Tuple     // Key of the next document to store.
	Values  tuple.Tuple     // Values of the next document to store.
	RawJSON json.RawMessage // Document to store.
	Exists  bool            // Does this document exist in the store already?
	// contains filtered or unexported fields
}

type TransactionRequest struct {
	Open                 *TransactionRequest_Open        `protobuf:"bytes,1,opt,name=open,proto3" json:"open,omitempty"`
	Load                 *TransactionRequest_Load        `protobuf:"bytes,2,opt,name=load,proto3" json:"load,omitempty"`
	Prepare              *TransactionRequest_Prepare     `protobuf:"bytes,3,opt,name=prepare,proto3" json:"prepare,omitempty"`
	Store                *TransactionRequest_Store       `protobuf:"bytes,4,opt,name=store,proto3" json:"store,omitempty"`
	Commit               *TransactionRequest_Commit      `protobuf:"bytes,5,opt,name=commit,proto3" json:"commit,omitempty"`
	Acknowledge          *TransactionRequest_Acknowledge `protobuf:"bytes,6,opt,name=acknowledge,proto3" json:"acknowledge,omitempty"`
	XXX_NoUnkeyedLiteral struct{}                        `json:"-"`
	XXX_unrecognized     []byte                          `json:"-"`
	XXX_sizecache        int32                           `json:"-"`
}

type TransactionRequest_Acknowledge struct {
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

type TransactionRequest_Commit struct {
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

type TransactionRequest_Load struct {
	// The materialization binding for documents of this Load request.
	Binding uint32 `protobuf:"varint,1,opt,name=binding,proto3" json:"binding,omitempty"`
	// Byte arena of the request.
	Arena github_com_estuary_protocols_flow.Arena `protobuf:"bytes,2,opt,name=arena,proto3,casttype=github.com/estuary/protocols/flow.Arena" json:"arena,omitempty"`
	// Packed tuples of collection keys, enumerating the documents to load.
	PackedKeys           []flow.Slice `protobuf:"bytes,3,rep,name=packed_keys,json=packedKeys,proto3" json:"packed_keys"`
	XXX_NoUnkeyedLiteral struct{}     `json:"-"`
	XXX_unrecognized     []byte       `json:"-"`
	XXX_sizecache        int32        `json:"-"`
}

type TransactionRequest_Open struct {
	// Materialization to be transacted.
	Materialization *flow.MaterializationSpec `protobuf:"bytes,1,opt,name=materialization,proto3" json:"materialization,omitempty"`
	// Version of the opened MaterializationSpec.
	// The driver may want to require that this match the version last
	// provided to a successful Apply RPC. It's possible that it won't,
	// due to expected propagation races in Flow's distributed runtime.
	Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"`
	// [begin, end] inclusive range of keys processed by this transaction
	// stream. Ranges are with respect to a 32-bit hash of a packed document
	// key.
	KeyBegin uint32 `protobuf:"fixed32,3,opt,name=key_begin,json=keyBegin,proto3" json:"key_begin,omitempty"`
	KeyEnd   uint32 `protobuf:"fixed32,4,opt,name=key_end,json=keyEnd,proto3" json:"key_end,omitempty"`
	// Last-persisted driver checkpoint committed in the Flow runtime recovery
	// log. Or empty, if the driver has cleared or never set its checkpoint.
	DriverCheckpointJson encoding_json.RawMessage `` /* 153-byte string literal not displayed */
	XXX_NoUnkeyedLiteral struct{}                 `json:"-"`
	XXX_unrecognized     []byte                   `json:"-"`
	XXX_sizecache        int32                    `json:"-"`
}

type TransactionRequest_Prepare struct {
	// Flow checkpoint to commit with this transaction.
	FlowCheckpoint       []byte   `protobuf:"bytes,1,opt,name=flow_checkpoint,json=flowCheckpoint,proto3" json:"flow_checkpoint,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

type TransactionRequest_Store struct {
	// The materialization binding for documents of this Store request.
	Binding uint32 `protobuf:"varint,1,opt,name=binding,proto3" json:"binding,omitempty"`
	// Byte arena of the request.
	Arena github_com_estuary_protocols_flow.Arena `protobuf:"bytes,2,opt,name=arena,proto3,casttype=github.com/estuary/protocols/flow.Arena" json:"arena,omitempty"`
	// Packed tuples holding keys of each document.
	PackedKeys []flow.Slice `protobuf:"bytes,3,rep,name=packed_keys,json=packedKeys,proto3" json:"packed_keys"`
	// Packed tuples holding values for each document.
	PackedValues []flow.Slice `protobuf:"bytes,4,rep,name=packed_values,json=packedValues,proto3" json:"packed_values"`
	// JSON documents.
	DocsJson []flow.Slice `protobuf:"bytes,5,rep,name=docs_json,json=docsJson,proto3" json:"docs_json"`
	// Exists is true if this document as previously been loaded or stored.
	Exists               []bool   `protobuf:"varint,6,rep,packed,name=exists,proto3" json:"exists,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

type TransactionResponse struct {
	Opened *TransactionResponse_Opened `protobuf:"bytes,1,opt,name=opened,proto3" json:"opened,omitempty"`
	Loaded *TransactionResponse_Loaded `protobuf:"bytes,2,opt,name=loaded,proto3" json:"loaded,omitempty"`
	// Prepared responds to a TransactionRequest.Prepare of the client.
	// No further Loaded responses will be sent.
	Prepared             *flow.DriverCheckpoint               `protobuf:"bytes,3,opt,name=prepared,proto3" json:"prepared,omitempty"`
	DriverCommitted      *TransactionResponse_DriverCommitted `protobuf:"bytes,4,opt,name=driver_committed,json=driverCommitted,proto3" json:"driver_committed,omitempty"`
	Acknowledged         *TransactionResponse_Acknowledged    `protobuf:"bytes,5,opt,name=acknowledged,proto3" json:"acknowledged,omitempty"`
	XXX_NoUnkeyedLiteral struct{}                             `json:"-"`
	XXX_unrecognized     []byte                               `json:"-"`
	XXX_sizecache        int32                                `json:"-"`
}

type TransactionResponseError struct {
	*TransactionResponse
	Error error
}

type TransactionResponse_Acknowledged struct {
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

type TransactionResponse_DriverCommitted struct {
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

type TransactionResponse_Loaded struct {
	// The materialization binding for documents of this Loaded response.
	Binding uint32 `protobuf:"varint,1,opt,name=binding,proto3" json:"binding,omitempty"`
	// Byte arena of the request.
	Arena github_com_estuary_protocols_flow.Arena `protobuf:"bytes,2,opt,name=arena,proto3,casttype=github.com/estuary/protocols/flow.Arena" json:"arena,omitempty"`
	// Loaded JSON documents.
	DocsJson             []flow.Slice `protobuf:"bytes,3,rep,name=docs_json,json=docsJson,proto3" json:"docs_json"`
	XXX_NoUnkeyedLiteral struct{}     `json:"-"`
	XXX_unrecognized     []byte       `json:"-"`
	XXX_sizecache        int32        `json:"-"`
}

type TransactionResponse_Opened struct {
	// Flow checkpoint to begin processing from.
	// If empty, the most recent checkpoint of the Flow recovery log is used.
	//
	// Or, a driver may send the value []byte{0xf8, 0xff, 0xff, 0xff, 0xf, 0x1}
	// to explicitly begin processing from a zero-valued checkpoint, effectively
	// rebuilding the materialization from scratch. This sentinel is a trivial
	// encoding of the max-value 2^29-1 protobuf tag with boolean true.
	FlowCheckpoint       []byte   `protobuf:"bytes,1,opt,name=flow_checkpoint,json=flowCheckpoint,proto3" json:"flow_checkpoint,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

type Transactor interface {
	// Load implements the transaction load phase by consuming Load requests
	// from the LoadIterator until a Prepare request is read. Requested keys
	// are not known to exist in the store, and very often they won't.
	// Load can ignore keys which are not found in the store. Before Load returns,
	// though, it must ensure loaded() is called for all found documents.
	//
	// Loads of transaction T+1 will be invoked after Store of T+0, but
	// concurrently with the commit and acknowledgement of T+0. This is an
	// important optimization that allows the driver to immediately begin the work
	// of T+1, for instance by staging Load keys into a temporary table which will
	// be joined and queried once the LoadIterator is drained.
	//
	// But, the driver contract is that documents loaded in T+1 must reflect
	// stores of T+0 (a.k.a. "read committed"). The driver must ensure that the
	// prior transaction is fully reflected in the store before attempting to
	// load any keys.
	//
	// Load is given two channels, priorCommittedCh and priorAcknowledgedCh,
	// to help it understand where the prior transaction is in its lifecycle.
	//  - priorCommittedCh selects (is closed) upon the completion of Transactor.Commit.
	//  - priorAcknowledgedCh selects (is closed) upon completion of Transactor.Acknowledge.
	//
	// * In the simple case where the driver uses implementation pattern
	//   "Recovery Log with Non-Transactional Store", the driver is free
	//   to simply ignore these signals. It may issue lookups to the store and call
	//   back to loaded() when those lookups succeed.
	//
	// * If the driver uses implementation pattern "Remote Store is Authoritative",
	//   it MUST await priorCommittedCh before loading from the store to ensure
	//   read-committed behavior (priorAcknowledgedCh is ignored).
	//   For ease of implementation, it MAY delay reading from the LoadIterator
	//   until priorCommittedCh is signaled to ensure this behavior.
	//
	//   After priorCommittedCh it MAY immediately evaluate loads against the store,
	//   calling back to loaded().
	//
	//   For simplicity it may wish to stage all loads, draining the LoadIterator,
	//   and only then evaluate staged keys against the store. In this case it
	//   can ignore priorCommittedCh, as the LoadIterator will drain only after
	//   it has been signaled.
	//
	// * If the driver uses pattern "Recovery Log with Idempotent Apply",
	//   it MUST await priorAcknowledgedCh before loading from the store,
	//   which indicates that Acknowledge has completed and applied the prior
	//   commit to the store.
	//
	//   The LoadIterator needs to make progress in order for acknowledgement
	//   to occur, and the driver MUST read from LoadIterator and stage those
	//   loads for future evaluation once priorAcknowledgedCh is signaled.
	//   Otherwise the transaction will deadlock.
	//
	//   After priorAcknowledgedCh it MAY immediately evaluate loads against the
	//   store, calling back to loaded().
	//
	//   For simplicity it may wish to stage all loads, draining the LoadIterator,
	//   and only then evaluate staged keys against the store. In this case it
	//   can ignore priorAcknowledgedCh, as the LoadIterator will drain only after
	//   it has been signaled.
	//
	Load(
		it *LoadIterator,
		priorCommittedCh <-chan struct{},
		priorAcknowledgedCh <-chan struct{},
		loaded func(binding int, doc json.RawMessage) error,
	) error
	// Prepare begins the transaction store phase.
	//
	// If the remote store is authoritative, the driver must stage the
	// request's Flow checkpoint for its future driver commit.
	//
	// If the recovery log is authoritative, the driver may wish to provide a
	// driver update which will be included in the log's commit. At this stage
	// the transaction hasn't stored any documents yet, so the driver checkpoint
	// may want to include what the driver _plans_ to do.
	Prepare(context.Context, TransactionRequest_Prepare) (pf.DriverCheckpoint, error)
	// Store consumes Store requests from the StoreIterator.
	Store(*StoreIterator) error
	// Commit is called upon the Flow runtime's request that the driver commit
	// its transaction. Commit does so, and returns a final error status.
	// If the driver doesn't use store transactions, Commit may be a no-op.
	//
	// Note that Commit runs concurrently with Transactor.Load().
	Commit(context.Context) error
	// Acknowledge is called upon the Flow runtime's acknowledgement of its
	// recovery log commit. If the driver stages data for application after
	// commit, it must perform that apply now and return a final error status.
	//
	// Note that Acknowledge may be called multiple times in acknowledgement
	// of a single actual commit. The driver must account for this. If it applies
	// staged data as part of acknowledgement, it must ensure that apply is
	// idempotent.
	//
	// Note that Acknowledge runs concurrently with Transactor.Load().
	Acknowledge(context.Context) error
	// Destroy the Transactor, releasing any held resources.
	Destroy()
}

type TxnClient struct {
	// contains filtered or unexported fields
}

type UnimplementedDriverServer struct {
}

type ValidateRequest struct {
	// Name of the materialization being validated.
	Materialization github_com_estuary_protocols_flow.Materialization `` /* 142-byte string literal not displayed */
	// Endpoint type addressed by this request.
	EndpointType flow.EndpointType `protobuf:"varint,2,opt,name=endpoint_type,json=endpointType,proto3,enum=flow.EndpointType" json:"endpoint_type,omitempty"`
	// Driver specification, as an encoded JSON object.
	EndpointSpecJson     encoding_json.RawMessage   `` /* 141-byte string literal not displayed */
	Bindings             []*ValidateRequest_Binding `protobuf:"bytes,4,rep,name=bindings,proto3" json:"bindings,omitempty"`
	XXX_NoUnkeyedLiteral struct{}                   `json:"-"`
	XXX_unrecognized     []byte                     `json:"-"`
	XXX_sizecache        int32                      `json:"-"`
}

type ValidateRequest_Binding struct {
	// JSON-encoded object which specifies the endpoint resource to be
	// materialized.
	ResourceSpecJson encoding_json.RawMessage `` /* 141-byte string literal not displayed */
	// Collection to be materialized.
	Collection flow.CollectionSpec `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection"`
	// Projection configuration, keyed by the projection field name,
	// with JSON-encoded and driver-defined configuration objects.
	FieldConfigJson      map[string]encoding_json.RawMessage `` /* 227-byte string literal not displayed */
	XXX_NoUnkeyedLiteral struct{}                            `json:"-"`
	XXX_unrecognized     []byte                              `json:"-"`
	XXX_sizecache        int32                               `json:"-"`
}

type ValidateResponse struct {
	Bindings             []*ValidateResponse_Binding `protobuf:"bytes,1,rep,name=bindings,proto3" json:"bindings,omitempty"`
	XXX_NoUnkeyedLiteral struct{}                    `json:"-"`
	XXX_unrecognized     []byte                      `json:"-"`
	XXX_sizecache        int32                       `json:"-"`
}

type ValidateResponse_Binding struct {
	// Constraints over collection projections imposed by the Driver,
	// keyed by the projection field name. Projections of the CollectionSpec
	// which are missing from constraints are implicitly forbidden.
	Constraints map[string]*Constraint `` /* 163-byte string literal not displayed */
	// Components of the resource path which fully qualify the resource
	// identified by this binding.
	// - For an RDBMS, this might be []{dbname, schema, table}.
	// - For Kafka, this might be []{topic}.
	// - For Redis, this might be []{key_prefix}.
	ResourcePath []string `protobuf:"bytes,2,rep,name=resource_path,json=resourcePath,proto3" json:"resource_path,omitempty"`
	// Materialize combined delta updates of documents rather than full
	// reductions.
	//
	// When set, the Flow runtime will not attempt to load documents via
	// TransactionRequest.Load, and also disables re-use of cached documents
	// stored in prior transactions. Each stored document is exclusively
	// combined from updates processed by the runtime within the current
	// transaction only.
	//
	// This is appropriate for drivers over streams, WebHooks, and append-only
	// files.
	//
	// For example, given a collection which reduces a sum count for each key,
	// its materialization will produce a stream of delta updates to the count,
	// such that a reader of the stream will arrive at the correct total count.
	DeltaUpdates         bool     `protobuf:"varint,3,opt,name=delta_updates,json=deltaUpdates,proto3" json:"delta_updates,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}