casng

type BatchingUploader struct {
	// contains filtered or unexported fields
}

type GRPCConfig struct {
	// ConcurrentCallsLimit sets the upper bound of concurrent calls.
	// Must be > 0.
	ConcurrentCallsLimit int

	// BytesLimit sets the upper bound for the size of each request.
	// Comparisons against this value may not be exact due to padding and other serialization naunces.
	// Clients should choose a value that is sufficiently lower than the max size limit for the corresponding gRPC connection.
	// Any blob that does not fit in a batching request based on this value will be streamed using the ByteStream API.
	// Must be > 0.
	// This is defined as int rather than int64 because gRPC uses int for its limit.
	BytesLimit int

	// ItemsLimit sets the upper bound for the number of items per request.
	// Must be > 0.
	ItemsLimit int

	// BundleTimeout sets the maximum duration a call is delayed while bundling.
	// Bundling is used to ammortize the cost of a gRPC call over time. Instead of sending
	// many requests with few items, bundling attempt to maximize the number of items sent in a single request.
	// This includes waiting for a bit to see if more items are requested.
	BundleTimeout time.Duration

	// Timeout sets the upper bound of the total time spent processing a request.
	// For streaming calls, this applies to each Send/Recv call individually, not the whole streaming session.
	// This does not take into account the time it takes to abort the request upon timeout.
	Timeout time.Duration

	// RetryPolicy sets the retry policy for calls using this config.
	RetryPolicy retry.BackoffPolicy

	// RetryPredicate is called to determine if the error is retryable. If not set, nothing is retried.
	RetryPredicate func(error) bool
}

type IOConfig struct {
	// ConcurrentWalksLimit sets the upper bound of concurrent filesystem tree traversals.
	// This affects the number of concurrent upload requests for the uploader since each one requires a walk.
	// Must be > 0.
	ConcurrentWalksLimit int

	// OpenFilesLimit sets the upper bound for the number of files being simultanuously processed.
	// Must be > 0.
	OpenFilesLimit int

	// OpenLargeFilesLimit sets the upper bound for the number of large files being simultanuously processed.
	//
	// This value counts towards open files. I.e. the following inequality is always effectively true:
	// OpenFilesLimit >= OpenLargeFilesLimit
	// Must be > 0.
	OpenLargeFilesLimit int

	// SmallFileSizeThreshold sets the upper bound (inclusive) for the file size to be considered a small file.
	//
	// Files that are larger than this value (medium and large files) are uploaded via the streaming API.
	//
	// Small files are buffered entirely in memory and uploaded via the batching API.
	// However, it is still possible for a file to be small in size, but still results in a request that is larger than the gRPC size limit.
	// In that case, the file is uploaded via the streaming API instead.
	//
	// The amount of memory used to buffer files is affected by this value and OpenFilesLimit as well as bundling limits for gRPC.
	// The uploader will stop buffering once the OpenFilesLimit is reached, before which the number of buffered files is bound by
	// the number of blobs buffered for uploading (and whatever the GC hasn't freed yet).
	// In the extreme case, the number of buffered bytes for small files (not including streaming buffers) equals
	// the concurrency limit for the upload gRPC call, times the bytes limit per call, times this value.
	// Note that the amount of memory used to buffer bytes of a generated proto messages is not included in this estimate.
	//
	// Must be >= 0.
	SmallFileSizeThreshold int64

	// LargeFileSizeThreshold sets the lower bound (inclusive) for the file size to be considered a large file.
	// Such files are uploaded in chunks using the file streaming API.
	// Must be >= 0.
	LargeFileSizeThreshold int64

	// CompressionSizeThreshold sets the lower bound for the chunk size before it is subject to compression.
	// A value of 0 enables compression for any chunk size. To disable compression, use math.MaxInt64.
	// Must >= 0.
	CompressionSizeThreshold int64

	// BufferSize sets the buffer size for IO read/write operations.
	// Must be > 0.
	BufferSize int

	// OptimizeForDiskLocality enables sorting files by path before they are written to disk to optimize for disk locality.
	// Assuming files under the same directory are located close to each other on disk, then such files are batched together.
	OptimizeForDiskLocality bool
}

type MissingBlobsResponse struct {
	Digest  digest.Digest
	Missing bool
	Err     error
}

type Stats struct {
	// BytesRequested is the total number of bytes in a request.
	// It does not necessarily equal the total number of bytes uploaded/downloaded.
	BytesRequested int64

	// LogicalBytesMoved is the amount of BytesRequested that was processed.
	// It cannot be larger than BytesRequested, but may be smaller in case of a partial response.
	// The quantity is more granular for streaming than it is for batching. In streaming, it is an increment of the buffer size.
	// For batching, it is a sum of the size of items that were batched.
	LogicalBytesMoved int64

	// TotalBytesMoved is the total number of bytes moved over the wire.
	// This may not be accurate since a gRPC call may be interrupted in which case this number may be higher than the real one.
	// It may be larger than (retries) or smaller than BytesRequested (compression, cache hits or partial response).
	TotalBytesMoved int64

	// EffectiveBytesMoved is the total number of bytes moved over the wire, excluding retries.
	// This may not be accurate since a gRPC call may be interrupted in which case this number may be higher than the real one.
	// For failures, this is reported as 0.
	// It may be higher than BytesRequested (compression headers), but never higher than TotalBytesMoved.
	EffectiveBytesMoved int64

	// LogicalBytesCached is the total number of bytes not moved over the wire due to caching (either remotely or locally).
	// For failures, this is reported as 0.
	LogicalBytesCached int64

	// LogicalBytesStreamed is the total number of logical bytes moved by the streaming API.
	// It may be larger than (retries) or smaller than (cache hits or partial response) than the requested size.
	// For failures, this is reported as 0.
	LogicalBytesStreamed int64

	// LogicalBytesBatched is the total number of logical bytes moved by the batching API.
	// It may be larger than (retries) or smaller than (cache hits or partial response) the requested size.
	// For failures, this is reported as 0.
	LogicalBytesBatched int64

	// InputFileCount is the number of processed regular files.
	InputFileCount int64

	// InputDirCount is the number of processed directories.
	InputDirCount int64

	// InputSymlinkCount is the number of processed symlinks (not the number of symlinks in the uploaded merkle tree which may be lower).
	InputSymlinkCount int64

	// CacheHitCount is the number of cache hits.
	CacheHitCount int64

	// CacheMissCount is the number of cache misses.
	CacheMissCount int64

	// DigestCount is the number of processed digests.
	// The counter is incremened regardless of digestion failures.
	DigestCount int64

	// BatchedCount is the number of batched files.
	BatchedCount int64

	// StreamedCount is the number of streamed files.
	// For methods that accept bytes, the value is 1 upon success, 0 otherwise.
	StreamedCount int64
}

type StreamingUploader struct {
	// contains filtered or unexported fields
}

type UploadRequest struct {
	// Digest is for pre-digested requests. This digest is trusted to be the one for the associated Bytes or Path.
	//
	// If not set, it will be calculated.
	// If set, it implies that this request is a single blob. I.e. either Bytes is set or Path is a regular file and both SymlinkOptions and Exclude are ignored.
	Digest digest.Digest

	// Bytes is meant for small blobs. Using a large slice of bytes might cause memory thrashing.
	//
	// If Bytes is nil, BytesFileMode is ignored and Path is used for traversal.
	// If Bytes is not nil (may be empty), Path is used as the corresponding path for the bytes content and is not used for traversal.
	Bytes []byte

	// BytesFileMode describes the bytes content. It is ignored if Bytes is not set.
	BytesFileMode fs.FileMode

	// Path is used to access and read files if Bytes is nil. Otherwise, Bytes is assumed to be the paths content (even if empty).
	//
	// This must not be equal to impath.Root since this is considered a zero value (Path not set).
	// If Bytes is not nil and Path is not set, a node cannot be constructed and therefore no node is cached.
	Path impath.Absolute

	// SymlinkOptions are used to handle symlinks when Path is set and Bytes is not.
	SymlinkOptions slo.Options

	// Exclude is used to exclude paths during traversal when Path is set and Bytes is not.
	//
	// The filter ID is used in the keys of the node cache, even when Bytes is set.
	// Using the same ID for effectively different filters will cause erroneous cache hits.
	// Using a different ID for effectively identical filters will reduce cache hit rates and increase digestion compute cost.
	Exclude walker.Filter
	// contains filtered or unexported fields
}

type UploadResponse struct {
	// Digest identifies the blob associated with this response.
	// May be empty (created from an empty byte slice or from a composite literal), in which case Err is set.
	Digest digest.Digest

	// Stats may be zero if this response has not been updated yet. It should be ignored if Err is set.
	// If this response has been processed, then either CacheHitCount or CacheHitMiss is not zero.
	Stats Stats

	// Err indicates the error encountered while processing the request associated with Digest.
	// If set, Stats should be ignored.
	Err error
	// contains filtered or unexported fields
}