sources

type Chunk struct {
	// SourceName is the name of the Source that produced the chunk.
	SourceName string
	// SourceID is the ID of the source that the Chunk originated from.
	SourceID SourceID
	// JobID is the ID of the job that the Chunk originated from.
	JobID JobID
	// SourceType is the type of Source that produced the chunk.
	SourceType sourcespb.SourceType
	// SourceMetadata holds the context of where the Chunk was found.
	SourceMetadata *source_metadatapb.MetaData

	// Data is the data to decode and scan.
	Data []byte
	// Verify specifies whether any secrets in the Chunk should be verified.
	Verify bool
}

type ChunkError struct {
	// contains filtered or unexported fields
}

type ChunkFunc func(chunk *Chunk) error

type ChunkReader func(ctx context.Context, reader io.Reader) <-chan ChunkResult

type ChunkReporter interface {
	ChunkOk(ctx context.Context, chunk Chunk) error
	ChunkErr(ctx context.Context, err error) error
}

type ChunkResult struct {
	// contains filtered or unexported fields
}

type ChunkingTarget struct {
	// QueryCriteria represents specific parameters or conditions to target the chunking process.
	QueryCriteria *source_metadatapb.MetaData
}

type CommonSourceUnit struct {
	ID string `json:"source_unit_id"`
}

type CommonSourceUnitUnmarshaller struct{}

type ConfigOption func(*chunkReaderConfig)

type Fatal struct {
	// contains filtered or unexported fields
}

type FilesystemConfig struct {
	// Paths is the list of files and directories to scan.
	Paths []string
	// Filter is the filter to use to scan the source.
	Filter *common.Filter
}

type GCSConfig struct {
	// CloudCred determines whether to use cloud credentials.
	// This can NOT be used with a secret.
	CloudCred,

	WithoutAuth bool
	// ApiKey is the API key to use to authenticate with the source.
	ApiKey,

	ProjectID,

	ServiceAccount string
	// MaxObjectSize is the maximum object size to scan.
	MaxObjectSize int64
	// Concurrency is the number of concurrent workers to use to scan the source.
	Concurrency int
	// IncludeBuckets is a list of buckets to include in the scan.
	IncludeBuckets,

	ExcludeBuckets,

	IncludeObjects,

	ExcludeObjects []string
}

type GitConfig struct {
	// RepoPath is the path to the repository to scan.
	RepoPath,

	HeadRef,

	BaseRef string
	// MaxDepth is the maximum depth to scan the source.
	MaxDepth int
	// Bare is an indicator to handle bare repositories properly.
	Bare bool
	// Filter is the filter to use to scan the source.
	Filter *common.Filter
	// ExcludeGlobs is a list of globs to exclude from the scan.
	// This differs from the Filter exclusions as ExcludeGlobs is applied at the `git log -p` level
	ExcludeGlobs []string
}

type GithubConfig struct {
	// Endpoint is the endpoint of the source.
	Endpoint,

	Token string
	// IncludeForks indicates whether to include forks in the scan.
	IncludeForks,

	IncludeMembers bool
	// Concurrency is the number of concurrent workers to use to scan the source.
	Concurrency int
	// Repos is the list of repositories to scan.
	Repos,

	Orgs,

	ExcludeRepos,

	IncludeRepos []string
	// Filter is the filter to use to scan the source.
	Filter *common.Filter
	// IncludeIssueComments indicates whether to include GitHub issue comments in the scan.
	IncludeIssueComments,

	IncludePullRequestComments,

	IncludeGistComments bool
}

type GitlabConfig struct {
	// Endpoint is the endpoint of the source.
	Endpoint,

	Token string
	// Repos is the list of repositories to scan.
	Repos []string
	// Filter is the filter to use to scan the source.
	Filter *common.Filter
}

type JobID int64

type JobProgress struct {
	// Unique identifiers for this job.
	JobID      JobID
	SourceID   SourceID
	SourceName string
	// contains filtered or unexported fields
}

type JobProgressHook interface {
	// Start and End marks the overall start and end time for this job.
	Start(JobProgressRef, time.Time)
	End(JobProgressRef, time.Time)
	// StartEnumerating and EndEnumerating marks the start and end time for
	// calling the source's Enumerate method. If the source does not
	// support enumeration these methods will never be called.
	StartEnumerating(JobProgressRef, time.Time)
	EndEnumerating(JobProgressRef, time.Time)
	// StartUnitChunking and EndUnitChunking marks the start and end time
	// for calling the source's ChunkUnit method for a given unit. If the
	// source does not support enumeration these methods will never be
	// called.
	StartUnitChunking(JobProgressRef, SourceUnit, time.Time)
	EndUnitChunking(JobProgressRef, SourceUnit, time.Time)
	// ReportError is called when any general error is encountered, usually
	// from enumeration.
	ReportError(JobProgressRef, error)
	// ReportUnit is called when a unit has been enumerated. If the source
	// does not support enumeration this method will never be called.
	ReportUnit(JobProgressRef, SourceUnit)
	// ReportChunk is called when a chunk has been produced for the given
	// unit. The unit will be nil if the source does not support
	// enumeration.
	ReportChunk(JobProgressRef, SourceUnit, *Chunk)
	// Finish marks the job as done.
	Finish(JobProgressRef)
}

type JobProgressMetrics struct {
	StartTime time.Time
	EndTime   time.Time
	// Total number of units found by the Source.
	TotalUnits uint64
	// Total number of units that have finished chunking.
	FinishedUnits uint64
	// Total number of chunks produced. This metric updates before the
	// chunk is sent on the output channel.
	TotalChunks uint64
	// All errors encountered.
	Errors []error
	// Set to true if the source supports enumeration and has finished
	// enumerating. If the source does not support enumeration, this field
	// is always false.
	DoneEnumerating bool

	// Progress information reported by the source.
	SourcePercent           int64
	SourceMessage           string
	SourceEncodedResumeInfo string
	SourceSectionsCompleted int32
	SourceSectionsRemaining int32
}

type JobProgressRef struct {
	JobID      JobID
	SourceID   SourceID
	SourceName string
	// contains filtered or unexported fields
}

type Progress struct {
	PercentComplete   int64
	Message           string
	EncodedResumeInfo string
	SectionsCompleted int32
	SectionsRemaining int32
	// contains filtered or unexported fields
}

type S3Config struct {
	// CloudCred determines whether to use cloud credentials.
	// This can NOT be used with a secret.
	CloudCred bool
	// Key is any key to use to authenticate with the source.
	Key,

	Secret,

	SessionToken string
	// Buckets is the list of buckets to scan.
	Buckets []string
	// Roles is the list of Roles to use.
	Roles []string
	// MaxObjectSize is the maximum object size to scan.
	MaxObjectSize int64
}

type ScanErrors struct {
	// contains filtered or unexported fields
}

type Source interface {
	// Type returns the source type, used for matching against configuration and jobs.
	Type() sourcespb.SourceType
	// SourceID returns the initialized source ID used for tracking relationships in the DB.
	SourceID() SourceID
	// JobID returns the initialized job ID used for tracking relationships in the DB.
	JobID() JobID
	// Init initializes the source.
	Init(aCtx context.Context, name string, jobId JobID, sourceId SourceID, verify bool, connection *anypb.Any, concurrency int) error
	// Chunks emits data over a channel which is then decoded and scanned for secrets.
	// By default, data is obtained indiscriminately. However, by providing one or more
	// ChunkingTarget parameters, the caller can direct the function to retrieve
	// specific chunks of data. This targeted approach allows for efficient and
	// intentional data processing, beneficial when verifying or rechecking specific data points.
	Chunks(ctx context.Context, chunksChan chan *Chunk, targets ...ChunkingTarget) error
	// GetProgress is the completion progress (percentage) for Scanned Source.
	GetProgress() *Progress
}

type SourceID int64

type SourceManager struct {
	// contains filtered or unexported fields
}

type SourceUnit interface {
	// SourceUnitID uniquely identifies a source unit.
	SourceUnitID() string
}

type SourceUnitChunker interface {
	// ChunkUnit creates 0 or more chunks from a unit, reporting them or
	// any errors to the ChunkReporter. An error should only be returned
	// from this method in the case of context cancellation, fatal source
	// errors, or errors returned by the reporter. All other errors related
	// to unit chunking are tracked by the ChunkReporter.
	ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
}

type SourceUnitEnumChunker interface {
	SourceUnitEnumerator
	SourceUnitChunker
}

type SourceUnitEnumerator interface {
	// Enumerate creates 0 or more units from an initialized source,
	// reporting them or any errors to the UnitReporter. This method is
	// synchronous but can be called in a goroutine to support concurrent
	// enumeration and chunking. An error should only be returned from this
	// method in the case of context cancellation, fatal source errors, or
	// errors returned by the reporter All other errors related to unit
	// enumeration are tracked by the UnitReporter.
	Enumerate(ctx context.Context, reporter UnitReporter) error
}

type SourceUnitUnmarshaller interface {
	UnmarshalSourceUnit(data []byte) (SourceUnit, error)
}

type SyslogConfig struct {
	// Address used to connect to the source.
	Address,

	Protocol,

	CertPath,

	Format,

	KeyPath string
	// Concurrency is the number of concurrent workers to use to scan the source.
	Concurrency int
}

type UnitReporter interface {
	UnitOk(ctx context.Context, unit SourceUnit) error
	UnitErr(ctx context.Context, err error) error
}

type Validator interface {
	Validate(ctx context.Context) []error
}