Documentation ¶
Index ¶
Constants ¶
const BATCH_SIZE int = 10000
const MAX_BACKLOG int = 10000
Variables ¶
This section is empty.
Functions ¶
func UserCacheDir ¶
func UserCacheDir() string
UserCacheDir returns the cache directory for the current user. In order, the following are checked: 1. WANDB_CACHE_DIR environment variable 2. Platform-specific default home directory 3. ./.wandb-cache/wandb
Types ¶
type ArtifactBuilder ¶
type ArtifactBuilder struct {
// contains filtered or unexported fields
}
func NewArtifactBuilder ¶
func NewArtifactBuilder(artifactRecord *service.ArtifactRecord) *ArtifactBuilder
func (*ArtifactBuilder) AddData ¶
func (b *ArtifactBuilder) AddData(name string, dataMap map[string]interface{}) error
func (*ArtifactBuilder) GetArtifact ¶
func (b *ArtifactBuilder) GetArtifact() *service.ArtifactRecord
type ArtifactDownloader ¶
type ArtifactDownloader struct { // Resources Ctx context.Context GraphqlClient graphql.Client DownloadManager filetransfer.FileTransferManager FileCache Cache // Input ArtifactID string DownloadRoot string AllowMissingReferences bool // Currently unused SkipCache bool // Currently unused PathPrefix string // Currently unused }
func NewArtifactDownloader ¶
func NewArtifactDownloader( ctx context.Context, graphQLClient graphql.Client, downloadManager filetransfer.FileTransferManager, artifactID string, downloadRoot string, allowMissingReferences bool, skipCache bool, pathPrefix string, ) *ArtifactDownloader
func (*ArtifactDownloader) Download ¶
func (ad *ArtifactDownloader) Download() (rerr error)
type ArtifactLinker ¶
type ArtifactLinker struct { Ctx context.Context Logger *observability.CoreLogger LinkArtifact *service.LinkArtifactRecord GraphqlClient graphql.Client }
func (*ArtifactLinker) Link ¶
func (al *ArtifactLinker) Link() error
type ArtifactSaver ¶
type ArtifactSaver struct { // Resources. Ctx context.Context Logger *observability.CoreLogger GraphqlClient graphql.Client FileTransferManager filetransfer.FileTransferManager FileCache Cache // Input. Artifact *service.ArtifactRecord HistoryStep int64 StagingDir string // contains filtered or unexported fields }
func NewArtifactSaver ¶
func NewArtifactSaver( ctx context.Context, logger *observability.CoreLogger, graphQLClient graphql.Client, uploadManager filetransfer.FileTransferManager, artifact *service.ArtifactRecord, historyStep int64, stagingDir string, ) ArtifactSaver
type Cache ¶
type Cache interface { AddFile(path string) (string, error) AddFileAndCheckDigest(path string, digest string) error RestoreTo(entry ManifestEntry, dst string) bool Write(src io.Reader) (string, error) }
func NewFileCache ¶
func NewHashOnlyCache ¶
func NewHashOnlyCache() Cache
type FileCache ¶
type FileCache struct {
// contains filtered or unexported fields
}
func (*FileCache) AddFileAndCheckDigest ¶
AddFileAndCheckDigest copies a file into the cache. If a digest is provided, it also verifies that the file's MD5 hash matches the digest.
func (*FileCache) RestoreTo ¶
func (c *FileCache) RestoreTo(entry ManifestEntry, dst string) bool
RestoreTo tries to restore the file referenced in a manifest entry to the given destination.
If the file exists, it will be hashed and overwritten if the hash is different; if the hash is correct, RestoreTo leaves it alone and returns true.
type HashOnlyCache ¶
type HashOnlyCache struct{}
HashOnlyCache never writes data but still computes and compares hashes.
func (*HashOnlyCache) AddFile ¶
func (c *HashOnlyCache) AddFile(path string) (string, error)
AddFile computes the base-64 MD5 hash of the file and returns it. It doesn't write.
func (*HashOnlyCache) AddFileAndCheckDigest ¶
func (c *HashOnlyCache) AddFileAndCheckDigest(path string, digest string) error
func (*HashOnlyCache) RestoreTo ¶
func (c *HashOnlyCache) RestoreTo(entry ManifestEntry, dst string) bool
RestoreTo is the same as the FileCache version, but it doesn't copy the file, so it always returns false if the file is missing.
type Manifest ¶
type Manifest struct { Version int32 `json:"version"` StoragePolicy string `json:"storagePolicy"` StoragePolicyConfig StoragePolicyConfig `json:"storagePolicyConfig"` Contents map[string]ManifestEntry `json:"contents"` }
func NewManifestFromProto ¶
func NewManifestFromProto(proto *service.ArtifactManifest) (Manifest, error)
func (*Manifest) GetManifestEntryFromArtifactFilePath ¶
func (m *Manifest) GetManifestEntryFromArtifactFilePath(path string) (ManifestEntry, error)
type ManifestEntry ¶
type ManifestEntry struct { // Fields from the service.ArtifactManifestEntry proto. Digest string `json:"digest"` Ref *string `json:"ref,omitempty"` Size int64 `json:"size"` LocalPath *string `json:"-"` BirthArtifactID *string `json:"birthArtifactID"` SkipCache bool `json:"-"` Extra map[string]interface{} `json:"extra,omitempty"` // Added and used during download. DownloadURL *string `json:"-"` }
type StoragePolicyConfig ¶
type StoragePolicyConfig struct {
StorageLayout string `json:"storageLayout"`
}