artifacts

package
v0.0.0-...-d9a1f69 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 21, 2024 License: MIT Imports: 23 Imported by: 0

Documentation

Index

Constants

View Source
const BATCH_SIZE int = 10000
View Source
const MAX_BACKLOG int = 10000

Variables

This section is empty.

Functions

func UserCacheDir

func UserCacheDir() string

UserCacheDir returns the cache directory for the current user. In order, the following are checked: 1. WANDB_CACHE_DIR environment variable 2. Platform-specific default home directory 3. ./.wandb-cache/wandb

Types

type ArtifactBuilder

type ArtifactBuilder struct {
	// contains filtered or unexported fields
}

func NewArtifactBuilder

func NewArtifactBuilder(artifactRecord *service.ArtifactRecord) *ArtifactBuilder

func (*ArtifactBuilder) AddData

func (b *ArtifactBuilder) AddData(name string, dataMap map[string]interface{}) error

func (*ArtifactBuilder) AddFile

func (b *ArtifactBuilder) AddFile(path string, name string) error

func (*ArtifactBuilder) GetArtifact

func (b *ArtifactBuilder) GetArtifact() *service.ArtifactRecord

type ArtifactDownloader

type ArtifactDownloader struct {
	// Resources
	Ctx             context.Context
	GraphqlClient   graphql.Client
	DownloadManager filetransfer.FileTransferManager
	FileCache       Cache
	// Input
	ArtifactID             string
	DownloadRoot           string
	AllowMissingReferences bool   // Currently unused
	SkipCache              bool   // Currently unused
	PathPrefix             string // Currently unused
}

func NewArtifactDownloader

func NewArtifactDownloader(
	ctx context.Context,
	graphQLClient graphql.Client,
	downloadManager filetransfer.FileTransferManager,
	artifactID string,
	downloadRoot string,
	allowMissingReferences bool,
	skipCache bool,
	pathPrefix string,
) *ArtifactDownloader

func (*ArtifactDownloader) Download

func (ad *ArtifactDownloader) Download() (rerr error)

type ArtifactLinker

type ArtifactLinker struct {
	Ctx           context.Context
	Logger        *observability.CoreLogger
	LinkArtifact  *service.LinkArtifactRecord
	GraphqlClient graphql.Client
}
func (al *ArtifactLinker) Link() error

type ArtifactSaver

type ArtifactSaver struct {
	// Resources.
	Ctx                 context.Context
	Logger              *observability.CoreLogger
	GraphqlClient       graphql.Client
	FileTransferManager filetransfer.FileTransferManager
	FileCache           Cache
	// Input.
	Artifact    *service.ArtifactRecord
	HistoryStep int64
	StagingDir  string
	// contains filtered or unexported fields
}

func NewArtifactSaver

func NewArtifactSaver(
	ctx context.Context,
	logger *observability.CoreLogger,
	graphQLClient graphql.Client,
	uploadManager filetransfer.FileTransferManager,
	artifact *service.ArtifactRecord,
	historyStep int64,
	stagingDir string,
) ArtifactSaver

func (*ArtifactSaver) Save

func (as *ArtifactSaver) Save(ch chan<- *service.Record) (artifactID string, rerr error)

type Cache

type Cache interface {
	AddFile(path string) (string, error)
	AddFileAndCheckDigest(path string, digest string) error
	RestoreTo(entry ManifestEntry, dst string) bool
	Write(src io.Reader) (string, error)
}

func NewFileCache

func NewFileCache(cacheDir string) Cache

func NewHashOnlyCache

func NewHashOnlyCache() Cache

type FileCache

type FileCache struct {
	// contains filtered or unexported fields
}

func (*FileCache) AddFile

func (c *FileCache) AddFile(path string) (string, error)

AddFile copies a file into the cache and returns the B64MD5 cache key.

func (*FileCache) AddFileAndCheckDigest

func (c *FileCache) AddFileAndCheckDigest(path string, digest string) error

AddFileAndCheckDigest copies a file into the cache. If a digest is provided, it also verifies that the file's MD5 hash matches the digest.

func (*FileCache) RestoreTo

func (c *FileCache) RestoreTo(entry ManifestEntry, dst string) bool

RestoreTo tries to restore the file referenced in a manifest entry to the given destination.

If the file exists, it will be hashed and overwritten if the hash is different; if the hash is correct, RestoreTo leaves it alone and returns true.

func (*FileCache) Write

func (c *FileCache) Write(src io.Reader) (string, error)

Write copies the contents of the reader to the cache and returns the B64MD5 cache key.

type HashOnlyCache

type HashOnlyCache struct{}

HashOnlyCache never writes data but still computes and compares hashes.

func (*HashOnlyCache) AddFile

func (c *HashOnlyCache) AddFile(path string) (string, error)

AddFile computes the base-64 MD5 hash of the file and returns it. It doesn't write.

func (*HashOnlyCache) AddFileAndCheckDigest

func (c *HashOnlyCache) AddFileAndCheckDigest(path string, digest string) error

func (*HashOnlyCache) RestoreTo

func (c *HashOnlyCache) RestoreTo(entry ManifestEntry, dst string) bool

RestoreTo is the same as the FileCache version, but it doesn't copy the file, so it always returns false if the file is missing.

func (*HashOnlyCache) Write

func (c *HashOnlyCache) Write(src io.Reader) (string, error)

Write computes and returns the B64MD5 cache key. It doesn't write any data.

type Manifest

type Manifest struct {
	Version             int32                    `json:"version"`
	StoragePolicy       string                   `json:"storagePolicy"`
	StoragePolicyConfig StoragePolicyConfig      `json:"storagePolicyConfig"`
	Contents            map[string]ManifestEntry `json:"contents"`
}

func NewManifestFromProto

func NewManifestFromProto(proto *service.ArtifactManifest) (Manifest, error)

func (*Manifest) GetManifestEntryFromArtifactFilePath

func (m *Manifest) GetManifestEntryFromArtifactFilePath(path string) (ManifestEntry, error)

func (*Manifest) WriteToFile

func (m *Manifest) WriteToFile() (filename string, digest string, size int64, rerr error)

type ManifestEntry

type ManifestEntry struct {
	// Fields from the service.ArtifactManifestEntry proto.
	Digest          string                 `json:"digest"`
	Ref             *string                `json:"ref,omitempty"`
	Size            int64                  `json:"size"`
	LocalPath       *string                `json:"-"`
	BirthArtifactID *string                `json:"birthArtifactID"`
	SkipCache       bool                   `json:"-"`
	Extra           map[string]interface{} `json:"extra,omitempty"`
	// Added and used during download.
	DownloadURL *string `json:"-"`
}

type StoragePolicyConfig

type StoragePolicyConfig struct {
	StorageLayout string `json:"storageLayout"`
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL