dsfs

package
v0.0.0-...-5ddb3d2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 13, 2022 License: GPL-3.0 Imports: 30 Imported by: 0

Documentation

Overview

Package dsfs glues datsets to cafs (content-addressed-file-system)

Index

Constants

This section is empty.

Variables

View Source
var (

	// ErrNoChanges indicates a save failed because no values changed, and
	// force-saving was disabled
	ErrNoChanges = fmt.Errorf("no changes")
	// ErrNoReadme is the error for asking a dataset without a readme component
	// for readme info
	ErrNoReadme = fmt.Errorf("this dataset has no readme component")
	// ErrNoTransform is the error for asking a dataset without a tranform
	// component for transform info
	ErrNoTransform = fmt.Errorf("this dataset has no transform component")
	// ErrNoViz is the error for asking a dataset without a viz component for
	// viz info
	ErrNoViz = fmt.Errorf("this dataset has no viz component")
	// ErrStrictMode indicates a dataset failed validation when it is required to
	// pass (Structure.Strict == true)
	ErrStrictMode = fmt.Errorf("dataset body did not validate against schema in strict-mode")
)
View Source
var (
	// BodySizeSmallEnoughToDiff sets how small a body must be to generate a message from it
	BodySizeSmallEnoughToDiff = 20000000 // 20M or less is small
	// OpenFileTimeoutDuration determines the maximium amount of time to wait for
	// a Filestore to open a file. Some filestores (like IPFS) fallback to a
	// network request when it can't find a file locally. Setting a short timeout
	// prevents waiting for a slow network response, at the expense of leaving
	// files unresolved.
	// TODO (b5) - allow -1 duration as a sentinel value for no timeout
	OpenFileTimeoutDuration = time.Millisecond * 700
)
View Source
var Timestamp = func() time.Time {
	return time.Now().UTC()
}

Timestamp is an function for getting commit timestamps timestamps MUST be stored in UTC time zone

Functions

func CreateDataset

func CreateDataset(
	ctx context.Context,
	source qfs.Filesystem,
	destination qfs.Filesystem,
	pub event.Publisher,
	ds *dataset.Dataset,
	prev *dataset.Dataset,
	pk crypto.PrivKey,
	sw SaveSwitches,
) (string, error)

CreateDataset writes a dataset to a provided store. Store is where we're going to store the data Dataset to be saved Prev is the previous version or nil if there isn't one Pk is the private key for cryptographically signing Sw is switches that control how the save happens Returns the immutable path if no error

func DerefCommit

func DerefCommit(ctx context.Context, store qfs.Filesystem, ds *dataset.Dataset) error

DerefCommit derferences a dataset's Commit element if required should be a no-op if ds.Commit is nil or isn't a reference

func DerefDataset

func DerefDataset(ctx context.Context, store qfs.Filesystem, ds *dataset.Dataset) error

DerefDataset attempts to fully dereference a dataset

func DerefMeta

func DerefMeta(ctx context.Context, store qfs.Filesystem, ds *dataset.Dataset) error

DerefMeta derferences a dataset's transform element if required should be a no-op if ds.Meta is nil or isn't a reference

func DerefReadme

func DerefReadme(ctx context.Context, store qfs.Filesystem, ds *dataset.Dataset) error

DerefReadme dereferences a dataset's Readme element if required no-op if ds.Readme is nil or isn't a reference

func DerefStats

func DerefStats(ctx context.Context, store qfs.Filesystem, ds *dataset.Dataset) error

DerefStats derferences a dataset's stats component if required no-op if ds.Stats is nil or isn't a reference

func DerefStructure

func DerefStructure(ctx context.Context, store qfs.Filesystem, ds *dataset.Dataset) error

DerefStructure derferences a dataset's structure element if required should be a no-op if ds.Structure is nil or isn't a reference

func DerefTransform

func DerefTransform(ctx context.Context, store qfs.Filesystem, ds *dataset.Dataset) error

DerefTransform derferences a dataset's transform element if required should be a no-op if ds.Structure is nil or isn't a reference

func DerefViz

func DerefViz(ctx context.Context, store qfs.Filesystem, ds *dataset.Dataset) error

DerefViz dereferences a dataset's Viz element if required no-op if ds.Viz is nil or isn't a reference

func EnsureCommitTitleAndMessage

func EnsureCommitTitleAndMessage(ctx context.Context, fs qfs.Filesystem, ds, prev *dataset.Dataset, bodyAct BodyAction, fileHint string, forceIfNoChanges bool) error

EnsureCommitTitleAndMessage creates the commit and title, message, skipping if both title and message are set. If no values are provided a commit description is generated by examining changes between the two versions

func GetHashBase

func GetHashBase(in string) string

GetHashBase strips paths to return just the hash

func JSONFile

func JSONFile(name string, m json.Marshaler) (fs.File, error)

JSONFile is a convenenience method for creating a file from a json.Marshaller

func LoadBody

func LoadBody(ctx context.Context, fs qfs.Filesystem, ds *dataset.Dataset) (qfs.File, error)

LoadBody loads the data this dataset points to from the store

func LoadDataset

func LoadDataset(ctx context.Context, store qfs.Filesystem, path string) (*dataset.Dataset, error)

LoadDataset reads a dataset from a cafs and dereferences structure, transform, and commitMsg if they exist, returning a fully-hydrated dataset

func LoadDatasetRefs

func LoadDatasetRefs(ctx context.Context, fs qfs.Filesystem, path string) (*dataset.Dataset, error)

LoadDatasetRefs reads a dataset from a content addressed filesystem without dereferencing components

func LoadReadmeScript

func LoadReadmeScript(ctx context.Context, fs qfs.Filesystem, dspath string) (qfs.File, error)

LoadReadmeScript loads script data from a dataset path if the given dataset has a readme script is specified the returned qfs.File will be the value of dataset.Readme.ScriptPath

func NewFileWithInfo

func NewFileWithInfo(fi fs.FileInfo, r io.Reader) (fs.File, error)

NewFileWithInfo creates a new open file with provided file information

func NewMemfileBytes

func NewMemfileBytes(name string, data []byte) fs.File

NewMemfileBytes creates a file from a byte slice

func NewMemfileReader

func NewMemfileReader(name string, r io.Reader) fs.File

NewMemfileReader creates a file from an io.Reader

func PackageFilepath

func PackageFilepath(fs qfs.Filesystem, path string, pf PackageFile) string

PackageFilepath returns the path to a package file for a given base path It relies relies on package storage conventions and qfs.Filesystem path prefixes If you supply a path that does not match the filestore's naming conventions will return an invalid path

func WriteDataset

func WriteDataset(
	ctx context.Context,
	src qfs.Filesystem,
	dst qfs.Filesystem,
	prev *dataset.Dataset,
	ds *dataset.Dataset,
	publisher event.Publisher,
	pk crypto.PrivKey,
	sw SaveSwitches,
) (string, error)

WriteDataset persists a datasets to a destination filesystem

Types

type BodyAction

type BodyAction string

BodyAction represents the action that should be taken to understand how the body changed

const (
	// BodyDefault is the default action: compare them to get how much changed
	BodyDefault BodyAction = "default"
	// BodySame means that the bodies are the same, no need to compare
	BodySame BodyAction = "same"
	// BodyTooBig means the body is too big to directly compare, and should use
	// some other method
	BodyTooBig BodyAction = "too_big"
)

type PackageFile

type PackageFile int

PackageFile specifies the different types of files that are stored in a package

const (
	// PackageFileUnknown is the default package file, which
	// should be erroneous, as there is no sensible default
	// for PackageFile
	PackageFileUnknown PackageFile = iota
	// PackageFileDataset is the main dataset.json file
	// that contains all dataset metadata, and is the only
	// required file to constitute a dataset
	PackageFileDataset
	// PackageFileStructure isolates this dataset's structure
	// in it's own file
	PackageFileStructure
	// PackageFileAbstract is the abstract verion of
	// structure
	PackageFileAbstract
	// PackageFileResources lists the resource datasets
	// that went into creating a dataset
	// TODO - I think this can be removed now that Transform exists
	PackageFileResources
	// PackageFileCommit isolates the user-entered
	// documentation of the changes to this dataset's history
	PackageFileCommit
	// PackageFileTransform isloates the concrete transform that
	// generated this dataset
	PackageFileTransform
	// PackageFileAbstractTransform is the abstract version of
	// the operation performed to create this dataset
	PackageFileAbstractTransform
	// PackageFileMeta encapsulates human-readable metadata
	PackageFileMeta
	// PackageFileViz isolates the data related to representing a dataset as a
	// visualization
	PackageFileViz
	// PackageFileVizScript is the viz template
	PackageFileVizScript
	// PackageFileRenderedViz is the rendered visualization of the dataset
	PackageFileRenderedViz
	// PackageFileReadme connects readme data to the dataset package
	PackageFileReadme
	// PackageFileReadmeScript is the raw readme of the dataset
	PackageFileReadmeScript
	// PackageFileRenderedReadme is the rendered readme of the dataset
	PackageFileRenderedReadme
	// PackageFileStats isolates the statistical metadata component
	PackageFileStats
)

func (PackageFile) Filename

func (p PackageFile) Filename() string

Filename gives the canonical filename for a PackageFile

func (PackageFile) String

func (p PackageFile) String() string

String implements the io.Stringer interface for PackageFile

type SaveSwitches

type SaveSwitches struct {
	// Use a custom timestamp, defaults to time.Now if unset
	Time time.Time
	// Replace is whether the save is a full replacement or a set of patches to previous
	Replace bool
	// Pin is whether the dataset should be pinned
	Pin bool
	// ConvertFormatToPrev is whether the body should be converted to match the previous format
	ConvertFormatToPrev bool
	// ForceIfNoChanges is whether the save should be forced even if no changes are detected
	ForceIfNoChanges bool
	// ShouldRender is deprecated, controls whether viz should be rendered
	ShouldRender bool
	// NewName is whether a new dataset should be created, guaranteeing there's no previous version
	NewName bool
	// FileHint is a hint for what file is used for creating this dataset
	FileHint string
	// Drop is a string of components to remove before saving
	Drop string
	// contains filtered or unexported fields
}

SaveSwitches represents options for saving a dataset

Directories

Path Synopsis
Package dstest defines an interface for reading test cases from static files leveraging directories of test dataset input files & expected output files
Package dstest defines an interface for reading test cases from static files leveraging directories of test dataset input files & expected output files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL