dload

package
v1.3.19 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 6, 2023 License: MIT Imports: 31 Imported by: 0

Documentation

Overview

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package cmn provides common low-level types and utilities for all aistore projects

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Index

Constants

View Source
const (
	DiffResolverSend = iota
	DiffResolverRecv
	DiffResolverDelete
	DiffResolverSkip
	DiffResolverErr
	DiffResolverEOF
)
View Source
const DownloadProgressInterval = 10 * time.Second
View Source
const PrefixJobID = "dnl-"

Variables

This section is empty.

Functions

func AbortReq

func AbortReq(jobID string) cmn.HreqArgs

called via ais/prxnotifs generic mechanism

func CompareObjects

func CompareObjects(lom *cluster.LOM, dst *DstElement) (equal bool, err error)

Use all available metadata including {size, version, ETag, MD5, CRC} to compare local object with its remote counterpart.

func IsType

func IsType(a string) bool

func ListJobs

func ListJobs(regex *regexp.Regexp, onlyActive bool) (any, int, error)

func NormalizeObjName

func NormalizeObjName(objName string) (string, error)

Removes everything that goes after '?', eg. "?query=key..." so it will not be part of final object name.

func ParseStartRequest

func ParseStartRequest(t cluster.Target, bck *meta.Bck, id string, dlb Body, xdl *Xact) (jobif, error)

func SetDB

func SetDB(dbdrv kvdb.Driver)

func Xreg

func Xreg()

Types

type AdminBody

type AdminBody struct {
	ID         string `json:"id"`
	Regex      string `json:"regex"`
	OnlyActive bool   `json:"only_active_tasks"` // Skips detailed info about tasks finished/errored
}

func (*AdminBody) Validate

func (b *AdminBody) Validate(requireID bool) error

type BackendBody

type BackendBody struct {
	Base
	Prefix string `json:"prefix"`
	Suffix string `json:"suffix"`
	Sync   bool   `json:"sync"`
}

func (*BackendBody) Describe

func (b *BackendBody) Describe() string

func (*BackendBody) Validate

func (b *BackendBody) Validate() error

type BackendResource

type BackendResource struct {
	ObjName string
}

type Base

type Base struct {
	Description      string  `json:"description"`
	Bck              cmn.Bck `json:"bucket"`
	Timeout          string  `json:"timeout"`
	ProgressInterval string  `json:"progress_interval"`
	Limits           Limits  `json:"limits"`
}

func (*Base) Validate

func (b *Base) Validate() error

type Body

type Body struct {
	Type Type `json:"type"`
	json.RawMessage
}

NOTE: Changing this structure requires changes in `MarshalJSON` and `UnmarshalJSON` methods.

func (Body) MarshalJSON

func (db Body) MarshalJSON() ([]byte, error)

func (*Body) UnmarshalJSON

func (db *Body) UnmarshalJSON(b []byte) error

type DiffResolver

type DiffResolver struct {
	// contains filtered or unexported fields
}

DiffResolver is entity that computes difference between two streams of objects. The streams are expected to be in sorted order.

func NewDiffResolver

func NewDiffResolver(ctx DiffResolverCtx) *DiffResolver

func (*DiffResolver) Abort

func (dr *DiffResolver) Abort(err error)

func (*DiffResolver) CloseDst

func (dr *DiffResolver) CloseDst()

func (*DiffResolver) CloseSrc

func (dr *DiffResolver) CloseSrc()

func (*DiffResolver) Next

func (dr *DiffResolver) Next() (DiffResolverResult, error)

func (*DiffResolver) PushDst

func (dr *DiffResolver) PushDst(v any)

func (*DiffResolver) PushSrc

func (dr *DiffResolver) PushSrc(v any)

func (*DiffResolver) Start

func (dr *DiffResolver) Start()

func (*DiffResolver) Stop

func (dr *DiffResolver) Stop()

func (*DiffResolver) Stopped

func (dr *DiffResolver) Stopped() bool

type DiffResolverCtx

type DiffResolverCtx interface {
	CompareObjects(*cluster.LOM, *DstElement) (bool, error)
	IsObjFromRemote(*cluster.LOM) (bool, error)
}

type DiffResolverResult

type DiffResolverResult struct {
	Action uint8
	Src    *cluster.LOM
	Dst    *DstElement
	Err    error
}

type DlPostResp

type DlPostResp struct {
	ID string `json:"id"`
}

Download POST result returned to the user

type DstElement

type DstElement struct {
	ObjName string
	Version string
	Link    string
}

type Job

type Job struct {
	ID            string    `json:"id"`
	XactID        string    `json:"xaction_id"`
	Description   string    `json:"description"`
	StartedTime   time.Time `json:"started_time"`
	FinishedTime  time.Time `json:"finished_time"`
	FinishedCnt   int       `json:"finished_cnt"`
	ScheduledCnt  int       `json:"scheduled_cnt"` // tasks being processed or already processed by dispatched
	SkippedCnt    int       `json:"skipped_cnt"`   // number of tasks skipped
	ErrorCnt      int       `json:"error_cnt"`
	Total         int       `json:"total"`          // total number of tasks, negative if unknown
	AllDispatched bool      `json:"all_dispatched"` // if true, dispatcher has already scheduled all tasks for given job
	Aborted       bool      `json:"aborted"`
}

func (*Job) Aggregate

func (j *Job) Aggregate(rhs *Job)

func (*Job) DoneCnt

func (j *Job) DoneCnt() int

DoneCnt returns number of tasks that have finished (either successfully or with an error).

func (*Job) JobFinished

func (j *Job) JobFinished() bool

func (*Job) JobRunning

func (j *Job) JobRunning() bool

func (*Job) PendingCnt

func (j *Job) PendingCnt() int

PendingCnt returns number of tasks which are currently being processed.

func (*Job) String

func (j *Job) String() string

func (*Job) TotalCnt

func (j *Job) TotalCnt() int

type JobInfos

type JobInfos []*Job

func (JobInfos) Len

func (d JobInfos) Len() int

func (JobInfos) Less

func (d JobInfos) Less(i, j int) bool

func (JobInfos) Swap

func (d JobInfos) Swap(i, j int)

type Limits

type Limits struct {
	Connections  int `json:"connections"`
	BytesPerHour int `json:"bytes_per_hour"`
}

type MultiBody

type MultiBody struct {
	Base
	ObjectsPayload any `json:"objects"`
}

func (*MultiBody) Describe

func (b *MultiBody) Describe() string

func (*MultiBody) ExtractPayload

func (b *MultiBody) ExtractPayload() (cos.StrKVs, error)

func (*MultiBody) String

func (b *MultiBody) String() string

func (*MultiBody) Validate

func (b *MultiBody) Validate() error

type NotifDownload

type NotifDownload struct {
	nl.Base
	// contains filtered or unexported fields
}

func (*NotifDownload) ToNotifMsg

func (nd *NotifDownload) ToNotifMsg() cluster.NotifMsg

type NotifDownloadListerner

type NotifDownloadListerner struct {
	nl.ListenerBase
}

func NewDownloadNL

func NewDownloadNL(jobID, action string, smap *meta.Smap, progressInterval time.Duration) *NotifDownloadListerner

func (*NotifDownloadListerner) QueryArgs

func (nd *NotifDownloadListerner) QueryArgs() cmn.HreqArgs

func (*NotifDownloadListerner) UnmarshalStats

func (*NotifDownloadListerner) UnmarshalStats(rawMsg []byte) (stats any, finished, aborted bool, err error)

type RangeBody

type RangeBody struct {
	Base
	Template string `json:"template"`
	Subdir   string `json:"subdir"`
}

func (*RangeBody) Describe

func (b *RangeBody) Describe() string

func (*RangeBody) String

func (b *RangeBody) String() string

func (*RangeBody) Validate

func (b *RangeBody) Validate() error

type SingleBody

type SingleBody struct {
	Base
	SingleObj
}

func (*SingleBody) Describe

func (b *SingleBody) Describe() string

func (*SingleBody) ExtractPayload

func (b *SingleBody) ExtractPayload() (cos.StrKVs, error)

func (*SingleBody) String

func (b *SingleBody) String() string

func (*SingleBody) Validate

func (b *SingleBody) Validate() error

type SingleObj

type SingleObj struct {
	ObjName    string `json:"object_name"`
	Link       string `json:"link"`
	FromRemote bool   `json:"from_remote"`
}

func (*SingleObj) Validate

func (b *SingleObj) Validate() error

type StatusResp

type StatusResp struct {
	Job
	CurrentTasks  []TaskDlInfo  `json:"current_tasks,omitempty"`
	FinishedTasks []TaskDlInfo  `json:"finished_tasks,omitempty"`
	Errs          []TaskErrInfo `json:"download_errors,omitempty"`
}

func (*StatusResp) Aggregate

func (d *StatusResp) Aggregate(rhs StatusResp) *StatusResp

type TaskDlInfo

type TaskDlInfo struct {
	Name       string    `json:"name"`
	Downloaded int64     `json:"downloaded,string"`
	Total      int64     `json:"total,string,omitempty"`
	StartTime  time.Time `json:"start_time,omitempty"`
	EndTime    time.Time `json:"end_time,omitempty"`
	Running    bool      `json:"running"`
}

type TaskErrByName

type TaskErrByName []TaskErrInfo

func (TaskErrByName) Len

func (t TaskErrByName) Len() int

func (TaskErrByName) Less

func (t TaskErrByName) Less(i, j int) bool

func (TaskErrByName) Swap

func (t TaskErrByName) Swap(i, j int)

type TaskErrInfo

type TaskErrInfo struct {
	Name string `json:"name"`
	Err  string `json:"error"`
}

type TaskInfoByName

type TaskInfoByName []TaskDlInfo

func (TaskInfoByName) Len

func (t TaskInfoByName) Len() int

func (TaskInfoByName) Less

func (t TaskInfoByName) Less(i, j int) bool

func (TaskInfoByName) Swap

func (t TaskInfoByName) Swap(i, j int)

type Type

type Type string
const (
	TypeSingle  Type = "single"
	TypeRange   Type = "range"
	TypeMulti   Type = "multi"
	TypeBackend Type = "backend"
)

type WebResource

type WebResource struct {
	ObjName string
	Link    string
}

type Xact

type Xact struct {
	xact.DemandBase
	// contains filtered or unexported fields
}

func (*Xact) AbortJob

func (xld *Xact) AbortJob(id string) (resp any, statusCode int, err error)

func (*Xact) Download

func (xld *Xact) Download(job jobif) (resp any, statusCode int, err error)

func (*Xact) JobStatus

func (xld *Xact) JobStatus(id string, onlyActive bool) (resp any, statusCode int, err error)

func (*Xact) RemoveJob

func (xld *Xact) RemoveJob(id string) (resp any, statusCode int, err error)

func (*Xact) Run

func (xld *Xact) Run(*sync.WaitGroup)

func (*Xact) Snap

func (xld *Xact) Snap() (snap *cluster.Snap)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL