dpn

type BagBuilder struct {
	// LocalPath is the full, absolute path the the untarred bag
	// the builder will create. It will end with the bag's UUID,
	// so it should look something like this:
	// /mnt/dpn/bags/00000000-0000-0000-0000-000000000000.
	LocalPath string

	// IntellectualObject is the APTrust IntellectualObject that
	// we'll be repackaging as a DPN bag.
	IntellectualObject *bagman.IntellectualObject

	// DefaultMetadata is some metadata that goes into EVERY DPN
	// bag we create. This includes our name and address in the
	// DPN data section that describes who packaged the bag.
	// DefaultMetadata should be loaded from a JSON file using
	// the dpn.LoadConfig() function.
	DefaultMetadata *DefaultMetadata

	// UUID is the DPN identifier for this bag. This has nothing to
	// do with any APTrust UUID. It's generated in the constructor.
	UUID string

	// ErrorMessage describes what when wrong while trying to
	// package this bag. If it's an empty string, packaging
	// succeeded.
	ErrorMessage string

	// What type of bag is this? Data, rights or interpretive?
	BagType string

	// The underlying bag object.
	Bag *bagins.Bag
	// contains filtered or unexported fields
}

type BagListResult struct {
	Count    int32     `json:count`
	Next     *string   `json:next`
	Previous *string   `json:previous`
	Results  []*DPNBag `json:results`
}

type Cleanup struct {
	DPNConfig   *DPNConfig
	ProcUtil    *bagman.ProcessUtil
	LocalClient *DPNRestClient
}

type Copier struct {
	LookupChannel      chan *DPNResult
	CopyChannel        chan *DPNResult
	PostProcessChannel chan *DPNResult
	DPNConfig          *DPNConfig
	ProcUtil           *bagman.ProcessUtil
	LocalClient        *DPNRestClient
	RemoteClients      map[string]*DPNRestClient
	// WaitGroup is for running local tests only.
	WaitGroup sync.WaitGroup
}

type CopyResult struct {
	LocalPath    string
	ErrorMessage string
	RsyncStdout  string
	RsyncStderr  string
	InfoMessage  string
	BagWasCopied bool
}

type DPNBag struct {

	// UUID is the unique identifier for a bag
	UUID string `json:"uuid"`

	// LocalId is the depositor's local identifier for a bag.
	LocalId string `json:"local_id"`

	// Member is the UUID of the member who deposited this bag.
	Member string `json:"member"`

	// Size is the size, in bytes of the bag.
	Size uint64 `json:"size"`

	// FirstVersionUUID is the UUID of the first version
	// of this bag.
	FirstVersionUUID string `json:"first_version_uuid"`

	// Version is the version or revision number of the bag. Starts at 1.
	Version uint32 `json:"version"`

	// IngestNode is the node that first ingested or produced the bag.
	IngestNode string `json:"ingest_node"`

	// AdminNode is the authoritative node for this bag. If various nodes
	// have conflicting registry info for this bag, the admin node wins.
	// The admin node also has some authority in restoring and (if its ever
	// possible) deleting bags.
	AdminNode string `json:"admin_node"`

	// BagType is one of 'D' (Data), 'R' (Rights) or 'I' (Interpretive)
	BagType string `json:"bag_type"`

	// Rights is a list of UUIDs of rights objects for this bag.
	Rights []string `json:"rights"`

	// Interpretive is a list of UUIDs of interpretive objects for this bag.
	Interpretive []string `json:"interpretive"`

	// ReplicatingNodes is a list of one more nodes that has stored
	// copies of this bag. The items in the list are node namespaces,
	// which are strings. E.g. ['aptrust', 'chron', 'tdr']
	ReplicatingNodes []string `json:"replicating_nodes"`

	// Fixities are the checksum/fixity values for this bag.
	Fixities *DPNFixity `json:"fixities"`

	// CreatedAt is when this record was created.
	CreatedAt time.Time `json:"created_at"`

	// UpdatedAt is when this record was last updated.
	UpdatedAt time.Time `json:"updated_at"`
}

type DPNConfig struct {
	// EnvironmentName is the name of this configuration: "dev", "test",
	// "production", etc.
	EnvironmentName string
	// LocalNode is the namespace of the node this code is running on.
	// E.g. "aptrust", "chron", "hathi", "tdr", "sdr"
	LocalNode string
	// Where should DPN service logs go?
	LogDirectory string
	// Log level (4 = debug)
	LogLevel logging.Level
	// Should we log to Stderr in addition to writing to
	// the log file?
	LogToStderr bool
	// Number of nodes we should replicate bags to.
	ReplicateToNumNodes int
	// Should we accept self-signed and otherwise invalid SSL
	// certificates? We need to do this in testing, but it
	// should not be allowed in production. Bools in Go default
	// to false, so if this is not set in config, we should be
	// safe.
	AcceptInvalidSSLCerts bool
	// When copying bags from remote nodes, should we use rsync
	// over SSH (true) or just plain rsync (false)?
	UseSSHWithRsync bool
	// Default metadata that goes into bags produced at our node.
	DefaultMetadata *DefaultMetadata
	// Settings for connecting to our own REST service
	RestClient *RestClientConfig
	// Standard Auth token header format for REST services
	// is "token %s", where "%s" will be the token. Rails
	// REST services require the format "Token token=%s".
	// This map of formats lets us override the standard
	// "token %s" with whatever the remote REST service
	// expects. Since the default is "token %s", there is no
	// need to create entries in this map for most nodes.
	AuthTokenHeaderFormats map[string]string
	// RemoteNodeAdminTokensForTesting are used in integration
	// tests only, when we want to perform admin-only operations,
	// such as creating bags and replication requests on a remote
	// node in the test cluster.
	RemoteNodeAdminTokensForTesting map[string]string
	// API Tokens for connecting to remote nodes
	RemoteNodeTokens map[string]string
	// URLs for remote nodes. Set these only if you want to
	// override the node URLs we get back from our local
	// DPN REST server.
	RemoteNodeURLs map[string]string
}

type DPNFetchResult struct {
	FetchResult *bagman.FetchResult
	GenericFile *bagman.GenericFile
}

type DPNFixity struct {

	// The algorithm used to check the fixity. Usually 'sha256',
	// but others may be valid in the future.
	Sha256 string `json:"sha256"`
}

type DPNMember struct {

	// UUID is the unique identifier for a member
	UUID string `json:"uuid"`

	// Name is the member's name
	Name string `json:"name"`

	// Email is the member's email address
	Email string `json:"email"`

	// CreatedAt is when this record was created.
	CreatedAt time.Time `json:"created_at"`

	// UpdatedAt is when this record was last updated.
	UpdatedAt time.Time `json:"updated_at"`
}

type DPNNode struct {
	Name             string      `json:"name"`
	Namespace        string      `json:"namespace"`
	APIRoot          string      `json:"api_root"`
	SSHPubKey        string      `json:"ssh_pubkey"`
	ReplicateFrom    []string    `json:"replicate_from"`
	ReplicateTo      []string    `json:"replicate_to"`
	RestoreFrom      []string    `json:"restore_from"`
	RestoreTo        []string    `json:"restore_to"`
	Protocols        []string    `json:"protocols"`
	FixityAlgorithms []string    `json:"fixity_algorithms"`
	CreatedAt        time.Time   `json:"created_at"`
	UpdatedAt        time.Time   `json:"updated_at"`
	LastPullDate     time.Time   `json:"last_pull_date"`
	Storage          *DPNStorage `json:"storage"`
}

type DPNReplicationTransfer struct {

	// FromNode is the node where the bag is coming from.
	// The FromNode initiates the replication request.
	FromNode string `json:"from_node"`

	// ToNode is the node the bag is being transfered to
	ToNode string `json:"to_node"`

	// Bag is the UUID of the bag to be replicated
	BagId string `json:"uuid"`

	// ReplicationId is a unique id for this replication request.
	// It's a UUID in string format.
	ReplicationId string `json:"replication_id"`

	// FixityAlgorithm is the algorithm used to calculate the fixity digest.
	FixityAlgorithm string `json:"fixity_algorithm"`

	// FixityNonce is an optional nonce used to calculate the fixity digest.
	FixityNonce *string `json:"fixity_nonce"`

	// FixityValue is the fixity value calculated by the ToNode after
	// it receives the bag. This will be null/empty until the replicating
	// node sends the info back to the FromNode.
	FixityValue *string `json:"fixity_value"`

	// FixityAccept describes whether the FromNode accepts the fixity
	// value calculated by the ToNode. This is a nullable boolean,
	// so it has to be a pointer.
	FixityAccept *bool `json:"fixity_accept"`

	// BagValid is a value set by the ToNode to indicate whether
	// the bag it received was valid. This is a nullable boolean,
	// so it has to be a pointer.
	BagValid *bool `json:"bag_valid"`

	// Status is the status of the request, which can be any of:
	//
	// "requested"  - The FromNode has requested this transfer.
	//                This means the transfer is new, and no
	//                action has been taken yet.
	// "rejected"   - Set by the ToNode when it will not or cannot
	//                accept this transfer. (Usually due to disk space.)
	// "received"   - Set by the ToNode to indicate it has received the
	//                the bag.
	// "confirmed"  - Set by the FromNode after the bag has been confirmed
	//                valid, the fixity value has been approved, and the bag
	//                has been stored by the ToNode.
	// "stored"     - Set by the ToNode after the bag has been copied to
	//                long-term storage.
	// "cancelled"  - Can be set by either node for any reason. No further
	//                processing should occur on a cancelled request.
	Status string `json:"status"`

	// Protocol is the network protocol used to transfer the bag.
	// At launch, the only valid value for this is 'R' for rsync.
	Protocol string `json:"protocol"`

	// Link is a URL that the ToNode can use to copy the bag from the
	// FromNode. This value is set by the FromNode.
	Link string `json:"link"`

	// CreatedAt is the datetime when this record was created.
	CreatedAt time.Time `json:"created_at"`

	// UpdatedAt is the datetime when this record was last updated.
	UpdatedAt time.Time `json:"updated_at"`
}

type DPNRestClient struct {
	HostUrl    string
	APIVersion string
	APIKey     string
	Node       string
	// contains filtered or unexported fields
}

type DPNRestoreTransfer struct {

	// RestoreId is a unique id for this restoration request.
	RestoreId string `json:"restore_id"`

	// FromNode is the node from which the bag should be restored.
	FromNode string `json:"from_node"`

	// ToNode is the node to which the bag should be restored.
	// The ToNode initiates a restoration request.
	ToNode string `json:"to_node"`

	// Bag is the unique identifier of the bag to be restored.
	BagId string `json:"uuid"`

	// Status is the status of the restoration operation. It can
	// have any of the following values:
	//
	// "requested" - Default status used when record first created.
	// "accepted"  - Indicates the FromNode has accepted the request to
	//               restore the bag.
	// "rejected"  - Set by the FromNode if it cannot or will not restore
	//               the bag.
	// "prepared"  - Set by the FromNode when the content has been restored
	//               locally and staged for transfer back to the to_node.
	// "finished"  - Set by the ToNode after it has retrieved the restored
	//               bag from the FromNode.
	// "cancelled" - Set by either node to indicate the restore operation
	//               was cancelled.
	Status string `json:"status"`

	// Protocol is the network protocol used to transfer the bag.
	// At launch, the only valid value for this is 'R' for rsync.
	Protocol string `json:"protocol"`

	// Link is a URL that the ToNode can use to copy the bag from the
	// FromNode. This value is set by the FromNode.
	Link string `json:"link"`

	// CreatedAt is the datetime when this record was created.
	CreatedAt time.Time `json:"created_at"`

	// UpdatedAt is the datetime when this record was last updated.
	UpdatedAt time.Time `json:"updated_at"`
}

type DPNResult struct {
	// BagIdentifier is the APTrust bag identifier, composed of
	// the institution domain name, a slash, and the institution's
	// internal bag identifier. E.g. "test.edu/ncsu.1840.16-1004"
	// For bags coming from other nodes, this will be blank.
	BagIdentifier string

	// If this is an APTrust bag that we're ingesting into DPN,
	// this is the ID of the "send to DPN" request in the
	// ProcessedItem queue. For bags being replicated from
	// other nodes, this will be nil.
	ProcessedItemId int

	// LocalPath is where this bag is stored on disk. The bag
	// may be a file ending in .tar or a directory if the bag
	// is not tarred.
	LocalPath string

	// The bag's md5 digest. We need this to copy to Amazon S3/Glacier.
	BagMd5Digest string

	// Sha256Digest of the tarred bag file. No longer used.
	BagSha256Digest string

	// Sha256 digest of the tagmanifest-sha256.txt file. We use this
	// to verify that the bag was correctly copied by replicating
	// nodes.
	TagManifestDigest string

	// The size of the bag. Used when copying to S3/Glacier.
	BagSize int64

	// The NSQ message being processed. May be nil if we're
	// running tests.
	NsqMessage *nsq.Message `json:"-"`

	// The current stage of processing for this bag.
	Stage string

	// A general error message describing what went wrong with
	// processing. More specific errors will appear in the
	// PackageResult or ValidationResult, depending
	// on the stage where processing failed. If this is empty,
	// there was no error.
	ErrorMessage string

	// The DPN bag record for this object. This will be nil for
	// bags ingested through APTrust and in the packaging stage,
	// since the bag won't have a UUID until after it's packaged.
	DPNBag *DPNBag

	// The DPN transfer request associated with this bag. This will
	// be nil if it's a bag created at our own node. It will be
	// non-nil for bags we're replicating from other nodes.
	TransferRequest *DPNReplicationTransfer

	// The results of attempts to fetch the S3 files that make up this
	// bag. This will be nil for bags that we're replicating from
	// other nodes, because in those cases, we just copy the whole
	// tar file from the remote node. For bags ingested at APTrust,
	// this should contain a number of records, once we've reached
	// the fetch stage of the process. Before that stage, it will
	// be nil.
	//
	// It would be nice to serialize this, but we wind up with
	// a lot of JSON.
	FetchResults *FetchResultCollection `json:"-"`

	// The result of the attempt to package this object as a DPN
	// bag. We only package APTrust bags that we ingested and that
	// the depositor has indicated should go to DPN. Bags we
	// replicate from other nodes will already have been packaged
	// by the ingesting node, so the PackageResult for those will
	// be nil. On successful copy, check DPNResult.LocalPath to
	// find where we stored the file.
	PackageResult *PackageResult

	// The result of the attempt to copy the bag from its admin
	// or ingest node (typically ingest node). When a remote
	// node asks us to replicate a bag, we have to copy it from the
	// remote node to our staging area, usually via rsync. This
	// structure records the result of that copy. For bags that we
	// created at APTrust, this will be nil because we don't have
	// to copy from ourselves.
	CopyResult *CopyResult

	// The URL of this item in long-term storage. This will be an
	// AWS S3 or Glacier URL. An empty string indicates the bag
	// has not yet been copied to storage.
	StorageURL string

	// The result of the attempt to record information about the bag
	// in DPN and in APTrust. This object is defined in recorder.go.
	RecordResult *RecordResult

	// The result of the attempt to validate the bag. This includes
	// information about whether the bag's structure is valid, whether
	// all required tags are present, checksums checked out, etc.
	ValidationResult *ValidationResult

	// Indicates whether we should try to process this bag again.
	// For transient problems, such as network outages and lack of
	// disk space, this will be true. For fatal problems, such as
	// an invalid bag, this will be false.
	Retry bool
	// contains filtered or unexported fields
}

type DPNStorage struct {
	Region string `json:"region"`
	Type   string `json:"type"`
}

type DPNSync struct {
	LocalClient   *DPNRestClient
	RemoteClients map[string]*DPNRestClient
	Logger        *logging.Logger
	Config        *DPNConfig
}

type DefaultMetadata struct {
	Comment                string
	BagItVersion           string
	BagItEncoding          string
	IngestNodeName         string
	IngestNodeAddress      string
	IngestNodeContactName  string
	IngestNodeContactEmail string
}

type FetchResultCollection struct {
	Items []*DPNFetchResult
	// contains filtered or unexported fields
}

type MemberListResult struct {
	Count    int32        `json:count`
	Next     *string      `json:next`
	Previous *string      `json:previous`
	Results  []*DPNMember `json:results`
}

type NodeListResult struct {
	Count    int32      `json:count`
	Next     *string    `json:next`
	Previous *string    `json:previous`
	Results  []*DPNNode `json:results`
}

type PackageResult struct {
	BagBuilder      *BagBuilder
	DPNFetchResults []*DPNFetchResult
	TarFilePath     string
	ErrorMessage    string
}

type Packager struct {
	LookupChannel      chan *DPNResult
	FetchChannel       chan *DPNResult
	BuildChannel       chan *DPNResult
	TarChannel         chan *DPNResult
	CleanupChannel     chan *DPNResult
	PostProcessChannel chan *DPNResult
	DPNConfig          *DPNConfig
	ProcUtil           *bagman.ProcessUtil
	// WaitGroup is for running local tests only.
	WaitGroup sync.WaitGroup
}

type RecordResult struct {
	// Did we create the DPN bag record in our local DPN registry?
	// We do this only if APTrust was the ingest node. If we created
	// the bag, this should be set to the bag's CreatedAt timestamp,
	// as returned by the server.
	DPNBagCreatedAt time.Time
	// If we created replication requests for this bag, the
	// namespaces of the replicating nodes should go here.
	// Note that this just means we created replication requests;
	// it does not mean those requests have been fulfilled.
	DPNReplicationRequests []string
	// If this was a local APTrust bag, we create a PREMIS
	// event saying that the bag has been ingested to DPN.
	// The PREMIS event identifier is a UUID string.
	PremisIngestEventId string
	// PREMIS identifier assignment event ID for bags ingested
	// by APTrust.
	PremisIdentifierEventId string
	// What time did we update the processed item request for this bag?
	// This lets Fluctus know that the task is complete.
	ProcessedItemUpdatedAt time.Time
	// If this is not an APTrust bag, did we send the copy receipt
	// the remote node that asked us to replicate this bag?
	// If sent the copy receipt, this should be set to the
	// ReplicationTransfer object's UpdatedAt timestamp, as returned
	// by the remote DPN REST server.
	CopyReceiptSentAt time.Time
	// If this is not an APTrust bag, did we send a message to the
	// remote node describing the outcome of our attempt to copy
	// this bag into long-term storage? If so, set this to the
	// UpdatedAt timestamp of the ReplicationTransfer object, as
	// returned by the remote DPN REST server.
	StorageResultSentAt time.Time
	// ErrorMessage contains information about an error that occurred
	// at any step during the recording process. If ErrorMessage is
	// an empty string, no error occurred.
	ErrorMessage string
}

type Recorder struct {
	RecordChannel      chan *DPNResult
	PostProcessChannel chan *DPNResult
	ProcUtil           *bagman.ProcessUtil
	DPNConfig          *DPNConfig
	LocalRESTClient    *DPNRestClient
	RemoteClients      map[string]*DPNRestClient
	// WaitGroup is for running local tests only.
	WaitGroup sync.WaitGroup
}

type ReplicationListResult struct {
	Count    int32                     `json:count`
	Next     *string                   `json:next`
	Previous *string                   `json:previous`
	Results  []*DPNReplicationTransfer `json:results`
}

type RestClientConfig struct {
	Comment         string
	LocalServiceURL string
	LocalAPIRoot    string
	LocalAuthToken  string
}

type RestoreListResult struct {
	Count    int32                 `json:count`
	Next     *string               `json:next`
	Previous *string               `json:previous`
	Results  []*DPNRestoreTransfer `json:results`
}

type Storer struct {
	StorageChannel     chan *DPNResult
	CleanupChannel     chan *DPNResult
	BagCreateChannel   chan *DPNResult
	PostProcessChannel chan *DPNResult
	ProcUtil           *bagman.ProcessUtil
	DPNConfig          *DPNConfig
	LocalRESTClient    *DPNRestClient
	// WaitGroup is for running local tests only.
	WaitGroup sync.WaitGroup
}

type SyncResult struct {
	// Node is the node we are pulling information from.
	RemoteNode *DPNNode
	// Bags is a list of bags successfully synched.
	Bags []*DPNBag
	// ReplicationTransfers successfully synched.
	ReplicationTransfers []*DPNReplicationTransfer
	// RestoreTransfers successfully synched.
	RestoreTransfers []*DPNRestoreTransfer
	// BagSyncError contains the error (if any) that occurred
	// during the bag sync process. The first error will stop
	// the synching of all subsquent bags.
	BagSyncError error
	// ReplicationSyncError contains the error (if any) that occurred
	// during the synching of Replication Transfers. The first error
	// will stop the synching of all subsquent replication requests.
	ReplicationSyncError error
	// RestoreSyncError contains the error (if any) that occurred
	// during the synching of Restore Transfers. The first error
	// will stop the synching of all subsquent restore requests.
	RestoreSyncError error
}

type TroubleProcessor struct {
	ProcUtil *bagman.ProcessUtil
	// WaitGroup is for running local tests only.
	WaitGroup sync.WaitGroup
}

type ValidationResult struct {
	// TarFilePath is the path to the tarred bag we'll be validating.
	TarFilePath string

	// UntarredPath is the path to the untarred version of this bag.
	UntarredPath string

	// The NSQ message we're currently working on. This will be nil
	// outside of production. In production, we need to touch the
	// message periodically to keep it from timing out, especially
	// on very large bags.
	NsqMessage *nsq.Message `json:"-"`

	// TagManifestChecksum is the sha256 digest (calculated with a nonce)
	// that we need to send back to the originating node as a receipt
	// when we're fulfilling replication requests. Outside of fulfilling
	// replication requests, we don't need to even calculate this value.
	TagManifestChecksum string

	// ErrorMessages contains a list of everything that's wrong with the
	// bag. If this list is empty, the bag is valid.
	ErrorMessages []string

	// Warning messages about non-fatal issues we might want to look into.
	Warnings []string
}

type Validator struct {
	ValidationChannel  chan *DPNResult
	PostProcessChannel chan *DPNResult
	ProcUtil           *bagman.ProcessUtil
	DPNConfig          *DPNConfig
	LocalRESTClient    *DPNRestClient
	// WaitGroup is for running local tests only.
	WaitGroup sync.WaitGroup
}