models

type DPNBag struct {

	// UUID is the unique identifier for a bag
	UUID string `json:"uuid"`

	// LocalId is the depositor's local identifier for a bag.
	LocalId string `json:"local_id"`

	// Member is the UUID of the member who deposited this bag.
	Member string `json:"member"`

	// Size is the size, in bytes of the bag.
	Size uint64 `json:"size"`

	// FirstVersionUUID is the UUID of the first version
	// of this bag.
	FirstVersionUUID string `json:"first_version_uuid"`

	// Version is the version or revision number of the bag. Starts at 1.
	Version uint32 `json:"version"`

	// IngestNode is the node that first ingested or produced the bag.
	IngestNode string `json:"ingest_node"`

	// AdminNode is the authoritative node for this bag. If various nodes
	// have conflicting registry info for this bag, the admin node wins.
	// The admin node also has some authority in restoring and (if its ever
	// possible) deleting bags.
	AdminNode string `json:"admin_node"`

	// BagType is one of 'D' (Data), 'R' (Rights) or 'I' (Interpretive)
	BagType string `json:"bag_type"`

	// Rights is a list of UUIDs of rights objects for this bag.
	Rights []string `json:"rights"`

	// Interpretive is a list of UUIDs of interpretive objects for this bag.
	Interpretive []string `json:"interpretive"`

	// ReplicatingNodes is a list of one more nodes that has stored
	// copies of this bag. The items in the list are node namespaces,
	// which are strings. E.g. ['aptrust', 'chron', 'tdr']
	ReplicatingNodes []string `json:"replicating_nodes"`

	// MessageDigests are the digests calculated on ingest by various
	// nodes. (Is this calculated on the tag manifest?)
	MessageDigests []*MessageDigest `json:"message_digests"`

	// FixityChecks record the result of post-ingest periodic fixity
	// checks.
	FixityChecks []*FixityCheck `json:"fixity_checks"`

	// CreatedAt is when this record was created.
	CreatedAt time.Time `json:"created_at"`

	// UpdatedAt is when this record was last updated.
	UpdatedAt time.Time `json:"updated_at"`
}

type DPNIngestManifest struct {
	// NsqMessage is the NSQ message being processed to fulfill
	// this ingest request.
	NsqMessage *nsq.Message `json:"-"`

	// WorkItem is the WorkItem in Pharos that maintains
	// information about this ingest.
	WorkItem *apt_models.WorkItem

	// WorkItemState is the WorkItemState in Pharos that conntains
	// a JSON representation of this DPNIngestManifest.
	WorkItemState *apt_models.WorkItemState

	// IntellectualObject is the object we are pushing into DPN.
	// We don't serialize this because in cases where the object
	// has tens of thousands of files, this record is huge.
	IntellectualObject *apt_models.IntellectualObject `json:"-"`

	// LocalDir is the directory in which we are assembling the
	// contents of the DPN bag.
	LocalDir string

	// LocalTarFile is the path the tarred DPN bag that we have
	// built, and which we will copy to long-term storage.
	LocalTarFile string

	// StorageURL is the URL of the tarred bag that has been copied
	// into long-term storage.
	StorageURL string

	// DPNBag is the DPNBag record we will create for the bag we're
	// sending into long-term storage. Once the bag is stored, we'll
	// POST this DPNBag record to our local DPN REST server.
	DPNBag *DPNBag

	// UUID is the uuid that will become the identifier of the DPNBag.
	// This uuid is generated early in the packaging process, before
	// the DPN bag is even created. The packager needs to know this
	// uuid first when assembling the files for the bag, and later
	// when creating the bag's dpn-info.txt tag file.
	UUID string

	// PackageSummary describes what happened during the packaging
	// of this DPN bag. The process involves packing all of the
	// IntellectualObject's files into a DPN bag, whose structure
	// differs somewhat from the structure of an APTrust bag.
	PackageSummary *apt_models.WorkSummary

	// ValidateSummary describes the result of running our validator
	// over the DPN bag we just created.
	ValidateSummary *apt_models.WorkSummary

	// StoreSummary describes the result of the attempt to copy the
	// DPN bag to long-term storage.
	StoreSummary *apt_models.WorkSummary

	// RecordSummary describes the result of the attempt to record
	// info about the new bag in our local DPN server, and to close
	// out the WorkItem in Pharos.
	RecordSummary *apt_models.WorkSummary

	// ReplicationTransfers are replication transfer requests we
	// created for this newly ingested DPN bag on our own DPN node.
	// The DPN spec as of late 2016 says that when we create a bag,
	// we need to create two replication transfers with it. That
	// number may change over time. If ReplicationTransfers is
	// empty, the transfers have not yet been created.
	ReplicationTransfers []*ReplicationTransfer

	// DPNIdentifierEvent is the PremisEvent (stored in Pharos)
	// that says we assigned a DPN UUID to this bag. This will
	// be nil until we get to the dpn_ingest_record stage.
	DPNIdentifierEvent *apt_models.PremisEvent

	// DPNIngestEvent is the PremisEvent (stored in Pharos)
	// that says we stored this item in DPN. The StorageURL
	// is in the event's OutcomeDetail. This will
	// be nil until we get to the dpn_ingest_record stage.
	DPNIngestEvent *apt_models.PremisEvent
}

type DPNRetrievalManifest struct {
	// NsqMessage is the NSQ message being processed to fulfill
	// this DPN fixity check request.
	NsqMessage *nsq.Message `json:"-"`

	// DPNWorkItem is the DPNWorkItem in Pharos that contains
	// information about this fixity task.
	DPNWorkItem *apt_models.DPNWorkItem

	// DPNBag is the bag we'll whose fixity we're checking. This
	// comes from the DPN REST server.
	DPNBag *DPNBag

	// TaskType describes whether this retrieval is for a fixity
	// check or a restoration. Value should be one of:
	// constants.ActionFixityCheck or constants.ActionRestore
	TaskType string

	// GlacierBucket is the bucket that contains the item
	// we want to restore. Note that after we request restoration,
	// Amazon leaves the item in same Glacier bucket and simply
	// changes its storage class from Glacier to Standard.
	GlacierBucket string

	// GlacierKey is the name of this item in the Glacier bucket.
	// It should be the bag UUID with a ".tar" extension.
	GlacierKey string

	// RequestedFromGlacierAt is the timestamp of the last request to
	// Glacier to restore this object. That request may not have been
	// accepted. See GlacierRequestAccepted
	RequestedFromGlacierAt time.Time

	// GlacierRequestAccepted indicates whether Glacier accepted
	// our request to restore this object. This does not mean
	// the request is complete. It can take several hours for
	// AWS to push the file from Glacier to S3. Check the
	// property IsAvailableInS3 to see if AWS has actually
	// completed the request.
	GlacierRequestAccepted bool

	// EstimatedDeletionFromS3 describes approximately when
	// this item should be available at the RestorationURL.
	// This time can vary, depending on what level of Glacier
	// retrieval service we're using. Using the standard service
	// level, this should be about four hours after RequestedAt,
	// if the requests succeeded.
	EstimatedDeletionFromS3 time.Time

	// IsAvailableInS3 describes whether the file has been
	// made available in S3 for download, a process which typically
	// takes 3-5 hours. If RequestAccepted is true and IsAvailableInS3
	// is false, then the request is still in process.
	IsAvailableInS3 bool

	// LocalPath is the path to the downloaded file on the local
	// file system. This will be empty until we have actually
	// downloaded the file from S3.
	LocalPath string

	// RestorationURL is the URL from which the depositor can
	// retrieve the bag. Since depositors cannot access preservation
	// buckets, we'll have to copy the item into an S3 bucket they
	// can access. If this is an empty string, the item was never
	// copied to a restoration bucket. (This attribute will remain
	// empty for fixity check operations in which we only copy the file
	// to local storage for validation.)
	RestorationURL string

	// ExpectedFixityValue is the SHA-256 digest that was calculated
	// for this bag's tagmanifest-sha256.txt file when the bag was
	// first ingested by the Ingest Node. This value comes from the
	// DPN REST server.
	ExpectedFixityValue string

	// ActualFixityValue is the SHA-256 digest that the worker
	// calculated for our stored copy of the bag's
	// tagmanifest-sha256.txt file.
	ActualFixityValue string

	// GlacierRestoreSummary is a summary of attempts to restore
	// the bag from Glacier to S3.
	GlacierRestoreSummary *apt_models.WorkSummary

	// LocalCopySummary is a summary attempts to copy the bag from
	// S3 (after Glacier restoration) to local storage.
	LocalCopySummary *apt_models.WorkSummary

	// ValidationSummary is a summary of attempts to validate the
	// bag. Because validation is IO-intensive and requires a lot
	// of random access, we do this locally, after the bag has been
	// copied to LocalPath. DPN rules say that a fixity check must:
	// 1) Validate the entire bag, 2) calculate the sha-256 digest
	// of the tagmanifest-sha256.txt file, and 3) record a new fixity
	// record in the DPN registry with the calculated digest.
	ValidationSummary *apt_models.WorkSummary

	// RecordSummary contains information about the worker's attempt
	// to record the result of the fixity check in the local DPN
	// registry (our node's instance of the DPN REST server).
	RecordSummary *apt_models.WorkSummary

	// FixityCheck is the FixityCheck record saved the DPN after
	// the worker finishes validating a bag and calculating the
	// sha256 checksum of its tagmanifest-sha256.txt file.
	// This property will be nil until the very last phase of
	// the fixity check process. For DPN bag restorations, this
	// property will remain nil throughout.
	FixityCheck *FixityCheck

	// FixityCheckSavedAt describes when we saved the FixityCheck
	// record to the local DPN REST server.
	FixityCheckSavedAt time.Time
}

type DPNStoredFile struct {
	// Id is a unique identifier for this DPNStoredFile,
	// if it happens to be stored in a SQL database.
	// This can be zero for items not stored in a DB.
	Id int64 `json:"id"`
	// Key is the s3/glacier name of the file. A file
	// may be stored under more than one UUID if multiple
	// versions of it exist. Typically, we should retain
	// only the most recent version.
	Key string `json:"key"`
	// Bucket is the name of the bucket where the item is stored.
	Bucket string `json:"bucket"`
	// Size is the size, in bytes, of the object in
	// long-term storage. This should match the size
	// of the file in the GenericFiles table in Pharos.
	Size int64 `json:"size"`
	// ContentType is the object's mime type. E.g. image/jpeg.
	ContentType string `json:"content_type"`
	// Member is the name or UUID of the institution
	// that owns the file.
	Member string `json:"member"`
	// FromNode is the namespace of the node this bag was
	// replicated from. This will be empty if we ingested
	// the bag ourselves.
	FromNode string `json:"from_node"`
	// TransferId is the UUID of the ReplicationTransfer
	// request we fulfilled when copying this bag. This will
	// be empty if we ingested the bag ourselves.
	TransferId string `json:"transfer_id"`
	// LocalId is the depositor's identifier for this bag.
	// If the bag was ingested by APTrust, LocalId will be
	// the APTrust IntellectualObject.Identifier.
	LocalId string `json:"local_id"`
	// Version is the version number for this bag, in string
	// format.
	Version string `json:"version"`
	// ETag is Amazon's etag for this item. For multipart
	// uploads, the etag will contain a dash.
	ETag string `json:"etag"`
	// LastModified is when this file was last modified in
	// the long-term storage bucket.
	LastModified time.Time `json:"last_modified"`
	// LastSeenAt is when our system last saw this item in
	// the long-term storage bucket.
	LastSeenAt time.Time `json:"last_seen_at"`
	// CreatedAt is when this record was created.
	CreatedAt time.Time `json:"created_at"`
	// UpdatedAt is when this record was updated.
	UpdatedAt time.Time `json:"updated_at"`
	// DeletedAt is when this file was deleted from long-term
	// storage. This will almost always be an empty timestamp.
	DeletedAt time.Time `json:"deleted_at,omitempty"`
}

type FixityCheck struct {

	// FixityCheckId is the unique id of this fixity check record.
	// This is a UUID in string format.
	FixityCheckId string `json:"fixity_check_id"`

	// Bag is the UUID of the bag to which this fixity check belongs.
	Bag string `json:"bag"`

	// Node is the namespace of the node that performed the fixity check.
	Node string `json:"node"`

	// Success indicates whether the calculated fixity matched the
	// expected fixity value (from MessageDigest).
	Success bool `json:"success"`

	// FixityAt describes when this fixity check was performed.
	FixityAt time.Time `json:"fixity_at"`

	// CreatedAt describes when this fixity check record was saved
	// to the DPN registry.
	CreatedAt time.Time `json:"created_at"`
}

type Ingest struct {

	// IngestId is the Id of this ingest record. This is a UUID
	// in string format.
	IngestId string `json:"ingest_id"`

	// Bag is the UUID of the bag to which this record belongs.
	Bag string `json:"bag"`

	// Ingested describes whether or not the ingest process
	// completed successfully. This may be fals in cases where
	// the bag was not successfully replicated to the minimum
	// number of nodes.
	Ingested bool `json:"ingested"`

	// ReplicatingNodes is a list of namespaces of the nodes
	// that have successfully replicated this bag.
	ReplicatingNodes []string `json:"replicating_nodes"`

	// CreatedAt describes when this Ingest record was created
	// in the DPN registry.
	CreatedAt time.Time `json:"created_at"`
}

type Member struct {

	// MemberId is the unique identifier for a member.
	// This is a UUID in string format.
	MemberId string `json:"member_id"`

	// Name is the member's name
	Name string `json:"name"`

	// Email is the member's email address
	Email string `json:"email"`
}

type MessageDigest struct {

	// Bag is the UUID of the bag to which this message digest belongs.
	Bag string `json:"bag"`

	// Algorithm is the digest algorithm (usually sha256).
	Algorithm string `json:"algorithm"`

	// Node is the namespace of the node that calculated this message digest.
	Node string `json:"node"`

	// Value is the actual digest value.
	Value string `json:"value"`

	// CreatedAt is the DateTime this message digest was created.
	CreatedAt time.Time `json:"created_at"`
}

type Node struct {

	// Name is the full name of the node.
	Name string `json:"name"`

	// Namespace is the node's short name, and is generally
	// used as the node identifier on other record types.
	Namespace string `json:"namespace"`

	// APIRoot is the root URL of the node's DPN server.
	APIRoot string `json:"api_root"`

	// SSHPubKey is the public half of the SSH key that the
	// node uses to connect to other nodes to copy data via
	// rsync/ssh.
	SSHPubKey string `json:"ssh_pubkey"`

	// ReplicateFrom is a list of node namespaces from which
	// this node will replicate content.
	ReplicateFrom []string `json:"replicate_from"`

	// ReplicateTo is a list of node namespaces to which
	// this node will replicate content.
	ReplicateTo []string `json:"replicate_to"`

	// RestoreFrom is a list of node namespaces from which
	// this node will restore content.
	RestoreFrom []string `json:"restore_from"`

	// RestoreTo is a list of node namespaces to which
	// this node will restore content.
	RestoreTo []string `json:"restore_to"`

	// Protocols is a list of protocols this node supports for
	// copying files to and from other nodes. Initially, the
	// only supported protocol is rsync.
	Protocols []string `json:"protocols"`

	// FixityAlgorithms is a list of fixity algorithms this
	// node supports for calculating a bag's initial message
	// digest and subsequent periodic fixity checks. Initially,
	// all nodes support sha256.
	FixityAlgorithms []string `json:"fixity_algorithms"`

	// Storage describes the node's storage region and type.
	Storage *Storage `json:"storage"`

	// CreatedAt is the time at which this node record was
	// created in the DPN registry.
	CreatedAt time.Time `json:"created_at"`

	// UpdatedAt describes when this node record was last
	// updated.
	UpdatedAt time.Time `json:"updated_at"`

	// LastPullDate describes when we last pulled data from
	// this node. This property is not part of the DPN spec.
	// It is for APTrust internal use only. We don't save
	// this info to the DPN registry, because there's no
	// place for it there. We save it locally and use it in
	// the dpn_sync chron job.
	LastPullDate time.Time `json:"last_pull_date"`
}

type QueueItem struct {
	// Identifier is a UUID in the case of ReplicationTransfers and
	// RestoreTransfers, or an IntellectualObject.Identifier in the
	// case of Ingest requests. It identifies the subject of the request.
	Identifier string
	// The Id of the DPNWorkItem that corresponds to this replication or
	// restore requests. For ingest requests, it's the Id of the WorkItem.
	ItemId int
	// QueuedAt is the time at which this item was queued.
	QueuedAt time.Time
}

type QueueResult struct {
	// StartTime is the time at which dpn_queue started.
	StartTime time.Time
	// EndTime is the time at which dpn_queue finished its processing.
	EndTime time.Time
	// Replications is a list of ReplicationTransfer requests encountered
	// during this run.
	Replications []*QueueItem
	// Restores is a list of RestoreTransfer requests encountered
	// during this run.
	Restores []*QueueItem
	// Ingests is a list of ingest requests encountered during this run.
	Ingests []*QueueItem
	// Fixities is a list of items representing bags that need fixity checks.
	Fixities []*QueueItem
	// Errors is a list of errors that occurred during processing.
	Errors []string
}

type ReplicationManifest struct {

	// NsqMessage is the NSQ message being processed to fulfill
	// this replication.
	NsqMessage *nsq.Message `json:"-"`

	// DPNWorkItem is the DPNWorkItem in Pharos that maintains
	// information about this ReplicationTransfer.
	DPNWorkItem *apt_models.DPNWorkItem

	// ReplicationTransfer is the DPN ReplicationTransfer created
	// by a remote node, requesting our local node to replicate
	// a bag.
	ReplicationTransfer *ReplicationTransfer

	// DPNBag is the bag we are replicating.
	DPNBag *DPNBag

	// CopySummary contains information about the process of copying
	// the bag from the remote DPN node to our local staging area
	// for processing.
	CopySummary *apt_models.WorkSummary

	// ValidateSummary contains information about the outcome of
	// the validation process for this bag.
	ValidateSummary *apt_models.WorkSummary

	// StoreSummary contains information about the outcome of the
	// storage operation for this bag. That is, did we manage to
	// copy it to long-term storage or not?
	StoreSummary *apt_models.WorkSummary

	// LocalPath describes where the bag is on our local file system.
	// The bag is a tar file, and should be in the staging area.
	LocalPath string

	// StorageURL describes where the bag is stored in AWS long-term
	// storage. This will be empty until the bag is stored.
	StorageURL string

	// RsyncOutput is the output of the rsync command used to copy
	// the bag from the remote node to our staging area. This is
	// especially useful for debugging, since firewall changes,
	// ssh config problems, and ssh key problems are the main
	// causes of rsync copy failures.
	RsyncOutput string

	// Cancelled indicates whether the replication process was
	// cancelled.
	Cancelled bool
}

type ReplicationTransfer struct {

	// FromNode is the node where the bag is coming from.
	// The FromNode initiates the replication request.
	FromNode string `json:"from_node"`

	// ToNode is the node the bag is being transferred to
	ToNode string `json:"to_node"`

	// Bag is the UUID of the bag to be replicated
	Bag string `json:"bag"`

	// ReplicationId is a unique id for this replication request.
	// It's a UUID in string format.
	ReplicationId string `json:"replication_id"`

	// FixityAlgorithm is the algorithm used to calculate the fixity digest.
	FixityAlgorithm string `json:"fixity_algorithm"`

	// FixityNonce is an optional nonce used to calculate the fixity digest.
	FixityNonce *string `json:"fixity_nonce"`

	// FixityValue is the fixity value calculated by the ToNode after
	// it receives the bag. This will be null/empty until the replicating
	// node sends the info back to the FromNode.
	FixityValue *string `json:"fixity_value"`

	// Protocol is the network protocol used to transfer the bag.
	// At launch, the only valid value for this is 'R' for rsync.
	Protocol string `json:"protocol"`

	// Link is a URL that the ToNode can use to copy the bag from the
	// FromNode. This value is set by the FromNode.
	Link string `json:"link"`

	// Stored indicates whether the ToNode has stored this bag.
	Stored bool `json:"stored"`

	// StoreRequested indicates whether the FromNode wants the ToNode
	// to store the bag. This will be true only after the ToNode has
	// sent the correct FixityValue to the FromNode.
	StoreRequested bool `json:"store_requested"`

	// Cancelled indicates whether this replication request was
	// cancelled.
	Cancelled bool `json:"cancelled"`

	// CancelReason is free-form text describing why this replication
	// request was cancelled.
	CancelReason *string `json:"cancel_reason"`

	// CreatedAt is the datetime when this record was created.
	CreatedAt time.Time `json:"created_at"`

	// UpdatedAt is the datetime when this record was last updated.
	UpdatedAt time.Time `json:"updated_at"`
}

type RestoreTransfer struct {

	// RestoreId is the UUID for this restoration request.
	RestoreId string `json:"restore_id"`

	// FromNode is the node from which the bag should be restored.
	FromNode string `json:"from_node"`

	// ToNode is the node to which the bag should be restored.
	// The ToNode initiates a restoration request.
	ToNode string `json:"to_node"`

	// Bag is the unique identifier of the bag to be restored.
	Bag string `json:"bag"`

	// Protocol is the network protocol used to transfer the bag.
	// At launch, the only valid value for this is 'R' for rsync.
	Protocol string `json:"protocol"`

	// Link is a URL that the ToNode can use to copy the bag from the
	// FromNode. This value is set by the FromNode.
	Link string `json:"link"`

	// Accepted indicates whether the FromNode is willing to
	// restore the bag to the ToNode.
	Accepted bool `json:"accepted"`

	// Finished indicates whether this restore request has been
	// completed.
	Finished bool `json:"finished"`

	// Cancelled indicates whether this restore request was
	// cancelled.
	Cancelled bool `json:"cancelled"`

	// CancelReason is free-form text describing why this restore
	// request was cancelled.
	CancelReason *string `json:"cancel_reason"`

	// CreatedAt is the datetime when this record was created.
	CreatedAt time.Time `json:"created_at"`

	// UpdatedAt is the datetime when this record was last updated.
	UpdatedAt time.Time `json:"updated_at"`
}

type Storage struct {
	Region string `json:"region"`
	Type   string `json:"type"`
}

type SyncResult struct {
	NodeName    string
	FetchCounts map[dpn.DPNObjectType]int
	SyncCounts  map[dpn.DPNObjectType]int
	Errors      map[dpn.DPNObjectType][]error
}