mapper

type Controller struct {
	// MapperQueue is a name of the GAE task queue to use for mapping jobs.
	//
	// This queue will perform all "heavy" tasks. It should be configured
	// appropriately to allow desired number of shards to run in parallel.
	//
	// For example, if the largest submitted job is expected to have 128 shards,
	// max_concurrent_requests setting of the mapper queue should be at least 128,
	// otherwise some shards will be stalled waiting for others to finish
	// (defeating the purpose of having large number of shards).
	//
	// If empty, "default" is used.
	MapperQueue string

	// ControlQueue is a name of the GAE task queue to use for control signals.
	//
	// This queue is used very lightly when starting and stopping jobs (roughly
	// 2*Shards tasks overall per job). A default queue.yaml settings for such
	// queue should be sufficient (unless you run a lot of different jobs at
	// once).
	//
	// If empty, "default" is used.
	ControlQueue string
	// contains filtered or unexported fields
}

type Factory func(c context.Context, j *Job, shardIdx int) (Mapper, error)

type ID string

type Job struct {

	// ID is auto-generated unique identifier of the job.
	ID JobID `gae:"$id"`
	// Config is the configuration of this job. Doesn't change once set.
	Config JobConfig `gae:",noindex"`
	// State is used to track job's lifecycle, see the enum.
	State State
	// Created is when the job was created, FYI.
	Created time.Time
	// Updated is when the job was last touched, FYI.
	Updated time.Time
	// contains filtered or unexported fields
}

type JobConfig struct {
	Query      Query  // a query identifying a set of entities
	Mapper     ID     // ID of a registered mapper to apply to entities
	Params     []byte // arbitrary user-provided data to pass to the mapper
	ShardCount int    // number of shards to split the key range into
	PageSize   int    // how many entities to process at once in each shard

	// PagesPerTask is how many pages (each of PageSize entities) to process
	// inside a TQ task.
	//
	// Default is unlimited: process until the deadline.
	PagesPerTask int

	// TaskDuration is how long to run a single mapping TQ task before
	// checkpointing the state and launching the next mapping TQ task.
	//
	// Small values (e.g. 1 min) makes each processing TQ task relatively small,
	// so it doesn't eat a lot of memory, or produces gigantic unreadable logs.
	// It also makes TQ's "Pause queue" button more handy.
	//
	// Default is 1 min.
	TaskDuration time.Duration

	// TrackProgress enables calculating number of entities per shard before
	// launching mappers, and using it to calculate completion ETA.
	//
	// May be VERY slow if processing large amount of entities. Slowness manifests
	// as a delay between job's launch and it actual start of shards processing.
	//
	// Enable only if shards are relatively small (< 100K entities per shard).
	TrackProgress bool
}

type JobID int64

type JobInfo struct {
	Id                   int64                `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"`
	State                State                `protobuf:"varint,2,opt,name=state,proto3,enum=appengine.mapper.messages.State" json:"state,omitempty"`
	Created              *timestamp.Timestamp `protobuf:"bytes,3,opt,name=created,proto3" json:"created,omitempty"`
	Updated              *timestamp.Timestamp `protobuf:"bytes,4,opt,name=updated,proto3" json:"updated,omitempty"`
	Eta                  *timestamp.Timestamp `protobuf:"bytes,5,opt,name=eta,proto3" json:"eta,omitempty"`
	ProcessedEntities    int64                `protobuf:"varint,6,opt,name=processed_entities,json=processedEntities,proto3" json:"processed_entities,omitempty"`
	TotalEntities        int64                `protobuf:"varint,7,opt,name=total_entities,json=totalEntities,proto3" json:"total_entities,omitempty"`
	EntitiesPerSec       float32              `protobuf:"fixed32,8,opt,name=entities_per_sec,json=entitiesPerSec,proto3" json:"entities_per_sec,omitempty"`
	Shards               []*ShardInfo         `protobuf:"bytes,20,rep,name=shards,proto3" json:"shards,omitempty"`
	XXX_NoUnkeyedLiteral struct{}             `json:"-"`
	XXX_unrecognized     []byte               `json:"-"`
	XXX_sizecache        int32                `json:"-"`
}

type Mapper func(c context.Context, keys []*datastore.Key) error

type Query struct {
	Kind     string         // entity kind to limit the query, "" for kindless
	Ancestor *datastore.Key // entity group to limit the query to (or nil)
}

type ShardInfo struct {
	Index                int32                `protobuf:"varint,1,opt,name=index,proto3" json:"index,omitempty"`
	State                State                `protobuf:"varint,2,opt,name=state,proto3,enum=appengine.mapper.messages.State" json:"state,omitempty"`
	Error                string               `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"`
	Created              *timestamp.Timestamp `protobuf:"bytes,4,opt,name=created,proto3" json:"created,omitempty"`
	Updated              *timestamp.Timestamp `protobuf:"bytes,5,opt,name=updated,proto3" json:"updated,omitempty"`
	Eta                  *timestamp.Timestamp `protobuf:"bytes,6,opt,name=eta,proto3" json:"eta,omitempty"`
	ProcessedEntities    int64                `protobuf:"varint,7,opt,name=processed_entities,json=processedEntities,proto3" json:"processed_entities,omitempty"`
	TotalEntities        int64                `protobuf:"varint,8,opt,name=total_entities,json=totalEntities,proto3" json:"total_entities,omitempty"`
	EntitiesPerSec       float32              `protobuf:"fixed32,9,opt,name=entities_per_sec,json=entitiesPerSec,proto3" json:"entities_per_sec,omitempty"`
	XXX_NoUnkeyedLiteral struct{}             `json:"-"`
	XXX_unrecognized     []byte               `json:"-"`
	XXX_sizecache        int32                `json:"-"`
}

type State int32