Documentation
¶
Index ¶
- Constants
- Variables
- func AddLogStreamArchivedFilter(q *ds.Query, v bool) *ds.Query
- func AddLogStreamPathFilter(q *ds.Query, path string) (*ds.Query, error)
- func AddLogStreamPurgedFilter(q *ds.Query, v bool) *ds.Query
- func AddLogStreamTagFilter(q *ds.Query, key string, value string) *ds.Query
- func AddLogStreamTerminatedFilter(q *ds.Query, v bool) *ds.Query
- func AddNewerFilter(q *ds.Query, t time.Time) *ds.Query
- func AddOlderFilter(q *ds.Query, t time.Time) *ds.Query
- func CurrentProject(c context.Context) types.ProjectName
- func CurrentProjectConfig(c context.Context) (*svcconfig.ProjectConfig, error)
- func IsAdminUser(c context.Context) error
- func IsMembershipError(e error) bool
- func IsProjectReader(c context.Context, pcfg *svcconfig.ProjectConfig) error
- func IsProjectWriter(c context.Context, pcfg *svcconfig.ProjectConfig) error
- func IsServiceUser(c context.Context) error
- func Project(c context.Context) types.ProjectName
- func ProjectFromNamespace(ns string) types.ProjectName
- func ProjectNamespace(project types.ProjectName) string
- func WithConfigProvider(c context.Context, s ConfigProvider) context.Context
- func WithProjectNamespace(c *context.Context, project types.ProjectName, at NamespaceAccessType) error
- type ArchivalParams
- type ArchivalPublisher
- type ArchivalState
- type ConfigProvider
- type HashID
- type LUCIConfigProvider
- type LogPrefix
- type LogStream
- func (s *LogStream) DescriptorProto() (*logpb.LogStreamDescriptor, error)
- func (s *LogStream) DescriptorValue() (*logpb.LogStreamDescriptor, error)
- func (s *LogStream) Load(pmap ds.PropertyMap) error
- func (s *LogStream) LoadDescriptor(desc *logpb.LogStreamDescriptor) error
- func (s *LogStream) LogPrefix() *LogPrefix
- func (s *LogStream) Path() types.StreamPath
- func (s *LogStream) PopulateState(c context.Context, lst *LogStreamState)
- func (s *LogStream) Save(withMeta bool) (ds.PropertyMap, error)
- func (s *LogStream) SetDSValidate(v bool)
- func (s *LogStream) State(c context.Context) *LogStreamState
- func (s *LogStream) Validate() error
- type LogStreamState
- func (lst *LogStreamState) ArchivalState() ArchivalState
- func (lst *LogStreamState) ID() HashID
- func (lst *LogStreamState) Load(pmap ds.PropertyMap) error
- func (lst *LogStreamState) Save(withMeta bool) (ds.PropertyMap, error)
- func (lst *LogStreamState) Terminated() bool
- func (lst *LogStreamState) Validate() error
- type MembershipError
- type NamespaceAccessType
- type PubsubArchivalPublisher
- type SigningStorage
- type TagMap
- type URLSigningRequest
- type URLSigningResponse
Constants ¶
const CurrentSchemaVersion = "1"
CurrentSchemaVersion is the current schema version of the LogStream. Changes that are not backward-compatible should update this field so migration logic and scripts can translate appropriately.
const ( // RegistrationNonceTimeout is how long LogPrefix.IsRetry will consider // a matching nonce to be valid. RegistrationNonceTimeout = 15 * time.Minute )
Variables ¶
var ErrArchiveTasked = errors.New("archival already tasked for this stream")
ErrArchiveTasked is returned by ArchivalParams' PublishTask if the supplied LogStream indicates that it has already had an archival request dispatched.
var ErrPathNotFound = grpcutil.Errf(codes.NotFound, "path not found")
ErrPathNotFound is the canonical error returned when a Log Stream Path is not found.
Functions ¶
func AddLogStreamArchivedFilter ¶
AddLogStreamArchivedFilter returns a derived query that asserts that a log stream has been archived.
func AddLogStreamPathFilter ¶
AddLogStreamPathFilter constructs a compiled LogStreamPathQuery. It will return an error if the supllied query string describes an invalid query.
func AddLogStreamPurgedFilter ¶
AddLogStreamPurgedFilter returns a derived query that asserts that a log stream has been archived.
func AddLogStreamTagFilter ¶
AddLogStreamTagFilter adds a tag filter to a Query object.
This method will only add equality filters to the query. If value is empty, a presence filter will be added; otherwise, an equality filter will be added.
This incorporates the encoding expressed by TagMap.
func AddLogStreamTerminatedFilter ¶
AddLogStreamTerminatedFilter returns a derived query that asserts that a log stream has been terminated.
func AddNewerFilter ¶
AddNewerFilter adds a filter to queries that restricts them to results that were created after the supplied time.
func AddOlderFilter ¶
AddOlderFilter adds a filter to queries that restricts them to results that were created before the supplied time.
func CurrentProject ¶
func CurrentProject(c context.Context) types.ProjectName
CurrentProject returns the current project based on the currently-loaded namespace.
If there is no current namespace, or if the current namespace is not a valid project namespace, an empty string will be returned.
func CurrentProjectConfig ¶
func CurrentProjectConfig(c context.Context) (*svcconfig.ProjectConfig, error)
CurrentProjectConfig returns the project-specific configuration for the current project.
If there is no current project namespace, or if the current project has no configuration, config.ErrInvalidConfig will be returned.
func IsAdminUser ¶
IsAdminUser tests whether the current user belongs to the administrative users group.
If the user is not, a MembershipError will be returned.
func IsMembershipError ¶
IsMembershipError returns whether a given error is a membership error.
func IsProjectReader ¶
func IsProjectReader(c context.Context, pcfg *svcconfig.ProjectConfig) error
IsProjectReader tests whether the current user belongs to one of the project's declared reader groups.
If the user is not, a MembershipError will be returned.
func IsProjectWriter ¶
func IsProjectWriter(c context.Context, pcfg *svcconfig.ProjectConfig) error
IsProjectWriter tests whether the current user belongs to one of the project's declared writer groups.
If the user is not a member of any of the groups, a MembershipError will be returned.
func IsServiceUser ¶
IsServiceUser tests whether the current user belongs to the backend services users group.
If the user is not, a MembershipError will be returned.
func Project ¶
func Project(c context.Context) types.ProjectName
Project returns the current project installed in the supplied Context's namespace.
This function is called with the expectation that the Context is in a namespace conforming to ProjectNamespace. If this is not the case, this method will panic.
func ProjectFromNamespace ¶
func ProjectFromNamespace(ns string) types.ProjectName
ProjectFromNamespace returns the current project installed in the supplied Context's namespace.
If the namespace does not have a project namespace prefix, this function will return an empty string.
func ProjectNamespace ¶
func ProjectNamespace(project types.ProjectName) string
ProjectNamespace returns the AppEngine namespace for a given luci-config project name.
func WithConfigProvider ¶
func WithConfigProvider(c context.Context, s ConfigProvider) context.Context
WithConfigProvider installs the supplied ConfigProvider instance into a Context.
func WithProjectNamespace ¶
func WithProjectNamespace(c *context.Context, project types.ProjectName, at NamespaceAccessType) error
WithProjectNamespace sets the current namespace to the project name.
It will return a user-facing wrapped gRPC error on failure:
- InvalidArgument if the project name is invalid.
- If the project exists, then
- nil, if the user has the requested access.
- Unauthenticated if the user does not have the requested access, but is also not authenticated. This lets them know they should try again after authenticating.
- PermissionDenied if the user does not have the requested access.
- PermissionDenied if the project doesn't exist.
- Internal if an internal error occurred.
Types ¶
type ArchivalParams ¶
type ArchivalParams struct { // RequestID is the unique request ID to use as a random base for the // archival key. RequestID string // SettleDelay is the amount of settle delay to attach to this request. SettleDelay time.Duration // CompletePeriod is the amount of time after the initial archival task is // executed when the task should fail if the stream is incomplete. After this // period has expired, the archival may complete successfully even if the // stream is missing log entries. CompletePeriod time.Duration }
ArchivalParams is the archival configuration.
func (*ArchivalParams) PublishTask ¶
func (p *ArchivalParams) PublishTask(c context.Context, ap ArchivalPublisher, lst *LogStreamState) error
PublishTask creates and dispatches a task queue task for the supplied LogStream. PublishTask is goroutine-safe.
This should be run within a transaction on lst. On success, lst's state will be updated to reflect the archival tasking. This will NOT update lst's datastore entity; the caller must make sure to call Put within the same transaction for transactional safety.
If the task is created successfully, this will return nil. If the LogStream already had a task dispatched, it will return ErrArchiveTasked.
type ArchivalPublisher ¶
type ArchivalPublisher interface { // Close shutdowns this publisher instance, releasing all its resources. Close() error // Publish publishes the supplied ArchiveTask. Publish(context.Context, *logdog.ArchiveTask) error // NewPublishIndex returns a new publish index. Each publish index is unique // within its request. NewPublishIndex() uint64 }
ArchivalPublisher is capable of publishing archival requests.
type ArchivalState ¶
type ArchivalState int
ArchivalState describes the archival state of a LogStream.
const ( // NotArchived means that the stream is not archived, and that no archival has // been tasked. NotArchived ArchivalState = iota // ArchiveTasked is true if the log stream has an archival tasked, but has // not yet been archived. ArchiveTasked // ArchivedPartial means that the stream is archived, but that some log // entries are missing. ArchivedPartial // ArchivedComplete means that the stream is archived and all log entries are // present. ArchivedComplete )
func (ArchivalState) Archived ¶
func (as ArchivalState) Archived() bool
Archived returns true if this ArchivalState implies that the log stream is archived.
type ConfigProvider ¶
type ConfigProvider interface { // Config returns the current instance and application configuration // instances. // // The production instance will cache the results for the duration of the // request. Config(context.Context) (*config.Config, error) // ProjectConfig returns the project configuration for the named project. // // The production instance will cache the results for the duration of the // request. // // Returns the same error codes as config.ProjectConfig. ProjectConfig(context.Context, types.ProjectName) (*svcconfig.ProjectConfig, error) }
ConfigProvider is a set of support services used by Coordinator to fetch configurations.
Each instance is valid for a single request, but can be re-used throughout that request. This is advised, as the Services instance may optionally cache values.
ConfigServices methods are goroutine-safe.
func GetConfigProvider ¶
func GetConfigProvider(c context.Context) ConfigProvider
GetConfigProvider gets the ConfigProvider instance installed in the supplied Context.
If no Services has been installed, it will panic.
type HashID ¶
type HashID string
HashID is a hex-encoded SHA256 hash.
func LogPrefixID ¶
func LogPrefixID(prefix types.StreamName) HashID
LogPrefixID returns the HashID for a specific prefix.
func LogStreamID ¶
func LogStreamID(path types.StreamPath) HashID
LogStreamID returns the HashID for a given log stream path.
type LUCIConfigProvider ¶
type LUCIConfigProvider struct {
// contains filtered or unexported fields
}
LUCIConfigProvider is a ConfigProvider implementation that loads its configuration from the LUCI Config Service.
func (*LUCIConfigProvider) ProjectConfig ¶
func (s *LUCIConfigProvider) ProjectConfig(c context.Context, project types.ProjectName) (*svcconfig.ProjectConfig, error)
ProjectConfig implements ConfigProvider.
type LogPrefix ¶
type LogPrefix struct { // ID is the LogPrefix's ID. It is an encoded hash value generated from the // stream's Prefix field. ID HashID `gae:"$id"` // Schema is the datastore schema version for this object. This can be used // to facilitate schema migrations. // // The current schema is currentSchemaVersion. Schema string // Created is the time when this stream was created. Created time.Time `gae:",noindex"` // Prefix is this log stream's prefix value. Log streams with the same prefix // are logically grouped. // // This value should not be changed once populated, as it will invalidate the // HashID. Prefix string `gae:",noindex"` // Source is the (indexed) set of source strings sent by the prefix registrar. Source []string // Expiration is the time when this log prefix expires. Stream registrations // for this prefix will fail after this point. Expiration time.Time // Secret is the Butler secret value for this prefix. All streams within // the prefix share this secret value. // // This value may only be returned to LogDog services; it is not user-visible. Secret []byte `gae:",noindex"` // OpNonce is provided by the client when calling RegisterPrefix. If the // client provides the same nonce on a subsequent invocation of // RegisterPrefix, the server will respond with success instead of // AlreadyExists. // // This must have a length of either 0 or types.OpNonceLength. // // The nonce has a valid lifetime of RegistrationNonceTimeout after Created. OpNonce []byte `gae:",noindex"` // contains filtered or unexported fields }
LogPrefix is a datastore model for a prefix space. All log streams sharing a prefix will have a LogPrefix entry to group under.
A LogPrefix is keyed on the hash of its Prefix property.
Prefix-scoped properties are used to control creation and modification attributes of log streams sharing the prefix.
func (*LogPrefix) IsRetry ¶
IsRetry checks to see if this LogPrefix is still in the OpNonce window, and if nonce matches the one in this LogPrefix.
func (*LogPrefix) Load ¶
func (p *LogPrefix) Load(pmap ds.PropertyMap) error
Load implements ds.PropertyLoadSaver.
type LogStream ¶
type LogStream struct { // ID is the LogStream ID. It is generated from the stream's Prefix/Name // fields. ID HashID `gae:"$id"` // Schema is the datastore schema version for this object. This can be used // to facilitate schema migrations. // // The current schema is currentSchemaVersion. Schema string // Prefix is this log stream's prefix value. Log streams with the same prefix // are logically grouped. // // This value should not be changed once populated, as it will invalidate the // ID. Prefix string // Name is the unique name of this log stream within the Prefix scope. // // This value should not be changed once populated, as it will invalidate the // ID. Name string // Created is the time when this stream was created. Created time.Time // Purged, if true, indicates that this log stream has been marked as purged. // Non-administrative queries and requests for this stream will operate as // if this entry doesn't exist. Purged bool // PurgedTime is the time when this stream was purged. PurgedTime time.Time `gae:",noindex"` // ProtoVersion is the version string of the protobuf, as reported by the // Collector (and ultimately self-identified by the Butler). ProtoVersion string // Descriptor is the binary protobuf data LogStreamDescriptor. Descriptor []byte `gae:",noindex"` // ContentType is the MIME-style content type string for this stream. ContentType string // StreamType is the data type of the stream. StreamType logpb.StreamType // Timestamp is the Descriptor's recorded client-side timestamp. Timestamp time.Time // Tags is a set of arbitrary key/value tags associated with this stream. Tags // can be queried against. // // The serialization/deserialization is handled manually in order to enable // key/value queries. Tags TagMap `gae:"-"` // contains filtered or unexported fields }
LogStream is the primary datastore model containing information and state of an individual log stream.
This structure contains the standard queryable fields, and is the source of truth for log stream state. Writes to LogStream should be done via Put, which will ensure that the LogStream's related query objects are kept in sync.
This structure has additional datastore fields imposed by the PropertyLoadSaver. These fields enable querying against some of the complex data types:
_C breaks the the Prefix and Name fields into positionally-queryable entries. It is used to build globbing queries.
It is composed of entries detailing (T is the path [P]refix or [N]ame):
TF:n:value ("T" has a component, "value", at index "n").
TR:n:value ("T" has a component, "value", at reverse-index "n").
TC:count ("T" has "count" total elements).
For example, the path "foo/bar/+/baz" would break into: ["PF:0:foo", "PF:1:bar", "PR:0:bar", "PR:1:foo", "PC:2", "NF:0:baz", "NR:0:baz", "NC:1"].
_Tags is a string slice containing:
KEY=[VALUE] key/value tags.
KEY key presence tags.
func WithStream ¶
func WithStream(c context.Context, project types.ProjectName, path types.StreamPath, at NamespaceAccessType) (context.Context, *LogStream, error)
WithStream performs all ACL checks on a stream, including:
- Project Namespace check
- Logs purged check
This returns the stream metadata, and a new context tied to the project. The new context can be used to fetch the LogStreamState from the datastore. The returned error is a gRPC error, parsable with grpcutil.
func (*LogStream) DescriptorProto ¶
func (s *LogStream) DescriptorProto() (*logpb.LogStreamDescriptor, error)
DescriptorProto unmarshals a LogStreamDescriptor from the stream's Descriptor field. It will return an error if the unmarshalling fails.
func (*LogStream) DescriptorValue ¶
func (s *LogStream) DescriptorValue() (*logpb.LogStreamDescriptor, error)
DescriptorValue returns the unmarshalled Descriptor field protobuf.
func (*LogStream) Load ¶
func (s *LogStream) Load(pmap ds.PropertyMap) error
Load implements ds.PropertyLoadSaver.
func (*LogStream) LoadDescriptor ¶
func (s *LogStream) LoadDescriptor(desc *logpb.LogStreamDescriptor) error
LoadDescriptor loads the fields in the log stream descriptor into this LogStream entry. These fields are:
- Prefix
- Name
- ContentType
- StreamType
- Descriptor
- Timestamp
- Tags
func (*LogStream) LogPrefix ¶
LogPrefix returns a keyed (but not loaded) LogPrefix struct for this LogStream's Prefix.
func (*LogStream) Path ¶
func (s *LogStream) Path() types.StreamPath
Path returns the LogDog path for this log stream.
func (*LogStream) PopulateState ¶
func (s *LogStream) PopulateState(c context.Context, lst *LogStreamState)
PopulateState populates the datastore key fields for the supplied LogStreamState, binding them to the current LogStream.
func (*LogStream) Save ¶
func (s *LogStream) Save(withMeta bool) (ds.PropertyMap, error)
Save implements ds.PropertyLoadSaver.
func (*LogStream) SetDSValidate ¶
SetDSValidate controls whether this LogStream is validated prior to being read from or written to datastore.
This is a testing parameter, and should NOT be used in production code.
type LogStreamState ¶
type LogStreamState struct { // Parent is the key of the corresponding LogStream. Parent *ds.Key `gae:"$parent"` // Schema is the datastore schema version for this object. This can be used // to facilitate schema migrations. // // The current schema is CurrentSchemaVersion. Schema string `gae:",noindex"` // Created is the last time that this state has been created. Created time.Time `gae:",noindex"` // Updated is the last time that this state has been updated. Updated time.Time `gae:",noindex"` // Secret is the Butler secret value for this stream. // // This value may only be returned to LogDog services; it is not user-visible. Secret []byte `gae:",noindex"` // TerminatedTime is the Coordinator's record of when this log stream was // terminated. TerminatedTime time.Time `gae:",noindex"` // TerminalIndex is the index of the last log entry in the stream. // // If this is <0, the log stream is either still streaming or has been // archived with no log entries. TerminalIndex int64 `gae:",noindex"` // ArchivedTime is the Coordinator's record of when this log stream was // archived. If this is non-zero, it means that the log entry has been // archived. ArchivedTime time.Time `gae:",noindex"` // ArchiveLogEntryCount is the number of LogEntry records that were archived // for this log stream. // // This is valid only if the log stream is Archived. ArchiveLogEntryCount int64 `gae:",noindex"` // ArchivalKey is the archival key for this log stream. This is used to // differentiate the real archival request from those that were dispatched, // but that ultimately failed to update state. // // See createArchivalKey for details on its generation and usage. ArchivalKey []byte `gae:",noindex"` // ArchiveIndexURL is the Google Storage URL where the log stream's index is // archived. ArchiveIndexURL string `gae:",noindex"` // ArchiveIndexSize is the size, in bytes, of the archived Index. It will be // zero if the file is not archived. ArchiveIndexSize int64 `gae:",noindex"` // ArchiveStreamURL is the Google Storage URL where the log stream's raw // stream data is archived. If this is not empty, the log stream is considered // archived. ArchiveStreamURL string `gae:",noindex"` // ArchiveStreamSize is the size, in bytes, of the archived stream. It will be // zero if the file is not archived. ArchiveStreamSize int64 `gae:",noindex"` // ArchiveDataURL is the Google Storage URL where the log stream's assembled // data is archived. If this is not empty, the log stream is considered // archived. ArchiveDataURL string `gae:",noindex"` // ArchiveDataSize is the size, in bytes, of the archived data. It will be // zero if the file is not archived. ArchiveDataSize int64 `gae:",noindex"` // contains filtered or unexported fields }
LogStreamState contains the current state of a LogStream.
This structure has additional datastore fields imposed by the PropertyLoadSaver.
- _Terminated is true if the LogStream has been terminated.
- _ArchivePending is true if the LogStream currently has an archive task dispatched.
- _ArchivalState is true if the LogStream has been archived.
See services API's LogStreamState message type.
func NewLogStreamState ¶
func NewLogStreamState(c context.Context, id HashID) *LogStreamState
NewLogStreamState returns a LogStreamState with its parent key populated to the LogStream with the supplied ID.
func (*LogStreamState) ArchivalState ¶
func (lst *LogStreamState) ArchivalState() ArchivalState
ArchivalState returns the archival state of the log stream.
func (*LogStreamState) ID ¶
func (lst *LogStreamState) ID() HashID
ID returns the LogStream ID for the LogStream that owns this LogStreamState.
func (*LogStreamState) Load ¶
func (lst *LogStreamState) Load(pmap ds.PropertyMap) error
Load implements ds.PropertyLoadSaver.
func (*LogStreamState) Save ¶
func (lst *LogStreamState) Save(withMeta bool) (ds.PropertyMap, error)
Save implements ds.PropertyLoadSaver.
func (*LogStreamState) Terminated ¶
func (lst *LogStreamState) Terminated() bool
Terminated returns true if this stream has been terminated.
func (*LogStreamState) Validate ¶
func (lst *LogStreamState) Validate() error
Validate evaluates the state and data contents of the LogStreamState and returns an error if it is invalid.
type MembershipError ¶
MembershipError is an error returned by group membership checking functions if the current identity is not a member of the requested group.
func (*MembershipError) Error ¶
func (e *MembershipError) Error() string
type NamespaceAccessType ¶
type NamespaceAccessType int
NamespaceAccessType specifies the type of namespace access that is being requested for WithProjectNamespace.
const ( // NamespaceAccessNoAuth grants unconditional access to a project's namespace. // This bypasses all ACL checks, and must only be used by service endpoints // that explicitly apply ACLs elsewhere. NamespaceAccessNoAuth NamespaceAccessType = iota // NamespaceAccessAllTesting is an extension of NamespaceAccessNoAuth that, // in addition to doing no ACL checks, also does no project existence checks. // // This must ONLY be used for testing. NamespaceAccessAllTesting // NamespaceAccessREAD enforces READ permission access to a project's // namespace. NamespaceAccessREAD // NamespaceAccessWRITE enforces WRITE permission access to a project's // namespace. NamespaceAccessWRITE )
type PubsubArchivalPublisher ¶
type PubsubArchivalPublisher struct { // Publisher is the client used to publish messages. Publisher pubsub.Publisher // AECtx is the AppEngine Context to use for publish operations. AECtx context.Context // PublishIndexFunc is a function that will return a unique publish index // for this request. PublishIndexFunc func() uint64 }
PubsubArchivalPublisher is an ArchivalPublisher that uses Cloud Pub/Sub for task archival.
func (*PubsubArchivalPublisher) Close ¶
func (p *PubsubArchivalPublisher) Close() error
Close implements ArchivalPublisher.
func (*PubsubArchivalPublisher) NewPublishIndex ¶
func (p *PubsubArchivalPublisher) NewPublishIndex() uint64
NewPublishIndex implements ArchivalPublisher.
type SigningStorage ¶
type SigningStorage interface { // Storage is the base Storage instance. storage.Storage // GetSignedURLs attempts to sign the storage's stream's RecordIO archive // stream storage URL. // // If signing is not supported by this Storage instance, this will return // a nil signing response and no error. GetSignedURLs(context.Context, *URLSigningRequest) (*URLSigningResponse, error) }
SigningStorage is an interface to storage used by the Coordinator.
type TagMap ¶
TagMap is tag map that stores log stream tags into the datastore.
Tags are stored both as presence entries (Key) and as equality entries (Key=Value). Both entry contents are encoded via encodeKey.
type URLSigningRequest ¶
type URLSigningRequest struct { // Lifetime is the signed URL expiration time. Lifetime time.Duration // Stream, if true, requests a signed log stream URL. Stream bool // Index, if true, requests a signed log stream index URL. Index bool }
URLSigningRequest is the set of URL signing parameters passed to a SigningStorage.GetSignedURLs call.
func (*URLSigningRequest) HasWork ¶
func (r *URLSigningRequest) HasWork() bool
HasWork returns true if this signing request actually has work that is requested.
type URLSigningResponse ¶
type URLSigningResponse struct { // Expriation is the signed URL expiration time. Expiration time.Time // Stream is the signed URL for the log stream, if requested. Stream string // Index is the signed URL for the log stream index, if requested. Index string }
URLSigningResponse is the resulting signed URLs from a SigningStorage.GetSignedURLs call.