contentwriter

package
v1.0.0-beta9 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 28, 2019 License: Apache-2.0 Imports: 9 Imported by: 1

Documentation

Index

Constants

This section is empty.

Variables

View Source
var RecordType_name = map[int32]string{
	0: "WARCINFO",
	1: "RESPONSE",
	2: "RESOURCE",
	3: "REQUEST",
	4: "METADATA",
	5: "REVISIT",
	6: "CONVERSION",
	7: "CONTINUATION",
}
View Source
var RecordType_value = map[string]int32{
	"WARCINFO":     0,
	"RESPONSE":     1,
	"RESOURCE":     2,
	"REQUEST":      3,
	"METADATA":     4,
	"REVISIT":      5,
	"CONVERSION":   6,
	"CONTINUATION": 7,
}

Functions

func RegisterContentWriterServer

func RegisterContentWriterServer(s *grpc.Server, srv ContentWriterServer)

Types

type ContentWriterClient

type ContentWriterClient interface {
	Write(ctx context.Context, opts ...grpc.CallOption) (ContentWriter_WriteClient, error)
}

ContentWriterClient is the client API for ContentWriter service.

For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.

func NewContentWriterClient

func NewContentWriterClient(cc *grpc.ClientConn) ContentWriterClient

type ContentWriterServer

type ContentWriterServer interface {
	Write(ContentWriter_WriteServer) error
}

ContentWriterServer is the server API for ContentWriter service.

type ContentWriter_WriteClient

type ContentWriter_WriteClient interface {
	Send(*WriteRequest) error
	CloseAndRecv() (*WriteReply, error)
	grpc.ClientStream
}

type ContentWriter_WriteServer

type ContentWriter_WriteServer interface {
	SendAndClose(*WriteReply) error
	Recv() (*WriteRequest, error)
	grpc.ServerStream
}

type CrawledContent

type CrawledContent struct {
	Digest               string               `protobuf:"bytes,1,opt,name=digest,proto3" json:"digest,omitempty"`
	WarcId               string               `protobuf:"bytes,2,opt,name=warc_id,json=warcId,proto3" json:"warc_id,omitempty"`
	TargetUri            string               `protobuf:"bytes,3,opt,name=target_uri,json=targetUri,proto3" json:"target_uri,omitempty"`
	Date                 *timestamp.Timestamp `protobuf:"bytes,4,opt,name=date,proto3" json:"date,omitempty"`
	XXX_NoUnkeyedLiteral struct{}             `json:"-"`
	XXX_unrecognized     []byte               `json:"-"`
	XXX_sizecache        int32                `json:"-"`
}

func (*CrawledContent) Descriptor

func (*CrawledContent) Descriptor() ([]byte, []int)

func (*CrawledContent) GetDate

func (m *CrawledContent) GetDate() *timestamp.Timestamp

func (*CrawledContent) GetDigest

func (m *CrawledContent) GetDigest() string

func (*CrawledContent) GetTargetUri

func (m *CrawledContent) GetTargetUri() string

func (*CrawledContent) GetWarcId

func (m *CrawledContent) GetWarcId() string

func (*CrawledContent) ProtoMessage

func (*CrawledContent) ProtoMessage()

func (*CrawledContent) Reset

func (m *CrawledContent) Reset()

func (*CrawledContent) String

func (m *CrawledContent) String() string

func (*CrawledContent) XXX_DiscardUnknown

func (m *CrawledContent) XXX_DiscardUnknown()

func (*CrawledContent) XXX_Marshal

func (m *CrawledContent) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*CrawledContent) XXX_Merge

func (m *CrawledContent) XXX_Merge(src proto.Message)

func (*CrawledContent) XXX_Size

func (m *CrawledContent) XXX_Size() int

func (*CrawledContent) XXX_Unmarshal

func (m *CrawledContent) XXX_Unmarshal(b []byte) error

type Data

type Data struct {
	RecordNum            int32    `protobuf:"varint,1,opt,name=record_num,json=recordNum,proto3" json:"record_num,omitempty"`
	Data                 []byte   `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*Data) Descriptor

func (*Data) Descriptor() ([]byte, []int)

func (*Data) GetData

func (m *Data) GetData() []byte

func (*Data) GetRecordNum

func (m *Data) GetRecordNum() int32

func (*Data) ProtoMessage

func (*Data) ProtoMessage()

func (*Data) Reset

func (m *Data) Reset()

func (*Data) String

func (m *Data) String() string

func (*Data) XXX_DiscardUnknown

func (m *Data) XXX_DiscardUnknown()

func (*Data) XXX_Marshal

func (m *Data) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Data) XXX_Merge

func (m *Data) XXX_Merge(src proto.Message)

func (*Data) XXX_Size

func (m *Data) XXX_Size() int

func (*Data) XXX_Unmarshal

func (m *Data) XXX_Unmarshal(b []byte) error

type RecordType

type RecordType int32
const (
	RecordType_WARCINFO     RecordType = 0
	RecordType_RESPONSE     RecordType = 1
	RecordType_RESOURCE     RecordType = 2
	RecordType_REQUEST      RecordType = 3
	RecordType_METADATA     RecordType = 4
	RecordType_REVISIT      RecordType = 5
	RecordType_CONVERSION   RecordType = 6
	RecordType_CONTINUATION RecordType = 7
)

func (RecordType) EnumDescriptor

func (RecordType) EnumDescriptor() ([]byte, []int)

func (RecordType) String

func (x RecordType) String() string

type StorageRef

type StorageRef struct {
	WarcId               string     `protobuf:"bytes,1,opt,name=warc_id,json=warcId,proto3" json:"warc_id,omitempty"`
	RecordType           RecordType `` /* 139-byte string literal not displayed */
	StorageRef           string     `protobuf:"bytes,3,opt,name=storage_ref,json=storageRef,proto3" json:"storage_ref,omitempty"`
	XXX_NoUnkeyedLiteral struct{}   `json:"-"`
	XXX_unrecognized     []byte     `json:"-"`
	XXX_sizecache        int32      `json:"-"`
}

func (*StorageRef) Descriptor

func (*StorageRef) Descriptor() ([]byte, []int)

func (*StorageRef) GetRecordType

func (m *StorageRef) GetRecordType() RecordType

func (*StorageRef) GetStorageRef

func (m *StorageRef) GetStorageRef() string

func (*StorageRef) GetWarcId

func (m *StorageRef) GetWarcId() string

func (*StorageRef) ProtoMessage

func (*StorageRef) ProtoMessage()

func (*StorageRef) Reset

func (m *StorageRef) Reset()

func (*StorageRef) String

func (m *StorageRef) String() string

func (*StorageRef) XXX_DiscardUnknown

func (m *StorageRef) XXX_DiscardUnknown()

func (*StorageRef) XXX_Marshal

func (m *StorageRef) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*StorageRef) XXX_Merge

func (m *StorageRef) XXX_Merge(src proto.Message)

func (*StorageRef) XXX_Size

func (m *StorageRef) XXX_Size() int

func (*StorageRef) XXX_Unmarshal

func (m *StorageRef) XXX_Unmarshal(b []byte) error

type WriteReply

type WriteReply struct {
	Meta                 *WriteResponseMeta `protobuf:"bytes,1,opt,name=meta,proto3" json:"meta,omitempty"`
	XXX_NoUnkeyedLiteral struct{}           `json:"-"`
	XXX_unrecognized     []byte             `json:"-"`
	XXX_sizecache        int32              `json:"-"`
}

func (*WriteReply) Descriptor

func (*WriteReply) Descriptor() ([]byte, []int)

func (*WriteReply) GetMeta

func (m *WriteReply) GetMeta() *WriteResponseMeta

func (*WriteReply) ProtoMessage

func (*WriteReply) ProtoMessage()

func (*WriteReply) Reset

func (m *WriteReply) Reset()

func (*WriteReply) String

func (m *WriteReply) String() string

func (*WriteReply) XXX_DiscardUnknown

func (m *WriteReply) XXX_DiscardUnknown()

func (*WriteReply) XXX_Marshal

func (m *WriteReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*WriteReply) XXX_Merge

func (m *WriteReply) XXX_Merge(src proto.Message)

func (*WriteReply) XXX_Size

func (m *WriteReply) XXX_Size() int

func (*WriteReply) XXX_Unmarshal

func (m *WriteReply) XXX_Unmarshal(b []byte) error

type WriteRequest

type WriteRequest struct {
	// Types that are valid to be assigned to Value:
	//	*WriteRequest_Meta
	//	*WriteRequest_ProtocolHeader
	//	*WriteRequest_Payload
	//	*WriteRequest_Cancel
	Value                isWriteRequest_Value `protobuf_oneof:"value"`
	XXX_NoUnkeyedLiteral struct{}             `json:"-"`
	XXX_unrecognized     []byte               `json:"-"`
	XXX_sizecache        int32                `json:"-"`
}

func (*WriteRequest) Descriptor

func (*WriteRequest) Descriptor() ([]byte, []int)

func (*WriteRequest) GetCancel

func (m *WriteRequest) GetCancel() string

func (*WriteRequest) GetMeta

func (m *WriteRequest) GetMeta() *WriteRequestMeta

func (*WriteRequest) GetPayload

func (m *WriteRequest) GetPayload() *Data

func (*WriteRequest) GetProtocolHeader

func (m *WriteRequest) GetProtocolHeader() *Data

func (*WriteRequest) GetValue

func (m *WriteRequest) GetValue() isWriteRequest_Value

func (*WriteRequest) ProtoMessage

func (*WriteRequest) ProtoMessage()

func (*WriteRequest) Reset

func (m *WriteRequest) Reset()

func (*WriteRequest) String

func (m *WriteRequest) String() string

func (*WriteRequest) XXX_DiscardUnknown

func (m *WriteRequest) XXX_DiscardUnknown()

func (*WriteRequest) XXX_Marshal

func (m *WriteRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*WriteRequest) XXX_Merge

func (m *WriteRequest) XXX_Merge(src proto.Message)

func (*WriteRequest) XXX_OneofWrappers

func (*WriteRequest) XXX_OneofWrappers() []interface{}

XXX_OneofWrappers is for the internal use of the proto package.

func (*WriteRequest) XXX_Size

func (m *WriteRequest) XXX_Size() int

func (*WriteRequest) XXX_Unmarshal

func (m *WriteRequest) XXX_Unmarshal(b []byte) error

type WriteRequestMeta

type WriteRequestMeta struct {
	ExecutionId          string                                 `protobuf:"bytes,1,opt,name=execution_id,json=executionId,proto3" json:"execution_id,omitempty"`
	TargetUri            string                                 `protobuf:"bytes,2,opt,name=target_uri,json=targetUri,proto3" json:"target_uri,omitempty"`
	RecordMeta           map[int32]*WriteRequestMeta_RecordMeta `` /* 180-byte string literal not displayed */
	FetchTimeStamp       *timestamp.Timestamp                   `protobuf:"bytes,4,opt,name=fetch_time_stamp,json=fetchTimeStamp,proto3" json:"fetch_time_stamp,omitempty"`
	IpAddress            string                                 `protobuf:"bytes,5,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"`
	CollectionRef        *v1.ConfigRef                          `protobuf:"bytes,7,opt,name=collection_ref,json=collectionRef,proto3" json:"collection_ref,omitempty"`
	XXX_NoUnkeyedLiteral struct{}                               `json:"-"`
	XXX_unrecognized     []byte                                 `json:"-"`
	XXX_sizecache        int32                                  `json:"-"`
}

func (*WriteRequestMeta) Descriptor

func (*WriteRequestMeta) Descriptor() ([]byte, []int)

func (*WriteRequestMeta) GetCollectionRef

func (m *WriteRequestMeta) GetCollectionRef() *v1.ConfigRef

func (*WriteRequestMeta) GetExecutionId

func (m *WriteRequestMeta) GetExecutionId() string

func (*WriteRequestMeta) GetFetchTimeStamp

func (m *WriteRequestMeta) GetFetchTimeStamp() *timestamp.Timestamp

func (*WriteRequestMeta) GetIpAddress

func (m *WriteRequestMeta) GetIpAddress() string

func (*WriteRequestMeta) GetRecordMeta

func (m *WriteRequestMeta) GetRecordMeta() map[int32]*WriteRequestMeta_RecordMeta

func (*WriteRequestMeta) GetTargetUri

func (m *WriteRequestMeta) GetTargetUri() string

func (*WriteRequestMeta) ProtoMessage

func (*WriteRequestMeta) ProtoMessage()

func (*WriteRequestMeta) Reset

func (m *WriteRequestMeta) Reset()

func (*WriteRequestMeta) String

func (m *WriteRequestMeta) String() string

func (*WriteRequestMeta) XXX_DiscardUnknown

func (m *WriteRequestMeta) XXX_DiscardUnknown()

func (*WriteRequestMeta) XXX_Marshal

func (m *WriteRequestMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*WriteRequestMeta) XXX_Merge

func (m *WriteRequestMeta) XXX_Merge(src proto.Message)

func (*WriteRequestMeta) XXX_Size

func (m *WriteRequestMeta) XXX_Size() int

func (*WriteRequestMeta) XXX_Unmarshal

func (m *WriteRequestMeta) XXX_Unmarshal(b []byte) error

type WriteRequestMeta_RecordMeta

type WriteRequestMeta_RecordMeta struct {
	RecordNum            int32                           `protobuf:"varint,1,opt,name=record_num,json=recordNum,proto3" json:"record_num,omitempty"`
	Type                 RecordType                      `protobuf:"varint,2,opt,name=type,proto3,enum=veidemann.api.contentwriter.v1.RecordType" json:"type,omitempty"`
	RecordContentType    string                          `protobuf:"bytes,3,opt,name=record_content_type,json=recordContentType,proto3" json:"record_content_type,omitempty"`
	BlockDigest          string                          `protobuf:"bytes,4,opt,name=block_digest,json=blockDigest,proto3" json:"block_digest,omitempty"`
	PayloadDigest        string                          `protobuf:"bytes,5,opt,name=payload_digest,json=payloadDigest,proto3" json:"payload_digest,omitempty"`
	Size                 int64                           `protobuf:"varint,6,opt,name=size,proto3" json:"size,omitempty"`
	SubCollection        v1.Collection_SubCollectionType `` /* 159-byte string literal not displayed */
	WarcConcurrentTo     []string                        `protobuf:"bytes,8,rep,name=warc_concurrent_to,json=warcConcurrentTo,proto3" json:"warc_concurrent_to,omitempty"`
	XXX_NoUnkeyedLiteral struct{}                        `json:"-"`
	XXX_unrecognized     []byte                          `json:"-"`
	XXX_sizecache        int32                           `json:"-"`
}

func (*WriteRequestMeta_RecordMeta) Descriptor

func (*WriteRequestMeta_RecordMeta) Descriptor() ([]byte, []int)

func (*WriteRequestMeta_RecordMeta) GetBlockDigest

func (m *WriteRequestMeta_RecordMeta) GetBlockDigest() string

func (*WriteRequestMeta_RecordMeta) GetPayloadDigest

func (m *WriteRequestMeta_RecordMeta) GetPayloadDigest() string

func (*WriteRequestMeta_RecordMeta) GetRecordContentType

func (m *WriteRequestMeta_RecordMeta) GetRecordContentType() string

func (*WriteRequestMeta_RecordMeta) GetRecordNum

func (m *WriteRequestMeta_RecordMeta) GetRecordNum() int32

func (*WriteRequestMeta_RecordMeta) GetSize

func (m *WriteRequestMeta_RecordMeta) GetSize() int64

func (*WriteRequestMeta_RecordMeta) GetSubCollection

func (*WriteRequestMeta_RecordMeta) GetType

func (*WriteRequestMeta_RecordMeta) GetWarcConcurrentTo

func (m *WriteRequestMeta_RecordMeta) GetWarcConcurrentTo() []string

func (*WriteRequestMeta_RecordMeta) ProtoMessage

func (*WriteRequestMeta_RecordMeta) ProtoMessage()

func (*WriteRequestMeta_RecordMeta) Reset

func (m *WriteRequestMeta_RecordMeta) Reset()

func (*WriteRequestMeta_RecordMeta) String

func (m *WriteRequestMeta_RecordMeta) String() string

func (*WriteRequestMeta_RecordMeta) XXX_DiscardUnknown

func (m *WriteRequestMeta_RecordMeta) XXX_DiscardUnknown()

func (*WriteRequestMeta_RecordMeta) XXX_Marshal

func (m *WriteRequestMeta_RecordMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*WriteRequestMeta_RecordMeta) XXX_Merge

func (m *WriteRequestMeta_RecordMeta) XXX_Merge(src proto.Message)

func (*WriteRequestMeta_RecordMeta) XXX_Size

func (m *WriteRequestMeta_RecordMeta) XXX_Size() int

func (*WriteRequestMeta_RecordMeta) XXX_Unmarshal

func (m *WriteRequestMeta_RecordMeta) XXX_Unmarshal(b []byte) error

type WriteRequest_Cancel

type WriteRequest_Cancel struct {
	Cancel string `protobuf:"bytes,4,opt,name=cancel,proto3,oneof"`
}

type WriteRequest_Meta

type WriteRequest_Meta struct {
	Meta *WriteRequestMeta `protobuf:"bytes,1,opt,name=meta,proto3,oneof"`
}

type WriteRequest_Payload

type WriteRequest_Payload struct {
	Payload *Data `protobuf:"bytes,3,opt,name=payload,proto3,oneof"`
}

type WriteRequest_ProtocolHeader

type WriteRequest_ProtocolHeader struct {
	ProtocolHeader *Data `protobuf:"bytes,2,opt,name=protocol_header,json=protocolHeader,proto3,oneof"`
}

type WriteResponseMeta

type WriteResponseMeta struct {
	RecordMeta           map[int32]*WriteResponseMeta_RecordMeta `` /* 180-byte string literal not displayed */
	XXX_NoUnkeyedLiteral struct{}                                `json:"-"`
	XXX_unrecognized     []byte                                  `json:"-"`
	XXX_sizecache        int32                                   `json:"-"`
}

func (*WriteResponseMeta) Descriptor

func (*WriteResponseMeta) Descriptor() ([]byte, []int)

func (*WriteResponseMeta) GetRecordMeta

func (m *WriteResponseMeta) GetRecordMeta() map[int32]*WriteResponseMeta_RecordMeta

func (*WriteResponseMeta) ProtoMessage

func (*WriteResponseMeta) ProtoMessage()

func (*WriteResponseMeta) Reset

func (m *WriteResponseMeta) Reset()

func (*WriteResponseMeta) String

func (m *WriteResponseMeta) String() string

func (*WriteResponseMeta) XXX_DiscardUnknown

func (m *WriteResponseMeta) XXX_DiscardUnknown()

func (*WriteResponseMeta) XXX_Marshal

func (m *WriteResponseMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*WriteResponseMeta) XXX_Merge

func (m *WriteResponseMeta) XXX_Merge(src proto.Message)

func (*WriteResponseMeta) XXX_Size

func (m *WriteResponseMeta) XXX_Size() int

func (*WriteResponseMeta) XXX_Unmarshal

func (m *WriteResponseMeta) XXX_Unmarshal(b []byte) error

type WriteResponseMeta_RecordMeta

type WriteResponseMeta_RecordMeta struct {
	RecordNum     int32      `protobuf:"varint,1,opt,name=record_num,json=recordNum,proto3" json:"record_num,omitempty"`
	Type          RecordType `protobuf:"varint,2,opt,name=type,proto3,enum=veidemann.api.contentwriter.v1.RecordType" json:"type,omitempty"`
	WarcId        string     `protobuf:"bytes,3,opt,name=warc_id,json=warcId,proto3" json:"warc_id,omitempty"`
	StorageRef    string     `protobuf:"bytes,4,opt,name=storage_ref,json=storageRef,proto3" json:"storage_ref,omitempty"`
	BlockDigest   string     `protobuf:"bytes,5,opt,name=block_digest,json=blockDigest,proto3" json:"block_digest,omitempty"`
	PayloadDigest string     `protobuf:"bytes,6,opt,name=payload_digest,json=payloadDigest,proto3" json:"payload_digest,omitempty"`
	// If the stored record was detected as a revisit, this field contains the id of the revisited record.
	RevisitReferenceId   string   `protobuf:"bytes,7,opt,name=revisit_reference_id,json=revisitReferenceId,proto3" json:"revisit_reference_id,omitempty"`
	CollectionFinalName  string   `protobuf:"bytes,8,opt,name=collection_final_name,json=collectionFinalName,proto3" json:"collection_final_name,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*WriteResponseMeta_RecordMeta) Descriptor

func (*WriteResponseMeta_RecordMeta) Descriptor() ([]byte, []int)

func (*WriteResponseMeta_RecordMeta) GetBlockDigest

func (m *WriteResponseMeta_RecordMeta) GetBlockDigest() string

func (*WriteResponseMeta_RecordMeta) GetCollectionFinalName

func (m *WriteResponseMeta_RecordMeta) GetCollectionFinalName() string

func (*WriteResponseMeta_RecordMeta) GetPayloadDigest

func (m *WriteResponseMeta_RecordMeta) GetPayloadDigest() string

func (*WriteResponseMeta_RecordMeta) GetRecordNum

func (m *WriteResponseMeta_RecordMeta) GetRecordNum() int32

func (*WriteResponseMeta_RecordMeta) GetRevisitReferenceId

func (m *WriteResponseMeta_RecordMeta) GetRevisitReferenceId() string

func (*WriteResponseMeta_RecordMeta) GetStorageRef

func (m *WriteResponseMeta_RecordMeta) GetStorageRef() string

func (*WriteResponseMeta_RecordMeta) GetType

func (*WriteResponseMeta_RecordMeta) GetWarcId

func (m *WriteResponseMeta_RecordMeta) GetWarcId() string

func (*WriteResponseMeta_RecordMeta) ProtoMessage

func (*WriteResponseMeta_RecordMeta) ProtoMessage()

func (*WriteResponseMeta_RecordMeta) Reset

func (m *WriteResponseMeta_RecordMeta) Reset()

func (*WriteResponseMeta_RecordMeta) String

func (*WriteResponseMeta_RecordMeta) XXX_DiscardUnknown

func (m *WriteResponseMeta_RecordMeta) XXX_DiscardUnknown()

func (*WriteResponseMeta_RecordMeta) XXX_Marshal

func (m *WriteResponseMeta_RecordMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*WriteResponseMeta_RecordMeta) XXX_Merge

func (m *WriteResponseMeta_RecordMeta) XXX_Merge(src proto.Message)

func (*WriteResponseMeta_RecordMeta) XXX_Size

func (m *WriteResponseMeta_RecordMeta) XXX_Size() int

func (*WriteResponseMeta_RecordMeta) XXX_Unmarshal

func (m *WriteResponseMeta_RecordMeta) XXX_Unmarshal(b []byte) error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL