Documentation ¶
Overview ¶
Go version of a Disco Distributed Filesystem (DDFS) client
Author: Tait Clarridge <tait@clarridge.ca>
Index ¶
- Constants
- Variables
- func ChunkToTag(ddfs *DDFSClient, tag string, urls []string, size int, conf *TagConfig) ([][]string, error)
- func DelTagAttr(ddfs *DDFSClient, tag string, attr string) error
- func GetUploadTargets(ddfs *DDFSClient, blob string) (*[]string, error)
- func ListTags(ddfs *DDFSClient, prefix string) ([]string, error)
- func SetTagAttr(ddfs *DDFSClient, tag string, attr string, val interface{}) error
- func TagBlobs(ddfs *DDFSClient, tag string, u [][]string, conf *TagConfig) ([]byte, [][]string, error)
- func TagDelete(ddfs *DDFSClient, tag string) error
- func Upload(ddfs *DDFSClient, blob string, data []byte) ([]string, error)
- func Version() string
- type Chunker
- type DDFSClient
- func (ddfs *DDFSClient) Chunk(tag string, urls []string, size int, conf *TagConfig) ([][]string, error)
- func (ddfs *DDFSClient) Communicate(method string, url string, data []byte) ([]byte, error)
- func (ddfs *DDFSClient) DelTagAttr(tag string, attr string) error
- func (ddfs *DDFSClient) Delete(tag string) error
- func (ddfs *DDFSClient) Exists(tag string) bool
- func (ddfs *DDFSClient) GetBlobs(tag string) ([][]string, error)
- func (ddfs *DDFSClient) GetTagAttr(tag string, attr string) (string, error)
- func (ddfs *DDFSClient) GetTagAttrs(tag string) (map[string]string, error)
- func (ddfs *DDFSClient) List(prefix string) ([]string, error)
- func (ddfs *DDFSClient) Push(tag string, urls []string, conf *TagConfig) (map[string][][]string, error)
- func (ddfs *DDFSClient) SetTagAttr(tag string, attr string, val interface{}) error
- func (ddfs *DDFSClient) Tag(tag string, urls [][]string, conf *TagConfig) error
- type DDFSTag
- type DiscoOutputStream
- type TagConfig
Constants ¶
const ( KILOBYTE = 1024.0 MEGABYTE = 1024 * KILOBYTE GIGABYTE = 1024 * MEGABYTE TERABYTE = 1024 * GIGABYTE )
Variables ¶
var UNSAFE_RE = regexp.MustCompile("[^A-Za-z0-9_\\-@:]")
Regexp for unsafe characters to replace for blob naming in DDFS
Functions ¶
func ChunkToTag ¶
func ChunkToTag(ddfs *DDFSClient, tag string, urls []string, size int, conf *TagConfig) ([][]string, error)
Chunk the files denoted by urls []string to a DDFS tag
func DelTagAttr ¶
func DelTagAttr(ddfs *DDFSClient, tag string, attr string) error
Delete a tag attribute
func GetUploadTargets ¶
func GetUploadTargets(ddfs *DDFSClient, blob string) (*[]string, error)
Query disco for the locations to upload blobs to
func SetTagAttr ¶
func SetTagAttr(ddfs *DDFSClient, tag string, attr string, val interface{}) error
Set a tag attribute
func TagBlobs ¶
func TagBlobs(ddfs *DDFSClient, tag string, u [][]string, conf *TagConfig) ([]byte, [][]string, error)
Tag the blobs to DDFS
func TagDelete ¶
func TagDelete(ddfs *DDFSClient, tag string) error
Types ¶
type Chunker ¶
type Chunker struct { DDFS *DDFSClient Urls [][]string Scanner *bufio.Scanner OutputStream *DiscoOutputStream Location string Index int ChunkSize int TagConf *TagConfig }
func NewChunker ¶
func NewChunker(ddfs *DDFSClient, loc string, size int, conf *TagConfig) *Chunker
Returns a new Chunker struct for chunking data
func (*Chunker) Flush ¶
func (chunker *Chunker) Flush()
Write the chunk to DDFS and reset the Output stream
func (*Chunker) WriteChunk ¶
Write the Chunk to DDFS
type DDFSClient ¶
type DDFSClient struct { Master string Port string Url string // contains filtered or unexported fields }
func NewDDFSClient ¶
func NewDDFSClient(master string, port string, timeout time.Duration) *DDFSClient
Returns a new DDFSClient struct for interaction with DDFS
func (*DDFSClient) Chunk ¶
func (ddfs *DDFSClient) Chunk(tag string, urls []string, size int, conf *TagConfig) ([][]string, error)
Chunk items to a tag, returns the urls the inputs were tagged to
func (*DDFSClient) Communicate ¶
Wrapper for communication with disco
func (*DDFSClient) DelTagAttr ¶
func (ddfs *DDFSClient) DelTagAttr(tag string, attr string) error
Delete a single tag attribute
func (*DDFSClient) GetBlobs ¶
func (ddfs *DDFSClient) GetBlobs(tag string) ([][]string, error)
Get blob locations for the tag
func (*DDFSClient) GetTagAttr ¶
func (ddfs *DDFSClient) GetTagAttr(tag string, attr string) (string, error)
Get a single named attribute
func (*DDFSClient) GetTagAttrs ¶
func (ddfs *DDFSClient) GetTagAttrs(tag string) (map[string]string, error)
Get all the tag attributes
func (*DDFSClient) List ¶
func (ddfs *DDFSClient) List(prefix string) ([]string, error)
List DDFS tags, can pass in a prefix that the tags start with
func (*DDFSClient) Push ¶
func (ddfs *DDFSClient) Push(tag string, urls []string, conf *TagConfig) (map[string][][]string, error)
Push raw files as blobs to DDFS
func (*DDFSClient) SetTagAttr ¶
func (ddfs *DDFSClient) SetTagAttr(tag string, attr string, val interface{}) error
Set a single named attribute
type DDFSTag ¶
type DDFSTag struct { Version int `json:"version"` ID string `json:"id"` LastModified string `json:"last-modified"` Urls [][]string `json:"urls"` UserData map[string]string `json:"user-data"` }
Struct for tag information unmarshaling
type DiscoOutputStream ¶
type DiscoOutputStream struct { Hunk *bytes.Buffer Output *bytes.Buffer Stream *bufio.Scanner PickleBuf *bytes.Buffer BinaryBuf *bytes.Buffer CompressBuf *bytes.Buffer HunkSize int MaxRecordSize int MinHunkSize int }
func NewOutputStream ¶
func NewOutputStream(stream *bufio.Scanner) *DiscoOutputStream
Returns a new output stream, used for chunking data
func (*DiscoOutputStream) Append ¶
func (output *DiscoOutputStream) Append(rec string)
Append a line to the output stream
func (*DiscoOutputStream) Flush ¶
func (output *DiscoOutputStream) Flush()
Assemble the hunk(s) to be a chunk
func (*DiscoOutputStream) Size ¶
func (output *DiscoOutputStream) Size() int
Get the size of the output stream buffer