goddfs

package module
v0.0.0-...-f57ee9b Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 5, 2015 License: MIT Imports: 17 Imported by: 0

README

goddfs

goddfs is a Go API for Disco DDFS. Requires https://github.com/hydrogen18/stalecucumber.

WARNING: This is still under development and is subject to change.

GODOC: https://godoc.org/github.com/chango/goddfs

Usage

go get github.com/chango/goddfs
go get github.com/hydrogen18/stalecucumber
import "github.com/chango/goddfs"
import "time"
import "fmt"
...
var err error
timeout := time.Second * 100
ddfs := goddfs.NewDDFSClient("localhost", "8989", timeout)

// Set tag attribute
err = ddfs.SetTagAttr("test:tag", "test", "test attribute")
if err != nil {
    fmt.Printf("Failed to set tag attr: %s\n", err)
}

// Get tag attributes
attrs, _ := ddfs.GetTagAttrs("test:tag")
for k, v := range attrs {
    fmt.Printf("ATTR: %s\tVAL: %s\n", k, v)
}

// Get single tag attribute
aaa, _ := ddfs.GetTagAttr("test:tag", "test")
fmt.Println(aaa)

// Delete tag attribute
err = ddfs.DelTagAttr("test:tag", "test") 
if err != nil {
    fmt.Printf("Failed to del attr: %s\n", err)
}

// Create the tag operation config (delayed bool, update bool)
tconf := goddfs.NewTagConfig(true, false)

// Chunk to DDFS (tag, path_to_file, chunk_size, tag_config)
urls, err := ddfs.Chunk("test:tag", []string{"/tmp/fileofstuff"}, 1048576, tconf)
if err != nil {
    fmt.Printf("Failed to chunk: %s\n", err)
}

fmt.Println("Chunked to: ", urls)

// Tag chunked blobs to another tag
ddfs.Tag("test:tag2", urls, tconf)

Documentation

Overview

Go version of a Disco Distributed Filesystem (DDFS) client

Author: Tait Clarridge <tait@clarridge.ca>

Index

Constants

View Source
const (
	KILOBYTE = 1024.0
	MEGABYTE = 1024 * KILOBYTE
	GIGABYTE = 1024 * MEGABYTE
	TERABYTE = 1024 * GIGABYTE
)

Variables

View Source
var UNSAFE_RE = regexp.MustCompile("[^A-Za-z0-9_\\-@:]")

Regexp for unsafe characters to replace for blob naming in DDFS

Functions

func ChunkToTag

func ChunkToTag(ddfs *DDFSClient, tag string, urls []string, size int, conf *TagConfig) ([][]string, error)

Chunk the files denoted by urls []string to a DDFS tag

func DelTagAttr

func DelTagAttr(ddfs *DDFSClient, tag string, attr string) error

Delete a tag attribute

func GetUploadTargets

func GetUploadTargets(ddfs *DDFSClient, blob string) (*[]string, error)

Query disco for the locations to upload blobs to

func ListTags

func ListTags(ddfs *DDFSClient, prefix string) ([]string, error)

func SetTagAttr

func SetTagAttr(ddfs *DDFSClient, tag string, attr string, val interface{}) error

Set a tag attribute

func TagBlobs

func TagBlobs(ddfs *DDFSClient, tag string, u [][]string, conf *TagConfig) ([]byte, [][]string, error)

Tag the blobs to DDFS

func TagDelete

func TagDelete(ddfs *DDFSClient, tag string) error

func Upload

func Upload(ddfs *DDFSClient, blob string, data []byte) ([]string, error)

Upload (push) the blobs to DDFS

func Version

func Version() string

Types

type Chunker

type Chunker struct {
	DDFS         *DDFSClient
	Urls         [][]string
	Scanner      *bufio.Scanner
	OutputStream *DiscoOutputStream
	Location     string
	Index        int
	ChunkSize    int
	TagConf      *TagConfig
}

func NewChunker

func NewChunker(ddfs *DDFSClient, loc string, size int, conf *TagConfig) *Chunker

Returns a new Chunker struct for chunking data

func (*Chunker) ChunkIter

func (chunker *Chunker) ChunkIter(tag string)

Iterates through the file, flushing when the chunk size (cs) is reached

func (*Chunker) Flush

func (chunker *Chunker) Flush()

Write the chunk to DDFS and reset the Output stream

func (*Chunker) WriteChunk

func (chunker *Chunker) WriteChunk(blob string, data []byte)

Write the Chunk to DDFS

type DDFSClient

type DDFSClient struct {
	Master string
	Port   string
	Url    string
	// contains filtered or unexported fields
}

func NewDDFSClient

func NewDDFSClient(master string, port string, timeout time.Duration) *DDFSClient

Returns a new DDFSClient struct for interaction with DDFS

func (*DDFSClient) Chunk

func (ddfs *DDFSClient) Chunk(tag string, urls []string, size int, conf *TagConfig) ([][]string, error)

Chunk items to a tag, returns the urls the inputs were tagged to

func (*DDFSClient) Communicate

func (ddfs *DDFSClient) Communicate(method string, url string, data []byte) ([]byte, error)

Wrapper for communication with disco

func (*DDFSClient) DelTagAttr

func (ddfs *DDFSClient) DelTagAttr(tag string, attr string) error

Delete a single tag attribute

func (*DDFSClient) Delete

func (ddfs *DDFSClient) Delete(tag string) error

Delete a tag from DDFS

func (*DDFSClient) Exists

func (ddfs *DDFSClient) Exists(tag string) bool

Check if tag exists

func (*DDFSClient) GetBlobs

func (ddfs *DDFSClient) GetBlobs(tag string) ([][]string, error)

Get blob locations for the tag

func (*DDFSClient) GetTagAttr

func (ddfs *DDFSClient) GetTagAttr(tag string, attr string) (string, error)

Get a single named attribute

func (*DDFSClient) GetTagAttrs

func (ddfs *DDFSClient) GetTagAttrs(tag string) (map[string]string, error)

Get all the tag attributes

func (*DDFSClient) List

func (ddfs *DDFSClient) List(prefix string) ([]string, error)

List DDFS tags, can pass in a prefix that the tags start with

func (*DDFSClient) Push

func (ddfs *DDFSClient) Push(tag string, urls []string, conf *TagConfig) (map[string][][]string, error)

Push raw files as blobs to DDFS

func (*DDFSClient) SetTagAttr

func (ddfs *DDFSClient) SetTagAttr(tag string, attr string, val interface{}) error

Set a single named attribute

func (*DDFSClient) Tag

func (ddfs *DDFSClient) Tag(tag string, urls [][]string, conf *TagConfig) error

Tag urls of uploaded blobs to a tag. Passing in nil for conf will create a default config

type DDFSTag

type DDFSTag struct {
	Version      int               `json:"version"`
	ID           string            `json:"id"`
	LastModified string            `json:"last-modified"`
	Urls         [][]string        `json:"urls"`
	UserData     map[string]string `json:"user-data"`
}

Struct for tag information unmarshaling

func GetTag

func GetTag(ddfs *DDFSClient, tag string) (*DDFSTag, error)

Get all the tag information

type DiscoOutputStream

type DiscoOutputStream struct {
	Hunk          *bytes.Buffer
	Output        *bytes.Buffer
	Stream        *bufio.Scanner
	PickleBuf     *bytes.Buffer
	BinaryBuf     *bytes.Buffer
	CompressBuf   *bytes.Buffer
	HunkSize      int
	MaxRecordSize int
	MinHunkSize   int
}

func NewOutputStream

func NewOutputStream(stream *bufio.Scanner) *DiscoOutputStream

Returns a new output stream, used for chunking data

func (*DiscoOutputStream) Append

func (output *DiscoOutputStream) Append(rec string)

Append a line to the output stream

func (*DiscoOutputStream) Flush

func (output *DiscoOutputStream) Flush()

Assemble the hunk(s) to be a chunk

func (*DiscoOutputStream) Size

func (output *DiscoOutputStream) Size() int

Get the size of the output stream buffer

type TagConfig

type TagConfig struct {
	Delayed string
	Update  string
}

Struct for pushing and tagging operations

func NewTagConfig

func NewTagConfig(d bool, u bool) *TagConfig

Get a new tag config for use with pushing

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL