hadoop

package module

v0.0.0-...-00d3ff7 Latest Latest Go to latest Published: Jun 23, 2016 License: BSD-3-Clause Imports: 8 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/librato/go-hadoop-io

Links

Open Source Insights

README ¶

go-hadoop-io

Hadoop SequenceFile reader for Go. This project is still in early stage and may be subject to change.

Installation

Ubuntu

sudo apt-get install libbz2-dev liblz4-dev
go get github.com/eiiches/go-hadoop-io

CentOS

sudo yum install bzip2-devel
git clone https://github.com/Cyan4973/lz4
cd lz4
make
sudo make install
sudo ldconfig
go get github.com/eiiches/go-hadoop-io

Usage

Some examples are located in examples.

License

This project is licensed under the terms of the 3-clause BSD license. See LICENSE.txt for the full license text.

Documentation ¶

Index ¶

Constants
Variables
func DecodeVIntSize(value byte) int
func IsNegativeVInt(fst byte) bool
func ReadBoolean(r io.Reader) (bool, error)
func ReadBuffer(r io.Reader) ([]byte, error)
func ReadByte(r io.Reader) (byte, error)
func ReadInt(r io.Reader) (int32, error)
func ReadVLong(r io.Reader) (int64, error)
func WriteBoolean(w io.Writer, v bool) error
func WriteBuffer(w io.Writer, buf []byte) (int, error)
func WriteByte(w io.Writer, b byte) error
func WriteInt(w io.Writer, v int32) error
func WriteVLong(w io.Writer, i int64) (int, error)
type BytesWritable
- func (self *BytesWritable) Read(r io.Reader) error
- func (self *BytesWritable) Write(w io.Writer) (int, error)
type Bzip2Codec
- func (c *Bzip2Codec) Compress(dst, src []byte) ([]byte, error)
- func (c *Bzip2Codec) Uncompress(dst, src []byte) ([]byte, error)
type Codec
type IntWritable
- func (self *IntWritable) Read(r io.Reader) error
- func (self *IntWritable) Write(w io.Writer) (int, error)
type LongWritable
- func (self *LongWritable) Read(r io.Reader) error
- func (self *LongWritable) Write(w io.Writer) (int, error)
type Lz4Codec
- func (c *Lz4Codec) Compress(dst, src []byte) ([]byte, error)
- func (c *Lz4Codec) Uncompress(dst, src []byte) ([]byte, error)
type SequenceFileReader
- func NewSequenceFileReader(r io.Reader) (*SequenceFileReader, error)
- func (self *SequenceFileReader) Close() error
- func (self *SequenceFileReader) Read(key Writable, value Writable) error
type SequenceFileWriter
- func NewSequenceFileWriter(w io.Writer, opts *SequenceFileWriterOpts) (*SequenceFileWriter, error)
- func (self *SequenceFileWriter) Close() error
- func (self *SequenceFileWriter) Write(key Writable, value Writable) error
type SequenceFileWriterOpts
type SnappyCodec
type TextWritable
- func (self *TextWritable) Read(r io.Reader) error
- func (self *TextWritable) Write(w io.Writer) (int, error)
type Writable
type ZlibCodec
- func (c *ZlibCodec) Compress(dst, src []byte) ([]byte, error)
- func (c *ZlibCodec) Uncompress(dst, src []byte) ([]byte, error)

Constants ¶

View Source

const (
	VERSION_BLOCK_COMPRESS  = 4
	VERSION_CUSTOM_COMPRESS = 5
	VERSION_WITH_METADATA   = 6
)

View Source

const BLOCK_SIZE_MIN = 1 << 20 // Corresponds roughly to io.seqfile.compress.blocksize

View Source

const SYNC_HASH_SIZE = 16

Variables ¶

View Source

var (
	Codecs map[string]Codec = map[string]Codec{
		"org.apache.hadoop.io.compress.DefaultCodec": &ZlibCodec{},
		"org.apache.hadoop.io.compress.Lz4Codec":     &Lz4Codec{},
		"org.apache.hadoop.io.compress.BZip2Codec":   &Bzip2Codec{},
	}
)

View Source

var SEQ_MAGIC = []byte("SEQ")

Functions ¶

func DecodeVIntSize ¶

func DecodeVIntSize(value byte) int

func IsNegativeVInt ¶

func IsNegativeVInt(fst byte) bool

func ReadBoolean ¶

func ReadBoolean(r io.Reader) (bool, error)

func ReadBuffer ¶

func ReadBuffer(r io.Reader) ([]byte, error)

func ReadByte ¶

func ReadByte(r io.Reader) (byte, error)

func ReadInt ¶

func ReadInt(r io.Reader) (int32, error)

func ReadVLong ¶

func ReadVLong(r io.Reader) (int64, error)

func WriteBoolean ¶

func WriteBoolean(w io.Writer, v bool) error

func WriteBuffer ¶

func WriteBuffer(w io.Writer, buf []byte) (int, error)

func WriteByte ¶

func WriteByte(w io.Writer, b byte) error

func WriteInt ¶

func WriteInt(w io.Writer, v int32) error

func WriteVLong ¶

func WriteVLong(w io.Writer, i int64) (int, error)

Ported from https://hadoop.apache.org/docs/r2.6.2/api/src-html/org/apache/hadoop/io/WritableUtils.html#line.271

Types ¶

type BytesWritable ¶

type BytesWritable struct {
	Buf []byte
}

func (*BytesWritable) Read ¶

func (self *BytesWritable) Read(r io.Reader) error

func (*BytesWritable) Write ¶

func (self *BytesWritable) Write(w io.Writer) (int, error)

type Bzip2Codec ¶

type Bzip2Codec struct {
}

func (*Bzip2Codec) Compress ¶

func (c *Bzip2Codec) Compress(dst, src []byte) ([]byte, error)

func (*Bzip2Codec) Uncompress ¶

func (c *Bzip2Codec) Uncompress(dst, src []byte) ([]byte, error)

type Codec ¶

type Codec interface {
	Uncompress(dst, src []byte) ([]byte, error)
	Compress(dst, src []byte) ([]byte, error)
}

type IntWritable ¶

type IntWritable int32

func (*IntWritable) Read ¶

func (self *IntWritable) Read(r io.Reader) error

func (*IntWritable) Write ¶

func (self *IntWritable) Write(w io.Writer) (int, error)

type LongWritable ¶

type LongWritable int64

func (*LongWritable) Read ¶

func (self *LongWritable) Read(r io.Reader) error

func (*LongWritable) Write ¶

func (self *LongWritable) Write(w io.Writer) (int, error)

type Lz4Codec ¶

type Lz4Codec struct {
}

func (*Lz4Codec) Compress ¶

func (c *Lz4Codec) Compress(dst, src []byte) ([]byte, error)

func (*Lz4Codec) Uncompress ¶

func (c *Lz4Codec) Uncompress(dst, src []byte) ([]byte, error)

type SequenceFileReader ¶

type SequenceFileReader struct {
	// contains filtered or unexported fields
}

func NewSequenceFileReader ¶

func NewSequenceFileReader(r io.Reader) (*SequenceFileReader, error)

func (*SequenceFileReader) Close ¶

func (self *SequenceFileReader) Close() error

func (*SequenceFileReader) Read ¶

func (self *SequenceFileReader) Read(key Writable, value Writable) error

type SequenceFileWriter ¶

type SequenceFileWriter struct {
	// contains filtered or unexported fields
}

func NewSequenceFileWriter ¶

func NewSequenceFileWriter(w io.Writer, opts *SequenceFileWriterOpts) (*SequenceFileWriter, error)

func (*SequenceFileWriter) Close ¶

func (self *SequenceFileWriter) Close() error

func (*SequenceFileWriter) Write ¶

func (self *SequenceFileWriter) Write(key Writable, value Writable) error

type SequenceFileWriterOpts ¶

type SequenceFileWriterOpts struct {
	KeyClassName     string
	ValueClassName   string
	CompressionCodec string
}

type SnappyCodec ¶

type SnappyCodec struct {
}

type TextWritable ¶

type TextWritable struct {
	Buf []byte
}

func (*TextWritable) Read ¶

func (self *TextWritable) Read(r io.Reader) error

func (*TextWritable) Write ¶

func (self *TextWritable) Write(w io.Writer) (int, error)

type Writable ¶

type Writable interface {
	Write(w io.Writer) (int, error)
	Read(r io.Reader) error
}

type ZlibCodec ¶

type ZlibCodec struct{}

func (*ZlibCodec) Compress ¶

func (c *ZlibCodec) Compress(dst, src []byte) ([]byte, error)

func (*ZlibCodec) Uncompress ¶

func (c *ZlibCodec) Uncompress(dst, src []byte) ([]byte, error)

Source Files ¶

View all Source files

Directories ¶

Path	Synopsis
examples

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL