gowfs

package module
v0.0.0-...-875761d Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 3, 2018 License: MIT Imports: 14 Imported by: 0

README

gowfs

A Go client binding for Hadoop HDFS using WebHDFS.

Documentation

Index

Constants

View Source
const (
	OP_OPEN                  = "OPEN"
	OP_CREATE                = "CREATE"
	OP_APPEND                = "APPEND"
	OP_CONCAT                = "CONCAT"
	OP_RENAME                = "RENAME"
	OP_DELETE                = "DELETE"
	OP_SETPERMISSION         = "SETPERMISSION"
	OP_SETOWNER              = "SETOWNER"
	OP_SETREPLICATION        = "SETREPLICATION"
	OP_SETTIMES              = "SETTIMES"
	OP_MKDIRS                = "MKDIRS"
	OP_CREATESYMLINK         = "CREATESYMLINK"
	OP_LISTSTATUS            = "LISTSTATUS"
	OP_GETFILESTATUS         = "GETFILESTATUS"
	OP_GETCONTENTSUMMARY     = "GETCONTENTSUMMARY"
	OP_GETFILECHECKSUM       = "GETFILECHECKSUM"
	OP_GETDELEGATIONTOKEN    = "GETDELEGATIONTOKEN"
	OP_GETDELEGATIONTOKENS   = "GETDELEGATIONTOKENS"
	OP_RENEWDELEGATIONTOKEN  = "RENEWDELEGATIONTOKEN"
	OP_CANCELDELEGATIONTOKEN = "CANCELDELEGATIONTOKEN"
)

Operations of WebHDFS

View Source
const (
	JMX_URL = "http://%s/jmx?get=Hadoop:service=NameNode,name=FSNamesystem::tag.HAState"
)

Variables

This section is empty.

Functions

This section is empty.

Types

type By

type By func(f1, f2 *FileStatus) bool

By is the type of a "less" function that defines the ordering of its FileStatus arguments.

func (By) Sort

func (by By) Sort(fs []FileStatus)

Sort is a method on the function type By, that sorts the argument slice according to the function.

type Client

type Client struct {
	Token string
	// contains filtered or unexported fields
}

func NewClient

func NewClient(nameNodes []string, opts ...ClientOption) (*Client, error)

NewClient returns a client wrapping webhdfs APIs. If your HDFS cluster support HA, please at least set two name nodes, and the Client use jmx metrics to get the available one.

func (*Client) GetDelegationToken

func (cl *Client) GetDelegationToken() (token string, err error)

func (*Client) ListStatus

func (cl *Client) ListStatus(path string) ([]FileStatus, error)

func (*Client) Open

func (cl *Client) Open(path string, offset, length uint64, bufferSize uint32) (io.ReadCloser, error)

func (*Client) RenewDelegationToken

func (cl *Client) RenewDelegationToken() (expiration int64, err error)

type ClientOption

type ClientOption func(*clientOptions)

func WithKerberos

func WithKerberos(cl *gokrb5cl.Client, autoGenToken bool) ClientOption

func WithTransport

func WithTransport(t *http.Transport) ClientOption

func WithUser

func WithUser(user string) ClientOption

type ContentSummary

type ContentSummary struct {
	DirectoryCount int64
	FileCount      int64
	Length         int64
	Quota          int64
	SpaceConsumed  int64
	SpaceQuota     int64
}

type FileChecksum

type FileChecksum struct {
	Algorithm string
	Bytes     string
	Length    int64
}

type FileStatus

type FileStatus struct {
	AccesTime        int64
	BlockSize        int64
	Group            string
	Length           int64
	ModificationTime int64
	Owner            string
	PathSuffix       string
	Permission       string
	Replication      int64
	Type             string
}

type FileStatusSorter

type FileStatusSorter struct {
	// contains filtered or unexported fields
}

func (*FileStatusSorter) Len

func (s *FileStatusSorter) Len() int

Len is part of sort.Interface.

func (*FileStatusSorter) Less

func (s *FileStatusSorter) Less(i, j int) bool

Less is part of sort.Interface. It is implemented by calling the "by" closure in the sorter.

func (*FileStatusSorter) Swap

func (s *FileStatusSorter) Swap(i, j int)

Swap is part of sort.Interface.

type FileStatuses

type FileStatuses struct {
	FileStatus []FileStatus
}

type OperationParams

type OperationParams struct {
	Addr       string
	Op         string
	Path       string
	UserName   string
	Delegation string
	Offset     uint64
	Length     uint64
	BufferSize uint32
}

type RemoteException

type RemoteException struct {
	Exception     string
	JavaClassName string
	Message       string
}

func (*RemoteException) Error

func (re *RemoteException) Error() string

Implementation of error type. Returns string representation of RemoteException

type Response

type Response struct {
	Boolean         bool             `json:"boolean"`
	Long            int64            `json:"long"`
	FileStatus      *FileStatus      `json:",omitempty"`
	FileStatuses    *FileStatuses    `json:",omitempty"`
	FileChecksum    *FileChecksum    `json:",omitempty"`
	ContentSummary  *ContentSummary  `json:",omitempty"`
	Token           *Token           `json:",omitempty"`
	Tokens          *Tokens          `json:",omitempty"`
	RemoteException *RemoteException `json:",omitempty"`
}

type Token

type Token struct {
	UrlString string
}

type Tokens

type Tokens struct {
	Token []Token
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL