hdfs

type ContentSummary ¶

type ContentSummary struct {
	DirectoryCount int64
	FileCount      int64
	Length         int64
	Quota          int64
	SpaceConsumed  int64
	SpaceQuota     int64
}

Type for HDFS FileSystem content summary (FileSystem.getContentSummary())
See http://hadoop.apache.org/docs/r2.2.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#ContentSummary_JSON_Schema

Example:

{
  "ContentSummary":
  {
    "directoryCount": 2,
    "fileCount"     : 1,
    "length"        : 24930,
    "quota"         : -1,
    "spaceConsumed" : 24930,
    "spaceQuota"    : -1
  }
}

type FileChecksum ¶

type FileChecksum struct {
	Algorithm string
	Bytes     string
	Length    int64
}

Type for HDFS FileSystem.getFileChecksum()
See http://hadoop.apache.org/docs/r2.2.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#FileChecksum_JSON_Schema

Example:

{
  "FileChecksum":
  {
    "algorithm": "MD5-of-1MD5-of-512CRC32",
    "bytes"    : "eadb10de24aa315748930df6e185c0d ...",
    "length"   : 28
  }
}

type FileStatus ¶

type FileStatus struct {
	AccesTime        int64
	BlockSize        int64
	Group            string
	Length           int64
	ModificationTime int64
	Owner            string
	PathSuffix       string
	Permission       string
	Replication      int64
	Type             string
}

Represents HDFS FileStatus (FileSystem.getStatus()) See http://hadoop.apache.org/docs/r2.2.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#FileStatus_JSON_Schema

Example:

{
  "FileStatus":
  {
    "accessTime"      : 0, 				// integer
    "blockSize"       : 0, 				// integer
    "group"           : "grp",			// string
    "length"          : 0,             	// integer - zero for directories
    "modificationTime": 1320173277227,	// integer
    "owner"           : "webuser",		// string
    "pathSuffix"      : "",				// string
    "permission"      : "777",			// string
    "replication"     : 0,				// integer
    "type"            : "DIRECTORY"    	// string - enum {FILE, DIRECTORY, SYMLINK}
  }
}

type FileStatuses ¶

type FileStatuses struct {
	FileStatus []FileStatus
}

Container type for multiple FileStatus for directory, etc (see HDFS FileSystem.listStatus()) NOTE: the confusing naming and Plurality is to match WebHDFS schema.

type Hdfs ¶

type Hdfs struct {
	Config *HdfsConfig
	// contains filtered or unexported fields
}

func NewHdfs(config *HdfsConfig) (*Hdfs, error)

func (h *Hdfs) Append(localfile string, destination string) error

func (h *Hdfs) Delete(path string, recursive bool) error

func (h *Hdfs) Deletes(paths []string, recursive bool) map[string]error

func (h *Hdfs) Get(path string) ([]byte, error)

func (h *Hdfs) GetToLocal(path string, destination string, permission string) error

func (h *Hdfs) List(path string) (*HdfsData, error)

func (h *Hdfs) MakeDir(path string, permission string) error

func (h *Hdfs) MakeDirs(paths []string, permission string) map[string]error

func (h *Hdfs) Put(localfile string, destination string, permission string, parms map[string]string) error

func (h *Hdfs) PutDir(dirname string, destination string) (error, map[string]error)

func (h *Hdfs) Puts(paths []string, destinationFolder string, permission string, parms map[string]string) map[string]error

func (h *Hdfs) Rename(path string, destination string) error

func (h *Hdfs) SetOwner(path string, owner string, group string) error

func (h *Hdfs) SetPermission(path string, user string) error

type HdfsConfig ¶

type HdfsConfig struct {
	Host     string
	UserId   string
	Password string
	Token    string
	Method   string
	TimeOut  time.Duration
	PoolSize int
}

func NewHdfsConfig(host, userid string) *HdfsConfig

type HdfsData ¶

type HdfsData struct {
	Boolean         bool
	FileStatus      FileStatus
	FileStatuses    FileStatuses
	FileChecksum    FileChecksum
	ContentSummary  ContentSummary
	Token           Token
	Tokens          Tokens
	Long            int64
	RemoteException RemoteException
}

Root level struct for data JSON data from WebHDFS.

type Path ¶

type Path struct {
	Name       string  // Relative path representation (/root/leaf)
	RefererUrl url.URL // URL related to path (http://server:port/root/leaf)
}

Represents a remote webHDFS path

type RemoteException ¶

type RemoteException struct {
	Exception     string
	JavaClassName string
	Message       string
}

Example:

{
  "RemoteException":
  {
    "exception"    : "FileNotFoundException",
    "javaClassName": "java.io.FileNotFoundException",
    "message"      : "File does not exist: /foo/a.patch"
  }
}

func (re RemoteException) Error() string

type Token ¶

type Token struct {
	UrlString string
}

Example:

{
  "Token":
  {
    "urlString": "JQAIaG9y..."
  }
}

type Tokens ¶

type Tokens struct {
	Token []Token
}

Container type for Token