Documentation
¶
Index ¶
- type FileInfo
- type FileSystem
- type FileSystemType
- type LocalFileSystem
- func (l *LocalFileSystem) Delete(filePath string) error
- func (l *LocalFileSystem) Init() error
- func (l *LocalFileSystem) Join(elem ...string) string
- func (l *LocalFileSystem) ListFiles(pathGlob string) ([]FileInfo, error)
- func (l *LocalFileSystem) OpenReader(filePath string, startAt int64) (io.ReadCloser, error)
- func (l *LocalFileSystem) OpenWriter(filePath string) (io.WriteCloser, error)
- func (l *LocalFileSystem) Stat(filePath string) (FileInfo, error)
- type S3FileSystem
- func (s *S3FileSystem) Delete(filePath string) error
- func (s *S3FileSystem) Init() error
- func (s *S3FileSystem) Join(elem ...string) string
- func (s *S3FileSystem) ListFiles(pathGlob string) ([]FileInfo, error)
- func (s *S3FileSystem) OpenReader(filePath string, startAt int64) (io.ReadCloser, error)
- func (s *S3FileSystem) OpenWriter(filePath string) (io.WriteCloser, error)
- func (s *S3FileSystem) Stat(filePath string) (FileInfo, error)
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type FileSystem ¶
type FileSystem interface {
ListFiles(pathGlob string) ([]FileInfo, error)
Stat(filePath string) (FileInfo, error)
OpenReader(filePath string, startAt int64) (io.ReadCloser, error)
OpenWriter(filePath string) (io.WriteCloser, error)
Delete(filePath string) error
Join(elem ...string) string
Init() error
}
FileSystem provides the file backend for MapReduce jobs. Input data is read from a file system. Intermediate and output data is written to a file system. This is abstracted to allow remote filesystems like S3 to be supported.
func InferFilesystem ¶
func InferFilesystem(location string) FileSystem
InferFilesystem initializes a filesystem by inferring its type from a file address. For example, locations starting with "s3://" will resolve to an S3 filesystem.
func InitFilesystem ¶
func InitFilesystem(fsType FileSystemType) FileSystem
InitFilesystem intializes a filesystem of the given type
type FileSystemType ¶
type FileSystemType int
FileSystemType is an identifier for supported FileSystems
const ( Local FileSystemType = iota S3 )
Identifiers for supported FileSystemTypes
type LocalFileSystem ¶
type LocalFileSystem struct{}
LocalFileSystem wraps "os" to provide access to the local filesystem.
func (*LocalFileSystem) Delete ¶
func (l *LocalFileSystem) Delete(filePath string) error
Delete deletes the file at filePath.
func (*LocalFileSystem) Init ¶
func (l *LocalFileSystem) Init() error
Init initializes the filesystem.
func (*LocalFileSystem) Join ¶
func (l *LocalFileSystem) Join(elem ...string) string
Join joins file path elements
func (*LocalFileSystem) ListFiles ¶
func (l *LocalFileSystem) ListFiles(pathGlob string) ([]FileInfo, error)
ListFiles lists files that match pathGlob.
func (*LocalFileSystem) OpenReader ¶
func (l *LocalFileSystem) OpenReader(filePath string, startAt int64) (io.ReadCloser, error)
OpenReader opens a reader to the file at filePath. The reader is initially seeked to "startAt" bytes into the file.
func (*LocalFileSystem) OpenWriter ¶
func (l *LocalFileSystem) OpenWriter(filePath string) (io.WriteCloser, error)
OpenWriter opens a writer to the file at filePath.
type S3FileSystem ¶
type S3FileSystem struct {
// contains filtered or unexported fields
}
S3FileSystem abstracts AWS S3 as a filesystem
func (*S3FileSystem) Delete ¶
func (s *S3FileSystem) Delete(filePath string) error
Delete deletes the file at filePath.
func (*S3FileSystem) Join ¶
func (s *S3FileSystem) Join(elem ...string) string
Join joins file path elements
func (*S3FileSystem) ListFiles ¶
func (s *S3FileSystem) ListFiles(pathGlob string) ([]FileInfo, error)
ListFiles lists files that match pathGlob.
func (*S3FileSystem) OpenReader ¶
func (s *S3FileSystem) OpenReader(filePath string, startAt int64) (io.ReadCloser, error)
OpenReader opens a reader to the file at filePath. The reader is initially seeked to "startAt" bytes into the file.
func (*S3FileSystem) OpenWriter ¶
func (s *S3FileSystem) OpenWriter(filePath string) (io.WriteCloser, error)
OpenWriter opens a writer to the file at filePath.
Source Files