README
¶
filestorage
- upload/store/download files, and clean files that are not linked to any objects.
Features
- Sync files to multiple machines using scp command.
- Download files using Nginx "X-Accel-Redirect" or sent file directly in response body.
- Clean files that are not linked to any object.
Documentation
¶
Overview ¶
Package filestorage upload/store/download files, and clean files that are not linked to any objects.
Index ¶
- func CheckHash(hashes ...string) error
- func Download(req *http.Request, resp http.ResponseWriter) error
- func FileHash(str string) (string, error)
- func FileHashes(strs []string) ([]string, error)
- func ImgSrcToFileHash(html string) (result string, hashes []string)
- func IsFileNotExists(err error) bool
- func IsHash(s string) bool
- func IsNotLinked(err error) bool
- func ReplaceImgSrc(html string, fn func(src string) string) string
- func TryFileHash(str string) (string, error)
- func UploadImages(req *http.Request, lang string) ([]string, error)
- func UploadWithMaxSize(req *http.Request, lang string, maxSize int64) ([]string, error)
- type Bucket
- func (b *Bucket) CheckFile(db DB, files ...string) error
- func (b *Bucket) Download(db DB, resp http.ResponseWriter, file string, object string) error
- func (b *Bucket) DownloadURL(linkObject interface{}, fileHash string, hosts ...string) string
- func (b *Bucket) DownloadURLs(linkObject interface{}, fileHashes []string, hosts ...string) []string
- func (b *Bucket) EnsureLinked(db DB, object, file string) error
- func (b *Bucket) FileDir(hash string) string
- func (b *Bucket) FilePath(hash string) string
- func (b *Bucket) FilesOf(db DB, object string) ([]string, error)
- func (b *Bucket) ImgSrcToDownloadURL(linkObject interface{}, html string) string
- func (b *Bucket) Init(db DB) error
- func (b *Bucket) Link(db DB, object string, files ...string) error
- func (b *Bucket) LinkOnly(db DB, object string, files ...string) error
- func (b *Bucket) Linked(db DB, object, file string) (bool, error)
- func (b *Bucket) Save(db DB, fileCheck func(string, int64) error, object string, files ...File) (fileHashes []string, err error)
- func (b *Bucket) SaveFiles(db DB, fileCheck func(string, int64) error, object string, paths ...string) (fileHashes []string, err error)
- func (b *Bucket) StartClean(cleanInterval, cleanAfter time.Duration, logger Logger)
- func (b *Bucket) Unlink(db DB, object string, files ...string) error
- func (b *Bucket) UnlinkAllOf(db DB, object string) error
- func (b *Bucket) Upload(db DB, fileCheck func(string, int64) error, object string, ...) ([]string, error)
- func (b *Bucket) UploadDefault(req *http.Request, lang string, maxSize int64) ([]string, error)
- type DB
- type File
- type LinkObject
- type Logger
Examples ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func CheckHash ¶ added in v0.0.6
CheckHash checks if hashes is in file hash format(43 urlsafe base64 characters).
func Download ¶ added in v0.2.8
func Download(req *http.Request, resp http.ResponseWriter) error
Download file according to the requested bucket, file, link object
func FileHashes ¶ added in v0.0.7
FileHashes returns file hashes from urls or file hashes
func ImgSrcToFileHash ¶ added in v0.2.8
func IsFileNotExists ¶ added in v0.2.8
func IsHash ¶ added in v0.0.6
IsHash returns if string s is in file hash format(43 urlsafe base64 characters).
func IsNotLinked ¶
IsNotLinked check if an error is the not linked Error.
func ReplaceImgSrc ¶ added in v0.2.8
Example ¶
var html = `<img> <img/>
<img src="1">
<img src="2"/>
<imG src="3" />
<IMG Src="4" / >
<img a src="5" b>
<img a src="x"b>
<img +src="x">
<img+ src="x">
`
fmt.Println(ReplaceImgSrc(html, func(src string) string { return src + "~" }))
Output: <img> <img/> <img src="1~"> <img src="2~"/> <imG src="3~" /> <IMG Src="4~" / > <img a src="5~" b> <img a src="x"b> <img +src="x"> <img+ src="x">
func TryFileHash ¶ added in v0.2.8
TryFileHash try to returns file hash from a url or file hash.
func UploadImages ¶ added in v0.2.8
Types ¶
type Bucket ¶ added in v0.2.8
type Bucket struct {
Name string
Machines []string
Dir string
DirDepth uint8
ScpUser string
DownloadURLPrefix string
// Path prefix for "X-Accel-Redirect" response header when downloading.
// If this path prefix is present, only file path is sent in the "X-Accel-Redirect" header,
// and nginx is responsible for file downloading for a better performance.
// Otherwise, file is sent directly in the response body.
RedirectPathPrefix string
DB DB
FilesTable string
LinksTable string
// contains filtered or unexported fields
}
Bucket store file on disk and infomation in database tables.
func (*Bucket) Download ¶ added in v0.2.8
Download file, if object is not empty, the file must be linked to it, otherwise an error is returned. If RedirectPathPrefix is not empty, an location like following is required in nginx virtual server config.
location /fs/ {
internal;
alias /data/file-storage;
}
The location prefix and alias path should be set according to RedirectPathPrefix and Dir.
func (*Bucket) DownloadURL ¶ added in v0.2.8
DownloadURL make the url for file download hosts support specified download url host
func (*Bucket) DownloadURLs ¶ added in v0.2.8
func (b *Bucket) DownloadURLs(linkObject interface{}, fileHashes []string, hosts ...string) []string
DownloadURLs make the urls for files download hosts support specified download url host
func (*Bucket) EnsureLinked ¶ added in v0.2.8
EnsureLinked ensure file is linked to object.
func (*Bucket) ImgSrcToDownloadURL ¶ added in v0.2.8
func (*Bucket) Init ¶ added in v0.2.8
Init validate storage fields and create tables if not created.
func (*Bucket) Link ¶ added in v0.2.8
Link files to object.
Example ¶
testBucket.insertFileRecords(nil, []fileRecord{
{Hash: "TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF1"},
{Hash: "TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF2"},
{Hash: "TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF3"},
})
fmt.Println(testBucket.UnlinkAllOf(nil, "object"))
fmt.Println(testBucket.Link(nil, "object", testFile1, testFile2, testFile3))
if files, err := testBucket.FilesOf(nil, "object"); err != nil {
fmt.Println(err)
} else {
for _, v := range files {
fmt.Println(v)
}
}
Output: <nil> <nil> TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF1 TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF2 TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF3
func (*Bucket) LinkOnly ¶ added in v0.2.8
LinkOnly make sure these files and only these files are linked to object.
Example ¶
testBucket.insertFileRecords(nil, []fileRecord{
{Hash: "TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF3"},
{Hash: "TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF4"},
})
fmt.Println(testBucket.LinkOnly(nil, "object", testFile3, testFile4))
if files, err := testBucket.FilesOf(nil, "object"); err != nil {
fmt.Println(err)
} else {
for _, v := range files {
fmt.Println(v)
}
}
fmt.Println(testBucket.EnsureLinked(nil, "object", testFile3))
fmt.Println(testBucket.Unlink(nil, "object", testFile3, testFile4))
fmt.Println(testBucket.Linked(nil, "object", testFile3))
Output: <nil> TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF3 TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF4 <nil> <nil> false <nil>
func (*Bucket) Save ¶ added in v0.2.8
func (b *Bucket) Save( db DB, fileCheck func(string, int64) error, object string, files ...File, ) (fileHashes []string, err error)
Save save files into bucket.
func (*Bucket) SaveFiles ¶ added in v0.2.8
func (b *Bucket) SaveFiles( db DB, fileCheck func(string, int64) error, object string, paths ...string, ) (fileHashes []string, err error)
SaveFiles save files specified by paths into bucket.
Example ¶
files, err := testBucket.SaveFiles(nil, nil, "linkObject", "LICENSE")
if err != nil {
panic(err)
}
fmt.Println(files)
Output: [y_-6r_79lAd8cpzmKuK-W0u7_ZulcxaquXsi308-mqk]
func (*Bucket) StartClean ¶ added in v0.2.8
Example ¶
testUpload() testBucket.StartClean(time.Second, time.Nanosecond, logger.New(os.Stdout)) time.Sleep(time.Second)
Output:
func (*Bucket) UnlinkAllOf ¶ added in v0.2.8
UnlinkAllOf unlink all linked files from an object.
func (*Bucket) Upload ¶ added in v0.2.8
func (b *Bucket) Upload( db DB, fileCheck func(string, int64) error, object string, fileHeaders ...*multipart.FileHeader, ) ([]string, error)
Upload files, if object is not empty, the files are linked to it.
Example ¶
fmt.Println(testUpload())
Output: [TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF0 HEbi8PV2cRPf8QeB8lesh6gWPAmiAda7xGarbjAv8v4]
type DB ¶
type DB interface {
QueryRow(query string, args ...interface{}) *sql.Row
Query(query string, args ...interface{}) (*sql.Rows, error)
Exec(query string, args ...interface{}) (sql.Result, error)
}
DB represents *sql.DB or *sql.Tx
type File ¶ added in v0.0.6
type File struct {
IO io.ReadSeeker // bytes.Reader and strings.Reader implemented this interface.
Size int64
}
File reprents the file to store.
type LinkObject ¶ added in v0.0.3
type LinkObject struct {
Table string // Table name, required.
ID int64 // Primary key or unique key, required.
Field string // If the table has multiple file storage fields, Field is required, Otherwise Field can be omitted.
}
LinkObject is a structured reference implementaion for link object string.
func (LinkObject) MarshalJSON ¶ added in v0.0.4
func (o LinkObject) MarshalJSON() ([]byte, error)
MarshalJSON implements json.Marshaler
Example ¶
object := LinkObject{Table: "x", ID: 1}
b, err := json.Marshal(object)
fmt.Println(string(b), err)
Output: "x|1" <nil>
func (LinkObject) String ¶ added in v0.0.3
func (o LinkObject) String() string
func (*LinkObject) UnmarshalJSON ¶ added in v0.0.3
func (o *LinkObject) UnmarshalJSON(b []byte) error
UnmarshalJSON implements json.Unmarshaler
Example ¶
object := LinkObject{}
err := json.Unmarshal([]byte(`"a.x|1|b"`), &object)
fmt.Println(err)
fmt.Printf("%#v\n", object)
Output: <nil> filestorage.LinkObject{Table:"a.x", ID:1, Field:"b"}
Source Files
Click to show internal directories.
Click to hide internal directories.