rmsgo

package module
v0.1.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 31, 2023 License: Unlicense Imports: 21 Imported by: 0

README

Go remoteStorage Library

An implementation of the remoteStorage protocol written in Go.

go get -u github.com/cvanloo/rmsgo

Example Usage

package main

import (
    "os"
    "github.com/cvanloo/rmsgo"
)

const (
    PersistFile = "/var/rms/persist"
    RemoteRoot  = "/storage/"
    StorageRoot = "/var/rms/storage/"
)

func main() {
    opts, err := rmsgo.Configure(RemoteRoot, StorageRoot)
    if err != nil {
        log.Fatal(err)
    }
    opts.UseErrorHandler(func(err error) {
        log.Fatalf("remote storage: unhandled error: %v", err)
    })
    opts.UseAuthentication(func(r *http.Request, bearer string) (rmsgo.User, bool) {
        // [!] TODO: Your authentication logic here...
        //       Return one of your own users.
        return rmsgo.ReadWriteUser{}, true
    })

    persistFile, err := os.Open(PersistFile)
    if err != nil {
        log.Fatal(err)
    }

    // Restore server state
    err = rmsgo.Load(persistFile)
    if err != nil {
        log.Fatal(err)
    }

    // Register remote storage endpoints to the http.DefaultServeMux
    rmsgo.Register(nil)
    http.ListenAndServe(":8080", nil) // [!] TODO: Use TLS

    // At shutdown: persist server state
    err = rmsgo.Persist(persistFile)
    if err != nil {
        log.Fatal(err)
    }
}

With Request Logging

func logger(next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        start := time.Now()
        lrw := rmsgo.NewLoggingResponseWriter(w)

        // [!] pass request on to remote storage server
        next.ServeHTTP(lrw, r)

        duration := time.Since(start)

        // - Mom! Can we have slog?
        // - No darling, we have slog at home.
        // slog at home:
        log.Printf("%v", map[string]any{
            "method":   r.Method,
            "uri":      r.RequestURI,
            "duration": duration,
            "status":   lrw.Status,
            "size":     lrw.Size,
        })
    })
}

func main() {
    opts, err := rmsgo.Configure(RemoteRoot, StorageRoot)
    if err != nil {
        log.Fatal(err)
    }

    // [!] Register custom middleware
    opts.UseMiddleware(logger)

    // [!] Other configuration...

    rmsgo.Register(nil)
    http.ListenAndServe(":8080", nil) // [!] TODO: Use TLS
}

All Configuration Options

  • [Required] Setup
    • remoteRoot: URL path below which the server is accessible. (e.g. "/storage/")
    • storageRoot: Location on server's file system to store remoteStorage documents. (e.g. "/var/rms/storage/")
  • [Recommended] UseAuthentication configure how requests are authenticated and control access permissions of users.
  • [Recommended] UseAllowedOrigins allow-list of hosts that may make requests to the server. Per default any host is allowed.
  • [Optional] UseAllowOrigin for more control, specify a function that decides based on the request if it is allowed or not. If this option is specified, UseAllowedOrigins has no effect.
  • [Not Recommended] AllowAnyReadWrite allow even unauthenticated requests to create, read, and delete any documents on the server. Has no effect if UseAuthentication is specified.
  • [Optional] UseErrorHandler to catch unhandled errors. Default behavior is to log.Printf the error.
  • [Optional] UseMiddleware to intercept requests before they are passed to the remote storage handler.

Register registers the remote storage handler to a ServeMux.

Documentation

Index

Examples

Constants

This section is empty.

Variables

View Source
var (
	ErrServerError         = errors.New("internal server error")
	ErrNotImplemented      = errors.New("not implemented")
	ErrNotModified         = errors.New("not modified")
	ErrUnauthorized        = errors.New("missing or invalid bearer token")
	ErrForbidden           = errors.New("insufficient scope")
	ErrNotFound            = errors.New("resource not found")
	ErrConflict            = errors.New("conflicting document/folder names")
	ErrPreconditionFailed  = errors.New("precondition failed")
	ErrTooLarge            = errors.New("request entity too large")
	ErrUriTooLong          = errors.New("request uri too long")
	ErrRangeNotSatisfiable = errors.New("request range not satisfiable")
	ErrTooManyRequests     = errors.New("too many requests")
	ErrMethodNotAllowed    = errors.New("method not allowed")
	ErrInsufficientStorage = errors.New("insufficient storage")
	ErrBadRequest          = errors.New("bad request")
)

Sentinel error values

View Source
var ErrNotExist = errors.New("no such document or folder")

StatusCodes maps errors to their respective HTTP status codes

Functions

func AddDocument

func AddDocument(rname, sname string, fsize int64, mime string) (*node, error)

AddDocument adds a new document to the storage tree and returns a reference to it. ETags of ancestors are invalidated. If the document name conflicts with any other document or folder an error of type ConflictPath is returned and the *node is set to nil.

func DeleteDocument

func DeleteDocument(w http.ResponseWriter, r *http.Request) error

func GetDocument

func GetDocument(w http.ResponseWriter, r *http.Request) error

func GetFolder

func GetFolder(w http.ResponseWriter, r *http.Request) error
Example
mockServer()

mux := http.NewServeMux()
Register(mux)
ts := httptest.NewServer(mux)
defer ts.Close()

// server url + remote root
remoteRoot := ts.URL + g.rroot

// GET the currently empty root folder
{
	r, err := http.Get(remoteRoot + "/")
	if err != nil {
		log.Fatal(err)
	}
	if r.StatusCode != http.StatusOK {
		log.Fatalf("%s %s: %s", r.Request.Method, r.Request.URL, r.Status)
	}
	bs, err := io.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("Root ETag: %s\n", r.Header.Get("ETag"))
	fmt.Print(string(bs))
	// Root ETag: 03d871638b18f0b459bf8fd12a58f1d8
	// {
	//   "@context": "http://remotestorage.io/spec/folder-description",
	//   "items": {}
	// }
}

// PUT a document
{
	req, err := http.NewRequest(http.MethodPut, remoteRoot+"/Documents/First.txt", bytes.NewReader([]byte("My first document.")))
	if err != nil {
		log.Fatal(err)
	}
	req.Header.Set("Content-Type", "funny/format") // mime type is auto-detected if not specified
	r, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	if r.StatusCode != http.StatusCreated {
		log.Fatalf("%s %s: %s", r.Request.Method, r.Request.URL, r.Status)
	}
	fmt.Printf("Created ETag: %s\n", r.Header.Get("ETag"))
	// Created ETag: f0d0f717619b09cc081bb0c11d9b9c6b
}

// GET the now NON-empty root folder
{
	r, err := http.Get(remoteRoot + "/")
	if err != nil {
		log.Fatal(err)
	}
	if r.StatusCode != http.StatusOK {
		log.Fatalf("%s %s: %s", r.Request.Method, r.Request.URL, r.Status)
	}
	bs, err := io.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("Root ETag: %s\n", r.Header.Get("ETag"))
	fmt.Print(string(bs))
	// Root ETag: ef528a27b48c1b187ef7116f7306358b
	// {
	//   "@context": "http://remotestorage.io/spec/folder-description",
	//   "items": {
	//     "Documents/": {
	//       "ETag": "cc4c6d3bbf39189be874992479b60e2a"
	//     }
	//   }
	// }
}

// GET the document's folder
{
	r, err := http.Get(remoteRoot + "/Documents/")
	if err != nil {
		log.Fatal(err)
	}
	if r.StatusCode != http.StatusOK {
		log.Fatalf("%s %s: %s", r.Request.Method, r.Request.URL, r.Status)
	}
	bs, err := io.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("Documents/ ETag: %s\n", r.Header.Get("ETag"))
	fmt.Print(string(bs))
	// Documents/ ETag: cc4c6d3bbf39189be874992479b60e2a
	// {
	//   "@context": "http://remotestorage.io/spec/folder-description",
	//   "items": {
	//     "First.txt": {
	//       "Content-Length": 18,
	//       "Content-Type": "funny/format",
	//       "ETag": "f0d0f717619b09cc081bb0c11d9b9c6b",
	//       "Last-Modified": "Mon, 01 Jan 0001 00:00:00 UTC"
	//     }
	//   }
	// }
}
Output:

Root ETag: 03d871638b18f0b459bf8fd12a58f1d8
{"@context":"http://remotestorage.io/spec/folder-description","items":{}}
Created ETag: f0d0f717619b09cc081bb0c11d9b9c6b
Root ETag: ef528a27b48c1b187ef7116f7306358b
{"@context":"http://remotestorage.io/spec/folder-description","items":{"Documents/":{"ETag":"cc4c6d3bbf39189be874992479b60e2a"}}}
Documents/ ETag: cc4c6d3bbf39189be874992479b60e2a
{"@context":"http://remotestorage.io/spec/folder-description","items":{"First.txt":{"Content-Length":18,"Content-Type":"funny/format","ETag":"f0d0f717619b09cc081bb0c11d9b9c6b","Last-Modified":"Mon, 01 Jan 0001 00:00:00 UTC"}}}

func LDGet

func LDGet[T any](ld LDjson, keys ...string) (t T, err error)

LDGet retrieves a value of type T from a nested ld+json map. It recursively follows the keys to reach the final value.

func Load

func Load(persistFile io.Reader) error

Load deserializes XML data from persistFile and adds the documents and folders to the storage tree. If storage has not been initialized before, Reset must be invoked before calling Load.

func Migrate

func Migrate(root string) (errs []error)

Migrate traverses the root directory and copies any files contained therein into the remoteStorage root (cfg.Sroot).

func Persist

func Persist(persistFile io.Writer) (err error)

Persist serializes the storage tree to XML. The generated XML is written to persistFile.

Example
mockServer()

panicIf := func(err error) {
	if err != nil {
		panic(err)
	}
}

const (
	testContent1 = "Whole life's a test."
	testContent2 = "Hello, World!"
)

{
	sname := genpath()
	err := FS.WriteFile(sname, []byte(testContent1), 0666)
	panicIf(err)
	_, err = AddDocument("/Documents/test.txt", sname, int64(len(testContent1)), "text/plain")
	panicIf(err)
}

{
	sname := genpath()
	err := FS.WriteFile(sname, []byte(testContent2), 0666)
	panicIf(err)
	_, err = AddDocument("/Documents/hello.txt", sname, int64(len(testContent2)), "text/plain")
	panicIf(err)
}

fd, err := FS.Create(g.sroot + "/marshalled.xml")
panicIf(err)
defer fd.Close()
err = Persist(fd)
panicIf(err)

fd.Seek(0, io.SeekStart)
bs, err := io.ReadAll(fd)
panicIf(err)
fmt.Printf("XML follows:\n%s\n", bs)

fd.Seek(0, io.SeekStart)
Reset()
err = Load(fd)
panicIf(err)
fmt.Printf("Storage listing follows:\n%s", root)
Output:

XML follows:
<Root>
	<Nodes IsFolder="true">
		<Name>Documents/</Name>
		<Rname>/Documents</Rname>
		<ETag>86f32f54096e02778610b22d1d6c56db</ETag>
		<Mime>inode/directory</Mime>
		<ParentRName>/</ParentRName>
	</Nodes>
	<Nodes IsFolder="false">
		<Name>hello.txt</Name>
		<Rname>/Documents/hello.txt</Rname>
		<Sname>/tmp/rms/storage/32000000-0000-0000-0000-000000000000</Sname>
		<ETag>ea724748ce53d55deb465a6d045fd160</ETag>
		<Mime>text/plain</Mime>
		<Length>13</Length>
		<LastMod>0001-01-01T00:00:00Z</LastMod>
		<ParentRName>/Documents</ParentRName>
	</Nodes>
	<Nodes IsFolder="false">
		<Name>test.txt</Name>
		<Rname>/Documents/test.txt</Rname>
		<Sname>/tmp/rms/storage/31000000-0000-0000-0000-000000000000</Sname>
		<ETag>10b3bf730d787feceec1d534a876dc5f</ETag>
		<Mime>text/plain</Mime>
		<Length>20</Length>
		<LastMod>0001-01-01T00:00:00Z</LastMod>
		<ParentRName>/Documents</ParentRName>
	</Nodes>
</Root>
Storage listing follows:
{F} / [/] [6330643033303764]
  {F} Documents/ [/Documents] [3836663332663534]
    {D} hello.txt (text/plain, 13) [/Documents/hello.txt -> /tmp/rms/storage/32000000-0000-0000-0000-000000000000] [6561373234373438]
    {D} test.txt (text/plain, 20) [/Documents/test.txt -> /tmp/rms/storage/31000000-0000-0000-0000-000000000000] [3130623362663733]

func PutDocument

func PutDocument(w http.ResponseWriter, r *http.Request) error

func Register

func Register(mux *http.ServeMux)

Register the remote storage server (with middleware if configured) to the mux using Rroot + '/' as pattern. If mux is nil, http.DefaultServeMux is used.

Example

ExampleRegister demonstrates how to register the remote storage endpoints to a serve mux.

package main

import (
	"bytes"
	"log"
	"net/http"

	"github.com/cvanloo/rmsgo"
)

func main() {
	const (
		remoteRoot  = "/storage/"
		storageRoot = "/var/rms/storage/"
	)

	// [!] TODO: Use a real file
	persistFile := &bytes.Buffer{}

	// Restore server state at startup
	err := rmsgo.Load(persistFile)
	if err != nil {
		log.Fatal(err)
	}

	_, err = rmsgo.Configure(remoteRoot, storageRoot)
	if err != nil {
		log.Fatal(err)
	}

	mux := http.NewServeMux()
	// TODO: Other mux.Handle setup

	rmsgo.Register(mux)
	http.ListenAndServe(":8080", mux) // [!] TODO: Use TLS

	// Persist server state at shutdown
	err = rmsgo.Persist(persistFile)
	if err != nil {
		log.Fatal(err)
	}
}
Output:

Example (UsingDefaultServeMux)

Alternatively, the endpoints can be registered to the http.DefaultServeMux by passing nil to Register.

package main

import (
	"bytes"
	"log"
	"net/http"

	"github.com/cvanloo/rmsgo"
)

func main() {
	const (
		remoteRoot  = "/storage/"
		storageRoot = "/var/rms/storage/"
	)

	// [!] TODO: Use a real file
	persistFile := &bytes.Buffer{}

	// Restore server state at startup
	err := rmsgo.Load(persistFile)
	if err != nil {
		log.Fatal(err)
	}

	_, err = rmsgo.Configure(remoteRoot, storageRoot)
	if err != nil {
		log.Fatal(err)
	}

	rmsgo.Register(nil)
	http.ListenAndServe(":8080", nil) // [!] TODO: Use TLS

	// Persist server state at shutdown
	err = rmsgo.Persist(persistFile)
	if err != nil {
		log.Fatal(err)
	}
}
Output:

func RemoveDocument

func RemoveDocument(n *node)

RemoveDocument deletes a document from the storage tree and invalidates the etags of its ancestors.

func Reset

func Reset()

Reset (re-) initializes the storage tree, so that it only contains a root folder.

func Retrieve

func Retrieve(rname string) (*node, error)

Retrieve a document or folder identified by rname. Returns ErrNotExist if rname can't be found.

func UpdateDocument

func UpdateDocument(n *node, mime string, fsize int64)

UpdateDocument updates an existing document in the storage tree with new information and invalidates etags of the document and its ancestors.

func WriteError

func WriteError(w http.ResponseWriter, err error) error

WriteError formats and writes err to w. If err is of type HttpError, its fields are formatted into an ld+json map and written to w. The status code is decided upon based on (HttpError).Cause: if Cause is one of the sentinel error values, status is looked up in the StatusCodes mapping. Else, if Cause in an unknown error, ErrServerError (500) is used and Cause is returned for further error handling. If err is NOT of type HttpError, only the response status is determined in the same manner as for HttpErrors, but no response body is written.

Types

type AllowOriginFunc

type AllowOriginFunc func(r *http.Request, origin string) bool

AllowOriginFunc decides whether an origin is allowed (returns true) or forbidden (returns false).

type AuthenticateFunc

type AuthenticateFunc func(r *http.Request, bearer string) (User, bool)

AuthenticateFunc authenticates a request (usually with the bearer token). If the request is correctly authenticated, a User and true must be returned, otherwise the returned values must be nil and false.

type ConflictError

type ConflictError struct {
	Path         string
	ConflictPath string
}

func (ConflictError) Error

func (e ConflictError) Error() string

type ETag

type ETag []byte

ETag is a short and unique identifier assigned to a specific version of a remoteStorage resource.

func ParseETag

func ParseETag(s string) (ETag, error)

ParseETag decodes an ETag previously encoded by (ETag).String()

func (ETag) Equal

func (e ETag) Equal(other ETag) bool

func (ETag) String

func (e ETag) String() string

String creates a string from an Etag e. To go the opposite way an obtain an ETag from a string, use ParseETag.

type ErrorHandlerFunc

type ErrorHandlerFunc func(err error)

Any errors that the remoteStorage server doesn't know how to handle itself are passed to the ErrorHandlerFunc.

type HttpError

type HttpError struct {
	// Msg is a human readable error message.
	Msg string
	// Desc provides additional information to the error.
	Desc string
	// A URL where further details or help for the solution can be found.
	URL string
	// Additonal Data related to the error.
	Data LDjson
	// Underlying error that caused the exception.
	// Cause is used to look up a response status in StatusCodes.
	// If not contained in StatusCodes, ErrServerError is used instead, and the
	// Cause is passed to the library user for further handling.
	Cause error
}

HttpError contains detailed error information, intended to be shown to users. No sensitive data should be contained by any of its fields (with Cause being the only exception).

func (HttpError) Error

func (e HttpError) Error() string

func (HttpError) Unwrap

func (e HttpError) Unwrap() error

type LDjson

type LDjson = map[string]any

LDjson aliases instead of defines a new type to make unmarshalling easier.

type Level

type Level string
var (
	LevelNone      Level = ""
	LevelRead      Level = ":r"
	LevelReadWrite Level = ":rw"
)

type LoggingResponseWriter

type LoggingResponseWriter struct {
	http.ResponseWriter // compose original ResponseWriter
	Status, Size        int
}

func NewLoggingResponseWriter

func NewLoggingResponseWriter(w http.ResponseWriter) *LoggingResponseWriter

func (*LoggingResponseWriter) Write

func (lrw *LoggingResponseWriter) Write(b []byte) (int, error)

func (*LoggingResponseWriter) WriteHeader

func (lrw *LoggingResponseWriter) WriteHeader(statusCode int)

type MiddlewareFunc

type MiddlewareFunc func(next http.Handler) http.Handler

A MiddlewareFunc is inserted into a chain of other http.Handler. This way, different parts of handling a request can be separated each into its own handler.

type NodeDTO

type NodeDTO struct {
	IsFolder    bool `xml:"IsFolder,attr"`
	Name        string
	Rname       string
	Sname       string `xml:"Sname,omitempty"`
	ETag        string
	Mime        string
	Length      int64      `xml:"Length,omitempty"`
	LastMod     *time.Time `xml:"LastMod,omitempty"`
	ParentRName string
}

type Options added in v0.1.1

type Options struct {
	// contains filtered or unexported fields
}
Example

Configure returns a reference to an options object. This can be used to customize the configuration, e.g., to configure CORS, and to setup authentication, additional middleware, and more.

package main

import (
	"log"
	"net/http"
	"time"

	"github.com/cvanloo/rmsgo"
)

func main() {
	const (
		remoteRoot  = "/storage/"
		storageRoot = "/var/rms/storage/"
	)

	opts, err := rmsgo.Configure(remoteRoot, storageRoot)
	if err != nil {
		log.Fatal(err)
	}

	opts.UseErrorHandler(func(err error) {
		log.Panicf("remote storage: unhandled error: %v", err)
	})

	opts.UseMiddleware(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			start := time.Now()
			lrw := rmsgo.NewLoggingResponseWriter(w)

			// [!] Pass request on to remote storage server
			next.ServeHTTP(lrw, r)

			duration := time.Since(start)

			// maybe use an actual library for structured logging
			log.Printf("%v", map[string]any{
				"method":   r.Method,
				"uri":      r.RequestURI,
				"duration": duration,
				"status":   lrw.Status,
				"size":     lrw.Size,
			})
		})
	})

	opts.UseAuthentication(func(r *http.Request, bearer string) (rmsgo.User, bool) {
		// [!] TODO: Your authentication logic here...
		//       Return one of your own users.
		return rmsgo.ReadWriteUser{}, true
	})

	rmsgo.Register(nil)
	http.ListenAndServe(":8080", nil) // [!] TODO: Use TLS
}
Output:

func Configure added in v0.1.1

func Configure(remoteRoot, storageRoot string) (*Options, error)

Configure initializes the remote storage server with the default configuration. remoteRoot is the URL path below which remote storage is accessible, and storageRoot is a folder on the server's file system where remoteStorage documents are written to and read from. A pointer to the Options object is returned and allows for further configuration beyond the default settings.

func (*Options) AllowAnyReadWrite added in v0.1.1

func (o *Options) AllowAnyReadWrite()

AllowAnyReadWrite allows even unauthenticated requests to create, read, and delete any documents on the server. This option has no effect if UseAuthentication is used. Per default, i.e if no other option is configured, any GET and HEAD requests are allowed.

func (*Options) Rroot added in v0.1.1

func (o *Options) Rroot() string

Rroot specifies the URL path at which remoteStorage is rooted. E.g., if Rroot is "/storage" then a document "/Picture/Kittens.png" can be accessed using the URL "https://example.com/storage/Picture/Kittens.png". Rroot does not have a trailing slash.

func (*Options) Sroot added in v0.1.1

func (o *Options) Sroot() string

Sroot is a path specifying the location on the server's file system where all of remoteStorage's files are stored. Sroot does not have a trailing slash.

func (*Options) UseAllowOrigin added in v0.1.1

func (o *Options) UseAllowOrigin(f AllowOriginFunc)

UseAllowOrigin configures the remote storage server to use f to decide whether an origin is allowed or not. If this option is set up, the list of origins set by AllowOrigins is ignored.

func (*Options) UseAllowedOrigins added in v0.1.1

func (o *Options) UseAllowedOrigins(origins []string)

UseAllowedOrigins configures a list of allowed origins. By default, i.e if UseAllowedOrigins is never called, all origins are allowed.

func (*Options) UseAuthentication added in v0.1.1

func (o *Options) UseAuthentication(a AuthenticateFunc)

UseAuthentication configures the function to use for authenticating requests.

func (*Options) UseErrorHandler added in v0.1.1

func (o *Options) UseErrorHandler(h ErrorHandlerFunc)

UseErrorHandler configures the error handler to use.

func (*Options) UseMiddleware added in v0.1.1

func (o *Options) UseMiddleware(m MiddlewareFunc)

UseMiddleware configures middleware (e.g., for logging) in front of the remote storage server. The middleware is responsible for passing the request on to the rms server using next.ServeHTTP(w, r).

type ReadOnlyUser

type ReadOnlyUser struct{}

ReadOnlyUser is a User with read access to any folder.

func (ReadOnlyUser) Permission

func (ReadOnlyUser) Permission(name string) Level

type ReadPublicUser added in v0.1.1

type ReadPublicUser struct{}

ReadPublicUser is a User with read permissions only to public folders.

func (ReadPublicUser) Permission added in v0.1.1

func (ReadPublicUser) Permission(name string) Level

type ReadWriteUser

type ReadWriteUser struct{}

ReadWriteUser is a User with read and write access to any folder.

func (ReadWriteUser) Permission

func (ReadWriteUser) Permission(name string) Level

type User

type User interface {
	Permission(name string) Level
}

func UserFromContext

func UserFromContext(ctx context.Context) (User, bool)

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL