scroll

package module
v1.2.2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 13, 2018 License: Apache-2.0 Imports: 24 Imported by: 15

README

scroll

Build Status

Scroll is a lightweight library for building Go HTTP services at Mailgun. It is built on top of mux and adds:

  • Service Discovery
  • Graceful Shutdown
  • Configurable Logging
  • Request Metrics

Scroll is a work in progress. Use at your own risk.

Installation

go get github.com/mailgun/scroll

Getting Started

Building an application with Scroll is simple. Here's a server that listens for GET or POST requests to http://0.0.0.0:8080/resources/{resourceID} and echoes back the resource ID provided in the URL.

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/mailgun/holster/etcdutil"
	"github.com/mailgun/metrics"
	"github.com/mailgun/scroll"
	"github.com/mailgun/scroll/vulcand"
)

const (
	APPNAME = "example"
)

func Example() {
	// These environment variables provided by the environment,
	// we set them here to only to illustrate how `NewEtcdConfig()`
	// uses the environment to create a new etcd config
	os.Setenv("ETCD3_USER", "root")
	os.Setenv("ETCD3_PASSWORD", "rootpw")
	os.Setenv("ETCD3_ENDPOINT", "localhost:2379")
	os.Setenv("ETCD3_SKIP_VERIFY", "true")

	// If this is set to anything but empty string "", scroll will attempt
	// to retrieve the applications config from '/mailgun/configs/{env}/APPNAME'
	// and fill in the PublicAPI, ProtectedAPI, etc.. fields from that config
	os.Setenv("MG_ENV", "")

	// Create a new etc config from available environment variables
	cfg, err := etcdutil.NewEtcdConfig(nil)
	if err != nil {
		fmt.Fprintf(os.Stderr, "while creating etcd config: %s\n", err)
		return
	}

	hostname, err := os.Hostname()
	if err != nil {
		fmt.Fprintf(os.Stderr, "while obtaining hostname: %s\n", err)
		return
	}

	// Send metrics to statsd @ localhost
	mc, err := metrics.NewWithOptions("localhost:8125",
		fmt.Sprintf("%s.%v", APPNAME, strings.Replace(hostname, ".", "_", -1)),
		metrics.Options{UseBuffering: true, FlushPeriod: time.Second})
	if err != nil {
		fmt.Fprintf(os.Stderr, "while initializing metrics: %s\n", err)
		return
	}

	app, err := scroll.NewAppWithConfig(scroll.AppConfig{
		Vulcand:          &vulcand.Config{Etcd: cfg},
		PublicAPIURL:     "http://api.mailgun.net:1212",
		ProtectedAPIURL:  "http://localhost:1212",
		PublicAPIHost:    "api.mailgun.net",
		ProtectedAPIHost: "localhost",
		Name:             APPNAME,
		ListenPort:       1212,
		Client:           mc,
	})
	if err != nil {
		fmt.Fprintf(os.Stderr, "while initializing scroll: %s\n", err)
		return
	}

	app.AddHandler(
		scroll.Spec{
			Methods: []string{"GET"},
			Paths:   []string{"/hello"},
			Handler: func(w http.ResponseWriter, r *http.Request, params map[string]string) (interface{}, error) {
				return scroll.Response{"message": "Hello World"}, nil
			},
		},
	)

	// Start serving requests
	app.Run()

Documentation

Overview

Example
package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/mailgun/holster/etcdutil"
	"github.com/mailgun/metrics"
	"github.com/mailgun/scroll"
	"github.com/mailgun/scroll/vulcand"
)

const (
	APPNAME = "example"
)

func main() {
	// These environment variables provided by the environment,
	// we set them here to only to illustrate how `NewEtcdConfig()`
	// uses the environment to create a new etcd config
	os.Setenv("ETCD3_USER", "root")
	os.Setenv("ETCD3_PASSWORD", "rootpw")
	os.Setenv("ETCD3_ENDPOINT", "https://localhost:2379")
	// Set this to force connecting with TLS, but without cert verification
	os.Setenv("ETCD3_SKIP_VERIFY", "true")

	// If this is set to anything but empty string "", scroll will attempt
	// to retrieve the applications config from '/mailgun/configs/{env}/APPNAME'
	// and fill in the PublicAPI, ProtectedAPI, etc.. fields from that config
	os.Setenv("MG_ENV", "")

	// Create a new etc config from available environment variables
	cfg, err := etcdutil.NewConfig(nil)
	if err != nil {
		fmt.Fprintf(os.Stderr, "while creating etcd config: %s\n", err)
		return
	}

	hostname, err := os.Hostname()
	if err != nil {
		fmt.Fprintf(os.Stderr, "while obtaining hostname: %s\n", err)
		return
	}

	// Send metrics to statsd @ localhost
	mc, err := metrics.NewWithOptions("localhost:8125",
		fmt.Sprintf("%s.%v", APPNAME, strings.Replace(hostname, ".", "_", -1)),
		metrics.Options{UseBuffering: true, FlushPeriod: time.Second})
	if err != nil {
		fmt.Fprintf(os.Stderr, "while initializing metrics: %s\n", err)
		return
	}

	app, err := scroll.NewAppWithConfig(scroll.AppConfig{
		Vulcand:          &vulcand.Config{Etcd: cfg},
		PublicAPIURL:     "http://api.mailgun.net:12121",
		ProtectedAPIURL:  "http://localhost:12121",
		PublicAPIHost:    "api.mailgun.net",
		ProtectedAPIHost: "localhost",
		Name:             APPNAME,
		ListenPort:       12121,
		Client:           mc,
	})
	if err != nil {
		fmt.Fprintf(os.Stderr, "while initializing scroll: %s\n", err)
		return
	}

	app.AddHandler(
		scroll.Spec{
			Methods: []string{"GET"},
			Paths:   []string{"/hello"},
			Handler: func(w http.ResponseWriter, r *http.Request, params map[string]string) (interface{}, error) {
				return scroll.Response{"message": "Hello World"}, nil
			},
		},
	)

	// Start serving requests
	go func() {
		if err := app.Run(); err != nil {
			fmt.Fprintf(os.Stderr, "while serving requests: %s\n", err)
		}
	}()

	// Wait for the endpoint to begin accepting connections
	waitFor("http://localhost:12121")

	// Get the response
	r, err := http.Get("http://localhost:12121/hello")
	if err != nil {
		fmt.Fprintf(os.Stderr, "GET request failed with: %s\n", err)
		return
	}
	content, _ := ioutil.ReadAll(r.Body)

	fmt.Println(string(content))

	// Shutdown the server and de-register routes with vulcand
	app.Stop()

}

// Waits for the endpoint to start accepting connections
func waitFor(url string) {
	after := time.After(time.Second * 10)
	_, err := http.Get(url)
	for err != nil {
		_, err = http.Get(url)
		select {
		case <-after:
			fmt.Fprintf(os.Stderr, "endpoint timeout: %s\n", err)
			os.Exit(1)
		default:
		}
	}
}
Output:

{"message":"Hello World"}

Index

Examples

Constants

View Source
const (
	// Suggested result set limit for APIs that may return many entries (e.g. paging).
	DefaultLimit = 100

	// Suggested max allowed result set limit for APIs that may return many entries (e.g. paging).
	MaxLimit = 10000

	// Suggested max allowed amount of entries that batch APIs can accept (e.g. batch uploads).
	MaxBatchSize = 1000
)

Variables

View Source
var LogRequest func(*http.Request, int, time.Duration, error)

When Handler or HandlerWithBody is used, this function will be called after every request with a log message. If nil, defaults to github.com/mailgun/log.Infof.

Functions

func DecodeParams

func DecodeParams(src map[string]string) map[string]string

Given a map of parameters url decode each parameter

func GetDurationField

func GetDurationField(r *http.Request, fieldName string) (time.Duration, error)

GetDurationField retrieves a request field as a time.Duration, which is not allowed to be negative. Returns `MissingFieldError` if requested field is missing.

func GetFloatField

func GetFloatField(r *http.Request, fieldName string) (float64, error)

Retrieve a request field as a float. Returns `MissingFieldError` if requested field is missing.

func GetIntField

func GetIntField(r *http.Request, fieldName string) (int, error)

Retrieve a POST request field as an integer. Returns `MissingFieldError` if requested field is missing.

func GetMultipleFields

func GetMultipleFields(r *http.Request, fieldName string) ([]string, error)

Retrieve fields with the same name as an array of strings.

func GetStringField

func GetStringField(r *http.Request, fieldName string) (string, error)

Retrieve a POST request field as a string. Returns `MissingFieldError` if requested field is missing.

func GetStringFieldSafe

func GetStringFieldSafe(r *http.Request, fieldName string, allowSet AllowSet) (string, error)

Retrieves requested field as a string, allowSet provides input sanitization. If an error occurs, returns either a `MissingFieldError` or an `UnsafeFieldError`.

func GetStringFieldWithDefault

func GetStringFieldWithDefault(r *http.Request, fieldName, defaultValue string) string

Retrieve a POST request field as a string. If the requested field is missing, returns provided default value.

func GetTimestampField

func GetTimestampField(r *http.Request, fieldName string) (time.Time, error)

Helper method to retrieve an optional timestamp from POST request field. If no timestamp provided, returns current time. Returns `InvalidFormatError` if provided timestamp can't be parsed.

func GetVarSafe

func GetVarSafe(r *http.Request, variableName string, allowSet AllowSet) (string, error)

GetVarSafe is a helper function that returns the requested variable from URI with allowSet providing input sanitization. If an error occurs, returns either a `MissingFieldError` or an `UnsafeFieldError`.

func HasField

func HasField(r *http.Request, fieldName string) bool

func MakeHandler

func MakeHandler(app *App, fn HandlerFunc, spec Spec) http.HandlerFunc

Wraps the provided handler function encapsulating boilerplate code so handlers do not have to implement it themselves: parsing a request's form, formatting a proper JSON response, emitting the request stats, etc.

func MakeHandlerWithBody

func MakeHandlerWithBody(app *App, fn HandlerWithBodyFunc, spec Spec) http.HandlerFunc

Make a handler out of HandlerWithBodyFunc, just like regular MakeHandler function.

func Reply

func Reply(w http.ResponseWriter, response interface{}, status int)

Reply with the provided HTTP response and status code.

Response body must be JSON-marshallable, otherwise the response will be "Internal Server Error".

func ReplyError

func ReplyError(w http.ResponseWriter, err error)

ReplyError converts registered error into HTTP response code and writes it back.

func ReplyInternalError

func ReplyInternalError(w http.ResponseWriter, message string)

ReplyInternalError logs the error message and replies with a 500 status code.

Types

type AllowSet

type AllowSet interface {
	IsSafe(string) error
}

The AllowSet interface is implemented to detect if input is safe or not.

type AllowSetBytes

type AllowSetBytes struct {
	// contains filtered or unexported fields
}

AllowSetBytes allows the definition of a set of safe allowed ASCII characters. AllowSetBytes does not support unicode code points. If you pass the string "ü" (which encodes as 0xc3 0xbc) they will be skipped over.

func NewAllowSetBytes

func NewAllowSetBytes(s string, maxlen int) AllowSetBytes

func (AllowSetBytes) IsSafe

func (a AllowSetBytes) IsSafe(s string) error

type AllowSetStrings

type AllowSetStrings struct {
	// contains filtered or unexported fields
}

AllowSetStrings allows the definition of a set of safe allowed strings.

func NewAllowSetStrings

func NewAllowSetStrings(s []string) AllowSetStrings

func (AllowSetStrings) IsSafe

func (a AllowSetStrings) IsSafe(s string) error

type App

type App struct {
	Config AppConfig
	// contains filtered or unexported fields
}

Represents an app.

func NewApp

func NewApp() (*App, error)

Create a new app.

func NewAppWithConfig

func NewAppWithConfig(config AppConfig) (*App, error)

Create a new app with the provided configuration.

func (*App) AddHandler

func (app *App) AddHandler(spec Spec) error

Register a handler function.

If vulcan registration is enabled in the both app config and handler spec, the handler will be registered in the local etcd instance.

func (*App) GetHandler

func (app *App) GetHandler() http.Handler

GetHandler returns HTTP compatible Handler interface.

func (*App) IsPublicRequest

func (app *App) IsPublicRequest(request *http.Request) bool

IsPublicRequest determines whether the provided request came through the public HTTP endpoint.

func (*App) Run

func (app *App) Run() error

Start the app on the configured host/port.

Supports graceful shutdown on 'kill' and 'int' signals.

func (*App) SetNotFoundHandler

func (app *App) SetNotFoundHandler(fn http.HandlerFunc)

SetNotFoundHandler sets the handler for the case when URL can not be matched by the router.

func (*App) Stop

func (app *App) Stop()

type AppConfig

type AppConfig struct {
	// name of the app being created
	Name string

	// IP/port the app will bind to
	ListenIP   string
	ListenPort int

	// optional router to use
	Router *mux.Router

	// host names of the public and protected API entrypoints used for vulcand registration
	PublicAPIHost    string
	PublicAPIURL     string // NOT USED, included for completeness
	ProtectedAPIHost string
	ProtectedAPIURL  string

	// Vulcand config must be provided to enable registration in etcd.
	Vulcand *vulcand.Config

	// metrics service used for emitting the app's real-time metrics
	Client metrics.Client

	HTTP struct {
		ReadTimeout  time.Duration
		WriteTimeout time.Duration
		IdleTimeout  time.Duration
	}
}

Represents a configuration object an app is created with.

type ConflictError

type ConflictError struct {
	Description string
}

func (ConflictError) Error

func (e ConflictError) Error() string

type GenericAPIError

type GenericAPIError struct {
	Reason string
}

func (GenericAPIError) Error

func (e GenericAPIError) Error() string

type HandlerFunc

type HandlerFunc func(http.ResponseWriter, *http.Request, map[string]string) (interface{}, error)

Defines the signature of a handler function that can be registered by an app.

The 3rd parameter is a map of variables extracted from the request path, e.g. if a request path was:

/resources/{resourceID}

and the request was made to:

/resources/1

then the map will contain the resource ID value:

{"resourceID": 1}

A handler function should return a JSON marshallable object, e.g. Response.

type HandlerWithBodyFunc

type HandlerWithBodyFunc func(http.ResponseWriter, *http.Request, map[string]string, []byte) (interface{}, error)

Defines a signature of a handler function, just like HandlerFunc.

In addition to the HandlerFunc a request's body is passed into this function as a 4th parameter.

type InvalidFormatError

type InvalidFormatError struct {
	Field string
	Value string
}

func (InvalidFormatError) Error

func (e InvalidFormatError) Error() string

type InvalidParameterError

type InvalidParameterError struct {
	Field string
	Value string
}

func (InvalidParameterError) Error

func (e InvalidParameterError) Error() string

type JSONConfig added in v1.2.0

type JSONConfig struct {
	PublicAPIHost    string `json:"public_api_host"`
	PublicAPIURL     string `json:"public_api_url"`
	ProtectedAPIHost string `json:"protected_api_host"`
	ProtectedAPIURL  string `json:"protected_api_url"`

	// Retrieved from via etcd
	VulcandNamespace string `json:"vulcand_namespace"`
}

This is a separate struct because JSON unmarshal() throws errors on the functions in AppConfig.Client

type MissingFieldError

type MissingFieldError struct {
	Field string
}

func (MissingFieldError) Error

func (e MissingFieldError) Error() string

type NotFoundError

type NotFoundError struct {
	Description string
}

func (NotFoundError) Error

func (e NotFoundError) Error() string

type RateLimitError

type RateLimitError struct {
	Description string
}

func (RateLimitError) Error

func (e RateLimitError) Error() string

type Response

type Response map[string]interface{}

Response objects that apps' handlers are advised to return.

Allows to easily return JSON-marshallable responses, e.g.:

Response{"message": "OK"}

type Scope

type Scope int
const (
	ScopePublic Scope = iota
	ScopeProtected
)

func (Scope) String

func (scope Scope) String() string

type Spec

type Spec struct {
	// List of HTTP methods the handler should match.
	Methods []string

	// List of paths the handler should match. A separate handler will be registered for each one of them.
	Paths []string

	// Key/value pairs of specific HTTP headers the handler should match (e.g. Content-Type).
	Headers []string

	// A handler function to use. Just one of these should be provided.
	RawHandler      http.HandlerFunc
	Handler         HandlerFunc
	HandlerWithBody HandlerWithBodyFunc

	// Unique identifier used when emitting performance metrics for the handler.
	MetricName string

	// Controls the handler's accessibility via vulcan (public or protected). If not specified, public is assumed.
	Scope Scope

	// Vulcan middlewares to register with the handler. When registering, middlewares are assigned priorities
	// according to their positions in the list: a middleware that appears in the list earlier is executed first.
	Middlewares []vulcand.Middleware

	// When Handler or HandlerWithBody is used, this function will be called after every request with a log message.
	// If nil, defaults to github.com/mailgun/log.Infof.
	LogRequest func(r *http.Request, status int, elapsedTime time.Duration, err error)
}

Represents handler's specification.

type UnsafeFieldError

type UnsafeFieldError struct {
	Field       string
	Description string
}

func (UnsafeFieldError) Error

func (e UnsafeFieldError) Error() string

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL