photon

package module
v0.0.0-...-367f6b9 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 20, 2019 License: MIT Imports: 10 Imported by: 0

README

photon

Build Status codecov.io Go Report codebeat

a fast lightweight and customizable crawler framework

License

The MIT License (MIT)

Credits

This repo relies on the following third-party projects:

Document

TODO:

  • add log
  • complete document and comment
  • complete test

Documentation

Overview

Example
package main

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"

	"github.com/n0trace/photon"
)

func main() {
	p := photon.New()
	p.Get(newTestServer().URL+"/users?id=hello", func(ctx photon.Context) {
		text, _ := ctx.Text()
		fmt.Println(text)
	})
	p.Wait()
}

func newTestServer() *httptest.Server {
	mux := http.NewServeMux()
	mux.HandleFunc("/users", func(w http.ResponseWriter, r *http.Request) {
		u, err := url.Parse(r.RequestURI)
		if err != nil {
			panic(err)
		}
		w.Write([]byte(u.Query().Get("id")))
	})

	mux.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) {
		cookie := http.Cookie{Name: "username", Value: r.FormValue("username"), Path: "/", MaxAge: 86400}
		http.SetCookie(w, &cookie)
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("ok"))
	})

	mux.HandleFunc("/must-login", func(w http.ResponseWriter, r *http.Request) {
		cookie, _ := r.Cookie("username")
		w.Write([]byte("hello " + cookie.Value))
	})

	mux.HandleFunc("/user-agent", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(r.Header.Get("User-Agent")))
	})

	return httptest.NewServer(mux)
}
Output:

hello
Example (KeepAuth)
package main

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"

	"github.com/n0trace/photon"
	"github.com/n0trace/photon/middleware"
)

func main() {
	p := photon.New()

	reader := strings.NewReader("username=foo&password=bar")

	lastCtx := p.Post(newTestServer().URL+"/login",
		"application/x-www-form-urlencoded", reader,
		func(ctx photon.Context) {
			text, _ := ctx.Text()
			fmt.Println(text)
		})

	p.Get(newTestServer().URL+"/must-login", func(ctx photon.Context) {
		text, _ := ctx.Text()
		fmt.Println(text)
	}, middleware.FromContext(lastCtx))

	p.Wait()

}

func newTestServer() *httptest.Server {
	mux := http.NewServeMux()
	mux.HandleFunc("/users", func(w http.ResponseWriter, r *http.Request) {
		u, err := url.Parse(r.RequestURI)
		if err != nil {
			panic(err)
		}
		w.Write([]byte(u.Query().Get("id")))
	})

	mux.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) {
		cookie := http.Cookie{Name: "username", Value: r.FormValue("username"), Path: "/", MaxAge: 86400}
		http.SetCookie(w, &cookie)
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("ok"))
	})

	mux.HandleFunc("/must-login", func(w http.ResponseWriter, r *http.Request) {
		cookie, _ := r.Cookie("username")
		w.Write([]byte("hello " + cookie.Value))
	})

	mux.HandleFunc("/user-agent", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(r.Header.Get("User-Agent")))
	})

	return httptest.NewServer(mux)
}
Output:

ok
hello foo
Example (UseMiddleware)
package main

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"time"

	"github.com/n0trace/photon"
	"github.com/n0trace/photon/middleware"
)

func main() {
	rootURL := newTestServer().URL
	p := photon.New()
	p.Use(middleware.Limit(200*time.Millisecond), middleware.UserAgent("diy-agent"))
	for i := 0; i != 3; i++ {
		url := fmt.Sprintf("%s/user-agent", rootURL)
		p.Get(url, func(ctx photon.Context) {
			text, _ := ctx.Text()
			fmt.Println(text)
		})
	}
	//or:
	//p.Get(url,callback,middleware...)
	p.Wait()
}

func newTestServer() *httptest.Server {
	mux := http.NewServeMux()
	mux.HandleFunc("/users", func(w http.ResponseWriter, r *http.Request) {
		u, err := url.Parse(r.RequestURI)
		if err != nil {
			panic(err)
		}
		w.Write([]byte(u.Query().Get("id")))
	})

	mux.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) {
		cookie := http.Cookie{Name: "username", Value: r.FormValue("username"), Path: "/", MaxAge: 86400}
		http.SetCookie(w, &cookie)
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("ok"))
	})

	mux.HandleFunc("/must-login", func(w http.ResponseWriter, r *http.Request) {
		cookie, _ := r.Cookie("username")
		w.Write([]byte("hello " + cookie.Value))
	})

	mux.HandleFunc("/user-agent", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(r.Header.Get("User-Agent")))
	})

	return httptest.NewServer(mux)
}
Output:

diy-agent
diy-agent
diy-agent

Index

Examples

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Context

type Context interface {
	Request() *http.Request
	SetRequest(*http.Request)
	Client() *http.Client
	SetClient(*http.Client)
	Reset()
	Error() error
	SetError(error)
	SetStdResponse(*http.Response)
	StdResponse() *http.Response
	Text() (string, error)
	Bytes() ([]byte, error)
	JSON(interface{}) error
	XML(interface{}) error
	Document() (*goquery.Document, error)
	Photon() *Photon
	Set(string, interface{})
	Get(string) (interface{}, bool)
	FromContext(interface{})
	Downloaded() bool
	SetDownload(bool)
	Finish()
	WaitFinish()
}

type HandlerFunc

type HandlerFunc func(Context)

type Jar

type Jar struct {
	// contains filtered or unexported fields
}

func (*Jar) Cookies

func (jar *Jar) Cookies(u *url.URL) []*http.Cookie

func (*Jar) SetCookies

func (jar *Jar) SetCookies(u *url.URL, cookies []*http.Cookie)

type MiddlewareFunc

type MiddlewareFunc func(HandlerFunc) HandlerFunc

type Photon

type Photon struct {
	// contains filtered or unexported fields
}

func New

func New() (p *Photon)

New new Photon Instance

func (*Photon) Get

func (p *Photon) Get(url string, handle HandlerFunc, middlewares ...MiddlewareFunc) Context

Get Get

func (*Photon) NewContext

func (p *Photon) NewContext() *context

func (*Photon) NewStdClient

func (p *Photon) NewStdClient() *http.Client

func (*Photon) Post

func (p *Photon) Post(url string, contentType string, body io.Reader, handle HandlerFunc, middlewares ...MiddlewareFunc) Context

Post Post

func (*Photon) Request

func (p *Photon) Request(req *http.Request, cb HandlerFunc, middleware ...MiddlewareFunc) Context

Request Request

func (*Photon) SetParallel

func (p *Photon) SetParallel(parallel int)

SetParallel set parallel

func (*Photon) Use

func (p *Photon) Use(middleware ...MiddlewareFunc)

func (*Photon) Wait

func (p *Photon) Wait()

Wait Wait

type Response

type Response struct {
	*http.Response
	// contains filtered or unexported fields
}

Response wraps an http.ResponseWriter and implements its interface to be used

func (*Response) Bytes

func (resp *Response) Bytes() (bodyBytes []byte, err error)

Bytes get []byte from response.Body

func (*Response) Document

func (resp *Response) Document() (document *goquery.Document, err error)

Document get query element from resp.Body

func (*Response) JSON

func (resp *Response) JSON(i interface{}) (err error)

JSON parses the resp.Body data and stores the result

func (*Response) Text

func (resp *Response) Text() (text string, err error)

Text get text from response.Body

func (*Response) XML

func (resp *Response) XML(i interface{}) (err error)

XML parses the resp.Body data and stores the result

Directories

Path Synopsis
_examples

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL