Documentation ¶
Index ¶
- Variables
- func GetInsertQuery(table string, fields ...string) string
- func MimeAny(string) bool
- func MimeIsText(mime string) bool
- func NewURLStore(db *sql.DB, opts ...URLStoreOpt) (*urlStore, error)
- func OnlyTLD(ending string) func(*url.URL) bool
- type ActionStore
- type BodyStore
- type Compressor
- type ConsoleStore
- type FileStore
- type FileStoreOpt
- type Hasher
- type HeaderStore
- type IDStore
- type InitiatorStackStore
- type MimeValidator
- type PostDataStore
- type Sampler
- type ScreenStore
- type ScreenshotStore
- type SecurityStore
- type SessionStore
- type Store
- type StoredFile
- type URLFilter
- type URLStoreOpt
- type UrlStore
- type WarehouseInserter
Constants ¶
This section is empty.
Variables ¶
View Source
var GzipCompression = gzipComp{}
View Source
var NoCompression = noComp{}
View Source
var (
NotAllowedMimeErr = errors.New("mime type is not allowed to be stored")
)
View Source
var Sha256Hasher = &hasher{sum: func(d []byte) string { return fmt.Sprintf("%x", sha256.Sum256(d)) }, name: "sha256"}
View Source
var (
StoreIsEmptyErr = errors.New("store is empty")
)
Functions ¶
func GetInsertQuery ¶
func MimeIsText ¶
func NewURLStore ¶
func NewURLStore(db *sql.DB, opts ...URLStoreOpt) (*urlStore, error)
Types ¶
type ActionStore ¶
type ActionStore struct {
// contains filtered or unexported fields
}
func NewActionStore ¶
func NewActionStore(db *sql.DB, fs *FileStore) (*ActionStore, error)
func (*ActionStore) Save ¶
func (as *ActionStore) Save(tx *sql.Tx, id int64, actions []*kraaler.CrawlAction) error
type Compressor ¶
type ConsoleStore ¶
type ConsoleStore struct {
// contains filtered or unexported fields
}
func NewConsoleStore ¶
func NewConsoleStore(db *sql.DB) (*ConsoleStore, error)
func (*ConsoleStore) Save ¶
func (cs *ConsoleStore) Save(tx *sql.Tx, id int64, console []*kraaler.JavaScriptConsole) error
type FileStore ¶
type FileStore struct {
// contains filtered or unexported fields
}
func NewFileStore ¶
func NewFileStore(root string, opts ...FileStoreOpt) (*FileStore, error)
type FileStoreOpt ¶
type FileStoreOpt func(fs *FileStore)
func WithCompression ¶
func WithCompression(c Compressor) FileStoreOpt
func WithMimeTypes ¶
func WithMimeTypes(types ...MimeValidator) FileStoreOpt
type HeaderStore ¶
type HeaderStore struct {
// contains filtered or unexported fields
}
func NewHeaderStore ¶
func NewHeaderStore(db *sql.DB) (*HeaderStore, error)
func (*HeaderStore) SaveRequest ¶
func (*HeaderStore) SaveResponse ¶
type InitiatorStackStore ¶
type InitiatorStackStore struct{}
func NewInitiatorStackStore ¶
func NewInitiatorStackStore(db *sql.DB) (*InitiatorStackStore, error)
type MimeValidator ¶
type PostDataStore ¶
type PostDataStore struct{}
func NewPostDataStore ¶
func NewPostDataStore(db *sql.DB) (*PostDataStore, error)
type ScreenStore ¶
type ScreenStore struct {
// contains filtered or unexported fields
}
func NewScreenStore ¶
func NewScreenStore(db *sql.DB, ss *ScreenshotStore) (*ScreenStore, error)
func (*ScreenStore) Save ¶
func (ss *ScreenStore) Save(tx *sql.Tx, id int64, urlstr string, screenshots []*kraaler.BrowserScreenshot) error
type ScreenshotStore ¶
type ScreenshotStore struct {
// contains filtered or unexported fields
}
func NewScreenshotStore ¶
func NewScreenshotStore(dir string) *ScreenshotStore
func (*ScreenshotStore) Store ¶
func (ss *ScreenshotStore) Store(s *kraaler.BrowserScreenshot, domain string) (string, error)
type SecurityStore ¶
type SecurityStore struct {
// contains filtered or unexported fields
}
func NewSecurityStore ¶
func NewSecurityStore(db *sql.DB) (*SecurityStore, error)
func (*SecurityStore) Save ¶
func (ss *SecurityStore) Save(tx *sql.Tx, id int64, sd *network.SecurityDetails) error
type SessionStore ¶
type SessionStore struct {
// contains filtered or unexported fields
}
func NewSessionStore ¶
func NewSessionStore(db *sql.DB) (*SessionStore, error)
type StoredFile ¶
type URLStoreOpt ¶
type URLStoreOpt func(*urlStore)
func WithNoResampling ¶
func WithNoResampling() URLStoreOpt
func WithSampler ¶
func WithSampler(s Sampler) URLStoreOpt
func WithURLFilters ¶
func WithURLFilters(f ...URLFilter) URLStoreOpt
Click to show internal directories.
Click to hide internal directories.