Documentation
¶
Index ¶
Constants ¶
View Source
const ( //StateSuccess - page without error StateSuccess State = 0 //StateDisabledByRobotsTxt - URL disabled in robots.txt StateDisabledByRobotsTxt = 1 //StateConnectError - can not connect to server StateConnectError = 2 //StateErrorStatusCode - status code != 200 StateErrorStatusCode = 3 //StateUnsupportedFormat - MIME type != text/html StateUnsupportedFormat = 4 //StateAnswerError - can not read body StateAnswerError = 5 //StateParseError - can not parse body or body not html (body not save) StateParseError = 6 //StateEncodingError - can not find or apply encoding (body not save) StateEncodingError = 7 //StateDublicate - dublicate see "Origin" field for origin URL StateDublicate = 8 //StateExternal - after redirect - host is external (body not save) StateExternal = 9 //StateNoFollow - found meta tag nofollow (body not save) StateNoFollow = 10 )
View Source
const (
// ErrScanArgument - Scan argument is not array of bytes
ErrScanArgument = "Compressed.Scan: argument is not array of bytes"
)
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Compressed ¶
type Compressed struct {
Data []byte
}
Compressed - field compressed by zlib
func (*Compressed) Compress ¶
func (c *Compressed) Compress() []byte
Compress - compress c.Data value
func (*Compressed) Scan ¶
func (c *Compressed) Scan(value interface{}) error
Scan - load data from DB to value
type Content ¶
type Content struct {
URL int64 `gorm:"type:integer REFERENCES url(id);unique_index;not null"`
Hash string `gorm:"size:64;not null"`
Body Compressed `gorm:"not null"`
Title string `gorm:"size:100;not null"`
}
Content - store page content Hash - hash of uncompressed content
type Host ¶
type Host struct {
ID int64 `gorm:"primary_key;not null"`
Name string `gorm:"size:255;unique_index;not null"`
RobotsStatusCode int `gorm:"not null"`
RobotsData []byte
}
Host - host information
type Meta ¶
type Meta struct {
URL int64 `gorm:"type:integer REFERENCES url(id);unique_index;not null"`
State State `gorm:"not null"`
Origin sql.NullInt64 `gorm:"type:integer REFERENCES url(id)"`
RedirectCnt int
StatusCode sql.NullInt64
}
Meta - meta information about processed URL Origin - link to origin document (for State == CtStateDublicate)
Source Files
Click to show internal directories.
Click to hide internal directories.