crawler

package
v0.0.0-...-dcbe3c4 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 28, 2024 License: MIT Imports: 22 Imported by: 0

Documentation

Overview

Package crawler implements web crawler.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type BBT

type BBT struct {
	// contains filtered or unexported fields
}

BBT database for basebit

func NewBBTDB

func NewBBTDB(ctx context.Context, addr, dbName, user, pwd, docusColName string) (b *BBT, err error)

NewBBTDB new bbt db

type Dao

type Dao struct {
	DB *BBT
}

func NewDao

func NewDao(ctx context.Context, addr, dbName, user, pwd, docusColName string) (*Dao, error)

func (*Dao) RemoveLegacy

func (d *Dao) RemoveLegacy(ctx context.Context, updateBefore time.Time) error

func (*Dao) Save

func (d *Dao) Save(ctx context.Context, title, text, url string) error

func (*Dao) Search

func (d *Dao) Search(ctx context.Context, text string) (rets []SearchResult, err error)

Search search text in title and content

type Docu

type Docu struct {
	CreatedAt time.Time `bson:"created_at" json:"created_at"`
	UpdatedAt time.Time `bson:"updated_at" json:"updated_at"`
	Text      string    `bson:"text" json:"text"`
	Title     string    `bson:"title" json:"title"`
	URL       string    `bson:"url" json:"url"`
}

Docu database for bbt document

type SearchResult

type SearchResult struct {
	Context string `bson:"context" json:"context"`
	URL     string `bson:"url" json:"url"`
	Title   string `bson:"title" json:"title"`

	ID   primitive.ObjectID `bson:"_id,omitempty" json:"-"`
	Text string             `bson:"text" json:"-"`
}

type Service

type Service struct {
	// contains filtered or unexported fields
}

func NewService

func NewService(ctx context.Context, addr, dbName, user, pwd, docusColName string) (*Service, error)

func (*Service) CrawlAllPages

func (s *Service) CrawlAllPages(ctx context.Context, sitemaps []string) error

func (*Service) Search

func (s *Service) Search(ctx context.Context, text string) (rets []SearchResult, err error)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL