grab

package module

v0.0.0-...-8dbf7a7 Latest Latest Go to latest Published: Aug 6, 2021 License: MIT Imports: 5 Imported by: 2

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/GanEasy/grab

Links

Open Source Insights

README ¶

grab

非结构式数据抓取服务

core

核心代码

Documentation ¶

Index ¶

Variables
func DecodeURL(str string) (string, error)
func EncodeURL(str string) string
func ExplainLink(url string) (address, drive, page string)
func GetGuide(drive string) reader.Guide
func GetGuidex(drive string) reader.Guide
func GetReader(drive string) reader.Reader
func GetReaderx(drive string) reader.Reader
type Card
type Catalog
type DemoItem
type Item
type Link

Constants ¶

This section is empty.

Variables ¶

View Source

var GuideList = map[string]reader.Guide{
	"qidian":      &reader.QidianReader{},
	"zongheng":    &reader.ZonghengReader{},
	"17k":         &reader.SeventeenKReader{},
	"xxsy":        &reader.XxsyReader{},
	"baidu":       &reader.BaiduReader{},
	"hongxiu":     &reader.HongxiuReader{},
	"luoqiu":      &reader.MLuoqiuReader{},
	"booktxt":     &reader.BooktxtReader{},
	"paoshu8":     &reader.Paoshu8Reader{},
	"qkshu6":      &reader.Qkshu6Reader{},
	"shuge":       &reader.ShugeReader{},
	"qu":          &reader.QuReader{},
	"jx":          &reader.JxReader{},
	"uxiaoshuo":   &reader.UxiaoshuoReader{},
	"soe8":        &reader.Soe8Reader{},
	"bxks":        &reader.BxksReader{},
	"xin18":       &reader.Xin18Reader{},
	"bxwx":        &reader.BxwxReader{},
	"biqugeinfo":  &reader.BiqugeinfoReader{},
	"mcmssc":      &reader.McmsscReader{},
	"xs280":       &reader.Xs280Reader{},
	"xbiquge":     &reader.XbiqugeReader{},
	"biquyun":     &reader.BiquyunReader{},
	"r2hm":        &reader.R2hmReader{},
	"manwuyu":     &reader.ManwuyuReader{},
	"manhwa":      &reader.ManhwaReader{},
	"aimeizi5":    &reader.Aimeizi5Reader{},
	"kanmeizi":    &reader.KanmeiziReader{},
	"fuman":       &reader.FumanReader{},
	"weijiaoshou": &reader.WeijiaoshouReader{},
	"haimaoba":    &reader.HaimaobaReader{},
	"ssmh":        &reader.SsmhReader{},
	"hanmanku":    &reader.HanmankuReader{},
	"hanmanwo":    &reader.HanmanwoReader{},
	"laosijixs":   &reader.LaosijixsReader{},
}

GuideList 阅读器匹配组

View Source

var ReaderList = map[string]reader.Reader{
	"qidian":      &reader.QidianReader{},
	"zongheng":    &reader.ZonghengReader{},
	"17k":         &reader.SeventeenKReader{},
	"xxsy":        &reader.XxsyReader{},
	"hongxiu":     &reader.HongxiuReader{},
	"luoqiu":      &reader.MLuoqiuReader{},
	"booktxt":     &reader.BooktxtReader{},
	"booktxtnet":  &reader.BooktxtnetReader{},
	"paoshu8":     &reader.Paoshu8Reader{},
	"qkshu6":      &reader.Qkshu6Reader{},
	"shuge":       &reader.ShugeReader{},
	"qu":          &reader.QuReader{},
	"jx":          &reader.JxReader{},
	"uxiaoshuo":   &reader.UxiaoshuoReader{},
	"soe8":        &reader.Soe8Reader{},
	"bxks":        &reader.BxksReader{},
	"bxwx":        &reader.BxwxReader{},
	"xin18":       &reader.Xin18Reader{},
	"biqugeinfo":  &reader.BiqugeinfoReader{},
	"mcmssc":      &reader.McmsscReader{},
	"xs280":       &reader.Xs280Reader{},
	"xbiquge":     &reader.XbiqugeReader{},
	"biquyun":     &reader.BiquyunReader{},
	"book":        &reader.BookReader{},
	"article":     &reader.ArticleReader{},
	"rss":         &reader.RssReader{},
	"blog":        &reader.BlogReader{},
	"learnku":     &reader.LearnkuReader{},
	"github":      &reader.GithubReader{},
	"r2hm":        &reader.R2hmReader{},
	"manwuyu":     &reader.ManwuyuReader{},
	"manhwa":      &reader.ManhwaReader{},
	"aimeizi5":    &reader.Aimeizi5Reader{},
	"kanmeizi":    &reader.KanmeiziReader{},
	"fuman":       &reader.FumanReader{},
	"weijiaoshou": &reader.WeijiaoshouReader{},
	"haimaoba":    &reader.HaimaobaReader{},
	"ssmh":        &reader.SsmhReader{},
	"hanmanku":    &reader.HanmankuReader{},
	"hanmanwo":    &reader.HanmanwoReader{},
	"laosijixs":   &reader.LaosijixsReader{},
}

ReaderList 阅读器匹配组

Functions ¶

func DecodeURL ¶

func DecodeURL(str string) (string, error)

DecodeURL 把url decode

func EncodeURL ¶

func EncodeURL(str string) string

EncodeURL 把url encode

func ExplainLink ¶

func ExplainLink(url string) (address, drive, page string)

ExplainLink 解释链接地址

func GetGuide ¶

func GetGuide(drive string) reader.Guide

GetGuide 获取引导

func GetGuidex ¶

func GetGuidex(drive string) reader.Guide

GetGuidex 获取引导

func GetReader ¶

func GetReader(drive string) reader.Reader

GetReader 获取阅读器

func GetReaderx ¶

func GetReaderx(drive string) reader.Reader

GetReaderx 获取阅读器

Types ¶

type Card ¶

type Card struct {
	Title  string   `json:"title"`  // 标题
	WxTo   string   `json:"wxto"`   // 小程序跳转到目标页
	Intro  string   `json:"intro"`  //介绍
	Type   string   `json:"type"`   // card展示形式 media card text image images link
	Cover  string   `json:"cover"`  // 封面图片
	Images []string `json:"images"` // 图片组效果时图片列表
}

Card 使用卡片代替链接

type Catalog ¶

type Catalog struct {
	Title     string `json:"title"`
	Cards     []Card //`json:"links"`
	SourceURL string `json:"source_url"` // 数据抓取时，统一声明数据来源
	Hash      string `json:"hash"`       // 当前目录 Hash
	Previous  Link   `json:"previous"`   // 如果有上一页
	Next      Link   `json:"next"`       // 如果有下一页
}

* * base.go 放一些基础数据结构类,用于制定内外数据结构

Catalog 第三方资源目录(用户可任意订阅此目录)

type DemoItem ¶

type DemoItem struct {
	Title    string `json:"title"`
	URL      string `json:"url"`
	Category string `json:"category"`
}

DemoItem 示例详细

func ArticleDemoList ¶

func ArticleDemoList() []DemoItem

ArticleDemoList 文章示例列表

func BookDemoList ¶

func BookDemoList() []DemoItem

BookDemoList 小说示例列表

func RssDemoList ¶

func RssDemoList() []DemoItem

RssDemoList Rss示例列表

type Item ¶

type Item struct {
	Title string `json:"title"`
	WxTo  string `json:"wxto"`
	Intro string `json:"intro"`
	Type  string `json:"type"`
}

Item 小程序授受参数明细

func GetAbouts ¶

func GetAbouts() []Item

GetAbouts 获得关于我们的介绍

func GetBooks ¶

func GetBooks(url string) []Item

GetBooks 小说目录列表

func GetChapter ¶

func GetChapter(url string) []Item

GetChapter 小说章节详细

func GetChapters ¶

func GetChapters(url string) []Item

GetChapters 小说章节列表

func GetClassify ¶

func GetClassify() []Item

GetClassify 发现分类

func GetDrives ¶

func GetDrives() []Item

GetDrives 获取所有解释引擎

func GetHelps ¶

func GetHelps() []Item

GetHelps 获取帮助(常见问题)

func GetRanks ¶

func GetRanks() []Item

GetRanks 各类榜单

func GetResource ¶

func GetResource(url string) []Item

GetResource 获得资源详细(分类)

func GetResources ¶

func GetResources() []Item

GetResources 自定义资源列表(支持平台目录)

func GetTopics ¶

func GetTopics() []Item

GetTopics 获取专题列表

func GetWaitExamineClassify ¶

func GetWaitExamineClassify() []Item

GetWaitExamineClassify 获取做给审核检查的class

type Link ¶

type Link struct {
	// Basic
	Title string `json:"title"`
	URL   string `json:"url"`
	WxTo  string `json:"wxto"` // 小程序跳转到目标页
}

Link 链接

Source Files ¶

View all Source files

Directories ¶

Path	Synopsis
api
core
db
reader
serve
test
web Command text is a chromedp example demonstrating how to extract text from a specific element.	Command text is a chromedp example demonstrating how to extract text from a specific element.

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL