database

package

v0.0.0-...-3b54a73 Latest Latest Go to latest Published: Jul 23, 2015 License: MIT Imports: 9 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/hokutosei/news_crawlers

Links

Open Source Insights

Documentation ¶

Index ¶

Variables
func ExtractIDsFromResult(results ...TopNewsRankerResult) []string
func GetMongodbCluster(host chan string)
func GetRedisHost(host chan string)
func GoogleNewsFindIfExist(URL string, sc *mongodb.Session) bool
func GoogleNewsInsert(hn GoogleNews, URL string, wg *sync.WaitGroup) bool
func HackerNewsFindIfExist(title string, sc *mongodb.Session) bool
func HackerNewsInsert(hn HackerNews, hnTitle string, wg *sync.WaitGroup)
func IncrementNewsScore(paramsID string)
func MongodbStart()
func NewPool(server string) *redis.Pool
func NewsItemPage(paramsID string) (interface{}, error)
func SessionCopy() *mongodb.Session
func StartRedis()
func TopNewsRanker(from time.Duration, to time.Duration) []string
type AggregatedNews
- func GetCategorizedNews(initial string) (AggregatedNews, error)
- func GetterNewsMainTopScore() (AggregatedNews, error)
- func GoogleNewsIndexNews() (AggregatedNews, error)
- func HackerNewsFeedMore(contentType string, length int) (AggregatedNews, error)
- func HackerNewsIndexNews() (AggregatedNews, error)
- func HackerNewsLatestNews() (AggregatedNews, error)
- func NewsMainIndexNews() (AggregatedNews, error)
type GoogleNews
type HackerNews
type NewsIds
- func NewsIndexNewsIDS() ([]NewsIds, error)
type TopNewsRankerResult

Constants ¶

This section is empty.

Variables ¶

View Source

var (
	// MongodbSession main mongodb connection session
	MongodbSession *mongodb.Session

	// Db database name
	Db = "news_aggregator"
)

View Source

var (

	// NewsMainCollection collection name
	NewsMainCollection = "news_main"
)

View Source

var (
	// RedisPool main redis pool connection
	RedisPool *redis.Pool
)

Functions ¶

func ExtractIDsFromResult ¶

func ExtractIDsFromResult(results ...TopNewsRankerResult) []string

ExtractIDsFromResult utils from extracting IDS from result

func GetMongodbCluster ¶

func GetMongodbCluster(host chan string)

GetMongodbCluster retrieve mongodb cluster node from etcd

func GetRedisHost ¶

func GetRedisHost(host chan string)

GetRedisHost get redis host from etcd

func GoogleNewsFindIfExist ¶

func GoogleNewsFindIfExist(URL string, sc *mongodb.Session) bool

GoogleNewsFindIfExist check google news current data if exist before insert

func GoogleNewsInsert ¶

func GoogleNewsInsert(hn GoogleNews, URL string, wg *sync.WaitGroup) bool

GoogleNewsInsert insert data for google news

func HackerNewsFindIfExist ¶

func HackerNewsFindIfExist(title string, sc *mongodb.Session) bool

HackerNewsFindIfExist check if data exists already before saving

func HackerNewsInsert ¶

func HackerNewsInsert(hn HackerNews, hnTitle string, wg *sync.WaitGroup)

HackerNewsInsert insert data to mongodb

func IncrementNewsScore ¶

func IncrementNewsScore(paramsID string)

IncrementNewsScore increment news score increment news ite page view

func MongodbStart ¶

func MongodbStart()

MongodbStart start connecting to mongodb

func NewPool ¶

func NewPool(server string) *redis.Pool

NewPool create redis pool servers

func NewsItemPage ¶

func NewsItemPage(paramsID string) (interface{}, error)

NewsItemPage get news item data

func SessionCopy ¶

func SessionCopy() *mongodb.Session

SessionCopy make copy of a mongodb session

func StartRedis ¶

func StartRedis()

StartRedis start connecting to redis

func TopNewsRanker ¶

func TopNewsRanker(from time.Duration, to time.Duration) []string

TopNewsRanker main news top ranker

Types ¶

type AggregatedNews ¶

type AggregatedNews []interface{}

AggregatedNews interface struct for AggregatedNews

func GetCategorizedNews ¶

func GetCategorizedNews(initial string) (AggregatedNews, error)

GetCategorizedNews will get news with category news initials

func GetterNewsMainTopScore ¶

func GetterNewsMainTopScore() (AggregatedNews, error)

GetterNewsMainTopScore main top page news getter

func GoogleNewsIndexNews ¶

func GoogleNewsIndexNews() (AggregatedNews, error)

GoogleNewsIndexNews aggregated news list for google news

func HackerNewsFeedMore ¶

func HackerNewsFeedMore(contentType string, length int) (AggregatedNews, error)

HackerNewsFeedMore feed more data on index list

func HackerNewsIndexNews ¶

func HackerNewsIndexNews() (AggregatedNews, error)

HackerNewsIndexNews index page data

func HackerNewsLatestNews ¶

func HackerNewsLatestNews() (AggregatedNews, error)

HackerNewsLatestNews aggregate latest news

func NewsMainIndexNews ¶

func NewsMainIndexNews() (AggregatedNews, error)

NewsMainIndexNews responder for index news query

type GoogleNews ¶

type GoogleNews interface{}

GoogleNews interface for google news

type HackerNews ¶

type HackerNews interface{}

HackerNews interface for hn news item

type NewsIds ¶

type NewsIds struct {
	ID bson.ObjectId `bson:"_id"`
}

NewsIds struct for result IDS only

func NewsIndexNewsIDS ¶

func NewsIndexNewsIDS() ([]NewsIds, error)

NewsIndexNewsIDS retrieve all news index ids and save to cache

type TopNewsRankerResult ¶

type TopNewsRankerResult struct {
	ID    string `bson:"_id"`
	Total int
	Items []map[string]bson.ObjectId
}

TopNewsRankerResult data struct from mongodb aggregation result { "_id" : null, "total" : 326554 }

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL