Documentation
¶
Overview ¶
Copyright 2021 Wei (Sam) Wang <sam.wang.0723@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
Copyright 2021 Wei (Sam) Wang <sam.wang.0723@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
Copyright 2021 Wei (Sam) Wang <sam.wang.0723@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
Copyright 2021 Wei (Sam) Wang <sam.wang.0723@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
Copyright 2021 Wei (Sam) Wang <sam.wang.0723@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
Index ¶
Constants ¶
This section is empty.
Variables ¶
var ( ErrWorkerCountInvalid = errors.New("invalid value for fetch workers") ErrRateLimitIntervalInvalid = errors.New("invalid value for rate limit interval") )
Functions ¶
This section is empty.
Types ¶
type CrawlerConfig ¶
type CrawlerConfig struct {
// An API for performing HTTP requests. If not specified,
// crawler.DefaultHttpClient will be used instead.
URLGetter crawler.URLGetter
// The number of concurrent workers used for retrieving links.
FetchWorkers int
// The time between subsequent crawler passes.
RateLimitInterval int64
// Proxy for preventing remote site's rate limiting
Proxy *crawler.Proxy
// The logger to use. If not defined an output-discarding logger will
// be used instead.
Logger *zerolog.Logger
}
Config encapsulates the settings for configuring the web-crawler service.
type CronjobConfig ¶
type CronjobConfig struct {
// The logger to use. If not defined an output-discarding logger will
// be used instead.
Logger *zerolog.Logger
}
Config encapsulates the settings for configuring the redis service.
type IService ¶
type IService interface {
StartCron()
StopCron()
AddJob(ctx context.Context, spec string, job func()) error
DailyCloseThroughKafka(ctx context.Context, objs *[]interface{}) error
StockThroughKafka(ctx context.Context, objs *[]interface{}) error
ThreePrimaryThroughKafka(ctx context.Context, objs *[]interface{}) error
StakeConcentrationThroughKafka(ctx context.Context, objs *[]interface{}) error
ObtainLock(ctx context.Context, key string, expire time.Duration) *redislock.Lock
StopRedis() error
StopKafka() error
ListCrawlingConcentrationURLs(ctx context.Context, date string) ([]string, error)
Crawl(ctx context.Context, linkIt graph.LinkIterator, interceptChan ...chan convert.InterceptData) (int, error)
IsHoliday(ctx context.Context, date string) bool
ListeningDownloadRequest(ctx context.Context, downloadChan chan *dto.StartCronjobRequest)
}
type KafkaConfig ¶
type KafkaConfig struct {
// Kafka controller DNS hostname
Controller string
GroupID string
Brokers []string
Topics []string
// The logger to use. If not defined an output-discarding logger will
// be used instead.
Logger *zerolog.Logger
}
Config encapsulates the settings for configuring the kafka service.
type Option ¶
type Option func(o *serviceImpl)
func WithCrawler ¶
func WithCrawler(cfg CrawlerConfig) Option
func WithCronJob ¶
func WithCronJob(cfg CronjobConfig) Option
func WithKafka ¶
func WithKafka(cfg KafkaConfig) Option
func WithRedis ¶
func WithRedis(cfg RedisConfig) Option
type RedisConfig ¶
type RedisConfig struct {
// Redis master node DNS hostname
Master string
// Redis sentinel addresses
SentinelAddrs []string
// The logger to use. If not defined an output-discarding logger will
// be used instead.
Logger *zerolog.Logger
Password string
}
Config encapsulates the settings for configuring the redis service.
Source Files