Documentation ¶
Index ¶
- Constants
- Variables
- type Aggregations
- type Auth0Client
- type ESClientProvider
- type Enricher
- type Fetcher
- type HTTPClientProvider
- type Hits
- type LastDate
- type LoginResponse
- type Manager
- type NestedHits
- type Param
- type Params
- type Permissions
- type Repository
- type RepositoryEnrich
- type RepositoryRaw
- type RepositoryResponse
- type RepositorySearchFields
- type TopHits
- type Total
Constants ¶
const ( // APIURL dockerhub base url APIURL = "https://hub.docker.com" // APIVersion dockerhub API version APIVersion = "v2" // APILogin url APILogin = "users/login" // APIRepositories dockerhub repositories API APIRepositories = "repositories" // Category ... Category = "dockerhub-data" // Dockerhub - DS name Dockerhub string = "dockerhub" )
Variables ¶
var ( // DockerhubSearchFields - extra search fields DockerhubSearchFields = map[string][]string{ "name": {"name"}, "namespace": {"namespace"}, } // DockerhubRawMapping - Dockerhub raw index mapping DockerhubRawMapping = []byte(`{"mappings": {"dynamic":true,"properties":{"metadata__updated_on":{"type":"date"},"data":{"properties":{"description":{"type":"text","index":true},"full_description":{"type":"text","index":true}}}}}}`) // DockerhubRichMapping - Dockerhub rich index mapping DockerhubRichMapping = []byte(`{"mappings": {"properties":{"metadata__updated_on":{"type":"date"},"description":{"type":"text","index":true},"description_analyzed":{"type":"text","index":true},"full_description_analyzed":{"type":"text","index":true},"origin":{"type":"keyword"},"repository_type":{"type":"keyword"},"tag":{"type":"keyword"},"id":{"type":"keyword"},"metadata__backend_name":{"type":"keyword"},"user":{"type":"keyword"},"uuid":{"type":"keyword"},"project":{"type":"keyword"},"meta_title":{"type":"keyword"},"meta_type":{"type":"keyword"},"meta_state":{"type":"keyword"},"meta_program":{"type":"keyword"},"status":{"type":"keyword"}}}}`) )
Functions ¶
This section is empty.
Types ¶
type Aggregations ¶
type Aggregations struct {
LastDate LastDate `json:"last_date"`
}
Aggregations result
type ESClientProvider ¶
type ESClientProvider interface { Add(index string, documentID string, body []byte) ([]byte, error) CreateIndex(index string, body []byte) ([]byte, error) Bulk(body []byte) ([]byte, error) Get(index string, query map[string]interface{}, result interface{}) (err error) GetStat(index string, field string, aggType string, mustConditions []map[string]interface{}, mustNotConditions []map[string]interface{}) (result time.Time, err error) BulkInsert(data []elastic.BulkData) ([]byte, error) DelayOfCreateIndex(ex func(str string, b []byte) ([]byte, error), uin uint, du time.Duration, index string, data []byte) error BulkUpdate(data []elastic.BulkData) ([]byte, error) }
ESClientProvider used in connecting to ES Client server
type Enricher ¶
type Enricher struct { DSName string // Datasource will be used as key for ES ElasticSearchProvider ESClientProvider BackendVersion string }
Enricher contains dockerhub datasource enrich logic
func NewEnricher ¶
func NewEnricher(backendVersion string, esClientProvider ESClientProvider) *Enricher
NewEnricher initiates a new Enricher
func (*Enricher) EnrichItem ¶
func (e *Enricher) EnrichItem(rawItem RepositoryRaw, project string, now time.Time) (*RepositoryEnrich, error)
EnrichItem enriches raw item
func (*Enricher) GetFetchedDataItem ¶
func (e *Enricher) GetFetchedDataItem(repo *Repository, cmdLastDate *time.Time, lastDate *time.Time, noIncremental bool) (result *TopHits, err error)
GetFetchedDataItem gets fetched data items starting from lastDate
func (*Enricher) HandleMapping ¶
HandleMapping creates rich mapping
type Fetcher ¶
type Fetcher struct { DSName string // Datasource will be used as key for ES IncludeArchived bool MultiOrigin bool // can we store multiple endpoints in a single index? HTTPClientProvider HTTPClientProvider ElasticSearchProvider ESClientProvider Username string Password string Token string BackendVersion string }
Fetcher contains dockerhub datasource fetch logic
func NewFetcher ¶
func NewFetcher(params *Params, httpClientProvider HTTPClientProvider, esClientProvider ESClientProvider) *Fetcher
NewFetcher initiates a new dockerhub fetcher
func (*Fetcher) GetLastDate ¶
GetLastDate gets fetching lastDate
type HTTPClientProvider ¶
type HTTPClientProvider interface {
Request(url string, method string, header map[string]string, body []byte, params map[string]string) (statusCode int, resBody []byte, err error)
}
HTTPClientProvider used in connecting to remote http server
type Hits ¶
type Hits struct { Total Total `json:"total"` MaxScore float32 `json:"max_score"` Hits []NestedHits `json:"hits"` }
Hits result
type LoginResponse ¶
type LoginResponse struct {
Token string `json:"token"`
}
LoginResponse from login dockerhub web API
type Manager ¶
type Manager struct { Username string Password string FetcherBackendVersion string EnricherBackendVersion string EnrichOnly bool Enrich bool ESUrl string ESUsername string ESPassword string HTTPTimeout time.Duration Repositories []*Repository FromDate *time.Time NoIncremental bool AffAPI string ProjectSlug string AffBaseURL string ESCacheURL string ESCacheUsername string ESCachePassword string AuthGrantType string AuthClientID string AuthClientSecret string AuthAudience string Auth0URL string Environment string Slug string Retries uint Delay time.Duration GapURL string WebHookURL string }
Manager describes dockerhub manager
func NewManager ¶
NewManager initiates dockerhub manager instance
type NestedHits ¶
type NestedHits struct { Index string `json:"_index"` Type string `json:"_type"` ID string `json:"_id"` Score float64 `json:"_score"` Source *RepositoryRaw `json:"_source"` }
NestedHits result
type Param ¶
type Param struct { Username string Password string EndPoint string FetcherBackendVersion string EnricherBackendVersion string Fetch bool Enrich bool ESUrl string EsUser string EsPassword string EsIndex string FromDate *time.Time Project string Retries uint Delay time.Duration GapURL string AffAPI string ProjectSlug string AffBaseURL string ESCacheURL string ESCacheUsername string ESCachePassword string AuthGrantType string AuthClientID string AuthClientSecret string AuthAudience string Auth0URL string Environment string Slug string EnrichOnly bool HTTPTimeout time.Duration Repositories []*Repository NoIncremental bool SlackWebHookURL string }
Param required for creating a new instance of Bugzilla manager
type Permissions ¶
type Permissions struct { Read bool `json:"read"` Write bool `json:"write"` Admin bool `json:"admin"` }
Permissions response
type Repository ¶
Repository represents dockerhub repository data
type RepositoryEnrich ¶
type RepositoryEnrich struct { ID string `json:"id"` Project string `json:"project"` Affiliation string `json:"affiliation"` Description string `json:"description"` IsPrivate bool `json:"is_private"` IsAutomated bool `json:"is_automated"` PullCount int `json:"pull_count"` RepositoryType string `json:"repository_type"` User string `json:"user"` Status int `json:"status"` StarCount int `json:"star_count"` IsEvent int `json:"is_event"` IsDockerImage int `json:"is_docker_image"` DescriptionAnalyzed string `json:"description_analyzed"` FullDescriptionAnalyzed string `json:"full_description_analyzed"` CreationDate time.Time `json:"creation_date"` IsDockerhubDockerhub int `json:"is_dockerhub_dockerhub"` RepositoryLabels *[]string `json:"repository_labels"` MetadataFilterRaw *string `json:"metadata__filter_raw"` LastUpdated time.Time `json:"last_updated"` Offset *string `json:"offset"` MetadataEnrichedOn time.Time `json:"metadata__enriched_on"` BackendVersion string `json:"backend_version"` Tag string `json:"tag"` UUID string `json:"uuid"` Origin string `json:"origin"` MetadataUpdatedOn time.Time `json:"metadata__updated_on"` MetadataBackendName string `json:"metadata__backend_name"` MetadataTimestamp time.Time `json:"metadata__timestamp"` BuildOnCloud *string `json:"build_on_cloud"` ProjectTS int64 `json:"project_ts"` }
RepositoryEnrich represents dockerhub repository enriched model
type RepositoryRaw ¶
type RepositoryRaw struct { BackendVersion string `json:"backend_version"` Data *RepositoryResponse `json:"data"` Tag string `json:"tag"` UUID string `json:"uuid"` SearchFields *RepositorySearchFields `json:"search_fields"` Origin string `json:"origin"` UpdatedOn float64 `json:"updated_on"` MetadataUpdatedOn time.Time `json:"metadata__updated_on"` BackendName string `json:"backend_name"` MetadataTimestamp time.Time `json:"metadata__timestamp"` Timestamp float64 `json:"timestamp"` Category string `json:"category"` ClassifiedFieldsFiltered *string `json:"classified_fields_filtered"` }
RepositoryRaw represents dockerhub repository raw model
type RepositoryResponse ¶
type RepositoryResponse struct { User string `json:"user"` Name string `json:"name"` Namespace string `json:"namespace"` RepositoryType string `json:"repository_type"` Status *int `json:"status"` Description string `json:"description"` IsPrivate *bool `json:"is_private"` IsAutomated bool `json:"is_automated"` CanEdit bool `json:"can_edit"` StarCount *int `json:"star_count"` PullCount *int `json:"pull_count"` LastUpdated time.Time `json:"last_updated"` IsMigrated bool `json:"is_migrated"` HasStarred bool `json:"has_starred"` FullDescription string `json:"full_description"` Affiliation string `json:"affiliation"` Permissions Permissions `json:"permissions"` FetchedOn float64 `json:"fetched_on"` }
RepositoryResponse data model represents dockerhub get repository results
type RepositorySearchFields ¶
type RepositorySearchFields struct { Name string `json:"name"` ItemID string `json:"item_id"` Namespace string `json:"namespace"` }
RepositorySearchFields ...
type TopHits ¶
type TopHits struct { Took int `json:"took"` Hits Hits `json:"hits"` Aggregations Aggregations `json:"aggregations"` }
TopHits result