bugzilla

package
v0.0.0-...-b175f30 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 23, 2022 License: Apache-2.0 Imports: 19 Imported by: 0

README

Bugzilla Datasource

Bugzilla datasource is a package to fetch data from dockerhub API and save it into Elasticsearch and Enrich saved data.

Bugzilla Running instructions

To run Bugzilla datasource from dads you must set proper environment variables to select Bugzilla as an engine and other parameters that determine the intended behavior.

These are the needed environment variables to run Bugzilla DA-DS:

  • DA_BUGZILLA_ENDPOINT={}
    • bugzilla origin url
  • DA_BUGZILLA_AFFILIATION_CONN_STRING={}
    • Affiliation database connection string
  • DA_BUGZILLA_FETCHER_BACKEND_VERSION={}
    • Fetcher version
  • DA_BUGZILLA_ENRICHER_BACKEND_VERSION={}
    • Enricher version
  • DA_BUGZILLA_FETCH={1,0}
    • To decide whether will fetch raw data or not.
  • DA_BUGZILLA_ENRICH={1,0}
    • To decide whether will do enrich raw data or not.
  • DA_BUGZILLA_ES_URL=''
    • Elastic search url.
  • DA_BUGZILLA_ES_USERNAME=''
    • Elastic search credentials
  • DA_BUGZILLA_ES_PASSWORD=''
    • Elastic search credentials
  • DA_BUGZILLA_ES_INDEX=''
    • Elastic search index name .
  • DA_BUGZILLA_FROM_DATE=''
    • Optional, date to start syncing from.
  • DA_BUGZILLA_PROJECT=''
    • Slug name of a project e.g. yocto.
  • DA_BUGZILLA_FETCH_SIZE=25
    • total number of fetched items per request.
  • DA_BUGZILLA_ENRICH_SIZE=25
    • total number of enriched items per request

Documentation

Index

Constants

View Source 
const (
	// Category type
	Category = "bug"
	// Bugzilla - DS name
	Bugzilla string = "bugzilla"
)

Variables

View Source 
var (

	// BugzillaRawMapping - bugzilla raw index mapping
	BugzillaRawMapping = []byte(`{"mappings": 
{"dynamic":true,
"properties":{
"metadata__updated_on":{"type":"date"},
"metadata__timestamp":{"type":"date"},
"creation_ts":{"type":"date"},
"changed_at":{"type":"date"},
"delta_ts":{"type":"date"},
"short_description":{"type":"text","index":true},
"backend_version":{"type":"keyword"},
"backend_name":{"type":"keyword"},
"bug_status":{"type":"keyword"},
  "priority":{"type":"keyword"},
  "severity":{"type":"keyword"}
}}}`)

	// BugzillaEnrichMapping - bugzilla enriched index mapping
	BugzillaEnrichMapping = []byte(`{"mappings":{"dynamic_templates":[{"notanalyzed":{"match":"*","match_mapping_type":"string","mapping":{"type":"keyword"}}},{"int_to_float":{"match":"*","match_mapping_type":"long","mapping":{"type":"float"}}},{"formatdate":{"match":"*","match_mapping_type":"date","mapping":{"format":"strict_date_optional_time||epoch_millis","type":"date"}}}]}}`)
)

Functions

This section is empty.

Types

type Activity 

type Activity struct {
	Who     string
	When    string
	What    string
	Removed string
	Added   string
}

Activity ...

func GetActivityLen 

func GetActivityLen(query string, body []byte) (int, []Activity, error)

GetActivityLen gets count of searched items

type AffiliationClient 

type AffiliationClient interface {
	GetIdentityByUser(key string, value string) (*libAffiliations.AffIdentity, error)
	AddIdentity(identity *libAffiliations.Identity) bool
}

AffiliationClient manages user identity

type AssignedTo 

type AssignedTo struct {
	Value string `xml:",chardata"`
	Name  string `xml:"name,attr"`
}

AssignedTo get assigned to xml field tag name and value

type AssigneeResponse 

type AssigneeResponse struct {
	Name  string `json:"name"`
	Email string `json:"email"`
}

AssigneeResponse represents Assignee response model

type Auth0Client 

type Auth0Client interface {
	GetToken() (string, error)
}

Auth0Client ...

type BugDetailResponse 

type BugDetailResponse struct {
	Bug BugDetailXML `xml:"bug"`
}

BugDetailResponse data model represents Bugzilla get bugDetail results

type BugDetailXML 

type BugDetailXML struct {
	ID               int        `xml:"bug_id"`
	CreationTS       string     `xml:"creation_ts"`
	DeltaTS          string     `xml:"delta_ts"`
	Priority         string     `xml:"priority"`
	Severity         string     `xml:"bug_severity"`
	OpSys            string     `xml:"op_sys"`
	RepPlatform      string     `xml:"rep_platform"`
	Keywords         []string   `xml:"keywords"`
	StatusWhiteboard string     `xml:"status_whiteboard"`
	Resolution       string     `xml:"resolution"`
	Reporter         AssignedTo `xml:"reporter"`
	AssignedTo       AssignedTo `xml:"assigned_to"`
	Summary          string     `xml:"summary"`
	LongDesc         []Comment  `xml:"long_desc"`
}

BugDetailXML ...

type BugEnrich 

type BugEnrich struct {
	UUID           string    `json:"uuid"`
	Project        string    `json:"project"`
	Labels         []string  `json:"labels"`
	Changes        int       `json:"changes"`
	Priority       string    `json:"priority"`
	Severity       string    `json:"severity"`
	OpSys          string    `json:"op_sys"`
	ChangedAt      string    `json:"changed_at"`
	Product        string    `json:"product"`
	Component      string    `json:"component"`
	Platform       string    `json:"platform"`
	BugID          int       `json:"bug_id"`
	Status         string    `json:"status"`
	TimeOpenDays   float64   `json:"timeopen_days"`
	Category       string    `json:"category"`
	ChangedDate    time.Time `json:"changed_date"`
	Tag            string    `json:"tag"`
	IsBugzillaBug  int       `json:"is_bugzilla_bug"`
	URL            string    `json:"url"`
	ResolutionDays float64   `json:"resolution_days"`
	CreationDate   time.Time `json:"creation_date"`
	DeltaTs        time.Time `json:"delta_ts"`
	Whiteboard     string    `json:"whiteboard"`
	Resolution     string    `json:"resolution"`
	Assigned       string    `json:"assigned"`

	ReporterID           string   `json:"reporter_id"`
	ReporterUUID         string   `json:"reporter_uuid"`
	ReporterName         string   `json:"reporter_name"`
	ReporterUserName     string   `json:"reporter_user_name"`
	ReporterDomain       string   `json:"reporter_domain"`
	ReporterOrgName      string   `json:"reporter_org_name"`
	ReporterMultiOrgName []string `json:"reporter_multi_org_names"`
	ReporterBot          bool     `json:"reporter_bot"`

	AuthorID           string   `json:"author_id"`
	AuthorUUID         string   `json:"author_uuid"`
	AuthorName         string   `json:"author_name"`
	AuthorUserName     string   `json:"author_user_name"`
	AuthorDomain       string   `json:"author_domain"`
	AuthorOrgName      string   `json:"author_org_name"`
	AuthorMultiOrgName []string `json:"author_multi_org_names"`
	AuthorBot          bool     `json:"author_bot"`

	AssignedToID           string   `json:"assigned_to_id"`
	AssignedToUUID         string   `json:"assigned_to_uuid"`
	AssignedToName         string   `json:"assigned_to_name"`
	AssignedToUserName     string   `json:"assigned_to_user_name"`
	AssignedToDomain       string   `json:"assigned_to_domain"`
	AssignedToOrgName      string   `json:"assigned_to_org_name"`
	AssignedToMultiOrgName []string `json:"assigned_to_multi_org_names"`
	AssignedToBot          bool     `json:"assigned_to_bot"`

	MainDescription         string    `json:"main_description"`
	MainDescriptionAnalyzed string    `json:"main_description_analyzed"`
	Summary                 string    `json:"summary"`
	SummaryAnalyzed         string    `json:"summary_analyzed"`
	Comments                int       `json:"comments"`
	RepositoryLabels        *[]string `json:"repository_labels"`

	MetadataUpdatedOn      time.Time `json:"metadata__updated_on"`
	MetadataTimestamp      time.Time `json:"metadata__timestamp"`
	MetadataEnrichedOn     time.Time `json:"metadata__enriched_on"`
	MetadataFilterRaw      *string   `json:"metadata__filter_raw"`
	MetadataBackendName    string    `json:"metadata__backend_name"`
	MetadataBackendVersion string    `json:"metadata__backend_version"`
	TimeToClose            float64   `json:"time_to_close"`
}

BugEnrich ...

type BugRaw 

type BugRaw struct {
	BackendVersion    string `json:"backend_version"`
	BackendName       string `json:"backend_name"`
	UUID              string `json:"uuid"`
	BugID             int    `json:"bug_id"`
	Origin            string `json:"origin"`
	Tag               string `json:"tag"`
	Product           string `json:"product"`
	Component         string `json:"component"`
	Assignee          Person
	ShortDescription  string        `json:"short_description"`
	LongDesc          []Comment     `json:"long_desc"`
	BugStatus         string        `json:"bug_status"`
	MetadataUpdatedOn time.Time     `json:"metadata__updated_on"`
	MetadataTimestamp time.Time     `json:"metadata__timestamp"`
	Timestamp         float64       `json:"timestamp"`
	Category          string        `json:"category"`
	CreationTS        time.Time     `json:"creation_ts"`
	Priority          string        `json:"priority"`
	Severity          string        `json:"severity"`
	OpSys             string        `json:"op_sys"`
	ChangedAt         time.Time     `json:"changed_at"`
	ActivityCount     int           `json:"activity_count"`
	SearchFields      *SearchFields `json:"search_fields"`
	DeltaTs           time.Time     `json:"delta_ts"`
	Keywords          []string      `json:"keywords"`
	RepPlatform       string        `json:"rep_platform"`
	StatusWhiteboard  string        `json:"status_whiteboard"`
	Resolution        string        `json:"resolution"`
	Reporter          Person        `json:"reporter"`
	AssignedTo        string        `json:"assigned_to"`
	Summary           string        `json:"summary"`
	Activities        []Activity    `json:"activities"`
}

BugRaw data model represents es schema

type BugResponse 

type BugResponse struct {
	ID               int               `json:"id"`
	Product          string            `json:"product"`
	Component        string            `json:"component"`
	AssignedTo       *AssigneeResponse `json:"assigned_to"`
	ShortDescription string            `json:"short_description"`
	CreationTS       time.Time         `json:"creation_ts"`
	Priority         string            `json:"priority"`
	BugStatus        string            `json:"bug_status"`
	ChangedAt        string            `json:"changed_at"`
}

BugResponse data model represents Bugzilla get bugsList results

type Comment 

type Comment struct {
	Commentid int    `xml:"commentid"`
	Who       string `xml:"who"`
	BugWhen   string `xml:"bug_when"`
	Thetext   string `xml:"thetext"`
}

Comment of a bug

type ESClientProvider 

type ESClientProvider interface {
	Add(index string, documentID string, body []byte) ([]byte, error)
	CreateIndex(index string, body []byte) ([]byte, error)
	Bulk(body []byte) ([]byte, error)
	Get(index string, query map[string]interface{}, result interface{}) (err error)
	GetStat(index string, field string, aggType string, mustConditions []map[string]interface{}, mustNotConditions []map[string]interface{}) (result time.Time, err error)
	BulkInsert(data []elastic.BulkData) ([]byte, error)
	DelayOfCreateIndex(ex func(str string, b []byte) ([]byte, error), uin uint, du time.Duration, index string, data []byte) error
}

ESClientProvider used in connecting to ES server

type Enricher 

type Enricher struct {
	DSName         string
	BackendVersion string
	Project        string
	// contains filtered or unexported fields
}

Enricher enrich Bugzilla raw

func NewEnricher 

func NewEnricher(backendVersion string, project string, affiliationsClient AffiliationClient, auth0Client Auth0Client, httpClientProvider HTTPClientProvider, affBaseURL string, projectSlug string) *Enricher

NewEnricher intiate a new enricher instance

func (*Enricher) EnrichItem 

func (e *Enricher) EnrichItem(rawItem BugRaw, now time.Time) (*BugEnrich, error)

EnrichItem enrich Bugzilla raw item

type Fetcher 

type Fetcher struct {
	DSName                string // Datasource will be used as key for ES
	HTTPClientProvider    HTTPClientProvider
	ElasticSearchProvider ESClientProvider
	BackendVersion        string
	Endpoint              string
}

Fetcher contains Bugzilla fetch logic

func NewFetcher 

func NewFetcher(params *Params, httpClientProvider HTTPClientProvider, esClientProvider ESClientProvider) *Fetcher

NewFetcher initiates a new bugZilla fetcher

func (*Fetcher) FetchItem 

func (f *Fetcher) FetchItem(fromDate time.Time, limit int, now time.Time) ([]*BugRaw, error)

FetchItem fetches bugs and save it into ES

func (*Fetcher) HandleMapping 

func (f *Fetcher) HandleMapping(index string) error

HandleMapping updates bugzilla raw mapping

func (*Fetcher) Query 

func (f *Fetcher) Query(index string, query map[string]interface{}) (*RawHits, error)

Query query saved raw data from ES

type HTTPClientProvider 

type HTTPClientProvider interface {
	Request(url string, method string, header map[string]string, body []byte, params map[string]string) (statusCode int, resBody []byte, err error)
	RequestCSV(url string) ([][]string, error)
}

HTTPClientProvider used in connecting to remote http server

type HitSource 

type HitSource struct {
	ID        string    `json:"id"`
	ChangedAt time.Time `json:"changed_at"`
}

HitSource is the document _source data

type Hits 

type Hits struct {
	Hits []NestedHits `json:"hits"`
}

Hits result

type Manager 

type Manager struct {
	Endpoint               string
	FetcherBackendVersion  string
	EnricherBackendVersion string
	Fetch                  bool
	Enrich                 bool
	ESUrl                  string
	ESUsername             string
	ESPassword             string
	ESIndex                string
	FromDate               *time.Time
	HTTPTimeout            time.Duration
	Project                string
	FetchSize              int
	EnrichSize             int
	Retries                uint
	Delay                  time.Duration
	GapURL                 string
	AffAPI                 string
	ProjectSlug            string
	AffBaseURL             string
	ESCacheURL             string
	ESCacheUsername        string
	ESCachePassword        string
	AuthGrantType          string
	AuthClientID           string
	AuthClientSecret       string
	AuthAudience           string
	Auth0URL               string
	Environment            string
	Slug                   string
	WebHookURL             string

	Auth0 Auth0Client
	// contains filtered or unexported fields
}

Manager describes bugzilla manager

func NewManager 

func NewManager(param Param) (*Manager, error)

NewManager initiates bugzilla manager instance

func (*Manager) Sync 

func (m *Manager) Sync() error

Sync starts fetch and enrich processes

type NHits 

type NHits struct {
	Hits []NestedRawHits `json:"hits"`
}

NHits result

type NestedHits 

type NestedHits struct {
	ID     string    `json:"_id"`
	Source HitSource `json:"_source"`
}

NestedHits is the actual hit data

type NestedRawHits 

type NestedRawHits struct {
	ID     string `json:"_id"`
	Source BugRaw `json:"_source"`
}

NestedRawHits is the actual hit data

type Param 

type Param struct {
	EndPoint               string
	FetcherBackendVersion  string
	EnricherBackendVersion string
	Fetch                  bool
	Enrich                 bool
	ESUrl                  string
	EsUser                 string
	EsPassword             string
	EsIndex                string
	FromDate               *time.Time
	Project                string
	FetchSize              int
	EnrichSize             int
	Retries                uint
	Delay                  time.Duration
	GapURL                 string
	AffAPI                 string
	ProjectSlug            string
	AffBaseURL             string
	ESCacheURL             string
	ESCacheUsername        string
	ESCachePassword        string
	AuthGrantType          string
	AuthClientID           string
	AuthClientSecret       string
	AuthAudience           string
	Auth0URL               string
	Environment            string
	Slug                   string
	WebHookURL             string
}

Param required for creating a new instance of Bugzilla manager

type Params 

type Params struct {
	Name           string
	Endpoint       string
	FromDate       time.Time
	Order          string
	Project        string
	Component      string
	Category       string
	BackendVersion string
}

Params required parameters for bugzilla fetcher

type Person 

type Person struct {
	Name     string `json:"name"`
	Username string `json:"username"`
}

Person describe affiliation person data

type RawHits 

type RawHits struct {
	Hits NHits `json:"hits"`
}

RawHits result

type SearchFields 

type SearchFields struct {
	Component string `json:"component"`
	Product   string `json:"product"`
	ItemID    string `json:"item_id"`
}

SearchFields ...

type SlackProvider 

type SlackProvider interface {
	SendText(text string) error
}

SlackProvider ...

type TopHits 

type TopHits struct {
	Hits Hits `json:"hits"`
}

TopHits result

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.