googlegroups

package
v0.0.0-...-b175f30 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 23, 2022 License: Apache-2.0 Imports: 24 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// GoogleGroups ...
	GoogleGroups = "googlegroups"

	// Unknown ...
	Unknown = "Unknown"
	// CredentialsSSMParamName from ssm
	CredentialsSSMParamName = "insights_googlegroups_credentials"
	// TokenSSMParamName from ssm
	TokenSSMParamName = "insights_googlegroups_token"
	// MaxNumberOfMessages from gmail
	MaxNumberOfMessages = 10000000
	// MaxConcurrentRequests ...
	MaxConcurrentRequests = 10000
)

Variables

View Source
var (
	// GoogleGroupRichMapping ...
	GoogleGroupRichMapping = []byte(`{"mappings":{"dynamic_templates":[{"notanalyzed":{"match":"*","match_mapping_type":"string","mapping":{"type":"keyword"}}},{"int_to_float":{"match":"*","match_mapping_type":"long","mapping":{"type":"float"}}},{"formatdate":{"match":"*","match_mapping_type":"date","mapping":{"format":"strict_date_optional_time||epoch_millis","type":"date"}}}]}}`)
	// GoogleGroupRawMapping ...
	GoogleGroupRawMapping = []byte(`{"mappings":{"dynamic":true,"properties":{"metadata__updated_on":{"type":"date"},"data":{"properties":{"body":{"dynamic":false,"properties":{}}}}}}}`)

	// DefaultDateTime ...
	DefaultDateTime = time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
)

Functions

This section is empty.

Types

type AffiliationClient

type AffiliationClient interface {
	GetIdentityByUser(key string, value string) (*affiliation.AffIdentity, error)
	AddIdentity(identity *affiliation.Identity) bool
	GetOrganizations(uuid string, projectSlug string) *[]affiliation.Enrollment
}

AffiliationClient manages user identity

type AuthClientProvider

type AuthClientProvider interface {
	GetToken(env string) (string, error)
}

AuthClientProvider interacts with auth0 server

type EnrichedMessage

type EnrichedMessage struct {
	From                 string    `json:"from"`
	Date                 time.Time `json:"date"`
	To                   []string  `json:"to"`
	MessageID            string    `json:"message_id"`
	InReplyTo            string    `json:"in_reply_to"`
	References           string    `json:"references"`
	Subject              string    `json:"subject"`
	Topic                string    `json:"topic"`
	MessageBody          string    `json:"message_body"`
	TopicID              string    `json:"topic_id"`
	BackendVersion       string    `json:"backend_version"`
	UUID                 string    `json:"uuid"`
	Origin               string    `json:"origin"`
	MetadataUpdatedOn    time.Time `json:"metadata__updated_on"`
	BackendName          string    `json:"backend_name"`
	MetadataTimestamp    time.Time `json:"metadata__timestamp"`
	MetadataEnrichedOn   time.Time `json:"metadata__enriched_on"`
	ProjectSlug          string    `json:"project_slug"`
	GroupName            string    `json:"group_name"`
	Project              string    `json:"project"`
	Root                 bool      `json:"root"`
	FromBot              bool      `json:"from_bot"`
	ChangedAt            time.Time `json:"changed_at"`
	AuthorName           string    `json:"author_name"`
	AuthorID             string    `json:"author_id"`
	AuthorUUID           string    `json:"author_uuid"`
	AuthorOrgName        string    `json:"author_org_name"`
	AuthorUserName       string    `json:"author_user_name"`
	AuthorBot            bool      `json:"author_bot"`
	AuthorMultiOrgNames  []string  `json:"author_multi_org_names"`
	MboxAuthorDomain     string    `json:"mbox_author_domain"`
	IsGoogleGroupMessage int       `json:"is_google_group_message"`
	Timezone             int       `json:"timezone"`
	ViaCommunityGroup    bool      `json:"via_community_group"`
}

EnrichedMessage ...

type Enricher

type Enricher struct {
	DSName                string // Datasource will be used as key for ES
	ElasticSearchProvider *elastic.ClientProvider
	// contains filtered or unexported fields
}

Enricher contains google groups datasource enrich logic

func NewEnricher

func NewEnricher(esClientProvider *elastic.ClientProvider, affiliationsClientProvider *affiliation.Affiliation) *Enricher

NewEnricher initiates a new Enricher

func (*Enricher) EnrichMessage

func (e *Enricher) EnrichMessage(rawMessage *RawMessage, now time.Time) (*EnrichedMessage, error)

EnrichMessage enriches raw message

func (*Enricher) Find

func (e *Enricher) Find(slice []string, val string) bool

Find takes a slice and looks for an element in it. If found it will return it's true, otherwise it will return false.

func (*Enricher) GetEmailAddress

func (e *Enricher) GetEmailAddress(rawMailString string) (mail *string)

GetEmailAddress ...

func (*Enricher) GetEmailDomain

func (e *Enricher) GetEmailDomain(email string) string

GetEmailDomain ...

func (*Enricher) GetEmailUsername

func (e *Enricher) GetEmailUsername(email string) string

GetEmailUsername ...

func (*Enricher) GetUserName

func (e *Enricher) GetUserName(rawMailString string) (username string)

GetUserName ...

func (*Enricher) IsValidEmail

func (e *Enricher) IsValidEmail(rawMailString string) bool

IsValidEmail validates email string

func (*Enricher) RemoveSpecialCharactersFromString

func (e *Enricher) RemoveSpecialCharactersFromString(s string) (val *string)

RemoveSpecialCharactersFromString ...

func (*Enricher) SanitizeEmails

func (e *Enricher) SanitizeEmails(emails []string) string

SanitizeEmails returns the first well formed email address for a given user it filters out norely emails and emails with ellipsis

type Fetcher

type Fetcher struct {
	DSName                string
	HTTPClientProvider    *http.ClientProvider
	ElasticSearchProvider *elastic.ClientProvider
	BackendVersion        string
	DateFrom              time.Time
	GroupName             string
	ProjectSlug           string
	Project               string
}

Fetcher contains GoogleGroups datasource fetch logic

func NewFetcher

func NewFetcher(groupName, projectSlug, project string, httpClientProvider *http.ClientProvider, esClientProvider *elastic.ClientProvider) *Fetcher

NewFetcher initiates a new GoogleGroups fetcher

func (*Fetcher) Fetch

func (f *Fetcher) Fetch(fromDate, now *time.Time) ([]*RawMessage, error)

Fetch ...

type GoogleGroupMessage

type GoogleGroupMessage struct {
	ID      string `json:"id"`
	Author  string `json:"author"`
	Date    string `json:"date"`
	File    string `json:"file"`
	Message string `json:"message"`
}

GoogleGroupMessage ...

type GoogleGroupMessageThread

type GoogleGroupMessageThread struct {
	Topic    string                `json:"topic"`
	ID       string                `json:"id"`
	Messages []*GoogleGroupMessage `json:"messages"`
}

GoogleGroupMessageThread ...

type GoogleGroupMessages

type GoogleGroupMessages struct {
	Messages []*GoogleGroupMessageThread
}

GoogleGroupMessages ...

type HeadersData

type HeadersData struct {
	// Date is the date the message was originally sent
	Date string
	// MessageID is the message id
	MessageID string
	// InReplyTo is who the email was sent to. This can contain multiple
	// addresses if the email was forwarded.
	InReplyTo string
	// References
	References string
	// Sender is the entity that originally created and sent the message
	Sender string
	// From is the name - email address combo of the email author
	From string
	// Subject is the subject of the email
	Subject string
	// To is the email recipient.
	To []string
	// DeliveredTo is to whom the email was sent to. This can contain multiple
	// addresses if the email was forwarded.
	DeliveredTo []string
	MailingList string
}

HeadersData struct

type HitSource

type HitSource struct {
	ID        string    `json:"id"`
	ChangedAt time.Time `json:"changed_at"`
}

HitSource is the document _source data

type Hits

type Hits struct {
	Hits []NestedHits `json:"hits"`
}

Hits result

type Manager

type Manager struct {
	Slug                   string
	GroupName              string
	SHConnString           string
	FetcherBackendVersion  string
	EnricherBackendVersion string
	Fetch                  bool
	Enrich                 bool
	ESUrl                  string
	ESUsername             string
	ESPassword             string
	ESIndex                string
	FromDate               *time.Time
	HTTPTimeout            time.Duration
	Project                string
	FetchSize              int
	EnrichSize             int
	AffBaseURL             string
	ESCacheURL             string
	ESCacheUsername        string
	ESCachePassword        string
	AuthGrantType          string
	AuthClientID           string
	AuthClientSecret       string
	AuthAudience           string
	Auth0URL               string
	Environment            string
	WebHookURL             string
	// contains filtered or unexported fields
}

Manager describes google groups manager

func NewManager

func NewManager(slug, groupName, shConnStr, fetcherBackendVersion, enricherBackendVersion string, fetch bool, enrich bool, eSUrl string, esUser string, esPassword string, esIndex string, fromDate *time.Time, project string, fetchSize int, enrichSize int, affBaseURL, esCacheURL, esCacheUsername, esCachePassword, authGrantType, authClientID, authClientSecret, authAudience, authURL, env string) (*Manager, error)

NewManager initiates google groups manager instance

func (*Manager) AddTask

func (m *Manager) AddTask(task func())

AddTask adds task to worker pool

func (*Manager) GetTotalQueuedTask

func (m *Manager) GetTotalQueuedTask() int

GetTotalQueuedTask get total number of queued tasks

func (*Manager) Sync

func (m *Manager) Sync() error

Sync runs google groups fetch and enrich according to passed parameters

type NHits

type NHits struct {
	Hits []NestedRawHits `json:"hits"`
}

NHits result

type NestedHits

type NestedHits struct {
	ID     string    `json:"_id"`
	Source HitSource `json:"_source"`
}

NestedHits is the actual hit data

type NestedRawHits

type NestedRawHits struct {
	ID     string     `json:"_id"`
	Source RawMessage `json:"_source"`
}

NestedRawHits is the actual hit data

type RawHits

type RawHits struct {
	Hits NHits `json:"hits"`
}

RawHits result

type RawMessage

type RawMessage struct {
	From              string    `json:"from"`
	Date              time.Time `json:"date"`
	To                []string  `json:"to"`
	MessageID         string    `json:"message_id"`
	InReplyTo         string    `json:"in_reply_to"`
	References        string    `json:"references"`
	Subject           string    `json:"subject"`
	MessageBody       string    `json:"message_body"`
	TopicID           string    `json:"topic_id"`
	Topic             string    `json:"topic"`
	BackendVersion    string    `json:"backend_version"`
	UUID              string    `json:"uuid"`
	Origin            string    `json:"origin"`
	MetadataUpdatedOn time.Time `json:"metadata__updated_on"`
	BackendName       string    `json:"backend_name"`
	MetadataTimestamp time.Time `json:"metadata__timestamp"`
	ProjectSlug       string    `json:"project_slug"`
	GroupName         string    `json:"group_name"`
	Project           string    `json:"project"`
	ChangedAt         time.Time `json:"changed_at"`
	Timezone          int       `json:"timezone"`
}

RawMessage represents GoogleGroups raw message

type TopHits

type TopHits struct {
	Hits Hits `json:"hits"`
}

TopHits result

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL