zannotate

package module
v0.0.0-...-69b89b4 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 7, 2022 License: Apache-2.0 Imports: 14 Imported by: 0

README

ZAnnotate

ZAnnotate is a Go utility that facilitates annotating large datasets with network metadata. Right now this includes:

  • Maxmind GeoIP2
  • AS/Routing Data (based on an MRT routing table)

For example, you can add Maxmind geolocation data to a list of IPs:

cat ips.csv | zannotate --geoip2 --geoip2-database=geoip2.mmdb

Documentation

Index

Constants

This section is empty.

Variables

View Source
var Annotators []AnnotatorFactory

Functions

func AnnotateInputDecode

func AnnotateInputDecode(conf *GlobalConf, inChan <-chan string,
	outChan chan<- inProcessIP, wg *sync.WaitGroup, i int)

multiple workers that decode raw lines from AnnotateRead from JSON/CSV into native golang objects

func AnnotateOutputEncode

func AnnotateOutputEncode(conf *GlobalConf, inChan <-chan inProcessIP,
	outChan chan<- string, wg *sync.WaitGroup, i int)

func AnnotateRead

func AnnotateRead(conf *GlobalConf, path string, in chan<- string)

single worker that reads from file and queues raw lines

func AnnotateWorker

func AnnotateWorker(a Annotator, inChan <-chan inProcessIP,
	outChan chan<- inProcessIP, fieldName string, wg *sync.WaitGroup, i int)

func AnnotateWrite

func AnnotateWrite(path string, out <-chan string, wg *sync.WaitGroup)

func DoAnnotation

func DoAnnotation(conf *GlobalConf)

func RegisterAnnotator

func RegisterAnnotator(af AnnotatorFactory)

Types

type Annotator

type Annotator interface {
	Initialize() error
	Annotate(ip net.IP) interface{}
	GetFieldName() string
	Close() error
}

type AnnotatorFactory

type AnnotatorFactory interface {
	Initialize(c *GlobalConf) error
	AddFlags(flags *flag.FlagSet)
	GetWorkers() int
	IsEnabled() bool
	MakeAnnotator(i int) Annotator
	Close() error
}

type BasePluginConf

type BasePluginConf struct {
	Threads int
	Enabled bool
}

type GeoIP2Annotator

type GeoIP2Annotator struct {
	Factory *GeoIP2AnnotatorFactory
	Reader  *geoip2.Reader
	Id      int
}

func (*GeoIP2Annotator) Annotate

func (a *GeoIP2Annotator) Annotate(ip net.IP) interface{}

func (*GeoIP2Annotator) Close

func (a *GeoIP2Annotator) Close() error

func (*GeoIP2Annotator) GeoIP2FillStruct

func (a *GeoIP2Annotator) GeoIP2FillStruct(in *geoip2.City) *GeoIP2Output

func (*GeoIP2Annotator) GetFieldName

func (a *GeoIP2Annotator) GetFieldName() string

func (*GeoIP2Annotator) Initialize

func (a *GeoIP2Annotator) Initialize() error

type GeoIP2AnnotatorFactory

type GeoIP2AnnotatorFactory struct {
	BasePluginConf
	Path       string
	Mode       string
	Language   string
	RawInclude string

	Conf *GlobalConf
	// what data to include
	IncludeCity               bool
	IncludeCountry            bool
	IncludeContinent          bool
	IncludePostal             bool
	IncludeLatLong            bool
	IncludeTraits             bool
	IncludeSubdivisions       bool
	IncludeRepresentedCountry bool
	IncludeRegisteredCountry  bool
}

func (*GeoIP2AnnotatorFactory) AddFlags

func (a *GeoIP2AnnotatorFactory) AddFlags(flags *flag.FlagSet)

GeoIP2 Annotator Factory (Global)

func (*GeoIP2AnnotatorFactory) Close

func (a *GeoIP2AnnotatorFactory) Close() error

func (*GeoIP2AnnotatorFactory) GetWorkers

func (a *GeoIP2AnnotatorFactory) GetWorkers() int

func (*GeoIP2AnnotatorFactory) Initialize

func (a *GeoIP2AnnotatorFactory) Initialize(conf *GlobalConf) error

func (*GeoIP2AnnotatorFactory) IsEnabled

func (a *GeoIP2AnnotatorFactory) IsEnabled() bool

func (*GeoIP2AnnotatorFactory) MakeAnnotator

func (a *GeoIP2AnnotatorFactory) MakeAnnotator(i int) Annotator

type GeoIP2City

type GeoIP2City struct {
	Name      string `json:"name,omitempty"`
	GeoNameId uint   `json:"id,omitempty"`
}

type GeoIP2Country

type GeoIP2Country struct {
	Name      string `json:"name,omitempty"`
	Code      string `json:"code,omitempty"`
	GeoNameId uint   `json:"id,omitempty"`
}

type GeoIP2LatLong

type GeoIP2LatLong struct {
	AccuracyRadius uint16  `json:"accuracy_radius,omitempty"`
	Latitude       float64 `json:"latitude,omitempty"`
	Longitude      float64 `json:"longitude,omitempty"`
	MetroCode      uint    `json:"metro_code,omitempty"`
	TimeZone       string  `json:"time_zone,omitempty"`
}

type GeoIP2Output

type GeoIP2Output struct {
	City               *GeoIP2City    `json:"city,omitempty"`
	Country            *GeoIP2Country `json:"country,omitempty"`
	Continent          *GeoIP2Country `json:"continent,omitempty"`
	Postal             *GeoIP2Postal  `json:"postal,omitempty"`
	LatLong            *GeoIP2LatLong `json:"latlong,omitempty"`
	RepresentedCountry *GeoIP2Country `json:"represented_country,omitempty"`
	RegisteredCountry  *GeoIP2Country `json:"represented_country,omitempty"`
	Traits             *GeoIP2Traits  `json:"metadata,omitempty"`
}

type GeoIP2Postal

type GeoIP2Postal struct {
	Code string `json:"code,omitempty"`
}

type GeoIP2Traits

type GeoIP2Traits struct {
	IsAnonymousProxy    bool `json:"is_anonymous_proxy,omitempty"`
	IsSatelliteProvider bool `json:"is_satellite_provider,omitempty"`
}

type GeoIPASNAnnotator

type GeoIPASNAnnotator struct {
	Factory *GeoIPASNAnnotatorFactory
	Reader  *geoip2.Reader
	Id      int
}

func (*GeoIPASNAnnotator) Annotate

func (anno *GeoIPASNAnnotator) Annotate(ip net.IP) interface{}

func (*GeoIPASNAnnotator) Close

func (anno *GeoIPASNAnnotator) Close() error

func (*GeoIPASNAnnotator) GetFieldName

func (anno *GeoIPASNAnnotator) GetFieldName() string

func (*GeoIPASNAnnotator) Initialize

func (anno *GeoIPASNAnnotator) Initialize() error

type GeoIPASNAnnotatorFactory

type GeoIPASNAnnotatorFactory struct {
	BasePluginConf
	Path string
	Mode string
}

func (*GeoIPASNAnnotatorFactory) AddFlags

func (fact *GeoIPASNAnnotatorFactory) AddFlags(flags *flag.FlagSet)

func (*GeoIPASNAnnotatorFactory) Close

func (fact *GeoIPASNAnnotatorFactory) Close() error

func (*GeoIPASNAnnotatorFactory) GetWorkers

func (fact *GeoIPASNAnnotatorFactory) GetWorkers() int

func (*GeoIPASNAnnotatorFactory) Initialize

func (fact *GeoIPASNAnnotatorFactory) Initialize(conf *GlobalConf) error

func (*GeoIPASNAnnotatorFactory) IsEnabled

func (fact *GeoIPASNAnnotatorFactory) IsEnabled() bool

func (*GeoIPASNAnnotatorFactory) MakeAnnotator

func (fact *GeoIPASNAnnotatorFactory) MakeAnnotator(i int) Annotator

type GeoIPASNOutput

type GeoIPASNOutput struct {
	ASN    uint   `json:"asn,omitempty"`
	ASNOrg string `json:"org,omitempty"`
}

type GlobalConf

type GlobalConf struct {
	InputFilePath           string
	InputFileType           string
	OutputFilePath          string
	MetadataFilePath        string
	LogFilePath             string
	Verbosity               int
	Threads                 int
	JSONIPFieldName         string
	JSONAnnotationFieldName string
	InputDecodeThreads      int
	OutputEncodeThreads     int
}

type RDNSAnnotator

type RDNSAnnotator struct {
	Factory *RDNSAnnotatorFactory
	Id      int
}

func (*RDNSAnnotator) Annotate

func (a *RDNSAnnotator) Annotate(ip net.IP) interface{}

func (*RDNSAnnotator) Close

func (a *RDNSAnnotator) Close() error

func (*RDNSAnnotator) GetFieldName

func (a *RDNSAnnotator) GetFieldName() string

func (*RDNSAnnotator) Initialize

func (a *RDNSAnnotator) Initialize() error

type RDNSAnnotatorFactory

type RDNSAnnotatorFactory struct {
	BasePluginConf
	RawResolvers string
}

func (*RDNSAnnotatorFactory) AddFlags

func (a *RDNSAnnotatorFactory) AddFlags(flags *flag.FlagSet)

func (*RDNSAnnotatorFactory) Close

func (a *RDNSAnnotatorFactory) Close() error

func (*RDNSAnnotatorFactory) GetWorkers

func (a *RDNSAnnotatorFactory) GetWorkers() int

func (*RDNSAnnotatorFactory) Initialize

func (a *RDNSAnnotatorFactory) Initialize(conf *GlobalConf) error

func (*RDNSAnnotatorFactory) IsEnabled

func (a *RDNSAnnotatorFactory) IsEnabled() bool

func (*RDNSAnnotatorFactory) MakeAnnotator

func (a *RDNSAnnotatorFactory) MakeAnnotator(i int) Annotator

type RoutingAnnotator

type RoutingAnnotator struct {
	Factory *RoutingAnnotatorFactory

	Id int
	// contains filtered or unexported fields
}

func (*RoutingAnnotator) Annotate

func (a *RoutingAnnotator) Annotate(ip net.IP) interface{}

func (*RoutingAnnotator) Close

func (a *RoutingAnnotator) Close() error

func (*RoutingAnnotator) GetFieldName

func (a *RoutingAnnotator) GetFieldName() string

func (*RoutingAnnotator) Initialize

func (a *RoutingAnnotator) Initialize() error

type RoutingAnnotatorFactory

type RoutingAnnotatorFactory struct {
	BasePluginConf
	RoutingTablePath string
	ASNamesPath      string
	ASDataPath       string
	// contains filtered or unexported fields
}

func (*RoutingAnnotatorFactory) AddFlags

func (a *RoutingAnnotatorFactory) AddFlags(flags *flag.FlagSet)

Routing Annotator Factory (Global)

func (*RoutingAnnotatorFactory) Close

func (a *RoutingAnnotatorFactory) Close() error

func (*RoutingAnnotatorFactory) GetWorkers

func (a *RoutingAnnotatorFactory) GetWorkers() int

func (*RoutingAnnotatorFactory) Initialize

func (a *RoutingAnnotatorFactory) Initialize(conf *GlobalConf) error

func (*RoutingAnnotatorFactory) IsEnabled

func (a *RoutingAnnotatorFactory) IsEnabled() bool

func (*RoutingAnnotatorFactory) MakeAnnotator

func (a *RoutingAnnotatorFactory) MakeAnnotator(i int) Annotator

Directories

Path Synopsis
cmd

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL