aws

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 6, 2022 License: Apache-2.0 Imports: 27 Imported by: 0

Documentation

Index

Constants

View Source
const (
	MIN_AWS_FIELD = 7
	ADDR_LEN      = 17

	AWS_LOG_PREFIX = "AWSLogs"

	AWS_ACTION       = "action"
	AWS_STATUS       = "log-status"
	AWS_VERSION      = "version"
	AWS_VPC_ID       = "vpc-id"
	AWS_SUBNET_ID    = "subnet-id"
	AWS_INSTANCE_ID  = "instance-id"
	AWS_INTERFACE_ID = "interface-id"
	AWS_ACCOUNT_ID   = "account-id"
	AWS_TYPE         = "type"
	AWS_SRC_ADDR     = "srcaddr"
	AWS_DST_ADDR     = "dstaddr"
	AWS_SRC_PORT     = "srcport"
	AWS_DST_PORT     = "dstport"
	AWS_PKT_SRC_ADDR = "pkt-srcaddr"
	AWS_PKT_DST_ADDR = "pkt-dstaddr"
	AWS_PROTOCOL     = "protocol"
	AWS_BYTES        = "bytes"
	AWS_PACKETS      = "packets"
	AWS_START        = "start"
	AWS_END          = "end"
	AWS_LINE_ACTION  = "action"
	AWS_TCP_FLAGS    = "tcp-flags"
	AWS_LOG_STATUS   = "log-status"

	// v4 fields
	AWS_REGION           = "region"
	AWS_AZ_ID            = "az-id"
	AWS_SUBLOCATION_TYPE = "sublocation-type"
	AWS_SUBLOCATION_ID   = "sublocation-id"

	// v5 fields
	AWS_PKT_SRC_AWS_SERVICE = "pkt-src-aws-service"
	AWS_PKT_DST_AWS_SERVICE = "pkt-dst-aws-service"
	AWS_FLOW_DIRECTION      = "flow-direction"
	AWS_TRAFFIC_PATH        = "traffic-path"

	AWS_VPC_TYPE = "AWS_VPC"
)

Variables

View Source
var (
	IamRole  = flag.String("iam_role", "", "IAM Role to use for processing flow")
	SqsName  = flag.String("sqs_name", "", "Listen for events from this queue for new objects to look at.")
	Regions  = flag.String("aws_regions", "us-east-1", "CSV list of region to run in. Will look for metadata in all regions, run SQS in first region.")
	IsLambda = flag.Bool("aws_lambda", kt.LookupEnvBool("AWS_IS_LAMBDA", false), "Run as a AWS Lambda function")

	ERROR_SLEEP_TIME     = 20 * time.Second
	MappingCheckDuration = 30 * 60 * time.Second
)

Functions

func NewAws

func NewAws(lineMap AwsLineMap, raw *string, log logger.ContextL) ([]*AWSLogLine, AwsLineMap, error)

func NewAwsFromKinesis

func NewAwsFromKinesis(lineMap AwsLineMap, raw *string, log logger.ContextL) ([]*AWSLogLine, AwsLineMap, error)

func NewAwsFromV2

func NewAwsFromV2(lineMap AwsLineMap, pts []string, log logger.ContextL) ([]*AWSLogLine, AwsLineMap, error)

2 391389995465 eni-0939c7c9e1255db73 10.236.54.140 10.236.57.28 31547 27068 6 2 112 1571081770 1571081799 ACCEPT OK

func NewAwsFromV345

func NewAwsFromV345(version int, lineMap AwsLineMap, pts []string, log logger.ContextL) ([]*AWSLogLine, AwsLineMap, error)

version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status

func NewAwsHeader

func NewAwsHeader(pts []string) ([]*AWSLogLine, AwsLineMap, error)

Types

type AWSEntities

type AWSEntities struct {
	Vpcs                      map[string]ec2.Vpc                      `json:"Vpcs"`
	AvailabilityZones         map[string]ec2.AvailabilityZone         `json:"AvailabilityZones"`
	Subnets                   map[string]ec2.Subnet                   `json:"Subnets"`
	InternetGateways          map[string]ec2.InternetGateway          `json:"InternetGateways"`
	NatGateways               map[string]ec2.NatGateway               `json:"NatGateways"`
	TransitGateways           map[string]ec2.TransitGateway           `json:"TransitGateways"`
	TransitGatewayAttachments map[string]ec2.TransitGatewayAttachment `json:"TransitGatewayAttachments"`
	VpnGateways               map[string]ec2.VpnGateway               `json:"VpnGateways"`
	VpcPeeringConnections     map[string]ec2.VpcPeeringConnection     `json:"VpcPeeringConnections"`
}

AWSEntities holds all of the entities fetched for a company

func NewAWSEntities

func NewAWSEntities() AWSEntities

type AWSHierarchy

type AWSHierarchy struct {
	Regions      map[string]RegionSkel `json:"Regions"`
	SubnetTrieV4 *tree.TreeV4
	SubnetTrieV6 *tree.TreeV6
}

func NewAWSHierarchy

func NewAWSHierarchy() AWSHierarchy

type AWSLogLine

type AWSLogLine struct {
	Version         int
	AccountID       string
	InterfaceID     string
	SrcAddr         net.IP
	DstAddr         net.IP
	SrcPktAddr      net.IP
	DstPktAddr      net.IP
	TcpFlags        uint32
	SrcPort         uint32
	DstPort         uint32
	Protocol        uint32
	Packets         uint64
	Bytes           uint64
	StartTime       time.Time
	EndTime         time.Time
	Action          string
	Status          string
	Sample          uint32
	VPCID           string
	SubnetID        string
	InstanceID      string
	Region          string
	AzID            string
	SublocationType string
	SublocationID   string
	SrcPktService   string
	DstPktService   string
	FlowDirection   string
	TrafficPath     string
}

func (*AWSLogLine) ToFlow

func (m *AWSLogLine) ToFlow(log logger.ContextL, topo *AWSTopology) (in *kt.JCHF)

type AWSTopology

type AWSTopology struct {
	Hierarchy AWSHierarchy `json:"Hierarchy"`
	Entities  AWSEntities  `json:"Entities"`
}

func FetchAllEntities

func FetchAllEntities(ctx context.Context, log logger.ContextL, arnName string, regions []string) (AWSTopology, bool)

FetchAllEntities fetches all the things, and returns whether it was a total success

func NewAWSTopology

func NewAWSTopology() AWSTopology

type AvailabilityZoneSkel

type AvailabilityZoneSkel struct {
	ZoneId string `json:"ZoneId"`
}

type AwsLineMap

type AwsLineMap map[string]int

type AwsVpc

type AwsVpc struct {
	logger.ContextL
	// contains filtered or unexported fields
}

func NewVpc

func NewVpc(ctx context.Context, log logger.Underlying, registry go_metrics.Registry, jchfChan chan []*kt.JCHF, apic *api.KentikApi, lambdaHandler func([]*kt.JCHF, func(error))) (*AwsVpc, error)

func (*AwsVpc) Close

func (vpc *AwsVpc) Close()

func (*AwsVpc) HttpInfo

func (vpc *AwsVpc) HttpInfo() map[string]float64

type ExtractedField

type ExtractedField struct {
	SrcAddr     string `json:"srcaddr"`
	DstPort     string `json:"dstport"`
	StartTime   string `json:"start"`
	DstAddr     string `json:"dstaddr"`
	Version     string `json:"version"`
	Packets     string `json:"packets"`
	Protocol    string `json:"protocol"`
	AccountId   string `json:"account_id"`
	InterfaceId string `json:"interface_id"`
	Status      string `json:"log_status"`
	Bytes       string `json:"bytes"`
	SrcPort     string `json:"srcport"`
	Action      string `json:"action"`
	EndTime     string `json:"end"`
}

type FlowSet

type FlowSet struct {
	Bucket string
	Key    string
	Lines  []*AWSLogLine `json:"lines"`
}

func (*FlowSet) GetDeviceKey

func (fs *FlowSet) GetDeviceKey() (string, error)

What is the kentik name of this device?

func (*FlowSet) ProcessKey

func (fs *FlowSet) ProcessKey(bucket string, key string) error

type InternetGatewayAttachmentSkel

type InternetGatewayAttachmentSkel struct {
	InternetGatewayId string `json:"InternetGatewayId"`
	VpcId             string `json:"VpcId"`
	State             string `json:"State"`
}

type InternetGatewaySkel

type InternetGatewaySkel struct {
	InternetGatewayId          string                                   `json:"InternetGatewayId"`
	InternetGatewayAttachments map[string]InternetGatewayAttachmentSkel `json:"InternetGatewayAttachments"`
}

func NewInternetGatewaySkel

func NewInternetGatewaySkel(id string) InternetGatewaySkel

type KinesisLog

type KinesisLog struct {
	Id              string         `json:"id"`
	Message         string         `json:"message"`
	ExtractedFields ExtractedField `json:"extractedFields"`
}

type KinesisLogWrapper

type KinesisLogWrapper struct {
	MessageType string       `json:"messageType"`
	Owner       string       `json:"owner"`
	LogGroup    string       `json:"logGroup"`
	LogEvents   []KinesisLog `json:"logEvents"`
}

type NatGatewaySkel

type NatGatewaySkel struct {
	NatGatewayId string `json:"NatGatewayId"`
}

type OrangeMetric

type OrangeMetric struct {
	ObjectsSeen       go_metrics.Meter
	Flows             go_metrics.Meter
	DroppedFlows      go_metrics.Meter
	RateSent          go_metrics.Meter
	DispatchCount     go_metrics.Counter
	DispatchRecsCount go_metrics.Counter
}

type RegionSkel

type RegionSkel struct {
	Name              string                          `json:"Name"`
	Vpcs              map[string]VpcSkel              `json:"Vpcs"`
	AvailabilityZones map[string]AvailabilityZoneSkel `json:"AvailabilityZones"`
	InternetGateways  map[string]InternetGatewaySkel  `json:"InternetGateways"`
	TransitGateways   map[string]TransitGatewaySkel   `json:"TransitGateways"`
	VpnGateways       map[string]VpnGatewaySkel       `json:"VpnGateways"`
}

func NewRegionSkel

func NewRegionSkel(name string) RegionSkel

type SQSBucket

type SQSBucket struct {
	Name string `json:"name"`
}

type SQSEvent

type SQSEvent struct {
	Records []SQSRecord `json:"Records"`
}

* {"Records":[{"eventVersion":"2.1","eventSource":"aws:s3","awsRegion":"us-west-2","eventTime":"2020-09-17T18:24:24.619Z","eventName":"ObjectCreated:Put","userIdentity":{"principalId":"AWS:AROAV2CJ256E23ZOBRSAF:prod.pdx.dbs.datafeeds.aws.internal"},"requestParameters":{"sourceIPAddress":"172.19.15.211"},"responseElements":{"x-amz-request-id":"AD5D47CDA09091B3","x-amz-id-2":"gMLgqszXsmKN41Ou3/l330BEXa+ARrbIP8UkW9VqW21WdC42ie4Ki1WMP5Zm8M6R1TuAgkjPJFtlFB2HX+Ui3yeNqrJ5oBz0"},"s3":{"s3SchemaVersion":"1.0","configurationId":"Flow","bucket":{"name":"kentik-test-orangeflow","ownerIdentity":{"principalId":"A2L4QHGC7GJYP3"},"arn":"arn:aws:s3:::kentik-test-orangeflow"},"object":{"key":"AWSLogs/451031991406/vpcflowlogs/us-west-2/2020/09/17/451031991406_vpcflowlogs_us-west-2_fl-0ac5de8260cdc0575_20200917T1820Z_c127bbf5.log.gz","size":1797,"eTag":"1638e127fe977dacb06958f144f0b549","sequencer":"005F63A9DAF94F4D9E"}}}]}

type SQSObject

type SQSObject struct {
	Key string `json:"key"`
}

type SQSRecord

type SQSRecord struct {
	EventName string `json:"eventName"`
	S3        SQSS3  `json:"s3"`
}

type SQSS3

type SQSS3 struct {
	Bucket SQSBucket `json:"bucket"`
	Object SQSObject `json:"object"`
}

type SubnetSkel

type SubnetSkel struct {
	SubnetId    string                    `json:"SubnetId"`
	NatGateways map[string]NatGatewaySkel `json:"NatGateways"`
}

func NewSubnetSkel

func NewSubnetSkel(subnetId string) SubnetSkel

type TransitGatewayAttachmentSkel

type TransitGatewayAttachmentSkel struct {
	TransitGatewayAttachmentId string `json:"TransitGatewayAttachmentId"`
}

type TransitGatewaySkel

type TransitGatewaySkel struct {
	TransitGatewayId          string                                  `json:"TransitGatewayId "`
	TransitGatewayAttachments map[string]TransitGatewayAttachmentSkel `json:"TransitGatewayAttachment"`
}

func NewTransitGatewaySkel

func NewTransitGatewaySkel(id string) TransitGatewaySkel

type VpcPeeringConnectionSkel

type VpcPeeringConnectionSkel struct {
	RequesterVpcId string `json:"RequesterVpcId"`
	AccepterVpcId  string `json:"AccepterVpcId"`
}

type VpcSkel

type VpcSkel struct {
	VpcId                      string                                   `json:"VpcId"`
	Subnets                    map[string]SubnetSkel                    `json:"Subnets"`
	TransitGatewayAttachments  map[string]TransitGatewayAttachmentSkel  `json:"TransitGatewayAttachments"`  // by GatewayAttachmentId
	InternetGatewayAttachments map[string]InternetGatewayAttachmentSkel `json:"InternetGatewayAttachments"` // by InternetGatewayId
	VpcPeeringConnections      map[string]VpcPeeringConnectionSkel      `json:"VpcPeeringConnections"`      // by VpcPeeringConnectionId
}

func NewVpcSkel

func NewVpcSkel(id string) VpcSkel

type VpnGatewaySkel

type VpnGatewaySkel struct {
	VpnGatewayId string `json:"VpnGatewayId"`
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL