kafka_manager_exporter

package module
v0.0.0-...-03a02a9 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 15, 2023 License: MIT Imports: 0 Imported by: 0

README

kafka-manager-exporter

image

A Prometheus Exporter For Kafka Manager

Usage

$ kafka-manager-exporter --help
  -addr string
    	http server listening address (default "127.0.0.1:9001")
  -interval duration
    	interval of collecting (default 15s)
  -urls string
    	Kafka Manager URLs. Separate with comma (default "http://127.0.0.1:9000")

$ kafka-manager-exporter -urls http://KAFKA_MANAGER_URL -interval 15s -addr 0.0.0.1:9001

2019/04/02 17:56:00 Server Launched 127.0.0.1:9001 ...
^C2019/04/02 17:56:18 Server Shutdown

# If you prefer docker

$ docker run -it --rm -p 9001:9001 dreampuf/kafka-manager-exporter -urls http://KAFKA_MANAGER_URL -interval 15s -addr 0.0.0.0:9001
2019/04/02 22:28:03 Server Launched 0.0.0.0:9001 ...

$ curl localhost:9001/metrics
topic_summed_offset{topic="pick_ack_pair"} 3.0283061e+07
topic_summed_offset{topic="test"} 8.080727e+06
topic_summed_offset{topic="test1"} 1.087054e+06
topic_summed_offset{topic="test2"} 45
topic_summed_offset{topic="test_broker"} 108723
topic_summed_offset{topic="test_new_consumer"} 1.0209415389e+10
topic_summed_offset{topic="test_performance"} 79514
topic_summed_offset{topic="test_single_partition"} 4.725704e+06
topic_summed_offset{topic="test_topic"} 41021

Why

We already have a kafka-exporter. But we always have multiple kafka cluster to manage. kafka manager becomes a kind of must have component. Rather than setup a batch of indivitually kafka-exporter instances, I can gather these data in the center place (Yes, it's break your best practice. Brain).

PLUS, if you also want some bandwidth metrics, you may need to set up jmx-exporter as well. That's the efforts.

All these ideas come to this project. PR are welcome.

Author: Dreampuf

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type APIClusters 

type APIClusters struct {
	Clusters struct {
		Active []struct {
			Name          string `json:"name"`
			CuratorConfig struct {
				ZkConnect       string `json:"zkConnect"`
				ZkMaxRetry      int    `json:"zkMaxRetry"`
				BaseSleepTimeMs int    `json:"baseSleepTimeMs"`
				MaxSleepTimeMs  int    `json:"maxSleepTimeMs"`
			} `json:"curatorConfig"`
			Enabled bool `json:"enabled"`
			Version struct {
			} `json:"version"`
			JmxEnabled               bool `json:"jmxEnabled"`
			JmxSsl                   bool `json:"jmxSsl"`
			PollConsumers            bool `json:"pollConsumers"`
			FilterConsumers          bool `json:"filterConsumers"`
			LogkafkaEnabled          bool `json:"logkafkaEnabled"`
			ActiveOffsetCacheEnabled bool `json:"activeOffsetCacheEnabled"`
			DisplaySizeEnabled       bool `json:"displaySizeEnabled"`
			Tuning                   struct {
				BrokerViewUpdatePeriodSeconds       int `json:"brokerViewUpdatePeriodSeconds"`
				ClusterManagerThreadPoolSize        int `json:"clusterManagerThreadPoolSize"`
				ClusterManagerThreadPoolQueueSize   int `json:"clusterManagerThreadPoolQueueSize"`
				KafkaCommandThreadPoolSize          int `json:"kafkaCommandThreadPoolSize"`
				KafkaCommandThreadPoolQueueSize     int `json:"kafkaCommandThreadPoolQueueSize"`
				LogkafkaCommandThreadPoolSize       int `json:"logkafkaCommandThreadPoolSize"`
				LogkafkaCommandThreadPoolQueueSize  int `json:"logkafkaCommandThreadPoolQueueSize"`
				LogkafkaUpdatePeriodSeconds         int `json:"logkafkaUpdatePeriodSeconds"`
				PartitionOffsetCacheTimeoutSecs     int `json:"partitionOffsetCacheTimeoutSecs"`
				BrokerViewThreadPoolSize            int `json:"brokerViewThreadPoolSize"`
				BrokerViewThreadPoolQueueSize       int `json:"brokerViewThreadPoolQueueSize"`
				OffsetCacheThreadPoolSize           int `json:"offsetCacheThreadPoolSize"`
				OffsetCacheThreadPoolQueueSize      int `json:"offsetCacheThreadPoolQueueSize"`
				KafkaAdminClientThreadPoolSize      int `json:"kafkaAdminClientThreadPoolSize"`
				KafkaAdminClientThreadPoolQueueSize int `json:"kafkaAdminClientThreadPoolQueueSize"`
			} `json:"tuning"`
			SecurityProtocol struct {
			} `json:"securityProtocol"`
		} `json:"active"`
		Pending []interface{} `json:"pending"`
	} `json:"clusters"`
}

type APIConsumers 

type APIConsumers struct {
	Consumers []struct {
		Name   string   `json:"name"`
		Type   string   `json:"type"`
		Topics []string `json:"topics"`
	} `json:"consumers"`
}

type APITopicIdentities 

type APITopicIdentities struct {
	TopicIdentities []struct {
		Topic              string `json:"topic"`
		ReadVersion        int    `json:"readVersion"`
		Partitions         int    `json:"partitions"`
		PartitionsIdentity []struct {
			PartNum           int     `json:"partNum"`
			Leader            int     `json:"leader"`
			LatestOffset      int     `json:"latestOffset"`
			RateOfChange      float64 `json:"rateOfChange"`
			Isr               []int   `json:"isr"`
			Replicas          []int   `json:"replicas"`
			IsPreferredLeader bool    `json:"isPreferredLeader"`
			IsUnderReplicated bool    `json:"isUnderReplicated"`
		} `json:"partitionsIdentity"`
		NumBrokers        int        `json:"numBrokers"`
		ConfigReadVersion int        `json:"configReadVersion"`
		Config            [][]string `json:"config"`
		ClusterContext    struct {
			ClusterFeatures struct {
				Features []struct {
				} `json:"features"`
			} `json:"clusterFeatures"`
			Config struct {
				Name          string `json:"name"`
				CuratorConfig struct {
					ZkConnect       string `json:"zkConnect"`
					ZkMaxRetry      int    `json:"zkMaxRetry"`
					BaseSleepTimeMs int    `json:"baseSleepTimeMs"`
					MaxSleepTimeMs  int    `json:"maxSleepTimeMs"`
				} `json:"curatorConfig"`
				Enabled bool `json:"enabled"`
				Version struct {
				} `json:"version"`
				JmxEnabled               bool `json:"jmxEnabled"`
				JmxSsl                   bool `json:"jmxSsl"`
				PollConsumers            bool `json:"pollConsumers"`
				FilterConsumers          bool `json:"filterConsumers"`
				LogkafkaEnabled          bool `json:"logkafkaEnabled"`
				ActiveOffsetCacheEnabled bool `json:"activeOffsetCacheEnabled"`
				DisplaySizeEnabled       bool `json:"displaySizeEnabled"`
				Tuning                   struct {
					BrokerViewUpdatePeriodSeconds       int `json:"brokerViewUpdatePeriodSeconds"`
					ClusterManagerThreadPoolSize        int `json:"clusterManagerThreadPoolSize"`
					ClusterManagerThreadPoolQueueSize   int `json:"clusterManagerThreadPoolQueueSize"`
					KafkaCommandThreadPoolSize          int `json:"kafkaCommandThreadPoolSize"`
					KafkaCommandThreadPoolQueueSize     int `json:"kafkaCommandThreadPoolQueueSize"`
					LogkafkaCommandThreadPoolSize       int `json:"logkafkaCommandThreadPoolSize"`
					LogkafkaCommandThreadPoolQueueSize  int `json:"logkafkaCommandThreadPoolQueueSize"`
					LogkafkaUpdatePeriodSeconds         int `json:"logkafkaUpdatePeriodSeconds"`
					PartitionOffsetCacheTimeoutSecs     int `json:"partitionOffsetCacheTimeoutSecs"`
					BrokerViewThreadPoolSize            int `json:"brokerViewThreadPoolSize"`
					BrokerViewThreadPoolQueueSize       int `json:"brokerViewThreadPoolQueueSize"`
					OffsetCacheThreadPoolSize           int `json:"offsetCacheThreadPoolSize"`
					OffsetCacheThreadPoolQueueSize      int `json:"offsetCacheThreadPoolQueueSize"`
					KafkaAdminClientThreadPoolSize      int `json:"kafkaAdminClientThreadPoolSize"`
					KafkaAdminClientThreadPoolQueueSize int `json:"kafkaAdminClientThreadPoolQueueSize"`
				} `json:"tuning"`
				SecurityProtocol struct {
				} `json:"securityProtocol"`
			} `json:"config"`
		} `json:"clusterContext"`
		Size               interface{} `json:"size"`
		ReplicationFactor  int         `json:"replicationFactor"`
		PartitionsByBroker []struct {
			ID             int   `json:"id"`
			Partitions     []int `json:"partitions"`
			IsSkewed       bool  `json:"isSkewed"`
			Leaders        []int `json:"leaders"`
			IsLeaderSkewed bool  `json:"isLeaderSkewed"`
		} `json:"partitionsByBroker"`
		SummedTopicOffsets          int64  `json:"summedTopicOffsets"`
		PreferredReplicasPercentage int    `json:"preferredReplicasPercentage"`
		UnderReplicatedPercentage   int    `json:"underReplicatedPercentage"`
		TopicBrokers                int    `json:"topicBrokers"`
		BrokersSkewPercentage       int    `json:"brokersSkewPercentage"`
		BrokersSpreadPercentage     int    `json:"brokersSpreadPercentage"`
		ProducerRate                string `json:"producerRate"`
	} `json:"topicIdentities"`
}

type APITopicSummary 

type APITopicSummary struct {
	TotalLag               int64    `json:"totalLag"`
	PercentageCovered      int      `json:"percentageCovered"`
	PartitionOffsets       []int64  `json:"partitionOffsets"`
	PartitionLatestOffsets []int64  `json:"partitionLatestOffsets"`
	Owners                 []string `json:"owners"`
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.