kube-promethues添加自定义指标

通过promethues-golang创建自定义指标

https://github.com/prometheus/client_golang/blob/master/examples/random/main.go
直接go run main.go运行起来即可，此时通过IP:8080/metrics可以获取到数据

添加额外监控组件配置scrape_configs

新建prometheus-additional.yaml  文件

- job_name: 'my_metrics'
  static_configs:
    - targets:
      - 172.20.16.185:8080

创建secret

kubectl create secret generic additional-scrape-configs --from-file=prometheus-additional.yaml -n monitoring

名字为additional-scrape-configs， 使用的文件为prometheus-additional.yaml 

修改promethues-promethues.yaml

该文件来自https://github.com/prometheus-operator/kube-prometheus

podMonitorNamespaceSelector: {}
podMonitorSelector: {}
//添加以下三行
additionalScrapeConfigs:      
  name: additional-scrape-configs
  key: prometheus-additional.yaml

应用到promethues

kubectl apply -f prometheus-prometheus.yaml

页面查看

promql查询并展示

promethues client_golang简单使用

大致流程为：创建指标->注册指标->定时循环获取数据

package main

import (
	"flag"
	"log"
	"math"
	"math/rand"
	"net/http"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	addr = flag.String("listen-address", ":8080", "The address to listen on for HTTP requests.")
	oscillationPeriod = flag.Duration("oscillation-period", 10*time.Minute, "The duration of the rate oscillation period.")
)

var (
	myGuage = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "myGuage",
		Help: "my test guage type",
	}, []string{"myGuage"})
)

func init() {
	// Add Go module build info.
	prometheus.MustRegister(myGuage)
}

func main() {
	flag.Parse()

	start := time.Now()

	oscillationFactor := func() float64 {
		return 2 + math.Sin(math.Sin(2*math.Pi*float64(time.Since(start))/float64(*oscillationPeriod)))
	}

	go func() {
		for {
			v := rand.ExpFloat64() / 1e6
			myGuage.WithLabelValues("myguage").Set(v)
			time.Sleep(time.Duration(50*oscillationFactor()) * time.Millisecond)

		}
	}()

	// Expose the registered metrics via HTTP.
	http.Handle("/metrics", promhttp.HandlerFor(
		prometheus.DefaultGatherer,
		promhttp.HandlerOpts{
			// Opt into OpenMetrics to support exemplars.
			EnableOpenMetrics: true,
		},
	))
	log.Fatal(http.ListenAndServe(*addr, nil))
}

查看新指标

promethues client_golang小案例

        在大多数情况下人们都倾向于使用某些量化指标的平均值，例如CPU的平均使用率、页面的平均响应时间。这种方式的问题很明显，以系统API调用的平均响应时间为例：如果大多数API请求都维持在100ms的响应时间范围内，而个别请求的响应时间需要5s，那么就会导致某些WEB页面的响应时间落到中位数的情况，而这种现象被称为长尾问题。

        为了区分是平均的慢还是长尾的慢，最简单的方式就是按照请求延迟的范围进行分组。例如，统计延迟在0~10ms之间的请求数有多少而10~20ms之间的请求数又有多少。Histogram和Summary都是为了能够解决这样问题的存在，通过Histogram和Summary类型的监控指标，我们可以快速了解监控样本的分布情况。

        此处小案例监控http请求的响应时间、请求次数、响应时间所处区间，将promethues的counter类型、guage类型、Summary类型、Histogram类型都做了演示。

• counter：只增不减的计数器
• guage：可憎可减的仪表盘
• histogram和summary用于分析数据分布
  • histogram：只展示每个区间的结果的个数，只计数，资源消耗小；也可计算分位数的结果，使用promql查询时在promethues服务端计算
  • summary：会直接计算出分位数的结果然后展示，在客户端计算

        因此对于分位数的计算而言，Summary在通过PromQL进行查询时有更好的性能表现，而Histogram则会消耗更多的资源。反之对于客户端而言Histogram消耗的资源更少。在选择这两种方式时用户应该按照自己的实际场景进行选择。

package main

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"strings"
	"time"

	"github.com/gorilla/mux"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	addr = flag.String("listen-address", ":8080", "The address to listen on for HTTP requests.")
)

var (
	requestTimes = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "request_times",
		Help: "请求次数",
	}, []string{"method", "url", "host"})
	// 多个label，需要与requestTimes.WithLabelValues(strings.ToLower(r.Method), r.URL.String())一一对应，
	// 效果为可用promql  request_times{method="get",url="/metrics",host="172.20.16.185:8080"}查询

	requestTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "request_time",
		Help: "每次请求花费的时间",
	}, []string{"method", "url", "host"})

	// 需要计算，开销大
	requestSummary = prometheus.NewSummaryVec(prometheus.SummaryOpts{
		Name:        "request_time_summary",
		Help:        "summary test",
		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
		// 0.5，0.9，0.99表示分位数，0.05，0.01，0.001表示可接收的误差，比如0.5分位的值为10，其实这个10可能是(0.45,0.55)这个区间内的某个值，太过精确会导致cpu压力过大
	}, []string{"request_time"})

	// Histogram只计数，不计算，开销小
	// 线性桶
	requestHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name:        "request_time_histogram",
		Help:        "histogram LinearBuckets test",
		Buckets:     prometheus.LinearBuckets(1, 2, 20),   // 初始为1，间隔为2，一共20个取值，Buckets会隐式添加 `+Inf` 值作为取值区间的最大值
		// request_time_histogram_bucket{instance="172.20.16.185:8080", job="my_metrics", le="1.0", request_time="request_time"}
		// request_time_histogram_bucket{instance="172.20.16.185:8080", job="my_metrics", le="3.0", request_time="request_time"}
		// ...
		// 会生成如上所示的指标，le: 小于等于
		// 即，比如第一个数据的值，表示为小于等于1.0的个数，此处即为响应时间小于等于1.0的请求的个数
	}, []string{"request_time"})

	// 默认桶
	requestHistogramDef = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name:        "request_time_histogram_def_buckets",
		Help:        "histogram DefBuckets test",
		Buckets:     prometheus.DefBuckets,
	}, []string{"request_time"})

	// 指数桶
	requestHistogramExponentialBuckets = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name:        "request_time_histogram_exponential_buckets",
		Help:        "histogram ExponentialBuckets test",
		Buckets:     prometheus.ExponentialBuckets(1, 2, 20),    // 初始值为1，指数为2增长，20个取值
	}, []string{"request_time"})
	// 不同的桶根据具体情况而定
)

func init() {
	// Add Go module build info.
	// 也可prometheus.MustRegister(xx, yy)
	prometheus.MustRegister(requestTime)
	prometheus.MustRegister(requestTimes)
	prometheus.MustRegister(requestSummary)
	prometheus.MustRegister(requestHistogram, requestHistogramDef, requestHistogramExponentialBuckets)
}

func main() {
	flag.Parse()
	r := mux.NewRouter()
	r.HandleFunc("/", handler)
	r.Use(metricsMiddleware)

	r.Handle("/metrics", promhttp.HandlerFor(
		prometheus.DefaultGatherer,
		promhttp.HandlerOpts{
			// Opt into OpenMetrics to support exemplars.
			EnableOpenMetrics: true,
		},
	))
	log.Fatal(http.ListenAndServe(*addr, r))
}


func metricsMiddleware(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		start := time.Now().Unix()
		next.ServeHTTP(w, r)
		end := time.Now().Unix()
		requestTime.WithLabelValues(strings.ToLower(r.Method), r.URL.String(), r.Host).Set(float64(end-start))
		requestTimes.WithLabelValues(strings.ToLower(r.Method), r.URL.String(), r.Host).Add(1)
		requestSummary.WithLabelValues("request_time").Observe(float64(end-start))
		requestHistogram.WithLabelValues("request_time").Observe(float64(end-start))
		requestHistogramDef.WithLabelValues("request_time").Observe(float64(end-start))
		requestHistogramExponentialBuckets.WithLabelValues("request_time").Observe(float64(end-start))
	})
}


func handler(w http.ResponseWriter, r *http.Request) {
	x := rand.Intn(3)
	time.Sleep(time.Duration(x) * time.Second)
	_, _ = w.Write([]byte("handler response"))
	fmt.Println("print handler")
}

// 清除数据，各项数据都会清零
func dataClear()  {
	requestTime.Reset()
	requestTimes.Reset()
	requestHistogram.Reset()
	requestHistogramExponentialBuckets.Reset()
	requestHistogramDef.Reset()
	requestSummary.Reset()
}

参考链接

Go进阶31:Prometheus Client教程 | ️

kube-prometheus 监控kubernetes集群 - 流年晕开时光 - 博客园
​​​​​​Metrics类型 · Prometheus中文技术文档