From bbde2e8bcc214892ad53c495b2eff2300753206a Mon Sep 17 00:00:00 2001 From: Sathvik Date: Tue, 3 Nov 2026 12:08:29 +0530 Subject: [PATCH 01/20] prometheus: serve protobuf exposition with created timestamps The handler now records when each series was created, at its first update, and serves the Prometheus protobuf exposition when a scraper ranks it first in its Accept header. Counters and histograms carry created_timestamp, so a scraper with start-timestamp zero ingestion writes a zero at that time and increase() counts a new series' first value instead of taking it as the baseline. A series re-created after MetricTimeout gets a new created time, so comebacks are counted too. Text 0.0.4 stays the default and is unchanged. Scrapers on their default protocols (Prometheus, Alloy) never rank protobuf first, so they keep receiving text. Encoding uses the generated types from prometheus/client_model, which depends only on google.golang.org/protobuf. Co-Authored-By: Claude Opus 5.5 --- go.mod | 1 + go.sum | 2 + prometheus/handler.go | 14 +- prometheus/metric.go | 12 ++ prometheus/protobuf.go | 229 +++++++++++++++++++++++++++++++ prometheus/protobuf_test.go | 264 ++++++++++++++++++++++++++++++++++++ 6 files changed, 521 insertions(+), 1 deletion(-) create mode 100644 prometheus/protobuf.go create mode 100644 prometheus/protobuf_test.go diff --git a/go.mod b/go.mod index 6d81e2f..395d196 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.0 require ( github.com/mdlayher/taskstats v0.0.0-20241219020249-a291fa5f5a69 + github.com/prometheus/client_model v0.6.2 github.com/segmentio/encoding v0.5.4 github.com/segmentio/fasthash v1.0.3 github.com/segmentio/vpcinfo v0.2.0 diff --git a/go.sum b/go.sum index 34df050..f4e3646 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,8 @@ github.com/mdlayher/socket v0.7.0 h1:qVREPVwtUMg17pwvveQxpurq0PVisMxi3FGgpsorMYQ github.com/mdlayher/socket v0.7.0/go.mod h1:f7iKql2EK/rfsWYDKofKjk2Ig7I+6HQWdMIdn1tO4A4= github.com/mdlayher/taskstats v0.0.0-20241219020249-a291fa5f5a69 h1:A33BCEd5ClzJCnUcicB1VRuRrgikbUc8J/v1qrtd2gw= github.com/mdlayher/taskstats v0.0.0-20241219020249-a291fa5f5a69/go.mod h1:0gPvvNyurVKWZFI09u1YSPCqhSnj6QwlwEijSrHXYag= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= diff --git a/prometheus/handler.go b/prometheus/handler.go index 39aca27..b608874 100644 --- a/prometheus/handler.go +++ b/prometheus/handler.go @@ -153,7 +153,15 @@ func (h *Handler) ServeHTTP(res http.ResponseWriter, req *http.Request) { } w := io.Writer(res) - res.Header().Set("Content-Type", "text/plain; version=0.0.4") + protobuf := acceptsProtobuf(req.Header.Get("Accept")) + + // Text stays the default. Only the protobuf exposition carries created + // timestamps, and a scraper gets it only by asking for it first. + if protobuf { + res.Header().Set("Content-Type", protobufContentType) + } else { + res.Header().Set("Content-Type", "text/plain; version=0.0.4") + } if acceptEncoding(req.Header.Get("Accept-Encoding"), "gzip") { res.Header().Set("Content-Encoding", "gzip") @@ -162,6 +170,10 @@ func (h *Handler) ServeHTTP(res http.ResponseWriter, req *http.Request) { w = zw } + if protobuf { + _ = h.writeProtobuf(w) + return + } h.WriteStats(w) } diff --git a/prometheus/metric.go b/prometheus/metric.go index 8975f91..8fa222a 100644 --- a/prometheus/metric.go +++ b/prometheus/metric.go @@ -311,6 +311,12 @@ type metricState struct { sum float64 count uint64 time time.Time + + // created is when the series started counting from zero, at its first + // update. The protobuf exposition publishes it so that a scraper can write a zero + // sample at that time, which lets increase() count the first value. A + // state re-created after MetricTimeout gets a new one. + created time.Time } func newMetricState(labels labels) *metricState { @@ -334,6 +340,9 @@ func (state *metricState) update(mtype metricType, value float64, time time.Time // entry than the registry slice. Comparing against len(buckets) here // would rebuild — and zero the counts — on every observation. if len(state.buckets) != len(buckets)+1 { + // A rebuild zeroes the bucket counts, but not _sum and _count. + // Moving the created time forward would make a scraper write a + // zero under a _count that kept counting, so it stays put. state.buckets = makeMetricBuckets(buckets, state.labels) } state.buckets.update(value) @@ -341,6 +350,9 @@ func (state *metricState) update(mtype metricType, value float64, time time.Time state.count++ } + if state.created.IsZero() { + state.created = time + } state.time = time state.mutex.Unlock() } diff --git a/prometheus/protobuf.go b/prometheus/protobuf.go new file mode 100644 index 0000000..f4d49b5 --- /dev/null +++ b/prometheus/protobuf.go @@ -0,0 +1,229 @@ +package prometheus + +import ( + "cmp" + "io" + "math" + "mime" + "slices" + "strconv" + "strings" + + dto "github.com/prometheus/client_model/go" + "google.golang.org/protobuf/encoding/protodelim" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// protobufContentType is the Prometheus protobuf exposition: a stream of +// length-delimited io.prometheus.client.MetricFamily messages. +const protobufContentType = "application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited" + +// acceptsProtobuf reports whether the Accept header prefers the protobuf +// exposition over the text format. +// +// Protobuf is served only when the scraper ranks it above every text type it +// accepts. Prometheus and Alloy list their scrape protocols with descending q +// values, and neither puts protobuf first unless configured to, so a scraper +// on its defaults keeps receiving text. Equal q values keep header order. +func acceptsProtobuf(accept string) bool { + best, isProto := 0.0, false + + for part := range strings.SplitSeq(accept, ",") { + mediatype, params, err := mime.ParseMediaType(strings.TrimSpace(part)) + if err != nil { + continue + } + + q := 1.0 + if s, ok := params["q"]; ok { + if q, err = strconv.ParseFloat(s, 64); err != nil { + continue + } + } + + var pb bool + switch mediatype { + case "application/vnd.google.protobuf": + if params["proto"] != "io.prometheus.client.MetricFamily" || params["encoding"] != "delimited" { + continue + } + pb = true + case "text/plain", "text/*", "*/*": + default: + continue + } + + if q > best { + best, isProto = q, pb + } + } + + return isProto +} + +// writeProtobuf writes every metric as a length-delimited MetricFamily. +// +// Counters and histograms carry created_timestamp, the time their series +// started from zero. A scraper with start-timestamp zero ingestion enabled +// writes a zero sample at that time, so the first value of a new series is +// counted by increase() instead of being taken as its baseline. +// +// Names and labels are rendered exactly as WriteStats renders them, so +// switching format does not change any series. +func (h *Handler) writeProtobuf(w io.Writer) error { + for _, mf := range h.metrics.families() { + if _, err := protodelim.MarshalTo(w, mf); err != nil { + return err + } + } + return nil +} + +type protoFamilyKey struct { + name string + mtype metricType +} + +type protoSeries struct { + labels labels + metric *dto.Metric +} + +type protoFamily struct { + family *dto.MetricFamily + series []protoSeries +} + +func (store *metricStore) families() []*dto.MetricFamily { + byKey := make(map[protoFamilyKey]*protoFamily) + + store.mutex.RLock() + + for key, entry := range store.entries { + var mtype dto.MetricType + switch entry.mtype { + case counter: + mtype = dto.MetricType_COUNTER + case gauge: + mtype = dto.MetricType_GAUGE + case histogram: + mtype = dto.MetricType_HISTOGRAM + default: + // WriteStats publishes only these three types. + continue + } + + // Two entries can render to the same name, a.b and a_b for example. + // The text format declares the family once and lists both under it, + // so they are merged here too. + name := string(appendMetricScopedName(nil, entry.scope, store.exposedName(key, entry))) + fk := protoFamilyKey{name: name, mtype: entry.mtype} + + f := byKey[fk] + if f == nil { + f = &protoFamily{family: &dto.MetricFamily{Name: proto.String(name), Type: mtype.Enum()}} + byKey[fk] = f + } + + entry.mutex.RLock() + for _, states := range entry.states { + for _, state := range states { + f.series = append(f.series, protoSeries{ + labels: state.labels, + metric: state.protoMetric(entry.mtype), + }) + } + } + entry.mutex.RUnlock() + } + + store.mutex.RUnlock() + + families := make([]*dto.MetricFamily, 0, len(byKey)) + for _, f := range byKey { + if len(f.series) == 0 { + continue + } + slices.SortFunc(f.series, func(a, b protoSeries) int { + switch { + case a.labels.less(b.labels): + return -1 + case b.labels.less(a.labels): + return 1 + } + return 0 + }) + for _, s := range f.series { + f.family.Metric = append(f.family.Metric, s.metric) + } + families = append(families, f.family) + } + + slices.SortFunc(families, func(a, b *dto.MetricFamily) int { + return cmp.Compare(a.GetName(), b.GetName()) + }) + return families +} + +func (state *metricState) protoMetric(mtype metricType) *dto.Metric { + state.mutex.Lock() + defer state.mutex.Unlock() + + m := &dto.Metric{Label: protoLabels(state.labels)} + + switch mtype { + case counter: + m.Counter = &dto.Counter{ + Value: proto.Float64(state.value), + CreatedTimestamp: protoCreated(state), + } + + case gauge: + m.Gauge = &dto.Gauge{Value: proto.Float64(state.value)} + + case histogram: + // The +Inf bucket is left out, as client_golang does: its count is + // sample_count, and the scraper adds the le="+Inf" series itself. + h := &dto.Histogram{ + SampleCount: proto.Uint64(state.count), + SampleSum: proto.Float64(state.sum), + CreatedTimestamp: protoCreated(state), + } + var cumulativeCount uint64 + for _, bucket := range state.buckets { + cumulativeCount += bucket.count + if math.IsInf(bucket.limit, 1) { + continue + } + h.Bucket = append(h.Bucket, &dto.Bucket{ + CumulativeCount: proto.Uint64(cumulativeCount), + UpperBound: proto.Float64(bucket.limit), + }) + } + m.Histogram = h + } + + return m +} + +func protoCreated(state *metricState) *timestamppb.Timestamp { + if state.created.IsZero() { + return nil + } + return timestamppb.New(state.created) +} + +func protoLabels(l labels) []*dto.LabelPair { + if len(l) == 0 { + return nil + } + pairs := make([]*dto.LabelPair, len(l)) + for i, label := range l { + pairs[i] = &dto.LabelPair{ + Name: proto.String(string(appendLabelName(nil, label.name))), + Value: proto.String(label.value), + } + } + return pairs +} diff --git a/prometheus/protobuf_test.go b/prometheus/protobuf_test.go new file mode 100644 index 0000000..2578477 --- /dev/null +++ b/prometheus/protobuf_test.go @@ -0,0 +1,264 @@ +package prometheus + +import ( + "bufio" + "compress/gzip" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + dto "github.com/prometheus/client_model/go" + "google.golang.org/protobuf/encoding/protodelim" + + "github.com/segmentio/stats/v5" +) + +// The Accept header Alloy sends with scrape_protocols set to +// [PrometheusProto, OpenMetricsText1.0.0, OpenMetricsText0.0.1, PrometheusText0.0.4]. +const acceptProtoFirst = "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.5," + + "application/openmetrics-text;version=1.0.0;q=0.4,application/openmetrics-text;version=0.0.1;q=0.3," + + "text/plain;version=0.0.4;q=0.2,*/*;q=0.1" + +func TestAcceptsProtobuf(t *testing.T) { + tests := []struct { + name string + accept string + expect bool + }{ + {name: "none", accept: "", expect: false}, + {name: "curl", accept: "*/*", expect: false}, + {name: "protobuf first", accept: acceptProtoFirst, expect: true}, + { + name: "scraper defaults", + accept: "application/openmetrics-text;version=1.0.0;q=0.5,application/openmetrics-text;version=0.0.1;q=0.4,text/plain;version=0.0.4;q=0.3,*/*;q=0.2", + expect: false, + }, + { + name: "protobuf ranked below text", + accept: "text/plain;version=0.0.4;q=0.5,application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.4", + expect: false, + }, + { + name: "protobuf only", + accept: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited", + expect: true, + }, + { + name: "protobuf refused", + accept: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0", + expect: false, + }, + { + name: "other message", + accept: "application/vnd.google.protobuf;proto=io.prometheus.client.Other;encoding=delimited", + expect: false, + }, + { + name: "not delimited", + accept: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=text", + expect: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if ok := acceptsProtobuf(test.accept); ok != test.expect { + t.Errorf("acceptsProtobuf(%q) = %t, expected %t", test.accept, ok, test.expect) + } + }) + } +} + +func scrapeProtobuf(t *testing.T, handler *Handler, gzipped bool) []*dto.MetricFamily { + t.Helper() + + server := httptest.NewServer(handler) + defer server.Close() + + req, err := http.NewRequest("GET", server.URL, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Accept", acceptProtoFirst) + if gzipped { + req.Header.Set("Accept-Encoding", "gzip") + } + + // A Transport left to itself would negotiate and undo gzip on its own. + res, err := (&http.Transport{DisableCompression: true}).RoundTrip(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + + if ct := res.Header.Get("Content-Type"); ct != protobufContentType { + t.Fatalf("Content-Type = %q, expected %q", ct, protobufContentType) + } + + body := io.Reader(res.Body) + if gzipped { + if ce := res.Header.Get("Content-Encoding"); ce != "gzip" { + t.Fatalf("Content-Encoding = %q, expected gzip", ce) + } + zr, err := gzip.NewReader(res.Body) + if err != nil { + t.Fatal(err) + } + body = zr + } + + var families []*dto.MetricFamily + r := bufio.NewReader(body) + for { + mf := &dto.MetricFamily{} + if err := protodelim.UnmarshalFrom(r, mf); err != nil { + if errors.Is(err, io.EOF) { + return families + } + t.Fatal(err) + } + families = append(families, mf) + } +} + +func TestServeHTTPProtobuf(t *testing.T) { + first := time.Date(2017, 6, 4, 22, 12, 0, 0, time.UTC) + later := first.Add(30 * time.Second) + + handler := &Handler{ + Buckets: map[stats.Key][]stats.Value{ + {Field: "C"}: { + stats.ValueOf(0.25), + stats.ValueOf(0.5), + stats.ValueOf(0.75), + stats.ValueOf(1.0), + }, + }, + } + + handler.HandleMeasures(first, + stats.Measure{Fields: []stats.Field{stats.MakeField("A", 1, stats.Counter)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("C", 0.1, stats.Histogram)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("B", 1, stats.Gauge)}, Tags: []stats.Tag{stats.T("a", "1"), stats.T("b", "2")}}, + ) + handler.HandleMeasures(later, + stats.Measure{Fields: []stats.Field{stats.MakeField("A", 2, stats.Counter)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("A", 4, stats.Counter)}, Tags: []stats.Tag{stats.T("id", "123")}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("B", 42, stats.Gauge)}, Tags: []stats.Tag{stats.T("a", "1")}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("C", 0.1, stats.Histogram)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("C", 0.5, stats.Histogram)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("C", 10, stats.Histogram)}}, + ) + + for _, gzipped := range []bool{false, true} { + families := scrapeProtobuf(t, handler, gzipped) + + if len(families) != 3 { + t.Fatalf("found %d families, expected 3: %v", len(families), families) + } + a, b, c := families[0], families[1], families[2] + + // Counter: the name keeps _total, and each series carries the time of + // its own first update. + if a.GetName() != "A_total" || a.GetType() != dto.MetricType_COUNTER || len(a.Metric) != 2 { + t.Fatalf("bad counter family: %v", a) + } + if m := a.Metric[0]; len(m.Label) != 0 || m.GetCounter().GetValue() != 3 || + !m.GetCounter().GetCreatedTimestamp().AsTime().Equal(first) { + t.Errorf("bad unlabeled counter: %v", m) + } + if m := a.Metric[1]; m.Label[0].GetName() != "id" || m.Label[0].GetValue() != "123" || + m.GetCounter().GetValue() != 4 || !m.GetCounter().GetCreatedTimestamp().AsTime().Equal(later) { + t.Errorf("bad labeled counter: %v", m) + } + + // Gauge: no created timestamp. + if b.GetName() != "B" || b.GetType() != dto.MetricType_GAUGE || len(b.Metric) != 2 { + t.Fatalf("bad gauge family: %v", b) + } + if m := b.Metric[0]; m.GetGauge().GetValue() != 42 || len(m.Label) != 1 { + t.Errorf("bad gauge: %v", m) + } + if m := b.Metric[1]; m.GetGauge().GetValue() != 1 || len(m.Label) != 2 { + t.Errorf("bad gauge: %v", m) + } + + // Histogram: cumulative buckets without +Inf, which the scraper + // derives from sample_count. + if c.GetName() != "C" || c.GetType() != dto.MetricType_HISTOGRAM || len(c.Metric) != 1 { + t.Fatalf("bad histogram family: %v", c) + } + h := c.Metric[0].GetHistogram() + if h.GetSampleCount() != 4 || h.GetSampleSum() != 10.7 || + !h.GetCreatedTimestamp().AsTime().Equal(first) { + t.Errorf("bad histogram: %v", h) + } + expect := []struct { + le float64 + count uint64 + }{{0.25, 2}, {0.5, 3}, {0.75, 3}, {1, 3}} + if len(h.Bucket) != len(expect) { + t.Fatalf("found %d buckets, expected %d: %v", len(h.Bucket), len(expect), h.Bucket) + } + for i, e := range expect { + if h.Bucket[i].GetUpperBound() != e.le || h.Bucket[i].GetCumulativeCount() != e.count { + t.Errorf("bucket %d = %v, expected le=%g count=%d", i, h.Bucket[i], e.le, e.count) + } + } + } +} + +// TestCreatedTimestampAfterTimeout covers a comeback: a series MetricTimeout +// deleted and that is then updated again starts from zero, so it has to carry +// a new created time or the scraper has no zero to write for it. +func TestCreatedTimestampAfterTimeout(t *testing.T) { + first := time.Date(2017, 6, 4, 22, 12, 0, 0, time.UTC) + comeback := first.Add(10 * time.Minute) + + handler := &Handler{} + handler.HandleMeasures(first, stats.Measure{Fields: []stats.Field{stats.MakeField("A", 1, stats.Counter)}}) + handler.metrics.cleanup(first.Add(time.Minute)) + handler.HandleMeasures(comeback, stats.Measure{Fields: []stats.Field{stats.MakeField("A", 1, stats.Counter)}}) + + families := scrapeProtobuf(t, handler, false) + if len(families) != 1 || len(families[0].Metric) != 1 { + t.Fatalf("unexpected families: %v", families) + } + c := families[0].Metric[0].GetCounter() + if c.GetValue() != 1 || !c.GetCreatedTimestamp().AsTime().Equal(comeback) { + t.Errorf("counter after timeout = %v, expected value 1 created at %s", c, comeback) + } +} + +// TestProtobufMatchesTextNames checks that the protobuf exposition renders +// family and label names the way the text format does, so a scraper that +// switches format keeps writing to the same series. +func TestProtobufMatchesTextNames(t *testing.T) { + now := time.Date(2017, 6, 4, 22, 12, 0, 0, time.UTC) + + handler := &Handler{} + handler.HandleMeasures(now, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("requests.total", 1, stats.Counter)}, Tags: []stats.Tag{stats.T("status.code", "200")}}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("hits", 1, stats.Counter)}}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("hits_total", 1, stats.Counter)}}, + ) + + var text strings.Builder + handler.WriteStats(&text) + + for _, mf := range scrapeProtobuf(t, handler, false) { + for _, m := range mf.Metric { + series := mf.GetName() + if len(m.Label) != 0 { + series += "{" + m.Label[0].GetName() + "=\"" + m.Label[0].GetValue() + "\"}" + } + if !strings.Contains(text.String(), "\n"+series+" ") && !strings.HasPrefix(text.String(), series+" ") { + t.Errorf("protobuf series %s not in text output:\n%s", series, text.String()) + } + } + } +} From 64da5c6453f67d8151b8ec37104824ebbeac1fc2 Mon Sep 17 00:00:00 2001 From: Sathvik Date: Tue, 3 Nov 2026 13:26:46 +0530 Subject: [PATCH 02/20] HISTORY: add v5.13.0 entry for protobuf exposition Co-Authored-By: Claude Opus 5.5 --- HISTORY.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 802b74b..104790b 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,31 @@ # History +### v5.13.0 + +**The `prometheus` handler can serve the protobuf exposition, with created +timestamps.** `increase()` and `rate()` never count a counter's first sample: +it is taken as the starting point. A series that starts at 5 and stays there +shows an increase of 0, and a series that goes idle long enough for +`MetricTimeout` to delete it loses its first update again when it comes back. + +The handler now records when each counter and histogram series started from +zero (its first update, or its first update after `MetricTimeout` deleted it) +and publishes it as `created_timestamp` in the protobuf exposition +(`application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; +encoding=delimited`). A scraper with created-timestamp zero ingestion enabled +(Prometheus `created-timestamp-zero-ingestion`, Alloy +`start_timestamp_zero_ingestion`) writes a zero at that time, so the first +update is counted. + +Protobuf is served only to a scraper that ranks it above every text type in +its `Accept` header, which Prometheus and Alloy do only when configured to. +Every other scraper keeps getting the text format, unchanged. Both formats +render the same metric and label names, so a scraper that switches format +keeps writing to the same series. Sample timestamps are still left to the +scraper. Gauges carry no created timestamp. + +The module now depends on `github.com/prometheus/client_model`. + ### v5.12.0 **`procstats` now reads CPU and memory limits on cgroup v2.** It only knew the From 04d508bac761910a637b33e5092a356baf3abaf2 Mon Sep 17 00:00:00 2001 From: Sathvik Date: Tue, 3 Nov 2026 13:47:55 +0530 Subject: [PATCH 03/20] prometheus: say which scrapers put protobuf first by default Co-Authored-By: Claude Opus 5.5 --- HISTORY.md | 8 ++++++-- prometheus/protobuf.go | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 104790b..abdeef7 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -18,8 +18,12 @@ encoding=delimited`). A scraper with created-timestamp zero ingestion enabled update is counted. Protobuf is served only to a scraper that ranks it above every text type in -its `Accept` header, which Prometheus and Alloy do only when configured to. -Every other scraper keeps getting the text format, unchanged. Both formats +its `Accept` header. That is a scraper whose scrape protocols put +`PrometheusProto` first, or a Prometheus with native histograms or +created-timestamp zero ingestion on, which puts it first by default: such a +Prometheus switches to protobuf when the application upgrades, with no change +to its configuration. Every other scraper keeps getting the text format, +unchanged. Both formats render the same metric and label names, so a scraper that switches format keeps writing to the same series. Sample timestamps are still left to the scraper. Gauges carry no created timestamp. diff --git a/prometheus/protobuf.go b/prometheus/protobuf.go index f4d49b5..7e3ae41 100644 --- a/prometheus/protobuf.go +++ b/prometheus/protobuf.go @@ -24,8 +24,10 @@ const protobufContentType = "application/vnd.google.protobuf; proto=io.prometheu // // Protobuf is served only when the scraper ranks it above every text type it // accepts. Prometheus and Alloy list their scrape protocols with descending q -// values, and neither puts protobuf first unless configured to, so a scraper -// on its defaults keeps receiving text. Equal q values keep header order. +// values. A scraper puts protobuf first when its scrape protocols are set that +// way, and Prometheus also does so by default when native histograms or +// created-timestamp zero ingestion are on. Any other scraper keeps receiving +// text. Equal q values keep header order. func acceptsProtobuf(accept string) bool { best, isProto := 0.0, false From 9ae25aada46afb145c44a639c55415561c32dd4d Mon Sep 17 00:00:00 2001 From: Sathvik Date: Tue, 3 Nov 2026 13:55:19 +0530 Subject: [PATCH 04/20] prometheus: say the Accept ranking ignores formats the handler does not serve Co-Authored-By: Claude Opus 5.5 --- prometheus/protobuf.go | 5 +++-- prometheus/protobuf_test.go | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/prometheus/protobuf.go b/prometheus/protobuf.go index 7e3ae41..3499ddc 100644 --- a/prometheus/protobuf.go +++ b/prometheus/protobuf.go @@ -22,8 +22,9 @@ const protobufContentType = "application/vnd.google.protobuf; proto=io.prometheu // acceptsProtobuf reports whether the Accept header prefers the protobuf // exposition over the text format. // -// Protobuf is served only when the scraper ranks it above every text type it -// accepts. Prometheus and Alloy list their scrape protocols with descending q +// Protobuf is served only when the scraper ranks it above the text format. +// Those are the only two formats the handler serves, so other types, such as +// OpenMetrics, are not ranked. Prometheus and Alloy list their scrape protocols with descending q // values. A scraper puts protobuf first when its scrape protocols are set that // way, and Prometheus also does so by default when native histograms or // created-timestamp zero ingestion are on. Any other scraper keeps receiving diff --git a/prometheus/protobuf_test.go b/prometheus/protobuf_test.go index 2578477..8a3a865 100644 --- a/prometheus/protobuf_test.go +++ b/prometheus/protobuf_test.go @@ -42,6 +42,11 @@ func TestAcceptsProtobuf(t *testing.T) { accept: "text/plain;version=0.0.4;q=0.5,application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.4", expect: false, }, + { + name: "openmetrics above protobuf", + accept: "application/openmetrics-text;version=1.0.0;q=0.5,application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.4,text/plain;version=0.0.4;q=0.3", + expect: true, + }, { name: "protobuf only", accept: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited", From 1ccab1949085d2a1f32b6c4904c235a930b107a3 Mon Sep 17 00:00:00 2001 From: Sathvik Date: Tue, 3 Nov 2026 17:21:27 +0530 Subject: [PATCH 05/20] prometheus: build protobuf messages after releasing the store lock families held store.mutex.RLock while it built every dto message. A writer waiting on the store (a new metric name, or cleanup) then blocked every HandleMeasures call queued behind it for the length of the build. The series are now copied out under the locks, as collect does, and the messages are built after. Co-Authored-By: Claude Opus 5.5 --- prometheus/protobuf.go | 143 ++++++++++++++++++++++++++---------- prometheus/protobuf_test.go | 55 ++++++++++++++ 2 files changed, 158 insertions(+), 40 deletions(-) diff --git a/prometheus/protobuf.go b/prometheus/protobuf.go index 3499ddc..90f9571 100644 --- a/prometheus/protobuf.go +++ b/prometheus/protobuf.go @@ -8,6 +8,7 @@ import ( "slices" "strconv" "strings" + "time" dto "github.com/prometheus/client_model/go" "google.golang.org/protobuf/encoding/protodelim" @@ -88,22 +89,38 @@ type protoFamilyKey struct { mtype metricType } -type protoSeries struct { - labels labels - metric *dto.Metric -} - type protoFamily struct { family *dto.MetricFamily series []protoSeries } +// protoEntry is a store entry copied out under the locks, so that the +// messages can be built after they are released. +type protoEntry struct { + scope string + name string + mtype metricType + series []protoSeries +} + +type protoSeries struct { + labels labels + value float64 + sum float64 + count uint64 + created time.Time + buckets []protoBucket +} + +type protoBucket struct { + limit float64 + count uint64 +} + func (store *metricStore) families() []*dto.MetricFamily { byKey := make(map[protoFamilyKey]*protoFamily) - store.mutex.RLock() - - for key, entry := range store.entries { + for _, entry := range store.protoEntries() { var mtype dto.MetricType switch entry.mtype { case counter: @@ -112,15 +129,12 @@ func (store *metricStore) families() []*dto.MetricFamily { mtype = dto.MetricType_GAUGE case histogram: mtype = dto.MetricType_HISTOGRAM - default: - // WriteStats publishes only these three types. - continue } // Two entries can render to the same name, a.b and a_b for example. // The text format declares the family once and lists both under it, // so they are merged here too. - name := string(appendMetricScopedName(nil, entry.scope, store.exposedName(key, entry))) + name := string(appendMetricScopedName(nil, entry.scope, entry.name)) fk := protoFamilyKey{name: name, mtype: entry.mtype} f := byKey[fk] @@ -128,23 +142,11 @@ func (store *metricStore) families() []*dto.MetricFamily { f = &protoFamily{family: &dto.MetricFamily{Name: proto.String(name), Type: mtype.Enum()}} byKey[fk] = f } - - entry.mutex.RLock() - for _, states := range entry.states { - for _, state := range states { - f.series = append(f.series, protoSeries{ - labels: state.labels, - metric: state.protoMetric(entry.mtype), - }) - } - } - entry.mutex.RUnlock() + f.series = append(f.series, entry.series...) } - store.mutex.RUnlock() - families := make([]*dto.MetricFamily, 0, len(byKey)) - for _, f := range byKey { + for fk, f := range byKey { if len(f.series) == 0 { continue } @@ -157,8 +159,9 @@ func (store *metricStore) families() []*dto.MetricFamily { } return 0 }) - for _, s := range f.series { - f.family.Metric = append(f.family.Metric, s.metric) + f.family.Metric = make([]*dto.Metric, len(f.series)) + for i := range f.series { + f.family.Metric[i] = f.series[i].protoMetric(fk.mtype) } families = append(families, f.family) } @@ -169,32 +172,92 @@ func (store *metricStore) families() []*dto.MetricFamily { return families } -func (state *metricState) protoMetric(mtype metricType) *dto.Metric { +// protoEntries copies out every series the protobuf exposition publishes. +// +// The locks are held only for the copy, as collect holds them. A writer +// waiting on store.mutex blocks every reader queued behind it, HandleMeasures +// included, so building the messages under the read lock would stall the +// application for as long as the build takes. +func (store *metricStore) protoEntries() []protoEntry { + // The bucket copies share one backing array, so that copying a histogram + // does not cost an allocation per series. + var buckets []protoBucket + + store.mutex.RLock() + entries := make([]protoEntry, 0, len(store.entries)) + + for key, entry := range store.entries { + switch entry.mtype { + case counter, gauge, histogram: + default: + // WriteStats publishes only these three types. + continue + } + + e := protoEntry{scope: entry.scope, name: store.exposedName(key, entry), mtype: entry.mtype} + + entry.mutex.RLock() + for _, states := range entry.states { + for _, state := range states { + var series protoSeries + series, buckets = state.protoSeries(buckets) + e.series = append(e.series, series) + } + } + entry.mutex.RUnlock() + + entries = append(entries, e) + } + + store.mutex.RUnlock() + return entries +} + +// protoSeries copies the state, appending its buckets to arena. +func (state *metricState) protoSeries(arena []protoBucket) (protoSeries, []protoBucket) { state.mutex.Lock() defer state.mutex.Unlock() - m := &dto.Metric{Label: protoLabels(state.labels)} + s := protoSeries{ + labels: state.labels, + value: state.value, + sum: state.sum, + count: state.count, + created: state.created, + } + if len(state.buckets) != 0 { + start := len(arena) + for _, bucket := range state.buckets { + arena = append(arena, protoBucket{limit: bucket.limit, count: bucket.count}) + } + s.buckets = arena[start:len(arena):len(arena)] + } + return s, arena +} + +func (s *protoSeries) protoMetric(mtype metricType) *dto.Metric { + m := &dto.Metric{Label: protoLabels(s.labels)} switch mtype { case counter: m.Counter = &dto.Counter{ - Value: proto.Float64(state.value), - CreatedTimestamp: protoCreated(state), + Value: proto.Float64(s.value), + CreatedTimestamp: protoCreated(s.created), } case gauge: - m.Gauge = &dto.Gauge{Value: proto.Float64(state.value)} + m.Gauge = &dto.Gauge{Value: proto.Float64(s.value)} case histogram: // The +Inf bucket is left out, as client_golang does: its count is // sample_count, and the scraper adds the le="+Inf" series itself. h := &dto.Histogram{ - SampleCount: proto.Uint64(state.count), - SampleSum: proto.Float64(state.sum), - CreatedTimestamp: protoCreated(state), + SampleCount: proto.Uint64(s.count), + SampleSum: proto.Float64(s.sum), + CreatedTimestamp: protoCreated(s.created), } var cumulativeCount uint64 - for _, bucket := range state.buckets { + for _, bucket := range s.buckets { cumulativeCount += bucket.count if math.IsInf(bucket.limit, 1) { continue @@ -210,11 +273,11 @@ func (state *metricState) protoMetric(mtype metricType) *dto.Metric { return m } -func protoCreated(state *metricState) *timestamppb.Timestamp { - if state.created.IsZero() { +func protoCreated(created time.Time) *timestamppb.Timestamp { + if created.IsZero() { return nil } - return timestamppb.New(state.created) + return timestamppb.New(created) } func protoLabels(l labels) []*dto.LabelPair { diff --git a/prometheus/protobuf_test.go b/prometheus/protobuf_test.go index 8a3a865..ad4e8d2 100644 --- a/prometheus/protobuf_test.go +++ b/prometheus/protobuf_test.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "net/http/httptest" + "strconv" "strings" "testing" "time" @@ -267,3 +268,57 @@ func TestProtobufMatchesTextNames(t *testing.T) { } } } + +// benchmarkHandler returns a handler holding n series: counters with three +// labels, and one histogram series for every ten of them. +func benchmarkHandler(n int) *Handler { + handler := &Handler{} + now := time.Now() + for i := range n { + tags := []stats.Tag{ + stats.T("method", "GET"), + stats.T("status_code", strconv.Itoa(200+i%5)), + stats.T("url", "/v1/workspaces/"+strconv.Itoa(i)), + } + handler.HandleMeasures(now, stats.Measure{ + Name: "http", + Fields: []stats.Field{stats.MakeField("requests", 1, stats.Counter)}, + Tags: tags, + }) + if i%10 == 0 { + handler.HandleMeasures(now, stats.Measure{ + Name: "http", + Fields: []stats.Field{stats.MakeField("duration", 0.1, stats.Histogram)}, + Tags: tags, + }) + } + } + return handler +} + +// BenchmarkCollect is the text path's time under the store lock. +func BenchmarkCollect(b *testing.B) { + handler := benchmarkHandler(100000) + b.ReportAllocs() + for b.Loop() { + handler.metrics.collect(make([]metric, 0, 10000)) + } +} + +// BenchmarkFamilies is the whole protobuf build. +func BenchmarkFamilies(b *testing.B) { + handler := benchmarkHandler(100000) + b.ReportAllocs() + for b.Loop() { + handler.metrics.families() + } +} + +// BenchmarkProtoEntries is the protobuf path's time under the store lock. +func BenchmarkProtoEntries(b *testing.B) { + handler := benchmarkHandler(100000) + b.ReportAllocs() + for b.Loop() { + handler.metrics.protoEntries() + } +} From c8f1cb398ff73565d31883ad9a9bc679a7cb2d91 Mon Sep 17 00:00:00 2001 From: Sathvik Date: Tue, 3 Nov 2026 17:23:20 +0530 Subject: [PATCH 06/20] prometheus: limit the same-series claim to Prometheus 3.x Co-Authored-By: Claude Opus 5.5 --- HISTORY.md | 14 +++++++++----- prometheus/protobuf.go | 5 ++++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index abdeef7..06e8331 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -17,16 +17,20 @@ encoding=delimited`). A scraper with created-timestamp zero ingestion enabled `start_timestamp_zero_ingestion`) writes a zero at that time, so the first update is counted. -Protobuf is served only to a scraper that ranks it above every text type in +Protobuf is served only to a scraper that ranks it above the text format in its `Accept` header. That is a scraper whose scrape protocols put `PrometheusProto` first, or a Prometheus with native histograms or created-timestamp zero ingestion on, which puts it first by default: such a Prometheus switches to protobuf when the application upgrades, with no change to its configuration. Every other scraper keeps getting the text format, -unchanged. Both formats -render the same metric and label names, so a scraper that switches format -keeps writing to the same series. Sample timestamps are still left to the -scraper. Gauges carry no created timestamp. +unchanged. Sample timestamps are still left to the scraper. Gauges carry no +created timestamp. + +Both formats render the same metric and label names, so a Prometheus 3.x +scraper that switches format keeps writing to the same series. On Prometheus +2.x, histogram buckets with integer bounds start new series: the scraper +formats the bound of a protobuf bucket as `le="1.0"`, while the text format +writes `le="1"`, and only 3.x normalizes the text value to match. The module now depends on `github.com/prometheus/client_model`. diff --git a/prometheus/protobuf.go b/prometheus/protobuf.go index 90f9571..2728a52 100644 --- a/prometheus/protobuf.go +++ b/prometheus/protobuf.go @@ -74,7 +74,10 @@ func acceptsProtobuf(accept string) bool { // counted by increase() instead of being taken as its baseline. // // Names and labels are rendered exactly as WriteStats renders them, so -// switching format does not change any series. +// switching format does not change any series on Prometheus 3.x. Histogram +// bounds are the exception on 2.x: the scraper formats UpperBound itself, as +// le="1.0" where the text format writes le="1", and only 3.x normalizes the +// text value to match. func (h *Handler) writeProtobuf(w io.Writer) error { for _, mf := range h.metrics.families() { if _, err := protodelim.MarshalTo(w, mf); err != nil { From ac502e6e6631706a30b7110820c6478bd229c620 Mon Sep 17 00:00:00 2001 From: Sathvik Date: Tue, 3 Nov 2026 17:29:43 +0530 Subject: [PATCH 07/20] prometheus: compare full series sets between protobuf and text The parity test read only a series' first label and matched it as a substring, so histograms, gauges and further labels were never compared. It now expands both outputs into the series a scraper stores, with full label sets and values, and requires them to be equal. Co-Authored-By: Claude Opus 5.5 --- prometheus/protobuf_test.go | 155 +++++++++++++++++++++++++++++++++--- 1 file changed, 142 insertions(+), 13 deletions(-) diff --git a/prometheus/protobuf_test.go b/prometheus/protobuf_test.go index ad4e8d2..2221b70 100644 --- a/prometheus/protobuf_test.go +++ b/prometheus/protobuf_test.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "strconv" "strings" "testing" @@ -240,33 +241,161 @@ func TestCreatedTimestampAfterTimeout(t *testing.T) { } } -// TestProtobufMatchesTextNames checks that the protobuf exposition renders -// family and label names the way the text format does, so a scraper that -// switches format keeps writing to the same series. -func TestProtobufMatchesTextNames(t *testing.T) { +// TestProtobufMatchesText checks that a scraper ingests the same series from +// the protobuf exposition as from the text format: the same names, label sets +// and values, so a scraper that switches format keeps writing to the same +// series. +// +// le is compared as a number. The scraper formats a protobuf bound itself, as +// le="1.0" where the text format writes le="1", and Prometheus 3.x normalizes +// the text value to the same form. +func TestProtobufMatchesText(t *testing.T) { now := time.Date(2017, 6, 4, 22, 12, 0, 0, time.UTC) - handler := &Handler{} + handler := &Handler{ + Buckets: map[stats.Key][]stats.Value{ + {Measure: "http.server", Field: "duration"}: { + stats.ValueOf(0.25), + stats.ValueOf(1), + stats.ValueOf(5), + }, + }, + } + tags := []stats.Tag{stats.T("method", "GET"), stats.T("status.code", "200")} handler.HandleMeasures(now, - stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("requests.total", 1, stats.Counter)}, Tags: []stats.Tag{stats.T("status.code", "200")}}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("requests.total", 3, stats.Counter)}, Tags: tags}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("requests.total", 1, stats.Counter)}, Tags: []stats.Tag{stats.T("method", "POST")}}, stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("hits", 1, stats.Counter)}}, - stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("hits_total", 1, stats.Counter)}}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("hits_total", 2, stats.Counter)}}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("inflight", 7, stats.Gauge)}, Tags: tags}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("duration", 0.1, stats.Histogram)}, Tags: tags}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("duration", 2, stats.Histogram)}, Tags: tags}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("duration", 10, stats.Histogram)}, Tags: tags}, ) var text strings.Builder handler.WriteStats(&text) + fromText := textSeries(t, text.String()) + fromProto := protoSeriesSet(scrapeProtobuf(t, handler, false)) + + for series, value := range fromText { + if v, ok := fromProto[series]; !ok { + t.Errorf("text series %s missing from protobuf", series) + } else if v != value { + t.Errorf("series %s = %g in protobuf, %g in text", series, v, value) + } + } + for series := range fromProto { + if _, ok := fromText[series]; !ok { + t.Errorf("protobuf series %s missing from text", series) + } + } + if t.Failed() { + t.Log("text output:\n" + text.String()) + } +} + +// seriesKey renders a series as name{label="value",...} with the labels +// sorted and le normalized, so the same series is the same string whichever +// format it came from. +func seriesKey(name string, labels map[string]string) string { + names := make([]string, 0, len(labels)) + for n := range labels { + names = append(names, n) + } + slices.Sort(names) + + var b strings.Builder + b.WriteString(name) + b.WriteByte('{') + for i, n := range names { + if i != 0 { + b.WriteByte(',') + } + v := labels[n] + if n == "le" { + if f, err := strconv.ParseFloat(v, 64); err == nil { + v = strconv.FormatFloat(f, 'g', -1, 64) + } + } + b.WriteString(n + "=" + strconv.Quote(v)) + } + b.WriteByte('}') + return b.String() +} + +// textSeries parses the text exposition into its series and values. +func textSeries(t *testing.T, text string) map[string]float64 { + t.Helper() + set := make(map[string]float64) + + for line := range strings.SplitSeq(text, "\n") { + if line == "" || strings.HasPrefix(line, "#") { + continue + } + sep := strings.LastIndexByte(line, ' ') + value, err := strconv.ParseFloat(line[sep+1:], 64) + if err != nil { + t.Fatalf("bad value in %q: %v", line, err) + } + series := line[:sep] + + name, labels := series, map[string]string{} + if i := strings.IndexByte(series, '{'); i >= 0 { + name = series[:i] + rest := strings.TrimSuffix(series[i+1:], "}") + for rest != "" { + eq := strings.IndexByte(rest, '=') + v, err := strconv.QuotedPrefix(rest[eq+1:]) + if err != nil { + t.Fatalf("bad label in %q: %v", line, err) + } + labels[rest[:eq]], _ = strconv.Unquote(v) + rest = strings.TrimPrefix(rest[eq+1+len(v):], ",") + } + } + set[seriesKey(name, labels)] = value + } + return set +} - for _, mf := range scrapeProtobuf(t, handler, false) { +// protoSeriesSet expands decoded families into the series a scraper stores +// for them: one per counter and gauge, and _bucket (with +Inf from +// sample_count), _sum and _count per histogram. +func protoSeriesSet(families []*dto.MetricFamily) map[string]float64 { + set := make(map[string]float64) + + for _, mf := range families { for _, m := range mf.Metric { - series := mf.GetName() - if len(m.Label) != 0 { - series += "{" + m.Label[0].GetName() + "=\"" + m.Label[0].GetValue() + "\"}" + labels := func(extra ...string) map[string]string { + l := make(map[string]string, len(m.Label)+1) + for _, p := range m.Label { + l[p.GetName()] = p.GetValue() + } + if len(extra) == 2 { + l[extra[0]] = extra[1] + } + return l } - if !strings.Contains(text.String(), "\n"+series+" ") && !strings.HasPrefix(text.String(), series+" ") { - t.Errorf("protobuf series %s not in text output:\n%s", series, text.String()) + + switch mf.GetType() { + case dto.MetricType_COUNTER: + set[seriesKey(mf.GetName(), labels())] = m.GetCounter().GetValue() + case dto.MetricType_GAUGE: + set[seriesKey(mf.GetName(), labels())] = m.GetGauge().GetValue() + case dto.MetricType_HISTOGRAM: + h := m.GetHistogram() + for _, b := range h.Bucket { + le := strconv.FormatFloat(b.GetUpperBound(), 'g', -1, 64) + set[seriesKey(mf.GetName()+"_bucket", labels("le", le))] = float64(b.GetCumulativeCount()) + } + set[seriesKey(mf.GetName()+"_bucket", labels("le", "+Inf"))] = float64(h.GetSampleCount()) + set[seriesKey(mf.GetName()+"_sum", labels())] = h.GetSampleSum() + set[seriesKey(mf.GetName()+"_count", labels())] = float64(h.GetSampleCount()) } } } + return set } // benchmarkHandler returns a handler holding n series: counters with three From 11e002168adad089f1c87950ea25502da610eb74 Mon Sep 17 00:00:00 2001 From: Sathvik Date: Tue, 3 Nov 2026 23:38:36 +0530 Subject: [PATCH 08/20] prometheus: order colliding protobuf series and families deterministically Entries came out of a map and both sorts were unstable, so where two entries rendered to the same name and labels the scraper kept a different sample on each scrape. Sort entries the way WriteStats does, sort series stably and break family ties on their first entry. Co-Authored-By: Claude Opus 5.5 --- prometheus/protobuf.go | 44 +++++++++++++++++++++++++++------- prometheus/protobuf_test.go | 48 +++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 8 deletions(-) diff --git a/prometheus/protobuf.go b/prometheus/protobuf.go index 2728a52..7071340 100644 --- a/prometheus/protobuf.go +++ b/prometheus/protobuf.go @@ -95,6 +95,7 @@ type protoFamilyKey struct { type protoFamily struct { family *dto.MetricFamily series []protoSeries + first int // position of the family's first entry, for ordering } // protoEntry is a store entry copied out under the locks, so that the @@ -123,7 +124,22 @@ type protoBucket struct { func (store *metricStore) families() []*dto.MetricFamily { byKey := make(map[protoFamilyKey]*protoFamily) - for _, entry := range store.protoEntries() { + // Entries come out of a map, so they are put in the order WriteStats + // uses before anything depends on it. Where two entries render to the + // same name and labels, a scraper keeps the first sample and drops the + // other: an order that changed between scrapes would make the stored + // value alternate between two different series, and increase() would + // read every switch as a reset. + entries := store.protoEntries() + slices.SortFunc(entries, func(a, b protoEntry) int { + return cmp.Or( + cmp.Compare(a.scope, b.scope), + cmp.Compare(a.name, b.name), + cmp.Compare(a.mtype, b.mtype), + ) + }) + + for i, entry := range entries { var mtype dto.MetricType switch entry.mtype { case counter: @@ -142,18 +158,20 @@ func (store *metricStore) families() []*dto.MetricFamily { f := byKey[fk] if f == nil { - f = &protoFamily{family: &dto.MetricFamily{Name: proto.String(name), Type: mtype.Enum()}} + f = &protoFamily{family: &dto.MetricFamily{Name: proto.String(name), Type: mtype.Enum()}, first: i} byKey[fk] = f } f.series = append(f.series, entry.series...) } - families := make([]*dto.MetricFamily, 0, len(byKey)) + families := make([]*protoFamily, 0, len(byKey)) for fk, f := range byKey { if len(f.series) == 0 { continue } - slices.SortFunc(f.series, func(a, b protoSeries) int { + // Stable, so that series with the same labels keep the order of the + // entries they came from. + slices.SortStableFunc(f.series, func(a, b protoSeries) int { switch { case a.labels.less(b.labels): return -1 @@ -166,13 +184,23 @@ func (store *metricStore) families() []*dto.MetricFamily { for i := range f.series { f.family.Metric[i] = f.series[i].protoMetric(fk.mtype) } - families = append(families, f.family) + families = append(families, f) } - slices.SortFunc(families, func(a, b *dto.MetricFamily) int { - return cmp.Compare(a.GetName(), b.GetName()) + // A counter and a gauge can render to the same name too; their families + // are ordered by their first entry. + slices.SortFunc(families, func(a, b *protoFamily) int { + return cmp.Or( + cmp.Compare(a.family.GetName(), b.family.GetName()), + cmp.Compare(a.first, b.first), + ) }) - return families + + out := make([]*dto.MetricFamily, len(families)) + for i, f := range families { + out[i] = f.family + } + return out } // protoEntries copies out every series the protobuf exposition publishes. diff --git a/prometheus/protobuf_test.go b/prometheus/protobuf_test.go index 2221b70..cbe7f08 100644 --- a/prometheus/protobuf_test.go +++ b/prometheus/protobuf_test.go @@ -2,6 +2,7 @@ package prometheus import ( "bufio" + "bytes" "compress/gzip" "errors" "io" @@ -295,6 +296,53 @@ func TestProtobufMatchesText(t *testing.T) { } } +// TestProtobufCollisionOrder covers entries that render to the same name. A +// scraper keeps the first of two samples with the same name and labels, so +// the order has to be the same on every scrape, and the same as the text +// format's, or the stored value alternates between the two series. +func TestProtobufCollisionOrder(t *testing.T) { + now := time.Date(2017, 6, 4, 22, 12, 0, 0, time.UTC) + + handler := &Handler{} + handler.HandleMeasures(now, + // Both render to a_b_total. + stats.Measure{Fields: []stats.Field{stats.MakeField("a.b", 1, stats.Counter)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("a_b", 2, stats.Counter)}}, + // A counter and a gauge that both render to c_d_total. + stats.Measure{Fields: []stats.Field{stats.MakeField("c_d", 3, stats.Counter)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("c.d_total", 4, stats.Gauge)}}, + ) + + var first bytes.Buffer + if err := handler.writeProtobuf(&first); err != nil { + t.Fatal(err) + } + for range 100 { + var b bytes.Buffer + if err := handler.writeProtobuf(&b); err != nil { + t.Fatal(err) + } + if !bytes.Equal(b.Bytes(), first.Bytes()) { + t.Fatal("protobuf exposition changed between scrapes") + } + } + + // The text format lists a.b first, and the c.d_total gauge before the + // c_d counter, so a scraper keeps the same samples from either format. + families := scrapeProtobuf(t, handler, false) + if len(families) != 3 { + t.Fatalf("found %d families, expected 3: %v", len(families), families) + } + if m := families[0].Metric; families[0].GetName() != "a_b_total" || len(m) != 2 || + m[0].GetCounter().GetValue() != 1 || m[1].GetCounter().GetValue() != 2 { + t.Errorf("bad a_b_total family: %v", families[0]) + } + if families[1].GetName() != "c_d_total" || families[1].GetType() != dto.MetricType_GAUGE || + families[2].GetName() != "c_d_total" || families[2].GetType() != dto.MetricType_COUNTER { + t.Errorf("bad c_d_total families: %v %v", families[1], families[2]) + } +} + // seriesKey renders a series as name{label="value",...} with the labels // sorted and le normalized, so the same series is the same string whichever // format it came from. From 20b23c4c452a5fcc2cfea80213f5b1942f1905f1 Mon Sep 17 00:00:00 2001 From: Sathvik Date: Tue, 3 Nov 2026 23:39:51 +0530 Subject: [PATCH 09/20] prometheus: stop claiming the text format merges colliding families Co-Authored-By: Claude Opus 5.5 --- prometheus/protobuf.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prometheus/protobuf.go b/prometheus/protobuf.go index 7071340..a109d2f 100644 --- a/prometheus/protobuf.go +++ b/prometheus/protobuf.go @@ -151,8 +151,8 @@ func (store *metricStore) families() []*dto.MetricFamily { } // Two entries can render to the same name, a.b and a_b for example. - // The text format declares the family once and lists both under it, - // so they are merged here too. + // Their series are merged into one family, so the name is declared + // once. name := string(appendMetricScopedName(nil, entry.scope, entry.name)) fk := protoFamilyKey{name: name, mtype: entry.mtype} From 8d11553231d42ddb8cf861ab252282a02de090c3 Mon Sep 17 00:00:00 2001 From: Sathvik Date: Tue, 3 Nov 2026 23:46:06 +0530 Subject: [PATCH 10/20] prometheus: give a zero-time first update a created time A zero created time means none is set, so a first update with a zero time left it unset and the next update set it after the series already held a value. Fall back to the wall clock instead. Co-Authored-By: Claude Opus 5.5 --- prometheus/metric.go | 12 +++++++++--- prometheus/protobuf_test.go | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/prometheus/metric.go b/prometheus/metric.go index 8fa222a..1488e7b 100644 --- a/prometheus/metric.go +++ b/prometheus/metric.go @@ -325,7 +325,7 @@ func newMetricState(labels labels) *metricState { } } -func (state *metricState) update(mtype metricType, value float64, time time.Time, buckets []stats.Value) { +func (state *metricState) update(mtype metricType, value float64, t time.Time, buckets []stats.Value) { state.mutex.Lock() switch mtype { @@ -351,9 +351,15 @@ func (state *metricState) update(mtype metricType, value float64, time time.Time } if state.created.IsZero() { - state.created = time + // A zero created time means none has been set, so a measure with a + // zero time cannot set one: the next update would take its place, + // and the series would claim to start later than its first value. + if t.IsZero() { + t = time.Now() + } + state.created = t } - state.time = time + state.time = t state.mutex.Unlock() } diff --git a/prometheus/protobuf_test.go b/prometheus/protobuf_test.go index cbe7f08..8c7ee90 100644 --- a/prometheus/protobuf_test.go +++ b/prometheus/protobuf_test.go @@ -242,6 +242,29 @@ func TestCreatedTimestampAfterTimeout(t *testing.T) { } } +// TestCreatedTimestampZeroTime covers a first update with a zero time. A zero +// created time means none is set, so the series has to take the time it was +// first seen instead, or the next update would set created after the series +// already held a value and the scraper would write a zero under it. +func TestCreatedTimestampZeroTime(t *testing.T) { + handler := &Handler{} + + before := time.Now() + handler.HandleMeasures(time.Time{}, stats.Measure{Fields: []stats.Field{stats.MakeField("A", 5, stats.Counter)}}) + after := time.Now() + handler.HandleMeasures(after.Add(time.Minute), stats.Measure{Fields: []stats.Field{stats.MakeField("A", 1, stats.Counter)}}) + + families := scrapeProtobuf(t, handler, false) + if len(families) != 1 || len(families[0].Metric) != 1 { + t.Fatalf("unexpected families: %v", families) + } + c := families[0].Metric[0].GetCounter() + created := c.GetCreatedTimestamp().AsTime() + if c.GetValue() != 6 || created.Before(before) || created.After(after) { + t.Errorf("counter = %v, expected value 6 created between %s and %s", c, before, after) + } +} + // TestProtobufMatchesText checks that a scraper ingests the same series from // the protobuf exposition as from the text format: the same names, label sets // and values, so a scraper that switches format keeps writing to the same From e1b21310f95066a022baa74a4be123dc1d821599 Mon Sep 17 00:00:00 2001 From: Sathvik Date: Tue, 3 Nov 2026 23:48:21 +0530 Subject: [PATCH 11/20] prometheus: write the text le="+Inf" bucket from _count A NaN observation lands in no bucket and a rebuild zeroes the buckets but not _count, so the bucket total could fall short of _count. The scraper derives +Inf from sample_count in the protobuf exposition, so the two formats disagreed. Write +Inf from _count in text too. Co-Authored-By: Claude Opus 5.5 --- prometheus/metric.go | 7 +++++++ prometheus/protobuf_test.go | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/prometheus/metric.go b/prometheus/metric.go index 1488e7b..7a4ca89 100644 --- a/prometheus/metric.go +++ b/prometheus/metric.go @@ -388,6 +388,13 @@ func (state *metricState) collect(metrics []metric, entry *metricEntry, name str var cumulativeCount uint64 for _, bucket := range state.buckets { cumulativeCount += bucket.count + if math.IsInf(bucket.limit, 1) { + // le="+Inf" is _count, as the scraper derives it from a + // protobuf exposition. The bucket total can fall short of it: + // a NaN lands in no bucket, and a rebuild zeroes the buckets + // but not _count. + cumulativeCount = state.count + } metrics = append(metrics, metric{ mtype: entry.mtype, scope: entry.scope, diff --git a/prometheus/protobuf_test.go b/prometheus/protobuf_test.go index 8c7ee90..2ef5038 100644 --- a/prometheus/protobuf_test.go +++ b/prometheus/protobuf_test.go @@ -6,6 +6,7 @@ import ( "compress/gzip" "errors" "io" + "math" "net/http" "net/http/httptest" "slices" @@ -265,6 +266,47 @@ func TestCreatedTimestampZeroTime(t *testing.T) { } } +// TestHistogramInfBucket checks that le="+Inf" is _count in both formats, +// where the bucket total falls short of it: a NaN observation lands in no +// bucket, and a rebuild zeroes the buckets but not _count. +func TestHistogramInfBucket(t *testing.T) { + now := time.Date(2017, 6, 4, 22, 12, 0, 0, time.UTC) + key := stats.Key{Field: "H"} + observe := func(h *Handler, kind string, v float64) { + h.HandleMeasures(now, stats.Measure{ + Fields: []stats.Field{stats.MakeField("H", v, stats.Histogram)}, + Tags: []stats.Tag{stats.T("kind", kind)}, + }) + } + + handler := &Handler{Buckets: map[stats.Key][]stats.Value{key: {stats.ValueOf(1)}}} + observe(handler, "nan", 0.5) + observe(handler, "nan", 2) + observe(handler, "nan", math.NaN()) + observe(handler, "rebuild", 0.5) + observe(handler, "rebuild", 2) + handler.Buckets[key] = []stats.Value{stats.ValueOf(1), stats.ValueOf(5)} + observe(handler, "rebuild", 0.5) + + var text strings.Builder + handler.WriteStats(&text) + fromText := textSeries(t, text.String()) + fromProto := protoSeriesSet(scrapeProtobuf(t, handler, false)) + + for kind, count := range map[string]float64{"nan": 3, "rebuild": 3} { + inf := seriesKey("H_bucket", map[string]string{"kind": kind, "le": "+Inf"}) + if v := fromText[inf]; v != count { + t.Errorf("text %s = %g, expected %g", inf, v, count) + } + if v := fromProto[inf]; v != count { + t.Errorf("protobuf %s = %g, expected %g", inf, v, count) + } + } + if t.Failed() { + t.Log("text output:\n" + text.String()) + } +} + // TestProtobufMatchesText checks that a scraper ingests the same series from // the protobuf exposition as from the text format: the same names, label sets // and values, so a scraper that switches format keeps writing to the same From 470da00d18336a2996bcb90ac60604b6b9e8470e Mon Sep 17 00:00:00 2001 From: Sathvik Date: Wed, 4 Nov 2026 00:26:34 +0530 Subject: [PATCH 12/20] prometheus: write protobuf families one at a time, and send Vary Build each family's messages just before writing it, so only one family is held at a time; peak heap at 100k series drops 12-21%. Send Vary: Accept, Accept-Encoding, since both headers choose the response, and say why write errors are ignored. Co-Authored-By: Claude Opus 5.5 --- prometheus/handler.go | 10 +++++ prometheus/protobuf.go | 63 ++++++++++++++++------------- prometheus/protobuf_test.go | 80 ++++++++++++++++++++++++++++++++++--- 3 files changed, 120 insertions(+), 33 deletions(-) diff --git a/prometheus/handler.go b/prometheus/handler.go index b608874..60568ff 100644 --- a/prometheus/handler.go +++ b/prometheus/handler.go @@ -155,6 +155,11 @@ func (h *Handler) ServeHTTP(res http.ResponseWriter, req *http.Request) { w := io.Writer(res) protobuf := acceptsProtobuf(req.Header.Get("Accept")) + // The format follows Accept and the compression Accept-Encoding, so a + // cache must not serve one response in place of the other. + res.Header().Add("Vary", "Accept") + res.Header().Add("Vary", "Accept-Encoding") + // Text stays the default. Only the protobuf exposition carries created // timestamps, and a scraper gets it only by asking for it first. if protobuf { @@ -170,6 +175,11 @@ func (h *Handler) ServeHTTP(res http.ResponseWriter, req *http.Request) { w = zw } + // Write errors, here and from WriteStats and the deferred gzip Close, + // are ignored: the only cause is the connection failing, and by then + // the status and part of the body have been sent, so there is nothing + // left to report the error with. The scraper sees a response that ends + // early and records a failed scrape. if protobuf { _ = h.writeProtobuf(w) return diff --git a/prometheus/protobuf.go b/prometheus/protobuf.go index a109d2f..06f3ebe 100644 --- a/prometheus/protobuf.go +++ b/prometheus/protobuf.go @@ -78,9 +78,12 @@ func acceptsProtobuf(accept string) bool { // bounds are the exception on 2.x: the scraper formats UpperBound itself, as // le="1.0" where the text format writes le="1", and only 3.x normalizes the // text value to match. +// +// Each family's messages are built just before it is written and dropped +// after, so only one family's worth is held at a time. func (h *Handler) writeProtobuf(w io.Writer) error { - for _, mf := range h.metrics.families() { - if _, err := protodelim.MarshalTo(w, mf); err != nil { + for _, f := range h.metrics.families() { + if _, err := protodelim.MarshalTo(w, f.message()); err != nil { return err } } @@ -93,11 +96,34 @@ type protoFamilyKey struct { } type protoFamily struct { - family *dto.MetricFamily + name string + mtype metricType series []protoSeries first int // position of the family's first entry, for ordering } +func (f *protoFamily) message() *dto.MetricFamily { + var mtype dto.MetricType + switch f.mtype { + case counter: + mtype = dto.MetricType_COUNTER + case gauge: + mtype = dto.MetricType_GAUGE + case histogram: + mtype = dto.MetricType_HISTOGRAM + } + + mf := &dto.MetricFamily{ + Name: proto.String(f.name), + Type: mtype.Enum(), + Metric: make([]*dto.Metric, len(f.series)), + } + for i := range f.series { + mf.Metric[i] = f.series[i].protoMetric(f.mtype) + } + return mf +} + // protoEntry is a store entry copied out under the locks, so that the // messages can be built after they are released. type protoEntry struct { @@ -121,7 +147,9 @@ type protoBucket struct { count uint64 } -func (store *metricStore) families() []*dto.MetricFamily { +// families returns the families to publish in order, each with its series +// sorted, without building their messages. +func (store *metricStore) families() []*protoFamily { byKey := make(map[protoFamilyKey]*protoFamily) // Entries come out of a map, so they are put in the order WriteStats @@ -140,16 +168,6 @@ func (store *metricStore) families() []*dto.MetricFamily { }) for i, entry := range entries { - var mtype dto.MetricType - switch entry.mtype { - case counter: - mtype = dto.MetricType_COUNTER - case gauge: - mtype = dto.MetricType_GAUGE - case histogram: - mtype = dto.MetricType_HISTOGRAM - } - // Two entries can render to the same name, a.b and a_b for example. // Their series are merged into one family, so the name is declared // once. @@ -158,14 +176,14 @@ func (store *metricStore) families() []*dto.MetricFamily { f := byKey[fk] if f == nil { - f = &protoFamily{family: &dto.MetricFamily{Name: proto.String(name), Type: mtype.Enum()}, first: i} + f = &protoFamily{name: name, mtype: entry.mtype, first: i} byKey[fk] = f } f.series = append(f.series, entry.series...) } families := make([]*protoFamily, 0, len(byKey)) - for fk, f := range byKey { + for _, f := range byKey { if len(f.series) == 0 { continue } @@ -180,10 +198,6 @@ func (store *metricStore) families() []*dto.MetricFamily { } return 0 }) - f.family.Metric = make([]*dto.Metric, len(f.series)) - for i := range f.series { - f.family.Metric[i] = f.series[i].protoMetric(fk.mtype) - } families = append(families, f) } @@ -191,16 +205,11 @@ func (store *metricStore) families() []*dto.MetricFamily { // are ordered by their first entry. slices.SortFunc(families, func(a, b *protoFamily) int { return cmp.Or( - cmp.Compare(a.family.GetName(), b.family.GetName()), + cmp.Compare(a.name, b.name), cmp.Compare(a.first, b.first), ) }) - - out := make([]*dto.MetricFamily, len(families)) - for i, f := range families { - out[i] = f.family - } - return out + return families } // protoEntries copies out every series the protobuf exposition publishes. diff --git a/prometheus/protobuf_test.go b/prometheus/protobuf_test.go index 2ef5038..b4ea13c 100644 --- a/prometheus/protobuf_test.go +++ b/prometheus/protobuf_test.go @@ -9,6 +9,7 @@ import ( "math" "net/http" "net/http/httptest" + "runtime" "slices" "strconv" "strings" @@ -107,6 +108,9 @@ func scrapeProtobuf(t *testing.T, handler *Handler, gzipped bool) []*dto.MetricF if ct := res.Header.Get("Content-Type"); ct != protobufContentType { t.Fatalf("Content-Type = %q, expected %q", ct, protobufContentType) } + if vary := res.Header.Values("Vary"); !slices.Equal(vary, []string{"Accept", "Accept-Encoding"}) { + t.Errorf("Vary = %q, expected Accept and Accept-Encoding", vary) + } body := io.Reader(res.Body) if gzipped { @@ -538,6 +542,35 @@ func benchmarkHandler(n int) *Handler { return handler } +// benchmarkHandlerFamilies is benchmarkHandler with its counters spread +// across families/2 names and its histograms across the other half. +func benchmarkHandlerFamilies(n, families int) *Handler { + handler := &Handler{} + now := time.Now() + names := max(families/2, 1) + for i := range n { + tags := []stats.Tag{ + stats.T("method", "GET"), + stats.T("status_code", strconv.Itoa(200+i%5)), + stats.T("url", "/v1/workspaces/"+strconv.Itoa(i)), + } + suffix := strconv.Itoa(i % names) + handler.HandleMeasures(now, stats.Measure{ + Name: "http", + Fields: []stats.Field{stats.MakeField("requests"+suffix, 1, stats.Counter)}, + Tags: tags, + }) + if i%10 == 0 { + handler.HandleMeasures(now, stats.Measure{ + Name: "http", + Fields: []stats.Field{stats.MakeField("duration"+suffix, 0.1, stats.Histogram)}, + Tags: tags, + }) + } + } + return handler +} + // BenchmarkCollect is the text path's time under the store lock. func BenchmarkCollect(b *testing.B) { handler := benchmarkHandler(100000) @@ -547,13 +580,48 @@ func BenchmarkCollect(b *testing.B) { } } -// BenchmarkFamilies is the whole protobuf build. -func BenchmarkFamilies(b *testing.B) { - handler := benchmarkHandler(100000) - b.ReportAllocs() - for b.Loop() { - handler.metrics.families() +// BenchmarkWriteProtobuf is the whole protobuf exposition, built and +// marshalled. peak-heap-B is the most heap in use at any write, above what +// it was before the call: the messages built but not yet written. +func BenchmarkWriteProtobuf(b *testing.B) { + for _, families := range []int{2, 100} { + b.Run(strconv.Itoa(families)+"-families", func(b *testing.B) { + handler := benchmarkHandlerFamilies(100000, families) + w := &peakHeapWriter{} + b.ReportAllocs() + for b.Loop() { + b.StopTimer() + w.start() + b.StartTimer() + if err := handler.writeProtobuf(w); err != nil { + b.Fatal(err) + } + } + b.ReportMetric(float64(w.max), "peak-heap-B") + }) + } +} + +// peakHeapWriter discards what it is given and records the heap in use at +// each write. +type peakHeapWriter struct { + base, max uint64 +} + +func (w *peakHeapWriter) start() { + var m runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&m) + w.base = m.HeapAlloc +} + +func (w *peakHeapWriter) Write(p []byte) (int, error) { + var m runtime.MemStats + runtime.ReadMemStats(&m) + if m.HeapAlloc > w.base && m.HeapAlloc-w.base > w.max { + w.max = m.HeapAlloc - w.base } + return len(p), nil } // BenchmarkProtoEntries is the protobuf path's time under the store lock. From 1ca7a60a102ed937f9ec37c3819ad0082f5fc210 Mon Sep 17 00:00:00 2001 From: Sathvik Date: Wed, 4 Nov 2026 00:51:50 +0530 Subject: [PATCH 13/20] prometheus: test real Prometheus Accept headers and a concurrent scrape Add the Accept headers Prometheus 3.x sends by default and with protobuf first, a scrape racing measures and expiry, and bump client_model to v0.6.3. Co-Authored-By: Claude Opus 5.5 --- go.mod | 2 +- go.sum | 4 +- prometheus/protobuf_test.go | 73 +++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 395d196..b26e1c4 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.26.0 require ( github.com/mdlayher/taskstats v0.0.0-20241219020249-a291fa5f5a69 - github.com/prometheus/client_model v0.6.2 + github.com/prometheus/client_model v0.6.3 github.com/segmentio/encoding v0.5.4 github.com/segmentio/fasthash v1.0.3 github.com/segmentio/vpcinfo v0.2.0 diff --git a/go.sum b/go.sum index f4e3646..caa6354 100644 --- a/go.sum +++ b/go.sum @@ -23,8 +23,8 @@ github.com/mdlayher/socket v0.7.0 h1:qVREPVwtUMg17pwvveQxpurq0PVisMxi3FGgpsorMYQ github.com/mdlayher/socket v0.7.0/go.mod h1:f7iKql2EK/rfsWYDKofKjk2Ig7I+6HQWdMIdn1tO4A4= github.com/mdlayher/taskstats v0.0.0-20241219020249-a291fa5f5a69 h1:A33BCEd5ClzJCnUcicB1VRuRrgikbUc8J/v1qrtd2gw= github.com/mdlayher/taskstats v0.0.0-20241219020249-a291fa5f5a69/go.mod h1:0gPvvNyurVKWZFI09u1YSPCqhSnj6QwlwEijSrHXYag= -github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= -github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= +github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= diff --git a/prometheus/protobuf_test.go b/prometheus/protobuf_test.go index b4ea13c..d355536 100644 --- a/prometheus/protobuf_test.go +++ b/prometheus/protobuf_test.go @@ -13,6 +13,7 @@ import ( "slices" "strconv" "strings" + "sync" "testing" "time" @@ -42,6 +43,31 @@ func TestAcceptsProtobuf(t *testing.T) { accept: "application/openmetrics-text;version=1.0.0;q=0.5,application/openmetrics-text;version=0.0.1;q=0.4,text/plain;version=0.0.4;q=0.3,*/*;q=0.2", expect: false, }, + { + // Prometheus' acceptHeader output for DefaultScrapeProtocols. Its + // weights start at one above the number of protocols it knows, so + // they shift between releases: q=0.8 on main, q=0.7 on 3.0. + name: "prometheus 3.x defaults", + accept: "application/openmetrics-text;version=1.0.0;escaping=allow-utf-8;q=0.8,application/openmetrics-text;version=0.0.1;q=0.7," + + "text/plain;version=1.0.0;escaping=allow-utf-8;q=0.6,text/plain;version=0.0.4;q=0.5,*/*;q=0.4", + expect: false, + }, + { + // DefaultProtoFirstScrapeProtocols, used with scrape_native_histograms + // or created-timestamp zero ingestion. + name: "prometheus 3.x proto first", + accept: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.8," + + "application/openmetrics-text;version=1.0.0;escaping=allow-utf-8;q=0.7,application/openmetrics-text;version=0.0.1;q=0.6," + + "text/plain;version=1.0.0;escaping=allow-utf-8;q=0.5,text/plain;version=0.0.4;q=0.4,*/*;q=0.3", + expect: true, + }, + { + name: "prometheus 3.0 proto first", + accept: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7," + + "application/openmetrics-text;version=1.0.0;escaping=allow-utf-8;q=0.6,application/openmetrics-text;version=0.0.1;q=0.5," + + "text/plain;version=1.0.0;escaping=allow-utf-8;q=0.4,text/plain;version=0.0.4;q=0.3,*/*;q=0.2", + expect: true, + }, { name: "protobuf ranked below text", accept: "text/plain;version=0.0.4;q=0.5,application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.4", @@ -311,6 +337,53 @@ func TestHistogramInfBucket(t *testing.T) { } } +// TestProtobufConcurrentScrape scrapes while measures arrive and series +// expire, for the race detector to check the copy taken under the locks. +func TestProtobufConcurrentScrape(t *testing.T) { + handler := &Handler{} + start := time.Now() + stop := make(chan struct{}) + var wg sync.WaitGroup + + wg.Add(2) + go func() { + defer wg.Done() + for i := 0; ; i++ { + select { + case <-stop: + return + default: + } + now := start.Add(time.Duration(i) * time.Millisecond) + tags := []stats.Tag{stats.T("id", strconv.Itoa(i%50))} + handler.HandleMeasures(now, + stats.Measure{Fields: []stats.Field{stats.MakeField("c", 1, stats.Counter)}, Tags: tags}, + stats.Measure{Fields: []stats.Field{stats.MakeField("g", float64(i), stats.Gauge)}, Tags: tags}, + stats.Measure{Fields: []stats.Field{stats.MakeField("h", 0.1, stats.Histogram)}, Tags: tags}, + ) + } + }() + go func() { + defer wg.Done() + for i := 0; ; i++ { + select { + case <-stop: + return + default: + } + handler.metrics.cleanup(start.Add(time.Duration(i) * time.Millisecond)) + } + }() + + for range 50 { + if err := handler.writeProtobuf(io.Discard); err != nil { + t.Error(err) + } + } + close(stop) + wg.Wait() +} + // TestProtobufMatchesText checks that a scraper ingests the same series from // the protobuf exposition as from the text format: the same names, label sets // and values, so a scraper that switches format keeps writing to the same From 0f58552ff74b3a484497d13b0b37757bdf30f385 Mon Sep 17 00:00:00 2001 From: Sathvik Date: Wed, 4 Nov 2026 00:53:28 +0530 Subject: [PATCH 14/20] HISTORY: clarify which scrapers put protobuf first, and note the +Inf and Vary changes Co-Authored-By: Claude Opus 5.5 --- HISTORY.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 06e8331..d194aef 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -12,19 +12,22 @@ The handler now records when each counter and histogram series started from zero (its first update, or its first update after `MetricTimeout` deleted it) and publishes it as `created_timestamp` in the protobuf exposition (`application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; -encoding=delimited`). A scraper with created-timestamp zero ingestion enabled -(Prometheus `created-timestamp-zero-ingestion`, Alloy -`start_timestamp_zero_ingestion`) writes a zero at that time, so the first -update is counted. +encoding=delimited`). Prometheus is renaming created timestamps to start +timestamps; the wire field is the same. A scraper with created (start) +timestamp zero ingestion enabled (Prometheus +`created-timestamp-zero-ingestion`, Alloy `start_timestamp_zero_ingestion`) +writes a zero at that time, so the first update is counted. Protobuf is served only to a scraper that ranks it above the text format in its `Accept` header. That is a scraper whose scrape protocols put -`PrometheusProto` first, or a Prometheus with native histograms or +`PrometheusProto` first, or a Prometheus with `scrape_native_histograms` or created-timestamp zero ingestion on, which puts it first by default: such a Prometheus switches to protobuf when the application upgrades, with no change -to its configuration. Every other scraper keeps getting the text format, -unchanged. Sample timestamps are still left to the scraper. Gauges carry no -created timestamp. +to its configuration. Alloy puts it first by default only with +`scrape_native_histograms`; `start_timestamp_zero_ingestion` alone does not +change the order, so `scrape_protocols` has to list `PrometheusProto` first. Every other scraper keeps getting the text format. +Sample timestamps are still left to the scraper. Gauges carry no created +timestamp. Both formats render the same metric and label names, so a Prometheus 3.x scraper that switches format keeps writing to the same series. On Prometheus @@ -32,6 +35,12 @@ scraper that switches format keeps writing to the same series. On Prometheus formats the bound of a protobuf bucket as `le="1.0"`, while the text format writes `le="1"`, and only 3.x normalizes the text value to match. +The text format now writes a histogram's `le="+Inf"` bucket from `_count` +rather than from the total of the other buckets. The two differed after a NaN +observation, which lands in no bucket, or after a change to the registered +buckets, which resets them but not `_count`. Responses now carry +`Vary: Accept, Accept-Encoding`. + The module now depends on `github.com/prometheus/client_model`. ### v5.12.0 From c6c5df46affb87a491023af401a634d5c793d855 Mon Sep 17 00:00:00 2001 From: Sathvik Date: Wed, 4 Nov 2026 00:53:42 +0530 Subject: [PATCH 15/20] HISTORY: rewrap Co-Authored-By: Claude Opus 5.5 --- HISTORY.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index d194aef..0f87c8c 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -25,9 +25,9 @@ created-timestamp zero ingestion on, which puts it first by default: such a Prometheus switches to protobuf when the application upgrades, with no change to its configuration. Alloy puts it first by default only with `scrape_native_histograms`; `start_timestamp_zero_ingestion` alone does not -change the order, so `scrape_protocols` has to list `PrometheusProto` first. Every other scraper keeps getting the text format. -Sample timestamps are still left to the scraper. Gauges carry no created -timestamp. +change the order, so `scrape_protocols` has to list `PrometheusProto` first. +Every other scraper keeps getting the text format. Sample timestamps are still +left to the scraper. Gauges carry no created timestamp. Both formats render the same metric and label names, so a Prometheus 3.x scraper that switches format keeps writing to the same series. On Prometheus From 164519f5c76d95c0c5ead84505561ac68ba43925 Mon Sep 17 00:00:00 2001 From: Sathvik Date: Wed, 4 Nov 2026 01:14:38 +0530 Subject: [PATCH 16/20] prometheus: fix misspelling flagged by golangci-lint Co-Authored-By: Claude Opus 5.5 --- prometheus/protobuf_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus/protobuf_test.go b/prometheus/protobuf_test.go index d355536..bec84c4 100644 --- a/prometheus/protobuf_test.go +++ b/prometheus/protobuf_test.go @@ -654,7 +654,7 @@ func BenchmarkCollect(b *testing.B) { } // BenchmarkWriteProtobuf is the whole protobuf exposition, built and -// marshalled. peak-heap-B is the most heap in use at any write, above what +// marshaled. peak-heap-B is the most heap in use at any write, above what // it was before the call: the messages built but not yet written. func BenchmarkWriteProtobuf(b *testing.B) { for _, families := range []int{2, 100} { From 5fcbb1ac253e9b0eb553448062555eb86bfcc1fa Mon Sep 17 00:00:00 2001 From: Sathvik Date: Wed, 4 Nov 2026 23:13:30 +0530 Subject: [PATCH 17/20] prometheus: break text sort ties so a histogram's series comes first A gauge "y_count" and the _count series of a histogram "y" tie on scope, name and labels. sort.Sort is not stable and metrics come out of a map, so the text format listed them in a different order between scrapes, and the value a scraper kept alternated between the two. Break the tie on the root name, then the type, which lists the histogram first as the protobuf format already does. Document the collision the two formats still resolve differently: a gauge "x.count" next to a histogram "x". Co-Authored-By: Claude Opus 5.5 --- HISTORY.md | 6 +++++- prometheus/metric.go | 18 +++++++++++++++- prometheus/protobuf.go | 8 ++++++++ prometheus/protobuf_test.go | 41 +++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 0f87c8c..41224b9 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -38,7 +38,11 @@ writes `le="1"`, and only 3.x normalizes the text value to match. The text format now writes a histogram's `le="+Inf"` bucket from `_count` rather than from the total of the other buckets. The two differed after a NaN observation, which lands in no bucket, or after a change to the registered -buckets, which resets them but not `_count`. Responses now carry +buckets, which resets them but not `_count`. Where a gauge and a histogram +series render to the same name and labels (a gauge `y_count` next to a +histogram `y`), the text format now always lists the histogram's first, as +the protobuf format does; their order used to change between scrapes, so the +value a scraper kept alternated between the two. Responses now carry `Vary: Accept, Accept-Encoding`. The module now depends on `github.com/prometheus/client_model`. diff --git a/prometheus/metric.go b/prometheus/metric.go index 7a4ca89..b34ff64 100644 --- a/prometheus/metric.go +++ b/prometheus/metric.go @@ -549,6 +549,13 @@ func (metrics byNameAndLabels) Swap(i, j int) { // // Comparing the two parts in turn rather than the joined "scope_name" avoids // building a string for every comparison in the sort. +// +// Two metrics can still tie on all three, a gauge "y_count" and the _count +// series of a histogram "y" for example. A scraper keeps the first of the two +// samples, and metrics come out of a map, so the tie is broken on the root +// name and then the type: otherwise the kept value would alternate between +// scrapes. The root name puts the histogram first, as the protobuf format +// does. func (metrics byNameAndLabels) Less(i, j int) bool { m1 := &metrics[i] m2 := &metrics[j] @@ -559,5 +566,14 @@ func (metrics byNameAndLabels) Less(i, j int) bool { if m1.name != m2.name { return m1.name < m2.name } - return m1.labels.less(m2.labels) + if m1.labels.less(m2.labels) { + return true + } + if m2.labels.less(m1.labels) { + return false + } + if r1, r2 := m1.rootName(), m2.rootName(); r1 != r2 { + return r1 < r2 + } + return m1.mtype < m2.mtype } diff --git a/prometheus/protobuf.go b/prometheus/protobuf.go index 06f3ebe..0d1bfe3 100644 --- a/prometheus/protobuf.go +++ b/prometheus/protobuf.go @@ -158,6 +158,14 @@ func (store *metricStore) families() []*protoFamily { // other: an order that changed between scrapes would make the stored // value alternate between two different series, and increase() would // read every switch as a reset. + // + // One collision still keeps a different series than the text format + // does, though the same one on every scrape. A histogram sorts here under + // its root name, and in the text format each of its series sorts under + // its own: next to a histogram "x", a gauge "x.count" is listed after the + // histogram here and before its x_count in text, because '.' sorts before + // '_'. Matching it would mean reordering the text format for every + // metric. entries := store.protoEntries() slices.SortFunc(entries, func(a, b protoEntry) int { return cmp.Or( diff --git a/prometheus/protobuf_test.go b/prometheus/protobuf_test.go index bec84c4..b35f32e 100644 --- a/prometheus/protobuf_test.go +++ b/prometheus/protobuf_test.go @@ -485,6 +485,47 @@ func TestProtobufCollisionOrder(t *testing.T) { } } +// TestHistogramCollisionOrder covers a gauge that renders to the same name +// as one of a histogram's series. Both formats list the histogram first, so +// a scraper keeps its y_count on every scrape, whichever format it reads. +func TestHistogramCollisionOrder(t *testing.T) { + now := time.Date(2017, 6, 4, 22, 12, 0, 0, time.UTC) + + handler := &Handler{} + handler.HandleMeasures(now, + stats.Measure{Fields: []stats.Field{stats.MakeField("y_count", 70, stats.Gauge)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("y", 0.5, stats.Histogram)}}, + ) + + var first bytes.Buffer + handler.WriteStats(&first) + for range 100 { + var b bytes.Buffer + handler.WriteStats(&b) + if !bytes.Equal(b.Bytes(), first.Bytes()) { + t.Fatalf("text exposition changed between scrapes:\n%s\nthen:\n%s", first.String(), b.String()) + } + } + + var firstCount string + for line := range strings.SplitSeq(first.String(), "\n") { + if strings.HasPrefix(line, "y_count ") { + firstCount = line + break + } + } + if firstCount != "y_count 1" { + t.Errorf("text lists %q first, expected the histogram's y_count 1:\n%s", firstCount, first.String()) + } + + families := scrapeProtobuf(t, handler, false) + if len(families) != 2 || + families[0].GetName() != "y" || families[0].GetType() != dto.MetricType_HISTOGRAM || + families[1].GetName() != "y_count" || families[1].GetType() != dto.MetricType_GAUGE { + t.Errorf("protobuf does not list the histogram first: %v", families) + } +} + // seriesKey renders a series as name{label="value",...} with the labels // sorted and le normalized, so the same series is the same string whichever // format it came from. From 516b50177f240c478e944c9575880c27f8ac8c2b Mon Sep 17 00:00:00 2001 From: Sathvik Date: Wed, 4 Nov 2026 23:16:14 +0530 Subject: [PATCH 18/20] prometheus: keep a zero update time when falling back for the created time A zero-time first update took time.Now() as its created time, but the fallback overwrote t, so state.time was set to now as well. Later zero-time updates set it back to zero, and MetricTimeout expiry reads it, so the first update kept a series alive that the others would not. Use a separate local for the created time. Co-Authored-By: Claude Opus 5.5 --- prometheus/metric.go | 9 ++++++--- prometheus/protobuf_test.go | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/prometheus/metric.go b/prometheus/metric.go index b34ff64..d97cf52 100644 --- a/prometheus/metric.go +++ b/prometheus/metric.go @@ -354,10 +354,13 @@ func (state *metricState) update(mtype metricType, value float64, t time.Time, b // A zero created time means none has been set, so a measure with a // zero time cannot set one: the next update would take its place, // and the series would claim to start later than its first value. - if t.IsZero() { - t = time.Now() + // state.time keeps the zero, so expiry treats this update like any + // other zero-time update. + created := t + if created.IsZero() { + created = time.Now() } - state.created = t + state.created = created } state.time = t state.mutex.Unlock() diff --git a/prometheus/protobuf_test.go b/prometheus/protobuf_test.go index b35f32e..ac26b37 100644 --- a/prometheus/protobuf_test.go +++ b/prometheus/protobuf_test.go @@ -296,6 +296,21 @@ func TestCreatedTimestampZeroTime(t *testing.T) { } } +// TestZeroTimeUpdateKeepsZeroTime checks that the created time fallback +// leaves the update time alone: MetricTimeout expiry reads the update time, +// so a zero-time first update has to expire like any other zero-time update. +func TestZeroTimeUpdateKeepsZeroTime(t *testing.T) { + var state metricState + state.update(counter, 5, time.Time{}, nil) + + if state.created.IsZero() { + t.Error("created time not set by a zero-time first update") + } + if !state.time.IsZero() { + t.Errorf("update time = %s, expected zero", state.time) + } +} + // TestHistogramInfBucket checks that le="+Inf" is _count in both formats, // where the bucket total falls short of it: a NaN observation lands in no // bucket, and a rebuild zeroes the buckets but not _count. From 4b983a76bcfd98b16e1533c28d92ab8b673b1326 Mon Sep 17 00:00:00 2001 From: Sathvik Date: Wed, 4 Nov 2026 23:19:29 +0530 Subject: [PATCH 19/20] prometheus: fix the Prometheus versions in the Accept header test acceptHeader starts its weights at one above the number of scrape protocols Prometheus knows: 5 on 3.0 to 3.14, 6 on 3.15, and 7 on main. The q=0.7 header is therefore 3.15, not 3.0. Rename the cases, and add 3.0's header, which starts at q=0.6 and appends escaping=allow-utf-8 to every protocol. Co-Authored-By: Claude Opus 5.5 --- prometheus/protobuf_test.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/prometheus/protobuf_test.go b/prometheus/protobuf_test.go index ac26b37..7b96777 100644 --- a/prometheus/protobuf_test.go +++ b/prometheus/protobuf_test.go @@ -46,8 +46,9 @@ func TestAcceptsProtobuf(t *testing.T) { { // Prometheus' acceptHeader output for DefaultScrapeProtocols. Its // weights start at one above the number of protocols it knows, so - // they shift between releases: q=0.8 on main, q=0.7 on 3.0. - name: "prometheus 3.x defaults", + // they shift between releases: q=0.6 on 3.0 to 3.14 (5 protocols), + // q=0.7 on 3.15 (6), and q=0.8 on main (7). + name: "prometheus main defaults", accept: "application/openmetrics-text;version=1.0.0;escaping=allow-utf-8;q=0.8,application/openmetrics-text;version=0.0.1;q=0.7," + "text/plain;version=1.0.0;escaping=allow-utf-8;q=0.6,text/plain;version=0.0.4;q=0.5,*/*;q=0.4", expect: false, @@ -55,19 +56,29 @@ func TestAcceptsProtobuf(t *testing.T) { { // DefaultProtoFirstScrapeProtocols, used with scrape_native_histograms // or created-timestamp zero ingestion. - name: "prometheus 3.x proto first", + name: "prometheus main proto first", accept: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.8," + "application/openmetrics-text;version=1.0.0;escaping=allow-utf-8;q=0.7,application/openmetrics-text;version=0.0.1;q=0.6," + "text/plain;version=1.0.0;escaping=allow-utf-8;q=0.5,text/plain;version=0.0.4;q=0.4,*/*;q=0.3", expect: true, }, { - name: "prometheus 3.0 proto first", + name: "prometheus 3.15 proto first", accept: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7," + "application/openmetrics-text;version=1.0.0;escaping=allow-utf-8;q=0.6,application/openmetrics-text;version=0.0.1;q=0.5," + "text/plain;version=1.0.0;escaping=allow-utf-8;q=0.4,text/plain;version=0.0.4;q=0.3,*/*;q=0.2", expect: true, }, + { + // 3.0 to 3.3 append escaping=allow-utf-8 to every protocol under + // the default UTF-8 name validation, including protobuf, and twice + // to text 1.0.0, whose header already carries it. + name: "prometheus 3.0 proto first", + accept: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;escaping=allow-utf-8;q=0.6," + + "application/openmetrics-text;version=1.0.0;escaping=allow-utf-8;q=0.5,application/openmetrics-text;version=0.0.1;escaping=allow-utf-8;q=0.4," + + "text/plain;version=1.0.0;escaping=allow-utf-8;escaping=allow-utf-8;q=0.3,text/plain;version=0.0.4;escaping=allow-utf-8;q=0.2,*/*;q=0.1", + expect: true, + }, { name: "protobuf ranked below text", accept: "text/plain;version=0.0.4;q=0.5,application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.4", From f52214334eff9274501165c1b543681b931e44a5 Mon Sep 17 00:00:00 2001 From: Sathvik Date: Wed, 4 Nov 2026 23:21:52 +0530 Subject: [PATCH 20/20] go.mod: go back to client_model v0.6.2 The bump to v0.6.3 came in with a test commit and nothing needs it: its generated Go code is the same as v0.6.2's. Requiring only what is used keeps consumers of stats from being moved to v0.6.3 for nothing. Co-Authored-By: Claude Opus 5.5 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b26e1c4..395d196 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.26.0 require ( github.com/mdlayher/taskstats v0.0.0-20241219020249-a291fa5f5a69 - github.com/prometheus/client_model v0.6.3 + github.com/prometheus/client_model v0.6.2 github.com/segmentio/encoding v0.5.4 github.com/segmentio/fasthash v1.0.3 github.com/segmentio/vpcinfo v0.2.0 diff --git a/go.sum b/go.sum index caa6354..f4e3646 100644 --- a/go.sum +++ b/go.sum @@ -23,8 +23,8 @@ github.com/mdlayher/socket v0.7.0 h1:qVREPVwtUMg17pwvveQxpurq0PVisMxi3FGgpsorMYQ github.com/mdlayher/socket v0.7.0/go.mod h1:f7iKql2EK/rfsWYDKofKjk2Ig7I+6HQWdMIdn1tO4A4= github.com/mdlayher/taskstats v0.0.0-20241219020249-a291fa5f5a69 h1:A33BCEd5ClzJCnUcicB1VRuRrgikbUc8J/v1qrtd2gw= github.com/mdlayher/taskstats v0.0.0-20241219020249-a291fa5f5a69/go.mod h1:0gPvvNyurVKWZFI09u1YSPCqhSnj6QwlwEijSrHXYag= -github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= -github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0=