diff --git a/HISTORY.md b/HISTORY.md index 802b74b..41224b9 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,52 @@ # History +### v5.13.0 + +**The `prometheus` handler can serve the protobuf exposition, with created +timestamps.** `increase()` and `rate()` never count a counter's first sample: +it is taken as the starting point. A series that starts at 5 and stays there +shows an increase of 0, and a series that goes idle long enough for +`MetricTimeout` to delete it loses its first update again when it comes back. + +The handler now records when each counter and histogram series started from +zero (its first update, or its first update after `MetricTimeout` deleted it) +and publishes it as `created_timestamp` in the protobuf exposition +(`application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; +encoding=delimited`). Prometheus is renaming created timestamps to start +timestamps; the wire field is the same. A scraper with created (start) +timestamp zero ingestion enabled (Prometheus +`created-timestamp-zero-ingestion`, Alloy `start_timestamp_zero_ingestion`) +writes a zero at that time, so the first update is counted. + +Protobuf is served only to a scraper that ranks it above the text format in +its `Accept` header. That is a scraper whose scrape protocols put +`PrometheusProto` first, or a Prometheus with `scrape_native_histograms` or +created-timestamp zero ingestion on, which puts it first by default: such a +Prometheus switches to protobuf when the application upgrades, with no change +to its configuration. Alloy puts it first by default only with +`scrape_native_histograms`; `start_timestamp_zero_ingestion` alone does not +change the order, so `scrape_protocols` has to list `PrometheusProto` first. +Every other scraper keeps getting the text format. Sample timestamps are still +left to the scraper. Gauges carry no created timestamp. + +Both formats render the same metric and label names, so a Prometheus 3.x +scraper that switches format keeps writing to the same series. On Prometheus +2.x, histogram buckets with integer bounds start new series: the scraper +formats the bound of a protobuf bucket as `le="1.0"`, while the text format +writes `le="1"`, and only 3.x normalizes the text value to match. + +The text format now writes a histogram's `le="+Inf"` bucket from `_count` +rather than from the total of the other buckets. The two differed after a NaN +observation, which lands in no bucket, or after a change to the registered +buckets, which resets them but not `_count`. Where a gauge and a histogram +series render to the same name and labels (a gauge `y_count` next to a +histogram `y`), the text format now always lists the histogram's first, as +the protobuf format does; their order used to change between scrapes, so the +value a scraper kept alternated between the two. Responses now carry +`Vary: Accept, Accept-Encoding`. + +The module now depends on `github.com/prometheus/client_model`. + ### v5.12.0 **`procstats` now reads CPU and memory limits on cgroup v2.** It only knew the diff --git a/go.mod b/go.mod index 6d81e2f..395d196 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.0 require ( github.com/mdlayher/taskstats v0.0.0-20241219020249-a291fa5f5a69 + github.com/prometheus/client_model v0.6.2 github.com/segmentio/encoding v0.5.4 github.com/segmentio/fasthash v1.0.3 github.com/segmentio/vpcinfo v0.2.0 diff --git a/go.sum b/go.sum index 34df050..f4e3646 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,8 @@ github.com/mdlayher/socket v0.7.0 h1:qVREPVwtUMg17pwvveQxpurq0PVisMxi3FGgpsorMYQ github.com/mdlayher/socket v0.7.0/go.mod h1:f7iKql2EK/rfsWYDKofKjk2Ig7I+6HQWdMIdn1tO4A4= github.com/mdlayher/taskstats v0.0.0-20241219020249-a291fa5f5a69 h1:A33BCEd5ClzJCnUcicB1VRuRrgikbUc8J/v1qrtd2gw= github.com/mdlayher/taskstats v0.0.0-20241219020249-a291fa5f5a69/go.mod h1:0gPvvNyurVKWZFI09u1YSPCqhSnj6QwlwEijSrHXYag= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= diff --git a/prometheus/handler.go b/prometheus/handler.go index 39aca27..60568ff 100644 --- a/prometheus/handler.go +++ b/prometheus/handler.go @@ -153,7 +153,20 @@ func (h *Handler) ServeHTTP(res http.ResponseWriter, req *http.Request) { } w := io.Writer(res) - res.Header().Set("Content-Type", "text/plain; version=0.0.4") + protobuf := acceptsProtobuf(req.Header.Get("Accept")) + + // The format follows Accept and the compression Accept-Encoding, so a + // cache must not serve one response in place of the other. + res.Header().Add("Vary", "Accept") + res.Header().Add("Vary", "Accept-Encoding") + + // Text stays the default. Only the protobuf exposition carries created + // timestamps, and a scraper gets it only by asking for it first. + if protobuf { + res.Header().Set("Content-Type", protobufContentType) + } else { + res.Header().Set("Content-Type", "text/plain; version=0.0.4") + } if acceptEncoding(req.Header.Get("Accept-Encoding"), "gzip") { res.Header().Set("Content-Encoding", "gzip") @@ -162,6 +175,15 @@ func (h *Handler) ServeHTTP(res http.ResponseWriter, req *http.Request) { w = zw } + // Write errors, here and from WriteStats and the deferred gzip Close, + // are ignored: the only cause is the connection failing, and by then + // the status and part of the body have been sent, so there is nothing + // left to report the error with. The scraper sees a response that ends + // early and records a failed scrape. + if protobuf { + _ = h.writeProtobuf(w) + return + } h.WriteStats(w) } diff --git a/prometheus/metric.go b/prometheus/metric.go index 8975f91..d97cf52 100644 --- a/prometheus/metric.go +++ b/prometheus/metric.go @@ -311,6 +311,12 @@ type metricState struct { sum float64 count uint64 time time.Time + + // created is when the series started counting from zero, at its first + // update. The protobuf exposition publishes it so that a scraper can write a zero + // sample at that time, which lets increase() count the first value. A + // state re-created after MetricTimeout gets a new one. + created time.Time } func newMetricState(labels labels) *metricState { @@ -319,7 +325,7 @@ func newMetricState(labels labels) *metricState { } } -func (state *metricState) update(mtype metricType, value float64, time time.Time, buckets []stats.Value) { +func (state *metricState) update(mtype metricType, value float64, t time.Time, buckets []stats.Value) { state.mutex.Lock() switch mtype { @@ -334,6 +340,9 @@ func (state *metricState) update(mtype metricType, value float64, time time.Time // entry than the registry slice. Comparing against len(buckets) here // would rebuild — and zero the counts — on every observation. if len(state.buckets) != len(buckets)+1 { + // A rebuild zeroes the bucket counts, but not _sum and _count. + // Moving the created time forward would make a scraper write a + // zero under a _count that kept counting, so it stays put. state.buckets = makeMetricBuckets(buckets, state.labels) } state.buckets.update(value) @@ -341,7 +350,19 @@ func (state *metricState) update(mtype metricType, value float64, time time.Time state.count++ } - state.time = time + if state.created.IsZero() { + // A zero created time means none has been set, so a measure with a + // zero time cannot set one: the next update would take its place, + // and the series would claim to start later than its first value. + // state.time keeps the zero, so expiry treats this update like any + // other zero-time update. + created := t + if created.IsZero() { + created = time.Now() + } + state.created = created + } + state.time = t state.mutex.Unlock() } @@ -370,6 +391,13 @@ func (state *metricState) collect(metrics []metric, entry *metricEntry, name str var cumulativeCount uint64 for _, bucket := range state.buckets { cumulativeCount += bucket.count + if math.IsInf(bucket.limit, 1) { + // le="+Inf" is _count, as the scraper derives it from a + // protobuf exposition. The bucket total can fall short of it: + // a NaN lands in no bucket, and a rebuild zeroes the buckets + // but not _count. + cumulativeCount = state.count + } metrics = append(metrics, metric{ mtype: entry.mtype, scope: entry.scope, @@ -524,6 +552,13 @@ func (metrics byNameAndLabels) Swap(i, j int) { // // Comparing the two parts in turn rather than the joined "scope_name" avoids // building a string for every comparison in the sort. +// +// Two metrics can still tie on all three, a gauge "y_count" and the _count +// series of a histogram "y" for example. A scraper keeps the first of the two +// samples, and metrics come out of a map, so the tie is broken on the root +// name and then the type: otherwise the kept value would alternate between +// scrapes. The root name puts the histogram first, as the protobuf format +// does. func (metrics byNameAndLabels) Less(i, j int) bool { m1 := &metrics[i] m2 := &metrics[j] @@ -534,5 +569,14 @@ func (metrics byNameAndLabels) Less(i, j int) bool { if m1.name != m2.name { return m1.name < m2.name } - return m1.labels.less(m2.labels) + if m1.labels.less(m2.labels) { + return true + } + if m2.labels.less(m1.labels) { + return false + } + if r1, r2 := m1.rootName(), m2.rootName(); r1 != r2 { + return r1 < r2 + } + return m1.mtype < m2.mtype } diff --git a/prometheus/protobuf.go b/prometheus/protobuf.go new file mode 100644 index 0000000..0d1bfe3 --- /dev/null +++ b/prometheus/protobuf.go @@ -0,0 +1,343 @@ +package prometheus + +import ( + "cmp" + "io" + "math" + "mime" + "slices" + "strconv" + "strings" + "time" + + dto "github.com/prometheus/client_model/go" + "google.golang.org/protobuf/encoding/protodelim" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// protobufContentType is the Prometheus protobuf exposition: a stream of +// length-delimited io.prometheus.client.MetricFamily messages. +const protobufContentType = "application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited" + +// acceptsProtobuf reports whether the Accept header prefers the protobuf +// exposition over the text format. +// +// Protobuf is served only when the scraper ranks it above the text format. +// Those are the only two formats the handler serves, so other types, such as +// OpenMetrics, are not ranked. Prometheus and Alloy list their scrape protocols with descending q +// values. A scraper puts protobuf first when its scrape protocols are set that +// way, and Prometheus also does so by default when native histograms or +// created-timestamp zero ingestion are on. Any other scraper keeps receiving +// text. Equal q values keep header order. +func acceptsProtobuf(accept string) bool { + best, isProto := 0.0, false + + for part := range strings.SplitSeq(accept, ",") { + mediatype, params, err := mime.ParseMediaType(strings.TrimSpace(part)) + if err != nil { + continue + } + + q := 1.0 + if s, ok := params["q"]; ok { + if q, err = strconv.ParseFloat(s, 64); err != nil { + continue + } + } + + var pb bool + switch mediatype { + case "application/vnd.google.protobuf": + if params["proto"] != "io.prometheus.client.MetricFamily" || params["encoding"] != "delimited" { + continue + } + pb = true + case "text/plain", "text/*", "*/*": + default: + continue + } + + if q > best { + best, isProto = q, pb + } + } + + return isProto +} + +// writeProtobuf writes every metric as a length-delimited MetricFamily. +// +// Counters and histograms carry created_timestamp, the time their series +// started from zero. A scraper with start-timestamp zero ingestion enabled +// writes a zero sample at that time, so the first value of a new series is +// counted by increase() instead of being taken as its baseline. +// +// Names and labels are rendered exactly as WriteStats renders them, so +// switching format does not change any series on Prometheus 3.x. Histogram +// bounds are the exception on 2.x: the scraper formats UpperBound itself, as +// le="1.0" where the text format writes le="1", and only 3.x normalizes the +// text value to match. +// +// Each family's messages are built just before it is written and dropped +// after, so only one family's worth is held at a time. +func (h *Handler) writeProtobuf(w io.Writer) error { + for _, f := range h.metrics.families() { + if _, err := protodelim.MarshalTo(w, f.message()); err != nil { + return err + } + } + return nil +} + +type protoFamilyKey struct { + name string + mtype metricType +} + +type protoFamily struct { + name string + mtype metricType + series []protoSeries + first int // position of the family's first entry, for ordering +} + +func (f *protoFamily) message() *dto.MetricFamily { + var mtype dto.MetricType + switch f.mtype { + case counter: + mtype = dto.MetricType_COUNTER + case gauge: + mtype = dto.MetricType_GAUGE + case histogram: + mtype = dto.MetricType_HISTOGRAM + } + + mf := &dto.MetricFamily{ + Name: proto.String(f.name), + Type: mtype.Enum(), + Metric: make([]*dto.Metric, len(f.series)), + } + for i := range f.series { + mf.Metric[i] = f.series[i].protoMetric(f.mtype) + } + return mf +} + +// protoEntry is a store entry copied out under the locks, so that the +// messages can be built after they are released. +type protoEntry struct { + scope string + name string + mtype metricType + series []protoSeries +} + +type protoSeries struct { + labels labels + value float64 + sum float64 + count uint64 + created time.Time + buckets []protoBucket +} + +type protoBucket struct { + limit float64 + count uint64 +} + +// families returns the families to publish in order, each with its series +// sorted, without building their messages. +func (store *metricStore) families() []*protoFamily { + byKey := make(map[protoFamilyKey]*protoFamily) + + // Entries come out of a map, so they are put in the order WriteStats + // uses before anything depends on it. Where two entries render to the + // same name and labels, a scraper keeps the first sample and drops the + // other: an order that changed between scrapes would make the stored + // value alternate between two different series, and increase() would + // read every switch as a reset. + // + // One collision still keeps a different series than the text format + // does, though the same one on every scrape. A histogram sorts here under + // its root name, and in the text format each of its series sorts under + // its own: next to a histogram "x", a gauge "x.count" is listed after the + // histogram here and before its x_count in text, because '.' sorts before + // '_'. Matching it would mean reordering the text format for every + // metric. + entries := store.protoEntries() + slices.SortFunc(entries, func(a, b protoEntry) int { + return cmp.Or( + cmp.Compare(a.scope, b.scope), + cmp.Compare(a.name, b.name), + cmp.Compare(a.mtype, b.mtype), + ) + }) + + for i, entry := range entries { + // Two entries can render to the same name, a.b and a_b for example. + // Their series are merged into one family, so the name is declared + // once. + name := string(appendMetricScopedName(nil, entry.scope, entry.name)) + fk := protoFamilyKey{name: name, mtype: entry.mtype} + + f := byKey[fk] + if f == nil { + f = &protoFamily{name: name, mtype: entry.mtype, first: i} + byKey[fk] = f + } + f.series = append(f.series, entry.series...) + } + + families := make([]*protoFamily, 0, len(byKey)) + for _, f := range byKey { + if len(f.series) == 0 { + continue + } + // Stable, so that series with the same labels keep the order of the + // entries they came from. + slices.SortStableFunc(f.series, func(a, b protoSeries) int { + switch { + case a.labels.less(b.labels): + return -1 + case b.labels.less(a.labels): + return 1 + } + return 0 + }) + families = append(families, f) + } + + // A counter and a gauge can render to the same name too; their families + // are ordered by their first entry. + slices.SortFunc(families, func(a, b *protoFamily) int { + return cmp.Or( + cmp.Compare(a.name, b.name), + cmp.Compare(a.first, b.first), + ) + }) + return families +} + +// protoEntries copies out every series the protobuf exposition publishes. +// +// The locks are held only for the copy, as collect holds them. A writer +// waiting on store.mutex blocks every reader queued behind it, HandleMeasures +// included, so building the messages under the read lock would stall the +// application for as long as the build takes. +func (store *metricStore) protoEntries() []protoEntry { + // The bucket copies share one backing array, so that copying a histogram + // does not cost an allocation per series. + var buckets []protoBucket + + store.mutex.RLock() + entries := make([]protoEntry, 0, len(store.entries)) + + for key, entry := range store.entries { + switch entry.mtype { + case counter, gauge, histogram: + default: + // WriteStats publishes only these three types. + continue + } + + e := protoEntry{scope: entry.scope, name: store.exposedName(key, entry), mtype: entry.mtype} + + entry.mutex.RLock() + for _, states := range entry.states { + for _, state := range states { + var series protoSeries + series, buckets = state.protoSeries(buckets) + e.series = append(e.series, series) + } + } + entry.mutex.RUnlock() + + entries = append(entries, e) + } + + store.mutex.RUnlock() + return entries +} + +// protoSeries copies the state, appending its buckets to arena. +func (state *metricState) protoSeries(arena []protoBucket) (protoSeries, []protoBucket) { + state.mutex.Lock() + defer state.mutex.Unlock() + + s := protoSeries{ + labels: state.labels, + value: state.value, + sum: state.sum, + count: state.count, + created: state.created, + } + if len(state.buckets) != 0 { + start := len(arena) + for _, bucket := range state.buckets { + arena = append(arena, protoBucket{limit: bucket.limit, count: bucket.count}) + } + s.buckets = arena[start:len(arena):len(arena)] + } + return s, arena +} + +func (s *protoSeries) protoMetric(mtype metricType) *dto.Metric { + m := &dto.Metric{Label: protoLabels(s.labels)} + + switch mtype { + case counter: + m.Counter = &dto.Counter{ + Value: proto.Float64(s.value), + CreatedTimestamp: protoCreated(s.created), + } + + case gauge: + m.Gauge = &dto.Gauge{Value: proto.Float64(s.value)} + + case histogram: + // The +Inf bucket is left out, as client_golang does: its count is + // sample_count, and the scraper adds the le="+Inf" series itself. + h := &dto.Histogram{ + SampleCount: proto.Uint64(s.count), + SampleSum: proto.Float64(s.sum), + CreatedTimestamp: protoCreated(s.created), + } + var cumulativeCount uint64 + for _, bucket := range s.buckets { + cumulativeCount += bucket.count + if math.IsInf(bucket.limit, 1) { + continue + } + h.Bucket = append(h.Bucket, &dto.Bucket{ + CumulativeCount: proto.Uint64(cumulativeCount), + UpperBound: proto.Float64(bucket.limit), + }) + } + m.Histogram = h + } + + return m +} + +func protoCreated(created time.Time) *timestamppb.Timestamp { + if created.IsZero() { + return nil + } + return timestamppb.New(created) +} + +func protoLabels(l labels) []*dto.LabelPair { + if len(l) == 0 { + return nil + } + pairs := make([]*dto.LabelPair, len(l)) + for i, label := range l { + pairs[i] = &dto.LabelPair{ + Name: proto.String(string(appendLabelName(nil, label.name))), + Value: proto.String(label.value), + } + } + return pairs +} diff --git a/prometheus/protobuf_test.go b/prometheus/protobuf_test.go new file mode 100644 index 0000000..7b96777 --- /dev/null +++ b/prometheus/protobuf_test.go @@ -0,0 +1,774 @@ +package prometheus + +import ( + "bufio" + "bytes" + "compress/gzip" + "errors" + "io" + "math" + "net/http" + "net/http/httptest" + "runtime" + "slices" + "strconv" + "strings" + "sync" + "testing" + "time" + + dto "github.com/prometheus/client_model/go" + "google.golang.org/protobuf/encoding/protodelim" + + "github.com/segmentio/stats/v5" +) + +// The Accept header Alloy sends with scrape_protocols set to +// [PrometheusProto, OpenMetricsText1.0.0, OpenMetricsText0.0.1, PrometheusText0.0.4]. +const acceptProtoFirst = "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.5," + + "application/openmetrics-text;version=1.0.0;q=0.4,application/openmetrics-text;version=0.0.1;q=0.3," + + "text/plain;version=0.0.4;q=0.2,*/*;q=0.1" + +func TestAcceptsProtobuf(t *testing.T) { + tests := []struct { + name string + accept string + expect bool + }{ + {name: "none", accept: "", expect: false}, + {name: "curl", accept: "*/*", expect: false}, + {name: "protobuf first", accept: acceptProtoFirst, expect: true}, + { + name: "scraper defaults", + accept: "application/openmetrics-text;version=1.0.0;q=0.5,application/openmetrics-text;version=0.0.1;q=0.4,text/plain;version=0.0.4;q=0.3,*/*;q=0.2", + expect: false, + }, + { + // Prometheus' acceptHeader output for DefaultScrapeProtocols. Its + // weights start at one above the number of protocols it knows, so + // they shift between releases: q=0.6 on 3.0 to 3.14 (5 protocols), + // q=0.7 on 3.15 (6), and q=0.8 on main (7). + name: "prometheus main defaults", + accept: "application/openmetrics-text;version=1.0.0;escaping=allow-utf-8;q=0.8,application/openmetrics-text;version=0.0.1;q=0.7," + + "text/plain;version=1.0.0;escaping=allow-utf-8;q=0.6,text/plain;version=0.0.4;q=0.5,*/*;q=0.4", + expect: false, + }, + { + // DefaultProtoFirstScrapeProtocols, used with scrape_native_histograms + // or created-timestamp zero ingestion. + name: "prometheus main proto first", + accept: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.8," + + "application/openmetrics-text;version=1.0.0;escaping=allow-utf-8;q=0.7,application/openmetrics-text;version=0.0.1;q=0.6," + + "text/plain;version=1.0.0;escaping=allow-utf-8;q=0.5,text/plain;version=0.0.4;q=0.4,*/*;q=0.3", + expect: true, + }, + { + name: "prometheus 3.15 proto first", + accept: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7," + + "application/openmetrics-text;version=1.0.0;escaping=allow-utf-8;q=0.6,application/openmetrics-text;version=0.0.1;q=0.5," + + "text/plain;version=1.0.0;escaping=allow-utf-8;q=0.4,text/plain;version=0.0.4;q=0.3,*/*;q=0.2", + expect: true, + }, + { + // 3.0 to 3.3 append escaping=allow-utf-8 to every protocol under + // the default UTF-8 name validation, including protobuf, and twice + // to text 1.0.0, whose header already carries it. + name: "prometheus 3.0 proto first", + accept: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;escaping=allow-utf-8;q=0.6," + + "application/openmetrics-text;version=1.0.0;escaping=allow-utf-8;q=0.5,application/openmetrics-text;version=0.0.1;escaping=allow-utf-8;q=0.4," + + "text/plain;version=1.0.0;escaping=allow-utf-8;escaping=allow-utf-8;q=0.3,text/plain;version=0.0.4;escaping=allow-utf-8;q=0.2,*/*;q=0.1", + expect: true, + }, + { + name: "protobuf ranked below text", + accept: "text/plain;version=0.0.4;q=0.5,application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.4", + expect: false, + }, + { + name: "openmetrics above protobuf", + accept: "application/openmetrics-text;version=1.0.0;q=0.5,application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.4,text/plain;version=0.0.4;q=0.3", + expect: true, + }, + { + name: "protobuf only", + accept: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited", + expect: true, + }, + { + name: "protobuf refused", + accept: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0", + expect: false, + }, + { + name: "other message", + accept: "application/vnd.google.protobuf;proto=io.prometheus.client.Other;encoding=delimited", + expect: false, + }, + { + name: "not delimited", + accept: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=text", + expect: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if ok := acceptsProtobuf(test.accept); ok != test.expect { + t.Errorf("acceptsProtobuf(%q) = %t, expected %t", test.accept, ok, test.expect) + } + }) + } +} + +func scrapeProtobuf(t *testing.T, handler *Handler, gzipped bool) []*dto.MetricFamily { + t.Helper() + + server := httptest.NewServer(handler) + defer server.Close() + + req, err := http.NewRequest("GET", server.URL, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Accept", acceptProtoFirst) + if gzipped { + req.Header.Set("Accept-Encoding", "gzip") + } + + // A Transport left to itself would negotiate and undo gzip on its own. + res, err := (&http.Transport{DisableCompression: true}).RoundTrip(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + + if ct := res.Header.Get("Content-Type"); ct != protobufContentType { + t.Fatalf("Content-Type = %q, expected %q", ct, protobufContentType) + } + if vary := res.Header.Values("Vary"); !slices.Equal(vary, []string{"Accept", "Accept-Encoding"}) { + t.Errorf("Vary = %q, expected Accept and Accept-Encoding", vary) + } + + body := io.Reader(res.Body) + if gzipped { + if ce := res.Header.Get("Content-Encoding"); ce != "gzip" { + t.Fatalf("Content-Encoding = %q, expected gzip", ce) + } + zr, err := gzip.NewReader(res.Body) + if err != nil { + t.Fatal(err) + } + body = zr + } + + var families []*dto.MetricFamily + r := bufio.NewReader(body) + for { + mf := &dto.MetricFamily{} + if err := protodelim.UnmarshalFrom(r, mf); err != nil { + if errors.Is(err, io.EOF) { + return families + } + t.Fatal(err) + } + families = append(families, mf) + } +} + +func TestServeHTTPProtobuf(t *testing.T) { + first := time.Date(2017, 6, 4, 22, 12, 0, 0, time.UTC) + later := first.Add(30 * time.Second) + + handler := &Handler{ + Buckets: map[stats.Key][]stats.Value{ + {Field: "C"}: { + stats.ValueOf(0.25), + stats.ValueOf(0.5), + stats.ValueOf(0.75), + stats.ValueOf(1.0), + }, + }, + } + + handler.HandleMeasures(first, + stats.Measure{Fields: []stats.Field{stats.MakeField("A", 1, stats.Counter)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("C", 0.1, stats.Histogram)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("B", 1, stats.Gauge)}, Tags: []stats.Tag{stats.T("a", "1"), stats.T("b", "2")}}, + ) + handler.HandleMeasures(later, + stats.Measure{Fields: []stats.Field{stats.MakeField("A", 2, stats.Counter)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("A", 4, stats.Counter)}, Tags: []stats.Tag{stats.T("id", "123")}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("B", 42, stats.Gauge)}, Tags: []stats.Tag{stats.T("a", "1")}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("C", 0.1, stats.Histogram)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("C", 0.5, stats.Histogram)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("C", 10, stats.Histogram)}}, + ) + + for _, gzipped := range []bool{false, true} { + families := scrapeProtobuf(t, handler, gzipped) + + if len(families) != 3 { + t.Fatalf("found %d families, expected 3: %v", len(families), families) + } + a, b, c := families[0], families[1], families[2] + + // Counter: the name keeps _total, and each series carries the time of + // its own first update. + if a.GetName() != "A_total" || a.GetType() != dto.MetricType_COUNTER || len(a.Metric) != 2 { + t.Fatalf("bad counter family: %v", a) + } + if m := a.Metric[0]; len(m.Label) != 0 || m.GetCounter().GetValue() != 3 || + !m.GetCounter().GetCreatedTimestamp().AsTime().Equal(first) { + t.Errorf("bad unlabeled counter: %v", m) + } + if m := a.Metric[1]; m.Label[0].GetName() != "id" || m.Label[0].GetValue() != "123" || + m.GetCounter().GetValue() != 4 || !m.GetCounter().GetCreatedTimestamp().AsTime().Equal(later) { + t.Errorf("bad labeled counter: %v", m) + } + + // Gauge: no created timestamp. + if b.GetName() != "B" || b.GetType() != dto.MetricType_GAUGE || len(b.Metric) != 2 { + t.Fatalf("bad gauge family: %v", b) + } + if m := b.Metric[0]; m.GetGauge().GetValue() != 42 || len(m.Label) != 1 { + t.Errorf("bad gauge: %v", m) + } + if m := b.Metric[1]; m.GetGauge().GetValue() != 1 || len(m.Label) != 2 { + t.Errorf("bad gauge: %v", m) + } + + // Histogram: cumulative buckets without +Inf, which the scraper + // derives from sample_count. + if c.GetName() != "C" || c.GetType() != dto.MetricType_HISTOGRAM || len(c.Metric) != 1 { + t.Fatalf("bad histogram family: %v", c) + } + h := c.Metric[0].GetHistogram() + if h.GetSampleCount() != 4 || h.GetSampleSum() != 10.7 || + !h.GetCreatedTimestamp().AsTime().Equal(first) { + t.Errorf("bad histogram: %v", h) + } + expect := []struct { + le float64 + count uint64 + }{{0.25, 2}, {0.5, 3}, {0.75, 3}, {1, 3}} + if len(h.Bucket) != len(expect) { + t.Fatalf("found %d buckets, expected %d: %v", len(h.Bucket), len(expect), h.Bucket) + } + for i, e := range expect { + if h.Bucket[i].GetUpperBound() != e.le || h.Bucket[i].GetCumulativeCount() != e.count { + t.Errorf("bucket %d = %v, expected le=%g count=%d", i, h.Bucket[i], e.le, e.count) + } + } + } +} + +// TestCreatedTimestampAfterTimeout covers a comeback: a series MetricTimeout +// deleted and that is then updated again starts from zero, so it has to carry +// a new created time or the scraper has no zero to write for it. +func TestCreatedTimestampAfterTimeout(t *testing.T) { + first := time.Date(2017, 6, 4, 22, 12, 0, 0, time.UTC) + comeback := first.Add(10 * time.Minute) + + handler := &Handler{} + handler.HandleMeasures(first, stats.Measure{Fields: []stats.Field{stats.MakeField("A", 1, stats.Counter)}}) + handler.metrics.cleanup(first.Add(time.Minute)) + handler.HandleMeasures(comeback, stats.Measure{Fields: []stats.Field{stats.MakeField("A", 1, stats.Counter)}}) + + families := scrapeProtobuf(t, handler, false) + if len(families) != 1 || len(families[0].Metric) != 1 { + t.Fatalf("unexpected families: %v", families) + } + c := families[0].Metric[0].GetCounter() + if c.GetValue() != 1 || !c.GetCreatedTimestamp().AsTime().Equal(comeback) { + t.Errorf("counter after timeout = %v, expected value 1 created at %s", c, comeback) + } +} + +// TestCreatedTimestampZeroTime covers a first update with a zero time. A zero +// created time means none is set, so the series has to take the time it was +// first seen instead, or the next update would set created after the series +// already held a value and the scraper would write a zero under it. +func TestCreatedTimestampZeroTime(t *testing.T) { + handler := &Handler{} + + before := time.Now() + handler.HandleMeasures(time.Time{}, stats.Measure{Fields: []stats.Field{stats.MakeField("A", 5, stats.Counter)}}) + after := time.Now() + handler.HandleMeasures(after.Add(time.Minute), stats.Measure{Fields: []stats.Field{stats.MakeField("A", 1, stats.Counter)}}) + + families := scrapeProtobuf(t, handler, false) + if len(families) != 1 || len(families[0].Metric) != 1 { + t.Fatalf("unexpected families: %v", families) + } + c := families[0].Metric[0].GetCounter() + created := c.GetCreatedTimestamp().AsTime() + if c.GetValue() != 6 || created.Before(before) || created.After(after) { + t.Errorf("counter = %v, expected value 6 created between %s and %s", c, before, after) + } +} + +// TestZeroTimeUpdateKeepsZeroTime checks that the created time fallback +// leaves the update time alone: MetricTimeout expiry reads the update time, +// so a zero-time first update has to expire like any other zero-time update. +func TestZeroTimeUpdateKeepsZeroTime(t *testing.T) { + var state metricState + state.update(counter, 5, time.Time{}, nil) + + if state.created.IsZero() { + t.Error("created time not set by a zero-time first update") + } + if !state.time.IsZero() { + t.Errorf("update time = %s, expected zero", state.time) + } +} + +// TestHistogramInfBucket checks that le="+Inf" is _count in both formats, +// where the bucket total falls short of it: a NaN observation lands in no +// bucket, and a rebuild zeroes the buckets but not _count. +func TestHistogramInfBucket(t *testing.T) { + now := time.Date(2017, 6, 4, 22, 12, 0, 0, time.UTC) + key := stats.Key{Field: "H"} + observe := func(h *Handler, kind string, v float64) { + h.HandleMeasures(now, stats.Measure{ + Fields: []stats.Field{stats.MakeField("H", v, stats.Histogram)}, + Tags: []stats.Tag{stats.T("kind", kind)}, + }) + } + + handler := &Handler{Buckets: map[stats.Key][]stats.Value{key: {stats.ValueOf(1)}}} + observe(handler, "nan", 0.5) + observe(handler, "nan", 2) + observe(handler, "nan", math.NaN()) + observe(handler, "rebuild", 0.5) + observe(handler, "rebuild", 2) + handler.Buckets[key] = []stats.Value{stats.ValueOf(1), stats.ValueOf(5)} + observe(handler, "rebuild", 0.5) + + var text strings.Builder + handler.WriteStats(&text) + fromText := textSeries(t, text.String()) + fromProto := protoSeriesSet(scrapeProtobuf(t, handler, false)) + + for kind, count := range map[string]float64{"nan": 3, "rebuild": 3} { + inf := seriesKey("H_bucket", map[string]string{"kind": kind, "le": "+Inf"}) + if v := fromText[inf]; v != count { + t.Errorf("text %s = %g, expected %g", inf, v, count) + } + if v := fromProto[inf]; v != count { + t.Errorf("protobuf %s = %g, expected %g", inf, v, count) + } + } + if t.Failed() { + t.Log("text output:\n" + text.String()) + } +} + +// TestProtobufConcurrentScrape scrapes while measures arrive and series +// expire, for the race detector to check the copy taken under the locks. +func TestProtobufConcurrentScrape(t *testing.T) { + handler := &Handler{} + start := time.Now() + stop := make(chan struct{}) + var wg sync.WaitGroup + + wg.Add(2) + go func() { + defer wg.Done() + for i := 0; ; i++ { + select { + case <-stop: + return + default: + } + now := start.Add(time.Duration(i) * time.Millisecond) + tags := []stats.Tag{stats.T("id", strconv.Itoa(i%50))} + handler.HandleMeasures(now, + stats.Measure{Fields: []stats.Field{stats.MakeField("c", 1, stats.Counter)}, Tags: tags}, + stats.Measure{Fields: []stats.Field{stats.MakeField("g", float64(i), stats.Gauge)}, Tags: tags}, + stats.Measure{Fields: []stats.Field{stats.MakeField("h", 0.1, stats.Histogram)}, Tags: tags}, + ) + } + }() + go func() { + defer wg.Done() + for i := 0; ; i++ { + select { + case <-stop: + return + default: + } + handler.metrics.cleanup(start.Add(time.Duration(i) * time.Millisecond)) + } + }() + + for range 50 { + if err := handler.writeProtobuf(io.Discard); err != nil { + t.Error(err) + } + } + close(stop) + wg.Wait() +} + +// TestProtobufMatchesText checks that a scraper ingests the same series from +// the protobuf exposition as from the text format: the same names, label sets +// and values, so a scraper that switches format keeps writing to the same +// series. +// +// le is compared as a number. The scraper formats a protobuf bound itself, as +// le="1.0" where the text format writes le="1", and Prometheus 3.x normalizes +// the text value to the same form. +func TestProtobufMatchesText(t *testing.T) { + now := time.Date(2017, 6, 4, 22, 12, 0, 0, time.UTC) + + handler := &Handler{ + Buckets: map[stats.Key][]stats.Value{ + {Measure: "http.server", Field: "duration"}: { + stats.ValueOf(0.25), + stats.ValueOf(1), + stats.ValueOf(5), + }, + }, + } + tags := []stats.Tag{stats.T("method", "GET"), stats.T("status.code", "200")} + handler.HandleMeasures(now, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("requests.total", 3, stats.Counter)}, Tags: tags}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("requests.total", 1, stats.Counter)}, Tags: []stats.Tag{stats.T("method", "POST")}}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("hits", 1, stats.Counter)}}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("hits_total", 2, stats.Counter)}}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("inflight", 7, stats.Gauge)}, Tags: tags}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("duration", 0.1, stats.Histogram)}, Tags: tags}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("duration", 2, stats.Histogram)}, Tags: tags}, + stats.Measure{Name: "http.server", Fields: []stats.Field{stats.MakeField("duration", 10, stats.Histogram)}, Tags: tags}, + ) + + var text strings.Builder + handler.WriteStats(&text) + fromText := textSeries(t, text.String()) + fromProto := protoSeriesSet(scrapeProtobuf(t, handler, false)) + + for series, value := range fromText { + if v, ok := fromProto[series]; !ok { + t.Errorf("text series %s missing from protobuf", series) + } else if v != value { + t.Errorf("series %s = %g in protobuf, %g in text", series, v, value) + } + } + for series := range fromProto { + if _, ok := fromText[series]; !ok { + t.Errorf("protobuf series %s missing from text", series) + } + } + if t.Failed() { + t.Log("text output:\n" + text.String()) + } +} + +// TestProtobufCollisionOrder covers entries that render to the same name. A +// scraper keeps the first of two samples with the same name and labels, so +// the order has to be the same on every scrape, and the same as the text +// format's, or the stored value alternates between the two series. +func TestProtobufCollisionOrder(t *testing.T) { + now := time.Date(2017, 6, 4, 22, 12, 0, 0, time.UTC) + + handler := &Handler{} + handler.HandleMeasures(now, + // Both render to a_b_total. + stats.Measure{Fields: []stats.Field{stats.MakeField("a.b", 1, stats.Counter)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("a_b", 2, stats.Counter)}}, + // A counter and a gauge that both render to c_d_total. + stats.Measure{Fields: []stats.Field{stats.MakeField("c_d", 3, stats.Counter)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("c.d_total", 4, stats.Gauge)}}, + ) + + var first bytes.Buffer + if err := handler.writeProtobuf(&first); err != nil { + t.Fatal(err) + } + for range 100 { + var b bytes.Buffer + if err := handler.writeProtobuf(&b); err != nil { + t.Fatal(err) + } + if !bytes.Equal(b.Bytes(), first.Bytes()) { + t.Fatal("protobuf exposition changed between scrapes") + } + } + + // The text format lists a.b first, and the c.d_total gauge before the + // c_d counter, so a scraper keeps the same samples from either format. + families := scrapeProtobuf(t, handler, false) + if len(families) != 3 { + t.Fatalf("found %d families, expected 3: %v", len(families), families) + } + if m := families[0].Metric; families[0].GetName() != "a_b_total" || len(m) != 2 || + m[0].GetCounter().GetValue() != 1 || m[1].GetCounter().GetValue() != 2 { + t.Errorf("bad a_b_total family: %v", families[0]) + } + if families[1].GetName() != "c_d_total" || families[1].GetType() != dto.MetricType_GAUGE || + families[2].GetName() != "c_d_total" || families[2].GetType() != dto.MetricType_COUNTER { + t.Errorf("bad c_d_total families: %v %v", families[1], families[2]) + } +} + +// TestHistogramCollisionOrder covers a gauge that renders to the same name +// as one of a histogram's series. Both formats list the histogram first, so +// a scraper keeps its y_count on every scrape, whichever format it reads. +func TestHistogramCollisionOrder(t *testing.T) { + now := time.Date(2017, 6, 4, 22, 12, 0, 0, time.UTC) + + handler := &Handler{} + handler.HandleMeasures(now, + stats.Measure{Fields: []stats.Field{stats.MakeField("y_count", 70, stats.Gauge)}}, + stats.Measure{Fields: []stats.Field{stats.MakeField("y", 0.5, stats.Histogram)}}, + ) + + var first bytes.Buffer + handler.WriteStats(&first) + for range 100 { + var b bytes.Buffer + handler.WriteStats(&b) + if !bytes.Equal(b.Bytes(), first.Bytes()) { + t.Fatalf("text exposition changed between scrapes:\n%s\nthen:\n%s", first.String(), b.String()) + } + } + + var firstCount string + for line := range strings.SplitSeq(first.String(), "\n") { + if strings.HasPrefix(line, "y_count ") { + firstCount = line + break + } + } + if firstCount != "y_count 1" { + t.Errorf("text lists %q first, expected the histogram's y_count 1:\n%s", firstCount, first.String()) + } + + families := scrapeProtobuf(t, handler, false) + if len(families) != 2 || + families[0].GetName() != "y" || families[0].GetType() != dto.MetricType_HISTOGRAM || + families[1].GetName() != "y_count" || families[1].GetType() != dto.MetricType_GAUGE { + t.Errorf("protobuf does not list the histogram first: %v", families) + } +} + +// seriesKey renders a series as name{label="value",...} with the labels +// sorted and le normalized, so the same series is the same string whichever +// format it came from. +func seriesKey(name string, labels map[string]string) string { + names := make([]string, 0, len(labels)) + for n := range labels { + names = append(names, n) + } + slices.Sort(names) + + var b strings.Builder + b.WriteString(name) + b.WriteByte('{') + for i, n := range names { + if i != 0 { + b.WriteByte(',') + } + v := labels[n] + if n == "le" { + if f, err := strconv.ParseFloat(v, 64); err == nil { + v = strconv.FormatFloat(f, 'g', -1, 64) + } + } + b.WriteString(n + "=" + strconv.Quote(v)) + } + b.WriteByte('}') + return b.String() +} + +// textSeries parses the text exposition into its series and values. +func textSeries(t *testing.T, text string) map[string]float64 { + t.Helper() + set := make(map[string]float64) + + for line := range strings.SplitSeq(text, "\n") { + if line == "" || strings.HasPrefix(line, "#") { + continue + } + sep := strings.LastIndexByte(line, ' ') + value, err := strconv.ParseFloat(line[sep+1:], 64) + if err != nil { + t.Fatalf("bad value in %q: %v", line, err) + } + series := line[:sep] + + name, labels := series, map[string]string{} + if i := strings.IndexByte(series, '{'); i >= 0 { + name = series[:i] + rest := strings.TrimSuffix(series[i+1:], "}") + for rest != "" { + eq := strings.IndexByte(rest, '=') + v, err := strconv.QuotedPrefix(rest[eq+1:]) + if err != nil { + t.Fatalf("bad label in %q: %v", line, err) + } + labels[rest[:eq]], _ = strconv.Unquote(v) + rest = strings.TrimPrefix(rest[eq+1+len(v):], ",") + } + } + set[seriesKey(name, labels)] = value + } + return set +} + +// protoSeriesSet expands decoded families into the series a scraper stores +// for them: one per counter and gauge, and _bucket (with +Inf from +// sample_count), _sum and _count per histogram. +func protoSeriesSet(families []*dto.MetricFamily) map[string]float64 { + set := make(map[string]float64) + + for _, mf := range families { + for _, m := range mf.Metric { + labels := func(extra ...string) map[string]string { + l := make(map[string]string, len(m.Label)+1) + for _, p := range m.Label { + l[p.GetName()] = p.GetValue() + } + if len(extra) == 2 { + l[extra[0]] = extra[1] + } + return l + } + + switch mf.GetType() { + case dto.MetricType_COUNTER: + set[seriesKey(mf.GetName(), labels())] = m.GetCounter().GetValue() + case dto.MetricType_GAUGE: + set[seriesKey(mf.GetName(), labels())] = m.GetGauge().GetValue() + case dto.MetricType_HISTOGRAM: + h := m.GetHistogram() + for _, b := range h.Bucket { + le := strconv.FormatFloat(b.GetUpperBound(), 'g', -1, 64) + set[seriesKey(mf.GetName()+"_bucket", labels("le", le))] = float64(b.GetCumulativeCount()) + } + set[seriesKey(mf.GetName()+"_bucket", labels("le", "+Inf"))] = float64(h.GetSampleCount()) + set[seriesKey(mf.GetName()+"_sum", labels())] = h.GetSampleSum() + set[seriesKey(mf.GetName()+"_count", labels())] = float64(h.GetSampleCount()) + } + } + } + return set +} + +// benchmarkHandler returns a handler holding n series: counters with three +// labels, and one histogram series for every ten of them. +func benchmarkHandler(n int) *Handler { + handler := &Handler{} + now := time.Now() + for i := range n { + tags := []stats.Tag{ + stats.T("method", "GET"), + stats.T("status_code", strconv.Itoa(200+i%5)), + stats.T("url", "/v1/workspaces/"+strconv.Itoa(i)), + } + handler.HandleMeasures(now, stats.Measure{ + Name: "http", + Fields: []stats.Field{stats.MakeField("requests", 1, stats.Counter)}, + Tags: tags, + }) + if i%10 == 0 { + handler.HandleMeasures(now, stats.Measure{ + Name: "http", + Fields: []stats.Field{stats.MakeField("duration", 0.1, stats.Histogram)}, + Tags: tags, + }) + } + } + return handler +} + +// benchmarkHandlerFamilies is benchmarkHandler with its counters spread +// across families/2 names and its histograms across the other half. +func benchmarkHandlerFamilies(n, families int) *Handler { + handler := &Handler{} + now := time.Now() + names := max(families/2, 1) + for i := range n { + tags := []stats.Tag{ + stats.T("method", "GET"), + stats.T("status_code", strconv.Itoa(200+i%5)), + stats.T("url", "/v1/workspaces/"+strconv.Itoa(i)), + } + suffix := strconv.Itoa(i % names) + handler.HandleMeasures(now, stats.Measure{ + Name: "http", + Fields: []stats.Field{stats.MakeField("requests"+suffix, 1, stats.Counter)}, + Tags: tags, + }) + if i%10 == 0 { + handler.HandleMeasures(now, stats.Measure{ + Name: "http", + Fields: []stats.Field{stats.MakeField("duration"+suffix, 0.1, stats.Histogram)}, + Tags: tags, + }) + } + } + return handler +} + +// BenchmarkCollect is the text path's time under the store lock. +func BenchmarkCollect(b *testing.B) { + handler := benchmarkHandler(100000) + b.ReportAllocs() + for b.Loop() { + handler.metrics.collect(make([]metric, 0, 10000)) + } +} + +// BenchmarkWriteProtobuf is the whole protobuf exposition, built and +// marshaled. peak-heap-B is the most heap in use at any write, above what +// it was before the call: the messages built but not yet written. +func BenchmarkWriteProtobuf(b *testing.B) { + for _, families := range []int{2, 100} { + b.Run(strconv.Itoa(families)+"-families", func(b *testing.B) { + handler := benchmarkHandlerFamilies(100000, families) + w := &peakHeapWriter{} + b.ReportAllocs() + for b.Loop() { + b.StopTimer() + w.start() + b.StartTimer() + if err := handler.writeProtobuf(w); err != nil { + b.Fatal(err) + } + } + b.ReportMetric(float64(w.max), "peak-heap-B") + }) + } +} + +// peakHeapWriter discards what it is given and records the heap in use at +// each write. +type peakHeapWriter struct { + base, max uint64 +} + +func (w *peakHeapWriter) start() { + var m runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&m) + w.base = m.HeapAlloc +} + +func (w *peakHeapWriter) Write(p []byte) (int, error) { + var m runtime.MemStats + runtime.ReadMemStats(&m) + if m.HeapAlloc > w.base && m.HeapAlloc-w.base > w.max { + w.max = m.HeapAlloc - w.base + } + return len(p), nil +} + +// BenchmarkProtoEntries is the protobuf path's time under the store lock. +func BenchmarkProtoEntries(b *testing.B) { + handler := benchmarkHandler(100000) + b.ReportAllocs() + for b.Loop() { + handler.metrics.protoEntries() + } +}