diff --git a/HISTORY.md b/HISTORY.md index 41224b9..953ae91 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,16 @@ # History +### Unreleased + +**`prometheus.StartExporter` serves `/metrics` and tags the engine in one call.** +It registers a `Handler` on the engine (the default engine unless `Engine` is +set), adds an app name tag (`appName` unless `AppNameTag` is set), and serves the +handler on `Addr` with a graceful `Shutdown`. Services that scrape through a +collector no longer each carry their own copy of this setup. `Done` and `Err` +report a server that stops on its own, `WriteTimeout` and `IdleTimeout` default +to 30s and 60s, and `Shutdown` closes connections still open when its context +expires. + ### v5.13.0 **The `prometheus` handler can serve the protobuf exposition, with created diff --git a/README.md b/README.md index 8114862..4fb9331 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,22 @@ stats.Register(prometheus.DefaultHandler) http.Handle("/metrics", prometheus.DefaultHandler) ``` +To serve `/metrics` on its own port, and tag every series with the owning service +so a collector can name it, `StartExporter` registers the handler and runs the +server. It binds the listener before returning, so a bad address is an error: + +```go +exp, err := prometheus.StartExporter(prometheus.ExporterConfig{ + Addr: ":9102", + AppName: "my-service", // adds appName="my-service" to the engine's tags +}) +if err != nil { ... } +defer exp.Shutdown(ctx) +``` + +Call it before deriving engines with `WithTags` or `WithPrefix`, which inherit the +tags the engine has at that moment. + `Handler` has no `NewHandler` constructor; `DefaultHandler` is a ready-to-use zero-config instance. Construct your own `&prometheus.Handler{}` literal instead if you need to set `TrimPrefix`, `MetricTimeout`, or `Buckets`. diff --git a/prometheus/exporter.go b/prometheus/exporter.go new file mode 100644 index 0000000..6d7fc62 --- /dev/null +++ b/prometheus/exporter.go @@ -0,0 +1,174 @@ +package prometheus + +import ( + "cmp" + "context" + "errors" + "net" + "net/http" + "time" + + "github.com/segmentio/stats/v5" +) + +// DefaultAppNameTag is the tag the exporter adds to the engine to identify the +// service that owns the series. Collectors that derive a service name from the +// metrics read "service", then "appName"; "appName" is the default because many +// services already use "service" for the name of a downstream. +const DefaultAppNameTag = "appName" + +// ExporterConfig configures StartExporter. +type ExporterConfig struct { + // Addr is the listen address of the metrics server, for example ":9102". + // It is required. + Addr string + + // AppName is the value of the app name tag added to every series. Empty + // adds no tag. + AppName string + + // AppNameTag is the name of the tag carrying AppName. The default is + // DefaultAppNameTag. + AppNameTag string + + // Engine is the engine the handler is registered on and the tag is added + // to. The default is stats.DefaultEngine. + Engine *stats.Engine + + // Handler serves the metrics. The default is a new Handler using + // MetricTimeout. + Handler *Handler + + // MetricTimeout is how long a series is exposed after its last update. + // It is ignored when Handler is set. + MetricTimeout time.Duration + + // ReadHeaderTimeout bounds reading request headers. The default is 10s. + ReadHeaderTimeout time.Duration + + // WriteTimeout bounds writing a response, so a slow reader of /metrics + // cannot hold a connection open. The default is 30s. + WriteTimeout time.Duration + + // IdleTimeout bounds how long an idle keep-alive connection is kept. The + // default is 60s. + IdleTimeout time.Duration +} + +// Exporter is a running metrics server started by StartExporter. +type Exporter struct { + // Handler is the handler registered on the engine. + Handler *Handler + + server *http.Server + addr net.Addr + listener net.Listener + done chan struct{} + err error // set before done is closed +} + +// StartExporter serves the engine's metrics at /metrics on cfg.Addr, in +// addition to any handler already registered on the engine, which keeps running +// unchanged. +// +// It adds the app name tag to the engine's tags and registers the handler on the +// engine. Neither write is synchronized with the engine's readers, so call it +// before the engine is used from other goroutines, and before creating any engine +// derived from cfg.Engine (WithTags, WithPrefix), which would not inherit the tag. +// Calling it again on the same engine is safe: the tag is deduplicated by name. +// +// It returns once the listener is bound, so a bad address is an error here rather +// than a log line later. If the server stops on its own later, Done is closed and +// Err returns the cause. +func StartExporter(cfg ExporterConfig) (*Exporter, error) { + if cfg.Addr == "" { + return nil, errors.New("prometheus: ExporterConfig.Addr is required") + } + + engine := cfg.Engine + if engine == nil { + engine = stats.DefaultEngine + } + + if cfg.AppName != "" { + tag := cfg.AppNameTag + if tag == "" { + tag = DefaultAppNameTag + } + engine.Tags = stats.SortTags(append(append([]stats.Tag{}, engine.Tags...), stats.T(tag, cfg.AppName))) + } + + handler := cfg.Handler + if handler == nil { + handler = &Handler{MetricTimeout: cfg.MetricTimeout} + } + + readHeaderTimeout := cmp.Or(cfg.ReadHeaderTimeout, 10*time.Second) + writeTimeout := cmp.Or(cfg.WriteTimeout, 30*time.Second) + idleTimeout := cmp.Or(cfg.IdleTimeout, 60*time.Second) + + ln, err := net.Listen("tcp", cfg.Addr) + if err != nil { + return nil, err + } + + mux := http.NewServeMux() + mux.Handle("/metrics", handler) + + e := &Exporter{ + Handler: handler, + server: &http.Server{ + Handler: mux, + ReadHeaderTimeout: readHeaderTimeout, + WriteTimeout: writeTimeout, + IdleTimeout: idleTimeout, + }, + addr: ln.Addr(), + listener: ln, + done: make(chan struct{}), + } + + engine.Register(handler) + + go func() { + defer close(e.done) + if err := e.server.Serve(ln); !errors.Is(err, http.ErrServerClosed) { + e.err = err + } + }() + + return e, nil +} + +// Addr returns the address the server is listening on. +func (e *Exporter) Addr() net.Addr { return e.addr } + +// Done is closed when the server has stopped, either through Shutdown or because +// it failed. +func (e *Exporter) Done() <-chan struct{} { return e.done } + +// Err returns the error that stopped the server, or nil if it was shut down +// normally or is still running. It is only meaningful once Done is closed. +func (e *Exporter) Err() error { + select { + case <-e.done: + return e.err + default: + return nil + } +} + +// Shutdown stops the server, waiting for in-flight scrapes until ctx is done, then +// closing any connection still open. It returns the error that stopped the server +// early, if any. The handler stays registered on the engine. +func (e *Exporter) Shutdown(ctx context.Context) error { + err := e.server.Shutdown(ctx) + if err != nil { + e.server.Close() // ctx expired with scrapes in flight: do not leave them open + } + <-e.done + if err == nil { + err = e.err + } + return err +} diff --git a/prometheus/exporter_test.go b/prometheus/exporter_test.go new file mode 100644 index 0000000..bd8efac --- /dev/null +++ b/prometheus/exporter_test.go @@ -0,0 +1,302 @@ +package prometheus + +import ( + "context" + "io" + "net" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/segmentio/stats/v5" +) + +func TestStartExporter(t *testing.T) { + engine := stats.NewEngine("svc", stats.Discard, stats.T("env", "test")) + + e, err := StartExporter(ExporterConfig{Addr: "127.0.0.1:0", AppName: "my-app", Engine: engine}) + if err != nil { + t.Fatal(err) + } + defer e.Shutdown(context.Background()) + + engine.Add("requests", 3) + engine.Flush() + + res, err := http.Get("http://" + e.Addr().String() + "/metrics") + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + body, _ := io.ReadAll(res.Body) + + for _, want := range []string{"svc_requests", `appName="my-app"`, `env="test"`} { + if !strings.Contains(string(body), want) { + t.Errorf("missing %q in\n%s", want, body) + } + } +} + +func TestStartExporterAppNameTag(t *testing.T) { + engine := stats.NewEngine("svc", stats.Discard) + + e, err := StartExporter(ExporterConfig{Addr: "127.0.0.1:0", AppName: "x", AppNameTag: "service", Engine: engine}) + if err != nil { + t.Fatal(err) + } + defer e.Shutdown(context.Background()) + + if got := engine.Tags; len(got) != 1 || got[0] != stats.T("service", "x") { + t.Errorf("tags = %v", got) + } +} + +func TestStartExporterKeepsExistingHandler(t *testing.T) { + var seen int + engine := stats.NewEngine("svc", stats.HandlerFunc(func(time.Time, ...stats.Measure) { seen++ })) + + e, err := StartExporter(ExporterConfig{Addr: "127.0.0.1:0", Engine: engine}) + if err != nil { + t.Fatal(err) + } + defer e.Shutdown(context.Background()) + + engine.Incr("n") + engine.Flush() + if seen == 0 { + t.Error("existing handler no longer receives measures") + } +} + +func TestStartExporterErrors(t *testing.T) { + if _, err := StartExporter(ExporterConfig{}); err == nil { + t.Error("missing Addr: want error") + } + + e, err := StartExporter(ExporterConfig{Addr: "127.0.0.1:0", Engine: stats.NewEngine("a", stats.Discard)}) + if err != nil { + t.Fatal(err) + } + defer e.Shutdown(context.Background()) + + if _, err := StartExporter(ExporterConfig{Addr: e.Addr().String(), Engine: stats.NewEngine("b", stats.Discard)}); err == nil { + t.Error("address in use: want error") + } +} + +func TestExporterShutdownTwice(t *testing.T) { + e, err := StartExporter(ExporterConfig{Addr: "127.0.0.1:0", Engine: stats.NewEngine("a", stats.Discard)}) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if err := e.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown %d: %v", i, err) + } + } +} + +func scrapeBody(t *testing.T, e *Exporter) string { + t.Helper() + + res, err := http.Get("http://" + e.Addr().String() + "/metrics") + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + t.Fatalf("status = %d", res.StatusCode) + } + body, err := io.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + return string(body) +} + +func TestStartExporterUsesCallerHandler(t *testing.T) { + engine := stats.NewEngine("svc", stats.Discard) + h := &Handler{MetricTimeout: time.Hour} + + e, err := StartExporter(ExporterConfig{ + Addr: "127.0.0.1:0", + Engine: engine, + Handler: h, + MetricTimeout: time.Millisecond, // ignored when Handler is set + }) + if err != nil { + t.Fatal(err) + } + defer e.Shutdown(context.Background()) + + if e.Handler != h { + t.Fatal("Exporter.Handler is not the caller's handler") + } + if h.MetricTimeout != time.Hour { + t.Errorf("MetricTimeout = %v, the caller's handler was modified", h.MetricTimeout) + } + + engine.Add("hits", 1) + engine.Flush() + if body := scrapeBody(t, e); !strings.Contains(body, "svc_hits") { + t.Errorf("measure missing from the caller's handler output:\n%s", body) + } +} + +func TestStartExporterMetricTimeout(t *testing.T) { + e, err := StartExporter(ExporterConfig{ + Addr: "127.0.0.1:0", + Engine: stats.NewEngine("svc", stats.Discard), + MetricTimeout: 7 * time.Minute, + }) + if err != nil { + t.Fatal(err) + } + defer e.Shutdown(context.Background()) + + if e.Handler.MetricTimeout != 7*time.Minute { + t.Errorf("MetricTimeout = %v", e.Handler.MetricTimeout) + } +} + +func TestStartExporterDefaultsAndTimeouts(t *testing.T) { + e, err := StartExporter(ExporterConfig{Addr: "127.0.0.1:0", Engine: stats.NewEngine("svc", stats.Discard)}) + if err != nil { + t.Fatal(err) + } + defer e.Shutdown(context.Background()) + + if e.server.ReadHeaderTimeout != 10*time.Second || e.server.WriteTimeout != 30*time.Second || e.server.IdleTimeout != 60*time.Second { + t.Errorf("default timeouts = %v, %v, %v", e.server.ReadHeaderTimeout, e.server.WriteTimeout, e.server.IdleTimeout) + } + + custom, err := StartExporter(ExporterConfig{ + Addr: "127.0.0.1:0", + Engine: stats.NewEngine("svc2", stats.Discard), + ReadHeaderTimeout: time.Second, + WriteTimeout: 2 * time.Second, + IdleTimeout: 3 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + defer custom.Shutdown(context.Background()) + + if custom.server.ReadHeaderTimeout != time.Second || custom.server.WriteTimeout != 2*time.Second || custom.server.IdleTimeout != 3*time.Second { + t.Errorf("custom timeouts = %v, %v, %v", custom.server.ReadHeaderTimeout, custom.server.WriteTimeout, custom.server.IdleTimeout) + } +} + +func TestStartExporterDefaultEngine(t *testing.T) { + saved := stats.DefaultEngine.Tags + t.Cleanup(func() { stats.DefaultEngine.Tags = saved }) + + e, err := StartExporter(ExporterConfig{Addr: "127.0.0.1:0", AppName: "my-app"}) + if err != nil { + t.Fatal(err) + } + defer e.Shutdown(context.Background()) + + for _, tag := range stats.DefaultEngine.Tags { + if tag == stats.T(DefaultAppNameTag, "my-app") { + return + } + } + t.Errorf("default engine tags = %v, want %s=my-app", stats.DefaultEngine.Tags, DefaultAppNameTag) +} + +func TestStartExporterIsIdempotentOnTheTag(t *testing.T) { + engine := stats.NewEngine("svc", stats.Discard) + for i := 0; i < 2; i++ { + e, err := StartExporter(ExporterConfig{Addr: "127.0.0.1:0", AppName: "x", Engine: engine}) + if err != nil { + t.Fatal(err) + } + defer e.Shutdown(context.Background()) + } + if len(engine.Tags) != 1 { + t.Errorf("tags = %v, want one appName tag", engine.Tags) + } +} + +func TestExporterDoneAndErr(t *testing.T) { + e, err := StartExporter(ExporterConfig{Addr: "127.0.0.1:0", Engine: stats.NewEngine("svc", stats.Discard)}) + if err != nil { + t.Fatal(err) + } + + select { + case <-e.Done(): + t.Fatal("Done is closed while the server is running") + default: + } + if err := e.Err(); err != nil { + t.Fatalf("Err = %v while running", err) + } + + if err := e.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } + select { + case <-e.Done(): + case <-time.After(time.Second): + t.Fatal("Done not closed after Shutdown") + } + if err := e.Err(); err != nil { + t.Errorf("Err = %v after a clean shutdown", err) + } +} + +func TestExporterReportsServeFailure(t *testing.T) { + e, err := StartExporter(ExporterConfig{Addr: "127.0.0.1:0", Engine: stats.NewEngine("svc", stats.Discard)}) + if err != nil { + t.Fatal(err) + } + // Closing the listener out from under the server makes Serve return a non-ErrServerClosed error. + if err := e.listener.Close(); err != nil { + t.Fatal(err) + } + + select { + case <-e.Done(): + case <-time.After(5 * time.Second): + t.Fatal("Done not closed after the listener failed") + } + if e.Err() == nil { + t.Error("Err is nil after the listener failed") + } + if err := e.Shutdown(context.Background()); err == nil { + t.Error("Shutdown did not report the serve failure") + } +} + +func TestExporterShutdownClosesConnectionsOnTimeout(t *testing.T) { + e, err := StartExporter(ExporterConfig{Addr: "127.0.0.1:0", Engine: stats.NewEngine("svc", stats.Discard)}) + if err != nil { + t.Fatal(err) + } + + // Hold a request open: send the headers of a request but never finish it, so it stays active. + conn, err := net.Dial("tcp", e.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if _, err := conn.Write([]byte("GET /metrics HTTP/1.1\r\nHost: x\r\n")); err != nil { + t.Fatal(err) + } + time.Sleep(50 * time.Millisecond) + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if err := e.Shutdown(ctx); err == nil { + t.Fatal("Shutdown with an in-flight request and a short deadline returned nil") + } + + conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + if _, err := conn.Read(make([]byte, 1)); err == nil || os.IsTimeout(err) { + t.Errorf("connection still open after Shutdown timed out: %v", err) + } +}