From 46e4a5b029d3d1ffe703e236a947578475176e05 Mon Sep 17 00:00:00 2001 From: Tsu-Shiuan Date: Sat, 2 May 2026 07:05:51 +0200 Subject: [PATCH 1/3] feat: add JSON log handler for Logstash / Elasticsearch ingestion Adds Zexbox.Logging.install_json_handler!/1 (and the underlying Zexbox.Logging.JsonHandler) that swaps the default :logger handler's formatter for a JSON one wrapping LoggerJSON.Formatters.Basic. Mirrors the Ruby-side opsbox JsonFormatter so Phoenix logs land in Elasticsearch as one structured document per event instead of fanning multi-line content (Elixir struct inspections, multi-line SQL, stack traces) out across many docs at the Filebeat-ingest layer. Bumps the :elixir constraint from ~> 1.14 to ~> 1.15 to match logger_json 7.x's requirement. Adds logger_json ~> 7.0 and jason ~> 1.4 as dependencies. Companion changes that need to land separately to deliver the end-to-end pipeline: - kubernetes-infra: extend the Logstash filter that JSON-parses the rails container's message field to also match the phoenix and elixir container names. The existing block already handles the JSON shape this handler produces. - per-app: 3-line addition to runtime.exs, e.g. if config_env() == :prod do Zexbox.Logging.install_json_handler!() end Tests: 6 new tests covering formatter swap, default options, metadata allow-list, redactor passthrough, idempotency, and the parent-module delegate. Full zexbox suite: 37 tests, 0 failures. Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 17 +++++ README.md | 33 ++++++++++ lib/zexbox/logging.ex | 21 +++++++ lib/zexbox/logging/json_handler.ex | 76 +++++++++++++++++++++++ mix.exs | 4 +- mix.lock | 1 + test/zexbox/logging/json_handler_test.exs | 73 ++++++++++++++++++++++ 7 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 lib/zexbox/logging/json_handler.ex create mode 100644 test/zexbox/logging/json_handler_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 72adff8..2e8dd2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added +- `Zexbox.Logging.install_json_handler!/1` (and the underlying + `Zexbox.Logging.JsonHandler`) — swaps the default `:logger` handler's + formatter for a JSON one wrapping `LoggerJSON.Formatters.Basic`. Mirrors + the Ruby-side opsbox `JsonFormatter` so Phoenix logs land in + Elasticsearch as one structured document per event instead of fanning + multi-line content out into many. + +### Changed +- `:elixir` constraint bumped from `~> 1.14` to `~> 1.15` to match the + `logger_json` 7.x requirement. + +### Dependencies +- Adds `logger_json ~> 7.0` and `jason ~> 1.4`. + ## 1.5.1 - 2026-02-05 - Handles cases where `$callers` and `$ancestors` may not be pids to avoid crashing metric handler. diff --git a/README.md b/README.md index 7b01eab..4fc5dfc 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,39 @@ Adding your own logs is as simple as calling the `Zexbox.Telementry.attach/4` (w Zexbox.Telemetry.attach(:my_event, [:my, :event], &MyAppHandler.my_handler/3, nil) ``` +### JSON-formatted logs for Logstash / Elasticsearch + +By default the Elixir Logger emits multi-line plain-text output. Filebeat +ships every line as a separate Elasticsearch document, so a single struct +inspection or stack trace can fan out into dozens of indexed docs. The +`Zexbox.Logging` JSON handler swaps the default `:logger` formatter for a +JSON one — every log event becomes a single line of JSON, so multi-line +content collapses into one ES document at the ingest layer. This mirrors +the behaviour of opsbox's `JsonFormatter` on the Ruby side. + +In `config/runtime.exs`: + +```elixir +if config_env() == :prod do + Zexbox.Logging.install_json_handler!() +end +``` + +Output (one line per event): + +```json +{"time":"2026-05-02T01:23:45.678Z","severity":"info","message":"Hello","metadata":{...}} +``` + +To filter Logger metadata down to a specific allow-list, pass it through: + +```elixir +Zexbox.Logging.install_json_handler!(metadata: [:request_id, :trace_id, :user_id]) +``` + +See `Zexbox.Logging.JsonHandler` for the full option list, including +redactor support for stripping sensitive metadata before serialisation. + ## Metrics In order to setup metrics with InfluxDB you'll need to add the following configuration: diff --git a/lib/zexbox/logging.ex b/lib/zexbox/logging.ex index 758b508..c4e2201 100644 --- a/lib/zexbox/logging.ex +++ b/lib/zexbox/logging.ex @@ -3,9 +3,30 @@ defmodule Zexbox.Logging do Module for logging events in Zexbox. """ + alias Zexbox.Logging.JsonHandler alias Zexbox.Logging.LogHandler alias Zexbox.Telemetry + @doc """ + Installs a JSON formatter on the default `:logger` handler so every + log event is emitted as a single JSON line. + + Designed to mirror the behaviour of opsbox's `JsonFormatter` on the Ruby + side. Multi-line content (Elixir struct inspections, multi-line SQL, + stack traces) collapses into a single Elasticsearch document at the + ingest layer rather than fanning out into many. + + Typical usage in `config/runtime.exs`: + + if config_env() == :prod do + Zexbox.Logging.install_json_handler!() + end + + See `Zexbox.Logging.JsonHandler` for the full option list. + """ + @spec install_json_handler!(keyword()) :: :ok | {:error, term()} + defdelegate install_json_handler!(opts \\ []), to: JsonHandler, as: :install! + @doc """ Attaches Telemetry handlers for Phoenix controller events. diff --git a/lib/zexbox/logging/json_handler.ex b/lib/zexbox/logging/json_handler.ex new file mode 100644 index 0000000..5a6f3d6 --- /dev/null +++ b/lib/zexbox/logging/json_handler.ex @@ -0,0 +1,76 @@ +defmodule Zexbox.Logging.JsonHandler do + @moduledoc """ + Replaces the default `:logger` handler's formatter with a JSON formatter + suitable for Logstash / Elasticsearch ingestion. + + Emits one JSON object per line, so multi-line content (Elixir struct + inspections, multi-line SQL, stack traces) collapses into a single log + event at the ingest layer instead of fanning out into N separate + Elasticsearch documents. + + This is the Phoenix / Elixir equivalent of opsbox's `JsonFormatter` for Ruby. + Wraps `LoggerJSON.Formatters.Basic` with sensible defaults for the + Zappi log-ingest pipeline. + + ## Setup + + In your application's `config/runtime.exs`: + + if config_env() == :prod do + Zexbox.Logging.JsonHandler.install!() + end + + After install, every log line is a single JSON object: + + {"time":"...","severity":"info","message":"...","metadata":{...}} + + Logstash's existing `kubernetes.container.name` rules can then JSON-parse + these into structured fields the same way they do for opsbox-formatted + Ruby logs. + + ## Options + + * `:metadata` - which `Logger` metadata keys to include. Defaults to + `:all` (every key set via `Logger.metadata/1` or + `Logger.put_application_level/2`). Pass a list (e.g. + `[:request_id, :trace_id]`) to filter, or `[]` to omit metadata. + + * `:redactors` - a list of `LoggerJSON.Redactor` modules applied to + metadata values before serialisation, e.g. for stripping sensitive + keys. Defaults to `[]`. + + ## Idempotency + + Safe to call multiple times — `install!/1` simply swaps the formatter + on the existing default handler. Subsequent calls overwrite the previous + formatter config. + """ + + @doc """ + Install the JSON formatter on the `:default` `:logger` handler. + + Returns `:ok` on success or `{:error, reason}` if the default handler + hasn't been configured (typically only in unusual test setups). + + ## Examples + + iex> Zexbox.Logging.JsonHandler.install!() + :ok + + iex> Zexbox.Logging.JsonHandler.install!(metadata: [:request_id, :trace_id]) + :ok + """ + @spec install!(keyword()) :: :ok | {:error, term()} + def install!(opts \\ []) do + formatter_config = %{ + metadata: Keyword.get(opts, :metadata, :all), + redactors: Keyword.get(opts, :redactors, []) + } + + :logger.update_handler_config( + :default, + :formatter, + {LoggerJSON.Formatters.Basic, formatter_config} + ) + end +end diff --git a/mix.exs b/mix.exs index 5e5dcb2..9c85153 100644 --- a/mix.exs +++ b/mix.exs @@ -5,7 +5,7 @@ defmodule Zexbox.MixProject do [ app: :zexbox, version: "1.5.1", - elixir: "~> 1.14", + elixir: "~> 1.15", start_permanent: Mix.env() == :prod, dialyzer: [plt_add_apps: [:mix, :ex_unit]], description: description(), @@ -37,7 +37,9 @@ defmodule Zexbox.MixProject do {:doctor, "~> 0.22.0", only: [:dev, :test]}, {:ex_doc, "~> 0.35.1", only: :dev, runtime: false}, {:instream, "~> 2.2"}, + {:jason, "~> 1.4"}, {:ldclient, "~> 3.8.0", hex: :launchdarkly_server_sdk}, + {:logger_json, "~> 7.0"}, {:mix_audit, "~> 2.0", only: [:dev, :test], runtime: false}, {:mock, "~> 0.3.0", only: :test}, {:sobelow, "~> 0.8", only: [:dev, :test]}, diff --git a/mix.lock b/mix.lock index f3472fc..f5170de 100644 --- a/mix.lock +++ b/mix.lock @@ -19,6 +19,7 @@ "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "jsx": {:hex, :jsx, "3.1.0", "d12516baa0bb23a59bb35dccaf02a1bd08243fcbb9efe24f2d9d056ccff71268", [:rebar3], [], "hexpm", "0c5cc8fdc11b53cc25cf65ac6705ad39e54ecc56d1c22e4adb8f5a53fb9427f3"}, "ldclient": {:hex, :launchdarkly_server_sdk, "3.8.0", "4e900c33cfa9fdcd80f01de371d1108eb8f3834f41a1c56d1f739597a6d8bf57", [:rebar3], [{:certifi, "~> 2.14", [hex: :certifi, repo: "hexpm", optional: false]}, {:eredis, "1.7.1", [hex: :eredis, repo: "hexpm", optional: false]}, {:jsx, "3.1.0", [hex: :jsx, repo: "hexpm", optional: false]}, {:lru, "2.4.0", [hex: :lru, repo: "hexpm", optional: false]}, {:shotgun, "1.2.1", [hex: :shotgun, repo: "hexpm", optional: false]}, {:uuid, "~> 2.0.2", [hex: :uuid_erl, repo: "hexpm", optional: false]}, {:verl, "1.0.1", [hex: :verl, repo: "hexpm", optional: false]}, {:yamerl, "0.10.0", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "015deb283e9e964ebd02cc3e95c21828ff41ce62b6115793d4e93c01cd6af661"}, + "logger_json": {:hex, :logger_json, "7.0.4", "e315f2b9a755504658a745f3eab90d88d2cd7ac2ecfd08c8da94d8893965ab5c", [:mix], [{:decimal, ">= 0.0.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:ecto, "~> 3.11", [hex: :ecto, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "d1369f8094e372db45d50672c3b91e8888bcd695fdc444a37a0734e96717c45c"}, "lru": {:hex, :lru, "2.4.0", "a8f9967ca9b6f260baa19e2efb2aeb3853a3f5bd5f8416f537a672294b38c1bc", [:rebar3], [], "hexpm", "4fcf77e882b5e57eca068999acba4386a20dbce8e446c98c6a0f8fb3d170afeb"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, diff --git a/test/zexbox/logging/json_handler_test.exs b/test/zexbox/logging/json_handler_test.exs new file mode 100644 index 0000000..a420328 --- /dev/null +++ b/test/zexbox/logging/json_handler_test.exs @@ -0,0 +1,73 @@ +defmodule Zexbox.Logging.JsonHandlerTest do + use ExUnit.Case, async: false + + alias Zexbox.Logging.JsonHandler + + setup do + {:ok, original_config} = :logger.get_handler_config(:default) + on_exit(fn -> :logger.update_handler_config(:default, original_config) end) + :ok + end + + describe "install!/1" do + test "swaps the default handler's formatter to LoggerJSON.Formatters.Basic" do + assert :ok = JsonHandler.install!() + + {:ok, %{formatter: {formatter_module, _config}}} = + :logger.get_handler_config(:default) + + assert formatter_module == LoggerJSON.Formatters.Basic + end + + test "passes metadata: :all by default" do + assert :ok = JsonHandler.install!() + + {:ok, %{formatter: {_module, config}}} = + :logger.get_handler_config(:default) + + assert config.metadata == :all + end + + test "passes through a metadata allow-list" do + assert :ok = JsonHandler.install!(metadata: [:request_id, :trace_id]) + + {:ok, %{formatter: {_module, config}}} = + :logger.get_handler_config(:default) + + assert config.metadata == [:request_id, :trace_id] + end + + test "passes through redactors" do + redactors = [{LoggerJSON.Redactors.RedactKeys, ["password"]}] + assert :ok = JsonHandler.install!(redactors: redactors) + + {:ok, %{formatter: {_module, config}}} = + :logger.get_handler_config(:default) + + assert config.redactors == redactors + end + + test "is idempotent across repeated calls" do + assert :ok = JsonHandler.install!() + assert :ok = JsonHandler.install!() + assert :ok = JsonHandler.install!(metadata: [:request_id]) + + {:ok, %{formatter: {formatter_module, config}}} = + :logger.get_handler_config(:default) + + assert formatter_module == LoggerJSON.Formatters.Basic + assert config.metadata == [:request_id] + end + end + + describe "Zexbox.Logging.install_json_handler!/1 delegate" do + test "the parent module exposes the same function" do + assert :ok = Zexbox.Logging.install_json_handler!() + + {:ok, %{formatter: {formatter_module, _config}}} = + :logger.get_handler_config(:default) + + assert formatter_module == LoggerJSON.Formatters.Basic + end + end +end From 933e59e53db051447baf9f63a3b1c0d4f0036e22 Mon Sep 17 00:00:00 2001 From: Tsu-Shiuan Date: Sat, 2 May 2026 07:29:26 +0200 Subject: [PATCH 2/3] chore: appease credo --strict on multi-alias consistency Group Zexbox.Logging.JsonHandler + Zexbox.Logging.LogHandler into the brace-expansion form so the file uses a single alias style for that namespace. [Consistency] Most of the time you are using the multi-alias/require/import/use syntax, but here you are using multiple single directives. No behaviour change. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/zexbox/logging.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/zexbox/logging.ex b/lib/zexbox/logging.ex index c4e2201..e569b6f 100644 --- a/lib/zexbox/logging.ex +++ b/lib/zexbox/logging.ex @@ -3,8 +3,7 @@ defmodule Zexbox.Logging do Module for logging events in Zexbox. """ - alias Zexbox.Logging.JsonHandler - alias Zexbox.Logging.LogHandler + alias Zexbox.Logging.{JsonHandler, LogHandler} alias Zexbox.Telemetry @doc """ From ff9a7eee9948ce6a691fd7eb5a01046be1bd60a0 Mon Sep 17 00:00:00 2001 From: Tsu-Shiuan Date: Tue, 5 May 2026 08:38:35 +0200 Subject: [PATCH 3/3] refactor: replace install_json_handler! with declarative JsonFormatter.config/1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses Brendon's review on PR #57. Three changes, all in service of keeping the logging shape declarative and idiomatic: 1. Drop the runtime installer in favour of a config-returning helper. `:logger` is configured before `Application.start/2` runs, so a `Foo.install!()` call in user code activates the JSON formatter *after* boot logs have already gone out in plain text. Returning the formatter tuple to splat into `config :logger, :default_handler, formatter: ...` makes JSON active from the first log line emitted by the BEAM and keeps log-shape config visible in `runtime.exs` rather than buried in application code. 2. Drop the bang on the public function. The previous `install!/1 :: :ok | {:error, term()}` violated Elixir naming convention — `!` means "raises on failure". `JsonFormatter.config/1` returns a value and never raises. 3. Stop pinning `:jason` directly. `logger_json` declares Jason as optional; pinning it in zexbox forces the encoder choice on consumer apps. On Elixir 1.18+ apps can now opt into the stdlib `JSON` module via `config :logger_json, encoder: JSON`. Jason still arrives transitively via `instream`, `ldclient`, `mix_audit`, and `sobelow`, so dev/test continues to work. Defaults now live in `Zexbox.Logging.JsonFormatter`: - `default_metadata/0` — curated allow-list (request_id, trace_id, span_id, user_id, pid, module, function, line, application). - `default_redactors/0` — a single `RedactKeys` redactor stripping common credential keys (password, secret, token, authorization, api_key, session). Both overridable via `config/1` opts. Tests are now `async: true` because the new function doesn't touch global `:logger` state. Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 21 +++-- README.md | 47 ++++++++--- lib/zexbox/logging.ex | 28 +++---- lib/zexbox/logging/json_formatter.ex | 89 +++++++++++++++++++++ lib/zexbox/logging/json_handler.ex | 76 ------------------ mix.exs | 1 - test/zexbox/logging/json_formatter_test.exs | 34 ++++++++ test/zexbox/logging/json_handler_test.exs | 73 ----------------- 8 files changed, 188 insertions(+), 181 deletions(-) create mode 100644 lib/zexbox/logging/json_formatter.ex delete mode 100644 lib/zexbox/logging/json_handler.ex create mode 100644 test/zexbox/logging/json_formatter_test.exs delete mode 100644 test/zexbox/logging/json_handler_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e8dd2c..cf5c5e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,19 +8,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased ### Added -- `Zexbox.Logging.install_json_handler!/1` (and the underlying - `Zexbox.Logging.JsonHandler`) — swaps the default `:logger` handler's - formatter for a JSON one wrapping `LoggerJSON.Formatters.Basic`. Mirrors - the Ruby-side opsbox `JsonFormatter` so Phoenix logs land in - Elasticsearch as one structured document per event instead of fanning - multi-line content out into many. +- `Zexbox.Logging.JsonFormatter.config/1` (and the + `Zexbox.Logging.json_formatter_config/1` delegate) — returns a + `:logger` formatter tuple wrapping `LoggerJSON.Formatters.Basic` with + Zappi-standard metadata and redactor defaults. Designed to be splatted + into `config :logger, :default_handler, formatter: ...` in + `runtime.exs`. Mirrors the Ruby-side opsbox `JsonFormatter` so Phoenix + logs land in Elasticsearch as one structured document per event + instead of fanning multi-line content out into many. ### Changed - `:elixir` constraint bumped from `~> 1.14` to `~> 1.15` to match the `logger_json` 7.x requirement. ### Dependencies -- Adds `logger_json ~> 7.0` and `jason ~> 1.4`. +- Adds `logger_json ~> 7.0`. + +### Notes +- This release does not pin `:jason`. Consumers may include it directly + (the default `LoggerJSON` encoder) or, on Elixir 1.18+, set + `config :logger_json, encoder: JSON` to use the stdlib `JSON` module. ## 1.5.1 - 2026-02-05 diff --git a/README.md b/README.md index 4fc5dfc..a74f08a 100644 --- a/README.md +++ b/README.md @@ -147,17 +147,25 @@ Zexbox.Telemetry.attach(:my_event, [:my, :event], &MyAppHandler.my_handler/3, ni By default the Elixir Logger emits multi-line plain-text output. Filebeat ships every line as a separate Elasticsearch document, so a single struct -inspection or stack trace can fan out into dozens of indexed docs. The -`Zexbox.Logging` JSON handler swaps the default `:logger` formatter for a -JSON one — every log event becomes a single line of JSON, so multi-line -content collapses into one ES document at the ingest layer. This mirrors -the behaviour of opsbox's `JsonFormatter` on the Ruby side. +inspection or stack trace can fan out into dozens of indexed docs. +`Zexbox.Logging.JsonFormatter` returns a `:logger` formatter tuple that +swaps the default formatter for a JSON one — every log event becomes a +single line of JSON, so multi-line content collapses into one ES +document at the ingest layer. This mirrors the behaviour of opsbox's +`JsonFormatter` on the Ruby side. + +The configuration is declarative — `:logger` is configured before +`Application.start/2` runs, so logs emitted during application boot use +the formatter that has been *configured*, not one installed at runtime. In `config/runtime.exs`: ```elixir +import Config + if config_env() == :prod do - Zexbox.Logging.install_json_handler!() + config :logger, :default_handler, + formatter: Zexbox.Logging.json_formatter_config() end ``` @@ -167,14 +175,33 @@ Output (one line per event): {"time":"2026-05-02T01:23:45.678Z","severity":"info","message":"Hello","metadata":{...}} ``` -To filter Logger metadata down to a specific allow-list, pass it through: +To override the metadata allow-list or redactors: + +```elixir +config :logger, :default_handler, + formatter: + Zexbox.Logging.json_formatter_config( + metadata: [:request_id, :trace_id, :user_id], + redactors: [] + ) +``` + +The defaults include a `RedactKeys` redactor stripping common credential +metadata keys; see `Zexbox.Logging.JsonFormatter.default_metadata/0` and +`default_redactors/0` for the full lists. + +### Encoder choice + +`logger_json` uses Jason as the JSON encoder by default. On Elixir 1.18+ +you can opt into the stdlib `JSON` module — set this in +`config/config.exs` (compile-time): ```elixir -Zexbox.Logging.install_json_handler!(metadata: [:request_id, :trace_id, :user_id]) +config :logger_json, encoder: JSON ``` -See `Zexbox.Logging.JsonHandler` for the full option list, including -redactor support for stripping sensitive metadata before serialisation. +Otherwise add `{:jason, "~> 1.4"}` to your application's deps. Zexbox +intentionally does not pin an encoder so consumers can choose. ## Metrics diff --git a/lib/zexbox/logging.ex b/lib/zexbox/logging.ex index e569b6f..a589405 100644 --- a/lib/zexbox/logging.ex +++ b/lib/zexbox/logging.ex @@ -3,28 +3,28 @@ defmodule Zexbox.Logging do Module for logging events in Zexbox. """ - alias Zexbox.Logging.{JsonHandler, LogHandler} + alias Zexbox.Logging.{JsonFormatter, LogHandler} alias Zexbox.Telemetry @doc """ - Installs a JSON formatter on the default `:logger` handler so every - log event is emitted as a single JSON line. - - Designed to mirror the behaviour of opsbox's `JsonFormatter` on the Ruby - side. Multi-line content (Elixir struct inspections, multi-line SQL, - stack traces) collapses into a single Elasticsearch document at the - ingest layer rather than fanning out into many. - - Typical usage in `config/runtime.exs`: + Returns a `:logger` formatter tuple wrapping `LoggerJSON.Formatters.Basic` + with Zappi-standard defaults. Designed to be splatted into + `config :logger, :default_handler, formatter: ...` in `config/runtime.exs`: if config_env() == :prod do - Zexbox.Logging.install_json_handler!() + config :logger, :default_handler, + formatter: Zexbox.Logging.json_formatter_config() end - See `Zexbox.Logging.JsonHandler` for the full option list. + Mirrors opsbox's Ruby-side `JsonFormatter`: every log event becomes one + JSON object on one line, so multi-line content (struct inspections, + multi-line SQL, stack traces) collapses into a single Elasticsearch + document at the ingest layer. + + See `Zexbox.Logging.JsonFormatter` for the full option list. """ - @spec install_json_handler!(keyword()) :: :ok | {:error, term()} - defdelegate install_json_handler!(opts \\ []), to: JsonHandler, as: :install! + @spec json_formatter_config(keyword()) :: {module(), map()} + defdelegate json_formatter_config(opts \\ []), to: JsonFormatter, as: :config @doc """ Attaches Telemetry handlers for Phoenix controller events. diff --git a/lib/zexbox/logging/json_formatter.ex b/lib/zexbox/logging/json_formatter.ex new file mode 100644 index 0000000..0a903e4 --- /dev/null +++ b/lib/zexbox/logging/json_formatter.ex @@ -0,0 +1,89 @@ +defmodule Zexbox.Logging.JsonFormatter do + @moduledoc """ + Returns a `:logger` formatter tuple suitable for Logstash / Elasticsearch + ingestion, wrapping `LoggerJSON.Formatters.Basic` with Zappi-standard + defaults. + + Splat into `config/runtime.exs`: + + config :logger, :default_handler, + formatter: Zexbox.Logging.JsonFormatter.config() + + Mirrors opsbox's Ruby-side `JsonFormatter`: every log event becomes one + JSON object on one line, so multi-line content (struct inspections, + multi-line SQL, stack traces) collapses into a single Elasticsearch + document at the ingest layer. + + ## Why declarative + + `:logger` is configured before `Application.start/2` runs. Logs emitted + during application boot use the formatter that has been *configured* — + not one installed imperatively at runtime. Pure config makes JSON + active from the first log line emitted by the BEAM. + + ## Encoder + + Consumers choose the JSON encoder. The default for `LoggerJSON` is + Jason; on Elixir 1.18+ you can opt into the stdlib `JSON` module with: + + config :logger_json, encoder: JSON + + This is a compile-time setting in `config/config.exs`. + """ + + alias LoggerJSON.{Formatters.Basic, Redactors.RedactKeys} + + @default_metadata [ + :request_id, + :trace_id, + :span_id, + :user_id, + :pid, + :module, + :function, + :line, + :application + ] + + @default_redactors [ + {RedactKeys, ["password", "secret", "token", "authorization", "api_key", "session"]} + ] + + @doc """ + Returns the formatter tuple for `config :logger, :default_handler, formatter: ...`. + + ## Options + + * `:metadata` — list of `Logger` metadata keys to include in the JSON + output, or `:all` to include every key set via `Logger.metadata/1`. + Defaults to a curated Zappi list (see `default_metadata/0`). Pass + an explicit list to override; metadata is a security-adjacent + surface and `:all` is **not** recommended for production. + + * `:redactors` — list of `LoggerJSON.Redactor` tuples. Defaults to a + single `RedactKeys` redactor stripping common credential keys + (see `default_redactors/0`). Pass `[]` to disable redaction; pass + a list to extend or override. + + ## Examples + + iex> {mod, _config} = Zexbox.Logging.JsonFormatter.config() + iex> mod + LoggerJSON.Formatters.Basic + """ + @spec config(keyword()) :: {module(), map()} + def config(opts \\ []) do + Basic.new( + metadata: Keyword.get(opts, :metadata, @default_metadata), + redactors: Keyword.get(opts, :redactors, @default_redactors) + ) + end + + @doc "The default metadata allow-list, exposed for inspection or extension." + @spec default_metadata() :: list(atom()) + def default_metadata, do: @default_metadata + + @doc "The default redactor list, exposed for inspection or extension." + @spec default_redactors() :: list({module(), term()}) + def default_redactors, do: @default_redactors +end diff --git a/lib/zexbox/logging/json_handler.ex b/lib/zexbox/logging/json_handler.ex deleted file mode 100644 index 5a6f3d6..0000000 --- a/lib/zexbox/logging/json_handler.ex +++ /dev/null @@ -1,76 +0,0 @@ -defmodule Zexbox.Logging.JsonHandler do - @moduledoc """ - Replaces the default `:logger` handler's formatter with a JSON formatter - suitable for Logstash / Elasticsearch ingestion. - - Emits one JSON object per line, so multi-line content (Elixir struct - inspections, multi-line SQL, stack traces) collapses into a single log - event at the ingest layer instead of fanning out into N separate - Elasticsearch documents. - - This is the Phoenix / Elixir equivalent of opsbox's `JsonFormatter` for Ruby. - Wraps `LoggerJSON.Formatters.Basic` with sensible defaults for the - Zappi log-ingest pipeline. - - ## Setup - - In your application's `config/runtime.exs`: - - if config_env() == :prod do - Zexbox.Logging.JsonHandler.install!() - end - - After install, every log line is a single JSON object: - - {"time":"...","severity":"info","message":"...","metadata":{...}} - - Logstash's existing `kubernetes.container.name` rules can then JSON-parse - these into structured fields the same way they do for opsbox-formatted - Ruby logs. - - ## Options - - * `:metadata` - which `Logger` metadata keys to include. Defaults to - `:all` (every key set via `Logger.metadata/1` or - `Logger.put_application_level/2`). Pass a list (e.g. - `[:request_id, :trace_id]`) to filter, or `[]` to omit metadata. - - * `:redactors` - a list of `LoggerJSON.Redactor` modules applied to - metadata values before serialisation, e.g. for stripping sensitive - keys. Defaults to `[]`. - - ## Idempotency - - Safe to call multiple times — `install!/1` simply swaps the formatter - on the existing default handler. Subsequent calls overwrite the previous - formatter config. - """ - - @doc """ - Install the JSON formatter on the `:default` `:logger` handler. - - Returns `:ok` on success or `{:error, reason}` if the default handler - hasn't been configured (typically only in unusual test setups). - - ## Examples - - iex> Zexbox.Logging.JsonHandler.install!() - :ok - - iex> Zexbox.Logging.JsonHandler.install!(metadata: [:request_id, :trace_id]) - :ok - """ - @spec install!(keyword()) :: :ok | {:error, term()} - def install!(opts \\ []) do - formatter_config = %{ - metadata: Keyword.get(opts, :metadata, :all), - redactors: Keyword.get(opts, :redactors, []) - } - - :logger.update_handler_config( - :default, - :formatter, - {LoggerJSON.Formatters.Basic, formatter_config} - ) - end -end diff --git a/mix.exs b/mix.exs index 9c85153..ee3a439 100644 --- a/mix.exs +++ b/mix.exs @@ -37,7 +37,6 @@ defmodule Zexbox.MixProject do {:doctor, "~> 0.22.0", only: [:dev, :test]}, {:ex_doc, "~> 0.35.1", only: :dev, runtime: false}, {:instream, "~> 2.2"}, - {:jason, "~> 1.4"}, {:ldclient, "~> 3.8.0", hex: :launchdarkly_server_sdk}, {:logger_json, "~> 7.0"}, {:mix_audit, "~> 2.0", only: [:dev, :test], runtime: false}, diff --git a/test/zexbox/logging/json_formatter_test.exs b/test/zexbox/logging/json_formatter_test.exs new file mode 100644 index 0000000..2de2950 --- /dev/null +++ b/test/zexbox/logging/json_formatter_test.exs @@ -0,0 +1,34 @@ +defmodule Zexbox.Logging.JsonFormatterTest do + use ExUnit.Case, async: true + + alias Zexbox.Logging.JsonFormatter + + describe "config/1" do + test "returns LoggerJSON.Formatters.Basic with sensible defaults" do + assert {LoggerJSON.Formatters.Basic, opts} = JsonFormatter.config() + assert :request_id in opts.metadata + assert :user_id in opts.metadata + refute opts.redactors == [] + end + + test "metadata override is honoured" do + assert {_module, %{metadata: [:foo, :bar]}} = + JsonFormatter.config(metadata: [:foo, :bar]) + end + + test "redactors override is honoured" do + assert {_module, %{redactors: []}} = JsonFormatter.config(redactors: []) + end + + test "default lists are exposed for inspection" do + assert :request_id in JsonFormatter.default_metadata() + assert [{LoggerJSON.Redactors.RedactKeys, _keys}] = JsonFormatter.default_redactors() + end + end + + describe "Zexbox.Logging.json_formatter_config/1 delegate" do + test "returns the same config" do + assert Zexbox.Logging.json_formatter_config() == JsonFormatter.config() + end + end +end diff --git a/test/zexbox/logging/json_handler_test.exs b/test/zexbox/logging/json_handler_test.exs deleted file mode 100644 index a420328..0000000 --- a/test/zexbox/logging/json_handler_test.exs +++ /dev/null @@ -1,73 +0,0 @@ -defmodule Zexbox.Logging.JsonHandlerTest do - use ExUnit.Case, async: false - - alias Zexbox.Logging.JsonHandler - - setup do - {:ok, original_config} = :logger.get_handler_config(:default) - on_exit(fn -> :logger.update_handler_config(:default, original_config) end) - :ok - end - - describe "install!/1" do - test "swaps the default handler's formatter to LoggerJSON.Formatters.Basic" do - assert :ok = JsonHandler.install!() - - {:ok, %{formatter: {formatter_module, _config}}} = - :logger.get_handler_config(:default) - - assert formatter_module == LoggerJSON.Formatters.Basic - end - - test "passes metadata: :all by default" do - assert :ok = JsonHandler.install!() - - {:ok, %{formatter: {_module, config}}} = - :logger.get_handler_config(:default) - - assert config.metadata == :all - end - - test "passes through a metadata allow-list" do - assert :ok = JsonHandler.install!(metadata: [:request_id, :trace_id]) - - {:ok, %{formatter: {_module, config}}} = - :logger.get_handler_config(:default) - - assert config.metadata == [:request_id, :trace_id] - end - - test "passes through redactors" do - redactors = [{LoggerJSON.Redactors.RedactKeys, ["password"]}] - assert :ok = JsonHandler.install!(redactors: redactors) - - {:ok, %{formatter: {_module, config}}} = - :logger.get_handler_config(:default) - - assert config.redactors == redactors - end - - test "is idempotent across repeated calls" do - assert :ok = JsonHandler.install!() - assert :ok = JsonHandler.install!() - assert :ok = JsonHandler.install!(metadata: [:request_id]) - - {:ok, %{formatter: {formatter_module, config}}} = - :logger.get_handler_config(:default) - - assert formatter_module == LoggerJSON.Formatters.Basic - assert config.metadata == [:request_id] - end - end - - describe "Zexbox.Logging.install_json_handler!/1 delegate" do - test "the parent module exposes the same function" do - assert :ok = Zexbox.Logging.install_json_handler!() - - {:ok, %{formatter: {formatter_module, _config}}} = - :logger.get_handler_config(:default) - - assert formatter_module == LoggerJSON.Formatters.Basic - end - end -end