diff --git a/.gitattributes b/.gitattributes index dc105a8..0ab9226 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,22 +1,17 @@ -# Line endings: normalise to LF everywhere (checkout as-is on all platforms). * text=auto eol=lf -# Sample/corpora data ships as PLAIN git blobs - deliberately NOT LFS, so a -# public clone needs no git-lfs and burns no LFS bandwidth quota. The data is -# small (a few MB) and write-once. +# Plain git blobs, not LFS, so a public clone needs no git-lfs. data/samples/*.log text eol=lf linguist-generated tests/testdata/*.log.gz binary tests/testdata/manifest.json linguist-generated -# Binary types (no diff/eol mangling) *.png binary *.gz binary *.zip binary -# Merge drivers: generated by the release tagger - keep ours on merge. +# Written by the release tagger: keep ours on merge. CHANGELOG.md merge=ours VERSION merge=ours -# Language statistics docs/* linguist-documentation data/* linguist-documentation diff --git a/.gitleaks.toml b/.gitleaks.toml index 75158ab..4c3b100 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -7,8 +7,7 @@ [extend] useDefault = true -# Sample log files and test fixtures contain token-like strings that are not -# real secrets - logreducer's whole job is chewing through log lines. +# Sample logs and test fixtures hold token-like strings that are not secrets. [[allowlists]] description = "logreducer false positives" paths = [ @@ -16,8 +15,7 @@ paths = [ '''tests/.*''', ] -# docs/JIRA_OAUTH_SETUP.md shows a curl example with the literal placeholder -# ACCESS_TOKEN in an Authorization header - documentation, not a credential. +# A placeholder ACCESS_TOKEN in a curl example, not a credential. [[allowlists]] description = "doc placeholder tokens in curl examples" paths = [ diff --git a/.hyperi-ci.yaml b/.hyperi-ci.yaml index 71d596e..7efcaa2 100644 --- a/.hyperi-ci.yaml +++ b/.hyperi-ci.yaml @@ -1,4 +1,3 @@ -# HyperI CI configuration -- logreducer # Only values that differ from the hyperi-ci defaults. classification: oss diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e409a39..1d06963 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,45 +2,29 @@ Thanks for your interest in improving logreducer. -## Development setup +## Setup and checks -logreducer uses [uv](https://docs.astral.sh/uv/) for dependency management. +logreducer uses [uv](https://docs.astral.sh/uv/) for dependency management. There is no Makefile, and the local gate is the one CI runs: ```bash git clone https://github.com/hyperi-io/logreducer.git cd logreducer uv sync --all-extras # core + ALL extras (enhanced/sql/clickhouse/kafka) + the dev group +hyperi-ci check # lint, types, security, tests ``` -## Running the checks - -There is no Makefile - the canonical local gate is the same one CI runs: - -```bash -hyperi-ci check # full local gate: lint, types, security, tests -``` - -Or run the tools individually: - -```bash -uv run ruff format # format -uv run ruff check --fix # lint (autofix) -uv run mypy src/logreducer # strict type check -uv run ty check src/logreducer # Astral type check -uv run pytest -q # tests -uv build # build wheel + sdist -``` +To run its tools one at a time, see the command list under Context below. ## Tests -- `tests/unit/` - fast, server-free (real SQLite, real corpora, no mocks of +- `tests/unit/` - fast, server-free (SQLite and the corpora, no mocks of internal code). `tests/integration/` - marked `integration`, run against - real services. `tests/testdata/` holds gzipped, PII-cleansed slices of REAL + running services. `tests/testdata/` holds gzipped, PII-cleansed slices of public log datasets (see its README for provenance and the rebuild tool). -- ClickHouse and Kafka come from scalo's shared test services (`scalo.deployment.test_support.pytest_plugin`): one container each per run, shared by every xdist worker, or the always-present server the `HYPERI_TEST_*` settings in `.env.example` name. Each test gets its own database or topics, dropped afterwards. Without Docker those tests skip locally and fail in CI. +- ClickHouse and Kafka come from scalo's shared test services (`scalo.deployment.test_support.pytest_plugin`): one container each per run, shared by every xdist worker, or the always-present server the `HYPERI_TEST_*` settings in `.env.example` name. Each test gets a separate database or set of topics, dropped afterwards. Without Docker those tests skip locally and fail in CI. - PostgreSQL and MySQL are throwaway testcontainers, stopped at the end of the session unless `LOGREDUCER_KEEP_CONTAINERS=1`, and skip without Docker. -## Commit messages +## Commits, pull requests and code style We use [Conventional Commits](https://www.conventionalcommits.org/). Keep the subject to 50 characters or fewer. Common types: `fix`, `feat`, `docs`, @@ -48,20 +32,18 @@ subject to 50 characters or fewer. Common types: `fix`, `feat`, `docs`, Versioning and releases are automated - do NOT hand-edit the version. semantic-release derives the next version from the commit history, then tags, -builds and publishes. Just land well-formed commits on a feature branch and -open a pull request to `main`. - -## Pull requests +builds and publishes. Land well-formed commits on a feature branch and +open a pull request to `main`: 1. Branch from `main` (e.g. `fix/memory-leak`, `feat/clickhouse-source`). -2. Make your change with tests; keep `hyperi-ci check` green. +2. Make your change with tests, and keep `hyperi-ci check` passing. 3. Open a pull request to `main`. -## Code style +Code style: - Python at the `requires-python` floor, [ruff](https://docs.astral.sh/ruff/) formatted at the `line-length` in `pyproject.toml`. -- Fully typed - `mypy` runs strict; annotate new code. -- Australian English in prose; ASCII-only in comments, commit messages and docs. +- Fully typed. `mypy` runs strict, so annotate every function you add. +- Australian English in prose, and ASCII only in comments, commit messages and docs. ## Context @@ -69,8 +51,8 @@ Depth: [docs/architecture.md](docs/architecture.md). ### What this is -A reduction engine over an abstraction, plus a CLI on top of it. The core carries -zero IO dependencies and reduces any re-iterable stream of `str` lines, so it +A reduction engine over an abstraction, plus a CLI on top of it. The core has +no IO dependencies and reduces any re-iterable stream of `str` lines, so it never owns a connection or a loading path. It is not a log shipper, not a parser, and not an aggregator front end -- it picks a representative subset of the lines it is given and drops the rest. @@ -88,8 +70,8 @@ it is given and drops the rest. | `src/logreducer/sampling.py`, `target.py` | dialect-aware SQL sampling, and the `reduce_to_target` batch loop | | `src/logreducer/config.py` | `BigDialConfig`, the level presets, the `from_env` cascade | | `src/logreducer/cli.py` | the `logreducer` entry point, dispatching on the `--dsn` scheme | -| `tests/unit/`, `tests/integration/` | server-free, and `integration`-marked against real services | -| `tests/testdata/` | gzipped, PII-cleansed slices of real public log datasets, with a manifest and a rebuild tool | +| `tests/unit/`, `tests/integration/` | server-free, and `integration`-marked against running services | +| `tests/testdata/` | gzipped, PII-cleansed slices of public log datasets, with a manifest and a rebuild tool | ### Commands that prove a change @@ -100,23 +82,23 @@ hyperi-ci check # the canonical local gate -- there is no Makefile Individually: `uv run ruff format`, `uv run ruff check --fix`, `uv run mypy src/logreducer`, `uv run ty check src/logreducer`, -`uv run pytest -q`, `uv build`. Three ways green lies: +`uv run pytest -q`, `uv build`. Three ways a passing run misleads: | It looks like | It is actually | |---|---| -| `pytest -q` covered the adapters | Nothing is deselected by default -- `addopts` is only `-ra -q --strict-markers --strict-config`, so `tests/integration/` is collected and then SKIPS when there is neither a configured endpoint nor Docker. CI sets no env vars, so CI always takes the docker path and a local run may not have | +| `pytest -q` covered the adapters | Nothing is deselected by default -- `addopts` is `-ra -q --strict-markers --strict-config`, so `tests/integration/` is collected and then SKIPS when there is neither a configured endpoint nor Docker. CI sets no env vars, so CI always takes the docker path and a local run may not have | | the tree tells you the version | It does not. The version in `pyproject.toml` and `VERSION` lags the newest tag and PyPI. semantic-release resolves the version at release time | -| coverage is at the stated floor | `min_coverage` is the gate, not the measured figure. `vulture` is set to `warn`, not fail, because a library's public API reads as unused to dead-code analysis | +| coverage is at the stated floor | `min_coverage` is the gate, not the coverage a run reports. `vulture` is set to `warn`, not fail, because a library's public API reads as unused to dead-code analysis | ### What tends to bite | Don't | Do | Why | |---|---|---| -| Hand `reduce()` a bare generator | Hand it something whose `__iter__` returns a FRESH iterator each call | `reduce()` counts lines in one pass then re-reads to process, and hybrid reads again. A one-shot iterator is drained by the count and leaves nothing behind, so it now raises `TypeError` rather than returning an empty result (`core.py`) | -| Assume a reused `LogReducer` starts clean | Nothing -- `_reset_components()` rebuilds them per run | The dedup seen-set, both Drain3 miners and the fuzzy LSH accumulate per line. Carried into the next run they drop every line as already-seen and return empty (`core.py`) | -| Share one deduplicator across hybrid's two passes | Rebuild it before the anomaly pass | The pattern pass consumes the seen-set, so the anomaly pass sees everything as seen and no anomalies survive. Hybrid silently degrades to pattern-only (`core.py`) | +| Hand `reduce()` a bare generator | Hand it an object with an `__iter__` that returns a FRESH iterator each call | `reduce()` counts lines in one pass then re-reads to process, and hybrid reads again. A one-shot iterator is drained by the count and leaves nothing behind, so it raises `TypeError` rather than returning an empty result (`core.py`) | +| Assume a reused `LogReducer` starts clean | Nothing -- `_reset_components()` rebuilds them per run | The dedup seen-set, both Drain3 miners and the fuzzy LSH accumulate per line. Carried into the next run they drop every line as seen and return empty (`core.py`) | +| Share one deduplicator across hybrid's two passes | Rebuild it before the anomaly pass | The pattern pass consumes the seen-set, so the anomaly pass sees everything as seen and no anomalies survive. Hybrid silently degrades to pattern mode (`core.py`) | | Assume an unknown config keyword is ignored | Read the `ValueError` and fix the name | A silently dropped override means a user's tuning quietly does nothing (`core.py`) | -| Assume every template is the final Drain3 one | Allow for a cluster the `max_clusters` cap evicted | `extract_patterns` re-reads each template from the live cluster once the feed ends, so a varying cluster carries its `<*>` slots and the priority boost counts them. A cluster LRU-evicted mid-feed is gone by then and keeps the template it had when first seen (`patterns.py`) | +| Assume every template is the final Drain3 one | Allow for a cluster the `max_clusters` cap evicted | `extract_patterns` re-reads each template from the live cluster once the feed ends, so a varying cluster keeps its `<*>` slots and the priority boost counts them. A cluster LRU-evicted mid-feed is gone by then and keeps the template it had when first seen (`patterns.py`) | | Hand-edit the version | Land a conventional commit and let semantic-release do it | semantic-release writes `VERSION`, the changelog and the tag from the commit type, so a hand edit is overwritten at the next release | ### Where this sits @@ -125,13 +107,9 @@ One declared edge in `dfe-infra/suite.yaml`. | Repo | Direction | Kind | Mechanism | |---|---|---|---| -| dfe-engine | outbound | `python-dep` | dfe-engine declares `logreducer[enhanced]` by range in its `pyproject.toml` and imports it lazily, inside the `smart` and `anomaly` sampler modes: `LogReducer` and `logreducer.clickhouse` in `sampling/service.py`, `logreducer.kafka` in `sampling/kafka_reader.py`. A new release is checked against that range, then taken with `uv lock --upgrade-package logreducer` and the engine's tests | +| dfe-engine | outbound | `python-dep` | dfe-engine declares `logreducer[enhanced]` with a version floor in its `pyproject.toml` and imports it lazily, inside the `smart` and `anomaly` sampler modes: `LogReducer` and `logreducer.clickhouse` in `src/dfe_engine/sampling/service.py`, `logreducer.kafka` in `src/dfe_engine/sampling/kafka_reader.py`. Each release is checked against that floor, then taken with `uv lock --upgrade-package logreducer` and the engine's tests | `dfe-stack suite --consumer logreducer` returns an empty edge list: no repo in that graph is something this library depends on. The runtime dependencies (psutil, numpy, scikit-learn, typer) are packages, declared in `pyproject.toml`, which is where they belong. - -The reason dfe-engine gives for not declaring the dependency -- pin it "once it -hits public PyPI" -- has been met since 3.4.0 was published on 2026-07-03, so -expect that edge to become an ordinary pinned dependency. diff --git a/README.md b/README.md index 24ca3b8..362d2d2 100644 --- a/README.md +++ b/README.md @@ -11,20 +11,22 @@ Reduces gigabytes of logs to a small, representative sample, as a CLI or a Pytho - By default you get a couple of examples of each recurring pattern, and the repeats are dropped. In a DFE deployment dfe-engine's sampler uses it for the smart and anomaly samples of a source's data. - A `logreducer` command that reduces a file, a SQL or ClickHouse query, or a Kafka topic, straight from the shell. - Four modes: pattern (Drain3 template mining), anomaly (Isolation Forest), temporal (one candidate per template per time window, ranked, up to `max_patterns`) and hybrid. -- Pattern mode streams through a bounded dedup cache, fuzzy index and template store, so memory does not grow with the line count or the number of distinct lines. -- Lines naming ERROR, FATAL or CRITICAL as a whole word always come through, one per distinct shape and ahead of the reduced output, in every mode. -- A core with no IO dependencies: anything re-iterable that yields `str` lines is a source, and anything with `write(lines)` is a sink. SQL, ClickHouse and Kafka adapters are extras, so you install only the one you use. +- Pattern mode streams through a bounded dedup cache, fuzzy index and template store, so memory does not grow with the line count. The one store that grows is the kept severe lines, one per distinct message. +- Lines naming ERROR, FATAL or CRITICAL as a whole word, in any case, always come through, one per distinct message once standalone numbers, IPs, UUIDs, hex IDs and ISO timestamps are masked, ahead of the reduced output, in every mode and however a file is read. A SQL or ClickHouse source samples in the database first, so a line the sample drops is never seen. +- A core with no IO dependencies: anything re-iterable that yields `str` lines is a source, and anything with `write(lines)` is a sink. SQL, ClickHouse and Kafka adapters are extras, so you install just the one you use. +- Seeded SQL sampling, PostgreSQL `TABLESAMPLE` and ClickHouse `SAMPLE` for large tables, and `reduce_to_target` to collect about N lines from them. +- Hooks for a host application's config cascade and logging. -## Installation +## Quick start -As a tool, isolated from your other Python projects (needs the Python `requires-python` in `pyproject.toml` names): +Install it as a tool, isolated from your other Python projects (needs the Python version `requires-python` names in `pyproject.toml`): ```bash uv tool install logreducer # or: pipx install logreducer uv tool install "logreducer[clickhouse]" # with an adapter extra ``` -As a library: +Or add it to a project as a library: ```bash uv add logreducer # or: pip install logreducer @@ -34,7 +36,7 @@ uv add "logreducer[clickhouse]" # ClickHouseSource (clickhouse-connect) uv add "logreducer[kafka]" # KafkaSource and KafkaSink (confluent-kafka) ``` -## Quick start +Then reduce a file: ```bash logreducer app.log # reduced lines to stdout @@ -59,12 +61,21 @@ reduced = reducer.reduce(lines) `reduce()` returns the reduced lines in memory. Pass `return_metadata=True` for a dict of `{"lines", "stats", "config"}`. -A database or a Kafka topic is a source too. The CLI dispatches on the `--dsn` scheme, and a library caller constructs the source directly: +A database query or a Kafka topic is a source in the same way. The CLI dispatches on the `--dsn` scheme, and a library caller constructs the source directly: ```bash -logreducer --dsn postgresql://user@host/db --query "SELECT message FROM logs" # logreducer[sql] + a driver -logreducer --dsn clickhouse://user@host:8123/db --query "SELECT message FROM logs" -logreducer --dsn kafka://broker:9092 --topic app-logs --group logreducer +logreducer --dsn postgresql://user@host/db --query "SELECT message FROM logs" # logreducer[sql] + a driver +logreducer --dsn clickhouse://user@host:8123/db --query "SELECT message FROM logs" # logreducer[clickhouse] +logreducer --dsn kafka://broker:9092 --topic app-logs --group logreducer # logreducer[kafka] +``` + +```python +from logreducer import LogReducer +from logreducer.clickhouse import ClickHouseSource + +reducer = LogReducer(level="enhanced", mode="hybrid") +with ClickHouseSource("clickhouse://user@host:8123/db", "SELECT message FROM logs") as source: + reduced = reducer.reduce(source) ``` The query selects the log line as its first column. Sources are re-iterable (the engine makes several passes), so a database source re-runs its query per pass and a Kafka source re-reads from the earliest offset without committing. @@ -73,31 +84,67 @@ The query selects the log line as its first column. Sources are re-iterable (the Pattern mode streams: lines flow through dedup and template mining as one generator pipeline, so the unique-line set is never materialised. Memory is set by `max_clusters`, `max_patterns` and `dedup_cache_size`, not by the input. +This diagram shows the two paths a line can take through the reducer. + +```mermaid +flowchart LR + accTitle: L4 Detail -- Reduction pipeline + accDescr: Source lines pass through exact dedup. The pattern path runs fuzzy dedup and Drain3 template mining. The anomaly path caps the deduplicated lines with a reservoir and scores them with an Isolation Forest. Both paths end in the reduced lines. + + source["Source"] + dedup["Exact dedup"] + fuzzy["Fuzzy dedup"] + cap["Reservoir cap"] + drain["Drain3 mining"] + forest["Isolation Forest"] + output["Reduced lines"] + + source --> dedup + dedup --> fuzzy + dedup -.-> cap + fuzzy --> drain + cap -.-> forest + drain --> output + forest -.-> output + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A +``` + +*Figure 1. L4 Detail -- Reduction pipeline. Dotted arrows are the anomaly path, which anomaly and hybrid mode take.* Up: [L3 Components -- logreducer](docs/architecture.md#a-reduction-engine-behind-two-protocols). + +Pattern mode takes the solid path, anomaly mode the dotted one, and hybrid both. Fuzzy dedup runs at the `enhanced` and `maximum` levels. The reservoir cap is `anomaly_max_rows`, off by default. Severe lines are kept out of both paths and lead the output, one per distinct message. + +The anomaly path and temporal mode do not stream. Isolation Forest is batch ML, so cap its input with `anomaly_max_rows` on very high-cardinality sources. Temporal mode holds every line in memory to group them by time window. + +This diagram shows which mode to pick. + ```mermaid flowchart LR - subgraph sources ["Source (re-iterable str lines)"] - direction TB - F[file] - L["list[str]"] - Q[SQL / ClickHouse query] - K[Kafka topic] - end - sources --> D["exact dedup
(bounded hash cache)"] - D --> FZ["fuzzy dedup
(MinHash LSH, enhanced+)"] - FZ --> DR["Drain3 template mining
(LRU-bounded clusters)"] - DR --> P["rank patterns
(CRITICAL > ERROR > WARN)"] - P --> O["top patterns
x examples"] - D -. "anomaly / hybrid" .-> A["TF-IDF + Isolation Forest
(batch, capped by anomaly_max_rows)"] - A --> O - O --> out["stdout / file + meta.json / Sink"] + accTitle: L4 Detail -- Choosing a mode + accDescr: Pick pattern for recurring structure, anomaly for rare events, temporal for bursts in time windows and hybrid when unsure. + + want{"What matters
most?"} + pattern(["pattern"]) + anomaly(["anomaly"]) + temporal(["temporal"]) + hybrid(["hybrid"]) + + want -->|structure| pattern + want -->|rare events| anomaly + want -->|bursts| temporal + want -->|unsure| hybrid + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef decision fill:#FCF1D6,stroke:#9A6700,stroke-width:1px,color:#3D2A00 + class want decision ``` -The anomaly branch is the one part that cannot stream (Isolation Forest is batch ML). Cap its input with `anomaly_max_rows` on very high-cardinality sources. +*Figure 2. L4 Detail -- Choosing a mode.* Up: [L3 Components -- logreducer](docs/architecture.md#a-reduction-engine-behind-two-protocols). | Mode | Description | Best for | |------|-------------|----------| | `pattern` | Drain3 template mining | Structured and application logs (fastest) | -| `anomaly` | Isolation Forest outlier detection, needs 10 or more unique lines | Security and error logs | +| `anomaly` | Isolation Forest outlier detection. Below 10 unique lines (after the `anomaly_max_rows` cap) nothing is scored and they all come through, up to `max_patterns` | Security and error logs | | `temporal` | One candidate per time window and template, most important first, up to `max_patterns` (`temporal_window_minutes`) | Time-series and monitoring logs | | `hybrid` | Pattern plus anomaly | Maximum coverage | @@ -107,15 +154,15 @@ The anomaly branch is the one part that cannot stream (Isolation Forest is batch | `enhanced` | Moderate | 1 GB | plus fuzzy dedup | | `maximum` | Thorough | 2 GB | plus a wider pattern budget | +Fuzzy dedup needs the `enhanced` extra. Without it the `enhanced` and `maximum` levels skip fuzzy dedup and log a warning. + ## Documentation -- [docs/library-guide.md](docs/library-guide.md): sampling large tables, collecting a target number of lines, bounding memory, your own sources and sinks, embedding in a host app, every config field, logging. +- [docs/library-guide.md](docs/library-guide.md): sampling large tables, collecting a target number of lines, bounding memory, custom sources and sinks, embedding in a host app, configuration, logging. - [docs/architecture.md](docs/architecture.md): how the pieces fit. - [docs/pypi.md](docs/pypi.md): the PyPI page. -## Help and contributing - -How to work in this repo, the toolchain, the test layout (including the real-log corpora under `tests/testdata/`) and how the integration tests find their services are in [CONTRIBUTING.md](CONTRIBUTING.md). A question that is not a bug goes in a new issue. +How to work in this repo, the toolchain, the test layout (including the log corpora under `tests/testdata/`) and how the integration tests find their services are in [CONTRIBUTING.md](CONTRIBUTING.md). A question that is not a bug goes in an issue. Open bugs: [this repo](https://github.com/hyperi-io/logreducer/issues?q=is%3Aopen+is%3Aissue+type%3ABug+-label%3Abacklog+-label%3Anext-minor) | [whole DFE suite](https://github.com/search?type=issues&q=is%3Aopen+is%3Aissue+type%3ABug+-label%3Abacklog+-label%3Anext-minor+repo%3Ahyperi-io%2Fdfe-infra+repo%3Ahyperi-io%2Fdfe-docker+repo%3Ahyperi-io%2Fdfe-deploy+repo%3Ahyperi-io%2Fdfe-engine+repo%3Ahyperi-io%2Fdfe-loader+repo%3Ahyperi-io%2Fdfe-receiver+repo%3Ahyperi-io%2Fdfe-fetcher+repo%3Ahyperi-io%2Fdfe-archiver+repo%3Ahyperi-io%2Fdfe-transform-vrl+repo%3Ahyperi-io%2Fdfe-transform-vector+repo%3Ahyperi-io%2Fdfe-transform-elastic+repo%3Ahyperi-io%2Fdfe-ui+repo%3Ahyperi-io%2Fdfe-hyperdx+repo%3Ahyperi-io%2Fscalo-rs+repo%3Ahyperi-io%2Fscalo-py+repo%3Ahyperi-io%2Flogreducer+repo%3Ahyperi-io%2Fdfe-schemas+repo%3Ahyperi-io%2Fvector-vrl+repo%3Ahyperi-io%2Fculvert+repo%3Ahyperi-io%2Fdfe-clickhouse-rs+repo%3Ahyperi-io%2Ffactbook-rs) @@ -123,4 +170,4 @@ Open bugs: [this repo](https://github.com/hyperi-io/logreducer/issues?q=is%3Aope [Apache-2.0](LICENSE). Third-party attributions are recorded in [NOTICE](NOTICE). Copyright 2026 HYPERI PTY LIMITED. -Credits: template mining (`src/logreducer/template_mining.py`) is derived from [Drain3](https://github.com/logpai/Drain3), Copyright (c) 2020-2022 International Business Machines and the Drain3 project contributors, MIT licence. Drain3 builds on the Drain parser in LogPAI's [logparser](https://github.com/logpai/logparser). The module header carries Drain3's licence text in full. +Credits: template mining (`src/logreducer/template_mining.py`) is derived from [Drain3](https://github.com/logpai/Drain3), Copyright (c) 2020-2022 International Business Machines and the Drain3 project contributors, MIT licence. Drain3 builds on the Drain parser in LogPAI's [logparser](https://github.com/logpai/logparser). The module header has Drain3's licence text in full. diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 5876971..0000000 --- a/docs/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# logreducer documentation - -Everything past the README. - -- [library-guide.md](library-guide.md) - sampling, collecting a target, memory, sources and sinks, embedding, configuration, logging -- [architecture.md](architecture.md) - how the pieces fit -- [pypi.md](pypi.md) - the PyPI page diff --git a/docs/architecture.md b/docs/architecture.md index 4a9461b..c2901f7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,15 +1,13 @@ # logreducer architecture -Why the engine is shaped the way it is. The README covers what it does and how to +Why the engine is built the way it is. The README covers what it does and how to drive it -- this page is the reasoning and the invariants a caller cannot infer from the API. -## The problem - You have gigabytes of logs and a question. Reading all of it is not an option, and a random sample is worse than it sounds: random throws away the rare lines, which are usually the interesting ones, and keeps thousands of copies of the line -that repeats. What a reader actually wants is one of each SHAPE, plus the +that repeats. What a reader actually wants is one of each kind of line, plus the outliers. Two constraints make that harder than picking a sample. @@ -20,15 +18,15 @@ Two constraints make that harder than picking a sample. - Some reduction modes need more than one look at the data, and the data might be a database query or a Kafka topic rather than a file. -## The shape: a reduction engine behind two protocols +## A reduction engine behind two protocols -The core does no IO at all. Input is a `Source` -- anything whose `__iter__` +The core does no IO at all. Input is a `Source` -- anything with an `__iter__` that yields `str` and can be iterated more than once. Output is a `Sink` -- -`write(lines) -> int`. Those two structural protocols are the whole integration +`write(lines) -> int`. Those two structural protocols are the whole IO surface. -They are protocols rather than base classes on purpose. A `list[str]` is already -a `Source`, and an application's own object becomes one without importing +They are protocols rather than base classes on purpose. A `list[str]` is a `Source` +as it stands, and an application's object becomes one without importing anything from here. The point of that boundary is ownership. The engine never manages a connection, @@ -37,15 +35,30 @@ credential inside it. The adapters that DO own connections -- SQLAlchemy, clickhouse-connect, confluent-kafka -- sit behind optional extras, and the core never imports them. +This diagram shows the parts of logreducer and what passes between them. + ```mermaid flowchart LR - A["any re-iterable
of str lines"] -->|Source| C["core
zero IO deps"] - C -->|"list[str]"| R["caller"] - C -->|"Sink.write()"| S["anywhere"] - AD["sql / clickhouse / kafka
adapters (extras)"] -.->|"also just Sources"| A + accTitle: L3 Components -- logreducer + accDescr: Sources hand lines to the LogReducer core on every pass. The core returns the reduced lines to its caller and can also write them to a sink. + + sources[["Sources"]] + core[["LogReducer"]] + caller["Caller"] + sinks[["Sinks"]] + + sources -->|lines| core + core -->|reduced lines| caller + core -->|reduced lines| sinks + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef ext fill:#EEF0F2,stroke:#6B7280,stroke-width:1px,stroke-dasharray:4 3,color:#1F2937 + class caller ext ``` -## Re-iterability is the load that protocol carries +*Figure 1. L3 Components -- logreducer. The dashed box is the application or CLI that called `reduce()`.* Up: top of logreducer. Down: [L4 Detail -- Reduction pipeline](../README.md#how-it-works), [L4 Detail -- Choosing a mode](../README.md#how-it-works), [L4 Detail -- Sources and sinks](library-guide.md#sources-and-sinks), [L4 Detail -- Collecting a target](library-guide.md#collecting-a-target-number-of-lines). + +## Every source is re-iterable `reduce()` counts lines in one pass to compute the reduction ratio, then re-reads the source to process it. Hybrid mode reads it a third time. So `__iter__` has to @@ -56,32 +69,32 @@ silent in the worst way -- the counting pass drains it and the processing pass finds nothing, so the result is an empty list rather than an error. That is why `reduce()` tests `iter(source) is source` and raises `TypeError` up front. -The same requirement shapes every adapter. `SQLSource` re-runs its query per +Every adapter follows from the same requirement. `SQLSource` re-runs its query per pass. `KafkaSource` re-reads from the earliest offset and never commits. -`FileSource` re-opens the file and streams it again. A seeded sample is required -for the same reason: an unseeded one would hand pass two a different set of rows -than pass one, and the reducer would be comparing two populations. +`FileSource` re-opens the file and streams it again. A SQL sample needs a seed +for the same reason: an unseeded one hands pass two a different set of rows +than pass one, and the reducer then compares two populations. ## What streams, and what cannot -Pattern mode streams end to end. Exact dedup, optional fuzzy dedup and Drain3 +Pattern mode streams. Exact dedup, optional fuzzy dedup and Drain3 mining run as one generator pipeline, so the unique-line set is never collected -into a list. Peak memory is the bounded dedup cache, the fuzzy index (the same `dedup_cache_size` window), the Drain3 template store and the results of evicted templates, and none of them grows with the number of unique lines the source has. +into a list. Peak memory is the bounded dedup cache, the fuzzy index (the same `dedup_cache_size` window), the Drain3 template store and the results of evicted templates, and none of them grows with the number of unique lines the source has. The one exception is the severe-line store, which keeps one line per distinct ERROR, FATAL or CRITICAL message in every mode. -An evicted template's result is kept only while it can still make the `max_patterns` cut. Its count stops changing once the dedup cache has dropped the lines that fed it, so from then on it competes for one of `max_patterns` places, and a result that loses one can never win it back. That keeps the output identical to ranking every template ever seen. +An evicted template's result is kept as long as it can still make the `max_patterns` cut. Its count stops changing once the dedup cache has dropped the lines that fed it, so from then on it competes for one of `max_patterns` places, and a result that loses one can never win it back. That keeps the output identical to ranking every template ever seen. -The fuzzy index measured about 1.8 KB per entry, so the preset windows hold about 180 MB at enhanced (100K lines) and 360 MB at maximum (200K), inside their 1 GB and 2 GB budgets. +In a 2026-11-01 test run the fuzzy index used about 1.8 KB per entry, so the preset windows hold about 180 MB at enhanced (100K lines) and 360 MB at maximum (200K), inside their 1 GB and 2 GB budgets. Drain3 compares each line with every template stored at the leaf it reaches, so time grows with the square of the template count when lines share no tokens. `max_clusters` (10,000 by default) bounds the template store and with it the -comparisons per line. Measured on 16,000 lines that share nothing, 86 s with no -bound and 4.6 s with `max_clusters=500`, so 10,000 caps growth on long inputs +comparisons per line. A 2026-10-31 test run on 16,000 lines that share nothing ran about 19 times +slower with no bound than with `max_clusters=500`, so 10,000 caps growth on long inputs and a lower value is the speed setting for high-cardinality sources. Temporal mode returns one example per time window and template, ranked by severity and count, and does not stream. It reads every line and a parsed entry per line -into memory before grouping, so it ignores `max_memory_gb`. Measured on an 80 MB -file, 832 MB peak against 156 MB for pattern mode. +into memory before grouping, so it ignores `max_memory_gb`. A 2026-10-31 test run on an 80 MB +file peaked at 832 MB, against 156 MB for pattern mode. Anomaly mode cannot stream. Isolation Forest over a TF-IDF matrix is batch ML and needs its rows at once. That is the one place a hard memory ceiling needs an @@ -89,9 +102,6 @@ explicit sample, which is exactly what `anomaly_max_rows` is -- a reservoir cap with a fixed seed, trading anomaly recall for a bounded matrix, and off unless asked for. -So picking a mode is also picking a memory shape. Worth knowing before reaching -for the tuning knobs. - ## Collecting a target instead of reducing everything `reduce_to_target` is a different loop for a different question: about N @@ -106,26 +116,30 @@ mean very different things about the answer you are holding. ## Determinism -Every sample in the engine is seeded. The anomaly reservoir and the -normal-context sample both use a fixed seed, and SQL sampling takes an explicit -`sample_seed`. +Every sample the engine draws itself is seeded. The anomaly reservoir, the +normal-context sample and a sampled `FileSource` use a fixed seed. A SQL sample +is reproducible when you pass `sample_seed`: an unseeded one is accepted and may +differ between passes. ClickHouse `SAMPLE` is deterministic with no seed. +`reduce_to_target` is the exception: +each round wants fresh rows, so a database draws its batch unseeded, and the +client-side reservoir for any other source is seeded when you pass `seed`, and unseeded otherwise. That is not a security property, and there is no security requirement in picking log lines. It is a correctness property the multi-pass design depends on: pass two has to see what pass one saw. -It is also why SQLite raises `SamplingNotSupported` for a seeded sample instead +The same property is why SQLite raises `SamplingNotSupported` for a seeded sample instead of quietly giving an unseeded one. SQLite has no seedable RNG, so it cannot keep the promise, and pretending otherwise would produce a result that looks reproducible and is not. ## Embedding in a host application -Three seams, none of which needs the library to know anything about the host. +Three hooks, none of which needs the library to know anything about the host. -| Seam | How | +| Hook | How | |---|---| -| Config | Build a `BigDialConfig` from the host's own cascade and inject it, or call `BigDialConfig.from_env(prefix, fallback)` where the prefixed name beats the bare one. Keyword arguments still win on top | +| Config | Build a `BigDialConfig` from the host's cascade and inject it, or call `BigDialConfig.from_env(prefix, fallback)` where the prefixed name beats the bare one. Keyword arguments still win on top | | Logging | Off by default. `own_sinks=False` registers nothing, so records flow through the host's handlers formatted by the host's standard | | IO | The `Source` and `Sink` protocols above | @@ -133,16 +147,14 @@ Three seams, none of which needs the library to know anything about the host. 1. `__iter__` returns a fresh iterator, checked at the entry point rather than documented and hoped for. 2. The core imports no IO library. An adapter's dependency is an extra, and the core never imports the adapter modules. -3. Analysis state is per run. `_reset_components()` rebuilds the dedup seen-set, both Drain3 miners and the fuzzy LSH, so one `LogReducer` is reusable across calls and hybrid's two passes cannot poison each other. Without it every line reads as already-seen and the result is empty. +3. Analysis state is per run. `_reset_components()` rebuilds the dedup seen-set, both Drain3 miners and the fuzzy LSH, so one `LogReducer` is reusable across calls and hybrid's two passes cannot poison each other. Without it every line reads as seen and the result is empty. 4. An unknown config keyword raises. A silently dropped override is tuning that quietly does nothing, which is worse than a failure. 5. The row-to-line convention is shared. The database adapters take the FIRST column, skip NULLs and skip blank lines, matching `FileSource`, so the same data reduces identically whichever source carried it. -6. Every sample is seeded. See above. +6. Samples the engine draws are seeded, and a SQL sample is reproducible when you pass `sample_seed`. See above. 7. Config enums serialise as their `.value`, so metadata survives `json.dumps` and no `OutputFormat.LINE`-style repr leaks into output. 8. A memory ceiling above 70% of available RAM is clamped with a warning rather than accepted, whether it comes from the config or a `LogReducer` argument. -### Where the numbers are approximate - -`input_lines` and the reduction ratio come from the counting pass over the +Three figures are approximate. `input_lines` and the reduction ratio come from the counting pass over the source. A size-sampled `FileSource` yields its SAMPLED line count, so on a very large file both figures describe the sample rather than the file. Say which when quoting them. diff --git a/docs/library-guide.md b/docs/library-guide.md index 6d44a83..f9ba67f 100644 --- a/docs/library-guide.md +++ b/docs/library-guide.md @@ -4,16 +4,16 @@ The README covers the install, the CLI and the first `LogReducer` call. This pag ## Sampling large sources -For a table too large to scan in full, sample a fraction of rows. SQL sampling is deterministic when you pass a seed (so the reducer's multi-pass modes see a stable input). ClickHouse uses its native `SAMPLE` clause. +For a large table, sample a fraction of rows. SQL sampling is deterministic when you pass a seed (so the reducer's multi-pass modes see a stable input). ClickHouse uses its native `SAMPLE` clause. ```python from logreducer.sql import SQLSource -# Wrap any query in a seeded random predicate (PostgreSQL/MySQL) +# Wrap any query in a predicate on a seeded hash of each row (PostgreSQL/MySQL) source = SQLSource("postgresql://user@host/db", "SELECT msg FROM logs", sample=0.01, sample_seed=42) # Or sample a TABLE with the engine's native sampler - PostgreSQL -# TABLESAMPLE SYSTEM is page-level and genuinely sub-linear: +# TABLESAMPLE SYSTEM is page-level, so it reads only the pages it keeps: source = SQLSource.from_table("postgresql://user@host/db", "logs", "msg", sample=0.01, sample_seed=42) ``` @@ -21,30 +21,42 @@ source = SQLSource.from_table("postgresql://user@host/db", "logs", "msg", sample logreducer --dsn postgresql://user@host/db --query "SELECT msg FROM logs" --sample 0.01 --sample-seed 42 ``` -Notes: SQLite has no seedable RNG, so a seeded sample raises `SamplingNotSupported` (unseeded, best-effort sampling still works). ClickHouse `SAMPLE` needs the table to declare a `SAMPLE BY` key. For an arbitrary complex query, put the sampling clause in your own SQL instead. +Notes: SQLite has no seedable RNG, so a seeded sample raises `SamplingNotSupported` (unseeded, best-effort sampling still works). ClickHouse `SAMPLE` needs the table to declare a `SAMPLE BY` key, and `sample=` takes a plain `SELECT ... FROM table` (`ClickHouseSource.from_table` adds a `where`). For any other query, put the sampling clause in your SQL. ## Collecting a target number of lines When you want *about N* representative lines rather than "reduce everything", `reduce_to_target` pulls fresh random batches and reduces each, accumulating distinct representatives until a stop condition fires. Peak memory is bounded to roughly one batch plus the accumulator. +This diagram shows one round of the loop. + ```mermaid -flowchart TD - PULL["pull fresh random batch
(sample_batch in-engine,
else client-side reservoir)"] --> E{empty?} - E -->|yes| SE([stop: exhausted]) - E -->|no| SIZE["resize next batch from
observed avg row bytes"] - SIZE --> RED["reduce the batch"] - RED --> ACC["add new distinct
representatives"] - ACC --> TGT{"target
reached?"} - TGT -->|yes| ST([stop: target]) - TGT -->|no| PLAT{"no new lines for
plateau_rounds?"} - PLAT -->|yes| SP([stop: plateau]) - PLAT -->|no| MEM{"memory
watchdog"} - MEM -->|"soft: shrink batch"| FET - MEM -->|"hard limit"| SM([stop: memory]) - FET{"max_fetches
hit?"} -->|yes| SF([stop: max_fetches]) - FET -->|no| PULL +flowchart LR + accTitle: L4 Detail -- Collecting a target + accDescr: Each round pulls a fresh random batch. An empty batch stops the loop as exhausted. Otherwise the batch is reduced and its new lines kept, then the stop conditions are checked, and the loop pulls again until one fires. + + pull["Pull random
batch"] + empty{"Empty?"} + reduce["Reduce, keep
new lines"] + exhausted(["Stop: exhausted"]) + check{"Stop
condition?"} + stop(["Stop with
its reason"]) + + pull --> empty + empty -->|no| reduce + empty -.->|yes| exhausted + reduce --> check + check -->|yes| stop + check -.->|no| pull + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef decision fill:#FCF1D6,stroke:#9A6700,stroke-width:1px,color:#3D2A00 + class empty,check decision ``` +*Figure 1. L4 Detail -- Collecting a target. The dotted arrow back to the start is the next round.* Up: [L3 Components -- logreducer](architecture.md#a-reduction-engine-behind-two-protocols). + +A batch comes from the source's `sample_batch` where it has one (SQL, ClickHouse), else from a client-side reservoir over the whole source. Unless `batch_rows` fixes it, the average row size of a batch sets the next batch size. The stop conditions run in this order: `target` reached, `plateau` after `plateau_rounds` batches that add no unseen line, then the memory watchdog. Still past the soft limit after a garbage collection, it halves the next batch, to no fewer than 1,000 rows, and past the hard limit it stops with `memory`. The loop ends with `max_fetches` once that many batches have been pulled. + ```python from logreducer import LogReducer, reduce_to_target from logreducer.sql import SQLSource @@ -65,9 +77,9 @@ logreducer --dsn postgresql://user@host/db --query "SELECT msg FROM logs" \ Pattern mode streams, and its memory is fixed by the knobs below, not by lines or distinct templates. Temporal mode holds the whole source in memory, and anomaly mode holds its unique lines. The knobs: -- `max_memory_gb` - the overall cap (also sizes file read strategy and the watchdog). Presets: 0.5 / 1 / 2 GB by level. Values above 70% of available RAM are clamped with a warning. +- `max_memory_gb` - the overall cap, which sets the file read strategy and the watchdog. Presets: 0.5 / 1 / 2 GB by level. Values above 70% of available RAM are clamped with a warning. - `dedup_cache_size` - the lines exact dedup remembers, and at enhanced and maximum the lines the fuzzy index holds. Both drop the oldest first, so a repeat or near-repeat older than the window comes through again. -- `max_clusters` - LRU-bound the Drain3 template store (10,000 by default, `None` for unbounded, which also keeps every evicted template's result). A lower value bounds the template store and cuts the work per line on high-cardinality sources. +- `max_clusters` - LRU-bound the Drain3 template store (10,000 by default, `None` for unbounded, so no template is evicted). A lower value bounds the template store and cuts the work per line on high-cardinality sources. - `anomaly_max_rows` - reservoir-cap the rows fed to anomaly detection (off by default, and it trades anomaly recall for a bounded matrix). ```python @@ -76,32 +88,48 @@ reducer = LogReducer(level="enhanced", mode="hybrid", max_clusters=50_000, anoma ## Sources and sinks -The engine is built around two tiny structural protocols - that is the whole integration surface: +The engine is built around two structural protocols, `Source` and `Sink`, and they are its whole IO surface. + +This diagram shows the built-in sources and sinks on either side of the reducer. ```mermaid flowchart LR - subgraph inputs ["Sources"] + accTitle: L4 Detail -- Sources and sinks + accDescr: FileSource, SQLSource, ClickHouseSource, KafkaSource or your own re-iterable hand lines to the LogReducer core on every pass. The core writes the reduced lines to FileSink, KafkaSink or your own sink. + + subgraph sources["Sources"] direction TB - FS["FileSource"] - SS["SQLSource [sql]"] - CS["ClickHouseSource [clickhouse]"] - KS["KafkaSource [kafka]"] - MY["your own re-iterable"] - end - subgraph core ["LogReducer core"] - C["zero IO dependencies
Source in -> list[str] out"] + filesrc["FileSource"] + sqlsrc["SQLSource"] + chsrc["ClickHouseSource"] + kafkasrc["KafkaSource"] + ownsrc["Your re-iterable"] + filesrc ~~~ sqlsrc ~~~ chsrc ~~~ kafkasrc ~~~ ownsrc end - subgraph outputs ["Sinks"] + + core["LogReducer"] + + subgraph sinks["Sinks"] direction TB - FK["FileSink"] - KK["KafkaSink [kafka]"] - MS["your own write()"] + filesink["FileSink"] + kafkasink["KafkaSink"] + ownsink["Your write()"] + filesink ~~~ kafkasink ~~~ ownsink end - inputs -->|"__iter__() -> str
fresh iterator per pass"| core - core -->|"write(lines) -> int"| outputs + + sources -->|lines, every pass| core + core -->|reduced lines| sinks + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef core fill:#DCEBF7,stroke:#0072B2,stroke-width:2px,color:#0B2A3F + classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 + class core core + class sources,sinks group ``` -An application can hand the reducer its own IO - anything that yields `str` and can be iterated more than once is a `Source`: +*Figure 2. L4 Detail -- Sources and sinks. SQLSource, ClickHouseSource and the Kafka pair each need their extra.* Up: [L3 Components -- logreducer](architecture.md#a-reduction-engine-behind-two-protocols). + +An application can hand the reducer custom IO - anything that yields `str` and can be iterated more than once is a `Source`: ```python class MySource: @@ -119,9 +147,9 @@ from logreducer import LogReducer, FileSink reducer.reduce(source, sink=FileSink("reduced.jsonl", output_format="jsonl")) ``` -## Embedding in a host application +## Configuration, logging and embedding -A host app that owns its own config cascade and logging standard drives logreducer with three hooks - no logreducer-side knowledge of the host required: +A host app with its config cascade and logging standard drives logreducer with three hooks - no logreducer-side knowledge of the host required: ```python import logreducer @@ -139,9 +167,7 @@ cfg = logreducer.BigDialConfig.from_env("MYAPP", "LOGREDUCER") # MYAPP_MAX_MEM logreducer.setup_logging(enable=True, own_sinks=False) ``` -## Configuration - -Override any config field as a keyword argument (unknown names raise - typos never silently no-op): +Any config field can be overridden as a keyword argument, and unknown names raise, so a typo never silently does nothing: ```python reducer = LogReducer( @@ -154,12 +180,12 @@ reducer = LogReducer( anomaly_contamination=0.1, # expected anomaly fraction temporal_window_minutes=60, # grouping window for temporal mode max_patterns=2000, # cap on extracted patterns + examples_per_pattern=3, # example lines kept per pattern + typed_masking=False, # opt-in: typed template slots (, , ...) instead of <*> min_pattern_occurrences=2, # patterns seen fewer times are dropped, except CRITICAL/FATAL/ERROR ones ) ``` -## Logging - Logging is off by default. Enable it and pick a format via env vars: ```bash diff --git a/docs/pypi.md b/docs/pypi.md index 8b0724d..93c719d 100644 --- a/docs/pypi.md +++ b/docs/pypi.md @@ -10,13 +10,12 @@ It ships as a CLI and as a Python library with an IO-agnostic core. - Pattern mode streams through a bounded dedup cache, fuzzy index and template store, so memory does not grow with the line count or the number of distinct lines. - A memory budget that decides how a file is read: whole, in chunks, or reservoir-sampled once it passes five times the budget. - Four modes: pattern (Drain3 template mining), anomaly (Isolation Forest), temporal (one example per template per time window) and hybrid. -- Lines naming ERROR, FATAL or CRITICAL as a whole word, in any case, always come through, one per distinct shape and ahead of the reduced output, in every mode and however a file is read. A SQL or ClickHouse source samples in the database first, so only the lines it returns are seen. +- Lines naming ERROR, FATAL or CRITICAL as a whole word, in any case, always come through, one per distinct message once standalone numbers, IPs, UUIDs, hex IDs and ISO timestamps are masked, ahead of the reduced output, in every mode and however a file is read. A SQL or ClickHouse source samples in the database first, so a line the sample drops is never seen. - A core with no IO dependencies: anything re-iterable that yields `str` lines is a source, and anything with `write(lines)` is a sink. -- Seeded SQL sampling and ClickHouse `SAMPLE` for tables too big to scan in full. -- `reduce_to_target` to collect about N representative lines from a large source, holding roughly one batch in memory. -- Injection points for a host application's own config cascade and logging. +- Seeded SQL sampling, PostgreSQL `TABLESAMPLE` and ClickHouse `SAMPLE` for large tables, and `reduce_to_target` to collect about N representative lines from a large source, holding roughly one batch in memory. +- Injection points for a host application's config cascade and logging. -## Example +Install it and reduce a list of lines: ```console uv add logreducer @@ -34,7 +33,7 @@ for line in reduced: print(line) ``` -```text +```console 540 -> 4 WARN slow query table=events ms=300 WARN slow query table=events ms=301 @@ -44,15 +43,15 @@ INFO GET /health 200 req=1 From the shell it is `logreducer app.log -o reduced.log`. `level` is `standard`, `enhanced` or `maximum`, trading speed for wider coverage on a 0.5, 1 or 2 GB budget. -## Extras +Extras add adapters and fuzzy dedup: | Extra | Adds | |---|---| -| `enhanced` | Fuzzy dedup with MinHash LSH, whose index holds the last `dedup_cache_size` near-unique lines, and faster hashing | -| `sql` | `SQLSource` over SQLAlchemy, with your own DBAPI driver | +| `enhanced` | Fuzzy dedup with MinHash LSH (the index holds the last `dedup_cache_size` near-unique lines) and faster hashing | +| `sql` | `SQLSource` over SQLAlchemy, with a DBAPI driver you install | | `clickhouse` | `ClickHouseSource` over clickhouse-connect | | `kafka` | `KafkaSource` and `KafkaSink` over confluent-kafka | Template mining is derived from [Drain3](https://github.com/logpai/Drain3), MIT licence, with its licence text in the module header. -Docs and source are at . The Python floor is `requires-python` in `pyproject.toml`. Licensed Apache-2.0: . +Docs and source are at . PyPI lists the Python floor under Meta. Licensed Apache-2.0: . diff --git a/pyproject.toml b/pyproject.toml index 7aab4f5..a8ae534 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,30 +31,26 @@ classifiers = [ ] keywords = ["log", "logging", "reduction", "analysis", "pattern-extraction", "memory-efficient", "streaming", "anomaly-detection"] dependencies = [ - "psutil>=7.2.2", # memory monitoring (MemoryMonitor) + "psutil>=7.2.2", "numpy>=2.5.3", - "scikit-learn>=1.9.1", # imports as sklearn - "typer>=0.27.3", # CLI + "scikit-learn>=1.9.1", + "typer>=0.27.3", ] +# What each extra adds: docs/pypi.md "Extras". [project.optional-dependencies] enhanced = [ - "xxhash>=4.0.1", # Fast xxh64 dedup hashing (memory.BoundedDeduplicator) - "datasketch>=2.0.0", # MinHash/LSH similarity in fuzzy dedup + "xxhash>=4.0.1", + "datasketch>=2.0.0", ] - -# IO adapters - thin OPTIONAL conveniences over the Source/Sink abstraction. -# The core engine has zero IO deps; these only matter to the CLI or an app that -# wants a ready-made source/sink. Each needs its own running service, so they -# are covered by docker-gated integration tests. sql = [ - "sqlalchemy>=2.1.3", # cross-DB server-side-cursor streaming (SQLSource) + "sqlalchemy>=2.1.3", ] clickhouse = [ - "clickhouse-connect>=1.9.0", # native block streaming (ClickHouseSource) + "clickhouse-connect>=1.9.0", ] kafka = [ - "confluent-kafka>=2.15.1", # librdkafka client (KafkaSource/Sink) + "confluent-kafka>=2.15.1", ] [project.urls] Homepage = "https://github.com/hyperi-io/logreducer" @@ -90,15 +86,14 @@ ignore = [ "SIM105", "SIM108", "SIM102", "F821", "F601", "TRY300", "C901", "UP035", "W191", - # Intentional security relaxations in this codebase - "S104", # bind-all-interfaces (containers) - "S110", # try-except-pass (intentional error handling) - "S112", # try-except-continue (intentional skip) - "S311", # non-crypto random (jitter/sampling) - "S603", # subprocess without shell=True (helper scripts) - "S607", # partial executable path (helper scripts) - "S608", # hardcoded SQL (query builders, not user input) - "T201", # print (CLI output) + "S104", + "S110", + "S112", + "S311", # sampling, never anything secret + "S603", # the template-mining test runs a subprocess + "S607", + "S608", # the samplers build SQL around the caller's query + "T201", # the CLI prints its output "RUF012", "RUF059", "RUF001", "RUF002", "RUF005", "RUF006", "N806", "N818", "PIE810", @@ -136,7 +131,7 @@ indent-style = "space" skip-magic-trailing-comma = false line-ending = "auto" -# Strict. ignore_missing_imports covers third-party libs without stubs. +# ignore_missing_imports: datasketch and other third-party libs ship no stubs. [tool.mypy] python_version = "3.14" ignore_missing_imports = true @@ -153,7 +148,6 @@ warn_no_return = true warn_unreachable = true strict_equality = true -# pytest configuration [tool.pytest.ini_options] minversion = "8.0.0" addopts = "-ra -q --strict-markers --strict-config" @@ -167,7 +161,6 @@ markers = [ ] shared_service_repo = "logreducer" -# Coverage configuration [tool.coverage.run] source = ["logreducer"] omit = [ @@ -191,7 +184,6 @@ exclude_lines = [ "@(abc\\.)?abstractmethod", ] -# Bandit security linting configuration [tool.bandit] exclude_dirs = [ "tests", @@ -205,14 +197,11 @@ exclude_dirs = [ ] skips = ["B101", "B603"] -# uv dependency resolution. Bleeding-edge policy: force latest majors even -# when transitive deps pin older lower bounds; revisit each /deps pass. [tool.uv] index-strategy = "unsafe-best-match" # Supply-chain cooldown. A CVE fix bypasses it with a per-package entry. exclude-newer = "7 days" -# scalo 2.32.4 carries the shared test services the suite needs, released 2026-11-01. A -# per-package entry replaces the global span rather than widening it, so remove this one on 2026-11-08. +# Lets in the scalo release the test suite needs before the cooldown would. exclude-newer-package = { scalo = "2026-11-01T06:00:00Z" } override-dependencies = [] @@ -221,7 +210,6 @@ name = "pypi" url = "https://pypi.org/simple" default = true -# ty: Astral type checker, runs alongside mypy. [tool.ty.environment] python-version = "3.14" @@ -234,38 +222,34 @@ exclude = ["examples/", "docs/", "scripts/", "tests/testdata/build_quiet_golden. [tool.vulture] min_confidence = 80 -# vermin: verify no syntax/features newer than the minimum Python. [tool.vermin] targets = ["3.14"] no_tips = true [tool.bandit.assert_used] -# Allow asserts in test files skips = ["*test*.py", "*tests*.py"] -# Local dev and test tooling, not in the published wheel. [dependency-groups] dev = [ - "pytest>=9.0.0", # Test framework (9.0 changed warning/path semantics) - "pytest-cov>=7.0.0", # Coverage reporting - "pytest-xdist>=3.8.0", # Parallel test workers (hyperi-ci adds -n) - "ruff>=0.16.10", # Lint + format (replaces black/flake8/isort) - "ty>=0.0.34", # Astral type checker (runs alongside mypy) - "mypy>=2.0", # Static type checking (strict; 2.0 flipped defaults) - "bandit[toml]>=1.7.0", # Security linting - "pip-audit>=2.6.0", # Dependency vulnerability scanning - "vulture>=2.16", # Dead-code detection - "vermin>=1.6.0", # Python version-compat detection - # Adapter + integration test deps (TEST ONLY - never in the published - # wheel; these mirror the optional extras so the adapters can be exercised). - "xxhash>=4.0.2", # enhanced extra: xxh64 dedup hashing - "datasketch>=2.0.0", # enhanced extra: MinHash/LSH fuzzy dedup - "sqlalchemy>=2.1.3", # SQLSource against SQLite (real, no server) - "clickhouse-connect>=1.9.0", # ClickHouseSource (local cluster or docker) - "confluent-kafka>=2.15.1", # KafkaSource/KafkaSink - "scalo>=2.32.4", # the run's shared ClickHouse and Kafka (scalo.deployment.test_support) - "testcontainers[postgres,mysql]>=4.15.0", # PostgreSQL and MySQL, which scalo does not provide - "psycopg[binary]>=3.2", # PostgreSQL driver (real TABLESAMPLE/setseed sampling tests) - "pymysql>=1.1.0", # MySQL driver (real RAND(seed) sampling tests) - "python-dotenv>=1.0.0", # load the local .env (CLICKHOUSE_*) in tests + "pytest>=9.0.0", + "pytest-cov>=7.0.0", + "pytest-xdist>=3.8.0", # hyperi-ci adds -n + "ruff>=0.16.10", + "ty>=0.0.34", + "mypy>=2.0", + "bandit[toml]>=1.7.0", + "pip-audit>=2.6.0", + "vulture>=2.16", + "vermin>=1.6.0", + # The extras again, so the adapter tests run. Test services: tests/TESTING.md. + "xxhash>=4.0.2", + "datasketch>=2.0.0", + "sqlalchemy>=2.1.3", + "clickhouse-connect>=1.9.0", + "confluent-kafka>=2.15.1", + "scalo>=2.32.4", + "testcontainers[postgres,mysql]>=4.15.0", + "psycopg[binary]>=3.2", + "pymysql>=1.1.0", + "python-dotenv>=1.0.0", # tests/conftest.py loads a local .env ]