diff --git a/.github/workflows/benchmark-rdf.yml b/.github/workflows/benchmark-rdf.yml
new file mode 100644
index 0000000..8f81d82
--- /dev/null
+++ b/.github/workflows/benchmark-rdf.yml
@@ -0,0 +1,59 @@
+name: Benchmark RDF
+
+# Triggers like the main workflow. The tests in tests/perf already run a 1k smoke there; this
+# job runs the sizes up to 10k. 100k is left out of CI (run it locally with make benchmark-rdf, or
+# pass it in the sizes input). Each run is capped at 120 s, and a size predicted to exceed that
+# is skipped, so slow scenarios stay cheap.
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ types: [opened, synchronize, reopened, ready_for_review]
+ workflow_dispatch:
+ inputs:
+ sizes:
+ description: "Triple counts, space separated. Empty uses 1000 2000 4000 10000"
+ default: ""
+
+jobs:
+ benchmark-rdf:
+ if: github.event.pull_request.draft == false
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ steps:
+ - name: Check out
+ uses: actions/checkout@v4
+
+ - name: Set up the environment
+ uses: ./.github/actions/setup-python-env
+ with:
+ python-version: "3.12"
+
+ - name: Run the RDF benchmark
+ env:
+ SIZES: ${{ inputs.sizes || '1000 2000 4000 10000' }}
+ run: |
+ # Without pipefail a failing benchmark would be hidden behind tee.
+ set -o pipefail
+ # shellcheck disable=SC2086 - SIZES is a space separated list on purpose
+ PYTHONPATH=tests uv run python -m perf --sizes $SIZES --out benchmark_rdf_results.json \
+ | tee benchmark_rdf_report.md
+
+ - name: Publish the report in the job summary
+ if: always()
+ run: |
+ if [ -f benchmark_rdf_report.md ]; then
+ cat benchmark_rdf_report.md >> "$GITHUB_STEP_SUMMARY"
+ fi
+
+ - name: Upload the results
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: benchmark-rdf-results
+ path: |
+ benchmark_rdf_results.json
+ benchmark_rdf_report.md
+ if-no-files-found: warn
+ retention-days: 90
diff --git a/.gitignore b/.gitignore
index 9b99e31..571ae17 100644
--- a/.gitignore
+++ b/.gitignore
@@ -213,6 +213,7 @@ __marimo__/
# Benchmarks (pytest-benchmark)
.benchmarks/
benchmark_results.json
+benchmark_rdf_results.json
baseline_benchmark.json
current_benchmark.json
benchmark_comparison.txt
diff --git a/Makefile b/Makefile
index 6c77c7a..a861d88 100644
--- a/Makefile
+++ b/Makefile
@@ -40,6 +40,11 @@ benchmark: ## Run performance benchmarks with pytest-benchmark
--benchmark-autosave \
--benchmark-json=benchmark_results.json
+.PHONY: benchmark-rdf
+benchmark-rdf: ## Benchmark RDF export and import at 1k, 10k and 100k triples (own harness, not pytest-benchmark)
+ @echo "🚀 Benchmarking RDF export and import"
+ @PYTHONPATH=tests uv run python -m perf --out benchmark_rdf_results.json
+
.PHONY: build
build: clean-build ## Build wheel file
@echo "🚀 Creating wheel file"
diff --git a/docs/contributing.md b/docs/contributing.md
index eaaa755..3923e7c 100644
--- a/docs/contributing.md
+++ b/docs/contributing.md
@@ -76,6 +76,19 @@ The tests marked `postgres` need a live server and skip unless `OOLD_TEST_POSTGR
OOLD_TEST_POSTGRES_URL=postgresql://user:password@localhost:5432/postgres make test
```
+### Benchmarking RDF performance
+
+A separate harness measures RDF export, store and import at 1k, 10k and 100k triples. It runs each scenario in a fresh subprocess and records time, CPU, peak memory, GC pauses and the scaling exponent. It is not `pytest-benchmark`, which `make benchmark` uses.
+
+```bash
+make benchmark-rdf # 1k to 100k, each run capped at 120 s, writes benchmark_rdf_results.json
+PYTHONPATH=tests uv run python -m perf --sizes 1000 2000 --scenarios export store
+PYTHONPATH=tests uv run python -m perf.worker import 1000 --profile 30 # cProfile, top 30 by cumulative time
+uv run pytest -m scaling --no-cov # scaling guards, skipped by default
+```
+
+The code lives in `tests/perf/`. CI runs the 1k sizes with the normal tests. The `Benchmark RDF` workflow runs sizes up to 10k, triggers like the main workflow and can also be started from the Actions tab (100k is left out of CI, use `make benchmark-rdf` locally), which publishes the report in the job summary and uploads `benchmark_rdf_results.json` as an artifact.
+
### Docs
Doc sources live in `docs/`. The site is configured in `zensical.toml`.
diff --git a/pyproject.toml b/pyproject.toml
index 5526f82..0d9df25 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -123,6 +123,7 @@ dev = [
"pytest-cov>=7.0.0",
"pytest-benchmark",
"pytest-playwright",
+ "psutil>=5.9",
"jsondiff",
"panel",
"pre-commit>=4.5.1",
@@ -227,6 +228,8 @@ markers = [
"parity: compares this port against the reference harness",
# Needs a live PostgreSQL; skips unless OOLD_TEST_POSTGRES_URL is set.
"postgres: runs against a live PostgreSQL server",
+ # Slow scaling checks over the RDF scenarios; they skip unless selected with `-m scaling`.
+ "scaling: slow checks of how the RDF scenarios scale",
]
[tool.deptry]
diff --git a/src/oold/backend/sparql.py b/src/oold/backend/sparql.py
index 51ca90c..9f363ac 100644
--- a/src/oold/backend/sparql.py
+++ b/src/oold/backend/sparql.py
@@ -2,7 +2,7 @@
from typing import Any
from pydantic import ConfigDict
-from rdflib import Graph
+from rdflib import Graph, URIRef
from SPARQLWrapper import JSON, JSONLD, SPARQLWrapper
from oold.backend.auth import UserPwdCredential, get_credential
@@ -30,6 +30,19 @@
"""
+_EX_NAMESPACE = "https://example.com/"
+
+
+def _subject(iri: str) -> URIRef:
+ """Full IRIs pass through, ``ex:`` expands to the one namespace this module assumes."""
+ if iri.startswith("http"):
+ return URIRef(iri)
+ prefix, _, local = iri.partition(":")
+ if prefix != "ex":
+ raise ValueError(f"Unknown namespace prefix : {prefix}")
+ return URIRef(_EX_NAMESPACE + local)
+
+
class LocalSparqlResolver(Resolver):
model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -55,41 +68,33 @@ def query(self, param: QueryParam) -> ResolveResult:
return self.resolve(ResolveParam(iris=iris, model_cls=model_cls))
def resolve_iris(self, iris: list[str]) -> dict[str, dict]:
- # sparql query to get a node by IRI with all its properties
- # using CONSTRUCT to get the full node
- # format the result as json-ld
+ """Every triple of each subject as JSON-LD, or None for a subject with no triples.
+
+ A direct index lookup rather than a CONSTRUCT with a FILTER: the FILTER
+ is evaluated against every triple of the graph, once per IRI, which made
+ resolving quadratic in the size of the graph.
+ """
jsonld_dicts = {}
for iri in iris:
- iri_filter = f"FILTER (?s = {iri})"
- # check if the iri is a full IRI or a prefix
- if iri.startswith("http"):
- iri_filter = f"FILTER (?s = <{iri}>)"
- # todo: build full iri / prefix mapping from model context
- qres = self.graph.query(_construct_node(iri_filter, "PREFIX ex: "))
- jsonld_dict = json.loads(qres.serialize(format="json-ld"))[0]
- jsonld_dicts[iri] = jsonld_dict
+ triples = list(self.graph.triples((_subject(iri), None, None)))
+ if not triples:
+ jsonld_dicts[iri] = None
+ continue
+ node = Graph(bind_namespaces="none")
+ for triple in triples:
+ node.add(triple)
+ jsonld_dicts[iri] = json.loads(node.serialize(format="json-ld"))[0]
return jsonld_dicts
class LocalSparqlBackend(LocalSparqlResolver, Backend):
def store_jsonld_dicts(self, jsonld_dicts: dict[str, dict]) -> StoreResult:
- # delete all triples with the given iris as subject
- for iri in jsonld_dicts:
- iri_filter = f"{iri}"
- # check if the iri is a full IRI or a prefix
- if iri.startswith("http"):
- iri_filter = f"<{iri}>"
- query = """
- PREFIX ex:
- DELETE WHERE {
- {{{iri_filter}}} ?p ?o .
- }
- """.replace("{{{iri_filter}}}", iri_filter)
- self.graph.update(query)
- # convert jsonld_dict to rdflib triples and add to graph
- g = Graph()
- g.parse(data=json.dumps(jsonld_dicts[iri]), format="json-ld")
- self.graph += g
+ # replace: drop the subject's triples, then parse the new document into the graph itself.
+ # None means delete, like the other stores.
+ for iri, jsonld_dict in jsonld_dicts.items():
+ self.graph.remove((_subject(iri), None, None))
+ if jsonld_dict is not None:
+ self.graph.parse(data=json.dumps(jsonld_dict), format="json-ld")
return StoreResult(success=True)
@@ -212,7 +217,7 @@ def _post_process(self, jsonld_dict: dict) -> dict:
def _construct_node(iri_filter: str, prefixes: str = "") -> str:
- """CONSTRUCT every triple of one subject. Shared by all three resolvers."""
+ """CONSTRUCT every triple of one subject. Used by the remote resolvers."""
return f"""
{prefixes}
CONSTRUCT {{ ?s ?p ?o . }}
diff --git a/tests/perf/__init__.py b/tests/perf/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/perf/__main__.py b/tests/perf/__main__.py
new file mode 100644
index 0000000..57e5822
--- /dev/null
+++ b/tests/perf/__main__.py
@@ -0,0 +1,39 @@
+"""``python -m perf``: run the scenarios over several sizes, print a report, write JSON."""
+
+import argparse
+import sys
+
+from .dataset import BENCHMARK_SIZES
+from .harness import dump
+from .report import format_report
+from .runner import DEFAULT_TIMEOUT_S, SLOW_RUN_S, collect
+from .scenarios import SCENARIOS
+
+
+def main(argv=None) -> None:
+ parser = argparse.ArgumentParser(prog="python -m perf")
+ parser.add_argument("--sizes", type=int, nargs="+", default=list(BENCHMARK_SIZES))
+ parser.add_argument("--scenarios", nargs="+", choices=sorted(SCENARIOS), default=list(SCENARIOS))
+ parser.add_argument("--repeats", type=int, default=3)
+ parser.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT_S, help="seconds per run")
+ parser.add_argument(
+ "--slow", type=float, default=SLOW_RUN_S, help="runs slower than this are not repeated or traced"
+ )
+ parser.add_argument("--out", default="benchmark_rdf_results.json")
+ args = parser.parse_args(argv)
+
+ collected = collect(
+ args.sizes,
+ args.scenarios,
+ repeats=args.repeats,
+ timeout=args.timeout,
+ slow_s=args.slow,
+ progress=lambda message: print(message, file=sys.stderr, flush=True),
+ )
+ dump(collected.results, args.out, [s.as_tuple() for s in collected.skipped])
+ print(format_report(collected))
+ print(f"\nwrote {args.out}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/perf/conftest.py b/tests/perf/conftest.py
new file mode 100644
index 0000000..86b4554
--- /dev/null
+++ b/tests/perf/conftest.py
@@ -0,0 +1,18 @@
+from pathlib import Path
+
+import pytest
+
+HERE = Path(__file__).parent
+RDFLIB_NOISE = "ignore:ConjunctiveGraph is deprecated:DeprecationWarning"
+
+
+def pytest_collection_modifyitems(config, items):
+ """Silence rdflib's deprecation noise here, and skip ``scaling`` tests unless selected with ``-m scaling``."""
+ selected = "scaling" in (config.getoption("-m") or "")
+ skip = pytest.mark.skip(reason="slow, run with `pytest -m scaling`")
+ for item in items:
+ if not item.path.is_relative_to(HERE):
+ continue
+ item.add_marker(pytest.mark.filterwarnings(RDFLIB_NOISE))
+ if item.get_closest_marker("scaling") and not selected:
+ item.add_marker(skip)
diff --git a/tests/perf/dataset.py b/tests/perf/dataset.py
new file mode 100644
index 0000000..24303c3
--- /dev/null
+++ b/tests/perf/dataset.py
@@ -0,0 +1,85 @@
+"""Deterministic datasets for the RDF performance scenarios.
+
+The scale unit is the triple, not the entity: each person contributes one type
+triple, one name triple and one triple per ``knows`` edge.
+"""
+
+from pydantic import ConfigDict, Field
+
+TRIPLE_SIZES = (1_000, 10_000, 100_000)
+# The intermediate sizes give a scaling exponent even for a scenario too slow to reach 10k.
+BENCHMARK_SIZES = (1_000, 2_000, 4_000, 10_000, 100_000)
+PREFIX = "perf"
+NAMESPACE = "https://perf.example.org/"
+BASE_TRIPLES = 2
+
+
+def build_person_model():
+ """Define the Person model. Call it inside the test or run that needs it.
+
+ Defining it registers the class by type IRI, so the conftest fixture that
+ restores the registries must be active.
+ """
+ from oold.model import LinkedBaseModel
+
+ class Entity(LinkedBaseModel):
+ model_config = ConfigDict(
+ json_schema_extra={
+ "@context": {
+ "id": "@id",
+ "type": "@type",
+ "schema": "https://schema.org/",
+ PREFIX: NAMESPACE,
+ "name": "schema:name",
+ },
+ "$id": f"{NAMESPACE}Entity",
+ }
+ )
+ type: str | None = f"{PREFIX}:Entity"
+ name: str
+
+ def get_iri(self):
+ return f"{PREFIX}:{self.name}"
+
+ class Person(Entity):
+ model_config = ConfigDict(
+ json_schema_extra={
+ "@context": [
+ f"{NAMESPACE}Entity",
+ {"knows": {"@id": "schema:knows", "@type": "@id", "@container": "@set"}},
+ ],
+ "$id": f"{NAMESPACE}Person",
+ }
+ )
+ type: str | None = f"{PREFIX}:Person"
+ knows: list["Person"] | None = Field(None, json_schema_extra={"range": f"{PREFIX}:Person"})
+
+ return Person
+
+
+def edges_per_person(triples: int, knows: int = 2) -> list[int]:
+ """Edge count per person so the dataset has exactly ``triples`` triples."""
+ per_person = BASE_TRIPLES + knows
+ people = triples // per_person
+ if people <= knows + 1:
+ raise ValueError(f"{triples} triples is too few for {knows} edges per person")
+ counts = [knows] * people
+ for i in range(triples - people * per_person):
+ counts[i] += 1
+ return counts
+
+
+def make_people(person_cls, triples: int, knows: int = 2) -> list:
+ """Persons ``p0..pN`` where person i knows the next persons, wrapping around.
+
+ Edges are IRI strings, so the people can be built in any order.
+ """
+ counts = edges_per_person(triples, knows)
+ total = len(counts)
+ return [
+ person_cls(
+ name=f"p{i}",
+ knows=[f"{PREFIX}:p{(i + step) % total}" for step in range(1, edges + 1)],
+ )
+ for i, edges in enumerate(counts)
+ ]
diff --git a/tests/perf/harness.py b/tests/perf/harness.py
new file mode 100644
index 0000000..2e6505a
--- /dev/null
+++ b/tests/perf/harness.py
@@ -0,0 +1,278 @@
+"""Resource metrics for one measured run, and the JSON result schema.
+
+Peak RSS is only meaningful in a fresh process, so scenarios run one per
+subprocess. Heap tracing slows a run down, so it is a separate, opt-in pass:
+traced runs are flagged and only contribute their heap peak to the results.
+"""
+
+import gc
+import json
+import math
+import platform
+import statistics
+import sys
+import threading
+import time
+import tracemalloc
+from collections.abc import Callable, Iterable, Iterator, Sequence
+from contextlib import contextmanager
+from dataclasses import asdict, dataclass, field
+
+import psutil
+
+try:
+ import resource
+except ImportError: # Windows
+ resource = None
+
+RSS_SAMPLE_INTERVAL_S = 0.002
+
+
+@dataclass
+class Metrics:
+ scenario: str
+ triples: int
+ wall_s: float
+ cpu_s: float
+ rss_start_bytes: int
+ rss_peak_bytes: int
+ gc_collections: list[int]
+ gc_pause_s: float
+ heap_peak_bytes: int | None = None
+ traced: bool = False
+ latency_s: dict[str, float] = field(default_factory=dict)
+ extra: dict[str, float] = field(default_factory=dict)
+
+ @property
+ def cpu_ratio(self) -> float:
+ return self.cpu_s / self.wall_s if self.wall_s else 0.0
+
+ @property
+ def throughput(self) -> float:
+ """Triples per second."""
+ return self.triples / self.wall_s if self.wall_s else 0.0
+
+ @property
+ def rss_delta_bytes(self) -> int:
+ return self.rss_peak_bytes - self.rss_start_bytes
+
+ @property
+ def bytes_per_triple(self) -> float:
+ return self.rss_delta_bytes / self.triples if self.triples else 0.0
+
+ @classmethod
+ def from_dict(cls, data: dict) -> "Metrics":
+ """Inverse of ``to_dict``, dropping the derived values."""
+ return cls(**{name: data[name] for name in cls.__dataclass_fields__ if name in data})
+
+ def to_dict(self) -> dict:
+ data = asdict(self)
+ data.update(
+ cpu_ratio=self.cpu_ratio,
+ throughput=self.throughput,
+ rss_delta_bytes=self.rss_delta_bytes,
+ bytes_per_triple=self.bytes_per_triple,
+ )
+ return data
+
+
+def os_peak_rss() -> int:
+ """Process-lifetime RSS high-water mark as tracked by the OS, in bytes."""
+ if resource is None:
+ return psutil.Process().memory_info().peak_wset
+ peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+ return peak if sys.platform == "darwin" else peak * 1024
+
+
+class _RssSampler:
+ """Polls RSS on a thread, since the peak is not readable portably after the fact."""
+
+ def __init__(self):
+ self._process = psutil.Process()
+ self._stop = threading.Event()
+ self._thread = threading.Thread(target=self._run, daemon=True)
+ self.start_bytes = self._process.memory_info().rss
+ self._os_peak_before = os_peak_rss()
+ self.peak_bytes = self.start_bytes
+
+ def _sample(self):
+ self.peak_bytes = max(self.peak_bytes, self._process.memory_info().rss)
+
+ def _run(self):
+ while not self._stop.wait(RSS_SAMPLE_INTERVAL_S):
+ self._sample()
+
+ def __enter__(self):
+ self._thread.start()
+ return self
+
+ def __exit__(self, *exc):
+ self._stop.set()
+ self._thread.join()
+ self._sample()
+ # The OS mark is exact but lifetime-wide: it is the peak of this window
+ # only when the window raised it, otherwise the polled value stands.
+ if os_peak_rss() > self._os_peak_before:
+ self.peak_bytes = max(self.peak_bytes, os_peak_rss())
+
+
+class _GcWatch:
+ """Counts collections per generation and the time spent in them."""
+
+ def __init__(self):
+ self.collections = [0, 0, 0]
+ self.pause_s = 0.0
+ self._started = 0.0
+
+ def _callback(self, phase, info):
+ if phase == "start":
+ self._started = time.perf_counter()
+ else:
+ self.pause_s += time.perf_counter() - self._started
+ self.collections[info["generation"]] += 1
+
+ def __enter__(self):
+ gc.callbacks.append(self._callback)
+ return self
+
+ def __exit__(self, *exc):
+ gc.callbacks.remove(self._callback)
+
+
+def scaling_exponent(points: list[tuple[int, float]]) -> float | None:
+ """Log-log least-squares slope of wall time over triples: 1 is linear, 2 is quadratic."""
+ if len({triples for triples, _ in points}) < 2:
+ return None
+ xs = [math.log(triples) for triples, _ in points]
+ ys = [math.log(wall) for _, wall in points]
+ mean_x, mean_y = statistics.fmean(xs), statistics.fmean(ys)
+ return sum((x - mean_x) * (y - mean_y) for x, y in zip(xs, ys, strict=True)) / sum((x - mean_x) ** 2 for x in xs)
+
+
+class CallCount:
+ count = 0
+
+
+@contextmanager
+def count_calls(targets: Sequence[tuple[type, str]]) -> Iterator[CallCount]:
+ """Count calls to ``owner.name`` for each target while the block runs, then restore them."""
+ counter = CallCount()
+ originals = [(owner, name, getattr(owner, name)) for owner, name in targets]
+
+ def wrap(original):
+ def counted(*args, **kwargs):
+ counter.count += 1
+ return original(*args, **kwargs)
+
+ return counted
+
+ for owner, name, original in originals:
+ setattr(owner, name, wrap(original))
+ try:
+ yield counter
+ finally:
+ for owner, name, original in originals:
+ setattr(owner, name, original)
+
+
+def percentiles(samples: Iterable[float]) -> dict[str, float]:
+ """Nearest-rank p50, p95 and p99."""
+ ordered = sorted(samples)
+ if not ordered:
+ return {}
+ return {f"p{p}": ordered[max(math.ceil(p / 100 * len(ordered)) - 1, 0)] for p in (50, 95, 99)}
+
+
+def time_each(fn: Callable, items: Iterable) -> list[float]:
+ """Seconds taken by ``fn`` for each item, to expose per-object tail cost."""
+ samples = []
+ for item in items:
+ started = time.perf_counter()
+ fn(item)
+ samples.append(time.perf_counter() - started)
+ return samples
+
+
+def measure(
+ scenario: str,
+ fn: Callable[[], object],
+ triples: int,
+ *,
+ trace_heap: bool = False,
+ latency_samples: Callable[[], Iterable[float]] | None = None,
+) -> Metrics:
+ """Run ``fn`` once and record time, CPU, RSS, GC and optionally the Python heap peak.
+
+ ``latency_samples`` is called after the run and returns per-object seconds
+ that ``fn`` collected itself.
+ """
+ gc.collect()
+ if trace_heap:
+ tracemalloc.start()
+ try:
+ with _RssSampler() as rss, _GcWatch() as gc_watch:
+ cpu_started = time.process_time()
+ wall_started = time.perf_counter()
+ fn()
+ wall_s = time.perf_counter() - wall_started
+ cpu_s = time.process_time() - cpu_started
+ heap_peak = tracemalloc.get_traced_memory()[1] if trace_heap else None
+ finally:
+ if trace_heap:
+ tracemalloc.stop()
+ return Metrics(
+ scenario=scenario,
+ triples=triples,
+ wall_s=wall_s,
+ cpu_s=cpu_s,
+ rss_start_bytes=rss.start_bytes,
+ rss_peak_bytes=rss.peak_bytes,
+ gc_collections=gc_watch.collections,
+ gc_pause_s=gc_watch.pause_s,
+ heap_peak_bytes=heap_peak,
+ traced=trace_heap,
+ latency_s=percentiles(latency_samples()) if latency_samples else {},
+ )
+
+
+def environment() -> dict[str, str]:
+ return {"python": platform.python_version(), "platform": platform.platform(), "machine": platform.machine()}
+
+
+def to_benchmark_json(results: list[Metrics], skipped: Iterable[tuple[str, int, str]] = ()) -> dict:
+ """The pytest-benchmark shape that ``scripts/compare_benchmarks.py`` reads.
+
+ Repeated untraced runs of one scenario and size are folded into one entry,
+ which gives the stddev. Traced runs are slowed by tracemalloc, so they never
+ feed the stats: only their heap peak is attached, under ``extra_info``.
+ """
+ grouped: dict[str, list[Metrics]] = {}
+ heap_peaks: dict[str, int] = {}
+ for result in results:
+ name = f"{result.scenario}[{result.triples}]"
+ if result.traced:
+ heap_peaks[name] = max(heap_peaks.get(name, 0), result.heap_peak_bytes or 0)
+ else:
+ grouped.setdefault(name, []).append(result)
+ benchmarks = []
+ for name, runs in grouped.items():
+ walls = [run.wall_s for run in runs]
+ benchmarks.append({
+ "name": name,
+ "stats": {
+ "mean": statistics.fmean(walls),
+ "median": statistics.median(walls),
+ "stddev": statistics.stdev(walls) if len(walls) > 1 else 0.0,
+ "min": min(walls),
+ "max": max(walls),
+ "rounds": len(walls),
+ },
+ "extra_info": {"runs": [run.to_dict() for run in runs], "heap_peak_bytes": heap_peaks.get(name)},
+ })
+ skips = [{"name": f"{scenario}[{triples}]", "reason": reason} for scenario, triples, reason in skipped]
+ return {"machine_info": environment(), "benchmarks": benchmarks, "skipped": skips}
+
+
+def dump(results: list[Metrics], path, skipped: Iterable[tuple[str, int, str]] = ()) -> None:
+ with open(path, "w") as handle:
+ json.dump(to_benchmark_json(results, skipped), handle, indent=2)
diff --git a/tests/perf/report.py b/tests/perf/report.py
new file mode 100644
index 0000000..7f0425f
--- /dev/null
+++ b/tests/perf/report.py
@@ -0,0 +1,68 @@
+"""Text report over collected runs: per-size metrics and the scaling exponent."""
+
+import statistics
+
+from .harness import Metrics, scaling_exponent
+from .runner import Collected
+
+MB = 1024 * 1024
+
+
+def verdict(exponent: float | None) -> str:
+ if exponent is None:
+ return "n/a"
+ if exponent < 1.2:
+ return "linear"
+ return "superlinear" if exponent < 1.6 else "quadratic-like"
+
+
+def _mean(runs: list[Metrics], attr: str) -> float:
+ return statistics.fmean(getattr(run, attr) for run in runs)
+
+
+def _extra(runs: list[Metrics], key: str, per_triple: bool = False) -> str:
+ values = [run.extra[key] / (run.triples if per_triple else 1) for run in runs if key in run.extra]
+ return f"{statistics.fmean(values):.0f}" if values else "-"
+
+
+def _untraced(results: list[Metrics], scenario: str) -> dict[int, list[Metrics]]:
+ grouped: dict[int, list[Metrics]] = {}
+ for run in results:
+ if run.scenario == scenario and not run.traced:
+ grouped.setdefault(run.triples, []).append(run)
+ return dict(sorted(grouped.items()))
+
+
+def exponents(collected: Collected) -> dict[str, float | None]:
+ scenarios = dict.fromkeys(run.scenario for run in collected.results)
+ return {
+ scenario: scaling_exponent([
+ (triples, statistics.fmean(r.wall_s for r in runs))
+ for triples, runs in _untraced(collected.results, scenario).items()
+ ])
+ for scenario in scenarios
+ }
+
+
+def format_report(collected: Collected) -> str:
+ heap = {(r.scenario, r.triples): r.heap_peak_bytes for r in collected.results if r.traced}
+ lines = [
+ "| scenario | triples | wall s | triples/s | cpu/wall | rss MB | B/triple | heap MB | p95 ms | gc s | sparql calls | out B/triple |",
+ "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
+ ]
+ for scenario in dict.fromkeys(run.scenario for run in collected.results):
+ for triples, runs in _untraced(collected.results, scenario).items():
+ peak = heap.get((scenario, triples))
+ p95 = [r.latency_s["p95"] for r in runs if "p95" in r.latency_s]
+ lines.append(
+ f"| {scenario} | {triples} | {_mean(runs, 'wall_s'):.3f} | {_mean(runs, 'throughput'):.0f} | {_mean(runs, 'cpu_ratio'):.2f} "
+ f"| {_mean(runs, 'rss_delta_bytes') / MB:.1f} | {_mean(runs, 'bytes_per_triple'):.0f} "
+ f"| {f'{peak / MB:.1f}' if peak else '-'} | {f'{statistics.fmean(p95) * 1000:.2f}' if p95 else '-'} "
+ f"| {_mean(runs, 'gc_pause_s'):.3f} | {_extra(runs, 'sparql_calls')} | {_extra(runs, 'output_bytes', per_triple=True)} |"
+ )
+ lines += ["", "| scenario | exponent | verdict |", "| --- | ---: | --- |"]
+ for scenario, exponent in exponents(collected).items():
+ lines.append(f"| {scenario} | {'-' if exponent is None else f'{exponent:.2f}'} | {verdict(exponent)} |")
+ if collected.skipped:
+ lines += ["", "Skipped:", *(f"- {s.scenario}[{s.triples}]: {s.reason}" for s in collected.skipped)]
+ return "\n".join(lines)
diff --git a/tests/perf/runner.py b/tests/perf/runner.py
new file mode 100644
index 0000000..4836807
--- /dev/null
+++ b/tests/perf/runner.py
@@ -0,0 +1,109 @@
+"""Runs scenarios one per subprocess, so peak RSS and imports never leak between runs."""
+
+import json
+import os
+import statistics
+import subprocess
+import sys
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .harness import Metrics, scaling_exponent
+from .scenarios import SCENARIOS
+
+TESTS_DIR = Path(__file__).parents[1]
+DEFAULT_TIMEOUT_S = 120
+SLOW_RUN_S = 20.0
+
+
+def run_scenario(
+ scenario: str, triples: int, *, trace_heap: bool = False, timeout: float = DEFAULT_TIMEOUT_S
+) -> Metrics:
+ command = [sys.executable, "-m", "perf.worker", scenario, str(triples)]
+ if trace_heap:
+ command.append("--trace-heap")
+ env = {**os.environ, "PYTHONPATH": os.pathsep.join(filter(None, [str(TESTS_DIR), os.environ.get("PYTHONPATH")]))}
+ done = subprocess.run( # noqa: S603 - fixed argv, no shell
+ command, capture_output=True, text=True, env=env, timeout=timeout
+ )
+ if done.returncode != 0:
+ raise RuntimeError(f"{scenario}[{triples}] failed:\n{done.stderr[-2000:]}")
+ return Metrics.from_dict(json.loads(done.stdout.strip().splitlines()[-1]))
+
+
+@dataclass
+class Skipped:
+ scenario: str
+ triples: int
+ reason: str
+
+ def as_tuple(self) -> tuple[str, int, str]:
+ return self.scenario, self.triples, self.reason
+
+
+@dataclass
+class Collected:
+ results: list[Metrics] = field(default_factory=list)
+ skipped: list[Skipped] = field(default_factory=list)
+
+
+def _predict(points: list[tuple[int, float]], triples: int) -> float | None:
+ """Expected wall time at ``triples``, extrapolated from the sizes already run."""
+ if not points:
+ return None
+ last_triples, last_wall = points[-1]
+ exponent = scaling_exponent(points) or 1.0
+ return last_wall * (triples / last_triples) ** max(exponent, 1.0)
+
+
+def collect(
+ sizes,
+ scenarios=None,
+ *,
+ repeats: int = 1,
+ timeout: float = DEFAULT_TIMEOUT_S,
+ slow_s: float = SLOW_RUN_S,
+ progress: Callable[[str], None] | None = None,
+) -> Collected:
+ """Per scenario and size: untraced runs, then one traced run for the heap peak.
+
+ Sizes run in ascending order. A run is skipped, not attempted, when the
+ sizes before it predict a wall time over ``timeout``, or when a smaller
+ size already timed out. Runs slower than ``slow_s`` are neither repeated
+ nor traced.
+ """
+ say = progress or (lambda _: None)
+ out = Collected()
+ for scenario in scenarios or SCENARIOS:
+ points: list[tuple[int, float]] = []
+ timed_out = False
+ for triples in sorted(sizes):
+ label = f"{scenario}[{triples}]"
+ predicted = _predict(points, triples)
+ if timed_out or (predicted is not None and predicted > timeout):
+ reason = (
+ "a smaller size timed out" if timed_out else f"predicted {predicted:.0f}s exceeds {timeout:.0f}s"
+ )
+ out.skipped.append(Skipped(scenario, triples, reason))
+ say(f"{label} skipped: {reason}")
+ continue
+ say(f"{label} running")
+ try:
+ runs = [run_scenario(scenario, triples, timeout=timeout)]
+ if runs[0].wall_s < slow_s:
+ runs += [run_scenario(scenario, triples, timeout=timeout) for _ in range(repeats - 1)]
+ traced = run_scenario(scenario, triples, trace_heap=True, timeout=timeout)
+ else:
+ traced = None
+ out.skipped.append(
+ Skipped(scenario, triples, f"repeats and heap trace skipped, run takes {runs[0].wall_s:.0f}s")
+ )
+ except subprocess.TimeoutExpired:
+ timed_out = True
+ out.skipped.append(Skipped(scenario, triples, f"timed out after {timeout:.0f}s"))
+ say(f"{label} timed out")
+ continue
+ out.results += runs + ([traced] if traced else [])
+ points.append((triples, statistics.fmean(run.wall_s for run in runs)))
+ return out
diff --git a/tests/perf/scenarios.py b/tests/perf/scenarios.py
new file mode 100644
index 0000000..973d91e
--- /dev/null
+++ b/tests/perf/scenarios.py
@@ -0,0 +1,156 @@
+"""RDF export, store and import scenarios, plus rdflib-only floors.
+
+A scenario's ``prepare`` builds everything outside the measured window and
+returns the callable to measure. ``verify`` checks the result afterwards, so a
+fast but wrong run cannot pass.
+"""
+
+import json
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+from rdflib import Graph
+
+from .dataset import NAMESPACE, build_person_model, make_people
+
+
+@dataclass
+class Prepared:
+ run: Callable[[], None]
+ verify: Callable[[], None]
+ samples: list[float] = field(default_factory=list)
+ output_bytes: Callable[[], int] | None = None
+
+
+def _iri(person) -> str:
+ return NAMESPACE + person.name
+
+
+def _export_all(people) -> dict[str, dict]:
+ return {_iri(person): person.to_jsonld() for person in people}
+
+
+def _timed(samples: list[float], fn, *args):
+ started = time.perf_counter()
+ result = fn(*args)
+ samples.append(time.perf_counter() - started)
+ return result
+
+
+def _stored_backend(docs: dict[str, dict]):
+ from oold.backend.sparql import LocalSparqlBackend
+
+ backend = LocalSparqlBackend()
+ backend.store_jsonld_dicts(docs)
+ return backend
+
+
+def _count_triples(docs: dict[str, dict]) -> int:
+ graph = Graph()
+ for doc in docs.values():
+ graph.parse(data=doc, format="json-ld")
+ return len(graph)
+
+
+def export(triples: int) -> Prepared:
+ """Models to expanded JSON-LD, one ``to_jsonld`` per person."""
+ people = make_people(build_person_model(), triples)
+ docs: dict[str, dict] = {}
+ samples: list[float] = []
+
+ def run():
+ for person in people:
+ docs[_iri(person)] = _timed(samples, person.to_jsonld)
+
+ def verify():
+ assert len(docs) == len(people)
+ assert _count_triples(docs) == triples
+
+ return Prepared(run, verify, samples, output_bytes=lambda: len(json.dumps(docs)))
+
+
+def store(triples: int) -> Prepared:
+ """JSON-LD into ``LocalSparqlBackend``, one call per IRI."""
+ from oold.backend.sparql import LocalSparqlBackend
+
+ docs = _export_all(make_people(build_person_model(), triples))
+ backend = LocalSparqlBackend()
+ samples: list[float] = []
+
+ def run():
+ for iri, doc in docs.items():
+ _timed(samples, backend.store_jsonld_dicts, {iri: doc})
+
+ def verify():
+ assert len(backend.graph) == triples
+
+ return Prepared(run, verify, samples)
+
+
+def import_(triples: int) -> Prepared:
+ """Resolve every IRI from the backend back into models, one query per IRI."""
+ from oold.backend.interface import ResolveParam
+
+ person_cls = build_person_model()
+ people = make_people(person_cls, triples)
+ backend = _stored_backend(_export_all(people))
+ nodes: dict = {}
+ samples: list[float] = []
+
+ def run():
+ for person in people:
+ iri = _iri(person)
+ result = _timed(samples, backend.resolve, ResolveParam(iris=[iri], model_cls=person_cls))
+ nodes.update(result.nodes)
+
+ def verify():
+ assert len(nodes) == len(people)
+ for person in people:
+ expected = person.model_dump()
+ node = nodes[_iri(person)].model_dump()
+ assert node["name"] == expected["name"]
+ assert sorted(node["knows"]) == sorted(expected["knows"])
+
+ return Prepared(run, verify, samples)
+
+
+def rdflib_parse(triples: int) -> Prepared:
+ """Floor for store: the same JSON-LD parsed into a plain rdflib graph."""
+ docs = _export_all(make_people(build_person_model(), triples))
+ graph = Graph()
+ samples: list[float] = []
+
+ def run():
+ for doc in docs.values():
+ _timed(samples, lambda doc=doc: graph.parse(data=doc, format="json-ld"))
+
+ def verify():
+ assert len(graph) == triples
+
+ return Prepared(run, verify, samples)
+
+
+def rdflib_serialize(triples: int) -> Prepared:
+ """Floor for import: the whole graph serialized once instead of one query per IRI."""
+ people = make_people(build_person_model(), triples)
+ graph = _stored_backend(_export_all(people)).graph
+ out: list[str] = []
+
+ def run():
+ out.append(graph.serialize(format="json-ld"))
+
+ def verify():
+ assert out
+ assert len(json.loads(out[0])) == len(people)
+
+ return Prepared(run, verify, output_bytes=lambda: len(out[0].encode()))
+
+
+SCENARIOS: dict[str, Callable[[int], Prepared]] = {
+ "export": export,
+ "store": store,
+ "import": import_,
+ "rdflib_parse": rdflib_parse,
+ "rdflib_serialize": rdflib_serialize,
+}
diff --git a/tests/perf/test_dataset.py b/tests/perf/test_dataset.py
new file mode 100644
index 0000000..4904971
--- /dev/null
+++ b/tests/perf/test_dataset.py
@@ -0,0 +1,32 @@
+import pytest
+from rdflib import Graph
+
+from .dataset import BASE_TRIPLES, TRIPLE_SIZES, build_person_model, edges_per_person, make_people
+
+
+@pytest.mark.parametrize("triples", [*TRIPLE_SIZES, 1_001, 1_003])
+def test_edge_plan_hits_the_target(triples):
+ counts = edges_per_person(triples)
+ assert sum(BASE_TRIPLES + edges for edges in counts) == triples
+
+
+def test_edge_plan_rejects_tiny_datasets():
+ with pytest.raises(ValueError):
+ edges_per_person(8)
+
+
+def test_people_are_deterministic_and_loop_free():
+ person_cls = build_person_model()
+ first = [p.model_dump() for p in make_people(person_cls, 1_001)]
+ assert first == [p.model_dump() for p in make_people(person_cls, 1_001)]
+ for person in first:
+ assert f"perf:{person['name']}" not in person["knows"]
+ assert len(set(person["knows"])) == len(person["knows"])
+
+
+@pytest.mark.parametrize("triples", [1_000, 1_001])
+def test_exported_graph_has_exactly_the_target_triples(triples):
+ graph = Graph()
+ for person in make_people(build_person_model(), triples):
+ graph.parse(data=person.to_jsonld(), format="json-ld")
+ assert len(graph) == triples
diff --git a/tests/perf/test_harness.py b/tests/perf/test_harness.py
new file mode 100644
index 0000000..185873b
--- /dev/null
+++ b/tests/perf/test_harness.py
@@ -0,0 +1,131 @@
+import gc
+import importlib.util
+import json
+import time
+import tracemalloc
+from pathlib import Path
+
+import pytest
+
+from .harness import Metrics, count_calls, dump, measure, os_peak_rss, percentiles, time_each, to_benchmark_json
+
+MB = 1024 * 1024
+
+
+def test_measure_records_time_and_cpu():
+ def busy():
+ sum(i * i for i in range(300_000))
+
+ result = measure("busy", busy, triples=1_000)
+ assert result.wall_s > 0
+ assert 0 < result.cpu_s <= result.wall_s * 1.5
+ assert result.throughput == pytest.approx(1_000 / result.wall_s)
+ assert result.heap_peak_bytes is None
+
+
+def test_measure_sees_a_memory_spike():
+ held = []
+
+ def allocate():
+ held.append(b"x" * (60 * MB))
+ time.sleep(0.05)
+
+ result = measure("alloc", allocate, triples=1_000)
+ assert result.rss_delta_bytes >= 40 * MB
+ assert result.bytes_per_triple == result.rss_delta_bytes / 1_000
+
+
+def test_measure_counts_gc_collections():
+ before = list(gc.callbacks)
+ result = measure("gc", gc.collect, triples=1)
+ assert sum(result.gc_collections) >= 1
+ assert result.gc_pause_s > 0
+ assert gc.callbacks == before
+
+
+def test_heap_tracing_is_opt_in_and_cleaned_up():
+ result = measure("heap", lambda: list(range(500_000)), triples=1, trace_heap=True)
+ assert result.heap_peak_bytes >= 10 * MB
+ assert not tracemalloc.is_tracing()
+
+
+def test_latency_samples_are_summarized():
+ samples = time_each(lambda _: None, range(100))
+ result = measure("lat", lambda: None, 1, latency_samples=lambda: samples)
+ assert set(result.latency_s) == {"p50", "p95", "p99"}
+
+
+def test_percentiles_use_nearest_rank():
+ assert percentiles(range(1, 101)) == {"p50": 50, "p95": 95, "p99": 99}
+ assert percentiles([]) == {}
+
+
+def _result(wall, **kwargs):
+ return Metrics("export", 1_000, wall, wall, 100, 300, [1, 0, 0], 0.0, **kwargs)
+
+
+def test_benchmark_json_folds_repeats_and_keeps_metrics(tmp_path):
+ out = to_benchmark_json([_result(1.0), _result(3.0)])
+ (bench,) = out["benchmarks"]
+ assert bench["name"] == "export[1000]"
+ assert bench["stats"]["mean"] == 2.0
+ assert bench["stats"]["stddev"] > 0
+ assert len(bench["extra_info"]["runs"]) == 2
+ path = tmp_path / "out.json"
+ dump([_result(1.0)], path)
+ assert json.loads(path.read_text())["benchmarks"][0]["extra_info"]["runs"][0]["rss_delta_bytes"] == 200
+
+
+def test_traced_runs_never_feed_the_stats_but_supply_the_heap_peak():
+ traced = _result(9.0, heap_peak_bytes=5 * MB, traced=True)
+ (bench,) = to_benchmark_json([_result(1.0), traced, _result(3.0)])["benchmarks"]
+ assert bench["stats"]["mean"] == 2.0
+ assert bench["stats"]["rounds"] == 2
+ assert bench["extra_info"]["heap_peak_bytes"] == 5 * MB
+ assert all(not run["traced"] for run in bench["extra_info"]["runs"])
+
+
+def test_traced_only_results_produce_no_entry():
+ assert to_benchmark_json([_result(9.0, heap_peak_bytes=MB, traced=True)])["benchmarks"] == []
+
+
+def test_measure_flags_traced_runs():
+ assert measure("t", lambda: None, 1, trace_heap=True).traced
+ assert not measure("t", lambda: None, 1).traced
+
+
+def test_result_file_is_readable_by_compare_benchmarks(tmp_path):
+ script = Path(__file__).parents[2] / "scripts" / "compare_benchmarks.py"
+ spec = importlib.util.spec_from_file_location("compare_benchmarks", script)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ benchmarks = module.extract_benchmarks(to_benchmark_json([_result(1.0)]))
+ assert benchmarks["export[1000]"]["mean"] == 1.0
+
+
+def test_os_peak_rss_is_a_monotonic_high_water_mark():
+ before = os_peak_rss()
+ assert before > 0
+ keep = b"x" * (40 * MB)
+ assert os_peak_rss() >= before
+ del keep
+
+
+def test_metrics_round_trip_through_a_dict():
+ original = _result(2.0, heap_peak_bytes=MB, traced=True)
+ assert Metrics.from_dict(original.to_dict()) == original
+
+
+def test_count_calls_counts_and_restores():
+ class Target:
+ def work(self, value):
+ return value * 2
+
+ original = Target.work
+ with count_calls([(Target, "work")]) as counter:
+ assert Target().work(2) == 4
+ Target().work(3)
+ assert counter.count == 2
+ assert Target.work is original
+ Target().work(1)
+ assert counter.count == 2
diff --git a/tests/perf/test_scaling.py b/tests/perf/test_scaling.py
new file mode 100644
index 0000000..6325d9c
--- /dev/null
+++ b/tests/perf/test_scaling.py
@@ -0,0 +1,105 @@
+import pytest
+
+from .dataset import BENCHMARK_SIZES, TRIPLE_SIZES
+from .harness import Metrics, scaling_exponent
+from .report import exponents, format_report, verdict
+from .runner import Collected, Skipped, collect
+
+
+def _metrics(scenario, triples, wall, **kwargs):
+ return Metrics(scenario, triples, wall, wall, 100, 200, [0, 0, 0], 0.0, **kwargs)
+
+
+def test_benchmark_sizes_cover_the_issue_sizes():
+ assert set(TRIPLE_SIZES) <= set(BENCHMARK_SIZES)
+
+
+def test_scaling_exponent_recovers_known_slopes():
+ assert scaling_exponent([(1_000, 1.0), (10_000, 10.0), (100_000, 100.0)]) == pytest.approx(1.0)
+ assert scaling_exponent([(1_000, 1.0), (10_000, 100.0)]) == pytest.approx(2.0)
+ assert scaling_exponent([(1_000, 1.0)]) is None
+
+
+@pytest.mark.parametrize(
+ ("exponent", "label"), [(None, "n/a"), (1.0, "linear"), (1.4, "superlinear"), (1.9, "quadratic-like")]
+)
+def test_verdict(exponent, label):
+ assert verdict(exponent) == label
+
+
+def test_report_lists_sizes_exponent_and_skips():
+ collected = Collected(
+ results=[
+ _metrics("export", 1_000, 1.0),
+ _metrics("export", 10_000, 10.0),
+ _metrics("export", 10_000, 9.0, heap_peak_bytes=1, traced=True),
+ ]
+ )
+ collected.skipped.append(Skipped("export", 100_000, "why"))
+ report = format_report(collected)
+ assert "| export | 10000 | 10.000 |" in report
+ assert "| export | 1.00 | linear |" in report
+ assert "export[100000]: why" in report
+ assert exponents(collected)["export"] == pytest.approx(1.0)
+
+
+def test_report_shows_sparql_calls_and_output_size():
+ run = _metrics("export", 1_000, 1.0)
+ run.extra.update(sparql_calls=0, output_bytes=600_000)
+ report = format_report(Collected(results=[run, _metrics("store", 1_000, 1.0)]))
+ assert "| 0 | 600 |" in report
+ store_row = next(line for line in report.splitlines() if line.startswith("| store"))
+ assert store_row.endswith("| - | - |")
+
+
+def _fake_runner(monkeypatch, wall_of, *, timeout_at=None):
+ import subprocess
+
+ from . import runner
+
+ calls = []
+
+ def fake(scenario, triples, *, trace_heap=False, timeout=0):
+ calls.append((scenario, triples, trace_heap))
+ if triples == timeout_at:
+ raise subprocess.TimeoutExpired("x", timeout)
+ return _metrics(scenario, triples, wall_of(triples), traced=trace_heap)
+
+ monkeypatch.setattr(runner, "run_scenario", fake)
+ return calls
+
+
+def test_collect_repeats_and_traces_fast_runs(monkeypatch):
+ calls = _fake_runner(monkeypatch, lambda n: 0.001 * n)
+ collected = collect([1_000], ["export"], repeats=3)
+ assert calls == [("export", 1_000, False)] * 3 + [("export", 1_000, True)]
+ assert len(collected.results) == 4
+
+
+def test_collect_skips_sizes_predicted_to_exceed_the_timeout(monkeypatch):
+ calls = _fake_runner(monkeypatch, lambda n: n * n / 1e6)
+ collected = collect([1_000, 10_000, 100_000], ["import"], timeout=500, slow_s=1e9)
+ assert {size for _, size, _ in calls} == {1_000, 10_000}
+ assert [s.triples for s in collected.skipped] == [100_000]
+ assert "predicted" in collected.skipped[0].reason
+
+
+def test_collect_skips_larger_sizes_after_a_timeout(monkeypatch):
+ calls = _fake_runner(monkeypatch, lambda n: 0.001 * n, timeout_at=10_000)
+ collected = collect([1_000, 10_000, 100_000], ["export"], timeout=50)
+ assert 100_000 not in {size for _, size, _ in calls}
+ assert [s.reason.split(" after")[0] for s in collected.skipped] == ["timed out", "a smaller size timed out"]
+
+
+def test_collect_does_not_repeat_or_trace_slow_runs(monkeypatch):
+ calls = _fake_runner(monkeypatch, lambda n: 100.0)
+ collected = collect([1_000], ["import"], repeats=3, slow_s=60)
+ assert calls == [("import", 1_000, False)]
+ assert "repeats and heap trace skipped" in collected.skipped[0].reason
+
+
+@pytest.mark.scaling
+@pytest.mark.parametrize("scenario", ["export", "store", "import", "rdflib_parse", "rdflib_serialize"])
+def test_scenario_scales_roughly_linearly(scenario):
+ collected = collect([1_000, 10_000], [scenario])
+ assert exponents(collected)[scenario] < 1.5
diff --git a/tests/perf/test_scenarios.py b/tests/perf/test_scenarios.py
new file mode 100644
index 0000000..aa9affc
--- /dev/null
+++ b/tests/perf/test_scenarios.py
@@ -0,0 +1,86 @@
+import json
+
+import pytest
+
+from .dataset import TRIPLE_SIZES
+from .harness import dump
+from .runner import collect, run_scenario
+from .scenarios import SCENARIOS
+
+SMOKE_TRIPLES = TRIPLE_SIZES[0]
+
+
+@pytest.mark.parametrize("scenario", SCENARIOS)
+def test_scenario_is_correct_in_process(scenario):
+ prepared = SCENARIOS[scenario](200)
+ prepared.run()
+ prepared.verify()
+
+
+@pytest.mark.parametrize("scenario", SCENARIOS)
+def test_verify_fails_when_nothing_ran(scenario):
+ with pytest.raises(AssertionError):
+ SCENARIOS[scenario](200).verify()
+
+
+@pytest.mark.parametrize("scenario", SCENARIOS)
+def test_scenario_runs_in_a_subprocess_at_1k(scenario):
+ result = run_scenario(scenario, SMOKE_TRIPLES)
+ assert result.scenario == scenario
+ assert result.triples == SMOKE_TRIPLES
+ assert not result.traced
+ assert result.wall_s > 0
+ assert result.rss_peak_bytes >= result.rss_start_bytes
+ assert result.throughput > 0
+
+
+def test_traced_pass_reports_the_heap_peak():
+ result = run_scenario("export", 200, trace_heap=True)
+ assert result.traced
+ assert result.heap_peak_bytes > 0
+
+
+def test_a_failing_scenario_surfaces_its_error():
+ with pytest.raises(RuntimeError, match="failed"):
+ run_scenario("export", 5)
+
+
+def test_collect_writes_one_entry_per_scenario_and_size(tmp_path):
+ collected = collect([200], ["export", "store"])
+ path = tmp_path / "result.json"
+ dump(collected.results, path)
+ benchmarks = json.loads(path.read_text())["benchmarks"]
+ assert [b["name"] for b in benchmarks] == ["export[200]", "store[200]"]
+ assert all(b["extra_info"]["heap_peak_bytes"] > 0 for b in benchmarks)
+
+
+def test_worker_profile_prints_the_hot_functions(capsys):
+ from .worker import main
+
+ main(["export", "200", "--profile", "5"])
+ assert "cumulative" in capsys.readouterr().out
+
+
+@pytest.mark.parametrize("scenario", ["store", "import"])
+def test_local_store_and_import_make_no_sparql_calls(scenario):
+ assert run_scenario(scenario, 200).extra["sparql_calls"] == 0
+
+
+def test_a_sparql_query_is_counted():
+ from rdflib import Graph
+
+ from .harness import count_calls
+
+ with count_calls([(Graph, "query")]) as counter:
+ Graph().query("SELECT * WHERE { ?s ?p ?o }")
+ assert counter.count == 1
+
+
+@pytest.mark.parametrize("scenario", ["export", "rdflib_serialize"])
+def test_scenarios_with_an_output_report_its_size(scenario):
+ assert run_scenario(scenario, 200).extra["output_bytes"] > 0
+
+
+@pytest.mark.parametrize("scenario", ["store", "import", "rdflib_parse"])
+def test_scenarios_without_an_output_report_no_size(scenario):
+ assert "output_bytes" not in run_scenario(scenario, 200).extra
diff --git a/tests/perf/worker.py b/tests/perf/worker.py
new file mode 100644
index 0000000..4b39474
--- /dev/null
+++ b/tests/perf/worker.py
@@ -0,0 +1,47 @@
+"""Runs one scenario in this process and prints its Metrics as JSON on the last line."""
+
+import argparse
+import cProfile
+import json
+import pstats
+
+from rdflib import Graph
+
+from .harness import count_calls, measure
+from .scenarios import SCENARIOS
+
+
+def main(argv=None) -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("scenario", choices=sorted(SCENARIOS))
+ parser.add_argument("triples", type=int)
+ parser.add_argument("--trace-heap", action="store_true")
+ parser.add_argument(
+ "--profile", type=int, metavar="TOP", help="print the TOP cumulative-time functions instead of metrics"
+ )
+ args = parser.parse_args(argv)
+
+ prepared = SCENARIOS[args.scenario](args.triples)
+ if args.profile:
+ profiler = cProfile.Profile()
+ profiler.runcall(prepared.run)
+ pstats.Stats(profiler).sort_stats("cumulative").print_stats(args.profile)
+ return
+ with count_calls([(Graph, "query"), (Graph, "update")]) as sparql:
+ metrics = measure(
+ args.scenario,
+ prepared.run,
+ args.triples,
+ trace_heap=args.trace_heap,
+ latency_samples=lambda: prepared.samples,
+ )
+ metrics.extra["sparql_calls"] = sparql.count
+ if prepared.output_bytes:
+ metrics.extra["output_bytes"] = prepared.output_bytes()
+ if not args.trace_heap:
+ prepared.verify()
+ print(json.dumps(metrics.to_dict()))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/test_local_sparql_backend.py b/tests/test_local_sparql_backend.py
new file mode 100644
index 0000000..b418d63
--- /dev/null
+++ b/tests/test_local_sparql_backend.py
@@ -0,0 +1,114 @@
+"""Pins the observable behaviour of LocalSparqlBackend, so its internals can be optimized."""
+
+import json
+
+import pytest
+from rdflib import Graph, Literal, URIRef
+from rdflib.compare import isomorphic
+
+from oold.backend.sparql import LocalSparqlBackend, _construct_node
+
+EX = "https://example.com/"
+NAME = URIRef("https://schema.org/name")
+KNOWS = URIRef("https://schema.org/knows")
+
+
+def _doc(name, knows=(), label=None):
+ doc = {"@id": EX + name, str(NAME): [{"@value": label or name}]}
+ if knows:
+ doc[str(KNOWS)] = [{"@id": EX + other} for other in knows]
+ return doc
+
+
+def _backend(*docs):
+ backend = LocalSparqlBackend()
+ for doc in docs:
+ backend.store_jsonld_dicts({doc["@id"]: doc})
+ return backend
+
+
+def _as_graph(jsonld_dict) -> Graph:
+ graph = Graph()
+ graph.parse(data=json.dumps(jsonld_dict), format="json-ld")
+ return graph
+
+
+def test_store_replaces_the_previous_triples_of_a_subject():
+ backend = _backend(_doc("a", knows=["b"]))
+ backend.store_jsonld_dicts({EX + "a": _doc("a", label="renamed")})
+ assert set(backend.graph.predicate_objects(URIRef(EX + "a"))) == {(NAME, Literal("renamed"))}
+
+
+def test_store_leaves_other_subjects_alone():
+ backend = _backend(_doc("a"), _doc("b", knows=["a"]))
+ backend.store_jsonld_dicts({EX + "a": _doc("a")})
+ assert len(backend.graph) == 3
+
+
+def test_store_twice_is_idempotent():
+ backend = _backend(_doc("a", knows=["b"]))
+ before = len(backend.graph)
+ backend.store_jsonld_dicts({EX + "a": _doc("a", knows=["b"])})
+ assert len(backend.graph) == before
+
+
+def test_resolve_accepts_full_iris_and_the_ex_prefix():
+ backend = _backend(_doc("a"))
+ by_iri = backend.resolve_iris([EX + "a"])[EX + "a"]
+ assert backend.resolve_iris(["ex:a"])["ex:a"] == by_iri
+ assert by_iri["@id"] == EX + "a"
+
+
+def test_resolve_returns_one_entry_per_requested_iri():
+ backend = _backend(_doc("a"), _doc("b"), _doc("c"))
+ assert list(backend.resolve_iris([EX + "c", EX + "a"])) == [EX + "c", EX + "a"]
+
+
+def test_resolve_matches_the_sparql_construct_it_replaces():
+ backend = _backend(_doc("a", knows=["b", "c"]), _doc("b", knows=["a"]), _doc("c"))
+ for name in "abc":
+ iri = EX + name
+ expected = backend.graph.query(_construct_node(f"FILTER (?s = <{iri}>)")).graph
+ assert isomorphic(_as_graph(backend.resolve_iris([iri])[iri]), expected)
+
+
+def test_an_unknown_prefix_still_fails_loudly():
+ with pytest.raises(Exception, match="Unknown namespace prefix"):
+ LocalSparqlBackend().resolve_iris(["foo:a"])
+
+
+def test_resolving_a_missing_iri_gives_none():
+ backend = _backend(_doc("a"))
+ assert backend.resolve_iris([EX + "missing"]) == {EX + "missing": None}
+ assert backend.resolve_iris(["ex:missing"]) == {"ex:missing": None}
+
+
+def test_resolving_found_and_missing_iris_together():
+ backend = _backend(_doc("a"))
+ result = backend.resolve_iris([EX + "missing", EX + "a"])
+ assert list(result) == [EX + "missing", EX + "a"]
+ assert result[EX + "missing"] is None
+ assert result[EX + "a"]["@id"] == EX + "a"
+
+
+def test_storing_none_deletes_the_subject_only():
+ backend = _backend(_doc("a", knows=["b"]), _doc("b"))
+ assert backend.store_jsonld_dicts({EX + "a": None}).success
+ assert (URIRef(EX + "a"), None, None) not in backend.graph
+ assert backend.resolve_iris([EX + "a"]) == {EX + "a": None}
+ assert backend.resolve_iris([EX + "b"])[EX + "b"]["@id"] == EX + "b"
+
+
+def test_storing_none_for_an_unknown_subject_is_a_no_op():
+ backend = _backend(_doc("a"))
+ before = len(backend.graph)
+ assert backend.store_jsonld_dicts({EX + "unknown": None}).success
+ assert len(backend.graph) == before
+
+
+def test_store_through_the_backend_interface_deletes_on_none():
+ from oold.backend.interface import StoreParam
+
+ backend = _backend(_doc("a"))
+ backend.store(StoreParam(nodes={EX + "a": None}))
+ assert len(backend.graph) == 0
diff --git a/uv.lock b/uv.lock
index 6dcd529..db993bd 100644
--- a/uv.lock
+++ b/uv.lock
@@ -2811,6 +2811,7 @@ dev = [
{ name = "mkdocstrings-python" },
{ name = "panel" },
{ name = "pre-commit" },
+ { name = "psutil" },
{ name = "psycopg", extra = ["binary"] },
{ name = "pytest" },
{ name = "pytest-benchmark" },
@@ -2884,6 +2885,7 @@ dev = [
{ name = "mkdocstrings-python", specifier = ">=1.0.3" },
{ name = "panel" },
{ name = "pre-commit", specifier = ">=4.5.1" },
+ { name = "psutil", specifier = ">=5.9" },
{ name = "psycopg", extras = ["binary"], specifier = ">=3.2" },
{ name = "pytest" },
{ name = "pytest-benchmark" },