diff --git a/.github/workflows/benchmark-rdf.yml b/.github/workflows/benchmark-rdf.yml new file mode 100644 index 0000000..8f81d82 --- /dev/null +++ b/.github/workflows/benchmark-rdf.yml @@ -0,0 +1,59 @@ +name: Benchmark RDF + +# Triggers like the main workflow. The tests in tests/perf already run a 1k smoke there; this +# job runs the sizes up to 10k. 100k is left out of CI (run it locally with make benchmark-rdf, or +# pass it in the sizes input). Each run is capped at 120 s, and a size predicted to exceed that +# is skipped, so slow scenarios stay cheap. +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: + inputs: + sizes: + description: "Triple counts, space separated. Empty uses 1000 2000 4000 10000" + default: "" + +jobs: + benchmark-rdf: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up the environment + uses: ./.github/actions/setup-python-env + with: + python-version: "3.12" + + - name: Run the RDF benchmark + env: + SIZES: ${{ inputs.sizes || '1000 2000 4000 10000' }} + run: | + # Without pipefail a failing benchmark would be hidden behind tee. + set -o pipefail + # shellcheck disable=SC2086 - SIZES is a space separated list on purpose + PYTHONPATH=tests uv run python -m perf --sizes $SIZES --out benchmark_rdf_results.json \ + | tee benchmark_rdf_report.md + + - name: Publish the report in the job summary + if: always() + run: | + if [ -f benchmark_rdf_report.md ]; then + cat benchmark_rdf_report.md >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload the results + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-rdf-results + path: | + benchmark_rdf_results.json + benchmark_rdf_report.md + if-no-files-found: warn + retention-days: 90 diff --git a/.gitignore b/.gitignore index 9b99e31..571ae17 100644 --- a/.gitignore +++ b/.gitignore @@ -213,6 +213,7 @@ __marimo__/ # Benchmarks (pytest-benchmark) .benchmarks/ benchmark_results.json +benchmark_rdf_results.json baseline_benchmark.json current_benchmark.json benchmark_comparison.txt diff --git a/Makefile b/Makefile index 6c77c7a..a861d88 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,11 @@ benchmark: ## Run performance benchmarks with pytest-benchmark --benchmark-autosave \ --benchmark-json=benchmark_results.json +.PHONY: benchmark-rdf +benchmark-rdf: ## Benchmark RDF export and import at 1k, 10k and 100k triples (own harness, not pytest-benchmark) + @echo "🚀 Benchmarking RDF export and import" + @PYTHONPATH=tests uv run python -m perf --out benchmark_rdf_results.json + .PHONY: build build: clean-build ## Build wheel file @echo "🚀 Creating wheel file" diff --git a/docs/contributing.md b/docs/contributing.md index eaaa755..3923e7c 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -76,6 +76,19 @@ The tests marked `postgres` need a live server and skip unless `OOLD_TEST_POSTGR OOLD_TEST_POSTGRES_URL=postgresql://user:password@localhost:5432/postgres make test ``` +### Benchmarking RDF performance + +A separate harness measures RDF export, store and import at 1k, 10k and 100k triples. It runs each scenario in a fresh subprocess and records time, CPU, peak memory, GC pauses and the scaling exponent. It is not `pytest-benchmark`, which `make benchmark` uses. + +```bash +make benchmark-rdf # 1k to 100k, each run capped at 120 s, writes benchmark_rdf_results.json +PYTHONPATH=tests uv run python -m perf --sizes 1000 2000 --scenarios export store +PYTHONPATH=tests uv run python -m perf.worker import 1000 --profile 30 # cProfile, top 30 by cumulative time +uv run pytest -m scaling --no-cov # scaling guards, skipped by default +``` + +The code lives in `tests/perf/`. CI runs the 1k sizes with the normal tests. The `Benchmark RDF` workflow runs sizes up to 10k, triggers like the main workflow and can also be started from the Actions tab (100k is left out of CI, use `make benchmark-rdf` locally), which publishes the report in the job summary and uploads `benchmark_rdf_results.json` as an artifact. + ### Docs Doc sources live in `docs/`. The site is configured in `zensical.toml`. diff --git a/pyproject.toml b/pyproject.toml index 5526f82..0d9df25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,6 +123,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-benchmark", "pytest-playwright", + "psutil>=5.9", "jsondiff", "panel", "pre-commit>=4.5.1", @@ -227,6 +228,8 @@ markers = [ "parity: compares this port against the reference harness", # Needs a live PostgreSQL; skips unless OOLD_TEST_POSTGRES_URL is set. "postgres: runs against a live PostgreSQL server", + # Slow scaling checks over the RDF scenarios; they skip unless selected with `-m scaling`. + "scaling: slow checks of how the RDF scenarios scale", ] [tool.deptry] diff --git a/src/oold/backend/sparql.py b/src/oold/backend/sparql.py index 51ca90c..9f363ac 100644 --- a/src/oold/backend/sparql.py +++ b/src/oold/backend/sparql.py @@ -2,7 +2,7 @@ from typing import Any from pydantic import ConfigDict -from rdflib import Graph +from rdflib import Graph, URIRef from SPARQLWrapper import JSON, JSONLD, SPARQLWrapper from oold.backend.auth import UserPwdCredential, get_credential @@ -30,6 +30,19 @@ """ +_EX_NAMESPACE = "https://example.com/" + + +def _subject(iri: str) -> URIRef: + """Full IRIs pass through, ``ex:`` expands to the one namespace this module assumes.""" + if iri.startswith("http"): + return URIRef(iri) + prefix, _, local = iri.partition(":") + if prefix != "ex": + raise ValueError(f"Unknown namespace prefix : {prefix}") + return URIRef(_EX_NAMESPACE + local) + + class LocalSparqlResolver(Resolver): model_config = ConfigDict(arbitrary_types_allowed=True) @@ -55,41 +68,33 @@ def query(self, param: QueryParam) -> ResolveResult: return self.resolve(ResolveParam(iris=iris, model_cls=model_cls)) def resolve_iris(self, iris: list[str]) -> dict[str, dict]: - # sparql query to get a node by IRI with all its properties - # using CONSTRUCT to get the full node - # format the result as json-ld + """Every triple of each subject as JSON-LD, or None for a subject with no triples. + + A direct index lookup rather than a CONSTRUCT with a FILTER: the FILTER + is evaluated against every triple of the graph, once per IRI, which made + resolving quadratic in the size of the graph. + """ jsonld_dicts = {} for iri in iris: - iri_filter = f"FILTER (?s = {iri})" - # check if the iri is a full IRI or a prefix - if iri.startswith("http"): - iri_filter = f"FILTER (?s = <{iri}>)" - # todo: build full iri / prefix mapping from model context - qres = self.graph.query(_construct_node(iri_filter, "PREFIX ex: ")) - jsonld_dict = json.loads(qres.serialize(format="json-ld"))[0] - jsonld_dicts[iri] = jsonld_dict + triples = list(self.graph.triples((_subject(iri), None, None))) + if not triples: + jsonld_dicts[iri] = None + continue + node = Graph(bind_namespaces="none") + for triple in triples: + node.add(triple) + jsonld_dicts[iri] = json.loads(node.serialize(format="json-ld"))[0] return jsonld_dicts class LocalSparqlBackend(LocalSparqlResolver, Backend): def store_jsonld_dicts(self, jsonld_dicts: dict[str, dict]) -> StoreResult: - # delete all triples with the given iris as subject - for iri in jsonld_dicts: - iri_filter = f"{iri}" - # check if the iri is a full IRI or a prefix - if iri.startswith("http"): - iri_filter = f"<{iri}>" - query = """ - PREFIX ex: - DELETE WHERE { - {{{iri_filter}}} ?p ?o . - } - """.replace("{{{iri_filter}}}", iri_filter) - self.graph.update(query) - # convert jsonld_dict to rdflib triples and add to graph - g = Graph() - g.parse(data=json.dumps(jsonld_dicts[iri]), format="json-ld") - self.graph += g + # replace: drop the subject's triples, then parse the new document into the graph itself. + # None means delete, like the other stores. + for iri, jsonld_dict in jsonld_dicts.items(): + self.graph.remove((_subject(iri), None, None)) + if jsonld_dict is not None: + self.graph.parse(data=json.dumps(jsonld_dict), format="json-ld") return StoreResult(success=True) @@ -212,7 +217,7 @@ def _post_process(self, jsonld_dict: dict) -> dict: def _construct_node(iri_filter: str, prefixes: str = "") -> str: - """CONSTRUCT every triple of one subject. Shared by all three resolvers.""" + """CONSTRUCT every triple of one subject. Used by the remote resolvers.""" return f""" {prefixes} CONSTRUCT {{ ?s ?p ?o . }} diff --git a/tests/perf/__init__.py b/tests/perf/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/perf/__main__.py b/tests/perf/__main__.py new file mode 100644 index 0000000..57e5822 --- /dev/null +++ b/tests/perf/__main__.py @@ -0,0 +1,39 @@ +"""``python -m perf``: run the scenarios over several sizes, print a report, write JSON.""" + +import argparse +import sys + +from .dataset import BENCHMARK_SIZES +from .harness import dump +from .report import format_report +from .runner import DEFAULT_TIMEOUT_S, SLOW_RUN_S, collect +from .scenarios import SCENARIOS + + +def main(argv=None) -> None: + parser = argparse.ArgumentParser(prog="python -m perf") + parser.add_argument("--sizes", type=int, nargs="+", default=list(BENCHMARK_SIZES)) + parser.add_argument("--scenarios", nargs="+", choices=sorted(SCENARIOS), default=list(SCENARIOS)) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT_S, help="seconds per run") + parser.add_argument( + "--slow", type=float, default=SLOW_RUN_S, help="runs slower than this are not repeated or traced" + ) + parser.add_argument("--out", default="benchmark_rdf_results.json") + args = parser.parse_args(argv) + + collected = collect( + args.sizes, + args.scenarios, + repeats=args.repeats, + timeout=args.timeout, + slow_s=args.slow, + progress=lambda message: print(message, file=sys.stderr, flush=True), + ) + dump(collected.results, args.out, [s.as_tuple() for s in collected.skipped]) + print(format_report(collected)) + print(f"\nwrote {args.out}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/tests/perf/conftest.py b/tests/perf/conftest.py new file mode 100644 index 0000000..86b4554 --- /dev/null +++ b/tests/perf/conftest.py @@ -0,0 +1,18 @@ +from pathlib import Path + +import pytest + +HERE = Path(__file__).parent +RDFLIB_NOISE = "ignore:ConjunctiveGraph is deprecated:DeprecationWarning" + + +def pytest_collection_modifyitems(config, items): + """Silence rdflib's deprecation noise here, and skip ``scaling`` tests unless selected with ``-m scaling``.""" + selected = "scaling" in (config.getoption("-m") or "") + skip = pytest.mark.skip(reason="slow, run with `pytest -m scaling`") + for item in items: + if not item.path.is_relative_to(HERE): + continue + item.add_marker(pytest.mark.filterwarnings(RDFLIB_NOISE)) + if item.get_closest_marker("scaling") and not selected: + item.add_marker(skip) diff --git a/tests/perf/dataset.py b/tests/perf/dataset.py new file mode 100644 index 0000000..24303c3 --- /dev/null +++ b/tests/perf/dataset.py @@ -0,0 +1,85 @@ +"""Deterministic datasets for the RDF performance scenarios. + +The scale unit is the triple, not the entity: each person contributes one type +triple, one name triple and one triple per ``knows`` edge. +""" + +from pydantic import ConfigDict, Field + +TRIPLE_SIZES = (1_000, 10_000, 100_000) +# The intermediate sizes give a scaling exponent even for a scenario too slow to reach 10k. +BENCHMARK_SIZES = (1_000, 2_000, 4_000, 10_000, 100_000) +PREFIX = "perf" +NAMESPACE = "https://perf.example.org/" +BASE_TRIPLES = 2 + + +def build_person_model(): + """Define the Person model. Call it inside the test or run that needs it. + + Defining it registers the class by type IRI, so the conftest fixture that + restores the registries must be active. + """ + from oold.model import LinkedBaseModel + + class Entity(LinkedBaseModel): + model_config = ConfigDict( + json_schema_extra={ + "@context": { + "id": "@id", + "type": "@type", + "schema": "https://schema.org/", + PREFIX: NAMESPACE, + "name": "schema:name", + }, + "$id": f"{NAMESPACE}Entity", + } + ) + type: str | None = f"{PREFIX}:Entity" + name: str + + def get_iri(self): + return f"{PREFIX}:{self.name}" + + class Person(Entity): + model_config = ConfigDict( + json_schema_extra={ + "@context": [ + f"{NAMESPACE}Entity", + {"knows": {"@id": "schema:knows", "@type": "@id", "@container": "@set"}}, + ], + "$id": f"{NAMESPACE}Person", + } + ) + type: str | None = f"{PREFIX}:Person" + knows: list["Person"] | None = Field(None, json_schema_extra={"range": f"{PREFIX}:Person"}) + + return Person + + +def edges_per_person(triples: int, knows: int = 2) -> list[int]: + """Edge count per person so the dataset has exactly ``triples`` triples.""" + per_person = BASE_TRIPLES + knows + people = triples // per_person + if people <= knows + 1: + raise ValueError(f"{triples} triples is too few for {knows} edges per person") + counts = [knows] * people + for i in range(triples - people * per_person): + counts[i] += 1 + return counts + + +def make_people(person_cls, triples: int, knows: int = 2) -> list: + """Persons ``p0..pN`` where person i knows the next persons, wrapping around. + + Edges are IRI strings, so the people can be built in any order. + """ + counts = edges_per_person(triples, knows) + total = len(counts) + return [ + person_cls( + name=f"p{i}", + knows=[f"{PREFIX}:p{(i + step) % total}" for step in range(1, edges + 1)], + ) + for i, edges in enumerate(counts) + ] diff --git a/tests/perf/harness.py b/tests/perf/harness.py new file mode 100644 index 0000000..2e6505a --- /dev/null +++ b/tests/perf/harness.py @@ -0,0 +1,278 @@ +"""Resource metrics for one measured run, and the JSON result schema. + +Peak RSS is only meaningful in a fresh process, so scenarios run one per +subprocess. Heap tracing slows a run down, so it is a separate, opt-in pass: +traced runs are flagged and only contribute their heap peak to the results. +""" + +import gc +import json +import math +import platform +import statistics +import sys +import threading +import time +import tracemalloc +from collections.abc import Callable, Iterable, Iterator, Sequence +from contextlib import contextmanager +from dataclasses import asdict, dataclass, field + +import psutil + +try: + import resource +except ImportError: # Windows + resource = None + +RSS_SAMPLE_INTERVAL_S = 0.002 + + +@dataclass +class Metrics: + scenario: str + triples: int + wall_s: float + cpu_s: float + rss_start_bytes: int + rss_peak_bytes: int + gc_collections: list[int] + gc_pause_s: float + heap_peak_bytes: int | None = None + traced: bool = False + latency_s: dict[str, float] = field(default_factory=dict) + extra: dict[str, float] = field(default_factory=dict) + + @property + def cpu_ratio(self) -> float: + return self.cpu_s / self.wall_s if self.wall_s else 0.0 + + @property + def throughput(self) -> float: + """Triples per second.""" + return self.triples / self.wall_s if self.wall_s else 0.0 + + @property + def rss_delta_bytes(self) -> int: + return self.rss_peak_bytes - self.rss_start_bytes + + @property + def bytes_per_triple(self) -> float: + return self.rss_delta_bytes / self.triples if self.triples else 0.0 + + @classmethod + def from_dict(cls, data: dict) -> "Metrics": + """Inverse of ``to_dict``, dropping the derived values.""" + return cls(**{name: data[name] for name in cls.__dataclass_fields__ if name in data}) + + def to_dict(self) -> dict: + data = asdict(self) + data.update( + cpu_ratio=self.cpu_ratio, + throughput=self.throughput, + rss_delta_bytes=self.rss_delta_bytes, + bytes_per_triple=self.bytes_per_triple, + ) + return data + + +def os_peak_rss() -> int: + """Process-lifetime RSS high-water mark as tracked by the OS, in bytes.""" + if resource is None: + return psutil.Process().memory_info().peak_wset + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return peak if sys.platform == "darwin" else peak * 1024 + + +class _RssSampler: + """Polls RSS on a thread, since the peak is not readable portably after the fact.""" + + def __init__(self): + self._process = psutil.Process() + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + self.start_bytes = self._process.memory_info().rss + self._os_peak_before = os_peak_rss() + self.peak_bytes = self.start_bytes + + def _sample(self): + self.peak_bytes = max(self.peak_bytes, self._process.memory_info().rss) + + def _run(self): + while not self._stop.wait(RSS_SAMPLE_INTERVAL_S): + self._sample() + + def __enter__(self): + self._thread.start() + return self + + def __exit__(self, *exc): + self._stop.set() + self._thread.join() + self._sample() + # The OS mark is exact but lifetime-wide: it is the peak of this window + # only when the window raised it, otherwise the polled value stands. + if os_peak_rss() > self._os_peak_before: + self.peak_bytes = max(self.peak_bytes, os_peak_rss()) + + +class _GcWatch: + """Counts collections per generation and the time spent in them.""" + + def __init__(self): + self.collections = [0, 0, 0] + self.pause_s = 0.0 + self._started = 0.0 + + def _callback(self, phase, info): + if phase == "start": + self._started = time.perf_counter() + else: + self.pause_s += time.perf_counter() - self._started + self.collections[info["generation"]] += 1 + + def __enter__(self): + gc.callbacks.append(self._callback) + return self + + def __exit__(self, *exc): + gc.callbacks.remove(self._callback) + + +def scaling_exponent(points: list[tuple[int, float]]) -> float | None: + """Log-log least-squares slope of wall time over triples: 1 is linear, 2 is quadratic.""" + if len({triples for triples, _ in points}) < 2: + return None + xs = [math.log(triples) for triples, _ in points] + ys = [math.log(wall) for _, wall in points] + mean_x, mean_y = statistics.fmean(xs), statistics.fmean(ys) + return sum((x - mean_x) * (y - mean_y) for x, y in zip(xs, ys, strict=True)) / sum((x - mean_x) ** 2 for x in xs) + + +class CallCount: + count = 0 + + +@contextmanager +def count_calls(targets: Sequence[tuple[type, str]]) -> Iterator[CallCount]: + """Count calls to ``owner.name`` for each target while the block runs, then restore them.""" + counter = CallCount() + originals = [(owner, name, getattr(owner, name)) for owner, name in targets] + + def wrap(original): + def counted(*args, **kwargs): + counter.count += 1 + return original(*args, **kwargs) + + return counted + + for owner, name, original in originals: + setattr(owner, name, wrap(original)) + try: + yield counter + finally: + for owner, name, original in originals: + setattr(owner, name, original) + + +def percentiles(samples: Iterable[float]) -> dict[str, float]: + """Nearest-rank p50, p95 and p99.""" + ordered = sorted(samples) + if not ordered: + return {} + return {f"p{p}": ordered[max(math.ceil(p / 100 * len(ordered)) - 1, 0)] for p in (50, 95, 99)} + + +def time_each(fn: Callable, items: Iterable) -> list[float]: + """Seconds taken by ``fn`` for each item, to expose per-object tail cost.""" + samples = [] + for item in items: + started = time.perf_counter() + fn(item) + samples.append(time.perf_counter() - started) + return samples + + +def measure( + scenario: str, + fn: Callable[[], object], + triples: int, + *, + trace_heap: bool = False, + latency_samples: Callable[[], Iterable[float]] | None = None, +) -> Metrics: + """Run ``fn`` once and record time, CPU, RSS, GC and optionally the Python heap peak. + + ``latency_samples`` is called after the run and returns per-object seconds + that ``fn`` collected itself. + """ + gc.collect() + if trace_heap: + tracemalloc.start() + try: + with _RssSampler() as rss, _GcWatch() as gc_watch: + cpu_started = time.process_time() + wall_started = time.perf_counter() + fn() + wall_s = time.perf_counter() - wall_started + cpu_s = time.process_time() - cpu_started + heap_peak = tracemalloc.get_traced_memory()[1] if trace_heap else None + finally: + if trace_heap: + tracemalloc.stop() + return Metrics( + scenario=scenario, + triples=triples, + wall_s=wall_s, + cpu_s=cpu_s, + rss_start_bytes=rss.start_bytes, + rss_peak_bytes=rss.peak_bytes, + gc_collections=gc_watch.collections, + gc_pause_s=gc_watch.pause_s, + heap_peak_bytes=heap_peak, + traced=trace_heap, + latency_s=percentiles(latency_samples()) if latency_samples else {}, + ) + + +def environment() -> dict[str, str]: + return {"python": platform.python_version(), "platform": platform.platform(), "machine": platform.machine()} + + +def to_benchmark_json(results: list[Metrics], skipped: Iterable[tuple[str, int, str]] = ()) -> dict: + """The pytest-benchmark shape that ``scripts/compare_benchmarks.py`` reads. + + Repeated untraced runs of one scenario and size are folded into one entry, + which gives the stddev. Traced runs are slowed by tracemalloc, so they never + feed the stats: only their heap peak is attached, under ``extra_info``. + """ + grouped: dict[str, list[Metrics]] = {} + heap_peaks: dict[str, int] = {} + for result in results: + name = f"{result.scenario}[{result.triples}]" + if result.traced: + heap_peaks[name] = max(heap_peaks.get(name, 0), result.heap_peak_bytes or 0) + else: + grouped.setdefault(name, []).append(result) + benchmarks = [] + for name, runs in grouped.items(): + walls = [run.wall_s for run in runs] + benchmarks.append({ + "name": name, + "stats": { + "mean": statistics.fmean(walls), + "median": statistics.median(walls), + "stddev": statistics.stdev(walls) if len(walls) > 1 else 0.0, + "min": min(walls), + "max": max(walls), + "rounds": len(walls), + }, + "extra_info": {"runs": [run.to_dict() for run in runs], "heap_peak_bytes": heap_peaks.get(name)}, + }) + skips = [{"name": f"{scenario}[{triples}]", "reason": reason} for scenario, triples, reason in skipped] + return {"machine_info": environment(), "benchmarks": benchmarks, "skipped": skips} + + +def dump(results: list[Metrics], path, skipped: Iterable[tuple[str, int, str]] = ()) -> None: + with open(path, "w") as handle: + json.dump(to_benchmark_json(results, skipped), handle, indent=2) diff --git a/tests/perf/report.py b/tests/perf/report.py new file mode 100644 index 0000000..7f0425f --- /dev/null +++ b/tests/perf/report.py @@ -0,0 +1,68 @@ +"""Text report over collected runs: per-size metrics and the scaling exponent.""" + +import statistics + +from .harness import Metrics, scaling_exponent +from .runner import Collected + +MB = 1024 * 1024 + + +def verdict(exponent: float | None) -> str: + if exponent is None: + return "n/a" + if exponent < 1.2: + return "linear" + return "superlinear" if exponent < 1.6 else "quadratic-like" + + +def _mean(runs: list[Metrics], attr: str) -> float: + return statistics.fmean(getattr(run, attr) for run in runs) + + +def _extra(runs: list[Metrics], key: str, per_triple: bool = False) -> str: + values = [run.extra[key] / (run.triples if per_triple else 1) for run in runs if key in run.extra] + return f"{statistics.fmean(values):.0f}" if values else "-" + + +def _untraced(results: list[Metrics], scenario: str) -> dict[int, list[Metrics]]: + grouped: dict[int, list[Metrics]] = {} + for run in results: + if run.scenario == scenario and not run.traced: + grouped.setdefault(run.triples, []).append(run) + return dict(sorted(grouped.items())) + + +def exponents(collected: Collected) -> dict[str, float | None]: + scenarios = dict.fromkeys(run.scenario for run in collected.results) + return { + scenario: scaling_exponent([ + (triples, statistics.fmean(r.wall_s for r in runs)) + for triples, runs in _untraced(collected.results, scenario).items() + ]) + for scenario in scenarios + } + + +def format_report(collected: Collected) -> str: + heap = {(r.scenario, r.triples): r.heap_peak_bytes for r in collected.results if r.traced} + lines = [ + "| scenario | triples | wall s | triples/s | cpu/wall | rss MB | B/triple | heap MB | p95 ms | gc s | sparql calls | out B/triple |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + ] + for scenario in dict.fromkeys(run.scenario for run in collected.results): + for triples, runs in _untraced(collected.results, scenario).items(): + peak = heap.get((scenario, triples)) + p95 = [r.latency_s["p95"] for r in runs if "p95" in r.latency_s] + lines.append( + f"| {scenario} | {triples} | {_mean(runs, 'wall_s'):.3f} | {_mean(runs, 'throughput'):.0f} | {_mean(runs, 'cpu_ratio'):.2f} " + f"| {_mean(runs, 'rss_delta_bytes') / MB:.1f} | {_mean(runs, 'bytes_per_triple'):.0f} " + f"| {f'{peak / MB:.1f}' if peak else '-'} | {f'{statistics.fmean(p95) * 1000:.2f}' if p95 else '-'} " + f"| {_mean(runs, 'gc_pause_s'):.3f} | {_extra(runs, 'sparql_calls')} | {_extra(runs, 'output_bytes', per_triple=True)} |" + ) + lines += ["", "| scenario | exponent | verdict |", "| --- | ---: | --- |"] + for scenario, exponent in exponents(collected).items(): + lines.append(f"| {scenario} | {'-' if exponent is None else f'{exponent:.2f}'} | {verdict(exponent)} |") + if collected.skipped: + lines += ["", "Skipped:", *(f"- {s.scenario}[{s.triples}]: {s.reason}" for s in collected.skipped)] + return "\n".join(lines) diff --git a/tests/perf/runner.py b/tests/perf/runner.py new file mode 100644 index 0000000..4836807 --- /dev/null +++ b/tests/perf/runner.py @@ -0,0 +1,109 @@ +"""Runs scenarios one per subprocess, so peak RSS and imports never leak between runs.""" + +import json +import os +import statistics +import subprocess +import sys +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path + +from .harness import Metrics, scaling_exponent +from .scenarios import SCENARIOS + +TESTS_DIR = Path(__file__).parents[1] +DEFAULT_TIMEOUT_S = 120 +SLOW_RUN_S = 20.0 + + +def run_scenario( + scenario: str, triples: int, *, trace_heap: bool = False, timeout: float = DEFAULT_TIMEOUT_S +) -> Metrics: + command = [sys.executable, "-m", "perf.worker", scenario, str(triples)] + if trace_heap: + command.append("--trace-heap") + env = {**os.environ, "PYTHONPATH": os.pathsep.join(filter(None, [str(TESTS_DIR), os.environ.get("PYTHONPATH")]))} + done = subprocess.run( # noqa: S603 - fixed argv, no shell + command, capture_output=True, text=True, env=env, timeout=timeout + ) + if done.returncode != 0: + raise RuntimeError(f"{scenario}[{triples}] failed:\n{done.stderr[-2000:]}") + return Metrics.from_dict(json.loads(done.stdout.strip().splitlines()[-1])) + + +@dataclass +class Skipped: + scenario: str + triples: int + reason: str + + def as_tuple(self) -> tuple[str, int, str]: + return self.scenario, self.triples, self.reason + + +@dataclass +class Collected: + results: list[Metrics] = field(default_factory=list) + skipped: list[Skipped] = field(default_factory=list) + + +def _predict(points: list[tuple[int, float]], triples: int) -> float | None: + """Expected wall time at ``triples``, extrapolated from the sizes already run.""" + if not points: + return None + last_triples, last_wall = points[-1] + exponent = scaling_exponent(points) or 1.0 + return last_wall * (triples / last_triples) ** max(exponent, 1.0) + + +def collect( + sizes, + scenarios=None, + *, + repeats: int = 1, + timeout: float = DEFAULT_TIMEOUT_S, + slow_s: float = SLOW_RUN_S, + progress: Callable[[str], None] | None = None, +) -> Collected: + """Per scenario and size: untraced runs, then one traced run for the heap peak. + + Sizes run in ascending order. A run is skipped, not attempted, when the + sizes before it predict a wall time over ``timeout``, or when a smaller + size already timed out. Runs slower than ``slow_s`` are neither repeated + nor traced. + """ + say = progress or (lambda _: None) + out = Collected() + for scenario in scenarios or SCENARIOS: + points: list[tuple[int, float]] = [] + timed_out = False + for triples in sorted(sizes): + label = f"{scenario}[{triples}]" + predicted = _predict(points, triples) + if timed_out or (predicted is not None and predicted > timeout): + reason = ( + "a smaller size timed out" if timed_out else f"predicted {predicted:.0f}s exceeds {timeout:.0f}s" + ) + out.skipped.append(Skipped(scenario, triples, reason)) + say(f"{label} skipped: {reason}") + continue + say(f"{label} running") + try: + runs = [run_scenario(scenario, triples, timeout=timeout)] + if runs[0].wall_s < slow_s: + runs += [run_scenario(scenario, triples, timeout=timeout) for _ in range(repeats - 1)] + traced = run_scenario(scenario, triples, trace_heap=True, timeout=timeout) + else: + traced = None + out.skipped.append( + Skipped(scenario, triples, f"repeats and heap trace skipped, run takes {runs[0].wall_s:.0f}s") + ) + except subprocess.TimeoutExpired: + timed_out = True + out.skipped.append(Skipped(scenario, triples, f"timed out after {timeout:.0f}s")) + say(f"{label} timed out") + continue + out.results += runs + ([traced] if traced else []) + points.append((triples, statistics.fmean(run.wall_s for run in runs))) + return out diff --git a/tests/perf/scenarios.py b/tests/perf/scenarios.py new file mode 100644 index 0000000..973d91e --- /dev/null +++ b/tests/perf/scenarios.py @@ -0,0 +1,156 @@ +"""RDF export, store and import scenarios, plus rdflib-only floors. + +A scenario's ``prepare`` builds everything outside the measured window and +returns the callable to measure. ``verify`` checks the result afterwards, so a +fast but wrong run cannot pass. +""" + +import json +import time +from collections.abc import Callable +from dataclasses import dataclass, field + +from rdflib import Graph + +from .dataset import NAMESPACE, build_person_model, make_people + + +@dataclass +class Prepared: + run: Callable[[], None] + verify: Callable[[], None] + samples: list[float] = field(default_factory=list) + output_bytes: Callable[[], int] | None = None + + +def _iri(person) -> str: + return NAMESPACE + person.name + + +def _export_all(people) -> dict[str, dict]: + return {_iri(person): person.to_jsonld() for person in people} + + +def _timed(samples: list[float], fn, *args): + started = time.perf_counter() + result = fn(*args) + samples.append(time.perf_counter() - started) + return result + + +def _stored_backend(docs: dict[str, dict]): + from oold.backend.sparql import LocalSparqlBackend + + backend = LocalSparqlBackend() + backend.store_jsonld_dicts(docs) + return backend + + +def _count_triples(docs: dict[str, dict]) -> int: + graph = Graph() + for doc in docs.values(): + graph.parse(data=doc, format="json-ld") + return len(graph) + + +def export(triples: int) -> Prepared: + """Models to expanded JSON-LD, one ``to_jsonld`` per person.""" + people = make_people(build_person_model(), triples) + docs: dict[str, dict] = {} + samples: list[float] = [] + + def run(): + for person in people: + docs[_iri(person)] = _timed(samples, person.to_jsonld) + + def verify(): + assert len(docs) == len(people) + assert _count_triples(docs) == triples + + return Prepared(run, verify, samples, output_bytes=lambda: len(json.dumps(docs))) + + +def store(triples: int) -> Prepared: + """JSON-LD into ``LocalSparqlBackend``, one call per IRI.""" + from oold.backend.sparql import LocalSparqlBackend + + docs = _export_all(make_people(build_person_model(), triples)) + backend = LocalSparqlBackend() + samples: list[float] = [] + + def run(): + for iri, doc in docs.items(): + _timed(samples, backend.store_jsonld_dicts, {iri: doc}) + + def verify(): + assert len(backend.graph) == triples + + return Prepared(run, verify, samples) + + +def import_(triples: int) -> Prepared: + """Resolve every IRI from the backend back into models, one query per IRI.""" + from oold.backend.interface import ResolveParam + + person_cls = build_person_model() + people = make_people(person_cls, triples) + backend = _stored_backend(_export_all(people)) + nodes: dict = {} + samples: list[float] = [] + + def run(): + for person in people: + iri = _iri(person) + result = _timed(samples, backend.resolve, ResolveParam(iris=[iri], model_cls=person_cls)) + nodes.update(result.nodes) + + def verify(): + assert len(nodes) == len(people) + for person in people: + expected = person.model_dump() + node = nodes[_iri(person)].model_dump() + assert node["name"] == expected["name"] + assert sorted(node["knows"]) == sorted(expected["knows"]) + + return Prepared(run, verify, samples) + + +def rdflib_parse(triples: int) -> Prepared: + """Floor for store: the same JSON-LD parsed into a plain rdflib graph.""" + docs = _export_all(make_people(build_person_model(), triples)) + graph = Graph() + samples: list[float] = [] + + def run(): + for doc in docs.values(): + _timed(samples, lambda doc=doc: graph.parse(data=doc, format="json-ld")) + + def verify(): + assert len(graph) == triples + + return Prepared(run, verify, samples) + + +def rdflib_serialize(triples: int) -> Prepared: + """Floor for import: the whole graph serialized once instead of one query per IRI.""" + people = make_people(build_person_model(), triples) + graph = _stored_backend(_export_all(people)).graph + out: list[str] = [] + + def run(): + out.append(graph.serialize(format="json-ld")) + + def verify(): + assert out + assert len(json.loads(out[0])) == len(people) + + return Prepared(run, verify, output_bytes=lambda: len(out[0].encode())) + + +SCENARIOS: dict[str, Callable[[int], Prepared]] = { + "export": export, + "store": store, + "import": import_, + "rdflib_parse": rdflib_parse, + "rdflib_serialize": rdflib_serialize, +} diff --git a/tests/perf/test_dataset.py b/tests/perf/test_dataset.py new file mode 100644 index 0000000..4904971 --- /dev/null +++ b/tests/perf/test_dataset.py @@ -0,0 +1,32 @@ +import pytest +from rdflib import Graph + +from .dataset import BASE_TRIPLES, TRIPLE_SIZES, build_person_model, edges_per_person, make_people + + +@pytest.mark.parametrize("triples", [*TRIPLE_SIZES, 1_001, 1_003]) +def test_edge_plan_hits_the_target(triples): + counts = edges_per_person(triples) + assert sum(BASE_TRIPLES + edges for edges in counts) == triples + + +def test_edge_plan_rejects_tiny_datasets(): + with pytest.raises(ValueError): + edges_per_person(8) + + +def test_people_are_deterministic_and_loop_free(): + person_cls = build_person_model() + first = [p.model_dump() for p in make_people(person_cls, 1_001)] + assert first == [p.model_dump() for p in make_people(person_cls, 1_001)] + for person in first: + assert f"perf:{person['name']}" not in person["knows"] + assert len(set(person["knows"])) == len(person["knows"]) + + +@pytest.mark.parametrize("triples", [1_000, 1_001]) +def test_exported_graph_has_exactly_the_target_triples(triples): + graph = Graph() + for person in make_people(build_person_model(), triples): + graph.parse(data=person.to_jsonld(), format="json-ld") + assert len(graph) == triples diff --git a/tests/perf/test_harness.py b/tests/perf/test_harness.py new file mode 100644 index 0000000..185873b --- /dev/null +++ b/tests/perf/test_harness.py @@ -0,0 +1,131 @@ +import gc +import importlib.util +import json +import time +import tracemalloc +from pathlib import Path + +import pytest + +from .harness import Metrics, count_calls, dump, measure, os_peak_rss, percentiles, time_each, to_benchmark_json + +MB = 1024 * 1024 + + +def test_measure_records_time_and_cpu(): + def busy(): + sum(i * i for i in range(300_000)) + + result = measure("busy", busy, triples=1_000) + assert result.wall_s > 0 + assert 0 < result.cpu_s <= result.wall_s * 1.5 + assert result.throughput == pytest.approx(1_000 / result.wall_s) + assert result.heap_peak_bytes is None + + +def test_measure_sees_a_memory_spike(): + held = [] + + def allocate(): + held.append(b"x" * (60 * MB)) + time.sleep(0.05) + + result = measure("alloc", allocate, triples=1_000) + assert result.rss_delta_bytes >= 40 * MB + assert result.bytes_per_triple == result.rss_delta_bytes / 1_000 + + +def test_measure_counts_gc_collections(): + before = list(gc.callbacks) + result = measure("gc", gc.collect, triples=1) + assert sum(result.gc_collections) >= 1 + assert result.gc_pause_s > 0 + assert gc.callbacks == before + + +def test_heap_tracing_is_opt_in_and_cleaned_up(): + result = measure("heap", lambda: list(range(500_000)), triples=1, trace_heap=True) + assert result.heap_peak_bytes >= 10 * MB + assert not tracemalloc.is_tracing() + + +def test_latency_samples_are_summarized(): + samples = time_each(lambda _: None, range(100)) + result = measure("lat", lambda: None, 1, latency_samples=lambda: samples) + assert set(result.latency_s) == {"p50", "p95", "p99"} + + +def test_percentiles_use_nearest_rank(): + assert percentiles(range(1, 101)) == {"p50": 50, "p95": 95, "p99": 99} + assert percentiles([]) == {} + + +def _result(wall, **kwargs): + return Metrics("export", 1_000, wall, wall, 100, 300, [1, 0, 0], 0.0, **kwargs) + + +def test_benchmark_json_folds_repeats_and_keeps_metrics(tmp_path): + out = to_benchmark_json([_result(1.0), _result(3.0)]) + (bench,) = out["benchmarks"] + assert bench["name"] == "export[1000]" + assert bench["stats"]["mean"] == 2.0 + assert bench["stats"]["stddev"] > 0 + assert len(bench["extra_info"]["runs"]) == 2 + path = tmp_path / "out.json" + dump([_result(1.0)], path) + assert json.loads(path.read_text())["benchmarks"][0]["extra_info"]["runs"][0]["rss_delta_bytes"] == 200 + + +def test_traced_runs_never_feed_the_stats_but_supply_the_heap_peak(): + traced = _result(9.0, heap_peak_bytes=5 * MB, traced=True) + (bench,) = to_benchmark_json([_result(1.0), traced, _result(3.0)])["benchmarks"] + assert bench["stats"]["mean"] == 2.0 + assert bench["stats"]["rounds"] == 2 + assert bench["extra_info"]["heap_peak_bytes"] == 5 * MB + assert all(not run["traced"] for run in bench["extra_info"]["runs"]) + + +def test_traced_only_results_produce_no_entry(): + assert to_benchmark_json([_result(9.0, heap_peak_bytes=MB, traced=True)])["benchmarks"] == [] + + +def test_measure_flags_traced_runs(): + assert measure("t", lambda: None, 1, trace_heap=True).traced + assert not measure("t", lambda: None, 1).traced + + +def test_result_file_is_readable_by_compare_benchmarks(tmp_path): + script = Path(__file__).parents[2] / "scripts" / "compare_benchmarks.py" + spec = importlib.util.spec_from_file_location("compare_benchmarks", script) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + benchmarks = module.extract_benchmarks(to_benchmark_json([_result(1.0)])) + assert benchmarks["export[1000]"]["mean"] == 1.0 + + +def test_os_peak_rss_is_a_monotonic_high_water_mark(): + before = os_peak_rss() + assert before > 0 + keep = b"x" * (40 * MB) + assert os_peak_rss() >= before + del keep + + +def test_metrics_round_trip_through_a_dict(): + original = _result(2.0, heap_peak_bytes=MB, traced=True) + assert Metrics.from_dict(original.to_dict()) == original + + +def test_count_calls_counts_and_restores(): + class Target: + def work(self, value): + return value * 2 + + original = Target.work + with count_calls([(Target, "work")]) as counter: + assert Target().work(2) == 4 + Target().work(3) + assert counter.count == 2 + assert Target.work is original + Target().work(1) + assert counter.count == 2 diff --git a/tests/perf/test_scaling.py b/tests/perf/test_scaling.py new file mode 100644 index 0000000..6325d9c --- /dev/null +++ b/tests/perf/test_scaling.py @@ -0,0 +1,105 @@ +import pytest + +from .dataset import BENCHMARK_SIZES, TRIPLE_SIZES +from .harness import Metrics, scaling_exponent +from .report import exponents, format_report, verdict +from .runner import Collected, Skipped, collect + + +def _metrics(scenario, triples, wall, **kwargs): + return Metrics(scenario, triples, wall, wall, 100, 200, [0, 0, 0], 0.0, **kwargs) + + +def test_benchmark_sizes_cover_the_issue_sizes(): + assert set(TRIPLE_SIZES) <= set(BENCHMARK_SIZES) + + +def test_scaling_exponent_recovers_known_slopes(): + assert scaling_exponent([(1_000, 1.0), (10_000, 10.0), (100_000, 100.0)]) == pytest.approx(1.0) + assert scaling_exponent([(1_000, 1.0), (10_000, 100.0)]) == pytest.approx(2.0) + assert scaling_exponent([(1_000, 1.0)]) is None + + +@pytest.mark.parametrize( + ("exponent", "label"), [(None, "n/a"), (1.0, "linear"), (1.4, "superlinear"), (1.9, "quadratic-like")] +) +def test_verdict(exponent, label): + assert verdict(exponent) == label + + +def test_report_lists_sizes_exponent_and_skips(): + collected = Collected( + results=[ + _metrics("export", 1_000, 1.0), + _metrics("export", 10_000, 10.0), + _metrics("export", 10_000, 9.0, heap_peak_bytes=1, traced=True), + ] + ) + collected.skipped.append(Skipped("export", 100_000, "why")) + report = format_report(collected) + assert "| export | 10000 | 10.000 |" in report + assert "| export | 1.00 | linear |" in report + assert "export[100000]: why" in report + assert exponents(collected)["export"] == pytest.approx(1.0) + + +def test_report_shows_sparql_calls_and_output_size(): + run = _metrics("export", 1_000, 1.0) + run.extra.update(sparql_calls=0, output_bytes=600_000) + report = format_report(Collected(results=[run, _metrics("store", 1_000, 1.0)])) + assert "| 0 | 600 |" in report + store_row = next(line for line in report.splitlines() if line.startswith("| store")) + assert store_row.endswith("| - | - |") + + +def _fake_runner(monkeypatch, wall_of, *, timeout_at=None): + import subprocess + + from . import runner + + calls = [] + + def fake(scenario, triples, *, trace_heap=False, timeout=0): + calls.append((scenario, triples, trace_heap)) + if triples == timeout_at: + raise subprocess.TimeoutExpired("x", timeout) + return _metrics(scenario, triples, wall_of(triples), traced=trace_heap) + + monkeypatch.setattr(runner, "run_scenario", fake) + return calls + + +def test_collect_repeats_and_traces_fast_runs(monkeypatch): + calls = _fake_runner(monkeypatch, lambda n: 0.001 * n) + collected = collect([1_000], ["export"], repeats=3) + assert calls == [("export", 1_000, False)] * 3 + [("export", 1_000, True)] + assert len(collected.results) == 4 + + +def test_collect_skips_sizes_predicted_to_exceed_the_timeout(monkeypatch): + calls = _fake_runner(monkeypatch, lambda n: n * n / 1e6) + collected = collect([1_000, 10_000, 100_000], ["import"], timeout=500, slow_s=1e9) + assert {size for _, size, _ in calls} == {1_000, 10_000} + assert [s.triples for s in collected.skipped] == [100_000] + assert "predicted" in collected.skipped[0].reason + + +def test_collect_skips_larger_sizes_after_a_timeout(monkeypatch): + calls = _fake_runner(monkeypatch, lambda n: 0.001 * n, timeout_at=10_000) + collected = collect([1_000, 10_000, 100_000], ["export"], timeout=50) + assert 100_000 not in {size for _, size, _ in calls} + assert [s.reason.split(" after")[0] for s in collected.skipped] == ["timed out", "a smaller size timed out"] + + +def test_collect_does_not_repeat_or_trace_slow_runs(monkeypatch): + calls = _fake_runner(monkeypatch, lambda n: 100.0) + collected = collect([1_000], ["import"], repeats=3, slow_s=60) + assert calls == [("import", 1_000, False)] + assert "repeats and heap trace skipped" in collected.skipped[0].reason + + +@pytest.mark.scaling +@pytest.mark.parametrize("scenario", ["export", "store", "import", "rdflib_parse", "rdflib_serialize"]) +def test_scenario_scales_roughly_linearly(scenario): + collected = collect([1_000, 10_000], [scenario]) + assert exponents(collected)[scenario] < 1.5 diff --git a/tests/perf/test_scenarios.py b/tests/perf/test_scenarios.py new file mode 100644 index 0000000..aa9affc --- /dev/null +++ b/tests/perf/test_scenarios.py @@ -0,0 +1,86 @@ +import json + +import pytest + +from .dataset import TRIPLE_SIZES +from .harness import dump +from .runner import collect, run_scenario +from .scenarios import SCENARIOS + +SMOKE_TRIPLES = TRIPLE_SIZES[0] + + +@pytest.mark.parametrize("scenario", SCENARIOS) +def test_scenario_is_correct_in_process(scenario): + prepared = SCENARIOS[scenario](200) + prepared.run() + prepared.verify() + + +@pytest.mark.parametrize("scenario", SCENARIOS) +def test_verify_fails_when_nothing_ran(scenario): + with pytest.raises(AssertionError): + SCENARIOS[scenario](200).verify() + + +@pytest.mark.parametrize("scenario", SCENARIOS) +def test_scenario_runs_in_a_subprocess_at_1k(scenario): + result = run_scenario(scenario, SMOKE_TRIPLES) + assert result.scenario == scenario + assert result.triples == SMOKE_TRIPLES + assert not result.traced + assert result.wall_s > 0 + assert result.rss_peak_bytes >= result.rss_start_bytes + assert result.throughput > 0 + + +def test_traced_pass_reports_the_heap_peak(): + result = run_scenario("export", 200, trace_heap=True) + assert result.traced + assert result.heap_peak_bytes > 0 + + +def test_a_failing_scenario_surfaces_its_error(): + with pytest.raises(RuntimeError, match="failed"): + run_scenario("export", 5) + + +def test_collect_writes_one_entry_per_scenario_and_size(tmp_path): + collected = collect([200], ["export", "store"]) + path = tmp_path / "result.json" + dump(collected.results, path) + benchmarks = json.loads(path.read_text())["benchmarks"] + assert [b["name"] for b in benchmarks] == ["export[200]", "store[200]"] + assert all(b["extra_info"]["heap_peak_bytes"] > 0 for b in benchmarks) + + +def test_worker_profile_prints_the_hot_functions(capsys): + from .worker import main + + main(["export", "200", "--profile", "5"]) + assert "cumulative" in capsys.readouterr().out + + +@pytest.mark.parametrize("scenario", ["store", "import"]) +def test_local_store_and_import_make_no_sparql_calls(scenario): + assert run_scenario(scenario, 200).extra["sparql_calls"] == 0 + + +def test_a_sparql_query_is_counted(): + from rdflib import Graph + + from .harness import count_calls + + with count_calls([(Graph, "query")]) as counter: + Graph().query("SELECT * WHERE { ?s ?p ?o }") + assert counter.count == 1 + + +@pytest.mark.parametrize("scenario", ["export", "rdflib_serialize"]) +def test_scenarios_with_an_output_report_its_size(scenario): + assert run_scenario(scenario, 200).extra["output_bytes"] > 0 + + +@pytest.mark.parametrize("scenario", ["store", "import", "rdflib_parse"]) +def test_scenarios_without_an_output_report_no_size(scenario): + assert "output_bytes" not in run_scenario(scenario, 200).extra diff --git a/tests/perf/worker.py b/tests/perf/worker.py new file mode 100644 index 0000000..4b39474 --- /dev/null +++ b/tests/perf/worker.py @@ -0,0 +1,47 @@ +"""Runs one scenario in this process and prints its Metrics as JSON on the last line.""" + +import argparse +import cProfile +import json +import pstats + +from rdflib import Graph + +from .harness import count_calls, measure +from .scenarios import SCENARIOS + + +def main(argv=None) -> None: + parser = argparse.ArgumentParser() + parser.add_argument("scenario", choices=sorted(SCENARIOS)) + parser.add_argument("triples", type=int) + parser.add_argument("--trace-heap", action="store_true") + parser.add_argument( + "--profile", type=int, metavar="TOP", help="print the TOP cumulative-time functions instead of metrics" + ) + args = parser.parse_args(argv) + + prepared = SCENARIOS[args.scenario](args.triples) + if args.profile: + profiler = cProfile.Profile() + profiler.runcall(prepared.run) + pstats.Stats(profiler).sort_stats("cumulative").print_stats(args.profile) + return + with count_calls([(Graph, "query"), (Graph, "update")]) as sparql: + metrics = measure( + args.scenario, + prepared.run, + args.triples, + trace_heap=args.trace_heap, + latency_samples=lambda: prepared.samples, + ) + metrics.extra["sparql_calls"] = sparql.count + if prepared.output_bytes: + metrics.extra["output_bytes"] = prepared.output_bytes() + if not args.trace_heap: + prepared.verify() + print(json.dumps(metrics.to_dict())) + + +if __name__ == "__main__": + main() diff --git a/tests/test_local_sparql_backend.py b/tests/test_local_sparql_backend.py new file mode 100644 index 0000000..b418d63 --- /dev/null +++ b/tests/test_local_sparql_backend.py @@ -0,0 +1,114 @@ +"""Pins the observable behaviour of LocalSparqlBackend, so its internals can be optimized.""" + +import json + +import pytest +from rdflib import Graph, Literal, URIRef +from rdflib.compare import isomorphic + +from oold.backend.sparql import LocalSparqlBackend, _construct_node + +EX = "https://example.com/" +NAME = URIRef("https://schema.org/name") +KNOWS = URIRef("https://schema.org/knows") + + +def _doc(name, knows=(), label=None): + doc = {"@id": EX + name, str(NAME): [{"@value": label or name}]} + if knows: + doc[str(KNOWS)] = [{"@id": EX + other} for other in knows] + return doc + + +def _backend(*docs): + backend = LocalSparqlBackend() + for doc in docs: + backend.store_jsonld_dicts({doc["@id"]: doc}) + return backend + + +def _as_graph(jsonld_dict) -> Graph: + graph = Graph() + graph.parse(data=json.dumps(jsonld_dict), format="json-ld") + return graph + + +def test_store_replaces_the_previous_triples_of_a_subject(): + backend = _backend(_doc("a", knows=["b"])) + backend.store_jsonld_dicts({EX + "a": _doc("a", label="renamed")}) + assert set(backend.graph.predicate_objects(URIRef(EX + "a"))) == {(NAME, Literal("renamed"))} + + +def test_store_leaves_other_subjects_alone(): + backend = _backend(_doc("a"), _doc("b", knows=["a"])) + backend.store_jsonld_dicts({EX + "a": _doc("a")}) + assert len(backend.graph) == 3 + + +def test_store_twice_is_idempotent(): + backend = _backend(_doc("a", knows=["b"])) + before = len(backend.graph) + backend.store_jsonld_dicts({EX + "a": _doc("a", knows=["b"])}) + assert len(backend.graph) == before + + +def test_resolve_accepts_full_iris_and_the_ex_prefix(): + backend = _backend(_doc("a")) + by_iri = backend.resolve_iris([EX + "a"])[EX + "a"] + assert backend.resolve_iris(["ex:a"])["ex:a"] == by_iri + assert by_iri["@id"] == EX + "a" + + +def test_resolve_returns_one_entry_per_requested_iri(): + backend = _backend(_doc("a"), _doc("b"), _doc("c")) + assert list(backend.resolve_iris([EX + "c", EX + "a"])) == [EX + "c", EX + "a"] + + +def test_resolve_matches_the_sparql_construct_it_replaces(): + backend = _backend(_doc("a", knows=["b", "c"]), _doc("b", knows=["a"]), _doc("c")) + for name in "abc": + iri = EX + name + expected = backend.graph.query(_construct_node(f"FILTER (?s = <{iri}>)")).graph + assert isomorphic(_as_graph(backend.resolve_iris([iri])[iri]), expected) + + +def test_an_unknown_prefix_still_fails_loudly(): + with pytest.raises(Exception, match="Unknown namespace prefix"): + LocalSparqlBackend().resolve_iris(["foo:a"]) + + +def test_resolving_a_missing_iri_gives_none(): + backend = _backend(_doc("a")) + assert backend.resolve_iris([EX + "missing"]) == {EX + "missing": None} + assert backend.resolve_iris(["ex:missing"]) == {"ex:missing": None} + + +def test_resolving_found_and_missing_iris_together(): + backend = _backend(_doc("a")) + result = backend.resolve_iris([EX + "missing", EX + "a"]) + assert list(result) == [EX + "missing", EX + "a"] + assert result[EX + "missing"] is None + assert result[EX + "a"]["@id"] == EX + "a" + + +def test_storing_none_deletes_the_subject_only(): + backend = _backend(_doc("a", knows=["b"]), _doc("b")) + assert backend.store_jsonld_dicts({EX + "a": None}).success + assert (URIRef(EX + "a"), None, None) not in backend.graph + assert backend.resolve_iris([EX + "a"]) == {EX + "a": None} + assert backend.resolve_iris([EX + "b"])[EX + "b"]["@id"] == EX + "b" + + +def test_storing_none_for_an_unknown_subject_is_a_no_op(): + backend = _backend(_doc("a")) + before = len(backend.graph) + assert backend.store_jsonld_dicts({EX + "unknown": None}).success + assert len(backend.graph) == before + + +def test_store_through_the_backend_interface_deletes_on_none(): + from oold.backend.interface import StoreParam + + backend = _backend(_doc("a")) + backend.store(StoreParam(nodes={EX + "a": None})) + assert len(backend.graph) == 0 diff --git a/uv.lock b/uv.lock index 6dcd529..db993bd 100644 --- a/uv.lock +++ b/uv.lock @@ -2811,6 +2811,7 @@ dev = [ { name = "mkdocstrings-python" }, { name = "panel" }, { name = "pre-commit" }, + { name = "psutil" }, { name = "psycopg", extra = ["binary"] }, { name = "pytest" }, { name = "pytest-benchmark" }, @@ -2884,6 +2885,7 @@ dev = [ { name = "mkdocstrings-python", specifier = ">=1.0.3" }, { name = "panel" }, { name = "pre-commit", specifier = ">=4.5.1" }, + { name = "psutil", specifier = ">=5.9" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2" }, { name = "pytest" }, { name = "pytest-benchmark" },