From 8d3e7e5a50162ff872811235a6b14252dbc6bb97 Mon Sep 17 00:00:00 2001 From: Rue <343175018+thatsrue@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:58:46 -0500 Subject: [PATCH 1/3] test: run the VAD tests and a tiny-model whisper test in CI --- .github/workflows/ci.yml | 2 ++ AGENTS.md | 34 ++++++++++++++++++++++++++-------- docs/README.md | 6 +++--- pyproject.toml | 2 +- scripts/check | 2 +- tests/test_vad.py | 4 ---- tests/test_vad_streaming.py | 5 ----- tests/test_whisper_tiny.py | 32 ++++++++++++++++++++++++++++++++ 8 files changed, 65 insertions(+), 22 deletions(-) create mode 100644 tests/test_whisper_tiny.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ee4d92..4da92d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,8 @@ jobs: enable-cache: true - run: uv sync --locked --extra dev - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends ffmpeg + # The tiny model (about 75 MB) is the smoke test's real whisper; checksum-verified. + - run: WHISPER_MODEL=tiny uv run --no-sync combobulator setup-model - run: uv run --no-sync python -m pytest -q --tb=short --no-header --color=no -m "not slow and not mlx" build: runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index 8425c01..d024f9c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -98,7 +98,7 @@ Always use the Makefile. `make help` lists every target; this is the full set: | `make create-sample-files` | Populate `tmp/input/jfk-sample/` | | `make sample-ami` | Fetch the 4-speaker AMI sample into `tmp/input/ami-en2002b/` (network, checksum-verified) | | `make check-sample` | Transcribe the AMI sample; print WER against `samples/ami/EN2002b-human.txt` (no gate); fail if any word differs from `samples/ami/EN2002b-reference.txt` | -| `make test-fast` | pytest without the `slow` (real inference) tests | +| `make test-fast` | pytest as CI runs it: without the `slow` (large-model) tests; needs the tiny model (`WHISPER_MODEL=tiny make setup-whisper`) | | `make test` | The suite `scripts/check` runs, real inference included; never touches `tmp/output` | | `make test-mlx` | Opt-in MLX inference test; needs `mlx_whisper` and the mlx checkpoint | | `make build` | `uv build` into `dist/`, as CI does | @@ -344,7 +344,8 @@ fails loudly. - mlx-whisper is pinned in the `mac` extra (`mlx-whisper==0.4.3`; `pip check` is clean next to `torch==2.15.0`). `whisper_mlx.py` relies on its `ModelHolder`, so a bump must keep `tests/test_whisper_runtime.py` green and - rerun the slow mlx test. + rerun the slow mlx test. CI does not run it: run `make test-mlx` before + merging a bump. - `transcript_combobulator.whisper.load_whisper_model` loads by file path on purpose. Loading by name makes whisper sha256 the whole checkpoint per worker per run. - On macOS, `multiprocessing.set_start_method("spawn")` is mandatory for torch. @@ -368,10 +369,20 @@ fails loudly. - `tests/test_batch.py` is fast (mocks + dir fixtures). Safe to run on every change. - `tests/test_combine.py` is fast (pure Python over synthetic VTTs). -- `tests/test_vad.py`, `tests/test_transcription.py`, `tests/test_whisper.py` - are marked `slow` (real inference, about 20 s). `scripts/check` and `make test` - run them; CI and `make test-fast` skip them (no models there). The MLX test is - marked `mlx` and runs only with `make test-mlx`. +- CI runs everything except `slow` and `mlx`. That includes real inference on + two small models: `tests/test_vad.py` and `tests/test_vad_streaming.py` run + Silero VAD, which ships inside the `silero-vad` wheel, and + `tests/test_whisper_tiny.py` transcribes the JFK clip with whisper `tiny` + and asserts keywords, not exact text. The tiny test needs `models/tiny.pt`: + CI downloads it (`WHISPER_MODEL=tiny combobulator setup-model`, + checksum-verified) and locally you run `WHISPER_MODEL=tiny make setup-whisper` + once (a `WHISPER_MODEL` set in your `.env` wins over the shell, so it would + download that model instead). Without the file the test fails and says so. +- `tests/test_transcription.py` and `tests/test_whisper.py` are marked `slow` + (real `large-v3-turbo` inference, about 20 s, needs `models/large-v3-turbo.pt`). + `scripts/check` and `make test` run them; CI and `make test-fast` skip them. + The MLX test is marked `mlx` and runs only with `make test-mlx`. CI time for + the added tiny download and test is not measured until a pull request runs. - `make test` never cleans `tmp/output`. Slow tests write under `tmp/output/test_jfk*/`; `make clean-output` removes everything. - Whisper segment count is nondeterministic — use range assertions, not exact @@ -388,12 +399,19 @@ fails loudly. `ffmpeg`, so `torchcodec` (ABI-locked to torch, no aarch64 wheel before 0.11) is deliberately not a dependency. Do not reintroduce `torchaudio.load`/`save`. +- Dependabot patch and minor updates of `torch`, `torchaudio`, `onnxruntime` + and `silero-vad` merge once CI passes. CI exercises Silero VAD and whisper + `tiny` on the CPU, so it catches a broken install, a broken VAD or a broken + whisper load and decode. It does not run `large-v3-turbo` or MLX: a + regression only those show (changed words, an MLX failure) is caught by the + next local `scripts/check` (and `make test-mlx` on Apple Silicon), not before + the merge. Run both before a release. - `silero-vad` bundles both its models inside the wheel; `load_silero_vad()` needs no network access. It does not declare `onnxruntime`, which its ONNX build needs, so `pyproject.toml` pins it. - `transcript_combobulator.vad._speech_regions_from_probs` is a verbatim port of silero's region logic because silero only accepts a whole-file tensor. Bumping `silero-vad` - must keep `tests/test_vad_streaming.py` green; if silero changes the + must keep `tests/test_vad_streaming.py` green (CI runs it); if silero changes the algorithm, port the change, do not paper over the diff. - `ffmpeg` must be on `PATH`, and its `ffprobe`: libsndfile cannot read `.m4a` or `.aac`, so conversion and the timing report take their duration @@ -423,7 +441,7 @@ fails loudly. extend one that already covers it. Use synthetic fixtures where possible, and the slow whisper tests only when testing whisper behavior. 6. Run `make lint`, `make typecheck` and `make test` before declaring done. - `scripts/check` runs CI's steps (ruff, pyrefly, tests, build) plus the slow real-inference tests CI can't run (it has no models) and fails when `.venv` is missing or off the pyproject pins; `.githooks/pre-push` runs it before every push. Enable the hook once per clone with `git config core.hooksPath .githooks`. A new worktree has no `.venv`: `python3.14 -m venv .venv && make install` (the Python in `.python-version`). + `scripts/check` runs CI's steps (ruff, pyrefly, tests, build) plus the `slow` large-model tests CI doesn't run (it has no large model) and fails when `.venv` is missing or off the pyproject pins; `.githooks/pre-push` runs it before every push. Enable the hook once per clone with `git config core.hooksPath .githooks`. A new worktree has no `.venv`: `python3.14 -m venv .venv && make install` (the Python in `.python-version`). ## Common Pitfalls diff --git a/docs/README.md b/docs/README.md index 76b91d0..65e95fb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -122,15 +122,15 @@ make filter-vtt file=path/to/file.flac threshold=60 # Re-emit a VTT above a con make compare-transcripts ref="old-1.txt old-2.txt" hyp="new-1.txt new-2.txt" # Per-speaker differences between two runs # Dev -make test-fast # Suite without real inference (seconds) -make test # Suite scripts/check runs, real inference included (about 20 s) +make test-fast # Suite as CI runs it: VAD and whisper tiny inference, no large model (needs WHISPER_MODEL=tiny make setup-whisper once) +make test # Suite scripts/check runs, large-model inference included (about 20 s more) make test-mlx # Opt-in MLX inference test (needs the mlx checkpoint) make check-sample # Transcribe the 4-speaker AMI sample; print WER against AMI's human transcript; fail if any word changed (samples/ami/NOTICE.md) make lint # ruff check and format check make format # ruff format make typecheck # pyrefly check make build # sdist and wheel into dist/, as CI does -scripts/check # CI's checks: ruff, pyrefly, fast tests, build; fails on a missing or stale .venv +scripts/check # CI's checks: ruff, pyrefly, tests (large model included), build; fails on a missing or stale .venv git config core.hooksPath .githooks # Once per clone: run scripts/check before every push make clean-output # Delete every session's outputs under tmp/output ``` diff --git a/pyproject.toml b/pyproject.toml index 754775f..ccc5f88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,7 @@ python_files = ["test_*.py"] addopts = "-v" pythonpath = ["."] markers = [ - "slow: real whisper or VAD inference; needs models/ (about 20 s on Apple Silicon)", + "slow: real inference with the large-v3-turbo model; needs models/large-v3-turbo.pt, local only (about 20 s on Apple Silicon)", "mlx: real MLX inference; needs mlx_whisper and the mlx checkpoint, opt-in: make test-mlx", ] diff --git a/scripts/check b/scripts/check index b410492..d2a1a50 100755 --- a/scripts/check +++ b/scripts/check @@ -54,7 +54,7 @@ run pins python -I -c "$pins" run ruff-check ruff check --output-format concise . run ruff-format ruff format --check . run pyrefly pyrefly check -# Real whisper and VAD inference included (about 20 s). The mlx test is opt-in: make test-mlx. +# Real inference included: VAD, whisper tiny and large-v3-turbo (about 20 s more than CI). The mlx test is opt-in: make test-mlx. run pytest python -m pytest -q --tb=short --no-header --color=no -m "not mlx" run build uv build exit $status diff --git a/tests/test_vad.py b/tests/test_vad.py index f12b715..3ff9fb1 100644 --- a/tests/test_vad.py +++ b/tests/test_vad.py @@ -5,13 +5,9 @@ from pathlib import Path from unittest.mock import patch -import pytest - from transcript_combobulator.config import INPUT_DIR from transcript_combobulator.vad import load_vad_model, process_audio -pytestmark = pytest.mark.slow # real whisper/VAD inference - def test_vad_model_loading(): """Test that the VAD model loads successfully.""" diff --git a/tests/test_vad_streaming.py b/tests/test_vad_streaming.py index 2cd9205..f2d2750 100644 --- a/tests/test_vad_streaming.py +++ b/tests/test_vad_streaming.py @@ -4,7 +4,6 @@ from pathlib import Path import numpy as np -import pytest import soundfile as sf import torch from silero_vad import get_speech_timestamps @@ -22,10 +21,6 @@ process_audio, ) -pytestmark = ( - pytest.mark.slow -) # real silero inference on a few minutes of audio (seconds) - def _composite(tmp_path: Path) -> Path: """Speech islands with gaps around every threshold the region logic has.""" diff --git a/tests/test_whisper_tiny.py b/tests/test_whisper_tiny.py new file mode 100644 index 0000000..a1b21eb --- /dev/null +++ b/tests/test_whisper_tiny.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Smoke test: the real whisper path transcribes speech with the tiny model.""" + +import pytest + +from transcript_combobulator.config import INPUT_DIR, WHISPER_MODELS_DIR +from transcript_combobulator.whisper import load_whisper_model, transcribe_segment + +TINY_MODEL = WHISPER_MODELS_DIR / "tiny.pt" + + +def test_tiny_model_transcribes_jfk_speech(): + """Fails when a torch, torchaudio or openai-whisper update breaks inference. + + Regression: after the update, transcription no longer runs, or it returns + empty or garbage text. The tiny model keeps this cheap enough for CI; it + checks keywords, not exact text, because tiny's wording and numerics drift + across torch versions. The large-model tests cover accuracy locally. + """ + if not TINY_MODEL.exists(): + pytest.fail( + f"{TINY_MODEL} is missing. Download it first: " + "WHISPER_MODEL=tiny make setup-whisper" + ) + + model = load_whisper_model("tiny") + segments = transcribe_segment(INPUT_DIR / "test_jfk.wav", model=model) + + text = " ".join(segment["text"] for segment in segments).lower() + assert text.strip(), "tiny transcribed no text from the JFK clip" + for keyword in ("americans", "country"): + assert keyword in text, f"{keyword!r} not in transcript: {text!r}" From 0b098ef38dfd72897ecc2fab9cc6eb24967f8cf7 Mon Sep 17 00:00:00 2001 From: Rue <343175018+thatsrue@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:39:13 -0500 Subject: [PATCH 2/3] feat(setup): setup-model takes a model name; the tiny test no longer depends on .env --- .github/workflows/ci.yml | 2 +- AGENTS.md | 33 +++++++++++++++----------- Makefile | 12 ++++++---- docs/README.md | 9 +++---- tests/test_cli.py | 31 ++++++++++++++++++++++++ tests/test_whisper_tiny.py | 29 ++++++++++++++++++---- transcript_combobulator/cli.py | 2 +- transcript_combobulator/model_setup.py | 13 +++++++++- 8 files changed, 102 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4da92d2..c1ebc6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: - run: uv sync --locked --extra dev - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends ffmpeg # The tiny model (about 75 MB) is the smoke test's real whisper; checksum-verified. - - run: WHISPER_MODEL=tiny uv run --no-sync combobulator setup-model + - run: uv run --no-sync combobulator setup-model tiny - run: uv run --no-sync python -m pytest -q --tb=short --no-header --color=no -m "not slow and not mlx" build: runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index d024f9c..b943b12 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,8 +86,9 @@ Always use the Makefile. `make help` lists every target; this is the full set: | Command | Purpose | |---------|---------| | `make check-deps` | Verify `uv` and `ffmpeg` on the host | -| `make setup` | Check deps, `make install`, download whisper model | +| `make setup` | Check deps, `make install`, download the whisper model and tiny (`setup-tiny`) | | `make install [EXTRAS=dev,mac]` | `uv sync --locked` with those extras into `.venv`, from `uv.lock` | +| `make setup-tiny` | Download `tiny` (openai checkpoint, whatever `.env` says) into `models/`; the CI smoke test needs it | | `make setup-whisper` | Download `WHISPER_MODEL` into `models/` (checksum-verified, no model load); with `WHISPER_RUNTIME=mlx`, the pinned mlx checkpoint into `models/mlx//` | | `make run folder=path/ [force=1]` | Full parallel pipeline for one session; the only thing automation calls | | `make run-single file=path [force=1]` | Convert → VAD → transcribe one file, no combine | @@ -98,8 +99,8 @@ Always use the Makefile. `make help` lists every target; this is the full set: | `make create-sample-files` | Populate `tmp/input/jfk-sample/` | | `make sample-ami` | Fetch the 4-speaker AMI sample into `tmp/input/ami-en2002b/` (network, checksum-verified) | | `make check-sample` | Transcribe the AMI sample; print WER against `samples/ami/EN2002b-human.txt` (no gate); fail if any word differs from `samples/ami/EN2002b-reference.txt` | -| `make test-fast` | pytest as CI runs it: without the `slow` (large-model) tests; needs the tiny model (`WHISPER_MODEL=tiny make setup-whisper`) | -| `make test` | The suite `scripts/check` runs, real inference included; never touches `tmp/output` | +| `make test-fast` | pytest as CI runs it: without the `slow` (large-model) tests; needs `make setup-tiny` | +| `make test` | The suite `scripts/check` runs: large-v3-turbo, tiny and VAD inference; needs `make setup` (both models); never touches `tmp/output` | | `make test-mlx` | Opt-in MLX inference test; needs `mlx_whisper` and the mlx checkpoint | | `make build` | `uv build` into `dist/`, as CI does | | `make lint` | `ruff check` and `ruff format --check` | @@ -343,9 +344,12 @@ fails loudly. threads across the active workers. - mlx-whisper is pinned in the `mac` extra (`mlx-whisper==0.4.3`; `pip check` is clean next to `torch==2.15.0`). `whisper_mlx.py` relies on its - `ModelHolder`, so a bump must keep `tests/test_whisper_runtime.py` green and - rerun the slow mlx test. CI does not run it: run `make test-mlx` before - merging a bump. + `ModelHolder`, so a bump can break it. `tests/test_whisper_runtime.py` runs + in CI against a stand-in and does not show that; the slow mlx test does. + Patch and minor updates of `mlx` and `mlx-whisper` merge with the rest once + CI passes, and CI cannot exercise them. The GPU path is opt-in and rarely + used, so nothing guards it: run `make test-mlx` before relying on it after + an update, and fix what it finds then. - `transcript_combobulator.whisper.load_whisper_model` loads by file path on purpose. Loading by name makes whisper sha256 the whole checkpoint per worker per run. - On macOS, `multiprocessing.set_start_method("spawn")` is mandatory for torch. @@ -373,16 +377,16 @@ fails loudly. two small models: `tests/test_vad.py` and `tests/test_vad_streaming.py` run Silero VAD, which ships inside the `silero-vad` wheel, and `tests/test_whisper_tiny.py` transcribes the JFK clip with whisper `tiny` - and asserts keywords, not exact text. The tiny test needs `models/tiny.pt`: - CI downloads it (`WHISPER_MODEL=tiny combobulator setup-model`, - checksum-verified) and locally you run `WHISPER_MODEL=tiny make setup-whisper` - once (a `WHISPER_MODEL` set in your `.env` wins over the shell, so it would - download that model instead). Without the file the test fails and says so. + and asserts keywords, not exact text. It runs the openai runtime whatever + `.env` says, and checks the loaded model is tiny. It needs `models/tiny.pt`: + CI runs `combobulator setup-model tiny` (checksum-verified), and `make setup` + or `make setup-tiny` does the same locally. A model name given to + `setup-model` ignores `WHISPER_MODEL`, `WHISPER_RUNTIME` and `.env`. Without + the file the test fails and says so. - `tests/test_transcription.py` and `tests/test_whisper.py` are marked `slow` (real `large-v3-turbo` inference, about 20 s, needs `models/large-v3-turbo.pt`). `scripts/check` and `make test` run them; CI and `make test-fast` skip them. - The MLX test is marked `mlx` and runs only with `make test-mlx`. CI time for - the added tiny download and test is not measured until a pull request runs. + The MLX test is marked `mlx` and runs only with `make test-mlx`. - `make test` never cleans `tmp/output`. Slow tests write under `tmp/output/test_jfk*/`; `make clean-output` removes everything. - Whisper segment count is nondeterministic — use range assertions, not exact @@ -402,7 +406,8 @@ fails loudly. - Dependabot patch and minor updates of `torch`, `torchaudio`, `onnxruntime` and `silero-vad` merge once CI passes. CI exercises Silero VAD and whisper `tiny` on the CPU, so it catches a broken install, a broken VAD or a broken - whisper load and decode. It does not run `large-v3-turbo` or MLX: a + whisper load and decode, on Linux x86_64 with the CPU wheels CI installs; + it does not cover macOS arm64 or aarch64. It does not run `large-v3-turbo` or MLX: a regression only those show (changed words, an MLX failure) is caught by the next local `scripts/check` (and `make test-mlx` on Apple Silicon), not before the merge. Run both before a release. diff --git a/Makefile b/Makefile index 5ca134b..3dfe7cc 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help check-deps setup install setup-whisper run run-single \ +.PHONY: help check-deps setup install setup-whisper setup-tiny run run-single \ combine-transcripts filter-vtt session-env compare-transcripts create-sample-files sample-ami check-sample test test-fast build lint format typecheck \ clean-output clean-tmp @@ -23,9 +23,10 @@ help: ## Show this help echo "Per-campaign config: ENV_FILE=.env. make run folder=tmp/input/" # ── Setup ── -setup: check-deps ## Install [$(EXTRAS)] into .venv from uv.lock, download the whisper model +setup: check-deps ## Install [$(EXTRAS)] into .venv from uv.lock, download the whisper model and tiny $(MAKE) install $(MAKE) setup-whisper + $(MAKE) setup-tiny # Verify host prerequisites without touching the venv. Recipes run under # /bin/sh, so keep this POSIX (no bashisms like `&>`). @@ -48,6 +49,9 @@ install: ## Install the project into .venv from uv.lock (EXTRAS=dev or EXTRAS=de setup-whisper: ## Download WHISPER_MODEL into models/ without loading it $(CLI) setup-model +setup-tiny: ## Download the tiny model the CI smoke test uses (openai checkpoint, whatever .env says) + $(CLI) setup-model tiny + # ── Pipeline ── run: ## Full session pipeline: folder=tmp/input/ [force=1] if [ -z "$$folder" ]; then echo "Usage: make run folder=tmp/input/ [force=1]"; exit 1; fi @@ -114,13 +118,13 @@ check-sample: sample-ami ## Transcribe the AMI sample and fail if it differs fro $(VENV_CMD) ENV_FILE="$$env_file" combobulator compare --ref samples/ami/EN2002b-reference.txt \ --hyp tmp/output/ami-en2002b/ami-en2002b-combined.txt --strict -test: ## Run the suite scripts/check runs, including real whisper/VAD inference +test: ## Run the suite scripts/check runs: large-v3-turbo, tiny and VAD inference (needs setup-whisper and setup-tiny) $(VENV_CMD) python -m pytest tests/ -v -m "not mlx" test-mlx: ## Run the opt-in MLX inference test (needs the mlx checkpoint) $(VENV_CMD) python -m pytest tests/ -v -m mlx -test-fast: ## Run the suite without the slow inference tests +test-fast: ## Run the suite as CI does: no large-model tests; tiny and VAD inference (needs setup-tiny) $(VENV_CMD) python -m pytest tests/ -v -m "not slow and not mlx" build: ## Build the sdist and wheel into dist/, as CI does diff --git a/docs/README.md b/docs/README.md index 65e95fb..4e666ab 100644 --- a/docs/README.md +++ b/docs/README.md @@ -32,7 +32,8 @@ make setup `make setup` verifies prerequisites, installs the packages in `uv.lock` into `.venv` (`uv sync --locked`), and downloads the Whisper model -(`WHISPER_MODEL`, ~1.6 GB for the default `large-v3-turbo`). `pip install .` +(`WHISPER_MODEL`, ~1.6 GB for the default `large-v3-turbo`) and `tiny` (~75 MB, +what the tests that CI runs use; `make setup-tiny` fetches it alone). `pip install .` still works from the exact pins in `pyproject.toml`. Apple Silicon can run Whisper on the GPU instead of the CPU. `WHISPER_DEVICE=mps` @@ -108,7 +109,7 @@ make help # Every target with a one-line descripti # Setup make check-deps # Verify uv and ffmpeg -make setup # Install from uv.lock into .venv, download the Whisper model +make setup # Install from uv.lock into .venv, download the Whisper model and tiny make install EXTRAS=dev,mac # Reinstall with extras (mac adds mlx-whisper) # Processing @@ -122,8 +123,8 @@ make filter-vtt file=path/to/file.flac threshold=60 # Re-emit a VTT above a con make compare-transcripts ref="old-1.txt old-2.txt" hyp="new-1.txt new-2.txt" # Per-speaker differences between two runs # Dev -make test-fast # Suite as CI runs it: VAD and whisper tiny inference, no large model (needs WHISPER_MODEL=tiny make setup-whisper once) -make test # Suite scripts/check runs, large-model inference included (about 20 s more) +make test-fast # Suite as CI runs it: VAD and whisper tiny inference, no large model (needs make setup-tiny once) +make test # Suite scripts/check runs, large-model inference included (about 20 s more; needs make setup) make test-mlx # Opt-in MLX inference test (needs the mlx checkpoint) make check-sample # Transcribe the 4-speaker AMI sample; print WER against AMI's human transcript; fail if any word changed (samples/ami/NOTICE.md) make lint # ruff check and format check diff --git a/tests/test_cli.py b/tests/test_cli.py index f93138f..44e5102 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -51,3 +51,34 @@ def test_help_does_not_import_torch() -> None: text=True, ) assert probe.stdout.strip().endswith("False"), probe.stdout + + +def test_setup_model_name_overrides_the_configured_model_and_runtime() -> None: + """A named model is fetched as the openai checkpoint even when .env says + otherwise. Regression: the name is ignored and .env's model (or its mlx + runtime) is fetched instead.""" + from unittest.mock import patch + + from transcript_combobulator import model_setup + + with ( + patch.object(model_setup, "WHISPER_MODEL", "large-v3-turbo"), + patch.object(model_setup, "WHISPER_RUNTIME", "mlx"), + patch.object(model_setup, "setup_whisper", return_value=True) as openai, + patch.object(model_setup, "setup_mlx_whisper") as mlx, + ): + model_setup.main(["tiny"]) + + openai.assert_called_once_with("tiny", model_setup.WHISPER_MODELS_DIR) + mlx.assert_not_called() + + +def test_setup_model_name_failure_exits_nonzero() -> None: + from unittest.mock import patch + + from transcript_combobulator import model_setup + + with patch.object(model_setup, "setup_whisper", return_value=False): + with pytest.raises(SystemExit) as exc: + model_setup.main(["no-such-model"]) + assert exc.value.code == 1 diff --git a/tests/test_whisper_tiny.py b/tests/test_whisper_tiny.py index a1b21eb..39b9e2f 100644 --- a/tests/test_whisper_tiny.py +++ b/tests/test_whisper_tiny.py @@ -1,10 +1,17 @@ #!/usr/bin/env python3 """Smoke test: the real whisper path transcribes speech with the tiny model.""" +from unittest.mock import patch + import pytest +from transcript_combobulator import config from transcript_combobulator.config import INPUT_DIR, WHISPER_MODELS_DIR -from transcript_combobulator.whisper import load_whisper_model, transcribe_segment +from transcript_combobulator.whisper import ( + _load_whisper_model, + load_whisper_model, + transcribe_segment, +) TINY_MODEL = WHISPER_MODELS_DIR / "tiny.pt" @@ -20,11 +27,25 @@ def test_tiny_model_transcribes_jfk_speech(): if not TINY_MODEL.exists(): pytest.fail( f"{TINY_MODEL} is missing. Download it first: " - "WHISPER_MODEL=tiny make setup-whisper" + "combobulator setup-model tiny (make setup-tiny)" ) - model = load_whisper_model("tiny") - segments = transcribe_segment(INPUT_DIR / "test_jfk.wav", model=model) + # Stock openai runtime whatever .env says (the loader and the options each + # read it from their own module). + _load_whisper_model.cache_clear() + try: + with ( + patch.object(config, "WHISPER_RUNTIME", "openai"), + patch("transcript_combobulator.whisper.WHISPER_RUNTIME", "openai"), + ): + model = load_whisper_model("tiny") + segments = transcribe_segment(INPUT_DIR / "test_jfk.wav", model=model) + finally: + _load_whisper_model.cache_clear() + + # tiny has 4 encoder layers; large-v3-turbo has 32. A loader that ignored + # its argument and loaded the default model would fail here. + assert model.dims.n_audio_layer == 4, "the loaded model is not tiny" text = " ".join(segment["text"] for segment in segments).lower() assert text.strip(), "tiny transcribed no text from the JFK clip" diff --git a/transcript_combobulator/cli.py b/transcript_combobulator/cli.py index ff0ed43..e299cd9 100644 --- a/transcript_combobulator/cli.py +++ b/transcript_combobulator/cli.py @@ -17,7 +17,7 @@ "filter-vtt": "Rewrite a speaker's VTT from its saved JSON above a confidence", "compare": "Compare two combined transcripts of one session, per speaker", "session-env": "Write a session's env file from a recording bundle and a speakers file", - "setup-model": "Download WHISPER_MODEL into models/ without loading it", + "setup-model": "Download WHISPER_MODEL (or a named model) into models/ without loading it", "samples": "Build sample and test audio from samples/", } diff --git a/transcript_combobulator/model_setup.py b/transcript_combobulator/model_setup.py index 81beff8..8e4987e 100644 --- a/transcript_combobulator/model_setup.py +++ b/transcript_combobulator/model_setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Download the configured whisper model into models/ without loading it. +A model name argument fetches that openai checkpoint instead (tests use tiny). whisper.load_model would also instantiate the fp32 model (about 3x the file size in RAM); on a Pi that is most of the memory for a step that only needs @@ -80,7 +81,17 @@ def main(argv: list[str] | None = None) -> None: prog="combobulator setup-model", description="Download WHISPER_MODEL into models/ without loading it (checksum-verified).", ) - parser.parse_args(argv) + parser.add_argument( + "model", + nargs="?", + help="Model to fetch instead of WHISPER_MODEL; always the openai (torch) checkpoint", + ) + opts = parser.parse_args(argv) + if opts.model: + # A named model ignores WHISPER_MODEL and WHISPER_RUNTIME (and .env's). + if not setup_whisper(opts.model, WHISPER_MODELS_DIR): + sys.exit(1) + return if not WHISPER_MODEL: print("Error: WHISPER_MODEL is not set") sys.exit(1) From f339f9a142a31ac6d3337f015d1749712a3f40bf Mon Sep 17 00:00:00 2001 From: Rue <343175018+thatsrue@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:50:55 -0500 Subject: [PATCH 3/3] docs: the update rule covers the whole uv group; setup-model help names the argument --- AGENTS.md | 16 ++++++++-------- transcript_combobulator/model_setup.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b943b12..fb4485b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -403,14 +403,14 @@ fails loudly. `ffmpeg`, so `torchcodec` (ABI-locked to torch, no aarch64 wheel before 0.11) is deliberately not a dependency. Do not reintroduce `torchaudio.load`/`save`. -- Dependabot patch and minor updates of `torch`, `torchaudio`, `onnxruntime` - and `silero-vad` merge once CI passes. CI exercises Silero VAD and whisper - `tiny` on the CPU, so it catches a broken install, a broken VAD or a broken - whisper load and decode, on Linux x86_64 with the CPU wheels CI installs; - it does not cover macOS arm64 or aarch64. It does not run `large-v3-turbo` or MLX: a - regression only those show (changed words, an MLX failure) is caught by the - next local `scripts/check` (and `make test-mlx` on Apple Silicon), not before - the merge. Run both before a release. +- Every Dependabot patch and minor update in the uv group merges once CI + passes. CI exercises Silero VAD and whisper `tiny` on the CPU, so it catches + a broken install, a broken VAD or a broken whisper load and decode, on + Linux x86_64 with the CPU wheels CI installs; it does not cover macOS arm64 + or aarch64. It does not run `large-v3-turbo`: a regression only that model + shows (changed words) is caught by the next local `scripts/check`, not + before the merge. Run `scripts/check` before a release. The MLX path has + its own rule above. - `silero-vad` bundles both its models inside the wheel; `load_silero_vad()` needs no network access. It does not declare `onnxruntime`, which its ONNX build needs, so `pyproject.toml` pins it. diff --git a/transcript_combobulator/model_setup.py b/transcript_combobulator/model_setup.py index 8e4987e..4202a36 100644 --- a/transcript_combobulator/model_setup.py +++ b/transcript_combobulator/model_setup.py @@ -79,7 +79,7 @@ def setup_mlx_whisper(model_name: str, target_dir: Path) -> bool: def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser( prog="combobulator setup-model", - description="Download WHISPER_MODEL into models/ without loading it (checksum-verified).", + description="Download WHISPER_MODEL, or the named model, into models/ without loading it (checksum-verified).", ) parser.add_argument( "model",