From 107b92928cd10c06b8b9b4db89c1ad3d30865549 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 10:47:14 +0000 Subject: [PATCH 1/6] fix(skill-reach): a case name with a lone surrogate is written as ? in SKILL.md (#755) build_fixture wrote each SKILL.md with a strict text.encode(), so a catalogue case whose name holds a lone surrogate (a \ud800 or \udcff escape, which load_catalogue accepts) raised UnicodeEncodeError, in the regular branch and in the symlink branch. Both writes now encode with errors="replace": the lone surrogate becomes ? and the file stays valid UTF-8, as guidance_arms.py already does for its SKILL.md. surrogatepass is not used because it would write bytes that are not UTF-8. Refusing the name was not taken: it would turn a catalogue that loads today into an error. The output of a name that encodes as UTF-8 is unchanged. The new test builds both names, with and without a symlink case, and reads the written file back as UTF-8. Signed-off-by: Anton Volkov --- ...7_120000_antonwolfy_lone-surrogate-encode.md | 3 +++ dokimi/dokimi/repos/skill_reach.py | 4 ++-- dokimi/tests/test_skill_reach.py | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md diff --git a/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md b/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md new file mode 100644 index 000000000..048b80ece --- /dev/null +++ b/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md @@ -0,0 +1,3 @@ +### Fixed + +- **A skill-reach catalogue case whose name holds a lone surrogate no longer stops `build_fixture`** — a `\ud800` or `\udcff` escape in a case `name` raised `UnicodeEncodeError` when the case's `SKILL.md` was written. The file is written with the lone surrogate replaced by `?`, so it stays valid UTF-8 and the run goes on; the file of a name that encodes as UTF-8 is unchanged. diff --git a/dokimi/dokimi/repos/skill_reach.py b/dokimi/dokimi/repos/skill_reach.py index f3e0088bc..3a36f5ed7 100644 --- a/dokimi/dokimi/repos/skill_reach.py +++ b/dokimi/dokimi/repos/skill_reach.py @@ -289,7 +289,7 @@ def build_fixture(cases: list[Case], profile: HarnessProfile, roots: dict[str, P if case.link == "symlink": target = outside / f"case-{n}" / case.dirname # the index, not the id: an id is not a path target.mkdir(parents=True) - (target / "SKILL.md").write_bytes(text.encode()) + (target / "SKILL.md").write_bytes(text.encode("utf-8", "replace")) # a lone surrogate is not UTF-8 dest.parent.mkdir(parents=True, exist_ok=True) try: os.symlink(target, dest, target_is_directory=True) @@ -299,7 +299,7 @@ def build_fixture(cases: list[Case], profile: HarnessProfile, roots: dict[str, P link_kind = "symlink" else: dest.mkdir(parents=True, exist_ok=True) - (dest / "SKILL.md").write_bytes(text.encode()) + (dest / "SKILL.md").write_bytes(text.encode("utf-8", "replace")) fixture.placed.append(Placed(case, scope, rel, tok, body_tok, link_kind)) return fixture diff --git a/dokimi/tests/test_skill_reach.py b/dokimi/tests/test_skill_reach.py index cd70c080a..7ff6e7aa6 100644 --- a/dokimi/tests/test_skill_reach.py +++ b/dokimi/tests/test_skill_reach.py @@ -784,6 +784,23 @@ def test_the_skill_name_loads_back_as_the_same_string(name): assert yaml.safe_load(text.split("---")[1])["name"] == name +@pytest.mark.parametrize("link", ["", "symlink"]) +@pytest.mark.parametrize("name", [pytest.param("n\udcff", id="dcff"), pytest.param("n\ud800", id="d800")]) +def test_a_case_name_with_a_lone_surrogate_is_written_as_a_question_mark(tmp_path, name, link): + if link: + try: + os.symlink(tmp_path, tmp_path / "probe-link", target_is_directory=True) + except (OSError, NotImplementedError) as exc: + pytest.skip(f"cannot create a symlink here: {exc}") + profile = load_profile(str(_profile(tmp_path))) + roots = _roots(tmp_path) + case = Case(id="c", at="project:x", name=name, dirname="d1", link=link or None) + fixture = build_fixture([case], profile, roots, tmp_path / "outside", "s") + assert [p.case.id for p in fixture.placed] == ["c"] + text = (roots["project"] / "x/d1/SKILL.md").read_bytes().decode("utf-8") + assert "\n# n?\n" in text + + def test_a_skill_dir_above_the_workspace_is_refused(tmp_path): (tmp_path / ".claude" / "skills").mkdir(parents=True) workspace = tmp_path / "ws" From 772eafc1db27ff1f5d541193c364516cf7c13e71 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 10:47:36 +0000 Subject: [PATCH 2/6] fix(harbor): export writes task.toml with a lone surrogate as ? (#759) harbor_view stopped with UnicodeEncodeError when export wrote task.toml for a task.yaml whose metadata holds a lone surrogate (a \udcff or \ud800 YAML escape): Path.write_text encodes strictly. The content key was already a digest (#720), the write was not. The instruction, written by the line above, raised the same way for an instruction with such an escape. task.toml and instruction.md are now written through open(..., "w", encoding="utf-8", errors="replace"), which keeps write_text's newline handling: the lone surrogate becomes ? and both files stay valid UTF-8. surrogatepass is not used because it would write a task.toml that is not UTF-8. Refusing the value was not taken: it would turn a task that loads today into an error. The files of a task that encodes as UTF-8 are unchanged. The unreleased fragment that said such an export still stops no longer says so. The new tests read the written task.toml back with tomllib and the instruction as UTF-8. Signed-off-by: Anton Volkov --- ...61114_120000_antonwolfy_surrogate-digests.md | 2 +- ...7_120000_antonwolfy_lone-surrogate-encode.md | 1 + dokimi/dokimi/harnesses/harbor_export.py | 10 ++++++++-- dokimi/tests/test_tbench_v1.py | 17 +++++++++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/dokimi/changelog.d/20261114_120000_antonwolfy_surrogate-digests.md b/dokimi/changelog.d/20261114_120000_antonwolfy_surrogate-digests.md index 97c47824e..0ae84b002 100644 --- a/dokimi/changelog.d/20261114_120000_antonwolfy_surrogate-digests.md +++ b/dokimi/changelog.d/20261114_120000_antonwolfy_surrogate-digests.md @@ -7,4 +7,4 @@ - **A matrix cell name with a lone surrogate no longer stops the plugin-delta pairing** — the pair suffix of the delta artifact in `eval/matrix_rollup.py` is hashed with `surrogatepass`, so a JSON escape such as `"n\ud800"` or `"n\udcff"` in the matrix config gets its own suffix, and the suffix of a name that encodes as UTF-8 is unchanged. - **A symbol with a non-UTF-8 byte no longer stops `task_id` of the graph tasks** — the hash of the symbols in `repos/graph_tasks.py` uses their UTF-8 bytes (`surrogateescape`); the id of a UTF-8 symbol set is unchanged. - **A bundle file whose name is not UTF-8 no longer stops `Bundle.measured_digest`** — the digest in `novelty/bundle.py` is taken over the UTF-8 bytes (`surrogateescape`) of the path rows; the digest of a UTF-8 path is unchanged. -- **A `\udXXX` escape in a loaded JSON or YAML file no longer stops the digests of the parsed data** — `item_digest` and `assignment_digest` (leak gate), `graph_digest` (eval coverage), `expectations_digest` (conformance), `response_cache_key` and the Harbor export's content key encode with `surrogatepass`, so a lone surrogate gets a digest of its own and every digest that encoded before is unchanged. A terminal-bench task whose `task.yaml` metadata holds such an escape still stops the Harbor export when it writes `task.toml`. +- **A `\udXXX` escape in a loaded JSON or YAML file no longer stops the digests of the parsed data** — `item_digest` and `assignment_digest` (leak gate), `graph_digest` (eval coverage), `expectations_digest` (conformance), `response_cache_key` and the Harbor export's content key encode with `surrogatepass`, so a lone surrogate gets a digest of its own and every digest that encoded before is unchanged. diff --git a/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md b/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md index 048b80ece..4bbec4250 100644 --- a/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md +++ b/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md @@ -1,3 +1,4 @@ ### Fixed - **A skill-reach catalogue case whose name holds a lone surrogate no longer stops `build_fixture`** — a `\ud800` or `\udcff` escape in a case `name` raised `UnicodeEncodeError` when the case's `SKILL.md` was written. The file is written with the lone surrogate replaced by `?`, so it stays valid UTF-8 and the run goes on; the file of a name that encodes as UTF-8 is unchanged. +- **A terminal-bench task with a lone surrogate in its `task.yaml` no longer stops the Harbor export** — a `\udcff` or `\ud800` escape in the metadata or the instruction raised `UnicodeEncodeError` when `harbor_view` wrote `task.toml` or `instruction.md`. Both files are written with the lone surrogate replaced by `?`, so they stay valid UTF-8; the files of a task that encodes as UTF-8 are unchanged. diff --git a/dokimi/dokimi/harnesses/harbor_export.py b/dokimi/dokimi/harnesses/harbor_export.py index 50eddd938..c20ef36bf 100644 --- a/dokimi/dokimi/harnesses/harbor_export.py +++ b/dokimi/dokimi/harnesses/harbor_export.py @@ -137,11 +137,17 @@ def export(task: TaskSpec, dest: Path) -> Path: if script.is_file(): script.chmod(0o755) instruction = task.instruction if task.instruction.endswith("\n") else task.instruction + "\n" - (dest / "instruction.md").write_text(instruction, encoding="utf-8") - (dest / "task.toml").write_text(_task_toml(task), encoding="utf-8") + _write_utf8(dest / "instruction.md", instruction) + _write_utf8(dest / "task.toml", _task_toml(task)) return dest +def _write_utf8(path: Path, text: str) -> None: + """``write_text`` in UTF-8, a lone surrogate (a YAML ``\\udcff`` escape) written as ``?``: the file stays valid UTF-8.""" + with path.open("w", encoding="utf-8", errors="replace") as out: + out.write(text) + + def _copy_context(task: TaskSpec, env_dir: Path) -> None: """The task's files into ``environment/``, the build context's at its top, as Harbor's mapper does. diff --git a/dokimi/tests/test_tbench_v1.py b/dokimi/tests/test_tbench_v1.py index 72451f4a1..ef64e7e73 100644 --- a/dokimi/tests/test_tbench_v1.py +++ b/dokimi/tests/test_tbench_v1.py @@ -316,6 +316,23 @@ def test_exported_task_toml_parses_with_a_non_bmp_character(tmp_path): assert tomllib.loads((dest / "task.toml").read_text(encoding="utf-8"))["metadata"]["author_email"] == author +@pytest.mark.parametrize("escape", ["\\udcff", "\\ud800"]) +def test_harbor_view_writes_a_lone_surrogate_in_the_metadata_as_a_question_mark(tmp_path, escape): + import tomllib + + task = load_task(_tb1(tmp_path / "tasks", yaml_extra=f'author_email: "n{escape}"\n')) + view = harbor_export.harbor_view(task, cache_root=tmp_path / "cache") + raw = (view.path / "task.toml").read_bytes() + assert tomllib.loads(raw.decode("utf-8"))["metadata"]["author_email"] == "n?" + + +def test_harbor_view_writes_a_lone_surrogate_in_the_instruction_as_a_question_mark(tmp_path): + path = _tb1(tmp_path / "tasks") + _write(path / "task.yaml", 'instruction: "Create n\\udcff.txt"\nauthor_name: a\n') + view = harbor_export.harbor_view(load_task(path), cache_root=tmp_path / "cache") + assert (view.path / "instruction.md").read_bytes().decode("utf-8") == "Create n?.txt\n" + + def test_export_puts_a_subdir_context_and_its_named_dockerfile_where_harbor_builds(tmp_path): compose = _COMPOSE.replace(" dockerfile: Dockerfile\n", " context: ./env\n dockerfile: Dockerfile.x\n working_dir: /srv\n") From 92d3a1d3bc6bd30dbdb4a151c9aeb6d1338c9bf9 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 10:49:06 +0000 Subject: [PATCH 3/6] fix(process): a string input with a lone surrogate is sent as ? instead of raising (#760) process.run encoded a string input with a strict input.encode("utf-8"), so a prompt built from a question or an artifact text with a \ud800 JSON escape raised UnicodeEncodeError before the child started. FakeRunner.__call__ and reply, the test adapter, raised the same way for a string input and a string stdout or stderr. All three now encode with errors="replace": the lone surrogate becomes ? and the child gets valid UTF-8, as a surrogatepass encoding would not give. Refusing the input where the prompt is built was not taken: it would turn a question set that loads today into an error. The bytes of a string that encodes as UTF-8 are unchanged. The new tests run a child that echoes its stdin for the two escapes, and check the bytes FakeRunner records and reply holds. Signed-off-by: Anton Volkov --- ...120000_antonwolfy_lone-surrogate-encode.md | 1 + dokimi/dokimi/process.py | 6 +++--- dokimi/tests/test_process.py | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md b/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md index 4bbec4250..d15810cd1 100644 --- a/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md +++ b/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md @@ -2,3 +2,4 @@ - **A skill-reach catalogue case whose name holds a lone surrogate no longer stops `build_fixture`** — a `\ud800` or `\udcff` escape in a case `name` raised `UnicodeEncodeError` when the case's `SKILL.md` was written. The file is written with the lone surrogate replaced by `?`, so it stays valid UTF-8 and the run goes on; the file of a name that encodes as UTF-8 is unchanged. - **A terminal-bench task with a lone surrogate in its `task.yaml` no longer stops the Harbor export** — a `\udcff` or `\ud800` escape in the metadata or the instruction raised `UnicodeEncodeError` when `harbor_view` wrote `task.toml` or `instruction.md`. Both files are written with the lone surrogate replaced by `?`, so they stay valid UTF-8; the files of a task that encodes as UTF-8 are unchanged. +- **An input string with a lone surrogate no longer stops `process.run`** — a prompt built from a question or an artifact text with a `\ud800` JSON escape raised `UnicodeEncodeError` before the child started. The string is sent to the child's stdin with the lone surrogate replaced by `?`, so the child gets valid UTF-8; `FakeRunner` and `reply` of the test adapter encode the same way. The bytes of a string that encodes as UTF-8 are unchanged. diff --git a/dokimi/dokimi/process.py b/dokimi/dokimi/process.py index 5652f02bf..27942a7ee 100644 --- a/dokimi/dokimi/process.py +++ b/dokimi/dokimi/process.py @@ -187,7 +187,7 @@ def run( """Run ``argv`` (no shell) to completion or ``timeout`` seconds; see the module doc.""" args = _check(argv, timeout, capture_limit, term_grace) preexec = _limits_preexec(limits) if limits else None - data = input.encode("utf-8") if isinstance(input, str) else input + data = input.encode("utf-8", "replace") if isinstance(input, str) else input with _Spool(stdout_to) as out_file, _Spool(stderr_to) as err_file: start = time.monotonic() try: @@ -743,7 +743,7 @@ class Call: def reply(stdout: str | bytes = "", stderr: str | bytes = "", returncode: int = 0, *, timed_out: bool = False) -> Completed: """A :class:`FakeRunner` answer (``argv`` and timing are filled in from the call).""" - enc = (lambda s: s.encode("utf-8") if isinstance(s, str) else s) + enc = (lambda s: s.encode("utf-8", "replace") if isinstance(s, str) else s) return Completed((), TIMEOUT_RC if timed_out else returncode, enc(stdout), enc(stderr), timed_out) @@ -774,7 +774,7 @@ def __call__( term_grace: float = 0.0, ) -> Completed: args = _check(argv, timeout, capture_limit, term_grace) - data = input.encode("utf-8") if isinstance(input, str) else input + data = input.encode("utf-8", "replace") if isinstance(input, str) else input path = (lambda p: Path(p) if p is not None else None) call = Call(tuple(args), path(cwd), env.resolve(), timeout, data, capture_stdout, limits, capture_limit, path(stdout_to), path(stderr_to), term_grace) diff --git a/dokimi/tests/test_process.py b/dokimi/tests/test_process.py index eb146107c..5df3a79bc 100644 --- a/dokimi/tests/test_process.py +++ b/dokimi/tests/test_process.py @@ -754,6 +754,25 @@ def handler(call): assert fake.argvs() == [("ok", str(tmp_path)), ("slow",), ("missing",)] +@pytest.mark.parametrize("text", [pytest.param("n\ud800", id="d800"), pytest.param("n\udcff", id="dcff")]) +def test_a_lone_surrogate_in_the_input_reaches_the_child_as_a_question_mark(text): + echo = "import sys; sys.stdout.buffer.write(sys.stdin.buffer.read())" + done = _py(echo, input=text) + assert done.returncode == 0 and done.stdout_bytes == b"n?" + + +@pytest.mark.parametrize("text", [pytest.param("n\ud800", id="d800"), pytest.param("n\udcff", id="dcff")]) +def test_fake_runner_records_a_lone_surrogate_in_the_input_as_a_question_mark(text): + fake = FakeRunner() + assert fake(["x"], timeout=1, env=INHERIT, input=text).ok + assert fake.calls[0].input == b"n?" + + +@pytest.mark.parametrize("text", [pytest.param("n\ud800", id="d800"), pytest.param("n\udcff", id="dcff")]) +def test_reply_encodes_a_lone_surrogate_in_the_output_as_a_question_mark(text): + assert reply(text, text).stdout_bytes == b"n?" and reply(text, text).stderr_bytes == b"n?" + + def test_input_larger_than_the_pipe_reaches_a_child_that_reads_late(): data = b"x" * 300_000 done = _py("import sys, time; time.sleep(0.6); sys.stdout.buffer.write(sys.stdin.buffer.read())", From 6db33d6ee7b16032c1307457b75a2c3e99af3b0b Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 10:49:40 +0000 Subject: [PATCH 4/6] fix(context7): a query or library id with a lone surrogate is sent as %3F (#794) Context7Client.get_library_docs built the request URL with urllib.parse.quote(query), which encodes strictly, so a question with a \ud800 or \udcff escape raised UnicodeEncodeError on a cache miss, after the cache key (a digest since #796) was computed. The library id went into the URL path unquoted, and httpx raised the same error for a lone surrogate there when it built the request. The query is quoted with errors="replace" and the library id is quoted for the path with errors="replace" and the path characters left as they are, so the lone surrogate is sent as %3F in both. surrogatepass is not used because it would send bytes that are not UTF-8; refusing was not taken because it would turn a question set that loads today into an error. The cache key still hashes the text, so two queries that send the same URL keep separate cache entries. The URL of a query or id that holds only letters, digits and those path characters is unchanged; a library id with another character (a space, ? or #) is percent-encoded now. The new tests mock httpx.get with an empty cache directory and check the URL requested for both escapes in the query and in a GitHub-style and a website-style library id. Signed-off-by: Anton Volkov --- ...120000_antonwolfy_lone-surrogate-encode.md | 1 + dokimi/dokimi/mcp/context7.py | 9 ++++---- dokimi/tests/test_mcp_context7.py | 22 +++++++++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md b/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md index d15810cd1..12512451d 100644 --- a/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md +++ b/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md @@ -3,3 +3,4 @@ - **A skill-reach catalogue case whose name holds a lone surrogate no longer stops `build_fixture`** — a `\ud800` or `\udcff` escape in a case `name` raised `UnicodeEncodeError` when the case's `SKILL.md` was written. The file is written with the lone surrogate replaced by `?`, so it stays valid UTF-8 and the run goes on; the file of a name that encodes as UTF-8 is unchanged. - **A terminal-bench task with a lone surrogate in its `task.yaml` no longer stops the Harbor export** — a `\udcff` or `\ud800` escape in the metadata or the instruction raised `UnicodeEncodeError` when `harbor_view` wrote `task.toml` or `instruction.md`. Both files are written with the lone surrogate replaced by `?`, so they stay valid UTF-8; the files of a task that encodes as UTF-8 are unchanged. - **An input string with a lone surrogate no longer stops `process.run`** — a prompt built from a question or an artifact text with a `\ud800` JSON escape raised `UnicodeEncodeError` before the child started. The string is sent to the child's stdin with the lone surrogate replaced by `?`, so the child gets valid UTF-8; `FakeRunner` and `reply` of the test adapter encode the same way. The bytes of a string that encodes as UTF-8 are unchanged. +- **A query or library id with a lone surrogate no longer stops `Context7Client.get_library_docs` on a cache miss** — `urllib.parse.quote(query)` raised `UnicodeEncodeError` for a `\ud800` or `\udcff` escape in a question, and a library id with one failed when the request was built. Both now go into the URL with the lone surrogate as `%3F`; the library id is percent-encoded for the URL path (letters, digits, `/` and the path characters `%:@!$&'()*+,;=-._~` stay as they are), and the cache key still tells two such queries apart. The URL of a query or id that encodes as UTF-8 and holds only those characters is unchanged. diff --git a/dokimi/dokimi/mcp/context7.py b/dokimi/dokimi/mcp/context7.py index 8a5b456bc..672640b10 100644 --- a/dokimi/dokimi/mcp/context7.py +++ b/dokimi/dokimi/mcp/context7.py @@ -153,20 +153,21 @@ def get_library_docs( # Website IDs: no slash + (underscore OR 'github_io' pattern) # This avoids misclassifying GitHub repos with underscores (e.g., some_org/repo) is_website = "/" not in library_id and ("_" in library_id or "github_io" in library_id) + path_id = urllib.parse.quote(library_id, safe="/%:@!$&'()*+,;=-._~", errors="replace") # a lone surrogate -> %3F if is_website: # Website format: websites/intelpython_github_io_dpnp url = ( - f"{self.endpoint}/websites/{library_id}/llms.txt" + f"{self.endpoint}/websites/{path_id}/llms.txt" f"?tokens={max_tokens}" - f"&topic={urllib.parse.quote(query)}" + f"&topic={urllib.parse.quote(query, errors='replace')}" ) else: # GitHub repo format: uxlfoundation/oneTBB (or some_org/repo) url = ( - f"{self.endpoint}/{library_id}/llms.txt" + f"{self.endpoint}/{path_id}/llms.txt" f"?tokens={max_tokens}" - f"&topic={urllib.parse.quote(query)}" + f"&topic={urllib.parse.quote(query, errors='replace')}" ) logger.info(f"Fetching from Context7: {library_id} query={query[:50]}...") diff --git a/dokimi/tests/test_mcp_context7.py b/dokimi/tests/test_mcp_context7.py index 5cb859b9b..3690bb2ba 100644 --- a/dokimi/tests/test_mcp_context7.py +++ b/dokimi/tests/test_mcp_context7.py @@ -112,6 +112,28 @@ def test_get_library_docs_with_cache(self, mock_get, tmp_path): assert docs2[0]["content"] == content assert mock_get.call_count == 1 # No additional HTTP call + @patch('httpx.get') + def test_get_library_docs_sends_a_lone_surrogate_in_the_query_as_a_question_mark(self, mock_get, tmp_path): + mock_get.return_value = Mock(text="Documentation about parallel_for. " * 4) + client = Context7Client(cache_dir=tmp_path / "cache") + for query in ("n\ud800", "n\udcff"): + assert not client.get_library_docs("org/x", query)[0]["cached"] + assert mock_get.call_args[0][0].endswith("&topic=n%3F") + assert mock_get.call_count == 2 # two queries that send the same URL keep their own cache entry + + @patch('httpx.get') + @pytest.mark.parametrize(("library_id", "path"), [ + ("org/n\ud800", "org/n%3F/llms.txt"), + ("n\udcff_x", "websites/n%3F_x/llms.txt"), + ("org/a-b.c~d", "org/a-b.c~d/llms.txt"), + ("org/x%20y:z", "org/x%20y:z/llms.txt"), + ]) + def test_get_library_docs_sends_a_lone_surrogate_in_the_library_id_as_a_question_mark( + self, mock_get, tmp_path, library_id, path): + mock_get.return_value = Mock(text="Documentation about parallel_for. " * 4) + Context7Client(cache_dir=tmp_path / "cache").get_library_docs(library_id, "n\ud800") + assert mock_get.call_args[0][0].endswith("/" + path + "?tokens=8000&topic=n%3F") + @patch('httpx.get') def test_get_library_docs_404(self, mock_get): """Test handling of library not found (404).""" From 7e0ca3fb6db545afe55dff79db3f8453bee4cd7e Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 12:31:49 +0000 Subject: [PATCH 5/6] test(harbor): read instruction.md as text so the lone-surrogate test passes on Windows Settles the exai-bot thread on test_tbench_v1.py:333. _write_utf8 writes in text mode, so on Windows the file has CRLF line ends and the exact-bytes comparison against "\n" failed. read_text(encoding="utf-8") normalizes the line ends and still decodes strictly. Signed-off-by: Anton Volkov --- dokimi/tests/test_tbench_v1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/tests/test_tbench_v1.py b/dokimi/tests/test_tbench_v1.py index ef64e7e73..91041d772 100644 --- a/dokimi/tests/test_tbench_v1.py +++ b/dokimi/tests/test_tbench_v1.py @@ -330,7 +330,7 @@ def test_harbor_view_writes_a_lone_surrogate_in_the_instruction_as_a_question_ma path = _tb1(tmp_path / "tasks") _write(path / "task.yaml", 'instruction: "Create n\\udcff.txt"\nauthor_name: a\n') view = harbor_export.harbor_view(load_task(path), cache_root=tmp_path / "cache") - assert (view.path / "instruction.md").read_bytes().decode("utf-8") == "Create n?.txt\n" + assert (view.path / "instruction.md").read_text(encoding="utf-8") == "Create n?.txt\n" def test_export_puts_a_subdir_context_and_its_named_dockerfile_where_harbor_builds(tmp_path): From a28d84b005e3795a6a8ac01a25d1e0b5205dbced Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 12:47:03 +0000 Subject: [PATCH 6/6] fix(process): join a JSON surrogate pair before a lone one becomes ? MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settles the review threads on the lone-surrogate encode. - A JSON escape pair for a non-BMP character (U+1F600 as `😀`) loads as two lone surrogates in PyYAML and json, so `errors="replace"` wrote `??`. `process_io.joined_pairs` joins each pair into its character first (a lone surrogate stays and becomes `?`); `utf8_bytes` is the bytes form. `process.run`, `FakeRunner`, `reply`, `build_fixture`, the Harbor export and the Context7 URL use them. - `harbor_export._write_utf8` is gone: `write_text(..., errors="replace")` does the same. - `llm_cli.call` writes the system prompt file with the same replacement, so a lone surrogate in a system message no longer raises (#861). - The `process` module doc says a `str` input goes out as UTF-8 with a lone surrogate as `?`. Signed-off-by: Anton Volkov --- ..._120000_antonwolfy_lone-surrogate-encode.md | 9 +++++---- dokimi/dokimi/harnesses/harbor_export.py | 13 +++++-------- dokimi/dokimi/llm_cli.py | 3 ++- dokimi/dokimi/mcp/context7.py | 10 +++++++--- dokimi/dokimi/process.py | 12 ++++++------ dokimi/dokimi/process_io.py | 11 +++++++++++ dokimi/dokimi/repos/skill_reach.py | 5 +++-- dokimi/tests/test_llm_cli.py | 11 +++++++++++ dokimi/tests/test_mcp_context7.py | 6 ++++++ dokimi/tests/test_process.py | 18 ++++++++++++++++++ dokimi/tests/test_skill_reach.py | 8 ++++++++ dokimi/tests/test_tbench_v1.py | 11 +++++++++++ 12 files changed, 93 insertions(+), 24 deletions(-) diff --git a/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md b/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md index 12512451d..5cb3553a0 100644 --- a/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md +++ b/dokimi/changelog.d/20261117_120000_antonwolfy_lone-surrogate-encode.md @@ -1,6 +1,7 @@ ### Fixed -- **A skill-reach catalogue case whose name holds a lone surrogate no longer stops `build_fixture`** — a `\ud800` or `\udcff` escape in a case `name` raised `UnicodeEncodeError` when the case's `SKILL.md` was written. The file is written with the lone surrogate replaced by `?`, so it stays valid UTF-8 and the run goes on; the file of a name that encodes as UTF-8 is unchanged. -- **A terminal-bench task with a lone surrogate in its `task.yaml` no longer stops the Harbor export** — a `\udcff` or `\ud800` escape in the metadata or the instruction raised `UnicodeEncodeError` when `harbor_view` wrote `task.toml` or `instruction.md`. Both files are written with the lone surrogate replaced by `?`, so they stay valid UTF-8; the files of a task that encodes as UTF-8 are unchanged. -- **An input string with a lone surrogate no longer stops `process.run`** — a prompt built from a question or an artifact text with a `\ud800` JSON escape raised `UnicodeEncodeError` before the child started. The string is sent to the child's stdin with the lone surrogate replaced by `?`, so the child gets valid UTF-8; `FakeRunner` and `reply` of the test adapter encode the same way. The bytes of a string that encodes as UTF-8 are unchanged. -- **A query or library id with a lone surrogate no longer stops `Context7Client.get_library_docs` on a cache miss** — `urllib.parse.quote(query)` raised `UnicodeEncodeError` for a `\ud800` or `\udcff` escape in a question, and a library id with one failed when the request was built. Both now go into the URL with the lone surrogate as `%3F`; the library id is percent-encoded for the URL path (letters, digits, `/` and the path characters `%:@!$&'()*+,;=-._~` stay as they are), and the cache key still tells two such queries apart. The URL of a query or id that encodes as UTF-8 and holds only those characters is unchanged. +- **A skill-reach catalogue case whose name holds a lone surrogate no longer stops `build_fixture`** — a `\ud800` or `\udcff` escape in a case `name` raised `UnicodeEncodeError` when the case's `SKILL.md` was written. The file is written with the lone surrogate replaced by `?`, so it stays valid UTF-8 and the run goes on; a JSON-style pair such as `\ud83d\ude00` is joined into its character first; the file of a name that encodes as UTF-8 is unchanged. +- **A terminal-bench task with a lone surrogate in its `task.yaml` no longer stops the Harbor export** — a `\udcff` or `\ud800` escape in the metadata or the instruction raised `UnicodeEncodeError` when `harbor_view` wrote `task.toml` or `instruction.md`. Both files are written with the lone surrogate replaced by `?`, so they stay valid UTF-8; a JSON-style pair such as `\ud83d\ude00` is joined into its character first; the files of a task that encodes as UTF-8 are unchanged. +- **An input string with a lone surrogate no longer stops `process.run`** — a prompt built from a question or an artifact text with a `\ud800` JSON escape raised `UnicodeEncodeError` before the child started. The string is sent to the child's stdin with the lone surrogate replaced by `?`, so the child gets valid UTF-8; a JSON-style pair such as `\ud83d\ude00` is sent as its character; `FakeRunner` and `reply` of the test adapter encode the same way. The bytes of a string that encodes as UTF-8 are unchanged. +- **A system prompt with a lone surrogate no longer stops `llm_cli.call`** — the system prompt file of a CLI call was written with a strict UTF-8 encode, so a `\ud800` escape in an imported question's system message raised `UnicodeEncodeError` before the child started. It is written with the lone surrogate replaced by `?`; a JSON-style pair such as `\ud83d\ude00` is joined into its character. +- **A query or library id with a lone surrogate no longer stops `Context7Client.get_library_docs` on a cache miss** — `urllib.parse.quote(query)` raised `UnicodeEncodeError` for a `\ud800` or `\udcff` escape in a question, and a library id with one failed when the request was built. Both now go into the URL with the lone surrogate as `%3F` (a JSON-style pair such as `\ud83d\ude00` as its UTF-8 percent-encoding); the library id is percent-encoded for the URL path (letters, digits, `/` and the path characters `%:@!$&'()*+,;=-._~` stay as they are), and the cache key still tells two such queries apart. The URL of a query or id that encodes as UTF-8 and holds only those characters is unchanged. diff --git a/dokimi/dokimi/harnesses/harbor_export.py b/dokimi/dokimi/harnesses/harbor_export.py index c20ef36bf..380199fa5 100644 --- a/dokimi/dokimi/harnesses/harbor_export.py +++ b/dokimi/dokimi/harnesses/harbor_export.py @@ -35,6 +35,8 @@ import yaml +from dokimi.process_io import joined_pairs + from . import tbench_v1 from .tasks import FORMAT_HARBOR, FORMAT_TBENCH_V1, TaskSpec @@ -137,17 +139,12 @@ def export(task: TaskSpec, dest: Path) -> Path: if script.is_file(): script.chmod(0o755) instruction = task.instruction if task.instruction.endswith("\n") else task.instruction + "\n" - _write_utf8(dest / "instruction.md", instruction) - _write_utf8(dest / "task.toml", _task_toml(task)) + # A `😀` escape is one character; a lone surrogate is written as `?`, so the file stays valid UTF-8. + (dest / "instruction.md").write_text(joined_pairs(instruction), encoding="utf-8", errors="replace") + (dest / "task.toml").write_text(joined_pairs(_task_toml(task)), encoding="utf-8", errors="replace") return dest -def _write_utf8(path: Path, text: str) -> None: - """``write_text`` in UTF-8, a lone surrogate (a YAML ``\\udcff`` escape) written as ``?``: the file stays valid UTF-8.""" - with path.open("w", encoding="utf-8", errors="replace") as out: - out.write(text) - - def _copy_context(task: TaskSpec, env_dir: Path) -> None: """The task's files into ``environment/``, the build context's at its top, as Harbor's mapper does. diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py index 396e0271d..6f74bfc1c 100644 --- a/dokimi/dokimi/llm_cli.py +++ b/dokimi/dokimi/llm_cli.py @@ -52,6 +52,7 @@ from dokimi import process from dokimi.metrics.usage import UsageRecord +from dokimi.process_io import joined_pairs #: Variables a Claude Code session sets for its own children, plus the API key. Dropped, so #: a call made from inside a session is a session of its own and runs on the CLI's login. @@ -431,7 +432,7 @@ def call(provider: str, *, model: str, prompt: str, system: Optional[str] = None # A private temp dir (mkdtemp, mode 0700) for the one call: the system prompt goes by file. with tempfile.TemporaryDirectory(prefix="dokimi-cli-") as tmp: system_file = Path(tmp) / "system-prompt.txt" - system_file.write_text(system or DEFAULT_SYSTEM, encoding="utf-8") + system_file.write_text(joined_pairs(system or DEFAULT_SYSTEM), encoding="utf-8", errors="replace") # lone surrogate: `?` try: done = (runner or process.run)(cli.build_argv(model, str(system_file)), timeout=timeout, env=_call_env(cli, cwd), cwd=cwd, input=prompt, capture_limit=None) diff --git a/dokimi/dokimi/mcp/context7.py b/dokimi/dokimi/mcp/context7.py index 672640b10..1a3b195f8 100644 --- a/dokimi/dokimi/mcp/context7.py +++ b/dokimi/dokimi/mcp/context7.py @@ -10,6 +10,8 @@ import hashlib import logging +from dokimi.process_io import joined_pairs + from . import MCPClient, MCPConnectionError, MCPLibraryNotFoundError from .utils import read_cache_text, write_cache_text @@ -153,21 +155,23 @@ def get_library_docs( # Website IDs: no slash + (underscore OR 'github_io' pattern) # This avoids misclassifying GitHub repos with underscores (e.g., some_org/repo) is_website = "/" not in library_id and ("_" in library_id or "github_io" in library_id) - path_id = urllib.parse.quote(library_id, safe="/%:@!$&'()*+,;=-._~", errors="replace") # a lone surrogate -> %3F + # A surrogate pair is one character, a lone surrogate is %3F. + path_id = urllib.parse.quote(joined_pairs(library_id), safe="/%:@!$&'()*+,;=-._~", errors="replace") + topic = urllib.parse.quote(joined_pairs(query), errors="replace") if is_website: # Website format: websites/intelpython_github_io_dpnp url = ( f"{self.endpoint}/websites/{path_id}/llms.txt" f"?tokens={max_tokens}" - f"&topic={urllib.parse.quote(query, errors='replace')}" + f"&topic={topic}" ) else: # GitHub repo format: uxlfoundation/oneTBB (or some_org/repo) url = ( f"{self.endpoint}/{path_id}/llms.txt" f"?tokens={max_tokens}" - f"&topic={urllib.parse.quote(query, errors='replace')}" + f"&topic={topic}" ) logger.info(f"Fetching from Context7: {library_id} query={query[:50]}...") diff --git a/dokimi/dokimi/process.py b/dokimi/dokimi/process.py index 27942a7ee..6a43a2190 100644 --- a/dokimi/dokimi/process.py +++ b/dokimi/dokimi/process.py @@ -40,8 +40,8 @@ * **The environment is a stated policy.** There is no default ``env``: a call site passes :meth:`Env.inherit` (the parent's environment plus overrides) or :meth:`Env.only` (an allow-list plus overrides). -* **stdin is closed** (``/dev/null``) unless ``input`` is given, so a child that - reads stdin gets EOF instead of the caller's terminal. +* **stdin is closed** (``/dev/null``) unless ``input`` is given (a child gets EOF, not the caller's terminal). + A ``str`` input is sent as UTF-8, a lone surrogate as ``?``: pass bytes for a name from ``os.fsdecode``. Errors: a command that cannot be started (missing binary, missing ``cwd``, no permission) raises :class:`OSError`; a non-zero exit never raises. After @@ -55,7 +55,7 @@ from __future__ import annotations -from .process_io import PipeReader, PipeReaderOwner, path_text +from .process_io import PipeReader, PipeReaderOwner, path_text, utf8_bytes import dataclasses import os @@ -187,7 +187,7 @@ def run( """Run ``argv`` (no shell) to completion or ``timeout`` seconds; see the module doc.""" args = _check(argv, timeout, capture_limit, term_grace) preexec = _limits_preexec(limits) if limits else None - data = input.encode("utf-8", "replace") if isinstance(input, str) else input + data = utf8_bytes(input) if isinstance(input, str) else input with _Spool(stdout_to) as out_file, _Spool(stderr_to) as err_file: start = time.monotonic() try: @@ -743,7 +743,7 @@ class Call: def reply(stdout: str | bytes = "", stderr: str | bytes = "", returncode: int = 0, *, timed_out: bool = False) -> Completed: """A :class:`FakeRunner` answer (``argv`` and timing are filled in from the call).""" - enc = (lambda s: s.encode("utf-8", "replace") if isinstance(s, str) else s) + enc = (lambda s: utf8_bytes(s) if isinstance(s, str) else s) return Completed((), TIMEOUT_RC if timed_out else returncode, enc(stdout), enc(stderr), timed_out) @@ -774,7 +774,7 @@ def __call__( term_grace: float = 0.0, ) -> Completed: args = _check(argv, timeout, capture_limit, term_grace) - data = input.encode("utf-8", "replace") if isinstance(input, str) else input + data = utf8_bytes(input) if isinstance(input, str) else input path = (lambda p: Path(p) if p is not None else None) call = Call(tuple(args), path(cwd), env.resolve(), timeout, data, capture_stdout, limits, capture_limit, path(stdout_to), path(stderr_to), term_grace) diff --git a/dokimi/dokimi/process_io.py b/dokimi/dokimi/process_io.py index ee812bdfb..f3fdc25df 100644 --- a/dokimi/dokimi/process_io.py +++ b/dokimi/dokimi/process_io.py @@ -13,6 +13,17 @@ _POSIX = sys.platform != "win32" +def joined_pairs(text: str) -> str: + """*text* with each high surrogate followed by a low one (a JSON ``\\ud83d\\ude00`` escape, which PyYAML and ``json`` + load as two code points) joined into its character; a lone surrogate stays, for ``errors="replace"`` to write as ``?``.""" + return text.encode("utf-16-le", "surrogatepass").decode("utf-16-le", "surrogatepass") + + +def utf8_bytes(text: str) -> bytes: + """*text* as UTF-8 with its surrogate pairs joined (:func:`joined_pairs`) and a lone surrogate sent as ``?``.""" + return joined_pairs(text).encode("utf-8", "replace") + + def path_text(raw: bytes) -> str: """A file name git printed, as the path the file system opens: on POSIX a non-UTF-8 byte stays one (``os.fsdecode``), on Windows it becomes U+FFFD. Every reader of a ``-z`` listing decodes a path this way.""" diff --git a/dokimi/dokimi/repos/skill_reach.py b/dokimi/dokimi/repos/skill_reach.py index 3a36f5ed7..d47e5f077 100644 --- a/dokimi/dokimi/repos/skill_reach.py +++ b/dokimi/dokimi/repos/skill_reach.py @@ -71,6 +71,7 @@ import yaml from dokimi import process +from dokimi.process_io import joined_pairs from dokimi.utils import key_bytes, retry_writable from .events import convert @@ -284,12 +285,12 @@ def build_fixture(cases: list[Case], profile: HarnessProfile, roots: dict[str, P continue taken[dest] = case.id tok, body_tok = token(salt, case.id, "description"), token(salt, case.id, "body") - text = skill_text(case, tok, body_tok) + text = joined_pairs(skill_text(case, tok, body_tok)) # a `😀` escape is one character; a lone surrogate is `?` link_kind = None if case.link == "symlink": target = outside / f"case-{n}" / case.dirname # the index, not the id: an id is not a path target.mkdir(parents=True) - (target / "SKILL.md").write_bytes(text.encode("utf-8", "replace")) # a lone surrogate is not UTF-8 + (target / "SKILL.md").write_bytes(text.encode("utf-8", "replace")) dest.parent.mkdir(parents=True, exist_ok=True) try: os.symlink(target, dest, target_is_directory=True) diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 40cbde1a7..5e1ba346a 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -118,6 +118,17 @@ def test_the_call_is_isolated_and_the_prompt_goes_on_stdin(runner, monkeypatch): assert "--system-prompt-file" in second.argv and runner.system_texts[1] == llm_cli.DEFAULT_SYSTEM +@pytest.mark.parametrize(("system", "written"), [ + pytest.param("sys \ud800", "sys ?", id="lone-high"), + pytest.param("sys \udcff", "sys ?", id="lone-low"), + pytest.param("sys \ud83d\ude00", "sys \U0001f600", id="json-pair"), +]) +def test_a_surrogate_in_the_system_prompt_is_written_as_utf8_not_raised(runner, system, written): + runner.answers.append(reply(_ok())) + llm_cli.call("claude-cli", model="sonnet", prompt="q", system=system) + assert runner.system_texts == [written] + + def test_a_long_system_prompt_goes_in_a_file_not_an_argument(runner): # One argument is limited to 128 KiB on Linux; a system prompt of this size must not reach argv. system = "s" * 150_000 diff --git a/dokimi/tests/test_mcp_context7.py b/dokimi/tests/test_mcp_context7.py index 3690bb2ba..6df6f6f12 100644 --- a/dokimi/tests/test_mcp_context7.py +++ b/dokimi/tests/test_mcp_context7.py @@ -134,6 +134,12 @@ def test_get_library_docs_sends_a_lone_surrogate_in_the_library_id_as_a_question Context7Client(cache_dir=tmp_path / "cache").get_library_docs(library_id, "n\ud800") assert mock_get.call_args[0][0].endswith("/" + path + "?tokens=8000&topic=n%3F") + @patch('httpx.get') + def test_get_library_docs_sends_a_json_surrogate_pair_as_one_character(self, mock_get, tmp_path): + mock_get.return_value = Mock(text="Documentation about parallel_for. " * 4) + Context7Client(cache_dir=tmp_path / "cache").get_library_docs("org/n\ud83d\ude00", "q\ud83d\ude00") + assert mock_get.call_args[0][0].endswith("/org/n%F0%9F%98%80/llms.txt?tokens=8000&topic=q%F0%9F%98%80") + @patch('httpx.get') def test_get_library_docs_404(self, mock_get): """Test handling of library not found (404).""" diff --git a/dokimi/tests/test_process.py b/dokimi/tests/test_process.py index 5df3a79bc..58745eb12 100644 --- a/dokimi/tests/test_process.py +++ b/dokimi/tests/test_process.py @@ -773,6 +773,24 @@ def test_reply_encodes_a_lone_surrogate_in_the_output_as_a_question_mark(text): assert reply(text, text).stdout_bytes == b"n?" and reply(text, text).stderr_bytes == b"n?" +_PAIR = "a\ud83d\ude00" # what PyYAML and json load for a JSON-style escape of U+1F600 + + +def test_a_surrogate_pair_in_the_input_reaches_the_child_as_one_character(): + echo = "import sys; sys.stdout.buffer.write(sys.stdin.buffer.read())" + assert _py(echo, input=_PAIR).stdout_bytes == "a\U0001f600".encode() + + +def test_fake_runner_records_a_surrogate_pair_in_the_input_as_one_character(): + fake = FakeRunner() + assert fake(["x"], timeout=1, env=INHERIT, input=_PAIR).ok + assert fake.calls[0].input == "a\U0001f600".encode() + + +def test_reply_encodes_a_surrogate_pair_as_one_character(): + assert reply(_PAIR, _PAIR).stdout_bytes == reply(_PAIR, _PAIR).stderr_bytes == "a\U0001f600".encode() + + def test_input_larger_than_the_pipe_reaches_a_child_that_reads_late(): data = b"x" * 300_000 done = _py("import sys, time; time.sleep(0.6); sys.stdout.buffer.write(sys.stdin.buffer.read())", diff --git a/dokimi/tests/test_skill_reach.py b/dokimi/tests/test_skill_reach.py index 7ff6e7aa6..e47c60b67 100644 --- a/dokimi/tests/test_skill_reach.py +++ b/dokimi/tests/test_skill_reach.py @@ -801,6 +801,14 @@ def test_a_case_name_with_a_lone_surrogate_is_written_as_a_question_mark(tmp_pat assert "\n# n?\n" in text +def test_a_case_name_written_as_a_json_surrogate_pair_is_one_character(tmp_path): + profile = load_profile(str(_profile(tmp_path))) + roots = _roots(tmp_path) + case = Case(id="c", at="project:x", name="n\ud83d\ude00", dirname="d1") + build_fixture([case], profile, roots, tmp_path / "outside", "s") + assert "\n# n\U0001f600\n" in (roots["project"] / "x/d1/SKILL.md").read_bytes().decode("utf-8") + + def test_a_skill_dir_above_the_workspace_is_refused(tmp_path): (tmp_path / ".claude" / "skills").mkdir(parents=True) workspace = tmp_path / "ws" diff --git a/dokimi/tests/test_tbench_v1.py b/dokimi/tests/test_tbench_v1.py index 91041d772..5c65f8b51 100644 --- a/dokimi/tests/test_tbench_v1.py +++ b/dokimi/tests/test_tbench_v1.py @@ -333,6 +333,17 @@ def test_harbor_view_writes_a_lone_surrogate_in_the_instruction_as_a_question_ma assert (view.path / "instruction.md").read_text(encoding="utf-8") == "Create n?.txt\n" +def test_harbor_view_joins_a_json_surrogate_pair_in_the_metadata_and_the_instruction(tmp_path): + import tomllib + + path = _tb1(tmp_path / "tasks", yaml_extra='author_email: "n\\ud83d\\ude00"\n') + _write(path / "task.yaml", 'instruction: "Reply with \\ud83d\\ude00"\nauthor_email: "n\\ud83d\\ude00"\n') + view = harbor_export.harbor_view(load_task(path), cache_root=tmp_path / "cache") + toml = tomllib.loads((view.path / "task.toml").read_bytes().decode("utf-8")) + assert toml["metadata"]["author_email"] == "n\U0001f600" + assert (view.path / "instruction.md").read_text(encoding="utf-8") == "Reply with \U0001f600\n" + + def test_export_puts_a_subdir_context_and_its_named_dockerfile_where_harbor_builds(tmp_path): compose = _COMPOSE.replace(" dockerfile: Dockerfile\n", " context: ./env\n dockerfile: Dockerfile.x\n working_dir: /srv\n")