diff --git a/dokimi/changelog.d/20261117_170000_lone-surrogate-leftovers.md b/dokimi/changelog.d/20261117_170000_lone-surrogate-leftovers.md new file mode 100644 index 000000000..d7e830d4b --- /dev/null +++ b/dokimi/changelog.d/20261117_170000_lone-surrogate-leftovers.md @@ -0,0 +1,5 @@ +### Fixed + +- **A lone surrogate in an argv entry** — `process.run(['echo', 'n\ud800'])` raised `UnicodeEncodeError` from `subprocess` before the child started, so a prompt passed as an argument (`claude -p `) with a JSON or YAML `\ud800` escape stopped the run. The entry now reaches the child with each character `os.fsencode` cannot encode as `?`, and a `😀` escape pair as its character, as on stdin; `U+DC80..U+DCFF` on POSIX stay, since `os.fsdecode` makes them of a non-UTF-8 file name. `Completed.argv` holds the text sent; an entry that needed no change is unchanged. +- **A lone surrogate in a task instruction** — `DockerSolverHarness` wrote `prompt.txt` with a strict UTF-8 encode, so an instruction with a `\ud800` or `\udcff` escape raised `UnicodeEncodeError`. The file now holds the instruction with an escape pair joined and a lone surrogate as `?`, as the Harbor export writes it, and `claude -p` gets that same text. +- **A lone surrogate in a package name** — `catalog check --online` built the registry URL of a package named in a skill with a strict `quote`, so a name with a `\ud800` or `\udcff` escape raised `UnicodeEncodeError`. The lookup now quotes the name with an escape pair joined and a lone surrogate as `%3F`; the registry answers 404 and the usual `package-not-found` warning follows. diff --git a/dokimi/dokimi/external/online_checks.py b/dokimi/dokimi/external/online_checks.py index d251bef81..c2153fafc 100644 --- a/dokimi/dokimi/external/online_checks.py +++ b/dokimi/dokimi/external/online_checks.py @@ -32,6 +32,7 @@ from dokimi.external.skill_markdown import outside_fences from dokimi.external.validate import Finding, SkillReview +from dokimi.process_io import joined_pairs from dokimi.utils import InputError if TYPE_CHECKING: @@ -574,7 +575,9 @@ class _Outcome: def _package_finding(client: httpx.Client, eco: str, name: str) -> _Outcome: - url = PYPI_URL.format(name=quote(name)) if eco == "pypi" else NPM_URL.format(name=quote(name, safe="@")) + name_text = joined_pairs(name) + url = (PYPI_URL.format(name=quote(name_text, errors="replace")) if eco == "pypi" + else NPM_URL.format(name=quote(name_text, safe="@", errors="replace"))) try: status = _status(client, url) except _InternalHop: diff --git a/dokimi/dokimi/harnesses/docker_solver.py b/dokimi/dokimi/harnesses/docker_solver.py index 0bb95430c..b1e83e5c4 100644 --- a/dokimi/dokimi/harnesses/docker_solver.py +++ b/dokimi/dokimi/harnesses/docker_solver.py @@ -57,7 +57,7 @@ from pathlib import Path from typing import Any, Optional -from dokimi import docker_cmd, process +from dokimi import docker_cmd, process, process_io from dokimi.container_policy import task_container_policy from dokimi.metrics.usage import UsageRecord @@ -554,7 +554,7 @@ def _claude_args(self, task: TaskSpec, treatment: Optional[Treatment], output_dir: Path) -> list[str]: """``claude`` argv after the executable; the prompt is also saved as ``prompt.txt``.""" # The task instruction as the user wrote it, behind the treatment notice only. - prompt = f"{treatment_notice(treatment) if self.skill_notice else ''}{task.instruction}" + prompt = process_io.utf8_bytes(f"{treatment_notice(treatment) if self.skill_notice else ''}{task.instruction}").decode() (output_dir / "prompt.txt").write_text(prompt, encoding="utf-8") args = ["-p", prompt, "--output-format", "stream-json", "--verbose", *solver_cli_flags()] args += ["--plugin-dir", plugin_dir] if plugin_dir else [] diff --git a/dokimi/dokimi/process.py b/dokimi/dokimi/process.py index 6a43a2190..03b54786d 100644 --- a/dokimi/dokimi/process.py +++ b/dokimi/dokimi/process.py @@ -42,11 +42,10 @@ :meth:`Env.only` (an allow-list plus overrides). * **stdin is closed** (``/dev/null``) unless ``input`` is given (a child gets EOF, not the caller's terminal). A ``str`` input is sent as UTF-8, a lone surrogate as ``?``: pass bytes for a name from ``os.fsdecode``. + An ``argv`` entry likewise (:func:`argv_text`; U+DC80..U+DCFF stay, the bytes of a non-UTF-8 file name), and ``Completed.argv`` holds the text sent. -Errors: a command that cannot be started (missing binary, missing ``cwd``, no -permission) raises :class:`OSError`; a non-zero exit never raises. After -:func:`kill_children`, a call raises :class:`ShuttingDown` instead of starting anything. ``argv`` must -be non-empty and ``timeout`` positive, else :class:`ValueError`. +Errors: a command that cannot be started (missing binary, missing ``cwd``, no permission) raises :class:`OSError`; a non-zero exit never raises. +After :func:`kill_children`, a call raises :class:`ShuttingDown` instead of starting anything. ``argv`` must be non-empty and ``timeout`` positive, else :class:`ValueError`. Two adapters satisfy :class:`Runner`: :func:`run` (real processes) and :class:`FakeRunner` (records calls and answers from a handler, for tests). @@ -70,6 +69,7 @@ from functools import cached_property from pathlib import Path from typing import cast, IO, Protocol +from .process_io import argv_text TIMEOUT_RC = 124 DRAIN_GRACE_SEC = 2.0 @@ -422,7 +422,7 @@ def _feed(sink, data: bytes) -> None: def _check(argv: Sequence[str | os.PathLike[str]], timeout: float, capture_limit: int | None, term_grace: float = 0.0) -> list[str]: - args = [os.fspath(a) for a in argv] + args = [argv_text(os.fspath(a)) for a in argv] if not args: raise ValueError("empty argv") if not timeout > 0: diff --git a/dokimi/dokimi/process_io.py b/dokimi/dokimi/process_io.py index f3fdc25df..489e8fcc6 100644 --- a/dokimi/dokimi/process_io.py +++ b/dokimi/dokimi/process_io.py @@ -24,6 +24,25 @@ def utf8_bytes(text: str) -> bytes: return joined_pairs(text).encode("utf-8", "replace") +def argv_text(text: str) -> str: + """An argv entry as the child gets it: pairs joined (:func:`joined_pairs`), and each character ``os.fsencode`` cannot + encode (a lone surrogate) a ``?``. On POSIX U+DC80..U+DCFF stay: ``os.fsdecode`` makes them of a non-UTF-8 file name.""" + joined = joined_pairs(text) + try: + os.fsencode(joined) + except UnicodeEncodeError: + return "".join(char if _fs_encodable(char) else "?" for char in joined) + return joined + + +def _fs_encodable(char: str) -> bool: + try: + os.fsencode(char) + except UnicodeEncodeError: + return False + return True + + def path_text(raw: bytes) -> str: """A file name git printed, as the path the file system opens: on POSIX a non-UTF-8 byte stays one (``os.fsdecode``), on Windows it becomes U+FFFD. Every reader of a ``-z`` listing decodes a path this way.""" diff --git a/dokimi/tests/test_docker_solver_harness.py b/dokimi/tests/test_docker_solver_harness.py index 2e922bb0b..4080d1aa0 100644 --- a/dokimi/tests/test_docker_solver_harness.py +++ b/dokimi/tests/test_docker_solver_harness.py @@ -16,6 +16,7 @@ import json import os from pathlib import Path +from types import SimpleNamespace import pytest @@ -89,6 +90,19 @@ def _fixture_task(tmp_path: Path) -> Path: return task +@pytest.mark.parametrize(("instruction", "saved"), [ + pytest.param("n\ud800", "n?", id="d800"), + pytest.param("n\udcff", "n?", id="dcff"), + pytest.param("a" + chr(0xD83D) + chr(0xDE00), "a\U0001f600", id="pair"), # a JSON escape pair of U+1F600 +]) +def test_prompt_txt_holds_the_instruction_with_a_lone_surrogate_as_a_question_mark(tmp_path, instruction, saved): + harness = DockerSolverHarness("docker-claude", skill_notice=False) + task = SimpleNamespace(instruction=instruction) + args = harness._claude_args(task, None, None, None, tmp_path) + assert (tmp_path / "prompt.txt").read_bytes() == saved.encode("utf-8") + assert args[:2] == ["-p", saved] + + # ── telemetry parsing ─────────────────────────────────────────────────────── diff --git a/dokimi/tests/test_online_package_surrogate.py b/dokimi/tests/test_online_package_surrogate.py new file mode 100644 index 000000000..6db712bc5 --- /dev/null +++ b/dokimi/tests/test_online_package_surrogate.py @@ -0,0 +1,47 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A package name with a lone surrogate is looked up with a ``?`` instead of raising (#862).""" + +from __future__ import annotations + +import httpx +import pytest + +from dokimi.external import online_checks + +PYPI = "https://pypi.org/simple/{name}/" +NPM = "https://registry.npmjs.org/{name}" + + +@pytest.mark.parametrize("eco", ["pypi", "npm"]) +@pytest.mark.parametrize(("name", "quoted"), [ + pytest.param("n\ud800", "n%3F", id="d800"), + pytest.param("n\udcff", "n%3F", id="dcff"), + pytest.param("n" + chr(0xD83D) + chr(0xDE00), "n%F0%9F%98%80", id="pair"), # a JSON escape pair of U+1F600 +]) +def test_a_package_name_with_a_lone_surrogate_is_looked_up_with_a_question_mark(eco, name, quoted): + asked = [] + + def handle(request: httpx.Request) -> httpx.Response: + asked.append(str(request.url)) + return httpx.Response(404) + + client = httpx.Client(transport=httpx.MockTransport(handle)) + outcome = online_checks._package_finding(client, eco, name) + assert set(asked) == {(PYPI if eco == "pypi" else NPM).format(name=quoted)} # a 404 to HEAD is asked again with GET + assert outcome.finding is not None and outcome.finding.code == "package-not-found" + outcome.finding.message.encode("utf-8") + + +def test_a_scoped_npm_name_with_a_lone_surrogate_keeps_its_at_sign(): + asked = [] + + def handle(request: httpx.Request) -> httpx.Response: + asked.append(str(request.url)) + return httpx.Response(404) + + client = httpx.Client(transport=httpx.MockTransport(handle)) + outcome = online_checks._package_finding(client, "npm", "@n\ud800/x") + assert set(asked) == {NPM.format(name="@n%3F%2Fx")} + assert outcome.finding is not None and outcome.finding.code == "package-not-found" diff --git a/dokimi/tests/test_process_argv.py b/dokimi/tests/test_process_argv.py new file mode 100644 index 000000000..0b088c85f --- /dev/null +++ b/dokimi/tests/test_process_argv.py @@ -0,0 +1,44 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""An argv entry with a lone surrogate reaches the child as ``?``, a surrogateescape byte as itself (#859).""" + +from __future__ import annotations + +import sys + +import pytest + +from dokimi import process +from dokimi.process import Env, FakeRunner + +PY = sys.executable +INHERIT = Env.inherit() +_PAIR = "a" + chr(0xD83D) + chr(0xDE00) # what PyYAML and json load for a JSON-style escape of U+1F600 +_POSIX_ONLY = pytest.mark.skipif(sys.platform == "win32", reason="Windows encodes argv entries with surrogatepass") +_ARGV_BYTES = "import os, sys; sys.stdout.buffer.write(os.fsencode(sys.argv[1]))" + + +@_POSIX_ONLY +@pytest.mark.parametrize(("text", "sent"), [ + pytest.param("n\ud800", b"n?", id="d800-becomes-question-mark"), + pytest.param("n\udcff", b"n\xff", id="dcff-stays-the-file-name-byte"), + pytest.param("\udcff\ud800", b"\xff?", id="dcff-stays-beside-a-replaced-d800"), + pytest.param(_PAIR, "a\U0001f600".encode(), id="pair-joined"), +]) +def test_a_lone_surrogate_in_an_argv_entry_reaches_the_child_as_a_question_mark(text, sent): + done = process.run([PY, "-c", _ARGV_BYTES, text], timeout=30, env=INHERIT) + assert done.returncode == 0 and done.stdout_bytes == sent + + +@_POSIX_ONLY +def test_completed_argv_holds_the_text_sent(): + done = process.run([PY, "-c", "pass", "n\ud800", "plain"], timeout=30, env=INHERIT) + assert done.argv[-2:] == ("n?", "plain") + + +@_POSIX_ONLY +def test_fake_runner_records_an_argv_entry_as_sent(): + fake = FakeRunner() + assert fake(["x", "n\ud800", "n\udcff", _PAIR], timeout=1, env=INHERIT).argv == ("x", "n?", "n\udcff", "a\U0001f600") + assert fake.calls[0].argv == ("x", "n?", "n\udcff", "a\U0001f600")