Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
### Fixed

- **A lone surrogate in an argv entry** — `process.run(['echo', 'n\ud800'])` raised `UnicodeEncodeError` from `subprocess` before the child started, so a prompt passed as an argument (`claude -p <prompt>`) with a JSON or YAML `\ud800` escape stopped the run. The entry now reaches the child with each character `os.fsencode` cannot encode as `?`, and a `😀` escape pair as its character, as on stdin; `U+DC80..U+DCFF` on POSIX stay, since `os.fsdecode` makes them of a non-UTF-8 file name. `Completed.argv` holds the text sent; an entry that needed no change is unchanged.
- **A lone surrogate in a task instruction** — `DockerSolverHarness` wrote `prompt.txt` with a strict UTF-8 encode, so an instruction with a `\ud800` or `\udcff` escape raised `UnicodeEncodeError`. The file now holds the instruction with an escape pair joined and a lone surrogate as `?`, as the Harbor export writes it, and `claude -p` gets that same text.
- **A lone surrogate in a package name** — `catalog check --online` built the registry URL of a package named in a skill with a strict `quote`, so a name with a `\ud800` or `\udcff` escape raised `UnicodeEncodeError`. The lookup now quotes the name with an escape pair joined and a lone surrogate as `%3F`; the registry answers 404 and the usual `package-not-found` warning follows.
5 changes: 4 additions & 1 deletion dokimi/dokimi/external/online_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

from dokimi.external.skill_markdown import outside_fences
from dokimi.external.validate import Finding, SkillReview
from dokimi.process_io import joined_pairs
from dokimi.utils import InputError

if TYPE_CHECKING:
Expand Down Expand Up @@ -574,7 +575,9 @@ class _Outcome:


def _package_finding(client: httpx.Client, eco: str, name: str) -> _Outcome:
url = PYPI_URL.format(name=quote(name)) if eco == "pypi" else NPM_URL.format(name=quote(name, safe="@"))
name_text = joined_pairs(name)
url = (PYPI_URL.format(name=quote(name_text, errors="replace")) if eco == "pypi"
else NPM_URL.format(name=quote(name_text, safe="@", errors="replace")))
try:
status = _status(client, url)
except _InternalHop:
Expand Down
4 changes: 2 additions & 2 deletions dokimi/dokimi/harnesses/docker_solver.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
from pathlib import Path
from typing import Any, Optional

from dokimi import docker_cmd, process
from dokimi import docker_cmd, process, process_io
from dokimi.container_policy import task_container_policy
from dokimi.metrics.usage import UsageRecord

Expand Down Expand Up @@ -554,7 +554,7 @@ def _claude_args(self, task: TaskSpec, treatment: Optional[Treatment],
output_dir: Path) -> list[str]:
"""``claude`` argv after the executable; the prompt is also saved as ``prompt.txt``."""
# The task instruction as the user wrote it, behind the treatment notice only.
prompt = f"{treatment_notice(treatment) if self.skill_notice else ''}{task.instruction}"
prompt = process_io.utf8_bytes(f"{treatment_notice(treatment) if self.skill_notice else ''}{task.instruction}").decode()
(output_dir / "prompt.txt").write_text(prompt, encoding="utf-8")
args = ["-p", prompt, "--output-format", "stream-json", "--verbose", *solver_cli_flags()]
args += ["--plugin-dir", plugin_dir] if plugin_dir else []
Expand Down
10 changes: 5 additions & 5 deletions dokimi/dokimi/process.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,10 @@
:meth:`Env.only` (an allow-list plus overrides).
* **stdin is closed** (``/dev/null``) unless ``input`` is given (a child gets EOF, not the caller's terminal).
A ``str`` input is sent as UTF-8, a lone surrogate as ``?``: pass bytes for a name from ``os.fsdecode``.
An ``argv`` entry likewise (:func:`argv_text`; U+DC80..U+DCFF stay, the bytes of a non-UTF-8 file name), and ``Completed.argv`` holds the text sent.

Errors: a command that cannot be started (missing binary, missing ``cwd``, no
permission) raises :class:`OSError`; a non-zero exit never raises. After
:func:`kill_children`, a call raises :class:`ShuttingDown` instead of starting anything. ``argv`` must
be non-empty and ``timeout`` positive, else :class:`ValueError`.
Errors: a command that cannot be started (missing binary, missing ``cwd``, no permission) raises :class:`OSError`; a non-zero exit never raises.
After :func:`kill_children`, a call raises :class:`ShuttingDown` instead of starting anything. ``argv`` must be non-empty and ``timeout`` positive, else :class:`ValueError`.

Two adapters satisfy :class:`Runner`: :func:`run` (real processes) and
:class:`FakeRunner` (records calls and answers from a handler, for tests).
Expand All @@ -70,6 +69,7 @@
from functools import cached_property
from pathlib import Path
from typing import cast, IO, Protocol
from .process_io import argv_text

TIMEOUT_RC = 124
DRAIN_GRACE_SEC = 2.0
Expand Down Expand Up @@ -422,7 +422,7 @@ def _feed(sink, data: bytes) -> None:

def _check(argv: Sequence[str | os.PathLike[str]], timeout: float, capture_limit: int | None,
term_grace: float = 0.0) -> list[str]:
args = [os.fspath(a) for a in argv]
args = [argv_text(os.fspath(a)) for a in argv]
if not args:
raise ValueError("empty argv")
if not timeout > 0:
Expand Down
19 changes: 19 additions & 0 deletions dokimi/dokimi/process_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,25 @@ def utf8_bytes(text: str) -> bytes:
return joined_pairs(text).encode("utf-8", "replace")


def argv_text(text: str) -> str:
"""An argv entry as the child gets it: pairs joined (:func:`joined_pairs`), and each character ``os.fsencode`` cannot
encode (a lone surrogate) a ``?``. On POSIX U+DC80..U+DCFF stay: ``os.fsdecode`` makes them of a non-UTF-8 file name."""
joined = joined_pairs(text)
try:
os.fsencode(joined)
except UnicodeEncodeError:
return "".join(char if _fs_encodable(char) else "?" for char in joined)
return joined


def _fs_encodable(char: str) -> bool:
try:
os.fsencode(char)
except UnicodeEncodeError:
return False
return True


def path_text(raw: bytes) -> str:
"""A file name git printed, as the path the file system opens: on POSIX a non-UTF-8 byte stays one
(``os.fsdecode``), on Windows it becomes U+FFFD. Every reader of a ``-z`` listing decodes a path this way."""
Expand Down
14 changes: 14 additions & 0 deletions dokimi/tests/test_docker_solver_harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import json
import os
from pathlib import Path
from types import SimpleNamespace

import pytest

Expand Down Expand Up @@ -89,6 +90,19 @@ def _fixture_task(tmp_path: Path) -> Path:
return task


@pytest.mark.parametrize(("instruction", "saved"), [
pytest.param("n\ud800", "n?", id="d800"),
pytest.param("n\udcff", "n?", id="dcff"),
pytest.param("a" + chr(0xD83D) + chr(0xDE00), "a\U0001f600", id="pair"), # a JSON escape pair of U+1F600
])
def test_prompt_txt_holds_the_instruction_with_a_lone_surrogate_as_a_question_mark(tmp_path, instruction, saved):
harness = DockerSolverHarness("docker-claude", skill_notice=False)
task = SimpleNamespace(instruction=instruction)
args = harness._claude_args(task, None, None, None, tmp_path)
assert (tmp_path / "prompt.txt").read_bytes() == saved.encode("utf-8")
assert args[:2] == ["-p", saved]


# ── telemetry parsing ───────────────────────────────────────────────────────


Expand Down
47 changes: 47 additions & 0 deletions dokimi/tests/test_online_package_surrogate.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

"""A package name with a lone surrogate is looked up with a ``?`` instead of raising (#862)."""

from __future__ import annotations

import httpx
import pytest

from dokimi.external import online_checks

PYPI = "https://pypi.org/simple/{name}/"
NPM = "https://registry.npmjs.org/{name}"


@pytest.mark.parametrize("eco", ["pypi", "npm"])
@pytest.mark.parametrize(("name", "quoted"), [
pytest.param("n\ud800", "n%3F", id="d800"),
pytest.param("n\udcff", "n%3F", id="dcff"),
pytest.param("n" + chr(0xD83D) + chr(0xDE00), "n%F0%9F%98%80", id="pair"), # a JSON escape pair of U+1F600
])
def test_a_package_name_with_a_lone_surrogate_is_looked_up_with_a_question_mark(eco, name, quoted):
asked = []

def handle(request: httpx.Request) -> httpx.Response:
asked.append(str(request.url))
return httpx.Response(404)

client = httpx.Client(transport=httpx.MockTransport(handle))
outcome = online_checks._package_finding(client, eco, name)
assert set(asked) == {(PYPI if eco == "pypi" else NPM).format(name=quoted)} # a 404 to HEAD is asked again with GET
assert outcome.finding is not None and outcome.finding.code == "package-not-found"
outcome.finding.message.encode("utf-8")


def test_a_scoped_npm_name_with_a_lone_surrogate_keeps_its_at_sign():
asked = []

def handle(request: httpx.Request) -> httpx.Response:
asked.append(str(request.url))
return httpx.Response(404)

client = httpx.Client(transport=httpx.MockTransport(handle))
outcome = online_checks._package_finding(client, "npm", "@n\ud800/x")
assert set(asked) == {NPM.format(name="@n%3F%2Fx")}
assert outcome.finding is not None and outcome.finding.code == "package-not-found"
44 changes: 44 additions & 0 deletions dokimi/tests/test_process_argv.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

"""An argv entry with a lone surrogate reaches the child as ``?``, a surrogateescape byte as itself (#859)."""

from __future__ import annotations

import sys

import pytest

from dokimi import process
from dokimi.process import Env, FakeRunner

PY = sys.executable
INHERIT = Env.inherit()
_PAIR = "a" + chr(0xD83D) + chr(0xDE00) # what PyYAML and json load for a JSON-style escape of U+1F600
_POSIX_ONLY = pytest.mark.skipif(sys.platform == "win32", reason="Windows encodes argv entries with surrogatepass")
_ARGV_BYTES = "import os, sys; sys.stdout.buffer.write(os.fsencode(sys.argv[1]))"


@_POSIX_ONLY
@pytest.mark.parametrize(("text", "sent"), [
pytest.param("n\ud800", b"n?", id="d800-becomes-question-mark"),
pytest.param("n\udcff", b"n\xff", id="dcff-stays-the-file-name-byte"),
pytest.param("\udcff\ud800", b"\xff?", id="dcff-stays-beside-a-replaced-d800"),
pytest.param(_PAIR, "a\U0001f600".encode(), id="pair-joined"),
])
def test_a_lone_surrogate_in_an_argv_entry_reaches_the_child_as_a_question_mark(text, sent):
done = process.run([PY, "-c", _ARGV_BYTES, text], timeout=30, env=INHERIT)
assert done.returncode == 0 and done.stdout_bytes == sent


@_POSIX_ONLY
def test_completed_argv_holds_the_text_sent():
done = process.run([PY, "-c", "pass", "n\ud800", "plain"], timeout=30, env=INHERIT)
assert done.argv[-2:] == ("n?", "plain")


@_POSIX_ONLY
def test_fake_runner_records_an_argv_entry_as_sent():
fake = FakeRunner()
assert fake(["x", "n\ud800", "n\udcff", _PAIR], timeout=1, env=INHERIT).argv == ("x", "n?", "n\udcff", "a\U0001f600")
assert fake.calls[0].argv == ("x", "n?", "n\udcff", "a\U0001f600")
Loading