diff --git a/validation/ai_checker/BUILD b/validation/ai_checker/BUILD index 8fa84219..8e4f061c 100644 --- a/validation/ai_checker/BUILD +++ b/validation/ai_checker/BUILD @@ -48,6 +48,11 @@ py_library( name = "copilot_langchain", srcs = [ "src/copilot_adapter/__init__.py", + "src/copilot_adapter/_client_manager.py", + "src/copilot_adapter/_errors.py", + "src/copilot_adapter/_message_converter.py", + "src/copilot_adapter/_preflight.py", + "src/copilot_adapter/_tool_converter.py", "src/copilot_adapter/copilot_langchain.py", ], imports = ["src"], diff --git a/validation/ai_checker/ai_checker.bzl b/validation/ai_checker/ai_checker.bzl index 42631f9f..34434cba 100644 --- a/validation/ai_checker/ai_checker.bzl +++ b/validation/ai_checker/ai_checker.bzl @@ -164,7 +164,7 @@ fi _COMMON_AI_TEST_ATTRS = { "model": attr.string( doc = "AI model name to use for analysis.", - default = "anthropic/claude-sonnet-4-5", + default = "claude-sonnet-4.6", ), "score_threshold": attr.string( doc = "Minimum average score required to pass the test (0-10).", diff --git a/validation/ai_checker/requirements.txt b/validation/ai_checker/requirements.txt index 4018feaa..a3c3e83f 100644 --- a/validation/ai_checker/requirements.txt +++ b/validation/ai_checker/requirements.txt @@ -138,13 +138,13 @@ charset-normalizer==3.4.4 \ --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \ --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 # via requests -github-copilot-sdk==0.1.25 \ - --hash=sha256:13ef99fa8c709c5f80d820672bf36ee9176bc33f0efce6a2b5cbf6d1bb2369e8 \ - --hash=sha256:1a90ee583309ff308fea42f9edec61203645a33ca1d3dc42953628fb8c3eda07 \ - --hash=sha256:5249a63d1ac1e4d325c70c9902e81327b0baca53afa46010f52ac3fd3b5a111b \ - --hash=sha256:7af33d3afbe09a78dfc9d65a843526e47aba15631e90926c42a21a200fab12da \ - --hash=sha256:bc74a3d08ee45313ac02a3f7159c583ec41fc16090ec5f27f88c4b737f03139e \ - --hash=sha256:d32c3fc2c393f70923a645a133607da2e562d078b87437f499100d5bb8c1902f +github-copilot-sdk==0.3.0 \ + --hash=sha256:7e241d9b00ebf8bb4d10b2d6101c75fcef38de04d144d729e07fa48394270ee1 \ + --hash=sha256:93b07c46f60cebbbb003d5bddba22eab886849b1d052b98037b52b6434a5bc07 \ + --hash=sha256:b591546d789f9f8243fb59ca71b08cb0bb1dbec818fbef060c3830c6787de2c8 \ + --hash=sha256:c5712d57a2c6291b805c79e039c55c48d858034b1a37fc8e1653925403a028e9 \ + --hash=sha256:ed8f27989158824c754d7febb473bdf25744a1e6bc07a06f114f7e7deebd2c22 \ + --hash=sha256:f4d98a67b8f038885ddd38bd7033d1ac20c3010f04c72ee0fc74ba4984b69ffa # via -r validation/ai_checker/requirements.txt.in h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ @@ -173,10 +173,14 @@ jsonpointer==3.0.0 \ --hash=sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942 \ --hash=sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef # via jsonpatch -langchain-core==1.2.13 \ - --hash=sha256:b31823e28d3eff1e237096d0bd3bf80c6f9624eb471a9496dbfbd427779f8d82 \ - --hash=sha256:d2773d0d0130a356378db9a858cfeef64c3d64bc03722f1d4d6c40eb46fdf01b +langchain-core==1.4.0 \ + --hash=sha256:1dc341eed802ed9c117c0df3923c991e5e9e226571e5725c194eeb5bd93d1a7f \ + --hash=sha256:23cbbdb46e38ddd1dd5247e6167e96013eae74bea4c5949c550809970a9e565c # via -r validation/ai_checker/requirements.txt.in +langchain-protocol==0.0.15 \ + --hash=sha256:461eb794358f83d5e42635a5797799ffec7b4702314e34edf73ac21e75d3ef79 \ + --hash=sha256:9ab2d11ee73944754f10e037e717098d3a6796f0e58afa9cadda6154e7655ade + # via langchain-core langsmith==0.6.3 \ --hash=sha256:33246769c0bb24e2c17e0c34bb21931084437613cd37faf83bd0978a297b826f \ --hash=sha256:44fdf8084165513e6bede9dda715e7b460b1b3f57ac69f2ca3f03afa911233ec @@ -519,8 +523,8 @@ typing-extensions==4.15.0 \ --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 # via # anyio - # github-copilot-sdk # langchain-core + # langchain-protocol # pydantic # pydantic-core # typing-inspection diff --git a/validation/ai_checker/requirements.txt.in b/validation/ai_checker/requirements.txt.in index 4c9f008a..8d143c56 100644 --- a/validation/ai_checker/requirements.txt.in +++ b/validation/ai_checker/requirements.txt.in @@ -5,5 +5,5 @@ pydantic pyyaml # LangChain + GitHub Copilot SDK -github-copilot-sdk>=0.1.23 -langchain-core>=1.2.9 +github-copilot-sdk>=0.3.0 +langchain-core>=1.4.0 diff --git a/validation/ai_checker/src/ai_checker/constants.py b/validation/ai_checker/src/ai_checker/constants.py index bbc8cb20..83bd1b00 100644 --- a/validation/ai_checker/src/ai_checker/constants.py +++ b/validation/ai_checker/src/ai_checker/constants.py @@ -17,4 +17,4 @@ """ # Default AI model to use for all analysis operations -DEFAULT_MODEL = "anthropic/claude-sonnet-4-5" +DEFAULT_MODEL = "claude-sonnet-4.6" diff --git a/validation/ai_checker/src/copilot_adapter/__init__.py b/validation/ai_checker/src/copilot_adapter/__init__.py index e69de29b..600db7c0 100644 --- a/validation/ai_checker/src/copilot_adapter/__init__.py +++ b/validation/ai_checker/src/copilot_adapter/__init__.py @@ -0,0 +1,18 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Public API for the copilot_adapter package.""" + +from .copilot_langchain import ChatCopilot +from ._errors import CopilotSetupError + +__all__ = ["ChatCopilot", "CopilotSetupError"] diff --git a/validation/ai_checker/src/copilot_adapter/_client_manager.py b/validation/ai_checker/src/copilot_adapter/_client_manager.py new file mode 100644 index 00000000..60e0c757 --- /dev/null +++ b/validation/ai_checker/src/copilot_adapter/_client_manager.py @@ -0,0 +1,237 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Lifecycle management for the Copilot CLI subprocess and SDK client.""" + +from __future__ import annotations + +import logging +from typing import Any, Optional + +from copilot import CopilotClient, SubprocessConfig + +from ._errors import CopilotSetupError +from ._preflight import ( + check_auth_sources, + check_cli_binary, + check_environment, + describe_auth_sources, + resolve_copilot_cli_path, +) + +logger = logging.getLogger(__name__) + + +class CopilotClientManager: + """Owns the lifecycle of a single CopilotClient / CLI subprocess. + + Responsibilities: + - Resolve the CLI binary path (rules_python copy_executables workaround) + - Run pre-flight checks before spawning the process + - Start the subprocess and verify authentication + - Expose the live client for callers + - Shut the process down cleanly on close + + This class is intentionally not a Pydantic model — it holds mutable + runtime state that must not be serialised. + """ + + def __init__(self, copilot_client_options: dict[str, Any] | None = None) -> None: + self._options: dict[str, Any] = dict(copilot_client_options or {}) + self._client: Optional[CopilotClient] = None + self._started: bool = False + + # ------------------------------------------------------------------ + # Public interface + # ------------------------------------------------------------------ + + async def ensure_client(self) -> CopilotClient: + """Return a started, authenticated CopilotClient. + + Creates and starts the client on the first call; subsequent calls + return the cached instance immediately. + + Pre-flight sequence (runs once, before the CLI is spawned): + 1. Resolve the CLI binary path + 2. Validate the binary exists and is executable + 3. Hard-fail if no auth source is available at all + 4. Warn about missing $HOME / HTTPS_PROXY (non-fatal) + 5. Start the CLI subprocess + 6. Verify authentication via get_auth_status() + + Raises: + CopilotSetupError: With a detailed, actionable message for any + failure that prevents the CLI from being used. + """ + if self._client is None: + self._client = self._create_client() + + if not self._started: + await self._start_and_verify() + + return self._client + + async def close(self) -> None: + """Stop the CLI subprocess if it is running.""" + if self._client and self._started: + await self._client.stop() + self._started = False + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _create_client(self) -> CopilotClient: + """Run pre-flight checks and construct (but do not start) the client.""" + opts = dict(self._options) + + # --- Resolve CLI binary path ---------------------------------- + if "cli_path" not in opts and "cli_url" not in opts: + resolved = resolve_copilot_cli_path() + if resolved: + opts["cli_path"] = resolved + logger.info("Resolved Copilot CLI path: %s", resolved) + else: + logger.warning( + "Could not find copilot_cli (copy_executables target). " + "Falling back to bundled binary — this may fail with " + "PermissionError if the executable bit was stripped." + ) + + # --- Check binary -------------------------------------------- + cli_path = opts.get("cli_path") + if cli_path: + problems = check_cli_binary(cli_path) + if problems: + raise CopilotSetupError( + "Copilot CLI binary check failed:\n" + + "\n".join(f" - {p}" for p in problems) + ) + + # --- Hard-fail if no auth source available ------------------- + auth_problems = check_auth_sources() + if auth_problems: + raise CopilotSetupError( + "Copilot authentication pre-flight check failed — " + "the CLI process will not be started:\n" + + "\n".join(f" - {p}" for p in auth_problems) + + "\n\n" + + describe_auth_sources() + ) + + # --- Warn about non-fatal env issues ------------------------- + env_problems = check_environment() + if env_problems: + logger.warning( + "Environment issues detected:\n%s\n%s", + "\n".join(f" - {p}" for p in env_problems), + describe_auth_sources(), + ) + + logger.info("Starting CopilotClient...\n%s", describe_auth_sources()) + _subprocess_fields = frozenset( + { + "cli_path", + "cli_args", + "cwd", + "use_stdio", + "port", + "log_level", + "env", + "github_token", + "use_logged_in_user", + "telemetry", + "session_fs", + "session_idle_timeout_seconds", + } + ) + subprocess_kwargs = {k: v for k, v in opts.items() if k in _subprocess_fields} + return CopilotClient(SubprocessConfig(**subprocess_kwargs)) + + async def _start_and_verify(self) -> None: + """Start the CLI subprocess and verify authentication.""" + assert self._client is not None + + try: + await self._client.start() + except PermissionError as exc: + raise CopilotSetupError( + f"PermissionError starting Copilot CLI: {exc}\n" + " The CLI binary is not executable. Make sure\n" + " pip.whl_mods / copy_executables is configured in MODULE.bazel\n" + " to create an executable copy of copilot/bin/copilot." + ) from exc + except RuntimeError as exc: + if "timeout" in str(exc).lower() or "Timeout" in str(exc): + raise CopilotSetupError( + f"Timeout starting Copilot CLI server: {exc}\n" + " The CLI started but did not become ready in time.\n" + " This usually means the CLI cannot authenticate.\n\n" + + describe_auth_sources() + + "\n\n" + " Possible fixes:\n" + " 1. Run 'copilot' in a terminal and sign in interactively.\n" + " 2. Set COPILOT_GITHUB_TOKEN (or GH_TOKEN / GITHUB_TOKEN)\n" + " and pass it via --action_env=COPILOT_GITHUB_TOKEN.\n" + " 3. Ensure HOME is available in the action environment\n" + " (use_default_shell_env = True in the Bazel rule).\n" + " See: https://github.com/github/copilot-sdk/blob/main/docs/auth/index.md" + ) from exc + raise + except Exception as exc: + raise CopilotSetupError( + f"Failed to start CopilotClient: {type(exc).__name__}: {exc}\n\n" + + describe_auth_sources() + ) from exc + + self._started = True + await self._verify_auth() + + async def _verify_auth(self) -> None: + """Log the result of get_auth_status() as a diagnostic; never hard-fail. + + Rationale: get_auth_status() can return isAuthenticated=False even when + the CLI is fully functional — for example: + - The auth state is resolved lazily on the first real request. + - GitHub Enterprise hosts (*.ghe.com) may not be reflected immediately. + - There is a brief window after start() where the status is not yet set. + + A false-positive hard-fail here would block valid requests. The actual + LLM call (send_and_wait) will fail with a clear error if auth is truly + broken, so we demote this check to a warning-only diagnostic. + """ + assert self._client is not None + try: + auth_status = await self._client.get_auth_status() + # The SDK uses camelCase on some versions, snake_case on others. + is_auth = getattr(auth_status, "isAuthenticated", None) or getattr( + auth_status, "is_authenticated", None + ) + if is_auth: + user = getattr(auth_status, "login", "unknown") + logger.info("Copilot authenticated as: %s", user) + else: + # Log as a warning only — do not raise. The CLI may still work. + logger.warning( + "get_auth_status() reports isAuthenticated=False — " + "continuing anyway; auth may be resolved on first request.\n" + " Auth status: %s\n%s", + auth_status, + describe_auth_sources(), + ) + except Exception as exc: + # get_auth_status itself failed — log but do not block. + logger.warning( + "Could not verify auth status (non-fatal): %s: %s", + type(exc).__name__, + exc, + ) diff --git a/validation/ai_checker/src/copilot_adapter/_errors.py b/validation/ai_checker/src/copilot_adapter/_errors.py new file mode 100644 index 00000000..2ed12dae --- /dev/null +++ b/validation/ai_checker/src/copilot_adapter/_errors.py @@ -0,0 +1,26 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Shared error types and constants for the Copilot adapter.""" + +from __future__ import annotations + +# Auth-related environment variables checked by the Copilot CLI (priority order) +AUTH_ENV_VARS: list[str] = [ + "COPILOT_GITHUB_TOKEN", # Recommended for explicit Copilot usage + "GH_TOKEN", # GitHub CLI compatible + "GITHUB_TOKEN", # GitHub Actions compatible +] + + +class CopilotSetupError(RuntimeError): + """Raised when the Copilot SDK environment is not correctly configured.""" diff --git a/validation/ai_checker/src/copilot_adapter/_message_converter.py b/validation/ai_checker/src/copilot_adapter/_message_converter.py new file mode 100644 index 00000000..8a1d4d3b --- /dev/null +++ b/validation/ai_checker/src/copilot_adapter/_message_converter.py @@ -0,0 +1,68 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Conversion between LangChain message types and Copilot SDK prompt format.""" + +from __future__ import annotations + +import json +from typing import Optional + +from langchain_core.messages import ( + AIMessage, + BaseMessage, + HumanMessage, + SystemMessage, + ToolMessage, +) + + +def messages_to_prompt(messages: list[BaseMessage]) -> str: + """Convert a list of LangChain messages into a single prompt string. + + The Copilot SDK accepts a plain text prompt rather than a structured + message array. We serialise the conversation into a tagged format so + the model can distinguish roles. + """ + parts: list[str] = [] + for msg in messages: + content = ( + msg.content if isinstance(msg.content, str) else json.dumps(msg.content) + ) + + if isinstance(msg, SystemMessage): + parts.append(f"[system]\n{content}") + elif isinstance(msg, HumanMessage): + parts.append(f"[user]\n{content}") + elif isinstance(msg, AIMessage): + text_parts = [f"[assistant]\n{content}"] if content else ["[assistant]"] + if msg.tool_calls: + for tc in msg.tool_calls: + text_parts.append( + f"[tool_call id={tc['id']} name={tc['name']}]\n" + f"{json.dumps(tc['args'])}" + ) + parts.append("\n".join(text_parts)) + elif isinstance(msg, ToolMessage): + parts.append(f"[tool_result id={msg.tool_call_id}]\n{content}") + else: + parts.append(f"[{msg.type}]\n{content}") + + return "\n\n".join(parts) + + +def extract_system_message(messages: list[BaseMessage]) -> Optional[str]: + """Return the content of the first message if it is a SystemMessage.""" + if messages and isinstance(messages[0], SystemMessage): + content = messages[0].content + return content if isinstance(content, str) else json.dumps(content) + return None diff --git a/validation/ai_checker/src/copilot_adapter/_preflight.py b/validation/ai_checker/src/copilot_adapter/_preflight.py new file mode 100644 index 00000000..cb15e721 --- /dev/null +++ b/validation/ai_checker/src/copilot_adapter/_preflight.py @@ -0,0 +1,188 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Pre-flight environment and authentication checks for the Copilot CLI.""" + +from __future__ import annotations + +import logging +import os +import stat +from pathlib import Path +from typing import Optional + +from ._errors import AUTH_ENV_VARS, CopilotSetupError + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# CLI binary +# --------------------------------------------------------------------------- + + +def resolve_copilot_cli_path() -> Optional[str]: + """Find the executable copy of the copilot CLI created by copy_executables. + + rules_python strips the executable bit from binaries inside wheels. + The pip.whl_mods / copy_executables mechanism creates an executable + copy called ``copilot_cli`` next to the package. We walk up from + ``copilot.__file__`` until we find it. + + IMPORTANT: we must NOT resolve symlinks (Path.resolve()) because in + the Bazel runfiles tree the symlinks point back to the source repo + where the genrule output does not exist. The raw __file__ path + stays inside the execution root where the copy IS present. + """ + import copilot as _copilot_pkg + + pkg_file = Path(_copilot_pkg.__file__) # .../site-packages/copilot/__init__.py + current = pkg_file.parent + for _ in range(10): + candidate = current / "copilot_cli" + if candidate.exists(): + return str(candidate) + current = current.parent + return None + + +def check_cli_binary(cli_path: str) -> list[str]: + """Validate that the CLI binary exists and is executable. + + Returns a list of problem descriptions (empty = all good). + """ + problems: list[str] = [] + p = Path(cli_path) + if not p.exists(): + problems.append(f"Copilot CLI binary not found at: {cli_path}") + return problems + if not p.is_file(): + problems.append(f"Copilot CLI path is not a file: {cli_path}") + return problems + mode = p.stat().st_mode + if not (mode & stat.S_IXUSR): + problems.append( + f"Copilot CLI binary is NOT executable (mode {oct(mode)}): {cli_path}\n" + " Hint: rules_python strips +x from wheel binaries. Make sure\n" + " pip.whl_mods / copy_executables is configured in MODULE.bazel." + ) + return problems + + +# --------------------------------------------------------------------------- +# Authentication sources +# --------------------------------------------------------------------------- + + +def check_auth_sources() -> list[str]: + """Check that at least one authentication source is available. + + Returns a list of problem descriptions (empty = at least one source present). + Hard-fails only when both token env vars AND $HOME are completely absent — + the only case where the CLI is guaranteed to time-out authenticating. + This check runs *before* the CLI is spawned to avoid wasting time. + """ + for var in AUTH_ENV_VARS: + if os.environ.get(var): + return [] # A token is set — auth is possible + + if os.environ.get("HOME"): + return [] # No token, but $HOME is set — stored OAuth may work + + return [ + "No authentication source is available for the Copilot CLI.\n" + " None of the token environment variables are set and $HOME is missing.\n" + " The Copilot CLI cannot authenticate without at least one of:\n" + " - $COPILOT_GITHUB_TOKEN (recommended)\n" + " - $GH_TOKEN\n" + " - $GITHUB_TOKEN\n" + " - $HOME set so the CLI can read stored OAuth credentials\n" + " Fix: add --action_env=COPILOT_GITHUB_TOKEN to .bazelrc.ai_checker\n" + " and export COPILOT_GITHUB_TOKEN= in your shell.\n" + " See: https://github.com/github/copilot-sdk/blob/main/docs/auth/index.md" + ] + + +def describe_auth_sources() -> str: + """Return a human-readable summary of all available auth sources.""" + lines = ["Authentication sources detected:"] + found_any = False + + for var in AUTH_ENV_VARS: + val = os.environ.get(var) + if val: + masked = val[:4] + "..." + val[-4:] if len(val) > 10 else "****" + lines.append(f" [OK] ${var} = {masked}") + found_any = True + else: + lines.append(f" [ ] ${var} — not set") + + home = os.environ.get("HOME", "") + if home: + lines.append(f" [OK] $HOME = {home} (CLI can search system keychain)") + else: + lines.append( + " [ ] $HOME — not set (CLI cannot find stored OAuth credentials)" + ) + + if not found_any and not home: + lines.append("") + lines.append(" ** No authentication source available! **") + lines.append( + " Fix: set COPILOT_GITHUB_TOKEN, or ensure HOME is passed to the action." + ) + lines.append( + " See: https://github.com/github/copilot-sdk/blob/main/docs/auth/index.md" + ) + + lines.append("") + proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("https_proxy") + if proxy: + lines.append(f" [OK] HTTPS_PROXY = {proxy}") + else: + lines.append( + " [ ] HTTPS_PROXY — not set (may cause 'fetch failed' behind a proxy)" + ) + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Runtime environment +# --------------------------------------------------------------------------- + + +def check_environment() -> list[str]: + """Check that the runtime environment has what the Copilot CLI needs. + + Returns a list of problem descriptions (empty = all good). + These are warnings, not hard failures — a token env var may still work + even when $HOME or HTTPS_PROXY are missing. + """ + problems: list[str] = [] + + if not os.environ.get("HOME"): + problems.append( + "HOME environment variable is not set.\n" + " The Copilot CLI needs HOME to locate stored OAuth credentials.\n" + " Ensure .bazelrc.ai_checker contains: build --action_env=HOME" + ) + + if not os.environ.get("HTTPS_PROXY") and not os.environ.get("https_proxy"): + problems.append( + "HTTPS_PROXY / https_proxy environment variable is not set.\n" + " If you are behind a corporate proxy the Copilot CLI cannot\n" + " reach api.github.com and will fail with 'TypeError: fetch failed'.\n" + " Ensure .bazelrc.ai_checker contains: build --action_env=HTTPS_PROXY" + ) + + return problems diff --git a/validation/ai_checker/src/copilot_adapter/_tool_converter.py b/validation/ai_checker/src/copilot_adapter/_tool_converter.py new file mode 100644 index 00000000..026b1a4a --- /dev/null +++ b/validation/ai_checker/src/copilot_adapter/_tool_converter.py @@ -0,0 +1,97 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Conversion between LangChain tool specs and Copilot SDK Tool objects.""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from typing import Any, Callable + +from copilot.tools import Tool as CopilotTool, ToolInvocation, ToolResult +from langchain_core.tools import BaseTool +from langchain_core.utils.function_calling import convert_to_openai_tool + + +def convert_tools_to_openai_format( + tools: Sequence[dict[str, Any] | type | Callable | BaseTool], +) -> list[dict[str, Any]]: + """Convert LangChain tool specs to OpenAI-format tool definitions.""" + result = [] + for tool in tools: + if isinstance(tool, dict): + result.append(tool) + else: + result.append(convert_to_openai_tool(tool)) + return result + + +def build_copilot_tools( + openai_tools: list[dict[str, Any]], +) -> list[CopilotTool]: + """Convert OpenAI-format tool dicts into Copilot SDK Tool objects. + + The handler is a no-op because we never let the Copilot agent + autonomously execute tools — we only need the definitions so the + model can emit tool_calls in its response. + """ + copilot_tools = [] + for t in openai_tools: + fn = t.get("function", t) + name = fn["name"] + description = fn.get("description", "") + parameters = fn.get("parameters") + + def _make_noop_handler(tool_name: str): + async def _noop_handler(invocation: ToolInvocation) -> ToolResult: + return ToolResult( + text_result_for_llm="Tool execution is managed by LangChain.", + result_type="success", + ) + + return _noop_handler + + copilot_tools.append( + CopilotTool( + name=name, + description=description, + handler=_make_noop_handler(name), + parameters=parameters, + ) + ) + return copilot_tools + + +def deep_decode_json_strings(obj: Any) -> Any: + """Recursively decode values that are JSON-encoded strings. + + Some LLMs (e.g. Claude via the Copilot SDK) double-encode nested + lists or objects as JSON strings inside the outer tool-call arguments + dict. This function walks the structure and replaces any string value + that successfully parses as a JSON array or object with the decoded + Python value, leaving plain strings untouched. + """ + if isinstance(obj, dict): + return {k: deep_decode_json_strings(v) for k, v in obj.items()} + if isinstance(obj, list): + return [deep_decode_json_strings(v) for v in obj] + if isinstance(obj, str): + stripped = obj.strip() + if stripped and stripped[0] in ("{", "["): + try: + decoded = json.loads(stripped) + if isinstance(decoded, (dict, list)): + return deep_decode_json_strings(decoded) + except (json.JSONDecodeError, ValueError): + pass + return obj diff --git a/validation/ai_checker/src/copilot_adapter/architecture.md b/validation/ai_checker/src/copilot_adapter/architecture.md new file mode 100644 index 00000000..ebd3bea3 --- /dev/null +++ b/validation/ai_checker/src/copilot_adapter/architecture.md @@ -0,0 +1,206 @@ + + +# copilot_adapter — Architecture + +## Overview + +`copilot_adapter` is a [LangChain](https://python.langchain.com/) integration layer that +bridges the **GitHub Copilot SDK** (`github-copilot-sdk`) to the LangChain ecosystem. +It exposes a single public class, `ChatCopilot`, which is a drop-in replacement for any +other LangChain `BaseChatModel` (e.g. `ChatOpenAI`). + +The adapter translates between two different worlds: + +| LangChain side | Copilot SDK side | +|---|---| +| `list[BaseMessage]` (typed message objects) | A single plain-text prompt string with role tags | +| `SystemMessage` | `SessionConfig.system_message` (injected once per session) | +| `BaseTool` / OpenAI tool dict | `copilot.tools.Tool` with async handler | +| Pydantic `BaseModel` schema | JSON schema embedded in the system prompt | +| `AIMessage` with `tool_calls` | `ExternalToolRequestedData` broadcast events | + +--- + +## Component Diagram + + + +--- + +## Module Responsibilities + +### `copilot_langchain.py` — `ChatCopilot` + +The central public class. Inherits from LangChain's `BaseChatModel` so it can be used +anywhere a standard LangChain model is expected. + +| Method | Role | +|---|---| +| `with_structured_output(schema)` | Returns a composed `Runnable` chain for structured JSON output (see below) | +| `bind_tools(tools)` | Returns a new `ChatCopilot` instance with the given tools pre-registered | +| `_agenerate(messages)` | **Async core** — creates a Copilot session, sends the prompt, collects the response | +| `_generate(messages)` | Sync bridge: runs `_agenerate` in a thread-pool executor if an event loop is already running | + +### `_client_manager.py` — `CopilotClientManager` + +Owns the lifecycle of the single `CopilotClient` / CLI subprocess. The same subprocess +is reused across calls (cached in `_client`). + +Pre-flight sequence executed once before the first request: +1. Resolve the `copilot_cli` binary path (Bazel `copy_executables` workaround) +2. Verify the binary exists and is executable +3. Hard-fail if no auth source is found (`COPILOT_GITHUB_TOKEN`, `GH_TOKEN`, `GITHUB_TOKEN`, or `~/.copilot/config.json`) +4. Warn (non-fatal) about missing `$HOME` or `HTTPS_PROXY` +5. Spawn the subprocess and authenticate via `get_auth_status()` + +### `_message_converter.py` + +Converts the LangChain message list into the formats the Copilot SDK accepts. + +- **`extract_system_message(messages)`** — Pulls out the first `SystemMessage` and + returns its string content. This is passed as `SessionConfig.system_message` so the + Copilot CLI handles it as a true system prompt (not just prepended text). + +- **`messages_to_prompt(messages)`** — Serialises all remaining messages into a single + tagged plain-text string: + ``` + [user] + What is 2 + 2? + + [assistant] + 4 + [tool_call id=abc name=add] + {"a": 2, "b": 2} + + [tool_result id=abc] + 4 + ``` + +### `_tool_converter.py` + +Converts tool definitions between three representations: + +``` +LangChain BaseTool / Callable / type + │ + ▼ convert_to_openai_tool() +OpenAI function dict {"type": "function", "function": {"name": ..., "parameters": ...}} + │ + ▼ build_copilot_tools() +copilot.tools.Tool (with async no-op handler) +``` + +The handler is always a no-op because when using `with_structured_output` tool execution +is bypassed entirely; when using `bind_tools` directly, tool execution is managed by the +LangChain agent loop, not by the Copilot CLI. + +`deep_decode_json_strings` recursively unwraps values that the model has +double-encoded as JSON strings inside the outer tool-call arguments dict. + +### `_preflight.py` + +Stateless helper functions called by `CopilotClientManager` before startup: + +- `resolve_copilot_cli_path()` — walks up from `copilot.__file__` to find the + `copilot_cli` binary that Bazel's `copy_executables` placed next to the package. +- `check_cli_binary(path)` — checks existence and executable bit. +- `check_auth_sources()` — scans env vars and `~/.copilot/config.json`; hard-fails if + none are present. +- `check_environment()` — warns about missing `HOME`, `HTTPS_PROXY`, or proxy vars. +- `describe_auth_sources()` — formats a human-readable description for error messages. + +### `_errors.py` + +- `CopilotSetupError` — a `RuntimeError` subclass raised for any configuration or + startup failure. Carries an actionable message for the user. +- `AUTH_ENV_VARS` — ordered list of accepted auth environment variables. + +--- + +## Data Flow: `with_structured_output` + +The primary usage path (used by `ai_checker_core`). `with_structured_output(schema)` +returns a composed chain: `_inject | ChatCopilot | _parse`. + +1. **`_inject`** — appends the Pydantic schema (serialised to JSON) to the + `SystemMessage`, instructing the model to respond with only a matching JSON object. +2. **`ChatCopilot._agenerate`** — extracts the system message into + `SessionConfig.system_message`, serialises remaining messages into a role-tagged + plain-text prompt, creates a Copilot session, and calls `send_and_wait`. Returns an + `AIMessage` whose `.content` is the model's raw text reply. +3. **`_parse`** — strips any markdown fences, extracts the outermost `{…}` substring, + parses it with `json.loads`, and validates it with `schema.model_validate`. On any + failure it raises a `ValueError` containing the full raw LLM output and the specific + error (JSON byte position or Pydantic field detail). + +> **Why JSON-in-prompt instead of tool calling?** +> The Copilot CLI uses a Claude model with reasoning enabled. That model ignores +> `tool_choice="any"` and always responds in plain text. Embedding the JSON schema +> directly in the system prompt is reliably followed. + +--- + +## Data Flow: `bind_tools` (direct tool use) + +Used when the LangChain agent loop — not the adapter — executes tools. + +1. `bind_tools(tools)` converts each tool to OpenAI format, then to a `copilot.tools.Tool` + with a no-op handler, and registers them in `SessionConfig`. +2. Tool calls arrive as `ExternalToolRequestedData` broadcast events (SDK protocol v3) + or in `AssistantMessageData.tool_requests` (legacy fallback). +3. Both sources are merged, deduplicated by `tool_call_id`, and returned as + `AIMessage.tool_calls` for the LangChain agent loop to dispatch. + +--- + +## Authentication + +The Copilot CLI requires a valid GitHub OAuth token to contact the Copilot API. +`_preflight.py` checks the following sources in priority order during pre-flight: + +| Priority | Source | Notes | +|---|---|---| +| 1 | `COPILOT_GITHUB_TOKEN` env var | Recommended for explicit Copilot usage | +| 2 | `GH_TOKEN` env var | GitHub CLI compatible | +| 3 | `GITHUB_TOKEN` env var | GitHub Actions compatible | +| 4 | `~/.copilot/config.json` | Written by `gh copilot` interactive login | + +If **none** of these sources is present, `CopilotClientManager` raises a +`CopilotSetupError` before spawning the subprocess (hard fail — no point starting +the CLI without credentials). + +If at least one source exists, the CLI is started and `get_auth_status()` is called to +confirm the token is accepted by the GitHub API. A failed status also raises +`CopilotSetupError` with an actionable message. + +### Bazel / headless environments + +In the CI/Bazel setup the `--config=copilot` bazelrc flag forwards `HOME` and the +proxy environment variables (`HTTPS_PROXY`, `HTTP_PROXY`, `NO_PROXY`) to the action +sandbox via `--action_env`. Without `HOME` the CLI cannot locate `~/.copilot/config.json`, +so a token env var must be set instead. `_preflight.py` emits a warning (non-fatal) +when `HOME` is unset. + +--- + +## Error Handling Summary + +| Failure point | Exception type | What is logged | +|---|---|---| +| CLI binary missing / not executable | `CopilotSetupError` | Path checked, alternatives suggested | +| No auth source found | `CopilotSetupError` | Lists all env vars and config file path | +| Copilot SDK startup error | `CopilotSetupError` | Wraps original exception + auth description | +| Model returns no JSON object | `ValueError` | Full LLM output | +| Model returns malformed JSON | `ValueError` | `json.JSONDecodeError` position + full LLM output | +| Model returns wrong JSON structure | `ValueError` | Pydantic field-level `ValidationError` + full LLM output | diff --git a/validation/ai_checker/src/copilot_adapter/component_diagram.puml b/validation/ai_checker/src/copilot_adapter/component_diagram.puml new file mode 100644 index 00000000..58c5d70c --- /dev/null +++ b/validation/ai_checker/src/copilot_adapter/component_diagram.puml @@ -0,0 +1,61 @@ +' ******************************************************************************* +' Copyright (c) 2026 Contributors to the Eclipse Foundation +' +' See the NOTICE file(s) distributed with this work for additional +' information regarding copyright ownership. +' +' This program and the accompanying materials are made available under the +' terms of the Apache License Version 2.0 which is available at +' https://www.apache.org/licenses/LICENSE-2.0 +' +' SPDX-License-Identifier: Apache-2.0 +' ******************************************************************************* + +@startuml copilot_adapter +!theme plain +skinparam componentStyle rectangle +skinparam defaultFontName Monospaced +skinparam ArrowFontSize 11 + +package "langchain_core" <> { + component BaseChatModel + component "Messages / Tools" as LCTypes +} + +package "github-copilot-sdk" <> { + component CopilotClient + component "Session API" as SessionAPI + component "Tool types" as CopilotTools +} + +package "pydantic" <> { + component BaseModel +} + +package "copilot_adapter" { + component ChatCopilot + component CopilotClientManager as ClientMgr + component _message_converter as MsgConv + component _tool_converter as ToolConv + component _preflight as Preflight + component _errors as Errors +} + +actor "Caller" as Caller + +Caller --> ChatCopilot : ainvoke(messages) +ChatCopilot -up-> BaseChatModel +ChatCopilot --> ClientMgr : ensure_client() +ChatCopilot --> MsgConv : convert messages +ChatCopilot --> ToolConv : convert tools +ChatCopilot --> SessionAPI : create_session / send_and_wait +ChatCopilot --> BaseModel : inject schema / validate + +ClientMgr --> CopilotClient +ClientMgr --> Preflight : pre-flight checks +ClientMgr --> Errors : CopilotSetupError + +ToolConv --> CopilotTools +ToolConv --> LCTypes +MsgConv --> LCTypes +@enduml diff --git a/validation/ai_checker/src/copilot_adapter/copilot_langchain.py b/validation/ai_checker/src/copilot_adapter/copilot_langchain.py index 541ecab8..3d419329 100644 --- a/validation/ai_checker/src/copilot_adapter/copilot_langchain.py +++ b/validation/ai_checker/src/copilot_adapter/copilot_langchain.py @@ -24,298 +24,36 @@ from __future__ import annotations import asyncio +import concurrent.futures import json import logging -import os -import stat -import uuid from collections.abc import Sequence -from pathlib import Path from typing import Any, Callable, Optional -from copilot import CopilotClient -from copilot.generated.session_events import SessionEvent, SessionEventType -from copilot.types import SessionConfig, Tool as CopilotTool, ToolInvocation, ToolResult - -logger = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# Auth-related environment variables checked by the Copilot CLI (priority order) -# --------------------------------------------------------------------------- -_AUTH_ENV_VARS = [ - "COPILOT_GITHUB_TOKEN", # Recommended for explicit Copilot usage - "GH_TOKEN", # GitHub CLI compatible - "GITHUB_TOKEN", # GitHub Actions compatible -] - - -class CopilotSetupError(RuntimeError): - """Raised when the Copilot SDK environment is not correctly configured.""" - - -def _resolve_copilot_cli_path() -> Optional[str]: - """Find the executable copy of the copilot CLI created by copy_executables. - - rules_python strips the executable bit from binaries inside wheels. - The pip.whl_mods / copy_executables mechanism creates an executable - copy called ``copilot_cli`` next to the package. We walk up from - ``copilot.__file__`` until we find it. - - IMPORTANT: we must NOT resolve symlinks (Path.resolve()) because in - the Bazel runfiles tree the symlinks point back to the source repo - where the genrule output does not exist. The raw __file__ path - stays inside the execution root where the copy IS present. - """ - import copilot as _copilot_pkg - - pkg_file = Path(_copilot_pkg.__file__) # .../site-packages/copilot/__init__.py - # Walk up: copilot/ -> site-packages/ -> lib/ -> ... -> repo root - current = pkg_file.parent - for _ in range(10): - candidate = current / "copilot_cli" - if candidate.exists(): - return str(candidate) - current = current.parent - return None - - -def _check_cli_binary(cli_path: str) -> list[str]: - """Validate that the CLI binary exists and is executable. - - Returns a list of problem descriptions (empty = all good). - """ - problems: list[str] = [] - p = Path(cli_path) - if not p.exists(): - problems.append(f"Copilot CLI binary not found at: {cli_path}") - return problems - if not p.is_file(): - problems.append(f"Copilot CLI path is not a file: {cli_path}") - return problems - mode = p.stat().st_mode - if not (mode & stat.S_IXUSR): - problems.append( - f"Copilot CLI binary is NOT executable (mode {oct(mode)}): {cli_path}\n" - " Hint: rules_python strips +x from wheel binaries. Make sure\n" - " pip.whl_mods / copy_executables is configured in MODULE.bazel." - ) - return problems - - -def _check_environment() -> list[str]: - """Check that the runtime environment has what the Copilot CLI needs. - - Returns a list of problem descriptions (empty = all good). - """ - problems: list[str] = [] - - if not os.environ.get("HOME"): - problems.append( - "HOME environment variable is not set.\n" - " The Copilot CLI needs HOME to locate stored OAuth credentials.\n" - " Ensure .bazelrc.ai_checker contains: build --action_env=HOME" - ) - - # The Copilot CLI binary (Node.js) uses fetch() to reach api.github.com. - # Behind a corporate proxy it needs HTTPS_PROXY. - if not os.environ.get("HTTPS_PROXY") and not os.environ.get("https_proxy"): - problems.append( - "HTTPS_PROXY / https_proxy environment variable is not set.\n" - " If you are behind a corporate proxy the Copilot CLI cannot\n" - " reach api.github.com and will fail with 'TypeError: fetch failed'.\n" - " Ensure .bazelrc.ai_checker contains: build --action_env=HTTPS_PROXY" - ) - - return problems - - -def _describe_auth_sources() -> str: - """Return a human-readable summary of available auth sources.""" - lines = ["Authentication sources detected:"] - found_any = False - - for var in _AUTH_ENV_VARS: - val = os.environ.get(var) - if val: - # Mask the token for security - masked = val[:4] + "..." + val[-4:] if len(val) > 10 else "****" - lines.append(f" [OK] ${var} = {masked}") - found_any = True - else: - lines.append(f" [ ] ${var} — not set") - - home = os.environ.get("HOME", "") - if home: - lines.append(f" [OK] $HOME = {home} (CLI can search system keychain)") - else: - lines.append( - " [ ] $HOME — not set (CLI cannot find stored OAuth credentials)" - ) - - if not found_any and not home: - lines.append("") - lines.append(" ** No authentication source available! **") - lines.append( - " Fix: set COPILOT_GITHUB_TOKEN, or ensure HOME is passed to the action." - ) - lines.append( - " See: https://github.com/github/copilot-sdk/blob/main/docs/auth/index.md" - ) - - # Network / proxy info - lines.append("") - proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("https_proxy") - if proxy: - lines.append(f" [OK] HTTPS_PROXY = {proxy}") - else: - lines.append( - " [ ] HTTPS_PROXY — not set (may cause 'fetch failed' behind a proxy)" - ) - - return "\n".join(lines) - +from copilot.generated.session_events import ExternalToolRequestedData, SessionEventType +from copilot.session import PermissionHandler, SessionConfig from langchain_core.callbacks import ( - CallbackManagerForLLMRun, AsyncCallbackManagerForLLMRun, + CallbackManagerForLLMRun, ) from langchain_core.language_models.chat_models import BaseChatModel -from langchain_core.messages import ( - AIMessage, - BaseMessage, - HumanMessage, - SystemMessage, - ToolMessage, -) +from langchain_core.messages import AIMessage, BaseMessage, SystemMessage from langchain_core.outputs import ChatGeneration, ChatResult from langchain_core.tools import BaseTool -from langchain_core.utils.function_calling import convert_to_openai_tool -from pydantic import BaseModel, Field, PrivateAttr - - -def _convert_tools_to_openai_format( - tools: Sequence[dict[str, Any] | type | Callable | BaseTool], -) -> list[dict[str, Any]]: - """Convert LangChain tool specs to OpenAI-format tool definitions.""" - result = [] - for tool in tools: - if isinstance(tool, dict): - # Already a dict — assume it's in OpenAI format or close enough - result.append(tool) - else: - result.append(convert_to_openai_tool(tool)) - return result - - -def _build_copilot_tools( - openai_tools: list[dict[str, Any]], -) -> list[CopilotTool]: - """Convert OpenAI-format tool dicts into Copilot SDK Tool objects. - - The handler is a no-op because we never let the Copilot agent - autonomously execute tools — we only need the definitions so the - model can emit tool_calls in its response. - """ - copilot_tools = [] - for t in openai_tools: - fn = t.get("function", t) - name = fn["name"] - description = fn.get("description", "") - parameters = fn.get("parameters") - - # Capture loop variables explicitly to avoid the closure-over-loop-variable - # pitfall. Although _noop_handler is never actually invoked (tool - # execution is intercepted at the LangChain level), the correct capture - # pattern is important for correctness and future maintainability. - def _make_noop_handler(tool_name: str): - async def _noop_handler(invocation: ToolInvocation) -> ToolResult: - # This handler should never actually be invoked because we - # intercept tool requests at the LangChain level. - return ToolResult( - textResultForLlm="Tool execution is managed by LangChain.", - resultType="success", - ) - - return _noop_handler - - copilot_tools.append( - CopilotTool( - name=name, - description=description, - handler=_make_noop_handler(name), - parameters=parameters, - ) - ) - return copilot_tools - - -def _deep_decode_json_strings(obj: Any) -> Any: - """Recursively decode values that are JSON-encoded strings. - - Some LLMs (e.g. Claude via the Copilot SDK) double-encode nested - lists or objects as JSON strings inside the outer tool-call arguments - dict. This function walks the structure and replaces any string value - that successfully parses as a JSON array or object with the decoded - Python value, leaving plain strings untouched. - """ - if isinstance(obj, dict): - return {k: _deep_decode_json_strings(v) for k, v in obj.items()} - if isinstance(obj, list): - return [_deep_decode_json_strings(v) for v in obj] - if isinstance(obj, str): - stripped = obj.strip() - if stripped and stripped[0] in ("{", "["): - try: - decoded = json.loads(stripped) - # Only substitute if the result is a richer structure - if isinstance(decoded, (dict, list)): - return _deep_decode_json_strings(decoded) - except (json.JSONDecodeError, ValueError): - pass - return obj - - -def _messages_to_prompt(messages: list[BaseMessage]) -> str: - """Convert a list of LangChain messages into a single prompt string. - - The Copilot SDK accepts a plain text prompt rather than a structured - message array. We serialise the conversation into a tagged format so - the model can distinguish roles. - """ - parts: list[str] = [] - for msg in messages: - content = ( - msg.content if isinstance(msg.content, str) else json.dumps(msg.content) - ) - - if isinstance(msg, SystemMessage): - parts.append(f"[system]\n{content}") - elif isinstance(msg, HumanMessage): - parts.append(f"[user]\n{content}") - elif isinstance(msg, AIMessage): - text_parts = [f"[assistant]\n{content}"] if content else ["[assistant]"] - # Include any tool calls the AI made previously - if msg.tool_calls: - for tc in msg.tool_calls: - text_parts.append( - f"[tool_call id={tc['id']} name={tc['name']}]\n" - f"{json.dumps(tc['args'])}" - ) - parts.append("\n".join(text_parts)) - elif isinstance(msg, ToolMessage): - parts.append(f"[tool_result id={msg.tool_call_id}]\n{content}") - else: - parts.append(f"[{msg.type}]\n{content}") - - return "\n\n".join(parts) - +from pydantic import Field, PrivateAttr + +from ._client_manager import CopilotClientManager +from ._errors import CopilotSetupError +from ._message_converter import extract_system_message, messages_to_prompt +from ._preflight import describe_auth_sources +from ._tool_converter import ( + build_copilot_tools, + convert_tools_to_openai_format, + deep_decode_json_strings, +) -def _extract_system_message(messages: list[BaseMessage]) -> Optional[str]: - """Extract the system message content if the first message is a SystemMessage.""" - if messages and isinstance(messages[0], SystemMessage): - content = messages[0].content - return content if isinstance(content, str) else json.dumps(content) - return None +logger = logging.getLogger(__name__) class ChatCopilot(BaseChatModel): @@ -361,12 +99,14 @@ class ChatCopilot(BaseChatModel): """Options passed to CopilotClient() constructor.""" # Private attributes (not serialised by Pydantic) - _client: Optional[CopilotClient] = PrivateAttr(default=None) - _client_started: bool = PrivateAttr(default=False) + _manager: CopilotClientManager = PrivateAttr(default=None) _bound_tools: list[dict[str, Any]] = PrivateAttr(default_factory=list) _tool_choice: Optional[str] = PrivateAttr(default=None) _ls_structured_output_format: Optional[dict[str, Any]] = PrivateAttr(default=None) + def model_post_init(self, __context: Any) -> None: + self._manager = CopilotClientManager(self.copilot_client_options) + # ------------------------------------------------------------------ # # LangChain required properties # ------------------------------------------------------------------ # @@ -380,143 +120,95 @@ def _identifying_params(self) -> dict[str, Any]: return {"model": self.model} # ------------------------------------------------------------------ # - # Client lifecycle + # Lifecycle # ------------------------------------------------------------------ # - async def _ensure_client(self) -> CopilotClient: - """Lazily create, start, and verify the CopilotClient. + async def aclose(self) -> None: + """Shut down the underlying Copilot CLI process.""" + await self._manager.close() + + # ------------------------------------------------------------------ # + # Structured output (JSON-based, bypasses tool calling) + # ------------------------------------------------------------------ # - Performs pre-flight checks before starting the CLI: - 1. Resolves the CLI binary path (copy_executables workaround) - 2. Validates the binary exists and is executable - 3. Checks required environment variables (HOME, token vars) - 4. Starts the CLI server - 5. Verifies authentication via ``get_auth_status()`` + def with_structured_output( + self, + schema: Any, + *, + include_raw: bool = False, + **kwargs: Any, + ) -> Any: + """Return a chain that produces structured output via JSON text parsing. - Raises: - CopilotSetupError: If any pre-flight check fails with a - detailed, actionable error message. + The Copilot CLI's model ignores tool-calling instructions and produces + natural language responses even when tools are registered. This override + injects a JSON schema requirement directly into the system prompt and + parses the model's text response, which is far more reliable. """ - if self._client is None: - opts = dict(self.copilot_client_options or {}) - - # --- Resolve CLI binary path -------------------------------- - if "cli_path" not in opts and "cli_url" not in opts: - resolved = _resolve_copilot_cli_path() - if resolved: - opts["cli_path"] = resolved - logger.info("Resolved Copilot CLI path: %s", resolved) - else: - logger.warning( - "Could not find copilot_cli (copy_executables target). " - "Falling back to bundled binary — this may fail with " - "PermissionError if the executable bit was stripped." - ) - - # --- Pre-flight: check binary ------------------------------- - cli_path = opts.get("cli_path") - if cli_path: - problems = _check_cli_binary(cli_path) - if problems: - raise CopilotSetupError( - "Copilot CLI binary check failed:\n" - + "\n".join(f" - {p}" for p in problems) - ) - - # --- Pre-flight: check environment -------------------------- - env_problems = _check_environment() - if env_problems: - logger.warning( - "Environment issues detected:\n%s\n%s", - "\n".join(f" - {p}" for p in env_problems), - _describe_auth_sources(), - ) - # Don't hard-fail here — the user may have a token env var. - # We'll verify auth after starting the client. + from pydantic import BaseModel as PydanticBaseModel - logger.info("Starting CopilotClient...\n%s", _describe_auth_sources()) - self._client = CopilotClient(opts or None) + from langchain_core.runnables import RunnableLambda - if not self._client_started: - try: - await self._client.start() - except PermissionError as exc: - raise CopilotSetupError( - f"PermissionError starting Copilot CLI: {exc}\n" - " The CLI binary is not executable. Make sure\n" - " pip.whl_mods / copy_executables is configured in MODULE.bazel\n" - " to create an executable copy of copilot/bin/copilot." - ) from exc - except RuntimeError as exc: - if "timeout" in str(exc).lower() or "Timeout" in str(exc): - raise CopilotSetupError( - f"Timeout starting Copilot CLI server: {exc}\n" - " The CLI started but did not become ready in time.\n" - " This usually means the CLI cannot authenticate.\n\n" - + _describe_auth_sources() - + "\n\n" - " Possible fixes:\n" - " 1. Run 'copilot' in a terminal and sign in interactively.\n" - " 2. Set COPILOT_GITHUB_TOKEN (or GH_TOKEN / GITHUB_TOKEN)\n" - " and pass it via --action_env=COPILOT_GITHUB_TOKEN.\n" - " 3. Ensure HOME is available in the action environment\n" - " (use_default_shell_env = True in the Bazel rule).\n" - " See: https://github.com/github/copilot-sdk/blob/main/docs/auth/index.md" - ) from exc - raise - except Exception as exc: - raise CopilotSetupError( - f"Failed to start CopilotClient: {type(exc).__name__}: {exc}\n\n" - + _describe_auth_sources() - ) from exc + is_pydantic = isinstance(schema, type) and issubclass(schema, PydanticBaseModel) + schema_json = schema.model_json_schema() if is_pydantic else schema + schema_str = json.dumps(schema_json, indent=2) - self._client_started = True + json_instruction = ( + "\n\n# CRITICAL OUTPUT FORMAT REQUIREMENT\n" + "You MUST respond with ONLY a valid JSON object. No prose, no markdown, " + "no explanations, no code fences.\n" + "Your ENTIRE response must be a single valid JSON object matching this schema:\n" + f"{schema_str}\n" + "Start your response immediately with `{` and end with `}`." + ) - # --- Post-start: verify authentication ---------------------- - try: - auth_status = await self._client.get_auth_status() - if ( - hasattr(auth_status, "isAuthenticated") - and auth_status.isAuthenticated - ): - user = getattr(auth_status, "login", "unknown") - logger.info("Copilot authenticated as: %s", user) - elif ( - hasattr(auth_status, "is_authenticated") - and auth_status.is_authenticated - ): - user = getattr(auth_status, "login", "unknown") - logger.info("Copilot authenticated as: %s", user) + def _inject(messages: list[BaseMessage]) -> list[BaseMessage]: + out: list[BaseMessage] = [] + injected = False + for msg in messages: + if isinstance(msg, SystemMessage) and not injected: + out.append(SystemMessage(content=msg.content + json_instruction)) + injected = True else: - raise CopilotSetupError( - "Copilot CLI started but is NOT authenticated.\n" - f" Auth status: {auth_status}\n\n" - + _describe_auth_sources() - + "\n\n" - " Possible fixes:\n" - " 1. Run 'copilot' in a terminal and sign in interactively.\n" - " 2. Set COPILOT_GITHUB_TOKEN (or GH_TOKEN / GITHUB_TOKEN).\n" - " See: https://github.com/github/copilot-sdk/blob/main/docs/auth/index.md" - ) - except CopilotSetupError: - raise - except Exception as exc: - # get_auth_status itself failed — log but don't block. - # The actual LLM call will fail with a clearer error if auth - # is truly broken. - logger.warning( - "Could not verify auth status (non-fatal): %s: %s", - type(exc).__name__, - exc, + out.append(msg) + if not injected: + out.insert(0, SystemMessage(content=json_instruction.lstrip())) + return out + + def _parse(ai_message: AIMessage) -> Any: + content = (ai_message.content or "").strip() + # Strip markdown code fences if present + if content.startswith("```"): + lines = content.split("\n") + content = "\n".join(lines[1:-1]).strip() + # Extract outermost JSON object + start = content.find("{") + end = content.rfind("}") + 1 + if start == -1 or end == 0: + raise ValueError( + f"No JSON object found in model response.\n" + f"--- LLM output ---\n{content}\n--- end ---" ) + json_text = content[start:end] + try: + parsed = json.loads(json_text) + except json.JSONDecodeError as exc: + raise ValueError( + f"Model returned invalid JSON: {exc}\n" + f"--- LLM output ---\n{content}\n--- end ---" + ) from exc + if not is_pydantic: + return parsed + try: + return schema.model_validate(parsed) + except Exception as exc: + raise ValueError( + f"Model output did not match the expected schema: {exc}\n" + f"--- LLM output ---\n{content}\n--- end ---" + ) from exc - return self._client - - async def aclose(self) -> None: - """Shut down the underlying Copilot CLI process.""" - if self._client and self._client_started: - await self._client.stop() - self._client_started = False + chain = RunnableLambda(_inject) | self | RunnableLambda(_parse) + return chain # ------------------------------------------------------------------ # # Tool binding @@ -539,14 +231,13 @@ def bind_tools( Returns: A new ChatCopilot instance with the tools bound. """ - openai_tools = _convert_tools_to_openai_format(tools) - # Create a shallow copy with the tools attached + openai_tools = convert_tools_to_openai_format(tools) new = self.model_copy() new._bound_tools = openai_tools new._tool_choice = tool_choice new._ls_structured_output_format = kwargs.get("ls_structured_output_format") - new._client = self._client - new._client_started = self._client_started + # Share the same client manager so the subprocess is not restarted + new._manager = self._manager return new # ------------------------------------------------------------------ # @@ -561,101 +252,89 @@ async def _agenerate( **kwargs: Any, ) -> ChatResult: try: - client = await self._ensure_client() + client = await self._manager.ensure_client() except CopilotSetupError: - raise # Already has a clear message + raise except Exception as exc: raise CopilotSetupError( f"Unexpected error initialising Copilot SDK: {type(exc).__name__}: {exc}\n\n" - + _describe_auth_sources() + + describe_auth_sources() ) from exc # Build session config session_config: SessionConfig = { "model": kwargs.get("model", self.model), + "available_tools": [], # Disable built-in tools } - # Disable all built-in tools so only our bound tools are available - session_config["available_tools"] = [] - # Merge any extra tools from kwargs with bound tools extra_tools = kwargs.get("tools", []) all_openai_tools = self._bound_tools + ( - _convert_tools_to_openai_format(extra_tools) if extra_tools else [] + convert_tools_to_openai_format(extra_tools) if extra_tools else [] ) - if all_openai_tools: - session_config["tools"] = _build_copilot_tools(all_openai_tools) + session_config["tools"] = build_copilot_tools(all_openai_tools) - # Use system message from the conversation if present - system_content = _extract_system_message(messages) + # System message + system_content = extract_system_message(messages) if system_content: base_system = system_content - # Remove system message from prompt construction prompt_messages = [m for m in messages if not isinstance(m, SystemMessage)] else: base_system = "You are a helpful assistant." prompt_messages = messages - # When tool_choice="any" (structured output), force the model to - # respond exclusively via tool calls. - if self._tool_choice == "any" and all_openai_tools: - tool_names = [t.get("function", t)["name"] for t in all_openai_tools] - base_system += ( - "\n\nIMPORTANT: You MUST respond by calling one of the following " - f"tools: {', '.join(tool_names)}. " - "Do NOT respond with plain text. You MUST use a tool call. " - "Pass your entire answer as arguments to the tool." - ) - - session_config["system_message"] = { - "mode": "replace", - "content": base_system, - } - - # Disable infinite sessions for simple request/response + session_config["system_message"] = {"mode": "replace", "content": base_system} session_config["infinite_sessions"] = {"enabled": False} - # Create session - session = await client.create_session(session_config) - + session = await client.create_session( + on_permission_request=PermissionHandler.approve_all, + **session_config, + ) try: - # Build the prompt from messages - prompt = _messages_to_prompt(prompt_messages) + prompt = messages_to_prompt(prompt_messages) - # Collect streaming events for tool calls tool_requests: list[Any] = [] def _event_handler(event: Any) -> None: + # New SDK (protocol v3): custom tool calls come as ExternalToolRequestedData + # broadcast events rather than AssistantMessageData.tool_requests. + if isinstance(event.data, ExternalToolRequestedData): + tool_requests.append(event.data) + return + # Legacy fallback: tool calls in AssistantMessageData.tool_requests if event.type == SessionEventType.ASSISTANT_MESSAGE: if event.data.tool_requests: tool_requests.extend(event.data.tool_requests) unsubscribe = session.on(_event_handler) - try: response = await session.send_and_wait( - {"prompt": prompt}, + prompt, timeout=self.timeout, ) finally: unsubscribe() - # Extract content content = "" if response and response.data and response.data.content: content = response.data.content - # Check for tool requests on the response itself if response and response.data and response.data.tool_requests: for tr in response.data.tool_requests: if tr not in tool_requests: tool_requests.append(tr) - # Build tool_calls for the AIMessage tool_calls = [] + seen_ids: set[str] = set() for tr in tool_requests: - args = tr.arguments + if isinstance(tr, ExternalToolRequestedData): + name, call_id, args = tr.tool_name, tr.tool_call_id, tr.arguments + else: + name, call_id, args = tr.name, tr.tool_call_id, tr.arguments + if call_id in seen_ids: + continue + seen_ids.add(call_id) if isinstance(args, str): try: args = json.loads(args) @@ -663,34 +342,22 @@ def _event_handler(event: Any) -> None: args = {"raw": args} elif args is None: args = {} - - # Deep-decode: some models (e.g. Claude via Copilot SDK) return - # nested lists/objects as JSON-encoded strings inside the outer - # tool-call arguments dict. Un-double-encode them so LangChain's - # structured-output parser receives proper Python objects. if isinstance(args, dict): - args = _deep_decode_json_strings(args) - + args = deep_decode_json_strings(args) tool_calls.append( { - "name": tr.name, + "name": name, "args": args if isinstance(args, dict) else {"raw": args}, - "id": tr.tool_call_id, + "id": call_id, } ) - # Build the AIMessage ai_message = AIMessage( content=content, tool_calls=tool_calls if tool_calls else [], - response_metadata={ - "model": self.model, - }, - ) - - return ChatResult( - generations=[ChatGeneration(message=ai_message)], + response_metadata={"model": self.model}, ) + return ChatResult(generations=[ChatGeneration(message=ai_message)]) finally: await session.destroy() @@ -712,10 +379,6 @@ def _generate( loop = None if loop and loop.is_running(): - # We're already in an async context — use a helper to run in - # a new thread to avoid blocking the event loop. - import concurrent.futures - with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: future = pool.submit( asyncio.run,