From ebae3107ea4e2542f0e5c4ca8ba4c52b6bf932ae Mon Sep 17 00:00:00 2001 From: Li Kong Date: Tue, 17 Nov 2026 14:09:44 +0800 Subject: [PATCH 1/4] fix: honor configured provider for flash summarizer Resolve the Flash step summarizer through the configured summarizer agent and preserve its provider when applying a model override. Add regression coverage for OpenAI routing. Co-Authored-By: Claude Code --- artemis/agents/flash/summarizer.py | 17 ++++++------ artemis/services/llm.py | 8 +++++- tests/unit/agents/test_flash_runner.py | 3 ++- .../unit/agents/test_flash_step_summarizer.py | 19 +++++++++++++- tests/unit/test_llm_grounding.py | 26 +++++++++++++++++++ third_party/mobile_use/llm_service.py | 3 +++ 6 files changed, 64 insertions(+), 12 deletions(-) diff --git a/artemis/agents/flash/summarizer.py b/artemis/agents/flash/summarizer.py index 5b8a3cf98..90cf49cf5 100644 --- a/artemis/agents/flash/summarizer.py +++ b/artemis/agents/flash/summarizer.py @@ -36,7 +36,7 @@ from artemis.context import ArtemisContext from artemis.memory.step_memory import JobKey, StepMemoryService -from artemis.services.llm import RobustChatModelWrapper, get_google_llm, get_llm +from artemis.services.llm import RobustChatModelWrapper, get_llm from artemis.services.token_meter import record_llm_usage from artemis.utils.task_tree import format_actions_clean from artemis.utils.visualization import draw_action_overlay_on_image @@ -161,16 +161,15 @@ def __init__( flush_timeout_s=flush_timeout_s, ) - # Initialize lightweight VLM: prioritize explicit model_name + # Resolve the configured summarizer provider, applying an optional model override. target_model = model_name or "gemini-2.5-flash-lite" self._model_name = target_model - try: - if model_name: - self._llm = get_google_llm(model_name=target_model, temperature=0.0) - else: - self._llm = get_llm(ctx, name="summarizer", is_utils=True) - except Exception: - self._llm = get_google_llm(model_name=target_model, temperature=0.0) + self._llm = get_llm( + ctx, + name="summarizer", + model_name=model_name, + temperature=0.0, + ) try: configured = getattr(self._llm, "model", None) or getattr(self._llm, "model_name", None) if isinstance(configured, str) and configured: diff --git a/artemis/services/llm.py b/artemis/services/llm.py index ec91e6fa0..c2f0f2d62 100644 --- a/artemis/services/llm.py +++ b/artemis/services/llm.py @@ -945,11 +945,17 @@ def _get_llm( is_utils: bool = False, use_fallback: bool = False, temperature: float | None = None, + model_name: str | None = None, ) -> BaseChatModel: """Resolves and instantiates the appropriate LLM wrapper for the given agent role.""" endpoint = _resolve_endpoint(ctx, name, is_utils=is_utils, use_fallback=use_fallback) + updates: dict[str, Any] = {} if temperature is not None: - endpoint = endpoint.model_copy(update={"temperature": temperature}) + updates["temperature"] = temperature + if model_name is not None: + updates["model_name"] = model_name + if updates: + endpoint = endpoint.model_copy(update=updates) raw_model = ModelFactory.get_model(endpoint) handler = DataEngineCallbackHandler(ctx) diff --git a/tests/unit/agents/test_flash_runner.py b/tests/unit/agents/test_flash_runner.py index f188b5e1b..edcc02344 100644 --- a/tests/unit/agents/test_flash_runner.py +++ b/tests/unit/agents/test_flash_runner.py @@ -34,7 +34,8 @@ def mock_context(): ctx = Mock(spec=ArtemisContext) ctx.llm_config = Mock() mock_llm_cfg = Mock() - mock_llm_cfg.model = "gemini-2.5-flash" + mock_llm_cfg.provider = "openai" + mock_llm_cfg.model = "gpt-4o-mini" mock_llm_cfg.temperature = 0.1 ctx.llm_config.get_agent.return_value = mock_llm_cfg diff --git a/tests/unit/agents/test_flash_step_summarizer.py b/tests/unit/agents/test_flash_step_summarizer.py index a26bc57b0..c6bcf7bf2 100644 --- a/tests/unit/agents/test_flash_step_summarizer.py +++ b/tests/unit/agents/test_flash_step_summarizer.py @@ -14,7 +14,7 @@ """Unit tests for VisualStepSummarizer and Context Compressor in Flash profile.""" -from unittest.mock import ANY, AsyncMock, Mock, call +from unittest.mock import ANY, AsyncMock, Mock, call, patch from uuid import uuid4 import pytest from langchain_core.messages import AIMessage, HumanMessage, ToolMessage @@ -38,6 +38,23 @@ def mock_context(): return ctx +def test_explicit_summarizer_model_uses_configured_provider(mock_context): + """A model override must not force the Google provider.""" + llm = Mock() + model_name = "copilot/gpt-5.6-sol" + + with patch("artemis.agents.flash.summarizer.get_llm", return_value=llm) as get_llm: + summarizer = VisualStepSummarizer(mock_context, model_name=model_name) + + get_llm.assert_called_once_with( + mock_context, + name="summarizer", + model_name=model_name, + temperature=0.0, + ) + assert summarizer._llm is llm + + @pytest.mark.asyncio async def test_summarizer_dispatch_and_caching(mock_context): """Verify non-blocking dispatch and summary generation.""" diff --git a/tests/unit/test_llm_grounding.py b/tests/unit/test_llm_grounding.py index c99b01d27..9af3502d8 100644 --- a/tests/unit/test_llm_grounding.py +++ b/tests/unit/test_llm_grounding.py @@ -14,6 +14,9 @@ """Unit tests for ModelFactory instantiation and grounding tool wiring.""" +from types import SimpleNamespace +from unittest.mock import Mock, patch + from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider @@ -43,6 +46,29 @@ def test_model_factory_openai_instantiation(): assert getattr(model, "reasoning_effort", None) == "medium" +def test_get_llm_model_override_preserves_configured_provider(): + """An explicit model name changes the model but not its configured provider.""" + from artemis.services.llm import get_llm + + llm_config = SimpleNamespace( + get_agent=lambda _: SimpleNamespace(provider="openai", model="configured-model") + ) + ctx = SimpleNamespace(llm_config=llm_config, data_engine=None) + model_name = "copilot/gpt-5.6-sol" + + with ( + patch("artemis.services.llm.ModelFactory.get_model", return_value=Mock()) as get_model, + patch("artemis.services.llm.DataEngineCallbackHandler"), + patch("artemis.services.llm.RobustChatModelWrapper", return_value=Mock()), + ): + get_llm(ctx, "summarizer", model_name=model_name, temperature=0.0) + + endpoint = get_model.call_args.args[0] + assert endpoint.provider is ModelProvider.OPENAI + assert endpoint.model_name == model_name + assert endpoint.temperature == 0.0 + + def test_robust_chat_model_wrapper_grounding_google(): """Verify RobustChatModelWrapper auto-injects google_search and server-side tool config for Gemini.""" from artemis.services.llm import RobustChatModelWrapper diff --git a/third_party/mobile_use/llm_service.py b/third_party/mobile_use/llm_service.py index e06f61005..6c71f8e15 100644 --- a/third_party/mobile_use/llm_service.py +++ b/third_party/mobile_use/llm_service.py @@ -169,6 +169,7 @@ def __call__( *, use_fallback: bool = False, temperature: float | None = None, + model_name: str | None = None, ) -> BaseChatModel: ... @overload @@ -179,6 +180,7 @@ def __call__( *, is_utils: Literal[True], temperature: float | None = None, + model_name: str | None = None, ) -> BaseChatModel: ... @overload @@ -190,4 +192,5 @@ def __call__( is_utils: Literal[True], use_fallback: bool = False, temperature: float | None = None, + model_name: str | None = None, ) -> BaseChatModel: ... From e4e0f4ce52356f58455a9c1d13b74492d3dd3d71 Mon Sep 17 00:00:00 2001 From: Li Kong Date: Tue, 17 Nov 2026 14:19:47 +0800 Subject: [PATCH 2/4] fix: omit temperature for Copilot reasoning models Copilot's OpenAI-compatible reasoning models reject temperature in chat completion requests. Omit the parameter for copilot-prefixed model names and cover the behavior with a regression test. Co-Authored-By: Claude Code --- artemis/llm/router.py | 8 +++++++- tests/unit/test_llm_grounding.py | 13 +++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/artemis/llm/router.py b/artemis/llm/router.py index f2e6b42b6..3d962c9db 100644 --- a/artemis/llm/router.py +++ b/artemis/llm/router.py @@ -291,9 +291,15 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: base_url = endpoint.api_base or ( str(settings.OPENAI_BASE_URL) if settings.OPENAI_BASE_URL else None ) + # Copilot's OpenAI-compatible reasoning models reject temperature. + temperature = ( + None + if endpoint.model_name.strip().lower().startswith("copilot/") + else endpoint.temperature + ) kwargs = { "model": endpoint.model_name, - "temperature": endpoint.temperature, + "temperature": temperature, "max_tokens": endpoint.max_tokens, "api_key": api_key, "base_url": base_url, diff --git a/tests/unit/test_llm_grounding.py b/tests/unit/test_llm_grounding.py index 9af3502d8..9dbc1df10 100644 --- a/tests/unit/test_llm_grounding.py +++ b/tests/unit/test_llm_grounding.py @@ -46,6 +46,19 @@ def test_model_factory_openai_instantiation(): assert getattr(model, "reasoning_effort", None) == "medium" +def test_model_factory_copilot_model_omits_temperature(): + """Copilot reasoning models reject the temperature parameter.""" + endpoint = ModelEndpoint( + provider=ModelProvider.OPENAI, + model_name="copilot/gpt-6.1-sol", + api_key="sk-test-key", + ) + + model = ModelFactory.create_model(endpoint) + + assert model.temperature is None + + def test_get_llm_model_override_preserves_configured_provider(): """An explicit model name changes the model but not its configured provider.""" from artemis.services.llm import get_llm From f49b7c953d3571500e3576006be5a901c4198e58 Mon Sep 17 00:00:00 2001 From: Li Kong Date: Tue, 17 Nov 2026 14:31:11 +0800 Subject: [PATCH 3/4] config: add Copilot model defaults Apply the workspace OpenAI-compatible Copilot model configuration and document the optional OpenAI-compatible base URL in the sanitized environment template. Do not commit the local .env credentials. Co-Authored-By: Claude Code --- .env.example | 2 ++ config/artemis.jsonc | 58 ++++++++++++++++++++++---------------------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/.env.example b/.env.example index 8b23d0a53..48c019916 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,8 @@ GEMINI_API_KEY= GOOGLE_API_KEY= OPENAI_API_KEY= +# Optional OpenAI-compatible API endpoint base URL (for example, a local gateway) +OPENAI_BASE_URL= ANTHROPIC_API_KEY= OPEN_ROUTER_API_KEY= XAI_API_KEY= diff --git a/config/artemis.jsonc b/config/artemis.jsonc index 517d73840..38e68af60 100644 --- a/config/artemis.jsonc +++ b/config/artemis.jsonc @@ -7,12 +7,12 @@ // 1. Global Default Model (Applied to all sub-agents unless overridden below) // ------------------------------------------------------------------------------ "default": { - "provider": "google", - "model": "gemini-3.8-flash", + "provider": "openai", + "model": "copilot/gpt-6.1-sol", "thinking_level": "medium", "fallback": { - "provider": "google", - "model": "gemini-3.7-flash", + "provider": "openai", + "model": "copilot/gpt-6.1-terra", "thinking_level": "medium" } }, @@ -22,29 +22,29 @@ // ------------------------------------------------------------------------------ "presets": { "gemini-flagship": { - "provider": "google", - "model": "gemini-3.8-flash", - "fallback": { "provider": "google", "model": "gemini-3.7-flash" } + "provider": "openai", + "model": "copilot/gpt-6.1-sol", + "fallback": { "provider": "openai", "model": "copilot/gpt-6.1-terra" } }, "gemini-flash": { - "provider": "google", - "model": "gemini-2.5-flash", - "fallback": { "provider": "google", "model": "gemini-2.0-flash" } + "provider": "openai", + "model": "copilot/gpt-6-sol", + "fallback": { "provider": "openai", "model": "copilot/gpt-6.1-terra" } }, "openai-gpt4o": { "provider": "openai", - "model": "gpt-4o", - "fallback": { "provider": "openai", "model": "gpt-4o-mini" } + "model": "copilot/gpt-6.1-sol", + "fallback": { "provider": "openai", "model": "copilot/gpt-6-sol" } }, "cost-saving": { - "provider": "google", - "model": "gemini-3.5-flash-lite", - "fallback": { "provider": "google", "model": "gemini-3.1-flash-lite" } + "provider": "openai", + "model": "copilot/gpt-5.6-sol", + "fallback": { "provider": "openai", "model": "copilot/gpt-5.6-terra" } }, "local-ollama": { "provider": "openai", - "model": "llama3.2-vision", - "fallback": { "provider": "openai", "model": "llama3.2-vision" } + "model": "copilot/gpt-6.1-sol", + "fallback": { "provider": "openai", "model": "copilot/gpt-6.1-terra" } } }, @@ -71,14 +71,13 @@ "video_analyzer": {}, // 👁️ Object Detector & Explorer (Flash Mode): Visual coordinates grounding and bounding boxes - // CRITICAL REQUIREMENT: Standard LLMs (including standard Gemini Flash, GPT-4o, Claude 3.7) lack sub-pixel spatial coordinate fine-tuning. - // To achieve accurate [x, y] point and bounding box localization, this node MUST use specialized Gemini ER (Embodied Reasoning / Robotics) models: - // e.g., "gemini-robotics-er-2-preview". Non-ER models will fail spatial coordinate detection. + // This uses a Copilot model rather than a specialized Gemini ER model; coordinate localization may be less reliable. "object_detector": { - "model": "gemini-robotics-er-2-preview", + "provider": "openai", + "model": "copilot/gpt-6.1-sol", "fallback": { - "provider": "google", - "model": "gemini-robotics-er-2-preview" + "provider": "openai", + "model": "copilot/gpt-6.1-terra" } }, @@ -95,10 +94,11 @@ // 🔽 Hopper: Fast lightweight package name locator "hopper": { - "model": "gemini-3.5-flash-lite", + "provider": "openai", + "model": "copilot/gpt-5.6-sol", "fallback": { - "provider": "google", - "model": "gemini-3.1-flash-lite" + "provider": "openai", + "model": "copilot/gpt-5.6-terra" } }, @@ -125,7 +125,7 @@ } // 🎯 Planner Validation: Advisory review of milestone changes. // Defaults to the same lightweight model as the pixel safety net - // (gemini-3.5-flash-lite, temperature 0); add "planner_validation": {...} here to override. + // (copilot/gpt-5.6-sol, temperature 0); add "planner_validation": {...} here to override. }, // ------------------------------------------------------------------------------ @@ -145,7 +145,7 @@ // replacing images only after strictly objective, high-density summaries are ready. "step_summarizer": { "enabled": true, // Whether background step summarization is active - "model": "gemini-3.5-flash-lite", // Lightweight model for fast background inference + "model": "copilot/gpt-5.6-sol", // Lightweight model for fast background inference "prune_history_xml": true // Prune outdated heavy XML trees from historical steps } }, @@ -246,7 +246,7 @@ // call each); milestone closes are exempt, the hard line waives it. "min_steps": 3, "target_source_tokens": 2000, - "model": "gemini-3.8-flash", + "model": "copilot/gpt-6.1-sol", "max_chunks": 8 // "max_eras": 8 // independent era cap; omitted = follows max_chunks }, From a90d57b41abd7e55e9e3caf4ded08a917ffdcb92 Mon Sep 17 00:00:00 2001 From: Li Kong Date: Tue, 17 Nov 2026 15:05:22 +0800 Subject: [PATCH 4/4] fix(ui): persist model setup selection Co-Authored-By: Claude Code --- .../src/app/pages/home/home.component.html | 4 +- .../src/app/pages/home/home.component.spec.ts | 78 +++++++++++++++++++ .../src/app/pages/home/home.component.ts | 34 +++++++- 3 files changed, 111 insertions(+), 5 deletions(-) create mode 100644 apps/showcase_ui/src/app/pages/home/home.component.spec.ts diff --git a/apps/showcase_ui/src/app/pages/home/home.component.html b/apps/showcase_ui/src/app/pages/home/home.component.html index 060a01d25..5dc304c0b 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.html +++ b/apps/showcase_ui/src/app/pages/home/home.component.html @@ -578,9 +578,9 @@

AI Model Setup

Default Model
- {{ modelConfigEnv()?.default_model?.provider || 'google' }} + {{ modelConfigEnv()?.default_model?.provider || '—' }} / - {{ modelConfigEnv()?.default_model?.model || 'gemini-3.8-flash' }} + {{ modelConfigEnv()?.default_model?.model || '—' }}
diff --git a/apps/showcase_ui/src/app/pages/home/home.component.spec.ts b/apps/showcase_ui/src/app/pages/home/home.component.spec.ts new file mode 100644 index 000000000..516c5aa3d --- /dev/null +++ b/apps/showcase_ui/src/app/pages/home/home.component.spec.ts @@ -0,0 +1,78 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { signal } from '@angular/core'; +import { TestBed } from '@angular/core/testing'; +import { provideRouter } from '@angular/router'; +import { of } from 'rxjs'; + +import { AgentService } from '../../services/agent.service'; +import { SystemService } from '../../services/system.service'; +import { TaskRecommendationService } from '../../core/services/task-recommendation.service'; +import { HomeComponent } from './home.component'; + +describe('HomeComponent model setup mode', () => { + const storageKey = 'artemis_model_setup_mode'; + let systemService: { + apiKeysMap: ReturnType>>; + currentApiKey: ReturnType>; + setSkipCredentialsCheck: jasmine.Spy; + fetchReadiness: jasmine.Spy; + fetchModelConfigEnv: jasmine.Spy; + fetchAdbServerStatus: jasmine.Spy; + }; + + beforeEach(async () => { + localStorage.removeItem(storageKey); + systemService = { + apiKeysMap: signal>({}), + currentApiKey: signal(''), + setSkipCredentialsCheck: jasmine.createSpy('setSkipCredentialsCheck'), + fetchReadiness: jasmine.createSpy('fetchReadiness').and.returnValue(of(null)), + fetchModelConfigEnv: jasmine.createSpy('fetchModelConfigEnv').and.returnValue(of(null)), + fetchAdbServerStatus: jasmine.createSpy('fetchAdbServerStatus').and.returnValue( + of({ endpoint: { mode: 'local' } }) + ) + }; + + await TestBed.configureTestingModule({ + imports: [HomeComponent], + providers: [ + provideRouter([]), + { provide: AgentService, useValue: { getProTuningDefaults: () => of({}) } }, + { provide: SystemService, useValue: systemService }, + { provide: TaskRecommendationService, useValue: {} } + ] + }).compileComponents(); + }); + + afterEach(() => localStorage.removeItem(storageKey)); + + it('persists and restores the custom setup choice', () => { + const first = TestBed.createComponent(HomeComponent); + first.componentInstance.setModelSetupMode('custom'); + + expect(localStorage.getItem(storageKey)).toBe('custom'); + first.destroy(); + + const second = TestBed.createComponent(HomeComponent); + second.componentInstance.ngOnInit(); + + expect(second.componentInstance.modelSetupMode()).toBe('custom'); + expect(systemService.setSkipCredentialsCheck).toHaveBeenCalledWith(true); + second.destroy(); + }); +}); diff --git a/apps/showcase_ui/src/app/pages/home/home.component.ts b/apps/showcase_ui/src/app/pages/home/home.component.ts index e189b9ce5..3e4344ece 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.ts +++ b/apps/showcase_ui/src/app/pages/home/home.component.ts @@ -98,7 +98,9 @@ export interface TuningSliderVm { export type { AppReference, SmartSuggestion, SuggestionCategory }; type AdbGuideTab = 'emulator' | 'usb' | 'wifi' | 'remote'; +type ModelSetupMode = 'gemini' | 'custom'; +const MODEL_SETUP_MODE_STORAGE_KEY = 'artemis_model_setup_mode'; @Component({ selector: 'app-home', @@ -121,8 +123,8 @@ export class HomeComponent implements OnInit, OnDestroy { public activeAdbGuideTab = signal('emulator'); public emulatorSetupMode = signal<'studio' | 'cli'>('studio'); - // Interactive guide tab for LLM / OCR credentials: 'gemini' | 'ocr' - public modelSetupMode = signal<'gemini' | 'custom'>('gemini'); + // Interactive guide tab for LLM credentials: 'gemini' | 'custom' + public modelSetupMode = signal('gemini'); public showOcrConfig = signal(false); public showFullConfigFile = signal(false); @@ -536,6 +538,9 @@ export class HomeComponent implements OnInit, OnDestroy { } ngOnInit(): void { + const savedMode = this.getSavedModelSetupMode(); + this.modelSetupMode.set(savedMode); + this.systemService.setSkipCredentialsCheck(savedMode === 'custom'); this.loadProTuningDefaults(); // Initial fetch of system readiness & model configuration @@ -581,8 +586,9 @@ export class HomeComponent implements OnInit, OnDestroy { this.emulatorSetupMode.set(mode); } - public setModelSetupMode(mode: 'gemini' | 'custom'): void { + public setModelSetupMode(mode: ModelSetupMode): void { this.modelSetupMode.set(mode); + this.saveModelSetupMode(mode); if (mode === 'custom') { this.systemService.setSkipCredentialsCheck(true); this.systemService.fetchModelConfigEnv().subscribe(); @@ -591,6 +597,28 @@ export class HomeComponent implements OnInit, OnDestroy { } } + private getSavedModelSetupMode(): ModelSetupMode { + if (typeof localStorage === 'undefined') { + return 'gemini'; + } + try { + return localStorage.getItem(MODEL_SETUP_MODE_STORAGE_KEY) === 'custom' ? 'custom' : 'gemini'; + } catch { + return 'gemini'; + } + } + + private saveModelSetupMode(mode: ModelSetupMode): void { + if (typeof localStorage === 'undefined') { + return; + } + try { + localStorage.setItem(MODEL_SETUP_MODE_STORAGE_KEY, mode); + } catch { + // Keep the in-memory selection if browser storage is unavailable. + } + } + public toggleFullConfigFile(): void { this.showFullConfigFile.update(v => !v); }