From 3d1faa7e19df80ac91f4007c63c2c6bc5492bfa9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 09:28:33 +0000 Subject: [PATCH 1/3] fix(remote): scroll full-screen agent UIs from the phone Claude Code's fullscreen renderer runs in the alternate screen, which has no terminal history. The desktop pane scrolls it by sending mouse-wheel reports to the agent; the phone only scrolled its local xterm buffer, so it showed a single page. A swipe over an alternate-screen terminal now sends a `scroll` message. The server's mirror tracks the mouse encoding the agent asked for and writes matching wheel reports, only while the agent reads the wheel. Scrolling bypasses draft tracking so it never holds back peer prompts. Like view-size, it needs a paired phone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017VZQeYWdHG5Mpbg1FEyvZk --- electron/ipc/pty.test.ts | 21 +++++++++ electron/ipc/pty.ts | 14 ++++++ electron/ipc/terminal-query-responder.test.ts | 24 ++++++++++ electron/ipc/terminal-query-responder.ts | 43 ++++++++++++++++++ electron/remote/protocol.test.ts | 25 +++++++++++ electron/remote/protocol.ts | 33 ++++++++++++++ electron/remote/server-ws.test.ts | 18 ++++++++ electron/remote/server.ts | 15 +++++-- src/remote/AgentDetail.client.test.tsx | 45 ++++++++++++++++++- src/remote/AgentDetail.tsx | 27 +++++++++-- src/remote/ws.ts | 8 ++++ 11 files changed, 264 insertions(+), 9 deletions(-) diff --git a/electron/ipc/pty.test.ts b/electron/ipc/pty.test.ts index d67afe6c2..341f91a81 100644 --- a/electron/ipc/pty.test.ts +++ b/electron/ipc/pty.test.ts @@ -111,6 +111,7 @@ import { setAgentRemoteSize, setRemoteHistory, resolveProjectDockerfile, + scrollAgent, spawnAgent, setAgentHookRuntime, subscribeToAgent, @@ -1257,6 +1258,26 @@ describe('peer prompt delivery', () => { }, ); + it('scrolls a TUI that reads the wheel without holding back peer prompts', async () => { + const { args, proc } = await launch(); + scrollAgent(args.agentId, -2, 10, 5); + expect(proc.write).not.toHaveBeenCalled(); + proc.emitData('\x1b[?1049h\x1b[?1000;1006h'); + await vi.waitFor(() => expect(getAgentPromptSnapshot(args.agentId)).not.toBeNull()); + scrollAgent(args.agentId, -2, 10, 5); + expect(proc.write.mock.calls).toEqual([['\x1b[<64;10;5M\x1b[<64;10;5M']]); + const delivery = writeAgentPrompt(args.agentId, 'peer', () => {}); + // A report inside the peer's paste would be typed into the composer. + scrollAgent(args.agentId, 1, 10, 5); + await vi.advanceTimersByTimeAsync(50); + expect(await delivery).toBe(true); + expect(proc.write.mock.calls.slice(1)).toEqual([ + ['\x1b[I'], + ['\x1b[200~peer\x1b[201~'], + ['\r'], + ]); + }); + it('defers to recent raw input for five seconds', async () => { const { args, proc } = await launch(); writeToAgent(args.agentId, 'user submission\r'); diff --git a/electron/ipc/pty.ts b/electron/ipc/pty.ts index 26870be89..7e54108d9 100644 --- a/electron/ipc/pty.ts +++ b/electron/ipc/pty.ts @@ -899,6 +899,20 @@ export function writeToAgent(agentId: string, data: string): void { writeSessionInput(session, data); } +/** + * Turn the mouse wheel over the agent's screen, as the desktop pane does. A + * full-screen TUI keeps its transcript out of the terminal's history, so this + * is the only way a phone can scroll it. Does nothing unless the TUI reads the wheel. + */ +export function scrollAgent(agentId: string, lines: number, col: number, row: number): void { + const session = sessions.get(agentId); + // Inside a peer prompt's paste, a report would be typed into the composer. + if (!session || session.peerInputQueue || handingOff.has(agentId)) return; + const report = session.queries.wheelReport(lines, col, row); + // Not through writeSessionInput: a scroll is not a draft that holds back peer prompts. + if (report) session.proc.write(report); +} + function writeSessionInput(session: PtySession, data: string): void { const userActivity = hasTerminalUserActivity(data); if (userActivity) session.lastInputAt = Date.now(); diff --git a/electron/ipc/terminal-query-responder.test.ts b/electron/ipc/terminal-query-responder.test.ts index f75a750f5..f2f5f5cbe 100644 --- a/electron/ipc/terminal-query-responder.test.ts +++ b/electron/ipc/terminal-query-responder.test.ts @@ -94,6 +94,30 @@ describe('createTerminalQueryResponder', () => { responder.dispose(); }); + it('reports wheel turns only while the program reads them, in the encoding it asked for', async () => { + const { responder } = setup(); + expect(responder.wheelReport(-1, 10, 5)).toBeNull(); + // Serializing waits for everything fed so far to be parsed. + responder.feed('\x1b[?1049h\x1b[?1000;1006h'); + await responder.serialize(); + expect(responder.wheelReport(-2, 10, 5)).toBe('\x1b[<64;10;5M\x1b[<64;10;5M'); + expect(responder.wheelReport(1, 500, 0)).toBe('\x1b[<65;80;1M'); + responder.feed('\x1b[?1006l'); + await responder.serialize(); + expect(responder.wheelReport(1, 10, 5)).toBe('\x1b[Ma*%'); + responder.feed('\x1b[?1016h'); + await responder.serialize(); + expect(responder.wheelReport(1, 10, 5)).toBeNull(); + responder.feed('\x1bc\x1b[?1002h'); + await responder.serialize(); + expect(responder.wheelReport(-1, 1, 1)).toBe('\x1b[M`!!'); + responder.feed('\x1b[?1002l\x1b[?9h'); + await responder.serialize(); + expect(responder.wheelReport(-1, 1, 1)).toBeNull(); + responder.dispose(); + expect(responder.wheelReport(-1, 1, 1)).toBeNull(); + }); + it('stops answering once disposed', async () => { const { responder, replies } = setup(); responder.dispose(); diff --git a/electron/ipc/terminal-query-responder.ts b/electron/ipc/terminal-query-responder.ts index b68a0438c..d8eefbb52 100644 --- a/electron/ipc/terminal-query-responder.ts +++ b/electron/ipc/terminal-query-responder.ts @@ -23,6 +23,9 @@ export const REMOTE_SCROLLBACK_LINES = 10_000; // eslint-disable-next-line no-control-regex const CURSOR_POSITION_REPORT = /^\x1b\[\??\d+;\d+R$/; +/** How the program asked for mouse reports: DECSET 1006 (SGR), 1016 (SGR pixels), or neither. */ +type MouseEncoding = 'default' | 'sgr' | 'sgr-pixels'; + /** Answers a PTY's cursor-position queries from the main process. */ export interface TerminalQueryResponder { /** Parse process output; a cursor-position query in it is answered through `reply`. */ @@ -43,6 +46,11 @@ export interface TerminalQueryResponder { resize(cols: number, rows: number): void; /** Change how many lines are kept above the viewport; lowering it drops the oldest. */ setScrollback(lines: number): void; + /** + * What the renderer's xterm sends for `lines` wheel turns over a 1-based cell, + * negative scrolling up. Null when the program is not reading the wheel. + */ + wheelReport(lines: number, col: number, row: number): string | null; dispose(): void; } @@ -70,6 +78,25 @@ export function createTerminalQueryResponder(opts: { let muted = 0; let disposed = false; let pendingWrites = 0; + // xterm keeps the report encoding private, so follow the same modes it does. + let mouseEncoding: MouseEncoding = 'default'; + const trackMouseEncoding = (params: (number | number[])[], set: boolean) => { + for (const param of params) { + if (param === 1006) mouseEncoding = set ? 'sgr' : 'default'; + else if (param === 1016) mouseEncoding = set ? 'sgr-pixels' : 'default'; + } + return false; + }; + term.parser.registerCsiHandler({ prefix: '?', final: 'h' }, (params) => + trackMouseEncoding(params, true), + ); + term.parser.registerCsiHandler({ prefix: '?', final: 'l' }, (params) => + trackMouseEncoding(params, false), + ); + term.parser.registerEscHandler({ final: 'c' }, () => { + mouseEncoding = 'default'; + return false; + }); term.onData((data) => { if (disposed || muted > 0 || !CURSOR_POSITION_REPORT.test(data)) return; @@ -116,6 +143,22 @@ export function createTerminalQueryResponder(opts: { setScrollback(lines) { if (!disposed) term.options.scrollback = lines; }, + wheelReport(lines, col, row) { + const tracking = term.modes.mouseTrackingMode; + // X10 tracking reports presses only; pixel reports need the renderer's cell size. + if (disposed || !lines || tracking === 'none' || tracking === 'x10') return null; + if (mouseEncoding === 'sgr-pixels') return null; + const button = lines < 0 ? 64 : 65; + const x = Math.min(term.cols, Math.max(1, col)); + const y = Math.min(term.rows, Math.max(1, row)); + let report = `\x1b[<${button};${x};${y}M`; + if (mouseEncoding === 'default') { + // Each value is one byte offset by 32, so cells past 223 cannot be reported. + if (x > 223 || y > 223) return null; + report = `\x1b[M${String.fromCharCode(button + 32, x + 32, y + 32)}`; + } + return report.repeat(Math.abs(lines)); + }, dispose() { if (disposed) return; disposed = true; diff --git a/electron/remote/protocol.test.ts b/electron/remote/protocol.test.ts index 9c5150160..54b175f69 100644 --- a/electron/remote/protocol.test.ts +++ b/electron/remote/protocol.test.ts @@ -85,3 +85,28 @@ describe('parseClientMessage view-size', () => { } }); }); + +describe('parseClientMessage scroll', () => { + it('accepts wheel lines over a cell', () => { + expect( + parseClientMessage( + JSON.stringify({ type: 'scroll', agentId: 'a', lines: -3, col: 10, row: 5 }), + ), + ).toEqual({ type: 'scroll', agentId: 'a', lines: -3, col: 10, row: 5 }); + }); + + it('rejects empty, fractional, oversized, or off-screen scrolls', () => { + for (const scroll of [ + { lines: 0, col: 1, row: 1 }, + { lines: 1.5, col: 1, row: 1 }, + { lines: -101, col: 1, row: 1 }, + { lines: '1', col: 1, row: 1 }, + { lines: 1, col: 0, row: 1 }, + { lines: 1, col: 1 }, + ]) { + expect( + parseClientMessage(JSON.stringify({ type: 'scroll', agentId: 'a', ...scroll })), + ).toBeNull(); + } + }); +}); diff --git a/electron/remote/protocol.ts b/electron/remote/protocol.ts index 546afaa95..93ac8e1ba 100644 --- a/electron/remote/protocol.ts +++ b/electron/remote/protocol.ts @@ -191,6 +191,22 @@ export interface ViewSizeCommand { rows?: number; } +/** Wheel turns in one scroll message, so each stays a small terminal write. */ +export const MAX_SCROLL_LINES = 100; + +/** + * A paired phone's swipe over a full-screen TUI, sent on as mouse-wheel turns + * so the agent scrolls its own transcript. Negative lines scroll up; col and + * row are the 1-based cell under the finger. + */ +export interface ScrollCommand { + type: 'scroll'; + agentId: string; + lines: number; + col: number; + row: number; +} + export interface KillCommand { type: 'kill'; agentId: string; @@ -230,6 +246,7 @@ export type ClientMessage = | InputCommand | ResizeCommand | ViewSizeCommand + | ScrollCommand | KillCommand | SubscribeCommand | UnsubscribeCommand @@ -287,6 +304,22 @@ export function parseClientMessage(raw: string): ClientMessage | null { if (!isTerminalDimension(msg.cols) || !isTerminalDimension(msg.rows)) return null; return { type: 'view-size', agentId: msg.agentId, cols: msg.cols, rows: msg.rows }; } + case 'scroll': + if ( + typeof msg.lines !== 'number' || + !Number.isInteger(msg.lines) || + msg.lines === 0 || + Math.abs(msg.lines) > MAX_SCROLL_LINES + ) + return null; + if (!isTerminalDimension(msg.col) || !isTerminalDimension(msg.row)) return null; + return { + type: 'scroll', + agentId: msg.agentId, + lines: msg.lines, + col: msg.col, + row: msg.row, + }; case 'kill': return { type: 'kill', agentId: msg.agentId }; case 'subscribe': diff --git a/electron/remote/server-ws.test.ts b/electron/remote/server-ws.test.ts index 9e7db38dd..e3e0abcc4 100644 --- a/electron/remote/server-ws.test.ts +++ b/electron/remote/server-ws.test.ts @@ -15,6 +15,7 @@ vi.mock('../ipc/pty.js', () => ({ writeToAgent: vi.fn(), resizeAgent: vi.fn(), setAgentRemoteSize: vi.fn(), + scrollAgent: vi.fn(), killAgent: vi.fn(), subscribeToAgent: vi.fn(), subscribeToAgentRendered: vi.fn(() => null), @@ -185,6 +186,14 @@ describe('mobile token over WebSocket', () => { expect(pty.setAgentRemoteSize).not.toHaveBeenCalled(); }); + it('rejects scroll with 4003 and sends nothing to the PTY', async () => { + const ws = await connectAndAuth(mobileToken); + const closed = waitForClose(ws); + ws.send(JSON.stringify({ type: 'scroll', agentId: 'agent-1', lines: -3, col: 10, row: 5 })); + expect(await closed).toBe(4003); + expect(pty.scrollAgent).not.toHaveBeenCalled(); + }); + it('rejects kill with 4003 and does not kill the agent', async () => { const ws = await connectAndAuth(mobileToken); const closed = waitForClose(ws); @@ -253,6 +262,15 @@ describe('paired phone view size', () => { }); }); +describe('paired phone scroll', () => { + it('turns the wheel over the agent screen', async () => { + const ws = await connectAndAuth(await pair()); + ws.send(JSON.stringify({ type: 'scroll', agentId: 'agent-1', lines: -3, col: 10, row: 5 })); + await vi.waitFor(() => expect(pty.scrollAgent).toHaveBeenCalledWith('agent-1', -3, 10, 5)); + ws.close(); + }); +}); + describe('heartbeat', () => { async function startWithHeartbeat() { const server = await startRemoteServer({ diff --git a/electron/remote/server.ts b/electron/remote/server.ts index c547f1db4..19ffdf8df 100644 --- a/electron/remote/server.ts +++ b/electron/remote/server.ts @@ -13,6 +13,7 @@ import { writeToAgent, resizeAgent, setAgentRemoteSize, + scrollAgent, killAgent, subscribeToAgent, subscribeToAgentRendered, @@ -2185,9 +2186,13 @@ export function startRemoteServer(opts: { ws.close(4003, 'Pairing required'); return; } - // A viewing phone may size the terminal once paired: it changes what the - // desktop pane shows, and typing already needs the same trust. - if (msg.type === 'view-size' && tokenType !== 'coordinator' && tokenType !== 'paired') { + // A viewing phone may size or scroll the terminal once paired: both change + // what the desktop pane shows, and typing already needs the same trust. + if ( + (msg.type === 'view-size' || msg.type === 'scroll') && + tokenType !== 'coordinator' && + tokenType !== 'paired' + ) { ws.close(4003, 'Pairing required'); return; } @@ -2323,6 +2328,10 @@ export function startRemoteServer(opts: { break; } + case 'scroll': + scrollAgent(msg.agentId, msg.lines, msg.col, msg.row); + break; + case 'kill': try { killAgent(msg.agentId); diff --git a/src/remote/AgentDetail.client.test.tsx b/src/remote/AgentDetail.client.test.tsx index daf69e96e..1b361a705 100644 --- a/src/remote/AgentDetail.client.test.tsx +++ b/src/remote/AgentDetail.client.test.tsx @@ -3,7 +3,7 @@ import { createSignal } from 'solid-js'; import { render } from 'solid-js/web'; import { AgentDetail } from './AgentDetail'; import { fetchMergeReadiness, fetchNotes, fetchTaskDiff, mergeTask, closeTask } from './api'; -import { agents, sendInput } from './ws'; +import { agents, scrollAgent, sendInput } from './ws'; import type { RemoteAgent } from '../../electron/remote/protocol'; const terminalMocks = vi.hoisted(() => ({ @@ -11,6 +11,8 @@ const terminalMocks = vi.hoisted(() => ({ refresh: vi.fn(), options: { fontSize: 14 }, scrollback: undefined as ((data: string, cols: number, rows: number) => void) | undefined, + bufferType: 'normal' as 'normal' | 'alternate', + element: undefined as HTMLElement | undefined, })); vi.mock('@xterm/xterm', () => ({ @@ -19,7 +21,20 @@ vi.mock('@xterm/xterm', () => ({ rows = 24; modes = { bracketedPasteMode: true }; options = terminalMocks.options; - buffer = { active: { length: 0, viewportY: 0, baseY: 0, getLine: () => undefined } }; + buffer = { + active: { + get type() { + return terminalMocks.bufferType; + }, + length: 0, + viewportY: 0, + baseY: 0, + getLine: () => undefined, + }, + }; + get element() { + return terminalMocks.element; + } open() {} refresh(start: number, end: number) { terminalMocks.refresh(start, end); @@ -62,6 +77,7 @@ vi.mock('./ws', () => ({ subscribeAgent: vi.fn(), unsubscribeAgent: vi.fn(), sendInput: vi.fn(), + scrollAgent: vi.fn(), onOutput: () => () => {}, onScrollback: (_id: string, callback: (data: string, cols: number, rows: number) => void) => { terminalMocks.scrollback = callback; @@ -112,6 +128,8 @@ beforeEach(() => { }, ]); terminalMocks.options.fontSize = 14; + terminalMocks.bufferType = 'normal'; + terminalMocks.element = undefined; localStorage.clear(); vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue(null); host = document.createElement('div'); @@ -717,6 +735,29 @@ describe('phone terminal viewport', () => { expect(sendInput).not.toHaveBeenCalled(); }); + it('scrolls a full-screen agent UI by turning the wheel over the cell under the finger', () => { + const element = document.createElement('div'); + const screen = document.createElement('div'); + screen.className = 'xterm-screen'; + element.append(screen); + // 80x24 cells of 10px, starting 100px down the page. + vi.spyOn(screen, 'getBoundingClientRect').mockReturnValue( + DOMRect.fromRect({ x: 0, y: 100, width: 800, height: 240 }), + ); + terminalMocks.element = element; + terminalMocks.bufferType = 'alternate'; + mount(); + const { scroller } = viewport(300); + touch(scroller, 'touchstart', 100, 200); + touch(scroller, 'touchmove', 100, 260); + expect(scrollAgent).toHaveBeenCalledWith('a1', -3, 11, 17); + // Past the grid, the nearest edge cell still gets the wheel. + touch(scroller, 'touchmove', 900, 1100); + expect(scrollAgent).toHaveBeenLastCalledWith('a1', expect.any(Number), 80, 24); + expect(terminalMocks.scrollLines).not.toHaveBeenCalled(); + expect(sendInput).not.toHaveBeenCalled(); + }); + it('handles wheel scrolling before xterm consumes it, while preserving browser zoom', () => { mount(); const { content } = viewport(300); diff --git a/src/remote/AgentDetail.tsx b/src/remote/AgentDetail.tsx index 2c544eeef..78318d1e3 100644 --- a/src/remote/AgentDetail.tsx +++ b/src/remote/AgentDetail.tsx @@ -10,11 +10,13 @@ import { readLocal, writeLocal } from './storage'; import { messageForTerminal } from './terminalText'; import { TaskHeader } from './TaskHeader'; import { needsYou } from './attention'; +import { MAX_SCROLL_LINES } from '../../electron/remote/protocol'; import { subscribeAgent, unsubscribeAgent, onOutput, onScrollback, + scrollAgent, sendInput, agents, status, @@ -208,7 +210,23 @@ export function AgentDetail(props: AgentDetailProps) { let touchY = 0; let touchId: number | undefined; let historyPixels = 0; - const scrollTerminal = (dx: number, dy: number) => { + // A full-screen agent UI keeps its transcript out of the terminal history: + // turn the wheel over the cell under the finger, as the desktop pane does. + const scrollFullScreen = (lines: number, x: number, y: number) => { + if (!term) return; + const rect = term.element?.querySelector('.xterm-screen')?.getBoundingClientRect(); + const cell = (offset: number, size: number | undefined, cells: number) => + size + ? Math.min(cells, Math.max(1, Math.floor((offset / size) * cells) + 1)) + : Math.ceil(cells / 2); + scrollAgent( + props.agentId, + Math.max(-MAX_SCROLL_LINES, Math.min(MAX_SCROLL_LINES, lines)), + cell(x - (rect?.left ?? 0), rect?.width, term.cols), + cell(y - (rect?.top ?? 0), rect?.height, term.rows), + ); + }; + const scrollTerminal = (dx: number, dy: number, x: number, y: number) => { if (!term || !terminalScroller) return; terminalScroller.scrollLeft += dx; const previousTop = terminalScroller.scrollTop; @@ -220,7 +238,8 @@ export function AgentDetail(props: AgentDetailProps) { historyPixels += dy - (terminalScroller.scrollTop - previousTop); const lines = Math.trunc(historyPixels / terminalLineHeight); if (lines) { - term.scrollLines(lines); + if (term.buffer.active.type === 'alternate') scrollFullScreen(lines, x, y); + else term.scrollLines(lines); historyPixels -= lines * terminalLineHeight; } } @@ -249,7 +268,7 @@ export function AgentDetail(props: AgentDetailProps) { return; } e.preventDefault(); - scrollTerminal(touchX - touch.clientX, touchY - touch.clientY); + scrollTerminal(touchX - touch.clientX, touchY - touch.clientY, touch.clientX, touch.clientY); touchX = touch.clientX; touchY = touch.clientY; }; @@ -268,7 +287,7 @@ export function AgentDetail(props: AgentDetailProps) { : e.deltaMode === WheelEvent.DOM_DELTA_PAGE ? (terminalScroller?.clientHeight ?? terminalLineHeight) : 1; - scrollTerminal(e.deltaX * scale, e.deltaY * scale); + scrollTerminal(e.deltaX * scale, e.deltaY * scale, e.clientX, e.clientY); }; terminalScroller.addEventListener('touchstart', touchStart, { passive: true }); terminalScroller.addEventListener('touchmove', touchMove, { passive: false }); diff --git a/src/remote/ws.ts b/src/remote/ws.ts index 9308b8a75..75b37c83e 100644 --- a/src/remote/ws.ts +++ b/src/remote/ws.ts @@ -280,6 +280,14 @@ function request(msg: Record): Promise { }); } +/** + * Turn the mouse wheel over a full-screen agent UI, which keeps its transcript + * out of the terminal history. Needs a paired phone; the server closes others. + */ +export function scrollAgent(agentId: string, lines: number, col: number, row: number): void { + if (canControl()) send({ type: 'scroll', agentId, lines, col, row }); +} + export function subscribeAgent(agentId: string): void { send({ type: 'subscribe', agentId }); } From 690f89140f45b66aa96986497b8a9c01e24159c7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 09:37:21 +0000 Subject: [PATCH 2/3] fix(android): scroll full-screen agents from the Android app The Android terminal had the same single-page limit as the phone web UI in Claude Code's fullscreen mode, and it showed the shell's main-screen history above the alternate screen, which the desktop never shows. The alternate screen now shows alone, as in xterm. While it is up on a paired phone, a vertical drag the terminal view cannot follow turns into the server's `scroll` message, so the agent scrolls its own transcript. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017VZQeYWdHG5Mpbg1FEyvZk --- android/README.md | 3 +- .../com/parallelcode/phone/RemoteClient.kt | 21 +++++++++ .../java/com/parallelcode/phone/Screens.kt | 45 +++++++++++++++++++ .../com/parallelcode/phone/TerminalScreen.kt | 12 ++++- .../com/parallelcode/phone/AgentWheelTest.kt | 36 +++++++++++++++ .../parallelcode/phone/TerminalScreenTest.kt | 11 +++++ 6 files changed, 125 insertions(+), 3 deletions(-) create mode 100644 android/app/src/test/java/com/parallelcode/phone/AgentWheelTest.kt diff --git a/android/README.md b/android/README.md index cfb691426..f73757e35 100644 --- a/android/README.md +++ b/android/README.md @@ -13,7 +13,7 @@ Native companion app for the desktop's **Connect Phone** (Remote Access) feature - **Settings:** theme and looks, keep the screen on, widget background transparency and card color, connection status, wait for VPN (skipped on your home Wi-Fi, which needs location access to read the network name, and "Allow all the time" for agent notifications in the background), and forget this computer. - **Swipe between tasks:** with a task open, swipe sideways to the previous or next one in the list; the header shows its position ("2 of 5"). When another task needs you, **Next task →** at the bottom of the terminal jumps straight to it. - **Terminal:** an agent's terminal in the colors of the look you picked, matching the desktop. Once paired: a reply box and keys a phone keyboard lacks (Enter, Esc, Tab, arrows, Ctrl+C). As on the desktop, typing `!` into an empty reply switches to the agent's shell mode; tap the `!` to leave it. With "Fit the terminal to this phone" on (Settings, off by default), the terminal takes the phone's size while open so full-screen agents such as Claude Code fill it; the computer's own terminal shifts meanwhile and gets its size back when you leave. -- **Terminal space and zoom:** pinch with two fingers to magnify the whole terminal up to 400%, and pan with two fingers (or sideways with one) to read a part of it; one finger still scrolls the history. Swiping between tasks pauses while zoomed; double-tap or the zoom pill resets it. The expand icon in the title bar hides the title, tabs, and quick keys while keeping the message field and Send/Stop available; the restore icon or Android Back returns to the normal layout without losing your draft. Terminal and chat replies use full-width multiline fields, with action buttons below. +- **Terminal space and zoom:** pinch with two fingers to magnify the whole terminal up to 400%, and pan with two fingers (or sideways with one) to read a part of it; one finger still scrolls the history. A full-screen agent such as Claude Code's fullscreen mode keeps its history to itself, so once paired, one finger scrolls the agent's own transcript instead, as the mouse wheel does on the computer (the computer's view scrolls with it). Swiping between tasks pauses while zoomed; double-tap or the zoom pill resets it. The expand icon in the title bar hides the title, tabs, and quick keys while keeping the message field and Send/Stop available; the restore icon or Android Back returns to the normal layout without losing your draft. Terminal and chat replies use full-width multiline fields, with action buttons below. - **History:** opening a terminal loads the desktop's history (up to 10,000 lines), and the phone keeps up to 20,000 lines while the terminal stays open. - **Changes:** the task's diff against its base branch, file by file with added and removed lines. - **Quick replies and voice:** saved replies above the built-in chat's reply box (edit them in Settings; the terminal leaves them out to make room) and a mic button that dictates with Android's speech recognizer. @@ -93,6 +93,7 @@ See `electron/remote/server.ts` and `electron/remote/protocol.ts`. | Connect | WebSocket `/ws`; first message `{ type: "auth", token }`. The paired token is used when present | | Watch | `subscribe` / `unsubscribe`; the server sends `scrollback`, then `output` (base64 PTY bytes) | | View size | `view-size` with `{ cols, rows }` (paired) while a terminal is open; without them, or on disconnect, the desktop size returns | +| Scroll | `scroll` with `{ lines, col, row }` (paired) over a full-screen agent; the desktop sends it mouse-wheel turns | | Projects | `GET /api/mobile/projects` (paired) | | Agents | `GET /api/mobile/agents` (paired) → `[{ id, name, isDefault, models }]`; `403` on older desktops | | New task | `POST /api/mobile/tasks` with `{ projectId, name, prompt, agentId?, model? }` (paired); returns `{ taskId }` | diff --git a/android/app/src/main/java/com/parallelcode/phone/RemoteClient.kt b/android/app/src/main/java/com/parallelcode/phone/RemoteClient.kt index ff305c0b9..b26268401 100644 --- a/android/app/src/main/java/com/parallelcode/phone/RemoteClient.kt +++ b/android/app/src/main/java/com/parallelcode/phone/RemoteClient.kt @@ -606,6 +606,25 @@ class RemoteClient( terminalBuffers[agentId]?.resize(cols, rows) } + /** + * Turn the mouse wheel over an agent's full-screen TUI, as scrolling over the desktop pane + * does: the TUI keeps its transcript out of the terminal history. Negative [lines] scroll up. + * Needs pairing; the server closes a view-only socket that tries. Agents scroll their + * transcript wherever the wheel turns, so it turns over the middle of the screen. + */ + fun scrollTerminal(agentId: String, lines: Int) { + val screen = terminalBuffers[agentId]?.screen ?: return + if (lines == 0 || !_state.value.canControl) return + send( + JSONObject() + .put("type", "scroll") + .put("agentId", agentId) + .put("lines", lines.coerceIn(-MAX_SCROLL_LINES, MAX_SCROLL_LINES)) + .put("col", screen.cols / 2 + 1) + .put("row", screen.rows / 2 + 1), + ) + } + /** Stream an agent's terminal. Retained for backwards compatibility. */ fun watchTerminal(agentId: String, listener: TerminalListener): () -> Unit { getTerminalBuffer(agentId) @@ -883,6 +902,8 @@ class RemoteClient( // The server drops a socket message past 64 KiB; leave room for the envelope. const val MAX_CHAT_MESSAGE_LENGTH = 60_000 const val MAX_INPUT_LENGTH = 4096 + // The server's MAX_SCROLL_LINES: wheel turns in one scroll message. + const val MAX_SCROLL_LINES = 100 const val PREFIX_KEY_DELAY_MS = 50L } } diff --git a/android/app/src/main/java/com/parallelcode/phone/Screens.kt b/android/app/src/main/java/com/parallelcode/phone/Screens.kt index 005636c44..224daa3f7 100644 --- a/android/app/src/main/java/com/parallelcode/phone/Screens.kt +++ b/android/app/src/main/java/com/parallelcode/phone/Screens.kt @@ -104,6 +104,9 @@ import androidx.compose.foundation.gestures.calculateZoom import androidx.compose.foundation.gestures.detectTapGestures import androidx.compose.ui.geometry.Offset import androidx.compose.ui.graphics.TransformOrigin +import androidx.compose.ui.input.nestedscroll.NestedScrollConnection +import androidx.compose.ui.input.nestedscroll.NestedScrollSource +import androidx.compose.ui.input.nestedscroll.nestedScroll import androidx.compose.ui.input.pointer.PointerEventPass import androidx.compose.ui.input.pointer.pointerInput import androidx.compose.ui.input.pointer.positionChange @@ -111,6 +114,7 @@ import androidx.compose.runtime.Composable import androidx.compose.runtime.collectAsState import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.SideEffect import androidx.compose.runtime.derivedStateOf import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableFloatStateOf @@ -144,6 +148,7 @@ import androidx.compose.ui.text.input.TextFieldValue import androidx.compose.ui.text.TextRange import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.Dp +import androidx.compose.ui.unit.Velocity import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import androidx.compose.animation.core.Animatable @@ -965,6 +970,7 @@ fun AgentScreen( val version by buffer.version.collectAsState() // Recomposition is batched per frame, so a burst of output renders the text once. val lines = remember(version) { buffer.screen.styledLines() } + val fullScreen = remember(version) { buffer.screen.alternateScreen } var tab by rememberSaveable { mutableStateOf(AgentTab.TERMINAL) } var closing by remember { mutableStateOf(false) } var merging by remember { mutableStateOf(false) } @@ -1160,6 +1166,11 @@ fun AgentScreen( fitToView = sizeTerminal, onViewSize = { cols, rows -> viewSize = cols to rows }, onZoomedChange = onZoomedChange, + onWheel = if (fullScreen && state.canControl) { + { wheelLines: Int -> client.scrollTerminal(agentId, wheelLines) } + } else { + null + }, // Keeps an agent's prompt clear of the Next task pill below it. bottomInset = if (nextNeedingYou != null) 44.dp else 0.dp, ) @@ -1340,6 +1351,8 @@ private fun TerminalText( fitToView: Boolean, onViewSize: (cols: Int, rows: Int) -> Unit, onZoomedChange: (Boolean) -> Unit = {}, + /** Takes the drag the terminal cannot follow, as mouse-wheel lines for a full-screen agent. */ + onWheel: ((lines: Int) -> Unit)? = null, /** Room kept under the newest line for a pill floating over the terminal's bottom. */ bottomInset: Dp = 0.dp, ) { @@ -1362,6 +1375,8 @@ private fun TerminalText( val currentOnZoomedChange by rememberUpdatedState(onZoomedChange) LaunchedEffect(zoomed) { currentOnZoomedChange(zoomed) } DisposableEffect(Unit) { onDispose { currentOnZoomedChange(false) } } + val wheel = remember { AgentWheel() } + SideEffect { wheel.onWheel = onWheel } // Follow output only while parked at the bottom, decided where each scroll ends. Deciding at // the start instead kept follow on for a scroll that began at the bottom (the usual case), so @@ -1389,6 +1404,7 @@ private fun TerminalText( .fillMaxSize() .background(Color(palette.background)) .clipToBounds() + .nestedScroll(wheel) .pointerInput(Unit) { awaitEachGesture { awaitFirstDown(requireUnconsumed = false, pass = PointerEventPass.Initial) @@ -1437,6 +1453,7 @@ private fun TerminalText( val autoFont = (10f * textWidthPx / (cols.coerceAtLeast(1) * charPxAt10)).coerceIn(8f, 14f).sp val phoneFont = TERMINAL_FONT val fontSize = if (fitToView) phoneFont else autoFont + SideEffect { wheel.linePx = with(density) { (fontSize * 1.27f).toPx() } } val viewPaddingPx = with(density) { 16.dp.roundToPx() } val viewHeightPx = constraints.maxHeight @@ -1558,6 +1575,34 @@ private fun TerminalText( } } +/** + * Turns the drag a full-screen agent's terminal cannot follow into mouse-wheel lines for the agent, + * as scrolling over the desktop pane does: the agent keeps its transcript out of the history. + */ +internal class AgentWheel : NestedScrollConnection { + var onWheel: ((lines: Int) -> Unit)? = null + var linePx = 0f + private var pending = 0f + + override fun onPostScroll(consumed: Offset, available: Offset, source: NestedScrollSource): Offset { + val turn = onWheel + if (turn == null || source != NestedScrollSource.UserInput || linePx <= 0f) return Offset.Zero + // Dragging down reveals what came before, so the wheel turns up (negative lines). + pending -= available.y + val lines = (pending / linePx).toInt() + if (lines != 0) { + pending -= lines * linePx + turn(lines) + } + return Offset(0f, available.y) + } + + override suspend fun onPreFling(available: Velocity): Velocity { + pending = 0f + return Velocity.Zero + } +} + private val TERMINAL_PADDING_H = 12.dp private val TERMINAL_FONT = 11.sp private const val MAX_TERMINAL_ZOOM = 4f diff --git a/android/app/src/main/java/com/parallelcode/phone/TerminalScreen.kt b/android/app/src/main/java/com/parallelcode/phone/TerminalScreen.kt index 36d85b3e5..8dfa1f50e 100644 --- a/android/app/src/main/java/com/parallelcode/phone/TerminalScreen.kt +++ b/android/app/src/main/java/com/parallelcode/phone/TerminalScreen.kt @@ -22,6 +22,13 @@ class TerminalScreen(cols: Int = 80, rows: Int = 24) { var bracketedPaste = false private set + /** + * A full-screen program (Claude Code's fullscreen mode, vim) holds the alternate screen. It + * keeps its history to itself, so the phone scrolls it by sending it the mouse wheel. + */ + val alternateScreen: Boolean + get() = savedMainGrid != null + private class Line(cols: Int, style: Long) { val chars = CharArray(cols) { ' ' } val styles = LongArray(cols) { style } @@ -129,10 +136,11 @@ class TerminalScreen(cols: Int = 80, rows: Int = 24) { /** * History then screen, each line split into style runs, without trailing blank lines. Built - * for every output frame, so the history is shared from a snapshot rather than copied. + * for every output frame, so the history is shared from a snapshot rather than copied. The + * alternate screen shows alone, as in xterm: the history above it is the shell's, not the program's. */ fun styledLines(): List> { - val past = history.snapshot() + val past = if (alternateScreen) emptyList() else history.snapshot() val screen = grid.map(::spans) var count = past.size + screen.size fun line(i: Int) = if (i < past.size) past[i] else screen[i - past.size] diff --git a/android/app/src/test/java/com/parallelcode/phone/AgentWheelTest.kt b/android/app/src/test/java/com/parallelcode/phone/AgentWheelTest.kt new file mode 100644 index 000000000..e66c48193 --- /dev/null +++ b/android/app/src/test/java/com/parallelcode/phone/AgentWheelTest.kt @@ -0,0 +1,36 @@ +package com.parallelcode.phone + +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.input.nestedscroll.NestedScrollSource +import org.junit.Assert.assertEquals +import org.junit.Test + +class AgentWheelTest { + private fun wheel(turns: MutableList) = AgentWheel().apply { + linePx = 10f + onWheel = { turns += it } + } + + @Test + fun turnsTheWheelUpForADragDownTheTerminalCannotFollow() { + val turns = mutableListOf() + val wheel = wheel(turns) + val left = wheel.onPostScroll(Offset.Zero, Offset(0f, 25f), NestedScrollSource.UserInput) + assertEquals(listOf(-2), turns) + assertEquals(Offset(0f, 25f), left) + // The half line left over carries into the next drag step. + wheel.onPostScroll(Offset.Zero, Offset(0f, 5f), NestedScrollSource.UserInput) + wheel.onPostScroll(Offset.Zero, Offset(0f, -40f), NestedScrollSource.UserInput) + assertEquals(listOf(-2, -1, 4), turns) + } + + @Test + fun leavesFlingsAndOrdinaryTerminalsAlone() { + val turns = mutableListOf() + val wheel = wheel(turns) + assertEquals(Offset.Zero, wheel.onPostScroll(Offset.Zero, Offset(0f, 50f), NestedScrollSource.SideEffect)) + wheel.onWheel = null + assertEquals(Offset.Zero, wheel.onPostScroll(Offset.Zero, Offset(0f, 50f), NestedScrollSource.UserInput)) + assertEquals(emptyList(), turns) + } +} diff --git a/android/app/src/test/java/com/parallelcode/phone/TerminalScreenTest.kt b/android/app/src/test/java/com/parallelcode/phone/TerminalScreenTest.kt index 9fe4335df..4c7c26f99 100644 --- a/android/app/src/test/java/com/parallelcode/phone/TerminalScreenTest.kt +++ b/android/app/src/test/java/com/parallelcode/phone/TerminalScreenTest.kt @@ -1,6 +1,7 @@ package com.parallelcode.phone import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Test @@ -71,6 +72,16 @@ class TerminalScreenTest { assertEquals("shell", s.text()) } + @Test + fun showsAFullScreenProgramWithoutTheShellHistoryBehindIt() { + val s = screen(rows = 2, data = "1\r\n2\r\n3\u001b[?1049h\u001b[Hclaude") + assertTrue(s.alternateScreen) + assertEquals("claude", s.text()) + s.feed("\u001b[?1049l".toByteArray()) + assertFalse(s.alternateScreen) + assertEquals("1\n2\n3", s.text()) + } + @Test fun decodesUtf8SplitAcrossChunks() { val s = screen() From 7e334433edc139d9834d3d56b4e31d7a19d1a858 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 11:35:18 +0000 Subject: [PATCH 3/3] fix(remote): send phone wheel scrolls only as SGR reports Default-encoded mouse reports carry cell positions as raw bytes. Written to the PTY as a string, a column or row of 96 or more became two UTF-8 bytes, and the program read a broken report followed by a stray key. The desktop pane never sends those reports at all: xterm emits them as binary events, which the renderer does not forward. Match it and send wheel turns only to programs that asked for SGR reports. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017VZQeYWdHG5Mpbg1FEyvZk --- electron/ipc/terminal-query-responder.test.ts | 18 +++++++---- electron/ipc/terminal-query-responder.ts | 32 +++++++------------ 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/electron/ipc/terminal-query-responder.test.ts b/electron/ipc/terminal-query-responder.test.ts index f2f5f5cbe..5e9ea7819 100644 --- a/electron/ipc/terminal-query-responder.test.ts +++ b/electron/ipc/terminal-query-responder.test.ts @@ -94,24 +94,28 @@ describe('createTerminalQueryResponder', () => { responder.dispose(); }); - it('reports wheel turns only while the program reads them, in the encoding it asked for', async () => { + it('reports wheel turns only while the program reads them as SGR reports', async () => { const { responder } = setup(); expect(responder.wheelReport(-1, 10, 5)).toBeNull(); // Serializing waits for everything fed so far to be parsed. - responder.feed('\x1b[?1049h\x1b[?1000;1006h'); + responder.feed('\x1b[?1049h\x1b[?1000h'); + await responder.serialize(); + // Default-encoded reports go out as binary events the desktop pane never forwards. + expect(responder.wheelReport(-1, 10, 5)).toBeNull(); + responder.feed('\x1b[?1006h'); await responder.serialize(); expect(responder.wheelReport(-2, 10, 5)).toBe('\x1b[<64;10;5M\x1b[<64;10;5M'); expect(responder.wheelReport(1, 500, 0)).toBe('\x1b[<65;80;1M'); - responder.feed('\x1b[?1006l'); - await responder.serialize(); - expect(responder.wheelReport(1, 10, 5)).toBe('\x1b[Ma*%'); responder.feed('\x1b[?1016h'); await responder.serialize(); expect(responder.wheelReport(1, 10, 5)).toBeNull(); + responder.feed('\x1bc\x1b[?1002;1006h'); + await responder.serialize(); + expect(responder.wheelReport(-1, 1, 1)).toBe('\x1b[<64;1;1M'); responder.feed('\x1bc\x1b[?1002h'); await responder.serialize(); - expect(responder.wheelReport(-1, 1, 1)).toBe('\x1b[M`!!'); - responder.feed('\x1b[?1002l\x1b[?9h'); + expect(responder.wheelReport(-1, 1, 1)).toBeNull(); + responder.feed('\x1b[?1006h\x1b[?1002l\x1b[?9h'); await responder.serialize(); expect(responder.wheelReport(-1, 1, 1)).toBeNull(); responder.dispose(); diff --git a/electron/ipc/terminal-query-responder.ts b/electron/ipc/terminal-query-responder.ts index d8eefbb52..3c78248ee 100644 --- a/electron/ipc/terminal-query-responder.ts +++ b/electron/ipc/terminal-query-responder.ts @@ -23,9 +23,6 @@ export const REMOTE_SCROLLBACK_LINES = 10_000; // eslint-disable-next-line no-control-regex const CURSOR_POSITION_REPORT = /^\x1b\[\??\d+;\d+R$/; -/** How the program asked for mouse reports: DECSET 1006 (SGR), 1016 (SGR pixels), or neither. */ -type MouseEncoding = 'default' | 'sgr' | 'sgr-pixels'; - /** Answers a PTY's cursor-position queries from the main process. */ export interface TerminalQueryResponder { /** Parse process output; a cursor-position query in it is answered through `reply`. */ @@ -48,7 +45,7 @@ export interface TerminalQueryResponder { setScrollback(lines: number): void; /** * What the renderer's xterm sends for `lines` wheel turns over a 1-based cell, - * negative scrolling up. Null when the program is not reading the wheel. + * negative scrolling up. Null when the program is not reading SGR wheel reports. */ wheelReport(lines: number, col: number, row: number): string | null; dispose(): void; @@ -78,12 +75,13 @@ export function createTerminalQueryResponder(opts: { let muted = 0; let disposed = false; let pendingWrites = 0; - // xterm keeps the report encoding private, so follow the same modes it does. - let mouseEncoding: MouseEncoding = 'default'; + // xterm keeps the report encoding private, so follow the same modes it does: + // 1006 selects SGR reports, 1016 SGR pixel reports, and resetting either the default. + let sgrReports = false; const trackMouseEncoding = (params: (number | number[])[], set: boolean) => { for (const param of params) { - if (param === 1006) mouseEncoding = set ? 'sgr' : 'default'; - else if (param === 1016) mouseEncoding = set ? 'sgr-pixels' : 'default'; + if (param === 1006) sgrReports = set; + else if (param === 1016) sgrReports = false; } return false; }; @@ -94,7 +92,7 @@ export function createTerminalQueryResponder(opts: { trackMouseEncoding(params, false), ); term.parser.registerEscHandler({ final: 'c' }, () => { - mouseEncoding = 'default'; + sgrReports = false; return false; }); @@ -145,19 +143,13 @@ export function createTerminalQueryResponder(opts: { }, wheelReport(lines, col, row) { const tracking = term.modes.mouseTrackingMode; - // X10 tracking reports presses only; pixel reports need the renderer's cell size. - if (disposed || !lines || tracking === 'none' || tracking === 'x10') return null; - if (mouseEncoding === 'sgr-pixels') return null; - const button = lines < 0 ? 64 : 65; + // X10 tracking reports presses only. The renderer sends other encodings as + // binary events the desktop pane never forwards, so the wheel does nothing there. + if (disposed || !lines || !sgrReports || tracking === 'none' || tracking === 'x10') + return null; const x = Math.min(term.cols, Math.max(1, col)); const y = Math.min(term.rows, Math.max(1, row)); - let report = `\x1b[<${button};${x};${y}M`; - if (mouseEncoding === 'default') { - // Each value is one byte offset by 32, so cells past 223 cannot be reported. - if (x > 223 || y > 223) return null; - report = `\x1b[M${String.fromCharCode(button + 32, x + 32, y + 32)}`; - } - return report.repeat(Math.abs(lines)); + return `\x1b[<${lines < 0 ? 64 : 65};${x};${y}M`.repeat(Math.abs(lines)); }, dispose() { if (disposed) return;