From a9605afdb2630e8f3563ee46dcbfdde1994b85ec Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 6 Nov 2026 04:38:07 +0000 Subject: [PATCH 1/6] Add opt-in Android instrumentation host for Drive native input --- platform/android/build.gradle.kts | 1 + .../android/drive-sample/build.gradle.kts | 29 ++ platform/android/drive-sample/run-drive.sh | 52 ++ .../src/debug/AndroidManifest.xml | 4 + .../kotlin/dev/lui/drive/sample/DriveHook.kt | 10 + .../drive-sample/src/main/AndroidManifest.xml | 10 + .../dev/lui/drive/sample/MainActivity.kt | 61 +++ .../kotlin/dev/lui/drive/sample/DriveHook.kt | 8 + platform/android/drive/README.md | 141 +++++ platform/android/drive/build.gradle.kts | 19 + .../drive/src/main/AndroidManifest.xml | 10 + .../kotlin/dev/lui/drive/DriveInputMethod.kt | 75 +++ .../dev/lui/drive/DriveInstrumentation.kt | 491 ++++++++++++++++++ .../kotlin/dev/lui/drive/DriveProtocol.kt | 60 +++ .../main/kotlin/dev/lui/drive/DriveTarget.kt | 44 ++ .../src/main/res/xml/drive_input_method.xml | 2 + .../kotlin/dev/lui/drive/DriveProtocolTest.kt | 39 ++ .../lui/src/main/kotlin/dev/lui/LuiBackend.kt | 18 + .../src/main/kotlin/dev/lui/LuiInspection.kt | 75 +++ .../src/main/kotlin/dev/lui/LuiNodeView.kt | 1 + platform/android/settings.gradle.kts | 1 + 21 files changed, 1151 insertions(+) create mode 100644 platform/android/drive-sample/build.gradle.kts create mode 100755 platform/android/drive-sample/run-drive.sh create mode 100644 platform/android/drive-sample/src/debug/AndroidManifest.xml create mode 100644 platform/android/drive-sample/src/debug/kotlin/dev/lui/drive/sample/DriveHook.kt create mode 100644 platform/android/drive-sample/src/main/AndroidManifest.xml create mode 100644 platform/android/drive-sample/src/main/kotlin/dev/lui/drive/sample/MainActivity.kt create mode 100644 platform/android/drive-sample/src/release/kotlin/dev/lui/drive/sample/DriveHook.kt create mode 100644 platform/android/drive/README.md create mode 100644 platform/android/drive/build.gradle.kts create mode 100644 platform/android/drive/src/main/AndroidManifest.xml create mode 100644 platform/android/drive/src/main/kotlin/dev/lui/drive/DriveInputMethod.kt create mode 100644 platform/android/drive/src/main/kotlin/dev/lui/drive/DriveInstrumentation.kt create mode 100644 platform/android/drive/src/main/kotlin/dev/lui/drive/DriveProtocol.kt create mode 100644 platform/android/drive/src/main/kotlin/dev/lui/drive/DriveTarget.kt create mode 100644 platform/android/drive/src/main/res/xml/drive_input_method.xml create mode 100644 platform/android/drive/src/test/kotlin/dev/lui/drive/DriveProtocolTest.kt create mode 100644 platform/android/lui/src/main/kotlin/dev/lui/LuiInspection.kt diff --git a/platform/android/build.gradle.kts b/platform/android/build.gradle.kts index 7802a7c5..8d5604f0 100644 --- a/platform/android/build.gradle.kts +++ b/platform/android/build.gradle.kts @@ -1,5 +1,6 @@ plugins { id("com.android.library") version "9.1.1" apply false + id("com.android.application") version "9.1.1" apply false id("org.jetbrains.kotlin.plugin.compose") version "2.4.0" apply false id("app.cash.paparazzi") version "2.0.0-alpha04" apply false } diff --git a/platform/android/drive-sample/build.gradle.kts b/platform/android/drive-sample/build.gradle.kts new file mode 100644 index 00000000..2604d95b --- /dev/null +++ b/platform/android/drive-sample/build.gradle.kts @@ -0,0 +1,29 @@ +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.plugin.compose") +} + +android { + namespace = "dev.lui.drive.sample" + compileSdk { version = release(37) { minorApiLevel = 0 } } + defaultConfig { + applicationId = "dev.lui.drive.sample" + minSdk = 30 + targetSdk = 37 + versionCode = 1 + versionName = "1.0" + } + buildFeatures { compose = true } + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } +} + +dependencies { + implementation(project(":lui")) + implementation(platform("androidx.compose:compose-bom:2026.09.00")) + implementation("androidx.activity:activity-compose:1.10.1") + implementation("androidx.compose.material3:material3:1.5.0-beta01") + debugImplementation(project(":drive")) +} diff --git a/platform/android/drive-sample/run-drive.sh b/platform/android/drive-sample/run-drive.sh new file mode 100755 index 00000000..e3c17e40 --- /dev/null +++ b/platform/android/drive-sample/run-drive.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ ${1:-} != --test-device || $# != 2 ]]; then + echo "Usage: DRIVE_BIN=/path/to/drive $0 --test-device scenario.drive" >&2 + echo "This explicitly selects a test IME on the connected test device; do not use a personal device." >&2 + exit 2 +fi + +scenario=$2 +port=49377 +package=dev.lui.drive.sample +ime=$package/dev.lui.drive.DriveInputMethod +old_ime=$(adb shell settings get secure default_input_method | tr -d '\r') +[[ -n "$old_ime" && "$old_ime" != null ]] || { echo "Cannot preserve current IME; refusing setup" >&2; exit 1; } +printf 'Original IME (restore if the launcher is killed): %s\n' "$old_ime" >&2 +enabled=false +if adb shell ime list -s | tr -d '\r' | grep -Fxq "$ime"; then enabled=true; fi +log=$(mktemp) +host_pid= +forwarded=false +cleanup() { + local status=$? + trap - EXIT INT TERM + adb shell ime set "$old_ime" >/dev/null || echo "Restore the original IME manually: $old_ime" >&2 + if ! $enabled; then adb shell ime disable "$ime" >/dev/null || true; fi + adb shell am force-stop "$package" >/dev/null || true + if $forwarded; then adb forward --remove "tcp:$port" >/dev/null || true; fi + if [[ -n "$host_pid" ]]; then kill "$host_pid" 2>/dev/null || true; wait "$host_pid" || true; fi + cat "$log" + rm -f "$log" + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +adb shell ime enable "$ime" >/dev/null +adb shell ime set "$ime" >/dev/null +adb forward --no-rebind "tcp:$port" "tcp:$port" +forwarded=true +adb shell am instrument -w -e drive true -e port "$port" \ + -e activity "$package/.MainActivity" \ + "$package/dev.lui.drive.DriveInstrumentation" >"$log" 2>&1 & +host_pid=$! +ready=false +for ((i=0; i<150; i++)); do + if grep -q 'listening on 127.0.0.1' "$log"; then ready=true; break; fi + if ! kill -0 "$host_pid" 2>/dev/null; then break; fi + sleep 0.1 +done +$ready || { echo "Drive instrumentation did not become ready" >&2; exit 1; } +"${DRIVE_BIN:-drive}" --tcp "127.0.0.1:$port" --input native "$scenario" diff --git a/platform/android/drive-sample/src/debug/AndroidManifest.xml b/platform/android/drive-sample/src/debug/AndroidManifest.xml new file mode 100644 index 00000000..54f56f1d --- /dev/null +++ b/platform/android/drive-sample/src/debug/AndroidManifest.xml @@ -0,0 +1,4 @@ + + + diff --git a/platform/android/drive-sample/src/debug/kotlin/dev/lui/drive/sample/DriveHook.kt b/platform/android/drive-sample/src/debug/kotlin/dev/lui/drive/sample/DriveHook.kt new file mode 100644 index 00000000..d1e0c9f0 --- /dev/null +++ b/platform/android/drive-sample/src/debug/kotlin/dev/lui/drive/sample/DriveHook.kt @@ -0,0 +1,10 @@ +package dev.lui.drive.sample + +import android.app.Activity +import dev.lui.LuiBackend +import dev.lui.drive.DriveTarget + +internal object DriveHook { + fun attach(activity: Activity, backend: LuiBackend): AutoCloseable? = + if (activity.intent.getBooleanExtra("luiDrive", false)) DriveTarget.attach(activity, backend) else null +} diff --git a/platform/android/drive-sample/src/main/AndroidManifest.xml b/platform/android/drive-sample/src/main/AndroidManifest.xml new file mode 100644 index 00000000..7f30df76 --- /dev/null +++ b/platform/android/drive-sample/src/main/AndroidManifest.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/platform/android/drive-sample/src/main/kotlin/dev/lui/drive/sample/MainActivity.kt b/platform/android/drive-sample/src/main/kotlin/dev/lui/drive/sample/MainActivity.kt new file mode 100644 index 00000000..dd431654 --- /dev/null +++ b/platform/android/drive-sample/src/main/kotlin/dev/lui/drive/sample/MainActivity.kt @@ -0,0 +1,61 @@ +package dev.lui.drive.sample + +import android.os.Bundle +import androidx.activity.ComponentActivity +import androidx.activity.compose.setContent +import androidx.compose.material3.MaterialTheme +import dev.lui.LuiBackend +import dev.lui.LuiEvent +import dev.lui.LuiNodeKind +import dev.lui.LuiPatchBatch +import dev.lui.LuiPatchOp +import dev.lui.LuiWireValue + +class MainActivity : ComponentActivity() { + private lateinit var backend: LuiBackend + private var drive: AutoCloseable? = null + private var generation = 0 + private var count = 0 + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + backend = LuiBackend(onEvent = { event -> + when (event) { + is LuiEvent.Press -> if (event.node == 3L) { + count++ + apply(listOf(LuiPatchOp.SetProp(4, "text", LuiWireValue.Str("Count: $count")))) + } + is LuiEvent.TextChanged -> apply(listOf(LuiPatchOp.SetProp(event.node, "text", LuiWireValue.Str(event.text)))) + else -> Unit + } + }) + val nodes = listOf(1L to LuiNodeKind.root, 2L to LuiNodeKind.column, + 3L to LuiNodeKind.button, 4L to LuiNodeKind.text, 5L to LuiNodeKind.textField, + 6L to LuiNodeKind.button) + apply(buildList { + nodes.forEach { (id, kind) -> add(LuiPatchOp.CreateNode(id, kind)) } + add(LuiPatchOp.InsertChild(1, 2, 0)) + listOf(3L, 4L, 5L, 6L).forEachIndexed { index, id -> add(LuiPatchOp.InsertChild(2, id, index)) } + add(LuiPatchOp.SetProp(2, "padding", LuiWireValue.IntValue(24))) + add(LuiPatchOp.SetProp(2, "gap", LuiWireValue.IntValue(16))) + add(LuiPatchOp.SetProp(3, "text", LuiWireValue.Str("Increment"))) + add(LuiPatchOp.SetProp(3, "accessibility-identifier", LuiWireValue.Str("increment"))) + add(LuiPatchOp.SetProp(4, "text", LuiWireValue.Str("Count: 0"))) + add(LuiPatchOp.SetProp(5, "text", LuiWireValue.Str("initial"))) + add(LuiPatchOp.SetProp(5, "accessibility-identifier", LuiWireValue.Str("editor"))) + add(LuiPatchOp.SetProp(6, "text", LuiWireValue.Str("Disabled"))) + add(LuiPatchOp.SetProp(6, "enabled", LuiWireValue.Bool(false))) + }) + setContent { MaterialTheme { backend.Content() } } + drive = DriveHook.attach(this, backend) + } + + private fun apply(ops: List) { + check(backend.applyBatch(LuiPatchBatch(++generation, ops))) { backend.lastError ?: "patch failed" } + } + + override fun onDestroy() { + drive?.close() + super.onDestroy() + } +} diff --git a/platform/android/drive-sample/src/release/kotlin/dev/lui/drive/sample/DriveHook.kt b/platform/android/drive-sample/src/release/kotlin/dev/lui/drive/sample/DriveHook.kt new file mode 100644 index 00000000..189df3be --- /dev/null +++ b/platform/android/drive-sample/src/release/kotlin/dev/lui/drive/sample/DriveHook.kt @@ -0,0 +1,8 @@ +package dev.lui.drive.sample + +import android.app.Activity +import dev.lui.LuiBackend + +internal object DriveHook { + fun attach(activity: Activity, backend: LuiBackend): AutoCloseable? = null +} diff --git a/platform/android/drive/README.md b/platform/android/drive/README.md new file mode 100644 index 00000000..2f8bc0d9 --- /dev/null +++ b/platform/android/drive/README.md @@ -0,0 +1,141 @@ +# Android native Drive host (explicit test opt-in) + +The optional `:drive` Android library runs under real Android Instrumentation, +not inside the OCaml reducer. It requires API 30+, a debuggable target, a +registered `DriveTarget`, and an explicit `-e drive true` launch. It binds only +`127.0.0.1:49377` (override with `-e port`). There is no manifest receiver, +application initializer, release dependency, or automatic listener startup. + +## Hook up a debug host + +Add `debugImplementation(project(":drive"))` to the Android app. Keep the +library, its INTERNET permission, IME service and instrumentation declaration +out of the release variant. The instrumented test variant must have minSdk 30+. +In debug source, register after `setContent` and keep the handle until Activity +destruction. In release source, use a no-op hook. For example: + +```kotlin +// Debug source only, and only for an explicitly enabled test Activity. +val drive = if (intent.getBooleanExtra("luiDrive", false)) { + DriveTarget.attach(this, backend) +} else null +// In onDestroy, on the UI thread: +drive?.close() +``` + +Declare a self-instrumenting runner in the app's **debug** manifest, replacing +the target package: + +```xml + +``` + +The runner starts the supplied Activity with `luiDrive=true` in a new cleared +task. Use an isolated debug/test app, not a production task. The SDK never +enables or selects an IME, changes accessibility settings, or grants itself +permissions through shell commands. + +## Build and run the sample + +From `platform/android`, with an Android SDK and JDK 17+: + +```sh +./gradlew :drive-sample:assembleDebug +adb install -r drive-sample/build/outputs/apk/debug/drive-sample-debug.apk +``` + +On an isolated, unlocked API 30+ emulator/test device only, use the explicit +sample launcher. It saves the currently selected/enabled IME state and restores +it on normal completion, errors and shell signals. Do not terminate it with +SIGKILL; if interrupted outside normal cleanup, restore the printed original +IME manually with `adb shell ime set ORIGINAL_COMPONENT`. + +```sh +DRIVE_BIN=/path/to/drive ./drive-sample/run-drive.sh --test-device scenario.drive +``` + +The Drive binary needs loopback TCP support (`--tcp 127.0.0.1:49377`). The +equivalent manual input-free launch (without IME setup, so no `type` capability): + +```sh +adb forward tcp:49377 tcp:49377 +adb shell am instrument -w -e drive true -e port 49377 \ + -e activity dev.lui.drive.sample/.MainActivity \ + dev.lui.drive.sample/dev.lui.drive.DriveInstrumentation +# In another terminal: +drive --tcp 127.0.0.1:49377 --input native scenario.drive +# Afterwards: +adb forward --remove tcp:49377 +``` + +For text, explicitly enable/select +`dev.lui.drive.sample/dev.lui.drive.DriveInputMethod` on the test device, saving +the previous `default_input_method` first and restoring it afterwards. Prefer +the launcher so cleanup is not forgotten. Missing selection is rejected before +the target is tapped. `type` is omitted from hello unless this test IME is +selected. No clipboard, accessibility set-text, property assignment or LUI +event injection is used. Empty text replaces the full value with an empty +string; Unicode is committed through the real editor `InputConnection`. + +## Protocol and behavior + +* Native protocol v1 is unchanged. Initial writes are hello, full frames, then + authoritative ops (the readiness marker). Updates remove previously + published IDs, recreate all current nodes/properties/links, and replace the + complete frame table. A single worker serializes actions and the ordered + writer publishes fresh ops and frames before each correlated result, + including screenshot and rejected actions. Changes between actions are + published while reading the connection. +* Coordinates/deltas are logical viewport units (Android dp), origin at the + content view's top-left. Geometry uses clipped actual Compose bounds. The + adapter converts dp plus the current content view screen offset to physical + screen coordinates for OS input, including status bar/inset movement and + resized IME layouts. PNG captures the content viewport in physical pixels. +* `UiAutomation.injectInputEvent` routes MotionEvent/KeyEvent through Android's + input dispatcher and the real Window/View/Compose input path. Clicks, double + clicks, long clicks, coordinate taps and drags never invoke semantics actions. + Positive scroll dx/dy means content right/down, using an opposite finger swipe. + Deltas must fit in the target's visible bounds; use multiple smaller swipes. +* The read-only accessibility hierarchy verifies native window layering, + visibility, enabled state and tagged hit ancestry. Model/ancestor visibility, + enabled state, clipped bounds and retained-node incarnation are checked too. + Missing geometry, untagged occluders, different windows, destroyed targets, + stale/reused IDs and unknown actions fail explicitly. There is no auto-scroll + or semantic fallback. Extensions/modals without measured/tagged nodes are not + selector-input targets; offscreen/unmaterialized virtual rows must be brought + into view by native scrolling first. +* `type` requires native editable focus and the selected test IME, complete + `getExtractedText`, `setSelection`, `beginBatchEdit` and `commitText` support. + Read-only, partial-extraction and unsupported input connections are rejected. + InputConnection queries run off the editor UI looper to avoid deadlock. +* Keys: letters, digits, Enter, Tab, Escape, Backspace, Delete, Space, + ArrowLeft/Right/Up/Down, Home, End, PageUp/Down. Combine with `ctrl`/`mod` + (Control on Android), `shift`, `alt`, or `meta`/`cmd`. Unknown keys/modifiers + fail; use `type` for arbitrary Unicode. Physical key-up and gesture cancel + cleanup run even after injection failure or timeout. +* `settle` means at least 100ms of retained-model, measured-layout, host draw + and accessibility-event quiet, bounded by a 3s action deadline. It is not + application-task/network idle and cannot prove completion of every + render-thread/paint-only animation. Continuous redraws fail the deadline; + use Drive `wait` for business effects. +* One connection is accepted per instrumentation launch. Accept/read idle + deadline is 120s; action lines are limited to 1 MiB; writes and UI-thread + waits are bounded to 1s. Bad envelopes without a correlatable request close + the connection. A timeout returns an error when possible, then closes rather + than retrying possibly applied input. EOF closes inspection and the Activity. + Timed-out, not-yet-started UI callbacks are removed; already running native + work cannot be rolled back. + +## Evidence and limitations + +Compile/lint and the pure v1 validation contract test are not device input +evidence. Run real-device/emulator native tests before relying on a downstream +app. Accessibility inspection cannot establish the hit behavior of arbitrary +custom, accessibility-invisible input interceptors; those need application- +specific instrumentation. Multi-display, split-screen, system UI automation, +external keyboard/IME behavior, clipboard, OS drag-and-drop and semantic-only +Drive actions are not supported. Pointer drag is a touch gesture, not a promise +of application drag-and-drop semantics. Secure-window screenshots are rejected. +The adapter never claims input changed product state: assert resulting state +with Drive expectations. diff --git a/platform/android/drive/build.gradle.kts b/platform/android/drive/build.gradle.kts new file mode 100644 index 00000000..897f5289 --- /dev/null +++ b/platform/android/drive/build.gradle.kts @@ -0,0 +1,19 @@ +plugins { + id("com.android.library") +} + +android { + namespace = "dev.lui.drive" + compileSdk { version = release(37) { minorApiLevel = 0 } } + defaultConfig { minSdk = 30 } + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } +} + +dependencies { + implementation(project(":lui")) + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.7.3") + testImplementation("junit:junit:4.13.2") +} diff --git a/platform/android/drive/src/main/AndroidManifest.xml b/platform/android/drive/src/main/AndroidManifest.xml new file mode 100644 index 00000000..ed5f7f42 --- /dev/null +++ b/platform/android/drive/src/main/AndroidManifest.xml @@ -0,0 +1,10 @@ + + + + + + + + + diff --git a/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveInputMethod.kt b/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveInputMethod.kt new file mode 100644 index 00000000..ec96b567 --- /dev/null +++ b/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveInputMethod.kt @@ -0,0 +1,75 @@ +package dev.lui.drive + +import android.content.pm.ApplicationInfo +import android.inputmethodservice.InputMethodService +import android.view.inputmethod.EditorInfo +import android.view.inputmethod.ExtractedTextRequest +import android.view.inputmethod.InputConnection + +/** An explicitly selected test-device IME. No clipboard or semantic editing is used. */ +class DriveInputMethod : InputMethodService() { + internal var session = 0L + private set + internal var editor: EditorInfo? = null + private set + + override fun onCreate() { + super.onCreate() + check(applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE != 0) { + "Drive IME requires a debuggable application" + } + active = this + } + + override fun onStartInput(attribute: EditorInfo, restarting: Boolean) { + super.onStartInput(attribute, restarting) + session++ + editor = attribute + } + + override fun onFinishInput() { + editor = null + session++ + super.onFinishInput() + } + + override fun onDestroy() { + if (active === this) active = null + super.onDestroy() + } + + override fun onEvaluateInputViewShown(): Boolean { + super.onEvaluateInputViewShown() + return false + } + + internal fun connection(packageName: String): InputConnection { + check(editor?.packageName == packageName) { "test IME is not connected to the target editor" } + return checkNotNull(currentInputConnection) { "no native InputConnection" } + } + + companion object { + @Volatile internal var active: DriveInputMethod? = null + private set + + // RemoteInputConnection queries must not block the editor's UI looper. + internal fun replace(connection: InputConnection, text: String, checkCurrent: () -> Unit) { + val extracted = checkNotNull(connection.getExtractedText(ExtractedTextRequest().apply { + hintMaxChars = 1_048_576 + hintMaxLines = 1_048_576 + }, 0)) { "editor does not support complete native text extraction" } + check(extracted.startOffset == 0 && extracted.partialStartOffset == -1) { + "editor returned a partial text value; replacement is unsupported" + } + checkCurrent() + check(connection.beginBatchEdit()) { "editor rejected native batch edit" } + try { + check(connection.setSelection(0, extracted.text.length)) { "editor rejected native selection" } + checkCurrent() + check(connection.commitText(text, 1)) { "editor rejected native Unicode commit" } + } finally { + connection.endBatchEdit() + } + } + } +} diff --git a/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveInstrumentation.kt b/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveInstrumentation.kt new file mode 100644 index 00000000..b42563b9 --- /dev/null +++ b/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveInstrumentation.kt @@ -0,0 +1,491 @@ +package dev.lui.drive + +import android.app.Activity +import android.app.Instrumentation +import android.content.ComponentName +import android.content.Intent +import android.content.pm.ApplicationInfo +import android.graphics.Bitmap +import android.graphics.Rect +import android.graphics.RectF +import android.os.Build +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.provider.Settings +import android.util.Base64 +import android.view.InputDevice +import android.view.KeyEvent +import android.view.MotionEvent +import android.view.ViewConfiguration +import android.view.accessibility.AccessibilityNodeInfo +import android.view.accessibility.AccessibilityWindowInfo +import dev.lui.LuiInspection +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonArray +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.longOrNull +import kotlinx.serialization.json.put +import java.io.ByteArrayOutputStream +import java.net.InetAddress +import java.net.ServerSocket +import java.net.Socket +import java.net.SocketTimeoutException +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean + +/** Self-instrumenting debug runner. Starts only with the explicit drive=true argument. */ +class DriveInstrumentation : Instrumentation() { + private lateinit var arguments: Bundle + private val main = Handler(Looper.getMainLooper()) + private var deadline = Long.MAX_VALUE + private var lastRequest = 0L + private var published = emptyMap() + private var lastSnapshot = "" + private var generation = 0 + private lateinit var target: DriveTarget + private lateinit var socket: Socket + private val transportWatchdog = Executors.newSingleThreadScheduledExecutor() + + override fun onCreate(arguments: Bundle?) { + this.arguments = arguments ?: Bundle() + start() + } + + override fun onStart() { + var activity: Activity? = null + var server: ServerSocket? = null + var failure: String? = null + try { + check(arguments.getString("drive") == "true") { "explicit -e drive true is required" } + check(Build.VERSION.SDK_INT >= 30) { "Drive native host requires Android API 30+" } + check(targetContext.applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE != 0) { "Drive requires a debug application" } + val port = arguments.getString("port", "49377").toInt() + require(port in 1024..65535) { "invalid loopback port" } + val component = ComponentName.unflattenFromString(checkNotNull(arguments.getString("activity")) { "-e activity package/.Activity is required" }) + check(component?.packageName == targetContext.packageName) { "activity must belong to instrumentation target" } + activity = startActivitySync(Intent().setComponent(component) + .putExtra("luiDrive", true) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TASK)) + target = ui { checkNotNull(DriveTarget.current) { "Activity did not register DriveTarget" } } + uiAutomation.serviceInfo = uiAutomation.serviceInfo.apply { + flags = flags or android.accessibilityservice.AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS or + android.accessibilityservice.AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS + } + server = ServerSocket(port, 1, InetAddress.getByName("127.0.0.1")).apply { soTimeout = 120_000 } + sendStatus(0, Bundle().apply { putString("drive", "listening on 127.0.0.1:$port") }) + socket = server.accept().apply { soTimeout = 100; tcpNoDelay = true } + server.close() + deadline = SystemClock.uptimeMillis() + 3_000 + settle() + check(ui { target.inspection.screenFrames().values.any { it.width() > 0 && it.height() > 0 } }) { "attach has no node geometry" } + val capabilities = DriveAction.capabilities.filter { it != "type" || selectedIme() } + send(buildJsonObject { + put("drive", "hello"); put("version", 1) + put("capabilities", JsonArray(capabilities.map { JsonPrimitive(it) })) + }) + publish(force = true, initial = true) + deadline = Long.MAX_VALUE + serve() + } catch (error: Exception) { + failure = error.message ?: error.javaClass.simpleName + } finally { + if (::socket.isInitialized) socket.close() + transportWatchdog.shutdownNow() + server?.close() + deadline = SystemClock.uptimeMillis() + 1_000 + runCatching { ui { DriveTarget.current?.close(); activity?.finish() } } + finish(if (failure == null) Activity.RESULT_OK else Activity.RESULT_CANCELED, + Bundle().apply { putString("drive", failure ?: "closed") }) + } + } + + private fun selectedIme(): Boolean = ComponentName.unflattenFromString( + Settings.Secure.getString(targetContext.contentResolver, Settings.Secure.DEFAULT_INPUT_METHOD) ?: "" + ) == ComponentName(targetContext, DriveInputMethod::class.java) + + private fun serve() { + val pending = ByteArrayOutputStream() + val input = socket.getInputStream() + var activityAt = SystemClock.uptimeMillis() + while (true) { + val byte = try { input.read() } catch (_: SocketTimeoutException) { + check(SystemClock.uptimeMillis() - activityAt < 120_000) { "Drive connection idle timeout" } + publish(force = false) + continue + } + if (byte == -1) return + activityAt = SystemClock.uptimeMillis() + if (byte == 10) { + val line = pending.toString(Charsets.UTF_8.name()) + pending.reset() + handle(line) + } else { + check(pending.size() < 1_048_576) { "Drive action line exceeds 1 MiB" } + pending.write(byte) + } + } + } + + private fun handle(line: String) { + val request = runCatching { (Json.parseToJsonElement(line) as JsonObject)["request"] as? JsonPrimitive } + .getOrNull()?.takeUnless { it.isString }?.longOrNull + if (request == null || request <= 0) error("missing positive Drive request; closing") + deadline = SystemClock.uptimeMillis() + 3_000 + var png: String? = null + var failure: String? = null + try { + val action = DriveAction.parse(line) + require(action.request > lastRequest) { "Drive request is stale or duplicated" } + lastRequest = action.request + execute(action) + settle() + if (action.name == "screenshot") png = screenshot() + } catch (error: Exception) { + failure = error.message ?: error.javaClass.simpleName + } + // A failed action still publishes the authoritative post-input state before its result. + deadline = SystemClock.uptimeMillis() + 500 + try { publish(force = true) } catch (error: Exception) { + failure = failure ?: error.message ?: "snapshot publication failed" + } + send(buildJsonObject { + put("drive", "result"); put("version", 1); put("request", request) + put("ok", failure == null) + if (failure != null) put("error", failure) + if (failure == null && png != null) put("png", png) + }) + if (failure?.contains("timeout") == true) error("native request timeout; closing") + deadline = Long.MAX_VALUE + } + + private fun execute(action: DriveAction) { + when (action.name) { + "settle", "screenshot" -> Unit + "key" -> key(action.string("combo")) + "tap" -> { + val viewport = ui { viewport() } + val density = targetContext.resources.displayMetrics.density + val x = viewport.left + action.number("x") * density + val y = viewport.top + action.number("y") * density + check(viewport.contains(x, y)) { "tap is outside the viewport" } + val id = hit(x, y) + validate(id, x, y) + gesture(x, y, x, y, 60) + } + else -> { + if (action.name == "type") { + check(selectedIme()) { "type requires the explicitly selected Drive test IME" } + ui { + val node = target.inspection.snapshot().firstOrNull { it.id == action.id() } + check(node?.properties?.get("read-only")?.boolValue != true) { "target editor is read-only" } + } + } + val frame = validate(action.id()) + val x = frame.centerX(); val y = frame.centerY() + when (action.name) { + "click" -> gesture(x, y, x, y, 60) + "double-click" -> { + gesture(x, y, x, y, 40) + pause(50) + validate(action.id(), x, y) + gesture(x, y, x, y, 40) + } + "long-click" -> gesture(x, y, x, y, ViewConfiguration.getLongPressTimeout().toLong() + 100) + "drag" -> { + val end = validate(action.id("target")) + gesture(x, y, end.centerX(), end.centerY(), 400) + } + "scroll" -> { + val scale = targetContext.resources.displayMetrics.density + val dx = action.number("dx") * scale; val dy = action.number("dy") * scale + check(dx != 0f || dy != 0f) { "scroll delta must be nonzero" } + val startX = x + dx / 2; val startY = y + dy / 2 + val endX = x - dx / 2; val endY = y - dy / 2 + check(frame.contains(startX, startY) && frame.contains(endX, endY)) { "scroll gesture exceeds visible target; use smaller deltas" } + validate(action.id(), startX, startY) + gesture(startX, startY, endX, endY, 300) + } + "type" -> type(action.id(), action.string("text"), x, y) + } + } + } + } + + private fun type(id: Long, text: String, x: Float, y: Float) { + val before = ui { DriveInputMethod.active?.session } + val wasFocused = focusedId() == id + gesture(x, y, x, y, 60) + while (true) { + checkTime() + val ready = ui { + DriveInputMethod.active?.let { ime -> + ime.editor?.packageName == targetContext.packageName && (wasFocused || ime.session != before) + } == true + } + if (focusedId() == id && ready) break + pause(20) + } + validate(id) + val (ime, session, connection) = ui { + val ime = checkNotNull(DriveInputMethod.active) { "test IME is not running" } + Triple(ime, ime.session, ime.connection(targetContext.packageName)) + } + DriveInputMethod.replace(connection, text) { + checkTime() + check(focusedId() == id && selectedIme()) { "native editor focus changed before replacement" } + ui { check(DriveInputMethod.active === ime && ime.session == session) { "native InputConnection was replaced" } } + } + } + + private fun focusedId(): Long? { + val root = uiAutomation.rootInActiveWindow ?: return null + val focused = root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT) ?: return null + if (!focused.isEditable) return null + var node: AccessibilityNodeInfo? = focused + while (node != null) { + tagId(node)?.let { return it } + node = node.parent + } + return null + } + + private fun validate(id: Long, atX: Float? = null, atY: Float? = null): RectF { + val frame = ui { + check(DriveTarget.current === target && !target.activity.isFinishing && target.viewport.isShown) { "target was detached" } + val nodes = target.inspection.snapshot().associateBy { it.id } + val node = checkNotNull(nodes[id]) { "stale node $id" } + check(published[id] == node.incarnation) { "stale incarnation for node $id" } + val parents = nodes.values.flatMap { n -> n.children.map { it to n.id } }.toMap() + var current: LuiInspection.Node? = node + while (current != null) { + check(current.properties["enabled"]?.boolValue != false) { "disabled target $id" } + check(current.properties["display"]?.stringValue != "none" && + current.properties["visibility"]?.stringValue != "hidden" && + (current.properties["opacity"]?.numberValue ?: 1.0) > 0.0) { "hidden target $id" } + current = parents[current.id]?.let { nodes[it] } + } + val bounds = checkNotNull(target.inspection.screenFrames()[id]) { "node $id has no attached geometry" } + check(bounds.intersect(viewport()) && bounds.width() > 0 && bounds.height() > 0) { "node $id is hidden or outside viewport" } + bounds + } + val x = atX ?: frame.centerX(); val y = atY ?: frame.centerY() + check(frame.contains(x, y)) { "point is outside node $id" } + val hitId = hit(x, y) + val descendant = ui { + val nodes = target.inspection.snapshot().associateBy { it.id } + val parents = nodes.values.flatMap { n -> n.children.map { it to n.id } }.toMap() + var candidate: Long? = hitId + while (candidate != null && candidate != id) candidate = parents[candidate] + candidate == id + } + check(descendant) { "node $id is occluded or has no native accessibility hit target" } + return frame + } + + private fun tagId(node: AccessibilityNodeInfo): Long? = node.viewIdResourceName?.removePrefix("lui-drive-")?.toLongOrNull() + + /** Read-only platform window/hit inspection. No accessibility action is ever invoked. */ + private fun hit(x: Float, y: Float): Long { + val windows = uiAutomation.windows.sortedByDescending { it.layer } + for (window in windows) { + val bounds = Rect(); window.getBoundsInScreen(bounds) + if (!bounds.contains(x.toInt(), y.toInt())) continue + val root = window.root ?: error("occluding window has no readable native root") + check(root.packageName?.toString() == targetContext.packageName && window.type == AccessibilityWindowInfo.TYPE_APPLICATION) { "target is occluded by another native window" } + fun descend(node: AccessibilityNodeInfo, inherited: Long?): Long? { + val rect = Rect(); node.getBoundsInScreen(rect) + if (!node.isVisibleToUser || !rect.contains(x.toInt(), y.toInt())) return null + check(node.isEnabled) { "native hit target is disabled" } + val own = tagId(node) ?: inherited + val children = (0 until node.childCount).mapNotNull { index -> node.getChild(index)?.let { index to it } } + .sortedWith(compareByDescending> { it.second.drawingOrder }.thenByDescending { it.first }) + for ((_, child) in children) { + val childBounds = Rect(); child.getBoundsInScreen(childBounds) + if (child.isVisibleToUser && childBounds.contains(x.toInt(), y.toInt())) return descend(child, own) + } + return own + } + return checkNotNull(descend(root, null)) { "native hit is untagged or occluded" } + } + error("no native application window at input point") + } + + private fun gesture(x: Float, y: Float, endX: Float, endY: Float, duration: Long) { + val downTime = SystemClock.uptimeMillis() + var down = false + fun inject(action: Int, atX: Float, atY: Float) { + val event = MotionEvent.obtain(downTime, SystemClock.uptimeMillis(), action, atX, atY, 0) + event.source = InputDevice.SOURCE_TOUCHSCREEN + try { check(uiAutomation.injectInputEvent(event, true)) { "native MotionEvent injection was rejected" } } + finally { event.recycle() } + } + try { + down = true + inject(MotionEvent.ACTION_DOWN, x, y) + val steps = (duration / 16).toInt().coerceAtLeast(1) + for (step in 1..steps) { + pause(duration / steps) + val t = step.toFloat() / steps + if (x != endX || y != endY) inject(MotionEvent.ACTION_MOVE, x + (endX - x) * t, y + (endY - y) * t) + } + inject(MotionEvent.ACTION_UP, endX, endY) + down = false + } finally { + if (down) runCatching { inject(MotionEvent.ACTION_CANCEL, endX, endY) } + } + } + + private fun key(combo: String) { + check(uiAutomation.rootInActiveWindow?.packageName?.toString() == targetContext.packageName) { "target window is not active" } + val tokens = combo.lowercase().split('+') + require(tokens.none { it.isEmpty() } && tokens.distinct().size == tokens.size) { "invalid key combo" } + val modifierCodes = mapOf("ctrl" to KeyEvent.KEYCODE_CTRL_LEFT, "mod" to KeyEvent.KEYCODE_CTRL_LEFT, + "shift" to KeyEvent.KEYCODE_SHIFT_LEFT, "alt" to KeyEvent.KEYCODE_ALT_LEFT, "meta" to KeyEvent.KEYCODE_META_LEFT, + "cmd" to KeyEvent.KEYCODE_META_LEFT) + val metaFlags = mapOf(KeyEvent.KEYCODE_CTRL_LEFT to KeyEvent.META_CTRL_ON, KeyEvent.KEYCODE_SHIFT_LEFT to KeyEvent.META_SHIFT_ON, + KeyEvent.KEYCODE_ALT_LEFT to KeyEvent.META_ALT_ON, KeyEvent.KEYCODE_META_LEFT to KeyEvent.META_META_ON) + val modifiers = tokens.dropLast(1).map { checkNotNull(modifierCodes[it]) { "unsupported key modifier: $it" } } + require(modifiers.distinct().size == modifiers.size) { "duplicate native key modifier" } + val name = tokens.last() + val code = when { + name.length == 1 && name[0] in 'a'..'z' -> KeyEvent.KEYCODE_A + (name[0] - 'a') + name.length == 1 && name[0] in '0'..'9' -> KeyEvent.KEYCODE_0 + (name[0] - '0') + else -> mapOf("enter" to KeyEvent.KEYCODE_ENTER, "tab" to KeyEvent.KEYCODE_TAB, "escape" to KeyEvent.KEYCODE_ESCAPE, + "backspace" to KeyEvent.KEYCODE_DEL, "delete" to KeyEvent.KEYCODE_FORWARD_DEL, "space" to KeyEvent.KEYCODE_SPACE, + "arrowleft" to KeyEvent.KEYCODE_DPAD_LEFT, "arrowright" to KeyEvent.KEYCODE_DPAD_RIGHT, + "arrowup" to KeyEvent.KEYCODE_DPAD_UP, "arrowdown" to KeyEvent.KEYCODE_DPAD_DOWN, + "home" to KeyEvent.KEYCODE_MOVE_HOME, "end" to KeyEvent.KEYCODE_MOVE_END, + "pageup" to KeyEvent.KEYCODE_PAGE_UP, "pagedown" to KeyEvent.KEYCODE_PAGE_DOWN)[name] + ?: error("unsupported native key: $name; use type for Unicode") + } + val held = mutableListOf() + var meta = 0 + val time = SystemClock.uptimeMillis() + fun inject(key: Int, action: Int) { + check(uiAutomation.injectInputEvent(KeyEvent(time, SystemClock.uptimeMillis(), action, key, 0, meta, + -1, 0, 0, InputDevice.SOURCE_KEYBOARD), true)) { "native KeyEvent injection rejected" } + } + try { + modifiers.forEach { modifier -> meta = meta or metaFlags.getValue(modifier); held += modifier; inject(modifier, KeyEvent.ACTION_DOWN) } + held += code + inject(code, KeyEvent.ACTION_DOWN) + } finally { + held.asReversed().forEach { key -> runCatching { inject(key, KeyEvent.ACTION_UP) }; meta = meta and (metaFlags[key] ?: 0).inv() } + } + } + + private fun settle() { + var previous = "" + var quietSince = SystemClock.uptimeMillis() + while (true) { + checkTime() + val state = ui { target.inspection.snapshot().toString() + target.inspection.screenFrames().toString() + target.draws } + val now = SystemClock.uptimeMillis() + if (state != previous) { previous = state; quietSince = now } + if (now - quietSince >= 100) { + uiAutomation.waitForIdle(100, (deadline - now).coerceAtMost(300).coerceAtLeast(1)) + if (ui { target.inspection.snapshot().toString() + target.inspection.screenFrames().toString() + target.draws } == state) return + quietSince = SystemClock.uptimeMillis() + } + pause(16) + } + } + + private fun viewport(): RectF { + val origin = IntArray(2); target.viewport.getLocationOnScreen(origin) + check(target.viewport.width > 0 && target.viewport.height > 0) { "empty target viewport" } + return RectF(origin[0].toFloat(), origin[1].toFloat(), (origin[0] + target.viewport.width).toFloat(), (origin[1] + target.viewport.height).toFloat()) + } + + private fun screenshot(): String { + check(ui { target.activity.window.attributes.flags and android.view.WindowManager.LayoutParams.FLAG_SECURE == 0 }) { + "native screenshot is unsupported for a secure window" + } + val crop = ui { viewport() } + val bitmap = checkNotNull(uiAutomation.takeScreenshot()) { "native screenshot unavailable (secure window?)" } + try { + check(crop.left >= 0 && crop.top >= 0 && crop.right <= bitmap.width && crop.bottom <= bitmap.height) { "viewport is outside screenshot" } + val clipped = Bitmap.createBitmap(bitmap, crop.left.toInt(), crop.top.toInt(), crop.width().toInt(), crop.height().toInt()) + try { + val bytes = ByteArrayOutputStream() + check(clipped.compress(Bitmap.CompressFormat.PNG, 100, bytes)) { "PNG encoding failed" } + return Base64.encodeToString(bytes.toByteArray(), Base64.NO_WRAP) + } finally { if (clipped !== bitmap) clipped.recycle() } + } finally { bitmap.recycle() } + } + + private fun publish(force: Boolean, initial: Boolean = false) { + val (nodes, frames, viewport) = ui { Triple(target.inspection.snapshot(), target.inspection.screenFrames(), viewport()) } + val state = nodes.toString() + frames.toString() + viewport + if (!force && state == lastSnapshot) return + val scale = targetContext.resources.displayMetrics.density + val ops = buildJsonObject { + put("generation", ++generation) + put("ops", buildJsonArray { + published.keys.forEach { id -> add(buildJsonObject { put("op", "drop-node"); put("id", id) }) } + nodes.forEach { node -> + add(buildJsonObject { + put("op", if (node.kind.startsWith("extension:")) "create-extension" else "create-node"); put("id", node.id) + put(if (node.kind.startsWith("extension:")) "identifier" else "kind", node.kind.removePrefix("extension:")) + }) + node.properties.forEach { (name, value) -> add(buildJsonObject { + put("op", "set-prop"); put("id", node.id); put("property", name) + put("value", Json.parseToJsonElement(value.toJsonLiteral())) + }) } + } + nodes.forEach { node -> node.children.forEachIndexed { index, child -> add(buildJsonObject { + put("op", "insert-child"); put("parent", node.id); put("child", child); put("index", index) + }) } } + }) + } + val geometry = buildJsonObject { put("frames", buildJsonArray { + nodes.forEach { node -> frames[node.id]?.let { original -> + val frame = RectF(original) + if (frame.intersect(viewport) && frame.width() > 0 && frame.height() > 0) add(buildJsonArray { + add(JsonPrimitive(node.id)); add(JsonPrimitive((frame.left - viewport.left) / scale)) + add(JsonPrimitive((frame.top - viewport.top) / scale)); add(JsonPrimitive(frame.width() / scale)); add(JsonPrimitive(frame.height() / scale)) + }) + } } + }) } + if (initial) { send(geometry); send(ops) } else { send(ops); send(geometry) } + published = nodes.associate { it.id to it.incarnation } + lastSnapshot = state + } + + private fun send(json: JsonObject) { + val timeout = (deadline - SystemClock.uptimeMillis()).coerceIn(1, 1_000) + val watchdog = transportWatchdog.schedule({ runCatching { socket.close() } }, timeout, TimeUnit.MILLISECONDS) + try { + socket.getOutputStream().write((json.toString() + "\n").toByteArray(Charsets.UTF_8)) + socket.getOutputStream().flush() + } finally { watchdog.cancel(false) } + } + + private fun checkTime() { check(SystemClock.uptimeMillis() < deadline) { "native action timeout" } } + private fun pause(ms: Long) { checkTime(); SystemClock.sleep(ms); checkTime() } + + private fun ui(block: () -> T): T { + check(Looper.myLooper() != Looper.getMainLooper()) { "Drive worker must not run on UI thread" } + val cancelled = AtomicBoolean(false) + val done = CountDownLatch(1) + var result: Result? = null + val task = Runnable { + if (!cancelled.get()) result = runCatching(block) + done.countDown() + } + main.post(task) + val remaining = (deadline - SystemClock.uptimeMillis()).coerceIn(1, 1_000) + if (!done.await(remaining, TimeUnit.MILLISECONDS)) { + cancelled.set(true); main.removeCallbacks(task) + error("UI thread timeout; closing native host") + } + return checkNotNull(result).getOrThrow() + } +} diff --git a/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveProtocol.kt b/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveProtocol.kt new file mode 100644 index 00000000..3dd96c55 --- /dev/null +++ b/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveProtocol.kt @@ -0,0 +1,60 @@ +package dev.lui.drive + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.doubleOrNull +import kotlinx.serialization.json.longOrNull + +internal data class DriveAction(val request: Long, val name: String, val fields: JsonObject) { + fun id(name: String = "id"): Long = (fields.getValue(name) as JsonPrimitive).longOrNull!! + fun number(name: String): Float = (fields.getValue(name) as JsonPrimitive).doubleOrNull!!.toFloat() + fun string(name: String): String = (fields.getValue(name) as JsonPrimitive).content + + companion object { + val capabilities = listOf("click", "double-click", "long-click", "tap", "type", "key", "scroll", "drag", "screenshot", "settle") + fun parse(source: String): DriveAction { + val obj = try { Json.parseToJsonElement(source) as? JsonObject } catch (_: Exception) { null } + requireNotNull(obj) { "invalid action JSON object" } + fun primitive(key: String) = obj[key] as? JsonPrimitive + fun string(key: String): String { + val value = primitive(key) + require(value?.isString == true) { "missing or invalid $key" } + return value.content + } + fun integer(key: String, min: Long): Long { + val value = primitive(key) + val number = value?.takeUnless { it.isString }?.longOrNull + require(number != null && number >= min) { "missing or invalid $key" } + return number + } + require(string("drive") == "action") { "expected drive action" } + require(integer("version", 1) == 1L) { "unsupported Drive version" } + val request = integer("request", 1) + val action = string("action") + val extra = when (action) { + "click", "double-click", "long-click" -> setOf("id") + "type" -> setOf("id", "text") + "key" -> setOf("combo") + "tap" -> setOf("x", "y") + "scroll" -> setOf("id", "dx", "dy") + "drag" -> setOf("id", "target") + "screenshot", "settle" -> emptySet() + else -> throw IllegalArgumentException("unsupported native action: $action") + } + require(obj.keys == setOf("drive", "version", "request", "action") + extra) { "unexpected or missing action fields" } + extra.forEach { key -> + when (key) { + "id", "target" -> integer(key, 0) + "text", "combo" -> string(key) + else -> { + val value = primitive(key) + val number = value?.takeUnless { it.isString }?.doubleOrNull + require(number != null && number.isFinite() && number.toFloat().isFinite()) { "invalid $key" } + } + } + } + return DriveAction(request, action, obj) + } + } +} diff --git a/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveTarget.kt b/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveTarget.kt new file mode 100644 index 00000000..685cb6c3 --- /dev/null +++ b/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveTarget.kt @@ -0,0 +1,44 @@ +package dev.lui.drive + +import android.app.Activity +import android.content.pm.ApplicationInfo +import android.os.Looper +import android.view.View +import android.view.ViewTreeObserver +import dev.lui.LuiBackend +import dev.lui.LuiInspection +import java.lang.ref.WeakReference + +/** Register from a debug Activity after setContent; registration does not open a socket. */ +class DriveTarget private constructor( + val activity: Activity, + val inspection: LuiInspection, +) : AutoCloseable { + val viewport: View = activity.findViewById(android.R.id.content) + internal var draws = 0L + private set + private val drawListener = ViewTreeObserver.OnDrawListener { draws++ } + + init { viewport.viewTreeObserver.addOnDrawListener(drawListener) } + + override fun close() { + check(Looper.myLooper() == Looper.getMainLooper()) + viewport.viewTreeObserver.takeIf { it.isAlive }?.removeOnDrawListener(drawListener) + inspection.close() + if (current === this) reference.clear() + } + + companion object { + private var reference = WeakReference(null) + internal val current: DriveTarget? get() = reference.get() + + fun attach(activity: Activity, backend: LuiBackend): DriveTarget { + check(Looper.myLooper() == Looper.getMainLooper()) + check(activity.applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE != 0) { + "Drive requires a debuggable application" + } + check(current == null) { "Drive target is already registered" } + return DriveTarget(activity, backend.inspect()).also { reference = WeakReference(it) } + } + } +} diff --git a/platform/android/drive/src/main/res/xml/drive_input_method.xml b/platform/android/drive/src/main/res/xml/drive_input_method.xml new file mode 100644 index 00000000..5f57ee0e --- /dev/null +++ b/platform/android/drive/src/main/res/xml/drive_input_method.xml @@ -0,0 +1,2 @@ + diff --git a/platform/android/drive/src/test/kotlin/dev/lui/drive/DriveProtocolTest.kt b/platform/android/drive/src/test/kotlin/dev/lui/drive/DriveProtocolTest.kt new file mode 100644 index 00000000..5fbf5d5c --- /dev/null +++ b/platform/android/drive/src/test/kotlin/dev/lui/drive/DriveProtocolTest.kt @@ -0,0 +1,39 @@ +package dev.lui.drive + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertThrows +import org.junit.Test + +class DriveProtocolTest { + @Test fun strictV1ActionBoundary() { + val prefix = "\"drive\":\"action\",\"version\":1,\"request\":1" + listOf( + "\"action\":\"click\",\"id\":37", + "\"action\":\"double-click\",\"id\":37", + "\"action\":\"long-click\",\"id\":37", + "\"action\":\"tap\",\"x\":1.5,\"y\":0", + "\"action\":\"type\",\"id\":37,\"text\":\"汉字😀\"", + "\"action\":\"key\",\"combo\":\"ctrl+a\"", + "\"action\":\"scroll\",\"id\":37,\"dx\":0,\"dy\":200", + "\"action\":\"drag\",\"id\":37,\"target\":38", + "\"action\":\"screenshot\"", + "\"action\":\"settle\"", + ).forEach { assertEquals(1L, DriveAction.parse("{$prefix,$it}").request) } + listOf( + "{$prefix,\"action\":\"click\",\"id\":\"37\"}", + "{$prefix,\"action\":\"click\",\"id\":37.0}", + "{$prefix,\"action\":\"click\",\"id\":-1}", + "{$prefix,\"action\":\"tap\",\"x\":\"1\",\"y\":2}", + "{$prefix,\"action\":\"tap\",\"x\":1e999,\"y\":2}", + "{$prefix,\"action\":\"type\",\"id\":1,\"text\":true}", + "{$prefix,\"action\":\"click\",\"id\":37,\"event\":\"press\"}", + "{$prefix,\"action\":\"toggle\",\"id\":37}", + "{$prefix,\"action\":\"drag\",\"id\":37}", + "{\"drive\":\"action\",\"version\":2,\"request\":1,\"action\":\"settle\"}", + "{\"drive\":\"action\",\"version\":1,\"request\":0,\"action\":\"settle\"}", + "{\"drive\":\"action\",\"version\":\"1\",\"request\":1,\"action\":\"settle\"}", + ).forEach { source -> + assertThrows(source, IllegalArgumentException::class.java) { DriveAction.parse(source) } + } + } +} diff --git a/platform/android/lui/src/main/kotlin/dev/lui/LuiBackend.kt b/platform/android/lui/src/main/kotlin/dev/lui/LuiBackend.kt index bba3f41b..28c096d0 100644 --- a/platform/android/lui/src/main/kotlin/dev/lui/LuiBackend.kt +++ b/platform/android/lui/src/main/kotlin/dev/lui/LuiBackend.kt @@ -31,6 +31,13 @@ class LuiBackend( private set internal var snapshotEpoch by mutableStateOf(0L) private set + internal var inspection by mutableStateOf(null) + + /** Enable layout tagging only while an explicit native inspection host is attached. */ + fun inspect(): LuiInspection { + check(inspection == null) { "a native inspection host is already attached" } + return LuiInspection(this).also { inspection = it } + } /** Structural half of a published node: kind, parent link and child * order. A batch that only mutates properties leaves this state * untouched, so parent scans of `kind`/`children` are not invalidated @@ -142,6 +149,13 @@ class LuiBackend( * thread and posts the batch for apply+publish here). */ fun applyBatch(batch: LuiPatchBatch): Boolean = try { tree.apply(batch) + inspection?.changed(batch.ops.mapNotNull { + when (it) { + is LuiPatchOp.CreateNode -> it.id + is LuiPatchOp.CreateExtension -> it.id + else -> null + } + }, tree.lastDropped) publish(tree.lastChanged) lastError = null true @@ -163,6 +177,10 @@ class LuiBackend( replacement.applyInitialSnapshot(batch) val observed = observedNodes.keys.toSet() tree = replacement + inspection?.changed( + replacement.nodes.keys.toList() + replacement.extensionNodes.keys, + observed + (inspection?.frames?.keys ?: emptySet()), + ) snapshotEpoch++ publish( changed = observed, diff --git a/platform/android/lui/src/main/kotlin/dev/lui/LuiInspection.kt b/platform/android/lui/src/main/kotlin/dev/lui/LuiInspection.kt new file mode 100644 index 00000000..df843ef1 --- /dev/null +++ b/platform/android/lui/src/main/kotlin/dev/lui/LuiInspection.kt @@ -0,0 +1,75 @@ +package dev.lui + +import android.graphics.RectF +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.ui.Modifier +import androidx.compose.ui.layout.boundsInWindow +import androidx.compose.ui.layout.onGloballyPositioned +import androidx.compose.ui.platform.LocalView +import androidx.compose.ui.semantics.semantics +import androidx.compose.ui.semantics.testTag +import androidx.compose.ui.semantics.testTagsAsResourceId + +/** Optional, read-only native-host inspection. All access must be on the UI thread. */ +class LuiInspection internal constructor(private val backend: LuiBackend) : AutoCloseable { + data class Node( + val id: Long, + val incarnation: Long, + val kind: String, + val properties: Map, + val children: List, + ) + + private var nextIncarnation = 0L + private val incarnations = mutableMapOf() + internal val frames = mutableMapOf() + + internal fun changed(created: List, dropped: Set) { + dropped.forEach { incarnations.remove(it); frames.remove(it) } + created.forEach { incarnations[it] = ++nextIncarnation; frames.remove(it) } + } + + fun snapshot(): List { + val tree = backend.tree + return (tree.nodes.map { (id, node) -> + Node(id, incarnation(id), node.kind.wireName, node.properties.toMap(), node.children.toList()) + } + tree.extensionNodes.map { (id, node) -> + Node(id, incarnation(id), "extension:${node.identifier}", node.properties.toMap(), node.children.toList()) + }).sortedBy { it.id } + } + + private fun incarnation(id: Long): Long = incarnations.getOrPut(id) { ++nextIncarnation } + + /** Clipped bounds in physical screen pixels; wire hosts convert to viewport units. */ + fun screenFrames(): Map = frames.mapValues { RectF(it.value) } + + override fun close() { + if (backend.inspection === this) backend.inspection = null + frames.clear() + incarnations.clear() + } +} + +@Composable +internal fun Modifier.luiInspection(backend: LuiBackend, id: Long): Modifier { + val inspection = backend.inspection ?: return this + val view = LocalView.current + DisposableEffect(inspection, id) { + onDispose { inspection.frames.remove(id) } + } + return semantics { + testTag = "lui-drive-$id" + testTagsAsResourceId = true + }.onGloballyPositioned { coordinates -> + val screen = IntArray(2) + val window = IntArray(2) + view.getLocationOnScreen(screen) + view.getLocationInWindow(window) + val bounds = coordinates.boundsInWindow() + inspection.frames[id] = RectF( + bounds.left + screen[0] - window[0], bounds.top + screen[1] - window[1], + bounds.right + screen[0] - window[0], bounds.bottom + screen[1] - window[1], + ) + } +} diff --git a/platform/android/lui/src/main/kotlin/dev/lui/LuiNodeView.kt b/platform/android/lui/src/main/kotlin/dev/lui/LuiNodeView.kt index d87563d8..f457530a 100644 --- a/platform/android/lui/src/main/kotlin/dev/lui/LuiNodeView.kt +++ b/platform/android/lui/src/main/kotlin/dev/lui/LuiNodeView.kt @@ -197,6 +197,7 @@ private fun LuiChromeSurface(backend: LuiBackend, node: LuiNodeLike, id: Long) { } } else Modifier val modifier = retainedVisibility + .luiInspection(backend, id) .luiChrome(node) .luiGestures(backend, node, id) { offset -> contextMenuAt = offset diff --git a/platform/android/settings.gradle.kts b/platform/android/settings.gradle.kts index 68eac8c2..6e04556a 100644 --- a/platform/android/settings.gradle.kts +++ b/platform/android/settings.gradle.kts @@ -16,3 +16,4 @@ dependencyResolutionManagement { } include(":lui") +include(":drive", ":drive-sample") From 2dcebaf98a0ab33ed7a2adcc86621e06e6f24ee8 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 6 Nov 2026 04:39:48 +0000 Subject: [PATCH 2/6] Reject hidden geometry and unsupported text targets in Android Drive --- .../kotlin/dev/lui/drive/DriveInstrumentation.kt | 15 +++++++++++---- .../lui/src/main/kotlin/dev/lui/LuiInspection.kt | 15 ++++++++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveInstrumentation.kt b/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveInstrumentation.kt index b42563b9..7cbe803c 100644 --- a/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveInstrumentation.kt +++ b/platform/android/drive/src/main/kotlin/dev/lui/drive/DriveInstrumentation.kt @@ -120,7 +120,10 @@ class DriveInstrumentation : Instrumentation() { publish(force = false) continue } - if (byte == -1) return + if (byte == -1) { + check(pending.size() == 0) { "Drive disconnected with an incomplete action line" } + return + } activityAt = SystemClock.uptimeMillis() if (byte == 10) { val line = pending.toString(Charsets.UTF_8.name()) @@ -148,7 +151,8 @@ class DriveInstrumentation : Instrumentation() { settle() if (action.name == "screenshot") png = screenshot() } catch (error: Exception) { - failure = error.message ?: error.javaClass.simpleName + failure = if (error is java.util.concurrent.TimeoutException) "native action timeout waiting for accessibility idle" + else error.message ?: error.javaClass.simpleName } // A failed action still publishes the authoritative post-input state before its result. deadline = SystemClock.uptimeMillis() + 500 @@ -183,8 +187,11 @@ class DriveInstrumentation : Instrumentation() { if (action.name == "type") { check(selectedIme()) { "type requires the explicitly selected Drive test IME" } ui { - val node = target.inspection.snapshot().firstOrNull { it.id == action.id() } - check(node?.properties?.get("read-only")?.boolValue != true) { "target editor is read-only" } + val node = checkNotNull(target.inspection.snapshot().firstOrNull { it.id == action.id() }) { "stale text target" } + check(node.kind in setOf("text-field", "secure-field", "input", "search-field", "textarea", "combobox")) { + "type requires a supported native text editor" + } + check(node.properties["read-only"]?.boolValue != true) { "target editor is read-only" } } } val frame = validate(action.id()) diff --git a/platform/android/lui/src/main/kotlin/dev/lui/LuiInspection.kt b/platform/android/lui/src/main/kotlin/dev/lui/LuiInspection.kt index df843ef1..355a890e 100644 --- a/platform/android/lui/src/main/kotlin/dev/lui/LuiInspection.kt +++ b/platform/android/lui/src/main/kotlin/dev/lui/LuiInspection.kt @@ -42,7 +42,20 @@ class LuiInspection internal constructor(private val backend: LuiBackend) : Auto private fun incarnation(id: Long): Long = incarnations.getOrPut(id) { ++nextIncarnation } /** Clipped bounds in physical screen pixels; wire hosts convert to viewport units. */ - fun screenFrames(): Map = frames.mapValues { RectF(it.value) } + fun screenFrames(): Map { + val nodes = snapshot().associateBy { it.id } + val parents = nodes.values.flatMap { node -> node.children.map { it to node.id } }.toMap() + val visible = mutableMapOf() + fun isVisible(id: Long): Boolean = visible.getOrPut(id) { + val node = nodes[id] + node != null && node.properties["display"]?.stringValue != "none" && + node.properties["visibility"]?.stringValue != "hidden" && + (node.properties["opacity"]?.numberValue ?: 1.0) > 0.0 && + !(node.kind == "virtual-list" && node.properties["selected"]?.boolValue == false) && + (parents[id]?.let { isVisible(it) } ?: true) + } + return frames.filterKeys { isVisible(it) }.mapValues { RectF(it.value) } + } override fun close() { if (backend.inspection === this) backend.inspection = null From 4e9f20fa7941bb441ffa154ba24815997cc1f1fa Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 6 Nov 2026 04:41:46 +0000 Subject: [PATCH 3/6] Gate Android Drive host and sample builds in CI --- .github/workflows/ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ce53929..1baad6fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -228,10 +228,17 @@ jobs: packages: platform-tools - uses: gradle/actions/setup-gradle@v4 - name: Install Android compile SDK - run: sdkmanager 'platforms;android-37.0' 'build-tools;36.0.0' + run: sdkmanager 'platforms;android-37.0' 'build-tools;36.0.0' 'build-tools;37.0.0' - name: Android store and actual Compose regressions working-directory: platform/android run: ./gradlew --no-daemon :lui:testDebugUnitTest + - name: Android Drive protocol, sample builds and scoped lint + working-directory: platform/android + run: >- + ./gradlew --no-daemon + :drive:testDebugUnitTest + :drive-sample:assembleDebug :drive-sample:assembleRelease + :drive:lintDebug :drive-sample:lintDebug :lui:lintDebug c-bridge-stress: runs-on: ubuntu-latest From f819db0b85c40d2130998f618fb1b0f9824fc451 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 6 Nov 2026 04:43:21 +0000 Subject: [PATCH 4/6] Add opt-in DEBUG iOS read-only Drive snapshot probe --- platform/apple/DriveProbe.md | 142 +++++++++++++ .../LUIAppleBackend/LUIAppleBackend.swift | 55 ++++++ .../LUIDriveSnapshotProbe.swift | 187 ++++++++++++++++++ 3 files changed, 384 insertions(+) create mode 100644 platform/apple/DriveProbe.md create mode 100644 platform/apple/Sources/LUIAppleBackend/LUIDriveSnapshotProbe.swift diff --git a/platform/apple/DriveProbe.md b/platform/apple/DriveProbe.md new file mode 100644 index 00000000..117f88cc --- /dev/null +++ b/platform/apple/DriveProbe.md @@ -0,0 +1,142 @@ +# DEBUG iOS Drive snapshot probe (source-only) + +The new API and probe are source implementation awaiting Xcode compilation and +Simulator validation. They are excluded by `#if DEBUG && os(iOS)` from Release, +macOS, and other builds. No app starts a listener automatically. + +## Public API + +```swift +@MainActor +public func LUIAppleBackend.debugDriveSnapshot() throws -> Data + +@MainActor +public static func LUIDriveSnapshotProbe.startIfEnabled( + backend: LUIAppleBackend, window: UIWindow, + environment: [String: String] = ProcessInfo.processInfo.environment +) throws -> LUIDriveSnapshotProbe? + +// On the retained probe: +public private(set) var failure: String? +public func stop() +``` + +These are additive DEBUG iOS APIs; production APIs, wire events, and input +defaults do not change. Downstream callers must gate references with `#if DEBUG` +(and `os(iOS)` if sharing source with macOS). Do not expose backend models, +retained tree objects, or mutable frame tables to the UI-test process. + +`debugDriveSnapshot()` returns an immutable JSON data snapshot of the current +retained standard/extension nodes, properties (existing scalar wire types), +child order, roots, generation, commit sequence/pending-commit state, collected +frames, and effective accessibility identifiers. Standard identifiers follow +the renderer's explicit identifier then accessibility-label fallback. The +metadata distinguishes explicit identifiers; Drive target actions require those, +not fallback labels. Extension metadata reflects declared identifier properties; +the XCTest runner must validate that the actual native extension control exposes +that identifier uniquely. Removed nodes are excluded from frame output. + +The data API itself is read-only and does not force pending commits, enable +geometry probes, change callbacks, or dispatch input/events. The standalone +probe temporarily enables frame reporting and resumes collection, leaves +`onFramesReport` untouched, and restores the saved flags on `stop()` or isolated +deinitialization. Only one probe may be active per backend. Retain the probe for +the desired debug session and stop it during host teardown; do not create it as +an unretained temporary. + +## Navigation demo host hookup (no view edits) + +In `examples/navigation/ios-swiftui/Sources/NavigationDemo/NavigationDemo.swift`, +add `import UIKit`, and put this stored property in `NavigationHost`: + +```swift +#if DEBUG +private var driveProbe: LUIDriveSnapshotProbe? +#endif +``` + +At the end of `NavigationHost.start()`, after `startOCaml(...)` succeeds, add: + +```swift +#if DEBUG +if ProcessInfo.processInfo.environment["LUI_DRIVE_PROBE"] == "1" { + let windows = UIApplication.shared.connectedScenes + .compactMap { $0 as? UIWindowScene } + .filter { $0.activationState == .foregroundActive } + .flatMap(\.windows) + .filter(\.isKeyWindow) + guard windows.count == 1, let window = windows.first else { + preconditionFailure("Drive requires one foreground key window") + } + do { + driveProbe = try LUIDriveSnapshotProbe.startIfEnabled(backend: backend, window: window) + } catch { + preconditionFailure("Drive probe failed: \(error)") + } +} +#endif +``` + +This is a downstream integration snippet, **not an already-applied demo change**. +The app must already have its foreground key window when `start()` runs; hosts +with a different lifecycle should call the factory at their window-ready hook. +If the host has explicit shutdown, call `driveProbe?.stop()` there. The existing +LUI geometry probes report SwiftUI scene-global/window-local logical points; +use one fixed-orientation window. Hoisted toolbar nodes intentionally do not +report SwiftUI geometry; the native XCTest adapter obtains their actual frames. +No root view or OCaml `Lui_elements` view change is needed. + +Enable only through the UI test's app launch environment: + +```swift +app.launchEnvironment["LUI_DRIVE_PROBE"] = "1" +app.launchEnvironment["LUI_DRIVE_PROBE_PORT"] = "49378" // optional default +``` + +Do not add these to production launch defaults. Ports must be `1024...65535`. +The factory returns `nil` without `LUI_DRIVE_PROBE=1`. Network listener failures +set `failure` and stop/restore the probe; inspect this in the integrating host's +debug diagnostics. A missing/hidden/non-key window rejects the snapshot request. + +## Private probe transport + +The probe binds Foundation Network TCP strictly to IPv4 `127.0.0.1`, accepts a +single loopback peer, and implements only a read request (not Drive actions): + +```json +{"probe":"snapshot","version":1,"request":1} +``` + +It responds with the same probe/version/request and a `snapshot` object that +also includes `viewport:[0,0,width,height]` and +`coordinateSpace:"window-top-left-points"`. The runner maps native screen frames +through the matching XCTest window before publishing Drive frame snapshots. +No physical screenshot scaling is applied. + +Requests start at 1 and increase by exactly 1 on each connection. Non-snapshot, +malformed/version-mismatched, duplicate/out-of-order, pipelined requests and +non-loopback peers are rejected by closing that connection. Input is limited to +4096 bytes; a partial request/response has a 5s absolute deadline that is not +extended by each received chunk. An otherwise idle connection is allowed. + +The probe is not an authenticated service: another local process can read the +opt-in debug app's retained content. Use isolated development environments. It +has no injection command, shell launch, private HID/touch API, event dispatch, +or native input responsibility. The separate Drive XCTest UI-testing target +owns all input and must never be linked into the production application. + +## Migration-note text for the integration owner + +Add an entry to the repository-level API migrations document when integrating: + +> Apple DEBUG iOS builds add a read-only `debugDriveSnapshot()` data API and an +> explicit `LUIDriveSnapshotProbe.startIfEnabled(backend:window:environment:)` +> lifecycle. Retain the probe at a single-window-ready host hook; gate references +> with `#if DEBUG`. Without `LUI_DRIVE_PROBE=1` no listener is opened; Release +> builds contain no probe/API. The loopback-only probe temporarily enables frame +> collection and restores settings without replacing callbacks. It exposes LUI +> wire snapshots and expected/effective accessibility identifiers, not input. +> Native Drive input belongs exclusively to a separate XCTest UI-testing target +> with explicit stable identifiers; there is no semantic or coordinate fallback +> for target actions. Existing production behavior and event contracts are +> unchanged. Xcode compilation/Simulator verification remain required. diff --git a/platform/apple/Sources/LUIAppleBackend/LUIAppleBackend.swift b/platform/apple/Sources/LUIAppleBackend/LUIAppleBackend.swift index 414f35d2..edb95169 100644 --- a/platform/apple/Sources/LUIAppleBackend/LUIAppleBackend.swift +++ b/platform/apple/Sources/LUIAppleBackend/LUIAppleBackend.swift @@ -653,6 +653,61 @@ public final class LUIAppleBackend { private var nodeFrames: [Int: CGRect] = [:] private var framesReportScheduled = false +#if DEBUG && os(iOS) + var debugDriveProbeActive = false + + /// A read-only retained-tree snapshot for the opt-in XCTest probe. + /// Frames are SwiftUI global/window logical points, never screenshot pixels. + public func debugDriveSnapshot() throws -> Data { + var ops: [[String: Any]] = [] + var identifiers: [String: [String: Any]] = [:] + let ids = (Array(tree.nodes.keys) + Array(tree.extensionNodes.keys)).sorted() + for id in ids { + if let node = tree.nodes[id] { + ops.append(["op": "create-node", "id": id, "kind": node.kind.rawValue]) + for (property, value) in node.properties.sorted(by: { $0.key.rawValue < $1.key.rawValue }) { + ops.append(["op": "set-prop", "id": id, "property": property.rawValue, + "value": value.debugDriveJSONValue]) + } + let explicit = node.properties[.accessibilityIdentifier]?.stringValue + if let effective = explicit ?? node.properties[.accessibilityLabel]?.stringValue { + identifiers[String(id)] = ["identifier": effective, "explicit": explicit != nil] + } + } else if let node = tree.extensionNodes[id] { + ops.append(["op": "create-extension", "id": id, + "identifier": node.identifier, "fingerprint": node.fingerprint]) + for (property, value) in node.properties.sorted(by: { $0.key < $1.key }) { + ops.append(["op": "set-extension-prop", "id": id, "property": property, + "value": value.debugDriveJSONValue]) + } + let explicit = node.properties["accessibility-identifier"]?.stringValue + if let effective = explicit ?? node.properties["accessibility-label"]?.stringValue { + identifiers[String(id)] = ["identifier": effective, "explicit": explicit != nil] + } + } + } + for id in ids { + let children = tree.nodes[id]?.children ?? tree.extensionNodes[id]?.children ?? [] + for (index, child) in children.enumerated() { + ops.append(["op": "insert-child", "parent": id, "child": child, "index": index]) + } + } + let frames: [[Any]] = ids.compactMap { id in + guard let rect = nodeFrames[id], !rect.isNull, !rect.isInfinite, + rect.minX.isFinite, rect.minY.isFinite, + rect.width.isFinite, rect.height.isFinite else { return nil } + return [id, rect.minX, rect.minY, rect.width, rect.height] + } + return try JSONSerialization.data(withJSONObject: [ + "generation": generation, "commitSequence": commitSequence, + "committed": pendingCommitTouched.isEmpty && pendingCommitDropped.isEmpty && + !pendingCommitStructural && pendingCommitModalRelevant.isEmpty, + "ops": ops, "nodeIDs": ids, "roots": tree.rootIDs.sorted(), + "frames": frames, "nativeIdentifiers": identifiers, + ], options: [.sortedKeys]) + } +#endif + // MARK: - file-picker operations /// In-flight and completed file-picker requests keyed by node id. Held diff --git a/platform/apple/Sources/LUIAppleBackend/LUIDriveSnapshotProbe.swift b/platform/apple/Sources/LUIAppleBackend/LUIDriveSnapshotProbe.swift new file mode 100644 index 00000000..4767c6ff --- /dev/null +++ b/platform/apple/Sources/LUIAppleBackend/LUIDriveSnapshotProbe.swift @@ -0,0 +1,187 @@ +#if DEBUG && os(iOS) +import Foundation +import CoreFoundation +import Network +import UIKit + +extension LUIWireValue { + var debugDriveJSONValue: Any { + switch self { + case let .string(value): value + case let .bool(value): value + case let .int(value): value + case let .double(value): value + } + } +} + +/// Read-only, single-client loopback probe. Retain it in a DEBUG app host only. +@MainActor +public final class LUIDriveSnapshotProbe { + public private(set) var failure: String? + private let backend: LUIAppleBackend + private weak var window: UIWindow? + private let savedReporting: Bool + private let savedSuspension: Bool + private var listener: NWListener? + private var connection: NWConnection? + private var buffer = Data() + private var request = 0 + private var timeout: Task? + private var stopped = false + + isolated deinit { stop() } + + /// No listener is created unless LUI_DRIVE_PROBE=1 is explicitly supplied. + public static func startIfEnabled( + backend: LUIAppleBackend, window: UIWindow, + environment: [String: String] = ProcessInfo.processInfo.environment + ) throws -> LUIDriveSnapshotProbe? { + guard environment["LUI_DRIVE_PROBE"] == "1" else { return nil } + let rawPort = environment["LUI_DRIVE_PROBE_PORT"] ?? "49378" + guard let port = UInt16(rawPort), port >= 1024, + !backend.debugDriveProbeActive else { + throw LUIBackendError.invalidBatch("invalid Drive probe port or probe already active") + } + return try LUIDriveSnapshotProbe(backend: backend, window: window, port: port) + } + + private init(backend: LUIAppleBackend, window: UIWindow, port: UInt16) throws { + self.backend = backend + self.window = window + savedReporting = backend.frameReportingEnabled + savedSuspension = backend.frameCollectionSuspended + let parameters = NWParameters.tcp + parameters.requiredLocalEndpoint = .hostPort( + host: .ipv4(.loopback), port: NWEndpoint.Port(rawValue: port)! + ) + let listener = try NWListener(using: parameters) + self.listener = listener + backend.debugDriveProbeActive = true + backend.frameCollectionSuspended = false + backend.frameReportingEnabled = true + listener.newConnectionHandler = { [weak self] connection in + Task { @MainActor in self?.accept(connection) } + } + listener.stateUpdateHandler = { [weak self] state in + Task { @MainActor in + if case let .failed(error) = state { + self?.failure = String(describing: error) + self?.stop() + } + } + } + listener.start(queue: .main) + } + + /// Call on host teardown. The pre-existing frame callback is never replaced. + public func stop() { + guard !stopped else { return } + stopped = true + closeClient() + listener?.cancel() + listener = nil + backend.frameReportingEnabled = savedReporting + backend.frameCollectionSuspended = savedSuspension + backend.debugDriveProbeActive = false + } + + private func accept(_ client: NWConnection) { + guard !stopped, connection == nil, + case let .hostPort(host, _) = client.endpoint, + host == .ipv4(.loopback) else { client.cancel(); return } + connection = client + buffer.removeAll(keepingCapacity: true) + request = 0 + client.stateUpdateHandler = { [weak self, weak client] state in + Task { @MainActor in + guard let self, let client, self.connection === client else { return } + switch state { + case .failed, .cancelled: self.closeClient() + default: break + } + } + } + client.start(queue: .main) + receive(client) + } + + private func closeClient() { + timeout?.cancel() + timeout = nil + connection?.cancel() + connection = nil + buffer.removeAll(keepingCapacity: true) + } + + private struct Request: Decodable { + let probe: String + let version: Int + let request: Int + } + + private func receive(_ client: NWConnection) { + client.receive(minimumIncompleteLength: 1, maximumLength: 4096) { [weak self] data, _, done, error in + Task { @MainActor in + guard let self, self.connection === client else { return } + if let data { self.buffer.append(data) } + if self.timeout == nil, !self.buffer.isEmpty { + self.timeout = Task { [weak self, weak client] in + try? await Task.sleep(for: .seconds(5)) + guard !Task.isCancelled, let self, let client, + self.connection === client else { return } + self.closeClient() + } + } + guard error == nil, !done, self.buffer.count <= 4096 else { + self.closeClient(); return + } + guard let newline = self.buffer.firstIndex(of: 10) else { + self.receive(client); return + } + let line = self.buffer.prefix(upTo: newline) + self.buffer.removeSubrange(...newline) + do { + let command = try JSONDecoder().decode(Request.self, from: line) + guard let raw = try JSONSerialization.jsonObject(with: line) as? [String: Any], + Set(raw.keys) == Set(["probe", "version", "request"]), + ["version", "request"].allSatisfy({ key in + guard let number = raw[key] as? NSNumber else { return false } + return CFGetTypeID(number) != CFBooleanGetTypeID() && + !["f", "d"].contains(String(cString: number.objCType)) + }) else { self.closeClient(); return } + guard command.probe == "snapshot", command.version == 1, + self.request < Int.max, command.request == self.request + 1, + self.buffer.isEmpty, let window = self.window, + window.isKeyWindow, window.windowScene?.activationState == .foregroundActive, + window.windowScene?.windows.filter({ !$0.isHidden && $0.windowLevel == .normal }).count == 1 + else { self.closeClient(); return } + self.request = command.request + guard var snapshot = try JSONSerialization.jsonObject( + with: self.backend.debugDriveSnapshot() + ) as? [String: Any] else { self.closeClient(); return } + snapshot["viewport"] = [0, 0, window.bounds.width, window.bounds.height] + snapshot["coordinateSpace"] = "window-top-left-points" + var bytes = try JSONSerialization.data(withJSONObject: [ + "probe": "snapshot", "version": 1, "request": command.request, + "snapshot": snapshot, + ], options: [.sortedKeys]) + bytes.append(10) + client.send(content: bytes, completion: .contentProcessed { [weak self] error in + Task { @MainActor in + guard let self, self.connection === client else { return } + if error != nil { self.closeClient() } + else { + self.timeout?.cancel() + self.timeout = nil + self.receive(client) + } + } + }) + } catch { self.closeClient() } + } + } + } + +} +#endif From 3db7a7e8cf186336434abd3e75bb3f2d118e5d07 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 6 Nov 2026 04:44:06 +0000 Subject: [PATCH 5/6] Document optional Android native Drive integration --- README.md | 15 +++++++++++++ docs/api-migrations.md | 50 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/README.md b/README.md index 25de33d3..4568bc6b 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,21 @@ Per-platform backend builds and test recipes are documented under each `platform/` directory (the Gradle wrapper under `platform/android` for Android). +### Native Drive testing + +[Drive](https://github.com/logseq/drive) keeps fast semantic/headless tests +separate from native input. Android has an optional, explicitly launched +[instrumentation host](platform/android/drive/README.md) for API 30+ debug +variants. It injects OS MotionEvent/KeyEvent input and uses a deliberately +selected test IME for Unicode/empty text replacement; it never emits LUI +semantic events as a fallback. + +With the host running and an explicit `adb forward`, use a Drive version +that supports `drive --tcp 127.0.0.1:49377 --input native scenario.drive`. +The host, IME, permission and runner must remain outside release artifacts. +Its compile/protocol/lint checks do not establish real-device input behavior; +see the host guide for setup, cleanup, geometry and settling limits. + ## Component schema `schema/components.json` is the single source of truth for node kinds, diff --git a/docs/api-migrations.md b/docs/api-migrations.md index ba89e0b1..885e2f50 100644 --- a/docs/api-migrations.md +++ b/docs/api-migrations.md @@ -1,5 +1,55 @@ # LUI API migration guide +## Unreleased: Optional Android native Drive host + +Android renderer defaults and existing semantic integrations are unchanged. +`LuiBackend.inspect()` is a new opt-in, UI-thread-only read-only inspection +API. Close its `LuiInspection` handle to remove inspection semantics and +layout observers. Snapshots copy retained nodes and carry host-local +incarnation guards; native targeting requires attached, visible geometry. + +For an API 30+ debuggable test variant, add +`debugImplementation(project(":drive"))`, register +`DriveTarget.attach(activity, backend)` after `setContent` behind an explicit +debug intent flag, and close the handle in `onDestroy`. Declare +`dev.lui.drive.DriveInstrumentation` only in the debug manifest. Keep the +optional library, INTERNET permission, runner and test IME out of release +artifacts. The sample supplies separate debug/release hooks. + +```sh +# The debug app must already be installed; this clears its Activity task. +adb forward tcp:49377 tcp:49377 +adb shell am instrument -w -e drive true \ + -e activity your.debug.package/.MainActivity \ + your.debug.package/dev.lui.drive.DriveInstrumentation +# In another terminal, with a Drive version supporting loopback TCP: +drive --tcp 127.0.0.1:49377 --input native scenario.drive +# After the run: +adb forward --remove tcp:49377 +``` + +Native protocol v1 is unchanged: initial hello, full frames, then ops establish +readiness; fresh ops and full frames precede each correlated result. Wire +coordinates/deltas are content-viewport dp, converted to screen pixels at OS +MotionEvent/KeyEvent injection. No action falls back to LUI semantic events. + +Text replacement uses a real InputConnection and requires deliberately +selecting the test-device IME; `type` is not advertised otherwise. The host +never enables/selects an IME itself. Preserve and restore the original IME, +or use the explicit `--test-device` sample launcher. Missing selection or +geometry, stale incarnations, disabled/hidden targets, detectable native +occlusion and unsupported input fail explicitly. Offscreen targets need +native scrolling; there is no selector auto-scroll. + +Settling is bounded model/layout/draw/accessibility quiet, not application or +network idle, or proof of every render-thread animation. Accessibility cannot +prove the behavior of arbitrary invisible custom touch interceptors; +unmeasured modals/extensions and system UI need separate instrumentation. +Already running input cannot be rolled back. Compile/protocol/lint checks +are not device-input evidence. See +[`platform/android/drive/README.md`](../platform/android/drive/README.md) +for complete integration, input and cleanup limits. + ## Unreleased: Web submenu placement through layout wrappers Web dropdowns owned by a `menu-trigger` or `menu-item` now find their semantic From 3e5dce6bce8b42b3421c2fa08b64ffe226a48328 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 6 Nov 2026 04:46:02 +0000 Subject: [PATCH 6/6] Document DEBUG iOS Drive probe lifecycle --- README.md | 5 +++++ docs/api-migrations.md | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/README.md b/README.md index 4568bc6b..ba73d061 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,11 @@ The host, IME, permission and runner must remain outside release artifacts. Its compile/protocol/lint checks do not establish real-device input behavior; see the host guide for setup, cleanup, geometry and settling limits. +For iOS, a DEBUG-only, opt-in [read-only snapshot probe](platform/apple/DriveProbe.md) +connects to Drive's separate XCTest UI-testing runner. The probe does not inject +input. This integration still needs Apple-SDK compilation and Simulator/device +validation; adding the sources is not evidence of native input support. + ## Component schema `schema/components.json` is the single source of truth for node kinds, diff --git a/docs/api-migrations.md b/docs/api-migrations.md index 885e2f50..88e7aca7 100644 --- a/docs/api-migrations.md +++ b/docs/api-migrations.md @@ -1,5 +1,37 @@ # LUI API migration guide +## Unreleased: DEBUG iOS read-only Drive probe + +DEBUG iOS builds add `LUIAppleBackend.debugDriveSnapshot()` and +`LUIDriveSnapshotProbe.startIfEnabled(backend:window:environment:)`. These APIs +are absent in Release and on other platforms. Retain the probe after the +single foreground key window is ready, and call `stop()` during host teardown. +Wrap host references in `#if DEBUG`: + +```swift +#if DEBUG +driveProbe = try LUIDriveSnapshotProbe.startIfEnabled( + backend: backend, window: window +) +#endif +``` + +No listener opens unless `LUI_DRIVE_PROBE=1` is explicitly supplied. The probe +temporarily enables frame collection and restores its settings without +replacing existing callbacks. It binds only IPv4 loopback (default port 49378) +and accepts versioned read-only snapshot requests; it has no Drive action or +semantic event endpoint. This is debug access for trusted local processes, +not authentication against other local applications. + +Native input belongs to the separate Drive XCTest UI-testing target, never +the app binary. Selector actions require explicit stable native accessibility +identifiers and do not fall back to labels, coordinates or semantic events. +Frame coordinates are window-local logical points, not screenshot pixels. +Existing production event behavior and transport contracts do not change. +See [`platform/apple/DriveProbe.md`](../platform/apple/DriveProbe.md) for a +complete host hookup, protocol and lifecycle example. Apple-SDK compilation +and Simulator/device input verification are separate integration gates. + ## Unreleased: Optional Android native Drive host Android renderer defaults and existing semantic integrations are unchanged.