liuyu/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-06-06 05:51:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: harden mobile a2ui bridge trust

Browse Source
This commit is contained in:
joshavant
2026-06-03 16:49:48 -07:00
parent 0b6fff44f5
commit d5d3e9983e
24 changed files with 12946 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
apps/android/README.md
View File

@@ -253,9 +253,9 @@ Pre-req checklist:
4) Open the app **Screen** tab and keep it active during the run (canvas/A2UI commands require the canvas WebView attached there).
5) Grant runtime permissions for capabilities you expect to pass (camera/mic/location/notification listener/location, etc.).
6) No interactive system dialogs should be pending before test start.
7) Canvas host is enabled and reachable from the device (do not run gateway with `OPENCLAW_SKIP_CANVAS_HOST=1`; startup logs should include `canvas host mounted at .../__openclaw__/`).
7) Canvas host is enabled and reachable from the device for remote Canvas checks (do not run gateway with `OPENCLAW_SKIP_CANVAS_HOST=1`; startup logs should include `canvas host mounted at .../__openclaw__/`).
8) Local operator test client pairing is approved. If first run fails with `pairing required`, preview the latest pending request, approve the printed request ID, then rerun:
9) For A2UI checks, keep the app on **Screen** tab; the node now auto-refreshes canvas capability once on first A2UI reachability failure (TTL-safe retry).
9) For A2UI checks, keep the app on **Screen** tab; the node uses its bundled app-owned A2UI page for message application.
```bash
openclaw devices list
@@ -287,8 +287,8 @@ Common failure quick-fixes:
- `pairing required` before tests start:
- list pending requests (`openclaw devices list`), then approve with the exact ID (`openclaw devices approve <requestId>`) and rerun.
- `A2UI host not reachable` / `A2UI_HOST_NOT_CONFIGURED`:
- ensure the Canvas plugin host is running and reachable, keep the app on the **Screen** tab. The app refreshes the Canvas plugin surface URL once before failing; if it still fails, reconnect app and rerun.
- `A2UI host not reachable` / `A2UI_HOST_UNAVAILABLE`:
- keep the app foregrounded on the **Screen** tab and rerun. A2UI commands use the bundled app-owned A2UI page; the Gateway Canvas host is still needed for remote Canvas checks, but not for A2UI message application.
- `NODE_BACKGROUND_UNAVAILABLE: canvas unavailable`:
- app is not effectively ready for canvas commands; keep app foregrounded and **Screen** tab active.

3
apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt
View File

@@ -189,8 +189,6 @@ class NodeRuntime(
A2UIHandler(
canvas = canvas,
json = json,
getNodeCanvasHostUrl = { nodeSession.currentCanvasHostUrl() },
getOperatorCanvasHostUrl = { operatorSession.currentCanvasHostUrl() },
)
private val connectionManager: ConnectionManager =
@@ -254,7 +252,6 @@ class NodeRuntime(
_canvasRehydrateErrorText.value = null
},
onCanvasA2uiReset = { _canvasA2uiHydrated.value = false },
refreshCanvasHostUrl = { nodeSession.refreshCanvasHostUrl() },
motionActivityAvailable = { motionHandler.isActivityAvailable() },
motionPedometerAvailable = { motionHandler.isPedometerAvailable() },
)

45
apps/android/app/src/main/java/ai/openclaw/app/node/A2UIHandler.kt
View File

@@ -12,47 +12,30 @@ import kotlinx.serialization.json.JsonPrimitive
class A2UIHandler(
private val canvas: CanvasController,
private val json: Json,
private val getNodeCanvasHostUrl: () -> String?,
private val getOperatorCanvasHostUrl: () -> String?,
) {
fun isTrustedCanvasActionUrl(rawUrl: String?): Boolean =
CanvasActionTrust.isTrustedCanvasActionUrl(
rawUrl = rawUrl,
trustedA2uiUrls = listOfNotNull(resolveA2uiHostUrl()),
)
fun isTrustedCanvasActionUrl(rawUrl: String?): Boolean = CanvasActionTrust.isTrustedCanvasActionUrl(rawUrl)
fun resolveA2uiHostUrl(): String? {
val nodeRaw = getNodeCanvasHostUrl()?.trim().orEmpty()
val operatorRaw = getOperatorCanvasHostUrl()?.trim().orEmpty()
// Prefer node-advertised canvas host; operator URL is a fallback for older hello payloads.
val raw = if (nodeRaw.isNotBlank()) nodeRaw else operatorRaw
if (raw.isBlank()) return null
val base = raw.trimEnd('/')
return "$base/__openclaw__/a2ui/?platform=android"
}
suspend fun ensureA2uiReady(a2uiUrl: String): Boolean {
try {
val already = canvas.eval(a2uiReadyCheckJS)
if (already == "true") return true
} catch (_: Throwable) {
// ignore
suspend fun ensureA2uiReady(): Boolean {
if (canvas.currentUrl()?.trim() == CanvasActionTrust.localA2uiAssetUrl && isA2uiReady()) {
return true
}
canvas.navigate(a2uiUrl)
// A2UI host bootstraps asynchronously after navigation; poll briefly before failing the command.
canvas.showLocalA2ui()
// The bundled A2UI host bootstraps asynchronously after navigation; poll briefly before failing the command.
repeat(50) {
try {
val ready = canvas.eval(a2uiReadyCheckJS)
if (ready == "true") return true
} catch (_: Throwable) {
// ignore
}
if (isA2uiReady()) return true
delay(120)
}
return false
}
private suspend fun isA2uiReady(): Boolean =
try {
canvas.eval(a2uiReadyCheckJS) == "true"
} catch (_: Throwable) {
false
}
fun decodeA2uiMessages(
command: String,
paramsJson: String?,

63
apps/android/app/src/main/java/ai/openclaw/app/node/CanvasActionTrust.kt
View File

@@ -1,7 +1,5 @@
package ai.openclaw.app.node
import java.net.URI
/**
* Trust helper for WebView-originated canvas/A2UI actions.
*/
@@ -9,62 +7,15 @@ object CanvasActionTrust {
/** Local canvas scaffold is the only trusted file URL. */
const val scaffoldAssetUrl: String = "file:///android_asset/CanvasScaffold/scaffold.html"
/** Accepts local scaffold or exact remote A2UI URLs advertised by the gateway. */
fun isTrustedCanvasActionUrl(
rawUrl: String?,
trustedA2uiUrls: List<String>,
): Boolean {
/** Local bundled A2UI is the only action-capable A2UI host. */
const val localA2uiAssetUrl: String = "file:///android_asset/CanvasA2UI/index.html"
/** Accepts only app-owned bundled pages. Remote WebView content is render-only. */
fun isTrustedCanvasActionUrl(rawUrl: String?): Boolean {
val candidate = rawUrl?.trim().orEmpty()
if (candidate.isEmpty()) return false
if (candidate == scaffoldAssetUrl) return true
val candidateUri = parseUri(candidate) ?: return false
if (candidateUri.scheme.equals("file", ignoreCase = true)) {
return false
}
val normalizedCandidate = normalizeTrustedRemoteA2uiUri(candidateUri) ?: return false
return trustedA2uiUrls.any { trusted ->
matchesTrustedRemoteA2uiUrlExact(normalizedCandidate, trusted)
}
if (candidate == localA2uiAssetUrl) return true
return false
}
private fun matchesTrustedRemoteA2uiUrlExact(
candidateUri: URI,
trustedUrl: String,
): Boolean {
// Gateway-advertised URLs are capabilities. Treat malformed entries as
// absent instead of broadening trust to same-origin or prefix matches.
val trustedUri = parseUri(trustedUrl) ?: return false
val normalizedTrusted = normalizeTrustedRemoteA2uiUri(trustedUri) ?: return false
return candidateUri == normalizedTrusted
}
/** Normalizes only the URL parts allowed to vary across trusted remote A2UI URLs. */
private fun normalizeTrustedRemoteA2uiUri(uri: URI): URI? {
// Keep Android trust normalization aligned with iOS ScreenController:
// exact remote URL match, scheme/host normalized, fragment ignored.
val scheme = uri.scheme?.lowercase() ?: return null
if (scheme != "http" && scheme != "https") return null
val host =
uri.host
?.trim()
?.takeIf { it.isNotEmpty() }
?.lowercase() ?: return null
return try {
URI(scheme, uri.userInfo, host, uri.port, uri.rawPath, uri.rawQuery, null)
} catch (_: Throwable) {
null
}
}
/** Parses untrusted WebView/gateway URL text without throwing into UI event handlers. */
private fun parseUri(raw: String): URI? =
try {
URI(raw)
} catch (_: Throwable) {
null
}
}

10
apps/android/app/src/main/java/ai/openclaw/app/node/CanvasController.kt
View File

@@ -48,7 +48,8 @@ class CanvasController {
private val _currentUrl = MutableStateFlow<String?>(null)
val currentUrl: StateFlow<String?> = _currentUrl.asStateFlow()
private val scaffoldAssetUrl = "file:///android_asset/CanvasScaffold/scaffold.html"
private val scaffoldAssetUrl = CanvasActionTrust.scaffoldAssetUrl
private val localA2uiAssetUrl = CanvasActionTrust.localA2uiAssetUrl
private fun clampJpegQuality(quality: Double?): Int {
val q = (quality ?: 0.82).coerceIn(0.1, 1.0)
@@ -87,6 +88,13 @@ class CanvasController {
reload()
}
/** Shows the app-owned A2UI renderer that is allowed to dispatch native actions. */
fun showLocalA2ui() {
this.url = localA2uiAssetUrl
_currentUrl.value = localA2uiAssetUrl
reload()
}
fun currentUrl(): String? = url
fun isDefaultCanvas(): Boolean = url == null

24
apps/android/app/src/main/java/ai/openclaw/app/node/InvokeDispatcher.kt
View File

@@ -89,7 +89,6 @@ class InvokeDispatcher(
private val debugBuild: () -> Boolean,
private val onCanvasA2uiPush: () -> Unit,
private val onCanvasA2uiReset: () -> Unit,
private val refreshCanvasHostUrl: suspend () -> String?,
private val motionActivityAvailable: () -> Boolean,
private val motionPedometerAvailable: () -> Boolean,
) {
@@ -242,24 +241,11 @@ class InvokeDispatcher(
}
private suspend fun withReadyA2ui(block: suspend () -> GatewaySession.InvokeResult): GatewaySession.InvokeResult {
var a2uiUrl =
a2uiHandler.resolveA2uiHostUrl()
?: refreshCanvasHostUrl().let { a2uiHandler.resolveA2uiHostUrl() }
?: return GatewaySession.InvokeResult.error(
code = "A2UI_HOST_NOT_CONFIGURED",
message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
)
val readyOnFirstCheck = a2uiHandler.ensureA2uiReady(a2uiUrl)
if (!readyOnFirstCheck) {
// Gateway canvas host metadata can lag reconnects; refresh once before failing the command.
refreshCanvasHostUrl()
a2uiUrl = a2uiHandler.resolveA2uiHostUrl() ?: a2uiUrl
if (!a2uiHandler.ensureA2uiReady(a2uiUrl)) {
return GatewaySession.InvokeResult.error(
code = "A2UI_HOST_UNAVAILABLE",
message = "A2UI_HOST_UNAVAILABLE: A2UI host not reachable",
)
}
if (!a2uiHandler.ensureA2uiReady()) {
return GatewaySession.InvokeResult.error(
code = "A2UI_HOST_UNAVAILABLE",
message = "A2UI_HOST_UNAVAILABLE: bundled A2UI host not reachable",
)
}
return block()
}

5
apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasScreen.kt
View File

@@ -152,9 +152,8 @@ fun CanvasScreen(
}
}
// The listener accepts any WebView origin at registration time because
// gateway A2UI URLs are dynamic; CanvasActionTrust validates the live URL
// before forwarding each message.
// The listener accepts any WebView origin at registration time; native
// dispatch still requires the live URL to be an app-owned bundled page.
val bridge =
CanvasA2UIActionBridge(
isTrustedPage = { viewModel.isTrustedCanvasActionUrl(currentPageUrlRef.get()) },

57
apps/android/app/src/test/java/ai/openclaw/app/node/CanvasActionTrustTest.kt
View File

@@ -7,66 +7,57 @@ import org.junit.Test
class CanvasActionTrustTest {
@Test
fun acceptsBundledScaffoldAsset() {
assertTrue(CanvasActionTrust.isTrustedCanvasActionUrl(CanvasActionTrust.scaffoldAssetUrl, emptyList()))
assertTrue(CanvasActionTrust.isTrustedCanvasActionUrl(CanvasActionTrust.scaffoldAssetUrl))
}
@Test
fun acceptsTrustedA2uiPageOnAdvertisedCanvasHost() {
assertTrue(
fun acceptsBundledA2uiAsset() {
assertTrue(CanvasActionTrust.isTrustedCanvasActionUrl(CanvasActionTrust.localA2uiAssetUrl))
}
@Test
fun rejectsRemoteHttpA2uiPageEvenWhenGatewayAdvertised() {
assertFalse(
CanvasActionTrust.isTrustedCanvasActionUrl(
rawUrl = "http://canvas.example.com:9443/__openclaw__/cap/token/__openclaw__/a2ui/?platform=android",
),
)
}
@Test
fun rejectsRemoteHttpsA2uiPageEvenWhenGatewayAdvertised() {
assertFalse(
CanvasActionTrust.isTrustedCanvasActionUrl(
rawUrl = "https://canvas.example.com:9443/__openclaw__/cap/token/__openclaw__/a2ui/?platform=android",
trustedA2uiUrls = listOf("https://canvas.example.com:9443/__openclaw__/cap/token/__openclaw__/a2ui/?platform=android"),
),
)
}
@Test
fun rejectsDifferentOriginEvenIfPathMatches() {
fun rejectsRemoteCanvasPage() {
assertFalse(
CanvasActionTrust.isTrustedCanvasActionUrl(
rawUrl = "https://evil.example.com:9443/__openclaw__/cap/token/__openclaw__/a2ui/?platform=android",
trustedA2uiUrls = listOf("https://canvas.example.com:9443/__openclaw__/cap/token/__openclaw__/a2ui/?platform=android"),
rawUrl = "https://canvas.example.com:9443/__openclaw__/canvas/",
),
)
}
@Test
fun rejectsUntrustedCanvasPagePathOnTrustedOrigin() {
fun rejectsDescendantPathUnderBundledA2uiRoot() {
assertFalse(
CanvasActionTrust.isTrustedCanvasActionUrl(
rawUrl = "https://canvas.example.com:9443/untrusted/index.html",
trustedA2uiUrls = listOf("https://canvas.example.com:9443/__openclaw__/cap/token/__openclaw__/a2ui/?platform=android"),
rawUrl = "file:///android_asset/CanvasA2UI/child/index.html",
),
)
}
@Test
fun acceptsFragmentOnlyDifferenceForTrustedA2uiPage() {
assertTrue(
CanvasActionTrust.isTrustedCanvasActionUrl(
rawUrl = "https://canvas.example.com:9443/__openclaw__/cap/token/__openclaw__/a2ui/?platform=android#step2",
trustedA2uiUrls = listOf("https://canvas.example.com:9443/__openclaw__/cap/token/__openclaw__/a2ui/?platform=android"),
),
)
}
@Test
fun rejectsQueryMismatchOnTrustedOriginAndPath() {
fun rejectsQueryOrFragmentChangesToBundledA2uiAsset() {
assertFalse(
CanvasActionTrust.isTrustedCanvasActionUrl(
rawUrl = "https://canvas.example.com:9443/__openclaw__/cap/token/__openclaw__/a2ui/?platform=ios",
trustedA2uiUrls = listOf("https://canvas.example.com:9443/__openclaw__/cap/token/__openclaw__/a2ui/?platform=android"),
),
)
}
@Test
fun rejectsDescendantPathUnderTrustedA2uiRoot() {
assertFalse(
CanvasActionTrust.isTrustedCanvasActionUrl(
rawUrl = "https://canvas.example.com:9443/__openclaw__/cap/token/__openclaw__/a2ui/child/index.html?platform=android",
trustedA2uiUrls = listOf("https://canvas.example.com:9443/__openclaw__/cap/token/__openclaw__/a2ui/?platform=android"),
rawUrl = "${CanvasActionTrust.localA2uiAssetUrl}?platform=android",
),
)
assertFalse(CanvasActionTrust.isTrustedCanvasActionUrl("${CanvasActionTrust.localA2uiAssetUrl}#step2"))
}
}

3
apps/android/app/src/test/java/ai/openclaw/app/node/InvokeDispatcherTest.kt
View File

@@ -299,8 +299,6 @@ class InvokeDispatcherTest {
A2UIHandler(
canvas = canvas,
json = Json { ignoreUnknownKeys = true },
getNodeCanvasHostUrl = { null },
getOperatorCanvasHostUrl = { null },
),
debugHandler = DebugHandler(appContext, DeviceIdentityStore(appContext)),
callLogHandler = CallLogHandler.forTesting(appContext, InvokeDispatcherFakeCallLogDataSource()),
@@ -317,7 +315,6 @@ class InvokeDispatcherTest {
debugBuild = { debugBuild },
onCanvasA2uiPush = {},
onCanvasA2uiReset = {},
refreshCanvasHostUrl = { null },
motionActivityAvailable = { motionActivityAvailable },
motionPedometerAvailable = { motionPedometerAvailable },
)

5
apps/ios/Sources/Gateway/GatewayConnectionController.swift
View File

@@ -990,7 +990,10 @@ extension GatewayConnectionController {
}
private func currentCaps() -> [String] {
var caps = [OpenClawCapability.canvas.rawValue, OpenClawCapability.screen.rawValue]
var caps = [
OpenClawCapability.canvas.rawValue,
OpenClawCapability.screen.rawValue,
]
// Default-on: if the key doesn't exist yet, treat it as enabled.
let cameraEnabled =

87
apps/ios/Sources/Model/NodeAppModel+Canvas.swift
View File

@@ -1,106 +1,35 @@
import Foundation
import Network
import OpenClawKit
enum A2UIReadyState {
case ready(String)
case hostNotConfigured
case ready
case hostUnavailable
}
extension NodeAppModel {
func resolveCanvasHostURL() async -> String? {
guard let raw = await self.gatewaySession.currentCanvasHostUrl() else { return nil }
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty, let base = URL(string: trimmed) else { return nil }
if let host = base.host, LoopbackHost.isLoopback(host) {
return nil
}
return base.appendingPathComponent("__openclaw__/canvas/").absoluteString
}
func _test_resolveA2UIHostURL() async -> String? {
await self.resolveA2UIHostURL()
}
func resolveA2UIHostURL() async -> String? {
guard let raw = await self.gatewaySession.currentCanvasHostUrl() else { return nil }
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty, let base = URL(string: trimmed) else { return nil }
if let host = base.host, LoopbackHost.isLoopback(host) {
return nil
}
return base.appendingPathComponent("__openclaw__/a2ui/").absoluteString + "?platform=ios"
}
/// Normalize a URL string for trust comparison: lowercase scheme/host and strip fragment.
/// This matches the normalization applied by ScreenController.isTrustedCanvasUIURL so that
/// SPA hash-routing fragments and scheme/host casing do not silently prevent trust being set.
static func normalizeURLForTrustComparison(_ raw: String) -> String {
guard let url = URL(string: raw),
var components = URLComponents(url: url, resolvingAgainstBaseURL: false)
else { return raw }
components.fragment = nil
components.scheme = components.scheme?.lowercased()
components.host = components.host?.lowercased()
return components.url?.absoluteString ?? raw
}
func showA2UIOnConnectIfNeeded() async {
await MainActor.run {
// Keep the bundled home canvas as the default connected view.
// Agents can still explicitly present a remote or local canvas later.
self.lastAutoA2uiURL = nil
self.screen.showDefaultCanvas()
}
}
func ensureA2UIReadyWithCapabilityRefresh(timeoutMs: Int = 5000) async -> A2UIReadyState {
guard let initialUrl = await self.resolveA2UIHostURLWithCapabilityRefresh() else {
return .hostNotConfigured
if self.screen.isShowingLocalA2UI(),
await self.screen.waitForA2UIReady(timeoutMs: timeoutMs)
{
return .ready
}
self.screen.navigate(to: initialUrl, trustA2UIActions: true)
self.screen.showLocalA2UI()
if await self.screen.waitForA2UIReady(timeoutMs: timeoutMs) {
return .ready(initialUrl)
}
guard let refreshedUrl = await self.resolveA2UIHostURLWithCapabilityRefresh(forceRefresh: true) else {
return .hostUnavailable
}
self.screen.navigate(to: refreshedUrl, trustA2UIActions: true)
if await self.screen.waitForA2UIReady(timeoutMs: timeoutMs) {
return .ready(refreshedUrl)
return .ready
}
return .hostUnavailable
}
func showLocalCanvasOnDisconnect() {
self.lastAutoA2uiURL = nil
self.screen.showDefaultCanvas()
}
private func resolveA2UIHostURLWithCapabilityRefresh(forceRefresh: Bool = false) async -> String? {
if !forceRefresh, let current = await self.resolveA2UIHostURL() {
return current
}
_ = await self.gatewaySession.refreshCanvasHostUrl()
return await self.resolveA2UIHostURL()
}
private func resolveCanvasHostURLWithCapabilityRefresh(forceRefresh: Bool = false) async -> String? {
if !forceRefresh, let current = await self.resolveCanvasHostURL() {
return current
}
_ = await self.gatewaySession.refreshCanvasHostUrl()
return await self.resolveCanvasHostURL()
}
private static func probeTCP(url: URL, timeoutSeconds: Double) async -> Bool {
guard let host = url.host, !host.isEmpty else { return false }
let portInt = url.port ?? ((url.scheme ?? "").lowercased() == "wss" ? 443 : 80)
return await TCPProbe.probe(
host: host,
port: portInt,
timeoutSeconds: timeoutSeconds,
queueLabel: "a2ui.preflight")
}
}

29
apps/ios/Sources/Model/NodeAppModel.swift
View File

@@ -173,7 +173,6 @@ final class NodeAppModel {
private let remindersService: any RemindersServicing
private let motionService: any MotionServicing
private let watchMessagingService: any WatchMessagingServicing
var lastAutoA2uiURL: String?
private var pttVoiceWakeSuspended = false
private var talkVoiceWakeSuspended = false
private var backgroundVoiceWakeSuspended = false
@@ -1037,10 +1036,7 @@ final class NodeAppModel {
if url.isEmpty {
self.screen.showDefaultCanvas()
} else {
let trustedA2UIURL = await self.resolveA2UIHostURL()
self.screen.navigate(
to: url,
trustA2UIActions: trustedA2UIURL == Self.normalizeURLForTrustComparison(url))
self.screen.navigate(to: url)
}
return BridgeInvokeResponse(id: req.id, ok: true)
case OpenClawCanvasCommand.hide.rawValue:
@@ -1049,10 +1045,7 @@ final class NodeAppModel {
case OpenClawCanvasCommand.navigate.rawValue:
let params = try Self.decodeParams(OpenClawCanvasNavigateParams.self, from: req.paramsJSON)
let trimmedURL = params.url.trimmingCharacters(in: .whitespacesAndNewlines)
let trustedA2UIURL = await self.resolveA2UIHostURL()
self.screen.navigate(
to: trimmedURL,
trustA2UIActions: trustedA2UIURL == Self.normalizeURLForTrustComparison(trimmedURL))
self.screen.navigate(to: trimmedURL)
return BridgeInvokeResponse(id: req.id, ok: true)
case OpenClawCanvasCommand.evalJS.rawValue:
let params = try Self.decodeParams(OpenClawCanvasEvalParams.self, from: req.paramsJSON)
@@ -1095,20 +1088,13 @@ final class NodeAppModel {
switch await self.ensureA2UIReadyWithCapabilityRefresh(timeoutMs: 5000) {
case .ready:
break
case .hostNotConfigured:
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: OpenClawNodeError(
code: .unavailable,
message: "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host"))
case .hostUnavailable:
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: OpenClawNodeError(
code: .unavailable,
message: "A2UI_HOST_UNAVAILABLE: A2UI host not reachable"))
message: "A2UI_HOST_UNAVAILABLE: bundled A2UI host not reachable"))
}
let json = try await self.screen.eval(javaScript: """
(() => {
@@ -1138,20 +1124,13 @@ final class NodeAppModel {
switch await self.ensureA2UIReadyWithCapabilityRefresh(timeoutMs: 5000) {
case .ready:
break
case .hostNotConfigured:
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: OpenClawNodeError(
code: .unavailable,
message: "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host"))
case .hostUnavailable:
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: OpenClawNodeError(
code: .unavailable,
message: "A2UI_HOST_UNAVAILABLE: A2UI host not reachable"))
message: "A2UI_HOST_UNAVAILABLE: bundled A2UI host not reachable"))
}
let messagesJSON = try OpenClawCanvasA2UIJSONL.encodeMessagesJSONArray(messages)

56
apps/ios/Sources/Screen/ScreenController.swift
View File

@@ -7,7 +7,6 @@ import WebKit
@Observable
final class ScreenController {
private weak var activeWebView: WKWebView?
private var trustedRemoteA2UIURL: URL?
var urlString: String = ""
var errorText: String?
@@ -27,11 +26,10 @@ final class ScreenController {
self.reload()
}
func navigate(to urlString: String, trustA2UIActions: Bool = false) {
func navigate(to urlString: String, trustA2UIActions _: Bool = false) {
let trimmed = urlString.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty {
self.urlString = ""
self.trustedRemoteA2UIURL = nil
self.reload()
return
}
@@ -45,7 +43,6 @@ final class ScreenController {
return
}
self.urlString = (trimmed == "/" ? "" : trimmed)
self.trustedRemoteA2UIURL = trustA2UIActions ? Self.normalizeTrustedRemoteA2UIURL(from: trimmed) : nil
self.reload()
}
@@ -75,10 +72,26 @@ final class ScreenController {
func showDefaultCanvas() {
self.urlString = ""
self.trustedRemoteA2UIURL = nil
self.reload()
}
func showLocalA2UI() {
guard let url = Self.localA2UIURL else {
self.showDefaultCanvas()
return
}
self.urlString = url.absoluteString
self.reload()
}
func isShowingLocalA2UI() -> Bool {
guard let url = URL(string: self.urlString),
url.isFileURL,
let expected = Self.localA2UIURL
else { return false }
return url.standardizedFileURL == expected.standardizedFileURL
}
func setDebugStatusEnabled(_ enabled: Bool) {
self.debugStatusEnabled = enabled
self.applyDebugStatusIfNeeded()
@@ -239,6 +252,11 @@ final class ScreenController {
ext: "html",
subdirectory: "CanvasScaffold")
private static let localA2UIURL: URL? = ScreenController.bundledResourceURL(
name: "index",
ext: "html",
subdirectory: "CanvasA2UI")
func isTrustedCanvasUIURL(_ url: URL) -> Bool {
if url.isFileURL {
let std = url.standardizedFileURL
@@ -247,10 +265,14 @@ final class ScreenController {
{
return true
}
if let expected = Self.localA2UIURL,
std == expected.standardizedFileURL
{
return true
}
return false
}
guard let trusted = self.trustedRemoteA2UIURL else { return false }
return Self.normalizeTrustedRemoteA2UIURL(from: url) == trusted
return false
}
nonisolated static func parseA2UIActionBody(_ body: Any) -> [String: Any]? {
@@ -280,26 +302,6 @@ final class ScreenController {
scrollView.isScrollEnabled = allowScroll
scrollView.bounces = allowScroll
}
private static func normalizeTrustedRemoteA2UIURL(from raw: String) -> URL? {
guard let url = URL(string: raw) else { return nil }
return self.normalizeTrustedRemoteA2UIURL(from: url)
}
private static func normalizeTrustedRemoteA2UIURL(from url: URL) -> URL? {
guard !url.isFileURL else { return nil }
guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else {
return nil
}
guard let host = url.host?.trimmingCharacters(in: .whitespacesAndNewlines), !host.isEmpty else {
return nil
}
var components = URLComponents(url: url, resolvingAgainstBaseURL: false)
components?.scheme = scheme
components?.host = host.lowercased()
components?.fragment = nil
return components?.url
}
}
extension Double {

6
apps/ios/Tests/NodeAppModelInvokeTests.swift
View File

@@ -623,13 +623,13 @@ private final class MockBootstrapNotificationCenter: NotificationCentering, @unc
#expect(appModel.screen.urlString.isEmpty)
}
@Test @MainActor func handleInvokeA2UICommandsFailWhenHostMissing() async throws {
@Test @MainActor func handleInvokeA2UICommandsFailWhenLocalHostUnavailable() async throws {
let appModel = NodeAppModel()
let reset = BridgeInvokeRequest(id: "reset", command: OpenClawCanvasA2UICommand.reset.rawValue)
let resetRes = await appModel._test_handleInvoke(reset)
#expect(resetRes.ok == false)
#expect(resetRes.error?.message.contains("A2UI_HOST_NOT_CONFIGURED") == true)
#expect(resetRes.error?.message.contains("A2UI_HOST_UNAVAILABLE") == true)
let jsonl = "{\"beginRendering\":{}}"
let pushParams = OpenClawCanvasA2UIPushJSONLParams(jsonl: jsonl)
@@ -641,7 +641,7 @@ private final class MockBootstrapNotificationCenter: NotificationCentering, @unc
paramsJSON: pushJSON)
let pushRes = await appModel._test_handleInvoke(push)
#expect(pushRes.ok == false)
#expect(pushRes.error?.message.contains("A2UI_HOST_NOT_CONFIGURED") == true)
#expect(pushRes.error?.message.contains("A2UI_HOST_UNAVAILABLE") == true)
}
@Test @MainActor func handleInvokeUnknownCommandReturnsInvalidRequest() async {

35
apps/ios/Tests/ScreenControllerTests.swift
View File

@@ -66,26 +66,37 @@ private func mountScreen(_ screen: ScreenController) throws -> (ScreenWebViewCoo
}
}
@Test @MainActor func trustedRemoteA2UIURLMustMatchExactly() {
@Test("remote A2UI URL is not trusted for native actions")
@MainActor func remoteA2UIURLIsNotTrustedForNativeActions() throws {
let screen = ScreenController()
let trusted = "https://node.ts.net:18789/__openclaw__/a2ui/?platform=ios"
screen.navigate(to: trusted, trustA2UIActions: true)
#expect(screen.isTrustedCanvasUIURL(URL(string: trusted)!) == true)
// Fragment differences must not affect trust (SPA hash routing).
#expect(screen.isTrustedCanvasUIURL(URL(string: "https://node.ts.net:18789/__openclaw__/a2ui/?platform=ios#step2")!) == true)
#expect(screen.isTrustedCanvasUIURL(URL(string: "https://node.ts.net:18789/__openclaw__/a2ui/?platform=android")!) == false)
#expect(screen.isTrustedCanvasUIURL(URL(string: "https://node.ts.net:18789/__openclaw__/canvas/")!) == false)
#expect(screen.isTrustedCanvasUIURL(URL(string: "https://evil.ts.net:18789/__openclaw__/a2ui/?platform=ios")!) == false)
#expect(screen.isTrustedCanvasUIURL(URL(string: "http://192.168.0.10:18789/")!) == false)
#expect(screen.isShowingLocalA2UI() == false)
let urls = try [
trusted,
"https://node.ts.net:18789/__openclaw__/a2ui/?platform=ios#step2",
"http://192.168.0.10:18789/__openclaw__/a2ui/?platform=ios",
"https://node.ts.net:18789/__openclaw__/a2ui/?platform=android",
"https://node.ts.net:18789/__openclaw__/canvas/",
"https://evil.ts.net:18789/__openclaw__/a2ui/?platform=ios",
].map { try #require(URL(string: $0)) }
for url in urls {
#expect(screen.isTrustedCanvasUIURL(url) == false)
}
}
@Test @MainActor func genericNavigationClearsTrustedRemoteA2UIURL() {
@Test("local A2UI URL is trusted for native actions")
@MainActor func localA2UIURLIsTrustedForNativeActions() throws {
let screen = ScreenController()
screen.navigate(to: "https://node.ts.net:18789/__openclaw__/a2ui/?platform=ios", trustA2UIActions: true)
screen.navigate(to: "https://evil.ts.net:18789/")
screen.showLocalA2UI()
#expect(screen.isTrustedCanvasUIURL(URL(string: "https://node.ts.net:18789/__openclaw__/a2ui/?platform=ios")!) == false)
let url = try #require(URL(string: screen.urlString))
#expect(url.isFileURL)
#expect(screen.isShowingLocalA2UI() == true)
#expect(screen.isTrustedCanvasUIURL(url) == true)
}
@Test func parseA2UIActionBodyAcceptsJSONString() throws {

5
apps/macos/Sources/OpenClaw/NodeMode/MacNodeModeCoordinator.swift
View File

@@ -139,7 +139,10 @@ final class MacNodeModeCoordinator {
locationMode: OpenClawLocationMode,
connectionMode: AppState.ConnectionMode) -> [String]
{
var caps: [String] = [OpenClawCapability.canvas.rawValue, OpenClawCapability.screen.rawValue]
var caps: [String] = [
OpenClawCapability.canvas.rawValue,
OpenClawCapability.screen.rawValue,
]
if browserControlEnabled, connectionMode == .local {
caps.append(OpenClawCapability.browser.rawValue)
}

12488
apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/CanvasA2UI/a2ui.bundle.js Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/CanvasA2UI/assets/providers/google.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 86 KiB

BIN
apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/CanvasA2UI/assets/providers/x.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 14 KiB

BIN
apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/CanvasA2UI/granola.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 258 KiB

311
apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/CanvasA2UI/index.html Normal file
View File

@@ -0,0 +1,311 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>OpenClaw Canvas</title>
<script>
(() => {
const normalizeLower = (value) => {
const trimmed = String(value || "").trim();
return trimmed.toLocaleLowerCase();
};
try {
const params = new URLSearchParams(window.location.search);
const platform = normalizeLower(params.get("platform"));
if (platform) {
document.documentElement.dataset.platform = platform;
return;
}
if (/android/i.test(navigator.userAgent || "")) {
document.documentElement.dataset.platform = "android";
}
} catch (_) {}
})();
</script>
<style>
:root {
color-scheme: dark;
}
@media (prefers-reduced-motion: reduce) {
body::before,
body::after {
animation: none !important;
}
}
html,
body {
height: 100%;
margin: 0;
}
body {
font:
14px system-ui,
-apple-system,
BlinkMacSystemFont,
"Roboto",
sans-serif;
background:
radial-gradient(1200px 900px at 15% 20%, rgba(42, 113, 255, 0.18), rgba(0, 0, 0, 0) 55%),
radial-gradient(900px 700px at 85% 30%, rgba(255, 0, 138, 0.14), rgba(0, 0, 0, 0) 60%),
radial-gradient(1000px 900px at 60% 90%, rgba(0, 209, 255, 0.1), rgba(0, 0, 0, 0) 60%),
#000;
color: #e5e7eb;
overflow: hidden;
}
:root[data-platform="android"] body {
background:
radial-gradient(1200px 900px at 15% 20%, rgba(42, 113, 255, 0.62), rgba(0, 0, 0, 0) 55%),
radial-gradient(900px 700px at 85% 30%, rgba(255, 0, 138, 0.52), rgba(0, 0, 0, 0) 60%),
radial-gradient(1000px 900px at 60% 90%, rgba(0, 209, 255, 0.48), rgba(0, 0, 0, 0) 60%),
#0b1328;
}
body::before {
content: "";
position: fixed;
inset: -20%;
background:
repeating-linear-gradient(
0deg,
rgba(255, 255, 255, 0.03) 0,
rgba(255, 255, 255, 0.03) 1px,
transparent 1px,
transparent 48px
),
repeating-linear-gradient(
90deg,
rgba(255, 255, 255, 0.03) 0,
rgba(255, 255, 255, 0.03) 1px,
transparent 1px,
transparent 48px
);
transform: translate3d(0, 0, 0) rotate(-7deg);
will-change: transform, opacity;
-webkit-backface-visibility: hidden;
backface-visibility: hidden;
opacity: 0.45;
pointer-events: none;
animation: openclaw-grid-drift 140s ease-in-out infinite alternate;
}
:root[data-platform="android"] body::before {
opacity: 0.8;
}
body::after {
content: "";
position: fixed;
inset: -35%;
background:
radial-gradient(900px 700px at 30% 30%, rgba(42, 113, 255, 0.16), rgba(0, 0, 0, 0) 60%),
radial-gradient(800px 650px at 70% 35%, rgba(255, 0, 138, 0.12), rgba(0, 0, 0, 0) 62%),
radial-gradient(900px 800px at 55% 75%, rgba(0, 209, 255, 0.1), rgba(0, 0, 0, 0) 62%);
filter: blur(28px);
opacity: 0.52;
will-change: transform, opacity;
-webkit-backface-visibility: hidden;
backface-visibility: hidden;
transform: translate3d(0, 0, 0);
pointer-events: none;
animation: openclaw-glow-drift 110s ease-in-out infinite alternate;
}
:root[data-platform="android"] body::after {
opacity: 0.85;
}
@supports (mix-blend-mode: screen) {
body::after {
mix-blend-mode: screen;
}
}
@supports not (mix-blend-mode: screen) {
body::after {
opacity: 0.7;
}
}
@keyframes openclaw-grid-drift {
0% {
transform: translate3d(-12px, 8px, 0) rotate(-7deg);
opacity: 0.4;
}
50% {
transform: translate3d(10px, -7px, 0) rotate(-6.6deg);
opacity: 0.56;
}
100% {
transform: translate3d(-8px, 6px, 0) rotate(-7.2deg);
opacity: 0.42;
}
}
@keyframes openclaw-glow-drift {
0% {
transform: translate3d(-18px, 12px, 0) scale(1.02);
opacity: 0.4;
}
50% {
transform: translate3d(14px, -10px, 0) scale(1.05);
opacity: 0.52;
}
100% {
transform: translate3d(-10px, 8px, 0) scale(1.03);
opacity: 0.43;
}
}
canvas {
position: fixed;
inset: 0;
display: block;
width: 100vw;
height: 100vh;
touch-action: none;
z-index: 1;
}
:root[data-platform="android"] #openclaw-canvas {
background:
radial-gradient(1100px 800px at 20% 15%, rgba(42, 113, 255, 0.78), rgba(0, 0, 0, 0) 58%),
radial-gradient(900px 650px at 82% 28%, rgba(255, 0, 138, 0.66), rgba(0, 0, 0, 0) 62%),
radial-gradient(1000px 900px at 60% 88%, rgba(0, 209, 255, 0.58), rgba(0, 0, 0, 0) 62%),
#141c33;
}
#openclaw-status {
position: fixed;
inset: 0;
display: none;
align-items: center;
justify-content: center;
flex-direction: column;
padding: 24px;
box-sizing: border-box;
pointer-events: none;
z-index: 3;
}
#openclaw-status .card {
width: min(560px, 88vw);
text-align: left;
padding: 14px 16px 12px;
border-radius: 16px;
background: linear-gradient(140deg, rgba(23, 24, 35, 0.78), rgba(18, 19, 28, 0.55));
border: 1px solid rgba(255, 255, 255, 0.12);
box-shadow:
0 16px 46px rgba(0, 0, 0, 0.52),
inset 0 1px 0 rgba(255, 255, 255, 0.06);
-webkit-backdrop-filter: blur(18px) saturate(140%);
backdrop-filter: blur(18px) saturate(140%);
}
#openclaw-status .title {
font:
600 12px/1.2 -apple-system,
BlinkMacSystemFont,
"SF Pro Text",
system-ui,
sans-serif;
letter-spacing: 0.45px;
text-transform: uppercase;
color: rgba(255, 255, 255, 0.7);
}
#openclaw-status .subtitle {
margin-top: 8px;
font:
500 13px/1.45 -apple-system,
BlinkMacSystemFont,
"SF Pro Text",
system-ui,
sans-serif;
color: rgba(255, 255, 255, 0.9);
white-space: pre-wrap;
overflow-wrap: anywhere;
}
openclaw-a2ui-host {
display: block;
height: 100%;
position: fixed;
inset: 0;
z-index: 4;
--openclaw-a2ui-inset-top: 28px;
--openclaw-a2ui-inset-right: 0px;
--openclaw-a2ui-inset-bottom: 0px;
--openclaw-a2ui-inset-left: 0px;
--openclaw-a2ui-scroll-pad-bottom: 0px;
--openclaw-a2ui-status-top: calc(50% - 18px);
--openclaw-a2ui-empty-top: 18px;
}
</style>
</head>
<body>
<canvas id="openclaw-canvas"></canvas>
<div id="openclaw-status" role="status" aria-live="polite">
<section class="card">
<div class="title" id="openclaw-status-title">Ready</div>
<div class="subtitle" id="openclaw-status-subtitle">Waiting for agent</div>
</section>
</div>
<openclaw-a2ui-host></openclaw-a2ui-host>
<script src="a2ui.bundle.js"></script>
<script>
(() => {
const canvas = document.getElementById("openclaw-canvas");
const ctx = canvas.getContext("2d");
const statusEl = document.getElementById("openclaw-status");
const titleEl = document.getElementById("openclaw-status-title");
const subtitleEl = document.getElementById("openclaw-status-subtitle");
const debugStatusEnabledByQuery = (() => {
try {
const params = new URLSearchParams(window.location.search);
const raw = params.get("debugStatus") ?? params.get("debug");
if (!raw) return false;
const normalized = normalizeLower(raw);
return normalized === "1" || normalized === "true" || normalized === "yes";
} catch (_) {
return false;
}
})();
let debugStatusEnabled = debugStatusEnabledByQuery;
function resize() {
const dpr = window.devicePixelRatio || 1;
const w = Math.max(1, Math.floor(window.innerWidth * dpr));
const h = Math.max(1, Math.floor(window.innerHeight * dpr));
canvas.width = w;
canvas.height = h;
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
}
window.addEventListener("resize", resize);
resize();
const setDebugStatusEnabled = (enabled) => {
debugStatusEnabled = !!enabled;
if (!statusEl) return;
if (!debugStatusEnabled) {
statusEl.style.display = "none";
}
};
if (statusEl && !debugStatusEnabled) {
statusEl.style.display = "none";
}
window.__openclaw = {
canvas,
ctx,
setDebugStatusEnabled,
setStatus: (title, subtitle) => {
if (!statusEl || !debugStatusEnabled) return;
if (!title && !subtitle) {
statusEl.style.display = "none";
return;
}
statusEl.style.display = "flex";
if (titleEl && typeof title === "string") titleEl.textContent = title;
if (subtitleEl && typeof subtitle === "string") subtitleEl.textContent = subtitle;
if (!debugStatusEnabled) {
clearTimeout(window.__statusTimeout);
window.__statusTimeout = setTimeout(() => {
statusEl.style.display = "none";
}, 3000);
} else {
clearTimeout(window.__statusTimeout);
}
},
};
})();
</script>
</body>
</html>

2
docs/nodes/index.md
View File

@@ -249,7 +249,9 @@ openclaw nodes canvas a2ui reset --node <idOrNameOrIp>
Notes:
- Mobile nodes use a bundled app-owned A2UI page for action-capable rendering.
- Only A2UI v0.8 JSONL is supported (v0.9/createSurface is rejected).
- iOS and Android render remote Gateway Canvas pages, but A2UI button actions are dispatched only from the bundled app-owned A2UI page. Gateway-hosted HTTP/HTTPS A2UI pages are render-only on those mobile clients.
## Photos + videos (node camera)

4
docs/platforms/android.md
View File

@@ -198,12 +198,12 @@ openclaw nodes invoke --node "<Android Node>" --command canvas.navigate --params
Tailnet (optional): if both devices are on Tailscale, use a MagicDNS name or tailnet IP instead of `.local`, e.g. `http://<gateway-magicdns>:18789/__openclaw__/canvas/`.
This server injects a live-reload client into HTML and reloads on file changes.
The A2UI host lives at `http://<gateway-host>:18789/__openclaw__/a2ui/`.
The Gateway also serves `/__openclaw__/a2ui/`, but the Android app treats remote A2UI pages as render-only. Action-capable A2UI commands use the bundled app-owned A2UI page before applying messages.
Canvas commands (foreground only):
- `canvas.eval`, `canvas.snapshot`, `canvas.navigate` (use `{"url":""}` or `{"url":"/"}` to return to the default scaffold). `canvas.snapshot` returns `{ format, base64 }` (default `format="jpeg"`).
- A2UI: `canvas.a2ui.push`, `canvas.a2ui.reset` (`canvas.a2ui.pushJSONL` legacy alias)
- A2UI: `canvas.a2ui.push`, `canvas.a2ui.reset` (`canvas.a2ui.pushJSONL` legacy alias). These commands use the bundled app-owned A2UI page for action-capable rendering.
Camera commands (foreground only; permission-gated):

5
docs/platforms/ios.md
View File

@@ -238,7 +238,8 @@ Notes:
- The Gateway canvas host serves `/__openclaw__/canvas/` and `/__openclaw__/a2ui/`.
- It is served from the Gateway HTTP server (same port as `gateway.port`, default `18789`).
- The iOS node auto-navigates to A2UI on connect when a canvas host URL is advertised.
- The iOS node keeps the built-in scaffold as the connected default view. `canvas.a2ui.push` and `canvas.a2ui.reset` use the bundled app-owned A2UI page.
- Remote Gateway A2UI pages are render-only on iOS; native A2UI button actions are accepted only from bundled app-owned pages.
- Return to the built-in scaffold with `canvas.navigate` and `{"url":""}`.
## Computer Use relationship
@@ -275,7 +276,7 @@ openclaw nodes invoke --node "iOS Node" --command canvas.snapshot --params '{"ma
## Common errors
- `NODE_BACKGROUND_UNAVAILABLE`: bring the iOS app to the foreground (canvas/camera/screen commands require it).
- `A2UI_HOST_NOT_CONFIGURED`: the Gateway did not advertise the Canvas plugin surface URL; check `plugins.entries.canvas.config.host` in [Gateway configuration](/gateway/configuration).
- `A2UI_HOST_UNAVAILABLE`: the bundled A2UI page was not reachable in the app WebView; keep the app foregrounded on the Screen tab and retry.
- Pairing prompt never appears: run `openclaw devices list` and approve manually.
- Reconnect fails after reinstall: the Keychain pairing token was cleared; re-pair the node.