diff --git a/apps/mobile/modules/t3-device-runtime/android/src/main/AndroidManifest.xml b/apps/mobile/modules/t3-device-runtime/android/src/main/AndroidManifest.xml index adb61bfb51f7..7341d5ab37d7 100644 --- a/apps/mobile/modules/t3-device-runtime/android/src/main/AndroidManifest.xml +++ b/apps/mobile/modules/t3-device-runtime/android/src/main/AndroidManifest.xml @@ -33,6 +33,11 @@ + + + + + diff --git a/apps/mobile/modules/t3-device-runtime/android/src/main/assets/android-bridge.mjs b/apps/mobile/modules/t3-device-runtime/android/src/main/assets/android-bridge.mjs index b9fdb661b348..d5f59579fada 100644 --- a/apps/mobile/modules/t3-device-runtime/android/src/main/assets/android-bridge.mjs +++ b/apps/mobile/modules/t3-device-runtime/android/src/main/assets/android-bridge.mjs @@ -155,6 +155,79 @@ const open = { }, }; +const AM_VALUES = { + "-a": "action", + "-d": "data", + "-t": "type", + "-n": "component", + "-p": "package", +}; +const AM_EXTRAS = { + "-e": "string", + "--es": "string", + "--ez": "boolean", + "--ei": "int", + "--el": "long", + "--ef": "float", +}; + +/** Android's am flags. As there, a last argument is a URI, a package/class or a package. */ +const am = { + usage: + "start|broadcast [-a action] [-d uri] [-t type] [-c category]... [-n package/class] " + + "[-p package] [--es|--ez|--ei|--el|--ef key value]... [uri | package/class | package]", + request: ([mode, ...args]) => { + if (mode !== "start" && mode !== "broadcast") + throw new UsageError("expected start or broadcast"); + const body = { mode }; + const categories = []; + const extras = []; + let index = 0; + const value = (offset) => { + const found = args[index + offset]; + if (found === undefined) throw new UsageError(`${args[index]} needs a value`); + return found; + }; + while (index < args.length) { + const arg = args[index]; + if (AM_VALUES[arg]) { + body[AM_VALUES[arg]] = value(1); + index += 2; + } else if (arg === "-c") { + categories.push(value(1)); + index += 2; + } else if (AM_EXTRAS[arg]) { + extras.push({ type: AM_EXTRAS[arg], key: value(1), value: value(2) }); + index += 3; + } else if (arg.startsWith("-")) { + throw new UsageError(`unknown option ${arg}`); + } else if (index === args.length - 1) { + const field = URL_SCHEME.test(arg) ? "data" : arg.includes("/") ? "component" : "package"; + body[field] = arg; + index += 1; + } else { + throw new UsageError(`unexpected argument ${arg}`); + } + } + if (categories.length > 0) body.categories = categories; + if (extras.length > 0) body.extras = extras; + return { path: "am", body }; + }, +}; + +const pm = { + usage: "list packages [-3 | -s] [filter]", + request: ([list, packages, ...args]) => { + if (list !== "list" || packages !== "packages") { + throw new UsageError("only pm list packages is supported"); + } + const { options, rest } = parseOptions(args, {}, { "-3": "thirdParty", "-s": "system" }); + if (rest.length > 1) throw new UsageError("expected at most one filter"); + return { path: "pm", body: { ...options, ...(rest[0] ? { filter: rest[0] } : {}) } }; + }, + print: (names) => names.forEach((name) => printLine(`package:${name}`)), +}; + const VOLUME_STREAMS = ["alarm", "music", "notification", "ring", "system", "call"]; const SPEECH_STREAMS = ["ALARM", "MUSIC", "NOTIFICATION", "RING", "SYSTEM", "VOICE_CALL"]; const SMS_TYPES = ["all", "inbox", "sent", "draft", "outbox", "failed", "queued"]; @@ -516,6 +589,9 @@ const COMMANDS = { }, "termux-telephony-deviceinfo": jsonCommand("telephony-deviceinfo"), "termux-telephony-cellinfo": jsonCommand("telephony-cellinfo"), + am, + "termux-am": am, + pm, }; async function call(path, body) { diff --git a/apps/mobile/modules/t3-device-runtime/android/src/main/assets/phone-agent-guide.md b/apps/mobile/modules/t3-device-runtime/android/src/main/assets/phone-agent-guide.md index f9453e346ad0..29f7c91bb311 100644 --- a/apps/mobile/modules/t3-device-runtime/android/src/main/assets/phone-agent-guide.md +++ b/apps/mobile/modules/t3-device-runtime/android/src/main/assets/phone-agent-guide.md @@ -23,11 +23,16 @@ There is no Termux, Termux:API app or `pkg`: install packages with `apt-get` (ru always allow or never): `termux-location`, `termux-telephony-cellinfo`, `termux-camera-photo`, `termux-microphone-record`, `termux-contact-list`, `termux-sms-list`, `termux-sms-send`, `termux-call-log`, `termux-telephony-call`, - `termux-telephony-deviceinfo`, `termux-notification-list`. With Ask each time, each run - waits up to a minute for the user to answer on the phone, and fails if nobody does. The - first use also shows Android's own permission prompt. If one fails because it's set to - Never, wasn't approved or Android wasn't allowed, pass on the place the error names - instead of working around it. + `termux-telephony-deviceinfo`, `termux-notification-list`, and `am` and `pm` (below). + With Ask each time, each run waits up to a minute for the user to answer on the phone, + and fails if nobody does. The first use also shows Android's own permission prompt. If + one fails because it's set to Never, wasn't approved or Android wasn't allowed, pass on + the place the error names instead of working around it. +- `am start` and `am broadcast` (also `termux-am`) send intents with Android's flags `-a`, + `-d`, `-t`, `-c`, `-n`, `-p`, `--es`, `--ez`, `--ei`, `--el` and `--ef`, and a last + argument that's a URI, package/class or package. `am start ` opens that app. + Broadcasts need `-p` or `-n`. `pm list packages [-3 | -s] [filter]` lists apps that have + a launcher icon; others are hidden from T3 Code. Calls go through `termux-telephony-call`. - While T3 Code isn't on screen, the clipboard can't be read, the camera, microphone and - location fail, and links, files, shares and calls arrive as a notification for the user + location fail, and links, files, apps, shares and calls arrive as a notification for the user to tap. diff --git a/apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AccessApproval.kt b/apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AccessApproval.kt index 3dc3ca7e3b21..07335263e552 100644 --- a/apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AccessApproval.kt +++ b/apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AccessApproval.kt @@ -100,7 +100,7 @@ internal object AccessApproval { private fun waitingFor(category: AccessCategory, command: String, details: String): WaitingApproval { val message = buildString { - append("termux-$command") + append(AgentAccess.commandName(command)) if (details.isNotEmpty()) append("\n\n$details") append("\n\nAn agent or another device using this phone's server asked for this.") } diff --git a/apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AgentAccess.kt b/apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AgentAccess.kt index 25bbcf23d955..fee30dc3f25a 100644 --- a/apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AgentAccess.kt +++ b/apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AgentAccess.kt @@ -64,8 +64,18 @@ object AgentAccess { ), // Notification access is a special permission, granted in system settings. AccessCategory(NOTIFICATIONS, "Notifications", listOf("notification-list"), emptyList()), + AccessCategory("apps", "Other apps", listOf("am", "pm"), emptyList()), ) + /** The guest commands that reach a bridge command: termux-, but am and pm keep Android's names. */ + fun guestCommands(command: String): List = when (command) { + "am" -> listOf("am", "termux-am") + "pm" -> listOf("pm") + else -> listOf("termux-$command") + } + + fun commandName(command: String) = guestCommands(command).first() + fun category(id: String): AccessCategory? = categories.firstOrNull { it.id == id } fun policy(context: Context, category: AccessCategory): AccessPolicy { @@ -153,7 +163,7 @@ object AgentAccess { } private fun rules(category: AccessCategory) = - category.commands.flatMap { listOf("Bash(termux-$it)", "Bash(termux-$it *)") } + category.commands.flatMap(::guestCommands).flatMap { listOf("Bash($it)", "Bash($it *)") } private fun preferences(context: Context) = context.getSharedPreferences(PREFERENCES, Context.MODE_PRIVATE) } diff --git a/apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AndroidBridge.kt b/apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AndroidBridge.kt index f5de6e420bee..b5de15101765 100644 --- a/apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AndroidBridge.kt +++ b/apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AndroidBridge.kt @@ -27,6 +27,7 @@ import android.webkit.URLUtil import android.widget.Toast import androidx.core.app.NotificationCompat import androidx.core.content.FileProvider +import org.json.JSONArray import org.json.JSONException import org.json.JSONObject import java.io.ByteArrayOutputStream @@ -85,6 +86,9 @@ private val GUEST_COMMANDS = listOf( "termux-telephony-call", "termux-telephony-deviceinfo", "termux-telephony-cellinfo", + "am", + "termux-am", + "pm", ) // Commands that wait on hardware, an engine or a database, so they run on the request's thread. @@ -104,6 +108,7 @@ private val WAITING_COMMANDS = setOf( "telephony-deviceinfo", "telephony-cellinfo", "notification-list", + "pm", ) private const val NOTIFICATION_CHANNEL = "t3-agent-notifications" private const val REQUEST_CHANNEL = "t3-agent-requests" @@ -305,9 +310,28 @@ class AndroidBridge private constructor(context: Context, private val rootfs: Fi "telephony-deviceinfo" -> data.deviceInfo() "telephony-cellinfo" -> data.cellInfo() "notification-list" -> data.notifications() + "am" -> am(body) + "pm" -> JSONArray(launchablePackages(context, body)) else -> throw BridgeError("Unknown command: $command", 404) }.takeUnless { it == Unit } + private fun am(body: JSONObject): Any? { + val intent = appIntent(context, body) + return when (val mode = body.getString("mode")) { + "start" -> launchIntent(context, intent).let { startOrNotify(it, "Open an app from an agent", describeIntent(it)) } + "broadcast" -> { + requireTarget(intent) + context.sendBroadcast(intent) + if (hasDeclaredReceiver(context, intent)) null + else JSONObject().put( + "message", + "Sent, but no app on this phone declares a receiver for it, so nothing may have received it.", + ) + } + else -> throw BridgeError("am takes start or broadcast, not $mode") + } + } + private fun openUrl(url: String): Any? = startOrNotify(Intent(Intent.ACTION_VIEW, webUri(url)), "Open a link from an agent", url) @@ -424,13 +448,13 @@ class AndroidBridge private constructor(context: Context, private val rootfs: Fi context.startActivity(intent) } else { val tap = PendingIntent.getActivity(context, notificationId, intent, PendingIntent.FLAG_IMMUTABLE) - notify(REQUEST_CHANNEL, REQUEST_TAG, notificationId, "Allow $label?", "An agent wants to run termux-$command. Tap to answer.", tap) + notify(REQUEST_CHANNEL, REQUEST_TAG, notificationId, "Allow $label?", "An agent wants to run ${AgentAccess.commandName(command)}. Tap to answer.", tap) } }, dismiss = { notificationManager().cancel(REQUEST_TAG, notificationId) }, ) if (!approved) { - throw BridgeError("The user didn't allow termux-$command on the phone. ${AgentAccess.where(category)} controls this.", 403) + throw BridgeError("The user didn't allow ${AgentAccess.commandName(command)} on the phone. ${AgentAccess.where(category)} controls this.", 403) } } diff --git a/apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AppIntents.kt b/apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AppIntents.kt new file mode 100644 index 000000000000..becb1551ef73 --- /dev/null +++ b/apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AppIntents.kt @@ -0,0 +1,108 @@ +package expo.modules.t3deviceruntime + +import android.content.ComponentName +import android.content.Context +import android.content.Intent +import android.content.pm.ApplicationInfo +import android.net.Uri +import org.json.JSONObject + +// T3 Code holds CALL_PHONE, so these would place a call outside the Phone category. +private val CALL_ACTIONS = setOf( + Intent.ACTION_CALL, + "android.intent.action.CALL_PRIVILEGED", + "android.intent.action.CALL_EMERGENCY", +) + +/** + * The intent an am command describes. Android sends it as T3 Code, with T3 Code's permissions, + * so it's built from am's flags rather than parsed: it carries no flags, URI grants, selector or + * Parcelable extras, and can't reach T3 Code's own components. + */ +internal fun appIntent(context: Context, body: JSONObject): Intent { + val intent = Intent() + body.optString("action").takeIf { it.isNotEmpty() }?.let { intent.action = it } + if (intent.action in CALL_ACTIONS) throw BridgeError("Calls go through termux-telephony-call.") + val data = body.optString("data").takeIf { it.isNotEmpty() }?.let(Uri::parse) + intent.setDataAndType(data, body.optString("type").takeIf { it.isNotEmpty() }) + body.optJSONArray("categories")?.let { categories -> + for (index in 0 until categories.length()) intent.addCategory(categories.getString(index)) + } + body.optString("component").takeIf { it.isNotEmpty() }?.let { name -> + intent.component = ComponentName.unflattenFromString(name) + ?: throw BridgeError("-n takes package/class, such as com.example/.MainActivity, not $name") + } + body.optString("package").takeIf { it.isNotEmpty() }?.let { intent.setPackage(it) } + body.optJSONArray("extras")?.let { extras -> + for (index in 0 until extras.length()) putExtra(intent, extras.getJSONObject(index)) + } + val own = context.packageName + if (intent.`package` == own || intent.component?.packageName == own) { + throw BridgeError("am can't send intents to T3 Code itself.") + } + if (data?.scheme == "content" && data?.authority.orEmpty().startsWith(own)) { + throw BridgeError("T3 Code's own files go through termux-open or termux-share.") + } + return intent +} + +/** am start -p with nothing else opens the app, as its launcher icon does. */ +internal fun launchIntent(context: Context, intent: Intent): Intent { + val target = intent.`package` + val onlyPackage = target != null && intent.action == null && intent.data == null && + intent.component == null && intent.categories.isNullOrEmpty() + if (!onlyPackage) return intent + val launch = context.packageManager.getLaunchIntentForPackage(target) + ?: throw BridgeError("$target isn't installed or has no app to open. pm list packages lists the ones that do.") + return launch.putExtras(intent) +} + +/** What a notification or error names the intent by. */ +internal fun describeIntent(intent: Intent): String = + intent.component?.flattenToShortString() ?: intent.`package` ?: intent.dataString ?: intent.action ?: "this intent" + +/** A broadcast names its app, so one can't reach every app that listens for its action. */ +internal fun requireTarget(intent: Intent) { + if (intent.`package` == null && intent.component == null) { + throw BridgeError("am broadcast needs -p or -n to name the app it's for.") + } +} + +/** Whether an installed app declares a receiver for it; apps can also register one while they run. */ +internal fun hasDeclaredReceiver(context: Context, intent: Intent): Boolean = + @Suppress("DEPRECATION") + context.packageManager.queryBroadcastReceivers(intent, 0).isNotEmpty() + +/** + * pm list packages, for the apps with a launcher icon: the manifest's queries make only those + * visible, as Play allows QUERY_ALL_PACKAGES for few kinds of app. + */ +internal fun launchablePackages(context: Context, body: JSONObject): List { + val launcher = Intent(Intent.ACTION_MAIN).addCategory(Intent.CATEGORY_LAUNCHER) + val filter = body.optString("filter") + @Suppress("DEPRECATION") + return context.packageManager.queryIntentActivities(launcher, 0) + .map { it.activityInfo.applicationInfo } + .filter { app -> + val isSystem = app.flags and ApplicationInfo.FLAG_SYSTEM != 0 + !(body.optBoolean("thirdParty") && isSystem) && !(body.optBoolean("system") && !isSystem) + } + .map { it.packageName } + .filter { filter in it } + .distinct() + .sorted() +} + +private fun putExtra(intent: Intent, extra: JSONObject) { + val key = extra.getString("key") + val value = extra.getString("value") + fun invalid(): Nothing = throw BridgeError("The extra $key isn't a valid ${extra.getString("type")}: $value") + when (extra.getString("type")) { + "string" -> intent.putExtra(key, value) + "boolean" -> intent.putExtra(key, value.toBooleanStrictOrNull() ?: invalid()) + "int" -> intent.putExtra(key, value.toIntOrNull() ?: invalid()) + "long" -> intent.putExtra(key, value.toLongOrNull() ?: invalid()) + "float" -> intent.putExtra(key, value.toFloatOrNull() ?: invalid()) + else -> throw BridgeError("Unknown extra type ${extra.getString("type")}") + } +} diff --git a/apps/mobile/src/features/device-runtime/AgentAccessRouteScreen.tsx b/apps/mobile/src/features/device-runtime/AgentAccessRouteScreen.tsx index f19f44485e84..6628d2ee00e6 100644 --- a/apps/mobile/src/features/device-runtime/AgentAccessRouteScreen.tsx +++ b/apps/mobile/src/features/device-runtime/AgentAccessRouteScreen.tsx @@ -23,6 +23,7 @@ const CATEGORIES: Readonly { await using bridge = await startBridge((path) => { if (path === "/clipboard-get") return { ok: true, result: "copied" }; if (path === "/battery-status") return { ok: true, result: { percentage: 80 } }; + if (path === "/pm") return { ok: true, result: ["com.tailscale.ipn", "org.mozilla.firefox"] }; return { ok: true, result: { notified: true, message: "Tap the notification." } }; }); expect((await bridge.run("termux-clipboard-get", [])).stdout).toBe("copied"); expect((await bridge.run("termux-battery-status", [])).stdout).toBe( '{\n "percentage": 80\n}\n', ); + expect((await bridge.run("pm", ["list", "packages"])).stdout).toBe( + "package:com.tailscale.ipn\npackage:org.mozilla.firefox\n", + ); expect(await bridge.run("xdg-open", ["https://example.com"])).toEqual({ code: 0, stdout: "", @@ -219,9 +223,13 @@ describe("android-bridge.mjs", () => { await bridge.run("termux-volume", ["music", "loud"]), await bridge.run("termux-location", ["-p", "wifi"]), await bridge.run("termux-torch", ["bright"]), + await bridge.run("am", ["start", "-W", "com.example"]), + await bridge.run("am", ["broadcast", "--ez", "enabled"]), + await bridge.run("am", ["startservice", "-p", "com.example"]), + await bridge.run("pm", ["uninstall", "com.example"]), ]; expect(recording.stdout).toBe("Recording started\n"); - expect(refused.map(({ code }) => code)).toEqual([1, 1, 1, 1]); + expect(refused.map(({ code }) => code)).toEqual([1, 1, 1, 1, 1, 1, 1, 1]); expect(sent(bridge)).toEqual([ { path: "/microphone-record", @@ -231,6 +239,44 @@ describe("android-bridge.mjs", () => { ]); }); + it("reads am's intent flags, and a last argument as a URI, component or package", async () => { + await using bridge = await startBridge(succeed); + await bridge.run("am", [ + "broadcast", + "-a", + "com.tailscale.ipn.CONNECT_VPN", + "--ez", + "quiet", + "true", + "-c", + "a", + "-c", + "b", + "com.tailscale.ipn/.IPNReceiver", + ]); + await bridge.run("termux-am", ["start", "-a", "android.intent.action.VIEW", "geo:0,0?q=cafe"]); + await bridge.run("am", ["start", "com.tailscale.ipn"]); + await bridge.run("pm", ["list", "packages", "-3", "tail"]); + expect(sent(bridge)).toEqual([ + { + path: "/am", + body: { + mode: "broadcast", + action: "com.tailscale.ipn.CONNECT_VPN", + component: "com.tailscale.ipn/.IPNReceiver", + categories: ["a", "b"], + extras: [{ type: "boolean", key: "quiet", value: "true" }], + }, + }, + { + path: "/am", + body: { mode: "start", action: "android.intent.action.VIEW", data: "geo:0,0?q=cafe" }, + }, + { path: "/am", body: { mode: "start", package: "com.tailscale.ipn" } }, + { path: "/pm", body: { thirdParty: true, filter: "tail" } }, + ]); + }); + it("reads sensors once per -n and cleans up without calling Android", async () => { await using bridge = await startBridge(() => ({ ok: true, diff --git a/apps/mobile/src/features/settings/settings-search.ts b/apps/mobile/src/features/settings/settings-search.ts index 2137a0355f60..240f7507c6f6 100644 --- a/apps/mobile/src/features/settings/settings-search.ts +++ b/apps/mobile/src/features/settings/settings-search.ts @@ -134,6 +134,8 @@ const SETTINGS_SEARCH_TERMS = { "sms", "call log", "notifications", + "apps", + "intents", ], } satisfies Partial>; const termsById: ReadonlyMap = new Map( diff --git a/docs/user/providers-claude.md b/docs/user/providers-claude.md index 164427f2366b..a425d46f0397 100644 --- a/docs/user/providers-claude.md +++ b/docs/user/providers-claude.md @@ -74,8 +74,10 @@ phone becomes an environment named after the device. `termux-share`, `termux-download`, `termux-tts-speak`, `termux-location` and `termux-camera-photo`. While T3 Code isn't on screen, links, files and calls arrive as a notification to tap, and the clipboard, camera, microphone and location can't be used. -- Commands that reach your location, camera, microphone, contacts, SMS, call log, phone or - notifications ask you first, as an approval in the thread, even with full access. The +- Agents can open other apps and send them intents with Android's `am start` and + `am broadcast`, and list apps with `pm list packages`. +- Commands that reach your location, camera, microphone, contacts, SMS, call log, phone, + notifications or other apps ask you first on the phone, even with full access. The first time, Android also asks you to give T3 Code the permission. In **Settings > Agent access** you can set each to **Always allow** or **Never**. If you installed T3 Code from an APK file, Android may block notification access until you turn on **Allow restricted