feat(mobile): agents on the phone can open apps and send intents with am

apps/mobile/modules/t3-device-runtime/android/src/main/AndroidManifest.xml

@@ -33,6 +33,11 @@
     <intent>
       <action android:name="android.intent.action.TTS_SERVICE" />
     </intent>
+    <!-- am and pm: apps with a launcher icon, without QUERY_ALL_PACKAGES. -->
+    <intent>
+      <action android:name="android.intent.action.MAIN" />
+      <category android:name="android.intent.category.LAUNCHER" />
+    </intent>
   </queries>
 
   <application>

apps/mobile/modules/t3-device-runtime/android/src/main/assets/android-bridge.mjs

@@ -155,6 +155,79 @@ const open = {
   },
 };
 
+const AM_VALUES = {
+  "-a": "action",
+  "-d": "data",
+  "-t": "type",
+  "-n": "component",
+  "-p": "package",
+};
+const AM_EXTRAS = {
+  "-e": "string",
+  "--es": "string",
+  "--ez": "boolean",
+  "--ei": "int",
+  "--el": "long",
+  "--ef": "float",
+};
+
+/** Android's am flags. As there, a last argument is a URI, a package/class or a package. */
+const am = {
+  usage:
+    "start|broadcast [-a action] [-d uri] [-t type] [-c category]... [-n package/class] " +
+    "[-p package] [--es|--ez|--ei|--el|--ef key value]... [uri | package/class | package]",
+  request: ([mode, ...args]) => {
+    if (mode !== "start" && mode !== "broadcast")
+      throw new UsageError("expected start or broadcast");
+    const body = { mode };
+    const categories = [];
+    const extras = [];
+    let index = 0;
+    const value = (offset) => {
+      const found = args[index + offset];
+      if (found === undefined) throw new UsageError(`${args[index]} needs a value`);
+      return found;
+    };
+    while (index < args.length) {
+      const arg = args[index];
+      if (AM_VALUES[arg]) {
+        body[AM_VALUES[arg]] = value(1);
+        index += 2;
+      } else if (arg === "-c") {
+        categories.push(value(1));
+        index += 2;
+      } else if (AM_EXTRAS[arg]) {
+        extras.push({ type: AM_EXTRAS[arg], key: value(1), value: value(2) });
+        index += 3;
+      } else if (arg.startsWith("-")) {
+        throw new UsageError(`unknown option ${arg}`);
+      } else if (index === args.length - 1) {
+        const field = URL_SCHEME.test(arg) ? "data" : arg.includes("/") ? "component" : "package";
+        body[field] = arg;
+        index += 1;
+      } else {
+        throw new UsageError(`unexpected argument ${arg}`);
+      }
+    }
+    if (categories.length > 0) body.categories = categories;
+    if (extras.length > 0) body.extras = extras;
+    return { path: "am", body };
+  },
+};
+
+const pm = {
+  usage: "list packages [-3 | -s] [filter]",
+  request: ([list, packages, ...args]) => {
+    if (list !== "list" || packages !== "packages") {
+      throw new UsageError("only pm list packages is supported");
+    }
+    const { options, rest } = parseOptions(args, {}, { "-3": "thirdParty", "-s": "system" });
+    if (rest.length > 1) throw new UsageError("expected at most one filter");
+    return { path: "pm", body: { ...options, ...(rest[0] ? { filter: rest[0] } : {}) } };
+  },
+  print: (names) => names.forEach((name) => printLine(`package:${name}`)),
+};
+
 const VOLUME_STREAMS = ["alarm", "music", "notification", "ring", "system", "call"];
 const SPEECH_STREAMS = ["ALARM", "MUSIC", "NOTIFICATION", "RING", "SYSTEM", "VOICE_CALL"];
 const SMS_TYPES = ["all", "inbox", "sent", "draft", "outbox", "failed", "queued"];
@@ -516,6 +589,9 @@ const COMMANDS = {
   },
   "termux-telephony-deviceinfo": jsonCommand("telephony-deviceinfo"),
   "termux-telephony-cellinfo": jsonCommand("telephony-cellinfo"),
+  am,
+  "termux-am": am,
+  pm,
 };
 
 async function call(path, body) {

apps/mobile/modules/t3-device-runtime/android/src/main/assets/phone-agent-guide.md

@@ -23,11 +23,16 @@ There is no Termux, Termux:API app or `pkg`: install packages with `apt-get` (ru
     always allow or never): `termux-location`, `termux-telephony-cellinfo`,
     `termux-camera-photo`, `termux-microphone-record`, `termux-contact-list`,
     `termux-sms-list`, `termux-sms-send`, `termux-call-log`, `termux-telephony-call`,
-    `termux-telephony-deviceinfo`, `termux-notification-list`. With Ask each time, each run
-    waits up to a minute for the user to answer on the phone, and fails if nobody does. The
-    first use also shows Android's own permission prompt. If one fails because it's set to
-    Never, wasn't approved or Android wasn't allowed, pass on the place the error names
-    instead of working around it.
+    `termux-telephony-deviceinfo`, `termux-notification-list`, and `am` and `pm` (below).
+    With Ask each time, each run waits up to a minute for the user to answer on the phone,
+    and fails if nobody does. The first use also shows Android's own permission prompt. If
+    one fails because it's set to Never, wasn't approved or Android wasn't allowed, pass on
+    the place the error names instead of working around it.
+- `am start` and `am broadcast` (also `termux-am`) send intents with Android's flags `-a`,
+  `-d`, `-t`, `-c`, `-n`, `-p`, `--es`, `--ez`, `--ei`, `--el` and `--ef`, and a last
+  argument that's a URI, package/class or package. `am start <package>` opens that app.
+  Broadcasts need `-p` or `-n`. `pm list packages [-3 | -s] [filter]` lists apps that have
+  a launcher icon; others are hidden from T3 Code. Calls go through `termux-telephony-call`.
 - While T3 Code isn't on screen, the clipboard can't be read, the camera, microphone and
-  location fail, and links, files, shares and calls arrive as a notification for the user
+  location fail, and links, files, apps, shares and calls arrive as a notification for the user
   to tap.

apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AccessApproval.kt

@@ -100,7 +100,7 @@ internal object AccessApproval {
 
   private fun waitingFor(category: AccessCategory, command: String, details: String): WaitingApproval {
     val message = buildString {
-      append("termux-$command")
+      append(AgentAccess.commandName(command))
       if (details.isNotEmpty()) append("\n\n$details")
       append("\n\nAn agent or another device using this phone's server asked for this.")
     }

apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AgentAccess.kt

@@ -64,8 +64,18 @@ object AgentAccess {
     ),
     // Notification access is a special permission, granted in system settings.
     AccessCategory(NOTIFICATIONS, "Notifications", listOf("notification-list"), emptyList()),
+    AccessCategory("apps", "Other apps", listOf("am", "pm"), emptyList()),
   )
 
+  /** The guest commands that reach a bridge command: termux-<command>, but am and pm keep Android's names. */
+  fun guestCommands(command: String): List<String> = when (command) {
+    "am" -> listOf("am", "termux-am")
+    "pm" -> listOf("pm")
+    else -> listOf("termux-$command")
+  }
+
+  fun commandName(command: String) = guestCommands(command).first()
+
   fun category(id: String): AccessCategory? = categories.firstOrNull { it.id == id }
 
   fun policy(context: Context, category: AccessCategory): AccessPolicy {
@@ -153,7 +163,7 @@ object AgentAccess {
   }
 
   private fun rules(category: AccessCategory) =
-    category.commands.flatMap { listOf("Bash(termux-$it)", "Bash(termux-$it *)") }
+    category.commands.flatMap(::guestCommands).flatMap { listOf("Bash($it)", "Bash($it *)") }
 
   private fun preferences(context: Context) = context.getSharedPreferences(PREFERENCES, Context.MODE_PRIVATE)
 }

apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AndroidBridge.kt

@@ -27,6 +27,7 @@ import android.webkit.URLUtil
 import android.widget.Toast
 import androidx.core.app.NotificationCompat
 import androidx.core.content.FileProvider
+import org.json.JSONArray
 import org.json.JSONException
 import org.json.JSONObject
 import java.io.ByteArrayOutputStream
@@ -85,6 +86,9 @@ private val GUEST_COMMANDS = listOf(
   "termux-telephony-call",
   "termux-telephony-deviceinfo",
   "termux-telephony-cellinfo",
+  "am",
+  "termux-am",
+  "pm",
 )
 
 // Commands that wait on hardware, an engine or a database, so they run on the request's thread.
@@ -104,6 +108,7 @@ private val WAITING_COMMANDS = setOf(
   "telephony-deviceinfo",
   "telephony-cellinfo",
   "notification-list",
+  "pm",
 )
 private const val NOTIFICATION_CHANNEL = "t3-agent-notifications"
 private const val REQUEST_CHANNEL = "t3-agent-requests"
@@ -305,9 +310,28 @@ class AndroidBridge private constructor(context: Context, private val rootfs: Fi
     "telephony-deviceinfo" -> data.deviceInfo()
     "telephony-cellinfo" -> data.cellInfo()
     "notification-list" -> data.notifications()
+    "am" -> am(body)
+    "pm" -> JSONArray(launchablePackages(context, body))
     else -> throw BridgeError("Unknown command: $command", 404)
   }.takeUnless { it == Unit }
 
+  private fun am(body: JSONObject): Any? {
+    val intent = appIntent(context, body)
+    return when (val mode = body.getString("mode")) {
+      "start" -> launchIntent(context, intent).let { startOrNotify(it, "Open an app from an agent", describeIntent(it)) }
+      "broadcast" -> {
+        requireTarget(intent)
+        context.sendBroadcast(intent)
+        if (hasDeclaredReceiver(context, intent)) null
+        else JSONObject().put(
+          "message",
+          "Sent, but no app on this phone declares a receiver for it, so nothing may have received it.",
+        )
+      }
+      else -> throw BridgeError("am takes start or broadcast, not $mode")
+    }
+  }
+
   private fun openUrl(url: String): Any? =
     startOrNotify(Intent(Intent.ACTION_VIEW, webUri(url)), "Open a link from an agent", url)
 
@@ -424,13 +448,13 @@ class AndroidBridge private constructor(context: Context, private val rootfs: Fi
           context.startActivity(intent)
         } else {
           val tap = PendingIntent.getActivity(context, notificationId, intent, PendingIntent.FLAG_IMMUTABLE)
-          notify(REQUEST_CHANNEL, REQUEST_TAG, notificationId, "Allow $label?", "An agent wants to run termux-$command. Tap to answer.", tap)
+          notify(REQUEST_CHANNEL, REQUEST_TAG, notificationId, "Allow $label?", "An agent wants to run ${AgentAccess.commandName(command)}. Tap to answer.", tap)
         }
       },
       dismiss = { notificationManager().cancel(REQUEST_TAG, notificationId) },
     )
     if (!approved) {
-      throw BridgeError("The user didn't allow termux-$command on the phone. ${AgentAccess.where(category)} controls this.", 403)
+      throw BridgeError("The user didn't allow ${AgentAccess.commandName(command)} on the phone. ${AgentAccess.where(category)} controls this.", 403)
     }
   }
 

apps/mobile/modules/t3-device-runtime/android/src/main/java/expo/modules/t3deviceruntime/AppIntents.kt

@@ -0,0 +1,108 @@
+package expo.modules.t3deviceruntime
+
+import android.content.ComponentName
+import android.content.Context
+import android.content.Intent
+import android.content.pm.ApplicationInfo
+import android.net.Uri
+import org.json.JSONObject
+
+// T3 Code holds CALL_PHONE, so these would place a call outside the Phone category.
+private val CALL_ACTIONS = setOf(
+  Intent.ACTION_CALL,
+  "android.intent.action.CALL_PRIVILEGED",
+  "android.intent.action.CALL_EMERGENCY",
+)
+
+/**
+ * The intent an am command describes. Android sends it as T3 Code, with T3 Code's permissions,
+ * so it's built from am's flags rather than parsed: it carries no flags, URI grants, selector or
+ * Parcelable extras, and can't reach T3 Code's own components.
+ */
+internal fun appIntent(context: Context, body: JSONObject): Intent {
+  val intent = Intent()
+  body.optString("action").takeIf { it.isNotEmpty() }?.let { intent.action = it }
+  if (intent.action in CALL_ACTIONS) throw BridgeError("Calls go through termux-telephony-call.")
+  val data = body.optString("data").takeIf { it.isNotEmpty() }?.let(Uri::parse)
+  intent.setDataAndType(data, body.optString("type").takeIf { it.isNotEmpty() })
+  body.optJSONArray("categories")?.let { categories ->
+    for (index in 0 until categories.length()) intent.addCategory(categories.getString(index))
+  }
+  body.optString("component").takeIf { it.isNotEmpty() }?.let { name ->
+    intent.component = ComponentName.unflattenFromString(name)
+      ?: throw BridgeError("-n takes package/class, such as com.example/.MainActivity, not $name")
+  }
+  body.optString("package").takeIf { it.isNotEmpty() }?.let { intent.setPackage(it) }
+  body.optJSONArray("extras")?.let { extras ->
+    for (index in 0 until extras.length()) putExtra(intent, extras.getJSONObject(index))
+  }
+  val own = context.packageName
+  if (intent.`package` == own || intent.component?.packageName == own) {
+    throw BridgeError("am can't send intents to T3 Code itself.")
+  }
+  if (data?.scheme == "content" && data?.authority.orEmpty().startsWith(own)) {
+    throw BridgeError("T3 Code's own files go through termux-open or termux-share.")
+  }
+  return intent
+}
+
+/** am start -p <package> with nothing else opens the app, as its launcher icon does. */
+internal fun launchIntent(context: Context, intent: Intent): Intent {
+  val target = intent.`package`
+  val onlyPackage = target != null && intent.action == null && intent.data == null &&
+    intent.component == null && intent.categories.isNullOrEmpty()
+  if (!onlyPackage) return intent
+  val launch = context.packageManager.getLaunchIntentForPackage(target)
+    ?: throw BridgeError("$target isn't installed or has no app to open. pm list packages lists the ones that do.")
+  return launch.putExtras(intent)
+}
+
+/** What a notification or error names the intent by. */
+internal fun describeIntent(intent: Intent): String =
+  intent.component?.flattenToShortString() ?: intent.`package` ?: intent.dataString ?: intent.action ?: "this intent"
+
+/** A broadcast names its app, so one can't reach every app that listens for its action. */
+internal fun requireTarget(intent: Intent) {
+  if (intent.`package` == null && intent.component == null) {
+    throw BridgeError("am broadcast needs -p or -n to name the app it's for.")
+  }
+}
+
+/** Whether an installed app declares a receiver for it; apps can also register one while they run. */
+internal fun hasDeclaredReceiver(context: Context, intent: Intent): Boolean =
+  @Suppress("DEPRECATION")
+  context.packageManager.queryBroadcastReceivers(intent, 0).isNotEmpty()
+
+/**
+ * pm list packages, for the apps with a launcher icon: the manifest's queries make only those
+ * visible, as Play allows QUERY_ALL_PACKAGES for few kinds of app.
+ */
+internal fun launchablePackages(context: Context, body: JSONObject): List<String> {
+  val launcher = Intent(Intent.ACTION_MAIN).addCategory(Intent.CATEGORY_LAUNCHER)
+  val filter = body.optString("filter")
+  @Suppress("DEPRECATION")
+  return context.packageManager.queryIntentActivities(launcher, 0)
+    .map { it.activityInfo.applicationInfo }
+    .filter { app ->
+      val isSystem = app.flags and ApplicationInfo.FLAG_SYSTEM != 0
+      !(body.optBoolean("thirdParty") && isSystem) && !(body.optBoolean("system") && !isSystem)
+    }
+    .map { it.packageName }
+    .filter { filter in it }
+    .distinct()
+    .sorted()
+}
+
+private fun putExtra(intent: Intent, extra: JSONObject) {
+  val key = extra.getString("key")
+  val value = extra.getString("value")
+  fun invalid(): Nothing = throw BridgeError("The extra $key isn't a valid ${extra.getString("type")}: $value")
+  when (extra.getString("type")) {
+    "string" -> intent.putExtra(key, value)
+    "boolean" -> intent.putExtra(key, value.toBooleanStrictOrNull() ?: invalid())
+    "int" -> intent.putExtra(key, value.toIntOrNull() ?: invalid())
+    "long" -> intent.putExtra(key, value.toLongOrNull() ?: invalid())
+    "float" -> intent.putExtra(key, value.toFloatOrNull() ?: invalid())
+    else -> throw BridgeError("Unknown extra type ${extra.getString("type")}")
+  }
+}

apps/mobile/src/features/device-runtime/AgentAccessRouteScreen.tsx

@@ -23,6 +23,7 @@ const CATEGORIES: Readonly<Record<string, { readonly label: string; readonly com
       commands: "termux-telephony-call, termux-telephony-deviceinfo",
     },
     notifications: { label: "Notifications", commands: "termux-notification-list" },
+    apps: { label: "Other apps", commands: "am (also termux-am), pm" },
   };
 
 const POLICIES: ReadonlyArray<{