From b8f7aa6cf296a6a6de8f7d5c0cdaf4c45d793948 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 04:32:27 +0000 Subject: [PATCH 01/24] ADFA-7135: Define the interface between the SQLite debugger extension and its server sqlitedbg.h is what the extension implements and its JNI layer calls; DebugEngine.kt is what the server (ADFA-7136) builds against, with a fake for JVM tests. Not yet built by Gradle: the module has no build file. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- .../sqlite-debugger/src/main/cpp/sqlitedbg.h | 147 ++++++++++++++++++ .../sqlitedebugger/engine/DebugEngine.kt | 136 ++++++++++++++++ .../sqlitedebugger/engine/FakeDebugEngine.kt | 94 +++++++++++ 3 files changed, 377 insertions(+) create mode 100644 subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h create mode 100644 subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/DebugEngine.kt create mode 100644 subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/FakeDebugEngine.kt diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h new file mode 100644 index 0000000000..2153afaa05 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h @@ -0,0 +1,147 @@ +// The SQLite debugger's extension (ADFA-7135): the interface its JNI layer and its host tests +// build against. Debug builds of apps built with Code on the Go only (ADFA-7131, D1). +// +// The extension runs on every connection the app's SQLite (requery's libsqlite3x.so) opens after +// sqlitedbg_install(). It keeps a list of the app's open connections and a buffer of statement +// events, and opens connections for the server (ADFA-7136). It never sets the progress handler or +// the update hook of an app connection: requery sets and clears the progress handler around +// cancellable statements, and the update hook belongs to the app. +// +// Threads: every function here may be called from any thread. requery builds SQLite with +// SQLITE_THREADSAFE=2, so a connection is used by one thread at a time; the extension's own state +// is guarded by its own lock, which is never held while SQLite runs a statement. + +#ifndef SQLITEDBG_H +#define SQLITEDBG_H + +#include +#include +#include "sqlite3.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// The interface version. Raise it when anything below changes shape. +#define SQLITEDBG_INTERFACE_VERSION 1 + +// sqlite3_auto_extension's type. On a device the JNI layer looks it up in libsqlite3x.so with +// dlsym (requery exports every sqlite3_ function); host tests pass SQLite's own. +typedef int (*sqlitedbg_auto_extension_fn)(void (*entry_point)(void)); + +typedef struct sqlitedbg_config { + // Size of the event buffer. Events are dropped, oldest first, when it is full. 0: 4 MiB. + size_t buffer_bytes; + // Longest SQL text kept in one event; longer text is cut at a UTF-8 character boundary and + // the event gets SQLITEDBG_EVENT_TRUNCATED. 0: 4 KiB. + uint32_t max_sql_bytes; +} sqlitedbg_config; + +// Registers the extension for every connection opened from now on, and sets up the buffer. +// Connections already open are not seen (a known limit, ADFA-7131). A second call does nothing and +// returns SQLITE_OK; config is read on the first call only. config may be NULL for the defaults. +// Returns SQLITE_OK, SQLITE_NOMEM, or what auto_extension returns. +int sqlitedbg_install(sqlitedbg_auto_extension_fn auto_extension, const sqlitedbg_config *config); + +// The SQLite entry point the extension registers. Exposed for host tests only: the app never +// calls it, and it is never loaded with load_extension(). +int sqlitedbg_extension_init(sqlite3 *db, char **err, const sqlite3_api_routines *api); + +// Connections + +typedef struct sqlitedbg_connection { + // Unique within the process, starting at 1, never reused. + uint64_t id; + // sqlite3_db_filename(db, "main") when the connection opened: "" for an in-memory or temporary + // database. Valid only during the sqlitedbg_each_connection callback. + const char *path; + // 1 if the main database is read-only (sqlite3_db_readonly). + int read_only; + // 1 once the trace "close" event has fired. The close may still fail, and the app may retry + // it, so the connection stays listed until SQLite frees it. + int closing; + // CLOCK_REALTIME, nanoseconds, when the extension first saw the connection. + int64_t opened_at_ns; + // gettid() of the thread that opened it. + int64_t open_thread_id; +} sqlitedbg_connection; + +// Calls fn once for each app connection SQLite has not yet freed, oldest first. Server connections +// (sqlitedbg_open_server) are not listed. fn runs with the extension's lock held: it must be quick +// and must not call into SQLite or this interface. Returns how many connections it reported. +size_t sqlitedbg_each_connection(void (*fn)(void *context, const sqlitedbg_connection *connection), + void *context); + +// Events + +enum { + // A statement starts running (SQLITE_TRACE_STMT); also each trigger it fires, whose sql starts + // with "--". + SQLITEDBG_EVENT_STATEMENT = 1, + // A statement finished (SQLITE_TRACE_PROFILE); duration_ns is set. + SQLITEDBG_EVENT_PROFILE = 2, + // The app asked SQLite to close the connection (SQLITE_TRACE_CLOSE); sql is empty. + SQLITEDBG_EVENT_CLOSE = 3, +}; + +enum { + // sql was cut at max_sql_bytes. + SQLITEDBG_EVENT_TRUNCATED = 1u << 0, + // sql has its bound values written in (sqlite3_expanded_sql). Only while a client is + // connected (sqlitedbg_set_client_connected): expanding a 1 MB value costs 2 MB on every run. + SQLITEDBG_EVENT_EXPANDED = 1u << 1, +}; + +// One event as sqlitedbg_read_events copies it out: this header, then sql_bytes of UTF-8 (no +// terminating NUL), then zero padding to a multiple of 8 bytes. Fields are in the CPU's byte order +// (little-endian on every ABI we build). The JNI layer hands the bytes to Kotlin unchanged, so +// EventDecoder.kt mirrors this layout; change both together. +typedef struct sqlitedbg_event_record { + uint32_t record_bytes; // the whole record, padding included + uint32_t kind; // SQLITEDBG_EVENT_STATEMENT, _PROFILE or _CLOSE + uint64_t seq; // 1, 2, 3...: every event made, dropped or not, gets the next one + uint64_t connection_id; // sqlitedbg_connection.id + int64_t thread_id; // gettid() of the thread running the statement + int64_t time_ns; // CLOCK_REALTIME when the event happened + int64_t duration_ns; // SQLITEDBG_EVENT_PROFILE only; -1 for the others + uint32_t flags; // SQLITEDBG_EVENT_TRUNCATED, SQLITEDBG_EVENT_EXPANDED + uint32_t sql_bytes; +} sqlitedbg_event_record; // 56 bytes; the SQL text follows + +typedef struct sqlitedbg_read_result { + // The seq of the last event copied out, or `after` if none was: pass it as `after` next time. + uint64_t next; + // Events after `after` that left the buffer before they could be read. + uint64_t dropped; + // Bytes of records written to out. + size_t bytes; + // Records written to out. + size_t count; +} sqlitedbg_read_result; + +// Copies out the events with seq > after, oldest first, as whole records, until out is full. +// The copy is made under the extension's lock with no SQLite call, so it never waits on a +// statement. A record larger than out_bytes is never split: an out smaller than one record of +// max_sql_bytes may return count 0 with events still waiting. Never fails. +void sqlitedbg_read_events(uint64_t after, void *out, size_t out_bytes, sqlitedbg_read_result *result); + +// Whether a client is connected. While it is, statements are recorded with their bound values +// written in. Off at install. +void sqlitedbg_set_client_connected(int connected); + +// Server connections + +// Opens path for the server with the same SQLite the app uses: SQLITE_OPEN_READONLY unless +// writable, PRAGMA query_only when read-only, and an authorizer that refuses ordinary writes when +// read-only. Read-only mode guards against accidents; it is not a security boundary (ADFA-7131, +// D11). Never traced and never listed. The caller owns *db and closes it with sqlite3_close_v2. +// A server connection, unlike an app connection, may use a progress handler (for time limits). +// Returns SQLITE_MISUSE before any app connection has opened (the extension then has no SQLite +// API to call), or what sqlite3_open_v2 returns. +int sqlitedbg_open_server(const char *path, int writable, sqlite3 **db); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/DebugEngine.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/DebugEngine.kt new file mode 100644 index 0000000000..65e476ad52 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/DebugEngine.kt @@ -0,0 +1,136 @@ +package org.appdevforall.sqlitedebugger.engine + +import java.io.Closeable + +/** + * What the MCP server (ADFA-7136) needs from the SQLite extension (ADFA-7135). The device + * implementation is a thin JNI layer over `sqlitedbg.h`; JVM tests use a fake. Every call may + * block briefly and must not run on the main thread. + */ +interface DebugEngine { + /** The app's connections SQLite has not yet freed, oldest first. Server connections are not listed. */ + fun connections(): List + + /** + * Events with [Event.seq] greater than [after], oldest first, up to about [maxBytes] of + * native records. Pass [EventBatch.next] as [after] next time; start from 0. + */ + fun readEvents( + after: Long, + maxBytes: Int, + ): EventBatch + + /** While true, statements are recorded with their bound values written in. */ + fun setClientConnected(connected: Boolean) + + /** + * Opens [path] for the server: read-only (with an authorizer refusing writes) unless + * [writable]. Only a path one of [connections] has open may be passed: opening a database that + * Android's own SQLite has open, with requery's, can corrupt it (ADFA-7131, Design 3). + * + * @throws DebugEngineException if SQLite cannot open it, or no app connection has opened yet. + */ + fun openServerConnection( + path: String, + writable: Boolean, + ): ServerConnection +} + +/** A connection the server opened with [DebugEngine.openServerConnection]. Use from one thread at a time. */ +interface ServerConnection : Closeable { + /** + * Runs [sql] with [args] bound (null, [Long], [Double], [String] or [ByteArray]), skips + * [offset] rows, and returns at most [QueryLimits.maxRows]. Paging re-runs the statement, so a + * later page can differ if the data changed. Stops the statement (sqlite3_interrupt, through a + * progress handler on this server connection) once [deadlineMillis] (System.currentTimeMillis + * clock) has passed. + * + * @throws DebugEngineException for SQLite errors, a refused write, or the deadline. + */ + fun query( + sql: String, + args: List, + offset: Long, + limits: QueryLimits, + deadlineMillis: Long, + ): QueryResult +} + +data class AppConnection( + /** Unique within the process, never reused. */ + val id: Long, + /** Empty for an in-memory or temporary database. */ + val path: String, + val readOnly: Boolean, + /** The app asked to close it; the close may still fail, so it stays listed until freed. */ + val closing: Boolean, + val openedAtEpochNanos: Long, + val openThreadId: Long, +) + +enum class EventKind( + /** The `kind` value in `sqlitedbg_event_record`. */ + val nativeValue: Int, +) { + /** A statement starts; also each trigger it fires, whose SQL starts with "--". */ + STATEMENT(1), + + /** A statement finished; [Event.durationNanos] is set. */ + PROFILE(2), + + /** The app asked SQLite to close the connection; [Event.sql] is empty. */ + CLOSE(3), +} + +data class Event( + /** 1, 2, 3...; a gap means events were dropped. */ + val seq: Long, + val kind: EventKind, + val connectionId: Long, + val threadId: Long, + val timeEpochNanos: Long, + /** Set for [EventKind.PROFILE] only. */ + val durationNanos: Long?, + val sql: String, + /** [sql] was cut at the extension's limit. */ + val truncated: Boolean, + /** [sql] has its bound values written in. */ + val expanded: Boolean, +) + +data class EventBatch( + val events: List, + /** Pass as `after` to the next [DebugEngine.readEvents]. */ + val next: Long, + /** Events after the requested cursor that left the buffer before they were read. */ + val dropped: Long, +) + +data class QueryLimits( + val maxRows: Int, + /** Longer TEXT values are cut at a UTF-8 character boundary and counted in [QueryResult.truncatedCells]. */ + val maxTextBytes: Int, + /** Longer BLOB values are cut and counted in [QueryResult.truncatedCells]. */ + val maxBlobBytes: Int, +) + +data class QueryResult( + val columns: List, + /** Each value is null, [Long], [Double], [String] or [ByteArray]. Doubles may be NaN or infinite. */ + val rows: List>, + val truncatedCells: Int, + /** More rows follow the last one returned. */ + val hasMore: Boolean, +) + +class DebugEngineException( + /** The SQLite result code, or [SQLITE_MISUSE] when no app connection has opened yet. */ + val sqliteCode: Int, + message: String, +) : Exception(message) { + companion object { + const val SQLITE_AUTH = 23 + const val SQLITE_INTERRUPT = 9 + const val SQLITE_MISUSE = 21 + } +} diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/FakeDebugEngine.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/FakeDebugEngine.kt new file mode 100644 index 0000000000..e0b61ce460 --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/FakeDebugEngine.kt @@ -0,0 +1,94 @@ +package org.appdevforall.sqlitedebugger.engine + +/** + * A [DebugEngine] for JVM tests of the server (ADFA-7136): connections and events are set by the + * test, and each query is answered by [answer]. It keeps the real engine's rules that the server + * relies on: cursors and dropped counts, and refusing to open a path no app connection has open. + */ +class FakeDebugEngine( + var answer: (path: String, writable: Boolean, sql: String, args: List) -> QueryResult = { _, _, _, _ -> + QueryResult(emptyList(), emptyList(), 0, false) + }, +) : DebugEngine { + val appConnections = mutableListOf() + + /** Events still in the buffer, oldest first; [lastSeq] counts every event made. */ + private val buffered = ArrayDeque() + private var lastSeq = 0L + + var clientConnected = false + private set + + val openedServerConnections = mutableListOf>() + + fun addEvent( + kind: EventKind, + sql: String, + connectionId: Long = 1, + durationNanos: Long? = null, + ) { + lastSeq++ + buffered.addLast(Event(lastSeq, kind, connectionId, 1, lastSeq, durationNanos, sql, false, clientConnected)) + } + + /** Drops the [count] oldest buffered events, as a full native buffer would. */ + fun dropOldest(count: Int) = repeat(count) { buffered.removeFirstOrNull() } + + override fun connections() = appConnections.toList() + + override fun readEvents( + after: Long, + maxBytes: Int, + ): EventBatch { + val oldest = buffered.firstOrNull()?.seq ?: (lastSeq + 1) + val dropped = (oldest - 1 - after).coerceAtLeast(0) + val events = mutableListOf() + var bytes = 0 + for (event in buffered) { + if (event.seq <= after) continue + // the native record: a 56-byte header, then the SQL padded to 8 bytes + val size = 56 + (event.sql.toByteArray().size + 7) / 8 * 8 + if (bytes + size > maxBytes) break + bytes += size + events += event + } + return EventBatch(events, events.lastOrNull()?.seq ?: maxOf(after, oldest - 1), dropped) + } + + override fun setClientConnected(connected: Boolean) { + clientConnected = connected + } + + override fun openServerConnection( + path: String, + writable: Boolean, + ): ServerConnection { + if (appConnections.isEmpty()) { + throw DebugEngineException(DebugEngineException.SQLITE_MISUSE, "no app connection has opened yet") + } + if (appConnections.none { it.path == path && path.isNotEmpty() }) { + throw DebugEngineException(14, "unable to open database file: $path") // SQLITE_CANTOPEN + } + openedServerConnections += path to writable + return object : ServerConnection { + private var closed = false + + override fun query( + sql: String, + args: List, + offset: Long, + limits: QueryLimits, + deadlineMillis: Long, + ): QueryResult { + check(!closed) { "closed" } + val all = answer(path, writable, sql, args) + val rows = all.rows.drop(offset.toInt()) + return all.copy(rows = rows.take(limits.maxRows), hasMore = rows.size > limits.maxRows) + } + + override fun close() { + closed = true + } + } + } +} From c603f89882b3e8464f9713d8181fdb8149712bf1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 04:32:28 +0000 Subject: [PATCH 02/24] ADFA-7136: Copy the MCP server from ShareYourApps, renamed McpHttp.kt and McpProtocol.kt from Local Store Server Combined with only the package, server name, title, instructions and 401 texts changed. McpProtocolTest.kt still names that project's tools and does not compile; ADFA-7136 rewrites it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- .../sqlitedebugger/mcp/McpHttp.kt | 313 +++++++++++++++ .../sqlitedebugger/mcp/McpProtocol.kt | 172 +++++++++ .../sqlitedebugger/mcp/McpProtocolTest.kt | 363 ++++++++++++++++++ 3 files changed, 848 insertions(+) create mode 100644 subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt create mode 100644 subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt create mode 100644 subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt new file mode 100644 index 0000000000..c04c7fe6c3 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt @@ -0,0 +1,313 @@ +package org.appdevforall.sqlitedebugger.mcp + +import java.io.BufferedInputStream +import java.io.ByteArrayOutputStream +import java.io.IOException +import java.io.InputStream +import java.io.OutputStream +import java.net.InetAddress +import java.net.InetSocketAddress +import java.net.ServerSocket +import java.net.Socket +import java.net.SocketTimeoutException +import java.net.URI +import java.security.MessageDigest +import java.util.Collections +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import kotlin.concurrent.thread +import org.json.JSONObject + +/** + * The MCP endpoint's HTTP side: a small HTTP/1.1 listener on 127.0.0.1 only, apart from the store's + * own HttpServer, so it's never reachable from another device. Before [McpProtocol] sees a request, + * it checks who's asking: + * - the peer is a loopback address (it can't be otherwise, but it's checked anyway); + * - Host names 127.0.0.1 or localhost, and Origin, if sent, does too, so a web page can't reach it + * by DNS rebinding; + * - Authorization: Bearer , the token shown on the server screen, compared in constant time. + * + * POST /mcp carries JSON-RPC; DELETE /mcp ends a session; GET gets 405, since this server never + * opens an event stream. Each connection has its own thread, as a tool may wait a while; at most + * [MAX_CONNECTIONS] at once. Plain Kotlin, so unit tests can run it without Android. + */ +class McpHttp( + private val protocol: McpProtocol, + /** The current token: read on every request, so a new one takes effect at once. */ + private val token: () -> String, + private val port: Int = DEFAULT_PORT, + /** Problems worth a line in the log. Never given the token. */ + private val log: (String) -> Unit = {}, +) { + companion object { + const val DEFAULT_PORT = 6176 + const val PATH = "/mcp" + const val MAX_CONNECTIONS = 8 + private const val MAX_HEADER_BYTES = 16 * 1024 + const val MAX_BODY_BYTES = 1024 * 1024 + /** A connection with no request for this long is closed. */ + const val IDLE_TIMEOUT_MS = 30_000 + private val STATUS = mapOf( + 200 to "OK", 202 to "Accepted", 204 to "No Content", 400 to "Bad Request", 401 to "Unauthorized", + 403 to "Forbidden", 404 to "Not Found", 405 to "Method Not Allowed", 413 to "Payload Too Large", + 415 to "Unsupported Media Type", 431 to "Request Header Fields Too Large", 500 to "Internal Server Error", + 501 to "Not Implemented", 503 to "Service Unavailable", + ) + private val LOOPBACK_NAMES = setOf("127.0.0.1", "localhost") + } + + private class Request(val method: String, val target: String, val version: String, val headers: Map, val body: ByteArray) + private class HttpError(val status: Int, message: String) : Exception(message) + + @Volatile private var socket: ServerSocket? = null + @Volatile private var running = false + private val open = AtomicInteger(0) + private val clients: MutableSet = Collections.newSetFromMap(ConcurrentHashMap()) + + /** The port actually bound (useful when [port] is 0, as in tests). */ + val boundPort: Int get() = socket?.localPort ?: port + + /** Binds 127.0.0.1:[port]. Throws IOException with a readable message if the port is taken. */ + fun start() { + val s = ServerSocket() + try { + s.reuseAddress = true + s.bind(InetSocketAddress(InetAddress.getByAddress(byteArrayOf(127, 0, 0, 1)), port), 16) + } catch (e: IOException) { + try { s.close() } catch (ignored: IOException) { } + throw IOException("Can't listen on 127.0.0.1:$port: ${e.message}", e) + } + socket = s + running = true + thread(name = "mcp-accept", isDaemon = true) { acceptLoop(s) } + } + + /** Stops listening and closes every connection. */ + fun stop() { + running = false + try { socket?.close() } catch (ignored: IOException) { } + socket = null + clients.forEach { close(it) } + clients.clear() + } + + private fun acceptLoop(s: ServerSocket) { + while (running && !s.isClosed) { + val c = try { s.accept() } catch (e: IOException) { + if (running && !s.isClosed) log("couldn't accept a connection: ${e.message}") + continue + } + if (!c.inetAddress.isLoopbackAddress) { close(c); continue } + if (open.incrementAndGet() > MAX_CONNECTIONS) { + open.decrementAndGet() + try { + c.soTimeout = 1000 + respond(c.getOutputStream(), 503, rpcError("Too many connections at once; try again."), emptyMap(), close = true) + } catch (ignored: IOException) { } + close(c) + continue + } + clients += c + thread(name = "mcp-connection", isDaemon = true) { + try { + serve(c) + } catch (ignored: IOException) { + // The client went away. + } catch (e: Exception) { + log("a connection failed: ${e.javaClass.simpleName}: ${e.message}") + } finally { + clients -= c + close(c) + open.decrementAndGet() + } + } + } + } + + private fun serve(c: Socket) { + c.soTimeout = IDLE_TIMEOUT_MS + c.tcpNoDelay = true + val input = BufferedInputStream(c.getInputStream()) + val out = c.getOutputStream() + while (running) { + val req = try { + readRequest(input) ?: return + } catch (e: SocketTimeoutException) { + return + } catch (e: HttpError) { + respond(out, e.status, rpcError(e.message ?: "Bad request."), emptyMap(), close = true) + return + } + if (!handle(req, out)) return + } + } + + /** Answers one request. Returns whether the connection stays open for another. */ + private fun handle(r: Request, out: OutputStream): Boolean { + val close = r.version == "HTTP/1.0" || r.headers["connection"]?.lowercase()?.contains("close") == true + fun send(status: Int, body: String?, headers: Map = emptyMap()): Boolean { + respond(out, status, body, headers, close) + return !close + } + if (!hostAllowed(r.headers["host"])) return send(403, rpcError("The Host header must be 127.0.0.1 or localhost.")) + val origin = r.headers["origin"] + if (origin != null && !originAllowed(origin)) return send(403, rpcError("Requests from web pages aren't accepted.")) + if (r.target.substringBefore('?') != PATH) return send(404, rpcError("Not found: the MCP endpoint is $PATH.")) + if (!authorized(r.headers["authorization"])) { + return send(401, rpcError("Missing or wrong token. Code on the Go passes it to its Agent; send it as Authorization: Bearer ."), + mapOf("WWW-Authenticate" to "Bearer realm=\"SQLite debugger\"")) + } + val session = r.headers["mcp-session-id"] + val reply = when (r.method) { + "POST" -> { + val type = r.headers["content-type"]?.substringBefore(';')?.trim()?.lowercase() + if (type != null && type != "application/json") return send(415, rpcError("Send the request as application/json.")) + try { + protocol.post(String(r.body, Charsets.UTF_8), session, r.headers["mcp-protocol-version"]) + } catch (e: Exception) { + log("a request failed: ${e.javaClass.simpleName}: ${e.message}") + McpProtocol.Reply(500, rpcError("The server failed: ${e.message ?: e.javaClass.simpleName}")) + } + } + "DELETE" -> protocol.delete(session) + "GET" -> return send(405, rpcError("This server doesn't offer an event stream: use POST."), mapOf("Allow" to "POST, DELETE")) + else -> return send(405, rpcError("Use POST."), mapOf("Allow" to "POST, DELETE")) + } + return send(reply.status, reply.body, reply.headers) + } + + // ---------------------------------------------------------------- checks + + /** 127.0.0.1 or localhost, on any port: adb forward may bring the request in from another port. */ + private fun hostAllowed(host: String?): Boolean { + val h = host?.trim()?.lowercase() ?: return false + val name = if (h.count { it == ':' } == 1) h.substringBefore(':') else h + if (h.count { it == ':' } == 1 && h.substringAfter(':').let { it.isEmpty() || it.any { ch -> !ch.isDigit() } }) return false + return name in LOOPBACK_NAMES + } + + private fun originAllowed(origin: String): Boolean = try { + val u = URI(origin.trim()) + (u.scheme == "http" || u.scheme == "https") && u.host?.lowercase() in LOOPBACK_NAMES + } catch (e: Exception) { false } + + private fun authorized(header: String?): Boolean { + val expected = token() + if (header == null || expected.isEmpty()) return false + val h = header.trim() + if (!h.regionMatches(0, "Bearer ", 0, 7, ignoreCase = true)) return false + return MessageDigest.isEqual(h.substring(7).trim().toByteArray(Charsets.UTF_8), expected.toByteArray(Charsets.UTF_8)) + } + + // ---------------------------------------------------------------- reading and writing HTTP + + /** One request, or null if the connection closed before one began. */ + private fun readRequest(input: InputStream): Request? { + var budget = MAX_HEADER_BYTES + fun line(): String? { + val b = ByteArrayOutputStream() + while (true) { + val ch = input.read() + if (ch < 0) { if (b.size() == 0) return null else throw HttpError(400, "The request ended early.") } + if (--budget < 0) throw HttpError(431, "The request's headers are too long.") + if (ch == '\n'.code) break + b.write(ch) + } + return b.toString("ISO-8859-1").removeSuffix("\r") + } + var first = line() ?: return null + while (first.isEmpty()) first = line() ?: return null // stray line ends between requests + val parts = first.split(' ') + if (parts.size != 3 || !parts[2].startsWith("HTTP/1.")) throw HttpError(400, "That isn't an HTTP/1.x request.") + val headers = HashMap() + while (true) { + val l = line() ?: throw HttpError(400, "The request ended early.") + if (l.isEmpty()) break + val colon = l.indexOf(':') + if (colon <= 0) throw HttpError(400, "A header line without a colon.") + val name = l.substring(0, colon).trim().lowercase() + val value = l.substring(colon + 1).trim() + // A repeated header is joined, so two Host headers fail the Host check. + headers[name] = headers[name]?.let { "$it, $value" } ?: value + } + return Request(parts[0], parts[1], parts[2], headers, readBody(input, headers)) + } + + private fun readBody(input: InputStream, h: Map): ByteArray { + val te = h["transfer-encoding"] + if (te != null) { + if (!te.equals("chunked", ignoreCase = true)) throw HttpError(501, "Transfer-Encoding $te isn't supported.") + if (h.containsKey("content-length")) throw HttpError(400, "Both Content-Length and Transfer-Encoding were sent.") + return readChunked(input) + } + val len = h["content-length"] ?: return ByteArray(0) + val n = len.toLongOrNull()?.takeIf { it >= 0 } ?: throw HttpError(400, "Content-Length isn't a number.") + if (n > MAX_BODY_BYTES) throw HttpError(413, "The request is larger than ${MAX_BODY_BYTES / 1024} KB.") + return readExactly(input, n.toInt()) + } + + private fun readChunked(input: InputStream): ByteArray { + val out = ByteArrayOutputStream() + while (true) { + val sizeLine = shortLine(input, 1024) ?: throw HttpError(400, "The request ended early.") + val size = sizeLine.substringBefore(';').trim().toIntOrNull(16)?.takeIf { it >= 0 } ?: throw HttpError(400, "A bad chunk size.") + if (out.size().toLong() + size > MAX_BODY_BYTES) throw HttpError(413, "The request is larger than ${MAX_BODY_BYTES / 1024} KB.") + if (size == 0) { + // Trailers, if any, up to a blank line. + var lines = 0 + while (true) { + val t = shortLine(input, 1024) ?: break + if (t.isEmpty()) break + if (++lines > 32) throw HttpError(431, "Too many trailers.") + } + return out.toByteArray() + } + out.write(readExactly(input, size)) + if (shortLine(input, 2) != "") throw HttpError(400, "A chunk without its line end.") + } + } + + private fun shortLine(input: InputStream, max: Int): String? { + val b = ByteArrayOutputStream() + var left = max + while (true) { + val ch = input.read() + if (ch < 0) return if (b.size() == 0) null else throw HttpError(400, "The request ended early.") + if (--left < 0) throw HttpError(400, "A line in the request is too long.") + if (ch == '\n'.code) break + b.write(ch) + } + return b.toString("ISO-8859-1").removeSuffix("\r") + } + + private fun readExactly(input: InputStream, n: Int): ByteArray { + val b = ByteArray(n) + var off = 0 + while (off < n) { + val r = input.read(b, off, n - off) + if (r < 0) throw HttpError(400, "The request ended early.") + off += r + } + return b + } + + private fun respond(out: OutputStream, status: Int, body: String?, headers: Map, close: Boolean) { + val bytes = body?.toByteArray(Charsets.UTF_8) ?: ByteArray(0) + val head = StringBuilder("HTTP/1.1 $status ${STATUS[status] ?: "Status"}\r\n") + if (body != null) head.append("Content-Type: application/json; charset=utf-8\r\n") + if (status != 204) head.append("Content-Length: ${bytes.size}\r\n") + head.append("Cache-Control: no-store\r\n") + if (close) head.append("Connection: close\r\n") + for ((k, v) in headers) head.append(k).append(": ").append(v.replace("\r", "").replace("\n", "")).append("\r\n") + head.append("\r\n") + out.write(head.toString().toByteArray(Charsets.ISO_8859_1)) + if (status != 204) out.write(bytes) + out.flush() + } + + /** An HTTP-level refusal, as a JSON-RPC error with no id, which MCP clients know how to show. */ + private fun rpcError(message: String): String = JSONObject().put("jsonrpc", "2.0").put("id", JSONObject.NULL) + .put("error", JSONObject().put("code", McpProtocol.INVALID_REQUEST).put("message", message)).toString() + + private fun close(s: Socket) { try { s.close() } catch (ignored: IOException) { } } +} diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt new file mode 100644 index 0000000000..273d27e5d3 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt @@ -0,0 +1,172 @@ +package org.appdevforall.sqlitedebugger.mcp + +import org.json.JSONArray +import org.json.JSONObject +import org.json.JSONTokener + +/** + * A tool an MCP client can call. [run] gets the call's arguments (an empty object when none were + * sent) and returns what happened, as JSON; it throws [McpToolError] for a failure the client + * should read, such as "a person is needed". + */ +class McpTool( + val name: String, + val title: String, + val description: String, + val inputSchema: JSONObject = JSONObject().put("type", "object").put("properties", JSONObject()).put("additionalProperties", false), + /** Only reports; changes nothing. Shown to clients as a hint, not relied on. */ + val readOnly: Boolean = true, + val run: (JSONObject) -> JSONObject, +) { + fun describe(): JSONObject = JSONObject() + .put("name", name).put("title", title).put("description", description) + .put("inputSchema", inputSchema) + .put("annotations", JSONObject().put("title", title).put("readOnlyHint", readOnly) + .put("destructiveHint", false).put("openWorldHint", false)) +} + +/** A tool's failure, told to the client as a tool result with isError, with [details] when there are any. */ +class McpToolError(message: String, val details: JSONObject? = null) : Exception(message) + +/** + * The Model Context Protocol, as the SQLite debugger speaks it: JSON-RPC 2.0 over the Streamable HTTP + * transport, answering every POST with plain JSON, never an event stream. Plain Kotlin and org.json, + * so unit tests run it without Android. [McpHttp] does the sockets and checks who's asking; this + * decides what to answer, and keeps the sessions. + * + * Methods: initialize, ping, tools/list, tools/call. Notifications, and responses a client sends, + * are accepted and need no answer (HTTP 202). + */ +class McpProtocol( + private val tools: List, + private val serverVersion: String, + private val newSessionId: () -> String, + private val maxSessions: Int = 16, +) { + companion object { + /** Newest first: offered when a client asks for a version this server doesn't know. */ + val VERSIONS = listOf("2025-06-18", "2025-03-26", "2024-11-05") + const val SERVER_NAME = "sqlite-debugger" + const val SESSION_HEADER = "Mcp-Session-Id" + const val PARSE_ERROR = -32700 + const val INVALID_REQUEST = -32600 + const val METHOD_NOT_FOUND = -32601 + const val INVALID_PARAMS = -32602 + const val INSTRUCTIONS = "Inspects and queries the SQLite databases of the app this server runs in, a debug " + + "build made with Code on the Go. Read-only unless the app's developer turned on read-write mode." + } + + /** What to send back: an HTTP status, a JSON body (or none), and extra headers. */ + class Reply(val status: Int, val body: String?, val headers: Map = emptyMap()) + + /** Live sessions, oldest first; the oldest is dropped when there are too many. */ + private val sessions = LinkedHashSet() + + /** A POST to the endpoint: one JSON-RPC message, or a batch of them. */ + fun post(body: String, sessionId: String?, versionHeader: String?): Reply { + val parsed = try { JSONTokener(body).nextValue() } catch (e: Exception) { null } + if (parsed !is JSONObject && parsed !is JSONArray) return Reply(400, error(null, PARSE_ERROR, "That isn't a JSON object or array.").toString()) + val messages: List = if (parsed is JSONArray) (0 until parsed.length()).map { parsed.opt(it) } else listOf(parsed) + if (messages.isEmpty()) return Reply(400, error(null, INVALID_REQUEST, "An empty batch.").toString()) + + if (messages.any { (it as? JSONObject)?.optString("method") == "initialize" }) { + if (messages.size != 1) return Reply(400, error(null, INVALID_REQUEST, "Send initialize on its own, not in a batch.").toString()) + return initialize(messages[0] as JSONObject) + } + if (sessionId.isNullOrEmpty()) return Reply(400, error(null, INVALID_REQUEST, "No $SESSION_HEADER header: send initialize first.").toString()) + if (!touch(sessionId)) return Reply(404, error(null, INVALID_REQUEST, "That session has ended or doesn't exist: send initialize again.").toString()) + if (versionHeader != null && versionHeader !in VERSIONS) { + return Reply(400, error(null, INVALID_REQUEST, "Unsupported MCP-Protocol-Version $versionHeader; this server speaks ${VERSIONS.joinToString()}.").toString()) + } + val replies = messages.mapNotNull { handle(it) } + if (replies.isEmpty()) return Reply(202, null) + return Reply(200, if (parsed is JSONArray) JSONArray(replies).toString() else replies[0].toString()) + } + + /** A DELETE: the client is done with its session. */ + fun delete(sessionId: String?): Reply { + if (sessionId.isNullOrEmpty()) return Reply(400, error(null, INVALID_REQUEST, "No $SESSION_HEADER header.").toString()) + val removed = synchronized(sessions) { sessions.remove(sessionId) } + return if (removed) Reply(204, null) else Reply(404, error(null, INVALID_REQUEST, "That session has ended or doesn't exist.").toString()) + } + + /** Ends every session, so clients must initialize again. */ + fun endAll() = synchronized(sessions) { sessions.clear() } + + fun sessionCount(): Int = synchronized(sessions) { sessions.size } + + private fun touch(id: String): Boolean = synchronized(sessions) { + if (!sessions.remove(id)) return false + sessions += id // most recently used goes last, so it's dropped last + true + } + + private fun initialize(m: JSONObject): Reply { + val id = m.opt("id") + if (!validId(id)) return Reply(400, error(null, INVALID_REQUEST, "initialize needs an id, a string or a number.").toString()) + if (m.optString("jsonrpc") != "2.0") return Reply(400, error(id, INVALID_REQUEST, "jsonrpc must be \"2.0\".").toString()) + val asked = m.optJSONObject("params")?.optString("protocolVersion", "") ?: "" + val version = if (asked in VERSIONS) asked else VERSIONS[0] + val session = newSessionId() + synchronized(sessions) { + sessions += session + while (sessions.size > maxSessions) sessions.remove(sessions.first()) + } + val result = JSONObject() + .put("protocolVersion", version) + .put("capabilities", JSONObject().put("tools", JSONObject().put("listChanged", false))) + .put("serverInfo", JSONObject().put("name", SERVER_NAME).put("title", "SQLite debugger").put("version", serverVersion)) + .put("instructions", INSTRUCTIONS) + return Reply(200, ok(id, result).toString(), mapOf(SESSION_HEADER to session)) + } + + /** One message of a POST; null when it needs no answer. */ + private fun handle(raw: Any?): JSONObject? { + val m = raw as? JSONObject ?: return error(null, INVALID_REQUEST, "Each message must be a JSON object.") + val hasId = m.has("id") + val id = m.opt("id") + val method = m.opt("method") as? String + if (method == null) { + // A response to something this server asked: it never asks, so there's nothing to do. + if (m.has("result") || m.has("error")) return null + return error(if (validId(id)) id else null, INVALID_REQUEST, "A request needs a method.") + } + if (!hasId) return null // a notification, such as notifications/initialized: no answer + if (!validId(id)) return error(null, INVALID_REQUEST, "id must be a string or a number.") + if (m.optString("jsonrpc") != "2.0") return error(id, INVALID_REQUEST, "jsonrpc must be \"2.0\".") + return when (method) { + "ping" -> ok(id, JSONObject()) + "tools/list" -> ok(id, JSONObject().put("tools", JSONArray().also { a -> tools.forEach { a.put(it.describe()) } })) + "tools/call" -> call(id, m.opt("params")) + else -> error(id, METHOD_NOT_FOUND, "Unknown method \"$method\".") + } + } + + private fun call(id: Any, rawParams: Any?): JSONObject { + val params = rawParams as? JSONObject ?: return error(id, INVALID_PARAMS, "tools/call needs params, with the tool's name.") + val name = params.opt("name") as? String ?: return error(id, INVALID_PARAMS, "tools/call needs the tool's name.") + val tool = tools.firstOrNull { it.name == name } ?: return error(id, INVALID_PARAMS, "Unknown tool \"$name\".") + val args = when (val a = params.opt("arguments")) { + null, JSONObject.NULL -> JSONObject() + is JSONObject -> a + else -> return error(id, INVALID_PARAMS, "arguments must be an object.") + } + val result = try { + val out = tool.run(args) + JSONObject().put("content", JSONArray().put(text(out.toString(2)))).put("structuredContent", out).put("isError", false) + } catch (e: McpToolError) { + JSONObject().put("content", JSONArray().put(text(e.message ?: "The tool failed."))).put("isError", true).also { r -> + e.details?.let { r.put("structuredContent", it) } + } + } catch (e: Exception) { + JSONObject().put("content", JSONArray().put(text("The tool failed: ${e.message ?: e.javaClass.simpleName}"))).put("isError", true) + } + return ok(id, result) + } + + private fun validId(id: Any?) = id is String || id is Number + private fun text(t: String) = JSONObject().put("type", "text").put("text", t) + private fun ok(id: Any, result: JSONObject) = JSONObject().put("jsonrpc", "2.0").put("id", id).put("result", result) + private fun error(id: Any?, code: Int, message: String) = JSONObject().put("jsonrpc", "2.0").put("id", id ?: JSONObject.NULL) + .put("error", JSONObject().put("code", code).put("message", message)) +} diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt new file mode 100644 index 0000000000..a1bf367dbd --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt @@ -0,0 +1,363 @@ +package org.appdevforall.sqlitedebugger.mcp + +import org.appdevforall.sqlitedebugger.mcp.McpHttp +import org.appdevforall.sqlitedebugger.mcp.McpProtocol +import org.appdevforall.sqlitedebugger.mcp.McpTool +import org.appdevforall.sqlitedebugger.mcp.McpToolError +import org.appdevforall.sqlitedebugger.mcp.McpTools +import org.json.JSONArray +import org.json.JSONObject +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import java.net.Socket + +/** + * MCP control without Android: the protocol (sessions, methods, tool results and errors) and the + * HTTP side (token, Host and Origin checks, methods and paths), on a real loopback socket. + */ +class McpProtocolTest { + + private var sessionNumber = 0 + private val echo = McpTool("echo", "Echo", "Returns its arguments.") { args -> JSONObject().put("got", args) } + private val needsPerson = McpTool("needs_person", "Needs a person", "Always says a person is needed.", readOnly = false) { _ -> + throw McpToolError("A person is needed: tap Allow on the sharing phone.", JSONObject().put("person_needed", true)) + } + private val crashes = McpTool("crashes", "Crashes", "Throws an unexpected exception.") { _ -> throw IllegalStateException("boom") } + + private fun protocol(maxSessions: Int = 16) = + McpProtocol(listOf(echo, needsPerson, crashes), "test", { "s${++sessionNumber}" }, maxSessions) + + private fun initBody(version: String = "2025-06-18") = JSONObject().put("jsonrpc", "2.0").put("id", 1).put("method", "initialize") + .put("params", JSONObject().put("protocolVersion", version).put("capabilities", JSONObject()) + .put("clientInfo", JSONObject().put("name", "test").put("version", "1"))).toString() + + private fun request(id: Any, method: String, params: JSONObject? = null) = JSONObject().put("jsonrpc", "2.0").put("id", id) + .put("method", method).also { if (params != null) it.put("params", params) } + + private fun session(p: McpProtocol): String { + val r = p.post(initBody(), null, null) + assertEquals(200, r.status) + return r.headers[McpProtocol.SESSION_HEADER]!! + } + + private fun call(p: McpProtocol, s: String, tool: String, args: JSONObject? = null): JSONObject { + val params = JSONObject().put("name", tool).also { if (args != null) it.put("arguments", args) } + val r = p.post(request(7, "tools/call", params).toString(), s, null) + assertEquals(200, r.status) + return JSONObject(r.body!!) + } + + // ---------------------------------------------------------------- the protocol + + @Test fun initializeAgreesVersionAndStartsSession() { + val p = protocol() + val r = p.post(initBody("2025-03-26"), null, null) + assertEquals(200, r.status) + assertEquals("s1", r.headers[McpProtocol.SESSION_HEADER]) + val result = JSONObject(r.body!!).getJSONObject("result") + assertEquals("2025-03-26", result.getString("protocolVersion")) + assertEquals(McpProtocol.SERVER_NAME, result.getJSONObject("serverInfo").getString("name")) + assertTrue(result.getJSONObject("capabilities").has("tools")) + assertEquals(1, p.sessionCount()) + } + + @Test fun unknownVersionGetsTheNewest() { + val r = protocol().post(initBody("1999-01-01"), null, null) + assertEquals(McpProtocol.VERSIONS[0], JSONObject(r.body!!).getJSONObject("result").getString("protocolVersion")) + } + + @Test fun requestsNeedALiveSession() { + val p = protocol() + assertEquals(400, p.post(request(1, "ping").toString(), null, null).status) + assertEquals(404, p.post(request(1, "ping").toString(), "nope", null).status) + val s = session(p) + assertEquals(200, p.post(request(1, "ping").toString(), s, null).status) + } + + @Test fun unsupportedVersionHeaderIsRefused() { + val p = protocol(); val s = session(p) + assertEquals(400, p.post(request(1, "ping").toString(), s, "1999-01-01").status) + assertEquals(200, p.post(request(1, "ping").toString(), s, "2025-06-18").status) + } + + @Test fun notificationsGetNoAnswer() { + val p = protocol(); val s = session(p) + val r = p.post(JSONObject().put("jsonrpc", "2.0").put("method", "notifications/initialized").toString(), s, null) + assertEquals(202, r.status) + assertNull(r.body) + } + + @Test fun pingAnswersEmptyResult() { + val p = protocol(); val s = session(p) + val r = JSONObject(p.post(request("a", "ping").toString(), s, null).body!!) + assertEquals("a", r.get("id")) + assertEquals(0, r.getJSONObject("result").length()) + } + + @Test fun toolsListDescribesEveryTool() { + val p = protocol(); val s = session(p) + val tools = JSONObject(p.post(request(2, "tools/list").toString(), s, null).body!!).getJSONObject("result").getJSONArray("tools") + val names = (0 until tools.length()).map { tools.getJSONObject(it).getString("name") } + assertEquals(listOf("echo", "needs_person", "crashes"), names) + val first = tools.getJSONObject(0) + assertEquals("object", first.getJSONObject("inputSchema").getString("type")) + assertTrue(first.getJSONObject("annotations").getBoolean("readOnlyHint")) + assertFalse(tools.getJSONObject(1).getJSONObject("annotations").getBoolean("readOnlyHint")) + } + + @Test fun toolCallReturnsStructuredContent() { + val p = protocol(); val s = session(p) + val result = call(p, s, "echo", JSONObject().put("x", 3)).getJSONObject("result") + assertFalse(result.getBoolean("isError")) + assertEquals(3, result.getJSONObject("structuredContent").getJSONObject("got").getInt("x")) + assertEquals("text", result.getJSONArray("content").getJSONObject(0).getString("type")) + } + + @Test fun toolCallWithoutArgumentsGetsEmptyObject() { + val p = protocol(); val s = session(p) + val got = call(p, s, "echo").getJSONObject("result").getJSONObject("structuredContent").getJSONObject("got") + assertEquals(0, got.length()) + } + + @Test fun toolErrorIsAResultWithIsError() { + val p = protocol(); val s = session(p) + val result = call(p, s, "needs_person").getJSONObject("result") + assertTrue(result.getBoolean("isError")) + assertTrue(result.getJSONArray("content").getJSONObject(0).getString("text").contains("person is needed")) + assertTrue(result.getJSONObject("structuredContent").getBoolean("person_needed")) + } + + @Test fun unexpectedExceptionIsAResultWithIsError() { + val p = protocol(); val s = session(p) + val result = call(p, s, "crashes").getJSONObject("result") + assertTrue(result.getBoolean("isError")) + assertTrue(result.getJSONArray("content").getJSONObject(0).getString("text").contains("boom")) + } + + @Test fun unknownToolAndBadArgumentsAreInvalidParams() { + val p = protocol(); val s = session(p) + assertEquals(McpProtocol.INVALID_PARAMS, call(p, s, "nope").getJSONObject("error").getInt("code")) + val bad = request(3, "tools/call", JSONObject().put("name", "echo").put("arguments", JSONArray())).toString() + assertEquals(McpProtocol.INVALID_PARAMS, JSONObject(p.post(bad, s, null).body!!).getJSONObject("error").getInt("code")) + } + + @Test fun unknownMethodIsMethodNotFound() { + val p = protocol(); val s = session(p) + val r = JSONObject(p.post(request(4, "resources/list").toString(), s, null).body!!) + assertEquals(McpProtocol.METHOD_NOT_FOUND, r.getJSONObject("error").getInt("code")) + } + + @Test fun badJsonIsAParseError() { + val r = protocol().post("{not json", null, null) + assertEquals(400, r.status) + assertEquals(McpProtocol.PARSE_ERROR, JSONObject(r.body!!).getJSONObject("error").getInt("code")) + } + + @Test fun batchGetsAnArrayOfReplies() { + val p = protocol(); val s = session(p) + val batch = JSONArray().put(request(1, "ping")).put(JSONObject().put("jsonrpc", "2.0").put("method", "notifications/x")).put(request(2, "tools/list")) + val r = p.post(batch.toString(), s, null) + assertEquals(200, r.status) + val replies = JSONArray(r.body!!) + assertEquals(2, replies.length()) + assertEquals(1, replies.getJSONObject(0).getInt("id")) + assertEquals(2, replies.getJSONObject(1).getInt("id")) + } + + @Test fun initializeMustBeAlone() { + val batch = JSONArray().put(JSONObject(initBody())).put(request(2, "ping")) + assertEquals(400, protocol().post(batch.toString(), null, null).status) + } + + @Test fun deleteEndsTheSession() { + val p = protocol(); val s = session(p) + assertEquals(204, p.delete(s).status) + assertEquals(404, p.delete(s).status) + assertEquals(404, p.post(request(1, "ping").toString(), s, null).status) + } + + @Test fun oldestSessionIsDroppedBeyondTheLimit() { + val p = protocol(maxSessions = 2) + val a = session(p); val b = session(p) + p.post(request(1, "ping").toString(), a, null) // a is now the most recently used + val c = session(p) // drops b + assertEquals(2, p.sessionCount()) + assertEquals(200, p.post(request(1, "ping").toString(), a, null).status) + assertEquals(404, p.post(request(1, "ping").toString(), b, null).status) + assertEquals(200, p.post(request(1, "ping").toString(), c, null).status) + } + + @Test fun endAllEndsEverySession() { + val p = protocol(); val s = session(p) + p.endAll() + assertEquals(0, p.sessionCount()) + assertEquals(404, p.post(request(1, "ping").toString(), s, null).status) + } + + @Test fun statusLineBecomesOneWord() { + assertEquals("serving", McpTools.stateOf("Serving Store_12345678", "http://192.168.1.5:6175/")) + assertEquals("failed", McpTools.stateOf("Failed: storage access hasn't been granted", null)) + assertEquals("paused", McpTools.stateOf("Paused while the screen is off.", null)) + assertEquals("stopped", McpTools.stateOf("Stopped", null)) + assertEquals("starting", McpTools.stateOf("Starting…", null)) + assertEquals("starting", McpTools.stateOf("Restarting after the server stopped unexpectedly…", null)) + assertEquals("starting", McpTools.stateOf("Not on Wi-Fi — starting a hotspot…", null)) + } + + // ---------------------------------------------------------------- step 3: share and server tools + + private val share = org.appdevforall.localstore.server.ShareControl + + @Test fun shareStateBecomesOneWord() { + assertEquals("waiting_for_phone", share.stateWord("OPEN", 0, false)) + assertEquals("preparing", share.stateWord("PREPARING", 0, false)) + assertEquals("phone_connecting", share.stateWord("ASKED", 0, false)) + assertEquals("waiting_for_allow", share.stateWord("REVEALED", 0, false)) + assertEquals("sending", share.stateWord("ALLOWED", 0, false)) + assertEquals("transfer_failed", share.stateWord("ALLOWED", 2, false)) + assertEquals("done", share.stateWord("DONE", 0, false)) + assertEquals("denied", share.stateWord("DENIED", 0, false)) + assertEquals("expired", share.stateWord("CLOSED", 0, false)) + assertEquals("not_serving", share.stateWord("NONE", 0, false)) + assertEquals("not_serving", share.stateWord(null, 0, false)) + assertEquals("cant_share", share.stateWord("ERROR", 0, true)) + assertEquals("closed", share.stateWord("ERROR", 0, false)) + } + + /** The rule MCP control rests on: ShareControl may open, poll and close a share, and never allow or deny one. */ + @Test fun shareControlCanNeverAllowOrDeny() { + val ipc = org.appdevforall.localstore.server.Ipc + val field = share.javaClass.getDeclaredField("ALLOWED_MESSAGES").apply { isAccessible = true } + val allowed = field.get(null) as Set<*> + assertEquals(setOf(ipc.MSG_SHARE_OPEN, ipc.MSG_SHARE_POLL, ipc.MSG_SHARE_CLOSE), allowed) + assertFalse(ipc.MSG_SHARE_ALLOW in allowed) + assertFalse(ipc.MSG_SHARE_DENY in allowed) + } + + @Test fun waitSecondsIsCheckedAndCapped() { + assertEquals(10, McpTools.waitArg(JSONObject(), 10)) + assertEquals(10, McpTools.waitArg(JSONObject().put("wait_seconds", JSONObject.NULL), 10)) + assertEquals(5, McpTools.waitArg(JSONObject().put("wait_seconds", 5), 10)) + assertEquals(2, McpTools.waitArg(JSONObject().put("wait_seconds", 2.7), 10)) + assertEquals(share.MAX_WAIT_S, McpTools.waitArg(JSONObject().put("wait_seconds", 999), 10)) + assertEquals(0, McpTools.waitArg(JSONObject().put("wait_seconds", -3), 10)) + assertTrue(runCatching { McpTools.waitArg(JSONObject().put("wait_seconds", "5"), 10) }.exceptionOrNull() is McpToolError) + } + + @Test fun storeArgumentIsOwnOrCopy() { + assertNull(McpTools.storeArg(JSONObject())) + assertNull(McpTools.storeArg(JSONObject().put("store", JSONObject.NULL))) + assertEquals("own", McpTools.storeArg(JSONObject().put("store", "own"))) + assertEquals("copy", McpTools.storeArg(JSONObject().put("store", "copy"))) + assertTrue(runCatching { McpTools.storeArg(JSONObject().put("store", "both")) }.exceptionOrNull() is McpToolError) + assertTrue(runCatching { McpTools.storeArg(JSONObject().put("store", 1)) }.exceptionOrNull() is McpToolError) + } + + // ---------------------------------------------------------------- the HTTP side + + private val token = "secret-token" + private var server: McpHttp? = null + + @After fun stopServer() { server?.stop(); server = null } + + private fun startServer(): Int { + val s = McpHttp(protocol(), { token }, port = 0) + s.start() + server = s + return s.boundPort + } + + private class Answer(val status: Int, val headers: Map, val body: String) + + /** One request on its own connection; [headers] may replace Host. */ + private fun http(port: Int, method: String, path: String = McpHttp.PATH, headers: Map = emptyMap(), body: String? = null): Answer { + Socket("127.0.0.1", port).use { s -> + s.soTimeout = 5000 + val bytes = body?.toByteArray(Charsets.UTF_8) ?: ByteArray(0) + val all = linkedMapOf("Host" to "127.0.0.1:$port", "Connection" to "close") + all.putAll(headers) + if (body != null) { all["Content-Type"] = "application/json"; all["Content-Length"] = bytes.size.toString() } + val head = StringBuilder("$method $path HTTP/1.1\r\n") + for ((k, v) in all) head.append(k).append(": ").append(v).append("\r\n") + head.append("\r\n") + s.getOutputStream().apply { write(head.toString().toByteArray(Charsets.ISO_8859_1)); write(bytes); flush() } + val text = s.getInputStream().readBytes().toString(Charsets.UTF_8) + val lines = text.substringBefore("\r\n\r\n").split("\r\n") + val status = lines[0].split(' ')[1].toInt() + val h = lines.drop(1).associate { it.substringBefore(':').trim().lowercase() to it.substringAfter(':').trim() } + return Answer(status, h, text.substringAfter("\r\n\r\n", "")) + } + } + + private val auth get() = mapOf("Authorization" to "Bearer $token") + + @Test fun httpNeedsTheToken() { + val port = startServer() + val none = http(port, "POST", body = initBody()) + assertEquals(401, none.status) + assertNotNull(none.headers["www-authenticate"]) + assertEquals(401, http(port, "POST", headers = mapOf("Authorization" to "Bearer wrong"), body = initBody()).status) + assertFalse("the token never appears in a refusal", none.body.contains(token)) + } + + @Test fun httpInitializeWithTheToken() { + val port = startServer() + val r = http(port, "POST", headers = auth, body = initBody()) + assertEquals(200, r.status) + assertNotNull(r.headers["mcp-session-id"]) + assertEquals("2025-06-18", JSONObject(r.body).getJSONObject("result").getString("protocolVersion")) + } + + @Test fun httpLocalhostHostIsAccepted() { + val port = startServer() + assertEquals(200, http(port, "POST", headers = auth + ("Host" to "localhost:$port"), body = initBody()).status) + } + + @Test fun httpOtherHostIsRefused() { + val port = startServer() + assertEquals(403, http(port, "POST", headers = auth + ("Host" to "evil.example"), body = initBody()).status) + assertEquals(403, http(port, "POST", headers = auth + ("Host" to "127.0.0.1:x"), body = initBody()).status) + } + + @Test fun httpWebPageOriginIsRefused() { + val port = startServer() + assertEquals(403, http(port, "POST", headers = auth + ("Origin" to "https://evil.example"), body = initBody()).status) + assertEquals(200, http(port, "POST", headers = auth + ("Origin" to "http://localhost:3000"), body = initBody()).status) + } + + @Test fun httpGetIsNotAllowed() { + val port = startServer() + val r = http(port, "GET", headers = auth) + assertEquals(405, r.status) + assertEquals("POST, DELETE", r.headers["allow"]) + } + + @Test fun httpOtherPathIsNotFound() { + assertEquals(404, http(startServer(), "POST", path = "/other", headers = auth, body = initBody()).status) + } + + @Test fun httpWrongContentTypeIsRefused() { + val port = startServer() + Socket("127.0.0.1", port).use { s -> + val body = initBody().toByteArray() + s.getOutputStream().write(("POST /mcp HTTP/1.1\r\nHost: 127.0.0.1\r\nAuthorization: Bearer $token\r\nConnection: close\r\n" + + "Content-Type: text/plain\r\nContent-Length: ${body.size}\r\n\r\n").toByteArray() + body) + val text = s.getInputStream().readBytes().toString(Charsets.UTF_8) + assertTrue(text.startsWith("HTTP/1.1 415")) + } + } + + @Test fun httpSessionThenDelete() { + val port = startServer() + val s = http(port, "POST", headers = auth, body = initBody()).headers["mcp-session-id"]!! + val ping = request(1, "ping").toString() + assertEquals(200, http(port, "POST", headers = auth + ("Mcp-Session-Id" to s), body = ping).status) + assertEquals(204, http(port, "DELETE", headers = auth + ("Mcp-Session-Id" to s)).status) + assertEquals(404, http(port, "POST", headers = auth + ("Mcp-Session-Id" to s), body = ping).status) + } +} From fab6cff8525428e18512cbb8c27cc4a4963f70b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 04:41:11 +0000 Subject: [PATCH 03/24] ADFA-7135 Add the sqlite-debugger module skeleton Module build file modelled on sqlite-js (no ABI flavors, both ABIs, no STL, 16 KiB pages), plus Kotlin at API 2.0 for apps built with CoGo. Registered in settings, modulesWithoutAbiFlavors and disableCoreLibDesugaringForModules. The CMake file arrives with the extension in the next commit. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- .../plugins/conf/AndroidModuleConf.kt | 8 ++- settings.gradle.kts | 1 + subprojects/sqlite-debugger/build.gradle.kts | 63 +++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 subprojects/sqlite-debugger/build.gradle.kts diff --git a/composite-builds/build-logic/plugins/src/main/java/com/itsaky/androidide/plugins/conf/AndroidModuleConf.kt b/composite-builds/build-logic/plugins/src/main/java/com/itsaky/androidide/plugins/conf/AndroidModuleConf.kt index 6dd8daa757..e418cb787a 100644 --- a/composite-builds/build-logic/plugins/src/main/java/com/itsaky/androidide/plugins/conf/AndroidModuleConf.kt +++ b/composite-builds/build-logic/plugins/src/main/java/com/itsaky/androidide/plugins/conf/AndroidModuleConf.kt @@ -53,13 +53,17 @@ private val disableCoreLibDesugaringForModules = ":quickbuild:runtime", // The AAR is a dependency of apps built with CoGo (ADFA-7082). ":subprojects:sqlite-js", + // Injected into debug builds of apps built with CoGo like :logsender (ADFA-7131). + ":subprojects:sqlite-debugger", ) /** * Modules without the v7/v8 flavors: libraries for other projects, which need one artifact. - * :subprojects:sqlite-js is an AAR for apps built with CoGo, with all the ABIs they run on. + * :subprojects:sqlite-js and :subprojects:sqlite-debugger are AARs for apps built with CoGo, with + * all the ABIs they run on. */ -private val modulesWithoutAbiFlavors = setOf(":plugin-api", ":subprojects:sqlite-js") +private val modulesWithoutAbiFlavors = + setOf(":plugin-api", ":subprojects:sqlite-js", ":subprojects:sqlite-debugger") /** * The name of the build type used for automated testing. diff --git a/settings.gradle.kts b/settings.gradle.kts index 5538bb5e0a..a3b77f05a5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -170,6 +170,7 @@ include( ":subprojects:shizuku-server-shared", ":subprojects:shizuku-shared", ":subprojects:shizuku-starter", + ":subprojects:sqlite-debugger", ":subprojects:sqlite-js", ":subprojects:tooling-api", ":subprojects:tooling-api-events", diff --git a/subprojects/sqlite-debugger/build.gradle.kts b/subprojects/sqlite-debugger/build.gradle.kts new file mode 100644 index 0000000000..cfcf5bd174 --- /dev/null +++ b/subprojects/sqlite-debugger/build.gradle.kts @@ -0,0 +1,63 @@ +import com.itsaky.androidide.build.config.BuildConfig + +plugins { + id("com.android.library") + id("kotlin-android") +} + +description = "SQLite debugger for debug builds of apps built with Code on the Go (ADFA-7131)" + +android { + namespace = "org.appdevforall.sqlitedebugger" + ndkVersion = BuildConfig.NDK_VERSION + + defaultConfig { + // requery's minSdk, not MIN_SDK_FOR_APPS_BUILT_WITH_COGO: the debugger runs on requery's SQLite + minSdk = 21 + // One AAR for both ABIs of the phones apps built with CoGo run on, unlike CoGo's own + // modules: this module has no v7/v8 flavors (AndroidModuleConf). + ndk { + abiFilters.clear() + abiFilters += listOf("arm64-v8a", "armeabi-v7a") + } + externalNativeBuild { + cmake { + arguments += "-DANDROID_STL=none" + } + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + + buildFeatures { + viewBinding = false + buildConfig = false + } + + externalNativeBuild { + cmake { + path = file("src/main/cpp/CMakeLists.txt") + version = "3.31.0+" + } + } +} + +kotlin { + compilerOptions { + jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_1_8) + // Apps compile against this library with Code on the Go's on-device Kotlin compiler, + // which can be older than the one that builds it (as in :plugin-api). + apiVersion.set(org.jetbrains.kotlin.gradle.dsl.KotlinVersion.KOTLIN_2_0) + languageVersion.set(org.jetbrains.kotlin.gradle.dsl.KotlinVersion.KOTLIN_2_0) + } +} + +dependencies { + // Android provides org.json on a device + testImplementation(libs.org.json) + testImplementation(libs.tests.junit) + testImplementation(libs.tests.google.truth) +} From 1b53c596af436e5bb50116c90e1cec26463bfa81 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 04:48:49 +0000 Subject: [PATCH 04/24] ADFA-7136: MCP server with the eight read-only database tools Binds 127.0.0.1 on port 0 with a new token per process start, and serves list_databases, list_connections, get_schema, query, explain_query_plan, get_stats, integrity_check and get_events against DebugEngine. JVM tests run against FakeDebugEngine (51 pass). DebuggerStartup.kt and the manifest are Android-only and not yet compiled. Formatting follows in its own commit. Co-Authored-By: Claude Sonnet 5.5 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- .../src/main/AndroidManifest.xml | 15 + .../sqlitedebugger/DebuggerStartup.kt | 68 ++ .../sqlitedebugger/mcp/DatabaseTools.kt | 382 ++++++++++ .../sqlitedebugger/mcp/McpHttp.kt | 541 ++++++++------- .../sqlitedebugger/mcp/McpProtocol.kt | 284 ++++---- .../sqlitedebugger/mcp/DatabaseToolsTest.kt | 318 +++++++++ .../sqlitedebugger/mcp/McpProtocolTest.kt | 656 +++++++++--------- 7 files changed, 1520 insertions(+), 744 deletions(-) create mode 100644 subprojects/sqlite-debugger/src/main/AndroidManifest.xml create mode 100644 subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/DebuggerStartup.kt create mode 100644 subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt create mode 100644 subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt diff --git a/subprojects/sqlite-debugger/src/main/AndroidManifest.xml b/subprojects/sqlite-debugger/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..18de3c738f --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/AndroidManifest.xml @@ -0,0 +1,15 @@ + + + + + + + + + + diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/DebuggerStartup.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/DebuggerStartup.kt new file mode 100644 index 0000000000..2328ca17d1 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/DebuggerStartup.kt @@ -0,0 +1,68 @@ +package org.appdevforall.sqlitedebugger + +import android.content.ContentProvider +import android.content.ContentValues +import android.content.pm.ApplicationInfo +import android.database.Cursor +import android.net.Uri +import android.util.Log +import java.io.File +import kotlin.concurrent.thread +import org.appdevforall.sqlitedebugger.engine.NativeDebugEngine +import org.appdevforall.sqlitedebugger.mcp.DatabaseTools +import org.appdevforall.sqlitedebugger.mcp.McpHttp +import org.appdevforall.sqlitedebugger.mcp.McpProtocol +import java.util.UUID + +/** + * Starts the debugger when the app's process starts. Refuses to run in a build that is not + * debuggable, so a release build that includes this library by mistake crashes at once instead of + * opening a local server. + */ +class DebuggerStartup : ContentProvider() { + override fun onCreate(): Boolean { + val context = context ?: return false + check(context.applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE != 0) { + "The SQLite debugger is in a build that is not debuggable. Remove it from release builds." + } + val databaseDir = File(context.applicationInfo.dataDir, "databases") + thread(name = "sqlite-debugger-start", isDaemon = true) { + try { + val engine = NativeDebugEngine() + engine.install() + val tools = DatabaseTools(engine, databaseDir) + val token = McpHttp.newToken() + val protocol = McpProtocol(tools.tools(), "1", { UUID.randomUUID().toString() }) + val http = McpHttp(protocol, { token }, log = { Log.w(TAG, it) }) + http.start() + port = http.boundPort + endpointToken = token + Log.i(TAG, "listening on 127.0.0.1:${http.boundPort}") + } catch (e: Exception) { + Log.e(TAG, "could not start", e) + } + } + return true + } + + override fun query(uri: Uri, projection: Array?, selection: String?, selectionArgs: Array?, sortOrder: String?): Cursor? = null + + override fun getType(uri: Uri): String? = null + + override fun insert(uri: Uri, values: ContentValues?): Uri? = null + + override fun delete(uri: Uri, selection: String?, selectionArgs: Array?): Int = 0 + + override fun update(uri: Uri, values: ContentValues?, selection: String?, selectionArgs: Array?): Int = 0 + + companion object { + private const val TAG = "SqliteDebugger" + + /** Set once the server listens; the broadcast to Code on the Go (ADFA-7137) reads them. */ + @Volatile var port: Int = 0 + private set + + @Volatile var endpointToken: String? = null + private set + } +} diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt new file mode 100644 index 0000000000..477b8d9d60 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt @@ -0,0 +1,382 @@ +package org.appdevforall.sqlitedebugger.mcp + +import java.io.File +import org.appdevforall.sqlitedebugger.engine.DebugEngine +import org.appdevforall.sqlitedebugger.engine.DebugEngineException +import org.appdevforall.sqlitedebugger.engine.QueryLimits +import org.appdevforall.sqlitedebugger.engine.QueryResult +import org.appdevforall.sqlitedebugger.engine.ServerConnection +import org.json.JSONArray +import org.json.JSONObject + +/** + * The debugger's MCP tools, all read-only: they open server connections through [engine] with + * `writable = false`, so SQLite's authorizer refuses a write whatever SQL a client sends. A tool + * names a database by the path one of the app's own connections has open; any other path is + * refused before the engine is called (opening a file Android's SQLite has open, with the + * extension's SQLite, can corrupt it). + */ +class DatabaseTools( + private val engine: DebugEngine, + /** The app's database folder, to list files no connection has opened; null on the JVM. */ + private val databaseDir: File? = null, + private val clock: () -> Long = System::currentTimeMillis, +) { + companion object { + const val DEFAULT_MAX_ROWS = 100 + const val MAX_ROWS = 1000 + const val MAX_TEXT_BYTES = 4096 + const val MAX_BLOB_BYTES = 1024 + + /** Below the client's 60 s limit on a tool call. */ + const val DEADLINE_MS = 50_000L + const val EVENT_BATCH_BYTES = 256 * 1024 + const val MAX_TABLES_FOR_COUNTS = 10 + + private const val MAX_SAFE_INTEGER = 9007199254740992L // 2^53 + private val SIDE_FILE_SUFFIXES = listOf("-wal", "-shm", "-journal") + + const val PAGING_NOTE = "Paging re-runs the query for each page, so a later page can differ if the data changed." + + /** A SQLite value as JSON: org.json throws on NaN and infinities, and JSON numbers lose integers past 2^53. */ + fun encodeValue( + value: Any?, + maxBlobBytes: Int = MAX_BLOB_BYTES, + ): Any = + when (value) { + null -> JSONObject.NULL + is Long -> encodeLong(value) + is Int -> value.toLong() + is Double -> + when { + value.isNaN() -> "NaN" + value == Double.POSITIVE_INFINITY -> "Infinity" + value == Double.NEGATIVE_INFINITY -> "-Infinity" + else -> value + } + is String -> value + is ByteArray -> + JSONObject() + .put("base64", McpHttp.base64(value, urlSafe = false, pad = true)) + .put("bytes", value.size) + // the engine cuts at the limit without saying how long the BLOB was + .put("truncated", value.size >= maxBlobBytes) + else -> value.toString() + } + + fun encodeLong(value: Long): Any = if (value > MAX_SAFE_INTEGER || value < -MAX_SAFE_INTEGER) value.toString() else value + + fun quoteIdentifier(name: String) = "\"" + name.replace("\"", "\"\"") + "\"" + } + + private val lock = Any() + private val serverConnections = HashMap() + private var clientConnected = false + + /** Closes the server connections and tells the engine no client is watching. Call when the server stops. */ + fun onServerStopped() { + synchronized(lock) { + serverConnections.values.forEach { runCatching { it.close() } } + serverConnections.clear() + if (clientConnected) { + clientConnected = false + engine.setClientConnected(false) + } + } + } + + fun tools(): List = + listOf( + tool("list_databases", "List databases", "Lists the app's databases: each one an app connection has open (available), and the files in the app's database folder that none has opened (not available: not inspectable until the app opens them).") { listDatabases() }, + tool("list_connections", "List connections", "Lists the app's open SQLite connections (not yet freed by SQLite), oldest first.") { listConnections() }, + tool("get_schema", "Get schema", "Returns the tables, indexes, views and triggers of a database, with their CREATE statements, from sqlite_schema.", databaseSchema()) { getSchema(it) }, + tool( + "query", + "Run a query", + "Runs one read-only SQL statement. Returns at most max_rows rows (default $DEFAULT_MAX_ROWS, at most $MAX_ROWS); " + + "TEXT cells are cut at $MAX_TEXT_BYTES bytes and BLOBs at $MAX_BLOB_BYTES. Use offset to page. $PAGING_NOTE Writes are refused.", + databaseSchema( + "sql" to prop("string", "The SQL statement."), + "args" to JSONObject().put("type", "array").put("description", "Values for ? parameters: null, numbers or strings."), + "offset" to prop("integer", "Rows to skip. Default 0."), + "max_rows" to prop("integer", "Rows to return. Default $DEFAULT_MAX_ROWS, at most $MAX_ROWS."), + required = listOf("sql"), + ), + ) { query(it) }, + tool( + "explain_query_plan", + "Explain a query plan", + "Runs EXPLAIN QUERY PLAN on a SQL statement and returns SQLite's plan. The statement is not run.", + databaseSchema( + "sql" to prop("string", "The SQL statement."), + "args" to JSONObject().put("type", "array").put("description", "Values for ? parameters."), + required = listOf("sql"), + ), + ) { explain(it) }, + tool("get_stats", "Get statistics", "Returns page_count, page_size, freelist_count, and the row count of each table when the database has fewer than $MAX_TABLES_FOR_COUNTS tables.", databaseSchema()) { getStats(it) }, + tool( + "integrity_check", + "Check integrity", + "Runs PRAGMA quick_check, or the slower PRAGMA integrity_check when full is true. ok is true when SQLite reports no problem.", + databaseSchema("full" to prop("boolean", "Run the full integrity_check. Default false."), required = emptyList()), + ) { integrityCheck(it) }, + tool( + "get_events", + "Get SQL events", + "Returns the SQL the app ran, oldest first, after the cursor. Pass the returned next as after to continue; dropped counts events that left the buffer unread. The first call starts recording bound values into the SQL.", + JSONObject().put("type", "object").put("additionalProperties", false) + .put("properties", JSONObject().put("after", prop("integer", "Cursor: the next from the last call. Default 0."))) + .put("required", JSONArray()), + ) { getEvents(it) }, + ) + + private fun tool( + name: String, + title: String, + description: String, + inputSchema: JSONObject = JSONObject().put("type", "object").put("properties", JSONObject()).put("additionalProperties", false), + run: (JSONObject) -> JSONObject, + ) = McpTool(name, title, description, inputSchema, readOnly = true, run = run) + + private fun prop( + type: String, + description: String, + ) = JSONObject().put("type", type).put("description", description) + + private fun databaseSchema( + vararg extra: Pair, + required: List = emptyList(), + ): JSONObject { + val props = JSONObject().put("database", prop("string", "The database path, as list_databases gives it.")) + extra.forEach { props.put(it.first, it.second) } + return JSONObject().put("type", "object").put("properties", props).put("additionalProperties", false) + .put("required", JSONArray(listOf("database") + required)) + } + + private fun listDatabases(): JSONObject { + val connections = engine.connections() + val byPath = LinkedHashMap>() + for (c in connections) if (c.path.isNotEmpty()) byPath.getOrPut(c.path) { mutableListOf() } += c.id + val list = JSONArray() + for ((path, ids) in byPath) { + list.put(JSONObject().put("path", path).put("available", true).put("connections", ids.size)) + } + databaseDir?.listFiles()?.sortedBy { it.name }?.forEach { f -> + if (!f.isFile || SIDE_FILE_SUFFIXES.any { f.name.endsWith(it) }) return@forEach + if (f.absolutePath in byPath || f.path in byPath) return@forEach + list.put(JSONObject().put("path", f.absolutePath).put("available", false).put("connections", 0)) + } + return JSONObject().put("databases", list) + } + + private fun listConnections(): JSONObject { + val list = JSONArray() + for (c in engine.connections()) { + list.put( + JSONObject().put("id", encodeLong(c.id)).put("path", c.path).put("read_only", c.readOnly).put("closing", c.closing) + .put("opened_at_epoch_nanos", encodeLong(c.openedAtEpochNanos)).put("open_thread_id", encodeLong(c.openThreadId)), + ) + } + return JSONObject().put("connections", list) + } + + private fun getSchema(args: JSONObject): JSONObject { + val result = run(args, "SELECT type, name, tbl_name, sql FROM sqlite_schema ORDER BY type, name", emptyList(), 0, QueryLimits(MAX_ROWS, 65536, MAX_BLOB_BYTES)) + val groups = linkedMapOf("table" to JSONArray(), "index" to JSONArray(), "view" to JSONArray(), "trigger" to JSONArray()) + for (row in result.rows) { + val item = JSONObject().put("name", encodeValue(row[1])).put("table", encodeValue(row[2])).put("sql", encodeValue(row[3])) + groups[row[0] as? String]?.put(item) + } + return JSONObject().put("tables", groups["table"]).put("indexes", groups["index"]).put("views", groups["view"]) + .put("triggers", groups["trigger"]).put("truncated", result.hasMore) + } + + private fun query(args: JSONObject): JSONObject { + val sql = stringArg(args, "sql") + val offset = intArg(args, "offset", 0, 0, Long.MAX_VALUE) + val maxRows = intArg(args, "max_rows", DEFAULT_MAX_ROWS.toLong(), 1, MAX_ROWS.toLong()).toInt() + val result = run(args, sql, bindArgs(args), offset, QueryLimits(maxRows, MAX_TEXT_BYTES, MAX_BLOB_BYTES)) + val out = encodeResult(result, offset) + out.put("note", PAGING_NOTE) + return out + } + + private fun explain(args: JSONObject): JSONObject { + val result = run(args, "EXPLAIN QUERY PLAN " + stringArg(args, "sql"), bindArgs(args), 0, QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES)) + return JSONObject().put("plan", rowsAsObjects(result)) + } + + private fun getStats(args: JSONObject): JSONObject { + val limits = QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES) + val deadline = clock() + DEADLINE_MS + val path = checkedPath(args) + fun pragma(name: String) = runChecked(path, "PRAGMA $name", emptyList(), 0, limits, deadline).rows.firstOrNull()?.firstOrNull() + val pageCount = pragma("page_count") as? Long + val pageSize = pragma("page_size") as? Long + val out = JSONObject().put("page_count", encodeValue(pageCount)).put("page_size", encodeValue(pageSize)) + .put("freelist_count", encodeValue(pragma("freelist_count"))) + if (pageCount != null && pageSize != null) out.put("size_bytes", encodeLong(pageCount * pageSize)) + val tables = runChecked(path, "SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", emptyList(), 0, limits, deadline) + .rows.map { it[0] as String } + if (tables.size < MAX_TABLES_FOR_COUNTS) { + val counts = JSONObject() + for (t in tables) { + val n = try { + runChecked(path, "SELECT COUNT(*) FROM ${quoteIdentifier(t)}", emptyList(), 0, limits, deadline).rows.firstOrNull()?.firstOrNull() + } catch (e: McpToolError) { + e.message + } + counts.put(t, encodeValue(n)) + } + out.put("row_counts", counts) + } else { + out.put("row_counts_skipped", "The database has ${tables.size} tables; counts are given only under $MAX_TABLES_FOR_COUNTS.") + } + return out + } + + private fun integrityCheck(args: JSONObject): JSONObject { + val full = when (val v = args.opt("full")) { + null, JSONObject.NULL -> false + is Boolean -> v + else -> throw McpToolError("full must be true or false.") + } + val pragma = if (full) "PRAGMA integrity_check" else "PRAGMA quick_check" + val result = run(args, pragma, emptyList(), 0, QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES)) + val messages = result.rows.map { it.firstOrNull()?.toString() ?: "" } + return JSONObject().put("check", if (full) "integrity_check" else "quick_check") + .put("ok", messages == listOf("ok")).put("messages", JSONArray(messages)) + } + + private fun getEvents(args: JSONObject): JSONObject { + val after = intArg(args, "after", 0, 0, Long.MAX_VALUE) + synchronized(lock) { + if (!clientConnected) { + engine.setClientConnected(true) + clientConnected = true + } + } + val batch = try { + engine.readEvents(after, EVENT_BATCH_BYTES) + } catch (e: DebugEngineException) { + throw engineError(e) + } + val events = JSONArray() + for (e in batch.events) { + events.put( + JSONObject().put("seq", encodeLong(e.seq)).put("kind", e.kind.name.lowercase()).put("connection_id", encodeLong(e.connectionId)) + .put("thread_id", encodeLong(e.threadId)).put("time_epoch_nanos", encodeLong(e.timeEpochNanos)) + .put("duration_nanos", e.durationNanos?.let { encodeLong(it) } ?: JSONObject.NULL) + .put("sql", e.sql).put("truncated", e.truncated).put("expanded", e.expanded), + ) + } + return JSONObject().put("next", encodeLong(batch.next)).put("dropped", encodeLong(batch.dropped)).put("events", events) + } + + private fun encodeResult( + result: QueryResult, + offset: Long, + ): JSONObject { + val rows = JSONArray() + for (row in result.rows) rows.put(JSONArray().also { a -> row.forEach { a.put(encodeValue(it)) } }) + val out = JSONObject().put("columns", JSONArray(result.columns)).put("rows", rows).put("row_count", result.rows.size) + .put("offset", encodeLong(offset)).put("has_more", result.hasMore).put("truncated_cells", result.truncatedCells) + if (result.hasMore) out.put("next_offset", encodeLong(offset + result.rows.size)) + return out + } + + private fun rowsAsObjects(result: QueryResult): JSONArray { + val arr = JSONArray() + for (row in result.rows) { + val o = JSONObject() + result.columns.forEachIndexed { i, c -> o.put(c, encodeValue(row[i])) } + arr.put(o) + } + return arr + } + + private fun checkedPath(args: JSONObject): String { + val path = stringArg(args, "database") + val known = engine.connections().map { it.path }.filter { it.isNotEmpty() }.distinct() + if (path !in known) { + throw McpToolError( + "Unknown database \"$path\": use a path from list_databases that is available.", + JSONObject().put("available", JSONArray(known)), + ) + } + return path + } + + private fun run( + args: JSONObject, + sql: String, + bind: List, + offset: Long, + limits: QueryLimits, + ): QueryResult = runChecked(checkedPath(args), sql, bind, offset, limits, clock() + DEADLINE_MS) + + private fun runChecked( + path: String, + sql: String, + bind: List, + offset: Long, + limits: QueryLimits, + deadline: Long, + ): QueryResult = + synchronized(lock) { + try { + val conn = serverConnections.getOrPut(path) { engine.openServerConnection(path, writable = false) } + conn.query(sql, bind, offset, limits, deadline) + } catch (e: DebugEngineException) { + throw engineError(e) + } + } + + private fun engineError(e: DebugEngineException) = + McpToolError(e.message ?: "SQLite error ${e.sqliteCode}", JSONObject().put("sqlite_code", e.sqliteCode)) + + private fun stringArg( + args: JSONObject, + name: String, + ): String { + val v = args.opt(name) + if (v !is String || v.isEmpty()) throw McpToolError("$name is required and must be a non-empty string.") + return v + } + + private fun intArg( + args: JSONObject, + name: String, + default: Long, + min: Long, + max: Long, + ): Long { + val v = args.opt(name) + if (v == null || v == JSONObject.NULL) return default + val n = v as? Number ?: throw McpToolError("$name must be an integer.") + // org.json parses a decimal as BigDecimal, not Double + if (n !is Int && n !is Long && n !is Short && n !is Byte) { + val d = n.toDouble() + if (d != Math.floor(d) || d.isInfinite() || Math.abs(d) >= 9.2e18) throw McpToolError("$name must be an integer.") + } + val l = n.toLong() + if (l < min || l > max) throw McpToolError("$name must be between $min and ${if (max == Long.MAX_VALUE) "2^63-1" else max.toString()}.") + return l + } + + private fun bindArgs(args: JSONObject): List { + val raw = args.opt("args") + if (raw == null || raw == JSONObject.NULL) return emptyList() + val arr = raw as? JSONArray ?: throw McpToolError("args must be an array.") + return (0 until arr.length()).map { i -> + when (val v = arr.opt(i)) { + null, JSONObject.NULL -> null + is Int, is Long, is Short -> (v as Number).toLong() + is Number -> v.toDouble() + is String -> v + is Boolean -> if (v) 1L else 0L + else -> throw McpToolError("args[$i] must be null, a number or a string.") + } + } + } +} diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt index c04c7fe6c3..7d58e73556 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt @@ -19,295 +19,318 @@ import kotlin.concurrent.thread import org.json.JSONObject /** - * The MCP endpoint's HTTP side: a small HTTP/1.1 listener on 127.0.0.1 only, apart from the store's - * own HttpServer, so it's never reachable from another device. Before [McpProtocol] sees a request, + * The MCP endpoint's HTTP side: a small HTTP/1.1 listener on 127.0.0.1 only, so it's never + * reachable from another device. Before [McpProtocol] sees a request, * it checks who's asking: * - the peer is a loopback address (it can't be otherwise, but it's checked anyway); * - Host names 127.0.0.1 or localhost, and Origin, if sent, does too, so a web page can't reach it * by DNS rebinding; - * - Authorization: Bearer , the token shown on the server screen, compared in constant time. + * - Authorization: Bearer , the token from [newToken], compared in constant time. * * POST /mcp carries JSON-RPC; DELETE /mcp ends a session; GET gets 405, since this server never * opens an event stream. Each connection has its own thread, as a tool may wait a while; at most * [MAX_CONNECTIONS] at once. Plain Kotlin, so unit tests can run it without Android. */ class McpHttp( - private val protocol: McpProtocol, - /** The current token: read on every request, so a new one takes effect at once. */ - private val token: () -> String, - private val port: Int = DEFAULT_PORT, - /** Problems worth a line in the log. Never given the token. */ - private val log: (String) -> Unit = {}, + private val protocol: McpProtocol, + /** The current token: read on every request, so a new one takes effect at once. */ + private val token: () -> String, + /** 0 lets the system pick a free port; read it from [boundPort] after [start]. */ + private val port: Int = 0, + /** Problems worth a line in the log. Never given the token. */ + private val log: (String) -> Unit = {}, ) { - companion object { - const val DEFAULT_PORT = 6176 - const val PATH = "/mcp" - const val MAX_CONNECTIONS = 8 - private const val MAX_HEADER_BYTES = 16 * 1024 - const val MAX_BODY_BYTES = 1024 * 1024 - /** A connection with no request for this long is closed. */ - const val IDLE_TIMEOUT_MS = 30_000 - private val STATUS = mapOf( - 200 to "OK", 202 to "Accepted", 204 to "No Content", 400 to "Bad Request", 401 to "Unauthorized", - 403 to "Forbidden", 404 to "Not Found", 405 to "Method Not Allowed", 413 to "Payload Too Large", - 415 to "Unsupported Media Type", 431 to "Request Header Fields Too Large", 500 to "Internal Server Error", - 501 to "Not Implemented", 503 to "Service Unavailable", - ) - private val LOOPBACK_NAMES = setOf("127.0.0.1", "localhost") - } + companion object { + private const val B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" - private class Request(val method: String, val target: String, val version: String, val headers: Map, val body: ByteArray) - private class HttpError(val status: Int, message: String) : Exception(message) + /** 32 random bytes, URL-safe base64 without padding. A new one on every process start. */ + fun newToken(): String = base64(ByteArray(32).also { java.security.SecureRandom().nextBytes(it) }, urlSafe = true, pad = false) - @Volatile private var socket: ServerSocket? = null - @Volatile private var running = false - private val open = AtomicInteger(0) - private val clients: MutableSet = Collections.newSetFromMap(ConcurrentHashMap()) + /** By hand: java.util.Base64 needs API 26 and this library's minSdk is 21. */ + fun base64(bytes: ByteArray, urlSafe: Boolean, pad: Boolean): String { + val out = StringBuilder() + var i = 0 + while (i < bytes.size) { + val n = minOf(3, bytes.size - i) + var v = 0 + for (k in 0 until 3) v = (v shl 8) or (if (k < n) bytes[i + k].toInt() and 0xff else 0) + for (k in 0..n) { + val idx = (v shr (18 - 6 * k)) and 63 + out.append(if (idx < 62) B64[idx] else if (idx == 62) (if (urlSafe) '-' else '+') else (if (urlSafe) '_' else '/')) + } + if (pad) repeat(3 - n) { out.append('=') } + i += 3 + } + return out.toString() + } - /** The port actually bound (useful when [port] is 0, as in tests). */ - val boundPort: Int get() = socket?.localPort ?: port + const val PATH = "/mcp" + const val MAX_CONNECTIONS = 8 + private const val MAX_HEADER_BYTES = 16 * 1024 + const val MAX_BODY_BYTES = 1024 * 1024 + /** A connection with no request for this long is closed. */ + const val IDLE_TIMEOUT_MS = 30_000 + private val STATUS = mapOf( + 200 to "OK", 202 to "Accepted", 204 to "No Content", 400 to "Bad Request", 401 to "Unauthorized", + 403 to "Forbidden", 404 to "Not Found", 405 to "Method Not Allowed", 413 to "Payload Too Large", + 415 to "Unsupported Media Type", 431 to "Request Header Fields Too Large", 500 to "Internal Server Error", + 501 to "Not Implemented", 503 to "Service Unavailable", + ) + private val LOOPBACK_NAMES = setOf("127.0.0.1", "localhost") + } - /** Binds 127.0.0.1:[port]. Throws IOException with a readable message if the port is taken. */ - fun start() { - val s = ServerSocket() - try { - s.reuseAddress = true - s.bind(InetSocketAddress(InetAddress.getByAddress(byteArrayOf(127, 0, 0, 1)), port), 16) - } catch (e: IOException) { - try { s.close() } catch (ignored: IOException) { } - throw IOException("Can't listen on 127.0.0.1:$port: ${e.message}", e) - } - socket = s - running = true - thread(name = "mcp-accept", isDaemon = true) { acceptLoop(s) } - } + private class Request(val method: String, val target: String, val version: String, val headers: Map, val body: ByteArray) + private class HttpError(val status: Int, message: String) : Exception(message) - /** Stops listening and closes every connection. */ - fun stop() { - running = false - try { socket?.close() } catch (ignored: IOException) { } - socket = null - clients.forEach { close(it) } - clients.clear() - } + @Volatile private var socket: ServerSocket? = null + @Volatile private var running = false + private val open = AtomicInteger(0) + private val clients: MutableSet = Collections.newSetFromMap(ConcurrentHashMap()) - private fun acceptLoop(s: ServerSocket) { - while (running && !s.isClosed) { - val c = try { s.accept() } catch (e: IOException) { - if (running && !s.isClosed) log("couldn't accept a connection: ${e.message}") - continue - } - if (!c.inetAddress.isLoopbackAddress) { close(c); continue } - if (open.incrementAndGet() > MAX_CONNECTIONS) { - open.decrementAndGet() - try { - c.soTimeout = 1000 - respond(c.getOutputStream(), 503, rpcError("Too many connections at once; try again."), emptyMap(), close = true) - } catch (ignored: IOException) { } - close(c) - continue - } - clients += c - thread(name = "mcp-connection", isDaemon = true) { - try { - serve(c) - } catch (ignored: IOException) { - // The client went away. - } catch (e: Exception) { - log("a connection failed: ${e.javaClass.simpleName}: ${e.message}") - } finally { - clients -= c - close(c) - open.decrementAndGet() - } - } - } - } + /** The port actually bound (useful when [port] is 0, as in tests). */ + val boundPort: Int get() = socket?.localPort ?: port - private fun serve(c: Socket) { - c.soTimeout = IDLE_TIMEOUT_MS - c.tcpNoDelay = true - val input = BufferedInputStream(c.getInputStream()) - val out = c.getOutputStream() - while (running) { - val req = try { - readRequest(input) ?: return - } catch (e: SocketTimeoutException) { - return - } catch (e: HttpError) { - respond(out, e.status, rpcError(e.message ?: "Bad request."), emptyMap(), close = true) - return - } - if (!handle(req, out)) return - } - } + /** Binds 127.0.0.1:[port]. Throws IOException with a readable message if the port is taken. */ + fun start() { + val s = ServerSocket() + try { + s.reuseAddress = true + s.bind(InetSocketAddress(InetAddress.getByAddress(byteArrayOf(127, 0, 0, 1)), port), 16) + } catch (e: IOException) { + try { s.close() } catch (ignored: IOException) { } + throw IOException("Can't listen on 127.0.0.1:$port: ${e.message}", e) + } + socket = s + running = true + thread(name = "mcp-accept", isDaemon = true) { acceptLoop(s) } + } - /** Answers one request. Returns whether the connection stays open for another. */ - private fun handle(r: Request, out: OutputStream): Boolean { - val close = r.version == "HTTP/1.0" || r.headers["connection"]?.lowercase()?.contains("close") == true - fun send(status: Int, body: String?, headers: Map = emptyMap()): Boolean { - respond(out, status, body, headers, close) - return !close - } - if (!hostAllowed(r.headers["host"])) return send(403, rpcError("The Host header must be 127.0.0.1 or localhost.")) - val origin = r.headers["origin"] - if (origin != null && !originAllowed(origin)) return send(403, rpcError("Requests from web pages aren't accepted.")) - if (r.target.substringBefore('?') != PATH) return send(404, rpcError("Not found: the MCP endpoint is $PATH.")) - if (!authorized(r.headers["authorization"])) { - return send(401, rpcError("Missing or wrong token. Code on the Go passes it to its Agent; send it as Authorization: Bearer ."), - mapOf("WWW-Authenticate" to "Bearer realm=\"SQLite debugger\"")) - } - val session = r.headers["mcp-session-id"] - val reply = when (r.method) { - "POST" -> { - val type = r.headers["content-type"]?.substringBefore(';')?.trim()?.lowercase() - if (type != null && type != "application/json") return send(415, rpcError("Send the request as application/json.")) - try { - protocol.post(String(r.body, Charsets.UTF_8), session, r.headers["mcp-protocol-version"]) - } catch (e: Exception) { - log("a request failed: ${e.javaClass.simpleName}: ${e.message}") - McpProtocol.Reply(500, rpcError("The server failed: ${e.message ?: e.javaClass.simpleName}")) - } - } - "DELETE" -> protocol.delete(session) - "GET" -> return send(405, rpcError("This server doesn't offer an event stream: use POST."), mapOf("Allow" to "POST, DELETE")) - else -> return send(405, rpcError("Use POST."), mapOf("Allow" to "POST, DELETE")) - } - return send(reply.status, reply.body, reply.headers) - } + /** Stops listening and closes every connection. */ + fun stop() { + running = false + try { socket?.close() } catch (ignored: IOException) { } + socket = null + clients.forEach { close(it) } + clients.clear() + } - // ---------------------------------------------------------------- checks + private fun acceptLoop(s: ServerSocket) { + while (running && !s.isClosed) { + val c = try { s.accept() } catch (e: IOException) { + if (running && !s.isClosed) log("couldn't accept a connection: ${e.message}") + continue + } + if (!c.inetAddress.isLoopbackAddress) { close(c); continue } + if (open.incrementAndGet() > MAX_CONNECTIONS) { + open.decrementAndGet() + try { + c.soTimeout = 1000 + respond(c.getOutputStream(), 503, rpcError("Too many connections at once; try again."), emptyMap(), close = true) + } catch (ignored: IOException) { } + close(c) + continue + } + clients += c + thread(name = "mcp-connection", isDaemon = true) { + try { + serve(c) + } catch (ignored: IOException) { + // The client went away. + } catch (e: Exception) { + log("a connection failed: ${e.javaClass.simpleName}: ${e.message}") + } finally { + clients -= c + close(c) + open.decrementAndGet() + } + } + } + } - /** 127.0.0.1 or localhost, on any port: adb forward may bring the request in from another port. */ - private fun hostAllowed(host: String?): Boolean { - val h = host?.trim()?.lowercase() ?: return false - val name = if (h.count { it == ':' } == 1) h.substringBefore(':') else h - if (h.count { it == ':' } == 1 && h.substringAfter(':').let { it.isEmpty() || it.any { ch -> !ch.isDigit() } }) return false - return name in LOOPBACK_NAMES - } + private fun serve(c: Socket) { + c.soTimeout = IDLE_TIMEOUT_MS + c.tcpNoDelay = true + val input = BufferedInputStream(c.getInputStream()) + val out = c.getOutputStream() + while (running) { + val req = try { + readRequest(input) ?: return + } catch (e: SocketTimeoutException) { + return + } catch (e: HttpError) { + respond(out, e.status, rpcError(e.message ?: "Bad request."), emptyMap(), close = true) + return + } + if (!handle(req, out)) return + } + } - private fun originAllowed(origin: String): Boolean = try { - val u = URI(origin.trim()) - (u.scheme == "http" || u.scheme == "https") && u.host?.lowercase() in LOOPBACK_NAMES - } catch (e: Exception) { false } + /** Answers one request. Returns whether the connection stays open for another. */ + private fun handle(r: Request, out: OutputStream): Boolean { + val close = r.version == "HTTP/1.0" || r.headers["connection"]?.lowercase()?.contains("close") == true + fun send(status: Int, body: String?, headers: Map = emptyMap()): Boolean { + respond(out, status, body, headers, close) + return !close + } + if (!hostAllowed(r.headers["host"])) return send(403, rpcError("The Host header must be 127.0.0.1 or localhost.")) + val origin = r.headers["origin"] + if (origin != null && !originAllowed(origin)) return send(403, rpcError("Requests from web pages aren't accepted.")) + if (r.target.substringBefore('?') != PATH) return send(404, rpcError("Not found: the MCP endpoint is $PATH.")) + if (!authorized(r.headers["authorization"])) { + return send(401, rpcError("Missing or wrong token. Code on the Go gives it to its Agent; send it as Authorization: Bearer ."), + mapOf("WWW-Authenticate" to "Bearer realm=\"SQLite debugger\"")) + } + val session = r.headers["mcp-session-id"] + val reply = when (r.method) { + "POST" -> { + val type = r.headers["content-type"]?.substringBefore(';')?.trim()?.lowercase() + if (type != null && type != "application/json") return send(415, rpcError("Send the request as application/json.")) + try { + protocol.post(String(r.body, Charsets.UTF_8), session, r.headers["mcp-protocol-version"]) + } catch (e: Exception) { + log("a request failed: ${e.javaClass.simpleName}: ${e.message}") + McpProtocol.Reply(500, rpcError("The server failed: ${e.message ?: e.javaClass.simpleName}")) + } + } + "DELETE" -> protocol.delete(session) + "GET" -> return send(405, rpcError("This server doesn't offer an event stream: use POST."), mapOf("Allow" to "POST, DELETE")) + else -> return send(405, rpcError("Use POST."), mapOf("Allow" to "POST, DELETE")) + } + return send(reply.status, reply.body, reply.headers) + } - private fun authorized(header: String?): Boolean { - val expected = token() - if (header == null || expected.isEmpty()) return false - val h = header.trim() - if (!h.regionMatches(0, "Bearer ", 0, 7, ignoreCase = true)) return false - return MessageDigest.isEqual(h.substring(7).trim().toByteArray(Charsets.UTF_8), expected.toByteArray(Charsets.UTF_8)) - } + // ---------------------------------------------------------------- checks - // ---------------------------------------------------------------- reading and writing HTTP + /** 127.0.0.1 or localhost, on any port: adb forward may bring the request in from another port. */ + private fun hostAllowed(host: String?): Boolean { + val h = host?.trim()?.lowercase() ?: return false + val name = if (h.count { it == ':' } == 1) h.substringBefore(':') else h + if (h.count { it == ':' } == 1 && h.substringAfter(':').let { it.isEmpty() || it.any { ch -> !ch.isDigit() } }) return false + return name in LOOPBACK_NAMES + } - /** One request, or null if the connection closed before one began. */ - private fun readRequest(input: InputStream): Request? { - var budget = MAX_HEADER_BYTES - fun line(): String? { - val b = ByteArrayOutputStream() - while (true) { - val ch = input.read() - if (ch < 0) { if (b.size() == 0) return null else throw HttpError(400, "The request ended early.") } - if (--budget < 0) throw HttpError(431, "The request's headers are too long.") - if (ch == '\n'.code) break - b.write(ch) - } - return b.toString("ISO-8859-1").removeSuffix("\r") - } - var first = line() ?: return null - while (first.isEmpty()) first = line() ?: return null // stray line ends between requests - val parts = first.split(' ') - if (parts.size != 3 || !parts[2].startsWith("HTTP/1.")) throw HttpError(400, "That isn't an HTTP/1.x request.") - val headers = HashMap() - while (true) { - val l = line() ?: throw HttpError(400, "The request ended early.") - if (l.isEmpty()) break - val colon = l.indexOf(':') - if (colon <= 0) throw HttpError(400, "A header line without a colon.") - val name = l.substring(0, colon).trim().lowercase() - val value = l.substring(colon + 1).trim() - // A repeated header is joined, so two Host headers fail the Host check. - headers[name] = headers[name]?.let { "$it, $value" } ?: value - } - return Request(parts[0], parts[1], parts[2], headers, readBody(input, headers)) - } + private fun originAllowed(origin: String): Boolean = try { + val u = URI(origin.trim()) + (u.scheme == "http" || u.scheme == "https") && u.host?.lowercase() in LOOPBACK_NAMES + } catch (e: Exception) { false } - private fun readBody(input: InputStream, h: Map): ByteArray { - val te = h["transfer-encoding"] - if (te != null) { - if (!te.equals("chunked", ignoreCase = true)) throw HttpError(501, "Transfer-Encoding $te isn't supported.") - if (h.containsKey("content-length")) throw HttpError(400, "Both Content-Length and Transfer-Encoding were sent.") - return readChunked(input) - } - val len = h["content-length"] ?: return ByteArray(0) - val n = len.toLongOrNull()?.takeIf { it >= 0 } ?: throw HttpError(400, "Content-Length isn't a number.") - if (n > MAX_BODY_BYTES) throw HttpError(413, "The request is larger than ${MAX_BODY_BYTES / 1024} KB.") - return readExactly(input, n.toInt()) - } + private fun authorized(header: String?): Boolean { + val expected = token() + if (header == null || expected.isEmpty()) return false + val h = header.trim() + if (!h.regionMatches(0, "Bearer ", 0, 7, ignoreCase = true)) return false + return MessageDigest.isEqual(h.substring(7).trim().toByteArray(Charsets.UTF_8), expected.toByteArray(Charsets.UTF_8)) + } - private fun readChunked(input: InputStream): ByteArray { - val out = ByteArrayOutputStream() - while (true) { - val sizeLine = shortLine(input, 1024) ?: throw HttpError(400, "The request ended early.") - val size = sizeLine.substringBefore(';').trim().toIntOrNull(16)?.takeIf { it >= 0 } ?: throw HttpError(400, "A bad chunk size.") - if (out.size().toLong() + size > MAX_BODY_BYTES) throw HttpError(413, "The request is larger than ${MAX_BODY_BYTES / 1024} KB.") - if (size == 0) { - // Trailers, if any, up to a blank line. - var lines = 0 - while (true) { - val t = shortLine(input, 1024) ?: break - if (t.isEmpty()) break - if (++lines > 32) throw HttpError(431, "Too many trailers.") - } - return out.toByteArray() - } - out.write(readExactly(input, size)) - if (shortLine(input, 2) != "") throw HttpError(400, "A chunk without its line end.") - } - } + // ---------------------------------------------------------------- reading and writing HTTP - private fun shortLine(input: InputStream, max: Int): String? { - val b = ByteArrayOutputStream() - var left = max - while (true) { - val ch = input.read() - if (ch < 0) return if (b.size() == 0) null else throw HttpError(400, "The request ended early.") - if (--left < 0) throw HttpError(400, "A line in the request is too long.") - if (ch == '\n'.code) break - b.write(ch) - } - return b.toString("ISO-8859-1").removeSuffix("\r") - } + /** One request, or null if the connection closed before one began. */ + private fun readRequest(input: InputStream): Request? { + var budget = MAX_HEADER_BYTES + fun line(): String? { + val b = ByteArrayOutputStream() + while (true) { + val ch = input.read() + if (ch < 0) { if (b.size() == 0) return null else throw HttpError(400, "The request ended early.") } + if (--budget < 0) throw HttpError(431, "The request's headers are too long.") + if (ch == '\n'.code) break + b.write(ch) + } + return b.toString("ISO-8859-1").removeSuffix("\r") + } + var first = line() ?: return null + while (first.isEmpty()) first = line() ?: return null // stray line ends between requests + val parts = first.split(' ') + if (parts.size != 3 || !parts[2].startsWith("HTTP/1.")) throw HttpError(400, "That isn't an HTTP/1.x request.") + val headers = HashMap() + while (true) { + val l = line() ?: throw HttpError(400, "The request ended early.") + if (l.isEmpty()) break + val colon = l.indexOf(':') + if (colon <= 0) throw HttpError(400, "A header line without a colon.") + val name = l.substring(0, colon).trim().lowercase() + val value = l.substring(colon + 1).trim() + // A repeated header is joined, so two Host headers fail the Host check. + headers[name] = headers[name]?.let { "$it, $value" } ?: value + } + return Request(parts[0], parts[1], parts[2], headers, readBody(input, headers)) + } - private fun readExactly(input: InputStream, n: Int): ByteArray { - val b = ByteArray(n) - var off = 0 - while (off < n) { - val r = input.read(b, off, n - off) - if (r < 0) throw HttpError(400, "The request ended early.") - off += r - } - return b - } + private fun readBody(input: InputStream, h: Map): ByteArray { + val te = h["transfer-encoding"] + if (te != null) { + if (!te.equals("chunked", ignoreCase = true)) throw HttpError(501, "Transfer-Encoding $te isn't supported.") + if (h.containsKey("content-length")) throw HttpError(400, "Both Content-Length and Transfer-Encoding were sent.") + return readChunked(input) + } + val len = h["content-length"] ?: return ByteArray(0) + val n = len.toLongOrNull()?.takeIf { it >= 0 } ?: throw HttpError(400, "Content-Length isn't a number.") + if (n > MAX_BODY_BYTES) throw HttpError(413, "The request is larger than ${MAX_BODY_BYTES / 1024} KB.") + return readExactly(input, n.toInt()) + } - private fun respond(out: OutputStream, status: Int, body: String?, headers: Map, close: Boolean) { - val bytes = body?.toByteArray(Charsets.UTF_8) ?: ByteArray(0) - val head = StringBuilder("HTTP/1.1 $status ${STATUS[status] ?: "Status"}\r\n") - if (body != null) head.append("Content-Type: application/json; charset=utf-8\r\n") - if (status != 204) head.append("Content-Length: ${bytes.size}\r\n") - head.append("Cache-Control: no-store\r\n") - if (close) head.append("Connection: close\r\n") - for ((k, v) in headers) head.append(k).append(": ").append(v.replace("\r", "").replace("\n", "")).append("\r\n") - head.append("\r\n") - out.write(head.toString().toByteArray(Charsets.ISO_8859_1)) - if (status != 204) out.write(bytes) - out.flush() - } + private fun readChunked(input: InputStream): ByteArray { + val out = ByteArrayOutputStream() + while (true) { + val sizeLine = shortLine(input, 1024) ?: throw HttpError(400, "The request ended early.") + val size = sizeLine.substringBefore(';').trim().toIntOrNull(16)?.takeIf { it >= 0 } ?: throw HttpError(400, "A bad chunk size.") + if (out.size().toLong() + size > MAX_BODY_BYTES) throw HttpError(413, "The request is larger than ${MAX_BODY_BYTES / 1024} KB.") + if (size == 0) { + // Trailers, if any, up to a blank line. + var lines = 0 + while (true) { + val t = shortLine(input, 1024) ?: break + if (t.isEmpty()) break + if (++lines > 32) throw HttpError(431, "Too many trailers.") + } + return out.toByteArray() + } + out.write(readExactly(input, size)) + if (shortLine(input, 2) != "") throw HttpError(400, "A chunk without its line end.") + } + } - /** An HTTP-level refusal, as a JSON-RPC error with no id, which MCP clients know how to show. */ - private fun rpcError(message: String): String = JSONObject().put("jsonrpc", "2.0").put("id", JSONObject.NULL) - .put("error", JSONObject().put("code", McpProtocol.INVALID_REQUEST).put("message", message)).toString() + private fun shortLine(input: InputStream, max: Int): String? { + val b = ByteArrayOutputStream() + var left = max + while (true) { + val ch = input.read() + if (ch < 0) return if (b.size() == 0) null else throw HttpError(400, "The request ended early.") + if (--left < 0) throw HttpError(400, "A line in the request is too long.") + if (ch == '\n'.code) break + b.write(ch) + } + return b.toString("ISO-8859-1").removeSuffix("\r") + } - private fun close(s: Socket) { try { s.close() } catch (ignored: IOException) { } } + private fun readExactly(input: InputStream, n: Int): ByteArray { + val b = ByteArray(n) + var off = 0 + while (off < n) { + val r = input.read(b, off, n - off) + if (r < 0) throw HttpError(400, "The request ended early.") + off += r + } + return b + } + + private fun respond(out: OutputStream, status: Int, body: String?, headers: Map, close: Boolean) { + val bytes = body?.toByteArray(Charsets.UTF_8) ?: ByteArray(0) + val head = StringBuilder("HTTP/1.1 $status ${STATUS[status] ?: "Status"}\r\n") + if (body != null) head.append("Content-Type: application/json; charset=utf-8\r\n") + if (status != 204) head.append("Content-Length: ${bytes.size}\r\n") + head.append("Cache-Control: no-store\r\n") + if (close) head.append("Connection: close\r\n") + for ((k, v) in headers) head.append(k).append(": ").append(v.replace("\r", "").replace("\n", "")).append("\r\n") + head.append("\r\n") + out.write(head.toString().toByteArray(Charsets.ISO_8859_1)) + if (status != 204) out.write(bytes) + out.flush() + } + + /** An HTTP-level refusal, as a JSON-RPC error with no id, which MCP clients know how to show. */ + private fun rpcError(message: String): String = JSONObject().put("jsonrpc", "2.0").put("id", JSONObject.NULL) + .put("error", JSONObject().put("code", McpProtocol.INVALID_REQUEST).put("message", message)).toString() + + private fun close(s: Socket) { try { s.close() } catch (ignored: IOException) { } } } diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt index 273d27e5d3..01e0877b1d 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt @@ -10,19 +10,19 @@ import org.json.JSONTokener * should read, such as "a person is needed". */ class McpTool( - val name: String, - val title: String, - val description: String, - val inputSchema: JSONObject = JSONObject().put("type", "object").put("properties", JSONObject()).put("additionalProperties", false), - /** Only reports; changes nothing. Shown to clients as a hint, not relied on. */ - val readOnly: Boolean = true, - val run: (JSONObject) -> JSONObject, + val name: String, + val title: String, + val description: String, + val inputSchema: JSONObject = JSONObject().put("type", "object").put("properties", JSONObject()).put("additionalProperties", false), + /** Only reports; changes nothing. Shown to clients as a hint, not relied on. */ + val readOnly: Boolean = true, + val run: (JSONObject) -> JSONObject, ) { - fun describe(): JSONObject = JSONObject() - .put("name", name).put("title", title).put("description", description) - .put("inputSchema", inputSchema) - .put("annotations", JSONObject().put("title", title).put("readOnlyHint", readOnly) - .put("destructiveHint", false).put("openWorldHint", false)) + fun describe(): JSONObject = JSONObject() + .put("name", name).put("title", title).put("description", description) + .put("inputSchema", inputSchema) + .put("annotations", JSONObject().put("title", title).put("readOnlyHint", readOnly) + .put("destructiveHint", false).put("openWorldHint", false)) } /** A tool's failure, told to the client as a tool result with isError, with [details] when there are any. */ @@ -38,135 +38,135 @@ class McpToolError(message: String, val details: JSONObject? = null) : Exception * are accepted and need no answer (HTTP 202). */ class McpProtocol( - private val tools: List, - private val serverVersion: String, - private val newSessionId: () -> String, - private val maxSessions: Int = 16, + private val tools: List, + private val serverVersion: String, + private val newSessionId: () -> String, + private val maxSessions: Int = 16, ) { - companion object { - /** Newest first: offered when a client asks for a version this server doesn't know. */ - val VERSIONS = listOf("2025-06-18", "2025-03-26", "2024-11-05") - const val SERVER_NAME = "sqlite-debugger" - const val SESSION_HEADER = "Mcp-Session-Id" - const val PARSE_ERROR = -32700 - const val INVALID_REQUEST = -32600 - const val METHOD_NOT_FOUND = -32601 - const val INVALID_PARAMS = -32602 - const val INSTRUCTIONS = "Inspects and queries the SQLite databases of the app this server runs in, a debug " + - "build made with Code on the Go. Read-only unless the app's developer turned on read-write mode." - } - - /** What to send back: an HTTP status, a JSON body (or none), and extra headers. */ - class Reply(val status: Int, val body: String?, val headers: Map = emptyMap()) - - /** Live sessions, oldest first; the oldest is dropped when there are too many. */ - private val sessions = LinkedHashSet() - - /** A POST to the endpoint: one JSON-RPC message, or a batch of them. */ - fun post(body: String, sessionId: String?, versionHeader: String?): Reply { - val parsed = try { JSONTokener(body).nextValue() } catch (e: Exception) { null } - if (parsed !is JSONObject && parsed !is JSONArray) return Reply(400, error(null, PARSE_ERROR, "That isn't a JSON object or array.").toString()) - val messages: List = if (parsed is JSONArray) (0 until parsed.length()).map { parsed.opt(it) } else listOf(parsed) - if (messages.isEmpty()) return Reply(400, error(null, INVALID_REQUEST, "An empty batch.").toString()) - - if (messages.any { (it as? JSONObject)?.optString("method") == "initialize" }) { - if (messages.size != 1) return Reply(400, error(null, INVALID_REQUEST, "Send initialize on its own, not in a batch.").toString()) - return initialize(messages[0] as JSONObject) - } - if (sessionId.isNullOrEmpty()) return Reply(400, error(null, INVALID_REQUEST, "No $SESSION_HEADER header: send initialize first.").toString()) - if (!touch(sessionId)) return Reply(404, error(null, INVALID_REQUEST, "That session has ended or doesn't exist: send initialize again.").toString()) - if (versionHeader != null && versionHeader !in VERSIONS) { - return Reply(400, error(null, INVALID_REQUEST, "Unsupported MCP-Protocol-Version $versionHeader; this server speaks ${VERSIONS.joinToString()}.").toString()) - } - val replies = messages.mapNotNull { handle(it) } - if (replies.isEmpty()) return Reply(202, null) - return Reply(200, if (parsed is JSONArray) JSONArray(replies).toString() else replies[0].toString()) - } - - /** A DELETE: the client is done with its session. */ - fun delete(sessionId: String?): Reply { - if (sessionId.isNullOrEmpty()) return Reply(400, error(null, INVALID_REQUEST, "No $SESSION_HEADER header.").toString()) - val removed = synchronized(sessions) { sessions.remove(sessionId) } - return if (removed) Reply(204, null) else Reply(404, error(null, INVALID_REQUEST, "That session has ended or doesn't exist.").toString()) - } - - /** Ends every session, so clients must initialize again. */ - fun endAll() = synchronized(sessions) { sessions.clear() } - - fun sessionCount(): Int = synchronized(sessions) { sessions.size } - - private fun touch(id: String): Boolean = synchronized(sessions) { - if (!sessions.remove(id)) return false - sessions += id // most recently used goes last, so it's dropped last - true - } - - private fun initialize(m: JSONObject): Reply { - val id = m.opt("id") - if (!validId(id)) return Reply(400, error(null, INVALID_REQUEST, "initialize needs an id, a string or a number.").toString()) - if (m.optString("jsonrpc") != "2.0") return Reply(400, error(id, INVALID_REQUEST, "jsonrpc must be \"2.0\".").toString()) - val asked = m.optJSONObject("params")?.optString("protocolVersion", "") ?: "" - val version = if (asked in VERSIONS) asked else VERSIONS[0] - val session = newSessionId() - synchronized(sessions) { - sessions += session - while (sessions.size > maxSessions) sessions.remove(sessions.first()) - } - val result = JSONObject() - .put("protocolVersion", version) - .put("capabilities", JSONObject().put("tools", JSONObject().put("listChanged", false))) - .put("serverInfo", JSONObject().put("name", SERVER_NAME).put("title", "SQLite debugger").put("version", serverVersion)) - .put("instructions", INSTRUCTIONS) - return Reply(200, ok(id, result).toString(), mapOf(SESSION_HEADER to session)) - } - - /** One message of a POST; null when it needs no answer. */ - private fun handle(raw: Any?): JSONObject? { - val m = raw as? JSONObject ?: return error(null, INVALID_REQUEST, "Each message must be a JSON object.") - val hasId = m.has("id") - val id = m.opt("id") - val method = m.opt("method") as? String - if (method == null) { - // A response to something this server asked: it never asks, so there's nothing to do. - if (m.has("result") || m.has("error")) return null - return error(if (validId(id)) id else null, INVALID_REQUEST, "A request needs a method.") - } - if (!hasId) return null // a notification, such as notifications/initialized: no answer - if (!validId(id)) return error(null, INVALID_REQUEST, "id must be a string or a number.") - if (m.optString("jsonrpc") != "2.0") return error(id, INVALID_REQUEST, "jsonrpc must be \"2.0\".") - return when (method) { - "ping" -> ok(id, JSONObject()) - "tools/list" -> ok(id, JSONObject().put("tools", JSONArray().also { a -> tools.forEach { a.put(it.describe()) } })) - "tools/call" -> call(id, m.opt("params")) - else -> error(id, METHOD_NOT_FOUND, "Unknown method \"$method\".") - } - } - - private fun call(id: Any, rawParams: Any?): JSONObject { - val params = rawParams as? JSONObject ?: return error(id, INVALID_PARAMS, "tools/call needs params, with the tool's name.") - val name = params.opt("name") as? String ?: return error(id, INVALID_PARAMS, "tools/call needs the tool's name.") - val tool = tools.firstOrNull { it.name == name } ?: return error(id, INVALID_PARAMS, "Unknown tool \"$name\".") - val args = when (val a = params.opt("arguments")) { - null, JSONObject.NULL -> JSONObject() - is JSONObject -> a - else -> return error(id, INVALID_PARAMS, "arguments must be an object.") - } - val result = try { - val out = tool.run(args) - JSONObject().put("content", JSONArray().put(text(out.toString(2)))).put("structuredContent", out).put("isError", false) - } catch (e: McpToolError) { - JSONObject().put("content", JSONArray().put(text(e.message ?: "The tool failed."))).put("isError", true).also { r -> - e.details?.let { r.put("structuredContent", it) } - } - } catch (e: Exception) { - JSONObject().put("content", JSONArray().put(text("The tool failed: ${e.message ?: e.javaClass.simpleName}"))).put("isError", true) - } - return ok(id, result) - } - - private fun validId(id: Any?) = id is String || id is Number - private fun text(t: String) = JSONObject().put("type", "text").put("text", t) - private fun ok(id: Any, result: JSONObject) = JSONObject().put("jsonrpc", "2.0").put("id", id).put("result", result) - private fun error(id: Any?, code: Int, message: String) = JSONObject().put("jsonrpc", "2.0").put("id", id ?: JSONObject.NULL) - .put("error", JSONObject().put("code", code).put("message", message)) + companion object { + /** Newest first: offered when a client asks for a version this server doesn't know. */ + val VERSIONS = listOf("2025-06-18", "2025-03-26", "2024-11-05") + const val SERVER_NAME = "sqlite-debugger" + const val SESSION_HEADER = "Mcp-Session-Id" + const val PARSE_ERROR = -32700 + const val INVALID_REQUEST = -32600 + const val METHOD_NOT_FOUND = -32601 + const val INVALID_PARAMS = -32602 + const val INSTRUCTIONS = "Inspects and queries the SQLite databases of the app this server runs in, a debug " + + "build made with Code on the Go. Read-only unless the app's developer turned on read-write mode." + } + + /** What to send back: an HTTP status, a JSON body (or none), and extra headers. */ + class Reply(val status: Int, val body: String?, val headers: Map = emptyMap()) + + /** Live sessions, oldest first; the oldest is dropped when there are too many. */ + private val sessions = LinkedHashSet() + + /** A POST to the endpoint: one JSON-RPC message, or a batch of them. */ + fun post(body: String, sessionId: String?, versionHeader: String?): Reply { + val parsed = try { JSONTokener(body).nextValue() } catch (e: Exception) { null } + if (parsed !is JSONObject && parsed !is JSONArray) return Reply(400, error(null, PARSE_ERROR, "That isn't a JSON object or array.").toString()) + val messages: List = if (parsed is JSONArray) (0 until parsed.length()).map { parsed.opt(it) } else listOf(parsed) + if (messages.isEmpty()) return Reply(400, error(null, INVALID_REQUEST, "An empty batch.").toString()) + + if (messages.any { (it as? JSONObject)?.optString("method") == "initialize" }) { + if (messages.size != 1) return Reply(400, error(null, INVALID_REQUEST, "Send initialize on its own, not in a batch.").toString()) + return initialize(messages[0] as JSONObject) + } + if (sessionId.isNullOrEmpty()) return Reply(400, error(null, INVALID_REQUEST, "No $SESSION_HEADER header: send initialize first.").toString()) + if (!touch(sessionId)) return Reply(404, error(null, INVALID_REQUEST, "That session has ended or doesn't exist: send initialize again.").toString()) + if (versionHeader != null && versionHeader !in VERSIONS) { + return Reply(400, error(null, INVALID_REQUEST, "Unsupported MCP-Protocol-Version $versionHeader; this server speaks ${VERSIONS.joinToString()}.").toString()) + } + val replies = messages.mapNotNull { handle(it) } + if (replies.isEmpty()) return Reply(202, null) + return Reply(200, if (parsed is JSONArray) JSONArray(replies).toString() else replies[0].toString()) + } + + /** A DELETE: the client is done with its session. */ + fun delete(sessionId: String?): Reply { + if (sessionId.isNullOrEmpty()) return Reply(400, error(null, INVALID_REQUEST, "No $SESSION_HEADER header.").toString()) + val removed = synchronized(sessions) { sessions.remove(sessionId) } + return if (removed) Reply(204, null) else Reply(404, error(null, INVALID_REQUEST, "That session has ended or doesn't exist.").toString()) + } + + /** Ends every session, so clients must initialize again. */ + fun endAll() = synchronized(sessions) { sessions.clear() } + + fun sessionCount(): Int = synchronized(sessions) { sessions.size } + + private fun touch(id: String): Boolean = synchronized(sessions) { + if (!sessions.remove(id)) return false + sessions += id // most recently used goes last, so it's dropped last + true + } + + private fun initialize(m: JSONObject): Reply { + val id = m.opt("id") + if (!validId(id)) return Reply(400, error(null, INVALID_REQUEST, "initialize needs an id, a string or a number.").toString()) + if (m.optString("jsonrpc") != "2.0") return Reply(400, error(id, INVALID_REQUEST, "jsonrpc must be \"2.0\".").toString()) + val asked = m.optJSONObject("params")?.optString("protocolVersion", "") ?: "" + val version = if (asked in VERSIONS) asked else VERSIONS[0] + val session = newSessionId() + synchronized(sessions) { + sessions += session + while (sessions.size > maxSessions) sessions.remove(sessions.first()) + } + val result = JSONObject() + .put("protocolVersion", version) + .put("capabilities", JSONObject().put("tools", JSONObject().put("listChanged", false))) + .put("serverInfo", JSONObject().put("name", SERVER_NAME).put("title", "SQLite debugger").put("version", serverVersion)) + .put("instructions", INSTRUCTIONS) + return Reply(200, ok(id, result).toString(), mapOf(SESSION_HEADER to session)) + } + + /** One message of a POST; null when it needs no answer. */ + private fun handle(raw: Any?): JSONObject? { + val m = raw as? JSONObject ?: return error(null, INVALID_REQUEST, "Each message must be a JSON object.") + val hasId = m.has("id") + val id = m.opt("id") + val method = m.opt("method") as? String + if (method == null) { + // A response to something this server asked: it never asks, so there's nothing to do. + if (m.has("result") || m.has("error")) return null + return error(if (validId(id)) id else null, INVALID_REQUEST, "A request needs a method.") + } + if (!hasId) return null // a notification, such as notifications/initialized: no answer + if (!validId(id)) return error(null, INVALID_REQUEST, "id must be a string or a number.") + if (m.optString("jsonrpc") != "2.0") return error(id, INVALID_REQUEST, "jsonrpc must be \"2.0\".") + return when (method) { + "ping" -> ok(id, JSONObject()) + "tools/list" -> ok(id, JSONObject().put("tools", JSONArray().also { a -> tools.forEach { a.put(it.describe()) } })) + "tools/call" -> call(id, m.opt("params")) + else -> error(id, METHOD_NOT_FOUND, "Unknown method \"$method\".") + } + } + + private fun call(id: Any, rawParams: Any?): JSONObject { + val params = rawParams as? JSONObject ?: return error(id, INVALID_PARAMS, "tools/call needs params, with the tool's name.") + val name = params.opt("name") as? String ?: return error(id, INVALID_PARAMS, "tools/call needs the tool's name.") + val tool = tools.firstOrNull { it.name == name } ?: return error(id, INVALID_PARAMS, "Unknown tool \"$name\".") + val args = when (val a = params.opt("arguments")) { + null, JSONObject.NULL -> JSONObject() + is JSONObject -> a + else -> return error(id, INVALID_PARAMS, "arguments must be an object.") + } + val result = try { + val out = tool.run(args) + JSONObject().put("content", JSONArray().put(text(out.toString(2)))).put("structuredContent", out).put("isError", false) + } catch (e: McpToolError) { + JSONObject().put("content", JSONArray().put(text(e.message ?: "The tool failed."))).put("isError", true).also { r -> + e.details?.let { r.put("structuredContent", it) } + } + } catch (e: Exception) { + JSONObject().put("content", JSONArray().put(text("The tool failed: ${e.message ?: e.javaClass.simpleName}"))).put("isError", true) + } + return ok(id, result) + } + + private fun validId(id: Any?) = id is String || id is Number + private fun text(t: String) = JSONObject().put("type", "text").put("text", t) + private fun ok(id: Any, result: JSONObject) = JSONObject().put("jsonrpc", "2.0").put("id", id).put("result", result) + private fun error(id: Any?, code: Int, message: String) = JSONObject().put("jsonrpc", "2.0").put("id", id ?: JSONObject.NULL) + .put("error", JSONObject().put("code", code).put("message", message)) } diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt new file mode 100644 index 0000000000..7e15669b14 --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt @@ -0,0 +1,318 @@ +package org.appdevforall.sqlitedebugger.mcp + +import java.io.File +import org.appdevforall.sqlitedebugger.engine.AppConnection +import org.appdevforall.sqlitedebugger.engine.DebugEngineException +import org.appdevforall.sqlitedebugger.engine.EventKind +import org.appdevforall.sqlitedebugger.engine.FakeDebugEngine +import org.appdevforall.sqlitedebugger.engine.QueryResult +import org.json.JSONArray +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class DatabaseToolsTest { + private val engine = FakeDebugEngine() + private var now = 1_000_000L + private var dir: File? = null + private val tools get() = DatabaseTools(engine, dir, { now }) + private val seenSql = mutableListOf() + + private fun conn( + id: Long, + path: String, + closing: Boolean = false, + ) = AppConnection(id, path, false, closing, 1_700_000_000_000_000_000L + id, 5) + + private fun runTool( + t: DatabaseTools, + name: String, + args: JSONObject = JSONObject(), + ): JSONObject { + val protocol = McpProtocol(t.tools(), "test", { "s" }, 4) + val init = JSONObject().put("jsonrpc", "2.0").put("id", 1).put("method", "initialize") + .put("params", JSONObject().put("protocolVersion", "2025-06-18")).toString() + val session = protocol.post(init, null, null).headers[McpProtocol.SESSION_HEADER]!! + val call = JSONObject().put("jsonrpc", "2.0").put("id", 2).put("method", "tools/call") + .put("params", JSONObject().put("name", name).put("arguments", args)).toString() + return JSONObject(protocol.post(call, session, null).body!!).getJSONObject("result") + } + + private fun ok( + name: String, + args: JSONObject = JSONObject(), + t: DatabaseTools = tools, + ): JSONObject { + val r = runTool(t, name, args) + assertFalse(r.toString(), r.getBoolean("isError")) + return r.getJSONObject("structuredContent") + } + + private fun db(path: String = "/data/a.db") = JSONObject().put("database", path) + + private fun answerWith(vararg byPrefix: Pair) { + engine.answer = { _, _, sql, _ -> + seenSql += sql + byPrefix.firstOrNull { sql.startsWith(it.first) }?.second ?: QueryResult(emptyList(), emptyList(), 0, false) + } + } + + @Test fun everyToolIsReadOnly() { + val list = tools.tools() + assertEquals( + listOf("list_databases", "list_connections", "get_schema", "query", "explain_query_plan", "get_stats", "integrity_check", "get_events"), + list.map { it.name }, + ) + list.forEach { assertTrue(it.name, it.describe().getJSONObject("annotations").getBoolean("readOnlyHint")) } + } + + @Test fun listDatabasesGroupsByPathAndSkipsEmpty() { + engine.appConnections += listOf(conn(1, "/data/a.db"), conn(2, "/data/a.db"), conn(3, ""), conn(4, "/data/b.db")) + val dbs = ok("list_databases").getJSONArray("databases") + assertEquals(2, dbs.length()) + assertEquals("/data/a.db", dbs.getJSONObject(0).getString("path")) + assertEquals(2, dbs.getJSONObject(0).getInt("connections")) + assertTrue(dbs.getJSONObject(1).getBoolean("available")) + } + + @Test fun listDatabasesAddsUnopenedFilesButNotSideFiles() { + val d = kotlin.io.path.createTempDirectory("dbs").toFile() + try { + listOf("open.db", "closed.db", "closed.db-wal", "closed.db-shm", "closed.db-journal").forEach { File(d, it).writeText("x") } + dir = d + engine.appConnections += conn(1, File(d, "open.db").absolutePath) + val dbs = ok("list_databases").getJSONArray("databases") + assertEquals(2, dbs.length()) + assertTrue(dbs.getJSONObject(0).getBoolean("available")) + val closed = dbs.getJSONObject(1) + assertEquals(File(d, "closed.db").absolutePath, closed.getString("path")) + assertFalse(closed.getBoolean("available")) + } finally { + d.deleteRecursively() + } + } + + @Test fun listConnectionsEncodesBigIntegersAsStrings() { + engine.appConnections += conn(1, "/data/a.db", closing = true) + val c = ok("list_connections").getJSONArray("connections").getJSONObject(0) + assertEquals(1, c.getInt("id")) + assertTrue(c.getBoolean("closing")) + assertEquals("1700000000000000001", c.getString("opened_at_epoch_nanos")) + } + + @Test fun everyDatabaseToolRefusesAnUnknownPathBeforeTheEngine() { + engine.appConnections += conn(1, "/data/a.db") + for ((name, extra) in listOf("get_schema" to JSONObject(), "query" to JSONObject().put("sql", "select 1"), "explain_query_plan" to JSONObject().put("sql", "select 1"), "get_stats" to JSONObject(), "integrity_check" to JSONObject())) { + for (bad in listOf("/data/other.db", "", "/data/a.db-wal")) { + val r = runTool(tools, name, db(bad).also { o -> extra.keys().forEach { k -> o.put(k, extra.get(k)) } }) + assertTrue("$name $bad", r.getBoolean("isError")) + } + val missing = runTool(tools, name, extra) + assertTrue(name, missing.getBoolean("isError")) + } + assertTrue(engine.openedServerConnections.isEmpty()) + } + + @Test fun emptyPathOfAnInMemoryConnectionIsNotOpenable() { + engine.appConnections += conn(1, "") + assertTrue(runTool(tools, "get_stats", db("")).getBoolean("isError")) + assertTrue(engine.openedServerConnections.isEmpty()) + } + + @Test fun queryOpensReadOnlyAndPages() { + engine.appConnections += conn(1, "/data/a.db") + engine.answer = { _, _, _, _ -> QueryResult(listOf("n"), (1L..5L).map { listOf(it) }, 0, false) } + val r = ok("query", db().put("sql", "select n from t").put("max_rows", 2).put("offset", 1)) + assertEquals(listOf(listOf("/data/a.db", false)), engine.openedServerConnections.map { listOf(it.first, it.second) }) + assertEquals(2, r.getInt("row_count")) + assertEquals(2, r.getJSONArray("rows").getJSONArray(0).getInt(0)) + assertTrue(r.getBoolean("has_more")) + assertEquals(3, r.getInt("next_offset")) + assertTrue(r.getString("note").contains("re-runs")) + assertTrue(tools.tools().first { it.name == "query" }.description.contains("re-runs")) + } + + @Test fun queryChecksItsArguments() { + engine.appConnections += conn(1, "/data/a.db") + fun bad(vararg kv: Pair) = assertTrue(kv.toString(), runTool(tools, "query", db().also { o -> kv.forEach { o.put(it.first, it.second) } }).getBoolean("isError")) + bad() + bad("sql" to "x", "max_rows" to 1001) + bad("sql" to "x", "max_rows" to 0) + bad("sql" to "x", "offset" to -1) + bad("sql" to "x", "max_rows" to 1.5) + bad("sql" to "x", "args" to "nope") + bad("sql" to "x", "args" to JSONArray().put(JSONObject())) + assertTrue(engine.openedServerConnections.isEmpty()) + } + + @Test fun queryBindsArgsByType() { + engine.appConnections += conn(1, "/data/a.db") + var got: List = emptyList() + engine.answer = { _, _, _, a -> got = a; QueryResult(emptyList(), emptyList(), 0, false) } + ok("query", db().put("sql", "x").put("args", JSONArray().put(JSONObject.NULL).put(3).put(2.5).put("s").put(true))) + assertEquals(listOf(null, 3L, 2.5, "s", 1L), got) + } + + @Test fun queryErrorsCarryTheSqliteCode() { + engine.appConnections += conn(1, "/data/a.db") + engine.answer = { _, _, _, _ -> throw DebugEngineException(DebugEngineException.SQLITE_AUTH, "not authorized") } + val r = runTool(tools, "query", db().put("sql", "delete from t")) + assertTrue(r.getBoolean("isError")) + assertEquals(23, r.getJSONObject("structuredContent").getInt("sqlite_code")) + } + + @Test fun deadlineIsFiftySecondsAhead() { + engine.appConnections += conn(1, "/data/a.db") + var deadline = 0L + val real = engine + engine.answer = { _, _, _, _ -> QueryResult(emptyList(), emptyList(), 0, false) } + // the fake ignores the deadline, so check the arithmetic through the constant and a wrapper engine + val wrapper = object : org.appdevforall.sqlitedebugger.engine.DebugEngine by real { + override fun openServerConnection(path: String, writable: Boolean) = real.openServerConnection(path, writable).let { c -> + object : org.appdevforall.sqlitedebugger.engine.ServerConnection by c { + override fun query(sql: String, args: List, offset: Long, limits: org.appdevforall.sqlitedebugger.engine.QueryLimits, deadlineMillis: Long): QueryResult { + deadline = deadlineMillis + return c.query(sql, args, offset, limits, deadlineMillis) + } + } + } + } + ok("query", db().put("sql", "x"), DatabaseTools(wrapper, null, { now })) + assertEquals(now + 50_000, deadline) + } + + @Test fun jsonEncodesNonFiniteDoublesBigIntegersAndBlobs() { + engine.appConnections += conn(1, "/data/a.db") + val big = 9_007_199_254_740_993L + engine.answer = { _, _, _, _ -> + QueryResult( + listOf("a", "b", "c", "d", "e", "f", "g", "h"), + listOf(listOf(Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, big, -big, 9_007_199_254_740_992L, byteArrayOf(1, 2, 3), null)), + 0, + false, + ) + } + val row = ok("query", db().put("sql", "x")).getJSONArray("rows").getJSONArray(0) + assertEquals("NaN", row.get(0)) + assertEquals("Infinity", row.get(1)) + assertEquals("-Infinity", row.get(2)) + assertEquals("9007199254740993", row.get(3)) + assertEquals("-9007199254740993", row.get(4)) + assertEquals(9_007_199_254_740_992L, row.getLong(5)) + val blob = row.getJSONObject(6) + assertEquals("AQID", blob.getString("base64")) + assertEquals(3, blob.getInt("bytes")) + assertFalse(blob.getBoolean("truncated")) + assertTrue(row.isNull(7)) + assertEquals(true, DatabaseTools.encodeValue(ByteArray(1024).also { }).let { (it as JSONObject).getBoolean("truncated") }) + assertEquals(1.5, DatabaseTools.encodeValue(1.5)) + } + + @Test fun getSchemaGroupsByType() { + engine.appConnections += conn(1, "/data/a.db") + answerWith( + "SELECT type" to QueryResult( + listOf("type", "name", "tbl_name", "sql"), + listOf( + listOf("index", "i1", "t", "CREATE INDEX i1 ON t(a)"), + listOf("table", "t", "t", "CREATE TABLE t(a)"), + listOf("trigger", "tr", "t", "CREATE TRIGGER tr ..."), + listOf("view", "v", "v", "CREATE VIEW v AS ..."), + ), + 0, + false, + ), + ) + val r = ok("get_schema", db()) + assertEquals("t", r.getJSONArray("tables").getJSONObject(0).getString("name")) + assertEquals("CREATE INDEX i1 ON t(a)", r.getJSONArray("indexes").getJSONObject(0).getString("sql")) + assertEquals(1, r.getJSONArray("views").length()) + assertEquals(1, r.getJSONArray("triggers").length()) + assertTrue(seenSql.single().contains("sqlite_schema")) + } + + @Test fun explainQueryPlanPrefixesTheSql() { + engine.appConnections += conn(1, "/data/a.db") + answerWith("EXPLAIN QUERY PLAN" to QueryResult(listOf("id", "parent", "notused", "detail"), listOf(listOf(3L, 0L, 0L, "SCAN t")), 0, false)) + val plan = ok("explain_query_plan", db().put("sql", "select * from t")).getJSONArray("plan") + assertEquals("SCAN t", plan.getJSONObject(0).getString("detail")) + assertEquals("EXPLAIN QUERY PLAN select * from t", seenSql.single()) + } + + @Test fun getStatsCountsRowsUnderTenTables() { + engine.appConnections += conn(1, "/data/a.db") + fun one(v: Long) = QueryResult(listOf("x"), listOf(listOf(v)), 0, false) + answerWith( + "PRAGMA page_count" to one(10), + "PRAGMA page_size" to one(4096), + "PRAGMA freelist_count" to one(2), + "SELECT name" to QueryResult(listOf("name"), listOf(listOf("a\"b"), listOf("c")), 0, false), + "SELECT COUNT" to one(7), + ) + val r = ok("get_stats", db()) + assertEquals(10, r.getInt("page_count")) + assertEquals(4096, r.getInt("page_size")) + assertEquals(2, r.getInt("freelist_count")) + assertEquals(40960, r.getInt("size_bytes")) + assertEquals(7, r.getJSONObject("row_counts").getInt("a\"b")) + assertTrue(seenSql.contains("SELECT COUNT(*) FROM \"a\"\"b\"")) + } + + @Test fun getStatsSkipsCountsFromTenTables() { + engine.appConnections += conn(1, "/data/a.db") + answerWith("SELECT name" to QueryResult(listOf("name"), (1..10).map { listOf("t$it") }, 0, false)) + val r = ok("get_stats", db()) + assertFalse(r.has("row_counts")) + assertTrue(r.has("row_counts_skipped")) + assertTrue(seenSql.none { it.startsWith("SELECT COUNT") }) + } + + @Test fun integrityCheckIsQuickUnlessAskedForFull() { + engine.appConnections += conn(1, "/data/a.db") + answerWith("PRAGMA" to QueryResult(listOf("quick_check"), listOf(listOf("ok")), 0, false)) + val quick = ok("integrity_check", db()) + assertTrue(quick.getBoolean("ok")) + assertEquals("quick_check", quick.getString("check")) + ok("integrity_check", db().put("full", true)) + assertEquals(listOf("PRAGMA quick_check", "PRAGMA integrity_check"), seenSql) + answerWith("PRAGMA" to QueryResult(listOf("c"), listOf(listOf("row 1 missing from index i1")), 0, false)) + assertFalse(ok("integrity_check", db()).getBoolean("ok")) + } + + @Test fun getEventsReturnsCursorDroppedAndTurnsOnExpansion() { + assertFalse(engine.clientConnected) + engine.addEvent(EventKind.STATEMENT, "select 1") + engine.addEvent(EventKind.PROFILE, "select 1", durationNanos = 99) + engine.addEvent(EventKind.STATEMENT, "select 2") + engine.dropOldest(1) + val t = tools + val first = ok("get_events", t = t) + assertTrue(engine.clientConnected) + assertEquals(1, first.getInt("dropped")) + assertEquals(3, first.getInt("next")) + val events = first.getJSONArray("events") + assertEquals(2, events.length()) + assertEquals("profile", events.getJSONObject(0).getString("kind")) + assertEquals(99, events.getJSONObject(0).getInt("duration_nanos")) + assertTrue(events.getJSONObject(1).isNull("duration_nanos")) + val second = ok("get_events", JSONObject().put("after", 3), t) + assertEquals(0, second.getJSONArray("events").length()) + assertEquals(0, second.getInt("dropped")) + assertEquals(3, second.getInt("next")) + t.onServerStopped() + assertFalse(engine.clientConnected) + } + + @Test fun stoppingTheServerClosesServerConnections() { + engine.appConnections += conn(1, "/data/a.db") + val t = tools + ok("get_stats", db(), t) + ok("get_stats", db(), t) + assertEquals(1, engine.openedServerConnections.size) + t.onServerStopped() + ok("get_stats", db(), t) + assertEquals(2, engine.openedServerConnections.size) + } +} diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt index a1bf367dbd..9d705d3bda 100644 --- a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt @@ -1,10 +1,5 @@ package org.appdevforall.sqlitedebugger.mcp -import org.appdevforall.sqlitedebugger.mcp.McpHttp -import org.appdevforall.sqlitedebugger.mcp.McpProtocol -import org.appdevforall.sqlitedebugger.mcp.McpTool -import org.appdevforall.sqlitedebugger.mcp.McpToolError -import org.appdevforall.sqlitedebugger.mcp.McpTools import org.json.JSONArray import org.json.JSONObject import org.junit.After @@ -22,342 +17,317 @@ import java.net.Socket */ class McpProtocolTest { - private var sessionNumber = 0 - private val echo = McpTool("echo", "Echo", "Returns its arguments.") { args -> JSONObject().put("got", args) } - private val needsPerson = McpTool("needs_person", "Needs a person", "Always says a person is needed.", readOnly = false) { _ -> - throw McpToolError("A person is needed: tap Allow on the sharing phone.", JSONObject().put("person_needed", true)) - } - private val crashes = McpTool("crashes", "Crashes", "Throws an unexpected exception.") { _ -> throw IllegalStateException("boom") } - - private fun protocol(maxSessions: Int = 16) = - McpProtocol(listOf(echo, needsPerson, crashes), "test", { "s${++sessionNumber}" }, maxSessions) - - private fun initBody(version: String = "2025-06-18") = JSONObject().put("jsonrpc", "2.0").put("id", 1).put("method", "initialize") - .put("params", JSONObject().put("protocolVersion", version).put("capabilities", JSONObject()) - .put("clientInfo", JSONObject().put("name", "test").put("version", "1"))).toString() - - private fun request(id: Any, method: String, params: JSONObject? = null) = JSONObject().put("jsonrpc", "2.0").put("id", id) - .put("method", method).also { if (params != null) it.put("params", params) } - - private fun session(p: McpProtocol): String { - val r = p.post(initBody(), null, null) - assertEquals(200, r.status) - return r.headers[McpProtocol.SESSION_HEADER]!! - } - - private fun call(p: McpProtocol, s: String, tool: String, args: JSONObject? = null): JSONObject { - val params = JSONObject().put("name", tool).also { if (args != null) it.put("arguments", args) } - val r = p.post(request(7, "tools/call", params).toString(), s, null) - assertEquals(200, r.status) - return JSONObject(r.body!!) - } - - // ---------------------------------------------------------------- the protocol - - @Test fun initializeAgreesVersionAndStartsSession() { - val p = protocol() - val r = p.post(initBody("2025-03-26"), null, null) - assertEquals(200, r.status) - assertEquals("s1", r.headers[McpProtocol.SESSION_HEADER]) - val result = JSONObject(r.body!!).getJSONObject("result") - assertEquals("2025-03-26", result.getString("protocolVersion")) - assertEquals(McpProtocol.SERVER_NAME, result.getJSONObject("serverInfo").getString("name")) - assertTrue(result.getJSONObject("capabilities").has("tools")) - assertEquals(1, p.sessionCount()) - } - - @Test fun unknownVersionGetsTheNewest() { - val r = protocol().post(initBody("1999-01-01"), null, null) - assertEquals(McpProtocol.VERSIONS[0], JSONObject(r.body!!).getJSONObject("result").getString("protocolVersion")) - } - - @Test fun requestsNeedALiveSession() { - val p = protocol() - assertEquals(400, p.post(request(1, "ping").toString(), null, null).status) - assertEquals(404, p.post(request(1, "ping").toString(), "nope", null).status) - val s = session(p) - assertEquals(200, p.post(request(1, "ping").toString(), s, null).status) - } - - @Test fun unsupportedVersionHeaderIsRefused() { - val p = protocol(); val s = session(p) - assertEquals(400, p.post(request(1, "ping").toString(), s, "1999-01-01").status) - assertEquals(200, p.post(request(1, "ping").toString(), s, "2025-06-18").status) - } - - @Test fun notificationsGetNoAnswer() { - val p = protocol(); val s = session(p) - val r = p.post(JSONObject().put("jsonrpc", "2.0").put("method", "notifications/initialized").toString(), s, null) - assertEquals(202, r.status) - assertNull(r.body) - } - - @Test fun pingAnswersEmptyResult() { - val p = protocol(); val s = session(p) - val r = JSONObject(p.post(request("a", "ping").toString(), s, null).body!!) - assertEquals("a", r.get("id")) - assertEquals(0, r.getJSONObject("result").length()) - } - - @Test fun toolsListDescribesEveryTool() { - val p = protocol(); val s = session(p) - val tools = JSONObject(p.post(request(2, "tools/list").toString(), s, null).body!!).getJSONObject("result").getJSONArray("tools") - val names = (0 until tools.length()).map { tools.getJSONObject(it).getString("name") } - assertEquals(listOf("echo", "needs_person", "crashes"), names) - val first = tools.getJSONObject(0) - assertEquals("object", first.getJSONObject("inputSchema").getString("type")) - assertTrue(first.getJSONObject("annotations").getBoolean("readOnlyHint")) - assertFalse(tools.getJSONObject(1).getJSONObject("annotations").getBoolean("readOnlyHint")) - } - - @Test fun toolCallReturnsStructuredContent() { - val p = protocol(); val s = session(p) - val result = call(p, s, "echo", JSONObject().put("x", 3)).getJSONObject("result") - assertFalse(result.getBoolean("isError")) - assertEquals(3, result.getJSONObject("structuredContent").getJSONObject("got").getInt("x")) - assertEquals("text", result.getJSONArray("content").getJSONObject(0).getString("type")) - } - - @Test fun toolCallWithoutArgumentsGetsEmptyObject() { - val p = protocol(); val s = session(p) - val got = call(p, s, "echo").getJSONObject("result").getJSONObject("structuredContent").getJSONObject("got") - assertEquals(0, got.length()) - } - - @Test fun toolErrorIsAResultWithIsError() { - val p = protocol(); val s = session(p) - val result = call(p, s, "needs_person").getJSONObject("result") - assertTrue(result.getBoolean("isError")) - assertTrue(result.getJSONArray("content").getJSONObject(0).getString("text").contains("person is needed")) - assertTrue(result.getJSONObject("structuredContent").getBoolean("person_needed")) - } - - @Test fun unexpectedExceptionIsAResultWithIsError() { - val p = protocol(); val s = session(p) - val result = call(p, s, "crashes").getJSONObject("result") - assertTrue(result.getBoolean("isError")) - assertTrue(result.getJSONArray("content").getJSONObject(0).getString("text").contains("boom")) - } - - @Test fun unknownToolAndBadArgumentsAreInvalidParams() { - val p = protocol(); val s = session(p) - assertEquals(McpProtocol.INVALID_PARAMS, call(p, s, "nope").getJSONObject("error").getInt("code")) - val bad = request(3, "tools/call", JSONObject().put("name", "echo").put("arguments", JSONArray())).toString() - assertEquals(McpProtocol.INVALID_PARAMS, JSONObject(p.post(bad, s, null).body!!).getJSONObject("error").getInt("code")) - } - - @Test fun unknownMethodIsMethodNotFound() { - val p = protocol(); val s = session(p) - val r = JSONObject(p.post(request(4, "resources/list").toString(), s, null).body!!) - assertEquals(McpProtocol.METHOD_NOT_FOUND, r.getJSONObject("error").getInt("code")) - } - - @Test fun badJsonIsAParseError() { - val r = protocol().post("{not json", null, null) - assertEquals(400, r.status) - assertEquals(McpProtocol.PARSE_ERROR, JSONObject(r.body!!).getJSONObject("error").getInt("code")) - } - - @Test fun batchGetsAnArrayOfReplies() { - val p = protocol(); val s = session(p) - val batch = JSONArray().put(request(1, "ping")).put(JSONObject().put("jsonrpc", "2.0").put("method", "notifications/x")).put(request(2, "tools/list")) - val r = p.post(batch.toString(), s, null) - assertEquals(200, r.status) - val replies = JSONArray(r.body!!) - assertEquals(2, replies.length()) - assertEquals(1, replies.getJSONObject(0).getInt("id")) - assertEquals(2, replies.getJSONObject(1).getInt("id")) - } - - @Test fun initializeMustBeAlone() { - val batch = JSONArray().put(JSONObject(initBody())).put(request(2, "ping")) - assertEquals(400, protocol().post(batch.toString(), null, null).status) - } - - @Test fun deleteEndsTheSession() { - val p = protocol(); val s = session(p) - assertEquals(204, p.delete(s).status) - assertEquals(404, p.delete(s).status) - assertEquals(404, p.post(request(1, "ping").toString(), s, null).status) - } - - @Test fun oldestSessionIsDroppedBeyondTheLimit() { - val p = protocol(maxSessions = 2) - val a = session(p); val b = session(p) - p.post(request(1, "ping").toString(), a, null) // a is now the most recently used - val c = session(p) // drops b - assertEquals(2, p.sessionCount()) - assertEquals(200, p.post(request(1, "ping").toString(), a, null).status) - assertEquals(404, p.post(request(1, "ping").toString(), b, null).status) - assertEquals(200, p.post(request(1, "ping").toString(), c, null).status) - } - - @Test fun endAllEndsEverySession() { - val p = protocol(); val s = session(p) - p.endAll() - assertEquals(0, p.sessionCount()) - assertEquals(404, p.post(request(1, "ping").toString(), s, null).status) - } - - @Test fun statusLineBecomesOneWord() { - assertEquals("serving", McpTools.stateOf("Serving Store_12345678", "http://192.168.1.5:6175/")) - assertEquals("failed", McpTools.stateOf("Failed: storage access hasn't been granted", null)) - assertEquals("paused", McpTools.stateOf("Paused while the screen is off.", null)) - assertEquals("stopped", McpTools.stateOf("Stopped", null)) - assertEquals("starting", McpTools.stateOf("Starting…", null)) - assertEquals("starting", McpTools.stateOf("Restarting after the server stopped unexpectedly…", null)) - assertEquals("starting", McpTools.stateOf("Not on Wi-Fi — starting a hotspot…", null)) - } - - // ---------------------------------------------------------------- step 3: share and server tools - - private val share = org.appdevforall.localstore.server.ShareControl - - @Test fun shareStateBecomesOneWord() { - assertEquals("waiting_for_phone", share.stateWord("OPEN", 0, false)) - assertEquals("preparing", share.stateWord("PREPARING", 0, false)) - assertEquals("phone_connecting", share.stateWord("ASKED", 0, false)) - assertEquals("waiting_for_allow", share.stateWord("REVEALED", 0, false)) - assertEquals("sending", share.stateWord("ALLOWED", 0, false)) - assertEquals("transfer_failed", share.stateWord("ALLOWED", 2, false)) - assertEquals("done", share.stateWord("DONE", 0, false)) - assertEquals("denied", share.stateWord("DENIED", 0, false)) - assertEquals("expired", share.stateWord("CLOSED", 0, false)) - assertEquals("not_serving", share.stateWord("NONE", 0, false)) - assertEquals("not_serving", share.stateWord(null, 0, false)) - assertEquals("cant_share", share.stateWord("ERROR", 0, true)) - assertEquals("closed", share.stateWord("ERROR", 0, false)) - } - - /** The rule MCP control rests on: ShareControl may open, poll and close a share, and never allow or deny one. */ - @Test fun shareControlCanNeverAllowOrDeny() { - val ipc = org.appdevforall.localstore.server.Ipc - val field = share.javaClass.getDeclaredField("ALLOWED_MESSAGES").apply { isAccessible = true } - val allowed = field.get(null) as Set<*> - assertEquals(setOf(ipc.MSG_SHARE_OPEN, ipc.MSG_SHARE_POLL, ipc.MSG_SHARE_CLOSE), allowed) - assertFalse(ipc.MSG_SHARE_ALLOW in allowed) - assertFalse(ipc.MSG_SHARE_DENY in allowed) - } - - @Test fun waitSecondsIsCheckedAndCapped() { - assertEquals(10, McpTools.waitArg(JSONObject(), 10)) - assertEquals(10, McpTools.waitArg(JSONObject().put("wait_seconds", JSONObject.NULL), 10)) - assertEquals(5, McpTools.waitArg(JSONObject().put("wait_seconds", 5), 10)) - assertEquals(2, McpTools.waitArg(JSONObject().put("wait_seconds", 2.7), 10)) - assertEquals(share.MAX_WAIT_S, McpTools.waitArg(JSONObject().put("wait_seconds", 999), 10)) - assertEquals(0, McpTools.waitArg(JSONObject().put("wait_seconds", -3), 10)) - assertTrue(runCatching { McpTools.waitArg(JSONObject().put("wait_seconds", "5"), 10) }.exceptionOrNull() is McpToolError) - } - - @Test fun storeArgumentIsOwnOrCopy() { - assertNull(McpTools.storeArg(JSONObject())) - assertNull(McpTools.storeArg(JSONObject().put("store", JSONObject.NULL))) - assertEquals("own", McpTools.storeArg(JSONObject().put("store", "own"))) - assertEquals("copy", McpTools.storeArg(JSONObject().put("store", "copy"))) - assertTrue(runCatching { McpTools.storeArg(JSONObject().put("store", "both")) }.exceptionOrNull() is McpToolError) - assertTrue(runCatching { McpTools.storeArg(JSONObject().put("store", 1)) }.exceptionOrNull() is McpToolError) - } - - // ---------------------------------------------------------------- the HTTP side - - private val token = "secret-token" - private var server: McpHttp? = null - - @After fun stopServer() { server?.stop(); server = null } - - private fun startServer(): Int { - val s = McpHttp(protocol(), { token }, port = 0) - s.start() - server = s - return s.boundPort - } - - private class Answer(val status: Int, val headers: Map, val body: String) - - /** One request on its own connection; [headers] may replace Host. */ - private fun http(port: Int, method: String, path: String = McpHttp.PATH, headers: Map = emptyMap(), body: String? = null): Answer { - Socket("127.0.0.1", port).use { s -> - s.soTimeout = 5000 - val bytes = body?.toByteArray(Charsets.UTF_8) ?: ByteArray(0) - val all = linkedMapOf("Host" to "127.0.0.1:$port", "Connection" to "close") - all.putAll(headers) - if (body != null) { all["Content-Type"] = "application/json"; all["Content-Length"] = bytes.size.toString() } - val head = StringBuilder("$method $path HTTP/1.1\r\n") - for ((k, v) in all) head.append(k).append(": ").append(v).append("\r\n") - head.append("\r\n") - s.getOutputStream().apply { write(head.toString().toByteArray(Charsets.ISO_8859_1)); write(bytes); flush() } - val text = s.getInputStream().readBytes().toString(Charsets.UTF_8) - val lines = text.substringBefore("\r\n\r\n").split("\r\n") - val status = lines[0].split(' ')[1].toInt() - val h = lines.drop(1).associate { it.substringBefore(':').trim().lowercase() to it.substringAfter(':').trim() } - return Answer(status, h, text.substringAfter("\r\n\r\n", "")) - } - } - - private val auth get() = mapOf("Authorization" to "Bearer $token") - - @Test fun httpNeedsTheToken() { - val port = startServer() - val none = http(port, "POST", body = initBody()) - assertEquals(401, none.status) - assertNotNull(none.headers["www-authenticate"]) - assertEquals(401, http(port, "POST", headers = mapOf("Authorization" to "Bearer wrong"), body = initBody()).status) - assertFalse("the token never appears in a refusal", none.body.contains(token)) - } - - @Test fun httpInitializeWithTheToken() { - val port = startServer() - val r = http(port, "POST", headers = auth, body = initBody()) - assertEquals(200, r.status) - assertNotNull(r.headers["mcp-session-id"]) - assertEquals("2025-06-18", JSONObject(r.body).getJSONObject("result").getString("protocolVersion")) - } - - @Test fun httpLocalhostHostIsAccepted() { - val port = startServer() - assertEquals(200, http(port, "POST", headers = auth + ("Host" to "localhost:$port"), body = initBody()).status) - } - - @Test fun httpOtherHostIsRefused() { - val port = startServer() - assertEquals(403, http(port, "POST", headers = auth + ("Host" to "evil.example"), body = initBody()).status) - assertEquals(403, http(port, "POST", headers = auth + ("Host" to "127.0.0.1:x"), body = initBody()).status) - } - - @Test fun httpWebPageOriginIsRefused() { - val port = startServer() - assertEquals(403, http(port, "POST", headers = auth + ("Origin" to "https://evil.example"), body = initBody()).status) - assertEquals(200, http(port, "POST", headers = auth + ("Origin" to "http://localhost:3000"), body = initBody()).status) - } - - @Test fun httpGetIsNotAllowed() { - val port = startServer() - val r = http(port, "GET", headers = auth) - assertEquals(405, r.status) - assertEquals("POST, DELETE", r.headers["allow"]) - } - - @Test fun httpOtherPathIsNotFound() { - assertEquals(404, http(startServer(), "POST", path = "/other", headers = auth, body = initBody()).status) - } - - @Test fun httpWrongContentTypeIsRefused() { - val port = startServer() - Socket("127.0.0.1", port).use { s -> - val body = initBody().toByteArray() - s.getOutputStream().write(("POST /mcp HTTP/1.1\r\nHost: 127.0.0.1\r\nAuthorization: Bearer $token\r\nConnection: close\r\n" + - "Content-Type: text/plain\r\nContent-Length: ${body.size}\r\n\r\n").toByteArray() + body) - val text = s.getInputStream().readBytes().toString(Charsets.UTF_8) - assertTrue(text.startsWith("HTTP/1.1 415")) - } - } - - @Test fun httpSessionThenDelete() { - val port = startServer() - val s = http(port, "POST", headers = auth, body = initBody()).headers["mcp-session-id"]!! - val ping = request(1, "ping").toString() - assertEquals(200, http(port, "POST", headers = auth + ("Mcp-Session-Id" to s), body = ping).status) - assertEquals(204, http(port, "DELETE", headers = auth + ("Mcp-Session-Id" to s)).status) - assertEquals(404, http(port, "POST", headers = auth + ("Mcp-Session-Id" to s), body = ping).status) - } + private var sessionNumber = 0 + private val echo = McpTool("echo", "Echo", "Returns its arguments.") { args -> JSONObject().put("got", args) } + private val needsPerson = McpTool("needs_person", "Needs a person", "Always says a person is needed.", readOnly = false) { _ -> + throw McpToolError("A person is needed: tap Allow on the sharing phone.", JSONObject().put("person_needed", true)) + } + private val crashes = McpTool("crashes", "Crashes", "Throws an unexpected exception.") { _ -> throw IllegalStateException("boom") } + + private fun protocol(maxSessions: Int = 16) = + McpProtocol(listOf(echo, needsPerson, crashes), "test", { "s${++sessionNumber}" }, maxSessions) + + private fun initBody(version: String = "2025-06-18") = JSONObject().put("jsonrpc", "2.0").put("id", 1).put("method", "initialize") + .put("params", JSONObject().put("protocolVersion", version).put("capabilities", JSONObject()) + .put("clientInfo", JSONObject().put("name", "test").put("version", "1"))).toString() + + private fun request(id: Any, method: String, params: JSONObject? = null) = JSONObject().put("jsonrpc", "2.0").put("id", id) + .put("method", method).also { if (params != null) it.put("params", params) } + + private fun session(p: McpProtocol): String { + val r = p.post(initBody(), null, null) + assertEquals(200, r.status) + return r.headers[McpProtocol.SESSION_HEADER]!! + } + + private fun call(p: McpProtocol, s: String, tool: String, args: JSONObject? = null): JSONObject { + val params = JSONObject().put("name", tool).also { if (args != null) it.put("arguments", args) } + val r = p.post(request(7, "tools/call", params).toString(), s, null) + assertEquals(200, r.status) + return JSONObject(r.body!!) + } + + // ---------------------------------------------------------------- the protocol + + @Test fun initializeAgreesVersionAndStartsSession() { + val p = protocol() + val r = p.post(initBody("2025-03-26"), null, null) + assertEquals(200, r.status) + assertEquals("s1", r.headers[McpProtocol.SESSION_HEADER]) + val result = JSONObject(r.body!!).getJSONObject("result") + assertEquals("2025-03-26", result.getString("protocolVersion")) + assertEquals(McpProtocol.SERVER_NAME, result.getJSONObject("serverInfo").getString("name")) + assertTrue(result.getJSONObject("capabilities").has("tools")) + assertEquals(1, p.sessionCount()) + } + + @Test fun unknownVersionGetsTheNewest() { + val r = protocol().post(initBody("1999-01-01"), null, null) + assertEquals(McpProtocol.VERSIONS[0], JSONObject(r.body!!).getJSONObject("result").getString("protocolVersion")) + } + + @Test fun requestsNeedALiveSession() { + val p = protocol() + assertEquals(400, p.post(request(1, "ping").toString(), null, null).status) + assertEquals(404, p.post(request(1, "ping").toString(), "nope", null).status) + val s = session(p) + assertEquals(200, p.post(request(1, "ping").toString(), s, null).status) + } + + @Test fun unsupportedVersionHeaderIsRefused() { + val p = protocol(); val s = session(p) + assertEquals(400, p.post(request(1, "ping").toString(), s, "1999-01-01").status) + assertEquals(200, p.post(request(1, "ping").toString(), s, "2025-06-18").status) + } + + @Test fun notificationsGetNoAnswer() { + val p = protocol(); val s = session(p) + val r = p.post(JSONObject().put("jsonrpc", "2.0").put("method", "notifications/initialized").toString(), s, null) + assertEquals(202, r.status) + assertNull(r.body) + } + + @Test fun pingAnswersEmptyResult() { + val p = protocol(); val s = session(p) + val r = JSONObject(p.post(request("a", "ping").toString(), s, null).body!!) + assertEquals("a", r.get("id")) + assertEquals(0, r.getJSONObject("result").length()) + } + + @Test fun toolsListDescribesEveryTool() { + val p = protocol(); val s = session(p) + val tools = JSONObject(p.post(request(2, "tools/list").toString(), s, null).body!!).getJSONObject("result").getJSONArray("tools") + val names = (0 until tools.length()).map { tools.getJSONObject(it).getString("name") } + assertEquals(listOf("echo", "needs_person", "crashes"), names) + val first = tools.getJSONObject(0) + assertEquals("object", first.getJSONObject("inputSchema").getString("type")) + assertTrue(first.getJSONObject("annotations").getBoolean("readOnlyHint")) + assertFalse(tools.getJSONObject(1).getJSONObject("annotations").getBoolean("readOnlyHint")) + } + + @Test fun toolCallReturnsStructuredContent() { + val p = protocol(); val s = session(p) + val result = call(p, s, "echo", JSONObject().put("x", 3)).getJSONObject("result") + assertFalse(result.getBoolean("isError")) + assertEquals(3, result.getJSONObject("structuredContent").getJSONObject("got").getInt("x")) + assertEquals("text", result.getJSONArray("content").getJSONObject(0).getString("type")) + } + + @Test fun toolCallWithoutArgumentsGetsEmptyObject() { + val p = protocol(); val s = session(p) + val got = call(p, s, "echo").getJSONObject("result").getJSONObject("structuredContent").getJSONObject("got") + assertEquals(0, got.length()) + } + + @Test fun toolErrorIsAResultWithIsError() { + val p = protocol(); val s = session(p) + val result = call(p, s, "needs_person").getJSONObject("result") + assertTrue(result.getBoolean("isError")) + assertTrue(result.getJSONArray("content").getJSONObject(0).getString("text").contains("person is needed")) + assertTrue(result.getJSONObject("structuredContent").getBoolean("person_needed")) + } + + @Test fun unexpectedExceptionIsAResultWithIsError() { + val p = protocol(); val s = session(p) + val result = call(p, s, "crashes").getJSONObject("result") + assertTrue(result.getBoolean("isError")) + assertTrue(result.getJSONArray("content").getJSONObject(0).getString("text").contains("boom")) + } + + @Test fun unknownToolAndBadArgumentsAreInvalidParams() { + val p = protocol(); val s = session(p) + assertEquals(McpProtocol.INVALID_PARAMS, call(p, s, "nope").getJSONObject("error").getInt("code")) + val bad = request(3, "tools/call", JSONObject().put("name", "echo").put("arguments", JSONArray())).toString() + assertEquals(McpProtocol.INVALID_PARAMS, JSONObject(p.post(bad, s, null).body!!).getJSONObject("error").getInt("code")) + } + + @Test fun unknownMethodIsMethodNotFound() { + val p = protocol(); val s = session(p) + val r = JSONObject(p.post(request(4, "resources/list").toString(), s, null).body!!) + assertEquals(McpProtocol.METHOD_NOT_FOUND, r.getJSONObject("error").getInt("code")) + } + + @Test fun badJsonIsAParseError() { + val r = protocol().post("{not json", null, null) + assertEquals(400, r.status) + assertEquals(McpProtocol.PARSE_ERROR, JSONObject(r.body!!).getJSONObject("error").getInt("code")) + } + + @Test fun batchGetsAnArrayOfReplies() { + val p = protocol(); val s = session(p) + val batch = JSONArray().put(request(1, "ping")).put(JSONObject().put("jsonrpc", "2.0").put("method", "notifications/x")).put(request(2, "tools/list")) + val r = p.post(batch.toString(), s, null) + assertEquals(200, r.status) + val replies = JSONArray(r.body!!) + assertEquals(2, replies.length()) + assertEquals(1, replies.getJSONObject(0).getInt("id")) + assertEquals(2, replies.getJSONObject(1).getInt("id")) + } + + @Test fun initializeMustBeAlone() { + val batch = JSONArray().put(JSONObject(initBody())).put(request(2, "ping")) + assertEquals(400, protocol().post(batch.toString(), null, null).status) + } + + @Test fun deleteEndsTheSession() { + val p = protocol(); val s = session(p) + assertEquals(204, p.delete(s).status) + assertEquals(404, p.delete(s).status) + assertEquals(404, p.post(request(1, "ping").toString(), s, null).status) + } + + @Test fun oldestSessionIsDroppedBeyondTheLimit() { + val p = protocol(maxSessions = 2) + val a = session(p); val b = session(p) + p.post(request(1, "ping").toString(), a, null) // a is now the most recently used + val c = session(p) // drops b + assertEquals(2, p.sessionCount()) + assertEquals(200, p.post(request(1, "ping").toString(), a, null).status) + assertEquals(404, p.post(request(1, "ping").toString(), b, null).status) + assertEquals(200, p.post(request(1, "ping").toString(), c, null).status) + } + + @Test fun endAllEndsEverySession() { + val p = protocol(); val s = session(p) + p.endAll() + assertEquals(0, p.sessionCount()) + assertEquals(404, p.post(request(1, "ping").toString(), s, null).status) + } + + // ---------------------------------------------------------------- the HTTP side + + private val token = "secret-token" + private var server: McpHttp? = null + + @After fun stopServer() { server?.stop(); server = null } + + private fun startServer(): Int { + val s = McpHttp(protocol(), { token }, port = 0) + s.start() + server = s + return s.boundPort + } + + private class Answer(val status: Int, val headers: Map, val body: String) + + /** One request on its own connection; [headers] may replace Host. */ + private fun http(port: Int, method: String, path: String = McpHttp.PATH, headers: Map = emptyMap(), body: String? = null): Answer { + Socket("127.0.0.1", port).use { s -> + s.soTimeout = 5000 + val bytes = body?.toByteArray(Charsets.UTF_8) ?: ByteArray(0) + val all = linkedMapOf("Host" to "127.0.0.1:$port", "Connection" to "close") + all.putAll(headers) + if (body != null) { all["Content-Type"] = "application/json"; all["Content-Length"] = bytes.size.toString() } + val head = StringBuilder("$method $path HTTP/1.1\r\n") + for ((k, v) in all) head.append(k).append(": ").append(v).append("\r\n") + head.append("\r\n") + s.getOutputStream().apply { write(head.toString().toByteArray(Charsets.ISO_8859_1)); write(bytes); flush() } + val text = s.getInputStream().readBytes().toString(Charsets.UTF_8) + val lines = text.substringBefore("\r\n\r\n").split("\r\n") + val status = lines[0].split(' ')[1].toInt() + val h = lines.drop(1).associate { it.substringBefore(':').trim().lowercase() to it.substringAfter(':').trim() } + return Answer(status, h, text.substringAfter("\r\n\r\n", "")) + } + } + + private val auth get() = mapOf("Authorization" to "Bearer $token") + + @Test fun tokenIs32BytesUrlSafeAndNew() { + val a = McpHttp.newToken() + assertEquals(43, a.length) + assertTrue(a.matches(Regex("[A-Za-z0-9_-]{43}"))) + assertTrue(a != McpHttp.newToken()) + } + + @Test fun base64MatchesTheJdkEncoder() { + for (n in 0..40) { + val bytes = ByteArray(n) { (it * 37 + 250).toByte() } + assertEquals(java.util.Base64.getEncoder().encodeToString(bytes), McpHttp.base64(bytes, urlSafe = false, pad = true)) + assertEquals(java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(bytes), McpHttp.base64(bytes, urlSafe = true, pad = false)) + } + } + + @Test fun httpPortZeroBindsAFreePortOnLoopback() { + val port = startServer() + assertTrue(port in 1..65535) + assertEquals("127.0.0.1", java.net.Socket("127.0.0.1", port).use { it.inetAddress.hostAddress }) + } + + @Test fun httpNinthConnectionIsRefused() { + val port = startServer() + val held = (1..McpHttp.MAX_CONNECTIONS).map { Socket("127.0.0.1", port) } + try { + Thread.sleep(300) + Socket("127.0.0.1", port).use { s -> + s.soTimeout = 5000 + val text = s.getInputStream().readBytes().toString(Charsets.UTF_8) + assertTrue(text.startsWith("HTTP/1.1 503")) + } + } finally { held.forEach { it.close() } } + } + + @Test fun httpNeedsTheToken() { + val port = startServer() + val none = http(port, "POST", body = initBody()) + assertEquals(401, none.status) + assertNotNull(none.headers["www-authenticate"]) + assertEquals(401, http(port, "POST", headers = mapOf("Authorization" to "Bearer wrong"), body = initBody()).status) + assertFalse("the token never appears in a refusal", none.body.contains(token)) + } + + @Test fun httpInitializeWithTheToken() { + val port = startServer() + val r = http(port, "POST", headers = auth, body = initBody()) + assertEquals(200, r.status) + assertNotNull(r.headers["mcp-session-id"]) + assertEquals("2025-06-18", JSONObject(r.body).getJSONObject("result").getString("protocolVersion")) + } + + @Test fun httpLocalhostHostIsAccepted() { + val port = startServer() + assertEquals(200, http(port, "POST", headers = auth + ("Host" to "localhost:$port"), body = initBody()).status) + } + + @Test fun httpOtherHostIsRefused() { + val port = startServer() + assertEquals(403, http(port, "POST", headers = auth + ("Host" to "evil.example"), body = initBody()).status) + assertEquals(403, http(port, "POST", headers = auth + ("Host" to "127.0.0.1:x"), body = initBody()).status) + } + + @Test fun httpWebPageOriginIsRefused() { + val port = startServer() + assertEquals(403, http(port, "POST", headers = auth + ("Origin" to "https://evil.example"), body = initBody()).status) + assertEquals(200, http(port, "POST", headers = auth + ("Origin" to "http://localhost:3000"), body = initBody()).status) + } + + @Test fun httpGetIsNotAllowed() { + val port = startServer() + val r = http(port, "GET", headers = auth) + assertEquals(405, r.status) + assertEquals("POST, DELETE", r.headers["allow"]) + } + + @Test fun httpOtherPathIsNotFound() { + assertEquals(404, http(startServer(), "POST", path = "/other", headers = auth, body = initBody()).status) + } + + @Test fun httpWrongContentTypeIsRefused() { + val port = startServer() + Socket("127.0.0.1", port).use { s -> + val body = initBody().toByteArray() + s.getOutputStream().write(("POST /mcp HTTP/1.1\r\nHost: 127.0.0.1\r\nAuthorization: Bearer $token\r\nConnection: close\r\n" + + "Content-Type: text/plain\r\nContent-Length: ${body.size}\r\n\r\n").toByteArray() + body) + val text = s.getInputStream().readBytes().toString(Charsets.UTF_8) + assertTrue(text.startsWith("HTTP/1.1 415")) + } + } + + @Test fun httpSessionThenDelete() { + val port = startServer() + val s = http(port, "POST", headers = auth, body = initBody()).headers["mcp-session-id"]!! + val ping = request(1, "ping").toString() + assertEquals(200, http(port, "POST", headers = auth + ("Mcp-Session-Id" to s), body = ping).status) + assertEquals(204, http(port, "DELETE", headers = auth + ("Mcp-Session-Id" to s)).status) + assertEquals(404, http(port, "POST", headers = auth + ("Mcp-Session-Id" to s), body = ping).status) + } } From acddbd2586e2f1e91a2a240254af262df500055d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 04:54:59 +0000 Subject: [PATCH 05/24] ADFA-7135 Implement the SQLite debugger extension and its host tests sqlitedbg.c implements sqlitedbg.h as a loadable-extension client. Per-connection state is freed only from the xDestroy of sqlitedbg_connection_id; the trace CLOSE event only marks it closing. Events live in a ring of records in their exported layout. Server connections are recognised by a thread-local flag set around sqlitedbg_open_server's own open. test.sh builds SQLite 3.49.1 from the amalgamation (pinned) and runs sqlitedbg_test.c under ASan+UBSan and under TSan. The same run writes or checks the fixture the EventDecoder JVM test reads. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- .../src/main/cpp/CMakeLists.txt | 17 + .../sqlite-debugger/src/main/cpp/sqlitedbg.c | 419 +++++++++ .../src/main/cpp/sqlitedbg.map | 6 + .../src/test/c/sqlitedbg_test.c | 841 ++++++++++++++++++ .../sqlitedebugger/engine/events.bin | Bin 0 -> 1792 bytes .../sqlitedebugger/engine/events.tsv | 21 + subprojects/sqlite-debugger/test.sh | 67 ++ 7 files changed, 1371 insertions(+) create mode 100644 subprojects/sqlite-debugger/src/main/cpp/CMakeLists.txt create mode 100644 subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c create mode 100644 subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.map create mode 100644 subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c create mode 100644 subprojects/sqlite-debugger/src/test/resources/org/appdevforall/sqlitedebugger/engine/events.bin create mode 100644 subprojects/sqlite-debugger/src/test/resources/org/appdevforall/sqlitedebugger/engine/events.tsv create mode 100755 subprojects/sqlite-debugger/test.sh diff --git a/subprojects/sqlite-debugger/src/main/cpp/CMakeLists.txt b/subprojects/sqlite-debugger/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000000..bbe9393af9 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/cpp/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 3.22.1) + +project(sqlitedbg C) + +# Only the headers: the extension runs inside requery's libsqlite3x.so and reaches SQLite through +# the api table it is given (sqlite3ext.h), so no SQLite is compiled or linked here. +set(SQLITE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../../../../sqlite-js/upstream/libs) + +add_library(sqlitedbg SHARED + sqlitedbg.c) + +target_include_directories(sqlitedbg PRIVATE ${SQLITE_HEADERS} ${CMAKE_CURRENT_SOURCE_DIR}) +target_compile_options(sqlitedbg PRIVATE -Wall -Wextra -fvisibility=hidden) +# Only JNI_OnLoad is exported, so sqlite3_api and our other symbols cannot bind to another library's. +target_link_options(sqlitedbg PRIVATE + -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/sqlitedbg.map + -Wl,-z,max-page-size=16384) diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c new file mode 100644 index 0000000000..1442df0990 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c @@ -0,0 +1,419 @@ +// The SQLite debugger's extension (ADFA-7135). The contract is sqlitedbg.h. +// +// Built as a loadable-extension client (sqlite3ext.h): on a device every sqlite3_ call goes +// through the api table SQLite hands to the first sqlitedbg_extension_init, because libsqlitedbg.so +// links no SQLite of its own. Host tests define SQLITE_CORE, where the macros call SQLite directly. + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include +#include + +#include "sqlite3ext.h" +#include "sqlitedbg.h" + +SQLITE_EXTENSION_INIT1 + +#define DEFAULT_BUFFER_BYTES ((size_t)4 << 20) +#define DEFAULT_MAX_SQL_BYTES 4096u +#define RECORD_HEADER_BYTES 56u + +_Static_assert(sizeof(sqlitedbg_event_record) == RECORD_HEADER_BYTES, "event record layout"); + +// Padding record written where the next record would not fit before the end of the ring, so a +// record is always contiguous. kind 0 is never a real event. +#define KIND_PAD 0u + +typedef struct conn_state { + struct conn_state *prev, *next; + uint64_t id; + char *path; + int read_only; + int closing; + int64_t opened_at_ns; + int64_t open_thread_id; +} conn_state; + +// Taken only for the fields below; never while SQLite runs. +static pthread_mutex_t g_lock = PTHREAD_MUTEX_INITIALIZER; +// Serialises sqlitedbg_install so a second caller does not return before the first registered. +static pthread_mutex_t g_install_lock = PTHREAD_MUTEX_INITIALIZER; + +static int g_installed; // g_install_lock +static int g_api_ready; // g_lock: sqlite3_api has been saved +static uint32_t g_max_sql; // set before g_installed, then read-only +static int g_client_connected; // atomic + +static conn_state *g_first, *g_last; +static uint64_t g_next_conn_id = 1; + +static unsigned char *g_ring; +static size_t g_cap; +static size_t g_head; // where the next record goes +static size_t g_tail; // the oldest record; never a padding record while g_used > 0 +static size_t g_used; // bytes between tail and head, padding included +static uint64_t g_next_seq = 1; + +// Set around the server's own sqlite3_open_v2: auto extensions run on the opening thread, inside it. +static __thread int tls_opening_server; + +static int64_t now_ns(void) { + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + return (int64_t)ts.tv_sec * 1000000000LL + ts.tv_nsec; +} + +static int64_t this_thread_id(void) { +#ifdef __ANDROID__ + return (int64_t)gettid(); +#else + return (int64_t)syscall(SYS_gettid); +#endif +} + +static sqlitedbg_event_record *record_at(size_t offset) { + return (sqlitedbg_event_record *)(void *)(g_ring + offset); +} + +// Drops the oldest record, then any padding that is now the oldest. g_lock held. +static void evict_oldest(void) { + do { + uint32_t bytes = record_at(g_tail)->record_bytes; + g_used -= bytes; + g_tail += bytes; + if (g_tail >= g_cap) g_tail = 0; + } while (g_used > 0 && record_at(g_tail)->kind == KIND_PAD); +} + +// The seq of the oldest event still buffered, or the next one to be made if none is. g_lock held. +static uint64_t oldest_seq(void) { + return g_used > 0 ? record_at(g_tail)->seq : g_next_seq; +} + +// Makes room for one record of `need` bytes at g_head. g_lock held; need <= g_cap. +static void make_room(size_t need) { + for (;;) { + if (g_used == 0) { + g_head = g_tail = 0; + return; + } + if (g_head > g_tail) { + if (g_cap - g_head >= need) return; + if (g_tail >= need) { + sqlitedbg_event_record *pad = record_at(g_head); + pad->record_bytes = (uint32_t)(g_cap - g_head); + pad->kind = KIND_PAD; + g_used += g_cap - g_head; + g_head = 0; + return; + } + } else if (g_tail - g_head >= need) { + return; + } + evict_oldest(); + } +} + +// Longest prefix of sql, at most max bytes, that ends on a character boundary. +static size_t utf8_cut(const char *sql, size_t len, size_t max) { + if (len <= max) return len; + while (max > 0 && ((unsigned char)sql[max] & 0xC0) == 0x80) max--; + return max; +} + +static void add_event(uint32_t kind, uint64_t connection_id, int64_t duration_ns, uint32_t flags, + const char *sql, size_t sql_len) { + int64_t time = now_ns(); + int64_t tid = this_thread_id(); + size_t kept = utf8_cut(sql, sql_len, g_max_sql); + if (kept < sql_len) flags |= SQLITEDBG_EVENT_TRUNCATED; + size_t need = (RECORD_HEADER_BYTES + kept + 7) & ~(size_t)7; + + pthread_mutex_lock(&g_lock); + uint64_t seq = g_next_seq++; + if (need <= g_cap) { + make_room(need); + sqlitedbg_event_record *r = record_at(g_head); + r->record_bytes = (uint32_t)need; + r->kind = kind; + r->seq = seq; + r->connection_id = connection_id; + r->thread_id = tid; + r->time_ns = time; + r->duration_ns = duration_ns; + r->flags = flags; + r->sql_bytes = (uint32_t)kept; + memcpy((unsigned char *)r + RECORD_HEADER_BYTES, sql, kept); + memset((unsigned char *)r + RECORD_HEADER_BYTES + kept, 0, need - RECORD_HEADER_BYTES - kept); + g_head += need; + g_used += need; + if (g_head >= g_cap) g_head = 0; + } + pthread_mutex_unlock(&g_lock); +} + +// sqlite3_trace_v2 callback. C is the connection's conn_state, which lives until SQLite frees the +// connection, so it outlives every callback. +static int trace_cb(unsigned type, void *c, void *p, void *x) { + conn_state *cs = (conn_state *)c; + switch (type) { + case SQLITE_TRACE_STMT: + case SQLITE_TRACE_PROFILE: { + sqlite3_stmt *stmt = (sqlite3_stmt *)p; + const char *sql = NULL; + char *expanded = NULL; + uint32_t flags = 0; + int is_stmt = type == SQLITE_TRACE_STMT; + if (is_stmt) sql = (const char *)x; + else sql = sqlite3_sql(stmt); + if (!sql) sql = ""; + // A trigger's text starts with "--" and P is the outer statement, whose expansion would + // be the wrong text. + if (__atomic_load_n(&g_client_connected, __ATOMIC_RELAXED) && !(sql[0] == '-' && sql[1] == '-')) { + expanded = sqlite3_expanded_sql(stmt); + if (expanded) { + sql = expanded; + flags |= SQLITEDBG_EVENT_EXPANDED; + } + } + int64_t duration = is_stmt ? -1 : *(sqlite3_int64 *)x; + add_event(is_stmt ? SQLITEDBG_EVENT_STATEMENT : SQLITEDBG_EVENT_PROFILE, cs->id, duration, flags, sql, + strlen(sql)); + if (expanded) sqlite3_free(expanded); + break; + } + case SQLITE_TRACE_CLOSE: + // The close can still fail or leave a zombie, so only mark it: xDestroy frees. + pthread_mutex_lock(&g_lock); + cs->closing = 1; + pthread_mutex_unlock(&g_lock); + add_event(SQLITEDBG_EVENT_CLOSE, cs->id, -1, 0, "", 0); + break; + } + return 0; +} + +// SQLite calls this when it really frees the connection (or if registering the function failed). +static void conn_destroy(void *p) { + conn_state *cs = (conn_state *)p; + pthread_mutex_lock(&g_lock); + if (cs->prev) cs->prev->next = cs->next; + else g_first = cs->next; + if (cs->next) cs->next->prev = cs->prev; + else g_last = cs->prev; + pthread_mutex_unlock(&g_lock); + free(cs->path); + free(cs); +} + +static void connection_id_fn(sqlite3_context *ctx, int argc, sqlite3_value **argv) { + (void)argc; + (void)argv; + sqlite3_result_int64(ctx, (sqlite3_int64)((conn_state *)sqlite3_user_data(ctx))->id); +} + +int sqlitedbg_extension_init(sqlite3 *db, char **err, const sqlite3_api_routines *api) { + (void)err; + if (tls_opening_server) return SQLITE_OK; + + pthread_mutex_lock(&g_lock); + if (!g_api_ready) { + SQLITE_EXTENSION_INIT2(api); + g_api_ready = 1; + } + pthread_mutex_unlock(&g_lock); + + conn_state *cs = (conn_state *)calloc(1, sizeof *cs); + if (!cs) return SQLITE_NOMEM; + const char *filename = sqlite3_db_filename(db, "main"); + cs->path = strdup(filename ? filename : ""); + if (!cs->path) { + free(cs); + return SQLITE_NOMEM; + } + cs->read_only = sqlite3_db_readonly(db, "main") == 1; + cs->opened_at_ns = now_ns(); + cs->open_thread_id = this_thread_id(); + + pthread_mutex_lock(&g_lock); + cs->id = g_next_conn_id++; + cs->prev = g_last; + if (g_last) g_last->next = cs; + else g_first = cs; + g_last = cs; + pthread_mutex_unlock(&g_lock); + + // On failure SQLite has already called conn_destroy, so cs is gone. + int rc = sqlite3_create_function_v2(db, "sqlitedbg_connection_id", 0, SQLITE_UTF8 | SQLITE_INNOCUOUS, cs, + connection_id_fn, NULL, NULL, conn_destroy); + if (rc != SQLITE_OK) return rc; + return sqlite3_trace_v2(db, SQLITE_TRACE_STMT | SQLITE_TRACE_PROFILE | SQLITE_TRACE_CLOSE, trace_cb, cs); +} + +int sqlitedbg_install(sqlitedbg_auto_extension_fn auto_extension, const sqlitedbg_config *config) { + if (!auto_extension) return SQLITE_MISUSE; + pthread_mutex_lock(&g_install_lock); + if (g_installed) { + pthread_mutex_unlock(&g_install_lock); + return SQLITE_OK; + } + size_t cap = config && config->buffer_bytes ? config->buffer_bytes : DEFAULT_BUFFER_BYTES; + uint32_t max_sql = config && config->max_sql_bytes ? config->max_sql_bytes : DEFAULT_MAX_SQL_BYTES; + // Room for two of the largest records, or the ring could never hold one beside a pad. + size_t min_cap = 2 * ((RECORD_HEADER_BYTES + (size_t)max_sql + 7) & ~(size_t)7); + if (cap < min_cap) cap = min_cap; + cap &= ~(size_t)7; + unsigned char *ring = (unsigned char *)calloc(1, cap); + if (!ring) { + pthread_mutex_unlock(&g_install_lock); + return SQLITE_NOMEM; + } + pthread_mutex_lock(&g_lock); + g_ring = ring; + g_cap = cap; + g_head = g_tail = g_used = 0; + g_max_sql = max_sql; + pthread_mutex_unlock(&g_lock); + + int rc = auto_extension((void (*)(void))sqlitedbg_extension_init); + if (rc == SQLITE_OK) { + g_installed = 1; + } else { + pthread_mutex_lock(&g_lock); + g_ring = NULL; + g_cap = 0; + pthread_mutex_unlock(&g_lock); + free(ring); + } + pthread_mutex_unlock(&g_install_lock); + return rc; +} + +size_t sqlitedbg_each_connection(void (*fn)(void *context, const sqlitedbg_connection *connection), + void *context) { + size_t n = 0; + pthread_mutex_lock(&g_lock); + for (conn_state *cs = g_first; cs; cs = cs->next) { + sqlitedbg_connection c; + c.id = cs->id; + c.path = cs->path; + c.read_only = cs->read_only; + c.closing = cs->closing; + c.opened_at_ns = cs->opened_at_ns; + c.open_thread_id = cs->open_thread_id; + fn(context, &c); + n++; + } + pthread_mutex_unlock(&g_lock); + return n; +} + +void sqlitedbg_read_events(uint64_t after, void *out, size_t out_bytes, sqlitedbg_read_result *result) { + result->next = after; + result->dropped = 0; + result->bytes = 0; + result->count = 0; + pthread_mutex_lock(&g_lock); + if (g_ring) { + uint64_t oldest = oldest_seq(); + if (oldest > after && oldest - after > 1) result->dropped = oldest - after - 1; + size_t offset = g_tail, left = g_used; + while (left > 0) { + const sqlitedbg_event_record *r = record_at(offset); + uint32_t bytes = r->record_bytes; + if (r->kind != KIND_PAD && r->seq > after) { + if (result->bytes + bytes > out_bytes) break; + memcpy((unsigned char *)out + result->bytes, r, bytes); + result->bytes += bytes; + result->count++; + result->next = r->seq; + } + left -= bytes; + offset += bytes; + if (offset >= g_cap) offset = 0; + } + } + pthread_mutex_unlock(&g_lock); +} + +void sqlitedbg_set_client_connected(int connected) { + __atomic_store_n(&g_client_connected, connected ? 1 : 0, __ATOMIC_RELAXED); +} + +static int server_authorizer(void *unused, int action, const char *a1, const char *a2, const char *db, + const char *trigger) { + (void)unused; + (void)a2; + (void)db; + (void)trigger; + switch (action) { + case SQLITE_INSERT: + case SQLITE_UPDATE: + case SQLITE_DELETE: + case SQLITE_CREATE_INDEX: + case SQLITE_CREATE_TABLE: + case SQLITE_CREATE_TEMP_INDEX: + case SQLITE_CREATE_TEMP_TABLE: + case SQLITE_CREATE_TEMP_TRIGGER: + case SQLITE_CREATE_TEMP_VIEW: + case SQLITE_CREATE_TRIGGER: + case SQLITE_CREATE_VIEW: + case SQLITE_CREATE_VTABLE: + case SQLITE_DROP_INDEX: + case SQLITE_DROP_TABLE: + case SQLITE_DROP_TEMP_INDEX: + case SQLITE_DROP_TEMP_TABLE: + case SQLITE_DROP_TEMP_TRIGGER: + case SQLITE_DROP_TEMP_VIEW: + case SQLITE_DROP_TRIGGER: + case SQLITE_DROP_VIEW: + case SQLITE_DROP_VTABLE: + case SQLITE_ALTER_TABLE: + case SQLITE_REINDEX: + case SQLITE_ANALYZE: + case SQLITE_ATTACH: + case SQLITE_DETACH: + return SQLITE_DENY; + case SQLITE_PRAGMA: + // PRAGMA query_only=0 would lift the guard, and a PRAGMA that stores a value writes. + // PRAGMA table_info(t) also has a second argument, so only these names are checked. + if (a1 && (sqlite3_stricmp(a1, "query_only") == 0 || sqlite3_stricmp(a1, "writable_schema") == 0 || + sqlite3_stricmp(a1, "wal_checkpoint") == 0 || + sqlite3_stricmp(a1, "incremental_vacuum") == 0 || sqlite3_stricmp(a1, "optimize") == 0)) + return SQLITE_DENY; + if (a1 && a2 && + (sqlite3_stricmp(a1, "journal_mode") == 0 || sqlite3_stricmp(a1, "user_version") == 0 || + sqlite3_stricmp(a1, "application_id") == 0 || sqlite3_stricmp(a1, "schema_version") == 0)) + return SQLITE_DENY; + return SQLITE_OK; + default: + return SQLITE_OK; + } +} + +int sqlitedbg_open_server(const char *path, int writable, sqlite3 **db) { + *db = NULL; + pthread_mutex_lock(&g_lock); + int ready = g_api_ready; + pthread_mutex_unlock(&g_lock); + if (!ready) return SQLITE_MISUSE; + + tls_opening_server = 1; + int rc = sqlite3_open_v2(path, db, writable ? SQLITE_OPEN_READWRITE : SQLITE_OPEN_READONLY, NULL); + tls_opening_server = 0; + if (rc == SQLITE_OK && !writable) { + rc = sqlite3_exec(*db, "PRAGMA query_only=1", NULL, NULL, NULL); + if (rc == SQLITE_OK) rc = sqlite3_set_authorizer(*db, server_authorizer, NULL); + } + if (rc != SQLITE_OK && *db) { + sqlite3_close_v2(*db); + *db = NULL; + } + return rc; +} diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.map b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.map new file mode 100644 index 0000000000..b6518f1c1d --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.map @@ -0,0 +1,6 @@ +{ + global: + JNI_OnLoad; + local: + *; +}; diff --git a/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c b/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c new file mode 100644 index 0000000000..f2efe08d7a --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c @@ -0,0 +1,841 @@ +// Host tests of the SQLite debugger's extension (ADFA-7135), run by test.sh against SQLite built +// from the amalgamation as requery builds it (SQLITE_THREADSAFE=2). The extension keeps process +// state and installs once, so the tests share one install and run in order. +// +// Usage: sqlitedbg_test [--write-fixture BIN TSV | --check-fixture BIN TSV] +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sqlite3.h" +#include "sqlitedbg.h" + +static int g_failures; +static int g_checks; + +#define CHECK(cond) \ + do { \ + g_checks++; \ + if (!(cond)) { \ + g_failures++; \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + } \ + } while (0) + +#define CHECK_EQ(a, b) \ + do { \ + long long va_ = (long long)(a), vb_ = (long long)(b); \ + g_checks++; \ + if (va_ != vb_) { \ + g_failures++; \ + fprintf(stderr, "FAIL %s:%d: %s == %s (%lld vs %lld)\n", __FILE__, __LINE__, \ + #a, #b, va_, vb_); \ + } \ + } while (0) + +#define MAX_SQL 64 +#define BUFFER_BYTES 8192 + +static char g_dir[256]; +static char g_db_path[300]; + +static long long tid(void) { return (long long)syscall(SYS_gettid); } + +static void exec_ok(sqlite3 *db, const char *sql) { + char *err = NULL; + int rc = sqlite3_exec(db, sql, NULL, NULL, &err); + if (rc != SQLITE_OK) fprintf(stderr, "exec(%s): %s\n", sql, err ? err : "?"); + sqlite3_free(err); + CHECK_EQ(rc, SQLITE_OK); +} + +static sqlite3 *open_app(const char *path) { + sqlite3 *db = NULL; + int rc = sqlite3_open(path, &db); + CHECK_EQ(rc, SQLITE_OK); + return db; +} + +// Events + +typedef struct ev { + uint32_t kind, flags; + uint64_t seq, conn; + int64_t tid, time_ns, duration_ns; + char sql[MAX_SQL * 4 + 8]; + uint32_t sql_bytes; +} ev; + +typedef struct batch { + ev *events; + size_t count; + uint64_t next, dropped; +} batch; + +static void batch_free(batch *b) { free(b->events); } + +// Reads everything after `after`, checking the layout of each record on the way. +static void read_all(uint64_t after, batch *b) { + size_t cap = 1 << 20; + unsigned char *buf = malloc(cap); + sqlitedbg_read_result r; + sqlitedbg_read_events(after, buf, cap, &r); + CHECK_EQ(r.bytes % 8, 0); + b->events = calloc(r.count + 1, sizeof(ev)); + b->count = r.count; + b->next = r.next; + b->dropped = r.dropped; + size_t off = 0; + uint64_t prev = after; + for (size_t i = 0; i < r.count; i++) { + const sqlitedbg_event_record *rec = (const sqlitedbg_event_record *)(void *)(buf + off); + CHECK(rec->record_bytes >= sizeof *rec); + CHECK_EQ(rec->record_bytes % 8, 0); + CHECK(rec->sql_bytes <= MAX_SQL); + CHECK(rec->record_bytes >= sizeof *rec + rec->sql_bytes); + CHECK(rec->seq > prev); + prev = rec->seq; + ev *e = &b->events[i]; + e->kind = rec->kind; + e->flags = rec->flags; + e->seq = rec->seq; + e->conn = rec->connection_id; + e->tid = rec->thread_id; + e->time_ns = rec->time_ns; + e->duration_ns = rec->duration_ns; + e->sql_bytes = rec->sql_bytes; + memcpy(e->sql, (const char *)buf + off + sizeof *rec, rec->sql_bytes); + e->sql[rec->sql_bytes] = 0; + off += rec->record_bytes; + } + CHECK_EQ(off, r.bytes); + free(buf); +} + +static uint64_t cursor_now(void) { + batch b; + read_all(0, &b); + uint64_t n = b.next; + batch_free(&b); + return n; +} + +static int utf8_valid(const char *s, size_t n) { + size_t i = 0; + while (i < n) { + unsigned char c = (unsigned char)s[i]; + size_t len = c < 0x80 ? 1 : (c >> 5) == 6 ? 2 : (c >> 4) == 14 ? 3 : (c >> 3) == 30 ? 4 : 0; + if (!len || i + len > n) return 0; + for (size_t k = 1; k < len; k++) + if (((unsigned char)s[i + k] & 0xC0) != 0x80) return 0; + i += len; + } + return 1; +} + +// Connections + +typedef struct listing { + sqlitedbg_connection c[64]; + char path[64][300]; + size_t n; +} listing; + +static void list_cb(void *context, const sqlitedbg_connection *c) { + listing *l = context; + if (l->n < 64) { + l->c[l->n] = *c; + snprintf(l->path[l->n], sizeof l->path[l->n], "%s", c->path); + l->c[l->n].path = l->path[l->n]; + } + l->n++; +} + +static size_t list_conns(listing *l) { + l->n = 0; + return sqlitedbg_each_connection(list_cb, l); +} + +static const sqlitedbg_connection *find_conn(const listing *l, uint64_t id) { + for (size_t i = 0; i < l->n && i < 64; i++) + if (l->c[i].id == id) return &l->c[i]; + return NULL; +} + +static uint64_t conn_id_of(sqlite3 *db) { + sqlite3_stmt *st; + uint64_t id = 0; + CHECK_EQ(sqlite3_prepare_v2(db, "SELECT sqlitedbg_connection_id()", -1, &st, NULL), SQLITE_OK); + if (sqlite3_step(st) == SQLITE_ROW) id = (uint64_t)sqlite3_column_int64(st, 0); + sqlite3_finalize(st); + return id; +} + +// Tests + +static sqlitedbg_config g_config = {BUFFER_BYTES, MAX_SQL}; +static sqlite3 *g_main; + +static void test_install(void) { + sqlite3 *srv = NULL; + CHECK_EQ(sqlitedbg_open_server(g_db_path, 0, &srv), SQLITE_MISUSE); + CHECK(srv == NULL); + CHECK_EQ(sqlitedbg_install(sqlite3_auto_extension, &g_config), SQLITE_OK); + sqlitedbg_config other = {1 << 20, 1000}; + CHECK_EQ(sqlitedbg_install(sqlite3_auto_extension, &other), SQLITE_OK); + CHECK_EQ(sqlitedbg_install(NULL, NULL), SQLITE_MISUSE); + // still no app connection: the extension has no SQLite API to call + CHECK_EQ(sqlitedbg_open_server(g_db_path, 0, &srv), SQLITE_MISUSE); + + listing l; + CHECK_EQ(list_conns(&l), 0); + g_main = open_app(g_db_path); + // a second install registered the entry point once: one connection, one listing, one id + CHECK_EQ(list_conns(&l), 1); + CHECK_EQ(l.c[0].id, 1); + CHECK(strcmp(l.c[0].path, g_db_path) == 0 || strstr(l.c[0].path, "app.db") != NULL); + CHECK_EQ(l.c[0].read_only, 0); + CHECK_EQ(l.c[0].closing, 0); + CHECK_EQ(l.c[0].open_thread_id, tid()); + CHECK(l.c[0].opened_at_ns > 0); + CHECK_EQ(conn_id_of(g_main), 1); + // the second config was ignored: the 64-byte limit still applies + uint64_t cur = cursor_now(); + char sql[200]; + snprintf(sql, sizeof sql, "SELECT '%0100d'", 0); + exec_ok(g_main, sql); + batch b; + read_all(cur, &b); + CHECK_EQ(b.count, 2); + CHECK(b.events[0].flags & SQLITEDBG_EVENT_TRUNCATED); + CHECK_EQ(b.events[0].sql_bytes, MAX_SQL); + batch_free(&b); +} + +static void test_events(void) { + exec_ok(g_main, "CREATE TABLE t(a)"); + exec_ok(g_main, "CREATE TABLE log(a)"); + exec_ok(g_main, "CREATE TRIGGER trg AFTER INSERT ON t BEGIN INSERT INTO log VALUES(new.a); END"); + uint64_t cur = cursor_now(); + + sqlite3_stmt *st; + CHECK_EQ(sqlite3_prepare_v2(g_main, "INSERT INTO t VALUES(?)", -1, &st, NULL), SQLITE_OK); + sqlite3_bind_int(st, 1, 42); + CHECK_EQ(sqlite3_step(st), SQLITE_DONE); + sqlite3_finalize(st); + + batch b; + read_all(cur, &b); + // the insert, the trigger it fires (its header, then its one step), then the insert's profile + CHECK_EQ(b.count, 4); + CHECK_EQ(b.events[0].kind, SQLITEDBG_EVENT_STATEMENT); + CHECK(strcmp(b.events[0].sql, "INSERT INTO t VALUES(?)") == 0); + CHECK_EQ(b.events[0].duration_ns, -1); + CHECK_EQ(b.events[0].flags, 0); + CHECK_EQ(b.events[1].kind, SQLITEDBG_EVENT_STATEMENT); + CHECK(strcmp(b.events[1].sql, "-- TRIGGER trg") == 0); + CHECK_EQ(b.events[2].kind, SQLITEDBG_EVENT_STATEMENT); + CHECK(strcmp(b.events[2].sql, "-- INSERT INTO log VALUES(new.a)") == 0); + CHECK_EQ(b.events[3].kind, SQLITEDBG_EVENT_PROFILE); + CHECK(strcmp(b.events[3].sql, "INSERT INTO t VALUES(?)") == 0); + CHECK(b.events[3].duration_ns >= 0); + for (size_t i = 0; i < b.count; i++) { + CHECK_EQ(b.events[i].conn, 1); + CHECK_EQ(b.events[i].tid, tid()); + CHECK_EQ(b.events[i].seq, cur + 1 + i); + CHECK(b.events[i].time_ns > 0); + } + CHECK_EQ(b.next, cur + 4); + CHECK_EQ(b.dropped, 0); + batch_free(&b); + + // nothing new: the cursor stays + read_all(cur + 4, &b); + CHECK_EQ(b.count, 0); + CHECK_EQ(b.next, cur + 4); + batch_free(&b); + + // one buffer too small for a record returns none and does not split one + sqlitedbg_read_result r; + char tiny[8]; + sqlitedbg_read_events(cur, tiny, sizeof tiny, &r); + CHECK_EQ(r.count, 0); + CHECK_EQ(r.bytes, 0); + CHECK_EQ(r.next, cur); +} + +static void test_expanded(void) { + sqlite3_stmt *st; + uint64_t cur = cursor_now(); + sqlitedbg_set_client_connected(1); + CHECK_EQ(sqlite3_prepare_v2(g_main, "INSERT INTO t VALUES(?)", -1, &st, NULL), SQLITE_OK); + sqlite3_bind_int(st, 1, 7); + CHECK_EQ(sqlite3_step(st), SQLITE_DONE); + sqlite3_finalize(st); + sqlitedbg_set_client_connected(0); + batch b; + read_all(cur, &b); + CHECK_EQ(b.count, 4); + CHECK(strcmp(b.events[0].sql, "INSERT INTO t VALUES(7)") == 0); + CHECK(b.events[0].flags & SQLITEDBG_EVENT_EXPANDED); + // a trigger's text is not expanded: P is the outer statement + CHECK(strncmp(b.events[1].sql, "--", 2) == 0); + CHECK(strncmp(b.events[2].sql, "--", 2) == 0); + CHECK_EQ(b.events[1].flags & SQLITEDBG_EVENT_EXPANDED, 0); + CHECK_EQ(b.events[2].flags & SQLITEDBG_EVENT_EXPANDED, 0); + CHECK(strcmp(b.events[3].sql, "INSERT INTO t VALUES(7)") == 0); + CHECK(b.events[3].flags & SQLITEDBG_EVENT_EXPANDED); + batch_free(&b); + + // after the client leaves, the bound value is not written in again + cur = cursor_now(); + CHECK_EQ(sqlite3_prepare_v2(g_main, "INSERT INTO t VALUES(?)", -1, &st, NULL), SQLITE_OK); + sqlite3_bind_int(st, 1, 8); + CHECK_EQ(sqlite3_step(st), SQLITE_DONE); + sqlite3_finalize(st); + read_all(cur, &b); + CHECK(strcmp(b.events[0].sql, "INSERT INTO t VALUES(?)") == 0); + CHECK_EQ(b.events[0].flags, 0); + batch_free(&b); + + // A long bound value does not outgrow the limit: expansion is cut like any other text + char big[300]; + memset(big, 'x', sizeof big - 1); + big[sizeof big - 1] = 0; + cur = cursor_now(); + sqlitedbg_set_client_connected(1); + CHECK_EQ(sqlite3_prepare_v2(g_main, "INSERT INTO t VALUES(?)", -1, &st, NULL), SQLITE_OK); + sqlite3_bind_text(st, 1, big, -1, SQLITE_STATIC); + CHECK_EQ(sqlite3_step(st), SQLITE_DONE); + sqlite3_finalize(st); + sqlitedbg_set_client_connected(0); + read_all(cur, &b); + CHECK(b.events[0].flags & SQLITEDBG_EVENT_EXPANDED); + CHECK(b.events[0].flags & SQLITEDBG_EVENT_TRUNCATED); + CHECK_EQ(b.events[0].sql_bytes, MAX_SQL); + batch_free(&b); +} + +static void test_truncation(void) { + char sql[256]; + batch b; + uint64_t cur; + + // "SELECT '" is 8 bytes; each euro sign is 3, so byte 64 is the third byte of one + // that starts at 62: the cut must fall back to 62 + size_t n = (size_t)snprintf(sql, sizeof sql, "SELECT '"); + for (int i = 0; i < 25; i++) n += (size_t)snprintf(sql + n, sizeof sql - n, "\xE2\x82\xAC"); + snprintf(sql + n, sizeof sql - n, "'"); + cur = cursor_now(); + exec_ok(g_main, sql); + read_all(cur, &b); + CHECK_EQ(b.count, 2); + for (int i = 0; i < 2; i++) { + CHECK_EQ(b.events[i].sql_bytes, 62); + CHECK(b.events[i].flags & SQLITEDBG_EVENT_TRUNCATED); + CHECK(utf8_valid(b.events[i].sql, b.events[i].sql_bytes)); + } + batch_free(&b); + + // a 4-byte character straddling the limit: 11 bytes of prefix, then 4-byte characters, + // so byte 64 is the second byte of one that starts at 63 + n = (size_t)snprintf(sql, sizeof sql, "SELECT 1 /*"); + for (int i = 0; i < 20; i++) n += (size_t)snprintf(sql + n, sizeof sql - n, "\xF0\x9F\x98\x80"); + snprintf(sql + n, sizeof sql - n, "*/"); + cur = cursor_now(); + exec_ok(g_main, sql); + read_all(cur, &b); + CHECK_EQ(b.count, 2); + CHECK_EQ(b.events[0].sql_bytes, 63); + CHECK(utf8_valid(b.events[0].sql, b.events[0].sql_bytes)); + batch_free(&b); + + // exactly at the limit is whole, one byte over is cut + for (int over = 0; over < 2; over++) { + memset(sql, 0, sizeof sql); + n = (size_t)snprintf(sql, sizeof sql, "SELECT 1 /*"); + while (n < (size_t)(MAX_SQL - 2 + over)) sql[n++] = 'x'; + sql[n++] = '*'; + sql[n++] = '/'; + sql[n] = 0; + CHECK_EQ(strlen(sql), MAX_SQL + over); + cur = cursor_now(); + exec_ok(g_main, sql); + read_all(cur, &b); + CHECK_EQ(b.events[0].sql_bytes, MAX_SQL); + CHECK_EQ((b.events[0].flags & SQLITEDBG_EVENT_TRUNCATED) != 0, over); + batch_free(&b); + } +} + +static void test_burst(void) { + uint64_t cur = cursor_now(); + const int stmts = 2000; + for (int i = 0; i < stmts; i++) exec_ok(g_main, "SELECT 1"); + const uint64_t made = 2 * (uint64_t)stmts; + batch b; + read_all(cur, &b); + CHECK(b.count > 0); + CHECK(b.count < made); + CHECK_EQ(b.dropped, made - b.count); + CHECK_EQ(b.next, cur + made); + CHECK_EQ(b.events[0].seq, cur + made - b.count + 1); + CHECK_EQ(b.events[b.count - 1].seq, cur + made); + for (size_t i = 1; i < b.count; i++) CHECK_EQ(b.events[i].seq, b.events[i - 1].seq + 1); + batch_free(&b); + + // nothing is reported twice + uint64_t next = cur + made; + read_all(next, &b); + CHECK_EQ(b.count, 0); + CHECK_EQ(b.dropped, 0); + batch_free(&b); + + // a cursor from before the start counts everything that left + read_all(0, &b); + CHECK_EQ(b.dropped, b.events[0].seq - 1); + CHECK_EQ(b.dropped + b.count, next); + batch_free(&b); + + // a reader that kept up loses nothing: read after every 10 statements + uint64_t pos = next, lost = 0, got = 0; + for (int round = 0; round < 50; round++) { + for (int i = 0; i < 10; i++) exec_ok(g_main, "SELECT 2"); + read_all(pos, &b); + lost += b.dropped; + got += b.count; + pos = b.next; + batch_free(&b); + } + CHECK_EQ(lost, 0); + CHECK_EQ(got, 50 * 10 * 2); +} + +static void test_close_and_list(void) { + listing l; + size_t before = list_conns(&l); + sqlite3 *c = open_app(g_db_path); + uint64_t id = conn_id_of(c); + CHECK(id > 1); + CHECK_EQ(list_conns(&l), before + 1); + CHECK(find_conn(&l, id) != NULL); + CHECK_EQ(find_conn(&l, id)->closing, 0); + + sqlite3_stmt *st; + CHECK_EQ(sqlite3_prepare_v2(c, "SELECT * FROM t", -1, &st, NULL), SQLITE_OK); + CHECK_EQ(sqlite3_step(st), SQLITE_ROW); + uint64_t cur = cursor_now(); + CHECK_EQ(sqlite3_close(c), SQLITE_BUSY); + // the close event fired, but the connection is still there: listed, marked closing + CHECK_EQ(list_conns(&l), before + 1); + CHECK_EQ(find_conn(&l, id)->closing, 1); + batch b; + read_all(cur, &b); + CHECK_EQ(b.count, 1); + CHECK_EQ(b.events[0].kind, SQLITEDBG_EVENT_CLOSE); + CHECK_EQ(b.events[0].conn, id); + CHECK_EQ(b.events[0].sql_bytes, 0); + batch_free(&b); + + // the app retries once the statement is gone + sqlite3_finalize(st); + CHECK_EQ(list_conns(&l), before + 1); + CHECK_EQ(sqlite3_close(c), SQLITE_OK); + CHECK_EQ(list_conns(&l), before); + CHECK(find_conn(&l, id) == NULL); + + // close_v2 with a statement open leaves a zombie, listed until it is finalized + c = open_app(g_db_path); + uint64_t id2 = conn_id_of(c); + CHECK(id2 > id); + CHECK_EQ(sqlite3_prepare_v2(c, "SELECT * FROM t", -1, &st, NULL), SQLITE_OK); + CHECK_EQ(sqlite3_close_v2(c), SQLITE_OK); + CHECK_EQ(list_conns(&l), before + 1); + CHECK_EQ(find_conn(&l, id2)->closing, 1); + sqlite3_finalize(st); + CHECK_EQ(list_conns(&l), before); + + // ids are never reused + c = open_app(g_db_path); + CHECK(conn_id_of(c) > id2); + CHECK_EQ(sqlite3_close(c), SQLITE_OK); + + // an in-memory database lists with an empty path, a read-only one as read-only + c = open_app(":memory:"); + list_conns(&l); + CHECK(find_conn(&l, conn_id_of(c)) != NULL); + CHECK_EQ(find_conn(&l, conn_id_of(c))->path[0], 0); + CHECK_EQ(sqlite3_close(c), SQLITE_OK); + sqlite3 *ro = NULL; + CHECK_EQ(sqlite3_open_v2(g_db_path, &ro, SQLITE_OPEN_READONLY, NULL), SQLITE_OK); + list_conns(&l); + CHECK_EQ(find_conn(&l, conn_id_of(ro))->read_only, 1); + CHECK_EQ(sqlite3_close(ro), SQLITE_OK); + CHECK_EQ(list_conns(&l), before); +} + +static int step_all(sqlite3 *db, const char *sql) { + sqlite3_stmt *st; + int rc = sqlite3_prepare_v2(db, sql, -1, &st, NULL); + if (rc != SQLITE_OK) return rc; + while ((rc = sqlite3_step(st)) == SQLITE_ROW) {} + sqlite3_finalize(st); + return rc; +} + +static void test_server(void) { + listing l; + size_t before = list_conns(&l); + uint64_t cur = cursor_now(); + sqlite3 *srv = NULL; + + CHECK_EQ(sqlitedbg_open_server(g_db_path, 0, &srv), SQLITE_OK); + CHECK(srv != NULL); + CHECK_EQ(sqlite3_db_readonly(srv, "main"), 1); + // not listed, and it has no connection id function + CHECK_EQ(list_conns(&l), before); + CHECK(sqlite3_prepare_v2(srv, "SELECT sqlitedbg_connection_id()", -1, &(sqlite3_stmt *){NULL}, NULL) != SQLITE_OK); + + CHECK_EQ(step_all(srv, "SELECT 'server-marker', count(*) FROM t"), SQLITE_DONE); + CHECK_EQ(step_all(srv, "SELECT name FROM sqlite_schema"), SQLITE_DONE); + CHECK_EQ(step_all(srv, "PRAGMA table_info(t)"), SQLITE_DONE); + CHECK_EQ(step_all(srv, "PRAGMA quick_check"), SQLITE_DONE); + // not traced + batch b; + read_all(cur, &b); + CHECK_EQ(b.count, 0); + batch_free(&b); + + // writes are refused, and so are the ways to lift the guard + CHECK_EQ(step_all(srv, "INSERT INTO t VALUES(99)"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "UPDATE t SET a=1"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "DELETE FROM t"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "CREATE TABLE x(a)"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "DROP TABLE t"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "PRAGMA query_only=0"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "PRAGMA user_version=5"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "ATTACH ':memory:' AS m"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "SELECT 'still works'"), SQLITE_DONE); + sqlite3_stmt *st; + CHECK_EQ(sqlite3_prepare_v2(srv, "SELECT count(*) FROM t WHERE a=99", -1, &st, NULL), SQLITE_OK); + CHECK_EQ(sqlite3_step(st), SQLITE_ROW); + CHECK_EQ(sqlite3_column_int(st, 0), 0); + sqlite3_finalize(st); + CHECK_EQ(sqlite3_close_v2(srv), SQLITE_OK); + CHECK_EQ(list_conns(&l), before); + + // a writable server connection writes, to what the app sees, and is still unlisted + CHECK_EQ(sqlitedbg_open_server(g_db_path, 1, &srv), SQLITE_OK); + CHECK_EQ(sqlite3_db_readonly(srv, "main"), 0); + CHECK_EQ(step_all(srv, "INSERT INTO t VALUES(1234)"), SQLITE_DONE); + CHECK_EQ(list_conns(&l), before); + CHECK_EQ(sqlite3_close_v2(srv), SQLITE_OK); + CHECK_EQ(sqlite3_prepare_v2(g_main, "SELECT count(*) FROM t WHERE a=1234", -1, &st, NULL), SQLITE_OK); + CHECK_EQ(sqlite3_step(st), SQLITE_ROW); + CHECK_EQ(sqlite3_column_int(st, 0), 1); + sqlite3_finalize(st); + + // a missing file is not created by a read-only open, and nothing leaks + char missing[320]; + snprintf(missing, sizeof missing, "%s/missing.db", g_dir); + CHECK_EQ(sqlitedbg_open_server(missing, 0, &srv), SQLITE_CANTOPEN); + CHECK(srv == NULL); + CHECK_EQ(sqlitedbg_open_server(missing, 1, &srv), SQLITE_CANTOPEN); + CHECK(srv == NULL); + + // the flag is per thread and per open: an app connection opened after is listed + sqlite3 *app = open_app(":memory:"); + CHECK_EQ(list_conns(&l), before + 1); + CHECK_EQ(sqlite3_close(app), SQLITE_OK); +} + +typedef struct worker { + int index; + atomic_int *stop; + long long stmts, closes; + int failed; +} worker; + +static void *worker_main(void *arg) { + worker *w = arg; + sqlite3 *db = NULL; + if (sqlite3_open(":memory:", &db) != SQLITE_OK) { + w->failed = 1; + return NULL; + } + sqlite3_stmt *st; + sqlite3_exec(db, "CREATE TABLE w(a, b)", NULL, NULL, NULL); + w->stmts++; + long long n = 0; + while (!atomic_load(w->stop)) { + if (sqlite3_prepare_v2(db, "INSERT INTO w VALUES(?, ?)", -1, &st, NULL) != SQLITE_OK) { + w->failed = 1; + break; + } + sqlite3_bind_int64(st, 1, n); + sqlite3_bind_text(st, 2, "value", -1, SQLITE_STATIC); + if (sqlite3_step(st) != SQLITE_DONE) w->failed = 1; + sqlite3_finalize(st); + w->stmts++; + if (n % 7 == 0) { + if (sqlite3_exec(db, "SELECT count(*) FROM w", NULL, NULL, NULL) != SQLITE_OK) w->failed = 1; + w->stmts++; + } + if (n % 50 == 0) { + // connection churn beside the other threads' statements + sqlite3 *extra = NULL; + if (sqlite3_open(":memory:", &extra) != SQLITE_OK) { + w->failed = 1; + break; + } + if (sqlite3_exec(extra, "SELECT 1", NULL, NULL, NULL) != SQLITE_OK) w->failed = 1; + w->stmts++; + if (sqlite3_close(extra) != SQLITE_OK) w->failed = 1; + w->closes++; + } + n++; + } + if (sqlite3_close(db) != SQLITE_OK) w->failed = 1; + w->closes++; + return NULL; +} + +typedef struct reader { + atomic_int *stop; + int failed; + long long reads; +} reader; + +static void list_noop(void *context, const sqlitedbg_connection *c) { + (void)context; + if (c->id == 0 || !c->path) *(int *)context = 1; +} + +static void *reader_main(void *arg) { + reader *r = arg; + static unsigned char buf[2048]; + uint64_t after = 0; + while (!atomic_load(r->stop)) { + sqlitedbg_read_result res; + sqlitedbg_read_events(after, buf, sizeof buf, &res); + size_t off = 0; + uint64_t prev = after; + for (size_t i = 0; i < res.count; i++) { + const sqlitedbg_event_record *rec = (const sqlitedbg_event_record *)(void *)(buf + off); + if (rec->seq <= prev || rec->record_bytes % 8 || rec->kind < 1 || rec->kind > 3 || + rec->record_bytes < sizeof *rec + rec->sql_bytes) + r->failed = 1; + prev = rec->seq; + off += rec->record_bytes; + } + if (off != res.bytes) r->failed = 1; + after = res.next; + int bad = 0; + sqlitedbg_each_connection(list_noop, &bad); + if (bad) r->failed = 1; + sqlitedbg_set_client_connected(r->reads++ % 3 == 0); + } + sqlitedbg_set_client_connected(0); + return NULL; +} + +static void test_threads(void) { + listing l; + size_t before = list_conns(&l); + uint64_t cur = cursor_now(); + enum { N = 8 }; + pthread_t th[N], rt; + worker w[N]; + reader rd; + atomic_int stop = 0; + memset(w, 0, sizeof w); + memset(&rd, 0, sizeof rd); + rd.stop = &stop; + pthread_create(&rt, NULL, reader_main, &rd); + for (int i = 0; i < N; i++) { + w[i].index = i; + w[i].stop = &stop; + pthread_create(&th[i], NULL, worker_main, &w[i]); + } + struct timespec two = {2, 0}; + nanosleep(&two, NULL); + atomic_store(&stop, 1); + for (int i = 0; i < N; i++) pthread_join(th[i], NULL); + pthread_join(rt, NULL); + + long long stmts = 0, closes = 0; + for (int i = 0; i < N; i++) { + CHECK_EQ(w[i].failed, 0); + stmts += w[i].stmts; + closes += w[i].closes; + } + CHECK_EQ(rd.failed, 0); + CHECK(stmts > 1000); + // every thread's connections are gone, and every event made was numbered + CHECK_EQ(list_conns(&l), before); + batch b; + read_all(0, &b); + CHECK_EQ(b.next - cur, 2 * (uint64_t)stmts + (uint64_t)closes); + batch_free(&b); + printf(" threads: %lld statements, %lld closes, %lld reads\n", stmts, closes, rd.reads); +} + +// Fixture for EventDecoder's JVM test: a known run, as bytes plus a table of what they hold. + +static void write_tsv_row(FILE *f, const sqlitedbg_event_record *r, const char *sql) { + fprintf(f, "%llu\t%u\t%llu\t%lld\t%lld\t%lld\t%u\t%.*s\n", (unsigned long long)r->seq, r->kind, + (unsigned long long)r->connection_id, (long long)r->thread_id, (long long)r->time_ns, + (long long)r->duration_ns, r->flags, (int)r->sql_bytes, sql); +} + +static void run_fixture_scenario(void) { + char sql[256]; + exec_ok(g_main, "CREATE TABLE fx(a)"); + sqlite3_stmt *st; + CHECK_EQ(sqlite3_prepare_v2(g_main, "INSERT INTO fx VALUES(?)", -1, &st, NULL), SQLITE_OK); + sqlite3_bind_int(st, 1, 5); + sqlite3_step(st); + sqlite3_reset(st); + sqlitedbg_set_client_connected(1); + sqlite3_bind_int(st, 1, 6); + sqlite3_step(st); + sqlite3_finalize(st); + sqlitedbg_set_client_connected(0); + exec_ok(g_main, "SELECT 'h\xC3\xA9llo \xE2\x82\xAC \xF0\x9F\x98\x80'"); + exec_ok(g_main, "CREATE TABLE fxlog(a)"); + exec_ok(g_main, "CREATE TRIGGER fxt AFTER INSERT ON fx BEGIN INSERT INTO fxlog VALUES(new.a); END"); + exec_ok(g_main, "INSERT INTO fx VALUES(1)"); + size_t n = (size_t)snprintf(sql, sizeof sql, "SELECT '"); + for (int i = 0; i < 25; i++) n += (size_t)snprintf(sql + n, sizeof sql - n, "\xE2\x82\xAC"); + snprintf(sql + n, sizeof sql - n, "'"); + exec_ok(g_main, sql); + sqlite3 *c = open_app(":memory:"); + exec_ok(c, "SELECT 1"); + CHECK_EQ(sqlite3_close(c), SQLITE_OK); +} + +static unsigned char *slurp(const char *path, size_t *n) { + FILE *f = fopen(path, "rb"); + if (!f) { + perror(path); + return NULL; + } + fseek(f, 0, SEEK_END); + long len = ftell(f); + fseek(f, 0, SEEK_SET); + unsigned char *buf = malloc((size_t)len + 1); + if (fread(buf, 1, (size_t)len, f) != (size_t)len) len = 0; + buf[len] = 0; + fclose(f); + *n = (size_t)len; + return buf; +} + +// Walks records in bin and rewrites them as the table; used both to write and to check the fixture. +static int bin_to_tsv(const unsigned char *bin, size_t n, char **out) { + size_t cap = 1 << 16; + char *s = malloc(cap); + FILE *f = fmemopen(s, cap, "w"); + size_t off = 0; + int ok = 1; + while (off < n) { + const sqlitedbg_event_record *r = (const sqlitedbg_event_record *)(const void *)(bin + off); + if (off + sizeof *r > n || r->record_bytes % 8 || off + r->record_bytes > n || + r->record_bytes < sizeof *r + r->sql_bytes) { + ok = 0; + break; + } + write_tsv_row(f, r, (const char *)bin + off + sizeof *r); + off += r->record_bytes; + } + fclose(f); + *out = s; + return ok; +} + +static void write_fixture(const char *bin_path, const char *tsv_path) { + uint64_t cur = cursor_now(); + run_fixture_scenario(); + size_t cap = 1 << 16; + unsigned char *buf = malloc(cap); + sqlitedbg_read_result r; + sqlitedbg_read_events(cur, buf, cap, &r); + CHECK_EQ(r.dropped, 0); + CHECK(r.count > 10); + FILE *f = fopen(bin_path, "wb"); + CHECK(f != NULL); + if (f) { + fwrite(buf, 1, r.bytes, f); + fclose(f); + } + char *tsv; + CHECK(bin_to_tsv(buf, r.bytes, &tsv)); + f = fopen(tsv_path, "wb"); + CHECK(f != NULL); + if (f) { + fputs(tsv, f); + fclose(f); + } + free(tsv); + free(buf); +} + +// The committed fixture still has the layout of the header: its table is what its bytes hold +static void check_fixture(const char *bin_path, const char *tsv_path) { + size_t bn, tn; + unsigned char *bin = slurp(bin_path, &bn); + unsigned char *tsv = slurp(tsv_path, &tn); + CHECK(bin && tsv); + if (!bin || !tsv) return; + char *again; + CHECK(bin_to_tsv(bin, bn, &again)); + CHECK(strcmp(again, (char *)tsv) == 0); + free(again); + free(bin); + free(tsv); +} + +int main(int argc, char **argv) { + snprintf(g_dir, sizeof g_dir, "/tmp/sqlitedbg-test-XXXXXX"); + if (!mkdtemp(g_dir)) { + perror("mkdtemp"); + return 2; + } + snprintf(g_db_path, sizeof g_db_path, "%s/app.db", g_dir); + + if (argc == 4 && strcmp(argv[1], "--check-fixture") == 0) { + check_fixture(argv[2], argv[3]); + } else { + printf("# install\n"); + test_install(); + printf("# events\n"); + test_events(); + test_expanded(); + printf("# truncation\n"); + test_truncation(); + printf("# burst\n"); + test_burst(); + printf("# close and list\n"); + test_close_and_list(); + printf("# server connections\n"); + test_server(); + printf("# threads\n"); + test_threads(); + if (argc == 4 && strcmp(argv[1], "--write-fixture") == 0) write_fixture(argv[2], argv[3]); + sqlite3_close(g_main); + } + + char cmd[300]; + snprintf(cmd, sizeof cmd, "rm -rf '%s'", g_dir); + if (system(cmd) != 0) fprintf(stderr, "could not remove %s\n", g_dir); + printf("%d checks, %d failures\n", g_checks, g_failures); + return g_failures ? 1 : 0; +} diff --git a/subprojects/sqlite-debugger/src/test/resources/org/appdevforall/sqlitedebugger/engine/events.bin b/subprojects/sqlite-debugger/src/test/resources/org/appdevforall/sqlitedebugger/engine/events.bin new file mode 100644 index 0000000000000000000000000000000000000000..0441b915006ad5921acdf11c603ea101862533ed GIT binary patch literal 1792 zcmb`IIY|tT3Cpsjg5_kjkUhFvlFvfUKDZ2n=CVLzW2|Yx3gi!SRPm+SqQy+#($Rsd&Wxd zHD_w&wfxg;#u#+oh;ZwI>uyg_@WF(m*NHQXg~9T{W(eMXu*2YKo)UC7S}F9oc&Kld z;EkjtFH>JR_yziKwI5sr1beypOg%o4%iTsBfO&U5-hf(ucXB zd39-h3j}ZIb;6?WH$2q0NbtseleY6keNhn9?&n{<+$y+CQjqS&l{2{Lxtfq$R-??L~2gXp>2EiHcv~wyX$9fgW z^cuIp=)iPs_Pw=9@T6W>^0Y(eJ N_K5YKM|*q(eF53lvcUiV literal 0 HcmV?d00001 diff --git a/subprojects/sqlite-debugger/src/test/resources/org/appdevforall/sqlitedebugger/engine/events.tsv b/subprojects/sqlite-debugger/src/test/resources/org/appdevforall/sqlitedebugger/engine/events.tsv new file mode 100644 index 0000000000..4405849603 --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/resources/org/appdevforall/sqlitedebugger/engine/events.tsv @@ -0,0 +1,21 @@ +426616 1 1 10289 1794545527187098386 -1 0 CREATE TABLE fx(a) +426617 2 1 10289 1794545527188573325 1000000 0 CREATE TABLE fx(a) +426618 1 1 10289 1794545527188596333 -1 0 INSERT INTO fx VALUES(?) +426619 2 1 10289 1794545527189296629 1000000 0 INSERT INTO fx VALUES(?) +426620 1 1 10289 1794545527189303467 -1 2 INSERT INTO fx VALUES(6) +426621 2 1 10289 1794545527190447975 1000000 2 INSERT INTO fx VALUES(6) +426622 1 1 10289 1794545527190472305 -1 0 SELECT 'héllo € 😀' +426623 2 1 10289 1794545527190474851 0 0 SELECT 'héllo € 😀' +426624 1 1 10289 1794545527190557276 -1 0 CREATE TABLE fxlog(a) +426625 2 1 10289 1794545527191607925 1000000 0 CREATE TABLE fxlog(a) +426626 1 1 10289 1794545527191694489 -1 1 CREATE TRIGGER fxt AFTER INSERT ON fx BEGIN INSERT INTO fxlog VA +426627 2 1 10289 1794545527192759261 1000000 1 CREATE TRIGGER fxt AFTER INSERT ON fx BEGIN INSERT INTO fxlog VA +426628 1 1 10289 1794545527192791863 -1 0 INSERT INTO fx VALUES(1) +426629 1 1 10289 1794545527192835643 -1 0 -- TRIGGER fxt +426630 1 1 10289 1794545527192836209 -1 0 -- INSERT INTO fxlog VALUES(new.a) +426631 2 1 10289 1794545527193623084 1000000 0 INSERT INTO fx VALUES(1) +426632 1 1 10289 1794545527193646993 -1 1 SELECT '€€€€€€€€€€€€€€€€€€ +426633 2 1 10289 1794545527193649365 0 1 SELECT '€€€€€€€€€€€€€€€€€€ +426634 1 3616 10289 1794545527193782816 -1 0 SELECT 1 +426635 2 3616 10289 1794545527193784200 0 0 SELECT 1 +426636 3 3616 10289 1794545527193785154 -1 0 diff --git a/subprojects/sqlite-debugger/test.sh b/subprojects/sqlite-debugger/test.sh new file mode 100755 index 0000000000..18b42811af --- /dev/null +++ b/subprojects/sqlite-debugger/test.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Host tests of the SQLite debugger's extension (ADFA-7135): src/test/c/sqlitedbg_test.c, with +# SQLite built from the amalgamation as requery builds it (SQLITE_THREADSAFE=2), under +# AddressSanitizer + UBSan and then ThreadSanitizer. +# Usage: test.sh [--update-fixture] [AMALGAMATION_TARBALL] (without the tarball, downloads it) +# --update-fixture rewrites src/test/resources/.../events.bin and events.tsv, which the JVM test +# of EventDecoder reads; without it the committed pair is checked against the +# current record layout instead. +# Needs gcc, curl, tar and sha256sum. +set -euo pipefail +HERE=$(cd "$(dirname "$0")" && pwd) + +# SQLite 3.49.1, the version of the vendored headers in ../sqlite-js/upstream/libs (the script +# checks they are the same). sqlite.org's own zip (sqlite-amalgamation-3490100.zip) is the usual +# source; this is the same amalgamation as bundled in the sqlean.py sdist, which PyPI serves where +# sqlite.org is not reachable. sqlean appends its own init code after SQLite's last line, which is +# cut off below. SHA256 is of the tarball, SQLITE3_C_SHA256 of the cut sqlite3.c. +VERSION=3.49.1 +SHA256=210d89989226b988d7d6391f837387d3b81e8cd608c997e0bd37826e395970e7 +URL=https://files.pythonhosted.org/packages/67/eb/ac95fab0bc4658124b4ec8fbc31fc494165ab4544606ae91b9a489907dad/sqlean_py-3.49.1.tar.gz +SQLITE3_C_SHA256=ff80c36ef1bb44eb357c7ff1d15be77540d41c28fb671088215a6cd12785c5d3 +SQLITE_SOURCE_ID="2025-02-18 13:38:58 873d4e274b4988d260ba8354a9718324a1c26187a4ab4c1cc0227c03d0f10e70" + +UPDATE=0 +if [ "${1:-}" = "--update-fixture" ]; then UPDATE=1; shift; fi + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +TARBALL=${1:-$WORK/sqlite-$VERSION.tar.gz} +[ -f "$TARBALL" ] || curl -fsSL -o "$TARBALL" "$URL" +echo "$SHA256 $TARBALL" | sha256sum -c --quiet - +mkdir "$WORK/src" +tar -xzf "$TARBALL" -C "$WORK/src" --strip-components=1 +AMAL=$WORK/src/sqlite +sed -i '/^\/\*.* End of sqlite3\.c \*/q' "$AMAL/sqlite3.c" +echo "$SQLITE3_C_SHA256 $AMAL/sqlite3.c" | sha256sum -c --quiet - +grep -q "SQLITE_SOURCE_ID \"$SQLITE_SOURCE_ID\"" "$AMAL/sqlite3.h" || { echo "unexpected SQLite source id"; exit 1; } + +LIBS=$HERE/../sqlite-js/upstream/libs +for h in sqlite3.h sqlite3ext.h; do + cmp -s "$AMAL/$h" "$LIBS/$h" || { echo "$h differs from $LIBS/$h"; exit 1; } +done + +build () { "$@" >"$WORK/build.log" 2>&1 || { cat "$WORK/build.log"; exit 1; }; } +CPP=$HERE/src/main/cpp +DEFS=(-DSQLITE_CORE -DSQLITE_THREADSAFE=2 -DSQLITE_DQS=0 -DSQLITE_DEFAULT_MEMSTATUS=0) + +run_with () { + local name=$1 sanitize=$2 + echo "# $name" + mkdir "$WORK/$name" + build gcc -O1 -g -w -fsanitize="$sanitize" "${DEFS[@]}" -I"$AMAL" -c "$AMAL/sqlite3.c" -o "$WORK/$name/sqlite3.o" + build gcc -O1 -g -Wall -Wextra -Werror -fsanitize="$sanitize" -fno-sanitize-recover=all "${DEFS[@]}" \ + -I"$AMAL" -I"$CPP" "$CPP/sqlitedbg.c" "$HERE/src/test/c/sqlitedbg_test.c" "$WORK/$name/sqlite3.o" \ + -o "$WORK/$name/test" -lpthread -ldl -lm + TSAN_OPTIONS=halt_on_error=1 "$WORK/$name/test" ${3:+"${@:3}"} +} + +RES=$HERE/src/test/resources/org/appdevforall/sqlitedebugger/engine +if [ "$UPDATE" = 1 ]; then + mkdir -p "$RES" + run_with asan-ubsan address,undefined --write-fixture "$RES/events.bin" "$RES/events.tsv" +else + run_with asan-ubsan address,undefined + "$WORK/asan-ubsan/test" --check-fixture "$RES/events.bin" "$RES/events.tsv" +fi +run_with tsan thread From ff0312f85b8121d9e1ac867b279b33db63e4de25 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 05:00:25 +0000 Subject: [PATCH 06/24] ADFA-7135 Add the JNI layer and NativeDebugEngine sqlitedbg_jni.c registers NativeBridge's natives from JNI_OnLoad (the version script exports nothing else) and reaches SQLite only through the extension's api table. ServerConnection.query steps the statement in C, binds args by type, applies QueryLimits, and stops at the deadline through a progress handler on the server connection. EventDecoder mirrors the exported record layout and is tested against the fixture the host C test writes. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- .../sqlite-debugger/consumer-rules.pro | 3 + .../src/main/cpp/CMakeLists.txt | 5 +- .../src/main/cpp/sqlitedbg_jni.c | 420 ++++++++++++++++++ .../sqlitedebugger/engine/EventDecoder.kt | 69 +++ .../sqlitedebugger/engine/NativeBridge.kt | 52 +++ .../engine/NativeDebugEngine.kt | 127 ++++++ .../sqlitedebugger/engine/EventDecoderTest.kt | 92 ++++ 7 files changed, 767 insertions(+), 1 deletion(-) create mode 100644 subprojects/sqlite-debugger/consumer-rules.pro create mode 100644 subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c create mode 100644 subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/EventDecoder.kt create mode 100644 subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeBridge.kt create mode 100644 subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeDebugEngine.kt create mode 100644 subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/EventDecoderTest.kt diff --git a/subprojects/sqlite-debugger/consumer-rules.pro b/subprojects/sqlite-debugger/consumer-rules.pro new file mode 100644 index 0000000000..4ba638bc89 --- /dev/null +++ b/subprojects/sqlite-debugger/consumer-rules.pro @@ -0,0 +1,3 @@ +# JNI_OnLoad registers these by name, and the native code builds this exception +-keep class org.appdevforall.sqlitedebugger.engine.NativeBridge { *; } +-keep class org.appdevforall.sqlitedebugger.engine.DebugEngineException { (int, java.lang.String); } diff --git a/subprojects/sqlite-debugger/src/main/cpp/CMakeLists.txt b/subprojects/sqlite-debugger/src/main/cpp/CMakeLists.txt index bbe9393af9..779767a2bb 100644 --- a/subprojects/sqlite-debugger/src/main/cpp/CMakeLists.txt +++ b/subprojects/sqlite-debugger/src/main/cpp/CMakeLists.txt @@ -7,10 +7,13 @@ project(sqlitedbg C) set(SQLITE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../../../../sqlite-js/upstream/libs) add_library(sqlitedbg SHARED - sqlitedbg.c) + sqlitedbg.c + sqlitedbg_jni.c) target_include_directories(sqlitedbg PRIVATE ${SQLITE_HEADERS} ${CMAKE_CURRENT_SOURCE_DIR}) target_compile_options(sqlitedbg PRIVATE -Wall -Wextra -fvisibility=hidden) +# dlopen/dlsym to find sqlite3_auto_extension in requery's libsqlite3x.so +target_link_libraries(sqlitedbg dl) # Only JNI_OnLoad is exported, so sqlite3_api and our other symbols cannot bind to another library's. target_link_options(sqlitedbg PRIVATE -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/sqlitedbg.map diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c new file mode 100644 index 0000000000..d5fede7078 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c @@ -0,0 +1,420 @@ +// JNI layer of the SQLite debugger (ADFA-7135): NativeBridge.kt over sqlitedbg.h. Like the +// extension it calls SQLite only through the api table (sqlite3ext.h), which sqlitedbg.c saves. +// Every handle here comes from sqlitedbg_open_server, which refuses to open before that. + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include + +#include "sqlite3ext.h" +#include "sqlitedbg.h" + +SQLITE_EXTENSION_INIT3 + +#define BRIDGE_CLASS "org/appdevforall/sqlitedebugger/engine/NativeBridge" +#define EXCEPTION_CLASS "org/appdevforall/sqlitedebugger/engine/DebugEngineException" + +static jclass g_object, g_string, g_array_list, g_long, g_double, g_exception; +static jmethodID g_string_from_bytes, g_list_init, g_list_add, g_long_value_of, g_double_value_of, g_exception_init; +static jstring g_utf8_name; + +static jclass global_class(JNIEnv *env, const char *name) { + jclass local = (*env)->FindClass(env, name); + if (!local) return NULL; + jclass global = (jclass)(*env)->NewGlobalRef(env, local); + (*env)->DeleteLocalRef(env, local); + return global; +} + +// A Java String from UTF-8 bytes. NewStringUTF takes modified UTF-8, which differs for NUL and for +// characters outside the BMP, and SQLite text can hold both. +static jstring utf8_string(JNIEnv *env, const char *bytes, int n) { + jbyteArray array = (*env)->NewByteArray(env, n); + if (!array) return NULL; + (*env)->SetByteArrayRegion(env, array, 0, n, (const jbyte *)bytes); + jstring s = (jstring)(*env)->NewObject(env, g_string, g_string_from_bytes, array, g_utf8_name); + (*env)->DeleteLocalRef(env, array); + return s; +} + +static void throw_engine(JNIEnv *env, int code, const char *message) { + if ((*env)->ExceptionCheck(env)) return; + jstring text = utf8_string(env, message, (int)strlen(message)); + if (!text) return; + jobject ex = (*env)->NewObject(env, g_exception, g_exception_init, (jint)code, text); + if (ex) (*env)->Throw(env, (jthrowable)ex); +} + +static int64_t now_ms(void) { + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + return (int64_t)ts.tv_sec * 1000 + ts.tv_nsec / 1000000; +} + +// install + +static jint bridge_install(JNIEnv *env, jclass cls) { + (void)env; + (void)cls; + // requery's library is already loaded by System.loadLibrary; NOLOAD only finds it, and its + // sqlite3_ functions are all exported, so no separate handle to SQLite is needed + void *lib = dlopen("libsqlite3x.so", RTLD_NOW | RTLD_NOLOAD); + if (!lib) return -1; + sqlitedbg_auto_extension_fn auto_extension = (sqlitedbg_auto_extension_fn)dlsym(lib, "sqlite3_auto_extension"); + int rc = auto_extension ? sqlitedbg_install(auto_extension, NULL) : -1; + dlclose(lib); + return rc; +} + +// connections + +typedef struct conn_list { + jlong *meta; + char **paths; + size_t n, cap; + int oom; +} conn_list; + +static void collect_connection(void *context, const sqlitedbg_connection *c) { + conn_list *l = (conn_list *)context; + if (l->oom) return; + if (l->n == l->cap) { + size_t cap = l->cap ? l->cap * 2 : 16; + jlong *meta = realloc(l->meta, cap * 5 * sizeof(jlong)); + if (meta) l->meta = meta; + char **paths = realloc(l->paths, cap * sizeof(char *)); + if (paths) l->paths = paths; + if (!meta || !paths) { + l->oom = 1; + return; + } + l->cap = cap; + } + char *path = strdup(c->path); + if (!path) { + l->oom = 1; + return; + } + jlong *m = l->meta + l->n * 5; + m[0] = (jlong)c->id; + m[1] = c->read_only; + m[2] = c->closing; + m[3] = (jlong)c->opened_at_ns; + m[4] = (jlong)c->open_thread_id; + l->paths[l->n++] = path; +} + +static jobjectArray bridge_connections(JNIEnv *env, jclass cls) { + (void)cls; + conn_list l = {0}; + sqlitedbg_each_connection(collect_connection, &l); + jobjectArray result = NULL; + if (l.oom) { + throw_engine(env, SQLITE_NOMEM, "out of memory"); + goto done; + } + jlongArray meta = (*env)->NewLongArray(env, (jsize)(l.n * 5)); + jobjectArray paths = meta ? (*env)->NewObjectArray(env, (jsize)l.n, g_string, NULL) : NULL; + if (!paths) goto done; + (*env)->SetLongArrayRegion(env, meta, 0, (jsize)(l.n * 5), l.meta ? l.meta : (jlong[]){0}); + for (size_t i = 0; i < l.n; i++) { + jstring s = utf8_string(env, l.paths[i], (int)strlen(l.paths[i])); + if (!s) goto done; + (*env)->SetObjectArrayElement(env, paths, (jsize)i, s); + (*env)->DeleteLocalRef(env, s); + } + result = (*env)->NewObjectArray(env, 2, g_object, NULL); + if (result) { + (*env)->SetObjectArrayElement(env, result, 0, meta); + (*env)->SetObjectArrayElement(env, result, 1, paths); + } +done: + for (size_t i = 0; i < l.n; i++) free(l.paths[i]); + free(l.paths); + free(l.meta); + return result; +} + +// events + +static void bridge_read_events(JNIEnv *env, jclass cls, jlong after, jbyteArray buf, jlongArray out) { + (void)cls; + jsize cap = (*env)->GetArrayLength(env, buf); + void *tmp = malloc((size_t)cap); + if (!tmp) { + throw_engine(env, SQLITE_NOMEM, "out of memory"); + return; + } + sqlitedbg_read_result r; + sqlitedbg_read_events((uint64_t)after, tmp, (size_t)cap, &r); + (*env)->SetByteArrayRegion(env, buf, 0, (jsize)r.bytes, (const jbyte *)tmp); + free(tmp); + jlong values[3] = {(jlong)r.next, (jlong)r.dropped, (jlong)r.bytes}; + (*env)->SetLongArrayRegion(env, out, 0, 3, values); +} + +static void bridge_set_client_connected(JNIEnv *env, jclass cls, jboolean connected) { + (void)env; + (void)cls; + sqlitedbg_set_client_connected(connected ? 1 : 0); +} + +// server connections + +static jlong bridge_open_server(JNIEnv *env, jclass cls, jbyteArray path, jboolean writable) { + (void)cls; + jsize n = (*env)->GetArrayLength(env, path); + char *c_path = malloc((size_t)n + 1); + if (!c_path) { + throw_engine(env, SQLITE_NOMEM, "out of memory"); + return 0; + } + (*env)->GetByteArrayRegion(env, path, 0, n, (jbyte *)c_path); + c_path[n] = 0; + sqlite3 *db = NULL; + int rc = sqlitedbg_open_server(c_path, writable ? 1 : 0, &db); + free(c_path); + if (rc != SQLITE_OK) { + throw_engine(env, rc, rc == SQLITE_MISUSE ? "no app connection has opened yet" : sqlite3_errstr(rc)); + return 0; + } + return (jlong)(intptr_t)db; +} + +static void bridge_close(JNIEnv *env, jclass cls, jlong handle) { + (void)env; + (void)cls; + sqlite3_close_v2((sqlite3 *)(intptr_t)handle); +} + +typedef struct deadline { + int64_t at_ms; + int hit; +} deadline; + +static int progress_cb(void *p) { + deadline *d = (deadline *)p; + if (now_ms() < d->at_ms) return 0; + d->hit = 1; + return 1; +} + +// Binds args[i] to parameter i + 1. Returns SQLITE_OK or a code with *message set. +static int bind_args(JNIEnv *env, sqlite3 *db, sqlite3_stmt *st, jobjectArray args, const char **message) { + jsize count = (*env)->GetArrayLength(env, args); + for (jsize i = 0; i < count; i++) { + jobject v = (*env)->GetObjectArrayElement(env, args, i); + int rc; + if (!v) { + rc = sqlite3_bind_null(st, i + 1); + } else if ((*env)->IsInstanceOf(env, v, g_long)) { + jmethodID m = (*env)->GetMethodID(env, g_long, "longValue", "()J"); + rc = sqlite3_bind_int64(st, i + 1, (sqlite3_int64)(*env)->CallLongMethod(env, v, m)); + } else if ((*env)->IsInstanceOf(env, v, g_double)) { + jmethodID m = (*env)->GetMethodID(env, g_double, "doubleValue", "()D"); + rc = sqlite3_bind_double(st, i + 1, (*env)->CallDoubleMethod(env, v, m)); + } else if ((*env)->IsInstanceOf(env, v, g_string)) { + jsize len = (*env)->GetStringLength(env, (jstring)v); + const jchar *chars = (*env)->GetStringChars(env, (jstring)v, NULL); + if (!chars) return SQLITE_NOMEM; + rc = sqlite3_bind_text16(st, i + 1, chars, len * 2, SQLITE_TRANSIENT); + (*env)->ReleaseStringChars(env, (jstring)v, chars); + } else { + jclass bytes_class = (*env)->FindClass(env, "[B"); + int is_bytes = bytes_class && (*env)->IsInstanceOf(env, v, bytes_class); + if (bytes_class) (*env)->DeleteLocalRef(env, bytes_class); + if (!is_bytes) { + *message = "unsupported argument type"; + return SQLITE_MISUSE; + } + jsize len = (*env)->GetArrayLength(env, (jbyteArray)v); + void *copy = malloc(len ? (size_t)len : 1); + if (!copy) return SQLITE_NOMEM; + (*env)->GetByteArrayRegion(env, (jbyteArray)v, 0, len, (jbyte *)copy); + rc = sqlite3_bind_blob64(st, i + 1, copy, (sqlite3_uint64)len, SQLITE_TRANSIENT); + free(copy); + } + if (v) (*env)->DeleteLocalRef(env, v); + if (rc != SQLITE_OK) { + *message = sqlite3_errmsg(db); + return rc; + } + if ((*env)->ExceptionCheck(env)) return SQLITE_ERROR; + } + return SQLITE_OK; +} + +// A UTF-8 prefix of at most max bytes ending on a character boundary. +static int utf8_prefix(const unsigned char *s, int len, int max) { + if (len <= max) return len; + while (max > 0 && (s[max] & 0xC0) == 0x80) max--; + return max; +} + +// One cell as a Java object, or NULL for SQL NULL (or on failure, with an exception pending). +static jobject cell_value(JNIEnv *env, sqlite3_stmt *st, int col, jint max_text, jint max_blob, int *truncated) { + switch (sqlite3_column_type(st, col)) { + case SQLITE_INTEGER: + return (*env)->CallStaticObjectMethod(env, g_long, g_long_value_of, (jlong)sqlite3_column_int64(st, col)); + case SQLITE_FLOAT: + return (*env)->CallStaticObjectMethod(env, g_double, g_double_value_of, (jdouble)sqlite3_column_double(st, col)); + case SQLITE_TEXT: { + const unsigned char *text = sqlite3_column_text(st, col); + int len = sqlite3_column_bytes(st, col); + int keep = utf8_prefix(text, len, max_text); + if (keep < len) (*truncated)++; + return utf8_string(env, (const char *)text, keep); + } + case SQLITE_BLOB: { + const void *blob = sqlite3_column_blob(st, col); + int len = sqlite3_column_bytes(st, col); + int keep = len > max_blob ? max_blob : len; + if (keep < len) (*truncated)++; + jbyteArray array = (*env)->NewByteArray(env, keep); + if (array && keep > 0) (*env)->SetByteArrayRegion(env, array, 0, keep, (const jbyte *)blob); + return array; + } + default: + return NULL; + } +} + +static jobjectArray bridge_query(JNIEnv *env, jclass cls, jlong handle, jstring sql, jobjectArray args, jlong offset, + jint max_rows, jint max_text, jint max_blob, jlong deadline_ms) { + (void)cls; + sqlite3 *db = (sqlite3 *)(intptr_t)handle; + sqlite3_stmt *st = NULL; + jobjectArray result = NULL; + deadline dl = {(int64_t)deadline_ms, 0}; + int handler_set = 0; + const char *message = NULL; + int rc; + + if (now_ms() >= dl.at_ms) { + throw_engine(env, SQLITE_INTERRUPT, "the time limit passed before the query started"); + return NULL; + } + jsize sql_len = (*env)->GetStringLength(env, sql); + const jchar *sql_chars = (*env)->GetStringChars(env, sql, NULL); + if (!sql_chars) return NULL; + rc = sqlite3_prepare16_v2(db, sql_chars, sql_len * 2, &st, NULL); + (*env)->ReleaseStringChars(env, sql, sql_chars); + if (rc != SQLITE_OK) { + throw_engine(env, rc, sqlite3_errmsg(db)); + return NULL; + } + + // SQLite checks the handler every 1000 virtual machine steps and stops the statement when it + // returns non-zero; it is cleared before this returns, so it never outlives `dl`. + sqlite3_progress_handler(db, 1000, progress_cb, &dl); + handler_set = 1; + + rc = st ? bind_args(env, db, st, args, &message) : SQLITE_OK; + if (rc != SQLITE_OK) { + if (!(*env)->ExceptionCheck(env)) throw_engine(env, rc, message ? message : "cannot bind"); + goto done; + } + + int ncols = st ? sqlite3_column_count(st) : 0; + jobjectArray names = (*env)->NewObjectArray(env, ncols, g_string, NULL); + jobject rows = names ? (*env)->NewObject(env, g_array_list, g_list_init) : NULL; + if (!rows) goto done; + for (int c = 0; c < ncols; c++) { + const char *name = sqlite3_column_name(st, c); + jstring s = utf8_string(env, name ? name : "", name ? (int)strlen(name) : 0); + if (!s) goto done; + (*env)->SetObjectArrayElement(env, names, c, s); + (*env)->DeleteLocalRef(env, s); + } + + int truncated = 0, has_more = 0, row_count = 0; + jlong skipped = 0; + while (st && (rc = sqlite3_step(st)) == SQLITE_ROW) { + if (skipped < offset) { + skipped++; + continue; + } + if (row_count >= max_rows) { + has_more = 1; + break; + } + jobjectArray row = (*env)->NewObjectArray(env, ncols, g_object, NULL); + if (!row) goto done; + for (int c = 0; c < ncols; c++) { + jobject cell = cell_value(env, st, c, max_text, max_blob, &truncated); + if ((*env)->ExceptionCheck(env)) goto done; + if (cell) { + (*env)->SetObjectArrayElement(env, row, c, cell); + (*env)->DeleteLocalRef(env, cell); + } + } + (*env)->CallBooleanMethod(env, rows, g_list_add, row); + (*env)->DeleteLocalRef(env, row); + if ((*env)->ExceptionCheck(env)) goto done; + row_count++; + } + if (st && !has_more && rc != SQLITE_DONE) { + if (dl.hit && rc == SQLITE_INTERRUPT) throw_engine(env, SQLITE_INTERRUPT, "the query ran past its time limit"); + else throw_engine(env, rc, sqlite3_errmsg(db)); + goto done; + } + + jlongArray counts = (*env)->NewLongArray(env, 2); + result = counts ? (*env)->NewObjectArray(env, 3, g_object, NULL) : NULL; + if (!result) goto done; + jlong values[2] = {truncated, has_more}; + (*env)->SetLongArrayRegion(env, counts, 0, 2, values); + (*env)->SetObjectArrayElement(env, result, 0, names); + (*env)->SetObjectArrayElement(env, result, 1, rows); + (*env)->SetObjectArrayElement(env, result, 2, counts); + +done: + if (handler_set) sqlite3_progress_handler(db, 0, NULL, NULL); + sqlite3_finalize(st); + return result; +} + +JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) { + (void)reserved; + JNIEnv *env; + if ((*vm)->GetEnv(vm, (void **)&env, JNI_VERSION_1_6) != JNI_OK) return JNI_ERR; + + g_object = global_class(env, "java/lang/Object"); + g_string = global_class(env, "java/lang/String"); + g_array_list = global_class(env, "java/util/ArrayList"); + g_long = global_class(env, "java/lang/Long"); + g_double = global_class(env, "java/lang/Double"); + g_exception = global_class(env, EXCEPTION_CLASS); + jstring utf8 = (*env)->NewStringUTF(env, "UTF-8"); + if (!g_object || !g_string || !g_array_list || !g_long || !g_double || !g_exception || !utf8) return JNI_ERR; + g_utf8_name = (jstring)(*env)->NewGlobalRef(env, utf8); + + g_string_from_bytes = (*env)->GetMethodID(env, g_string, "", "([BLjava/lang/String;)V"); + g_list_init = (*env)->GetMethodID(env, g_array_list, "", "()V"); + g_list_add = (*env)->GetMethodID(env, g_array_list, "add", "(Ljava/lang/Object;)Z"); + g_long_value_of = (*env)->GetStaticMethodID(env, g_long, "valueOf", "(J)Ljava/lang/Long;"); + g_double_value_of = (*env)->GetStaticMethodID(env, g_double, "valueOf", "(D)Ljava/lang/Double;"); + g_exception_init = (*env)->GetMethodID(env, g_exception, "", "(ILjava/lang/String;)V"); + if (!g_string_from_bytes || !g_list_init || !g_list_add || !g_long_value_of || !g_double_value_of || !g_exception_init) + return JNI_ERR; + + static const JNINativeMethod methods[] = { + {"install", "()I", (void *)bridge_install}, + {"connections", "()[Ljava/lang/Object;", (void *)bridge_connections}, + {"readEvents", "(J[B[J)V", (void *)bridge_read_events}, + {"setClientConnected", "(Z)V", (void *)bridge_set_client_connected}, + {"openServer", "([BZ)J", (void *)bridge_open_server}, + {"query", "(JLjava/lang/String;[Ljava/lang/Object;JIIIJ)[Ljava/lang/Object;", (void *)bridge_query}, + {"close", "(J)V", (void *)bridge_close}, + }; + jclass bridge = (*env)->FindClass(env, BRIDGE_CLASS); + if (!bridge || (*env)->RegisterNatives(env, bridge, methods, sizeof methods / sizeof methods[0]) != JNI_OK) + return JNI_ERR; + return JNI_VERSION_1_6; +} diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/EventDecoder.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/EventDecoder.kt new file mode 100644 index 0000000000..73af648598 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/EventDecoder.kt @@ -0,0 +1,69 @@ +package org.appdevforall.sqlitedebugger.engine + +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** + * Decodes the records `sqlitedbg_read_events` copies out. Mirrors `sqlitedbg_event_record` in + * `sqlitedbg.h`: change both together. Little-endian, because the CPU is on every ABI we build. + */ +internal object EventDecoder { + /** `sizeof(sqlitedbg_event_record)`. */ + const val HEADER_BYTES = 56 + + private const val OFFSET_RECORD_BYTES = 0 + private const val OFFSET_KIND = 4 + private const val OFFSET_SEQ = 8 + private const val OFFSET_CONNECTION_ID = 16 + private const val OFFSET_THREAD_ID = 24 + private const val OFFSET_TIME_NS = 32 + private const val OFFSET_DURATION_NS = 40 + private const val OFFSET_FLAGS = 48 + private const val OFFSET_SQL_BYTES = 52 + + private const val FLAG_TRUNCATED = 1 + private const val FLAG_EXPANDED = 2 + + /** + * The records in the first [length] bytes of [bytes]. + * + * @throws IllegalArgumentException if they are not whole, well-formed records. + */ + fun decode( + bytes: ByteArray, + length: Int, + ): List { + require(length in 0..bytes.size) { "length $length outside 0..${bytes.size}" } + val buffer = ByteBuffer.wrap(bytes, 0, length).order(ByteOrder.LITTLE_ENDIAN) + val events = ArrayList() + var offset = 0 + while (offset < length) { + require(length - offset >= HEADER_BYTES) { "record at $offset is cut off by the end of the data" } + val recordBytes = buffer.getInt(offset + OFFSET_RECORD_BYTES) + val sqlBytes = buffer.getInt(offset + OFFSET_SQL_BYTES) + require(sqlBytes >= 0 && recordBytes >= 0 && recordBytes % 8 == 0 && sqlBytes <= recordBytes - HEADER_BYTES) { + "record at $offset has record_bytes=$recordBytes sql_bytes=$sqlBytes" + } + require(recordBytes <= length - offset) { "record at $offset runs past the end of the data" } + val kindValue = buffer.getInt(offset + OFFSET_KIND) + val kind = + EventKind.values().firstOrNull { it.nativeValue == kindValue } + ?: throw IllegalArgumentException("record at $offset has unknown kind $kindValue") + val flags = buffer.getInt(offset + OFFSET_FLAGS) + events += + Event( + seq = buffer.getLong(offset + OFFSET_SEQ), + kind = kind, + connectionId = buffer.getLong(offset + OFFSET_CONNECTION_ID), + threadId = buffer.getLong(offset + OFFSET_THREAD_ID), + timeEpochNanos = buffer.getLong(offset + OFFSET_TIME_NS), + durationNanos = if (kind == EventKind.PROFILE) buffer.getLong(offset + OFFSET_DURATION_NS) else null, + sql = String(bytes, offset + HEADER_BYTES, sqlBytes, Charsets.UTF_8), + truncated = flags and FLAG_TRUNCATED != 0, + expanded = flags and FLAG_EXPANDED != 0, + ) + offset += recordBytes + } + return events + } +} diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeBridge.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeBridge.kt new file mode 100644 index 0000000000..578ac83086 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeBridge.kt @@ -0,0 +1,52 @@ +package org.appdevforall.sqlitedebugger.engine + +/** + * The JNI entry points of libsqlitedbg.so, registered by name in its JNI_OnLoad (sqlitedbg_jni.c). + * Signatures there must match these. Only [NativeDebugEngine] calls them. + */ +internal object NativeBridge { + /** Registers the extension with requery's SQLite. 0 (SQLITE_OK) or the SQLite code; -1 if libsqlite3x.so is not loaded. */ + @JvmStatic + external fun install(): Int + + /** `[LongArray, Array]`: 5 longs per connection (id, readOnly, closing, openedAtNs, openThreadId), and the paths. */ + @JvmStatic + external fun connections(): Array + + /** Fills [buf] with records after [after]; [out] gets next, dropped and the bytes written. */ + @JvmStatic + external fun readEvents( + after: Long, + buf: ByteArray, + out: LongArray, + ) + + @JvmStatic + external fun setClientConnected(connected: Boolean) + + /** A handle for [query] and [close]. Throws [DebugEngineException]. */ + @JvmStatic + external fun openServer( + path: ByteArray, + writable: Boolean, + ): Long + + /** + * `[Array, ArrayList>, LongArray]`: the column names, the rows, and + * `{truncatedCells, hasMore}`. Throws [DebugEngineException]. + */ + @JvmStatic + external fun query( + handle: Long, + sql: String, + args: Array, + offset: Long, + maxRows: Int, + maxTextBytes: Int, + maxBlobBytes: Int, + deadlineMillis: Long, + ): Array + + @JvmStatic + external fun close(handle: Long) +} diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeDebugEngine.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeDebugEngine.kt new file mode 100644 index 0000000000..00e95c20d1 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeDebugEngine.kt @@ -0,0 +1,127 @@ +package org.appdevforall.sqlitedebugger.engine + +/** + * [DebugEngine] over the JNI layer of libsqlitedbg.so. Needs requery's libsqlite3x.so in the process: + * the extension runs inside it, on connections it opens after [install]. Call [install] before + * anything else; the native state is per process, so every instance shares it. + */ +class NativeDebugEngine : DebugEngine { + /** + * Loads requery's SQLite and the debugger's library and registers the extension. Connections + * opened before this are never seen, so call it as early as the process allows. Calling it + * again, on any instance, does nothing. + * + * @throws DebugEngineException if SQLite refuses the extension. + * @throws UnsatisfiedLinkError if a library is missing. + */ + fun install() { + synchronized(lock) { + if (installed) return + System.loadLibrary("sqlite3x") + System.loadLibrary("sqlitedbg") + val rc = NativeBridge.install() + if (rc != 0) { + throw DebugEngineException( + if (rc < 0) DebugEngineException.SQLITE_MISUSE else rc, + if (rc < 0) "libsqlite3x.so is not loaded" else "sqlite3_auto_extension failed with $rc", + ) + } + installed = true + } + } + + override fun connections(): List { + val result = NativeBridge.connections() + + @Suppress("UNCHECKED_CAST") + val paths = result[1] as Array + val meta = result[0] as LongArray + return paths.indices.map { i -> + val m = i * META_PER_CONNECTION + AppConnection( + id = meta[m], + path = paths[i], + readOnly = meta[m + 1] != 0L, + closing = meta[m + 2] != 0L, + openedAtEpochNanos = meta[m + 3], + openThreadId = meta[m + 4], + ) + } + } + + override fun readEvents( + after: Long, + maxBytes: Int, + ): EventBatch { + val buffer = ByteArray(maxBytes.coerceIn(1, MAX_READ_BYTES)) + val out = LongArray(3) + NativeBridge.readEvents(after, buffer, out) + return EventBatch(events = EventDecoder.decode(buffer, out[2].toInt()), next = out[0], dropped = out[1]) + } + + override fun setClientConnected(connected: Boolean) = NativeBridge.setClientConnected(connected) + + override fun openServerConnection( + path: String, + writable: Boolean, + ): ServerConnection = NativeServerConnection(NativeBridge.openServer(path.toByteArray(Charsets.UTF_8), writable)) + + private class NativeServerConnection( + private var handle: Long, + ) : ServerConnection { + override fun query( + sql: String, + args: List, + offset: Long, + limits: QueryLimits, + deadlineMillis: Long, + ): QueryResult { + check(handle != 0L) { "the connection is closed" } + require(offset >= 0) { "offset $offset is negative" } + require(limits.maxRows > 0 && limits.maxTextBytes >= 0 && limits.maxBlobBytes >= 0) { "bad limits $limits" } + val result = + NativeBridge.query( + handle, + sql, + Array(args.size) { bindable(args[it]) }, + offset, + limits.maxRows, + limits.maxTextBytes, + limits.maxBlobBytes, + deadlineMillis, + ) + + @Suppress("UNCHECKED_CAST") + val columns = (result[0] as Array).asList() + + @Suppress("UNCHECKED_CAST") + val rows = (result[1] as List>).map { it.asList() } + val counts = result[2] as LongArray + return QueryResult(columns, rows, truncatedCells = counts[0].toInt(), hasMore = counts[1] != 0L) + } + + override fun close() { + val h = handle + handle = 0 + if (h != 0L) NativeBridge.close(h) + } + + private fun bindable(value: Any?): Any? = + when (value) { + null, is Long, is Double, is String, is ByteArray -> value + is Int -> value.toLong() + is Short -> value.toLong() + is Byte -> value.toLong() + is Float -> value.toDouble() + else -> throw IllegalArgumentException("cannot bind a ${value.javaClass.name}") + } + } + + private companion object { + const val META_PER_CONNECTION = 5 + const val MAX_READ_BYTES = 16 shl 20 + + val lock = Any() + var installed = false + } +} diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/EventDecoderTest.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/EventDecoderTest.kt new file mode 100644 index 0000000000..555e566544 --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/EventDecoderTest.kt @@ -0,0 +1,92 @@ +package org.appdevforall.sqlitedebugger.engine + +import com.google.common.truth.Truth.assertThat +import org.junit.Assert.assertThrows +import org.junit.Test +import java.nio.ByteBuffer +import java.nio.ByteOrder + +class EventDecoderTest { + // events.bin is what sqlitedbg_read_events copied out in sqlitedbg_test.c (test.sh --update-fixture); + // events.tsv is the same records as the C struct reads them: seq, kind, connection, thread, time, + // duration, flags, sql. + private val bytes = resource("events.bin") + private val table = + String(resource("events.tsv"), Charsets.UTF_8) + .trimEnd('\n') + .split('\n') + .map { it.split('\t', limit = 8) } + + private fun resource(name: String): ByteArray = + checkNotNull(javaClass.getResourceAsStream(name)) { "missing fixture $name" }.use { it.readBytes() } + + @Test + fun `decodes every field of the record the C layer wrote`() { + val events = EventDecoder.decode(bytes, bytes.size) + + assertThat(events).hasSize(table.size) + events.zip(table).forEach { (event, row) -> + assertThat(event.seq).isEqualTo(row[0].toLong()) + assertThat(event.kind.nativeValue).isEqualTo(row[1].toInt()) + assertThat(event.connectionId).isEqualTo(row[2].toLong()) + assertThat(event.threadId).isEqualTo(row[3].toLong()) + assertThat(event.timeEpochNanos).isEqualTo(row[4].toLong()) + assertThat(event.durationNanos).isEqualTo(if (event.kind == EventKind.PROFILE) row[5].toLong() else null) + assertThat(event.truncated).isEqualTo(row[6].toInt() and 1 != 0) + assertThat(event.expanded).isEqualTo(row[6].toInt() and 2 != 0) + assertThat(event.sql).isEqualTo(row[7]) + } + } + + @Test + fun `the fixture covers every kind and flag and a wide id`() { + val events = EventDecoder.decode(bytes, bytes.size) + + assertThat(events.map { it.kind }.toSet()).containsExactlyElementsIn(EventKind.values().toList()) + assertThat(events.any { it.truncated }).isTrue() + assertThat(events.any { it.expanded }).isTrue() + assertThat(events.any { it.sql.startsWith("--") }).isTrue() + // multi-byte text survives, including a character outside the BMP + assertThat(events.any { it.sql.contains("héllo € 😀") }).isTrue() + assertThat(events.map { it.seq }).isInOrder() + } + + @Test + fun `a prefix ending on a record boundary decodes to those records`() { + val firstRecordBytes = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).getInt(0) + + assertThat(EventDecoder.decode(bytes, firstRecordBytes)).hasSize(1) + assertThat(EventDecoder.decode(bytes, 0)).isEmpty() + } + + @Test + fun `a cut-off record is refused`() { + val firstRecordBytes = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).getInt(0) + + assertThrows(IllegalArgumentException::class.java) { EventDecoder.decode(bytes, firstRecordBytes - 8) } + assertThrows(IllegalArgumentException::class.java) { EventDecoder.decode(bytes, EventDecoder.HEADER_BYTES - 1) } + assertThrows(IllegalArgumentException::class.java) { EventDecoder.decode(bytes, bytes.size + 1) } + } + + @Test + fun `a malformed header is refused`() { + fun patched( + offset: Int, + value: Int, + ): ByteArray { + val copy = bytes.copyOf() + ByteBuffer.wrap(copy).order(ByteOrder.LITTLE_ENDIAN).putInt(offset, value) + return copy + } + + // record_bytes not a multiple of 8, smaller than the header, and an unknown kind + for (broken in listOf(patched(0, 60), patched(0, 48), patched(4, 9), patched(4, 0))) { + assertThrows(IllegalArgumentException::class.java) { EventDecoder.decode(broken, broken.size) } + } + // sql_bytes longer than the record holds + val big = patched(52, 100000) + assertThrows(IllegalArgumentException::class.java) { EventDecoder.decode(big, big.size) } + val negative = patched(52, -1) + assertThrows(IllegalArgumentException::class.java) { EventDecoder.decode(negative, negative.size) } + } +} From 31dc721e05c3871b68da8ebe909072a43897e08a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 05:06:19 +0000 Subject: [PATCH 07/24] style: spotless reformat of the SQLite debugger module, no functional change Co-Authored-By: Claude Haiku 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- .../sqlitedebugger/DebuggerStartup.kt | 30 ++- .../sqlitedebugger/mcp/DatabaseTools.kt | 213 ++++++++++++---- .../sqlitedebugger/mcp/McpHttp.kt | 232 +++++++++++++----- .../sqlitedebugger/mcp/McpProtocol.kt | 150 +++++++---- .../sqlitedebugger/mcp/DatabaseToolsTest.kt | 148 ++++++++--- .../sqlitedebugger/mcp/McpProtocolTest.kt | 176 +++++++++---- 6 files changed, 718 insertions(+), 231 deletions(-) diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/DebuggerStartup.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/DebuggerStartup.kt index 2328ca17d1..30f7177dad 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/DebuggerStartup.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/DebuggerStartup.kt @@ -6,13 +6,13 @@ import android.content.pm.ApplicationInfo import android.database.Cursor import android.net.Uri import android.util.Log -import java.io.File -import kotlin.concurrent.thread import org.appdevforall.sqlitedebugger.engine.NativeDebugEngine import org.appdevforall.sqlitedebugger.mcp.DatabaseTools import org.appdevforall.sqlitedebugger.mcp.McpHttp import org.appdevforall.sqlitedebugger.mcp.McpProtocol +import java.io.File import java.util.UUID +import kotlin.concurrent.thread /** * Starts the debugger when the app's process starts. Refuses to run in a build that is not @@ -45,15 +45,33 @@ class DebuggerStartup : ContentProvider() { return true } - override fun query(uri: Uri, projection: Array?, selection: String?, selectionArgs: Array?, sortOrder: String?): Cursor? = null + override fun query( + uri: Uri, + projection: Array?, + selection: String?, + selectionArgs: Array?, + sortOrder: String?, + ): Cursor? = null override fun getType(uri: Uri): String? = null - override fun insert(uri: Uri, values: ContentValues?): Uri? = null + override fun insert( + uri: Uri, + values: ContentValues?, + ): Uri? = null - override fun delete(uri: Uri, selection: String?, selectionArgs: Array?): Int = 0 + override fun delete( + uri: Uri, + selection: String?, + selectionArgs: Array?, + ): Int = 0 - override fun update(uri: Uri, values: ContentValues?, selection: String?, selectionArgs: Array?): Int = 0 + override fun update( + uri: Uri, + values: ContentValues?, + selection: String?, + selectionArgs: Array?, + ): Int = 0 companion object { private const val TAG = "SqliteDebugger" diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt index 477b8d9d60..2ff44ec868 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt @@ -1,6 +1,5 @@ package org.appdevforall.sqlitedebugger.mcp -import java.io.File import org.appdevforall.sqlitedebugger.engine.DebugEngine import org.appdevforall.sqlitedebugger.engine.DebugEngineException import org.appdevforall.sqlitedebugger.engine.QueryLimits @@ -8,6 +7,7 @@ import org.appdevforall.sqlitedebugger.engine.QueryResult import org.appdevforall.sqlitedebugger.engine.ServerConnection import org.json.JSONArray import org.json.JSONObject +import java.io.File /** * The debugger's MCP tools, all read-only: they open server connections through [engine] with @@ -44,24 +44,42 @@ class DatabaseTools( maxBlobBytes: Int = MAX_BLOB_BYTES, ): Any = when (value) { - null -> JSONObject.NULL - is Long -> encodeLong(value) - is Int -> value.toLong() - is Double -> + null -> { + JSONObject.NULL + } + + is Long -> { + encodeLong(value) + } + + is Int -> { + value.toLong() + } + + is Double -> { when { value.isNaN() -> "NaN" value == Double.POSITIVE_INFINITY -> "Infinity" value == Double.NEGATIVE_INFINITY -> "-Infinity" else -> value } - is String -> value - is ByteArray -> + } + + is String -> { + value + } + + is ByteArray -> { JSONObject() .put("base64", McpHttp.base64(value, urlSafe = false, pad = true)) .put("bytes", value.size) // the engine cuts at the limit without saying how long the BLOB was .put("truncated", value.size >= maxBlobBytes) - else -> value.toString() + } + + else -> { + value.toString() + } } fun encodeLong(value: Long): Any = if (value > MAX_SAFE_INTEGER || value < -MAX_SAFE_INTEGER) value.toString() else value @@ -87,14 +105,29 @@ class DatabaseTools( fun tools(): List = listOf( - tool("list_databases", "List databases", "Lists the app's databases: each one an app connection has open (available), and the files in the app's database folder that none has opened (not available: not inspectable until the app opens them).") { listDatabases() }, - tool("list_connections", "List connections", "Lists the app's open SQLite connections (not yet freed by SQLite), oldest first.") { listConnections() }, - tool("get_schema", "Get schema", "Returns the tables, indexes, views and triggers of a database, with their CREATE statements, from sqlite_schema.", databaseSchema()) { getSchema(it) }, + tool( + "list_databases", + "List databases", + "Lists the app's databases: each one an app connection has open (available), and the files in the app's database folder " + + "that none has opened (not available: not inspectable until the app opens them).", + ) { listDatabases() }, + tool( + "list_connections", + "List connections", + "Lists the app's open SQLite connections (not yet freed by SQLite), oldest first.", + ) { listConnections() }, + tool( + "get_schema", + "Get schema", + "Returns the tables, indexes, views and triggers of a database, with their CREATE statements, from sqlite_schema.", + databaseSchema(), + ) { getSchema(it) }, tool( "query", "Run a query", "Runs one read-only SQL statement. Returns at most max_rows rows (default $DEFAULT_MAX_ROWS, at most $MAX_ROWS); " + - "TEXT cells are cut at $MAX_TEXT_BYTES bytes and BLOBs at $MAX_BLOB_BYTES. Use offset to page. $PAGING_NOTE Writes are refused.", + "TEXT cells are cut at $MAX_TEXT_BYTES bytes and BLOBs at $MAX_BLOB_BYTES. Use offset to page. " + + "$PAGING_NOTE Writes are refused.", databaseSchema( "sql" to prop("string", "The SQL statement."), "args" to JSONObject().put("type", "array").put("description", "Values for ? parameters: null, numbers or strings."), @@ -113,18 +146,28 @@ class DatabaseTools( required = listOf("sql"), ), ) { explain(it) }, - tool("get_stats", "Get statistics", "Returns page_count, page_size, freelist_count, and the row count of each table when the database has fewer than $MAX_TABLES_FOR_COUNTS tables.", databaseSchema()) { getStats(it) }, + tool( + "get_stats", + "Get statistics", + "Returns page_count, page_size, freelist_count, and the row count of each table when the database has fewer than " + + "$MAX_TABLES_FOR_COUNTS tables.", + databaseSchema(), + ) { getStats(it) }, tool( "integrity_check", "Check integrity", - "Runs PRAGMA quick_check, or the slower PRAGMA integrity_check when full is true. ok is true when SQLite reports no problem.", + "Runs PRAGMA quick_check, or the slower PRAGMA integrity_check when full is true. " + + "ok is true when SQLite reports no problem.", databaseSchema("full" to prop("boolean", "Run the full integrity_check. Default false."), required = emptyList()), ) { integrityCheck(it) }, tool( "get_events", "Get SQL events", - "Returns the SQL the app ran, oldest first, after the cursor. Pass the returned next as after to continue; dropped counts events that left the buffer unread. The first call starts recording bound values into the SQL.", - JSONObject().put("type", "object").put("additionalProperties", false) + "Returns the SQL the app ran, oldest first, after the cursor. Pass the returned next as after to continue; " + + "dropped counts events that left the buffer unread. The first call starts recording bound values into the SQL.", + JSONObject() + .put("type", "object") + .put("additionalProperties", false) .put("properties", JSONObject().put("after", prop("integer", "Cursor: the next from the last call. Default 0."))) .put("required", JSONArray()), ) { getEvents(it) }, @@ -149,7 +192,10 @@ class DatabaseTools( ): JSONObject { val props = JSONObject().put("database", prop("string", "The database path, as list_databases gives it.")) extra.forEach { props.put(it.first, it.second) } - return JSONObject().put("type", "object").put("properties", props).put("additionalProperties", false) + return JSONObject() + .put("type", "object") + .put("properties", props) + .put("additionalProperties", false) .put("required", JSONArray(listOf("database") + required)) } @@ -173,22 +219,38 @@ class DatabaseTools( val list = JSONArray() for (c in engine.connections()) { list.put( - JSONObject().put("id", encodeLong(c.id)).put("path", c.path).put("read_only", c.readOnly).put("closing", c.closing) - .put("opened_at_epoch_nanos", encodeLong(c.openedAtEpochNanos)).put("open_thread_id", encodeLong(c.openThreadId)), + JSONObject() + .put("id", encodeLong(c.id)) + .put("path", c.path) + .put("read_only", c.readOnly) + .put("closing", c.closing) + .put("opened_at_epoch_nanos", encodeLong(c.openedAtEpochNanos)) + .put("open_thread_id", encodeLong(c.openThreadId)), ) } return JSONObject().put("connections", list) } private fun getSchema(args: JSONObject): JSONObject { - val result = run(args, "SELECT type, name, tbl_name, sql FROM sqlite_schema ORDER BY type, name", emptyList(), 0, QueryLimits(MAX_ROWS, 65536, MAX_BLOB_BYTES)) + val result = + run( + args, + "SELECT type, name, tbl_name, sql FROM sqlite_schema ORDER BY type, name", + emptyList(), + 0, + QueryLimits(MAX_ROWS, 65536, MAX_BLOB_BYTES), + ) val groups = linkedMapOf("table" to JSONArray(), "index" to JSONArray(), "view" to JSONArray(), "trigger" to JSONArray()) for (row in result.rows) { val item = JSONObject().put("name", encodeValue(row[1])).put("table", encodeValue(row[2])).put("sql", encodeValue(row[3])) groups[row[0] as? String]?.put(item) } - return JSONObject().put("tables", groups["table"]).put("indexes", groups["index"]).put("views", groups["view"]) - .put("triggers", groups["trigger"]).put("truncated", result.hasMore) + return JSONObject() + .put("tables", groups["table"]) + .put("indexes", groups["index"]) + .put("views", groups["view"]) + .put("triggers", groups["trigger"]) + .put("truncated", result.hasMore) } private fun query(args: JSONObject): JSONObject { @@ -202,7 +264,14 @@ class DatabaseTools( } private fun explain(args: JSONObject): JSONObject { - val result = run(args, "EXPLAIN QUERY PLAN " + stringArg(args, "sql"), bindArgs(args), 0, QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES)) + val result = + run( + args, + "EXPLAIN QUERY PLAN " + stringArg(args, "sql"), + bindArgs(args), + 0, + QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES), + ) return JSONObject().put("plan", rowsAsObjects(result)) } @@ -210,22 +279,42 @@ class DatabaseTools( val limits = QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES) val deadline = clock() + DEADLINE_MS val path = checkedPath(args) + fun pragma(name: String) = runChecked(path, "PRAGMA $name", emptyList(), 0, limits, deadline).rows.firstOrNull()?.firstOrNull() val pageCount = pragma("page_count") as? Long val pageSize = pragma("page_size") as? Long - val out = JSONObject().put("page_count", encodeValue(pageCount)).put("page_size", encodeValue(pageSize)) - .put("freelist_count", encodeValue(pragma("freelist_count"))) + val out = + JSONObject() + .put("page_count", encodeValue(pageCount)) + .put("page_size", encodeValue(pageSize)) + .put("freelist_count", encodeValue(pragma("freelist_count"))) if (pageCount != null && pageSize != null) out.put("size_bytes", encodeLong(pageCount * pageSize)) - val tables = runChecked(path, "SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", emptyList(), 0, limits, deadline) - .rows.map { it[0] as String } + val tables = + runChecked( + path, + "SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", + emptyList(), + 0, + limits, + deadline, + ).rows + .map { it[0] as String } if (tables.size < MAX_TABLES_FOR_COUNTS) { val counts = JSONObject() for (t in tables) { - val n = try { - runChecked(path, "SELECT COUNT(*) FROM ${quoteIdentifier(t)}", emptyList(), 0, limits, deadline).rows.firstOrNull()?.firstOrNull() - } catch (e: McpToolError) { - e.message - } + val n = + try { + runChecked( + path, + "SELECT COUNT(*) FROM ${quoteIdentifier(t)}", + emptyList(), + 0, + limits, + deadline, + ).rows.firstOrNull()?.firstOrNull() + } catch (e: McpToolError) { + e.message + } counts.put(t, encodeValue(n)) } out.put("row_counts", counts) @@ -236,16 +325,19 @@ class DatabaseTools( } private fun integrityCheck(args: JSONObject): JSONObject { - val full = when (val v = args.opt("full")) { - null, JSONObject.NULL -> false - is Boolean -> v - else -> throw McpToolError("full must be true or false.") - } + val full = + when (val v = args.opt("full")) { + null, JSONObject.NULL -> false + is Boolean -> v + else -> throw McpToolError("full must be true or false.") + } val pragma = if (full) "PRAGMA integrity_check" else "PRAGMA quick_check" val result = run(args, pragma, emptyList(), 0, QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES)) val messages = result.rows.map { it.firstOrNull()?.toString() ?: "" } - return JSONObject().put("check", if (full) "integrity_check" else "quick_check") - .put("ok", messages == listOf("ok")).put("messages", JSONArray(messages)) + return JSONObject() + .put("check", if (full) "integrity_check" else "quick_check") + .put("ok", messages == listOf("ok")) + .put("messages", JSONArray(messages)) } private fun getEvents(args: JSONObject): JSONObject { @@ -256,18 +348,25 @@ class DatabaseTools( clientConnected = true } } - val batch = try { - engine.readEvents(after, EVENT_BATCH_BYTES) - } catch (e: DebugEngineException) { - throw engineError(e) - } + val batch = + try { + engine.readEvents(after, EVENT_BATCH_BYTES) + } catch (e: DebugEngineException) { + throw engineError(e) + } val events = JSONArray() for (e in batch.events) { events.put( - JSONObject().put("seq", encodeLong(e.seq)).put("kind", e.kind.name.lowercase()).put("connection_id", encodeLong(e.connectionId)) - .put("thread_id", encodeLong(e.threadId)).put("time_epoch_nanos", encodeLong(e.timeEpochNanos)) + JSONObject() + .put("seq", encodeLong(e.seq)) + .put("kind", e.kind.name.lowercase()) + .put("connection_id", encodeLong(e.connectionId)) + .put("thread_id", encodeLong(e.threadId)) + .put("time_epoch_nanos", encodeLong(e.timeEpochNanos)) .put("duration_nanos", e.durationNanos?.let { encodeLong(it) } ?: JSONObject.NULL) - .put("sql", e.sql).put("truncated", e.truncated).put("expanded", e.expanded), + .put("sql", e.sql) + .put("truncated", e.truncated) + .put("expanded", e.expanded), ) } return JSONObject().put("next", encodeLong(batch.next)).put("dropped", encodeLong(batch.dropped)).put("events", events) @@ -279,8 +378,14 @@ class DatabaseTools( ): JSONObject { val rows = JSONArray() for (row in result.rows) rows.put(JSONArray().also { a -> row.forEach { a.put(encodeValue(it)) } }) - val out = JSONObject().put("columns", JSONArray(result.columns)).put("rows", rows).put("row_count", result.rows.size) - .put("offset", encodeLong(offset)).put("has_more", result.hasMore).put("truncated_cells", result.truncatedCells) + val out = + JSONObject() + .put("columns", JSONArray(result.columns)) + .put("rows", rows) + .put("row_count", result.rows.size) + .put("offset", encodeLong(offset)) + .put("has_more", result.hasMore) + .put("truncated_cells", result.truncatedCells) if (result.hasMore) out.put("next_offset", encodeLong(offset + result.rows.size)) return out } @@ -297,7 +402,12 @@ class DatabaseTools( private fun checkedPath(args: JSONObject): String { val path = stringArg(args, "database") - val known = engine.connections().map { it.path }.filter { it.isNotEmpty() }.distinct() + val known = + engine + .connections() + .map { it.path } + .filter { it.isNotEmpty() } + .distinct() if (path !in known) { throw McpToolError( "Unknown database \"$path\": use a path from list_databases that is available.", @@ -360,7 +470,8 @@ class DatabaseTools( if (d != Math.floor(d) || d.isInfinite() || Math.abs(d) >= 9.2e18) throw McpToolError("$name must be an integer.") } val l = n.toLong() - if (l < min || l > max) throw McpToolError("$name must be between $min and ${if (max == Long.MAX_VALUE) "2^63-1" else max.toString()}.") + val upper = if (max == Long.MAX_VALUE) "2^63-1" else max.toString() + if (l < min || l > max) throw McpToolError("$name must be between $min and $upper.") return l } diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt index 7d58e73556..f29b5acad3 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt @@ -1,5 +1,6 @@ package org.appdevforall.sqlitedebugger.mcp +import org.json.JSONObject import java.io.BufferedInputStream import java.io.ByteArrayOutputStream import java.io.IOException @@ -16,7 +17,6 @@ import java.util.Collections import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger import kotlin.concurrent.thread -import org.json.JSONObject /** * The MCP endpoint's HTTP side: a small HTTP/1.1 listener on 127.0.0.1 only, so it's never @@ -47,7 +47,11 @@ class McpHttp( fun newToken(): String = base64(ByteArray(32).also { java.security.SecureRandom().nextBytes(it) }, urlSafe = true, pad = false) /** By hand: java.util.Base64 needs API 26 and this library's minSdk is 21. */ - fun base64(bytes: ByteArray, urlSafe: Boolean, pad: Boolean): String { + fun base64( + bytes: ByteArray, + urlSafe: Boolean, + pad: Boolean, + ): String { val out = StringBuilder() var i = 0 while (i < bytes.size) { @@ -56,7 +60,15 @@ class McpHttp( for (k in 0 until 3) v = (v shl 8) or (if (k < n) bytes[i + k].toInt() and 0xff else 0) for (k in 0..n) { val idx = (v shr (18 - 6 * k)) and 63 - out.append(if (idx < 62) B64[idx] else if (idx == 62) (if (urlSafe) '-' else '+') else (if (urlSafe) '_' else '/')) + out.append( + if (idx < 62) { + B64[idx] + } else if (idx == 62) { + (if (urlSafe) '-' else '+') + } else { + (if (urlSafe) '_' else '/') + }, + ) } if (pad) repeat(3 - n) { out.append('=') } i += 3 @@ -68,21 +80,44 @@ class McpHttp( const val MAX_CONNECTIONS = 8 private const val MAX_HEADER_BYTES = 16 * 1024 const val MAX_BODY_BYTES = 1024 * 1024 + /** A connection with no request for this long is closed. */ const val IDLE_TIMEOUT_MS = 30_000 - private val STATUS = mapOf( - 200 to "OK", 202 to "Accepted", 204 to "No Content", 400 to "Bad Request", 401 to "Unauthorized", - 403 to "Forbidden", 404 to "Not Found", 405 to "Method Not Allowed", 413 to "Payload Too Large", - 415 to "Unsupported Media Type", 431 to "Request Header Fields Too Large", 500 to "Internal Server Error", - 501 to "Not Implemented", 503 to "Service Unavailable", - ) + private val STATUS = + mapOf( + 200 to "OK", + 202 to "Accepted", + 204 to "No Content", + 400 to "Bad Request", + 401 to "Unauthorized", + 403 to "Forbidden", + 404 to "Not Found", + 405 to "Method Not Allowed", + 413 to "Payload Too Large", + 415 to "Unsupported Media Type", + 431 to "Request Header Fields Too Large", + 500 to "Internal Server Error", + 501 to "Not Implemented", + 503 to "Service Unavailable", + ) private val LOOPBACK_NAMES = setOf("127.0.0.1", "localhost") } - private class Request(val method: String, val target: String, val version: String, val headers: Map, val body: ByteArray) - private class HttpError(val status: Int, message: String) : Exception(message) + private class Request( + val method: String, + val target: String, + val version: String, + val headers: Map, + val body: ByteArray, + ) + + private class HttpError( + val status: Int, + message: String, + ) : Exception(message) @Volatile private var socket: ServerSocket? = null + @Volatile private var running = false private val open = AtomicInteger(0) private val clients: MutableSet = Collections.newSetFromMap(ConcurrentHashMap()) @@ -97,7 +132,10 @@ class McpHttp( s.reuseAddress = true s.bind(InetSocketAddress(InetAddress.getByAddress(byteArrayOf(127, 0, 0, 1)), port), 16) } catch (e: IOException) { - try { s.close() } catch (ignored: IOException) { } + try { + s.close() + } catch (ignored: IOException) { + } throw IOException("Can't listen on 127.0.0.1:$port: ${e.message}", e) } socket = s @@ -108,7 +146,10 @@ class McpHttp( /** Stops listening and closes every connection. */ fun stop() { running = false - try { socket?.close() } catch (ignored: IOException) { } + try { + socket?.close() + } catch (ignored: IOException) { + } socket = null clients.forEach { close(it) } clients.clear() @@ -116,17 +157,24 @@ class McpHttp( private fun acceptLoop(s: ServerSocket) { while (running && !s.isClosed) { - val c = try { s.accept() } catch (e: IOException) { - if (running && !s.isClosed) log("couldn't accept a connection: ${e.message}") + val c = + try { + s.accept() + } catch (e: IOException) { + if (running && !s.isClosed) log("couldn't accept a connection: ${e.message}") + continue + } + if (!c.inetAddress.isLoopbackAddress) { + close(c) continue } - if (!c.inetAddress.isLoopbackAddress) { close(c); continue } if (open.incrementAndGet() > MAX_CONNECTIONS) { open.decrementAndGet() try { c.soTimeout = 1000 respond(c.getOutputStream(), 503, rpcError("Too many connections at once; try again."), emptyMap(), close = true) - } catch (ignored: IOException) { } + } catch (ignored: IOException) { + } close(c) continue } @@ -153,22 +201,31 @@ class McpHttp( val input = BufferedInputStream(c.getInputStream()) val out = c.getOutputStream() while (running) { - val req = try { - readRequest(input) ?: return - } catch (e: SocketTimeoutException) { - return - } catch (e: HttpError) { - respond(out, e.status, rpcError(e.message ?: "Bad request."), emptyMap(), close = true) - return - } + val req = + try { + readRequest(input) ?: return + } catch (e: SocketTimeoutException) { + return + } catch (e: HttpError) { + respond(out, e.status, rpcError(e.message ?: "Bad request."), emptyMap(), close = true) + return + } if (!handle(req, out)) return } } /** Answers one request. Returns whether the connection stays open for another. */ - private fun handle(r: Request, out: OutputStream): Boolean { + private fun handle( + r: Request, + out: OutputStream, + ): Boolean { val close = r.version == "HTTP/1.0" || r.headers["connection"]?.lowercase()?.contains("close") == true - fun send(status: Int, body: String?, headers: Map = emptyMap()): Boolean { + + fun send( + status: Int, + body: String?, + headers: Map = emptyMap(), + ): Boolean { respond(out, status, body, headers, close) return !close } @@ -177,25 +234,45 @@ class McpHttp( if (origin != null && !originAllowed(origin)) return send(403, rpcError("Requests from web pages aren't accepted.")) if (r.target.substringBefore('?') != PATH) return send(404, rpcError("Not found: the MCP endpoint is $PATH.")) if (!authorized(r.headers["authorization"])) { - return send(401, rpcError("Missing or wrong token. Code on the Go gives it to its Agent; send it as Authorization: Bearer ."), - mapOf("WWW-Authenticate" to "Bearer realm=\"SQLite debugger\"")) + return send( + 401, + rpcError( + "Missing or wrong token. Code on the Go gives it to its Agent; " + + "send it as Authorization: Bearer .", + ), + mapOf("WWW-Authenticate" to "Bearer realm=\"SQLite debugger\""), + ) } val session = r.headers["mcp-session-id"] - val reply = when (r.method) { - "POST" -> { - val type = r.headers["content-type"]?.substringBefore(';')?.trim()?.lowercase() - if (type != null && type != "application/json") return send(415, rpcError("Send the request as application/json.")) - try { - protocol.post(String(r.body, Charsets.UTF_8), session, r.headers["mcp-protocol-version"]) - } catch (e: Exception) { - log("a request failed: ${e.javaClass.simpleName}: ${e.message}") - McpProtocol.Reply(500, rpcError("The server failed: ${e.message ?: e.javaClass.simpleName}")) + val reply = + when (r.method) { + "POST" -> { + val type = + r.headers["content-type"] + ?.substringBefore(';') + ?.trim() + ?.lowercase() + if (type != null && type != "application/json") return send(415, rpcError("Send the request as application/json.")) + try { + protocol.post(String(r.body, Charsets.UTF_8), session, r.headers["mcp-protocol-version"]) + } catch (e: Exception) { + log("a request failed: ${e.javaClass.simpleName}: ${e.message}") + McpProtocol.Reply(500, rpcError("The server failed: ${e.message ?: e.javaClass.simpleName}")) + } + } + + "DELETE" -> { + protocol.delete(session) + } + + "GET" -> { + return send(405, rpcError("This server doesn't offer an event stream: use POST."), mapOf("Allow" to "POST, DELETE")) + } + + else -> { + return send(405, rpcError("Use POST."), mapOf("Allow" to "POST, DELETE")) } } - "DELETE" -> protocol.delete(session) - "GET" -> return send(405, rpcError("This server doesn't offer an event stream: use POST."), mapOf("Allow" to "POST, DELETE")) - else -> return send(405, rpcError("Use POST."), mapOf("Allow" to "POST, DELETE")) - } return send(reply.status, reply.body, reply.headers) } @@ -209,10 +286,13 @@ class McpHttp( return name in LOOPBACK_NAMES } - private fun originAllowed(origin: String): Boolean = try { - val u = URI(origin.trim()) - (u.scheme == "http" || u.scheme == "https") && u.host?.lowercase() in LOOPBACK_NAMES - } catch (e: Exception) { false } + private fun originAllowed(origin: String): Boolean = + try { + val u = URI(origin.trim()) + (u.scheme == "http" || u.scheme == "https") && u.host?.lowercase() in LOOPBACK_NAMES + } catch (e: Exception) { + false + } private fun authorized(header: String?): Boolean { val expected = token() @@ -227,11 +307,14 @@ class McpHttp( /** One request, or null if the connection closed before one began. */ private fun readRequest(input: InputStream): Request? { var budget = MAX_HEADER_BYTES + fun line(): String? { val b = ByteArrayOutputStream() while (true) { val ch = input.read() - if (ch < 0) { if (b.size() == 0) return null else throw HttpError(400, "The request ended early.") } + if (ch < 0) { + if (b.size() == 0) return null else throw HttpError(400, "The request ended early.") + } if (--budget < 0) throw HttpError(431, "The request's headers are too long.") if (ch == '\n'.code) break b.write(ch) @@ -239,7 +322,7 @@ class McpHttp( return b.toString("ISO-8859-1").removeSuffix("\r") } var first = line() ?: return null - while (first.isEmpty()) first = line() ?: return null // stray line ends between requests + while (first.isEmpty()) first = line() ?: return null // stray line ends between requests val parts = first.split(' ') if (parts.size != 3 || !parts[2].startsWith("HTTP/1.")) throw HttpError(400, "That isn't an HTTP/1.x request.") val headers = HashMap() @@ -256,7 +339,10 @@ class McpHttp( return Request(parts[0], parts[1], parts[2], headers, readBody(input, headers)) } - private fun readBody(input: InputStream, h: Map): ByteArray { + private fun readBody( + input: InputStream, + h: Map, + ): ByteArray { val te = h["transfer-encoding"] if (te != null) { if (!te.equals("chunked", ignoreCase = true)) throw HttpError(501, "Transfer-Encoding $te isn't supported.") @@ -273,7 +359,12 @@ class McpHttp( val out = ByteArrayOutputStream() while (true) { val sizeLine = shortLine(input, 1024) ?: throw HttpError(400, "The request ended early.") - val size = sizeLine.substringBefore(';').trim().toIntOrNull(16)?.takeIf { it >= 0 } ?: throw HttpError(400, "A bad chunk size.") + val size = + sizeLine + .substringBefore(';') + .trim() + .toIntOrNull(16) + ?.takeIf { it >= 0 } ?: throw HttpError(400, "A bad chunk size.") if (out.size().toLong() + size > MAX_BODY_BYTES) throw HttpError(413, "The request is larger than ${MAX_BODY_BYTES / 1024} KB.") if (size == 0) { // Trailers, if any, up to a blank line. @@ -290,7 +381,10 @@ class McpHttp( } } - private fun shortLine(input: InputStream, max: Int): String? { + private fun shortLine( + input: InputStream, + max: Int, + ): String? { val b = ByteArrayOutputStream() var left = max while (true) { @@ -303,7 +397,10 @@ class McpHttp( return b.toString("ISO-8859-1").removeSuffix("\r") } - private fun readExactly(input: InputStream, n: Int): ByteArray { + private fun readExactly( + input: InputStream, + n: Int, + ): ByteArray { val b = ByteArray(n) var off = 0 while (off < n) { @@ -314,14 +411,26 @@ class McpHttp( return b } - private fun respond(out: OutputStream, status: Int, body: String?, headers: Map, close: Boolean) { + private fun respond( + out: OutputStream, + status: Int, + body: String?, + headers: Map, + close: Boolean, + ) { val bytes = body?.toByteArray(Charsets.UTF_8) ?: ByteArray(0) val head = StringBuilder("HTTP/1.1 $status ${STATUS[status] ?: "Status"}\r\n") if (body != null) head.append("Content-Type: application/json; charset=utf-8\r\n") if (status != 204) head.append("Content-Length: ${bytes.size}\r\n") head.append("Cache-Control: no-store\r\n") if (close) head.append("Connection: close\r\n") - for ((k, v) in headers) head.append(k).append(": ").append(v.replace("\r", "").replace("\n", "")).append("\r\n") + for ((k, v) in headers) { + head + .append(k) + .append(": ") + .append(v.replace("\r", "").replace("\n", "")) + .append("\r\n") + } head.append("\r\n") out.write(head.toString().toByteArray(Charsets.ISO_8859_1)) if (status != 204) out.write(bytes) @@ -329,8 +438,17 @@ class McpHttp( } /** An HTTP-level refusal, as a JSON-RPC error with no id, which MCP clients know how to show. */ - private fun rpcError(message: String): String = JSONObject().put("jsonrpc", "2.0").put("id", JSONObject.NULL) - .put("error", JSONObject().put("code", McpProtocol.INVALID_REQUEST).put("message", message)).toString() + private fun rpcError(message: String): String = + JSONObject() + .put("jsonrpc", "2.0") + .put("id", JSONObject.NULL) + .put("error", JSONObject().put("code", McpProtocol.INVALID_REQUEST).put("message", message)) + .toString() - private fun close(s: Socket) { try { s.close() } catch (ignored: IOException) { } } + private fun close(s: Socket) { + try { + s.close() + } catch (ignored: IOException) { + } + } } diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt index 01e0877b1d..6ee7c1b128 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt @@ -18,15 +18,27 @@ class McpTool( val readOnly: Boolean = true, val run: (JSONObject) -> JSONObject, ) { - fun describe(): JSONObject = JSONObject() - .put("name", name).put("title", title).put("description", description) - .put("inputSchema", inputSchema) - .put("annotations", JSONObject().put("title", title).put("readOnlyHint", readOnly) - .put("destructiveHint", false).put("openWorldHint", false)) + fun describe(): JSONObject = + JSONObject() + .put("name", name) + .put("title", title) + .put("description", description) + .put("inputSchema", inputSchema) + .put( + "annotations", + JSONObject() + .put("title", title) + .put("readOnlyHint", readOnly) + .put("destructiveHint", false) + .put("openWorldHint", false), + ) } /** A tool's failure, told to the client as a tool result with isError, with [details] when there are any. */ -class McpToolError(message: String, val details: JSONObject? = null) : Exception(message) +class McpToolError( + message: String, + val details: JSONObject? = null, +) : Exception(message) /** * The Model Context Protocol, as the SQLite debugger speaks it: JSON-RPC 2.0 over the Streamable HTTP @@ -52,31 +64,54 @@ class McpProtocol( const val INVALID_REQUEST = -32600 const val METHOD_NOT_FOUND = -32601 const val INVALID_PARAMS = -32602 - const val INSTRUCTIONS = "Inspects and queries the SQLite databases of the app this server runs in, a debug " + - "build made with Code on the Go. Read-only unless the app's developer turned on read-write mode." + const val INSTRUCTIONS = + "Inspects and queries the SQLite databases of the app this server runs in, a debug " + + "build made with Code on the Go. Read-only unless the app's developer turned on read-write mode." } /** What to send back: an HTTP status, a JSON body (or none), and extra headers. */ - class Reply(val status: Int, val body: String?, val headers: Map = emptyMap()) + class Reply( + val status: Int, + val body: String?, + val headers: Map = emptyMap(), + ) /** Live sessions, oldest first; the oldest is dropped when there are too many. */ private val sessions = LinkedHashSet() /** A POST to the endpoint: one JSON-RPC message, or a batch of them. */ - fun post(body: String, sessionId: String?, versionHeader: String?): Reply { - val parsed = try { JSONTokener(body).nextValue() } catch (e: Exception) { null } - if (parsed !is JSONObject && parsed !is JSONArray) return Reply(400, error(null, PARSE_ERROR, "That isn't a JSON object or array.").toString()) + fun post( + body: String, + sessionId: String?, + versionHeader: String?, + ): Reply { + val parsed = + try { + JSONTokener(body).nextValue() + } catch (e: Exception) { + null + } + if (parsed !is JSONObject && parsed !is JSONArray) { + return Reply(400, error(null, PARSE_ERROR, "That isn't a JSON object or array.").toString()) + } val messages: List = if (parsed is JSONArray) (0 until parsed.length()).map { parsed.opt(it) } else listOf(parsed) if (messages.isEmpty()) return Reply(400, error(null, INVALID_REQUEST, "An empty batch.").toString()) if (messages.any { (it as? JSONObject)?.optString("method") == "initialize" }) { - if (messages.size != 1) return Reply(400, error(null, INVALID_REQUEST, "Send initialize on its own, not in a batch.").toString()) + if (messages.size != 1) { + return Reply(400, error(null, INVALID_REQUEST, "Send initialize on its own, not in a batch.").toString()) + } return initialize(messages[0] as JSONObject) } - if (sessionId.isNullOrEmpty()) return Reply(400, error(null, INVALID_REQUEST, "No $SESSION_HEADER header: send initialize first.").toString()) - if (!touch(sessionId)) return Reply(404, error(null, INVALID_REQUEST, "That session has ended or doesn't exist: send initialize again.").toString()) + if (sessionId.isNullOrEmpty()) { + return Reply(400, error(null, INVALID_REQUEST, "No $SESSION_HEADER header: send initialize first.").toString()) + } + if (!touch(sessionId)) { + return Reply(404, error(null, INVALID_REQUEST, "That session has ended or doesn't exist: send initialize again.").toString()) + } if (versionHeader != null && versionHeader !in VERSIONS) { - return Reply(400, error(null, INVALID_REQUEST, "Unsupported MCP-Protocol-Version $versionHeader; this server speaks ${VERSIONS.joinToString()}.").toString()) + val detail = "Unsupported MCP-Protocol-Version $versionHeader; this server speaks ${VERSIONS.joinToString()}." + return Reply(400, error(null, INVALID_REQUEST, detail).toString()) } val replies = messages.mapNotNull { handle(it) } if (replies.isEmpty()) return Reply(202, null) @@ -87,7 +122,11 @@ class McpProtocol( fun delete(sessionId: String?): Reply { if (sessionId.isNullOrEmpty()) return Reply(400, error(null, INVALID_REQUEST, "No $SESSION_HEADER header.").toString()) val removed = synchronized(sessions) { sessions.remove(sessionId) } - return if (removed) Reply(204, null) else Reply(404, error(null, INVALID_REQUEST, "That session has ended or doesn't exist.").toString()) + return if (removed) { + Reply(204, null) + } else { + Reply(404, error(null, INVALID_REQUEST, "That session has ended or doesn't exist.").toString()) + } } /** Ends every session, so clients must initialize again. */ @@ -95,11 +134,12 @@ class McpProtocol( fun sessionCount(): Int = synchronized(sessions) { sessions.size } - private fun touch(id: String): Boolean = synchronized(sessions) { - if (!sessions.remove(id)) return false - sessions += id // most recently used goes last, so it's dropped last - true - } + private fun touch(id: String): Boolean = + synchronized(sessions) { + if (!sessions.remove(id)) return false + sessions += id // most recently used goes last, so it's dropped last + true + } private fun initialize(m: JSONObject): Reply { val id = m.opt("id") @@ -112,11 +152,12 @@ class McpProtocol( sessions += session while (sessions.size > maxSessions) sessions.remove(sessions.first()) } - val result = JSONObject() - .put("protocolVersion", version) - .put("capabilities", JSONObject().put("tools", JSONObject().put("listChanged", false))) - .put("serverInfo", JSONObject().put("name", SERVER_NAME).put("title", "SQLite debugger").put("version", serverVersion)) - .put("instructions", INSTRUCTIONS) + val result = + JSONObject() + .put("protocolVersion", version) + .put("capabilities", JSONObject().put("tools", JSONObject().put("listChanged", false))) + .put("serverInfo", JSONObject().put("name", SERVER_NAME).put("title", "SQLite debugger").put("version", serverVersion)) + .put("instructions", INSTRUCTIONS) return Reply(200, ok(id, result).toString(), mapOf(SESSION_HEADER to session)) } @@ -131,7 +172,7 @@ class McpProtocol( if (m.has("result") || m.has("error")) return null return error(if (validId(id)) id else null, INVALID_REQUEST, "A request needs a method.") } - if (!hasId) return null // a notification, such as notifications/initialized: no answer + if (!hasId) return null // a notification, such as notifications/initialized: no answer if (!validId(id)) return error(null, INVALID_REQUEST, "id must be a string or a number.") if (m.optString("jsonrpc") != "2.0") return error(id, INVALID_REQUEST, "jsonrpc must be \"2.0\".") return when (method) { @@ -142,31 +183,50 @@ class McpProtocol( } } - private fun call(id: Any, rawParams: Any?): JSONObject { + private fun call( + id: Any, + rawParams: Any?, + ): JSONObject { val params = rawParams as? JSONObject ?: return error(id, INVALID_PARAMS, "tools/call needs params, with the tool's name.") val name = params.opt("name") as? String ?: return error(id, INVALID_PARAMS, "tools/call needs the tool's name.") val tool = tools.firstOrNull { it.name == name } ?: return error(id, INVALID_PARAMS, "Unknown tool \"$name\".") - val args = when (val a = params.opt("arguments")) { - null, JSONObject.NULL -> JSONObject() - is JSONObject -> a - else -> return error(id, INVALID_PARAMS, "arguments must be an object.") - } - val result = try { - val out = tool.run(args) - JSONObject().put("content", JSONArray().put(text(out.toString(2)))).put("structuredContent", out).put("isError", false) - } catch (e: McpToolError) { - JSONObject().put("content", JSONArray().put(text(e.message ?: "The tool failed."))).put("isError", true).also { r -> - e.details?.let { r.put("structuredContent", it) } + val args = + when (val a = params.opt("arguments")) { + null, JSONObject.NULL -> JSONObject() + is JSONObject -> a + else -> return error(id, INVALID_PARAMS, "arguments must be an object.") + } + val result = + try { + val out = tool.run(args) + JSONObject().put("content", JSONArray().put(text(out.toString(2)))).put("structuredContent", out).put("isError", false) + } catch (e: McpToolError) { + JSONObject().put("content", JSONArray().put(text(e.message ?: "The tool failed."))).put("isError", true).also { r -> + e.details?.let { r.put("structuredContent", it) } + } + } catch (e: Exception) { + JSONObject() + .put("content", JSONArray().put(text("The tool failed: ${e.message ?: e.javaClass.simpleName}"))) + .put("isError", true) } - } catch (e: Exception) { - JSONObject().put("content", JSONArray().put(text("The tool failed: ${e.message ?: e.javaClass.simpleName}"))).put("isError", true) - } return ok(id, result) } private fun validId(id: Any?) = id is String || id is Number + private fun text(t: String) = JSONObject().put("type", "text").put("text", t) - private fun ok(id: Any, result: JSONObject) = JSONObject().put("jsonrpc", "2.0").put("id", id).put("result", result) - private fun error(id: Any?, code: Int, message: String) = JSONObject().put("jsonrpc", "2.0").put("id", id ?: JSONObject.NULL) + + private fun ok( + id: Any, + result: JSONObject, + ) = JSONObject().put("jsonrpc", "2.0").put("id", id).put("result", result) + + private fun error( + id: Any?, + code: Int, + message: String, + ) = JSONObject() + .put("jsonrpc", "2.0") + .put("id", id ?: JSONObject.NULL) .put("error", JSONObject().put("code", code).put("message", message)) } diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt index 7e15669b14..702bc4c75f 100644 --- a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt @@ -1,6 +1,5 @@ package org.appdevforall.sqlitedebugger.mcp -import java.io.File import org.appdevforall.sqlitedebugger.engine.AppConnection import org.appdevforall.sqlitedebugger.engine.DebugEngineException import org.appdevforall.sqlitedebugger.engine.EventKind @@ -12,6 +11,7 @@ import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Test +import java.io.File class DatabaseToolsTest { private val engine = FakeDebugEngine() @@ -32,11 +32,21 @@ class DatabaseToolsTest { args: JSONObject = JSONObject(), ): JSONObject { val protocol = McpProtocol(t.tools(), "test", { "s" }, 4) - val init = JSONObject().put("jsonrpc", "2.0").put("id", 1).put("method", "initialize") - .put("params", JSONObject().put("protocolVersion", "2025-06-18")).toString() + val init = + JSONObject() + .put("jsonrpc", "2.0") + .put("id", 1) + .put("method", "initialize") + .put("params", JSONObject().put("protocolVersion", "2025-06-18")) + .toString() val session = protocol.post(init, null, null).headers[McpProtocol.SESSION_HEADER]!! - val call = JSONObject().put("jsonrpc", "2.0").put("id", 2).put("method", "tools/call") - .put("params", JSONObject().put("name", name).put("arguments", args)).toString() + val call = + JSONObject() + .put("jsonrpc", "2.0") + .put("id", 2) + .put("method", "tools/call") + .put("params", JSONObject().put("name", name).put("arguments", args)) + .toString() return JSONObject(protocol.post(call, session, null).body!!).getJSONObject("result") } @@ -62,7 +72,16 @@ class DatabaseToolsTest { @Test fun everyToolIsReadOnly() { val list = tools.tools() assertEquals( - listOf("list_databases", "list_connections", "get_schema", "query", "explain_query_plan", "get_stats", "integrity_check", "get_events"), + listOf( + "list_databases", + "list_connections", + "get_schema", + "query", + "explain_query_plan", + "get_stats", + "integrity_check", + "get_events", + ), list.map { it.name }, ) list.forEach { assertTrue(it.name, it.describe().getJSONObject("annotations").getBoolean("readOnlyHint")) } @@ -78,7 +97,10 @@ class DatabaseToolsTest { } @Test fun listDatabasesAddsUnopenedFilesButNotSideFiles() { - val d = kotlin.io.path.createTempDirectory("dbs").toFile() + val d = + kotlin.io.path + .createTempDirectory("dbs") + .toFile() try { listOf("open.db", "closed.db", "closed.db-wal", "closed.db-shm", "closed.db-journal").forEach { File(d, it).writeText("x") } dir = d @@ -104,7 +126,15 @@ class DatabaseToolsTest { @Test fun everyDatabaseToolRefusesAnUnknownPathBeforeTheEngine() { engine.appConnections += conn(1, "/data/a.db") - for ((name, extra) in listOf("get_schema" to JSONObject(), "query" to JSONObject().put("sql", "select 1"), "explain_query_plan" to JSONObject().put("sql", "select 1"), "get_stats" to JSONObject(), "integrity_check" to JSONObject())) { + val cases = + listOf( + "get_schema" to JSONObject(), + "query" to JSONObject().put("sql", "select 1"), + "explain_query_plan" to JSONObject().put("sql", "select 1"), + "get_stats" to JSONObject(), + "integrity_check" to JSONObject(), + ) + for ((name, extra) in cases) { for (bad in listOf("/data/other.db", "", "/data/a.db-wal")) { val r = runTool(tools, name, db(bad).also { o -> extra.keys().forEach { k -> o.put(k, extra.get(k)) } }) assertTrue("$name $bad", r.getBoolean("isError")) @@ -131,12 +161,23 @@ class DatabaseToolsTest { assertTrue(r.getBoolean("has_more")) assertEquals(3, r.getInt("next_offset")) assertTrue(r.getString("note").contains("re-runs")) - assertTrue(tools.tools().first { it.name == "query" }.description.contains("re-runs")) + assertTrue( + tools + .tools() + .first { it.name == "query" } + .description + .contains("re-runs"), + ) } @Test fun queryChecksItsArguments() { engine.appConnections += conn(1, "/data/a.db") - fun bad(vararg kv: Pair) = assertTrue(kv.toString(), runTool(tools, "query", db().also { o -> kv.forEach { o.put(it.first, it.second) } }).getBoolean("isError")) + + fun bad(vararg kv: Pair) = + assertTrue( + kv.toString(), + runTool(tools, "query", db().also { o -> kv.forEach { o.put(it.first, it.second) } }).getBoolean("isError"), + ) bad() bad("sql" to "x", "max_rows" to 1001) bad("sql" to "x", "max_rows" to 0) @@ -150,8 +191,22 @@ class DatabaseToolsTest { @Test fun queryBindsArgsByType() { engine.appConnections += conn(1, "/data/a.db") var got: List = emptyList() - engine.answer = { _, _, _, a -> got = a; QueryResult(emptyList(), emptyList(), 0, false) } - ok("query", db().put("sql", "x").put("args", JSONArray().put(JSONObject.NULL).put(3).put(2.5).put("s").put(true))) + engine.answer = { _, _, _, a -> + got = a + QueryResult(emptyList(), emptyList(), 0, false) + } + ok( + "query", + db().put("sql", "x").put( + "args", + JSONArray() + .put(JSONObject.NULL) + .put(3) + .put(2.5) + .put("s") + .put(true), + ), + ) assertEquals(listOf(null, 3L, 2.5, "s", 1L), got) } @@ -169,16 +224,26 @@ class DatabaseToolsTest { val real = engine engine.answer = { _, _, _, _ -> QueryResult(emptyList(), emptyList(), 0, false) } // the fake ignores the deadline, so check the arithmetic through the constant and a wrapper engine - val wrapper = object : org.appdevforall.sqlitedebugger.engine.DebugEngine by real { - override fun openServerConnection(path: String, writable: Boolean) = real.openServerConnection(path, writable).let { c -> - object : org.appdevforall.sqlitedebugger.engine.ServerConnection by c { - override fun query(sql: String, args: List, offset: Long, limits: org.appdevforall.sqlitedebugger.engine.QueryLimits, deadlineMillis: Long): QueryResult { - deadline = deadlineMillis - return c.query(sql, args, offset, limits, deadlineMillis) + val wrapper = + object : org.appdevforall.sqlitedebugger.engine.DebugEngine by real { + override fun openServerConnection( + path: String, + writable: Boolean, + ) = real.openServerConnection(path, writable).let { c -> + object : org.appdevforall.sqlitedebugger.engine.ServerConnection by c { + override fun query( + sql: String, + args: List, + offset: Long, + limits: org.appdevforall.sqlitedebugger.engine.QueryLimits, + deadlineMillis: Long, + ): QueryResult { + deadline = deadlineMillis + return c.query(sql, args, offset, limits, deadlineMillis) + } } } } - } ok("query", db().put("sql", "x"), DatabaseTools(wrapper, null, { now })) assertEquals(now + 50_000, deadline) } @@ -189,7 +254,18 @@ class DatabaseToolsTest { engine.answer = { _, _, _, _ -> QueryResult( listOf("a", "b", "c", "d", "e", "f", "g", "h"), - listOf(listOf(Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, big, -big, 9_007_199_254_740_992L, byteArrayOf(1, 2, 3), null)), + listOf( + listOf( + Double.NaN, + Double.POSITIVE_INFINITY, + Double.NEGATIVE_INFINITY, + big, + -big, + 9_007_199_254_740_992L, + byteArrayOf(1, 2, 3), + null, + ), + ), 0, false, ) @@ -213,17 +289,18 @@ class DatabaseToolsTest { @Test fun getSchemaGroupsByType() { engine.appConnections += conn(1, "/data/a.db") answerWith( - "SELECT type" to QueryResult( - listOf("type", "name", "tbl_name", "sql"), - listOf( - listOf("index", "i1", "t", "CREATE INDEX i1 ON t(a)"), - listOf("table", "t", "t", "CREATE TABLE t(a)"), - listOf("trigger", "tr", "t", "CREATE TRIGGER tr ..."), - listOf("view", "v", "v", "CREATE VIEW v AS ..."), + "SELECT type" to + QueryResult( + listOf("type", "name", "tbl_name", "sql"), + listOf( + listOf("index", "i1", "t", "CREATE INDEX i1 ON t(a)"), + listOf("table", "t", "t", "CREATE TABLE t(a)"), + listOf("trigger", "tr", "t", "CREATE TRIGGER tr ..."), + listOf("view", "v", "v", "CREATE VIEW v AS ..."), + ), + 0, + false, ), - 0, - false, - ), ) val r = ok("get_schema", db()) assertEquals("t", r.getJSONArray("tables").getJSONObject(0).getString("name")) @@ -235,7 +312,15 @@ class DatabaseToolsTest { @Test fun explainQueryPlanPrefixesTheSql() { engine.appConnections += conn(1, "/data/a.db") - answerWith("EXPLAIN QUERY PLAN" to QueryResult(listOf("id", "parent", "notused", "detail"), listOf(listOf(3L, 0L, 0L, "SCAN t")), 0, false)) + answerWith( + "EXPLAIN QUERY PLAN" to + QueryResult( + listOf("id", "parent", "notused", "detail"), + listOf(listOf(3L, 0L, 0L, "SCAN t")), + 0, + false, + ), + ) val plan = ok("explain_query_plan", db().put("sql", "select * from t")).getJSONArray("plan") assertEquals("SCAN t", plan.getJSONObject(0).getString("detail")) assertEquals("EXPLAIN QUERY PLAN select * from t", seenSql.single()) @@ -243,6 +328,7 @@ class DatabaseToolsTest { @Test fun getStatsCountsRowsUnderTenTables() { engine.appConnections += conn(1, "/data/a.db") + fun one(v: Long) = QueryResult(listOf("x"), listOf(listOf(v)), 0, false) answerWith( "PRAGMA page_count" to one(10), diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt index 9d705d3bda..ecad15a633 100644 --- a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt @@ -16,23 +16,39 @@ import java.net.Socket * HTTP side (token, Host and Origin checks, methods and paths), on a real loopback socket. */ class McpProtocolTest { - private var sessionNumber = 0 private val echo = McpTool("echo", "Echo", "Returns its arguments.") { args -> JSONObject().put("got", args) } - private val needsPerson = McpTool("needs_person", "Needs a person", "Always says a person is needed.", readOnly = false) { _ -> - throw McpToolError("A person is needed: tap Allow on the sharing phone.", JSONObject().put("person_needed", true)) - } + private val needsPerson = + McpTool("needs_person", "Needs a person", "Always says a person is needed.", readOnly = false) { _ -> + throw McpToolError("A person is needed: tap Allow on the sharing phone.", JSONObject().put("person_needed", true)) + } private val crashes = McpTool("crashes", "Crashes", "Throws an unexpected exception.") { _ -> throw IllegalStateException("boom") } private fun protocol(maxSessions: Int = 16) = McpProtocol(listOf(echo, needsPerson, crashes), "test", { "s${++sessionNumber}" }, maxSessions) - private fun initBody(version: String = "2025-06-18") = JSONObject().put("jsonrpc", "2.0").put("id", 1).put("method", "initialize") - .put("params", JSONObject().put("protocolVersion", version).put("capabilities", JSONObject()) - .put("clientInfo", JSONObject().put("name", "test").put("version", "1"))).toString() - - private fun request(id: Any, method: String, params: JSONObject? = null) = JSONObject().put("jsonrpc", "2.0").put("id", id) - .put("method", method).also { if (params != null) it.put("params", params) } + private fun initBody(version: String = "2025-06-18") = + JSONObject() + .put("jsonrpc", "2.0") + .put("id", 1) + .put("method", "initialize") + .put( + "params", + JSONObject() + .put("protocolVersion", version) + .put("capabilities", JSONObject()) + .put("clientInfo", JSONObject().put("name", "test").put("version", "1")), + ).toString() + + private fun request( + id: Any, + method: String, + params: JSONObject? = null, + ) = JSONObject() + .put("jsonrpc", "2.0") + .put("id", id) + .put("method", method) + .also { if (params != null) it.put("params", params) } private fun session(p: McpProtocol): String { val r = p.post(initBody(), null, null) @@ -40,7 +56,12 @@ class McpProtocolTest { return r.headers[McpProtocol.SESSION_HEADER]!! } - private fun call(p: McpProtocol, s: String, tool: String, args: JSONObject? = null): JSONObject { + private fun call( + p: McpProtocol, + s: String, + tool: String, + args: JSONObject? = null, + ): JSONObject { val params = JSONObject().put("name", tool).also { if (args != null) it.put("arguments", args) } val r = p.post(request(7, "tools/call", params).toString(), s, null) assertEquals(200, r.status) @@ -75,27 +96,31 @@ class McpProtocolTest { } @Test fun unsupportedVersionHeaderIsRefused() { - val p = protocol(); val s = session(p) + val p = protocol() + val s = session(p) assertEquals(400, p.post(request(1, "ping").toString(), s, "1999-01-01").status) assertEquals(200, p.post(request(1, "ping").toString(), s, "2025-06-18").status) } @Test fun notificationsGetNoAnswer() { - val p = protocol(); val s = session(p) + val p = protocol() + val s = session(p) val r = p.post(JSONObject().put("jsonrpc", "2.0").put("method", "notifications/initialized").toString(), s, null) assertEquals(202, r.status) assertNull(r.body) } @Test fun pingAnswersEmptyResult() { - val p = protocol(); val s = session(p) + val p = protocol() + val s = session(p) val r = JSONObject(p.post(request("a", "ping").toString(), s, null).body!!) assertEquals("a", r.get("id")) assertEquals(0, r.getJSONObject("result").length()) } @Test fun toolsListDescribesEveryTool() { - val p = protocol(); val s = session(p) + val p = protocol() + val s = session(p) val tools = JSONObject(p.post(request(2, "tools/list").toString(), s, null).body!!).getJSONObject("result").getJSONArray("tools") val names = (0 until tools.length()).map { tools.getJSONObject(it).getString("name") } assertEquals(listOf("echo", "needs_person", "crashes"), names) @@ -106,7 +131,8 @@ class McpProtocolTest { } @Test fun toolCallReturnsStructuredContent() { - val p = protocol(); val s = session(p) + val p = protocol() + val s = session(p) val result = call(p, s, "echo", JSONObject().put("x", 3)).getJSONObject("result") assertFalse(result.getBoolean("isError")) assertEquals(3, result.getJSONObject("structuredContent").getJSONObject("got").getInt("x")) @@ -114,35 +140,52 @@ class McpProtocolTest { } @Test fun toolCallWithoutArgumentsGetsEmptyObject() { - val p = protocol(); val s = session(p) + val p = protocol() + val s = session(p) val got = call(p, s, "echo").getJSONObject("result").getJSONObject("structuredContent").getJSONObject("got") assertEquals(0, got.length()) } @Test fun toolErrorIsAResultWithIsError() { - val p = protocol(); val s = session(p) + val p = protocol() + val s = session(p) val result = call(p, s, "needs_person").getJSONObject("result") assertTrue(result.getBoolean("isError")) - assertTrue(result.getJSONArray("content").getJSONObject(0).getString("text").contains("person is needed")) + assertTrue( + result + .getJSONArray("content") + .getJSONObject(0) + .getString("text") + .contains("person is needed"), + ) assertTrue(result.getJSONObject("structuredContent").getBoolean("person_needed")) } @Test fun unexpectedExceptionIsAResultWithIsError() { - val p = protocol(); val s = session(p) + val p = protocol() + val s = session(p) val result = call(p, s, "crashes").getJSONObject("result") assertTrue(result.getBoolean("isError")) - assertTrue(result.getJSONArray("content").getJSONObject(0).getString("text").contains("boom")) + assertTrue( + result + .getJSONArray("content") + .getJSONObject(0) + .getString("text") + .contains("boom"), + ) } @Test fun unknownToolAndBadArgumentsAreInvalidParams() { - val p = protocol(); val s = session(p) + val p = protocol() + val s = session(p) assertEquals(McpProtocol.INVALID_PARAMS, call(p, s, "nope").getJSONObject("error").getInt("code")) val bad = request(3, "tools/call", JSONObject().put("name", "echo").put("arguments", JSONArray())).toString() assertEquals(McpProtocol.INVALID_PARAMS, JSONObject(p.post(bad, s, null).body!!).getJSONObject("error").getInt("code")) } @Test fun unknownMethodIsMethodNotFound() { - val p = protocol(); val s = session(p) + val p = protocol() + val s = session(p) val r = JSONObject(p.post(request(4, "resources/list").toString(), s, null).body!!) assertEquals(McpProtocol.METHOD_NOT_FOUND, r.getJSONObject("error").getInt("code")) } @@ -154,8 +197,13 @@ class McpProtocolTest { } @Test fun batchGetsAnArrayOfReplies() { - val p = protocol(); val s = session(p) - val batch = JSONArray().put(request(1, "ping")).put(JSONObject().put("jsonrpc", "2.0").put("method", "notifications/x")).put(request(2, "tools/list")) + val p = protocol() + val s = session(p) + val batch = + JSONArray() + .put(request(1, "ping")) + .put(JSONObject().put("jsonrpc", "2.0").put("method", "notifications/x")) + .put(request(2, "tools/list")) val r = p.post(batch.toString(), s, null) assertEquals(200, r.status) val replies = JSONArray(r.body!!) @@ -170,7 +218,8 @@ class McpProtocolTest { } @Test fun deleteEndsTheSession() { - val p = protocol(); val s = session(p) + val p = protocol() + val s = session(p) assertEquals(204, p.delete(s).status) assertEquals(404, p.delete(s).status) assertEquals(404, p.post(request(1, "ping").toString(), s, null).status) @@ -178,9 +227,10 @@ class McpProtocolTest { @Test fun oldestSessionIsDroppedBeyondTheLimit() { val p = protocol(maxSessions = 2) - val a = session(p); val b = session(p) - p.post(request(1, "ping").toString(), a, null) // a is now the most recently used - val c = session(p) // drops b + val a = session(p) + val b = session(p) + p.post(request(1, "ping").toString(), a, null) // a is now the most recently used + val c = session(p) // drops b assertEquals(2, p.sessionCount()) assertEquals(200, p.post(request(1, "ping").toString(), a, null).status) assertEquals(404, p.post(request(1, "ping").toString(), b, null).status) @@ -188,7 +238,8 @@ class McpProtocolTest { } @Test fun endAllEndsEverySession() { - val p = protocol(); val s = session(p) + val p = protocol() + val s = session(p) p.endAll() assertEquals(0, p.sessionCount()) assertEquals(404, p.post(request(1, "ping").toString(), s, null).status) @@ -199,7 +250,10 @@ class McpProtocolTest { private val token = "secret-token" private var server: McpHttp? = null - @After fun stopServer() { server?.stop(); server = null } + @After fun stopServer() { + server?.stop() + server = null + } private fun startServer(): Int { val s = McpHttp(protocol(), { token }, port = 0) @@ -208,20 +262,43 @@ class McpProtocolTest { return s.boundPort } - private class Answer(val status: Int, val headers: Map, val body: String) + private class Answer( + val status: Int, + val headers: Map, + val body: String, + ) /** One request on its own connection; [headers] may replace Host. */ - private fun http(port: Int, method: String, path: String = McpHttp.PATH, headers: Map = emptyMap(), body: String? = null): Answer { + private fun http( + port: Int, + method: String, + path: String = McpHttp.PATH, + headers: Map = emptyMap(), + body: String? = null, + ): Answer { Socket("127.0.0.1", port).use { s -> s.soTimeout = 5000 val bytes = body?.toByteArray(Charsets.UTF_8) ?: ByteArray(0) val all = linkedMapOf("Host" to "127.0.0.1:$port", "Connection" to "close") all.putAll(headers) - if (body != null) { all["Content-Type"] = "application/json"; all["Content-Length"] = bytes.size.toString() } + if (body != null) { + all["Content-Type"] = "application/json" + all["Content-Length"] = bytes.size.toString() + } val head = StringBuilder("$method $path HTTP/1.1\r\n") - for ((k, v) in all) head.append(k).append(": ").append(v).append("\r\n") + for ((k, v) in all) { + head + .append(k) + .append(": ") + .append(v) + .append("\r\n") + } head.append("\r\n") - s.getOutputStream().apply { write(head.toString().toByteArray(Charsets.ISO_8859_1)); write(bytes); flush() } + s.getOutputStream().apply { + write(head.toString().toByteArray(Charsets.ISO_8859_1)) + write(bytes) + flush() + } val text = s.getInputStream().readBytes().toString(Charsets.UTF_8) val lines = text.substringBefore("\r\n\r\n").split("\r\n") val status = lines[0].split(' ')[1].toInt() @@ -242,8 +319,19 @@ class McpProtocolTest { @Test fun base64MatchesTheJdkEncoder() { for (n in 0..40) { val bytes = ByteArray(n) { (it * 37 + 250).toByte() } - assertEquals(java.util.Base64.getEncoder().encodeToString(bytes), McpHttp.base64(bytes, urlSafe = false, pad = true)) - assertEquals(java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(bytes), McpHttp.base64(bytes, urlSafe = true, pad = false)) + assertEquals( + java.util.Base64 + .getEncoder() + .encodeToString(bytes), + McpHttp.base64(bytes, urlSafe = false, pad = true), + ) + assertEquals( + java.util.Base64 + .getUrlEncoder() + .withoutPadding() + .encodeToString(bytes), + McpHttp.base64(bytes, urlSafe = true, pad = false), + ) } } @@ -263,7 +351,9 @@ class McpProtocolTest { val text = s.getInputStream().readBytes().toString(Charsets.UTF_8) assertTrue(text.startsWith("HTTP/1.1 503")) } - } finally { held.forEach { it.close() } } + } finally { + held.forEach { it.close() } + } } @Test fun httpNeedsTheToken() { @@ -315,8 +405,12 @@ class McpProtocolTest { val port = startServer() Socket("127.0.0.1", port).use { s -> val body = initBody().toByteArray() - s.getOutputStream().write(("POST /mcp HTTP/1.1\r\nHost: 127.0.0.1\r\nAuthorization: Bearer $token\r\nConnection: close\r\n" + - "Content-Type: text/plain\r\nContent-Length: ${body.size}\r\n\r\n").toByteArray() + body) + s.getOutputStream().write( + ( + "POST /mcp HTTP/1.1\r\nHost: 127.0.0.1\r\nAuthorization: Bearer $token\r\nConnection: close\r\n" + + "Content-Type: text/plain\r\nContent-Length: ${body.size}\r\n\r\n" + ).toByteArray() + body, + ) val text = s.getInputStream().readBytes().toString(Charsets.UTF_8) assertTrue(text.startsWith("HTTP/1.1 415")) } From 16a006fe603e6f9d7a6d7a836ee5acf89b42fd18 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 05:07:48 +0000 Subject: [PATCH 08/24] ADFA-7135 Fix three review findings in the extension - read_events moves the cursor past dropped events when it copies nothing, so a drop is reported once. - An internal failure in sqlitedbg_extension_init is logged and the connection left untracked; the app's own sqlite3_open no longer fails because of it. - A trigger is told apart from app SQL by X differing from sqlite3_sql(P), not by a "--" prefix, so statements that start with a comment are expanded. test.sh builds the extension with SQLITEDBG_TEST for the failure hook. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- .../sqlite-debugger/src/main/cpp/sqlitedbg.c | 45 ++++++++++++++-- .../sqlite-debugger/src/main/cpp/sqlitedbg.h | 4 +- .../src/test/c/sqlitedbg_test.c | 53 +++++++++++++++++++ subprojects/sqlite-debugger/test.sh | 2 +- 4 files changed, 97 insertions(+), 7 deletions(-) diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c index 1442df0990..6e3f0f78f8 100644 --- a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c @@ -173,9 +173,15 @@ static int trace_cb(unsigned type, void *c, void *p, void *x) { if (is_stmt) sql = (const char *)x; else sql = sqlite3_sql(stmt); if (!sql) sql = ""; - // A trigger's text starts with "--" and P is the outer statement, whose expansion would - // be the wrong text. - if (__atomic_load_n(&g_client_connected, __ATOMIC_RELAXED) && !(sql[0] == '-' && sql[1] == '-')) { + // For a trigger, X is its own "-- ..." text while P is the outer statement, whose expansion + // would be the wrong text. For the statement itself X is its SQL. Not a prefix test: app SQL + // may start with a comment. + int is_trigger = 0; + if (is_stmt) { + const char *own = sqlite3_sql(stmt); + is_trigger = !own || strcmp(own, sql) != 0; + } + if (__atomic_load_n(&g_client_connected, __ATOMIC_RELAXED) && !is_trigger) { expanded = sqlite3_expanded_sql(stmt); if (expanded) { sql = expanded; @@ -218,9 +224,24 @@ static void connection_id_fn(sqlite3_context *ctx, int argc, sqlite3_value **arg sqlite3_result_int64(ctx, (sqlite3_int64)((conn_state *)sqlite3_user_data(ctx))->id); } +#ifdef SQLITEDBG_TEST +// Host tests: 1 fails the allocation, 2 makes registering the function fail. +int sqlitedbg_test_fail_init; +#endif + +static int init_connection(sqlite3 *db, const sqlite3_api_routines *api); + +// An internal failure must never fail the app's own sqlite3_open: log it and leave the connection +// untracked. int sqlitedbg_extension_init(sqlite3 *db, char **err, const sqlite3_api_routines *api) { (void)err; if (tls_opening_server) return SQLITE_OK; + int rc = init_connection(db, api); + if (rc != SQLITE_OK) sqlite3_log(rc, "sqlitedbg: connection not tracked (%d)", rc); + return SQLITE_OK; +} + +static int init_connection(sqlite3 *db, const sqlite3_api_routines *api) { pthread_mutex_lock(&g_lock); if (!g_api_ready) { @@ -230,6 +251,12 @@ int sqlitedbg_extension_init(sqlite3 *db, char **err, const sqlite3_api_routines pthread_mutex_unlock(&g_lock); conn_state *cs = (conn_state *)calloc(1, sizeof *cs); +#ifdef SQLITEDBG_TEST + if (sqlitedbg_test_fail_init == 1) { + free(cs); + cs = NULL; + } +#endif if (!cs) return SQLITE_NOMEM; const char *filename = sqlite3_db_filename(db, "main"); cs->path = strdup(filename ? filename : ""); @@ -250,7 +277,11 @@ int sqlitedbg_extension_init(sqlite3 *db, char **err, const sqlite3_api_routines pthread_mutex_unlock(&g_lock); // On failure SQLite has already called conn_destroy, so cs is gone. - int rc = sqlite3_create_function_v2(db, "sqlitedbg_connection_id", 0, SQLITE_UTF8 | SQLITE_INNOCUOUS, cs, + const char *fn_name = "sqlitedbg_connection_id"; +#ifdef SQLITEDBG_TEST + if (sqlitedbg_test_fail_init == 2) fn_name = NULL; // SQLite refuses a NULL name +#endif + int rc = sqlite3_create_function_v2(db, fn_name, 0, SQLITE_UTF8 | SQLITE_INNOCUOUS, cs, connection_id_fn, NULL, NULL, conn_destroy); if (rc != SQLITE_OK) return rc; return sqlite3_trace_v2(db, SQLITE_TRACE_STMT | SQLITE_TRACE_PROFILE | SQLITE_TRACE_CLOSE, trace_cb, cs); @@ -322,7 +353,11 @@ void sqlitedbg_read_events(uint64_t after, void *out, size_t out_bytes, sqlitedb pthread_mutex_lock(&g_lock); if (g_ring) { uint64_t oldest = oldest_seq(); - if (oldest > after && oldest - after > 1) result->dropped = oldest - after - 1; + if (oldest > after && oldest - after > 1) { + result->dropped = oldest - after - 1; + // Past the dropped events, so they are reported once even if nothing is copied below. + result->next = oldest - 1; + } size_t offset = g_tail, left = g_used; while (left > 0) { const sqlitedbg_event_record *r = record_at(offset); diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h index 2153afaa05..5a40fd7877 100644 --- a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h @@ -109,7 +109,9 @@ typedef struct sqlitedbg_event_record { } sqlitedbg_event_record; // 56 bytes; the SQL text follows typedef struct sqlitedbg_read_result { - // The seq of the last event copied out, or `after` if none was: pass it as `after` next time. + // The seq of the last event copied out, or if none was, `after` moved past any dropped events + // (so they are reported once): pass it as `after` next time. A record too big for out is not + // passed. uint64_t next; // Events after `after` that left the buffer before they could be read. uint64_t dropped; diff --git a/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c b/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c index f2efe08d7a..d2606e6997 100644 --- a/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c +++ b/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c @@ -181,6 +181,8 @@ static uint64_t conn_id_of(sqlite3 *db) { // Tests +extern int sqlitedbg_test_fail_init; // sqlitedbg.c, built with SQLITEDBG_TEST + static sqlitedbg_config g_config = {BUFFER_BYTES, MAX_SQL}; static sqlite3 *g_main; @@ -295,6 +297,20 @@ static void test_expanded(void) { CHECK(b.events[3].flags & SQLITEDBG_EVENT_EXPANDED); batch_free(&b); + // app SQL that starts with a comment is not a trigger: it is expanded like any other + cur = cursor_now(); + sqlitedbg_set_client_connected(1); + CHECK_EQ(sqlite3_prepare_v2(g_main, "-- note\nINSERT INTO t VALUES(?)", -1, &st, NULL), SQLITE_OK); + sqlite3_bind_int(st, 1, 9); + CHECK_EQ(sqlite3_step(st), SQLITE_DONE); + sqlite3_finalize(st); + sqlitedbg_set_client_connected(0); + read_all(cur, &b); + CHECK(b.count >= 2); + CHECK(strstr(b.events[0].sql, "INSERT INTO t VALUES(9)") != NULL); + CHECK(b.events[0].flags & SQLITEDBG_EVENT_EXPANDED); + batch_free(&b); + // after the client leaves, the bound value is not written in again cur = cursor_now(); CHECK_EQ(sqlite3_prepare_v2(g_main, "INSERT INTO t VALUES(?)", -1, &st, NULL), SQLITE_OK); @@ -390,8 +406,25 @@ static void test_burst(void) { CHECK_EQ(b.events[0].seq, cur + made - b.count + 1); CHECK_EQ(b.events[b.count - 1].seq, cur + made); for (size_t i = 1; i < b.count; i++) CHECK_EQ(b.events[i].seq, b.events[i - 1].seq + 1); + const uint64_t first_seq = b.events[0].seq; + const size_t stored = b.count; batch_free(&b); + // drops are reported once even when nothing is copied: a record too big for out stays + // unpassed, and the cursor moves up to it + { + sqlitedbg_read_result r; + char tiny[8]; + sqlitedbg_read_events(cur, tiny, sizeof tiny, &r); + CHECK_EQ(r.count, 0); + CHECK_EQ(r.dropped, made - stored); + CHECK_EQ(r.next, first_seq - 1); + sqlitedbg_read_result again; + sqlitedbg_read_events(r.next, tiny, sizeof tiny, &again); + CHECK_EQ(again.dropped, 0); + CHECK_EQ(again.next, r.next); + } + // nothing is reported twice uint64_t next = cur + made; read_all(next, &b); @@ -482,6 +515,24 @@ static void test_close_and_list(void) { CHECK_EQ(list_conns(&l), before); } +static void test_init_failure(void) { + listing l; + size_t before = list_conns(&l); + for (int mode = 1; mode <= 2; mode++) { + sqlitedbg_test_fail_init = mode; + sqlite3 *db = NULL; + // the app's open must not fail because the debugger could not track the connection + CHECK_EQ(sqlite3_open(":memory:", &db), SQLITE_OK); + sqlitedbg_test_fail_init = 0; + CHECK(db != NULL); + if (!db) continue; + CHECK_EQ(list_conns(&l), before); + exec_ok(db, "CREATE TABLE u(a)"); + CHECK_EQ(sqlite3_close(db), SQLITE_OK); + } + CHECK_EQ(list_conns(&l), before); +} + static int step_all(sqlite3 *db, const char *sql) { sqlite3_stmt *st; int rc = sqlite3_prepare_v2(db, sql, -1, &st, NULL); @@ -825,6 +876,8 @@ int main(int argc, char **argv) { test_burst(); printf("# close and list\n"); test_close_and_list(); + printf("# init failure\n"); + test_init_failure(); printf("# server connections\n"); test_server(); printf("# threads\n"); diff --git a/subprojects/sqlite-debugger/test.sh b/subprojects/sqlite-debugger/test.sh index 18b42811af..455dffa987 100755 --- a/subprojects/sqlite-debugger/test.sh +++ b/subprojects/sqlite-debugger/test.sh @@ -51,7 +51,7 @@ run_with () { mkdir "$WORK/$name" build gcc -O1 -g -w -fsanitize="$sanitize" "${DEFS[@]}" -I"$AMAL" -c "$AMAL/sqlite3.c" -o "$WORK/$name/sqlite3.o" build gcc -O1 -g -Wall -Wextra -Werror -fsanitize="$sanitize" -fno-sanitize-recover=all "${DEFS[@]}" \ - -I"$AMAL" -I"$CPP" "$CPP/sqlitedbg.c" "$HERE/src/test/c/sqlitedbg_test.c" "$WORK/$name/sqlite3.o" \ + -DSQLITEDBG_TEST -I"$AMAL" -I"$CPP" "$CPP/sqlitedbg.c" "$HERE/src/test/c/sqlitedbg_test.c" "$WORK/$name/sqlite3.o" \ -o "$WORK/$name/test" -lpthread -ldl -lm TSAN_OPTIONS=halt_on_error=1 "$WORK/$name/test" ${3:+"${@:3}"} } From f03521dc2a03b684333c89e37d7ee9510ea06e96 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 05:20:10 +0000 Subject: [PATCH 09/24] ADFA-7136: Document the SQLite debugger module Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- ARCHITECTURE.md | 1 + subprojects/sqlite-debugger/README.md | 53 +++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 subprojects/sqlite-debugger/README.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b4046e491d..b83d711c35 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -108,6 +108,7 @@ These structural facts shape every module. Day-to-day build *commands* live in ` - **SDK levels** (`build-logic/.../build/config/BuildConfig.kt`): `COMPILE_SDK=36`, `MIN_SDK=28`, `TARGET_SDK=28`. **`TARGET_SDK` is deliberately pinned at 28:** higher targets enforce W^X (write-xor-execute), which blocks executing code from app-writable files. That is fatal for an on-device IDE that compiles and runs code (Gradle, `javac`, Termux binaries), so it is a hard requirement, not tech debt. `MIN_SDK_FOR_APPS_BUILT_WITH_COGO=16` is the floor for the apps a *user* builds with CoGo — distinct from CoGo's own `MIN_SDK`. - **Native asset bundling.** The on-device LLM (`llama-impl`) ships as a per-flavor native AAR, wired through the root `build.gradle.kts` (`bundleLlamaV8Assets` / `assembleV8Assets`, …); prebuilt per-flavor assets live under `assets/release/v7/` and `assets/release/v8/`. - **sqlite-js for apps built with CoGo** (ADFA-7082). `subprojects:sqlite-js` builds the AAR that the JavaScript-in-SQLite project template depends on: sqlite-js vendored with our patches (see its README), for arm64-v8a and armeabi-v7a, minSdk 26, without core-library desugaring. Nothing in CoGo depends on it. Its default entry point runs no code stored in a database; loading stored functions is a separate entry point, only for databases the app made, since loading runs their code. `test-sqlite-js.yml` runs its host tests on pull requests. The manual `release-sqlite-js.yml` publishes it to the moving `sqlite-js-latest` release, from which dev-assets fetches it into the template, as it fetches `plugin-api-latest` for the plugin template. +- **SQLite debugger for apps built with CoGo** (ADFA-7131). `subprojects:sqlite-debugger` builds an AAR with a debug-only SQLite debugger and MCP server for apps built with CoGo: a native extension on requery's SQLite plus a loopback HTTP server. Like sqlite-js it has no ABI flavors, and nothing in CoGo depends on it; see its README. - **Native lib compression** (ADFA-2306, ADFA-4729). The app manifest hard-codes `android:extractNativeLibs="true"` (required: the installer must materialize libs in `nativeLibraryDir`, e.g. `libshizuku.so` is an executable the adb shell runs from there). That attribute overrides the `jniLibs.useLegacyPackaging` DSL, so AGP packages `lib//*.so` deflate-compressed in **every** APK — ~5.9 MB smaller (`libtree-sitter-kotlin.so` alone is 4.18 MB → 339 kB). The trap is the `recompressApk` post-step (release always, debug in CI only): its no-compress lists in `app/build.gradle.kts` must NOT contain `"so"`, or it silently re-stores the libs and undoes the saving — which is what ADFA-2306 fixed for release and ADFA-4729 for CI debug. Locally built debug APKs (including the e2e farm's) never run that step and were always fine. - **Quick Build shares the Gradle distribution's toolchain** (ADFA-4931). `:app`'s `quickBuildDaemonZip` leaves the Kotlin compiler and eight other jars out of the daemon zip, because the on-device Gradle distribution already carries them; `QuickBuildArtifactStager` links them in at provision time. That makes the daemon's Kotlin version the distribution's, so `:quickbuild:daemon` takes its Kotlin artifacts at `org.adfa.constants.GRADLE_DISTRIBUTION_KOTLIN_VERSION`, and `:app:quickBuildDistJarCheck` fails the build when the distribution it packages lacks a linked jar. See [ADR 0016](docs/adr/0016-quick-build-compiles-outside-gradle.md). - **`app` package layout is by concern, not feature:** `activities`, `fragments`, `services`, `di`, `agent`, `viewmodel(s)`, `repositories`, `roomData`, `localWebServer`, `preferences`, `ui` (Compose screens live under `ui/compose`), `templates/manager` (the Manager screen's `.cgt`-parsing data layer, with direct filesystem access to `Environment.TEMPLATES_DIR` — distinct from the plugin-facing `IdeTemplateService` in `plugin-api`/`plugin-manager`), `utils`, …. diff --git a/subprojects/sqlite-debugger/README.md b/subprojects/sqlite-debugger/README.md new file mode 100644 index 0000000000..406bcd17ab --- /dev/null +++ b/subprojects/sqlite-debugger/README.md @@ -0,0 +1,53 @@ +# SQLite debugger for apps + +A debug-only SQLite debugger and MCP server that runs inside apps built with Code on the Go (ADFA-7131). It lets an AI agent list an app's databases, read their schema and rows, see the SQL the app runs, and check integrity, all read-only. Phase 1 is the native extension (ADFA-7135) and the server (ADFA-7136). Nothing in CoGo itself depends on it: apps do, in debug builds. + +The module builds an AAR with `libsqlitedbg.so` for arm64-v8a and armeabi-v7a in one artifact, so it has no v7/v8 flavors (`AndroidModuleConf`). Its minSdk is 21, requery's, because the debugger runs on requery's SQLite. The unit test task is `:subprojects:sqlite-debugger:testDebugUnitTest`. + +## How it fits together + +``` +MCP client --HTTP, 127.0.0.1--> McpHttp -> McpProtocol -> DatabaseTools -> DebugEngine + | + NativeDebugEngine -> NativeBridge (JNI) + | + sqlitedbg_jni.c -> sqlitedbg.c (sqlitedbg.h) + | + requery's libsqlite3x.so (the app's SQLite) +``` + +- `src/main/cpp/sqlitedbg.h` is the contract between the extension, its JNI layer and its host tests, with the semantics of each call. `SQLITEDBG_INTERFACE_VERSION` is raised when anything in it changes shape. +- `sqlitedbg.c` is a SQLite extension. `sqlitedbg_install` registers it with `sqlite3_auto_extension`, so it runs on every connection the app opens afterwards. It keeps the list of app connections and a ring buffer of statement events (trace: statement, profile, close), and opens connections for the server (`sqlitedbg_open_server`). +- `sqlitedbg_jni.c` finds `sqlite3_auto_extension` in requery's `libsqlite3x.so` with `dlopen(RTLD_NOLOAD)` and `dlsym`, so no SQLite is compiled or linked here (CMake takes only the headers from `../sqlite-js/upstream/libs`). Its version script exports only `JNI_OnLoad`; the functions of `NativeBridge` are registered there by name, and `consumer-rules.pro` keeps them. +- `engine/`: `DebugEngine` is what the server needs from the extension, `NativeDebugEngine` implements it over JNI, and `EventDecoder` turns the event buffer's bytes into `Event`s. +- `mcp/`: `McpHttp` is the HTTP/1.1 listener, `McpProtocol` is JSON-RPC and the MCP session handling, `DatabaseTools` is the tools: `list_databases`, `list_connections`, `get_schema`, `query`, `explain_query_plan`, `get_stats`, `integrity_check`, `get_events`. +- `DebuggerStartup` is a `ContentProvider` with the highest `initOrder`, declared in the AAR's manifest. It runs before the app's own providers and `Application.onCreate`, so the extension is registered before the app opens a database. It starts the engine and the server on a background thread. + +### The event record + +The layout of `sqlitedbg_event_record` lives in `sqlitedbg.h`. The JNI layer hands the bytes to Kotlin unchanged, so `EventDecoder.kt` mirrors the layout, field offsets included: change both together. A committed fixture pins them (see Testing). + +## Decisions that shape it + +- **Debug builds only.** `DebuggerStartup.onCreate` throws if the process is not debuggable, so a release build that includes the library by mistake crashes at once instead of opening a local server. +- **Loopback HTTP, a random port and token per start.** `McpHttp` binds 127.0.0.1 only (port 0, so the system picks one) and requires `Authorization: Bearer `, a new 32-byte random token each process start. It also checks that the Host and any Origin name 127.0.0.1 or localhost, against DNS rebinding. At most 8 connections at once. It opens no event stream: GET gets 405. +- **Read-only is a guard against accidents, not a security boundary.** Every tool opens its server connection with `writable = false`: `SQLITE_OPEN_READONLY`, `PRAGMA query_only` and an authorizer that refuses ordinary writes. Anyone holding the token can still run SQL, so the token is the boundary. +- **Only databases an app connection has open.** `DatabaseTools` refuses any other path before the engine is called: opening a file Android's own SQLite has open, with requery's, can corrupt it. The extension has no SQLite API until an app connection has opened, so `sqlitedbg_open_server` returns `SQLITE_MISUSE` before that. +- **The app's connections are never altered.** The extension never sets the progress handler or the update hook of an app connection: requery sets and clears the progress handler around cancellable statements, and the update hook belongs to the app. Only server connections use a progress handler, for the time limit. +- **Bound values are recorded only while a client is connected.** Expanding the SQL costs memory on every run, so `sqlitedbg_set_client_connected` turns it on; the first `get_events` call does. +- **Limits.** Connections opened before `install` are not seen, so `DebuggerStartup` runs as early as the process allows. Events are dropped, oldest first, when the buffer (4 MiB by default) is full, and `get_events` reports how many. + +## Testing + +- **Host C tests of the extension:** `subprojects/sqlite-debugger/test.sh [--update-fixture] [AMALGAMATION_TARBALL]`. It builds SQLite 3.49.1 from the amalgamation as requery builds it (`SQLITE_THREADSAFE=2`), checks its headers against `sqlite-js/upstream/libs`, and runs `src/test/c/sqlitedbg_test.c` under AddressSanitizer with UBSan, then under ThreadSanitizer. It needs gcc, curl, tar and sha256sum, and downloads the tarball unless you pass one. +- **The fixture:** `src/test/resources/.../engine/events.bin` and `events.tsv` are written by the C test and read by `EventDecoderTest`. Without `--update-fixture`, `test.sh` checks the committed pair against the current record layout, so a layout change fails until you regenerate it and update `EventDecoder.kt`. +- **JVM tests:** `flox activate -d flox/local -- ./gradlew :subprojects:sqlite-debugger:testDebugUnitTest`. They cover `EventDecoder`, `McpProtocol` and `DatabaseTools` (with `FakeDebugEngine`). + +No workflow in `.github/workflows` runs either yet. + +## Not done yet + +- Broadcasting the port and token to Code on the Go (ADFA-7137). `DebuggerStartup` only keeps them in `DebuggerStartup.port` and `endpointToken` and logs the port, never the token. +- Gradle plugin wiring and the build-time rewrite that adds the library to debug builds (ADFA-7138). +- The AI-Agent-MCP side that connects to the server (ADFA-7139). +- Verification on a device. The C tests run on the host and the Kotlin tests on the JVM; nothing has run in an app on a device or emulator. From e21aae35a83ac43c4d159b107ac8063bff9fe95c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 05:20:32 +0000 Subject: [PATCH 10/24] ADFA-7136: List sqlite-debugger among the modules without ABI flavors AndroidModuleConf.kt already excludes it; ARCHITECTURE.md and CLAUDE.md named only plugin-api and sqlite-js. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- ARCHITECTURE.md | 4 ++-- CLAUDE.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b83d711c35..e8247b2966 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -95,7 +95,7 @@ Strategy: **layer-and-subsystem based**, not feature-by-feature. The Gradle buil - **`app` depends inward; libraries never depend on `app`.** Subsystems are consumed by `app`, not vice versa. - **Vendored forks are substituted, not imported ad hoc.** `composite-builds/build-deps` and `build-deps-common` provide forked `javac`/`jdt`/`layoutlib`/etc.; `settings.gradle.kts` substitutes them in for `com.itsaky.androidide.build:*`. Don't add a Maven coordinate for something already substituted. Pebble is likewise built from source, as `subprojects:pebble`, for a lexer fix Android needs (its `NOTICE` says what changed); depend on `projects.subprojects.pebble`, not `io.pebbletemplates:pebble`. -- **All module config flows through `composite-builds/build-logic`.** Every Android module gets the `v7`/`v8` ABI flavors centrally (`AndroidModuleConf.kt`) — there is no flavorless `assembleDebug`. `:plugin-api` and `:subprojects:sqlite-js`, libraries for other projects that need one artifact, are intentionally excluded from flavors. +- **All module config flows through `composite-builds/build-logic`.** Every Android module gets the `v7`/`v8` ABI flavors centrally (`AndroidModuleConf.kt`) — there is no flavorless `assembleDebug`. `:plugin-api`, `:subprojects:sqlite-js` and `:subprojects:sqlite-debugger`, libraries for other projects that need one artifact, are intentionally excluded from flavors. - **`RepositoriesMode.FAIL_ON_PROJECT_REPOS`** — repositories are declared once in `settings.gradle.kts`; modules must not declare their own. - **Avoid new dependencies.** Check `gradle/libs.versions.toml` first; the build almost certainly already has it. @@ -104,7 +104,7 @@ Strategy: **layer-and-subsystem based**, not feature-by-feature. The Gradle buil These structural facts shape every module. Day-to-day build *commands* live in `CLAUDE.md`; the rules that produce them live here. - **Centralized convention logic.** All Android module setup flows through `composite-builds/build-logic` (`conf/AndroidModuleConf.kt`). Modules stay thin and share configuration, so understanding any module's setup starts here. -- **ABI product flavors.** Every Android module *except* `:plugin-api` and `:subprojects:sqlite-js` gets two flavors on the `abi` dimension — `v7` (`armeabi-v7a`) and `v8` (`arm64-v8a`) — defined centrally. There is no flavorless variant; tasks are `assembleV8Debug`, `assembleV7Release`, etc. +- **ABI product flavors.** Every Android module *except* `:plugin-api`, `:subprojects:sqlite-js` and `:subprojects:sqlite-debugger` gets two flavors on the `abi` dimension — `v7` (`armeabi-v7a`) and `v8` (`arm64-v8a`) — defined centrally. There is no flavorless variant; tasks are `assembleV8Debug`, `assembleV7Release`, etc. - **SDK levels** (`build-logic/.../build/config/BuildConfig.kt`): `COMPILE_SDK=36`, `MIN_SDK=28`, `TARGET_SDK=28`. **`TARGET_SDK` is deliberately pinned at 28:** higher targets enforce W^X (write-xor-execute), which blocks executing code from app-writable files. That is fatal for an on-device IDE that compiles and runs code (Gradle, `javac`, Termux binaries), so it is a hard requirement, not tech debt. `MIN_SDK_FOR_APPS_BUILT_WITH_COGO=16` is the floor for the apps a *user* builds with CoGo — distinct from CoGo's own `MIN_SDK`. - **Native asset bundling.** The on-device LLM (`llama-impl`) ships as a per-flavor native AAR, wired through the root `build.gradle.kts` (`bundleLlamaV8Assets` / `assembleV8Assets`, …); prebuilt per-flavor assets live under `assets/release/v7/` and `assets/release/v8/`. - **sqlite-js for apps built with CoGo** (ADFA-7082). `subprojects:sqlite-js` builds the AAR that the JavaScript-in-SQLite project template depends on: sqlite-js vendored with our patches (see its README), for arm64-v8a and armeabi-v7a, minSdk 26, without core-library desugaring. Nothing in CoGo depends on it. Its default entry point runs no code stored in a database; loading stored functions is a separate entry point, only for databases the app made, since loading runs their code. `test-sqlite-js.yml` runs its host tests on pull requests. The manual `release-sqlite-js.yml` publishes it to the moving `sqlite-js-latest` release, from which dev-assets fetches it into the template, as it fetches `plugin-api-latest` for the plugin template. diff --git a/CLAUDE.md b/CLAUDE.md index eaae5b3de6..7334588a8c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,7 +15,7 @@ flox activate -d flox/local -- ./gradlew ``` - **Debug APK (arm64, the usual target):** `flox activate -d flox/local -- ./gradlew :app:assembleV8Debug --parallel --max-workers=6` -- **Module unit tests:** `flox activate -d flox/local -- ./gradlew :editor:testV7DebugUnitTest` - unit tests are per-flavor, there is no plain `test` task. `:plugin-api` and `:subprojects:sqlite-js` are the modules without flavors: `:plugin-api:testDebugUnitTest`. +- **Module unit tests:** `flox activate -d flox/local -- ./gradlew :editor:testV7DebugUnitTest` - unit tests are per-flavor, there is no plain `test` task. `:plugin-api`, `:subprojects:sqlite-js` and `:subprojects:sqlite-debugger` are the modules without flavors: `:plugin-api:testDebugUnitTest`. - **Single unit test:** append `--tests "com.itsaky.androidide.SomeTest"` to the flavored task above. - **Fast iteration:** during multi-file/multi-module changes, verify with targeted `:module:compileV8DebugKotlin`/`compileV8DebugJavaWithJavac` invocations (batch several modules into one Gradle call) rather than a full assemble. Reserve `:app:assembleV8Debug` for final end-to-end verification — it's slow (multi-minute) in this multi-module project, and running it after every small change adds up. - **Long-running commands: narrate them, and background only the read-only ones.** Anything that can exceed ~60s — `assembleV8Debug`, a full test sweep, a cold Gradle invocation of any kind (daemon start plus configuring this many modules is ~20s before any task runs, and far worse under memory pressure), and `git push`, which runs Spotless through the hook — gets a line before it starts saying what is running and roughly how long it takes, and a status line every couple of minutes while it runs: elapsed time, the last output line, whether it is still progressing. A silent terminal is indistinguishable from a hang, and saying which it is is your job, not the user's to ask. @@ -26,7 +26,7 @@ When the user names a CI/CD job ("the sonar job", "the analyze workflow"), read ### ABI product flavors (v7 / v8) -Every module except `:plugin-api` and `:subprojects:sqlite-js` carries `v7` (`armeabi-v7a`) and `v8` (`arm64-v8a`) flavors, so build tasks are `assembleV8Debug`, `assembleV7Release`, etc. — there is **no** flavorless `assembleDebug`. Those two build one artifact for other projects. See [ARCHITECTURE.md](ARCHITECTURE.md) (Build & Module Configuration) for flavors, native asset bundling, and SDK levels. +Every module except `:plugin-api`, `:subprojects:sqlite-js` and `:subprojects:sqlite-debugger` carries `v7` (`armeabi-v7a`) and `v8` (`arm64-v8a`) flavors, so build tasks are `assembleV8Debug`, `assembleV7Release`, etc. — there is **no** flavorless `assembleDebug`. Those three build one artifact for other projects. See [ARCHITECTURE.md](ARCHITECTURE.md) (Build & Module Configuration) for flavors, native asset bundling, and SDK levels. ### Emulator / device From 8dd5efb4d9f4046461eedc2e2665618488f7b732 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 05:28:26 +0000 Subject: [PATCH 11/24] ADFA-7136 Never let the debugger crash or miss the host app's early connections - DebuggerStartup catches Throwable after the debuggable check, so a missing requery library (UnsatisfiedLinkError) disables the debugger instead of killing the app. The check itself still throws. - install() runs synchronously in onCreate; only the server start is on the thread, so connections opened in Application.onCreate are tracked. - McpHttp: connection and accept threads catch Throwable; an Error while handling a request is answered with a 500. - McpHttp: a failed accept sleeps 100 ms, doubling to 2 s while failures continue, and resets on success (no spin on EMFILE). Tests: an Error thrown by a tool gives HTTP 500 and the server keeps serving; a ServerSocket whose accept fails is retried with growing gaps, then serves. DebuggerStartup imports android.* and has no JVM test. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- subprojects/sqlite-debugger/README.md | 2 +- .../sqlitedebugger/DebuggerStartup.kt | 43 +++++++++++++------ .../sqlitedebugger/mcp/McpHttp.kt | 42 ++++++++++++++---- .../sqlitedebugger/mcp/McpProtocolTest.kt | 37 +++++++++++++++- 4 files changed, 99 insertions(+), 25 deletions(-) diff --git a/subprojects/sqlite-debugger/README.md b/subprojects/sqlite-debugger/README.md index 406bcd17ab..fbd8034fb4 100644 --- a/subprojects/sqlite-debugger/README.md +++ b/subprojects/sqlite-debugger/README.md @@ -21,7 +21,7 @@ MCP client --HTTP, 127.0.0.1--> McpHttp -> McpProtocol -> DatabaseTools -> Debug - `sqlitedbg_jni.c` finds `sqlite3_auto_extension` in requery's `libsqlite3x.so` with `dlopen(RTLD_NOLOAD)` and `dlsym`, so no SQLite is compiled or linked here (CMake takes only the headers from `../sqlite-js/upstream/libs`). Its version script exports only `JNI_OnLoad`; the functions of `NativeBridge` are registered there by name, and `consumer-rules.pro` keeps them. - `engine/`: `DebugEngine` is what the server needs from the extension, `NativeDebugEngine` implements it over JNI, and `EventDecoder` turns the event buffer's bytes into `Event`s. - `mcp/`: `McpHttp` is the HTTP/1.1 listener, `McpProtocol` is JSON-RPC and the MCP session handling, `DatabaseTools` is the tools: `list_databases`, `list_connections`, `get_schema`, `query`, `explain_query_plan`, `get_stats`, `integrity_check`, `get_events`. -- `DebuggerStartup` is a `ContentProvider` with the highest `initOrder`, declared in the AAR's manifest. It runs before the app's own providers and `Application.onCreate`, so the extension is registered before the app opens a database. It starts the engine and the server on a background thread. +- `DebuggerStartup` is a `ContentProvider` with the highest `initOrder`, declared in the AAR's manifest. It runs before the app's own providers and `Application.onCreate`, so the extension is registered before the app opens a database. It installs the extension synchronously in `onCreate` (so an app that opens its database in `Application.onCreate` is still seen) and starts the server on a background thread. Everything after the debuggable check is wrapped in `catch (Throwable)`: a missing requery library (`UnsatisfiedLinkError`) means the debugger is disabled and logged, never a crash of the host app. ### The event record diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/DebuggerStartup.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/DebuggerStartup.kt index 30f7177dad..a2605df810 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/DebuggerStartup.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/DebuggerStartup.kt @@ -26,25 +26,40 @@ class DebuggerStartup : ContentProvider() { "The SQLite debugger is in a build that is not debuggable. Remove it from release builds." } val databaseDir = File(context.applicationInfo.dataDir, "databases") - thread(name = "sqlite-debugger-start", isDaemon = true) { - try { - val engine = NativeDebugEngine() - engine.install() - val tools = DatabaseTools(engine, databaseDir) - val token = McpHttp.newToken() - val protocol = McpProtocol(tools.tools(), "1", { UUID.randomUUID().toString() }) - val http = McpHttp(protocol, { token }, log = { Log.w(TAG, it) }) - http.start() - port = http.boundPort - endpointToken = token - Log.i(TAG, "listening on 127.0.0.1:${http.boundPort}") - } catch (e: Exception) { - Log.e(TAG, "could not start", e) + // Throwable, not Exception: a missing library is an UnsatisfiedLinkError, and an Error that + // escapes here kills the host app. A debugger that cannot start is disabled, never a crash. + try { + // Synchronous: an app that opens its database in Application.onCreate or another + // provider would otherwise be missed. + val engine = NativeDebugEngine() + engine.install() + thread(name = "sqlite-debugger-start", isDaemon = true) { + try { + startServer(engine, databaseDir) + } catch (t: Throwable) { + Log.e(TAG, "could not start the server", t) + } } + } catch (t: Throwable) { + Log.e(TAG, "the SQLite debugger is disabled", t) } return true } + private fun startServer( + engine: NativeDebugEngine, + databaseDir: File, + ) { + val tools = DatabaseTools(engine, databaseDir) + val token = McpHttp.newToken() + val protocol = McpProtocol(tools.tools(), "1", { UUID.randomUUID().toString() }) + val http = McpHttp(protocol, { token }, log = { Log.w(TAG, it) }) + http.start() + port = http.boundPort + endpointToken = token + Log.i(TAG, "listening on 127.0.0.1:${http.boundPort}") + } + override fun query( uri: Uri, projection: Array?, diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt index f29b5acad3..ef1028cc7f 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt @@ -39,6 +39,8 @@ class McpHttp( private val port: Int = 0, /** Problems worth a line in the log. Never given the token. */ private val log: (String) -> Unit = {}, + /** Creates the listening socket; tests pass one whose accept fails. */ + private val socketFactory: () -> ServerSocket = { ServerSocket() }, ) { companion object { private const val B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" @@ -83,6 +85,10 @@ class McpHttp( /** A connection with no request for this long is closed. */ const val IDLE_TIMEOUT_MS = 30_000 + + /** After a failed accept the loop sleeps this long, doubling up to the maximum while failures continue. */ + const val ACCEPT_BACKOFF_MIN_MS = 100L + const val ACCEPT_BACKOFF_MAX_MS = 2_000L private val STATUS = mapOf( 200 to "OK", @@ -127,7 +133,7 @@ class McpHttp( /** Binds 127.0.0.1:[port]. Throws IOException with a readable message if the port is taken. */ fun start() { - val s = ServerSocket() + val s = socketFactory() try { s.reuseAddress = true s.bind(InetSocketAddress(InetAddress.getByAddress(byteArrayOf(127, 0, 0, 1)), port), 16) @@ -156,12 +162,23 @@ class McpHttp( } private fun acceptLoop(s: ServerSocket) { + var backoffMs = 0L while (running && !s.isClosed) { val c = try { - s.accept() + s.accept().also { backoffMs = 0 } } catch (e: IOException) { - if (running && !s.isClosed) log("couldn't accept a connection: ${e.message}") + if (running && !s.isClosed) { + log("couldn't accept a connection: ${e.message}") + // A persistent failure (EMFILE) would otherwise spin; keep trying, slowly. + backoffMs = if (backoffMs == 0L) ACCEPT_BACKOFF_MIN_MS else minOf(backoffMs * 2, ACCEPT_BACKOFF_MAX_MS) + sleepQuietly(backoffMs) + } + continue + } catch (t: Throwable) { + log("accept failed: ${t.javaClass.simpleName}: ${t.message}") + backoffMs = if (backoffMs == 0L) ACCEPT_BACKOFF_MIN_MS else minOf(backoffMs * 2, ACCEPT_BACKOFF_MAX_MS) + sleepQuietly(backoffMs) continue } if (!c.inetAddress.isLoopbackAddress) { @@ -184,8 +201,9 @@ class McpHttp( serve(c) } catch (ignored: IOException) { // The client went away. - } catch (e: Exception) { - log("a connection failed: ${e.javaClass.simpleName}: ${e.message}") + } catch (t: Throwable) { + // An Error here would kill the host app: log it and drop the connection. + log("a connection failed: ${t.javaClass.simpleName}: ${t.message}") } finally { clients -= c close(c) @@ -255,9 +273,9 @@ class McpHttp( if (type != null && type != "application/json") return send(415, rpcError("Send the request as application/json.")) try { protocol.post(String(r.body, Charsets.UTF_8), session, r.headers["mcp-protocol-version"]) - } catch (e: Exception) { - log("a request failed: ${e.javaClass.simpleName}: ${e.message}") - McpProtocol.Reply(500, rpcError("The server failed: ${e.message ?: e.javaClass.simpleName}")) + } catch (t: Throwable) { + log("a request failed: ${t.javaClass.simpleName}: ${t.message}") + McpProtocol.Reply(500, rpcError("The server failed: ${t.message ?: t.javaClass.simpleName}")) } } @@ -445,6 +463,14 @@ class McpHttp( .put("error", JSONObject().put("code", McpProtocol.INVALID_REQUEST).put("message", message)) .toString() + private fun sleepQuietly(ms: Long) { + try { + Thread.sleep(ms) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + } + } + private fun close(s: Socket) { try { s.close() diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt index ecad15a633..b6db250247 100644 --- a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt @@ -23,9 +23,10 @@ class McpProtocolTest { throw McpToolError("A person is needed: tap Allow on the sharing phone.", JSONObject().put("person_needed", true)) } private val crashes = McpTool("crashes", "Crashes", "Throws an unexpected exception.") { _ -> throw IllegalStateException("boom") } + private val linkFails = McpTool("link_fails", "Link fails", "Throws an Error.") { _ -> throw UnsatisfiedLinkError("no library") } private fun protocol(maxSessions: Int = 16) = - McpProtocol(listOf(echo, needsPerson, crashes), "test", { "s${++sessionNumber}" }, maxSessions) + McpProtocol(listOf(echo, needsPerson, crashes, linkFails), "test", { "s${++sessionNumber}" }, maxSessions) private fun initBody(version: String = "2025-06-18") = JSONObject() @@ -123,7 +124,7 @@ class McpProtocolTest { val s = session(p) val tools = JSONObject(p.post(request(2, "tools/list").toString(), s, null).body!!).getJSONObject("result").getJSONArray("tools") val names = (0 until tools.length()).map { tools.getJSONObject(it).getString("name") } - assertEquals(listOf("echo", "needs_person", "crashes"), names) + assertEquals(listOf("echo", "needs_person", "crashes", "link_fails"), names) val first = tools.getJSONObject(0) assertEquals("object", first.getJSONObject("inputSchema").getString("type")) assertTrue(first.getJSONObject("annotations").getBoolean("readOnlyHint")) @@ -416,6 +417,38 @@ class McpProtocolTest { } } + @Test fun httpAnErrorInAToolIsA500AndTheServerKeepsServing() { + val port = startServer() + val s = http(port, "POST", headers = auth, body = initBody()).headers["mcp-session-id"]!! + val params = JSONObject().put("name", "link_fails") + val r = http(port, "POST", headers = auth + ("Mcp-Session-Id" to s), body = request(3, "tools/call", params).toString()) + assertEquals(500, r.status) + assertTrue(r.body, r.body.contains("no library")) + assertEquals(200, http(port, "POST", headers = auth + ("Mcp-Session-Id" to s), body = request(4, "ping").toString()).status) + } + + @Test fun httpAcceptBacksOffAfterFailuresAndKeepsServing() { + val attempts = java.util.Collections.synchronizedList(mutableListOf()) + val failures = 4 + val flaky = + object : java.net.ServerSocket() { + override fun accept(): Socket { + attempts += System.nanoTime() + if (attempts.size <= failures) throw java.io.IOException("Too many open files") + return super.accept() + } + } + val s = McpHttp(protocol(), { token }, port = 0, socketFactory = { flaky }) + s.start() + server = s + Thread.sleep(1_200) // 100 + 200 + 400 ms of sleeps, then the fourth failure sleeps 800 ms + // a spinning loop would have made thousands of attempts by now + assertTrue("attempts so far: ${attempts.size}", attempts.size in 4..6) + val gaps = attempts.zipWithNext { a, b -> (b - a) / 1_000_000 } + assertTrue("gaps $gaps", gaps[0] >= 90 && gaps[1] >= 190 && gaps[2] >= 390) + assertEquals(200, http(s.boundPort, "POST", headers = auth, body = initBody()).status) + } + @Test fun httpSessionThenDelete() { val port = startServer() val s = http(port, "POST", headers = auth, body = initBody()).headers["mcp-session-id"]!! From c862c5c2ae7dc090ca7469e9bba9ac5864c05c9a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 05:31:33 +0000 Subject: [PATCH 12/24] ADFA-7135 Keep read-only server connections from holding locks The read-only authorizer allowed BEGIN, SAVEPOINT and PRAGMA locking_mode, so a client could hold a lock on the app's database: BEGIN; SELECT left the app's INSERT failing with SQLITE_BUSY until ROLLBACK. - sqlitedbg.c: deny SQLITE_TRANSACTION, SQLITE_SAVEPOINT, PRAGMA locking_mode and PRAGMA busy_timeout (any form) on read-only server connections. - sqlitedbg_jni.c: bridge_query rolls back if the connection is not in autocommit when it returns (covers writable connections). - Host C test: each denied statement returns SQLITE_AUTH and the app can still write after a read on that connection. Without the authorizer change the test fails on all 8 statements (BEGIN and SAVEPOINT: "database is locked" for the app's INSERT). The backstop is tested with the JNI test. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- subprojects/sqlite-debugger/README.md | 2 +- .../sqlite-debugger/src/main/cpp/sqlitedbg.c | 6 +++ .../sqlite-debugger/src/main/cpp/sqlitedbg.h | 3 +- .../src/main/cpp/sqlitedbg_jni.c | 3 ++ .../src/test/c/sqlitedbg_test.c | 41 +++++++++++++++++++ 5 files changed, 53 insertions(+), 2 deletions(-) diff --git a/subprojects/sqlite-debugger/README.md b/subprojects/sqlite-debugger/README.md index fbd8034fb4..1cfc9e69f2 100644 --- a/subprojects/sqlite-debugger/README.md +++ b/subprojects/sqlite-debugger/README.md @@ -31,7 +31,7 @@ The layout of `sqlitedbg_event_record` lives in `sqlitedbg.h`. The JNI layer han - **Debug builds only.** `DebuggerStartup.onCreate` throws if the process is not debuggable, so a release build that includes the library by mistake crashes at once instead of opening a local server. - **Loopback HTTP, a random port and token per start.** `McpHttp` binds 127.0.0.1 only (port 0, so the system picks one) and requires `Authorization: Bearer `, a new 32-byte random token each process start. It also checks that the Host and any Origin name 127.0.0.1 or localhost, against DNS rebinding. At most 8 connections at once. It opens no event stream: GET gets 405. -- **Read-only is a guard against accidents, not a security boundary.** Every tool opens its server connection with `writable = false`: `SQLITE_OPEN_READONLY`, `PRAGMA query_only` and an authorizer that refuses ordinary writes. Anyone holding the token can still run SQL, so the token is the boundary. +- **Read-only is a guard against accidents, not a security boundary.** Every tool opens its server connection with `writable = false`: `SQLITE_OPEN_READONLY`, `PRAGMA query_only` and an authorizer that refuses ordinary writes, transactions and savepoints, and `PRAGMA locking_mode` and `busy_timeout`, so a client cannot hold a lock on the app's database; the JNI query path also rolls back any transaction left open. Anyone holding the token can still run SQL, so the token is the boundary. - **Only databases an app connection has open.** `DatabaseTools` refuses any other path before the engine is called: opening a file Android's own SQLite has open, with requery's, can corrupt it. The extension has no SQLite API until an app connection has opened, so `sqlitedbg_open_server` returns `SQLITE_MISUSE` before that. - **The app's connections are never altered.** The extension never sets the progress handler or the update hook of an app connection: requery sets and clears the progress handler around cancellable statements, and the update hook belongs to the app. Only server connections use a progress handler, for the time limit. - **Bound values are recorded only while a client is connected.** Expanding the SQL costs memory on every run, so `sqlitedbg_set_client_connected` turns it on; the first `get_events` call does. diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c index 6e3f0f78f8..929be274a2 100644 --- a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c @@ -414,11 +414,17 @@ static int server_authorizer(void *unused, int action, const char *a1, const cha case SQLITE_ANALYZE: case SQLITE_ATTACH: case SQLITE_DETACH: + // A client could open a transaction or savepoint and hold a lock on the app's database. + case SQLITE_TRANSACTION: + case SQLITE_SAVEPOINT: return SQLITE_DENY; case SQLITE_PRAGMA: // PRAGMA query_only=0 would lift the guard, and a PRAGMA that stores a value writes. // PRAGMA table_info(t) also has a second argument, so only these names are checked. + // locking_mode=EXCLUSIVE keeps the lock after a read, and busy_timeout (any form) lets a + // statement wait on the app's lock for as long as the client likes. if (a1 && (sqlite3_stricmp(a1, "query_only") == 0 || sqlite3_stricmp(a1, "writable_schema") == 0 || + sqlite3_stricmp(a1, "locking_mode") == 0 || sqlite3_stricmp(a1, "busy_timeout") == 0 || sqlite3_stricmp(a1, "wal_checkpoint") == 0 || sqlite3_stricmp(a1, "incremental_vacuum") == 0 || sqlite3_stricmp(a1, "optimize") == 0)) return SQLITE_DENY; diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h index 5a40fd7877..d87890daa0 100644 --- a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h @@ -135,7 +135,8 @@ void sqlitedbg_set_client_connected(int connected); // Opens path for the server with the same SQLite the app uses: SQLITE_OPEN_READONLY unless // writable, PRAGMA query_only when read-only, and an authorizer that refuses ordinary writes when -// read-only. Read-only mode guards against accidents; it is not a security boundary (ADFA-7131, +// read-only, and also BEGIN, SAVEPOINT, PRAGMA locking_mode and PRAGMA busy_timeout: a client must not +// be able to hold a lock on the app's database. Read-only mode guards against accidents; it is not a security boundary (ADFA-7131, // D11). Never traced and never listed. The caller owns *db and closes it with sqlite3_close_v2. // A server connection, unlike an app connection, may use a progress handler (for time limits). // Returns SQLITE_MISUSE before any app connection has opened (the extension then has no SQLite diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c index d5fede7078..7793e2e277 100644 --- a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c @@ -377,6 +377,9 @@ static jobjectArray bridge_query(JNIEnv *env, jclass cls, jlong handle, jstring done: if (handler_set) sqlite3_progress_handler(db, 0, NULL, NULL); sqlite3_finalize(st); + // Backstop: a transaction left open (a writable connection, or a way past the authorizer) + // would hold a lock on the app's database until the next query. + if (sqlite3_get_autocommit(db) == 0) sqlite3_exec(db, "ROLLBACK", NULL, NULL, NULL); return result; } diff --git a/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c b/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c index d2606e6997..bcb1c8460d 100644 --- a/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c +++ b/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c @@ -608,6 +608,45 @@ static void test_server(void) { CHECK_EQ(sqlite3_close(app), SQLITE_OK); } +// Each of these would let a client keep a lock on the app's database (ADFA-7135 review, H3). +static void test_server_holds_no_locks(void) { + static const char *const denied[] = { + "BEGIN", + "BEGIN IMMEDIATE", + "BEGIN EXCLUSIVE", + "SAVEPOINT s", + "PRAGMA locking_mode=EXCLUSIVE", + "PRAGMA locking_mode", + "PRAGMA busy_timeout=60000", + "PRAGMA busy_timeout", + }; + for (size_t i = 0; i < sizeof denied / sizeof denied[0]; i++) { + sqlite3 *srv = NULL; + CHECK_EQ(sqlitedbg_open_server(g_db_path, 0, &srv), SQLITE_OK); + int rc = step_all(srv, denied[i]); + if (rc != SQLITE_AUTH) fprintf(stderr, "not refused: %s (%d)\n", denied[i], rc); + CHECK_EQ(rc, SQLITE_AUTH); + // a read on the same connection, then the app must still be able to write + CHECK_EQ(step_all(srv, "SELECT count(*) FROM t"), SQLITE_DONE); + char *err = NULL; + rc = sqlite3_exec(g_main, "INSERT INTO t VALUES(4242)", NULL, NULL, &err); + if (rc != SQLITE_OK) fprintf(stderr, "app write after \"%s\": %s\n", denied[i], err ? err : "?"); + sqlite3_free(err); + CHECK_EQ(rc, SQLITE_OK); + exec_ok(g_main, "DELETE FROM t WHERE a=4242"); + CHECK_EQ(sqlite3_close_v2(srv), SQLITE_OK); + } + + // an ordinary read leaves the server connection with no lock and in autocommit + sqlite3 *srv = NULL; + CHECK_EQ(sqlitedbg_open_server(g_db_path, 0, &srv), SQLITE_OK); + CHECK_EQ(step_all(srv, "SELECT * FROM t"), SQLITE_DONE); + CHECK_EQ(sqlite3_get_autocommit(srv), 1); + exec_ok(g_main, "INSERT INTO t VALUES(4243)"); + exec_ok(g_main, "DELETE FROM t WHERE a=4243"); + CHECK_EQ(sqlite3_close_v2(srv), SQLITE_OK); +} + typedef struct worker { int index; atomic_int *stop; @@ -880,6 +919,8 @@ int main(int argc, char **argv) { test_init_failure(); printf("# server connections\n"); test_server(); + printf("# server holds no locks\n"); + test_server_holds_no_locks(); printf("# threads\n"); test_threads(); if (argc == 4 && strcmp(argv[1], "--write-fixture") == 0) write_fixture(argv[2], argv[3]); From 05c2a2b018b3c3123f5601ede147214658d0c498 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 05:34:42 +0000 Subject: [PATCH 13/24] ADFA-7136 Bound query results and report truncation per cell - M4: QueryLimits.maxTotalBytes (DatabaseTools passes 4 MiB). bridge_query counts the TEXT/BLOB bytes kept and 8 per number, and stops with hasMore before the row that would pass the budget; the first row always goes in so a page makes progress. The tool result text is serialised compact: it was sent twice, pretty and as structuredContent. - L11: the engine reports cut cells (QueryResult.truncatedCellIndexes, row * columns + column) instead of DatabaseTools guessing from size >= limit, so a BLOB of exactly maxBlobBytes is no longer flagged. - L12: get_schema and get_stats read sqlite_master (sqlite_schema needs 3.33). - L13: sqlite3_column_text/blob returning NULL (out of memory) is checked. JVM tests: blob at the limit vs one byte over, the budget pages a result of 3 MiB rows, sqlite_master in the SQL sent, compact result text. Each fails with its fix reverted. The native side (budget, cut cells) is tested by the JNI test in a later commit; L13 has no test (needs an allocation failure). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- subprojects/sqlite-debugger/README.md | 2 +- .../src/main/cpp/sqlitedbg_jni.c | 67 ++++++++++++++++--- .../sqlitedebugger/engine/DebugEngine.kt | 20 +++++- .../sqlitedebugger/engine/NativeBridge.kt | 4 +- .../engine/NativeDebugEngine.kt | 9 ++- .../sqlitedebugger/mcp/DatabaseTools.kt | 35 +++++----- .../sqlitedebugger/mcp/McpProtocol.kt | 2 +- .../sqlitedebugger/engine/FakeDebugEngine.kt | 34 +++++++++- .../sqlitedebugger/mcp/DatabaseToolsTest.kt | 36 +++++++++- .../sqlitedebugger/mcp/McpProtocolTest.kt | 9 +++ 10 files changed, 182 insertions(+), 36 deletions(-) diff --git a/subprojects/sqlite-debugger/README.md b/subprojects/sqlite-debugger/README.md index 1cfc9e69f2..efdae621c8 100644 --- a/subprojects/sqlite-debugger/README.md +++ b/subprojects/sqlite-debugger/README.md @@ -35,7 +35,7 @@ The layout of `sqlitedbg_event_record` lives in `sqlitedbg.h`. The JNI layer han - **Only databases an app connection has open.** `DatabaseTools` refuses any other path before the engine is called: opening a file Android's own SQLite has open, with requery's, can corrupt it. The extension has no SQLite API until an app connection has opened, so `sqlitedbg_open_server` returns `SQLITE_MISUSE` before that. - **The app's connections are never altered.** The extension never sets the progress handler or the update hook of an app connection: requery sets and clears the progress handler around cancellable statements, and the update hook belongs to the app. Only server connections use a progress handler, for the time limit. - **Bound values are recorded only while a client is connected.** Expanding the SQL costs memory on every run, so `sqlitedbg_set_client_connected` turns it on; the first `get_events` call does. -- **Limits.** Connections opened before `install` are not seen, so `DebuggerStartup` runs as early as the process allows. Events are dropped, oldest first, when the buffer (4 MiB by default) is full, and `get_events` reports how many. +- **Limits.** Connections opened before `install` are not seen, so `DebuggerStartup` runs as early as the process allows. Events are dropped, oldest first, when the buffer (4 MiB by default) is full, and `get_events` reports how many. A query result holds at most 4 MiB of cell bytes (`QueryLimits.maxTotalBytes`: TEXT and BLOB bytes kept, 8 per number), so a wide SELECT cannot exhaust the app's heap; the rest is paged (`has_more`). The engine reports which cells were cut (`QueryResult.truncatedCellIndexes`), so a BLOB of exactly the limit is not flagged. ## Testing diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c index 7793e2e277..7033766bd6 100644 --- a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c @@ -258,24 +258,40 @@ static int utf8_prefix(const unsigned char *s, int len, int max) { } // One cell as a Java object, or NULL for SQL NULL (or on failure, with an exception pending). -static jobject cell_value(JNIEnv *env, sqlite3_stmt *st, int col, jint max_text, jint max_blob, int *truncated) { +// *cut is set when a TEXT or BLOB was cut at its limit, and *bytes grows by what the cell costs the +// result: the TEXT or BLOB bytes kept, 8 for a number. +static jobject cell_value(JNIEnv *env, sqlite3_stmt *st, int col, jint max_text, jint max_blob, int *cut, + int64_t *bytes) { + *cut = 0; switch (sqlite3_column_type(st, col)) { case SQLITE_INTEGER: + *bytes += 8; return (*env)->CallStaticObjectMethod(env, g_long, g_long_value_of, (jlong)sqlite3_column_int64(st, col)); case SQLITE_FLOAT: + *bytes += 8; return (*env)->CallStaticObjectMethod(env, g_double, g_double_value_of, (jdouble)sqlite3_column_double(st, col)); case SQLITE_TEXT: { const unsigned char *text = sqlite3_column_text(st, col); + if (!text) { // out of memory + throw_engine(env, SQLITE_NOMEM, "out of memory"); + return NULL; + } int len = sqlite3_column_bytes(st, col); int keep = utf8_prefix(text, len, max_text); - if (keep < len) (*truncated)++; + if (keep < len) *cut = 1; + *bytes += keep; return utf8_string(env, (const char *)text, keep); } case SQLITE_BLOB: { const void *blob = sqlite3_column_blob(st, col); int len = sqlite3_column_bytes(st, col); + if (!blob && len > 0) { // out of memory + throw_engine(env, SQLITE_NOMEM, "out of memory"); + return NULL; + } int keep = len > max_blob ? max_blob : len; - if (keep < len) (*truncated)++; + if (keep < len) *cut = 1; + *bytes += keep; jbyteArray array = (*env)->NewByteArray(env, keep); if (array && keep > 0) (*env)->SetByteArrayRegion(env, array, 0, keep, (const jbyte *)blob); return array; @@ -286,11 +302,13 @@ static jobject cell_value(JNIEnv *env, sqlite3_stmt *st, int col, jint max_text, } static jobjectArray bridge_query(JNIEnv *env, jclass cls, jlong handle, jstring sql, jobjectArray args, jlong offset, - jint max_rows, jint max_text, jint max_blob, jlong deadline_ms) { + jint max_rows, jint max_text, jint max_blob, jint max_total, jlong deadline_ms) { (void)cls; sqlite3 *db = (sqlite3 *)(intptr_t)handle; sqlite3_stmt *st = NULL; jobjectArray result = NULL; + jlong *cut_cells = NULL; // row * columns + column of each cut cell + size_t cut_n = 0, cut_cap = 0; deadline dl = {(int64_t)deadline_ms, 0}; int handler_set = 0; const char *message = NULL; @@ -333,7 +351,8 @@ static jobjectArray bridge_query(JNIEnv *env, jclass cls, jlong handle, jstring (*env)->DeleteLocalRef(env, s); } - int truncated = 0, has_more = 0, row_count = 0; + int has_more = 0, row_count = 0; + int64_t total = 0; jlong skipped = 0; while (st && (rc = sqlite3_step(st)) == SQLITE_ROW) { if (skipped < offset) { @@ -346,14 +365,39 @@ static jobjectArray bridge_query(JNIEnv *env, jclass cls, jlong handle, jstring } jobjectArray row = (*env)->NewObjectArray(env, ncols, g_object, NULL); if (!row) goto done; + int64_t row_bytes = 0; + size_t cut_before = cut_n; for (int c = 0; c < ncols; c++) { - jobject cell = cell_value(env, st, c, max_text, max_blob, &truncated); + int cut; + jobject cell = cell_value(env, st, c, max_text, max_blob, &cut, &row_bytes); if ((*env)->ExceptionCheck(env)) goto done; + if (cut) { + if (cut_n == cut_cap) { + size_t cap = cut_cap ? cut_cap * 2 : 16; + jlong *grown = realloc(cut_cells, cap * sizeof(jlong)); + if (!grown) { + throw_engine(env, SQLITE_NOMEM, "out of memory"); + goto done; + } + cut_cells = grown; + cut_cap = cap; + } + cut_cells[cut_n++] = (jlong)row_count * ncols + c; + } if (cell) { (*env)->SetObjectArrayElement(env, row, c, cell); (*env)->DeleteLocalRef(env, cell); } } + // The total bounds the host app's heap. The first row always goes in, so a page makes + // progress; one row is bounded by the per-cell limits. + if (row_count > 0 && total + row_bytes > max_total) { + (*env)->DeleteLocalRef(env, row); + cut_n = cut_before; + has_more = 1; + break; + } + total += row_bytes; (*env)->CallBooleanMethod(env, rows, g_list_add, row); (*env)->DeleteLocalRef(env, row); if ((*env)->ExceptionCheck(env)) goto done; @@ -365,11 +409,13 @@ static jobjectArray bridge_query(JNIEnv *env, jclass cls, jlong handle, jstring goto done; } - jlongArray counts = (*env)->NewLongArray(env, 2); + // {cut cell count, hasMore, then the index of each cut cell} + jlongArray counts = (*env)->NewLongArray(env, (jsize)(2 + cut_n)); result = counts ? (*env)->NewObjectArray(env, 3, g_object, NULL) : NULL; if (!result) goto done; - jlong values[2] = {truncated, has_more}; - (*env)->SetLongArrayRegion(env, counts, 0, 2, values); + jlong head[2] = {(jlong)cut_n, has_more}; + (*env)->SetLongArrayRegion(env, counts, 0, 2, head); + if (cut_n) (*env)->SetLongArrayRegion(env, counts, 2, (jsize)cut_n, cut_cells); (*env)->SetObjectArrayElement(env, result, 0, names); (*env)->SetObjectArrayElement(env, result, 1, rows); (*env)->SetObjectArrayElement(env, result, 2, counts); @@ -377,6 +423,7 @@ static jobjectArray bridge_query(JNIEnv *env, jclass cls, jlong handle, jstring done: if (handler_set) sqlite3_progress_handler(db, 0, NULL, NULL); sqlite3_finalize(st); + free(cut_cells); // Backstop: a transaction left open (a writable connection, or a way past the authorizer) // would hold a lock on the app's database until the next query. if (sqlite3_get_autocommit(db) == 0) sqlite3_exec(db, "ROLLBACK", NULL, NULL, NULL); @@ -413,7 +460,7 @@ JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) { {"readEvents", "(J[B[J)V", (void *)bridge_read_events}, {"setClientConnected", "(Z)V", (void *)bridge_set_client_connected}, {"openServer", "([BZ)J", (void *)bridge_open_server}, - {"query", "(JLjava/lang/String;[Ljava/lang/Object;JIIIJ)[Ljava/lang/Object;", (void *)bridge_query}, + {"query", "(JLjava/lang/String;[Ljava/lang/Object;JIIIIJ)[Ljava/lang/Object;", (void *)bridge_query}, {"close", "(J)V", (void *)bridge_close}, }; jclass bridge = (*env)->FindClass(env, BRIDGE_CLASS); diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/DebugEngine.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/DebugEngine.kt index 65e476ad52..7aeabc53fa 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/DebugEngine.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/DebugEngine.kt @@ -112,6 +112,12 @@ data class QueryLimits( val maxTextBytes: Int, /** Longer BLOB values are cut and counted in [QueryResult.truncatedCells]. */ val maxBlobBytes: Int, + /** + * A bound on the cell bytes of one result, so a wide SELECT cannot exhaust the app's heap: the + * TEXT and BLOB bytes kept, plus 8 for each number. Rows are added until the next one would pass + * it; the rest is [QueryResult.hasMore]. The first row is always returned. + */ + val maxTotalBytes: Int, ) data class QueryResult( @@ -119,9 +125,19 @@ data class QueryResult( /** Each value is null, [Long], [Double], [String] or [ByteArray]. Doubles may be NaN or infinite. */ val rows: List>, val truncatedCells: Int, - /** More rows follow the last one returned. */ + /** More rows follow the last one returned: the row limit or [QueryLimits.maxTotalBytes] was reached. */ val hasMore: Boolean, -) + /** + * Which cells were cut, as `row * columns.size + column`: a BLOB of exactly [QueryLimits.maxBlobBytes] + * is whole and not listed, one byte more is. [truncatedCells] is its size. + */ + val truncatedCellIndexes: Set = emptySet(), +) { + fun isTruncated( + row: Int, + column: Int, + ) = (row.toLong() * columns.size + column) in truncatedCellIndexes +} class DebugEngineException( /** The SQLite result code, or [SQLITE_MISUSE] when no app connection has opened yet. */ diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeBridge.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeBridge.kt index 578ac83086..fa95abee7b 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeBridge.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeBridge.kt @@ -33,7 +33,8 @@ internal object NativeBridge { /** * `[Array, ArrayList>, LongArray]`: the column names, the rows, and - * `{truncatedCells, hasMore}`. Throws [DebugEngineException]. + * `{truncatedCells, hasMore, then the index of each cut cell}` (row * columns + column). + * Throws [DebugEngineException]. */ @JvmStatic external fun query( @@ -44,6 +45,7 @@ internal object NativeBridge { maxRows: Int, maxTextBytes: Int, maxBlobBytes: Int, + maxTotalBytes: Int, deadlineMillis: Long, ): Array diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeDebugEngine.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeDebugEngine.kt index 00e95c20d1..f020c2d1e7 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeDebugEngine.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeDebugEngine.kt @@ -78,7 +78,9 @@ class NativeDebugEngine : DebugEngine { ): QueryResult { check(handle != 0L) { "the connection is closed" } require(offset >= 0) { "offset $offset is negative" } - require(limits.maxRows > 0 && limits.maxTextBytes >= 0 && limits.maxBlobBytes >= 0) { "bad limits $limits" } + require(limits.maxRows > 0 && limits.maxTextBytes >= 0 && limits.maxBlobBytes >= 0 && limits.maxTotalBytes > 0) { + "bad limits $limits" + } val result = NativeBridge.query( handle, @@ -88,6 +90,7 @@ class NativeDebugEngine : DebugEngine { limits.maxRows, limits.maxTextBytes, limits.maxBlobBytes, + limits.maxTotalBytes, deadlineMillis, ) @@ -97,7 +100,9 @@ class NativeDebugEngine : DebugEngine { @Suppress("UNCHECKED_CAST") val rows = (result[1] as List>).map { it.asList() } val counts = result[2] as LongArray - return QueryResult(columns, rows, truncatedCells = counts[0].toInt(), hasMore = counts[1] != 0L) + val cut = HashSet() + for (i in 2 until counts.size) cut += counts[i] + return QueryResult(columns, rows, truncatedCells = counts[0].toInt(), hasMore = counts[1] != 0L, truncatedCellIndexes = cut) } override fun close() { diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt index 2ff44ec868..53ebf6419a 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt @@ -28,6 +28,9 @@ class DatabaseTools( const val MAX_TEXT_BYTES = 4096 const val MAX_BLOB_BYTES = 1024 + /** The most cell bytes one tool result holds (the engine's `maxTotalBytes`); the rest is paged. */ + const val MAX_TOTAL_BYTES = 4 * 1024 * 1024 + /** Below the client's 60 s limit on a tool call. */ const val DEADLINE_MS = 50_000L const val EVENT_BATCH_BYTES = 256 * 1024 @@ -41,7 +44,7 @@ class DatabaseTools( /** A SQLite value as JSON: org.json throws on NaN and infinities, and JSON numbers lose integers past 2^53. */ fun encodeValue( value: Any?, - maxBlobBytes: Int = MAX_BLOB_BYTES, + truncated: Boolean = false, ): Any = when (value) { null -> { @@ -73,8 +76,7 @@ class DatabaseTools( JSONObject() .put("base64", McpHttp.base64(value, urlSafe = false, pad = true)) .put("bytes", value.size) - // the engine cuts at the limit without saying how long the BLOB was - .put("truncated", value.size >= maxBlobBytes) + .put("truncated", truncated) } else -> { @@ -119,14 +121,15 @@ class DatabaseTools( tool( "get_schema", "Get schema", - "Returns the tables, indexes, views and triggers of a database, with their CREATE statements, from sqlite_schema.", + "Returns the tables, indexes, views and triggers of a database, with their CREATE statements, from sqlite_master.", databaseSchema(), ) { getSchema(it) }, tool( "query", "Run a query", "Runs one read-only SQL statement. Returns at most max_rows rows (default $DEFAULT_MAX_ROWS, at most $MAX_ROWS); " + - "TEXT cells are cut at $MAX_TEXT_BYTES bytes and BLOBs at $MAX_BLOB_BYTES. Use offset to page. " + + "TEXT cells are cut at $MAX_TEXT_BYTES bytes and BLOBs at $MAX_BLOB_BYTES. A result also stops at about " + + "${MAX_TOTAL_BYTES / (1024 * 1024)} MiB of cell bytes (has_more is then true). Use offset to page. " + "$PAGING_NOTE Writes are refused.", databaseSchema( "sql" to prop("string", "The SQL statement."), @@ -235,10 +238,10 @@ class DatabaseTools( val result = run( args, - "SELECT type, name, tbl_name, sql FROM sqlite_schema ORDER BY type, name", + "SELECT type, name, tbl_name, sql FROM sqlite_master ORDER BY type, name", emptyList(), 0, - QueryLimits(MAX_ROWS, 65536, MAX_BLOB_BYTES), + QueryLimits(MAX_ROWS, 65536, MAX_BLOB_BYTES, MAX_TOTAL_BYTES), ) val groups = linkedMapOf("table" to JSONArray(), "index" to JSONArray(), "view" to JSONArray(), "trigger" to JSONArray()) for (row in result.rows) { @@ -257,7 +260,7 @@ class DatabaseTools( val sql = stringArg(args, "sql") val offset = intArg(args, "offset", 0, 0, Long.MAX_VALUE) val maxRows = intArg(args, "max_rows", DEFAULT_MAX_ROWS.toLong(), 1, MAX_ROWS.toLong()).toInt() - val result = run(args, sql, bindArgs(args), offset, QueryLimits(maxRows, MAX_TEXT_BYTES, MAX_BLOB_BYTES)) + val result = run(args, sql, bindArgs(args), offset, QueryLimits(maxRows, MAX_TEXT_BYTES, MAX_BLOB_BYTES, MAX_TOTAL_BYTES)) val out = encodeResult(result, offset) out.put("note", PAGING_NOTE) return out @@ -270,13 +273,13 @@ class DatabaseTools( "EXPLAIN QUERY PLAN " + stringArg(args, "sql"), bindArgs(args), 0, - QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES), + QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES, MAX_TOTAL_BYTES), ) return JSONObject().put("plan", rowsAsObjects(result)) } private fun getStats(args: JSONObject): JSONObject { - val limits = QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES) + val limits = QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES, MAX_TOTAL_BYTES) val deadline = clock() + DEADLINE_MS val path = checkedPath(args) @@ -292,7 +295,7 @@ class DatabaseTools( val tables = runChecked( path, - "SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", emptyList(), 0, limits, @@ -332,7 +335,7 @@ class DatabaseTools( else -> throw McpToolError("full must be true or false.") } val pragma = if (full) "PRAGMA integrity_check" else "PRAGMA quick_check" - val result = run(args, pragma, emptyList(), 0, QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES)) + val result = run(args, pragma, emptyList(), 0, QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES, MAX_TOTAL_BYTES)) val messages = result.rows.map { it.firstOrNull()?.toString() ?: "" } return JSONObject() .put("check", if (full) "integrity_check" else "quick_check") @@ -377,7 +380,9 @@ class DatabaseTools( offset: Long, ): JSONObject { val rows = JSONArray() - for (row in result.rows) rows.put(JSONArray().also { a -> row.forEach { a.put(encodeValue(it)) } }) + result.rows.forEachIndexed { r, row -> + rows.put(JSONArray().also { a -> row.forEachIndexed { c, v -> a.put(encodeValue(v, result.isTruncated(r, c))) } }) + } val out = JSONObject() .put("columns", JSONArray(result.columns)) @@ -392,9 +397,9 @@ class DatabaseTools( private fun rowsAsObjects(result: QueryResult): JSONArray { val arr = JSONArray() - for (row in result.rows) { + result.rows.forEachIndexed { r, row -> val o = JSONObject() - result.columns.forEachIndexed { i, c -> o.put(c, encodeValue(row[i])) } + result.columns.forEachIndexed { i, c -> o.put(c, encodeValue(row[i], result.isTruncated(r, i))) } arr.put(o) } return arr diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt index 6ee7c1b128..21ca3deaa9 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt @@ -199,7 +199,7 @@ class McpProtocol( val result = try { val out = tool.run(args) - JSONObject().put("content", JSONArray().put(text(out.toString(2)))).put("structuredContent", out).put("isError", false) + JSONObject().put("content", JSONArray().put(text(out.toString()))).put("structuredContent", out).put("isError", false) } catch (e: McpToolError) { JSONObject().put("content", JSONArray().put(text(e.message ?: "The tool failed."))).put("isError", true).also { r -> e.details?.let { r.put("structuredContent", it) } diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/FakeDebugEngine.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/FakeDebugEngine.kt index e0b61ce460..ea815dbc2b 100644 --- a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/FakeDebugEngine.kt +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/FakeDebugEngine.kt @@ -34,6 +34,14 @@ class FakeDebugEngine( /** Drops the [count] oldest buffered events, as a full native buffer would. */ fun dropOldest(count: Int) = repeat(count) { buffered.removeFirstOrNull() } + private fun cellBytes(value: Any?): Long = + when (value) { + is String -> value.toByteArray().size.toLong() + is ByteArray -> value.size.toLong() + is Long, is Int, is Double -> 8 + else -> 0 + } + override fun connections() = appConnections.toList() override fun readEvents( @@ -82,8 +90,30 @@ class FakeDebugEngine( ): QueryResult { check(!closed) { "closed" } val all = answer(path, writable, sql, args) - val rows = all.rows.drop(offset.toInt()) - return all.copy(rows = rows.take(limits.maxRows), hasMore = rows.size > limits.maxRows) + val skip = offset.toInt().coerceAtMost(all.rows.size) + val rest = all.rows.drop(skip) + // the native rule: the first row always goes in, then rows while the total stays within the budget + var taken = 0 + var total = 0L + for (row in rest) { + if (taken >= limits.maxRows) break + val bytes = row.sumOf { cellBytes(it) } + if (taken > 0 && total + bytes > limits.maxTotalBytes) break + total += bytes + taken++ + } + val columns = all.columns.size + val cut = + all.truncatedCellIndexes + .map { it - skip.toLong() * columns } + .filter { it >= 0 && it < taken.toLong() * columns } + .toSet() + return all.copy( + rows = rest.take(taken), + hasMore = rest.size > taken, + truncatedCells = if (all.truncatedCellIndexes.isEmpty()) all.truncatedCells else cut.size, + truncatedCellIndexes = cut, + ) } override fun close() { diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt index 702bc4c75f..87c63b9f4b 100644 --- a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt @@ -282,10 +282,39 @@ class DatabaseToolsTest { assertEquals(3, blob.getInt("bytes")) assertFalse(blob.getBoolean("truncated")) assertTrue(row.isNull(7)) - assertEquals(true, DatabaseTools.encodeValue(ByteArray(1024).also { }).let { (it as JSONObject).getBoolean("truncated") }) + assertEquals(true, (DatabaseTools.encodeValue(ByteArray(8), truncated = true) as JSONObject).getBoolean("truncated")) + assertEquals(false, (DatabaseTools.encodeValue(ByteArray(1024)) as JSONObject).getBoolean("truncated")) assertEquals(1.5, DatabaseTools.encodeValue(1.5)) } + @Test fun aBlobExactlyAtTheLimitIsNotTruncatedButOneByteMoreIs() { + engine.appConnections += conn(1, "/data/a.db") + val whole = ByteArray(DatabaseTools.MAX_BLOB_BYTES) + val cut = ByteArray(DatabaseTools.MAX_BLOB_BYTES) // the engine kept maxBlobBytes of a longer value + engine.answer = { _, _, _, _ -> + QueryResult(listOf("a", "b"), listOf(listOf(whole, cut)), 1, false, setOf(1L)) + } + val r = ok("query", db().put("sql", "x")) + val row = r.getJSONArray("rows").getJSONArray(0) + assertFalse(row.getJSONObject(0).getBoolean("truncated")) + assertTrue(row.getJSONObject(1).getBoolean("truncated")) + assertEquals(1, r.getInt("truncated_cells")) + } + + @Test fun aResultStopsAtTheTotalBudgetAndPages() { + engine.appConnections += conn(1, "/data/a.db") + val big = "x".repeat(3 * 1024 * 1024) + engine.answer = { _, _, _, _ -> QueryResult(listOf("v"), listOf(listOf(big), listOf(big), listOf(big)), 0, false) } + val r = ok("query", db().put("sql", "x")) + assertEquals(1, r.getInt("row_count")) + assertTrue(r.getBoolean("has_more")) + assertEquals(1, r.getInt("next_offset")) + val next = ok("query", db().put("sql", "x").put("offset", 1)) + assertEquals(1, next.getInt("row_count")) + assertEquals(2, next.getInt("next_offset")) + assertEquals(4 * 1024 * 1024, DatabaseTools.MAX_TOTAL_BYTES) + } + @Test fun getSchemaGroupsByType() { engine.appConnections += conn(1, "/data/a.db") answerWith( @@ -307,7 +336,8 @@ class DatabaseToolsTest { assertEquals("CREATE INDEX i1 ON t(a)", r.getJSONArray("indexes").getJSONObject(0).getString("sql")) assertEquals(1, r.getJSONArray("views").length()) assertEquals(1, r.getJSONArray("triggers").length()) - assertTrue(seenSql.single().contains("sqlite_schema")) + assertTrue(seenSql.single().contains("FROM sqlite_master")) + assertFalse(seenSql.single().contains("sqlite_schema")) // an alias that needs SQLite 3.33 } @Test fun explainQueryPlanPrefixesTheSql() { @@ -344,6 +374,8 @@ class DatabaseToolsTest { assertEquals(40960, r.getInt("size_bytes")) assertEquals(7, r.getJSONObject("row_counts").getInt("a\"b")) assertTrue(seenSql.contains("SELECT COUNT(*) FROM \"a\"\"b\"")) + assertTrue(seenSql.any { it.contains("FROM sqlite_master") }) + assertTrue(seenSql.none { it.contains("sqlite_schema") }) } @Test fun getStatsSkipsCountsFromTenTables() { diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt index b6db250247..bb7b2c1e53 100644 --- a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt @@ -140,6 +140,15 @@ class McpProtocolTest { assertEquals("text", result.getJSONArray("content").getJSONObject(0).getString("type")) } + @Test fun toolResultTextIsCompactSoTheResultIsNotSentTwiceInFull() { + val p = protocol() + val s = session(p) + val result = call(p, s, "echo", JSONObject().put("x", 3)).getJSONObject("result") + val text = result.getJSONArray("content").getJSONObject(0).getString("text") + assertFalse(text, text.contains("\n")) + assertEquals(result.getJSONObject("structuredContent").toString(), text) + } + @Test fun toolCallWithoutArgumentsGetsEmptyObject() { val p = protocol() val s = session(p) From d309d28af6462f86d0926ed06bb6070b7c548e3a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 05:36:17 +0000 Subject: [PATCH 14/24] ADFA-7136 Drop stale server connections and stop expanding SQL when idle - M7: DatabaseTools cached a server connection per path forever, so after the app deleted and recreated its database the server read the old file. Each cached connection remembers the ids of the app connections to its path when it opened. Before each tool call it is closed and dropped when no live app connection has the path, or none of those ids is left. - L14: the first get_events turned on bound-value expansion until the server stopped, so every app statement was expanded twice forever. It now goes off 60 s after the last get_events call, checked on every tool call and by a daemon timer; the tool description says so. - L10 (one lock for all tool calls) is documented in the class KDoc. Tests: recreated database reopens (opened 2, closed 1), a connection survives while a seen app connection remains, a vanished path closes the cached one, and expansion stops after 60 s. Each fails with its fix reverted (expected 2 but was 1; expected 1 but was 0; "60 s after the last get_events"). Limit accepted: a database replaced in place while the same app connections stay open is not detected. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- subprojects/sqlite-debugger/README.md | 2 +- .../sqlitedebugger/mcp/DatabaseTools.kt | 91 ++++++++++++++++++- .../sqlitedebugger/engine/FakeDebugEngine.kt | 5 + .../sqlitedebugger/mcp/DatabaseToolsTest.kt | 57 ++++++++++++ 4 files changed, 149 insertions(+), 6 deletions(-) diff --git a/subprojects/sqlite-debugger/README.md b/subprojects/sqlite-debugger/README.md index efdae621c8..bd33b17f4c 100644 --- a/subprojects/sqlite-debugger/README.md +++ b/subprojects/sqlite-debugger/README.md @@ -34,7 +34,7 @@ The layout of `sqlitedbg_event_record` lives in `sqlitedbg.h`. The JNI layer han - **Read-only is a guard against accidents, not a security boundary.** Every tool opens its server connection with `writable = false`: `SQLITE_OPEN_READONLY`, `PRAGMA query_only` and an authorizer that refuses ordinary writes, transactions and savepoints, and `PRAGMA locking_mode` and `busy_timeout`, so a client cannot hold a lock on the app's database; the JNI query path also rolls back any transaction left open. Anyone holding the token can still run SQL, so the token is the boundary. - **Only databases an app connection has open.** `DatabaseTools` refuses any other path before the engine is called: opening a file Android's own SQLite has open, with requery's, can corrupt it. The extension has no SQLite API until an app connection has opened, so `sqlitedbg_open_server` returns `SQLITE_MISUSE` before that. - **The app's connections are never altered.** The extension never sets the progress handler or the update hook of an app connection: requery sets and clears the progress handler around cancellable statements, and the update hook belongs to the app. Only server connections use a progress handler, for the time limit. -- **Bound values are recorded only while a client is connected.** Expanding the SQL costs memory on every run, so `sqlitedbg_set_client_connected` turns it on; the first `get_events` call does. +- **Bound values are recorded only while a client is connected.** Expanding the SQL costs memory on every run, so `sqlitedbg_set_client_connected` turns it on; the first `get_events` call does, and it goes off again 60 s after the last one (checked on every tool call and by a timer). - **Limits.** Connections opened before `install` are not seen, so `DebuggerStartup` runs as early as the process allows. Events are dropped, oldest first, when the buffer (4 MiB by default) is full, and `get_events` reports how many. A query result holds at most 4 MiB of cell bytes (`QueryLimits.maxTotalBytes`: TEXT and BLOB bytes kept, 8 per number), so a wide SELECT cannot exhaust the app's heap; the rest is paged (`has_more`). The engine reports which cells were cut (`QueryResult.truncatedCellIndexes`), so a BLOB of exactly the limit is not flagged. ## Testing diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt index 53ebf6419a..5bbed62ebf 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt @@ -1,5 +1,6 @@ package org.appdevforall.sqlitedebugger.mcp +import org.appdevforall.sqlitedebugger.engine.AppConnection import org.appdevforall.sqlitedebugger.engine.DebugEngine import org.appdevforall.sqlitedebugger.engine.DebugEngineException import org.appdevforall.sqlitedebugger.engine.QueryLimits @@ -8,6 +9,8 @@ import org.appdevforall.sqlitedebugger.engine.ServerConnection import org.json.JSONArray import org.json.JSONObject import java.io.File +import java.util.Timer +import java.util.TimerTask /** * The debugger's MCP tools, all read-only: they open server connections through [engine] with @@ -15,6 +18,10 @@ import java.io.File * names a database by the path one of the app's own connections has open; any other path is * refused before the engine is called (opening a file Android's SQLite has open, with the * extension's SQLite, can corrupt it). + * + * One lock guards the cached server connections and every query, so tool calls run one at a time: + * a slow query (up to [DEADLINE_MS]) makes the others wait. That is accepted: the debugger serves + * one agent, and a connection per call would open the app's file many times over. */ class DatabaseTools( private val engine: DebugEngine, @@ -36,6 +43,9 @@ class DatabaseTools( const val EVENT_BATCH_BYTES = 256 * 1024 const val MAX_TABLES_FOR_COUNTS = 10 + /** Bound values stop being recorded this long after the last get_events call. */ + const val EVENTS_IDLE_MS = 60_000L + private const val MAX_SAFE_INTEGER = 9007199254740992L // 2^53 private val SIDE_FILE_SUFFIXES = listOf("-wal", "-shm", "-journal") @@ -90,14 +100,24 @@ class DatabaseTools( } private val lock = Any() - private val serverConnections = HashMap() + private val serverConnections = HashMap() private var clientConnected = false + private var lastEventsCallMs = 0L + private var idleTimer: Timer? = null + + /** A server connection and the ids of the app connections to its path when it was opened. */ + private class CachedConnection( + val connection: ServerConnection, + val appConnectionIds: Set, + ) /** Closes the server connections and tells the engine no client is watching. Call when the server stops. */ fun onServerStopped() { synchronized(lock) { - serverConnections.values.forEach { runCatching { it.close() } } + serverConnections.values.forEach { runCatching { it.connection.close() } } serverConnections.clear() + idleTimer?.cancel() + idleTimer = null if (clientConnected) { clientConnected = false engine.setClientConnected(false) @@ -105,6 +125,35 @@ class DatabaseTools( } } + /** + * Stops recording bound values once [EVENTS_IDLE_MS] have passed since the last get_events call: + * expanding every statement costs memory, and nobody is reading. Checked on every tool call and + * by a timer, so it also happens when no call comes. + */ + fun expireIdleClient() { + synchronized(lock) { + if (clientConnected && clock() - lastEventsCallMs >= EVENTS_IDLE_MS) { + clientConnected = false + engine.setClientConnected(false) + } + } + } + + private fun scheduleIdleCheck() { + val timer = idleTimer ?: Timer("sqlite-debugger-idle", true).also { idleTimer = it } + timer.schedule( + object : TimerTask() { + override fun run() { + expireIdleClient() + synchronized(lock) { + if (clientConnected && idleTimer === timer) scheduleIdleCheck() + } + } + }, + EVENTS_IDLE_MS / 4, + ) + } + fun tools(): List = listOf( tool( @@ -167,7 +216,8 @@ class DatabaseTools( "get_events", "Get SQL events", "Returns the SQL the app ran, oldest first, after the cursor. Pass the returned next as after to continue; " + - "dropped counts events that left the buffer unread. The first call starts recording bound values into the SQL.", + "dropped counts events that left the buffer unread. The first call starts recording bound values into the SQL; " + + "that stops ${EVENTS_IDLE_MS / 1000} s after the last get_events call, so call it at least that often while watching.", JSONObject() .put("type", "object") .put("additionalProperties", false) @@ -182,7 +232,10 @@ class DatabaseTools( description: String, inputSchema: JSONObject = JSONObject().put("type", "object").put("properties", JSONObject()).put("additionalProperties", false), run: (JSONObject) -> JSONObject, - ) = McpTool(name, title, description, inputSchema, readOnly = true, run = run) + ) = McpTool(name, title, description, inputSchema, readOnly = true, run = { args -> + expireIdleClient() + run(args) + }) private fun prop( type: String, @@ -346,9 +399,11 @@ class DatabaseTools( private fun getEvents(args: JSONObject): JSONObject { val after = intArg(args, "after", 0, 0, Long.MAX_VALUE) synchronized(lock) { + lastEventsCallMs = clock() if (!clientConnected) { engine.setClientConnected(true) clientConnected = true + scheduleIdleCheck() } } val batch = @@ -413,6 +468,7 @@ class DatabaseTools( .map { it.path } .filter { it.isNotEmpty() } .distinct() + synchronized(lock) { dropStaleServerConnections(engine.connections()) } if (path !in known) { throw McpToolError( "Unknown database \"$path\": use a path from list_databases that is available.", @@ -440,13 +496,38 @@ class DatabaseTools( ): QueryResult = synchronized(lock) { try { - val conn = serverConnections.getOrPut(path) { engine.openServerConnection(path, writable = false) } + val live = engine.connections() + dropStaleServerConnections(live) + val conn = + serverConnections + .getOrPut(path) { + val ids = live.filter { it.path == path && !it.closing }.map { it.id }.toSet() + CachedConnection(engine.openServerConnection(path, writable = false), ids) + }.connection conn.query(sql, bind, offset, limits, deadline) } catch (e: DebugEngineException) { throw engineError(e) } } + /** + * Closes cached server connections whose file may be gone: after the app deletes and recreates a + * database, a connection opened before still reads the old file. A connection is kept while some + * app connection to its path that was there when it opened is still open; a path no app connection + * has any more, or only new ones, is stale. Call with [lock] held. + */ + private fun dropStaleServerConnections(live: List) { + val it = serverConnections.entries.iterator() + while (it.hasNext()) { + val (path, cached) = it.next() + val ids = live.filter { c -> c.path == path && !c.closing }.map { c -> c.id } + if (ids.none { id -> id in cached.appConnectionIds }) { + runCatching { cached.connection.close() } + it.remove() + } + } + } + private fun engineError(e: DebugEngineException) = McpToolError(e.message ?: "SQLite error ${e.sqliteCode}", JSONObject().put("sqlite_code", e.sqliteCode)) diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/FakeDebugEngine.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/FakeDebugEngine.kt index ea815dbc2b..b6d9a59484 100644 --- a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/FakeDebugEngine.kt +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/FakeDebugEngine.kt @@ -21,6 +21,10 @@ class FakeDebugEngine( val openedServerConnections = mutableListOf>() + /** How many server connections were closed. */ + var closedServerConnections = 0 + private set + fun addEvent( kind: EventKind, sql: String, @@ -117,6 +121,7 @@ class FakeDebugEngine( } override fun close() { + if (!closed) closedServerConnections++ closed = true } } diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt index 87c63b9f4b..21cae2b0a0 100644 --- a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt @@ -423,6 +423,63 @@ class DatabaseToolsTest { assertFalse(engine.clientConnected) } + @Test fun aServerConnectionIsDroppedWhenTheAppRecreatesTheDatabase() { + engine.appConnections += conn(1, "/data/a.db") + val t = tools + ok("get_stats", db(), t) + assertEquals(1, engine.openedServerConnections.size) + // the app closed connection 1, deleted the file and opened a new one: no overlap with {1} + engine.appConnections.clear() + engine.appConnections += conn(2, "/data/a.db") + ok("get_stats", db(), t) + assertEquals(2, engine.openedServerConnections.size) + assertEquals(1, engine.closedServerConnections) + } + + @Test fun aServerConnectionIsKeptWhileASeenAppConnectionRemains() { + engine.appConnections += conn(1, "/data/a.db") + val t = tools + ok("get_stats", db(), t) + engine.appConnections += conn(2, "/data/a.db") + ok("get_stats", db(), t) + engine.appConnections.removeAll { it.id == 2L } + ok("get_stats", db(), t) + assertEquals(1, engine.openedServerConnections.size) + assertEquals(0, engine.closedServerConnections) + } + + @Test fun aServerConnectionIsDroppedWhenNoAppConnectionHasThePathAnyMore() { + engine.appConnections += conn(1, "/data/a.db") + val t = tools + ok("get_stats", db(), t) + engine.appConnections.clear() + engine.appConnections += conn(3, "/data/b.db") + assertTrue(runTool(t, "get_stats", db()).getBoolean("isError")) // refused: no longer a known path + assertEquals(1, engine.closedServerConnections) + } + + @Test fun boundValuesStopBeingRecordedSixtySecondsAfterTheLastGetEvents() { + val t = tools + ok("get_events", t = t) + assertTrue(engine.clientConnected) + now += 59_000 + ok("list_connections", t = t) + assertTrue("still within 60 s", engine.clientConnected) + ok("get_events", t = t) // renews the 60 s + now += 59_000 + ok("list_connections", t = t) + assertTrue(engine.clientConnected) + now += 2_000 + ok("list_connections", t = t) + assertFalse("60 s after the last get_events", engine.clientConnected) + ok("get_events", t = t) + assertTrue("the next get_events turns it on again", engine.clientConnected) + now += 61_000 + t.expireIdleClient() // what the timer calls when no tool call comes + assertFalse(engine.clientConnected) + t.onServerStopped() + } + @Test fun stoppingTheServerClosesServerConnections() { engine.appConnections += conn(1, "/data/a.db") val t = tools From d1f0feb397fc7348603094006ae171a75dc0a638 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 05:54:35 +0000 Subject: [PATCH 15/24] ADFA-7136 Test the JNI query path test.sh now also builds the JNI layer as it ships (libsqlitedbg.so without SQLITE_CORE, with the version script) against a requery-like libsqlite3x.so (amalgamation, THREADSAFE=2, all symbols exported) and runs src/test/jni/JniQueryTest.java on the host JVM through NativeBridge. It is skipped with a message when no JDK is on PATH. It uses Java stubs of NativeBridge and DebugEngineException (same names and native signatures) rather than compiling the Kotlin, so only a JDK is needed; test.sh fails if the native names in NativeBridge.kt and the stub differ. A change of a Kotlin parameter type alone is not caught. Covers: value types, paging and hasMore, the total-bytes budget (including "first row always goes in" and the budget stopping on a row boundary), per-cell truncation at, over and under the limit with row * columns + column indexes and a budget-dropped row's cells, bound types, the deadline giving SQLITE_INTERRUPT, refused writes giving SQLITE_AUTH, BEGIN/SAVEPOINT/PRAGMA refusals leaving the app able to write, and the rollback backstop. Checked to fail with each native fix reverted: budget (50 rows vs 4), cut cell indexes ([] vs [1, 3]), rollback backstop (app INSERT 5 vs 0, SQLITE_BUSY), authorizer (BEGIN: 0 vs 23), dropped row's cut cells ([0, 1] vs [0]). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- .../sqlite-debugger/src/test/jni/Helper.java | 8 + .../src/test/jni/JniQueryTest.java | 255 ++++++++++++++++++ .../sqlite-debugger/src/test/jni/helper.c | 27 ++ .../engine/DebugEngineException.java | 11 + .../sqlitedebugger/engine/NativeBridge.java | 26 ++ subprojects/sqlite-debugger/test.sh | 36 +++ 6 files changed, 363 insertions(+) create mode 100644 subprojects/sqlite-debugger/src/test/jni/Helper.java create mode 100644 subprojects/sqlite-debugger/src/test/jni/JniQueryTest.java create mode 100644 subprojects/sqlite-debugger/src/test/jni/helper.c create mode 100644 subprojects/sqlite-debugger/src/test/jni/org/appdevforall/sqlitedebugger/engine/DebugEngineException.java create mode 100644 subprojects/sqlite-debugger/src/test/jni/org/appdevforall/sqlitedebugger/engine/NativeBridge.java diff --git a/subprojects/sqlite-debugger/src/test/jni/Helper.java b/subprojects/sqlite-debugger/src/test/jni/Helper.java new file mode 100644 index 0000000000..e283c57768 --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/jni/Helper.java @@ -0,0 +1,8 @@ +/** Opens and uses "app" connections with the same libsqlite3x.so the debugger runs in. */ +public class Helper { + static native long open(String path); + + static native int exec(long db, String sql); + + static native int close(long db); +} diff --git a/subprojects/sqlite-debugger/src/test/jni/JniQueryTest.java b/subprojects/sqlite-debugger/src/test/jni/JniQueryTest.java new file mode 100644 index 0000000000..061d813ccb --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/jni/JniQueryTest.java @@ -0,0 +1,255 @@ +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import org.appdevforall.sqlitedebugger.engine.DebugEngineException; +import org.appdevforall.sqlitedebugger.engine.NativeBridge; + +/** + * The real JNI query path (sqlitedbg_jni.c over sqlitedbg.c) on a host JVM, against a SQLite built + * like requery's. test.sh builds libsqlite3x.so, libsqlitedbg.so (no SQLITE_CORE) and libhosthelper.so + * and runs this with them. It calls NativeBridge directly; the Kotlin above it is covered by the + * JVM unit tests with a fake engine. + */ +public class JniQueryTest { + static int failures, checks; + + static void check(boolean ok, String what) { + checks++; + if (!ok) { + failures++; + System.out.println("FAIL " + what); + } + } + + static void eq(Object a, Object b, String what) { + check(Objects.equals(a, b), what + ": " + show(a) + " vs " + show(b)); + } + + static String show(Object o) { + return o instanceof byte[] ? Arrays.toString((byte[]) o) : String.valueOf(o); + } + + static final long FAR = Long.MAX_VALUE / 2; + static final int MAX = 1 << 30; + + /** What NativeBridge.query returns, unpacked. */ + static class Result { + String[] columns; + List rows; + long truncated; + boolean hasMore; + Set cut = new HashSet<>(); + } + + @SuppressWarnings("unchecked") + static Result query(long h, String sql, Object[] args, long offset, int maxRows, int maxText, int maxBlob, int maxTotal, + long deadline) throws DebugEngineException { + Object[] r = NativeBridge.query(h, sql, args, offset, maxRows, maxText, maxBlob, maxTotal, deadline); + Result out = new Result(); + out.columns = (String[]) r[0]; + out.rows = (List) r[1]; + long[] counts = (long[]) r[2]; + out.truncated = counts[0]; + out.hasMore = counts[1] != 0; + eq(counts.length, 2 + (int) counts[0], "one index per cut cell"); + for (int i = 2; i < counts.length; i++) out.cut.add(counts[i]); + return out; + } + + static Result q(long h, String sql) throws DebugEngineException { + return query(h, sql, new Object[0], 0, 1000, 4096, 1024, MAX, FAR); + } + + static int codeOf(long h, String sql, Object[] args, long deadline) { + try { + query(h, sql, args, 0, 100, 4096, 1024, MAX, deadline); + return 0; + } catch (DebugEngineException e) { + return e.sqliteCode; + } + } + + static Set set(Long... v) { + return new HashSet<>(Arrays.asList(v)); + } + + public static void main(String[] a) throws Exception { + System.loadLibrary("sqlite3x"); + System.loadLibrary("sqlitedbg"); + System.loadLibrary("hosthelper"); + + try { + NativeBridge.openServer("/nonexistent/x.db".getBytes(StandardCharsets.UTF_8), false); + check(false, "open before install"); + } catch (DebugEngineException e) { + eq(e.sqliteCode, 21, "SQLITE_MISUSE before any app connection"); + } + eq(NativeBridge.install(), 0, "install"); + eq(NativeBridge.install(), 0, "install twice"); + + Path dir = Files.createTempDirectory("jniquery"); + String path = dir.resolve("app.db").toString(); + long app = Helper.open(path); + eq(Helper.exec(app, "CREATE TABLE t(id INTEGER, r REAL, s TEXT, b BLOB)"), 0, "create"); + eq(Helper.exec(app, "INSERT INTO t VALUES(1, 1.5, 'one', x'0102'),(2, NULL, 'héllo €', x''),(3, 3e999, NULL, NULL)"), 0, + "insert"); + eq(Helper.exec(app, "WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM c WHERE x<300) INSERT INTO t(id) SELECT x+10 FROM c"), + 0, "insert 300"); + + Object[] conns = NativeBridge.connections(); + eq(((String[]) conns[1]).length, 1, "one app connection"); + eq(((String[]) conns[1])[0], path, "its path"); + + byte[] bytes = path.getBytes(StandardCharsets.UTF_8); + long srv = NativeBridge.openServer(bytes, false); + check(srv != 0, "server handle"); + + // value types + Result r = q(srv, "SELECT id, r, s, b FROM t WHERE id < 10 ORDER BY id"); + eq(Arrays.asList(r.columns), Arrays.asList("id", "r", "s", "b"), "columns"); + eq(r.rows.size(), 3, "rows"); + eq(r.rows.get(0)[0], 1L, "long"); + eq(r.rows.get(0)[1], 1.5, "double"); + eq(r.rows.get(0)[2], "one", "text"); + check(Arrays.equals((byte[]) r.rows.get(0)[3], new byte[] {1, 2}), "blob"); + eq(r.rows.get(1)[1], null, "null"); + eq(r.rows.get(1)[2], "héllo €", "utf8 text"); + eq(((byte[]) r.rows.get(1)[3]).length, 0, "empty blob"); + eq(r.rows.get(2)[1], Double.POSITIVE_INFINITY, "infinity"); + eq(r.hasMore, false, "no more"); + eq(r.truncated, 0L, "nothing cut"); + + // paging: 300 rows with id >= 10 + String all = "SELECT id FROM t WHERE id >= 10 ORDER BY id"; + r = query(srv, all, new Object[0], 0, 100, 4096, 1024, MAX, FAR); + eq(r.rows.size(), 100, "page 1"); + eq(r.hasMore, true, "has more after a full page"); + r = query(srv, all, new Object[0], 100, 100, 4096, 1024, MAX, FAR); + eq(r.rows.get(0)[0], 111L, "page 2 starts after the offset"); + r = query(srv, all, new Object[0], 200, 100, 4096, 1024, MAX, FAR); + eq(r.rows.size(), 100, "page 3"); + eq(r.hasMore, false, "exactly the last row of the last page: no more"); + r = query(srv, all, new Object[0], 299, 100, 4096, 1024, MAX, FAR); + eq(r.rows.size(), 1, "last page"); + eq(r.rows.get(0)[0], 310L, "last id"); + r = query(srv, all, new Object[0], 5000, 100, 4096, 1024, MAX, FAR); + eq(r.rows.size(), 0, "past the end"); + eq(r.hasMore, false, "past the end has no more"); + + // M4: the total-bytes budget. 8 bytes per number: 3 numbers = 24 a row, 4 rows fit in 100. + String nums = "WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM c WHERE x<50) SELECT x, x, x FROM c"; + r = query(srv, nums, new Object[0], 0, 1000, 4096, 1024, 100, FAR); + eq(r.rows.size(), 4, "budget 100 holds 4 rows of 24"); + eq(r.hasMore, true, "budget reached: has more"); + r = query(srv, nums, new Object[0], 4, 1000, 4096, 1024, 100, FAR); + eq(r.rows.get(0)[0], 5L, "the next page starts where the budget stopped"); + r = query(srv, nums, new Object[0], 0, 1000, 4096, 1024, 24 * 50, FAR); + eq(r.rows.size(), 50, "a budget of exactly the whole result holds it"); + eq(r.hasMore, false, "and has no more"); + // text and blob count the bytes kept, not the bytes stored + String fat = "WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM c WHERE x<5) " + + "SELECT substr(hex(zeroblob(1000)), 1, 1000), zeroblob(500) FROM c"; + r = query(srv, fat, new Object[0], 0, 1000, 4096, 1024, 3200, FAR); + eq(r.rows.size(), 2, "1500 bytes a row: 2 rows in 3200"); + eq(r.hasMore, true, "fat rows: has more"); + r = query(srv, fat, new Object[0], 0, 1000, 100, 100, 3200, FAR); + eq(r.rows.size(), 5, "cells cut to 100 bytes cost 200 a row: all 5 fit"); + // the first row always goes in, so a page makes progress + r = query(srv, fat, new Object[0], 0, 1000, 4096, 1024, 10, FAR); + eq(r.rows.size(), 1, "one row over the budget is still returned"); + eq(r.hasMore, true, "and more follows"); + + // L11: per-cell truncation, exactly at the limit is whole + r = query(srv, "SELECT x'01020304', x'0102030405', 'abcd', 'abcde'", new Object[0], 0, 10, 4, 4, MAX, FAR); + eq(((byte[]) r.rows.get(0)[0]).length, 4, "blob at the limit"); + eq(((byte[]) r.rows.get(0)[1]).length, 4, "blob over the limit is cut"); + eq(r.rows.get(0)[2], "abcd", "text at the limit"); + eq(r.rows.get(0)[3], "abcd", "text over the limit is cut"); + eq(r.truncated, 2L, "two cells cut"); + eq(r.cut, set(1L, 3L), "cut cells are the over-limit blob and text, not the ones at the limit"); + r = query(srv, "SELECT x'01020304', x'0102030405', 'abcd', 'abcde' UNION ALL SELECT x'0102030405', x'01', 'abcde', 'a'", + new Object[0], 0, 10, 4, 4, MAX, FAR); + eq(r.cut, set(1L, 3L, 4L, 6L), "indexes are row * columns + column"); + eq(r.truncated, 4L, "four cells cut"); + // a character is never split: 'a' + 3 emoji of 4 bytes, 6 bytes allowed + r = query(srv, "SELECT 'a' || '😀😀😀', x'0102030405'", new Object[0], 0, 10, 6, 3, MAX, FAR); + eq(r.rows.get(0)[0], "a😀", "text cut at a character boundary"); + eq(((byte[]) r.rows.get(0)[1]).length, 3, "blob cut"); + eq(r.cut, set(0L, 1L), "both cut"); + // a row left out by the budget takes its cut cells with it + r = query(srv, "SELECT 'abcde' UNION ALL SELECT 'abcde'", new Object[0], 0, 10, 4, 4, 6, FAR); + eq(r.rows.size(), 1, "the second row would pass the budget"); + eq(r.cut, set(0L), "only the kept row's cut cell is reported"); + eq(r.truncated, 1L, "and counted"); + + // bound values of every type + r = query(srv, "SELECT ?, ?, ?, ?, ?", new Object[] {7L, 2.25, "a😀b", new byte[] {9, 8, 7}, null}, 0, 10, 4096, 1024, + MAX, FAR); + Object[] row = r.rows.get(0); + eq(row[0], 7L, "arg long"); + eq(row[1], 2.25, "arg double"); + eq(row[2], "a😀b", "arg text"); + check(Arrays.equals((byte[]) row[3], new byte[] {9, 8, 7}), "arg blob"); + eq(row[4], null, "arg null"); + eq(q(srv, "SELECT ?").rows.get(0)[0], null, "an unbound parameter is NULL"); + eq(codeOf(srv, "SELECT ?", new Object[] {1L, 2L}, FAR), 25, "too many args: SQLITE_RANGE"); + eq(codeOf(srv, "SELECT ?", new Object[] {new Object()}, FAR), 21, "unsupported arg type: SQLITE_MISUSE"); + eq(codeOf(srv, "SELEKT 1", new Object[0], FAR), 1, "syntax error: SQLITE_ERROR"); + + // a refused write is SQLITE_AUTH, and the data is untouched + eq(codeOf(srv, "INSERT INTO t(id) VALUES(5)", new Object[0], FAR), 23, "write refused: SQLITE_AUTH"); + eq(codeOf(srv, "DELETE FROM t", new Object[0], FAR), 23, "delete refused: SQLITE_AUTH"); + eq(q(srv, "SELECT count(*) FROM t WHERE id = 5").rows.get(0)[0], 0L, "nothing written"); + + // H3: a transaction cannot be opened, so the app is never locked out + for (String sql : new String[] {"BEGIN", "BEGIN IMMEDIATE", "SAVEPOINT s", "PRAGMA locking_mode=EXCLUSIVE", "PRAGMA busy_timeout=60000"}) { + eq(codeOf(srv, sql, new Object[0], FAR), 23, sql + " refused: SQLITE_AUTH"); + q(srv, "SELECT count(*) FROM t"); + eq(Helper.exec(app, "INSERT INTO t(id) VALUES(4242)"), 0, "the app can write after " + sql); + Helper.exec(app, "DELETE FROM t WHERE id = 4242"); + } + + // the deadline stops a statement that never ends, and the connection stays usable + long t0 = System.currentTimeMillis(); + eq(codeOf(srv, "WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM c) SELECT count(*) FROM c", new Object[0], t0 + 300), 9, + "deadline: SQLITE_INTERRUPT"); + long took = System.currentTimeMillis() - t0; + check(took >= 250 && took < 3000, "the deadline stopped it in " + took + " ms"); + eq(codeOf(srv, "SELECT 1", new Object[0], t0 - 1), 9, "a deadline already past: SQLITE_INTERRUPT"); + eq(q(srv, "SELECT 41 + 1").rows.get(0)[0], 42L, "usable after the deadline"); + eq(Helper.exec(app, "INSERT INTO t(id) VALUES(4243)"), 0, "the app can write after an interrupted query"); + Helper.exec(app, "DELETE FROM t WHERE id = 4243"); + + // the server connection is not listed + eq(((String[]) NativeBridge.connections()[1]).length, 1, "server not listed"); + NativeBridge.close(srv); + + // the backstop: a writable connection that opens a transaction does not keep it + long w = NativeBridge.openServer(bytes, true); + q(w, "BEGIN IMMEDIATE"); + eq(Helper.exec(app, "INSERT INTO t(id) VALUES(4244)"), 0, "the app can write after a writable BEGIN IMMEDIATE"); + Helper.exec(app, "DELETE FROM t WHERE id = 4244"); + q(w, "BEGIN"); + q(w, "INSERT INTO t(id) VALUES(4245)"); + eq(q(w, "SELECT count(*) FROM t WHERE id = 4245").rows.get(0)[0], 1L, "an autocommit insert on the writable connection stays"); + NativeBridge.close(w); + Helper.exec(app, "DELETE FROM t WHERE id = 4245"); + + try { + NativeBridge.openServer(dir.resolve("none.db").toString().getBytes(StandardCharsets.UTF_8), false); + check(false, "missing file"); + } catch (DebugEngineException e) { + eq(e.sqliteCode, 14, "SQLITE_CANTOPEN"); + } + + eq(Helper.close(app), 0, "close the app connection"); + eq(((String[]) NativeBridge.connections()[1]).length, 0, "all freed"); + System.out.println(checks + " checks, " + failures + " failures"); + System.exit(failures == 0 ? 0 : 1); + } +} diff --git a/subprojects/sqlite-debugger/src/test/jni/helper.c b/subprojects/sqlite-debugger/src/test/jni/helper.c new file mode 100644 index 0000000000..9fe88d041a --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/jni/helper.c @@ -0,0 +1,27 @@ +// The "app" side of the JNI test: plain sqlite3 calls into libsqlite3x.so, as requery's do. +#include +#include +#include "sqlite3.h" + +JNIEXPORT jlong JNICALL Java_Helper_open(JNIEnv *e, jclass c, jstring p) { + (void)c; + const char *s = (*e)->GetStringUTFChars(e, p, 0); + sqlite3 *db = 0; + sqlite3_open(s, &db); + (*e)->ReleaseStringUTFChars(e, p, s); + return (jlong)(intptr_t)db; +} + +JNIEXPORT jint JNICALL Java_Helper_exec(JNIEnv *e, jclass c, jlong db, jstring q) { + (void)c; + const char *s = (*e)->GetStringUTFChars(e, q, 0); + int rc = sqlite3_exec((sqlite3 *)(intptr_t)db, s, 0, 0, 0); + (*e)->ReleaseStringUTFChars(e, q, s); + return rc; +} + +JNIEXPORT jint JNICALL Java_Helper_close(JNIEnv *e, jclass c, jlong db) { + (void)e; + (void)c; + return sqlite3_close((sqlite3 *)(intptr_t)db); +} diff --git a/subprojects/sqlite-debugger/src/test/jni/org/appdevforall/sqlitedebugger/engine/DebugEngineException.java b/subprojects/sqlite-debugger/src/test/jni/org/appdevforall/sqlitedebugger/engine/DebugEngineException.java new file mode 100644 index 0000000000..bbdbdf276f --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/jni/org/appdevforall/sqlitedebugger/engine/DebugEngineException.java @@ -0,0 +1,11 @@ +package org.appdevforall.sqlitedebugger.engine; + +/** Stands in for DebugEngineException in DebugEngine.kt: the JNI layer constructs it as (int, String). */ +public class DebugEngineException extends Exception { + public final int sqliteCode; + + public DebugEngineException(int sqliteCode, String message) { + super(message); + this.sqliteCode = sqliteCode; + } +} diff --git a/subprojects/sqlite-debugger/src/test/jni/org/appdevforall/sqlitedebugger/engine/NativeBridge.java b/subprojects/sqlite-debugger/src/test/jni/org/appdevforall/sqlitedebugger/engine/NativeBridge.java new file mode 100644 index 0000000000..bc4ee25852 --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/jni/org/appdevforall/sqlitedebugger/engine/NativeBridge.java @@ -0,0 +1,26 @@ +package org.appdevforall.sqlitedebugger.engine; + +/** + * Stands in for NativeBridge.kt in the host JNI test (test.sh): the same class name and native + * signatures, which sqlitedbg_jni.c registers by name in JNI_OnLoad, so a Java-only toolchain can + * run the real C. test.sh checks that the native names here match the ones in NativeBridge.kt. + */ +public final class NativeBridge { + private NativeBridge() {} + + public static native int install(); + + public static native Object[] connections(); + + public static native void readEvents(long after, byte[] buf, long[] out); + + public static native void setClientConnected(boolean connected); + + public static native long openServer(byte[] path, boolean writable) throws DebugEngineException; + + public static native Object[] query(long handle, String sql, Object[] args, long offset, int maxRows, int maxTextBytes, + int maxBlobBytes, int maxTotalBytes, long deadlineMillis) + throws DebugEngineException; + + public static native void close(long handle); +} diff --git a/subprojects/sqlite-debugger/test.sh b/subprojects/sqlite-debugger/test.sh index 455dffa987..ce80e1702a 100755 --- a/subprojects/sqlite-debugger/test.sh +++ b/subprojects/sqlite-debugger/test.sh @@ -6,6 +6,8 @@ # --update-fixture rewrites src/test/resources/.../events.bin and events.tsv, which the JVM test # of EventDecoder reads; without it the committed pair is checked against the # current record layout instead. +# Then, if a JDK is on PATH (javac, java, jni.h), the JNI layer is built as on a device (no SQLITE_CORE, +# against a requery-like libsqlite3x.so) and src/test/jni/JniQueryTest.java runs the real query path. # Needs gcc, curl, tar and sha256sum. set -euo pipefail HERE=$(cd "$(dirname "$0")" && pwd) @@ -65,3 +67,37 @@ else "$WORK/asan-ubsan/test" --check-fixture "$RES/events.bin" "$RES/events.tsv" fi run_with tsan thread + +# The JNI query path on a host JVM. The Java stubs of NativeBridge and DebugEngineException under +# src/test/jni stand in for the Kotlin classes (no Kotlin compiler needed); the native names are +# checked against NativeBridge.kt below. +run_jni () { + local jdk + jdk=$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")") + [ -f "$jdk/include/jni.h" ] || { echo "# jni: skipped, no jni.h under $jdk"; return 0; } + echo "# jni" + local jni=$HERE/src/test/jni out=$WORK/jni + mkdir -p "$out/lib" "$out/classes" + local kotlin_names java_names + kotlin_names=$(sed -n 's/.*external fun \([A-Za-z]*\)(.*/\1/p' "$HERE/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeBridge.kt" | sort | tr '\n' ' ') + java_names=$(sed -n 's/.*static native .* \([A-Za-z]*\)(.*/\1/p' "$jni/org/appdevforall/sqlitedebugger/engine/NativeBridge.java" | sort | tr '\n' ' ') + [ "$kotlin_names" = "$java_names" ] || { echo "NativeBridge.kt has ($kotlin_names) but the test stub has ($java_names)"; exit 1; } + local jinc=(-I"$jdk/include" -I"$jdk/include/linux") + # as requery builds it: everything exported, so dlsym finds sqlite3_auto_extension + build gcc -O1 -w -shared -fPIC -DSQLITE_THREADSAFE=2 -DSQLITE_DQS=0 -DSQLITE_DEFAULT_MEMSTATUS=0 -I"$AMAL" \ + "$AMAL/sqlite3.c" -o "$out/lib/libsqlite3x.so" -lpthread -ldl -lm + build gcc -O1 -Wall -Wextra -Werror -shared -fPIC -fvisibility=hidden -I"$AMAL" -I"$CPP" "${jinc[@]}" \ + "$CPP/sqlitedbg.c" "$CPP/sqlitedbg_jni.c" -Wl,--version-script="$CPP/sqlitedbg.map" \ + -o "$out/lib/libsqlitedbg.so" -lpthread -ldl + build gcc -O1 -Wall -Wextra -Werror -shared -fPIC -I"$AMAL" "${jinc[@]}" "$jni/helper.c" \ + -L"$out/lib" -lsqlite3x -Wl,-rpath,"$out/lib" -o "$out/lib/libhosthelper.so" + build javac -d "$out/classes" "$jni/org/appdevforall/sqlitedebugger/engine/NativeBridge.java" \ + "$jni/org/appdevforall/sqlitedebugger/engine/DebugEngineException.java" "$jni/Helper.java" "$jni/JniQueryTest.java" + LD_LIBRARY_PATH="$out/lib" java -Djava.library.path="$out/lib" -cp "$out/classes" JniQueryTest +} + +if command -v javac >/dev/null && command -v java >/dev/null; then + run_jni +else + echo "# jni: skipped, no JDK (javac and java) on PATH" +fi From 7684331c2bcb54a7ce21c4823fc8bbe4a81ad9d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 05:54:43 +0000 Subject: [PATCH 16/24] ADFA-7136 Document the JNI test and the requery trace limit README: the JNI test and how it stands in for the Kotlin, and that requery's own sqlite3_trace (log.tag.SQLiteStatements or SQLiteTime VERBOSE) replaces ours on a connection, so its events stop. No code change for that. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAZ8yszEP4MqbR9PLba --- subprojects/sqlite-debugger/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/subprojects/sqlite-debugger/README.md b/subprojects/sqlite-debugger/README.md index bd33b17f4c..7421b66283 100644 --- a/subprojects/sqlite-debugger/README.md +++ b/subprojects/sqlite-debugger/README.md @@ -35,13 +35,14 @@ The layout of `sqlitedbg_event_record` lives in `sqlitedbg.h`. The JNI layer han - **Only databases an app connection has open.** `DatabaseTools` refuses any other path before the engine is called: opening a file Android's own SQLite has open, with requery's, can corrupt it. The extension has no SQLite API until an app connection has opened, so `sqlitedbg_open_server` returns `SQLITE_MISUSE` before that. - **The app's connections are never altered.** The extension never sets the progress handler or the update hook of an app connection: requery sets and clears the progress handler around cancellable statements, and the update hook belongs to the app. Only server connections use a progress handler, for the time limit. - **Bound values are recorded only while a client is connected.** Expanding the SQL costs memory on every run, so `sqlitedbg_set_client_connected` turns it on; the first `get_events` call does, and it goes off again 60 s after the last one (checked on every tool call and by a timer). -- **Limits.** Connections opened before `install` are not seen, so `DebuggerStartup` runs as early as the process allows. Events are dropped, oldest first, when the buffer (4 MiB by default) is full, and `get_events` reports how many. A query result holds at most 4 MiB of cell bytes (`QueryLimits.maxTotalBytes`: TEXT and BLOB bytes kept, 8 per number), so a wide SELECT cannot exhaust the app's heap; the rest is paged (`has_more`). The engine reports which cells were cut (`QueryResult.truncatedCellIndexes`), so a BLOB of exactly the limit is not flagged. +- **Limits.** Connections opened before `install` are not seen, so `DebuggerStartup` runs as early as the process allows. requery's own `sqlite3_trace` replaces ours on a connection when `log.tag.SQLiteStatements` or `log.tag.SQLiteTime` is VERBOSE, so events from such connections stop. Events are dropped, oldest first, when the buffer (4 MiB by default) is full, and `get_events` reports how many. A query result holds at most 4 MiB of cell bytes (`QueryLimits.maxTotalBytes`: TEXT and BLOB bytes kept, 8 per number), so a wide SELECT cannot exhaust the app's heap; the rest is paged (`has_more`). The engine reports which cells were cut (`QueryResult.truncatedCellIndexes`), so a BLOB of exactly the limit is not flagged. ## Testing - **Host C tests of the extension:** `subprojects/sqlite-debugger/test.sh [--update-fixture] [AMALGAMATION_TARBALL]`. It builds SQLite 3.49.1 from the amalgamation as requery builds it (`SQLITE_THREADSAFE=2`), checks its headers against `sqlite-js/upstream/libs`, and runs `src/test/c/sqlitedbg_test.c` under AddressSanitizer with UBSan, then under ThreadSanitizer. It needs gcc, curl, tar and sha256sum, and downloads the tarball unless you pass one. - **The fixture:** `src/test/resources/.../engine/events.bin` and `events.tsv` are written by the C test and read by `EventDecoderTest`. Without `--update-fixture`, `test.sh` checks the committed pair against the current record layout, so a layout change fails until you regenerate it and update `EventDecoder.kt`. -- **JVM tests:** `flox activate -d flox/local -- ./gradlew :subprojects:sqlite-debugger:testDebugUnitTest`. They cover `EventDecoder`, `McpProtocol` and `DatabaseTools` (with `FakeDebugEngine`). +- **JNI test:** when a JDK is on PATH, `test.sh` also builds `libsqlitedbg.so` as it ships (no `SQLITE_CORE`) against a requery-like `libsqlite3x.so` and runs `src/test/jni/JniQueryTest.java` through `NativeBridge`: paging, the total-bytes budget, per-cell truncation, bound types, the deadline, refused writes and transactions. It uses Java stubs of `NativeBridge` and `DebugEngineException` instead of the Kotlin, so a JDK is all it needs; `test.sh` checks the native names match `NativeBridge.kt`. Otherwise it is skipped with a message. +- **JVM tests:** `flox activate -d flox/local -- ./gradlew :subprojects:sqlite-debugger:testDebugUnitTest`. They cover `EventDecoder`, `McpProtocol`, `McpHttp` and `DatabaseTools` (with `FakeDebugEngine`). No workflow in `.github/workflows` runs either yet. From 9d87b3e4163d0b04de3f671ac845e5fe3c88cd66 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Thu, 12 Nov 2026 21:07:11 -0800 Subject: [PATCH 17/24] ADFA-7135 Set the Kotlin JVM target on the compile tasks The root build sets jvmTarget 17 on every KotlinCompile task, which overrides the module's kotlin { compilerOptions } and made compileDebugKotlin fail against compileOptions' Java 1.8 ("Inconsistent JVM-target compatibility"). Setting it on the tasks keeps 1.8, as sqlite-js uses. First compile of the Android-only files (NativeDebugEngine.kt, DebuggerStartup.kt, sqlitedbg_jni.c, the manifest): with this change :subprojects:sqlite-debugger:assembleDebug builds both ABIs, the C compiles without warnings under -Wall -Wextra, libsqlitedbg.so exports only JNI_OnLoad, and testDebugUnitTest passes 56 tests. Co-Authored-By: Claude Opus 5.5 --- subprojects/sqlite-debugger/build.gradle.kts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/subprojects/sqlite-debugger/build.gradle.kts b/subprojects/sqlite-debugger/build.gradle.kts index cfcf5bd174..2acf1dd65c 100644 --- a/subprojects/sqlite-debugger/build.gradle.kts +++ b/subprojects/sqlite-debugger/build.gradle.kts @@ -45,9 +45,14 @@ android { } } +// Set on the tasks, not the extension: the root build sets every KotlinCompile task's jvmTarget +// to 17, which overrides the extension and no longer matches compileOptions above. +tasks.withType().configureEach { + compilerOptions.jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_1_8) +} + kotlin { compilerOptions { - jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_1_8) // Apps compile against this library with Code on the Go's on-device Kotlin compiler, // which can be older than the one that builds it (as in :plugin-api). apiVersion.set(org.jetbrains.kotlin.gradle.dsl.KotlinVersion.KOTLIN_2_0) From 3e1830fbec93b4ce5f87d2ab15c726fea9e9cb73 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 15 Nov 2026 22:45:22 -0800 Subject: [PATCH 18/24] ADFA-7136 Bound the request head and refuse before reading the body; say the server is read-only A local app could hold all 8 connection slots without authenticating: the 30 s timeout applied per read and the token was checked after the whole request. The head now has an overall deadline (5 s), and Host, Origin, path and token are checked before the body. The initialize instructions no longer mention a read-write mode that does not exist. Tests: sockets that send nothing or trickle bytes are closed within the deadline and the real client is then served; an unauthenticated request with an unsent body gets 401; the instructions do not mention read-write. All three fail with the fix reverted. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAYT7GPoC2cDzo8LErn --- subprojects/sqlite-debugger/README.md | 2 +- .../sqlitedebugger/mcp/McpHttp.kt | 93 +++++++++++++++---- .../sqlitedebugger/mcp/McpProtocol.kt | 2 +- .../sqlitedebugger/mcp/McpProtocolTest.kt | 63 +++++++++++++ 4 files changed, 138 insertions(+), 22 deletions(-) diff --git a/subprojects/sqlite-debugger/README.md b/subprojects/sqlite-debugger/README.md index 7421b66283..9e8aaaa95a 100644 --- a/subprojects/sqlite-debugger/README.md +++ b/subprojects/sqlite-debugger/README.md @@ -30,7 +30,7 @@ The layout of `sqlitedbg_event_record` lives in `sqlitedbg.h`. The JNI layer han ## Decisions that shape it - **Debug builds only.** `DebuggerStartup.onCreate` throws if the process is not debuggable, so a release build that includes the library by mistake crashes at once instead of opening a local server. -- **Loopback HTTP, a random port and token per start.** `McpHttp` binds 127.0.0.1 only (port 0, so the system picks one) and requires `Authorization: Bearer `, a new 32-byte random token each process start. It also checks that the Host and any Origin name 127.0.0.1 or localhost, against DNS rebinding. At most 8 connections at once. It opens no event stream: GET gets 405. +- **Loopback HTTP, a random port and token per start.** `McpHttp` binds 127.0.0.1 only (port 0, so the system picks one) and requires `Authorization: Bearer `, a new 32-byte random token each process start. It also checks that the Host and any Origin name 127.0.0.1 or localhost, against DNS rebinding. At most 8 connections at once. The head of a request (request line and headers) must arrive within 5 s, from connecting for a connection's first request and from its first byte for later ones, and Host, Origin, path and token are checked before the body is read, so a client that trickles bytes or never authenticates frees its slot; a refused request closes the connection. It opens no event stream: GET gets 405. - **Read-only is a guard against accidents, not a security boundary.** Every tool opens its server connection with `writable = false`: `SQLITE_OPEN_READONLY`, `PRAGMA query_only` and an authorizer that refuses ordinary writes, transactions and savepoints, and `PRAGMA locking_mode` and `busy_timeout`, so a client cannot hold a lock on the app's database; the JNI query path also rolls back any transaction left open. Anyone holding the token can still run SQL, so the token is the boundary. - **Only databases an app connection has open.** `DatabaseTools` refuses any other path before the engine is called: opening a file Android's own SQLite has open, with requery's, can corrupt it. The extension has no SQLite API until an app connection has opened, so `sqlitedbg_open_server` returns `SQLITE_MISUSE` before that. - **The app's connections are never altered.** The extension never sets the progress handler or the update hook of an app connection: requery sets and clears the progress handler around cancellable statements, and the update hook belongs to the app. Only server connections use a progress handler, for the time limit. diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt index ef1028cc7f..dc7db46d46 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt @@ -41,6 +41,8 @@ class McpHttp( private val log: (String) -> Unit = {}, /** Creates the listening socket; tests pass one whose accept fails. */ private val socketFactory: () -> ServerSocket = { ServerSocket() }, + /** How long a request's head may take to arrive; see [HEAD_DEADLINE_MS]. Tests shorten it. */ + private val headDeadlineMs: Int = HEAD_DEADLINE_MS, ) { companion object { private const val B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" @@ -86,6 +88,13 @@ class McpHttp( /** A connection with no request for this long is closed. */ const val IDLE_TIMEOUT_MS = 30_000 + /** + * The whole head of a request (request line and headers) must arrive within this, counted from + * the connection for its first request and from the first byte for later ones, so a client + * that trickles bytes or never authenticates frees its slot. + */ + const val HEAD_DEADLINE_MS = 5_000 + /** After a failed accept the loop sleeps this long, doubling up to the maximum while failures continue. */ const val ACCEPT_BACKOFF_MIN_MS = 100L const val ACCEPT_BACKOFF_MAX_MS = 2_000L @@ -114,7 +123,7 @@ class McpHttp( val target: String, val version: String, val headers: Map, - val body: ByteArray, + var body: ByteArray = ByteArray(0), ) private class HttpError( @@ -218,10 +227,26 @@ class McpHttp( c.tcpNoDelay = true val input = BufferedInputStream(c.getInputStream()) val out = c.getOutputStream() + var headStart = System.currentTimeMillis() + var first = true while (running) { val req = try { - readRequest(input) ?: return + if (!first) { + // Between requests the client may idle; the head's clock starts at its first byte. + c.soTimeout = IDLE_TIMEOUT_MS + input.mark(1) + if (input.read() < 0) return + input.reset() + headStart = System.currentTimeMillis() + } + first = false + val head = readHead(input, c, headStart + headDeadlineMs) ?: return + // Refuse before reading a body, so an unauthenticated client never gets that far. + if (!allowed(head, out)) return + c.soTimeout = IDLE_TIMEOUT_MS + head.body = readBody(input, head.headers) + head } catch (e: SocketTimeoutException) { return } catch (e: HttpError) { @@ -232,6 +257,34 @@ class McpHttp( } } + /** Host, Origin, path and token. Answers and returns false when the request is refused; the connection then closes. */ + private fun allowed( + r: Request, + out: OutputStream, + ): Boolean { + fun refuse( + status: Int, + message: String, + headers: Map = emptyMap(), + ): Boolean { + respond(out, status, rpcError(message), headers, close = true) + return false + } + if (!hostAllowed(r.headers["host"])) return refuse(403, "The Host header must be 127.0.0.1 or localhost.") + val origin = r.headers["origin"] + if (origin != null && !originAllowed(origin)) return refuse(403, "Requests from web pages aren't accepted.") + if (r.target.substringBefore('?') != PATH) return refuse(404, "Not found: the MCP endpoint is $PATH.") + if (!authorized(r.headers["authorization"])) { + return refuse( + 401, + "Missing or wrong token. Code on the Go gives it to its Agent; " + + "send it as Authorization: Bearer .", + mapOf("WWW-Authenticate" to "Bearer realm=\"SQLite debugger\""), + ) + } + return true + } + /** Answers one request. Returns whether the connection stays open for another. */ private fun handle( r: Request, @@ -247,20 +300,6 @@ class McpHttp( respond(out, status, body, headers, close) return !close } - if (!hostAllowed(r.headers["host"])) return send(403, rpcError("The Host header must be 127.0.0.1 or localhost.")) - val origin = r.headers["origin"] - if (origin != null && !originAllowed(origin)) return send(403, rpcError("Requests from web pages aren't accepted.")) - if (r.target.substringBefore('?') != PATH) return send(404, rpcError("Not found: the MCP endpoint is $PATH.")) - if (!authorized(r.headers["authorization"])) { - return send( - 401, - rpcError( - "Missing or wrong token. Code on the Go gives it to its Agent; " + - "send it as Authorization: Bearer .", - ), - mapOf("WWW-Authenticate" to "Bearer realm=\"SQLite debugger\""), - ) - } val session = r.headers["mcp-session-id"] val reply = when (r.method) { @@ -322,14 +361,28 @@ class McpHttp( // ---------------------------------------------------------------- reading and writing HTTP - /** One request, or null if the connection closed before one began. */ - private fun readRequest(input: InputStream): Request? { + /** The head of one request (no body yet), or null if the connection closed before one began. */ + private fun readHead( + input: InputStream, + c: Socket, + deadlineAt: Long, + ): Request? { var budget = MAX_HEADER_BYTES + fun read(): Int { + // Each blocking read may wait only as long as the head has left, however slowly bytes come. + if (input.available() == 0) { + val left = deadlineAt - System.currentTimeMillis() + if (left <= 0) throw SocketTimeoutException("the request's head took too long") + c.soTimeout = minOf(left, IDLE_TIMEOUT_MS.toLong()).toInt() + } + return input.read() + } + fun line(): String? { val b = ByteArrayOutputStream() while (true) { - val ch = input.read() + val ch = read() if (ch < 0) { if (b.size() == 0) return null else throw HttpError(400, "The request ended early.") } @@ -354,7 +407,7 @@ class McpHttp( // A repeated header is joined, so two Host headers fail the Host check. headers[name] = headers[name]?.let { "$it, $value" } ?: value } - return Request(parts[0], parts[1], parts[2], headers, readBody(input, headers)) + return Request(parts[0], parts[1], parts[2], headers) } private fun readBody( diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt index 21ca3deaa9..34cf7b122b 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt @@ -66,7 +66,7 @@ class McpProtocol( const val INVALID_PARAMS = -32602 const val INSTRUCTIONS = "Inspects and queries the SQLite databases of the app this server runs in, a debug " + - "build made with Code on the Go. Read-only unless the app's developer turned on read-write mode." + "build made with Code on the Go. Every tool is read-only." } /** What to send back: an HTTP status, a JSON body (or none), and extra headers. */ diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt index bb7b2c1e53..b1d48bf176 100644 --- a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt @@ -80,6 +80,8 @@ class McpProtocolTest { assertEquals("2025-03-26", result.getString("protocolVersion")) assertEquals(McpProtocol.SERVER_NAME, result.getJSONObject("serverInfo").getString("name")) assertTrue(result.getJSONObject("capabilities").has("tools")) + // no read-write mode exists, so the agent must not be told one may be on + assertFalse(result.getString("instructions").contains("read-write", ignoreCase = true)) assertEquals(1, p.sessionCount()) } @@ -466,4 +468,65 @@ class McpProtocolTest { assertEquals(204, http(port, "DELETE", headers = auth + ("Mcp-Session-Id" to s)).status) assertEquals(404, http(port, "POST", headers = auth + ("Mcp-Session-Id" to s), body = ping).status) } + + @Test fun clientsThatTrickleOrNeverAuthenticateAreDroppedAndDoNotBlockTheRealOne() { + val s = McpHttp(protocol(), { token }, port = 0, headDeadlineMs = 600) + s.start() + server = s + val port = s.boundPort + val started = System.nanoTime() + val held = mutableListOf() + try { + repeat(McpHttp.MAX_CONNECTIONS) { i -> + val c = Socket("127.0.0.1", port) + c.soTimeout = 5000 + held += c + // half of them send a byte now and then, which resets a per-read timeout but not the deadline + if (i % 2 == 0) { + c.getOutputStream().apply { + write("POST /mcp HTTP/1.1\r\nHost: 127.0.0.1\r\n".toByteArray()) + flush() + } + } + } + // the slots are full: a real client is refused with 503 + assertEquals(503, http(port, "POST", headers = auth, body = initBody()).status) + val trickler = + Thread { + try { + repeat(20) { + Thread.sleep(200) + held[0].getOutputStream().apply { + write("X-Slow: 1\r\n".toByteArray()) + flush() + } + } + } catch (ignored: Exception) { + } + } + trickler.isDaemon = true + trickler.start() + // every held connection is closed by the server within the deadline plus slack + for (c in held) assertEquals(-1, c.getInputStream().read()) + val elapsedMs = (System.nanoTime() - started) / 1_000_000 + assertTrue("took $elapsedMs ms", elapsedMs < 3_000) + assertEquals(200, http(port, "POST", headers = auth, body = initBody()).status) + } finally { + held.forEach { it.close() } + } + } + + @Test fun anUnauthenticatedRequestIsRefusedBeforeItsBodyIsRead() { + val port = startServer() + Socket("127.0.0.1", port).use { c -> + c.soTimeout = 3000 + c.getOutputStream().apply { + write("POST /mcp HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Length: 1000\r\n\r\n".toByteArray()) + flush() + } + // the body never comes, yet the answer does + val line = c.getInputStream().bufferedReader().readLine() + assertTrue(line, line.startsWith("HTTP/1.1 401")) + } + } } From 7cf251bcc66d23ec8f93fcbd5d69f27de36f1c99 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 15 Nov 2026 22:48:28 -0800 Subject: [PATCH 19/24] ADFA-7135 Refuse the process-wide heap limit and temp directory PRAGMAs on server connections SQLite applies hard_heap_limit, soft_heap_limit and temp_store_directory to the whole process, so a client's PRAGMA hard_heap_limit=1 could make the app's own connections fail with SQLITE_NOMEM. Test: each form (set and read) returns SQLITE_AUTH on a server connection; without the change all five return SQLITE_OK. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAYT7GPoC2cDzo8LErn --- subprojects/sqlite-debugger/README.md | 2 +- subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c | 6 ++++-- subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h | 3 ++- subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c | 6 ++++++ 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/subprojects/sqlite-debugger/README.md b/subprojects/sqlite-debugger/README.md index 9e8aaaa95a..e2e9fcd7a4 100644 --- a/subprojects/sqlite-debugger/README.md +++ b/subprojects/sqlite-debugger/README.md @@ -31,7 +31,7 @@ The layout of `sqlitedbg_event_record` lives in `sqlitedbg.h`. The JNI layer han - **Debug builds only.** `DebuggerStartup.onCreate` throws if the process is not debuggable, so a release build that includes the library by mistake crashes at once instead of opening a local server. - **Loopback HTTP, a random port and token per start.** `McpHttp` binds 127.0.0.1 only (port 0, so the system picks one) and requires `Authorization: Bearer `, a new 32-byte random token each process start. It also checks that the Host and any Origin name 127.0.0.1 or localhost, against DNS rebinding. At most 8 connections at once. The head of a request (request line and headers) must arrive within 5 s, from connecting for a connection's first request and from its first byte for later ones, and Host, Origin, path and token are checked before the body is read, so a client that trickles bytes or never authenticates frees its slot; a refused request closes the connection. It opens no event stream: GET gets 405. -- **Read-only is a guard against accidents, not a security boundary.** Every tool opens its server connection with `writable = false`: `SQLITE_OPEN_READONLY`, `PRAGMA query_only` and an authorizer that refuses ordinary writes, transactions and savepoints, and `PRAGMA locking_mode` and `busy_timeout`, so a client cannot hold a lock on the app's database; the JNI query path also rolls back any transaction left open. Anyone holding the token can still run SQL, so the token is the boundary. +- **Read-only is a guard against accidents, not a security boundary.** Every tool opens its server connection with `writable = false`: `SQLITE_OPEN_READONLY`, `PRAGMA query_only` and an authorizer that refuses ordinary writes, transactions and savepoints, and `PRAGMA locking_mode` and `busy_timeout`, so a client cannot hold a lock on the app's database, and `hard_heap_limit`, `soft_heap_limit` and `temp_store_directory`, which SQLite applies process-wide, to the app's connections too; the JNI query path also rolls back any transaction left open. Anyone holding the token can still run SQL, so the token is the boundary. - **Only databases an app connection has open.** `DatabaseTools` refuses any other path before the engine is called: opening a file Android's own SQLite has open, with requery's, can corrupt it. The extension has no SQLite API until an app connection has opened, so `sqlitedbg_open_server` returns `SQLITE_MISUSE` before that. - **The app's connections are never altered.** The extension never sets the progress handler or the update hook of an app connection: requery sets and clears the progress handler around cancellable statements, and the update hook belongs to the app. Only server connections use a progress handler, for the time limit. - **Bound values are recorded only while a client is connected.** Expanding the SQL costs memory on every run, so `sqlitedbg_set_client_connected` turns it on; the first `get_events` call does, and it goes off again 60 s after the last one (checked on every tool call and by a timer). diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c index 929be274a2..468529a604 100644 --- a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c @@ -422,10 +422,12 @@ static int server_authorizer(void *unused, int action, const char *a1, const cha // PRAGMA query_only=0 would lift the guard, and a PRAGMA that stores a value writes. // PRAGMA table_info(t) also has a second argument, so only these names are checked. // locking_mode=EXCLUSIVE keeps the lock after a read, and busy_timeout (any form) lets a - // statement wait on the app's lock for as long as the client likes. + // statement wait on the app's lock for as long as the client likes. The heap limits and + // temp_store_directory are process-wide: they would change the app's own connections. if (a1 && (sqlite3_stricmp(a1, "query_only") == 0 || sqlite3_stricmp(a1, "writable_schema") == 0 || sqlite3_stricmp(a1, "locking_mode") == 0 || sqlite3_stricmp(a1, "busy_timeout") == 0 || - sqlite3_stricmp(a1, "wal_checkpoint") == 0 || + sqlite3_stricmp(a1, "hard_heap_limit") == 0 || sqlite3_stricmp(a1, "soft_heap_limit") == 0 || + sqlite3_stricmp(a1, "temp_store_directory") == 0 || sqlite3_stricmp(a1, "wal_checkpoint") == 0 || sqlite3_stricmp(a1, "incremental_vacuum") == 0 || sqlite3_stricmp(a1, "optimize") == 0)) return SQLITE_DENY; if (a1 && a2 && diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h index d87890daa0..d46e7ac585 100644 --- a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h @@ -136,7 +136,8 @@ void sqlitedbg_set_client_connected(int connected); // Opens path for the server with the same SQLite the app uses: SQLITE_OPEN_READONLY unless // writable, PRAGMA query_only when read-only, and an authorizer that refuses ordinary writes when // read-only, and also BEGIN, SAVEPOINT, PRAGMA locking_mode and PRAGMA busy_timeout: a client must not -// be able to hold a lock on the app's database. Read-only mode guards against accidents; it is not a security boundary (ADFA-7131, +// be able to hold a lock on the app's database. The process-wide PRAGMAs hard_heap_limit, soft_heap_limit and +// temp_store_directory are refused too: they would change the app's own connections. Read-only mode guards against accidents; it is not a security boundary (ADFA-7131, // D11). Never traced and never listed. The caller owns *db and closes it with sqlite3_close_v2. // A server connection, unlike an app connection, may use a progress handler (for time limits). // Returns SQLITE_MISUSE before any app connection has opened (the extension then has no SQLite diff --git a/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c b/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c index bcb1c8460d..63bf3f736e 100644 --- a/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c +++ b/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c @@ -619,6 +619,12 @@ static void test_server_holds_no_locks(void) { "PRAGMA locking_mode", "PRAGMA busy_timeout=60000", "PRAGMA busy_timeout", + // process-wide settings that would reach the app's own connections + "PRAGMA hard_heap_limit=1000000000", + "PRAGMA hard_heap_limit", + "PRAGMA soft_heap_limit=1000000000", + "PRAGMA soft_heap_limit", + "PRAGMA temp_store_directory='/tmp'", }; for (size_t i = 0; i < sizeof denied / sizeof denied[0]; i++) { sqlite3 *srv = NULL; From f785f969fd547e5dedacdd764ff4eafce0133562 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 15 Nov 2026 22:51:39 -0800 Subject: [PATCH 20/24] ADFA-7135 Give server connections a 3 s busy timeout With the app's database in rollback-journal mode, a server read failed at once with SQLITE_BUSY while the app held a write lock, exactly when the agent wants to look. The timeout is set in C and PRAGMA busy_timeout stays refused, so the client cannot change it. Test: an app connection holds BEGIN EXCLUSIVE; a server read waits about 500 ms and succeeds once it is released, and reports SQLITE_BUSY after about 3 s if it is not. Without the timeout the first read failed after 0 ms with SQLITE_BUSY. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAYT7GPoC2cDzo8LErn --- subprojects/sqlite-debugger/README.md | 2 +- .../sqlite-debugger/src/main/cpp/sqlitedbg.c | 3 + .../sqlite-debugger/src/main/cpp/sqlitedbg.h | 4 + .../src/test/c/sqlitedbg_test.c | 74 +++++++++++++++++++ 4 files changed, 82 insertions(+), 1 deletion(-) diff --git a/subprojects/sqlite-debugger/README.md b/subprojects/sqlite-debugger/README.md index e2e9fcd7a4..047a0cecf5 100644 --- a/subprojects/sqlite-debugger/README.md +++ b/subprojects/sqlite-debugger/README.md @@ -31,7 +31,7 @@ The layout of `sqlitedbg_event_record` lives in `sqlitedbg.h`. The JNI layer han - **Debug builds only.** `DebuggerStartup.onCreate` throws if the process is not debuggable, so a release build that includes the library by mistake crashes at once instead of opening a local server. - **Loopback HTTP, a random port and token per start.** `McpHttp` binds 127.0.0.1 only (port 0, so the system picks one) and requires `Authorization: Bearer `, a new 32-byte random token each process start. It also checks that the Host and any Origin name 127.0.0.1 or localhost, against DNS rebinding. At most 8 connections at once. The head of a request (request line and headers) must arrive within 5 s, from connecting for a connection's first request and from its first byte for later ones, and Host, Origin, path and token are checked before the body is read, so a client that trickles bytes or never authenticates frees its slot; a refused request closes the connection. It opens no event stream: GET gets 405. -- **Read-only is a guard against accidents, not a security boundary.** Every tool opens its server connection with `writable = false`: `SQLITE_OPEN_READONLY`, `PRAGMA query_only` and an authorizer that refuses ordinary writes, transactions and savepoints, and `PRAGMA locking_mode` and `busy_timeout`, so a client cannot hold a lock on the app's database, and `hard_heap_limit`, `soft_heap_limit` and `temp_store_directory`, which SQLite applies process-wide, to the app's connections too; the JNI query path also rolls back any transaction left open. Anyone holding the token can still run SQL, so the token is the boundary. +- **Read-only is a guard against accidents, not a security boundary.** Every tool opens its server connection with `writable = false`: `SQLITE_OPEN_READONLY`, `PRAGMA query_only` and an authorizer that refuses ordinary writes, transactions and savepoints, and `PRAGMA locking_mode` and `busy_timeout`, so a client cannot hold a lock on the app's database, and `hard_heap_limit`, `soft_heap_limit` and `temp_store_directory`, which SQLite applies process-wide, to the app's connections too; the JNI query path also rolls back any transaction left open. A server connection waits up to 3 s (`SQLITEDBG_SERVER_BUSY_MS`, fixed in C) for a lock the app holds, as in rollback-journal mode, and then reports `SQLITE_BUSY`. Anyone holding the token can still run SQL, so the token is the boundary. - **Only databases an app connection has open.** `DatabaseTools` refuses any other path before the engine is called: opening a file Android's own SQLite has open, with requery's, can corrupt it. The extension has no SQLite API until an app connection has opened, so `sqlitedbg_open_server` returns `SQLITE_MISUSE` before that. - **The app's connections are never altered.** The extension never sets the progress handler or the update hook of an app connection: requery sets and clears the progress handler around cancellable statements, and the update hook belongs to the app. Only server connections use a progress handler, for the time limit. - **Bound values are recorded only while a client is connected.** Expanding the SQL costs memory on every run, so `sqlitedbg_set_client_connected` turns it on; the first `get_events` call does, and it goes off again 60 s after the last one (checked on every tool call and by a timer). diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c index 468529a604..c69f5b9fdf 100644 --- a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c @@ -454,6 +454,9 @@ int sqlitedbg_open_server(const char *path, int writable, sqlite3 **db) { rc = sqlite3_exec(*db, "PRAGMA query_only=1", NULL, NULL, NULL); if (rc == SQLITE_OK) rc = sqlite3_set_authorizer(*db, server_authorizer, NULL); } + // In rollback-journal mode a reader fails at once while the app holds a write lock. Wait a little, + // well under the tools' 50 s deadline; the client cannot change it (PRAGMA busy_timeout is refused). + if (rc == SQLITE_OK) rc = sqlite3_busy_timeout(*db, SQLITEDBG_SERVER_BUSY_MS); if (rc != SQLITE_OK && *db) { sqlite3_close_v2(*db); *db = NULL; diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h index d46e7ac585..22eb2025dd 100644 --- a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h @@ -133,6 +133,8 @@ void sqlitedbg_set_client_connected(int connected); // Server connections +#define SQLITEDBG_SERVER_BUSY_MS 3000 + // Opens path for the server with the same SQLite the app uses: SQLITE_OPEN_READONLY unless // writable, PRAGMA query_only when read-only, and an authorizer that refuses ordinary writes when // read-only, and also BEGIN, SAVEPOINT, PRAGMA locking_mode and PRAGMA busy_timeout: a client must not @@ -140,6 +142,8 @@ void sqlitedbg_set_client_connected(int connected); // temp_store_directory are refused too: they would change the app's own connections. Read-only mode guards against accidents; it is not a security boundary (ADFA-7131, // D11). Never traced and never listed. The caller owns *db and closes it with sqlite3_close_v2. // A server connection, unlike an app connection, may use a progress handler (for time limits). +// The connection waits up to SQLITEDBG_SERVER_BUSY_MS for a lock the app holds (rollback-journal mode) +// and then reports SQLITE_BUSY. // Returns SQLITE_MISUSE before any app connection has opened (the extension then has no SQLite // API to call), or what sqlite3_open_v2 returns. int sqlitedbg_open_server(const char *path, int writable, sqlite3 **db); diff --git a/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c b/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c index 63bf3f736e..298fdd592e 100644 --- a/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c +++ b/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c @@ -653,6 +653,78 @@ static void test_server_holds_no_locks(void) { CHECK_EQ(sqlite3_close_v2(srv), SQLITE_OK); } +typedef struct locker { + char path[320]; + atomic_int locked, release; +} locker; + +// An app connection that holds an exclusive lock (rollback-journal mode) until told to release it. +static void *locker_main(void *arg) { + locker *l = arg; + sqlite3 *db = open_app(l->path); + exec_ok(db, "BEGIN EXCLUSIVE"); + atomic_store(&l->locked, 1); + while (!atomic_load(&l->release)) usleep(1000); + exec_ok(db, "COMMIT"); + sqlite3_close(db); + return NULL; +} + +static double since_ms(const struct timespec *t0) { + struct timespec t1; + clock_gettime(CLOCK_MONOTONIC, &t1); + return (t1.tv_sec - t0->tv_sec) * 1e3 + (t1.tv_nsec - t0->tv_nsec) / 1e6; +} + +static void *release_later(void *arg) { + usleep(500 * 1000); + atomic_store(&((locker *)arg)->release, 1); + return NULL; +} + +// A server read waits for the app's lock (rollback-journal mode) instead of failing at once. +static void test_server_waits_for_locks(void) { + locker l = {0}; + snprintf(l.path, sizeof l.path, "%s/journal.db", g_dir); + sqlite3 *app = open_app(l.path); + exec_ok(app, "PRAGMA journal_mode=DELETE"); + exec_ok(app, "CREATE TABLE j(a); INSERT INTO j VALUES(1)"); + sqlite3_close(app); + + // the lock is released after 500 ms: the read waits, then succeeds + pthread_t holder, releaser; + pthread_create(&holder, NULL, locker_main, &l); + while (!atomic_load(&l.locked)) usleep(1000); + sqlite3 *srv = NULL; + CHECK_EQ(sqlitedbg_open_server(l.path, 0, &srv), SQLITE_OK); + struct timespec t0; + clock_gettime(CLOCK_MONOTONIC, &t0); + pthread_create(&releaser, NULL, release_later, &l); + CHECK_EQ(step_all(srv, "SELECT * FROM j"), SQLITE_DONE); + double waited = since_ms(&t0); + CHECK(waited >= 300); + CHECK(waited < SQLITEDBG_SERVER_BUSY_MS); + pthread_join(releaser, NULL); + pthread_join(holder, NULL); + CHECK_EQ(sqlite3_close_v2(srv), SQLITE_OK); + + // a lock held past the timeout gives SQLITE_BUSY after about that long, not at once + atomic_store(&l.locked, 0); + atomic_store(&l.release, 0); + pthread_create(&holder, NULL, locker_main, &l); + while (!atomic_load(&l.locked)) usleep(1000); + CHECK_EQ(sqlitedbg_open_server(l.path, 0, &srv), SQLITE_OK); + clock_gettime(CLOCK_MONOTONIC, &t0); + CHECK_EQ(step_all(srv, "SELECT * FROM j"), SQLITE_BUSY); + waited = since_ms(&t0); + CHECK(waited >= SQLITEDBG_SERVER_BUSY_MS - 500); + CHECK(waited < SQLITEDBG_SERVER_BUSY_MS + 2000); + atomic_store(&l.release, 1); + pthread_join(holder, NULL); + CHECK_EQ(step_all(srv, "SELECT * FROM j"), SQLITE_DONE); + CHECK_EQ(sqlite3_close_v2(srv), SQLITE_OK); +} + typedef struct worker { int index; atomic_int *stop; @@ -927,6 +999,8 @@ int main(int argc, char **argv) { test_server(); printf("# server holds no locks\n"); test_server_holds_no_locks(); + printf("# server waits for locks\n"); + test_server_waits_for_locks(); printf("# threads\n"); test_threads(); if (argc == 4 && strcmp(argv[1], "--write-fixture") == 0) write_fixture(argv[2], argv[3]); From dcd342e3b29c3791e69ae551cf2c619740818d88 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 15 Nov 2026 22:55:39 -0800 Subject: [PATCH 21/24] ADFA-7135 Fail a query whose argument count differs from its parameter count Too few arguments left the missing parameters unbound, which SQLite reads as NULL, so the query ran and returned a plausible empty result. It now throws SQLITE_RANGE with a message naming both counts. Test (JniQueryTest, the real JNI query path on a host JVM, run by test.sh): zero args for one parameter and one for two give SQLITE_RANGE; the first expectation was NULL before, and both fail with the check disabled. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAYT7GPoC2cDzo8LErn --- .../sqlite-debugger/src/main/cpp/sqlitedbg_jni.c | 10 ++++++++++ .../sqlite-debugger/src/test/jni/JniQueryTest.java | 9 ++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c index 7033766bd6..703f103d57 100644 --- a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -333,6 +334,15 @@ static jobjectArray bridge_query(JNIEnv *env, jclass cls, jlong handle, jstring sqlite3_progress_handler(db, 1000, progress_cb, &dl); handler_set = 1; + // A missing argument would be bound as NULL and the query would run, returning a plausible empty result. + int wanted = st ? sqlite3_bind_parameter_count(st) : 0; + jsize given = (*env)->GetArrayLength(env, args); + if (given != wanted) { + char msg[96]; + snprintf(msg, sizeof msg, "the statement has %d parameter(s) but %d argument(s) were given", wanted, (int)given); + throw_engine(env, SQLITE_RANGE, msg); + goto done; + } rc = st ? bind_args(env, db, st, args, &message) : SQLITE_OK; if (rc != SQLITE_OK) { if (!(*env)->ExceptionCheck(env)) throw_engine(env, rc, message ? message : "cannot bind"); diff --git a/subprojects/sqlite-debugger/src/test/jni/JniQueryTest.java b/subprojects/sqlite-debugger/src/test/jni/JniQueryTest.java index 061d813ccb..776287b498 100644 --- a/subprojects/sqlite-debugger/src/test/jni/JniQueryTest.java +++ b/subprojects/sqlite-debugger/src/test/jni/JniQueryTest.java @@ -196,7 +196,14 @@ public static void main(String[] a) throws Exception { eq(row[2], "a😀b", "arg text"); check(Arrays.equals((byte[]) row[3], new byte[] {9, 8, 7}), "arg blob"); eq(row[4], null, "arg null"); - eq(q(srv, "SELECT ?").rows.get(0)[0], null, "an unbound parameter is NULL"); + eq(codeOf(srv, "SELECT ?", new Object[0], FAR), 25, "too few args: SQLITE_RANGE, not a NULL parameter"); + eq(codeOf(srv, "SELECT ?, ?", new Object[] {1L}, FAR), 25, "one arg of two: SQLITE_RANGE"); + eq(codeOf(srv, "SELECT 1", new Object[] {1L}, FAR), 25, "an arg for no parameter: SQLITE_RANGE"); + try { + query(srv, "SELECT ?, ?", new Object[] {1L}, 0, 100, 4096, 1024, MAX, FAR); + } catch (DebugEngineException e) { + check(e.getMessage().contains("2 parameter") && e.getMessage().contains("1 argument"), "the message names both counts: " + e.getMessage()); + } eq(codeOf(srv, "SELECT ?", new Object[] {1L, 2L}, FAR), 25, "too many args: SQLITE_RANGE"); eq(codeOf(srv, "SELECT ?", new Object[] {new Object()}, FAR), 21, "unsupported arg type: SQLITE_MISUSE"); eq(codeOf(srv, "SELEKT 1", new Object[0], FAR), 1, "syntax error: SQLITE_ERROR"); From d90b7b420f91bd64d05c2d50bacd0a170a54bca6 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 15 Nov 2026 23:06:17 -0800 Subject: [PATCH 22/24] ADFA-7136 List an open database once when the databases folder is reached through a link list_databases matched the files of the app's databases folder against connection paths as strings, so on a phone (/data/user/0/ against /data/data/) every open database was listed again as available: false. Both sides are now canonicalised before matching; paths are reported as the connections gave them. Test: a folder listed through a symlink, and the reverse, list the open database once. Without the canonicalisation it was listed twice. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAYT7GPoC2cDzo8LErn --- .../sqlitedebugger/mcp/DatabaseTools.kt | 6 +++- .../sqlitedebugger/mcp/DatabaseToolsTest.kt | 35 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt index 5bbed62ebf..19ceceaa98 100644 --- a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt @@ -255,17 +255,21 @@ class DatabaseTools( .put("required", JSONArray(listOf("database") + required)) } + private fun canonical(path: String): String = runCatching { File(path).canonicalPath }.getOrDefault(path) + private fun listDatabases(): JSONObject { val connections = engine.connections() val byPath = LinkedHashMap>() for (c in connections) if (c.path.isNotEmpty()) byPath.getOrPut(c.path) { mutableListOf() } += c.id + // The folder may be reached through a link (/data/user/0/ and /data/data/), so match resolved paths. + val openCanonical = byPath.keys.map { canonical(it) }.toSet() val list = JSONArray() for ((path, ids) in byPath) { list.put(JSONObject().put("path", path).put("available", true).put("connections", ids.size)) } databaseDir?.listFiles()?.sortedBy { it.name }?.forEach { f -> if (!f.isFile || SIDE_FILE_SUFFIXES.any { f.name.endsWith(it) }) return@forEach - if (f.absolutePath in byPath || f.path in byPath) return@forEach + if (canonical(f.absolutePath) in openCanonical) return@forEach list.put(JSONObject().put("path", f.absolutePath).put("available", false).put("connections", 0)) } return JSONObject().put("databases", list) diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt index 21cae2b0a0..b53bfed6d9 100644 --- a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt @@ -116,6 +116,41 @@ class DatabaseToolsTest { } } + @Test fun listDatabasesMatchesOpenFilesThroughASymlinkedFolder() { + val root = + kotlin.io.path + .createTempDirectory("dbs") + try { + val real = + java.nio.file.Files + .createDirectory(root.resolve("real")) + .toFile() + val link = root.resolve("link") + java.nio.file.Files + .createSymbolicLink(link, real.toPath()) + listOf("open.db", "closed.db").forEach { File(real, it).writeText("x") } + // the folder is listed through the link, the connection reports the real path + dir = link.toFile() + engine.appConnections += conn(1, File(real, "open.db").absolutePath) + var dbs = ok("list_databases").getJSONArray("databases") + assertEquals(dbs.toString(), 2, dbs.length()) + assertEquals(File(real, "open.db").absolutePath, dbs.getJSONObject(0).getString("path")) + assertTrue(dbs.getJSONObject(0).getBoolean("available")) + assertFalse(dbs.getJSONObject(1).getBoolean("available")) + assertTrue(dbs.getJSONObject(1).getString("path").endsWith("closed.db")) + + // and the other way round: the connection reports the link, the folder is the real one + engine.appConnections.clear() + dir = real + engine.appConnections += conn(2, File(link.toFile(), "open.db").absolutePath) + dbs = ok("list_databases").getJSONArray("databases") + assertEquals(dbs.toString(), 2, dbs.length()) + assertEquals(File(link.toFile(), "open.db").absolutePath, dbs.getJSONObject(0).getString("path")) + } finally { + root.toFile().deleteRecursively() + } + } + @Test fun listConnectionsEncodesBigIntegersAsStrings() { engine.appConnections += conn(1, "/data/a.db", closing = true) val c = ok("list_connections").getJSONArray("connections").getJSONObject(0) From 3527173a3e5d1a483e160ee2696c8f833006056c Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 15 Nov 2026 23:06:17 -0800 Subject: [PATCH 23/24] ADFA-7135 Run the sqlite-debugger host tests in CI test.sh (the extension's C tests under ASan/UBSan and TSan, and the JNI query test) ran in no workflow. The new workflow follows test-sqlite-js.yml and also triggers on the vendored SQLite headers test.sh compares against. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAYT7GPoC2cDzo8LErn --- .github/workflows/test-sqlite-debugger.yml | 27 ++++++++++++++++++++++ subprojects/sqlite-debugger/README.md | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-sqlite-debugger.yml diff --git a/.github/workflows/test-sqlite-debugger.yml b/.github/workflows/test-sqlite-debugger.yml new file mode 100644 index 0000000000..b119984e49 --- /dev/null +++ b/.github/workflows/test-sqlite-debugger.yml @@ -0,0 +1,27 @@ +name: Test sqlite-debugger + +# The tests the Android build cannot run (ADFA-7135): the extension's C tests under AddressSanitizer, +# UBSan and ThreadSanitizer, and the JNI query test on a host JVM. See subprojects/sqlite-debugger/test.sh, +# which downloads the SQLite amalgamation itself and checks its hash. + +on: + pull_request: + paths: + - 'subprojects/sqlite-debugger/**' + - 'subprojects/sqlite-js/upstream/libs/**' + - '.github/workflows/test-sqlite-debugger.yml' + +permissions: + contents: read + +jobs: + + test-sqlite-debugger: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Run the sqlite-debugger tests + run: subprojects/sqlite-debugger/test.sh diff --git a/subprojects/sqlite-debugger/README.md b/subprojects/sqlite-debugger/README.md index 047a0cecf5..55b60427e7 100644 --- a/subprojects/sqlite-debugger/README.md +++ b/subprojects/sqlite-debugger/README.md @@ -41,7 +41,7 @@ The layout of `sqlitedbg_event_record` lives in `sqlitedbg.h`. The JNI layer han - **Host C tests of the extension:** `subprojects/sqlite-debugger/test.sh [--update-fixture] [AMALGAMATION_TARBALL]`. It builds SQLite 3.49.1 from the amalgamation as requery builds it (`SQLITE_THREADSAFE=2`), checks its headers against `sqlite-js/upstream/libs`, and runs `src/test/c/sqlitedbg_test.c` under AddressSanitizer with UBSan, then under ThreadSanitizer. It needs gcc, curl, tar and sha256sum, and downloads the tarball unless you pass one. - **The fixture:** `src/test/resources/.../engine/events.bin` and `events.tsv` are written by the C test and read by `EventDecoderTest`. Without `--update-fixture`, `test.sh` checks the committed pair against the current record layout, so a layout change fails until you regenerate it and update `EventDecoder.kt`. -- **JNI test:** when a JDK is on PATH, `test.sh` also builds `libsqlitedbg.so` as it ships (no `SQLITE_CORE`) against a requery-like `libsqlite3x.so` and runs `src/test/jni/JniQueryTest.java` through `NativeBridge`: paging, the total-bytes budget, per-cell truncation, bound types, the deadline, refused writes and transactions. It uses Java stubs of `NativeBridge` and `DebugEngineException` instead of the Kotlin, so a JDK is all it needs; `test.sh` checks the native names match `NativeBridge.kt`. Otherwise it is skipped with a message. +- **JNI test:** when a JDK is on PATH, `test.sh` also builds `libsqlitedbg.so` as it ships (no `SQLITE_CORE`) against a requery-like `libsqlite3x.so` and runs `src/test/jni/JniQueryTest.java` through `NativeBridge`: paging, the total-bytes budget, per-cell truncation, bound types, the deadline, refused writes and transactions. It uses Java stubs of `NativeBridge` and `DebugEngineException` instead of the Kotlin, so a JDK is all it needs; `test.sh` checks the native names match `NativeBridge.kt`. Otherwise it is skipped with a message. CI runs `test.sh` in `.github/workflows/test-sqlite-debugger.yml` for changes under this module. - **JVM tests:** `flox activate -d flox/local -- ./gradlew :subprojects:sqlite-debugger:testDebugUnitTest`. They cover `EventDecoder`, `McpProtocol`, `McpHttp` and `DatabaseTools` (with `FakeDebugEngine`). No workflow in `.github/workflows` runs either yet. From 5b54c337903b8ff18f3be2b16942b5b36ace6ded Mon Sep 17 00:00:00 2001 From: David Schachter Date: Mon, 16 Nov 2026 10:58:37 -0800 Subject: [PATCH 24/24] ADFA-7135: Correct two README lines the CI and device work made false "No workflow runs either yet" has been false since the test-sqlite-debugger workflow: it runs test.sh, the C and JNI tests. The JVM tests are still in no workflow, which the line now says. "Nothing has run in an app on a device" has been false since the device runs recorded on ADFA-7135. What is left is a device run of the current code, because those results predate the review fixes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAYT7GPoC2cDzo8LErn --- subprojects/sqlite-debugger/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subprojects/sqlite-debugger/README.md b/subprojects/sqlite-debugger/README.md index 55b60427e7..9b32e7c718 100644 --- a/subprojects/sqlite-debugger/README.md +++ b/subprojects/sqlite-debugger/README.md @@ -44,11 +44,11 @@ The layout of `sqlitedbg_event_record` lives in `sqlitedbg.h`. The JNI layer han - **JNI test:** when a JDK is on PATH, `test.sh` also builds `libsqlitedbg.so` as it ships (no `SQLITE_CORE`) against a requery-like `libsqlite3x.so` and runs `src/test/jni/JniQueryTest.java` through `NativeBridge`: paging, the total-bytes budget, per-cell truncation, bound types, the deadline, refused writes and transactions. It uses Java stubs of `NativeBridge` and `DebugEngineException` instead of the Kotlin, so a JDK is all it needs; `test.sh` checks the native names match `NativeBridge.kt`. Otherwise it is skipped with a message. CI runs `test.sh` in `.github/workflows/test-sqlite-debugger.yml` for changes under this module. - **JVM tests:** `flox activate -d flox/local -- ./gradlew :subprojects:sqlite-debugger:testDebugUnitTest`. They cover `EventDecoder`, `McpProtocol`, `McpHttp` and `DatabaseTools` (with `FakeDebugEngine`). -No workflow in `.github/workflows` runs either yet. +`.github/workflows/test-sqlite-debugger.yml` runs `test.sh` (the C and JNI tests) on pull requests that touch the module or the SQLite headers. No workflow runs the JVM tests. ## Not done yet - Broadcasting the port and token to Code on the Go (ADFA-7137). `DebuggerStartup` only keeps them in `DebuggerStartup.port` and `endpointToken` and logs the port, never the token. - Gradle plugin wiring and the build-time rewrite that adds the library to debug builds (ADFA-7138). - The AI-Agent-MCP side that connects to the server (ADFA-7139). -- Verification on a device. The C tests run on the host and the Kotlin tests on the JVM; nothing has run in an app on a device or emulator. +- A device run of the current code. The device results on ADFA-7135 (statements captured on both ABIs, the overhead) predate the review fixes.