diff --git a/.github/workflows/test-sqlite-debugger.yml b/.github/workflows/test-sqlite-debugger.yml new file mode 100644 index 0000000000..b119984e49 --- /dev/null +++ b/.github/workflows/test-sqlite-debugger.yml @@ -0,0 +1,27 @@ +name: Test sqlite-debugger + +# The tests the Android build cannot run (ADFA-7135): the extension's C tests under AddressSanitizer, +# UBSan and ThreadSanitizer, and the JNI query test on a host JVM. See subprojects/sqlite-debugger/test.sh, +# which downloads the SQLite amalgamation itself and checks its hash. + +on: + pull_request: + paths: + - 'subprojects/sqlite-debugger/**' + - 'subprojects/sqlite-js/upstream/libs/**' + - '.github/workflows/test-sqlite-debugger.yml' + +permissions: + contents: read + +jobs: + + test-sqlite-debugger: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Run the sqlite-debugger tests + run: subprojects/sqlite-debugger/test.sh diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 0a7b2ef3eb..4158fa6bb4 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -95,7 +95,7 @@ Strategy: **layer-and-subsystem based**, not feature-by-feature. The Gradle buil - **`app` depends inward; libraries never depend on `app`.** Subsystems are consumed by `app`, not vice versa. - **Vendored forks are substituted, not imported ad hoc.** `composite-builds/build-deps` and `build-deps-common` provide forked `javac`/`jdt`/`layoutlib`/etc.; `settings.gradle.kts` substitutes them in for `com.itsaky.androidide.build:*`. Don't add a Maven coordinate for something already substituted. Pebble is likewise built from source, as `subprojects:pebble`, for a lexer fix Android needs (its `NOTICE` says what changed); depend on `projects.subprojects.pebble`, not `io.pebbletemplates:pebble`. -- **All module config flows through `composite-builds/build-logic`.** Every Android module gets the `v7`/`v8` ABI flavors centrally (`AndroidModuleConf.kt`) — there is no flavorless `assembleDebug`. `:plugin-api` and `:subprojects:sqlite-js`, libraries for other projects that need one artifact, are intentionally excluded from flavors. +- **All module config flows through `composite-builds/build-logic`.** Every Android module gets the `v7`/`v8` ABI flavors centrally (`AndroidModuleConf.kt`) — there is no flavorless `assembleDebug`. `:plugin-api`, `:subprojects:sqlite-js` and `:subprojects:sqlite-debugger`, libraries for other projects that need one artifact, are intentionally excluded from flavors. - **`RepositoriesMode.FAIL_ON_PROJECT_REPOS`** — repositories are declared once in `settings.gradle.kts`; modules must not declare their own. - **Avoid new dependencies.** Check `gradle/libs.versions.toml` first; the build almost certainly already has it. @@ -104,10 +104,11 @@ Strategy: **layer-and-subsystem based**, not feature-by-feature. The Gradle buil These structural facts shape every module. Day-to-day build *commands* live in `CLAUDE.md`; the rules that produce them live here. - **Centralized convention logic.** All Android module setup flows through `composite-builds/build-logic` (`conf/AndroidModuleConf.kt`). Modules stay thin and share configuration, so understanding any module's setup starts here. -- **ABI product flavors.** Every Android module *except* `:plugin-api` and `:subprojects:sqlite-js` gets two flavors on the `abi` dimension — `v7` (`armeabi-v7a`) and `v8` (`arm64-v8a`) — defined centrally. There is no flavorless variant; tasks are `assembleV8Debug`, `assembleV7Release`, etc. +- **ABI product flavors.** Every Android module *except* `:plugin-api`, `:subprojects:sqlite-js` and `:subprojects:sqlite-debugger` gets two flavors on the `abi` dimension — `v7` (`armeabi-v7a`) and `v8` (`arm64-v8a`) — defined centrally. There is no flavorless variant; tasks are `assembleV8Debug`, `assembleV7Release`, etc. - **SDK levels** (`build-logic/.../build/config/BuildConfig.kt`): `COMPILE_SDK=36`, `MIN_SDK=28`, `TARGET_SDK=28`. **`TARGET_SDK` is deliberately pinned at 28:** higher targets enforce W^X (write-xor-execute), which blocks executing code from app-writable files. That is fatal for an on-device IDE that compiles and runs code (Gradle, `javac`, Termux binaries), so it is a hard requirement, not tech debt. `MIN_SDK_FOR_APPS_BUILT_WITH_COGO=16` is the floor for the apps a *user* builds with CoGo — distinct from CoGo's own `MIN_SDK`. - **Native asset bundling.** The on-device LLM (`llama-impl`) ships as a per-flavor native AAR, wired through the root `build.gradle.kts` (`bundleLlamaV8Assets` / `assembleV8Assets`, …); prebuilt per-flavor assets live under `assets/release/v7/` and `assets/release/v8/`. - **sqlite-js for apps built with CoGo** (ADFA-7082). `subprojects:sqlite-js` builds the AAR that the JavaScript-in-SQLite project template depends on: the sqlite-js release vendored (see its README), for arm64-v8a and armeabi-v7a, minSdk 26, without core-library desugaring. Nothing in CoGo depends on it. Its default entry point runs no code stored in a database; loading stored functions is a separate entry point, only for databases the app made, since loading runs their code. `test-sqlite-js.yml` runs its host tests on pull requests. The manual `release-sqlite-js.yml` publishes it to the moving `sqlite-js-latest` release, from which dev-assets fetches it into the template, as it fetches `plugin-api-latest` for the plugin template. +- **SQLite debugger for apps built with CoGo** (ADFA-7131). `subprojects:sqlite-debugger` builds an AAR with a debug-only SQLite debugger and MCP server for apps built with CoGo: a native extension on requery's SQLite plus a loopback HTTP server. Like sqlite-js it has no ABI flavors, and nothing in CoGo depends on it; see its README. - **Native lib compression** (ADFA-2306, ADFA-4729). The app manifest hard-codes `android:extractNativeLibs="true"` (required: the installer must materialize libs in `nativeLibraryDir`, e.g. `libshizuku.so` is an executable the adb shell runs from there). That attribute overrides the `jniLibs.useLegacyPackaging` DSL, so AGP packages `lib//*.so` deflate-compressed in **every** APK — ~5.9 MB smaller (`libtree-sitter-kotlin.so` alone is 4.18 MB → 339 kB). The trap is the `recompressApk` post-step (release always, debug in CI only): its no-compress lists in `app/build.gradle.kts` must NOT contain `"so"`, or it silently re-stores the libs and undoes the saving — which is what ADFA-2306 fixed for release and ADFA-4729 for CI debug. Locally built debug APKs (including the e2e farm's) never run that step and were always fine. - **Quick Build shares the Gradle distribution's toolchain** (ADFA-4931). `:app`'s `quickBuildDaemonZip` leaves the Kotlin compiler and eight other jars out of the daemon zip, because the on-device Gradle distribution already carries them; `QuickBuildArtifactStager` links them in at provision time. That makes the daemon's Kotlin version the distribution's, so `:quickbuild:daemon` takes its Kotlin artifacts at `org.adfa.constants.GRADLE_DISTRIBUTION_KOTLIN_VERSION`, and `:app:quickBuildDistJarCheck` fails the build when the distribution it packages lacks a linked jar. See [ADR 0016](docs/adr/0016-quick-build-compiles-outside-gradle.md). - **`app` package layout is by concern, not feature:** `activities`, `fragments`, `services`, `di`, `agent`, `viewmodel(s)`, `repositories`, `roomData`, `localWebServer`, `preferences`, `ui` (Compose screens live under `ui/compose`), `templates/manager` (the Manager screen's `.cgt`-parsing data layer, with direct filesystem access to `Environment.TEMPLATES_DIR` — distinct from the plugin-facing `IdeTemplateService` in `plugin-api`/`plugin-manager`), `utils`, …. diff --git a/CLAUDE.md b/CLAUDE.md index eaae5b3de6..7334588a8c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,7 +15,7 @@ flox activate -d flox/local -- ./gradlew ``` - **Debug APK (arm64, the usual target):** `flox activate -d flox/local -- ./gradlew :app:assembleV8Debug --parallel --max-workers=6` -- **Module unit tests:** `flox activate -d flox/local -- ./gradlew :editor:testV7DebugUnitTest` - unit tests are per-flavor, there is no plain `test` task. `:plugin-api` and `:subprojects:sqlite-js` are the modules without flavors: `:plugin-api:testDebugUnitTest`. +- **Module unit tests:** `flox activate -d flox/local -- ./gradlew :editor:testV7DebugUnitTest` - unit tests are per-flavor, there is no plain `test` task. `:plugin-api`, `:subprojects:sqlite-js` and `:subprojects:sqlite-debugger` are the modules without flavors: `:plugin-api:testDebugUnitTest`. - **Single unit test:** append `--tests "com.itsaky.androidide.SomeTest"` to the flavored task above. - **Fast iteration:** during multi-file/multi-module changes, verify with targeted `:module:compileV8DebugKotlin`/`compileV8DebugJavaWithJavac` invocations (batch several modules into one Gradle call) rather than a full assemble. Reserve `:app:assembleV8Debug` for final end-to-end verification — it's slow (multi-minute) in this multi-module project, and running it after every small change adds up. - **Long-running commands: narrate them, and background only the read-only ones.** Anything that can exceed ~60s — `assembleV8Debug`, a full test sweep, a cold Gradle invocation of any kind (daemon start plus configuring this many modules is ~20s before any task runs, and far worse under memory pressure), and `git push`, which runs Spotless through the hook — gets a line before it starts saying what is running and roughly how long it takes, and a status line every couple of minutes while it runs: elapsed time, the last output line, whether it is still progressing. A silent terminal is indistinguishable from a hang, and saying which it is is your job, not the user's to ask. @@ -26,7 +26,7 @@ When the user names a CI/CD job ("the sonar job", "the analyze workflow"), read ### ABI product flavors (v7 / v8) -Every module except `:plugin-api` and `:subprojects:sqlite-js` carries `v7` (`armeabi-v7a`) and `v8` (`arm64-v8a`) flavors, so build tasks are `assembleV8Debug`, `assembleV7Release`, etc. — there is **no** flavorless `assembleDebug`. Those two build one artifact for other projects. See [ARCHITECTURE.md](ARCHITECTURE.md) (Build & Module Configuration) for flavors, native asset bundling, and SDK levels. +Every module except `:plugin-api`, `:subprojects:sqlite-js` and `:subprojects:sqlite-debugger` carries `v7` (`armeabi-v7a`) and `v8` (`arm64-v8a`) flavors, so build tasks are `assembleV8Debug`, `assembleV7Release`, etc. — there is **no** flavorless `assembleDebug`. Those three build one artifact for other projects. See [ARCHITECTURE.md](ARCHITECTURE.md) (Build & Module Configuration) for flavors, native asset bundling, and SDK levels. ### Emulator / device diff --git a/composite-builds/build-logic/plugins/src/main/java/com/itsaky/androidide/plugins/conf/AndroidModuleConf.kt b/composite-builds/build-logic/plugins/src/main/java/com/itsaky/androidide/plugins/conf/AndroidModuleConf.kt index 6dd8daa757..e418cb787a 100644 --- a/composite-builds/build-logic/plugins/src/main/java/com/itsaky/androidide/plugins/conf/AndroidModuleConf.kt +++ b/composite-builds/build-logic/plugins/src/main/java/com/itsaky/androidide/plugins/conf/AndroidModuleConf.kt @@ -53,13 +53,17 @@ private val disableCoreLibDesugaringForModules = ":quickbuild:runtime", // The AAR is a dependency of apps built with CoGo (ADFA-7082). ":subprojects:sqlite-js", + // Injected into debug builds of apps built with CoGo like :logsender (ADFA-7131). + ":subprojects:sqlite-debugger", ) /** * Modules without the v7/v8 flavors: libraries for other projects, which need one artifact. - * :subprojects:sqlite-js is an AAR for apps built with CoGo, with all the ABIs they run on. + * :subprojects:sqlite-js and :subprojects:sqlite-debugger are AARs for apps built with CoGo, with + * all the ABIs they run on. */ -private val modulesWithoutAbiFlavors = setOf(":plugin-api", ":subprojects:sqlite-js") +private val modulesWithoutAbiFlavors = + setOf(":plugin-api", ":subprojects:sqlite-js", ":subprojects:sqlite-debugger") /** * The name of the build type used for automated testing. diff --git a/settings.gradle.kts b/settings.gradle.kts index 5538bb5e0a..a3b77f05a5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -170,6 +170,7 @@ include( ":subprojects:shizuku-server-shared", ":subprojects:shizuku-shared", ":subprojects:shizuku-starter", + ":subprojects:sqlite-debugger", ":subprojects:sqlite-js", ":subprojects:tooling-api", ":subprojects:tooling-api-events", diff --git a/subprojects/sqlite-debugger/README.md b/subprojects/sqlite-debugger/README.md new file mode 100644 index 0000000000..9b32e7c718 --- /dev/null +++ b/subprojects/sqlite-debugger/README.md @@ -0,0 +1,54 @@ +# SQLite debugger for apps + +A debug-only SQLite debugger and MCP server that runs inside apps built with Code on the Go (ADFA-7131). It lets an AI agent list an app's databases, read their schema and rows, see the SQL the app runs, and check integrity, all read-only. Phase 1 is the native extension (ADFA-7135) and the server (ADFA-7136). Nothing in CoGo itself depends on it: apps do, in debug builds. + +The module builds an AAR with `libsqlitedbg.so` for arm64-v8a and armeabi-v7a in one artifact, so it has no v7/v8 flavors (`AndroidModuleConf`). Its minSdk is 21, requery's, because the debugger runs on requery's SQLite. The unit test task is `:subprojects:sqlite-debugger:testDebugUnitTest`. + +## How it fits together + +``` +MCP client --HTTP, 127.0.0.1--> McpHttp -> McpProtocol -> DatabaseTools -> DebugEngine + | + NativeDebugEngine -> NativeBridge (JNI) + | + sqlitedbg_jni.c -> sqlitedbg.c (sqlitedbg.h) + | + requery's libsqlite3x.so (the app's SQLite) +``` + +- `src/main/cpp/sqlitedbg.h` is the contract between the extension, its JNI layer and its host tests, with the semantics of each call. `SQLITEDBG_INTERFACE_VERSION` is raised when anything in it changes shape. +- `sqlitedbg.c` is a SQLite extension. `sqlitedbg_install` registers it with `sqlite3_auto_extension`, so it runs on every connection the app opens afterwards. It keeps the list of app connections and a ring buffer of statement events (trace: statement, profile, close), and opens connections for the server (`sqlitedbg_open_server`). +- `sqlitedbg_jni.c` finds `sqlite3_auto_extension` in requery's `libsqlite3x.so` with `dlopen(RTLD_NOLOAD)` and `dlsym`, so no SQLite is compiled or linked here (CMake takes only the headers from `../sqlite-js/upstream/libs`). Its version script exports only `JNI_OnLoad`; the functions of `NativeBridge` are registered there by name, and `consumer-rules.pro` keeps them. +- `engine/`: `DebugEngine` is what the server needs from the extension, `NativeDebugEngine` implements it over JNI, and `EventDecoder` turns the event buffer's bytes into `Event`s. +- `mcp/`: `McpHttp` is the HTTP/1.1 listener, `McpProtocol` is JSON-RPC and the MCP session handling, `DatabaseTools` is the tools: `list_databases`, `list_connections`, `get_schema`, `query`, `explain_query_plan`, `get_stats`, `integrity_check`, `get_events`. +- `DebuggerStartup` is a `ContentProvider` with the highest `initOrder`, declared in the AAR's manifest. It runs before the app's own providers and `Application.onCreate`, so the extension is registered before the app opens a database. It installs the extension synchronously in `onCreate` (so an app that opens its database in `Application.onCreate` is still seen) and starts the server on a background thread. Everything after the debuggable check is wrapped in `catch (Throwable)`: a missing requery library (`UnsatisfiedLinkError`) means the debugger is disabled and logged, never a crash of the host app. + +### The event record + +The layout of `sqlitedbg_event_record` lives in `sqlitedbg.h`. The JNI layer hands the bytes to Kotlin unchanged, so `EventDecoder.kt` mirrors the layout, field offsets included: change both together. A committed fixture pins them (see Testing). + +## Decisions that shape it + +- **Debug builds only.** `DebuggerStartup.onCreate` throws if the process is not debuggable, so a release build that includes the library by mistake crashes at once instead of opening a local server. +- **Loopback HTTP, a random port and token per start.** `McpHttp` binds 127.0.0.1 only (port 0, so the system picks one) and requires `Authorization: Bearer `, a new 32-byte random token each process start. It also checks that the Host and any Origin name 127.0.0.1 or localhost, against DNS rebinding. At most 8 connections at once. The head of a request (request line and headers) must arrive within 5 s, from connecting for a connection's first request and from its first byte for later ones, and Host, Origin, path and token are checked before the body is read, so a client that trickles bytes or never authenticates frees its slot; a refused request closes the connection. It opens no event stream: GET gets 405. +- **Read-only is a guard against accidents, not a security boundary.** Every tool opens its server connection with `writable = false`: `SQLITE_OPEN_READONLY`, `PRAGMA query_only` and an authorizer that refuses ordinary writes, transactions and savepoints, and `PRAGMA locking_mode` and `busy_timeout`, so a client cannot hold a lock on the app's database, and `hard_heap_limit`, `soft_heap_limit` and `temp_store_directory`, which SQLite applies process-wide, to the app's connections too; the JNI query path also rolls back any transaction left open. A server connection waits up to 3 s (`SQLITEDBG_SERVER_BUSY_MS`, fixed in C) for a lock the app holds, as in rollback-journal mode, and then reports `SQLITE_BUSY`. Anyone holding the token can still run SQL, so the token is the boundary. +- **Only databases an app connection has open.** `DatabaseTools` refuses any other path before the engine is called: opening a file Android's own SQLite has open, with requery's, can corrupt it. The extension has no SQLite API until an app connection has opened, so `sqlitedbg_open_server` returns `SQLITE_MISUSE` before that. +- **The app's connections are never altered.** The extension never sets the progress handler or the update hook of an app connection: requery sets and clears the progress handler around cancellable statements, and the update hook belongs to the app. Only server connections use a progress handler, for the time limit. +- **Bound values are recorded only while a client is connected.** Expanding the SQL costs memory on every run, so `sqlitedbg_set_client_connected` turns it on; the first `get_events` call does, and it goes off again 60 s after the last one (checked on every tool call and by a timer). +- **Limits.** Connections opened before `install` are not seen, so `DebuggerStartup` runs as early as the process allows. requery's own `sqlite3_trace` replaces ours on a connection when `log.tag.SQLiteStatements` or `log.tag.SQLiteTime` is VERBOSE, so events from such connections stop. Events are dropped, oldest first, when the buffer (4 MiB by default) is full, and `get_events` reports how many. A query result holds at most 4 MiB of cell bytes (`QueryLimits.maxTotalBytes`: TEXT and BLOB bytes kept, 8 per number), so a wide SELECT cannot exhaust the app's heap; the rest is paged (`has_more`). The engine reports which cells were cut (`QueryResult.truncatedCellIndexes`), so a BLOB of exactly the limit is not flagged. + +## Testing + +- **Host C tests of the extension:** `subprojects/sqlite-debugger/test.sh [--update-fixture] [AMALGAMATION_TARBALL]`. It builds SQLite 3.49.1 from the amalgamation as requery builds it (`SQLITE_THREADSAFE=2`), checks its headers against `sqlite-js/upstream/libs`, and runs `src/test/c/sqlitedbg_test.c` under AddressSanitizer with UBSan, then under ThreadSanitizer. It needs gcc, curl, tar and sha256sum, and downloads the tarball unless you pass one. +- **The fixture:** `src/test/resources/.../engine/events.bin` and `events.tsv` are written by the C test and read by `EventDecoderTest`. Without `--update-fixture`, `test.sh` checks the committed pair against the current record layout, so a layout change fails until you regenerate it and update `EventDecoder.kt`. +- **JNI test:** when a JDK is on PATH, `test.sh` also builds `libsqlitedbg.so` as it ships (no `SQLITE_CORE`) against a requery-like `libsqlite3x.so` and runs `src/test/jni/JniQueryTest.java` through `NativeBridge`: paging, the total-bytes budget, per-cell truncation, bound types, the deadline, refused writes and transactions. It uses Java stubs of `NativeBridge` and `DebugEngineException` instead of the Kotlin, so a JDK is all it needs; `test.sh` checks the native names match `NativeBridge.kt`. Otherwise it is skipped with a message. CI runs `test.sh` in `.github/workflows/test-sqlite-debugger.yml` for changes under this module. +- **JVM tests:** `flox activate -d flox/local -- ./gradlew :subprojects:sqlite-debugger:testDebugUnitTest`. They cover `EventDecoder`, `McpProtocol`, `McpHttp` and `DatabaseTools` (with `FakeDebugEngine`). + +`.github/workflows/test-sqlite-debugger.yml` runs `test.sh` (the C and JNI tests) on pull requests that touch the module or the SQLite headers. No workflow runs the JVM tests. + +## Not done yet + +- Broadcasting the port and token to Code on the Go (ADFA-7137). `DebuggerStartup` only keeps them in `DebuggerStartup.port` and `endpointToken` and logs the port, never the token. +- Gradle plugin wiring and the build-time rewrite that adds the library to debug builds (ADFA-7138). +- The AI-Agent-MCP side that connects to the server (ADFA-7139). +- A device run of the current code. The device results on ADFA-7135 (statements captured on both ABIs, the overhead) predate the review fixes. diff --git a/subprojects/sqlite-debugger/build.gradle.kts b/subprojects/sqlite-debugger/build.gradle.kts new file mode 100644 index 0000000000..2acf1dd65c --- /dev/null +++ b/subprojects/sqlite-debugger/build.gradle.kts @@ -0,0 +1,68 @@ +import com.itsaky.androidide.build.config.BuildConfig + +plugins { + id("com.android.library") + id("kotlin-android") +} + +description = "SQLite debugger for debug builds of apps built with Code on the Go (ADFA-7131)" + +android { + namespace = "org.appdevforall.sqlitedebugger" + ndkVersion = BuildConfig.NDK_VERSION + + defaultConfig { + // requery's minSdk, not MIN_SDK_FOR_APPS_BUILT_WITH_COGO: the debugger runs on requery's SQLite + minSdk = 21 + // One AAR for both ABIs of the phones apps built with CoGo run on, unlike CoGo's own + // modules: this module has no v7/v8 flavors (AndroidModuleConf). + ndk { + abiFilters.clear() + abiFilters += listOf("arm64-v8a", "armeabi-v7a") + } + externalNativeBuild { + cmake { + arguments += "-DANDROID_STL=none" + } + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + + buildFeatures { + viewBinding = false + buildConfig = false + } + + externalNativeBuild { + cmake { + path = file("src/main/cpp/CMakeLists.txt") + version = "3.31.0+" + } + } +} + +// Set on the tasks, not the extension: the root build sets every KotlinCompile task's jvmTarget +// to 17, which overrides the extension and no longer matches compileOptions above. +tasks.withType().configureEach { + compilerOptions.jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_1_8) +} + +kotlin { + compilerOptions { + // Apps compile against this library with Code on the Go's on-device Kotlin compiler, + // which can be older than the one that builds it (as in :plugin-api). + apiVersion.set(org.jetbrains.kotlin.gradle.dsl.KotlinVersion.KOTLIN_2_0) + languageVersion.set(org.jetbrains.kotlin.gradle.dsl.KotlinVersion.KOTLIN_2_0) + } +} + +dependencies { + // Android provides org.json on a device + testImplementation(libs.org.json) + testImplementation(libs.tests.junit) + testImplementation(libs.tests.google.truth) +} diff --git a/subprojects/sqlite-debugger/consumer-rules.pro b/subprojects/sqlite-debugger/consumer-rules.pro new file mode 100644 index 0000000000..4ba638bc89 --- /dev/null +++ b/subprojects/sqlite-debugger/consumer-rules.pro @@ -0,0 +1,3 @@ +# JNI_OnLoad registers these by name, and the native code builds this exception +-keep class org.appdevforall.sqlitedebugger.engine.NativeBridge { *; } +-keep class org.appdevforall.sqlitedebugger.engine.DebugEngineException { (int, java.lang.String); } diff --git a/subprojects/sqlite-debugger/src/main/AndroidManifest.xml b/subprojects/sqlite-debugger/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..18de3c738f --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/AndroidManifest.xml @@ -0,0 +1,15 @@ + + + + + + + + + + diff --git a/subprojects/sqlite-debugger/src/main/cpp/CMakeLists.txt b/subprojects/sqlite-debugger/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000000..779767a2bb --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/cpp/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 3.22.1) + +project(sqlitedbg C) + +# Only the headers: the extension runs inside requery's libsqlite3x.so and reaches SQLite through +# the api table it is given (sqlite3ext.h), so no SQLite is compiled or linked here. +set(SQLITE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../../../../sqlite-js/upstream/libs) + +add_library(sqlitedbg SHARED + sqlitedbg.c + sqlitedbg_jni.c) + +target_include_directories(sqlitedbg PRIVATE ${SQLITE_HEADERS} ${CMAKE_CURRENT_SOURCE_DIR}) +target_compile_options(sqlitedbg PRIVATE -Wall -Wextra -fvisibility=hidden) +# dlopen/dlsym to find sqlite3_auto_extension in requery's libsqlite3x.so +target_link_libraries(sqlitedbg dl) +# Only JNI_OnLoad is exported, so sqlite3_api and our other symbols cannot bind to another library's. +target_link_options(sqlitedbg PRIVATE + -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/sqlitedbg.map + -Wl,-z,max-page-size=16384) diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c new file mode 100644 index 0000000000..c69f5b9fdf --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.c @@ -0,0 +1,465 @@ +// The SQLite debugger's extension (ADFA-7135). The contract is sqlitedbg.h. +// +// Built as a loadable-extension client (sqlite3ext.h): on a device every sqlite3_ call goes +// through the api table SQLite hands to the first sqlitedbg_extension_init, because libsqlitedbg.so +// links no SQLite of its own. Host tests define SQLITE_CORE, where the macros call SQLite directly. + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include +#include + +#include "sqlite3ext.h" +#include "sqlitedbg.h" + +SQLITE_EXTENSION_INIT1 + +#define DEFAULT_BUFFER_BYTES ((size_t)4 << 20) +#define DEFAULT_MAX_SQL_BYTES 4096u +#define RECORD_HEADER_BYTES 56u + +_Static_assert(sizeof(sqlitedbg_event_record) == RECORD_HEADER_BYTES, "event record layout"); + +// Padding record written where the next record would not fit before the end of the ring, so a +// record is always contiguous. kind 0 is never a real event. +#define KIND_PAD 0u + +typedef struct conn_state { + struct conn_state *prev, *next; + uint64_t id; + char *path; + int read_only; + int closing; + int64_t opened_at_ns; + int64_t open_thread_id; +} conn_state; + +// Taken only for the fields below; never while SQLite runs. +static pthread_mutex_t g_lock = PTHREAD_MUTEX_INITIALIZER; +// Serialises sqlitedbg_install so a second caller does not return before the first registered. +static pthread_mutex_t g_install_lock = PTHREAD_MUTEX_INITIALIZER; + +static int g_installed; // g_install_lock +static int g_api_ready; // g_lock: sqlite3_api has been saved +static uint32_t g_max_sql; // set before g_installed, then read-only +static int g_client_connected; // atomic + +static conn_state *g_first, *g_last; +static uint64_t g_next_conn_id = 1; + +static unsigned char *g_ring; +static size_t g_cap; +static size_t g_head; // where the next record goes +static size_t g_tail; // the oldest record; never a padding record while g_used > 0 +static size_t g_used; // bytes between tail and head, padding included +static uint64_t g_next_seq = 1; + +// Set around the server's own sqlite3_open_v2: auto extensions run on the opening thread, inside it. +static __thread int tls_opening_server; + +static int64_t now_ns(void) { + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + return (int64_t)ts.tv_sec * 1000000000LL + ts.tv_nsec; +} + +static int64_t this_thread_id(void) { +#ifdef __ANDROID__ + return (int64_t)gettid(); +#else + return (int64_t)syscall(SYS_gettid); +#endif +} + +static sqlitedbg_event_record *record_at(size_t offset) { + return (sqlitedbg_event_record *)(void *)(g_ring + offset); +} + +// Drops the oldest record, then any padding that is now the oldest. g_lock held. +static void evict_oldest(void) { + do { + uint32_t bytes = record_at(g_tail)->record_bytes; + g_used -= bytes; + g_tail += bytes; + if (g_tail >= g_cap) g_tail = 0; + } while (g_used > 0 && record_at(g_tail)->kind == KIND_PAD); +} + +// The seq of the oldest event still buffered, or the next one to be made if none is. g_lock held. +static uint64_t oldest_seq(void) { + return g_used > 0 ? record_at(g_tail)->seq : g_next_seq; +} + +// Makes room for one record of `need` bytes at g_head. g_lock held; need <= g_cap. +static void make_room(size_t need) { + for (;;) { + if (g_used == 0) { + g_head = g_tail = 0; + return; + } + if (g_head > g_tail) { + if (g_cap - g_head >= need) return; + if (g_tail >= need) { + sqlitedbg_event_record *pad = record_at(g_head); + pad->record_bytes = (uint32_t)(g_cap - g_head); + pad->kind = KIND_PAD; + g_used += g_cap - g_head; + g_head = 0; + return; + } + } else if (g_tail - g_head >= need) { + return; + } + evict_oldest(); + } +} + +// Longest prefix of sql, at most max bytes, that ends on a character boundary. +static size_t utf8_cut(const char *sql, size_t len, size_t max) { + if (len <= max) return len; + while (max > 0 && ((unsigned char)sql[max] & 0xC0) == 0x80) max--; + return max; +} + +static void add_event(uint32_t kind, uint64_t connection_id, int64_t duration_ns, uint32_t flags, + const char *sql, size_t sql_len) { + int64_t time = now_ns(); + int64_t tid = this_thread_id(); + size_t kept = utf8_cut(sql, sql_len, g_max_sql); + if (kept < sql_len) flags |= SQLITEDBG_EVENT_TRUNCATED; + size_t need = (RECORD_HEADER_BYTES + kept + 7) & ~(size_t)7; + + pthread_mutex_lock(&g_lock); + uint64_t seq = g_next_seq++; + if (need <= g_cap) { + make_room(need); + sqlitedbg_event_record *r = record_at(g_head); + r->record_bytes = (uint32_t)need; + r->kind = kind; + r->seq = seq; + r->connection_id = connection_id; + r->thread_id = tid; + r->time_ns = time; + r->duration_ns = duration_ns; + r->flags = flags; + r->sql_bytes = (uint32_t)kept; + memcpy((unsigned char *)r + RECORD_HEADER_BYTES, sql, kept); + memset((unsigned char *)r + RECORD_HEADER_BYTES + kept, 0, need - RECORD_HEADER_BYTES - kept); + g_head += need; + g_used += need; + if (g_head >= g_cap) g_head = 0; + } + pthread_mutex_unlock(&g_lock); +} + +// sqlite3_trace_v2 callback. C is the connection's conn_state, which lives until SQLite frees the +// connection, so it outlives every callback. +static int trace_cb(unsigned type, void *c, void *p, void *x) { + conn_state *cs = (conn_state *)c; + switch (type) { + case SQLITE_TRACE_STMT: + case SQLITE_TRACE_PROFILE: { + sqlite3_stmt *stmt = (sqlite3_stmt *)p; + const char *sql = NULL; + char *expanded = NULL; + uint32_t flags = 0; + int is_stmt = type == SQLITE_TRACE_STMT; + if (is_stmt) sql = (const char *)x; + else sql = sqlite3_sql(stmt); + if (!sql) sql = ""; + // For a trigger, X is its own "-- ..." text while P is the outer statement, whose expansion + // would be the wrong text. For the statement itself X is its SQL. Not a prefix test: app SQL + // may start with a comment. + int is_trigger = 0; + if (is_stmt) { + const char *own = sqlite3_sql(stmt); + is_trigger = !own || strcmp(own, sql) != 0; + } + if (__atomic_load_n(&g_client_connected, __ATOMIC_RELAXED) && !is_trigger) { + expanded = sqlite3_expanded_sql(stmt); + if (expanded) { + sql = expanded; + flags |= SQLITEDBG_EVENT_EXPANDED; + } + } + int64_t duration = is_stmt ? -1 : *(sqlite3_int64 *)x; + add_event(is_stmt ? SQLITEDBG_EVENT_STATEMENT : SQLITEDBG_EVENT_PROFILE, cs->id, duration, flags, sql, + strlen(sql)); + if (expanded) sqlite3_free(expanded); + break; + } + case SQLITE_TRACE_CLOSE: + // The close can still fail or leave a zombie, so only mark it: xDestroy frees. + pthread_mutex_lock(&g_lock); + cs->closing = 1; + pthread_mutex_unlock(&g_lock); + add_event(SQLITEDBG_EVENT_CLOSE, cs->id, -1, 0, "", 0); + break; + } + return 0; +} + +// SQLite calls this when it really frees the connection (or if registering the function failed). +static void conn_destroy(void *p) { + conn_state *cs = (conn_state *)p; + pthread_mutex_lock(&g_lock); + if (cs->prev) cs->prev->next = cs->next; + else g_first = cs->next; + if (cs->next) cs->next->prev = cs->prev; + else g_last = cs->prev; + pthread_mutex_unlock(&g_lock); + free(cs->path); + free(cs); +} + +static void connection_id_fn(sqlite3_context *ctx, int argc, sqlite3_value **argv) { + (void)argc; + (void)argv; + sqlite3_result_int64(ctx, (sqlite3_int64)((conn_state *)sqlite3_user_data(ctx))->id); +} + +#ifdef SQLITEDBG_TEST +// Host tests: 1 fails the allocation, 2 makes registering the function fail. +int sqlitedbg_test_fail_init; +#endif + +static int init_connection(sqlite3 *db, const sqlite3_api_routines *api); + +// An internal failure must never fail the app's own sqlite3_open: log it and leave the connection +// untracked. +int sqlitedbg_extension_init(sqlite3 *db, char **err, const sqlite3_api_routines *api) { + (void)err; + if (tls_opening_server) return SQLITE_OK; + int rc = init_connection(db, api); + if (rc != SQLITE_OK) sqlite3_log(rc, "sqlitedbg: connection not tracked (%d)", rc); + return SQLITE_OK; +} + +static int init_connection(sqlite3 *db, const sqlite3_api_routines *api) { + + pthread_mutex_lock(&g_lock); + if (!g_api_ready) { + SQLITE_EXTENSION_INIT2(api); + g_api_ready = 1; + } + pthread_mutex_unlock(&g_lock); + + conn_state *cs = (conn_state *)calloc(1, sizeof *cs); +#ifdef SQLITEDBG_TEST + if (sqlitedbg_test_fail_init == 1) { + free(cs); + cs = NULL; + } +#endif + if (!cs) return SQLITE_NOMEM; + const char *filename = sqlite3_db_filename(db, "main"); + cs->path = strdup(filename ? filename : ""); + if (!cs->path) { + free(cs); + return SQLITE_NOMEM; + } + cs->read_only = sqlite3_db_readonly(db, "main") == 1; + cs->opened_at_ns = now_ns(); + cs->open_thread_id = this_thread_id(); + + pthread_mutex_lock(&g_lock); + cs->id = g_next_conn_id++; + cs->prev = g_last; + if (g_last) g_last->next = cs; + else g_first = cs; + g_last = cs; + pthread_mutex_unlock(&g_lock); + + // On failure SQLite has already called conn_destroy, so cs is gone. + const char *fn_name = "sqlitedbg_connection_id"; +#ifdef SQLITEDBG_TEST + if (sqlitedbg_test_fail_init == 2) fn_name = NULL; // SQLite refuses a NULL name +#endif + int rc = sqlite3_create_function_v2(db, fn_name, 0, SQLITE_UTF8 | SQLITE_INNOCUOUS, cs, + connection_id_fn, NULL, NULL, conn_destroy); + if (rc != SQLITE_OK) return rc; + return sqlite3_trace_v2(db, SQLITE_TRACE_STMT | SQLITE_TRACE_PROFILE | SQLITE_TRACE_CLOSE, trace_cb, cs); +} + +int sqlitedbg_install(sqlitedbg_auto_extension_fn auto_extension, const sqlitedbg_config *config) { + if (!auto_extension) return SQLITE_MISUSE; + pthread_mutex_lock(&g_install_lock); + if (g_installed) { + pthread_mutex_unlock(&g_install_lock); + return SQLITE_OK; + } + size_t cap = config && config->buffer_bytes ? config->buffer_bytes : DEFAULT_BUFFER_BYTES; + uint32_t max_sql = config && config->max_sql_bytes ? config->max_sql_bytes : DEFAULT_MAX_SQL_BYTES; + // Room for two of the largest records, or the ring could never hold one beside a pad. + size_t min_cap = 2 * ((RECORD_HEADER_BYTES + (size_t)max_sql + 7) & ~(size_t)7); + if (cap < min_cap) cap = min_cap; + cap &= ~(size_t)7; + unsigned char *ring = (unsigned char *)calloc(1, cap); + if (!ring) { + pthread_mutex_unlock(&g_install_lock); + return SQLITE_NOMEM; + } + pthread_mutex_lock(&g_lock); + g_ring = ring; + g_cap = cap; + g_head = g_tail = g_used = 0; + g_max_sql = max_sql; + pthread_mutex_unlock(&g_lock); + + int rc = auto_extension((void (*)(void))sqlitedbg_extension_init); + if (rc == SQLITE_OK) { + g_installed = 1; + } else { + pthread_mutex_lock(&g_lock); + g_ring = NULL; + g_cap = 0; + pthread_mutex_unlock(&g_lock); + free(ring); + } + pthread_mutex_unlock(&g_install_lock); + return rc; +} + +size_t sqlitedbg_each_connection(void (*fn)(void *context, const sqlitedbg_connection *connection), + void *context) { + size_t n = 0; + pthread_mutex_lock(&g_lock); + for (conn_state *cs = g_first; cs; cs = cs->next) { + sqlitedbg_connection c; + c.id = cs->id; + c.path = cs->path; + c.read_only = cs->read_only; + c.closing = cs->closing; + c.opened_at_ns = cs->opened_at_ns; + c.open_thread_id = cs->open_thread_id; + fn(context, &c); + n++; + } + pthread_mutex_unlock(&g_lock); + return n; +} + +void sqlitedbg_read_events(uint64_t after, void *out, size_t out_bytes, sqlitedbg_read_result *result) { + result->next = after; + result->dropped = 0; + result->bytes = 0; + result->count = 0; + pthread_mutex_lock(&g_lock); + if (g_ring) { + uint64_t oldest = oldest_seq(); + if (oldest > after && oldest - after > 1) { + result->dropped = oldest - after - 1; + // Past the dropped events, so they are reported once even if nothing is copied below. + result->next = oldest - 1; + } + size_t offset = g_tail, left = g_used; + while (left > 0) { + const sqlitedbg_event_record *r = record_at(offset); + uint32_t bytes = r->record_bytes; + if (r->kind != KIND_PAD && r->seq > after) { + if (result->bytes + bytes > out_bytes) break; + memcpy((unsigned char *)out + result->bytes, r, bytes); + result->bytes += bytes; + result->count++; + result->next = r->seq; + } + left -= bytes; + offset += bytes; + if (offset >= g_cap) offset = 0; + } + } + pthread_mutex_unlock(&g_lock); +} + +void sqlitedbg_set_client_connected(int connected) { + __atomic_store_n(&g_client_connected, connected ? 1 : 0, __ATOMIC_RELAXED); +} + +static int server_authorizer(void *unused, int action, const char *a1, const char *a2, const char *db, + const char *trigger) { + (void)unused; + (void)a2; + (void)db; + (void)trigger; + switch (action) { + case SQLITE_INSERT: + case SQLITE_UPDATE: + case SQLITE_DELETE: + case SQLITE_CREATE_INDEX: + case SQLITE_CREATE_TABLE: + case SQLITE_CREATE_TEMP_INDEX: + case SQLITE_CREATE_TEMP_TABLE: + case SQLITE_CREATE_TEMP_TRIGGER: + case SQLITE_CREATE_TEMP_VIEW: + case SQLITE_CREATE_TRIGGER: + case SQLITE_CREATE_VIEW: + case SQLITE_CREATE_VTABLE: + case SQLITE_DROP_INDEX: + case SQLITE_DROP_TABLE: + case SQLITE_DROP_TEMP_INDEX: + case SQLITE_DROP_TEMP_TABLE: + case SQLITE_DROP_TEMP_TRIGGER: + case SQLITE_DROP_TEMP_VIEW: + case SQLITE_DROP_TRIGGER: + case SQLITE_DROP_VIEW: + case SQLITE_DROP_VTABLE: + case SQLITE_ALTER_TABLE: + case SQLITE_REINDEX: + case SQLITE_ANALYZE: + case SQLITE_ATTACH: + case SQLITE_DETACH: + // A client could open a transaction or savepoint and hold a lock on the app's database. + case SQLITE_TRANSACTION: + case SQLITE_SAVEPOINT: + return SQLITE_DENY; + case SQLITE_PRAGMA: + // PRAGMA query_only=0 would lift the guard, and a PRAGMA that stores a value writes. + // PRAGMA table_info(t) also has a second argument, so only these names are checked. + // locking_mode=EXCLUSIVE keeps the lock after a read, and busy_timeout (any form) lets a + // statement wait on the app's lock for as long as the client likes. The heap limits and + // temp_store_directory are process-wide: they would change the app's own connections. + if (a1 && (sqlite3_stricmp(a1, "query_only") == 0 || sqlite3_stricmp(a1, "writable_schema") == 0 || + sqlite3_stricmp(a1, "locking_mode") == 0 || sqlite3_stricmp(a1, "busy_timeout") == 0 || + sqlite3_stricmp(a1, "hard_heap_limit") == 0 || sqlite3_stricmp(a1, "soft_heap_limit") == 0 || + sqlite3_stricmp(a1, "temp_store_directory") == 0 || sqlite3_stricmp(a1, "wal_checkpoint") == 0 || + sqlite3_stricmp(a1, "incremental_vacuum") == 0 || sqlite3_stricmp(a1, "optimize") == 0)) + return SQLITE_DENY; + if (a1 && a2 && + (sqlite3_stricmp(a1, "journal_mode") == 0 || sqlite3_stricmp(a1, "user_version") == 0 || + sqlite3_stricmp(a1, "application_id") == 0 || sqlite3_stricmp(a1, "schema_version") == 0)) + return SQLITE_DENY; + return SQLITE_OK; + default: + return SQLITE_OK; + } +} + +int sqlitedbg_open_server(const char *path, int writable, sqlite3 **db) { + *db = NULL; + pthread_mutex_lock(&g_lock); + int ready = g_api_ready; + pthread_mutex_unlock(&g_lock); + if (!ready) return SQLITE_MISUSE; + + tls_opening_server = 1; + int rc = sqlite3_open_v2(path, db, writable ? SQLITE_OPEN_READWRITE : SQLITE_OPEN_READONLY, NULL); + tls_opening_server = 0; + if (rc == SQLITE_OK && !writable) { + rc = sqlite3_exec(*db, "PRAGMA query_only=1", NULL, NULL, NULL); + if (rc == SQLITE_OK) rc = sqlite3_set_authorizer(*db, server_authorizer, NULL); + } + // In rollback-journal mode a reader fails at once while the app holds a write lock. Wait a little, + // well under the tools' 50 s deadline; the client cannot change it (PRAGMA busy_timeout is refused). + if (rc == SQLITE_OK) rc = sqlite3_busy_timeout(*db, SQLITEDBG_SERVER_BUSY_MS); + if (rc != SQLITE_OK && *db) { + sqlite3_close_v2(*db); + *db = NULL; + } + return rc; +} diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h new file mode 100644 index 0000000000..22eb2025dd --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.h @@ -0,0 +1,155 @@ +// The SQLite debugger's extension (ADFA-7135): the interface its JNI layer and its host tests +// build against. Debug builds of apps built with Code on the Go only (ADFA-7131, D1). +// +// The extension runs on every connection the app's SQLite (requery's libsqlite3x.so) opens after +// sqlitedbg_install(). It keeps a list of the app's open connections and a buffer of statement +// events, and opens connections for the server (ADFA-7136). It never sets the progress handler or +// the update hook of an app connection: requery sets and clears the progress handler around +// cancellable statements, and the update hook belongs to the app. +// +// Threads: every function here may be called from any thread. requery builds SQLite with +// SQLITE_THREADSAFE=2, so a connection is used by one thread at a time; the extension's own state +// is guarded by its own lock, which is never held while SQLite runs a statement. + +#ifndef SQLITEDBG_H +#define SQLITEDBG_H + +#include +#include +#include "sqlite3.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// The interface version. Raise it when anything below changes shape. +#define SQLITEDBG_INTERFACE_VERSION 1 + +// sqlite3_auto_extension's type. On a device the JNI layer looks it up in libsqlite3x.so with +// dlsym (requery exports every sqlite3_ function); host tests pass SQLite's own. +typedef int (*sqlitedbg_auto_extension_fn)(void (*entry_point)(void)); + +typedef struct sqlitedbg_config { + // Size of the event buffer. Events are dropped, oldest first, when it is full. 0: 4 MiB. + size_t buffer_bytes; + // Longest SQL text kept in one event; longer text is cut at a UTF-8 character boundary and + // the event gets SQLITEDBG_EVENT_TRUNCATED. 0: 4 KiB. + uint32_t max_sql_bytes; +} sqlitedbg_config; + +// Registers the extension for every connection opened from now on, and sets up the buffer. +// Connections already open are not seen (a known limit, ADFA-7131). A second call does nothing and +// returns SQLITE_OK; config is read on the first call only. config may be NULL for the defaults. +// Returns SQLITE_OK, SQLITE_NOMEM, or what auto_extension returns. +int sqlitedbg_install(sqlitedbg_auto_extension_fn auto_extension, const sqlitedbg_config *config); + +// The SQLite entry point the extension registers. Exposed for host tests only: the app never +// calls it, and it is never loaded with load_extension(). +int sqlitedbg_extension_init(sqlite3 *db, char **err, const sqlite3_api_routines *api); + +// Connections + +typedef struct sqlitedbg_connection { + // Unique within the process, starting at 1, never reused. + uint64_t id; + // sqlite3_db_filename(db, "main") when the connection opened: "" for an in-memory or temporary + // database. Valid only during the sqlitedbg_each_connection callback. + const char *path; + // 1 if the main database is read-only (sqlite3_db_readonly). + int read_only; + // 1 once the trace "close" event has fired. The close may still fail, and the app may retry + // it, so the connection stays listed until SQLite frees it. + int closing; + // CLOCK_REALTIME, nanoseconds, when the extension first saw the connection. + int64_t opened_at_ns; + // gettid() of the thread that opened it. + int64_t open_thread_id; +} sqlitedbg_connection; + +// Calls fn once for each app connection SQLite has not yet freed, oldest first. Server connections +// (sqlitedbg_open_server) are not listed. fn runs with the extension's lock held: it must be quick +// and must not call into SQLite or this interface. Returns how many connections it reported. +size_t sqlitedbg_each_connection(void (*fn)(void *context, const sqlitedbg_connection *connection), + void *context); + +// Events + +enum { + // A statement starts running (SQLITE_TRACE_STMT); also each trigger it fires, whose sql starts + // with "--". + SQLITEDBG_EVENT_STATEMENT = 1, + // A statement finished (SQLITE_TRACE_PROFILE); duration_ns is set. + SQLITEDBG_EVENT_PROFILE = 2, + // The app asked SQLite to close the connection (SQLITE_TRACE_CLOSE); sql is empty. + SQLITEDBG_EVENT_CLOSE = 3, +}; + +enum { + // sql was cut at max_sql_bytes. + SQLITEDBG_EVENT_TRUNCATED = 1u << 0, + // sql has its bound values written in (sqlite3_expanded_sql). Only while a client is + // connected (sqlitedbg_set_client_connected): expanding a 1 MB value costs 2 MB on every run. + SQLITEDBG_EVENT_EXPANDED = 1u << 1, +}; + +// One event as sqlitedbg_read_events copies it out: this header, then sql_bytes of UTF-8 (no +// terminating NUL), then zero padding to a multiple of 8 bytes. Fields are in the CPU's byte order +// (little-endian on every ABI we build). The JNI layer hands the bytes to Kotlin unchanged, so +// EventDecoder.kt mirrors this layout; change both together. +typedef struct sqlitedbg_event_record { + uint32_t record_bytes; // the whole record, padding included + uint32_t kind; // SQLITEDBG_EVENT_STATEMENT, _PROFILE or _CLOSE + uint64_t seq; // 1, 2, 3...: every event made, dropped or not, gets the next one + uint64_t connection_id; // sqlitedbg_connection.id + int64_t thread_id; // gettid() of the thread running the statement + int64_t time_ns; // CLOCK_REALTIME when the event happened + int64_t duration_ns; // SQLITEDBG_EVENT_PROFILE only; -1 for the others + uint32_t flags; // SQLITEDBG_EVENT_TRUNCATED, SQLITEDBG_EVENT_EXPANDED + uint32_t sql_bytes; +} sqlitedbg_event_record; // 56 bytes; the SQL text follows + +typedef struct sqlitedbg_read_result { + // The seq of the last event copied out, or if none was, `after` moved past any dropped events + // (so they are reported once): pass it as `after` next time. A record too big for out is not + // passed. + uint64_t next; + // Events after `after` that left the buffer before they could be read. + uint64_t dropped; + // Bytes of records written to out. + size_t bytes; + // Records written to out. + size_t count; +} sqlitedbg_read_result; + +// Copies out the events with seq > after, oldest first, as whole records, until out is full. +// The copy is made under the extension's lock with no SQLite call, so it never waits on a +// statement. A record larger than out_bytes is never split: an out smaller than one record of +// max_sql_bytes may return count 0 with events still waiting. Never fails. +void sqlitedbg_read_events(uint64_t after, void *out, size_t out_bytes, sqlitedbg_read_result *result); + +// Whether a client is connected. While it is, statements are recorded with their bound values +// written in. Off at install. +void sqlitedbg_set_client_connected(int connected); + +// Server connections + +#define SQLITEDBG_SERVER_BUSY_MS 3000 + +// Opens path for the server with the same SQLite the app uses: SQLITE_OPEN_READONLY unless +// writable, PRAGMA query_only when read-only, and an authorizer that refuses ordinary writes when +// read-only, and also BEGIN, SAVEPOINT, PRAGMA locking_mode and PRAGMA busy_timeout: a client must not +// be able to hold a lock on the app's database. The process-wide PRAGMAs hard_heap_limit, soft_heap_limit and +// temp_store_directory are refused too: they would change the app's own connections. Read-only mode guards against accidents; it is not a security boundary (ADFA-7131, +// D11). Never traced and never listed. The caller owns *db and closes it with sqlite3_close_v2. +// A server connection, unlike an app connection, may use a progress handler (for time limits). +// The connection waits up to SQLITEDBG_SERVER_BUSY_MS for a lock the app holds (rollback-journal mode) +// and then reports SQLITE_BUSY. +// Returns SQLITE_MISUSE before any app connection has opened (the extension then has no SQLite +// API to call), or what sqlite3_open_v2 returns. +int sqlitedbg_open_server(const char *path, int writable, sqlite3 **db); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.map b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.map new file mode 100644 index 0000000000..b6518f1c1d --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg.map @@ -0,0 +1,6 @@ +{ + global: + JNI_OnLoad; + local: + *; +}; diff --git a/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c new file mode 100644 index 0000000000..703f103d57 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/cpp/sqlitedbg_jni.c @@ -0,0 +1,480 @@ +// JNI layer of the SQLite debugger (ADFA-7135): NativeBridge.kt over sqlitedbg.h. Like the +// extension it calls SQLite only through the api table (sqlite3ext.h), which sqlitedbg.c saves. +// Every handle here comes from sqlitedbg_open_server, which refuses to open before that. + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include +#include + +#include "sqlite3ext.h" +#include "sqlitedbg.h" + +SQLITE_EXTENSION_INIT3 + +#define BRIDGE_CLASS "org/appdevforall/sqlitedebugger/engine/NativeBridge" +#define EXCEPTION_CLASS "org/appdevforall/sqlitedebugger/engine/DebugEngineException" + +static jclass g_object, g_string, g_array_list, g_long, g_double, g_exception; +static jmethodID g_string_from_bytes, g_list_init, g_list_add, g_long_value_of, g_double_value_of, g_exception_init; +static jstring g_utf8_name; + +static jclass global_class(JNIEnv *env, const char *name) { + jclass local = (*env)->FindClass(env, name); + if (!local) return NULL; + jclass global = (jclass)(*env)->NewGlobalRef(env, local); + (*env)->DeleteLocalRef(env, local); + return global; +} + +// A Java String from UTF-8 bytes. NewStringUTF takes modified UTF-8, which differs for NUL and for +// characters outside the BMP, and SQLite text can hold both. +static jstring utf8_string(JNIEnv *env, const char *bytes, int n) { + jbyteArray array = (*env)->NewByteArray(env, n); + if (!array) return NULL; + (*env)->SetByteArrayRegion(env, array, 0, n, (const jbyte *)bytes); + jstring s = (jstring)(*env)->NewObject(env, g_string, g_string_from_bytes, array, g_utf8_name); + (*env)->DeleteLocalRef(env, array); + return s; +} + +static void throw_engine(JNIEnv *env, int code, const char *message) { + if ((*env)->ExceptionCheck(env)) return; + jstring text = utf8_string(env, message, (int)strlen(message)); + if (!text) return; + jobject ex = (*env)->NewObject(env, g_exception, g_exception_init, (jint)code, text); + if (ex) (*env)->Throw(env, (jthrowable)ex); +} + +static int64_t now_ms(void) { + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + return (int64_t)ts.tv_sec * 1000 + ts.tv_nsec / 1000000; +} + +// install + +static jint bridge_install(JNIEnv *env, jclass cls) { + (void)env; + (void)cls; + // requery's library is already loaded by System.loadLibrary; NOLOAD only finds it, and its + // sqlite3_ functions are all exported, so no separate handle to SQLite is needed + void *lib = dlopen("libsqlite3x.so", RTLD_NOW | RTLD_NOLOAD); + if (!lib) return -1; + sqlitedbg_auto_extension_fn auto_extension = (sqlitedbg_auto_extension_fn)dlsym(lib, "sqlite3_auto_extension"); + int rc = auto_extension ? sqlitedbg_install(auto_extension, NULL) : -1; + dlclose(lib); + return rc; +} + +// connections + +typedef struct conn_list { + jlong *meta; + char **paths; + size_t n, cap; + int oom; +} conn_list; + +static void collect_connection(void *context, const sqlitedbg_connection *c) { + conn_list *l = (conn_list *)context; + if (l->oom) return; + if (l->n == l->cap) { + size_t cap = l->cap ? l->cap * 2 : 16; + jlong *meta = realloc(l->meta, cap * 5 * sizeof(jlong)); + if (meta) l->meta = meta; + char **paths = realloc(l->paths, cap * sizeof(char *)); + if (paths) l->paths = paths; + if (!meta || !paths) { + l->oom = 1; + return; + } + l->cap = cap; + } + char *path = strdup(c->path); + if (!path) { + l->oom = 1; + return; + } + jlong *m = l->meta + l->n * 5; + m[0] = (jlong)c->id; + m[1] = c->read_only; + m[2] = c->closing; + m[3] = (jlong)c->opened_at_ns; + m[4] = (jlong)c->open_thread_id; + l->paths[l->n++] = path; +} + +static jobjectArray bridge_connections(JNIEnv *env, jclass cls) { + (void)cls; + conn_list l = {0}; + sqlitedbg_each_connection(collect_connection, &l); + jobjectArray result = NULL; + if (l.oom) { + throw_engine(env, SQLITE_NOMEM, "out of memory"); + goto done; + } + jlongArray meta = (*env)->NewLongArray(env, (jsize)(l.n * 5)); + jobjectArray paths = meta ? (*env)->NewObjectArray(env, (jsize)l.n, g_string, NULL) : NULL; + if (!paths) goto done; + (*env)->SetLongArrayRegion(env, meta, 0, (jsize)(l.n * 5), l.meta ? l.meta : (jlong[]){0}); + for (size_t i = 0; i < l.n; i++) { + jstring s = utf8_string(env, l.paths[i], (int)strlen(l.paths[i])); + if (!s) goto done; + (*env)->SetObjectArrayElement(env, paths, (jsize)i, s); + (*env)->DeleteLocalRef(env, s); + } + result = (*env)->NewObjectArray(env, 2, g_object, NULL); + if (result) { + (*env)->SetObjectArrayElement(env, result, 0, meta); + (*env)->SetObjectArrayElement(env, result, 1, paths); + } +done: + for (size_t i = 0; i < l.n; i++) free(l.paths[i]); + free(l.paths); + free(l.meta); + return result; +} + +// events + +static void bridge_read_events(JNIEnv *env, jclass cls, jlong after, jbyteArray buf, jlongArray out) { + (void)cls; + jsize cap = (*env)->GetArrayLength(env, buf); + void *tmp = malloc((size_t)cap); + if (!tmp) { + throw_engine(env, SQLITE_NOMEM, "out of memory"); + return; + } + sqlitedbg_read_result r; + sqlitedbg_read_events((uint64_t)after, tmp, (size_t)cap, &r); + (*env)->SetByteArrayRegion(env, buf, 0, (jsize)r.bytes, (const jbyte *)tmp); + free(tmp); + jlong values[3] = {(jlong)r.next, (jlong)r.dropped, (jlong)r.bytes}; + (*env)->SetLongArrayRegion(env, out, 0, 3, values); +} + +static void bridge_set_client_connected(JNIEnv *env, jclass cls, jboolean connected) { + (void)env; + (void)cls; + sqlitedbg_set_client_connected(connected ? 1 : 0); +} + +// server connections + +static jlong bridge_open_server(JNIEnv *env, jclass cls, jbyteArray path, jboolean writable) { + (void)cls; + jsize n = (*env)->GetArrayLength(env, path); + char *c_path = malloc((size_t)n + 1); + if (!c_path) { + throw_engine(env, SQLITE_NOMEM, "out of memory"); + return 0; + } + (*env)->GetByteArrayRegion(env, path, 0, n, (jbyte *)c_path); + c_path[n] = 0; + sqlite3 *db = NULL; + int rc = sqlitedbg_open_server(c_path, writable ? 1 : 0, &db); + free(c_path); + if (rc != SQLITE_OK) { + throw_engine(env, rc, rc == SQLITE_MISUSE ? "no app connection has opened yet" : sqlite3_errstr(rc)); + return 0; + } + return (jlong)(intptr_t)db; +} + +static void bridge_close(JNIEnv *env, jclass cls, jlong handle) { + (void)env; + (void)cls; + sqlite3_close_v2((sqlite3 *)(intptr_t)handle); +} + +typedef struct deadline { + int64_t at_ms; + int hit; +} deadline; + +static int progress_cb(void *p) { + deadline *d = (deadline *)p; + if (now_ms() < d->at_ms) return 0; + d->hit = 1; + return 1; +} + +// Binds args[i] to parameter i + 1. Returns SQLITE_OK or a code with *message set. +static int bind_args(JNIEnv *env, sqlite3 *db, sqlite3_stmt *st, jobjectArray args, const char **message) { + jsize count = (*env)->GetArrayLength(env, args); + for (jsize i = 0; i < count; i++) { + jobject v = (*env)->GetObjectArrayElement(env, args, i); + int rc; + if (!v) { + rc = sqlite3_bind_null(st, i + 1); + } else if ((*env)->IsInstanceOf(env, v, g_long)) { + jmethodID m = (*env)->GetMethodID(env, g_long, "longValue", "()J"); + rc = sqlite3_bind_int64(st, i + 1, (sqlite3_int64)(*env)->CallLongMethod(env, v, m)); + } else if ((*env)->IsInstanceOf(env, v, g_double)) { + jmethodID m = (*env)->GetMethodID(env, g_double, "doubleValue", "()D"); + rc = sqlite3_bind_double(st, i + 1, (*env)->CallDoubleMethod(env, v, m)); + } else if ((*env)->IsInstanceOf(env, v, g_string)) { + jsize len = (*env)->GetStringLength(env, (jstring)v); + const jchar *chars = (*env)->GetStringChars(env, (jstring)v, NULL); + if (!chars) return SQLITE_NOMEM; + rc = sqlite3_bind_text16(st, i + 1, chars, len * 2, SQLITE_TRANSIENT); + (*env)->ReleaseStringChars(env, (jstring)v, chars); + } else { + jclass bytes_class = (*env)->FindClass(env, "[B"); + int is_bytes = bytes_class && (*env)->IsInstanceOf(env, v, bytes_class); + if (bytes_class) (*env)->DeleteLocalRef(env, bytes_class); + if (!is_bytes) { + *message = "unsupported argument type"; + return SQLITE_MISUSE; + } + jsize len = (*env)->GetArrayLength(env, (jbyteArray)v); + void *copy = malloc(len ? (size_t)len : 1); + if (!copy) return SQLITE_NOMEM; + (*env)->GetByteArrayRegion(env, (jbyteArray)v, 0, len, (jbyte *)copy); + rc = sqlite3_bind_blob64(st, i + 1, copy, (sqlite3_uint64)len, SQLITE_TRANSIENT); + free(copy); + } + if (v) (*env)->DeleteLocalRef(env, v); + if (rc != SQLITE_OK) { + *message = sqlite3_errmsg(db); + return rc; + } + if ((*env)->ExceptionCheck(env)) return SQLITE_ERROR; + } + return SQLITE_OK; +} + +// A UTF-8 prefix of at most max bytes ending on a character boundary. +static int utf8_prefix(const unsigned char *s, int len, int max) { + if (len <= max) return len; + while (max > 0 && (s[max] & 0xC0) == 0x80) max--; + return max; +} + +// One cell as a Java object, or NULL for SQL NULL (or on failure, with an exception pending). +// *cut is set when a TEXT or BLOB was cut at its limit, and *bytes grows by what the cell costs the +// result: the TEXT or BLOB bytes kept, 8 for a number. +static jobject cell_value(JNIEnv *env, sqlite3_stmt *st, int col, jint max_text, jint max_blob, int *cut, + int64_t *bytes) { + *cut = 0; + switch (sqlite3_column_type(st, col)) { + case SQLITE_INTEGER: + *bytes += 8; + return (*env)->CallStaticObjectMethod(env, g_long, g_long_value_of, (jlong)sqlite3_column_int64(st, col)); + case SQLITE_FLOAT: + *bytes += 8; + return (*env)->CallStaticObjectMethod(env, g_double, g_double_value_of, (jdouble)sqlite3_column_double(st, col)); + case SQLITE_TEXT: { + const unsigned char *text = sqlite3_column_text(st, col); + if (!text) { // out of memory + throw_engine(env, SQLITE_NOMEM, "out of memory"); + return NULL; + } + int len = sqlite3_column_bytes(st, col); + int keep = utf8_prefix(text, len, max_text); + if (keep < len) *cut = 1; + *bytes += keep; + return utf8_string(env, (const char *)text, keep); + } + case SQLITE_BLOB: { + const void *blob = sqlite3_column_blob(st, col); + int len = sqlite3_column_bytes(st, col); + if (!blob && len > 0) { // out of memory + throw_engine(env, SQLITE_NOMEM, "out of memory"); + return NULL; + } + int keep = len > max_blob ? max_blob : len; + if (keep < len) *cut = 1; + *bytes += keep; + jbyteArray array = (*env)->NewByteArray(env, keep); + if (array && keep > 0) (*env)->SetByteArrayRegion(env, array, 0, keep, (const jbyte *)blob); + return array; + } + default: + return NULL; + } +} + +static jobjectArray bridge_query(JNIEnv *env, jclass cls, jlong handle, jstring sql, jobjectArray args, jlong offset, + jint max_rows, jint max_text, jint max_blob, jint max_total, jlong deadline_ms) { + (void)cls; + sqlite3 *db = (sqlite3 *)(intptr_t)handle; + sqlite3_stmt *st = NULL; + jobjectArray result = NULL; + jlong *cut_cells = NULL; // row * columns + column of each cut cell + size_t cut_n = 0, cut_cap = 0; + deadline dl = {(int64_t)deadline_ms, 0}; + int handler_set = 0; + const char *message = NULL; + int rc; + + if (now_ms() >= dl.at_ms) { + throw_engine(env, SQLITE_INTERRUPT, "the time limit passed before the query started"); + return NULL; + } + jsize sql_len = (*env)->GetStringLength(env, sql); + const jchar *sql_chars = (*env)->GetStringChars(env, sql, NULL); + if (!sql_chars) return NULL; + rc = sqlite3_prepare16_v2(db, sql_chars, sql_len * 2, &st, NULL); + (*env)->ReleaseStringChars(env, sql, sql_chars); + if (rc != SQLITE_OK) { + throw_engine(env, rc, sqlite3_errmsg(db)); + return NULL; + } + + // SQLite checks the handler every 1000 virtual machine steps and stops the statement when it + // returns non-zero; it is cleared before this returns, so it never outlives `dl`. + sqlite3_progress_handler(db, 1000, progress_cb, &dl); + handler_set = 1; + + // A missing argument would be bound as NULL and the query would run, returning a plausible empty result. + int wanted = st ? sqlite3_bind_parameter_count(st) : 0; + jsize given = (*env)->GetArrayLength(env, args); + if (given != wanted) { + char msg[96]; + snprintf(msg, sizeof msg, "the statement has %d parameter(s) but %d argument(s) were given", wanted, (int)given); + throw_engine(env, SQLITE_RANGE, msg); + goto done; + } + rc = st ? bind_args(env, db, st, args, &message) : SQLITE_OK; + if (rc != SQLITE_OK) { + if (!(*env)->ExceptionCheck(env)) throw_engine(env, rc, message ? message : "cannot bind"); + goto done; + } + + int ncols = st ? sqlite3_column_count(st) : 0; + jobjectArray names = (*env)->NewObjectArray(env, ncols, g_string, NULL); + jobject rows = names ? (*env)->NewObject(env, g_array_list, g_list_init) : NULL; + if (!rows) goto done; + for (int c = 0; c < ncols; c++) { + const char *name = sqlite3_column_name(st, c); + jstring s = utf8_string(env, name ? name : "", name ? (int)strlen(name) : 0); + if (!s) goto done; + (*env)->SetObjectArrayElement(env, names, c, s); + (*env)->DeleteLocalRef(env, s); + } + + int has_more = 0, row_count = 0; + int64_t total = 0; + jlong skipped = 0; + while (st && (rc = sqlite3_step(st)) == SQLITE_ROW) { + if (skipped < offset) { + skipped++; + continue; + } + if (row_count >= max_rows) { + has_more = 1; + break; + } + jobjectArray row = (*env)->NewObjectArray(env, ncols, g_object, NULL); + if (!row) goto done; + int64_t row_bytes = 0; + size_t cut_before = cut_n; + for (int c = 0; c < ncols; c++) { + int cut; + jobject cell = cell_value(env, st, c, max_text, max_blob, &cut, &row_bytes); + if ((*env)->ExceptionCheck(env)) goto done; + if (cut) { + if (cut_n == cut_cap) { + size_t cap = cut_cap ? cut_cap * 2 : 16; + jlong *grown = realloc(cut_cells, cap * sizeof(jlong)); + if (!grown) { + throw_engine(env, SQLITE_NOMEM, "out of memory"); + goto done; + } + cut_cells = grown; + cut_cap = cap; + } + cut_cells[cut_n++] = (jlong)row_count * ncols + c; + } + if (cell) { + (*env)->SetObjectArrayElement(env, row, c, cell); + (*env)->DeleteLocalRef(env, cell); + } + } + // The total bounds the host app's heap. The first row always goes in, so a page makes + // progress; one row is bounded by the per-cell limits. + if (row_count > 0 && total + row_bytes > max_total) { + (*env)->DeleteLocalRef(env, row); + cut_n = cut_before; + has_more = 1; + break; + } + total += row_bytes; + (*env)->CallBooleanMethod(env, rows, g_list_add, row); + (*env)->DeleteLocalRef(env, row); + if ((*env)->ExceptionCheck(env)) goto done; + row_count++; + } + if (st && !has_more && rc != SQLITE_DONE) { + if (dl.hit && rc == SQLITE_INTERRUPT) throw_engine(env, SQLITE_INTERRUPT, "the query ran past its time limit"); + else throw_engine(env, rc, sqlite3_errmsg(db)); + goto done; + } + + // {cut cell count, hasMore, then the index of each cut cell} + jlongArray counts = (*env)->NewLongArray(env, (jsize)(2 + cut_n)); + result = counts ? (*env)->NewObjectArray(env, 3, g_object, NULL) : NULL; + if (!result) goto done; + jlong head[2] = {(jlong)cut_n, has_more}; + (*env)->SetLongArrayRegion(env, counts, 0, 2, head); + if (cut_n) (*env)->SetLongArrayRegion(env, counts, 2, (jsize)cut_n, cut_cells); + (*env)->SetObjectArrayElement(env, result, 0, names); + (*env)->SetObjectArrayElement(env, result, 1, rows); + (*env)->SetObjectArrayElement(env, result, 2, counts); + +done: + if (handler_set) sqlite3_progress_handler(db, 0, NULL, NULL); + sqlite3_finalize(st); + free(cut_cells); + // Backstop: a transaction left open (a writable connection, or a way past the authorizer) + // would hold a lock on the app's database until the next query. + if (sqlite3_get_autocommit(db) == 0) sqlite3_exec(db, "ROLLBACK", NULL, NULL, NULL); + return result; +} + +JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) { + (void)reserved; + JNIEnv *env; + if ((*vm)->GetEnv(vm, (void **)&env, JNI_VERSION_1_6) != JNI_OK) return JNI_ERR; + + g_object = global_class(env, "java/lang/Object"); + g_string = global_class(env, "java/lang/String"); + g_array_list = global_class(env, "java/util/ArrayList"); + g_long = global_class(env, "java/lang/Long"); + g_double = global_class(env, "java/lang/Double"); + g_exception = global_class(env, EXCEPTION_CLASS); + jstring utf8 = (*env)->NewStringUTF(env, "UTF-8"); + if (!g_object || !g_string || !g_array_list || !g_long || !g_double || !g_exception || !utf8) return JNI_ERR; + g_utf8_name = (jstring)(*env)->NewGlobalRef(env, utf8); + + g_string_from_bytes = (*env)->GetMethodID(env, g_string, "", "([BLjava/lang/String;)V"); + g_list_init = (*env)->GetMethodID(env, g_array_list, "", "()V"); + g_list_add = (*env)->GetMethodID(env, g_array_list, "add", "(Ljava/lang/Object;)Z"); + g_long_value_of = (*env)->GetStaticMethodID(env, g_long, "valueOf", "(J)Ljava/lang/Long;"); + g_double_value_of = (*env)->GetStaticMethodID(env, g_double, "valueOf", "(D)Ljava/lang/Double;"); + g_exception_init = (*env)->GetMethodID(env, g_exception, "", "(ILjava/lang/String;)V"); + if (!g_string_from_bytes || !g_list_init || !g_list_add || !g_long_value_of || !g_double_value_of || !g_exception_init) + return JNI_ERR; + + static const JNINativeMethod methods[] = { + {"install", "()I", (void *)bridge_install}, + {"connections", "()[Ljava/lang/Object;", (void *)bridge_connections}, + {"readEvents", "(J[B[J)V", (void *)bridge_read_events}, + {"setClientConnected", "(Z)V", (void *)bridge_set_client_connected}, + {"openServer", "([BZ)J", (void *)bridge_open_server}, + {"query", "(JLjava/lang/String;[Ljava/lang/Object;JIIIIJ)[Ljava/lang/Object;", (void *)bridge_query}, + {"close", "(J)V", (void *)bridge_close}, + }; + jclass bridge = (*env)->FindClass(env, BRIDGE_CLASS); + if (!bridge || (*env)->RegisterNatives(env, bridge, methods, sizeof methods / sizeof methods[0]) != JNI_OK) + return JNI_ERR; + return JNI_VERSION_1_6; +} diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/DebuggerStartup.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/DebuggerStartup.kt new file mode 100644 index 0000000000..a2605df810 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/DebuggerStartup.kt @@ -0,0 +1,101 @@ +package org.appdevforall.sqlitedebugger + +import android.content.ContentProvider +import android.content.ContentValues +import android.content.pm.ApplicationInfo +import android.database.Cursor +import android.net.Uri +import android.util.Log +import org.appdevforall.sqlitedebugger.engine.NativeDebugEngine +import org.appdevforall.sqlitedebugger.mcp.DatabaseTools +import org.appdevforall.sqlitedebugger.mcp.McpHttp +import org.appdevforall.sqlitedebugger.mcp.McpProtocol +import java.io.File +import java.util.UUID +import kotlin.concurrent.thread + +/** + * Starts the debugger when the app's process starts. Refuses to run in a build that is not + * debuggable, so a release build that includes this library by mistake crashes at once instead of + * opening a local server. + */ +class DebuggerStartup : ContentProvider() { + override fun onCreate(): Boolean { + val context = context ?: return false + check(context.applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE != 0) { + "The SQLite debugger is in a build that is not debuggable. Remove it from release builds." + } + val databaseDir = File(context.applicationInfo.dataDir, "databases") + // Throwable, not Exception: a missing library is an UnsatisfiedLinkError, and an Error that + // escapes here kills the host app. A debugger that cannot start is disabled, never a crash. + try { + // Synchronous: an app that opens its database in Application.onCreate or another + // provider would otherwise be missed. + val engine = NativeDebugEngine() + engine.install() + thread(name = "sqlite-debugger-start", isDaemon = true) { + try { + startServer(engine, databaseDir) + } catch (t: Throwable) { + Log.e(TAG, "could not start the server", t) + } + } + } catch (t: Throwable) { + Log.e(TAG, "the SQLite debugger is disabled", t) + } + return true + } + + private fun startServer( + engine: NativeDebugEngine, + databaseDir: File, + ) { + val tools = DatabaseTools(engine, databaseDir) + val token = McpHttp.newToken() + val protocol = McpProtocol(tools.tools(), "1", { UUID.randomUUID().toString() }) + val http = McpHttp(protocol, { token }, log = { Log.w(TAG, it) }) + http.start() + port = http.boundPort + endpointToken = token + Log.i(TAG, "listening on 127.0.0.1:${http.boundPort}") + } + + override fun query( + uri: Uri, + projection: Array?, + selection: String?, + selectionArgs: Array?, + sortOrder: String?, + ): Cursor? = null + + override fun getType(uri: Uri): String? = null + + override fun insert( + uri: Uri, + values: ContentValues?, + ): Uri? = null + + override fun delete( + uri: Uri, + selection: String?, + selectionArgs: Array?, + ): Int = 0 + + override fun update( + uri: Uri, + values: ContentValues?, + selection: String?, + selectionArgs: Array?, + ): Int = 0 + + companion object { + private const val TAG = "SqliteDebugger" + + /** Set once the server listens; the broadcast to Code on the Go (ADFA-7137) reads them. */ + @Volatile var port: Int = 0 + private set + + @Volatile var endpointToken: String? = null + private set + } +} diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/DebugEngine.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/DebugEngine.kt new file mode 100644 index 0000000000..7aeabc53fa --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/DebugEngine.kt @@ -0,0 +1,152 @@ +package org.appdevforall.sqlitedebugger.engine + +import java.io.Closeable + +/** + * What the MCP server (ADFA-7136) needs from the SQLite extension (ADFA-7135). The device + * implementation is a thin JNI layer over `sqlitedbg.h`; JVM tests use a fake. Every call may + * block briefly and must not run on the main thread. + */ +interface DebugEngine { + /** The app's connections SQLite has not yet freed, oldest first. Server connections are not listed. */ + fun connections(): List + + /** + * Events with [Event.seq] greater than [after], oldest first, up to about [maxBytes] of + * native records. Pass [EventBatch.next] as [after] next time; start from 0. + */ + fun readEvents( + after: Long, + maxBytes: Int, + ): EventBatch + + /** While true, statements are recorded with their bound values written in. */ + fun setClientConnected(connected: Boolean) + + /** + * Opens [path] for the server: read-only (with an authorizer refusing writes) unless + * [writable]. Only a path one of [connections] has open may be passed: opening a database that + * Android's own SQLite has open, with requery's, can corrupt it (ADFA-7131, Design 3). + * + * @throws DebugEngineException if SQLite cannot open it, or no app connection has opened yet. + */ + fun openServerConnection( + path: String, + writable: Boolean, + ): ServerConnection +} + +/** A connection the server opened with [DebugEngine.openServerConnection]. Use from one thread at a time. */ +interface ServerConnection : Closeable { + /** + * Runs [sql] with [args] bound (null, [Long], [Double], [String] or [ByteArray]), skips + * [offset] rows, and returns at most [QueryLimits.maxRows]. Paging re-runs the statement, so a + * later page can differ if the data changed. Stops the statement (sqlite3_interrupt, through a + * progress handler on this server connection) once [deadlineMillis] (System.currentTimeMillis + * clock) has passed. + * + * @throws DebugEngineException for SQLite errors, a refused write, or the deadline. + */ + fun query( + sql: String, + args: List, + offset: Long, + limits: QueryLimits, + deadlineMillis: Long, + ): QueryResult +} + +data class AppConnection( + /** Unique within the process, never reused. */ + val id: Long, + /** Empty for an in-memory or temporary database. */ + val path: String, + val readOnly: Boolean, + /** The app asked to close it; the close may still fail, so it stays listed until freed. */ + val closing: Boolean, + val openedAtEpochNanos: Long, + val openThreadId: Long, +) + +enum class EventKind( + /** The `kind` value in `sqlitedbg_event_record`. */ + val nativeValue: Int, +) { + /** A statement starts; also each trigger it fires, whose SQL starts with "--". */ + STATEMENT(1), + + /** A statement finished; [Event.durationNanos] is set. */ + PROFILE(2), + + /** The app asked SQLite to close the connection; [Event.sql] is empty. */ + CLOSE(3), +} + +data class Event( + /** 1, 2, 3...; a gap means events were dropped. */ + val seq: Long, + val kind: EventKind, + val connectionId: Long, + val threadId: Long, + val timeEpochNanos: Long, + /** Set for [EventKind.PROFILE] only. */ + val durationNanos: Long?, + val sql: String, + /** [sql] was cut at the extension's limit. */ + val truncated: Boolean, + /** [sql] has its bound values written in. */ + val expanded: Boolean, +) + +data class EventBatch( + val events: List, + /** Pass as `after` to the next [DebugEngine.readEvents]. */ + val next: Long, + /** Events after the requested cursor that left the buffer before they were read. */ + val dropped: Long, +) + +data class QueryLimits( + val maxRows: Int, + /** Longer TEXT values are cut at a UTF-8 character boundary and counted in [QueryResult.truncatedCells]. */ + val maxTextBytes: Int, + /** Longer BLOB values are cut and counted in [QueryResult.truncatedCells]. */ + val maxBlobBytes: Int, + /** + * A bound on the cell bytes of one result, so a wide SELECT cannot exhaust the app's heap: the + * TEXT and BLOB bytes kept, plus 8 for each number. Rows are added until the next one would pass + * it; the rest is [QueryResult.hasMore]. The first row is always returned. + */ + val maxTotalBytes: Int, +) + +data class QueryResult( + val columns: List, + /** Each value is null, [Long], [Double], [String] or [ByteArray]. Doubles may be NaN or infinite. */ + val rows: List>, + val truncatedCells: Int, + /** More rows follow the last one returned: the row limit or [QueryLimits.maxTotalBytes] was reached. */ + val hasMore: Boolean, + /** + * Which cells were cut, as `row * columns.size + column`: a BLOB of exactly [QueryLimits.maxBlobBytes] + * is whole and not listed, one byte more is. [truncatedCells] is its size. + */ + val truncatedCellIndexes: Set = emptySet(), +) { + fun isTruncated( + row: Int, + column: Int, + ) = (row.toLong() * columns.size + column) in truncatedCellIndexes +} + +class DebugEngineException( + /** The SQLite result code, or [SQLITE_MISUSE] when no app connection has opened yet. */ + val sqliteCode: Int, + message: String, +) : Exception(message) { + companion object { + const val SQLITE_AUTH = 23 + const val SQLITE_INTERRUPT = 9 + const val SQLITE_MISUSE = 21 + } +} diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/EventDecoder.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/EventDecoder.kt new file mode 100644 index 0000000000..73af648598 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/EventDecoder.kt @@ -0,0 +1,69 @@ +package org.appdevforall.sqlitedebugger.engine + +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** + * Decodes the records `sqlitedbg_read_events` copies out. Mirrors `sqlitedbg_event_record` in + * `sqlitedbg.h`: change both together. Little-endian, because the CPU is on every ABI we build. + */ +internal object EventDecoder { + /** `sizeof(sqlitedbg_event_record)`. */ + const val HEADER_BYTES = 56 + + private const val OFFSET_RECORD_BYTES = 0 + private const val OFFSET_KIND = 4 + private const val OFFSET_SEQ = 8 + private const val OFFSET_CONNECTION_ID = 16 + private const val OFFSET_THREAD_ID = 24 + private const val OFFSET_TIME_NS = 32 + private const val OFFSET_DURATION_NS = 40 + private const val OFFSET_FLAGS = 48 + private const val OFFSET_SQL_BYTES = 52 + + private const val FLAG_TRUNCATED = 1 + private const val FLAG_EXPANDED = 2 + + /** + * The records in the first [length] bytes of [bytes]. + * + * @throws IllegalArgumentException if they are not whole, well-formed records. + */ + fun decode( + bytes: ByteArray, + length: Int, + ): List { + require(length in 0..bytes.size) { "length $length outside 0..${bytes.size}" } + val buffer = ByteBuffer.wrap(bytes, 0, length).order(ByteOrder.LITTLE_ENDIAN) + val events = ArrayList() + var offset = 0 + while (offset < length) { + require(length - offset >= HEADER_BYTES) { "record at $offset is cut off by the end of the data" } + val recordBytes = buffer.getInt(offset + OFFSET_RECORD_BYTES) + val sqlBytes = buffer.getInt(offset + OFFSET_SQL_BYTES) + require(sqlBytes >= 0 && recordBytes >= 0 && recordBytes % 8 == 0 && sqlBytes <= recordBytes - HEADER_BYTES) { + "record at $offset has record_bytes=$recordBytes sql_bytes=$sqlBytes" + } + require(recordBytes <= length - offset) { "record at $offset runs past the end of the data" } + val kindValue = buffer.getInt(offset + OFFSET_KIND) + val kind = + EventKind.values().firstOrNull { it.nativeValue == kindValue } + ?: throw IllegalArgumentException("record at $offset has unknown kind $kindValue") + val flags = buffer.getInt(offset + OFFSET_FLAGS) + events += + Event( + seq = buffer.getLong(offset + OFFSET_SEQ), + kind = kind, + connectionId = buffer.getLong(offset + OFFSET_CONNECTION_ID), + threadId = buffer.getLong(offset + OFFSET_THREAD_ID), + timeEpochNanos = buffer.getLong(offset + OFFSET_TIME_NS), + durationNanos = if (kind == EventKind.PROFILE) buffer.getLong(offset + OFFSET_DURATION_NS) else null, + sql = String(bytes, offset + HEADER_BYTES, sqlBytes, Charsets.UTF_8), + truncated = flags and FLAG_TRUNCATED != 0, + expanded = flags and FLAG_EXPANDED != 0, + ) + offset += recordBytes + } + return events + } +} diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeBridge.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeBridge.kt new file mode 100644 index 0000000000..fa95abee7b --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeBridge.kt @@ -0,0 +1,54 @@ +package org.appdevforall.sqlitedebugger.engine + +/** + * The JNI entry points of libsqlitedbg.so, registered by name in its JNI_OnLoad (sqlitedbg_jni.c). + * Signatures there must match these. Only [NativeDebugEngine] calls them. + */ +internal object NativeBridge { + /** Registers the extension with requery's SQLite. 0 (SQLITE_OK) or the SQLite code; -1 if libsqlite3x.so is not loaded. */ + @JvmStatic + external fun install(): Int + + /** `[LongArray, Array]`: 5 longs per connection (id, readOnly, closing, openedAtNs, openThreadId), and the paths. */ + @JvmStatic + external fun connections(): Array + + /** Fills [buf] with records after [after]; [out] gets next, dropped and the bytes written. */ + @JvmStatic + external fun readEvents( + after: Long, + buf: ByteArray, + out: LongArray, + ) + + @JvmStatic + external fun setClientConnected(connected: Boolean) + + /** A handle for [query] and [close]. Throws [DebugEngineException]. */ + @JvmStatic + external fun openServer( + path: ByteArray, + writable: Boolean, + ): Long + + /** + * `[Array, ArrayList>, LongArray]`: the column names, the rows, and + * `{truncatedCells, hasMore, then the index of each cut cell}` (row * columns + column). + * Throws [DebugEngineException]. + */ + @JvmStatic + external fun query( + handle: Long, + sql: String, + args: Array, + offset: Long, + maxRows: Int, + maxTextBytes: Int, + maxBlobBytes: Int, + maxTotalBytes: Int, + deadlineMillis: Long, + ): Array + + @JvmStatic + external fun close(handle: Long) +} diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeDebugEngine.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeDebugEngine.kt new file mode 100644 index 0000000000..f020c2d1e7 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeDebugEngine.kt @@ -0,0 +1,132 @@ +package org.appdevforall.sqlitedebugger.engine + +/** + * [DebugEngine] over the JNI layer of libsqlitedbg.so. Needs requery's libsqlite3x.so in the process: + * the extension runs inside it, on connections it opens after [install]. Call [install] before + * anything else; the native state is per process, so every instance shares it. + */ +class NativeDebugEngine : DebugEngine { + /** + * Loads requery's SQLite and the debugger's library and registers the extension. Connections + * opened before this are never seen, so call it as early as the process allows. Calling it + * again, on any instance, does nothing. + * + * @throws DebugEngineException if SQLite refuses the extension. + * @throws UnsatisfiedLinkError if a library is missing. + */ + fun install() { + synchronized(lock) { + if (installed) return + System.loadLibrary("sqlite3x") + System.loadLibrary("sqlitedbg") + val rc = NativeBridge.install() + if (rc != 0) { + throw DebugEngineException( + if (rc < 0) DebugEngineException.SQLITE_MISUSE else rc, + if (rc < 0) "libsqlite3x.so is not loaded" else "sqlite3_auto_extension failed with $rc", + ) + } + installed = true + } + } + + override fun connections(): List { + val result = NativeBridge.connections() + + @Suppress("UNCHECKED_CAST") + val paths = result[1] as Array + val meta = result[0] as LongArray + return paths.indices.map { i -> + val m = i * META_PER_CONNECTION + AppConnection( + id = meta[m], + path = paths[i], + readOnly = meta[m + 1] != 0L, + closing = meta[m + 2] != 0L, + openedAtEpochNanos = meta[m + 3], + openThreadId = meta[m + 4], + ) + } + } + + override fun readEvents( + after: Long, + maxBytes: Int, + ): EventBatch { + val buffer = ByteArray(maxBytes.coerceIn(1, MAX_READ_BYTES)) + val out = LongArray(3) + NativeBridge.readEvents(after, buffer, out) + return EventBatch(events = EventDecoder.decode(buffer, out[2].toInt()), next = out[0], dropped = out[1]) + } + + override fun setClientConnected(connected: Boolean) = NativeBridge.setClientConnected(connected) + + override fun openServerConnection( + path: String, + writable: Boolean, + ): ServerConnection = NativeServerConnection(NativeBridge.openServer(path.toByteArray(Charsets.UTF_8), writable)) + + private class NativeServerConnection( + private var handle: Long, + ) : ServerConnection { + override fun query( + sql: String, + args: List, + offset: Long, + limits: QueryLimits, + deadlineMillis: Long, + ): QueryResult { + check(handle != 0L) { "the connection is closed" } + require(offset >= 0) { "offset $offset is negative" } + require(limits.maxRows > 0 && limits.maxTextBytes >= 0 && limits.maxBlobBytes >= 0 && limits.maxTotalBytes > 0) { + "bad limits $limits" + } + val result = + NativeBridge.query( + handle, + sql, + Array(args.size) { bindable(args[it]) }, + offset, + limits.maxRows, + limits.maxTextBytes, + limits.maxBlobBytes, + limits.maxTotalBytes, + deadlineMillis, + ) + + @Suppress("UNCHECKED_CAST") + val columns = (result[0] as Array).asList() + + @Suppress("UNCHECKED_CAST") + val rows = (result[1] as List>).map { it.asList() } + val counts = result[2] as LongArray + val cut = HashSet() + for (i in 2 until counts.size) cut += counts[i] + return QueryResult(columns, rows, truncatedCells = counts[0].toInt(), hasMore = counts[1] != 0L, truncatedCellIndexes = cut) + } + + override fun close() { + val h = handle + handle = 0 + if (h != 0L) NativeBridge.close(h) + } + + private fun bindable(value: Any?): Any? = + when (value) { + null, is Long, is Double, is String, is ByteArray -> value + is Int -> value.toLong() + is Short -> value.toLong() + is Byte -> value.toLong() + is Float -> value.toDouble() + else -> throw IllegalArgumentException("cannot bind a ${value.javaClass.name}") + } + } + + private companion object { + const val META_PER_CONNECTION = 5 + const val MAX_READ_BYTES = 16 shl 20 + + val lock = Any() + var installed = false + } +} diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt new file mode 100644 index 0000000000..19ceceaa98 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/DatabaseTools.kt @@ -0,0 +1,583 @@ +package org.appdevforall.sqlitedebugger.mcp + +import org.appdevforall.sqlitedebugger.engine.AppConnection +import org.appdevforall.sqlitedebugger.engine.DebugEngine +import org.appdevforall.sqlitedebugger.engine.DebugEngineException +import org.appdevforall.sqlitedebugger.engine.QueryLimits +import org.appdevforall.sqlitedebugger.engine.QueryResult +import org.appdevforall.sqlitedebugger.engine.ServerConnection +import org.json.JSONArray +import org.json.JSONObject +import java.io.File +import java.util.Timer +import java.util.TimerTask + +/** + * The debugger's MCP tools, all read-only: they open server connections through [engine] with + * `writable = false`, so SQLite's authorizer refuses a write whatever SQL a client sends. A tool + * names a database by the path one of the app's own connections has open; any other path is + * refused before the engine is called (opening a file Android's SQLite has open, with the + * extension's SQLite, can corrupt it). + * + * One lock guards the cached server connections and every query, so tool calls run one at a time: + * a slow query (up to [DEADLINE_MS]) makes the others wait. That is accepted: the debugger serves + * one agent, and a connection per call would open the app's file many times over. + */ +class DatabaseTools( + private val engine: DebugEngine, + /** The app's database folder, to list files no connection has opened; null on the JVM. */ + private val databaseDir: File? = null, + private val clock: () -> Long = System::currentTimeMillis, +) { + companion object { + const val DEFAULT_MAX_ROWS = 100 + const val MAX_ROWS = 1000 + const val MAX_TEXT_BYTES = 4096 + const val MAX_BLOB_BYTES = 1024 + + /** The most cell bytes one tool result holds (the engine's `maxTotalBytes`); the rest is paged. */ + const val MAX_TOTAL_BYTES = 4 * 1024 * 1024 + + /** Below the client's 60 s limit on a tool call. */ + const val DEADLINE_MS = 50_000L + const val EVENT_BATCH_BYTES = 256 * 1024 + const val MAX_TABLES_FOR_COUNTS = 10 + + /** Bound values stop being recorded this long after the last get_events call. */ + const val EVENTS_IDLE_MS = 60_000L + + private const val MAX_SAFE_INTEGER = 9007199254740992L // 2^53 + private val SIDE_FILE_SUFFIXES = listOf("-wal", "-shm", "-journal") + + const val PAGING_NOTE = "Paging re-runs the query for each page, so a later page can differ if the data changed." + + /** A SQLite value as JSON: org.json throws on NaN and infinities, and JSON numbers lose integers past 2^53. */ + fun encodeValue( + value: Any?, + truncated: Boolean = false, + ): Any = + when (value) { + null -> { + JSONObject.NULL + } + + is Long -> { + encodeLong(value) + } + + is Int -> { + value.toLong() + } + + is Double -> { + when { + value.isNaN() -> "NaN" + value == Double.POSITIVE_INFINITY -> "Infinity" + value == Double.NEGATIVE_INFINITY -> "-Infinity" + else -> value + } + } + + is String -> { + value + } + + is ByteArray -> { + JSONObject() + .put("base64", McpHttp.base64(value, urlSafe = false, pad = true)) + .put("bytes", value.size) + .put("truncated", truncated) + } + + else -> { + value.toString() + } + } + + fun encodeLong(value: Long): Any = if (value > MAX_SAFE_INTEGER || value < -MAX_SAFE_INTEGER) value.toString() else value + + fun quoteIdentifier(name: String) = "\"" + name.replace("\"", "\"\"") + "\"" + } + + private val lock = Any() + private val serverConnections = HashMap() + private var clientConnected = false + private var lastEventsCallMs = 0L + private var idleTimer: Timer? = null + + /** A server connection and the ids of the app connections to its path when it was opened. */ + private class CachedConnection( + val connection: ServerConnection, + val appConnectionIds: Set, + ) + + /** Closes the server connections and tells the engine no client is watching. Call when the server stops. */ + fun onServerStopped() { + synchronized(lock) { + serverConnections.values.forEach { runCatching { it.connection.close() } } + serverConnections.clear() + idleTimer?.cancel() + idleTimer = null + if (clientConnected) { + clientConnected = false + engine.setClientConnected(false) + } + } + } + + /** + * Stops recording bound values once [EVENTS_IDLE_MS] have passed since the last get_events call: + * expanding every statement costs memory, and nobody is reading. Checked on every tool call and + * by a timer, so it also happens when no call comes. + */ + fun expireIdleClient() { + synchronized(lock) { + if (clientConnected && clock() - lastEventsCallMs >= EVENTS_IDLE_MS) { + clientConnected = false + engine.setClientConnected(false) + } + } + } + + private fun scheduleIdleCheck() { + val timer = idleTimer ?: Timer("sqlite-debugger-idle", true).also { idleTimer = it } + timer.schedule( + object : TimerTask() { + override fun run() { + expireIdleClient() + synchronized(lock) { + if (clientConnected && idleTimer === timer) scheduleIdleCheck() + } + } + }, + EVENTS_IDLE_MS / 4, + ) + } + + fun tools(): List = + listOf( + tool( + "list_databases", + "List databases", + "Lists the app's databases: each one an app connection has open (available), and the files in the app's database folder " + + "that none has opened (not available: not inspectable until the app opens them).", + ) { listDatabases() }, + tool( + "list_connections", + "List connections", + "Lists the app's open SQLite connections (not yet freed by SQLite), oldest first.", + ) { listConnections() }, + tool( + "get_schema", + "Get schema", + "Returns the tables, indexes, views and triggers of a database, with their CREATE statements, from sqlite_master.", + databaseSchema(), + ) { getSchema(it) }, + tool( + "query", + "Run a query", + "Runs one read-only SQL statement. Returns at most max_rows rows (default $DEFAULT_MAX_ROWS, at most $MAX_ROWS); " + + "TEXT cells are cut at $MAX_TEXT_BYTES bytes and BLOBs at $MAX_BLOB_BYTES. A result also stops at about " + + "${MAX_TOTAL_BYTES / (1024 * 1024)} MiB of cell bytes (has_more is then true). Use offset to page. " + + "$PAGING_NOTE Writes are refused.", + databaseSchema( + "sql" to prop("string", "The SQL statement."), + "args" to JSONObject().put("type", "array").put("description", "Values for ? parameters: null, numbers or strings."), + "offset" to prop("integer", "Rows to skip. Default 0."), + "max_rows" to prop("integer", "Rows to return. Default $DEFAULT_MAX_ROWS, at most $MAX_ROWS."), + required = listOf("sql"), + ), + ) { query(it) }, + tool( + "explain_query_plan", + "Explain a query plan", + "Runs EXPLAIN QUERY PLAN on a SQL statement and returns SQLite's plan. The statement is not run.", + databaseSchema( + "sql" to prop("string", "The SQL statement."), + "args" to JSONObject().put("type", "array").put("description", "Values for ? parameters."), + required = listOf("sql"), + ), + ) { explain(it) }, + tool( + "get_stats", + "Get statistics", + "Returns page_count, page_size, freelist_count, and the row count of each table when the database has fewer than " + + "$MAX_TABLES_FOR_COUNTS tables.", + databaseSchema(), + ) { getStats(it) }, + tool( + "integrity_check", + "Check integrity", + "Runs PRAGMA quick_check, or the slower PRAGMA integrity_check when full is true. " + + "ok is true when SQLite reports no problem.", + databaseSchema("full" to prop("boolean", "Run the full integrity_check. Default false."), required = emptyList()), + ) { integrityCheck(it) }, + tool( + "get_events", + "Get SQL events", + "Returns the SQL the app ran, oldest first, after the cursor. Pass the returned next as after to continue; " + + "dropped counts events that left the buffer unread. The first call starts recording bound values into the SQL; " + + "that stops ${EVENTS_IDLE_MS / 1000} s after the last get_events call, so call it at least that often while watching.", + JSONObject() + .put("type", "object") + .put("additionalProperties", false) + .put("properties", JSONObject().put("after", prop("integer", "Cursor: the next from the last call. Default 0."))) + .put("required", JSONArray()), + ) { getEvents(it) }, + ) + + private fun tool( + name: String, + title: String, + description: String, + inputSchema: JSONObject = JSONObject().put("type", "object").put("properties", JSONObject()).put("additionalProperties", false), + run: (JSONObject) -> JSONObject, + ) = McpTool(name, title, description, inputSchema, readOnly = true, run = { args -> + expireIdleClient() + run(args) + }) + + private fun prop( + type: String, + description: String, + ) = JSONObject().put("type", type).put("description", description) + + private fun databaseSchema( + vararg extra: Pair, + required: List = emptyList(), + ): JSONObject { + val props = JSONObject().put("database", prop("string", "The database path, as list_databases gives it.")) + extra.forEach { props.put(it.first, it.second) } + return JSONObject() + .put("type", "object") + .put("properties", props) + .put("additionalProperties", false) + .put("required", JSONArray(listOf("database") + required)) + } + + private fun canonical(path: String): String = runCatching { File(path).canonicalPath }.getOrDefault(path) + + private fun listDatabases(): JSONObject { + val connections = engine.connections() + val byPath = LinkedHashMap>() + for (c in connections) if (c.path.isNotEmpty()) byPath.getOrPut(c.path) { mutableListOf() } += c.id + // The folder may be reached through a link (/data/user/0/ and /data/data/), so match resolved paths. + val openCanonical = byPath.keys.map { canonical(it) }.toSet() + val list = JSONArray() + for ((path, ids) in byPath) { + list.put(JSONObject().put("path", path).put("available", true).put("connections", ids.size)) + } + databaseDir?.listFiles()?.sortedBy { it.name }?.forEach { f -> + if (!f.isFile || SIDE_FILE_SUFFIXES.any { f.name.endsWith(it) }) return@forEach + if (canonical(f.absolutePath) in openCanonical) return@forEach + list.put(JSONObject().put("path", f.absolutePath).put("available", false).put("connections", 0)) + } + return JSONObject().put("databases", list) + } + + private fun listConnections(): JSONObject { + val list = JSONArray() + for (c in engine.connections()) { + list.put( + JSONObject() + .put("id", encodeLong(c.id)) + .put("path", c.path) + .put("read_only", c.readOnly) + .put("closing", c.closing) + .put("opened_at_epoch_nanos", encodeLong(c.openedAtEpochNanos)) + .put("open_thread_id", encodeLong(c.openThreadId)), + ) + } + return JSONObject().put("connections", list) + } + + private fun getSchema(args: JSONObject): JSONObject { + val result = + run( + args, + "SELECT type, name, tbl_name, sql FROM sqlite_master ORDER BY type, name", + emptyList(), + 0, + QueryLimits(MAX_ROWS, 65536, MAX_BLOB_BYTES, MAX_TOTAL_BYTES), + ) + val groups = linkedMapOf("table" to JSONArray(), "index" to JSONArray(), "view" to JSONArray(), "trigger" to JSONArray()) + for (row in result.rows) { + val item = JSONObject().put("name", encodeValue(row[1])).put("table", encodeValue(row[2])).put("sql", encodeValue(row[3])) + groups[row[0] as? String]?.put(item) + } + return JSONObject() + .put("tables", groups["table"]) + .put("indexes", groups["index"]) + .put("views", groups["view"]) + .put("triggers", groups["trigger"]) + .put("truncated", result.hasMore) + } + + private fun query(args: JSONObject): JSONObject { + val sql = stringArg(args, "sql") + val offset = intArg(args, "offset", 0, 0, Long.MAX_VALUE) + val maxRows = intArg(args, "max_rows", DEFAULT_MAX_ROWS.toLong(), 1, MAX_ROWS.toLong()).toInt() + val result = run(args, sql, bindArgs(args), offset, QueryLimits(maxRows, MAX_TEXT_BYTES, MAX_BLOB_BYTES, MAX_TOTAL_BYTES)) + val out = encodeResult(result, offset) + out.put("note", PAGING_NOTE) + return out + } + + private fun explain(args: JSONObject): JSONObject { + val result = + run( + args, + "EXPLAIN QUERY PLAN " + stringArg(args, "sql"), + bindArgs(args), + 0, + QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES, MAX_TOTAL_BYTES), + ) + return JSONObject().put("plan", rowsAsObjects(result)) + } + + private fun getStats(args: JSONObject): JSONObject { + val limits = QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES, MAX_TOTAL_BYTES) + val deadline = clock() + DEADLINE_MS + val path = checkedPath(args) + + fun pragma(name: String) = runChecked(path, "PRAGMA $name", emptyList(), 0, limits, deadline).rows.firstOrNull()?.firstOrNull() + val pageCount = pragma("page_count") as? Long + val pageSize = pragma("page_size") as? Long + val out = + JSONObject() + .put("page_count", encodeValue(pageCount)) + .put("page_size", encodeValue(pageSize)) + .put("freelist_count", encodeValue(pragma("freelist_count"))) + if (pageCount != null && pageSize != null) out.put("size_bytes", encodeLong(pageCount * pageSize)) + val tables = + runChecked( + path, + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", + emptyList(), + 0, + limits, + deadline, + ).rows + .map { it[0] as String } + if (tables.size < MAX_TABLES_FOR_COUNTS) { + val counts = JSONObject() + for (t in tables) { + val n = + try { + runChecked( + path, + "SELECT COUNT(*) FROM ${quoteIdentifier(t)}", + emptyList(), + 0, + limits, + deadline, + ).rows.firstOrNull()?.firstOrNull() + } catch (e: McpToolError) { + e.message + } + counts.put(t, encodeValue(n)) + } + out.put("row_counts", counts) + } else { + out.put("row_counts_skipped", "The database has ${tables.size} tables; counts are given only under $MAX_TABLES_FOR_COUNTS.") + } + return out + } + + private fun integrityCheck(args: JSONObject): JSONObject { + val full = + when (val v = args.opt("full")) { + null, JSONObject.NULL -> false + is Boolean -> v + else -> throw McpToolError("full must be true or false.") + } + val pragma = if (full) "PRAGMA integrity_check" else "PRAGMA quick_check" + val result = run(args, pragma, emptyList(), 0, QueryLimits(MAX_ROWS, MAX_TEXT_BYTES, MAX_BLOB_BYTES, MAX_TOTAL_BYTES)) + val messages = result.rows.map { it.firstOrNull()?.toString() ?: "" } + return JSONObject() + .put("check", if (full) "integrity_check" else "quick_check") + .put("ok", messages == listOf("ok")) + .put("messages", JSONArray(messages)) + } + + private fun getEvents(args: JSONObject): JSONObject { + val after = intArg(args, "after", 0, 0, Long.MAX_VALUE) + synchronized(lock) { + lastEventsCallMs = clock() + if (!clientConnected) { + engine.setClientConnected(true) + clientConnected = true + scheduleIdleCheck() + } + } + val batch = + try { + engine.readEvents(after, EVENT_BATCH_BYTES) + } catch (e: DebugEngineException) { + throw engineError(e) + } + val events = JSONArray() + for (e in batch.events) { + events.put( + JSONObject() + .put("seq", encodeLong(e.seq)) + .put("kind", e.kind.name.lowercase()) + .put("connection_id", encodeLong(e.connectionId)) + .put("thread_id", encodeLong(e.threadId)) + .put("time_epoch_nanos", encodeLong(e.timeEpochNanos)) + .put("duration_nanos", e.durationNanos?.let { encodeLong(it) } ?: JSONObject.NULL) + .put("sql", e.sql) + .put("truncated", e.truncated) + .put("expanded", e.expanded), + ) + } + return JSONObject().put("next", encodeLong(batch.next)).put("dropped", encodeLong(batch.dropped)).put("events", events) + } + + private fun encodeResult( + result: QueryResult, + offset: Long, + ): JSONObject { + val rows = JSONArray() + result.rows.forEachIndexed { r, row -> + rows.put(JSONArray().also { a -> row.forEachIndexed { c, v -> a.put(encodeValue(v, result.isTruncated(r, c))) } }) + } + val out = + JSONObject() + .put("columns", JSONArray(result.columns)) + .put("rows", rows) + .put("row_count", result.rows.size) + .put("offset", encodeLong(offset)) + .put("has_more", result.hasMore) + .put("truncated_cells", result.truncatedCells) + if (result.hasMore) out.put("next_offset", encodeLong(offset + result.rows.size)) + return out + } + + private fun rowsAsObjects(result: QueryResult): JSONArray { + val arr = JSONArray() + result.rows.forEachIndexed { r, row -> + val o = JSONObject() + result.columns.forEachIndexed { i, c -> o.put(c, encodeValue(row[i], result.isTruncated(r, i))) } + arr.put(o) + } + return arr + } + + private fun checkedPath(args: JSONObject): String { + val path = stringArg(args, "database") + val known = + engine + .connections() + .map { it.path } + .filter { it.isNotEmpty() } + .distinct() + synchronized(lock) { dropStaleServerConnections(engine.connections()) } + if (path !in known) { + throw McpToolError( + "Unknown database \"$path\": use a path from list_databases that is available.", + JSONObject().put("available", JSONArray(known)), + ) + } + return path + } + + private fun run( + args: JSONObject, + sql: String, + bind: List, + offset: Long, + limits: QueryLimits, + ): QueryResult = runChecked(checkedPath(args), sql, bind, offset, limits, clock() + DEADLINE_MS) + + private fun runChecked( + path: String, + sql: String, + bind: List, + offset: Long, + limits: QueryLimits, + deadline: Long, + ): QueryResult = + synchronized(lock) { + try { + val live = engine.connections() + dropStaleServerConnections(live) + val conn = + serverConnections + .getOrPut(path) { + val ids = live.filter { it.path == path && !it.closing }.map { it.id }.toSet() + CachedConnection(engine.openServerConnection(path, writable = false), ids) + }.connection + conn.query(sql, bind, offset, limits, deadline) + } catch (e: DebugEngineException) { + throw engineError(e) + } + } + + /** + * Closes cached server connections whose file may be gone: after the app deletes and recreates a + * database, a connection opened before still reads the old file. A connection is kept while some + * app connection to its path that was there when it opened is still open; a path no app connection + * has any more, or only new ones, is stale. Call with [lock] held. + */ + private fun dropStaleServerConnections(live: List) { + val it = serverConnections.entries.iterator() + while (it.hasNext()) { + val (path, cached) = it.next() + val ids = live.filter { c -> c.path == path && !c.closing }.map { c -> c.id } + if (ids.none { id -> id in cached.appConnectionIds }) { + runCatching { cached.connection.close() } + it.remove() + } + } + } + + private fun engineError(e: DebugEngineException) = + McpToolError(e.message ?: "SQLite error ${e.sqliteCode}", JSONObject().put("sqlite_code", e.sqliteCode)) + + private fun stringArg( + args: JSONObject, + name: String, + ): String { + val v = args.opt(name) + if (v !is String || v.isEmpty()) throw McpToolError("$name is required and must be a non-empty string.") + return v + } + + private fun intArg( + args: JSONObject, + name: String, + default: Long, + min: Long, + max: Long, + ): Long { + val v = args.opt(name) + if (v == null || v == JSONObject.NULL) return default + val n = v as? Number ?: throw McpToolError("$name must be an integer.") + // org.json parses a decimal as BigDecimal, not Double + if (n !is Int && n !is Long && n !is Short && n !is Byte) { + val d = n.toDouble() + if (d != Math.floor(d) || d.isInfinite() || Math.abs(d) >= 9.2e18) throw McpToolError("$name must be an integer.") + } + val l = n.toLong() + val upper = if (max == Long.MAX_VALUE) "2^63-1" else max.toString() + if (l < min || l > max) throw McpToolError("$name must be between $min and $upper.") + return l + } + + private fun bindArgs(args: JSONObject): List { + val raw = args.opt("args") + if (raw == null || raw == JSONObject.NULL) return emptyList() + val arr = raw as? JSONArray ?: throw McpToolError("args must be an array.") + return (0 until arr.length()).map { i -> + when (val v = arr.opt(i)) { + null, JSONObject.NULL -> null + is Int, is Long, is Short -> (v as Number).toLong() + is Number -> v.toDouble() + is String -> v + is Boolean -> if (v) 1L else 0L + else -> throw McpToolError("args[$i] must be null, a number or a string.") + } + } + } +} diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt new file mode 100644 index 0000000000..dc7db46d46 --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpHttp.kt @@ -0,0 +1,533 @@ +package org.appdevforall.sqlitedebugger.mcp + +import org.json.JSONObject +import java.io.BufferedInputStream +import java.io.ByteArrayOutputStream +import java.io.IOException +import java.io.InputStream +import java.io.OutputStream +import java.net.InetAddress +import java.net.InetSocketAddress +import java.net.ServerSocket +import java.net.Socket +import java.net.SocketTimeoutException +import java.net.URI +import java.security.MessageDigest +import java.util.Collections +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import kotlin.concurrent.thread + +/** + * The MCP endpoint's HTTP side: a small HTTP/1.1 listener on 127.0.0.1 only, so it's never + * reachable from another device. Before [McpProtocol] sees a request, + * it checks who's asking: + * - the peer is a loopback address (it can't be otherwise, but it's checked anyway); + * - Host names 127.0.0.1 or localhost, and Origin, if sent, does too, so a web page can't reach it + * by DNS rebinding; + * - Authorization: Bearer , the token from [newToken], compared in constant time. + * + * POST /mcp carries JSON-RPC; DELETE /mcp ends a session; GET gets 405, since this server never + * opens an event stream. Each connection has its own thread, as a tool may wait a while; at most + * [MAX_CONNECTIONS] at once. Plain Kotlin, so unit tests can run it without Android. + */ +class McpHttp( + private val protocol: McpProtocol, + /** The current token: read on every request, so a new one takes effect at once. */ + private val token: () -> String, + /** 0 lets the system pick a free port; read it from [boundPort] after [start]. */ + private val port: Int = 0, + /** Problems worth a line in the log. Never given the token. */ + private val log: (String) -> Unit = {}, + /** Creates the listening socket; tests pass one whose accept fails. */ + private val socketFactory: () -> ServerSocket = { ServerSocket() }, + /** How long a request's head may take to arrive; see [HEAD_DEADLINE_MS]. Tests shorten it. */ + private val headDeadlineMs: Int = HEAD_DEADLINE_MS, +) { + companion object { + private const val B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + + /** 32 random bytes, URL-safe base64 without padding. A new one on every process start. */ + fun newToken(): String = base64(ByteArray(32).also { java.security.SecureRandom().nextBytes(it) }, urlSafe = true, pad = false) + + /** By hand: java.util.Base64 needs API 26 and this library's minSdk is 21. */ + fun base64( + bytes: ByteArray, + urlSafe: Boolean, + pad: Boolean, + ): String { + val out = StringBuilder() + var i = 0 + while (i < bytes.size) { + val n = minOf(3, bytes.size - i) + var v = 0 + for (k in 0 until 3) v = (v shl 8) or (if (k < n) bytes[i + k].toInt() and 0xff else 0) + for (k in 0..n) { + val idx = (v shr (18 - 6 * k)) and 63 + out.append( + if (idx < 62) { + B64[idx] + } else if (idx == 62) { + (if (urlSafe) '-' else '+') + } else { + (if (urlSafe) '_' else '/') + }, + ) + } + if (pad) repeat(3 - n) { out.append('=') } + i += 3 + } + return out.toString() + } + + const val PATH = "/mcp" + const val MAX_CONNECTIONS = 8 + private const val MAX_HEADER_BYTES = 16 * 1024 + const val MAX_BODY_BYTES = 1024 * 1024 + + /** A connection with no request for this long is closed. */ + const val IDLE_TIMEOUT_MS = 30_000 + + /** + * The whole head of a request (request line and headers) must arrive within this, counted from + * the connection for its first request and from the first byte for later ones, so a client + * that trickles bytes or never authenticates frees its slot. + */ + const val HEAD_DEADLINE_MS = 5_000 + + /** After a failed accept the loop sleeps this long, doubling up to the maximum while failures continue. */ + const val ACCEPT_BACKOFF_MIN_MS = 100L + const val ACCEPT_BACKOFF_MAX_MS = 2_000L + private val STATUS = + mapOf( + 200 to "OK", + 202 to "Accepted", + 204 to "No Content", + 400 to "Bad Request", + 401 to "Unauthorized", + 403 to "Forbidden", + 404 to "Not Found", + 405 to "Method Not Allowed", + 413 to "Payload Too Large", + 415 to "Unsupported Media Type", + 431 to "Request Header Fields Too Large", + 500 to "Internal Server Error", + 501 to "Not Implemented", + 503 to "Service Unavailable", + ) + private val LOOPBACK_NAMES = setOf("127.0.0.1", "localhost") + } + + private class Request( + val method: String, + val target: String, + val version: String, + val headers: Map, + var body: ByteArray = ByteArray(0), + ) + + private class HttpError( + val status: Int, + message: String, + ) : Exception(message) + + @Volatile private var socket: ServerSocket? = null + + @Volatile private var running = false + private val open = AtomicInteger(0) + private val clients: MutableSet = Collections.newSetFromMap(ConcurrentHashMap()) + + /** The port actually bound (useful when [port] is 0, as in tests). */ + val boundPort: Int get() = socket?.localPort ?: port + + /** Binds 127.0.0.1:[port]. Throws IOException with a readable message if the port is taken. */ + fun start() { + val s = socketFactory() + try { + s.reuseAddress = true + s.bind(InetSocketAddress(InetAddress.getByAddress(byteArrayOf(127, 0, 0, 1)), port), 16) + } catch (e: IOException) { + try { + s.close() + } catch (ignored: IOException) { + } + throw IOException("Can't listen on 127.0.0.1:$port: ${e.message}", e) + } + socket = s + running = true + thread(name = "mcp-accept", isDaemon = true) { acceptLoop(s) } + } + + /** Stops listening and closes every connection. */ + fun stop() { + running = false + try { + socket?.close() + } catch (ignored: IOException) { + } + socket = null + clients.forEach { close(it) } + clients.clear() + } + + private fun acceptLoop(s: ServerSocket) { + var backoffMs = 0L + while (running && !s.isClosed) { + val c = + try { + s.accept().also { backoffMs = 0 } + } catch (e: IOException) { + if (running && !s.isClosed) { + log("couldn't accept a connection: ${e.message}") + // A persistent failure (EMFILE) would otherwise spin; keep trying, slowly. + backoffMs = if (backoffMs == 0L) ACCEPT_BACKOFF_MIN_MS else minOf(backoffMs * 2, ACCEPT_BACKOFF_MAX_MS) + sleepQuietly(backoffMs) + } + continue + } catch (t: Throwable) { + log("accept failed: ${t.javaClass.simpleName}: ${t.message}") + backoffMs = if (backoffMs == 0L) ACCEPT_BACKOFF_MIN_MS else minOf(backoffMs * 2, ACCEPT_BACKOFF_MAX_MS) + sleepQuietly(backoffMs) + continue + } + if (!c.inetAddress.isLoopbackAddress) { + close(c) + continue + } + if (open.incrementAndGet() > MAX_CONNECTIONS) { + open.decrementAndGet() + try { + c.soTimeout = 1000 + respond(c.getOutputStream(), 503, rpcError("Too many connections at once; try again."), emptyMap(), close = true) + } catch (ignored: IOException) { + } + close(c) + continue + } + clients += c + thread(name = "mcp-connection", isDaemon = true) { + try { + serve(c) + } catch (ignored: IOException) { + // The client went away. + } catch (t: Throwable) { + // An Error here would kill the host app: log it and drop the connection. + log("a connection failed: ${t.javaClass.simpleName}: ${t.message}") + } finally { + clients -= c + close(c) + open.decrementAndGet() + } + } + } + } + + private fun serve(c: Socket) { + c.soTimeout = IDLE_TIMEOUT_MS + c.tcpNoDelay = true + val input = BufferedInputStream(c.getInputStream()) + val out = c.getOutputStream() + var headStart = System.currentTimeMillis() + var first = true + while (running) { + val req = + try { + if (!first) { + // Between requests the client may idle; the head's clock starts at its first byte. + c.soTimeout = IDLE_TIMEOUT_MS + input.mark(1) + if (input.read() < 0) return + input.reset() + headStart = System.currentTimeMillis() + } + first = false + val head = readHead(input, c, headStart + headDeadlineMs) ?: return + // Refuse before reading a body, so an unauthenticated client never gets that far. + if (!allowed(head, out)) return + c.soTimeout = IDLE_TIMEOUT_MS + head.body = readBody(input, head.headers) + head + } catch (e: SocketTimeoutException) { + return + } catch (e: HttpError) { + respond(out, e.status, rpcError(e.message ?: "Bad request."), emptyMap(), close = true) + return + } + if (!handle(req, out)) return + } + } + + /** Host, Origin, path and token. Answers and returns false when the request is refused; the connection then closes. */ + private fun allowed( + r: Request, + out: OutputStream, + ): Boolean { + fun refuse( + status: Int, + message: String, + headers: Map = emptyMap(), + ): Boolean { + respond(out, status, rpcError(message), headers, close = true) + return false + } + if (!hostAllowed(r.headers["host"])) return refuse(403, "The Host header must be 127.0.0.1 or localhost.") + val origin = r.headers["origin"] + if (origin != null && !originAllowed(origin)) return refuse(403, "Requests from web pages aren't accepted.") + if (r.target.substringBefore('?') != PATH) return refuse(404, "Not found: the MCP endpoint is $PATH.") + if (!authorized(r.headers["authorization"])) { + return refuse( + 401, + "Missing or wrong token. Code on the Go gives it to its Agent; " + + "send it as Authorization: Bearer .", + mapOf("WWW-Authenticate" to "Bearer realm=\"SQLite debugger\""), + ) + } + return true + } + + /** Answers one request. Returns whether the connection stays open for another. */ + private fun handle( + r: Request, + out: OutputStream, + ): Boolean { + val close = r.version == "HTTP/1.0" || r.headers["connection"]?.lowercase()?.contains("close") == true + + fun send( + status: Int, + body: String?, + headers: Map = emptyMap(), + ): Boolean { + respond(out, status, body, headers, close) + return !close + } + val session = r.headers["mcp-session-id"] + val reply = + when (r.method) { + "POST" -> { + val type = + r.headers["content-type"] + ?.substringBefore(';') + ?.trim() + ?.lowercase() + if (type != null && type != "application/json") return send(415, rpcError("Send the request as application/json.")) + try { + protocol.post(String(r.body, Charsets.UTF_8), session, r.headers["mcp-protocol-version"]) + } catch (t: Throwable) { + log("a request failed: ${t.javaClass.simpleName}: ${t.message}") + McpProtocol.Reply(500, rpcError("The server failed: ${t.message ?: t.javaClass.simpleName}")) + } + } + + "DELETE" -> { + protocol.delete(session) + } + + "GET" -> { + return send(405, rpcError("This server doesn't offer an event stream: use POST."), mapOf("Allow" to "POST, DELETE")) + } + + else -> { + return send(405, rpcError("Use POST."), mapOf("Allow" to "POST, DELETE")) + } + } + return send(reply.status, reply.body, reply.headers) + } + + // ---------------------------------------------------------------- checks + + /** 127.0.0.1 or localhost, on any port: adb forward may bring the request in from another port. */ + private fun hostAllowed(host: String?): Boolean { + val h = host?.trim()?.lowercase() ?: return false + val name = if (h.count { it == ':' } == 1) h.substringBefore(':') else h + if (h.count { it == ':' } == 1 && h.substringAfter(':').let { it.isEmpty() || it.any { ch -> !ch.isDigit() } }) return false + return name in LOOPBACK_NAMES + } + + private fun originAllowed(origin: String): Boolean = + try { + val u = URI(origin.trim()) + (u.scheme == "http" || u.scheme == "https") && u.host?.lowercase() in LOOPBACK_NAMES + } catch (e: Exception) { + false + } + + private fun authorized(header: String?): Boolean { + val expected = token() + if (header == null || expected.isEmpty()) return false + val h = header.trim() + if (!h.regionMatches(0, "Bearer ", 0, 7, ignoreCase = true)) return false + return MessageDigest.isEqual(h.substring(7).trim().toByteArray(Charsets.UTF_8), expected.toByteArray(Charsets.UTF_8)) + } + + // ---------------------------------------------------------------- reading and writing HTTP + + /** The head of one request (no body yet), or null if the connection closed before one began. */ + private fun readHead( + input: InputStream, + c: Socket, + deadlineAt: Long, + ): Request? { + var budget = MAX_HEADER_BYTES + + fun read(): Int { + // Each blocking read may wait only as long as the head has left, however slowly bytes come. + if (input.available() == 0) { + val left = deadlineAt - System.currentTimeMillis() + if (left <= 0) throw SocketTimeoutException("the request's head took too long") + c.soTimeout = minOf(left, IDLE_TIMEOUT_MS.toLong()).toInt() + } + return input.read() + } + + fun line(): String? { + val b = ByteArrayOutputStream() + while (true) { + val ch = read() + if (ch < 0) { + if (b.size() == 0) return null else throw HttpError(400, "The request ended early.") + } + if (--budget < 0) throw HttpError(431, "The request's headers are too long.") + if (ch == '\n'.code) break + b.write(ch) + } + return b.toString("ISO-8859-1").removeSuffix("\r") + } + var first = line() ?: return null + while (first.isEmpty()) first = line() ?: return null // stray line ends between requests + val parts = first.split(' ') + if (parts.size != 3 || !parts[2].startsWith("HTTP/1.")) throw HttpError(400, "That isn't an HTTP/1.x request.") + val headers = HashMap() + while (true) { + val l = line() ?: throw HttpError(400, "The request ended early.") + if (l.isEmpty()) break + val colon = l.indexOf(':') + if (colon <= 0) throw HttpError(400, "A header line without a colon.") + val name = l.substring(0, colon).trim().lowercase() + val value = l.substring(colon + 1).trim() + // A repeated header is joined, so two Host headers fail the Host check. + headers[name] = headers[name]?.let { "$it, $value" } ?: value + } + return Request(parts[0], parts[1], parts[2], headers) + } + + private fun readBody( + input: InputStream, + h: Map, + ): ByteArray { + val te = h["transfer-encoding"] + if (te != null) { + if (!te.equals("chunked", ignoreCase = true)) throw HttpError(501, "Transfer-Encoding $te isn't supported.") + if (h.containsKey("content-length")) throw HttpError(400, "Both Content-Length and Transfer-Encoding were sent.") + return readChunked(input) + } + val len = h["content-length"] ?: return ByteArray(0) + val n = len.toLongOrNull()?.takeIf { it >= 0 } ?: throw HttpError(400, "Content-Length isn't a number.") + if (n > MAX_BODY_BYTES) throw HttpError(413, "The request is larger than ${MAX_BODY_BYTES / 1024} KB.") + return readExactly(input, n.toInt()) + } + + private fun readChunked(input: InputStream): ByteArray { + val out = ByteArrayOutputStream() + while (true) { + val sizeLine = shortLine(input, 1024) ?: throw HttpError(400, "The request ended early.") + val size = + sizeLine + .substringBefore(';') + .trim() + .toIntOrNull(16) + ?.takeIf { it >= 0 } ?: throw HttpError(400, "A bad chunk size.") + if (out.size().toLong() + size > MAX_BODY_BYTES) throw HttpError(413, "The request is larger than ${MAX_BODY_BYTES / 1024} KB.") + if (size == 0) { + // Trailers, if any, up to a blank line. + var lines = 0 + while (true) { + val t = shortLine(input, 1024) ?: break + if (t.isEmpty()) break + if (++lines > 32) throw HttpError(431, "Too many trailers.") + } + return out.toByteArray() + } + out.write(readExactly(input, size)) + if (shortLine(input, 2) != "") throw HttpError(400, "A chunk without its line end.") + } + } + + private fun shortLine( + input: InputStream, + max: Int, + ): String? { + val b = ByteArrayOutputStream() + var left = max + while (true) { + val ch = input.read() + if (ch < 0) return if (b.size() == 0) null else throw HttpError(400, "The request ended early.") + if (--left < 0) throw HttpError(400, "A line in the request is too long.") + if (ch == '\n'.code) break + b.write(ch) + } + return b.toString("ISO-8859-1").removeSuffix("\r") + } + + private fun readExactly( + input: InputStream, + n: Int, + ): ByteArray { + val b = ByteArray(n) + var off = 0 + while (off < n) { + val r = input.read(b, off, n - off) + if (r < 0) throw HttpError(400, "The request ended early.") + off += r + } + return b + } + + private fun respond( + out: OutputStream, + status: Int, + body: String?, + headers: Map, + close: Boolean, + ) { + val bytes = body?.toByteArray(Charsets.UTF_8) ?: ByteArray(0) + val head = StringBuilder("HTTP/1.1 $status ${STATUS[status] ?: "Status"}\r\n") + if (body != null) head.append("Content-Type: application/json; charset=utf-8\r\n") + if (status != 204) head.append("Content-Length: ${bytes.size}\r\n") + head.append("Cache-Control: no-store\r\n") + if (close) head.append("Connection: close\r\n") + for ((k, v) in headers) { + head + .append(k) + .append(": ") + .append(v.replace("\r", "").replace("\n", "")) + .append("\r\n") + } + head.append("\r\n") + out.write(head.toString().toByteArray(Charsets.ISO_8859_1)) + if (status != 204) out.write(bytes) + out.flush() + } + + /** An HTTP-level refusal, as a JSON-RPC error with no id, which MCP clients know how to show. */ + private fun rpcError(message: String): String = + JSONObject() + .put("jsonrpc", "2.0") + .put("id", JSONObject.NULL) + .put("error", JSONObject().put("code", McpProtocol.INVALID_REQUEST).put("message", message)) + .toString() + + private fun sleepQuietly(ms: Long) { + try { + Thread.sleep(ms) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + } + } + + private fun close(s: Socket) { + try { + s.close() + } catch (ignored: IOException) { + } + } +} diff --git a/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt new file mode 100644 index 0000000000..34cf7b122b --- /dev/null +++ b/subprojects/sqlite-debugger/src/main/java/org/appdevforall/sqlitedebugger/mcp/McpProtocol.kt @@ -0,0 +1,232 @@ +package org.appdevforall.sqlitedebugger.mcp + +import org.json.JSONArray +import org.json.JSONObject +import org.json.JSONTokener + +/** + * A tool an MCP client can call. [run] gets the call's arguments (an empty object when none were + * sent) and returns what happened, as JSON; it throws [McpToolError] for a failure the client + * should read, such as "a person is needed". + */ +class McpTool( + val name: String, + val title: String, + val description: String, + val inputSchema: JSONObject = JSONObject().put("type", "object").put("properties", JSONObject()).put("additionalProperties", false), + /** Only reports; changes nothing. Shown to clients as a hint, not relied on. */ + val readOnly: Boolean = true, + val run: (JSONObject) -> JSONObject, +) { + fun describe(): JSONObject = + JSONObject() + .put("name", name) + .put("title", title) + .put("description", description) + .put("inputSchema", inputSchema) + .put( + "annotations", + JSONObject() + .put("title", title) + .put("readOnlyHint", readOnly) + .put("destructiveHint", false) + .put("openWorldHint", false), + ) +} + +/** A tool's failure, told to the client as a tool result with isError, with [details] when there are any. */ +class McpToolError( + message: String, + val details: JSONObject? = null, +) : Exception(message) + +/** + * The Model Context Protocol, as the SQLite debugger speaks it: JSON-RPC 2.0 over the Streamable HTTP + * transport, answering every POST with plain JSON, never an event stream. Plain Kotlin and org.json, + * so unit tests run it without Android. [McpHttp] does the sockets and checks who's asking; this + * decides what to answer, and keeps the sessions. + * + * Methods: initialize, ping, tools/list, tools/call. Notifications, and responses a client sends, + * are accepted and need no answer (HTTP 202). + */ +class McpProtocol( + private val tools: List, + private val serverVersion: String, + private val newSessionId: () -> String, + private val maxSessions: Int = 16, +) { + companion object { + /** Newest first: offered when a client asks for a version this server doesn't know. */ + val VERSIONS = listOf("2025-06-18", "2025-03-26", "2024-11-05") + const val SERVER_NAME = "sqlite-debugger" + const val SESSION_HEADER = "Mcp-Session-Id" + const val PARSE_ERROR = -32700 + const val INVALID_REQUEST = -32600 + const val METHOD_NOT_FOUND = -32601 + const val INVALID_PARAMS = -32602 + const val INSTRUCTIONS = + "Inspects and queries the SQLite databases of the app this server runs in, a debug " + + "build made with Code on the Go. Every tool is read-only." + } + + /** What to send back: an HTTP status, a JSON body (or none), and extra headers. */ + class Reply( + val status: Int, + val body: String?, + val headers: Map = emptyMap(), + ) + + /** Live sessions, oldest first; the oldest is dropped when there are too many. */ + private val sessions = LinkedHashSet() + + /** A POST to the endpoint: one JSON-RPC message, or a batch of them. */ + fun post( + body: String, + sessionId: String?, + versionHeader: String?, + ): Reply { + val parsed = + try { + JSONTokener(body).nextValue() + } catch (e: Exception) { + null + } + if (parsed !is JSONObject && parsed !is JSONArray) { + return Reply(400, error(null, PARSE_ERROR, "That isn't a JSON object or array.").toString()) + } + val messages: List = if (parsed is JSONArray) (0 until parsed.length()).map { parsed.opt(it) } else listOf(parsed) + if (messages.isEmpty()) return Reply(400, error(null, INVALID_REQUEST, "An empty batch.").toString()) + + if (messages.any { (it as? JSONObject)?.optString("method") == "initialize" }) { + if (messages.size != 1) { + return Reply(400, error(null, INVALID_REQUEST, "Send initialize on its own, not in a batch.").toString()) + } + return initialize(messages[0] as JSONObject) + } + if (sessionId.isNullOrEmpty()) { + return Reply(400, error(null, INVALID_REQUEST, "No $SESSION_HEADER header: send initialize first.").toString()) + } + if (!touch(sessionId)) { + return Reply(404, error(null, INVALID_REQUEST, "That session has ended or doesn't exist: send initialize again.").toString()) + } + if (versionHeader != null && versionHeader !in VERSIONS) { + val detail = "Unsupported MCP-Protocol-Version $versionHeader; this server speaks ${VERSIONS.joinToString()}." + return Reply(400, error(null, INVALID_REQUEST, detail).toString()) + } + val replies = messages.mapNotNull { handle(it) } + if (replies.isEmpty()) return Reply(202, null) + return Reply(200, if (parsed is JSONArray) JSONArray(replies).toString() else replies[0].toString()) + } + + /** A DELETE: the client is done with its session. */ + fun delete(sessionId: String?): Reply { + if (sessionId.isNullOrEmpty()) return Reply(400, error(null, INVALID_REQUEST, "No $SESSION_HEADER header.").toString()) + val removed = synchronized(sessions) { sessions.remove(sessionId) } + return if (removed) { + Reply(204, null) + } else { + Reply(404, error(null, INVALID_REQUEST, "That session has ended or doesn't exist.").toString()) + } + } + + /** Ends every session, so clients must initialize again. */ + fun endAll() = synchronized(sessions) { sessions.clear() } + + fun sessionCount(): Int = synchronized(sessions) { sessions.size } + + private fun touch(id: String): Boolean = + synchronized(sessions) { + if (!sessions.remove(id)) return false + sessions += id // most recently used goes last, so it's dropped last + true + } + + private fun initialize(m: JSONObject): Reply { + val id = m.opt("id") + if (!validId(id)) return Reply(400, error(null, INVALID_REQUEST, "initialize needs an id, a string or a number.").toString()) + if (m.optString("jsonrpc") != "2.0") return Reply(400, error(id, INVALID_REQUEST, "jsonrpc must be \"2.0\".").toString()) + val asked = m.optJSONObject("params")?.optString("protocolVersion", "") ?: "" + val version = if (asked in VERSIONS) asked else VERSIONS[0] + val session = newSessionId() + synchronized(sessions) { + sessions += session + while (sessions.size > maxSessions) sessions.remove(sessions.first()) + } + val result = + JSONObject() + .put("protocolVersion", version) + .put("capabilities", JSONObject().put("tools", JSONObject().put("listChanged", false))) + .put("serverInfo", JSONObject().put("name", SERVER_NAME).put("title", "SQLite debugger").put("version", serverVersion)) + .put("instructions", INSTRUCTIONS) + return Reply(200, ok(id, result).toString(), mapOf(SESSION_HEADER to session)) + } + + /** One message of a POST; null when it needs no answer. */ + private fun handle(raw: Any?): JSONObject? { + val m = raw as? JSONObject ?: return error(null, INVALID_REQUEST, "Each message must be a JSON object.") + val hasId = m.has("id") + val id = m.opt("id") + val method = m.opt("method") as? String + if (method == null) { + // A response to something this server asked: it never asks, so there's nothing to do. + if (m.has("result") || m.has("error")) return null + return error(if (validId(id)) id else null, INVALID_REQUEST, "A request needs a method.") + } + if (!hasId) return null // a notification, such as notifications/initialized: no answer + if (!validId(id)) return error(null, INVALID_REQUEST, "id must be a string or a number.") + if (m.optString("jsonrpc") != "2.0") return error(id, INVALID_REQUEST, "jsonrpc must be \"2.0\".") + return when (method) { + "ping" -> ok(id, JSONObject()) + "tools/list" -> ok(id, JSONObject().put("tools", JSONArray().also { a -> tools.forEach { a.put(it.describe()) } })) + "tools/call" -> call(id, m.opt("params")) + else -> error(id, METHOD_NOT_FOUND, "Unknown method \"$method\".") + } + } + + private fun call( + id: Any, + rawParams: Any?, + ): JSONObject { + val params = rawParams as? JSONObject ?: return error(id, INVALID_PARAMS, "tools/call needs params, with the tool's name.") + val name = params.opt("name") as? String ?: return error(id, INVALID_PARAMS, "tools/call needs the tool's name.") + val tool = tools.firstOrNull { it.name == name } ?: return error(id, INVALID_PARAMS, "Unknown tool \"$name\".") + val args = + when (val a = params.opt("arguments")) { + null, JSONObject.NULL -> JSONObject() + is JSONObject -> a + else -> return error(id, INVALID_PARAMS, "arguments must be an object.") + } + val result = + try { + val out = tool.run(args) + JSONObject().put("content", JSONArray().put(text(out.toString()))).put("structuredContent", out).put("isError", false) + } catch (e: McpToolError) { + JSONObject().put("content", JSONArray().put(text(e.message ?: "The tool failed."))).put("isError", true).also { r -> + e.details?.let { r.put("structuredContent", it) } + } + } catch (e: Exception) { + JSONObject() + .put("content", JSONArray().put(text("The tool failed: ${e.message ?: e.javaClass.simpleName}"))) + .put("isError", true) + } + return ok(id, result) + } + + private fun validId(id: Any?) = id is String || id is Number + + private fun text(t: String) = JSONObject().put("type", "text").put("text", t) + + private fun ok( + id: Any, + result: JSONObject, + ) = JSONObject().put("jsonrpc", "2.0").put("id", id).put("result", result) + + private fun error( + id: Any?, + code: Int, + message: String, + ) = JSONObject() + .put("jsonrpc", "2.0") + .put("id", id ?: JSONObject.NULL) + .put("error", JSONObject().put("code", code).put("message", message)) +} diff --git a/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c b/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c new file mode 100644 index 0000000000..298fdd592e --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/c/sqlitedbg_test.c @@ -0,0 +1,1015 @@ +// Host tests of the SQLite debugger's extension (ADFA-7135), run by test.sh against SQLite built +// from the amalgamation as requery builds it (SQLITE_THREADSAFE=2). The extension keeps process +// state and installs once, so the tests share one install and run in order. +// +// Usage: sqlitedbg_test [--write-fixture BIN TSV | --check-fixture BIN TSV] +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sqlite3.h" +#include "sqlitedbg.h" + +static int g_failures; +static int g_checks; + +#define CHECK(cond) \ + do { \ + g_checks++; \ + if (!(cond)) { \ + g_failures++; \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + } \ + } while (0) + +#define CHECK_EQ(a, b) \ + do { \ + long long va_ = (long long)(a), vb_ = (long long)(b); \ + g_checks++; \ + if (va_ != vb_) { \ + g_failures++; \ + fprintf(stderr, "FAIL %s:%d: %s == %s (%lld vs %lld)\n", __FILE__, __LINE__, \ + #a, #b, va_, vb_); \ + } \ + } while (0) + +#define MAX_SQL 64 +#define BUFFER_BYTES 8192 + +static char g_dir[256]; +static char g_db_path[300]; + +static long long tid(void) { return (long long)syscall(SYS_gettid); } + +static void exec_ok(sqlite3 *db, const char *sql) { + char *err = NULL; + int rc = sqlite3_exec(db, sql, NULL, NULL, &err); + if (rc != SQLITE_OK) fprintf(stderr, "exec(%s): %s\n", sql, err ? err : "?"); + sqlite3_free(err); + CHECK_EQ(rc, SQLITE_OK); +} + +static sqlite3 *open_app(const char *path) { + sqlite3 *db = NULL; + int rc = sqlite3_open(path, &db); + CHECK_EQ(rc, SQLITE_OK); + return db; +} + +// Events + +typedef struct ev { + uint32_t kind, flags; + uint64_t seq, conn; + int64_t tid, time_ns, duration_ns; + char sql[MAX_SQL * 4 + 8]; + uint32_t sql_bytes; +} ev; + +typedef struct batch { + ev *events; + size_t count; + uint64_t next, dropped; +} batch; + +static void batch_free(batch *b) { free(b->events); } + +// Reads everything after `after`, checking the layout of each record on the way. +static void read_all(uint64_t after, batch *b) { + size_t cap = 1 << 20; + unsigned char *buf = malloc(cap); + sqlitedbg_read_result r; + sqlitedbg_read_events(after, buf, cap, &r); + CHECK_EQ(r.bytes % 8, 0); + b->events = calloc(r.count + 1, sizeof(ev)); + b->count = r.count; + b->next = r.next; + b->dropped = r.dropped; + size_t off = 0; + uint64_t prev = after; + for (size_t i = 0; i < r.count; i++) { + const sqlitedbg_event_record *rec = (const sqlitedbg_event_record *)(void *)(buf + off); + CHECK(rec->record_bytes >= sizeof *rec); + CHECK_EQ(rec->record_bytes % 8, 0); + CHECK(rec->sql_bytes <= MAX_SQL); + CHECK(rec->record_bytes >= sizeof *rec + rec->sql_bytes); + CHECK(rec->seq > prev); + prev = rec->seq; + ev *e = &b->events[i]; + e->kind = rec->kind; + e->flags = rec->flags; + e->seq = rec->seq; + e->conn = rec->connection_id; + e->tid = rec->thread_id; + e->time_ns = rec->time_ns; + e->duration_ns = rec->duration_ns; + e->sql_bytes = rec->sql_bytes; + memcpy(e->sql, (const char *)buf + off + sizeof *rec, rec->sql_bytes); + e->sql[rec->sql_bytes] = 0; + off += rec->record_bytes; + } + CHECK_EQ(off, r.bytes); + free(buf); +} + +static uint64_t cursor_now(void) { + batch b; + read_all(0, &b); + uint64_t n = b.next; + batch_free(&b); + return n; +} + +static int utf8_valid(const char *s, size_t n) { + size_t i = 0; + while (i < n) { + unsigned char c = (unsigned char)s[i]; + size_t len = c < 0x80 ? 1 : (c >> 5) == 6 ? 2 : (c >> 4) == 14 ? 3 : (c >> 3) == 30 ? 4 : 0; + if (!len || i + len > n) return 0; + for (size_t k = 1; k < len; k++) + if (((unsigned char)s[i + k] & 0xC0) != 0x80) return 0; + i += len; + } + return 1; +} + +// Connections + +typedef struct listing { + sqlitedbg_connection c[64]; + char path[64][300]; + size_t n; +} listing; + +static void list_cb(void *context, const sqlitedbg_connection *c) { + listing *l = context; + if (l->n < 64) { + l->c[l->n] = *c; + snprintf(l->path[l->n], sizeof l->path[l->n], "%s", c->path); + l->c[l->n].path = l->path[l->n]; + } + l->n++; +} + +static size_t list_conns(listing *l) { + l->n = 0; + return sqlitedbg_each_connection(list_cb, l); +} + +static const sqlitedbg_connection *find_conn(const listing *l, uint64_t id) { + for (size_t i = 0; i < l->n && i < 64; i++) + if (l->c[i].id == id) return &l->c[i]; + return NULL; +} + +static uint64_t conn_id_of(sqlite3 *db) { + sqlite3_stmt *st; + uint64_t id = 0; + CHECK_EQ(sqlite3_prepare_v2(db, "SELECT sqlitedbg_connection_id()", -1, &st, NULL), SQLITE_OK); + if (sqlite3_step(st) == SQLITE_ROW) id = (uint64_t)sqlite3_column_int64(st, 0); + sqlite3_finalize(st); + return id; +} + +// Tests + +extern int sqlitedbg_test_fail_init; // sqlitedbg.c, built with SQLITEDBG_TEST + +static sqlitedbg_config g_config = {BUFFER_BYTES, MAX_SQL}; +static sqlite3 *g_main; + +static void test_install(void) { + sqlite3 *srv = NULL; + CHECK_EQ(sqlitedbg_open_server(g_db_path, 0, &srv), SQLITE_MISUSE); + CHECK(srv == NULL); + CHECK_EQ(sqlitedbg_install(sqlite3_auto_extension, &g_config), SQLITE_OK); + sqlitedbg_config other = {1 << 20, 1000}; + CHECK_EQ(sqlitedbg_install(sqlite3_auto_extension, &other), SQLITE_OK); + CHECK_EQ(sqlitedbg_install(NULL, NULL), SQLITE_MISUSE); + // still no app connection: the extension has no SQLite API to call + CHECK_EQ(sqlitedbg_open_server(g_db_path, 0, &srv), SQLITE_MISUSE); + + listing l; + CHECK_EQ(list_conns(&l), 0); + g_main = open_app(g_db_path); + // a second install registered the entry point once: one connection, one listing, one id + CHECK_EQ(list_conns(&l), 1); + CHECK_EQ(l.c[0].id, 1); + CHECK(strcmp(l.c[0].path, g_db_path) == 0 || strstr(l.c[0].path, "app.db") != NULL); + CHECK_EQ(l.c[0].read_only, 0); + CHECK_EQ(l.c[0].closing, 0); + CHECK_EQ(l.c[0].open_thread_id, tid()); + CHECK(l.c[0].opened_at_ns > 0); + CHECK_EQ(conn_id_of(g_main), 1); + // the second config was ignored: the 64-byte limit still applies + uint64_t cur = cursor_now(); + char sql[200]; + snprintf(sql, sizeof sql, "SELECT '%0100d'", 0); + exec_ok(g_main, sql); + batch b; + read_all(cur, &b); + CHECK_EQ(b.count, 2); + CHECK(b.events[0].flags & SQLITEDBG_EVENT_TRUNCATED); + CHECK_EQ(b.events[0].sql_bytes, MAX_SQL); + batch_free(&b); +} + +static void test_events(void) { + exec_ok(g_main, "CREATE TABLE t(a)"); + exec_ok(g_main, "CREATE TABLE log(a)"); + exec_ok(g_main, "CREATE TRIGGER trg AFTER INSERT ON t BEGIN INSERT INTO log VALUES(new.a); END"); + uint64_t cur = cursor_now(); + + sqlite3_stmt *st; + CHECK_EQ(sqlite3_prepare_v2(g_main, "INSERT INTO t VALUES(?)", -1, &st, NULL), SQLITE_OK); + sqlite3_bind_int(st, 1, 42); + CHECK_EQ(sqlite3_step(st), SQLITE_DONE); + sqlite3_finalize(st); + + batch b; + read_all(cur, &b); + // the insert, the trigger it fires (its header, then its one step), then the insert's profile + CHECK_EQ(b.count, 4); + CHECK_EQ(b.events[0].kind, SQLITEDBG_EVENT_STATEMENT); + CHECK(strcmp(b.events[0].sql, "INSERT INTO t VALUES(?)") == 0); + CHECK_EQ(b.events[0].duration_ns, -1); + CHECK_EQ(b.events[0].flags, 0); + CHECK_EQ(b.events[1].kind, SQLITEDBG_EVENT_STATEMENT); + CHECK(strcmp(b.events[1].sql, "-- TRIGGER trg") == 0); + CHECK_EQ(b.events[2].kind, SQLITEDBG_EVENT_STATEMENT); + CHECK(strcmp(b.events[2].sql, "-- INSERT INTO log VALUES(new.a)") == 0); + CHECK_EQ(b.events[3].kind, SQLITEDBG_EVENT_PROFILE); + CHECK(strcmp(b.events[3].sql, "INSERT INTO t VALUES(?)") == 0); + CHECK(b.events[3].duration_ns >= 0); + for (size_t i = 0; i < b.count; i++) { + CHECK_EQ(b.events[i].conn, 1); + CHECK_EQ(b.events[i].tid, tid()); + CHECK_EQ(b.events[i].seq, cur + 1 + i); + CHECK(b.events[i].time_ns > 0); + } + CHECK_EQ(b.next, cur + 4); + CHECK_EQ(b.dropped, 0); + batch_free(&b); + + // nothing new: the cursor stays + read_all(cur + 4, &b); + CHECK_EQ(b.count, 0); + CHECK_EQ(b.next, cur + 4); + batch_free(&b); + + // one buffer too small for a record returns none and does not split one + sqlitedbg_read_result r; + char tiny[8]; + sqlitedbg_read_events(cur, tiny, sizeof tiny, &r); + CHECK_EQ(r.count, 0); + CHECK_EQ(r.bytes, 0); + CHECK_EQ(r.next, cur); +} + +static void test_expanded(void) { + sqlite3_stmt *st; + uint64_t cur = cursor_now(); + sqlitedbg_set_client_connected(1); + CHECK_EQ(sqlite3_prepare_v2(g_main, "INSERT INTO t VALUES(?)", -1, &st, NULL), SQLITE_OK); + sqlite3_bind_int(st, 1, 7); + CHECK_EQ(sqlite3_step(st), SQLITE_DONE); + sqlite3_finalize(st); + sqlitedbg_set_client_connected(0); + batch b; + read_all(cur, &b); + CHECK_EQ(b.count, 4); + CHECK(strcmp(b.events[0].sql, "INSERT INTO t VALUES(7)") == 0); + CHECK(b.events[0].flags & SQLITEDBG_EVENT_EXPANDED); + // a trigger's text is not expanded: P is the outer statement + CHECK(strncmp(b.events[1].sql, "--", 2) == 0); + CHECK(strncmp(b.events[2].sql, "--", 2) == 0); + CHECK_EQ(b.events[1].flags & SQLITEDBG_EVENT_EXPANDED, 0); + CHECK_EQ(b.events[2].flags & SQLITEDBG_EVENT_EXPANDED, 0); + CHECK(strcmp(b.events[3].sql, "INSERT INTO t VALUES(7)") == 0); + CHECK(b.events[3].flags & SQLITEDBG_EVENT_EXPANDED); + batch_free(&b); + + // app SQL that starts with a comment is not a trigger: it is expanded like any other + cur = cursor_now(); + sqlitedbg_set_client_connected(1); + CHECK_EQ(sqlite3_prepare_v2(g_main, "-- note\nINSERT INTO t VALUES(?)", -1, &st, NULL), SQLITE_OK); + sqlite3_bind_int(st, 1, 9); + CHECK_EQ(sqlite3_step(st), SQLITE_DONE); + sqlite3_finalize(st); + sqlitedbg_set_client_connected(0); + read_all(cur, &b); + CHECK(b.count >= 2); + CHECK(strstr(b.events[0].sql, "INSERT INTO t VALUES(9)") != NULL); + CHECK(b.events[0].flags & SQLITEDBG_EVENT_EXPANDED); + batch_free(&b); + + // after the client leaves, the bound value is not written in again + cur = cursor_now(); + CHECK_EQ(sqlite3_prepare_v2(g_main, "INSERT INTO t VALUES(?)", -1, &st, NULL), SQLITE_OK); + sqlite3_bind_int(st, 1, 8); + CHECK_EQ(sqlite3_step(st), SQLITE_DONE); + sqlite3_finalize(st); + read_all(cur, &b); + CHECK(strcmp(b.events[0].sql, "INSERT INTO t VALUES(?)") == 0); + CHECK_EQ(b.events[0].flags, 0); + batch_free(&b); + + // A long bound value does not outgrow the limit: expansion is cut like any other text + char big[300]; + memset(big, 'x', sizeof big - 1); + big[sizeof big - 1] = 0; + cur = cursor_now(); + sqlitedbg_set_client_connected(1); + CHECK_EQ(sqlite3_prepare_v2(g_main, "INSERT INTO t VALUES(?)", -1, &st, NULL), SQLITE_OK); + sqlite3_bind_text(st, 1, big, -1, SQLITE_STATIC); + CHECK_EQ(sqlite3_step(st), SQLITE_DONE); + sqlite3_finalize(st); + sqlitedbg_set_client_connected(0); + read_all(cur, &b); + CHECK(b.events[0].flags & SQLITEDBG_EVENT_EXPANDED); + CHECK(b.events[0].flags & SQLITEDBG_EVENT_TRUNCATED); + CHECK_EQ(b.events[0].sql_bytes, MAX_SQL); + batch_free(&b); +} + +static void test_truncation(void) { + char sql[256]; + batch b; + uint64_t cur; + + // "SELECT '" is 8 bytes; each euro sign is 3, so byte 64 is the third byte of one + // that starts at 62: the cut must fall back to 62 + size_t n = (size_t)snprintf(sql, sizeof sql, "SELECT '"); + for (int i = 0; i < 25; i++) n += (size_t)snprintf(sql + n, sizeof sql - n, "\xE2\x82\xAC"); + snprintf(sql + n, sizeof sql - n, "'"); + cur = cursor_now(); + exec_ok(g_main, sql); + read_all(cur, &b); + CHECK_EQ(b.count, 2); + for (int i = 0; i < 2; i++) { + CHECK_EQ(b.events[i].sql_bytes, 62); + CHECK(b.events[i].flags & SQLITEDBG_EVENT_TRUNCATED); + CHECK(utf8_valid(b.events[i].sql, b.events[i].sql_bytes)); + } + batch_free(&b); + + // a 4-byte character straddling the limit: 11 bytes of prefix, then 4-byte characters, + // so byte 64 is the second byte of one that starts at 63 + n = (size_t)snprintf(sql, sizeof sql, "SELECT 1 /*"); + for (int i = 0; i < 20; i++) n += (size_t)snprintf(sql + n, sizeof sql - n, "\xF0\x9F\x98\x80"); + snprintf(sql + n, sizeof sql - n, "*/"); + cur = cursor_now(); + exec_ok(g_main, sql); + read_all(cur, &b); + CHECK_EQ(b.count, 2); + CHECK_EQ(b.events[0].sql_bytes, 63); + CHECK(utf8_valid(b.events[0].sql, b.events[0].sql_bytes)); + batch_free(&b); + + // exactly at the limit is whole, one byte over is cut + for (int over = 0; over < 2; over++) { + memset(sql, 0, sizeof sql); + n = (size_t)snprintf(sql, sizeof sql, "SELECT 1 /*"); + while (n < (size_t)(MAX_SQL - 2 + over)) sql[n++] = 'x'; + sql[n++] = '*'; + sql[n++] = '/'; + sql[n] = 0; + CHECK_EQ(strlen(sql), MAX_SQL + over); + cur = cursor_now(); + exec_ok(g_main, sql); + read_all(cur, &b); + CHECK_EQ(b.events[0].sql_bytes, MAX_SQL); + CHECK_EQ((b.events[0].flags & SQLITEDBG_EVENT_TRUNCATED) != 0, over); + batch_free(&b); + } +} + +static void test_burst(void) { + uint64_t cur = cursor_now(); + const int stmts = 2000; + for (int i = 0; i < stmts; i++) exec_ok(g_main, "SELECT 1"); + const uint64_t made = 2 * (uint64_t)stmts; + batch b; + read_all(cur, &b); + CHECK(b.count > 0); + CHECK(b.count < made); + CHECK_EQ(b.dropped, made - b.count); + CHECK_EQ(b.next, cur + made); + CHECK_EQ(b.events[0].seq, cur + made - b.count + 1); + CHECK_EQ(b.events[b.count - 1].seq, cur + made); + for (size_t i = 1; i < b.count; i++) CHECK_EQ(b.events[i].seq, b.events[i - 1].seq + 1); + const uint64_t first_seq = b.events[0].seq; + const size_t stored = b.count; + batch_free(&b); + + // drops are reported once even when nothing is copied: a record too big for out stays + // unpassed, and the cursor moves up to it + { + sqlitedbg_read_result r; + char tiny[8]; + sqlitedbg_read_events(cur, tiny, sizeof tiny, &r); + CHECK_EQ(r.count, 0); + CHECK_EQ(r.dropped, made - stored); + CHECK_EQ(r.next, first_seq - 1); + sqlitedbg_read_result again; + sqlitedbg_read_events(r.next, tiny, sizeof tiny, &again); + CHECK_EQ(again.dropped, 0); + CHECK_EQ(again.next, r.next); + } + + // nothing is reported twice + uint64_t next = cur + made; + read_all(next, &b); + CHECK_EQ(b.count, 0); + CHECK_EQ(b.dropped, 0); + batch_free(&b); + + // a cursor from before the start counts everything that left + read_all(0, &b); + CHECK_EQ(b.dropped, b.events[0].seq - 1); + CHECK_EQ(b.dropped + b.count, next); + batch_free(&b); + + // a reader that kept up loses nothing: read after every 10 statements + uint64_t pos = next, lost = 0, got = 0; + for (int round = 0; round < 50; round++) { + for (int i = 0; i < 10; i++) exec_ok(g_main, "SELECT 2"); + read_all(pos, &b); + lost += b.dropped; + got += b.count; + pos = b.next; + batch_free(&b); + } + CHECK_EQ(lost, 0); + CHECK_EQ(got, 50 * 10 * 2); +} + +static void test_close_and_list(void) { + listing l; + size_t before = list_conns(&l); + sqlite3 *c = open_app(g_db_path); + uint64_t id = conn_id_of(c); + CHECK(id > 1); + CHECK_EQ(list_conns(&l), before + 1); + CHECK(find_conn(&l, id) != NULL); + CHECK_EQ(find_conn(&l, id)->closing, 0); + + sqlite3_stmt *st; + CHECK_EQ(sqlite3_prepare_v2(c, "SELECT * FROM t", -1, &st, NULL), SQLITE_OK); + CHECK_EQ(sqlite3_step(st), SQLITE_ROW); + uint64_t cur = cursor_now(); + CHECK_EQ(sqlite3_close(c), SQLITE_BUSY); + // the close event fired, but the connection is still there: listed, marked closing + CHECK_EQ(list_conns(&l), before + 1); + CHECK_EQ(find_conn(&l, id)->closing, 1); + batch b; + read_all(cur, &b); + CHECK_EQ(b.count, 1); + CHECK_EQ(b.events[0].kind, SQLITEDBG_EVENT_CLOSE); + CHECK_EQ(b.events[0].conn, id); + CHECK_EQ(b.events[0].sql_bytes, 0); + batch_free(&b); + + // the app retries once the statement is gone + sqlite3_finalize(st); + CHECK_EQ(list_conns(&l), before + 1); + CHECK_EQ(sqlite3_close(c), SQLITE_OK); + CHECK_EQ(list_conns(&l), before); + CHECK(find_conn(&l, id) == NULL); + + // close_v2 with a statement open leaves a zombie, listed until it is finalized + c = open_app(g_db_path); + uint64_t id2 = conn_id_of(c); + CHECK(id2 > id); + CHECK_EQ(sqlite3_prepare_v2(c, "SELECT * FROM t", -1, &st, NULL), SQLITE_OK); + CHECK_EQ(sqlite3_close_v2(c), SQLITE_OK); + CHECK_EQ(list_conns(&l), before + 1); + CHECK_EQ(find_conn(&l, id2)->closing, 1); + sqlite3_finalize(st); + CHECK_EQ(list_conns(&l), before); + + // ids are never reused + c = open_app(g_db_path); + CHECK(conn_id_of(c) > id2); + CHECK_EQ(sqlite3_close(c), SQLITE_OK); + + // an in-memory database lists with an empty path, a read-only one as read-only + c = open_app(":memory:"); + list_conns(&l); + CHECK(find_conn(&l, conn_id_of(c)) != NULL); + CHECK_EQ(find_conn(&l, conn_id_of(c))->path[0], 0); + CHECK_EQ(sqlite3_close(c), SQLITE_OK); + sqlite3 *ro = NULL; + CHECK_EQ(sqlite3_open_v2(g_db_path, &ro, SQLITE_OPEN_READONLY, NULL), SQLITE_OK); + list_conns(&l); + CHECK_EQ(find_conn(&l, conn_id_of(ro))->read_only, 1); + CHECK_EQ(sqlite3_close(ro), SQLITE_OK); + CHECK_EQ(list_conns(&l), before); +} + +static void test_init_failure(void) { + listing l; + size_t before = list_conns(&l); + for (int mode = 1; mode <= 2; mode++) { + sqlitedbg_test_fail_init = mode; + sqlite3 *db = NULL; + // the app's open must not fail because the debugger could not track the connection + CHECK_EQ(sqlite3_open(":memory:", &db), SQLITE_OK); + sqlitedbg_test_fail_init = 0; + CHECK(db != NULL); + if (!db) continue; + CHECK_EQ(list_conns(&l), before); + exec_ok(db, "CREATE TABLE u(a)"); + CHECK_EQ(sqlite3_close(db), SQLITE_OK); + } + CHECK_EQ(list_conns(&l), before); +} + +static int step_all(sqlite3 *db, const char *sql) { + sqlite3_stmt *st; + int rc = sqlite3_prepare_v2(db, sql, -1, &st, NULL); + if (rc != SQLITE_OK) return rc; + while ((rc = sqlite3_step(st)) == SQLITE_ROW) {} + sqlite3_finalize(st); + return rc; +} + +static void test_server(void) { + listing l; + size_t before = list_conns(&l); + uint64_t cur = cursor_now(); + sqlite3 *srv = NULL; + + CHECK_EQ(sqlitedbg_open_server(g_db_path, 0, &srv), SQLITE_OK); + CHECK(srv != NULL); + CHECK_EQ(sqlite3_db_readonly(srv, "main"), 1); + // not listed, and it has no connection id function + CHECK_EQ(list_conns(&l), before); + CHECK(sqlite3_prepare_v2(srv, "SELECT sqlitedbg_connection_id()", -1, &(sqlite3_stmt *){NULL}, NULL) != SQLITE_OK); + + CHECK_EQ(step_all(srv, "SELECT 'server-marker', count(*) FROM t"), SQLITE_DONE); + CHECK_EQ(step_all(srv, "SELECT name FROM sqlite_schema"), SQLITE_DONE); + CHECK_EQ(step_all(srv, "PRAGMA table_info(t)"), SQLITE_DONE); + CHECK_EQ(step_all(srv, "PRAGMA quick_check"), SQLITE_DONE); + // not traced + batch b; + read_all(cur, &b); + CHECK_EQ(b.count, 0); + batch_free(&b); + + // writes are refused, and so are the ways to lift the guard + CHECK_EQ(step_all(srv, "INSERT INTO t VALUES(99)"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "UPDATE t SET a=1"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "DELETE FROM t"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "CREATE TABLE x(a)"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "DROP TABLE t"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "PRAGMA query_only=0"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "PRAGMA user_version=5"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "ATTACH ':memory:' AS m"), SQLITE_AUTH); + CHECK_EQ(step_all(srv, "SELECT 'still works'"), SQLITE_DONE); + sqlite3_stmt *st; + CHECK_EQ(sqlite3_prepare_v2(srv, "SELECT count(*) FROM t WHERE a=99", -1, &st, NULL), SQLITE_OK); + CHECK_EQ(sqlite3_step(st), SQLITE_ROW); + CHECK_EQ(sqlite3_column_int(st, 0), 0); + sqlite3_finalize(st); + CHECK_EQ(sqlite3_close_v2(srv), SQLITE_OK); + CHECK_EQ(list_conns(&l), before); + + // a writable server connection writes, to what the app sees, and is still unlisted + CHECK_EQ(sqlitedbg_open_server(g_db_path, 1, &srv), SQLITE_OK); + CHECK_EQ(sqlite3_db_readonly(srv, "main"), 0); + CHECK_EQ(step_all(srv, "INSERT INTO t VALUES(1234)"), SQLITE_DONE); + CHECK_EQ(list_conns(&l), before); + CHECK_EQ(sqlite3_close_v2(srv), SQLITE_OK); + CHECK_EQ(sqlite3_prepare_v2(g_main, "SELECT count(*) FROM t WHERE a=1234", -1, &st, NULL), SQLITE_OK); + CHECK_EQ(sqlite3_step(st), SQLITE_ROW); + CHECK_EQ(sqlite3_column_int(st, 0), 1); + sqlite3_finalize(st); + + // a missing file is not created by a read-only open, and nothing leaks + char missing[320]; + snprintf(missing, sizeof missing, "%s/missing.db", g_dir); + CHECK_EQ(sqlitedbg_open_server(missing, 0, &srv), SQLITE_CANTOPEN); + CHECK(srv == NULL); + CHECK_EQ(sqlitedbg_open_server(missing, 1, &srv), SQLITE_CANTOPEN); + CHECK(srv == NULL); + + // the flag is per thread and per open: an app connection opened after is listed + sqlite3 *app = open_app(":memory:"); + CHECK_EQ(list_conns(&l), before + 1); + CHECK_EQ(sqlite3_close(app), SQLITE_OK); +} + +// Each of these would let a client keep a lock on the app's database (ADFA-7135 review, H3). +static void test_server_holds_no_locks(void) { + static const char *const denied[] = { + "BEGIN", + "BEGIN IMMEDIATE", + "BEGIN EXCLUSIVE", + "SAVEPOINT s", + "PRAGMA locking_mode=EXCLUSIVE", + "PRAGMA locking_mode", + "PRAGMA busy_timeout=60000", + "PRAGMA busy_timeout", + // process-wide settings that would reach the app's own connections + "PRAGMA hard_heap_limit=1000000000", + "PRAGMA hard_heap_limit", + "PRAGMA soft_heap_limit=1000000000", + "PRAGMA soft_heap_limit", + "PRAGMA temp_store_directory='/tmp'", + }; + for (size_t i = 0; i < sizeof denied / sizeof denied[0]; i++) { + sqlite3 *srv = NULL; + CHECK_EQ(sqlitedbg_open_server(g_db_path, 0, &srv), SQLITE_OK); + int rc = step_all(srv, denied[i]); + if (rc != SQLITE_AUTH) fprintf(stderr, "not refused: %s (%d)\n", denied[i], rc); + CHECK_EQ(rc, SQLITE_AUTH); + // a read on the same connection, then the app must still be able to write + CHECK_EQ(step_all(srv, "SELECT count(*) FROM t"), SQLITE_DONE); + char *err = NULL; + rc = sqlite3_exec(g_main, "INSERT INTO t VALUES(4242)", NULL, NULL, &err); + if (rc != SQLITE_OK) fprintf(stderr, "app write after \"%s\": %s\n", denied[i], err ? err : "?"); + sqlite3_free(err); + CHECK_EQ(rc, SQLITE_OK); + exec_ok(g_main, "DELETE FROM t WHERE a=4242"); + CHECK_EQ(sqlite3_close_v2(srv), SQLITE_OK); + } + + // an ordinary read leaves the server connection with no lock and in autocommit + sqlite3 *srv = NULL; + CHECK_EQ(sqlitedbg_open_server(g_db_path, 0, &srv), SQLITE_OK); + CHECK_EQ(step_all(srv, "SELECT * FROM t"), SQLITE_DONE); + CHECK_EQ(sqlite3_get_autocommit(srv), 1); + exec_ok(g_main, "INSERT INTO t VALUES(4243)"); + exec_ok(g_main, "DELETE FROM t WHERE a=4243"); + CHECK_EQ(sqlite3_close_v2(srv), SQLITE_OK); +} + +typedef struct locker { + char path[320]; + atomic_int locked, release; +} locker; + +// An app connection that holds an exclusive lock (rollback-journal mode) until told to release it. +static void *locker_main(void *arg) { + locker *l = arg; + sqlite3 *db = open_app(l->path); + exec_ok(db, "BEGIN EXCLUSIVE"); + atomic_store(&l->locked, 1); + while (!atomic_load(&l->release)) usleep(1000); + exec_ok(db, "COMMIT"); + sqlite3_close(db); + return NULL; +} + +static double since_ms(const struct timespec *t0) { + struct timespec t1; + clock_gettime(CLOCK_MONOTONIC, &t1); + return (t1.tv_sec - t0->tv_sec) * 1e3 + (t1.tv_nsec - t0->tv_nsec) / 1e6; +} + +static void *release_later(void *arg) { + usleep(500 * 1000); + atomic_store(&((locker *)arg)->release, 1); + return NULL; +} + +// A server read waits for the app's lock (rollback-journal mode) instead of failing at once. +static void test_server_waits_for_locks(void) { + locker l = {0}; + snprintf(l.path, sizeof l.path, "%s/journal.db", g_dir); + sqlite3 *app = open_app(l.path); + exec_ok(app, "PRAGMA journal_mode=DELETE"); + exec_ok(app, "CREATE TABLE j(a); INSERT INTO j VALUES(1)"); + sqlite3_close(app); + + // the lock is released after 500 ms: the read waits, then succeeds + pthread_t holder, releaser; + pthread_create(&holder, NULL, locker_main, &l); + while (!atomic_load(&l.locked)) usleep(1000); + sqlite3 *srv = NULL; + CHECK_EQ(sqlitedbg_open_server(l.path, 0, &srv), SQLITE_OK); + struct timespec t0; + clock_gettime(CLOCK_MONOTONIC, &t0); + pthread_create(&releaser, NULL, release_later, &l); + CHECK_EQ(step_all(srv, "SELECT * FROM j"), SQLITE_DONE); + double waited = since_ms(&t0); + CHECK(waited >= 300); + CHECK(waited < SQLITEDBG_SERVER_BUSY_MS); + pthread_join(releaser, NULL); + pthread_join(holder, NULL); + CHECK_EQ(sqlite3_close_v2(srv), SQLITE_OK); + + // a lock held past the timeout gives SQLITE_BUSY after about that long, not at once + atomic_store(&l.locked, 0); + atomic_store(&l.release, 0); + pthread_create(&holder, NULL, locker_main, &l); + while (!atomic_load(&l.locked)) usleep(1000); + CHECK_EQ(sqlitedbg_open_server(l.path, 0, &srv), SQLITE_OK); + clock_gettime(CLOCK_MONOTONIC, &t0); + CHECK_EQ(step_all(srv, "SELECT * FROM j"), SQLITE_BUSY); + waited = since_ms(&t0); + CHECK(waited >= SQLITEDBG_SERVER_BUSY_MS - 500); + CHECK(waited < SQLITEDBG_SERVER_BUSY_MS + 2000); + atomic_store(&l.release, 1); + pthread_join(holder, NULL); + CHECK_EQ(step_all(srv, "SELECT * FROM j"), SQLITE_DONE); + CHECK_EQ(sqlite3_close_v2(srv), SQLITE_OK); +} + +typedef struct worker { + int index; + atomic_int *stop; + long long stmts, closes; + int failed; +} worker; + +static void *worker_main(void *arg) { + worker *w = arg; + sqlite3 *db = NULL; + if (sqlite3_open(":memory:", &db) != SQLITE_OK) { + w->failed = 1; + return NULL; + } + sqlite3_stmt *st; + sqlite3_exec(db, "CREATE TABLE w(a, b)", NULL, NULL, NULL); + w->stmts++; + long long n = 0; + while (!atomic_load(w->stop)) { + if (sqlite3_prepare_v2(db, "INSERT INTO w VALUES(?, ?)", -1, &st, NULL) != SQLITE_OK) { + w->failed = 1; + break; + } + sqlite3_bind_int64(st, 1, n); + sqlite3_bind_text(st, 2, "value", -1, SQLITE_STATIC); + if (sqlite3_step(st) != SQLITE_DONE) w->failed = 1; + sqlite3_finalize(st); + w->stmts++; + if (n % 7 == 0) { + if (sqlite3_exec(db, "SELECT count(*) FROM w", NULL, NULL, NULL) != SQLITE_OK) w->failed = 1; + w->stmts++; + } + if (n % 50 == 0) { + // connection churn beside the other threads' statements + sqlite3 *extra = NULL; + if (sqlite3_open(":memory:", &extra) != SQLITE_OK) { + w->failed = 1; + break; + } + if (sqlite3_exec(extra, "SELECT 1", NULL, NULL, NULL) != SQLITE_OK) w->failed = 1; + w->stmts++; + if (sqlite3_close(extra) != SQLITE_OK) w->failed = 1; + w->closes++; + } + n++; + } + if (sqlite3_close(db) != SQLITE_OK) w->failed = 1; + w->closes++; + return NULL; +} + +typedef struct reader { + atomic_int *stop; + int failed; + long long reads; +} reader; + +static void list_noop(void *context, const sqlitedbg_connection *c) { + (void)context; + if (c->id == 0 || !c->path) *(int *)context = 1; +} + +static void *reader_main(void *arg) { + reader *r = arg; + static unsigned char buf[2048]; + uint64_t after = 0; + while (!atomic_load(r->stop)) { + sqlitedbg_read_result res; + sqlitedbg_read_events(after, buf, sizeof buf, &res); + size_t off = 0; + uint64_t prev = after; + for (size_t i = 0; i < res.count; i++) { + const sqlitedbg_event_record *rec = (const sqlitedbg_event_record *)(void *)(buf + off); + if (rec->seq <= prev || rec->record_bytes % 8 || rec->kind < 1 || rec->kind > 3 || + rec->record_bytes < sizeof *rec + rec->sql_bytes) + r->failed = 1; + prev = rec->seq; + off += rec->record_bytes; + } + if (off != res.bytes) r->failed = 1; + after = res.next; + int bad = 0; + sqlitedbg_each_connection(list_noop, &bad); + if (bad) r->failed = 1; + sqlitedbg_set_client_connected(r->reads++ % 3 == 0); + } + sqlitedbg_set_client_connected(0); + return NULL; +} + +static void test_threads(void) { + listing l; + size_t before = list_conns(&l); + uint64_t cur = cursor_now(); + enum { N = 8 }; + pthread_t th[N], rt; + worker w[N]; + reader rd; + atomic_int stop = 0; + memset(w, 0, sizeof w); + memset(&rd, 0, sizeof rd); + rd.stop = &stop; + pthread_create(&rt, NULL, reader_main, &rd); + for (int i = 0; i < N; i++) { + w[i].index = i; + w[i].stop = &stop; + pthread_create(&th[i], NULL, worker_main, &w[i]); + } + struct timespec two = {2, 0}; + nanosleep(&two, NULL); + atomic_store(&stop, 1); + for (int i = 0; i < N; i++) pthread_join(th[i], NULL); + pthread_join(rt, NULL); + + long long stmts = 0, closes = 0; + for (int i = 0; i < N; i++) { + CHECK_EQ(w[i].failed, 0); + stmts += w[i].stmts; + closes += w[i].closes; + } + CHECK_EQ(rd.failed, 0); + CHECK(stmts > 1000); + // every thread's connections are gone, and every event made was numbered + CHECK_EQ(list_conns(&l), before); + batch b; + read_all(0, &b); + CHECK_EQ(b.next - cur, 2 * (uint64_t)stmts + (uint64_t)closes); + batch_free(&b); + printf(" threads: %lld statements, %lld closes, %lld reads\n", stmts, closes, rd.reads); +} + +// Fixture for EventDecoder's JVM test: a known run, as bytes plus a table of what they hold. + +static void write_tsv_row(FILE *f, const sqlitedbg_event_record *r, const char *sql) { + fprintf(f, "%llu\t%u\t%llu\t%lld\t%lld\t%lld\t%u\t%.*s\n", (unsigned long long)r->seq, r->kind, + (unsigned long long)r->connection_id, (long long)r->thread_id, (long long)r->time_ns, + (long long)r->duration_ns, r->flags, (int)r->sql_bytes, sql); +} + +static void run_fixture_scenario(void) { + char sql[256]; + exec_ok(g_main, "CREATE TABLE fx(a)"); + sqlite3_stmt *st; + CHECK_EQ(sqlite3_prepare_v2(g_main, "INSERT INTO fx VALUES(?)", -1, &st, NULL), SQLITE_OK); + sqlite3_bind_int(st, 1, 5); + sqlite3_step(st); + sqlite3_reset(st); + sqlitedbg_set_client_connected(1); + sqlite3_bind_int(st, 1, 6); + sqlite3_step(st); + sqlite3_finalize(st); + sqlitedbg_set_client_connected(0); + exec_ok(g_main, "SELECT 'h\xC3\xA9llo \xE2\x82\xAC \xF0\x9F\x98\x80'"); + exec_ok(g_main, "CREATE TABLE fxlog(a)"); + exec_ok(g_main, "CREATE TRIGGER fxt AFTER INSERT ON fx BEGIN INSERT INTO fxlog VALUES(new.a); END"); + exec_ok(g_main, "INSERT INTO fx VALUES(1)"); + size_t n = (size_t)snprintf(sql, sizeof sql, "SELECT '"); + for (int i = 0; i < 25; i++) n += (size_t)snprintf(sql + n, sizeof sql - n, "\xE2\x82\xAC"); + snprintf(sql + n, sizeof sql - n, "'"); + exec_ok(g_main, sql); + sqlite3 *c = open_app(":memory:"); + exec_ok(c, "SELECT 1"); + CHECK_EQ(sqlite3_close(c), SQLITE_OK); +} + +static unsigned char *slurp(const char *path, size_t *n) { + FILE *f = fopen(path, "rb"); + if (!f) { + perror(path); + return NULL; + } + fseek(f, 0, SEEK_END); + long len = ftell(f); + fseek(f, 0, SEEK_SET); + unsigned char *buf = malloc((size_t)len + 1); + if (fread(buf, 1, (size_t)len, f) != (size_t)len) len = 0; + buf[len] = 0; + fclose(f); + *n = (size_t)len; + return buf; +} + +// Walks records in bin and rewrites them as the table; used both to write and to check the fixture. +static int bin_to_tsv(const unsigned char *bin, size_t n, char **out) { + size_t cap = 1 << 16; + char *s = malloc(cap); + FILE *f = fmemopen(s, cap, "w"); + size_t off = 0; + int ok = 1; + while (off < n) { + const sqlitedbg_event_record *r = (const sqlitedbg_event_record *)(const void *)(bin + off); + if (off + sizeof *r > n || r->record_bytes % 8 || off + r->record_bytes > n || + r->record_bytes < sizeof *r + r->sql_bytes) { + ok = 0; + break; + } + write_tsv_row(f, r, (const char *)bin + off + sizeof *r); + off += r->record_bytes; + } + fclose(f); + *out = s; + return ok; +} + +static void write_fixture(const char *bin_path, const char *tsv_path) { + uint64_t cur = cursor_now(); + run_fixture_scenario(); + size_t cap = 1 << 16; + unsigned char *buf = malloc(cap); + sqlitedbg_read_result r; + sqlitedbg_read_events(cur, buf, cap, &r); + CHECK_EQ(r.dropped, 0); + CHECK(r.count > 10); + FILE *f = fopen(bin_path, "wb"); + CHECK(f != NULL); + if (f) { + fwrite(buf, 1, r.bytes, f); + fclose(f); + } + char *tsv; + CHECK(bin_to_tsv(buf, r.bytes, &tsv)); + f = fopen(tsv_path, "wb"); + CHECK(f != NULL); + if (f) { + fputs(tsv, f); + fclose(f); + } + free(tsv); + free(buf); +} + +// The committed fixture still has the layout of the header: its table is what its bytes hold +static void check_fixture(const char *bin_path, const char *tsv_path) { + size_t bn, tn; + unsigned char *bin = slurp(bin_path, &bn); + unsigned char *tsv = slurp(tsv_path, &tn); + CHECK(bin && tsv); + if (!bin || !tsv) return; + char *again; + CHECK(bin_to_tsv(bin, bn, &again)); + CHECK(strcmp(again, (char *)tsv) == 0); + free(again); + free(bin); + free(tsv); +} + +int main(int argc, char **argv) { + snprintf(g_dir, sizeof g_dir, "/tmp/sqlitedbg-test-XXXXXX"); + if (!mkdtemp(g_dir)) { + perror("mkdtemp"); + return 2; + } + snprintf(g_db_path, sizeof g_db_path, "%s/app.db", g_dir); + + if (argc == 4 && strcmp(argv[1], "--check-fixture") == 0) { + check_fixture(argv[2], argv[3]); + } else { + printf("# install\n"); + test_install(); + printf("# events\n"); + test_events(); + test_expanded(); + printf("# truncation\n"); + test_truncation(); + printf("# burst\n"); + test_burst(); + printf("# close and list\n"); + test_close_and_list(); + printf("# init failure\n"); + test_init_failure(); + printf("# server connections\n"); + test_server(); + printf("# server holds no locks\n"); + test_server_holds_no_locks(); + printf("# server waits for locks\n"); + test_server_waits_for_locks(); + printf("# threads\n"); + test_threads(); + if (argc == 4 && strcmp(argv[1], "--write-fixture") == 0) write_fixture(argv[2], argv[3]); + sqlite3_close(g_main); + } + + char cmd[300]; + snprintf(cmd, sizeof cmd, "rm -rf '%s'", g_dir); + if (system(cmd) != 0) fprintf(stderr, "could not remove %s\n", g_dir); + printf("%d checks, %d failures\n", g_checks, g_failures); + return g_failures ? 1 : 0; +} diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/EventDecoderTest.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/EventDecoderTest.kt new file mode 100644 index 0000000000..555e566544 --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/EventDecoderTest.kt @@ -0,0 +1,92 @@ +package org.appdevforall.sqlitedebugger.engine + +import com.google.common.truth.Truth.assertThat +import org.junit.Assert.assertThrows +import org.junit.Test +import java.nio.ByteBuffer +import java.nio.ByteOrder + +class EventDecoderTest { + // events.bin is what sqlitedbg_read_events copied out in sqlitedbg_test.c (test.sh --update-fixture); + // events.tsv is the same records as the C struct reads them: seq, kind, connection, thread, time, + // duration, flags, sql. + private val bytes = resource("events.bin") + private val table = + String(resource("events.tsv"), Charsets.UTF_8) + .trimEnd('\n') + .split('\n') + .map { it.split('\t', limit = 8) } + + private fun resource(name: String): ByteArray = + checkNotNull(javaClass.getResourceAsStream(name)) { "missing fixture $name" }.use { it.readBytes() } + + @Test + fun `decodes every field of the record the C layer wrote`() { + val events = EventDecoder.decode(bytes, bytes.size) + + assertThat(events).hasSize(table.size) + events.zip(table).forEach { (event, row) -> + assertThat(event.seq).isEqualTo(row[0].toLong()) + assertThat(event.kind.nativeValue).isEqualTo(row[1].toInt()) + assertThat(event.connectionId).isEqualTo(row[2].toLong()) + assertThat(event.threadId).isEqualTo(row[3].toLong()) + assertThat(event.timeEpochNanos).isEqualTo(row[4].toLong()) + assertThat(event.durationNanos).isEqualTo(if (event.kind == EventKind.PROFILE) row[5].toLong() else null) + assertThat(event.truncated).isEqualTo(row[6].toInt() and 1 != 0) + assertThat(event.expanded).isEqualTo(row[6].toInt() and 2 != 0) + assertThat(event.sql).isEqualTo(row[7]) + } + } + + @Test + fun `the fixture covers every kind and flag and a wide id`() { + val events = EventDecoder.decode(bytes, bytes.size) + + assertThat(events.map { it.kind }.toSet()).containsExactlyElementsIn(EventKind.values().toList()) + assertThat(events.any { it.truncated }).isTrue() + assertThat(events.any { it.expanded }).isTrue() + assertThat(events.any { it.sql.startsWith("--") }).isTrue() + // multi-byte text survives, including a character outside the BMP + assertThat(events.any { it.sql.contains("héllo € 😀") }).isTrue() + assertThat(events.map { it.seq }).isInOrder() + } + + @Test + fun `a prefix ending on a record boundary decodes to those records`() { + val firstRecordBytes = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).getInt(0) + + assertThat(EventDecoder.decode(bytes, firstRecordBytes)).hasSize(1) + assertThat(EventDecoder.decode(bytes, 0)).isEmpty() + } + + @Test + fun `a cut-off record is refused`() { + val firstRecordBytes = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).getInt(0) + + assertThrows(IllegalArgumentException::class.java) { EventDecoder.decode(bytes, firstRecordBytes - 8) } + assertThrows(IllegalArgumentException::class.java) { EventDecoder.decode(bytes, EventDecoder.HEADER_BYTES - 1) } + assertThrows(IllegalArgumentException::class.java) { EventDecoder.decode(bytes, bytes.size + 1) } + } + + @Test + fun `a malformed header is refused`() { + fun patched( + offset: Int, + value: Int, + ): ByteArray { + val copy = bytes.copyOf() + ByteBuffer.wrap(copy).order(ByteOrder.LITTLE_ENDIAN).putInt(offset, value) + return copy + } + + // record_bytes not a multiple of 8, smaller than the header, and an unknown kind + for (broken in listOf(patched(0, 60), patched(0, 48), patched(4, 9), patched(4, 0))) { + assertThrows(IllegalArgumentException::class.java) { EventDecoder.decode(broken, broken.size) } + } + // sql_bytes longer than the record holds + val big = patched(52, 100000) + assertThrows(IllegalArgumentException::class.java) { EventDecoder.decode(big, big.size) } + val negative = patched(52, -1) + assertThrows(IllegalArgumentException::class.java) { EventDecoder.decode(negative, negative.size) } + } +} diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/FakeDebugEngine.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/FakeDebugEngine.kt new file mode 100644 index 0000000000..b6d9a59484 --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/engine/FakeDebugEngine.kt @@ -0,0 +1,129 @@ +package org.appdevforall.sqlitedebugger.engine + +/** + * A [DebugEngine] for JVM tests of the server (ADFA-7136): connections and events are set by the + * test, and each query is answered by [answer]. It keeps the real engine's rules that the server + * relies on: cursors and dropped counts, and refusing to open a path no app connection has open. + */ +class FakeDebugEngine( + var answer: (path: String, writable: Boolean, sql: String, args: List) -> QueryResult = { _, _, _, _ -> + QueryResult(emptyList(), emptyList(), 0, false) + }, +) : DebugEngine { + val appConnections = mutableListOf() + + /** Events still in the buffer, oldest first; [lastSeq] counts every event made. */ + private val buffered = ArrayDeque() + private var lastSeq = 0L + + var clientConnected = false + private set + + val openedServerConnections = mutableListOf>() + + /** How many server connections were closed. */ + var closedServerConnections = 0 + private set + + fun addEvent( + kind: EventKind, + sql: String, + connectionId: Long = 1, + durationNanos: Long? = null, + ) { + lastSeq++ + buffered.addLast(Event(lastSeq, kind, connectionId, 1, lastSeq, durationNanos, sql, false, clientConnected)) + } + + /** Drops the [count] oldest buffered events, as a full native buffer would. */ + fun dropOldest(count: Int) = repeat(count) { buffered.removeFirstOrNull() } + + private fun cellBytes(value: Any?): Long = + when (value) { + is String -> value.toByteArray().size.toLong() + is ByteArray -> value.size.toLong() + is Long, is Int, is Double -> 8 + else -> 0 + } + + override fun connections() = appConnections.toList() + + override fun readEvents( + after: Long, + maxBytes: Int, + ): EventBatch { + val oldest = buffered.firstOrNull()?.seq ?: (lastSeq + 1) + val dropped = (oldest - 1 - after).coerceAtLeast(0) + val events = mutableListOf() + var bytes = 0 + for (event in buffered) { + if (event.seq <= after) continue + // the native record: a 56-byte header, then the SQL padded to 8 bytes + val size = 56 + (event.sql.toByteArray().size + 7) / 8 * 8 + if (bytes + size > maxBytes) break + bytes += size + events += event + } + return EventBatch(events, events.lastOrNull()?.seq ?: maxOf(after, oldest - 1), dropped) + } + + override fun setClientConnected(connected: Boolean) { + clientConnected = connected + } + + override fun openServerConnection( + path: String, + writable: Boolean, + ): ServerConnection { + if (appConnections.isEmpty()) { + throw DebugEngineException(DebugEngineException.SQLITE_MISUSE, "no app connection has opened yet") + } + if (appConnections.none { it.path == path && path.isNotEmpty() }) { + throw DebugEngineException(14, "unable to open database file: $path") // SQLITE_CANTOPEN + } + openedServerConnections += path to writable + return object : ServerConnection { + private var closed = false + + override fun query( + sql: String, + args: List, + offset: Long, + limits: QueryLimits, + deadlineMillis: Long, + ): QueryResult { + check(!closed) { "closed" } + val all = answer(path, writable, sql, args) + val skip = offset.toInt().coerceAtMost(all.rows.size) + val rest = all.rows.drop(skip) + // the native rule: the first row always goes in, then rows while the total stays within the budget + var taken = 0 + var total = 0L + for (row in rest) { + if (taken >= limits.maxRows) break + val bytes = row.sumOf { cellBytes(it) } + if (taken > 0 && total + bytes > limits.maxTotalBytes) break + total += bytes + taken++ + } + val columns = all.columns.size + val cut = + all.truncatedCellIndexes + .map { it - skip.toLong() * columns } + .filter { it >= 0 && it < taken.toLong() * columns } + .toSet() + return all.copy( + rows = rest.take(taken), + hasMore = rest.size > taken, + truncatedCells = if (all.truncatedCellIndexes.isEmpty()) all.truncatedCells else cut.size, + truncatedCellIndexes = cut, + ) + } + + override fun close() { + if (!closed) closedServerConnections++ + closed = true + } + } + } +} diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt new file mode 100644 index 0000000000..b53bfed6d9 --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/DatabaseToolsTest.kt @@ -0,0 +1,528 @@ +package org.appdevforall.sqlitedebugger.mcp + +import org.appdevforall.sqlitedebugger.engine.AppConnection +import org.appdevforall.sqlitedebugger.engine.DebugEngineException +import org.appdevforall.sqlitedebugger.engine.EventKind +import org.appdevforall.sqlitedebugger.engine.FakeDebugEngine +import org.appdevforall.sqlitedebugger.engine.QueryResult +import org.json.JSONArray +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import java.io.File + +class DatabaseToolsTest { + private val engine = FakeDebugEngine() + private var now = 1_000_000L + private var dir: File? = null + private val tools get() = DatabaseTools(engine, dir, { now }) + private val seenSql = mutableListOf() + + private fun conn( + id: Long, + path: String, + closing: Boolean = false, + ) = AppConnection(id, path, false, closing, 1_700_000_000_000_000_000L + id, 5) + + private fun runTool( + t: DatabaseTools, + name: String, + args: JSONObject = JSONObject(), + ): JSONObject { + val protocol = McpProtocol(t.tools(), "test", { "s" }, 4) + val init = + JSONObject() + .put("jsonrpc", "2.0") + .put("id", 1) + .put("method", "initialize") + .put("params", JSONObject().put("protocolVersion", "2025-06-18")) + .toString() + val session = protocol.post(init, null, null).headers[McpProtocol.SESSION_HEADER]!! + val call = + JSONObject() + .put("jsonrpc", "2.0") + .put("id", 2) + .put("method", "tools/call") + .put("params", JSONObject().put("name", name).put("arguments", args)) + .toString() + return JSONObject(protocol.post(call, session, null).body!!).getJSONObject("result") + } + + private fun ok( + name: String, + args: JSONObject = JSONObject(), + t: DatabaseTools = tools, + ): JSONObject { + val r = runTool(t, name, args) + assertFalse(r.toString(), r.getBoolean("isError")) + return r.getJSONObject("structuredContent") + } + + private fun db(path: String = "/data/a.db") = JSONObject().put("database", path) + + private fun answerWith(vararg byPrefix: Pair) { + engine.answer = { _, _, sql, _ -> + seenSql += sql + byPrefix.firstOrNull { sql.startsWith(it.first) }?.second ?: QueryResult(emptyList(), emptyList(), 0, false) + } + } + + @Test fun everyToolIsReadOnly() { + val list = tools.tools() + assertEquals( + listOf( + "list_databases", + "list_connections", + "get_schema", + "query", + "explain_query_plan", + "get_stats", + "integrity_check", + "get_events", + ), + list.map { it.name }, + ) + list.forEach { assertTrue(it.name, it.describe().getJSONObject("annotations").getBoolean("readOnlyHint")) } + } + + @Test fun listDatabasesGroupsByPathAndSkipsEmpty() { + engine.appConnections += listOf(conn(1, "/data/a.db"), conn(2, "/data/a.db"), conn(3, ""), conn(4, "/data/b.db")) + val dbs = ok("list_databases").getJSONArray("databases") + assertEquals(2, dbs.length()) + assertEquals("/data/a.db", dbs.getJSONObject(0).getString("path")) + assertEquals(2, dbs.getJSONObject(0).getInt("connections")) + assertTrue(dbs.getJSONObject(1).getBoolean("available")) + } + + @Test fun listDatabasesAddsUnopenedFilesButNotSideFiles() { + val d = + kotlin.io.path + .createTempDirectory("dbs") + .toFile() + try { + listOf("open.db", "closed.db", "closed.db-wal", "closed.db-shm", "closed.db-journal").forEach { File(d, it).writeText("x") } + dir = d + engine.appConnections += conn(1, File(d, "open.db").absolutePath) + val dbs = ok("list_databases").getJSONArray("databases") + assertEquals(2, dbs.length()) + assertTrue(dbs.getJSONObject(0).getBoolean("available")) + val closed = dbs.getJSONObject(1) + assertEquals(File(d, "closed.db").absolutePath, closed.getString("path")) + assertFalse(closed.getBoolean("available")) + } finally { + d.deleteRecursively() + } + } + + @Test fun listDatabasesMatchesOpenFilesThroughASymlinkedFolder() { + val root = + kotlin.io.path + .createTempDirectory("dbs") + try { + val real = + java.nio.file.Files + .createDirectory(root.resolve("real")) + .toFile() + val link = root.resolve("link") + java.nio.file.Files + .createSymbolicLink(link, real.toPath()) + listOf("open.db", "closed.db").forEach { File(real, it).writeText("x") } + // the folder is listed through the link, the connection reports the real path + dir = link.toFile() + engine.appConnections += conn(1, File(real, "open.db").absolutePath) + var dbs = ok("list_databases").getJSONArray("databases") + assertEquals(dbs.toString(), 2, dbs.length()) + assertEquals(File(real, "open.db").absolutePath, dbs.getJSONObject(0).getString("path")) + assertTrue(dbs.getJSONObject(0).getBoolean("available")) + assertFalse(dbs.getJSONObject(1).getBoolean("available")) + assertTrue(dbs.getJSONObject(1).getString("path").endsWith("closed.db")) + + // and the other way round: the connection reports the link, the folder is the real one + engine.appConnections.clear() + dir = real + engine.appConnections += conn(2, File(link.toFile(), "open.db").absolutePath) + dbs = ok("list_databases").getJSONArray("databases") + assertEquals(dbs.toString(), 2, dbs.length()) + assertEquals(File(link.toFile(), "open.db").absolutePath, dbs.getJSONObject(0).getString("path")) + } finally { + root.toFile().deleteRecursively() + } + } + + @Test fun listConnectionsEncodesBigIntegersAsStrings() { + engine.appConnections += conn(1, "/data/a.db", closing = true) + val c = ok("list_connections").getJSONArray("connections").getJSONObject(0) + assertEquals(1, c.getInt("id")) + assertTrue(c.getBoolean("closing")) + assertEquals("1700000000000000001", c.getString("opened_at_epoch_nanos")) + } + + @Test fun everyDatabaseToolRefusesAnUnknownPathBeforeTheEngine() { + engine.appConnections += conn(1, "/data/a.db") + val cases = + listOf( + "get_schema" to JSONObject(), + "query" to JSONObject().put("sql", "select 1"), + "explain_query_plan" to JSONObject().put("sql", "select 1"), + "get_stats" to JSONObject(), + "integrity_check" to JSONObject(), + ) + for ((name, extra) in cases) { + for (bad in listOf("/data/other.db", "", "/data/a.db-wal")) { + val r = runTool(tools, name, db(bad).also { o -> extra.keys().forEach { k -> o.put(k, extra.get(k)) } }) + assertTrue("$name $bad", r.getBoolean("isError")) + } + val missing = runTool(tools, name, extra) + assertTrue(name, missing.getBoolean("isError")) + } + assertTrue(engine.openedServerConnections.isEmpty()) + } + + @Test fun emptyPathOfAnInMemoryConnectionIsNotOpenable() { + engine.appConnections += conn(1, "") + assertTrue(runTool(tools, "get_stats", db("")).getBoolean("isError")) + assertTrue(engine.openedServerConnections.isEmpty()) + } + + @Test fun queryOpensReadOnlyAndPages() { + engine.appConnections += conn(1, "/data/a.db") + engine.answer = { _, _, _, _ -> QueryResult(listOf("n"), (1L..5L).map { listOf(it) }, 0, false) } + val r = ok("query", db().put("sql", "select n from t").put("max_rows", 2).put("offset", 1)) + assertEquals(listOf(listOf("/data/a.db", false)), engine.openedServerConnections.map { listOf(it.first, it.second) }) + assertEquals(2, r.getInt("row_count")) + assertEquals(2, r.getJSONArray("rows").getJSONArray(0).getInt(0)) + assertTrue(r.getBoolean("has_more")) + assertEquals(3, r.getInt("next_offset")) + assertTrue(r.getString("note").contains("re-runs")) + assertTrue( + tools + .tools() + .first { it.name == "query" } + .description + .contains("re-runs"), + ) + } + + @Test fun queryChecksItsArguments() { + engine.appConnections += conn(1, "/data/a.db") + + fun bad(vararg kv: Pair) = + assertTrue( + kv.toString(), + runTool(tools, "query", db().also { o -> kv.forEach { o.put(it.first, it.second) } }).getBoolean("isError"), + ) + bad() + bad("sql" to "x", "max_rows" to 1001) + bad("sql" to "x", "max_rows" to 0) + bad("sql" to "x", "offset" to -1) + bad("sql" to "x", "max_rows" to 1.5) + bad("sql" to "x", "args" to "nope") + bad("sql" to "x", "args" to JSONArray().put(JSONObject())) + assertTrue(engine.openedServerConnections.isEmpty()) + } + + @Test fun queryBindsArgsByType() { + engine.appConnections += conn(1, "/data/a.db") + var got: List = emptyList() + engine.answer = { _, _, _, a -> + got = a + QueryResult(emptyList(), emptyList(), 0, false) + } + ok( + "query", + db().put("sql", "x").put( + "args", + JSONArray() + .put(JSONObject.NULL) + .put(3) + .put(2.5) + .put("s") + .put(true), + ), + ) + assertEquals(listOf(null, 3L, 2.5, "s", 1L), got) + } + + @Test fun queryErrorsCarryTheSqliteCode() { + engine.appConnections += conn(1, "/data/a.db") + engine.answer = { _, _, _, _ -> throw DebugEngineException(DebugEngineException.SQLITE_AUTH, "not authorized") } + val r = runTool(tools, "query", db().put("sql", "delete from t")) + assertTrue(r.getBoolean("isError")) + assertEquals(23, r.getJSONObject("structuredContent").getInt("sqlite_code")) + } + + @Test fun deadlineIsFiftySecondsAhead() { + engine.appConnections += conn(1, "/data/a.db") + var deadline = 0L + val real = engine + engine.answer = { _, _, _, _ -> QueryResult(emptyList(), emptyList(), 0, false) } + // the fake ignores the deadline, so check the arithmetic through the constant and a wrapper engine + val wrapper = + object : org.appdevforall.sqlitedebugger.engine.DebugEngine by real { + override fun openServerConnection( + path: String, + writable: Boolean, + ) = real.openServerConnection(path, writable).let { c -> + object : org.appdevforall.sqlitedebugger.engine.ServerConnection by c { + override fun query( + sql: String, + args: List, + offset: Long, + limits: org.appdevforall.sqlitedebugger.engine.QueryLimits, + deadlineMillis: Long, + ): QueryResult { + deadline = deadlineMillis + return c.query(sql, args, offset, limits, deadlineMillis) + } + } + } + } + ok("query", db().put("sql", "x"), DatabaseTools(wrapper, null, { now })) + assertEquals(now + 50_000, deadline) + } + + @Test fun jsonEncodesNonFiniteDoublesBigIntegersAndBlobs() { + engine.appConnections += conn(1, "/data/a.db") + val big = 9_007_199_254_740_993L + engine.answer = { _, _, _, _ -> + QueryResult( + listOf("a", "b", "c", "d", "e", "f", "g", "h"), + listOf( + listOf( + Double.NaN, + Double.POSITIVE_INFINITY, + Double.NEGATIVE_INFINITY, + big, + -big, + 9_007_199_254_740_992L, + byteArrayOf(1, 2, 3), + null, + ), + ), + 0, + false, + ) + } + val row = ok("query", db().put("sql", "x")).getJSONArray("rows").getJSONArray(0) + assertEquals("NaN", row.get(0)) + assertEquals("Infinity", row.get(1)) + assertEquals("-Infinity", row.get(2)) + assertEquals("9007199254740993", row.get(3)) + assertEquals("-9007199254740993", row.get(4)) + assertEquals(9_007_199_254_740_992L, row.getLong(5)) + val blob = row.getJSONObject(6) + assertEquals("AQID", blob.getString("base64")) + assertEquals(3, blob.getInt("bytes")) + assertFalse(blob.getBoolean("truncated")) + assertTrue(row.isNull(7)) + assertEquals(true, (DatabaseTools.encodeValue(ByteArray(8), truncated = true) as JSONObject).getBoolean("truncated")) + assertEquals(false, (DatabaseTools.encodeValue(ByteArray(1024)) as JSONObject).getBoolean("truncated")) + assertEquals(1.5, DatabaseTools.encodeValue(1.5)) + } + + @Test fun aBlobExactlyAtTheLimitIsNotTruncatedButOneByteMoreIs() { + engine.appConnections += conn(1, "/data/a.db") + val whole = ByteArray(DatabaseTools.MAX_BLOB_BYTES) + val cut = ByteArray(DatabaseTools.MAX_BLOB_BYTES) // the engine kept maxBlobBytes of a longer value + engine.answer = { _, _, _, _ -> + QueryResult(listOf("a", "b"), listOf(listOf(whole, cut)), 1, false, setOf(1L)) + } + val r = ok("query", db().put("sql", "x")) + val row = r.getJSONArray("rows").getJSONArray(0) + assertFalse(row.getJSONObject(0).getBoolean("truncated")) + assertTrue(row.getJSONObject(1).getBoolean("truncated")) + assertEquals(1, r.getInt("truncated_cells")) + } + + @Test fun aResultStopsAtTheTotalBudgetAndPages() { + engine.appConnections += conn(1, "/data/a.db") + val big = "x".repeat(3 * 1024 * 1024) + engine.answer = { _, _, _, _ -> QueryResult(listOf("v"), listOf(listOf(big), listOf(big), listOf(big)), 0, false) } + val r = ok("query", db().put("sql", "x")) + assertEquals(1, r.getInt("row_count")) + assertTrue(r.getBoolean("has_more")) + assertEquals(1, r.getInt("next_offset")) + val next = ok("query", db().put("sql", "x").put("offset", 1)) + assertEquals(1, next.getInt("row_count")) + assertEquals(2, next.getInt("next_offset")) + assertEquals(4 * 1024 * 1024, DatabaseTools.MAX_TOTAL_BYTES) + } + + @Test fun getSchemaGroupsByType() { + engine.appConnections += conn(1, "/data/a.db") + answerWith( + "SELECT type" to + QueryResult( + listOf("type", "name", "tbl_name", "sql"), + listOf( + listOf("index", "i1", "t", "CREATE INDEX i1 ON t(a)"), + listOf("table", "t", "t", "CREATE TABLE t(a)"), + listOf("trigger", "tr", "t", "CREATE TRIGGER tr ..."), + listOf("view", "v", "v", "CREATE VIEW v AS ..."), + ), + 0, + false, + ), + ) + val r = ok("get_schema", db()) + assertEquals("t", r.getJSONArray("tables").getJSONObject(0).getString("name")) + assertEquals("CREATE INDEX i1 ON t(a)", r.getJSONArray("indexes").getJSONObject(0).getString("sql")) + assertEquals(1, r.getJSONArray("views").length()) + assertEquals(1, r.getJSONArray("triggers").length()) + assertTrue(seenSql.single().contains("FROM sqlite_master")) + assertFalse(seenSql.single().contains("sqlite_schema")) // an alias that needs SQLite 3.33 + } + + @Test fun explainQueryPlanPrefixesTheSql() { + engine.appConnections += conn(1, "/data/a.db") + answerWith( + "EXPLAIN QUERY PLAN" to + QueryResult( + listOf("id", "parent", "notused", "detail"), + listOf(listOf(3L, 0L, 0L, "SCAN t")), + 0, + false, + ), + ) + val plan = ok("explain_query_plan", db().put("sql", "select * from t")).getJSONArray("plan") + assertEquals("SCAN t", plan.getJSONObject(0).getString("detail")) + assertEquals("EXPLAIN QUERY PLAN select * from t", seenSql.single()) + } + + @Test fun getStatsCountsRowsUnderTenTables() { + engine.appConnections += conn(1, "/data/a.db") + + fun one(v: Long) = QueryResult(listOf("x"), listOf(listOf(v)), 0, false) + answerWith( + "PRAGMA page_count" to one(10), + "PRAGMA page_size" to one(4096), + "PRAGMA freelist_count" to one(2), + "SELECT name" to QueryResult(listOf("name"), listOf(listOf("a\"b"), listOf("c")), 0, false), + "SELECT COUNT" to one(7), + ) + val r = ok("get_stats", db()) + assertEquals(10, r.getInt("page_count")) + assertEquals(4096, r.getInt("page_size")) + assertEquals(2, r.getInt("freelist_count")) + assertEquals(40960, r.getInt("size_bytes")) + assertEquals(7, r.getJSONObject("row_counts").getInt("a\"b")) + assertTrue(seenSql.contains("SELECT COUNT(*) FROM \"a\"\"b\"")) + assertTrue(seenSql.any { it.contains("FROM sqlite_master") }) + assertTrue(seenSql.none { it.contains("sqlite_schema") }) + } + + @Test fun getStatsSkipsCountsFromTenTables() { + engine.appConnections += conn(1, "/data/a.db") + answerWith("SELECT name" to QueryResult(listOf("name"), (1..10).map { listOf("t$it") }, 0, false)) + val r = ok("get_stats", db()) + assertFalse(r.has("row_counts")) + assertTrue(r.has("row_counts_skipped")) + assertTrue(seenSql.none { it.startsWith("SELECT COUNT") }) + } + + @Test fun integrityCheckIsQuickUnlessAskedForFull() { + engine.appConnections += conn(1, "/data/a.db") + answerWith("PRAGMA" to QueryResult(listOf("quick_check"), listOf(listOf("ok")), 0, false)) + val quick = ok("integrity_check", db()) + assertTrue(quick.getBoolean("ok")) + assertEquals("quick_check", quick.getString("check")) + ok("integrity_check", db().put("full", true)) + assertEquals(listOf("PRAGMA quick_check", "PRAGMA integrity_check"), seenSql) + answerWith("PRAGMA" to QueryResult(listOf("c"), listOf(listOf("row 1 missing from index i1")), 0, false)) + assertFalse(ok("integrity_check", db()).getBoolean("ok")) + } + + @Test fun getEventsReturnsCursorDroppedAndTurnsOnExpansion() { + assertFalse(engine.clientConnected) + engine.addEvent(EventKind.STATEMENT, "select 1") + engine.addEvent(EventKind.PROFILE, "select 1", durationNanos = 99) + engine.addEvent(EventKind.STATEMENT, "select 2") + engine.dropOldest(1) + val t = tools + val first = ok("get_events", t = t) + assertTrue(engine.clientConnected) + assertEquals(1, first.getInt("dropped")) + assertEquals(3, first.getInt("next")) + val events = first.getJSONArray("events") + assertEquals(2, events.length()) + assertEquals("profile", events.getJSONObject(0).getString("kind")) + assertEquals(99, events.getJSONObject(0).getInt("duration_nanos")) + assertTrue(events.getJSONObject(1).isNull("duration_nanos")) + val second = ok("get_events", JSONObject().put("after", 3), t) + assertEquals(0, second.getJSONArray("events").length()) + assertEquals(0, second.getInt("dropped")) + assertEquals(3, second.getInt("next")) + t.onServerStopped() + assertFalse(engine.clientConnected) + } + + @Test fun aServerConnectionIsDroppedWhenTheAppRecreatesTheDatabase() { + engine.appConnections += conn(1, "/data/a.db") + val t = tools + ok("get_stats", db(), t) + assertEquals(1, engine.openedServerConnections.size) + // the app closed connection 1, deleted the file and opened a new one: no overlap with {1} + engine.appConnections.clear() + engine.appConnections += conn(2, "/data/a.db") + ok("get_stats", db(), t) + assertEquals(2, engine.openedServerConnections.size) + assertEquals(1, engine.closedServerConnections) + } + + @Test fun aServerConnectionIsKeptWhileASeenAppConnectionRemains() { + engine.appConnections += conn(1, "/data/a.db") + val t = tools + ok("get_stats", db(), t) + engine.appConnections += conn(2, "/data/a.db") + ok("get_stats", db(), t) + engine.appConnections.removeAll { it.id == 2L } + ok("get_stats", db(), t) + assertEquals(1, engine.openedServerConnections.size) + assertEquals(0, engine.closedServerConnections) + } + + @Test fun aServerConnectionIsDroppedWhenNoAppConnectionHasThePathAnyMore() { + engine.appConnections += conn(1, "/data/a.db") + val t = tools + ok("get_stats", db(), t) + engine.appConnections.clear() + engine.appConnections += conn(3, "/data/b.db") + assertTrue(runTool(t, "get_stats", db()).getBoolean("isError")) // refused: no longer a known path + assertEquals(1, engine.closedServerConnections) + } + + @Test fun boundValuesStopBeingRecordedSixtySecondsAfterTheLastGetEvents() { + val t = tools + ok("get_events", t = t) + assertTrue(engine.clientConnected) + now += 59_000 + ok("list_connections", t = t) + assertTrue("still within 60 s", engine.clientConnected) + ok("get_events", t = t) // renews the 60 s + now += 59_000 + ok("list_connections", t = t) + assertTrue(engine.clientConnected) + now += 2_000 + ok("list_connections", t = t) + assertFalse("60 s after the last get_events", engine.clientConnected) + ok("get_events", t = t) + assertTrue("the next get_events turns it on again", engine.clientConnected) + now += 61_000 + t.expireIdleClient() // what the timer calls when no tool call comes + assertFalse(engine.clientConnected) + t.onServerStopped() + } + + @Test fun stoppingTheServerClosesServerConnections() { + engine.appConnections += conn(1, "/data/a.db") + val t = tools + ok("get_stats", db(), t) + ok("get_stats", db(), t) + assertEquals(1, engine.openedServerConnections.size) + t.onServerStopped() + ok("get_stats", db(), t) + assertEquals(2, engine.openedServerConnections.size) + } +} diff --git a/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt new file mode 100644 index 0000000000..b1d48bf176 --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/java/org/appdevforall/sqlitedebugger/mcp/McpProtocolTest.kt @@ -0,0 +1,532 @@ +package org.appdevforall.sqlitedebugger.mcp + +import org.json.JSONArray +import org.json.JSONObject +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import java.net.Socket + +/** + * MCP control without Android: the protocol (sessions, methods, tool results and errors) and the + * HTTP side (token, Host and Origin checks, methods and paths), on a real loopback socket. + */ +class McpProtocolTest { + private var sessionNumber = 0 + private val echo = McpTool("echo", "Echo", "Returns its arguments.") { args -> JSONObject().put("got", args) } + private val needsPerson = + McpTool("needs_person", "Needs a person", "Always says a person is needed.", readOnly = false) { _ -> + throw McpToolError("A person is needed: tap Allow on the sharing phone.", JSONObject().put("person_needed", true)) + } + private val crashes = McpTool("crashes", "Crashes", "Throws an unexpected exception.") { _ -> throw IllegalStateException("boom") } + private val linkFails = McpTool("link_fails", "Link fails", "Throws an Error.") { _ -> throw UnsatisfiedLinkError("no library") } + + private fun protocol(maxSessions: Int = 16) = + McpProtocol(listOf(echo, needsPerson, crashes, linkFails), "test", { "s${++sessionNumber}" }, maxSessions) + + private fun initBody(version: String = "2025-06-18") = + JSONObject() + .put("jsonrpc", "2.0") + .put("id", 1) + .put("method", "initialize") + .put( + "params", + JSONObject() + .put("protocolVersion", version) + .put("capabilities", JSONObject()) + .put("clientInfo", JSONObject().put("name", "test").put("version", "1")), + ).toString() + + private fun request( + id: Any, + method: String, + params: JSONObject? = null, + ) = JSONObject() + .put("jsonrpc", "2.0") + .put("id", id) + .put("method", method) + .also { if (params != null) it.put("params", params) } + + private fun session(p: McpProtocol): String { + val r = p.post(initBody(), null, null) + assertEquals(200, r.status) + return r.headers[McpProtocol.SESSION_HEADER]!! + } + + private fun call( + p: McpProtocol, + s: String, + tool: String, + args: JSONObject? = null, + ): JSONObject { + val params = JSONObject().put("name", tool).also { if (args != null) it.put("arguments", args) } + val r = p.post(request(7, "tools/call", params).toString(), s, null) + assertEquals(200, r.status) + return JSONObject(r.body!!) + } + + // ---------------------------------------------------------------- the protocol + + @Test fun initializeAgreesVersionAndStartsSession() { + val p = protocol() + val r = p.post(initBody("2025-03-26"), null, null) + assertEquals(200, r.status) + assertEquals("s1", r.headers[McpProtocol.SESSION_HEADER]) + val result = JSONObject(r.body!!).getJSONObject("result") + assertEquals("2025-03-26", result.getString("protocolVersion")) + assertEquals(McpProtocol.SERVER_NAME, result.getJSONObject("serverInfo").getString("name")) + assertTrue(result.getJSONObject("capabilities").has("tools")) + // no read-write mode exists, so the agent must not be told one may be on + assertFalse(result.getString("instructions").contains("read-write", ignoreCase = true)) + assertEquals(1, p.sessionCount()) + } + + @Test fun unknownVersionGetsTheNewest() { + val r = protocol().post(initBody("1999-01-01"), null, null) + assertEquals(McpProtocol.VERSIONS[0], JSONObject(r.body!!).getJSONObject("result").getString("protocolVersion")) + } + + @Test fun requestsNeedALiveSession() { + val p = protocol() + assertEquals(400, p.post(request(1, "ping").toString(), null, null).status) + assertEquals(404, p.post(request(1, "ping").toString(), "nope", null).status) + val s = session(p) + assertEquals(200, p.post(request(1, "ping").toString(), s, null).status) + } + + @Test fun unsupportedVersionHeaderIsRefused() { + val p = protocol() + val s = session(p) + assertEquals(400, p.post(request(1, "ping").toString(), s, "1999-01-01").status) + assertEquals(200, p.post(request(1, "ping").toString(), s, "2025-06-18").status) + } + + @Test fun notificationsGetNoAnswer() { + val p = protocol() + val s = session(p) + val r = p.post(JSONObject().put("jsonrpc", "2.0").put("method", "notifications/initialized").toString(), s, null) + assertEquals(202, r.status) + assertNull(r.body) + } + + @Test fun pingAnswersEmptyResult() { + val p = protocol() + val s = session(p) + val r = JSONObject(p.post(request("a", "ping").toString(), s, null).body!!) + assertEquals("a", r.get("id")) + assertEquals(0, r.getJSONObject("result").length()) + } + + @Test fun toolsListDescribesEveryTool() { + val p = protocol() + val s = session(p) + val tools = JSONObject(p.post(request(2, "tools/list").toString(), s, null).body!!).getJSONObject("result").getJSONArray("tools") + val names = (0 until tools.length()).map { tools.getJSONObject(it).getString("name") } + assertEquals(listOf("echo", "needs_person", "crashes", "link_fails"), names) + val first = tools.getJSONObject(0) + assertEquals("object", first.getJSONObject("inputSchema").getString("type")) + assertTrue(first.getJSONObject("annotations").getBoolean("readOnlyHint")) + assertFalse(tools.getJSONObject(1).getJSONObject("annotations").getBoolean("readOnlyHint")) + } + + @Test fun toolCallReturnsStructuredContent() { + val p = protocol() + val s = session(p) + val result = call(p, s, "echo", JSONObject().put("x", 3)).getJSONObject("result") + assertFalse(result.getBoolean("isError")) + assertEquals(3, result.getJSONObject("structuredContent").getJSONObject("got").getInt("x")) + assertEquals("text", result.getJSONArray("content").getJSONObject(0).getString("type")) + } + + @Test fun toolResultTextIsCompactSoTheResultIsNotSentTwiceInFull() { + val p = protocol() + val s = session(p) + val result = call(p, s, "echo", JSONObject().put("x", 3)).getJSONObject("result") + val text = result.getJSONArray("content").getJSONObject(0).getString("text") + assertFalse(text, text.contains("\n")) + assertEquals(result.getJSONObject("structuredContent").toString(), text) + } + + @Test fun toolCallWithoutArgumentsGetsEmptyObject() { + val p = protocol() + val s = session(p) + val got = call(p, s, "echo").getJSONObject("result").getJSONObject("structuredContent").getJSONObject("got") + assertEquals(0, got.length()) + } + + @Test fun toolErrorIsAResultWithIsError() { + val p = protocol() + val s = session(p) + val result = call(p, s, "needs_person").getJSONObject("result") + assertTrue(result.getBoolean("isError")) + assertTrue( + result + .getJSONArray("content") + .getJSONObject(0) + .getString("text") + .contains("person is needed"), + ) + assertTrue(result.getJSONObject("structuredContent").getBoolean("person_needed")) + } + + @Test fun unexpectedExceptionIsAResultWithIsError() { + val p = protocol() + val s = session(p) + val result = call(p, s, "crashes").getJSONObject("result") + assertTrue(result.getBoolean("isError")) + assertTrue( + result + .getJSONArray("content") + .getJSONObject(0) + .getString("text") + .contains("boom"), + ) + } + + @Test fun unknownToolAndBadArgumentsAreInvalidParams() { + val p = protocol() + val s = session(p) + assertEquals(McpProtocol.INVALID_PARAMS, call(p, s, "nope").getJSONObject("error").getInt("code")) + val bad = request(3, "tools/call", JSONObject().put("name", "echo").put("arguments", JSONArray())).toString() + assertEquals(McpProtocol.INVALID_PARAMS, JSONObject(p.post(bad, s, null).body!!).getJSONObject("error").getInt("code")) + } + + @Test fun unknownMethodIsMethodNotFound() { + val p = protocol() + val s = session(p) + val r = JSONObject(p.post(request(4, "resources/list").toString(), s, null).body!!) + assertEquals(McpProtocol.METHOD_NOT_FOUND, r.getJSONObject("error").getInt("code")) + } + + @Test fun badJsonIsAParseError() { + val r = protocol().post("{not json", null, null) + assertEquals(400, r.status) + assertEquals(McpProtocol.PARSE_ERROR, JSONObject(r.body!!).getJSONObject("error").getInt("code")) + } + + @Test fun batchGetsAnArrayOfReplies() { + val p = protocol() + val s = session(p) + val batch = + JSONArray() + .put(request(1, "ping")) + .put(JSONObject().put("jsonrpc", "2.0").put("method", "notifications/x")) + .put(request(2, "tools/list")) + val r = p.post(batch.toString(), s, null) + assertEquals(200, r.status) + val replies = JSONArray(r.body!!) + assertEquals(2, replies.length()) + assertEquals(1, replies.getJSONObject(0).getInt("id")) + assertEquals(2, replies.getJSONObject(1).getInt("id")) + } + + @Test fun initializeMustBeAlone() { + val batch = JSONArray().put(JSONObject(initBody())).put(request(2, "ping")) + assertEquals(400, protocol().post(batch.toString(), null, null).status) + } + + @Test fun deleteEndsTheSession() { + val p = protocol() + val s = session(p) + assertEquals(204, p.delete(s).status) + assertEquals(404, p.delete(s).status) + assertEquals(404, p.post(request(1, "ping").toString(), s, null).status) + } + + @Test fun oldestSessionIsDroppedBeyondTheLimit() { + val p = protocol(maxSessions = 2) + val a = session(p) + val b = session(p) + p.post(request(1, "ping").toString(), a, null) // a is now the most recently used + val c = session(p) // drops b + assertEquals(2, p.sessionCount()) + assertEquals(200, p.post(request(1, "ping").toString(), a, null).status) + assertEquals(404, p.post(request(1, "ping").toString(), b, null).status) + assertEquals(200, p.post(request(1, "ping").toString(), c, null).status) + } + + @Test fun endAllEndsEverySession() { + val p = protocol() + val s = session(p) + p.endAll() + assertEquals(0, p.sessionCount()) + assertEquals(404, p.post(request(1, "ping").toString(), s, null).status) + } + + // ---------------------------------------------------------------- the HTTP side + + private val token = "secret-token" + private var server: McpHttp? = null + + @After fun stopServer() { + server?.stop() + server = null + } + + private fun startServer(): Int { + val s = McpHttp(protocol(), { token }, port = 0) + s.start() + server = s + return s.boundPort + } + + private class Answer( + val status: Int, + val headers: Map, + val body: String, + ) + + /** One request on its own connection; [headers] may replace Host. */ + private fun http( + port: Int, + method: String, + path: String = McpHttp.PATH, + headers: Map = emptyMap(), + body: String? = null, + ): Answer { + Socket("127.0.0.1", port).use { s -> + s.soTimeout = 5000 + val bytes = body?.toByteArray(Charsets.UTF_8) ?: ByteArray(0) + val all = linkedMapOf("Host" to "127.0.0.1:$port", "Connection" to "close") + all.putAll(headers) + if (body != null) { + all["Content-Type"] = "application/json" + all["Content-Length"] = bytes.size.toString() + } + val head = StringBuilder("$method $path HTTP/1.1\r\n") + for ((k, v) in all) { + head + .append(k) + .append(": ") + .append(v) + .append("\r\n") + } + head.append("\r\n") + s.getOutputStream().apply { + write(head.toString().toByteArray(Charsets.ISO_8859_1)) + write(bytes) + flush() + } + val text = s.getInputStream().readBytes().toString(Charsets.UTF_8) + val lines = text.substringBefore("\r\n\r\n").split("\r\n") + val status = lines[0].split(' ')[1].toInt() + val h = lines.drop(1).associate { it.substringBefore(':').trim().lowercase() to it.substringAfter(':').trim() } + return Answer(status, h, text.substringAfter("\r\n\r\n", "")) + } + } + + private val auth get() = mapOf("Authorization" to "Bearer $token") + + @Test fun tokenIs32BytesUrlSafeAndNew() { + val a = McpHttp.newToken() + assertEquals(43, a.length) + assertTrue(a.matches(Regex("[A-Za-z0-9_-]{43}"))) + assertTrue(a != McpHttp.newToken()) + } + + @Test fun base64MatchesTheJdkEncoder() { + for (n in 0..40) { + val bytes = ByteArray(n) { (it * 37 + 250).toByte() } + assertEquals( + java.util.Base64 + .getEncoder() + .encodeToString(bytes), + McpHttp.base64(bytes, urlSafe = false, pad = true), + ) + assertEquals( + java.util.Base64 + .getUrlEncoder() + .withoutPadding() + .encodeToString(bytes), + McpHttp.base64(bytes, urlSafe = true, pad = false), + ) + } + } + + @Test fun httpPortZeroBindsAFreePortOnLoopback() { + val port = startServer() + assertTrue(port in 1..65535) + assertEquals("127.0.0.1", java.net.Socket("127.0.0.1", port).use { it.inetAddress.hostAddress }) + } + + @Test fun httpNinthConnectionIsRefused() { + val port = startServer() + val held = (1..McpHttp.MAX_CONNECTIONS).map { Socket("127.0.0.1", port) } + try { + Thread.sleep(300) + Socket("127.0.0.1", port).use { s -> + s.soTimeout = 5000 + val text = s.getInputStream().readBytes().toString(Charsets.UTF_8) + assertTrue(text.startsWith("HTTP/1.1 503")) + } + } finally { + held.forEach { it.close() } + } + } + + @Test fun httpNeedsTheToken() { + val port = startServer() + val none = http(port, "POST", body = initBody()) + assertEquals(401, none.status) + assertNotNull(none.headers["www-authenticate"]) + assertEquals(401, http(port, "POST", headers = mapOf("Authorization" to "Bearer wrong"), body = initBody()).status) + assertFalse("the token never appears in a refusal", none.body.contains(token)) + } + + @Test fun httpInitializeWithTheToken() { + val port = startServer() + val r = http(port, "POST", headers = auth, body = initBody()) + assertEquals(200, r.status) + assertNotNull(r.headers["mcp-session-id"]) + assertEquals("2025-06-18", JSONObject(r.body).getJSONObject("result").getString("protocolVersion")) + } + + @Test fun httpLocalhostHostIsAccepted() { + val port = startServer() + assertEquals(200, http(port, "POST", headers = auth + ("Host" to "localhost:$port"), body = initBody()).status) + } + + @Test fun httpOtherHostIsRefused() { + val port = startServer() + assertEquals(403, http(port, "POST", headers = auth + ("Host" to "evil.example"), body = initBody()).status) + assertEquals(403, http(port, "POST", headers = auth + ("Host" to "127.0.0.1:x"), body = initBody()).status) + } + + @Test fun httpWebPageOriginIsRefused() { + val port = startServer() + assertEquals(403, http(port, "POST", headers = auth + ("Origin" to "https://evil.example"), body = initBody()).status) + assertEquals(200, http(port, "POST", headers = auth + ("Origin" to "http://localhost:3000"), body = initBody()).status) + } + + @Test fun httpGetIsNotAllowed() { + val port = startServer() + val r = http(port, "GET", headers = auth) + assertEquals(405, r.status) + assertEquals("POST, DELETE", r.headers["allow"]) + } + + @Test fun httpOtherPathIsNotFound() { + assertEquals(404, http(startServer(), "POST", path = "/other", headers = auth, body = initBody()).status) + } + + @Test fun httpWrongContentTypeIsRefused() { + val port = startServer() + Socket("127.0.0.1", port).use { s -> + val body = initBody().toByteArray() + s.getOutputStream().write( + ( + "POST /mcp HTTP/1.1\r\nHost: 127.0.0.1\r\nAuthorization: Bearer $token\r\nConnection: close\r\n" + + "Content-Type: text/plain\r\nContent-Length: ${body.size}\r\n\r\n" + ).toByteArray() + body, + ) + val text = s.getInputStream().readBytes().toString(Charsets.UTF_8) + assertTrue(text.startsWith("HTTP/1.1 415")) + } + } + + @Test fun httpAnErrorInAToolIsA500AndTheServerKeepsServing() { + val port = startServer() + val s = http(port, "POST", headers = auth, body = initBody()).headers["mcp-session-id"]!! + val params = JSONObject().put("name", "link_fails") + val r = http(port, "POST", headers = auth + ("Mcp-Session-Id" to s), body = request(3, "tools/call", params).toString()) + assertEquals(500, r.status) + assertTrue(r.body, r.body.contains("no library")) + assertEquals(200, http(port, "POST", headers = auth + ("Mcp-Session-Id" to s), body = request(4, "ping").toString()).status) + } + + @Test fun httpAcceptBacksOffAfterFailuresAndKeepsServing() { + val attempts = java.util.Collections.synchronizedList(mutableListOf()) + val failures = 4 + val flaky = + object : java.net.ServerSocket() { + override fun accept(): Socket { + attempts += System.nanoTime() + if (attempts.size <= failures) throw java.io.IOException("Too many open files") + return super.accept() + } + } + val s = McpHttp(protocol(), { token }, port = 0, socketFactory = { flaky }) + s.start() + server = s + Thread.sleep(1_200) // 100 + 200 + 400 ms of sleeps, then the fourth failure sleeps 800 ms + // a spinning loop would have made thousands of attempts by now + assertTrue("attempts so far: ${attempts.size}", attempts.size in 4..6) + val gaps = attempts.zipWithNext { a, b -> (b - a) / 1_000_000 } + assertTrue("gaps $gaps", gaps[0] >= 90 && gaps[1] >= 190 && gaps[2] >= 390) + assertEquals(200, http(s.boundPort, "POST", headers = auth, body = initBody()).status) + } + + @Test fun httpSessionThenDelete() { + val port = startServer() + val s = http(port, "POST", headers = auth, body = initBody()).headers["mcp-session-id"]!! + val ping = request(1, "ping").toString() + assertEquals(200, http(port, "POST", headers = auth + ("Mcp-Session-Id" to s), body = ping).status) + assertEquals(204, http(port, "DELETE", headers = auth + ("Mcp-Session-Id" to s)).status) + assertEquals(404, http(port, "POST", headers = auth + ("Mcp-Session-Id" to s), body = ping).status) + } + + @Test fun clientsThatTrickleOrNeverAuthenticateAreDroppedAndDoNotBlockTheRealOne() { + val s = McpHttp(protocol(), { token }, port = 0, headDeadlineMs = 600) + s.start() + server = s + val port = s.boundPort + val started = System.nanoTime() + val held = mutableListOf() + try { + repeat(McpHttp.MAX_CONNECTIONS) { i -> + val c = Socket("127.0.0.1", port) + c.soTimeout = 5000 + held += c + // half of them send a byte now and then, which resets a per-read timeout but not the deadline + if (i % 2 == 0) { + c.getOutputStream().apply { + write("POST /mcp HTTP/1.1\r\nHost: 127.0.0.1\r\n".toByteArray()) + flush() + } + } + } + // the slots are full: a real client is refused with 503 + assertEquals(503, http(port, "POST", headers = auth, body = initBody()).status) + val trickler = + Thread { + try { + repeat(20) { + Thread.sleep(200) + held[0].getOutputStream().apply { + write("X-Slow: 1\r\n".toByteArray()) + flush() + } + } + } catch (ignored: Exception) { + } + } + trickler.isDaemon = true + trickler.start() + // every held connection is closed by the server within the deadline plus slack + for (c in held) assertEquals(-1, c.getInputStream().read()) + val elapsedMs = (System.nanoTime() - started) / 1_000_000 + assertTrue("took $elapsedMs ms", elapsedMs < 3_000) + assertEquals(200, http(port, "POST", headers = auth, body = initBody()).status) + } finally { + held.forEach { it.close() } + } + } + + @Test fun anUnauthenticatedRequestIsRefusedBeforeItsBodyIsRead() { + val port = startServer() + Socket("127.0.0.1", port).use { c -> + c.soTimeout = 3000 + c.getOutputStream().apply { + write("POST /mcp HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Length: 1000\r\n\r\n".toByteArray()) + flush() + } + // the body never comes, yet the answer does + val line = c.getInputStream().bufferedReader().readLine() + assertTrue(line, line.startsWith("HTTP/1.1 401")) + } + } +} diff --git a/subprojects/sqlite-debugger/src/test/jni/Helper.java b/subprojects/sqlite-debugger/src/test/jni/Helper.java new file mode 100644 index 0000000000..e283c57768 --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/jni/Helper.java @@ -0,0 +1,8 @@ +/** Opens and uses "app" connections with the same libsqlite3x.so the debugger runs in. */ +public class Helper { + static native long open(String path); + + static native int exec(long db, String sql); + + static native int close(long db); +} diff --git a/subprojects/sqlite-debugger/src/test/jni/JniQueryTest.java b/subprojects/sqlite-debugger/src/test/jni/JniQueryTest.java new file mode 100644 index 0000000000..776287b498 --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/jni/JniQueryTest.java @@ -0,0 +1,262 @@ +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import org.appdevforall.sqlitedebugger.engine.DebugEngineException; +import org.appdevforall.sqlitedebugger.engine.NativeBridge; + +/** + * The real JNI query path (sqlitedbg_jni.c over sqlitedbg.c) on a host JVM, against a SQLite built + * like requery's. test.sh builds libsqlite3x.so, libsqlitedbg.so (no SQLITE_CORE) and libhosthelper.so + * and runs this with them. It calls NativeBridge directly; the Kotlin above it is covered by the + * JVM unit tests with a fake engine. + */ +public class JniQueryTest { + static int failures, checks; + + static void check(boolean ok, String what) { + checks++; + if (!ok) { + failures++; + System.out.println("FAIL " + what); + } + } + + static void eq(Object a, Object b, String what) { + check(Objects.equals(a, b), what + ": " + show(a) + " vs " + show(b)); + } + + static String show(Object o) { + return o instanceof byte[] ? Arrays.toString((byte[]) o) : String.valueOf(o); + } + + static final long FAR = Long.MAX_VALUE / 2; + static final int MAX = 1 << 30; + + /** What NativeBridge.query returns, unpacked. */ + static class Result { + String[] columns; + List rows; + long truncated; + boolean hasMore; + Set cut = new HashSet<>(); + } + + @SuppressWarnings("unchecked") + static Result query(long h, String sql, Object[] args, long offset, int maxRows, int maxText, int maxBlob, int maxTotal, + long deadline) throws DebugEngineException { + Object[] r = NativeBridge.query(h, sql, args, offset, maxRows, maxText, maxBlob, maxTotal, deadline); + Result out = new Result(); + out.columns = (String[]) r[0]; + out.rows = (List) r[1]; + long[] counts = (long[]) r[2]; + out.truncated = counts[0]; + out.hasMore = counts[1] != 0; + eq(counts.length, 2 + (int) counts[0], "one index per cut cell"); + for (int i = 2; i < counts.length; i++) out.cut.add(counts[i]); + return out; + } + + static Result q(long h, String sql) throws DebugEngineException { + return query(h, sql, new Object[0], 0, 1000, 4096, 1024, MAX, FAR); + } + + static int codeOf(long h, String sql, Object[] args, long deadline) { + try { + query(h, sql, args, 0, 100, 4096, 1024, MAX, deadline); + return 0; + } catch (DebugEngineException e) { + return e.sqliteCode; + } + } + + static Set set(Long... v) { + return new HashSet<>(Arrays.asList(v)); + } + + public static void main(String[] a) throws Exception { + System.loadLibrary("sqlite3x"); + System.loadLibrary("sqlitedbg"); + System.loadLibrary("hosthelper"); + + try { + NativeBridge.openServer("/nonexistent/x.db".getBytes(StandardCharsets.UTF_8), false); + check(false, "open before install"); + } catch (DebugEngineException e) { + eq(e.sqliteCode, 21, "SQLITE_MISUSE before any app connection"); + } + eq(NativeBridge.install(), 0, "install"); + eq(NativeBridge.install(), 0, "install twice"); + + Path dir = Files.createTempDirectory("jniquery"); + String path = dir.resolve("app.db").toString(); + long app = Helper.open(path); + eq(Helper.exec(app, "CREATE TABLE t(id INTEGER, r REAL, s TEXT, b BLOB)"), 0, "create"); + eq(Helper.exec(app, "INSERT INTO t VALUES(1, 1.5, 'one', x'0102'),(2, NULL, 'héllo €', x''),(3, 3e999, NULL, NULL)"), 0, + "insert"); + eq(Helper.exec(app, "WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM c WHERE x<300) INSERT INTO t(id) SELECT x+10 FROM c"), + 0, "insert 300"); + + Object[] conns = NativeBridge.connections(); + eq(((String[]) conns[1]).length, 1, "one app connection"); + eq(((String[]) conns[1])[0], path, "its path"); + + byte[] bytes = path.getBytes(StandardCharsets.UTF_8); + long srv = NativeBridge.openServer(bytes, false); + check(srv != 0, "server handle"); + + // value types + Result r = q(srv, "SELECT id, r, s, b FROM t WHERE id < 10 ORDER BY id"); + eq(Arrays.asList(r.columns), Arrays.asList("id", "r", "s", "b"), "columns"); + eq(r.rows.size(), 3, "rows"); + eq(r.rows.get(0)[0], 1L, "long"); + eq(r.rows.get(0)[1], 1.5, "double"); + eq(r.rows.get(0)[2], "one", "text"); + check(Arrays.equals((byte[]) r.rows.get(0)[3], new byte[] {1, 2}), "blob"); + eq(r.rows.get(1)[1], null, "null"); + eq(r.rows.get(1)[2], "héllo €", "utf8 text"); + eq(((byte[]) r.rows.get(1)[3]).length, 0, "empty blob"); + eq(r.rows.get(2)[1], Double.POSITIVE_INFINITY, "infinity"); + eq(r.hasMore, false, "no more"); + eq(r.truncated, 0L, "nothing cut"); + + // paging: 300 rows with id >= 10 + String all = "SELECT id FROM t WHERE id >= 10 ORDER BY id"; + r = query(srv, all, new Object[0], 0, 100, 4096, 1024, MAX, FAR); + eq(r.rows.size(), 100, "page 1"); + eq(r.hasMore, true, "has more after a full page"); + r = query(srv, all, new Object[0], 100, 100, 4096, 1024, MAX, FAR); + eq(r.rows.get(0)[0], 111L, "page 2 starts after the offset"); + r = query(srv, all, new Object[0], 200, 100, 4096, 1024, MAX, FAR); + eq(r.rows.size(), 100, "page 3"); + eq(r.hasMore, false, "exactly the last row of the last page: no more"); + r = query(srv, all, new Object[0], 299, 100, 4096, 1024, MAX, FAR); + eq(r.rows.size(), 1, "last page"); + eq(r.rows.get(0)[0], 310L, "last id"); + r = query(srv, all, new Object[0], 5000, 100, 4096, 1024, MAX, FAR); + eq(r.rows.size(), 0, "past the end"); + eq(r.hasMore, false, "past the end has no more"); + + // M4: the total-bytes budget. 8 bytes per number: 3 numbers = 24 a row, 4 rows fit in 100. + String nums = "WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM c WHERE x<50) SELECT x, x, x FROM c"; + r = query(srv, nums, new Object[0], 0, 1000, 4096, 1024, 100, FAR); + eq(r.rows.size(), 4, "budget 100 holds 4 rows of 24"); + eq(r.hasMore, true, "budget reached: has more"); + r = query(srv, nums, new Object[0], 4, 1000, 4096, 1024, 100, FAR); + eq(r.rows.get(0)[0], 5L, "the next page starts where the budget stopped"); + r = query(srv, nums, new Object[0], 0, 1000, 4096, 1024, 24 * 50, FAR); + eq(r.rows.size(), 50, "a budget of exactly the whole result holds it"); + eq(r.hasMore, false, "and has no more"); + // text and blob count the bytes kept, not the bytes stored + String fat = "WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM c WHERE x<5) " + + "SELECT substr(hex(zeroblob(1000)), 1, 1000), zeroblob(500) FROM c"; + r = query(srv, fat, new Object[0], 0, 1000, 4096, 1024, 3200, FAR); + eq(r.rows.size(), 2, "1500 bytes a row: 2 rows in 3200"); + eq(r.hasMore, true, "fat rows: has more"); + r = query(srv, fat, new Object[0], 0, 1000, 100, 100, 3200, FAR); + eq(r.rows.size(), 5, "cells cut to 100 bytes cost 200 a row: all 5 fit"); + // the first row always goes in, so a page makes progress + r = query(srv, fat, new Object[0], 0, 1000, 4096, 1024, 10, FAR); + eq(r.rows.size(), 1, "one row over the budget is still returned"); + eq(r.hasMore, true, "and more follows"); + + // L11: per-cell truncation, exactly at the limit is whole + r = query(srv, "SELECT x'01020304', x'0102030405', 'abcd', 'abcde'", new Object[0], 0, 10, 4, 4, MAX, FAR); + eq(((byte[]) r.rows.get(0)[0]).length, 4, "blob at the limit"); + eq(((byte[]) r.rows.get(0)[1]).length, 4, "blob over the limit is cut"); + eq(r.rows.get(0)[2], "abcd", "text at the limit"); + eq(r.rows.get(0)[3], "abcd", "text over the limit is cut"); + eq(r.truncated, 2L, "two cells cut"); + eq(r.cut, set(1L, 3L), "cut cells are the over-limit blob and text, not the ones at the limit"); + r = query(srv, "SELECT x'01020304', x'0102030405', 'abcd', 'abcde' UNION ALL SELECT x'0102030405', x'01', 'abcde', 'a'", + new Object[0], 0, 10, 4, 4, MAX, FAR); + eq(r.cut, set(1L, 3L, 4L, 6L), "indexes are row * columns + column"); + eq(r.truncated, 4L, "four cells cut"); + // a character is never split: 'a' + 3 emoji of 4 bytes, 6 bytes allowed + r = query(srv, "SELECT 'a' || '😀😀😀', x'0102030405'", new Object[0], 0, 10, 6, 3, MAX, FAR); + eq(r.rows.get(0)[0], "a😀", "text cut at a character boundary"); + eq(((byte[]) r.rows.get(0)[1]).length, 3, "blob cut"); + eq(r.cut, set(0L, 1L), "both cut"); + // a row left out by the budget takes its cut cells with it + r = query(srv, "SELECT 'abcde' UNION ALL SELECT 'abcde'", new Object[0], 0, 10, 4, 4, 6, FAR); + eq(r.rows.size(), 1, "the second row would pass the budget"); + eq(r.cut, set(0L), "only the kept row's cut cell is reported"); + eq(r.truncated, 1L, "and counted"); + + // bound values of every type + r = query(srv, "SELECT ?, ?, ?, ?, ?", new Object[] {7L, 2.25, "a😀b", new byte[] {9, 8, 7}, null}, 0, 10, 4096, 1024, + MAX, FAR); + Object[] row = r.rows.get(0); + eq(row[0], 7L, "arg long"); + eq(row[1], 2.25, "arg double"); + eq(row[2], "a😀b", "arg text"); + check(Arrays.equals((byte[]) row[3], new byte[] {9, 8, 7}), "arg blob"); + eq(row[4], null, "arg null"); + eq(codeOf(srv, "SELECT ?", new Object[0], FAR), 25, "too few args: SQLITE_RANGE, not a NULL parameter"); + eq(codeOf(srv, "SELECT ?, ?", new Object[] {1L}, FAR), 25, "one arg of two: SQLITE_RANGE"); + eq(codeOf(srv, "SELECT 1", new Object[] {1L}, FAR), 25, "an arg for no parameter: SQLITE_RANGE"); + try { + query(srv, "SELECT ?, ?", new Object[] {1L}, 0, 100, 4096, 1024, MAX, FAR); + } catch (DebugEngineException e) { + check(e.getMessage().contains("2 parameter") && e.getMessage().contains("1 argument"), "the message names both counts: " + e.getMessage()); + } + eq(codeOf(srv, "SELECT ?", new Object[] {1L, 2L}, FAR), 25, "too many args: SQLITE_RANGE"); + eq(codeOf(srv, "SELECT ?", new Object[] {new Object()}, FAR), 21, "unsupported arg type: SQLITE_MISUSE"); + eq(codeOf(srv, "SELEKT 1", new Object[0], FAR), 1, "syntax error: SQLITE_ERROR"); + + // a refused write is SQLITE_AUTH, and the data is untouched + eq(codeOf(srv, "INSERT INTO t(id) VALUES(5)", new Object[0], FAR), 23, "write refused: SQLITE_AUTH"); + eq(codeOf(srv, "DELETE FROM t", new Object[0], FAR), 23, "delete refused: SQLITE_AUTH"); + eq(q(srv, "SELECT count(*) FROM t WHERE id = 5").rows.get(0)[0], 0L, "nothing written"); + + // H3: a transaction cannot be opened, so the app is never locked out + for (String sql : new String[] {"BEGIN", "BEGIN IMMEDIATE", "SAVEPOINT s", "PRAGMA locking_mode=EXCLUSIVE", "PRAGMA busy_timeout=60000"}) { + eq(codeOf(srv, sql, new Object[0], FAR), 23, sql + " refused: SQLITE_AUTH"); + q(srv, "SELECT count(*) FROM t"); + eq(Helper.exec(app, "INSERT INTO t(id) VALUES(4242)"), 0, "the app can write after " + sql); + Helper.exec(app, "DELETE FROM t WHERE id = 4242"); + } + + // the deadline stops a statement that never ends, and the connection stays usable + long t0 = System.currentTimeMillis(); + eq(codeOf(srv, "WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM c) SELECT count(*) FROM c", new Object[0], t0 + 300), 9, + "deadline: SQLITE_INTERRUPT"); + long took = System.currentTimeMillis() - t0; + check(took >= 250 && took < 3000, "the deadline stopped it in " + took + " ms"); + eq(codeOf(srv, "SELECT 1", new Object[0], t0 - 1), 9, "a deadline already past: SQLITE_INTERRUPT"); + eq(q(srv, "SELECT 41 + 1").rows.get(0)[0], 42L, "usable after the deadline"); + eq(Helper.exec(app, "INSERT INTO t(id) VALUES(4243)"), 0, "the app can write after an interrupted query"); + Helper.exec(app, "DELETE FROM t WHERE id = 4243"); + + // the server connection is not listed + eq(((String[]) NativeBridge.connections()[1]).length, 1, "server not listed"); + NativeBridge.close(srv); + + // the backstop: a writable connection that opens a transaction does not keep it + long w = NativeBridge.openServer(bytes, true); + q(w, "BEGIN IMMEDIATE"); + eq(Helper.exec(app, "INSERT INTO t(id) VALUES(4244)"), 0, "the app can write after a writable BEGIN IMMEDIATE"); + Helper.exec(app, "DELETE FROM t WHERE id = 4244"); + q(w, "BEGIN"); + q(w, "INSERT INTO t(id) VALUES(4245)"); + eq(q(w, "SELECT count(*) FROM t WHERE id = 4245").rows.get(0)[0], 1L, "an autocommit insert on the writable connection stays"); + NativeBridge.close(w); + Helper.exec(app, "DELETE FROM t WHERE id = 4245"); + + try { + NativeBridge.openServer(dir.resolve("none.db").toString().getBytes(StandardCharsets.UTF_8), false); + check(false, "missing file"); + } catch (DebugEngineException e) { + eq(e.sqliteCode, 14, "SQLITE_CANTOPEN"); + } + + eq(Helper.close(app), 0, "close the app connection"); + eq(((String[]) NativeBridge.connections()[1]).length, 0, "all freed"); + System.out.println(checks + " checks, " + failures + " failures"); + System.exit(failures == 0 ? 0 : 1); + } +} diff --git a/subprojects/sqlite-debugger/src/test/jni/helper.c b/subprojects/sqlite-debugger/src/test/jni/helper.c new file mode 100644 index 0000000000..9fe88d041a --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/jni/helper.c @@ -0,0 +1,27 @@ +// The "app" side of the JNI test: plain sqlite3 calls into libsqlite3x.so, as requery's do. +#include +#include +#include "sqlite3.h" + +JNIEXPORT jlong JNICALL Java_Helper_open(JNIEnv *e, jclass c, jstring p) { + (void)c; + const char *s = (*e)->GetStringUTFChars(e, p, 0); + sqlite3 *db = 0; + sqlite3_open(s, &db); + (*e)->ReleaseStringUTFChars(e, p, s); + return (jlong)(intptr_t)db; +} + +JNIEXPORT jint JNICALL Java_Helper_exec(JNIEnv *e, jclass c, jlong db, jstring q) { + (void)c; + const char *s = (*e)->GetStringUTFChars(e, q, 0); + int rc = sqlite3_exec((sqlite3 *)(intptr_t)db, s, 0, 0, 0); + (*e)->ReleaseStringUTFChars(e, q, s); + return rc; +} + +JNIEXPORT jint JNICALL Java_Helper_close(JNIEnv *e, jclass c, jlong db) { + (void)e; + (void)c; + return sqlite3_close((sqlite3 *)(intptr_t)db); +} diff --git a/subprojects/sqlite-debugger/src/test/jni/org/appdevforall/sqlitedebugger/engine/DebugEngineException.java b/subprojects/sqlite-debugger/src/test/jni/org/appdevforall/sqlitedebugger/engine/DebugEngineException.java new file mode 100644 index 0000000000..bbdbdf276f --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/jni/org/appdevforall/sqlitedebugger/engine/DebugEngineException.java @@ -0,0 +1,11 @@ +package org.appdevforall.sqlitedebugger.engine; + +/** Stands in for DebugEngineException in DebugEngine.kt: the JNI layer constructs it as (int, String). */ +public class DebugEngineException extends Exception { + public final int sqliteCode; + + public DebugEngineException(int sqliteCode, String message) { + super(message); + this.sqliteCode = sqliteCode; + } +} diff --git a/subprojects/sqlite-debugger/src/test/jni/org/appdevforall/sqlitedebugger/engine/NativeBridge.java b/subprojects/sqlite-debugger/src/test/jni/org/appdevforall/sqlitedebugger/engine/NativeBridge.java new file mode 100644 index 0000000000..bc4ee25852 --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/jni/org/appdevforall/sqlitedebugger/engine/NativeBridge.java @@ -0,0 +1,26 @@ +package org.appdevforall.sqlitedebugger.engine; + +/** + * Stands in for NativeBridge.kt in the host JNI test (test.sh): the same class name and native + * signatures, which sqlitedbg_jni.c registers by name in JNI_OnLoad, so a Java-only toolchain can + * run the real C. test.sh checks that the native names here match the ones in NativeBridge.kt. + */ +public final class NativeBridge { + private NativeBridge() {} + + public static native int install(); + + public static native Object[] connections(); + + public static native void readEvents(long after, byte[] buf, long[] out); + + public static native void setClientConnected(boolean connected); + + public static native long openServer(byte[] path, boolean writable) throws DebugEngineException; + + public static native Object[] query(long handle, String sql, Object[] args, long offset, int maxRows, int maxTextBytes, + int maxBlobBytes, int maxTotalBytes, long deadlineMillis) + throws DebugEngineException; + + public static native void close(long handle); +} diff --git a/subprojects/sqlite-debugger/src/test/resources/org/appdevforall/sqlitedebugger/engine/events.bin b/subprojects/sqlite-debugger/src/test/resources/org/appdevforall/sqlitedebugger/engine/events.bin new file mode 100644 index 0000000000..0441b91500 Binary files /dev/null and b/subprojects/sqlite-debugger/src/test/resources/org/appdevforall/sqlitedebugger/engine/events.bin differ diff --git a/subprojects/sqlite-debugger/src/test/resources/org/appdevforall/sqlitedebugger/engine/events.tsv b/subprojects/sqlite-debugger/src/test/resources/org/appdevforall/sqlitedebugger/engine/events.tsv new file mode 100644 index 0000000000..4405849603 --- /dev/null +++ b/subprojects/sqlite-debugger/src/test/resources/org/appdevforall/sqlitedebugger/engine/events.tsv @@ -0,0 +1,21 @@ +426616 1 1 10289 1794545527187098386 -1 0 CREATE TABLE fx(a) +426617 2 1 10289 1794545527188573325 1000000 0 CREATE TABLE fx(a) +426618 1 1 10289 1794545527188596333 -1 0 INSERT INTO fx VALUES(?) +426619 2 1 10289 1794545527189296629 1000000 0 INSERT INTO fx VALUES(?) +426620 1 1 10289 1794545527189303467 -1 2 INSERT INTO fx VALUES(6) +426621 2 1 10289 1794545527190447975 1000000 2 INSERT INTO fx VALUES(6) +426622 1 1 10289 1794545527190472305 -1 0 SELECT 'héllo € 😀' +426623 2 1 10289 1794545527190474851 0 0 SELECT 'héllo € 😀' +426624 1 1 10289 1794545527190557276 -1 0 CREATE TABLE fxlog(a) +426625 2 1 10289 1794545527191607925 1000000 0 CREATE TABLE fxlog(a) +426626 1 1 10289 1794545527191694489 -1 1 CREATE TRIGGER fxt AFTER INSERT ON fx BEGIN INSERT INTO fxlog VA +426627 2 1 10289 1794545527192759261 1000000 1 CREATE TRIGGER fxt AFTER INSERT ON fx BEGIN INSERT INTO fxlog VA +426628 1 1 10289 1794545527192791863 -1 0 INSERT INTO fx VALUES(1) +426629 1 1 10289 1794545527192835643 -1 0 -- TRIGGER fxt +426630 1 1 10289 1794545527192836209 -1 0 -- INSERT INTO fxlog VALUES(new.a) +426631 2 1 10289 1794545527193623084 1000000 0 INSERT INTO fx VALUES(1) +426632 1 1 10289 1794545527193646993 -1 1 SELECT '€€€€€€€€€€€€€€€€€€ +426633 2 1 10289 1794545527193649365 0 1 SELECT '€€€€€€€€€€€€€€€€€€ +426634 1 3616 10289 1794545527193782816 -1 0 SELECT 1 +426635 2 3616 10289 1794545527193784200 0 0 SELECT 1 +426636 3 3616 10289 1794545527193785154 -1 0 diff --git a/subprojects/sqlite-debugger/test.sh b/subprojects/sqlite-debugger/test.sh new file mode 100755 index 0000000000..ce80e1702a --- /dev/null +++ b/subprojects/sqlite-debugger/test.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Host tests of the SQLite debugger's extension (ADFA-7135): src/test/c/sqlitedbg_test.c, with +# SQLite built from the amalgamation as requery builds it (SQLITE_THREADSAFE=2), under +# AddressSanitizer + UBSan and then ThreadSanitizer. +# Usage: test.sh [--update-fixture] [AMALGAMATION_TARBALL] (without the tarball, downloads it) +# --update-fixture rewrites src/test/resources/.../events.bin and events.tsv, which the JVM test +# of EventDecoder reads; without it the committed pair is checked against the +# current record layout instead. +# Then, if a JDK is on PATH (javac, java, jni.h), the JNI layer is built as on a device (no SQLITE_CORE, +# against a requery-like libsqlite3x.so) and src/test/jni/JniQueryTest.java runs the real query path. +# Needs gcc, curl, tar and sha256sum. +set -euo pipefail +HERE=$(cd "$(dirname "$0")" && pwd) + +# SQLite 3.49.1, the version of the vendored headers in ../sqlite-js/upstream/libs (the script +# checks they are the same). sqlite.org's own zip (sqlite-amalgamation-3490100.zip) is the usual +# source; this is the same amalgamation as bundled in the sqlean.py sdist, which PyPI serves where +# sqlite.org is not reachable. sqlean appends its own init code after SQLite's last line, which is +# cut off below. SHA256 is of the tarball, SQLITE3_C_SHA256 of the cut sqlite3.c. +VERSION=3.49.1 +SHA256=210d89989226b988d7d6391f837387d3b81e8cd608c997e0bd37826e395970e7 +URL=https://files.pythonhosted.org/packages/67/eb/ac95fab0bc4658124b4ec8fbc31fc494165ab4544606ae91b9a489907dad/sqlean_py-3.49.1.tar.gz +SQLITE3_C_SHA256=ff80c36ef1bb44eb357c7ff1d15be77540d41c28fb671088215a6cd12785c5d3 +SQLITE_SOURCE_ID="2025-02-18 13:38:58 873d4e274b4988d260ba8354a9718324a1c26187a4ab4c1cc0227c03d0f10e70" + +UPDATE=0 +if [ "${1:-}" = "--update-fixture" ]; then UPDATE=1; shift; fi + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +TARBALL=${1:-$WORK/sqlite-$VERSION.tar.gz} +[ -f "$TARBALL" ] || curl -fsSL -o "$TARBALL" "$URL" +echo "$SHA256 $TARBALL" | sha256sum -c --quiet - +mkdir "$WORK/src" +tar -xzf "$TARBALL" -C "$WORK/src" --strip-components=1 +AMAL=$WORK/src/sqlite +sed -i '/^\/\*.* End of sqlite3\.c \*/q' "$AMAL/sqlite3.c" +echo "$SQLITE3_C_SHA256 $AMAL/sqlite3.c" | sha256sum -c --quiet - +grep -q "SQLITE_SOURCE_ID \"$SQLITE_SOURCE_ID\"" "$AMAL/sqlite3.h" || { echo "unexpected SQLite source id"; exit 1; } + +LIBS=$HERE/../sqlite-js/upstream/libs +for h in sqlite3.h sqlite3ext.h; do + cmp -s "$AMAL/$h" "$LIBS/$h" || { echo "$h differs from $LIBS/$h"; exit 1; } +done + +build () { "$@" >"$WORK/build.log" 2>&1 || { cat "$WORK/build.log"; exit 1; }; } +CPP=$HERE/src/main/cpp +DEFS=(-DSQLITE_CORE -DSQLITE_THREADSAFE=2 -DSQLITE_DQS=0 -DSQLITE_DEFAULT_MEMSTATUS=0) + +run_with () { + local name=$1 sanitize=$2 + echo "# $name" + mkdir "$WORK/$name" + build gcc -O1 -g -w -fsanitize="$sanitize" "${DEFS[@]}" -I"$AMAL" -c "$AMAL/sqlite3.c" -o "$WORK/$name/sqlite3.o" + build gcc -O1 -g -Wall -Wextra -Werror -fsanitize="$sanitize" -fno-sanitize-recover=all "${DEFS[@]}" \ + -DSQLITEDBG_TEST -I"$AMAL" -I"$CPP" "$CPP/sqlitedbg.c" "$HERE/src/test/c/sqlitedbg_test.c" "$WORK/$name/sqlite3.o" \ + -o "$WORK/$name/test" -lpthread -ldl -lm + TSAN_OPTIONS=halt_on_error=1 "$WORK/$name/test" ${3:+"${@:3}"} +} + +RES=$HERE/src/test/resources/org/appdevforall/sqlitedebugger/engine +if [ "$UPDATE" = 1 ]; then + mkdir -p "$RES" + run_with asan-ubsan address,undefined --write-fixture "$RES/events.bin" "$RES/events.tsv" +else + run_with asan-ubsan address,undefined + "$WORK/asan-ubsan/test" --check-fixture "$RES/events.bin" "$RES/events.tsv" +fi +run_with tsan thread + +# The JNI query path on a host JVM. The Java stubs of NativeBridge and DebugEngineException under +# src/test/jni stand in for the Kotlin classes (no Kotlin compiler needed); the native names are +# checked against NativeBridge.kt below. +run_jni () { + local jdk + jdk=$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")") + [ -f "$jdk/include/jni.h" ] || { echo "# jni: skipped, no jni.h under $jdk"; return 0; } + echo "# jni" + local jni=$HERE/src/test/jni out=$WORK/jni + mkdir -p "$out/lib" "$out/classes" + local kotlin_names java_names + kotlin_names=$(sed -n 's/.*external fun \([A-Za-z]*\)(.*/\1/p' "$HERE/src/main/java/org/appdevforall/sqlitedebugger/engine/NativeBridge.kt" | sort | tr '\n' ' ') + java_names=$(sed -n 's/.*static native .* \([A-Za-z]*\)(.*/\1/p' "$jni/org/appdevforall/sqlitedebugger/engine/NativeBridge.java" | sort | tr '\n' ' ') + [ "$kotlin_names" = "$java_names" ] || { echo "NativeBridge.kt has ($kotlin_names) but the test stub has ($java_names)"; exit 1; } + local jinc=(-I"$jdk/include" -I"$jdk/include/linux") + # as requery builds it: everything exported, so dlsym finds sqlite3_auto_extension + build gcc -O1 -w -shared -fPIC -DSQLITE_THREADSAFE=2 -DSQLITE_DQS=0 -DSQLITE_DEFAULT_MEMSTATUS=0 -I"$AMAL" \ + "$AMAL/sqlite3.c" -o "$out/lib/libsqlite3x.so" -lpthread -ldl -lm + build gcc -O1 -Wall -Wextra -Werror -shared -fPIC -fvisibility=hidden -I"$AMAL" -I"$CPP" "${jinc[@]}" \ + "$CPP/sqlitedbg.c" "$CPP/sqlitedbg_jni.c" -Wl,--version-script="$CPP/sqlitedbg.map" \ + -o "$out/lib/libsqlitedbg.so" -lpthread -ldl + build gcc -O1 -Wall -Wextra -Werror -shared -fPIC -I"$AMAL" "${jinc[@]}" "$jni/helper.c" \ + -L"$out/lib" -lsqlite3x -Wl,-rpath,"$out/lib" -o "$out/lib/libhosthelper.so" + build javac -d "$out/classes" "$jni/org/appdevforall/sqlitedebugger/engine/NativeBridge.java" \ + "$jni/org/appdevforall/sqlitedebugger/engine/DebugEngineException.java" "$jni/Helper.java" "$jni/JniQueryTest.java" + LD_LIBRARY_PATH="$out/lib" java -Djava.library.path="$out/lib" -cp "$out/classes" JniQueryTest +} + +if command -v javac >/dev/null && command -v java >/dev/null; then + run_jni +else + echo "# jni: skipped, no JDK (javac and java) on PATH" +fi