feat: add opt-in resumable SQLite RBU support

Package.swift

@@ -73,6 +73,7 @@ let phoneVersion = (opsqliteConfig["iosSqlite"] as? Bool) == true
 let sqliteFlags = (opsqliteConfig["sqliteFlags"] as? String) ?? ""
 let fts5 = (opsqliteConfig["fts5"] as? Bool) == true
 let rtree = (opsqliteConfig["rtree"] as? Bool) == true
+let rbu = (opsqliteConfig["rbu"] as? Bool) == true
 let useSqliteVec = (opsqliteConfig["sqliteVec"] as? Bool) == true
 let tokenizers = (opsqliteConfig["tokenizers"] as? [String]) ?? []
 
@@ -96,6 +97,9 @@ if useTurso && useSqliteVec {
 if useTurso && useLibsql {
   fatalError("[OP-SQLITE] You cannot enable both libsql and turso backend.")
 }
+if rbu && (useSqlcipher || useLibsql || useTurso || phoneVersion) {
+  fatalError("[OP-SQLITE] RBU currently supports only the bundled vanilla SQLite backend.")
+}
 if !tokenizers.isEmpty && useTurso {
   fatalError("[OP-SQLITE] Tokenizers are not supported with turso backend. Please disable tokenizers or do not enable turso.")
 }
@@ -294,6 +298,10 @@ if useSqlcipher {
 }
 if fts5 { defines.append(("SQLITE_ENABLE_FTS5", "1")) }
 if rtree { defines.append(("SQLITE_ENABLE_RTREE", "1")) }
+if rbu {
+  print("[OP-SQLITE] RBU enabled")
+  defines.append(("SQLITE_ENABLE_RBU", "1"))
+}
 if phoneVersion { defines.append(("OP_SQLITE_USE_PHONE_VERSION", "1")) }
 if performanceMode {
   print("[OP-SQLITE] Performance mode enabled")

README.md

@@ -20,6 +20,7 @@ Some of the big supported features:
 - SQLCipher is supported as a compilation target
 - FTS5 plugin
 - Rtree plugin
+- Opt-in resumable bulk updates (RBU)
 - sqlite-vec plugin
 - Reactive queries
 - Custom tokenizers

android/CMakeLists.txt

@@ -44,6 +44,7 @@ add_library(
   ../cpp/OPSqlite.cpp
   ../cpp/OPUtils.cpp
   ../cpp/OPThreadPool.cpp
+  ../cpp/OPRBU.cpp
   ../cpp/OPSmartHostObject.cpp
   ../cpp/OPPreparedStatementHostObject.cpp
   ../cpp/OPDumbHostObject.cpp

android/build.gradle

@@ -60,6 +60,7 @@ def sqliteFlags = ""
 def enableFTS5 = false
 def useSqliteVec = false
 def enableRtree = false
+def enableRBU = false
 def tokenizers = []
 
 // On the example app, the package.json is located at the root of the project
@@ -100,13 +101,18 @@ if(opsqliteConfig) {
   useLibsql = !!opsqliteConfig["libsql"]
   useTurso = !!opsqliteConfig["turso"]
   enableRtree = !!opsqliteConfig["rtree"]
+  enableRBU = !!opsqliteConfig["rbu"]
   tokenizers = opsqliteConfig["tokenizers"] ? opsqliteConfig["tokenizers"] : []
 }
 
 if(useLibsql && useTurso) {
   throw new GradleException("[OP-SQLITE] Error: libsql and turso backends are mutually exclusive.")
 }
 
+if(enableRBU && (useSQLCipher || useLibsql || useTurso)) {
+  throw new GradleException("[OP-SQLITE] Error: RBU currently supports only the bundled vanilla SQLite backend.")
+}
+
 if(useSQLCipher) {
   println "[OP-SQLITE] using sqlcipher."
 } else if(useTurso) {
@@ -127,6 +133,10 @@ if(enableRtree) {
   println "[OP-SQLITE] RTree enabled"
 }
 
+if(enableRBU) {
+  println "[OP-SQLITE] RBU enabled"
+}
+
 if(useSqliteVec) {
   println "[OP-SQLITE] Sqlite-vec enabled"
 }
@@ -158,6 +168,9 @@ if (enableFTS5) {
 if (enableRtree) {
   defaultSqliteFlags += "-DSQLITE_ENABLE_RTREE=1"
 }
+if (enableRBU) {
+  defaultSqliteFlags += "-DSQLITE_ENABLE_RBU=1"
+}
 
 android {
   namespace "com.op.sqlite"

cpp/OPRBU.cpp

@@ -0,0 +1,170 @@
+#include "OPRBU.hpp"
+
+#include "OPTypes.hpp"
+#include <sqlite3.h>
+#include <sys/stat.h>
+
+#if defined(SQLITE_ENABLE_RBU) && !defined(OP_SQLITE_USE_SQLCIPHER) &&         \
+    !defined(OP_SQLITE_USE_LIBSQL) && !defined(OP_SQLITE_USE_TURSO) &&         \
+    !defined(OP_SQLITE_USE_PHONE_VERSION)
+#define OP_SQLITE_RBU_AVAILABLE 1
+
+extern "C" {
+typedef struct sqlite3rbu sqlite3rbu;
+
+sqlite3rbu *sqlite3rbu_open(const char *target_path, const char *update_path,
+                            const char *state_path);
+int sqlite3rbu_step(sqlite3rbu *rbu);
+int sqlite3rbu_close(sqlite3rbu *rbu, char **error_message);
+sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *rbu);
+int sqlite3rbu_state(sqlite3rbu *rbu);
+}
+#else
+#define OP_SQLITE_RBU_AVAILABLE 0
+#endif
+
+namespace opsqlite {
+
+namespace {
+
+[[noreturn]] void throw_rbu_error(int code, const std::string &message) {
+  throw OPSQLiteError(code, "[op-sqlite][RBU] " + message);
+}
+
+#if OP_SQLITE_RBU_AVAILABLE
+
+constexpr int RBU_STATE_OAL = 1;
+constexpr int RBU_STATE_MOVE = 2;
+constexpr int RBU_STATE_CHECKPOINT = 3;
+constexpr int RBU_STATE_DONE = 4;
+constexpr int RBU_STATE_ERROR = 5;
+
+bool is_regular_file(const std::string &path) {
+  struct stat info {};
+  return stat(path.c_str(), &info) == 0 && S_ISREG(info.st_mode);
+}
+
+bool directory_exists(const std::string &path) {
+  struct stat info {};
+  return stat(path.c_str(), &info) == 0 && S_ISDIR(info.st_mode);
+}
+
+std::string parent_directory(const std::string &path) {
+  const auto separator = path.find_last_of('/');
+  if (separator == std::string::npos) {
+    return {};
+  }
+  if (separator == 0) {
+    return "/";
+  }
+  return path.substr(0, separator);
+}
+
+std::string state_name(int state) {
+  switch (state) {
+  case RBU_STATE_OAL:
+    return "oal";
+  case RBU_STATE_MOVE:
+    return "move";
+  case RBU_STATE_CHECKPOINT:
+    return "checkpoint";
+  case RBU_STATE_DONE:
+    return "done";
+  case RBU_STATE_ERROR:
+    return "error";
+  default:
+    return "unknown";
+  }
+}
+
+void validate_paths(const std::string &target_path,
+                    const std::string &update_path,
+                    const std::string &state_path) {
+  if (target_path.empty() || target_path.front() != '/') {
+    throw_rbu_error(SQLITE_MISUSE, "targetPath must be an absolute path");
+  }
+  if (update_path.empty() || update_path.front() != '/') {
+    throw_rbu_error(SQLITE_MISUSE, "updatePath must be an absolute path");
+  }
+  if (!is_regular_file(target_path)) {
+    throw_rbu_error(SQLITE_CANTOPEN,
+                    "target database does not exist or is not a file: " +
+                        target_path);
+  }
+  if (!is_regular_file(update_path)) {
+    throw_rbu_error(SQLITE_CANTOPEN,
+                    "RBU update database does not exist or is not a file: " +
+                        update_path);
+  }
+  if (target_path == update_path ||
+      (!state_path.empty() &&
+       (state_path == target_path || state_path == update_path))) {
+    throw_rbu_error(SQLITE_MISUSE,
+                    "targetPath, updatePath, and statePath must be distinct");
+  }
+  if (!state_path.empty()) {
+    if (state_path.front() != '/') {
+      throw_rbu_error(SQLITE_MISUSE, "statePath must be an absolute path");
+    }
+    const auto parent = parent_directory(state_path);
+    if (!parent.empty() && !directory_exists(parent)) {
+      throw_rbu_error(SQLITE_CANTOPEN,
+                      "statePath parent directory does not exist: " + parent);
+    }
+  }
+}
+
+#endif
+
+} // namespace
+
+bool is_rbu_enabled() { return OP_SQLITE_RBU_AVAILABLE == 1; }
+
+RBUResult apply_rbu(const std::string &target_path,
+                    const std::string &update_path,
+                    const std::string &state_path, std::uint64_t max_steps) {
+#if OP_SQLITE_RBU_AVAILABLE
+  validate_paths(target_path, update_path, state_path);
+
+  sqlite3rbu *rbu =
+      sqlite3rbu_open(target_path.c_str(), update_path.c_str(),
+                      state_path.empty() ? nullptr : state_path.c_str());
+  if (rbu == nullptr) {
+    throw_rbu_error(SQLITE_NOMEM, "could not allocate an RBU handle");
+  }
+
+  int step_code = SQLITE_OK;
+  std::uint64_t steps = 0;
+  while (step_code == SQLITE_OK && steps < max_steps) {
+    step_code = sqlite3rbu_step(rbu);
+    ++steps;
+  }
+
+  const auto progress = static_cast<double>(sqlite3rbu_progress(rbu));
+  const auto state = state_name(sqlite3rbu_state(rbu));
+  char *error_message = nullptr;
+  const int close_code = sqlite3rbu_close(rbu, &error_message);
+
+  if (close_code != SQLITE_OK && close_code != SQLITE_DONE) {
+    const std::string message =
+        error_message != nullptr ? error_message : sqlite3_errstr(close_code);
+    sqlite3_free(error_message);
+    throw_rbu_error(close_code, message);
+  }
+
+  sqlite3_free(error_message);
+  return {close_code == SQLITE_DONE ? "complete" : "paused", steps, progress,
+          state};
+#else
+  (void)target_path;
+  (void)update_path;
+  (void)state_path;
+  (void)max_steps;
+  throw_rbu_error(
+      SQLITE_MISUSE,
+      "RBU is unavailable. Enable the bundled SQLite backend with "
+      "\"op-sqlite\": { \"rbu\": true } and rebuild the native app");
+#endif
+}
+
+} // namespace opsqlite

cpp/OPRBU.hpp

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+
+namespace opsqlite {
+
+struct RBUResult {
+  std::string status;
+  std::uint64_t steps;
+  double progress;
+  std::string state;
+};
+
+bool is_rbu_enabled();
+
+RBUResult apply_rbu(const std::string &target_path,
+                    const std::string &update_path,
+                    const std::string &state_path, std::uint64_t max_steps);
+
+} // namespace opsqlite

cpp/OPSqlite.cpp

@@ -9,7 +9,9 @@
 #endif
 #include "OPLogs.h"
 #include "OPMacros.hpp"
+#include "OPRBU.hpp"
 #include "OPUtils.hpp"
+#include <cmath>
 #include <functional>
 #include <iostream>
 #include <string>
@@ -134,6 +136,84 @@ install(jsi::Runtime &rt, const std::shared_ptr<react::CallInvoker> &invoker,
 #endif
   });
 
+  auto rbu_thread_pool = std::make_shared<ThreadPool>();
+
+  auto is_rbu_enabled = HFN(=) { return opsqlite::is_rbu_enabled(); });
+
+  auto apply_rbu = HFN(rbu_thread_pool) {
+    if (count != 1 || !args[0].isObject()) {
+      throw OPSQLiteError(
+          SQLITE_MISUSE,
+          "[op-sqlite][RBU] applyRBU expects one options object");
+    }
+
+    auto options = args[0].asObject(rt);
+    if (!options.hasProperty(rt, "targetPath") ||
+        !options.getProperty(rt, "targetPath").isString()) {
+      throw OPSQLiteError(SQLITE_MISUSE,
+                          "[op-sqlite][RBU] targetPath must be a string");
+    }
+    if (!options.hasProperty(rt, "updatePath") ||
+        !options.getProperty(rt, "updatePath").isString()) {
+      throw OPSQLiteError(SQLITE_MISUSE,
+                          "[op-sqlite][RBU] updatePath must be a string");
+    }
+
+    const auto target_path =
+        options.getProperty(rt, "targetPath").asString(rt).utf8(rt);
+    const auto update_path =
+        options.getProperty(rt, "updatePath").asString(rt).utf8(rt);
+    std::string state_path;
+    if (options.hasProperty(rt, "statePath") &&
+        !options.getProperty(rt, "statePath").isUndefined() &&
+        !options.getProperty(rt, "statePath").isNull()) {
+      if (!options.getProperty(rt, "statePath").isString()) {
+        throw OPSQLiteError(SQLITE_MISUSE,
+                            "[op-sqlite][RBU] statePath must be a string");
+      }
+      state_path = options.getProperty(rt, "statePath").asString(rt).utf8(rt);
+    }
+
+    constexpr std::uint64_t default_max_steps = 1000;
+    auto max_steps = default_max_steps;
+    if (options.hasProperty(rt, "maxSteps") &&
+        !options.getProperty(rt, "maxSteps").isUndefined() &&
+        !options.getProperty(rt, "maxSteps").isNull()) {
+      const auto value = options.getProperty(rt, "maxSteps");
+      if (!value.isNumber()) {
+        throw OPSQLiteError(
+            SQLITE_MISUSE,
+            "[op-sqlite][RBU] maxSteps must be a positive integer");
+      }
+      const auto number = value.asNumber();
+      if (!std::isfinite(number) || number < 1 ||
+          std::floor(number) != number || number > 9007199254740991.0) {
+        throw OPSQLiteError(
+            SQLITE_MISUSE,
+            "[op-sqlite][RBU] maxSteps must be a positive integer");
+      }
+      max_steps = static_cast<std::uint64_t>(number);
+    }
+
+    return promisify(
+        rt, rbu_thread_pool,
+        [target_path, update_path, state_path, max_steps]() {
+          return std::any(opsqlite::apply_rbu(target_path, update_path,
+                                              state_path, max_steps));
+        },
+        [](jsi::Runtime &rt, std::any result) {
+          const auto rbu_result = std::any_cast<RBUResult>(result);
+          jsi::Object value(rt);
+          value.setProperty(rt, "status",
+                            jsi::String::createFromUtf8(rt, rbu_result.status));
+          value.setProperty(rt, "steps", static_cast<double>(rbu_result.steps));
+          value.setProperty(rt, "progress", rbu_result.progress);
+          value.setProperty(rt, "state",
+                            jsi::String::createFromUtf8(rt, rbu_result.state));
+          return value;
+        });
+  });
+
 #if defined(OP_SQLITE_USE_LIBSQL) || defined(OP_SQLITE_USE_TURSO)
   auto open_remote = HFN(=) {
     jsi::Object options = args[0].asObject(rt);
@@ -226,6 +306,8 @@ install(jsi::Runtime &rt, const std::shared_ptr<react::CallInvoker> &invoker,
   module.setProperty(rt, "isLibsql", std::move(is_libsql));
   module.setProperty(rt, "isTurso", std::move(is_turso));
   module.setProperty(rt, "isIOSEmbedded", std::move(is_ios_embedded));
+  module.setProperty(rt, "isRBUEnabled", std::move(is_rbu_enabled));
+  module.setProperty(rt, "applyRBU", std::move(apply_rbu));
 #if defined(OP_SQLITE_USE_LIBSQL) || defined(OP_SQLITE_USE_TURSO)
   module.setProperty(rt, "openRemote", std::move(open_remote));
   module.setProperty(rt, "openSync", std::move(open_sync));