diff --git a/.gitignore b/.gitignore index 345c852b..dad3e94e 100644 --- a/.gitignore +++ b/.gitignore @@ -84,8 +84,12 @@ android/gradle/ !.yarn/sdks !.yarn/versions +# Generated by the build pipelines from generate_tokenizers_header_file.js android/c_sources -# Android does not generate the sources on its own, leave this in for CI +# c_sources/ is also generated (copied out of the app's own c_sources by the +# podspec/Package.swift) but stays tracked: SwiftPM evaluates Package.swift in a +# sandbox that can deny that copy, so an SPM build off a fresh checkout needs the +# files to already be there. Android no longer depends on it. # c_sources/ scripts/sqlite-vec-* diff --git a/Package.swift b/Package.swift index f62ac90f..dd89ddc5 100644 --- a/Package.swift +++ b/Package.swift @@ -103,34 +103,51 @@ if !tokenizers.isEmpty && useTurso { print("[OP-SQLITE] SPM configuration found at \(appPackageJSONPath)") // MARK: - Tokenizer header generation -// Direct translation of generate_tokenizers_header_file.rb. +// The header contents come from generate_tokenizers_header_file.js, the one +// implementation shared by every build pipeline (CocoaPods and Gradle call the +// same script). This only has to find node and run it. func generateTokenizersHeaderFile(names: [String], filePath: String) { - let fileURL = URL(fileURLWithPath: filePath) - try? FileManager.default.createDirectory( - at: fileURL.deletingLastPathComponent(), withIntermediateDirectories: true) - - let tokenizerList = names.map { "opsqlite_\($0)_init(db,&errMsg,nullptr);" }.joined() - - var content = "" - content += "#ifndef TOKENIZERS_H\n" - content += "#define TOKENIZERS_H\n" - content += "\n" - content += "#define TOKENIZER_LIST \(tokenizerList)\n" - content += "\n" - content += "#include \n" - content += "\n" - content += "namespace opsqlite {\n" - content += "\n" - for name in names { - content += "int opsqlite_\(name)_init(sqlite3 *db, char **error, sqlite3_api_routines const *api);\n" + let scriptPath = packageRoot.appendingPathComponent("generate_tokenizers_header_file.js").path + + // NODE_BINARY is React Native's own convention (it writes one into + // ios/.xcode.env). Xcode does not necessarily inherit the user's shell PATH, + // so fall back to a PATH lookup and then to the usual install locations. + var candidates: [String] = [] + if let fromEnv = ProcessInfo.processInfo.environment["NODE_BINARY"], !fromEnv.isEmpty { + candidates.append(fromEnv) + } + candidates += ["node", "/opt/homebrew/bin/node", "/usr/local/bin/node", "/usr/bin/node"] + + for node in candidates { + let process = Process() + // Going through `env` is what resolves a bare "node" against PATH. + process.executableURL = URL(fileURLWithPath: "/usr/bin/env") + process.arguments = [node, scriptPath, filePath] + names + process.standardOutput = FileHandle.standardError + + do { + try process.run() + process.waitUntilExit() + if process.terminationStatus == 0 { + return + } + } catch { + continue + } } - content += "\n" - content += "} // namespace opsqlite\n" - content += "\n" - content += "#endif // TOKENIZERS_H\n" - try? content.write(to: fileURL, atomically: true, encoding: .utf8) + // Not fatal, on purpose. Manifest evaluation happens in contexts we do not + // control (Xcode's package resolution, `swift package dump-package`), and + // SwiftPM sandboxes writes there, so the generation can legitimately fail + // while an already up-to-date header sits on disk -- the script exits 0 + // without writing in that case, so reaching this point means it really could + // not produce the file. Killing the manifest would be worse than letting the + // compiler report the missing header. + FileHandle.standardError.write( + Data( + ("[OP-SQLITE] warning: could not generate \(filePath) with \(scriptPath). Make sure node is " + + "on your PATH or set NODE_BINARY to its location.\n").utf8)) } // Mirrors FileUtils.cp_r: copies the contents of `source` into `destination`, diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index f9a76cc5..3b52778e 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -93,6 +93,13 @@ find_library(LOG_LIB log) if (USER_DEFINED_SOURCE_FILES) target_sources(${PACKAGE_NAME} PRIVATE ${USER_DEFINED_SOURCE_FILES}) + # tokenizers.h is generated next to those sources, outside ../cpp, so it is + # found through the include path instead of a path relative to whichever file + # does the #include. + if (USER_DEFINED_TOKENIZERS_HEADER_DIR) + include_directories(${USER_DEFINED_TOKENIZERS_HEADER_DIR}) + endif() + add_definitions("-DTOKENIZERS_HEADER_PATH=\"${USER_DEFINED_TOKENIZERS_HEADER_PATH}\"") endif() diff --git a/android/build.gradle b/android/build.gradle index 3049da40..1ef8ae68 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -143,6 +143,38 @@ if(!tokenizers.isEmpty()) { println "[OP-SQLITE] Tokenizers enabled. Detected tokenizers: " + tokenizers } +// Generates the tokenizers.h that cpp/OPBridge.cpp includes (through +// TOKENIZERS_HEADER_PATH) and that the copied tokenizers.cpp includes by name. +// +// The generation itself lives in generate_tokenizers_header_file.js, shared with +// the CocoaPods (generate_tokenizers_header_file.rb) and SwiftPM (Package.swift) +// pipelines, so all three emit the same header. Before this, Android had no +// generator of its own and could only build if an iOS pod install (or a header +// committed to the repo) had produced the file first. +// +// Uses a plain ProcessBuilder rather than project.exec: this runs at +// configuration time, where that API is deprecated/removed in recent Gradle. +def generateTokenizersHeaderFile(List names, File headerFile) { + def packageDir = buildscript.sourceFile.parentFile.parentFile + def script = new File(packageDir, "generate_tokenizers_header_file.js") + // NODE_BINARY is React Native's own convention for pointing at a specific node. + def node = System.getenv("NODE_BINARY") + if (!node) { + node = "node" + } + + def command = [node, script.absolutePath, headerFile.absolutePath] + names + def process = new ProcessBuilder(command).redirectErrorStream(true).start() + def output = process.inputStream.text + def status = process.waitFor() + + if (status != 0) { + throw new GradleException( + "[OP-SQLITE] Could not generate ${headerFile}. Command: ${command.join(' ')}\n" + + "Make sure node is on your PATH or set NODE_BINARY to its location.\n${output}") + } +} + // Build the list of library-default SQLite compile flags driven by package.json // toggles. These are emitted via CMake add_definitions() BEFORE the user-supplied // sqliteFlags, so that user flags take precedence when both define the same macro @@ -194,17 +226,29 @@ android { // This are zeroes because they will be passed as C flags, so they become falsy def sourceFiles = 0 // def tokenizerInitStrings = 0 + def tokenizersHeaderDir = 0 def tokenizersHeaderPath = 0 if (!tokenizers.isEmpty()) { def sourceDir = isUserApp ? file("$rootDir/../../../c_sources") : file("$rootDir/../c_sources") def destDir = file("$buildscript.sourceFile.parentFile/c_sources") + // Generated into the app's own c_sources, exactly where the podspec and + // Package.swift put it, so the header the user's tokenizers.cpp includes + // is there no matter which platform was built last. The copy below then + // carries it over with the sources. + generateTokenizersHeaderFile(tokenizers, new File(sourceDir, "tokenizers.h")) copy { from sourceDir into destDir include "**/*.cpp", "**/*.h" } sourceFiles = fileTree(dir: destDir, include: ["**/*.cpp", "**/*.h"]).files.join(";") - tokenizersHeaderPath = "../c_sources/tokenizers.h" + // The header sits next to the copied tokenizer sources, not in cpp/, so + // it is reached through the include path. Passing a path relative to the + // file doing the #include would hardcode where in cpp/ that include is, + // and an absolute path in a C string literal would break on Windows + // separators. + tokenizersHeaderDir = destDir.absolutePath.replace('\\', '/') + tokenizersHeaderPath = "tokenizers.h" } cppFlags "-O3 -frtti -fexceptions -Wall -fstack-protector-all" @@ -217,6 +261,7 @@ android { "-DUSE_TURSO=${useTurso ? 1 : 0}", "-DUSE_SQLITE_VEC=${useSqliteVec ? 1 : 0}", "-DUSER_DEFINED_SOURCE_FILES=${sourceFiles}", + "-DUSER_DEFINED_TOKENIZERS_HEADER_DIR='${tokenizersHeaderDir}'", "-DUSER_DEFINED_TOKENIZERS_HEADER_PATH='${tokenizersHeaderPath}'", "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON" } diff --git a/c_sources/tokenizers.h b/c_sources/tokenizers.h index 2a41fbee..850e48ca 100644 --- a/c_sources/tokenizers.h +++ b/c_sources/tokenizers.h @@ -1,7 +1,7 @@ #ifndef TOKENIZERS_H #define TOKENIZERS_H -#define TOKENIZER_LIST opsqlite_wordtokenizer_init(db,&errMsg,nullptr);opsqlite_porter_init(db,&errMsg,nullptr); +#define TOKENIZER_LIST opsqlite_wordtokenizer_init(db,&err_msg,nullptr);opsqlite_porter_init(db,&err_msg,nullptr); #include diff --git a/cpp/OPBridge.cpp b/cpp/OPBridge.cpp index 1b2042d6..b6ac8084 100644 --- a/cpp/OPBridge.cpp +++ b/cpp/OPBridge.cpp @@ -140,7 +140,7 @@ sqlite3 *opsqlite_open(std::string const &name, std::string const &path, // Written to only on failure by both sqlite3_load_extension below and the // tokenizer init calls TOKENIZER_LIST expands into, so it starts out null. // Unused when neither of those is configured into the build. - [[maybe_unused]] char *errMsg = nullptr; + [[maybe_unused]] char *err_msg = nullptr; sqlite3 *db; int flags = SQLITE_OPEN_FULLMUTEX; @@ -187,11 +187,11 @@ sqlite3 *opsqlite_open(std::string const &name, std::string const &path, const char *vec_entry_point = "sqlite3_vec_init"; int vec_status = sqlite3_load_extension(db, _sqlite_vec_path.c_str(), - vec_entry_point, &errMsg); + vec_entry_point, &err_msg); if (vec_status != SQLITE_OK) { - std::string message = errMsg != nullptr ? errMsg : "unknown error"; - sqlite3_free(errMsg); + std::string message = err_msg != nullptr ? err_msg : "unknown error"; + sqlite3_free(err_msg); throw build_error("could not load sqlite-vec", message, vec_status & 0xff, vec_status); diff --git a/docs/docs/tokenizers.md b/docs/docs/tokenizers.md index 73be8b35..fe009cd2 100644 --- a/docs/docs/tokenizers.md +++ b/docs/docs/tokenizers.md @@ -18,7 +18,7 @@ op-sqlite has a novel way for you to create your tokenizers. } ``` -2. Run `pod install`. The podspec now contains a code generation step. It will create a `c_sources` folder at the root of your project. It will create a `tokenizers.h` file. DON’T TOUCH THIS FILE. It will be overwritten every time. You need to create a `c_sources/tokenizers.cpp` file. Here you need to provide your tokenizer implementation. The `tokenizer.h` file contains the function declaration that will be executed when registering your tokenizer. In this case here is a sample `tokenizers.cpp` implementation +2. Run `pod install` (iOS) or a Gradle build (Android). Both contain a code generation step. It will create a `c_sources` folder at the root of your project. It will create a `tokenizers.h` file. DON’T TOUCH THIS FILE. It will be overwritten every time. You need to create a `c_sources/tokenizers.cpp` file. Here you need to provide your tokenizer implementation. The `tokenizer.h` file contains the function declaration that will be executed when registering your tokenizer. In this case here is a sample `tokenizers.cpp` implementation ```cpp #include "tokenizers.h" @@ -102,7 +102,7 @@ op-sqlite has a novel way for you to create your tokenizers. You need to keep the namespace and the function signature intact. For now the `sqlite3_api_routines` parameter will always be a null pointer. 3. Once you are done. You need to run `pod install` again. It will then copy the files you created to the pod sources in order to compile `op-sqlite` together with your new C++ code in one go. -4. The code generation step is only implemented in Cocoapods. Every time you create/change a file inside of `c_sources` you will need to do a `pod install` to re-add the newly created files into the compilation process. This also applies for Android, at least the header file generation step. On your CI, you will also need to do a pod install even if your pods are cached, in order to copy the sources. +4. The `tokenizers.h` generation runs in every pipeline: CocoaPods, SwiftPM and Gradle all call the same `generate_tokenizers_header_file.js` from the package, so an Android build no longer needs an iOS `pod install` to have happened first. Adding or removing a **file** inside `c_sources` still needs a `pod install` on iOS, since the pod's file list is fixed at install time; Android picks up new files on the next Gradle build. On your CI, you will also need to do a pod install even if your pods are cached, in order to copy the sources. 5. You can then create a FTS5 virtual table with your tokenizer: ```tsx diff --git a/example/c_sources/tokenizers.h b/example/c_sources/tokenizers.h index 2a41fbee..850e48ca 100644 --- a/example/c_sources/tokenizers.h +++ b/example/c_sources/tokenizers.h @@ -1,7 +1,7 @@ #ifndef TOKENIZERS_H #define TOKENIZERS_H -#define TOKENIZER_LIST opsqlite_wordtokenizer_init(db,&errMsg,nullptr);opsqlite_porter_init(db,&errMsg,nullptr); +#define TOKENIZER_LIST opsqlite_wordtokenizer_init(db,&err_msg,nullptr);opsqlite_porter_init(db,&err_msg,nullptr); #include diff --git a/generate_tokenizers_header_file.js b/generate_tokenizers_header_file.js new file mode 100644 index 00000000..6f7b459e --- /dev/null +++ b/generate_tokenizers_header_file.js @@ -0,0 +1,80 @@ +#!/usr/bin/env node + +// Single source of truth for the generated `tokenizers.h`. +// +// Every build pipeline needs this file: CocoaPods (via +// generate_tokenizers_header_file.rb), SwiftPM (Package.swift) and Gradle +// (android/build.gradle). They all shell out to this script instead of +// reimplementing the generation, so the header cannot drift between platforms +// and TOKENIZER_LIST only has to be kept in sync with cpp/OPBridge.cpp here. +// +// Usage: node generate_tokenizers_header_file.js ... + +const fs = require("node:fs"); +const path = require("node:path"); + +/** + * The expansion of TOKENIZER_LIST is pasted into opsqlite_open(), so `db` and + * `err_msg` refer to that function's locals. + */ +function generateTokenizersHeader(names) { + const tokenizerList = names.map((name) => `opsqlite_${name}_init(db,&err_msg,nullptr);`).join(""); + + const declarations = names.map( + (name) => + `int opsqlite_${name}_init(sqlite3 *db, char **error, sqlite3_api_routines const *api);`, + ); + + return [ + "#ifndef TOKENIZERS_H", + "#define TOKENIZERS_H", + "", + `#define TOKENIZER_LIST ${tokenizerList}`, + "", + "#include ", + "", + "namespace opsqlite {", + "", + ...declarations, + "", + "} // namespace opsqlite", + "", + "#endif // TOKENIZERS_H", + "", + ].join("\n"); +} + +/** + * Returns true when the file was written, false when it was already up to date. + * + * Rewriting an identical file would bump its mtime and force a recompile of + * everything that includes it. Gradle in particular runs this on every + * configuration. + */ +function writeTokenizersHeader(names, filePath) { + const contents = generateTokenizersHeader(names); + + if (fs.existsSync(filePath) && fs.readFileSync(filePath, "utf8") === contents) { + return false; + } + + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, contents); + + return true; +} + +module.exports = { generateTokenizersHeader, writeTokenizersHeader }; + +if (require.main === module) { + const [filePath, ...names] = process.argv.slice(2); + + if (!filePath) { + console.error( + "[OP-SQLITE] usage: node generate_tokenizers_header_file.js ...", + ); + process.exit(1); + } + + writeTokenizersHeader(names, filePath); +} diff --git a/generate_tokenizers_header_file.rb b/generate_tokenizers_header_file.rb index 395f379a..fd3fcd13 100644 --- a/generate_tokenizers_header_file.rb +++ b/generate_tokenizers_header_file.rb @@ -1,29 +1,19 @@ -require 'fileutils' +require 'shellwords' +# Thin wrapper around generate_tokenizers_header_file.js, which holds the actual +# generation logic so CocoaPods, SwiftPM and Gradle all emit the exact same +# header. Node is already a hard requirement for building a React Native app. def generate_tokenizers_header_file(names, file_path) - # Ensure the directory exists - dir_path = File.dirname(file_path) - FileUtils.mkdir_p(dir_path) unless Dir.exist?(dir_path) - tokenizer_list = names.map { |name| "opsqlite_#{name}_init(db,&errMsg,nullptr);" }.join + script_path = File.join(__dir__, "generate_tokenizers_header_file.js") + # NODE_BINARY is React Native's own convention for pointing at a specific + # node (see the .xcode.env files it generates). + node = ENV["NODE_BINARY"] + node = "node" if node.nil? || node.empty? - File.open(file_path, 'w') do |file| - file.puts "#ifndef TOKENIZERS_H" - file.puts "#define TOKENIZERS_H" - file.puts - file.puts "#define TOKENIZER_LIST #{tokenizer_list}" - file.puts - file.puts "#include " - file.puts - file.puts "namespace opsqlite {" - file.puts + command = [node, script_path, file_path, *names] - names.each do |name| - file.puts "int opsqlite_#{name}_init(sqlite3 *db, char **error, sqlite3_api_routines const *api);" - end - - file.puts - file.puts "} // namespace opsqlite" - file.puts - file.puts "#endif // TOKENIZERS_H" + unless system(*command) + raise "[OP-SQLITE] Could not generate the tokenizers header. Command: #{command.shelljoin}. " \ + "Make sure node is on your PATH or set NODE_BINARY to its location." end -end \ No newline at end of file +end diff --git a/package.json b/package.json index 12343d18..4ac3f510 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "node/package.json", "*.podspec", "*.rb", + "generate_tokenizers_header_file.js", "react-native.config.js", "ios/**.xcframework", "Package.swift",