Add DuckLake storage extension with SQL pushdown support

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ concurrency:
 env:
   # pg_client requires libpq (PostgreSQL) at build time and a running
   # PostgreSQL instance at test time (provided by run_pgembed_fixture.py).
-  EXTENSION_LIST: adbc;azure;delta;duckdb;fts;httpfs;iceberg;json;llm;neo4j;pg_client;postgres;sqlite;unity_catalog;vector;algo;gql
+  EXTENSION_LIST: adbc;azure;delta;duckdb;ducklake;fts;httpfs;iceberg;json;llm;neo4j;pg_client;postgres;sqlite;unity_catalog;vector;algo;gql
 
 jobs:
   # ─────────────────────────────────────────────────────────────────

CMakeLists.txt

@@ -86,6 +86,7 @@ add_extension_if_enabled_and_skip_32bit("delta")
 add_extension_if_enabled_and_skip_32bit("iceberg")
 add_extension_if_enabled_and_skip_32bit("azure")
 add_extension_if_enabled_and_skip_32bit("unity_catalog")
+add_extension_if_enabled_and_skip_32bit("ducklake")
 add_extension_if_enabled("json")
 add_extension_if_enabled("fts")
 add_extension_if_enabled("vector")

ducklake/CMakeLists.txt

@@ -0,0 +1,37 @@
+if (WIN32 OR BUILD_STATIC_EXTENSION)
+    set(DuckDB_USE_STATIC_LIBS ON)
+else ()
+    set(DuckDB_USE_STATIC_LIBS OFF)
+endif ()
+find_package(DuckDB REQUIRED)
+
+include_directories(
+        ${PROJECT_SOURCE_DIR}/src/include
+        ${CMAKE_BINARY_DIR}/src/include
+        src/include
+        ${PROJECT_SOURCE_DIR}/extension/duckdb/src/include
+        ${PROJECT_SOURCE_DIR}/extension/httpfs/src/include  # For S3 configuration
+        ${DuckDB_INCLUDE_DIRS})
+
+add_subdirectory(src/installer)
+add_subdirectory(src/main)
+add_subdirectory(src/connector)
+add_subdirectory(src/storage)
+add_subdirectory(src/options)
+
+build_extension_lib(${BUILD_STATIC_EXTENSION} "ducklake")
+
+target_link_libraries(lbug_${EXTENSION_LIB_NAME}_extension
+        PRIVATE
+        ${DuckDB_LIBRARIES})
+
+if (NOT WIN32 AND NOT BUILD_STATIC_EXTENSION)
+    add_library(ducklake_loader
+            SHARED
+            ${PROJECT_SOURCE_DIR}/extension/duckdb/src/loader/duckdb_loader.cpp)
+    set_extension_properties(ducklake_loader ducklake_loader ducklake)
+endif ()
+
+if (APPLE AND NOT BUILD_STATIC_EXTENSION)
+    set_apple_dynamic_lookup(ducklake_loader)
+endif ()

ducklake/src/connector/CMakeLists.txt

@@ -0,0 +1,9 @@
+add_library(lbug_ducklake_connector
+        OBJECT
+        ducklake_connector.cpp
+        ${PROJECT_SOURCE_DIR}/extension/duckdb/src/connector/duckdb_connector.cpp
+        ${PROJECT_SOURCE_DIR}/extension/duckdb/src/connector/duckdb_result_converter.cpp)
+
+set(DUCKLAKE_EXTENSION_OBJECT_FILES
+        ${DUCKLAKE_EXTENSION_OBJECT_FILES} $<TARGET_OBJECTS:lbug_ducklake_connector>
+        PARENT_SCOPE)

ducklake/src/connector/ducklake_connector.cpp

@@ -0,0 +1,39 @@
+#include "connector/ducklake_connector.h"
+
+#include "options/ducklake_options.h"
+#include <format>
+
+namespace lbug {
+namespace ducklake_extension {
+
+// Ensures a DuckDB extension is loaded. LOAD is attempted first so that an
+// already-installed extension is reused as-is (no network, no repository
+// origin check). INSTALL runs only as a fallback when the extension is
+// missing, e.g. on a fresh machine.
+static void ensureExtensionLoaded(const duckdb_extension::DuckDBConnector& connector,
+    const std::string& extensionName) {
+    try {
+        connector.executeQuery(std::format("load {};", extensionName));
+        return;
+    } catch (const common::Exception&) {
+        // Not installed yet: fall through to INSTALL.
+    }
+    connector.executeQuery(std::format("install {};", extensionName));
+    connector.executeQuery(std::format("load {};", extensionName));
+}
+
+void DuckLakeConnector::connect(const std::string& dbPath, const std::string& catalogName,
+    const std::string& /*schemaName*/, main::ClientContext* context) {
+    // Creates an in-memory DuckDB instance, loads the ducklake extension
+    // (plus httpfs for remote DATA_PATHs such as s3://), then attaches the
+    // DuckLake catalog inside the embedded instance so its tables can be
+    // referenced by fully qualified name (e.g. my_lake.main.person).
+    instance = std::make_unique<duckdb::DuckDB>(nullptr);
+    connection = std::make_unique<duckdb::Connection>(*instance);
+    ensureExtensionLoaded(*this, "ducklake");
+    ensureExtensionLoaded(*this, "httpfs");
+    executeQuery(DuckLakeAttachHelper::getAttachQuery(dbPath, catalogName, context));
+}
+
+} // namespace ducklake_extension
+} // namespace lbug

ducklake/src/include/connector/ducklake_connector.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "connector/duckdb_connector.h"
+
+namespace lbug {
+namespace ducklake_extension {
+
+class DuckLakeConnector : public duckdb_extension::DuckDBConnector {
+public:
+    void connect(const std::string& dbPath, const std::string& catalogName,
+        const std::string& schemaName, main::ClientContext* context) override;
+};
+
+} // namespace ducklake_extension
+} // namespace lbug

ducklake/src/include/main/ducklake_extension.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "extension/extension.h"
+
+namespace lbug {
+namespace ducklake_extension {
+
+class DuckLakeExtension final : public extension::Extension {
+public:
+    static constexpr char EXTENSION_NAME[] = "DUCKLAKE";
+
+public:
+    static void load(main::ClientContext* context);
+};
+
+} // namespace ducklake_extension
+} // namespace lbug

ducklake/src/include/options/ducklake_options.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include "common/types/value/value.h"
+
+namespace lbug {
+namespace main {
+class Database;
+class ClientContext;
+} // namespace main
+
+namespace ducklake_extension {
+
+// Optional override for the DuckLake `DATA_PATH` ATTACH parameter. When set,
+// it is appended to the `ATTACH 'ducklake:...'` statement executed on the
+// embedded DuckDB instance as `(DATA_PATH '...', OVERRIDE_DATA_PATH true)`,
+// so the given directory is used instead of the data path stored in the
+// DuckLake metadata for the current connection (the stored value is left
+// untouched). Empty (the default) leaves the stored path untouched, which is
+// the common case for file-backed catalogs. The override form is required
+// because DuckDB rejects a plain `DATA_PATH` that differs from the stored
+// path, and it is what makes relocated catalogs (e.g. checked-in test data)
+// queryable.
+struct DuckLakeDataPath {
+    static constexpr const char* NAME = "ducklake_data_path";
+    static constexpr common::LogicalTypeID TYPE = common::LogicalTypeID::STRING;
+    static common::Value getDefaultValue() { return common::Value{std::string()}; }
+};
+
+// Optional read-only flag, forwarded as `READ_ONLY` to the embedded `ATTACH`.
+struct DuckLakeReadOnly {
+    static constexpr const char* NAME = "ducklake_read_only";
+    static constexpr common::LogicalTypeID TYPE = common::LogicalTypeID::BOOL;
+    static common::Value getDefaultValue() { return common::Value{false}; }
+};
+
+struct DuckLakeOptions {
+    static void registerExtensionOptions(main::Database* db);
+    static void setEnvValue(main::ClientContext* context);
+    static std::string getDataPath(main::ClientContext* context);
+    static bool getReadOnly(main::ClientContext* context);
+};
+
+// Translates the Ladybug ATTACH path and extension options into the
+// `ATTACH 'ducklake:...' AS <alias> (...)` statement executed on the embedded
+// DuckDB instance.
+struct DuckLakeAttachHelper {
+    static std::string getAttachQuery(const std::string& dbPath, const std::string& catalogAlias,
+        main::ClientContext* context);
+};
+
+} // namespace ducklake_extension
+} // namespace lbug

ducklake/src/include/storage/ducklake_storage.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "storage/storage_extension.h"
+
+namespace lbug {
+namespace main {
+class Database;
+} // namespace main
+
+namespace ducklake_extension {
+
+class DuckLakeStorageExtension final : public storage::StorageExtension {
+public:
+    static constexpr const char* DB_TYPE = "DUCKLAKE";
+
+    // DuckLake catalogs live in DuckDB, whose default schema is `main`.
+    static constexpr const char* DEFAULT_SCHEMA_NAME = "main";
+
+    explicit DuckLakeStorageExtension(main::Database& database);
+
+    bool canHandleDB(std::string dbType) const override;
+};
+
+} // namespace ducklake_extension
+} // namespace lbug

ducklake/src/installer/CMakeLists.txt

@@ -0,0 +1,11 @@
+if (NOT WIN32)
+    add_library(ducklake_installer
+            SHARED
+            ducklake_install_func.cpp
+            ${PROJECT_SOURCE_DIR}/extension/duckdb/src/installer/duckdb_installer.cpp)
+    set_extension_properties(ducklake_installer ducklake_installer ducklake)
+endif ()
+
+if (APPLE)
+    set_apple_dynamic_lookup(ducklake_installer)
+endif ()

ducklake/src/installer/ducklake_install_func.cpp

@@ -0,0 +1,16 @@
+#include "installer/duckdb_installer.h"
+
+extern "C" {
+// Because we link against the static library on windows, we implicitly inherit LBUG_STATIC_DEFINE,
+// which cancels out any exporting, so we can't use LBUG_API.
+#if defined(_WIN32)
+#define INIT_EXPORT __declspec(dllexport)
+#else
+#define INIT_EXPORT __attribute__((visibility("default")))
+#endif
+INIT_EXPORT void install(const std::string& repo, lbug::main::ClientContext& context) {
+    lbug::extension::InstallExtensionInfo info{"ducklake", repo, false /* forceInstall */};
+    lbug::duckdb_extension::DuckDBInstaller installer{info, context};
+    installer.install();
+}
+}

ducklake/src/main/CMakeLists.txt

@@ -0,0 +1,7 @@
+add_library(ducklake_extension_main
+        OBJECT
+        ducklake_extension.cpp)
+
+set(DUCKLAKE_EXTENSION_OBJECT_FILES
+        ${DUCKLAKE_EXTENSION_OBJECT_FILES} $<TARGET_OBJECTS:ducklake_extension_main>
+        PARENT_SCOPE)

ducklake/src/main/ducklake_extension.cpp

@@ -0,0 +1,38 @@
+
+#include "main/ducklake_extension.h"
+
+#include "main/client_context.h"
+#include "options/ducklake_options.h"
+#include "storage/ducklake_storage.h"
+
+namespace lbug {
+namespace ducklake_extension {
+
+void DuckLakeExtension::load(main::ClientContext* context) {
+    auto& db = *context->getDatabase();
+    db.registerStorageExtension(EXTENSION_NAME, std::make_unique<DuckLakeStorageExtension>(db));
+    DuckLakeOptions::registerExtensionOptions(&db);
+    DuckLakeOptions::setEnvValue(context);
+}
+
+} // namespace ducklake_extension
+} // namespace lbug
+
+#if defined(BUILD_DYNAMIC_LOAD)
+extern "C" {
+// Because we link against the static library on windows, we implicitly inherit LBUG_STATIC_DEFINE,
+// which cancels out any exporting, so we can't use LBUG_API.
+#if defined(_WIN32)
+#define INIT_EXPORT __declspec(dllexport)
+#else
+#define INIT_EXPORT __attribute__((visibility("default")))
+#endif
+INIT_EXPORT void init(lbug::main::ClientContext* context) {
+    lbug::ducklake_extension::DuckLakeExtension::load(context);
+}
+
+INIT_EXPORT const char* name() {
+    return lbug::ducklake_extension::DuckLakeExtension::EXTENSION_NAME;
+}
+}
+#endif

ducklake/src/options/CMakeLists.txt

@@ -0,0 +1,7 @@
+add_library(lbug_ducklake_options
+        OBJECT
+        ducklake_options.cpp)
+
+set(DUCKLAKE_EXTENSION_OBJECT_FILES
+        ${DUCKLAKE_EXTENSION_OBJECT_FILES} $<TARGET_OBJECTS:lbug_ducklake_options>
+        PARENT_SCOPE)

ducklake/src/options/ducklake_options.cpp

@@ -0,0 +1,97 @@
+#include "options/ducklake_options.h"
+
+#include "common/string_utils.h"
+#include "extension/extension.h"
+#include "main/client_context.h"
+#include "main/database.h"
+#include <format>
+
+namespace lbug {
+namespace ducklake_extension {
+
+using namespace common;
+
+namespace {
+
+std::string escapeSingleQuotes(std::string value) {
+    StringUtils::replaceAll(value, "'", "''");
+    return value;
+}
+
+// Reads an option value from the environment. Both the option name as-is and
+// its upper-case form are accepted (e.g. `ducklake_data_path` or
+// `DUCKLAKE_DATA_PATH`).
+std::string getEnvOption(const char* name) {
+    auto value = main::ClientContext::getEnvVariable(name);
+    if (value.empty()) {
+        value = main::ClientContext::getEnvVariable(StringUtils::getUpper(std::string(name)));
+    }
+    return value;
+}
+
+void setEnvOption(main::ClientContext* context, const char* name) {
+    auto value = getEnvOption(name);
+    if (!value.empty()) {
+        context->setExtensionOption(name, Value::createValue(value));
+    }
+}
+
+} // namespace
+
+void DuckLakeOptions::registerExtensionOptions(main::Database* db) {
+    ADD_EXTENSION_OPTION(DuckLakeDataPath);
+    ADD_EXTENSION_OPTION(DuckLakeReadOnly);
+}
+
+void DuckLakeOptions::setEnvValue(main::ClientContext* context) {
+    setEnvOption(context, DuckLakeDataPath::NAME);
+    auto readOnly = getEnvOption(DuckLakeReadOnly::NAME);
+    if (!readOnly.empty()) {
+        StringUtils::toLower(readOnly);
+        context->setExtensionOption(DuckLakeReadOnly::NAME,
+            Value::createValue(readOnly == "true" || readOnly == "1"));
+    }
+}
+
+std::string DuckLakeOptions::getDataPath(main::ClientContext* context) {
+    return context->getCurrentSetting(DuckLakeDataPath::NAME).toString();
+}
+
+bool DuckLakeOptions::getReadOnly(main::ClientContext* context) {
+    auto value = context->getCurrentSetting(DuckLakeReadOnly::NAME);
+    return value.getDataType().getLogicalTypeID() == LogicalTypeID::BOOL && value.getValue<bool>();
+}
+
+std::string DuckLakeAttachHelper::getAttachQuery(const std::string& dbPath,
+    const std::string& catalogAlias, main::ClientContext* context) {
+    // Normalize the ATTACH target. An empty path means "read the
+    // configuration from the default (unnamed) TYPE ducklake secret"
+    // (`ATTACH 'ducklake:'`). A path that already carries the `ducklake:`
+    // scheme (e.g. `ducklake:postgres:dbname=...`) is used as-is; anything
+    // else is treated as the metadata catalog path and prefixed.
+    std::string target;
+    if (dbPath.empty()) {
+        target = "ducklake:";
+    } else if (dbPath.rfind("ducklake:", 0) == 0) {
+        target = dbPath;
+    } else {
+        target = "ducklake:" + dbPath;
+    }
+    std::string options;
+    auto dataPath = DuckLakeOptions::getDataPath(context);
+    if (!dataPath.empty()) {
+        // OVERRIDE_DATA_PATH is required: DuckDB rejects a DATA_PATH that
+        // differs from the catalog's stored path without it.
+        options +=
+            std::format(", DATA_PATH '{}', OVERRIDE_DATA_PATH true", escapeSingleQuotes(dataPath));
+    }
+    if (DuckLakeOptions::getReadOnly(context)) {
+        options += ", READ_ONLY";
+    }
+    // Strip the leading ", " when options are present.
+    auto params = options.empty() ? std::string{} : " (" + options.substr(2) + ")";
+    return std::format("ATTACH '{}' AS {}{};", escapeSingleQuotes(target), catalogAlias, params);
+}
+
+} // namespace ducklake_extension
+} // namespace lbug