From 2c90d1379ff4c85a0b22f87b539c301277e0bf48 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Thu, 13 Aug 2026 11:37:00 +0200 Subject: [PATCH 01/18] forge: allow source.strip on url-download sources unpack_source already honors source.strip (the tar --strip-components equivalent), but the schema locked the url-source object to url-only, so no recipe could set it. Expose it: strip: 0 unpacks a prebuilt release tarball whose files sit at the archive root verbatim, instead of silently dropping the root-level files under the default strip=1 (which assumes a single top-level wrapper directory). --- src/forge/schema/meta-schema.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/forge/schema/meta-schema.yaml b/src/forge/schema/meta-schema.yaml index 5db919aa..0cc2e23f 100644 --- a/src/forge/schema/meta-schema.yaml +++ b/src/forge/schema/meta-schema.yaml @@ -61,6 +61,16 @@ properties: properties: url: type: string + strip: + type: integer + minimum: 0 + description: >- + Number of leading path components to strip when unpacking, like + `tar --strip-components`. Defaults to 1 (drop the archive's single + top-level wrapper directory, the common case for sdists and source + tarballs). Set to 0 for prebuilt release archives whose files sit + at the archive root with no wrapper directory, so root-level files + (e.g. a static library beside an `include/` dir) are not dropped. additionalProperties: false description: >- Download and unpack an archive from a URL. Use when there is no PyPI From b95d4ee5e48a83982afb0bac5318362f860136c3 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Thu, 13 Aug 2026 11:37:00 +0200 Subject: [PATCH 02/18] recipes: curl-cffi 0.16.0 + flet-libcurl-impersonate 2.0.0 Adds TLS-impersonation HTTP (curl_cffi, flet#6768) for iOS and Android. flet-libcurl-impersonate repackages lexiforest/curl-impersonate's prebuilt static mega-archive (patched curl 8.21.0 + BoringSSL + nghttp2/nghttp3 + ngtcp2 QUIC + brotli + zstd + zlib, all in one libcurl-impersonate.a) per slice; source.strip: 0 because the tarballs carry the .a + include/curl/ at the archive root. armeabi-v7a is excluded (no upstream 32-bit ARM binary). curl-cffi links that archive into its cffi _wrapper extension. mobile.patch adds one opt-in lever, IMPERSONATE_FORGE_TARGET, so scripts/build.py selects the target arch + static-link recipe from the forge target instead of the build host's uname (which is never the target under cross-compilation), and takes the prebuilt .a from the flet-libcurl-impersonate host_build dep via IMPERSONATE_BUILD_DIR (no network in the build). On iOS -- the first-ever curl-cffi iOS build -- it also links the Apple system libraries the archive needs (Security + CoreFoundation for USE_APPLE_SECTRUST, libiconv + libicucore for USE_APPLE_IDN). Built and verified on-device for all five slices (android arm64-v8a/x86_64, iOS device arm64, iOS sim arm64/x86_64): import loads the extension, the linked libcurl reports the impersonate build, and curl_easy_impersonate() applies a browser fingerprint. Runtime deps cffi>=2.0.0 + certifi resolve from pypi.flet.dev. [skip ci] --- recipes/curl-cffi/meta.yaml | 39 ++++++++++ recipes/curl-cffi/patches/mobile.patch | 90 ++++++++++++++++++++++ recipes/curl-cffi/tests/test_curl_cffi.py | 38 +++++++++ recipes/flet-libcurl-impersonate/build.sh | 28 +++++++ recipes/flet-libcurl-impersonate/meta.yaml | 31 ++++++++ 5 files changed, 226 insertions(+) create mode 100644 recipes/curl-cffi/meta.yaml create mode 100644 recipes/curl-cffi/patches/mobile.patch create mode 100644 recipes/curl-cffi/tests/test_curl_cffi.py create mode 100755 recipes/flet-libcurl-impersonate/build.sh create mode 100644 recipes/flet-libcurl-impersonate/meta.yaml diff --git a/recipes/curl-cffi/meta.yaml b/recipes/curl-cffi/meta.yaml new file mode 100644 index 00000000..0abd50d1 --- /dev/null +++ b/recipes/curl-cffi/meta.yaml @@ -0,0 +1,39 @@ +{% set version = "0.16.0" %} + +package: + name: curl-cffi + version: '{{ version }}' + # No upstream curl-impersonate binary exists for 32-bit ARM Android, so the + # flet-libcurl-impersonate host_build dep can't be built there either. + excluded_arches: + - armeabi-v7a + +build: + number: 0 + script_env: + # flet-libcurl-impersonate stages the prebuilt static mega-archive and + # include/curl/ into {platlib}/opt. Point curl-cffi's ffibuilder there so it + # links that libcurl-impersonate.a and skips the GitHub download entirely + # (offline, deterministic build). IMPERSONATE_FORGE_TARGET is consumed by the + # mobile.patch to pick the target arch + link recipe instead of the build + # host's uname. + IMPERSONATE_BUILD_DIR: '{platlib}/opt' + IMPERSONATE_LINK_TYPE: static +# {% if sdk == 'android' %} + IMPERSONATE_FORGE_TARGET: android +# {% else %} + IMPERSONATE_FORGE_TARGET: ios +# {% endif %} + +requirements: + host_build: + - flet-libcurl-impersonate 2.0.0 +# {% if sdk == 'android' %} + host: + # The extension links -lc++ -> DT_NEEDED libc++_shared.so on Android (iOS + # links the system libc++ statically, so this is Android-only). + - flet-libcpp-shared >=27.2.12479018 +# {% endif %} + +patches: + - mobile.patch diff --git a/recipes/curl-cffi/patches/mobile.patch b/recipes/curl-cffi/patches/mobile.patch new file mode 100644 index 00000000..22a98377 --- /dev/null +++ b/recipes/curl-cffi/patches/mobile.patch @@ -0,0 +1,90 @@ +Cross-compile curl-cffi for iOS and Android under mobile-forge. + +curl-cffi's scripts/build.py selects the target arch by inspecting the build +host's platform.uname(), and chooses the static-link recipe (Apple -force_load +vs ELF --whole-archive, and whether to add -lc++) from the host's +platform.system(). Under mobile-forge the build host (macOS or Linux) is never +the wheel's target, so both of those are wrong for every slice. + +This patch adds a single opt-in lever, IMPERSONATE_FORGE_TARGET (set to "ios" or +"android" by the recipe's build.script_env). When present: + + * detect_arch() returns a synthesized target entry (static link, obj_name + libcurl-impersonate.a) and takes the prebuilt mega-archive + headers from + IMPERSONATE_BUILD_DIR (staged into {platlib}/opt by the + flet-libcurl-impersonate recipe). The GitHub download is skipped because the + .a already exists there, so the build is fully offline and never leaks a + stale arch across forge's reused source tree. + + * the link recipe is forced to the target's: Darwin (-force_load + -lc++) for + iOS, ELF --whole-archive + -lc++ for Android. + + * on iOS (the first-ever curl-cffi iOS build) the extension additionally links + the Apple system libraries the curl-impersonate iOS archive depends on: + Security + CoreFoundation (USE_APPLE_SECTRUST) and libiconv + libicucore + (USE_APPLE_IDN). Real-macOS builds don't need these and are untouched. + +Upstream behaviour is unchanged whenever IMPERSONATE_FORGE_TARGET is unset. + +--- a/scripts/build.py 2026-08-01 14:46:08 ++++ b/scripts/build.py 2026-08-13 11:17:48 +@@ -31,6 +31,27 @@ + with open(Path(__file__).parent.parent / "libs.json") as f: + archs = json.loads(f.read()) + ++ # mobile-forge cross-compilation: the build host (macOS/Linux) is not the ++ # wheel's target, so the host uname cannot select the target arch. When ++ # IMPERSONATE_FORGE_TARGET is set, synthesize the target entry directly and ++ # take the prebuilt libcurl-impersonate.a + include/ from IMPERSONATE_BUILD_DIR ++ # (staged by the flet-libcurl-impersonate recipe); the network download is ++ # skipped because /libcurl-impersonate.a already exists. ++ forge_target = os.environ.get("IMPERSONATE_FORGE_TARGET") ++ if forge_target in ("ios", "android"): ++ build_dir = os.path.expanduser(os.environ["IMPERSONATE_BUILD_DIR"]) ++ return { ++ "system": "Darwin" if forge_target == "ios" else "Linux", ++ "machine": platform.uname().machine, ++ "pointer_size": struct.calcsize("P") * 8, ++ "link_type": "static", ++ "obj_name": "libcurl-impersonate.a", ++ "libc": "android" if forge_target == "android" else None, ++ "libdir": build_dir, ++ "sysname": "", ++ "arch": "", ++ } ++ + uname = platform.uname() + uname_system = "Android" if is_android_env() else uname.system + glibc_flavor = "gnueabihf" if uname.machine in ["armv7l", "armv6l"] else "gnu" +@@ -181,6 +202,13 @@ + + ffibuilder = FFI() + system = platform.system() ++# mobile-forge: the link recipe (force_load vs --whole-archive, -lc++) must match ++# the wheel's target, not the macOS/Linux build host running this script. ++_forge_target = os.environ.get("IMPERSONATE_FORGE_TARGET") ++if _forge_target == "android": ++ system = "Linux" # ELF --whole-archive branch; -lc++ added via is_android ++elif _forge_target == "ios": ++ system = "Darwin" # -force_load + -lc++, correct for the iOS toolchain + root_dir = Path(__file__).parent.parent + download_libcurl() + +@@ -193,6 +221,17 @@ + f"-Wl,-force_load,{static_libs[0]}", + "-lc++", + ] ++ if _forge_target == "ios": ++ # First-ever iOS build: curl-impersonate's iOS mega-archive is built ++ # with USE_APPLE_SECTRUST + USE_APPLE_IDN, so the consuming extension ++ # must pull in the matching Apple system libraries (Security + ++ # CoreFoundation for SecTrust, libiconv + libicucore for Apple IDN). ++ extra_link_args += [ ++ "-framework", "Security", ++ "-framework", "CoreFoundation", ++ "-liconv", ++ "-licucore", ++ ] + elif is_android: + extra_link_args = [ + "-Wl,--whole-archive", diff --git a/recipes/curl-cffi/tests/test_curl_cffi.py b/recipes/curl-cffi/tests/test_curl_cffi.py new file mode 100644 index 00000000..5c9fc0d0 --- /dev/null +++ b/recipes/curl-cffi/tests/test_curl_cffi.py @@ -0,0 +1,38 @@ +"""On-device smoke tests for curl_cffi — the cffi bindings around +curl-impersonate (a patched libcurl + BoringSSL + nghttp2/3 + ngtcp2 + brotli + +zstd + zlib, all statically linked into the `_wrapper` extension). These are +network-free: they exercise the compiled native library directly, without making +any HTTP request (an emulator/simulator has no guaranteed connectivity).""" + + +def test_import_loads_native_wrapper(): + """Importing curl_cffi loads its compiled cffi extension (_wrapper) — proves + the mega-archive linked and _cffi_backend / libc++_shared resolve on load.""" + import curl_cffi + from curl_cffi import _wrapper + + assert _wrapper.lib is not None + assert _wrapper.ffi is not None + + +def test_linked_libcurl_is_impersonate_build(): + """__curl_version__ is a real lib.curl_version() call into the statically + linked library; confirm it is the curl-impersonate build, not system curl.""" + import curl_cffi + + version = curl_cffi.__curl_version__ + assert version.startswith("libcurl/"), version + assert "impersonate" in version.lower(), version + + +def test_curl_easy_impersonate_applies_fingerprint(): + """curl_easy_impersonate() succeeds for a known browser target, proving the + TLS/HTTP2 fingerprint tables are compiled into the native library.""" + from curl_cffi import Curl + + curl = Curl() + try: + ret = curl.impersonate("chrome110") + assert ret == 0, f"curl_easy_impersonate returned {ret}" + finally: + curl.close() diff --git a/recipes/flet-libcurl-impersonate/build.sh b/recipes/flet-libcurl-impersonate/build.sh new file mode 100755 index 00000000..398d6134 --- /dev/null +++ b/recipes/flet-libcurl-impersonate/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -eu + +# Stage the prebuilt curl-impersonate static mega-archive + headers so the +# curl-cffi recipe can statically link it. curl-cffi's ffibuilder expects a +# single "libdir" that contains both libcurl-impersonate.a and an include/curl/ +# subtree; it points IMPERSONATE_BUILD_DIR at {platlib}/opt, so lay the files out +# exactly that way (the .a directly under opt/, headers under opt/include/). +# +# source.strip=0 left the archive's root-level files in place, and each upstream +# tarball is already thin for its slice (arm64 device, arm64/x86_64 simulator, +# aarch64/x86_64 android — all verified single-arch), so no lipo/thinning here. +# Nothing is shipped to the device: this wheel is a host_build (link-time) dep. + +if [ ! -f libcurl-impersonate.a ]; then + echo "flet-libcurl-impersonate: libcurl-impersonate.a not found after unpack" >&2 + ls -la >&2 + exit 1 +fi +if [ ! -d include/curl ]; then + echo "flet-libcurl-impersonate: include/curl/ not found after unpack" >&2 + ls -la >&2 + exit 1 +fi + +mkdir -p "$PREFIX" +cp libcurl-impersonate.a "$PREFIX/libcurl-impersonate.a" +cp -R include "$PREFIX/include" diff --git a/recipes/flet-libcurl-impersonate/meta.yaml b/recipes/flet-libcurl-impersonate/meta.yaml new file mode 100644 index 00000000..8664a946 --- /dev/null +++ b/recipes/flet-libcurl-impersonate/meta.yaml @@ -0,0 +1,31 @@ +{% set version = "2.0.0" %} + +package: + name: flet-libcurl-impersonate + version: '{{ version }}' + # Prebuilt curl-impersonate static mega-archive (a single libcurl-impersonate.a + # bundling patched curl + BoringSSL + nghttp2/nghttp3/ngtcp2 + brotli + zstd + + # zlib) for consumers that statically link it — the curl-cffi recipe. This is a + # link-time (host_build) dependency only; nothing is shipped to the device. + # + # Upstream publishes a per-(arch, platform) asset for exactly our slices, EXCEPT + # armeabi-v7a — there is no 32-bit ARM Android build of curl-impersonate. + excluded_arches: + - armeabi-v7a + +source: +# {% if sdk == 'iphoneos' %} + url: https://github.com/lexiforest/curl-impersonate/releases/download/v{{ version }}/libcurl-impersonate-v{{ version }}.arm64-apple-ios.tar.gz +# {% elif sdk == 'iphonesimulator' %} + url: https://github.com/lexiforest/curl-impersonate/releases/download/v{{ version }}/libcurl-impersonate-v{{ version }}.{{ arch }}-apple-ios-simulator.tar.gz +# {% else %} + # Android: arm64-v8a -> aarch64, x86_64 -> x86_64. + url: https://github.com/lexiforest/curl-impersonate/releases/download/v{{ version }}/libcurl-impersonate-v{{ version }}.{% if arch == 'arm64-v8a' %}aarch64{% else %}{{ arch }}{% endif %}-linux-android.tar.gz +# {% endif %} + # The release tarballs hold libcurl-impersonate.a + include/curl/ at the archive + # root (no wrapper directory), so unpack verbatim rather than stripping a + # nonexistent top-level component. + strip: 0 + +build: + number: 0 From 7a9686f69c685848df9fc20927e044dd688c85a6 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Thu, 13 Aug 2026 13:04:35 +0200 Subject: [PATCH 03/18] skills: capture curl-cffi patterns (cffi-consumer-of-prebuilt, iOS Apple-frameworks, source.strip:0) Fold the reusable findings from the curl-cffi recipe into the skills: - new-mobile-recipe: a shape row + deep-dive for a cffi/ctypes consumer whose own build self-detects arch via the host uname and downloads a prebuilt static lib (steer it off the forge target; stage the lib via a source.strip:0 flet-lib* dep). - forge-error-catalogue: iOS undefined _SecTrust*/_iconv/_uidna_* when linking an Apple-built static archive -> link Security/CoreFoundation/iconv/icucore; and the source.url root-level-tarball -> source.strip:0 fix. [skip ci] --- .claude/skills/forge-error-catalogue/SKILL.md | 5 +++ .../references/failure-catalogue.md | 31 +++++++++++++++++++ .claude/skills/new-mobile-recipe/SKILL.md | 13 ++++++++ 3 files changed, 49 insertions(+) diff --git a/.claude/skills/forge-error-catalogue/SKILL.md b/.claude/skills/forge-error-catalogue/SKILL.md index 8ed16081..7ec909e2 100644 --- a/.claude/skills/forge-error-catalogue/SKILL.md +++ b/.claude/skills/forge-error-catalogue/SKILL.md @@ -78,6 +78,11 @@ instead of re-deriving it. hand-written + gen2.py-generated code), **opencv-5 KleidiCV `armv8-a` on x86_64** and **hardcoded `CMAKE_SYSTEM_PROCESSOR` → ARM asm on the x86_64 sim** (per-arch fix), **Rust crate with no `target_os="ios"` backend** (`mac_address`, cfg-gate it), + **iOS undefined `_SecTrust*`/`_iconv`/`_uidna_*` linking a prebuilt Apple-built + static lib → `-framework Security -framework CoreFoundation -liconv -licucore`** + (curl-impersonate archive; iOS-only, the macOS build omits them), + **`source.url` prebuilt tarball unpacks with root-level `.a`/`.so` MISSING → + `source.strip: 0`** (default strip=1 drops files with no wrapper dir), **`User for pypi.flet.dev:` → `EOFError` at build-tool/host-dep resolution** (the index 401s, not the recipe — deterministic locally on a fresh cmake cross-venv, transient/scattered in CI where a rerun clears it). diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index 99a85066..85b555df 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -122,6 +122,37 @@ If upstream hardcodes `-L/usr/lib/X` somewhere, write a `mobile.patch` to strip --- +### iOS `Undefined symbols: _SecTrustEvaluateWithError / _iconv / _uidna_nameToASCII_UTF8 / _kCFTypeArrayCallBacks` linking a prebuilt Apple-built static lib + +**Cause:** a prebuilt static archive built for iOS with Apple's native TLS trust store (`USE_APPLE_SECTRUST`) and/or Apple IDN (`USE_APPLE_IDN`) — e.g. curl-impersonate's `libcurl-impersonate.a` — has undefined references into Apple **system** libraries that its objects do NOT carry. When you statically link that `.a` into a Python extension, those symbols surface as `ld: Undefined symbols for architecture arm64`. The symbol → library map: + +- `_SecTrust*`, `_SecCertificate*`, `_SecPolicy*` → `-framework Security` +- `_kCFType*`, `_CFRelease`, `CFString*`, `CFArray*` → `-framework CoreFoundation` +- `_iconv`, `_iconv_open`, `_iconv_close` → `-liconv` +- `_uidna_*` (ICU International Domain Names) → `-licucore` + +(The real-macOS build of the same package usually does NOT need these — macOS uses a different verify/IDN path — so upstream's link args omit them, and this is iOS-only.) + +**Fix:** append the frameworks/libs to the extension's link args, gated to iOS. They resolve from the SDK sysroot (`.tbd` stubs in `usr/lib` + `System/Library/Frameworks`) with no extra `-L`/`-F`. In a cffi recipe this is a `mobile.patch` hunk on the `extra_link_args` list; in a `setup.py`/CMake recipe it can also ride in `script_env` `LDFLAGS`. Precedent: `recipes/curl-cffi/patches/mobile.patch` (branch `curl-cffi`) adds `["-framework","Security","-framework","CoreFoundation","-liconv","-licucore"]` for the iOS slice only. + +--- + +### `source.url` prebuilt tarball unpacks with the top-level files MISSING (`.a`/`.so` gone, only `include/…` present) + +**Cause:** forge's `unpack_source` defaults to `strip=1` (`member.path.split("/", 1)[1]`), which assumes a single top-level wrapper directory. A prebuilt **release** tarball often has its files at the archive **root** (`libX.a`, `libX.so`, `include/X/…` with no wrapper dir). At `strip=1` the root-level files have no `/` → `split(...)[1]` IndexErrors → they are silently **dropped**, while `include/x.h` → `x.h`. build.sh then can't find the `.a`. + +**Symptoms:** build.sh's own guard (`[ ! -f libX.a ]`) trips, or a downstream link fails with the lib missing, even though the tarball clearly contains it. + +**Fix:** set `strip: 0` on the source object so forge extracts verbatim: +```yaml +source: + url: https://.../libX-.tar.gz + strip: 0 +``` +(The `strip` key requires forge with the `source.strip` schema addition — on the `curl-cffi` branch / after it merges.) Verify the actual behavior empirically before trusting either value: replicate `members()` from `src/forge/build.py` on the real tarball. Precedent: `recipes/flet-libcurl-impersonate/` (branch `curl-cffi`). + +--- + ### `ImportError: dlopen failed: library "/abs/path/.../libpython3.12.so" not found` (Android, at runtime) **Cause:** The Android `libpython3.12.so` from the `flet-dev/python-build` tarball is built without a SONAME (no `-Wl,-soname,libpython3.12.so` at libpython link time). When a recipe uses CMake's `target_link_libraries(... Python::Python)` to link explicitly against libpython (which pyzmq does on Android via `EXTRA_PYTHON_COMPONENT=Development.Embed`), the linker can't use a SONAME and falls back to using the input argument as DT_NEEDED. With `-DPython_LIBRARY=`, that path gets embedded verbatim. At runtime, Android's loader looks for the absolute build-host path and fails. diff --git a/.claude/skills/new-mobile-recipe/SKILL.md b/.claude/skills/new-mobile-recipe/SKILL.md index eea30e31..5b295c66 100644 --- a/.claude/skills/new-mobile-recipe/SKILL.md +++ b/.claude/skills/new-mobile-recipe/SKILL.md @@ -81,6 +81,7 @@ Match the package to one of these shapes. Each maps to a template in `templates/ | C-ext that links a lib via a `*-config` tool | Compiled C-ext whose `setup.py` shells out to `pg_config`/`mysql_config`/… (psycopg2→libpq, mysqlclient→libmysqlclient) | A **static+PIC** `flet-lib*` (`build-flet-lib.sh` + `-fPIC`) shipping a config-shim, + consumer `script_env`/patch; see Pattern I | | CMake giant, **no sdist AND no setup.py/pyproject.toml** | Upstream's only wheel path is a host==target build script (onnxruntime's `ci_build/build.py`, TF's `build_pip_package_with_cmake.sh`) | No template — copy from `recipes/onnxruntime/` or `recipes/tflite-runtime/` (branches `machine/onnxruntime` / `machine/tflite-runtime`); see "PEP 517 shim" deep-dive below | | **Prebuilt-repackage + host_build chain** | Upstream publishes official prebuilt mobile archives of the native lib AND the consumer's own cmake links + re-ships the `.so` (flet-libonnxruntime→sherpa-onnx) | `build.sh` repackager + consumer `requirements.host_build`; copy from `recipes/flet-libonnxruntime/` + `recipes/sherpa-onnx/` (branch `machine/sherpa-onnx`); see "prebuilt-repackage" deep-dive below | +| **cffi/ctypes consumer of a prebuilt archive** | The package's own build DOWNLOADS a prebuilt static lib keyed by the build host's `uname` and statically links it (curl-cffi → curl-impersonate); breaks under cross because host≠target | `flet-lib*` prebuilt-repackage dep (`source.strip: 0` for root-level tarballs) + `requirements.host_build` + a `mobile.patch` opt-in env lever that steers arch/link off the forge target; copy from `recipes/flet-libcurl-impersonate/` + `recipes/curl-cffi/` (branch `curl-cffi`); see deep-dive below | If unsure, start with **minimal C-extension** and let the build tell you what's missing. Iterate up the table as failures surface. @@ -111,6 +112,18 @@ The shim's load-bearing rules (each one bought with a failed build): **Real example:** `machine/sherpa-onnx:recipes/flet-libonnxruntime/` + `machine/sherpa-onnx:recipes/sherpa-onnx/` (patches `android-no-jni.patch` + `android-preload-ort.patch`). +### Shape deep-dive: cffi/ctypes consumer that self-detects arch + downloads a prebuilt archive (curl-cffi → flet-libcurl-impersonate) + +**When:** the Python package's own build **downloads a prebuilt static lib keyed by the build host's `platform.uname()`** and statically links it (curl-cffi's `scripts/build.py` matches `uname` against a `libs.json`, downloads `libcurl-impersonate-v{ver}.{arch}-{sysname}.tar.gz`, and `--whole-archive`/`-force_load`s it into a cffi extension). This CANNOT work unpatched under forge: the build host (macOS/Linux) is never the wheel's target, so `uname`-based arch selection and any host-`platform.system()`-driven link recipe are wrong for every slice. + +The shape that works — a `flet-lib*` prebuilt-repackage dep + a small opt-in patch: + +1. **A `flet-lib*` recipe stages the prebuilt archive per slice** via `source.url` (one tarball per `(sdk, arch)` — Jinja-map `arm64-v8a`→`aarch64` etc.) into `$PREFIX` (=`wheel/opt`), so the consumer sees `{platlib}/opt/`. **`source.strip: 0`** when the tarball's files sit at the archive **root** with no wrapper dir (a prebuilt `.a` beside an `include/` — forge's default `strip=1` splits `path.split("/",1)[1]` and IndexError-**drops** every root-level file, keeping only `include/…`→`…`; verify empirically, it fails silently otherwise). Each upstream tarball is usually already thin per-slice → no `lipo`. `excluded_arches` any slice with no upstream binary (curl-impersonate has no `armeabi-v7a`). +2. **The consumer declares it `requirements.host_build`** and points the package's own "where's the lib" env var at it (`IMPERSONATE_BUILD_DIR: '{platlib}/opt'`) so the package's **download is skipped** (the `.a` already exists there) — fully offline, and no CI `tmplibdir`-reuse arch leak. +3. **A minimal `mobile.patch` adds ONE opt-in lever** (an env var the recipe's `script_env` sets, e.g. `IMPERSONATE_FORGE_TARGET: ios|android`) that (a) makes the arch-detect return a synthesized target entry instead of scanning `uname`, and (b) forces the target's static-link recipe (Apple `-force_load`+`-lc++` for iOS, ELF `--whole-archive`+`-lc++` for Android — the Android branch is the one the macOS host gets wrong). Guard it so upstream behaviour is unchanged when the var is unset. **iOS extra:** a static archive built with Apple SecTrust / Apple IDN needs the consuming extension to link `-framework Security -framework CoreFoundation -liconv -licucore` (see `forge-error-catalogue` § iOS undefined `_SecTrust*`/`_iconv`/`_uidna_*`). + +**Real example:** branch `curl-cffi` — `recipes/flet-libcurl-impersonate/` (build.sh, `source.strip: 0`) + `recipes/curl-cffi/` (`patches/mobile.patch`, 3 hunks). World-first curl-cffi iOS wheels; on-device 3/3 both platforms. Needed one forge-core change: exposing `source.strip` in `src/forge/schema/meta-schema.yaml` (the code already honored it). + ### Naming - Recipe directory name and `package.name` in meta.yaml must match the PyPI sdist filename exactly (case-sensitive). `MarkupSafe`, not `markupsafe`. `cffi`, not `CFFI`. Look at `-.tar.gz` filename on PyPI for the ground truth. From e0d3fdb0c793752c3f8cd77f61d02867f78bdcc4 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 31 Aug 2026 15:04:40 +0200 Subject: [PATCH 04/18] recipe: ship the curl-impersonate licence notices [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `flet-libcurl-impersonate` stopped building when #116 turned the missing-licence warning into an error: upstream's release tarballs are a bare `libcurl-impersonate.a` plus headers, with no notice anywhere in the archive, the release or CMakeLists.txt. `about.license_file: []` is the one option not available here — the wheel really does carry the object code, and curl, BoringSSL, brotli and zstd all attach notice conditions to binary redistribution. So the notices are vendored, one file per component, fetched at each component's pinned version tag. The component list is upstream's `CMakeLists.txt`, confirmed against the archive itself: `ar t` is no help on a single `ld -r` blob, but the merge left 665 `STT_FILE` symbols and the build's own `deps/src//` paths in `.rodata`, which attribute every object. Looked for and absent: c-ares, libpsl, libidn2, libssh2, wolfSSL, mbedTLS, GnuTLS, rustls — curl's own backend files for those are present but compile to empty translation units. BoringSSL is the load-bearing one. It is Apache-2.0 at this commit, not ISC and not the OpenSSL licence (the `SSLeay` strings in the binary are API-compat shims), so §4(d)'s NOTICE requirement and its patent grant are the only non-inert terms in the set. zstd's GPL-2.0 alternative is deliberately not in the expression: the dual licence is a fact about the project, not about this artifact, and we take the BSD arm. Two file-choice traps, both the libiconv trap in different directions — nghttp2's `LICENSE` is a 12-byte "See COPYING", and zstd's `COPYING` is 18 KB of GPL-2.0. --- recipes/flet-libcurl-impersonate/COPYING.curl | 22 ++ .../flet-libcurl-impersonate/COPYING.nghttp2 | 23 ++ .../flet-libcurl-impersonate/COPYING.nghttp3 | 22 ++ .../flet-libcurl-impersonate/COPYING.ngtcp2 | 22 ++ .../LICENSE.boringssl | 238 ++++++++++++++++++ .../flet-libcurl-impersonate/LICENSE.brotli | 19 ++ .../LICENSE.curl-impersonate | 21 ++ recipes/flet-libcurl-impersonate/LICENSE.zlib | 22 ++ recipes/flet-libcurl-impersonate/LICENSE.zstd | 30 +++ recipes/flet-libcurl-impersonate/meta.yaml | 41 ++- 10 files changed, 458 insertions(+), 2 deletions(-) create mode 100644 recipes/flet-libcurl-impersonate/COPYING.curl create mode 100644 recipes/flet-libcurl-impersonate/COPYING.nghttp2 create mode 100644 recipes/flet-libcurl-impersonate/COPYING.nghttp3 create mode 100644 recipes/flet-libcurl-impersonate/COPYING.ngtcp2 create mode 100644 recipes/flet-libcurl-impersonate/LICENSE.boringssl create mode 100644 recipes/flet-libcurl-impersonate/LICENSE.brotli create mode 100644 recipes/flet-libcurl-impersonate/LICENSE.curl-impersonate create mode 100644 recipes/flet-libcurl-impersonate/LICENSE.zlib create mode 100644 recipes/flet-libcurl-impersonate/LICENSE.zstd diff --git a/recipes/flet-libcurl-impersonate/COPYING.curl b/recipes/flet-libcurl-impersonate/COPYING.curl new file mode 100644 index 00000000..2f71d999 --- /dev/null +++ b/recipes/flet-libcurl-impersonate/COPYING.curl @@ -0,0 +1,22 @@ +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1996 - 2026, Daniel Stenberg, , and many +contributors, see the THANKS file. + +All rights reserved. + +Permission to use, copy, modify, and distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright +notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall not +be used in advertising or otherwise to promote the sale, use or other dealings +in this Software without prior written authorization of the copyright holder. diff --git a/recipes/flet-libcurl-impersonate/COPYING.nghttp2 b/recipes/flet-libcurl-impersonate/COPYING.nghttp2 new file mode 100644 index 00000000..80201792 --- /dev/null +++ b/recipes/flet-libcurl-impersonate/COPYING.nghttp2 @@ -0,0 +1,23 @@ +The MIT License + +Copyright (c) 2012, 2014, 2015, 2016 Tatsuhiro Tsujikawa +Copyright (c) 2012, 2014, 2015, 2016 nghttp2 contributors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/recipes/flet-libcurl-impersonate/COPYING.nghttp3 b/recipes/flet-libcurl-impersonate/COPYING.nghttp3 new file mode 100644 index 00000000..37562ea5 --- /dev/null +++ b/recipes/flet-libcurl-impersonate/COPYING.nghttp3 @@ -0,0 +1,22 @@ +The MIT License + +Copyright (c) 2019 nghttp3 contributors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/recipes/flet-libcurl-impersonate/COPYING.ngtcp2 b/recipes/flet-libcurl-impersonate/COPYING.ngtcp2 new file mode 100644 index 00000000..9b367cdc --- /dev/null +++ b/recipes/flet-libcurl-impersonate/COPYING.ngtcp2 @@ -0,0 +1,22 @@ +The MIT License + +Copyright (c) 2016 ngtcp2 contributors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/recipes/flet-libcurl-impersonate/LICENSE.boringssl b/recipes/flet-libcurl-impersonate/LICENSE.boringssl new file mode 100644 index 00000000..37a5b743 --- /dev/null +++ b/recipes/flet-libcurl-impersonate/LICENSE.boringssl @@ -0,0 +1,238 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +Licenses for support code +------------------------- + +Parts of the TLS test suite are under the Go license. This code is not included +in BoringSSL (i.e. libcrypto and libssl) when compiled, however, so +distributing code linked against BoringSSL does not trigger this license: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/recipes/flet-libcurl-impersonate/LICENSE.brotli b/recipes/flet-libcurl-impersonate/LICENSE.brotli new file mode 100644 index 00000000..33b7cdd2 --- /dev/null +++ b/recipes/flet-libcurl-impersonate/LICENSE.brotli @@ -0,0 +1,19 @@ +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/recipes/flet-libcurl-impersonate/LICENSE.curl-impersonate b/recipes/flet-libcurl-impersonate/LICENSE.curl-impersonate new file mode 100644 index 00000000..7475f58f --- /dev/null +++ b/recipes/flet-libcurl-impersonate/LICENSE.curl-impersonate @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 curl_cffi developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/recipes/flet-libcurl-impersonate/LICENSE.zlib b/recipes/flet-libcurl-impersonate/LICENSE.zlib new file mode 100644 index 00000000..ab8ee6f7 --- /dev/null +++ b/recipes/flet-libcurl-impersonate/LICENSE.zlib @@ -0,0 +1,22 @@ +Copyright notice: + + (C) 1995-2022 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu diff --git a/recipes/flet-libcurl-impersonate/LICENSE.zstd b/recipes/flet-libcurl-impersonate/LICENSE.zstd new file mode 100644 index 00000000..75800288 --- /dev/null +++ b/recipes/flet-libcurl-impersonate/LICENSE.zstd @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/recipes/flet-libcurl-impersonate/meta.yaml b/recipes/flet-libcurl-impersonate/meta.yaml index 8664a946..264fcaa7 100644 --- a/recipes/flet-libcurl-impersonate/meta.yaml +++ b/recipes/flet-libcurl-impersonate/meta.yaml @@ -1,4 +1,4 @@ -{% set version = "2.0.0" %} +{% set version = "2.1.1" %} package: name: flet-libcurl-impersonate @@ -28,4 +28,41 @@ source: strip: 0 build: - number: 0 + number: 1 + +about: + # The release tarballs ship no licence text at all, so the notices are vendored + # here. libcurl-impersonate.a is a single `ld -r` blob merging nine upstream + # projects, pinned by upstream's CMakeLists.txt at this tag and confirmed in the + # archive by symbol probe: + # + # curl 8.21.0 curl COPYING.curl + # curl-impersonate patches MIT LICENSE.curl-impersonate + # BoringSSL 156c7b75 Apache-2.0 LICENSE.boringssl + # nghttp2 1.63.0 MIT COPYING.nghttp2 + # nghttp3 1.15.0 MIT COPYING.nghttp3 + # ngtcp2 1.20.0 MIT COPYING.ngtcp2 + # brotli 1.2.0 MIT LICENSE.brotli + # zstd 1.5.7 BSD-3-Clause LICENSE.zstd + # zlib 1.3.1 Zlib LICENSE.zlib + # + # On a bump, re-fetch each file from its project at the NEW pinned version and + # re-run the probe — the component set is upstream's to change. Two traps: take + # nghttp2's COPYING (its LICENSE is a 12-byte "See COPYING") and zstd's LICENSE + # (its COPYING is the GPL-2.0 arm of a dual licence; we take the BSD arm). + # + # Same set on both platforms — one superbuild, only BoringSSL's asm differs. + license_file: + - COPYING.curl + - LICENSE.curl-impersonate + - LICENSE.boringssl + - COPYING.nghttp2 + - COPYING.nghttp3 + - COPYING.ngtcp2 + - LICENSE.brotli + - LICENSE.zstd + - LICENSE.zlib + # Deduplicated set of the nine, ANDed because the archive holds all of them at + # once. zstd's GPL-2.0 alternative is deliberately not expressed: it is a fact + # about the project, not about this artifact. + license: curl AND MIT AND Apache-2.0 AND BSD-3-Clause AND Zlib From e8c22001c4787ee51039195a8ea7ed61096fa398 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 31 Aug 2026 15:04:58 +0200 Subject: [PATCH 05/18] recipes: curl-cffi 0.16.2 + flet-libcurl-impersonate 2.1.1 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The library version tracks curl-cffi's own pin rather than the newest release: `scripts/build.py::__version__` is "2.1.1" at 0.16.2, and that one line is the only change to the file the patch touches. Taking curl-impersonate 2.2.0 instead would have been silent — our patch skips the download that constant normally drives, and `ffi/cdef.c` compiles against curl-cffi's own bundled `include/curl/`, so a skew links a mismatched archive and still goes green. meta.yaml now says so next to the pin. `build.number` 0 -> 1 on both. These were the only two `number: 0` recipes in the tree against a schema default of 1, and 0 is not a smaller build number — for the Python path `build.py` only passes `--build-number` when the value is truthy, so curl-cffi's wheel shipped with no build tag at all and lost the PEP 427 tie-break. That matters here specifically: PyPI carries `curl_cffi-0.16.2-cp313/cp314-\ android_24_arm64_v8a`, so `flet build` sees two builds of one version. CI could not have caught it either — the mobile test rewrites local wheels' build tag to 9999. Adds `test_default_cacert_resolves_to_a_readable_file`, which backs the README's certificate claims on device: curl-cffi resolves its CA bundle once at import and hands the path to libcurl, so if that path were not a readable PEM every HTTPS request would fail with CURLE_SSL_CACERT_BADFILE instead of anything diagnosable. Verified on device, both platforms, with the recipe-tester and a throwaway probe: 4/4 tests EXIT 0 on an Android emulator (arm64, API 34) and an iOS simulator, and a real `curl_cffi.get(..., impersonate="chrome")` returns 200 on both. No certifi wiring is needed — `flet build`'s generated `lib/python.dart` exports SSL_CERT_FILE before the app runs, and certifi's zipimport path extracts `cacert.pem` to a real file anyway. --- recipes/curl-cffi/meta.yaml | 10 +++++++--- recipes/curl-cffi/patches/mobile.patch | 8 ++++---- recipes/curl-cffi/tests/test_curl_cffi.py | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/recipes/curl-cffi/meta.yaml b/recipes/curl-cffi/meta.yaml index 0abd50d1..ec84aff3 100644 --- a/recipes/curl-cffi/meta.yaml +++ b/recipes/curl-cffi/meta.yaml @@ -1,4 +1,4 @@ -{% set version = "0.16.0" %} +{% set version = "0.16.2" %} package: name: curl-cffi @@ -9,7 +9,7 @@ package: - armeabi-v7a build: - number: 0 + number: 1 script_env: # flet-libcurl-impersonate stages the prebuilt static mega-archive and # include/curl/ into {platlib}/opt. Point curl-cffi's ffibuilder there so it @@ -27,7 +27,11 @@ build: requirements: host_build: - - flet-libcurl-impersonate 2.0.0 + # Must equal scripts/build.py::__version__ in the pinned curl-cffi sdist. That + # constant normally drives the download URL, so a skew self-corrects; the patch + # skips the download, and cdef.c compiles against curl-cffi's own bundled + # include/curl/ headers — so a mismatch links a skewed archive and still goes green. + - flet-libcurl-impersonate 2.1.1 # {% if sdk == 'android' %} host: # The extension links -lc++ -> DT_NEEDED libc++_shared.so on Android (iOS diff --git a/recipes/curl-cffi/patches/mobile.patch b/recipes/curl-cffi/patches/mobile.patch index 22a98377..0fd19cd5 100644 --- a/recipes/curl-cffi/patches/mobile.patch +++ b/recipes/curl-cffi/patches/mobile.patch @@ -19,8 +19,8 @@ This patch adds a single opt-in lever, IMPERSONATE_FORGE_TARGET (set to "ios" or * the link recipe is forced to the target's: Darwin (-force_load + -lc++) for iOS, ELF --whole-archive + -lc++ for Android. - * on iOS (the first-ever curl-cffi iOS build) the extension additionally links - the Apple system libraries the curl-impersonate iOS archive depends on: + * on iOS the extension additionally links the Apple system libraries the + curl-impersonate iOS archive depends on: Security + CoreFoundation (USE_APPLE_SECTRUST) and libiconv + libicucore (USE_APPLE_IDN). Real-macOS builds don't need these and are untouched. @@ -75,8 +75,8 @@ Upstream behaviour is unchanged whenever IMPERSONATE_FORGE_TARGET is unset. "-lc++", ] + if _forge_target == "ios": -+ # First-ever iOS build: curl-impersonate's iOS mega-archive is built -+ # with USE_APPLE_SECTRUST + USE_APPLE_IDN, so the consuming extension ++ # curl-impersonate's iOS mega-archive is built with ++ # USE_APPLE_SECTRUST + USE_APPLE_IDN, so the consuming extension + # must pull in the matching Apple system libraries (Security + + # CoreFoundation for SecTrust, libiconv + libicucore for Apple IDN). + extra_link_args += [ diff --git a/recipes/curl-cffi/tests/test_curl_cffi.py b/recipes/curl-cffi/tests/test_curl_cffi.py index 5c9fc0d0..7731d206 100644 --- a/recipes/curl-cffi/tests/test_curl_cffi.py +++ b/recipes/curl-cffi/tests/test_curl_cffi.py @@ -36,3 +36,17 @@ def test_curl_easy_impersonate_applies_fingerprint(): assert ret == 0, f"curl_easy_impersonate returned {ret}" finally: curl.close() + + +def test_default_cacert_resolves_to_a_readable_file(): + """curl_cffi picks its CA bundle once, at import, and hands the path straight + to libcurl. Flet extracts certifi's bundle out of sitepackages.zip and points + SSL_CERT_FILE at it, so the path lands on disk — if it ever didn't, every + HTTPS request would fail with CURLE_SSL_CACERT_BADFILE instead.""" + import os + + from curl_cffi.curl import DEFAULT_CACERT + + assert os.path.isfile(DEFAULT_CACERT), DEFAULT_CACERT + with open(DEFAULT_CACERT, "rb") as f: + assert b"BEGIN CERTIFICATE" in f.read(), DEFAULT_CACERT From f92d13b67b2fb08f32f1d9e49b858e1bff67ba63 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 31 Aug 2026 15:05:11 +0200 Subject: [PATCH 06/18] docs: curl-cffi consumer README + fingerprint-fanout example [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recipe README per README.rst § Documenting a recipe, and a runnable example that fans five browser profiles at one fingerprinting endpoint under a single `asyncio.gather` — the shape an app querying several endpoints at once actually needs, and the one thing no desktop run can establish, since JA3/JA4 hash the ClientHello and only the far end of a handshake can report it. Measured on an Android emulator rather than asserted: 5/5 probes, 886 ms wall against 3,208 ms summed, five distinct JA4/JA3n/JA3 triples, `off` sending no User-Agent at all, and `chrome150`/`chrome131_android`/`off` sharing one Akamai HTTP/2 hash — two clients can share an h2 fingerprint and be nothing alike on the wire. The page carries the mandatory `target_arch` note (no armeabi-v7a wheel exists, and `flet build apk` defaults to all three ABIs), and a Licensing bullet, because the one thing a reader cannot discover is that an MIT package's extension statically contains nine projects whose notices ship in a different wheel. The certificates section is the part worth reading twice, and it is the opposite of what it looks like: nothing needs configuring, because Flet sets SSL_CERT_FILE before the app runs. What does bite is that the device then trusts certifi's roots and only those, so a corporate root or mitmproxy that works under `flet run` fails on a phone. --- recipes/curl-cffi/README.md | 350 ++++++++++++++++++ .../examples/fingerprint-fanout/.gitignore | 7 + .../examples/fingerprint-fanout/README.md | 90 +++++ .../fingerprint-fanout/pyproject.toml | 19 + .../fingerprint-fanout/src/impersonation.py | 147 ++++++++ .../examples/fingerprint-fanout/src/main.py | 202 ++++++++++ 6 files changed, 815 insertions(+) create mode 100644 recipes/curl-cffi/README.md create mode 100644 recipes/curl-cffi/examples/fingerprint-fanout/.gitignore create mode 100644 recipes/curl-cffi/examples/fingerprint-fanout/README.md create mode 100644 recipes/curl-cffi/examples/fingerprint-fanout/pyproject.toml create mode 100644 recipes/curl-cffi/examples/fingerprint-fanout/src/impersonation.py create mode 100644 recipes/curl-cffi/examples/fingerprint-fanout/src/main.py diff --git a/recipes/curl-cffi/README.md b/recipes/curl-cffi/README.md new file mode 100644 index 00000000..8ac8eb6a --- /dev/null +++ b/recipes/curl-cffi/README.md @@ -0,0 +1,350 @@ +# curl-cffi + +[`curl-cffi`](https://curl-cffi.readthedocs.io/en/latest/) is an HTTP client that passes itself +off as a real browser. It binds, through cffi, to +[curl-impersonate](https://github.com/lexiforest/curl-impersonate) — a patched libcurl built +against BoringSSL — so the TLS ClientHello, the HTTP/2 settings and the header order that go out +on the wire are a named browser's rather than a Python library's, and a server that fingerprints +its callers by [JA3/JA4](https://github.com/FoxIO-LLC/ja4) or by frame shape sees Chrome. Reach +for it in a Flet app when an API or page you need answers an ordinary client with a challenge +page, a 403 or a silent block. + +The API is `requests`-shaped and everything is re-exported at the top level: +[`curl_cffi.get(url, ...)`](https://curl-cffi.readthedocs.io/en/latest/quick_start.html) for a +one-shot, `Session` or +[`AsyncSession`](https://curl-cffi.readthedocs.io/en/latest/asyncio.html) when you want the +connection pool and the cookie jar to survive between calls, and +[`WebSocket`](https://curl-cffi.readthedocs.io/en/latest/websockets.html) for a socket. + +## Install + +Add curl-cffi to your `pyproject.toml`: + +```toml +dependencies = [ + "flet", + "curl-cffi", +] + +[tool.flet.android] +target_arch = ["arm64-v8a", "x86_64"] +``` + +**The [`target_arch`](https://flet.dev/docs/publish/android/#supported-target-architectures) line +is required, not optional.** No `armeabi-v7a` wheel is published — upstream ships no 32-bit ARM +build of curl-impersonate for anything to link against — so a default `flet build apk`, which +targets all three ABIs, fails at dependency resolution for the 32-bit one after the other two +have already resolved, which makes the failure look like a fluke. Spell the ABI names out in +full; `arm64` and `x64` are the macOS spellings and Flet rejects them here. Dropping 32-bit ARM +costs you old hardware rather than current users — 64-bit has been mandatory for Play Store +uploads since 2019. + +**On Python 3.13 and 3.14, Android arm64-v8a can resolve to upstream's own wheel rather than +this index's.** `flet build` installs with `pip install --upgrade --only-binary :all: +--extra-index-url https://pypi.flet.dev`, so pip sees PyPI too, and upstream publishes exactly +one mobile wheel per minor there — `cp313` and `cp314`, `android_24_arm64_v8a`, at the same +version this index carries; no cp312, no x86_64, no armeabi-v7a, no iOS. One Android build can +therefore carry two different builds of a single version across its ABIs, and a `==` pin cannot +separate them because the versions agree. They are not the same payload — upstream's arm64-v8a +wheel is 8.2 MB compressed and 25.2 MB unpacked against 2.7 MB and 6.3 MB here — so pin +`curl-cffi` to the version in [`meta.yaml`](meta.yaml) if you want the wheel this page describes on +every slice, as the [`fingerprint-fanout`](examples/fingerprint-fanout) example's `pyproject.toml` +does. Which one an unpinned resolve prefers when the versions agree was not measured. + +## Examples + +See runnable Flet apps in [`examples/`](examples): + +- [`fingerprint-fanout`](examples/fingerprint-fanout) — five browser profiles probed at once, + with the fingerprints each one produced. + +## Usage in a Flet app + +Build a session inside a handler, name a profile, and put the decoded body into a control: + +```python +import curl_cffi +import flet as ft + + +async def main(page: ft.Page): + async def load(e): + async with curl_cffi.AsyncSession(impersonate="chrome", timeout=15) as session: + response = await session.get("https://api.example.com/items") + feed.controls = [ft.Text(item["title"]) for item in response.json()] + page.update() + + page.add(ft.Button("Load", on_click=load), feed := ft.Column()) +``` + +The two arguments a mobile app should not leave to the defaults are both there. `impersonate=` +is the whole reason for the package and is off unless you pass it. `timeout=` defaults to 30 +seconds, which on a phone that just lost its signal is a spinner nobody waits out. + +The synchronous `Session` has the same shape, but its request blocks the calling thread outright +and must go through a worker — see [Threading](#threading). + +### Storage + +Bodies arrive in memory (`response.content` is `bytes`, `response.text` is `str`). Anything the +user expects to keep belongs in +[`FLET_APP_STORAGE_DATA`](https://flet.dev/docs/reference/environment-variables/#flet_app_storage_data), +which is never auto-deleted. + +Two paths the library picks for itself. The first is the response cache: `FileCacheBackend` +defaults to `tempfile.gettempdir() / "curl_cffi_cache"`, wherever the stdlib puts that on device. +Point it somewhere you chose: + +```python +import os +from datetime import timedelta + +from curl_cffi import FileCacheBackend, Session + +cache_dir = os.path.join(os.getenv("FLET_APP_STORAGE_CACHE", "."), "http-cache") +session = Session(cache=FileCacheBackend(expires=timedelta(hours=1), path=cache_dir)) +``` + +`expires` is required, and `cache=` also takes a bare `int` of seconds or a `timedelta` — which +builds a `FileCacheBackend` in the default directory, the one case where the path above is silently +skipped. [`FLET_APP_STORAGE_CACHE`](https://flet.dev/docs/reference/environment-variables/#flet_app_storage_cache) +is the right guarantee for bodies you can fetch again: the OS may purge it under storage pressure, +where [`FLET_APP_STORAGE_TEMP`](https://flet.dev/docs/reference/environment-variables/#flet_app_storage_temp) +can vanish between launches. + +The second is the fingerprint store. `FingerprintManager` resolves `$XDG_CONFIG_HOME/impersonate`, +falling back to `~/.config/impersonate`, and every request naming a target the wheel does not carry +natively reads `fingerprints.json` from there. Set `IMPERSONATE_CONFIG_DIR` to a directory under +`FLET_APP_STORAGE_DATA` before `import curl_cffi` if you ship extra fingerprints; `~` on device is +not a directory you chose. + +Caching is synchronous-only: `AsyncSession(cache=...)` raises +`NotImplementedError: AsyncSession does not support cache yet because CacheBackend I/O is +blocking.` Cookies live in the session and die with it, so persist any you need yourself. + +### Threading + +**Prefer `AsyncSession`, and drive it from Flet's own loop.** It does not use threads: it runs +`curl_multi` through the running asyncio loop's `add_reader`/`add_writer`, so it needs a loop and +takes the one it finds. Write `async def main(page)` and start work with +[`page.run_task(...)`](https://flet.dev/docs/controls/page/#flet.Page.run_task) — never +[`page.run_thread(...)`](https://flet.dev/docs/controls/page/#flet.Page.run_thread), which hands +the work to an executor with no running loop and then discards the future, so whatever the worker +raises disappears without a traceback. Four consequences follow: + +- **The first request binds the session to whichever loop is running then**, permanently. + Constructing one at module scope is harmless; using it from a second loop is not. +- **Close every session** — `async with`, or `await session.close()`. Each one keeps a background + task that wakes every 0.1 s until closed, which on a phone is battery spent on nothing. +- **Put the re-entrancy guard in the handler, not in the coroutine.** `run_task` only schedules, + so a `disabled = True` set inside the task has not happened when the handler returns and Flet + pushes the control's state. +- **End the task with an explicit + [`page.update()`](https://flet.dev/docs/controls/page/#flet.Page.update)** — auto-update fires only + around event handlers and around `main`. +- **Concurrency is capped at `max_clients`, 10 by default** — an `asyncio.gather` over more URLs + than that queues the rest rather than failing. + +For the synchronous `Session`, the request blocks the calling thread inside `curl_easy_perform`, +so it belongs in a `page.run_thread(...)` worker. A session is thread-safe — it mints a fresh +curl handle per thread — but its own docstring recommends one session per thread, and that is the +shape to copy. + +### Certificates + +curl-cffi resolves one CA bundle at **import** time and hands the path to libcurl at request +time. It tries `SSL_CERT_FILE`, `CURL_CA_BUNDLE` and `REQUESTS_CA_BUNDLE` in that order, then +OpenSSL's built-in path, then `certifi.where()` — and on device the first step wins, because +Flet's generated startup code sets `SSL_CERT_FILE` and `REQUESTS_CA_BUNDLE` to `certifi.where()` +before your module runs (read out of the `lib/python.dart` a `flet build` generates). So the +bundle is certifi's on both platforms, and `tests/` asserts on device that the resolved path is a +readable PEM. That it is *certifi's* follows from the startup code, not from the test. + +Two things follow. Overriding the bundle means setting `SSL_CERT_FILE` **above** your +`import curl_cffi` — the resolution happens at that import, not at Flet's startup — and a value +naming a file that does not exist is skipped silently. And **the trust anchors on device are only +certifi's**, so a corporate root or an intercepting debug proxy such as mitmproxy, which a +desktop `flet run` picks up out of the machine's own trust store, fails on the phone with a +`CertificateVerifyError` against a host the browser is perfectly happy with. Ship your own PEM in +`src/assets/` and name it per session or per request: + +```python +roots = os.path.join(os.getenv("FLET_ASSETS_DIR", "assets"), "roots.pem") +session = curl_cffi.Session(verify=roots) +``` + +[`FLET_ASSETS_DIR`](https://flet.dev/docs/reference/environment-variables/#flet_assets_dir) is where +`flet build` puts `src/assets/`. `verify=` covers the origin and the proxy alike, and unlike the env +var it cannot be defeated by import order. + +### Impersonation + +`impersonate="chrome"` is an alias that resolves to one particular Chrome build, chosen by the +release rather than by you; a versioned name such as `impersonate="chrome131_android"` says which. +[Upstream's table](https://curl-cffi.readthedocs.io/en/latest/impersonate/targets.html) is the +only complete list. + +**The built-in targets are compiled into the wheel, so that list is fixed at build time**, and +checkable offline: `Curl().impersonate(name)` returns `0` when that name's fingerprint tables are +present and a non-zero code when they are not, without raising — which separates "this build does +not know that target" from "the request failed" on a device with no connectivity. Through a +`Session` the same miss surfaces as `ImpersonateError` instead. + +That list is not the whole story: a name the wheel does not carry natively is looked up in +`fingerprints.json` under the directory in [Storage](#storage) and applied from Python, so extra +targets are shippable without a new wheel. Fetching them is +`FingerprintManager.update_fingerprints()` — importable on device, but it needs an +`IMPERSONATE_API_KEY` for impersonate.pro and writes into that config dir. Run it on a laptop and +ship the resulting `fingerprints.json`. + +### Android + +**The `INTERNET` permission is already there.** `flet build` starts its +[permission table](https://flet.dev/docs/publish/android/#permissions) from +`{"android.permission.INTERNET": True}` and merges your entries into it, so an outbound request +needs no `pyproject.toml` entry. This is the only platform where the question exists. + +**Site-packages is a ZIP here, a directory on iOS**, and curl-cffi runs out of it as-is: nothing +in the package needs a filesystem path of its own, so there is no +[`extract_packages`](https://flet.dev/docs/publish/android/#extract-packages) entry to write. The +CA bundle is the one file that does need one, and it is already a real path by the time curl-cffi +looks — `certifi.where()` unpacks its roughly 240 KB `cacert.pem` to a temporary file when the +package it lives in is zipped. + +### App size + +2.6–2.9 MB compressed and 6.3–7.0 MB unpacked per slice, measured across all five cp312 wheels +at this version (the x86_64 slices are the large end of each range; both platforms sit in it). Nearly all of it is the single +`_wrapper` extension, which carries the whole of curl, BoringSSL, nghttp2, nghttp3, ngtcp2, +brotli, zstd and zlib inside it — so +[`[tool.flet.cleanup]`](https://flet.dev/docs/publish/#compilation-and-cleanup) has nothing worth +removing. Android adds a shared C++ runtime of about 1.3 MB per ABI, which every C++ package in +your app shares rather than duplicating. + +The levers are an app bundle, split APKs, or the narrowed +[`target_arch`](https://flet.dev/docs/publish/android/#supported-target-architectures) that +[Install](#install) already forces you to write. These are payload figures, not the exact amount +added to the finished APK or IPA. + +### Other considerations + +A desktop `flet run` uses PyPI's wheel, and the native library behind it is the same +curl-impersonate release this recipe pins — `scripts/build.py` names the version and `meta.yaml` +matches it — so the impersonation targets are the same set in both places. + +The trust store is what differs, in the direction that produces "works on my laptop": the desktop +run finds the operating system's bundle where the device finds certifi's, so a root installed on +your machine is invisible on the phone. See [Certificates](#certificates). One real HTTPS request +against your own backend, on a device, is the thing to validate. + +## Things to know + +- **Every warning the library raises is silenced before you see it.** `curl_cffi/__init__.py` + ends with `config_warnings(on=False)`, a `simplefilter("ignore")` over the package's own warning + class — so the one it emits when a session built with its own `curl=` handle is used from a + second thread never reaches a log, on a platform where reading logs is already the hard part. Call + `curl_cffi.config_warnings(on=True)` while debugging, and do not read a silent run as a clean + one. + +- **Retries are off by default** (`retry=0`), and with the 30 s timeout that means a request made + as the radio drops fails once and stays failed. Both are worth setting deliberately for a phone; + see upstream's [advanced usage](https://curl-cffi.readthedocs.io/en/latest/advanced.html). + +- **HTTP/3 is compiled in but unproven on a device.** The shipped Android extension carries + ngtcp2 and nghttp3 symbols, and `http_version="v3"` exists in the API, but nothing here has run + a QUIC request from a phone or through an emulator's NAT. Read `curl_cffi.__curl_version__` on + the device you care about before designing around it — it is a real call into the linked + library, not a constant. + +- **Licensing:** curl_cffi is [MIT](https://spdx.org/licenses/MIT.html) and that is all its + metadata declares, but nine projects are compiled into the extension — patched curl, + curl-impersonate's own patches, BoringSSL, nghttp2, nghttp3, ngtcp2, brotli, zstd and zlib — + arriving as one static archive from + [`flet-libcurl-impersonate`](../flet-libcurl-impersonate). Every one of those licences + ([curl](https://spdx.org/licenses/curl.html), [MIT](https://spdx.org/licenses/MIT.html), + [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html), + [BSD-3-Clause](https://spdx.org/licenses/BSD-3-Clause.html), + [Zlib](https://spdx.org/licenses/Zlib.html)) is permissive and none restricts what you may + ship, but each asks that its copyright notice travel with the binary — and **those notices are + not in the curl_cffi wheel**. Its `dist-info/licenses/` holds one file, curl_cffi's own MIT + text, because the archive links in as a build-time dependency that ships nothing else to the + device. The nine texts are in `flet-libcurl-impersonate`'s wheel, under the same + `dist-info/licenses/`, and beside its `meta.yaml`; copy them into whatever acknowledgements + screen your app has. Flagging it, not advising you — we are not lawyers. + +## Build notes (maintainers) + +`patches/mobile.patch` explains its own hunks in its preamble, and `meta.yaml` justifies +`excluded_arches`, the `IMPERSONATE_*` environment and the host pins inline in both recipes. What +is left is shape, hazards, and what a green run does not prove. + +### Recipe shape + +**Two recipes because the payload is a prebuilt binary, not a build.** +`flet-libcurl-impersonate` repackages upstream's per-slice release tarball — a single +`libcurl-impersonate.a` merging nine projects — into `{platlib}/opt`, and curl-cffi links that. +Building curl and BoringSSL from source across five slices was rejected: BoringSSL's build system +is a project of its own, and a rebuild would drift from the fingerprints upstream actually tests. +The cost is that upstream's release assets are now the supply chain, which is what +[Upgrade hazards](#upgrade-hazards) is mostly about. + +It sits under `requirements.host_build`, so it links in without appearing in the consumer's +`Requires-Dist` and ships nothing to the device: `unzip -l` on the Android arm64-v8a wheel shows +36 files and no `opt/` directory, against 25 files and a 43 MB `opt/libcurl-impersonate.a` in the +library wheel it linked against. + +The platforms are asymmetric in one place, and it is why the Android wheel — and only the Android +wheel — declares `flet-libcpp-shared`: the arm64-v8a extension's `DT_NEEDED` is `libm`, +`libpython3.`, `libc++_shared`, `libdl`, `libc`, where iOS links libc++ statically. +`patches/mobile.patch` has the link recipes. + +### Upgrade hazards + +- **The two versions move independently.** A curl-cffi bump can expect a curl-impersonate release + whose assets `flet-libcurl-impersonate` does not pin, and a curl-impersonate bump can rename or + drop a per-slice asset. `scripts/build.py`'s own `__version__` is the number to compare the pin + against; when they diverge, the desktop-versus-device claim in + [Other considerations](#other-considerations) stops being true. +- **Upstream's release-asset naming and slice coverage is the single point of failure** for the + whole chain: `aarch64-linux-android`, `x86_64-linux-android`, `arm64-apple-ios` and the two + simulator assets. `excluded_arches: [armeabi-v7a]` is what makes `target_arch` mandatory in + [Install](#install); if a 32-bit ARM asset ever appears, that paragraph and the example's + `pyproject.toml` entry both stop being necessary. +- **`scripts/build.py` is the patch target.** A restructure of `detect_arch()` or of `libs.json` + makes the patch fail to apply — or worse, apply into a build that falls back to the host's + `uname` and downloads a host-architecture archive. +- **The target table, the trust anchors and the component set all move with the vendored + library**, without any recipe change — the last of those is what the Licensing bullet in + [Things to know](#things-to-know) rests on. So does upstream's own PyPI mobile coverage, which + [Install](#install) describes. + +### Re-verification checklist + +- `__curl_version__` on device still names an `impersonate` build, and + `curl_easy_impersonate("chrome110")` still returns 0 — `tests/` covers both. +- `DEFAULT_CACERT` is still a readable PEM on device (`tests/` covers this too), and Flet's + generated `lib/python.dart` still sets `SSL_CERT_FILE`. All of [Certificates](#certificates) + rests on that startup code, which is template output and can change under a Flet upgrade + without anything here failing. +- The consumer wheel still has no `opt/` directory and no second copy of anything from the + archive; the file count and `DT_NEEDED` set are unchanged. +- The licence texts actually present under `dist-info/licenses/` in both shipped wheels — the + Licensing bullet is a claim about what is and is not in them. +- Sizes re-measured per slice off the published wheels, summing file bytes rather than reading + `du`, which answers in binary units. +- Which wheel a bare `curl-cffi` resolves to, per slice and per minor, once this index publishes + one: `pip download --only-binary :all: --platform … --extra-index-url https://pypi.flet.dev`, + then read the filename. [Install](#install) currently says that is unmeasured. +- iOS `MH_DYLIB`; Android 16 KB `PT_LOAD` alignment. + +### Coverage gaps + +`tests/` is four network-free functions by design — an emulator has no guaranteed connectivity — +so **no test has ever made an HTTPS request from a device with this wheel.** They prove the +archive linked, that the library is the impersonate build, that one fingerprint table is present +and that the CA bundle path is readable. Nothing exercises `Session` or `AsyncSession`, a +handshake, cookies, proxies, HTTP/2 or HTTP/3, and nothing shows a fingerprint being accepted by a +real server. An import-and-construct suite passes even where the first request would fail; the +[`fingerprint-fanout`](examples/fingerprint-fanout) example is the thing to build and install to +close that gap. The current versions have also not been through a CI run — the established green +result is for the previous curl-cffi and curl-impersonate pair. diff --git a/recipes/curl-cffi/examples/fingerprint-fanout/.gitignore b/recipes/curl-cffi/examples/fingerprint-fanout/.gitignore new file mode 100644 index 00000000..429a8307 --- /dev/null +++ b/recipes/curl-cffi/examples/fingerprint-fanout/.gitignore @@ -0,0 +1,7 @@ +.venv/ +.flet/ +build/ +__pycache__/ +.pytest_cache/ +.ruff_cache/ +uv.lock diff --git a/recipes/curl-cffi/examples/fingerprint-fanout/README.md b/recipes/curl-cffi/examples/fingerprint-fanout/README.md new file mode 100644 index 00000000..3cf8526d --- /dev/null +++ b/recipes/curl-cffi/examples/fingerprint-fanout/README.md @@ -0,0 +1,90 @@ +# curl-cffi fingerprint fan-out + +Five browser profiles probe one fingerprinting endpoint at the same time, and the screen +lays the JA4, JA3n, JA3 and HTTP/2 hashes each handshake produced side by side. The band +above them is local and fills in with the radio off: the `libcurl-IMPERSONATE` version +string, whether this build holds fingerprint tables for each target, and the CA bundle +curl-cffi resolved at import. + +The probes are the only thing here that leaves the device. They go to +[tls.browserleaks.com](https://tls.browserleaks.com/), a third-party endpoint, because only +the far end of a handshake can report what the ClientHello looked like. + +What it demonstrates: + +- **What impersonation actually changes.** Five targets, five different ClientHellos, and + `off` — no impersonation — sends no `User-Agent` at all, which is the shape a filter looks + for when it decides a caller is a script. Setting + [`impersonate`](https://curl-cffi.readthedocs.io/en/latest/impersonate/_index.html) is the + whole of it. +- **The half you cannot see locally.** JA3 and JA4 hash the ClientHello — cipher suites, + extensions, curves, ALPN — so no socket the app opens itself can report them. That is why + the rows need the network and the header does not, and why a matching `User-Agent` alone + does not make a client look like Chrome. +- **Press "Probe all targets" again and read the tinted values**, which are the ones that + moved since the previous run. For both Chrome targets that is `ja3`, every time: Chrome has + [permuted its ClientHello extensions since version 110](https://curl-cffi.readthedocs.io/en/latest/faq.html#why-does-the-ja3-fingerprints-change-for-chrome-110-impersonation) + and curl-impersonate reproduces that, so the raw hash is different on every connection while + `ja3n` (extensions sorted) and `ja4` (sorted by construction) hold still. Safari, Firefox and + `off` do not move at all. Pinning a client by raw JA3 is pinning noise. +- **`chrome131_android` moves its `ja4` and `ja3n` too, now and then.** Roughly one handshake + in five carried one extra TLS extension — `t13d1517h2_…` instead of `t13d1516h2_…` — over 22 + handshakes, sequential and concurrent alike. No mechanism is claimed for it here; it is + another reason not to key anything off a single fingerprint, and it is why an occasional + tint on that row is not a bug in the app. +- **The `h2` line does not separate what the TLS lines do.** `chrome150`, + `chrome131_android` and `off` all came back with the same Akamai HTTP/2 hash. Two clients + can share an HTTP/2 fingerprint and be nothing alike on the wire. +- **The fan-out**, which is the point for anything that queries several endpoints at once. One + `asyncio.gather` over five tasks, and one + [`AsyncSession`](https://curl-cffi.readthedocs.io/en/latest/asyncio.html) per + target rather than one shared: a session pools connections, and a pooled connection is a + handshake that already happened, so sharing one would let an earlier target answer for a + later one. The footer prints the wall clock against the sum of the five, so the overlap is a + number rather than a claim — 886 ms against 3,208 ms, 3.6x, on an Android emulator. +- **Async on Flet's own loop.** + [`page.run_task(...)`](https://flet.dev/docs/controls/page/#flet.Page.run_task), not + `run_thread`: `AsyncSession` drives libcurl through the running loop's readers and writers, + a worker thread has no loop to give it, and `run_thread` discards whatever its worker + raises. The button is disabled in the handler rather than inside the coroutine, because + `run_task` only *schedules* — a `disabled` set inside the task has not happened when the + handler returns and Flet pushes the button's state, and a second tap in that window queues a + second fan-out into the same five rows. The task ends in the explicit + [`page.update()`](https://flet.dev/docs/controls/page/#flet.Page.update) a task needs, since + auto-update only fires around event handlers and around `main`. +- **With no network it still says something true.** Each probe is wrapped on its own, so an + unreachable endpoint is one red row reading `DNSError: … curl: (6) Could not resolve host …` + and never a crash or a blank screen. `timeout=15` is passed explicitly, because curl-cffi's + own default is 30 s and a phone that just lost signal spends all of it spinning. + +The CA line is worth reading on both. curl-cffi resolves its bundle once, at +`import curl_cffi`: `SSL_CERT_FILE` / `CURL_CA_BUNDLE` / `REQUESTS_CA_BUNDLE` first, then +OpenSSL's compiled-in path, then [certifi](https://github.com/certifi/python-certifi). A +desktop `flet run` reaches the second and reports `openssl default path`; on device the first +already wins, because `flet build` generates startup code that exports `SSL_CERT_FILE` and +`REQUESTS_CA_BUNDLE` pointing at certifi's bundle. Either way the device trusts certifi's roots +and nothing else — but the step differs, which is why the app prints the winner rather than +asserting one. + +`[tool.flet.android] target_arch = ["arm64-v8a", "x86_64"]` is required: no `armeabi-v7a` +wheel exists, and without it the APK build fails resolving that ABI. + +## Try it + +[Build](https://flet.dev/docs/publish/) the app, then install it on a device or emulator/simulator: + +```bash +# Android +uv run flet build apk + +# iOS +uv run flet build ipa + +# iOS-Simulator +uv run flet build ios-simulator +``` + +The line to read first on device is `fingerprint tables 4/4`. It answers offline, and it is +the one thing no desktop run can establish: that the fingerprint tables inside the static +curl-impersonate archive survived the cross-compile and linked into the wheel for that +platform. See the [recipe README](../../README.md) for the rest. diff --git a/recipes/curl-cffi/examples/fingerprint-fanout/pyproject.toml b/recipes/curl-cffi/examples/fingerprint-fanout/pyproject.toml new file mode 100644 index 00000000..022e74fa --- /dev/null +++ b/recipes/curl-cffi/examples/fingerprint-fanout/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "curl-cffi-fingerprint-fanout" +version = "1.0.0" +description = "Probes one endpoint as five browsers at once and compares the fingerprints it reports." +requires-python = ">=3.10" + +dependencies = [ + "flet==0.86.5", + "curl-cffi==0.16.2", +] + +[dependency-groups] +dev = ["flet-cli", "flet-desktop", "flet-web"] + +[tool.flet.app] +path = "src" + +[tool.flet.android] +target_arch = ["arm64-v8a", "x86_64"] diff --git a/recipes/curl-cffi/examples/fingerprint-fanout/src/impersonation.py b/recipes/curl-cffi/examples/fingerprint-fanout/src/impersonation.py new file mode 100644 index 00000000..7b91abc9 --- /dev/null +++ b/recipes/curl-cffi/examples/fingerprint-fanout/src/impersonation.py @@ -0,0 +1,147 @@ +"""The curl-cffi half: what this build knows offline, and what a server sees.""" + +import asyncio +import os +import platform +import ssl +import time + +import certifi +import curl_cffi +from curl_cffi import Curl, CurlHttpVersion +from curl_cffi.curl import DEFAULT_CACERT +from curl_cffi.requests import AsyncSession + +# Answers with the JA3/JA4 hashes of the ClientHello it just received, the +# HTTP/2 fingerprint, and the User-Agent it was sent. Only the far end of a +# handshake can see any of that, which is why this is the one thing in the +# example that leaves the device. +PROBE_URL = "https://tls.browserleaks.com/json" + +# "off" is the sentinel for no impersonation. The other four name fingerprints +# compiled into the linked curl-impersonate build, so a bump can retire one. +TARGETS = ("chrome150", "chrome131_android", "safari260_ios", "firefox147", "off") + +# curl-cffi's own default is 30 s, which on a phone that just lost signal is a +# spinner nobody waits out. +TIMEOUT = 15 + +FIELDS = ("ja4", "ja3n", "ja3", "h2", "ua") + +VERSION = f"curl_cffi {curl_cffi.__version__} — Python {platform.python_version()}" +CURL_VERSION = curl_cffi.__curl_version__ + +# Response.http_version is a plain int; CurlHttpVersion is an IntEnum, so these +# keys match one. +HTTP_NAMES = { + CurlHttpVersion.V1_0: "http/1.0", + CurlHttpVersion.V1_1: "http/1.1", + CurlHttpVersion.V2_0: "h2", + CurlHttpVersion.V2TLS: "h2", + CurlHttpVersion.V3: "h3", +} + + +def known_targets(): + """Ask the linked library which targets it holds fingerprints for, offline. + + curl_easy_impersonate() returns 0 when a target's TLS and HTTP/2 tables are + compiled in and 43 when the name is unknown — it does not raise — so this + separates "this build does not know that target" from "the request failed" + without opening a socket, and goes red on its own if a bump retires one. + """ + verdicts = {} + for target in TARGETS: + if target == "off": + continue + curl = Curl() + try: + verdicts[target] = curl.impersonate(target) == 0 + finally: + curl.close() + return verdicts + + +def cacert(): + """Describe the CA bundle curl-cffi resolved at import, and which source won. + + Its _default_cacert() tries SSL_CERT_FILE / CURL_CA_BUNDLE / + REQUESTS_CA_BUNDLE, then OpenSSL's compiled-in path, then certifi — and the + winner differs between a desktop `flet run` and a phone, so it is read back + rather than assumed. On Android certifi's cacert.pem lives inside + sitepackages.zip and certifi.where() unpacks it to a temporary file, so the + path is a real one either way. + """ + source = "unknown" + for name in ("SSL_CERT_FILE", "CURL_CA_BUNDLE", "REQUESTS_CA_BUNDLE"): + if os.environ.get(name) == DEFAULT_CACERT: + source = f"${name}" + break + else: + if ssl.get_default_verify_paths().cafile == DEFAULT_CACERT: + source = "openssl default path" + elif certifi.where() == DEFAULT_CACERT: + source = f"certifi {certifi.__version__}" + try: + with open(DEFAULT_CACERT, "rb") as bundle: + pem = bundle.read() + except OSError as error: + return source, DEFAULT_CACERT, f"unreadable: {error.strerror}" + return ( + source, + DEFAULT_CACERT, + f"{len(pem):,} B · {pem.count(b'BEGIN CERTIFICATE')} certificates", + ) + + +async def probe(target): + """Handshake once as `target` and return what the far end reports. + + One AsyncSession per target rather than one shared one: a session pools + connections, and a pooled connection is a handshake that already happened, + so a later target could be answered through an earlier one's fingerprint. + `async with` matters — each session keeps a 0.1 s watchdog task alive until + it is closed. + """ + started = time.perf_counter() + async with AsyncSession( + impersonate=None if target == "off" else target, timeout=TIMEOUT + ) as session: + response = await session.get(PROBE_URL) + seen = response.json() + return { + "status": response.status_code, + "http": HTTP_NAMES.get(response.http_version, str(response.http_version)), + "ms": (time.perf_counter() - started) * 1e3, + # ja3 hashes the ClientHello as sent; ja3n sorts the extensions first. + # Chrome shuffles that order every handshake and curl-impersonate copies + # it, so ja3 moves between runs for a Chrome target while ja3n and ja4 + # (sorted by construction) hold still. Showing both is the point. + "ja4": seen["ja4"], + "ja3n": seen["ja3n_hash"], + "ja3": seen["ja3_hash"], + "h2": seen["akamai_hash"], + "ua": seen["user_agent"] or "(none sent)", + } + + +async def fanout(on_result): + """Probe every target at once, calling `on_result(target, reading, error)`. + + One task per target under a single gather, so the wall clock is the slowest + handshake rather than the sum of five — the shape a search app fanning out + to several endpoints needs. Each task catches its own failure, so one + unreachable target costs one row. Returns the wall clock in milliseconds, + for the caller to compare against the summed per-probe times. + """ + + async def one(target): + """Run a single probe and report it, however it ends.""" + try: + on_result(target, await probe(target), None) + except Exception as error: + on_result(target, None, f"{type(error).__name__}: {error}") + + started = time.perf_counter() + await asyncio.gather(*(one(target) for target in TARGETS)) + return (time.perf_counter() - started) * 1e3 diff --git a/recipes/curl-cffi/examples/fingerprint-fanout/src/main.py b/recipes/curl-cffi/examples/fingerprint-fanout/src/main.py new file mode 100644 index 00000000..4c481afd --- /dev/null +++ b/recipes/curl-cffi/examples/fingerprint-fanout/src/main.py @@ -0,0 +1,202 @@ +import flet as ft +from impersonation import ( + CURL_VERSION, + FIELDS, + PROBE_URL, + TARGETS, + VERSION, + cacert, + fanout, + known_targets, +) + +# "monospace" is a generic family name that Android maps and iOS does not, and a +# hash only reads as a hash in a fixed-width face; Courier backs it up there. +MONO = {"font_family": "monospace", "font_family_fallback": ["Courier"]} + + +def result_row(target): + """Build one target's row, and return it with a callback that refills it. + + The five rows are laid out up front in a fixed order rather than appended as + probes land, which a fan-out would order at random and make uncomparable. + The callback closes over the previous run's values so a hash that moved can + be tinted — which is how a second run shows that it is the Chrome targets' + raw ja3 that changes from one handshake to the next. + """ + status = ft.Text("…", size=11, color=ft.Colors.OUTLINE) + failed = ft.Text(size=11, color=ft.Colors.ERROR) + # expand sits on a Text inside a Row, never on a direct child of the + # scrolling Column: there it collapses the whole viewport on iOS. + values = { + name: ft.Text(size=10, expand=True, selectable=True, **MONO) for name in FIELDS + } + fields = ft.Column( + spacing=1, + controls=[ + ft.Row( + vertical_alignment=ft.CrossAxisAlignment.START, + controls=[ + ft.Text(name, size=10, width=36, color=ft.Colors.OUTLINE), + values[name], + ], + ) + for name in FIELDS + ], + ) + control = ft.Column( + spacing=1, + controls=[ + ft.Row( + controls=[ + ft.Text(target, size=13, weight=ft.FontWeight.BOLD, expand=True), + status, + ] + ), + failed, + fields, + ft.Divider(height=8), + ], + ) + previous = {} + + def refill(reading=None, error=None): + """Show this row's outcome, tinting whatever moved since the last run. + + Called with neither argument to blank the row while a probe is in flight. + """ + failed.value = error or "" + fields.visible = error is None + if error: + status.value = "failed" + status.color = ft.Colors.ERROR + return + status.color = ft.Colors.OUTLINE + if reading is None: + status.value = "…" + for text in values.values(): + text.value = "" + return + status.value = ( + f"{reading['status']} · {reading['http']} · {reading['ms']:.0f} ms" + ) + for name in FIELDS: + moved = name in previous and previous[name] != reading[name] + values[name].value = reading[name] + values[name].color = ft.Colors.PRIMARY if moved else None + previous[name] = reading[name] + + return control, refill + + +async def main(page: ft.Page): + """Probe one fingerprinting endpoint as five browsers at the same time.""" + + async def run(): + """Refill every row from one concurrent fan-out, then total the timings. + + Ends in an explicit page.update() because a task gets none of the + auto-update an event handler does, and the `finally` releases the button + even when every probe failed. + """ + for refill in rows.values(): + refill() + footer.value = f"probing {len(TARGETS)} targets…" + spinner.visible = True + page.update() + + summed = 0.0 + landed = 0 + + def show(target, reading, error): + """Fill one row the moment its probe lands, from the same loop.""" + nonlocal summed, landed + if reading: + landed += 1 + summed += reading["ms"] + rows[target](reading, error) + page.update() + + try: + wall = await fanout(show) + footer.value = f"{landed}/{len(TARGETS)} probes · {wall:.0f} ms wall" + if landed: + footer.value += ( + f" · {summed:.0f} ms summed ({summed / wall:.1f}x overlap)" + ) + finally: + button.disabled = False + spinner.visible = False + page.update() + + def rerun(): + """Start a fan-out. The guard is set here, not inside `run`. + + `page.run_task` only schedules, so a `disabled` set inside the coroutine + has not happened when this handler returns and Flet pushes the button's + state — a second tap in that window queues a second fan-out into the same + rows. `run_thread` is not an alternative: AsyncSession drives libcurl + through the running loop's readers and writers, and a worker thread has + no loop. + """ + if button.disabled: + return + button.disabled = True + page.run_task(run) + + source, path, contents = cacert() + verdicts = known_targets() + rows = {} + row_controls = [] + for target in TARGETS: + control, refill = result_row(target) + rows[target] = refill + row_controls.append(control) + + page.appbar = ft.AppBar(title=ft.Text("Fingerprint fan-out"), center_title=True) + page.add( + ft.SafeArea( + expand=True, + content=ft.Column( + scroll=ft.ScrollMode.AUTO, + controls=[ + ft.Text(f"{VERSION} on {page.platform.value}", size=12), + ft.Text(CURL_VERSION, size=10, **MONO), + ft.Text( + f"fingerprint tables {sum(verdicts.values())}/{len(verdicts)} · " + f"{', '.join(t for t, ok in verdicts.items() if ok)}", + size=11, + ), + ft.Text(f"CA bundle via {source} — {contents}", size=11), + ft.Text(path, size=10, selectable=True, **MONO), + ft.Divider(), + ft.Row( + controls=[ + button := ft.Button( + "Probe all targets", + icon=ft.Icons.FINGERPRINT, + on_click=rerun, + ), + spinner := ft.ProgressRing( + width=20, height=20, visible=False + ), + ] + ), + ft.Text( + f"as seen by {PROBE_URL}", size=11, color=ft.Colors.OUTLINE + ), + *row_controls, + footer := ft.Text(size=11), + ], + ), + ) + ) + + # Scheduled, not awaited: Flet awaits `main` before its first update, so + # awaiting a fan-out here would hold the first frame for up to the timeout on + # a phone with no network. Everything above the button is already on screen. + rerun() + + +if __name__ == "__main__": + ft.run(main) From b64e867fd1ec65d8dbf5e58a67eea28e557411e0 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 31 Aug 2026 15:05:20 +0200 Subject: [PATCH 07/18] skills: the licence gate's no-notice-at-all case [skip ci] The catalogue covered upstream renaming or relocating its notice, and the `[]` opt-out. It did not cover a prebuilt-repackage recipe whose upstream ships no notice anywhere, where `[]` is exactly wrong and the answer is to vendor the notices into the recipe dir. Adds that, the symbol-probe recipe for recovering a mega-archive's component list, and the two file-choice traps (a stub `LICENSE` redirect, and a dual-licensed project's `COPYING` being the arm we do not take). --- .../references/failure-catalogue.md | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index 9d820494..ccef3e33 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -2066,6 +2066,37 @@ about: Note an **unset** `license_file` still means "discover for me" — only an empty **list** is the opt-out. +**The prebuilt-repackage case: upstream ships no notice ANYWHERE.** A recipe that repackages +a third-party *binary* release often gets a tarball holding nothing but the library and its +headers — no `LICENSE`, no `COPYING`, nothing to point `license_file` at. `[]` is the wrong +reflex here: the wheel really does contain that object code, so the notice has to come from +somewhere, and the schema says so — *"or to the recipe directory, for a notice the upstream +archive doesn't ship"*. **Vendor the notices into the recipe directory** and list them: + +```yaml +about: + license_file: [COPYING.curl, LICENSE.boringssl, LICENSE.zstd] # flet-libcurl-impersonate + license: curl AND MIT AND Apache-2.0 AND BSD-3-Clause AND Zlib +``` + +Keep them **flat in the recipe dir**, not under a `licenses/` subdir — paths are preserved +into the wheel, so a subdir yields `dist-info/licenses/licenses/COPYING.curl`. + +Getting the component list right is the actual work, and a mega-archive will not tell you +directly: `ar t` on a `ld -r` blob lists one member. Recover it from the symbols the linker +left behind, then corroborate against upstream's build config: + +```bash +llvm-nm -a libfoo.a | awk '$2=="-"||$2=="f"{print $3}' | sort -u # STT_FILE syms per project +strings libfoo.a | grep -oE '/build/deps/src/[a-z0-9-]+' | sort | uniq -c +``` + +Two traps when picking each file, both the libiconv trap in different directions: a project's +`LICENSE` may be a stub redirect (nghttp2's is 12 bytes, "See COPYING"), and a dual-licensed +project's `COPYING` may be the arm we do *not* take (zstd's is the full GPL-2.0; the BSD arm +is in `LICENSE`). Fetch every file at the **pinned version tag**, never `main` — the texts +carry copyright year ranges that drift. + **Related:** `about.license_file` naming a file that isn't there raises too (a typo or a moved file, not a preference). Changing licence metadata does not reach pypi.flet.dev until the recipe's **build number is bumped** — see `forge-ci` § Deploying, "Bump before From 01112f553491071359b283567eea4bc3ebfb72ac Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 31 Aug 2026 15:10:48 +0200 Subject: [PATCH 08/18] docs: pin the example to 3.12, and fix what a failed probe renders [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `requires-python = ">=3.10"` was the wrong default here. `flet build` takes the HIGHEST stable Python the spec admits (`python_versions.py`: `max(stable_matching)`), so it landed on 3.14 — precisely the range where PyPI also carries a curl-cffi `android_24_arm64_v8a` wheel at the same version, and the example that exists to demonstrate this recipe's wheel might not have contained it on the flagship ABI. Upstream ships no cp312 mobile wheel at all, so `==3.12.*` is the lever that settles it. The README said to pin `curl-cffi` instead, two sentences after explaining that a version pin cannot separate two builds of one version; it now names the Python pin. A failed probe rendered 153 characters, 74 of them libcurl's docs URL, on five stacked rows. Trimmed at the URL and capped at two lines: 76 characters, still naming the host and the curl error number. `certifi` is imported directly for the CA-bundle readout, so it is declared rather than relied on transitively through curl-cffi. Docstrings run through docformatter to match the other examples, and the test's `import curl_cffi` is now asserted on rather than tripping F401 — importing it is the point of that test. --- recipes/curl-cffi/README.md | 12 ++++--- .../fingerprint-fanout/pyproject.toml | 3 +- .../fingerprint-fanout/src/impersonation.py | 32 ++++++++++--------- .../examples/fingerprint-fanout/src/main.py | 12 +++---- recipes/curl-cffi/tests/test_curl_cffi.py | 32 +++++++++++-------- 5 files changed, 52 insertions(+), 39 deletions(-) diff --git a/recipes/curl-cffi/README.md b/recipes/curl-cffi/README.md index 8ac8eb6a..cd1d3771 100644 --- a/recipes/curl-cffi/README.md +++ b/recipes/curl-cffi/README.md @@ -46,10 +46,14 @@ one mobile wheel per minor there — `cp313` and `cp314`, `android_24_arm64_v8a` version this index carries; no cp312, no x86_64, no armeabi-v7a, no iOS. One Android build can therefore carry two different builds of a single version across its ABIs, and a `==` pin cannot separate them because the versions agree. They are not the same payload — upstream's arm64-v8a -wheel is 8.2 MB compressed and 25.2 MB unpacked against 2.7 MB and 6.3 MB here — so pin -`curl-cffi` to the version in [`meta.yaml`](meta.yaml) if you want the wheel this page describes on -every slice, as the [`fingerprint-fanout`](examples/fingerprint-fanout) example's `pyproject.toml` -does. Which one an unpinned resolve prefers when the versions agree was not measured. +wheel is 8.2 MB compressed and 25.2 MB unpacked against 2.7 MB and 6.3 MB here — and which one an +unpinned resolve prefers was not measured. + +The lever is the **Python** version, not a `curl-cffi` pin: upstream publishes no cp312 mobile wheel +at all, so building on 3.12 is what settles it. Note `flet build` takes the **highest** stable Python +your `requires-python` admits, so the common `>=3.10` lands on 3.14 — exactly the range where the +two builds overlap. Say `requires-python = "==3.12.*"`, or pass `--python-version 3.12`, as the +[`fingerprint-fanout`](examples/fingerprint-fanout) example does. ## Examples diff --git a/recipes/curl-cffi/examples/fingerprint-fanout/pyproject.toml b/recipes/curl-cffi/examples/fingerprint-fanout/pyproject.toml index 022e74fa..51e43c6a 100644 --- a/recipes/curl-cffi/examples/fingerprint-fanout/pyproject.toml +++ b/recipes/curl-cffi/examples/fingerprint-fanout/pyproject.toml @@ -2,11 +2,12 @@ name = "curl-cffi-fingerprint-fanout" version = "1.0.0" description = "Probes one endpoint as five browsers at once and compares the fingerprints it reports." -requires-python = ">=3.10" +requires-python = "==3.12.*" dependencies = [ "flet==0.86.5", "curl-cffi==0.16.2", + "certifi", ] [dependency-groups] diff --git a/recipes/curl-cffi/examples/fingerprint-fanout/src/impersonation.py b/recipes/curl-cffi/examples/fingerprint-fanout/src/impersonation.py index 7b91abc9..b671525d 100644 --- a/recipes/curl-cffi/examples/fingerprint-fanout/src/impersonation.py +++ b/recipes/curl-cffi/examples/fingerprint-fanout/src/impersonation.py @@ -45,10 +45,10 @@ def known_targets(): """Ask the linked library which targets it holds fingerprints for, offline. - curl_easy_impersonate() returns 0 when a target's TLS and HTTP/2 tables are - compiled in and 43 when the name is unknown — it does not raise — so this - separates "this build does not know that target" from "the request failed" - without opening a socket, and goes red on its own if a bump retires one. + curl_easy_impersonate() returns 0 when a target's TLS and HTTP/2 tables are compiled + in and 43 when the name is unknown — it does not raise — so this separates "this + build does not know that target" from "the request failed" without opening a socket, + and goes red on its own if a bump retires one. """ verdicts = {} for target in TARGETS: @@ -65,12 +65,11 @@ def known_targets(): def cacert(): """Describe the CA bundle curl-cffi resolved at import, and which source won. - Its _default_cacert() tries SSL_CERT_FILE / CURL_CA_BUNDLE / - REQUESTS_CA_BUNDLE, then OpenSSL's compiled-in path, then certifi — and the - winner differs between a desktop `flet run` and a phone, so it is read back - rather than assumed. On Android certifi's cacert.pem lives inside - sitepackages.zip and certifi.where() unpacks it to a temporary file, so the - path is a real one either way. + Its _default_cacert() tries SSL_CERT_FILE / CURL_CA_BUNDLE / REQUESTS_CA_BUNDLE, + then OpenSSL's compiled-in path, then certifi — and the winner differs between a + desktop `flet run` and a phone, so it is read back rather than assumed. On Android + certifi's cacert.pem lives inside sitepackages.zip and certifi.where() unpacks it to + a temporary file, so the path is a real one either way. """ source = "unknown" for name in ("SSL_CERT_FILE", "CURL_CA_BUNDLE", "REQUESTS_CA_BUNDLE"): @@ -129,10 +128,10 @@ async def fanout(on_result): """Probe every target at once, calling `on_result(target, reading, error)`. One task per target under a single gather, so the wall clock is the slowest - handshake rather than the sum of five — the shape a search app fanning out - to several endpoints needs. Each task catches its own failure, so one - unreachable target costs one row. Returns the wall clock in milliseconds, - for the caller to compare against the summed per-probe times. + handshake rather than the sum of five — the shape a search app fanning out to + several endpoints needs. Each task catches its own failure, so one unreachable + target costs one row. Returns the wall clock in milliseconds, for the caller to + compare against the summed per-probe times. """ async def one(target): @@ -140,7 +139,10 @@ async def one(target): try: on_result(target, await probe(target), None) except Exception as error: - on_result(target, None, f"{type(error).__name__}: {error}") + # libcurl tacks 74 characters of docs URL onto every message, which + # on five stacked rows is most of the screen. + detail = str(error).split(" See https://curl.se/", 1)[0] + on_result(target, None, f"{type(error).__name__}: {detail}") started = time.perf_counter() await asyncio.gather(*(one(target) for target in TARGETS)) diff --git a/recipes/curl-cffi/examples/fingerprint-fanout/src/main.py b/recipes/curl-cffi/examples/fingerprint-fanout/src/main.py index 4c481afd..fa083828 100644 --- a/recipes/curl-cffi/examples/fingerprint-fanout/src/main.py +++ b/recipes/curl-cffi/examples/fingerprint-fanout/src/main.py @@ -18,14 +18,14 @@ def result_row(target): """Build one target's row, and return it with a callback that refills it. - The five rows are laid out up front in a fixed order rather than appended as - probes land, which a fan-out would order at random and make uncomparable. - The callback closes over the previous run's values so a hash that moved can - be tinted — which is how a second run shows that it is the Chrome targets' - raw ja3 that changes from one handshake to the next. + The five rows are laid out up front in a fixed order rather than appended as probes + land, which a fan-out would order at random and make uncomparable. The callback + closes over the previous run's values so a hash that moved can be tinted — which is + how a second run shows that it is the Chrome targets' raw ja3 that changes from one + handshake to the next. """ status = ft.Text("…", size=11, color=ft.Colors.OUTLINE) - failed = ft.Text(size=11, color=ft.Colors.ERROR) + failed = ft.Text(size=11, color=ft.Colors.ERROR, max_lines=2) # expand sits on a Text inside a Row, never on a direct child of the # scrolling Column: there it collapses the whole viewport on iOS. values = { diff --git a/recipes/curl-cffi/tests/test_curl_cffi.py b/recipes/curl-cffi/tests/test_curl_cffi.py index 7731d206..7c21931e 100644 --- a/recipes/curl-cffi/tests/test_curl_cffi.py +++ b/recipes/curl-cffi/tests/test_curl_cffi.py @@ -1,23 +1,26 @@ -"""On-device smoke tests for curl_cffi — the cffi bindings around -curl-impersonate (a patched libcurl + BoringSSL + nghttp2/3 + ngtcp2 + brotli + -zstd + zlib, all statically linked into the `_wrapper` extension). These are -network-free: they exercise the compiled native library directly, without making -any HTTP request (an emulator/simulator has no guaranteed connectivity).""" +"""On-device smoke tests for curl_cffi — the cffi bindings around curl-impersonate (a +patched libcurl + BoringSSL + nghttp2/3 + ngtcp2 + brotli + zstd + zlib, all statically +linked into the `_wrapper` extension). + +They are network-free, exercising the compiled native library directly rather than +making an HTTP request, because an emulator or simulator has no guaranteed connectivity. +""" def test_import_loads_native_wrapper(): - """Importing curl_cffi loads its compiled cffi extension (_wrapper) — proves - the mega-archive linked and _cffi_backend / libc++_shared resolve on load.""" + """Importing curl_cffi loads its compiled cffi extension (_wrapper) — proves the + mega-archive linked and _cffi_backend / libc++_shared resolve on load.""" import curl_cffi from curl_cffi import _wrapper + assert curl_cffi.__file__ assert _wrapper.lib is not None assert _wrapper.ffi is not None def test_linked_libcurl_is_impersonate_build(): - """__curl_version__ is a real lib.curl_version() call into the statically - linked library; confirm it is the curl-impersonate build, not system curl.""" + """__curl_version__ is a real lib.curl_version() call into the statically linked + library; confirm it is the curl-impersonate build, not system curl.""" import curl_cffi version = curl_cffi.__curl_version__ @@ -39,10 +42,13 @@ def test_curl_easy_impersonate_applies_fingerprint(): def test_default_cacert_resolves_to_a_readable_file(): - """curl_cffi picks its CA bundle once, at import, and hands the path straight - to libcurl. Flet extracts certifi's bundle out of sitepackages.zip and points - SSL_CERT_FILE at it, so the path lands on disk — if it ever didn't, every - HTTPS request would fail with CURLE_SSL_CACERT_BADFILE instead.""" + """curl_cffi picks its CA bundle once, at import, and hands the path straight to + libcurl. + + Flet extracts certifi's bundle out of sitepackages.zip and points SSL_CERT_FILE at + it, so the path lands on disk — if it ever didn't, every HTTPS request would fail + with CURLE_SSL_CACERT_BADFILE instead. + """ import os from curl_cffi.curl import DEFAULT_CACERT From 38e409350c3b24e07766331b64672c6031c9407c Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 31 Aug 2026 15:12:38 +0200 Subject: [PATCH 09/18] skills: what `build.number: 0` actually costs, and the certifi non-gotcha [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recipe-patterns already said not to write `number: 0`. It did not say that the lost build tag forfeits a PEP 427 tie-break against upstream's own mobile wheels on an extra index, or — the part that makes it expensive — that CI cannot catch it, because the mobile test rewrites local build tags to 9999 before installing. Only the filename in dist/ tells you. Also records that a version bump resets the number to 1. Corrects the curl-cffi shape's on-device count, and notes that a real impersonated HTTPS request was verified on both platforms — that one is worth having written down, because the intuition that Flet 0.86's sitepackages.zip breaks certifi is wrong and cost this recipe a wrong README draft before a device run settled it. --- .claude/skills/new-mobile-recipe/SKILL.md | 2 +- .../skills/new-mobile-recipe/references/recipe-patterns.md | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.claude/skills/new-mobile-recipe/SKILL.md b/.claude/skills/new-mobile-recipe/SKILL.md index d8eb5479..c5910291 100644 --- a/.claude/skills/new-mobile-recipe/SKILL.md +++ b/.claude/skills/new-mobile-recipe/SKILL.md @@ -122,7 +122,7 @@ The shape that works — a `flet-lib*` prebuilt-repackage dep + a small opt-in p 2. **The consumer declares it `requirements.host_build`** and points the package's own "where's the lib" env var at it (`IMPERSONATE_BUILD_DIR: '{platlib}/opt'`) so the package's **download is skipped** (the `.a` already exists there) — fully offline, and no CI `tmplibdir`-reuse arch leak. 3. **A minimal `mobile.patch` adds ONE opt-in lever** (an env var the recipe's `script_env` sets, e.g. `IMPERSONATE_FORGE_TARGET: ios|android`) that (a) makes the arch-detect return a synthesized target entry instead of scanning `uname`, and (b) forces the target's static-link recipe (Apple `-force_load`+`-lc++` for iOS, ELF `--whole-archive`+`-lc++` for Android — the Android branch is the one the macOS host gets wrong). Guard it so upstream behaviour is unchanged when the var is unset. **iOS extra:** a static archive built with Apple SecTrust / Apple IDN needs the consuming extension to link `-framework Security -framework CoreFoundation -liconv -licucore` (see `forge-error-catalogue` § iOS undefined `_SecTrust*`/`_iconv`/`_uidna_*`). -**Real example:** branch `curl-cffi` — `recipes/flet-libcurl-impersonate/` (build.sh, `source.strip: 0`) + `recipes/curl-cffi/` (`patches/mobile.patch`, 3 hunks). World-first curl-cffi iOS wheels; on-device 3/3 both platforms. Needed one forge-core change: exposing `source.strip` in `src/forge/schema/meta-schema.yaml` (the code already honored it). +**Real example:** branch `curl-cffi` — `recipes/flet-libcurl-impersonate/` (build.sh, `source.strip: 0`) + `recipes/curl-cffi/` (`patches/mobile.patch`, 3 hunks). World-first curl-cffi iOS wheels; on-device 4/4 both platforms, and a real impersonated HTTPS request returns 200 on each. Needed one forge-core change: exposing `source.strip` in `src/forge/schema/meta-schema.yaml` (the code already honored it). ### Naming diff --git a/.claude/skills/new-mobile-recipe/references/recipe-patterns.md b/.claude/skills/new-mobile-recipe/references/recipe-patterns.md index b4ea3c18..55044afa 100644 --- a/.claude/skills/new-mobile-recipe/references/recipe-patterns.md +++ b/.claude/skills/new-mobile-recipe/references/recipe-patterns.md @@ -689,7 +689,9 @@ So: Integer in `build:`, schema default **1** (`src/forge/schema/meta-schema.yaml`) — and `1` is the repo convention (start every new recipe there). Bump when the recipe itself changes but the upstream version doesn't — pip prefers higher build numbers for the same version, and the new build gets a distinct filename (`--N-.whl`). Important for forcing redeploys when patching. -Don't write `number: 0`: forge only passes `--build-number` when the value is truthy (`build.py`), so `0` produces a wheel with NO build tag — losing the ability to supersede it later without a version bump. +Don't write `number: 0`: forge only passes `--build-number` when the value is truthy (`build.py`), so `0` produces a wheel with NO build tag — losing the ability to supersede it later without a version bump. It also forfeits a **PEP 427 tie-break you may be relying on**: pypi.flet.dev is an *extra* index, not a replacement, so where upstream publishes its own mobile wheel at the same version (curl-cffi ships `cp313`/`cp314` `android_24_arm64_v8a` on PyPI), an untagged forge wheel is competing with one hand tied. **CI cannot catch any of this** — the mobile test rewrites local wheels' build tag to `9999` before installing, so a `number: 0` recipe still goes green. Only the filename in `dist/` tells you. + +On an upstream **version** bump, reset `number` to 1 rather than carrying the old value forward. ### Source layout From c2be3edcb495c712614756a954078fc04df496b3 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 31 Aug 2026 15:24:24 +0200 Subject: [PATCH 10/18] docs: curl-cffi coverage gaps now that CI is green [skip ci] Run 33395094861: 10/10 jobs, 0.16.2 across 3.12/3.13/3.14 on all five slices, on-device 4/4 EXIT 0 on both the CI emulator and simulator. --- recipes/curl-cffi/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/curl-cffi/README.md b/recipes/curl-cffi/README.md index cd1d3771..f521f245 100644 --- a/recipes/curl-cffi/README.md +++ b/recipes/curl-cffi/README.md @@ -350,5 +350,5 @@ and that the CA bundle path is readable. Nothing exercises `Session` or `AsyncSe handshake, cookies, proxies, HTTP/2 or HTTP/3, and nothing shows a fingerprint being accepted by a real server. An import-and-construct suite passes even where the first request would fail; the [`fingerprint-fanout`](examples/fingerprint-fanout) example is the thing to build and install to -close that gap. The current versions have also not been through a CI run — the established green -result is for the previous curl-cffi and curl-impersonate pair. +close that gap. What the green CI run does cover is every Python: these versions built and passed +on 3.12, 3.13 and 3.14 across all five slices, with the on-device suite run on 3.12. From 9c804af062d2c4c30fd2dada408e876ea77dc3b8 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 31 Aug 2026 15:54:15 +0200 Subject: [PATCH 11/18] forge: only warn about `about` when a recipe actually declares one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The warning that `about` is ignored on non-build.sh recipes fired on almost every Python recipe in the tree — gdal, pyproj, pyogrio, rasterio and the rest — none of which declare it. Schema validation fills in defaults in place, and `about`'s children default to empty strings rather than to nothing, so by the time `fix_wheel` looked, every recipe had `about = {"license_file": "", "license": ""}` and the truthiness test could never be False. Net effect was that a warning written to catch one specific mistake became noise on every build, which is the reliable way to make the real case invisible. Records whether the recipe declared the key before defaults are merged, and tests that. Testing the values instead would have been close but not right: it would miss `about.license_file: []`, the deliberate opt-out, which is a thing an author wrote and which does nothing on this path. Verified by building lru-dict (a Python recipe) both ways — no warning as-is, warning present with an `about:` block appended — and flet-libomp, where the build.sh path still emits `License-Expression: Apache-2.0 WITH LLVM-exception` and its LICENSE file. --- src/forge/build.py | 4 +++- src/forge/package.py | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/forge/build.py b/src/forge/build.py index 6e6fe774..248c8450 100644 --- a/src/forge/build.py +++ b/src/forge/build.py @@ -905,7 +905,9 @@ def fix_wheel(self, wheel_dir: Path): # `about` drives metadata that only the build.sh path synthesises. A Python # package's own build backend already carries upstream's licence through, so # setting it there would be a no-op — say so rather than ignoring it silently. - if (self.package.meta.get("about") or {}) and not self.synthesizes_metadata: + # Test what the recipe DECLARED: schema defaults leave every recipe with a + # populated `about` dict, so testing the dict warned on all of them. + if self.package.declares_about and not self.synthesizes_metadata: log( self.log_file, f"[{self.cross_venv}] WARNING: `about` is only honoured for build.sh " diff --git a/src/forge/package.py b/src/forge/package.py index c3200111..8344f16e 100644 --- a/src/forge/package.py +++ b/src/forge/package.py @@ -102,6 +102,12 @@ def set_defaults(validator, properties, instance, schema): except KeyError: pass + # Whether the recipe actually wrote an `about:` block. Validation fills in + # schema defaults in place, and `about`'s are empty strings rather than + # nothing — so afterwards every recipe has a non-empty `about` dict and + # "did the author ask for this?" is no longer answerable from the metadata. + self.declares_about = "about" in meta + # Validate the metadata against the schema. with_defaults(Validator)(schema).validate(meta) From 68de4241c0051f849eb74374c5c7bb428e0b69aa Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 31 Aug 2026 16:16:30 +0200 Subject: [PATCH 12/18] improve --- recipes/curl-cffi/examples/fingerprint-fanout/src/main.py | 4 +--- recipes/flet-libcurl-impersonate/meta.yaml | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/recipes/curl-cffi/examples/fingerprint-fanout/src/main.py b/recipes/curl-cffi/examples/fingerprint-fanout/src/main.py index fa083828..05c9d0bf 100644 --- a/recipes/curl-cffi/examples/fingerprint-fanout/src/main.py +++ b/recipes/curl-cffi/examples/fingerprint-fanout/src/main.py @@ -90,8 +90,6 @@ def refill(reading=None, error=None): async def main(page: ft.Page): - """Probe one fingerprinting endpoint as five browsers at the same time.""" - async def run(): """Refill every row from one concurrent fan-out, then total the timings. @@ -153,7 +151,7 @@ def rerun(): rows[target] = refill row_controls.append(control) - page.appbar = ft.AppBar(title=ft.Text("Fingerprint fan-out"), center_title=True) + page.appbar = ft.AppBar(title="Fingerprint fan-out", center_title=True) page.add( ft.SafeArea( expand=True, diff --git a/recipes/flet-libcurl-impersonate/meta.yaml b/recipes/flet-libcurl-impersonate/meta.yaml index 264fcaa7..41759d38 100644 --- a/recipes/flet-libcurl-impersonate/meta.yaml +++ b/recipes/flet-libcurl-impersonate/meta.yaml @@ -13,6 +13,9 @@ package: excluded_arches: - armeabi-v7a +build: + number: 1 + source: # {% if sdk == 'iphoneos' %} url: https://github.com/lexiforest/curl-impersonate/releases/download/v{{ version }}/libcurl-impersonate-v{{ version }}.arm64-apple-ios.tar.gz @@ -27,8 +30,6 @@ source: # nonexistent top-level component. strip: 0 -build: - number: 1 about: # The release tarballs ship no licence text at all, so the notices are vendored From 00f9d29d0253b5b276c2c24becbd20d35f667d8a Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 31 Aug 2026 16:25:30 +0200 Subject: [PATCH 13/18] forge: collect a recipe's licenses/ folder, and move the vendored notices into one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nine notices sitting in the recipe root drowned out meta.yaml and build.sh, and the meta.yaml list naming them was a second copy of the folder's contents that a bump could put out of step. A folder settles both, alongside tests/ and examples/: everything in recipes//licenses/ ships, whatever each file is named, so the folder IS the list. The names are deliberately not licence-shaped — COPYING.curl, LICENSE.boringssl — because a recipe supplying notices for a prebuilt binary chooses them per bundled project, so the LICEN[CS]E/COPYING/COPYRIGHT/NOTICE match that guards top-level discovery would be the wrong test inside a folder that exists only to hold notices. The folder name is stripped from the destination, so a notice lands at dist-info/licenses/ rather than the doubled dist-info/licenses/licenses/ that preserving the relative path would give — which is what pushed these files into the recipe root in the first place. An explicit about.license_file naming a licenses/ path is stripped the same way, so both routes agree; a path into the SOURCE tree keeps its relative path, as flet-libjq's modules/oniguruma/COPYING needs. Verified: flet-libcurl-impersonate ships all nine at dist-info/licenses/ with nine License-File headers and no doubled paths, and flet-libomp's top-level recipe LICENSE still comes through. Every branch exercised against scratch trees — folder alone, folder plus an upstream LICENSE, a nested file, upstream-only, nothing at all (raises), explicit naming a licenses/ path, explicit source subpath, the [] opt-out, and an explicit missing file. --- .../references/failure-catalogue.md | 16 +++++-- .claude/skills/new-mobile-recipe/SKILL.md | 9 +++- README.rst | 6 +++ recipes/curl-cffi/meta.yaml | 7 ++- .../{ => licenses}/COPYING.curl | 0 .../{ => licenses}/COPYING.nghttp2 | 0 .../{ => licenses}/COPYING.nghttp3 | 0 .../{ => licenses}/COPYING.ngtcp2 | 0 .../{ => licenses}/LICENSE.boringssl | 0 .../{ => licenses}/LICENSE.brotli | 0 .../{ => licenses}/LICENSE.curl-impersonate | 0 .../{ => licenses}/LICENSE.zlib | 0 .../{ => licenses}/LICENSE.zstd | 0 recipes/flet-libcurl-impersonate/meta.yaml | 20 +++----- src/forge/build.py | 47 ++++++++++++++++--- src/forge/schema/meta-schema.yaml | 12 +++-- 16 files changed, 83 insertions(+), 34 deletions(-) rename recipes/flet-libcurl-impersonate/{ => licenses}/COPYING.curl (100%) rename recipes/flet-libcurl-impersonate/{ => licenses}/COPYING.nghttp2 (100%) rename recipes/flet-libcurl-impersonate/{ => licenses}/COPYING.nghttp3 (100%) rename recipes/flet-libcurl-impersonate/{ => licenses}/COPYING.ngtcp2 (100%) rename recipes/flet-libcurl-impersonate/{ => licenses}/LICENSE.boringssl (100%) rename recipes/flet-libcurl-impersonate/{ => licenses}/LICENSE.brotli (100%) rename recipes/flet-libcurl-impersonate/{ => licenses}/LICENSE.curl-impersonate (100%) rename recipes/flet-libcurl-impersonate/{ => licenses}/LICENSE.zlib (100%) rename recipes/flet-libcurl-impersonate/{ => licenses}/LICENSE.zstd (100%) diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index ccef3e33..bac64d46 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -2071,16 +2071,22 @@ a third-party *binary* release often gets a tarball holding nothing but the libr headers — no `LICENSE`, no `COPYING`, nothing to point `license_file` at. `[]` is the wrong reflex here: the wheel really does contain that object code, so the notice has to come from somewhere, and the schema says so — *"or to the recipe directory, for a notice the upstream -archive doesn't ship"*. **Vendor the notices into the recipe directory** and list them: +archive doesn't ship"*. **Vendor the notices into `recipes//licenses/`** — a folder +alongside `tests/` and `examples/`, whose entire contents ship whatever each file is named: + +``` +recipes/flet-libcurl-impersonate/ + licenses/COPYING.curl licenses/LICENSE.boringssl licenses/LICENSE.zstd ... +``` ```yaml about: - license_file: [COPYING.curl, LICENSE.boringssl, LICENSE.zstd] # flet-libcurl-impersonate - license: curl AND MIT AND Apache-2.0 AND BSD-3-Clause AND Zlib + license: curl AND MIT AND Apache-2.0 AND BSD-3-Clause AND Zlib # no license_file needed ``` -Keep them **flat in the recipe dir**, not under a `licenses/` subdir — paths are preserved -into the wheel, so a subdir yields `dist-info/licenses/licenses/COPYING.curl`. +No `license_file` list: the folder is the list, so it cannot fall out of step on a bump. +The folder name is stripped from the destination, so notices land at +`dist-info/licenses/`. Getting the component list right is the actual work, and a mega-archive will not tell you directly: `ar t` on a `ld -r` blob lists one member. Recover it from the symbols the linker diff --git a/.claude/skills/new-mobile-recipe/SKILL.md b/.claude/skills/new-mobile-recipe/SKILL.md index c5910291..c0a84fd5 100644 --- a/.claude/skills/new-mobile-recipe/SKILL.md +++ b/.claude/skills/new-mobile-recipe/SKILL.md @@ -285,7 +285,14 @@ accompany the binary, forge now bundles one automatically: any top-level copied into `.dist-info/licenses/` and listed as `License-File`. That covers almost every upstream unchanged — so usually you write nothing. -Three cases need a line in `meta.yaml`: +**When upstream ships no notice at all** — the usual case for a prebuilt-binary +repackage — put one file per bundled project in `recipes//licenses/`. Everything in +that folder ships, under any name, and the folder name is stripped from the destination. +Prefer it over a `license_file` list and over loose files in the recipe root: the folder +IS the list, so it cannot go stale on a bump, and the recipe root stays readable. +Precedent: `recipes/flet-libcurl-impersonate/licenses/` (nine notices, no list). + +Three cases still need a line in `meta.yaml`: - **The notice is not at the top level, or is named something else** — point at it: ```yaml diff --git a/README.rst b/README.rst index d8b09e05..2d4345ba 100644 --- a/README.rst +++ b/README.rst @@ -129,6 +129,12 @@ Inside the recipe directory, add the following files. pytest suite which imports the package and does some basic checks. * Optionally, one or more patch files in a folder named ``patches``. These patches will be applied when the source code is unpacked for a given platform. +* Optionally, a folder named ``licenses``. Every file in it is copied into the wheel's + ``.dist-info/licenses/`` under its own name, and the folder name is not part of that + destination. Use it when the upstream archive ships no licence text of its own — a + prebuilt binary release, usually — and the recipe has to supply one notice per bundled + project. A licence-shaped file at the top level of the source or recipe directory is + picked up without this, so most recipes need neither. * For non-Python packages, a ``build.sh`` script. This is the script that will be executed in the build environment build the package. This script should invoke any ``configure``, ``make``, or any other compilation steps needed to build the package. This script will be diff --git a/recipes/curl-cffi/meta.yaml b/recipes/curl-cffi/meta.yaml index ec84aff3..549d948f 100644 --- a/recipes/curl-cffi/meta.yaml +++ b/recipes/curl-cffi/meta.yaml @@ -13,10 +13,9 @@ build: script_env: # flet-libcurl-impersonate stages the prebuilt static mega-archive and # include/curl/ into {platlib}/opt. Point curl-cffi's ffibuilder there so it - # links that libcurl-impersonate.a and skips the GitHub download entirely - # (offline, deterministic build). IMPERSONATE_FORGE_TARGET is consumed by the - # mobile.patch to pick the target arch + link recipe instead of the build - # host's uname. + # links that libcurl-impersonate.a and skips the GitHub download entirely (offline, + # deterministic build). IMPERSONATE_FORGE_TARGET is consumed by the mobile.patch + # to pick the target arch + link recipe instead of the build host's uname. IMPERSONATE_BUILD_DIR: '{platlib}/opt' IMPERSONATE_LINK_TYPE: static # {% if sdk == 'android' %} diff --git a/recipes/flet-libcurl-impersonate/COPYING.curl b/recipes/flet-libcurl-impersonate/licenses/COPYING.curl similarity index 100% rename from recipes/flet-libcurl-impersonate/COPYING.curl rename to recipes/flet-libcurl-impersonate/licenses/COPYING.curl diff --git a/recipes/flet-libcurl-impersonate/COPYING.nghttp2 b/recipes/flet-libcurl-impersonate/licenses/COPYING.nghttp2 similarity index 100% rename from recipes/flet-libcurl-impersonate/COPYING.nghttp2 rename to recipes/flet-libcurl-impersonate/licenses/COPYING.nghttp2 diff --git a/recipes/flet-libcurl-impersonate/COPYING.nghttp3 b/recipes/flet-libcurl-impersonate/licenses/COPYING.nghttp3 similarity index 100% rename from recipes/flet-libcurl-impersonate/COPYING.nghttp3 rename to recipes/flet-libcurl-impersonate/licenses/COPYING.nghttp3 diff --git a/recipes/flet-libcurl-impersonate/COPYING.ngtcp2 b/recipes/flet-libcurl-impersonate/licenses/COPYING.ngtcp2 similarity index 100% rename from recipes/flet-libcurl-impersonate/COPYING.ngtcp2 rename to recipes/flet-libcurl-impersonate/licenses/COPYING.ngtcp2 diff --git a/recipes/flet-libcurl-impersonate/LICENSE.boringssl b/recipes/flet-libcurl-impersonate/licenses/LICENSE.boringssl similarity index 100% rename from recipes/flet-libcurl-impersonate/LICENSE.boringssl rename to recipes/flet-libcurl-impersonate/licenses/LICENSE.boringssl diff --git a/recipes/flet-libcurl-impersonate/LICENSE.brotli b/recipes/flet-libcurl-impersonate/licenses/LICENSE.brotli similarity index 100% rename from recipes/flet-libcurl-impersonate/LICENSE.brotli rename to recipes/flet-libcurl-impersonate/licenses/LICENSE.brotli diff --git a/recipes/flet-libcurl-impersonate/LICENSE.curl-impersonate b/recipes/flet-libcurl-impersonate/licenses/LICENSE.curl-impersonate similarity index 100% rename from recipes/flet-libcurl-impersonate/LICENSE.curl-impersonate rename to recipes/flet-libcurl-impersonate/licenses/LICENSE.curl-impersonate diff --git a/recipes/flet-libcurl-impersonate/LICENSE.zlib b/recipes/flet-libcurl-impersonate/licenses/LICENSE.zlib similarity index 100% rename from recipes/flet-libcurl-impersonate/LICENSE.zlib rename to recipes/flet-libcurl-impersonate/licenses/LICENSE.zlib diff --git a/recipes/flet-libcurl-impersonate/LICENSE.zstd b/recipes/flet-libcurl-impersonate/licenses/LICENSE.zstd similarity index 100% rename from recipes/flet-libcurl-impersonate/LICENSE.zstd rename to recipes/flet-libcurl-impersonate/licenses/LICENSE.zstd diff --git a/recipes/flet-libcurl-impersonate/meta.yaml b/recipes/flet-libcurl-impersonate/meta.yaml index 41759d38..db1012dc 100644 --- a/recipes/flet-libcurl-impersonate/meta.yaml +++ b/recipes/flet-libcurl-impersonate/meta.yaml @@ -32,10 +32,11 @@ source: about: - # The release tarballs ship no licence text at all, so the notices are vendored - # here. libcurl-impersonate.a is a single `ld -r` blob merging nine upstream - # projects, pinned by upstream's CMakeLists.txt at this tag and confirmed in the - # archive by symbol probe: + # The release tarballs ship no licence text at all, so the notices are vendored in + # licenses/ — every file there ships, so there is no list here to fall out of step + # with the folder. libcurl-impersonate.a is a single `ld -r` blob merging nine + # upstream projects, pinned by upstream's CMakeLists.txt at this tag and confirmed + # in the archive by symbol probe: # # curl 8.21.0 curl COPYING.curl # curl-impersonate patches MIT LICENSE.curl-impersonate @@ -53,16 +54,7 @@ about: # (its COPYING is the GPL-2.0 arm of a dual licence; we take the BSD arm). # # Same set on both platforms — one superbuild, only BoringSSL's asm differs. - license_file: - - COPYING.curl - - LICENSE.curl-impersonate - - LICENSE.boringssl - - COPYING.nghttp2 - - COPYING.nghttp3 - - COPYING.ngtcp2 - - LICENSE.brotli - - LICENSE.zstd - - LICENSE.zlib + # # Deduplicated set of the nine, ANDed because the archive holds all of them at # once. zstd's GPL-2.0 alternative is deliberately not expressed: it is a fact # about the project, not about this artifact. diff --git a/src/forge/build.py b/src/forge/build.py index 248c8450..681a10e5 100644 --- a/src/forge/build.py +++ b/src/forge/build.py @@ -41,6 +41,12 @@ # ASF-sourced archive (arrow) keeps its attributions there rather than in LICENSE. LICENSE_FILE_RE = re.compile(r"^(licen[cs]e|copying|copyright|notice)", re.IGNORECASE) +# A recipe subdirectory whose entire contents are licence notices, alongside tests/ +# and examples/. Everything in it ships, whatever it is named — a recipe supplying +# notices for a binary that carries none usually has one per bundled project, and +# those names (COPYING.curl, LICENSE.boringssl) are ours to choose. +LICENSES_DIR_NAME = "licenses" + class Builder(ABC): # Whether this builder writes the wheel's METADATA itself (and so is the one that @@ -1151,6 +1157,14 @@ def collect_license_files(self) -> list[tuple[Path, str]]: archive that ships none; a source file shadows a recipe file of the same name, which is how a recipe can carry a fallback without overriding upstream. + Everything under the recipe's `licenses/` directory is collected as well, + whatever each file is called. A recipe repackaging a prebuilt binary that + ships no notice at all has to supply one per bundled project, and that is a + set of files rather than one — a folder keeps them out of the recipe root + (next to tests/ and examples/) and out of a meta.yaml list that would go + stale on the next bump. The folder name is not part of the destination, so a + notice lands at `licenses/` in the wheel rather than doubled. + `about.license_file` (a path, or a list of them) replaces that discovery entirely, for the two cases it cannot get right on its own: excluding a notice that covers something the wheel does not contain, and including one that lives @@ -1194,7 +1208,15 @@ def collect_license_files(self) -> list[tuple[Path, str]]: for directory in search_dirs: candidate = directory / name if candidate.is_file(): - resolved.append((candidate, name)) + # Naming a file inside the recipe's licenses/ explicitly must + # land it where the folder convention would, not at + # licenses/licenses/. + dest = name + if directory == self.package.recipe_path: + parts = Path(name).parts + if parts[:1] == (LICENSES_DIR_NAME,): + dest = str(Path(*parts[1:])) + resolved.append((candidate, dest)) break else: raise RuntimeError( @@ -1214,20 +1236,31 @@ def collect_license_files(self) -> list[tuple[Path, str]]: if candidate.is_file() and LICENSE_FILE_RE.match(candidate.name): found.setdefault(candidate.name, candidate) + # Then everything the recipe vendored, under any name. + licenses_dir = self.package.recipe_path / LICENSES_DIR_NAME + if licenses_dir.is_dir(): + for candidate in sorted(licenses_dir.rglob("*")): + if candidate.is_file(): + rel = str(candidate.relative_to(licenses_dir)) + found.setdefault(rel, candidate) + if not found: raise RuntimeError( f"{self.package.name}: no licence file found, so this wheel would ship " f"the library's object code with no notice.\n" - f" searched (top level only):\n" + f" searched, for names starting with LICENSE / LICENCE / COPYING / " + f"COPYRIGHT / NOTICE (any case, top level only):\n" f" {self.build_path}\n" f" {self.package.recipe_path}\n" - f" for names starting with: LICENSE / LICENCE / COPYING / COPYRIGHT / " - f"NOTICE (any case)\n" - f" Fix by pointing at the real file, which is what an upstream that " - f"moved or renamed its notice needs:\n" + f" searched, every file whatever its name:\n" + f" {licenses_dir}\n" + f" An upstream that renamed or moved its notice needs it named:\n" f" about:\n" f" license_file: path/to/LICENSE # or a list of paths\n" - f" A recipe with genuinely nothing to ship says so explicitly instead:\n" + f" One that ships no notice at all — a prebuilt binary release, " + f"usually — wants the folder instead: drop a file in per bundled " + f"project and they all ship, with no list in meta.yaml to go stale.\n" + f" A recipe with genuinely nothing to ship says so explicitly:\n" f" about:\n" f" license_file: [] # deliberately none -- " ) diff --git a/src/forge/schema/meta-schema.yaml b/src/forge/schema/meta-schema.yaml index 995e10dc..272c6944 100644 --- a/src/forge/schema/meta-schema.yaml +++ b/src/forge/schema/meta-schema.yaml @@ -250,9 +250,15 @@ properties: ship), to add to the wheel's .dist-info/licenses/. Any top-level file in either directory whose name starts with "LICEN[CS]E", "COPYING", "COPYRIGHT" or "NOTICE" (case-insensitive) is included - automatically, so most recipes need nothing here. Setting this - REPLACES that discovery, which is what you want in the two cases - it cannot get right: excluding a notice covering something the + automatically, and so is every file in the recipe's `licenses/` + directory whatever it is named — so most recipes need nothing here. + Prefer that folder when the upstream archive ships no notice at all + and the recipe has to supply one per bundled project: the files + stay out of the recipe root, and there is no list here to fall out + of step with them. Its name is not part of the destination, so a + notice lands at .dist-info/licenses/, not doubled. + Setting this REPLACES discovery, which is what you want in the two + cases it cannot get right: excluding a notice covering something the wheel doesn't contain (libiconv's top-level COPYING is the GPL for the `iconv` program, not the LGPL library we build), and including one below the top level (a bundled library's own notice). From 7ccb78c01bee9bbe917000a503986b46b034e1ca Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 31 Aug 2026 18:34:05 +0200 Subject: [PATCH 14/18] docs: link curl-cffi's API references, drop the Build notes preamble [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Session, FileCacheBackend, Curl, ImpersonateError and CertificateVerifyError were named without links, against README.rst's rule to link every API reference the first time it appears. All five resolve to upstream's api.html anchors; `requests` now points at curl-cffi's own vs-requests page and certifi at its repository. FingerprintManager and config_warnings stay unlinked deliberately — upstream documents neither, and a guessed anchor is worse than none. The Build notes opener said patches/mobile.patch explains its own hunks and meta.yaml justifies its settings inline. Both are true, both are this repo's standing convention, and the reader of a maintainer section has both files open — so the paragraph spent three lines telling them nothing. The section now starts at Recipe shape. All 31 links and all 16 anchors verified to resolve. --- recipes/curl-cffi/README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/recipes/curl-cffi/README.md b/recipes/curl-cffi/README.md index f521f245..da88dfc2 100644 --- a/recipes/curl-cffi/README.md +++ b/recipes/curl-cffi/README.md @@ -9,9 +9,10 @@ its callers by [JA3/JA4](https://github.com/FoxIO-LLC/ja4) or by frame shape see for it in a Flet app when an API or page you need answers an ordinary client with a challenge page, a 403 or a silent block. -The API is `requests`-shaped and everything is re-exported at the top level: +The API is [`requests`-shaped](https://curl-cffi.readthedocs.io/en/latest/vs-requests.html) +and everything is re-exported at the top level: [`curl_cffi.get(url, ...)`](https://curl-cffi.readthedocs.io/en/latest/quick_start.html) for a -one-shot, `Session` or +one-shot, [`Session`](https://curl-cffi.readthedocs.io/en/latest/api.html#curl_cffi.requests.Session) or [`AsyncSession`](https://curl-cffi.readthedocs.io/en/latest/asyncio.html) when you want the connection pool and the cookie jar to survive between calls, and [`WebSocket`](https://curl-cffi.readthedocs.io/en/latest/websockets.html) for a socket. @@ -95,8 +96,8 @@ user expects to keep belongs in [`FLET_APP_STORAGE_DATA`](https://flet.dev/docs/reference/environment-variables/#flet_app_storage_data), which is never auto-deleted. -Two paths the library picks for itself. The first is the response cache: `FileCacheBackend` -defaults to `tempfile.gettempdir() / "curl_cffi_cache"`, wherever the stdlib puts that on device. +Two paths the library picks for itself. The first is the response cache: +[`FileCacheBackend`](https://curl-cffi.readthedocs.io/en/latest/api.html#curl_cffi.requests.FileCacheBackend) defaults to `tempfile.gettempdir() / "curl_cffi_cache"`, wherever the stdlib puts that on device. Point it somewhere you chose: ```python @@ -158,7 +159,8 @@ shape to copy. curl-cffi resolves one CA bundle at **import** time and hands the path to libcurl at request time. It tries `SSL_CERT_FILE`, `CURL_CA_BUNDLE` and `REQUESTS_CA_BUNDLE` in that order, then -OpenSSL's built-in path, then `certifi.where()` — and on device the first step wins, because +OpenSSL's built-in path, then [`certifi.where()`](https://github.com/certifi/python-certifi) +— and on device the first step wins, because Flet's generated startup code sets `SSL_CERT_FILE` and `REQUESTS_CA_BUNDLE` to `certifi.where()` before your module runs (read out of the `lib/python.dart` a `flet build` generates). So the bundle is certifi's on both platforms, and `tests/` asserts on device that the resolved path is a @@ -169,7 +171,8 @@ Two things follow. Overriding the bundle means setting `SSL_CERT_FILE` **above** naming a file that does not exist is skipped silently. And **the trust anchors on device are only certifi's**, so a corporate root or an intercepting debug proxy such as mitmproxy, which a desktop `flet run` picks up out of the machine's own trust store, fails on the phone with a -`CertificateVerifyError` against a host the browser is perfectly happy with. Ship your own PEM in +[`CertificateVerifyError`](https://curl-cffi.readthedocs.io/en/latest/api.html#curl_cffi.requests.exceptions.CertificateVerifyError) +against a host the browser is perfectly happy with. Ship your own PEM in `src/assets/` and name it per session or per request: ```python @@ -189,10 +192,11 @@ release rather than by you; a versioned name such as `impersonate="chrome131_and only complete list. **The built-in targets are compiled into the wheel, so that list is fixed at build time**, and -checkable offline: `Curl().impersonate(name)` returns `0` when that name's fingerprint tables are +checkable offline: [`Curl()`](https://curl-cffi.readthedocs.io/en/latest/api.html#curl_cffi.Curl)`.impersonate(name)` returns `0` when that name's fingerprint tables are present and a non-zero code when they are not, without raising — which separates "this build does not know that target" from "the request failed" on a device with no connectivity. Through a -`Session` the same miss surfaces as `ImpersonateError` instead. +`Session` the same miss surfaces as +[`ImpersonateError`](https://curl-cffi.readthedocs.io/en/latest/api.html#curl_cffi.requests.exceptions.ImpersonateError) instead. That list is not the whole story: a name the wheel does not carry natively is looked up in `fingerprints.json` under the directory in [Storage](#storage) and applied from Python, so extra @@ -278,10 +282,6 @@ against your own backend, on a device, is the thing to validate. ## Build notes (maintainers) -`patches/mobile.patch` explains its own hunks in its preamble, and `meta.yaml` justifies -`excluded_arches`, the `IMPERSONATE_*` environment and the host pins inline in both recipes. What -is left is shape, hazards, and what a green run does not prove. - ### Recipe shape **Two recipes because the payload is a prebuilt binary, not a build.** From 89313d8c1833d47cb9bb0bbc21ee1185ee26ee1f Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 31 Aug 2026 18:40:33 +0200 Subject: [PATCH 15/18] docs: measure which wheel pip prefers, and stop pinning the example to 3.12 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The page said an unpinned resolve's preference "was not measured" and that a `==` pin could not separate two builds of one version, then told the reader to pin Python instead. Measured now, with `pip download --platform android_24_arm64_v8a` against both indexes: pyzmq==27.1.0 cp313, identical platform tags, only a build tag between them -> pyzmq-27.1.0-1-cp313-cp313-android_24_arm64_v8a.whl (this index) lru-dict==1.4.1 cp313, android_24 here vs android_21 upstream -> lru_dict-1.4.1-1-cp313-cp313-android_24_arm64_v8a.whl (this index) pyzmq unpinned, upstream carries 27.2.0 and this index 27.1.0 -> pyzmq-27.2.0-cp313-cp313-android_24_arm64_v8a.whl (upstream) So a forge wheel is not the lower-priority candidate: at equal version it wins on both the build tag and the higher android_24 platform tag. Version is compared first, so the only thing that flips it is upstream shipping a version this index has not caught up to — and because upstream ships arm64-v8a alone, that yields a MIXED build rather than theirs outright. A `==` pin is therefore exactly the right lever, which is what the example already declares. `requires-python` goes back to `>=3.12`. Forcing 3.12 to dodge upstream's cp313/cp314 wheels was solving a problem that does not exist, at the cost of denying the example the newer runtimes this index builds for. --- recipes/curl-cffi/README.md | 30 +++++++++---------- .../fingerprint-fanout/pyproject.toml | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/recipes/curl-cffi/README.md b/recipes/curl-cffi/README.md index da88dfc2..079b1b3a 100644 --- a/recipes/curl-cffi/README.md +++ b/recipes/curl-cffi/README.md @@ -40,21 +40,21 @@ full; `arm64` and `x64` are the macOS spellings and Flet rejects them here. Drop costs you old hardware rather than current users — 64-bit has been mandatory for Play Store uploads since 2019. -**On Python 3.13 and 3.14, Android arm64-v8a can resolve to upstream's own wheel rather than -this index's.** `flet build` installs with `pip install --upgrade --only-binary :all: ---extra-index-url https://pypi.flet.dev`, so pip sees PyPI too, and upstream publishes exactly -one mobile wheel per minor there — `cp313` and `cp314`, `android_24_arm64_v8a`, at the same -version this index carries; no cp312, no x86_64, no armeabi-v7a, no iOS. One Android build can -therefore carry two different builds of a single version across its ABIs, and a `==` pin cannot -separate them because the versions agree. They are not the same payload — upstream's arm64-v8a -wheel is 8.2 MB compressed and 25.2 MB unpacked against 2.7 MB and 6.3 MB here — and which one an -unpinned resolve prefers was not measured. - -The lever is the **Python** version, not a `curl-cffi` pin: upstream publishes no cp312 mobile wheel -at all, so building on 3.12 is what settles it. Note `flet build` takes the **highest** stable Python -your `requires-python` admits, so the common `>=3.10` lands on 3.14 — exactly the range where the -two builds overlap. Say `requires-python = "==3.12.*"`, or pass `--python-version 3.12`, as the -[`fingerprint-fanout`](examples/fingerprint-fanout) example does. +**Upstream publishes an Android wheel of its own, and on `cp313`/`cp314` it competes with this +one.** `flet build` installs with `--extra-index-url https://pypi.flet.dev`, so pip sees PyPI as +well; upstream ships `android_24_arm64_v8a` only — no x86_64, no armeabi-v7a, no iOS — and it is a +different payload, 8.2 MB compressed against 2.7 MB here. + +**At the same version this wheel wins**, so nothing needs doing while the version here keeps up. +pip compares the version first, and only then the tags — where this index carries a build tag and a +higher `android_24` platform tag against upstream's none, both of which rank it first. What flips +it is upstream releasing a version this index has not caught up to: pip takes the higher version. +That is worth knowing because upstream ships only one ABI, so the result is not "their build" +but a **mixed** one — their arm64-v8a beside this index's x86_64. + +Pin `curl-cffi` to the version in [`meta.yaml`](meta.yaml) if you want that settled rather than +watched, as the [`fingerprint-fanout`](examples/fingerprint-fanout) example does. The Python +version is not the lever: this index carries `cp312`, `cp313` and `cp314` alike. ## Examples diff --git a/recipes/curl-cffi/examples/fingerprint-fanout/pyproject.toml b/recipes/curl-cffi/examples/fingerprint-fanout/pyproject.toml index 51e43c6a..d940c2bf 100644 --- a/recipes/curl-cffi/examples/fingerprint-fanout/pyproject.toml +++ b/recipes/curl-cffi/examples/fingerprint-fanout/pyproject.toml @@ -2,7 +2,7 @@ name = "curl-cffi-fingerprint-fanout" version = "1.0.0" description = "Probes one endpoint as five browsers at once and compares the fingerprints it reports." -requires-python = "==3.12.*" +requires-python = ">=3.12" dependencies = [ "flet==0.86.5", From 7abf9ad6b1dbcc9e98d1087f19b5ad6b30b72894 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 31 Aug 2026 18:40:59 +0200 Subject: [PATCH 16/18] skills: the forge-shadows-official rule holds only at equal versions [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Version is compared before any tag, so an upstream release the recipe has not caught up to wins outright — and because upstream ships arm64-v8a alone, that produces a mixed app rather than theirs outright. Measured against both indexes with pip download. --- .claude/skills/local-recipe-testing/SKILL.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.claude/skills/local-recipe-testing/SKILL.md b/.claude/skills/local-recipe-testing/SKILL.md index e0c23c6d..834ee628 100644 --- a/.claude/skills/local-recipe-testing/SKILL.md +++ b/.claude/skills/local-recipe-testing/SKILL.md @@ -177,7 +177,15 @@ Upstream packages now publish cibuildwheel-built iOS/Android wheels to PyPI (cp3 only). They are live pip candidates in every 0.86 build, **but while a forge recipe exists on pypi.flet.dev it deterministically shadows them** — pip's sort at equal versions is tag-priority (forge `android_24` > official `android_21`) then build tag -(forge `-1-` > none). To force the official wheel on-device without touching the index: +(forge `-1-` > none). **Only at equal versions.** Version is compared before any tag, so +an upstream release the recipe has not caught up to wins outright — and since upstream +publishes arm64-v8a alone, the result is a MIXED app: their arm64 beside forge's x86_64, +one version apart. A stale recipe is the failure mode here, not a resolution quirk. +Measured 2026-08-31 with `pip download --only-binary :all: --platform +android_24_arm64_v8a --python-version 313` against both indexes: `pyzmq==27.1.0` (same +platform tag, build tag alone between them) and `lru-dict==1.4.1` (`android_24` vs +`android_21`) both resolve to the forge wheel; unpinned `pyzmq` resolves to upstream's +newer 27.2.0. To force the official wheel on-device without touching the index: retag a downloaded copy into a find-links dir with a HIGH build tag — and on Android also lift the platform tag past forge's — From 815d1eef892d02d2c112781c0e69f672d4362e1f Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 31 Aug 2026 19:56:15 +0200 Subject: [PATCH 17/18] ci: trigger a run on this branch From a986866ed6bbf67f2b6e45625e508c74cee04823 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 31 Aug 2026 23:55:20 +0200 Subject: [PATCH 18/18] improve pycares example --- .../references/failure-catalogue.md | 1 - .../pycares/examples/dns-lookup/src/main.py | 57 +++++++++++-------- src/forge/build.py | 44 +++++++------- src/forge/package.py | 5 +- 4 files changed, 53 insertions(+), 54 deletions(-) diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index 53ca2949..0be23b6d 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -149,7 +149,6 @@ source: url: https://.../libX-.tar.gz strip: 0 ``` -(The `strip` key requires forge with the `source.strip` schema addition — on the `curl-cffi` branch / after it merges.) Verify the actual behavior empirically before trusting either value: replicate `members()` from `src/forge/build.py` on the real tarball. Precedent: `recipes/flet-libcurl-impersonate/` (branch `curl-cffi`). --- diff --git a/recipes/pycares/examples/dns-lookup/src/main.py b/recipes/pycares/examples/dns-lookup/src/main.py index 03ee8d3d..3e61a86b 100644 --- a/recipes/pycares/examples/dns-lookup/src/main.py +++ b/recipes/pycares/examples/dns-lookup/src/main.py @@ -3,7 +3,6 @@ async def main(page: ft.Page): - """Resolve a name on demand and list the answers.""" discovered = system_nameservers() # c-ares cannot read Android's resolver config, so it falls back to loopback. system_works = not all(s.startswith("127.") for s in discovered) @@ -28,37 +27,47 @@ async def resolve(_): spinner.visible = False page.update() - host = ft.TextField(label="Host", value="pypi.flet.dev", expand=True) - qtype = ft.Dropdown( - value="A", - width=110, - options=[ft.DropdownOption(key=t) for t in RECORD_TYPES], - ) - source = ft.RadioGroup( - value="system" if system_works else "public", - content=ft.Row( - controls=[ - ft.Radio(value="system", label="System DNS"), - ft.Radio(value="public", label=", ".join(PUBLIC_NAMESERVERS)), - ], - ), - ) - spinner = ft.ProgressRing(visible=False, width=18, height=18) - answers = ft.ListView(expand=True, spacing=6) - - page.appbar = ft.AppBar(title=ft.Text("DNS lookup"), center_title=True) + page.appbar = ft.AppBar(title="DNS lookup", center_title=True) page.add( ft.SafeArea( expand=True, content=ft.Column( controls=[ - ft.Row(controls=[host, qtype]), - source, ft.Row( - controls=[ft.Button("Look up", on_click=resolve), spinner] + controls=[ + host := ft.TextField( + label="Host", value="pypi.flet.dev", expand=True + ), + qtype := ft.Dropdown( + value="A", + width=110, + options=[ + ft.DropdownOption(key=t) for t in RECORD_TYPES + ], + ), + ] + ), + source := ft.RadioGroup( + value="system" if system_works else "public", + content=ft.Row( + controls=[ + ft.Radio(value="system", label="System DNS"), + ft.Radio( + value="public", label=", ".join(PUBLIC_NAMESERVERS) + ), + ], + ), + ), + ft.Row( + controls=[ + ft.Button("Look up", on_click=resolve), + spinner := ft.ProgressRing( + visible=False, width=18, height=18 + ), + ] ), ft.Divider(), - answers, + answers := ft.ListView(expand=True, spacing=6), ft.Text( "c-ares found: " + ", ".join(discovered), size=11, diff --git a/src/forge/build.py b/src/forge/build.py index 681a10e5..808f0d69 100644 --- a/src/forge/build.py +++ b/src/forge/build.py @@ -32,20 +32,24 @@ from forge.package import Package -# Names a project uses for its licence notice. Matches the prefix rather than the whole -# name so LICENSE.txt, COPYING.LGPL, LICENCE.md and LICENSE-THIRD-PARTY all qualify. -# COPYRIGHT is included because for several projects here it IS the licence grant and -# the only such file shipped — postgresql, libxml2 and libxslt each carry their full -# permission notice in a file by that name and nothing else. NOTICE is included because -# Apache-2.0 section 4(d) requires redistributing it alongside the licence, and an -# ASF-sourced archive (arrow) keeps its attributions there rather than in LICENSE. LICENSE_FILE_RE = re.compile(r"^(licen[cs]e|copying|copyright|notice)", re.IGNORECASE) +""" +Names a project uses for its licence notice. Matches the prefix rather than the whole +name so LICENSE.txt, COPYING.LGPL, LICENCE.md and LICENSE-THIRD-PARTY all qualify. +COPYRIGHT is included because for several projects here it IS the licence grant and +the only such file shipped — postgresql, libxml2 and libxslt each carry their full +permission notice in a file by that name and nothing else. NOTICE is included because +Apache-2.0 section 4(d) requires redistributing it alongside the licence, and an +ASF-sourced archive (arrow) keeps its attributions there rather than in LICENSE. +""" -# A recipe subdirectory whose entire contents are licence notices, alongside tests/ -# and examples/. Everything in it ships, whatever it is named — a recipe supplying -# notices for a binary that carries none usually has one per bundled project, and -# those names (COPYING.curl, LICENSE.boringssl) are ours to choose. LICENSES_DIR_NAME = "licenses" +""" +A recipe subdirectory whose entire contents are licence notices, alongside. +Everything in it ships, whatever it is named — a recipe supplying +notices for a binary that carries none usually has one per bundled project, and +those names (COPYING.curl, LICENSE.boringssl) are ours to choose. +""" class Builder(ABC): @@ -911,8 +915,6 @@ def fix_wheel(self, wheel_dir: Path): # `about` drives metadata that only the build.sh path synthesises. A Python # package's own build backend already carries upstream's licence through, so # setting it there would be a no-op — say so rather than ignoring it silently. - # Test what the recipe DECLARED: schema defaults leave every recipe with a - # populated `about` dict, so testing the dict warned on all of them. if self.package.declares_about and not self.synthesizes_metadata: log( self.log_file, @@ -1158,12 +1160,8 @@ def collect_license_files(self) -> list[tuple[Path, str]]: which is how a recipe can carry a fallback without overriding upstream. Everything under the recipe's `licenses/` directory is collected as well, - whatever each file is called. A recipe repackaging a prebuilt binary that - ships no notice at all has to supply one per bundled project, and that is a - set of files rather than one — a folder keeps them out of the recipe root - (next to tests/ and examples/) and out of a meta.yaml list that would go - stale on the next bump. The folder name is not part of the destination, so a - notice lands at `licenses/` in the wheel rather than doubled. + whatever each file is called. The folder name is not part of the destination, + so a notice lands at `licenses/` in the wheel rather than doubled. `about.license_file` (a path, or a list of them) replaces that discovery entirely, for the two cases it cannot get right on its own: excluding a notice @@ -1209,8 +1207,7 @@ def collect_license_files(self) -> list[tuple[Path, str]]: candidate = directory / name if candidate.is_file(): # Naming a file inside the recipe's licenses/ explicitly must - # land it where the folder convention would, not at - # licenses/licenses/. + # land it where the folder convention would, not at licenses/licenses/. dest = name if directory == self.package.recipe_path: parts = Path(name).parts @@ -1257,10 +1254,7 @@ def collect_license_files(self) -> list[tuple[Path, str]]: f" An upstream that renamed or moved its notice needs it named:\n" f" about:\n" f" license_file: path/to/LICENSE # or a list of paths\n" - f" One that ships no notice at all — a prebuilt binary release, " - f"usually — wants the folder instead: drop a file in per bundled " - f"project and they all ship, with no list in meta.yaml to go stale.\n" - f" A recipe with genuinely nothing to ship says so explicitly:\n" + f" A recipe with genuinely nothing to ship says so explicitly instead:\n" f" about:\n" f" license_file: [] # deliberately none -- " ) diff --git a/src/forge/package.py b/src/forge/package.py index 8344f16e..d694b10d 100644 --- a/src/forge/package.py +++ b/src/forge/package.py @@ -102,10 +102,7 @@ def set_defaults(validator, properties, instance, schema): except KeyError: pass - # Whether the recipe actually wrote an `about:` block. Validation fills in - # schema defaults in place, and `about`'s are empty strings rather than - # nothing — so afterwards every recipe has a non-empty `about` dict and - # "did the author ask for this?" is no longer answerable from the metadata. + # Whether the recipe actually wrote an `about:` block. self.declares_about = "about" in meta # Validate the metadata against the schema.