diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreDataSource.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreDataSource.java index eb8891339..6d7d7f5e0 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreDataSource.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreDataSource.java @@ -84,6 +84,11 @@ public JSONObject postImportStart(JSONObject body) throws Exception { return postJson(V1 + "/import/item", body); } + /** {@code POST /v1/import/discard} with {@code {app, entry}}: drop a given-up pull's kept bytes. */ + public JSONObject postImportDiscard(JSONObject body) throws Exception { + return postJson(V1 + "/import/discard", body); + } + /** * {@code GET /v1/import/status/{opId}} -> {@code {state, movedBytes, totalBytes, error?, code?, * result?}}. A 404 means the op was swept; it surfaces as {@link ConexHttpException} with diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreImportSource.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreImportSource.java index d8fed15e4..bbf7b05cb 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreImportSource.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreImportSource.java @@ -16,6 +16,7 @@ import org.appdevforall.k2go.contentexchange.domain.ImportStatus; import org.appdevforall.k2go.contentexchange.domain.ImportVerdict; import org.appdevforall.k2go.contentexchange.domain.ItemRef; +import org.appdevforall.k2go.contentexchange.domain.PullUnsupportedException; import org.json.JSONArray; import org.json.JSONObject; @@ -67,6 +68,46 @@ public String startImport(String app, String entryJson, String payloadBoxPath) t return opId; } + @Override + public String startPull(String app, String entryJson, String host, int port, String module, + String token) throws Exception { + JSONObject source = new JSONObject() + .put("host", host) + .put("port", port) + .put("module", module) + .put("token", token); + JSONObject body = new JSONObject() + .put("app", app) + .put("entry", new JSONObject(entryJson)) + .put("source", source); + JSONObject r; + try { + r = dataSource.postImportStart(body); + } catch (ConexCoreDataSource.ConexHttpException e) { + // 400: a core older than the pull wants a payloadPath. 501: no rsync on this box. + if (e.code() == 400 || e.code() == 501) { + throw new PullUnsupportedException(e.getMessage()); + } + throw e; + } + String opId = r.optString("opId", ""); + if (opId.isEmpty()) { + throw new Exception("pull start returned no opId"); + } + return opId; + } + + @Override + public void discardPull(String app, String entryJson) { + try { + dataSource.postImportDiscard(new JSONObject() + .put("app", app) + .put("entry", new JSONObject(entryJson))); + } catch (Exception ignored) { + // best effort: the box sweeps partials older than 48 h on its own + } + } + @Override public ImportStatus importStatus(String opId) throws Exception { JSONObject r; diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/NetworkImportCarrier.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/NetworkImportCarrier.java index afad31cd2..dae7859c9 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/NetworkImportCarrier.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/NetworkImportCarrier.java @@ -36,6 +36,9 @@ import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * One Share session as an {@link ImportCarrier}, read over the hotspot from the sender's gate. The @@ -81,6 +84,8 @@ public interface Cancellation { private final String token; private final String manifestUrl; private final Cancellation cancel; + /** Progress reports to the sender (K2GO-577). Its one thread ends on its own when idle. */ + private final ThreadPoolExecutor reporter; public NetworkImportCarrier(Context context, String host, int port, String token, String manifestUrl, Cancellation cancel) { @@ -90,6 +95,13 @@ public NetworkImportCarrier(Context context, String host, int port, String token this.token = token; this.manifestUrl = manifestUrl; this.cancel = cancel != null ? cancel : () -> false; + this.reporter = new ThreadPoolExecutor(1, 1, 10L, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(1), r -> { + Thread t = new Thread(r, "share-progress"); + t.setDaemon(true); + return t; + }, new ThreadPoolExecutor.DiscardOldestPolicy()); + this.reporter.allowCoreThreadTimeOut(true); } @Override @@ -179,6 +191,43 @@ public void onAllDelivered() { } } + /** + * Best-effort report of one item's progress to the sender's gate (K2GO-577): it feeds the + * sender's view and keeps its session fresh during a long rsync pull, which the gate cannot see. + * {@code state} is "running", "done" or "error". A failure is logged and ignored. + */ + public void reportProgress(String itemRef, long movedBytes, long totalBytes, String state) { + if (!ShareUrls.isValidItemRef(itemRef)) { + return; + } + // Off the import thread, so an unreachable sender never stalls the poll loop. One report + // in flight and one queued; an older queued report is dropped for the newer one. + reporter.execute(() -> postProgress(itemRef, movedBytes, totalBytes, state)); + } + + private void postProgress(String itemRef, long movedBytes, long totalBytes, String state) { + try { + JSONObject body = new JSONObject() + .put("itemRef", itemRef) + .put("movedBytes", Math.max(0L, movedBytes)) + .put("totalBytes", Math.max(0L, totalBytes)) + .put("state", state); + HttpURLConnection c = open(ShareUrls.progress(baseUrl, manifestUrl), "POST", READ_TIMEOUT_MS); + try { + c.setDoOutput(true); + c.setRequestProperty("Content-Type", "application/json"); + try (OutputStream os = c.getOutputStream()) { + os.write(body.toString().getBytes(StandardCharsets.UTF_8)); + } + c.getResponseCode(); + } finally { + c.disconnect(); + } + } catch (Exception e) { + Log.i(TAG, "share progress report failed: " + e.getMessage()); + } + } + // ---- HTTP plumbing ------------------------------------------------------------------------- /** Open a connection to the sender, pinned to the Network that reaches its hotspot when one exists, diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareRelay.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareRelay.java new file mode 100644 index 000000000..9d891422c --- /dev/null +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareRelay.java @@ -0,0 +1,143 @@ +/* + * ============================================================================ + * Name : ShareRelay.java + * Author : AppDevForAll + * Copyright : Copyright (c) 2026 AppDevForAll + * Description : K2GO-577. A loopback TCP relay from the box to the sender's rsync daemon. The box's + * rsync is a Linux process that Android routes by the default network, which on a + * hotspot without internet can be cellular, so it may never reach the sender. The relay + * accepts on 127.0.0.1 and opens each outbound socket on the Network whose subnet holds + * the sender (WifiNetworkBinder), the same pinning NetworkImportCarrier uses for HTTP. + * ============================================================================ + */ +package org.appdevforall.k2go.contentexchange.data; + +import android.content.Context; +import android.net.Network; +import android.util.Log; + +import org.appdevforall.k2go.sync.transport.WifiNetworkBinder; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Forwards each loopback connection to {@code host:port}. Lifecycle: the import service opens one per + * network run and closes it when the run ends; close() also ends every live connection, so a stopped + * run never leaves a transfer flowing. + */ +public final class ShareRelay implements Closeable { + + private static final String TAG = "IIAB-ShareRelay"; + private static final int CONNECT_TIMEOUT_MS = 15000; + private static final int BUFFER = 64 * 1024; + + private final Context appContext; + private final String host; + private final int port; + private final ServerSocket server; + private final Set live = ConcurrentHashMap.newKeySet(); + private volatile boolean closed; + + public ShareRelay(Context context, String host, int port) throws IOException { + this.appContext = context.getApplicationContext(); + this.host = host; + this.port = port; + this.server = new ServerSocket(0, 4, InetAddress.getByName("127.0.0.1")); + Thread accept = new Thread(this::acceptLoop, "share-relay-accept"); + accept.setDaemon(true); + accept.start(); + } + + /** The loopback port the box connects to. */ + public int localPort() { + return server.getLocalPort(); + } + + @Override + public void close() { + closed = true; + closeQuietly(server); + for (Socket s : live) { + closeQuietly(s); + } + live.clear(); + } + + private void acceptLoop() { + while (!closed) { + Socket inbound; + try { + inbound = server.accept(); + } catch (IOException e) { + return; // closed + } + live.add(inbound); + Thread t = new Thread(() -> bridge(inbound), "share-relay-conn"); + t.setDaemon(true); + t.start(); + } + } + + private void bridge(Socket inbound) { + Socket outbound = null; + try { + // Resolve the Network per connection, so a Wi-Fi reconnect is picked up on the next pull. + Network net = WifiNetworkBinder.findHostNetwork(appContext, host); + outbound = net != null ? net.getSocketFactory().createSocket() : new Socket(); + live.add(outbound); + outbound.connect(new InetSocketAddress(host, port), CONNECT_TIMEOUT_MS); + Socket out = outbound; + Thread up = new Thread(() -> pump(inbound, out), "share-relay-up"); + up.setDaemon(true); + up.start(); + pump(outbound, inbound); + } catch (IOException e) { + Log.i(TAG, "relay connection ended: " + e.getMessage()); + } finally { + closeQuietly(inbound); + closeQuietly(outbound); + live.remove(inbound); + if (outbound != null) { + live.remove(outbound); + } + } + } + + /** Copy one direction until EOF or an error, then close both ends so the peer pump ends too. */ + private static void pump(Socket from, Socket to) { + byte[] buf = new byte[BUFFER]; + try { + InputStream in = from.getInputStream(); + OutputStream out = to.getOutputStream(); + int n; + while ((n = in.read(buf)) != -1) { + out.write(buf, 0, n); + } + } catch (IOException ignored) { + // a closed socket on either side ends the pump + } finally { + closeQuietly(from); + closeQuietly(to); + } + } + + private static void closeQuietly(Closeable c) { + if (c == null) { + return; + } + try { + c.close(); + } catch (IOException ignored) { + // nothing to recover + } + } +} diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareRsyncTarget.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareRsyncTarget.java new file mode 100644 index 000000000..8b46cd60e --- /dev/null +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareRsyncTarget.java @@ -0,0 +1,64 @@ +/* + * ============================================================================ + * Name : ShareRsyncTarget.java + * Author : AppDevForAll + * Copyright : Copyright (c) 2026 AppDevForAll + * Description : K2GO-577. The rsync block of one share manifest entry ({"rsync": {port, module}}), + * read and validated at the network boundary. Absent or invalid means the item uses + * the HTTP fetch, so a sender that runs no rsync daemon still works. The manifest + * reader keeps the block in the entry JSON (it strips only itemRef and delivered). + * ============================================================================ + */ +package org.appdevforall.k2go.contentexchange.data; + +import androidx.annotation.Nullable; + +import org.json.JSONObject; + +import java.util.regex.Pattern; + +/** Where one shared item can be pulled with rsync: the sender's daemon port and module. */ +public final class ShareRsyncTarget { + + /** Same rule as the core's module names: "-". */ + private static final Pattern MODULE = Pattern.compile("^[A-Za-z0-9][A-Za-z0-9-]{0,80}$"); + + private final int port; + private final String module; + + private ShareRsyncTarget(int port, String module) { + this.port = port; + this.module = module; + } + + /** The target of a manifest entry, or null when it has no valid rsync block. */ + @Nullable + public static ShareRsyncTarget fromEntry(@Nullable String entryJson) { + if (entryJson == null) { + return null; + } + try { + JSONObject entry = new JSONObject(entryJson); + JSONObject rsync = entry.optJSONObject("rsync"); + if (rsync == null) { + return null; + } + int port = rsync.optInt("port", 0); + String module = rsync.optString("module", ""); + if (port < 1 || port > 65535 || !MODULE.matcher(module).matches()) { + return null; + } + return new ShareRsyncTarget(port, module); + } catch (Exception e) { + return null; + } + } + + public int port() { + return port; + } + + public String module() { + return module; + } +} diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ImportSource.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ImportSource.java index 2a08d6525..12dfa74d4 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ImportSource.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ImportSource.java @@ -27,6 +27,22 @@ public interface ImportSource { */ String startImport(String app, String entryJson, String payloadBoxPath) throws Exception; + /** + * Start a pull long operation (K2GO-577): the box pulls the item from a sender's rsync module + * ({@code host}:{@code port}, {@code module}, password {@code token}) straight into place, with + * no staged copy. The op is polled with {@link #importStatus} like an import; its movedBytes are + * the bytes on disk, so a resumed pull starts from the bytes it kept. + */ + String startPull(String app, String entryJson, String host, int port, String module, String token) + throws Exception; + + /** + * Drop what a given-up pull of this item kept for a resume (K2GO-577): the user ended the run, so + * the partial bytes must not hold the disk. Best effort: a failure is ignored (the box sweeps old + * partials on its own). + */ + void discardPull(String app, String entryJson); + /** * Poll one import operation's status. {@link ImportStatus.State#GONE} means the op was swept, so * the caller re-starts once. Throws on a transport failure (the caller tolerates a few blips). diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/PullUnsupportedException.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/PullUnsupportedException.java new file mode 100644 index 000000000..6a48e3390 --- /dev/null +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/PullUnsupportedException.java @@ -0,0 +1,12 @@ +package org.appdevforall.k2go.contentexchange.domain; + +/** + * The box refused to start an rsync pull (K2GO-577): its core is older than the pull, or it has no + * rsync. A typed signal so the import service moves the item over the HTTP fetch instead of failing + * it. The data-layer import source maps the core's 400 / 501 on a pull start to this exception. + */ +public final class PullUnsupportedException extends Exception { + public PullUnsupportedException(String message) { + super(message); + } +} diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ShareUrls.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ShareUrls.java index 08855386d..73dbe6e3c 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ShareUrls.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ShareUrls.java @@ -51,6 +51,12 @@ public static String complete(String baseUrl, String manifestPath) { return baseUrl + sessionPrefix(manifestPath) + "/complete"; } + /** Where the receiver reports per-item progress to the sender's gate (K2GO-577). */ + public static String progress(String baseUrl, String manifestPath) { + requireOrigin(baseUrl); + return baseUrl + sessionPrefix(manifestPath) + "/progress"; + } + /** * The share-session path prefix, the manifest path with its trailing {@code /manifest} removed * ({@code /v1/share//manifest} -> {@code /v1/share/}). Fails closed when the path is not a diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/presentation/ContentImportService.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/presentation/ContentImportService.java index 760c012cd..abd6be3bd 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/presentation/ContentImportService.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/presentation/ContentImportService.java @@ -8,8 +8,9 @@ * the core reads, POST /v1/import/item so the core ingests it (delegating to the app's own * import), then remove the staging copy. The mirror of ContentExportService (foreground * specialUse + per-item queue + retry + notification + grant release), with the bytes - * flowing drive -> phone staging -> box. The item is the unit of resume (per-item retry); - * pause/resume and within-item byte-resume are later work. + * flowing drive -> phone staging -> box. The item is the unit of resume (per-item retry). + * K2GO-577: a Share item the sender offers over rsync skips the staging copy: the box + * pulls it straight into place through a loopback relay, and a Retry resumes its bytes. * ============================================================================ */ package org.appdevforall.k2go.contentexchange.presentation; @@ -37,12 +38,15 @@ import org.appdevforall.k2go.contentexchange.data.ConexCoreImportSource; import org.appdevforall.k2go.contentexchange.data.NetworkImportCarrier; import org.appdevforall.k2go.contentexchange.data.SafImportCarrier; +import org.appdevforall.k2go.contentexchange.data.ShareRelay; +import org.appdevforall.k2go.contentexchange.data.ShareRsyncTarget; import org.appdevforall.k2go.contentexchange.domain.ExportJob; import org.appdevforall.k2go.contentexchange.domain.ImportCarrier; import org.appdevforall.k2go.contentexchange.domain.ImportNoSpaceException; import org.appdevforall.k2go.contentexchange.domain.ImportSource; import org.appdevforall.k2go.contentexchange.domain.ImportStatus; import org.appdevforall.k2go.contentexchange.domain.ItemRef; +import org.appdevforall.k2go.contentexchange.domain.PullUnsupportedException; import org.appdevforall.k2go.download.domain.DownloadEta; import org.appdevforall.k2go.notifications.NotificationIds; @@ -87,6 +91,10 @@ public final class ContentImportService extends Service { /** K2GO-376: consecutive status-poll failures tolerated before an item fails. The ingest runs * minutes, so one relay blip or a busy core must not discard it (mirrors ContentExportService). */ private static final int MAX_POLL_ERRORS = 10; + /** K2GO-577: gap between progress reports to the sender's gate during an rsync pull. */ + private static final long SENDER_REPORT_INTERVAL_MS = 5000L; + /** The box reaches the relay on loopback (the box shares the phone's network namespace). */ + private static final String LOOPBACK = "127.0.0.1"; /** The single run state the progress fragment observes. */ private static final ImportSession SESSION = new ImportSession(); @@ -161,9 +169,39 @@ public static void cancel(Context ctx) { /** Clear a settled session (the "Finish" button) and release its SAF grant. Safe when idle. */ public static void finishSession(Context ctx) { releaseGrant(ctx, SESSION.treeUri()); + discardGivenUpPulls(); // Finish after a failure gives up the items that did not arrive SESSION.clear(); } + /** + * K2GO-577 (decision a): the user ended the run (Cancel or Finish), so the items that did not + * arrive will not be resumed. Ask the box to drop the bytes their rsync pulls kept, instead of + * holding the disk until the 48 h sweep. A drop or a Retry never comes here, so those keep their + * bytes. Snapshot now (the session is cleared next), then call off the main thread. + */ + private static void discardGivenUpPulls() { + if (!SESSION.isNetwork()) { + return; + } + ExportJob job = SESSION.job(); + final java.util.List pending = new java.util.ArrayList<>(); + for (int i = 0; i < job.size(); i++) { + String entry = SESSION.entryJson(i); + if (job.status(i) != ExportJob.DONE && ShareRsyncTarget.fromEntry(entry) != null) { + pending.add(new String[]{SESSION.refAt(i).app(), entry}); + } + } + if (pending.isEmpty()) { + return; + } + new Thread(() -> { + ImportSource source = new ConexCoreImportSource(new ConexCoreDataSource()); + for (String[] p : pending) { + source.discardPull(p[0], p[1]); + } + }, "content-import-discard").start(); + } + /** * Release the persisted SAF grant for a finished import. Import is one-shot (pick a drive, read, * done), so the grant is not needed afterwards; releasing it keeps the app's persisted-permission @@ -208,6 +246,7 @@ public int onStartCommand(Intent intent, int flags, int startId) { // No worker owns teardown: a fresh service instance may be handling this cancel, so // satisfy the FGS start-foreground obligation before stopping. startForeground(NOTIFICATION_ID, buildNotification(null)); + discardGivenUpPulls(); SESSION.clear(); stopForeground(true); stopSelf(); @@ -294,10 +333,28 @@ private void drain() { return; } SESSION.setSource(carrier.sourceLabel()); + NetworkImportCarrier net = carrier instanceof NetworkImportCarrier + ? (NetworkImportCarrier) carrier : null; + // The rsync relay opens with the first pulled item and closes with the run on every exit + // path, which also ends any live pull connection. + ShareRelay[] relay = {null}; + try { + drainItems(source, staging, carrier, net, relay); + } finally { + if (relay[0] != null) { + relay[0].close(); + } + } + } + /** The per-item loop of {@link #drain()}. {@code relay} holds the rsync relay the caller closes. */ + private void drainItems(ImportSource source, BoxImportStaging staging, ImportCarrier carrier, + @Nullable NetworkImportCarrier net, ShareRelay[] relay) { ExportJob job = SESSION.job(); long lastSampleAt = SystemClock.elapsedRealtime(); long lastSampleBytes = job.copiedTotal(); + // False once this box refused a pull (an older core): the rest of the run uses the HTTP fetch. + boolean pullSupported = true; for (int i = job.firstPending(); i >= 0; i = job.firstPending()) { if (SESSION.isCancelled()) { @@ -309,9 +366,27 @@ private void drain() { notifyNow(); String stored = BoxImportStaging.safeName(SESSION.payloadName(i)); + ShareRsyncTarget target = net != null && pullSupported + ? ShareRsyncTarget.fromEntry(SESSION.entryJson(i)) : null; try { - copyToStaging(carrier, staging, SESSION.payloadName(i), stored, i, job); - ingest(source, ref, i, stored); + boolean pulled = false; + if (target != null) { + // The sender offers rsync: the box pulls the item into place, no staging copy. + if (relay[0] == null) { + relay[0] = new ShareRelay(getApplicationContext(), SESSION.shareHost(), target.port()); + } + try { + pull(source, ref, i, target, relay[0], net, job); + pulled = true; + } catch (PullUnsupportedException e) { + Log.i(TAG, "box cannot pull, using the HTTP fetch: " + e.getMessage()); + pullSupported = false; + } + } + if (!pulled) { + copyToStaging(carrier, staging, SESSION.payloadName(i), stored, i, job); + ingest(source, ref, i, stored); + } job.setDone(i); } catch (Exception e) { // The staging copy or the ingest failed: mark the item and keep its slot for a retry. @@ -340,6 +415,8 @@ private void drain() { } if (SESSION.isCancelled()) { + // The running pull was aborted (runBoxOp), so the box no longer writes its partial. + discardGivenUpPulls(); SESSION.clear(); stopForeground(true); stopSelf(); @@ -420,7 +497,68 @@ private void copyToStaging(ImportCarrier carrier, BoxImportStaging staging, Stri */ private void ingest(ImportSource source, ItemRef ref, int i, String storedName) throws Exception { String boxPath = BoxImportStaging.boxPath(storedName); - String opId = source.startImport(ref.app(), SESSION.entryJson(i), boxPath); + runBoxOp(source, () -> source.startImport(ref.app(), SESSION.entryJson(i), boxPath), null); + } + + /** + * Pull one Share item with rsync (K2GO-577): the box pulls it from the sender's daemon through the + * loopback relay, straight into place. The op's movedBytes are the bytes on disk, so the item bar + * moves with the transfer and a Retry starts from the bytes the last attempt kept. The sender gets + * a progress report every few seconds and a final done / error. + */ + private void pull(ImportSource source, ItemRef ref, int i, ShareRsyncTarget target, ShareRelay relay, + NetworkImportCarrier net, ExportJob job) throws Exception { + String itemRef = SESSION.payloadName(i); + long[] last = {0L, 0L}; // moved, total + long[] window = {-1L, 0L, 0L}; // at, bytes, reportAt + try { + runBoxOp(source, () -> source.startPull(ref.app(), SESSION.entryJson(i), LOOPBACK, + relay.localPort(), target.module(), SESSION.shareToken()), (moved, total) -> { + last[0] = moved; + last[1] = total; + job.setCopied(i, moved); + long now = SystemClock.elapsedRealtime(); + if (window[0] < 0) { + // Anchor the rate on the first reading: a resumed pull starts at its kept bytes, + // which must not count as bytes moved in this window. + window[0] = now; + window[1] = job.copiedTotal(); + } else if (now - window[0] >= RATE_WINDOW_MS) { + sampleRate(job, now - window[0], job.copiedTotal() - window[1]); + window[0] = now; + window[1] = job.copiedTotal(); + } + if (now - window[2] >= SENDER_REPORT_INTERVAL_MS) { + net.reportProgress(itemRef, moved, total, "running"); + window[2] = now; + } + }); + } catch (PullUnsupportedException e) { + throw e; // nothing moved: the item goes over the HTTP fetch, so no error report + } catch (Exception e) { + net.reportProgress(itemRef, last[0], last[1], "error"); + throw e; + } + net.reportProgress(itemRef, last[1], last[1], "done"); + } + + /** Starts one box long operation; returns its opId. */ + private interface OpStarter { + String start() throws Exception; + } + + /** Receives a running op's moved / total bytes on each poll. */ + private interface MovedSink { + void onMoved(long moved, long total); + } + + /** + * Drive one box long operation (import or pull) to its end: poll its status, re-start once if it + * was swept, and throw on an error. {@code sink} is null for an import of an already-staged item. + */ + private void runBoxOp(ImportSource source, OpStarter starter, @Nullable MovedSink sink) + throws Exception { + String opId = starter.start(); boolean restarted = false; int pollErrors = 0; while (true) { @@ -445,6 +583,9 @@ private void ingest(ImportSource source, ItemRef ref, int i, String storedName) pollErrors = 0; switch (st.state()) { case DONE: + if (sink != null) { + sink.onMoved(st.movedBytes(), st.totalBytes()); + } // Release the consumed op now (an idempotent drop on the box), so a finished // import does not linger until the TTL sweep. This is import's normal-completion // cleanup, since it has no fetch stream whose close drops the op as export does. @@ -463,10 +604,13 @@ private void ingest(ImportSource source, ItemRef ref, int i, String storedName) throw new Exception("the import operation was lost"); } restarted = true; - opId = source.startImport(ref.app(), SESSION.entryJson(i), boxPath); + opId = starter.start(); break; case RUNNING: default: + if (sink != null) { + sink.onMoved(st.movedBytes(), st.totalBytes()); + } // Keep the session fresh so the UI shows the item is still being imported. SESSION.publish(); maybeNotify(SystemClock.elapsedRealtime()); diff --git a/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/data/ShareRsyncTargetTest.java b/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/data/ShareRsyncTargetTest.java new file mode 100644 index 000000000..0875e61bd --- /dev/null +++ b/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/data/ShareRsyncTargetTest.java @@ -0,0 +1,40 @@ +package org.appdevforall.k2go.contentexchange.data; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import org.junit.Test; + +/** K2GO-577: the rsync block of a share manifest entry is read only when it is valid. */ +public class ShareRsyncTargetTest { + + private static String entry(String rsync) { + return "{\"app\":\"kiwix\",\"identity\":{\"localRef\":\"a.zim\"}" + + (rsync != null ? ",\"rsync\":" + rsync : "") + "}"; + } + + @Test + public void readsAValidBlock() { + ShareRsyncTarget t = ShareRsyncTarget.fromEntry(entry("{\"port\":9873,\"module\":\"abc123-0\"}")); + assertNotNull(t); + assertEquals(9873, t.port()); + assertEquals("abc123-0", t.module()); + } + + @Test + public void noBlockMeansHttpFetch() { + assertNull(ShareRsyncTarget.fromEntry(entry(null))); + assertNull(ShareRsyncTarget.fromEntry(null)); + assertNull(ShareRsyncTarget.fromEntry("not json")); + } + + @Test + public void rejectsABadPortOrModule() { + assertNull(ShareRsyncTarget.fromEntry(entry("{\"port\":0,\"module\":\"abc-0\"}"))); + assertNull(ShareRsyncTarget.fromEntry(entry("{\"port\":70000,\"module\":\"abc-0\"}"))); + assertNull(ShareRsyncTarget.fromEntry(entry("{\"port\":9873,\"module\":\"../x\"}"))); + assertNull(ShareRsyncTarget.fromEntry(entry("{\"port\":9873,\"module\":\"a b\"}"))); + assertNull(ShareRsyncTarget.fromEntry(entry("{\"port\":9873,\"module\":\"\"}"))); + } +} diff --git a/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/domain/ShareUrlsTest.java b/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/domain/ShareUrlsTest.java index 42d333246..1acd81b4c 100644 --- a/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/domain/ShareUrlsTest.java +++ b/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/domain/ShareUrlsTest.java @@ -22,6 +22,7 @@ public void composesManifestFetchAndComplete() { assertEquals(BASE + "/v1/share/abc123/fetch/0", ShareUrls.fetch(BASE, MANIFEST, "0")); assertEquals(BASE + "/v1/share/abc123/fetch/12", ShareUrls.fetch(BASE, MANIFEST, "12")); assertEquals(BASE + "/v1/share/abc123/complete", ShareUrls.complete(BASE, MANIFEST)); + assertEquals(BASE + "/v1/share/abc123/progress", ShareUrls.progress(BASE, MANIFEST)); } @Test diff --git a/controller/docs/ADR-content-exchange.md b/controller/docs/ADR-content-exchange.md new file mode 100644 index 000000000..06f7ca54c --- /dev/null +++ b/controller/docs/ADR-content-exchange.md @@ -0,0 +1,1208 @@ +# ADR: granular content exchange between boxes + +Status: ANALYSIS : Pillar 0 (contracts: identity + manifest + adapter port + core REST +API) FROZEN 2026-10-22. No ticket created; anchor is the existing K2GO-427. Decisions +below are settled for v1; later pillars build against this frozen Pillar 0 (topology A). + +This is a living design document. We pour the full analysis here (not in memory, +which stays a short index). No Jira ticket is attached yet: the work is larger +than one ticket. A direct ticket opens only when the first shippable deliverable +is ready to build: "install content from an offline disk". That deliverable needs +the whole mechanism first (you cannot cook breakfast without the stove, the pan, +the oil and the fire), so this ADR lands the whole mechanism; the pillars below +become area-level tickets later, under an Epic, each with explicit approval. + +Open decisions are marked `OPEN:`. Sections to fill in are marked `TO DETAIL`. + +ADR altitude: this doc fixes DECISIONS, CONTRACTS (as shapes, not frozen signatures) and +REJECTED alternatives : the things expensive to rediscover or re-litigate. Implementation +detail (exact field names, function signatures, UI layout) is left to the build and +marked `TO DETAIL` / "confirmed when the adapter is built". Generalist on code, precise +on boundaries and the why. + +## Context and problem + +Today a box shares its whole library or nothing: the clone flow (CloneFragment + +rsync) copies everything. Two real needs break that model: + +- Phones have different free space. A user with a small phone cannot take "the + whole library"; they need to pick items. +- Users want to move content between boxes offline (an external disk), add to a + box, remove items to free space, and avoid copying what the other box already + has. + +So we want a subsystem for GRANULAR content exchange: list what is indexed, pick +items, export them, share them selectively, import them with validation, and +delete them to reclaim space. It must work for BOTH Knowledge to Go (Android host) +and Internet-in-a-Box (no Android), and it must stay aware only of the apps that +are actually installed, and learn new apps as modules are installed. + +First concrete apps: Kolibri, Kiwix, Calibre-Web. + +First user-visible deliverable: import selected content from an offline external +disk into another K2Go or IIAB. + +## Tickets (anchor + constellation; no new ticket created by this ADR) + +The analysis is bigger than one ticket (it builds the whole mechanism), but the first +deliverable points at an EXISTING ticket so the work is not in the air: +- ANCHOR: K2GO-427 "Make K2Go & content installable from a connected drive" (To Do). Its + CONTENT side = our first deliverable (offline import), and it names the manage / remove + use case = our delete-to-reclaim pillar. Scope note: "& K2Go" also means sideloading + the APK itself from the drive : a separate concern, NOT the content-exchange mechanism. +Open sibling tickets this subsystem also serves (found, not created): +- K2GO-376 "When cloning, choose content instead of all-or-nothing duplication": the + granular selective-share pillar (Pillar 2 network share). +- K2GO-396 "Content / storage management so you don't have to uninstall the app to free + space": the delete-to-reclaim pillar (Pillar 1 delete). +- K2GO-425 "Support local installation / content update from a non-K2Go server": the + import-from-URL / non-K2Go-source carrier (Pillar 3 input; IIAB-as-source). +- K2GO-494 "Distribute Code On the Go plugins and other auxiliary content": the Code on + the Go adapters (trivial, static-file; see per-app notes). +The pillars become area tickets under these as the work starts, each with explicit OK. +This ADR creates no ticket. + +## Decision 1 (ACCEPTED): the core is an independent REST service in the box + +The core is a standalone service that runs IN THE BOX (the Debian system), exposes +its OWN REST API, and is client-agnostic. It is not part of any one front end. + +Portability is a first-class requirement: the same core runs on +- systemd IIAB hosts: AMD64 and ARM64, and +- our proot host (Android phones): ARM64 and ARM 32-bit. + +Clients talk to the core over its REST API. Two clients: +- K2Go: the Android app -> dash-node (the app's proxy/BFF, which we own) -> core. + A three-level path: app, dash-node (REST), core service (REST) in proot. +- IIAB: the IIAB admin Console -> core. We do NOT build the Console integration; + we leave the core "ready to connect" with a documented REST contract, so IIAB + can consume it the same way dash-node does. (The Console is not our code.) + +Why this shape: +- One brain, two environments and all four architectures. The core only has to + "speak REST": send and receive. Who calls it (dash-node or the Console) does not + change the core. +- The core sits next to Kolibri/Kiwix/Calibre in the box: it queries their indexes + locally, reads/writes their databases, and deletes their files. +- Each front keeps what it is good at: the app does UI/QR/transport; dash-node adds + K2Go orchestration; the Console serves IIAB. + +Scope now vs later: +- NOW: the core + its REST contract + the K2Go path (app -> dash-node -> core). +- LATER (deferred, the unknown part): how IIAB sends content, and the IIAB Console + UI. We design the core so this is a documented extension point, not a rewrite. + +### Sub-decisions (all ACCEPTED) +- D1a core runtime/language: lean Python. IIAB is Python/Ansible-native, Kolibri is + Python, and Python is trivially portable across AMD64/ARM64/ARM32. The core + exposes REST (e.g. Flask/FastAPI). dash-node (Node) stays as the K2Go proxy, so + the box runs both runtimes (IIAB already runs mixed runtimes). +- D1b app<->core path: app -> dash-node -> core (Luis's call). dash-node is a thin + relay, not a second brain. Watch the "one owner per fact" rule: the CORE owns + "what content exists" and the export/import/delete operations; dash-node must not + re-derive or re-own those. +- D1c (ACCEPTED) who orchestrates a long operation: NOT the core alone. The core + is BLIND to the world the operation runs in: the phone battery, the phone's real + free disk (the box writes into a subtree of the phone storage, so "server free + space" is a fiction: the final disk is the phone's), the network (metered / + offline), and even whether the box is still alive (the box can die while the app + looks healthy). So: + - The core exposes PRIMITIVES (list / manifest, stage-export one item, ingest- + import one item, delete one item) and reports its own in-box sub-progress. It + does not own "the transfer" and does not drive the whole long op by itself. + - On K2Go the APP orchestrates: a foreground service keeps the op alive through + Doze and the phantom-process killer; it preflights the PHONE disk, guards + battery / metered network, and it is the SENDER (rsync runs in the app process; + the app pulls on receive). The core feeds it information (what exists, item + bytes, dedup answers) and does the in-box half (read / write the app DB + files). + - On IIAB the Console runs inside the box (always-on Pi, shares the box fate), so + it can own more of the orchestration, and the core can "know everything" there: + the sender is the Console, co-located with the core. + - Ownership is SHARED, and the split differs per environment. Common ground: the + core primitive set + one progress / status vocabulary. The orchestration and the + real-world guards live in the FRONT, sized to that front's reality. + - Principle: whoever can see the real-world constraints owns the orchestration; the + core owns only the in-box execution primitives. The sender is the front (app for + K2Go, Console for IIAB), never the core. + - Grounded by existing hybrid precedents: see "Recon: hybrid precedents" below. +- D1d (ACCEPTED: dual) how the core is updated / deployed. Each new app adapter is a core + change, so the update cadence matters. Two mechanisms, with the tension Luis + raised: + - Ansible role: bakes into the rootfs, IIAB-native, but static: a change forces a + rootfs re-bake and a slow service stop / start. Good as the baseline installer. + - Live self-update: like dash-node's self-update (ADFA-5026 / K2GO-440), which + diffs the installed version against origin/main and rebuilds in place, no re-bake. + Good for shipping adapter additions to the deployed fleet. + Lean for v1: the role installs / owns the core baseline (IIAB parity) AND the core + carries a dash-node-style live self-update so deployed boxes pick up adapters + without a full re-bake. Dual mechanism, mirroring what dash-node already does. + +## Architecture: the pillars + +### Pillar 0: shared contracts (first; freeze before forking) + +Everything else consumes these. If we do not freeze them first, parallel work +diverges (repo rule: shared contracts land first). There are TWO contracts: +- Outer contract: the core's REST API (consumed by dash-node and the IIAB Console). + Platform-facing. Decision 1. +- Inner contract: the app-adapter port (core <-> Kolibri/Kiwix/Calibre). Content-facing. +Both are defined here; the manifest and content identity are shared by both. + +#### Identity semantics (grounded recon: where each id comes from, who computes it) + +The identity choice is not interchangeable per app: it has collision and deletion +risk, so we confirm the SOURCE of each id before we trust it. + +- Kiwix / ZIM: AUTHORITATIVE, read-not-computed. The ZIM header carries a UUID set by + the upstream scraper at build time; it is byte-copied on download, so the same file + always has the same UUID and we never generate it (no "same file, two UUIDs" risk). + Our `kiwix_catalog.csv` already keys the content LINE by (creator, lang, flavour) + and the VERSION by `date`. Dedup: same line, compare date / UUID. +- Kolibri: AUTHORITATIVE, upstream + reused. `channel_id` + integer `version` come + from Kolibri Studio (our catalog `source` is the Studio public API). Kolibri also + content-addresses its files by md5 in its own storage. We REUSE Kolibri's identity + and its content-addressing: one source of truth, no second scheme, nothing + recomputed on-device. This is the "reuse the app's own mechanism" case. +- Calibre: WEAK for cross-box dedup, and DYNAMIC. Calibre-Web is not a book store: one + "book" record can be any media (epub, pdf, image, video, audio) and can hold several + formats at once, so ISBN is NOT a usable key (most items have none). Split the two + identity needs that the other apps let us conflate: + - LOCAL identity (export / delete this exact item on this box): `metadata.db` + `books.uuid`, generated once when the item is added. Stable within the box, + authoritative-local, free. The autoincrement `id` is only a row number and collides + across boxes: never use it cross-box. + - CROSS-BOX dedup ("do you already have this?"): no native answer for arbitrary media. + Use a SOFT, advisory key from metadata Calibre ALREADY has (zero compute): + normalized title + authors + format + byte-size (`data.uncompressed_size`), plus any + external id in the `identifiers` table (isbn / doi / ...) WHEN present as a stronger + hint. Byte-size is a free, strong discriminator. idKind = soft: "looks like you may + already have this", never an automatic skip or delete. + - A true content hash is the only authoritative cross-box key for arbitrary media, but + it is the on-device cost we avoid. So: opt-in "strict" mode only, computed ONCE by + the SENDER at export and written into the manifest (like metalink for downloads); + the receiver hashes only its OWN candidates that already collided on the soft key, + never the whole library. Deferred, not v1 default. + The adapter boundary CONFINES all of this to the Calibre adapter; the manifest and + core stay app-agnostic. + +Consequence: identity is TWO handles, not one, because "grab / delete this exact item" +and "is this the same content as yours" are different questions (Calibre forces the +split; the other apps happen to answer both with the same id): +- `localRef` = the app's own handle to an item on its own box (export / delete). Always + authoritative-local, free. +- `dedup` = { line, build, version, idKind }. idKind authoritative for Kiwix / Kolibri + (hard skip) and soft for Calibre (advisory). We never pretend a weak id is strong. + +#### Manifest (draft): one small top-level file + one per-item entry + +Collection-level (read first, zero ingest cost): +- `format` (e.g. "k2go-content-manifest/1"), `generated`, `source` {box, env}, `items[]`. + +Per item: +- `app` (adapter key: kiwix|kolibri|calibre), `kind` (app content type), `title`, `bytes`. +- `identity`: two handles (see "Identity semantics"): + - `localRef`: the app's own handle to THIS item on THIS box, for export / delete. + Authoritative-local, free. Kiwix=ZIM UUID/file; Kolibri=channel_id+version; + Calibre=books.uuid. + - `dedup`: { `line`, `build`, `version`, `idKind` (authoritative|soft) } for cross-box + "already have this?". Kiwix line=(creator,lang,flavour) build=UUID version=date; + Kolibri line=channel_id build=version; Calibre line=soft key + (title+authors+format+size [+external id]) idKind=soft. +- `integrity`: { `algo`, `value`, `source` } or null : the UPSTREAM hash (metalink + `.meta4` / `.md5`) when present; never computed on-device by default. +- `payload`: relative path of this item's own archive (collection-open, item-atomic). +- `requires`: dependency item ids (usually empty; a Kolibri channel bundles its own + db + files inside its payload). + +#### App-adapter port (draft): the inner contract, one adapter per installed app + +The single owner of "how THIS app identifies, stores, exports, imports and deletes its +content". The core and the manifest stay app-agnostic; a new app = a new adapter. + +- `app()` / `isInstalled()` : the service loads an adapter only if its app is present. +- `list() -> [ManifestItem]` : enumerate indexed content with identity + size + + upstream-hash-if-any (Pillar 1 read side). +- `check(incoming) -> Presence` : ABSENT | SAME_VERSION | OLDER_HERE | NEWER_HERE, + using the identity envelope. Drives "you have an older version: replace or keep + both?". For a soft id (Calibre) the verdict is advisory. +- `export(id, destDir) -> ExportResult` : write THIS item's own archive + return its + manifest entry (Pillar 2). Per item, never a monolith. +- `importItem(entry, payload) -> ImportResult` : ingest one already-local item archive + into the app index, reusing the app's own import where it has one (Pillar 3). +- `delete(id) -> DeleteResult` : remove the item from BOTH the app index and the + physical files (Pillar 1). Where the app exposes its own delete, delegate to it. + +Design rule: where an app has an authoritative mechanism (Kolibri content-addressing, +Kiwix library.xml + UUID), the adapter DELEGATES to it rather than re-deriving : one +source of truth. The weak case (Calibre) is isolated behind its adapter. + +#### Outer contract (draft): the core REST API + +Client-agnostic, same on systemd (IIAB) and proot (K2Go), localhost-only in the box +(like dash-node today); reached by dash-node (K2Go) or the IIAB Console. JSON for +metadata, octet-stream for item payloads. Prefix `/v1`. The core NEVER opens a transfer +to another box and never owns the whole operation: it is a set of per-item PRIMITIVES +the FRONT sequences (D1c). The carrier (disk / rsync) is entirely front-side, so the +SAME API serves offline-disk and network-share: only the front's carrier changes. + +Discovery +- `GET /v1/apps` : installed content apps that have a loaded adapter (what the box can + handle). Honors "aware only of installed apps". + +Read (Pillar 1) +- `GET /v1/content` : the manifest of everything this box has, grouped by app, each item + with identity {localRef, dedup}, size, title. This is what the manager UI renders + (category + disk usage + per item). Filterable by app. +- `GET /v1/content/{app}/{localRef}` : one item detail. + +Dedup (receiver-side; used by share preflight + import) +- `POST /v1/dedup/check` {items:[dedup-key]} : per item ABSENT | SAME_VERSION | + OLDER_HERE | NEWER_HERE + idKind. Runs on the RECEIVER core (it knows what it has). + This is the preflight the QR / offline flow needs: "what can I import, what do I + already have". For a soft idKind the verdict is advisory. + +Export (Pillar 2) : the core STAGES, the FRONT moves the bytes +- `POST /v1/export/plan` {items:[localRef]} : returns manifest entries + EXACT bytes per + item (computes sizes, moves no payload), so the front preflights the PHONE disk and + the user picks BEFORE a byte moves. +- `GET /v1/export/item/{app}/{localRef}` : streams that ONE item's own archive. The + front pulls it under its foreground service + space guard, item by item, with pause / + resume / retry per item, and writes it to the disk or sends it. The core does not + choose the destination and does not own the sequence. + +Import (Pillar 3) : the FRONT hands the core a local payload +- `POST /v1/import/validate` {manifest} : per item importable | NO_APP | dedup verdict. + Partial import is first-class; nothing is moved. Lists what will import and what is + skipped and why. +- `POST /v1/import/item` {app, entry, payloadPath} : ingest ONE already-local item + archive into the app index (payloadPath is box-visible: on proot the front places it + where the box can read it, no double copy). Per item, so the front drives the + sequence, reports progress and retries one. + +Delete (Pillar 1) : reclaim space +- `DELETE /v1/content/{app}/{localRef}` : remove from the app index AND the physical + files; returns freed bytes. Destructive: the FRONT confirms with the user first, the + core executes. One item at a time. + +Sub-progress (reporting, NOT ownership) +- Where a single item is slow (assemble a Kolibri channel bundle, ingest one), the call + streams chunked progress or the front polls `GET /v1/op/{id}` for THAT item's in-box + sub-progress. The front still owns the whole operation, the foreground service and the + real-world guards (D1c). The core never runs a background job that owns the export / + import. + +Mapping: the outer API is just the core composing every loaded adapter; each endpoint +maps 1:1 to an adapter-port operation (`GET /content`->list, `dedup/check`->check, +`export`->export, `import/item`->importItem, `DELETE`->delete). One owner per fact: the +core owns "what exists / export / import / delete"; dash-node is a thin relay, not a +second brain. + +### Pillar 1: the service (query / index / delete) + +In the box. Responsibilities: +- Enumerate INSTALLED apps and load the matching adapter for each. +- Expose a unified read API: what content exists, grouped by category, with sizes + (so the UI can show disk usage by category and per item). +- Expose a delete API: remove an item from both the app index and the physical + files, reclaiming space. +- React to the module lifecycle: when a role/module is installed or removed, add + or drop its adapter, so the service is aware only of what is present. The source is + the IIAB installer's own rootfs flag files, NOT a probe or a cached list: see + "Installed-apps source" below (resolves review finding #2). +- Delete to reclaim space: DECIDED : delegate to each app's own delete, never drive the + app DB + files directly. See "Delete semantics" below. + +#### Installed-apps source (how the registry stays current; resolves review #2) + +The registry must know which apps are installed and stay current as modules are +installed / removed, WITHOUT becoming a second source of truth for "what is installed". +The box already has that owner, and the Android app already consumes it (ADFA-5104 / +K2GO-393): the IIAB installer's OWN flag files on the rootfs. +- `etc/iiab/local_vars.yml`: the INTENT flags, `_install: True`, written BEFORE a + role runs and reverted on failure (so it can be falsely true after a mid-install crash). +- `etc/iiab/iiab_state.yml`: the COMPLETION markers, `_installed: True`, written at + the END of the role play, after all downloads. This is the "did it finish" truth. + +Decision: the core resolves installed apps by READING these files (prefer the +`iiab_state.yml` completion marker; fall back to the `local_vars.yml` intent), restricted +to the apps the core has adapters for. NOT by probing the box (`os.path.isdir` / a live +HTTP probe), NOT by a cached list it must invalidate, NOT via the Android ModuleRegistry +(K2Go-only). + +Why this is the single source, not a second one: +- The files are written by the thing that installs; reading them is reading the owner, + not a parallel copy. Invalidation is free: they live inside the rootfs, so a reset / + restore / clone / reinstall replaces them with everything else: nothing to clear. This + is what the review-second-pass "no duplicate truth" rule actually asks for. +- They are IIAB's own files, present on BOTH systemd IIAB and proot K2Go, so the same + core logic serves dash-node and the IIAB Console: environment-agnostic by construction. + This is why #2 is a CORE decision, below the REST API, serving both clients. +- "Could not read" (null) stays distinct from "nothing installed" (empty): an unreadable + rootfs is not an empty box. + +Staying current: the completion file is small, so the core re-reads it on demand (per +request, or on an mtime change). Re-reading the owner's file is NOT a second source, so +the "do not re-derive / do not cache a parallel truth" rule (finding #2) is met: that +rule is about a PARALLEL cache, not about reading the one file. + +Rule reuse, not duplication: the Android app (Java `InstalledModules` / +`InstalledModulesReader`) and the core (Python) are both CLIENTS of IIAB's flag-file +contract (`_install` / `_installed`), which IIAB's installer owns. The contract +lives in one place; if it changes, both readers follow. The core does not copy the app's +reader; it consumes the same upstream contract. + +Supported-module contract (REQUIRED, the price of one source of truth): every module the +core supports MUST write the IIAB flags: `_install` (intent, in `local_vars.yml`) +and `_installed` (completion, in `iiab_state.yml`). A module that does not write them +is invisible to the core: there is no second place to look, on purpose. So a new supported +module either follows this or is changed to, before the core can see it. +Verified compliant: the Code on the Go family, which are our own roles and were the doubt +here. Both `tools/upstream-patches/overlays/roles/code_assets` and `.../code_addons` +assert `_install` from `local_vars.yml`, write it to `iiab.ini`, gate their install on +`_installed is undefined` (idempotent), and write `_installed: True` to +`iiab_state.yml` at the END of the play. So they carry the stronger completion marker the +core prefers. No role change needed. A future custom module must match this. + +OPEN (minor): the exact rootfs path the core sees for these files per environment (the +app resolves it via `SystemStateEvaluator.rootfsDir`; the core is already in the box, so +it is a fixed path there): confirm when wiring the registry. + +#### Delete semantics (reclaim space; resolves the Pillar 1 OPEN) + +Scope: delete an ITEM to reclaim space (K2GO-396: "free space without uninstalling the +app"), NOT uninstall a whole app. Module uninstall is a bigger, separate capability: the +box has NO uninstall today (`InstalledModules`: "A module install only ever adds. There is +no uninstall today"), and the install flags are add-only. So Code on the Go (a whole static +blob with no sub-items) has no item-level delete here; dropping it would be a module +uninstall, out of scope. + +Decision: DELEGATE to the app's own delete, never drive the app DB + files directly. The +app owns its index and storage invariants; the core calls the app's delete and the next +list() reflects it (no separate state to track : one owner). Confirmed per app: +- Kolibri: delegate to Kolibri's own delete task `kolibri.core.content.tasks.deletechannel` + (dash-node already does this: `sockets/kolibri.query.ts` deleteChannel). MANDATORY, not a + preference: Kolibri content files are shared / content-addressed across channels, so a + direct rm of a channel's files would corrupt other channels. The task also does the + shared-file refcounting, so it frees only files no other channel still uses. +- Kiwix: remove the .zim file, then re-run the indexer `iiab-make-kiwix-lib` to drop the + `library.xml` entry (reuse the KiwixManageController / ADFA-5004 path; mind the reindex + race noted in `routes.ts`). A ZIM is self-contained, so the file is the unit. +- Calibre: `calibredb` is ABSENT on the box (recon), so delete cannot shell out to it. Use + Calibre-Web's own remove (its API / management path), which drops the book row plus its FK + rows (data, authors link, identifiers) and its files; a direct DB + dir removal is the + careful fallback only if no Calibre-Web path exists. Confirm when built. (Export has the + same constraint: with no calibredb, the core tars the book's own directory from + `books.path`, which is what Calibre reads back.) + +Reclaimed bytes: report what the APP actually freed, not the item's nominal size. Deleting +a Kolibri channel that shares files with another frees only the non-shared bytes, so an +item's `bytes` (available size) is an upper bound on what a delete reclaims, not the amount. + +Lifecycle / long-op: Kolibri delete runs as an async Kolibri TASK, so delete is a long-ish +op. The FRONT owns its progress and its confirmation (destructive and outward-facing: the +front confirms with the user first, the core executes, one item at a time), and the core +reports its in-box sub-progress, the same shape as export / import (D1c). + +### Pillar 2: output (export + share) + +The content manager UI (served by the box, shown by the app / IIAB web): content +by category, disk usage, per-item selection. Two outputs over the SAME manifest: +- Export offline: write the archive (manifest + content + structure) to an + external disk. FIRST GOAL. +- Selective share: rsync-style like clone, but per selected item. The QR carries + the preflight manifest (what the receiver can and cannot import, and what it + already has) instead of only credentials. OPEN: QR size: a full index may not + fit in a QR, so the QR likely carries a pointer/summary and the sender serves + the full manifest. DETAILED CONTRACT: see "Pillar Share" below. + +### Pillar 3: input (import) + +Read an archive or a disk, validate the manifest against the INSTALLED apps, +import what is importable (per app), report/skip the rest ("you do not have the +app for this content"), and dedup against what already exists. Partial import is +first-class. + +### Cross-cutting concerns (not pillars, but they thread through all) + +- Space / disparate storage (the hard problem): preflight the space, import + partially, keep per-item granularity. This is the reason the subsystem exists. +- Unified transport: offline disk and rsync/QR should share ONE manifest and ONE + validation path; only the carrier changes. +- Environment parity (K2Go + IIAB): a consequence of decision 1. The core must + not assume Android. + +## Offline carrier and container (fragility vs practicality) + +The worry: a 20 GB monolith on a USB needs 20 GB just to LAND, then more to import, so +importing 10 GB of it needs ~30 GB. That breaks the whole granular goal. + +Root cause (the key fact): the "need the whole thing locally to decide or extract" cost +comes from ONE property, not from "a single file": a SOLID / opaque container that must +be fully copied local and read sequentially. Two independent properties remove it: +- Read / extract IN PLACE from the medium (USB / OTG), never copy the archive local + first: then you only write the items you import, never the whole archive. +- Random access, not sequential: a format with a directory lets you read the index + cheaply and extract only selected members. + - tar / tar.gz = sequential, `.gz` is solid: to reach the last member you process the + whole stream. This is the trap. + - zip = a CENTRAL DIRECTORY at the end: seek to it (KB), read the full listing without + the body, then seek to and extract ONLY the picked members, each with its own CRC32. + Our content (zim / mp4 / epub / pdf / webp) is ALREADY compressed, so zip STORE mode + (no compression) is zero CPU on a Pi 3B and still gives the index + per-entry CRC32. + +So neither extreme is forced. The format spectrum, ONE manifest underneath: +- v1 DEFAULT: open directory (BagIt-style). `manifest.json` (identity + per-item size + + upstream-hash-if-present) + one content file per item in a readable layout. Browsable, + usable without the app (open the mp4 / epub directly), per-item import (copy only what + you pick: no 30 GB), validate-and-skip on tamper. Carrier = USB / OTG read in place. +- OPTION (later): a single STORE-zip (manifest as the first entry + central directory). + For the user who wants one tamper-resistant file: still index-readable and + partial-extract-in-place, so it does NOT bring back the 30 GB cost. Same manifest. +- OPTION (later): import-from-URL. Any static HTTP server with Range support (Python + `http.server` has it since 3.7): fetch the small manifest, then HTTP Range GET only the + selected items. The network twin of zip random access / reading loose files in place. + (Between two phones the QR / rsync path wins; URL import shines for a laptop or a USB + server holding the content.) + +We are NOT reinventing: "loose content files + a manifest with per-file checksums, +validate-and-skip" is the mainstream pattern. +- BagIt (LoC RFC 8493): a spec for exactly offline file transfer with a manifest + + checksums for fixity (`bagit.txt` + `manifest-sha256.txt` + `data/`). Almost our design. +- APT / Debian repos: tiny `Release` (hashes + sizes) -> `Packages` (per-.deb hashes + + sizes) -> loose `.deb` in `pool/`. Read indices, validate, fetch only what you pick. +- OCI image layout (docker save to a dir): `index.json` + content-addressed + `blobs/sha256/` loose files; copy around offline, pull partially; the + content-address = automatic dedup + integrity + self-detecting moves. +- git-annex: built for large files over sneakernet (USB transport drives are first-class): + a light manifest + checksum-named content, sync only selected items, integrity by name. +- OPDS, and Kiwix (`library.xml` + loose `.zim`) and Calibre (`metadata.db` + loose + `Author/Title/` files): the apps we integrate ALREADY use manifest + loose files. + +Fragility mitigations (cheap, standard, in v1): +- The manifest is the authority: a missing / renamed / changed item is reported and + SKIPPED, never a hard error (the rest import fine). +- Per-item size + upstream-hash-if-present = a fixity check at import, zero on-device + compute by default (BagIt-style). +- Name each content file with its native id / version (not only a friendly name), so a + careless rename cannot silently swap one item for another and a move is detectable. +- Optional: a checksum of the manifest itself (tiny) to detect an edited manifest. + +v1 accepts limits and unfolds (endorsed): +- v1: open directory + USB / OTG + offline import + delete-to-reclaim; soft dedup; no + on-device hashing. This alone fully solves the 30 GB problem (item-atomic copy). +- v1.x: the store-zip container option; import-from-URL (Range); strict-mode hashing; + selective network share (QR / rsync per item). + +## Resilience: pause / resume / retry (sender and receiver) + +A phone transfer is interrupted constantly: the battery dies, the screen locks (Doze), +the user switches network (join a hotspot, then a 2.4 GHz AP, then find a 5 GHz AP to go +faster), the app is backgrounded or killed. BOTH ends must pause, switch, and continue +where they left off : never restart from zero. This is FRONT-owned (D1c); the core only +serves restartable primitives. + +Advantage we have: WE are the source. The files already exist on the sender (we do not +fetch them from the internet), so there is always something to resume from : no upstream +to re-request, no re-download. + +Per-item op-state, persisted by the FRONT (durable across process death, like the install +banking K2GO-408 / 187): + PENDING -> COPYING(offset) -> COPIED -> IMPORTING -> DONE | FAILED(retryable) +- The ITEM is the unit of resume: an interrupted item is retried on its own; the rest are + untouched. v1 does per-item retry (re-do the interrupted item). +- Within-item byte resume (continue a half-copied big file from its offset) uses the + mechanisms we already run: rsync partial-tree resume (clone), aria2 `--continue`, HTTP + Range. v1.x adds this so a 4 GB item does not restart from zero. + +Sender side (export / share): `GET /v1/export/item` is restartable (Range), so the front +pauses, drops the carrier, switches network, and resumes the SAME item from its offset. +The core holds no transfer state; the front does. + +Receiver side (import): a partially-copied item is re-copied (Range / rsync) then handed +to the core; ingest of ONE item is idempotent (re-import replaces / dedups, never +double-counts). The front banks the queue, so a battery death resumes the queue, not +restarts it. + +Network change mid-transfer: the front re-establishes the carrier and resumes from the +last offset; it must NOT silently restart or (for ADB) re-pair : K2GO-173 is the +cautionary precedent (reconnect silently re-triggered pairing). + +Precedents reused (see "Recon: hybrid precedents"): the foreground service (Clone +WatchdogService), durable app-side progress (DashboardRebuildService), per-item retry + +resume (RestContentClient, aria2). We are not inventing a resume engine; we drive the +existing ones per item. + +## Decisions status (pointers; detail lives in ONE place above) + +To avoid a duplicate-truth in this doc, each decision's detail lives in one section: +- Core in the box + D1a-d: "Decision 1" + the open-questions log. ACCEPTED. +- Content identity / dedup: "Identity semantics" (localRef + dedup + idKind). DIRECTION + SET; Calibre residual confirmed when its adapter is built. +- Archive format + offline carrier: "Offline carrier and container". DIRECTION SET + (open directory v1; store-zip + URL carriers later). The ITEM is the atom; a SOLID + monolith is rejected, but a store-zip (random-access directory) is a valid later option. +- Transport unification (one manifest, many carriers): DECIDED (same section). +- Delete semantics: DECIDED. Delegate to each app's own delete (Kolibri deletechannel + task, Kiwix rm + reindex, Calibre-Web remove : calibredb is ABSENT on the box); + item-level only, module uninstall out of scope (box has no uninstall). See "Delete + semantics". +- Pillar 2 export BUILT (fork, commit 777d401c): POST /v1/export/plan + GET + /v1/export/item streaming, per item. Per-item archive shape RESOLVED: Kiwix = the .zim + streamed as-is; Calibre = tar of the book dir (books.path; calibredb absent); Kolibri = + tar of `kolibri manage exportchannel` + `exportcontent` staged output (content-addressed, + so import dedups shared files). Export staging must live on the content volume, not /tmp, + and preflight the BOX free space before staging (code-review finding). Code-review done + (5 findings, fork applying fixes: HTTP/1.1 for chunked, per-member OOM, staging dir, body + cap, e2e test). + +## Build order and the first deliverable + +The first user-visible deliverable is OFFLINE IMPORT (plug an external disk, pick, +import). It sits on top of the whole stack, so the build order is bottom-up: + +1. Pillar 0 contracts (manifest + identity + adapter port + core REST API). +2. Pillar 1 service + the three adapters (read side first: list + identity + size). +3. Pillar 2 export to disk (write the archive) : the other half of the disk round trip. +4. Pillar 3 import from disk (validate + partial + dedup) : the deliverable lights up. +5. Then: selective network share (rsync/QR), delete-to-reclaim, the manager UI polish. + +So "install from offline" is the first thing that works end to end, but only after +the stack beneath it exists. + +## Work plan (session topology) + +A CONFIRMED (contracts first, then 2 forks). B and C kept for context. + +- Option A (CHOSEN): contracts first, then fork. This session owns Pillar 0 + (freeze the contracts + this ADR). Then two forks: Fork-Service (Pillar 1 + the + three adapters) and Fork-Flows (Pillars 2+3 + archive format + manager UX), both + building against the frozen contract, syncing through this session. +- Option B: three parallel forks now (Service / Output / Input) + this session as + integrator. Faster in appearance, but the forks design against a contract that + is not yet agreed, so they diverge. +- Option C: one session, sequential. No divergence, less parallelism. + +## Per-app notes (identity confirmed by recon; ops TO DETAIL) + +- Kolibri: content DB (channels/nodes), import/export already touched in + K2GO-475/476. Identity + read CONFIRMED on the box: `/library/kolibri/db.sqlite3` + (opened read-only), channel_id (32 hex) + integer version, size = SUM of DISTINCT + available local files (a file can hang off several nodes; a flat join double-counts). + The adapter reuses the same source and aggregation dash-node uses + (`static/dashboard/sockets/kolibri.query.ts`): one source of truth. Delete + export: + via Kolibri's own tools vs direct DB (TO DETAIL, Pillar 2/3). +- Kiwix: ZIM files. Identity CONFIRMED: ZIM UUID (book id) + `library.xml`; the dated + filename carries the version axis. Export = the ZIM file; delete = remove the file + + the library entry (the existing KiwixManageController path, ADFA-5004). +- Calibre-Web: NOT book-only : arbitrary media (epub/pdf/image/video/audio), multi- + format per record. Local identity = `metadata.db` `books.uuid` (free, authoritative- + local). Cross-box dedup = SOFT key (title+authors+format+`data.uncompressed_size` + [+ `identifiers` when present]); content hash opt-in / sender-computed only. Read + CONFIRMED on the box: `metadata.db` opened read-only (the + `static/dashboard/sockets/books.query.ts` pattern), reading `books.uuid`, `data.format` + / `data.uncompressed_size`, `authors`, and one `identifiers.val` as the optional + stronger hint. Note our `books_catalog.jsonl` (Get More) carries the weak local `id` + + title/author: that is the download catalog, not the installed-library identity. +- Code on the Go family (code-assets, code-addons): the TRIVIAL case : static files, no + database, no index semantics. Adapter = enumerate the served tree, copy / drop / remove + files. Identity = path + the per-file `.md5` the mirror already ships (reuse, no + compute). Proves the adapter model scales DOWN as well as up (K2GO-494). "Moving files + for the sake of moving files." +- Integrity source CONFIRMED: metalink `.meta4` ships upstream sha256 + size (used + today for rootfs and maps); reuse it in the manifest, do not recompute on-device. + +## Recon: hybrid precedents (why the front co-owns long operations) + +Every heavy or long operation on K2Go has ALREADY been forced out of the pure +in-box / Ansible-role model: the APP had to co-own it, because the backend two +levels down (under proot, sometimes with the system stopped) is blind to the phone +real world and can die without the app noticing. Evidence in the current code: + +- Disk guard (K2GO-386): the in-box healer "dies with the box: only an Android-side + guard, independent of the box, catches" a disk-fill (`diskguard/domain/DiskGuardPolicy.java` + header). The app reads the phone real free space (`StatFs`) and reserves a 2 GiB + op-floor (StorageGuard) before every app-driven heavy op. So the box view of free + space is a fiction; the phone owns the real-space truth. Direct support for D1c. +- Clone (`redesign/CloneFragment.java` + the rsync TransportEngine + WatchdogService): + the APP runs the rsync transport in its own process and holds a FOREGROUND service + to stop Android SIGKILLing rsync (exit 137, the phantom-process killer). The app + builds the QR handshake and is the SENDER. The backend does not move the bytes. +- Maps (K2GO-393 / 394): the heavy base-map (pmtiles) download was pulled OUT of the + Ansible role into the dash-node durable job engine (front-driven); the role in-proot + task only asserts the files are present and skips (`creates:`). The role cannot do a + heavy live download under a stopped system; the front does it, the role finalizes. +- Content download / Get More (`content/RestContentClient.java` + the per-area + foreground download services: Kolibri / Addons / CodeAssets / Forgejo seed): the APP + drives long in-box downloads via a foreground service + POST / poll, with per-item + retry and network-policy gating. The box engine does the fetch / resume / checksum, + but the APP owns the lifecycle, the user-facing progress, and the guards. +- Rebuild (`DashboardRebuildService`, K2GO-383 / 451): the APP is the persistent + progress owner (survives fragment recreation and minimize), drives the notification, + and self-heals rebuild-zombies: the box-side rebuild cannot report durable progress + to the user by itself. + +Design consequence for the core: it must be a set of PRIMITIVES the front drives, not +a self-contained in-box job runner. The content-exchange subsystem reuses these exact +seams (foreground service, phone-disk preflight, app-owned transport, durable +app-side progress) rather than re-inventing an in-box owner that cannot see the phone. + +## References (existing patterns to reuse) + +- Clone / rsync selective share: redesign/CloneFragment, RsyncManager (the + all-or-nothing baseline this generalizes). +- dash-node: the in-box REST service pattern the app already consumes + (static/dashboard). +- Backup / restore + manifest validation: RootfsArchiveValidator, the rootfs + identity manifest (.iiab-rootfs.json, docs/ROOTFS_MANIFEST.md) : precedent for a + manifest-validated archive. +- Content download durable job engine (RestContentClient, the dash-node jobs) : + how the app drives long in-box operations. + +## Pillar 4: the K2Go front (APK) and the content manager + +The backend (Pillars 0-3) is in main. The user-facing deliverable of K2GO-427 lives in the +APK: one content MANAGER with per-item actions, reusing the Clone / Get More lessons. + +### The manager (gestor): the home +Content grouped by app / category, each item with its size; a total used and the FREE disk; +per-item multi-select. From a selection the user ACTS. One surface: export / import / share / +delete are actions on it, not separate screens. + +### Actions (the modes) +- EXPORT (to a connected drive): write the OPEN-format bundle (manifest + one archive per + item) to a USB / OTG drive. Open on purpose: the content stays browsable, not a black box. + Core: POST /v1/export/plan + GET /v1/export/item; the app owns the disk IO, the foreground + service and the phone-disk preflight. +- IMPORT (from a connected drive): read the drive's manifest, show per item importable / + dedup verdict / NO_APP, select, import (partial first-class). The receiver half of export. + Core: POST /v1/import/validate + /v1/import/item. +- SHARE (phone to phone): clone-like. A QR handshake carries the preflight manifest (what the + receiver can / cannot import, what it already has), then a per-item rsync-style transfer. + Granular Clone. Reuses the Clone transport + QR + the receiver's /v1/dedup/check. +- DELETE (reclaim space): drop selected items; the core delegates to each app's own delete. + +### Shared infra (reuse, do not reinvent) +Foreground service (the Clone WatchdogService pattern), a per-item queue with pause / resume / +retry (the resilience design), progress UI (bar, %, ETA, bytes, speed), the PHONE-disk +preflight (StatFs, app-side), the QR handshake, Material 3, and the layered architecture. +These are the Clone / Get More / rebuild lessons, applied per item. + +### Android storage constraints (recon: what we can and cannot read / write) +- Reading disk SIZE is NOT restricted. `storage/StorageProbe` (StatFs getAvailableBytes / + getTotalBytes) is the app's shared, in-production free/total probe (ADFA-5105, used by the + ZIM / Maps / Kolibri screens). The manager's FREE-disk figure comes from StorageProbe, not + from the core: on K2Go the app owns the phone-disk truth (D1c). The core's `/v1/storage` + gives content totals (usedByContent, perApp) and the box free view (authoritative for IIAB). +- The REAL restriction is external-DRIVE (USB / OTG) access for export / import, and it is + FLAVOR-dependent: the app declares MANAGE_EXTERNAL_STORAGE (R+) / WRITE_EXTERNAL_STORAGE + (<=29), but the PLAY flavor REMOVES MANAGE_EXTERNAL_STORAGE (K2GO-442: + `permissions/StoragePermission.isRequired` self-corrects, "treat storage as optional"); + standard / fdroid keep it. And a USB / OTG volume on modern Android is reliably reached only + through SAF (the system document picker), even with broad access. +- Decision (OPEN, Luis's call): the export / import drive IO uses SAF (DocumentFile + an + ACTION_OPEN_DOCUMENT_TREE picker) : portable across ALL flavors incl. Play, and the reliable + path to a USB drive. The alternative (raw File paths on the broad-access flavors only) is + simpler but excludes Play and is unreliable for USB OTG. Lean: SAF. This shapes the Export / + Import slices (2 / 3), not the manager (slice 1). On-device SAF-sees-the-USB check is a + slice-2 verification (needs a physical drive on the 7T). + +### First pass (proposal): the DISK round-trip + the manager +The K2GO-427 story is "install content from a connected drive", so the first pass is the DISK +carrier end to end: Manager (list + sizes + free disk + multi-select) + EXPORT-to-disk + +IMPORT-from-disk + DELETE-to-reclaim. DEFER SHARE-over-QR (the network carrier) to a second +pass: same manager, a different carrier. (A tighter first pass is Manager + Import + Delete, +with Export right after. Luis's call.) + +### Core gaps the front reveals (fill in the core FIRST) +Two core capabilities the manager needs that do not exist yet: +1. DELETE endpoint: `DELETE /v1/content/{app}/{localRef}` -> { ok, freedBytes }. Designed + (delegate to the app's own delete: see "Delete semantics"), not coded. +2. STORAGE-STATUS endpoint: `GET /v1/storage` -> { volume, totalBytes, freeBytes, + usedByContentBytes, perApp: [{app, bytes}] }. The core has per-item sizes but no storage + summary (box disk via shutil.disk_usage; content totals from the adapters). The PHONE's own + free disk stays app-side (StatFs). +dash-node needs NO change for these: its relay is generic (`apiRouter.use('/v1', conexProxy)`), +so any new `/v1/*` core endpoint is auto-exposed. dash-node changes only if a path needs +non-relay logic (none here). + +### Import flow design (the receiver slice, mirror of Export) +The receiver half of the disk round-trip. Reuses Export's infra: the SAF carrier port (the +reader is the mirror of Export's writer), the foreground service, the progress UI. No new +permission (SAF). Core endpoints (all exist): POST /v1/import/validate, POST /v1/dedup/check, +POST /v1/import/item, and DELETE (for the replace-older flow). + +1. SAF: an ACTION_OPEN_DOCUMENT_TREE picker -> the user selects the drive/folder holding the + bundle -> DocumentFile read. +2. Read the top-level manifest.json from the tree (a few KB: the collection-open point is you + read the manifest WITHOUT ingesting the payloads). Parse items[]. +3. Validate against THIS box: POST /v1/import/validate {manifest} -> per item importable | + NO_APP | dedup verdict (ABSENT/SAME_VERSION/OLDER_HERE/NEWER_HERE + idKind). The UI groups + the manifest items and marks each: "can import", "already have (same)", "newer available", + "older than yours", "no app for this (skipped)". A soft (Calibre) verdict is advisory. +4. The user selects what to import (partial first-class). PHONE-DISK PREFLIGHT here is the + crux of the disparate-storage problem: StorageProbe (StatFs) gives the phone's free space; + the UI shows "selected total X / free on device Y" and warns / disables when it will not + fit, so the user picks what fits (a small phone takes a subset of a big drive). +5. Import the selected items, per item (the carrier streams FROM the drive; the core ingests): + - SAF-read the item's archive and write it to the box-visible import-staging path the core + reads (registry.import_root, app-private storage the app can write), then + POST /v1/import/item {app, entry, payloadPath} -> the core ingests it (delegating to the + app's own import: Kolibri importchannel, Kiwix place + reindex, Calibre-Web upload). + Remove the staging copy after ingest. Per item -> partial + retry; skip items the user + deselected or that validate SAME_VERSION. + - The "already have / newer / older" dynamic (Luis): SAME -> skip; NEWER incoming over an + OLDER_HERE -> offer "replace" (import the new, optionally DELETE the old via the delete + endpoint, the user's choice: keeping both is allowed, it costs double space); OLDER + incoming -> default skip with a "you already have a newer one" note. +6. A FOREGROUND service owns the queue + progress (reuse Export's), with per-item + pause/resume/retry (receiver-side resilience: battery dies -> resume where it left off, not + restart). ENOSPC on a copy/ingest -> stop/skip that item gracefully. + +Note: on K2Go the import writes into the phone storage (the box is a subtree of it), so the +box-side import preflight (registry.import_root) and the app's StorageProbe are the SAME +physical disk; the app's StorageProbe is authoritative (Android's real view). + +### Build topology (contracts first, then fork : same as the backend) +- Fork-core (now): implement the two endpoints above against these shapes. Independent of the + UI scope; unblocks the app. +- Fork-app (after the first-pass scope is set): the manager + the disk actions, against the + frozen core contract. Android, layered, Material 3. +- l10n gate (Luis, 2026-10-22): the manager is delivered across slices whose strings still + shift as the actions land, so UI strings stay parked in `strings_untranslated.xml` during + the slices and migrate to `strings.xml` (all 33 locales) IN THE CLOSING PR of the feature, + not per slice. The closing slice's checklist: migrate the strings, verify light/dark + on-device, and the 2nd code-review pass. + +## Open questions log + +- Decision 1 (core in the box): ACCEPTED. +- D1a (core runtime = lean Python): ACCEPTED. +- D1b (app -> dash-node -> core path): ACCEPTED. +- D1c (long-op orchestration): ACCEPTED. Shared / front-owned; the core owns only + in-box primitives, the front (app / Console) is the sender and the guard. +- D1d (core update): ACCEPTED. Dual: Ansible-role baseline + dash-node-style live + self-update. +- Dedup identity per app: DIRECTION SET. Two handles: `localRef` (export/delete, free) + + `dedup` (cross-box). Kiwix + Kolibri authoritative. Calibre (arbitrary media, no + ISBN): localRef = books.uuid; dedup = SOFT key (title+authors+format+size) + hash + opt-in. CONFIRMED on the box for all three (Kiwix/Kolibri/Calibre); field names and + soft-key normalization implemented and tested. +- Pillar 1 read side BUILT (fork, feat/K2GO-427-conex-core-read): Kiwix + Kolibri + Calibre + adapters, 43 tests, verified on-device. Transitional note: the Kolibri/Calibre adapters + reuse dash-node's own DB queries' shape (kolibri.query.ts / books.query.ts), so the + listing SQL now exists in both dash-node (TS) and the core (Python). Both read the one + authoritative DB, so it is not a duplicate SOURCE, but it is a duplicated query rule; the + end-state (D1b: the core owns "what content exists", dash-node is a thin relay) collapses + it by routing dash-node's content listing through the core. Track for the consolidation. +- Archive format: DIRECTION SET (collection-open, item-atomic: one manifest + one + archive per item). Per-item shape RESOLVED in Pillar 2: Kiwix = the .zim as-is; Calibre = + tar of the book dir; Kolibri = tar of exportchannel+exportcontent output. +- Pillar 0 FROZEN (2026-10-22): identity envelope (localRef + dedup/idKind), manifest, + adapter port (inner), core REST API (outer). Later pillars build against it (topology A). +- Session topology: A CONFIRMED (contracts first, then 2 forks). Fork 1 (read side) + building on feat/K2GO-427-conex-core-read. +- Installed-apps source (review #2): DECIDED. The core reads IIAB's own rootfs flag files + (`iiab_state.yml` completion, fallback `local_vars.yml` intent), the same single source + the app uses (ADFA-5104 / K2GO-393); a CORE decision serving both dash-node and IIAB. + Read side slice ships it as a documented startup snapshot; the live re-read is the next + Pillar 1 step. See "Installed-apps source". +- Does the QR carry a manifest pointer/summary, with the sender serving the full + index? + +## K2GO-507: export as a liveness-paced long operation (prepare / poll / fetch) + +Problem (device pass, A16, 2026-10-23): `GET /v1/export/item` calls the adapter's +`open_export()` and the server sends the 200 + chunked headers only AFTER it returns. +For Kolibri, `open_export` stages the whole channel synchronously (the Kolibri export +CLI into a `cxkolibri-*` temp) before returning, so a 2.8 GB channel takes minutes; the +app client (`openExportItem`, 20 s read timeout) times out in `getResponseCode()`. Kiwix +(`file_stream`) and Calibre (`tar_stream` of the book dir) stream directly, fast first +byte, so only large staged items fail. + +Decision: replace the single blocking export-item request with a long operation, uniform +for all apps (no per-app special case, no alias: nobody uses the feature yet). The front +owns the sequence (D1c); the core exposes primitives. The timeout becomes a client +inactivity (stall) timeout, not a wall clock: the op is alive while it makes progress. + +Contracts (outer REST, through dash-node /v1 relay): +- POST /v1/export/prepare {app, localRef} + -> 200 {opId, state, totalBytes}. state is "staging" (work started in a background + thread) or "ready" (nothing to stage: Kiwix/Calibre). 404 item absent/not exportable; + 507 when the staging disk cannot hold totalBytes (reuse has_room). +- GET /v1/export/status/{opId} + -> 200 {state, stagedBytes, totalBytes, error?}. state: staging | ready | error. + stagedBytes = du of the op temp: the liveness heartbeat AND the prepare-phase progress. + For "ready", stagedBytes == totalBytes. 404 unknown opId (swept) -> client re-prepares. +- GET /v1/export/fetch/{opId} + -> 200 chunked octet stream of the staged tar (Kolibri) or the direct stream + (Kiwix/Calibre). Content-Disposition carries the suggested name. on-close removes the + op temp and drops the op. 409 when state != ready. +- DELETE /v1/export/{opId} + -> 200 {ok}. Client abort: drop the op and remove its temp. + +Op lifecycle (one owner per transition; the hard part, per the design-coherence rule): +- CREATE: prepare allocates {opId -> temp, state, totalBytes, error, lastSeen} in an + in-memory registry and starts the staging thread (or marks ready). +- CLEAR, exactly one of: (1) fetch on-close (normal completion); (2) DELETE (client + abort); (3) a TTL sweep of ops whose lastSeen (bumped by status/fetch) exceeds a + timeout, so an abandoned prepare never orphans a multi-GB temp; (4) a STARTUP sweep of + {staging_dir}/cxkolibri-* (and stray op temps) on conex-core start, for ops lost to a + process death. +- The registry is in-memory; a conex-core restart loses ops (acceptable: the front + re-prepares; the startup sweep reclaims orphan temps). Matches the stateless core and + avoids a durable op-state. One source of truth: the filesystem for bytes, the registry + for the in-flight set. + +Single source of truth: op state + stagedBytes are owned by the core. The app's per-item +phase (prepare | write) is a PROJECTION read from the status poll; the app never stores a +competing "where is this export". No duplicate truth. + +Resilience: v1 recovers by RE-PREPARE (not re-attach). On app death or core restart the +opId is lost; the client prepares a fresh op and the old is swept. A later enhancement can +persist the opId and re-attach (the RestContentClient pattern: POST + poll, re-findable +job); not required for v1. + +App side: ConexCoreDataSource gets postExportPrepare, getExportStatus, openExportFetch (the +fetch uses a STALL read timeout that resets on each read, not a 20 s wall clock; the +foreground service + Cancel are the control). drain() becomes: per item, prepare -> poll +status (feed the prepare-phase bar + liveness) until ready/error -> fetch -> copy +(write-phase bar). ExportSession/ExportJob gain a per-item PHASE and a prepare percent +(stagedBytes/totalBytes); ExportProgressFragment shows the two-phase per-item bar (~50/50). + +conex-core version (folded in, this ticket already changes the core): add +conex_core.__version__, expose GET /v1/version {version} (and optionally in /v1/apps), and +have the conex_core role write it to the [conex_core] section of iiab.ini, mirroring dash-node's +package.json version and /system/version, so a box can report which core it runs. + +Reuse: RestContentClient POST+poll (content/RestContentClient, ADFA-4893/4897) is the +client precedent; tar_stream/file_stream (export.py) and _stage_export (kolibri.py) are +unchanged primitives the fetch reuses. Build order: conex-core endpoints + registry + sweep + +version first, then the app client + two-phase progress; integration verified on the A16 +(export a 2.8 GB Kolibri channel end to end). + +## Pillar Share: granular wireless send (network carrier + QR handshake) + +Status: DESIGN (deferred from v1; the v1.x "selective network share" in the carrier spectrum +above). Anchor ticket K2GO-376 (granular share). The Share button exists in the manager, +disabled ("not available yet"). This section is its contract; no new ticket from this ADR. + +The key fact (already decided under cross-cutting "unified transport"): Share is the NETWORK +TWIN of Export. Same bundle, same manifest, same validation and import path: ONLY the carrier +changes. The two carrier ports were built for exactly this (ExportCarrier / ImportCarrier +doc: "a future network carrier reuses the same port, so it is never hard-wired into the +sequence"). So Share adds a transport and a pairing step, NOT content logic. + +### Topology (ACCEPTED): the receiver PULLS from the sender + +The bundle format is already pull-shaped: import reads a manifest, then streams only the +payloads it picks. So Share does not push: the SENDER exposes its selected items, the +RECEIVER runs the existing import flow against a network source. Reuse, do not reinvent: the +sender's in-box conex-core already serves the export long operation (POST /v1/export/prepare, +GET /v1/export/status, GET /v1/export/fetch: K2GO-507) plus the manifest; the receiver +fetches those over the WLAN instead of over the loopback relay. No new server: the sender's +conex-core is the share server, scoped to the shared item set; the receiver's app points its +import at the sender's address. + +### The carriers + +- Sender: nothing new on the content side. It selects items (the same selection as Export) + and makes them fetchable: its conex-core already stages and streams per item (prepare / poll / + fetch). Share only opens that conex-core to the paired receiver on the hotspot / WLAN, scoped + to the shared items plus the session token, never the whole API. +- Receiver: a network ImportCarrier (the mirror of SafImportCarrier): readManifest() and + openPayload(name) become HTTP GETs (with Range) against the sender, instead of SAF reads. + Everything downstream is unchanged: validate against the INSTALLED apps, dedup, partial + import, stage to /library/.conex-staging, POST /v1/import/item. + +### Pairing (ACCEPTED): reuse the sync QR handshake + +The box-to-box sync handshake already exists and is hardened: SyncHandshakeHelper encodes +host / port / credentials in a QR, validated by SyncCredentialValidator (the security slice +already shipped). Share reuses it: +- The sender shows a QR carrying: its hotspot / WLAN address + port, a short-lived session + token, and a POINTER to the bundle manifest (NOT the full manifest: a full index does not + fit in a QR, the OPEN item from Pillar 2, resolved here by pointer + served manifest). +- The receiver scans, joins the sender's hotspot / WLAN (Connect's medium), fetches the + manifest over HTTP, then proceeds with the existing import review and flow. + +### Pairing is a pre-step, not the contract (ACCEPTED: v1 UX phone-to-phone, contract peer-agnostic) + +Establishing the network (who is on the same LAN) is SEPARATE from the transfer. An IIAB box on a +wired LAN already has the network, so it just transfers; two phones must first bring one up (hotspot / +wifi). That pre-step is Android's job and REUSES Connect / Clone's existing join flows (Connect is 2 +steps; Clone is 2 or 3, depending on wifi vs hotspot). In every one of those flows the LAST step is the +same: hand over the info to actually access (Connect: reach the service; Clone: the QR to pull). The +Share contract IS exactly that last step: the final QR carries the /v1/share address + token + manifest +pointer, and the pull follows. Everything before it (bringing up the network) is outside the conex core, +the way plugging the Ethernet cable is not part of content exchange. + +Consequences: +- The /v1/share contract + the network ImportCarrier are PEER-AGNOSTIC (plain HTTP + manifest + token). + v1 only builds and tests the phone-to-phone UX; a laptop / server (IIAB box as sender, or URL import + as receiver, "URL import shines for a laptop" above) is a later carrier, NOT excluded, and needs no + core change. +- Step 1 (the /v1/share contract + network carrier) is pure core + loopback and needs NO + network-establishment work. The Android pairing is a separate PRESENTATION slice that reuses + Connect / Clone and owns the "do not leave the hotspot on" lifecycle. The core never toggles the + hotspot. + +### Free space lives on the RECEIVER (ACCEPTED) + +The relevant free space for a send is the RECEIVER's, not the sender's. After fetching the +manifest (which carries per-item sizes), the receiver runs its OWN preflight with pieces that +already exist: ImportPreflight.fits(selectedBytes, StorageProbe.freeBytes(receiver)) plus the +installed-apps validation and dedup. The sender never needs the receiver's disk figure: the +receiver decides what it can take, item by item (partial is first-class). This is the "space +in Share" answer: the receiver-side import preflight on the fetched manifest. (Related but +separate gap for Export: the export flow does not preflight the DESTINATION drive's free +space today, it only reacts to ENOSPC mid-write. A destination preflight is a small +standalone improvement, not part of Share.) + +### Resilience (reuse) + +Per-item pause / resume / retry is already the model (see Resilience above), and the sender +is the source, so a send is always resumable: a dropped WLAN connection re-fetches the failed +item (HTTP Range or re-request the item), the rest stand. The receiver's import is already +per-item and partial. + +### Reuse ledger (do not reinvent) + +- SyncHandshakeHelper + SyncCredentialValidator: QR pairing + validation. +- Connect: the hotspot / WLAN as the transport medium. +- conex-core /v1/export/* (K2GO-507) + the manifest: the sender serves these; no new server. +- ContentImportService + the import flow: unchanged on the receiver. +- StorageProbe + ImportPreflight: the receiver-side free-space preflight. +- Clone is the whole-box twin; Share is Clone scoped to the selected items. + +### Gate / token lifecycle (closing the two-party pincer): ACCEPTED + +The failure to avoid (Clone's gap, confirmed in the next subsection): the host exposure is "open +until the host user closes it", with no deadline and no signal from the remote. One jaw only: if the +user walks away, it stays open. Share inverts the default: the gate EXPIRES by default, and either +party can close it. + +The gate separates TWO lifetimes (the refinement that makes large, resumable transfers work). +Collapsing them into one idle TTL was the first-cut error: a multi-hour transfer that pauses would +be reclaimed mid-way. + +CONNECTION lifetime (ephemeral, Layer 1): the idle TTL and the in_use pin belong to ONE open stream, +not to the session. A dead socket is reclaimed after the idle TTL (~10 min); a live stream is pinned +so the sweep never drops a temp under it (reuse last_seen / in_use / _sweep_expired from the export +op). Per-item resume = re-fetch that item (the export op re-stages). The connection carries the +token; the session never learns the client's IP, so a reconnect from a new IP (DHCP, AP change) with +the same token is just a new connection on the same session. IP-resilience is therefore free in the +core; re-finding the host at a new IP is the app's job (re-emit the QR with the current address). + +SESSION lifetime (durable, Layer 2): the session holds the token (the durable transfer handle + +resume code), the scoped item set (the "selection") and the per-item delivered state. It does NOT +expire on idle. It lives until ONE of: + : COMPLETE : the receiver signals it has every item (after a quick final verification) : the clean + success close; the host then releases the protection. + : CANCEL (DELETE, by host OR receiver) : abort; drops the protection and any in-flight stream. The + sender's foreground share service calls DELETE in onDestroy (the app-side closer). + : a long SAFETY CAP (hours), so a truly abandoned session is eventually reclaimed : NOT the + ~10-minute connection idle. + : process death : the session is in-memory (v1 DECISION), so a restart loses it and sweep_startup + reclaims the child temps on boot. + +v1 storage DECISION: in-memory + a long safety cap. The session metadata is tiny (token, a list of +(app, local_ref), a delivered set); the payloads stream from on-disk export temps, so there is NO +OOM risk from the registry. Persisting the selection to a disk file is a v1.x move purely for +cross-restart resume durability, to revisit by real-world performance, not by memory pressure. + +"Protection" = the Layer-2 session being open: while it is open, the token holder can fetch the +selection; CANCEL closes Layer 2 and with it the transfer. Resume hours later = the same token: the +host re-emits the QR (current address + same token) and the receiver continues, skipping items +already delivered. + +delivered is set ONLY on a CLEAN item close (full byte count), never on a mid-stream drop, so a +dropped item stays pending and resumes; the receiver drives the close with COMPLETE, not an implicit +drop-based auto-close. The manifest surfaces per-item delivered as an ADVISORY hint (progress + +what the sender streamed cleanly): it is NOT the receiver's resume authority. The receiver skips an +item only when its OWN persisted state has it, since the sender marking an item delivered (full body +written to the socket) does not prove the receiver saved it. This matters for the step-2 receiver +fork, not for the loopback core slice. + +The two-party pincer still closes (no actor's absence leaves it open): the receiver closes it +(COMPLETE or CANCEL), the host closes it (CANCEL / the foreground service onDestroy), and the host +self-bounds it (the safety cap + startup sweep). The connection idle TTL bounds a dead socket +independently. The token is bound to the session (revoking the session revokes the token) and scoped +to the item set, so there is no separate credential lifecycle to leak (the leak Clone mitigates with +RsyncManager.stop() clearing the secrets file). Default = closed; staying open requires an active +session, bounded by COMPLETE / CANCEL / cap. + +Ownership (app vs cx): the CORE owns the durable session (token + selection + delivered state, +IP-agnostic), the connection idle TTL, and the endpoints (create / manifest / fetch-with-resume / +complete / cancel / the safety cap). The APP owns the pairing (hotspot / wifi + QR), re-discovering +the host at a new IP, re-emitting the QR with the current address + same token, the background +foreground service + the client retry loop, and the final user-facing "done" confirm. + +### Why the pincer closes here but not in Clone (transport paradigms): analysis + +The root cause of Clone's gap is the TRANSPORT, not an oversight. Two paradigms: + +- Machine OFF (Clone): CloneFragment stops the server (stopApkServer, ServerController) and clones the + whole system into a static copy, so the live services are DOWN. The only transport available is a + STANDALONE static binary (rsync daemon), built by .github/workflows/build-static-binaries.yml and + pinned in controller/binary_version.txt. That binary is MUTE on the host side: RsyncManager + .startServer just launches `rsync --daemon` and returns a boolean, with no listener and no + onComplete. The client side DOES talk (startClient parses rsync stdout: onProgress / onComplete / + onError), so the RECEIVER knows when it finished, but the HOST never learns a client finished. One + blind jaw: the pincer cannot close without the user. + +- Machine ON (Share): selected-content share does NOT stop the system, so the live core (conex-core) is + available as a CONVERSATIONAL transport. conex-core already computes the total up front (prepare -> + totalBytes), streams per item, and knows when each fetch closes (on_close). So it can signal done + and close the gate itself. The pincer closes BY DESIGN, with the primitives already hardened for the + export op (K2GO-507). + +So the decision for Share is system-level (conex-core as the talking transport), not the APK-level +mute-binary model. The APK still owns UX, battery and network (foreground service, wake / wifi locks), +but delegates the transport-that-talks to the core. Clone (whole-system, machine off) cannot use this: +conex-core is down. Closing Clone's pincer would need to give the standalone daemon a VOICE: a wrapper / +sidecar that watches the daemon's connections and reports completion to the APK. That is a separate +effort (its own ticket), not part of Share; it is noted as a Clone follow-up (the +exposure-until-user-stops gap is real but out of scope here). + +### Readiness: what is pinned vs what we discover while building + +Method (endorsed, and already proven by v1 + K2GO-507): freeze the contracts that are expensive to +retrofit (the "shared contracts first" discipline of Pillar 0), then implement incrementally and let +code-review + on-device validation catch the rest. The test for "pin it first" is the design-coherence +rule: a missing FACT / OWNER / LIFECYCLE is expensive to retrofit (pin it); a missing CASE is cheap to +patch in review (discover it). + +Pinned (ACCEPTED, safe to build against): +- Bundle format + manifest + identity (built + validated for disk). +- Carrier seam (ExportCarrier / ImportCarrier: "a future network carrier reuses the same port"). +- Import flow (validate vs installed apps, dedup, partial, stage, /v1/import/item: built + validated). +- Topology: receiver pulls from the sender's conex-core. +- Gate / token lifecycle (the five closers + session-bound token). +- Free-space preflight on the receiver (ImportPreflight + StorageProbe on the fetched manifest). + +Open but DISCOVERABLE while building (cases, not facts: do NOT block): +- QR payload exact shape (pointer + address + token) and size limits. +- Per-item retry / re-request over a flaky WLAN (reuse the export stall timeout + resume). +- Progress UI reuse (the two-phase bar already exists). + +Open and BLOCKING-IF-WRONG (pin with a one-line decision before the slice that touches them): +- Trust on the local link: token scope enforcement + plain HTTP vs TLS on the hotspot. Pin before the + gate is exposed off loopback. +- WLAN on_close reliability: confirm the server _stream finally fires on a remote client drop (not + only loopback), or the in_use pin can stick. Confirm before relying on it. +- Hotspot ownership: Connect owns it; Share must borrow and not leave it on. Pin before Share toggles + the hotspot. + +Build order (contracts first, then fork: same as the backend and K2GO-507): +1. Shared contract: the /v1/share gate (scoped to item set + token, the five closers) in conex-core, plus + the network ImportCarrier interface. Merge this first. +2. Fork: sender (expose selection via the gate) and receiver (network carrier + QR scan + import), + each built against the frozen contract. +3. Resolve the three blocking-if-wrong items as their slice is reached. + +Recommendation: we have the primary directives. Start implementing step 1 (the shared contract) once +the three blocking items have a one-line decision each; discover the rest in review + on-device. Do +NOT wait to theorize everything: v1 and K2GO-507 show the incremental path with code-review as the net +works. + +### OPEN questions + +(The gate / token lifecycle and the scoped /v1/share gate are RESOLVED above; trust and +WLAN on_close and hotspot ownership are categorized as blocking-if-wrong in Readiness. What +remains genuinely open:) +- Who hosts: box-to-box (one box's hotspot) vs phone-to-phone (one phone's hotspot serving + its box). Clone's transport choices inform this. +- Discovery: QR only (v1.x) vs mDNS / zeroconf later. + +### Scope + +v1.x, after the disk round-trip (done). Ties to K2GO-376. When Share is scheduled, it reuses +this section as its contract. + +### Amendment K2GO-577: rsync transport, no temp copies, resume after a drop (ACCEPTED, Nov 2026) + +Status: ACCEPTED. ZIM shipped in conex-core 0.4.0 (PR 1); Kolibri is PR 2. This amendment +supersedes two lines above: "openPayload ... HTTP GETs (with Range)" in "The carriers" and the +re-fetch model in "Resilience (reuse)". The gate, the token, the two lifetimes and the five +closers are unchanged. + +Problem (measured, not assumed): +- The HTTP fetch stages a full copy before the import: a ZIM is written twice, a Kolibri channel + up to three times (sender export temp, receiver tar, receiver extract, then the Kolibri copy). +- A drop restarts the item from zero. On a phone, a multi-GB item needs the free space of two + or three copies and may never finish on a weak link. +- Kolibri's own network import is not drop-resilient (spike, 15 Nov, Kolibri 0.19.5): it does + not resume partial files, and after a sender Wi-Fi cut its retry appended to the in-flight + files, which then failed the checksum and the command ended in error. + +Decision: the transport is rsync, it runs IN THE BOX, and conex-core owns it, so IIAB reuses the +same delivery. +- Sender: one rsync daemon per box, alive only while a share session is open. One read-only + module per item ("-"); a filter admits only that item's files; the module + password is the session token; the daemon logs nothing. The session owns its modules: create + adds them, every closer removes them, the last close stops the daemon, the startup sweep kills + a daemon a dead core left. The manifest advertises the module per item. +- Receiver: the receiver's own conex-core pulls (POST /v1/import/item with a source instead of a + payloadPath), as the same import long operation (same status, cancel and TTL). +- The HTTP fetch stays for mixed versions: a sender without the daemon advertises no module, and + a receiver box whose core refuses the pull falls back to the staged import. + +This is the "voice" the Clone analysis above asks for. The daemon is still mute, but it is no +longer alone: conex-core supervises its lifecycle, and the receiver reports per-item progress +to the gate (POST .../progress), which keeps the session fresh and feeds the sender's view. +COMPLETE still closes the pincer. + +Per app: +- ZIM (one large file): rsync --inplace --append-verify into /library/.conex-partial/kiwix/, + then a rename into the zims dir. A kill of either side keeps the bytes; the next pull appends + and verifies the WHOLE file (a corrupt partial is detected and re-sent). The rename is the + commit, so Kiwix never sees a partial ZIM. +- Kolibri (PR 2): static files, no tar. The content files are content-addressed and immutable + (file name = md5), so they move once, straight into the receiver's KOLIBRI_HOME storage, with + partial names until verified (never --inplace into storage, never --delete: the storage is + shared across channels). The channel DB and Kolibri's manifest move into a small session dir. + The sender stages nothing: the DB already lives in content/databases, `exportcontent + --manifest-only` writes only the manifest, and the file list is the channel's available + local files in the main db. The receiver then links them with three stable Kolibri commands: + `importchannel disk` -> `scanforcontent --channels --channel-import-mode none` -> + `importcontent disk`. Source review of Kolibri 0.19.5 (proven on device): scanforcontent alone + marks a file available on existence only and does not set admin_imported (content without it + can be auto-removed by content requests); importcontent alone fails a free-space check that + counts files already in place. The scan first, then importcontent (which copies nothing, + sets admin_imported and honors the manifest) passes on a full disk. + +Integrity is the transport's job: Kolibri marks a file available if it exists. So a file gets +its final name only when it is complete and verified (rsync whole-file checksum; for Kolibri +the md5 in the name). + +Android (K2Go only): the box rsync is a Linux process that Android routes by the default +network, which can be cellular when the hotspot has no internet. The receiver app relays the +pull through a loopback relay pinned to the hotspot Network (ShareRelay), the same pinning the +HTTP carrier uses. conex-core stays platform-neutral. + +Lifecycle (who sets, who clears, what if the process dies): +- Daemon + secrets: set by the first open session; cleared by the last close, the safety cap, + or the startup sweep (pid file of the process group). A daemon that dies mid-session is + restarted on the next manifest or progress call. +- Partial files: set by the pull; cleared by the rename on success; kept on a drop or a cancel + so a Retry resumes; removed after 48 h by the sweep (at startup and at each pull start). +- Kolibri files of an abandoned pull (DECIDED: keep): they land straight in Kolibri storage, + so a pull that nobody retries leaves them there, unreferenced. Kept on purpose: a later Retry + or import of the channel reuses them (the name is the md5), and pulling into the work dir + first would add one move per file under proot and re-send files other channels hold. +- Relay: opened by the receiver's import run at its first pulled item; closed when the run ends. + +Exposure: one more listener (TCP 9873) while a session is open. See +conex-core-share-exposure.md "rsync transport". + +Evidence: OPPO + OnePlus 7T, 333 MB ZIM over Wi-Fi at about 20 MB/s; cut at 131 MB, Retry +resumed from 131 MB, md5 equal on both phones, no staging copy, session and daemon closed. + +Open: +- Resume of a multi-GB partial on a slow phone (the --append-verify pre-read against the rsync + I/O timeouts): to test. +- IIAB: the host firewall must admit TCP 9873 when the share exposure is on. +- Kolibri content location: the share assumes KOLIBRI_HOME/content (the K2Go and IIAB default). + A box that moves it with Kolibri's CONTENT_DIR or CONTENT_FALLBACK_DIRS is not covered yet. +- conex-core is delivered with the rootfs today; a live update like dash-node is planned, so + mixed versions become rare (the HTTP fallback covers the gap). diff --git a/controller/docs/conex-core-share-exposure.md b/controller/docs/conex-core-share-exposure.md index 4928221bc..2f97a923b 100644 --- a/controller/docs/conex-core-share-exposure.md +++ b/controller/docs/conex-core-share-exposure.md @@ -160,6 +160,31 @@ This matrix passed against the rendered template in a stock nginx 1.27 with Remove the forward when done (`adb -s forward --remove tcp:8085`). +## rsync transport: a second listener while a session is open (K2GO-577) + +The HTTP fetch stages and re-sends a whole item after a drop. Share now moves an item that an +adapter can list as files (a ZIM today) with rsync, which writes in place and resumes. This +adds one listener next to the HTTP gate: + +- conex-core runs `rsync --daemon` on `0.0.0.0:9873` (`CONEX_SHARE_RSYNC_PORT`; `0` turns it + off and Share stays HTTP only). The daemon runs ONLY while a share session is open: the + first session starts it, the last close (complete, cancel or the safety cap) stops it, and + the startup sweep kills a daemon that a dead core left behind. With no session, the port is + closed. +- One read-only module per shared item, named `-` (128-bit session id). + A filter admits only that item's files; `list = no` hides the module list. +- The module password is the session token. rsync authenticates by challenge and response, so + the token is not sent in clear. A session close removes its modules and secrets at once. +- The daemon logs nothing (`log file = /dev/null`), so no client address is kept (K2GO-559). +- The share manifest advertises the module per item (`rsync: {port, module}`). The receiver + pulls through its own box's conex-core (`POST /v1/import/item` with a `source`); a receiver + whose box cannot pull uses the HTTP fetch, so old and new peers still work together. +- On K2Go the receiver app relays the pull over the hotspot network (`ShareRelay`), because + Android may route a box process over cellular when the hotspot has no internet. + +Pending for IIAB: a host firewall must admit TCP 9873 when `conex_core_share_expose` is on, or +a receiver cannot reach the daemon. + ## Scope kept (what this change does NOT do) - Does not expose the whole API: only `/v1/share//...`. diff --git a/static/conex-core/CHANGELOG.md b/static/conex-core/CHANGELOG.md index dd709381d..013f78c79 100644 --- a/static/conex-core/CHANGELOG.md +++ b/static/conex-core/CHANGELOG.md @@ -5,6 +5,8 @@ One line per version, newest first. Every REST-facing change bumps `conex_core.V role), so this file is the human record of what each bump enables. Keep entries short: `version - change (TICKET)`. +- **0.4.1** - Share moves Kolibri channels with rsync too, with no tar and no staging (K2GO-577). The sender module holds the channel DB and the available content files, in place. The receiver pulls the DB into a work dir and the files straight into Kolibri storage (a file that exists is skipped: its name is its md5; temp and kept partial files stay in `/library/.conex-partial`), then links them with `importchannel disk`, `scanforcontent --channel-import-mode none` and `importcontent disk` on a manifest of the nodes that arrived. A drop keeps the received files and a retry continues. (K2GO-577) +- **0.4.0** - Share moves ZIMs with rsync (K2GO-577). While a share session is open, the box runs an rsync daemon (`CONEX_SHARE_RSYNC_PORT`, default 9873, 0 = off) with one read-only module per shareable item, password = the session token; the last session close stops it. The share manifest adds `rsync: {port, module}` per such item. `POST /v1/import/item` with `source: {host, port, module, token}` instead of `payloadPath` pulls the item straight into place: no staging copy, a drop or a cancel keeps the bytes in `/library/.conex-partial` and the next pull resumes and verifies the whole file. `POST /v1/share/{id}/progress` takes the receiver's per-item progress (keeps the session fresh, marks `done` items delivered); `GET /v1/share/{id}/progress` is the sender's view. `POST /v1/import/discard {app, entry}` drops the bytes a given-up pull kept (the user ended the run; refused while a pull of the item runs). Kolibri and Calibre still share over the HTTP fetch. (K2GO-577) - **0.3.5** - Kiwix counts as installed only when kiwix-serve is present, not just the zims dir, so a library that cannot open ZIMs answers NO_APP and does not import them. (K2GO-574) - **0.3.4** - import becomes a long operation like export: POST /v1/import/item now STARTS the ingest and returns an opId, the client polls GET /v1/import/status/{op}, and DELETE /v1/import/{op} cancels, so a 35-141 s Kolibri import no longer times out at the dash-node/nginx relay. A shared liveness watchdog supervises the Kolibri import and export CLIs by byte and CPU progress (no fixed timeout; the window is relative to the task) and names a signal-9 exit as a likely out-of-memory kill. (K2GO-376) - **0.3.3** - observability: log every error response and uncaught handler exception to stdout (the supervisor captures it to /var/log/conex-core.log), and line-buffer stdout/stderr in __main__ so output reaches the log while the server runs forever, so a failed import or share leaves a box-side trace; an uncaught error now returns a 500 JSON instead of dropping the connection. (K2GO-376) diff --git a/static/conex-core/README.md b/static/conex-core/README.md index 4c3d3dc0d..dd3dfd1ee 100644 --- a/static/conex-core/README.md +++ b/static/conex-core/README.md @@ -38,9 +38,18 @@ Environment (defaults match the box layout): - `CONEX_HOST` (default `127.0.0.1`), `CONEX_PORT` (default `9500`). - `CONEX_ZIMS_DIR` (default `/library/zims/content/`): where the ZIM files live. +- `CONEX_SHARE_RSYNC_PORT` (default `9873`, `0` = off): the share rsync daemon port. The + daemon listens on all interfaces only while a share session is open (K2GO-577). +- `CONEX_RSYNC_BIN` (default `/usr/bin/rsync`), `CONEX_PARTIAL_DIR` (default + `/library/.conex-partial`): the rsync binary and where a pull keeps resumable partial files. ## Endpoints +Share over rsync (K2GO-577): `POST /v1/import/item` with `{app, entry, source: {host, port, +module, token}}` pulls an item from a sender's rsync module into place (same op and status as an +import). `POST /v1/share/{id}/progress` takes the receiver's `{itemRef, movedBytes, totalBytes, +state}`; `GET /v1/share/{id}/progress` returns it for the sender. + Read: - `GET /v1/version`: the core build (mirrors dash-node `/system/version`). diff --git a/static/conex-core/conex_core/__init__.py b/static/conex-core/conex_core/__init__.py index 457a21fa9..32a75e962 100644 --- a/static/conex-core/conex_core/__init__.py +++ b/static/conex-core/conex_core/__init__.py @@ -7,5 +7,5 @@ # The core build. Bumped per core change so a box can report which core it runs (GET # /v1/version), mirroring dash-node's package.json version and /system/version. The # conex_core Ansible role writes this into the [conex_core] section of iiab.ini. -VERSION = "0.3.5" +VERSION = "0.4.1" __version__ = VERSION diff --git a/static/conex-core/conex_core/adapters/kiwix.py b/static/conex-core/conex_core/adapters/kiwix.py index d1f9fbd99..a68427afc 100644 --- a/static/conex-core/conex_core/adapters/kiwix.py +++ b/static/conex-core/conex_core/adapters/kiwix.py @@ -33,7 +33,9 @@ Identity, ManifestItem, ) -from ..ports import ContentAdapter, ExportPlan, ExportStream +from ..importing import volume_nearly_full +from ..ports import ContentAdapter, ExportPlan, ExportStream, ShareFiles +from ..watchdog import CANCELLED # openzim header: magic number is uint32 little-endian 0x044D495A at offset 0, and the # 16-byte UUID is at offset 8. We only need the first 24 bytes. @@ -238,6 +240,58 @@ def import_item(self, entry: dict, payload_path: str, should_cancel=None, self._reindex() return {"ok": True, "app": "kiwix", "localRef": local_ref, "bytes": os.path.getsize(dest)} + def share_files(self, local_ref: str) -> Optional[ShareFiles]: + # A ZIM is one file in the zims dir: the daemon serves only that file (K2GO-577). + path = self._export_path(local_ref) + if path is None: + return None + return ShareFiles(root=self._dir, files=[local_ref]) + + def pull_item(self, entry: dict, source: dict, partial_dir: str, puller, + should_cancel=None, on_progress=None) -> dict: + # Pull the ZIM into a partial file on the same volume (resumable), then rename it into + # the zims dir and re-index. The rename is the commit: Kiwix never sees a partial ZIM. + local_ref = (entry.get("identity") or {}).get("localRef", "") + if not is_safe_local_ref(local_ref) or not local_ref.lower().endswith(".zim"): + return {"ok": False, "reason": "invalid localRef"} + part_dir = os.path.join(partial_dir, "kiwix") + os.makedirs(part_dir, exist_ok=True) + part = os.path.join(part_dir, local_ref) + result = puller.pull_file(source, local_ref, part, should_cancel=should_cancel, + on_progress=on_progress) + if result.state == CANCELLED: + return {"ok": False, "reason": "import cancelled"} + if not result.ok: + failure = {"ok": False, "reason": f"transfer {result.detail}"} + if volume_nearly_full(part_dir): + failure["code"] = "no_space" + return failure + os.makedirs(self._dir, exist_ok=True) + dest = os.path.join(self._dir, local_ref) + try: + os.replace(part, dest) + except OSError as exc: + if exc.errno != errno.EXDEV: + return {"ok": False, "reason": f"move failed: {exc}"} + shutil.move(part, dest) # the partial dir is on another volume: copy once + self._reindex() + return {"ok": True, "app": "kiwix", "localRef": local_ref, "bytes": os.path.getsize(dest)} + + def discard_pull(self, entry: dict, partial_dir: str) -> dict: + # The kept partial is the only thing a ZIM pull leaves behind (K2GO-577). + local_ref = (entry.get("identity") or {}).get("localRef", "") + if not is_safe_local_ref(local_ref) or not local_ref.lower().endswith(".zim"): + return {"ok": False, "reason": "invalid localRef"} + part = os.path.join(partial_dir, "kiwix", local_ref) + try: + freed = os.path.getsize(part) + os.remove(part) + except FileNotFoundError: + freed = 0 + except OSError as exc: + return {"ok": False, "reason": f"discard failed: {exc}"} + return {"ok": True, "freedBytes": freed} + def delete(self, local_ref: str) -> dict: # A ZIM is self-contained, so the file IS the unit: remove it, then re-run the indexer # to drop its library.xml entry. freedBytes is the file size exactly (no shared bytes). diff --git a/static/conex-core/conex_core/adapters/kolibri.py b/static/conex-core/conex_core/adapters/kolibri.py index 64a6f1571..3e3e3bec7 100644 --- a/static/conex-core/conex_core/adapters/kolibri.py +++ b/static/conex-core/conex_core/adapters/kolibri.py @@ -13,6 +13,7 @@ from __future__ import annotations import errno +import json import logging import os import re @@ -36,7 +37,7 @@ Identity, ManifestItem, ) -from ..ports import ContentAdapter, ExportPlan, ExportStream +from ..ports import ContentAdapter, ExportPlan, ExportStream, ShareFiles _CHANNEL_ID_RE = re.compile(r"^[0-9a-fA-F]{32}$") @@ -66,6 +67,39 @@ def import_commands(channel_id: str, source: str, kolibri_bin: str) -> List[List ] +def pull_finalize_commands(channel_id: str, source: str, kolibri_bin: str) -> List[List[str]]: + """The Kolibri commands that link a pulled channel (K2GO-577), in order: register the channel + DB, mark the files already in storage available, then importcontent (copies nothing: every + file is in place) to set admin_imported on the manifest's nodes. The scan must run first: + importcontent alone counts files not yet marked available against free space, and fails on a + fuller disk. Pure (argv only). The manifest is written between the scan and importcontent.""" + base = [kolibri_bin, "manage"] + return [ + base + ["importchannel", "disk", channel_id, source], + base + ["scanforcontent", "--channels", channel_id, "--channel-import-mode", "none"], + base + ["importcontent", "disk", channel_id, source], + ] + + +# The files a channel holds on this box: its available local files (K2GO-577 share). +_CHANNEL_FILES_SQL = """ +SELECT DISTINCT lf.id, lf.extension +FROM content_localfile lf +JOIN content_file f ON f.local_file_id = lf.id +JOIN content_contentnode cn ON cn.id = f.contentnode_id +WHERE cn.channel_id = ? AND lf.available = 1 +""" + +# The nodes a channel has available after the scan, for the import manifest. +_AVAILABLE_NODES_SQL = """ +SELECT id FROM content_contentnode +WHERE channel_id = ? AND available = 1 AND kind != 'topic' +""" + +_FILE_ID_RE = re.compile(r"^[0-9a-f]{32}$") +_EXT_RE = re.compile(r"^[A-Za-z0-9]{1,16}$") + + def delete_command(channel_id: str, kolibri_bin: str) -> List[str]: """The Kolibri management command that deletes a channel and its unused content files. Pure (builds argv only). This is Kolibri's OWN delete: it does the shared-file refcounting, @@ -294,6 +328,125 @@ def _stage_import(self, channel_id: str, source: str, should_cancel=None, return result return SuperviseResult(OK, 0, "ok") + def share_files(self, local_ref: str) -> Optional[ShareFiles]: + # The channel DB plus the channel's available content files, relative to the content + # dir (K2GO-577). The files are content-addressed and immutable, and the DB already lives + # in content/databases, so the share stages nothing. A row with an unexpected id or + # extension is skipped, never written into the daemon filter. + if not _CHANNEL_ID_RE.match(local_ref or ""): + return None + content = os.path.join(self._home, "content") + db_rel = f"databases/{local_ref}.sqlite3" + if not os.path.isfile(os.path.join(content, db_rel)): + return None + rows = self._read(_CHANNEL_FILES_SQL, (local_ref,)) + if rows is None: + return None + files = [db_rel] + [f"storage/{fid[0]}/{fid[1]}/{fid}.{ext}" for fid, ext in rows + if _FILE_ID_RE.match(fid or "") and _EXT_RE.match(ext or "")] + return ShareFiles(root=content, files=files) + + def pull_item(self, entry: dict, source: dict, partial_dir: str, puller, + should_cancel=None, on_progress=None) -> dict: + # Pull the channel DB into a small work dir, the content files straight into Kolibri's + # storage, then link them with Kolibri's own commands (K2GO-577). The work dir is kept + # on a failure, so a Retry resumes; it is removed on success. + channel_id = (entry.get("identity") or {}).get("localRef", "") + if not _CHANNEL_ID_RE.match(channel_id or ""): + return {"ok": False, "reason": "invalid channel id"} + work = os.path.join(partial_dir, "kolibri", channel_id) + src = os.path.join(work, "src") + content = os.path.join(self._home, "content") + os.makedirs(os.path.join(src, "content"), exist_ok=True) + # The module root holds databases/ and storage/ (share_files). The DB goes to the work + # dir (the import source); the files go straight into Kolibri's storage. + steps = ( + lambda: puller.pull_tree( + source, os.path.join(src, "content"), work, ignore_existing=False, + filters=("+ /databases/", f"+ /databases/{channel_id}.sqlite3", "- *"), + should_cancel=should_cancel), + lambda: puller.pull_tree( + source, content, work, filters=("- /databases/",), + should_cancel=should_cancel, on_progress=on_progress), + ) + for step in steps: + result = step() + if result.state == CANCELLED: + return {"ok": False, "reason": "import cancelled"} + if not result.ok: + return self._pull_failure(f"transfer {result.detail}") + outcome = self._finalize_pull(channel_id, src, should_cancel) + if outcome.state == CANCELLED: + return {"ok": False, "reason": "import cancelled"} + if not outcome.ok: + return self._pull_failure(f"kolibri import {outcome.detail}") + shutil.rmtree(work, ignore_errors=True) + return {"ok": True, "app": "kolibri", "localRef": channel_id} + + def discard_pull(self, entry: dict, partial_dir: str) -> dict: + # Drop the work dir (the channel DB, the in-flight temp and a kept partial). The files that + # already arrived stay in Kolibri storage on purpose: a later pull of the channel reuses them + # (decision recorded in ADR-content-exchange, K2GO-577). + channel_id = (entry.get("identity") or {}).get("localRef", "") + if not _CHANNEL_ID_RE.match(channel_id or ""): + return {"ok": False, "reason": "invalid channel id"} + work = os.path.join(partial_dir, "kolibri", channel_id) + freed = 0 + for dirpath, _dirs, files in os.walk(work): + for name in files: + try: + freed += os.path.getsize(os.path.join(dirpath, name)) + except OSError: + pass + shutil.rmtree(work, ignore_errors=True) + return {"ok": True, "freedBytes": freed} + + def _pull_failure(self, reason: str) -> dict: + failure = {"ok": False, "reason": reason} + if volume_nearly_full(self._home): + failure["code"] = "no_space" + return failure + + def _finalize_pull(self, channel_id: str, src: str, should_cancel=None) -> SuperviseResult: + """Run the three Kolibri link commands under the liveness watchdog, writing the import + manifest after the scan. A seam, so a test need not run Kolibri.""" + env = dict(os.environ, KOLIBRI_HOME=self._home) + for argv in pull_finalize_commands(channel_id, src, self._bin): + if argv[2] == "importcontent": + self._write_pull_manifest(channel_id, src) + result = run_supervised(argv, env, dest_dir=self._home, should_cancel=should_cancel, + log=_LOG) + if not result.ok: + return result + return SuperviseResult(OK, 0, "ok") + + def _write_pull_manifest(self, channel_id: str, src: str) -> None: + """Kolibri's import manifest (content/manifest.json, which importcontent reads from its + source dir): the channel's nodes that are available after the scan, i.e. the nodes whose + files arrived. So importcontent marks exactly those admin-imported and never counts the + files the sender did not have.""" + rows = self._read(_AVAILABLE_NODES_SQL, (channel_id,)) or [] + version = self._read("SELECT version FROM content_channelmetadata WHERE id = ?", + (channel_id,)) or [(0,)] + manifest = {"channels": [{"id": channel_id, "version": int(version[0][0] or 0), + "include_node_ids": [r[0] for r in rows]}]} + path = os.path.join(src, "content", "manifest.json") + with open(path, "w", encoding="utf-8") as f: + json.dump(manifest, f) + + def _read(self, sql: str, params) -> Optional[list]: + """Rows of a read-only query on the main DB, or None when it cannot be read.""" + try: + con = sqlite3.connect(f"file:{self._db}?mode=ro", uri=True) + except sqlite3.Error: + return None + try: + return con.execute(sql, params).fetchall() + except sqlite3.Error: + return None + finally: + con.close() + def delete(self, local_ref: str) -> dict: # Delegate to Kolibri's own delete (it refcounts shared files). Measure the reclaim as a # free-space delta around the CLI call: the filesystem is the one source for bytes diff --git a/static/conex-core/conex_core/config.py b/static/conex-core/conex_core/config.py index f3377e705..a4fd74f55 100644 --- a/static/conex-core/conex_core/config.py +++ b/static/conex-core/conex_core/config.py @@ -7,16 +7,21 @@ from dataclasses import dataclass DEFAULT_PORT = 9500 +#: The share rsync daemon port (K2GO-577). Above 1024: a proot box cannot bind a low port. +DEFAULT_SHARE_RSYNC_PORT = 9873 -def _parse_port(raw: str) -> int: +def _parse_port(raw: str, default: int = DEFAULT_PORT, allow_off: bool = False) -> int: """Parse a TCP port from the environment, falling back to the default on an empty, - non-numeric or out-of-range value instead of crashing the service at startup.""" + non-numeric or out-of-range value instead of crashing the service at startup. With + ``allow_off``, 0 is kept (the feature is off).""" try: port = int(raw) except (TypeError, ValueError): - return DEFAULT_PORT - return port if 1 <= port <= 65535 else DEFAULT_PORT + return default + if allow_off and port == 0: + return 0 + return port if 1 <= port <= 65535 else default @dataclass(frozen=True) @@ -38,6 +43,12 @@ class Config: # multi-gigabyte channel export has room and the free-space preflight measures the right # disk. staging_dir: str = "/library/.conex-staging" + # K2GO-577: the share rsync daemon port (0 = off: share over HTTP fetch only), the rsync + # binary, and where a pull keeps resumable partial files (the content volume, so the final + # rename does not copy). + share_rsync_port: int = DEFAULT_SHARE_RSYNC_PORT + rsync_bin: str = "/usr/bin/rsync" + partial_dir: str = "/library/.conex-partial" @staticmethod def from_env() -> "Config": @@ -55,4 +66,9 @@ def from_env() -> "Config": credentials_file=os.environ.get( "K2GO_CREDENTIALS_FILE", "/library/dashboard/credentials.json"), staging_dir=os.environ.get("CONEX_STAGING_DIR", "/library/.conex-staging"), + share_rsync_port=_parse_port( + os.environ.get("CONEX_SHARE_RSYNC_PORT", str(DEFAULT_SHARE_RSYNC_PORT)), + DEFAULT_SHARE_RSYNC_PORT, allow_off=True), + rsync_bin=os.environ.get("CONEX_RSYNC_BIN", "/usr/bin/rsync"), + partial_dir=os.environ.get("CONEX_PARTIAL_DIR", "/library/.conex-partial"), ) diff --git a/static/conex-core/conex_core/import_ops.py b/static/conex-core/conex_core/import_ops.py index f3a038747..bd6caccaf 100644 --- a/static/conex-core/conex_core/import_ops.py +++ b/static/conex-core/conex_core/import_ops.py @@ -23,6 +23,7 @@ from typing import Any, Optional from .importing import is_within_dir +from .rsync_share import sweep_partials, valid_source from .ops_base import DONE, ERROR, RUNNING, Op, OpRegistry, Response # The import work-temp prefix, created by the adapters' import_item. The startup sweep reclaims an @@ -35,6 +36,7 @@ class _ImportOp(Op): moved_bytes: int = 0 result: Optional[dict] = None code: Optional[str] = None # a structural error code, e.g. "no_space" + local_ref: Optional[tuple] = None # (app, localRef) of a pull, so discard can refuse a live one def _entry_bytes(entry: Any) -> int: @@ -49,10 +51,15 @@ class ImportOps(OpRegistry): """Registry of in-flight import operations. One instance lives for the server's lifetime; the handler threads call in concurrently.""" - def __init__(self, registry, import_root: str, ttl_seconds: int = 600): + def __init__(self, registry, import_root: str, ttl_seconds: int = 600, + partial_root: Optional[str] = None, puller=None): super().__init__(ttl_seconds) self._registry = registry self._import_root = import_root + # K2GO-577: where a pull keeps its resumable partial files, and the rsync client. Both + # None disables pull (the HTTP-staged import still works). + self._partial_root = partial_root + self._puller = puller # ---- REST-facing operations (return (status, payload)) ---- @@ -75,6 +82,51 @@ def start(self, app: str, entry: Any, payload_path: str) -> Response: worker.start() return 200, {"opId": op_id, "state": RUNNING, "totalBytes": total} + def start_pull(self, app: str, entry: Any, source: Any) -> Response: + """Start a pull long operation (K2GO-577): the adapter pulls the item from the sender's + rsync module straight into place, then indexes it. Same op shape as ``start``, so the + client polls the same status; movedBytes is the bytes on disk, a resumed pull starts + from the kept bytes.""" + self._sweep_expired() + if not self._registry.has(app): + return 404, {"error": f"unknown app: {app}"} + if self._puller is None or self._partial_root is None or not self._puller.available(): + return 501, {"error": "pull unavailable on this box"} + if not isinstance(entry, dict): + return 400, {"error": "entry is required"} + reason = valid_source(source) + if reason is not None: + return 400, {"error": reason} + # Reclaim partials no retry picked up, also on a box that runs for weeks without a restart. + sweep_partials(self._partial_root) + total = _entry_bytes(entry) + op_id = self._new_id() + op = _ImportOp(app=app, state=RUNNING, total_bytes=total) + op.local_ref = (app, (entry.get("identity") or {}).get("localRef")) + self._put(op_id, op) + worker = threading.Thread( + target=self._run_pull, args=(op_id, app, entry, source, total), daemon=True) + worker.start() + return 200, {"opId": op_id, "state": RUNNING, "totalBytes": total} + + def discard(self, app: str, entry: Any) -> Response: + """Drop what a given-up pull kept for a resume (K2GO-577): the user ended the run (Cancel or + Finish), so the partial bytes must not hold the disk for 48 h. Refused while a pull of the + same item still runs: the client aborts the op first, and a resumed pull would lose its + bytes.""" + self._sweep_expired() + if not self._registry.has(app): + return 404, {"error": f"unknown app: {app}"} + if not isinstance(entry, dict) or self._partial_root is None: + return 400, {"error": "entry is required"} + local_ref = (entry.get("identity") or {}).get("localRef") + with self._lock: + busy = any(op.state == RUNNING and getattr(op, "local_ref", None) == (app, local_ref) + for op in self._ops.values()) + if busy: + return 409, {"error": "a pull of this item is still running"} + return 200, self._registry.get(app).discard_pull(entry, self._partial_root) + def status(self, op_id: str) -> Response: """Report the op state, the moved-bytes heartbeat, and the result once done. A 404 means the op is unknown (swept or never existed), so the client re-starts.""" @@ -105,6 +157,8 @@ def sweep_startup(self) -> None: for name in names: if name.startswith(TEMP_PREFIX): shutil.rmtree(os.path.join(self._import_root, name), ignore_errors=True) + if self._partial_root: + sweep_partials(self._partial_root) def _run_import(self, op_id: str, app: str, entry: Any, payload_path: str, total: int) -> None: """Ingest the item off the request thread, under the liveness watchdog, then flip the op to @@ -119,9 +173,25 @@ def on_progress(moved: int) -> None: # that fills and holds, never past 100%. self._touch(op_id, moved_bytes=min(moved, total) if total else moved) + self._finish_with(op_id, total, lambda: self._registry.get(app).import_item( + entry, payload_path, should_cancel=should_cancel, on_progress=on_progress)) + + def _run_pull(self, op_id: str, app: str, entry: Any, source: dict, total: int) -> None: + """Pull the item off the request thread. movedBytes is the partial file size, so it is + monotonic and already counts the bytes a resumed pull kept.""" + should_cancel = self.should_cancel(op_id) + + def on_progress(moved: int) -> None: + self._touch(op_id, moved_bytes=min(moved, total) if total else moved) + + self._finish_with(op_id, total, lambda: self._registry.get(app).pull_item( + entry, source, self._partial_root, self._puller, + should_cancel=should_cancel, on_progress=on_progress)) + + def _finish_with(self, op_id: str, total: int, work) -> None: + """Run the adapter work, then flip the op to done or error.""" try: - result = self._registry.get(app).import_item( - entry, payload_path, should_cancel=should_cancel, on_progress=on_progress) + result = work() except Exception as exc: # any adapter failure surfaces as an op error with its detail self._transition(op_id, RUNNING, ERROR, error=f"import failed: {exc}") return diff --git a/static/conex-core/conex_core/ports.py b/static/conex-core/conex_core/ports.py index d2d8fc323..fdf6e4a11 100644 --- a/static/conex-core/conex_core/ports.py +++ b/static/conex-core/conex_core/ports.py @@ -55,6 +55,14 @@ class ExportPlan: content_type: str +@dataclass +class ShareFiles: + """The files one item is made of, for the share rsync daemon (K2GO-577): a root dir and the + paths under it. The daemon serves only these, read-only.""" + root: str + files: List[str] + + class ContentAdapter(ABC): """Read-side port for one content app.""" @@ -119,6 +127,25 @@ def import_item(self, entry: dict, payload_path: str, may ignore both. Default: not supported.""" return {"ok": False, "reason": "import not supported"} + def share_files(self, local_ref: str) -> Optional[ShareFiles]: + """The files the share rsync daemon may serve for this item (K2GO-577). None: the item + is not shared over rsync, so a receiver uses the HTTP fetch. Default: None.""" + return None + + def pull_item(self, entry: dict, source: dict, partial_dir: str, puller, + should_cancel: Optional[Callable[[], bool]] = None, + on_progress: Optional[Callable[[int], None]] = None) -> dict: + """Pull one item from a sender's rsync module straight into place and index it + (K2GO-577). ``partial_dir`` is this app's dir for resumable partial files; ``puller`` is + the rsync client. Same result shape as ``import_item``. Default: not supported.""" + return {"ok": False, "reason": "pull not supported"} + + def discard_pull(self, entry: dict, partial_dir: str) -> dict: + """Remove what a given-up pull of this item kept for a resume (K2GO-577): the user ended + the run with Cancel or Finish, so the bytes will not be resumed. Returns {ok, freedBytes}. + Default: nothing kept.""" + return {"ok": True, "freedBytes": 0} + def delete(self, local_ref: str) -> dict: """Remove one item to reclaim space, delegating to the app's own delete (never driving the app DB + files directly): the app owns its storage invariants and the next list() diff --git a/static/conex-core/conex_core/router.py b/static/conex-core/conex_core/router.py index e3cbfc071..e709cf390 100644 --- a/static/conex-core/conex_core/router.py +++ b/static/conex-core/conex_core/router.py @@ -167,6 +167,11 @@ def dispatch(registry: Registry, method: str, path: str, query: dict, body: Any return 500, {"error": "share unavailable"} items = body.get("items") if isinstance(body, dict) else None return share.create(items) + # POST /v1/share/{sessionId}/progress : the receiver reports one item's progress. + if len(segments) == 4 and segments[0] == "v1" and segments[1] == "share" and segments[3] == "progress": + if share is None: + return 500, {"error": "share unavailable"} + return share.report_progress(segments[2], token, body) # POST /v1/share/{sessionId}/complete : the receiver signals a clean, done transfer. if len(segments) == 4 and segments[0] == "v1" and segments[1] == "share" and segments[3] == "complete": if share is None: @@ -178,6 +183,13 @@ def dispatch(registry: Registry, method: str, path: str, query: dict, body: Any # POST /v1/import/validate : what will import and what is skipped, nothing moved. if segments == ["v1", "import", "validate"]: return _import_validate(registry, body) + # POST /v1/import/discard : drop what a given-up pull kept for a resume (K2GO-577). + if segments == ["v1", "import", "discard"]: + if import_ops is None: + return 500, {"error": "import ops unavailable"} + if not isinstance(body, dict): + return 400, {"error": "body must be {app, entry}"} + return import_ops.discard(body.get("app"), body.get("entry")) # POST /v1/import/item : start the import long operation, return its opId. Import is a long # op (K2GO-376): the ingest runs minutes on a real channel, so a single blocking POST times # out at the relay. The op starts off the request thread; the client polls import status. @@ -185,7 +197,11 @@ def dispatch(registry: Registry, method: str, path: str, query: dict, body: Any if import_ops is None: return 500, {"error": "import ops unavailable"} if not isinstance(body, dict): - return 400, {"error": "body must be {app, entry, payloadPath}"} + return 400, {"error": "body must be {app, entry, payloadPath} or {app, entry, source}"} + # A source instead of a payloadPath: pull the item from a sender's rsync module + # straight into place (K2GO-577). + if "source" in body: + return import_ops.start_pull(body.get("app"), body.get("entry"), body.get("source")) return import_ops.start(body.get("app"), body.get("entry"), body.get("payloadPath")) return 404, {"error": "not found"} @@ -246,6 +262,12 @@ def dispatch(registry: Registry, method: str, path: str, query: dict, body: Any return 500, {"error": "share unavailable"} return share.manifest(segments[2], token) + # GET /v1/share/{sessionId}/progress : the receiver's last progress report per item. + if len(segments) == 4 and segments[0] == "v1" and segments[1] == "share" and segments[3] == "progress": + if share is None: + return 500, {"error": "share unavailable"} + return share.progress(segments[2], token) + # GET /v1/share/{sessionId}/fetch/{itemRef} : stream one shared item (payload is an # ExportStream the server streams). 409 while the item stages; the receiver retries. if len(segments) == 5 and segments[0] == "v1" and segments[1] == "share" and segments[3] == "fetch": diff --git a/static/conex-core/conex_core/rsync_share.py b/static/conex-core/conex_core/rsync_share.py new file mode 100644 index 000000000..2b179b160 --- /dev/null +++ b/static/conex-core/conex_core/rsync_share.py @@ -0,0 +1,388 @@ +"""rsync transport for Share (K2GO-577). + +Share moves content with rsync, not with a staged HTTP stream: rsync writes each file once into its +final place, keeps the partial bytes when the link drops, and resumes from them. The core owns the +whole mechanism, so IIAB and K2Go share it. + +Sender: one rsync daemon per box, alive only while a share session is open. Each shared item is a +read-only module whose filter admits only that item's files. The module password is the session +token. The daemon re-reads its config on every connection, so a session that opens or closes only +rewrites the config; the last close stops the daemon. + +Receiver: one rsync client per item. A single large file (a ZIM) is pulled with --inplace +--append-verify into a partial path on the content volume, then renamed into place: a kill of the +client or of the sender keeps the bytes, and the next pull appends to them and verifies the whole +file. The rename is the commit, so the app never sees a partial file under its final name. + +Lifecycle: + daemon: started by the first open session, stopped by the last close, the safety cap, or the + startup sweep (a pid file names the process group a dead core left behind). + partial file: made by the pull, renamed into place on success, kept on a drop or a cancel so a + retry resumes, removed by the startup sweep after PARTIAL_MAX_AGE_SECONDS. +""" + +from __future__ import annotations + +import logging +import os +import re +import shutil +import signal +import subprocess +import threading +import time +from dataclasses import dataclass +from typing import Callable, List, Optional + +from .watchdog import run_supervised + +_LOG = logging.getLogger("conex-core") +_SIGKILL = getattr(signal, "SIGKILL", 9) + +#: The rsync user name. The secret (the session token) is the auth; the name is fixed. +SHARE_USER = "share" +#: Module names are "-": hex, digits and a hyphen. +MODULE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]{0,80}$") +#: An IPv4 address or a host name. No IPv6: the hotspot and LAN paths are IPv4. +HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.-]{0,252}$") +#: rsync ends a transfer with no data for this long (a dead peer), keeping the partial. +IO_TIMEOUT_SECONDS = 60 +CONNECT_TIMEOUT_SECONDS = 20 +#: A partial that no retry picked up for this long is removed by the startup sweep. +PARTIAL_MAX_AGE_SECONDS = 48 * 3600 + + +@dataclass(frozen=True) +class Module: + """One shared item as a daemon module: the root dir and the files under it it may serve.""" + name: str + root: str + files: List[str] + secret: str + + +def _clean(value: str) -> bool: + """True when a value is safe to write into rsyncd.conf or a filter file: no control chars + (a newline would start a new directive) and no ']' (it would close a section header).""" + return bool(value) and "]" not in value and not any(ord(c) < 32 or ord(c) == 127 for c in value) + + +def render_filter(files: List[str]) -> str: + """Filter rules that admit only ``files`` (relative to the module root) and their parent + dirs, then exclude everything else. Anchored with '/', so a name matches only at its path.""" + rules: List[str] = [] + seen = set() + for rel in files: + parts = rel.strip("/").split("/") + for depth in range(1, len(parts)): + parent = "/" + "/".join(parts[:depth]) + "/" + if parent not in seen: + seen.add(parent) + rules.append("+ " + parent) + rules.append("+ /" + "/".join(parts)) + rules.append("- *") + return "\n".join(rules) + "\n" + + +def render_config(modules: List[Module], run_dir: str, uid: int, gid: int) -> str: + """The daemon config. log file /dev/null: the daemon logs no client address (K2GO-559). + uid/gid keep the core's own identity, so a root daemon does not drop to nobody and lose + read access to the content.""" + lines = [ + "use chroot = no", + "reverse lookup = no", + "log file = /dev/null", + "transfer logging = no", + f"timeout = {IO_TIMEOUT_SECONDS * 2}", + f"uid = {uid}", + f"gid = {gid}", + ] + for m in modules: + lines += [ + f"[{m.name}]", + f" path = {m.root}", + " read only = yes", + " list = no", + f" auth users = {SHARE_USER}", + f" secrets file = {os.path.join(run_dir, m.name + '.secrets')}", + " strict modes = no", + f" filter = merge {os.path.join(run_dir, m.name + '.filter')}", + ] + return "\n".join(lines) + "\n" + + +def _getid(name: str) -> int: + """The process uid / gid; 0 on a host without them (only the tests run there).""" + return getattr(os, name, lambda: 0)() + + +def _write(path: str, text: str, mode: int = 0o600) -> None: + """Write atomically, so a connection that reads the config mid-rewrite sees old or new.""" + tmp = path + ".tmp" + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode) + with os.fdopen(fd, "w") as f: + f.write(text) + os.replace(tmp, path) + + +class RsyncDaemon: + """The sender's rsync daemon. Thread-safe; one instance lives for the server's lifetime.""" + + def __init__(self, run_dir: str, port: int, rsync_bin: str = "/usr/bin/rsync", + address: str = "0.0.0.0", popen=subprocess.Popen, killpg=None): + self._run_dir = run_dir + self.port = port + self._bin = rsync_bin + self._address = address + self._popen = popen + # Seams: tests never start or signal a real process. getattr keeps this importable on a + # host without process groups, so the tests run anywhere. + self._killpg = killpg or getattr(os, "killpg", None) + self._lock = threading.Lock() + self._proc = None + + @property + def _conf(self) -> str: + return os.path.join(self._run_dir, "rsyncd.conf") + + @property + def _pidfile(self) -> str: + return os.path.join(self._run_dir, "daemon.pid") + + def available(self) -> bool: + return os.path.isfile(self._bin) + + def running(self) -> bool: + with self._lock: + return self._proc is not None and self._proc.poll() is None + + def set_modules(self, modules: List[Module]) -> bool: + """Serve exactly ``modules``. Starts the daemon for the first module and stops it when + none is left. Returns True when the daemon serves them. A module with an unsafe value is + skipped (logged), never written.""" + safe = [m for m in modules if MODULE_RE.match(m.name) and _clean(m.root) + and _clean(m.secret) and m.files and all(_clean(f) for f in m.files)] + if len(safe) != len(modules): + _LOG.warning("share rsync: skipped %d unsafe module(s)", len(modules) - len(safe)) + with self._lock: + if not safe: + self._stop_locked() + return False + if not self.available(): + return False + os.makedirs(self._run_dir, exist_ok=True) + names = set() + for m in safe: + names.add(m.name) + _write(os.path.join(self._run_dir, m.name + ".secrets"), f"{SHARE_USER}:{m.secret}\n") + _write(os.path.join(self._run_dir, m.name + ".filter"), render_filter(m.files)) + _write(self._conf, render_config(safe, self._run_dir, _getid("getuid"), _getid("getgid"))) + self._remove_stale(names) + if self._proc is None or self._proc.poll() is not None: + self._start_locked() + return self._proc is not None and self._proc.poll() is None + + def stop(self) -> None: + with self._lock: + self._stop_locked() + + def sweep_startup(self) -> None: + """Stop a daemon a dead core left behind and remove its config and secrets.""" + try: + with open(self._pidfile) as f: + pgid = int(f.read().strip()) + self._killpg(pgid, _SIGKILL) + except (OSError, ValueError): + pass + shutil.rmtree(self._run_dir, ignore_errors=True) + + def _start_locked(self) -> None: + argv = [self._bin, "--daemon", "--no-detach", f"--config={self._conf}", + f"--address={self._address}", f"--port={self.port}"] + try: + # Own process group: the per-connection children are in it, so a stop ends a live + # transfer too (killing only the parent leaves the children serving). + self._proc = self._popen(argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + start_new_session=True) + except OSError as exc: + _LOG.warning("share rsync: daemon did not start: %s", exc) + self._proc = None + return + _write(self._pidfile, str(self._proc.pid)) + time.sleep(0.3) # a busy port makes the daemon exit at once; report that as not serving + if self._proc.poll() is not None: + _LOG.warning("share rsync: daemon exited at start (port %d busy?)", self.port) + self._proc = None + + def _stop_locked(self) -> None: + proc, self._proc = self._proc, None + if proc is not None and proc.poll() is None: + for sig in (signal.SIGTERM, _SIGKILL): + try: + self._killpg(proc.pid, sig) + proc.wait(timeout=3) + break + except (OSError, subprocess.TimeoutExpired): + continue + shutil.rmtree(self._run_dir, ignore_errors=True) + + def _remove_stale(self, keep: set) -> None: + for name in os.listdir(self._run_dir): + stem, ext = os.path.splitext(name) + if ext in (".secrets", ".filter") and stem not in keep: + try: + os.remove(os.path.join(self._run_dir, name)) + except OSError: + pass + + +def valid_source(source) -> Optional[str]: + """None when a pull source {host, port, module, token} is safe to use, else the reason.""" + if not isinstance(source, dict): + return "source must be an object" + host, port, module, token = (source.get(k) for k in ("host", "port", "module", "token")) + if not isinstance(host, str) or not HOST_RE.match(host): + return "invalid source host" + if isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535: + return "invalid source port" + if not isinstance(module, str) or not MODULE_RE.match(module): + return "invalid source module" + if not isinstance(token, str) or not _clean(token): + return "invalid source token" + return None + + +class RsyncPuller: + """The receiver's rsync client. ``runner`` is the liveness supervisor (a seam for tests).""" + + def __init__(self, rsync_bin: str = "/usr/bin/rsync", runner=run_supervised): + self._bin = rsync_bin + self._runner = runner + + def available(self) -> bool: + return os.path.isfile(self._bin) + + def pull_file(self, source: dict, remote_path: str, dest_path: str, + should_cancel: Optional[Callable[[], bool]] = None, + on_progress: Optional[Callable[[int], None]] = None): + """Pull one file of a module into ``dest_path``, resuming from its bytes. Returns the + supervisor result: ok, cancelled (SIGTERM, partial kept), or the failure.""" + url = (f"rsync://{SHARE_USER}@{source['host']}:{source['port']}" + f"/{source['module']}/{remote_path}") + argv = [self._bin, "-t", "--inplace", "--append-verify", + f"--timeout={IO_TIMEOUT_SECONDS}", f"--contimeout={CONNECT_TIMEOUT_SECONDS}", + url, dest_path] + # The password goes in the environment, never argv (argv is visible to every process). + env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "RSYNC_PASSWORD": source["token"]} + + def on_disk() -> int: + try: + return os.path.getsize(dest_path) + except OSError: + return 0 + + def progress(_activity: int) -> None: + if on_progress is not None: + on_progress(on_disk()) + + # Liveness is the bytes on disk: in a pull the parent rsync is the generator, which moves + # almost nothing while the receiver child writes, so its own I/O would look stalled. + return self._runner(argv, env, dest_dir=os.path.dirname(dest_path), + should_cancel=should_cancel, on_progress=progress, log=_LOG, + activity_fn=on_disk, die_with_parent=True) + + def pull_tree(self, source: dict, dest_dir: str, work_dir: str, filters=(), + ignore_existing: bool = True, + should_cancel: Optional[Callable[[], bool]] = None, + on_progress: Optional[Callable[[int], None]] = None): + """Pull files of a module into ``dest_dir``, from the MODULE ROOT, picking them with the + client ``filters`` (rsync matches the daemon filter against paths relative to the + transfer root, so a subdir pull would miss the module-anchored rules). With + ``ignore_existing`` a file that already exists is skipped: for content-addressed files + (Kolibri, name = md5) it is the same file. The in-flight temp file and a kept partial + live in ``work_dir``, never under a final name in ``dest_dir``. Never deletes in + ``dest_dir`` and never touches its dir times (other content shares it). Progress is the + sum of the files completed, read from the client log (cumulative over retries).""" + kept, tmp = os.path.join(work_dir, "kept"), os.path.join(work_dir, "tmp") + log = os.path.join(work_dir, "transfer.log") + # A run that was killed (the app died) leaves its in-flight temp file here, and rsync never + # reuses it. Only one pull of this item runs at a time, so clear it before each run. + shutil.rmtree(tmp, ignore_errors=True) + for d in (kept, tmp, dest_dir): + os.makedirs(d, exist_ok=True) + url = f"rsync://{SHARE_USER}@{source['host']}:{source['port']}/{source['module']}/" + argv = [self._bin, "-r", "-t", "--omit-dir-times", f"--partial-dir={kept}", + f"--temp-dir={tmp}", f"--log-file={log}", "--log-file-format=%o %l", + f"--timeout={IO_TIMEOUT_SECONDS}", f"--contimeout={CONNECT_TIMEOUT_SECONDS}"] + if ignore_existing: + argv.append("--ignore-existing") + argv += [f"--filter={rule}" for rule in filters] + argv += [url, dest_dir.rstrip("/") + "/"] + env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "RSYNC_PASSWORD": source["token"]} + tally = _LogTally(log) + + def on_disk() -> int: + # Completed files plus the file in flight (its temp lives in tmp): the liveness + # signal, since the parent rsync (the generator) moves almost nothing itself. + return tally.read() + _dir_bytes(tmp) + + def progress(_activity: int) -> None: + if on_progress is not None: + on_progress(tally.read()) + + return self._runner(argv, env, dest_dir=None, should_cancel=should_cancel, + on_progress=progress, log=_LOG, activity_fn=on_disk) + + +def _dir_bytes(path: str) -> int: + """Bytes of the files directly in ``path`` (the rsync temp dir holds one in-flight file).""" + total = 0 + try: + with os.scandir(path) as it: + for e in it: + if e.is_file(follow_symlinks=False): + total += e.stat(follow_symlinks=False).st_size + except OSError: + pass + return total + + +class _LogTally: + """Sums the "recv " lines the rsync client appends to its log, reading only the + new bytes on each call (the log can hold thousands of lines).""" + + _LINE = re.compile(r"\brecv (\d+)\s*$") + + def __init__(self, path: str): + self._path = path + self._offset = 0 + self._total = 0 + + def read(self) -> int: + try: + with open(self._path, "rb") as f: + f.seek(self._offset) + chunk = f.read() + except OSError: + return self._total + end = chunk.rfind(b"\n") + 1 # leave a half-written last line for the next read + self._offset += end + for line in chunk[:end].decode("utf-8", "replace").splitlines(): + m = self._LINE.search(line) + if m: + self._total += int(m.group(1)) + return self._total + + +def sweep_partials(partial_root: str, max_age: float = PARTIAL_MAX_AGE_SECONDS, + now: Optional[float] = None) -> None: + """Remove partial files that no retry picked up within ``max_age``.""" + now = time.time() if now is None else now + for dirpath, _dirs, files in os.walk(partial_root): + for name in files: + path = os.path.join(dirpath, name) + try: + if now - os.path.getmtime(path) > max_age: + os.remove(path) + except OSError: + pass diff --git a/static/conex-core/conex_core/server.py b/static/conex-core/conex_core/server.py index 4dd74d876..acad4fe19 100644 --- a/static/conex-core/conex_core/server.py +++ b/static/conex-core/conex_core/server.py @@ -7,6 +7,7 @@ import json import logging +import os import sys from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import parse_qs, urlsplit @@ -18,6 +19,7 @@ from .ports import ExportStream from .registry import Registry from .router import dispatch +from .rsync_share import RsyncDaemon, RsyncPuller from .share_ops import ShareOps _MAX_BODY = 1 * 1024 * 1024 # a plan request is small; cap the read defensively @@ -163,8 +165,13 @@ def serve(config: Config) -> None: format="%(asctime)s conex-core %(levelname)s %(message)s") registry = Registry.load(config) ops = ExportOps(registry, config.staging_dir) - import_ops = ImportOps(registry, registry.import_root) - share = ShareOps(registry, ops) + import_ops = ImportOps(registry, registry.import_root, partial_root=config.partial_dir, + puller=RsyncPuller(config.rsync_bin)) + rsync = None + if config.share_rsync_port: + rsync = RsyncDaemon(os.path.join(config.staging_dir, "rsyncd"), config.share_rsync_port, + config.rsync_bin) + share = ShareOps(registry, ops, rsync=rsync) # Reclaim op temps left by a dead process before serving (the process-death cleanup owner): a # fresh process holds no ops, so any op temp on disk is an orphan. Export reclaims its staging # temps, import its work temps; the share sweep then drops any in-memory sessions (K2GO-376 diff --git a/static/conex-core/conex_core/share_ops.py b/static/conex-core/conex_core/share_ops.py index 8207979b0..301eaaa35 100644 --- a/static/conex-core/conex_core/share_ops.py +++ b/static/conex-core/conex_core/share_ops.py @@ -51,6 +51,15 @@ revokes the token) and scoped to the item set. The only surface a token holder reaches is /v1/share/* scoped to this session's items; the wider /v1 API is never widened. +rsync transport (K2GO-577): when the box runs the share rsync daemon, each item an adapter can +list as files is also a read-only daemon module, named "-", with the session +token as its password. The manifest advertises it per item ({"rsync": {port, module}}), and a +receiver that knows rsync pulls it straight into place (resumable) instead of the HTTP fetch. The +session owns its modules: create adds them, every close (COMPLETE, CANCEL, safety cap) removes +them, and the last close stops the daemon. The receiver reports per-item progress to the gate +(POST .../progress), which keeps the session fresh during a long pull (the daemon itself reports +nothing) and gives the sender its progress view (GET .../progress). + This slice is loopback only: pairing (QR, hotspot, WLAN) is a later presentation slice and is outside the core, the way plugging in the Ethernet cable is not part of content exchange. """ @@ -64,6 +73,7 @@ from typing import Any, Dict, List, Tuple, Union from .registry import Registry +from .rsync_share import Module Response = Tuple[int, Any] @@ -87,8 +97,12 @@ class _Session: delivered: set = field(default_factory=set) # itemRefs with an open (or opening) stream: single-flight + "do not cap mid-transfer". streaming: set = field(default_factory=set) - # Last activity, for the safety cap only (bumped on manifest / fetch / complete / close). + # Last activity, for the safety cap only (bumped on manifest / fetch / progress / complete). last_seen: float = field(default_factory=time.monotonic) + # itemRef -> the rsync daemon module that serves it (K2GO-577). Empty without the daemon. + modules: Dict[str, Module] = field(default_factory=dict) + # itemRef -> the receiver's last progress report {movedBytes, totalBytes, state}. + progress: Dict[str, dict] = field(default_factory=dict) class ShareOps: @@ -97,12 +111,16 @@ class ShareOps: delegates every per-item transfer (the connection lifetime) to it.""" def __init__(self, registry: Registry, export_ops, - safety_cap_seconds: int = DEFAULT_SAFETY_CAP_SECONDS): + safety_cap_seconds: int = DEFAULT_SAFETY_CAP_SECONDS, rsync=None): self._registry = registry self._export = export_ops self._cap = safety_cap_seconds + self._rsync = rsync # the RsyncDaemon, or None: share over HTTP fetch only self._lock = threading.Lock() self._sessions: Dict[str, _Session] = {} + # Serializes snapshot + apply, so a stale module list never lands after a newer one (a + # closed session's module must not keep serving). + self._sync_lock = threading.Lock() # ---- REST-facing operations (return (status, payload), or (200, ExportStream)) ---- @@ -126,8 +144,11 @@ def create(self, items: Any) -> Response: scoped.append(key) session_id = secrets.token_hex(16) token = secrets.token_urlsafe(32) + session = _Session(token=token, items=scoped) + session.modules = self._modules_for(session_id, token, scoped) with self._lock: - self._sessions[session_id] = _Session(token=token, items=scoped) + self._sessions[session_id] = session + self._sync_daemon() # The pointer carries no token: the QR carries the address, the token and this pointer # as separate fields (ADR), and the receiver sends the token as the X-Share-Token header. return 200, { @@ -151,6 +172,7 @@ def manifest(self, session_id: str, token: Any) -> Response: found: List[dict] = [] missing: List[dict] = [] total = 0 + serving = self._ensure_serving(session) for index, (app, local_ref) in enumerate(session.items): item_ref = str(index) if not self._registry.has(app): @@ -165,6 +187,9 @@ def manifest(self, session_id: str, token: Any) -> Response: entry = item.to_json() entry["itemRef"] = item_ref entry["delivered"] = item_ref in session.delivered + module = session.modules.get(item_ref) + if module is not None and serving: + entry["rsync"] = {"port": self._rsync.port, "module": module.name} found.append(entry) total += item.bytes return 200, {"sessionId": session_id, "items": found, "missing": missing, @@ -219,6 +244,46 @@ def fetch(self, session_id: str, item_ref: str, token: Any): if not keep: self._release_streaming(session_id, item_ref) + def report_progress(self, session_id: str, token: Any, body: Any) -> Response: + """The receiver reports one item's progress {itemRef, movedBytes, totalBytes, state}. + Advisory, like ``delivered``: it feeds the sender's view and keeps the session fresh; a + state "done" also marks the item delivered.""" + session = self._resolve(session_id, token) + if isinstance(session, tuple): + return session + if not isinstance(body, dict): + return 400, {"error": "body must be {itemRef, movedBytes, totalBytes, state}"} + item_ref = body.get("itemRef") + if not isinstance(item_ref, str) or self._item(session, item_ref) is None: + return 404, {"error": "unknown item"} + report = {} + for key in ("movedBytes", "totalBytes"): + value = body.get(key) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return 400, {"error": f"{key} must be a non-negative integer"} + report[key] = value + state = body.get("state") + if state not in ("running", "done", "error"): + return 400, {"error": "state must be running, done or error"} + report["state"] = state + if state == "running": + self._ensure_serving(session) # a Retry pull needs the daemon up + with self._lock: + session.progress[item_ref] = report + if state == "done": + session.delivered.add(item_ref) + return 200, {"ok": True} + + def progress(self, session_id: str, token: Any) -> Response: + """The sender's view: the last progress the receiver reported per item.""" + session = self._resolve(session_id, token) + if isinstance(session, tuple): + return session + with self._lock: + items = [dict(report, itemRef=ref, delivered=ref in session.delivered) + for ref, report in sorted(session.progress.items(), key=lambda kv: int(kv[0]))] + return 200, {"sessionId": session_id, "items": items} + def complete(self, session_id: str, token: Any) -> Response: """Clean-success close: the receiver has every item it wanted and releases the session. This is the receiver-driven close (supersedes any drop-based auto-close). Revokes the @@ -241,6 +306,8 @@ def sweep_startup(self) -> None: that single sweep avoids a second temp owner.""" with self._lock: self._sessions.clear() + if self._rsync is not None: + self._rsync.sweep_startup() # ---- token + item resolution ---- @@ -274,6 +341,8 @@ def _close(self, session_id: str, token: Any) -> Response: child_ops = list(session.item_ops.values()) for op_id in child_ops: self._export.delete(op_id) # drops each child op + removes its staged temp + if session.modules: + self._sync_daemon() # revoke its rsync modules too (and stop an idle daemon) return 200, {"ok": True} @staticmethod @@ -392,3 +461,41 @@ def _sweep_capped(self) -> None: for session in capped: for op_id in list(session.item_ops.values()): self._export.delete(op_id) + if any(session.modules for session in capped): + self._sync_daemon() + + # ---- rsync daemon (K2GO-577) ---- + + def _modules_for(self, session_id: str, token: str, items) -> Dict[str, Module]: + """One daemon module per item the adapter can list as files. Empty without the daemon.""" + if self._rsync is None: + return {} + modules: Dict[str, Module] = {} + for index, (app, local_ref) in enumerate(items): + if not self._registry.has(app): + continue + files = self._registry.get(app).share_files(local_ref) + if files is not None: + item_ref = str(index) + modules[item_ref] = Module(name=f"{session_id}-{item_ref}", root=files.root, + files=list(files.files), secret=token) + return modules + + def _ensure_serving(self, session: _Session) -> bool: + """True when the daemon serves this session's modules, restarting a daemon that died while + the session was open (set_modules restarts a dead process).""" + if self._rsync is None or not session.modules: + return False + if not self._rsync.running(): + self._sync_daemon() + return self._rsync.running() + + def _sync_daemon(self) -> None: + """Make the daemon serve exactly the modules of the live sessions.""" + if self._rsync is None: + return + with self._sync_lock: + with self._lock: + modules = [m for session in self._sessions.values() + for m in session.modules.values()] + self._rsync.set_modules(modules) diff --git a/static/conex-core/conex_core/watchdog.py b/static/conex-core/conex_core/watchdog.py index 639063e34..f6e2baed0 100644 --- a/static/conex-core/conex_core/watchdog.py +++ b/static/conex-core/conex_core/watchdog.py @@ -161,6 +161,17 @@ def _kill(proc: subprocess.Popen) -> None: pass +def _die_with_parent() -> None: + """In the child, before exec: get SIGTERM when the parent dies (Linux PR_SET_PDEATHSIG). A pull + must not outlive the core that owns it, or a Retry starts a second writer on the same partial + file. Best effort: a host without prctl keeps the default.""" + try: + import ctypes + ctypes.CDLL(None).prctl(1, signal.SIGTERM) # 1 = PR_SET_PDEATHSIG + except Exception: + pass + + def classify_exit(returncode: int) -> SuperviseResult: """Map a child return code to a result. A negative code is death by signal; signal 9 (SIGKILL) on a child that was not cancelled is almost always the OOM killer, so it is named as such.""" @@ -181,14 +192,16 @@ def run_supervised(argv, env, *, dest_dir: Optional[str] = None, poll_seconds: float = 1.0, floor_seconds: float = 120.0, gap_multiplier: float = 4.0, cpu_grace: float = 3.0, on_progress: Optional[Callable[[int], None]] = None, - log=None) -> SuperviseResult: + log=None, activity_fn: Optional[Callable[[], int]] = None, + die_with_parent: bool = False) -> SuperviseResult: """Run argv as a child and supervise it by liveness. The child's stdout/stderr go to DEVNULL (the caller owns user output). dest_dir is the directory whose growth is the byte-progress fallback when /proc//io is unreadable. should_cancel is polled each interval; True terminates the child and returns CANCELLED. on_progress(activity_bytes) is called each poll so the caller can surface progress.""" try: - proc = subprocess.Popen(argv, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + proc = subprocess.Popen(argv, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + preexec_fn=_die_with_parent if die_with_parent and os.name == "posix" else None) except OSError as exc: # The child could not start at all (a missing binary, a bad interpreter). Own it here as a # clean failure rather than letting the raw OSError escape the supervisor. @@ -198,6 +211,13 @@ def run_supervised(argv, env, *, dest_dir: Optional[str] = None, use_io = proc_io_bytes(pid) is not None def activity() -> int: + # A caller that knows where the work lands measures it directly (an rsync pull: the + # parent process is the generator and moves almost no bytes while a child receives). + if activity_fn is not None: + try: + return activity_fn() + except Exception: # a probe must never break supervision + return 0 if use_io: value = proc_io_bytes(pid) if value is not None: diff --git a/static/conex-core/tests/test_kolibri_pull.py b/static/conex-core/tests/test_kolibri_pull.py new file mode 100644 index 000000000..7feaabd64 --- /dev/null +++ b/static/conex-core/tests/test_kolibri_pull.py @@ -0,0 +1,181 @@ +"""Kolibri over rsync (K2GO-577): the share file list, the pull steps, the finalize commands and +the import manifest. No rsync and no Kolibri run: the transfer and the CLI are fakes.""" + +import json +import os +import sqlite3 +import tempfile +import unittest + +from conex_core.adapters.kolibri import KolibriAdapter, pull_finalize_commands +from conex_core.rsync_share import _LogTally +from conex_core.watchdog import CANCELLED, FAILED, OK, SuperviseResult + +CID = "9b3463eaa85354eeb26a184fe1d9a04b" +F1 = "8b0a7472152ff9b4b46cacd2761d52d1" +F2 = "c61f30d1875c3b4b9a883d304c553956" +F3 = "8d08d00cd420f7ce94411d685d2e553e" +SOURCE = {"host": "192.168.43.1", "port": 9873, "module": "abc-0", "token": "tok"} + + +def _home(): + """A KOLIBRI_HOME with a main DB: two available files and one missing, three nodes.""" + home = tempfile.mkdtemp() + os.makedirs(os.path.join(home, "content", "databases")) + open(os.path.join(home, "content", "databases", CID + ".sqlite3"), "w").close() + con = sqlite3.connect(os.path.join(home, "db.sqlite3")) + con.executescript(""" + CREATE TABLE content_channelmetadata (id TEXT, name TEXT, version INTEGER); + CREATE TABLE content_contentnode (id TEXT, channel_id TEXT, available INTEGER, kind TEXT); + CREATE TABLE content_file (contentnode_id TEXT, local_file_id TEXT); + CREATE TABLE content_localfile (id TEXT, extension TEXT, file_size INTEGER, available INTEGER); + """) + con.execute("INSERT INTO content_channelmetadata VALUES (?, 'Digital Awareness', 8)", (CID,)) + con.executemany("INSERT INTO content_contentnode VALUES (?, ?, ?, ?)", [ + ("root", CID, 1, "topic"), ("n1", CID, 1, "video"), ("n2", CID, 0, "h5p")]) + con.executemany("INSERT INTO content_file VALUES (?, ?)", [ + ("n1", F1), ("n1", F3), ("n2", F2), ("root", F1)]) + con.executemany("INSERT INTO content_localfile VALUES (?, ?, ?, ?)", [ + (F1, "png", 10, 1), (F3, "mp4", 30, 1), (F2, "h5p", 20, 0)]) + con.commit() + con.close() + return home + + +class ShareFilesTests(unittest.TestCase): + def test_lists_the_db_and_only_available_files(self): + home = _home() + files = KolibriAdapter(os.path.join(home, "db.sqlite3")).share_files(CID) + self.assertEqual(files.root, os.path.join(home, "content")) + self.assertEqual(files.files[0], f"databases/{CID}.sqlite3") + self.assertEqual(sorted(files.files[1:]), sorted([ + f"storage/8/b/{F1}.png", f"storage/8/d/{F3}.mp4"])) + + def test_unknown_or_unsafe_channel_is_not_shared(self): + adapter = KolibriAdapter(os.path.join(_home(), "db.sqlite3")) + self.assertIsNone(adapter.share_files("../etc")) + self.assertIsNone(adapter.share_files("0" * 32)) # no channel DB on disk + + +class _Puller: + def __init__(self, states=(OK, OK)): + self.states = list(states) + self.calls = [] + + def _result(self): + state = self.states.pop(0) + return SuperviseResult(state, 0 if state == OK else 12, "ok" if state == OK else "exited with code 12") + + def pull_tree(self, source, dest_dir, work_dir, filters=(), ignore_existing=True, + should_cancel=None, on_progress=None): + self.calls.append((dest_dir, tuple(filters), ignore_existing)) + if on_progress: + on_progress(40) + return self._result() + + +class _Adapter(KolibriAdapter): + def __init__(self, db, finalize_state=OK): + super().__init__(db) + self.finalized = [] + self._state = finalize_state + + def _finalize_pull(self, channel_id, src, should_cancel=None): + self.finalized.append((channel_id, src)) + return SuperviseResult(self._state, 0, "ok" if self._state == OK else "exited with code 1") + + +class PullItemTests(unittest.TestCase): + def setUp(self): + self.home = _home() + self.partial = tempfile.mkdtemp() + self.entry = {"identity": {"localRef": CID}} + + def test_pulls_db_then_storage_then_links(self): + adapter, puller = _Adapter(os.path.join(self.home, "db.sqlite3")), _Puller() + moved = [] + result = adapter.pull_item(self.entry, SOURCE, self.partial, puller, on_progress=moved.append) + self.assertTrue(result["ok"]) + db_dest, db_filters, db_ignore = puller.calls[0] + self.assertTrue(os.path.normpath(db_dest).endswith(os.path.join("src", "content"))) + self.assertEqual(db_filters, ("+ /databases/", f"+ /databases/{CID}.sqlite3", "- *")) + self.assertFalse(db_ignore) + self.assertEqual(puller.calls[1], (os.path.join(self.home, "content"), + ("- /databases/",), True)) + self.assertEqual(moved, [40]) + self.assertEqual(len(adapter.finalized), 1) + self.assertFalse(os.path.exists(os.path.join(self.partial, "kolibri", CID))) # work dir gone + + def test_transfer_drop_keeps_the_work_dir_and_skips_linking(self): + adapter, puller = _Adapter(os.path.join(self.home, "db.sqlite3")), _Puller((OK, FAILED)) + result = adapter.pull_item(self.entry, SOURCE, self.partial, puller) + self.assertFalse(result["ok"]) + self.assertIn("transfer", result["reason"]) + self.assertEqual(adapter.finalized, []) + self.assertTrue(os.path.isdir(os.path.join(self.partial, "kolibri", CID))) + + def test_cancel_and_link_failure(self): + adapter = _Adapter(os.path.join(self.home, "db.sqlite3")) + result = adapter.pull_item(self.entry, SOURCE, self.partial, _Puller((CANCELLED,))) + self.assertEqual(result["reason"], "import cancelled") + failing = _Adapter(os.path.join(self.home, "db.sqlite3"), finalize_state=FAILED) + result = failing.pull_item(self.entry, SOURCE, self.partial, _Puller()) + self.assertIn("kolibri import", result["reason"]) + + def test_invalid_channel_id(self): + adapter = _Adapter(os.path.join(self.home, "db.sqlite3")) + bad = {"identity": {"localRef": "../x"}} + self.assertFalse(adapter.pull_item(bad, SOURCE, self.partial, _Puller())["ok"]) + + +class DiscardTests(unittest.TestCase): + def test_discard_drops_the_work_dir_and_keeps_storage(self): + home, partial = _home(), tempfile.mkdtemp() + work = os.path.join(partial, "kolibri", CID, "tmp") + os.makedirs(work) + with open(os.path.join(work, ".x.mp4.ab12"), "wb") as f: + f.write(b"y" * 50) + storage_file = os.path.join(home, "content", "storage", "8", "b", F1 + ".png") + os.makedirs(os.path.dirname(storage_file)) + open(storage_file, "w").close() + result = KolibriAdapter(os.path.join(home, "db.sqlite3")).discard_pull( + {"identity": {"localRef": CID}}, partial) + self.assertEqual(result, {"ok": True, "freedBytes": 50}) + self.assertFalse(os.path.exists(os.path.join(partial, "kolibri", CID))) + self.assertTrue(os.path.exists(storage_file)) + + +class FinalizeTests(unittest.TestCase): + def test_commands_scan_before_importcontent(self): + cmds = pull_finalize_commands(CID, "/w/src", "/usr/bin/kolibri") + self.assertEqual([c[2] for c in cmds], ["importchannel", "scanforcontent", "importcontent"]) + self.assertEqual(cmds[1][3:], ["--channels", CID, "--channel-import-mode", "none"]) + self.assertEqual(cmds[0][3:], ["disk", CID, "/w/src"]) + + def test_manifest_lists_available_leaf_nodes(self): + home = _home() + src = os.path.join(tempfile.mkdtemp(), "src") + os.makedirs(os.path.join(src, "content")) + KolibriAdapter(os.path.join(home, "db.sqlite3"))._write_pull_manifest(CID, src) + with open(os.path.join(src, "content", "manifest.json")) as f: + manifest = json.load(f) + self.assertEqual(manifest, {"channels": [{"id": CID, "version": 8, + "include_node_ids": ["n1"]}]}) + + +class LogTallyTests(unittest.TestCase): + def test_sums_recv_lines_incrementally(self): + log = os.path.join(tempfile.mkdtemp(), "t.log") + tally = _LogTally(log) + self.assertEqual(tally.read(), 0) # no log yet + with open(log, "w") as f: + f.write("2026/11/16 10:00:00 [12] receiving file list\n") + f.write("2026/11/16 10:00:01 [12] recv 100\n2026/11/16 10:00:02 [12] recv 2") + self.assertEqual(tally.read(), 100) # the half-written last line waits + with open(log, "a") as f: + f.write("50\n2026/11/16 10:00:03 [12] sent 10 bytes received 999 bytes\n") + self.assertEqual(tally.read(), 350) + + +if __name__ == "__main__": + unittest.main() diff --git a/static/conex-core/tests/test_rsync_share.py b/static/conex-core/tests/test_rsync_share.py new file mode 100644 index 000000000..78df6d3ed --- /dev/null +++ b/static/conex-core/tests/test_rsync_share.py @@ -0,0 +1,465 @@ +"""rsync Share transport tests (K2GO-577): the daemon config and lifecycle, the pull client, the +partial sweep, the gate's rsync modules and progress, the pull import op and the Kiwix pull. +No real rsync runs: the process and the transfer are fakes behind the seams.""" + +import os +import tempfile +import time +import unittest + +from conex_core.adapters.kiwix import KiwixAdapter +from conex_core.import_ops import ImportOps +from conex_core.model import AUTHORITATIVE, Dedup, Identity, ManifestItem +from conex_core.ports import ContentAdapter, ShareFiles +from conex_core.registry import Registry +from conex_core.rsync_share import ( + SHARE_USER, + Module, + RsyncDaemon, + RsyncPuller, + render_config, + render_filter, + sweep_partials, + valid_source, +) +from conex_core.share_ops import ShareOps +from conex_core.watchdog import CANCELLED, FAILED, OK, SuperviseResult + +SOURCE = {"host": "192.168.43.1", "port": 9873, "module": "abc123-0", "token": "tok"} + + +class _FakeProc: + def __init__(self, pid=4242): + self.pid = pid + self.alive = True + + def poll(self): + return None if self.alive else 0 + + def wait(self, timeout=None): + return 0 + + +class _FakePopen: + def __init__(self): + self.calls = [] + self.procs = [] + + def __call__(self, argv, **kwargs): + self.calls.append((argv, kwargs)) + proc = _FakeProc(pid=1000 + len(self.calls)) + self.procs.append(proc) + return proc + + +class _Killer: + def __init__(self, popen): + self.popen = popen + self.signals = [] + + def __call__(self, pgid, sig): + self.signals.append((pgid, sig)) + for proc in self.popen.procs: + if proc.pid == pgid: + proc.alive = False + + +def _daemon(run_dir, available=True): + popen = _FakePopen() + killer = _Killer(popen) + rsync_bin = os.path.join(run_dir, "..", "rsync-bin") + if available: + open(rsync_bin, "w").close() + daemon = RsyncDaemon(run_dir, 9873, rsync_bin, popen=popen, killpg=killer) + return daemon, popen, killer + + +def _module(name="s1-0", files=("a.zim",), secret="tok", root="/library/zims/content"): + return Module(name=name, root=root, files=list(files), secret=secret) + + +class RenderTests(unittest.TestCase): + def test_filter_admits_only_the_file(self): + self.assertEqual(render_filter(["a.zim"]), "+ /a.zim\n- *\n") + + def test_filter_admits_nested_parents_once(self): + text = render_filter(["content/storage/a/b/x.mp4", "content/storage/a/c/y.png"]) + self.assertEqual(text.splitlines(), [ + "+ /content/", "+ /content/storage/", "+ /content/storage/a/", + "+ /content/storage/a/b/", "+ /content/storage/a/b/x.mp4", + "+ /content/storage/a/c/", "+ /content/storage/a/c/y.png", "- *"]) + + def test_config_is_read_only_authenticated_and_logs_nothing(self): + text = render_config([_module()], "/run/r", 10123, 10123) + for line in ("[s1-0]", " read only = yes", " list = no", f" auth users = {SHARE_USER}", + " path = /library/zims/content", "log file = /dev/null", "uid = 10123", + " secrets file = " + os.path.join("/run/r", "s1-0.secrets")): + self.assertIn(line, text.splitlines()) + + +class DaemonTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.run_dir = os.path.join(self.tmp, "rsyncd") + + def test_first_module_starts_last_close_stops(self): + daemon, popen, killer = _daemon(self.run_dir) + self.assertTrue(daemon.set_modules([_module()])) + self.assertEqual(len(popen.calls), 1) + argv, kwargs = popen.calls[0] + self.assertIn("--daemon", argv) + self.assertIn("--port=9873", argv) + self.assertTrue(kwargs.get("start_new_session")) + with open(os.path.join(self.run_dir, "s1-0.secrets")) as f: + self.assertEqual(f.read(), f"{SHARE_USER}:tok\n") + # A second session only rewrites the config: the running daemon re-reads it. + self.assertTrue(daemon.set_modules([_module(), _module(name="s2-0", secret="t2")])) + self.assertEqual(len(popen.calls), 1) + self.assertFalse(daemon.set_modules([])) + self.assertTrue(killer.signals) + self.assertFalse(os.path.exists(self.run_dir)) # secrets are gone with the daemon + + def test_closed_session_secrets_are_removed(self): + daemon, _popen, _killer = _daemon(self.run_dir) + daemon.set_modules([_module(), _module(name="s2-0")]) + daemon.set_modules([_module(name="s2-0")]) + self.assertFalse(os.path.exists(os.path.join(self.run_dir, "s1-0.secrets"))) + self.assertTrue(os.path.exists(os.path.join(self.run_dir, "s2-0.secrets"))) + + def test_unsafe_module_is_never_written(self): + daemon, popen, _killer = _daemon(self.run_dir) + bad = [_module(name="x]y"), _module(files=("a.zim\n[evil]",)), _module(secret="a\nb")] + self.assertFalse(daemon.set_modules(bad)) + self.assertEqual(popen.calls, []) + + def test_no_rsync_binary_means_not_serving(self): + daemon, popen, _killer = _daemon(self.run_dir, available=False) + self.assertFalse(daemon.set_modules([_module()])) + self.assertEqual(popen.calls, []) + + def test_startup_sweep_kills_the_recorded_group(self): + os.makedirs(self.run_dir) + with open(os.path.join(self.run_dir, "daemon.pid"), "w") as f: + f.write("777") + daemon, _popen, killer = _daemon(self.run_dir) + daemon.sweep_startup() + self.assertEqual(killer.signals[0][0], 777) + self.assertFalse(os.path.exists(self.run_dir)) + + +class SourceAndPullTests(unittest.TestCase): + def test_valid_source(self): + self.assertIsNone(valid_source(SOURCE)) + for key, value in (("host", "a b"), ("host", "-x"), ("port", 0), ("port", True), + ("port", "9873"), ("module", "../x"), ("token", ""), + ("token", "a\nb")): + self.assertIsNotNone(valid_source(dict(SOURCE, **{key: value})), (key, value)) + self.assertIsNotNone(valid_source(None)) + + def test_pull_keeps_the_password_out_of_argv_and_reports_file_size(self): + tmp = tempfile.mkdtemp() + dest = os.path.join(tmp, "a.zim") + seen = {} + + def runner(argv, env, **kwargs): + seen["argv"], seen["env"] = argv, env + with open(dest, "wb") as f: + f.write(b"x" * 10) + kwargs["on_progress"](0) + seen["activity"] = kwargs["activity_fn"]() + return SuperviseResult(OK, 0, "ok") + + moved = [] + result = RsyncPuller("/usr/bin/rsync", runner=runner).pull_file( + SOURCE, "a.zim", dest, on_progress=moved.append) + self.assertTrue(result.ok) + self.assertEqual(moved, [10]) + self.assertEqual(seen["activity"], 10) # liveness = bytes on disk, not the parent's I/O + self.assertNotIn("tok", " ".join(seen["argv"])) + self.assertEqual(seen["env"]["RSYNC_PASSWORD"], "tok") + self.assertIn("--inplace", seen["argv"]) + self.assertIn("--append-verify", seen["argv"]) + self.assertIn(f"rsync://{SHARE_USER}@192.168.43.1:9873/abc123-0/a.zim", seen["argv"]) + + def test_pull_tree_pulls_from_the_module_root_and_never_touches_other_content(self): + work, dest = tempfile.mkdtemp(), tempfile.mkdtemp() + seen = {} + + def runner(argv, env, **kwargs): + seen["argv"], seen["env"] = argv, env + os.makedirs(os.path.join(work, "tmp"), exist_ok=True) + with open(os.path.join(work, "tmp", ".x.mp4.ab12"), "wb") as f: + f.write(b"y" * 7) # the file in flight + with open(os.path.join(work, "transfer.log"), "w") as f: + f.write("2026/11/16 10:00:01 [9] recv 100" + chr(10)) + seen["activity"] = kwargs["activity_fn"]() + return SuperviseResult(OK, 0, "ok") + + os.makedirs(os.path.join(work, "tmp")) + with open(os.path.join(work, "tmp", ".orphan.mp4.zz99"), "wb") as f: + f.write(b"z" * 1000) # left by a killed run + RsyncPuller("/usr/bin/rsync", runner=runner).pull_tree( + SOURCE, dest, work, filters=("- /databases/",)) + self.assertFalse(os.path.exists(os.path.join(work, "tmp", ".orphan.mp4.zz99"))) + argv = seen["argv"] + self.assertEqual(argv[-2], f"rsync://{SHARE_USER}@192.168.43.1:9873/abc123-0/") + for flag in ("--ignore-existing", "--omit-dir-times", "--filter=- /databases/", + f"--temp-dir={os.path.join(work, 'tmp')}", + f"--partial-dir={os.path.join(work, 'kept')}"): + self.assertIn(flag, argv) + self.assertFalse(any(a.startswith("--delete") for a in argv)) + self.assertNotIn("tok", " ".join(argv)) + self.assertEqual(seen["activity"], 107) # completed files + the file in flight + + def test_sweep_removes_only_old_partials(self): + root = tempfile.mkdtemp() + os.makedirs(os.path.join(root, "kiwix")) + old, new = (os.path.join(root, "kiwix", n) for n in ("old.zim", "new.zim")) + for path in (old, new): + open(path, "w").close() + now = time.time() + os.utime(old, (now - 3 * 24 * 3600, now - 3 * 24 * 3600)) + sweep_partials(root, now=now) + self.assertFalse(os.path.exists(old)) + self.assertTrue(os.path.exists(new)) + + +def _mitem(app, local_ref, total): + return ManifestItem( + app=app, kind="zim", title=local_ref, bytes=total, + identity=Identity(local_ref, Dedup(line={}, build=None, version="", id_kind=AUTHORITATIVE))) + + +class _ShareFake(ContentAdapter): + app = "kiwix" + + def __init__(self, items, shareable=True): + self._items = dict(items) + self._shareable = shareable + + def is_installed(self): + return True + + def list(self): + return [_mitem(self.app, ref, size) for ref, size in self._items.items()] + + def share_files(self, local_ref): + if not self._shareable or local_ref not in self._items: + return None + return ShareFiles(root="/zims", files=[local_ref]) + + +class _RecDaemon: + port = 9873 + + def __init__(self, serving=True): + self.modules = None + self.serving = serving + self.swept = False + + def running(self): + return self.serving and bool(self.modules) + + def set_modules(self, modules): + self.modules = list(modules) + return self.running() + + def sweep_startup(self): + self.swept = True + + +class ShareGateRsyncTests(unittest.TestCase): + def _share(self, adapter, daemon): + return ShareOps(Registry([adapter]), export_ops=None, rsync=daemon) + + def test_manifest_advertises_the_module_and_close_revokes_it(self): + daemon = _RecDaemon() + share = self._share(_ShareFake({"a.zim": 5, "b.zim": 7}), daemon) + _, created = share.create([{"app": "kiwix", "localRef": "a.zim"}, + {"app": "kiwix", "localRef": "b.zim"}]) + sid, token = created["sessionId"], created["token"] + self.assertEqual([m.name for m in daemon.modules], [f"{sid}-0", f"{sid}-1"]) + self.assertTrue(all(m.secret == token for m in daemon.modules)) + _, manifest = share.manifest(sid, token) + self.assertEqual(manifest["items"][0]["rsync"], {"port": 9873, "module": f"{sid}-0"}) + self.assertEqual(share.delete(sid, token)[0], 200) + self.assertEqual(daemon.modules, []) + + def test_no_rsync_block_when_daemon_is_not_serving(self): + daemon = _RecDaemon(serving=False) + share = self._share(_ShareFake({"a.zim": 5}), daemon) + _, created = share.create([{"app": "kiwix", "localRef": "a.zim"}]) + _, manifest = share.manifest(created["sessionId"], created["token"]) + self.assertNotIn("rsync", manifest["items"][0]) + + def test_progress_report_and_view(self): + share = self._share(_ShareFake({"a.zim": 5}), _RecDaemon()) + _, created = share.create([{"app": "kiwix", "localRef": "a.zim"}]) + sid, token = created["sessionId"], created["token"] + report = {"itemRef": "0", "movedBytes": 3, "totalBytes": 5, "state": "running"} + self.assertEqual(share.report_progress(sid, token, report)[0], 200) + self.assertEqual(share.report_progress(sid, "bad", report)[0], 403) + self.assertEqual(share.report_progress(sid, token, dict(report, itemRef="9"))[0], 404) + self.assertEqual(share.report_progress(sid, token, dict(report, movedBytes=-1))[0], 400) + self.assertEqual(share.report_progress(sid, token, dict(report, state="x"))[0], 400) + share.report_progress(sid, token, dict(report, movedBytes=5, state="done")) + _, view = share.progress(sid, token) + self.assertEqual(view["items"], [{"itemRef": "0", "movedBytes": 5, "totalBytes": 5, + "state": "done", "delivered": True}]) + + def test_dead_daemon_is_restarted_by_manifest_and_progress(self): + daemon = _RecDaemon() + share = self._share(_ShareFake({"a.zim": 5}), daemon) + _, created = share.create([{"app": "kiwix", "localRef": "a.zim"}]) + sid, token = created["sessionId"], created["token"] + calls = [] + real_set = daemon.set_modules + daemon.set_modules = lambda modules: calls.append(len(modules)) or real_set(modules) + daemon.modules = [] # the daemon died: it serves nothing + _, manifest = share.manifest(sid, token) + self.assertEqual(calls, [1]) + self.assertIn("rsync", manifest["items"][0]) + daemon.modules = [] + share.report_progress(sid, token, {"itemRef": "0", "movedBytes": 1, "totalBytes": 5, + "state": "running"}) + self.assertEqual(calls, [1, 1]) + + def test_startup_sweep_reaches_the_daemon(self): + daemon = _RecDaemon() + self._share(_ShareFake({}), daemon).sweep_startup() + self.assertTrue(daemon.swept) + + +class _PullAdapter(_ShareFake): + def __init__(self): + super().__init__({}) + self.calls = [] + + def pull_item(self, entry, source, partial_dir, puller, should_cancel=None, on_progress=None): + self.calls.append((entry, source, partial_dir)) + on_progress(4) + return {"ok": True, "app": "kiwix", "localRef": "a.zim"} + + +class _OkPuller: + def available(self): + return True + + +class PullOpTests(unittest.TestCase): + def _wait(self, ops, op_id): + for _ in range(100): + _, st = ops.status(op_id) + if st["state"] != "running": + return st + time.sleep(0.01) + self.fail("op did not finish") + + def test_pull_runs_the_adapter_and_finishes(self): + adapter = _PullAdapter() + ops = ImportOps(Registry([adapter]), tempfile.mkdtemp(), partial_root="/p", + puller=_OkPuller()) + status, started = ops.start_pull("kiwix", {"bytes": 4}, SOURCE) + self.assertEqual(status, 200) + st = self._wait(ops, started["opId"]) + self.assertEqual(st["state"], "done") + self.assertEqual(adapter.calls[0][1], SOURCE) + self.assertEqual(adapter.calls[0][2], "/p") + + def test_pull_rejects_bad_requests(self): + ops = ImportOps(Registry([_PullAdapter()]), tempfile.mkdtemp(), partial_root="/p", + puller=_OkPuller()) + self.assertEqual(ops.start_pull("nope", {}, SOURCE)[0], 404) + self.assertEqual(ops.start_pull("kiwix", None, SOURCE)[0], 400) + self.assertEqual(ops.start_pull("kiwix", {}, dict(SOURCE, port=0))[0], 400) + no_pull = ImportOps(Registry([_PullAdapter()]), tempfile.mkdtemp()) + self.assertEqual(no_pull.start_pull("kiwix", {}, SOURCE)[0], 501) + + +class DiscardTests(unittest.TestCase): + def test_kiwix_discard_frees_the_kept_partial(self): + partial = tempfile.mkdtemp() + os.makedirs(os.path.join(partial, "kiwix")) + with open(os.path.join(partial, "kiwix", "a_en_all_2026-09.zim"), "wb") as f: + f.write(b"z" * 123) + adapter = KiwixAdapter(tempfile.mkdtemp(), indexer="") + entry = {"identity": {"localRef": "a_en_all_2026-09.zim"}} + self.assertEqual(adapter.discard_pull(entry, partial), {"ok": True, "freedBytes": 123}) + self.assertEqual(adapter.discard_pull(entry, partial), {"ok": True, "freedBytes": 0}) + self.assertFalse(adapter.discard_pull({"identity": {"localRef": "../x.zim"}}, partial)["ok"]) + + def test_discard_is_refused_while_a_pull_of_the_item_runs(self): + class _Slow(_PullAdapter): + def pull_item(self, entry, source, partial_dir, puller, should_cancel=None, on_progress=None): + while not should_cancel(): + time.sleep(0.01) + return {"ok": False, "reason": "import cancelled"} + + def discard_pull(self, entry, partial_dir): + return {"ok": True, "freedBytes": 9} + + ops = ImportOps(Registry([_Slow()]), tempfile.mkdtemp(), partial_root="/p", puller=_OkPuller()) + entry = {"identity": {"localRef": "a.zim"}} + _, started = ops.start_pull("kiwix", entry, SOURCE) + self.assertEqual(ops.discard("kiwix", entry)[0], 409) + ops.delete(started["opId"]) # the client aborts the pull first + self.assertEqual(ops.discard("kiwix", entry), (200, {"ok": True, "freedBytes": 9})) + + +class _FakePuller: + def __init__(self, state=OK, data=b"ZIMDATA"): + self.state = state + self.data = data + self.calls = [] + + def pull_file(self, source, remote_path, dest_path, should_cancel=None, on_progress=None): + self.calls.append((remote_path, dest_path)) + with open(dest_path, "ab") as f: + f.write(self.data) + if on_progress: + on_progress(os.path.getsize(dest_path)) + detail = "ok" if self.state == OK else "exited with code 12" + return SuperviseResult(self.state, 0 if self.state == OK else 12, detail) + + +class KiwixPullTests(unittest.TestCase): + def setUp(self): + self.zims = tempfile.mkdtemp() + self.partial = tempfile.mkdtemp() + self.adapter = KiwixAdapter(self.zims, indexer="") + self.entry = {"identity": {"localRef": "wiki_en_all_2026-09.zim"}} + + def test_success_renames_into_place(self): + puller = _FakePuller() + result = self.adapter.pull_item(self.entry, SOURCE, self.partial, puller) + self.assertTrue(result["ok"]) + self.assertEqual(puller.calls[0][0], "wiki_en_all_2026-09.zim") + self.assertTrue(os.path.isfile(os.path.join(self.zims, "wiki_en_all_2026-09.zim"))) + self.assertFalse(os.path.exists(puller.calls[0][1])) # the partial became the file + + def test_drop_keeps_the_partial_for_a_resume(self): + puller = _FakePuller(state=FAILED) + result = self.adapter.pull_item(self.entry, SOURCE, self.partial, puller) + self.assertFalse(result["ok"]) + self.assertTrue(os.path.isfile(puller.calls[0][1])) + self.assertFalse(os.path.exists(os.path.join(self.zims, "wiki_en_all_2026-09.zim"))) + + def test_cancel_keeps_the_partial(self): + puller = _FakePuller(state=CANCELLED) + result = self.adapter.pull_item(self.entry, SOURCE, self.partial, puller) + self.assertEqual(result["reason"], "import cancelled") + self.assertTrue(os.path.isfile(puller.calls[0][1])) + + def test_unsafe_local_ref_is_rejected(self): + bad = {"identity": {"localRef": "../x.zim"}} + self.assertFalse(self.adapter.pull_item(bad, SOURCE, self.partial, _FakePuller())["ok"]) + + def test_share_files_lists_only_the_zim(self): + open(os.path.join(self.zims, "a.zim"), "w").close() + files = self.adapter.share_files("a.zim") + self.assertEqual((files.root, files.files), (self.zims, ["a.zim"])) + self.assertIsNone(self.adapter.share_files("missing.zim")) + + +if __name__ == "__main__": + unittest.main() diff --git a/static/conex-core/tests/test_watchdog.py b/static/conex-core/tests/test_watchdog.py index d32dfa4f8..3bc57c761 100644 --- a/static/conex-core/tests/test_watchdog.py +++ b/static/conex-core/tests/test_watchdog.py @@ -118,5 +118,24 @@ def test_on_progress_is_called(self): self.assertTrue(seen) # at least one progress sample while the child ran + def test_activity_fn_is_the_liveness_signal(self): + # A child that does nothing itself, while the work it drives moves (an rsync generator + # whose receiver child writes): growing activity keeps it alive past the floor. + moved = [0] + + def growing(): + moved[0] += 1 + return moved[0] + + alive = run_supervised( + [sys.executable, "-c", "import time; time.sleep(0.5)"], None, poll_seconds=0.02, + floor_seconds=0.1, activity_fn=growing) + self.assertEqual(alive.state, OK) + stalled = run_supervised( + [sys.executable, "-c", "import time; time.sleep(5)"], None, poll_seconds=0.02, + floor_seconds=0.1, cpu_grace=1.0, activity_fn=lambda: 7) + self.assertEqual(stalled.state, STALLED) + + if __name__ == "__main__": unittest.main() diff --git a/static/dashboard/CHANGELOG.md b/static/dashboard/CHANGELOG.md index dfe1f268c..e02065676 100644 --- a/static/dashboard/CHANGELOG.md +++ b/static/dashboard/CHANGELOG.md @@ -4,6 +4,7 @@ One line per version, newest first. Every REST-facing change bumps the version i (the app surfaces it via `/system/dashboard/update-check` and the "Update available" pill), so this file is the human record of what each bump enables. Keep entries short: `version - change (TICKET)`. +- **1.6.3** - The `/v1` relay to conex-core forwards the `X-Share-Token` header (never logged). Before, it was dropped, so the sender's close (`DELETE /v1/share/`) got 401 and the share token stayed valid after Done. (K2GO-577) - **1.6.2** - Box logs no longer identify the user, on boxes installed before bake patches 0010-0012 (K2GO-559). At its first start, dash-node applies the same change once: nginx access log without client IP and User-Agent (`/etc/nginx/nginx.conf`), Forgejo router log off (`LOGGER_ROUTER_MODE` empty in `app.ini`), Calibre-Web log level DEBUG -> INFO (`app.db`). A value an admin changed is kept. Each change applies at the service restart (Forgejo: at the next app open, as a pending privacy change; nginx and Calibre-Web: at the next box start). After that restart, the older logs of each changed service are removed once (rotated copies deleted, live log emptied). Markers: `/library/dashboard/log-less.done`, `/library/dashboard/log-less-purge.done`. No REST change. (K2GO-559) - **1.6.1** - The box restarts conex-core (Content Exchange) on its own when its process is gone: service-heal probes `/v1/apps` and runs `pdsm restart conex-core` only on a refused connection (a busy, slow service is never restarted mid-operation), and skips boxes without the conex-core pdsm service. `POST /system/service/conex-core/restart` is allowed. Localhost-only. (K2GO-574) - **1.6.0** - Privacy switches for the box services that contact an outside server on their own (K2GO-559). `GET /system/privacy` reports each one (Kolibri statistics ping, Forgejo update check): installed, and whether it may contact its server. `POST /system/privacy/:service {outbound}` sets the service's own config key (Kolibri `[Deployment] DISABLE_PING` in `options.ini`, Forgejo `[cron.update_checker] ENABLED` in `app.ini`) and only writes the config; it never restarts a service. `pending` reports a change that waits for a restart (config newer than the running process; derived on each read, not stored). The app asks the user (Restart now / Later) and requests the restart through `POST /system/service/:svc/restart` (ADR-5343), which now runs one restart per service at a time; a pending change is restarted at the next app open. A missing key reads as the service default (on). On its first start a box sets every installed switch OFF once (new bakes ship OFF), then writes `/library/dashboard/privacy-defaults.done` so a switch the user turns on later stays on; it applies like any pending change. Localhost-only. (K2GO-559) diff --git a/static/dashboard/package.json b/static/dashboard/package.json index d965e168a..bebc0d240 100644 --- a/static/dashboard/package.json +++ b/static/dashboard/package.json @@ -1,6 +1,6 @@ { "name": "dashboard-console", - "version": "1.6.2", + "version": "1.6.3", "description": "", "main": "index.js", "scripts": { diff --git a/static/dashboard/sockets/conex-proxy.test.ts b/static/dashboard/sockets/conex-proxy.test.ts index f718aa94b..dec2f5056 100644 --- a/static/dashboard/sockets/conex-proxy.test.ts +++ b/static/dashboard/sockets/conex-proxy.test.ts @@ -125,6 +125,34 @@ test('conexProxy passes Content-Disposition through (the export filename)', asyn } }); +test('conexProxy forwards X-Share-Token (the sender close DELETE)', async () => { + const core = http.createServer((req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ token: req.headers['x-share-token'] ?? null, method: req.method })); + }); + const corePort = await listen(core); + process.env.CONEX_PORT = String(corePort); + const app = await startProxyApp(); + try { + const r = await new Promise<{ status: number; body: string }>((resolve, reject) => { + const req = http.request({ host: '127.0.0.1', port: app.port, path: '/v1/share/abc', + method: 'DELETE', headers: { 'x-share-token': 'tok-123' } }, (res) => { + let data = ''; + res.on('data', (c) => { data += c; }); + res.on('end', () => resolve({ status: res.statusCode || 0, body: data })); + }); + req.on('error', reject); + req.end(); + }); + assert.equal(r.status, 200); + assert.deepEqual(JSON.parse(r.body), { token: 'tok-123', method: 'DELETE' }); + const none = await get(app.port, '/v1/apps'); + assert.equal(JSON.parse(none.body).token, null); + } finally { + app.close(); core.close(); + } +}); + test('conexProxy returns 502 when the core is unreachable', async () => { // Point at a port with nothing listening. const dead = http.createServer(); diff --git a/static/dashboard/sockets/conex-proxy.ts b/static/dashboard/sockets/conex-proxy.ts index da22d3588..0e63c161a 100644 --- a/static/dashboard/sockets/conex-proxy.ts +++ b/static/dashboard/sockets/conex-proxy.ts @@ -33,6 +33,10 @@ export function conexProxy(req: Request, res: Response): void { headers['content-type'] = 'application/json'; headers['content-length'] = String(payload.length); } + // The share-session token is the gate's only auth (the sender's close DELETE carries it). + // Forward it, and never log it. + const shareToken = req.headers['x-share-token']; + if (typeof shareToken === 'string' && shareToken) headers['x-share-token'] = shareToken; const upstream = http.request( { host: CONEX_HOST, port: conexPort(), method: req.method, path: corePath, headers },