diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreDataSource.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreDataSource.java index eb8891339..6d7d7f5e0 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreDataSource.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreDataSource.java @@ -84,6 +84,11 @@ public JSONObject postImportStart(JSONObject body) throws Exception { return postJson(V1 + "/import/item", body); } + /** {@code POST /v1/import/discard} with {@code {app, entry}}: drop a given-up pull's kept bytes. */ + public JSONObject postImportDiscard(JSONObject body) throws Exception { + return postJson(V1 + "/import/discard", body); + } + /** * {@code GET /v1/import/status/{opId}} -> {@code {state, movedBytes, totalBytes, error?, code?, * result?}}. A 404 means the op was swept; it surfaces as {@link ConexHttpException} with diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreImportSource.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreImportSource.java index d8fed15e4..bbf7b05cb 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreImportSource.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ConexCoreImportSource.java @@ -16,6 +16,7 @@ import org.appdevforall.k2go.contentexchange.domain.ImportStatus; import org.appdevforall.k2go.contentexchange.domain.ImportVerdict; import org.appdevforall.k2go.contentexchange.domain.ItemRef; +import org.appdevforall.k2go.contentexchange.domain.PullUnsupportedException; import org.json.JSONArray; import org.json.JSONObject; @@ -67,6 +68,46 @@ public String startImport(String app, String entryJson, String payloadBoxPath) t return opId; } + @Override + public String startPull(String app, String entryJson, String host, int port, String module, + String token) throws Exception { + JSONObject source = new JSONObject() + .put("host", host) + .put("port", port) + .put("module", module) + .put("token", token); + JSONObject body = new JSONObject() + .put("app", app) + .put("entry", new JSONObject(entryJson)) + .put("source", source); + JSONObject r; + try { + r = dataSource.postImportStart(body); + } catch (ConexCoreDataSource.ConexHttpException e) { + // 400: a core older than the pull wants a payloadPath. 501: no rsync on this box. + if (e.code() == 400 || e.code() == 501) { + throw new PullUnsupportedException(e.getMessage()); + } + throw e; + } + String opId = r.optString("opId", ""); + if (opId.isEmpty()) { + throw new Exception("pull start returned no opId"); + } + return opId; + } + + @Override + public void discardPull(String app, String entryJson) { + try { + dataSource.postImportDiscard(new JSONObject() + .put("app", app) + .put("entry", new JSONObject(entryJson))); + } catch (Exception ignored) { + // best effort: the box sweeps partials older than 48 h on its own + } + } + @Override public ImportStatus importStatus(String opId) throws Exception { JSONObject r; diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/NetworkImportCarrier.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/NetworkImportCarrier.java index afad31cd2..dae7859c9 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/NetworkImportCarrier.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/NetworkImportCarrier.java @@ -36,6 +36,9 @@ import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * One Share session as an {@link ImportCarrier}, read over the hotspot from the sender's gate. The @@ -81,6 +84,8 @@ public interface Cancellation { private final String token; private final String manifestUrl; private final Cancellation cancel; + /** Progress reports to the sender (K2GO-577). Its one thread ends on its own when idle. */ + private final ThreadPoolExecutor reporter; public NetworkImportCarrier(Context context, String host, int port, String token, String manifestUrl, Cancellation cancel) { @@ -90,6 +95,13 @@ public NetworkImportCarrier(Context context, String host, int port, String token this.token = token; this.manifestUrl = manifestUrl; this.cancel = cancel != null ? cancel : () -> false; + this.reporter = new ThreadPoolExecutor(1, 1, 10L, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(1), r -> { + Thread t = new Thread(r, "share-progress"); + t.setDaemon(true); + return t; + }, new ThreadPoolExecutor.DiscardOldestPolicy()); + this.reporter.allowCoreThreadTimeOut(true); } @Override @@ -179,6 +191,43 @@ public void onAllDelivered() { } } + /** + * Best-effort report of one item's progress to the sender's gate (K2GO-577): it feeds the + * sender's view and keeps its session fresh during a long rsync pull, which the gate cannot see. + * {@code state} is "running", "done" or "error". A failure is logged and ignored. + */ + public void reportProgress(String itemRef, long movedBytes, long totalBytes, String state) { + if (!ShareUrls.isValidItemRef(itemRef)) { + return; + } + // Off the import thread, so an unreachable sender never stalls the poll loop. One report + // in flight and one queued; an older queued report is dropped for the newer one. + reporter.execute(() -> postProgress(itemRef, movedBytes, totalBytes, state)); + } + + private void postProgress(String itemRef, long movedBytes, long totalBytes, String state) { + try { + JSONObject body = new JSONObject() + .put("itemRef", itemRef) + .put("movedBytes", Math.max(0L, movedBytes)) + .put("totalBytes", Math.max(0L, totalBytes)) + .put("state", state); + HttpURLConnection c = open(ShareUrls.progress(baseUrl, manifestUrl), "POST", READ_TIMEOUT_MS); + try { + c.setDoOutput(true); + c.setRequestProperty("Content-Type", "application/json"); + try (OutputStream os = c.getOutputStream()) { + os.write(body.toString().getBytes(StandardCharsets.UTF_8)); + } + c.getResponseCode(); + } finally { + c.disconnect(); + } + } catch (Exception e) { + Log.i(TAG, "share progress report failed: " + e.getMessage()); + } + } + // ---- HTTP plumbing ------------------------------------------------------------------------- /** Open a connection to the sender, pinned to the Network that reaches its hotspot when one exists, diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareRelay.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareRelay.java new file mode 100644 index 000000000..9d891422c --- /dev/null +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareRelay.java @@ -0,0 +1,143 @@ +/* + * ============================================================================ + * Name : ShareRelay.java + * Author : AppDevForAll + * Copyright : Copyright (c) 2026 AppDevForAll + * Description : K2GO-577. A loopback TCP relay from the box to the sender's rsync daemon. The box's + * rsync is a Linux process that Android routes by the default network, which on a + * hotspot without internet can be cellular, so it may never reach the sender. The relay + * accepts on 127.0.0.1 and opens each outbound socket on the Network whose subnet holds + * the sender (WifiNetworkBinder), the same pinning NetworkImportCarrier uses for HTTP. + * ============================================================================ + */ +package org.appdevforall.k2go.contentexchange.data; + +import android.content.Context; +import android.net.Network; +import android.util.Log; + +import org.appdevforall.k2go.sync.transport.WifiNetworkBinder; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Forwards each loopback connection to {@code host:port}. Lifecycle: the import service opens one per + * network run and closes it when the run ends; close() also ends every live connection, so a stopped + * run never leaves a transfer flowing. + */ +public final class ShareRelay implements Closeable { + + private static final String TAG = "IIAB-ShareRelay"; + private static final int CONNECT_TIMEOUT_MS = 15000; + private static final int BUFFER = 64 * 1024; + + private final Context appContext; + private final String host; + private final int port; + private final ServerSocket server; + private final Set live = ConcurrentHashMap.newKeySet(); + private volatile boolean closed; + + public ShareRelay(Context context, String host, int port) throws IOException { + this.appContext = context.getApplicationContext(); + this.host = host; + this.port = port; + this.server = new ServerSocket(0, 4, InetAddress.getByName("127.0.0.1")); + Thread accept = new Thread(this::acceptLoop, "share-relay-accept"); + accept.setDaemon(true); + accept.start(); + } + + /** The loopback port the box connects to. */ + public int localPort() { + return server.getLocalPort(); + } + + @Override + public void close() { + closed = true; + closeQuietly(server); + for (Socket s : live) { + closeQuietly(s); + } + live.clear(); + } + + private void acceptLoop() { + while (!closed) { + Socket inbound; + try { + inbound = server.accept(); + } catch (IOException e) { + return; // closed + } + live.add(inbound); + Thread t = new Thread(() -> bridge(inbound), "share-relay-conn"); + t.setDaemon(true); + t.start(); + } + } + + private void bridge(Socket inbound) { + Socket outbound = null; + try { + // Resolve the Network per connection, so a Wi-Fi reconnect is picked up on the next pull. + Network net = WifiNetworkBinder.findHostNetwork(appContext, host); + outbound = net != null ? net.getSocketFactory().createSocket() : new Socket(); + live.add(outbound); + outbound.connect(new InetSocketAddress(host, port), CONNECT_TIMEOUT_MS); + Socket out = outbound; + Thread up = new Thread(() -> pump(inbound, out), "share-relay-up"); + up.setDaemon(true); + up.start(); + pump(outbound, inbound); + } catch (IOException e) { + Log.i(TAG, "relay connection ended: " + e.getMessage()); + } finally { + closeQuietly(inbound); + closeQuietly(outbound); + live.remove(inbound); + if (outbound != null) { + live.remove(outbound); + } + } + } + + /** Copy one direction until EOF or an error, then close both ends so the peer pump ends too. */ + private static void pump(Socket from, Socket to) { + byte[] buf = new byte[BUFFER]; + try { + InputStream in = from.getInputStream(); + OutputStream out = to.getOutputStream(); + int n; + while ((n = in.read(buf)) != -1) { + out.write(buf, 0, n); + } + } catch (IOException ignored) { + // a closed socket on either side ends the pump + } finally { + closeQuietly(from); + closeQuietly(to); + } + } + + private static void closeQuietly(Closeable c) { + if (c == null) { + return; + } + try { + c.close(); + } catch (IOException ignored) { + // nothing to recover + } + } +} diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareRsyncTarget.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareRsyncTarget.java new file mode 100644 index 000000000..8b46cd60e --- /dev/null +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareRsyncTarget.java @@ -0,0 +1,64 @@ +/* + * ============================================================================ + * Name : ShareRsyncTarget.java + * Author : AppDevForAll + * Copyright : Copyright (c) 2026 AppDevForAll + * Description : K2GO-577. The rsync block of one share manifest entry ({"rsync": {port, module}}), + * read and validated at the network boundary. Absent or invalid means the item uses + * the HTTP fetch, so a sender that runs no rsync daemon still works. The manifest + * reader keeps the block in the entry JSON (it strips only itemRef and delivered). + * ============================================================================ + */ +package org.appdevforall.k2go.contentexchange.data; + +import androidx.annotation.Nullable; + +import org.json.JSONObject; + +import java.util.regex.Pattern; + +/** Where one shared item can be pulled with rsync: the sender's daemon port and module. */ +public final class ShareRsyncTarget { + + /** Same rule as the core's module names: "-". */ + private static final Pattern MODULE = Pattern.compile("^[A-Za-z0-9][A-Za-z0-9-]{0,80}$"); + + private final int port; + private final String module; + + private ShareRsyncTarget(int port, String module) { + this.port = port; + this.module = module; + } + + /** The target of a manifest entry, or null when it has no valid rsync block. */ + @Nullable + public static ShareRsyncTarget fromEntry(@Nullable String entryJson) { + if (entryJson == null) { + return null; + } + try { + JSONObject entry = new JSONObject(entryJson); + JSONObject rsync = entry.optJSONObject("rsync"); + if (rsync == null) { + return null; + } + int port = rsync.optInt("port", 0); + String module = rsync.optString("module", ""); + if (port < 1 || port > 65535 || !MODULE.matcher(module).matches()) { + return null; + } + return new ShareRsyncTarget(port, module); + } catch (Exception e) { + return null; + } + } + + public int port() { + return port; + } + + public String module() { + return module; + } +} diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ImportSource.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ImportSource.java index 2a08d6525..12dfa74d4 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ImportSource.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ImportSource.java @@ -27,6 +27,22 @@ public interface ImportSource { */ String startImport(String app, String entryJson, String payloadBoxPath) throws Exception; + /** + * Start a pull long operation (K2GO-577): the box pulls the item from a sender's rsync module + * ({@code host}:{@code port}, {@code module}, password {@code token}) straight into place, with + * no staged copy. The op is polled with {@link #importStatus} like an import; its movedBytes are + * the bytes on disk, so a resumed pull starts from the bytes it kept. + */ + String startPull(String app, String entryJson, String host, int port, String module, String token) + throws Exception; + + /** + * Drop what a given-up pull of this item kept for a resume (K2GO-577): the user ended the run, so + * the partial bytes must not hold the disk. Best effort: a failure is ignored (the box sweeps old + * partials on its own). + */ + void discardPull(String app, String entryJson); + /** * Poll one import operation's status. {@link ImportStatus.State#GONE} means the op was swept, so * the caller re-starts once. Throws on a transport failure (the caller tolerates a few blips). diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/PullUnsupportedException.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/PullUnsupportedException.java new file mode 100644 index 000000000..6a48e3390 --- /dev/null +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/PullUnsupportedException.java @@ -0,0 +1,12 @@ +package org.appdevforall.k2go.contentexchange.domain; + +/** + * The box refused to start an rsync pull (K2GO-577): its core is older than the pull, or it has no + * rsync. A typed signal so the import service moves the item over the HTTP fetch instead of failing + * it. The data-layer import source maps the core's 400 / 501 on a pull start to this exception. + */ +public final class PullUnsupportedException extends Exception { + public PullUnsupportedException(String message) { + super(message); + } +} diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ShareUrls.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ShareUrls.java index 08855386d..73dbe6e3c 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ShareUrls.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ShareUrls.java @@ -51,6 +51,12 @@ public static String complete(String baseUrl, String manifestPath) { return baseUrl + sessionPrefix(manifestPath) + "/complete"; } + /** Where the receiver reports per-item progress to the sender's gate (K2GO-577). */ + public static String progress(String baseUrl, String manifestPath) { + requireOrigin(baseUrl); + return baseUrl + sessionPrefix(manifestPath) + "/progress"; + } + /** * The share-session path prefix, the manifest path with its trailing {@code /manifest} removed * ({@code /v1/share//manifest} -> {@code /v1/share/}). Fails closed when the path is not a diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/presentation/ContentImportService.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/presentation/ContentImportService.java index 760c012cd..abd6be3bd 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/presentation/ContentImportService.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/presentation/ContentImportService.java @@ -8,8 +8,9 @@ * the core reads, POST /v1/import/item so the core ingests it (delegating to the app's own * import), then remove the staging copy. The mirror of ContentExportService (foreground * specialUse + per-item queue + retry + notification + grant release), with the bytes - * flowing drive -> phone staging -> box. The item is the unit of resume (per-item retry); - * pause/resume and within-item byte-resume are later work. + * flowing drive -> phone staging -> box. The item is the unit of resume (per-item retry). + * K2GO-577: a Share item the sender offers over rsync skips the staging copy: the box + * pulls it straight into place through a loopback relay, and a Retry resumes its bytes. * ============================================================================ */ package org.appdevforall.k2go.contentexchange.presentation; @@ -37,12 +38,15 @@ import org.appdevforall.k2go.contentexchange.data.ConexCoreImportSource; import org.appdevforall.k2go.contentexchange.data.NetworkImportCarrier; import org.appdevforall.k2go.contentexchange.data.SafImportCarrier; +import org.appdevforall.k2go.contentexchange.data.ShareRelay; +import org.appdevforall.k2go.contentexchange.data.ShareRsyncTarget; import org.appdevforall.k2go.contentexchange.domain.ExportJob; import org.appdevforall.k2go.contentexchange.domain.ImportCarrier; import org.appdevforall.k2go.contentexchange.domain.ImportNoSpaceException; import org.appdevforall.k2go.contentexchange.domain.ImportSource; import org.appdevforall.k2go.contentexchange.domain.ImportStatus; import org.appdevforall.k2go.contentexchange.domain.ItemRef; +import org.appdevforall.k2go.contentexchange.domain.PullUnsupportedException; import org.appdevforall.k2go.download.domain.DownloadEta; import org.appdevforall.k2go.notifications.NotificationIds; @@ -87,6 +91,10 @@ public final class ContentImportService extends Service { /** K2GO-376: consecutive status-poll failures tolerated before an item fails. The ingest runs * minutes, so one relay blip or a busy core must not discard it (mirrors ContentExportService). */ private static final int MAX_POLL_ERRORS = 10; + /** K2GO-577: gap between progress reports to the sender's gate during an rsync pull. */ + private static final long SENDER_REPORT_INTERVAL_MS = 5000L; + /** The box reaches the relay on loopback (the box shares the phone's network namespace). */ + private static final String LOOPBACK = "127.0.0.1"; /** The single run state the progress fragment observes. */ private static final ImportSession SESSION = new ImportSession(); @@ -161,9 +169,39 @@ public static void cancel(Context ctx) { /** Clear a settled session (the "Finish" button) and release its SAF grant. Safe when idle. */ public static void finishSession(Context ctx) { releaseGrant(ctx, SESSION.treeUri()); + discardGivenUpPulls(); // Finish after a failure gives up the items that did not arrive SESSION.clear(); } + /** + * K2GO-577 (decision a): the user ended the run (Cancel or Finish), so the items that did not + * arrive will not be resumed. Ask the box to drop the bytes their rsync pulls kept, instead of + * holding the disk until the 48 h sweep. A drop or a Retry never comes here, so those keep their + * bytes. Snapshot now (the session is cleared next), then call off the main thread. + */ + private static void discardGivenUpPulls() { + if (!SESSION.isNetwork()) { + return; + } + ExportJob job = SESSION.job(); + final java.util.List pending = new java.util.ArrayList<>(); + for (int i = 0; i < job.size(); i++) { + String entry = SESSION.entryJson(i); + if (job.status(i) != ExportJob.DONE && ShareRsyncTarget.fromEntry(entry) != null) { + pending.add(new String[]{SESSION.refAt(i).app(), entry}); + } + } + if (pending.isEmpty()) { + return; + } + new Thread(() -> { + ImportSource source = new ConexCoreImportSource(new ConexCoreDataSource()); + for (String[] p : pending) { + source.discardPull(p[0], p[1]); + } + }, "content-import-discard").start(); + } + /** * Release the persisted SAF grant for a finished import. Import is one-shot (pick a drive, read, * done), so the grant is not needed afterwards; releasing it keeps the app's persisted-permission @@ -208,6 +246,7 @@ public int onStartCommand(Intent intent, int flags, int startId) { // No worker owns teardown: a fresh service instance may be handling this cancel, so // satisfy the FGS start-foreground obligation before stopping. startForeground(NOTIFICATION_ID, buildNotification(null)); + discardGivenUpPulls(); SESSION.clear(); stopForeground(true); stopSelf(); @@ -294,10 +333,28 @@ private void drain() { return; } SESSION.setSource(carrier.sourceLabel()); + NetworkImportCarrier net = carrier instanceof NetworkImportCarrier + ? (NetworkImportCarrier) carrier : null; + // The rsync relay opens with the first pulled item and closes with the run on every exit + // path, which also ends any live pull connection. + ShareRelay[] relay = {null}; + try { + drainItems(source, staging, carrier, net, relay); + } finally { + if (relay[0] != null) { + relay[0].close(); + } + } + } + /** The per-item loop of {@link #drain()}. {@code relay} holds the rsync relay the caller closes. */ + private void drainItems(ImportSource source, BoxImportStaging staging, ImportCarrier carrier, + @Nullable NetworkImportCarrier net, ShareRelay[] relay) { ExportJob job = SESSION.job(); long lastSampleAt = SystemClock.elapsedRealtime(); long lastSampleBytes = job.copiedTotal(); + // False once this box refused a pull (an older core): the rest of the run uses the HTTP fetch. + boolean pullSupported = true; for (int i = job.firstPending(); i >= 0; i = job.firstPending()) { if (SESSION.isCancelled()) { @@ -309,9 +366,27 @@ private void drain() { notifyNow(); String stored = BoxImportStaging.safeName(SESSION.payloadName(i)); + ShareRsyncTarget target = net != null && pullSupported + ? ShareRsyncTarget.fromEntry(SESSION.entryJson(i)) : null; try { - copyToStaging(carrier, staging, SESSION.payloadName(i), stored, i, job); - ingest(source, ref, i, stored); + boolean pulled = false; + if (target != null) { + // The sender offers rsync: the box pulls the item into place, no staging copy. + if (relay[0] == null) { + relay[0] = new ShareRelay(getApplicationContext(), SESSION.shareHost(), target.port()); + } + try { + pull(source, ref, i, target, relay[0], net, job); + pulled = true; + } catch (PullUnsupportedException e) { + Log.i(TAG, "box cannot pull, using the HTTP fetch: " + e.getMessage()); + pullSupported = false; + } + } + if (!pulled) { + copyToStaging(carrier, staging, SESSION.payloadName(i), stored, i, job); + ingest(source, ref, i, stored); + } job.setDone(i); } catch (Exception e) { // The staging copy or the ingest failed: mark the item and keep its slot for a retry. @@ -340,6 +415,8 @@ private void drain() { } if (SESSION.isCancelled()) { + // The running pull was aborted (runBoxOp), so the box no longer writes its partial. + discardGivenUpPulls(); SESSION.clear(); stopForeground(true); stopSelf(); @@ -420,7 +497,68 @@ private void copyToStaging(ImportCarrier carrier, BoxImportStaging staging, Stri */ private void ingest(ImportSource source, ItemRef ref, int i, String storedName) throws Exception { String boxPath = BoxImportStaging.boxPath(storedName); - String opId = source.startImport(ref.app(), SESSION.entryJson(i), boxPath); + runBoxOp(source, () -> source.startImport(ref.app(), SESSION.entryJson(i), boxPath), null); + } + + /** + * Pull one Share item with rsync (K2GO-577): the box pulls it from the sender's daemon through the + * loopback relay, straight into place. The op's movedBytes are the bytes on disk, so the item bar + * moves with the transfer and a Retry starts from the bytes the last attempt kept. The sender gets + * a progress report every few seconds and a final done / error. + */ + private void pull(ImportSource source, ItemRef ref, int i, ShareRsyncTarget target, ShareRelay relay, + NetworkImportCarrier net, ExportJob job) throws Exception { + String itemRef = SESSION.payloadName(i); + long[] last = {0L, 0L}; // moved, total + long[] window = {-1L, 0L, 0L}; // at, bytes, reportAt + try { + runBoxOp(source, () -> source.startPull(ref.app(), SESSION.entryJson(i), LOOPBACK, + relay.localPort(), target.module(), SESSION.shareToken()), (moved, total) -> { + last[0] = moved; + last[1] = total; + job.setCopied(i, moved); + long now = SystemClock.elapsedRealtime(); + if (window[0] < 0) { + // Anchor the rate on the first reading: a resumed pull starts at its kept bytes, + // which must not count as bytes moved in this window. + window[0] = now; + window[1] = job.copiedTotal(); + } else if (now - window[0] >= RATE_WINDOW_MS) { + sampleRate(job, now - window[0], job.copiedTotal() - window[1]); + window[0] = now; + window[1] = job.copiedTotal(); + } + if (now - window[2] >= SENDER_REPORT_INTERVAL_MS) { + net.reportProgress(itemRef, moved, total, "running"); + window[2] = now; + } + }); + } catch (PullUnsupportedException e) { + throw e; // nothing moved: the item goes over the HTTP fetch, so no error report + } catch (Exception e) { + net.reportProgress(itemRef, last[0], last[1], "error"); + throw e; + } + net.reportProgress(itemRef, last[1], last[1], "done"); + } + + /** Starts one box long operation; returns its opId. */ + private interface OpStarter { + String start() throws Exception; + } + + /** Receives a running op's moved / total bytes on each poll. */ + private interface MovedSink { + void onMoved(long moved, long total); + } + + /** + * Drive one box long operation (import or pull) to its end: poll its status, re-start once if it + * was swept, and throw on an error. {@code sink} is null for an import of an already-staged item. + */ + private void runBoxOp(ImportSource source, OpStarter starter, @Nullable MovedSink sink) + throws Exception { + String opId = starter.start(); boolean restarted = false; int pollErrors = 0; while (true) { @@ -445,6 +583,9 @@ private void ingest(ImportSource source, ItemRef ref, int i, String storedName) pollErrors = 0; switch (st.state()) { case DONE: + if (sink != null) { + sink.onMoved(st.movedBytes(), st.totalBytes()); + } // Release the consumed op now (an idempotent drop on the box), so a finished // import does not linger until the TTL sweep. This is import's normal-completion // cleanup, since it has no fetch stream whose close drops the op as export does. @@ -463,10 +604,13 @@ private void ingest(ImportSource source, ItemRef ref, int i, String storedName) throw new Exception("the import operation was lost"); } restarted = true; - opId = source.startImport(ref.app(), SESSION.entryJson(i), boxPath); + opId = starter.start(); break; case RUNNING: default: + if (sink != null) { + sink.onMoved(st.movedBytes(), st.totalBytes()); + } // Keep the session fresh so the UI shows the item is still being imported. SESSION.publish(); maybeNotify(SystemClock.elapsedRealtime()); diff --git a/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/data/ShareRsyncTargetTest.java b/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/data/ShareRsyncTargetTest.java new file mode 100644 index 000000000..0875e61bd --- /dev/null +++ b/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/data/ShareRsyncTargetTest.java @@ -0,0 +1,40 @@ +package org.appdevforall.k2go.contentexchange.data; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import org.junit.Test; + +/** K2GO-577: the rsync block of a share manifest entry is read only when it is valid. */ +public class ShareRsyncTargetTest { + + private static String entry(String rsync) { + return "{\"app\":\"kiwix\",\"identity\":{\"localRef\":\"a.zim\"}" + + (rsync != null ? ",\"rsync\":" + rsync : "") + "}"; + } + + @Test + public void readsAValidBlock() { + ShareRsyncTarget t = ShareRsyncTarget.fromEntry(entry("{\"port\":9873,\"module\":\"abc123-0\"}")); + assertNotNull(t); + assertEquals(9873, t.port()); + assertEquals("abc123-0", t.module()); + } + + @Test + public void noBlockMeansHttpFetch() { + assertNull(ShareRsyncTarget.fromEntry(entry(null))); + assertNull(ShareRsyncTarget.fromEntry(null)); + assertNull(ShareRsyncTarget.fromEntry("not json")); + } + + @Test + public void rejectsABadPortOrModule() { + assertNull(ShareRsyncTarget.fromEntry(entry("{\"port\":0,\"module\":\"abc-0\"}"))); + assertNull(ShareRsyncTarget.fromEntry(entry("{\"port\":70000,\"module\":\"abc-0\"}"))); + assertNull(ShareRsyncTarget.fromEntry(entry("{\"port\":9873,\"module\":\"../x\"}"))); + assertNull(ShareRsyncTarget.fromEntry(entry("{\"port\":9873,\"module\":\"a b\"}"))); + assertNull(ShareRsyncTarget.fromEntry(entry("{\"port\":9873,\"module\":\"\"}"))); + } +} diff --git a/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/domain/ShareUrlsTest.java b/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/domain/ShareUrlsTest.java index 42d333246..1acd81b4c 100644 --- a/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/domain/ShareUrlsTest.java +++ b/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/domain/ShareUrlsTest.java @@ -22,6 +22,7 @@ public void composesManifestFetchAndComplete() { assertEquals(BASE + "/v1/share/abc123/fetch/0", ShareUrls.fetch(BASE, MANIFEST, "0")); assertEquals(BASE + "/v1/share/abc123/fetch/12", ShareUrls.fetch(BASE, MANIFEST, "12")); assertEquals(BASE + "/v1/share/abc123/complete", ShareUrls.complete(BASE, MANIFEST)); + assertEquals(BASE + "/v1/share/abc123/progress", ShareUrls.progress(BASE, MANIFEST)); } @Test diff --git a/controller/docs/conex-core-share-exposure.md b/controller/docs/conex-core-share-exposure.md index 4928221bc..2f97a923b 100644 --- a/controller/docs/conex-core-share-exposure.md +++ b/controller/docs/conex-core-share-exposure.md @@ -160,6 +160,31 @@ This matrix passed against the rendered template in a stock nginx 1.27 with Remove the forward when done (`adb -s forward --remove tcp:8085`). +## rsync transport: a second listener while a session is open (K2GO-577) + +The HTTP fetch stages and re-sends a whole item after a drop. Share now moves an item that an +adapter can list as files (a ZIM today) with rsync, which writes in place and resumes. This +adds one listener next to the HTTP gate: + +- conex-core runs `rsync --daemon` on `0.0.0.0:9873` (`CONEX_SHARE_RSYNC_PORT`; `0` turns it + off and Share stays HTTP only). The daemon runs ONLY while a share session is open: the + first session starts it, the last close (complete, cancel or the safety cap) stops it, and + the startup sweep kills a daemon that a dead core left behind. With no session, the port is + closed. +- One read-only module per shared item, named `-` (128-bit session id). + A filter admits only that item's files; `list = no` hides the module list. +- The module password is the session token. rsync authenticates by challenge and response, so + the token is not sent in clear. A session close removes its modules and secrets at once. +- The daemon logs nothing (`log file = /dev/null`), so no client address is kept (K2GO-559). +- The share manifest advertises the module per item (`rsync: {port, module}`). The receiver + pulls through its own box's conex-core (`POST /v1/import/item` with a `source`); a receiver + whose box cannot pull uses the HTTP fetch, so old and new peers still work together. +- On K2Go the receiver app relays the pull over the hotspot network (`ShareRelay`), because + Android may route a box process over cellular when the hotspot has no internet. + +Pending for IIAB: a host firewall must admit TCP 9873 when `conex_core_share_expose` is on, or +a receiver cannot reach the daemon. + ## Scope kept (what this change does NOT do) - Does not expose the whole API: only `/v1/share//...`. diff --git a/static/conex-core/CHANGELOG.md b/static/conex-core/CHANGELOG.md index dd709381d..66ddc0e61 100644 --- a/static/conex-core/CHANGELOG.md +++ b/static/conex-core/CHANGELOG.md @@ -5,6 +5,7 @@ One line per version, newest first. Every REST-facing change bumps `conex_core.V role), so this file is the human record of what each bump enables. Keep entries short: `version - change (TICKET)`. +- **0.4.0** - Share moves ZIMs with rsync (K2GO-577). While a share session is open, the box runs an rsync daemon (`CONEX_SHARE_RSYNC_PORT`, default 9873, 0 = off) with one read-only module per shareable item, password = the session token; the last session close stops it. The share manifest adds `rsync: {port, module}` per such item. `POST /v1/import/item` with `source: {host, port, module, token}` instead of `payloadPath` pulls the item straight into place: no staging copy, a drop or a cancel keeps the bytes in `/library/.conex-partial` and the next pull resumes and verifies the whole file. `POST /v1/share/{id}/progress` takes the receiver's per-item progress (keeps the session fresh, marks `done` items delivered); `GET /v1/share/{id}/progress` is the sender's view. `POST /v1/import/discard {app, entry}` drops the bytes a given-up pull kept (the user ended the run; refused while a pull of the item runs). Kolibri and Calibre still share over the HTTP fetch. (K2GO-577) - **0.3.5** - Kiwix counts as installed only when kiwix-serve is present, not just the zims dir, so a library that cannot open ZIMs answers NO_APP and does not import them. (K2GO-574) - **0.3.4** - import becomes a long operation like export: POST /v1/import/item now STARTS the ingest and returns an opId, the client polls GET /v1/import/status/{op}, and DELETE /v1/import/{op} cancels, so a 35-141 s Kolibri import no longer times out at the dash-node/nginx relay. A shared liveness watchdog supervises the Kolibri import and export CLIs by byte and CPU progress (no fixed timeout; the window is relative to the task) and names a signal-9 exit as a likely out-of-memory kill. (K2GO-376) - **0.3.3** - observability: log every error response and uncaught handler exception to stdout (the supervisor captures it to /var/log/conex-core.log), and line-buffer stdout/stderr in __main__ so output reaches the log while the server runs forever, so a failed import or share leaves a box-side trace; an uncaught error now returns a 500 JSON instead of dropping the connection. (K2GO-376) diff --git a/static/conex-core/README.md b/static/conex-core/README.md index 4c3d3dc0d..dd3dfd1ee 100644 --- a/static/conex-core/README.md +++ b/static/conex-core/README.md @@ -38,9 +38,18 @@ Environment (defaults match the box layout): - `CONEX_HOST` (default `127.0.0.1`), `CONEX_PORT` (default `9500`). - `CONEX_ZIMS_DIR` (default `/library/zims/content/`): where the ZIM files live. +- `CONEX_SHARE_RSYNC_PORT` (default `9873`, `0` = off): the share rsync daemon port. The + daemon listens on all interfaces only while a share session is open (K2GO-577). +- `CONEX_RSYNC_BIN` (default `/usr/bin/rsync`), `CONEX_PARTIAL_DIR` (default + `/library/.conex-partial`): the rsync binary and where a pull keeps resumable partial files. ## Endpoints +Share over rsync (K2GO-577): `POST /v1/import/item` with `{app, entry, source: {host, port, +module, token}}` pulls an item from a sender's rsync module into place (same op and status as an +import). `POST /v1/share/{id}/progress` takes the receiver's `{itemRef, movedBytes, totalBytes, +state}`; `GET /v1/share/{id}/progress` returns it for the sender. + Read: - `GET /v1/version`: the core build (mirrors dash-node `/system/version`). diff --git a/static/conex-core/conex_core/__init__.py b/static/conex-core/conex_core/__init__.py index 457a21fa9..367fe14ff 100644 --- a/static/conex-core/conex_core/__init__.py +++ b/static/conex-core/conex_core/__init__.py @@ -7,5 +7,5 @@ # The core build. Bumped per core change so a box can report which core it runs (GET # /v1/version), mirroring dash-node's package.json version and /system/version. The # conex_core Ansible role writes this into the [conex_core] section of iiab.ini. -VERSION = "0.3.5" +VERSION = "0.4.0" __version__ = VERSION diff --git a/static/conex-core/conex_core/adapters/kiwix.py b/static/conex-core/conex_core/adapters/kiwix.py index d1f9fbd99..a68427afc 100644 --- a/static/conex-core/conex_core/adapters/kiwix.py +++ b/static/conex-core/conex_core/adapters/kiwix.py @@ -33,7 +33,9 @@ Identity, ManifestItem, ) -from ..ports import ContentAdapter, ExportPlan, ExportStream +from ..importing import volume_nearly_full +from ..ports import ContentAdapter, ExportPlan, ExportStream, ShareFiles +from ..watchdog import CANCELLED # openzim header: magic number is uint32 little-endian 0x044D495A at offset 0, and the # 16-byte UUID is at offset 8. We only need the first 24 bytes. @@ -238,6 +240,58 @@ def import_item(self, entry: dict, payload_path: str, should_cancel=None, self._reindex() return {"ok": True, "app": "kiwix", "localRef": local_ref, "bytes": os.path.getsize(dest)} + def share_files(self, local_ref: str) -> Optional[ShareFiles]: + # A ZIM is one file in the zims dir: the daemon serves only that file (K2GO-577). + path = self._export_path(local_ref) + if path is None: + return None + return ShareFiles(root=self._dir, files=[local_ref]) + + def pull_item(self, entry: dict, source: dict, partial_dir: str, puller, + should_cancel=None, on_progress=None) -> dict: + # Pull the ZIM into a partial file on the same volume (resumable), then rename it into + # the zims dir and re-index. The rename is the commit: Kiwix never sees a partial ZIM. + local_ref = (entry.get("identity") or {}).get("localRef", "") + if not is_safe_local_ref(local_ref) or not local_ref.lower().endswith(".zim"): + return {"ok": False, "reason": "invalid localRef"} + part_dir = os.path.join(partial_dir, "kiwix") + os.makedirs(part_dir, exist_ok=True) + part = os.path.join(part_dir, local_ref) + result = puller.pull_file(source, local_ref, part, should_cancel=should_cancel, + on_progress=on_progress) + if result.state == CANCELLED: + return {"ok": False, "reason": "import cancelled"} + if not result.ok: + failure = {"ok": False, "reason": f"transfer {result.detail}"} + if volume_nearly_full(part_dir): + failure["code"] = "no_space" + return failure + os.makedirs(self._dir, exist_ok=True) + dest = os.path.join(self._dir, local_ref) + try: + os.replace(part, dest) + except OSError as exc: + if exc.errno != errno.EXDEV: + return {"ok": False, "reason": f"move failed: {exc}"} + shutil.move(part, dest) # the partial dir is on another volume: copy once + self._reindex() + return {"ok": True, "app": "kiwix", "localRef": local_ref, "bytes": os.path.getsize(dest)} + + def discard_pull(self, entry: dict, partial_dir: str) -> dict: + # The kept partial is the only thing a ZIM pull leaves behind (K2GO-577). + local_ref = (entry.get("identity") or {}).get("localRef", "") + if not is_safe_local_ref(local_ref) or not local_ref.lower().endswith(".zim"): + return {"ok": False, "reason": "invalid localRef"} + part = os.path.join(partial_dir, "kiwix", local_ref) + try: + freed = os.path.getsize(part) + os.remove(part) + except FileNotFoundError: + freed = 0 + except OSError as exc: + return {"ok": False, "reason": f"discard failed: {exc}"} + return {"ok": True, "freedBytes": freed} + def delete(self, local_ref: str) -> dict: # A ZIM is self-contained, so the file IS the unit: remove it, then re-run the indexer # to drop its library.xml entry. freedBytes is the file size exactly (no shared bytes). diff --git a/static/conex-core/conex_core/config.py b/static/conex-core/conex_core/config.py index f3377e705..a4fd74f55 100644 --- a/static/conex-core/conex_core/config.py +++ b/static/conex-core/conex_core/config.py @@ -7,16 +7,21 @@ from dataclasses import dataclass DEFAULT_PORT = 9500 +#: The share rsync daemon port (K2GO-577). Above 1024: a proot box cannot bind a low port. +DEFAULT_SHARE_RSYNC_PORT = 9873 -def _parse_port(raw: str) -> int: +def _parse_port(raw: str, default: int = DEFAULT_PORT, allow_off: bool = False) -> int: """Parse a TCP port from the environment, falling back to the default on an empty, - non-numeric or out-of-range value instead of crashing the service at startup.""" + non-numeric or out-of-range value instead of crashing the service at startup. With + ``allow_off``, 0 is kept (the feature is off).""" try: port = int(raw) except (TypeError, ValueError): - return DEFAULT_PORT - return port if 1 <= port <= 65535 else DEFAULT_PORT + return default + if allow_off and port == 0: + return 0 + return port if 1 <= port <= 65535 else default @dataclass(frozen=True) @@ -38,6 +43,12 @@ class Config: # multi-gigabyte channel export has room and the free-space preflight measures the right # disk. staging_dir: str = "/library/.conex-staging" + # K2GO-577: the share rsync daemon port (0 = off: share over HTTP fetch only), the rsync + # binary, and where a pull keeps resumable partial files (the content volume, so the final + # rename does not copy). + share_rsync_port: int = DEFAULT_SHARE_RSYNC_PORT + rsync_bin: str = "/usr/bin/rsync" + partial_dir: str = "/library/.conex-partial" @staticmethod def from_env() -> "Config": @@ -55,4 +66,9 @@ def from_env() -> "Config": credentials_file=os.environ.get( "K2GO_CREDENTIALS_FILE", "/library/dashboard/credentials.json"), staging_dir=os.environ.get("CONEX_STAGING_DIR", "/library/.conex-staging"), + share_rsync_port=_parse_port( + os.environ.get("CONEX_SHARE_RSYNC_PORT", str(DEFAULT_SHARE_RSYNC_PORT)), + DEFAULT_SHARE_RSYNC_PORT, allow_off=True), + rsync_bin=os.environ.get("CONEX_RSYNC_BIN", "/usr/bin/rsync"), + partial_dir=os.environ.get("CONEX_PARTIAL_DIR", "/library/.conex-partial"), ) diff --git a/static/conex-core/conex_core/import_ops.py b/static/conex-core/conex_core/import_ops.py index f3a038747..bd6caccaf 100644 --- a/static/conex-core/conex_core/import_ops.py +++ b/static/conex-core/conex_core/import_ops.py @@ -23,6 +23,7 @@ from typing import Any, Optional from .importing import is_within_dir +from .rsync_share import sweep_partials, valid_source from .ops_base import DONE, ERROR, RUNNING, Op, OpRegistry, Response # The import work-temp prefix, created by the adapters' import_item. The startup sweep reclaims an @@ -35,6 +36,7 @@ class _ImportOp(Op): moved_bytes: int = 0 result: Optional[dict] = None code: Optional[str] = None # a structural error code, e.g. "no_space" + local_ref: Optional[tuple] = None # (app, localRef) of a pull, so discard can refuse a live one def _entry_bytes(entry: Any) -> int: @@ -49,10 +51,15 @@ class ImportOps(OpRegistry): """Registry of in-flight import operations. One instance lives for the server's lifetime; the handler threads call in concurrently.""" - def __init__(self, registry, import_root: str, ttl_seconds: int = 600): + def __init__(self, registry, import_root: str, ttl_seconds: int = 600, + partial_root: Optional[str] = None, puller=None): super().__init__(ttl_seconds) self._registry = registry self._import_root = import_root + # K2GO-577: where a pull keeps its resumable partial files, and the rsync client. Both + # None disables pull (the HTTP-staged import still works). + self._partial_root = partial_root + self._puller = puller # ---- REST-facing operations (return (status, payload)) ---- @@ -75,6 +82,51 @@ def start(self, app: str, entry: Any, payload_path: str) -> Response: worker.start() return 200, {"opId": op_id, "state": RUNNING, "totalBytes": total} + def start_pull(self, app: str, entry: Any, source: Any) -> Response: + """Start a pull long operation (K2GO-577): the adapter pulls the item from the sender's + rsync module straight into place, then indexes it. Same op shape as ``start``, so the + client polls the same status; movedBytes is the bytes on disk, a resumed pull starts + from the kept bytes.""" + self._sweep_expired() + if not self._registry.has(app): + return 404, {"error": f"unknown app: {app}"} + if self._puller is None or self._partial_root is None or not self._puller.available(): + return 501, {"error": "pull unavailable on this box"} + if not isinstance(entry, dict): + return 400, {"error": "entry is required"} + reason = valid_source(source) + if reason is not None: + return 400, {"error": reason} + # Reclaim partials no retry picked up, also on a box that runs for weeks without a restart. + sweep_partials(self._partial_root) + total = _entry_bytes(entry) + op_id = self._new_id() + op = _ImportOp(app=app, state=RUNNING, total_bytes=total) + op.local_ref = (app, (entry.get("identity") or {}).get("localRef")) + self._put(op_id, op) + worker = threading.Thread( + target=self._run_pull, args=(op_id, app, entry, source, total), daemon=True) + worker.start() + return 200, {"opId": op_id, "state": RUNNING, "totalBytes": total} + + def discard(self, app: str, entry: Any) -> Response: + """Drop what a given-up pull kept for a resume (K2GO-577): the user ended the run (Cancel or + Finish), so the partial bytes must not hold the disk for 48 h. Refused while a pull of the + same item still runs: the client aborts the op first, and a resumed pull would lose its + bytes.""" + self._sweep_expired() + if not self._registry.has(app): + return 404, {"error": f"unknown app: {app}"} + if not isinstance(entry, dict) or self._partial_root is None: + return 400, {"error": "entry is required"} + local_ref = (entry.get("identity") or {}).get("localRef") + with self._lock: + busy = any(op.state == RUNNING and getattr(op, "local_ref", None) == (app, local_ref) + for op in self._ops.values()) + if busy: + return 409, {"error": "a pull of this item is still running"} + return 200, self._registry.get(app).discard_pull(entry, self._partial_root) + def status(self, op_id: str) -> Response: """Report the op state, the moved-bytes heartbeat, and the result once done. A 404 means the op is unknown (swept or never existed), so the client re-starts.""" @@ -105,6 +157,8 @@ def sweep_startup(self) -> None: for name in names: if name.startswith(TEMP_PREFIX): shutil.rmtree(os.path.join(self._import_root, name), ignore_errors=True) + if self._partial_root: + sweep_partials(self._partial_root) def _run_import(self, op_id: str, app: str, entry: Any, payload_path: str, total: int) -> None: """Ingest the item off the request thread, under the liveness watchdog, then flip the op to @@ -119,9 +173,25 @@ def on_progress(moved: int) -> None: # that fills and holds, never past 100%. self._touch(op_id, moved_bytes=min(moved, total) if total else moved) + self._finish_with(op_id, total, lambda: self._registry.get(app).import_item( + entry, payload_path, should_cancel=should_cancel, on_progress=on_progress)) + + def _run_pull(self, op_id: str, app: str, entry: Any, source: dict, total: int) -> None: + """Pull the item off the request thread. movedBytes is the partial file size, so it is + monotonic and already counts the bytes a resumed pull kept.""" + should_cancel = self.should_cancel(op_id) + + def on_progress(moved: int) -> None: + self._touch(op_id, moved_bytes=min(moved, total) if total else moved) + + self._finish_with(op_id, total, lambda: self._registry.get(app).pull_item( + entry, source, self._partial_root, self._puller, + should_cancel=should_cancel, on_progress=on_progress)) + + def _finish_with(self, op_id: str, total: int, work) -> None: + """Run the adapter work, then flip the op to done or error.""" try: - result = self._registry.get(app).import_item( - entry, payload_path, should_cancel=should_cancel, on_progress=on_progress) + result = work() except Exception as exc: # any adapter failure surfaces as an op error with its detail self._transition(op_id, RUNNING, ERROR, error=f"import failed: {exc}") return diff --git a/static/conex-core/conex_core/ports.py b/static/conex-core/conex_core/ports.py index d2d8fc323..fdf6e4a11 100644 --- a/static/conex-core/conex_core/ports.py +++ b/static/conex-core/conex_core/ports.py @@ -55,6 +55,14 @@ class ExportPlan: content_type: str +@dataclass +class ShareFiles: + """The files one item is made of, for the share rsync daemon (K2GO-577): a root dir and the + paths under it. The daemon serves only these, read-only.""" + root: str + files: List[str] + + class ContentAdapter(ABC): """Read-side port for one content app.""" @@ -119,6 +127,25 @@ def import_item(self, entry: dict, payload_path: str, may ignore both. Default: not supported.""" return {"ok": False, "reason": "import not supported"} + def share_files(self, local_ref: str) -> Optional[ShareFiles]: + """The files the share rsync daemon may serve for this item (K2GO-577). None: the item + is not shared over rsync, so a receiver uses the HTTP fetch. Default: None.""" + return None + + def pull_item(self, entry: dict, source: dict, partial_dir: str, puller, + should_cancel: Optional[Callable[[], bool]] = None, + on_progress: Optional[Callable[[int], None]] = None) -> dict: + """Pull one item from a sender's rsync module straight into place and index it + (K2GO-577). ``partial_dir`` is this app's dir for resumable partial files; ``puller`` is + the rsync client. Same result shape as ``import_item``. Default: not supported.""" + return {"ok": False, "reason": "pull not supported"} + + def discard_pull(self, entry: dict, partial_dir: str) -> dict: + """Remove what a given-up pull of this item kept for a resume (K2GO-577): the user ended + the run with Cancel or Finish, so the bytes will not be resumed. Returns {ok, freedBytes}. + Default: nothing kept.""" + return {"ok": True, "freedBytes": 0} + def delete(self, local_ref: str) -> dict: """Remove one item to reclaim space, delegating to the app's own delete (never driving the app DB + files directly): the app owns its storage invariants and the next list() diff --git a/static/conex-core/conex_core/router.py b/static/conex-core/conex_core/router.py index e3cbfc071..e709cf390 100644 --- a/static/conex-core/conex_core/router.py +++ b/static/conex-core/conex_core/router.py @@ -167,6 +167,11 @@ def dispatch(registry: Registry, method: str, path: str, query: dict, body: Any return 500, {"error": "share unavailable"} items = body.get("items") if isinstance(body, dict) else None return share.create(items) + # POST /v1/share/{sessionId}/progress : the receiver reports one item's progress. + if len(segments) == 4 and segments[0] == "v1" and segments[1] == "share" and segments[3] == "progress": + if share is None: + return 500, {"error": "share unavailable"} + return share.report_progress(segments[2], token, body) # POST /v1/share/{sessionId}/complete : the receiver signals a clean, done transfer. if len(segments) == 4 and segments[0] == "v1" and segments[1] == "share" and segments[3] == "complete": if share is None: @@ -178,6 +183,13 @@ def dispatch(registry: Registry, method: str, path: str, query: dict, body: Any # POST /v1/import/validate : what will import and what is skipped, nothing moved. if segments == ["v1", "import", "validate"]: return _import_validate(registry, body) + # POST /v1/import/discard : drop what a given-up pull kept for a resume (K2GO-577). + if segments == ["v1", "import", "discard"]: + if import_ops is None: + return 500, {"error": "import ops unavailable"} + if not isinstance(body, dict): + return 400, {"error": "body must be {app, entry}"} + return import_ops.discard(body.get("app"), body.get("entry")) # POST /v1/import/item : start the import long operation, return its opId. Import is a long # op (K2GO-376): the ingest runs minutes on a real channel, so a single blocking POST times # out at the relay. The op starts off the request thread; the client polls import status. @@ -185,7 +197,11 @@ def dispatch(registry: Registry, method: str, path: str, query: dict, body: Any if import_ops is None: return 500, {"error": "import ops unavailable"} if not isinstance(body, dict): - return 400, {"error": "body must be {app, entry, payloadPath}"} + return 400, {"error": "body must be {app, entry, payloadPath} or {app, entry, source}"} + # A source instead of a payloadPath: pull the item from a sender's rsync module + # straight into place (K2GO-577). + if "source" in body: + return import_ops.start_pull(body.get("app"), body.get("entry"), body.get("source")) return import_ops.start(body.get("app"), body.get("entry"), body.get("payloadPath")) return 404, {"error": "not found"} @@ -246,6 +262,12 @@ def dispatch(registry: Registry, method: str, path: str, query: dict, body: Any return 500, {"error": "share unavailable"} return share.manifest(segments[2], token) + # GET /v1/share/{sessionId}/progress : the receiver's last progress report per item. + if len(segments) == 4 and segments[0] == "v1" and segments[1] == "share" and segments[3] == "progress": + if share is None: + return 500, {"error": "share unavailable"} + return share.progress(segments[2], token) + # GET /v1/share/{sessionId}/fetch/{itemRef} : stream one shared item (payload is an # ExportStream the server streams). 409 while the item stages; the receiver retries. if len(segments) == 5 and segments[0] == "v1" and segments[1] == "share" and segments[3] == "fetch": diff --git a/static/conex-core/conex_core/rsync_share.py b/static/conex-core/conex_core/rsync_share.py new file mode 100644 index 000000000..7c4ce8635 --- /dev/null +++ b/static/conex-core/conex_core/rsync_share.py @@ -0,0 +1,306 @@ +"""rsync transport for Share (K2GO-577). + +Share moves content with rsync, not with a staged HTTP stream: rsync writes each file once into its +final place, keeps the partial bytes when the link drops, and resumes from them. The core owns the +whole mechanism, so IIAB and K2Go share it. + +Sender: one rsync daemon per box, alive only while a share session is open. Each shared item is a +read-only module whose filter admits only that item's files. The module password is the session +token. The daemon re-reads its config on every connection, so a session that opens or closes only +rewrites the config; the last close stops the daemon. + +Receiver: one rsync client per item. A single large file (a ZIM) is pulled with --inplace +--append-verify into a partial path on the content volume, then renamed into place: a kill of the +client or of the sender keeps the bytes, and the next pull appends to them and verifies the whole +file. The rename is the commit, so the app never sees a partial file under its final name. + +Lifecycle: + daemon: started by the first open session, stopped by the last close, the safety cap, or the + startup sweep (a pid file names the process group a dead core left behind). + partial file: made by the pull, renamed into place on success, kept on a drop or a cancel so a + retry resumes, removed by the startup sweep after PARTIAL_MAX_AGE_SECONDS. +""" + +from __future__ import annotations + +import logging +import os +import re +import shutil +import signal +import subprocess +import threading +import time +from dataclasses import dataclass +from typing import Callable, List, Optional + +from .watchdog import run_supervised + +_LOG = logging.getLogger("conex-core") +_SIGKILL = getattr(signal, "SIGKILL", 9) + +#: The rsync user name. The secret (the session token) is the auth; the name is fixed. +SHARE_USER = "share" +#: Module names are "-": hex, digits and a hyphen. +MODULE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]{0,80}$") +#: An IPv4 address or a host name. No IPv6: the hotspot and LAN paths are IPv4. +HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.-]{0,252}$") +#: rsync ends a transfer with no data for this long (a dead peer), keeping the partial. +IO_TIMEOUT_SECONDS = 60 +CONNECT_TIMEOUT_SECONDS = 20 +#: A partial that no retry picked up for this long is removed by the startup sweep. +PARTIAL_MAX_AGE_SECONDS = 48 * 3600 + + +@dataclass(frozen=True) +class Module: + """One shared item as a daemon module: the root dir and the files under it it may serve.""" + name: str + root: str + files: List[str] + secret: str + + +def _clean(value: str) -> bool: + """True when a value is safe to write into rsyncd.conf or a filter file: no control chars + (a newline would start a new directive) and no ']' (it would close a section header).""" + return bool(value) and "]" not in value and not any(ord(c) < 32 or ord(c) == 127 for c in value) + + +def render_filter(files: List[str]) -> str: + """Filter rules that admit only ``files`` (relative to the module root) and their parent + dirs, then exclude everything else. Anchored with '/', so a name matches only at its path.""" + rules: List[str] = [] + seen = set() + for rel in files: + parts = rel.strip("/").split("/") + for depth in range(1, len(parts)): + parent = "/" + "/".join(parts[:depth]) + "/" + if parent not in seen: + seen.add(parent) + rules.append("+ " + parent) + rules.append("+ /" + "/".join(parts)) + rules.append("- *") + return "\n".join(rules) + "\n" + + +def render_config(modules: List[Module], run_dir: str, uid: int, gid: int) -> str: + """The daemon config. log file /dev/null: the daemon logs no client address (K2GO-559). + uid/gid keep the core's own identity, so a root daemon does not drop to nobody and lose + read access to the content.""" + lines = [ + "use chroot = no", + "reverse lookup = no", + "log file = /dev/null", + "transfer logging = no", + f"timeout = {IO_TIMEOUT_SECONDS * 2}", + f"uid = {uid}", + f"gid = {gid}", + ] + for m in modules: + lines += [ + f"[{m.name}]", + f" path = {m.root}", + " read only = yes", + " list = no", + f" auth users = {SHARE_USER}", + f" secrets file = {os.path.join(run_dir, m.name + '.secrets')}", + " strict modes = no", + f" filter = merge {os.path.join(run_dir, m.name + '.filter')}", + ] + return "\n".join(lines) + "\n" + + +def _getid(name: str) -> int: + """The process uid / gid; 0 on a host without them (only the tests run there).""" + return getattr(os, name, lambda: 0)() + + +def _write(path: str, text: str, mode: int = 0o600) -> None: + """Write atomically, so a connection that reads the config mid-rewrite sees old or new.""" + tmp = path + ".tmp" + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode) + with os.fdopen(fd, "w") as f: + f.write(text) + os.replace(tmp, path) + + +class RsyncDaemon: + """The sender's rsync daemon. Thread-safe; one instance lives for the server's lifetime.""" + + def __init__(self, run_dir: str, port: int, rsync_bin: str = "/usr/bin/rsync", + address: str = "0.0.0.0", popen=subprocess.Popen, killpg=None): + self._run_dir = run_dir + self.port = port + self._bin = rsync_bin + self._address = address + self._popen = popen + # Seams: tests never start or signal a real process. getattr keeps this importable on a + # host without process groups, so the tests run anywhere. + self._killpg = killpg or getattr(os, "killpg", None) + self._lock = threading.Lock() + self._proc = None + + @property + def _conf(self) -> str: + return os.path.join(self._run_dir, "rsyncd.conf") + + @property + def _pidfile(self) -> str: + return os.path.join(self._run_dir, "daemon.pid") + + def available(self) -> bool: + return os.path.isfile(self._bin) + + def running(self) -> bool: + with self._lock: + return self._proc is not None and self._proc.poll() is None + + def set_modules(self, modules: List[Module]) -> bool: + """Serve exactly ``modules``. Starts the daemon for the first module and stops it when + none is left. Returns True when the daemon serves them. A module with an unsafe value is + skipped (logged), never written.""" + safe = [m for m in modules if MODULE_RE.match(m.name) and _clean(m.root) + and _clean(m.secret) and m.files and all(_clean(f) for f in m.files)] + if len(safe) != len(modules): + _LOG.warning("share rsync: skipped %d unsafe module(s)", len(modules) - len(safe)) + with self._lock: + if not safe: + self._stop_locked() + return False + if not self.available(): + return False + os.makedirs(self._run_dir, exist_ok=True) + names = set() + for m in safe: + names.add(m.name) + _write(os.path.join(self._run_dir, m.name + ".secrets"), f"{SHARE_USER}:{m.secret}\n") + _write(os.path.join(self._run_dir, m.name + ".filter"), render_filter(m.files)) + _write(self._conf, render_config(safe, self._run_dir, _getid("getuid"), _getid("getgid"))) + self._remove_stale(names) + if self._proc is None or self._proc.poll() is not None: + self._start_locked() + return self._proc is not None and self._proc.poll() is None + + def stop(self) -> None: + with self._lock: + self._stop_locked() + + def sweep_startup(self) -> None: + """Stop a daemon a dead core left behind and remove its config and secrets.""" + try: + with open(self._pidfile) as f: + pgid = int(f.read().strip()) + self._killpg(pgid, _SIGKILL) + except (OSError, ValueError): + pass + shutil.rmtree(self._run_dir, ignore_errors=True) + + def _start_locked(self) -> None: + argv = [self._bin, "--daemon", "--no-detach", f"--config={self._conf}", + f"--address={self._address}", f"--port={self.port}"] + try: + # Own process group: the per-connection children are in it, so a stop ends a live + # transfer too (killing only the parent leaves the children serving). + self._proc = self._popen(argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + start_new_session=True) + except OSError as exc: + _LOG.warning("share rsync: daemon did not start: %s", exc) + self._proc = None + return + _write(self._pidfile, str(self._proc.pid)) + time.sleep(0.3) # a busy port makes the daemon exit at once; report that as not serving + if self._proc.poll() is not None: + _LOG.warning("share rsync: daemon exited at start (port %d busy?)", self.port) + self._proc = None + + def _stop_locked(self) -> None: + proc, self._proc = self._proc, None + if proc is not None and proc.poll() is None: + for sig in (signal.SIGTERM, _SIGKILL): + try: + self._killpg(proc.pid, sig) + proc.wait(timeout=3) + break + except (OSError, subprocess.TimeoutExpired): + continue + shutil.rmtree(self._run_dir, ignore_errors=True) + + def _remove_stale(self, keep: set) -> None: + for name in os.listdir(self._run_dir): + stem, ext = os.path.splitext(name) + if ext in (".secrets", ".filter") and stem not in keep: + try: + os.remove(os.path.join(self._run_dir, name)) + except OSError: + pass + + +def valid_source(source) -> Optional[str]: + """None when a pull source {host, port, module, token} is safe to use, else the reason.""" + if not isinstance(source, dict): + return "source must be an object" + host, port, module, token = (source.get(k) for k in ("host", "port", "module", "token")) + if not isinstance(host, str) or not HOST_RE.match(host): + return "invalid source host" + if isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535: + return "invalid source port" + if not isinstance(module, str) or not MODULE_RE.match(module): + return "invalid source module" + if not isinstance(token, str) or not _clean(token): + return "invalid source token" + return None + + +class RsyncPuller: + """The receiver's rsync client. ``runner`` is the liveness supervisor (a seam for tests).""" + + def __init__(self, rsync_bin: str = "/usr/bin/rsync", runner=run_supervised): + self._bin = rsync_bin + self._runner = runner + + def available(self) -> bool: + return os.path.isfile(self._bin) + + def pull_file(self, source: dict, remote_path: str, dest_path: str, + should_cancel: Optional[Callable[[], bool]] = None, + on_progress: Optional[Callable[[int], None]] = None): + """Pull one file of a module into ``dest_path``, resuming from its bytes. Returns the + supervisor result: ok, cancelled (SIGTERM, partial kept), or the failure.""" + url = (f"rsync://{SHARE_USER}@{source['host']}:{source['port']}" + f"/{source['module']}/{remote_path}") + argv = [self._bin, "-t", "--inplace", "--append-verify", + f"--timeout={IO_TIMEOUT_SECONDS}", f"--contimeout={CONNECT_TIMEOUT_SECONDS}", + url, dest_path] + # The password goes in the environment, never argv (argv is visible to every process). + env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "RSYNC_PASSWORD": source["token"]} + + def on_disk() -> int: + try: + return os.path.getsize(dest_path) + except OSError: + return 0 + + def progress(_activity: int) -> None: + if on_progress is not None: + on_progress(on_disk()) + + # Liveness is the bytes on disk: in a pull the parent rsync is the generator, which moves + # almost nothing while the receiver child writes, so its own I/O would look stalled. + return self._runner(argv, env, dest_dir=os.path.dirname(dest_path), + should_cancel=should_cancel, on_progress=progress, log=_LOG, + activity_fn=on_disk, die_with_parent=True) + + +def sweep_partials(partial_root: str, max_age: float = PARTIAL_MAX_AGE_SECONDS, + now: Optional[float] = None) -> None: + """Remove partial files that no retry picked up within ``max_age``.""" + now = time.time() if now is None else now + for dirpath, _dirs, files in os.walk(partial_root): + for name in files: + path = os.path.join(dirpath, name) + try: + if now - os.path.getmtime(path) > max_age: + os.remove(path) + except OSError: + pass diff --git a/static/conex-core/conex_core/server.py b/static/conex-core/conex_core/server.py index 4dd74d876..acad4fe19 100644 --- a/static/conex-core/conex_core/server.py +++ b/static/conex-core/conex_core/server.py @@ -7,6 +7,7 @@ import json import logging +import os import sys from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import parse_qs, urlsplit @@ -18,6 +19,7 @@ from .ports import ExportStream from .registry import Registry from .router import dispatch +from .rsync_share import RsyncDaemon, RsyncPuller from .share_ops import ShareOps _MAX_BODY = 1 * 1024 * 1024 # a plan request is small; cap the read defensively @@ -163,8 +165,13 @@ def serve(config: Config) -> None: format="%(asctime)s conex-core %(levelname)s %(message)s") registry = Registry.load(config) ops = ExportOps(registry, config.staging_dir) - import_ops = ImportOps(registry, registry.import_root) - share = ShareOps(registry, ops) + import_ops = ImportOps(registry, registry.import_root, partial_root=config.partial_dir, + puller=RsyncPuller(config.rsync_bin)) + rsync = None + if config.share_rsync_port: + rsync = RsyncDaemon(os.path.join(config.staging_dir, "rsyncd"), config.share_rsync_port, + config.rsync_bin) + share = ShareOps(registry, ops, rsync=rsync) # Reclaim op temps left by a dead process before serving (the process-death cleanup owner): a # fresh process holds no ops, so any op temp on disk is an orphan. Export reclaims its staging # temps, import its work temps; the share sweep then drops any in-memory sessions (K2GO-376 diff --git a/static/conex-core/conex_core/share_ops.py b/static/conex-core/conex_core/share_ops.py index 8207979b0..301eaaa35 100644 --- a/static/conex-core/conex_core/share_ops.py +++ b/static/conex-core/conex_core/share_ops.py @@ -51,6 +51,15 @@ revokes the token) and scoped to the item set. The only surface a token holder reaches is /v1/share/* scoped to this session's items; the wider /v1 API is never widened. +rsync transport (K2GO-577): when the box runs the share rsync daemon, each item an adapter can +list as files is also a read-only daemon module, named "-", with the session +token as its password. The manifest advertises it per item ({"rsync": {port, module}}), and a +receiver that knows rsync pulls it straight into place (resumable) instead of the HTTP fetch. The +session owns its modules: create adds them, every close (COMPLETE, CANCEL, safety cap) removes +them, and the last close stops the daemon. The receiver reports per-item progress to the gate +(POST .../progress), which keeps the session fresh during a long pull (the daemon itself reports +nothing) and gives the sender its progress view (GET .../progress). + This slice is loopback only: pairing (QR, hotspot, WLAN) is a later presentation slice and is outside the core, the way plugging in the Ethernet cable is not part of content exchange. """ @@ -64,6 +73,7 @@ from typing import Any, Dict, List, Tuple, Union from .registry import Registry +from .rsync_share import Module Response = Tuple[int, Any] @@ -87,8 +97,12 @@ class _Session: delivered: set = field(default_factory=set) # itemRefs with an open (or opening) stream: single-flight + "do not cap mid-transfer". streaming: set = field(default_factory=set) - # Last activity, for the safety cap only (bumped on manifest / fetch / complete / close). + # Last activity, for the safety cap only (bumped on manifest / fetch / progress / complete). last_seen: float = field(default_factory=time.monotonic) + # itemRef -> the rsync daemon module that serves it (K2GO-577). Empty without the daemon. + modules: Dict[str, Module] = field(default_factory=dict) + # itemRef -> the receiver's last progress report {movedBytes, totalBytes, state}. + progress: Dict[str, dict] = field(default_factory=dict) class ShareOps: @@ -97,12 +111,16 @@ class ShareOps: delegates every per-item transfer (the connection lifetime) to it.""" def __init__(self, registry: Registry, export_ops, - safety_cap_seconds: int = DEFAULT_SAFETY_CAP_SECONDS): + safety_cap_seconds: int = DEFAULT_SAFETY_CAP_SECONDS, rsync=None): self._registry = registry self._export = export_ops self._cap = safety_cap_seconds + self._rsync = rsync # the RsyncDaemon, or None: share over HTTP fetch only self._lock = threading.Lock() self._sessions: Dict[str, _Session] = {} + # Serializes snapshot + apply, so a stale module list never lands after a newer one (a + # closed session's module must not keep serving). + self._sync_lock = threading.Lock() # ---- REST-facing operations (return (status, payload), or (200, ExportStream)) ---- @@ -126,8 +144,11 @@ def create(self, items: Any) -> Response: scoped.append(key) session_id = secrets.token_hex(16) token = secrets.token_urlsafe(32) + session = _Session(token=token, items=scoped) + session.modules = self._modules_for(session_id, token, scoped) with self._lock: - self._sessions[session_id] = _Session(token=token, items=scoped) + self._sessions[session_id] = session + self._sync_daemon() # The pointer carries no token: the QR carries the address, the token and this pointer # as separate fields (ADR), and the receiver sends the token as the X-Share-Token header. return 200, { @@ -151,6 +172,7 @@ def manifest(self, session_id: str, token: Any) -> Response: found: List[dict] = [] missing: List[dict] = [] total = 0 + serving = self._ensure_serving(session) for index, (app, local_ref) in enumerate(session.items): item_ref = str(index) if not self._registry.has(app): @@ -165,6 +187,9 @@ def manifest(self, session_id: str, token: Any) -> Response: entry = item.to_json() entry["itemRef"] = item_ref entry["delivered"] = item_ref in session.delivered + module = session.modules.get(item_ref) + if module is not None and serving: + entry["rsync"] = {"port": self._rsync.port, "module": module.name} found.append(entry) total += item.bytes return 200, {"sessionId": session_id, "items": found, "missing": missing, @@ -219,6 +244,46 @@ def fetch(self, session_id: str, item_ref: str, token: Any): if not keep: self._release_streaming(session_id, item_ref) + def report_progress(self, session_id: str, token: Any, body: Any) -> Response: + """The receiver reports one item's progress {itemRef, movedBytes, totalBytes, state}. + Advisory, like ``delivered``: it feeds the sender's view and keeps the session fresh; a + state "done" also marks the item delivered.""" + session = self._resolve(session_id, token) + if isinstance(session, tuple): + return session + if not isinstance(body, dict): + return 400, {"error": "body must be {itemRef, movedBytes, totalBytes, state}"} + item_ref = body.get("itemRef") + if not isinstance(item_ref, str) or self._item(session, item_ref) is None: + return 404, {"error": "unknown item"} + report = {} + for key in ("movedBytes", "totalBytes"): + value = body.get(key) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return 400, {"error": f"{key} must be a non-negative integer"} + report[key] = value + state = body.get("state") + if state not in ("running", "done", "error"): + return 400, {"error": "state must be running, done or error"} + report["state"] = state + if state == "running": + self._ensure_serving(session) # a Retry pull needs the daemon up + with self._lock: + session.progress[item_ref] = report + if state == "done": + session.delivered.add(item_ref) + return 200, {"ok": True} + + def progress(self, session_id: str, token: Any) -> Response: + """The sender's view: the last progress the receiver reported per item.""" + session = self._resolve(session_id, token) + if isinstance(session, tuple): + return session + with self._lock: + items = [dict(report, itemRef=ref, delivered=ref in session.delivered) + for ref, report in sorted(session.progress.items(), key=lambda kv: int(kv[0]))] + return 200, {"sessionId": session_id, "items": items} + def complete(self, session_id: str, token: Any) -> Response: """Clean-success close: the receiver has every item it wanted and releases the session. This is the receiver-driven close (supersedes any drop-based auto-close). Revokes the @@ -241,6 +306,8 @@ def sweep_startup(self) -> None: that single sweep avoids a second temp owner.""" with self._lock: self._sessions.clear() + if self._rsync is not None: + self._rsync.sweep_startup() # ---- token + item resolution ---- @@ -274,6 +341,8 @@ def _close(self, session_id: str, token: Any) -> Response: child_ops = list(session.item_ops.values()) for op_id in child_ops: self._export.delete(op_id) # drops each child op + removes its staged temp + if session.modules: + self._sync_daemon() # revoke its rsync modules too (and stop an idle daemon) return 200, {"ok": True} @staticmethod @@ -392,3 +461,41 @@ def _sweep_capped(self) -> None: for session in capped: for op_id in list(session.item_ops.values()): self._export.delete(op_id) + if any(session.modules for session in capped): + self._sync_daemon() + + # ---- rsync daemon (K2GO-577) ---- + + def _modules_for(self, session_id: str, token: str, items) -> Dict[str, Module]: + """One daemon module per item the adapter can list as files. Empty without the daemon.""" + if self._rsync is None: + return {} + modules: Dict[str, Module] = {} + for index, (app, local_ref) in enumerate(items): + if not self._registry.has(app): + continue + files = self._registry.get(app).share_files(local_ref) + if files is not None: + item_ref = str(index) + modules[item_ref] = Module(name=f"{session_id}-{item_ref}", root=files.root, + files=list(files.files), secret=token) + return modules + + def _ensure_serving(self, session: _Session) -> bool: + """True when the daemon serves this session's modules, restarting a daemon that died while + the session was open (set_modules restarts a dead process).""" + if self._rsync is None or not session.modules: + return False + if not self._rsync.running(): + self._sync_daemon() + return self._rsync.running() + + def _sync_daemon(self) -> None: + """Make the daemon serve exactly the modules of the live sessions.""" + if self._rsync is None: + return + with self._sync_lock: + with self._lock: + modules = [m for session in self._sessions.values() + for m in session.modules.values()] + self._rsync.set_modules(modules) diff --git a/static/conex-core/conex_core/watchdog.py b/static/conex-core/conex_core/watchdog.py index 639063e34..f6e2baed0 100644 --- a/static/conex-core/conex_core/watchdog.py +++ b/static/conex-core/conex_core/watchdog.py @@ -161,6 +161,17 @@ def _kill(proc: subprocess.Popen) -> None: pass +def _die_with_parent() -> None: + """In the child, before exec: get SIGTERM when the parent dies (Linux PR_SET_PDEATHSIG). A pull + must not outlive the core that owns it, or a Retry starts a second writer on the same partial + file. Best effort: a host without prctl keeps the default.""" + try: + import ctypes + ctypes.CDLL(None).prctl(1, signal.SIGTERM) # 1 = PR_SET_PDEATHSIG + except Exception: + pass + + def classify_exit(returncode: int) -> SuperviseResult: """Map a child return code to a result. A negative code is death by signal; signal 9 (SIGKILL) on a child that was not cancelled is almost always the OOM killer, so it is named as such.""" @@ -181,14 +192,16 @@ def run_supervised(argv, env, *, dest_dir: Optional[str] = None, poll_seconds: float = 1.0, floor_seconds: float = 120.0, gap_multiplier: float = 4.0, cpu_grace: float = 3.0, on_progress: Optional[Callable[[int], None]] = None, - log=None) -> SuperviseResult: + log=None, activity_fn: Optional[Callable[[], int]] = None, + die_with_parent: bool = False) -> SuperviseResult: """Run argv as a child and supervise it by liveness. The child's stdout/stderr go to DEVNULL (the caller owns user output). dest_dir is the directory whose growth is the byte-progress fallback when /proc//io is unreadable. should_cancel is polled each interval; True terminates the child and returns CANCELLED. on_progress(activity_bytes) is called each poll so the caller can surface progress.""" try: - proc = subprocess.Popen(argv, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + proc = subprocess.Popen(argv, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + preexec_fn=_die_with_parent if die_with_parent and os.name == "posix" else None) except OSError as exc: # The child could not start at all (a missing binary, a bad interpreter). Own it here as a # clean failure rather than letting the raw OSError escape the supervisor. @@ -198,6 +211,13 @@ def run_supervised(argv, env, *, dest_dir: Optional[str] = None, use_io = proc_io_bytes(pid) is not None def activity() -> int: + # A caller that knows where the work lands measures it directly (an rsync pull: the + # parent process is the generator and moves almost no bytes while a child receives). + if activity_fn is not None: + try: + return activity_fn() + except Exception: # a probe must never break supervision + return 0 if use_io: value = proc_io_bytes(pid) if value is not None: diff --git a/static/conex-core/tests/test_rsync_share.py b/static/conex-core/tests/test_rsync_share.py new file mode 100644 index 000000000..85322fc54 --- /dev/null +++ b/static/conex-core/tests/test_rsync_share.py @@ -0,0 +1,435 @@ +"""rsync Share transport tests (K2GO-577): the daemon config and lifecycle, the pull client, the +partial sweep, the gate's rsync modules and progress, the pull import op and the Kiwix pull. +No real rsync runs: the process and the transfer are fakes behind the seams.""" + +import os +import tempfile +import time +import unittest + +from conex_core.adapters.kiwix import KiwixAdapter +from conex_core.import_ops import ImportOps +from conex_core.model import AUTHORITATIVE, Dedup, Identity, ManifestItem +from conex_core.ports import ContentAdapter, ShareFiles +from conex_core.registry import Registry +from conex_core.rsync_share import ( + SHARE_USER, + Module, + RsyncDaemon, + RsyncPuller, + render_config, + render_filter, + sweep_partials, + valid_source, +) +from conex_core.share_ops import ShareOps +from conex_core.watchdog import CANCELLED, FAILED, OK, SuperviseResult + +SOURCE = {"host": "192.168.43.1", "port": 9873, "module": "abc123-0", "token": "tok"} + + +class _FakeProc: + def __init__(self, pid=4242): + self.pid = pid + self.alive = True + + def poll(self): + return None if self.alive else 0 + + def wait(self, timeout=None): + return 0 + + +class _FakePopen: + def __init__(self): + self.calls = [] + self.procs = [] + + def __call__(self, argv, **kwargs): + self.calls.append((argv, kwargs)) + proc = _FakeProc(pid=1000 + len(self.calls)) + self.procs.append(proc) + return proc + + +class _Killer: + def __init__(self, popen): + self.popen = popen + self.signals = [] + + def __call__(self, pgid, sig): + self.signals.append((pgid, sig)) + for proc in self.popen.procs: + if proc.pid == pgid: + proc.alive = False + + +def _daemon(run_dir, available=True): + popen = _FakePopen() + killer = _Killer(popen) + rsync_bin = os.path.join(run_dir, "..", "rsync-bin") + if available: + open(rsync_bin, "w").close() + daemon = RsyncDaemon(run_dir, 9873, rsync_bin, popen=popen, killpg=killer) + return daemon, popen, killer + + +def _module(name="s1-0", files=("a.zim",), secret="tok", root="/library/zims/content"): + return Module(name=name, root=root, files=list(files), secret=secret) + + +class RenderTests(unittest.TestCase): + def test_filter_admits_only_the_file(self): + self.assertEqual(render_filter(["a.zim"]), "+ /a.zim\n- *\n") + + def test_filter_admits_nested_parents_once(self): + text = render_filter(["content/storage/a/b/x.mp4", "content/storage/a/c/y.png"]) + self.assertEqual(text.splitlines(), [ + "+ /content/", "+ /content/storage/", "+ /content/storage/a/", + "+ /content/storage/a/b/", "+ /content/storage/a/b/x.mp4", + "+ /content/storage/a/c/", "+ /content/storage/a/c/y.png", "- *"]) + + def test_config_is_read_only_authenticated_and_logs_nothing(self): + text = render_config([_module()], "/run/r", 10123, 10123) + for line in ("[s1-0]", " read only = yes", " list = no", f" auth users = {SHARE_USER}", + " path = /library/zims/content", "log file = /dev/null", "uid = 10123", + " secrets file = " + os.path.join("/run/r", "s1-0.secrets")): + self.assertIn(line, text.splitlines()) + + +class DaemonTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.run_dir = os.path.join(self.tmp, "rsyncd") + + def test_first_module_starts_last_close_stops(self): + daemon, popen, killer = _daemon(self.run_dir) + self.assertTrue(daemon.set_modules([_module()])) + self.assertEqual(len(popen.calls), 1) + argv, kwargs = popen.calls[0] + self.assertIn("--daemon", argv) + self.assertIn("--port=9873", argv) + self.assertTrue(kwargs.get("start_new_session")) + with open(os.path.join(self.run_dir, "s1-0.secrets")) as f: + self.assertEqual(f.read(), f"{SHARE_USER}:tok\n") + # A second session only rewrites the config: the running daemon re-reads it. + self.assertTrue(daemon.set_modules([_module(), _module(name="s2-0", secret="t2")])) + self.assertEqual(len(popen.calls), 1) + self.assertFalse(daemon.set_modules([])) + self.assertTrue(killer.signals) + self.assertFalse(os.path.exists(self.run_dir)) # secrets are gone with the daemon + + def test_closed_session_secrets_are_removed(self): + daemon, _popen, _killer = _daemon(self.run_dir) + daemon.set_modules([_module(), _module(name="s2-0")]) + daemon.set_modules([_module(name="s2-0")]) + self.assertFalse(os.path.exists(os.path.join(self.run_dir, "s1-0.secrets"))) + self.assertTrue(os.path.exists(os.path.join(self.run_dir, "s2-0.secrets"))) + + def test_unsafe_module_is_never_written(self): + daemon, popen, _killer = _daemon(self.run_dir) + bad = [_module(name="x]y"), _module(files=("a.zim\n[evil]",)), _module(secret="a\nb")] + self.assertFalse(daemon.set_modules(bad)) + self.assertEqual(popen.calls, []) + + def test_no_rsync_binary_means_not_serving(self): + daemon, popen, _killer = _daemon(self.run_dir, available=False) + self.assertFalse(daemon.set_modules([_module()])) + self.assertEqual(popen.calls, []) + + def test_startup_sweep_kills_the_recorded_group(self): + os.makedirs(self.run_dir) + with open(os.path.join(self.run_dir, "daemon.pid"), "w") as f: + f.write("777") + daemon, _popen, killer = _daemon(self.run_dir) + daemon.sweep_startup() + self.assertEqual(killer.signals[0][0], 777) + self.assertFalse(os.path.exists(self.run_dir)) + + +class SourceAndPullTests(unittest.TestCase): + def test_valid_source(self): + self.assertIsNone(valid_source(SOURCE)) + for key, value in (("host", "a b"), ("host", "-x"), ("port", 0), ("port", True), + ("port", "9873"), ("module", "../x"), ("token", ""), + ("token", "a\nb")): + self.assertIsNotNone(valid_source(dict(SOURCE, **{key: value})), (key, value)) + self.assertIsNotNone(valid_source(None)) + + def test_pull_keeps_the_password_out_of_argv_and_reports_file_size(self): + tmp = tempfile.mkdtemp() + dest = os.path.join(tmp, "a.zim") + seen = {} + + def runner(argv, env, **kwargs): + seen["argv"], seen["env"] = argv, env + with open(dest, "wb") as f: + f.write(b"x" * 10) + kwargs["on_progress"](0) + seen["activity"] = kwargs["activity_fn"]() + return SuperviseResult(OK, 0, "ok") + + moved = [] + result = RsyncPuller("/usr/bin/rsync", runner=runner).pull_file( + SOURCE, "a.zim", dest, on_progress=moved.append) + self.assertTrue(result.ok) + self.assertEqual(moved, [10]) + self.assertEqual(seen["activity"], 10) # liveness = bytes on disk, not the parent's I/O + self.assertNotIn("tok", " ".join(seen["argv"])) + self.assertEqual(seen["env"]["RSYNC_PASSWORD"], "tok") + self.assertIn("--inplace", seen["argv"]) + self.assertIn("--append-verify", seen["argv"]) + self.assertIn(f"rsync://{SHARE_USER}@192.168.43.1:9873/abc123-0/a.zim", seen["argv"]) + + def test_sweep_removes_only_old_partials(self): + root = tempfile.mkdtemp() + os.makedirs(os.path.join(root, "kiwix")) + old, new = (os.path.join(root, "kiwix", n) for n in ("old.zim", "new.zim")) + for path in (old, new): + open(path, "w").close() + now = time.time() + os.utime(old, (now - 3 * 24 * 3600, now - 3 * 24 * 3600)) + sweep_partials(root, now=now) + self.assertFalse(os.path.exists(old)) + self.assertTrue(os.path.exists(new)) + + +def _mitem(app, local_ref, total): + return ManifestItem( + app=app, kind="zim", title=local_ref, bytes=total, + identity=Identity(local_ref, Dedup(line={}, build=None, version="", id_kind=AUTHORITATIVE))) + + +class _ShareFake(ContentAdapter): + app = "kiwix" + + def __init__(self, items, shareable=True): + self._items = dict(items) + self._shareable = shareable + + def is_installed(self): + return True + + def list(self): + return [_mitem(self.app, ref, size) for ref, size in self._items.items()] + + def share_files(self, local_ref): + if not self._shareable or local_ref not in self._items: + return None + return ShareFiles(root="/zims", files=[local_ref]) + + +class _RecDaemon: + port = 9873 + + def __init__(self, serving=True): + self.modules = None + self.serving = serving + self.swept = False + + def running(self): + return self.serving and bool(self.modules) + + def set_modules(self, modules): + self.modules = list(modules) + return self.running() + + def sweep_startup(self): + self.swept = True + + +class ShareGateRsyncTests(unittest.TestCase): + def _share(self, adapter, daemon): + return ShareOps(Registry([adapter]), export_ops=None, rsync=daemon) + + def test_manifest_advertises_the_module_and_close_revokes_it(self): + daemon = _RecDaemon() + share = self._share(_ShareFake({"a.zim": 5, "b.zim": 7}), daemon) + _, created = share.create([{"app": "kiwix", "localRef": "a.zim"}, + {"app": "kiwix", "localRef": "b.zim"}]) + sid, token = created["sessionId"], created["token"] + self.assertEqual([m.name for m in daemon.modules], [f"{sid}-0", f"{sid}-1"]) + self.assertTrue(all(m.secret == token for m in daemon.modules)) + _, manifest = share.manifest(sid, token) + self.assertEqual(manifest["items"][0]["rsync"], {"port": 9873, "module": f"{sid}-0"}) + self.assertEqual(share.delete(sid, token)[0], 200) + self.assertEqual(daemon.modules, []) + + def test_no_rsync_block_when_daemon_is_not_serving(self): + daemon = _RecDaemon(serving=False) + share = self._share(_ShareFake({"a.zim": 5}), daemon) + _, created = share.create([{"app": "kiwix", "localRef": "a.zim"}]) + _, manifest = share.manifest(created["sessionId"], created["token"]) + self.assertNotIn("rsync", manifest["items"][0]) + + def test_progress_report_and_view(self): + share = self._share(_ShareFake({"a.zim": 5}), _RecDaemon()) + _, created = share.create([{"app": "kiwix", "localRef": "a.zim"}]) + sid, token = created["sessionId"], created["token"] + report = {"itemRef": "0", "movedBytes": 3, "totalBytes": 5, "state": "running"} + self.assertEqual(share.report_progress(sid, token, report)[0], 200) + self.assertEqual(share.report_progress(sid, "bad", report)[0], 403) + self.assertEqual(share.report_progress(sid, token, dict(report, itemRef="9"))[0], 404) + self.assertEqual(share.report_progress(sid, token, dict(report, movedBytes=-1))[0], 400) + self.assertEqual(share.report_progress(sid, token, dict(report, state="x"))[0], 400) + share.report_progress(sid, token, dict(report, movedBytes=5, state="done")) + _, view = share.progress(sid, token) + self.assertEqual(view["items"], [{"itemRef": "0", "movedBytes": 5, "totalBytes": 5, + "state": "done", "delivered": True}]) + + def test_dead_daemon_is_restarted_by_manifest_and_progress(self): + daemon = _RecDaemon() + share = self._share(_ShareFake({"a.zim": 5}), daemon) + _, created = share.create([{"app": "kiwix", "localRef": "a.zim"}]) + sid, token = created["sessionId"], created["token"] + calls = [] + real_set = daemon.set_modules + daemon.set_modules = lambda modules: calls.append(len(modules)) or real_set(modules) + daemon.modules = [] # the daemon died: it serves nothing + _, manifest = share.manifest(sid, token) + self.assertEqual(calls, [1]) + self.assertIn("rsync", manifest["items"][0]) + daemon.modules = [] + share.report_progress(sid, token, {"itemRef": "0", "movedBytes": 1, "totalBytes": 5, + "state": "running"}) + self.assertEqual(calls, [1, 1]) + + def test_startup_sweep_reaches_the_daemon(self): + daemon = _RecDaemon() + self._share(_ShareFake({}), daemon).sweep_startup() + self.assertTrue(daemon.swept) + + +class _PullAdapter(_ShareFake): + def __init__(self): + super().__init__({}) + self.calls = [] + + def pull_item(self, entry, source, partial_dir, puller, should_cancel=None, on_progress=None): + self.calls.append((entry, source, partial_dir)) + on_progress(4) + return {"ok": True, "app": "kiwix", "localRef": "a.zim"} + + +class _OkPuller: + def available(self): + return True + + +class PullOpTests(unittest.TestCase): + def _wait(self, ops, op_id): + for _ in range(100): + _, st = ops.status(op_id) + if st["state"] != "running": + return st + time.sleep(0.01) + self.fail("op did not finish") + + def test_pull_runs_the_adapter_and_finishes(self): + adapter = _PullAdapter() + ops = ImportOps(Registry([adapter]), tempfile.mkdtemp(), partial_root="/p", + puller=_OkPuller()) + status, started = ops.start_pull("kiwix", {"bytes": 4}, SOURCE) + self.assertEqual(status, 200) + st = self._wait(ops, started["opId"]) + self.assertEqual(st["state"], "done") + self.assertEqual(adapter.calls[0][1], SOURCE) + self.assertEqual(adapter.calls[0][2], "/p") + + def test_pull_rejects_bad_requests(self): + ops = ImportOps(Registry([_PullAdapter()]), tempfile.mkdtemp(), partial_root="/p", + puller=_OkPuller()) + self.assertEqual(ops.start_pull("nope", {}, SOURCE)[0], 404) + self.assertEqual(ops.start_pull("kiwix", None, SOURCE)[0], 400) + self.assertEqual(ops.start_pull("kiwix", {}, dict(SOURCE, port=0))[0], 400) + no_pull = ImportOps(Registry([_PullAdapter()]), tempfile.mkdtemp()) + self.assertEqual(no_pull.start_pull("kiwix", {}, SOURCE)[0], 501) + + +class DiscardTests(unittest.TestCase): + def test_kiwix_discard_frees_the_kept_partial(self): + partial = tempfile.mkdtemp() + os.makedirs(os.path.join(partial, "kiwix")) + with open(os.path.join(partial, "kiwix", "a_en_all_2026-09.zim"), "wb") as f: + f.write(b"z" * 123) + adapter = KiwixAdapter(tempfile.mkdtemp(), indexer="") + entry = {"identity": {"localRef": "a_en_all_2026-09.zim"}} + self.assertEqual(adapter.discard_pull(entry, partial), {"ok": True, "freedBytes": 123}) + self.assertEqual(adapter.discard_pull(entry, partial), {"ok": True, "freedBytes": 0}) + self.assertFalse(adapter.discard_pull({"identity": {"localRef": "../x.zim"}}, partial)["ok"]) + + def test_discard_is_refused_while_a_pull_of_the_item_runs(self): + class _Slow(_PullAdapter): + def pull_item(self, entry, source, partial_dir, puller, should_cancel=None, on_progress=None): + while not should_cancel(): + time.sleep(0.01) + return {"ok": False, "reason": "import cancelled"} + + def discard_pull(self, entry, partial_dir): + return {"ok": True, "freedBytes": 9} + + ops = ImportOps(Registry([_Slow()]), tempfile.mkdtemp(), partial_root="/p", puller=_OkPuller()) + entry = {"identity": {"localRef": "a.zim"}} + _, started = ops.start_pull("kiwix", entry, SOURCE) + self.assertEqual(ops.discard("kiwix", entry)[0], 409) + ops.delete(started["opId"]) # the client aborts the pull first + self.assertEqual(ops.discard("kiwix", entry), (200, {"ok": True, "freedBytes": 9})) + + +class _FakePuller: + def __init__(self, state=OK, data=b"ZIMDATA"): + self.state = state + self.data = data + self.calls = [] + + def pull_file(self, source, remote_path, dest_path, should_cancel=None, on_progress=None): + self.calls.append((remote_path, dest_path)) + with open(dest_path, "ab") as f: + f.write(self.data) + if on_progress: + on_progress(os.path.getsize(dest_path)) + detail = "ok" if self.state == OK else "exited with code 12" + return SuperviseResult(self.state, 0 if self.state == OK else 12, detail) + + +class KiwixPullTests(unittest.TestCase): + def setUp(self): + self.zims = tempfile.mkdtemp() + self.partial = tempfile.mkdtemp() + self.adapter = KiwixAdapter(self.zims, indexer="") + self.entry = {"identity": {"localRef": "wiki_en_all_2026-09.zim"}} + + def test_success_renames_into_place(self): + puller = _FakePuller() + result = self.adapter.pull_item(self.entry, SOURCE, self.partial, puller) + self.assertTrue(result["ok"]) + self.assertEqual(puller.calls[0][0], "wiki_en_all_2026-09.zim") + self.assertTrue(os.path.isfile(os.path.join(self.zims, "wiki_en_all_2026-09.zim"))) + self.assertFalse(os.path.exists(puller.calls[0][1])) # the partial became the file + + def test_drop_keeps_the_partial_for_a_resume(self): + puller = _FakePuller(state=FAILED) + result = self.adapter.pull_item(self.entry, SOURCE, self.partial, puller) + self.assertFalse(result["ok"]) + self.assertTrue(os.path.isfile(puller.calls[0][1])) + self.assertFalse(os.path.exists(os.path.join(self.zims, "wiki_en_all_2026-09.zim"))) + + def test_cancel_keeps_the_partial(self): + puller = _FakePuller(state=CANCELLED) + result = self.adapter.pull_item(self.entry, SOURCE, self.partial, puller) + self.assertEqual(result["reason"], "import cancelled") + self.assertTrue(os.path.isfile(puller.calls[0][1])) + + def test_unsafe_local_ref_is_rejected(self): + bad = {"identity": {"localRef": "../x.zim"}} + self.assertFalse(self.adapter.pull_item(bad, SOURCE, self.partial, _FakePuller())["ok"]) + + def test_share_files_lists_only_the_zim(self): + open(os.path.join(self.zims, "a.zim"), "w").close() + files = self.adapter.share_files("a.zim") + self.assertEqual((files.root, files.files), (self.zims, ["a.zim"])) + self.assertIsNone(self.adapter.share_files("missing.zim")) + + +if __name__ == "__main__": + unittest.main() diff --git a/static/conex-core/tests/test_watchdog.py b/static/conex-core/tests/test_watchdog.py index d32dfa4f8..3bc57c761 100644 --- a/static/conex-core/tests/test_watchdog.py +++ b/static/conex-core/tests/test_watchdog.py @@ -118,5 +118,24 @@ def test_on_progress_is_called(self): self.assertTrue(seen) # at least one progress sample while the child ran + def test_activity_fn_is_the_liveness_signal(self): + # A child that does nothing itself, while the work it drives moves (an rsync generator + # whose receiver child writes): growing activity keeps it alive past the floor. + moved = [0] + + def growing(): + moved[0] += 1 + return moved[0] + + alive = run_supervised( + [sys.executable, "-c", "import time; time.sleep(0.5)"], None, poll_seconds=0.02, + floor_seconds=0.1, activity_fn=growing) + self.assertEqual(alive.state, OK) + stalled = run_supervised( + [sys.executable, "-c", "import time; time.sleep(5)"], None, poll_seconds=0.02, + floor_seconds=0.1, cpu_grace=1.0, activity_fn=lambda: 7) + self.assertEqual(stalled.state, STALLED) + + if __name__ == "__main__": unittest.main() diff --git a/static/dashboard/CHANGELOG.md b/static/dashboard/CHANGELOG.md index dfe1f268c..e02065676 100644 --- a/static/dashboard/CHANGELOG.md +++ b/static/dashboard/CHANGELOG.md @@ -4,6 +4,7 @@ One line per version, newest first. Every REST-facing change bumps the version i (the app surfaces it via `/system/dashboard/update-check` and the "Update available" pill), so this file is the human record of what each bump enables. Keep entries short: `version - change (TICKET)`. +- **1.6.3** - The `/v1` relay to conex-core forwards the `X-Share-Token` header (never logged). Before, it was dropped, so the sender's close (`DELETE /v1/share/`) got 401 and the share token stayed valid after Done. (K2GO-577) - **1.6.2** - Box logs no longer identify the user, on boxes installed before bake patches 0010-0012 (K2GO-559). At its first start, dash-node applies the same change once: nginx access log without client IP and User-Agent (`/etc/nginx/nginx.conf`), Forgejo router log off (`LOGGER_ROUTER_MODE` empty in `app.ini`), Calibre-Web log level DEBUG -> INFO (`app.db`). A value an admin changed is kept. Each change applies at the service restart (Forgejo: at the next app open, as a pending privacy change; nginx and Calibre-Web: at the next box start). After that restart, the older logs of each changed service are removed once (rotated copies deleted, live log emptied). Markers: `/library/dashboard/log-less.done`, `/library/dashboard/log-less-purge.done`. No REST change. (K2GO-559) - **1.6.1** - The box restarts conex-core (Content Exchange) on its own when its process is gone: service-heal probes `/v1/apps` and runs `pdsm restart conex-core` only on a refused connection (a busy, slow service is never restarted mid-operation), and skips boxes without the conex-core pdsm service. `POST /system/service/conex-core/restart` is allowed. Localhost-only. (K2GO-574) - **1.6.0** - Privacy switches for the box services that contact an outside server on their own (K2GO-559). `GET /system/privacy` reports each one (Kolibri statistics ping, Forgejo update check): installed, and whether it may contact its server. `POST /system/privacy/:service {outbound}` sets the service's own config key (Kolibri `[Deployment] DISABLE_PING` in `options.ini`, Forgejo `[cron.update_checker] ENABLED` in `app.ini`) and only writes the config; it never restarts a service. `pending` reports a change that waits for a restart (config newer than the running process; derived on each read, not stored). The app asks the user (Restart now / Later) and requests the restart through `POST /system/service/:svc/restart` (ADR-5343), which now runs one restart per service at a time; a pending change is restarted at the next app open. A missing key reads as the service default (on). On its first start a box sets every installed switch OFF once (new bakes ship OFF), then writes `/library/dashboard/privacy-defaults.done` so a switch the user turns on later stays on; it applies like any pending change. Localhost-only. (K2GO-559) diff --git a/static/dashboard/package.json b/static/dashboard/package.json index d965e168a..bebc0d240 100644 --- a/static/dashboard/package.json +++ b/static/dashboard/package.json @@ -1,6 +1,6 @@ { "name": "dashboard-console", - "version": "1.6.2", + "version": "1.6.3", "description": "", "main": "index.js", "scripts": { diff --git a/static/dashboard/sockets/conex-proxy.test.ts b/static/dashboard/sockets/conex-proxy.test.ts index f718aa94b..dec2f5056 100644 --- a/static/dashboard/sockets/conex-proxy.test.ts +++ b/static/dashboard/sockets/conex-proxy.test.ts @@ -125,6 +125,34 @@ test('conexProxy passes Content-Disposition through (the export filename)', asyn } }); +test('conexProxy forwards X-Share-Token (the sender close DELETE)', async () => { + const core = http.createServer((req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ token: req.headers['x-share-token'] ?? null, method: req.method })); + }); + const corePort = await listen(core); + process.env.CONEX_PORT = String(corePort); + const app = await startProxyApp(); + try { + const r = await new Promise<{ status: number; body: string }>((resolve, reject) => { + const req = http.request({ host: '127.0.0.1', port: app.port, path: '/v1/share/abc', + method: 'DELETE', headers: { 'x-share-token': 'tok-123' } }, (res) => { + let data = ''; + res.on('data', (c) => { data += c; }); + res.on('end', () => resolve({ status: res.statusCode || 0, body: data })); + }); + req.on('error', reject); + req.end(); + }); + assert.equal(r.status, 200); + assert.deepEqual(JSON.parse(r.body), { token: 'tok-123', method: 'DELETE' }); + const none = await get(app.port, '/v1/apps'); + assert.equal(JSON.parse(none.body).token, null); + } finally { + app.close(); core.close(); + } +}); + test('conexProxy returns 502 when the core is unreachable', async () => { // Point at a port with nothing listening. const dead = http.createServer(); diff --git a/static/dashboard/sockets/conex-proxy.ts b/static/dashboard/sockets/conex-proxy.ts index da22d3588..0e63c161a 100644 --- a/static/dashboard/sockets/conex-proxy.ts +++ b/static/dashboard/sockets/conex-proxy.ts @@ -33,6 +33,10 @@ export function conexProxy(req: Request, res: Response): void { headers['content-type'] = 'application/json'; headers['content-length'] = String(payload.length); } + // The share-session token is the gate's only auth (the sender's close DELETE carries it). + // Forward it, and never log it. + const shareToken = req.headers['x-share-token']; + if (typeof shareToken === 'string' && shareToken) headers['x-share-token'] = shareToken; const upstream = http.request( { host: CONEX_HOST, port: conexPort(), method: req.method, path: corePath, headers },