diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/BoxImportStaging.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/BoxImportStaging.java index d63df5cc2..bc47db2ca 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/BoxImportStaging.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/BoxImportStaging.java @@ -17,11 +17,13 @@ import org.appdevforall.k2go.SystemStateEvaluator; import org.appdevforall.k2go.contentexchange.domain.ExportFilenames; +import org.appdevforall.k2go.contentexchange.domain.PartialPayload; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.util.Set; public final class BoxImportStaging { @@ -48,10 +50,34 @@ public static String safeName(String name) { /** Open the host staging file for writing, creating the staging directory if needed. */ public OutputStream open(String storedName) throws IOException { + return open(storedName, false); + } + + /** Open the host staging file, appending to the kept bytes when {@code append} (a resume). */ + public OutputStream open(String storedName, boolean append) throws IOException { if (!hostDir.exists() && !hostDir.mkdirs()) { throw new IOException("Could not create the import staging folder"); } - return new FileOutputStream(new File(hostDir, storedName)); + return new FileOutputStream(new File(hostDir, storedName), append); + } + + /** Bytes in a staged file, 0 when it does not exist. */ + public long length(String storedName) { + return new File(hostDir, storedName).length(); + } + + /** Remove every kept partial whose name is not in {@code keep} (K2GO-577). */ + public void sweepPartials(Set keep) { + File[] files = hostDir.listFiles(); + if (files == null) { + return; + } + for (File f : files) { + if (PartialPayload.isPartial(f.getName()) && !keep.contains(f.getName())) { + //noinspection ResultOfMethodCallIgnored + f.delete(); + } + } } /** Remove a staged file after the core has ingested it (best effort). */ diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/NetworkImportCarrier.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/NetworkImportCarrier.java index afad31cd2..653be1436 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/NetworkImportCarrier.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/NetworkImportCarrier.java @@ -6,7 +6,8 @@ * Description : K2GO-376. The network ImportCarrier: the mirror of SafImportCarrier, but each item is * read over HTTP from the sender's live conex-core gate on the shared hotspot, not from a * SAF drive. readManifest() GETs the invite's manifest URL; openPayload(itemRef) GETs - * its sibling .../fetch/, retrying while the sender stages the item; the token + * its sibling .../fetch/, retrying while the sender stages the item, with a + * Range header to resume a cut item (K2GO-577); the token * rides the X-Share-Token header, never the URL. Everything downstream (stage to * /library/.conex-staging, POST /v1/import/item) is unchanged from the disk path, so the * carrier is the only sender-specific dependency of the import service. On a clean finish @@ -20,6 +21,7 @@ import android.util.Log; import org.appdevforall.k2go.contentexchange.domain.ImportCarrier; +import org.appdevforall.k2go.contentexchange.domain.PartialPayload; import org.appdevforall.k2go.contentexchange.domain.ShareTransport; import org.appdevforall.k2go.contentexchange.domain.ShareUrls; import org.appdevforall.k2go.sync.transport.WifiNetworkBinder; @@ -121,6 +123,12 @@ public List listFiles() { @Override public InputStream openPayload(String itemRef) throws IOException { + return openPayloadFrom(itemRef, 0L).stream(); + } + + /** Fetch one item, asking for {@code Range: bytes=offset-} to resume a cut copy (K2GO-577). */ + @Override + public Payload openPayloadFrom(String itemRef, long offset) throws IOException { String url = ShareUrls.fetch(baseUrl, manifestUrl, itemRef); long deadline = System.currentTimeMillis() + FETCH_MAX_WAIT_MS; while (true) { @@ -128,6 +136,9 @@ public InputStream openPayload(String itemRef) throws IOException { throw new IOException("cancelled"); } HttpURLConnection c = open(url, "GET", FETCH_STALL_TIMEOUT_MS); + if (offset > 0) { + c.setRequestProperty("Range", "bytes=" + offset + "-"); + } int code; try { code = c.getResponseCode(); @@ -135,10 +146,17 @@ public InputStream openPayload(String itemRef) throws IOException { c.disconnect(); throw e; } - if (code == 200) { + if (code == 200 || code == 206) { + long start = PartialPayload.bodyStart(code, offset, c.getHeaderField("Content-Range")); + if (start < 0) { + c.disconnect(); + throw new IOException("Share fetch: reply does not continue byte " + offset); + } // The caller streams and closes this; closing disconnects the connection. try { - return new ConnectionInputStream(c, c.getInputStream()); + // The sender marks a byte-exact file (a ZIM) with Accept-Ranges; a tar has none. + boolean exact = "bytes".equalsIgnoreCase(c.getHeaderField("Accept-Ranges")); + return new Payload(new ConnectionInputStream(c, c.getInputStream()), start, exact); } catch (IOException e) { c.disconnect(); // never leak the socket if the body stream cannot be opened throw e; diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareManifestReader.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareManifestReader.java index 9709c000b..32f44cb5d 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareManifestReader.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/data/ShareManifestReader.java @@ -36,6 +36,27 @@ private ShareManifestReader() { * missing (unknown app / not found) are not in {@code items}, so they never produce an entry. An * item that cannot be addressed (no app, no localRef, or a non-numeric itemRef) is skipped. */ + /** + * The item's exact-build id (the ZIM UUID) from its entry JSON, or null when the identity is not + * authoritative or has no build. Keys the kept partial of a cut transfer (K2GO-577). + */ + public static String buildId(String entryJson) { + if (entryJson == null) { + return null; + } + try { + JSONObject dedup = new JSONObject(entryJson).optJSONObject("identity"); + dedup = dedup != null ? dedup.optJSONObject("dedup") : null; + if (dedup == null || !"authoritative".equals(dedup.optString("idKind"))) { + return null; + } + String build = dedup.optString("build", ""); + return build.isEmpty() || "null".equals(build) ? null : build; + } catch (Exception e) { + return null; + } + } + public static List parse(byte[] manifestBytes) throws Exception { List out = new ArrayList<>(); if (manifestBytes == null || manifestBytes.length == 0) { diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ImportCarrier.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ImportCarrier.java index 97e9abc58..95978af5c 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ImportCarrier.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/ImportCarrier.java @@ -26,6 +26,44 @@ public interface ImportCarrier { /** Open a readable stream for one payload file; the caller copies it, then closes it. */ InputStream openPayload(String name) throws IOException; + /** + * Open one payload from byte {@code offset} to resume a cut copy (K2GO-577). The source may send + * the whole item instead; {@link Payload#start()} says where the body starts. Default: the whole + * item, not resumable (a drive source has nothing to resume). + */ + default Payload openPayloadFrom(String name, long offset) throws IOException { + return new Payload(openPayload(name), 0L, false); + } + + /** + * A payload body, the item byte it starts at, and whether the item is one exact file the source + * can resume (so a cut copy is worth keeping). Immutable value. + */ + final class Payload { + + private final InputStream stream; + private final long start; + private final boolean exact; + + public Payload(InputStream stream, long start, boolean exact) { + this.stream = stream; + this.start = start; + this.exact = exact; + } + + public boolean exact() { + return exact; + } + + public InputStream stream() { + return stream; + } + + public long start() { + return start; + } + } + /** * Signal that the run finished with every requested item delivered (a clean success). The default * is a no-op (a drive source has nothing to close); the network carrier uses it to tell the sender diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/PartialPayload.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/PartialPayload.java new file mode 100644 index 000000000..87f57b746 --- /dev/null +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/domain/PartialPayload.java @@ -0,0 +1,91 @@ +/* + * ============================================================================ + * Name : PartialPayload.java + * Author : AppDevForAll + * Copyright : Copyright (c) 2026 AppDevForAll + * Description : K2GO-577. The rules for resuming a cut Share item: which items keep a partial file + * (an exact-build id, e.g. the ZIM UUID), where a resume starts, and whether the + * sender's reply continues the kept bytes. Pure JVM, unit-tested. + * ============================================================================ + */ +package org.appdevforall.k2go.contentexchange.domain; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A cut Share item keeps its bytes in a partial file named by the item's exact-build id, so a Retry, + * or a new share of the same build, continues it with {@code Range: bytes=N-}. The sender decides if + * it can resume (206) or must send the whole item (200); the box verifies the item before import. + */ +public final class PartialPayload { + + /** Staging-name prefix of every kept partial, so a run can find and sweep stale ones. */ + public static final String PREFIX = "partial-"; + + private static final Pattern APP = Pattern.compile("[a-z0-9]{1,16}"); + private static final Pattern BUILD = Pattern.compile("[A-Za-z0-9-]{1,64}"); + private static final Pattern CONTENT_RANGE = Pattern.compile("bytes (\\d{1,19})-\\d{1,19}/\\d{1,19}"); + + private PartialPayload() { + } + + /** + * The staging name of the partial for an item, or null when the item has no safe exact-build id + * (then it is not kept: a cut item restarts at 0, as before). + */ + public static String name(String app, String buildId) { + if (app == null || buildId == null || !APP.matcher(app).matches() + || !BUILD.matcher(buildId).matches()) { + return null; + } + return PREFIX + app + "-" + buildId; + } + + public static boolean isPartial(String storedName) { + return storedName != null && storedName.startsWith(PREFIX); + } + + /** + * The byte to resume from: the kept length when it is not more than the item size, else 0 (a + * longer file is not this item; start over). Equal to the size means nothing is left to fetch. + */ + public static long resumeFrom(long kept, long size) { + return kept > 0 && kept <= size ? kept : 0L; + } + + /** + * True when a failed item keeps its partial for a later resume: only an exact-file transfer that + * was cut. A failed ingest means the box rejected the bytes, and a cancel means the user gave up. + */ + public static boolean keepOnFailure(boolean resumable, boolean copied, boolean cancelled) { + return resumable && !copied && !cancelled; + } + + /** True when an exact-file copy holds the whole item, no more and no less. */ + public static boolean isComplete(long copied, long size) { + return copied == size; + } + + /** + * Where the reply body starts in the item: 0 for a whole item (200), the requested byte for a 206 + * whose Content-Range starts there, else -1 (a reply that does not continue the kept bytes). + */ + public static long bodyStart(int code, long requested, String contentRange) { + if (code == 200) { + return 0L; + } + if (code != 206 || contentRange == null) { + return -1L; + } + Matcher m = CONTENT_RANGE.matcher(contentRange.trim()); + if (!m.matches()) { + return -1L; + } + try { + return Long.parseLong(m.group(1)) == requested ? requested : -1L; + } catch (NumberFormatException e) { + return -1L; + } + } +} diff --git a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/presentation/ContentImportService.java b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/presentation/ContentImportService.java index 760c012cd..83f08bcc4 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/presentation/ContentImportService.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/contentexchange/presentation/ContentImportService.java @@ -8,8 +8,8 @@ * the core reads, POST /v1/import/item so the core ingests it (delegating to the app's own * import), then remove the staging copy. The mirror of ContentExportService (foreground * specialUse + per-item queue + retry + notification + grant release), with the bytes - * flowing drive -> phone staging -> box. The item is the unit of resume (per-item retry); - * pause/resume and within-item byte-resume are later work. + * flowing drive -> phone staging -> box. The item is the unit of retry; a cut network + * item with an exact-build id resumes from its kept bytes (K2GO-577). * ============================================================================ */ package org.appdevforall.k2go.contentexchange.presentation; @@ -37,8 +37,10 @@ import org.appdevforall.k2go.contentexchange.data.ConexCoreImportSource; import org.appdevforall.k2go.contentexchange.data.NetworkImportCarrier; import org.appdevforall.k2go.contentexchange.data.SafImportCarrier; +import org.appdevforall.k2go.contentexchange.data.ShareManifestReader; import org.appdevforall.k2go.contentexchange.domain.ExportJob; import org.appdevforall.k2go.contentexchange.domain.ImportCarrier; +import org.appdevforall.k2go.contentexchange.domain.PartialPayload; import org.appdevforall.k2go.contentexchange.domain.ImportNoSpaceException; import org.appdevforall.k2go.contentexchange.domain.ImportSource; import org.appdevforall.k2go.contentexchange.domain.ImportStatus; @@ -49,7 +51,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; - +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -208,6 +212,8 @@ public int onStartCommand(Intent intent, int flags, int startId) { // No worker owns teardown: a fresh service instance may be handling this cancel, so // satisfy the FGS start-foreground obligation before stopping. startForeground(NOTIFICATION_ID, buildNotification(null)); + // An explicit cancel drops kept partials; Finish keeps them for a new share (K2GO-577). + new BoxImportStaging(getApplicationContext()).sweepPartials(Collections.emptySet()); SESSION.clear(); stopForeground(true); stopSelf(); @@ -273,7 +279,8 @@ public int onStartCommand(Intent intent, int flags, int startId) { * ingest it, then clean up the staged copy. * *

v1 limitation (accepted): the run state is in-memory, so a process death mid-import loses it; - * v1 does within-run per-item retry only. Durable op-state and byte-resume are v1.x. + * a cut network item keeps its partial on the staging disk, so a later run of the same build + * resumes it (K2GO-577). */ private void drain() { ImportSource source = new ConexCoreImportSource(new ConexCoreDataSource()); @@ -296,6 +303,8 @@ private void drain() { SESSION.setSource(carrier.sourceLabel()); ExportJob job = SESSION.job(); + // A kept partial lives only while its item is in the current run (K2GO-577). + staging.sweepPartials(partialNames(job)); long lastSampleAt = SystemClock.elapsedRealtime(); long lastSampleBytes = job.copiedTotal(); @@ -308,10 +317,12 @@ private void drain() { SESSION.publish(); notifyNow(); - String stored = BoxImportStaging.safeName(SESSION.payloadName(i)); + Staged staged = new Staged(BoxImportStaging.safeName(SESSION.payloadName(i))); + boolean copied = false; try { - copyToStaging(carrier, staging, SESSION.payloadName(i), stored, i, job); - ingest(source, ref, i, stored); + copyToStaging(carrier, staging, SESSION.payloadName(i), partialName(i), staged, i, job); + copied = true; + ingest(source, ref, i, staged.name); job.setDone(i); } catch (Exception e) { // The staging copy or the ingest failed: mark the item and keep its slot for a retry. @@ -320,7 +331,11 @@ private void drain() { Log.w(TAG, "Import failed for " + ref.app() + "/" + ref.localRef() + ": " + e.getMessage(), e); job.setFailed(i); SESSION.publish(); - staging.remove(stored); + // A cut transfer keeps its partial so Retry resumes it. A failed ingest drops it: the + // box rejected the bytes (e.g. a checksum mismatch), so they must not be resumed. + if (!PartialPayload.keepOnFailure(staged.resumable, copied, SESSION.isCancelled())) { + staging.remove(staged.name); + } if (isNoSpace(e)) { // No room on the shared phone/box disk: stop, leave the rest pending for a Retry. finishWithError("Not enough space on the device."); @@ -328,7 +343,7 @@ private void drain() { } continue; } - staging.remove(stored); + staging.remove(staged.name); long now = SystemClock.elapsedRealtime(); if (now - lastSampleAt >= RATE_WINDOW_MS) { @@ -340,6 +355,7 @@ private void drain() { } if (SESSION.isCancelled()) { + staging.sweepPartials(Collections.emptySet()); // the user gave up: drop every partial SESSION.clear(); stopForeground(true); stopSelf(); @@ -369,20 +385,41 @@ private void drain() { * honoring cancellation. Both streams close inside try-with-resources so a close-time failure * (ENOSPC on the staging disk commonly surfaces at flush/close) propagates and the item is marked * failed, never a silent partial written as done. + * + *

K2GO-577: an item with a {@code partial} name resumes from its kept bytes. The bytes are + * staged under that name, and kept on a cut, only when the sender says the body is the exact file + * ({@link ImportCarrier.Payload#exact()}); a tar (Kolibri) is staged as before and restarts at 0. */ private void copyToStaging(ImportCarrier carrier, BoxImportStaging staging, String payloadName, - String storedName, int i, ExportJob job) + String partial, Staged staged, int i, ExportJob job) throws IOException { + long size = job.planned(i); + long kept = partial != null ? PartialPayload.resumeFrom(staging.length(partial), size) : 0L; + if (kept == size && size > 0) { + // A whole partial from an earlier run (it died before the ingest): the box verifies it. + staged.name = partial; + staged.resumable = true; + job.setCopied(i, kept); + return; + } + ImportCarrier.Payload payload = carrier.openPayloadFrom(payloadName, kept); + staged.resumable = partial != null && payload.exact(); + if (staged.resumable) { + staged.name = partial; + } else if (partial != null) { + staging.remove(partial); // not an exact file: kept bytes cannot continue it + } byte[] buffer = new byte[COPY_BUFFER]; - long total = 0L; + long total = payload.start(); + job.setCopied(i, total); long lastPublish = 0L; // Trailing-window rate: anchor at this item's copy start and re-anchor once the window is // wide enough. A cumulative average (all bytes / whole run elapsed) reads far too low on a // single large item that warms up slowly, so sample the recent throughput instead. long windowAt = SystemClock.elapsedRealtime(); long windowBytes = job.copiedTotal(); - try (InputStream in = carrier.openPayload(payloadName); - OutputStream out = staging.open(storedName)) { + try (InputStream in = payload.stream(); + OutputStream out = staging.open(staged.name, total > 0)) { int read; while ((read = in.read(buffer)) != -1) { if (SESSION.isCancelled()) { @@ -406,6 +443,41 @@ private void copyToStaging(ImportCarrier carrier, BoxImportStaging staging, Stri } out.flush(); } + if (staged.resumable && !PartialPayload.isComplete(total, size)) { + // A clean end short of the item (or past it) is a cut, not a copy: keep it for Retry, + // where a longer file restarts at 0. + throw new IOException("transfer ended at " + total + " of " + size + " bytes"); + } + } + + /** Where one item's bytes are staged, and whether they may be kept for a resume (K2GO-577). */ + private static final class Staged { + String name; + boolean resumable; + + Staged(String name) { + this.name = name; + } + } + + /** The kept-partial name of item {@code i} for a network run, or null (K2GO-577). */ + private static String partialName(int i) { + if (!SESSION.isNetwork()) { + return null; + } + ItemRef ref = SESSION.refAt(i); + return PartialPayload.name(ref.app(), ShareManifestReader.buildId(SESSION.entryJson(i))); + } + + private static Set partialNames(ExportJob job) { + Set names = new HashSet<>(); + for (int i = 0; i < job.size(); i++) { + String n = partialName(i); + if (n != null) { + names.add(n); + } + } + return names; } /** diff --git a/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/data/ShareManifestReaderTest.java b/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/data/ShareManifestReaderTest.java index db3010157..6bd2569c8 100644 --- a/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/data/ShareManifestReaderTest.java +++ b/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/data/ShareManifestReaderTest.java @@ -76,4 +76,25 @@ public void emptyOrNoItemsYieldsNoEntries() throws Exception { byte[] noItems = new JSONObject().put("sessionId", "abc").toString().getBytes(StandardCharsets.UTF_8); assertTrue(ShareManifestReader.parse(noItems).isEmpty()); } + + @Test + public void buildIdIsTheAuthoritativeExactBuild() throws Exception { + JSONObject dedup = new JSONObject().put("build", "0c7e2d4a-1b2c-4d5e-8f90-123456789abc") + .put("idKind", "authoritative"); + String entry = new JSONObject().put("identity", + new JSONObject().put("localRef", "a.zim").put("dedup", dedup)).toString(); + assertEquals("0c7e2d4a-1b2c-4d5e-8f90-123456789abc", ShareManifestReader.buildId(entry)); + } + + @Test + public void buildIdIsNullWhenSoftMissingOrBad() throws Exception { + JSONObject soft = new JSONObject().put("build", "x").put("idKind", "soft"); + JSONObject none = new JSONObject().put("build", JSONObject.NULL).put("idKind", "authoritative"); + for (JSONObject dedup : new JSONObject[] {soft, none}) { + String entry = new JSONObject().put("identity", new JSONObject().put("dedup", dedup)).toString(); + assertEquals(null, ShareManifestReader.buildId(entry)); + } + assertEquals(null, ShareManifestReader.buildId(null)); + assertEquals(null, ShareManifestReader.buildId("not json")); + } } diff --git a/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/domain/PartialPayloadTest.java b/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/domain/PartialPayloadTest.java new file mode 100644 index 000000000..2b71cd71a --- /dev/null +++ b/controller/app/src/test/java/org/appdevforall/k2go/contentexchange/domain/PartialPayloadTest.java @@ -0,0 +1,64 @@ +package org.appdevforall.k2go.contentexchange.domain; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** K2GO-577: partial naming, resume start and the sender's range reply. */ +public class PartialPayloadTest { + + @Test + public void nameUsesAppAndBuildId() { + String n = PartialPayload.name("kiwix", "0c7e2d4a-1b2c-4d5e-8f90-123456789abc"); + assertEquals("partial-kiwix-0c7e2d4a-1b2c-4d5e-8f90-123456789abc", n); + assertTrue(PartialPayload.isPartial(n)); + } + + @Test + public void noSafeBuildIdMeansNoPartial() { + assertNull(PartialPayload.name("kiwix", null)); + assertNull(PartialPayload.name("kiwix", "")); + assertNull(PartialPayload.name("kiwix", "../evil")); + assertNull(PartialPayload.name("kiwix", "a/b")); + assertNull(PartialPayload.name("Kiwix!", "abc")); + assertNull(PartialPayload.name(null, "abc")); + assertFalse(PartialPayload.isPartial("0")); + } + + @Test + public void resumeFromKeptPrefixOnly() { + assertEquals(0L, PartialPayload.resumeFrom(0L, 100L)); + assertEquals(40L, PartialPayload.resumeFrom(40L, 100L)); + assertEquals(100L, PartialPayload.resumeFrom(100L, 100L)); // nothing left to fetch + assertEquals(0L, PartialPayload.resumeFrom(101L, 100L)); // not this item: start over + } + + @Test + public void onlyACutExactTransferIsKept() { + assertTrue(PartialPayload.keepOnFailure(true, false, false)); + assertFalse(PartialPayload.keepOnFailure(false, false, false)); // a tar restarts at 0 + assertFalse(PartialPayload.keepOnFailure(true, true, false)); // the box rejected it + assertFalse(PartialPayload.keepOnFailure(true, false, true)); // the user cancelled + assertTrue(PartialPayload.isComplete(100L, 100L)); + assertFalse(PartialPayload.isComplete(99L, 100L)); + assertFalse(PartialPayload.isComplete(101L, 100L)); + } + + @Test + public void bodyStartAcceptsWholeItemOrMatchingRange() { + assertEquals(0L, PartialPayload.bodyStart(200, 16777216L, null)); + assertEquals(16777216L, + PartialPayload.bodyStart(206, 16777216L, "bytes 16777216-333429921/333429922")); + } + + @Test + public void bodyStartRejectsOtherReplies() { + assertEquals(-1L, PartialPayload.bodyStart(206, 100L, "bytes 0-99/200")); + assertEquals(-1L, PartialPayload.bodyStart(206, 100L, null)); + assertEquals(-1L, PartialPayload.bodyStart(206, 100L, "bytes */200")); + assertEquals(-1L, PartialPayload.bodyStart(416, 100L, "bytes */200")); + } +} diff --git a/static/conex-core/CHANGELOG.md b/static/conex-core/CHANGELOG.md index dd709381d..b94445ba1 100644 --- a/static/conex-core/CHANGELOG.md +++ b/static/conex-core/CHANGELOG.md @@ -5,6 +5,7 @@ One line per version, newest first. Every REST-facing change bumps `conex_core.V role), so this file is the human record of what each bump enables. Keep entries short: `version - change (TICKET)`. +- **0.3.6** - The share fetch resumes a cut direct item: `Range: bytes=N-` on `GET /v1/share/{id}/fetch/{ref}` answers 206 with `Content-Range` for a ZIM (a staged Kolibri tar still sends 200 from byte 0). The Kiwix import checks the ZIM's own MD5 while it copies and replaces the target only on a match; a mismatch returns code `corrupt` and keeps the existing ZIM. (K2GO-577) - **0.3.5** - Kiwix counts as installed only when kiwix-serve is present, not just the zims dir, so a library that cannot open ZIMs answers NO_APP and does not import them. (K2GO-574) - **0.3.4** - import becomes a long operation like export: POST /v1/import/item now STARTS the ingest and returns an opId, the client polls GET /v1/import/status/{op}, and DELETE /v1/import/{op} cancels, so a 35-141 s Kolibri import no longer times out at the dash-node/nginx relay. A shared liveness watchdog supervises the Kolibri import and export CLIs by byte and CPU progress (no fixed timeout; the window is relative to the task) and names a signal-9 exit as a likely out-of-memory kill. (K2GO-376) - **0.3.3** - observability: log every error response and uncaught handler exception to stdout (the supervisor captures it to /var/log/conex-core.log), and line-buffer stdout/stderr in __main__ so output reaches the log while the server runs forever, so a failed import or share leaves a box-side trace; an uncaught error now returns a 500 JSON instead of dropping the connection. (K2GO-376) diff --git a/static/conex-core/README.md b/static/conex-core/README.md index 4c3d3dc0d..ca6c5d7c7 100644 --- a/static/conex-core/README.md +++ b/static/conex-core/README.md @@ -96,7 +96,8 @@ only auth, so it must not reach access logs). - `GET /v1/share/{sessionId}/fetch/{itemRef}` -> the chunked octet stream for one item. `409 {state: staging}` while a staging item (Kolibri) stages (the receiver retries), `409 {state: busy}` when the item already has an open stream (single-flight), `200` streams - once ready. + once ready. `Range: bytes=N-` resumes a cut direct item (ZIM): `206` with `Content-Range`. + Any other range, or a staged item, gets `200` from byte 0. - `POST /v1/share/{sessionId}/complete` -> `{ok: true}`. The receiver's clean-success close. - `DELETE /v1/share/{sessionId}` -> `{ok: true}`. Cancel (abort) by host or receiver. Both closers revoke the token and every child temp; `404` on an unknown or already-closed session. diff --git a/static/conex-core/conex_core/__init__.py b/static/conex-core/conex_core/__init__.py index 457a21fa9..9aefebe72 100644 --- a/static/conex-core/conex_core/__init__.py +++ b/static/conex-core/conex_core/__init__.py @@ -7,5 +7,5 @@ # The core build. Bumped per core change so a box can report which core it runs (GET # /v1/version), mirroring dash-node's package.json version and /system/version. The # conex_core Ansible role writes this into the [conex_core] section of iiab.ini. -VERSION = "0.3.5" +VERSION = "0.3.6" __version__ = VERSION diff --git a/static/conex-core/conex_core/adapters/calibre.py b/static/conex-core/conex_core/adapters/calibre.py index 47598a1af..15a0ef97f 100644 --- a/static/conex-core/conex_core/adapters/calibre.py +++ b/static/conex-core/conex_core/adapters/calibre.py @@ -245,7 +245,8 @@ def prepare_export(self, local_ref: str) -> Optional[ExportPlan]: content_type="application/x-tar", ) - def open_fetch(self, local_ref: str, staged_dir: Optional[str] = None) -> Optional[ExportStream]: + def open_fetch(self, local_ref: str, staged_dir: Optional[str] = None, + offset: int = 0) -> Optional[ExportStream]: # Direct app: staged_dir is unused. Re-resolve at fetch time. book_dir = self._export_dir(local_ref) if book_dir is None: diff --git a/static/conex-core/conex_core/adapters/kiwix.py b/static/conex-core/conex_core/adapters/kiwix.py index d1f9fbd99..b32fc7586 100644 --- a/static/conex-core/conex_core/adapters/kiwix.py +++ b/static/conex-core/conex_core/adapters/kiwix.py @@ -14,9 +14,9 @@ from __future__ import annotations import errno +import hashlib import os import re -import shutil import subprocess import uuid from typing import List, Optional, Tuple @@ -40,6 +40,13 @@ ZIM_MAGIC = 0x044D495A _ZIM_HEADER_MIN = 24 +# The header's checksumPos (uint64 LE at offset 72) points at the file's MD5, the last 16 bytes. +# The MD5 covers every byte before it (openzim spec), so a whole-file check needs no extra data. +_ZIM_CHECKSUM_POS_AT = 72 +_ZIM_MD5_LEN = 16 +_COPY_CHUNK = 1024 * 1024 +_IMPORT_TEMP_SUFFIX = ".k2go-import" + # A trailing YYYY-MM or YYYY-MM-DD is the version (date) axis in the filename. _DATE_RE = re.compile(r"^\d{4}-\d{2}(-\d{2})?$") @@ -63,6 +70,55 @@ def read_zim_uuid_from_file(path: str) -> Optional[str]: return None +def zim_checksum_pos(header: bytes, size: int) -> Optional[int]: + """Where the ZIM's MD5 starts, or None when the header or the size do not match the openzim + layout (the MD5 is always the last 16 bytes). Pure.""" + end = _ZIM_CHECKSUM_POS_AT + 8 + if len(header) < end or int.from_bytes(header[0:4], "little") != ZIM_MAGIC: + return None + pos = int.from_bytes(header[_ZIM_CHECKSUM_POS_AT:end], "little") + return pos if pos + _ZIM_MD5_LEN == size and pos >= end else None + + +def copy_verified_zim(src: str, dest: str) -> bool: + """Copy src to dest and check the ZIM's own MD5 on the way. False (dest removed) when the + layout is wrong or the MD5 does not match: a resumed transfer that spliced bad bytes never + reaches the library. An OSError (ENOSPC) propagates, with dest removed.""" + size = os.path.getsize(src) + md5 = hashlib.md5() + try: + with open(src, "rb") as fin, open(dest, "wb") as fout: + pos = zim_checksum_pos(fin.read(_ZIM_CHECKSUM_POS_AT + 8), size) + if pos is None: + ok = False + else: + fin.seek(0) + done = 0 + while done < pos: + block = fin.read(min(_COPY_CHUNK, pos - done)) + if not block: + break + md5.update(block) + fout.write(block) + done += len(block) + stored = fin.read(_ZIM_MD5_LEN) + fout.write(stored) + ok = done == pos and md5.digest() == stored + except OSError: + _remove_quietly(dest) + raise + if not ok: + _remove_quietly(dest) + return ok + + +def _remove_quietly(path: str) -> None: + try: + os.remove(path) + except OSError: + pass + + def is_safe_local_ref(local_ref: str) -> bool: """True when local_ref is a plain filename contained in one directory: no path separators, no parent/current reference, no null byte. So joining it onto the @@ -177,16 +233,22 @@ def prepare_export(self, local_ref: str) -> Optional[ExportPlan]: content_type="application/octet-stream", ) - def open_fetch(self, local_ref: str, staged_dir: Optional[str] = None) -> Optional[ExportStream]: + def open_fetch(self, local_ref: str, staged_dir: Optional[str] = None, + offset: int = 0) -> Optional[ExportStream]: # Direct app: staged_dir is unused. Re-validate at fetch time (the file may have - # vanished between prepare and fetch). + # vanished between prepare and fetch). The ZIM is the file itself, so a resume offset + # inside it is honored (K2GO-577); any other offset streams the whole file. path = self._export_path(local_ref) if path is None: return None + size = os.path.getsize(path) + start = offset if 0 < offset < size else 0 return ExportStream( filename=local_ref, content_type="application/octet-stream", - chunks=file_stream(path), + chunks=file_stream(path, start=start), + size=size, + offset=start, ) def _export_path(self, local_ref: str) -> Optional[str]: @@ -218,7 +280,8 @@ def import_item(self, entry: dict, payload_path: str, should_cancel=None, # import is a file move plus a re-index, so it ignores both and runs to completion. # The payload is the .zim itself: place it in the zims dir under its own name, then # re-run the indexer so the library entry appears. Overwriting the same name is - # idempotent (same content, same UUID). + # idempotent (same content, same UUID). The copy goes to a temp name (no .zim, so list() + # skips it) and replaces the target only after the ZIM's MD5 matches (K2GO-577). local_ref = (entry.get("identity") or {}).get("localRef", "") if not is_safe_local_ref(local_ref) or not local_ref.lower().endswith(".zim"): return {"ok": False, "reason": "invalid localRef"} @@ -226,9 +289,14 @@ def import_item(self, entry: dict, payload_path: str, should_cancel=None, return {"ok": False, "reason": "payload not found"} os.makedirs(self._dir, exist_ok=True) dest = os.path.join(self._dir, local_ref) + temp = dest + _IMPORT_TEMP_SUFFIX + self._sweep_import_temps() # a temp left by a process that died mid-import try: - shutil.copyfile(payload_path, dest) + if not copy_verified_zim(payload_path, temp): + return {"ok": False, "reason": "ZIM checksum mismatch", "code": "corrupt"} + os.replace(temp, dest) except OSError as exc: + _remove_quietly(temp) # ENOSPC gets a structural code so the front stops the run with a space message, not a # per-item retry (a ZIM is one large file, the likeliest out-of-space import). failure = {"ok": False, "reason": f"copy failed: {exc}"} @@ -238,6 +306,16 @@ def import_item(self, entry: dict, payload_path: str, should_cancel=None, self._reindex() return {"ok": True, "app": "kiwix", "localRef": local_ref, "bytes": os.path.getsize(dest)} + def _sweep_import_temps(self) -> None: + """Remove import temps in the zims dir. The front serializes imports, so none is live here.""" + try: + names = os.listdir(self._dir) + except OSError: + return + for name in names: + if name.endswith(_IMPORT_TEMP_SUFFIX): + _remove_quietly(os.path.join(self._dir, name)) + def delete(self, local_ref: str) -> dict: # A ZIM is self-contained, so the file IS the unit: remove it, then re-run the indexer # to drop its library.xml entry. freedBytes is the file size exactly (no shared bytes). diff --git a/static/conex-core/conex_core/adapters/kolibri.py b/static/conex-core/conex_core/adapters/kolibri.py index 64a6f1571..dbd705ba9 100644 --- a/static/conex-core/conex_core/adapters/kolibri.py +++ b/static/conex-core/conex_core/adapters/kolibri.py @@ -199,7 +199,8 @@ def stage_export(self, local_ref: str, dest: str, should_cancel=None) -> bool: return False return self._stage_export(local_ref, dest, should_cancel) - def open_fetch(self, local_ref: str, staged_dir: Optional[str] = None) -> Optional[ExportStream]: + def open_fetch(self, local_ref: str, staged_dir: Optional[str] = None, + offset: int = 0) -> Optional[ExportStream]: # Tar-stream the staged tree. The op layer sets on_close (drop the op + remove temp). if not staged_dir or not os.path.isdir(staged_dir): return None diff --git a/static/conex-core/conex_core/export.py b/static/conex-core/conex_core/export.py index 1bccd421e..04a812dbd 100644 --- a/static/conex-core/conex_core/export.py +++ b/static/conex-core/conex_core/export.py @@ -12,6 +12,7 @@ import os import queue +import re import shutil import tarfile import threading @@ -40,9 +41,12 @@ def free_bytes(path: str) -> Optional[int]: return None -def file_stream(path: str, chunk: int = CHUNK) -> Iterator[bytes]: - """Yield a file's bytes in chunks. Opens lazily (on first iteration), read-only.""" +def file_stream(path: str, chunk: int = CHUNK, start: int = 0) -> Iterator[bytes]: + """Yield a file's bytes in chunks, from byte ``start``. Opens lazily (on first iteration), + read-only.""" with open(path, "rb") as f: + if start: + f.seek(start) while True: block = f.read(chunk) if not block: @@ -50,6 +54,17 @@ def file_stream(path: str, chunk: int = CHUNK) -> Iterator[bytes]: yield block +_RANGE_FROM = re.compile(r"^bytes=(\d{1,19})-$") + + +def parse_range_start(header: Optional[str]) -> int: + """The start byte of a single open range ``bytes=N-`` (resume a cut transfer), else 0. Any + other form (no header, a closed or multi range) reads as 0, so the server sends the whole + body with 200, which RFC 9110 allows for a range it does not support.""" + match = _RANGE_FROM.match((header or "").strip()) + return int(match.group(1)) if match else 0 + + def resolve_within(root: str, rel: str) -> Optional[str]: """Join a relative path onto root and return its real path only when it stays inside root. Guards a multi-segment relative path (e.g. a Calibre book path) against diff --git a/static/conex-core/conex_core/export_ops.py b/static/conex-core/conex_core/export_ops.py index 9e0a0160d..d26f3a0aa 100644 --- a/static/conex-core/conex_core/export_ops.py +++ b/static/conex-core/conex_core/export_ops.py @@ -104,9 +104,10 @@ def status(self, op_id: str) -> Response: payload["error"] = error return 200, payload - def fetch(self, op_id: str): - """Open the byte stream for a ready op. Returns (200, ExportStream); the stream's on_close - drops the op and removes the temp. 409 when not ready, 404 when unknown.""" + def fetch(self, op_id: str, offset: int = 0): + """Open the byte stream for a ready op, from ``offset`` when the adapter can resume there. + Returns (200, ExportStream); the stream's on_close drops the op and removes the temp. 409 + when not ready, 404 when unknown.""" self._sweep_expired() with self._lock: op = self._ops.get(op_id) @@ -120,7 +121,7 @@ def fetch(self, op_id: str): # actually begins streaming, so a failure here leaves the op merely fresh and reclaimable by # the normal sweep, never stuck in_use. try: - stream = self._registry.get(app).open_fetch(local_ref, temp) + stream = self._registry.get(app).open_fetch(local_ref, temp, offset=offset) except Exception: self._finish(op_id) return 500, {"error": "failed to open export"} diff --git a/static/conex-core/conex_core/ports.py b/static/conex-core/conex_core/ports.py index d2d8fc323..9496df728 100644 --- a/static/conex-core/conex_core/ports.py +++ b/static/conex-core/conex_core/ports.py @@ -29,7 +29,11 @@ class ExportStream: ``completed`` is set True by the server after the FULL body has been written (the clean terminator), and stays False on a mid-stream client drop. The export op ignores it (it drops the temp either way), but the share gate reads it so a share item counts as delivered only on - a clean close, never on a drop (K2GO-376).""" + a clean close, never on a drop (K2GO-376). + + ``size`` is the full byte length when the body is one exact file (a direct ZIM), else None (a + tar). ``offset`` is where the body starts in it: above 0 only when the adapter honored a resume + request, and then the server answers 206 with Content-Range (K2GO-577).""" filename: str content_type: str @@ -37,6 +41,8 @@ class ExportStream: on_close: Optional[Callable[[], None]] = None on_open: Optional[Callable[[], None]] = None completed: bool = False + size: Optional[int] = None + offset: int = 0 @dataclass @@ -95,10 +101,13 @@ def stage_export(self, local_ref: str, dest: str, to stage.""" return False - def open_fetch(self, local_ref: str, staged_dir: Optional[str] = None) -> Optional[ExportStream]: + def open_fetch(self, local_ref: str, staged_dir: Optional[str] = None, + offset: int = 0) -> Optional[ExportStream]: """Open the byte stream for a prepared export: the staged tar (``staged_dir`` set) or - the direct file / tar (``staged_dir`` None). The op layer sets ``on_close``. Returns - None when the item is no longer exportable. Default: not supported.""" + the direct file / tar (``staged_dir`` None). ``offset`` asks to resume from that byte; an + adapter honors it only for a byte-exact file and sets the stream's ``offset``, else it + streams from 0. The op layer sets ``on_close``. Returns None when the item is no longer + exportable. Default: not supported.""" return None def check(self, dedup: dict) -> str: diff --git a/static/conex-core/conex_core/router.py b/static/conex-core/conex_core/router.py index e3cbfc071..e9b6050b7 100644 --- a/static/conex-core/conex_core/router.py +++ b/static/conex-core/conex_core/router.py @@ -143,14 +143,16 @@ def _export_prepare(ops: Optional[ExportOps], body: Any) -> Response: def dispatch(registry: Registry, method: str, path: str, query: dict, body: Any = None, ops: Optional[ExportOps] = None, share: Optional[ShareOps] = None, - token: Optional[str] = None, import_ops: Optional[ImportOps] = None) -> Response: + token: Optional[str] = None, import_ops: Optional[ImportOps] = None, + range_start: int = 0) -> Response: """Route one request. ``query`` is a dict of first values; ``body`` is the parsed JSON for a POST; ``ops`` is the export long-operation registry, ``import_ops`` the import long-operation registry, and ``share`` the share-gate registry (the server supplies all; the routes that need them guard against None). ``token`` is the share session token: the server reads it from the ``X-Share-Token`` request header (never the URL, so it stays out of access logs) and passes it here, so the router stays header-agnostic and unit-testable without a - socket.""" + socket. ``range_start`` is the start byte of a ``Range: bytes=N-`` request header (0 when + absent); only the share fetch uses it, to resume a cut item (K2GO-577).""" trimmed = path.rstrip("/") or "/" segments = [unquote(s) for s in trimmed.split("/") if s != ""] @@ -251,7 +253,7 @@ def dispatch(registry: Registry, method: str, path: str, query: dict, body: Any if len(segments) == 5 and segments[0] == "v1" and segments[1] == "share" and segments[3] == "fetch": if share is None: return 500, {"error": "share unavailable"} - return share.fetch(segments[2], segments[4], token) + return share.fetch(segments[2], segments[4], token, range_start) # GET /v1/storage : box content-volume disk summary + per-app content totals. if segments == ["v1", "storage"]: diff --git a/static/conex-core/conex_core/server.py b/static/conex-core/conex_core/server.py index 4dd74d876..09009a5a6 100644 --- a/static/conex-core/conex_core/server.py +++ b/static/conex-core/conex_core/server.py @@ -13,6 +13,7 @@ from . import VERSION from .config import Config +from .export import parse_range_start from .export_ops import ExportOps from .import_ops import ImportOps from .ports import ExportStream @@ -44,12 +45,13 @@ def _share_token(self): # land in access logs (it is the only auth for the gate). return self.headers.get("X-Share-Token") - def _dispatch(self, method: str, path: str, query: dict, body=None): + def _dispatch(self, method: str, path: str, query: dict, body=None, range_start: int = 0): # Single dispatch boundary: an uncaught error in a handler becomes a logged 500 # with a JSON body, never a dropped connection the front reads as a network error. try: return dispatch(registry, method, path, query, body, ops=ops, share=share, - token=self._share_token(), import_ops=import_ops) + token=self._share_token(), import_ops=import_ops, + range_start=range_start) except Exception: log.exception("unhandled error: %s %s", method, path) return 500, {"error": "internal error"} @@ -57,7 +59,8 @@ def _dispatch(self, method: str, path: str, query: dict, body=None): def do_GET(self) -> None: # noqa: N802 (http.server naming) parts = urlsplit(self.path) query = {k: v[0] for k, v in parse_qs(parts.query).items()} - status, payload = self._dispatch("GET", parts.path, query) + status, payload = self._dispatch("GET", parts.path, query, + range_start=parse_range_start(self.headers.get("Range"))) if isinstance(payload, ExportStream): self._stream(status, payload) else: @@ -125,7 +128,15 @@ def _stream(self, status: int, stream: ExportStream) -> None: # must still release the op pin and the share single-flight slot (K2GO-376), # never strand them. on_open follows the headers, so a pre-header failure runs # on_close without a matching on_open: the op cleanup (_finish) is safe either way. - self.send_response(status) + if stream.offset > 0 and stream.size is not None: + # The adapter resumed inside a byte-exact file: a partial body (K2GO-577). + self.send_response(206) + self.send_header("Content-Range", + f"bytes {stream.offset}-{stream.size - 1}/{stream.size}") + else: + self.send_response(status) + if stream.size is not None: + self.send_header("Accept-Ranges", "bytes") self.send_header("Content-Type", stream.content_type) self.send_header("Content-Disposition", f'attachment; filename="{stream.filename}"') self.send_header("Transfer-Encoding", "chunked") diff --git a/static/conex-core/conex_core/share_ops.py b/static/conex-core/conex_core/share_ops.py index 8207979b0..6d9aeead5 100644 --- a/static/conex-core/conex_core/share_ops.py +++ b/static/conex-core/conex_core/share_ops.py @@ -170,8 +170,9 @@ def manifest(self, session_id: str, token: Any) -> Response: return 200, {"sessionId": session_id, "items": found, "missing": missing, "totalBytes": total} - def fetch(self, session_id: str, item_ref: str, token: Any): - """Stream one shared item. Lazily creates the child export op the first time an item is + def fetch(self, session_id: str, item_ref: str, token: Any, offset: int = 0): + """Stream one shared item, from ``offset`` when the receiver resumes a cut item and the + item is a byte-exact file (a direct ZIM, K2GO-577); a staged tar always starts at 0. Lazily creates the child export op the first time an item is fetched, then drives it: 409 {state: staging} while a staging app (Kolibri) stages (the receiver retries this same URL), 409 {state: busy} when the item already has an open stream (single-flight), (200, ExportStream) once ready. The stream's on_close releases @@ -203,7 +204,7 @@ def fetch(self, session_id: str, item_ref: str, token: Any): op_id = self._ensure_op(session_id, item_ref, app, local_ref) if isinstance(op_id, tuple): return op_id # prepare failed: 404 absent, 507 no space - status, result = self._export.fetch(op_id) + status, result = self._export.fetch(op_id, offset) if status == 404 and attempt == 1: self._drop_op(session_id, item_ref, op_id) continue diff --git a/static/conex-core/tests/test_export.py b/static/conex-core/tests/test_export.py index de3335e28..47c886088 100644 --- a/static/conex-core/tests/test_export.py +++ b/static/conex-core/tests/test_export.py @@ -4,7 +4,7 @@ import tempfile import unittest -from conex_core.export import file_stream, resolve_within, tar_stream +from conex_core.export import file_stream, parse_range_start, resolve_within, tar_stream class FileStreamTest(unittest.TestCase): @@ -17,6 +17,16 @@ def test_streams_whole_file(self): self.assertEqual(b"".join(file_stream(p, chunk=1024 * 1024)), data) +class ParseRangeStartTest(unittest.TestCase): + def test_open_range_gives_its_start(self): + self.assertEqual(parse_range_start("bytes=16777216-"), 16777216) + self.assertEqual(parse_range_start(" bytes=0- "), 0) + + def test_other_forms_read_as_zero(self): + for header in (None, "", "bytes=1-5", "bytes=-500", "bytes=1-,5-", "items=3-", "bytes=x-"): + self.assertEqual(parse_range_start(header), 0, header) + + class ResolveWithinTest(unittest.TestCase): def test_inside_is_resolved(self): with tempfile.TemporaryDirectory() as d: diff --git a/static/conex-core/tests/test_export_ops.py b/static/conex-core/tests/test_export_ops.py index 4fa439d5c..f7ee2529f 100644 --- a/static/conex-core/tests/test_export_ops.py +++ b/static/conex-core/tests/test_export_ops.py @@ -30,7 +30,7 @@ def prepare_export(self, local_ref): return None return ExportPlan(False, self._total, local_ref, "application/octet-stream") - def open_fetch(self, local_ref, staged_dir=None): + def open_fetch(self, local_ref, staged_dir=None, offset=0): if not self._present: return None return ExportStream(local_ref, "application/octet-stream", [b"x" * self._total]) @@ -67,7 +67,7 @@ def stage_export(self, local_ref, dest, should_cancel=None): f.write(b"x" * self._total) return True - def open_fetch(self, local_ref, staged_dir=None): + def open_fetch(self, local_ref, staged_dir=None, offset=0): path = os.path.join(staged_dir or "", "part.bin") data = b"" if os.path.isfile(path): @@ -286,7 +286,7 @@ def test_fetch_failure_before_streaming_does_not_leave_op_pinned(self): # If open_fetch raises (or the stream never starts), the op must not be stuck in_use: # it is cleaned up so it cannot survive the sweep forever. class _RaisingFetch(_StagingFake): - def open_fetch(self, local_ref, staged_dir=None): + def open_fetch(self, local_ref, staged_dir=None, offset=0): raise RuntimeError("boom") with tempfile.TemporaryDirectory() as d: diff --git a/static/conex-core/tests/test_kiwix.py b/static/conex-core/tests/test_kiwix.py index f3fcf0c47..042aed512 100644 --- a/static/conex-core/tests/test_kiwix.py +++ b/static/conex-core/tests/test_kiwix.py @@ -5,6 +5,7 @@ from conex_core.adapters.kiwix import ( KiwixAdapter, + zim_checksum_pos, build_item, is_safe_local_ref, parse_zim_filename, @@ -18,6 +19,29 @@ def _zim_header(u: uuid.UUID) -> bytes: return (0x044D495A).to_bytes(4, "little") + b"\x06\x00\x00\x00" + u.bytes +def _valid_zim(body_len: int = 64) -> bytes: + """A ZIM with the openzim layout the import checks: checksumPos at offset 72 points at the + trailing MD5 of every byte before it.""" + import hashlib + head = bytearray(_zim_header(uuid.uuid4()) + b"\x00" * (80 - 24)) + pos = 80 + body_len + head[72:80] = pos.to_bytes(8, "little") + data = bytes(head) + os.urandom(body_len) + return data + hashlib.md5(data).digest() + + +class ZimChecksumPosTest(unittest.TestCase): + def test_reads_pos_when_md5_is_the_last_16_bytes(self): + data = _valid_zim(10) + self.assertEqual(zim_checksum_pos(data[:80], len(data)), len(data) - 16) + + def test_rejects_size_mismatch_bad_magic_and_short_header(self): + data = _valid_zim(10) + self.assertIsNone(zim_checksum_pos(data[:80], len(data) + 1)) + self.assertIsNone(zim_checksum_pos(b"\x00" * 80, len(data))) + self.assertIsNone(zim_checksum_pos(data[:40], len(data))) + + class ReadZimUuidTest(unittest.TestCase): def test_reads_uuid_from_valid_header(self): u = uuid.UUID("abcdabcd-1234-5678-9abc-def012345678") @@ -175,6 +199,27 @@ def test_open_fetch_streams_the_zim_as_is(self): self.assertEqual(stream.content_type, "application/octet-stream") self.assertEqual(b"".join(stream.chunks), payload) + def test_open_fetch_resumes_inside_the_file(self): + with tempfile.TemporaryDirectory() as d: + name = "devdocs_en_angular_2026-05.zim" + payload = _zim_header(uuid.uuid4()) + os.urandom(200) + with open(os.path.join(d, name), "wb") as f: + f.write(payload) + stream = KiwixAdapter(d).open_fetch(name, offset=100) + self.assertEqual((stream.offset, stream.size), (100, len(payload))) + self.assertEqual(b"".join(stream.chunks), payload[100:]) + + def test_open_fetch_ignores_an_offset_outside_the_file(self): + with tempfile.TemporaryDirectory() as d: + name = "devdocs_en_angular_2026-05.zim" + payload = _zim_header(uuid.uuid4()) + os.urandom(200) + with open(os.path.join(d, name), "wb") as f: + f.write(payload) + for offset in (len(payload), len(payload) + 5, -1): + stream = KiwixAdapter(d).open_fetch(name, offset=offset) + self.assertEqual(stream.offset, 0) + self.assertEqual(b"".join(stream.chunks), payload) + def test_export_rejects_traversal_and_missing(self): with tempfile.TemporaryDirectory() as d: adapter = KiwixAdapter(d) @@ -216,7 +261,7 @@ class KiwixImportTest(unittest.TestCase): def test_import_places_zim_and_reports_ok(self): with tempfile.TemporaryDirectory() as src, tempfile.TemporaryDirectory() as zdir: payload = os.path.join(src, "payload.zim") - data = _zim_header(uuid.uuid4()) + os.urandom(64) + data = _valid_zim() with open(payload, "wb") as f: f.write(data) adapter = KiwixAdapter(zdir, indexer="/nonexistent/indexer") # reindex is a no-op @@ -227,6 +272,38 @@ def test_import_places_zim_and_reports_ok(self): self.assertTrue(os.path.isfile(placed)) with open(placed, "rb") as f: self.assertEqual(f.read(), data) + self.assertEqual(os.listdir(zdir), ["gutenberg_en_all_2024-01.zim"]) + + def test_import_removes_a_temp_left_by_a_dead_process(self): + with tempfile.TemporaryDirectory() as src, tempfile.TemporaryDirectory() as zdir: + with open(os.path.join(zdir, "old_en_all_2020-01.zim.k2go-import"), "wb") as f: + f.write(b"half") + payload = os.path.join(src, "payload.zim") + with open(payload, "wb") as f: + f.write(_valid_zim()) + adapter = KiwixAdapter(zdir, indexer="/nonexistent") + self.assertTrue(adapter.import_item( + {"identity": {"localRef": "gutenberg_en_all_2024-01.zim"}}, payload)["ok"]) + self.assertEqual(os.listdir(zdir), ["gutenberg_en_all_2024-01.zim"]) + + def test_import_rejects_a_bad_checksum_and_keeps_the_existing_zim(self): + with tempfile.TemporaryDirectory() as src, tempfile.TemporaryDirectory() as zdir: + name = "gutenberg_en_all_2024-01.zim" + good = _valid_zim() + with open(os.path.join(zdir, name), "wb") as f: + f.write(good) + bad = bytearray(good) + bad[100] ^= 0xFF # one spliced byte in the body + payload = os.path.join(src, "payload.zim") + with open(payload, "wb") as f: + f.write(bytes(bad)) + adapter = KiwixAdapter(zdir, indexer="/nonexistent") + result = adapter.import_item({"identity": {"localRef": name}}, payload) + self.assertFalse(result["ok"]) + self.assertEqual(result["code"], "corrupt") + self.assertEqual(os.listdir(zdir), [name]) # no temp left behind + with open(os.path.join(zdir, name), "rb") as f: + self.assertEqual(f.read(), good) def test_import_rejects_traversal_localref(self): with tempfile.TemporaryDirectory() as src, tempfile.TemporaryDirectory() as zdir: @@ -245,19 +322,19 @@ def test_import_enospc_sets_no_space_code(self): with tempfile.TemporaryDirectory() as src, tempfile.TemporaryDirectory() as zdir: payload = os.path.join(src, "payload.zim") with open(payload, "wb") as f: - f.write(_zim_header(uuid.uuid4()) + os.urandom(16)) + f.write(_valid_zim()) adapter = KiwixAdapter(zdir, indexer="/nonexistent") - orig = kiwix_mod.shutil.copyfile + orig = kiwix_mod.copy_verified_zim def boom(*_a, **_k): raise OSError(28, "No space left on device") - kiwix_mod.shutil.copyfile = boom + kiwix_mod.copy_verified_zim = boom try: result = adapter.import_item( {"identity": {"localRef": "gutenberg_en_all_2024-01.zim"}}, payload) finally: - kiwix_mod.shutil.copyfile = orig + kiwix_mod.copy_verified_zim = orig self.assertFalse(result["ok"]) self.assertEqual(result["code"], "no_space") diff --git a/static/conex-core/tests/test_server.py b/static/conex-core/tests/test_server.py index e80c4a748..65c2554e2 100644 --- a/static/conex-core/tests/test_server.py +++ b/static/conex-core/tests/test_server.py @@ -12,6 +12,7 @@ from conex_core.export_ops import ExportOps from conex_core.registry import Registry from conex_core.server import make_handler +from conex_core.share_ops import ShareOps def _zim(path, body_len=500): @@ -187,5 +188,56 @@ def test_uncaught_exception_returns_500_and_logs(self): httpd.server_close() + +class ShareRangeHttpTest(unittest.TestCase): + """A cut ZIM resumes over the share gate: Range bytes=N- gets 206 with the tail (K2GO-577).""" + + def setUp(self): + self._dir = tempfile.TemporaryDirectory() + self.name = "wikipedia_en_all_2024-01.zim" + self.data = _zim(os.path.join(self._dir.name, self.name), body_len=300000) + reg = Registry([KiwixAdapter(self._dir.name)]) + ops = ExportOps(reg, staging_dir=self._dir.name, ttl_seconds=60) + share = ShareOps(reg, ops) + self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), make_handler(reg, ops, share)) + self.port = self.httpd.server_address[1] + threading.Thread(target=self.httpd.serve_forever, daemon=True).start() + status, created = share.create([{"app": "kiwix", "localRef": self.name}]) + self.sid, self.token = created["sessionId"], created["token"] + self.share = share + + def tearDown(self): + self.httpd.shutdown() + self.httpd.server_close() + self._dir.cleanup() + + def _fetch(self, range_header=None): + c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10) + headers = {"Connection": "close", "X-Share-Token": self.token} + if range_header: + headers["Range"] = range_header + c.request("GET", f"/v1/share/{self.sid}/fetch/0", headers=headers) + resp = c.getresponse() + out = (resp.status, resp.getheader("Content-Range"), resp.getheader("Accept-Ranges"), + resp.read()) + c.close() + return out + + def test_open_range_resumes_with_206_and_marks_delivered(self): + cut = 123457 + status, content_range, accept, body = self._fetch(f"bytes={cut}-") + self.assertEqual(status, 206) + self.assertEqual(content_range, f"bytes {cut}-{len(self.data) - 1}/{len(self.data)}") + self.assertEqual(accept, "bytes") + self.assertEqual(body, self.data[cut:]) + _, man = self.share.manifest(self.sid, self.token) + self.assertTrue(man["items"][0]["delivered"]) + + def test_no_or_unsupported_range_sends_the_whole_file(self): + for header in (None, "bytes=0-10", f"bytes={len(self.data)}-"): + status, content_range, _, body = self._fetch(header) + self.assertEqual((status, content_range), (200, None), header) + self.assertEqual(body, self.data) + if __name__ == "__main__": unittest.main() diff --git a/static/conex-core/tests/test_share_ops.py b/static/conex-core/tests/test_share_ops.py index d1cdc8a49..0929cc293 100644 --- a/static/conex-core/tests/test_share_ops.py +++ b/static/conex-core/tests/test_share_ops.py @@ -49,7 +49,7 @@ def prepare_export(self, local_ref): return None return ExportPlan(False, self._items[local_ref], local_ref, "application/octet-stream") - def open_fetch(self, local_ref, staged_dir=None): + def open_fetch(self, local_ref, staged_dir=None, offset=0): if local_ref not in self._items: return None return ExportStream(local_ref, "application/octet-stream", [b"x" * self._items[local_ref]]) @@ -86,7 +86,7 @@ def stage_export(self, local_ref, dest, should_cancel=None): f.write(b"x" * self._items[local_ref]) return True - def open_fetch(self, local_ref, staged_dir=None): + def open_fetch(self, local_ref, staged_dir=None, offset=0): path = os.path.join(staged_dir or "", "part.bin") data = b"" if os.path.isfile(path): diff --git a/static/dashboard/CHANGELOG.md b/static/dashboard/CHANGELOG.md index dfe1f268c..e02065676 100644 --- a/static/dashboard/CHANGELOG.md +++ b/static/dashboard/CHANGELOG.md @@ -4,6 +4,7 @@ One line per version, newest first. Every REST-facing change bumps the version i (the app surfaces it via `/system/dashboard/update-check` and the "Update available" pill), so this file is the human record of what each bump enables. Keep entries short: `version - change (TICKET)`. +- **1.6.3** - The `/v1` relay to conex-core forwards the `X-Share-Token` header (never logged). Before, it was dropped, so the sender's close (`DELETE /v1/share/`) got 401 and the share token stayed valid after Done. (K2GO-577) - **1.6.2** - Box logs no longer identify the user, on boxes installed before bake patches 0010-0012 (K2GO-559). At its first start, dash-node applies the same change once: nginx access log without client IP and User-Agent (`/etc/nginx/nginx.conf`), Forgejo router log off (`LOGGER_ROUTER_MODE` empty in `app.ini`), Calibre-Web log level DEBUG -> INFO (`app.db`). A value an admin changed is kept. Each change applies at the service restart (Forgejo: at the next app open, as a pending privacy change; nginx and Calibre-Web: at the next box start). After that restart, the older logs of each changed service are removed once (rotated copies deleted, live log emptied). Markers: `/library/dashboard/log-less.done`, `/library/dashboard/log-less-purge.done`. No REST change. (K2GO-559) - **1.6.1** - The box restarts conex-core (Content Exchange) on its own when its process is gone: service-heal probes `/v1/apps` and runs `pdsm restart conex-core` only on a refused connection (a busy, slow service is never restarted mid-operation), and skips boxes without the conex-core pdsm service. `POST /system/service/conex-core/restart` is allowed. Localhost-only. (K2GO-574) - **1.6.0** - Privacy switches for the box services that contact an outside server on their own (K2GO-559). `GET /system/privacy` reports each one (Kolibri statistics ping, Forgejo update check): installed, and whether it may contact its server. `POST /system/privacy/:service {outbound}` sets the service's own config key (Kolibri `[Deployment] DISABLE_PING` in `options.ini`, Forgejo `[cron.update_checker] ENABLED` in `app.ini`) and only writes the config; it never restarts a service. `pending` reports a change that waits for a restart (config newer than the running process; derived on each read, not stored). The app asks the user (Restart now / Later) and requests the restart through `POST /system/service/:svc/restart` (ADR-5343), which now runs one restart per service at a time; a pending change is restarted at the next app open. A missing key reads as the service default (on). On its first start a box sets every installed switch OFF once (new bakes ship OFF), then writes `/library/dashboard/privacy-defaults.done` so a switch the user turns on later stays on; it applies like any pending change. Localhost-only. (K2GO-559) diff --git a/static/dashboard/package.json b/static/dashboard/package.json index d965e168a..bebc0d240 100644 --- a/static/dashboard/package.json +++ b/static/dashboard/package.json @@ -1,6 +1,6 @@ { "name": "dashboard-console", - "version": "1.6.2", + "version": "1.6.3", "description": "", "main": "index.js", "scripts": { diff --git a/static/dashboard/sockets/conex-proxy.test.ts b/static/dashboard/sockets/conex-proxy.test.ts index f718aa94b..dec2f5056 100644 --- a/static/dashboard/sockets/conex-proxy.test.ts +++ b/static/dashboard/sockets/conex-proxy.test.ts @@ -125,6 +125,34 @@ test('conexProxy passes Content-Disposition through (the export filename)', asyn } }); +test('conexProxy forwards X-Share-Token (the sender close DELETE)', async () => { + const core = http.createServer((req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ token: req.headers['x-share-token'] ?? null, method: req.method })); + }); + const corePort = await listen(core); + process.env.CONEX_PORT = String(corePort); + const app = await startProxyApp(); + try { + const r = await new Promise<{ status: number; body: string }>((resolve, reject) => { + const req = http.request({ host: '127.0.0.1', port: app.port, path: '/v1/share/abc', + method: 'DELETE', headers: { 'x-share-token': 'tok-123' } }, (res) => { + let data = ''; + res.on('data', (c) => { data += c; }); + res.on('end', () => resolve({ status: res.statusCode || 0, body: data })); + }); + req.on('error', reject); + req.end(); + }); + assert.equal(r.status, 200); + assert.deepEqual(JSON.parse(r.body), { token: 'tok-123', method: 'DELETE' }); + const none = await get(app.port, '/v1/apps'); + assert.equal(JSON.parse(none.body).token, null); + } finally { + app.close(); core.close(); + } +}); + test('conexProxy returns 502 when the core is unreachable', async () => { // Point at a port with nothing listening. const dead = http.createServer(); diff --git a/static/dashboard/sockets/conex-proxy.ts b/static/dashboard/sockets/conex-proxy.ts index da22d3588..0e63c161a 100644 --- a/static/dashboard/sockets/conex-proxy.ts +++ b/static/dashboard/sockets/conex-proxy.ts @@ -33,6 +33,10 @@ export function conexProxy(req: Request, res: Response): void { headers['content-type'] = 'application/json'; headers['content-length'] = String(payload.length); } + // The share-session token is the gate's only auth (the sender's close DELETE carries it). + // Forward it, and never log it. + const shareToken = req.headers['x-share-token']; + if (typeof shareToken === 'string' && shareToken) headers['x-share-token'] = shareToken; const upstream = http.request( { host: CONEX_HOST, port: conexPort(), method: req.method, path: corePath, headers },