Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
7a8f7df
K2GO-577 fix(dash-node): forward X-Share-Token to conex-core
luisguzman-adfa Nov 14, 2026
8a143df
K2GO-577 feat(conex-core): share ZIMs over rsync, resumable pull into…
luisguzman-adfa Nov 16, 2026
a1a1271
K2GO-577 feat(share): receiver pulls rsync items through a loopback r…
luisguzman-adfa Nov 16, 2026
94995b1
K2GO-577 docs(share): rsync daemon listener in the exposure doc
luisguzman-adfa Nov 16, 2026
a618162
K2GO-577 fix(conex-core): serialize daemon module updates, sweep old …
luisguzman-adfa Nov 16, 2026
328f4eb
K2GO-577 fix(share): review fixes for the rsync pull
luisguzman-adfa Nov 16, 2026
b3420c5
K2GO-577 feat(conex-core): share Kolibri channels over rsync, no tar
luisguzman-adfa Nov 16, 2026
cb0eb60
K2GO-577 docs(share): content exchange ADR with the rsync transport a…
luisguzman-adfa Nov 16, 2026
98bc600
Merge branch 'feat/K2GO-577-share-rsync' into feat/K2GO-577-kolibri-r…
luisguzman-adfa Nov 16, 2026
dd417dd
K2GO-577 fix(conex-core): pull liveness is the bytes on disk
luisguzman-adfa Nov 16, 2026
73b7aa7
K2GO-577 fix(conex-core): Kolibri pull liveness on disk, pull_tree te…
luisguzman-adfa Nov 16, 2026
b1ba097
Merge branch 'feat/K2GO-577-share-rsync' into feat/K2GO-577-kolibri-r…
luisguzman-adfa Nov 16, 2026
b365641
K2GO-577 fix(conex-core): an rsync pull dies with the core
luisguzman-adfa Nov 16, 2026
c092b8f
K2GO-577 fix(conex-core): clear the Kolibri pull temp dir left by a k…
luisguzman-adfa Nov 16, 2026
6b54df3
K2GO-577 feat(share): drop a given-up pull's partial on Cancel or Finish
luisguzman-adfa Nov 16, 2026
d735988
Merge branch 'feat/K2GO-577-share-rsync' into feat/K2GO-577-kolibri-r…
luisguzman-adfa Nov 16, 2026
08a2521
K2GO-577 feat(conex-core): a given-up Kolibri pull drops its work dir
luisguzman-adfa Nov 16, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,11 @@ public JSONObject postImportStart(JSONObject body) throws Exception {
return postJson(V1 + "/import/item", body);
}

/** {@code POST /v1/import/discard} with {@code {app, entry}}: drop a given-up pull's kept bytes. */
public JSONObject postImportDiscard(JSONObject body) throws Exception {
return postJson(V1 + "/import/discard", body);
}

/**
* {@code GET /v1/import/status/{opId}} -> {@code {state, movedBytes, totalBytes, error?, code?,
* result?}}. A 404 means the op was swept; it surfaces as {@link ConexHttpException} with
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.appdevforall.k2go.contentexchange.domain.ImportStatus;
import org.appdevforall.k2go.contentexchange.domain.ImportVerdict;
import org.appdevforall.k2go.contentexchange.domain.ItemRef;
import org.appdevforall.k2go.contentexchange.domain.PullUnsupportedException;
import org.json.JSONArray;
import org.json.JSONObject;

Expand Down Expand Up @@ -67,6 +68,46 @@ public String startImport(String app, String entryJson, String payloadBoxPath) t
return opId;
}

@Override
public String startPull(String app, String entryJson, String host, int port, String module,
String token) throws Exception {
JSONObject source = new JSONObject()
.put("host", host)
.put("port", port)
.put("module", module)
.put("token", token);
JSONObject body = new JSONObject()
.put("app", app)
.put("entry", new JSONObject(entryJson))
.put("source", source);
JSONObject r;
try {
r = dataSource.postImportStart(body);
} catch (ConexCoreDataSource.ConexHttpException e) {
// 400: a core older than the pull wants a payloadPath. 501: no rsync on this box.
if (e.code() == 400 || e.code() == 501) {
throw new PullUnsupportedException(e.getMessage());
}
throw e;
}
String opId = r.optString("opId", "");
if (opId.isEmpty()) {
throw new Exception("pull start returned no opId");
}
return opId;
}

@Override
public void discardPull(String app, String entryJson) {
try {
dataSource.postImportDiscard(new JSONObject()
.put("app", app)
.put("entry", new JSONObject(entryJson)));
} catch (Exception ignored) {
// best effort: the box sweeps partials older than 48 h on its own
}
}

@Override
public ImportStatus importStatus(String opId) throws Exception {
JSONObject r;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
* One Share session as an {@link ImportCarrier}, read over the hotspot from the sender's gate. The
Expand Down Expand Up @@ -81,6 +84,8 @@ public interface Cancellation {
private final String token;
private final String manifestUrl;
private final Cancellation cancel;
/** Progress reports to the sender (K2GO-577). Its one thread ends on its own when idle. */
private final ThreadPoolExecutor reporter;

public NetworkImportCarrier(Context context, String host, int port, String token,
String manifestUrl, Cancellation cancel) {
Expand All @@ -90,6 +95,13 @@ public NetworkImportCarrier(Context context, String host, int port, String token
this.token = token;
this.manifestUrl = manifestUrl;
this.cancel = cancel != null ? cancel : () -> false;
this.reporter = new ThreadPoolExecutor(1, 1, 10L, TimeUnit.SECONDS,
new ArrayBlockingQueue<>(1), r -> {
Thread t = new Thread(r, "share-progress");
t.setDaemon(true);
return t;
}, new ThreadPoolExecutor.DiscardOldestPolicy());
this.reporter.allowCoreThreadTimeOut(true);
}

@Override
Expand Down Expand Up @@ -179,6 +191,43 @@ public void onAllDelivered() {
}
}

/**
* Best-effort report of one item's progress to the sender's gate (K2GO-577): it feeds the
* sender's view and keeps its session fresh during a long rsync pull, which the gate cannot see.
* {@code state} is "running", "done" or "error". A failure is logged and ignored.
*/
public void reportProgress(String itemRef, long movedBytes, long totalBytes, String state) {
if (!ShareUrls.isValidItemRef(itemRef)) {
return;
}
// Off the import thread, so an unreachable sender never stalls the poll loop. One report
// in flight and one queued; an older queued report is dropped for the newer one.
reporter.execute(() -> postProgress(itemRef, movedBytes, totalBytes, state));
}

private void postProgress(String itemRef, long movedBytes, long totalBytes, String state) {
try {
JSONObject body = new JSONObject()
.put("itemRef", itemRef)
.put("movedBytes", Math.max(0L, movedBytes))
.put("totalBytes", Math.max(0L, totalBytes))
.put("state", state);
HttpURLConnection c = open(ShareUrls.progress(baseUrl, manifestUrl), "POST", READ_TIMEOUT_MS);
try {
c.setDoOutput(true);
c.setRequestProperty("Content-Type", "application/json");
try (OutputStream os = c.getOutputStream()) {
os.write(body.toString().getBytes(StandardCharsets.UTF_8));
}
c.getResponseCode();
} finally {
c.disconnect();
}
} catch (Exception e) {
Log.i(TAG, "share progress report failed: " + e.getMessage());
}
}

// ---- HTTP plumbing -------------------------------------------------------------------------

/** Open a connection to the sender, pinned to the Network that reaches its hotspot when one exists,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
/*
* ============================================================================
* Name : ShareRelay.java
* Author : AppDevForAll
* Copyright : Copyright (c) 2026 AppDevForAll
* Description : K2GO-577. A loopback TCP relay from the box to the sender's rsync daemon. The box's
* rsync is a Linux process that Android routes by the default network, which on a
* hotspot without internet can be cellular, so it may never reach the sender. The relay
* accepts on 127.0.0.1 and opens each outbound socket on the Network whose subnet holds
* the sender (WifiNetworkBinder), the same pinning NetworkImportCarrier uses for HTTP.
* ============================================================================
*/
package org.appdevforall.k2go.contentexchange.data;

import android.content.Context;
import android.net.Network;
import android.util.Log;

import org.appdevforall.k2go.sync.transport.WifiNetworkBinder;

import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

/**
* Forwards each loopback connection to {@code host:port}. Lifecycle: the import service opens one per
* network run and closes it when the run ends; close() also ends every live connection, so a stopped
* run never leaves a transfer flowing.
*/
public final class ShareRelay implements Closeable {

private static final String TAG = "IIAB-ShareRelay";
private static final int CONNECT_TIMEOUT_MS = 15000;
private static final int BUFFER = 64 * 1024;

private final Context appContext;
private final String host;
private final int port;
private final ServerSocket server;
private final Set<Socket> live = ConcurrentHashMap.newKeySet();
private volatile boolean closed;

public ShareRelay(Context context, String host, int port) throws IOException {
this.appContext = context.getApplicationContext();
this.host = host;
this.port = port;
this.server = new ServerSocket(0, 4, InetAddress.getByName("127.0.0.1"));
Thread accept = new Thread(this::acceptLoop, "share-relay-accept");
accept.setDaemon(true);
accept.start();
}

/** The loopback port the box connects to. */
public int localPort() {
return server.getLocalPort();
}

@Override
public void close() {
closed = true;
closeQuietly(server);
for (Socket s : live) {
closeQuietly(s);
}
live.clear();
}

private void acceptLoop() {
while (!closed) {
Socket inbound;
try {
inbound = server.accept();
} catch (IOException e) {
return; // closed
}
live.add(inbound);
Thread t = new Thread(() -> bridge(inbound), "share-relay-conn");
t.setDaemon(true);
t.start();
}
}

private void bridge(Socket inbound) {
Socket outbound = null;
try {
// Resolve the Network per connection, so a Wi-Fi reconnect is picked up on the next pull.
Network net = WifiNetworkBinder.findHostNetwork(appContext, host);
outbound = net != null ? net.getSocketFactory().createSocket() : new Socket();
live.add(outbound);
outbound.connect(new InetSocketAddress(host, port), CONNECT_TIMEOUT_MS);
Socket out = outbound;
Thread up = new Thread(() -> pump(inbound, out), "share-relay-up");
up.setDaemon(true);
up.start();
pump(outbound, inbound);
} catch (IOException e) {
Log.i(TAG, "relay connection ended: " + e.getMessage());
} finally {
closeQuietly(inbound);
closeQuietly(outbound);
live.remove(inbound);
if (outbound != null) {
live.remove(outbound);
}
}
}

/** Copy one direction until EOF or an error, then close both ends so the peer pump ends too. */
private static void pump(Socket from, Socket to) {
byte[] buf = new byte[BUFFER];
try {
InputStream in = from.getInputStream();
OutputStream out = to.getOutputStream();
int n;
while ((n = in.read(buf)) != -1) {
out.write(buf, 0, n);
}
} catch (IOException ignored) {
// a closed socket on either side ends the pump
} finally {
closeQuietly(from);
closeQuietly(to);
}
}

private static void closeQuietly(Closeable c) {
if (c == null) {
return;
}
try {
c.close();
} catch (IOException ignored) {
// nothing to recover
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/*
* ============================================================================
* Name : ShareRsyncTarget.java
* Author : AppDevForAll
* Copyright : Copyright (c) 2026 AppDevForAll
* Description : K2GO-577. The rsync block of one share manifest entry ({"rsync": {port, module}}),
* read and validated at the network boundary. Absent or invalid means the item uses
* the HTTP fetch, so a sender that runs no rsync daemon still works. The manifest
* reader keeps the block in the entry JSON (it strips only itemRef and delivered).
* ============================================================================
*/
package org.appdevforall.k2go.contentexchange.data;

import androidx.annotation.Nullable;

import org.json.JSONObject;

import java.util.regex.Pattern;

/** Where one shared item can be pulled with rsync: the sender's daemon port and module. */
public final class ShareRsyncTarget {

/** Same rule as the core's module names: "<sessionId>-<itemRef>". */
private static final Pattern MODULE = Pattern.compile("^[A-Za-z0-9][A-Za-z0-9-]{0,80}$");

private final int port;
private final String module;

private ShareRsyncTarget(int port, String module) {
this.port = port;
this.module = module;
}

/** The target of a manifest entry, or null when it has no valid rsync block. */
@Nullable
public static ShareRsyncTarget fromEntry(@Nullable String entryJson) {
if (entryJson == null) {
return null;
}
try {
JSONObject entry = new JSONObject(entryJson);
JSONObject rsync = entry.optJSONObject("rsync");
if (rsync == null) {
return null;
}
int port = rsync.optInt("port", 0);
String module = rsync.optString("module", "");
if (port < 1 || port > 65535 || !MODULE.matcher(module).matches()) {
return null;
}
return new ShareRsyncTarget(port, module);
} catch (Exception e) {
return null;
}
}

public int port() {
return port;
}

public String module() {
return module;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,22 @@ public interface ImportSource {
*/
String startImport(String app, String entryJson, String payloadBoxPath) throws Exception;

/**
* Start a pull long operation (K2GO-577): the box pulls the item from a sender's rsync module
* ({@code host}:{@code port}, {@code module}, password {@code token}) straight into place, with
* no staged copy. The op is polled with {@link #importStatus} like an import; its movedBytes are
* the bytes on disk, so a resumed pull starts from the bytes it kept.
*/
String startPull(String app, String entryJson, String host, int port, String module, String token)
throws Exception;

/**
* Drop what a given-up pull of this item kept for a resume (K2GO-577): the user ended the run, so
* the partial bytes must not hold the disk. Best effort: a failure is ignored (the box sweeps old
* partials on its own).
*/
void discardPull(String app, String entryJson);

/**
* Poll one import operation's status. {@link ImportStatus.State#GONE} means the op was swept, so
* the caller re-starts once. Throws on a transport failure (the caller tolerates a few blips).
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package org.appdevforall.k2go.contentexchange.domain;

/**
* The box refused to start an rsync pull (K2GO-577): its core is older than the pull, or it has no
* rsync. A typed signal so the import service moves the item over the HTTP fetch instead of failing
* it. The data-layer import source maps the core's 400 / 501 on a pull start to this exception.
*/
public final class PullUnsupportedException extends Exception {
public PullUnsupportedException(String message) {
super(message);
}
}
Loading
Loading