Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ public class ContentManagerFragment extends Fragment {
private RecyclerView list;
private TextView status;
private View actionBar;
private View entryBar;
private TextView selectionLabel;
private Button exportButton;
private Button importButton;
Expand Down Expand Up @@ -101,6 +102,7 @@ public View onCreateView(@NonNull LayoutInflater inflater, @Nullable ViewGroup c
list = root.findViewById(R.id.k2go_cx_list);
status = root.findViewById(R.id.k2go_cx_status);
actionBar = root.findViewById(R.id.k2go_cx_actionbar);
entryBar = root.findViewById(R.id.k2go_cx_entrybar);
selectionLabel = root.findViewById(R.id.k2go_cx_selection_label);
exportButton = root.findViewById(R.id.k2go_cx_export);
importButton = root.findViewById(R.id.k2go_cx_import);
Expand Down Expand Up @@ -183,20 +185,29 @@ private void render(ContentManagerUiState st) {
showStatus(getString(R.string.k2go_cx_loading));
return;
case ERROR:
// K2GO-574: nothing works while the content service is down, so offer no action; the box
// restarts conex-core on its own (dash-node service-heal) and a tap reloads.
list.setVisibility(View.GONE);
entryBar.setVisibility(View.GONE);
actionBar.setVisibility(View.GONE);
showStatus(getString(R.string.k2go_cx_error));
status.setOnClickListener(v -> viewModel.load());
return;
default:
break;
}
status.setVisibility(View.GONE);
entryBar.setVisibility(View.VISIBLE);
list.setVisibility(View.VISIBLE);
adapter.submit(buildRows(st.content, st.storage));
renderSelection(viewModel.selection().getValue()); // the ERROR branch hid the action bar
}

private void showStatus(String text) {
status.setText(text);
status.setVisibility(View.VISIBLE);
status.setOnClickListener(null);
status.setClickable(false);
}

/** Flattens the grouped content into the adapter's row list (header, groups, items, empty). */
Expand Down
1 change: 1 addition & 0 deletions static/conex-core/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ One line per version, newest first. Every REST-facing change bumps `conex_core.V
role), so this file is the human record of what each bump enables. Keep entries short:
`version - change (TICKET)`.

- **0.3.5** - Kiwix counts as installed only when kiwix-serve is present, not just the zims dir, so a library that cannot open ZIMs answers NO_APP and does not import them. (K2GO-574)
- **0.3.4** - import becomes a long operation like export: POST /v1/import/item now STARTS the ingest and returns an opId, the client polls GET /v1/import/status/{op}, and DELETE /v1/import/{op} cancels, so a 35-141 s Kolibri import no longer times out at the dash-node/nginx relay. A shared liveness watchdog supervises the Kolibri import and export CLIs by byte and CPU progress (no fixed timeout; the window is relative to the task) and names a signal-9 exit as a likely out-of-memory kill. (K2GO-376)
- **0.3.3** - observability: log every error response and uncaught handler exception to stdout (the supervisor captures it to /var/log/conex-core.log), and line-buffer stdout/stderr in __main__ so output reaches the log while the server runs forever, so a failed import or share leaves a box-side trace; an uncaught error now returns a 500 JSON instead of dropping the connection. (K2GO-376)
- **0.3.2** - /v1/import/item returns 507 (not 422) when a stage runs out of disk (adapters tag it code "no_space"), so the front stops the run with a space message; /v1/export/prepare drops the proot-based free-space preflight too (same false-507 as import), failing cleanly during staging instead. (K2GO-376)
Expand Down
2 changes: 1 addition & 1 deletion static/conex-core/conex_core/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,5 @@
# The core build. Bumped per core change so a box can report which core it runs (GET
# /v1/version), mirroring dash-node's package.json version and /system/version. The
# conex_core Ansible role writes this into the [conex_core] section of iiab.ini.
VERSION = "0.3.4"
VERSION = "0.3.5"
__version__ = VERSION
7 changes: 5 additions & 2 deletions static/conex-core/conex_core/adapters/kiwix.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,12 +123,15 @@ class KiwixAdapter(ContentAdapter):
app = "kiwix"
id_kind = AUTHORITATIVE

def __init__(self, zims_dir: str, indexer: str = "/usr/bin/iiab-make-kiwix-lib"):
def __init__(self, zims_dir: str, indexer: str = "/usr/bin/iiab-make-kiwix-lib",
server: str = "/opt/iiab/kiwix/bin/kiwix-serve"):
self._dir = zims_dir
self._indexer = indexer
self._server = server

def is_installed(self) -> bool:
return os.path.isdir(self._dir)
# The zims dir can exist without Kiwix (an old 32-bit library); a ZIM there has no reader.
return os.path.isdir(self._dir) and os.path.isfile(self._server)

def list(self) -> List[ManifestItem]:
items: List[ManifestItem] = []
Expand Down
2 changes: 2 additions & 0 deletions static/conex-core/conex_core/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ class Config:
content_dir: str = "/library"
zims_dir: str = "/library/zims/content/"
kiwix_indexer: str = "/usr/bin/iiab-make-kiwix-lib"
kiwix_serve: str = "/opt/iiab/kiwix/bin/kiwix-serve"
kolibri_db: str = "/library/kolibri/db.sqlite3"
kolibri_bin: str = "/usr/bin/kolibri"
calibre_db: str = "/library/calibre-web/metadata.db"
Expand All @@ -46,6 +47,7 @@ def from_env() -> "Config":
content_dir=os.environ.get("CONEX_CONTENT_DIR", "/library"),
zims_dir=os.environ.get("CONEX_ZIMS_DIR", "/library/zims/content/"),
kiwix_indexer=os.environ.get("CONEX_KIWIX_INDEXER", "/usr/bin/iiab-make-kiwix-lib"),
kiwix_serve=os.environ.get("CONEX_KIWIX_SERVE", "/opt/iiab/kiwix/bin/kiwix-serve"),
kolibri_db=os.environ.get("CONEX_KOLIBRI_DB", "/library/kolibri/db.sqlite3"),
kolibri_bin=os.environ.get("CONEX_KOLIBRI_BIN", "/usr/bin/kolibri"),
calibre_db=os.environ.get("CONEX_CALIBRE_DB", "/library/calibre-web/metadata.db"),
Expand Down
2 changes: 1 addition & 1 deletion static/conex-core/conex_core/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def __init__(
@staticmethod
def load(config: Config) -> "Registry":
candidates: List[ContentAdapter] = [
KiwixAdapter(config.zims_dir, config.kiwix_indexer),
KiwixAdapter(config.zims_dir, config.kiwix_indexer, config.kiwix_serve),
KolibriAdapter(config.kolibri_db, config.kolibri_bin, config.staging_dir),
CalibreAdapter(config.calibre_db, config.calibre_url, config.credentials_file, config.staging_dir),
]
Expand Down
9 changes: 8 additions & 1 deletion static/conex-core/tests/test_kiwix.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,9 @@ def test_list_reads_installed_zims_with_uuid_and_size(self):
with open(os.path.join(d, "readme.txt"), "w") as f:
f.write("x")

adapter = KiwixAdapter(d)
server = os.path.join(d, "kiwix-serve")
open(server, "w").close()
adapter = KiwixAdapter(d, server=server)
self.assertTrue(adapter.is_installed())
items = adapter.list()
self.assertEqual(len(items), 1)
Expand Down Expand Up @@ -136,6 +138,11 @@ def test_get_rejects_traversal_without_touching_fs(self):
self.assertIsNone(adapter.get("../secret_en_x_2020-01.zim"))
self.assertIsNone(adapter.get("..\\secret_en_x_2020-01.zim"))

def test_zims_dir_without_kiwix_serve_is_not_installed(self):
# K2GO-574: an old 32-bit library has the zims dir but no kiwix-serve.
with tempfile.TemporaryDirectory() as d:
self.assertFalse(KiwixAdapter(d, server=os.path.join(d, "missing")).is_installed())

def test_absent_dir_is_not_installed(self):
adapter = KiwixAdapter("/no/such/dir/for/conex-core")
self.assertFalse(adapter.is_installed())
Expand Down
1 change: 1 addition & 0 deletions static/dashboard/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ One line per version, newest first. Every REST-facing change bumps the version i
(the app surfaces it via `/system/dashboard/update-check` and the "Update available" pill), so this
file is the human record of what each bump enables. Keep entries short: `version - change (TICKET)`.

- **1.6.1** - The box restarts conex-core (Content Exchange) on its own when its process is gone: service-heal probes `/v1/apps` and runs `pdsm restart conex-core` only on a refused connection (a busy, slow service is never restarted mid-operation), and skips boxes without the conex-core pdsm service. `POST /system/service/conex-core/restart` is allowed. Localhost-only. (K2GO-574)
- **1.6.0** - Privacy switches for the box services that contact an outside server on their own (K2GO-559). `GET /system/privacy` reports each one (Kolibri statistics ping, Forgejo update check): installed, and whether it may contact its server. `POST /system/privacy/:service {outbound}` sets the service's own config key (Kolibri `[Deployment] DISABLE_PING` in `options.ini`, Forgejo `[cron.update_checker] ENABLED` in `app.ini`) and only writes the config; it never restarts a service. `pending` reports a change that waits for a restart (config newer than the running process; derived on each read, not stored). The app asks the user (Restart now / Later) and requests the restart through `POST /system/service/:svc/restart` (ADR-5343), which now runs one restart per service at a time; a pending change is restarted at the next app open. A missing key reads as the service default (on). On its first start a box sets every installed switch OFF once (new bakes ship OFF), then writes `/library/dashboard/privacy-defaults.done` so a switch the user turns on later stays on; it applies like any pending change. Localhost-only. (K2GO-559)
- **1.5.2** - FQR maps on the new tile-extract.py CLI (iiab#4574, carried as `tools/upstream-patches` 0004) (K2GO-566). The download, the size estimate and the delete follow the CLI of the installed script: `--bbox=`, `--yes`, `--progress json` and `--threads 8` when it has them, else the older positional form (a box can run a newer dash-node than its rootfs). Delete uses `--yes` with no stdin pipe (no EPIPE) and also removes an unfinished download; `POST /maps/delete` returns 409 `{busy: true}` while another FQR download or delete holds the box lock. The script runs one FQR download or delete at a time, so a second download now waits as `queued` and then runs (before, both ran in parallel). A purge after a cancel or a hard error never removes a region that the job did not make. A Cancel of a paused download now removes its unfinished files (new engine hook `registerDiscard`), and a Cancel during a pause's flush wins over the pause. The script's own catalog and mirror network errors and an outside SIGTERM (exit 3) keep the partial (paused) instead of a purge. A download that ends with no region (the script exits 0 because the box overlaps a region that an earlier queued job made) is an error, not `done`. Localhost-only. (K2GO-566)
- **1.5.1** - Pin better-sqlite3 to an exact version so the live self-update reuses the baked native binary (K2GO-557). The dependency was a caret range (`^12.8.0`), so a fresh `yarn install` on the box could resolve a newer 12.x than the one baked into the rootfs. When the resolved version differed from the live one, `tools/rebuild-dashboard.sh` recompiled better-sqlite3 from source (about 20 min on a low-RAM 32-bit box) instead of reusing the compiled `better_sqlite3.node`. The exact pin (`12.11.1`, the version that ships in the rootfs) makes the box resolve the same version every time, so the reuse path always applies. CI (`dashboard-check.yml`) now fails if the pin is relaxed back to a range. No REST-surface change; the version bump is the delivery mechanism for the pinned manifest. Localhost-only. (K2GO-557) Also repairs the in-app FQR list and delete for upstream's new maps storage format (K2GO-566): regions are now `<name>.fqr/` directories with `meta.json {"bbox":[...]}` instead of a central `extracts.json`. dash-node adds `GET /maps/regions` (enumerates the `.fqr` dirs) and the app reads it instead of the static catalog; the delete path is updated for upstream's now-interactive `tile-extract.py delete` (a noninteractive form via `tools/upstream-patches`). Localhost-only. (K2GO-566)
Expand Down
2 changes: 1 addition & 1 deletion static/dashboard/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "dashboard-console",
"version": "1.6.0",
"version": "1.6.1",
"description": "",
"main": "index.js",
"scripts": {
Expand Down
2 changes: 1 addition & 1 deletion static/dashboard/sockets/conex-proxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import http from 'http';
const CONEX_HOST = '127.0.0.1';

// The core's port, env-overridable to match its own CONEX_PORT default (9500).
function conexPort(): number {
export function conexPort(): number {
return Number(process.env.CONEX_PORT) || 9500;
}

Expand Down
33 changes: 27 additions & 6 deletions static/dashboard/sockets/service-heal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,22 @@
// box heals itself (§10). The app/reconciler owns box up/down and does NOT manage individual
// box services (§10 layering); this watcher is the dashboard REST core owning its own service
// tree — the single owner of "is my content service tree healthy?".
import { existsSync } from 'fs';
import { conexPort } from './conex-proxy';
import { restartService } from './services';

/** A watched content service: its pdsm name and the loopback URL that reflects its health.
* kiwix first (the device-confirmed wedge, §10); the others are added here as they are
* device-verified — the endpoint already accepts the full supported set. */
interface Watch { svc: string; probeUrl: string; }
interface Watch {
svc: string;
probeUrl: string;
/** Heal only a missing process: any reply is ok and only a refused connection is down, so a busy
* service (a slow reply) is never restarted in the middle of an import or export. */
processOnly?: boolean;
/** Not installed on this box when this file is missing: leave it alone. */
installedFile?: string;
}

// The box fronts content on nginx :8085 (dash-node-nginx.conf); a HEAD on the public path is
// exactly the served page's own probe, run here on loopback. Origin/timing are env-overridable
Expand All @@ -27,6 +37,9 @@ const PROBE_TIMEOUT_MS = Number(process.env.K2GO_HEAL_PROBE_TIMEOUT_MS) || 4_000

const WATCHED: Watch[] = [
{ svc: 'kiwix', probeUrl: `${PUBLIC_ORIGIN}/kiwix/` },
// K2GO-574: Content Exchange. /v1/apps is the route its pdsm health check uses.
{ svc: 'conex-core', probeUrl: `http://127.0.0.1:${conexPort()}/v1/apps`, processOnly: true,
installedFile: '/usr/local/pdsm/services-available/conex-core' },
];

/** Enough time has passed since the last restart attempt to try again. A per-service
Expand Down Expand Up @@ -54,13 +67,21 @@ export function classifyProbe(status: number | null): ProbeResult {
return 'down';
}

/** True when a fetch failed because nothing listens on the port (the process is gone). */
export function isRefused(e: unknown): boolean {
const err = e as { code?: string; cause?: { code?: string } } | null;
return err?.code === 'ECONNREFUSED' || err?.cause?.code === 'ECONNREFUSED';
}

/** HEAD the probe URL and classify the outcome. Only a 'down' verdict heals; 'absent' (404,
* not installed) and 'ok' are left alone. */
async function probe(url: string): Promise<ProbeResult> {
async function probe(w: Watch): Promise<ProbeResult> {
if (w.installedFile && !existsSync(w.installedFile)) return 'absent';
try {
const res = await fetch(url, { method: 'HEAD', signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) });
return classifyProbe(res.status);
} catch {
const res = await fetch(w.probeUrl, { method: 'HEAD', signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) });
return w.processOnly ? 'ok' : classifyProbe(res.status);
} catch (e) {
if (w.processOnly) return isRefused(e) ? 'down' : 'ok';
return classifyProbe(null);
}
}
Expand All @@ -72,7 +93,7 @@ export function startServiceHeal(): NodeJS.Timeout {

const tick = async (): Promise<void> => {
for (const w of WATCHED) {
if (await probe(w.probeUrl) !== 'down') continue; // 'ok'/'absent' need no heal
if (await probe(w) !== 'down') continue; // 'ok'/'absent' need no heal
const now = Date.now();
if (!dueForRestart(lastAttempt.get(w.svc) ?? 0, now, COOLDOWN_MS)) continue;
lastAttempt.set(w.svc, now);
Expand Down
15 changes: 14 additions & 1 deletion static/dashboard/sockets/services.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { isRestartableService, RESTARTABLE_SERVICES } from './services';
import { dueForRestart, classifyProbe } from './service-heal';
import { dueForRestart, classifyProbe, isRefused } from './service-heal';

// --- isRestartableService: the /system/service/:svc/restart allowlist -------------------------

Expand All @@ -13,6 +13,7 @@ test('isRestartableService: accepts every upstream content service', () => {
// The concrete cases §10 wires first.
assert.equal(isRestartableService('kiwix'), true);
assert.equal(isRestartableService('kolibri'), true);
assert.equal(isRestartableService('conex-core'), true); // K2GO-574: service-heal restarts it
});

test('isRestartableService: rejects dash-node (k2go\'s own service, not a content service)', () => {
Expand Down Expand Up @@ -67,3 +68,15 @@ test('classifyProbe: a wedged upstream (5xx) and no reply (null) both heal', ()
assert.equal(classifyProbe(504), 'down');
assert.equal(classifyProbe(null), 'down'); // timeout / connection refused
});

// --- isRefused: conex-core is healed only when its process is gone (K2GO-574) ---------------

test('isRefused: a refused connection means the process is gone', () => {
assert.equal(isRefused({ cause: { code: 'ECONNREFUSED' } }), true);
assert.equal(isRefused({ code: 'ECONNREFUSED' }), true);
});

test('isRefused: a timeout (busy service) or no error is not refused', () => {
assert.equal(isRefused({ name: 'TimeoutError', code: 23 }), false);
assert.equal(isRefused(null), false);
});
2 changes: 1 addition & 1 deletion static/dashboard/sockets/services.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import { spawn } from 'child_process';

export const RESTARTABLE_SERVICES: readonly string[] = [
'nginx', 'php-fpm', 'mariadb', 'kolibri', 'kiwix', 'calibre-web', 'forgejo',
'nginx', 'php-fpm', 'mariadb', 'kolibri', 'kiwix', 'calibre-web', 'forgejo', 'conex-core',
];

/** A name is restartable only when it is exactly one of the known upstream content
Expand Down
Loading