diff --git a/static/dashboard/CHANGELOG.md b/static/dashboard/CHANGELOG.md index 3b386a65a..dfe1f268c 100644 --- a/static/dashboard/CHANGELOG.md +++ b/static/dashboard/CHANGELOG.md @@ -4,6 +4,7 @@ One line per version, newest first. Every REST-facing change bumps the version i (the app surfaces it via `/system/dashboard/update-check` and the "Update available" pill), so this file is the human record of what each bump enables. Keep entries short: `version - change (TICKET)`. +- **1.6.2** - Box logs no longer identify the user, on boxes installed before bake patches 0010-0012 (K2GO-559). At its first start, dash-node applies the same change once: nginx access log without client IP and User-Agent (`/etc/nginx/nginx.conf`), Forgejo router log off (`LOGGER_ROUTER_MODE` empty in `app.ini`), Calibre-Web log level DEBUG -> INFO (`app.db`). A value an admin changed is kept. Each change applies at the service restart (Forgejo: at the next app open, as a pending privacy change; nginx and Calibre-Web: at the next box start). After that restart, the older logs of each changed service are removed once (rotated copies deleted, live log emptied). Markers: `/library/dashboard/log-less.done`, `/library/dashboard/log-less-purge.done`. No REST change. (K2GO-559) - **1.6.1** - The box restarts conex-core (Content Exchange) on its own when its process is gone: service-heal probes `/v1/apps` and runs `pdsm restart conex-core` only on a refused connection (a busy, slow service is never restarted mid-operation), and skips boxes without the conex-core pdsm service. `POST /system/service/conex-core/restart` is allowed. Localhost-only. (K2GO-574) - **1.6.0** - Privacy switches for the box services that contact an outside server on their own (K2GO-559). `GET /system/privacy` reports each one (Kolibri statistics ping, Forgejo update check): installed, and whether it may contact its server. `POST /system/privacy/:service {outbound}` sets the service's own config key (Kolibri `[Deployment] DISABLE_PING` in `options.ini`, Forgejo `[cron.update_checker] ENABLED` in `app.ini`) and only writes the config; it never restarts a service. `pending` reports a change that waits for a restart (config newer than the running process; derived on each read, not stored). The app asks the user (Restart now / Later) and requests the restart through `POST /system/service/:svc/restart` (ADR-5343), which now runs one restart per service at a time; a pending change is restarted at the next app open. A missing key reads as the service default (on). On its first start a box sets every installed switch OFF once (new bakes ship OFF), then writes `/library/dashboard/privacy-defaults.done` so a switch the user turns on later stays on; it applies like any pending change. Localhost-only. (K2GO-559) - **1.5.2** - FQR maps on the new tile-extract.py CLI (iiab#4574, carried as `tools/upstream-patches` 0004) (K2GO-566). The download, the size estimate and the delete follow the CLI of the installed script: `--bbox=`, `--yes`, `--progress json` and `--threads 8` when it has them, else the older positional form (a box can run a newer dash-node than its rootfs). Delete uses `--yes` with no stdin pipe (no EPIPE) and also removes an unfinished download; `POST /maps/delete` returns 409 `{busy: true}` while another FQR download or delete holds the box lock. The script runs one FQR download or delete at a time, so a second download now waits as `queued` and then runs (before, both ran in parallel). A purge after a cancel or a hard error never removes a region that the job did not make. A Cancel of a paused download now removes its unfinished files (new engine hook `registerDiscard`), and a Cancel during a pause's flush wins over the pause. The script's own catalog and mirror network errors and an outside SIGTERM (exit 3) keep the partial (paused) instead of a purge. A download that ends with no region (the script exits 0 because the box overlaps a region that an earlier queued job made) is an error, not `done`. Localhost-only. (K2GO-566) diff --git a/static/dashboard/package-lock.json b/static/dashboard/package-lock.json index 8b249d3b2..5277a40d3 100644 --- a/static/dashboard/package-lock.json +++ b/static/dashboard/package-lock.json @@ -1,12 +1,12 @@ { "name": "dashboard-console", - "version": "1.6.0", + "version": "1.6.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dashboard-console", - "version": "1.6.0", + "version": "1.6.2", "license": "GPL-3.0-or-later", "dependencies": { "better-sqlite3": "12.11.1", diff --git a/static/dashboard/package.json b/static/dashboard/package.json index 0c8e714e9..d965e168a 100644 --- a/static/dashboard/package.json +++ b/static/dashboard/package.json @@ -1,10 +1,10 @@ { "name": "dashboard-console", - "version": "1.6.1", + "version": "1.6.2", "description": "", "main": "index.js", "scripts": { - "test": "node --require ts-node/register --test sockets/maps.socket.test.ts sockets/rolling-log.test.ts sockets/kolibri.session.test.ts sockets/credentials.test.ts sockets/net-retry.test.ts sockets/services.test.ts sockets/log-rotate.test.ts sockets/rebuild-status.test.ts sockets/conex-proxy.test.ts sockets/maps.fqr-behavior.test.ts sockets/privacy.test.ts", + "test": "node --require ts-node/register --test sockets/maps.socket.test.ts sockets/rolling-log.test.ts sockets/kolibri.session.test.ts sockets/credentials.test.ts sockets/net-retry.test.ts sockets/services.test.ts sockets/log-rotate.test.ts sockets/rebuild-status.test.ts sockets/conex-proxy.test.ts sockets/maps.fqr-behavior.test.ts sockets/privacy.test.ts sockets/log-less.test.ts", "test:db": "node --require ts-node/register --test sockets/jobs.test.ts", "typecheck": "tsc --noEmit", "build": "tsc", diff --git a/static/dashboard/server.ts b/static/dashboard/server.ts index d53acd02d..44cc79d96 100644 --- a/static/dashboard/server.ts +++ b/static/dashboard/server.ts @@ -17,6 +17,7 @@ import { apiRouter } from './routes'; import { startServiceHeal } from './sockets/service-heal'; import { startLogRotation, stopLogRotation } from './sockets/log-rotate'; import { applyPrivacyDefaultsOnce, PRIVACY_DEFAULTS_MARKER, PRIVACY_SWITCHES } from './sockets/privacy'; +import { applyLogLessOnce, LOG_LESS_MARKER, LOG_LESS_TARGETS, LOG_PURGE_MARKER, purgeOldLogsOnce } from './sockets/log-less'; // K2GO-559: one-time privacy defaults, before the API answers. Only the live service runs it: the // rebuild's staged smoke instance sets PORT and must not change the box (tools/rebuild-dashboard.sh). @@ -27,6 +28,14 @@ if (process.env.PORT === undefined) { } catch (e) { console.error('[privacy] one-time defaults failed; retry at next start', e); } + try { + const r = applyLogLessOnce(LOG_LESS_TARGETS, LOG_LESS_MARKER); + if (r.ran) console.log(`[log-less] applied once; changed (applies at the service restart): ${r.changed.join(', ') || 'none'}`); + const p = purgeOldLogsOnce(LOG_LESS_MARKER, LOG_PURGE_MARKER); + if (p.ran) console.log(`[log-less] old logs removed: ${p.purged.join(', ') || 'none'}; waiting for a restart: ${p.waiting.join(', ') || 'none'}`); + } catch (e) { + console.error('[log-less] one-time step failed; retry at next start', e); + } } const app = express(); diff --git a/static/dashboard/sockets/log-less.test.ts b/static/dashboard/sockets/log-less.test.ts new file mode 100644 index 000000000..162cc65e6 --- /dev/null +++ b/static/dashboard/sockets/log-less.test.ts @@ -0,0 +1,160 @@ +/// +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { + applyLogLessOnce, forgejoWithoutRouterLog, nginxWithoutClientIds, purgeOldLogsOnce, rotatedBefore, + LogLessStep, ServiceLog, +} from './log-less'; + +const STOCK_NGINX = [ + 'http {', + ' log_format awstats', + ` '$remote_addr - $remote_user [$time_local] "$request" '`, + ` '$status $body_bytes_sent "$http_referer" '`, + ` '"$http_user_agent" "http_x_forwarded_for"';`, + '', + ' access_log /var/log/nginx/access.log awstats;', + ` log_format scripts '$request > $document_root$fastcgi_script_name $fastcgi_path_info';`, + '}', + '', +].join('\n'); + +test('nginx: the stock awstats format loses the client IP and User-Agent, same fields', () => { + const out = nginxWithoutClientIds(STOCK_NGINX)!; + assert.ok(out.includes(`'- - $remote_user [$time_local] "$request" '`)); + assert.ok(out.includes(`'"-" "http_x_forwarded_for"';`)); + assert.ok(!out.includes('$remote_addr') && !out.includes('$http_user_agent')); + assert.ok(out.includes(`log_format scripts '$request >`)); // other formats stay + assert.equal(nginxWithoutClientIds(out), undefined); // second run: nothing to do +}); + +test('nginx: a changed or missing awstats format is left alone', () => { + assert.equal(nginxWithoutClientIds('http {\n access_log off;\n}\n'), undefined); + assert.equal(nginxWithoutClientIds(`log_format awstats '$remote_user "$request"';`), undefined); +}); + +test('forgejo: the router log is turned off in [log]; a key the admin set is kept', () => { + const t = '[mailer]\nENABLED = false\n\n[log]\nROOT_PATH = /var/log/forgejo\nMODE = console\nLEVEL = Info\n'; + const out = forgejoWithoutRouterLog(t)!; + assert.ok(/\[log\][^[]*LOGGER_ROUTER_MODE = \s*\n/.test(out)); + assert.equal(forgejoWithoutRouterLog(out), undefined); + assert.equal(forgejoWithoutRouterLog('[log]\nDISABLE_ROUTER_LOG = false\n'), undefined); + assert.equal(forgejoWithoutRouterLog('[log]\nlogger.router.MODE = console\n'), undefined); +}); + +function box(files: { nginx?: string; forgejo?: string; calibre?: boolean }) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'logless-')); + const t = { + nginxConf: path.join(dir, 'nginx.conf'), + forgejoIni: path.join(dir, 'app.ini'), + calibreAppDb: path.join(dir, 'app.db'), + }; + if (files.nginx !== undefined) fs.writeFileSync(t.nginxConf, files.nginx); + if (files.forgejo !== undefined) fs.writeFileSync(t.forgejoIni, files.forgejo); + if (files.calibre) fs.writeFileSync(t.calibreAppDb, ''); + return { t, marker: path.join(dir, 'log-less.done') }; +} + +test('once: first run changes every installed service, then writes the marker; later runs do nothing', () => { + const b = box({ nginx: STOCK_NGINX, forgejo: '[log]\nMODE = console\n', calibre: true }); + const calls: string[] = []; + const calibre = (f: string) => { calls.push(f); return true; }; + assert.deepEqual(applyLogLessOnce(b.t, b.marker, calibre), { ran: true, changed: ['nginx', 'forgejo', 'calibre-web'] }); + assert.deepEqual(calls, [b.t.calibreAppDb]); + assert.ok(!fs.readFileSync(b.t.nginxConf, 'utf8').includes('$remote_addr')); + assert.equal(fs.existsSync(b.marker), true); + assert.deepEqual(applyLogLessOnce(b.t, b.marker, calibre), { ran: false, changed: [] }); + assert.equal(calls.length, 1); +}); + +test('once: services that are not installed are skipped', () => { + const b = box({}); + assert.deepEqual(applyLogLessOnce(b.t, b.marker, () => { throw new Error('not called'); }), { ran: true, changed: [] }); + assert.equal(fs.existsSync(b.t.forgejoIni), false); +}); + +test('once: a failed step leaves no marker, so the next start tries again', () => { + const b = box({ nginx: STOCK_NGINX, calibre: true }); + assert.throws(() => applyLogLessOnce(b.t, b.marker, () => { throw new Error('database is locked'); })); + assert.equal(fs.existsSync(b.marker), false); + assert.deepEqual(applyLogLessOnce(b.t, b.marker, () => false), { ran: true, changed: [] }); // nginx already done +}); + +test('once: the marker records which services changed (input of the purge)', () => { + const b = box({ nginx: STOCK_NGINX }); + applyLogLessOnce(b.t, b.marker, () => false); + assert.deepEqual(JSON.parse(fs.readFileSync(b.marker, 'utf8')).changed, ['nginx']); +}); + +// --- Purge of the logs written before the change --------------------------------------------------- + +function purgeBox(changed: LogLessStep[]) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'logpurge-')); + const marker = path.join(dir, 'log-less.done'); + const old = Date.now() / 1000 - 3600; // one hour before the change + const logs = {} as Record; + for (const svc of ['nginx', 'forgejo', 'calibre-web'] as LogLessStep[]) { + const log = path.join(dir, `${svc}.log`); + fs.writeFileSync(log, '192.168.1.20 - - "GET /" "Mozilla/5.0 (Linux; moto g)"\n'); + for (const r of [`${log}.1`, `${log}.2.gz`]) { fs.writeFileSync(r, 'old'); fs.utimesSync(r, old, old); } + logs[svc] = { log, process: svc }; + } + fs.writeFileSync(path.join(dir, 'nginx.log.bak'), 'not a rotated copy'); + fs.utimesSync(path.join(dir, 'nginx.log.bak'), old, old); + fs.writeFileSync(marker, JSON.stringify({ at: 'x', changed }) + '\n'); + return { dir, marker, purgeMarker: path.join(dir, 'log-less-purge.done'), logs }; +} + +test('rotatedBefore: only .N and .N.gz older than the change', () => { + const b = purgeBox([]); + const got = rotatedBefore(b.logs.nginx.log, Date.now()).map((f) => path.basename(f)).sort(); + assert.deepEqual(got, ['nginx.log.1', 'nginx.log.2.gz']); + assert.deepEqual(rotatedBefore(b.logs.nginx.log, Date.now() - 2 * 3600_000), []); +}); + +test('purge: a changed service on the new config loses its old logs; others are not touched', () => { + const b = purgeBox(['nginx', 'calibre-web']); + const started = Date.now() + 1000; // restarted after the change + const r = purgeOldLogsOnce(b.marker, b.purgeMarker, b.logs, () => started); + assert.deepEqual(r, { ran: true, purged: ['nginx', 'calibre-web'], waiting: [] }); + assert.equal(fs.statSync(b.logs.nginx.log).size, 0); + assert.equal(fs.existsSync(`${b.logs.nginx.log}.1`), false); + assert.equal(fs.existsSync(`${b.logs.nginx.log}.2.gz`), false); + assert.equal(fs.existsSync(path.join(b.dir, 'nginx.log.bak')), true); + assert.ok(fs.statSync(b.logs.forgejo.log).size > 0); // not changed: no purge + assert.equal(fs.existsSync(`${b.logs.forgejo.log}.1`), true); + assert.equal(fs.existsSync(b.purgeMarker), true); + assert.equal(purgeOldLogsOnce(b.marker, b.purgeMarker, b.logs, () => started).ran, false); +}); + +test('purge: a service still on the old config waits; the next start purges it', () => { + const b = purgeBox(['nginx', 'forgejo']); + const startOf = (p: string) => (p === 'forgejo' ? Date.now() - 3600_000 : null); // nginx not running + assert.deepEqual(purgeOldLogsOnce(b.marker, b.purgeMarker, b.logs, startOf), + { ran: true, purged: ['nginx'], waiting: ['forgejo'] }); + assert.ok(fs.statSync(b.logs.forgejo.log).size > 0); + assert.equal(fs.existsSync(b.purgeMarker), false); + assert.deepEqual(purgeOldLogsOnce(b.marker, b.purgeMarker, b.logs, () => Date.now() + 1000), + { ran: true, purged: ['nginx', 'forgejo'], waiting: [] }); + assert.equal(fs.statSync(b.logs.forgejo.log).size, 0); + assert.equal(fs.existsSync(b.purgeMarker), true); +}); + +test('purge: a copy rotated after the change but before the restart is removed too', () => { + const b = purgeBox(['nginx']); + const rotatedLater = `${b.logs.nginx.log}.3`; + fs.writeFileSync(rotatedLater, '10.0.0.7 - - "GET /" "Mozilla/5.0"'); // mtime now: after the marker + const r = purgeOldLogsOnce(b.marker, b.purgeMarker, b.logs, () => Date.now() + 1000); + assert.deepEqual(r.purged, ['nginx']); + assert.equal(fs.existsSync(rotatedLater), false); +}); + +test('purge: nothing before the log-less step, nothing to do when it changed nothing', () => { + const b = purgeBox([]); + assert.equal(purgeOldLogsOnce(path.join(b.dir, 'missing'), b.purgeMarker, b.logs, () => null).ran, false); + assert.deepEqual(purgeOldLogsOnce(b.marker, b.purgeMarker, b.logs, () => null), { ran: true, purged: [], waiting: [] }); + assert.ok(fs.statSync(b.logs.nginx.log).size > 0); +}); diff --git a/static/dashboard/sockets/log-less.ts b/static/dashboard/sockets/log-less.ts new file mode 100644 index 000000000..5d116805b --- /dev/null +++ b/static/dashboard/sockets/log-less.ts @@ -0,0 +1,148 @@ +// sockets/log-less.ts -- K2GO-559 +// +// Log less at the source: the box logs must not identify who used the server. New bakes ship this +// (tools/upstream-patches 0010-0012); this step brings boxes installed before that change to the +// same state, once. Each step edits the service's own config and keeps a value an admin changed. +// The change applies when the service restarts: Forgejo then reads as `pending` (privacy.ts) and the +// app restarts it at its next open; nginx and Calibre-Web apply it at the next box start. After a +// service runs on the new config, its older logs (written with client IDs) are removed once. +import fs from 'fs'; +import path from 'path'; +import Database from 'better-sqlite3'; +import { processStartMs, readIniOption, setIniOption, writeAtomic } from './privacy'; + +export type LogLessStep = 'nginx' | 'forgejo' | 'calibre-web'; + +export interface LogLessTargets { + nginxConf: string; + forgejoIni: string; + calibreAppDb: string; +} + +export const LOG_LESS_TARGETS: LogLessTargets = { + nginxConf: process.env.K2GO_NGINX_CONF || '/etc/nginx/nginx.conf', + forgejoIni: process.env.K2GO_FORGEJO_APP_INI || '/etc/forgejo/app.ini', + calibreAppDb: process.env.K2GO_CALIBRE_APP_DB || '/library/calibre-web/config/app.db', +}; + +// Owner of the marker: dash-node, at startup. It is written only after every step succeeds; a +// crash before it re-runs the steps at the next start, which is safe because each step is a no-op +// on a config that already has the change. It records which services changed (for the log purge). +export const LOG_LESS_MARKER = process.env.K2GO_LOG_LESS_MARKER || '/library/dashboard/log-less.done'; + +/** The IIAB `log_format awstats` with the client IP and the User-Agent replaced by "-" (same fields, + * so awstats still parses the log). undefined when the statement is not the stock one. */ +export function nginxWithoutClientIds(text: string): string | undefined { + const m = /log_format\s+awstats\b[^;]*;/.exec(text); + if (!m || !m[0].includes('$remote_addr') || !m[0].includes('"$http_user_agent"')) return undefined; + const fixed = m[0].replace('$remote_addr', '-').replace('"$http_user_agent"', '"-"'); + return text.slice(0, m.index) + fixed + text.slice(m.index + m[0].length); +} + +// Every key Forgejo reads for the router logger (modules/setting/log.go). Any of them set = the +// admin chose; we leave it. +const FORGEJO_ROUTER_KEYS = ['LOGGER_ROUTER_MODE', 'logger.router.MODE', 'ROUTER', 'DISABLE_ROUTER_LOG']; + +/** app.ini with the router log off (`LOGGER_ROUTER_MODE` empty). undefined when a router key is set. */ +export function forgejoWithoutRouterLog(text: string): string | undefined { + if (FORGEJO_ROUTER_KEYS.some((k) => readIniOption(text, 'log', k) !== undefined)) return undefined; + return setIniOption(text, 'log', 'LOGGER_ROUTER_MODE', ''); +} + +/** Python logging levels as Calibre-Web stores them (`settings.config_log_level`). */ +const CALIBRE_DEBUG_MAX = 19; +const CALIBRE_INFO = 20; + +/** Raise a DEBUG Calibre-Web log level to INFO. Returns true when the row changed. */ +export function sqliteCalibreInfo(dbFile: string): boolean { + const db = new Database(dbFile, { timeout: 5000 }); + try { + const r = db.prepare('UPDATE settings SET config_log_level = ? WHERE config_log_level <= ?') + .run(CALIBRE_INFO, CALIBRE_DEBUG_MAX); + return r.changes > 0; + } finally { + db.close(); + } +} + +function rewrite(file: string, edit: (text: string) => string | undefined): boolean { + if (!fs.existsSync(file)) return false; // the service is not on the box + const next = edit(fs.readFileSync(file, 'utf8')); + if (next === undefined) return false; + writeAtomic(file, next); + return true; +} + +function writeMarker(marker: string, data: string): void { + const tmp = marker + '.k2go-tmp'; + fs.writeFileSync(tmp, data); + fs.renameSync(tmp, marker); +} + +export function applyLogLessOnce( + t: LogLessTargets, marker: string, calibreInfo: (dbFile: string) => boolean = sqliteCalibreInfo, +): { ran: boolean; changed: LogLessStep[] } { + if (fs.existsSync(marker)) return { ran: false, changed: [] }; + const changed: LogLessStep[] = []; + if (rewrite(t.nginxConf, nginxWithoutClientIds)) changed.push('nginx'); + if (rewrite(t.forgejoIni, forgejoWithoutRouterLog)) changed.push('forgejo'); + if (fs.existsSync(t.calibreAppDb) && calibreInfo(t.calibreAppDb)) changed.push('calibre-web'); + writeMarker(marker, JSON.stringify({ at: new Date().toISOString(), changed }) + '\n'); + return { ran: true, changed }; +} + +// --- Remove the logs written before the change ---------------------------------------------------- +// They hold client IPs, User-Agents (nginx, Forgejo router) or DEBUG session data (Calibre-Web). +// Only a service that the step above changed is purged, and only when the new config is in effect: +// its process started after the change, or it does not run. Then the rotated copies from before the +// change are deleted and the live log is emptied in place (the writers append, as with logrotate +// copytruncate). Owner of the marker: dash-node, at startup; while a changed service still runs the +// old config, the next start tries again. +export const LOG_PURGE_MARKER = process.env.K2GO_LOG_PURGE_MARKER || '/library/dashboard/log-less-purge.done'; + +/** Per service: the log that held client IDs (rotated copies are `.N[.gz]`) and the `pgrep -f` + * pattern of its process. Paths: tools/setup-proot-logging.sh and the IIAB pdsm service wrappers. */ +export interface ServiceLog { log: string; process: string } + +export const SERVICE_LOGS: Record = { + nginx: { log: process.env.K2GO_NGINX_ACCESS_LOG || '/var/log/nginx/access.log', process: 'nginx: master' }, + forgejo: { log: process.env.K2GO_FORGEJO_LOG || '/library/forgejo/log/forgejo.log', process: 'forgejo web' }, + 'calibre-web': { log: process.env.K2GO_CALIBRE_LOG || '/var/log/calibre-web.log', process: 'calibre-web-py3/cps.py' }, +}; + +/** Rotated copies of `log` (`.N`, `.N.gz`) last written before `beforeMs`. */ +export function rotatedBefore(log: string, beforeMs: number): string[] { + const dir = path.dirname(log); + const base = path.basename(log); + let names: string[]; + try { names = fs.readdirSync(dir); } catch { return []; } + return names + .filter((n) => n.startsWith(base + '.') && /^\d+(\.gz)?$/.test(n.slice(base.length + 1))) + .map((n) => path.join(dir, n)) + .filter((f) => { try { return fs.statSync(f).mtimeMs < beforeMs; } catch { return false; } }); +} + +export function purgeOldLogsOnce( + marker: string, purgeMarker: string, logs: Record = SERVICE_LOGS, + startMs: (pattern: string) => number | null = processStartMs, +): { ran: boolean; purged: LogLessStep[]; waiting: LogLessStep[] } { + if (fs.existsSync(purgeMarker) || !fs.existsSync(marker)) return { ran: false, purged: [], waiting: [] }; + const changeMs = fs.statSync(marker).mtimeMs; + let changed: LogLessStep[] = []; + try { changed = JSON.parse(fs.readFileSync(marker, 'utf8')).changed || []; } catch { /* nothing to purge */ } + const purged: LogLessStep[] = []; + const waiting: LogLessStep[] = []; + for (const svc of changed) { + const s = logs[svc]; + if (!s) continue; + const started = startMs(s.process); + if (started !== null && started <= changeMs) { waiting.push(svc); continue; } // old config runs + // Until the restart the old config wrote the log, so a copy rotated after the change can + // still hold IDs: the cutoff is the restart (or now, when the service does not run). + for (const f of rotatedBefore(s.log, started ?? Date.now())) fs.rmSync(f, { force: true }); + if (fs.existsSync(s.log)) fs.truncateSync(s.log, 0); + purged.push(svc); + } + if (waiting.length === 0) writeMarker(purgeMarker, new Date().toISOString() + '\n'); + return { ran: true, purged, waiting }; +} diff --git a/static/dashboard/sockets/privacy.ts b/static/dashboard/sockets/privacy.ts index 6d014a562..1d8095e25 100644 --- a/static/dashboard/sockets/privacy.ts +++ b/static/dashboard/sockets/privacy.ts @@ -171,7 +171,7 @@ export function startEpochMs(ticks: number, selfTicks: number, selfStartMs: numb } /** Start time of the newest process that matches `pattern`, or null when none runs. */ -function processStartMs(pattern: string): number | null { +export function processStartMs(pattern: string): number | null { try { const r = spawnSync('pgrep', ['-f', pattern], { encoding: 'utf8', timeout: 3000 }); if (r.status !== 0 || !r.stdout) return null; @@ -194,7 +194,7 @@ function processStartMs(pattern: string): number | null { } /** Same mode and owner as the file it replaces; write + fsync + rename, so a crash leaves the old file. */ -function writeAtomic(path: string, data: string): void { +export function writeAtomic(path: string, data: string): void { const file = fs.realpathSync(path); // a symlinked config keeps its link: we replace the target const st = fs.statSync(file); const tmp = file + '.k2go-tmp'; diff --git a/tools/upstream-patches/patches/0010-nginx-access-log-no-ip-ua.patch b/tools/upstream-patches/patches/0010-nginx-access-log-no-ip-ua.patch new file mode 100644 index 000000000..4bd4ecf1b --- /dev/null +++ b/tools/upstream-patches/patches/0010-nginx-access-log-no-ip-ua.patch @@ -0,0 +1,24 @@ +Upstream-PR: not yet submitted +Upstream-Status: open +Applies-to: roles/nginx/templates/nginx.conf.j2 +Summary: Drop the client IP and the User-Agent from the nginx access log; the fields stay as "-" so awstats still parses it (K2GO-559). + +diff --git a/roles/nginx/templates/nginx.conf.j2 b/roles/nginx/templates/nginx.conf.j2 +index f4b9ec17f..b47dcb244 100644 +--- a/roles/nginx/templates/nginx.conf.j2 ++++ b/roles/nginx/templates/nginx.conf.j2 +@@ -43,10 +43,12 @@ http { + # Logging Settings + ## + ++ # No client IP and no User-Agent: the access log must not identify who used the server. ++ # The fields stay in place ("-") so awstats still parses the log (no visitor or browser counts). + log_format awstats +- '$remote_addr - $remote_user [$time_local] "$request" ' ++ '- - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' +- '"$http_user_agent" "http_x_forwarded_for"'; ++ '"-" "http_x_forwarded_for"'; + + access_log {{ nginx_log_dir }}/access.log awstats; + diff --git a/tools/upstream-patches/patches/0011-forgejo-router-log-off.patch b/tools/upstream-patches/patches/0011-forgejo-router-log-off.patch new file mode 100644 index 000000000..aef0eb28c --- /dev/null +++ b/tools/upstream-patches/patches/0011-forgejo-router-log-off.patch @@ -0,0 +1,15 @@ +Upstream-PR: not yet submitted +Upstream-Status: open +Applies-to: roles/forgejo/templates/app.ini.j2 +Summary: Turn off the Forgejo router log (LOGGER_ROUTER_MODE empty); it records the client address of each request (K2GO-559). + +diff --git a/roles/forgejo/templates/app.ini.j2 b/roles/forgejo/templates/app.ini.j2 +index 643fff769..d1eae8d09 100644 +--- a/roles/forgejo/templates/app.ini.j2 ++++ b/roles/forgejo/templates/app.ini.j2 +@@ -85,3 +85,5 @@ ENABLED = false + ROOT_PATH = {{ forgejo_log_root }} + MODE = console + LEVEL = Info ++; No router log: it records the client address of each request. ++LOGGER_ROUTER_MODE = diff --git a/tools/upstream-patches/patches/0012-calibre-web-log-level-info.patch b/tools/upstream-patches/patches/0012-calibre-web-log-level-info.patch new file mode 100644 index 000000000..551bbe28a --- /dev/null +++ b/tools/upstream-patches/patches/0012-calibre-web-log-level-info.patch @@ -0,0 +1,20 @@ +Upstream-PR: not yet submitted +Upstream-Status: open +Applies-to: roles/calibre-web/tasks/install.yml +Summary: Set the Calibre-Web log level to INFO in a newly provisioned app.db; the shipped app.db is DEBUG, which logs session data (K2GO-559). + +diff --git a/roles/calibre-web/tasks/install.yml b/roles/calibre-web/tasks/install.yml +index 452248da5..964b48c8e 100644 +--- a/roles/calibre-web/tasks/install.yml ++++ b/roles/calibre-web/tasks/install.yml +@@ -207,6 +207,10 @@ + backup: yes + when: not appdb.stat.exists + ++- name: Set the Calibre-Web log level to INFO in the new app.db (the shipped app.db is DEBUG, which logs session data) ++ command: python3 -c "import sqlite3, sys; c = sqlite3.connect(sys.argv[1]); c.execute('UPDATE settings SET config_log_level = 20 WHERE config_log_level < 20'); c.commit()" {{ calibreweb_config }}/app.db ++ when: not appdb.stat.exists ++ + + # RECORD Calibre-Web AS INSTALLED +