diff --git a/docker/services/logs.mjs b/docker/services/logs.mjs index 5a66935..2d07010 100644 --- a/docker/services/logs.mjs +++ b/docker/services/logs.mjs @@ -2,13 +2,20 @@ import { mkdirSync, statSync, renameSync, writeFileSync, appendFileSync, readFil import path from "node:path"; import { StringDecoder } from "node:string_decoder"; import { ServiceError } from './store.mjs'; +import { maskValues } from './redact.mjs'; const LIMIT = 256 * 1024; export function logWriter(root, id, runId, stream, masks = []) { const dir = path.join(root, "logs", id); mkdirSync(dir, { recursive: true, mode: 0o700 }); const file = path.join(dir, stream + ".jsonl"), decoder = new StringDecoder("utf8"); let buffered = "", pending = '', sequence = 0; - const redact = value => masks.filter(Boolean).sort((a, b) => b.length - a.length).reduce((result, mask) => result.split(mask).join("[redacted]"), value); + // The same encoded forms the Vault hides in command output: a service that + // prints `echo $TOKEN | base64` must not leave the value in its log. + // Preserve the old literal masking of short service passwords too. The + // external service owns its password policy; saved values must stay usable. + const values = maskValues(masks, { minimumLength: 1 }); + const pattern = values.length ? new RegExp(values.map(value => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|'), 'g') : null; + const redact = value => pattern ? value.replace(pattern, '[redacted]') : value; const line = text => { const record = JSON.stringify({ at: Date.now(), runId, stream, sequence: ++sequence, text: text.slice(0, 8192) }) + "\n"; try { @@ -19,12 +26,12 @@ export function logWriter(root, id, runId, stream, masks = []) { const drain = final => { // Redact before splitting lines, including multiline values and values // split across writes. Keep enough raw suffix for the next chunk. - const overlap = Math.max(1, ...masks.map(mask => mask.length)); + const overlap = Math.max(1, ...values.map(value => value.length)); let cut = final ? pending.length : Math.max(0, pending.length - overlap + 1); let changed; do { changed = false; - for (const mask of masks.filter(Boolean)) { + for (const mask of values) { const at = pending.lastIndexOf(mask, cut - 1); if (at >= 0 && at < cut && at + mask.length > cut) { cut = at; changed = true; } } diff --git a/docker/services/redact.mjs b/docker/services/redact.mjs new file mode 100644 index 0000000..fa31adc --- /dev/null +++ b/docker/services/redact.mjs @@ -0,0 +1,6 @@ +// Services and Setup ship in sibling /opt/t3-* directories in the image. +// Share the Vault's variants while also supporting the source-tree layout. +import { existsSync } from 'node:fs'; +const source = new URL('../setup/credentials/redact.mjs', import.meta.url); +const installed = new URL('../t3-setup/credentials/redact.mjs', import.meta.url); +export const { maskValues } = await import(existsSync(source) ? source.href : installed.href); diff --git a/docker/setup/credentials/redact.mjs b/docker/setup/credentials/redact.mjs index 0ddcb0c..738c655 100644 --- a/docker/setup/credentials/redact.mjs +++ b/docker/setup/credentials/redact.mjs @@ -6,10 +6,10 @@ const MIN_LINE = 16; // The forms a value commonly takes in output. A file's long lines count as well, // so printing part of a private key is caught too. -export function maskValues(secrets) { +export function maskValues(secrets, { minimumLength = MIN_LENGTH } = {}) { const values = new Set(); for (const secret of [secrets].flat()) { - if (typeof secret !== 'string' || secret.length < MIN_LENGTH) continue; + if (typeof secret !== 'string' || secret.length < minimumLength) continue; values.add(secret); values.add(secret.trim()); values.add(JSON.stringify(secret).slice(1, -1)); values.add(encodeURIComponent(secret)); @@ -17,7 +17,7 @@ export function maskValues(secrets) { values.add(Buffer.from(secret).toString('base64')); if (secret.includes('\n')) for (const line of secret.split(/\r?\n/)) if (line.trim().length >= MIN_LINE) values.add(line.trim()); } - return [...values].filter(value => value.length >= MIN_LENGTH).sort((a, b) => b.length - a.length); + return [...values].filter(value => value.length >= minimumLength).sort((a, b) => b.length - a.length); } function filterValues(values, write) { diff --git a/tests/services-redaction.test.mjs b/tests/services-redaction.test.mjs new file mode 100644 index 0000000..0538278 --- /dev/null +++ b/tests/services-redaction.test.mjs @@ -0,0 +1,40 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import * as fs from 'node:fs/promises'; +import path from 'node:path'; +import os from 'node:os'; +import {pathToFileURL} from 'node:url'; +import {normalize} from '../docker/setup/credentials/model.mjs'; +import {logWriter,readLogs} from '../docker/services/logs.mjs'; + +test('valid existing short credentials remain readable and editable',()=>{ + for(const token of ['abc','sk-one','1234567']){ + const saved=normalize({id:'existing',type:'token',secrets:{token}}); + assert.equal(normalize({...saved,description:'edited'},saved).secrets.token,token); + } +}); + +test('service logs redact short and encoded secrets across every split',async t=>{ + const root=await fs.mkdtemp(path.join(os.tmpdir(),'t3-services-redaction-')); + t.after(()=>fs.rm(root,{recursive:true,force:true})); + for(const secret of ['abc','a','p@ss word','quote"\\line\nnext']){ + const forms=[secret,Buffer.from(secret).toString('base64'),encodeURIComponent(secret),new URLSearchParams([['',secret]]).toString().slice(1),JSON.stringify(secret).slice(1,-1)]; + for(const value of new Set(forms))for(let at=0;at<=value.length;at++){ + const writer=logWriter(root,'test','run','stdout',[secret]); + writer.write(Buffer.from(value.slice(0,at)));writer.write(Buffer.from(value.slice(at)+'\n'));writer.end(); + assert.equal(readLogs(root,'test').entries.at(-1).text,'[redacted]',`split ${at} for ${JSON.stringify(secret)}`); + } + } +}); + +test('Services imports and masks logs in the installed image directory layout',async t=>{ + const root=await fs.mkdtemp(path.join(os.tmpdir(),'t3-services-image-layout-')); + t.after(()=>fs.rm(root,{recursive:true,force:true})); + await fs.cp(new URL('../docker/services/',import.meta.url),path.join(root,'t3-services'),{recursive:true}); + const setup=path.join(root,'t3-setup','credentials');await fs.mkdir(setup,{recursive:true}); + await fs.copyFile(new URL('../docker/setup/credentials/redact.mjs',import.meta.url),path.join(setup,'redact.mjs')); + const {logWriter:writer,readLogs:logs}=await import(pathToFileURL(path.join(root,'t3-services','logs.mjs'))); + const output=writer(path.join(root,'logs'),'demo','run','stdout',['private-token']); + output.write(Buffer.from(Buffer.from('private-token').toString('base64')+'\n'));output.end(); + assert.equal(logs(path.join(root,'logs'),'demo').entries[0].text,'[redacted]'); +}); diff --git a/tests/services.test.mjs b/tests/services.test.mjs index e592023..7bbfde2 100644 --- a/tests/services.test.mjs +++ b/tests/services.test.mjs @@ -169,6 +169,17 @@ test("logs redact split secrets, remain bounded and separate stdout from stderr" const dir = path.join(f.root, "logs/preview"); for (const file of await fs.readdir(dir)) assert.ok((await fs.stat(path.join(dir, file))).size <= 256 * 1024); }); +test("logs hide the encoded forms of a secret, even split across writes", async t => { + const f = await fixture(t), secret = "p@ss w0rd/val+ue"; + const writer = logWriter(f.root, "preview", "run", "stdout", [secret]); + const base64 = Buffer.from(secret).toString("base64"), url = encodeURIComponent(secret); + const form = new URLSearchParams([["", secret]]).toString().slice(1); + writer.write(Buffer.from("a " + base64.slice(0, 10))); writer.write(Buffer.from(base64.slice(10) + " b " + url.slice(0, 8))); + writer.write(Buffer.from(url.slice(8) + " c " + form + " d " + secret + "\n")); writer.end(); + const text = readLogs(f.root, "preview").entries.map(entry => entry.text).join("\n"); + for (const value of [secret, base64, url, form]) assert.ok(!text.includes(value), value); + assert.equal(text.match(/\[redacted\]/g).length, 4); +}); test("disabled registrations retain exact tool and Vault dependency references", async t => { const root = await fs.mkdtemp(path.join(os.tmpdir(), "t3-services-refs-")); t.after(() => fs.rm(root, { recursive: true, force: true }));