Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions docker/services/logs.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,20 @@ import { mkdirSync, statSync, renameSync, writeFileSync, appendFileSync, readFil
import path from "node:path";
import { StringDecoder } from "node:string_decoder";
import { ServiceError } from './store.mjs';
import { maskValues } from './redact.mjs';
const LIMIT = 256 * 1024;
export function logWriter(root, id, runId, stream, masks = []) {
const dir = path.join(root, "logs", id);
mkdirSync(dir, { recursive: true, mode: 0o700 });
const file = path.join(dir, stream + ".jsonl"), decoder = new StringDecoder("utf8");
let buffered = "", pending = '', sequence = 0;
const redact = value => masks.filter(Boolean).sort((a, b) => b.length - a.length).reduce((result, mask) => result.split(mask).join("[redacted]"), value);
// The same encoded forms the Vault hides in command output: a service that
// prints `echo $TOKEN | base64` must not leave the value in its log.
// Preserve the old literal masking of short service passwords too. The
// external service owns its password policy; saved values must stay usable.
const values = maskValues(masks, { minimumLength: 1 });
const pattern = values.length ? new RegExp(values.map(value => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|'), 'g') : null;
const redact = value => pattern ? value.replace(pattern, '[redacted]') : value;
const line = text => {
const record = JSON.stringify({ at: Date.now(), runId, stream, sequence: ++sequence, text: text.slice(0, 8192) }) + "\n";
try {
Expand All @@ -19,12 +26,12 @@ export function logWriter(root, id, runId, stream, masks = []) {
const drain = final => {
// Redact before splitting lines, including multiline values and values
// split across writes. Keep enough raw suffix for the next chunk.
const overlap = Math.max(1, ...masks.map(mask => mask.length));
const overlap = Math.max(1, ...values.map(value => value.length));
let cut = final ? pending.length : Math.max(0, pending.length - overlap + 1);
let changed;
do {
changed = false;
for (const mask of masks.filter(Boolean)) {
for (const mask of values) {
const at = pending.lastIndexOf(mask, cut - 1);
if (at >= 0 && at < cut && at + mask.length > cut) { cut = at; changed = true; }
}
Expand Down
6 changes: 6 additions & 0 deletions docker/services/redact.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
// Services and Setup ship in sibling /opt/t3-* directories in the image.
// Share the Vault's variants while also supporting the source-tree layout.
import { existsSync } from 'node:fs';
const source = new URL('../setup/credentials/redact.mjs', import.meta.url);
const installed = new URL('../t3-setup/credentials/redact.mjs', import.meta.url);
export const { maskValues } = await import(existsSync(source) ? source.href : installed.href);
6 changes: 3 additions & 3 deletions docker/setup/credentials/redact.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,18 @@ const MIN_LINE = 16;

// The forms a value commonly takes in output. A file's long lines count as well,
// so printing part of a private key is caught too.
export function maskValues(secrets) {
export function maskValues(secrets, { minimumLength = MIN_LENGTH } = {}) {
const values = new Set();
for (const secret of [secrets].flat()) {
if (typeof secret !== 'string' || secret.length < MIN_LENGTH) continue;
if (typeof secret !== 'string' || secret.length < minimumLength) continue;
values.add(secret); values.add(secret.trim());
values.add(JSON.stringify(secret).slice(1, -1));
values.add(encodeURIComponent(secret));
values.add(new URLSearchParams([['', secret]]).toString().slice(1));
values.add(Buffer.from(secret).toString('base64'));
if (secret.includes('\n')) for (const line of secret.split(/\r?\n/)) if (line.trim().length >= MIN_LINE) values.add(line.trim());
}
return [...values].filter(value => value.length >= MIN_LENGTH).sort((a, b) => b.length - a.length);
return [...values].filter(value => value.length >= minimumLength).sort((a, b) => b.length - a.length);
}

function filterValues(values, write) {
Expand Down
40 changes: 40 additions & 0 deletions tests/services-redaction.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import * as fs from 'node:fs/promises';
import path from 'node:path';
import os from 'node:os';
import {pathToFileURL} from 'node:url';
import {normalize} from '../docker/setup/credentials/model.mjs';
import {logWriter,readLogs} from '../docker/services/logs.mjs';

test('valid existing short credentials remain readable and editable',()=>{
for(const token of ['abc','sk-one','1234567']){
const saved=normalize({id:'existing',type:'token',secrets:{token}});
assert.equal(normalize({...saved,description:'edited'},saved).secrets.token,token);
}
});

test('service logs redact short and encoded secrets across every split',async t=>{
const root=await fs.mkdtemp(path.join(os.tmpdir(),'t3-services-redaction-'));
t.after(()=>fs.rm(root,{recursive:true,force:true}));
for(const secret of ['abc','a','p@ss word','quote"\\line\nnext']){
const forms=[secret,Buffer.from(secret).toString('base64'),encodeURIComponent(secret),new URLSearchParams([['',secret]]).toString().slice(1),JSON.stringify(secret).slice(1,-1)];
for(const value of new Set(forms))for(let at=0;at<=value.length;at++){
const writer=logWriter(root,'test','run','stdout',[secret]);
writer.write(Buffer.from(value.slice(0,at)));writer.write(Buffer.from(value.slice(at)+'\n'));writer.end();
assert.equal(readLogs(root,'test').entries.at(-1).text,'[redacted]',`split ${at} for ${JSON.stringify(secret)}`);
}
}
});

test('Services imports and masks logs in the installed image directory layout',async t=>{
const root=await fs.mkdtemp(path.join(os.tmpdir(),'t3-services-image-layout-'));
t.after(()=>fs.rm(root,{recursive:true,force:true}));
await fs.cp(new URL('../docker/services/',import.meta.url),path.join(root,'t3-services'),{recursive:true});
const setup=path.join(root,'t3-setup','credentials');await fs.mkdir(setup,{recursive:true});
await fs.copyFile(new URL('../docker/setup/credentials/redact.mjs',import.meta.url),path.join(setup,'redact.mjs'));
const {logWriter:writer,readLogs:logs}=await import(pathToFileURL(path.join(root,'t3-services','logs.mjs')));
const output=writer(path.join(root,'logs'),'demo','run','stdout',['private-token']);
output.write(Buffer.from(Buffer.from('private-token').toString('base64')+'\n'));output.end();
assert.equal(logs(path.join(root,'logs'),'demo').entries[0].text,'[redacted]');
});
11 changes: 11 additions & 0 deletions tests/services.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,17 @@ test("logs redact split secrets, remain bounded and separate stdout from stderr"
const dir = path.join(f.root, "logs/preview");
for (const file of await fs.readdir(dir)) assert.ok((await fs.stat(path.join(dir, file))).size <= 256 * 1024);
});
test("logs hide the encoded forms of a secret, even split across writes", async t => {
const f = await fixture(t), secret = "p@ss w0rd/val+ue";
const writer = logWriter(f.root, "preview", "run", "stdout", [secret]);
const base64 = Buffer.from(secret).toString("base64"), url = encodeURIComponent(secret);
const form = new URLSearchParams([["", secret]]).toString().slice(1);
writer.write(Buffer.from("a " + base64.slice(0, 10))); writer.write(Buffer.from(base64.slice(10) + " b " + url.slice(0, 8)));
writer.write(Buffer.from(url.slice(8) + " c " + form + " d " + secret + "\n")); writer.end();
const text = readLogs(f.root, "preview").entries.map(entry => entry.text).join("\n");
for (const value of [secret, base64, url, form]) assert.ok(!text.includes(value), value);
assert.equal(text.match(/\[redacted\]/g).length, 4);
});
test("disabled registrations retain exact tool and Vault dependency references", async t => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "t3-services-refs-"));
t.after(() => fs.rm(root, { recursive: true, force: true }));
Expand Down
Loading