diff --git a/README.md b/README.md index 6ec13378..9ea14b25 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ A self-hosted online file converter. Supports over a thousand different formats. - Process multiple files at once - Password protection - Multiple accounts +- A REST API for scripts and automation, documented with OpenAPI ## Converters supported @@ -142,6 +143,38 @@ Tutorial in chinese: Tutorial in polish: +## API + +ConvertX has a REST API at `/api/v1`, so scripts and tools like n8n can convert files without the web interface. If you set `WEBROOT`, the API is under it: with `WEBROOT=/convert`, it's at `/convert/api/v1`. + +1. In ConvertX, open **Account → API keys** and create a key. It's shown once, so copy it. +2. Send it as `Authorization: Bearer `. + +Convert one file and save the result: + +```bash +curl --fail -H "Authorization: Bearer $CONVERTX_KEY" \ + -F file=@report.docx -F to=pdf \ + -o report.pdf \ + http://localhost:3000/api/v1/convert +``` + +With `--fail`, curl exits with an error instead of saving an error message as `report.pdf`. Leave it out to see the message: errors are JSON, with an `error` code and a `message`. + +For several files, or conversions that take a while, start a job and follow it: + +```bash +curl -H "Authorization: Bearer $CONVERTX_KEY" \ + -F files=@one.png -F files=@two.png -F to=webp \ + http://localhost:3000/api/v1/jobs +# → {"id": 12, "status": "pending", "files": [...]} + +curl -H "Authorization: Bearer $CONVERTX_KEY" http://localhost:3000/api/v1/jobs/12 +# → each file's status, and a download URL for each result +``` + +`GET /api/v1/targets?from=docx` lists what a format can be converted to. The full reference, which you can try out in the browser, is at `/api/v1/docs`. + ## Screenshots ![ConvertX Preview](images/preview.png) diff --git a/bun.lock b/bun.lock index 21f91118..35832904 100644 --- a/bun.lock +++ b/bun.lock @@ -7,6 +7,7 @@ "dependencies": { "@elysiajs/html": "^1.4.2", "@elysiajs/jwt": "^1.4.2", + "@elysiajs/openapi": "1.4.16", "@elysiajs/static": "^1.4.10", "@kitajs/html": "^4.2.13", "elysia": "1.4.32", @@ -52,6 +53,8 @@ "@elysiajs/jwt": ["@elysiajs/jwt@1.4.2", "", { "dependencies": { "jose": "^6.0.11" }, "peerDependencies": { "elysia": ">= 1.4.27" } }, "sha512-gSPjF8KoL7cQIxL3kb+Ui9g5jpBdO4iUTe2+F52w66JVFct2K8cTLum8PEArxRZNg4PBrg100h6i1bJ2zfcqMg=="], + "@elysiajs/openapi": ["@elysiajs/openapi@1.4.16", "", { "peerDependencies": { "@sinclair/typebox": ">= 0.34.0 < 1", "elysia": ">= 1.4.0", "typescript": ">= 5.0.0" } }, "sha512-kj8RBffE4J6myjm7nPPj6Dw1ujWt45NkdGW5sG9u3T0OfUzKN71s630/LEjajGo5jRZqFF3VHyk40lwXH2AbmQ=="], + "@elysiajs/static": ["@elysiajs/static@1.4.12", "", { "peerDependencies": { "elysia": ">= 1.4.0" } }, "sha512-u9KnwjG+ZX6lOg95QudHqphBvHv8DiqY3/AYrvEDjTcJ1HH/innCwo/ssAgF2LZ66bKBkt7MrXPk+GZgDC2XZw=="], "@emnapi/core": ["@emnapi/core@1.11.2", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" } }, "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA=="], diff --git a/package.json b/package.json index 209a2f4e..f34f81dc 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "dependencies": { "@elysiajs/html": "^1.4.2", "@elysiajs/jwt": "^1.4.2", + "@elysiajs/openapi": "1.4.16", "@elysiajs/static": "^1.4.10", "@kitajs/html": "^4.2.13", "elysia": "1.4.32", diff --git a/src/api/index.ts b/src/api/index.ts new file mode 100644 index 00000000..9c3270a1 --- /dev/null +++ b/src/api/index.ts @@ -0,0 +1,417 @@ +import { openapi } from "@elysiajs/openapi"; +import { Elysia, t } from "elysia"; + +import { getAllInputs, getAllTargets, getPossibleTargets } from "../converters/main"; +import { WEBROOT } from "../helpers/env"; +import { normalizeFiletype } from "../helpers/normalizeFiletype"; +import { + ApiError, + archiveJob, + chooseConverter, + cleanTarget, + createJob, + deleteJob, + describeJob, + listJobs, + outputFile, +} from "./jobs"; +import { userForApiKey } from "./keys"; + +const API_ROOT = `${WEBROOT}/api/v1`; + +const ErrorBody = t.Object( + { + error: t.String({ description: "A stable code, such as `job_not_found`." }), + message: t.String({ description: "What went wrong, for people." }), + }, + { title: "Error" }, +); + +const JobFile = t.Object( + { + input: t.String({ description: "The uploaded file's name.", examples: ["report.docx"] }), + status: t.Union([t.Literal("converting"), t.Literal("done"), t.Literal("failed")]), + error: t.Optional(t.String({ description: "The converter's message when it failed." })), + outputs: t.Array( + t.Object({ + name: t.String({ examples: ["report.pdf"] }), + url: t.String({ description: "Where to download it, with the same API key." }), + }), + { description: "Converted files. Converting a PDF to images makes one per page." }, + ), + }, + { title: "JobFile" }, +); + +const Job = t.Object( + { + id: t.Number(), + status: t.Union([t.Literal("pending"), t.Literal("completed"), t.Literal("failed")], { + description: + "`pending` until every file is converted, then `completed`, or `failed` if any file failed. Each file has its own status.", + }), + createdAt: t.String({ format: "date-time" }), + files: t.Array(JobFile), + }, + { title: "Job" }, +); + +const JobSummary = t.Object( + { + id: t.Number(), + status: t.String(), + createdAt: t.String({ format: "date-time" }), + files: t.Number({ description: "How many files were uploaded." }), + }, + { title: "JobSummary" }, +); + +const unauthorized = { + error: "unauthorized", + message: + "Send an API key as `Authorization: Bearer `. Create one in ConvertX under Account → API keys.", +}; + +function escapeRegExp(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * A file as a download. A plain Response passes through Elysia untouched, unlike its file() helper, + * which the minified build in the Docker image sends as "[object Object]". + */ +function download(path: string, name: string, headers: Record = {}): Response { + return new Response(Bun.file(path), { + headers: { + "content-disposition": `attachment; filename*=UTF-8''${encodeURIComponent(name)}`, + ...headers, + }, + }); +} + +/** Turns one file or several from a multipart form into a list. */ +function asList(files: File | File[]): File[] { + return Array.isArray(files) ? files : [files]; +} + +/** An error in the API's `{ error, message }` shape. */ +function errorResponse(status: number, error: string, message: string): Response { + return Response.json({ error, message }, { status }); +} + +/** + * The REST API at /api/v1, for scripts and automation. Authenticated with API keys, documented with + * OpenAPI at /api/v1/docs. + */ +export const api = new Elysia({ name: "api", prefix: "/api/v1" }) + .use( + openapi({ + path: "/docs", + specPath: "/openapi.json", + provider: "swagger-ui", + // Only the API is documented, not the web pages. + exclude: { paths: [new RegExp(`^(?!${escapeRegExp(API_ROOT)}/)`)], staticFile: true }, + documentation: { + info: { + title: "ConvertX API", + version: "1", + description: + "Convert files between formats from scripts and automation tools. Create an API key in ConvertX under **Account → API keys** and send it as `Authorization: Bearer `.\n\n" + + "For one file, `POST /convert` uploads it and returns the result. For several, or for long conversions, `POST /jobs` starts a job you can follow with `GET /jobs/{id}`.", + }, + components: { + securitySchemes: { + apiKey: { type: "http", scheme: "bearer", description: "An API key, starting `cvx_`." }, + }, + }, + security: [{ apiKey: [] }], + tags: [ + { name: "Formats", description: "What can be converted into what." }, + { name: "Conversion", description: "Converting files." }, + ], + }, + }), + ) + .macro("apiKey", { + // Headers come from `request`: Elysia decides whether to parse `headers` by reading + // this function's source, which the minified build in the Docker image hides from it. + async resolve({ request, status }) { + const authorization = request.headers.get("authorization") ?? ""; + const key = /^bearer\s+/i.test(authorization) + ? authorization.replace(/^bearer\s+/i, "").trim() + : request.headers.get("x-api-key"); + const user = key ? userForApiKey(key) : null; + if (!user) return status(401, unauthorized); + return { user }; + }, + }) + // Every API error is JSON in the shape the docs describe, including Elysia's own validation and + // parse errors. The app's error handler is registered after the API, so it never sees these. + .onError(({ code, error }) => { + if (error instanceof ApiError) return error.toResponse(); + if (code === "VALIDATION") { + const field = error.valueError?.path.slice(1); + return errorResponse( + 400, + "invalid_request", + `${field ? `"${field}"` : "The request"} is missing or not valid. ${API_ROOT}/docs describes what each endpoint takes.`, + ); + } + if (code === "PARSE") { + return errorResponse(400, "invalid_request", "The request body couldn't be read."); + } + console.error(error); + return errorResponse( + 500, + "internal_error", + "Something went wrong. The server log has details.", + ); + }) + .get( + "/converters", + () => ({ + converters: Object.entries(getAllTargets()).map(([name, to]) => ({ + name, + from: [...new Set(getAllInputs(name))].sort(), + to: [...new Set(to)].sort(), + })), + }), + { + apiKey: true, + response: { + 200: t.Object({ + converters: t.Array( + t.Object({ name: t.String(), from: t.Array(t.String()), to: t.Array(t.String()) }), + ), + }), + 401: ErrorBody, + }, + detail: { + summary: "List converters", + description: "Every converter and the formats it reads and writes.", + tags: ["Formats"], + }, + }, + ) + .get( + "/targets", + ({ query }) => { + const from = normalizeFiletype(query.from.replace(/^\./, "")); + const byFormat = new Map(); + for (const [converter, targets] of Object.entries(getPossibleTargets(from))) { + // A converter can list a format twice, such as LibreOffice's pdf. + for (const target of new Set(targets)) + byFormat.set(target, [...(byFormat.get(target) ?? []), converter]); + } + return { + from, + targets: [...byFormat] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([format, converters]) => ({ format, converters })), + }; + }, + { + apiKey: true, + query: t.Object({ + from: t.String({ description: "A file extension, such as `docx`.", examples: ["docx"] }), + }), + response: { + 200: t.Object({ + from: t.String(), + targets: t.Array(t.Object({ format: t.String(), converters: t.Array(t.String()) })), + }), + 400: ErrorBody, + 401: ErrorBody, + }, + detail: { + summary: "List what a format can become", + description: + "The formats a file of this type can be converted to, and which converters can do it.", + tags: ["Formats"], + }, + }, + ) + .post( + "/convert", + async ({ body, user }) => { + const to = cleanTarget(body.to); + const converter = chooseConverter([body.file.name], to, body.converter); + const { id, done } = await createJob(user.id, [body.file], to, converter); + await done; + const job = describeJob(user.id, id, API_ROOT); + const converted = job.files[0]; + if (!converted || converted.status !== "done") { + return Response.json( + { error: "conversion_failed", message: converted?.error ?? "The conversion failed." }, + { status: 422, headers: { "x-convertx-job": String(id) } }, + ); + } + // Several outputs, such as one image per PDF page, come back as a tar. + const single = converted.outputs.length === 1 ? converted.outputs[0] : undefined; + const path = single + ? await outputFile(user.id, id, single.name) + : await archiveJob(user.id, id); + const name = single ? single.name : `converted_files_${id}.tar`; + return download(path, name, { "x-convertx-job": String(id) }); + }, + { + apiKey: true, + body: t.Object({ + file: t.File({ description: "The file to convert." }), + to: t.String({ + description: "The format to convert to, such as `pdf`.", + examples: ["pdf"], + }), + converter: t.Optional( + t.String({ description: "A converter to use. By default, the first that can do it." }), + ), + }), + response: { 200: t.File(), 400: ErrorBody, 401: ErrorBody, 422: ErrorBody }, + detail: { + summary: "Convert a file", + description: + "Uploads one file and returns the converted file once it's ready. The `X-ConvertX-Job` header gives the job, which stays in the history like any other.", + tags: ["Conversion"], + }, + }, + ) + .post( + "/jobs", + async ({ body, user, set }) => { + const files = asList(body.files); + const to = cleanTarget(body.to); + const converter = chooseConverter( + files.map((file) => file.name), + to, + body.converter, + ); + const { id } = await createJob(user.id, files, to, converter); + set.status = 202; + set.headers.location = `${API_ROOT}/jobs/${id}`; + return describeJob(user.id, id, API_ROOT); + }, + { + apiKey: true, + body: t.Object({ + files: t.Files({ description: "One or more files, all converted to the same format." }), + to: t.String({ + description: "The format to convert to, such as `pdf`.", + examples: ["pdf"], + }), + converter: t.Optional( + t.String({ + description: "A converter to use. By default, the first that handles every file.", + }), + ), + }), + response: { 202: Job, 400: ErrorBody, 401: ErrorBody, 422: ErrorBody }, + detail: { + summary: "Start a conversion job", + description: + "Uploads files and starts converting them in the background. Follow the job with `GET /jobs/{id}`.", + tags: ["Conversion"], + }, + }, + ) + .get( + "/jobs", + ({ query, user }) => { + const limit = query.limit ?? 20; + const offset = query.offset ?? 0; + const { jobs, total } = listJobs(user.id, limit, offset); + return { + total, + jobs: jobs.map((job) => ({ + id: job.id, + status: job.status, + createdAt: job.date_created, + files: job.num_files, + })), + }; + }, + { + apiKey: true, + query: t.Object({ + limit: t.Optional(t.Numeric({ minimum: 1, maximum: 100, default: 20 })), + offset: t.Optional(t.Numeric({ minimum: 0, default: 0 })), + }), + response: { + 200: t.Object({ total: t.Number(), jobs: t.Array(JobSummary) }), + 400: ErrorBody, + 401: ErrorBody, + }, + detail: { + summary: "List jobs", + description: "Your jobs, newest first.", + tags: ["Conversion"], + }, + }, + ) + .get("/jobs/:id", ({ params, user }) => describeJob(user.id, params.id, API_ROOT), { + apiKey: true, + params: t.Object({ id: t.Numeric() }), + response: { 200: Job, 400: ErrorBody, 401: ErrorBody, 404: ErrorBody }, + detail: { + summary: "Get a job", + description: "The job's progress and its converted files.", + tags: ["Conversion"], + }, + }) + .get( + "/jobs/:id/files/:name", + async ({ params, user }) => { + const path = await outputFile(user.id, params.id, params.name); + return download(path, params.name); + }, + { + apiKey: true, + params: t.Object({ id: t.Numeric(), name: t.String() }), + response: { 200: t.File(), 400: ErrorBody, 401: ErrorBody, 404: ErrorBody }, + detail: { summary: "Download a converted file", tags: ["Conversion"] }, + }, + ) + .get( + "/jobs/:id/archive", + async ({ params, user }) => { + const path = await archiveJob(user.id, params.id); + return download(path, `converted_files_${params.id}.tar`); + }, + { + apiKey: true, + params: t.Object({ id: t.Numeric() }), + response: { 200: t.File(), 400: ErrorBody, 401: ErrorBody, 404: ErrorBody }, + detail: { + summary: "Download every converted file", + description: "A tar of the job's converted files.", + tags: ["Conversion"], + }, + }, + ) + .delete( + "/jobs/:id", + ({ params, user, set }) => { + deleteJob(user.id, params.id); + set.status = 204; + }, + { + apiKey: true, + params: t.Object({ id: t.Numeric() }), + response: { 204: t.Void(), 400: ErrorBody, 401: ErrorBody, 404: ErrorBody }, + detail: { + summary: "Delete a job", + description: "Deletes the job, its uploads and its converted files.", + tags: ["Conversion"], + }, + }, + ) + // Without this, unknown API paths would get the web interface's plain-text 404. + .all( + "/*", + ({ request }) => + errorResponse( + 404, + "not_found", + `There's no ${request.method} ${new URL(request.url).pathname}. ${API_ROOT}/docs lists every endpoint.`, + ), + { detail: { hide: true } }, + ); diff --git a/src/api/jobs.ts b/src/api/jobs.ts new file mode 100644 index 00000000..47b65d9a --- /dev/null +++ b/src/api/jobs.ts @@ -0,0 +1,275 @@ +import { mkdirSync, rmSync } from "node:fs"; + +import type { Cookie } from "elysia"; +import sanitize from "sanitize-filename"; + +import { getPossibleTargets, handleConvert, outputFileName } from "../converters/main"; +import db from "../db/db"; +import { Filename, Jobs } from "../db/types"; +import { buildJobArchive } from "../helpers/buildJobArchive"; +import { normalizeFiletype } from "../helpers/normalizeFiletype"; +import { outputDir, uploadsDir } from "../index"; + +/** An error the API reports to the client as `{ error, message }` with this status. */ +export class ApiError extends Error { + constructor( + readonly status: 400 | 404 | 422, + readonly code: string, + message: string, + ) { + super(message); + } + + /** Elysia sends this instead of its default plain-text error. */ + toResponse(): Response { + return Response.json({ error: this.code, message: this.message }, { status: this.status }); + } +} + +interface JobFile { + /** The uploaded file's name. */ + input: string; + status: "converting" | "done" | "failed"; + /** The converter's message when the conversion failed. */ + error?: string; + /** Converted files. Most conversions make one; some, such as PDF to images, make several. */ + outputs: { name: string; url: string }[]; +} + +export interface Job { + id: number; + status: "pending" | "completed" | "failed"; + createdAt: string; + files: JobFile[]; +} + +function extension(fileName: string): string { + return fileName.includes(".") ? normalizeFiletype(fileName.split(".").pop() ?? "") : ""; +} + +/** + * Picks a converter that can turn every file into `to`: the one asked for, or else the first that + * handles them all, as the web UI offers them. + */ +export function chooseConverter(fileNames: string[], to: string, wanted?: string): string { + const types = [...new Set(fileNames.map(extension))]; + const candidates = Object.entries(getPossibleTargets(types[0] ?? "")) + .filter(([, targets]) => targets.includes(to)) + .map(([name]) => name); + const usable = candidates.filter((name) => + types.every((type) => getPossibleTargets(type)[name]?.includes(to)), + ); + + if (wanted) { + if (usable.includes(wanted)) return wanted; + throw new ApiError( + 422, + "unsupported_conversion", + `${wanted} can't convert ${types.map((type) => `.${type}`).join(", ")} to .${to}.` + + (usable.length ? ` Converters that can: ${usable.join(", ")}.` : ""), + ); + } + const chosen = usable[0]; + if (chosen) return chosen; + throw new ApiError( + 422, + "unsupported_conversion", + `No converter turns ${types.map((type) => `.${type || "(no extension)"}`).join(", ")} into .${to}. ` + + `GET /api/v1/targets?from=${types[0] ?? ""} lists what each type can become.`, + ); +} + +/** Checks the target format the same way the web UI does. */ +export function cleanTarget(to: string): string { + const target = normalizeFiletype(to.trim().replace(/^\./, "")); + if (!target || target.includes("/") || target.includes("\\") || target.includes("..")) { + throw new ApiError(400, "invalid_format", `"${to}" isn't a file format.`); + } + return target; +} + +/** + * Saves the uploads, records the job and starts converting in the background. `done` resolves when + * every file has been converted. It doesn't reject: POST /jobs leaves it running, and Bun stops the + * server on a rejection nothing handles. + */ +export async function createJob( + userId: string, + files: File[], + to: string, + converter: string, +): Promise<{ id: number; done: Promise }> { + const names = files.map((file) => sanitize(file.name)); + if (names.some((name) => name === "")) { + throw new ApiError(400, "invalid_file_name", "Every file needs a name with an extension."); + } + const duplicate = names.find((name, i) => names.indexOf(name) !== i); + if (duplicate) { + throw new ApiError( + 400, + "duplicate_file_name", + `Two files are called "${duplicate}". Rename one.`, + ); + } + // Names that differ only in extension, such as logo.png and logo.svg, give the same + // output name, and the second result would overwrite the first. + const inputFor = new Map(); + for (const name of names) { + const output = outputFileName(name, to); + const other = inputFor.get(output); + if (other) { + throw new ApiError( + 400, + "duplicate_output_name", + `"${other}" and "${name}" would both become "${output}". Rename one, or convert them in separate jobs.`, + ); + } + inputFor.set(output, name); + } + + const job = db + .query( + "INSERT INTO jobs (user_id, date_created, status, num_files) VALUES (?1, ?2, 'pending', ?3) RETURNING id", + ) + .get(userId, new Date().toISOString(), names.length) as { id: number }; + const userUploadsDir = `${uploadsDir}${userId}/${job.id}/`; + const userOutputDir = `${outputDir}${userId}/${job.id}/`; + try { + mkdirSync(userOutputDir, { recursive: true }); + await Promise.all(files.map((file, i) => Bun.write(`${userUploadsDir}${names[i]}`, file))); + } catch (error) { + // The client never gets this job's id, so nothing of it should be left behind. The record + // goes first: whatever stopped the upload may stop removing its folders too. + db.query("DELETE FROM jobs WHERE id = ?").run(job.id); + for (const dir of [userUploadsDir, userOutputDir]) { + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + // The error worth reporting is the one rethrown below. + } + } + throw error; + } + + // handleConvert reads the job id from the web UI's cookie; it only needs `.value`. + const jobId = { value: String(job.id) } as Cookie; + const done = handleConvert(names, userUploadsDir, userOutputDir, to, converter, jobId) + .then(() => { + // The job can be deleted while it converts. + if (!findJob(userId, job.id)) return; + // handleConvert also resolves when files fail; each file's record says how it went. + const allDone = describeJob(userId, job.id, "").files.every((f) => f.status === "done"); + db.query("UPDATE jobs SET status = ? WHERE id = ?").run( + allDone ? "completed" : "failed", + job.id, + ); + }) + .catch((error: unknown) => { + console.error(`API job ${job.id} failed:`, error); + db.query("UPDATE jobs SET status = 'failed' WHERE id = ?").run(job.id); + }); + return { id: job.id, done }; +} + +function findJob(userId: string, jobId: string | number): Jobs | null { + return db.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?").as(Jobs).get(userId, jobId); +} + +function notFound(jobId: string | number): ApiError { + return new ApiError(404, "job_not_found", `There's no job ${jobId}.`); +} + +/** A job with each file's progress and its converted files. */ +export function describeJob(userId: string, jobId: string | number, apiRoot: string): Job { + const job = findJob(userId, jobId); + if (!job) throw notFound(jobId); + + const rows = db + .query("SELECT * FROM file_names WHERE job_id = ? ORDER BY id") + .as(Filename) + .all(job.id); + let uploaded: string[] = []; + try { + uploaded = [...new Bun.Glob("*").scanSync({ cwd: `${uploadsDir}${userId}/${job.id}` })]; + } catch { + // The uploads may already have been cleaned up; the file records still say what happened. + } + const inputs = [...new Set([...uploaded, ...rows.map((row) => row.file_name)])].sort(); + + const files = inputs.map((input): JobFile => { + const results = rows.filter((row) => row.file_name === input); + if (results.length === 0) { + // Rows appear when a file finishes. A finished job with no row lost the file. + return job.status === "pending" + ? { input, status: "converting", outputs: [] } + : { + input, + status: "failed", + error: "The conversion stopped before this file.", + outputs: [], + }; + } + const failed = results.find((row) => row.status !== "Done"); + if (failed) return { input, status: "failed", error: failed.status, outputs: [] }; + return { + input, + status: "done", + outputs: results.map((row) => ({ + name: row.output_file_name, + url: `${apiRoot}/jobs/${job.id}/files/${encodeURIComponent(row.output_file_name)}`, + })), + }; + }); + + const status = job.status === "completed" || job.status === "failed" ? job.status : "pending"; + return { id: job.id, status, createdAt: job.date_created, files }; +} + +export function listJobs(userId: string, limit: number, offset: number) { + const jobs = db + .query( + "SELECT * FROM jobs WHERE user_id = ? AND num_files > 0 ORDER BY id DESC LIMIT ? OFFSET ?", + ) + .as(Jobs) + .all(userId, limit, offset); + const { total } = db + .query("SELECT count(*) AS total FROM jobs WHERE user_id = ? AND num_files > 0") + .get(userId) as { total: number }; + return { jobs, total }; +} + +/** The path of a converted file, after checking it belongs to the user's job. */ +export async function outputFile( + userId: string, + jobId: string | number, + name: string, +): Promise { + const job = findJob(userId, jobId); + if (!job) throw notFound(jobId); + const file = `${outputDir}${userId}/${job.id}/${sanitize(name)}`; + if (!(await Bun.file(file).exists())) { + throw new ApiError( + 404, + "file_not_found", + `Job ${job.id} has no converted file called "${name}".`, + ); + } + return file; +} + +/** A tar of every converted file in a job, shared with the web interface's archive download. */ +export async function archiveJob(userId: string, jobId: string | number): Promise { + const job = findJob(userId, jobId); + if (!job) throw notFound(jobId); + return buildJobArchive(`${outputDir}${userId}/${job.id}`, String(job.id)); +} + +/** Deletes a job, its uploads, its results and its file records. */ +export function deleteJob(userId: string, jobId: string | number): void { + const job = findJob(userId, jobId); + if (!job) throw notFound(jobId); + rmSync(`${outputDir}${userId}/${job.id}`, { recursive: true, force: true }); + rmSync(`${uploadsDir}${userId}/${job.id}`, { recursive: true, force: true }); + db.query("DELETE FROM file_names WHERE job_id = ?").run(job.id); + db.query("DELETE FROM jobs WHERE id = ?").run(job.id); +} diff --git a/src/api/keys.ts b/src/api/keys.ts new file mode 100644 index 00000000..bec3c02d --- /dev/null +++ b/src/api/keys.ts @@ -0,0 +1,76 @@ +import { createHash, randomBytes } from "node:crypto"; + +import db from "../db/db"; + +export interface ApiKey { + id: number; + user_id: number; + name: string; + /** The start of the key, shown so people can tell their keys apart. */ + prefix: string; + date_created: string; + last_used: string | null; +} + +const KEY_PREFIX = "cvx_"; + +function hashKey(key: string): string { + return createHash("sha256").update(key).digest("hex"); +} + +/** Creates a key. The key itself is returned once; only its hash is kept. */ +export function createApiKey( + userId: string | number, + name: string, +): { key: string; apiKey: ApiKey } { + const key = `${KEY_PREFIX}${randomBytes(32).toString("base64url")}`; + const apiKey = db + .query( + `INSERT INTO api_keys (user_id, name, prefix, key_hash, date_created) + VALUES (?1, ?2, ?3, ?4, ?5) + RETURNING id, user_id, name, prefix, date_created, last_used`, + ) + .get( + userId, + name.trim() || "API key", + key.slice(0, 12), + hashKey(key), + new Date().toISOString(), + ) as ApiKey; + return { key, apiKey }; +} + +export function listApiKeys(userId: string | number): ApiKey[] { + return db + .query( + "SELECT id, user_id, name, prefix, date_created, last_used FROM api_keys WHERE user_id = ? ORDER BY id DESC", + ) + .all(userId) as ApiKey[]; +} + +/** Revokes a key. Returns `false` if the user has no such key. */ +export function deleteApiKey(userId: string | number, id: string | number): boolean { + return db.query("DELETE FROM api_keys WHERE id = ? AND user_id = ?").run(id, userId).changes > 0; +} + +/** The user a key belongs to, or `null`. Records when the key was last used. */ +export function userForApiKey(key: string): { id: string } | null { + if (!key.startsWith(KEY_PREFIX)) return null; + // Only keys of registered accounts count, never those of anonymous sessions. + const row = db + .query( + `SELECT api_keys.id, api_keys.user_id, api_keys.last_used FROM api_keys + JOIN users ON users.id = api_keys.user_id WHERE api_keys.key_hash = ?`, + ) + .get(hashKey(key)) as { id: number; user_id: number; last_used: string | null } | null; + if (!row) return null; + // Writing on every request would be wasteful; a minute's precision is plenty. + const now = Date.now(); + if (!row.last_used || now - Date.parse(row.last_used) > 60_000) { + db.query("UPDATE api_keys SET last_used = ? WHERE id = ?").run( + new Date(now).toISOString(), + row.id, + ); + } + return { id: String(row.user_id) }; +} diff --git a/src/converters/main.ts b/src/converters/main.ts index a82adadd..de1484ea 100644 --- a/src/converters/main.ts +++ b/src/converters/main.ts @@ -163,6 +163,18 @@ function chunks(arr: T[], size: number): T[][] { ); } +/** The name a converted file gets: the input's name, with the target's extension. */ +export function outputFileName(fileName: string, convertTo: string): string { + const fileTypeOrig = fileName.includes(".") ? (fileName.split(".").pop() ?? "") : ""; + const newFileExt = normalizeOutputFiletype(convertTo); + if (fileTypeOrig === "") { + return `${fileName}.${newFileExt}`; + } + // The extension is everything after the last dot, so it can be sliced off. A RegExp built from + // it would break on names such as `scan.[` or `notes.$x`. + return `${fileName.slice(0, -fileTypeOrig.length)}${newFileExt}`; +} + export async function handleConvert( fileNames: string[], userUploadsDir: string, @@ -181,16 +193,7 @@ export async function handleConvert( const filePath = `${userUploadsDir}${fileName}`; const fileTypeOrig = fileName.includes(".") ? (fileName.split(".").pop() ?? "") : ""; const fileType = normalizeFiletype(fileTypeOrig); - const newFileExt = normalizeOutputFiletype(convertTo); - let newFileName: string; - if (fileTypeOrig === "") { - newFileName = `${fileName}.${newFileExt}`; - } else { - newFileName = fileName.replace( - new RegExp(`${fileTypeOrig}(?!.*${fileTypeOrig})`), - newFileExt, - ); - } + const newFileName = outputFileName(fileName, convertTo); const targetPath = `${userOutputDir}${newFileName}`; toProcess.push( new Promise((resolve, reject) => { diff --git a/src/db/db.ts b/src/db/db.ts index a99302c2..0e879042 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -46,6 +46,20 @@ export function initializeDatabase(db: Database): void { console.log("Updated database to version 1."); } + // Keys for the REST API. Only a hash of each key is stored. + db.exec(` + CREATE TABLE IF NOT EXISTS api_keys ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + name TEXT NOT NULL, + prefix TEXT NOT NULL, + key_hash TEXT NOT NULL UNIQUE, + date_created TEXT NOT NULL, + last_used TEXT, + FOREIGN KEY (user_id) REFERENCES users(id) + ); + `); + // enable WAL mode db.exec("PRAGMA journal_mode = WAL;"); } diff --git a/src/helpers/buildJobArchive.ts b/src/helpers/buildJobArchive.ts new file mode 100644 index 00000000..8dcba66d --- /dev/null +++ b/src/helpers/buildJobArchive.ts @@ -0,0 +1,46 @@ +import { randomUUID } from "node:crypto"; +import { rename, unlink } from "node:fs/promises"; +import path from "node:path"; + +const archiveBuilds = new Map>(); + +/** + * Writes `converted_files_.tar` with every file in a job's output folder and returns its + * path. Requests for a job whose archive is being built wait for that build instead of starting + * another. Used by the web interface's archive download and by the API. + */ +export async function buildJobArchive(outputPath: string, jobId: string): Promise { + const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`); + + let archiveBuild = archiveBuilds.get(outputPath); + if (!archiveBuild) { + const build = async () => { + const outputTarName = path.basename(outputTar); + const outputTarTempName = `${outputTarName}.tmp-${randomUUID()}`; + const outputTarTemp = path.join(outputPath, outputTarTempName); + + try { + const files: Record = {}; + const glob = new Bun.Glob("**/*"); + for await (const file of glob.scan({ cwd: outputPath, onlyFiles: true, dot: true })) { + if (file !== outputTarName && !file.startsWith(`${outputTarName}.tmp-`)) { + files[file] = new Uint8Array(await Bun.file(path.join(outputPath, file)).arrayBuffer()); + } + } + await Bun.Archive.write(outputTarTemp, files); + await rename(outputTarTemp, outputTar); + } finally { + if (await Bun.file(outputTarTemp).exists()) { + await unlink(outputTarTemp); + } + } + }; + + archiveBuild = build().finally(() => { + archiveBuilds.delete(outputPath); + }); + archiveBuilds.set(outputPath, archiveBuild); + } + await archiveBuild; + return outputTar; +} diff --git a/src/index.tsx b/src/index.tsx index 3fcff561..286f2800 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -4,11 +4,13 @@ import { html } from "@elysiajs/html"; import { staticPlugin } from "@elysiajs/static"; import { Elysia } from "elysia"; +import { api } from "./api"; import db from "./db/db"; import { Jobs } from "./db/types"; import { AUTO_DELETE_EVERY_N_HOURS, WEBROOT } from "./helpers/env"; import "./helpers/printVersions"; import { scheduleDelayed, ScheduledTask } from "./helpers/scheduleDelayed"; +import { apiKeys } from "./pages/apiKeys"; import { chooseConverter } from "./pages/chooseConverter"; import { convert } from "./pages/convert"; import { deleteFile } from "./pages/deleteFile"; @@ -53,6 +55,8 @@ const app = new Elysia({ .use(listConverters) .use(chooseConverter) .use(healthcheck) + .use(apiKeys) + .use(api) .onError(({ error, code, request }) => { if (code === "NOT_FOUND") { console.warn(`404: ${request.method} ${new URL(request.url).pathname}`); diff --git a/src/pages/apiKeys.tsx b/src/pages/apiKeys.tsx new file mode 100644 index 00000000..a5cf56d1 --- /dev/null +++ b/src/pages/apiKeys.tsx @@ -0,0 +1,144 @@ +import { Elysia, t } from "elysia"; + +import { type ApiKey, createApiKey, deleteApiKey } from "../api/keys"; +import db from "../db/db"; +import { LANGUAGE, TIMEZONE, WEBROOT } from "../helpers/env"; +import { userService } from "../services/user"; + +/** Dates in the configured language and time zone, like the history page. */ +function formatDate(iso: string): string { + return new Date(iso).toLocaleString(LANGUAGE, { timeZone: TIMEZONE }); +} + +/** The API keys part of the account page. */ +export function ApiKeys({ keys, newKey }: { keys: ApiKey[]; newKey?: string | undefined }) { + return ( +
+

API keys

+

+ Keys let scripts and tools convert files through the ConvertX API. Send one as{" "} + {/* Escaped by hand: JSX would turn < in markup back into a raw "<". */} + {"Authorization: Bearer <key>"}. See the{" "} + + API documentation + + . +

+ {newKey ? ( +
+ Copy your new key now. It won't be shown again. + + {newKey} + +
+ ) : null} +
+ + {/* Its own row, so the input and the button stretch to the same height. */} +
+ + +
+ {/* Suggestions only: any name works, and the name doesn't change what a key can do. */} + + +
+ {keys.length === 0 ? ( +

You have no API keys yet.

+ ) : ( + // Scrolls sideways on phones instead of widening the page. +
+ + + + + + + + + + + + {keys.map((key) => ( + + + + + + + + ))} + +
NameKeyCreatedLast used + Revoke +
+ {key.name} + + {`${key.prefix}…`} + + {formatDate(key.date_created)} + {key.last_used ? formatDate(key.last_used) : "Never"} +
+ +
+
+
+ )} +
+ ); +} + +/** Keys belong to accounts, so anonymous sessions (`ALLOW_UNAUTHENTICATED`) can't manage any. */ +function isRegistered(userId: string): boolean { + return db.query("SELECT 1 FROM users WHERE id = ?").get(userId) !== null; +} + +// Creating a key redirects to the account page, which shows the new key once. Reloading that +// page can't create another key. Until it's shown, the key waits here for at most a minute. +const newKeys = new Map(); + +/** The key the user just created, if the account page hasn't shown it yet. */ +export function takeNewKey(userId: string): string | undefined { + const entry = newKeys.get(userId); + newKeys.delete(userId); + return entry && entry.expires > Date.now() ? entry.key : undefined; +} + +// The account page (src/pages/user.tsx) lists the keys; these routes create and revoke them. +export const apiKeys = new Elysia() + .use(userService) + .post( + "/account/api-keys", + ({ body, user, redirect }) => { + if (!isRegistered(user.id)) return redirect(`${WEBROOT}/`, 302); + const { key } = createApiKey(user.id, body.name.slice(0, 100)); + const now = Date.now(); + for (const [id, entry] of newKeys) if (entry.expires <= now) newKeys.delete(id); + newKeys.set(user.id, { key, expires: now + 60_000 }); + return redirect(`${WEBROOT}/account#api-keys`, 303); + }, + { auth: true, body: t.Object({ name: t.String() }) }, + ) + // A POST, not a GET, so a link can't revoke a key behind the user's back. + .post( + "/account/api-keys/:id/delete", + ({ params, user, redirect }) => { + if (isRegistered(user.id)) deleteApiKey(user.id, params.id); + return redirect(`${WEBROOT}/account#api-keys`, 302); + }, + { auth: true, params: t.Object({ id: t.Numeric() }) }, + ); diff --git a/src/pages/download.tsx b/src/pages/download.tsx index 60cbf02c..11ae862e 100644 --- a/src/pages/download.tsx +++ b/src/pages/download.tsx @@ -1,17 +1,12 @@ -import { randomUUID } from "node:crypto"; -import { rename, unlink } from "node:fs/promises"; -import path from "node:path"; - import { Elysia } from "elysia"; import sanitize from "sanitize-filename"; import { outputDir } from ".."; import db from "../db/db"; +import { buildJobArchive } from "../helpers/buildJobArchive"; import { WEBROOT } from "../helpers/env"; import { userService } from "./user"; -const archiveBuilds = new Map>(); - export const download = new Elysia() .use(userService) .get( @@ -55,42 +50,7 @@ export const download = new Elysia() } const jobId = decodeURIComponent(params.jobId); - const outputPath = `${outputDir}${userId}/${jobId}`; - const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`); - - const archiveKey = `${userId}/${jobId}`; - let archiveBuild = archiveBuilds.get(archiveKey); - if (!archiveBuild) { - const build = async () => { - const outputTarName = path.basename(outputTar); - const outputTarTempName = `${outputTarName}.tmp-${randomUUID()}`; - const outputTarTemp = path.join(outputPath, outputTarTempName); - - try { - const files: Record = {}; - const glob = new Bun.Glob("**/*"); - for await (const file of glob.scan({ cwd: outputPath, onlyFiles: true, dot: true })) { - if (file !== outputTarName && !file.startsWith(`${outputTarName}.tmp-`)) { - files[file] = new Uint8Array( - await Bun.file(path.join(outputPath, file)).arrayBuffer(), - ); - } - } - await Bun.Archive.write(outputTarTemp, files); - await rename(outputTarTemp, outputTar); - } finally { - if (await Bun.file(outputTarTemp).exists()) { - await unlink(outputTarTemp); - } - } - }; - - archiveBuild = build().finally(() => { - archiveBuilds.delete(archiveKey); - }); - archiveBuilds.set(archiveKey, archiveBuild); - } - await archiveBuild; + const outputTar = await buildJobArchive(`${outputDir}${userId}/${jobId}`, jobId); return Bun.file(outputTar); }, { diff --git a/src/pages/user.tsx b/src/pages/user.tsx index 6405d257..426172ee 100644 --- a/src/pages/user.tsx +++ b/src/pages/user.tsx @@ -1,5 +1,6 @@ import { Elysia, t } from "elysia"; +import { type ApiKey, listApiKeys } from "../api/keys"; import { BaseHtml } from "../components/base"; import { Header } from "../components/header"; import db from "../db/db"; @@ -13,6 +14,7 @@ import { WEBROOT, } from "../helpers/env"; import { userService } from "../services/user"; +import { ApiKeys, takeNewKey } from "./apiKeys"; export { userService } from "../services/user"; @@ -95,7 +97,17 @@ function LoginPage({ error, email = "" }: { error?: string; email?: string }) { ); } -function AccountPage({ email, error }: { email: string; error?: string }) { +function AccountPage({ + email, + error, + keys, + newKey, +}: { + email: string; + error?: string; + keys: ApiKey[]; + newKey?: string | undefined; +}) { return ( <> @@ -156,6 +168,7 @@ function AccountPage({ email, error }: { email: string; error?: string }) { + @@ -425,7 +438,13 @@ export const user = new Elysia() return redirect(`${WEBROOT}/`, 302); } - return ; + return ( + + ); }, { auth: true, @@ -456,7 +475,13 @@ export const user = new Elysia() if (!validPassword) { set.status = 403; - return ; + return ( + + ); } const fields = []; @@ -469,7 +494,13 @@ export const user = new Elysia() .get(body.email); if (existingEmailUser && existingEmailUser.id.toString() !== user.id) { set.status = 409; - return ; + return ( + + ); } fields.push("email"); values.push(body.email); diff --git a/tests/api/api.test.ts b/tests/api/api.test.ts new file mode 100644 index 00000000..b319343d --- /dev/null +++ b/tests/api/api.test.ts @@ -0,0 +1,394 @@ +import { beforeAll, describe, expect, test } from "bun:test"; +import { existsSync, readdirSync } from "node:fs"; + +import { Elysia } from "elysia"; + +if (!process.env.CONVERTX_TEST_ROOT) { + // Without tests/preload.ts the API would use ./data/mydb.sqlite and write into ./data. + throw new Error("API tests must run via `bun test` so that tests/preload.ts is loaded."); +} + +// dynamic imports ensure that the guard above runs before the API is loaded +const { default: db } = await import("../../src/db/db"); +const { api } = await import("../../src/api"); +const { createApiKey, deleteApiKey, listApiKeys } = await import("../../src/api/keys"); +const { createJob } = await import("../../src/api/jobs"); +const { outputDir, uploadsDir } = await import("../../src/index"); +// Accounts come from the page tests' helper. Loading it walks through the first run, which +// tests/pages/firstRun.test.ts checks, so it must load before any account exists. +const { createUser } = await import("../pages/helpers/app"); + +// Some page, to check the API docs leave the web UI out. +const app = new Elysia().get("/history", () => "history page").use(api); + +const vcard = (name: string, email: string) => + `BEGIN:VCARD\nVERSION:3.0\nFN:${name}\nN:${name.split(" ")[1]};${name.split(" ")[0]}\nEMAIL:${email}\nEND:VCARD\n`; + +async function addUser(): Promise { + return String((await createUser()).id); +} + +const alice = await addUser(); +const bob = await addUser(); +const aliceKey = createApiKey(alice, "tests").key; +const bobKey = createApiKey(bob, "tests").key; + +function call(path: string, init: RequestInit & { key?: string | null } = {}) { + const { key = aliceKey, ...rest } = init; + const headers = new Headers(rest.headers); + if (key) headers.set("authorization", `Bearer ${key}`); + return app.handle(new Request(`http://localhost/api/v1${path}`, { ...rest, headers })); +} + +function form(fields: Record): FormData { + const data = new FormData(); + for (const [name, value] of Object.entries(fields)) { + for (const item of Array.isArray(value) ? value : [value]) data.append(name, item); + } + return data; +} + +async function waitForJob(id: number) { + for (let i = 0; i < 100; i++) { + const job = await (await call(`/jobs/${id}`)).json(); + if (job.status !== "pending") return job; + await Bun.sleep(20); + } + throw new Error(`Job ${id} didn't finish`); +} + +describe("authentication", () => { + test("rejects requests without a valid key", async () => { + for (const key of [null, "cvx_not-a-real-key", "not-even-the-format"]) { + const response = await call("/converters", { key }); + expect(response.status).toBe(401); + expect((await response.json()).error).toBe("unauthorized"); + } + }); + + test("accepts the key as a bearer token or in X-API-Key", async () => { + expect((await call("/converters")).status).toBe(200); + expect( + (await call("/converters", { key: null, headers: { "x-api-key": aliceKey } })).status, + ).toBe(200); + }); + + test("stores only a hash of each key, and revoking one locks it out", async () => { + const { key, apiKey } = createApiKey(alice, "temporary"); + const stored = JSON.stringify(db.query("SELECT * FROM api_keys").all()); + expect(stored).not.toContain(key); + expect(apiKey.prefix).toBe(key.slice(0, 12)); + expect((await call("/converters", { key })).status).toBe(200); + expect(listApiKeys(alice).find((entry) => entry.id === apiKey.id)?.last_used).toBeTruthy(); + expect(deleteApiKey(bob, apiKey.id)).toBe(false); + expect(deleteApiKey(alice, apiKey.id)).toBe(true); + expect((await call("/converters", { key })).status).toBe(401); + }); + + test("only accepts keys of registered accounts", async () => { + // e.g. a key made for an anonymous session's id, which has no account + const { key } = createApiKey(2 ** 24 + 7, "anonymous"); + expect((await call("/converters", { key })).status).toBe(401); + }); +}); + +describe("formats", () => { + test("lists converters with what they read and write", async () => { + const { converters } = await (await call("/converters")).json(); + expect(converters).toContainEqual({ name: "vcf", from: ["vcf"], to: ["csv"] }); + }); + + test("lists what a format can become", async () => { + expect(await (await call("/targets?from=.VCF")).json()).toEqual({ + from: "vcf", + targets: [{ format: "csv", converters: ["vcf"] }], + }); + }); + + test("names each converter once per format", async () => { + // LibreOffice lists pdf among both its document and its spreadsheet targets. + const { targets } = await (await call("/targets?from=docx")).json(); + for (const { converters } of targets) expect(converters).toEqual([...new Set(converters)]); + }); +}); + +describe("converting one file", () => { + test("returns the converted file", async () => { + const response = await call("/convert", { + method: "POST", + body: form({ + file: new File([vcard("Ada Lovelace", "ada@example.com")], "contacts.vcf"), + to: "csv", + }), + }); + expect(response.status).toBe(200); + expect(response.headers.get("content-disposition")).toBe( + "attachment; filename*=UTF-8''contacts.csv", + ); + expect(response.headers.get("x-convertx-job")).toMatch(/^\d+$/); + const csv = await response.text(); + expect(csv).toContain("Ada"); + expect(csv).toContain("ada@example.com"); + }); + + test("explains a conversion no converter can do", async () => { + const response = await call("/convert", { + method: "POST", + body: form({ file: new File(["x"], "contacts.vcf"), to: "pdf" }), + }); + expect(response.status).toBe(422); + expect(await response.json()).toEqual({ + error: "unsupported_conversion", + message: + "No converter turns .vcf into .pdf. GET /api/v1/targets?from=vcf lists what each type can become.", + }); + }); + + test("names the converters that can, when the one asked for can't", async () => { + const response = await call("/convert", { + method: "POST", + body: form({ file: new File(["x"], "contacts.vcf"), to: "csv", converter: "ffmpeg" }), + }); + expect(response.status).toBe(422); + expect(await response.json()).toEqual({ + error: "unsupported_conversion", + message: "ffmpeg can't convert .vcf to .csv. Converters that can: vcf.", + }); + }); + + test("reports a failed conversion", async () => { + // The test environment has no image tools, so this one fails like a broken install would. + const response = await call("/convert", { + method: "POST", + body: form({ file: new File([new Uint8Array([137, 80, 78, 71])], "pixel.png"), to: "jpg" }), + }); + expect(response.status).toBe(422); + expect((await response.json()).error).toBe("conversion_failed"); + }); + + test("rejects a format that looks like a path", async () => { + const response = await call("/convert", { + method: "POST", + body: form({ file: new File(["x"], "contacts.vcf"), to: "../csv" }), + }); + expect(response.status).toBe(400); + expect((await response.json()).error).toBe("invalid_format"); + }); +}); + +describe("jobs", () => { + let id: number; + + beforeAll(async () => { + const response = await call("/jobs", { + method: "POST", + body: form({ + files: [ + new File([vcard("Ada Lovelace", "ada@example.com")], "team.vcf"), + new File([vcard("Grace Hopper", "grace@example.com")], "more team.vcf"), + ], + to: "csv", + }), + }); + expect(response.status).toBe(202); + const job = await response.json(); + id = job.id; + expect(response.headers.get("location")).toBe(`/api/v1/jobs/${id}`); + }); + + test("converts every file and links each result", async () => { + const job = await waitForJob(id); + expect(job).toMatchObject({ + id, + status: "completed", + files: [ + { + input: "more team.vcf", + status: "done", + outputs: [{ name: "more team.csv", url: `/api/v1/jobs/${id}/files/more%20team.csv` }], + }, + { + input: "team.vcf", + status: "done", + outputs: [{ name: "team.csv", url: `/api/v1/jobs/${id}/files/team.csv` }], + }, + ], + }); + }); + + test("downloads one result, or all of them as a tar", async () => { + await waitForJob(id); + const one = await call(`/jobs/${id}/files/more%20team.csv`); + expect(one.status).toBe(200); + expect(await one.text()).toContain("grace@example.com"); + + // Twice, as the first archive is saved next to the results and mustn't end up in the second. + for (let i = 0; i < 2; i++) { + const response = await call(`/jobs/${id}/archive`); + const files = await new Bun.Archive(await response.arrayBuffer()).files(); + expect([...files.keys()].sort()).toEqual(["more team.csv", "team.csv"]); + expect(await files.get("team.csv")?.text()).toContain("ada@example.com"); + } + }); + + test("keeps results whose names start like the archive's", async () => { + const lookalike = `converted_files_${id}.tar.csv`; + await Bun.write(`${outputDir}${alice}/${id}/${lookalike}`, "kept"); + const response = await call(`/jobs/${id}/archive`); + const files = await new Bun.Archive(await response.arrayBuffer()).files(); + expect([...files.keys()].sort()).toEqual([lookalike, "more team.csv", "team.csv"]); + }); + + test("lists jobs, newest first", async () => { + const { total, jobs } = await (await call("/jobs?limit=1")).json(); + expect(total).toBeGreaterThanOrEqual(2); + expect(jobs).toHaveLength(1); + expect(jobs[0].id).toBe(id); + }); + + test("keeps jobs private to their owner", async () => { + for (const path of [`/jobs/${id}`, `/jobs/${id}/files/team.csv`, `/jobs/${id}/archive`]) { + const response = await call(path, { key: bobKey }); + expect(response.status).toBe(404); + expect((await response.json()).error).toBe("job_not_found"); + } + expect((await call(`/jobs/${id}`, { method: "DELETE", key: bobKey })).status).toBe(404); + }); + + test("rejects two files with the same name", async () => { + const response = await call("/jobs", { + method: "POST", + body: form({ files: [new File(["a"], "same.vcf"), new File(["b"], "same.vcf")], to: "csv" }), + }); + expect(response.status).toBe(400); + expect((await response.json()).error).toBe("duplicate_file_name"); + }); + + test("rejects two files that would get the same output name", async () => { + // Otherwise the second logo.webp would overwrite the first. + const response = await call("/jobs", { + method: "POST", + body: form({ files: [new File(["a"], "logo.png"), new File(["b"], "logo.svg")], to: "webp" }), + }); + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: "duplicate_output_name", + message: + '"logo.png" and "logo.svg" would both become "logo.webp". Rename one, or convert them in separate jobs.', + }); + }); + + test("deletes a job with its files", async () => { + await waitForJob(id); + expect((await call(`/jobs/${id}`, { method: "DELETE" })).status).toBe(204); + expect((await call(`/jobs/${id}`)).status).toBe(404); + expect(db.query("SELECT count(*) AS n FROM file_names WHERE job_id = ?").get(id)).toEqual({ + n: 0, + }); + expect(await Bun.file(`${uploadsDir}${alice}/${id}/team.vcf`).exists()).toBe(false); + expect(await Bun.file(`${outputDir}${alice}/${id}/team.csv`).exists()).toBe(false); + expect(existsSync(`${uploadsDir}${alice}/${id}`)).toBe(false); + expect(existsSync(`${outputDir}${alice}/${id}`)).toBe(false); + }); +}); + +describe("failures", () => { + test("marks a job failed when a file fails", async () => { + // The test environment has no image tools, so this conversion fails. + const response = await call("/jobs", { + method: "POST", + body: form({ files: new File([new Uint8Array([137, 80, 78, 71])], "pixel.png"), to: "jpg" }), + }); + const job = await waitForJob((await response.json()).id); + expect(job.status).toBe("failed"); + expect(job.files[0].status).toBe("failed"); + }); + + test("leaves nothing behind when the uploads can't be saved", async () => { + const carol = await addUser(); + const carolKey = createApiKey(carol, "tests").key; + // A file where carol's uploads folder belongs makes saving any upload fail. + await Bun.write(`${uploadsDir}${carol}`, "in the way"); + const response = await call("/jobs", { + method: "POST", + key: carolKey, + body: form({ + files: new File([vcard("Ada Lovelace", "ada@example.com")], "team.vcf"), + to: "csv", + }), + }); + expect(response.status).toBe(500); + expect((await response.json()).error).toBe("internal_error"); + expect(db.query("SELECT count(*) AS n FROM jobs WHERE user_id = ?").get(carol)).toEqual({ + n: 0, + }); + expect(existsSync(`${outputDir}${carol}`) ? readdirSync(`${outputDir}${carol}`) : []).toEqual( + [], + ); + }); + + test("survives a job deleted while it converts", async () => { + const files = [new File([vcard("Ada Lovelace", "ada@example.com")], "late.vcf")]; + const { id, done } = await createJob(alice, files, "csv", "vcf"); + // Deleting only the record lets the conversion finish, as it does when a converter + // creates its output folder again after the job's folders were removed. + db.query("DELETE FROM jobs WHERE id = ?").run(id); + // POST /jobs doesn't wait for `done`, so a rejection here would stop the server. + expect(await done).toBeUndefined(); + expect(db.query("SELECT 1 FROM jobs WHERE id = ?").get(id)).toBeNull(); + }); + + test("answers unknown paths with a JSON 404", async () => { + const response = await call("/nope"); + expect(response.status).toBe(404); + expect(await response.json()).toEqual({ + error: "not_found", + message: "There's no GET /api/v1/nope. /api/v1/docs lists every endpoint.", + }); + }); + + test("answers invalid requests with a JSON 400", async () => { + const missingTarget = await call("/convert", { + method: "POST", + body: form({ file: new File(["x"], "contacts.vcf") }), + }); + const badId = await call("/jobs/abc"); + for (const [response, field] of [ + [missingTarget, "to"], + [badId, "id"], + ] as const) { + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: "invalid_request", + message: `"${field}" is missing or not valid. /api/v1/docs describes what each endpoint takes.`, + }); + } + }); +}); + +describe("documentation", () => { + test("describes the API, and only the API, in OpenAPI", async () => { + const response = await app.handle(new Request("http://localhost/api/v1/openapi.json")); + expect(response.status).toBe(200); + const spec = await response.json(); + expect(Object.keys(spec.paths).sort()).toEqual([ + "/api/v1/convert", + "/api/v1/converters", + "/api/v1/jobs", + "/api/v1/jobs/{id}", + "/api/v1/jobs/{id}/archive", + "/api/v1/jobs/{id}/files/{name}", + "/api/v1/targets", + ]); + expect(spec.components.securitySchemes.apiKey).toMatchObject({ + type: "http", + scheme: "bearer", + }); + }); + + test("serves Swagger UI", async () => { + const response = await app.handle(new Request("http://localhost/api/v1/docs")); + expect(response.status).toBe(200); + expect(await response.text()).toContain("swagger-ui"); + }); +}); diff --git a/tests/converters/main.test.ts b/tests/converters/main.test.ts index 00b7bd92..c66699ae 100644 --- a/tests/converters/main.test.ts +++ b/tests/converters/main.test.ts @@ -17,8 +17,15 @@ if (!process.env.CONVERTX_TEST_ROOT) { // dynamic imports ensure that the guard above runs before the db module is loaded const { default: db } = await import("../../src/db/db"); -const { chunks, getAllInputs, getAllTargets, getPossibleTargets, handleConvert, mainConverter } = - await import("../../src/converters/main"); +const { + chunks, + getAllInputs, + getAllTargets, + getPossibleTargets, + handleConvert, + mainConverter, + outputFileName, +} = await import("../../src/converters/main"); const testRoot = mkdtempSync(join(tmpdir(), "convertx-main-test-")); @@ -158,6 +165,20 @@ describe("converter registry", () => { }); }); +describe("outputFileName", () => { + test("swaps the extension for the target's", () => { + expect(outputFileName("report.docx", "pdf")).toBe("report.pdf"); + expect(outputFileName("photo.backup.png", "jpeg")).toBe("photo.backup.jpg"); + expect(outputFileName("README", "pdf")).toBe("README.pdf"); + }); + + test("handles extensions with regex characters", () => { + expect(outputFileName("scan.[", "png")).toBe("scan.png"); + expect(outputFileName("notes.$x", "pdf")).toBe("notes.pdf"); + expect(outputFileName("main.c++", "txt")).toBe("main.txt"); + }); +}); + describe("chunks", () => { test("chunks with size 0 returns entire array as single chunk", () => { expect(chunks([1, 2, 3, 4, 5], 0)).toEqual([[1, 2, 3, 4, 5]]); diff --git a/tests/pages/apiKeys.test.ts b/tests/pages/apiKeys.test.ts new file mode 100644 index 00000000..590c0e7b --- /dev/null +++ b/tests/pages/apiKeys.test.ts @@ -0,0 +1,97 @@ +import { afterEach, describe, expect, test } from "bun:test"; + +import { createUser, db, request, resetEnv, setEnv, signToken } from "./helpers/app"; + +afterEach(() => { + resetEnv(); +}); + +// A whole key; the list only shows its first few characters. +const FULL_KEY = /cvx_[\w-]{20,}/; + +function keyCount(userId: number | string) { + return ( + db.query("SELECT count(*) AS n FROM api_keys WHERE user_id = ?").get(userId) as { + n: number; + } + ).n; +} + +describe("API keys on the account page", () => { + test("shows a new key once, so reloading doesn't create another", async () => { + const user = await createUser(); + const cookies = { auth: user.token }; + + const created = await request("/account/api-keys", { cookies, form: { name: "n8n" } }); + expect(created.status).toBe(303); + expect(created.headers.get("location")).toBe("/account#api-keys"); + + const first = await (await request("/account", { cookies })).text(); + expect(first).toContain("Copy your new key now"); + expect(first).toMatch(FULL_KEY); + + const reloaded = await (await request("/account", { cookies })).text(); + expect(reloaded).toContain("n8n"); + expect(reloaded).not.toMatch(FULL_KEY); + expect(keyCount(user.id)).toBe(1); + }); + + test("revokes a key", async () => { + const user = await createUser(); + const cookies = { auth: user.token }; + await request("/account/api-keys", { cookies, form: { name: "old script" } }); + const { id } = db.query("SELECT id FROM api_keys WHERE user_id = ?").get(user.id) as { + id: number; + }; + + const revoked = await request(`/account/api-keys/${id}/delete`, { cookies, method: "POST" }); + expect(revoked.status).toBe(302); + expect(revoked.headers.get("location")).toBe("/account#api-keys"); + expect(keyCount(user.id)).toBe(0); + expect(await (await request("/account", { cookies })).text()).toContain( + "You have no API keys yet.", + ); + }); + + test("still lists the keys when updating the account fails", async () => { + const user = await createUser(); + const cookies = { auth: user.token }; + await request("/account/api-keys", { cookies, form: { name: "backup job" } }); + + const response = await request("/account", { + cookies, + form: { email: user.email, newPassword: "", password: "wrong password" }, + }); + expect(response.status).toBe(403); + expect(await response.text()).toContain("backup job"); + }); + + test("shows dates in the configured language and time zone", async () => { + setEnv({ LANGUAGE: "de", TIMEZONE: "Asia/Tokyo" }); + const user = await createUser(); + await request("/account/api-keys", { cookies: { auth: user.token }, form: { name: "script" } }); + const { date_created } = db + .query("SELECT date_created FROM api_keys WHERE user_id = ?") + .get(user.id) as { date_created: string }; + + const page = await (await request("/account", { cookies: { auth: user.token } })).text(); + + expect(page).toContain(new Date(date_created).toLocaleString("de", { timeZone: "Asia/Tokyo" })); + }); + + test("are only for registered accounts, not anonymous sessions", async () => { + // ALLOW_UNAUTHENTICATED gives visitors a signed id that has no account; with + // UNAUTHENTICATED_USER_SHARING they all share id 0. + for (const id of [2 ** 24 + 5, 0]) { + const cookies = { auth: await signToken(id) }; + for (const response of [ + await request("/account", { cookies }), + await request("/account/api-keys", { cookies, form: { name: "anonymous" } }), + ]) { + expect(response.status).toBe(302); + expect(response.headers.get("location")).toBe("/"); + } + expect(keyCount(id)).toBe(0); + } + }); +}); diff --git a/tests/pages/helpers/app.ts b/tests/pages/helpers/app.ts index 82af5f82..1207e874 100644 --- a/tests/pages/helpers/app.ts +++ b/tests/pages/helpers/app.ts @@ -26,6 +26,8 @@ const { deleteFile } = await import("../../../src/pages/deleteFile"); const { listConverters } = await import("../../../src/pages/listConverters"); const { chooseConverter } = await import("../../../src/pages/chooseConverter"); const { healthcheck } = await import("../../../src/pages/healthcheck"); +const { apiKeys } = await import("../../../src/pages/apiKeys"); +const { api } = await import("../../../src/api"); export { db, uploadsDir }; @@ -43,7 +45,9 @@ export const app = new Elysia({ prefix: realEnvModule.WEBROOT }) .use(deleteFile) .use(listConverters) .use(chooseConverter) - .use(healthcheck); + .use(healthcheck) + .use(apiKeys) + .use(api); // --- environment flags --------------------------------------------------------- // src/helpers/env.ts reads process.env once at import, so flags are switched by