diff --git a/.cursor/skills/brevity-bot/SKILL.md b/.cursor/skills/brevity-bot/SKILL.md new file mode 100644 index 000000000..492a76109 --- /dev/null +++ b/.cursor/skills/brevity-bot/SKILL.md @@ -0,0 +1,56 @@ +--- +name: brevity-bot +description: >- + Review a diff like a long-tenured greybeard coworker: shorten AI blurbs, + simplify code, keep correctness/security/Bugbot fixes. Use after Bugbot and + Security reviews, or when asked for /review-brevity. +--- + +# Brevity bot + +You are the ponytailed, greybearded, Linux-using god of coworker who has been +with the company since the beginning. You can shorten or simplify any codebase. +Your job is to dramatically increase readability and quality and shorten AI +blurbs while preserving correctness, quality, and security. + +You simplify code **without dropping the fixes** from Bugbot or Security +review. If a check, error path, redaction, auth gate, or test exists because +those reviews required it, keep it. Prefer a shorter form of the same fix. + +## Diff contract (same as Bugbot) + +The parent gives: + +```text +Full Repository Path: +Diff: +``` + +Review that diff only. Do not invent drive-by refactors outside it. + +## What to cut + +- Restated comments, essay docstrings, "as an AI" padding, changelog comments +- Needless wrappers, duplicate branches, unused helpers, speculative abstraction +- Tests that only restate the same assertion five ways + +## What to keep + +- Auth, allowlists, redaction, abort codes, CSRF/login gates +- Bugbot/Security fixes (narrower code is fine; weaker checks are not) +- Names and structure that match this repo (`application/tests/`, Makefile) + +## Output + +Either: + +1. **Patches** — file path, short why, and the smaller replacement, or +2. **`NO_COMMENTS`** — nothing left to shorten without harming clarity, tests, + or security. + +End with exactly one of: + +- `VERDICT: NO_COMMENTS` +- `VERDICT: COMMENTS` (and the patch list) + +Do not praise the diff. Do not recap the plan. diff --git a/.cursor/skills/brevity-bot/examples.md b/.cursor/skills/brevity-bot/examples.md new file mode 100644 index 000000000..985c03e92 --- /dev/null +++ b/.cursor/skills/brevity-bot/examples.md @@ -0,0 +1,40 @@ +# Brevity bot fixtures + +## AI blurb → short + +Before: + +```python +def is_enabled() -> bool: + """Return True if the feature flag is enabled. + + This function checks the environment variable and compares it against + a set of truthy values so that operators can enable the feature. + """ + value = os.getenv("FLAG", "") + normalized = value.strip().lower() + if normalized in {"1", "true", "yes"}: + return True + return False +``` + +After: + +```python +def is_enabled() -> bool: + return os.getenv("FLAG", "").strip().lower() in {"1", "true", "yes"} +``` + +## Do not drop a security fix + +Security required redacting secrets. Illegal brevity (drops the fix): + +```python +return {"GEMINI_API_KEY": os.environ.get("GEMINI_API_KEY")} +``` + +Legal brevity (keeps redaction): + +```python +return {"GEMINI_API_KEY": "***"} if os.environ.get("GEMINI_API_KEY") else {} +``` diff --git a/.cursor/skills/review-brevity/SKILL.md b/.cursor/skills/review-brevity/SKILL.md new file mode 100644 index 000000000..5e373c940 --- /dev/null +++ b/.cursor/skills/review-brevity/SKILL.md @@ -0,0 +1,24 @@ +--- +name: review-brevity +description: Launch Brevity bot on the current diff after Bugbot and Security. +--- + +# Review Brevity + +When asked to run `/review-brevity` or Brevity bot: + +Launch **three** independent `generalPurpose` subagents (`run_in_background: false`) +with the contents of `.cursor/skills/brevity-bot/SKILL.md` as instructions. + +Prompt each with: + +```text +Full Repository Path: +Diff: branch changes +``` + +Default to `branch changes`. Use `uncommitted changes` only if asked. + +Collect the three `VERDICT:` lines. Majority `NO_COMMENTS` (2 of 3) is clean. +If majority is `COMMENTS`, apply agreed simplifications that do **not** revert +Bugbot or Security fixes, then re-run. diff --git a/.dockerignore b/.dockerignore index 1c274ff64..bdf1c7d61 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,6 +5,11 @@ tmp .vscode .github .git +.harvester_cache/ +logs/ +.cursor/ +.cursorignore +*.nohup.out # This file specifies files that are *not* uploaded to Google Cloud # using gcloud. It follows the same syntax as .gitignore, with the addition of diff --git a/.env.example b/.env.example index 17ad95d27..439c7027e 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,10 @@ # Never commit your real `.env` file to version control. # ------------------------------------------------------------------------------ # Database +# Local Docker (make docker-postgres / make admin-local): +# postgresql://cre:password@127.0.0.1:5432/cre +# `make admin-local` starts that container when the URL is unreachable and +# never falls back to SQLite for the admin showcase. DEV_DATABASE_URL=postgresql://user:password@localhost:5432/opencre @@ -41,6 +45,21 @@ CRE_ENABLE_MYOPENCRE=false CRE_ENABLE_LOGIN=false +# Experimental OIE OWASP metadata agent (chat + MCP only — never CRE graph UI). +# Off by default. Uses the main app Postgres (DEV_DATABASE_URL / DATABASE_URL) +# for the agent index + pgvector. Reachable from /completion and MCP owasp_meta_* tools. +OWASP_AGENT_ENABLED=false + +# Module C.1 graph filing (Automatically linked to). Off by default — leave Module C +# decisions in decision_queue for review; set true to auto-add high-confidence links. +OIE_GRAPH_FILING_ENABLED=false + +# Nest REST API key (https://nest.owasp.org — create via GraphQL while logged in) +NEST_API_KEY= +NEST_API_BASE=https://nest.owasp.org/api/v0 +# Set to 1 to crawl all www-chapter / www-project repos during sync (slow). +OWASP_AGENT_CRAWL_REPOS=0 + # Embeddings NO_GEN_EMBEDDINGS=false @@ -52,14 +71,25 @@ GOOGLE_CLIENT_SECRET=your-client-secret GOOGLE_SECRET_JSON=path/to/secret.json LOGIN_ALLOWED_DOMAINS=example.com -# GCP +# GCP / Gemini (optional — any LiteLLM provider works for Module B) GCP_NATIVE=false GEMINI_API_KEY=your-gemini-api-key VERTEX_CHAT_MODEL=gemini-2.5-flash VERTEX_EMBED_CONTENT_MODEL=gemini-embedding-001 -# Noise / Relevance Filter (Module B) +# Other chat providers (LiteLLM). Set the key that matches CRE_NOISE_FILTER_LLM_MODEL. +# OPENAI_API_KEY=sk-... +# ANTHROPIC_API_KEY=sk-ant-... +# GROQ_API_KEY=gsk_... +# MISTRAL_API_KEY=... +# OPENROUTER_API_KEY=... + +# Noise / Relevance Filter (Module B) — LiteLLM model id (provider/model) +# Examples: gemini/gemini-2.5-flash-lite | openai/gpt-4o-mini | +# anthropic/claude-haiku-4-5 | groq/llama-3.3-70b-versatile +# Any chat model LiteLLM supports; mid-size models usually work best +# (cheap + reliable for this classify gate — avoid tiny completion stubs). CRE_NOISE_FILTER_LLM_MODEL=gemini/gemini-2.5-flash-lite CRE_NOISE_FILTER_BATCH_SIZE=10 @@ -86,3 +116,89 @@ CRE_LIBRARIAN_LINK_THRESHOLD=0.8 CRE_LIBRARIAN_BATCH_SIZE=32 CRE_LIBRARIAN_ECE_TARGET=0.10 CRE_LIBRARIAN_CONFORMAL_ALPHA=0.10 + +# --- OWASP agent answers: ingestion-time comprehension (#1158) --------------------------- +# Values below are the measured winning combination on staging (full 589-question set: +# whole-project context 57.6% correct vs 54.7% with no flags, ~20 s end to end). Every new +# feature is off in code unless its flag is set, so production behaves as before without them. +# Retrieval depth for the OWASP agent lane (default 8) and for comprehension documents. +OWASP_AGENT_TOP_K=16 +OWASP_COMPREHENSION_TOP_K=24 +# Answer from ingestion-time comprehension documents; say "not found" instead of guessing. +OWASP_COMPREHENSION_ENABLED=1 +OWASP_COMPREHENSION_ABSTAIN=1 +# How many retrieved documents the answer reads (more did not help: 24 scored the same as 10). +OWASP_COMPREHENSION_SOURCES=10 +# Search only "raw" source chunks or only "digest" documents (default all). Raw chunks alone +# score within ~3 points of all, which is the cheap full-import option. +OWASP_COMPREHENSION_DOCS=all +# Model that writes the final answer ('' = the default chat model). Gemini Pro did not help. +OWASP_COMPREHENSION_MODEL= +# Scope retrieval to the project named in the question as a score boost. Hurt (-10 points): keep 0. +OWASP_COMPREHENSION_SCOPE=0 +# Half-precision HNSW vector search. Needs migration e5f6a7b8c9d0 (the index); 1 cuts ~10 s +# of database time per question on a large embeddings table. +OWASP_PGVECTOR_HALFVEC=0 + +# Feature flags (1 = on). Measured on the 252-question sample unless noted. +# L4 whole-project context: when a question names exactly one project that fits, the model +# reads the project's whole text instead of snippets (+3 points on the full set). +OWASP_CX_LONGCTX=1 +# Max characters of one project to read whole. +OWASP_CX_LONGCTX_CHARS=2000000 +# How many named projects to read whole at once (questions naming more fall back to retrieval). +# 1 scored 59.1% on the sample; 3 scored 56.3% (counts and multi-part got worse), so keep 1. +OWASP_CX_LONGCTX_PROJECTS=1 +# Off: hybrid keyword search (-2.5 points, +5 s), rerank (-2), query rewrite (-3), verify +# (+/-3, hurts with LONGCTX), per-project (0), expand (+1, noise), count (+1, slower), +# merged single call (-13, loses citations), in-process answer cache (real traffic only). +# Final-answer model with thinking off (latency); multi-part questions answered part by part. +OWASP_CX_FAST=0 +# Thinking budget for the final answer: '' (model default), low, medium or high. Measured: low and +# medium are NOT faster than the default (~7 s); only OWASP_CX_FAST=1 (thinking off, 2.5 s but +# 49% on the sample, 54% with OWASP_COMPREHENSION_MODEL=gemini/gemini-2.5-flash-lite) is. +OWASP_CX_THINK_EFFORT= +OWASP_CX_DECOMPOSE=0 +OWASP_CX_HYBRID=0 +OWASP_CX_RERANK=0 +OWASP_CX_QUERY=0 +OWASP_CX_VERIFY=0 +OWASP_CX_PERPROJECT=0 +OWASP_CX_EXPAND=0 +OWASP_CX_COUNT=0 +OWASP_CX_MERGED=0 +OWASP_CX_CACHE=0 +# Cheap model used by the rerank/query/verify helpers when those flags are on. +OWASP_CX_HELPER_MODEL=gemini/gemini-2.5-flash-lite + +# Ingestion (scripts/comprehend_standard.py): bulk digests and chunk enrichment use a cheap +# model with thinking off; repo overviews use a stronger one. Run with --estimate first. +CRE_COMPREHENSION_BULK_MODEL=gemini/gemini-2.5-flash +CRE_COMPREHENSION_OVERVIEW_MODEL=gemini/gemini-2.5-pro +CRE_COMPREHENSION_ENRICH_MODEL=gemini/gemini-2.5-flash-lite +CRE_COMPREHENSION_THINKING=off + +# Warm the answer path in the background at server start (the first question after a restart +# otherwise takes ~30 s). 0 disables. +OWASP_WARMUP=1 + +# Remote MCP server: how many questions may be answered at the same time (each in its own thread). +OPENCRE_MCP_ASK_CONCURRENCY=8 + +# --- Accounts and roles (web chatbot and MCP share one model; see docs/roles.md) --- +# Two roles: "user" = any account on an allowed domain; "admin" = the named accounts below. +# Leave ADMIN_EMAILS unset for the old behaviour (LOGIN_ALLOWED_EMAILS is the one allowlist, all admins). +# ADMIN_EMAILS=alice@owasp.org,bob@owasp.org +# LOGIN_ALLOWED_DOMAINS=owasp.org + +# --- Per-account question limits (chatbot + MCP ask; shared through Redis) --- +# Defaults per account: 5 / minute, 40 / hour, 150 / day; admins x4. Uncomment to change. +# RATE_LIMIT_PER_MINUTE=5 +# RATE_LIMIT_PER_HOUR=40 +# RATE_LIMIT_PER_DAY=150 +# RATE_LIMIT_ADMIN_MULTIPLIER=4 +# RATE_LIMIT_EXEMPT_EMAILS=evalbot@owasp.org +# RATE_LIMIT_ENABLED=0 # switch limits off + +# Warm each web worker in the background at start so the first chat is not slow (the MCP server always does). +# OWASP_WARMUP_WEB=1 diff --git a/.gitignore b/.gitignore index 9efaa738d..6d500678a 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,8 @@ v/ .cursor/* !.cursor/rules/ !.cursor/rules/** +!.cursor/skills/ +!.cursor/skills/** .claude/ ### Frontend @@ -67,11 +69,13 @@ standards_cache.sqlite ### Docs *.md !AGENTS.md +!.cursor/skills/**/*.md !docs/faq.md !docs/api/mcp.md !docs/Mid_eval_blog_gsoc2026/module_B_mideval_blog.md !docs/Final_eval_blog_gsoc2026/module_B_final_blog.md !application/utils/librarian/README.md +!application/utils/owasp_agent/README.md !docs/gsoc_2026_module_c/*.md !docs/gsoc_2026_module_a/*.md !docs/Final_eval_blog_gsoc2026/*.md @@ -79,6 +83,8 @@ standards_cache.sqlite !docs/gsoc_2026_module_b/module_c_contract.md !docs/gsoc_2026_module_b/module_b_runbook.md !docs/gsoc_2026_module_b/final_metrics.md +!docs/oie/*.md +!scripts/oie_owasp_eval/experiments/*.md ### Dev DBDumps *.sql @@ -91,6 +97,8 @@ neo4j/ .mypy_cache tmp/ +import_telemetry.json + ### CREs dir cres/* ### Local project management tooling diff --git a/Dockerfile b/Dockerfile index eb382def0..64bf8c3ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,10 @@ FROM python:3.11 as run COPY --from=build /code /code WORKDIR /code COPY ./scripts/prod-docker-entrypoint.sh /code -RUN pip install -r requirements.txt gunicorn +# Staging Compose may pass requirements-dev.txt (Librarian ML / OIE Module C.2). +# Default stays prod requirements.txt for Heroku slug size. +ARG REQUIREMENTS_FILE=requirements.txt +RUN pip install -r "${REQUIREMENTS_FILE}" gunicorn ENV INSECURE_REQUESTS=1 ENV FLASK_CONFIG="production" diff --git a/Makefile b/Makefile index 17b8c0ea1..9281eb1f9 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .ONESHELL: -.PHONY: run test covers install-deps dev docker lint frontend clean all e2e e2e-db +.PHONY: run test covers install-deps dev dev-flask docker lint frontend clean all e2e e2e-db admin-local owasp-agent-sync owasp-agent-merge-concepts owasp-agent-synth-eval prod-run: gunicorn cre:app --log-file=- @@ -70,15 +70,39 @@ docker-postgres: start-containers: docker-neo4j docker-redis +# RQ work-horses fork after Flask/torch load. On macOS, unset device → MPS and +# CrashReporter popups from Homebrew Python.app. Force CPU unless overridden. +CRE_LIBRARIAN_DEVICE?=cpu start-worker: - . ./venv/bin/activate && FLASK_APP=`pwd`/cre.py python cre.py --start_worker + . ./venv/bin/activate && \ + OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES \ + CRE_LIBRARIAN_DEVICE="$(CRE_LIBRARIAN_DEVICE)" \ + FLASK_APP=`pwd`/cre.py python cre.py --start_worker + +# Local OIE RQ fleet (queue ``oie``). Default 10; override with OIE_WORKER_COUNT=N. +# Double-fork via scripts/start_oie_workers.py so workers survive make exit (SIGHUP). +OIE_WORKER_COUNT?=10 +start-oie-workers: + . ./venv/bin/activate && \ + OIE_WORKER_COUNT="$(OIE_WORKER_COUNT)" \ + CRE_LIBRARIAN_DEVICE="$(CRE_LIBRARIAN_DEVICE)" \ + OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES \ + python scripts/start_oie_workers.py upstream-sync: - . ./venv/bin/activate && python cre.py --upstream_sync + . ./venv/bin/activate &&\ + python cre.py --upstream_sync --cache_file "$(or $(DEV_DATABASE_URL),$(CRE_CACHE_FILE),$(CURDIR)/standards_cache.sqlite)" PORT?=5000 -dev-flask: +# Local app against Docker Postgres: migrate, upstream-sync if cre/node empty, Flask. +# Same entrypoint as admin showcase / OIE local (no SQLite). macOS AirPlay often +# holds :5000 — use PORT=5001 make dev if bind fails. +dev dev-flask admin-local: + bash ./scripts/run_admin_local.sh --port $(PORT) $(ADMIN_LOCAL_ARGS) + +# Bare Flask without Postgres bootstrap (legacy / CI helpers). Prefer `make dev`. +dev-flask-sqlite: . ./venv/bin/activate && INSECURE_REQUESTS=1 FLASK_APP=`pwd`/cre.py FLASK_CONFIG=development flask run --port $(PORT) dev-flask-docker: @@ -136,15 +160,28 @@ install-deps-typescript: install-deps: install-deps-python install-deps-typescript install-python: - virtualenv -p python3 venv + @if [ ! -d "./venv" ]; then \ + if command -v virtualenv >/dev/null 2>&1; then \ + virtualenv -p python3 venv; \ + else \ + echo "virtualenv not on PATH — using python3 -m venv"; \ + python3 -m venv venv; \ + fi; \ + fi . ./venv/bin/activate &&\ make install-deps-python &&\ playwright install # Python embeddings/scraping (prompt_client); NOT frontend e2e — keep when migrating to Cypress -install-typescript: - yarn add webpack && cd application/frontend && yarn build +# Yarn install only. Do not `yarn add` or `yarn build` here: webpack is +# already in package.json, and a local build rewrites +# application/frontend/www/bundle.js (tracked so Heroku can serve the UI +# without a Node compile step). That was dirtying contributor trees after +# `make install`. Rebuild with `make frontend` when you change the UI. +install-typescript: install-deps-typescript -install: install-typescript install-python migrate-upgrade +# Deps + Docker Postgres schema + CRE upstream graph when empty (see run_admin_local.sh). +install: install-typescript install-python + bash ./scripts/run_admin_local.sh --migrate-only $(ADMIN_LOCAL_ARGS) docker-dev: docker build -f Dockerfile-dev -t opencre-dev:$(shell git rev-parse HEAD) . @@ -185,10 +222,39 @@ oie-pipeline: [ -d "./venv" ] && . ./venv/bin/activate &&\ PYTHONPATH=. python scripts/run_oie_pipeline.py --cache_file "$(or $(CACHE_FILE),sqlite:///$(CURDIR)/standards_cache.sqlite)" $(OIE_ARGS) +# Deterministic OIE base metadata (Node then CRE document_metadata.oie). Fill-if-missing. +# Example: +# make oie-tag-base CACHE_FILE=postgresql://cre:password@127.0.0.1:5432/cre_prodclone +oie-tag-base: + [ -d "./venv" ] && . ./venv/bin/activate &&\ + PYTHONPATH=. python scripts/oie_ensure_base_metadata.py \ + --cache-file "$(or $(CACHE_FILE),$(or $(DEV_DATABASE_URL),postgresql://cre:password@127.0.0.1:5432/cre))" \ + $(OIE_TAG_ARGS) + +# Local OIE bootstrap: start cre-postgres, migrate, sync CRE hub (local|upstream). +# Examples: +# make setup-oie +# make setup-oie SETUP_OIE_ARGS='--source upstream --install-ml --calibrate' +setup-oie: + bash ./scripts/setup_oie.sh $(SETUP_OIE_ARGS) + oie-e2e-smoke: [ -d "./venv" ] && . ./venv/bin/activate &&\ PYTHONPATH=. python scripts/run_oie_e2e_smoke.py +owasp-agent-sync: + [ -d "./venv" ] && . ./venv/bin/activate &&\ + PYTHONPATH=. python scripts/owasp_agent_sync.py $(OWASP_AGENT_SYNC_ARGS) + +owasp-agent-merge-concepts: + [ -d "./venv" ] && . ./venv/bin/activate &&\ + PYTHONPATH=. python scripts/owasp_agent_merge_concepts.py $(OWASP_AGENT_MERGE_ARGS) + +# Score fixtures/owasp_agent_synth_v2.jsonl against OWASP_AGENT_DB (see application/utils/owasp_agent/README.md). +owasp-agent-synth-eval: + [ -d "./venv" ] && . ./venv/bin/activate &&\ + OWASP_AGENT_ENABLED=1 PYTHONPATH=. python scripts/owasp_agent_synth_eval.py + openapi-generate: [ -d "./venv" ] && . ./venv/bin/activate &&\ python scripts/generate_openapi.py @@ -240,6 +306,21 @@ monitor-chatbot-health-prod: --base-url "https://opencre.org" \ --output-json "$(CURDIR)/tmp/prod-chatbot-health.json" +# Scheduled OIE jobs (Postgres only). Cron `make oie-tick` every 10 minutes: each +# job runs once per slot (owasp every 10 min, cre_expansion daily) and is a no-op +# otherwise. Overrides: OIE_JOB=owasp|cre_expansion|all, OIE_FORCE=1, OIE_DRY_RUN=1. +oie-tick: + @[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \ + export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \ + python cre.py --cache_file "$$CRE_CACHE_FILE" --run_scheduled "$${OIE_JOB:-all}" \ + $${OIE_FORCE:+--scheduled_force} $${OIE_DRY_RUN:+--scheduled_dry_run} + +monitor-oie-health: + @[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \ + export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \ + export PYTHONPATH="$(CURDIR)"; \ + python scripts/monitor_oie_health.py --output-json "$(CURDIR)/tmp/oie-health.json" + verify-ga-parity-local: @[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \ export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \ diff --git a/README.md b/README.md index 032a5f89c..8e2623b8a 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ If you want to develop on OpenCRE or docker is not available in your environment ### Command Line To run outside of Docker you need to install OpenCRE. -To install this application you need Python 3.11.9, Node.js, Yarn Classic (the repository uses a Yarn v1 lockfile), and `virtualenv`. The Python version is pinned in `.python-version`; the repository does not currently pin a Node.js version. +To install this application you need Python 3.11.9, Node.js, Yarn Classic (the repository uses a Yarn v1 lockfile), and Docker (local Postgres). `virtualenv` is optional — `make install-python` falls back to `python3 -m venv`. The Python version is pinned in `.python-version`; the repository does not currently pin a Node.js version. Clone the repository: @@ -57,23 +57,18 @@ Clone the repository: git clone https://github.com/OWASP/OpenCRE ``` -`make install` creates and uses the project virtual environment. Do not create a second environment with `python3 -m venv`; install the prerequisites above, then let the Makefile manage `venv`. - -Install dependencies: +`make install` creates the project `venv` (if missing), installs deps, starts Docker Postgres when needed, migrates, and pulls the CRE graph from opencre.org when the local DB is empty. ```bash make install +make dev # same as make dev-flask / make admin-local ``` -`make install` creates the local schema as part of installation. Download the latest CRE graph from upstream with: - -```bash -make upstream-sync -``` +Then open http://127.0.0.1:5000/admin and http://127.0.0.1:5000/explorer. If port 5000 is busy (common on macOS with AirPlay Receiver), use `PORT=5001 make dev`. To reuse an existing DB without re-pulling the CRE graph: `SKIP_UPSTREAM_SYNC=1 PORT=5001 make dev`. For contributors, this is the supported local data bootstrap path. You do **not** need access to the internal OpenCRE Google Sheet to work on the -project. +project. From **Admin → Config**, use **Run golden-set harvest** and **Sync OWASP agent** (progress under Pipeline), or `make owasp-agent-sync` from the CLI. You can precompute local gap-analysis cache after imports with: @@ -81,12 +76,6 @@ You can precompute local gap-analysis cache after imports with: make backfill-gap-analysis ``` -To run CRE locally then you can do: - -```bash -make dev-flask -``` - To run the CLI application, you can run: ```bash diff --git a/application/__init__.py b/application/__init__.py index 370c70327..1c318ee65 100644 --- a/application/__init__.py +++ b/application/__init__.py @@ -49,6 +49,22 @@ def create_app(mode: str = "production", conf: any = None) -> Any: letters = string.ascii_lowercase app.secret_key = "".join(random.choice(letters) for i in range(20)) + # JWT API-token signing secret: env, or mint+persist once on install/boot. + # Tests get a stable in-process default so create_app does not touch /code. + try: + from application.utils.api_tokens.secret import ensure_api_token_jwt_secret + + if ( + mode == "test" + and not (os.environ.get("API_TOKEN_JWT_SECRET") or "").strip() + ): + os.environ["API_TOKEN_JWT_SECRET"] = ( + "test-only-api-token-jwt-secret-32bytes" + ) + ensure_api_token_jwt_secret() + except Exception: + logger.exception("API_TOKEN_JWT_SECRET bootstrap failed") + # config[mode].init_app(app) sqla.init_app(app=app) from application.web.web_main import app as app_blueprint @@ -59,10 +75,25 @@ def create_app(mode: str = "production", conf: any = None) -> Any: init_openapi(app) - CORS(app) - app.config["CORS_HEADERS"] = "Content-Type" + # Credentialed browser calls are same-origin in prod/staging; keep CORS + # narrow so cross-site pages cannot drive cookie auth. Admin mutations also + # require CSRF (see web_main.admin_imports_enabled_required). + cors_origins = [ + o.strip() + for o in (os.environ.get("CORS_ALLOWED_ORIGINS") or "").split(",") + if o.strip() + ] + if cors_origins: + CORS(app, origins=cors_origins, supports_credentials=True) + else: + CORS(app, supports_credentials=True) + app.config["CORS_HEADERS"] = "Content-Type, X-CSRF-Token" compress.init_app(app) cache.init_app(app) + if mode != "test": + from application.utils.comprehension.shared import start_background_warm_up + + start_background_warm_up(app) # no-op unless OWASP_WARMUP_WEB=1 return app diff --git a/application/cmd/cre_main.py b/application/cmd/cre_main.py index 09abb66b4..69a97e33f 100644 --- a/application/cmd/cre_main.py +++ b/application/cmd/cre_main.py @@ -880,6 +880,39 @@ def backfill_gap_analysis_only( logger.info("GA backfill complete: final_missing_pairs=%s", final_missing) +def run_scheduled_cli(args: argparse.Namespace, collection: db.Node_collection) -> int: + """``--run_scheduled``: run the selected OIE job(s); return a process exit code. + + Postgres only -- the advisory-lock lease and the run table assume it, and the + default ``standards_cache.sqlite`` must not silently become the scheduler's + database. Exit 1 when a job errored, 2 for a non-Postgres database. + """ + import json + + from application.utils.oie_scheduler.runner import run_due_jobs + + dialect = collection.session.get_bind().dialect.name + if dialect != "postgresql": + logger.error( + "--run_scheduled is Postgres-only (got %s); point --cache_file at " + "a postgresql:// URL (make docker-postgres for a local one)", + dialect, + ) + return 2 + selected = args.run_scheduled + force = bool(getattr(args, "scheduled_force", False)) + outcomes = run_due_jobs( + collection, + cache_file=args.cache_file, + jobs=None if selected == "all" else [selected], + force=force, + dry_run=bool(getattr(args, "scheduled_dry_run", False)), + trigger="manual" if force else "scheduled", + ) + print(json.dumps([o.to_dict() for o in outcomes], indent=2)) + return 1 if any(o.failed for o in outcomes) else 0 + + def run(args: argparse.Namespace) -> None: # pragma: no cover script_path = os.path.dirname(os.path.realpath(__file__)) os.path.join(script_path, "../cres") @@ -892,6 +925,14 @@ def run(args: argparse.Namespace) -> None: # pragma: no cover logger.info("Exported %s rows to %s", rows, csv_out) return + if getattr(args, "run_scheduled", ""): + import sys + + code = run_scheduled_cli(args, db_connect(args.cache_file)) + if code: + sys.exit(code) + return + if getattr(args, "run_harvester", False): import sys @@ -905,12 +946,37 @@ def run(args: argparse.Namespace) -> None: # pragma: no cover args.run_id.strip(), repos_yaml=repos_yaml, dry_run=getattr(args, "harvester_dry_run", False), + kinds=[ + k.strip() + for k in (getattr(args, "harvester_kinds", "") or "").split(",") + if k.strip() + ] + or None, ) print(summary.to_json()) if summary.status == "degraded": sys.exit(1) return + if str(getattr(args, "ingest_github", "") or "").strip(): + import sys + + from application import sqla + from application.utils.harvester.github_url_ingest import ingest_github_url + + db_connect(args.cache_file) + counts = ingest_github_url( + str(args.ingest_github).strip(), + session=sqla.session, + cache_file=args.cache_file, + branch_override=(getattr(args, "ingest_branch", "") or "").strip() or None, + keep_all=bool(getattr(args, "ingest_keep_all", False)), + model_override=(getattr(args, "ingest_model", "") or "").strip() or None, + ) + if counts.errors or (counts.knowledge + counts.uncertain) == 0: + sys.exit(1) + return + if getattr(args, "run_noise_filter", False): import sys diff --git a/application/config.py b/application/config.py index de7c0036f..1ae9f7918 100644 --- a/application/config.py +++ b/application/config.py @@ -4,9 +4,30 @@ import os +from application.utils.postgres_url import ( + is_postgres_url, + main_db_env_keys, + sqlalchemy_postgres_url, +) + basedir = os.path.abspath(os.path.dirname(__file__)) +def _sqlalchemy_uri_from_env(raw: str) -> str: + """Bind Postgres env URLs to psycopg2; leave sqlite and other schemes alone.""" + if is_postgres_url(raw): + return sqlalchemy_postgres_url(raw) + return raw + + +def _first_env_url(*keys: str) -> str: + for key in keys: + raw = (os.environ.get(key) or "").strip() + if raw: + return raw + return "" + + class Config: SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_RECORD_QUERIES = False @@ -18,8 +39,8 @@ class DevelopmentConfig(Config): DEBUG = True ENVIRONMENT = "DEVELOPMENT" CACHE_TYPE = "SimpleCache" - SQLALCHEMY_DATABASE_URI = ( - os.environ.get("DEV_DATABASE_URL") + SQLALCHEMY_DATABASE_URI = _sqlalchemy_uri_from_env( + _first_env_url(*main_db_env_keys("development")) or f'sqlite:///{os.path.join(basedir, "../standards_cache.sqlite")}' ) @@ -35,8 +56,8 @@ class ProductionConfig(Config): ENVIRONMENT = "PRODUCTION" CACHE_TYPE = "SimpleCache" CACHE_DEFAULT_TIMEOUT = 3000 - SQLALCHEMY_DATABASE_URI = ( - os.environ.get("PROD_DATABASE_URL") + SQLALCHEMY_DATABASE_URI = _sqlalchemy_uri_from_env( + _first_env_url(*main_db_env_keys("production")) or f'sqlite:///{os.path.join(basedir, "../standards_cache.sqlite")}' ) @@ -46,11 +67,12 @@ class CMDConfig(Config): def __init__(self, db_uri: str): if "://" in db_uri: - # Heroku and some tools still emit ``postgres://``; SQLAlchemy 2 expects - # ``postgresql://`` for the psycopg dialect name. - if db_uri.startswith("postgres://"): - db_uri = "postgresql://" + db_uri[len("postgres://") :] - self.SQLALCHEMY_DATABASE_URI = db_uri + # Heroku still emits ``postgres://``; SQLAlchemy 2.1 maps bare + # ``postgresql://`` to psycopg v3 — pin psycopg2 (requirements.txt). + if is_postgres_url(db_uri): + self.SQLALCHEMY_DATABASE_URI = sqlalchemy_postgres_url(db_uri) + else: + self.SQLALCHEMY_DATABASE_URI = db_uri else: # Flask-SQLAlchemy 3+ resolves non-absolute sqlite URLs against # app.instance_path, not the shell cwd. CLI --cache_file should diff --git a/application/database/db.py b/application/database/db.py index 22e7358be..9bfe1b3f1 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -200,6 +200,9 @@ class Embeddings(BaseModel): # type: ignore # vectors (legacy CSV ``embeddings`` column is dropped in that migration). # SQLite/tests: Text holding a pgvector literal ``[1.0,2.0,...]``. embedding_vec = sqla.Column(sqla.Text, nullable=False) + # Multi-lane chat routing: cre_graph | corpus | owasp_agent_meta + # (Alembic c9d0e1f2a3b4). Nullable for legacy rows until backfilled. + source_channel = sqla.Column(sqla.String, nullable=True, default=None) class GapAnalysisResults(BaseModel): @@ -209,6 +212,16 @@ class GapAnalysisResults(BaseModel): __table_args__ = (sqla.UniqueConstraint(cache_key, name="unique_cache_key_field"),) +class OieLlmCache(BaseModel): # type: ignore + """Durable librarian LLM response cache (shortlist / edition / neighbor / safety).""" + + __tablename__ = "oie_llm_cache" + kind = sqla.Column(sqla.String, primary_key=True, nullable=False) + cache_key = sqla.Column(sqla.String, primary_key=True, nullable=False) + payload_json = sqla.Column(sqla.JSON, nullable=False) + updated_at = sqla.Column(sqla.DateTime, nullable=False) + + class ImportRun(BaseModel): # type: ignore """Tracks import runs for Module C diff/staging. Step 6.""" @@ -263,6 +276,74 @@ class StagedChangeSet(BaseModel): # type: ignore created_at = sqla.Column(sqla.DateTime, nullable=False) +class IngestionTarget(BaseModel): # type: ignore + __tablename__ = "ingestion_target" + id = sqla.Column(sqla.String, primary_key=True, default=generate_uuid) + # oie_repo | import_source | owasp_agent (built-in, not persisted) + kind = sqla.Column(sqla.String, nullable=False) + name = sqla.Column(sqla.String, nullable=False) + spec_json = sqla.Column(sqla.Text, nullable=False, default="{}") + enabled = sqla.Column(sqla.Boolean, nullable=False, default=True) + created_at = sqla.Column(sqla.DateTime, nullable=False) + + +class AdminPipelineEvent(BaseModel): # type: ignore + __tablename__ = "admin_pipeline_event" + id = sqla.Column(sqla.String, primary_key=True, default=generate_uuid) + run_id = sqla.Column(sqla.String, nullable=False) + stage = sqla.Column(sqla.String, nullable=False) + status = sqla.Column(sqla.String, nullable=False) + detail = sqla.Column(sqla.Text, nullable=True) + created_at = sqla.Column(sqla.DateTime, nullable=False) + + __table_args__ = (sqla.Index("ix_admin_pipeline_event_run", "run_id"),) + + +class AdminRuntimeSetting(BaseModel): # type: ignore + """Durable admin toggles (e.g. OIE_GRAPH_FILING_ENABLED) shared by all workers.""" + + __tablename__ = "admin_runtime_setting" + key = sqla.Column(sqla.String, primary_key=True) + value = sqla.Column(sqla.String, nullable=False) + updated_at = sqla.Column(sqla.DateTime, nullable=False) + updated_by = sqla.Column(sqla.String, nullable=True) + + +class AdminRuntimeSettingAudit(BaseModel): # type: ignore + __tablename__ = "admin_runtime_setting_audit" + id = sqla.Column(sqla.String, primary_key=True, default=generate_uuid) + key = sqla.Column(sqla.String, nullable=False) + old_value = sqla.Column(sqla.String, nullable=True) + new_value = sqla.Column(sqla.String, nullable=False) + actor = sqla.Column(sqla.String, nullable=True) + created_at = sqla.Column(sqla.DateTime, nullable=False) + + __table_args__ = (sqla.Index("ix_admin_runtime_setting_audit_key", "key"),) + + +class AnswerLog(BaseModel): # type: ignore + """Every answer the system gives, with an optional thumbs up/down from the asker.""" + + __tablename__ = "answer_log" + id = sqla.Column(sqla.String, primary_key=True, default=generate_uuid) + created_at = sqla.Column(sqla.DateTime, nullable=False) + source = sqla.Column(sqla.String, nullable=False) # "mcp" | "chat" + question = sqla.Column(sqla.Text, nullable=False) + answer = sqla.Column(sqla.Text, nullable=True) + citations = sqla.Column(sqla.Text, nullable=True) # JSON list of urls + channel = sqla.Column(sqla.String, nullable=True) + flags = sqla.Column(sqla.String, nullable=True) # active OWASP_CX_* flags + latency_ms = sqla.Column(sqla.Integer, nullable=True) + rating = sqla.Column(sqla.String, nullable=True) # "up" | "down" + comment = sqla.Column(sqla.Text, nullable=True) + rated_at = sqla.Column(sqla.DateTime, nullable=True) + + __table_args__ = ( + sqla.Index("ix_answer_log_created_at", "created_at"), + sqla.Index("ix_answer_log_rating", "rating"), + ) + + class User(BaseModel): # type: ignore """Persisted account identity for OIDC login (issue #586, RFC #876 TODO 1). @@ -327,6 +408,41 @@ class McpGrants(BaseModel): # type: ignore __table_args__ = (sqla.UniqueConstraint(user_id, scope, name="uq_mcp_grants"),) +class ApiKey(BaseModel): # type: ignore + """Metadata for a minted JWT API key (``jti`` claim). Token shown once at mint.""" + + __tablename__ = "api_keys" + jti = sqla.Column(sqla.String, primary_key=True) + user_id = sqla.Column( + sqla.String, + sqla.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), + nullable=False, + ) + name = sqla.Column(sqla.String, nullable=False) + expires_at = sqla.Column(sqla.DateTime, nullable=False) + revoked_at = sqla.Column(sqla.DateTime, nullable=True) + created_at = sqla.Column(sqla.DateTime, nullable=False) + last_used_at = sqla.Column(sqla.DateTime, nullable=True) + + +class ApiKeyAuditLog(BaseModel): # type: ignore + """One authenticated API-key request (method/path/status).""" + + __tablename__ = "api_key_audit_log" + id = sqla.Column(sqla.String, primary_key=True, default=generate_uuid) + jti = sqla.Column( + sqla.String, + sqla.ForeignKey("api_keys.jti", onupdate="CASCADE", ondelete="CASCADE"), + nullable=False, + ) + method = sqla.Column(sqla.String, nullable=False) + path = sqla.Column(sqla.String, nullable=False) + status_code = sqla.Column(sqla.Integer, nullable=False) + ip = sqla.Column(sqla.String, nullable=True) + user_agent = sqla.Column(sqla.String, nullable=True) + created_at = sqla.Column(sqla.DateTime, nullable=False) + + class ArtifactIngestEvent(BaseModel): # type: ignore """Tracks one harvested artifact persisted per import run.""" @@ -451,6 +567,10 @@ class HarvestInput(BaseModel): # type: ignore status = sqla.Column( sqla.String, nullable=False, default="pending" ) # pending | processed | error + # Denormalized from payload for OIE RQ per-document B jobs (fan-out by + # artifact_id). Kept in sync by harvest_writer; payload remains the contract. + artifact_id = sqla.Column(sqla.String, nullable=False, default="") + source_repo = sqla.Column(sqla.String, nullable=True) # A's ChangeRecord (contract v0.3). JSONB on Postgres, JSON on SQLite # (dev/CI/tests); Module B parses it with schemas.ChangeRecord directly. payload = sqla.Column(sqla.JSON().with_variant(JSONB, "postgresql"), nullable=False) @@ -459,6 +579,12 @@ class HarvestInput(BaseModel): # type: ignore ) __table_args__ = ( sqla.Index("ix_harvest_input_run_status", "pipeline_run_id", "status"), + sqla.Index( + "ix_harvest_input_run_status_artifact", + "pipeline_run_id", + "status", + "artifact_id", + ), ) @@ -561,6 +687,32 @@ class DecisionQueueItem(BaseModel): # type: ignore ) +class OieRun(BaseModel): # type: ignore + """One scheduled OIE job run: the audit row behind ``/admin/oie``. + + ``id`` is the run id (``-``), derived from the schedule slot so a + replayed tick lands on the same row instead of starting a second run. + ``summary`` carries the per-stage counts of the run as JSON. + """ + + __tablename__ = "oie_run" + id = sqla.Column(sqla.String, primary_key=True) + job = sqla.Column(sqla.String, nullable=False) + slot = sqla.Column(sqla.String, nullable=False) + status = sqla.Column(sqla.String, nullable=False) # running | ok | degraded | error + trigger = sqla.Column(sqla.String, nullable=False) # scheduled | manual + dry_run = sqla.Column(sqla.Boolean, nullable=False, default=False) + attempts = sqla.Column(sqla.Integer, nullable=False, default=1) + started_at = sqla.Column(sqla.DateTime, nullable=False) + finished_at = sqla.Column(sqla.DateTime, nullable=True) + summary = sqla.Column(sqla.JSON().with_variant(JSONB, "postgresql"), nullable=True) + error = sqla.Column(sqla.Text, nullable=True) + __table_args__ = ( + sqla.UniqueConstraint("job", "slot", name="uq_oie_run_job_slot"), + sqla.Index("ix_oie_run_job_started", "job", "started_at"), + ) + + def create_import_run(source: str, version: Optional[str] = None) -> ImportRun: """Create and persist an import run record. Returns the new ImportRun.""" from datetime import datetime, timezone @@ -733,12 +885,31 @@ def get_staged_change_set(*, run_id: str) -> Optional[StagedChangeSet]: ) +def ensure_staged_change_set( + *, + run_id: str, + changeset_json: str = "[]", + staging_status: str = "pending_review", +) -> StagedChangeSet: + """Return the staged set for ``run_id``, creating an empty one if missing.""" + cs = get_staged_change_set(run_id=run_id) + if cs: + return cs + return persist_staged_change_set( + run_id=run_id, + changeset_json=changeset_json, + has_conflicts=False, + staging_status=staging_status, + ) + + def update_staged_change_set( *, run_id: str, staging_status: Optional[str] = None, has_conflicts: Optional[bool] = None, apply_error: Optional[str] = None, + changeset_json: Optional[str] = None, ) -> Optional[StagedChangeSet]: cs = get_staged_change_set(run_id=run_id) if not cs: @@ -749,6 +920,8 @@ def update_staged_change_set( cs.has_conflicts = has_conflicts if apply_error is not None: cs.apply_error = apply_error + if changeset_json is not None: + cs.changeset_json = changeset_json sqla.session.add(cs) sqla.session.commit() return cs @@ -3022,7 +3195,10 @@ def get_embeddings_for_doc(self, doc: cre_defs.Node | cre_defs.CRE) -> Embedding obj = self.session.query(CRE).filter(CRE.external_id == doc.id).first() return ( self.session.query(Embeddings) - .filter(Embeddings.cre_id == obj.id) + .filter( + Embeddings.cre_id == obj.id, + Embeddings.doc_type == cre_defs.Credoctypes.CRE.value, + ) .first() ) else: @@ -3051,6 +3227,173 @@ def get_embedding(self, object_id: str) -> Optional[Embeddings]: .all() ) + def get_embedding_by_id(self, embedding_id: str) -> Optional[Embeddings]: + if not embedding_id: + return None + return ( + self.session.query(Embeddings).filter(Embeddings.id == embedding_id).first() + ) + + def get_oie_llm_cache(self, kind: str, cache_key: str) -> Optional[Any]: + if not kind or not cache_key: + return None + row = ( + self.session.query(OieLlmCache) + .filter(OieLlmCache.kind == kind, OieLlmCache.cache_key == cache_key) + .first() + ) + if row is None: + return None + return row.payload_json + + def put_oie_llm_cache(self, kind: str, cache_key: str, payload: Any) -> bool: + if not kind or not cache_key: + return False + from datetime import datetime, timezone + + now = datetime.now(timezone.utc).replace(tzinfo=None) + row = ( + self.session.query(OieLlmCache) + .filter(OieLlmCache.kind == kind, OieLlmCache.cache_key == cache_key) + .first() + ) + if row is None: + self.session.add( + OieLlmCache( + kind=kind, + cache_key=cache_key, + payload_json=payload, + updated_at=now, + ) + ) + else: + row.payload_json = payload + row.updated_at = now + from sqlalchemy.orm.attributes import flag_modified + + flag_modified(row, "payload_json") + self.session.commit() + return True + + def get_cre_summary_embedding(self, cre_id: str) -> Optional[Embeddings]: + """Hidden librarian summary vector row (``doc_type=CRE_SUMMARY``), not hub CRE.""" + from application.utils.librarian.cre_summary import DOC_TYPE_CRE_SUMMARY + + if not cre_id: + return None + return ( + self.session.query(Embeddings) + .filter( + Embeddings.cre_id == cre_id, + Embeddings.doc_type == DOC_TYPE_CRE_SUMMARY, + ) + .first() + ) + + def get_cre_oie_summary( + self, cre_id: str + ) -> Tuple[Optional[str], Optional[str], Optional[List[float]]]: + """Return ``(summary, embedding_id, vector)`` from CRE ``oie`` metadata.""" + from application.database.pgvector_utils import parse_stored_embedding_vec + from application.utils.librarian.cre_summary import read_oie_summary + + if not cre_id: + return None, None, None + row = self.session.query(CRE).filter(CRE.id == cre_id).first() + if row is None: + return None, None, None + summary, emb_id = read_oie_summary(row.metadata_json) + if not summary: + return None, emb_id, None + vec: Optional[List[float]] = None + if emb_id: + emb = self.get_embedding_by_id(emb_id) + if emb is not None and emb.embedding_vec is not None: + parsed = parse_stored_embedding_vec(emb.embedding_vec) + if parsed: + vec = parsed + if vec is None: + emb = self.get_cre_summary_embedding(cre_id) + if emb is not None and emb.embedding_vec is not None: + parsed = parse_stored_embedding_vec(emb.embedding_vec) + if parsed: + vec = parsed + emb_id = emb.id or emb_id + return summary, emb_id, vec + + def upsert_cre_oie_summary( + self, + cre_id: str, + summary: str, + embedding: Optional[List[float]] = None, + ) -> Optional[str]: + """Persist hidden ``oie.summary`` and optional ``CRE_SUMMARY`` embedding. + + Returns the summary embedding id when a vector is stored, ``""`` when only + text is stored, or ``None`` when the CRE row is missing / write failed. + Never updates hub ``doc_type=CRE`` embeddings or ``CRE.description``. + """ + from sqlalchemy.orm.attributes import flag_modified + + from application.database.pgvector_utils import ( + require_embedding_vec_store, + to_pgvector_literal, + ) + from application.utils.librarian.cre_summary import ( + DOC_TYPE_CRE_SUMMARY, + OIE_EMBEDDING_ID_KEY, + OIE_SUMMARY_KEY, + ) + + blob = (summary or "").strip() + if not cre_id or not blob: + return None + row = self.session.query(CRE).filter(CRE.id == cre_id).first() + if row is None: + return None + + emb_id: Optional[str] = None + if embedding is not None: + vec = [float(x) for x in embedding] + if not vec: + return None + require_embedding_vec_store( + self.session.get_bind(), context="upsert_cre_oie_summary" + ) + embedding_model_id = ( + os.environ.get("CRE_EMBED_MODEL", "") or "" + ).strip() or "gemini/gemini-embedding-001" + literal = to_pgvector_literal(vec) + existing = self.get_cre_summary_embedding(cre_id) + if existing is None: + existing = Embeddings( + cre_id=cre_id, + doc_type=DOC_TYPE_CRE_SUMMARY, + embeddings_content=blob, + embedding_model_id=embedding_model_id, + embedding_dim=len(vec), + embedding_vec=literal, + ) + self.session.add(existing) + self.session.flush() + else: + existing.embeddings_content = blob + existing.embedding_model_id = embedding_model_id + existing.embedding_dim = len(vec) + existing.embedding_vec = literal + emb_id = existing.id + + meta = dict(row.metadata_json or {}) + oie = dict(meta.get("oie") or {}) if isinstance(meta.get("oie"), dict) else {} + oie[OIE_SUMMARY_KEY] = blob + if emb_id: + oie[OIE_EMBEDDING_ID_KEY] = emb_id + meta["oie"] = oie + row.metadata_json = meta + flag_modified(row, "metadata_json") + self.session.commit() + return emb_id if emb_id is not None else "" + def delete_all_embeddings(self) -> int: """ Delete every row in ``embeddings`` so a subsequent ``PromptHandler(..., load_all_embeddings=True)`` @@ -3092,7 +3435,23 @@ def add_embedding( ) require_embedding_vec_store(self.session.get_bind(), context="add_embedding") - existing = self.get_embedding(db_object.id) + # Hub CRE vs hidden CRE_SUMMARY both set cre_id; match on doc_type so + # librarian summary rows never overwrite catalog CRE embeddings. + if doctype == cre_defs.Credoctypes.CRE: + existing_row = ( + self.session.query(Embeddings) + .filter( + Embeddings.cre_id == db_object.id, + Embeddings.doc_type == cre_defs.Credoctypes.CRE.value, + ) + .first() + ) + else: + existing_row = ( + self.session.query(Embeddings) + .filter(Embeddings.node_id == db_object.id) + .first() + ) # Sole store is ``embedding_vec`` (pgvector on Postgres; Text literal # on SQLite). Never LLM re-embed here — see pgvector_utils. embedding_vec_literal = to_pgvector_literal(embeddings) @@ -3102,7 +3461,7 @@ def add_embedding( embeddings_url if embeddings_url is not None else db_object.link ) - if not existing: + if not existing_row: emb = None if doctype == cre_defs.Credoctypes.CRE: emb = Embeddings( @@ -3112,6 +3471,7 @@ def add_embedding( embedding_model_id=embedding_model_id, embedding_dim=embedding_dim, embedding_vec=embedding_vec_literal, + source_channel="cre_graph", ) else: emb = Embeddings( @@ -3122,24 +3482,25 @@ def add_embedding( embedding_model_id=embedding_model_id, embedding_dim=embedding_dim, embedding_vec=embedding_vec_literal, + source_channel="corpus", ) self.session.add(emb) self.session.commit() return emb else: logger.debug(f"knew of embedding for object {db_object.id} ,updating") - existing[0].embeddings_content = embedding_text - existing[0].embedding_model_id = embedding_model_id - existing[0].embedding_dim = embedding_dim - existing[0].embedding_vec = embedding_vec_literal + existing_row.embeddings_content = embedding_text + existing_row.embedding_model_id = embedding_model_id + existing_row.embedding_dim = embedding_dim + existing_row.embedding_vec = embedding_vec_literal if doctype != cre_defs.Credoctypes.CRE: if embeddings_url is not None: - existing[0].embeddings_url = embeddings_url - elif getattr(existing[0], "embeddings_url", None) is None: - existing[0].embeddings_url = db_object.link + existing_row.embeddings_url = embeddings_url + elif getattr(existing_row, "embeddings_url", None) is None: + existing_row.embeddings_url = db_object.link self.session.commit() - return existing + return existing_row def can_use_pgvector_similarity(self) -> bool: """True when Postgres has ``embeddings.embedding_vec`` for SQL cosine.""" @@ -3440,12 +3801,14 @@ def CREfromDB(dbcre: CRE) -> cre_defs.CRE: tags = [] if dbcre.tags: tags = list(set(dbcre.tags.split(","))) + from application.utils.librarian.cre_summary import public_cre_metadata + return cre_defs.CRE( name=dbcre.name, description=dbcre.description, id=dbcre.external_id, tags=tags, - metadata=dbcre.metadata_json or {}, + metadata=public_cre_metadata(dbcre.metadata_json or {}), ) diff --git a/application/frontend/src/components/DocumentNode/DocumentNode.tsx b/application/frontend/src/components/DocumentNode/DocumentNode.tsx index c8318e205..1326cc660 100644 --- a/application/frontend/src/components/DocumentNode/DocumentNode.tsx +++ b/application/frontend/src/components/DocumentNode/DocumentNode.tsx @@ -36,7 +36,8 @@ export interface DocumentNode { const linkTypesToNest = [TYPE_IS_PART_OF, TYPE_RELATED, TYPE_AUTOLINKED_TO]; const linkTypesExcludedInNesting = [TYPE_CONTAINS]; const linkTypesExcludedWhenNestingRelatedTo = [TYPE_RELATED, TYPE_IS_PART_OF, TYPE_CONTAINS]; -const linkDisplayOrder = [TYPE_LINKED_TO, TYPE_AUTOLINKED_TO, TYPE_CONTAINS, TYPE_IS_PART_OF, TYPE_RELATED]; +// "Related" is more useful than the hierarchy, so it comes before "child of". +const linkDisplayOrder = [TYPE_LINKED_TO, TYPE_AUTOLINKED_TO, TYPE_CONTAINS, TYPE_RELATED, TYPE_IS_PART_OF]; export const DocumentNode: FunctionComponent = ({ node, diff --git a/application/frontend/src/hooks/useCapabilities.ts b/application/frontend/src/hooks/useCapabilities.ts index 85f3cf140..53fde5d2b 100644 --- a/application/frontend/src/hooks/useCapabilities.ts +++ b/application/frontend/src/hooks/useCapabilities.ts @@ -5,11 +5,13 @@ import { useEnvironment } from './useEnvironment'; export type Capabilities = { myopencre: boolean; login: boolean; + admin?: boolean; }; const DEFAULT_CAPABILITIES: Capabilities = { myopencre: false, login: false, + admin: false, }; export const useCapabilities = () => { @@ -26,6 +28,7 @@ export const useCapabilities = () => { setCapabilities({ myopencre: Boolean(data?.myopencre), login: Boolean(data?.login), + admin: Boolean(data?.admin), }) ) .catch(() => setCapabilities(DEFAULT_CAPABILITIES)) diff --git a/application/frontend/src/index.html b/application/frontend/src/index.html index afa85e5c7..16037dcf0 100755 --- a/application/frontend/src/index.html +++ b/application/frontend/src/index.html @@ -10,6 +10,7 @@ name="description" content="Open CRE - Open Cybersecurity Requirement Expression: Link security requirements across standards and frameworks." /> + Open CRE @@ -18,4 +19,7 @@
+ <% htmlWebpackPlugin.files.js.forEach(function (file) { %> + + <% }); %> diff --git a/application/frontend/src/pages/Admin/Admin.scss b/application/frontend/src/pages/Admin/Admin.scss new file mode 100644 index 000000000..897bf4110 --- /dev/null +++ b/application/frontend/src/pages/Admin/Admin.scss @@ -0,0 +1,480 @@ +/* Admin shell — CRE blues (#2563eb / #60a5fa) + widget layout */ + +.admin-shell { + display: flex; + min-height: calc(100vh - 4rem); + background: #f8fafc; +} + +.admin-side { + width: 220px; + flex: 0 0 220px; + background: linear-gradient(180deg, #1e3a8a, #1d4ed8 55%, #2563eb); + color: #eff6ff; + padding: 1.25rem 0.75rem 1.5rem; + display: flex; + flex-direction: column; + gap: 0.25rem; +} + +.admin-side__brand { + font-weight: 700; + font-size: 1.15rem; + letter-spacing: -0.02em; + padding: 0.25rem 0.75rem 0.85rem; +} + +.admin-side__brand span { + display: block; + font-size: 0.7rem; + font-weight: 500; + text-transform: uppercase; + letter-spacing: 0.05em; + opacity: 0.7; + margin-top: 0.25rem; +} + +.admin-side__label { + font-size: 0.65rem; + text-transform: uppercase; + letter-spacing: 0.06em; + opacity: 0.55; + padding: 0.75rem 0.75rem 0.2rem; + font-weight: 600; +} + +.admin-side__nav { + display: flex; + flex-direction: column; + gap: 0.15rem; +} + +.admin-side__nav button { + appearance: none; + border: 0; + background: transparent; + color: inherit; + text-align: left; + padding: 0.55rem 0.75rem; + border-radius: 0.5rem; + font: inherit; + font-weight: 500; + cursor: pointer; + opacity: 0.85; +} + +.admin-side__nav button:hover { + background: rgba(255, 255, 255, 0.1); + opacity: 1; +} + +.admin-side__nav button.is-active { + background: rgba(255, 255, 255, 0.16); + opacity: 1; + box-shadow: inset 3px 0 0 #93c5fd; +} + +.admin-side__foot { + margin-top: auto; + font-size: 0.75rem; + opacity: 0.55; + padding: 0.75rem; + line-height: 1.35; +} + +.admin-side__profile { + margin-top: 0.5rem; + padding-top: 0.75rem; + border-top: 1px solid rgba(255, 255, 255, 0.15); +} + +.admin-main { + flex: 1; + min-width: 0; + padding: 1.5rem 1.75rem 3rem; +} + +.admin-page { + max-width: 1100px; + margin: 0 auto; +} + +.admin-page__title { + margin: 0 0 0.25rem; + font-size: 1.6rem; + letter-spacing: -0.02em; + color: #0f172a; +} + +.admin-page__sub { + margin: 0 0 1.25rem; + color: #64748b; + font-size: 0.95rem; +} + +.admin-tabs { + display: none; +} + +.admin-kpi-row { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(120px, 1fr)); + gap: 0.65rem; + margin-bottom: 1.1rem; +} + +.admin-kpi { + background: #fff; + border: 1px solid #e2e8f0; + border-radius: 0.75rem; + padding: 0.75rem 0.85rem; + box-shadow: 0 1px 0 rgba(15, 23, 42, 0.03); +} + +.admin-kpi__label { + font-size: 0.68rem; + text-transform: uppercase; + letter-spacing: 0.05em; + color: #64748b; + font-weight: 600; +} + +.admin-kpi__value { + font-size: 1.35rem; + font-weight: 700; + color: #0f172a; + margin-top: 0.15rem; +} + +.admin-kpi__value.warn { + color: #b45309; +} + +.admin-kpi__value.danger { + color: #b91c1c; +} + +.admin-kpi__value.ok { + color: #15803d; +} + +.admin-card { + background: #fff; + border: 1px solid #e2e8f0; + border-radius: 0.85rem; + padding: 1rem 1.1rem; + margin-bottom: 1rem; + box-shadow: 0 1px 0 rgba(15, 23, 42, 0.03); +} + +.admin-card h3 { + margin: 0 0 0.35rem; + font-size: 1rem; + color: #0f172a; +} + +.admin-card__sub { + margin: 0 0 0.85rem; + color: #64748b; + font-size: 0.9rem; +} + +.admin-stat-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 0.65rem; + margin-bottom: 0.75rem; +} + +.admin-stat { + background: #f8fafc; + border: 1px solid #e2e8f0; + border-radius: 0.6rem; + padding: 0.65rem 0.75rem; +} + +.admin-stat__k { + font-size: 0.68rem; + text-transform: uppercase; + letter-spacing: 0.05em; + color: #64748b; + font-weight: 600; +} + +.admin-stat__v { + font-size: 1.15rem; + font-weight: 700; + margin-top: 0.1rem; +} + +.admin-table { + width: 100%; + border-collapse: collapse; +} + +.admin-table th, +.admin-table td { + border-bottom: 1px solid #e2e8f0; + padding: 0.4rem 0.5rem; + text-align: left; + vertical-align: top; +} + +.admin-table th { + font-size: 0.7rem; + text-transform: uppercase; + letter-spacing: 0.04em; + color: #64748b; +} + +.admin-pre { + white-space: pre-wrap; + font-size: 0.85rem; + background: #f8fafc; + padding: 0.75rem; + overflow: auto; + border: 1px solid #e2e8f0; + border-radius: 0.5rem; +} + +.admin-help { + font-size: 0.85rem; + color: #64748b; +} + +.admin-toolbar { + display: flex; + align-items: center; + gap: 0.75rem; + margin: 0.5rem 0 0.75rem; +} + +.admin-muted { + font-size: 0.85rem; + color: #94a3b8; +} + +.admin-config-diff { + margin-bottom: 1.25rem !important; +} + +.admin-yaml { + width: 100%; + min-height: 16rem; + font-family: monospace; + font-size: 0.85rem; + display: block; + margin: 0.4rem 0 0.75rem; + padding: 0.5rem; +} + +.admin-strip { + display: flex; + list-style: none; + padding: 0; + margin: 0.5rem 0 1rem; + gap: 0; +} + +.admin-strip-step { + flex: 1; + padding: 0.4rem 0.5rem; + text-align: center; + border: 1px solid #e2e8f0; + background: #f8fafc; + font-size: 0.85rem; +} + +.admin-strip-step.is-done { + background: #dcfce7; +} + +.admin-strip-step.is-current { + background: #dbeafe; + font-weight: 600; +} + +.admin-strip-step.is-failed { + background: #fee2e2; +} + +.admin-map-op textarea, +.admin-page textarea { + width: 100%; + min-height: 4rem; + font-family: monospace; +} + +.admin-pill { + display: inline-flex; + align-items: center; + padding: 0.15rem 0.5rem; + border-radius: 999px; + font-size: 0.72rem; + font-weight: 600; + border: 1px solid transparent; +} + +.admin-pill.ok { + background: #dcfce7; + color: #15803d; + border-color: #bbf7d0; +} + +.admin-pill.warn { + background: #fff7ed; + color: #b45309; + border-color: #fed7aa; +} + +.admin-pill.danger { + background: #fee2e2; + color: #b91c1c; + border-color: #fecaca; +} + +.admin-pill.muted { + background: #f1f5f9; + color: #475569; + border-color: #e2e8f0; +} + +.admin-diff-match td { + background: #f0fdf4; +} + +.admin-diff-diff td { + background: #fff7ed; +} + +.admin-diff-missing td { + background: #fef2f2; +} + +.admin-diff-process_only td { + background: #eff6ff; +} + +.admin-review-item { + border: 1px solid #e2e8f0; + border-radius: 0.65rem; + margin-bottom: 0.65rem; + background: #fff; + overflow: hidden; +} + +.admin-review-item details { + border-top: 1px solid #e2e8f0; + background: #f8fafc; +} + +.admin-review-item summary { + cursor: pointer; + padding: 0.55rem 0.75rem; + font-weight: 600; + color: #2563eb; +} + +.admin-review-item__body { + padding: 0 0.75rem 0.75rem; +} + +.admin-grid-2 { + display: grid; + grid-template-columns: 1.1fr 1fr; + gap: 1rem; +} + +.admin-ingest-strip { + margin-bottom: 1rem; +} + +.admin-ingest-list { + display: flex; + flex-direction: column; + gap: 0.75rem; +} + +.admin-ingest-card { + border: 1px solid #dbeafe; + border-radius: 0.5rem; + padding: 0.75rem 0.9rem; + background: #f8fbff; +} + +.admin-ingest-card__head { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 0.5rem 0.75rem; + margin-bottom: 0.4rem; +} + +.admin-ingest-chips { + display: flex; + flex-wrap: wrap; + gap: 0.35rem; + margin-bottom: 0.55rem; +} + +.admin-chip { + display: inline-block; + font-size: 0.72rem; + font-weight: 600; + padding: 0.15rem 0.45rem; + border-radius: 0.35rem; + background: #dbeafe; + color: #1e3a8a; +} + +.admin-badge { + display: inline-block; + font-size: 0.72rem; + font-weight: 700; + padding: 0.1rem 0.45rem; + border-radius: 0.35rem; + text-transform: uppercase; + letter-spacing: 0.04em; +} + +.admin-badge--stuck { + background: #fee2e2; + color: #991b1b; +} + +.admin-ingest-progress { + margin-top: 0.25rem; +} + +.admin-ingest-stuck-reason { + margin: 0.4rem 0 0; + color: #991b1b; +} + +.admin-linkish { + appearance: none; + border: 0; + background: none; + color: #1d4ed8; + font: inherit; + font-weight: 600; + cursor: pointer; + padding: 0; + text-decoration: underline; +} + +@media (max-width: 900px) { + .admin-shell { + flex-direction: column; + } + + .admin-side { + width: 100%; + flex: none; + } + + .admin-grid-2, + .admin-stat-grid { + grid-template-columns: 1fr; + } +} diff --git a/application/frontend/src/pages/Admin/Admin.test.tsx b/application/frontend/src/pages/Admin/Admin.test.tsx new file mode 100644 index 000000000..0ce005d03 --- /dev/null +++ b/application/frontend/src/pages/Admin/Admin.test.tsx @@ -0,0 +1,1037 @@ +import { fireEvent, render, waitFor } from '@testing-library/react'; +import React from 'react'; +import { MemoryRouter } from 'react-router-dom'; + +import { useCapabilities } from '../../hooks/useCapabilities'; +import { useUser } from '../../hooks/useUser'; +import { Admin } from './Admin'; + +jest.mock('../../hooks/useEnvironment', () => ({ + useEnvironment: () => ({ name: 'test', apiUrl: '/rest/v1' }), +})); +jest.mock('../../hooks/useUser'); +jest.mock('../../hooks/useCapabilities'); + +const mockUser = useUser as jest.Mock; +const mockCaps = useCapabilities as jest.Mock; + +function jsonRes(body: unknown, status = 200) { + return Promise.resolve({ + ok: status >= 200 && status < 300, + status, + statusText: 'OK', + text: () => Promise.resolve(JSON.stringify(body)), + json: () => Promise.resolve(body), + }); +} + +function loggedIn() { + mockUser.mockReturnValue({ + user: 'u', + isLoggedIn: true, + loading: false, + login: jest.fn(), + logout: jest.fn(), + }); + mockCaps.mockReturnValue({ + capabilities: { myopencre: true, login: true, admin: true }, + loading: false, + }); +} + +function mockRuns() { + return jsonRes({ runs: [{ id: 'r1', source: 'asvs', staging_status: 'pending_review' }] }); +} + +describe('Admin', () => { + beforeEach(() => { + mockCaps.mockReturnValue({ + capabilities: { myopencre: false, login: true, admin: false }, + loading: false, + }); + }); + + afterEach(() => jest.clearAllMocks()); + + it('hides admin tabs when import capability is off', () => { + loggedIn(); + mockCaps.mockReturnValue({ + capabilities: { myopencre: true, login: true, admin: false }, + loading: false, + }); + const { getByText, queryByText } = render( + + + + ); + expect(getByText(/CRE_ALLOW_IMPORT/)).toBeTruthy(); + expect(queryByText('Overview')).toBeNull(); + }); + + it('asks anonymous users to log in', () => { + mockUser.mockReturnValue({ + user: null, + isLoggedIn: false, + loading: false, + login: jest.fn(), + logout: jest.fn(), + }); + const { getByText } = render( + + + + ); + expect(getByText(/Sign in to continue/)).toBeTruthy(); + }); + + it('shows job schedules and saves an edited interval', async () => { + loggedIn(); + const row = { + job: 'owasp', + description: 'OWASP agent sync', + default_interval_minutes: 10, + interval_minutes: 10, + overridden: false, + enabled: true, + next_expected: '2026-10-19T12:10:00+00:00', + last_run: { status: 'ok', started_at: '2026-10-19T12:00:00' }, + }; + const calls: { url: string; init?: any }[] = []; + (global as any).fetch = jest.fn((url: string, init?: any) => { + calls.push({ url: String(url), init }); + if (String(url).endsWith('/admin/schedule/owasp') && init?.method === 'PUT') { + return jsonRes({ jobs: [{ ...row, interval_minutes: 30, overridden: true }] }); + } + if (String(url).endsWith('/admin/schedule')) { + return jsonRes({ jobs: [row] }); + } + return jsonRes({}); + }); + const { getByText, findByLabelText, findByText } = render( + + + + ); + fireEvent.click(getByText('Schedules')); + const input = (await findByLabelText('owasp interval minutes')) as HTMLInputElement; + expect(input.value).toBe('10'); + fireEvent.change(input, { target: { value: '30' } }); + fireEvent.click(getByText('Save')); + expect(await findByText('owasp schedule saved')).toBeTruthy(); + const put = calls.find((c) => c.init?.method === 'PUT'); + expect(JSON.parse(put!.init.body)).toEqual({ interval_minutes: 30, enabled: true }); + }); + + it('edits project authority weights and saves only the changed ones', async () => { + loggedIn(); + const calls: { url: string; init?: any }[] = []; + (global as any).fetch = jest.fn((url: string, init?: any) => { + calls.push({ url: String(url), init }); + if (String(url).endsWith('/admin/authority') && init?.method === 'PUT') { + return jsonRes({ weights: { 'www-project-ai-exchange': 1.2 } }); + } + if (String(url).endsWith('/admin/authority')) { + return jsonRes({ + projects: [ + { slug: 'www-project-ai-exchange', name: 'ai exchange', weight: 1 }, + { slug: 'AISVS', name: 'aisvs', weight: 1 }, + ], + }); + } + return jsonRes({}); + }); + const { getByText, findByLabelText, findByText } = render( + + + + ); + fireEvent.click(getByText('Authority')); + const input = (await findByLabelText('www-project-ai-exchange weight')) as HTMLInputElement; + fireEvent.change(input, { target: { value: '1.2' } }); + fireEvent.click(getByText(/^Save/)); + expect(await findByText(/1 project weight\(s\) saved/)).toBeTruthy(); + const put = calls.find((c) => c.init?.method === 'PUT'); + expect(JSON.parse(put!.init.body)).toEqual({ weights: { 'www-project-ai-exchange': 1.2 } }); + }); + + it('loads import runs and links to MyOpenCRE', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + if (String(url).includes('/admin/imports/runs')) { + return mockRuns(); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Review')); + await findByText('asvs'); + fireEvent.click(getByText('MyOpenCRE')); + expect(getByText('Open MyOpenCRE').closest('a')?.getAttribute('href')).toBe('/myopencre'); + }); + + it('loads agent status on the dashboard', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + if (String(url).includes('/admin/dashboard')) { + return jsonRes({ + running: [], + failed: [], + import_runs: [], + needs_attention: { staged_review: 2, failed_24h: 0, blocked: 1, ingest_errors: 0 }, + review_pulse: { + accepted_as_is_pct: 40, + adjusted_pct: 35, + rejected_pct: 25, + pending: 3, + }, + agent: { + enabled: true, + writes_cre_graph: false, + demo_path: '/chatbot', + package_present: false, + }, + }); + } + return jsonRes({}); + }); + const { getByText, findByText, queryByRole } = render( + + + + ); + await findByText(/writes CRE graph: false/); + expect(getByText('Needs attention')).toBeTruthy(); + expect(getByText('Review pulse')).toBeTruthy(); + expect(getByText('40%')).toBeTruthy(); + expect(getByText('Open chat demo').closest('a')?.getAttribute('href')).toBe('/chatbot'); + expect(queryByRole('button', { name: 'OWASP agent' })).toBeNull(); + fireEvent.click(getByText('Open review')); + expect(await findByText(/Approve, deny, or relink/)).toBeTruthy(); + }); + + it('reviews links per run and loads the changeset graph', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/changeset/graph')) { + return jsonRes({ + nodes: [{ id: 'n1', label: 'ASVS 1.1', type: 'Standard', status: 'updated' }], + edges: [], + }); + } + if (u.includes('/links') && init?.method === 'POST') { + return jsonRes({ + run_id: 'r1', + links: [ + { + op_index: 0, + link_index: 0, + added: { op: 'add_control', name: 'ASVS', section: '1.1' }, + linked_to: { id: '123', name: 'Auth' }, + decision: 'approved', + }, + ], + }); + } + if (u.includes('/links') && (!init || !init.method || init.method === 'GET')) { + return jsonRes({ + run_id: 'r1', + links: [ + { + op_index: 0, + link_index: 0, + added: { op: 'add_control', name: 'ASVS', section: '1.1' }, + linked_to: { id: '123', name: 'Auth' }, + decision: 'pending', + }, + ], + }); + } + if (u.includes('/changeset')) { + return jsonRes({ run_id: 'r1', changeset: [{ op: 'add_control', document: {} }] }); + } + if (u.includes('/admin/imports/runs')) { + return mockRuns(); + } + return jsonRes({}); + }); + const { getByText, findByText, getAllByText } = render( + + + + ); + fireEvent.click(getByText('Review')); + await findByText('asvs'); + fireEvent.click(getByText('Graph')); + expect(await findByText('ASVS 1.1')).toBeTruthy(); + fireEvent.click(getByText('Changeset')); + expect(await findByText('Staged links')).toBeTruthy(); + const accepts = getAllByText('Accept'); + fireEvent.click(accepts[accepts.length - 1]); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('/links'), + expect.objectContaining({ method: 'POST' }) + ) + ); + }); + + it('shows empty pipeline state', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + const u = String(url); + if (u.includes('/admin/ingest/status')) { + return jsonRes({ ingestions: [], active_count: 0, stuck_count: 0 }); + } + if (u.includes('/admin/pipeline')) { + return jsonRes({ + import_runs: [], + events: [], + latest_strip: [ + { id: 'queued', label: 'Queued', state: 'current' }, + { id: 'pending_review', label: 'Review', state: 'idle' }, + ], + oie: { unconsumed: 0, recent: [], knowledge: [] }, + }); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Pipeline')); + expect(await findByText('No pipeline runs yet.')).toBeTruthy(); + expect(getByText('Queued')).toBeTruthy(); + expect(getByText('New')).toBeTruthy(); + }); + + it('renders active ingestions with config progress and ETA', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + const u = String(url); + if (u.includes('/admin/ingest/status')) { + return jsonRes({ + active_count: 1, + stuck_count: 1, + ingestions: [ + { + run_id: 'run-abc-123', + source: 'owasp-asvs', + phase: 'oie/started', + stuck: true, + stuck_reason: 'remaining unchanged for 12m', + config: { + skip_c1: true, + max_repos: 0, + spec: { owner: 'OWASP', repo: 'ASVS', lane: 'owasp_agent' }, + }, + progress: { + harvest_pending: 2, + harvest_processed: 8, + knowledge_unconsumed: 1, + knowledge_consumed: 4, + rq_queued: 3, + rq_started: 1, + rq_failed: 0, + remaining: 6, + }, + eta: { note: '~40m left', eta_utc: '2026-10-13T12:00:00Z' }, + }, + ], + }); + } + if (u.includes('/admin/pipeline')) { + return jsonRes({ + import_runs: [], + events: [], + latest_strip: [{ id: 'queued', label: 'Queued', state: 'current' }], + oie: { unconsumed: 0, recent: [], knowledge: [] }, + }); + } + return jsonRes({}); + }); + const { getByText, findByText, getAllByText } = render( + + + + ); + fireEvent.click(getByText('Pipeline')); + expect(await findByText('owasp-asvs')).toBeTruthy(); + expect(getByText('skip_c1')).toBeTruthy(); + expect(getByText(/~40m left/)).toBeTruthy(); + expect(getAllByText('Stuck').length).toBeGreaterThan(0); + expect(getByText(/remaining unchanged/)).toBeTruthy(); + }); + + it('shows config restart instructions', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + const u = String(url); + if (u.includes('/admin/config/diff')) { + return jsonRes({ + diff: [ + { + key: 'CRE_ALLOW_IMPORT', + status: 'match', + process: '1', + dotenv: '1', + }, + ], + }); + } + if (u.includes('/admin/config')) { + return jsonRes({ + writable: false, + restart_instructions: 'HTTP cannot change process env.', + config: [ + { + key: 'CRE_ALLOW_IMPORT', + value: '1', + help_text: 'kill', + help_url: 'https://example.test', + secret: false, + }, + ], + }); + } + if (u.includes('/admin/repos.yaml')) { + return jsonRes({ yaml: 'sources: []\n', source: 'repos.yaml:abc' }); + } + if (String(url).includes('/rest/v1/api-keys')) { + return jsonRes({ keys: [] }); + } + return jsonRes({}); + }); + const { getByText, findByText, getAllByText } = render( + + + + ); + fireEvent.click(getByText('Config values')); + expect(await findByText('Environment vs .env')).toBeTruthy(); + expect(await findByText('HTTP cannot change process env.')).toBeTruthy(); + expect(getAllByText('CRE_ALLOW_IMPORT').length).toBeGreaterThan(0); + }); + + it('mints an API key from Config Manage API keys dialog', async () => { + loggedIn(); + let keys: any[] = []; + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/rest/v1/api-keys') && init?.method === 'POST') { + const body = JSON.parse(String(init.body || '{}')); + keys = [ + { + jti: 'abcd1234-eeee-ffff-0000-111122223333', + name: body.name || 'ci', + expires_at: '2026-12-01T00:00:00Z', + revoked_at: null, + created_at: '2026-10-11T00:00:00Z', + last_used_at: null, + }, + ]; + return jsonRes( + { + token: 'eyJhbGciOiJIUzI1NiJ9.test-token', + jti: keys[0].jti, + name: keys[0].name, + expires_at: keys[0].expires_at, + }, + 201 + ); + } + if (u.includes('/rest/v1/api-keys')) { + return jsonRes({ keys }); + } + if (u.includes('/admin/config')) { + return jsonRes({ config: [], restart_instructions: '' }); + } + if (u.includes('/admin/repos.yaml')) { + return jsonRes({ yaml: 'sources: []\n', source: 'repos.yaml:abc' }); + } + return jsonRes({}); + }); + const { getByText, getByLabelText, findByText, findByLabelText } = render( + + + + ); + fireEvent.click(getByText('Config values')); + fireEvent.click(await findByText('Manage API keys')); + expect(await findByText('No API keys yet.')).toBeTruthy(); + fireEvent.change(getByLabelText('API key name'), { target: { value: 'staging-ci' } }); + fireEvent.click(getByText('Mint API key')); + expect(await findByText('API key minted')).toBeTruthy(); + const tokenEl = await findByLabelText('minted API token'); + expect(tokenEl.textContent).toContain('eyJhbGciOiJIUzI1NiJ9.test-token'); + expect(await findByText('staging-ci')).toBeTruthy(); + expect(await findByText('2026-12-01T00:00:00Z')).toBeTruthy(); + expect((global as any).fetch).toHaveBeenCalledWith( + '/rest/v1/api-keys', + expect.objectContaining({ method: 'POST', credentials: 'include' }) + ); + }); + + it('toggles Module C.1 graph filing from Config', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/admin/config') && init?.method === 'PUT') { + return jsonRes({ + applied: ['OIE_GRAPH_FILING_ENABLED'], + rejected: [], + config: [ + { + key: 'OIE_GRAPH_FILING_ENABLED', + value: '1', + writable: true, + help_text: 'Module C.1', + help_url: '#', + secret: false, + }, + ], + }); + } + if (u.includes('/admin/config')) { + return jsonRes({ + writable: true, + config: [ + { + key: 'OIE_GRAPH_FILING_ENABLED', + value: '0', + writable: true, + help_text: 'Module C.1', + help_url: '#', + secret: false, + }, + ], + }); + } + if (u.includes('/admin/repos.yaml')) { + return jsonRes({ yaml: 'sources: []\n', source: 'repos.yaml:abc' }); + } + if (u.includes('/admin/agent/status')) { + return jsonRes({ enabled: true, db_url: null, counts: {} }); + } + return jsonRes({}); + }); + const { getByText, findByText, findByRole } = render( + + + + ); + fireEvent.click(getByText('Config values')); + expect(await findByText(/C\.1 auto-file high-confidence links \(off/)).toBeTruthy(); + const toggle = await findByRole('checkbox'); + fireEvent.click(toggle); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('/admin/config'), + expect.objectContaining({ + method: 'PUT', + body: JSON.stringify({ updates: { OIE_GRAPH_FILING_ENABLED: '1' } }), + }) + ) + ); + expect(await findByText(/C\.1 auto-filing on/)).toBeTruthy(); + }); + + it('runs golden-set harvest and OWASP agent sync from Config', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/admin/config') && (!init || init.method === 'GET' || !init.method)) { + return jsonRes({ + restart_instructions: 'HTTP cannot change process env.', + config: [{ key: 'OIE_GRAPH_FILING_ENABLED', value: '0', writable: true }], + }); + } + if (u.includes('/admin/repos.yaml')) { + return jsonRes({ yaml: 'sources: []\n', source: 'repos.yaml:abc' }); + } + if (u.includes('/admin/agent/status')) { + return jsonRes({ + enabled: true, + db_url: 'postgresql://cre:***@127.0.0.1:5432/cre', + counts: { projects: 1 }, + last_sync: '2026-01-01T00:00:00+00:00', + }); + } + if (u.includes('/admin/upstream-sync-and-ingest')) { + return jsonRes({ + source: 'repos.yaml:deadbeef', + async: true, + run_id: 'u1', + upstream_sync: true, + }); + } + if (u.includes('/admin/ingest/start')) { + return jsonRes({ source: 'repos.yaml:deadbeef', async: true, run_id: 'g1' }); + } + if (u.includes('/admin/agent/sync')) { + return jsonRes({ + run_id: 'a1b2c3d4-eeee', + async: true, + skip_nest: true, + source: 'owasp-agent-sync', + }); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Config values')); + expect(await findByText(/End-to-end bootstrap/)).toBeTruthy(); + fireEvent.click(getByText(/^Import upstream CRE \+ harvest$/)); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('/admin/upstream-sync-and-ingest'), + expect.objectContaining({ + method: 'POST', + body: JSON.stringify({ packaged: true, max_repos: 50 }), + }) + ) + ); + expect(await findByText(/Upstream CRE sync \+ harvest started/)).toBeTruthy(); + fireEvent.click(getByText('Run golden-set harvest')); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('/admin/ingest/start'), + expect.objectContaining({ + method: 'POST', + body: JSON.stringify({ packaged: true, max_repos: 50 }), + }) + ) + ); + expect(await findByText(/Golden-set harvest started/)).toBeTruthy(); + fireEvent.click(getByText('Sync OWASP agent')); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('/admin/agent/sync'), + expect.objectContaining({ method: 'POST' }) + ) + ); + expect(await findByText(/OWASP agent sync started/)).toBeTruthy(); + }); + + it('shows agent disabled when the flag is off', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + if (String(url).includes('/admin/dashboard')) { + return jsonRes({ + running: [], + failed: [], + import_runs: [], + agent: { enabled: false, writes_cre_graph: false, db_url: null }, + }); + } + return jsonRes({}); + }); + const { findByText } = render( + + + + ); + expect(await findByText(/Enabled: false/)).toBeTruthy(); + }); + + it('shows pipeline error state', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + const u = String(url); + if (u.includes('/admin/ingest/status')) { + return jsonRes({ ingestions: [], active_count: 0, stuck_count: 0 }); + } + if (u.includes('/admin/pipeline')) { + return jsonRes({ description: 'pipeline down' }, 500); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Pipeline')); + expect(await findByText(/pipeline down/)).toBeTruthy(); + }); + + it('accepts and discards a staged run', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.endsWith('/accept') || u.endsWith('/discard') || u.includes('apply?dry_run')) { + return jsonRes({ run_id: 'r1', staging_status: 'ok' }); + } + if (u.includes('/impact')) { + return jsonRes({ + run_id: 'r1', + operation_count: 1, + impacted_standard_names: [], + impacted_cre_external_ids: [], + warnings: ['Skipped 1 operation(s) with empty standard keys'], + }); + } + if (u.includes('/admin/imports/runs')) { + return mockRuns(); + } + return jsonRes({}); + }); + const { getByText, findByText, findAllByText } = render( + + + + ); + fireEvent.click(getByText('Review')); + await findByText('asvs'); + fireEvent.click(getByText('Impact')); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('/impact'), + expect.objectContaining({ method: 'GET' }) + ) + ); + expect(await findAllByText(/empty standard keys/)).toBeTruthy(); + fireEvent.click(getByText('Accept')); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('/accept'), + expect.objectContaining({ method: 'POST' }) + ) + ); + expect(await findByText(/accept →/i)).toBeTruthy(); + fireEvent.click(getByText('Discard')); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('/discard'), + expect.objectContaining({ method: 'POST' }) + ) + ); + fireEvent.click(getByText('Dry-run')); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('apply?dry_run=1'), + expect.objectContaining({ method: 'POST' }) + ) + ); + }); + + it('surfaces a failed changeset load', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + const u = String(url); + if (u.includes('/changeset')) { + return jsonRes({ description: 'run not found' }, 404); + } + if (u.includes('/admin/imports/runs')) { + return mockRuns(); + } + return jsonRes({}); + }); + const { getByText, findByText, queryByText } = render( + + + + ); + fireEvent.click(getByText('Review')); + await findByText('asvs'); + fireEvent.click(getByText('Changeset')); + expect(await findByText(/run not found/)).toBeTruthy(); + expect(queryByText('Staged links')).toBeNull(); + }); + + it('saves repos.yaml and expands an org from config', async () => { + loggedIn(); + let savedBody: any; + let expandBody: any; + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/expand-org') && init?.method === 'POST') { + expandBody = JSON.parse(String(init.body || '{}')); + return jsonRes({ + yaml: 'sources:\n - github.com/OWASP/\n', + added: 1, + skipped: 0, + source_url: 'github.com/OWASP/', + source: 'repos.yaml:newhash12ab', + }); + } + if (u.includes('/admin/repos.yaml') && init?.method === 'PUT') { + savedBody = JSON.parse(String(init.body || '{}')); + return jsonRes({ + yaml: savedBody.yaml, + source: 'repos.yaml:savedhash12', + saved: true, + }); + } + if (u.includes('/admin/repos.yaml')) { + return jsonRes({ + yaml: 'repositories:\n - id: owasp-asvs\n', + source: 'repos.yaml:abc123abc123', + }); + } + if (u.includes('/admin/config')) { + return jsonRes({ config: [], restart_instructions: '' }); + } + return jsonRes({}); + }); + const { getByText, getByLabelText, getByPlaceholderText, findByText } = render( + + + + ); + fireEvent.click(getByText('Config values')); + expect(await findByText('targets.yaml')).toBeTruthy(); + fireEvent.change(getByLabelText('repos.yaml'), { + target: { value: 'repositories:\n - id: custom\n' }, + }); + fireEvent.click(getByText('Save repos.yaml')); + expect(await findByText(/Saved repos.yaml:savedhash12/)).toBeTruthy(); + expect(savedBody.yaml).toContain('custom'); + fireEvent.change(getByPlaceholderText('GitHub org'), { target: { value: 'OWASP' } }); + fireEvent.click(getByText('Add org')); + expect(await findByText(/Saved source github.com\/OWASP/)).toBeTruthy(); + expect(expandBody.owner).toBe('OWASP'); + }); + + it('surfaces an inaccessible GitHub source immediately', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/expand-org') && init?.method === 'POST') { + return jsonRes({ description: 'GitHub source is not accessible: github.com/NoSuchOrgCcdd/' }, 400); + } + if (u.includes('/admin/repos.yaml')) { + return jsonRes({ yaml: 'sources: []\n', source: 'repos.yaml:abc123abc123' }); + } + if (u.includes('/admin/config')) { + return jsonRes({ config: [], restart_instructions: '' }); + } + return jsonRes({}); + }); + const { getByText, getByPlaceholderText, findByText } = render( + + + + ); + fireEvent.click(getByText('Config values')); + await findByText('targets.yaml'); + fireEvent.change(getByPlaceholderText('GitHub org'), { target: { value: 'NoSuchOrgCcdd' } }); + fireEvent.click(getByText('Add org')); + expect(await findByText(/not accessible: github.com\/NoSuchOrgCcdd\//)).toBeTruthy(); + }); + + it('starts a packaged ingest from Pipeline New', async () => { + loggedIn(); + let startBody: any; + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/admin/ingest/start') && init?.method === 'POST') { + startBody = JSON.parse(String(init.body || '{}')); + return jsonRes({ run_id: 'r-new', source: 'repos.yaml:abc', dry_run: true }); + } + if (u.includes('/admin/ingest/status')) { + return jsonRes({ ingestions: [], active_count: 0, stuck_count: 0 }); + } + if (u.includes('/admin/pipeline')) { + return jsonRes({ + import_runs: [], + events: [], + latest_strip: [{ id: 'queued', label: 'Queued', state: 'current' }], + oie: { unconsumed: 0, recent: [], knowledge: [] }, + }); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Pipeline')); + await findByText('No pipeline runs yet.'); + fireEvent.click(getByText('New')); + fireEvent.click(getByText('Start')); + expect(await findByText(/Started repos.yaml:abc/)).toBeTruthy(); + expect(startBody.packaged).toBe(true); + }); + + it('rejects empty drop-last and empty add-target', async () => { + loggedIn(); + const fetchMock = jest.fn((url: string) => { + if (String(url).includes('/admin/imports/runs')) { + return jsonRes({ runs: [] }); + } + if (String(url).includes('/admin/repos.yaml')) { + return jsonRes({ yaml: 'sources: []\n', source: 'repos.yaml:abc' }); + } + if (String(url).includes('/admin/config')) { + return jsonRes({ config: [], restart_instructions: '' }); + } + return jsonRes({}); + }); + (global as any).fetch = fetchMock; + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Review')); + expect(await findByText('Drop last')).toBeTruthy(); + fireEvent.click(getByText('Drop last')); + expect(await findByText(/source is required/)).toBeTruthy(); + fireEvent.click(getByText('Config values')); + fireEvent.click(getByText('Add target')); + fireEvent.click(getByText('Add')); + expect(await findByText(/owner and repo are required/)).toBeTruthy(); + const posts = fetchMock.mock.calls.filter((call: unknown[]) => { + const init = call[1] as RequestInit | undefined; + return init?.method === 'POST'; + }); + expect(posts).toEqual([]); + }); + + it('surfaces a failed graph load', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + const u = String(url); + if (u.includes('/changeset/graph')) { + return jsonRes({ description: 'graph unavailable' }, 500); + } + if (u.includes('/admin/imports/runs')) { + return mockRuns(); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Review')); + await findByText('asvs'); + fireEvent.click(getByText('Graph')); + expect(await findByText(/graph unavailable/)).toBeTruthy(); + }); + + it('adds a target from the config lightbox', async () => { + loggedIn(); + let addBody: any; + let savedYaml: string | undefined; + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/add-repo') && init?.method === 'POST') { + addBody = JSON.parse(String(init.body || '{}')); + return jsonRes({ + yaml: 'repositories:\n - id: owasp-asvs\n', + added: 'owasp-asvs', + source: 'repos.yaml:added12ab', + }); + } + if (u.includes('/admin/repos.yaml') && init?.method === 'PUT') { + savedYaml = JSON.parse(String(init.body || '{}')).yaml; + return jsonRes({ yaml: savedYaml, source: 'repos.yaml:saved12ab', saved: true }); + } + if (u.includes('/admin/repos.yaml')) { + return jsonRes({ yaml: 'sources: []\n', source: 'repos.yaml:abc' }); + } + if (u.includes('/admin/config')) { + return jsonRes({ config: [], restart_instructions: '' }); + } + return jsonRes({}); + }); + const { getByText, getByPlaceholderText, findByText } = render( + + + + ); + fireEvent.click(getByText('Config values')); + await findByText('targets.yaml'); + fireEvent.click(getByText('Add target')); + fireEvent.change(getByPlaceholderText('OWASP'), { target: { value: 'OWASP' } }); + fireEvent.change(getByPlaceholderText('ASVS'), { target: { value: 'ASVS' } }); + fireEvent.click(getByText('Add')); + expect(await findByText(/Added target owasp-asvs/)).toBeTruthy(); + expect(addBody.owner).toBe('OWASP'); + expect(addBody.repo).toBe('ASVS'); + expect(addBody.cron).toBe('0 2 * * *'); + expect(savedYaml).toContain('owasp-asvs'); + }); + + it('starts a one-off single repository from Pipeline New', async () => { + loggedIn(); + let startBody: any; + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/add-repo') && init?.method === 'POST') { + return jsonRes({ yaml: 'repositories:\n - id: one-off\n', added: 'one-off' }); + } + if (u.includes('/admin/ingest/start') && init?.method === 'POST') { + startBody = JSON.parse(String(init.body || '{}')); + return jsonRes({ run_id: 'r-one', source: 'one-off', dry_run: true }); + } + if (u.includes('/admin/ingest/status')) { + return jsonRes({ ingestions: [], active_count: 0, stuck_count: 0 }); + } + if (u.includes('/admin/pipeline')) { + return jsonRes({ + import_runs: [], + events: [], + latest_strip: [{ id: 'queued', label: 'Queued', state: 'current' }], + oie: { unconsumed: 0, recent: [], knowledge: [] }, + }); + } + return jsonRes({}); + }); + const { getByText, getByPlaceholderText, findByText } = render( + + + + ); + fireEvent.click(getByText('Pipeline')); + await findByText('No pipeline runs yet.'); + fireEvent.click(getByText('New')); + fireEvent.click(getByText('Single repository')); + fireEvent.change(getByPlaceholderText('OWASP'), { target: { value: 'OWASP' } }); + fireEvent.change(getByPlaceholderText('ASVS'), { target: { value: 'ASVS' } }); + fireEvent.click(getByText('Start')); + expect(await findByText(/Started one-off/)).toBeTruthy(); + expect(startBody.yaml).toContain('one-off'); + expect(startBody.name).toBe('owasp-asvs'); + }); + + it('shows an empty graph-management links table', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + const u = String(url); + if (u.includes('/links')) { + return jsonRes({ run_id: 'r1', links: [] }); + } + if (u.includes('/changeset')) { + return jsonRes({ run_id: 'r1', changeset: [] }); + } + if (u.includes('/admin/imports/runs')) { + return mockRuns(); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Review')); + await findByText('asvs'); + fireEvent.click(getByText('Changeset')); + expect(await findByText('No staged links on this run.')).toBeTruthy(); + }); +}); diff --git a/application/frontend/src/pages/Admin/Admin.tsx b/application/frontend/src/pages/Admin/Admin.tsx new file mode 100644 index 000000000..48c65ca45 --- /dev/null +++ b/application/frontend/src/pages/Admin/Admin.tsx @@ -0,0 +1,2481 @@ +import './Admin.scss'; + +import React, { useCallback, useEffect, useState } from 'react'; +import { Link } from 'react-router-dom'; +import { Button, Checkbox, Form, Message, Modal, Header as SUIHeader, Table } from 'semantic-ui-react'; + +import { useCapabilities, useEnvironment } from '../../hooks'; +import { useUser } from '../../hooks/useUser'; + +type Tab = + | 'dashboard' + | 'config' + | 'pipeline' + | 'oie' + | 'schedule' + | 'authority' + | 'graph' + | 'workbench' + | 'myopencre' + | 'profile'; + +const TAB_META: Record = { + dashboard: { label: 'Overview', sub: 'Queue health, needs-attention, review pulse.' }, + config: { label: 'Config values', sub: 'Process env vs .env — admin only.' }, + pipeline: { label: 'Pipeline', sub: 'Jobs, stage logs, ingest.' }, + oie: { label: 'OIE queue', sub: 'RQ / OIE worker status.' }, + authority: { + label: 'Authority', + sub: 'Feature or soften projects. 1 = neutral; above 1 = always list the closest passage among the references; below 1 = rank lower; 0 = hidden.', + }, + schedule: { label: 'Schedules', sub: 'How often the ingestion jobs run — change or switch off.' }, + graph: { label: 'Review', sub: 'Staged links — expand section text, accept/reject.' }, + workbench: { label: 'Workbench', sub: 'Accepted targets and re-ingest.' }, + myopencre: { label: 'MyOpenCRE', sub: 'Personal CRE workspace (admin shortcut).' }, + profile: { label: 'Profile', sub: 'Operator account — admin session required.' }, +}; + +type RepoForm = { + id: string; + owner: string; + repo: string; + branch: string; + include: string; + exclude: string; + strategy: string; + max_tokens: string; + overlap_tokens: string; + mode: string; + interval_minutes: string; + cron: string; +}; + +function emptyRepo(): RepoForm { + return { + id: '', + owner: '', + repo: '', + branch: 'main', + include: '**/*.md', + exclude: '', + strategy: 'markdown_heading', + max_tokens: '1200', + overlap_tokens: '100', + mode: 'incremental', + interval_minutes: '60', + cron: '0 2 * * *', + }; +} + +function repoSpec(form: RepoForm) { + return { + id: form.id.trim() || `${form.owner.trim()}-${form.repo.trim()}`.toLowerCase(), + owner: form.owner.trim(), + repo: form.repo.trim(), + branch: form.branch.trim() || 'main', + include: form.include, + exclude: form.exclude, + strategy: form.strategy, + max_tokens: Number(form.max_tokens), + overlap_tokens: Number(form.overlap_tokens), + mode: form.mode, + interval_minutes: Number(form.interval_minutes), + cron: form.cron.trim(), + }; +} + +function adminOrigin(apiUrl: string): string { + if (apiUrl.startsWith('http')) { + return apiUrl.replace(/\/rest\/v1\/?$/, ''); + } + return ''; +} + +async function readJson(res: Response) { + const text = await res.text(); + try { + return JSON.parse(text); + } catch { + // Flask HTML aborts (legacy) — surface a short line, not the whole page. + const plain = text + .replace(/<[^>]+>/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + const clipped = plain.slice(0, 240); + return { description: clipped || res.statusText, error: clipped || res.statusText }; + } +} + +function apiError(body: any, res: Response): string { + return body?.description || body?.error || res.statusText || 'Request failed'; +} + +/** Session CSRF from GET /admin/config (see web_main.admin_imports_enabled_required). */ +let _adminCsrf = ''; + +function setAdminCsrf(token: string) { + _adminCsrf = (token || '').trim(); +} + +function mutHeaders(extra: Record = {}): Record { + return { + Accept: 'application/json', + ...(_adminCsrf ? { 'X-CSRF-Token': _adminCsrf } : {}), + ...extra, + }; +} + +export const Admin = () => { + const { apiUrl } = useEnvironment(); + const { isLoggedIn, loading, login, user } = useUser(); + const { capabilities, loading: capsLoading } = useCapabilities(); + const [tab, setTab] = useState('dashboard'); + const origin = adminOrigin(apiUrl); + + useEffect(() => { + if (!origin || !isLoggedIn || !capabilities?.admin) return; + let cancelled = false; + fetch(`${origin}/admin/config`) + .then(async (res) => { + const body = await readJson(res); + if (!cancelled && typeof body.csrf_token === 'string') { + setAdminCsrf(body.csrf_token); + } + }) + .catch(() => { + /* ConfigTab will retry; mutations fail closed without token. */ + }); + return () => { + cancelled = true; + }; + }, [origin, isLoggedIn, capabilities?.admin]); + + if (loading || capsLoading) { + return ( +
+ Admin +

Loading…

+
+ ); + } + + if (!isLoggedIn) { + return ( +
+ Admin +

Sign in to continue. Admin APIs require login.

+ +
+ ); + } + + if (!capabilities?.admin) { + return ( +
+ Admin +

Admin APIs are off. Set CRE_ALLOW_IMPORT and restart the process.

+
+ ); + } + + const meta = TAB_META[tab]; + const nav: { id: Tab; label: string; group: string }[] = [ + { id: 'dashboard', label: 'Overview', group: 'Ops' }, + { id: 'config', label: 'Config values', group: 'Ops' }, + { id: 'pipeline', label: 'Pipeline', group: 'Ops' }, + { id: 'oie', label: 'OIE queue', group: 'Ops' }, + { id: 'schedule', label: 'Schedules', group: 'Ops' }, + { id: 'authority', label: 'Authority', group: 'Ops' }, + { id: 'graph', label: 'Review', group: 'Ops' }, + { id: 'workbench', label: 'Workbench', group: 'Ops' }, + { id: 'myopencre', label: 'MyOpenCRE', group: 'Apps' }, + { id: 'profile', label: 'Profile', group: 'Account' }, + ]; + + return ( +
+ +
+
+

{meta.label}

+

{meta.sub}

+ {tab === 'dashboard' && ( + setTab('pipeline')} + onOpenReview={() => setTab('graph')} + onOpenWorkbench={() => setTab('workbench')} + /> + )} + {tab === 'config' && } + {tab === 'pipeline' && } + {tab === 'oie' && } + {tab === 'schedule' && } + {tab === 'authority' && } + {tab === 'graph' && } + {tab === 'workbench' && } + {tab === 'myopencre' && ( +
+

MyOpenCRE

+

+ Personal workspace (also needs CRE_ENABLE_MYOPENCRE for non-admin users). +

+ + Open MyOpenCRE + +
+ )} + {tab === 'profile' && } +
+
+
+ ); +}; + +function StageStrip({ steps }: { steps?: { id: string; label: string; state: string }[] }) { + if (!steps || !steps.length) return null; + return ( +
    + {steps.map((step) => ( +
  1. + {step.label} +
    2. + ))} +
+ ); +} + +function RepoFormFields({ form, setForm }: { form: RepoForm; setForm: (next: RepoForm) => void }) { + const set = + (key: keyof RepoForm) => + (e: React.ChangeEvent) => + setForm({ ...form, [key]: e.target.value }); + return ( +
+ + + + + + + + + + + + + + + + + + + + +