Feature/tracking charts

scripts/append_dataset_snapshot.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""
+Append today's dataset/image counts to static/data/dataset_history.json, which powers the
+homepage growth charts. dataset_history.json is otherwise hand-maintained (see its `_readme`
+field) -- this just automates counting the current catalog so you don't have to.
+
+Usage:
+    python3 scripts/append_dataset_snapshot.py
+"""
+
+import json
+from datetime import date, datetime, timezone
+from pathlib import Path
+
+STATIC_DATA_DIR = Path(__file__).resolve().parent.parent / "static" / "data"
+HISTORY_PATH = STATIC_DATA_DIR / "dataset_history.json"
+
+
+def read_json(path: Path):
+    with path.open("r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def is_vlm_dataset(entry: dict) -> bool:
+    """Mirrors the isVlm inference in normalizeDataset (src/lib/datasets.ts): raw manifests
+    don't reliably set `dataset_type` itself, so VLM datasets are recognized by their
+    machine_learning_task/qa_type/task_dimensions instead."""
+    if entry.get("machine_learning_task") == "image-text-to-text":
+        return True
+    if entry.get("qa_type"):
+        return True
+    if entry.get("task_dimensions"):
+        return True
+    return False
+
+
+def is_child_dataset(entry: dict) -> bool:
+    return bool(entry.get("parent_dataset")) and not is_vlm_dataset(entry)
+
+
+def current_counts() -> tuple[int, int]:
+    """Same counting rules as computeDatasetStats in src/lib/datasets.ts: dedupe the two
+    manifests by name, drop child/variant datasets, and exclude iNatAg-mini's images as a
+    reduced duplicate of iNatAg's."""
+    datasets = read_json(STATIC_DATA_DIR / "datasets.json") + read_json(STATIC_DATA_DIR / "hf_datasets.json")
+
+    merged: dict[str, dict] = {}
+    for entry in datasets:
+        name = entry.get("name")
+        if name and name not in merged:
+            merged[name] = entry
+
+    top_level = [entry for entry in merged.values() if not is_child_dataset(entry)]
+    dataset_count = len(top_level)
+    image_count = sum(
+        entry.get("num_images") or 0
+        for entry in top_level
+        if not entry["name"].startswith("iNatAg-mini")
+    )
+    return dataset_count, image_count
+
+
+def main() -> None:
+    history = read_json(HISTORY_PATH)
+    dataset_count, image_count = current_counts()
+    today = date.today().isoformat()
+
+    points = history["points"]
+    last = points[-1] if points else None
+    if last and last.get("date") == today:
+        last["datasetCount"] = dataset_count
+        last["imageCount"] = image_count
+        print(f"Updated today's ({today}) snapshot: {dataset_count} datasets, {image_count} images")
+    else:
+        period = datetime.now(timezone.utc).strftime("%b ") + str(int(datetime.now(timezone.utc).strftime("%d")))
+        points.append({
+            "period": period,
+            "date": today,
+            "datasetCount": dataset_count,
+            "imageCount": image_count,
+        })
+        print(f"Appended {today} snapshot: {dataset_count} datasets, {image_count} images")
+
+    history["generatedAt"] = today
+    with HISTORY_PATH.open("w", encoding="utf-8") as f:
+        json.dump(history, f, indent=2, ensure_ascii=False)
+        f.write("\n")
+
+
+if __name__ == "__main__":
+    main()

src/components/DistributionBarChart.tsx

@@ -0,0 +1,93 @@
+import { useMemo } from 'react';
+import BrowserOnly from '@docusaurus/BrowserOnly';
+import { usePlotlyChrome } from '../lib/plotlyChrome';
+import type { DistributionEntry } from '../lib/datasets';
+import styles from './GrowthLineChart.module.css';
+
+export function DistributionBarChart({
+	entries,
+	color,
+	xaxisTitle,
+}: {
+	entries: DistributionEntry[];
+	color: 'primary' | 'teal';
+	xaxisTitle: string;
+}) {
+	const chrome = usePlotlyChrome();
+	const barColor = color === 'primary' ? chrome.accentPrimary : chrome.accentTeal;
+
+	// Reversed so the highest count renders at the top of the horizontal bar chart (Plotly's
+	// default category order for a horizontal bar runs bottom-to-top).
+	const ordered = useMemo(() => [...entries].reverse(), [entries]);
+
+	const trace = useMemo(
+		() => ({
+			type: 'bar',
+			orientation: 'h',
+			x: ordered.map((e) => e.count),
+			y: ordered.map((e) => e.label),
+			text: ordered.map((e) => e.count.toLocaleString()),
+			textposition: 'outside',
+			// The top bar's count label sits right at the plot's top edge — without this Plotly
+			// clips it there instead of letting it draw into the margin.
+			cliponaxis: false,
+			textfont: { color: chrome.text, family: chrome.fontFamily, size: 11 },
+			marker: { color: barColor },
+			hovertemplate: '%{y}<br>%{x:,} datasets<extra></extra>',
+			hoverlabel: {
+				bgcolor: chrome.tooltipBg,
+				bordercolor: chrome.tooltipBorder,
+				font: { color: chrome.tooltipText },
+			},
+		}),
+		[ordered, barColor, chrome],
+	);
+
+	return (
+		<div className={styles.chartTall}>
+			<BrowserOnly fallback={<div className={styles.fallback}>Loading chart…</div>}>
+				{() => {
+					const Plotly = require('plotly.js-dist-min');
+					const createPlotlyComponent = require('react-plotly.js/factory').default;
+					const Plot = createPlotlyComponent(Plotly);
+					return (
+						<Plot
+							data={[trace]}
+							layout={{
+								autosize: true,
+								margin: { l: 128, r: 32, t: 12, b: 34 },
+								paper_bgcolor: 'transparent',
+								plot_bgcolor: 'transparent',
+								xaxis: {
+									title: { text: xaxisTitle, font: { color: chrome.text, family: chrome.fontFamily, size: 11 } },
+									showgrid: true,
+									gridcolor: chrome.grid,
+									zeroline: false,
+									tickfont: { color: chrome.text, family: chrome.fontFamily, size: 10.5 },
+								},
+								yaxis: {
+									showgrid: false,
+									linecolor: chrome.grid,
+									tickfont: { color: chrome.text, family: chrome.fontFamily, size: 10.5 },
+									automargin: true,
+								},
+								showlegend: false,
+								hoverlabel: {
+									bgcolor: chrome.tooltipBg,
+									bordercolor: chrome.tooltipBorder,
+									font: { color: chrome.tooltipText },
+								},
+								uirevision: 'distribution-chart',
+							}}
+							config={{ displayModeBar: false, responsive: true }}
+							style={{ width: '100%', height: '100%' }}
+							useResizeHandler
+						/>
+					);
+				}}
+			</BrowserOnly>
+		</div>
+	);
+}
+
+export default DistributionBarChart;

src/components/GrowthLineChart.module.css

@@ -0,0 +1,19 @@
+.chart {
+	width: 100%;
+	height: 260px;
+}
+
+.chartTall {
+	width: 100%;
+	height: 320px;
+}
+
+.fallback {
+	display: flex;
+	align-items: center;
+	justify-content: center;
+	height: 100%;
+	font-family: var(--ifm-code-font-family);
+	font-size: 0.8rem;
+	color: var(--agml-muted);
+}