diff --git a/CLAUDE.md b/CLAUDE.md index e079c00..753b3b5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,7 @@ python3 install.py # --cli-only, --dev, --prefix, --verify, --uninstal # Pipe a single command (non-interactive; cli.py detects a non-tty stdin) echo '/search "q2 plan"' | python3 -m glean_code -# Run the full test suite (1,109 tests, stdlib unittest β€” works with or without pytest) +# Run the full test suite (1,148 tests, stdlib unittest β€” works with or without pytest) python3 -m pytest tests/ python3 -m unittest discover tests/ diff --git a/README.md b/README.md index 3333f45..d6570e6 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ python3 install.py && glean # installed, with a macOS Spotlight app | `/index.bulk-documents --path ./docs/ --datasource custom1 --object-type Article --dry-run` | Index a folder, inspecting the payload first | | `/metadata.attach --doc-id --group tickets --values owner=alice` | Enrich docs without re-uploading them | | `/personal index ~/Documents --label docs` | Index your own files, locally | +| `/graph "quarterly planning" --html graph.html` | Graph how that content relates β€” documents, authors, sources | | `/flow show` | Draw the investigations you ran | @@ -75,6 +76,8 @@ python3 install.py && glean # installed, with a macOS Spotlight app ### πŸ”Ž Search & chat Search, autocomplete, recommendations, threaded chat, summarize, answers, feedback β€” plus docs, people, entities, announcements, collections, pins, Go Links, and verification. +`/graph` turns any result set into a knowledge graph of documents, authors and sources. +[Knowledge graph](docs/GRAPH.md)
@@ -172,11 +175,12 @@ The full Glean Code REPL β€” slash commands, status bar, mock/live switching, se | πŸ“Š **[Insights](docs/INSIGHTS.md)** | Flags, output, and CSV export | | πŸ§ͺ **[Mock corpus](docs/MOCK_CORPUS.md)** | The offline corpus β€” inventory, ranking, bring-your-own format | | 🏠 **[Local indexing](docs/LOCAL_INDEXING.md)** Β· **[Personal](docs/PERSONAL.md)** | Index your own folders, search them, keep them current | +| πŸ•ΈοΈ **[Knowledge graph](docs/GRAPH.md)** | How `/graph` builds nodes and edges, and how shared terms are scored | | πŸ—ΊοΈ **[Flow mapper](docs/FLOW_MAPPER.md)** | Capturing investigations, linking them, retention questions | | πŸ” **[SSO / OAuth](docs/SSO_OAUTH.md)** Β· **[Secure tokens](docs/SECURE_TOKENS.md)** | Browser sign-in, secure refs, the masking matrix | | πŸ”Œ **[MCP server](docs/MCP.md)** | Glean as native tools in Claude Code, Claude Desktop, Cursor | | πŸ›οΈ **[Architecture](docs/ARCHITECTURE.md)** Β· **[REST paths](docs/REST_PATHS.md)** | Module map, request flow, endpoints, how to add a command | -| βœ… **[Testing](docs/TESTING.md)** | Running the 1,109-test suite and what it covers | +| βœ… **[Testing](docs/TESTING.md)** | Running the 1,148-test suite and what it covers | | πŸ›Ÿ **[Support](SUPPORT.md)** Β· **[Changelog](CHANGELOG.md)** | How to report a bug Β· release history | > [!NOTE] diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 7298dea..a0a4bf2 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -548,6 +548,57 @@ Search the Glean index and display ranked results with snippets. --- +#### /graph + +Build a knowledge graph over the documents a query returns β€” how they relate to each other, who wrote them, and where they live. Glean's Client API has no graph endpoint, so the graph is synthesised locally from the search response, which means it works identically in mock, local and live mode. + +```text +/graph [--html ] [--page-size ] [--datasource ] [--min-shared ] [--no-terms] +``` + +| Parameter | Description | +| --- | --- | +| `query` | Free text query. The graph covers the documents it returns, not the whole index. | +| `--html` | Write a self-contained interactive page to this path. No CDN, no framework, no network. | +| `--page-size` | How many results to graph. Default 25. | +| `--datasource` | Restrict the result set to one datasource. | +| `--min-shared` | Shared terms needed before two documents are linked. Default 2. | +| `--no-terms` | Structural edges only: author, datasource, container. | + +Four kinds of node, four kinds of edge, and every edge carries the evidence that produced it: + +| Node | From | +| --- | --- | +| `doc` | A document in the result set | +| `person` | Its author, from `metadata.author` | +| `source` | The datasource it was indexed from | +| `container` | The folder, channel or space it lives in | + +| Edge | Meaning | +| --- | --- | +| `authored_by` | `doc` β†’ `person` | +| `in_source` | `doc` β†’ `source` | +| `in_container` | `doc` β†’ `container` | +| `shares_term` | `doc` ↔ `doc`, weighted by how rare the shared vocabulary is across the result set | + +`shares_term` is the interesting one. Terms carried by most of the result set are dropped before scoring β€” those are usually the query itself β€” so what remains is what actually distinguishes one pair of documents from the rest. The edge's evidence names the shared words, so a link can be read rather than trusted. + +```text +/graph "quarterly planning" +/graph "checkout incident" --html incident-graph.html +/graph "access review" --datasource confluence --min-shared 3 +``` + +**Output** β€” A terminal summary: node and edge counts by kind, the hubs by degree, connected clusters with the node anchoring each, and the strongest content links with their shared terms. With `--html`, also an interactive page: pan, zoom, drag, and click a node to see every edge and why it exists. + +**Mock mode** β€” Fully supported. The mock corpus carries an author, datasource and container on all seventy documents, so the graph is complete offline. + +**Local mode** β€” `/search` resolves to the personal index, so the graph covers your own files. Local files have no author metadata, so the graph has no `person` nodes; the command says so rather than leaving you to notice. + +**Endpoint** β€” `POST /rest/api/v1/search` (the graph itself is built locally) + +--- + #### /datasources.list List all datasources visible to the current token, derived from a faceted search call. diff --git a/docs/GRAPH.md b/docs/GRAPH.md new file mode 100644 index 0000000..9aa52ef --- /dev/null +++ b/docs/GRAPH.md @@ -0,0 +1,105 @@ +# Knowledge Graph + +`/graph` draws how the content behind a query relates to itself: which documents share +vocabulary, who wrote them, and where they live. It answers the question a flat list of +search results cannot β€” *what is this cluster of material, and who owns it?* + +```text +/graph "quarterly planning" +/graph "checkout incident" --html incident-graph.html +``` + +## Why it is built locally + +The Glean Client API has no graph endpoint. What it does return, on every search result, is +the author, the datasource, the container, and snippets of the text. That is enough to build +a graph β€” and building it client-side has three consequences worth stating plainly: + +- **It is the graph of one query's result set**, not a crawl of the whole index. `--page-size` + is therefore the size of the graph. +- **It works identically in mock, local and live mode**, because all three return the same + response shape. No endpoint to stub, no shape to keep in sync. +- **Nothing leaves the process.** The graph, the layout and the HTML are all computed locally. + +## The model + +| Node | Comes from | +| --- | --- | +| `doc` | A document in the result set | +| `person` | `metadata.author` β€” name and email | +| `source` | `metadata.datasource` | +| `container` | `metadata.container` β€” a folder, channel or space | + +| Edge | Meaning | Evidence it carries | +| --- | --- | --- | +| `authored_by` | `doc` β†’ `person` | "author of this document" | +| `in_source` | `doc` β†’ `source` | "indexed from gdrive" | +| `in_container` | `doc` β†’ `container` | "lives in #planning" | +| `shares_term` | `doc` ↔ `doc` | The shared words themselves | + +### How `shares_term` is scored + +Every document's title and snippets are reduced to terms β€” lowercase, four characters or +more, stopwords dropped. A term is then weighted by how rare it is *within this result set*: + +``` +weight(term) = log(total_docs / docs_containing_term) +``` + +Terms appearing in more than 60% of the set are discarded before scoring, because those are +almost always the query itself: in a search for "quarterly planning", the words *quarterly* +and *planning* say nothing about how two of the results differ. An edge is drawn when two +documents share at least `--min-shared` of the surviving terms, and its score is the sum of +their weights. + +This is the same idea `/flow` uses to link investigations, and for the same reason: a link you +cannot inspect is a link you have to take on trust. Every edge names its evidence, so +`shares capacity, model, fy26 (4.7)` can be judged rather than believed. + +## Reading the terminal view + +```text +27 nodes 10 doc Β· 4 person Β· 4 source Β· 9 container +40 edges 10 authored_by Β· 10 in_source Β· 10 in_container Β· 10 shares_term + +hubs + β–ͺ doc Questions on the capacity model numbers 7 edges + β—† person Priya Raman 4 edges + +clusters (1) + 27 nodes β€” anchored on doc Questions on the capacity model numbers + +strongest content links + Headcount and Budget Model FY26 + ↓ shares feeds, capacity, model, fy26 (4.7) + PLAN-482 β€” Q4 FY26 planning: engineering capacity model +``` + +**Hubs** are ranked by degree, so the documents everything else touches surface first. +**Clusters** are connected components β€” more than one means the query pulled in unrelated +material, which is itself worth knowing. **Strongest content links** are the `shares_term` +edges in score order. + +## The HTML view + +`--html ` writes a self-contained page: one file, no CDN, no framework, no network β€” +the same constraint `/flow timeline` works under. Inside it: + +- A force-directed layout, seeded from deterministic ring positions so the same graph opens + the same way, then relaxed in the browser. +- Nodes coloured by kind and sized by degree. +- Pan by dragging the background, zoom on scroll, drag a node to pull the layout around it. + The view auto-frames the graph while it settles and stops the moment you take over. +- Click any node for a panel listing every edge it has, each with its evidence and score. +- Light and dark, following the reader's system setting. `prefers-reduced-motion` skips the + animated relaxation and draws the settled layout directly. + +## Limits worth knowing + +- **Local mode has no people.** Plain files carry no author, so a graph of the personal index + has `doc`, `source` and `container` nodes only. The command tells you instead of leaving a + silent gap. +- **Layout cost is O(nΒ²) per tick.** Fine to a few hundred nodes, which is well past a + sensible `--page-size`; it is not a whole-index visualiser. +- **Authorship is whoever the connector recorded.** A document indexed under a service account + is attributed to that account, exactly as Glean holds it. diff --git a/docs/TESTING.md b/docs/TESTING.md index e26b391..7344270 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -21,13 +21,13 @@ files, and they outrank the `Glean Code.app` launcher in `Cmd+Space`: export PYTHONPYCACHEPREFIX="$HOME/.cache/python" ``` -1,109 tests covering the client and every mock response, commands and dispatch, config, UI, auth, completion, help docs, the mock corpus, indexing-walk, scaffold, the installer, the MCP server, the flow mapper, the Pages site builder, and Glean Personal (text extraction, the index, the content graph, ranking explanations, local mode, and the local MCP tools). +1,148 tests covering the client and every mock response, commands and dispatch, config, UI, auth, completion, help docs, the mock corpus, indexing-walk, scaffold, the installer, the MCP server, the flow mapper, the Pages site builder, the knowledge graph, and Glean Personal (text extraction, the index, the content graph, ranking explanations, local mode, and the local MCP tools). ## Development notes Notes on the test suite added during development of glean-code-cli. -All 1,109 tests pass. Here's what was added across the development passes: +All 1,148 tests pass. Here's what was added across the development passes: `tests/test_commands_extended.py` (155 new tests) β€” covers all previously untested commands: diff --git a/glean_code/commands.py b/glean_code/commands.py index 87a70dd..d5b9978 100644 --- a/glean_code/commands.py +++ b/glean_code/commands.py @@ -23,6 +23,7 @@ from . import ui from . import _indexing_walk as _walk from . import flow as _flow +from . import graph as _graph from . import mcp_control as _mcp from . import personal as _personal from .client import GleanClient, GleanError @@ -796,6 +797,66 @@ def cmd_search(s: Session, pos, flags): print(ui.rule()) +@register("graph") +def cmd_graph(s: Session, pos, flags): + """Knowledge graph over a query's result set: documents, authors, sources.""" + if not pos: + ui.print_err('Usage: /graph [--html out.html]') + return + query = " ".join(pos) + page_size = int(flags.get("page-size") or flags.get("page_size") or 25) + datasource = flags.get("datasource") + min_shared = int(flags.get("min-shared") or flags.get("min_shared") or + _graph.DEFAULT_MIN_SHARED) + with_terms = not (flags.get("no-terms") or flags.get("no_terms")) + + try: + resp = s.client.search(query, page_size=page_size, datasource=datasource) + except GleanError as e: + ui.print_err(str(e)) + return + + results = resp.get("results", []) + # The banner belongs to the response, exactly as in /search: a graph drawn + # from the personal index must not read as one drawn from Glean. + if resp.get("localIndex"): + print(ui.style(_personal.LOCAL_BANNER, ui.C.YELLOW)) + print() + if not results: + ui.print_info("No results for that query, so there is nothing to graph.") + return + + mode = s.config.effective_mode + source = {"mock": "mock corpus", "local": "personal index"}.get( + mode, s.config.instance or "live") + built = _graph.build(results, query=query, source_label=source, + min_shared=min_shared, with_terms=with_terms) + summary = _graph.summarize(built) + + print(ui.rule(f"graph: {query}")) + print(ui.style(f" {source}", ui.C.GREY)) + print() + print(_graph.render_terminal(built, summary, ui.term_width())) + print(ui.rule()) + + if mode == "local": + ui.print_info("Local files carry no author, so this graph has no people in it.") + + out = flags.get("html") + if out: + if out is True: + ui.print_err("--html needs a path: /graph --html graph.html") + return + try: + written = _graph.write_html(Path(str(out)), built) + except OSError as e: + ui.print_err(f"Could not write {out}: {e}") + return + ui.print_ok(f"Wrote {written} ({summary['nodes']} nodes, {summary['edges']} edges)") + else: + ui.print_info("Add --html graph.html for the interactive view.") + + @register("datasources.list") def cmd_datasources_list(s: Session, pos, flags): sample = int(flags.get("sample") or 100) diff --git a/glean_code/graph.py b/glean_code/graph.py new file mode 100644 index 0000000..240d25c --- /dev/null +++ b/glean_code/graph.py @@ -0,0 +1,660 @@ +"""A knowledge graph over a set of search results. + +Glean's Client API has no graph endpoint, so the graph is synthesised from +what a search response already carries: the author, datasource and container +of every result, plus the language of its title and snippets. That keeps one +code path for mock, local and live β€” the same fields come back from all three +β€” and it keeps the claim honest: this is the graph of one query's result set, +not a crawl of the whole index. + +Four kinds of node, and every edge carries the evidence that produced it: + + doc a document in the result set + person an author + source a datasource + container a folder, channel or space + + authored_by doc -> person the author on the result + in_source doc -> source the datasource it came from + in_container doc -> container the space it lives in + shares_term doc <-> doc vocabulary shared by two documents, + weighted by how rare it is in the set + +Rendered two ways: a terminal summary (hubs, clusters, strongest links) and a +self-contained HTML page with a force-directed layout β€” no CDN, no framework, +no network, matching flow.render_timeline. +""" + +from __future__ import annotations + +import html +import json +import math +import re +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +# Words too common to say anything about how two documents relate. +STOPWORDS = frozenset(""" +about above after again against alone along already also although always among +another around because been before being below between both came come could +does doing done down during each either else enough even ever every from +further gave give given goes going gone have having here hers herself himself +however into itself just last later least less like made make many maybe mean +might more most much must myself never next none nothing once only onto other +ought ours ourselves over rather really same seem seen several shall should +since some such take taken than that their them themselves then there these +they this those though through thus together took toward under until upon used +uses using very want wants well were what when where which while whom whose +will with within without would your yours yourself +""".split()) + +TERM_RE = re.compile(r"[a-z][a-z0-9_.\-]{3,}") + +NODE_KINDS = ("doc", "person", "source", "container") +EDGE_KINDS = ("authored_by", "in_source", "in_container", "shares_term") + +DEFAULT_MIN_SHARED = 2 +DEFAULT_MAX_TERM_SHARE = 0.6 # a term in more than this share of docs says nothing + + +# -------------------------------------------------------------------------- +# building +# -------------------------------------------------------------------------- + +def _text_of(result: Dict[str, Any]) -> str: + parts = [str(result.get("title") or "")] + for snippet in result.get("snippets") or []: + parts.append(str(snippet.get("text") or "")) + return " ".join(parts) + + +def _terms(text: str) -> set: + return {t for t in TERM_RE.findall(text.lower()) if t not in STOPWORDS} + + +def _author_of(result: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]: + """(email, display name) for a result, from wherever the shape carries it.""" + meta = result.get("metadata") or {} + author = meta.get("author") or result.get("author") + if isinstance(author, dict): + email = author.get("email") or author.get("id") + return email, (author.get("name") or email) + if isinstance(author, str) and author: + return author, author + return None, None + + +def build(results: Sequence[Dict[str, Any]], query: str = "", + source_label: str = "", min_shared: int = DEFAULT_MIN_SHARED, + with_terms: bool = True) -> Dict[str, Any]: + """Turn search results into nodes and edges.""" + nodes: Dict[str, Dict[str, Any]] = {} + edges: List[Dict[str, Any]] = [] + + def node(node_id: str, kind: str, label: str, **meta) -> str: + existing = nodes.get(node_id) + if existing is None: + nodes[node_id] = {"id": node_id, "kind": kind, "label": label, "meta": meta} + elif meta: + existing["meta"].update({k: v for k, v in meta.items() if v}) + return node_id + + def edge(a: str, b: str, kind: str, score: float, why: str) -> None: + edges.append({"a": a, "b": b, "kind": kind, "score": round(float(score), 3), "why": why}) + + doc_terms: Dict[str, set] = {} + + for result in results: + raw_id = str(result.get("id") or result.get("url") or result.get("title") or "") + if not raw_id: + continue + doc = node( + "doc:" + raw_id, "doc", str(result.get("title") or raw_id), + url=result.get("url") or "", + datasource=str(result.get("datasource") or (result.get("metadata") or {}).get("datasource") or ""), + doc_type=str((result.get("metadata") or {}).get("documentType") or ""), + updated=str((result.get("metadata") or {}).get("updatedAgo") or ""), + ) + + email, name = _author_of(result) + if email: + person = node("person:" + email, "person", name or email, email=email) + edge(doc, person, "authored_by", 1.0, "author of this document") + + datasource = nodes[doc]["meta"].get("datasource") + if datasource: + source = node("source:" + datasource, "source", datasource) + edge(doc, source, "in_source", 0.5, "indexed from " + datasource) + + container = str((result.get("metadata") or {}).get("container") or "") + if container: + key = "container:" + (datasource or "") + "/" + container + box = node(key, "container", container, datasource=datasource or "") + edge(doc, box, "in_container", 0.8, "lives in " + container) + + if with_terms: + doc_terms[doc] = _terms(_text_of(result)) + + if with_terms and len(doc_terms) > 1: + edges.extend(_term_edges(doc_terms, min_shared)) + + return { + "query": query, + "source": source_label, + "nodes": sorted(nodes.values(), key=lambda n: (NODE_KINDS.index(n["kind"]), n["label"].lower())), + "edges": edges, + } + + +def _term_edges(doc_terms: Dict[str, set], min_shared: int) -> List[Dict[str, Any]]: + """doc <-> doc edges for shared vocabulary, weighted by rarity. + + A term carried by most of the result set is usually the query itself, so it + is dropped before scoring β€” what is left is what actually distinguishes one + pair of documents from the rest. + """ + total = len(doc_terms) + counts: Dict[str, int] = {} + for terms in doc_terms.values(): + for term in terms: + counts[term] = counts.get(term, 0) + 1 + + ceiling = max(2, int(total * DEFAULT_MAX_TERM_SHARE)) + weight = { + term: math.log(total / count) + for term, count in counts.items() + # count == total gives log(1) == 0: shared by everything, so it + # distinguishes nothing and must not draw a zero-weight edge + if 1 < count <= ceiling and count < total + } + + out: List[Dict[str, Any]] = [] + ids = sorted(doc_terms) + for i, a in enumerate(ids): + for b in ids[i + 1:]: + shared = doc_terms[a] & doc_terms[b] & weight.keys() + if len(shared) < min_shared: + continue + ranked = sorted(shared, key=lambda t: (-weight[t], t)) + score = sum(weight[t] for t in ranked) + if score <= 0: + continue + out.append({ + "a": a, "b": b, "kind": "shares_term", + "score": round(score, 3), + "why": ", ".join(ranked[:4]), + }) + out.sort(key=lambda e: -e["score"]) + return out + + +# -------------------------------------------------------------------------- +# reading it +# -------------------------------------------------------------------------- + +def _degrees(graph: Dict[str, Any]) -> Dict[str, int]: + degrees = {n["id"]: 0 for n in graph["nodes"]} + for e in graph["edges"]: + if e["a"] in degrees: + degrees[e["a"]] += 1 + if e["b"] in degrees: + degrees[e["b"]] += 1 + return degrees + + +def _clusters(graph: Dict[str, Any]) -> List[List[str]]: + parent = {n["id"]: n["id"] for n in graph["nodes"]} + + def find(x: str) -> str: + while parent[x] != x: + parent[x] = parent[parent[x]] + x = parent[x] + return x + + for e in graph["edges"]: + a, b = e["a"], e["b"] + if a in parent and b in parent: + ra, rb = find(a), find(b) + if ra != rb: + parent[rb] = ra + + groups: Dict[str, List[str]] = {} + for node_id in parent: + groups.setdefault(find(node_id), []).append(node_id) + return sorted(groups.values(), key=len, reverse=True) + + +def summarize(graph: Dict[str, Any], hubs: int = 6, links: int = 4) -> Dict[str, Any]: + by_kind = {kind: 0 for kind in NODE_KINDS} + for n in graph["nodes"]: + by_kind[n["kind"]] = by_kind.get(n["kind"], 0) + 1 + edge_kinds = {kind: 0 for kind in EDGE_KINDS} + for e in graph["edges"]: + edge_kinds[e["kind"]] = edge_kinds.get(e["kind"], 0) + 1 + + degrees = _degrees(graph) + labels = {n["id"]: n for n in graph["nodes"]} + ranked = sorted(graph["nodes"], key=lambda n: (-degrees[n["id"]], n["label"].lower())) + + groups = _clusters(graph) + described = [] + for group in groups: + anchor = max(group, key=lambda i: (degrees[i], labels[i]["label"])) + described.append({"size": len(group), "anchor": labels[anchor]["label"], + "anchor_kind": labels[anchor]["kind"]}) + + strongest = [e for e in graph["edges"] if e["kind"] == "shares_term"][:links] + return { + "nodes": len(graph["nodes"]), + "edges": len(graph["edges"]), + "by_kind": by_kind, + "edge_kinds": edge_kinds, + "hubs": [{"label": n["label"], "kind": n["kind"], "degree": degrees[n["id"]]} + for n in ranked[:hubs]], + "clusters": described, + "strongest": [{"a": labels[e["a"]]["label"], "b": labels[e["b"]]["label"], + "score": e["score"], "why": e["why"]} for e in strongest], + } + + +# -------------------------------------------------------------------------- +# terminal +# -------------------------------------------------------------------------- + +_KIND_MARK = {"doc": "β–ͺ", "person": "β—†", "source": "β–Έ", "container": "β–«"} + + +def _fit(text: str, width: int) -> str: + text = " ".join(str(text or "").split()) + if width <= 1 or len(text) <= width: + return text + return text[: max(1, width - 1)].rstrip() + "…" + + +def render_terminal(graph: Dict[str, Any], summary: Optional[Dict[str, Any]] = None, + width: int = 80) -> str: + """Plain text; the caller styles it. Kept free of ANSI so it stays testable.""" + summary = summary or summarize(graph) + if not graph["nodes"]: + return "No graph: the result set was empty." + + counts = " Β· ".join( + "%d %s" % (summary["by_kind"][k], k if summary["by_kind"][k] != 1 else k.rstrip("s")) + for k in NODE_KINDS if summary["by_kind"].get(k)) + edge_counts = " Β· ".join( + "%d %s" % (v, k) for k, v in summary["edge_kinds"].items() if v) + + lines = [ + "%d nodes %s" % (summary["nodes"], counts), + "%d edges %s" % (summary["edges"], edge_counts), + "", + "hubs", + ] + for hub in summary["hubs"]: + lines.append(" %s %-9s %s %d edges" % ( + _KIND_MARK.get(hub["kind"], "Β·"), hub["kind"], + _fit(hub["label"], max(20, width - 28)), hub["degree"])) + + if summary["clusters"]: + lines += ["", "clusters (%d)" % len(summary["clusters"])] + for cluster in summary["clusters"][:5]: + lines.append(" %d nodes β€” anchored on %s %s" % ( + cluster["size"], cluster["anchor_kind"], + _fit(cluster["anchor"], max(20, width - 34)))) + + if summary["strongest"]: + lines += ["", "strongest content links"] + for link in summary["strongest"]: + lines.append(" %s" % _fit(link["a"], width - 4)) + lines.append(" ↓ shares %s (%.1f)" % (link["why"], link["score"])) + lines.append(" %s" % _fit(link["b"], width - 4)) + lines.append("") + lines.pop() + + return "\n".join(lines) + + +# -------------------------------------------------------------------------- +# html +# -------------------------------------------------------------------------- + +def _layout(graph: Dict[str, Any]) -> List[Dict[str, Any]]: + """Deterministic starting positions, so the same graph opens the same way.""" + laid = [] + rings = {"doc": 0.62, "person": 0.30, "source": 0.90, "container": 0.78} + per_kind: Dict[str, int] = {} + totals: Dict[str, int] = {} + for n in graph["nodes"]: + totals[n["kind"]] = totals.get(n["kind"], 0) + 1 + for n in graph["nodes"]: + kind = n["kind"] + index = per_kind.get(kind, 0) + per_kind[kind] = index + 1 + count = max(1, totals[kind]) + angle = (2 * math.pi * index / count) + (0.7 if kind == "person" else 0.0) + radius = rings.get(kind, 0.7) + laid.append(dict(n, x=round(math.cos(angle) * radius, 4), + y=round(math.sin(angle) * radius, 4))) + return laid + + +HTML_TEMPLATE = """ + + +__TITLE__ + + +
+

__HEADING__

+ __SOURCE__ + __COUNTS__ + click a node for its edges Β· drag to move Β· scroll to zoom +
+
+ +
+
+
shares_term edges are drawn thicker the more distinctive the shared words
+
+ + + +""" + + +def render_html(graph: Dict[str, Any]) -> str: + """A self-contained page. No CDN, no framework, no network.""" + summary = summarize(graph) + query = graph.get("query") or "" + heading = ("graph: " + query) if query else "knowledge graph" + counts = "%d nodes Β· %d edges" % (summary["nodes"], summary["edges"]) + data = dict(graph, nodes=_layout(graph)) + return (HTML_TEMPLATE + .replace("__TITLE__", html.escape(heading)) + .replace("__HEADING__", html.escape(heading)) + .replace("__SOURCE__", html.escape(graph.get("source") or "")) + .replace("__COUNTS__", counts) + .replace("__DATA__", json.dumps(data, ensure_ascii=False).replace(" Path: + target = Path(path).expanduser() + if target.parent and not target.parent.exists(): + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(render_html(graph), encoding="utf-8") + return target diff --git a/glean_code/help_docs.py b/glean_code/help_docs.py index b81bbb3..3c7f4c6 100644 --- a/glean_code/help_docs.py +++ b/glean_code/help_docs.py @@ -17,6 +17,27 @@ DOCS: Dict[str, CommandDoc] = { # ---------------- core shell ---------------- + "graph": { + "summary": ("Knowledge graph over a query's result set β€” documents, authors, " + "datasources and containers, with the evidence behind every edge."), + "usage": ("/graph [--html ] [--page-size ] " + "[--datasource ] [--min-shared ] [--no-terms]"), + "params": [ + ["query", "Free text query. The graph covers the documents it returns."], + ["--html", "Write a self-contained interactive page to this path."], + ["--page-size", "How many results to graph. Default 25."], + ["--datasource", "Restrict the result set to one datasource."], + ["--min-shared", "Shared terms needed before two documents are linked. Default 2."], + ["--no-terms", "Only structural edges: author, datasource, container."], + ], + "examples": [ + '/graph "quarterly planning"', + '/graph "checkout incident" --html incident-graph.html', + '/graph "access review" --datasource confluence --min-shared 3', + ], + "endpoint": ("POST /rest/api/v1/search β€” the graph is built locally from the " + "results; there is no graph endpoint in the Client API."), + }, "help": { "summary": "Show help for a command, or list all commands.", "usage": "/help [command]", diff --git a/tests/test_graph.py b/tests/test_graph.py new file mode 100644 index 0000000..47cf246 --- /dev/null +++ b/tests/test_graph.py @@ -0,0 +1,309 @@ +"""Tests for the knowledge graph (glean_code/graph.py) and the /graph command. + +The graph is synthesised from a search response, so the fixtures here are +search-response shaped β€” the same shape mock, local and live all return. No +network, no files outside a temp directory. +""" + +import io +import json +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path +from unittest.mock import MagicMock + +from glean_code import graph +from glean_code.client import GleanError +from glean_code.commands import HANDLERS, Session +from glean_code.config import Config + + +def _result(doc_id, title, author_email=None, author_name=None, datasource="gdrive", + container=None, snippet=""): + meta = {"datasource": datasource, "documentType": "Document"} + if author_email: + meta["author"] = {"name": author_name or author_email, "email": author_email} + if container: + meta["container"] = container + return { + "id": doc_id, + "title": title, + "url": "https://example.com/" + doc_id, + "datasource": datasource, + "snippets": [{"text": snippet}] if snippet else [], + "metadata": meta, + } + + +RESULTS = [ + _result("d1", "Capacity model for planning", "priya@acme.com", "Priya Raman", + "gdrive", "Platform Eng / Planning", + "The capacity model feeds engineer-weeks into the tracker."), + _result("d2", "PLAN-482 capacity model review", "priya@acme.com", "Priya Raman", + "jira", "PLAN board", + "Reviewing the capacity model and its engineer-weeks estimates."), + _result("d3", "Security access review", "marcus@acme.com", "Marcus Webb", + "confluence", "SEC space", + "Quarterly service account rotation and audit evidence."), +] + + +def _session(): + session = Session(Config(mode="mock")) + session.client = MagicMock() + return session + + +class TestBuild(unittest.TestCase): + def setUp(self): + self.g = graph.build(RESULTS, query="planning", source_label="mock corpus") + + def _ids(self, kind): + return {n["id"] for n in self.g["nodes"] if n["kind"] == kind} + + def _edges(self, kind): + return [e for e in self.g["edges"] if e["kind"] == kind] + + def test_every_document_becomes_a_node(self): + self.assertEqual(self._ids("doc"), {"doc:d1", "doc:d2", "doc:d3"}) + + def test_authors_are_deduplicated_across_documents(self): + # Priya wrote two of the three; she is one node with two edges. + self.assertEqual(self._ids("person"), {"person:priya@acme.com", "person:marcus@acme.com"}) + priya = [e for e in self._edges("authored_by") if e["b"] == "person:priya@acme.com"] + self.assertEqual(len(priya), 2) + + def test_person_label_prefers_the_display_name(self): + person = next(n for n in self.g["nodes"] if n["id"] == "person:priya@acme.com") + self.assertEqual(person["label"], "Priya Raman") + self.assertEqual(person["meta"]["email"], "priya@acme.com") + + def test_sources_and_containers_become_nodes(self): + self.assertEqual(self._ids("source"), + {"source:gdrive", "source:jira", "source:confluence"}) + self.assertEqual(len(self._ids("container")), 3) + + def test_shared_terms_link_the_two_related_documents(self): + links = self._edges("shares_term") + self.assertTrue(links) + pairs = {(e["a"], e["b"]) for e in links} + self.assertIn(("doc:d1", "doc:d2"), pairs) + # the security document shares nothing distinctive with the planning pair + self.assertNotIn(("doc:d1", "doc:d3"), pairs) + + def test_every_edge_carries_evidence_and_a_score(self): + for edge in self.g["edges"]: + self.assertTrue(edge["why"], edge) + self.assertIsInstance(edge["score"], float) + + def test_shared_term_evidence_names_the_shared_words(self): + link = next(e for e in self._edges("shares_term")) + self.assertIn("capacity", link["why"]) + + def test_a_string_author_is_accepted_as_well_as_an_object(self): + g = graph.build([_result("d9", "Plain author", None) | {"author": "sam@acme.com"}]) + self.assertIn("person:sam@acme.com", {n["id"] for n in g["nodes"]}) + + def test_results_without_an_id_are_skipped_not_crashed_on(self): + g = graph.build([{"title": ""}, RESULTS[0]]) + self.assertEqual(len({n["id"] for n in g["nodes"] if n["kind"] == "doc"}), 1) + + def test_min_shared_raises_the_bar_for_a_link(self): + loose = graph.build(RESULTS, min_shared=2) + strict = graph.build(RESULTS, min_shared=8) + self.assertTrue([e for e in loose["edges"] if e["kind"] == "shares_term"]) + self.assertEqual([e for e in strict["edges"] if e["kind"] == "shares_term"], []) + + def test_no_terms_leaves_only_structural_edges(self): + g = graph.build(RESULTS, with_terms=False) + kinds = {e["kind"] for g_edge in [g["edges"]] for e in g_edge} + self.assertNotIn("shares_term", kinds) + self.assertIn("authored_by", kinds) + + def test_a_term_in_every_document_is_not_evidence_of_anything(self): + same = [_result("a", "planning planning", snippet="planning notes for planning"), + _result("b", "planning planning", snippet="planning notes for planning")] + g = graph.build(same, min_shared=1) + self.assertEqual([e for e in g["edges"] if e["kind"] == "shares_term"], []) + + def test_empty_results_give_an_empty_graph(self): + g = graph.build([]) + self.assertEqual(g["nodes"], []) + self.assertEqual(g["edges"], []) + + +class TestSummary(unittest.TestCase): + def setUp(self): + self.g = graph.build(RESULTS, query="planning") + self.s = graph.summarize(self.g) + + def test_counts_match_the_graph(self): + self.assertEqual(self.s["nodes"], len(self.g["nodes"])) + self.assertEqual(self.s["edges"], len(self.g["edges"])) + self.assertEqual(self.s["by_kind"]["doc"], 3) + self.assertEqual(self.s["by_kind"]["person"], 2) + + def test_hubs_are_ranked_by_degree(self): + degrees = [h["degree"] for h in self.s["hubs"]] + self.assertEqual(degrees, sorted(degrees, reverse=True)) + + def test_clusters_find_the_disconnected_component(self): + # the security document shares an author and source with nothing else + sizes = [c["size"] for c in self.s["clusters"]] + self.assertEqual(len(sizes), 2, self.s["clusters"]) + self.assertEqual(sum(sizes), len(self.g["nodes"])) + + def test_strongest_links_are_shared_term_edges_in_score_order(self): + scores = [l["score"] for l in self.s["strongest"]] + self.assertEqual(scores, sorted(scores, reverse=True)) + + +class TestTerminalRender(unittest.TestCase): + def test_render_names_counts_hubs_and_evidence(self): + g = graph.build(RESULTS, query="planning") + out = graph.render_terminal(g, width=100) + self.assertIn("nodes", out) + self.assertIn("hubs", out) + self.assertIn("strongest content links", out) + self.assertIn("capacity", out) + + def test_render_carries_no_ansi_so_it_stays_testable(self): + out = graph.render_terminal(graph.build(RESULTS), width=80) + self.assertNotIn("\x1b[", out) + + def test_empty_graph_says_so(self): + self.assertIn("No graph", graph.render_terminal(graph.build([]))) + + def test_long_labels_are_truncated_to_the_width(self): + long_title = "x" * 400 + g = graph.build([_result("d1", long_title, "a@b.c")]) + for line in graph.render_terminal(g, width=60).split("\n"): + self.assertLessEqual(len(line), 60, line) + + +class TestHtmlRender(unittest.TestCase): + def setUp(self): + self.html = graph.render_html(graph.build(RESULTS, query="planning", + source_label="mock corpus")) + + def test_page_is_self_contained(self): + # the same rule flow.render_timeline works under: no CDN, no framework + for forbidden in ("")[0] + data = json.loads(payload) + self.assertEqual(len(data["nodes"]), len(graph.build(RESULTS)["nodes"])) + self.assertTrue(all("x" in n and "y" in n for n in data["nodes"])) + + def test_layout_is_deterministic(self): + again = graph.render_html(graph.build(RESULTS, query="planning", + source_label="mock corpus")) + self.assertEqual(self.html, again) + + def test_query_and_source_reach_the_page(self): + self.assertIn("graph: planning", self.html) + self.assertIn("mock corpus", self.html) + + def test_closing_script_tags_in_data_are_escaped(self): + html = graph.render_html(graph.build([_result("d1", "x")])) + self.assertNotIn("", html) + + def test_titles_are_escaped_in_the_markup(self): + html = graph.render_html(graph.build([], query="")) + # the raw string may sit inside the inert JSON payload, but must never + # reach the markup itself + markup = html.split('id="data"')[0] + html.split("", 1)[1] + self.assertNotIn("