From 21ec281bdb0bb9cc13051353fbec6447ba752d87 Mon Sep 17 00:00:00 2001 From: Jeff Schnitter Date: Fri, 23 Oct 2026 14:21:44 -0700 Subject: [PATCH 01/10] feat: add custom-metrics definition-create command --- cortexapps_cli/commands/custom_metrics.py | 34 ++++++++++++++++++++++ tests/test_custom_metrics.py | 35 +++++++++++++++++++++-- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/cortexapps_cli/commands/custom_metrics.py b/cortexapps_cli/commands/custom_metrics.py index 4183ccaf..1dd61a2c 100644 --- a/cortexapps_cli/commands/custom_metrics.py +++ b/cortexapps_cli/commands/custom_metrics.py @@ -1,7 +1,9 @@ from collections import defaultdict from datetime import datetime import typer +from typing import Optional from typing_extensions import Annotated +from rich import print_json app = typer.Typer(help="Custom metrics commands") @@ -131,3 +133,35 @@ def delete( params = _convert_datetime_to_string(params) r = client.delete("api/v1/eng-intel/custom-metrics/" + custom_metric_key + "/entity/" + tag, params=params) + +@app.command() +def definition_create( + ctx: typer.Context, + key: str = typer.Option(..., "--key", "-k", help="Unique metric key (2-255 chars, letters/digits/hyphens, alphanumeric start/end)"), + name: str = typer.Option(..., "--name", "-n", help="Display name for the metric"), + valence: str = typer.Option(..., "--valence", "-V", help="Whether higher values are preferable: POSITIVE, NEGATIVE, or NEUTRAL"), + category: Optional[str] = typer.Option(None, "--category", "-c", help="Grouping label"), + description: Optional[str] = typer.Option(None, "--description", "-d", help="What the metric measures"), + is_draft: bool = typer.Option(False, "--is-draft", help="If true, metric accepts data but is hidden from Eng Intelligence"), +): + """ + Create a custom metric definition + """ + + client = ctx.obj["client"] + + data = { + "key": key, + "name": name, + "valence": valence, + "category": category, + "description": description, + "isDraft": is_draft, + "type": "PUBLIC_API" + } + + # Remove keys whose value is None + data = {k: v for k, v in data.items() if v is not None} + + result = client.post("api/v1/eng-intel/custom-metrics", data=data) + print_json(data=result) diff --git a/tests/test_custom_metrics.py b/tests/test_custom_metrics.py index 7074b392..7f85642d 100644 --- a/tests/test_custom_metrics.py +++ b/tests/test_custom_metrics.py @@ -1,4 +1,6 @@ from tests.helpers.utils import * +import os +import requests as _requests # As part of this testing, filed: # CET-19691: custom metrics POST API returns 200 response for un-processed metrics older than 6 months @@ -10,8 +12,7 @@ def test(): print("API key does not have access to custom metrics or feature not enabled in tenant, not running tests.") return - # No API support to create a custom metric. It can only be done in the UI, so check if this workspace has the - # 'vulnerabilities' custom metric defined. + # 'vulnerabilities' must already exist as a metric definition; use `cortex custom-metrics definition-create` to create it. result = cli(["custom-metrics", "get", "-t", "cli-test-service", "-k", "vulnerabilities"], ReturnType.STDOUT) if "HTTP Error 404: Not Found - CustomMetricKey not found" in result: print("Custom metric named 'vulnerabilities' does not exist. It has to be created in the UI for this test to run.") @@ -27,3 +28,33 @@ def test(): cli(["custom-metrics", "add-in-bulk", "-t", "cli-test-service", "-k", "vulnerabilities", "-v", f"{date}=1.0", "-v", f"{date}=2.0"]) result = cli(["custom-metrics", "get", "-t", "cli-test-service", "-k", "vulnerabilities"]) assert result['total'] == 3, "should have total of 3 metrics data points" + +def test_definition_create(): + # Skip if ENG_METRICS not enabled in tenant + access_check = cli(["custom-metrics", "get", "-t", "cli-test-service", "-k", "vulnerabilities"], ReturnType.STDOUT) + if "HTTP Error 403: Product access to [ENG_METRICS] not permitted" in access_check: + print("Skipping - no Eng Metrics access") + return + + base_url = os.environ.get("CORTEX_BASE_URL", "https://api.getcortexapp.com").rstrip("/") + api_key = os.environ.get("CORTEX_API_KEY", "") + # Pre-clean any leftover from a prior interrupted run (ignore result) + _requests.delete( + f"{base_url}/api/v1/eng-intel/custom-metrics/cli-test-definition", + headers={"Authorization": f"Bearer {api_key}"}, + ) + + result = cli(["custom-metrics", "definition-create", + "-k", "cli-test-definition", + "-n", "CLI Test Definition", + "-V", "POSITIVE", + "-c", "Test", + "-d", "Temporary definition for CLI testing", + "--is-draft"]) + assert result["key"] == "cli-test-definition" + assert result["name"] == "CLI Test Definition" + assert result["valence"] == "POSITIVE" + assert result["category"] == "Test" + assert result["description"] == "Temporary definition for CLI testing" + assert result["isDraft"] is True + assert result["type"] == "PUBLIC_API" From 222749570fd782e41b62f4afb94a1def244880b6 Mon Sep 17 00:00:00 2001 From: Jeff Schnitter Date: Fri, 23 Oct 2026 14:25:46 -0700 Subject: [PATCH 02/10] feat: add ai-spend setup.py to auto-create metric definition --- cortexapps_cli/solutions/ai-spend/setup.py | 92 ++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 cortexapps_cli/solutions/ai-spend/setup.py diff --git a/cortexapps_cli/solutions/ai-spend/setup.py b/cortexapps_cli/solutions/ai-spend/setup.py new file mode 100644 index 00000000..87f6dcbb --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/setup.py @@ -0,0 +1,92 @@ +""" +Post-install setup script for the ai-spend solution. +Creates the ai-spend custom metric definition in Cortex. +Run via: cortex solutions post-install -s ai-spend +""" + +SETUP_DESCRIPTION = ( + "This solution includes a setup script that will create " + "the 'ai-spend' custom metric definition in your Cortex instance." +) + +import sys +from pathlib import Path +import requests + +try: + from cortexapps_cli.solutions._lib.setup_base import SolutionSetup +except ImportError: + sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + from _lib.setup_base import SolutionSetup + +METRIC_PAYLOAD = { + "key": "ai-spend", + "name": "AI Spend", + "valence": "POSITIVE", + "category": "Cost", + "description": "Weekly Claude AI spend per entity in USD", +} + + +class AiSpendSetup(SolutionSetup): + solution_tag = "ai-spend" + + def __init__( + self, + cortex_api_key: str = None, + cortex_base_url: str = None, + no_prompt: bool = False, + **kwargs, + ): + super().__init__(no_prompt=no_prompt, **kwargs) + self._api_key = cortex_api_key or "" + self._base_url = (cortex_base_url or "https://api.getcortexapp.com").rstrip("/") + + def _cortex_headers(self) -> dict: + return { + "Authorization": f"Bearer {self._api_key}", + "Content-Type": "application/json", + } + + def collect_prompts(self) -> None: + pass + + def _create_metric_definition(self) -> None: + """Create the ai-spend metric definition via the Cortex API.""" + if self.already_done("create-definition"): + return "Already created (skipped)" + + r = requests.post( + f"{self._base_url}/api/v1/eng-intel/custom-metrics", + headers=self._cortex_headers(), + json=METRIC_PAYLOAD, + ) + + if r.status_code == 409: + print(" ai-spend metric definition already exists, skipping") + self.mark_done("create-definition") + return + + r.raise_for_status() + self.mark_done("create-definition") + print(" ✓ Created ai-spend metric definition") + + def steps(self) -> list: + return [ + ("Create ai-spend metric definition", self._create_metric_definition), + ] + + def post_steps(self) -> None: + print("\n✓ ai-spend metric definition is ready.\n") + + +def main(cortex_api_key=None, cortex_base_url=None, no_prompt=False, **kwargs): + AiSpendSetup( + cortex_api_key=cortex_api_key, + cortex_base_url=cortex_base_url, + no_prompt=no_prompt, + ).run() + + +if __name__ == "__main__": + main() From c2a11cadaa3c017c55ab7350110151dfe828c6b2 Mon Sep 17 00:00:00 2001 From: Jeff Schnitter Date: Fri, 23 Oct 2026 14:27:40 -0700 Subject: [PATCH 03/10] =?UTF-8?q?docs:=20update=20ai-spend=20README=20?= =?UTF-8?q?=E2=80=94=20metric=20definition=20is=20now=20auto-created=20on?= =?UTF-8?q?=20install?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cortexapps_cli/solutions/ai-spend/README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cortexapps_cli/solutions/ai-spend/README.md b/cortexapps_cli/solutions/ai-spend/README.md index cd5fc167..8a1155a7 100644 --- a/cortexapps_cli/solutions/ai-spend/README.md +++ b/cortexapps_cli/solutions/ai-spend/README.md @@ -65,11 +65,6 @@ Register every employee as a Cortex entity linked to their team, push weekly Cla | Sync script | `scripts/sync-claude-spend.py` | | GH Actions workflow | `.github/workflows/sync-claude-spend.yaml` | -## Prerequisites - -Before installing, create the `ai-spend` custom metric definition in your Cortex instance: -**Eng Intel → Custom Metrics → New Metric**, key: `ai-spend`. - ## Installation ``` @@ -161,5 +156,5 @@ The sample entities are fictional. Add your real employees as catalog entities w - Users who authenticate Claude Code with a personal API key (not Enterprise OAuth) show $0 spend in the Analytics API and are skipped automatically. - Cost data may take up to 24 hours to appear; dates at least 30 days old are considered final for billing purposes. -- The `ai-spend` custom metric definition must currently be created manually before installing. A future release will support auto-creation of custom metric definitions as part of `cortex solutions install`. +- The `ai-spend` custom metric definition is created automatically during install by the solution's setup script (`cortex solutions post-install -s ai-spend`). - The scorecard's Bronze rule uses a `P1Y` lookback to accommodate sample data. Once your weekly sync is running consistently, consider tightening it to `P8D` to ensure the rule only passes when data is fresh. The Silver and Gold rules use `P8D` and can similarly be adjusted to match your sync frequency. From f55bf9330b1a4cb02418dcc38b966f3d36061d85 Mon Sep 17 00:00:00 2001 From: Jeff Schnitter Date: Fri, 23 Oct 2026 14:31:59 -0700 Subject: [PATCH 04/10] fix: correct return type annotation and update stale test skip message --- cortexapps_cli/solutions/ai-spend/setup.py | 2 +- tests/test_custom_metrics.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cortexapps_cli/solutions/ai-spend/setup.py b/cortexapps_cli/solutions/ai-spend/setup.py index 87f6dcbb..7257edbd 100644 --- a/cortexapps_cli/solutions/ai-spend/setup.py +++ b/cortexapps_cli/solutions/ai-spend/setup.py @@ -51,7 +51,7 @@ def _cortex_headers(self) -> dict: def collect_prompts(self) -> None: pass - def _create_metric_definition(self) -> None: + def _create_metric_definition(self) -> str | None: """Create the ai-spend metric definition via the Cortex API.""" if self.already_done("create-definition"): return "Already created (skipped)" diff --git a/tests/test_custom_metrics.py b/tests/test_custom_metrics.py index 7f85642d..b9babe64 100644 --- a/tests/test_custom_metrics.py +++ b/tests/test_custom_metrics.py @@ -15,8 +15,7 @@ def test(): # 'vulnerabilities' must already exist as a metric definition; use `cortex custom-metrics definition-create` to create it. result = cli(["custom-metrics", "get", "-t", "cli-test-service", "-k", "vulnerabilities"], ReturnType.STDOUT) if "HTTP Error 404: Not Found - CustomMetricKey not found" in result: - print("Custom metric named 'vulnerabilities' does not exist. It has to be created in the UI for this test to run.") - print("To create: Settings -> Eng Intelligence -> General -> Custom -> Add Metric -> (select API toggle).") + print("Custom metric named 'vulnerabilities' does not exist. Create it with: cortex custom-metrics definition-create -k vulnerabilities -n Vulnerabilities -V POSITIVE") return date = today() From 61b6b57785575e22c91009eefbd729617352687b Mon Sep 17 00:00:00 2001 From: Jeff Schnitter Date: Fri, 23 Oct 2026 14:33:42 -0700 Subject: [PATCH 05/10] fix: fall back to config file for api key in test pre-clean --- tests/test_custom_metrics.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_custom_metrics.py b/tests/test_custom_metrics.py index b9babe64..a1acb6cf 100644 --- a/tests/test_custom_metrics.py +++ b/tests/test_custom_metrics.py @@ -35,8 +35,11 @@ def test_definition_create(): print("Skipping - no Eng Metrics access") return - base_url = os.environ.get("CORTEX_BASE_URL", "https://api.getcortexapp.com").rstrip("/") - api_key = os.environ.get("CORTEX_API_KEY", "") + import configparser + _cfg = configparser.ConfigParser() + _cfg.read(os.path.expanduser("~/.cortex/config")) + base_url = (os.environ.get("CORTEX_BASE_URL") or _cfg.get("default", "base_url", fallback="https://api.getcortexapp.com")).rstrip("/") + api_key = os.environ.get("CORTEX_API_KEY") or _cfg.get("default", "api_key", fallback="") # Pre-clean any leftover from a prior interrupted run (ignore result) _requests.delete( f"{base_url}/api/v1/eng-intel/custom-metrics/cli-test-definition", From 2cecc483c63844cc3dc93b2aa0f2833905500467 Mon Sep 17 00:00:00 2001 From: Jeff Schnitter Date: Fri, 23 Oct 2026 15:36:09 -0700 Subject: [PATCH 06/10] feat: add --file option to custom-metrics definition-create --- cortexapps_cli/commands/custom_metrics.py | 36 +++++++++++++---------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/cortexapps_cli/commands/custom_metrics.py b/cortexapps_cli/commands/custom_metrics.py index 1dd61a2c..ac80b508 100644 --- a/cortexapps_cli/commands/custom_metrics.py +++ b/cortexapps_cli/commands/custom_metrics.py @@ -1,5 +1,6 @@ from collections import defaultdict from datetime import datetime +import json import typer from typing import Optional from typing_extensions import Annotated @@ -137,9 +138,10 @@ def delete( @app.command() def definition_create( ctx: typer.Context, - key: str = typer.Option(..., "--key", "-k", help="Unique metric key (2-255 chars, letters/digits/hyphens, alphanumeric start/end)"), - name: str = typer.Option(..., "--name", "-n", help="Display name for the metric"), - valence: str = typer.Option(..., "--valence", "-V", help="Whether higher values are preferable: POSITIVE, NEGATIVE, or NEUTRAL"), + file_input: Annotated[typer.FileText, typer.Option("--file", "-f", help="JSON file containing the metric definition; can be passed as stdin with -, example: -f-")] = None, + key: Optional[str] = typer.Option(None, "--key", "-k", help="Unique metric key (2-255 chars, letters/digits/hyphens, alphanumeric start/end)"), + name: Optional[str] = typer.Option(None, "--name", "-n", help="Display name for the metric"), + valence: Optional[str] = typer.Option(None, "--valence", "-V", help="Whether higher values are preferable: POSITIVE, NEGATIVE, or NEUTRAL"), category: Optional[str] = typer.Option(None, "--category", "-c", help="Grouping label"), description: Optional[str] = typer.Option(None, "--description", "-d", help="What the metric measures"), is_draft: bool = typer.Option(False, "--is-draft", help="If true, metric accepts data but is hidden from Eng Intelligence"), @@ -150,18 +152,22 @@ def definition_create( client = ctx.obj["client"] - data = { - "key": key, - "name": name, - "valence": valence, - "category": category, - "description": description, - "isDraft": is_draft, - "type": "PUBLIC_API" - } - - # Remove keys whose value is None - data = {k: v for k, v in data.items() if v is not None} + if file_input: + data = json.loads("".join([line for line in file_input])) + else: + if not key or not name or not valence: + raise typer.BadParameter("--key, --name, and --valence are required when --file is not provided") + data = { + "key": key, + "name": name, + "valence": valence, + "category": category, + "description": description, + "isDraft": is_draft, + "type": "PUBLIC_API" + } + # Remove keys whose value is None + data = {k: v for k, v in data.items() if v is not None} result = client.post("api/v1/eng-intel/custom-metrics", data=data) print_json(data=result) From 68f8be88ce8e57e860092c1ea4e2848351e94244 Mon Sep 17 00:00:00 2001 From: Jeff Schnitter Date: Mon, 26 Oct 2026 16:09:50 -0700 Subject: [PATCH 07/10] feat: add definition-list, definition-get, definition-update, definition-delete commands Co-Authored-By: Claude Sonnet 4.6 --- cortexapps_cli/commands/custom_metrics.py | 69 +++++++++++++++++++ tests/test_custom_metrics.py | 82 +++++++++++++++++++++++ 2 files changed, 151 insertions(+) diff --git a/cortexapps_cli/commands/custom_metrics.py b/cortexapps_cli/commands/custom_metrics.py index ac80b508..5b7300a0 100644 --- a/cortexapps_cli/commands/custom_metrics.py +++ b/cortexapps_cli/commands/custom_metrics.py @@ -171,3 +171,72 @@ def definition_create( result = client.post("api/v1/eng-intel/custom-metrics", data=data) print_json(data=result) + +@app.command() +def definition_list( + ctx: typer.Context, + page: int | None = typer.Option(None, "--page", "-p", help="Page number, 0-indexed — omit to fetch all pages"), + page_size: int | None = typer.Option(None, "--page-size", "-z", help="Page size for results"), + prt: bool = typer.Option(True, "--print", help="If result should be printed to the terminal", hidden=True), +): + """List custom metric definitions""" + + client = ctx.obj["client"] + + params = {} + if page_size is not None: + params["pageSize"] = page_size + + client.fetch_or_get("api/v1/eng-intel/custom-metrics", page, prt, params=params) + +@app.command() +def definition_get( + ctx: typer.Context, + key: str = typer.Option(..., "--key", "-k", help="Metric key"), +): + """Get a custom metric definition""" + + client = ctx.obj["client"] + + result = client.get("api/v1/eng-intel/custom-metrics/" + key) + print_json(data=result) + +@app.command() +def definition_update( + ctx: typer.Context, + key: str = typer.Option(..., "--key", "-k", help="Metric key to update"), + file_input: Annotated[typer.FileText, typer.Option("--file", "-f", help="JSON file with updated definition; can be passed as stdin with -, example: -f-")] = None, + name: Optional[str] = typer.Option(None, "--name", "-n", help="Display name for the metric"), + valence: Optional[str] = typer.Option(None, "--valence", "-V", help="POSITIVE, NEGATIVE, or NEUTRAL"), + category: Optional[str] = typer.Option(None, "--category", "-c", help="Grouping label"), + description: Optional[str] = typer.Option(None, "--description", "-d", help="What the metric measures"), +): + """Update a custom metric definition""" + + client = ctx.obj["client"] + + if file_input: + data = json.loads("".join([line for line in file_input])) + else: + data = { + "name": name, + "valence": valence, + "category": category, + "description": description, + } + # Strip None-valued keys + data = {k: v for k, v in data.items() if v is not None} + + result = client.put("api/v1/eng-intel/custom-metrics/" + key, data=data) + print_json(data=result) + +@app.command() +def definition_delete( + ctx: typer.Context, + key: str = typer.Option(..., "--key", "-k", help="Metric key to delete"), +): + """Delete a custom metric definition""" + + client = ctx.obj["client"] + + client.delete("api/v1/eng-intel/custom-metrics/" + key) diff --git a/tests/test_custom_metrics.py b/tests/test_custom_metrics.py index a1acb6cf..319f1304 100644 --- a/tests/test_custom_metrics.py +++ b/tests/test_custom_metrics.py @@ -60,3 +60,85 @@ def test_definition_create(): assert result["description"] == "Temporary definition for CLI testing" assert result["isDraft"] is True assert result["type"] == "PUBLIC_API" + + +def test_definition_list(): + access_check = cli(["custom-metrics", "get", "-t", "cli-test-service", "-k", "vulnerabilities"], ReturnType.STDOUT) + if "HTTP Error 403: Product access to [ENG_METRICS] not permitted" in access_check: + print("Skipping - no Eng Metrics access") + return + result = cli(["custom-metrics", "definition-list"]) + assert "customMetrics" in result + assert isinstance(result["customMetrics"], list) + # Verify pagination params work + result_paged = cli(["custom-metrics", "definition-list", "--page", "0", "--page-size", "1"]) + assert "customMetrics" in result_paged + assert isinstance(result_paged["customMetrics"], list) + + +def test_definition_get(): + access_check = cli(["custom-metrics", "get", "-t", "cli-test-service", "-k", "vulnerabilities"], ReturnType.STDOUT) + if "HTTP Error 403: Product access to [ENG_METRICS] not permitted" in access_check: + print("Skipping - no Eng Metrics access") + return + import configparser + _cfg = configparser.ConfigParser() + _cfg.read(os.path.expanduser("~/.cortex/config")) + base_url = (os.environ.get("CORTEX_BASE_URL") or _cfg.get("default", "base_url", fallback="https://api.getcortexapp.com")).rstrip("/") + api_key = os.environ.get("CORTEX_API_KEY") or _cfg.get("default", "api_key", fallback="") + _requests.delete( + f"{base_url}/api/v1/eng-intel/custom-metrics/cli-test-def-get", + headers={"Authorization": f"Bearer {api_key}"}, + ) + cli(["custom-metrics", "definition-create", "-k", "cli-test-def-get", "-n", "CLI Test Get", "-V", "POSITIVE", "--is-draft"]) + result = cli(["custom-metrics", "definition-get", "-k", "cli-test-def-get"]) + assert result["key"] == "cli-test-def-get" + assert result["name"] == "CLI Test Get" + _requests.delete( + f"{base_url}/api/v1/eng-intel/custom-metrics/cli-test-def-get", + headers={"Authorization": f"Bearer {api_key}"}, + ) + + +def test_definition_update(): + access_check = cli(["custom-metrics", "get", "-t", "cli-test-service", "-k", "vulnerabilities"], ReturnType.STDOUT) + if "HTTP Error 403: Product access to [ENG_METRICS] not permitted" in access_check: + print("Skipping - no Eng Metrics access") + return + import configparser + _cfg = configparser.ConfigParser() + _cfg.read(os.path.expanduser("~/.cortex/config")) + base_url = (os.environ.get("CORTEX_BASE_URL") or _cfg.get("default", "base_url", fallback="https://api.getcortexapp.com")).rstrip("/") + api_key = os.environ.get("CORTEX_API_KEY") or _cfg.get("default", "api_key", fallback="") + _requests.delete( + f"{base_url}/api/v1/eng-intel/custom-metrics/cli-test-def-update", + headers={"Authorization": f"Bearer {api_key}"}, + ) + cli(["custom-metrics", "definition-create", "-k", "cli-test-def-update", "-n", "CLI Test Update", "-V", "POSITIVE", "--is-draft"]) + result = cli(["custom-metrics", "definition-update", "-k", "cli-test-def-update", "-n", "CLI Test Updated"]) + assert result["key"] == "cli-test-def-update" + assert result["name"] == "CLI Test Updated" + _requests.delete( + f"{base_url}/api/v1/eng-intel/custom-metrics/cli-test-def-update", + headers={"Authorization": f"Bearer {api_key}"}, + ) + + +def test_definition_delete(): + access_check = cli(["custom-metrics", "get", "-t", "cli-test-service", "-k", "vulnerabilities"], ReturnType.STDOUT) + if "HTTP Error 403: Product access to [ENG_METRICS] not permitted" in access_check: + print("Skipping - no Eng Metrics access") + return + import configparser + _cfg = configparser.ConfigParser() + _cfg.read(os.path.expanduser("~/.cortex/config")) + base_url = (os.environ.get("CORTEX_BASE_URL") or _cfg.get("default", "base_url", fallback="https://api.getcortexapp.com")).rstrip("/") + api_key = os.environ.get("CORTEX_API_KEY") or _cfg.get("default", "api_key", fallback="") + _requests.delete( + f"{base_url}/api/v1/eng-intel/custom-metrics/cli-test-def-delete", + headers={"Authorization": f"Bearer {api_key}"}, + ) + cli(["custom-metrics", "definition-create", "-k", "cli-test-def-delete", "-n", "CLI Test Delete", "-V", "POSITIVE", "--is-draft"]) + cli(["custom-metrics", "definition-delete", "-k", "cli-test-def-delete"]) + result = cli(["custom-metrics", "definition-get", "-k", "cli-test-def-delete"], ReturnType.STDOUT) + assert "404" in result or "Not Found" in result From 712da1ac4b12fd90fcbb273617ffd5dd545c0851 Mon Sep 17 00:00:00 2001 From: Jeff Schnitter Date: Mon, 26 Oct 2026 16:13:24 -0700 Subject: [PATCH 08/10] feat: add custom-metric-definitions to backup export and import Co-Authored-By: Claude Sonnet 4.6 --- cortexapps_cli/commands/backup.py | 52 +++++++++++++++++++++++++++++-- tests/test_backup.py | 43 +++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/cortexapps_cli/commands/backup.py b/cortexapps_cli/commands/backup.py index c7a22597..73500566 100644 --- a/cortexapps_cli/commands/backup.py +++ b/cortexapps_cli/commands/backup.py @@ -220,9 +220,18 @@ def _export_catalog_pages(ctx, directory): except Exception as e: print(f"Failed to export catalog page {slug}: {e}") +def _export_custom_metric_definitions(ctx, directory): + directory = _directory_name(directory, "custom-metric-definitions") + client = ctx.obj["client"] + data = client.fetch("api/v1/eng-intel/custom-metrics", params={}) + for definition in data.get("customMetrics", []): + key = definition["key"] + _file_name(directory, key, definition, "json") + backupTypes = sorted([ "catalog", "catalogs", + "custom-metric-definitions", "entity-relationship-types", "entity-relationships", "entity-types", @@ -273,6 +282,7 @@ def export( Exports the following objects: - catalog - catalogs + - custom-metric-definitions - entity-types - entity-relationship-types - entity-relationships @@ -293,7 +303,7 @@ def export( - api-keys - custom-events - custom-metadata created by the public API - - custom-metrics + - custom-metrics data points (definitions are backed up; per-entity data points are not) - dependencies created by the API - deploys - docs created by the API @@ -312,6 +322,10 @@ def export( _create_directory(directory) if "catalog" in export_types: _export_catalog(ctx, directory, catalog_types) + if "catalogs" in export_types: + _export_catalog_pages(ctx, directory) + if "custom-metric-definitions" in export_types: + _export_custom_metric_definitions(ctx, directory) if "entity-types" in export_types: _export_entity_types(ctx, directory) if "entity-relationship-types" in export_types: @@ -326,8 +340,6 @@ def export( _export_scorecards(ctx, directory) if "workflows" in export_types: _export_workflows(ctx, directory) - if "catalogs" in export_types: - _export_catalog_pages(ctx, directory) print("\nExport complete!") print("Contents available in " + directory) @@ -489,6 +501,33 @@ def import_relationships_file(file_info): return ("entity-relationships", len(results) - failed_count, [(fp, et, em) for rt, fp, et, em in results if et]) +def _import_custom_metric_definitions(ctx, directory): + imported = 0 + failed = [] + if os.path.isdir(directory): + print("Processing: " + directory) + client = ctx.obj["client"] + for filename in sorted(os.listdir(directory)): + if not filename.endswith(".json"): + continue + file_path = os.path.join(directory, filename) + if not os.path.isfile(file_path): + continue + try: + print(" Importing: " + filename) + with open(file_path) as f: + definition = json.load(f) + client.post("api/v1/eng-intel/custom-metrics", data=definition) + imported += 1 + except Exception as e: + if "409" in str(e): + print(f" Definition {filename} already exists, skipping") + imported += 1 + else: + print(f" Failed: {type(e).__name__} - {str(e)}") + failed.append((file_path, type(e).__name__, str(e))) + return ("custom-metric-definitions", imported, failed) + def _import_custom_metrics(ctx, directory): imported = 0 failed = [] @@ -846,6 +885,10 @@ def import_tenant( ): """ Import data into tenant + + If the import directory contains a custom-metric-definitions/ subdirectory, + definitions are created before custom-metrics data points are imported. + Existing definitions (HTTP 409) are skipped silently and counted as successes. """ client = ctx.obj["client"] @@ -857,6 +900,7 @@ def import_tenant( all_stats.append(_import_entity_relationship_types(ctx, directory + "/entity-relationship-types")) all_stats.append(_import_catalog(ctx, directory + "/catalog")) all_stats.append(_import_entity_relationships(ctx, directory + "/entity-relationships")) + all_stats.append(_import_custom_metric_definitions(ctx, directory + "/custom-metric-definitions")) all_stats.append(_import_custom_metrics(ctx, directory + "/custom-metrics")) all_stats.append(_import_plugins(ctx, directory + "/plugins")) all_stats.append(_import_scorecards(ctx, directory + "/scorecards")) @@ -910,6 +954,8 @@ def import_tenant( elif import_type == "entity-relationships": # These need special handling - would need the relationship type print(f"# Manual retry needed for entity-relationships: {file_path}") + elif import_type == "custom-metric-definitions": + print(f"cortex custom-metrics definition-create -f \"{file_path}\"") elif import_type == "custom-metrics": print(f"# Manual retry needed for custom-metrics: {file_path}") elif import_type == "plugins": diff --git a/tests/test_backup.py b/tests/test_backup.py index d0dcee75..4ea17d26 100644 --- a/tests/test_backup.py +++ b/tests/test_backup.py @@ -56,6 +56,49 @@ def test_backup_import_invalid_api_key(monkeypatch): assert result.exit_code != 0, f"backup import should exit with non-zero code on failure, got exit_code={result.exit_code}" +def test_backup_import_custom_metric_definitions_invalid_api_key(monkeypatch): + """ + Test that backup import of custom-metric-definitions fails cleanly with invalid API key. + """ + monkeypatch.setenv("CORTEX_API_KEY", "invalidKey") + + with tempfile.TemporaryDirectory() as tmpdir: + defs_dir = os.path.join(tmpdir, "custom-metric-definitions") + os.makedirs(defs_dir) + + def_file = os.path.join(defs_dir, "test-metric.json") + with open(def_file, "w") as f: + json.dump({ + "key": "test-metric", + "name": "Test Metric", + "valence": "POSITIVE", + "type": "PUBLIC_API", + "isDraft": True, + }, f) + + result = cli(["backup", "import", "-d", tmpdir], return_type=ReturnType.RAW) + assert result.exit_code != 0, ( + f"backup import should exit with non-zero code on failure, " + f"got exit_code={result.exit_code}" + ) + + +def test_backup_import_custom_metric_definitions_empty_dir(): + """ + Test that backup import with an empty custom-metric-definitions/ dir is a silent no-op. + """ + with tempfile.TemporaryDirectory() as tmpdir: + defs_dir = os.path.join(tmpdir, "custom-metric-definitions") + os.makedirs(defs_dir) + # No files — import should succeed with 0 imported and 0 failed + result = cli(["backup", "import", "-d", tmpdir], return_type=ReturnType.RAW) + # Empty dir is a no-op — exit code 0 (all_stats has 0 failed) + assert result.exit_code == 0, ( + f"backup import with empty custom-metric-definitions/ should exit 0, " + f"got exit_code={result.exit_code}\n{result.stdout}" + ) + + def test_backup_export_invalid_api_key(monkeypatch): """ Test that backup export exits with non-zero return code and clean error message when API calls fail. From a378a053d71441926e0bfd54181984ae8f53dceb Mon Sep 17 00:00:00 2001 From: Jeff Schnitter Date: Mon, 26 Oct 2026 16:16:54 -0700 Subject: [PATCH 09/10] feat: install ai-spend metric definition via solutions install, not post-install Co-Authored-By: Claude Sonnet 4.6 --- cortexapps_cli/solutions/ai-spend/README.md | 2 +- .../custom-metric-definitions/ai-spend.json | 9 ++++ cortexapps_cli/solutions/ai-spend/setup.py | 46 ++----------------- 3 files changed, 14 insertions(+), 43 deletions(-) create mode 100644 cortexapps_cli/solutions/ai-spend/custom-metric-definitions/ai-spend.json diff --git a/cortexapps_cli/solutions/ai-spend/README.md b/cortexapps_cli/solutions/ai-spend/README.md index 8a1155a7..4f3cfa5e 100644 --- a/cortexapps_cli/solutions/ai-spend/README.md +++ b/cortexapps_cli/solutions/ai-spend/README.md @@ -156,5 +156,5 @@ The sample entities are fictional. Add your real employees as catalog entities w - Users who authenticate Claude Code with a personal API key (not Enterprise OAuth) show $0 spend in the Analytics API and are skipped automatically. - Cost data may take up to 24 hours to appear; dates at least 30 days old are considered final for billing purposes. -- The `ai-spend` custom metric definition is created automatically during install by the solution's setup script (`cortex solutions post-install -s ai-spend`). +- The `ai-spend` custom metric definition is installed automatically via `custom-metric-definitions/ai-spend.json` when you run `cortex solutions install -s ai-spend`. - The scorecard's Bronze rule uses a `P1Y` lookback to accommodate sample data. Once your weekly sync is running consistently, consider tightening it to `P8D` to ensure the rule only passes when data is fresh. The Silver and Gold rules use `P8D` and can similarly be adjusted to match your sync frequency. diff --git a/cortexapps_cli/solutions/ai-spend/custom-metric-definitions/ai-spend.json b/cortexapps_cli/solutions/ai-spend/custom-metric-definitions/ai-spend.json new file mode 100644 index 00000000..257bbc8c --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/custom-metric-definitions/ai-spend.json @@ -0,0 +1,9 @@ +{ + "key": "ai-spend", + "name": "AI Spend", + "type": "PUBLIC_API", + "valence": "POSITIVE", + "category": "Cost", + "description": "Weekly Claude AI spend per entity in USD", + "isDraft": false +} diff --git a/cortexapps_cli/solutions/ai-spend/setup.py b/cortexapps_cli/solutions/ai-spend/setup.py index 7257edbd..1ba01bdc 100644 --- a/cortexapps_cli/solutions/ai-spend/setup.py +++ b/cortexapps_cli/solutions/ai-spend/setup.py @@ -1,17 +1,15 @@ """ Post-install setup script for the ai-spend solution. -Creates the ai-spend custom metric definition in Cortex. Run via: cortex solutions post-install -s ai-spend """ SETUP_DESCRIPTION = ( - "This solution includes a setup script that will create " - "the 'ai-spend' custom metric definition in your Cortex instance." + "The ai-spend solution installs custom metric definitions automatically. " + "This post-install script confirms the setup is complete." ) import sys from pathlib import Path -import requests try: from cortexapps_cli.solutions._lib.setup_base import SolutionSetup @@ -19,14 +17,6 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent)) from _lib.setup_base import SolutionSetup -METRIC_PAYLOAD = { - "key": "ai-spend", - "name": "AI Spend", - "valence": "POSITIVE", - "category": "Cost", - "description": "Weekly Claude AI spend per entity in USD", -} - class AiSpendSetup(SolutionSetup): solution_tag = "ai-spend" @@ -42,42 +32,14 @@ def __init__( self._api_key = cortex_api_key or "" self._base_url = (cortex_base_url or "https://api.getcortexapp.com").rstrip("/") - def _cortex_headers(self) -> dict: - return { - "Authorization": f"Bearer {self._api_key}", - "Content-Type": "application/json", - } - def collect_prompts(self) -> None: pass - def _create_metric_definition(self) -> str | None: - """Create the ai-spend metric definition via the Cortex API.""" - if self.already_done("create-definition"): - return "Already created (skipped)" - - r = requests.post( - f"{self._base_url}/api/v1/eng-intel/custom-metrics", - headers=self._cortex_headers(), - json=METRIC_PAYLOAD, - ) - - if r.status_code == 409: - print(" ai-spend metric definition already exists, skipping") - self.mark_done("create-definition") - return - - r.raise_for_status() - self.mark_done("create-definition") - print(" ✓ Created ai-spend metric definition") - def steps(self) -> list: - return [ - ("Create ai-spend metric definition", self._create_metric_definition), - ] + return [] def post_steps(self) -> None: - print("\n✓ ai-spend metric definition is ready.\n") + print("\n✓ ai-spend solution is ready.\n") def main(cortex_api_key=None, cortex_base_url=None, no_prompt=False, **kwargs): From 7c412a397fa25307e1b71e7b95a271a6f6cd048a Mon Sep 17 00:00:00 2001 From: Jeff Schnitter Date: Mon, 26 Oct 2026 16:36:28 -0700 Subject: [PATCH 10/10] =?UTF-8?q?remove:=20ai-spend=20setup.py=20=E2=80=94?= =?UTF-8?q?=20definition=20now=20installed=20via=20custom-metric-definitio?= =?UTF-8?q?ns/=20during=20solutions=20install?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cortexapps_cli/solutions/ai-spend/setup.py | 54 ---------------------- 1 file changed, 54 deletions(-) delete mode 100644 cortexapps_cli/solutions/ai-spend/setup.py diff --git a/cortexapps_cli/solutions/ai-spend/setup.py b/cortexapps_cli/solutions/ai-spend/setup.py deleted file mode 100644 index 1ba01bdc..00000000 --- a/cortexapps_cli/solutions/ai-spend/setup.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Post-install setup script for the ai-spend solution. -Run via: cortex solutions post-install -s ai-spend -""" - -SETUP_DESCRIPTION = ( - "The ai-spend solution installs custom metric definitions automatically. " - "This post-install script confirms the setup is complete." -) - -import sys -from pathlib import Path - -try: - from cortexapps_cli.solutions._lib.setup_base import SolutionSetup -except ImportError: - sys.path.insert(0, str(Path(__file__).parent.parent.parent)) - from _lib.setup_base import SolutionSetup - - -class AiSpendSetup(SolutionSetup): - solution_tag = "ai-spend" - - def __init__( - self, - cortex_api_key: str = None, - cortex_base_url: str = None, - no_prompt: bool = False, - **kwargs, - ): - super().__init__(no_prompt=no_prompt, **kwargs) - self._api_key = cortex_api_key or "" - self._base_url = (cortex_base_url or "https://api.getcortexapp.com").rstrip("/") - - def collect_prompts(self) -> None: - pass - - def steps(self) -> list: - return [] - - def post_steps(self) -> None: - print("\n✓ ai-spend solution is ready.\n") - - -def main(cortex_api_key=None, cortex_base_url=None, no_prompt=False, **kwargs): - AiSpendSetup( - cortex_api_key=cortex_api_key, - cortex_base_url=cortex_base_url, - no_prompt=no_prompt, - ).run() - - -if __name__ == "__main__": - main()