diff --git a/docs/how-to/just-run-blueapi-and-services-locally.md b/docs/how-to/just-run-blueapi-and-services-locally.md index 5a8303b42..005e73482 100644 --- a/docs/how-to/just-run-blueapi-and-services-locally.md +++ b/docs/how-to/just-run-blueapi-and-services-locally.md @@ -35,3 +35,23 @@ To log in through the BlueAPI CLI: 4. Run `blueapi controller plans` to check that the log in has succeeded By default the BlueAPI instance will be available via the OAuth2 proxy at `localhost:4180`, and Tiled through its OAuth2 proxy at `localhost:4181`. + +## Metrics and traces + +BlueAPI reports metrics and traces with OpenTelemetry, pushed over OTLP when `OTLP_EXPORT_ENABLED=true` (to wherever the standard `OTEL_EXPORTER_OTLP_*` variables point). To look at them locally, the compose file has an optional `monitoring` profile with Prometheus, Grafana, Jaeger and Tempo, behind an OpenTelemetry Collector that sends traces to both Jaeger and Tempo: + +1. `just compose --profile monitoring up -d` starts the usual services plus the monitoring stack +2. `just serve-monitored` runs the server with its telemetry pointed at that stack (see `tests/system_tests/services/monitoring/otel.env`) +3. Run some plans, e.g. `just run sleep '{"time": 2}'` +4. Open: + - Grafana at , which opens on the BlueAPI dashboard + - Prometheus at + - Jaeger at + - Tempo through Grafana: Explore, then the Tempo datasource + +| Metric | Labels | Meaning | +| --- | --- | --- | +| `blueapi.task.duration` (`blueapi_task_duration_seconds_*` in Prometheus) | `task.name`, `outcome` (`success`/`failure`) | Histogram of how long each task took; its count is the number of runs | +| `http.server.*` | method, route, status code | Request metrics for the REST API, from the FastAPI instrumentation | + +Every metric also carries `service.name` (`BlueAPI` for the API process, `BlueAPIWorker` for the worker subprocess) and `service.instrument`, taken from the `INSTRUMENT` or `BEAMLINE` environment variable. diff --git a/justfile b/justfile index 23c5aacd6..177c5a8ac 100644 --- a/justfile +++ b/justfile @@ -30,6 +30,12 @@ serve *OPTS: source tests/system_tests/.env uv run blueapi -c {{ TEST_CONFIG }} {{ OPTS }} serve +[doc('Run the server exporting metrics and traces to the monitoring stack (`just compose --profile monitoring up -d`)')] +serve-monitored *OPTS: + #!/usr/bin/env bash + source tests/system_tests/services/monitoring/otel.env + just serve {{ OPTS }} + [doc('Run a plan using the blueapi CLI')] run PLAN PARAMS: uv run blueapi -c {{ TEST_CONFIG }} controller run -i {{ SESSION }} {{ PLAN }} '{{ PARAMS }}' diff --git a/src/blueapi/cli/cli.py b/src/blueapi/cli/cli.py index d78a88d01..44a2afda3 100644 --- a/src/blueapi/cli/cli.py +++ b/src/blueapi/cli/cli.py @@ -38,6 +38,7 @@ ) from blueapi.core import OTLP_EXPORT_ENABLED, DataEvent from blueapi.log import set_up_logging +from blueapi.metrics import setup_metrics from blueapi.service.model import DeviceResponse, PlanResponse, SourceInfo, TaskRequest from blueapi.worker import ProgressEvent, WorkerEvent from blueapi.worker.event import TaskError, TaskResult @@ -249,6 +250,7 @@ def start_application(obj: dict): observability context. """ setup_tracing("BlueAPI", OTLP_EXPORT_ENABLED) + setup_metrics(OTLP_EXPORT_ENABLED) start(config) diff --git a/src/blueapi/metrics.py b/src/blueapi/metrics.py new file mode 100644 index 000000000..144ee25c0 --- /dev/null +++ b/src/blueapi/metrics.py @@ -0,0 +1,35 @@ +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.metrics import get_meter, set_meter_provider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.trace import get_tracer_provider + +METER = get_meter("blueapi") + +TASK_DURATION = METER.create_histogram( + "blueapi.task.duration", + unit="s", + description="Time taken to run a task, by task name and outcome", + # Plans run for anything between a fraction of a second and hours + explicit_bucket_boundaries_advisory=[ + 0.1, 0.5, 1, 5, 10, 30, 60, 300, 600, 1800, 3600, 7200, 14400 + ], +) # fmt: skip + + +def setup_metrics(with_otlp_export: bool = True) -> None: + """Push metrics over OTLP, to the same place and with the same resource + (service name, instrument) as the traces. Call after setup_tracing. + + Pushing rather than serving a /metrics endpoint means the API process and + the worker subprocess each report for themselves, with nothing shared. + """ + if with_otlp_export: + # The API only allows the global provider to be set once, later calls + # are ignored with a warning + set_meter_provider( + MeterProvider( + metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())], + resource=getattr(get_tracer_provider(), "resource", None), + ) + ) diff --git a/src/blueapi/worker/task_worker.py b/src/blueapi/worker/task_worker.py index 219a1c1ca..b364d711f 100644 --- a/src/blueapi/worker/task_worker.py +++ b/src/blueapi/worker/task_worker.py @@ -1,4 +1,5 @@ import logging +import time import uuid from collections.abc import Iterable, Mapping from concurrent.futures import Future, ThreadPoolExecutor @@ -32,6 +33,7 @@ ) from blueapi.core.bluesky_event_loop import configure_bluesky_event_loop from blueapi.log import plan_tag_filter_context +from blueapi.metrics import TASK_DURATION, setup_metrics from blueapi.utils.base_model import BlueapiBaseModel from blueapi.utils.thread_exception import handle_all_exceptions @@ -152,6 +154,7 @@ def __init__( self._broadcast_statuses = broadcast_statuses self._current_task_otel_context = None setup_tracing("BlueAPIWorker", OTLP_EXPORT_ENABLED) + setup_metrics(OTLP_EXPORT_ENABLED) @start_as_current_span(TRACER, "task_id") def clear_task(self, task_id: str) -> str: @@ -432,16 +435,24 @@ def process_task(): self._current = next_task self._current.is_pending = False meta = {"task_id": self._current.task_id} + start = time.monotonic() + outcome = "failure" try: result = self._current.task.do_task(self._ctx) LOGGER.info( "Task ran successfully - returned: %s", result, extra=meta ) self._current.set_result(result) + outcome = "success" except Exception as e: LOGGER.error("Task failed", extra=meta) self._current.set_exception(e) self._report_error(e) + finally: + TASK_DURATION.record( + time.monotonic() - start, + {"task.name": next_task.task.name, "outcome": outcome}, + ) with plan_tag_filter_context(next_task.task.name, LOGGER): if self._current_task_otel_context is not None: diff --git a/tests/system_tests/compose.yaml b/tests/system_tests/compose.yaml index 25570146b..9c7924034 100644 --- a/tests/system_tests/compose.yaml +++ b/tests/system_tests/compose.yaml @@ -154,3 +154,62 @@ services: condition: service_healthy security_opt: - label=disable + + # Local dev only: `just compose --profile monitoring up -d` then `just serve-monitored` + otel-collector: + profiles: [monitoring] + image: otel/opentelemetry-collector:0.139.0 + command: ["--config=/etc/otel-collector.yaml"] + ports: + - "4318:4318" # OTLP http, metrics and traces from blueapi + volumes: + - ./services/monitoring/otel-collector.yaml:/etc/otel-collector.yaml:ro + security_opt: + - label=disable + + jaeger: + profiles: [monitoring] + image: jaegertracing/jaeger:2.11.0 + ports: + - "16686:16686" # UI + security_opt: + - label=disable + + # Queried through Grafana, it has no UI of its own + tempo: + profiles: [monitoring] + image: grafana/tempo:2.9.0 + command: ["-config.file=/etc/tempo.yaml"] + volumes: + - ./services/monitoring/tempo.yaml:/etc/tempo.yaml:ro + security_opt: + - label=disable + + prometheus: + profiles: [monitoring] + image: prom/prometheus:v3.7.3 + command: + - --config.file=/etc/prometheus/prometheus.yaml + - --web.enable-otlp-receiver # metrics from the collector + ports: + - "9090:9090" + volumes: + - ./services/monitoring/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + security_opt: + - label=disable + + grafana: + profiles: [monitoring] + image: grafana/grafana:12.3.0 + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/blueapi.json + ports: + - "3000:3000" + volumes: + - ./services/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./services/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + security_opt: + - label=disable diff --git a/tests/system_tests/services/monitoring/grafana/dashboards/blueapi.json b/tests/system_tests/services/monitoring/grafana/dashboards/blueapi.json new file mode 100644 index 000000000..68cf15932 --- /dev/null +++ b/tests/system_tests/services/monitoring/grafana/dashboards/blueapi.json @@ -0,0 +1,361 @@ +{ + "uid": "blueapi", + "title": "BlueAPI", + "schemaVersion": 39, + "refresh": "5s", + "time": { + "from": "now-15m", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "instrument", + "label": "Instrument", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(target_info, service_instrument)", + "refId": "A" + }, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "refresh": 2 + } + ] + }, + "links": [ + { + "title": "Traces (Jaeger)", + "url": "http://localhost:16686", + "type": "link", + "targetBlank": true + } + ], + "panels": [ + { + "id": 1, + "title": "Tasks run", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 6, + "h": 5 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(blueapi_task_duration_seconds_count{service_instrument=~\"$instrument\"})", + "legendFormat": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "instant": true + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "description": "Since the worker process started" + }, + { + "id": 2, + "title": "Success rate", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 6, + "y": 0, + "w": 6, + "h": 5 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(blueapi_task_duration_seconds_count{outcome=\"success\",service_instrument=~\"$instrument\"}) / sum(blueapi_task_duration_seconds_count{service_instrument=~\"$instrument\"})", + "legendFormat": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": {}, + "description": "Since the worker process started" + }, + { + "id": 3, + "title": "Runs by plan and outcome", + "type": "bargauge", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 5 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (task_name, outcome) (blueapi_task_duration_seconds_count{service_instrument=~\"$instrument\"})", + "legendFormat": "{{task_name}} {{outcome}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "instant": true + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "basic" + }, + "description": "Since the worker process started" + }, + { + "id": 4, + "title": "Task rate by plan and outcome", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 5, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (task_name, outcome) (rate(blueapi_task_duration_seconds_count{service_instrument=~\"$instrument\"}[$__rate_interval]))", + "legendFormat": "{{task_name}} {{outcome}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 5, + "title": "Task duration by plan", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 5, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5, sum by (le, task_name) (rate(blueapi_task_duration_seconds_bucket{service_instrument=~\"$instrument\"}[$__rate_interval])))", + "legendFormat": "{{task_name}} p50", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum by (le, task_name) (rate(blueapi_task_duration_seconds_bucket{service_instrument=~\"$instrument\"}[$__rate_interval])))", + "legendFormat": "{{task_name}} p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": {}, + "description": "Estimated from histogram buckets, so only as precise as the bucket boundaries" + }, + { + "id": 6, + "title": "Mean task duration by plan", + "type": "bargauge", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 13, + "w": 12, + "h": 6 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (task_name) (blueapi_task_duration_seconds_sum{service_instrument=~\"$instrument\"}) / sum by (task_name) (blueapi_task_duration_seconds_count{service_instrument=~\"$instrument\"})", + "legendFormat": "{{task_name}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "basic" + }, + "description": "Since the worker process started" + }, + { + "id": 7, + "title": "Requests in flight", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 13, + "w": 12, + "h": 6 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(http_server_active_requests{service_instrument=~\"$instrument\"})", + "legendFormat": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "instant": true + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {} + }, + { + "id": 8, + "title": "HTTP request rate by route", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 19, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (http_method, http_target, http_status_code) (rate(http_server_duration_milliseconds_count{service_instrument=~\"$instrument\"}[$__rate_interval]))", + "legendFormat": "{{http_method}} {{http_target}} {{http_status_code}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 9, + "title": "HTTP p95 latency by route", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 19, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le, http_method, http_target) (rate(http_server_duration_milliseconds_bucket{service_instrument=~\"$instrument\"}[$__rate_interval])))", + "legendFormat": "{{http_method}} {{http_target}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "options": {} + } + ] +} diff --git a/tests/system_tests/services/monitoring/grafana/provisioning/dashboards/dashboards.yaml b/tests/system_tests/services/monitoring/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 000000000..72d910e73 --- /dev/null +++ b/tests/system_tests/services/monitoring/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,6 @@ +apiVersion: 1 +providers: + - name: blueapi + type: file + options: + path: /var/lib/grafana/dashboards diff --git a/tests/system_tests/services/monitoring/grafana/provisioning/datasources/datasources.yaml b/tests/system_tests/services/monitoring/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 000000000..3732e2555 --- /dev/null +++ b/tests/system_tests/services/monitoring/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,15 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + url: http://prometheus:9090 + isDefault: true + - name: Jaeger + uid: jaeger + type: jaeger + url: http://jaeger:16686 + - name: Tempo + uid: tempo + type: tempo + url: http://tempo:3200 diff --git a/tests/system_tests/services/monitoring/otel-collector.yaml b/tests/system_tests/services/monitoring/otel-collector.yaml new file mode 100644 index 000000000..764df81bb --- /dev/null +++ b/tests/system_tests/services/monitoring/otel-collector.yaml @@ -0,0 +1,26 @@ +# Local dev only. One OTLP endpoint for blueapi, fanned out to the backends. +# Traces go to both Jaeger and Tempo while the migration to Tempo is ongoing. +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 +exporters: + otlp/jaeger: + endpoint: jaeger:4317 + tls: + insecure: true + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + otlphttp/prometheus: + endpoint: http://prometheus:9090/api/v1/otlp +service: + pipelines: + traces: + receivers: [otlp] + exporters: [otlp/jaeger, otlp/tempo] + metrics: + receivers: [otlp] + exporters: [otlphttp/prometheus] diff --git a/tests/system_tests/services/monitoring/otel.env b/tests/system_tests/services/monitoring/otel.env new file mode 100644 index 000000000..6a046a622 --- /dev/null +++ b/tests/system_tests/services/monitoring/otel.env @@ -0,0 +1,7 @@ +# Point blueapi's telemetry at the local monitoring stack (compose profile "monitoring") +export OTLP_EXPORT_ENABLED=true +# The collector, which forwards metrics to Prometheus and traces to Jaeger and Tempo +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +# Default is 60s, too slow to watch a demo +export OTEL_METRIC_EXPORT_INTERVAL=5000 +export INSTRUMENT=adsim diff --git a/tests/system_tests/services/monitoring/prometheus.yaml b/tests/system_tests/services/monitoring/prometheus.yaml new file mode 100644 index 000000000..3d4c3e116 --- /dev/null +++ b/tests/system_tests/services/monitoring/prometheus.yaml @@ -0,0 +1,7 @@ +# Local dev only. Nothing is scraped: the collector pushes blueapi's metrics over OTLP +# (prometheus runs with --web.enable-otlp-receiver) +otlp: + # Turn these resource attributes into labels on every metric + promote_resource_attributes: + - service.name + - service.instrument diff --git a/tests/system_tests/services/monitoring/tempo.yaml b/tests/system_tests/services/monitoring/tempo.yaml new file mode 100644 index 000000000..1615da3c2 --- /dev/null +++ b/tests/system_tests/services/monitoring/tempo.yaml @@ -0,0 +1,16 @@ +# Local dev only: single binary, traces kept on the container's own disk +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 +storage: + trace: + backend: local + local: + path: /var/tempo/blocks + wal: + path: /var/tempo/wal diff --git a/tests/unit_tests/worker/test_task_worker.py b/tests/unit_tests/worker/test_task_worker.py index 044668d93..317742808 100644 --- a/tests/unit_tests/worker/test_task_worker.py +++ b/tests/unit_tests/worker/test_task_worker.py @@ -1197,3 +1197,16 @@ def plan_with_default_str(arg: str = "primary") -> MsgGenerator: # by mistake as the default plan value. The kwargs is provided by pydantic model # from schema. assert kwargs == {"arg": "primary"} + + +@pytest.mark.parametrize( + "task, outcome", [(_SIMPLE_TASK, "success"), (_FAILING_TASK, "failure")] +) +def test_task_duration_metric_recorded(worker: TaskWorker, task: Task, outcome: str): + with patch("blueapi.worker.task_worker.TASK_DURATION") as duration: + begin_task_and_wait_until_complete(worker, worker.submit_task(task)) + + duration.record.assert_called_once_with( + ANY, {"task.name": task.name, "outcome": outcome} + ) + assert duration.record.call_args.args[0] >= 0