HYPERFLEET-1410 - feat: expose operator metrics, health probes, and ServiceMonitor

Dockerfile

@@ -2,6 +2,15 @@ ARG BASE_IMAGE=registry.access.redhat.com/ubi9-micro:latest
 
 FROM registry.access.redhat.com/ubi9/go-toolset:9.8-1788409979 AS builder
 
+# APP_VERSION/GIT_SHA are injected into the binary below via -ldflags -X, so
+# hyperfleet_operator_build_info reports the real release/commit instead of
+# falling back to "dev"/"unknown" (see internal/version and docs/metrics.md).
+# The container build has no .git directory to source them from automatically,
+# unlike `make build`/`make run`, which get them from the toolchain's own VCS
+# stamping.
+ARG APP_VERSION="0.0.0-dev"
+ARG GIT_SHA="unknown"
+
 USER root
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -18,7 +27,10 @@ COPY internal/ internal/
 
 
 RUN CGO_ENABLED=1 GOEXPERIMENT=boringcrypto \
-    go build -trimpath -ldflags="-s -w" -o manager ./cmd/main.go
+    go build -trimpath -ldflags="-s -w \
+      -X github.com/openshift-hyperfleet/hyperfleet-operator/internal/version.version=${APP_VERSION} \
+      -X github.com/openshift-hyperfleet/hyperfleet-operator/internal/version.commit=${GIT_SHA}" \
+    -o manager ./cmd/main.go
 
 # Runtime stage
 FROM ${BASE_IMAGE} AS final

Makefile

@@ -169,12 +169,12 @@ GIT_DIRTY ?= $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] || echo "-m
 
 # Go build flags (FIPS compliant)
 CGO_ENABLED ?= 1
-GOEXPERIMENT ?= boringcrypto 
+GOEXPERIMENT ?= boringcrypto
 GOFLAGS ?= -trimpath
-# LDFLAGS := -s -w \
-#            -X github.com/openshift-hyperfleet/hyperfleet-operator/pkg/version.Version=$(APP_VERSION) \
-#            -X github.com/openshift-hyperfleet/hyperfleet-operator/pkg/version.Commit=$(GIT_SHA) \
-#            -X 'github.com/openshift-hyperfleet/hyperfleet-operator/pkg/version.BuildTime=$(BUILD_DATE)'
+# APP_VERSION/GIT_SHA are injected into the binary via the Dockerfile's own
+# -ldflags -X (see Dockerfile and internal/version); `make build`/`make run`
+# intentionally skip ldflags and rely on the Go toolchain's automatic VCS
+# stamping from the local .git checkout instead (see internal/version).
 
 .PHONY: check-container-tool
 check-container-tool:
@@ -190,6 +190,7 @@ image: check-container-tool manifests generate fmt vet ## Build container image
 		--platform $(PLATFORM) \
 		--build-arg BASE_IMAGE=$(BASE_IMAGE) \
 		--build-arg APP_VERSION=$(APP_VERSION) \
+		--build-arg GIT_SHA=$(GIT_SHA) \
 		-t $(IMG) .
 	@echo "Image built: $(IMG)"
 	@echo "$(IMG)"

README.md

@@ -49,6 +49,19 @@ kubectl apply -k config/samples/
 
 >**NOTE**: Ensure that the samples has default values to test it out.
 
+### Observability endpoints
+
+The manager exposes the standard HyperFleet observability endpoints (defaults):
+
+- **Liveness probe:**  `http://localhost:8080/healthz`
+- **Readiness probe:** `http://localhost:8080/readyz`
+- **Metrics:**         `http://localhost:9090/metrics`
+
+Metrics are served as plain HTTP under the `hyperfleet_operator_*` namespace. Ports
+are configurable via `--health-probe-bind-address` and `--metrics-bind-address`.
+See [docs/metrics.md](docs/metrics.md) for the full metric catalogue, labels, and
+example PromQL queries.
+
 ### To Uninstall
 **Delete the instances (CRs) from the cluster:**
 

bundle/manifests/hyperfleet-operator-controller-manager-metrics-service_v1_service.yaml

@@ -9,10 +9,10 @@ metadata:
   name: hyperfleet-operator-controller-manager-metrics-service
 spec:
   ports:
-  - name: https
-    port: 8443
+  - name: metrics
+    port: 9090
     protocol: TCP
-    targetPort: 8443
+    targetPort: metrics
   selector:
     app.kubernetes.io/name: hyperfleet-operator
     control-plane: controller-manager

bundle/manifests/hyperfleet-operator.clusterserviceversion.yaml

@@ -166,9 +166,10 @@ spec:
             spec:
               containers:
               - args:
-                - --metrics-bind-address=:8443
                 - --leader-elect
-                - --health-probe-bind-address=:8081
+                - --health-probe-bind-address=:8080
+                - --metrics-bind-address=:9090
+                - --metrics-secure=false
                 command:
                 - /manager
                 env:
@@ -180,18 +181,29 @@ spec:
                   value: quay.io/redhat-services-prod/hyperfleet-tenant/hyperfleet/hyperfleet-api:latest
                 image: quay.io/redhat-services-prod/hyperfleet-tenant/hyperfleet/hyperfleet-operator:latest
                 livenessProbe:
+                  failureThreshold: 3
                   httpGet:
                     path: /healthz
-                    port: 8081
+                    port: 8080
                   initialDelaySeconds: 15
                   periodSeconds: 20
+                  timeoutSeconds: 5
                 name: manager
+                ports:
+                - containerPort: 9090
+                  name: metrics
+                  protocol: TCP
+                - containerPort: 8080
+                  name: health
+                  protocol: TCP
                 readinessProbe:
+                  failureThreshold: 3
                   httpGet:
                     path: /readyz
-                    port: 8081
+                    port: 8080
                   initialDelaySeconds: 5
                   periodSeconds: 10
+                  timeoutSeconds: 3
                 resources:
                   limits:
                     cpu: 500m
@@ -209,7 +221,7 @@ spec:
                 seccompProfile:
                   type: RuntimeDefault
               serviceAccountName: hyperfleet-operator-controller-manager
-              terminationGracePeriodSeconds: 10
+              terminationGracePeriodSeconds: 30
       permissions:
       - rules:
         - apiGroups:
@@ -251,6 +263,17 @@ spec:
           - get
           - list
           - watch
+        - apiGroups:
+          - monitoring.coreos.com
+          resources:
+          - servicemonitors
+          verbs:
+          - create
+          - get
+          - list
+          - patch
+          - update
+          - watch
         - apiGroups:
           - networking.k8s.io
           resources:

cmd/main.go

@@ -43,6 +43,8 @@ import (
 
 	hyperfleetv1alpha1 "github.com/openshift-hyperfleet/hyperfleet-operator/api/v1alpha1"
 	"github.com/openshift-hyperfleet/hyperfleet-operator/internal/controller"
+	"github.com/openshift-hyperfleet/hyperfleet-operator/internal/metrics"
+	"github.com/openshift-hyperfleet/hyperfleet-operator/internal/servicemonitor"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -68,14 +70,18 @@ func main() {
 	var secureMetrics bool
 	var enableHTTP2 bool
 	var tlsOpts []func(*tls.Config)
-	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
-		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
-	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
+	// Defaults follow the HyperFleet health-endpoints / metrics standards, matching
+	// the API, Sentinel and Adapter components: metrics on :9090 over plain HTTP at
+	// /metrics, health/readiness on :8080. Set 0 on metrics-bind-address to disable.
+	flag.StringVar(&metricsAddr, "metrics-bind-address", ":9090", "The address the metrics endpoint binds to. "+
+		"Defaults to :9090 (HyperFleet standard). Set to 0 to disable the metrics service.")
+	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8080", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-	flag.BoolVar(&secureMetrics, "metrics-secure", true,
-		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+	flag.BoolVar(&secureMetrics, "metrics-secure", false,
+		"If set, the metrics endpoint is served securely via HTTPS with authn/authz. "+
+			"The HyperFleet standard scrapes plain HTTP, so this defaults to false.")
 	flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.")
 	flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.")
 	flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.")
@@ -93,6 +99,10 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	// Publish the build-info and up metrics into controller-runtime's registry so
+	// they are exposed on the same /metrics endpoint as the reconcile metrics.
+	metrics.Init()
+
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
 	// due to its vulnerabilities. More specifically, disabling http/2 will
 	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
@@ -256,6 +266,29 @@ func main() {
 	}
 	// +kubebuilder:scaffold:builder
 
+	// Create the operator's own ServiceMonitor at runtime, but only when the
+	// Prometheus Operator API is present. Shipping it in the OLM bundle would fail
+	// the InstallPlan on clusters without the monitoring.coreos.com CRD and block
+	// the operator install; this bootstrapper degrades gracefully instead. See the
+	// servicemonitor package doc for the full rationale.
+	//
+	// Skipped entirely when metrics are disabled (metricsAddr == "0", the same
+	// sentinel controller-runtime's metrics server itself checks — see
+	// metricsserver.NewServer): the ServiceMonitor always points at the metrics
+	// Service's "metrics" port, so with no metrics server listening it would only
+	// give Prometheus a target that fails every scrape.
+	if metricsAddr != "0" {
+		if err := mgr.Add(&servicemonitor.Bootstrapper{
+			Config:    mgr.GetConfig(),
+			Namespace: operatorNamespace,
+		}); err != nil {
+			setupLog.Error(err, "unable to add ServiceMonitor bootstrapper")
+			os.Exit(1)
+		}
+	} else {
+		setupLog.Info("metrics disabled (metrics-bind-address=0); skipping ServiceMonitor bootstrap")
+	}
+
 	if metricsCertWatcher != nil {
 		setupLog.Info("Adding metrics certificate watcher to manager")
 		if err := mgr.Add(metricsCertWatcher); err != nil {

config/default/kustomization.yaml

@@ -38,13 +38,9 @@ resources:
 # be able to communicate with the Webhook Server.
 #- ../network-policy
 
-# Uncomment the patches line if you enable Metrics
-patches:
-# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443.
-# More info: https://book.kubebuilder.io/reference/metrics
-- path: manager_metrics_patch.yaml
-  target:
-    kind: Deployment
+# The metrics endpoint (:9090, plain HTTP, /metrics) and health probes (:8080)
+# are configured directly on the manager Deployment (config/manager/manager.yaml)
+# per the HyperFleet standard, so no metrics args patch is needed here.
 
 # Uncomment the patches line if you enable Metrics and CertManager
 # [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.

config/default/metrics_service.yaml

@@ -9,10 +9,10 @@ metadata:
   namespace: system
 spec:
   ports:
-  - name: https
-    port: 8443
+  - name: metrics
+    port: 9090
     protocol: TCP
-    targetPort: 8443
+    targetPort: metrics
   selector:
     control-plane: controller-manager
     app.kubernetes.io/name: hyperfleet-operator

config/manager/manager.yaml

@@ -62,7 +62,11 @@ spec:
         - /manager
         args:
           - --leader-elect
-          - --health-probe-bind-address=:8081
+          # HyperFleet standard: metrics on :9090 (plain HTTP, /metrics) and
+          # health/readiness on :8080, matching the other components.
+          - --health-probe-bind-address=:8080
+          - --metrics-bind-address=:9090
+          - --metrics-secure=false
         image: controller:latest
         name: manager
         env:
@@ -72,24 +76,35 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        ports: []
+        ports:
+        - name: metrics
+          containerPort: 9090
+          protocol: TCP
+        - name: health
+          containerPort: 8080
+          protocol: TCP
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
             drop:
             - "ALL"
+        # Probe timing follows the HyperFleet health-endpoints standard.
         livenessProbe:
           httpGet:
             path: /healthz
-            port: 8081
+            port: 8080
           initialDelaySeconds: 15
           periodSeconds: 20
+          timeoutSeconds: 5
+          failureThreshold: 3
         readinessProbe:
           httpGet:
             path: /readyz
-            port: 8081
+            port: 8080
           initialDelaySeconds: 5
           periodSeconds: 10
+          timeoutSeconds: 3
+          failureThreshold: 3
         # TODO(user): Configure the resources accordingly based on the project requirements.
         # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
         resources:
@@ -102,4 +117,4 @@ spec:
         volumeMounts: []
       volumes: []
       serviceAccountName: controller-manager
-      terminationGracePeriodSeconds: 10
+      terminationGracePeriodSeconds: 30

config/manifests/kustomization.yaml

@@ -5,6 +5,14 @@ resources:
 - ../default
 - ../samples
 - ../scorecard
+# The Prometheus ServiceMonitor is deliberately NOT included in the bundle. OLM
+# applies a bundle's arbitrary manifests but does not install the CRDs they need,
+# so bundling the ServiceMonitor would fail the InstallPlan — and block the whole
+# operator install — on clusters without the Prometheus Operator CRD. HyperFleet
+# targets generic Kubernetes, so the operator creates the ServiceMonitor itself at
+# runtime only when the monitoring.coreos.com/v1 API is present (see the
+# internal/servicemonitor package). config/prometheus remains an optional GitOps
+# overlay for users who prefer to apply it statically.
 
 # [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix.
 # Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager.

config/network-policy/allow-metrics-traffic.yaml

@@ -23,5 +23,5 @@ spec:
           matchLabels:
             metrics: enabled  # Only from namespaces with this label
       ports:
-        - port: 8443
+        - port: 9090
           protocol: TCP

config/prometheus/monitor.yaml

@@ -1,4 +1,6 @@
 # Prometheus Monitor Service (Metrics)
+# Scrapes the operator's plain-HTTP metrics endpoint on :9090, per the HyperFleet
+# metrics standard (same exposition as the API, Sentinel and Adapter components).
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -11,16 +13,14 @@ metadata:
 spec:
   endpoints:
     - path: /metrics
-      port: https # Ensure this is the name of the port that exposes HTTPS metrics
-      scheme: https
-      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-      tlsConfig:
-        # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
-        # certificate verification, exposing the system to potential man-in-the-middle attacks.
-        # For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
-        # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
-        # which securely references the certificate from the 'metrics-server-cert' secret.
-        insecureSkipVerify: true
+      port: metrics # matches the metrics Service port name
+      # Hardcoded to the HyperFleet metrics standard's plain-HTTP default
+      # (--metrics-secure=false). If the operator is run with --metrics-secure=true,
+      # this scrape will fail: the endpoint requires HTTPS + authn/authz and this
+      # manifest is not updated automatically. Configure scheme/tlsConfig/bearerToken
+      # here yourself in that case (see config/prometheus/monitor_tls_patch.yaml).
+      scheme: http
+      interval: 30s
   selector:
     matchLabels:
       control-plane: controller-manager

config/rbac/role.yaml

@@ -81,6 +81,17 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - servicemonitors
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - networking.k8s.io
   resources: