Map unrecoverable Postgres branches as degraded

integration_tests/k8s_resources/postgres_branches/dev/someteamname/postgres_foobar_failed.yaml

@@ -0,0 +1,17 @@
+apiVersion: nais.io/v1
+kind: PostgresBranch
+metadata:
+  name: foobar-failed-68299523
+  namespace: someteamname
+spec:
+  postgres: foobar
+  branchName: failed
+status:
+  reconcilePhase: Failed
+  conditions:
+    - type: RecoveryFailed
+      status: "True"
+      reason: Unrecoverable
+    - type: Ready
+      status: "False"
+      reason: RecoveryFailed

integration_tests/postgres_branches.lua

@@ -41,7 +41,10 @@ Test.gql("List Postgres and their branches together", function(t)
     } } } }]]
 	t.check { data = { team = { postgreses = { nodes = {
 		{ name = "another-db",    majorVersion = "16", activeBranch = { name = "main" }, branches = { nodes = { { name = "main", state = "AVAILABLE", postgres = { name = "another-db" } } } } },
-		{ name = "foobar",        majorVersion = "17", activeBranch = { name = "main" }, branches = { nodes = { { name = "main", state = "AVAILABLE", postgres = { name = "foobar" } } } } },
+		{ name = "foobar", majorVersion = "17", activeBranch = { name = "main" }, branches = { nodes = {
+			{ name = "failed", state = "DEGRADED",  postgres = { name = "foobar" } },
+			{ name = "main",   state = "AVAILABLE", postgres = { name = "foobar" } },
+		} } },
 		{ name = "with-audit",    majorVersion = "16", activeBranch = { name = "main" }, branches = { nodes = { { name = "main", state = "AVAILABLE", postgres = { name = "with-audit" } } } } },
 		{ name = "without-audit", majorVersion = "15", activeBranch = { name = "main" }, branches = { nodes = { { name = "main", state = "AVAILABLE", postgres = { name = "without-audit" } } } } },
 	} } } } }
@@ -50,10 +53,23 @@ end)
 Test.gql("Retrieve logical Postgres and selected physical instance separately", function(t)
 	t.addHeader("x-user-email", user:email())
 	t.query [[{ team(slug:"someteamname") { environment(name:"dev") {
-        postgres(name:"foobar") { name activeBranch { name } majorVersion highAvailability resources { cpu memory diskSize } branch(name:"main") { name postgres { name } teamEnvironment { name } } branches { nodes { name } } }
+        postgres(name:"foobar") { name activeBranch { name } majorVersion highAvailability resources { cpu memory diskSize } branch(name:"main") { name postgres { name } teamEnvironment { name } } branches { nodes { name state } } }
     } } }]]
 	t.check { data = { team = { environment = {
-		postgres = { name = "foobar", activeBranch = { name = "main" }, majorVersion = "17", highAvailability = false, resources = { cpu = "100m", memory = "2Gi", diskSize = "2Gi" }, branch = { name = "main", postgres = { name = "foobar" }, teamEnvironment = { name = "dev" } }, branches = { nodes = { { name = "main" } } } },
+		postgres = { name = "foobar", activeBranch = { name = "main" }, majorVersion = "17", highAvailability = false, resources = { cpu = "100m", memory = "2Gi", diskSize = "2Gi" }, branch = { name = "main", postgres = { name = "foobar" }, teamEnvironment = { name = "dev" } }, branches = { nodes = {
+			{ name = "failed", state = "DEGRADED" },
+			{ name = "main",   state = "AVAILABLE" },
+		} } },
+	} } } }
+end)
+
+Test.gql("An unrecoverable Postgres branch is degraded", function(t)
+	t.addHeader("x-user-email", user:email())
+	t.query [[{ team(slug:"someteamname") { environment(name:"dev") {
+		postgres(name:"foobar") { branch(name:"failed") { name state } }
+	} } }]]
+	t.check { data = { team = { environment = {
+		postgres = { branch = { name = "failed", state = "DEGRADED" } },
 	} } } }
 end)
 
@@ -63,8 +79,11 @@ Test.gql("Branch state facets are scoped to their Postgres", function(t)
         branches { nodes { name state } facets { states { state count } } }
     } } } }]]
 	t.check { data = { team = { environment = { postgres = { branches = {
-		nodes = { { name = "main", state = "AVAILABLE" } },
-		facets = { states = { { state = "AVAILABLE", count = 1 } } },
+		nodes = {
+			{ name = "failed", state = "DEGRADED" },
+			{ name = "main",   state = "AVAILABLE" },
+		},
+		facets = { states = { { state = "AVAILABLE", count = 1 }, { state = "DEGRADED", count = 1 } } },
 	} } } } } }
 end)
 

internal/persistence/postgres/models.go

@@ -486,9 +486,14 @@ func toPostgres(u *unstructured.Unstructured, environmentName string) (*Postgres
 	}, nil
 }
 
-// postgresStateFromConditions interprets the CNPG phase mirrored by pgrator.
+// postgresStateFromConditions interprets pgrator and CNPG status conditions.
 // ObservedState=False means no phase has been observed, not a failed cluster.
 func postgresStateFromConditions(conditions []metav1.Condition, reconciled bool) PostgresBranchState {
+	for _, condition := range conditions {
+		if condition.Type == "RecoveryFailed" && condition.Status == metav1.ConditionTrue && condition.Reason == "Unrecoverable" {
+			return PostgresBranchStateDegraded
+		}
+	}
 	if !reconciled {
 		return PostgresBranchStateProgressing
 	}

internal/persistence/postgres/models_test.go

@@ -133,6 +133,9 @@ func TestPostgresStateFromConditions(t *testing.T) {
 		want       PostgresBranchState
 	}{
 		{name: "not reconciled", want: PostgresBranchStateProgressing},
+		{name: "unrecoverable recovery failure", conditions: []metav1.Condition{{Type: "RecoveryFailed", Status: metav1.ConditionTrue, Reason: "Unrecoverable"}}, want: PostgresBranchStateDegraded},
+		{name: "recovery failure not terminal", conditions: []metav1.Condition{{Type: "RecoveryFailed", Status: metav1.ConditionFalse, Reason: "Recovering"}}, want: PostgresBranchStateProgressing},
+		{name: "ready false without recovery failure", reconciled: true, conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, Reason: "RecoveryFailed"}}, want: PostgresBranchStateProgressing},
 		{name: "healthy", reconciled: true, conditions: []metav1.Condition{{Type: "cluster.postgresql.cnpg.io/ObservedState", Status: metav1.ConditionTrue, Message: "Cluster is in phase: Cluster in healthy state"}}, want: PostgresBranchStateAvailable},
 		{name: "still starting", reconciled: true, conditions: []metav1.Condition{{Type: "cluster.postgresql.cnpg.io/ObservedState", Status: metav1.ConditionFalse, Message: "Cluster is in phase: "}}, want: PostgresBranchStateProgressing},
 		{name: "unrecoverable", reconciled: true, conditions: []metav1.Condition{{Type: "cluster.postgresql.cnpg.io/ObservedState", Status: metav1.ConditionTrue, Message: "Cluster is in phase: Cluster is unrecoverable and needs manual intervention"}}, want: PostgresBranchStateDegraded},