From 6ec8a669b480b581c98bc36f624239c9f931dda6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Bj=C3=B8rnstad?= Date: Mon, 19 Oct 2026 14:52:18 +0200 Subject: [PATCH 1/2] Map unrecoverable Postgres branches as degraded Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../someteamname/postgres_foobar_failed.yaml | 17 +++++++++++ integration_tests/postgres_branches.lua | 29 +++++++++++++++---- internal/persistence/postgres/models.go | 7 ++++- internal/persistence/postgres/models_test.go | 3 ++ 4 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 integration_tests/k8s_resources/postgres_branches/dev/someteamname/postgres_foobar_failed.yaml diff --git a/integration_tests/k8s_resources/postgres_branches/dev/someteamname/postgres_foobar_failed.yaml b/integration_tests/k8s_resources/postgres_branches/dev/someteamname/postgres_foobar_failed.yaml new file mode 100644 index 000000000..017d58cac --- /dev/null +++ b/integration_tests/k8s_resources/postgres_branches/dev/someteamname/postgres_foobar_failed.yaml @@ -0,0 +1,17 @@ +apiVersion: nais.io/v1 +kind: PostgresBranch +metadata: + name: foobar-failed-68299523 + namespace: someteamname +spec: + postgres: foobar + branchName: failed +status: + reconcilePhase: Failed + conditions: + - type: RecoveryFailed + status: "True" + reason: Unrecoverable + - type: Ready + status: "False" + reason: RecoveryFailed diff --git a/integration_tests/postgres_branches.lua b/integration_tests/postgres_branches.lua index 22cb429dd..fed6470a1 100644 --- a/integration_tests/postgres_branches.lua +++ b/integration_tests/postgres_branches.lua @@ -41,7 +41,10 @@ Test.gql("List Postgres and their branches together", function(t) } } } }]] t.check { data = { team = { postgreses = { nodes = { { name = "another-db", majorVersion = "16", activeBranch = { name = "main" }, branches = { nodes = { { name = "main", state = "AVAILABLE", postgres = { name = "another-db" } } } } }, - { name = "foobar", majorVersion = "17", activeBranch = { name = "main" }, branches = { nodes = { { name = "main", state = "AVAILABLE", postgres = { name = "foobar" } } } } }, + { name = "foobar", majorVersion = "17", activeBranch = { name = "main" }, branches = { nodes = { + { name = "failed", state = "DEGRADED", postgres = { name = "foobar" } }, + { name = "main", state = "AVAILABLE", postgres = { name = "foobar" } }, + } } }, { name = "with-audit", majorVersion = "16", activeBranch = { name = "main" }, branches = { nodes = { { name = "main", state = "AVAILABLE", postgres = { name = "with-audit" } } } } }, { name = "without-audit", majorVersion = "15", activeBranch = { name = "main" }, branches = { nodes = { { name = "main", state = "AVAILABLE", postgres = { name = "without-audit" } } } } }, } } } } } @@ -50,10 +53,23 @@ end) Test.gql("Retrieve logical Postgres and selected physical instance separately", function(t) t.addHeader("x-user-email", user:email()) t.query [[{ team(slug:"someteamname") { environment(name:"dev") { - postgres(name:"foobar") { name activeBranch { name } majorVersion highAvailability resources { cpu memory diskSize } branch(name:"main") { name postgres { name } teamEnvironment { name } } branches { nodes { name } } } + postgres(name:"foobar") { name activeBranch { name } majorVersion highAvailability resources { cpu memory diskSize } branch(name:"main") { name postgres { name } teamEnvironment { name } } branches { nodes { name state } } } } } }]] t.check { data = { team = { environment = { - postgres = { name = "foobar", activeBranch = { name = "main" }, majorVersion = "17", highAvailability = false, resources = { cpu = "100m", memory = "2Gi", diskSize = "2Gi" }, branch = { name = "main", postgres = { name = "foobar" }, teamEnvironment = { name = "dev" } }, branches = { nodes = { { name = "main" } } } }, + postgres = { name = "foobar", activeBranch = { name = "main" }, majorVersion = "17", highAvailability = false, resources = { cpu = "100m", memory = "2Gi", diskSize = "2Gi" }, branch = { name = "main", postgres = { name = "foobar" }, teamEnvironment = { name = "dev" } }, branches = { nodes = { + { name = "failed", state = "DEGRADED" }, + { name = "main", state = "AVAILABLE" }, + } } }, + } } } } +end) + +Test.gql("An unrecoverable Postgres branch is degraded", function(t) + t.addHeader("x-user-email", user:email()) + t.query [[{ team(slug:"someteamname") { environment(name:"dev") { + postgres(name:"foobar") { branch(name:"failed") { name state } } + } } }]] + t.check { data = { team = { environment = { + postgres = { branch = { name = "failed", state = "DEGRADED" } }, } } } } end) @@ -63,8 +79,11 @@ Test.gql("Branch state facets are scoped to their Postgres", function(t) branches { nodes { name state } facets { states { state count } } } } } } }]] t.check { data = { team = { environment = { postgres = { branches = { - nodes = { { name = "main", state = "AVAILABLE" } }, - facets = { states = { { state = "AVAILABLE", count = 1 } } }, + nodes = { + { name = "failed", state = "DEGRADED" }, + { name = "main", state = "AVAILABLE" }, + }, + facets = { states = { { state = "AVAILABLE", count = 1 }, { state = "DEGRADED", count = 1 } } }, } } } } } } end) diff --git a/internal/persistence/postgres/models.go b/internal/persistence/postgres/models.go index dc4e3c61d..ecac98dd6 100644 --- a/internal/persistence/postgres/models.go +++ b/internal/persistence/postgres/models.go @@ -486,9 +486,14 @@ func toPostgres(u *unstructured.Unstructured, environmentName string) (*Postgres }, nil } -// postgresStateFromConditions interprets the CNPG phase mirrored by pgrator. +// postgresStateFromConditions interprets pgrator and CNPG status conditions. // ObservedState=False means no phase has been observed, not a failed cluster. func postgresStateFromConditions(conditions []metav1.Condition, reconciled bool) PostgresBranchState { + for _, condition := range conditions { + if condition.Type == "RecoveryFailed" && condition.Status == metav1.ConditionTrue && condition.Reason == "Unrecoverable" { + return PostgresBranchStateDegraded + } + } if !reconciled { return PostgresBranchStateProgressing } diff --git a/internal/persistence/postgres/models_test.go b/internal/persistence/postgres/models_test.go index 6f92222c8..0a785f777 100644 --- a/internal/persistence/postgres/models_test.go +++ b/internal/persistence/postgres/models_test.go @@ -133,6 +133,9 @@ func TestPostgresStateFromConditions(t *testing.T) { want PostgresBranchState }{ {name: "not reconciled", want: PostgresBranchStateProgressing}, + {name: "unrecoverable recovery failure", conditions: []metav1.Condition{{Type: "RecoveryFailed", Status: metav1.ConditionTrue, Reason: "Unrecoverable"}}, want: PostgresBranchStateDegraded}, + {name: "recovery failure not terminal", conditions: []metav1.Condition{{Type: "RecoveryFailed", Status: metav1.ConditionFalse, Reason: "Recovering"}}, want: PostgresBranchStateProgressing}, + {name: "ready false without recovery failure", reconciled: true, conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, Reason: "RecoveryFailed"}}, want: PostgresBranchStateProgressing}, {name: "healthy", reconciled: true, conditions: []metav1.Condition{{Type: "cluster.postgresql.cnpg.io/ObservedState", Status: metav1.ConditionTrue, Message: "Cluster is in phase: Cluster in healthy state"}}, want: PostgresBranchStateAvailable}, {name: "still starting", reconciled: true, conditions: []metav1.Condition{{Type: "cluster.postgresql.cnpg.io/ObservedState", Status: metav1.ConditionFalse, Message: "Cluster is in phase: "}}, want: PostgresBranchStateProgressing}, {name: "unrecoverable", reconciled: true, conditions: []metav1.Condition{{Type: "cluster.postgresql.cnpg.io/ObservedState", Status: metav1.ConditionTrue, Message: "Cluster is in phase: Cluster is unrecoverable and needs manual intervention"}}, want: PostgresBranchStateDegraded}, From d484612b775826bb69223853f28c786d457bf18f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Bj=C3=B8rnstad?= Date: Mon, 19 Oct 2026 15:00:39 +0200 Subject: [PATCH 2/2] fix(tests): standardize spacing in Postgres branch state checks --- integration_tests/postgres_branches.lua | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/integration_tests/postgres_branches.lua b/integration_tests/postgres_branches.lua index fed6470a1..3dcbf634c 100644 --- a/integration_tests/postgres_branches.lua +++ b/integration_tests/postgres_branches.lua @@ -41,9 +41,9 @@ Test.gql("List Postgres and their branches together", function(t) } } } }]] t.check { data = { team = { postgreses = { nodes = { { name = "another-db", majorVersion = "16", activeBranch = { name = "main" }, branches = { nodes = { { name = "main", state = "AVAILABLE", postgres = { name = "another-db" } } } } }, - { name = "foobar", majorVersion = "17", activeBranch = { name = "main" }, branches = { nodes = { - { name = "failed", state = "DEGRADED", postgres = { name = "foobar" } }, - { name = "main", state = "AVAILABLE", postgres = { name = "foobar" } }, + { name = "foobar", majorVersion = "17", activeBranch = { name = "main" }, branches = { nodes = { + { name = "failed", state = "DEGRADED", postgres = { name = "foobar" } }, + { name = "main", state = "AVAILABLE", postgres = { name = "foobar" } }, } } }, { name = "with-audit", majorVersion = "16", activeBranch = { name = "main" }, branches = { nodes = { { name = "main", state = "AVAILABLE", postgres = { name = "with-audit" } } } } }, { name = "without-audit", majorVersion = "15", activeBranch = { name = "main" }, branches = { nodes = { { name = "main", state = "AVAILABLE", postgres = { name = "without-audit" } } } } }, @@ -58,7 +58,7 @@ Test.gql("Retrieve logical Postgres and selected physical instance separately", t.check { data = { team = { environment = { postgres = { name = "foobar", activeBranch = { name = "main" }, majorVersion = "17", highAvailability = false, resources = { cpu = "100m", memory = "2Gi", diskSize = "2Gi" }, branch = { name = "main", postgres = { name = "foobar" }, teamEnvironment = { name = "dev" } }, branches = { nodes = { { name = "failed", state = "DEGRADED" }, - { name = "main", state = "AVAILABLE" }, + { name = "main", state = "AVAILABLE" }, } } }, } } } } end) @@ -81,7 +81,7 @@ Test.gql("Branch state facets are scoped to their Postgres", function(t) t.check { data = { team = { environment = { postgres = { branches = { nodes = { { name = "failed", state = "DEGRADED" }, - { name = "main", state = "AVAILABLE" }, + { name = "main", state = "AVAILABLE" }, }, facets = { states = { { state = "AVAILABLE", count = 1 }, { state = "DEGRADED", count = 1 } } }, } } } } } }