diff --git a/integration_tests/k8s_resources/postgres_branches/dev/someteamname/postgres_foobar_failed.yaml b/integration_tests/k8s_resources/postgres_branches/dev/someteamname/postgres_foobar_failed.yaml new file mode 100644 index 000000000..017d58cac --- /dev/null +++ b/integration_tests/k8s_resources/postgres_branches/dev/someteamname/postgres_foobar_failed.yaml @@ -0,0 +1,17 @@ +apiVersion: nais.io/v1 +kind: PostgresBranch +metadata: + name: foobar-failed-68299523 + namespace: someteamname +spec: + postgres: foobar + branchName: failed +status: + reconcilePhase: Failed + conditions: + - type: RecoveryFailed + status: "True" + reason: Unrecoverable + - type: Ready + status: "False" + reason: RecoveryFailed diff --git a/integration_tests/postgres_branches.lua b/integration_tests/postgres_branches.lua index 22cb429dd..3dcbf634c 100644 --- a/integration_tests/postgres_branches.lua +++ b/integration_tests/postgres_branches.lua @@ -41,7 +41,10 @@ Test.gql("List Postgres and their branches together", function(t) } } } }]] t.check { data = { team = { postgreses = { nodes = { { name = "another-db", majorVersion = "16", activeBranch = { name = "main" }, branches = { nodes = { { name = "main", state = "AVAILABLE", postgres = { name = "another-db" } } } } }, - { name = "foobar", majorVersion = "17", activeBranch = { name = "main" }, branches = { nodes = { { name = "main", state = "AVAILABLE", postgres = { name = "foobar" } } } } }, + { name = "foobar", majorVersion = "17", activeBranch = { name = "main" }, branches = { nodes = { + { name = "failed", state = "DEGRADED", postgres = { name = "foobar" } }, + { name = "main", state = "AVAILABLE", postgres = { name = "foobar" } }, + } } }, { name = "with-audit", majorVersion = "16", activeBranch = { name = "main" }, branches = { nodes = { { name = "main", state = "AVAILABLE", postgres = { name = "with-audit" } } } } }, { name = "without-audit", majorVersion = "15", activeBranch = { name = "main" }, branches = { nodes = { { name = "main", state = "AVAILABLE", postgres = { name = "without-audit" } } } } }, } } } } } @@ -50,10 +53,23 @@ end) Test.gql("Retrieve logical Postgres and selected physical instance separately", function(t) t.addHeader("x-user-email", user:email()) t.query [[{ team(slug:"someteamname") { environment(name:"dev") { - postgres(name:"foobar") { name activeBranch { name } majorVersion highAvailability resources { cpu memory diskSize } branch(name:"main") { name postgres { name } teamEnvironment { name } } branches { nodes { name } } } + postgres(name:"foobar") { name activeBranch { name } majorVersion highAvailability resources { cpu memory diskSize } branch(name:"main") { name postgres { name } teamEnvironment { name } } branches { nodes { name state } } } } } }]] t.check { data = { team = { environment = { - postgres = { name = "foobar", activeBranch = { name = "main" }, majorVersion = "17", highAvailability = false, resources = { cpu = "100m", memory = "2Gi", diskSize = "2Gi" }, branch = { name = "main", postgres = { name = "foobar" }, teamEnvironment = { name = "dev" } }, branches = { nodes = { { name = "main" } } } }, + postgres = { name = "foobar", activeBranch = { name = "main" }, majorVersion = "17", highAvailability = false, resources = { cpu = "100m", memory = "2Gi", diskSize = "2Gi" }, branch = { name = "main", postgres = { name = "foobar" }, teamEnvironment = { name = "dev" } }, branches = { nodes = { + { name = "failed", state = "DEGRADED" }, + { name = "main", state = "AVAILABLE" }, + } } }, + } } } } +end) + +Test.gql("An unrecoverable Postgres branch is degraded", function(t) + t.addHeader("x-user-email", user:email()) + t.query [[{ team(slug:"someteamname") { environment(name:"dev") { + postgres(name:"foobar") { branch(name:"failed") { name state } } + } } }]] + t.check { data = { team = { environment = { + postgres = { branch = { name = "failed", state = "DEGRADED" } }, } } } } end) @@ -63,8 +79,11 @@ Test.gql("Branch state facets are scoped to their Postgres", function(t) branches { nodes { name state } facets { states { state count } } } } } } }]] t.check { data = { team = { environment = { postgres = { branches = { - nodes = { { name = "main", state = "AVAILABLE" } }, - facets = { states = { { state = "AVAILABLE", count = 1 } } }, + nodes = { + { name = "failed", state = "DEGRADED" }, + { name = "main", state = "AVAILABLE" }, + }, + facets = { states = { { state = "AVAILABLE", count = 1 }, { state = "DEGRADED", count = 1 } } }, } } } } } } end) diff --git a/internal/persistence/postgres/models.go b/internal/persistence/postgres/models.go index dc4e3c61d..ecac98dd6 100644 --- a/internal/persistence/postgres/models.go +++ b/internal/persistence/postgres/models.go @@ -486,9 +486,14 @@ func toPostgres(u *unstructured.Unstructured, environmentName string) (*Postgres }, nil } -// postgresStateFromConditions interprets the CNPG phase mirrored by pgrator. +// postgresStateFromConditions interprets pgrator and CNPG status conditions. // ObservedState=False means no phase has been observed, not a failed cluster. func postgresStateFromConditions(conditions []metav1.Condition, reconciled bool) PostgresBranchState { + for _, condition := range conditions { + if condition.Type == "RecoveryFailed" && condition.Status == metav1.ConditionTrue && condition.Reason == "Unrecoverable" { + return PostgresBranchStateDegraded + } + } if !reconciled { return PostgresBranchStateProgressing } diff --git a/internal/persistence/postgres/models_test.go b/internal/persistence/postgres/models_test.go index 6f92222c8..0a785f777 100644 --- a/internal/persistence/postgres/models_test.go +++ b/internal/persistence/postgres/models_test.go @@ -133,6 +133,9 @@ func TestPostgresStateFromConditions(t *testing.T) { want PostgresBranchState }{ {name: "not reconciled", want: PostgresBranchStateProgressing}, + {name: "unrecoverable recovery failure", conditions: []metav1.Condition{{Type: "RecoveryFailed", Status: metav1.ConditionTrue, Reason: "Unrecoverable"}}, want: PostgresBranchStateDegraded}, + {name: "recovery failure not terminal", conditions: []metav1.Condition{{Type: "RecoveryFailed", Status: metav1.ConditionFalse, Reason: "Recovering"}}, want: PostgresBranchStateProgressing}, + {name: "ready false without recovery failure", reconciled: true, conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, Reason: "RecoveryFailed"}}, want: PostgresBranchStateProgressing}, {name: "healthy", reconciled: true, conditions: []metav1.Condition{{Type: "cluster.postgresql.cnpg.io/ObservedState", Status: metav1.ConditionTrue, Message: "Cluster is in phase: Cluster in healthy state"}}, want: PostgresBranchStateAvailable}, {name: "still starting", reconciled: true, conditions: []metav1.Condition{{Type: "cluster.postgresql.cnpg.io/ObservedState", Status: metav1.ConditionFalse, Message: "Cluster is in phase: "}}, want: PostgresBranchStateProgressing}, {name: "unrecoverable", reconciled: true, conditions: []metav1.Condition{{Type: "cluster.postgresql.cnpg.io/ObservedState", Status: metav1.ConditionTrue, Message: "Cluster is in phase: Cluster is unrecoverable and needs manual intervention"}}, want: PostgresBranchStateDegraded},