From 99d7108df0972fdb70420bca98bfdede78664952 Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Fri, 16 Oct 2026 19:10:37 +0000 Subject: [PATCH 1/2] DAOS-19761 control: Add rank/target details to check query In verbose mode, include: - Target status organized by rank. - Rank and target of inconsistancy reports. Features: recovery Signed-off-by: Kris Jacque --- src/control/cmd/dmg/pretty/check.go | 68 +++++++++++++- src/control/cmd/dmg/pretty/check_test.go | 96 ++++++++++++++++++- src/control/lib/control/check.go | 112 ++++++++++++++++++----- src/control/lib/control/check_test.go | 13 ++- 4 files changed, 263 insertions(+), 26 deletions(-) diff --git a/src/control/cmd/dmg/pretty/check.go b/src/control/cmd/dmg/pretty/check.go index 7860e694d26..85d98075ffd 100644 --- a/src/control/cmd/dmg/pretty/check.go +++ b/src/control/cmd/dmg/pretty/check.go @@ -1,6 +1,6 @@ // // (C) Copyright 2020-2023 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -16,6 +16,7 @@ import ( "github.com/daos-stack/daos/src/control/common" "github.com/daos-stack/daos/src/control/lib/control" + "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/lib/txtfmt" ) @@ -73,6 +74,62 @@ func countResultPools(resp *control.SystemCheckQueryResp) (int, int) { return len(checkedMap), len(uncheckedMap) } +func printSystemCheckTargetInfo(out io.Writer, targets []*control.SystemCheckTargetInfo) { + byRank := make(map[ranklist.Rank]map[string][]*control.SystemCheckTargetInfo) + allStatus := make(map[string]struct{}) + + for _, t := range targets { + if _, ok := byRank[t.Rank]; !ok { + byRank[t.Rank] = make(map[string][]*control.SystemCheckTargetInfo) + } + byRank[t.Rank][t.Status] = append(byRank[t.Rank][t.Status], t) + allStatus[t.Status] = struct{}{} + } + + sortedRanks := make([]ranklist.Rank, 0, len(byRank)) + for r := range byRank { + sortedRanks = append(sortedRanks, r) + } + sort.Slice(sortedRanks, func(i, j int) bool { + return sortedRanks[i] < sortedRanks[j] + }) + + sortedStatus := make([]string, 0, len(allStatus)) + for s := range allStatus { + sortedStatus = append(sortedStatus, s) + } + sort.Slice(sortedStatus, func(i, j int) bool { + return sortedStatus[i] < sortedStatus[j] + }) + + condensedTargetString := func(targets []*control.SystemCheckTargetInfo) string { + targetSet := ranklist.NewRankSet() + for _, target := range targets { + targetSet.Add(ranklist.Rank(target.Target)) + } + if len(targets) == 1 { + return "[" + targetSet.String() + "]" + } + return targetSet.RangedString() + } + + iw := txtfmt.NewIndentWriter(out) + for _, r := range sortedRanks { + fmt.Fprintf(iw, "Rank %d target status:\n", r) + + iw2 := txtfmt.NewIndentWriter(iw) + statusMap := byRank[r] + for _, s := range sortedStatus { + targets, ok := statusMap[s] + if !ok { + continue + } + + fmt.Fprintf(iw2, "%s: %s\n", s, condensedTargetString(targets)) + } + } +} + func printSystemCheckPoolInfo(out io.Writer, pools []*control.SystemCheckPoolInfo, verbose bool) { if verbose { fmt.Fprintln(out, "\nPer-Pool Checker Info:") @@ -80,8 +137,11 @@ func printSystemCheckPoolInfo(out io.Writer, pools []*control.SystemCheckPoolInf fmt.Fprintln(out, "\nUnchecked Pools:") } for _, pool := range pools { + iw := txtfmt.NewIndentWriter(out) if verbose || pool.Unchecked() { - fmt.Fprintf(out, " %+v\n", pool) + fmt.Fprintf(iw, "%+v\n", pool) + + printSystemCheckTargetInfo(iw, pool.Targets) } } } @@ -241,6 +301,10 @@ func printInconsistencyReportsVerbose(out io.Writer, resp *control.SystemCheckQu fmt.Fprintf(iw, "ID: 0x%x\n", report.Seq) fmt.Fprintf(iw, "Class: %s\n", cls) fmt.Fprintf(iw, "Message: %s\n", report.Msg) + if ranklist.Rank(report.Rank) != ranklist.NilRank { + fmt.Fprintf(iw, "Rank: %d\n", report.Rank) + fmt.Fprintf(iw, "Target: %d\n", report.Target) + } fmt.Fprintf(iw, "Pool: %s\n", checkerPoolID(report, true)) if report.ContUuid != "" { fmt.Fprintf(iw, "Container: %s\n", checkerContID(report, true)) diff --git a/src/control/cmd/dmg/pretty/check_test.go b/src/control/cmd/dmg/pretty/check_test.go index 3394206d020..1c22683ff66 100644 --- a/src/control/cmd/dmg/pretty/check_test.go +++ b/src/control/cmd/dmg/pretty/check_test.go @@ -1,6 +1,6 @@ // // (C) Copyright 2023 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -16,8 +16,10 @@ import ( "github.com/google/go-cmp/cmp" "github.com/daos-stack/daos/src/control/cmd/dmg/pretty" + "github.com/daos-stack/daos/src/control/common/proto/chk" chkpb "github.com/daos-stack/daos/src/control/common/proto/chk" "github.com/daos-stack/daos/src/control/lib/control" + "github.com/daos-stack/daos/src/control/lib/ranklist" ) func TestPretty_PrintCheckQueryResp(t *testing.T) { @@ -81,23 +83,86 @@ No reports to display. Status: chkpb.CheckPoolStatus_CPS_CHECKED.String(), Phase: chkpb.CheckScanPhase_CSP_DONE.String(), StartTime: checkTime, + Targets: []*control.SystemCheckTargetInfo{ + { + Target: 1, + Status: chk.CheckInstStatus_CIS_FAILED.String(), + }, + { + Target: 2, + Status: chk.CheckInstStatus_CIS_COMPLETED.String(), + }, + { + Target: 3, + Rank: 1, + Status: chk.CheckInstStatus_CIS_COMPLETED.String(), + }, + }, }, "pool-2": { UUID: "pool-2", Status: chkpb.CheckPoolStatus_CPS_CHECKED.String(), Phase: chkpb.CheckScanPhase_CSP_DONE.String(), StartTime: checkTime, + Targets: []*control.SystemCheckTargetInfo{ + { + Target: 4, + Status: chk.CheckInstStatus_CIS_COMPLETED.String(), + }, + { + Target: 5, + Status: chk.CheckInstStatus_CIS_COMPLETED.String(), + }, + { + Target: 6, + Status: chk.CheckInstStatus_CIS_COMPLETED.String(), + }, + }, }, "pool-3": { UUID: "pool-3", Status: chkpb.CheckPoolStatus_CPS_CHECKED.String(), Phase: chkpb.CheckScanPhase_CSP_DONE.String(), StartTime: checkTime, + Targets: []*control.SystemCheckTargetInfo{ + { + Rank: 3, + Target: 7, + Status: chk.CheckInstStatus_CIS_IMPLICATED.String(), + }, + { + Rank: 2, + Target: 8, + Status: chk.CheckInstStatus_CIS_STOPPED.String(), + }, + { + Rank: 1, + Target: 9, + Status: chk.CheckInstStatus_CIS_COMPLETED.String(), + }, + }, }, "pool-5": { UUID: "pool-5", Status: chkpb.CheckPoolStatus_CPS_UNCHECKED.String(), Phase: chkpb.CheckScanPhase_CSP_PREPARE.String(), + Targets: []*control.SystemCheckTargetInfo{ + { + Rank: 1, + Target: 10, + Status: chk.CheckInstStatus_CIS_RUNNING.String(), + }, + { + Rank: 1, + Target: 11, + Status: chk.CheckInstStatus_CIS_PAUSED.String(), + }, + { + Rank: 1, + Target: 12, + Status: chk.CheckInstStatus_CIS_RUNNING.String(), + }, + }, }, }, Reports: []*control.SystemCheckReport{ @@ -108,6 +173,8 @@ No reports to display. Action: chkpb.CheckInconsistAction_CIA_IGNORE, Msg: "message 1", PoolUuid: "pool-1", + Rank: 2, + Target: 0, }, }, { @@ -117,6 +184,7 @@ No reports to display. Action: chkpb.CheckInconsistAction_CIA_TRUST_MS, Msg: "message 2", PoolUuid: "pool-2", + Rank: uint32(ranklist.NilRank), }, }, { @@ -126,6 +194,8 @@ No reports to display. Action: chkpb.CheckInconsistAction_CIA_TRUST_PS, Msg: "message 3", PoolUuid: "pool-3", + Rank: 2, + Target: 4, }, }, { @@ -149,14 +219,32 @@ DAOS System Checker Info Per-Pool Checker Info: Pool pool-1: 0 ranks, status: CPS_CHECKED, phase: CSP_DONE, started: 2023-03-20T10:07:00.000-05:00 + Rank 0 target status: + CIS_COMPLETED: [2] + CIS_FAILED: [1] + Rank 1 target status: + CIS_COMPLETED: [3] Pool pool-2: 0 ranks, status: CPS_CHECKED, phase: CSP_DONE, started: 2023-03-20T10:07:00.000-05:00 + Rank 0 target status: + CIS_COMPLETED: [4-6] Pool pool-3: 0 ranks, status: CPS_CHECKED, phase: CSP_DONE, started: 2023-03-20T10:07:00.000-05:00 + Rank 1 target status: + CIS_COMPLETED: [9] + Rank 2 target status: + CIS_STOPPED: [8] + Rank 3 target status: + CIS_IMPLICATED: [7] Pool pool-5: 0 ranks, status: CPS_UNCHECKED, phase: CSP_PREPARE + Rank 1 target status: + CIS_PAUSED: [11] + CIS_RUNNING: [10,12] Inconsistency Reports: ID: 0x1 Class: POOL_BAD_SVCL Message: message 1 + Rank: 2 + Target: 0 Pool: pool-1 Resolution: IGNORE @@ -169,12 +257,16 @@ Inconsistency Reports: ID: 0x3 Class: POOL_LESS_SVC_WITHOUT_QUORUM Message: message 3 + Rank: 2 + Target: 4 Pool: pool-3 Resolution: TRUST_PS ID: 0x4 Class: POOL_NONEXIST_ON_ENGINE Message: message 4 + Rank: 0 + Target: 0 Pool: pool-4 Resolution: DISCARD (dry run - not applied) @@ -595,6 +687,8 @@ Inconsistency Reports: ID: 0x1 Class: POOL_BAD_LABEL Message: message 1 + Rank: 0 + Target: 0 Pool: pool-1 Potential resolution actions: 0: trust MS details diff --git a/src/control/lib/control/check.go b/src/control/lib/control/check.go index f173b46bf6c..8be9dd622dd 100644 --- a/src/control/lib/control/check.go +++ b/src/control/lib/control/check.go @@ -475,17 +475,63 @@ func (r *SystemCheckReport) Resolution() string { return msg } +// SystemCheckStatistics contains statistics about the inconsistencies discovered by the checker. +type SystemCheckStatistics struct { + NumTotal uint32 `json:"num_inconsist_total"` + NumRepaired uint32 `json:"num_inconsist_repaired"` + NumIgnored uint32 `json:"num_inconsist_ignored"` + NumFailed uint32 `json:"num_inconsist_failed"` +} + +// SystemCheckTargetInfo contains check info for an individual target. +type SystemCheckTargetInfo struct { + Rank ranklist.Rank `json:"rank"` + Target uint32 `json:"target"` + Status string `json:"status"` + StartTime time.Time `json:"-"` + StopTime time.Time `json:"-"` + Remaining time.Duration `json:"-"` + Stats *SystemCheckStatistics `json:"statistics"` +} + +func startTimeString(t time.Time) string { + return common.FormatTime(t) +} + +func stopTimeString(t time.Time) string { + if t.IsZero() { + return "" + } + return common.FormatTime(t) +} + +func (t *SystemCheckTargetInfo) MarshalJSON() ([]byte, error) { + type toJSON SystemCheckTargetInfo + return json.Marshal(&struct { + *toJSON + StartTime string `json:"start_time"` + StopTime string `json:"stop_time,omitempty"` + Remaining float64 `json:"remaining,omitempty"` + }{ + toJSON: (*toJSON)(t), + StartTime: startTimeString(t.StartTime), + StopTime: stopTimeString(t.StopTime), + Remaining: t.Remaining.Seconds(), + }) +} + type rawRankMap map[ranklist.Rank]*mgmtpb.CheckQueryPool type SystemCheckPoolInfo struct { - RawRankInfo rawRankMap `json:"-"` - UUID string `json:"uuid"` - Label string `json:"label,omitempty"` - Status string `json:"status"` - Phase string `json:"phase"` - StartTime time.Time `json:"-"` - StopTime time.Time `json:"-"` - Remaining time.Duration `json:"-"` + RawRankInfo rawRankMap `json:"-"` + UUID string `json:"uuid"` + Label string `json:"label,omitempty"` + Status string `json:"status"` + Phase string `json:"phase"` + StartTime time.Time `json:"-"` + StopTime time.Time `json:"-"` + Remaining time.Duration `json:"-"` + Targets []*SystemCheckTargetInfo `json:"targets"` } func (p *SystemCheckPoolInfo) MarshalJSON() ([]byte, error) { @@ -499,13 +545,8 @@ func (p *SystemCheckPoolInfo) MarshalJSON() ([]byte, error) { }{ toJSON: (*toJSON)(p), RankCount: len(p.RawRankInfo), - StartTime: common.FormatTime(p.StartTime), - StopTime: func() string { - if p.StopTime.IsZero() { - return "" - } - return common.FormatTime(p.StopTime) - }(), + StartTime: startTimeString(p.StartTime), + StopTime: stopTimeString(p.StopTime), Remaining: p.Remaining.Seconds(), }) } @@ -540,22 +581,26 @@ func isPoolCheckInProgress(status chkpb.CheckPoolStatus) bool { return status == chkpb.CheckPoolStatus_CPS_CHECKING || status == chkpb.CheckPoolStatus_CPS_PENDING } -func remainingDuration(status chkpb.CheckPoolStatus, seconds uint64) time.Duration { - if !isPoolCheckInProgress(status) { +func remainingDuration(inProgress bool, seconds uint64) time.Duration { + if !inProgress { // Done - no time remaining return 0 } return time.Duration(seconds) * time.Second } -func stopTime(status chkpb.CheckPoolStatus, timestamp uint64) time.Time { - if isPoolCheckInProgress(status) { +func stopTime(inProgress bool, timestamp uint64) time.Time { + if inProgress { // Hasn't stopped yet return time.Time{} } return time.Unix(int64(timestamp), 0) } +func isTargetCheckInProgress(status chkpb.CheckInstStatus) bool { + return status == chkpb.CheckInstStatus_CIS_RUNNING +} + func getPoolCheckInfo(pbPools []*mgmtpb.CheckQueryPool) map[string]*SystemCheckPoolInfo { pools := make(map[string]*SystemCheckPoolInfo) @@ -569,10 +614,35 @@ func getPoolCheckInfo(pbPools []*mgmtpb.CheckQueryPool) map[string]*SystemCheckP Status: pbPool.Status.String(), Phase: pbPool.Phase.String(), StartTime: time.Unix(int64(pbPool.Time.StartTime), 0), - StopTime: stopTime(pbPool.Status, pbPool.Time.MiscTime), - Remaining: remainingDuration(pbPool.Status, pbPool.Time.MiscTime), + StopTime: stopTime(isPoolCheckInProgress(pbPool.Status), pbPool.Time.MiscTime), + Remaining: remainingDuration(isPoolCheckInProgress(pbPool.Status), pbPool.Time.MiscTime), } } + for _, pbTarget := range pbPool.Targets { + tInfo := &SystemCheckTargetInfo{ + Rank: ranklist.Rank(pbTarget.Rank), + Target: pbTarget.Target, + Status: pbTarget.Status.String(), + StartTime: time.Unix(int64(pbTarget.Time.StartTime), 0), + StopTime: stopTime(isTargetCheckInProgress(pbTarget.Status), pbTarget.Time.MiscTime), + Remaining: remainingDuration(isTargetCheckInProgress(pbTarget.Status), pbPool.Time.MiscTime), + Stats: &SystemCheckStatistics{ + NumTotal: pbTarget.Inconsistency.Total, + NumRepaired: pbTarget.Inconsistency.Repaired, + NumIgnored: pbTarget.Inconsistency.Ignored, + NumFailed: pbTarget.Inconsistency.Failed, + }, + } + pools[pbPool.Uuid].Targets = append(pools[pbPool.Uuid].Targets, tInfo) + } + sortedTargets := pools[pbPool.Uuid].Targets + sort.Slice(sortedTargets, func(i, j int) bool { + if sortedTargets[i].Rank == sortedTargets[j].Rank { + return sortedTargets[i].Target < sortedTargets[j].Target + } + return sortedTargets[i].Rank < sortedTargets[j].Rank + }) + pools[pbPool.Uuid].Targets = sortedTargets pools[pbPool.Uuid].RawRankInfo[getQueryPoolRank(pbPool)] = pbPool } diff --git a/src/control/lib/control/check_test.go b/src/control/lib/control/check_test.go index b0040db84fc..e3503abc5d0 100644 --- a/src/control/lib/control/check_test.go +++ b/src/control/lib/control/check_test.go @@ -238,7 +238,7 @@ func TestControl_SystemCheckQuery(t *testing.T) { Inconsistency: &mgmtpb.CheckQueryInconsist{}, Time: &mgmtpb.CheckQueryTime{ StartTime: uint64(testTime.Unix()), - MiscTime: uint64(testTime.Unix() + 15), + MiscTime: uint64(testTime.Unix() + 12), }, }, }, @@ -276,7 +276,7 @@ func TestControl_SystemCheckQuery(t *testing.T) { Inconsistency: &mgmtpb.CheckQueryInconsist{}, Time: &mgmtpb.CheckQueryTime{ StartTime: uint64(testTime.Unix()), - MiscTime: uint64(testTime.Unix() + 15), + MiscTime: uint64(testTime.Unix() + 12), }, }, }, @@ -287,6 +287,15 @@ func TestControl_SystemCheckQuery(t *testing.T) { Phase: chkpb.CheckScanPhase_CSP_DONE.String(), StartTime: time.Unix(testTime.Unix(), 0), StopTime: time.Unix(testTime.Unix()+15, 0), + Targets: []*SystemCheckTargetInfo{ + { + Rank: 3, + Status: "CIS_COMPLETED", + StartTime: time.Unix(testTime.Unix(), 0), + StopTime: time.Unix(testTime.Unix()+12, 0), + Stats: &SystemCheckStatistics{}, + }, + }, }, }, Reports: []*SystemCheckReport{ From 5677f5d644d40c1b8a891c3e48bd45c110ef04da Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Fri, 23 Oct 2026 15:45:31 +0000 Subject: [PATCH 2/2] Address knard38's review comments Features: recovery Signed-off-by: Kris Jacque --- src/control/cmd/dmg/pretty/check.go | 4 +++- src/control/lib/control/check.go | 11 +++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/control/cmd/dmg/pretty/check.go b/src/control/cmd/dmg/pretty/check.go index 85d98075ffd..03b5c90f77f 100644 --- a/src/control/cmd/dmg/pretty/check.go +++ b/src/control/cmd/dmg/pretty/check.go @@ -141,7 +141,9 @@ func printSystemCheckPoolInfo(out io.Writer, pools []*control.SystemCheckPoolInf if verbose || pool.Unchecked() { fmt.Fprintf(iw, "%+v\n", pool) - printSystemCheckTargetInfo(iw, pool.Targets) + if verbose { + printSystemCheckTargetInfo(iw, pool.Targets) + } } } } diff --git a/src/control/lib/control/check.go b/src/control/lib/control/check.go index 8be9dd622dd..36a4c08a650 100644 --- a/src/control/lib/control/check.go +++ b/src/control/lib/control/check.go @@ -625,7 +625,7 @@ func getPoolCheckInfo(pbPools []*mgmtpb.CheckQueryPool) map[string]*SystemCheckP Status: pbTarget.Status.String(), StartTime: time.Unix(int64(pbTarget.Time.StartTime), 0), StopTime: stopTime(isTargetCheckInProgress(pbTarget.Status), pbTarget.Time.MiscTime), - Remaining: remainingDuration(isTargetCheckInProgress(pbTarget.Status), pbPool.Time.MiscTime), + Remaining: remainingDuration(isTargetCheckInProgress(pbTarget.Status), pbTarget.Time.MiscTime), Stats: &SystemCheckStatistics{ NumTotal: pbTarget.Inconsistency.Total, NumRepaired: pbTarget.Inconsistency.Repaired, @@ -635,15 +635,18 @@ func getPoolCheckInfo(pbPools []*mgmtpb.CheckQueryPool) map[string]*SystemCheckP } pools[pbPool.Uuid].Targets = append(pools[pbPool.Uuid].Targets, tInfo) } - sortedTargets := pools[pbPool.Uuid].Targets + pools[pbPool.Uuid].RawRankInfo[getQueryPoolRank(pbPool)] = pbPool + } + + for _, pool := range pools { + sortedTargets := pool.Targets sort.Slice(sortedTargets, func(i, j int) bool { if sortedTargets[i].Rank == sortedTargets[j].Rank { return sortedTargets[i].Target < sortedTargets[j].Target } return sortedTargets[i].Rank < sortedTargets[j].Rank }) - pools[pbPool.Uuid].Targets = sortedTargets - pools[pbPool.Uuid].RawRankInfo[getQueryPoolRank(pbPool)] = pbPool + pool.Targets = sortedTargets } return pools