From b33c67ea56172b0e627e54270e272805d32d44f2 Mon Sep 17 00:00:00 2001 From: Kenneth Cain Date: Thu, 17 Sep 2026 16:29:23 -0400 Subject: [PATCH 1/3] DAOS-19396 test: suite.yaml fix libfabric warning log routing Before this change, daos_test invocations from suite.py/yaml relied on FI_LOG_LEVEL and D_LOG_STDERR_IN_LOG environment variables in an attempt to capture libfabric-level warnings. However, that configuration seems to be ineffective - i.e., it is seen in test results that no such warnings are saved in engine logs. With this change, HG_LOG_SUBSYS (with value including 'libfabric') and HG_LOG_LEVEL environnment variables are set to correctly capture libfabric warnings - e.g., to log entries like the following: mercury->libfabric [warning] ... rxm_handle_error() fi_eq_readerr ... Test-tag: daos_core_test Skip-unit-tests: true Skip-fault-injection-test: true Skip-test-rpms: true Signed-off-by: Kenneth Cain --- src/tests/ftest/daos_test/suite.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tests/ftest/daos_test/suite.yaml b/src/tests/ftest/daos_test/suite.yaml index 15c3c40b8d7..41eee615a49 100644 --- a/src/tests/ftest/daos_test/suite.yaml +++ b/src/tests/ftest/daos_test/suite.yaml @@ -49,8 +49,8 @@ server_config: - D_LOG_FILE_APPEND_PID=1 - D_LOG_FILE_APPEND_RANK=1 - D_LOG_FLUSH=DEBUG - - FI_LOG_LEVEL=warn - - D_LOG_STDERR_IN_LOG=1 + - HG_LOG_LEVEL=warn + - HG_LOG_SUBSYS=hg,na,libfabric - DAOS_PIPELINE=1 storage: auto 1: @@ -63,8 +63,8 @@ server_config: - D_LOG_FILE_APPEND_PID=1 - D_LOG_FILE_APPEND_RANK=1 - D_LOG_FLUSH=DEBUG - - FI_LOG_LEVEL=warn - - D_LOG_STDERR_IN_LOG=1 + - HG_LOG_LEVEL=warn + - HG_LOG_SUBSYS=hg,na,libfabric - DAOS_PIPELINE=1 storage: auto transport_config: From 98ec0f16c635f5c5b1f77247fb47b78f489fdeb8 Mon Sep 17 00:00:00 2001 From: Kenneth Cain Date: Fri, 18 Sep 2026 17:45:01 -0400 Subject: [PATCH 2/3] When a peer engine restarts, its address remains cached on other engines and is never invalidated on a transient SWIM death+restart. RPCs then take the cache-hit path and reuse a possibly-stale endpoint from the peer's previous incarnation, which can silently fail (e.g. a CORPC timing out to a just-restarted rank) with no CaRT-level indication of the cause. Add a diagnostic-only D_WARN, emitted when a peer's incarnation increases (i.e. it restarted) while a cached address for it is still present for ctx0/tag0. The check is read-only and does not change behavior. The incarnation increase can be observed on either of two paths, and which one records it first is a race: - crt_swim_set_member_state() (SWIM gossip), and - crt_swim_rank_check() (primary-group membership / rejoin update). The warning is emitted from both sites so it fires regardless of which path observes the restart first. The gossip-path check is gated on a real incarnation increase and a transition to ALIVE to keep it low-volume (once per restart, not per ping). This is intended to be combined with HG_LOG_SUBSYS/HG_LOG_LEVEL and the existing RPC timeout logs to correlate a stale-endpoint reuse at runtime. Test-tag: daos_core_test Skip-unit-tests: true Skip-fault-injection-test: true Skip-test-rpms: true Signed-off-by: Kenneth Cain --- src/cart/crt_swim.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/cart/crt_swim.c b/src/cart/crt_swim.c index 298a753a130..cc1890064f4 100644 --- a/src/cart/crt_swim.c +++ b/src/cart/crt_swim.c @@ -1,5 +1,6 @@ /* * (C) Copyright 2019-2024 Intel Corporation. + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -935,6 +936,23 @@ static int crt_swim_get_member_state(struct swim_context *ctx, return rc; } +/* + * Diagnostic only (no behavior change): when a peer's incarnation increases + * (it restarted), warn if a cached address for it is still present, since RPCs + * may reuse a possibly-stale endpoint from the peer's previous incarnation. + */ +static void +crt_swim_warn_stale_addr(struct crt_grp_priv *grp_priv, d_rank_t rank, uint64_t incarnation) +{ + hg_addr_t stale_addr = NULL; + + crt_grp_lc_lookup(grp_priv, 0, rank, 0, NULL, &stale_addr); + if (stale_addr != NULL) + D_WARN("rank %u restarted (incarnation " DF_X64 "); cached address still " + "present for ctx0/tag0, RPCs may reuse a stale endpoint\n", + rank, incarnation); +} + static int crt_swim_set_member_state(struct swim_context *ctx, swim_id_t id, struct swim_member_state *state) @@ -967,6 +985,10 @@ static int crt_swim_set_member_state(struct swim_context *ctx, if (rc == 0) crt_swim_notify_rank_state((d_rank_t)id, &state_prev, state); + if (rc == 0 && state->sms_incarnation > state_prev.sms_incarnation && + state->sms_status == SWIM_MEMBER_ALIVE) + crt_swim_warn_stale_addr(grp_priv, (d_rank_t)id, state->sms_incarnation); + return rc; } @@ -1633,8 +1655,10 @@ crt_swim_rank_check(struct crt_grp_priv *grp_priv, d_rank_t rank, uint64_t incar } crt_swim_csm_unlock(csm); - if (updated) + if (updated) { crt_swim_notify_rank_state(rank, &state_prev, &state); + crt_swim_warn_stale_addr(grp_priv, rank, incarnation); + } return rc; } From 8b91287dd3db3fe7de84febeb3fa256fc08d20bb Mon Sep 17 00:00:00 2001 From: Kenneth Cain Date: Tue, 29 Sep 2026 07:55:40 -0400 Subject: [PATCH 3/3] revert crt_swim.c changes Test-tag: daos_core_test Skip-unit-tests: true Skip-fault-injection-test: true Skip-test-rpms: true Signed-off-by: Kenneth Cain --- src/cart/crt_swim.c | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/src/cart/crt_swim.c b/src/cart/crt_swim.c index cc1890064f4..298a753a130 100644 --- a/src/cart/crt_swim.c +++ b/src/cart/crt_swim.c @@ -1,6 +1,5 @@ /* * (C) Copyright 2019-2024 Intel Corporation. - * (C) Copyright 2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -936,23 +935,6 @@ static int crt_swim_get_member_state(struct swim_context *ctx, return rc; } -/* - * Diagnostic only (no behavior change): when a peer's incarnation increases - * (it restarted), warn if a cached address for it is still present, since RPCs - * may reuse a possibly-stale endpoint from the peer's previous incarnation. - */ -static void -crt_swim_warn_stale_addr(struct crt_grp_priv *grp_priv, d_rank_t rank, uint64_t incarnation) -{ - hg_addr_t stale_addr = NULL; - - crt_grp_lc_lookup(grp_priv, 0, rank, 0, NULL, &stale_addr); - if (stale_addr != NULL) - D_WARN("rank %u restarted (incarnation " DF_X64 "); cached address still " - "present for ctx0/tag0, RPCs may reuse a stale endpoint\n", - rank, incarnation); -} - static int crt_swim_set_member_state(struct swim_context *ctx, swim_id_t id, struct swim_member_state *state) @@ -985,10 +967,6 @@ static int crt_swim_set_member_state(struct swim_context *ctx, if (rc == 0) crt_swim_notify_rank_state((d_rank_t)id, &state_prev, state); - if (rc == 0 && state->sms_incarnation > state_prev.sms_incarnation && - state->sms_status == SWIM_MEMBER_ALIVE) - crt_swim_warn_stale_addr(grp_priv, (d_rank_t)id, state->sms_incarnation); - return rc; } @@ -1655,10 +1633,8 @@ crt_swim_rank_check(struct crt_grp_priv *grp_priv, d_rank_t rank, uint64_t incar } crt_swim_csm_unlock(csm); - if (updated) { + if (updated) crt_swim_notify_rank_state(rank, &state_prev, &state); - crt_swim_warn_stale_addr(grp_priv, rank, incarnation); - } return rc; }