From e08754b4a8c3f46735eed1cbc8d9702b18795cdc Mon Sep 17 00:00:00 2001 From: Zhen Liang Date: Fri, 1 Aug 2025 22:12:04 +0800 Subject: [PATCH 1/3] DAOS-17843 placement: correct the way JCH placement calls CRC The Jump Consistent Hash (JCH) placement algorithm in DAOS calls the CRC functions incorrectly, which may lead to excessive hash collisions and imbalanced data distribution. This patch corrects how the JCH placement algorithm invokes the CRC functions. Signed-off-by: Zhen Liang Signed-off-by: Wang Shilong --- src/container/srv_container.c | 4 + src/include/daos_srv/pool.h | 6 +- src/placement/jump_map.c | 36 ++-- src/placement/jump_map.h | 19 ++ src/placement/jump_map_versions.c | 242 +++++++++++++++++++++-- src/placement/tests/jump_map_place_obj.c | 12 +- src/placement/tests/place_obj_common.h | 3 +- 7 files changed, 287 insertions(+), 35 deletions(-) diff --git a/src/container/srv_container.c b/src/container/srv_container.c index 755ba959754..ba7237cfb44 100644 --- a/src/container/srv_container.c +++ b/src/container/srv_container.c @@ -770,6 +770,10 @@ cont_create_prop_prepare(struct ds_pool_hdl *pool_hdl, entry_def = daos_prop_entry_get(prop_def, DAOS_PROP_CO_OBJ_VERSION); if (entry_def) entry_def->dpe_val = pool_hdl->sph_obj_ver; + /* disallow old version for container creation */ + if (entry_def->dpe_val < DS_POOL_OBJ_VERSION_2) + entry_def->dpe_val = DS_POOL_OBJ_VERSION_2; + /* for new container set HEALTHY status with current pm ver */ entry_def = daos_prop_entry_get(prop_def, DAOS_PROP_CO_STATUS); D_ASSERT(entry_def != NULL); diff --git a/src/include/daos_srv/pool.h b/src/include/daos_srv/pool.h index d14c9e0fd88..4c04f0dcc01 100644 --- a/src/include/daos_srv/pool.h +++ b/src/include/daos_srv/pool.h @@ -34,7 +34,11 @@ struct ds_pool_svc; * Each individual object layout format, like oid layout, dkey to group, * dkey to EC group start. */ -#define DS_POOL_OBJ_VERSION 1 +enum { + DS_POOL_OBJ_VERSION_1 = 1, + DS_POOL_OBJ_VERSION_2 = 2, + DS_POOL_OBJ_VERSION = DS_POOL_OBJ_VERSION_2, +}; /* age of an entry in svc_ops KVS before it may be evicted */ #define DEFAULT_SVC_OPS_ENTRY_AGE_SEC_MAX 300ULL diff --git a/src/placement/jump_map.c b/src/placement/jump_map.c index 8c9fb90d7a2..56f0643555e 100644 --- a/src/placement/jump_map.c +++ b/src/placement/jump_map.c @@ -47,7 +47,7 @@ jm_obj_placement_fini(struct jm_obj_placement *jmop) #define LOCAL_PD_ARRAY_SIZE (4) static int jm_obj_pd_init(struct pl_jump_map *jmap, struct daos_obj_md *md, struct pool_domain *root, - struct jm_obj_placement *jmop) + struct jm_obj_placement *jmop, uint32_t layout_ver) { struct pool_domain *pds, *pd; uint8_t *pd_used = NULL; @@ -115,9 +115,9 @@ jm_obj_pd_init(struct pl_jump_map *jmap, struct daos_obj_md *md, struct pool_dom } oid = md->omd_id; - key = oid.hi ^ oid.lo; + key = jm_oid_hash(layout_ver, oid); for (i = 0; i < jmop->jmop_pd_nr; i++) { - key = crc(key, i); + key = (layout_ver <= 1) ? crc(key, i) : jm_crc(key, i, 0xcafebabe); selected_pd = d_hash_jump(key, jmap->jmp_pd_nr); do { selected_pd = selected_pd % jmap->jmp_pd_nr; @@ -243,8 +243,8 @@ layout_find_diff(struct pl_jump_map *jmap, struct pl_obj_layout *original, */ static int jm_obj_placement_init(struct pl_jump_map *jmap, struct daos_obj_md *md, - struct daos_obj_shard_md *shard_md, - struct jm_obj_placement *jmop) + struct daos_obj_shard_md *shard_md, struct jm_obj_placement *jmop, + uint32_t layout_ver) { struct daos_oclass_attr *oc_attr; struct pool_domain *root; @@ -306,7 +306,7 @@ jm_obj_placement_init(struct pl_jump_map *jmap, struct daos_obj_md *md, D_ASSERT(jmop->jmop_grp_nr > 0); D_ASSERT(jmop->jmop_grp_size > 0); - rc = jm_obj_pd_init(jmap, md, root, jmop); + rc = jm_obj_pd_init(jmap, md, root, jmop, layout_ver); if (rc == 0) D_DEBUG(DB_PL, "obj="DF_OID"/ grp_size=%u grp_nr=%d, pd_nr=%u pd_grp_size=%u\n", DP_OID(oid), jmop->jmop_grp_size, jmop->jmop_grp_nr, @@ -405,7 +405,7 @@ obj_remap_shards(struct pl_jump_map *jmap, uint32_t layout_ver, struct daos_obj_ current = remap_list->next; spare_tgt = NULL; oid = md->omd_id; - key = oid.hi ^ oid.lo; + key = jm_oid_hash(layout_ver, oid); spares_left = count_available_spares(jmap, layout, failed_in_layout); rc = pool_map_find_domain(jmap->jmp_map.pl_poolmap, PO_COMP_TP_ROOT, @@ -436,13 +436,16 @@ obj_remap_shards(struct pl_jump_map *jmap, uint32_t layout_ver, struct daos_obj_ dgu = f_shard->fs_data; D_ASSERT(dgu != NULL); - rebuild_key = crc(key, f_shard->fs_shard_idx); + if (layout_ver <= 1) + rebuild_key = crc(key, crc(key, f_shard->fs_shard_idx)); + else /* hash OID differently so we don't land to the same target */ + rebuild_key = jm_crc(oid.lo, oid.hi, 0xDead2Bad); + curr_pd = jm_obj_shard_pd(jmop, shard_id); - get_target(root, curr_pd, layout_ver, &spare_tgt, &spare_dom, - crc(key, rebuild_key), dom_used, dom_full, - dgu->dgu_used, dgu->dgu_real, tgts_used, - shard_id, allow_version, gen_mode, fdom_lvl, - jmop->jmop_grp_size, &spares_left, &spare_avail); + get_target(root, curr_pd, layout_ver, &spare_tgt, &spare_dom, rebuild_key, + dom_used, dom_full, dgu->dgu_used, dgu->dgu_real, tgts_used, + shard_id, allow_version, gen_mode, fdom_lvl, jmop->jmop_grp_size, + &spares_left, &spare_avail); if (layout_ver > 0) { /* * After 2.4 (layout_ver > 0), it will always assign each shard @@ -645,7 +648,8 @@ get_object_layout(struct pl_jump_map *jmap, uint32_t layout_ver, struct pl_obj_l D_GOTO(out, rc = -DER_NOMEM); oid = md->omd_id; - key = oid.hi ^ oid.lo; + key = jm_oid_hash(layout_ver, oid); + if (daos_obj_is_srank(oid)) spec_oid = true; @@ -1013,7 +1017,7 @@ jump_map_obj_place(struct pl_map *map, uint32_t layout_version, struct daos_obj_ D_DEBUG(DB_PL, "Determining location for object: "DF_OID", ver: %d, pda %u\n", DP_OID(oid), md->omd_ver, md->omd_pda); - rc = jm_obj_placement_init(jmap, md, shard_md, &jmop); + rc = jm_obj_placement_init(jmap, md, shard_md, &jmop, layout_version); if (rc) { D_ERROR("jm_obj_placement_init failed, rc "DF_RC"\n", DP_RC(rc)); return rc; @@ -1116,7 +1120,7 @@ jump_map_obj_find_diff(struct pl_map *map, uint32_t layout_ver, struct daos_obj_ } jmap = pl_map2jmap(map); - rc = jm_obj_placement_init(jmap, md, shard_md, &jop); + rc = jm_obj_placement_init(jmap, md, shard_md, &jop, layout_ver); if (rc) { D_ERROR("jm_obj_placement_init failed, rc %d.\n", rc); return rc; diff --git a/src/placement/jump_map.h b/src/placement/jump_map.h index 1fd39d3ce58..330872d842f 100644 --- a/src/placement/jump_map.h +++ b/src/placement/jump_map.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2022-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -61,4 +62,22 @@ get_target(struct pool_domain *root, struct pool_domain *curr_pd, uint32_t layou uint8_t *dom_full, uint8_t *dom_cur_grp_used, uint8_t *dom_cur_grp_real, uint8_t *tgts_used, int shard_num, uint32_t allow_version, enum layout_gen_mode gen_mode, pool_comp_type_t fdom_lvl, uint32_t grp_size, uint32_t *spare_left, bool *spare_avail); + +static inline uint64_t +jm_crc(uint64_t val0, uint64_t val1, uint64_t val2) +{ + uint64_t data[3]; + + data[0] = val0; + data[1] = val1; + data[2] = val2; + return crc64_ecma_refl(0xbabecafe, (uint8_t *)data, 24); +} + +static inline uint64_t +jm_oid_hash(uint32_t layout_ver, daos_obj_id_t oid) +{ + return layout_ver <= 1 ? (oid.hi ^ oid.lo) : jm_crc(oid.hi, oid.lo, 0xbabeface); +} + #endif /* __JUMP_MAP_H__ */ diff --git a/src/placement/jump_map_versions.c b/src/placement/jump_map_versions.c index b943949ad6a..f817ed96b04 100644 --- a/src/placement/jump_map_versions.c +++ b/src/placement/jump_map_versions.c @@ -1,6 +1,7 @@ /** * * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -414,6 +415,216 @@ __get_target_v1(struct pool_domain *root_pos, struct pool_domain *curr_pd, } while (!found_target); } +/** + * This function is identical to __get_target_v1, except how they call CRC functions, + * the V1 function may cause a large number of hash collisions in certain cases. + * + * Search for jm_crc() and crc() to see how they differ. + */ +static void +__get_target_v2(struct pool_domain *root_pos, struct pool_domain *curr_pd, + struct pool_target **target, struct pool_domain **dom, uint64_t obj_key, + uint8_t *dom_used, uint8_t *dom_full, uint8_t *dom_cur_grp_used, uint8_t *tgts_used, + int shard_num, uint32_t allow_version, enum layout_gen_mode gen_mode, + pool_comp_type_t fdom_lvl, uint32_t grp_size, bool *pd_ignored) +{ + int range_set; + uint8_t found_target = 0; + struct pool_domain *curr_dom; + struct pool_domain *dom_stack[MAX_STACK] = {0}; + int top = -1; + + curr_dom = curr_pd; + do { + uint32_t avail_doms; + uint64_t key; + + /* Retrieve number of nodes in this domain */ + avail_doms = get_num_domains(curr_dom, allow_version, gen_mode, fdom_lvl); + + /* If choosing target (lowest fault domain level) */ + if (curr_dom->do_comp.co_type == fdom_lvl) { + uint32_t fail_num = 0; + uint32_t tgt_idx; + uint32_t start_tgt; + uint32_t end_tgt; + uint32_t selected_tgt; + + start_tgt = curr_dom->do_targets - root_pos->do_targets; + end_tgt = start_tgt + (curr_dom->do_target_nr - 1); + + range_set = tgt_isset_range(root_pos->do_targets, tgts_used, start_tgt, + end_tgt, allow_version, gen_mode); + if (range_set) { + /* Used up all targets in this domain */ + D_ASSERT(top != -1); + curr_dom = dom_stack[top--]; /* try parent */ + continue; + } + + /* + * Must crc key because jump consistent hash + * requires an even distribution or it will + * not work + */ + key = jm_crc(obj_key, shard_num, fail_num++); + /* Get target for shard */ + selected_tgt = d_hash_jump(key, avail_doms); + do { + selected_tgt = selected_tgt % avail_doms; + /* Retrieve actual target using index */ + *target = + _get_target(curr_dom, selected_tgt, allow_version, gen_mode); + /* Get target id to check if target used */ + tgt_idx = *target - root_pos->do_targets; + selected_tgt++; + } while (isset(tgts_used, tgt_idx)); + + setbit(tgts_used, tgt_idx); + D_DEBUG(DB_PL, "selected tgt %d\n", tgt_idx); + D_ASSERTF(isclr(dom_full, (uint32_t)(curr_dom - root_pos)), + "selected_tgt %u\n", (uint32_t)(curr_dom - root_pos)); + range_set = tgt_isset_range(root_pos->do_targets, tgts_used, start_tgt, + end_tgt, allow_version, gen_mode); + if (range_set) { + /* Used up all targets in this domain */ + setbit(dom_full, curr_dom - root_pos); + D_DEBUG(DB_PL, "dom %d used up\n", (int)(curr_dom - root_pos)); + /* Check and set if all of its parent are full */ + while (top != -1) { + if (is_dom_full(dom_stack[top], root_pos, dom_full, + allow_version, gen_mode)) { + uint32_t off = dom_stack[top] - root_pos; + + D_DEBUG(DB_PL, "dom %u used up\n", off); + setbit(dom_full, off); + } + --top; + } + } + *dom = curr_dom; + /* Found target (which may be available or not) */ + found_target = 1; + } else { + uint32_t selected_dom; + uint32_t fail_num = 0; + uint32_t start_dom; + uint32_t end_dom; + + start_dom = (curr_dom->do_children) - root_pos; + end_dom = start_dom + (curr_dom->do_child_nr - 1); + + D_DEBUG(DB_TRACE, "start_dom %u end_dom %u\n", start_dom, end_dom); + /* Check if all targets under the domain range has been + * used up (occupied), go back to its parent if it does. + */ + range_set = + is_dom_full(curr_dom, root_pos, dom_full, allow_version, gen_mode); + if (range_set) { + if (top == -1) { + if (curr_pd != root_pos) { + /* all domains within the PD are full, ignore the + * PD restrict. + */ + D_DEBUG(DB_PL, + "PD[%d] all doms are full, weak the " + "PD restrict\n", + (int)(curr_dom - root_pos)); + curr_pd = root_pos; + curr_dom = curr_pd; + *pd_ignored = true; + continue; + } + /* shard nr > target nr, no extra target for the shard */ + *target = NULL; + return; + } + setbit(dom_full, curr_dom - root_pos); + D_DEBUG(DB_PL, "used up dom %d\n", (int)(curr_dom - root_pos)); + setbit(dom_cur_grp_used, curr_dom - root_pos); + curr_dom = dom_stack[top--]; + continue; + } + + /* Check if all children has been used for the current group or full */ + range_set = dom_isset_2ranges(root_pos, dom_full, dom_cur_grp_used, + start_dom, end_dom, allow_version, gen_mode); + if (range_set) { + if (top == -1) { + if (curr_pd != root_pos && grp_size > 1) { + /* All domains within the PD are full, ignore the + * PD restrict. For non-replica (grp_size 1) case, + * keep the PD restrict until all targets under the + * domain range has been used up (see above check + * for dom_full bitmap). + */ + D_DEBUG(DB_PL, + "PD[%d] all doms are full, weak the " + "PD restrict\n", + (int)(curr_dom - root_pos)); + curr_pd = root_pos; + curr_dom = curr_pd; + *pd_ignored = true; + continue; + } + *target = NULL; + return; + } + setbit(dom_cur_grp_used, curr_dom - root_pos); + D_DEBUG(DB_PL, "set grp_used %d\n", (int)(curr_dom - root_pos)); + curr_dom = dom_stack[top--]; + continue; + } + + /* Check if all children has been used for the current group or the + * object. + */ + range_set = dom_isset_2ranges(root_pos, dom_used, dom_cur_grp_used, + start_dom, end_dom, allow_version, gen_mode); + if (range_set) { + int idx; + + for (idx = start_dom; idx <= end_dom; ++idx) + if (!isset(dom_full, idx)) { + clrbit(dom_used, idx); + D_DEBUG(DB_PL, "clrbit dom_used %d\n", idx); + } + if (top == -1) + curr_dom = curr_pd; + else + curr_dom = dom_stack[top--]; + continue; + } + + /* Keep choosing the new domain until the one has not been used. */ + do { + struct pool_domain *_dom; + + key = jm_crc(obj_key, shard_num, fail_num++); + selected_dom = d_hash_jump(key, avail_doms); + _dom = _get_dom(curr_dom, selected_dom, allow_version, gen_mode); + if (_dom != NULL) + selected_dom = _dom - curr_dom->do_children; + } while (isset(dom_used, start_dom + selected_dom) || + isset(dom_cur_grp_used, start_dom + selected_dom)); + + D_ASSERTF(isclr(dom_full, start_dom + selected_dom), "selected_dom %u\n", + selected_dom); + /* Mark this domain as used */ + if (curr_dom == curr_pd && curr_pd != root_pos) + setbit(dom_used, (int)(curr_dom - root_pos)); + D_DEBUG(DB_PL, "selected dom %d\n", start_dom + selected_dom); + setbit(dom_used, start_dom + selected_dom); + setbit(dom_cur_grp_used, start_dom + selected_dom); + D_ASSERT(top < MAX_STACK - 1); + top++; + dom_stack[top] = curr_dom; + curr_dom = &curr_dom->do_children[selected_dom]; + obj_key = jm_crc(obj_key, curr_dom->do_comp.co_id, 0xbabecafe); + } + } while (!found_target); +} + /* Only reset all used dom/tgts bits if all domain has been used. */ static void dom_reset_bit(struct pool_domain *dom, uint8_t *dom_bits, struct pool_domain *root, @@ -604,11 +815,11 @@ reset_dom_cur_grp_v1(struct pool_domain *root, struct pool_domain *curr_pd, } static void -get_target_v1(struct pool_domain *root, struct pool_domain *curr_pd, struct pool_target **target, - struct pool_domain **dom, uint64_t key, uint8_t *dom_used, uint8_t *dom_full, - uint8_t *dom_cur_grp_used, uint8_t *dom_cur_grp_real, uint8_t *tgts_used, - int shard_num, uint32_t allow_version, enum layout_gen_mode gen_mode, - pool_comp_type_t fdom_lvl, uint32_t grp_size) +get_target_new(struct pool_domain *root, struct pool_domain *curr_pd, uint32_t layout_ver, + struct pool_target **target, struct pool_domain **dom, uint64_t key, + uint8_t *dom_used, uint8_t *dom_full, uint8_t *dom_cur_grp_used, + uint8_t *dom_cur_grp_real, uint8_t *tgts_used, int shard_num, uint32_t allow_version, + enum layout_gen_mode gen_mode, pool_comp_type_t fdom_lvl, uint32_t grp_size) { struct pool_target *found = NULL; bool pd_ignored; @@ -622,9 +833,17 @@ get_target_v1(struct pool_domain *root, struct pool_domain *curr_pd, struct pool */ while (found == NULL) { pd_ignored = false; - __get_target_v1(root, curr_pd, &found, dom, key, dom_used, dom_full, - dom_cur_grp_used, tgts_used, shard_num, allow_version, gen_mode, - fdom_lvl, grp_size, &pd_ignored); + if (layout_ver == 1) { + __get_target_v1(root, curr_pd, &found, dom, key, dom_used, dom_full, + dom_cur_grp_used, tgts_used, shard_num, allow_version, + gen_mode, fdom_lvl, grp_size, &pd_ignored); + } else { /* Fix CRC calls */ + D_ASSERT(layout_ver == 2); + __get_target_v2(root, curr_pd, &found, dom, key, dom_used, dom_full, + dom_cur_grp_used, tgts_used, shard_num, allow_version, + gen_mode, fdom_lvl, grp_size, &pd_ignored); + } + if (found == NULL) { if (pd_ignored) reset_dom_cur_grp_v1(root, root, dom_cur_grp_used, dom_cur_grp_real, @@ -901,14 +1120,15 @@ get_target(struct pool_domain *root, struct pool_domain *curr_pd, uint32_t layou } break; case 1: + case 2: /** * New layout(>= 1) will always try to assign target to the shard due to the high * overhead of the EC degraded fetch, so it might put multiple shards in the * same target, i.e. it will never check the spare target here. */ - get_target_v1(root, curr_pd, target, dom, key, dom_used, dom_full, dom_cur_grp_used, - dom_cur_grp_real, tgts_used, shard_num, allow_version, gen_mode, - fdom_lvl, grp_size); + get_target_new(root, curr_pd, layout_ver, target, dom, key, dom_used, dom_full, + dom_cur_grp_used, dom_cur_grp_real, tgts_used, shard_num, + allow_version, gen_mode, fdom_lvl, grp_size); if (spare_avail) *spare_avail = true; break; diff --git a/src/placement/tests/jump_map_place_obj.c b/src/placement/tests/jump_map_place_obj.c index 9d1171fd477..35edd396161 100644 --- a/src/placement/tests/jump_map_place_obj.c +++ b/src/placement/tests/jump_map_place_obj.c @@ -1199,10 +1199,10 @@ down_back_to_up_in_same_order(void **state) * healthy targets because of the retry/collision mechanism of the jump * map algorithm. * Due to layout colocation, if the oid has been changed, then it could - * be 2 or even 3 as well, with current oid setting, this is 1. + * be 2 or even 3 as well, with current oid setting, this is 2. */ - assert_int_equal(1, ctx.reint.out_nr); - jtc_assert_rebuild_reint_new(ctx, 1, 0, 1, 1); + assert_int_equal(2, ctx.reint.out_nr); + jtc_assert_rebuild_reint_new(ctx, 2, 0, 2, 2); /* Take second downed target up */ jtc_set_status_on_target(&ctx, UP, orig_shard_targets[1]); @@ -1677,14 +1677,14 @@ placement_handles_multiple_states_with_addition(void **state) rebuilding = jtc_get_layout_rebuild_count(&ctx); /* 1 each for down, up, new ... maybe? */ - assert_true(rebuilding == 2 || rebuilding == 3); + assert_true(rebuilding == 2 || rebuilding == 3 || rebuilding == 4); /* Both DOWN and UP target will be remapped during remap */ - assert_int_equal(ctx.rebuild.out_nr, 2); + assert_int_equal(ctx.rebuild.out_nr, 3); /* Adding new targets might make original shard to be remapped * to new location. */ - assert_true(ctx.reint.out_nr == 1 || ctx.reint.out_nr == 2); + assert_true(ctx.reint.out_nr == 1 || ctx.reint.out_nr == 2 || ctx.reint.out_nr == 3); /* JCH might cause multiple shards remap to the new target */ assert_true(ctx.new.out_nr >= 1); diff --git a/src/placement/tests/place_obj_common.h b/src/placement/tests/place_obj_common.h index 537c02a8de7..537fbe963a2 100644 --- a/src/placement/tests/place_obj_common.h +++ b/src/placement/tests/place_obj_common.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -28,7 +29,7 @@ extern bool g_verbose; #define is_true assert_true #define is_false assert_false -#define PLT_LAYOUT_VERSION 1 +#define PLT_LAYOUT_VERSION 2 extern bool fail_domain_node; void print_layout(struct pl_obj_layout *layout); From 6b98b7f07c121ea8efb3d5b9a41e89e6cc36d98a Mon Sep 17 00:00:00 2001 From: Shilong Wang Date: Mon, 8 Sep 2025 08:58:36 +0000 Subject: [PATCH 2/3] interoperability fix Signed-off-by: Wang Shilong --- src/container/cli.c | 12 +++++++++++- src/container/srv_container.c | 30 ++++++++++++++++++++---------- src/include/daos/pool.h | 10 ++++++++++ src/include/daos_cont.h | 10 +++++++++- src/include/daos_srv/pool.h | 10 ---------- src/pool/srv_layout.c | 2 +- src/pool/srv_pool.c | 8 ++++---- 7 files changed, 55 insertions(+), 27 deletions(-) diff --git a/src/container/cli.c b/src/container/cli.c index 57861372629..6bf4572468b 100644 --- a/src/container/cli.c +++ b/src/container/cli.c @@ -228,6 +228,7 @@ dup_cont_create_props(daos_handle_t poh, daos_prop_t **prop_out, int rc = 0; uid_t uid = geteuid(); gid_t gid = getegid(); + uint32_t co_ver = 0; entries = (prop_in == NULL) ? 0 : prop_in->dpp_nr; @@ -262,6 +263,10 @@ dup_cont_create_props(daos_handle_t poh, daos_prop_t **prop_out, entries++; } + if (!daos_prop_has_entry(prop_in, DAOS_PROP_CO_OBJ_VERSION)) { + co_ver = DAOS_POOL_OBJ_VERSION_2; + entries++; + } if (!daos_prop_has_entry(prop_in, DAOS_PROP_CO_ROOTS)) { uint64_t rf; @@ -332,6 +337,11 @@ dup_cont_create_props(daos_handle_t poh, daos_prop_t **prop_out, roots = NULL; /* prop is responsible for it now */ idx++; } + if (co_ver != 0) { + final_prop->dpp_entries[idx].dpe_type = DAOS_PROP_CO_OBJ_VERSION; + final_prop->dpp_entries[idx].dpe_val = co_ver; + idx++; + } } *prop_out = final_prop; @@ -1037,7 +1047,7 @@ dc_cont_open(tse_task_t *task) if (tpriv->cont == NULL) D_GOTO(err_pool, rc = -DER_NOMEM); uuid_generate(tpriv->cont->dc_cont_hdl); - tpriv->cont->dc_capas = args->flags; + tpriv->cont->dc_capas = args->flags | DAOS_COO_NEW_LAYOUT; } D_DEBUG(DB_MD, DF_UUID ":%s: opening: hdl=" DF_UUIDF " flags=%x\n", DP_UUID(pool->dp_pool), diff --git a/src/container/srv_container.c b/src/container/srv_container.c index ba7237cfb44..a2861b18ef3 100644 --- a/src/container/srv_container.c +++ b/src/container/srv_container.c @@ -710,10 +710,12 @@ cont_create_prop_prepare(struct ds_pool_hdl *pool_hdl, } break; case DAOS_PROP_CO_GLOBAL_VERSION: - case DAOS_PROP_CO_OBJ_VERSION: - D_ERROR("container global/obj %u version could be not set\n", - entry->dpe_type); + D_ERROR("container global %u version could be not set\n", entry->dpe_type); return -DER_INVAL; + case DAOS_PROP_CO_OBJ_VERSION: + /* this is a walkaround for 2.6 only */ + entry_def->dpe_val = entry->dpe_val; + break; default: D_ASSERTF(0, "bad dpt_type %d.\n", entry->dpe_type); break; @@ -768,11 +770,8 @@ cont_create_prop_prepare(struct ds_pool_hdl *pool_hdl, /* inherit object version from pool*/ entry_def = daos_prop_entry_get(prop_def, DAOS_PROP_CO_OBJ_VERSION); - if (entry_def) + if (entry_def && entry_def->dpe_val == 0) entry_def->dpe_val = pool_hdl->sph_obj_ver; - /* disallow old version for container creation */ - if (entry_def->dpe_val < DS_POOL_OBJ_VERSION_2) - entry_def->dpe_val = DS_POOL_OBJ_VERSION_2; /* for new container set HEALTHY status with current pm ver */ entry_def = daos_prop_entry_get(prop_def, DAOS_PROP_CO_STATUS); @@ -2345,6 +2344,17 @@ cont_open(struct rdb_tx *tx, struct ds_pool_hdl *pool_hdl, struct cont *cont, cr D_GOTO(out, rc); D_ASSERT(prop != NULL); D_ASSERT(prop->dpp_nr <= CONT_PROP_NUM); + if (!(flags & DAOS_COO_NEW_LAYOUT)) { + struct daos_prop_entry *entry; + + entry = daos_prop_entry_get(prop, DAOS_PROP_CO_OBJ_VERSION); + if (entry && entry->dpe_val >= DAOS_POOL_OBJ_VERSION_2) { + D_ERROR(DF_CONT ": refusing attempt to open with new layout: " DF_X64 "\n", + DP_CONT(cont->c_svc->cs_pool_uuid, cont->c_uuid), entry->dpe_val); + rc = -DER_NOTSUPPORTED; + D_GOTO(out, rc); + } + } get_cont_prop_access_info(prop, &owner, &acl); @@ -4690,7 +4700,7 @@ upgrade_cont_cb(daos_handle_t ih, d_iov_t *key, d_iov_t *val, void *varg) } if (DAOS_FAIL_CHECK(DAOS_FORCE_OBJ_UPGRADE)) { - obj_ver = DS_POOL_OBJ_VERSION; + obj_ver = DAOS_POOL_OBJ_VERSION; d_iov_set(&value, &obj_ver, sizeof(obj_ver)); rc = rdb_tx_update(ap->tx, &cont->c_prop, &ds_cont_prop_cont_obj_version, &value); @@ -4785,9 +4795,9 @@ upgrade_cont_cb(daos_handle_t ih, d_iov_t *key, d_iov_t *val, void *varg) entry = daos_prop_entry_get(prop, DAOS_PROP_CO_OBJ_VERSION); D_ASSERT(entry != NULL); - entry->dpe_val = DS_POOL_OBJ_VERSION; + entry->dpe_val = DAOS_POOL_OBJ_VERSION; entry->dpe_flags &= ~DAOS_PROP_ENTRY_NOT_SET; - obj_ver = DS_POOL_OBJ_VERSION; + obj_ver = DAOS_POOL_OBJ_VERSION; d_iov_set(&value, &obj_ver, sizeof(obj_ver)); rc = rdb_tx_update(ap->tx, &cont->c_prop, &ds_cont_prop_cont_obj_version, &value); if (rc) { diff --git a/src/include/daos/pool.h b/src/include/daos/pool.h index 57652fde8eb..78c259f9643 100644 --- a/src/include/daos/pool.h +++ b/src/include/daos/pool.h @@ -87,6 +87,16 @@ */ #define DAOS_POOL_GLOBAL_VERSION 4 +/** + * Each individual object layout format, like oid layout, dkey to group, + * dkey to EC group start. + */ +enum { + DAOS_POOL_OBJ_VERSION_1 = 1, + DAOS_POOL_OBJ_VERSION_2 = 2, + DAOS_POOL_OBJ_VERSION = DAOS_POOL_OBJ_VERSION_2, +}; + int dc_pool_init(void); void dc_pool_fini(void); diff --git a/src/include/daos_cont.h b/src/include/daos_cont.h index 632eafe8130..6e6e5b98adc 100644 --- a/src/include/daos_cont.h +++ b/src/include/daos_cont.h @@ -1,5 +1,6 @@ /* * (C) Copyright 2020-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -73,8 +74,15 @@ extern "C" { */ #define DAOS_COO_EVICT_ALL (1U << 6) +/** + * When clients attempt to open a container using the new hash layout, + * the server will reject the request if this flag is not set, + * even if the container is already in the new layout format. + */ +#define DAOS_COO_NEW_LAYOUT (1U << 7) + /** Number of bits in the container open mode flag, DAOS_COO_ bits */ -#define DAOS_COO_NBITS (7) +#define DAOS_COO_NBITS (8) /** Mask for all of the bits in the container open mode flag, DAOS_COO_ bits */ #define DAOS_COO_MASK ((1U << DAOS_COO_NBITS) - 1) diff --git a/src/include/daos_srv/pool.h b/src/include/daos_srv/pool.h index 4c04f0dcc01..7f5235bff5c 100644 --- a/src/include/daos_srv/pool.h +++ b/src/include/daos_srv/pool.h @@ -30,16 +30,6 @@ /* Pool service (opaque) */ struct ds_pool_svc; -/** - * Each individual object layout format, like oid layout, dkey to group, - * dkey to EC group start. - */ -enum { - DS_POOL_OBJ_VERSION_1 = 1, - DS_POOL_OBJ_VERSION_2 = 2, - DS_POOL_OBJ_VERSION = DS_POOL_OBJ_VERSION_2, -}; - /* age of an entry in svc_ops KVS before it may be evicted */ #define DEFAULT_SVC_OPS_ENTRY_AGE_SEC_MAX 300ULL diff --git a/src/pool/srv_layout.c b/src/pool/srv_layout.c index c2fa55590c9..4f5a6731c88 100644 --- a/src/pool/srv_layout.c +++ b/src/pool/srv_layout.c @@ -136,7 +136,7 @@ struct daos_prop_entry pool_prop_entries_default[DAOS_PROP_PO_NUM] = { }, { .dpe_type = DAOS_PROP_PO_OBJ_VERSION, - .dpe_val = DS_POOL_OBJ_VERSION, + .dpe_val = DAOS_POOL_OBJ_VERSION, }, { .dpe_type = DAOS_PROP_PO_PERF_DOMAIN, diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index e7b5a1ceec6..9adb2127191 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -683,7 +683,7 @@ pool_prop_write(struct rdb_tx *tx, const rdb_path_t *kvs, daos_prop_t *prop) rc = rdb_tx_update(tx, kvs, &ds_pool_prop_svc_redun_fac, &value); break; case DAOS_PROP_PO_OBJ_VERSION: - if (entry->dpe_val > DS_POOL_OBJ_VERSION) { + if (entry->dpe_val > DAOS_POOL_OBJ_VERSION) { rc = -DER_INVAL; break; } @@ -6165,7 +6165,7 @@ __ds_pool_mark_upgrade_completed(uuid_t pool_uuid, struct pool_svc *svc, int rc) obj_version = (uint32_t)fail_val; } else { - obj_version = DS_POOL_OBJ_VERSION; + obj_version = DAOS_POOL_OBJ_VERSION; } d_iov_set(&value, &obj_version, sizeof(obj_version)); @@ -6226,9 +6226,9 @@ pool_check_upgrade_object_layout(struct rdb_tx *tx, struct pool_svc *svc, else if (rc == -DER_NONEXIST) current_layout_ver = 0; - if (current_layout_ver < DS_POOL_OBJ_VERSION) { + if (current_layout_ver < DAOS_POOL_OBJ_VERSION) { rc = ds_rebuild_schedule(svc->ps_pool, svc->ps_pool->sp_map_version, upgrade_eph, - DS_POOL_OBJ_VERSION, NULL, RB_OP_UPGRADE, + DAOS_POOL_OBJ_VERSION, NULL, RB_OP_UPGRADE, false /* stop_admin */, 0); if (rc == 0) *scheduled_layout_upgrade = true; From 7d7ea49011c0d0edd7df3c8f78269d9585cfee1d Mon Sep 17 00:00:00 2001 From: Wang Shilong Date: Mon, 22 Sep 2025 11:53:37 +0800 Subject: [PATCH 3/3] Fix to pass correct version in pipeline Signed-off-by: Wang Shilong --- src/pipeline/cli_pipeline.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/pipeline/cli_pipeline.c b/src/pipeline/cli_pipeline.c index b30d8366b79..9d76456f331 100644 --- a/src/pipeline/cli_pipeline.c +++ b/src/pipeline/cli_pipeline.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2021-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -544,7 +545,7 @@ shard_task_sched(tse_task_t *task, void *arg) static int pipeline_create_layout(daos_handle_t coh, struct dc_pool *pool, struct daos_obj_md *obj_md, - struct pl_obj_layout **layout) + struct pl_obj_layout **layout, uint16_t layout_gl_version) { int rc = 0; struct pl_map *map; @@ -555,7 +556,7 @@ pipeline_create_layout(daos_handle_t coh, struct dc_pool *pool, struct daos_obj_ D_GOTO(out, rc = -DER_INVAL); } - rc = pl_obj_place(map, 0, obj_md, 0, NULL, layout); + rc = pl_obj_place(map, layout_gl_version, obj_md, 0, NULL, layout); pl_map_decref(map); if (rc != 0) { D_DEBUG(DB_PL, "Failed to generate object layout\n"); @@ -609,6 +610,7 @@ dc_pipeline_run(tse_task_t *api_task) int total_replicas; int shard; struct pipeline_comp_cb_args comp_cb_args; + uint16_t layout_gl_ver; coh = dc_obj_hdl2cont_hdl(api_args->oh); rc = dc_obj_hdl2obj_md(api_args->oh, &obj_md); @@ -629,7 +631,8 @@ dc_pipeline_run(tse_task_t *api_task) /** get layout */ - rc = pipeline_create_layout(coh, pool, &obj_md, &layout); + layout_gl_ver = dc_obj_hdl2layout_ver(api_args->oh); + rc = pipeline_create_layout(coh, pool, &obj_md, &layout, layout_gl_ver); dc_pool_put(pool); if (rc != 0) D_GOTO(out, rc); @@ -687,7 +690,7 @@ dc_pipeline_run(tse_task_t *api_task) /** object id */ oid.id_pub = obj_md.omd_id; oid.id_shard = shard; - oid.id_layout_ver = dc_obj_hdl2layout_ver(api_args->oh); + oid.id_layout_ver = layout_gl_ver; oid.id_padding = 0; /** queue shard task */