Skip to content

DAOS-17754 bio: store ctrlr data in SMD #16886

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
NiuYawei merged 3 commits into from
Oct 11, 2025
Merged

DAOS-17754 bio: store ctrlr data in SMD #16886

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f25c946
DAOS-17754 bio: store ctrlr data in SMD
NiuYawei Sep 19, 2025
43fb08a
DAOS-17754 bio: fix typo in set_ctrlr_data()
NiuYawei Oct 9, 2025
9289b46
Merge remote-tracking branch 'origin/master' into niu/DAOS-17754
NiuYawei Oct 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 103 additions & 59 deletions src/bio/bio_device.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ replace_dev(struct bio_xs_context *xs_ctxt, struct smd_dev_info *old_info,
struct bio_bdev *old_dev, struct bio_bdev *new_dev)
{
struct bio_blobstore *bbs = old_dev->bb_blobstore;
struct bio_dev_info *b_info;
unsigned int old_roles;
int rc;

Expand All @@ -86,14 +87,23 @@ replace_dev(struct bio_xs_context *xs_ctxt, struct smd_dev_info *old_info,
/* Avoid re-enter or being destroyed by hot remove callback */
new_dev->bb_replacing = 1;

b_info = alloc_dev_info(new_dev->bb_uuid, new_dev, NULL);
if (b_info == NULL) {
D_ERROR("Failed to alloc bio_dev_info.");
rc = -DER_NOMEM;
goto out;
}

old_roles = bio_nvme_configured(SMD_DEV_TYPE_META) ? old_dev->bb_roles : NVME_ROLE_DATA;
/* Replace old device with new device in SMD */
rc = smd_dev_replace(old_dev->bb_uuid, new_dev->bb_uuid, old_roles);
rc = smd_dev_replace(old_dev->bb_uuid, new_dev->bb_uuid, old_roles, b_info->bdi_ctrlr);
if (rc) {
DL_ERROR(rc, "Failed to replace dev: "DF_UUID" -> "DF_UUID". roles(%u)",
DP_UUID(old_dev->bb_uuid), DP_UUID(new_dev->bb_uuid), old_roles);
bio_free_dev_info(b_info);
goto out;
}
bio_free_dev_info(b_info);

/* Replace in-memory bio_bdev */
replace_bio_bdev(old_dev, new_dev);
Expand Down Expand Up @@ -331,40 +341,6 @@ fill_in_traddr(struct bio_dev_info *b_info, char *dev_name)
return rc;
}

static struct bio_dev_info *
alloc_dev_info(uuid_t dev_id, char *dev_name, struct smd_dev_info *s_info)
{
struct bio_dev_info *info;
int tgt_cnt = 0, i;

D_ALLOC_PTR(info);
if (info == NULL)
return NULL;

if (s_info != NULL) {
tgt_cnt = s_info->sdi_tgt_cnt;
info->bdi_flags |= NVME_DEV_FL_INUSE;
if (s_info->sdi_state == SMD_DEV_FAULTY)
info->bdi_flags |= NVME_DEV_FL_FAULTY;
}

if (tgt_cnt != 0) {
D_ALLOC_ARRAY(info->bdi_tgts, tgt_cnt);
if (info->bdi_tgts == NULL) {
bio_free_dev_info(info);
return NULL;
}
}

D_INIT_LIST_HEAD(&info->bdi_link);
uuid_copy(info->bdi_dev_id, dev_id);
info->bdi_tgt_cnt = tgt_cnt;
for (i = 0; i < info->bdi_tgt_cnt; i++)
info->bdi_tgts[i] = s_info->sdi_tgts[i];

return info;
}

static struct smd_dev_info *
find_smd_dev(uuid_t dev_id, d_list_t *s_dev_list)
{
Expand Down Expand Up @@ -461,7 +437,7 @@ fetch_pci_dev_info(struct nvme_ctrlr_t *w_ctrlr, const char *tr_addr)
}

static int
alloc_ctrlr_info(uuid_t dev_id, char *dev_name, struct bio_dev_info *b_info)
alloc_ctrlr_info(char *dev_name, struct bio_dev_info *b_info)
{
struct spdk_bdev *bdev;
uint32_t blk_sz;
Expand All @@ -470,14 +446,7 @@ alloc_ctrlr_info(uuid_t dev_id, char *dev_name, struct bio_dev_info *b_info)

D_ASSERT(b_info != NULL);
D_ASSERT(b_info->bdi_ctrlr == NULL);

if (dev_name == NULL) {
D_DEBUG(DB_MGMT,
"missing bdev device name for device " DF_UUID ", skipping ctrlr "
"info fetch\n",
DP_UUID(dev_id));
return 0;
}
D_ASSERT(dev_name != NULL);

bdev = spdk_bdev_get_by_name(dev_name);
if (bdev == NULL) {
Expand Down Expand Up @@ -516,6 +485,95 @@ alloc_ctrlr_info(uuid_t dev_id, char *dev_name, struct bio_dev_info *b_info)
return fetch_pci_dev_info(b_info->bdi_ctrlr, b_info->bdi_traddr);
}

static int
alloc_ctrlr_from_smd(struct bio_dev_info *b_info, struct smd_dev_info *s_info)
{
D_ASSERT(b_info != NULL);
D_ASSERT(b_info->bdi_ctrlr == NULL);

if (s_info->sdi_model == NULL && s_info->sdi_serial == NULL)
return 0;

D_ALLOC_PTR(b_info->bdi_ctrlr);
if (b_info->bdi_ctrlr == NULL)
return -DER_NOMEM;

if (s_info->sdi_model) {
D_STRNDUP(b_info->bdi_ctrlr->model, s_info->sdi_model, strlen(s_info->sdi_model));
if (b_info->bdi_ctrlr->model == NULL)
return -DER_NOMEM;
}

if (s_info->sdi_serial) {
D_STRNDUP(b_info->bdi_ctrlr->serial, s_info->sdi_serial,
strlen(s_info->sdi_serial));
if (b_info->bdi_ctrlr->serial == NULL)
return -DER_NOMEM;
}

return 0;
}

struct bio_dev_info *
alloc_dev_info(uuid_t dev_id, struct bio_bdev *d_bdev, struct smd_dev_info *s_info)
{
struct bio_dev_info *info;
int tgt_cnt = 0, i, rc;

D_ALLOC_PTR(info);
if (info == NULL)
return NULL;

if (s_info != NULL) {
tgt_cnt = s_info->sdi_tgt_cnt;
info->bdi_flags |= NVME_DEV_FL_INUSE;
if (s_info->sdi_state == SMD_DEV_FAULTY)
info->bdi_flags |= NVME_DEV_FL_FAULTY;

/* When the device isn't plugged, get ctrlr data from SMD */
if (d_bdev == NULL || d_bdev->bb_removed) {
rc = alloc_ctrlr_from_smd(info, s_info);
if (rc) {
DL_ERROR(rc, "Failed to get ctrlr data from SMD.");
bio_free_dev_info(info);
return NULL;
}
}
}

if (tgt_cnt != 0) {
D_ALLOC_ARRAY(info->bdi_tgts, tgt_cnt);
if (info->bdi_tgts == NULL) {
bio_free_dev_info(info);
return NULL;
}
}

if (d_bdev != NULL) {
info->bdi_dev_roles = d_bdev->bb_roles;
if (!d_bdev->bb_removed) {
info->bdi_flags |= NVME_DEV_FL_PLUGGED;
/* Get ctrlr data when the SSD is plugged */
rc = alloc_ctrlr_info(d_bdev->bb_name, info);
if (rc) {
DL_ERROR(rc, "Failed to get ctrlr details");
bio_free_dev_info(info);
return NULL;
}
}
if (d_bdev->bb_faulty)
info->bdi_flags |= NVME_DEV_FL_FAULTY;
}

D_INIT_LIST_HEAD(&info->bdi_link);
uuid_copy(info->bdi_dev_id, dev_id);
info->bdi_tgt_cnt = tgt_cnt;
for (i = 0; i < info->bdi_tgt_cnt; i++)
info->bdi_tgts[i] = s_info->sdi_tgts[i];

return info;
}

int
bio_dev_list(struct bio_xs_context *xs_ctxt, d_list_t *dev_list, int *dev_cnt)
{
Expand All @@ -541,28 +599,14 @@ bio_dev_list(struct bio_xs_context *xs_ctxt, d_list_t *dev_list, int *dev_cnt)

/* Scan all devices present in bio_bdev list */
d_list_for_each_entry(d_bdev, bio_bdev_list(), bb_link) {
char *dev_name = d_bdev->bb_removed ? NULL : d_bdev->bb_name;

s_info = find_smd_dev(d_bdev->bb_uuid, &s_dev_list);

b_info = alloc_dev_info(d_bdev->bb_uuid, dev_name, s_info);
b_info = alloc_dev_info(d_bdev->bb_uuid, d_bdev, s_info);
if (b_info == NULL) {
D_ERROR("Failed to allocate device info\n");
rc = -DER_NOMEM;
goto out;
}
b_info->bdi_dev_roles = d_bdev->bb_roles;
if (!d_bdev->bb_removed)
b_info->bdi_flags |= NVME_DEV_FL_PLUGGED;
if (d_bdev->bb_faulty)
b_info->bdi_flags |= NVME_DEV_FL_FAULTY;

rc = alloc_ctrlr_info(d_bdev->bb_uuid, dev_name, b_info);
if (rc) {
DL_ERROR(rc, "Failed to get ctrlr details");
bio_free_dev_info(b_info);
goto out;
}

d_list_add_tail(&b_info->bdi_link, dev_list);
(*dev_cnt)++;
Expand Down
2 changes: 2 additions & 0 deletions src/bio/bio_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -656,6 +656,8 @@ void trigger_faulty_reaction(struct bio_blobstore *bbs);

/* bio_device.c */
int fill_in_traddr(struct bio_dev_info *b_info, char *dev_name);
struct bio_dev_info *
alloc_dev_info(uuid_t dev_id, struct bio_bdev *d_bdev, struct smd_dev_info *s_info);

/* bio_config.c */
int
Expand Down
21 changes: 17 additions & 4 deletions src/bio/bio_xstream.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1250,14 +1250,21 @@ static int
assign_roles(struct bio_bdev *d_bdev, unsigned int tgt_id)
{
enum smd_dev_type st, failed_st;
struct bio_dev_info *b_info;
bool assigned = false;
int rc;

b_info = alloc_dev_info(d_bdev->bb_uuid, d_bdev, NULL);
if (b_info == NULL) {
D_ERROR("Failed to alloc bio_dev_info.");
return -DER_NOMEM;
}

for (st = SMD_DEV_TYPE_DATA; st < SMD_DEV_TYPE_MAX; st++) {
if (!is_role_match(d_bdev->bb_roles, smd_dev_type2role(st)))
continue;

rc = smd_dev_add_tgt(d_bdev->bb_uuid, tgt_id, st);
rc = smd_dev_add_tgt(d_bdev->bb_uuid, tgt_id, st, b_info->bdi_ctrlr);
if (rc) {
D_ERROR("Failed to map dev "DF_UUID" type:%u to tgt %d. "DF_RC"\n",
DP_UUID(d_bdev->bb_uuid), st, tgt_id, DP_RC(rc));
Expand Down Expand Up @@ -1292,8 +1299,11 @@ assign_roles(struct bio_bdev *d_bdev, unsigned int tgt_id)
break;
}

bio_free_dev_info(b_info);

return assigned ? 0 : -DER_INVAL;
error:
bio_free_dev_info(b_info);
for (st = SMD_DEV_TYPE_DATA; st < failed_st; st++) {
if (!is_role_match(d_bdev->bb_roles, smd_dev_type2role(st)))
continue;
Expand Down Expand Up @@ -1346,9 +1356,12 @@ assign_xs_bdev(struct bio_xs_context *ctxt, int tgt_id, enum smd_dev_type st,
*/
d_bdev = lookup_dev_by_id(dev_info->sdi_id);
if (d_bdev == NULL)
D_ERROR("Device "DF_UUID" for target %d type %d isn't plugged or the "
"SMD table is stale/corrupted.\n",
DP_UUID(dev_info->sdi_id), tgt_id, st);
ras_notify_eventf(RAS_DEVICE_UNPLUGGED, RAS_TYPE_INFO, RAS_SEV_ERROR, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL,
"Device: " DF_UUID " "
"[model:%s, serial:%s] for target:%d type:%d is unplugged\n",
DP_UUID(dev_info->sdi_id), dev_info->sdi_model,
dev_info->sdi_serial, tgt_id, st);

@wangshilong wangshilong Sep 25, 2025

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will it cause segfault if sdi_model or sdi_serial is NULL? or we gurante this two values are not NULL.

@NiuYawei NiuYawei Sep 25, 2025

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NULL string can be handled by printf, right?

@wangshilong wangshilong Sep 25, 2025

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are right, sorry for noise.

smd_dev_free_info(dev_info);

return d_bdev;
Expand Down
Loading
Loading