Skip to content

DAOS-19616 dtx: fix race issues for DTX resync #19240

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Nasf-Fan wants to merge 1 commit into
base: master
Choose a base branch
from
Open

DAOS-19616 dtx: fix race issues for DTX resync #19240

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 32 additions & 50 deletions src/dtx/dtx_coll.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,8 @@ dtx_coll_find_rank_range(struct pl_map *map, struct pl_obj_layout *layout, uint3
continue;

/* Skip the target that (re-)joined the system after the DTX. */
if (target->ta_comp.co_ver > version)
if (target->ta_comp.co_status == PO_COMP_ST_UP &&
target->ta_comp.co_in_ver > version)
continue;

/* Skip non-healthy one. */
Expand Down Expand Up @@ -176,61 +177,40 @@ dtx_coll_prep(uuid_t po_uuid, daos_unit_oid_t oid, struct dtx_id *xid, struct dt

dce->dce_bitmap_sz = dct->dct_bitmap_sz;

if (!for_check) {
memcpy(dce->dce_bitmap, dct->dct_tgts + dct->dct_tgt_nr, dct->dct_bitmap_sz);
} else {
map = pl_map_find(po_uuid, oid.id_pub);
if (map == NULL) {
D_ERROR("Failed to find valid placement map in pool "DF_UUID"\n",
DP_UUID(po_uuid));
D_GOTO(out, rc = -DER_INVAL);
}
map = pl_map_find(po_uuid, oid.id_pub);
if (map == NULL) {
D_ERROR("Failed to find valid placement map in pool " DF_UUID "\n",
DP_UUID(po_uuid));
D_GOTO(out, rc = -DER_INVAL);
}

for (i = 0, j = 0; i < dct->dct_tgt_nr; i++) {
rc = pool_map_find_target(map->pl_poolmap, dct->dct_tgts[i], &target);
D_ASSERT(rc == 1);

/* Skip the targets that reside on other engines. */
if (unlikely(target->ta_comp.co_rank != my_rank))
continue;

/* Skip the target that (re-)joined the system after the DTX. */
if (target->ta_comp.co_ver > dtx_ver)
continue;

/* Skip non-healthy one. */
if (target->ta_comp.co_status != PO_COMP_ST_UP &&
target->ta_comp.co_status != PO_COMP_ST_UPIN &&
target->ta_comp.co_status != PO_COMP_ST_DRAIN)
continue;

/* Skip current (new) leader target. */
if (my_tgtid != target->ta_comp.co_index) {
setbit(dce->dce_bitmap, target->ta_comp.co_index);
j++;
}
}
for (i = 0; i < dct->dct_tgt_nr; i++) {
rc = pool_map_find_target(map->pl_poolmap, dct->dct_tgts[i], &target);
D_ASSERT(rc == 1);

rc = 0;
/* Skip the targets that reside on other engines. */
if (unlikely(target->ta_comp.co_rank != my_rank))
continue;

if (unlikely(j == 0)) {
D_FREE(dce->dce_bitmap);
dce->dce_bitmap_sz = 0;
}
/* Skip the target that (re-)joined the system after the DTX. */
if (target->ta_comp.co_status == PO_COMP_ST_UP &&
target->ta_comp.co_in_ver > dtx_ver)
continue;

/* Skip non-healthy one. */
if (target->ta_comp.co_status != PO_COMP_ST_UP &&
target->ta_comp.co_status != PO_COMP_ST_UPIN &&
target->ta_comp.co_status != PO_COMP_ST_DRAIN)
continue;

setbit(dce->dce_bitmap, target->ta_comp.co_index);
}

rc = 0;

if (!need_hint)
goto out;

if (map == NULL) {
map = pl_map_find(po_uuid, oid.id_pub);
if (map == NULL) {
D_ERROR("Failed to find valid placement map in pool "DF_UUID"\n",
DP_UUID(po_uuid));
D_GOTO(out, rc = -DER_INVAL);
}
}

md.omd_id = oid.id_pub;
md.omd_ver = pm_ver;
md.omd_fdom_lvl = dct->dct_fdom_lvl;
Expand Down Expand Up @@ -289,7 +269,8 @@ dtx_coll_prep(uuid_t po_uuid, daos_unit_oid_t oid, struct dtx_id *xid, struct dt
continue;

/* Skip the target that (re-)joined the system after the DTX. */
if (target->ta_comp.co_ver > dtx_ver)
if (target->ta_comp.co_status == PO_COMP_ST_UP &&
target->ta_comp.co_in_ver > dtx_ver)
continue;

/* Skip non-healthy one. */
Expand Down Expand Up @@ -365,7 +346,8 @@ dtx_coll_local_one(void *args)
rc = vos_dtx_abort(cont->sc_hdl, &dcla->dcla_xid, dcla->dcla_epoch, dcla->dcla_ver);
break;
case DTX_COLL_CHECK:
rc = vos_dtx_check(cont->sc_hdl, &dcla->dcla_xid, NULL, NULL, NULL, false);
rc = vos_dtx_check(cont->sc_hdl, &dcla->dcla_xid, NULL, &dcla->dcla_ver, NULL,
DCI_RESYNC);
if (rc == DTX_ST_INITED) {
/*
* For DTX_CHECK, non-ready one is equal to non-exist. Do not directly
Expand Down
6 changes: 3 additions & 3 deletions src/dtx/dtx_common.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1357,9 +1357,9 @@ dtx_leader_end(struct dtx_leader_handle *dlh, struct ds_cont_child *cont, int re
*/
vos_dtx_cleanup(dth, true);
if (dlh->dlh_coll)
dtx_coll_abort(cont, dlh->dlh_coll_entry, dth->dth_epoch);
dtx_coll_abort(cont, dlh->dlh_coll_entry, dth->dth_epoch, dth->dth_ver);
else
dtx_abort(cont, &dth->dth_dte, dth->dth_epoch);
dtx_abort(cont, &dth->dth_dte, dth->dth_epoch, dth->dth_ver);
aborted = true;
}

Expand Down Expand Up @@ -1899,7 +1899,7 @@ dtx_handle_resend(daos_handle_t coh, struct dtx_id *dti,
*/
return -DER_NONEXIST;

rc = vos_dtx_check(coh, dti, epoch, pm_ver, NULL, false);
rc = vos_dtx_check(coh, dti, epoch, pm_ver, NULL, DCI_RESENT);
switch (rc) {
case DTX_ST_INITED:
return -DER_INPROGRESS;
Expand Down
25 changes: 18 additions & 7 deletions src/dtx/dtx_resync.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ dtx_resync_commit(struct ds_cont_child *cont,
* committed or aborted the DTX during we handling other
* DTXs. So double check the status before current commit.
*/
rc = vos_dtx_check(cont->sc_hdl, &dre->dre_xid, NULL, NULL, NULL, false);
rc = vos_dtx_check(cont->sc_hdl, &dre->dre_xid, NULL, NULL, NULL, DCI_DEFAULT);

/*
* Skip this DTX since it has been committed or aggregated.
Expand Down Expand Up @@ -225,6 +225,16 @@ dtx_status_handle_one(struct ds_cont_child *cont, struct dtx_entry *dte, daos_un
struct dtx_coll_entry *dce = NULL;
int rc = 0;

/*
* Current DTX may has been committed or aborted during checking other DTX entries.
* So check it locally before senting out CHECK RPCs to remote targets. Similarly,
* double check locally after CHECK RPC replied.
*/
rc = vos_dtx_check(cont->sc_hdl, &dte->dte_xid, NULL, &cont->sc_pool->spc_map_version, NULL,
DCI_RESYNC);
if (rc == DTX_ST_COMMITTED || rc == DTX_ST_COMMITTABLE || rc == -DER_NONEXIST)
D_GOTO(out, rc = DSHR_IGNORE);

if (mbs->dm_flags & DMF_COLL_TARGET) {
rc = dtx_coll_prep(cont->sc_pool_uuid, oid, &dte->dte_xid, mbs,
dss_get_module_info()->dmi_tgt_id, dte->dte_ver,
Expand Down Expand Up @@ -282,9 +292,9 @@ dtx_status_handle_one(struct ds_cont_child *cont, struct dtx_entry *dte, daos_un
* dtx_abort() with 0 @epoch.
*/
if (mbs->dm_flags & DMF_COLL_TARGET)
rc = dtx_coll_abort(cont, dce, 0);
rc = dtx_coll_abort(cont, dce, 0, 0);
else
rc = dtx_abort(cont, dte, 0);
rc = dtx_abort(cont, dte, 0, 0);
if (rc < 0 && err != NULL)
*err = rc;

Expand All @@ -298,7 +308,7 @@ dtx_status_handle_one(struct ds_cont_child *cont, struct dtx_entry *dte, daos_un
* committed or aborted the DTX during we handling other
* DTXs. So double check the status before next action.
*/
rc = vos_dtx_check(cont->sc_hdl, &dte->dte_xid, NULL, NULL, NULL, false);
rc = vos_dtx_check(cont->sc_hdl, &dte->dte_xid, NULL, NULL, NULL, DCI_DEFAULT);

/* Skip the DTX that may has been committed or aborted. */
if (rc == DTX_ST_COMMITTED || rc == DTX_ST_COMMITTABLE || rc == -DER_NONEXIST)
Expand Down Expand Up @@ -326,9 +336,9 @@ dtx_status_handle_one(struct ds_cont_child *cont, struct dtx_entry *dte, daos_un
* abort the DTXs one by one, not batched.
*/
if (mbs->dm_flags & DMF_COLL_TARGET)
rc = dtx_coll_abort(cont, dce, epoch);
rc = dtx_coll_abort(cont, dce, epoch, cont->sc_pool->spc_map_version);
else
rc = dtx_abort(cont, dte, epoch);
rc = dtx_abort(cont, dte, epoch, cont->sc_pool->spc_map_version);

DL_CDEBUG(rc != 0, DLOG_ERR, DB_TRACE, rc,
"As new leader for DTX " DF_DTI ", abort it (2)", DP_DTI(&dte->dte_xid));
Expand Down Expand Up @@ -393,8 +403,9 @@ dtx_status_handle(struct dtx_resync_args *dra)
again:
d_list_for_each_entry_safe(dre, next, &drh->drh_list, dre_link) {
if (dre->dre_dte.dte_ver < dra->discard_version) {
/* Use the latest known version to abort DTX locally to avoid rejection. */
err = vos_dtx_abort(cont->sc_hdl, &dre->dre_xid, dre->dre_epoch,
dre->dre_dte.dte_ver);
cont->sc_pool->spc_map_version);
if (err == -DER_NONEXIST)
err = 0;
if (err != 0)
Expand Down
Loading
Loading