git: 701d776c98e0 - main - mpi3mr: poll reply queue and add MPI3MR_DEV_REMOVE_HS_COMPLETED flag

The branch main has been updated by ssaxena:

URL: https://cgit.FreeBSD.org/src/commit/?id=701d776c98e015160c2d0e3f4fdaf980d1d5dc83

commit 701d776c98e015160c2d0e3f4fdaf980d1d5dc83
Author:     Chandrakanth patil <chandrakanth.patil@broadcom.com>
AuthorDate: 2024-03-14 17:47:06 +0000
Commit:     Sumit Saxena <ssaxena@FreeBSD.org>
CommitDate: 2024-06-06 10:39:14 +0000

    mpi3mr: poll reply queue and add MPI3MR_DEV_REMOVE_HS_COMPLETED flag
    
    An outstanding IO counter per target check has been added before deleting
    the target from the OS which will poll the reply queue if there are any
    outstanding IOs are found.
    
    A new flag, named "MPI3MR_DEV_REMOVE_HS_COMPLETED," is added. If a remove event
    for a target occurs and before the deletion of the target resource if the add event
    for another target arrives reusing the same target ID then this flag will prevent
    the removal of the target reference. This flag ensures synchronization between the interrupt
    top and bottom half during target removal and addition events.
    
    Reviewed by:            imp
    Approved by:            imp
    Differential revision:  https://reviews.freebsd.org/D44423
---
 sys/dev/mpi3mr/mpi3mr.c     | 19 ++++++++-----------
 sys/dev/mpi3mr/mpi3mr.h     |  3 +--
 sys/dev/mpi3mr/mpi3mr_cam.c | 27 ++++++++++++++++++---------
 3 files changed, 27 insertions(+), 22 deletions(-)

diff --git a/sys/dev/mpi3mr/mpi3mr.c b/sys/dev/mpi3mr/mpi3mr.c
index faa8b52bbb50..72abe76ad085 100644
--- a/sys/dev/mpi3mr/mpi3mr.c
+++ b/sys/dev/mpi3mr/mpi3mr.c
@@ -3477,6 +3477,7 @@ static void mpi3mr_dev_rmhs_complete_iou(struct mpi3mr_softc *sc,
 {
 	U16 cmd_idx = drv_cmd->host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
 	struct delayed_dev_rmhs_node *delayed_dev_rmhs = NULL;
+	struct mpi3mr_target *tgtdev = NULL;
 
 	mpi3mr_dprint(sc, MPI3MR_EVENT,
 	    "%s :dev_rmhs_iouctrl_complete:handle(0x%04x), ioc_status(0x%04x), loginfo(0x%08x)\n",
@@ -3497,6 +3498,13 @@ static void mpi3mr_dev_rmhs_complete_iou(struct mpi3mr_softc *sc,
 		    "%s :dev removal handshake failed after all retries: handle(0x%04x)\n",
 		    __func__, drv_cmd->dev_handle);
 	} else {
+		mtx_lock_spin(&sc->target_lock);
+		TAILQ_FOREACH(tgtdev, &sc->cam_sc->tgt_list, tgt_next) {
+		       if (tgtdev->dev_handle == drv_cmd->dev_handle)
+			       tgtdev->state = MPI3MR_DEV_REMOVE_HS_COMPLETED;
+		}
+		mtx_unlock_spin(&sc->target_lock);
+
 		mpi3mr_dprint(sc, MPI3MR_INFO,
 		    "%s :dev removal handshake completed successfully: handle(0x%04x)\n",
 		    __func__, drv_cmd->dev_handle);
@@ -3604,18 +3612,7 @@ static void mpi3mr_dev_rmhs_send_tm(struct mpi3mr_softc *sc, U16 handle,
 	U8 retrycount = 5;
 	struct mpi3mr_drvr_cmd *drv_cmd = cmdparam;
 	struct delayed_dev_rmhs_node *delayed_dev_rmhs = NULL;
-	struct mpi3mr_target *tgtdev = NULL;
 	
-	mtx_lock_spin(&sc->target_lock);
-	TAILQ_FOREACH(tgtdev, &sc->cam_sc->tgt_list, tgt_next) {
-		if ((tgtdev->dev_handle == handle) &&
-		    (iou_rc == MPI3_CTRL_OP_REMOVE_DEVICE)) {
-			tgtdev->state = MPI3MR_DEV_REMOVE_HS_STARTED;
-			break;
-		}
-	}
-	mtx_unlock_spin(&sc->target_lock);
-
 	if (drv_cmd)
 		goto issue_cmd;
 	do {
diff --git a/sys/dev/mpi3mr/mpi3mr.h b/sys/dev/mpi3mr/mpi3mr.h
index d93c53b286cb..1cf6718d9047 100644
--- a/sys/dev/mpi3mr/mpi3mr.h
+++ b/sys/dev/mpi3mr/mpi3mr.h
@@ -446,8 +446,7 @@ enum mpi3mr_cmd_state {
 
 enum mpi3mr_target_state {
 	MPI3MR_DEV_CREATED = 1,
-	MPI3MR_DEV_REMOVE_HS_STARTED = 2,
-	MPI3MR_DEV_DELETED = 3,
+	MPI3MR_DEV_REMOVE_HS_COMPLETED = 2,
 };
 
 struct mpi3mr_cmd {
diff --git a/sys/dev/mpi3mr/mpi3mr_cam.c b/sys/dev/mpi3mr/mpi3mr_cam.c
index b842e2a05bda..b8e4e350dfa1 100644
--- a/sys/dev/mpi3mr/mpi3mr_cam.c
+++ b/sys/dev/mpi3mr/mpi3mr_cam.c
@@ -1822,6 +1822,17 @@ int mpi3mr_remove_device_from_os(struct mpi3mr_softc *sc, U16 handle)
 
 	target->flags |= MPI3MRSAS_TARGET_INREMOVAL;
 
+	if (mpi3mr_atomic_read(&target->outstanding)) {
+		mpi3mr_dprint(sc, MPI3MR_ERROR, "there are [%2d] outstanding IOs on target: %d"
+			      "Poll reply queue once\n", mpi3mr_atomic_read(&target->outstanding),
+			      target->per_id);
+		mpi3mr_poll_pend_io_completions(sc);
+		if (mpi3mr_atomic_read(&target->outstanding))
+			mpi3mr_dprint(sc, MPI3MR_ERROR, "[%2d] outstanding IOs present on target: %d"
+				      "despite poll\n", mpi3mr_atomic_read(&target->outstanding),
+				      target->per_id);
+	}
+
 	while (mpi3mr_atomic_read(&target->outstanding) && (i < 30)) {
 		i++;
 		if (!(i % 2)) {
@@ -1848,18 +1859,16 @@ out:
 void mpi3mr_remove_device_from_list(struct mpi3mr_softc *sc,
 	struct mpi3mr_target *target, bool must_delete)
 {
+	if ((must_delete == false) &&
+	    (target->state != MPI3MR_DEV_REMOVE_HS_COMPLETED))
+		return;
+
 	mtx_lock_spin(&sc->target_lock);
-	if ((target->state == MPI3MR_DEV_REMOVE_HS_STARTED) ||
-	    (must_delete == true)) {
-		TAILQ_REMOVE(&sc->cam_sc->tgt_list, target, tgt_next);
-		target->state = MPI3MR_DEV_DELETED;
-	}
+	TAILQ_REMOVE(&sc->cam_sc->tgt_list, target, tgt_next);
 	mtx_unlock_spin(&sc->target_lock);
 
-	if (target->state == MPI3MR_DEV_DELETED) {
- 		free(target, M_MPI3MR);
- 		target = NULL;
- 	}
+	free(target, M_MPI3MR);
+	target = NULL;
 	
 	return;
 }