git: 5603ddb8299d - main - ufshci: detect completions with the completion notification register

The branch main has been updated by jaeyoon:

URL: https://cgit.FreeBSD.org/src/commit/?id=5603ddb8299d0fe3b8e81a5bee5cd1fa24a1806c

commit 5603ddb8299d0fe3b8e81a5bee5cd1fa24a1806c
Author:     Jaeyoon Choi <jaeyoon@FreeBSD.org>
AuthorDate: 2026-11-03 04:11:14 +0000
Commit:     Jaeyoon Choi <jaeyoon@FreeBSD.org>
CommitDate: 2026-11-03 04:29:41 +0000

    ufshci: detect completions with the completion notification register
    
    The single doorbell completion scan took a cleared doorbell bit as a
    completion. A bit that was never set reads the same, so a slot whose
    doorbell was not rung yet, or one that a reset wiped, looked complete
    and was finished with a stale descriptor.
    
    UFSHCI 3.0 added UTRLCNR, which the controller sets when it completes
    a request and the driver already clears afterwards. Use it to find
    completed slots on a 3.0 or later controller. Older controllers keep
    the doorbell check, and the task management list has no such register.
    
    Reviewed by:            imp
    Sponsored by:           Samsung Electronics
    Differential Revision:  https://reviews.freebsd.org/D61111
---
 sys/dev/ufshci/ufshci_private.h   |  6 +++---
 sys/dev/ufshci/ufshci_req_queue.c |  4 ++--
 sys/dev/ufshci/ufshci_req_sdb.c   | 22 +++++++++++++++-------
 3 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/sys/dev/ufshci/ufshci_private.h b/sys/dev/ufshci/ufshci_private.h
index 66cbe4a81bb5..d3a02f45f271 100644
--- a/sys/dev/ufshci/ufshci_private.h
+++ b/sys/dev/ufshci/ufshci_private.h
@@ -133,7 +133,7 @@ struct ufshci_qops {
 	    struct ufshci_tracker **tr);
 	void (*ring_doorbell)(struct ufshci_controller *ctrlr,
 	    struct ufshci_tracker *tr);
-	bool (*is_doorbell_cleared)(struct ufshci_controller *ctrlr,
+	bool (*is_completed)(struct ufshci_controller *ctrlr,
 	    uint8_t slot);
 	void (*clear_cpl_ntf)(struct ufshci_controller *ctrlr,
 	    struct ufshci_tracker *tr);
@@ -532,9 +532,9 @@ void ufshci_req_sdb_utmr_ring_doorbell(struct ufshci_controller *ctrlr,
     struct ufshci_tracker *tr);
 void ufshci_req_sdb_utr_ring_doorbell(struct ufshci_controller *ctrlr,
     struct ufshci_tracker *tr);
-bool ufshci_req_sdb_utmr_is_doorbell_cleared(struct ufshci_controller *ctrlr,
+bool ufshci_req_sdb_utmr_is_completed(struct ufshci_controller *ctrlr,
     uint8_t slot);
-bool ufshci_req_sdb_utr_is_doorbell_cleared(struct ufshci_controller *ctrlr,
+bool ufshci_req_sdb_utr_is_completed(struct ufshci_controller *ctrlr,
     uint8_t slot);
 void ufshci_req_sdb_utmr_clear_cpl_ntf(struct ufshci_controller *ctrlr,
     struct ufshci_tracker *tr);
diff --git a/sys/dev/ufshci/ufshci_req_queue.c b/sys/dev/ufshci/ufshci_req_queue.c
index f1b145f9791b..add399e98ed2 100644
--- a/sys/dev/ufshci/ufshci_req_queue.c
+++ b/sys/dev/ufshci/ufshci_req_queue.c
@@ -28,7 +28,7 @@ static const struct ufshci_qops sdb_utmr_qops = {
 	.reserve_slot = ufshci_req_sdb_reserve_slot,
 	.reserve_admin_slot = ufshci_req_sdb_reserve_slot,
 	.ring_doorbell = ufshci_req_sdb_utmr_ring_doorbell,
-	.is_doorbell_cleared = ufshci_req_sdb_utmr_is_doorbell_cleared,
+	.is_completed = ufshci_req_sdb_utmr_is_completed,
 	.clear_cpl_ntf = ufshci_req_sdb_utmr_clear_cpl_ntf,
 	.process_cpl = ufshci_req_sdb_process_cpl,
 	.get_inflight_io = ufshci_req_sdb_get_inflight_io,
@@ -43,7 +43,7 @@ static const struct ufshci_qops sdb_utr_qops = {
 	.reserve_slot = ufshci_req_sdb_reserve_slot,
 	.reserve_admin_slot = ufshci_req_sdb_reserve_slot,
 	.ring_doorbell = ufshci_req_sdb_utr_ring_doorbell,
-	.is_doorbell_cleared = ufshci_req_sdb_utr_is_doorbell_cleared,
+	.is_completed = ufshci_req_sdb_utr_is_completed,
 	.clear_cpl_ntf = ufshci_req_sdb_utr_clear_cpl_ntf,
 	.process_cpl = ufshci_req_sdb_process_cpl,
 	.get_inflight_io = ufshci_req_sdb_get_inflight_io,
diff --git a/sys/dev/ufshci/ufshci_req_sdb.c b/sys/dev/ufshci/ufshci_req_sdb.c
index b737a7081fd9..2404354bfade 100644
--- a/sys/dev/ufshci/ufshci_req_sdb.c
+++ b/sys/dev/ufshci/ufshci_req_sdb.c
@@ -551,21 +551,31 @@ ufshci_req_sdb_utr_ring_doorbell(struct ufshci_controller *ctrlr,
 }
 
 bool
-ufshci_req_sdb_utmr_is_doorbell_cleared(struct ufshci_controller *ctrlr,
+ufshci_req_sdb_utmr_is_completed(struct ufshci_controller *ctrlr,
     uint8_t slot)
 {
 	uint32_t utmrldbr;
 
+	/* The task management list has no completion notification. */
 	utmrldbr = ufshci_mmio_read_4(ctrlr, utmrldbr);
 	return (!(utmrldbr & (1 << slot)));
 }
 
 bool
-ufshci_req_sdb_utr_is_doorbell_cleared(struct ufshci_controller *ctrlr,
+ufshci_req_sdb_utr_is_completed(struct ufshci_controller *ctrlr,
     uint8_t slot)
 {
-	uint32_t utrldbr;
+	uint32_t utrlcnr, utrldbr;
 
+	/*
+	 * Since UFSHCI 3.0 the controller sets a notification bit when it
+	 * completes a request. Before that, a cleared doorbell bit was the
+	 * only sign, and a slot that was never rung looks the same.
+	 */
+	if (ctrlr->major_version >= 3) {
+		utrlcnr = ufshci_mmio_read_4(ctrlr, utrlcnr);
+		return ((utrlcnr & (1 << slot)) != 0);
+	}
 	utrldbr = ufshci_mmio_read_4(ctrlr, utrldbr);
 	return (!(utrldbr & (1 << slot)));
 }
@@ -592,15 +602,13 @@ ufshci_req_sdb_process_cpl(struct ufshci_req_queue *req_queue)
 
 		KASSERT(tr, ("there is no tracker assigned to the slot"));
 		/*
-		 * When the response is delivered from the device, the doorbell
-		 * is cleared. Check it under qlock so that a slot whose
+		 * Check the completion under qlock so that a slot whose
 		 * doorbell write is still in flight in the submit path is not
 		 * mistaken for a completed one.
 		 */
 		mtx_lock(&hwq->qlock);
 		completed = tr->slot_state == UFSHCI_SLOT_STATE_SCHEDULED &&
-		    req_queue->qops.is_doorbell_cleared(req_queue->ctrlr,
-			slot);
+		    req_queue->qops.is_completed(req_queue->ctrlr, slot);
 		mtx_unlock(&hwq->qlock);
 		if (completed) {
 			ufshci_req_queue_complete_tracker(tr);