git: dd6123ebf02b - main - cam/iosched: Log outlier latency events

From: Warner Losh <imp_at_FreeBSD.org>
Date: Sat, 20 Jul 2024 02:59:09 UTC
The branch main has been updated by imp:

URL: https://cgit.FreeBSD.org/src/commit/?id=dd6123ebf02b86cee90b75e54fc755f6c20caebf

commit dd6123ebf02b86cee90b75e54fc755f6c20caebf
Author:     Warner Losh <imp@FreeBSD.org>
AuthorDate: 2024-07-20 02:53:08 +0000
Commit:     Warner Losh <imp@FreeBSD.org>
CommitDate: 2024-07-20 02:53:37 +0000

    cam/iosched: Log outlier latency events
    
    Log outlier latency events to devd. In addition to counting, this will
    allow analysis of whether the problem is confined to a specific block
    range, or if it's a more general problem.
    
    Sponsored by:           Netflix
    Reviewed by:            jhb
    Differential Revision:  https://reviews.freebsd.org/D46036
---
 sys/cam/cam_iosched.c | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/sys/cam/cam_iosched.c b/sys/cam/cam_iosched.c
index cfefc8098d18..28b5375ae246 100644
--- a/sys/cam/cam_iosched.c
+++ b/sys/cam/cam_iosched.c
@@ -1813,7 +1813,6 @@ cam_iosched_bio_complete(struct cam_iosched_softc *isc, struct bio *bp,
 		if (isc->latfcn && isc->max_lat != 0 && sim_latency > isc->max_lat)
 			isc->latfcn(isc->latarg, sim_latency, bp);
 	}
-		
 #endif
 	return retval;
 }
@@ -1926,9 +1925,35 @@ static sbintime_t latencies[LAT_BUCKETS - 1] = {
 	BUCKET_BASE << 18	/* 5,242,880us */
 };
 
+#define CAM_IOSCHED_DEVD_MSG_SIZE	256
+
+static void
+cam_iosched_devctl_outlier(struct iop_stats *iop, sbintime_t sim_latency,
+    const struct bio *bp)
+{
+	daddr_t lba = bp->bio_pblkno;
+	daddr_t cnt = bp->bio_bcount / iop->softc->disk->d_sectorsize;
+	char *sbmsg;
+	struct sbuf sb;
+
+	sbmsg = malloc(CAM_IOSCHED_DEVD_MSG_SIZE, M_CAMSCHED, M_NOWAIT);
+	if (sbmsg == NULL)
+		return;
+	sbuf_new(&sb, sbmsg, CAM_IOSCHED_DEVD_MSG_SIZE, SBUF_FIXEDLEN);
+
+	sbuf_printf(&sb, "device=%s%d lba=%jd blocks=%jd latency=%jd",
+	    iop->softc->periph->periph_name,
+	    iop->softc->periph->unit_number,
+	    lba, cnt, sbttons(sim_latency));
+	if (sbuf_finish(&sb) == 0)
+		devctl_notify("CAM", "iosched", "latency", sbuf_data(&sb));
+	sbuf_delete(&sb);
+	free(sbmsg, M_CAMSCHED);
+}
+
 static void
 cam_iosched_update(struct iop_stats *iop, sbintime_t sim_latency,
-    const struct bio *bp __unused)
+    const struct bio *bp)
 {
 	sbintime_t y, deltasq, delta;
 	int i;
@@ -1938,6 +1963,7 @@ cam_iosched_update(struct iop_stats *iop, sbintime_t sim_latency,
 	 * configured threshold.
 	 */
 	if (sim_latency > iop->bad_latency) {
+		cam_iosched_devctl_outlier(iop, sim_latency, bp);
 		iop->too_long++;
 	}