git: c4c790949fbc - main - dtrace: Instrument more of the CAM device lifecycle

Go to: [ bottom of page ] [ top of archives ] [ this month ]
From: Warner Losh <imp_at_FreeBSD.org>
Date: Fri, 11 Sep 2026 05:32:41 UTC
The branch main has been updated by imp:

URL: https://cgit.FreeBSD.org/src/commit/?id=c4c790949fbcba9f8425351896e3b3047eaae0b8

commit c4c790949fbcba9f8425351896e3b3047eaae0b8
Author:     Warner Losh <imp@FreeBSD.org>
AuthorDate: 2026-09-11 05:13:30 +0000
Commit:     Warner Losh <imp@FreeBSD.org>
CommitDate: 2026-09-11 05:13:30 +0000

    dtrace: Instrument more of the CAM device lifecycle
    
    cam::periph:error, cam::{,a,n,sd}da:error
            Trace the error evolution for I/O errors the periph encounters
    
    cam::periph:recovery, cam::{,a,n,sd}da:recovery
            Trace the result of the error sorting
    
    cam::periph:invalidate
            Trace device invalidations
    
    cam::periph:hold-boot
    cam::periph:release-boot
    cam::xpt:hold-boot
    cam::xpt:release-boot
            Trace boot holds and releases at different levels
    
    cam::xpt:bus-register
            Trace new busses added
    
    These additions allow fairly complete instrumentation of the discovery
    process at boot through dtrace's anonymous tracing. It would be mildly
    better to have the boot holds have the periph, but dtrace sbts have to
    be static at compile time.
    
    Sponsored by:           Netflix
    Reviewed by:            dteske
    Differential Revision:  https://reviews.freebsd.org/D58523
---
 share/man/man4/dtrace_cam.4 | 140 ++++++++++++++++++++++++++++++++++++++++++--
 sys/cam/ata/ata_da.c        |  12 +++-
 sys/cam/cam.h               |   5 ++
 sys/cam/cam_periph.c        |  14 +++++
 sys/cam/cam_xpt.c           |  11 +++-
 sys/cam/mmc/mmc_da.c        |  12 +++-
 sys/cam/nvme/nvme_da.c      |  12 +++-
 sys/cam/scsi/scsi_da.c      |  14 ++++-
 8 files changed, 210 insertions(+), 10 deletions(-)

diff --git a/share/man/man4/dtrace_cam.4 b/share/man/man4/dtrace_cam.4
index 8e2790e72d22..5a117047b777 100644
--- a/share/man/man4/dtrace_cam.4
+++ b/share/man/man4/dtrace_cam.4
@@ -2,22 +2,38 @@
 .\"
 .\" SPDX-License-Identifier: BSD-2-Clause
 .\"
-.Dd May 7, 2026
+.Dd July 29, 2026
 .Dt DTRACE_CAM 4
 .Os
 .Sh NAME
 .Nm dtrace_cam
 .Nd a DTrace provider for tracing events related to CAM
 .Sh SYNOPSIS
-.Fn cam::xpt:action "union ccb *ccn"
+.Fn cam::xpt:action "union ccb *ccb"
 .Fn cam::xpt:done "union ccb *ccb"
-.Fn cam::xpt:async-cb "void *cbarg" "uint32_t async_code" "struct cam_path *path" "void *async_Arg"
+a.Fn cam::xpt:async-cb "void *cbarg" "uint32_t async_code" "struct cam_path *path" "void *async_arg"
+.Fn cam::periph:error "union ccb *ccb" "cam_flags camflags" "uint32_t sense_flags"
+.Fn cam::da:error "union ccb *ccb" "uint32_t cam_flags" "uint32_t sense_flags"
+.Fn cam::ada:error "union ccb *ccb" "uint32_t cam_flags" "uint32_t sense_flags"
+.Fn cam::nda:error "union ccb *ccb" "uint32_t cam_flags" "uint32_t sense_flags"
+.Fn cam::sdda:error "union ccb *ccb" "uint32_t cam_flags" "uint32_t sense_flags"
+.Fn cam::periph:recovery "union ccb *ccb" "int error"
+.Fn cam::da:recovery "union ccb *ccb" "int error"
+.Fn cam::ada:recovery "union ccb *ccb" "int error"
+.Fn cam::nda:recovery "union ccb *ccb" "int error"
+.Fn cam::sdda:recovery "union ccb *ccb" "int error"
+.Fn cam::periph:invalidate "struct cam_periph *periph"
+.Fn cam::periph:hold-boot "struct cam_periph *periph"
+.Fn cam::periph:release-boot "struct cam_periph *periph"
+.Fn cam::xpt:bus-register "struct cam_sim *sim" "path_id_t path_id"
+.Fn cam::xpt:hold-boot "int buses_to_config"
+.Fn cam::xpt:release-boot "int buses_to_config"
 .Sh DESCRIPTION
 The
 .Nm cam
 provider allows the tracing of CAM events.
 The
-.Fn cam::xpt_action
+.Fn cam::xpt:action
 probe fires when a CAM Control Block (ccb) is submitted to a CAM SIM driver.
 The
 .Fn cam::xpt:done
@@ -25,6 +41,122 @@ probe fires when that request completes.
 The
 .Fn cam::xpt:async-cb
 probe fires just before an async callback is called.
+The
+.Fn cam::periph:error
+probe fires on entry to
+.Fn cam_periph_error ,
+the generic CAM peripheral driver error handler, before any error
+recovery or retry decisions are made.
+The
+.Fn cam::da:error ,
+.Fn cam::ada:error ,
+.Fn cam::nda:error ,
+and
+.Fn cam::sdda:error
+probes fire on entry to the
+.Xr da 4 ,
+.Xr ada 4 ,
+.Xr nda 4 ,
+and sdda
+.Pq Xr mmc 4
+peripheral drivers' own error handlers, before they call
+.Fn cam_periph_error
+or otherwise process the failing ccb.
+.Pp
+The
+.Fn cam::periph:recovery
+probe fires once
+.Fn cam_periph_error
+has decided how to dispose of the failing ccb, reporting that decision
+as
+.Fa error :
+0 means the ccb is being treated as successful,
+.Dv ERESTART
+means the request is being retried, and any other value is the
+.Xr errno 2
+the request is failing with.
+Of those,
+.Er EIO
+means only this request is being failed,
+while
+.Er ENXIO
+means the device itself is now being treated as gone
+(a selection timeout, or an error seen after the peripheral was
+already invalidated).
+Other errno values can appear as well: CAM_SCSI_STATUS_ERROR handling
+maps specific SCSI sense key/ASC/ASCQ combinations to whatever errno
+is most appropriate
+.Pq Er EBUSY , Er ENOSPC , Er EINVAL , No and others
+via the table in
+.Pa sys/cam/scsi/scsi_all.c .
+The
+.Fn cam::da:recovery ,
+.Fn cam::ada:recovery ,
+.Fn cam::nda:recovery ,
+and
+.Fn cam::sdda:recovery
+probes report the same
+.Fa error
+decision from the corresponding peripheral driver's own error handler,
+which may resolve the error itself (for example
+.Xr da 4 Ns 's
+automatic READ/WRITE(6)-to-(10) upgrade) before ever calling
+.Fn cam_periph_error .
+The
+.Fn cam::periph:invalidate
+probe fires from
+.Fn cam_periph_invalidate
+when a peripheral is actually torn down as a result of an
+unrecoverable error, regardless of which peripheral driver or code
+path triggered it.
+.Pp
+The
+.Fn cam::periph:hold-boot
+and
+.Fn cam::periph:release-boot
+probes fire from
+.Fn cam_periph_hold_boot
+and
+.Fn cam_periph_release_boot ,
+reporting the
+.Vt cam_periph
+taking or releasing its own, per-peripheral root-mount hold, which is
+keyed by
+.Fa periph->periph_name
+(for example
+.Sq da
+or
+.Sq ada ) .
+Since the probe itself is necessarily generic
+.Pq SDT probe names are fixed at compile time, not runtime ,
+a consumer wanting to filter by peripheral type should match on
+.Fa periph->periph_name
+in the enabling's predicate, for example
+.Li stringof(args[0]->periph_name) == \(dqda\(dq .
+.Pp
+The
+.Fn cam::xpt:bus-register
+probe fires from
+.Fn xpt_bus_register
+once a new bus has been assigned a
+.Fa path_id
+and linked into the list of known busses, reporting the
+.Vt cam_sim
+being registered and that
+.Fa path_id .
+The
+.Fn cam::xpt:hold-boot
+and
+.Fn cam::xpt:release-boot
+probes report the CAM subsystem's shared boot-hold refcount
+.Pq Va xsoftc.buses_to_config
+immediately after it is incremented or decremented, respectively.
+This one refcount is shared by every bus's initial scan
+.Pq via Fn xpt_rescan
+as well as by peripheral drivers' own startup bookkeeping, so root
+mount can remain blocked on it long after any one contributor has
+finished; tracing both probes together shows exactly which holds are
+still outstanding and for how long.
 .Sh SEE ALSO
 .Xr dtrace 1 ,
 .Xr cam 4 ,
diff --git a/sys/cam/ata/ata_da.c b/sys/cam/ata/ata_da.c
index 9434756b87f9..76f8e26e43ae 100644
--- a/sys/cam/ata/ata_da.c
+++ b/sys/cam/ata/ata_da.c
@@ -68,6 +68,10 @@
 
 #include <cam/ata/ata_all.h>
 
+/* SDT Probes */
+SDT_PROBE_DEFINE3(cam, , ada, error, "union ccb *", "uint32_t", "uint32_t");
+SDT_PROBE_DEFINE2(cam, , ada, recovery, "union ccb *", "int");
+
 #ifdef _KERNEL
 
 #define ATA_MAX_28BIT_LBA               268435455UL
@@ -3479,6 +3483,10 @@ adadone(struct cam_periph *periph, union ccb *done_ccb)
 static int
 adaerror(union ccb *ccb, uint32_t cam_flags, uint32_t sense_flags)
 {
+	int error;
+
+	CAM_PROBE3(ada, error, ccb, cam_flags, sense_flags);
+
 #ifdef CAM_IO_STATS
 	struct ada_softc *softc;
 	struct cam_periph *periph;
@@ -3503,7 +3511,9 @@ adaerror(union ccb *ccb, uint32_t cam_flags, uint32_t sense_flags)
 	}
 #endif
 
-	return(cam_periph_error(ccb, cam_flags, sense_flags));
+	error = cam_periph_error(ccb, cam_flags, sense_flags);
+	CAM_PROBE2(ada, recovery, ccb, error);
+	return (error);
 }
 
 static void
diff --git a/sys/cam/cam.h b/sys/cam/cam.h
index 4d12debdd86f..701aa11024b3 100644
--- a/sys/cam/cam.h
+++ b/sys/cam/cam.h
@@ -388,6 +388,11 @@ SDT_PROVIDER_DECLARE(cam);
 	SDT_PROBE3(cam, , group, probe, arg0, arg1, arg2)
 #define CAM_PROBE4(group, probe, arg0, arg1, arg2, arg3)	\
 	SDT_PROBE4(cam, , group, probe, arg0, arg1, arg2, arg3)
+#else
+#define CAM_PROBE1(group, probe, arg0)
+#define CAM_PROBE2(group, probe, arg0, arg1)
+#define CAM_PROBE3(group, probe, arg0, arg1, arg2)
+#define CAM_PROBE4(group, probe, arg0, arg1, arg2, arg3)
 #endif
 union ccb;
 struct sbuf;
diff --git a/sys/cam/cam_periph.c b/sys/cam/cam_periph.c
index e2672fbff7c7..d79977249cf2 100644
--- a/sys/cam/cam_periph.c
+++ b/sys/cam/cam_periph.c
@@ -61,6 +61,14 @@
 #include <cam/scsi/scsi_message.h>
 #include <cam/scsi/scsi_pass.h>
 
+/* SDT Probes */
+SDT_PROBE_DEFINE3(cam, , periph, error, "union ccb *", "cam_flags",
+    "uint32_t");
+SDT_PROBE_DEFINE2(cam, , periph, recovery, "union ccb *", "int");
+SDT_PROBE_DEFINE1(cam, , periph, invalidate, "struct cam_periph *");
+SDT_PROBE_DEFINE1(cam, , periph, hold__boot, "struct cam_periph *");
+SDT_PROBE_DEFINE1(cam, , periph, release__boot, "struct cam_periph *");
+
 static	u_int		camperiphnextunit(struct periph_driver *p_drv,
 					  u_int newunit, bool wired,
 					  path_id_t pathid, target_id_t target,
@@ -536,6 +544,7 @@ void
 cam_periph_hold_boot(struct cam_periph *periph)
 {
 
+	CAM_PROBE1(periph, hold__boot, periph);
 	root_mount_hold_token(periph->periph_name, &periph->periph_rootmount);
 }
 
@@ -543,6 +552,7 @@ void
 cam_periph_release_boot(struct cam_periph *periph)
 {
 
+	CAM_PROBE1(periph, release__boot, periph);
 	root_mount_rel(&periph->periph_rootmount);
 }
 
@@ -690,6 +700,7 @@ cam_periph_invalidate(struct cam_periph *periph)
 	if ((periph->flags & CAM_PERIPH_INVALID) != 0)
 		return;
 
+	CAM_PROBE1(periph, invalidate, periph);
 	CAM_DEBUG(periph->path, CAM_DEBUG_INFO, ("Periph invalidated\n"));
 	if (!rebooting)
 		cam_periph_invalidate_devctl(periph);
@@ -1893,6 +1904,8 @@ cam_periph_error(union ccb *ccb, cam_flags camflags,
 	int	    error, openings, devctl_err;
 	uint32_t   action, relsim_flags, timeout;
 
+	CAM_PROBE3(periph, error, ccb, camflags, sense_flags);
+
 	action = SSQ_PRINT_SENSE;
 	periph = xpt_path_periph(ccb->ccb_h.path);
 	action_string = NULL;
@@ -2122,6 +2135,7 @@ cam_periph_error(union ccb *ccb, cam_flags camflags,
 					 /*getcount_only*/0);
 	}
 
+	CAM_PROBE2(periph, recovery, ccb, error);
 	return (error);
 }
 
diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c
index 0ae6adb8adf3..8a1da69c4acc 100644
--- a/sys/cam/cam_xpt.c
+++ b/sys/cam/cam_xpt.c
@@ -75,6 +75,9 @@ SDT_PROBE_DEFINE1(cam, , xpt, action, "union ccb *");
 SDT_PROBE_DEFINE1(cam, , xpt, done, "union ccb *");
 SDT_PROBE_DEFINE4(cam, , xpt, async__cb, "void *", "uint32_t",
     "struct cam_path *", "void *");
+SDT_PROBE_DEFINE2(cam, , xpt, bus__register, "struct cam_sim *", "path_id_t");
+SDT_PROBE_DEFINE1(cam, , xpt, hold__boot, "int");
+SDT_PROBE_DEFINE1(cam, , xpt, release__boot, "int");
 
 /* Wild guess based on not wanting to grow the stack too much */
 #define XPT_PRINT_MAXLEN	512
@@ -3941,6 +3944,8 @@ xpt_bus_register(struct cam_sim *sim, device_t parent, uint32_t bus)
 	xsoftc.bus_generation++;
 	xpt_unlock_buses();
 
+	CAM_PROBE2(xpt, bus__register, sim, new_bus->path_id);
+
 	/*
 	 * Set a default transport so that a PATH_INQ can be issued to
 	 * the SIM.  This will then allow for probing and attaching of
@@ -5092,6 +5097,7 @@ xpt_hold_boot_locked(void)
 
 	if (xsoftc.buses_to_config++ == 0)
 		root_mount_hold_token("CAM", &xsoftc.xpt_rootmount);
+	CAM_PROBE1(xpt, hold__boot, xsoftc.buses_to_config);
 }
 
 void
@@ -5108,10 +5114,13 @@ xpt_release_boot(void)
 {
 
 	xpt_lock_buses();
-	if (--xsoftc.buses_to_config == 0) {
+	--xsoftc.buses_to_config;
+	CAM_PROBE1(xpt, release__boot, xsoftc.buses_to_config);
+	if (xsoftc.buses_to_config == 0) {
 		if (xsoftc.buses_config_done == 0) {
 			xsoftc.buses_config_done = 1;
 			xsoftc.buses_to_config++;
+			CAM_PROBE1(xpt, hold__boot, xsoftc.buses_to_config);
 			TASK_INIT(&xsoftc.boot_task, 0, xpt_finishconfig_task,
 			    NULL);
 			taskqueue_enqueue(taskqueue_thread, &xsoftc.boot_task);
diff --git a/sys/cam/mmc/mmc_da.c b/sys/cam/mmc/mmc_da.c
index 322141a72707..3a5a7eedb4a1 100644
--- a/sys/cam/mmc/mmc_da.c
+++ b/sys/cam/mmc/mmc_da.c
@@ -74,6 +74,10 @@
 
 #include <cam/mmc/mmc_all.h>
 
+/* SDT Probes */
+SDT_PROBE_DEFINE3(cam, , sdda, error, "union ccb *", "uint32_t", "uint32_t");
+SDT_PROBE_DEFINE2(cam, , sdda, recovery, "union ccb *", "int");
+
 #ifdef _KERNEL
 
 typedef enum {
@@ -1983,7 +1987,13 @@ sddadone(struct cam_periph *periph, union ccb *done_ccb)
 static int
 sddaerror(union ccb *ccb, uint32_t cam_flags, uint32_t sense_flags)
 {
-	return(cam_periph_error(ccb, cam_flags, sense_flags));
+	int error;
+
+	CAM_PROBE3(sdda, error, ccb, cam_flags, sense_flags);
+
+	error = cam_periph_error(ccb, cam_flags, sense_flags);
+	CAM_PROBE2(sdda, recovery, ccb, error);
+	return (error);
 }
 
 static int
diff --git a/sys/cam/nvme/nvme_da.c b/sys/cam/nvme/nvme_da.c
index ebebe5bbc0a9..de1253c78fed 100644
--- a/sys/cam/nvme/nvme_da.c
+++ b/sys/cam/nvme/nvme_da.c
@@ -65,6 +65,10 @@
 
 #include <cam/nvme/nvme_all.h>
 
+/* SDT Probes */
+SDT_PROBE_DEFINE3(cam, , nda, error, "union ccb *", "uint32_t", "uint32_t");
+SDT_PROBE_DEFINE2(cam, , nda, recovery, "union ccb *", "int");
+
 typedef enum {
 	NDA_STATE_NORMAL
 } nda_state;
@@ -1371,6 +1375,10 @@ ndadone(struct cam_periph *periph, union ccb *done_ccb)
 static int
 ndaerror(union ccb *ccb, uint32_t cam_flags, uint32_t sense_flags)
 {
+	int error;
+
+	CAM_PROBE3(nda, error, ccb, cam_flags, sense_flags);
+
 #ifdef CAM_IO_STATS
 	struct nda_softc *softc;
 	struct cam_periph *periph;
@@ -1395,7 +1403,9 @@ ndaerror(union ccb *ccb, uint32_t cam_flags, uint32_t sense_flags)
 		break;
 	}
 
-	return(cam_periph_error(ccb, cam_flags, sense_flags));
+	error = cam_periph_error(ccb, cam_flags, sense_flags);
+	CAM_PROBE2(nda, recovery, ccb, error);
+	return (error);
 }
 
 /*
diff --git a/sys/cam/scsi/scsi_da.c b/sys/cam/scsi/scsi_da.c
index b7b07e922914..9a52ee96fa80 100644
--- a/sys/cam/scsi/scsi_da.c
+++ b/sys/cam/scsi/scsi_da.c
@@ -72,6 +72,10 @@
 #include <cam/scsi/scsi_da.h>
 
 #ifdef _KERNEL
+/* SDT Probes */
+SDT_PROBE_DEFINE3(cam, , da, error, "union ccb *", "uint32_t", "uint32_t");
+SDT_PROBE_DEFINE2(cam, , da, recovery, "union ccb *", "int");
+
 /*
  * Note that there are probe ordering dependencies here.  The order isn't
  * controlled by this enumeration, but by explicit state transitions in
@@ -6255,6 +6259,8 @@ daerror(union ccb *ccb, uint32_t cam_flags, uint32_t sense_flags)
 	struct cam_periph *periph;
 	int error, error_code, sense_key, asc, ascq;
 
+	CAM_PROBE3(da, error, ccb, cam_flags, sense_flags);
+
 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
 	if (ccb->csio.bio != NULL)
 		biotrack(ccb->csio.bio, __func__);
@@ -6320,8 +6326,10 @@ daerror(union ccb *ccb, uint32_t cam_flags, uint32_t sense_flags)
 			disk_media_gone(softc->disk, M_NOWAIT);
 		}
 	}
-	if (error == ERESTART)
+	if (error == ERESTART) {
+		CAM_PROBE2(da, recovery, ccb, error);
 		return (ERESTART);
+	}
 
 #ifdef CAM_IO_STATS
 	switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
@@ -6351,7 +6359,9 @@ daerror(union ccb *ccb, uint32_t cam_flags, uint32_t sense_flags)
 
 	if (softc->quirks & DA_Q_RETRY_BUSY)
 		sense_flags |= SF_RETRY_BUSY;
-	return(cam_periph_error(ccb, cam_flags, sense_flags));
+	error = cam_periph_error(ccb, cam_flags, sense_flags);
+	CAM_PROBE2(da, recovery, ccb, error);
+	return (error);
 }
 
 static void