TRIM visibility bugs [patches]

Hey All,

While toying with different storage options to use with bhyve, I've run 
across a few frustrations related to visibility into TRIM operations. 
Specifically, it's not always easy to tell if trim operations are being 
processed by drives due to the counters not being updated by the SCSI da 
device. The counters exported as sysctl values are updated by the SCCI 
UNMAP method, but not by the TRIM or WS methods. You can see the 
BIO_DELETE operations being processed in real time by running the gstat 
-d command, but the sysctl counters never change from 0. This gives the 
false illusion that nothing is happening. With the following patch, I'm 
now able to see the counters reflect trim operations that are processed 
by da devices ...

mgrooms@mgrooms:~/trim $ cat scsi_da.diff
--- scsi_da.c.orig    2024-03-13 11:32:32.098922000 -0500
+++ scsi_da.c    2024-03-13 11:31:37.255187000 -0500
@@ -4197,6 +4197,9 @@
                da_default_timeout * 1000);
      ccb->ccb_h.ccb_state = DA_CCB_DELETE;
      ccb->ccb_h.flags |= CAM_UNLOCKED;
+    softc->trim_count++;
+    softc->trim_ranges += ranges;
+    softc->trim_lbas += block_count;
      cam_iosched_submit_trim(softc->cam_iosched);
  }

@@ -4257,6 +4260,8 @@
              da_default_timeout * 1000);
      ccb->ccb_h.ccb_state = DA_CCB_DELETE;
      ccb->ccb_h.flags |= CAM_UNLOCKED;
+    softc->trim_count++;
+    softc->trim_lbas += count;
      cam_iosched_submit_trim(softc->cam_iosched);
  }

Additionally, while attempting to test geom mirror+stripe to provide 
software RAID10, the diskinfo utility reports that a mirror supports 
UNMAP/TRIM when at least one underlying devices supports it, but a 
stripe does not. I added a small patch that attempts to use the same 
logic as mirror so that it will report that UNMAP/TRIM is supported when 
one of the underlying devices does ...

--- g_stripe.c.orig    2024-03-12 18:23:52.960025000 -0500
+++ g_stripe.c    2024-03-12 18:25:01.009378000 -0500
@@ -26,6 +26,7 @@
   * SUCH DAMAGE.
   */

+#include <sys/cdefs.h>
  #include <sys/param.h>
  #include <sys/systm.h>
  #include <sys/kernel.h>
@@ -568,7 +569,7 @@
      off_t offset, start, length, nstripe, stripesize;
      struct g_stripe_softc *sc;
      u_int no;
-    int error, fast = 0;
+    int error, fast = 0, val = 0;

      sc = bp->bio_to->geom->softc;
      /*
@@ -591,6 +592,12 @@
          g_stripe_pushdown(sc, bp);
          return;
      case BIO_GETATTR:
+        if (!strcmp(bp->bio_attribute, "GEOM::candelete")) {
+            if (sc->sc_flags & G_STRIPE_FLAG_CANDELETE)
+                val = 1;
+            g_handleattr(bp, "GEOM::candelete", &val, sizeof(val));
+            return;
+        }
          /* To which provider it should be delivered? */
      default:
          g_io_deliver(bp, EOPNOTSUPP);
@@ -745,7 +752,7 @@
  {
      struct g_consumer *cp, *fcp;
      struct g_geom *gp;
-    int error;
+    int error, i;

      g_topology_assert();
      /* Metadata corrupted? */
@@ -792,8 +799,19 @@
              goto fail;
          }
      }
-
      sc->sc_disks[no] = cp;
+
+    /* cascade candelete */
+    error = g_access(cp, 1, 0, 0);
+    if (error == 0)
+    {
+        error = g_getattr("GEOM::candelete", cp, &i);
+        if (error == 0 && i != 0)
+            sc->sc_flags |= G_STRIPE_FLAG_CANDELETE;
+        G_STRIPE_DEBUG(1, "Provider %s candelete %i.", pp->name, i);
+        g_access(cp, -1, 0, 0);
+    }
+
      G_STRIPE_DEBUG(0, "Disk %s attached to %s.", pp->name, sc->sc_name);
      g_stripe_check_and_run(sc);

--- g_stripe.h.orig    2024-03-12 18:24:00.960741000 -0500
+++ g_stripe.h    2024-03-12 12:25:22.842925000 -0500
@@ -47,6 +47,8 @@
  #define    G_STRIPE_TYPE_MANUAL    0
  #define    G_STRIPE_TYPE_AUTOMATIC    1

+#define    G_STRIPE_FLAG_CANDELETE        0x00000001UL
+
  #define    G_STRIPE_DEBUG(lvl, ...) \
      _GEOM_DEBUG("GEOM_STRIPE", g_stripe_debug, (lvl), NULL, __VA_ARGS__)
  #define    G_STRIPE_LOGREQ(bp, ...) \
@@ -61,6 +63,7 @@
      uint16_t     sc_ndisks;
      off_t         sc_stripesize;
      uint32_t     sc_stripebits;
+    uint32_t     sc_flags;
      struct mtx     sc_lock;
  };
  #define    sc_name    sc_geom->name

Fair warning: I'm not a CAM or GEOM developer, so these should be 
reviewed before someone before they are applied anywhere that counts. In 
any case, I wanted to share this but here as I've seen some internet 
posts from other folks setting up virtual storage that ran into similar 
problems. I've also opened a bug report so that hopefully these 
visibility issues get fixed ...

https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=277673

Thanks,

-Matthew