geom write cache handling

[Richard Kojedzinszky]

Dear fs team,

I own an STEC SSD, which I use for ZFS SLOG. The device is broken in that 
if it gets a SCSI synchronize cache, it does something which is very slow. 
Actually with it in FreeBSD the reachable sync IOPS is around 100. When 
the device has write cache disabled, there is no need to send the SCSI 
synchronize cache commands to it, and without them I can reach 1400 IOPS. 
The device itself have power-loss-protection, so I risk no data corruption 
at all.

By the way, linux behave the same, if either ata or scsi disks have their 
write cache turned off, it even does not send the command to the drive.

Also, I have an Intel S3700, which is much faster, but have similar 
symptons. The drive handles synchronize cache commands much faster than 
STEC, maybe it is implemented in the drive as a simple NOP, but as for 4K 
sync writes a sync cache follows, it effectively halves the IOPS. I have 
it attached to a SATA2 controller only, and with WCE disabled, and sending 
the sync cache to it I can reach around 4500 IOPS, while disabling the 
sync cache it can reach >9000 IOPS.

I've attached a very simple patch for the ata layer, but I dont know how 
to implement it for the scsi subsystem also.

Regards,

Kojedzinszky Richard
-------------- next part --------------
diff --git a/sys/cam/ata/ata_da.c b/sys/cam/ata/ata_da.c
index cc28311..10e4f9b 100644
--- a/sys/cam/ata/ata_da.c
+++ b/sys/cam/ata/ata_da.c
@@ -1242,7 +1242,7 @@ adaregister(struct cam_periph *periph, void *arg)
 		maxio = min(maxio, 256 * softc->params.secsize);
 	softc->disk->d_maxsize = maxio;
 	softc->disk->d_unit = periph->unit_number;
-	softc->disk->d_flags = 0;
+	softc->disk->d_flags = DISKFLAG_WRITE_THROUGH;
 	if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE)
 		softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
 	if (softc->flags & ADA_FLAG_CAN_TRIM) {
@@ -1835,6 +1835,12 @@ adadone(struct cam_periph *periph, union ccb *done_ccb)
 			}
 		}
 
+		if (ataio->cmd.features == ATA_SF_ENAB_WCACHE) {
+			softc->disk->d_flags &= ~DISKFLAG_WRITE_THROUGH;
+		} else {
+			softc->disk->d_flags |= DISKFLAG_WRITE_THROUGH;
+		}
+
 		softc->state = ADA_STATE_NORMAL;
 		/*
 		 * Since our peripheral may be invalidated by an error
diff --git a/sys/geom/geom_disk.c b/sys/geom/geom_disk.c
index 16f6c44..2cb4cef 100644
--- a/sys/geom/geom_disk.c
+++ b/sys/geom/geom_disk.c
@@ -404,6 +404,10 @@ g_disk_start(struct bio *bp)
 	case BIO_FLUSH:
 		g_trace(G_T_BIO, "g_disk_flushcache(%s)",
 		    bp->bio_to->name);
+		if (dp->d_flags & DISKFLAG_WRITE_THROUGH) {
+			error = 0;
+			break;
+		}
 		if (!(dp->d_flags & DISKFLAG_CANFLUSHCACHE)) {
 			error = EOPNOTSUPP;
 			break;
diff --git a/sys/geom/geom_disk.h b/sys/geom/geom_disk.h
index 5e081c8..a53aa38 100644
--- a/sys/geom/geom_disk.h
+++ b/sys/geom/geom_disk.h
@@ -111,6 +111,7 @@ struct disk {
 #define DISKFLAG_LACKS_GONE	0x10
 #define DISKFLAG_UNMAPPED_BIO	0x20
 #define DISKFLAG_LACKS_DELMAX	0x40
+#define DISKFLAG_WRITE_THROUGH	0x80
 
 struct disk *disk_alloc(void);
 void disk_create(struct disk *disk, int version);