git: 3394d4239b85 - main - cam: allocate CCBs from UMA for SCSI and ATA IO
Edward Tomasz Napierala
trasz at FreeBSD.org
Sat May 15 11:05:50 UTC 2021
The branch main has been updated by trasz:
URL: https://cgit.FreeBSD.org/src/commit/?id=3394d4239b85b5577845d9e6de4e97b18d3dba58
commit 3394d4239b85b5577845d9e6de4e97b18d3dba58
Author: Edward Tomasz Napierala <trasz at FreeBSD.org>
AuthorDate: 2021-05-15 10:17:22 +0000
Commit: Edward Tomasz Napierala <trasz at FreeBSD.org>
CommitDate: 2021-05-15 11:03:49 +0000
cam: allocate CCBs from UMA for SCSI and ATA IO
This patch makes it possible for CAM to use small CCBs allocated
from an periph-specific UMA zone instead of the usual, huge ones.
The end result is that CCBs issued via da(4) take 544B (size of
ccb_scsiio) instead of the usual 2kB (size of 'union ccb', ~1.5kB,
rounded up by malloc(9)). For ATA it's 272B. We waste less
memory, we avoid zeroing the unused 1kB, and it should be easier
to allocate those CCBs in low memory conditions. It should also
be possible to use uma_zone_reserve(9) to improve behaviour
in low memory conditions even further.
Note that this does not change the size, or the layout, of CCBs
as such. CCBs get allocated in various different ways, in particular
on the stack, and I don't want to redo all that. Instead, this
provides an opt-in mechanism for the periph to declare "my start()
callback is fine with receiving a CCB allocated from this UMA zone".
In other words, most of the code works exactly as it used to; the
change only happens to IOs issued by xpt_run_allockq(), which
is - conveniently - pretty much all that matters for performance.
The reason for doing it this way is that it's pretty small, localized
change, and can be implemented gradually and iteratively: take a
periph, make sure its start() callback only casts the CCBs it takes
to a particular type of CCB, for example ccb_scsiio, and that it only
casts CCBs returned by cam_periph_getccb() to that type, then add UMA
zone for that size, and declare it safe to XPT.
This is disabled by default. Set 'kern.cam.ada.enable_uma_ccbs=1'
and 'kern.cam.da.enable_uma_ccbs=1' tunables to enable it. Testing
is welcome; I will flip the default to enable in two weeks from now.
Reviewed By: imp
Sponsored by: NetApp, Inc.
Sponsored by: Klara, Inc.
Differential Revision: https://reviews.freebsd.org/D28674
---
sys/cam/ata/ata_da.c | 18 ++++++++++++++++++
sys/cam/ata/ata_xpt.c | 7 +++++++
sys/cam/cam_ccb.h | 14 +++++++++++++-
sys/cam/cam_periph.h | 3 +++
sys/cam/cam_xpt.c | 32 +++++++++++++++++++++++++++++---
sys/cam/scsi/scsi_da.c | 18 ++++++++++++++++++
sys/cam/scsi/scsi_xpt.c | 7 +++++++
7 files changed, 95 insertions(+), 4 deletions(-)
diff --git a/sys/cam/ata/ata_da.c b/sys/cam/ata/ata_da.c
index c29235e64e81..a8b67b2254a6 100644
--- a/sys/cam/ata/ata_da.c
+++ b/sys/cam/ata/ata_da.c
@@ -297,6 +297,8 @@ struct ada_softc {
char announce_buffer[ADA_ANNOUNCE_SZ];
};
+static uma_zone_t ada_ccb_zone;
+
struct ada_quirk_entry {
struct scsi_inquiry_pattern inq_pat;
ada_quirks quirks;
@@ -902,6 +904,7 @@ static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
static int ada_enable_biospeedup = 1;
+static int ada_enable_uma_ccbs = 0;
static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
"CAM Direct Access Disk driver");
@@ -921,6 +924,8 @@ SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN,
&ada_write_cache, 0, "Enable disk write cache");
SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_biospeedup, CTLFLAG_RDTUN,
&ada_enable_biospeedup, 0, "Enable BIO_SPEEDUP processing");
+SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_uma_ccbs, CTLFLAG_RWTUN,
+ &ada_enable_uma_ccbs, 0, "Use UMA for CCBs");
/*
* ADA_ORDEREDTAG_INTERVAL determines how often, relative
@@ -1178,6 +1183,10 @@ adainit(void)
{
cam_status status;
+ ada_ccb_zone = uma_zcreate("ada_ccb",
+ sizeof(struct ccb_ataio), NULL, NULL, NULL, NULL,
+ UMA_ALIGN_PTR, 0);
+
/*
* Install a global async callback. This callback will
* receive async callbacks like "new device found".
@@ -1855,6 +1864,15 @@ adaregister(struct cam_periph *periph, void *arg)
"kern.cam.ada.%d.write_cache", periph->unit_number);
TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
+ /*
+ * Let XPT know we can use UMA-allocated CCBs.
+ */
+ if (ada_enable_uma_ccbs) {
+ KASSERT(ada_ccb_zone != NULL,
+ ("%s: NULL ada_ccb_zone", __func__));
+ periph->ccb_zone = ada_ccb_zone;
+ }
+
/*
* Set support flags based on the Identify data and quirks.
*/
diff --git a/sys/cam/ata/ata_xpt.c b/sys/cam/ata/ata_xpt.c
index 0f94e556745a..c13c7b493c78 100644
--- a/sys/cam/ata/ata_xpt.c
+++ b/sys/cam/ata/ata_xpt.c
@@ -1795,6 +1795,13 @@ static void
ata_action(union ccb *start_ccb)
{
+ if (start_ccb->ccb_h.func_code != XPT_ATA_IO) {
+ KASSERT((start_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) == 0,
+ ("%s: ccb %p, func_code %#x should not be allocated "
+ "from UMA zone\n",
+ __func__, start_ccb, start_ccb->ccb_h.func_code));
+ }
+
switch (start_ccb->ccb_h.func_code) {
case XPT_SET_TRAN_SETTINGS:
{
diff --git a/sys/cam/cam_ccb.h b/sys/cam/cam_ccb.h
index 221b24a7c187..2545e40e192d 100644
--- a/sys/cam/cam_ccb.h
+++ b/sys/cam/cam_ccb.h
@@ -58,6 +58,12 @@
/* Struct definitions for CAM control blocks */
/* Common CCB header */
+
+/* CCB memory allocation flags */
+typedef enum {
+ CAM_CCB_FROM_UMA = 0x00000001,/* CCB from a periph UMA zone */
+} ccb_alloc_flags;
+
/* CAM CCB flags */
typedef enum {
CAM_CDB_POINTER = 0x00000001,/* The CDB field is a pointer */
@@ -341,7 +347,13 @@ struct ccb_hdr {
camq_entry xpt_links; /* For chaining in the XPT layer */
camq_entry sim_links; /* For chaining in the SIM layer */
camq_entry periph_links; /* For chaining in the type driver */
- u_int32_t retry_count;
+#if BYTE_ORDER == LITTLE_ENDIAN
+ u_int16_t retry_count;
+ u_int16_t alloc_flags; /* ccb_alloc_flags */
+#else
+ u_int16_t alloc_flags; /* ccb_alloc_flags */
+ u_int16_t retry_count;
+#endif
void (*cbfcnp)(struct cam_periph *, union ccb *);
/* Callback on completion function */
xpt_opcode func_code; /* XPT function code */
diff --git a/sys/cam/cam_periph.h b/sys/cam/cam_periph.h
index 15a239decf0a..9c323394797c 100644
--- a/sys/cam/cam_periph.h
+++ b/sys/cam/cam_periph.h
@@ -42,6 +42,8 @@
#include <sys/sysctl.h>
#include <sys/taskqueue.h>
+#include <vm/uma.h>
+
#include <cam/cam_xpt.h>
struct devstat;
@@ -147,6 +149,7 @@ struct cam_periph {
ac_callback_t *deferred_callback;
ac_code deferred_ac;
struct task periph_run_task;
+ uma_zone_t ccb_zone;
};
#define CAM_PERIPH_MAXMAPS 2
diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c
index b7bc2b74da6c..33361cfb68a5 100644
--- a/sys/cam/cam_xpt.c
+++ b/sys/cam/cam_xpt.c
@@ -4693,7 +4693,17 @@ xpt_alloc_ccb_nowait(void)
void
xpt_free_ccb(union ccb *free_ccb)
{
- free(free_ccb, M_CAMCCB);
+ struct cam_periph *periph;
+
+ if (free_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) {
+ /*
+ * Looks like a CCB allocated from a periph UMA zone.
+ */
+ periph = free_ccb->ccb_h.path->periph;
+ uma_zfree(periph->ccb_zone, free_ccb);
+ } else {
+ free(free_ccb, M_CAMCCB);
+ }
}
/* Private XPT functions */
@@ -4707,10 +4717,18 @@ static union ccb *
xpt_get_ccb_nowait(struct cam_periph *periph)
{
union ccb *new_ccb;
+ int alloc_flags;
- new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
+ if (periph->ccb_zone != NULL) {
+ alloc_flags = CAM_CCB_FROM_UMA;
+ new_ccb = uma_zalloc(periph->ccb_zone, M_ZERO|M_NOWAIT);
+ } else {
+ alloc_flags = 0;
+ new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
+ }
if (new_ccb == NULL)
return (NULL);
+ new_ccb->ccb_h.alloc_flags = alloc_flags;
periph->periph_allocated++;
cam_ccbq_take_opening(&periph->path->device->ccbq);
return (new_ccb);
@@ -4720,9 +4738,17 @@ static union ccb *
xpt_get_ccb(struct cam_periph *periph)
{
union ccb *new_ccb;
+ int alloc_flags;
cam_periph_unlock(periph);
- new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
+ if (periph->ccb_zone != NULL) {
+ alloc_flags = CAM_CCB_FROM_UMA;
+ new_ccb = uma_zalloc(periph->ccb_zone, M_ZERO|M_WAITOK);
+ } else {
+ alloc_flags = 0;
+ new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
+ }
+ new_ccb->ccb_h.alloc_flags = alloc_flags;
cam_periph_lock(periph);
periph->periph_allocated++;
cam_ccbq_take_opening(&periph->path->device->ccbq);
diff --git a/sys/cam/scsi/scsi_da.c b/sys/cam/scsi/scsi_da.c
index baba5d9ed839..05dd9b6e7566 100644
--- a/sys/cam/scsi/scsi_da.c
+++ b/sys/cam/scsi/scsi_da.c
@@ -403,6 +403,8 @@ struct da_softc {
softc->delete_available &= ~(1 << delete_method); \
}
+static uma_zone_t da_ccb_zone;
+
struct da_quirk_entry {
struct scsi_inquiry_pattern inq_pat;
da_quirks quirks;
@@ -1557,6 +1559,7 @@ static sbintime_t da_default_softtimeout = DA_DEFAULT_SOFTTIMEOUT;
static int da_send_ordered = DA_DEFAULT_SEND_ORDERED;
static int da_disable_wp_detection = 0;
static int da_enable_biospeedup = 1;
+static int da_enable_uma_ccbs = 0;
static SYSCTL_NODE(_kern_cam, OID_AUTO, da, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
"CAM Direct Access Disk driver");
@@ -1573,6 +1576,8 @@ SYSCTL_INT(_kern_cam_da, OID_AUTO, disable_wp_detection, CTLFLAG_RWTUN,
"Disable detection of write-protected disks");
SYSCTL_INT(_kern_cam_da, OID_AUTO, enable_biospeedup, CTLFLAG_RDTUN,
&da_enable_biospeedup, 0, "Enable BIO_SPEEDUP processing");
+SYSCTL_INT(_kern_cam_da, OID_AUTO, enable_uma_ccbs, CTLFLAG_RWTUN,
+ &da_enable_uma_ccbs, 0, "Use UMA for CCBs");
SYSCTL_PROC(_kern_cam_da, OID_AUTO, default_softtimeout,
CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, NULL, 0,
@@ -2011,6 +2016,10 @@ dainit(void)
NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
printf("dainit: shutdown event registration failed!\n");
}
+
+ da_ccb_zone = uma_zcreate("da_ccb",
+ sizeof(struct ccb_scsiio), NULL, NULL, NULL, NULL,
+ UMA_ALIGN_PTR, 0);
}
/*
@@ -2848,6 +2857,15 @@ daregister(struct cam_periph *periph, void *arg)
TASK_INIT(&softc->sysctl_task, 0, dasysctlinit, periph);
+ /*
+ * Let XPT know we can use UMA-allocated CCBs.
+ */
+ if (da_enable_uma_ccbs) {
+ KASSERT(da_ccb_zone != NULL,
+ ("%s: NULL da_ccb_zone", __func__));
+ periph->ccb_zone = da_ccb_zone;
+ }
+
/*
* Take an exclusive section lock on the periph while dastart is called
* to finish the probe. The lock will be dropped in dadone at the end
diff --git a/sys/cam/scsi/scsi_xpt.c b/sys/cam/scsi/scsi_xpt.c
index 67b94488dff0..bdc23e4b51b7 100644
--- a/sys/cam/scsi/scsi_xpt.c
+++ b/sys/cam/scsi/scsi_xpt.c
@@ -2625,6 +2625,13 @@ static void
scsi_action(union ccb *start_ccb)
{
+ if (start_ccb->ccb_h.func_code != XPT_SCSI_IO) {
+ KASSERT((start_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) == 0,
+ ("%s: ccb %p, func_code %#x should not be allocated "
+ "from UMA zone\n",
+ __func__, start_ccb, start_ccb->ccb_h.func_code));
+ }
+
switch (start_ccb->ccb_h.func_code) {
case XPT_SET_TRAN_SETTINGS:
{
More information about the dev-commits-src-all
mailing list