git: 2c7dc6bae9fd - main - Refactor CTL datamove KPI.
Alexander Motin
mav at FreeBSD.org
Sun Feb 21 21:52:38 UTC 2021
The branch main has been updated by mav:
URL: https://cgit.FreeBSD.org/src/commit/?id=2c7dc6bae9fd5c2fa0a65768df8e4e99c2f159f1
commit 2c7dc6bae9fd5c2fa0a65768df8e4e99c2f159f1
Author: Alexander Motin <mav at FreeBSD.org>
AuthorDate: 2021-02-21 21:45:14 +0000
Commit: Alexander Motin <mav at FreeBSD.org>
CommitDate: 2021-02-21 21:52:33 +0000
Refactor CTL datamove KPI.
- Make frontends call unified CTL core method ctl_datamove_done()
to report move completion. It allows to reduce code duplication
in differerent backends by accounting DMA time in common code.
- Add to ctl_datamove_done() and be_move_done() callback samethr
argument, reporting whether the callback is called in the same
context as ctl_datamove(). It allows for some cases like iSCSI
write with immediate data or camsim frontend write save one context
switch, since we know that the context is sleepable.
- Remove data_move_done() methods from struct ctl_backend_driver,
unused since forever.
MFC after: 1 month
---
sys/cam/ctl/ctl.c | 112 +++++++++++++++++--------------------
sys/cam/ctl/ctl.h | 3 +-
sys/cam/ctl/ctl_backend.h | 1 -
sys/cam/ctl/ctl_backend_block.c | 64 ++++++---------------
sys/cam/ctl/ctl_backend_ramdisk.c | 36 ++----------
sys/cam/ctl/ctl_frontend_cam_sim.c | 2 +-
sys/cam/ctl/ctl_frontend_ioctl.c | 2 +-
sys/cam/ctl/ctl_frontend_iscsi.c | 23 ++++----
sys/cam/ctl/ctl_io.h | 2 +-
sys/cam/ctl/ctl_tpc_local.c | 2 +-
sys/cam/ctl/scsi_ctl.c | 3 +-
sys/dev/usb/storage/cfumass.c | 6 +-
12 files changed, 94 insertions(+), 162 deletions(-)
diff --git a/sys/cam/ctl/ctl.c b/sys/cam/ctl/ctl.c
index 464fd8dfc25e..9a9ae70c2173 100644
--- a/sys/cam/ctl/ctl.c
+++ b/sys/cam/ctl/ctl.c
@@ -535,9 +535,9 @@ static void ctl_done_timer_wakeup(void *arg);
static void ctl_send_datamove_done(union ctl_io *io, int have_lock);
static void ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq);
-static int ctl_datamove_remote_dm_write_cb(union ctl_io *io);
+static int ctl_datamove_remote_dm_write_cb(union ctl_io *io, bool samethr);
static void ctl_datamove_remote_write(union ctl_io *io);
-static int ctl_datamove_remote_dm_read_cb(union ctl_io *io);
+static int ctl_datamove_remote_dm_read_cb(union ctl_io *io, bool samethr);
static void ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq);
static int ctl_datamove_remote_sgl_setup(union ctl_io *io);
static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
@@ -736,7 +736,7 @@ ctl_ha_datamove(union ctl_io *io)
sizeof(struct ctl_sg_entry) * msg.dt.cur_sg_entries,
M_WAITOK) > CTL_HA_STATUS_SUCCESS) {
io->io_hdr.port_status = 31341;
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, true);
return;
}
msg.dt.sent_sg_entries = sg_entries_sent;
@@ -753,7 +753,7 @@ ctl_ha_datamove(union ctl_io *io)
if (lun)
mtx_unlock(&lun->lun_lock);
io->io_hdr.port_status = 31342;
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, true);
return;
}
io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
@@ -5028,7 +5028,7 @@ ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
* make it down to say RAIDCore's configuration code.
*/
int
-ctl_config_move_done(union ctl_io *io)
+ctl_config_move_done(union ctl_io *io, bool samethr)
{
int retval;
@@ -5036,18 +5036,6 @@ ctl_config_move_done(union ctl_io *io)
KASSERT(io->io_hdr.io_type == CTL_IO_SCSI,
("%s: unexpected I/O type %x", __func__, io->io_hdr.io_type));
- if ((io->io_hdr.port_status != 0) &&
- ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
- (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
- ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1,
- /*retry_count*/ io->io_hdr.port_status);
- } else if (io->scsiio.kern_data_resid != 0 &&
- (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT &&
- ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
- (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
- ctl_set_invalid_field_ciu(&io->scsiio);
- }
-
if (ctl_debug & CTL_DEBUG_CDB_DATA)
ctl_data_print(io);
if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) ||
@@ -12303,7 +12291,7 @@ ctl_handle_isc(union ctl_io *io)
ctl_datamove_remote(io);
break;
case CTL_MSG_DATAMOVE_DONE: /* Only used in XFER mode */
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, false);
break;
case CTL_MSG_FAILOVER:
ctl_failover_lun(io);
@@ -12462,6 +12450,45 @@ ctl_datamove_timer_wakeup(void *arg)
}
#endif /* CTL_IO_DELAY */
+static void
+ctl_datamove_done_process(union ctl_io *io)
+{
+#ifdef CTL_TIME_IO
+ struct bintime cur_bt;
+#endif
+
+ KASSERT(io->io_hdr.io_type == CTL_IO_SCSI,
+ ("%s: unexpected I/O type %x", __func__, io->io_hdr.io_type));
+
+#ifdef CTL_TIME_IO
+ getbinuptime(&cur_bt);
+ bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
+ bintime_add(&io->io_hdr.dma_bt, &cur_bt);
+#endif
+ io->io_hdr.num_dmas++;
+
+ if ((io->io_hdr.port_status != 0) &&
+ ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
+ (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
+ ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1,
+ /*retry_count*/ io->io_hdr.port_status);
+ } else if (io->scsiio.kern_data_resid != 0 &&
+ (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT &&
+ ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
+ (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
+ ctl_set_invalid_field_ciu(&io->scsiio);
+ } else if (ctl_debug & CTL_DEBUG_CDB_DATA)
+ ctl_data_print(io);
+}
+
+void
+ctl_datamove_done(union ctl_io *io, bool samethr)
+{
+
+ ctl_datamove_done_process(io);
+ io->scsiio.be_move_done(io, samethr);
+}
+
void
ctl_datamove(union ctl_io *io)
{
@@ -12475,39 +12502,7 @@ ctl_datamove(union ctl_io *io)
io->scsiio.kern_data_resid = io->scsiio.kern_data_len;
#ifdef CTL_TIME_IO
- if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
- char str[256];
- char path_str[64];
- struct sbuf sb;
-
- ctl_scsi_path_string(io, path_str, sizeof(path_str));
- sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
-
- sbuf_cat(&sb, path_str);
- switch (io->io_hdr.io_type) {
- case CTL_IO_SCSI:
- ctl_scsi_command_string(&io->scsiio, NULL, &sb);
- sbuf_printf(&sb, "\n");
- sbuf_cat(&sb, path_str);
- sbuf_printf(&sb, "Tag: 0x%04x/%d, Prio: %d\n",
- io->scsiio.tag_num, io->scsiio.tag_type,
- io->scsiio.priority);
- break;
- case CTL_IO_TASK:
- sbuf_printf(&sb, "Task Action: %d Tag: 0x%04x/%d\n",
- io->taskio.task_action,
- io->taskio.tag_num, io->taskio.tag_type);
- break;
- default:
- panic("%s: Invalid CTL I/O type %d\n",
- __func__, io->io_hdr.io_type);
- }
- sbuf_cat(&sb, path_str);
- sbuf_printf(&sb, "ctl_datamove: %jd seconds\n",
- (intmax_t)time_uptime - io->io_hdr.start_time);
- sbuf_finish(&sb);
- printf("%s", sbuf_data(&sb));
- }
+ getbinuptime(&io->io_hdr.dma_start_bt);
#endif /* CTL_TIME_IO */
#ifdef CTL_IO_DELAY
@@ -12542,18 +12537,15 @@ ctl_datamove(union ctl_io *io)
io->io_hdr.nexus.targ_port,
io->io_hdr.nexus.targ_lun);
io->io_hdr.port_status = 31337;
- /*
- * Note that the backend, in this case, will get the
- * callback in its context. In other cases it may get
- * called in the frontend's interrupt thread context.
- */
- io->scsiio.be_move_done(io);
+ ctl_datamove_done_process(io);
+ io->scsiio.be_move_done(io, true);
return;
}
/* Don't confuse frontend with zero length data move. */
if (io->scsiio.kern_data_len == 0) {
- io->scsiio.be_move_done(io);
+ ctl_datamove_done_process(io);
+ io->scsiio.be_move_done(io, true);
return;
}
@@ -12640,7 +12632,7 @@ ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq)
* need to push it over to the remote controller's memory.
*/
static int
-ctl_datamove_remote_dm_write_cb(union ctl_io *io)
+ctl_datamove_remote_dm_write_cb(union ctl_io *io, bool samethr)
{
int retval;
@@ -12679,7 +12671,7 @@ ctl_datamove_remote_write(union ctl_io *io)
}
static int
-ctl_datamove_remote_dm_read_cb(union ctl_io *io)
+ctl_datamove_remote_dm_read_cb(union ctl_io *io, bool samethr)
{
uint32_t i;
diff --git a/sys/cam/ctl/ctl.h b/sys/cam/ctl/ctl.h
index 56dd5313b4cb..be3e4a37b157 100644
--- a/sys/cam/ctl/ctl.h
+++ b/sys/cam/ctl/ctl.h
@@ -170,7 +170,8 @@ int ctl_sap_log_sense_handler(struct ctl_scsiio *ctsio,
int ctl_ie_log_sense_handler(struct ctl_scsiio *ctsio,
struct ctl_page_index *page_index,
int pc);
-int ctl_config_move_done(union ctl_io *io);
+int ctl_config_move_done(union ctl_io *io, bool samethr);
+void ctl_datamove_done(union ctl_io *io, bool samethr);
void ctl_datamove(union ctl_io *io);
void ctl_serseq_done(union ctl_io *io);
void ctl_done(union ctl_io *io);
diff --git a/sys/cam/ctl/ctl_backend.h b/sys/cam/ctl/ctl_backend.h
index be8ab4d1706b..05e65abe41f8 100644
--- a/sys/cam/ctl/ctl_backend.h
+++ b/sys/cam/ctl/ctl_backend.h
@@ -186,7 +186,6 @@ struct ctl_backend_driver {
be_init_t init; /* passed to CTL */
be_shutdown_t shutdown; /* passed to CTL */
be_func_t data_submit; /* passed to CTL */
- be_func_t data_move_done; /* passed to CTL */
be_func_t config_read; /* passed to CTL */
be_func_t config_write; /* passed to CTL */
be_ioctl_t ioctl; /* passed to CTL */
diff --git a/sys/cam/ctl/ctl_backend_block.c b/sys/cam/ctl/ctl_backend_block.c
index 5262c041bf62..7ec8e32f7338 100644
--- a/sys/cam/ctl/ctl_backend_block.c
+++ b/sys/cam/ctl/ctl_backend_block.c
@@ -235,7 +235,7 @@ SYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RWTUN,
static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc);
static void ctl_free_beio(struct ctl_be_block_io *beio);
static void ctl_complete_beio(struct ctl_be_block_io *beio);
-static int ctl_be_block_move_done(union ctl_io *io);
+static int ctl_be_block_move_done(union ctl_io *io, bool samethr);
static void ctl_be_block_biodone(struct bio *bio);
static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
struct ctl_be_block_io *beio);
@@ -291,7 +291,6 @@ static struct ctl_backend_driver ctl_be_block_driver =
.init = ctl_be_block_init,
.shutdown = ctl_be_block_shutdown,
.data_submit = ctl_be_block_submit,
- .data_move_done = ctl_be_block_move_done,
.config_read = ctl_be_block_config_read,
.config_write = ctl_be_block_config_write,
.ioctl = ctl_be_block_ioctl,
@@ -432,46 +431,23 @@ ctl_be_block_compare(union ctl_io *io)
}
static int
-ctl_be_block_move_done(union ctl_io *io)
+ctl_be_block_move_done(union ctl_io *io, bool samethr)
{
struct ctl_be_block_io *beio;
struct ctl_be_block_lun *be_lun;
struct ctl_lba_len_flags *lbalen;
-#ifdef CTL_TIME_IO
- struct bintime cur_bt;
-#endif
beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
be_lun = beio->lun;
DPRINTF("entered\n");
-
-#ifdef CTL_TIME_IO
- getbinuptime(&cur_bt);
- bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
- bintime_add(&io->io_hdr.dma_bt, &cur_bt);
-#endif
- io->io_hdr.num_dmas++;
io->scsiio.kern_rel_offset += io->scsiio.kern_data_len;
/*
- * We set status at this point for read commands, and write
- * commands with errors.
+ * We set status at this point for read and compare commands.
*/
- if (io->io_hdr.flags & CTL_FLAG_ABORT) {
- ;
- } else if ((io->io_hdr.port_status != 0) &&
- ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
- (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
- ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1,
- /*retry_count*/ io->io_hdr.port_status);
- } else if (io->scsiio.kern_data_resid != 0 &&
- (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT &&
- ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
- (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
- ctl_set_invalid_field_ciu(&io->scsiio);
- } else if ((io->io_hdr.port_status == 0) &&
- ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) {
+ if ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
+ (io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE) {
lbalen = ARGS(beio->io);
if (lbalen->flags & CTL_LLF_READ) {
ctl_set_success(&io->scsiio);
@@ -492,18 +468,22 @@ ctl_be_block_move_done(union ctl_io *io)
}
/*
- * At this point, we have a write and the DMA completed
- * successfully. We now have to queue it to the task queue to
+ * At this point, we have a write and the DMA completed successfully.
+ * If we were called synchronously in the original thread then just
+ * dispatch, otherwise we now have to queue it to the task queue to
* execute the backend I/O. That is because we do blocking
* memory allocations, and in the file backing case, blocking I/O.
* This move done routine is generally called in the SIM's
* interrupt context, and therefore we cannot block.
*/
- mtx_lock(&be_lun->queue_lock);
- STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links);
- mtx_unlock(&be_lun->queue_lock);
- taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
-
+ if (samethr) {
+ be_lun->dispatch(be_lun, beio);
+ } else {
+ mtx_lock(&be_lun->queue_lock);
+ STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links);
+ mtx_unlock(&be_lun->queue_lock);
+ taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
+ }
return (0);
}
@@ -598,9 +578,6 @@ ctl_be_block_biodone(struct bio *bio)
ctl_set_success(&io->scsiio);
ctl_serseq_done(io);
}
-#ifdef CTL_TIME_IO
- getbinuptime(&io->io_hdr.dma_start_bt);
-#endif
ctl_datamove(io);
}
}
@@ -811,9 +788,6 @@ ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
ctl_set_success(&io->scsiio);
ctl_serseq_done(io);
}
-#ifdef CTL_TIME_IO
- getbinuptime(&io->io_hdr.dma_start_bt);
-#endif
ctl_datamove(io);
}
}
@@ -980,9 +954,6 @@ ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun,
ctl_set_success(&io->scsiio);
ctl_serseq_done(io);
}
-#ifdef CTL_TIME_IO
- getbinuptime(&io->io_hdr.dma_start_bt);
-#endif
ctl_datamove(io);
}
}
@@ -1672,9 +1643,6 @@ ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
be_lun->dispatch(be_lun, beio);
} else {
SDT_PROBE0(cbb, , write, alloc_done);
-#ifdef CTL_TIME_IO
- getbinuptime(&io->io_hdr.dma_start_bt);
-#endif
ctl_datamove(io);
}
}
diff --git a/sys/cam/ctl/ctl_backend_ramdisk.c b/sys/cam/ctl/ctl_backend_ramdisk.c
index 2595aa0be00e..e67d699bda70 100644
--- a/sys/cam/ctl/ctl_backend_ramdisk.c
+++ b/sys/cam/ctl/ctl_backend_ramdisk.c
@@ -139,7 +139,7 @@ extern struct ctl_softc *control_softc;
static int ctl_backend_ramdisk_init(void);
static int ctl_backend_ramdisk_shutdown(void);
-static int ctl_backend_ramdisk_move_done(union ctl_io *io);
+static int ctl_backend_ramdisk_move_done(union ctl_io *io, bool samethr);
static void ctl_backend_ramdisk_compare(union ctl_io *io);
static void ctl_backend_ramdisk_rw(union ctl_io *io);
static int ctl_backend_ramdisk_submit(union ctl_io *io);
@@ -164,7 +164,6 @@ static struct ctl_backend_driver ctl_be_ramdisk_driver =
.init = ctl_backend_ramdisk_init,
.shutdown = ctl_backend_ramdisk_shutdown,
.data_submit = ctl_backend_ramdisk_submit,
- .data_move_done = ctl_backend_ramdisk_move_done,
.config_read = ctl_backend_ramdisk_config_read,
.config_write = ctl_backend_ramdisk_config_write,
.ioctl = ctl_backend_ramdisk_ioctl,
@@ -402,38 +401,17 @@ ctl_backend_ramdisk_cmp(union ctl_io *io)
}
static int
-ctl_backend_ramdisk_move_done(union ctl_io *io)
+ctl_backend_ramdisk_move_done(union ctl_io *io, bool samethr)
{
struct ctl_be_ramdisk_lun *be_lun =
(struct ctl_be_ramdisk_lun *)CTL_BACKEND_LUN(io);
-#ifdef CTL_TIME_IO
- struct bintime cur_bt;
-#endif
CTL_DEBUG_PRINT(("ctl_backend_ramdisk_move_done\n"));
-#ifdef CTL_TIME_IO
- getbinuptime(&cur_bt);
- bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
- bintime_add(&io->io_hdr.dma_bt, &cur_bt);
-#endif
- io->io_hdr.num_dmas++;
if (io->scsiio.kern_sg_entries > 0)
free(io->scsiio.kern_data_ptr, M_RAMDISK);
io->scsiio.kern_rel_offset += io->scsiio.kern_data_len;
- if (io->io_hdr.flags & CTL_FLAG_ABORT) {
- ;
- } else if (io->io_hdr.port_status != 0 &&
- ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
- (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
- ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1,
- /*retry_count*/ io->io_hdr.port_status);
- } else if (io->scsiio.kern_data_resid != 0 &&
- (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT &&
- ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
- (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
- ctl_set_invalid_field_ciu(&io->scsiio);
- } else if ((io->io_hdr.port_status == 0) &&
- ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) {
+ if ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
+ (io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE) {
if (ARGS(io)->flags & CTL_LLF_COMPARE) {
/* We have data block ready for comparison. */
if (ctl_backend_ramdisk_cmp(io))
@@ -471,9 +449,6 @@ ctl_backend_ramdisk_compare(union ctl_io *io)
io->scsiio.kern_sg_entries = 0;
io->io_hdr.flags |= CTL_FLAG_ALLOCATED;
PRIV(io)->len += lbas;
-#ifdef CTL_TIME_IO
- getbinuptime(&io->io_hdr.dma_start_bt);
-#endif
ctl_datamove(io);
}
@@ -534,9 +509,6 @@ nospc:
ctl_set_success(&io->scsiio);
ctl_serseq_done(io);
}
-#ifdef CTL_TIME_IO
- getbinuptime(&io->io_hdr.dma_start_bt);
-#endif
ctl_datamove(io);
}
diff --git a/sys/cam/ctl/ctl_frontend_cam_sim.c b/sys/cam/ctl/ctl_frontend_cam_sim.c
index fdcccee2f569..0e61a80e452c 100644
--- a/sys/cam/ctl/ctl_frontend_cam_sim.c
+++ b/sys/cam/ctl/ctl_frontend_cam_sim.c
@@ -415,7 +415,7 @@ cfcs_datamove(union ctl_io *io)
xpt_done(ccb);
}
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, true);
}
static void
diff --git a/sys/cam/ctl/ctl_frontend_ioctl.c b/sys/cam/ctl/ctl_frontend_ioctl.c
index ef5e2bd22a86..f326100cb013 100644
--- a/sys/cam/ctl/ctl_frontend_ioctl.c
+++ b/sys/cam/ctl/ctl_frontend_ioctl.c
@@ -566,7 +566,7 @@ cfi_submit_wait(union ctl_io *io)
* will immediately call back and wake us up,
* probably using our own context.
*/
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, false);
break;
case CTL_IOCTL_DONE:
mtx_unlock(¶ms.ioctl_mtx);
diff --git a/sys/cam/ctl/ctl_frontend_iscsi.c b/sys/cam/ctl/ctl_frontend_iscsi.c
index 73483fb155cc..fdbc06150f93 100644
--- a/sys/cam/ctl/ctl_frontend_iscsi.c
+++ b/sys/cam/ctl/ctl_frontend_iscsi.c
@@ -933,7 +933,7 @@ cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
cfiscsi_data_wait_free(cs, cdw);
io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
if (done)
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, false);
else
cfiscsi_datamove_out(io);
}
@@ -1146,7 +1146,7 @@ cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
*/
cdw->cdw_ctl_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42;
- cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
+ ctl_datamove_done(cdw->cdw_ctl_io, false);
cfiscsi_data_wait_free(cs, cdw);
CFISCSI_SESSION_LOCK(cs);
}
@@ -2487,7 +2487,7 @@ cfiscsi_datamove_in(union ctl_io *io)
CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "
"already sent the expected len", buffer_offset);
#endif
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, true);
return;
}
@@ -2508,7 +2508,7 @@ cfiscsi_datamove_in(union ctl_io *io)
CFISCSI_SESSION_WARN(cs, "failed to "
"allocate memory; dropping connection");
ctl_set_busy(&io->scsiio);
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, true);
cfiscsi_session_terminate(cs);
return;
}
@@ -2566,7 +2566,7 @@ cfiscsi_datamove_in(union ctl_io *io)
"allocate memory; dropping connection");
icl_pdu_free(response);
ctl_set_busy(&io->scsiio);
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, true);
cfiscsi_session_terminate(cs);
return;
}
@@ -2656,7 +2656,7 @@ cfiscsi_datamove_in(union ctl_io *io)
cfiscsi_pdu_queue_cb(response, cb);
}
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, true);
}
static void
@@ -2685,9 +2685,10 @@ cfiscsi_datamove_out(union ctl_io *io)
*/
expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
if (io->scsiio.kern_rel_offset >= expected_len) {
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, true);
return;
}
+
datamove_len = MIN(io->scsiio.kern_data_len,
expected_len - io->scsiio.kern_rel_offset);
@@ -2703,7 +2704,7 @@ cfiscsi_datamove_out(union ctl_io *io)
CFISCSI_SESSION_WARN(cs, "failed to "
"allocate memory; dropping connection");
ctl_set_busy(&io->scsiio);
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, true);
cfiscsi_session_terminate(cs);
return;
}
@@ -2750,7 +2751,7 @@ cfiscsi_datamove_out(union ctl_io *io)
done = cfiscsi_handle_data_segment(request, cdw);
if (done) {
cfiscsi_data_wait_free(cs, cdw);
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, true);
return;
}
}
@@ -2773,7 +2774,7 @@ cfiscsi_datamove_out(union ctl_io *io)
CFISCSI_SESSION_WARN(cs, "failed to "
"allocate memory; dropping connection");
ctl_set_busy(&io->scsiio);
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, true);
cfiscsi_session_terminate(cs);
return;
}
@@ -2954,7 +2955,7 @@ cfiscsi_task_management_done(union ctl_io *io)
cdw, cdw_next);
io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 43;
- cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
+ ctl_datamove_done(cdw->cdw_ctl_io, false);
cfiscsi_data_wait_free(cs, cdw);
}
CFISCSI_SESSION_UNLOCK(cs);
diff --git a/sys/cam/ctl/ctl_io.h b/sys/cam/ctl/ctl_io.h
index 52ba98f3a9bd..60f8aef82d02 100644
--- a/sys/cam/ctl/ctl_io.h
+++ b/sys/cam/ctl/ctl_io.h
@@ -329,7 +329,7 @@ struct ctl_scsiio {
ctl_tag_type tag_type; /* simple, ordered, head of queue,etc.*/
uint8_t cdb_len; /* CDB length */
uint8_t cdb[CTL_MAX_CDBLEN]; /* CDB */
- int (*be_move_done)(union ctl_io *io); /* called by fe */
+ int (*be_move_done)(union ctl_io *io, bool samethr); /* called by fe */
int (*io_cont)(union ctl_io *io); /* to continue processing */
ctl_ref kern_data_ref; /* Method to reference/release data */
void *kern_data_arg; /* Opaque argument for kern_data_ref() */
diff --git a/sys/cam/ctl/ctl_tpc_local.c b/sys/cam/ctl/ctl_tpc_local.c
index 714240fe8228..9b2c85ac8173 100644
--- a/sys/cam/ctl/ctl_tpc_local.c
+++ b/sys/cam/ctl/ctl_tpc_local.c
@@ -254,7 +254,7 @@ tpcl_datamove(union ctl_io *io)
__func__, ctsio->ext_data_len, ctsio->kern_data_len));
bailout:
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, true);
}
static void
diff --git a/sys/cam/ctl/scsi_ctl.c b/sys/cam/ctl/scsi_ctl.c
index 646b3fe07053..d3023f9a6c8c 100644
--- a/sys/cam/ctl/scsi_ctl.c
+++ b/sys/cam/ctl/scsi_ctl.c
@@ -1394,8 +1394,7 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb)
xpt_release_ccb(done_ccb);
mtx_unlock(mtx);
- /* Call the backend move done callback */
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, false);
}
return;
}
diff --git a/sys/dev/usb/storage/cfumass.c b/sys/dev/usb/storage/cfumass.c
index 59d744bd62d0..88b5a6156704 100644
--- a/sys/dev/usb/storage/cfumass.c
+++ b/sys/dev/usb/storage/cfumass.c
@@ -475,7 +475,7 @@ cfumass_terminate(struct cfumass_softc *sc)
if (sc->sc_ctl_io != NULL) {
CFUMASS_DEBUG(sc, "terminating CTL transfer");
ctl_set_data_phase_error(&sc->sc_ctl_io->scsiio);
- sc->sc_ctl_io->scsiio.be_move_done(sc->sc_ctl_io);
+ ctl_datamove_done(sc->sc_ctl_io, false);
sc->sc_ctl_io = NULL;
}
@@ -730,7 +730,7 @@ cfumass_t_data_callback(struct usb_xfer *xfer, usb_error_t usb_error)
sc->sc_current_residue == 0 ||
io->scsiio.kern_data_resid == 0) {
sc->sc_ctl_io = NULL;
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, false);
break;
}
/* FALLTHROUGH */
@@ -887,7 +887,7 @@ cfumass_datamove(union ctl_io *io)
fail:
ctl_set_data_phase_error(&io->scsiio);
- io->scsiio.be_move_done(io);
+ ctl_datamove_done(io, true);
sc->sc_ctl_io = NULL;
}
More information about the dev-commits-src-main
mailing list