svn commit: r275493 - stable/10/sys/cam/ctl
Alexander Motin
mav at FreeBSD.org
Fri Dec 5 07:23:28 UTC 2014
Author: mav
Date: Fri Dec 5 07:23:25 2014
New Revision: 275493
URL: https://svnweb.freebsd.org/changeset/base/275493
Log:
MFC r274785: Partially reconstruct Active/Standby clusting.
In this mode one head is in Active state, supporting all commands, while
another is in Standby state, supporting only minimal LUN discovery subset.
It is still incomplete since Standby state requires reservation support,
which is impossible to do right without having interlink between heads.
But it allows to run some basic experiments.
Modified:
stable/10/sys/cam/ctl/ctl.c
stable/10/sys/cam/ctl/ctl_cmd_table.c
stable/10/sys/cam/ctl/ctl_frontend.c
stable/10/sys/cam/ctl/ctl_frontend.h
stable/10/sys/cam/ctl/ctl_frontend_cam_sim.c
stable/10/sys/cam/ctl/ctl_frontend_internal.c
stable/10/sys/cam/ctl/ctl_frontend_iscsi.c
stable/10/sys/cam/ctl/ctl_ha.h
stable/10/sys/cam/ctl/ctl_private.h
stable/10/sys/cam/ctl/ctl_tpc_local.c
stable/10/sys/cam/ctl/scsi_ctl.c
Directory Properties:
stable/10/ (props changed)
Modified: stable/10/sys/cam/ctl/ctl.c
==============================================================================
--- stable/10/sys/cam/ctl/ctl.c Fri Dec 5 00:32:33 2014 (r275492)
+++ stable/10/sys/cam/ctl/ctl.c Fri Dec 5 07:23:25 2014 (r275493)
@@ -357,7 +357,6 @@ static struct ctl_logical_block_provisio
static int rcv_sync_msg;
static int persis_offset;
static uint8_t ctl_pause_rtr;
-static int ctl_is_single = 1;
SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer");
static int worker_threads = -1;
@@ -972,12 +971,42 @@ ctl_copy_sense_data(union ctl_ha_msg *sr
}
static int
+ctl_ha_state_sysctl(SYSCTL_HANDLER_ARGS)
+{
+ struct ctl_softc *softc = (struct ctl_softc *)arg1;
+ struct ctl_lun *lun;
+ int error, value, i;
+
+ if (softc->flags & CTL_FLAG_ACTIVE_SHELF)
+ value = 0;
+ else
+ value = 1;
+
+ error = sysctl_handle_int(oidp, &value, 0, req);
+ if ((error != 0) || (req->newptr == NULL))
+ return (error);
+
+ mtx_lock(&softc->ctl_lock);
+ if (value == 0)
+ softc->flags |= CTL_FLAG_ACTIVE_SHELF;
+ else
+ softc->flags &= ~CTL_FLAG_ACTIVE_SHELF;
+ STAILQ_FOREACH(lun, &softc->lun_list, links) {
+ mtx_lock(&lun->lun_lock);
+ for (i = 0; i < CTL_MAX_INITIATORS; i++)
+ lun->pending_ua[i] |= CTL_UA_ASYM_ACC_CHANGE;
+ mtx_unlock(&lun->lun_lock);
+ }
+ mtx_unlock(&softc->ctl_lock);
+ return (0);
+}
+
+static int
ctl_init(void)
{
struct ctl_softc *softc;
struct ctl_io_pool *internal_pool, *emergency_pool, *other_pool;
struct ctl_port *port;
- uint8_t sc_id =0;
int i, error, retval;
//int isc_retval;
@@ -1035,16 +1064,17 @@ ctl_init(void)
* In Copan's HA scheme, the "master" and "slave" roles are
* figured out through the slot the controller is in. Although it
* is an active/active system, someone has to be in charge.
- */
-#ifdef NEEDTOPORT
- scmicro_rw(SCMICRO_GET_SHELF_ID, &sc_id);
-#endif
-
- if (sc_id == 0) {
- softc->flags |= CTL_FLAG_MASTER_SHELF;
- persis_offset = 0;
+ */
+ SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
+ OID_AUTO, "ha_id", CTLFLAG_RDTUN, &softc->ha_id, 0,
+ "HA head ID (0 - no HA)");
+ if (softc->ha_id == 0) {
+ softc->flags |= CTL_FLAG_ACTIVE_SHELF;
+ softc->is_single = 1;
+ softc->port_offset = 0;
} else
- persis_offset = CTL_MAX_INITIATORS;
+ softc->port_offset = (softc->ha_id - 1) * CTL_MAX_PORTS;
+ persis_offset = softc->port_offset * CTL_MAX_INIT_PER_PORT;
/*
* XXX KDM need to figure out where we want to get our target ID
@@ -1157,12 +1187,15 @@ ctl_init(void)
port->max_targets = 15;
port->max_target_id = 15;
- if (ctl_port_register(&softc->ioctl_info.port,
- (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0) {
+ if (ctl_port_register(&softc->ioctl_info.port) != 0) {
printf("ctl: ioctl front end registration failed, will "
"continue anyway\n");
}
+ SYSCTL_ADD_PROC(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
+ OID_AUTO, "ha_state", CTLTYPE_INT | CTLFLAG_RWTUN,
+ softc, 0, ctl_ha_state_sysctl, "I", "HA state for this head");
+
#ifdef CTL_IO_DELAY
if (sizeof(struct callout) > CTL_TIMER_BYTES) {
printf("sizeof(struct callout) %zd > CTL_TIMER_BYTES %zd\n",
@@ -1264,10 +1297,10 @@ ctl_close(struct cdev *dev, int flags, i
int
ctl_port_enable(ctl_port_type port_type)
{
- struct ctl_softc *softc;
+ struct ctl_softc *softc = control_softc;
struct ctl_port *port;
- if (ctl_is_single == 0) {
+ if (softc->is_single == 0) {
union ctl_ha_msg msg_info;
int isc_retval;
@@ -1292,8 +1325,6 @@ ctl_port_enable(ctl_port_type port_type)
#endif
}
- softc = control_softc;
-
STAILQ_FOREACH(port, &softc->port_list, links) {
if (port_type & port->port_type)
{
@@ -7439,8 +7470,8 @@ ctl_report_tagret_port_groups(struct ctl
{
struct scsi_maintenance_in *cdb;
int retval;
- int alloc_len, ext, total_len = 0, g, p, pc, pg;
- int num_target_port_groups, num_target_ports, single;
+ int alloc_len, ext, total_len = 0, g, p, pc, pg, gs, os;
+ int num_target_port_groups, num_target_ports;
struct ctl_lun *lun;
struct ctl_softc *softc;
struct ctl_port *port;
@@ -7474,8 +7505,7 @@ ctl_report_tagret_port_groups(struct ctl
return(retval);
}
- single = ctl_is_single;
- if (single)
+ if (softc->is_single)
num_target_port_groups = 1;
else
num_target_port_groups = NUM_TARGET_PORT_GROUPS;
@@ -7531,18 +7561,26 @@ ctl_report_tagret_port_groups(struct ctl
tpg_desc = &rtg_ptr->groups[0];
}
- pg = ctsio->io_hdr.nexus.targ_port / CTL_MAX_PORTS;
mtx_lock(&softc->ctl_lock);
+ pg = softc->port_offset / CTL_MAX_PORTS;
+ if (softc->flags & CTL_FLAG_ACTIVE_SHELF) {
+ if (softc->ha_mode == CTL_HA_MODE_ACT_STBY) {
+ gs = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
+ os = TPG_ASYMMETRIC_ACCESS_STANDBY;
+ } else if (lun->flags & CTL_LUN_PRIMARY_SC) {
+ gs = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
+ os = TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
+ } else {
+ gs = TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
+ os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
+ }
+ } else {
+ gs = TPG_ASYMMETRIC_ACCESS_STANDBY;
+ os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
+ }
for (g = 0; g < num_target_port_groups; g++) {
- if (g == pg)
- tpg_desc->pref_state = TPG_PRIMARY |
- TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
- else
- tpg_desc->pref_state =
- TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
- tpg_desc->support = TPG_AO_SUP;
- if (!single)
- tpg_desc->support |= TPG_AN_SUP;
+ tpg_desc->pref_state = (g == pg) ? gs : os;
+ tpg_desc->support = TPG_AO_SUP | TPG_AN_SUP | TPG_S_SUP;
scsi_ulto2b(g + 1, tpg_desc->target_port_group);
tpg_desc->status = TPG_IMPLICIT;
pc = 0;
@@ -10181,12 +10219,11 @@ ctl_inquiry_evpd_scsi_ports(struct ctl_s
struct ctl_lun *lun;
struct ctl_port *port;
int data_len, num_target_ports, iid_len, id_len, g, pg, p;
- int num_target_port_groups, single;
+ int num_target_port_groups;
lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
- single = ctl_is_single;
- if (single)
+ if (softc->is_single)
num_target_port_groups = 1;
else
num_target_port_groups = NUM_TARGET_PORT_GROUPS;
@@ -10246,10 +10283,7 @@ ctl_inquiry_evpd_scsi_ports(struct ctl_s
pd = &sp->design[0];
mtx_lock(&softc->ctl_lock);
- if (softc->flags & CTL_FLAG_MASTER_SHELF)
- pg = 0;
- else
- pg = 1;
+ pg = softc->port_offset / CTL_MAX_PORTS;
for (g = 0; g < num_target_port_groups; g++) {
STAILQ_FOREACH(port, &softc->port_list, links) {
if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
@@ -11329,15 +11363,12 @@ ctl_scsiio_lun_check(struct ctl_softc *c
* If this shelf is a secondary shelf controller, we have to reject
* any media access commands.
*/
-#if 0
- /* No longer needed for HA */
- if (((ctl_softc->flags & CTL_FLAG_MASTER_SHELF) == 0)
- && ((entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0)) {
+ if ((ctl_softc->flags & CTL_FLAG_ACTIVE_SHELF) == 0 &&
+ (entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0) {
ctl_set_lun_standby(ctsio);
retval = 1;
goto bailout;
}
-#endif
if (entry->pattern & CTL_LUN_PAT_WRITE) {
if (lun->flags & CTL_LUN_READONLY) {
@@ -14392,7 +14423,7 @@ ctl_isc_start(struct ctl_ha_component *c
// UNKNOWN->HA or UNKNOWN->SINGLE (bootstrap)
if (c->state == CTL_HA_STATE_UNKNOWN ) {
- ctl_is_single = 0;
+ control_softc->is_single = 0;
if (ctl_ha_msg_create(CTL_HA_CHAN_CTL, ctl_isc_event_handler)
!= CTL_HA_STATUS_SUCCESS) {
printf("ctl_isc_start: ctl_ha_msg_create failed.\n");
@@ -14402,14 +14433,14 @@ ctl_isc_start(struct ctl_ha_component *c
&& CTL_HA_STATE_IS_SINGLE(state)){
// HA->SINGLE transition
ctl_failover();
- ctl_is_single = 1;
+ control_softc->is_single = 1;
} else {
printf("ctl_isc_start:Invalid state transition %X->%X\n",
c->state, state);
ret = CTL_HA_COMP_STATUS_ERROR;
}
if (CTL_HA_STATE_IS_SINGLE(state))
- ctl_is_single = 1;
+ control_softc->is_single = 1;
c->state = state;
c->status = ret;
Modified: stable/10/sys/cam/ctl/ctl_cmd_table.c
==============================================================================
--- stable/10/sys/cam/ctl/ctl_cmd_table.c Fri Dec 5 00:32:33 2014 (r275492)
+++ stable/10/sys/cam/ctl/ctl_cmd_table.c Fri Dec 5 07:23:25 2014 (r275493)
@@ -800,12 +800,18 @@ const struct ctl_cmd_entry ctl_cmd_table
/* 3B WRITE BUFFER */
{ctl_write_buffer, CTL_SERIDX_MD_SEL, CTL_CMD_FLAG_OK_ON_BOTH |
+ CTL_CMD_FLAG_OK_ON_STOPPED |
+ CTL_CMD_FLAG_OK_ON_INOPERABLE |
+ CTL_CMD_FLAG_OK_ON_OFFLINE |
CTL_FLAG_DATA_OUT,
CTL_LUN_PAT_NONE,
10, {0x1f, 0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x07}},
/* 3C READ BUFFER */
{ctl_read_buffer, CTL_SERIDX_MD_SNS, CTL_CMD_FLAG_OK_ON_BOTH |
+ CTL_CMD_FLAG_OK_ON_STOPPED |
+ CTL_CMD_FLAG_OK_ON_INOPERABLE |
+ CTL_CMD_FLAG_OK_ON_OFFLINE |
CTL_FLAG_DATA_IN |
CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
CTL_LUN_PAT_NONE,
Modified: stable/10/sys/cam/ctl/ctl_frontend.c
==============================================================================
--- stable/10/sys/cam/ctl/ctl_frontend.c Fri Dec 5 00:32:33 2014 (r275492)
+++ stable/10/sys/cam/ctl/ctl_frontend.c Fri Dec 5 07:23:25 2014 (r275493)
@@ -136,7 +136,7 @@ ctl_frontend_find(char *frontend_name)
}
int
-ctl_port_register(struct ctl_port *port, int master_shelf)
+ctl_port_register(struct ctl_port *port)
{
struct ctl_io_pool *pool;
int port_num;
@@ -193,7 +193,7 @@ error:
STAILQ_INIT(&port->options);
mtx_lock(&control_softc->ctl_lock);
- port->targ_port = port_num + (master_shelf != 0 ? 0 : CTL_MAX_PORTS);
+ port->targ_port = port_num + control_softc->port_offset;
STAILQ_INSERT_TAIL(&port->frontend->port_list, port, fe_links);
STAILQ_INSERT_TAIL(&control_softc->port_list, port, links);
control_softc->ctl_ports[port_num] = port;
Modified: stable/10/sys/cam/ctl/ctl_frontend.h
==============================================================================
--- stable/10/sys/cam/ctl/ctl_frontend.h Fri Dec 5 00:32:33 2014 (r275492)
+++ stable/10/sys/cam/ctl/ctl_frontend.h Fri Dec 5 07:23:25 2014 (r275493)
@@ -278,7 +278,7 @@ struct ctl_frontend * ctl_frontend_find(
* This may block until resources are allocated. Called at FETD module load
* time. Returns 0 for success, non-zero for failure.
*/
-int ctl_port_register(struct ctl_port *port, int master_SC);
+int ctl_port_register(struct ctl_port *port);
/*
* Called at FETD module unload time.
Modified: stable/10/sys/cam/ctl/ctl_frontend_cam_sim.c
==============================================================================
--- stable/10/sys/cam/ctl/ctl_frontend_cam_sim.c Fri Dec 5 00:32:33 2014 (r275492)
+++ stable/10/sys/cam/ctl/ctl_frontend_cam_sim.c Fri Dec 5 07:23:25 2014 (r275493)
@@ -164,7 +164,7 @@ cfcs_init(void)
port->max_targets = 1;
port->max_target_id = 15;
- retval = ctl_port_register(port, /*master_SC*/ 1);
+ retval = ctl_port_register(port);
if (retval != 0) {
printf("%s: ctl_port_register() failed with error %d!\n",
__func__, retval);
Modified: stable/10/sys/cam/ctl/ctl_frontend_internal.c
==============================================================================
--- stable/10/sys/cam/ctl/ctl_frontend_internal.c Fri Dec 5 00:32:33 2014 (r275492)
+++ stable/10/sys/cam/ctl/ctl_frontend_internal.c Fri Dec 5 07:23:25 2014 (r275493)
@@ -244,8 +244,6 @@ cfi_init(void)
memset(softc, 0, sizeof(*softc));
mtx_init(&softc->lock, "CTL frontend mutex", NULL, MTX_DEF);
- softc->flags |= CTL_FLAG_MASTER_SHELF;
-
STAILQ_INIT(&softc->lun_list);
STAILQ_INIT(&softc->metatask_list);
sprintf(softc->fe_name, "kernel");
@@ -264,7 +262,7 @@ cfi_init(void)
port->max_targets = 15;
port->max_target_id = 15;
- if (ctl_port_register(port, (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0)
+ if (ctl_port_register(port) != 0)
{
printf("%s: internal frontend registration failed\n", __func__);
return (0);
Modified: stable/10/sys/cam/ctl/ctl_frontend_iscsi.c
==============================================================================
--- stable/10/sys/cam/ctl/ctl_frontend_iscsi.c Fri Dec 5 00:32:33 2014 (r275492)
+++ stable/10/sys/cam/ctl/ctl_frontend_iscsi.c Fri Dec 5 07:23:25 2014 (r275493)
@@ -2054,7 +2054,7 @@ cfiscsi_ioctl_port_create(struct ctl_req
desc->length = idlen;
strlcpy(desc->identifier, target, idlen);
- retval = ctl_port_register(port, /*master_SC*/ 1);
+ retval = ctl_port_register(port);
if (retval != 0) {
ctl_free_opts(&port->options);
cfiscsi_target_release(ct);
Modified: stable/10/sys/cam/ctl/ctl_ha.h
==============================================================================
--- stable/10/sys/cam/ctl/ctl_ha.h Fri Dec 5 00:32:33 2014 (r275492)
+++ stable/10/sys/cam/ctl/ctl_ha.h Fri Dec 5 07:23:25 2014 (r275493)
@@ -38,6 +38,8 @@
/*
* CTL High Availability Modes:
*
+ * CTL_HA_MODE_ACT_STBY: One side is in Active state and processing commands,
+ * the other side is in Standby state, returning errors.
* CTL_HA_MODE_SER_ONLY: Commands are serialized to the other side. Write
* mirroring and read re-direction are assumed to
* happen in the back end.
@@ -46,6 +48,7 @@
*/
typedef enum {
+ CTL_HA_MODE_ACT_STBY,
CTL_HA_MODE_SER_ONLY,
CTL_HA_MODE_XFER
} ctl_ha_mode;
Modified: stable/10/sys/cam/ctl/ctl_private.h
==============================================================================
--- stable/10/sys/cam/ctl/ctl_private.h Fri Dec 5 00:32:33 2014 (r275492)
+++ stable/10/sys/cam/ctl/ctl_private.h Fri Dec 5 07:23:25 2014 (r275493)
@@ -442,7 +442,7 @@ struct ctl_lun {
typedef enum {
CTL_FLAG_REAL_SYNC = 0x02,
- CTL_FLAG_MASTER_SHELF = 0x04
+ CTL_FLAG_ACTIVE_SHELF = 0x04
} ctl_gen_flags;
#define CTL_MAX_THREADS 16
@@ -467,6 +467,10 @@ struct ctl_softc {
int num_luns;
ctl_gen_flags flags;
ctl_ha_mode ha_mode;
+ int ha_id;
+ int ha_state;
+ int is_single;
+ int port_offset;
int inquiry_pq_no_lun;
struct sysctl_ctx_list sysctl_ctx;
struct sysctl_oid *sysctl_tree;
Modified: stable/10/sys/cam/ctl/ctl_tpc_local.c
==============================================================================
--- stable/10/sys/cam/ctl/ctl_tpc_local.c Fri Dec 5 00:32:33 2014 (r275492)
+++ stable/10/sys/cam/ctl/ctl_tpc_local.c Fri Dec 5 07:23:25 2014 (r275493)
@@ -87,7 +87,6 @@ CTL_FRONTEND_DECLARE(ctltpc, tpcl_fronte
static int
tpcl_init(void)
{
- struct ctl_softc *softc = control_softc;
struct tpcl_softc *tsoftc = &tpcl_softc;
struct ctl_port *port;
struct scsi_transportid_spi *tid;
@@ -112,7 +111,7 @@ tpcl_init(void)
port->max_target_id = 0;
port->max_initiators = 1;
- if (ctl_port_register(port, (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0)
+ if (ctl_port_register(port) != 0)
{
printf("%s: tpc frontend registration failed\n", __func__);
return (0);
Modified: stable/10/sys/cam/ctl/scsi_ctl.c
==============================================================================
--- stable/10/sys/cam/ctl/scsi_ctl.c Fri Dec 5 00:32:33 2014 (r275492)
+++ stable/10/sys/cam/ctl/scsi_ctl.c Fri Dec 5 07:23:25 2014 (r275493)
@@ -425,7 +425,7 @@ ctlfeasync(void *callback_arg, uint32_t
printf("%s: calling ctl_port_register() for %s%d\n",
__func__, cpi->dev_name, cpi->unit_number);
#endif
- retval = ctl_port_register(port, /*master_SC*/ 1);
+ retval = ctl_port_register(port);
if (retval != 0) {
printf("%s: ctl_port_register() failed with "
"error %d!\n", __func__, retval);
More information about the svn-src-all
mailing list