git: 95c714c6c473 - stable/13 - ocs_fc: Support persistent topology feature

From: Ram Kishore Vegesna <ram_at_FreeBSD.org>
Date: Mon, 07 Mar 2022 11:03:06 UTC
The branch stable/13 has been updated by ram:

URL: https://cgit.FreeBSD.org/src/commit/?id=95c714c6c4735aacec1a683eb39c596a3ae7b2bd

commit 95c714c6c4735aacec1a683eb39c596a3ae7b2bd
Author:     Ram Kishore Vegesna <ram@FreeBSD.org>
AuthorDate: 2022-03-03 16:00:27 +0000
Commit:     Ram Kishore Vegesna <ram@FreeBSD.org>
CommitDate: 2022-03-07 10:54:45 +0000

    ocs_fc: Support persistent topology feature
    
    Summary: Enable persistent topology across power cycles/firmware resets.
    
    Reviewed by: mav
    
    MFC after: 3 days
    
    Differential Revision: https://reviews.freebsd.org/D34425
    
    (cherry picked from commit 965e2154e7682572e99e4c8fd4776dfe8f836b24)
---
 sys/dev/ocs_fc/ocs_hw.c    | 162 +++++++++++++++++++++++++++++++++++++++++++++
 sys/dev/ocs_fc/ocs_hw.h    |   2 +
 sys/dev/ocs_fc/ocs_ioctl.c |   2 +-
 sys/dev/ocs_fc/ocs_mgmt.c  |   5 ++
 sys/dev/ocs_fc/ocs_xport.c |  37 +++++++++++
 sys/dev/ocs_fc/sli4.c      | 110 ++++++++++++++++++++++--------
 sys/dev/ocs_fc/sli4.h      |  42 +++++++++++-
 7 files changed, 330 insertions(+), 30 deletions(-)

diff --git a/sys/dev/ocs_fc/ocs_hw.c b/sys/dev/ocs_fc/ocs_hw.c
index 8fcf31b21514..65fc01f62846 100644
--- a/sys/dev/ocs_fc/ocs_hw.c
+++ b/sys/dev/ocs_fc/ocs_hw.c
@@ -12071,6 +12071,168 @@ ocs_hw_get_def_wwn(ocs_t *ocs, uint32_t chan, uint64_t *wwpn, uint64_t *wwnn)
 	return 0;
 }
 
+uint32_t
+ocs_hw_get_config_persistent_topology(ocs_hw_t *hw)
+{
+        uint32_t topology = OCS_HW_TOPOLOGY_AUTO;
+	sli4_t *sli = &hw->sli;
+
+        if (!sli_persist_topology_enabled(sli))
+                return topology;
+
+        switch (sli->config.pt) {
+                case SLI4_INIT_LINK_F_P2P_ONLY:
+                        topology = OCS_HW_TOPOLOGY_NPORT;
+                        break;
+                case SLI4_INIT_LINK_F_FCAL_ONLY:
+                        topology = OCS_HW_TOPOLOGY_LOOP;
+                        break;
+                default:
+                        break;
+        }
+
+        return topology;
+}
+
+/*
+ * @brief Persistent topology configuration callback argument.
+ */
+typedef struct ocs_hw_persistent_topo_cb_arg {
+	ocs_sem_t semaphore;
+	int32_t status;
+} ocs_hw_persistent_topo_cb_arg_t;
+
+/*
+ * @brief Called after the completion of set persistent topology request
+ *
+ * @par Description
+ * This is callback fn for the set_persistent_topology
+ * function. This callback is called when the common feature mbx cmd
+ * completes.
+ *
+ * @param hw Hardware context.
+ * @param status The status from the MQE.
+ * @param mqe Pointer to mailbox command buffer.
+ * @param arg Pointer to a callback argument.
+ *
+ * @return 0 on success, non-zero otherwise
+ */
+static int32_t
+ocs_hw_set_persistent_topolgy_cb(ocs_hw_t *hw, int32_t status, uint8_t *mqe, void *arg)
+{
+	ocs_hw_persistent_topo_cb_arg_t *req = (ocs_hw_persistent_topo_cb_arg_t *)arg;
+
+	req->status = status;
+
+	ocs_sem_v(&req->semaphore);
+
+	return 0;
+}
+
+/**
+ * @brief Set persistent topology
+ *
+ * Sets the persistent topology(PT) feature using
+ * COMMON_SET_FEATURES cmd. If mbx cmd succeeds, update the
+ * topology into sli config. PT stores the value to be set into link_flags
+ * of the cmd INIT_LINK, to bring up the link.
+ *
+ * SLI specs defines following for PT:
+ *     When TF is set to 0:
+ *       0 Reserved
+ *       1 Attempt point-to-point initialization (direct attach or Fabric topology).
+ *       2 Attempt FC-AL loop initialization.
+ *       3 Reserved
+ *
+ *      When TF is set to 1:
+ *       0 Attempt FC-AL loop initialization; if it fails, attempt point-to-point initialization.
+ *       1 Attempt point-to-point initialization; if it fails, attempt FC-AL loop initialization.
+ *       2 Reserved
+ *      3 Reserved
+ *
+ *     Note: Topology failover is only available on Lancer G5. This command will fail
+ *     if TF is set to 1 on any other ASICs
+ *
+ * @param hw Pointer to hw
+ * @param topology topology value to be set, provided through
+ *        elxsdkutil set-topology cmd
+ *
+ * @return Returns 0 on success, or a non-zero value on failure.
+ */
+ocs_hw_rtn_e
+ocs_hw_set_persistent_topology(ocs_hw_t *hw, uint32_t topology, uint32_t opts)
+{
+	ocs_hw_rtn_e rc = OCS_HW_RTN_SUCCESS;
+	uint8_t buf[SLI4_BMBX_SIZE];
+	sli4_req_common_set_features_persistent_topo_param_t param;
+	ocs_hw_persistent_topo_cb_arg_t request;
+
+	ocs_memset(&param, 0, sizeof(param));
+	param.persistent_topo = topology;
+
+	switch (topology) {
+	case OCS_HW_TOPOLOGY_AUTO:
+		if (sli_get_asic_type(&hw->sli) == SLI4_ASIC_TYPE_LANCER) {
+			param.persistent_topo = SLI4_INIT_LINK_F_P2P_FAIL_OVER;
+			param.topo_failover = 1;
+		} else {
+			param.persistent_topo = SLI4_INIT_LINK_F_P2P_ONLY;;
+			param.topo_failover = 0;
+		}
+		break;
+
+	case OCS_HW_TOPOLOGY_NPORT:
+		param.persistent_topo = SLI4_INIT_LINK_F_P2P_ONLY;
+		param.topo_failover = 0;
+		break;
+
+	case OCS_HW_TOPOLOGY_LOOP:
+		param.persistent_topo = SLI4_INIT_LINK_F_FCAL_ONLY;
+		param.topo_failover = 0;
+		break;
+
+	default:
+		ocs_log_err(hw->os, "unsupported topology %#x\n", topology);
+		return -1;
+	}
+
+	ocs_sem_init(&request.semaphore, 0, "set_persistent_topo");
+
+	/* build the set_features command */
+	sli_cmd_common_set_features(&hw->sli, buf, SLI4_BMBX_SIZE,
+		SLI4_SET_FEATURES_PERSISTENT_TOPOLOGY, sizeof(param), &param);
+
+	if (opts == OCS_CMD_POLL) {
+		rc = ocs_hw_command(hw, buf, OCS_CMD_POLL, NULL, NULL);
+		if (rc) {
+			ocs_log_err(hw->os, "Failed to set persistent topology, rc: %#x\n", rc);
+			return rc;
+		}
+	} else {
+
+		// there's no response for this feature command
+		rc = ocs_hw_command(hw, buf, OCS_CMD_NOWAIT, ocs_hw_set_persistent_topolgy_cb, &request);
+		if (rc) {
+			ocs_log_err(hw->os, "Failed to set persistent topology, rc: %#x\n", rc);
+			return rc;
+		}
+
+		if (ocs_sem_p(&request.semaphore, OCS_SEM_FOREVER)) {
+			ocs_log_err(hw->os, "ocs_sem_p failed\n");
+			return -ENXIO;
+		}
+
+		if (request.status) {
+			ocs_log_err(hw->os, "set persistent topology failed; status: %d\n", request.status);
+			return -EFAULT;
+		}
+	}
+
+	sli_config_persistent_topology(&hw->sli, &param);
+
+	return rc;
+}
+
 /**
  * @page fc_hw_api_overview HW APIs
  * - @ref devInitShutdown
diff --git a/sys/dev/ocs_fc/ocs_hw.h b/sys/dev/ocs_fc/ocs_hw.h
index 14fe4bbd3e8b..287e45acd539 100644
--- a/sys/dev/ocs_fc/ocs_hw.h
+++ b/sys/dev/ocs_fc/ocs_hw.h
@@ -1374,6 +1374,8 @@ extern ocs_hw_io_t * ocs_hw_io_lookup(ocs_hw_t *hw, uint32_t indicator);
 extern uint32_t ocs_hw_xri_move_to_port_owned(ocs_hw_t *hw, uint32_t num_xri);
 extern ocs_hw_rtn_e ocs_hw_xri_move_to_host_owned(ocs_hw_t *hw, uint8_t num_xri);
 extern int32_t ocs_hw_reque_xri(ocs_hw_t *hw, ocs_hw_io_t *io);
+ocs_hw_rtn_e ocs_hw_set_persistent_topology(ocs_hw_t *hw, uint32_t topology, uint32_t opts);
+extern uint32_t ocs_hw_get_config_persistent_topology(ocs_hw_t *hw);
 
 typedef struct {
 	/* structure elements used by HW */
diff --git a/sys/dev/ocs_fc/ocs_ioctl.c b/sys/dev/ocs_fc/ocs_ioctl.c
index c0576bd6600f..19407d413d53 100644
--- a/sys/dev/ocs_fc/ocs_ioctl.c
+++ b/sys/dev/ocs_fc/ocs_ioctl.c
@@ -612,7 +612,7 @@ ocs_ioctl(struct cdev *cdev, u_long cmd, caddr_t addr, int flag, struct thread *
 			return -EFAULT;
 		}
 
-		req->result = ocs_mgmt_set(ocs, req->name, req->value);
+		req->result = ocs_mgmt_set(ocs, name, value);
 
 		break;
 	}
diff --git a/sys/dev/ocs_fc/ocs_mgmt.c b/sys/dev/ocs_fc/ocs_mgmt.c
index ff0e028caea2..f9db394c4f20 100644
--- a/sys/dev/ocs_fc/ocs_mgmt.c
+++ b/sys/dev/ocs_fc/ocs_mgmt.c
@@ -1807,6 +1807,11 @@ set_configured_topology(ocs_t *ocs, char *name, char *value)
 		if (hw_rc != OCS_HW_RTN_SUCCESS) {
 			ocs_log_test(ocs, "Topology set failed\n");
 			result = 1;
+		} else {
+			// Set the persistent topology before port is online
+			hw_rc = ocs_hw_set_persistent_topology(&ocs->hw, topo, OCS_CMD_NOWAIT);
+			if (hw_rc != OCS_HW_RTN_SUCCESS)
+				ocs_log_err(ocs, "Set persistent topology feature failed: %d\n", hw_rc);
 		}
 
 		/* If we failed to set the topology we still want to try to bring
diff --git a/sys/dev/ocs_fc/ocs_xport.c b/sys/dev/ocs_fc/ocs_xport.c
index 6b5ff6fa5b48..6170413a3d89 100644
--- a/sys/dev/ocs_fc/ocs_xport.c
+++ b/sys/dev/ocs_fc/ocs_xport.c
@@ -371,6 +371,36 @@ ocs_xport_initialize_auto_xfer_ready(ocs_xport_t *xport)
 	return 0;
 }
 
+/**
+ * @brief Initialize link topology config
+ *
+ * @par Description
+ * Topology can be fetched from mod-param or Persistet Topology(PT).
+ *  a. Mod-param value is used when the value is 1(P2P) or 2(LOOP).
+ *  a. PT is used if mod-param is not provided( i.e, default value of AUTO)
+ * Also, if mod-param is used, update PT.
+ *
+ * @param ocs Pointer to ocs
+ *
+ * @return Returns 0 on success, or a non-zero value on failure.
+ */
+static int
+ocs_topology_setup(ocs_t *ocs)
+{
+        uint32_t topology;
+
+        if (ocs->topology == OCS_HW_TOPOLOGY_AUTO) {
+                topology = ocs_hw_get_config_persistent_topology(&ocs->hw);
+        }  else {
+                topology = ocs->topology;
+                /* ignore failure here. link will come-up either in auto mode
+                 * if PT is not supported or last saved PT value */
+                ocs_hw_set_persistent_topology(&ocs->hw, topology, OCS_CMD_POLL);
+        }
+
+        return ocs_hw_set(&ocs->hw, OCS_HW_TOPOLOGY, topology);
+}
+
 /**
  * @brief Initializes the device.
  *
@@ -453,6 +483,13 @@ ocs_xport_initialize(ocs_xport_t *xport)
 		}
 	}
 
+	 /* Setup persistent topology based on topology mod-param value */
+        rc = ocs_topology_setup(ocs);
+        if (rc) {
+                ocs_log_err(ocs, "%s: Can't set the toplogy\n", ocs->desc);
+                return -1;
+        }
+
 	if (ocs_hw_set(&ocs->hw, OCS_HW_TOPOLOGY, ocs->topology) != OCS_HW_RTN_SUCCESS) {
 		ocs_log_err(ocs, "%s: Can't set the toplogy\n", ocs->desc);
 		return -1;
diff --git a/sys/dev/ocs_fc/sli4.c b/sys/dev/ocs_fc/sli4.c
index b40c4cfa2946..e4ce20c9beac 100644
--- a/sys/dev/ocs_fc/sli4.c
+++ b/sys/dev/ocs_fc/sli4.c
@@ -764,6 +764,63 @@ sli_cmd_fw_initialize(sli4_t *sli4, void *buf, size_t size)
 	return sizeof(sli4_fw_initialize);
 }
 
+/**
+ * @ingroup sli
+ * @brief update INIT_LINK flags with the sli config topology.
+ *
+ * @param sli4 SLI context pointer.
+ * @param init_link Pointer to the init link command
+ *
+ * @return Returns 0 on success, -1 on failure
+ */
+static int32_t
+sli4_set_link_flags_config_topo(sli4_t *sli4, sli4_cmd_init_link_t *init_link)
+{
+
+	switch (sli4->config.topology) {
+	case SLI4_READ_CFG_TOPO_FC:
+		// Attempt P2P but failover to FC-AL
+		init_link->link_flags.enable_topology_failover = TRUE;
+		init_link->link_flags.topology = SLI4_INIT_LINK_F_P2P_FAIL_OVER;
+		break;
+	case SLI4_READ_CFG_TOPO_FC_AL:
+		init_link->link_flags.topology = SLI4_INIT_LINK_F_FCAL_ONLY;
+		return (!sli_fcal_is_speed_supported(init_link->link_speed_selection_code));
+
+	case SLI4_READ_CFG_TOPO_FC_DA:
+		init_link->link_flags.topology = FC_TOPOLOGY_P2P;
+		break;
+	default:
+		ocs_log_err(sli4->os, "unsupported topology %#x\n", sli4->config.topology);
+		return -1;
+	}
+
+	return 0;
+}
+
+/**
+ * @ingroup sli
+ * @brief update INIT_LINK flags with the persistent topology.
+ * PT stores value in compatible form, directly assign to link_flags
+ *
+ * @param sli4 SLI context pointer.
+ * @param init_link Pointer to the init link command
+ *
+ * @return Returns 0 on success, -1 on failure
+ */
+static int32_t
+sli4_set_link_flags_persistent_topo(sli4_t *sli4, sli4_cmd_init_link_t *init_link)
+{
+	if ((sli4->config.pt == SLI4_INIT_LINK_F_FCAL_ONLY) &&
+	    (!sli_fcal_is_speed_supported(init_link->link_speed_selection_code)))
+		return -1;
+
+	init_link->link_flags.enable_topology_failover = sli4->config.tf;
+	init_link->link_flags.topology = sli4->config.pt;
+
+	return 0;
+}
+
 /**
  * @ingroup sli
  * @brief Write an INIT_LINK command to the provided buffer.
@@ -780,6 +837,7 @@ int32_t
 sli_cmd_init_link(sli4_t *sli4, void *buf, size_t size, uint32_t speed, uint8_t reset_alpa)
 {
 	sli4_cmd_init_link_t	*init_link = buf;
+	int32_t rc = 0;
 
 	ocs_memset(buf, 0, size);
 
@@ -805,41 +863,23 @@ sli_cmd_init_link(sli4_t *sli4, void *buf, size_t size, uint32_t speed, uint8_t
 			return 0;
 		}
 
-		switch (sli4->config.topology) {
-		case SLI4_READ_CFG_TOPO_FC:
-			/* Attempt P2P but failover to FC-AL */
-			init_link->link_flags.enable_topology_failover = TRUE;
-
-			if (sli_get_asic_type(sli4) == SLI4_ASIC_TYPE_LANCER)
-				init_link->link_flags.topology = SLI4_INIT_LINK_F_FCAL_FAIL_OVER;
-			else
-				init_link->link_flags.topology = SLI4_INIT_LINK_F_P2P_FAIL_OVER;
-
-			break;
-		case SLI4_READ_CFG_TOPO_FC_AL:
-			init_link->link_flags.topology = SLI4_INIT_LINK_F_FCAL_ONLY;
-			if ((init_link->link_speed_selection_code == FC_LINK_SPEED_16G) ||
-			    (init_link->link_speed_selection_code == FC_LINK_SPEED_32G)) {
-				ocs_log_test(sli4->os, "unsupported FC-AL speed %d\n", speed);
-				return 0;
-			}
-			break;
-		case SLI4_READ_CFG_TOPO_FC_DA:
-			init_link->link_flags.topology = FC_TOPOLOGY_P2P;
-			break;
-		default:
-			ocs_log_test(sli4->os, "unsupported topology %#x\n", sli4->config.topology);
-			return 0;
-		}
-
 		init_link->link_flags.unfair = FALSE;
 		init_link->link_flags.skip_lirp_lilp = FALSE;
 		init_link->link_flags.gen_loop_validity_check = FALSE;
 		init_link->link_flags.skip_lisa = FALSE;
 		init_link->link_flags.select_hightest_al_pa = FALSE;
+
+		//update topology in the link flags for link bring up
+		ocs_log_info(sli4->os, "bring up link with topology: %d, PTV: %d, TF: %d, PT: %d \n",
+			     sli4->config.topology, sli4->config.ptv, sli4->config.tf, sli4->config.pt);
+		if (sli4->config.ptv)
+			rc = sli4_set_link_flags_persistent_topo(sli4, init_link);
+		else
+			rc = sli4_set_link_flags_config_topo(sli4, init_link);
+
 	}
 
-	return sizeof(sli4_cmd_init_link_t);
+	return rc ? 0 : sizeof(sli4_cmd_init_link_t);
 }
 
 /**
@@ -3724,6 +3764,14 @@ sli_get_config(sli4_t *sli4)
 		}
 
 		sli4->config.topology = read_config->topology;
+		sli4->config.ptv = read_config->ptv;
+		if (sli4->config.ptv){
+			sli4->config.tf = read_config->tf;
+			sli4->config.pt = read_config->pt;
+		}
+		ocs_log_info(sli4->os, "Persistent Topology: PTV: %d, TF: %d, PT: %d \n",
+			     sli4->config.topology, sli4->config.ptv, sli4->config.tf, sli4->config.pt);
+
 		switch (sli4->config.topology) {
 		case SLI4_READ_CFG_TOPO_FCOE:
 			ocs_log_debug(sli4->os, "FCoE\n");
@@ -4162,6 +4210,12 @@ sli_setup(sli4_t *sli4, ocs_os_handle_t os, sli4_port_type_e port_type)
 	return 0;
 }
 
+bool
+sli_persist_topology_enabled(sli4_t *sli4)
+{
+        return (sli4->config.ptv);
+}
+
 int32_t
 sli_init(sli4_t *sli4)
 {
diff --git a/sys/dev/ocs_fc/sli4.h b/sys/dev/ocs_fc/sli4.h
index 54e1915a715c..c8bb9ab5c015 100644
--- a/sys/dev/ocs_fc/sli4.h
+++ b/sys/dev/ocs_fc/sli4.h
@@ -794,7 +794,10 @@ typedef struct sli4_res_read_config_s {
 #if BYTE_ORDER == LITTLE_ENDIAN
 	uint32_t	:31,
 			ext:1;		/** Resource Extents */
-	uint32_t	:24,
+	uint32_t	:20,
+			pt:2,
+			tf:1,
+			ptv:1,
 			topology:8;
 	uint32_t	rsvd3;
 	uint32_t	e_d_tov:16,
@@ -2538,6 +2541,7 @@ typedef struct sli4_res_common_set_dump_location_s {
 #define SLI4_SET_FEATURES_ENABLE_MULTI_RECEIVE_QUEUE	0x0D
 #define SLI4_SET_FEATURES_SET_FTD_XFER_HINT		0x0F
 #define SLI4_SET_FEATURES_SLI_PORT_HEALTH_CHECK		0x11
+#define SLI4_SET_FEATURES_PERSISTENT_TOPOLOGY          0x20
 
 typedef struct sli4_req_common_set_features_s {
 	sli4_req_hdr_t	hdr;
@@ -2615,6 +2619,16 @@ typedef struct sli4_req_common_set_features_set_fdt_xfer_hint_s {
 #endif
 } sli4_req_common_set_features_set_fdt_xfer_hint_t;
 
+typedef struct sli4_req_common_set_features_persistent_topo_param_s {
+#if BYTE_ORDER == LITTLE_ENDIAN
+       uint32_t        persistent_topo:2,
+                       topo_failover:1,
+                       :29;
+#else
+#error big endian version not defined
+#endif
+} sli4_req_common_set_features_persistent_topo_param_t;
+
 /**
  * @brief DMTF_EXEC_CLP_CMD
  */
@@ -3228,6 +3242,10 @@ typedef struct sli4_s {
 		uint16_t		rq_min_buf_size;
 		uint32_t		rq_max_buf_size;
 		uint8_t			topology;
+		uint8_t                 pt:4,
+					tf:1,
+					ptv:1,
+					:2;
 		uint8_t			wwpn[8];
 		uint8_t			wwnn[8];
 		uint32_t		fw_rev[2];
@@ -3565,6 +3583,13 @@ sli_set_topology(sli4_t *sli4, uint32_t value)
 	return rc;
 }
 
+static inline void
+sli_config_persistent_topology(sli4_t *sli4, sli4_req_common_set_features_persistent_topo_param_t *req)
+{
+	sli4->config.pt = req->persistent_topo;
+	sli4->config.tf = req->topo_failover;
+}
+
 static inline uint16_t
 sli_get_link_module_type(sli4_t *sli4)
 {
@@ -3598,6 +3623,19 @@ sli_convert_mask_to_count(uint32_t method, uint32_t mask)
 	return count;
 }
 
+static inline bool
+sli_fcal_is_speed_supported(uint32_t link_speed)
+{
+	if ((link_speed == FC_LINK_SPEED_16G) ||
+	    (link_speed == FC_LINK_SPEED_32G) ||
+	    (link_speed >= FC_LINK_SPEED_AUTO_32_16)) {
+		ocs_log_err(NULL, "unsupported FC-AL speed (speed_code: %d)\n", link_speed);
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
 /**
  * @brief Common Create Queue function prototype
  */
@@ -5569,6 +5607,8 @@ extern int32_t sli_xmit_bls_rsp64_wqe(sli4_t *, void *, size_t, sli_bls_payload_
 extern int32_t sli_xmit_els_rsp64_wqe(sli4_t *, void *, size_t, ocs_dma_t *, uint32_t, uint16_t, uint16_t, uint16_t, uint16_t, ocs_remote_node_t *, uint32_t, uint32_t);
 extern int32_t sli_requeue_xri_wqe(sli4_t *, void *, size_t, uint16_t, uint16_t, uint16_t);
 extern void sli4_cmd_lowlevel_set_watchdog(sli4_t *sli4, void *buf, size_t size, uint16_t timeout);
+extern bool sli_persist_topology_enabled(sli4_t *sli4);
+
 
 /**
  * @ingroup sli_fc