svn commit: r330607 - in head/sys/dev/mlx5: . mlx5_core mlx5_en
Hans Petter Selasky
hselasky at FreeBSD.org
Wed Mar 7 15:17:38 UTC 2018
Author: hselasky
Date: Wed Mar 7 15:17:36 2018
New Revision: 330607
URL: https://svnweb.freebsd.org/changeset/base/330607
Log:
Implement rate limit per traffic class in mlx5core.
Add support for rate limiting traffic class via sysctl.
Submitted by: Slava Shwartsman <slavash at mellanox.com>
MFC after: 1 week
Sponsored by: Mellanox Technologies
Modified:
head/sys/dev/mlx5/mlx5_core/mlx5_port.c
head/sys/dev/mlx5/mlx5_en/en.h
head/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c
head/sys/dev/mlx5/mlx5_ifc.h
head/sys/dev/mlx5/port.h
Modified: head/sys/dev/mlx5/mlx5_core/mlx5_port.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_core/mlx5_port.c Wed Mar 7 15:03:11 2018 (r330606)
+++ head/sys/dev/mlx5/mlx5_core/mlx5_port.c Wed Mar 7 15:17:36 2018 (r330607)
@@ -860,6 +860,89 @@ int mlx5_query_port_cong_params(struct mlx5_core_dev *
out, out_size);
}
+static int mlx5_query_port_qetcr_reg(struct mlx5_core_dev *mdev, u32 *out,
+ int outlen)
+{
+ u32 in[MLX5_ST_SZ_DW(qtct_reg)];
+
+ if (!MLX5_CAP_GEN(mdev, ets))
+ return -ENOTSUPP;
+
+ memset(in, 0, sizeof(in));
+ return mlx5_core_access_reg(mdev, in, sizeof(in), out, outlen,
+ MLX5_REG_QETCR, 0, 0);
+}
+
+int mlx5_max_tc(struct mlx5_core_dev *mdev)
+{
+ u8 num_tc = MLX5_CAP_GEN(mdev, max_tc) ? : 8;
+
+ return num_tc - 1;
+}
+EXPORT_SYMBOL_GPL(mlx5_max_tc);
+
+static int mlx5_set_port_qetcr_reg(struct mlx5_core_dev *mdev, u32 *in,
+ int inlen)
+{
+ u32 out[MLX5_ST_SZ_DW(qtct_reg)];
+
+ if (!MLX5_CAP_GEN(mdev, ets))
+ return -ENOTSUPP;
+
+ return mlx5_core_access_reg(mdev, in, inlen, out, sizeof(out),
+ MLX5_REG_QETCR, 0, 1);
+}
+
+int mlx5_query_port_tc_rate_limit(struct mlx5_core_dev *mdev,
+ u8 *max_bw_value,
+ u8 *max_bw_units)
+{
+ u32 out[MLX5_ST_SZ_DW(qetc_reg)];
+ void *ets_tcn_conf;
+ int err;
+ int i;
+
+ err = mlx5_query_port_qetcr_reg(mdev, out, sizeof(out));
+ if (err)
+ return err;
+
+ for (i = 0; i <= mlx5_max_tc(mdev); i++) {
+ ets_tcn_conf = MLX5_ADDR_OF(qetc_reg, out, tc_configuration[i]);
+
+ max_bw_value[i] = MLX5_GET(ets_tcn_config_reg, ets_tcn_conf,
+ max_bw_value);
+ max_bw_units[i] = MLX5_GET(ets_tcn_config_reg, ets_tcn_conf,
+ max_bw_units);
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_port_tc_rate_limit);
+
+int mlx5_modify_port_tc_rate_limit(struct mlx5_core_dev *mdev,
+ const u8 *max_bw_value,
+ const u8 *max_bw_units)
+{
+ u32 in[MLX5_ST_SZ_DW(qetc_reg)] = {};
+ void *ets_tcn_conf;
+ int i;
+
+ MLX5_SET(qetc_reg, in, port_number, 1);
+
+ for (i = 0; i <= mlx5_max_tc(mdev); i++) {
+ ets_tcn_conf = MLX5_ADDR_OF(qetc_reg, in, tc_configuration[i]);
+
+ MLX5_SET(ets_tcn_config_reg, ets_tcn_conf, r, 1);
+ MLX5_SET(ets_tcn_config_reg, ets_tcn_conf, max_bw_units,
+ max_bw_units[i]);
+ MLX5_SET(ets_tcn_config_reg, ets_tcn_conf, max_bw_value,
+ max_bw_value[i]);
+ }
+
+ return mlx5_set_port_qetcr_reg(mdev, in, sizeof(in));
+}
+EXPORT_SYMBOL_GPL(mlx5_modify_port_tc_rate_limit);
+
int mlx5_modify_port_cong_params(struct mlx5_core_dev *mdev,
void *in, int in_size)
{
Modified: head/sys/dev/mlx5/mlx5_en/en.h
==============================================================================
--- head/sys/dev/mlx5/mlx5_en/en.h Wed Mar 7 15:03:11 2018 (r330606)
+++ head/sys/dev/mlx5/mlx5_en/en.h Wed Mar 7 15:17:36 2018 (r330607)
@@ -70,6 +70,8 @@
#include <dev/mlx5/mlx5_core/transobj.h>
#include <dev/mlx5/mlx5_core/mlx5_core.h>
+#define IEEE_8021QAZ_MAX_TCS 8
+
#define MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE 0x7
#define MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE 0xa
#define MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE 0xe
@@ -114,6 +116,9 @@
(MLX5E_MAX_TX_HEADER - sizeof(struct mlx5e_tx_wqe) + \
sizeof(((struct mlx5e_tx_wqe *)0)->eth.inline_hdr_start)) /* bytes */
+#define MLX5E_100MB (100000)
+#define MLX5E_1GB (1000000)
+
MALLOC_DECLARE(M_MLX5EN);
struct mlx5_core_dev;
@@ -417,11 +422,13 @@ struct mlx5e_params {
m(+1, u64 mc_local_lb, "mc_local_lb", "0: Local multicast loopback enabled 1: Disabled") \
m(+1, u64 uc_local_lb, "uc_local_lb", "0: Local unicast loopback enabled 1: Disabled")
+
#define MLX5E_PARAMS_NUM (0 MLX5E_PARAMS(MLX5E_STATS_COUNT))
struct mlx5e_params_ethtool {
u64 arg [0];
MLX5E_PARAMS(MLX5E_STATS_VAR)
+ u64 max_bw_value[IEEE_8021QAZ_MAX_TCS];
};
/* EEPROM Standards for plug in modules */
@@ -629,6 +636,12 @@ enum {
MLX5E_STATE_OPENED,
};
+enum {
+ MLX5_BW_NO_LIMIT = 0,
+ MLX5_100_MBPS_UNIT = 3,
+ MLX5_GBPS_UNIT = 4,
+};
+
struct mlx5e_vlan_db {
unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
struct mlx5_flow_rule *active_vlans_ft_rule[VLAN_N_VID];
@@ -747,6 +760,19 @@ struct mlx5e_eeprom {
int page_valid;
u32 *data;
};
+
+/*
+ * This structure contains rate limit extension to the IEEE 802.1Qaz ETS
+ * managed object.
+ * Values are 64 bits long and specified in Kbps to enable usage over both
+ * slow and very fast networks.
+ *
+ * @tc_maxrate: maximal tc tx bandwidth indexed by traffic class
+ */
+struct ieee_maxrate {
+ __u64 tc_maxrate[IEEE_8021QAZ_MAX_TCS];
+};
+
#define MLX5E_FLD_MAX(typ, fld) ((1ULL << __mlx5_bit_sz(typ, fld)) - 1ULL)
Modified: head/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c Wed Mar 7 15:03:11 2018 (r330606)
+++ head/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c Wed Mar 7 15:17:36 2018 (r330607)
@@ -84,6 +84,97 @@ mlx5e_ethtool_sync_tx_completion_fact(struct mlx5e_pri
priv->params_ethtool.tx_completion_fact = max;
}
+static int
+mlx5e_getmaxrate(struct mlx5e_priv *priv)
+{
+ struct mlx5_core_dev *mdev = priv->mdev;
+ u8 max_bw_unit[IEEE_8021QAZ_MAX_TCS];
+ u8 max_bw_value[IEEE_8021QAZ_MAX_TCS];
+ int err;
+ int i;
+
+ PRIV_LOCK(priv);
+ err = -mlx5_query_port_tc_rate_limit(mdev, max_bw_value, max_bw_unit);
+ if (err)
+ goto done;
+
+ for (i = 0; i <= mlx5_max_tc(mdev); i++) {
+ switch (max_bw_unit[i]) {
+ case MLX5_100_MBPS_UNIT:
+ priv->params_ethtool.max_bw_value[i] = max_bw_value[i] * MLX5E_100MB;
+ break;
+ case MLX5_GBPS_UNIT:
+ priv->params_ethtool.max_bw_value[i] = max_bw_value[i] * MLX5E_1GB;
+ break;
+ case MLX5_BW_NO_LIMIT:
+ priv->params_ethtool.max_bw_value[i] = 0;
+ break;
+ default:
+ priv->params_ethtool.max_bw_value[i] = -1;
+ WARN_ONCE(true, "non-supported BW unit");
+ break;
+ }
+ }
+done:
+ PRIV_UNLOCK(priv);
+ return (err);
+}
+
+static int
+mlx5e_tc_maxrate_handler(SYSCTL_HANDLER_ARGS)
+{
+ struct mlx5e_priv *priv = arg1;
+ int prio_index = arg2;
+ struct mlx5_core_dev *mdev = priv->mdev;
+ u8 max_bw_unit[IEEE_8021QAZ_MAX_TCS];
+ u8 max_bw_value[IEEE_8021QAZ_MAX_TCS];
+ int i, err;
+ u64 bw_val;
+ u64 result = priv->params_ethtool.max_bw_value[prio_index];
+ const u64 upper_limit_mbps = 255 * MLX5E_100MB;
+ const u64 upper_limit_gbps = 255 * MLX5E_1GB;
+
+ PRIV_LOCK(priv);
+ err = sysctl_handle_64(oidp, &result, 0, req);
+ if (err || !req->newptr ||
+ result == priv->params_ethtool.max_bw_value[prio_index])
+ goto done;
+
+ if (result % MLX5E_100MB) {
+ err = ERANGE;
+ goto done;
+ }
+
+ memset(max_bw_value, 0, sizeof(max_bw_value));
+ memset(max_bw_unit, 0, sizeof(max_bw_unit));
+
+ for (i = 0; i <= mlx5_max_tc(mdev); i++) {
+ bw_val = (i == prio_index) ? result : priv->params_ethtool.max_bw_value[i];
+
+ if (!bw_val) {
+ max_bw_unit[i] = MLX5_BW_NO_LIMIT;
+ } else if (bw_val > upper_limit_gbps) {
+ result = 0;
+ max_bw_unit[i] = MLX5_BW_NO_LIMIT;
+ } else if (bw_val <= upper_limit_mbps) {
+ max_bw_value[i] = howmany(bw_val, MLX5E_100MB);
+ max_bw_unit[i] = MLX5_100_MBPS_UNIT;
+ } else {
+ max_bw_value[i] = howmany(bw_val, MLX5E_1GB);
+ max_bw_unit[i] = MLX5_GBPS_UNIT;
+ }
+ }
+
+ err = -mlx5_modify_port_tc_rate_limit(mdev, max_bw_value, max_bw_unit);
+ if (err)
+ goto done;
+
+ priv->params_ethtool.max_bw_value[prio_index] = result;
+done:
+ PRIV_UNLOCK(priv);
+ return (err);
+}
+
#define MLX5_PARAM_OFFSET(n) \
__offsetof(struct mlx5e_priv, params_ethtool.n)
@@ -734,9 +825,11 @@ mlx5e_create_diagnostics(struct mlx5e_priv *priv)
void
mlx5e_create_ethtool(struct mlx5e_priv *priv)
{
- struct sysctl_oid *node;
+ struct mlx5_core_dev *mdev = priv->mdev;
+ struct sysctl_oid *node, *qos_node;
const char *pnameunit;
unsigned x;
+ int i;
/* set some defaults */
priv->params_ethtool.tx_queue_size_max = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE;
@@ -830,4 +923,25 @@ mlx5e_create_ethtool(struct mlx5e_priv *priv)
/* Diagnostics support */
mlx5e_create_diagnostics(priv);
+
+ /* create qos node */
+ qos_node = SYSCTL_ADD_NODE(&priv->sysctl_ctx,
+ SYSCTL_CHILDREN(node), OID_AUTO,
+ "qos", CTLFLAG_RW, NULL, "Quality Of Service configuration");
+ if (node == NULL)
+ return;
+
+ /* Prioriry rate limit support */
+ if (mlx5e_getmaxrate(priv))
+ return;
+
+ for (i = 0; i <= mlx5_max_tc(mdev); i++) {
+ char name[32];
+ snprintf(name, sizeof(name), "tc_%d_max_rate", i);
+ SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(qos_node),
+ OID_AUTO, name, CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE,
+ priv, i, mlx5e_tc_maxrate_handler, "QU",
+ "Max rate for priority, specified in kilobits, where kilo=1000, \
+ max_rate must be divisible by 100000");
+ }
}
Modified: head/sys/dev/mlx5/mlx5_ifc.h
==============================================================================
--- head/sys/dev/mlx5/mlx5_ifc.h Wed Mar 7 15:03:11 2018 (r330606)
+++ head/sys/dev/mlx5/mlx5_ifc.h Wed Mar 7 15:17:36 2018 (r330607)
@@ -9096,6 +9096,15 @@ struct mlx5_ifc_ets_global_config_reg_bits {
u8 max_bw_value[0x8];
};
+struct mlx5_ifc_qetc_reg_bits {
+ u8 reserved_at_0[0x8];
+ u8 port_number[0x8];
+ u8 reserved_at_10[0x30];
+
+ struct mlx5_ifc_ets_tcn_config_reg_bits tc_configuration[0x8];
+ struct mlx5_ifc_ets_global_config_reg_bits global_configuration;
+};
+
struct mlx5_ifc_nodnic_mac_filters_bits {
struct mlx5_ifc_mac_address_layout_bits mac_filter0;
Modified: head/sys/dev/mlx5/port.h
==============================================================================
--- head/sys/dev/mlx5/port.h Wed Mar 7 15:03:11 2018 (r330606)
+++ head/sys/dev/mlx5/port.h Wed Mar 7 15:17:36 2018 (r330607)
@@ -144,4 +144,11 @@ int mlx5_query_eeprom(struct mlx5_core_dev *dev, int i
int device_addr, int size, int module_num, u32 *data,
int *size_read);
+int mlx5_max_tc(struct mlx5_core_dev *mdev);
+int mlx5_query_port_tc_rate_limit(struct mlx5_core_dev *mdev,
+ u8 *max_bw_value,
+ u8 *max_bw_units);
+int mlx5_modify_port_tc_rate_limit(struct mlx5_core_dev *mdev,
+ const u8 *max_bw_value,
+ const u8 *max_bw_units);
#endif /* __MLX5_PORT_H__ */
More information about the svn-src-all
mailing list