git: 01fbb86991d5 - main - irdma(4): upgrade to 1.2.17-k
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Fri, 18 Aug 2023 18:30:44 UTC
The branch main has been updated by erj:
URL: https://cgit.FreeBSD.org/src/commit/?id=01fbb86991d524288a785b544c29d9c5ea1b61fb
commit 01fbb86991d524288a785b544c29d9c5ea1b61fb
Author: Bartosz Sobczak <bartosz.sobczak@intel.com>
AuthorDate: 2023-08-18 18:14:33 +0000
Commit: Eric Joyner <erj@FreeBSD.org>
CommitDate: 2023-08-18 18:28:58 +0000
irdma(4): upgrade to 1.2.17-k
Update Intel irdma driver to version 1.2.17-k
Notable changes:
- pf-reset handling improvements, including fixes in communication with if_ice(4)
- avoid racing when handling various events
- adding sw stats sysctls
- hand over pe_criterr handling from ice(4) to irdma(4)
- debug prints adjustments
- fix crash after changes in irdma_add_mqh_ifa_cb
Signed-off-by: Bartosz Sobczak <bartosz.sobczak@intel.com>
Signed-off-by: Eric Joyner <erj@FreeBSD.org>
Reviewed by: erj@
MFC after: 1 month
Sponsored by: Intel Corporation
Differential Revision: https://reviews.freebsd.org/D41425
---
contrib/ofed/libirdma/irdma_defs.h | 25 +-
contrib/ofed/libirdma/irdma_uk.c | 59 +----
contrib/ofed/libirdma/irdma_umain.c | 4 +-
contrib/ofed/libirdma/irdma_user.h | 3 +-
contrib/ofed/libirdma/irdma_uverbs.c | 61 ++---
contrib/ofed/libirdma/osdep.h | 82 +++----
sys/dev/ice/ice_rdma.c | 59 ++++-
sys/dev/ice/ice_rdma.h | 11 +-
sys/dev/ice/ice_rdma_internal.h | 2 +
sys/dev/ice/if_ice_iflib.c | 15 +-
sys/dev/irdma/fbsd_kcompat.c | 282 ++++++++++++++++++++++-
sys/dev/irdma/fbsd_kcompat.h | 16 +-
sys/dev/irdma/icrdma.c | 283 +++++++++++++++--------
sys/dev/irdma/irdma.h | 4 +-
sys/dev/irdma/irdma_cm.c | 303 ++++++++++++------------
sys/dev/irdma/irdma_cm.h | 15 +-
sys/dev/irdma/irdma_ctrl.c | 160 +++++--------
sys/dev/irdma/irdma_defs.h | 87 +++----
sys/dev/irdma/irdma_hmc.c | 6 +-
sys/dev/irdma/irdma_hmc.h | 6 +-
sys/dev/irdma/irdma_hw.c | 99 ++++----
sys/dev/irdma/irdma_kcompat.c | 121 +++++-----
sys/dev/irdma/irdma_main.h | 12 +-
sys/dev/irdma/irdma_pble.c | 16 +-
sys/dev/irdma/irdma_pble.h | 6 +-
sys/dev/irdma/irdma_protos.h | 8 +-
sys/dev/irdma/irdma_puda.c | 8 +-
sys/dev/irdma/irdma_puda.h | 6 +-
sys/dev/irdma/irdma_type.h | 21 +-
sys/dev/irdma/irdma_uda.c | 17 +-
sys/dev/irdma/irdma_uda.h | 4 +-
sys/dev/irdma/irdma_uk.c | 84 +------
sys/dev/irdma/irdma_user.h | 3 +-
sys/dev/irdma/irdma_utils.c | 230 ++++++++++++++-----
sys/dev/irdma/irdma_verbs.c | 431 +++++++++++++++++++----------------
sys/dev/irdma/irdma_verbs.h | 91 ++++++--
sys/dev/irdma/irdma_ws.c | 4 +-
sys/dev/irdma/osdep.h | 61 +++--
38 files changed, 1586 insertions(+), 1119 deletions(-)
diff --git a/contrib/ofed/libirdma/irdma_defs.h b/contrib/ofed/libirdma/irdma_defs.h
index 3d8b59c4b78e..9276490793a0 100644
--- a/contrib/ofed/libirdma/irdma_defs.h
+++ b/contrib/ofed/libirdma/irdma_defs.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -312,7 +312,7 @@
#define IRDMA_GET_CQ_ELEM_AT_OFFSET(_cq, _i, _cqe) \
{ \
- register __u32 offset; \
+ __u32 offset; \
offset = IRDMA_GET_RING_OFFSET((_cq)->cq_ring, _i); \
(_cqe) = (_cq)->cq_base[offset].buf; \
}
@@ -338,7 +338,7 @@
#define IRDMA_RING_MOVE_HEAD(_ring, _retcode) \
{ \
- register u32 size; \
+ u32 size; \
size = (_ring).size; \
if (!IRDMA_RING_FULL_ERR(_ring)) { \
(_ring).head = ((_ring).head + 1) % size; \
@@ -349,7 +349,7 @@
}
#define IRDMA_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \
{ \
- register u32 size; \
+ u32 size; \
size = (_ring).size; \
if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < size) { \
(_ring).head = ((_ring).head + (_count)) % size; \
@@ -360,7 +360,7 @@
}
#define IRDMA_SQ_RING_MOVE_HEAD(_ring, _retcode) \
{ \
- register u32 size; \
+ u32 size; \
size = (_ring).size; \
if (!IRDMA_SQ_RING_FULL_ERR(_ring)) { \
(_ring).head = ((_ring).head + 1) % size; \
@@ -371,7 +371,7 @@
}
#define IRDMA_SQ_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \
{ \
- register u32 size; \
+ u32 size; \
size = (_ring).size; \
if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < (size - 256)) { \
(_ring).head = ((_ring).head + (_count)) % size; \
@@ -457,6 +457,19 @@ enum irdma_qp_wqe_size {
IRDMA_WQE_SIZE_256 = 256,
};
+enum irdma_ws_op_type {
+ IRDMA_WS_OP_TYPE_NODE = 0,
+ IRDMA_WS_OP_TYPE_LEAF_NODE_GROUP,
+};
+
+enum irdma_ws_rate_limit_flags {
+ IRDMA_WS_RATE_LIMIT_FLAGS_VALID = 0x1,
+ IRDMA_WS_NO_RDMA_RATE_LIMIT = 0x2,
+ IRDMA_WS_LEAF_NODE_IS_PART_GROUP = 0x4,
+ IRDMA_WS_TREE_RATE_LIMITING = 0x8,
+ IRDMA_WS_PACING_CONTROL = 0x10,
+};
+
/**
* set_64bit_val - set 64 bit value to hw wqe
* @wqe_words: wqe addr to write
diff --git a/contrib/ofed/libirdma/irdma_uk.c b/contrib/ofed/libirdma/irdma_uk.c
index 97e3ac553c26..a1b796b318f4 100644
--- a/contrib/ofed/libirdma/irdma_uk.c
+++ b/contrib/ofed/libirdma/irdma_uk.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -275,7 +275,8 @@ irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && wqe_quanta == 1 &&
(IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) {
wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem;
- wqe_0[3] = htole64(FIELD_PREP(IRDMAQPSQ_VALID, !qp->swqe_polarity));
+ wqe_0[3] = htole64(FIELD_PREP(IRDMAQPSQ_VALID,
+ qp->swqe_polarity ? 0 : 1));
}
qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id;
qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size;
@@ -683,8 +684,8 @@ irdma_set_mw_bind_wqe(__le64 * wqe,
* @polarity: polarity of wqe valid bit
*/
static void
-irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, u32 num_sges,
- u8 polarity)
+irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list,
+ u32 num_sges, u8 polarity)
{
u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
u32 quanta_bytes_remaining = 8;
@@ -1173,7 +1174,7 @@ irdma_repost_rq_wqes(struct irdma_qp_uk *qp, u32 start_idx,
u32 end_idx)
{
__le64 *dst_wqe, *src_wqe;
- u32 wqe_idx;
+ u32 wqe_idx = 0;
u8 wqe_quanta = qp->rq_wqe_size_multiplier;
bool flip_polarity;
u64 val;
@@ -1480,7 +1481,8 @@ irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
sw_wqe = qp->sq_base[tail].elem;
get_64bit_val(sw_wqe, IRDMA_BYTE_24,
&wqe_qword);
- info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword);
+ info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE,
+ wqe_qword);
IRDMA_RING_SET_TAIL(qp->sq_ring,
tail + qp->sq_wrtrk_array[tail].quanta);
if (info->op_type != IRDMAQP_OP_NOP) {
@@ -1834,6 +1836,9 @@ irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq)
if (polarity != temp)
break;
+ /* Ensure CQE contents are read after valid bit is checked */
+ udma_from_device_barrier();
+
get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx);
if ((void *)(irdma_uintptr) comp_ctx == q)
set_64bit_val(cqe, IRDMA_BYTE_8, 0);
@@ -1845,48 +1850,6 @@ irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq)
return 0;
}
-/**
- * irdma_nop - post a nop
- * @qp: hw qp ptr
- * @wr_id: work request id
- * @signaled: signaled for completion
- * @post_sq: ring doorbell
- */
-int
-irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq)
-{
- __le64 *wqe;
- u64 hdr;
- u32 wqe_idx;
- struct irdma_post_sq_info info = {0};
- u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
-
- info.push_wqe = qp->push_db ? true : false;
- info.wr_id = wr_id;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, &info);
- if (!wqe)
- return ENOSPC;
-
- set_64bit_val(wqe, IRDMA_BYTE_0, 0);
- set_64bit_val(wqe, IRDMA_BYTE_8, 0);
- set_64bit_val(wqe, IRDMA_BYTE_16, 0);
-
- hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
- FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) |
- FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
-
- udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
-
- if (info.push_wqe)
- irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- else if (post_sq)
- irdma_uk_qp_post_wr(qp);
-
- return 0;
-}
-
/**
* irdma_fragcnt_to_quanta_sq - calculate quanta based on fragment count for SQ
* @frag_cnt: number of fragments
diff --git a/contrib/ofed/libirdma/irdma_umain.c b/contrib/ofed/libirdma/irdma_umain.c
index 6c823646b375..9e223cae429f 100644
--- a/contrib/ofed/libirdma/irdma_umain.c
+++ b/contrib/ofed/libirdma/irdma_umain.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2021 - 2022 Intel Corporation
+ * Copyright (c) 2021 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -49,7 +49,7 @@
/**
* Driver version
*/
-char libirdma_version[] = "1.1.11-k";
+char libirdma_version[] = "1.2.17-k";
unsigned int irdma_dbg;
diff --git a/contrib/ofed/libirdma/irdma_user.h b/contrib/ofed/libirdma/irdma_user.h
index 8eb9ae7990e7..8d71a37d7bf4 100644
--- a/contrib/ofed/libirdma/irdma_user.h
+++ b/contrib/ofed/libirdma/irdma_user.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -572,7 +572,6 @@ void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
u32 inline_data, u8 *shift);
int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth);
int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth);
-int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift, u32 *srqdepth);
void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta,
u32 wqe_idx, bool post_sq);
void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);
diff --git a/contrib/ofed/libirdma/irdma_uverbs.c b/contrib/ofed/libirdma/irdma_uverbs.c
index 14efab96a107..c59d409e72cd 100644
--- a/contrib/ofed/libirdma/irdma_uverbs.c
+++ b/contrib/ofed/libirdma/irdma_uverbs.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (C) 2019 - 2022 Intel Corporation
+ * Copyright (C) 2019 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -266,11 +266,13 @@ irdma_ualloc_mw(struct ibv_pd *pd, enum ibv_mw_type type)
if (!mw)
return NULL;
- if (ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), &resp,
- sizeof(resp))) {
+ err = ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), &resp,
+ sizeof(resp));
+ if (err) {
printf("%s: Failed to alloc memory window\n",
__func__);
free(mw);
+ errno = err;
return NULL;
}
@@ -446,8 +448,10 @@ ucreate_cq(struct ibv_context *context,
if (!iwucq)
return NULL;
- if (pthread_spin_init(&iwucq->lock, PTHREAD_PROCESS_PRIVATE)) {
+ ret = pthread_spin_init(&iwucq->lock, PTHREAD_PROCESS_PRIVATE);
+ if (ret) {
free(iwucq);
+ errno = ret;
return NULL;
}
@@ -464,8 +468,10 @@ ucreate_cq(struct ibv_context *context,
iwucq->buf_size = total_size;
info.cq_base = irdma_alloc_hw_buf(total_size);
- if (!info.cq_base)
+ if (!info.cq_base) {
+ ret = ENOMEM;
goto err_cq_base;
+ }
memset(info.cq_base, 0, total_size);
reg_mr_cmd.reg_type = IRDMA_MEMREG_TYPE_CQ;
@@ -476,17 +482,17 @@ ucreate_cq(struct ibv_context *context,
IBV_ACCESS_LOCAL_WRITE, &iwucq->vmr.ibv_mr,
®_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd),
®_mr_resp, sizeof(reg_mr_resp));
- if (ret) {
- errno = ret;
+ if (ret)
goto err_dereg_mr;
- }
iwucq->vmr.ibv_mr.pd = &iwvctx->iwupd->ibv_pd;
if (uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE) {
info.shadow_area = irdma_alloc_hw_buf(IRDMA_DB_SHADOW_AREA_SIZE);
- if (!info.shadow_area)
+ if (!info.shadow_area) {
+ ret = ENOMEM;
goto err_alloc_shadow;
+ }
memset(info.shadow_area, 0, IRDMA_DB_SHADOW_AREA_SIZE);
reg_mr_shadow_cmd.reg_type = IRDMA_MEMREG_TYPE_CQ;
@@ -499,7 +505,6 @@ ucreate_cq(struct ibv_context *context,
®_mr_shadow_resp, sizeof(reg_mr_shadow_resp));
if (ret) {
irdma_free_hw_buf(info.shadow_area, IRDMA_DB_SHADOW_AREA_SIZE);
- errno = ret;
goto err_alloc_shadow;
}
@@ -517,10 +522,8 @@ ucreate_cq(struct ibv_context *context,
&cmd.ibv_cmd, sizeof(cmd.ibv_cmd), sizeof(cmd), &resp.ibv_resp,
sizeof(resp.ibv_resp), sizeof(resp));
attr_ex->cqe = ncqe;
- if (ret) {
- errno = ret;
+ if (ret)
goto err_create_cq;
- }
if (ext_cq)
irdma_ibvcq_ex_fill_priv_funcs(iwucq, attr_ex);
@@ -548,6 +551,7 @@ err_cq_base:
free(iwucq);
+ errno = ret;
return NULL;
}
@@ -1560,7 +1564,8 @@ irdma_ucreate_qp(struct ibv_pd *pd,
memset(iwuqp, 0, sizeof(*iwuqp));
- if (pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE))
+ status = pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE);
+ if (status)
goto err_free_qp;
info.sq_size = info.sq_depth >> info.sq_shift;
@@ -1575,35 +1580,37 @@ irdma_ucreate_qp(struct ibv_pd *pd,
}
iwuqp->recv_sges = calloc(attr->cap.max_recv_sge, sizeof(*iwuqp->recv_sges));
- if (!iwuqp->recv_sges)
+ if (!iwuqp->recv_sges) {
+ status = errno; /* preserve errno */
goto err_destroy_lock;
+ }
info.wqe_alloc_db = (u32 *)iwvctx->db;
info.legacy_mode = iwvctx->legacy_mode;
info.sq_wrtrk_array = calloc(info.sq_depth, sizeof(*info.sq_wrtrk_array));
- if (!info.sq_wrtrk_array)
+ if (!info.sq_wrtrk_array) {
+ status = errno; /* preserve errno */
goto err_free_rsges;
+ }
info.rq_wrid_array = calloc(info.rq_depth, sizeof(*info.rq_wrid_array));
- if (!info.rq_wrid_array)
+ if (!info.rq_wrid_array) {
+ status = errno; /* preserve errno */
goto err_free_sq_wrtrk;
+ }
iwuqp->sq_sig_all = attr->sq_sig_all;
iwuqp->qp_type = attr->qp_type;
status = irdma_vmapped_qp(iwuqp, pd, attr, &info, iwvctx->legacy_mode);
- if (status) {
- errno = status;
+ if (status)
goto err_free_rq_wrid;
- }
iwuqp->qp.back_qp = iwuqp;
iwuqp->qp.lock = &iwuqp->lock;
status = irdma_uk_qp_init(&iwuqp->qp, &info);
- if (status) {
- errno = status;
+ if (status)
goto err_free_vmap_qp;
- }
attr->cap.max_send_wr = (info.sq_depth - IRDMA_SQ_RSVD) >> info.sq_shift;
attr->cap.max_recv_wr = (info.rq_depth - IRDMA_RQ_RSVD) >> info.rq_shift;
@@ -1625,6 +1632,7 @@ err_free_qp:
printf("%s: failed to create QP\n", __func__);
free(iwuqp);
+ errno = status;
return NULL;
}
@@ -2081,11 +2089,10 @@ irdma_ucreate_ah(struct ibv_pd *ibpd, struct ibv_ah_attr *attr)
struct irdma_ucreate_ah_resp resp = {};
int err;
- err = ibv_query_gid(ibpd->context, attr->port_num, attr->grh.sgid_index,
- &sgid);
- if (err) {
+ if (ibv_query_gid(ibpd->context, attr->port_num, attr->grh.sgid_index,
+ &sgid)) {
fprintf(stderr, "irdma: Error from ibv_query_gid.\n");
- errno = err;
+ errno = ENOENT;
return NULL;
}
diff --git a/contrib/ofed/libirdma/osdep.h b/contrib/ofed/libirdma/osdep.h
index 1bbf6573b800..f7b8a96b93d2 100644
--- a/contrib/ofed/libirdma/osdep.h
+++ b/contrib/ofed/libirdma/osdep.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2021 - 2022 Intel Corporation
+ * Copyright (c) 2021 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -46,7 +46,6 @@
#include <sys/bus_dma.h>
#include <sys/endian.h>
-#define ATOMIC atomic_t
#define IOMEM
#define IRDMA_NTOHL(a) ntohl(a)
#define IRDMA_NTOHS(a) ntohs(a)
@@ -80,23 +79,13 @@
#define STATS_TIMER_DELAY 60000
/* a couple of linux size defines */
-#define SZ_128 128
-#define SZ_2K SZ_128 * 16
-#define SZ_1G (SZ_1K * SZ_1K * SZ_1K)
-#define SPEED_1000 1000
-#define SPEED_10000 10000
-#define SPEED_20000 20000
-#define SPEED_25000 25000
-#define SPEED_40000 40000
-#define SPEED_100000 100000
-
#define BIT_ULL(a) (1ULL << (a))
#define min(a, b) ((a) > (b) ? (b) : (a))
#ifndef likely
-#define likely(x) __builtin_expect((x), 1)
+#define likely(x) __builtin_expect((x), 1)
#endif
#ifndef unlikely
-#define unlikely(x) __builtin_expect((x), 0)
+#define unlikely(x) __builtin_expect((x), 0)
#endif
#define __aligned_u64 uint64_t __aligned(8)
@@ -112,7 +101,7 @@
#define irdma_print(S, ...) printf("%s:%d "S, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#define irdma_debug_buf(dev, mask, desc, buf, size) \
do { \
- u32 i; \
+ u32 i; \
if (!((mask) & (dev)->debug_mask)) { \
break; \
} \
@@ -122,20 +111,20 @@ do { \
irdma_debug(dev, mask, "index %03d val: %016lx\n", i, ((unsigned long *)(buf))[i / 8]); \
} while(0)
-#define irdma_debug(h, m, s, ...) \
-do { \
- if (!(h)) { \
- if ((m) == IRDMA_DEBUG_INIT) \
+#define irdma_debug(h, m, s, ...) \
+do { \
+ if (!(h)) { \
+ if ((m) == IRDMA_DEBUG_INIT) \
printf("irdma INIT " s, ##__VA_ARGS__); \
- } else if (((m) & (h)->debug_mask)) { \
- printf("irdma " s, ##__VA_ARGS__); \
- } \
+ } else if (((m) & (h)->debug_mask)) { \
+ printf("irdma " s, ##__VA_ARGS__); \
+ } \
} while (0)
extern unsigned int irdma_dbg;
-#define libirdma_debug(fmt, args...) \
-do { \
- if (irdma_dbg) \
- printf("libirdma-%s: " fmt, __func__, ##args); \
+#define libirdma_debug(fmt, args...) \
+do { \
+ if (irdma_dbg) \
+ printf("libirdma-%s: " fmt, __func__, ##args); \
} while (0)
#define irdma_dev_err(ibdev, fmt, ...) \
pr_err("%s:%s:%d ERR "fmt, (ibdev)->name, __func__, __LINE__, ##__VA_ARGS__)
@@ -144,18 +133,6 @@ do { \
#define irdma_dev_info(a, b, ...) printf(b, ##__VA_ARGS__)
#define irdma_pr_warn printf
-#define dump_struct(s, sz, name) \
-do { \
- unsigned char *a; \
- printf("%s %u", (name), (unsigned int)(sz)); \
- for (a = (unsigned char*)(s); a < (unsigned char *)(s) + (sz) ; a ++) { \
- if ((u64)a % 8 == 0) \
- printf("\n%p ", a); \
- printf("%2x ", *a); \
- } \
- printf("\n"); \
-}while(0)
-
/*
* debug definition end
*/
@@ -179,13 +156,6 @@ struct irdma_sc_vsi;
#define rt_tos2priority(tos) (tos >> 5)
#define ah_attr_to_dmac(attr) ((attr).dmac)
-#define kc_ib_modify_qp_is_ok(cur_state, next_state, type, mask, ll) \
- ib_modify_qp_is_ok(cur_state, next_state, type, mask)
-#define kc_typeq_ib_wr const
-#define kc_ifp_find ip_ifp_find
-#define kc_ifp6_find ip6_ifp_find
-#define kc_rdma_gid_attr_network_type(sgid_attr, gid_type, gid) \
- ib_gid_to_network_type(gid_type, gid)
#define irdma_del_timer_compat(tt) del_timer((tt))
#define IRDMA_TAILQ_FOREACH CK_STAILQ_FOREACH
#define IRDMA_TAILQ_FOREACH_SAFE CK_STAILQ_FOREACH_SAFE
@@ -193,36 +163,36 @@ struct irdma_sc_vsi;
static inline void db_wr32(__u32 val, __u32 *wqe_word)
{
- *wqe_word = val;
+ *wqe_word = val;
}
void *hw_to_dev(struct irdma_hw *hw);
struct irdma_dma_mem {
- void *va;
- u64 pa;
+ void *va;
+ u64 pa;
bus_dma_tag_t tag;
bus_dmamap_t map;
bus_dma_segment_t seg;
bus_size_t size;
- int nseg;
- int flags;
+ int nseg;
+ int flags;
};
struct irdma_virt_mem {
- void *va;
- u32 size;
+ void *va;
+ u32 size;
};
#ifndef verbs_mr
enum ibv_mr_type {
- IBV_MR_TYPE_MR,
- IBV_MR_TYPE_NULL_MR,
+ IBV_MR_TYPE_MR,
+ IBV_MR_TYPE_NULL_MR,
};
struct verbs_mr {
- struct ibv_mr ibv_mr;
- enum ibv_mr_type mr_type;
+ struct ibv_mr ibv_mr;
+ enum ibv_mr_type mr_type;
int access;
};
#define verbs_get_mr(mr) container_of((mr), struct verbs_mr, ibv_mr)
diff --git a/sys/dev/ice/ice_rdma.c b/sys/dev/ice/ice_rdma.c
index decf9ea490d8..0bb4b79bb143 100644
--- a/sys/dev/ice/ice_rdma.c
+++ b/sys/dev/ice/ice_rdma.c
@@ -121,11 +121,10 @@ ice_rdma_pf_reset(struct ice_rdma_peer *peer)
{
struct ice_softc *sc = ice_rdma_peer_to_sc(peer);
- /*
- * Request that the driver re-initialize by bringing the interface
- * down and up.
- */
- ice_request_stack_reinit(sc);
+ /* Tell the base driver that RDMA is requesting a PFR */
+ ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
+
+ /* XXX: Base driver will notify RDMA when it's done */
return (0);
}
@@ -331,6 +330,7 @@ ice_rdma_request_handler(struct ice_rdma_peer *peer,
switch(req->type) {
case ICE_RDMA_EVENT_RESET:
+ ice_rdma_pf_reset(peer);
break;
case ICE_RDMA_EVENT_QSET_REGISTER:
ice_rdma_qset_register_request(peer, &req->res);
@@ -865,3 +865,52 @@ ice_rdma_dcb_qos_update(struct ice_softc *sc, struct ice_port_info *pi)
IRDMA_EVENT_HANDLER(peer, &event);
sx_xunlock(&ice_rdma.mtx);
}
+
+/**
+ * ice_rdma_notify_pe_intr - notify irdma on incoming interrupts regarding PE
+ * @sc: the ice driver softc
+ * @oicr: interrupt cause
+ *
+ * Pass the information about received interrupt to RDMA driver if it was
+ * relating to PE. Specifically PE_CRITERR and HMC_ERR.
+ * The irdma driver shall decide what should be done upon these interrupts.
+ */
+void
+ice_rdma_notify_pe_intr(struct ice_softc *sc, uint32_t oicr)
+{
+ struct ice_rdma_peer *peer = &sc->rdma_entry.peer;
+ struct ice_rdma_event event;
+
+ memset(&event, 0, sizeof(struct ice_rdma_event));
+ event.type = ICE_RDMA_EVENT_CRIT_ERR;
+ event.oicr_reg = oicr;
+
+ sx_xlock(&ice_rdma.mtx);
+ if (sc->rdma_entry.attached && ice_rdma.registered)
+ IRDMA_EVENT_HANDLER(peer, &event);
+ sx_xunlock(&ice_rdma.mtx);
+}
+
+/**
+ * ice_rdma_notify_reset - notify irdma on incoming pf-reset
+ * @sc: the ice driver softc
+ *
+ * Inform irdma driver of an incoming PF reset.
+ * The irdma driver shall set its state to reset, and avoid using CQP
+ * anymore. Next step should be to call ice_rdma_pf_stop in order to
+ * remove resources.
+ */
+void
+ice_rdma_notify_reset(struct ice_softc *sc)
+{
+ struct ice_rdma_peer *peer = &sc->rdma_entry.peer;
+ struct ice_rdma_event event;
+
+ memset(&event, 0, sizeof(struct ice_rdma_event));
+ event.type = ICE_RDMA_EVENT_RESET;
+
+ sx_xlock(&ice_rdma.mtx);
+ if (sc->rdma_entry.attached && ice_rdma.registered)
+ IRDMA_EVENT_HANDLER(peer, &event);
+ sx_xunlock(&ice_rdma.mtx);
+}
diff --git a/sys/dev/ice/ice_rdma.h b/sys/dev/ice/ice_rdma.h
index f83c30b33f6c..38e2ef491e8e 100644
--- a/sys/dev/ice/ice_rdma.h
+++ b/sys/dev/ice/ice_rdma.h
@@ -64,7 +64,7 @@
* considered stable.
*/
#define ICE_RDMA_MAJOR_VERSION 1
-#define ICE_RDMA_MINOR_VERSION 0
+#define ICE_RDMA_MINOR_VERSION 1
#define ICE_RDMA_PATCH_VERSION 0
/**
@@ -273,18 +273,19 @@ struct ice_rdma_event {
uint64_t baudrate;
};
/* MTU change event */
- struct {
- int mtu;
- };
+ int mtu;
/*
* TC/QoS/DCB change event
- * RESET event use prep variable only
* prep: if true, this is a pre-event, post-event otherwise
*/
struct {
struct ice_qos_params port_qos;
bool prep;
};
+ /*
+ * CRIT_ERR event
+ */
+ uint32_t oicr_reg;
};
};
diff --git a/sys/dev/ice/ice_rdma_internal.h b/sys/dev/ice/ice_rdma_internal.h
index c80d4540194a..b36544609f45 100644
--- a/sys/dev/ice/ice_rdma_internal.h
+++ b/sys/dev/ice/ice_rdma_internal.h
@@ -98,4 +98,6 @@ int ice_rdma_pf_stop(struct ice_softc *sc);
void ice_rdma_link_change(struct ice_softc *sc, int linkstate, uint64_t baudrate);
void ice_rdma_notify_dcb_qos_change(struct ice_softc *sc);
void ice_rdma_dcb_qos_update(struct ice_softc *sc, struct ice_port_info *pi);
+void ice_rdma_notify_pe_intr(struct ice_softc *sc, uint32_t oicr);
+void ice_rdma_notify_reset(struct ice_softc *sc);
#endif
diff --git a/sys/dev/ice/if_ice_iflib.c b/sys/dev/ice/if_ice_iflib.c
index cb4931c4d54e..2397fce11596 100644
--- a/sys/dev/ice/if_ice_iflib.c
+++ b/sys/dev/ice/if_ice_iflib.c
@@ -1278,9 +1278,11 @@ ice_msix_admin(void *arg)
ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
}
- if (oicr & PFINT_OICR_PE_CRITERR_M) {
- device_printf(dev, "Critical Protocol Engine Error detected!\n");
- ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
+ if (oicr & (PFINT_OICR_PE_CRITERR_M | PFINT_OICR_HMC_ERR_M)) {
+ if (oicr & PFINT_OICR_HMC_ERR_M)
+ /* Log the HMC errors */
+ ice_log_hmc_error(hw, dev);
+ ice_rdma_notify_pe_intr(sc, oicr);
}
if (oicr & PFINT_OICR_PCI_EXCEPTION_M) {
@@ -1288,11 +1290,6 @@ ice_msix_admin(void *arg)
ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
}
- if (oicr & PFINT_OICR_HMC_ERR_M) {
- /* Log the HMC errors, but don't disable the interrupt cause */
- ice_log_hmc_error(hw, dev);
- }
-
return (FILTER_SCHEDULE_THREAD);
}
@@ -2298,6 +2295,8 @@ ice_prepare_for_reset(struct ice_softc *sc)
if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
return;
+ /* inform the RDMA client */
+ ice_rdma_notify_reset(sc);
/* stop the RDMA client */
ice_rdma_pf_stop(sc);
diff --git a/sys/dev/irdma/fbsd_kcompat.c b/sys/dev/irdma/fbsd_kcompat.c
index 325429cdd993..013f9279952e 100644
--- a/sys/dev/irdma/fbsd_kcompat.c
+++ b/sys/dev/irdma/fbsd_kcompat.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2021 - 2022 Intel Corporation
+ * Copyright (c) 2021 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -273,12 +273,15 @@ irdma_add_ip(struct irdma_device *iwdev)
{
struct ifnet *ifp = iwdev->netdev;
struct ifnet *ifv;
+ struct epoch_tracker et;
int i;
irdma_add_ipv4_addr(iwdev, ifp);
irdma_add_ipv6_addr(iwdev, ifp);
for (i = 0; if_getvlantrunk(ifp) != NULL && i < VLAN_N_VID; ++i) {
+ NET_EPOCH_ENTER(et);
ifv = VLAN_DEVAT(ifp, i);
+ NET_EPOCH_EXIT(et);
if (!ifv)
continue;
irdma_add_ipv4_addr(iwdev, ifv);
@@ -361,7 +364,8 @@ irdma_get_route_ifp(struct sockaddr *dst_sin, struct ifnet *netdev,
struct nhop_object *nh;
if (dst_sin->sa_family == AF_INET6)
- nh = fib6_lookup(RT_DEFAULT_FIB, &((struct sockaddr_in6 *)dst_sin)->sin6_addr, 0, NHR_NONE, 0);
+ nh = fib6_lookup(RT_DEFAULT_FIB, &((struct sockaddr_in6 *)dst_sin)->sin6_addr,
+ ((struct sockaddr_in6 *)dst_sin)->sin6_scope_id, NHR_NONE, 0);
else
nh = fib4_lookup(RT_DEFAULT_FIB, ((struct sockaddr_in *)dst_sin)->sin_addr, 0, NHR_NONE, 0);
if (!nh || (nh->nh_ifp != netdev &&
@@ -591,6 +595,188 @@ irdma_sysctl_dcqcn_update(SYSCTL_HANDLER_ARGS)
return 0;
}
+enum irdma_cqp_stats_info {
+ IRDMA_CQP_REQ_CMDS = 28,
+ IRDMA_CQP_CMPL_CMDS = 29
+};
+
+static int
+irdma_sysctl_cqp_stats(SYSCTL_HANDLER_ARGS)
+{
+ struct irdma_sc_cqp *cqp = (struct irdma_sc_cqp *)arg1;
+ char rslt[192] = "no cqp available yet";
+ int rslt_size = sizeof(rslt) - 1;
+ int option = (int)arg2;
+
+ if (!cqp) {
+ return sysctl_handle_string(oidp, rslt, sizeof(rslt), req);
+ }
+
+ snprintf(rslt, sizeof(rslt), "");
+ switch (option) {
+ case IRDMA_CQP_REQ_CMDS:
+ snprintf(rslt, rslt_size, "%lu", cqp->requested_ops);
+ break;
+ case IRDMA_CQP_CMPL_CMDS:
+ snprintf(rslt, rslt_size, "%lu", atomic64_read(&cqp->completed_ops));
+ break;
+ }
+
+ return sysctl_handle_string(oidp, rslt, sizeof(rslt), req);
+}
+
+struct irdma_sw_stats_tunable_info {
+ u8 op_type;
+ const char name[32];
+ const char desc[32];
+ uintptr_t value;
+};
+
+static const struct irdma_sw_stats_tunable_info irdma_sws_list[] = {
+ {IRDMA_OP_CEQ_DESTROY, "ceq_destroy", "ceq_destroy", 0},
+ {IRDMA_OP_AEQ_DESTROY, "aeq_destroy", "aeq_destroy", 0},
+ {IRDMA_OP_DELETE_ARP_CACHE_ENTRY, "delete_arp_cache_entry",
+ "delete_arp_cache_entry", 0},
+ {IRDMA_OP_MANAGE_APBVT_ENTRY, "manage_apbvt_entry",
+ "manage_apbvt_entry", 0},
+ {IRDMA_OP_CEQ_CREATE, "ceq_create", "ceq_create", 0},
+ {IRDMA_OP_AEQ_CREATE, "aeq_create", "aeq_create", 0},
+ {IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY, "manage_qhash_table_entry",
+ "manage_qhash_table_entry", 0},
+ {IRDMA_OP_QP_MODIFY, "qp_modify", "qp_modify", 0},
+ {IRDMA_OP_QP_UPLOAD_CONTEXT, "qp_upload_context", "qp_upload_context",
+ 0},
+ {IRDMA_OP_CQ_CREATE, "cq_create", "cq_create", 0},
+ {IRDMA_OP_CQ_DESTROY, "cq_destroy", "cq_destroy", 0},
+ {IRDMA_OP_QP_CREATE, "qp_create", "qp_create", 0},
+ {IRDMA_OP_QP_DESTROY, "qp_destroy", "qp_destroy", 0},
+ {IRDMA_OP_ALLOC_STAG, "alloc_stag", "alloc_stag", 0},
+ {IRDMA_OP_MR_REG_NON_SHARED, "mr_reg_non_shared", "mr_reg_non_shared",
+ 0},
+ {IRDMA_OP_DEALLOC_STAG, "dealloc_stag", "dealloc_stag", 0},
+ {IRDMA_OP_MW_ALLOC, "mw_alloc", "mw_alloc", 0},
+ {IRDMA_OP_QP_FLUSH_WQES, "qp_flush_wqes", "qp_flush_wqes", 0},
+ {IRDMA_OP_ADD_ARP_CACHE_ENTRY, "add_arp_cache_entry",
+ "add_arp_cache_entry", 0},
+ {IRDMA_OP_MANAGE_PUSH_PAGE, "manage_push_page", "manage_push_page", 0},
+ {IRDMA_OP_UPDATE_PE_SDS, "update_pe_sds", "update_pe_sds", 0},
+ {IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE, "manage_hmc_pm_func_table",
+ "manage_hmc_pm_func_table", 0},
+ {IRDMA_OP_SUSPEND, "suspend", "suspend", 0},
+ {IRDMA_OP_RESUME, "resume", "resume", 0},
+ {IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP, "manage_vchnl_req_pble_bp",
+ "manage_vchnl_req_pble_bp", 0},
+ {IRDMA_OP_QUERY_FPM_VAL, "query_fpm_val", "query_fpm_val", 0},
+ {IRDMA_OP_COMMIT_FPM_VAL, "commit_fpm_val", "commit_fpm_val", 0},
+ {IRDMA_OP_AH_CREATE, "ah_create", "ah_create", 0},
+ {IRDMA_OP_AH_MODIFY, "ah_modify", "ah_modify", 0},
+ {IRDMA_OP_AH_DESTROY, "ah_destroy", "ah_destroy", 0},
+ {IRDMA_OP_MC_CREATE, "mc_create", "mc_create", 0},
+ {IRDMA_OP_MC_DESTROY, "mc_destroy", "mc_destroy", 0},
+ {IRDMA_OP_MC_MODIFY, "mc_modify", "mc_modify", 0},
+ {IRDMA_OP_STATS_ALLOCATE, "stats_allocate", "stats_allocate", 0},
+ {IRDMA_OP_STATS_FREE, "stats_free", "stats_free", 0},
+ {IRDMA_OP_STATS_GATHER, "stats_gather", "stats_gather", 0},
+ {IRDMA_OP_WS_ADD_NODE, "ws_add_node", "ws_add_node", 0},
+ {IRDMA_OP_WS_MODIFY_NODE, "ws_modify_node", "ws_modify_node", 0},
+ {IRDMA_OP_WS_DELETE_NODE, "ws_delete_node", "ws_delete_node", 0},
+ {IRDMA_OP_WS_FAILOVER_START, "ws_failover_start", "ws_failover_start",
+ 0},
+ {IRDMA_OP_WS_FAILOVER_COMPLETE, "ws_failover_complete",
+ "ws_failover_complete", 0},
+ {IRDMA_OP_SET_UP_MAP, "set_up_map", "set_up_map", 0},
+ {IRDMA_OP_GEN_AE, "gen_ae", "gen_ae", 0},
+ {IRDMA_OP_QUERY_RDMA_FEATURES, "query_rdma_features",
+ "query_rdma_features", 0},
+ {IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY, "alloc_local_mac_entry",
+ "alloc_local_mac_entry", 0},
+ {IRDMA_OP_ADD_LOCAL_MAC_ENTRY, "add_local_mac_entry",
+ "add_local_mac_entry", 0},
+ {IRDMA_OP_DELETE_LOCAL_MAC_ENTRY, "delete_local_mac_entry",
+ "delete_local_mac_entry", 0},
+ {IRDMA_OP_CQ_MODIFY, "cq_modify", "cq_modify", 0}
+};
+
+static const struct irdma_sw_stats_tunable_info irdma_cmcs_list[] = {
+ {0, "cm_nodes_created", "cm_nodes_created",
+ offsetof(struct irdma_cm_core, stats_nodes_created)},
+ {0, "cm_nodes_destroyed", "cm_nodes_destroyed",
+ offsetof(struct irdma_cm_core, stats_nodes_destroyed)},
+ {0, "cm_listen_created", "cm_listen_created",
+ offsetof(struct irdma_cm_core, stats_listen_created)},
+ {0, "cm_listen_destroyed", "cm_listen_destroyed",
+ offsetof(struct irdma_cm_core, stats_listen_destroyed)},
+ {0, "cm_listen_nodes_created", "cm_listen_nodes_created",
+ offsetof(struct irdma_cm_core, stats_listen_nodes_created)},
+ {0, "cm_listen_nodes_destroyed", "cm_listen_nodes_destroyed",
+ offsetof(struct irdma_cm_core, stats_listen_nodes_destroyed)},
+ {0, "cm_lpbs", "cm_lpbs", offsetof(struct irdma_cm_core, stats_lpbs)},
+ {0, "cm_accepts", "cm_accepts", offsetof(struct irdma_cm_core,
+ stats_accepts)},
+ {0, "cm_rejects", "cm_rejects", offsetof(struct irdma_cm_core,
+ stats_rejects)},
*** 4485 LINES SKIPPED ***