git: c051c1ff703c - stable/13 - irdma(4): Upgrade driver to 1.1.5-k
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Wed, 08 Feb 2023 00:26:19 UTC
The branch stable/13 has been updated by erj:
URL: https://cgit.FreeBSD.org/src/commit/?id=c051c1ff703cf279c80490c143a99bec0ce98e7f
commit c051c1ff703cf279c80490c143a99bec0ce98e7f
Author: Bartosz Sobczak <bartosz.sobczak@intel.com>
AuthorDate: 2022-12-22 01:10:15 +0000
Commit: Eric Joyner <erj@FreeBSD.org>
CommitDate: 2023-02-08 00:24:06 +0000
irdma(4): Upgrade driver to 1.1.5-k
This is to upgrade current irdma driver version (in support of RDMA on
Intel(R) Ethernet Controller E810) to 1.1.5-k
change summary:
- refactor defines for hardware registers
- rereg_mr verb added in libirdma
- fix print warning during compilation
- rt_ros2priority macro fix
- irdma.4 validated with mandoc
- fixing nd6_resolve usage
- added libirdma_query_device
- sysctl for irdma version
- aeq_alloc_db fix
- dwork_flush protected with qp refcount
- PFC fixes
Signed-off-by: Eric Joyner <erj@FreeBSD.org>
Reviewed by: erj@
Relnotes: yes
Sponsored by: Intel Corporation
Differential Revision: https://reviews.freebsd.org/D36944
(cherry picked from commit 777e472cd86b9394d07bf96c19dbafc2e1ff4fdc)
---
contrib/ofed/libirdma/abi.h | 10 +-
contrib/ofed/libirdma/i40iw_hw.h | 4 +-
contrib/ofed/libirdma/irdma.h | 18 +-
contrib/ofed/libirdma/irdma_defs.h | 259 ++----
contrib/ofed/libirdma/irdma_uk.c | 774 +++++++++-------
contrib/ofed/libirdma/irdma_umain.c | 28 +-
contrib/ofed/libirdma/irdma_umain.h | 30 +-
contrib/ofed/libirdma/irdma_uquery.h | 50 +
contrib/ofed/libirdma/irdma_user.h | 227 ++++-
contrib/ofed/libirdma/irdma_uverbs.c | 377 ++++----
contrib/ofed/libirdma/libirdma.map | 8 +-
contrib/ofed/libirdma/osdep.h | 17 +-
share/man/man4/irdma.4 | 168 ++--
sys/dev/irdma/fbsd_kcompat.c | 17 +-
sys/dev/irdma/fbsd_kcompat.h | 19 +-
sys/dev/irdma/icrdma.c | 7 +-
sys/dev/irdma/icrdma_hw.c | 39 +-
sys/dev/irdma/icrdma_hw.h | 35 +-
sys/dev/irdma/irdma-abi.h | 2 +-
sys/dev/irdma/irdma.h | 158 ++--
sys/dev/irdma/irdma_cm.c | 110 +--
sys/dev/irdma/irdma_cm.h | 22 +-
sys/dev/irdma/irdma_ctrl.c | 1330 +++++++++++++--------------
sys/dev/irdma/irdma_defs.h | 1678 ++++++++++------------------------
sys/dev/irdma/irdma_hmc.c | 33 +-
sys/dev/irdma/irdma_hmc.h | 12 +-
sys/dev/irdma/irdma_hw.c | 205 ++---
sys/dev/irdma/irdma_kcompat.c | 348 ++++---
sys/dev/irdma/irdma_main.h | 23 +-
sys/dev/irdma/irdma_pble.c | 4 +-
sys/dev/irdma/irdma_protos.h | 13 +-
sys/dev/irdma/irdma_puda.c | 146 +--
sys/dev/irdma/irdma_type.h | 99 +-
sys/dev/irdma/irdma_uda.c | 79 +-
sys/dev/irdma/irdma_uda_d.h | 361 ++------
sys/dev/irdma/irdma_uk.c | 766 +++++++++-------
sys/dev/irdma/irdma_user.h | 227 ++++-
sys/dev/irdma/irdma_utils.c | 160 +++-
sys/dev/irdma/irdma_verbs.c | 706 +++++++++-----
sys/dev/irdma/irdma_verbs.h | 27 +-
sys/dev/irdma/irdma_ws.c | 4 +-
sys/dev/irdma/osdep.h | 6 +-
42 files changed, 4321 insertions(+), 4285 deletions(-)
diff --git a/contrib/ofed/libirdma/abi.h b/contrib/ofed/libirdma/abi.h
index ff7a2828efe0..e45a7b49caf8 100644
--- a/contrib/ofed/libirdma/abi.h
+++ b/contrib/ofed/libirdma/abi.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (C) 2019 - 2020 Intel Corporation
+ * Copyright (C) 2019 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -145,6 +145,14 @@ struct irdma_ureg_mr {
__u16 rq_pages;
__u16 sq_pages;
+};
+struct irdma_urereg_mr {
+ struct ibv_rereg_mr ibv_cmd;
+ __u16 reg_type; /* enum irdma_memreg_type */
+ __u16 cq_pages;
+ __u16 rq_pages;
+ __u16 sq_pages;
+
};
struct irdma_ucreate_ah_resp {
struct ibv_create_ah_resp ibv_resp;
diff --git a/contrib/ofed/libirdma/i40iw_hw.h b/contrib/ofed/libirdma/i40iw_hw.h
index 38c7e37c35c9..c51d89a0fcb2 100644
--- a/contrib/ofed/libirdma/i40iw_hw.h
+++ b/contrib/ofed/libirdma/i40iw_hw.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2020 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -50,11 +50,11 @@ enum i40iw_device_caps_const {
I40IW_MAX_CQ_SIZE = 1048575,
I40IW_MAX_OUTBOUND_MSG_SIZE = 2147483647,
I40IW_MAX_INBOUND_MSG_SIZE = 2147483647,
+ I40IW_MIN_WQ_SIZE = 4 /* WQEs */,
};
#define I40IW_QP_WQE_MIN_SIZE 32
#define I40IW_QP_WQE_MAX_SIZE 128
-#define I40IW_QP_SW_MIN_WQSIZE 4
#define I40IW_MAX_RQ_WQE_SHIFT 2
#define I40IW_MAX_QUANTA_PER_WR 2
diff --git a/contrib/ofed/libirdma/irdma.h b/contrib/ofed/libirdma/irdma.h
index 27fa3d53d3e8..1dd09c36c7ea 100644
--- a/contrib/ofed/libirdma/irdma.h
+++ b/contrib/ofed/libirdma/irdma.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2017 - 2021 Intel Corporation
+ * Copyright (c) 2017 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -39,13 +39,15 @@
#define RDMA_BIT2(type, a) ((u##type) 1UL << a)
#define RDMA_MASK3(type, mask, shift) ((u##type) mask << shift)
#define MAKEMASK(m, s) ((m) << (s))
-#define IRDMA_WQEALLOC_WQE_DESC_INDEX_S 20
-#define IRDMA_WQEALLOC_WQE_DESC_INDEX_M (0xfff << IRDMA_WQEALLOC_WQE_DESC_INDEX_S)
+
+#define IRDMA_WQEALLOC_WQE_DESC_INDEX_S 20
+#define IRDMA_WQEALLOC_WQE_DESC_INDEX GENMASK(31, 20)
enum irdma_vers {
- IRDMA_GEN_RSVD,
- IRDMA_GEN_1,
- IRDMA_GEN_2,
+ IRDMA_GEN_RSVD = 0,
+ IRDMA_GEN_1 = 1,
+ IRDMA_GEN_2 = 2,
+ IRDMA_GEN_MAX = 2,
};
struct irdma_uk_attrs {
@@ -58,8 +60,7 @@ struct irdma_uk_attrs {
u32 min_hw_cq_size;
u32 max_hw_cq_size;
u16 max_hw_sq_chunk;
- u16 max_hw_wq_size;
- u16 min_sw_wq_size;
+ u16 min_hw_wq_size;
u8 hw_rev;
};
@@ -68,6 +69,7 @@ struct irdma_hw_attrs {
u64 max_hw_outbound_msg_size;
u64 max_hw_inbound_msg_size;
u64 max_mr_size;
+ u64 page_size_cap;
u32 min_hw_qp_id;
u32 min_hw_aeq_size;
u32 max_hw_aeq_size;
diff --git a/contrib/ofed/libirdma/irdma_defs.h b/contrib/ofed/libirdma/irdma_defs.h
index 8fb9f1e2b622..932993fd44ce 100644
--- a/contrib/ofed/libirdma/irdma_defs.h
+++ b/contrib/ofed/libirdma/irdma_defs.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2021 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -75,7 +75,6 @@
#define IRDMA_CQE_QTYPE_RQ 0
#define IRDMA_CQE_QTYPE_SQ 1
-#define IRDMA_QP_SW_MIN_WQSIZE 8u /* in WRs*/
#define IRDMA_QP_WQE_MIN_SIZE 32
#define IRDMA_QP_WQE_MAX_SIZE 256
#define IRDMA_QP_WQE_MIN_QUANTA 1
@@ -85,9 +84,11 @@
#define IRDMA_SQ_RSVD 258
#define IRDMA_RQ_RSVD 1
-#define IRDMA_FEATURE_RTS_AE 1ULL
-#define IRDMA_FEATURE_CQ_RESIZE 2ULL
-#define IRDMA_FEATURE_RELAX_RQ_ORDER 4ULL
+#define IRDMA_FEATURE_RTS_AE BIT_ULL(0)
+#define IRDMA_FEATURE_CQ_RESIZE BIT_ULL(1)
+#define IRDMA_FEATURE_RELAX_RQ_ORDER BIT_ULL(2)
+#define IRDMA_FEATURE_64_BYTE_CQE BIT_ULL(5)
+
#define IRDMAQP_OP_RDMA_WRITE 0x00
#define IRDMAQP_OP_RDMA_READ 0x01
#define IRDMAQP_OP_RDMA_SEND 0x03
@@ -106,262 +107,198 @@
#define LS_32_1(val, bits) ((u32)((val) << (bits)))
#define RS_32_1(val, bits) ((u32)((val) >> (bits)))
#endif
-#define LS_64(val, field) (((u64)(val) << field ## _S) & (field ## _M))
-#define RS_64(val, field) ((u64)((val) & field ## _M) >> field ## _S)
-#define LS_32(val, field) (((val) << field ## _S) & (field ## _M))
-#define RS_32(val, field) (((val) & field ## _M) >> field ## _S)
+#ifndef GENMASK_ULL
+#define GENMASK_ULL(high, low) ((0xFFFFFFFFFFFFFFFFULL >> (64ULL - ((high) - (low) + 1ULL))) << (low))
+#endif /* GENMASK_ULL */
+#ifndef GENMASK
+#define GENMASK(high, low) ((0xFFFFFFFFUL >> (32UL - ((high) - (low) + 1UL))) << (low))
+#endif /* GENMASK */
+#ifndef FIELD_PREP
+#define FIELD_PREP(mask, val) (((u64)(val) << mask##_S) & (mask))
+#define FIELD_GET(mask, val) (((val) & mask) >> mask##_S)
+#endif /* FIELD_PREP */
#define IRDMA_CQPHC_QPCTX_S 0
-#define IRDMA_CQPHC_QPCTX_M \
- (0xffffffffffffffffULL << IRDMA_CQPHC_QPCTX_S)
-
-/* iWARP QP Doorbell shadow area */
+#define IRDMA_CQPHC_QPCTX GENMASK_ULL(63, 0)
#define IRDMA_QP_DBSA_HW_SQ_TAIL_S 0
-#define IRDMA_QP_DBSA_HW_SQ_TAIL_M \
- (0x7fffULL << IRDMA_QP_DBSA_HW_SQ_TAIL_S)
-
-/* Completion Queue Doorbell shadow area */
+#define IRDMA_QP_DBSA_HW_SQ_TAIL GENMASK_ULL(14, 0)
#define IRDMA_CQ_DBSA_CQEIDX_S 0
-#define IRDMA_CQ_DBSA_CQEIDX_M (0xfffffULL << IRDMA_CQ_DBSA_CQEIDX_S)
-
+#define IRDMA_CQ_DBSA_CQEIDX GENMASK_ULL(19, 0)
#define IRDMA_CQ_DBSA_SW_CQ_SELECT_S 0
-#define IRDMA_CQ_DBSA_SW_CQ_SELECT_M \
- (0x3fffULL << IRDMA_CQ_DBSA_SW_CQ_SELECT_S)
-
+#define IRDMA_CQ_DBSA_SW_CQ_SELECT GENMASK_ULL(13, 0)
#define IRDMA_CQ_DBSA_ARM_NEXT_S 14
-#define IRDMA_CQ_DBSA_ARM_NEXT_M BIT_ULL(IRDMA_CQ_DBSA_ARM_NEXT_S)
-
+#define IRDMA_CQ_DBSA_ARM_NEXT BIT_ULL(14)
#define IRDMA_CQ_DBSA_ARM_NEXT_SE_S 15
-#define IRDMA_CQ_DBSA_ARM_NEXT_SE_M BIT_ULL(IRDMA_CQ_DBSA_ARM_NEXT_SE_S)
-
+#define IRDMA_CQ_DBSA_ARM_NEXT_SE BIT_ULL(15)
#define IRDMA_CQ_DBSA_ARM_SEQ_NUM_S 16
-#define IRDMA_CQ_DBSA_ARM_SEQ_NUM_M \
- (0x3ULL << IRDMA_CQ_DBSA_ARM_SEQ_NUM_S)
+#define IRDMA_CQ_DBSA_ARM_SEQ_NUM GENMASK_ULL(17, 16)
/* CQP and iWARP Completion Queue */
#define IRDMA_CQ_QPCTX_S IRDMA_CQPHC_QPCTX_S
-#define IRDMA_CQ_QPCTX_M IRDMA_CQPHC_QPCTX_M
+#define IRDMA_CQ_QPCTX IRDMA_CQPHC_QPCTX
#define IRDMA_CQ_MINERR_S 0
-#define IRDMA_CQ_MINERR_M (0xffffULL << IRDMA_CQ_MINERR_S)
-
+#define IRDMA_CQ_MINERR GENMASK_ULL(15, 0)
#define IRDMA_CQ_MAJERR_S 16
-#define IRDMA_CQ_MAJERR_M (0xffffULL << IRDMA_CQ_MAJERR_S)
-
+#define IRDMA_CQ_MAJERR GENMASK_ULL(31, 16)
#define IRDMA_CQ_WQEIDX_S 32
-#define IRDMA_CQ_WQEIDX_M (0x7fffULL << IRDMA_CQ_WQEIDX_S)
-
+#define IRDMA_CQ_WQEIDX GENMASK_ULL(46, 32)
#define IRDMA_CQ_EXTCQE_S 50
-#define IRDMA_CQ_EXTCQE_M BIT_ULL(IRDMA_CQ_EXTCQE_S)
-
+#define IRDMA_CQ_EXTCQE BIT_ULL(50)
#define IRDMA_OOO_CMPL_S 54
-#define IRDMA_OOO_CMPL_M BIT_ULL(IRDMA_OOO_CMPL_S)
-
+#define IRDMA_OOO_CMPL BIT_ULL(54)
#define IRDMA_CQ_ERROR_S 55
-#define IRDMA_CQ_ERROR_M BIT_ULL(IRDMA_CQ_ERROR_S)
-
+#define IRDMA_CQ_ERROR BIT_ULL(55)
#define IRDMA_CQ_SQ_S 62
-#define IRDMA_CQ_SQ_M BIT_ULL(IRDMA_CQ_SQ_S)
+#define IRDMA_CQ_SQ BIT_ULL(62)
#define IRDMA_CQ_VALID_S 63
-#define IRDMA_CQ_VALID_M BIT_ULL(IRDMA_CQ_VALID_S)
-
-#define IRDMA_CQ_IMMVALID_S 62
-#define IRDMA_CQ_IMMVALID_M BIT_ULL(IRDMA_CQ_IMMVALID_S)
-
+#define IRDMA_CQ_VALID BIT_ULL(63)
+#define IRDMA_CQ_IMMVALID BIT_ULL(62)
#define IRDMA_CQ_UDSMACVALID_S 61
-#define IRDMA_CQ_UDSMACVALID_M BIT_ULL(IRDMA_CQ_UDSMACVALID_S)
-
+#define IRDMA_CQ_UDSMACVALID BIT_ULL(61)
#define IRDMA_CQ_UDVLANVALID_S 60
-#define IRDMA_CQ_UDVLANVALID_M BIT_ULL(IRDMA_CQ_UDVLANVALID_S)
-
+#define IRDMA_CQ_UDVLANVALID BIT_ULL(60)
#define IRDMA_CQ_UDSMAC_S 0
-#define IRDMA_CQ_UDSMAC_M (0xffffffffffffULL << IRDMA_CQ_UDSMAC_S)
-
+#define IRDMA_CQ_UDSMAC GENMASK_ULL(47, 0)
#define IRDMA_CQ_UDVLAN_S 48
-#define IRDMA_CQ_UDVLAN_M (0xffffULL << IRDMA_CQ_UDVLAN_S)
+#define IRDMA_CQ_UDVLAN GENMASK_ULL(63, 48)
#define IRDMA_CQ_IMMDATA_S 0
-#define IRDMA_CQ_IMMDATA_M (0xffffffffffffffffULL << IRDMA_CQ_IMMVALID_S)
-
+#define IRDMA_CQ_IMMVALID_S 62
+#define IRDMA_CQ_IMMDATA GENMASK_ULL(125, 62)
#define IRDMA_CQ_IMMDATALOW32_S 0
-#define IRDMA_CQ_IMMDATALOW32_M (0xffffffffULL << IRDMA_CQ_IMMDATALOW32_S)
-
+#define IRDMA_CQ_IMMDATALOW32 GENMASK_ULL(31, 0)
#define IRDMA_CQ_IMMDATAUP32_S 32
-#define IRDMA_CQ_IMMDATAUP32_M (0xffffffffULL << IRDMA_CQ_IMMDATAUP32_S)
-
+#define IRDMA_CQ_IMMDATAUP32 GENMASK_ULL(63, 32)
#define IRDMACQ_PAYLDLEN_S 0
-#define IRDMACQ_PAYLDLEN_M (0xffffffffULL << IRDMACQ_PAYLDLEN_S)
-
-#define IRDMACQ_TCPSEQNUMRTT_S 32
-#define IRDMACQ_TCPSEQNUMRTT_M (0xffffffffULL << IRDMACQ_TCPSEQNUMRTT_S)
-
+#define IRDMACQ_PAYLDLEN GENMASK_ULL(31, 0)
+#define IRDMACQ_TCPSQN_ROCEPSN_RTT_TS_S 32
+#define IRDMACQ_TCPSQN_ROCEPSN_RTT_TS GENMASK_ULL(63, 32)
#define IRDMACQ_INVSTAG_S 0
-#define IRDMACQ_INVSTAG_M (0xffffffffULL << IRDMACQ_INVSTAG_S)
-
+#define IRDMACQ_INVSTAG GENMASK_ULL(31, 0)
#define IRDMACQ_QPID_S 32
-#define IRDMACQ_QPID_M (0xffffffULL << IRDMACQ_QPID_S)
+#define IRDMACQ_QPID GENMASK_ULL(55, 32)
#define IRDMACQ_UDSRCQPN_S 0
-#define IRDMACQ_UDSRCQPN_M (0xffffffffULL << IRDMACQ_UDSRCQPN_S)
-
+#define IRDMACQ_UDSRCQPN GENMASK_ULL(31, 0)
#define IRDMACQ_PSHDROP_S 51
-#define IRDMACQ_PSHDROP_M BIT_ULL(IRDMACQ_PSHDROP_S)
-
+#define IRDMACQ_PSHDROP BIT_ULL(51)
#define IRDMACQ_STAG_S 53
-#define IRDMACQ_STAG_M BIT_ULL(IRDMACQ_STAG_S)
-
+#define IRDMACQ_STAG BIT_ULL(53)
#define IRDMACQ_IPV4_S 53
-#define IRDMACQ_IPV4_M BIT_ULL(IRDMACQ_IPV4_S)
-
+#define IRDMACQ_IPV4 BIT_ULL(53)
#define IRDMACQ_SOEVENT_S 54
-#define IRDMACQ_SOEVENT_M BIT_ULL(IRDMACQ_SOEVENT_S)
-
+#define IRDMACQ_SOEVENT BIT_ULL(54)
#define IRDMACQ_OP_S 56
-#define IRDMACQ_OP_M (0x3fULL << IRDMACQ_OP_S)
+#define IRDMACQ_OP GENMASK_ULL(61, 56)
/* Manage Push Page - MPP */
#define IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1 0xffff
#define IRDMA_INVALID_PUSH_PAGE_INDEX 0xffffffff
-/* iwarp QP SQ WQE common fields */
#define IRDMAQPSQ_OPCODE_S 32
-#define IRDMAQPSQ_OPCODE_M (0x3fULL << IRDMAQPSQ_OPCODE_S)
-
+#define IRDMAQPSQ_OPCODE GENMASK_ULL(37, 32)
#define IRDMAQPSQ_COPY_HOST_PBL_S 43
-#define IRDMAQPSQ_COPY_HOST_PBL_M BIT_ULL(IRDMAQPSQ_COPY_HOST_PBL_S)
-
+#define IRDMAQPSQ_COPY_HOST_PBL BIT_ULL(43)
#define IRDMAQPSQ_ADDFRAGCNT_S 38
-#define IRDMAQPSQ_ADDFRAGCNT_M (0xfULL << IRDMAQPSQ_ADDFRAGCNT_S)
-
+#define IRDMAQPSQ_ADDFRAGCNT GENMASK_ULL(41, 38)
#define IRDMAQPSQ_PUSHWQE_S 56
-#define IRDMAQPSQ_PUSHWQE_M BIT_ULL(IRDMAQPSQ_PUSHWQE_S)
-
+#define IRDMAQPSQ_PUSHWQE BIT_ULL(56)
#define IRDMAQPSQ_STREAMMODE_S 58
-#define IRDMAQPSQ_STREAMMODE_M BIT_ULL(IRDMAQPSQ_STREAMMODE_S)
-
+#define IRDMAQPSQ_STREAMMODE BIT_ULL(58)
#define IRDMAQPSQ_WAITFORRCVPDU_S 59
-#define IRDMAQPSQ_WAITFORRCVPDU_M BIT_ULL(IRDMAQPSQ_WAITFORRCVPDU_S)
-
+#define IRDMAQPSQ_WAITFORRCVPDU BIT_ULL(59)
#define IRDMAQPSQ_READFENCE_S 60
-#define IRDMAQPSQ_READFENCE_M BIT_ULL(IRDMAQPSQ_READFENCE_S)
-
+#define IRDMAQPSQ_READFENCE BIT_ULL(60)
#define IRDMAQPSQ_LOCALFENCE_S 61
-#define IRDMAQPSQ_LOCALFENCE_M BIT_ULL(IRDMAQPSQ_LOCALFENCE_S)
-
+#define IRDMAQPSQ_LOCALFENCE BIT_ULL(61)
#define IRDMAQPSQ_UDPHEADER_S 61
-#define IRDMAQPSQ_UDPHEADER_M BIT_ULL(IRDMAQPSQ_UDPHEADER_S)
-
+#define IRDMAQPSQ_UDPHEADER BIT_ULL(61)
#define IRDMAQPSQ_L4LEN_S 42
-#define IRDMAQPSQ_L4LEN_M ((u64)0xF << IRDMAQPSQ_L4LEN_S)
-
+#define IRDMAQPSQ_L4LEN GENMASK_ULL(45, 42)
#define IRDMAQPSQ_SIGCOMPL_S 62
-#define IRDMAQPSQ_SIGCOMPL_M BIT_ULL(IRDMAQPSQ_SIGCOMPL_S)
-
+#define IRDMAQPSQ_SIGCOMPL BIT_ULL(62)
#define IRDMAQPSQ_VALID_S 63
-#define IRDMAQPSQ_VALID_M BIT_ULL(IRDMAQPSQ_VALID_S)
+#define IRDMAQPSQ_VALID BIT_ULL(63)
#define IRDMAQPSQ_FRAG_TO_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPSQ_FRAG_TO_M IRDMA_CQPHC_QPCTX_M
-
+#define IRDMAQPSQ_FRAG_TO IRDMA_CQPHC_QPCTX
#define IRDMAQPSQ_FRAG_VALID_S 63
-#define IRDMAQPSQ_FRAG_VALID_M BIT_ULL(IRDMAQPSQ_FRAG_VALID_S)
-
+#define IRDMAQPSQ_FRAG_VALID BIT_ULL(63)
#define IRDMAQPSQ_FRAG_LEN_S 32
-#define IRDMAQPSQ_FRAG_LEN_M (0x7fffffffULL << IRDMAQPSQ_FRAG_LEN_S)
-
+#define IRDMAQPSQ_FRAG_LEN GENMASK_ULL(62, 32)
#define IRDMAQPSQ_FRAG_STAG_S 0
-#define IRDMAQPSQ_FRAG_STAG_M (0xffffffffULL << IRDMAQPSQ_FRAG_STAG_S)
-
+#define IRDMAQPSQ_FRAG_STAG GENMASK_ULL(31, 0)
#define IRDMAQPSQ_GEN1_FRAG_LEN_S 0
-#define IRDMAQPSQ_GEN1_FRAG_LEN_M (0xffffffffULL << IRDMAQPSQ_GEN1_FRAG_LEN_S)
-
+#define IRDMAQPSQ_GEN1_FRAG_LEN GENMASK_ULL(31, 0)
#define IRDMAQPSQ_GEN1_FRAG_STAG_S 32
-#define IRDMAQPSQ_GEN1_FRAG_STAG_M (0xffffffffULL << IRDMAQPSQ_GEN1_FRAG_STAG_S)
-
+#define IRDMAQPSQ_GEN1_FRAG_STAG GENMASK_ULL(63, 32)
#define IRDMAQPSQ_REMSTAGINV_S 0
-#define IRDMAQPSQ_REMSTAGINV_M (0xffffffffULL << IRDMAQPSQ_REMSTAGINV_S)
-
+#define IRDMAQPSQ_REMSTAGINV GENMASK_ULL(31, 0)
#define IRDMAQPSQ_DESTQKEY_S 0
-#define IRDMAQPSQ_DESTQKEY_M (0xffffffffULL << IRDMAQPSQ_DESTQKEY_S)
-
+#define IRDMAQPSQ_DESTQKEY GENMASK_ULL(31, 0)
#define IRDMAQPSQ_DESTQPN_S 32
-#define IRDMAQPSQ_DESTQPN_M (0x00ffffffULL << IRDMAQPSQ_DESTQPN_S)
-
+#define IRDMAQPSQ_DESTQPN GENMASK_ULL(55, 32)
#define IRDMAQPSQ_AHID_S 0
-#define IRDMAQPSQ_AHID_M (0x0001ffffULL << IRDMAQPSQ_AHID_S)
-
+#define IRDMAQPSQ_AHID GENMASK_ULL(16, 0)
#define IRDMAQPSQ_INLINEDATAFLAG_S 57
-#define IRDMAQPSQ_INLINEDATAFLAG_M BIT_ULL(IRDMAQPSQ_INLINEDATAFLAG_S)
+#define IRDMAQPSQ_INLINEDATAFLAG BIT_ULL(57)
#define IRDMA_INLINE_VALID_S 7
-
#define IRDMAQPSQ_INLINEDATALEN_S 48
-#define IRDMAQPSQ_INLINEDATALEN_M \
- (0xffULL << IRDMAQPSQ_INLINEDATALEN_S)
+#define IRDMAQPSQ_INLINEDATALEN GENMASK_ULL(55, 48)
#define IRDMAQPSQ_IMMDATAFLAG_S 47
-#define IRDMAQPSQ_IMMDATAFLAG_M \
- BIT_ULL(IRDMAQPSQ_IMMDATAFLAG_S)
+#define IRDMAQPSQ_IMMDATAFLAG BIT_ULL(47)
#define IRDMAQPSQ_REPORTRTT_S 46
-#define IRDMAQPSQ_REPORTRTT_M \
- BIT_ULL(IRDMAQPSQ_REPORTRTT_S)
+#define IRDMAQPSQ_REPORTRTT BIT_ULL(46)
#define IRDMAQPSQ_IMMDATA_S 0
-#define IRDMAQPSQ_IMMDATA_M \
- (0xffffffffffffffffULL << IRDMAQPSQ_IMMDATA_S)
-
-/* rdma write */
+#define IRDMAQPSQ_IMMDATA GENMASK_ULL(63, 0)
#define IRDMAQPSQ_REMSTAG_S 0
-#define IRDMAQPSQ_REMSTAG_M (0xffffffffULL << IRDMAQPSQ_REMSTAG_S)
+#define IRDMAQPSQ_REMSTAG GENMASK_ULL(31, 0)
#define IRDMAQPSQ_REMTO_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPSQ_REMTO_M IRDMA_CQPHC_QPCTX_M
+#define IRDMAQPSQ_REMTO IRDMA_CQPHC_QPCTX
-/* memory window */
#define IRDMAQPSQ_STAGRIGHTS_S 48
-#define IRDMAQPSQ_STAGRIGHTS_M (0x1fULL << IRDMAQPSQ_STAGRIGHTS_S)
-
+#define IRDMAQPSQ_STAGRIGHTS GENMASK_ULL(52, 48)
#define IRDMAQPSQ_VABASEDTO_S 53
-#define IRDMAQPSQ_VABASEDTO_M BIT_ULL(IRDMAQPSQ_VABASEDTO_S)
-
+#define IRDMAQPSQ_VABASEDTO BIT_ULL(53)
#define IRDMAQPSQ_MEMWINDOWTYPE_S 54
-#define IRDMAQPSQ_MEMWINDOWTYPE_M BIT_ULL(IRDMAQPSQ_MEMWINDOWTYPE_S)
+#define IRDMAQPSQ_MEMWINDOWTYPE BIT_ULL(54)
#define IRDMAQPSQ_MWLEN_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPSQ_MWLEN_M IRDMA_CQPHC_QPCTX_M
-
+#define IRDMAQPSQ_MWLEN IRDMA_CQPHC_QPCTX
#define IRDMAQPSQ_PARENTMRSTAG_S 32
-#define IRDMAQPSQ_PARENTMRSTAG_M \
- (0xffffffffULL << IRDMAQPSQ_PARENTMRSTAG_S)
-
+#define IRDMAQPSQ_PARENTMRSTAG GENMASK_ULL(63, 32)
#define IRDMAQPSQ_MWSTAG_S 0
-#define IRDMAQPSQ_MWSTAG_M (0xffffffffULL << IRDMAQPSQ_MWSTAG_S)
+#define IRDMAQPSQ_MWSTAG GENMASK_ULL(31, 0)
#define IRDMAQPSQ_BASEVA_TO_FBO_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPSQ_BASEVA_TO_FBO_M IRDMA_CQPHC_QPCTX_M
+#define IRDMAQPSQ_BASEVA_TO_FBO IRDMA_CQPHC_QPCTX
-/* Local Invalidate */
#define IRDMAQPSQ_LOCSTAG_S 0
-#define IRDMAQPSQ_LOCSTAG_M (0xffffffffULL << IRDMAQPSQ_LOCSTAG_S)
+#define IRDMAQPSQ_LOCSTAG GENMASK_ULL(31, 0)
/* iwarp QP RQ WQE common fields */
#define IRDMAQPRQ_ADDFRAGCNT_S IRDMAQPSQ_ADDFRAGCNT_S
-#define IRDMAQPRQ_ADDFRAGCNT_M IRDMAQPSQ_ADDFRAGCNT_M
+#define IRDMAQPRQ_ADDFRAGCNT IRDMAQPSQ_ADDFRAGCNT
#define IRDMAQPRQ_VALID_S IRDMAQPSQ_VALID_S
-#define IRDMAQPRQ_VALID_M IRDMAQPSQ_VALID_M
+#define IRDMAQPRQ_VALID IRDMAQPSQ_VALID
#define IRDMAQPRQ_COMPLCTX_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPRQ_COMPLCTX_M IRDMA_CQPHC_QPCTX_M
+#define IRDMAQPRQ_COMPLCTX IRDMA_CQPHC_QPCTX
#define IRDMAQPRQ_FRAG_LEN_S IRDMAQPSQ_FRAG_LEN_S
-#define IRDMAQPRQ_FRAG_LEN_M IRDMAQPSQ_FRAG_LEN_M
+#define IRDMAQPRQ_FRAG_LEN IRDMAQPSQ_FRAG_LEN
#define IRDMAQPRQ_STAG_S IRDMAQPSQ_FRAG_STAG_S
-#define IRDMAQPRQ_STAG_M IRDMAQPSQ_FRAG_STAG_M
+#define IRDMAQPRQ_STAG IRDMAQPSQ_FRAG_STAG
#define IRDMAQPRQ_TO_S IRDMAQPSQ_FRAG_TO_S
-#define IRDMAQPRQ_TO_M IRDMAQPSQ_FRAG_TO_M
+#define IRDMAQPRQ_TO IRDMAQPSQ_FRAG_TO
#define IRDMAPFINT_OICR_HMC_ERR_M BIT(26)
#define IRDMAPFINT_OICR_PE_PUSH_M BIT(27)
@@ -500,6 +437,12 @@
IRDMA_RING_MOVE_HEAD(_ring, _retcode); \
}
+enum irdma_protocol_used {
+ IRDMA_ANY_PROTOCOL = 0,
+ IRDMA_IWARP_PROTOCOL_ONLY = 1,
+ IRDMA_ROCE_PROTOCOL_ONLY = 2,
+};
+
enum irdma_qp_wqe_size {
IRDMA_WQE_SIZE_32 = 32,
IRDMA_WQE_SIZE_64 = 64,
diff --git a/contrib/ofed/libirdma/irdma_uk.c b/contrib/ofed/libirdma/irdma_uk.c
index 2f77c132d296..5201ad692dc1 100644
--- a/contrib/ofed/libirdma/irdma_uk.c
+++ b/contrib/ofed/libirdma/irdma_uk.c
@@ -51,15 +51,15 @@ irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge,
{
if (sge) {
set_64bit_val(wqe, offset,
- LS_64(sge->tag_off, IRDMAQPSQ_FRAG_TO));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
- LS_64(valid, IRDMAQPSQ_VALID) |
- LS_64(sge->len, IRDMAQPSQ_FRAG_LEN) |
- LS_64(sge->stag, IRDMAQPSQ_FRAG_STAG));
+ FIELD_PREP(IRDMAQPSQ_VALID, valid) |
+ FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->len) |
+ FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->stag));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
- LS_64(valid, IRDMAQPSQ_VALID));
+ FIELD_PREP(IRDMAQPSQ_VALID, valid));
}
}
@@ -76,16 +76,26 @@ irdma_set_fragment_gen_1(__le64 * wqe, u32 offset,
{
if (sge) {
set_64bit_val(wqe, offset,
- LS_64(sge->tag_off, IRDMAQPSQ_FRAG_TO));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
- LS_64(sge->len, IRDMAQPSQ_GEN1_FRAG_LEN) |
- LS_64(sge->stag, IRDMAQPSQ_GEN1_FRAG_STAG));
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->len) |
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->stag));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8, 0);
}
}
+/**
+ * irdma_nop_hdr - Format header section of noop WQE
+ * @qp: hw qp ptr
+ */
+static inline u64 irdma_nop_hdr(struct irdma_qp_uk *qp){
+ return FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, false) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
+}
+
/**
* irdma_nop_1 - insert a NOP wqe
* @qp: hw qp ptr
@@ -93,10 +103,8 @@ irdma_set_fragment_gen_1(__le64 * wqe, u32 offset,
static int
irdma_nop_1(struct irdma_qp_uk *qp)
{
- u64 hdr;
__le64 *wqe;
u32 wqe_idx;
- bool signaled = false;
if (!qp->sq_ring.head)
return EINVAL;
@@ -110,14 +118,10 @@ irdma_nop_1(struct irdma_qp_uk *qp)
set_64bit_val(wqe, IRDMA_BYTE_8, 0);
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
- hdr = LS_64(IRDMAQP_OP_NOP, IRDMAQPSQ_OPCODE) |
- LS_64(signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
-
/* make sure WQE is written before valid bit is set */
udma_to_device_barrier();
- set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
+ set_64bit_val(wqe, IRDMA_BYTE_24, irdma_nop_hdr(qp));
return 0;
}
@@ -160,7 +164,7 @@ irdma_uk_qp_post_wr(struct irdma_qp_uk *qp)
/* read the doorbell shadow area */
get_64bit_val(qp->shadow_area, IRDMA_BYTE_0, &temp);
- hw_sq_tail = (u32)RS_64(temp, IRDMA_QP_DBSA_HW_SQ_TAIL);
+ hw_sq_tail = (u32)FIELD_GET(IRDMA_QP_DBSA_HW_SQ_TAIL, temp);
sw_sq_head = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
if (sw_sq_head != qp->initial_ring.head) {
if (qp->push_dropped) {
@@ -191,7 +195,7 @@ static void
irdma_qp_ring_push_db(struct irdma_qp_uk *qp, u32 wqe_idx)
{
set_32bit_val(qp->push_db, 0,
- LS_32(wqe_idx >> 3, IRDMA_WQEALLOC_WQE_DESC_INDEX) | qp->qp_id);
+ FIELD_PREP(IRDMA_WQEALLOC_WQE_DESC_INDEX, wqe_idx >> 3) | qp->qp_id);
qp->initial_ring.head = qp->sq_ring.head;
qp->push_mode = true;
qp->push_dropped = false;
@@ -220,31 +224,32 @@ irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 * wqe, u16 quanta,
* irdma_qp_get_next_send_wqe - pad with NOP if needed, return where next WR should go
* @qp: hw qp ptr
* @wqe_idx: return wqe index
- * @quanta: size of WR in quanta
+ * @quanta: (in/out) ptr to size of WR in quanta. Modified in case pad is needed
* @total_size: size of WR in bytes
* @info: info on WR
*/
__le64 *
irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
- u16 quanta, u32 total_size,
+ u16 *quanta, u32 total_size,
struct irdma_post_sq_info *info)
{
__le64 *wqe;
__le64 *wqe_0 = NULL;
u32 nop_wqe_idx;
- u16 avail_quanta;
+ u16 avail_quanta, wqe_quanta = *quanta;
u16 i;
avail_quanta = qp->uk_attrs->max_hw_sq_chunk -
(IRDMA_RING_CURRENT_HEAD(qp->sq_ring) %
qp->uk_attrs->max_hw_sq_chunk);
- if (quanta <= avail_quanta) {
+
+ if (*quanta <= avail_quanta) {
/* WR fits in current chunk */
- if (quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
+ if (*quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
return NULL;
} else {
/* Need to pad with NOP */
- if (quanta + avail_quanta >
+ if (*quanta + avail_quanta >
IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
return NULL;
@@ -262,17 +267,19 @@ irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
if (!*wqe_idx)
qp->swqe_polarity = !qp->swqe_polarity;
- IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, quanta);
+ IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, *quanta);
+
+ irdma_clr_wqes(qp, *wqe_idx);
wqe = qp->sq_base[*wqe_idx].elem;
- if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && quanta == 1 &&
+ if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && wqe_quanta == 1 &&
(IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) {
wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem;
- wqe_0[3] = htole64(LS_64(!qp->swqe_polarity, IRDMAQPSQ_VALID));
+ wqe_0[3] = htole64(FIELD_PREP(IRDMAQPSQ_VALID, !qp->swqe_polarity));
}
qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id;
qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size;
- qp->sq_wrtrk_array[*wqe_idx].quanta = quanta;
+ qp->sq_wrtrk_array[*wqe_idx].quanta = wqe_quanta;
qp->sq_wrtrk_array[*wqe_idx].signaled = info->signaled;
return wqe;
@@ -344,20 +351,17 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
if (ret_code)
return ret_code;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
- info);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
if (!wqe)
return ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(op_info->rem_addr.tag_off, IRDMAQPSQ_FRAG_TO));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
if (info->imm_data_valid) {
set_64bit_val(wqe, IRDMA_BYTE_0,
- LS_64(info->imm_data, IRDMAQPSQ_IMMDATA));
+ FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
i = 0;
} else {
qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0,
@@ -382,28 +386,24 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
++addl_frag_cnt;
}
- if (!op_info->rem_addr.stag && !total_size)
- op_info->rem_addr.stag = 0x1234;
- hdr = LS_64(op_info->rem_addr.stag, IRDMAQPSQ_REMSTAG) |
- LS_64(info->op_type, IRDMAQPSQ_OPCODE) |
- LS_64((info->imm_data_valid ? 1 : 0), IRDMAQPSQ_IMMDATAFLAG) |
- LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) |
- LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) |
- LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(read_fence, IRDMAQPSQ_READFENCE) |
- LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
+ FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) |
+ FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
+ if (info->push_wqe)
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -443,8 +443,7 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
if (ret_code)
return ret_code;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
- info);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
if (!wqe)
return ENOSPC;
@@ -453,8 +452,6 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
qp->ord_cnt = 0;
}
- irdma_clr_wqes(qp, wqe_idx);
-
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
addl_frag_cnt = op_info->num_lo_sges > 1 ?
(op_info->num_lo_sges - 1) : 0;
@@ -478,28 +475,26 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
++addl_frag_cnt;
}
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(op_info->rem_addr.tag_off, IRDMAQPSQ_FRAG_TO));
- hdr = LS_64(op_info->rem_addr.stag, IRDMAQPSQ_REMSTAG) |
- LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) |
- LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) |
- LS_64((inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ),
- IRDMAQPSQ_OPCODE) |
- LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(info->read_fence || qp->force_fence || ord_fence ? 1 : 0,
- IRDMAQPSQ_READFENCE) |
- LS_64(local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE,
+ (inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ)) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE,
+ info->read_fence || ord_fence ? 1 : 0) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
+ if (info->push_wqe)
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -540,21 +535,19 @@ irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
if (ret_code)
return ret_code;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
- info);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
if (!wqe)
return ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
read_fence |= info->read_fence;
addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0;
if (info->imm_data_valid) {
set_64bit_val(wqe, IRDMA_BYTE_0,
- LS_64(info->imm_data, IRDMAQPSQ_IMMDATA));
+ FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
i = 0;
} else {
- qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, op_info->sg_list,
+ qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0,
+ frag_cnt ? op_info->sg_list : NULL,
qp->swqe_polarity);
i = 1;
}
@@ -575,31 +568,30 @@ irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
}
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(op_info->qkey, IRDMAQPSQ_DESTQKEY) |
- LS_64(op_info->dest_qp, IRDMAQPSQ_DESTQPN));
- hdr = LS_64(info->stag_to_inv, IRDMAQPSQ_REMSTAG) |
- LS_64(op_info->ah_id, IRDMAQPSQ_AHID) |
- LS_64((info->imm_data_valid ? 1 : 0), IRDMAQPSQ_IMMDATAFLAG) |
- LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) |
- LS_64(info->op_type, IRDMAQPSQ_OPCODE) |
- LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) |
- LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(read_fence, IRDMAQPSQ_READFENCE) |
- LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(info->udp_hdr, IRDMAQPSQ_UDPHEADER) |
- LS_64(info->l4len, IRDMAQPSQ_L4LEN) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) |
+ FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp));
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) |
+ FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) |
+ FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,
+ (info->imm_data_valid ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
+ FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) |
+ FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
+ if (info->push_wqe)
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -615,29 +607,45 @@ irdma_set_mw_bind_wqe_gen_1(__le64 * wqe,
{
set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)op_info->va);
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(op_info->mw_stag, IRDMAQPSQ_PARENTMRSTAG) |
- LS_64(op_info->mr_stag, IRDMAQPSQ_MWSTAG));
+ FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mw_stag) |
+ FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mr_stag));
set_64bit_val(wqe, IRDMA_BYTE_16, op_info->bind_len);
}
/**
* irdma_copy_inline_data_gen_1 - Copy inline data to wqe
- * @dest: pointer to wqe
- * @src: pointer to inline data
- * @len: length of inline data to copy
+ * @wqe: pointer to wqe
+ * @sge_list: table of pointers to inline data
+ * @num_sges: Total inline data length
* @polarity: compatibility parameter
*/
static void
-irdma_copy_inline_data_gen_1(u8 *dest, u8 *src, u32 len,
- u8 polarity)
+irdma_copy_inline_data_gen_1(u8 *wqe, struct irdma_sge *sge_list,
+ u32 num_sges, u8 polarity)
{
- if (len <= IRDMA_BYTE_16) {
- irdma_memcpy(dest, src, len);
- } else {
*** 14300 LINES SKIPPED ***