git: a977168c48d4 - main - qat: Add Intel® 4xxx Series platform support
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 24 Jan 2023 15:37:17 UTC
The branch main has been updated by markj:
URL: https://cgit.FreeBSD.org/src/commit/?id=a977168c48d45085cdf0c40f9b9bde3850b1f3ea
commit a977168c48d45085cdf0c40f9b9bde3850b1f3ea
Author: Michal Gulbicki <michalx.gulbicki@intel.com>
AuthorDate: 2023-01-24 14:31:38 +0000
Commit: Mark Johnston <markj@FreeBSD.org>
CommitDate: 2023-01-24 15:33:50 +0000
qat: Add Intel® 4xxx Series platform support
Overview:
Intel(R) QuickAssist Technology (Intel(R) QAT) provides hardware
acceleration for offloading security, authentication and compression
services from the CPU, thus significantly increasing the performance and
efficiency of standard platform solutions.
This commit introduces:
- Intel® 4xxx Series platform support.
- QuickAssist kernel API implementation update for Generation 4 device.
Enabled services: symmetric cryptography and data compression.
- Increased default number of crypto instances in static configuration
for performance purposes.
OCF backend changes:
- changed GCM/CCM MAC validation policy to generate MAC by HW
and validate by SW due to the QAT HW limitations.
Patch co-authored by: Krzysztof Zdziarski <krzysztofx.zdziarski@intel.com>
Patch co-authored by: Michal Jaraczewski <michalx.jaraczewski@intel.com>
Patch co-authored by: Michal Gulbicki <michalx.gulbicki@intel.com>
Patch co-authored by: Julian Grajkowski <julianx.grajkowski@intel.com>
Patch co-authored by: Piotr Kasierski <piotrx.kasierski@intel.com>
Patch co-authored by: Adam Czupryna <adamx.czupryna@intel.com>
Patch co-authored by: Konrad Zelazny <konradx.zelazny@intel.com>
Patch co-authored by: Katarzyna Rucinska <katarzynax.kargol@intel.com>
Patch co-authored by: Lukasz Kolodzinski <lukaszx.kolodzinski@intel.com>
Patch co-authored by: Zbigniew Jedlinski <zbigniewx.jedlinski@intel.com>
Sponsored by: Intel Corporation
Reviewed by: markj, jhb
Differential Revision: https://reviews.freebsd.org/D36254
---
share/man/man4/qat.4 | 6 +-
sys/contrib/dev/qat/qat_4xxx.bin | Bin 0 -> 532308 bytes
sys/contrib/dev/qat/qat_4xxx_mmp.bin | Bin 0 -> 150084 bytes
sys/dev/qat/include/adf_cfg_device.h | 2 +-
sys/dev/qat/include/common/adf_accel_devices.h | 89 +-
sys/dev/qat/include/common/adf_cfg.h | 10 +
sys/dev/qat/include/common/adf_cfg_common.h | 6 +-
sys/dev/qat/include/common/adf_cfg_strings.h | 4 +
sys/dev/qat/include/common/adf_common_drv.h | 16 +-
sys/dev/qat/include/common/adf_gen2_hw_data.h | 172 +++
sys/dev/qat/include/common/adf_gen4_hw_data.h | 132 ++
.../qat/include/common/icp_qat_fw_loader_handle.h | 1 +
sys/dev/qat/include/common/icp_qat_hal.h | 79 +-
sys/dev/qat/include/common/icp_qat_uclo.h | 48 +-
sys/dev/qat/include/icp_qat_fw_init_admin.h | 4 -
sys/dev/qat/include/qat_ocf_utils.h | 7 +-
sys/dev/qat/qat/qat_ocf.c | 31 +-
.../qat/qat_api/common/compression/dc_buffers.c | 64 +-
.../qat/qat_api/common/compression/dc_datapath.c | 350 ++++-
sys/dev/qat/qat_api/common/compression/dc_dp.c | 29 +-
.../qat/qat_api/common/compression/dc_session.c | 387 ++++-
.../common/compression/include/dc_datapath.h | 14 +
.../common/compression/include/dc_session.h | 204 ++-
.../common/crypto/sym/include/lac_session.h | 106 +-
.../common/crypto/sym/include/lac_sym_cipher.h | 24 +-
.../crypto/sym/include/lac_sym_cipher_defs.h | 26 +-
.../common/crypto/sym/include/lac_sym_hash_defs.h | 61 +-
.../common/crypto/sym/include/lac_sym_qat.h | 39 +
.../common/crypto/sym/include/lac_sym_qat_cipher.h | 27 +-
.../sym/include/lac_sym_qat_constants_table.h | 111 ++
.../common/crypto/sym/include/lac_sym_qat_hash.h | 5 +
.../qat_api/common/crypto/sym/key/lac_sym_key.c | 11 +-
.../qat_api/common/crypto/sym/lac_sym_alg_chain.c | 1505 +++++++++++++-------
.../qat/qat_api/common/crypto/sym/lac_sym_api.c | 96 +-
sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cb.c | 130 +-
.../qat/qat_api/common/crypto/sym/lac_sym_cipher.c | 302 ++--
sys/dev/qat/qat_api/common/crypto/sym/lac_sym_dp.c | 335 +++--
.../qat/qat_api/common/crypto/sym/lac_sym_hash.c | 155 +-
.../qat/qat_api/common/crypto/sym/lac_sym_queue.c | 11 +-
.../qat_api/common/crypto/sym/qat/lac_sym_qat.c | 129 +-
.../common/crypto/sym/qat/lac_sym_qat_cipher.c | 869 ++++++-----
.../crypto/sym/qat/lac_sym_qat_constants_table.c | 257 ++++
.../common/crypto/sym/qat/lac_sym_qat_hash.c | 126 +-
.../crypto/sym/qat/lac_sym_qat_hash_defs_lookup.c | 510 +++----
sys/dev/qat/qat_api/common/ctrl/sal_compression.c | 69 +-
.../qat/qat_api/common/ctrl/sal_create_services.c | 3 +
sys/dev/qat/qat_api/common/ctrl/sal_crypto.c | 251 +++-
.../qat/qat_api/common/ctrl/sal_ctrl_services.c | 38 +-
sys/dev/qat/qat_api/common/include/lac_common.h | 3 -
sys/dev/qat/qat_api/common/include/lac_sal.h | 3 +-
sys/dev/qat/qat_api/common/include/lac_sal_types.h | 14 +
.../qat_api/common/include/lac_sal_types_crypto.h | 12 +
sys/dev/qat/qat_api/common/include/lac_sync.h | 2 +-
sys/dev/qat/qat_api/common/include/sal_hw_gen.h | 88 ++
.../qat_api/common/include/sal_types_compression.h | 9 +
sys/dev/qat/qat_api/firmware/include/icp_qat_fw.h | 116 +-
.../qat/qat_api/firmware/include/icp_qat_fw_comp.h | 132 +-
.../qat/qat_api/firmware/include/icp_qat_fw_la.h | 408 ++++--
sys/dev/qat/qat_api/firmware/include/icp_qat_hw.h | 201 ++-
.../qat_api/firmware/include/icp_qat_hw_20_comp.h | 292 ++++
.../firmware/include/icp_qat_hw_20_comp_defs.h | 443 ++++++
sys/dev/qat/qat_api/include/icp_sal_user.h | 37 +
sys/dev/qat/qat_api/include/icp_sal_versions.h | 4 +-
.../qat_api/qat_direct/include/icp_accel_devices.h | 3 +-
sys/dev/qat/qat_api/qat_kernel/src/qat_transport.c | 15 +-
sys/dev/qat/qat_api/qat_utils/include/qat_utils.h | 21 +
sys/dev/qat/qat_api/qat_utils/src/QatUtilsCrypto.c | 39 +
sys/dev/qat/qat_common/adf_accel_engine.c | 32 +-
sys/dev/qat/qat_common/adf_cfg_bundle.c | 49 +-
sys/dev/qat/qat_common/adf_cfg_bundle.h | 24 +-
sys/dev/qat/qat_common/adf_cfg_device.c | 33 +-
sys/dev/qat/qat_common/adf_cfg_section.c | 29 +-
sys/dev/qat/qat_common/adf_freebsd_admin.c | 9 +-
.../qat/qat_common/adf_freebsd_transport_debug.c | 31 +-
sys/dev/qat/qat_common/adf_gen2_hw_data.c | 132 ++
sys/dev/qat/qat_common/adf_gen4_hw_data.c | 176 +++
sys/dev/qat/qat_common/adf_heartbeat.c | 63 +-
sys/dev/qat/qat_common/adf_hw_arbiter.c | 44 +-
sys/dev/qat/qat_common/adf_init.c | 11 +-
sys/dev/qat/qat_common/adf_isr.c | 8 +-
sys/dev/qat/qat_common/adf_transport.c | 109 +-
sys/dev/qat/qat_common/qat_hal.c | 168 ++-
sys/dev/qat/qat_common/qat_uclo.c | 320 ++++-
sys/dev/qat/qat_hw/qat_200xx/adf_200xx_hw_data.c | 4 +
sys/dev/qat/qat_hw/qat_200xx/adf_drv.c | 1 +
sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c | 973 +++++++++++++
sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.h | 111 ++
sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c | 267 ++++
sys/dev/qat/qat_hw/qat_c3xxx/adf_c3xxx_hw_data.c | 4 +
sys/dev/qat/qat_hw/qat_c3xxx/adf_drv.c | 1 +
sys/dev/qat/qat_hw/qat_c4xxx/adf_c4xxx_hw_data.c | 76 +-
sys/dev/qat/qat_hw/qat_c4xxx/adf_drv.c | 1 +
sys/dev/qat/qat_hw/qat_c62x/adf_c62x_hw_data.c | 4 +
sys/dev/qat/qat_hw/qat_c62x/adf_drv.c | 1 +
.../qat/qat_hw/qat_dh895xcc/adf_dh895xcc_hw_data.c | 4 +
sys/dev/qat/qat_hw/qat_dh895xcc/adf_drv.c | 1 +
sys/modules/qat/qat_api/Makefile | 1 +
sys/modules/qat/qat_common/Makefile | 2 +
sys/modules/qat/qat_hw/Makefile | 1 +
sys/modules/qatfw/Makefile | 3 +-
sys/modules/qatfw/qat_4xxx/Makefile | 10 +
101 files changed, 8941 insertions(+), 2455 deletions(-)
diff --git a/share/man/man4/qat.4 b/share/man/man4/qat.4
index c6082f873a44..e8a46a99e949 100644
--- a/share/man/man4/qat.4
+++ b/share/man/man4/qat.4
@@ -1,7 +1,7 @@
.\" SPDX-License-Identifier: BSD-3-Clause
.\" Copyright(c) 2007-2022 Intel Corporation
.\" $FreeBSD$
-.Dd June 30, 2022
+.Dd September 1, 2022
.Dt QAT 4
.Os
.Sh NAME
@@ -30,6 +30,8 @@ qat_c62x_fw_load="YES"
.It
qat_dh895xcc_fw_load="YES"
.It
+qat_4xxx_fw_load="YES"
+.It
qat_load="YES"
.El
.Sh DESCRIPTION
@@ -53,6 +55,8 @@ Intel (R) QuickAssist Adapter 8960/Intel (R) QuickAssist Adapter 8970
Intel (R) Communications Chipset 8925 to 8955 Series
.It
Intel (R) Atom P5300 processor product family
+.It
+Intel (R) QAT 4xxx Series
.El
.Pp
The
diff --git a/sys/contrib/dev/qat/qat_4xxx.bin b/sys/contrib/dev/qat/qat_4xxx.bin
new file mode 100644
index 000000000000..baec3ad9ca32
Binary files /dev/null and b/sys/contrib/dev/qat/qat_4xxx.bin differ
diff --git a/sys/contrib/dev/qat/qat_4xxx_mmp.bin b/sys/contrib/dev/qat/qat_4xxx_mmp.bin
new file mode 100644
index 000000000000..7b7c560b97d0
Binary files /dev/null and b/sys/contrib/dev/qat/qat_4xxx_mmp.bin differ
diff --git a/sys/dev/qat/include/adf_cfg_device.h b/sys/dev/qat/include/adf_cfg_device.h
index 40fb91119f03..9def937cc9db 100644
--- a/sys/dev/qat/include/adf_cfg_device.h
+++ b/sys/dev/qat/include/adf_cfg_device.h
@@ -28,7 +28,7 @@
#define ADF_CFG_STATIC_CONF_NUM_DC_ACCEL_UNITS 2
#define ADF_CFG_STATIC_CONF_NUM_INLINE_ACCEL_UNITS 0
#define ADF_CFG_STATIC_CONF_INST_NUM_DC 2
-#define ADF_CFG_STATIC_CONF_INST_NUM_CY_POLL 2
+#define ADF_CFG_STATIC_CONF_INST_NUM_CY_POLL 6
#define ADF_CFG_STATIC_CONF_INST_NUM_CY_IRQ 2
#define ADF_CFG_FW_STRING_TO_ID(str, acc, id) \
diff --git a/sys/dev/qat/include/common/adf_accel_devices.h b/sys/dev/qat/include/common/adf_accel_devices.h
index ad0e74335259..9503069ac2a2 100644
--- a/sys/dev/qat/include/common/adf_accel_devices.h
+++ b/sys/dev/qat/include/common/adf_accel_devices.h
@@ -19,6 +19,7 @@
#define ADF_200XXVF_DEVICE_NAME "200xxvf"
#define ADF_C4XXX_DEVICE_NAME "c4xxx"
#define ADF_C4XXXVF_DEVICE_NAME "c4xxxvf"
+#define ADF_4XXX_DEVICE_NAME "4xxx"
#define ADF_DH895XCC_PCI_DEVICE_ID 0x435
#define ADF_DH895XCCIOV_PCI_DEVICE_ID 0x443
#define ADF_C62X_PCI_DEVICE_ID 0x37c8
@@ -31,8 +32,17 @@
#define ADF_D15XXIOV_PCI_DEVICE_ID 0x6f55
#define ADF_C4XXX_PCI_DEVICE_ID 0x18a0
#define ADF_C4XXXIOV_PCI_DEVICE_ID 0x18a1
+#define ADF_4XXX_PCI_DEVICE_ID 0x4940
+#define ADF_401XX_PCI_DEVICE_ID 0x4942
#define IS_QAT_GEN3(ID) ({ (ID == ADF_C4XXX_PCI_DEVICE_ID); })
+static inline bool
+IS_QAT_GEN4(const unsigned int id)
+{
+ return (id == ADF_4XXX_PCI_DEVICE_ID || id == ADF_401XX_PCI_DEVICE_ID);
+}
+
+#define IS_QAT_GEN3_OR_GEN4(ID) (IS_QAT_GEN3(ID) || IS_QAT_GEN4(ID))
#define ADF_VF2PF_SET_SIZE 32
#define ADF_MAX_VF2PF_SET 4
#define ADF_VF2PF_SET_OFFSET(set_nr) ((set_nr)*ADF_VF2PF_SET_SIZE)
@@ -50,7 +60,7 @@
#define ADF_PCI_MAX_BARS 3
#define ADF_DEVICE_NAME_LENGTH 32
#define ADF_ETR_MAX_RINGS_PER_BANK 16
-#define ADF_MAX_MSIX_VECTOR_NAME 16
+#define ADF_MAX_MSIX_VECTOR_NAME 32
#define ADF_DEVICE_NAME_PREFIX "qat_"
#define ADF_STOP_RETRY 50
#define ADF_NUM_THREADS_PER_AE (8)
@@ -58,7 +68,6 @@
#define ADF_NUM_PKE_STRAND (2)
#define ADF_AE_STRAND0_THREAD (8)
#define ADF_AE_STRAND1_THREAD (9)
-#define ADF_NUM_HB_CNT_PER_AE (ADF_NUM_THREADS_PER_AE + ADF_NUM_PKE_STRAND)
#define ADF_CFG_NUM_SERVICES 4
#define ADF_SRV_TYPE_BIT_LEN 3
#define ADF_SRV_TYPE_MASK 0x7
@@ -75,6 +84,8 @@
#define GET_SRV_TYPE(ena_srv_mask, srv) \
(((ena_srv_mask) >> (ADF_SRV_TYPE_BIT_LEN * (srv))) & ADF_SRV_TYPE_MASK)
+#define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_info.csr_ops)
+
#define ADF_DEFAULT_RING_TO_SRV_MAP \
(CRYPTO | CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
NA << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
@@ -156,7 +167,9 @@ enum adf_accel_unit_services {
ADF_ACCEL_SERVICE_NULL = 0,
ADF_ACCEL_INLINE_CRYPTO = 1,
ADF_ACCEL_CRYPTO = 2,
- ADF_ACCEL_COMPRESSION = 4
+ ADF_ACCEL_COMPRESSION = 4,
+ ADF_ACCEL_ASYM = 8,
+ ADF_ACCEL_ADMIN = 16
};
struct adf_ae_info {
@@ -182,6 +195,7 @@ struct adf_accel_unit_info {
u32 dc_ae_msk;
u8 num_cy_au;
u8 num_dc_au;
+ u8 num_asym_au;
u8 num_inline_au;
struct adf_accel_unit *au;
const struct adf_ae_info *ae_info;
@@ -231,6 +245,60 @@ struct admin_info {
u32 mailbox_offset;
} __packed;
+struct adf_hw_csr_ops {
+ u64 (*build_csr_ring_base_addr)(bus_addr_t addr, u32 size);
+ u32 (*read_csr_ring_head)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring);
+ void (*write_csr_ring_head)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ u32 value);
+ u32 (*read_csr_ring_tail)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring);
+ void (*write_csr_ring_tail)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ u32 value);
+ u32 (*read_csr_e_stat)(struct resource *csr_base_addr, u32 bank);
+ void (*write_csr_ring_config)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ u32 value);
+ void (*write_csr_ring_base)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ bus_addr_t addr);
+ void (*write_csr_int_flag)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 value);
+ void (*write_csr_int_srcsel)(struct resource *csr_base_addr, u32 bank);
+ void (*write_csr_int_col_en)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 value);
+ void (*write_csr_int_col_ctl)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 value);
+ void (*write_csr_int_flag_and_col)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 value);
+ u32 (*read_csr_ring_srv_arb_en)(struct resource *csr_base_addr,
+ u32 bank);
+ void (*write_csr_ring_srv_arb_en)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 value);
+};
+
+struct adf_hw_csr_info {
+ struct adf_hw_csr_ops csr_ops;
+ u32 csr_addr_offset;
+ u32 ring_bundle_size;
+ u32 bank_int_flag_clear_mask;
+ u32 num_rings_per_int_srcsel;
+ u32 arb_enable_mask;
+};
+
struct adf_cfg_device_data;
struct adf_accel_dev;
struct adf_etr_data;
@@ -282,8 +350,10 @@ struct adf_hw_device_data {
void (*exit_arb)(struct adf_accel_dev *accel_dev);
void (*get_arb_mapping)(struct adf_accel_dev *accel_dev,
const uint32_t **cfg);
+ int (*init_device)(struct adf_accel_dev *accel_dev);
int (*get_heartbeat_status)(struct adf_accel_dev *accel_dev);
uint32_t (*get_ae_clock)(struct adf_hw_device_data *self);
+ uint32_t (*get_hb_clock)(struct adf_hw_device_data *self);
void (*disable_iov)(struct adf_accel_dev *accel_dev);
void (*configure_iov_threads)(struct adf_accel_dev *accel_dev,
bool enable);
@@ -298,6 +368,8 @@ struct adf_hw_device_data {
void (*restore_device)(struct adf_accel_dev *accel_dev);
uint32_t (*get_obj_cfg_ae_mask)(struct adf_accel_dev *accel_dev,
enum adf_accel_unit_services services);
+ enum adf_accel_unit_services (
+ *get_service_type)(struct adf_accel_dev *accel_dev, s32 obj_num);
int (*add_pke_stats)(struct adf_accel_dev *accel_dev);
void (*remove_pke_stats)(struct adf_accel_dev *accel_dev);
int (*add_misc_error)(struct adf_accel_dev *accel_dev);
@@ -311,6 +383,14 @@ struct adf_hw_device_data {
enum adf_accel_unit_services services);
void (*pre_reset)(struct adf_accel_dev *accel_dev);
void (*post_reset)(struct adf_accel_dev *accel_dev);
+ void (*set_msix_rttable)(struct adf_accel_dev *accel_dev);
+ void (*get_ring_svc_map_data)(int ring_pair_index,
+ u16 ring_to_svc_map,
+ u8 *serv_type,
+ int *ring_index,
+ int *num_rings_per_srv,
+ int bundle_num);
+ struct adf_hw_csr_info csr_info;
const char *fw_name;
const char *fw_mmp_name;
bool reset_ack;
@@ -320,7 +400,10 @@ struct adf_hw_device_data {
uint16_t accel_mask;
u32 aerucm_mask;
u32 ae_mask;
+ u32 admin_ae_mask;
u32 service_mask;
+ u32 service_to_load_mask;
+ u32 heartbeat_ctr_num;
uint16_t tx_rings_mask;
uint8_t tx_rx_gap;
uint8_t num_banks;
diff --git a/sys/dev/qat/include/common/adf_cfg.h b/sys/dev/qat/include/common/adf_cfg.h
index edc4813cb69e..58502c8605b8 100644
--- a/sys/dev/qat/include/common/adf_cfg.h
+++ b/sys/dev/qat/include/common/adf_cfg.h
@@ -76,4 +76,14 @@ int adf_cfg_get_services_enabled(struct adf_accel_dev *accel_dev,
int adf_cfg_restore_section(struct adf_accel_dev *accel_dev,
struct adf_cfg_section *section);
void adf_cfg_keyval_del_all(struct list_head *head);
+
+static inline int
+adf_cy_inst_cross_banks(struct adf_accel_dev *accel_dev)
+{
+ if (accel_dev->hw_device->num_rings_per_bank == 2)
+ return 1;
+ else
+ return 0;
+}
+
#endif
diff --git a/sys/dev/qat/include/common/adf_cfg_common.h b/sys/dev/qat/include/common/adf_cfg_common.h
index 68fb5e8a98b3..65fc60fc8c3d 100644
--- a/sys/dev/qat/include/common/adf_cfg_common.h
+++ b/sys/dev/qat/include/common/adf_cfg_common.h
@@ -27,7 +27,7 @@
#define ADF_MAX_ACCELENGINES 12
#define ADF_CFG_STORAGE_ENABLED 1
#define ADF_DEVS_ARRAY_SIZE BITS_TO_LONGS(ADF_MAX_DEVICES)
-#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x3000000
+#define ADF_GEN2_SSM_WDT_PKE_DEFAULT_VALUE 0x3000000
#define ADF_WDT_TIMER_SYM_COMP_MS 3
#define ADF_MIN_HB_TIMER_MS 100
#define ADF_CFG_MAX_NUM_OF_SECTIONS 16
@@ -87,7 +87,8 @@ enum adf_device_type {
DEV_200XX,
DEV_200XXVF,
DEV_C4XXX,
- DEV_C4XXXVF
+ DEV_C4XXXVF,
+ DEV_4XXX
};
enum adf_cfg_fw_image_type {
@@ -158,6 +159,7 @@ struct adf_cfg_bundle {
/* contains all the info about rings */
struct adf_cfg_ring **rings;
u16 in_use;
+ u16 max_cfg_svc_num;
};
struct adf_cfg_instance {
diff --git a/sys/dev/qat/include/common/adf_cfg_strings.h b/sys/dev/qat/include/common/adf_cfg_strings.h
index 2f05dadadc45..933ffe0ba6ad 100644
--- a/sys/dev/qat/include/common/adf_cfg_strings.h
+++ b/sys/dev/qat/include/common/adf_cfg_strings.h
@@ -22,6 +22,8 @@
#define ADF_RING_DC_RX "RingRx"
#define ADF_ETRMGR_BANK "Bank"
#define ADF_RING_BANK_NUM "BankNumber"
+#define ADF_RING_BANK_NUM_ASYM "BankNumberAsym"
+#define ADF_RING_BANK_NUM_SYM "BankNumberSym"
#define ADF_CY "Cy"
#define ADF_DC "Dc"
#define ADF_DC_EXTENDED_FEATURES "Device_DcExtendedFeatures"
@@ -112,6 +114,8 @@
#define ADF_CY_CORE_AFFINITY_FORMAT ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY
#define ADF_DC_CORE_AFFINITY_FORMAT ADF_DC "%d" ADF_ETRMGR_CORE_AFFINITY
#define ADF_CY_BANK_NUM_FORMAT ADF_CY "%d" ADF_RING_BANK_NUM
+#define ADF_CY_ASYM_BANK_NUM_FORMAT ADF_CY "%d" ADF_RING_BANK_NUM_ASYM
+#define ADF_CY_SYM_BANK_NUM_FORMAT ADF_CY "%d" ADF_RING_BANK_NUM_SYM
#define ADF_DC_BANK_NUM_FORMAT ADF_DC "%d" ADF_RING_BANK_NUM
#define ADF_CY_ASYM_TX_FORMAT ADF_CY "%d" ADF_RING_ASYM_TX
#define ADF_CY_SYM_TX_FORMAT ADF_CY "%d" ADF_RING_SYM_TX
diff --git a/sys/dev/qat/include/common/adf_common_drv.h b/sys/dev/qat/include/common/adf_common_drv.h
index 3bb35ed55da3..7ec380540336 100644
--- a/sys/dev/qat/include/common/adf_common_drv.h
+++ b/sys/dev/qat/include/common/adf_common_drv.h
@@ -203,10 +203,14 @@ int adf_init_gen2_arb(struct adf_accel_dev *accel_dev);
void adf_exit_arb(struct adf_accel_dev *accel_dev);
void adf_disable_arb(struct adf_accel_dev *accel_dev);
void adf_update_ring_arb(struct adf_etr_ring_data *ring);
-void
-adf_enable_ring_arb(void *csr_addr, unsigned int bank_nr, unsigned int mask);
-void
-adf_disable_ring_arb(void *csr_addr, unsigned int bank_nr, unsigned int mask);
+void adf_enable_ring_arb(struct adf_accel_dev *accel_dev,
+ void *csr_addr,
+ unsigned int bank_nr,
+ unsigned int mask);
+void adf_disable_ring_arb(struct adf_accel_dev *accel_dev,
+ void *csr_addr,
+ unsigned int bank_nr,
+ unsigned int mask);
int adf_set_ssm_wdtimer(struct adf_accel_dev *accel_dev);
struct adf_accel_dev *adf_devmgr_get_dev_by_bdf(struct adf_pci_address *addr);
struct adf_accel_dev *adf_devmgr_get_dev_by_pci_bus(u8 bus);
@@ -238,9 +242,7 @@ void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev);
int qat_hal_init(struct adf_accel_dev *accel_dev);
void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle);
-void qat_hal_start(struct icp_qat_fw_loader_handle *handle,
- unsigned char ae,
- unsigned int ctx_mask);
+int qat_hal_start(struct icp_qat_fw_loader_handle *handle);
void qat_hal_stop(struct icp_qat_fw_loader_handle *handle,
unsigned char ae,
unsigned int ctx_mask);
diff --git a/sys/dev/qat/include/common/adf_gen2_hw_data.h b/sys/dev/qat/include/common/adf_gen2_hw_data.h
new file mode 100644
index 000000000000..395abec81b9f
--- /dev/null
+++ b/sys/dev/qat/include/common/adf_gen2_hw_data.h
@@ -0,0 +1,172 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+/* Copyright(c) 2021 Intel Corporation */
+/* $FreeBSD$ */
+#ifndef ADF_GEN2_HW_DATA_H_
+#define ADF_GEN2_HW_DATA_H_
+
+#include "adf_accel_devices.h"
+#include "adf_cfg_common.h"
+
+/* Transport access */
+#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL
+#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL
+#define ADF_RING_CSR_RING_CONFIG 0x000
+#define ADF_RING_CSR_RING_LBASE 0x040
+#define ADF_RING_CSR_RING_UBASE 0x080
+#define ADF_RING_CSR_RING_HEAD 0x0C0
+#define ADF_RING_CSR_RING_TAIL 0x100
+#define ADF_RING_CSR_E_STAT 0x14C
+#define ADF_RING_CSR_INT_FLAG 0x170
+#define ADF_RING_CSR_INT_SRCSEL 0x174
+#define ADF_RING_CSR_INT_SRCSEL_2 0x178
+#define ADF_RING_CSR_INT_COL_EN 0x17C
+#define ADF_RING_CSR_INT_COL_CTL 0x180
+#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184
+#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000
+#define ADF_RING_BUNDLE_SIZE 0x1000
+#define ADF_GEN2_RX_RINGS_OFFSET 8
+#define ADF_GEN2_TX_RINGS_MASK 0xFF
+
+#define BUILD_RING_BASE_ADDR(addr, size) \
+ (((addr) >> 6) & (GENMASK_ULL(63, 0) << (size)))
+#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \
+ ADF_CSR_RD(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_HEAD + \
+ ((ring) << 2))
+#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \
+ ADF_CSR_RD(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_TAIL + \
+ ((ring) << 2))
+#define READ_CSR_E_STAT(csr_base_addr, bank) \
+ ADF_CSR_RD(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_E_STAT)
+#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + \
+ ADF_RING_CSR_RING_CONFIG + ((ring) << 2), \
+ value)
+#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \
+ do { \
+ u32 l_base = 0, u_base = 0; \
+ l_base = (u32)((value)&0xFFFFFFFF); \
+ u_base = (u32)(((value)&0xFFFFFFFF00000000ULL) >> 32); \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + \
+ ADF_RING_CSR_RING_LBASE + ((ring) << 2), \
+ l_base); \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + \
+ ADF_RING_CSR_RING_UBASE + ((ring) << 2), \
+ u_base); \
+ } while (0)
+
+#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_HEAD + \
+ ((ring) << 2), \
+ value)
+#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_TAIL + \
+ ((ring) << 2), \
+ value)
+#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_INT_FLAG, \
+ value)
+#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \
+ do { \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + \
+ ADF_RING_CSR_INT_SRCSEL, \
+ ADF_BANK_INT_SRC_SEL_MASK_0); \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + \
+ ADF_RING_CSR_INT_SRCSEL_2, \
+ ADF_BANK_INT_SRC_SEL_MASK_X); \
+ } while (0)
+#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_INT_COL_EN, \
+ value)
+#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_INT_COL_CTL, \
+ ADF_RING_CSR_INT_COL_CTL_ENABLE | (value))
+#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + \
+ ADF_RING_CSR_INT_FLAG_AND_COL, \
+ value)
+
+/* AE to function map */
+#define AE2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190)
+#define AE2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310)
+#define AE2FUNCTION_MAP_REG_SIZE 4
+#define AE2FUNCTION_MAP_VALID BIT(7)
+
+#define READ_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index) \
+ ADF_CSR_RD(pmisc_bar_addr, \
+ AE2FUNCTION_MAP_A_OFFSET + \
+ AE2FUNCTION_MAP_REG_SIZE * (index))
+#define WRITE_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index, value) \
+ ADF_CSR_WR(pmisc_bar_addr, \
+ AE2FUNCTION_MAP_A_OFFSET + \
+ AE2FUNCTION_MAP_REG_SIZE * (index), \
+ value)
+#define READ_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index) \
+ ADF_CSR_RD(pmisc_bar_addr, \
+ AE2FUNCTION_MAP_B_OFFSET + \
+ AE2FUNCTION_MAP_REG_SIZE * (index))
+#define WRITE_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index, value) \
+ ADF_CSR_WR(pmisc_bar_addr, \
+ AE2FUNCTION_MAP_B_OFFSET + \
+ AE2FUNCTION_MAP_REG_SIZE * (index), \
+ value)
+
+/* Admin Interface Offsets */
+#define ADF_ADMINMSGUR_OFFSET (0x3A000 + 0x574)
+#define ADF_ADMINMSGLR_OFFSET (0x3A000 + 0x578)
+#define ADF_MAILBOX_BASE_OFFSET 0x20970
+
+/* Arbiter configuration */
+#define ADF_ARB_OFFSET 0x30000
+#define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180
+#define ADF_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0))
+#define ADF_ARB_REG_SLOT 0x1000
+#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C
+
+#define READ_CSR_RING_SRV_ARB_EN(csr_addr, index) \
+ ADF_CSR_RD(csr_addr, \
+ ADF_ARB_RINGSRVARBEN_OFFSET + (ADF_ARB_REG_SLOT * (index)))
+
+#define WRITE_CSR_RING_SRV_ARB_EN(csr_addr, index, value) \
+ ADF_CSR_WR(csr_addr, \
+ ADF_ARB_RINGSRVARBEN_OFFSET + (ADF_ARB_REG_SLOT * (index)), \
+ value)
+
+/* Power gating */
+#define ADF_POWERGATE_DC BIT(23)
+#define ADF_POWERGATE_PKE BIT(24)
+
+/* Default ring mapping */
+#define ADF_GEN2_DEFAULT_RING_TO_SRV_MAP \
+ (CRYPTO << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \
+ CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
+ UNUSED << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
+ COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
+
+/* Error detection and correction */
+#define ADF_GEN2_AE_CTX_ENABLES(i) ((i)*0x1000 + 0x20818)
+#define ADF_GEN2_AE_MISC_CONTROL(i) ((i)*0x1000 + 0x20960)
+#define ADF_GEN2_ENABLE_AE_ECC_ERR BIT(28)
+#define ADF_GEN2_ENABLE_AE_ECC_PARITY_CORR (BIT(24) | BIT(12))
+#define ADF_GEN2_UERRSSMSH(i) ((i)*0x4000 + 0x18)
+#define ADF_GEN2_CERRSSMSH(i) ((i)*0x4000 + 0x10)
+#define ADF_GEN2_ERRSSMSH_EN BIT(3)
+
+#define ADF_NUM_HB_CNT_PER_AE (ADF_NUM_THREADS_PER_AE + ADF_NUM_PKE_STRAND)
+
+void adf_gen2_init_hw_csr_info(struct adf_hw_csr_info *csr_info);
+
+#endif
diff --git a/sys/dev/qat/include/common/adf_gen4_hw_data.h b/sys/dev/qat/include/common/adf_gen4_hw_data.h
new file mode 100644
index 000000000000..c0ef0c92772e
--- /dev/null
+++ b/sys/dev/qat/include/common/adf_gen4_hw_data.h
@@ -0,0 +1,132 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+/* Copyright(c) 2021 Intel Corporation */
+/* $FreeBSD$ */
+#ifndef ADF_GEN4_HW_CSR_DATA_H_
+#define ADF_GEN4_HW_CSR_DATA_H_
+
+#include "adf_accel_devices.h"
+
+/* Transport access */
+#define ADF_BANK_INT_SRC_SEL_MASK 0x44UL
+#define ADF_RING_CSR_RING_CONFIG 0x1000
+#define ADF_RING_CSR_RING_LBASE 0x1040
+#define ADF_RING_CSR_RING_UBASE 0x1080
+#define ADF_RING_CSR_RING_HEAD 0x0C0
+#define ADF_RING_CSR_RING_TAIL 0x100
+#define ADF_RING_CSR_E_STAT 0x14C
+#define ADF_RING_CSR_INT_FLAG 0x170
+#define ADF_RING_CSR_INT_SRCSEL 0x174
+#define ADF_RING_CSR_INT_COL_CTL 0x180
+#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184
+#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000
+#define ADF_RING_CSR_INT_COL_EN 0x17C
+#define ADF_RING_CSR_ADDR_OFFSET 0x100000
+#define ADF_RING_BUNDLE_SIZE 0x2000
+
+#define BUILD_RING_BASE_ADDR(addr, size) \
+ ((((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) << 6)
+#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \
+ ADF_CSR_RD((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_RING_HEAD + ((ring) << 2))
+#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \
+ ADF_CSR_RD((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_RING_TAIL + ((ring) << 2))
+#define READ_CSR_E_STAT(csr_base_addr, bank) \
+ ADF_CSR_RD((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_E_STAT)
+#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_RING_CONFIG + ((ring) << 2), \
+ value)
+#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \
+ do { \
+ struct resource *_csr_base_addr = csr_base_addr; \
+ u32 _bank = bank; \
+ u32 _ring = ring; \
+ dma_addr_t _value = value; \
+ u32 l_base = 0, u_base = 0; \
+ l_base = lower_32_bits(_value); \
+ u_base = upper_32_bits(_value); \
+ ADF_CSR_WR((_csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + \
+ ADF_RING_BUNDLE_SIZE * (_bank) + \
+ ADF_RING_CSR_RING_LBASE + ((_ring) << 2), \
+ l_base); \
+ ADF_CSR_WR((_csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + \
+ ADF_RING_BUNDLE_SIZE * (_bank) + \
+ ADF_RING_CSR_RING_UBASE + ((_ring) << 2), \
+ u_base); \
+ } while (0)
+
+#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_RING_HEAD + ((ring) << 2), \
+ value)
+#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_RING_TAIL + ((ring) << 2), \
+ value)
+#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_INT_FLAG, \
+ (value))
+#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_INT_SRCSEL, \
+ ADF_BANK_INT_SRC_SEL_MASK)
+#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_INT_COL_EN, \
+ (value))
+#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_INT_COL_CTL, \
+ ADF_RING_CSR_INT_COL_CTL_ENABLE | (value))
+#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_INT_FLAG_AND_COL, \
+ (value))
+
+/* Arbiter configuration */
+#define ADF_RING_CSR_RING_SRV_ARB_EN 0x19C
+
+#define READ_CSR_RING_SRV_ARB_EN(csr_base_addr, bank) \
+ ADF_CSR_RD((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_RING_SRV_ARB_EN)
+
+#define WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_RING_SRV_ARB_EN, \
+ (value))
+
+/* WDT timers
+ *
+ * Timeout is in cycles. Clock speed may vary across products but this
+ * value should be a few milli-seconds.
+ */
+#define ADF_SSM_WDT_DEFAULT_VALUE 0x7000000ULL
+#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x8000000
+#define ADF_SSMWDTL_OFFSET 0x54
+#define ADF_SSMWDTH_OFFSET 0x5C
+#define ADF_SSMWDTPKEL_OFFSET 0x58
+#define ADF_SSMWDTPKEH_OFFSET 0x60
+
+#define ADF_NUM_HB_CNT_PER_AE (ADF_NUM_THREADS_PER_AE)
+
+int adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev);
+void adf_gen4_init_hw_csr_info(struct adf_hw_csr_info *csr_info);
+#endif
diff --git a/sys/dev/qat/include/common/icp_qat_fw_loader_handle.h b/sys/dev/qat/include/common/icp_qat_fw_loader_handle.h
index a8afb5a4b377..7f6e80eeb431 100644
--- a/sys/dev/qat/include/common/icp_qat_fw_loader_handle.h
+++ b/sys/dev/qat/include/common/icp_qat_fw_loader_handle.h
@@ -16,6 +16,7 @@ struct icp_qat_fw_loader_ae_data {
struct icp_qat_fw_loader_hal_handle {
struct icp_qat_fw_loader_ae_data aes[ICP_QAT_UCLO_MAX_AE];
unsigned int ae_mask;
+ unsigned int admin_ae_mask;
unsigned int slice_mask;
unsigned int revision_id;
unsigned int ae_max_num;
diff --git a/sys/dev/qat/include/common/icp_qat_hal.h b/sys/dev/qat/include/common/icp_qat_hal.h
index 3a7475f25333..68e12826a7e8 100644
--- a/sys/dev/qat/include/common/icp_qat_hal.h
+++ b/sys/dev/qat/include/common/icp_qat_hal.h
@@ -52,23 +52,32 @@ enum hal_ae_csr {
};
enum fcu_csr {
- FCU_CONTROL = 0x0,
- FCU_STATUS = 0x4,
- FCU_DRAM_ADDR_LO = 0xc,
+ FCU_CONTROL = 0x00,
+ FCU_STATUS = 0x04,
+ FCU_DRAM_ADDR_LO = 0x0c,
FCU_DRAM_ADDR_HI = 0x10,
FCU_RAMBASE_ADDR_HI = 0x14,
FCU_RAMBASE_ADDR_LO = 0x18
};
enum fcu_csr_c4xxx {
- FCU_CONTROL_C4XXX = 0x0,
- FCU_STATUS_C4XXX = 0x4,
- FCU_STATUS1_C4XXX = 0xc,
+ FCU_CONTROL_C4XXX = 0x00,
+ FCU_STATUS_C4XXX = 0x04,
+ FCU_STATUS1_C4XXX = 0x0c,
FCU_AE_LOADED_C4XXX = 0x10,
FCU_DRAM_ADDR_LO_C4XXX = 0x14,
FCU_DRAM_ADDR_HI_C4XXX = 0x18,
};
+enum fcu_csr_4xxx {
+ FCU_CONTROL_4XXX = 0x00,
+ FCU_STATUS_4XXX = 0x04,
+ FCU_ME_BROADCAST_MASK_TYPE = 0x08,
+ FCU_AE_LOADED_4XXX = 0x10,
+ FCU_DRAM_ADDR_LO_4XXX = 0x14,
+ FCU_DRAM_ADDR_HI_4XXX = 0x18,
+};
+
enum fcu_cmd {
FCU_CTRL_CMD_NOOP = 0,
FCU_CTRL_CMD_AUTH = 1,
@@ -104,6 +113,7 @@ enum fcu_sts {
#define LCS_STATUS (0x1)
#define MMC_SHARE_CS_BITPOS 2
#define GLOBAL_CSR 0xA00
+#define FCU_CTRL_BROADCAST_POS 0x4
#define FCU_CTRL_AE_POS 0x8
#define FCU_AUTH_STS_MASK 0x7
#define FCU_STS_DONE_POS 0x9
@@ -111,20 +121,26 @@ enum fcu_sts {
#define FCU_LOADED_AE_POS 0x16
#define FW_AUTH_WAIT_PERIOD 10
#define FW_AUTH_MAX_RETRY 300
+#define FW_BROADCAST_MAX_RETRY 300
#define FCU_OFFSET 0x8c0
#define FCU_OFFSET_C4XXX 0x1000
+#define FCU_OFFSET_4XXX 0x1000
#define MAX_CPP_NUM 2
#define AE_CPP_NUM 2
#define AES_PER_CPP 16
#define SLICES_PER_CPP 6
#define ICP_QAT_AE_OFFSET 0x20000
#define ICP_QAT_AE_OFFSET_C4XXX 0x40000
+#define ICP_QAT_AE_OFFSET_4XXX 0x600000
#define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000)
#define ICP_QAT_CAP_OFFSET_C4XXX 0x70000
+#define ICP_QAT_CAP_OFFSET_4XXX 0x640000
#define LOCAL_TO_XFER_REG_OFFSET 0x800
#define ICP_QAT_EP_OFFSET 0x3a000
#define ICP_QAT_EP_OFFSET_C4XXX 0x60000
+#define ICP_QAT_EP_OFFSET_4XXX 0x200000 /* HI MMIO CSRs */
#define MEM_CFG_ERR_BIT 0x20
+#define AE_TG_NUM_CPM2X 4
#define CAP_CSR_ADDR(csr) (csr + handle->hal_cap_g_ctl_csr_addr_v)
#define SET_CAP_CSR(handle, csr, val) \
@@ -133,20 +149,17 @@ enum fcu_sts {
ADF_CSR_RD(handle->hal_misc_addr_v, CAP_CSR_ADDR(csr))
#define SET_GLB_CSR(handle, csr, val) \
({ \
- typeof(handle) handle_ = (handle); \
- typeof(csr) csr_ = (csr); \
- typeof(val) val_ = (val); \
- (IS_QAT_GEN3(pci_get_device(GET_DEV(handle_->accel_dev)))) ? \
- SET_CAP_CSR(handle_, (csr_), (val_)) : \
- SET_CAP_CSR(handle_, csr_ + GLOBAL_CSR, val_); \
+ u32 dev_id = pci_get_device(GET_DEV((handle)->accel_dev)); \
+ (IS_QAT_GEN3_OR_GEN4(dev_id)) ? \
+ SET_CAP_CSR((handle), (csr), (val)) : \
+ SET_CAP_CSR((handle), (csr) + GLOBAL_CSR, val); \
})
#define GET_GLB_CSR(handle, csr) \
({ \
- typeof(handle) handle_ = (handle); \
- typeof(csr) csr_ = (csr); \
- (IS_QAT_GEN3(pci_get_device(GET_DEV(handle_->accel_dev)))) ? \
- (GET_CAP_CSR(handle_, (csr_))) : \
- (GET_CAP_CSR(handle_, (GLOBAL_CSR + (csr_)))); \
+ u32 dev_id = pci_get_device(GET_DEV((handle)->accel_dev)); \
+ (IS_QAT_GEN3_OR_GEN4(dev_id)) ? \
+ GET_CAP_CSR((handle), (csr)) : \
+ GET_CAP_CSR((handle), (csr) + GLOBAL_CSR); \
})
#define SET_FCU_CSR(handle, csr, val) \
({ \
@@ -157,7 +170,12 @@ enum fcu_sts {
SET_CAP_CSR(handle_, \
((csr_) + FCU_OFFSET_C4XXX), \
(val_)) : \
- SET_CAP_CSR(handle_, ((csr_) + FCU_OFFSET), (val_)); \
+ ((IS_QAT_GEN4( \
+ pci_get_device(GET_DEV(handle_->accel_dev)))) ? \
+ SET_CAP_CSR(handle_, \
+ ((csr_) + FCU_OFFSET_4XXX), \
+ (val_)) : \
+ SET_CAP_CSR(handle_, ((csr_) + FCU_OFFSET), (val_))); \
})
#define GET_FCU_CSR(handle, csr) \
({ \
@@ -165,7 +183,10 @@ enum fcu_sts {
typeof(csr) csr_ = (csr); \
(IS_QAT_GEN3(pci_get_device(GET_DEV(handle_->accel_dev)))) ? \
GET_CAP_CSR(handle_, (FCU_OFFSET_C4XXX + (csr_))) : \
- GET_CAP_CSR(handle_, (FCU_OFFSET + (csr_))); \
+ ((IS_QAT_GEN4( \
+ pci_get_device(GET_DEV(handle_->accel_dev)))) ? \
+ GET_CAP_CSR(handle_, (FCU_OFFSET_4XXX + (csr_))) : \
+ GET_CAP_CSR(handle_, (FCU_OFFSET + (csr_)))); \
})
#define AE_CSR(handle, ae) \
((handle)->hal_cap_ae_local_csr_addr_v + ((ae) << 12))
@@ -184,13 +205,19 @@ enum fcu_sts {
ADF_CSR_WR((handle)->hal_sram_addr_v, addr, val)
#define GET_CSR_OFFSET(device_id, cap_offset_, ae_offset_, ep_offset_) \
({ \
- int gen3 = IS_QAT_GEN3(device_id); \
- cap_offset_ = \
- (gen3 ? ICP_QAT_CAP_OFFSET_C4XXX : ICP_QAT_CAP_OFFSET); \
- ae_offset_ = \
- (gen3 ? ICP_QAT_AE_OFFSET_C4XXX : ICP_QAT_AE_OFFSET); \
- ep_offset_ = \
- (gen3 ? ICP_QAT_EP_OFFSET_C4XXX : ICP_QAT_EP_OFFSET); \
+ if (IS_QAT_GEN3(device_id)) { \
+ cap_offset_ = ICP_QAT_CAP_OFFSET_C4XXX; \
+ ae_offset_ = ICP_QAT_AE_OFFSET_C4XXX; \
+ ep_offset_ = ICP_QAT_EP_OFFSET_C4XXX; \
+ } else if (IS_QAT_GEN4(device_id)) { \
+ cap_offset_ = ICP_QAT_CAP_OFFSET_4XXX; \
+ ae_offset_ = ICP_QAT_AE_OFFSET_4XXX; \
+ ep_offset_ = ICP_QAT_EP_OFFSET_4XXX; \
+ } else { \
+ cap_offset_ = ICP_QAT_CAP_OFFSET; \
+ ae_offset_ = ICP_QAT_AE_OFFSET; \
+ ep_offset_ = ICP_QAT_EP_OFFSET; \
+ } \
})
#endif
diff --git a/sys/dev/qat/include/common/icp_qat_uclo.h b/sys/dev/qat/include/common/icp_qat_uclo.h
index 21a1c2fc8ace..1bdddce1d85e 100644
--- a/sys/dev/qat/include/common/icp_qat_uclo.h
+++ b/sys/dev/qat/include/common/icp_qat_uclo.h
@@ -9,6 +9,7 @@
#define ICP_QAT_AC_C3XXX_DEV_TYPE 0x02000000
#define ICP_QAT_AC_200XX_DEV_TYPE 0x02000000
#define ICP_QAT_AC_C4XXX_DEV_TYPE 0x04000000
+#define ICP_QAT_AC_4XXX_A_DEV_TYPE 0x08000000
#define ICP_QAT_UCLO_MAX_AE 32
#define ICP_QAT_UCLO_MAX_CTX 8
#define ICP_QAT_UCLO_MAX_CPPNUM 2
*** 16121 LINES SKIPPED ***