git: 73bb24792747 - main - dpaa: Add QorIQ Security Engine (SEC) driver
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Fri, 18 Sep 2026 13:03:53 UTC
The branch main has been updated by jhibbits:
URL: https://cgit.FreeBSD.org/src/commit/?id=73bb247927470cb2f96cb06fc00a24019cca2607
commit 73bb247927470cb2f96cb06fc00a24019cca2607
Author: Justin Hibbits <jhibbits@FreeBSD.org>
AuthorDate: 2026-07-30 02:19:52 +0000
Commit: Justin Hibbits <jhibbits@FreeBSD.org>
CommitDate: 2026-09-18 13:03:29 +0000
dpaa: Add QorIQ Security Engine (SEC) driver
The QorIQ Security Engine (SEC) generally fits into the Data Path
Acceleration Architecture, accelerating cryptographic operations.
Currently this driver does not use the QMan interface, instead relying
on the job rings, as Linux also does, to reduce complexity until needed
for network protocol acceleration, like IPSec and OpenVPN.
Initial testing via `openssl speed -engine devcrpto -evp aes-128-cbc`
yields a ~200x throughput improvement, from 105MB/s to more than 20GB/s
for 16k block sizes.
Differential Revision: https://reviews.freebsd.org/D59581
---
sys/conf/files.powerpc | 4 +-
sys/dev/dpaa/dpaa_common.h | 4 +
sys/dev/dpaa/sec_dev.c | 2291 ++++++++++++++++++++++++++++++++++++++++++++
sys/dev/dpaa/sec_jr.c | 400 ++++++++
sys/dev/dpaa/sec_var.h | 162 ++++
5 files changed, 2860 insertions(+), 1 deletion(-)
diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc
index 06b189b95edb..5cd9625d268c 100644
--- a/sys/conf/files.powerpc
+++ b/sys/conf/files.powerpc
@@ -113,6 +113,8 @@ dev/dpaa/qman_portal_if.m optional dpaa
dev/dpaa/qman_portals.c optional dpaa fdt
dev/dpaa/qman.c optional dpaa
dev/dpaa/qman_fdt.c optional dpaa fdt
+dev/dpaa/sec_dev.c optional dpaa crypto
+dev/dpaa/sec_jr.c optional dpaa crypto
dev/fb/fb.c optional sc
dev/gpio/qoriq_gpio.c optional mpc85xx gpio
dev/hwpmc/hwpmc_e500.c optional hwpmc
@@ -247,7 +249,7 @@ dev/quicc/quicc_bfe_fdt.c optional quicc mpc85xx
dev/random/darn.c optional powerpc64 !random_loadable | powerpc64le !random_loadable
dev/scc/scc_bfe_macio.c optional scc powermac
dev/sdhci/sdhci_fsl_fdt.c optional mpc85xx sdhci
-dev/sec/sec.c optional sec mpc85xx
+dev/sec/sec.c optional sec mpc85xx powerpc
dev/sound/macio/aoa.c optional snd_davbus | snd_ai2s powermac
dev/sound/macio/davbus.c optional snd_davbus powermac
dev/sound/macio/i2s.c optional snd_ai2s powermac
diff --git a/sys/dev/dpaa/dpaa_common.h b/sys/dev/dpaa/dpaa_common.h
index 8fa43a034ef5..5693789b113b 100644
--- a/sys/dev/dpaa/dpaa_common.h
+++ b/sys/dev/dpaa/dpaa_common.h
@@ -24,7 +24,11 @@ struct dpaa_fd {
uint32_t cmd_stat;
} __packed;
+#define DPAA_FD_FORMAT_SHORT_SBSF 0
+#define DPAA_FD_FORMAT_COMPOUND 1
+#define DPAA_FD_FORMAT_LONG_SBSF 2
#define DPAA_FD_FORMAT_SHORT_MBSF 4
+#define DPAA_FD_FORMAT_LONG_MBSF 6
#define DPAA_FD_RX_STATUS_DCL4C 0x10000000
#define DPAA_FD_RX_STATUS_DME 0x01000000
diff --git a/sys/dev/dpaa/sec_dev.c b/sys/dev/dpaa/sec_dev.c
new file mode 100644
index 000000000000..81b325806c3b
--- /dev/null
+++ b/sys/dev/dpaa/sec_dev.c
@@ -0,0 +1,2291 @@
+/*
+ * Copyright (c) 2026 Justin Hibbits <jhibbits@FreeBSD.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/callout.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/queue.h>
+#include <sys/rman.h>
+#include <sys/smp.h>
+
+#include <machine/atomic.h>
+#include <machine/bus.h>
+#include <machine/resource.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <sys/endian.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
+
+#include "sec_var.h"
+#include "cryptodev_if.h"
+
+/*
+ * Most of this work is based on the T2080 Security (SEC) Reference Manual.
+ *
+ * The driver uses the Job Ring interface for all jobs. The QI interface can be
+ * added if IPSec, OVPN, or kTLS acceleration is added.
+ */
+
+/* From T2080 Security Reference Manual */
+#define SEC_MAX_SHDESC_WORDS 62
+
+#define SEC_MAX_JR 4 /* T2080 exposes four Job Rings */
+
+/* CCSR register offsets. */
+#define SEC_MCFGR 0x0004
+#define MCFGR_SWRST 0x80000000 /* Software reset */
+#define MCFGR_WDE 0x40000000 /* DECO watchdog enable */
+#define MCFGR_WDF 0x20000000 /* Watchdog fast (test only) */
+#define MCFGR_DMARST 0x10000000 /* DMA reset (with SWRST) */
+#define MCFGR_WRHD 0x08000000 /* Write handoff disable */
+#define MCFGR_DJPC 0x00200000 /* Disable job perf ctrs */
+#define MCFGR_DBPC 0x00100000 /* Disable byte perf ctrs */
+#define MCFGR_PS 0x00010000 /* Large pointers */
+#define MCFGR_ARCACHE_M 0x0000f000 /* AXI read cache attrs */
+#define MCFGR_AWCACHE_M 0x00000f00 /* AXI write cache attrs */
+#define MCFGR_AXIPRI 0x00000008 /* AXI master priority */
+#define MCFGR_LARGE_BURST 0x00000004 /* Enable 256B bursts */
+#define SEC_SCFGR 0x000c
+#define SCFGR_VIRT_EN 0x00008000 /* Virtualization enabled */
+
+#define SEC_RDSTA 0x06c0 /* RNG DRNG Status */
+#define RDSTA_IF0 0x00000001 /* State handle 0 up */
+#define RDSTA_IF1 0x00000002 /* State handle 1 up */
+#define RDSTA_ERRCODE_M 0x000f0000
+#define RDSTA_ERRCODE_S 16
+#define RDSTA_CE 0x00100000 /* Catastrophic error */
+
+/* DECO direct-access registers */
+#define SEC_DECORR 0x009c /* DECO Request Register */
+#define DECORR_DEN0 0x00010000 /* DECO0 enable (RO, bit 16) */
+#define DECORR_RQD0 0x00000001 /* DECO0 request */
+#define SEC_D0LIODNR_MS 0x00a0
+#define SEC_D0LIODNR_LS 0x00a4
+#define SEC_D0JQCR_MS 0x8800 /* JQCR upper: WHL/FOUR/SOB */
+#define DAJQCR_MS_WHL 0x20000000 /* Whole descriptor loaded */
+#define DAJQCR_MS_FOUR 0x10000000 /* >= 4 words in first burst */
+#define DAJQCR_MS_SOB 0x00010000 /* Shared/burst loaded */
+#define DAJQCR_MS_SRC_M 0x00000700 /* Job source */
+#define DAJQCR_MS_SRC_S 8
+#define SEC_D0JQCR_LS 0x8804
+#define SEC_D0DAR_MS 0x8808 /* Descriptor address, upper */
+#define SEC_D0DAR_LS 0x880c
+#define SEC_D0DESB(n) (0x8a00 + (n) * 4) /* n = 0..63 */
+#define SEC_D0DDR 0x8e04 /* Debug status */
+#define DADDR_VALID 0x80000000 /* Job currently running */
+#define DADDR_DECO_STATE_M 0x00f00000 /* Main state machine */
+#define DADDR_DECO_STATE_S 20
+
+/* Fault-address registers. */
+#define SEC_FAR_HI 0x0fc0 /* Fault Address, upper */
+#define SEC_FAR_LO 0x0fc4 /* Fault Address, lower */
+#define SEC_FALR 0x0fc8 /* Fault Address LIODN */
+#define SEC_FADR 0x0fcc /* Fault Address Detail */
+#define FADR_FERR_M 0xc0000000 /* AXI error response */
+#define FADR_FERR_S 30
+#define FADR_FSZ_EXT_M 0x00070000 /* Transfer size high 3 bits */
+#define FADR_FSZ_EXT_S 16
+#define FADR_DTYP 0x00008000 /* 0=message, 1=control */
+#define FADR_JSRC_M 0x00007000 /* Job source */
+#define FADR_JSRC_S 12
+#define FADR_BLKID_M 0x00000f00 /* SEC internal block ID */
+#define FADR_BLKID_S 8
+#define FADR_TYP 0x00000080 /* 0=read, 1=write */
+#define FADR_FSZ_M 0x0000007f /* Transfer size low 7 bits */
+
+#define SEC_RD4(sc, off) bus_read_4((sc)->sc_rres, (off))
+#define SEC_WR4(sc, off, v) bus_write_4((sc)->sc_rres, (off), (v))
+
+/* Descriptor command components */
+/* SEQ commands are intended for network protocols */
+#define CMD_DESC(n) ((n) << 27)
+#define CMD_KEY 0x00 /* Pointer/key follows descriptor */
+#define CMD_SEQ_KEY 0x01
+#define KEY_CLASS_M 0x06000000
+#define KEY_CLASS_1 0x02000000
+#define KEY_CLASS_2 0x04000000
+#define KEY_SGF 0x01000000 /* KEY - Pointer to SGT */
+#define KEY_VLF 0x01000000 /* SK - variable length */
+#define KEY_IMM 0x00800000 /* KEY - Key follows descriptor */
+#define KEY_AIDF 0x00800000 /* SK - Already in Input FIFO */
+#define KEY_ENC 0x00400000 /* Key is encrypted */
+#define KEY_NWB 0x00200000 /* No write back */
+#define KEY_EKT 0x00100000 /* Encrypted Key Type:
+ * 0 - AES-CCB
+ * 1 - AES-CCM
+ */
+#define KEY_KDEST_M 0x00030000 /* Key Destination */
+#define KEY_KDEST_REG 0x00000000 /* Dest is Key register */
+#define KEY_KDEST_PKHA 0x00010000 /* Dest is PKHA E-memory */
+#define KEY_KDEST_AFHA 0x00020000 /* Dest is AFHA S-Box */
+#define KEY_KDEST_MDHA_SPLIT 0x00030000 /* Key is MDHA split key */
+#define KEY_TK 0x00008000 /* Trusted Key */
+#define KEY_LENGTH_M 0x000003ff /* Key length */
+#define CMD_LOAD 0x02
+#define CMD_SEQ_LOAD 0x03
+#define LOAD_CLASS_M 0x06000000
+#define LOAD_CLASS_1 0x02000000
+#define LOAD_CLASS_2 0x04000000
+#define LOAD_CLASS_3 0x06000000
+#define LOAD_SGF 0x01000000 /* LOAD - Pointer to SGT */
+#define LOAD_VLF 0x01000000 /* SL - variable length */
+#define LOAD_IMM 0x00800000 /* LOAD - Data follows descriptor */
+#define LOAD_DST_M 0x007f0000 /* Destination register */
+#define LOAD_DST_S 16
+#define LOAD_KSR 0x00010000 /* Key Size Register (C1/C2) */
+#define LOAD_DSR 0x00020000 /* Data Size Register (C1/C2) */
+#define LOAD_ICVS 0x00030000 /* ICV Size Register (C1/C2) */
+#define LOAD_LSR 0x00040000 /* LIODN Status Register (C3) */
+#define LOAD_DCTRL2 0x00050000 /* DECO Control Register 2(C3) */
+#define LOAD_CCTRL 0x00060000 /* CHA Control Register (C1) */
+#define LOAD_DCTRL 0x00060000 /* DECO Control Register (C3) */
+#define LOAD_ICTRL 0x00070000 /* IRQ Control Register (C0) */
+#define LOAD_DPOVRD 0x00070000 /* DECO Protocol Override (C3) */
+#define LOAD_CLRW 0x00080000 /* Clear Written Register (C0) */
+#define LOAD_MATH0W 0x00080000 /* DECO Math Register 0 (C3) */
+#define LOAD_MATH1W 0x00090000 /* DECO Math Register 1 (C3) */
+#define LOAD_MATH2W 0x000a0000 /* DECO Math Register 2 (C3) */
+#define LOAD_CISEL 0x000a0000 /* CHA Instance Select Reg (C0) */
+#define LOAD_AADSZ 0x000b0000 /* AAD Size Register (C1) */
+#define LOAD_MAT3W 0x000b0000 /* DECO Math Register 3 (C3) */
+#define LOAD_C1VSZ 0x000c0000 /* Class 1 IV SIze Register (C1) */
+#define LOAD_ALTDS1 0x000f0000 /* Alternate Data Size C1 (C1) */
+#define LOAD_PKASZ 0x00100000 /* PKHA A Size Register (C1) */
+#define LOAD_PKBSZ 0x00110000 /* PKHA B Size Register (C1) */
+#define LOAD_PKNSZ 0x00120000 /* PKHA N Size Register (C1) */
+#define LOAD_PKESZ 0x00130000 /* PKHA E Size Register (C1) */
+#define LOAD_CTX 0x00200000 /* Context Register (C1/C2) */
+#define LOAD_KEY 0x00400000 /* Key Register (C1/C2) */
+#define LOAD_DESC_BUF 0x00400000 /* DECO Descriptor Buffer (C3) */
+#define LOAD_NFSL 0x00700000 /* NFIFO and size registers (C0) */
+#define LOAD_NFSM 0x00710000 /* NFIFO and size registers (C0) */
+#define LOAD_NFL 0x00720000 /* NFIFO (C0) */
+#define LOAD_NFM 0x00730000 /* NFIFO (C0) */
+#define LOAD_SL 0x00740000 /* Size register(s) (C0) */
+#define LOAD_SM 0x00750000 /* Size register(s) (C0) */
+#define LOAD_IDFNS 0x00760000 /* Input Data FIFO Nibble Shift (C0) */
+#define LOAD_ODFNS 0x00770000 /* Output Data FIFO Nibble Shift (C0) */
+#define LOAD_AUXDATA 0x00780000 /* Aux Data FIFO (C0) */
+#define LOAD_NFIFO 0x007a0000 /* NFIFO (C0) */
+#define LOAD_IFIFO 0x007c0000 /* Input Data FIFO (C0) */
+#define LOAD_OFIFO 0x007e0000 /* Output Data FIFO (C0) */
+#define LOAD_LENGTH_M 0x000000ff /* Data length (8 bits) */
+#define LOAD_OFFSET_S 8 /* OFFSET field shift (bits 8-15) */
+#define CMD_FIFO_LOAD 0x04
+#define CMD_SEQ_FIFO_LOAD 0x05
+#define CMD_STORE 0x0a
+#define CMD_SEQ_STORE 0x0b
+#define CMD_FIFO_STORE 0x0c
+#define CMD_SEQ_FIFO_STORE 0x0d
+#define CMD_MOVE 0x0e
+#define CMD_MOVE_LEN 0x0f
+#define CMD_OPERATION 0x10
+#define OPTYPE_M 0x07000000
+#define OPTYPE_S 24
+#define OPTYPE_CLASS1_ALG 0x02000000
+#define OPTYPE_CLASS2_ALG 0x04000000
+#define ALG_S 16
+#define CMD_ALGORITHM(m, n) ((m) | ((n) << ALG_S))
+/* Class 1 algorithms */
+#define ALG_AES CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x10)
+#define ALG_DES CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x20)
+#define ALG_3DES CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x21)
+#define ALG_ARC4 CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x30)
+#define ALG_RNG CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x50)
+#define ALG_SNOW3G_F8 CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x60)
+#define ALG_KASUMI CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x70)
+#define ALG_ZUC_ENC CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0xb0)
+/* Class 2 algorithms */
+#define ALG_MD5 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x40)
+#define ALG_SHA1 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x41)
+#define ALG_SHA224 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x42)
+#define ALG_SHA256 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x43)
+#define ALG_SHA384 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x44)
+#define ALG_SHA512 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x45)
+#define ALG_CRC CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x90)
+#define ALG_SNOW3G_F9 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0xa0)
+#define ALG_ZUC_AUTH CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0xc0)
+/* AAI (Additional Algorithm Information) codes. */
+#define AAI_S 4
+/* AES modes */
+#define AAI_AES_CTR (0x00 << AAI_S)
+#define AAI_AES_CBC (0x10 << AAI_S)
+#define AAI_AES_ECB (0x20 << AAI_S)
+#define AAI_AES_CFB (0x30 << AAI_S)
+#define AAI_AES_OFB (0x40 << AAI_S)
+#define AAI_AES_XTS (0x50 << AAI_S)
+#define AAI_AES_CMAC (0x60 << AAI_S)
+#define AAI_AES_XCBC_MAC (0x70 << AAI_S)
+#define AAI_AES_CCM (0x80 << AAI_S)
+#define AAI_AES_GCM (0x90 << AAI_S)
+#define AAI_AES_DK (0x100 << AAI_S) /* Decrypt-key derive */
+/* DES/3DES modes */
+#define AAI_DES_CBC (0x10 << AAI_S)
+#define AAI_DES_ECB (0x20 << AAI_S)
+/* MDHA modes */
+#define AAI_HASH (0x00 << AAI_S)
+#define AAI_HMAC (0x01 << AAI_S)
+#define AAI_HMAC_PRECOMP (0x04 << AAI_S) /* Precomputed IPAD/OPAD */
+/* Algorithm State field (bits 2-3): what phase to run */
+#define AS_S 2
+#define AS_UPDATE (0x0 << AS_S)
+#define AS_INIT (0x1 << AS_S)
+#define AS_FINAL (0x2 << AS_S)
+#define AS_INIT_FINAL (0x3 << AS_S)
+/* RNG-specific: State-Handle field. */
+#define OP_RNG_SH_S 4
+#define OP_RNG_SH(n) ((n) << OP_RNG_SH_S)
+/* Direction / ICV */
+#define OP_ICV 0x00000002
+#define OP_ENC 0x00000001
+
+/* SEQ FIFO LOAD command bits. */
+#define FIFOLD_CLASS_1 0x02000000 /* CLASS = 01b (Class 1) */
+#define FIFOLD_CLASS_2 0x04000000 /* CLASS = 10b (Class 2) */
+#define FIFOLD_CLASS_BOTH 0x06000000 /* CLASS = 11b (snooping) */
+#define FIFOLD_VLF 0x01000000 /* Variable-length flag */
+/*
+ * Input data type: top 3 bits = type,
+ * bottom 3 bits = LC2/LC1/FC1 flags.
+ */
+#define FIFOLD_TYPE_S 16
+#define FIFOLD_TYPE_MSG (0x10 << FIFOLD_TYPE_S) /* 010_000 */
+/* Class 1 output fed straight into Class 2, i.e. MAC over ciphertext. */
+#define FIFOLD_TYPE_MSG_C1OUT (0x18 << FIFOLD_TYPE_S) /* 011_000 */
+#define FIFOLD_TYPE_IV (0x20 << FIFOLD_TYPE_S) /* 100_000 */
+#define FIFOLD_TYPE_AAD (0x30 << FIFOLD_TYPE_S) /* 110_000 */
+#define FIFOLD_TYPE_ICV (0x38 << FIFOLD_TYPE_S) /* 111_000 */
+#define FIFOLD_FC1 (0x01 << FIFOLD_TYPE_S) /* Flush class 1 */
+#define FIFOLD_LC1 (0x02 << FIFOLD_TYPE_S) /* Last for Class 1 */
+#define FIFOLD_LC2 (0x04 << FIFOLD_TYPE_S) /* Last for Class 2 */
+/* Length moves to a 32-bit word after the command. */
+#define FIFO_EXT 0x00400000
+
+/* SEQ FIFO STORE command bits. */
+#define FIFOST_VLF 0x01000000
+#define FIFOST_TYPE_S 16
+#define FIFOST_TYPE_MSG_DATA (0x30 << FIFOST_TYPE_S)
+
+#define CMD_SIGNATURE 0x12
+#define CMD_JUMP 0x14
+#define CMD_MATH 0x15
+#define MATH_FN_ADD (0x0 << 20) /* SRC0 + SRC1 */
+#define MATH_SRC0_SIL (0x8 << 16) /* Sequence In Length */
+#define MATH_SRC1_ZERO (0xF << 12) /* Constant zero */
+#define MATH_DEST_VSIL (0xA << 8) /* Variable SIL */
+#define MATH_DEST_VSOL (0xB << 8) /* Variable SOL */
+#define MATH_LEN_4 0x4
+/* J - Job Descriptor, S - Shared Descriptor */
+#define CMD_DESC_HEADER 0x16
+#define HEADER_EXT 0x04000000 /* Has Extension (J) */
+#define HEADER_RSL 0x02000000 /* Require SEQ LIODN (J) */
+#define HEADER_DNR 0x01000000 /* Do Not Run (J/S) */
+#define HEADER_ONE 0x00800000 /* Must be 1 (J/S) */
+#define HEADER_START_INDEX(n) ((n) << 16) /* Start Index (J/S) */
+#define HEADER_SHR_DESC_L(n) ((n) << 16) /* Shared Desc len (J) */
+/* Bit 16 must be 0 */
+#define HEADER_TDES_M 0x00006000 /* Trusted Descriptor Mask (J) */
+#define HEADER_TDES 0x00004000 /* Trusted Descriptor (J) */
+#define HEADER_TDES_CAND 0x00006000 /* Candidate Trust Desc (J) */
+#define HEADER_SHR 0x00001000 /* Has Shared Descriptor (J) */
+#define HEADER_REO 0x00000800 /* Reverse Execution Order (J) */
+#define HEADER_SHARE_M 0x00000700 /* Share State (J/S) */
+#define HEADER_SHARE_WAIT 0x00000100 /* Wait to share (J/S) */
+#define HEADER_SHARE_SERIAL 0x00000200 /* Serialize (J/S) */
+#define HEADER_SHARE_ALWAYS 0x00000300 /* Always share (stateless) (J/S) */
+#define HEADER_SHARE_DEFER 0x00000400 /* Defer to shared desc (J) */
+#define HEADER_DESCLEN_M 0x0000007f /* Descriptor length */
+#define HEADER_DESCLEN_S 0
+#define HEADER_EXT_FTD 0x00000100 /* Fake Trusted Descriptor */
+#define HEADER_EXT_DSELVALID 0x00000080 /* DECO_SELECT field valid */
+#define HEADER_EXT_DSEL_M 0x0000000f /* DECO Select */
+#define CMD_SHARED_HEADER 0x17
+#define HEADER_RIF 0x02000000 /* Read Input Frame */
+#define HEADER_CIF 0x00002000 /* Clear Input FIFO */
+#define HEADER_SC 0x00001000 /* Save Context */
+#define HEADER_PD 0x00000800 /* Propagate DNR */
+#define CMD_MATHI 0x1d
+#define CMD_SEQ_IN_PTR 0x1e
+#define SEQ_SGF 0x01000000 /* Pointer is SGT (bit 7 NXP) */
+#define SEQ_EXT 0x00400000 /* 32-bit extended length (bit 9 NXP) */
+#define CMD_SEQ_OUT_PTR 0x1f
+
+/* Shared descriptor container. */
+struct sec_context {
+ uint32_t shd[SEC_MAX_SHDESC_WORDS];
+};
+
+
+/*
+ * Session state: one shared descriptor per direction. The shared
+ * descriptor holds just KEY + OPERATION; the per-job JD adds LOAD-IV
+ * and SEQ_IN_PTR / SEQ_OUT_PTR inline.
+ */
+#define SEC_MAX_SPLIT_KEY 128 /* SHA-512 AES-ECB encrypted */
+
+#define SEC_CCM_AAD_MAX 0xfeff
+
+struct sec_session {
+ struct sec_softc *sess_sc;
+ struct sec_context ctx[2]; /* [0]=dec, [1]=enc */
+ uint32_t sdlen[2]; /* words per direction */
+ uint8_t digestlen; /* HMAC output size (0 if none) */
+ uint8_t skeylen; /* HMAC split key size (0 if none) */
+ uint8_t skey[SEC_MAX_SPLIT_KEY];
+};
+
+static device_probe_t sec_probe;
+static device_attach_t sec_attach;
+static device_detach_t sec_detach;
+static cryptodev_probesession_t sec_probe_session;
+static cryptodev_newsession_t sec_new_session;
+static cryptodev_freesession_t sec_free_session;
+static cryptodev_process_t sec_process;
+
+static void sec_intr(void *);
+
+/* Register-level bring-up. Filled in from the SEC reference manual. */
+static int sec_reset(struct sec_softc *);
+static int sec_rng_init(struct sec_softc *);
+
+static struct ofw_compat_data compats[] = {
+ { "fsl,sec-v5.2", 52 },
+ { "fsl,sec-v5.0", 50 },
+ { "fsl,sec-v4.0", 40 },
+ { NULL, 0 }
+};
+
+static device_method_t sec_methods[] = {
+ /* Device methods */
+ DEVMETHOD(device_probe, sec_probe),
+ DEVMETHOD(device_attach, sec_attach),
+ DEVMETHOD(device_detach, sec_detach),
+
+ /* Cryptodev methods */
+ DEVMETHOD(cryptodev_probesession, sec_probe_session),
+ DEVMETHOD(cryptodev_newsession, sec_new_session),
+ DEVMETHOD(cryptodev_freesession, sec_free_session),
+ DEVMETHOD(cryptodev_process, sec_process),
+
+ DEVMETHOD_END
+};
+
+static DEFINE_CLASS_0(sec, sec_driver, sec_methods, sizeof(struct sec_softc));
+DRIVER_MODULE(sec, simplebus, sec_driver, NULL, NULL);
+MODULE_DEPEND(sec, crypto, 1, 1, 1);
+
+MALLOC_DEFINE(M_SEC, "sec", "SEC driver");
+
+static int
+sec_probe(device_t dev)
+{
+ const struct ofw_compat_data *cd;
+
+ cd = ofw_bus_search_compatible(dev, compats);
+ if (cd->ocd_data == 0)
+ return (ENXIO);
+
+ device_set_descf(dev, "Freescale Security Engine v%d.%d",
+ (int)cd->ocd_data / 10, (int)cd->ocd_data % 10);
+
+ return (BUS_PROBE_DEFAULT);
+}
+
+static int
+sec_attach(device_t dev)
+{
+ struct sec_softc *sc = device_get_softc(dev);
+ const struct ofw_compat_data *cd;
+
+ sc->sc_dev = dev;
+ sc->sc_cid = -1;
+
+ cd = ofw_bus_search_compatible(dev, compats);
+ sc->sc_version = cd->ocd_data;
+
+ sc->sc_rrid = 0;
+ sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid,
+ RF_ACTIVE);
+ if (sc->sc_rres == NULL) {
+ device_printf(dev, "could not allocate register resource\n");
+ goto fail;
+ }
+
+ /* TODO: Error IRQ handling. */
+ sc->sc_irid = 0;
+ sc->sc_ires = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->sc_irid,
+ RF_ACTIVE | RF_SHAREABLE);
+ if (sc->sc_ires == NULL) {
+ device_printf(dev, "could not allocate error interrupt\n");
+ goto fail;
+ }
+
+ if (bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0,
+ BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
+ SEC_MAX_SIZE, SEC_MAX_SEGMENTS, SEC_MAX_SIZE, BUS_DMA_ALLOCNOW,
+ NULL, NULL, &sc->sc_dmatag) != 0) {
+ device_printf(dev, "could not create DMA tag\n");
+ goto fail;
+ }
+
+ if (sec_reset(sc) != 0) {
+ device_printf(dev, "SEC reset failed\n");
+ goto fail;
+ }
+ if (sec_rng_init(sc) != 0) {
+ device_printf(dev, "SEC RNG instantiation failed\n");
+ goto fail;
+ }
+ if (sec_init_rings(sc) == 0) {
+ device_printf(dev, "SEC job ring init failed\n");
+ goto fail;
+ }
+
+ /*
+ * Clear any fault-address latch left over from the bootloader before
+ * enabling the error IRQ. FADR, FAR_HI/LO, and FALR must all be read
+ * before they're all cleared, per the RM.
+ */
+ (void)SEC_RD4(sc, SEC_FADR);
+ (void)SEC_RD4(sc, SEC_FAR_HI);
+ (void)SEC_RD4(sc, SEC_FAR_LO);
+ (void)SEC_RD4(sc, SEC_FALR);
+
+ if (bus_setup_intr(dev, sc->sc_ires, INTR_TYPE_MISC | INTR_MPSAFE,
+ NULL, sec_intr, sc, &sc->sc_icookie) != 0) {
+ device_printf(dev, "could not install error interrupt\n");
+ goto fail;
+ }
+
+ sc->sc_cid = crypto_get_driverid(dev, sizeof(struct sec_session),
+ CRYPTOCAP_F_HARDWARE);
+ if (sc->sc_cid < 0) {
+ device_printf(dev, "could not get crypto driver id\n");
+ goto fail;
+ }
+
+ return (0);
+
+fail:
+ sec_detach(dev);
+ return (ENXIO);
+}
+
+static int
+sec_detach(device_t dev)
+{
+ struct sec_softc *sc = device_get_softc(dev);
+ u_int i;
+
+ if (sc->sc_cid >= 0)
+ crypto_unregister_all(sc->sc_cid);
+
+ /* Silence the rings before halting them. */
+ for (i = 0; i < sc->sc_njr; i++) {
+ struct sec_jr *jr = &sc->sc_jr[i];
+
+ if (jr->jr_icookie != NULL)
+ bus_teardown_intr(dev, jr->jr_ires, jr->jr_icookie);
+ if (jr->jr_ires != NULL)
+ bus_release_resource(dev, SYS_RES_IRQ, jr->jr_irid,
+ jr->jr_ires);
+ sec_jr_teardown(sc, jr);
+ }
+ free(sc->sc_jr, M_SEC);
+
+ if (sc->sc_dmatag != NULL)
+ bus_dma_tag_destroy(sc->sc_dmatag);
+ if (sc->sc_icookie != NULL)
+ bus_teardown_intr(dev, sc->sc_ires, sc->sc_icookie);
+ if (sc->sc_ires != NULL)
+ bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irid,
+ sc->sc_ires);
+ if (sc->sc_rres != NULL)
+ bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid,
+ sc->sc_rres);
+
+ return (0);
+}
+
+static const char *sec_ferr_str[] = {
+ "OKAY", "reserved", "SLVERR", "DECERR",
+};
+
+static const char *sec_jsrc_str[] = {
+ "JR0", "JR1", "JR2", "JR3", "RTIC", "QI", "rsvd6", "rsvd7",
+};
+
+static void
+sec_intr(void *arg)
+{
+ struct sec_softc *sc = arg;
+ uint32_t fadr, falr;
+ uint64_t far;
+
+ fadr = SEC_RD4(sc, SEC_FADR);
+ if ((fadr & FADR_FERR_M) != 0) {
+ /*
+ * All fault registers are latched by hardware until all are
+ * read, in any order.
+ */
+ far = (uint64_t)SEC_RD4(sc, SEC_FAR_HI) << 32;
+ far |= SEC_RD4(sc, SEC_FAR_LO);
+ falr = SEC_RD4(sc, SEC_FALR);
+
+ device_printf(sc->sc_dev,
+ "bus fault: FADR=%#x FAR=%#jx FALR=%#x "
+ "(%s, %s, src=%s, blkid=%#x, %s, size=%u)\n",
+ fadr, (uintmax_t)far, falr,
+ sec_ferr_str[(fadr & FADR_FERR_M) >> FADR_FERR_S],
+ (fadr & FADR_DTYP) ? "control" : "message",
+ sec_jsrc_str[(fadr & FADR_JSRC_M) >> FADR_JSRC_S],
+ (fadr & FADR_BLKID_M) >> FADR_BLKID_S,
+ (fadr & FADR_TYP) ? "write" : "read",
+ (unsigned)(((fadr & FADR_FSZ_EXT_M) >>
+ (FADR_FSZ_EXT_S - 7)) | (fadr & FADR_FSZ_M)));
+ }
+
+}
+
+/*
+ * Decode a SEC job termination status word.
+ *
+ * Bits 0-3 (MSB) are the "source" of the report; the remaining bits are
+ * source-specific. Zero means clean completion.
+ *
+ * Two cases we care to distinguish:
+ * - CCB (source 2), ERRID field bits 28-31
+ * value 0xA is "ICV check failed" -> EBADMSG.
+ * - DECO (source 4), Error Code bits 24-31
+ * values F0h/F1h/FFh are informational warnings (IPsec TTL,
+ * 3GPP HFN, output-length rollover). The job actually completed,
+ * so map those to success.
+ *
+ * Everything else is logged and reported as EIO. Real per-code
+ * decoding of DECO/QI errors can be layered on as we hit them.
+ */
+#define SEC_STAT_SOURCE(s) (((s) >> 28) & 0xf)
+#define SEC_SRC_NONE 0x0
+#define SEC_SRC_CCB 0x2
+#define SEC_SRC_DECO 0x4
+#define SEC_SRC_QI 0x5
+#define SEC_SRC_JR 0x6
+#define SEC_CCB_ERR_ICV_FAIL 0x0a
+#define SEC_DECO_ERR_WARN_MIN 0xf0
+
+static int
+sec_decode_status(struct sec_softc *sc, uint32_t status)
+{
+ uint32_t source;
+
+ if (status == 0)
+ return (0);
+
+ source = SEC_STAT_SOURCE(status);
+
+ switch (source) {
+ case SEC_SRC_CCB:
+ if ((status & 0xf) == SEC_CCB_ERR_ICV_FAIL)
+ return (EBADMSG);
+ break;
+ case SEC_SRC_DECO:
+ if ((status & 0xff) >= SEC_DECO_ERR_WARN_MIN)
+ return (0);
+ break;
+ }
+
+ device_printf(sc->sc_dev,
+ "job termination status %#x (source %#x)\n", status, source);
+ return (EIO);
+}
+
+/*
+ * Complete one job that SEC has finished processing.
+ */
+void
+sec_complete_one(struct sec_softc *sc, uint64_t desc_pa, uint32_t status)
+{
+ struct sec_job *job;
+ struct cryptop *crp;
+ const struct crypto_session_params *csp;
+ uint8_t expected[SEC_MAX_DIGEST];
+ int dlen;
+
+ job = (struct sec_job *)PHYS_TO_DMAP((vm_paddr_t)desc_pa);
+ crp = job->crp;
+
+ crp->crp_etype = sec_decode_status(sc, status);
+
+ bus_dmamap_sync(sc->sc_dmatag, job->map,
+ BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+ bus_dmamap_unload(sc->sc_dmatag, job->map);
+ bus_dmamap_destroy(sc->sc_dmatag, job->map);
+
+ if (crp->crp_etype == 0) {
+ csp = crypto_get_params(crp->crp_session);
+ dlen = csp->csp_auth_mlen != 0 ? csp->csp_auth_mlen :
+ job->sess->digestlen;
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) {
+ crypto_copydata(crp, crp->crp_digest_start,
+ dlen, expected);
+ if (timingsafe_bcmp(job->digest, expected,
+ dlen) != 0)
+ crp->crp_etype = EBADMSG;
+ } else {
+ crypto_copyback(crp, crp->crp_digest_start,
+ dlen, job->digest);
+ }
+ break;
+ case CSP_MODE_ETA:
+ if ((crp->crp_op & CRYPTO_OP_ENCRYPT) != 0) {
+ crypto_copyback(crp, crp->crp_digest_start,
+ dlen, job->digest);
+ break;
+ }
+ crypto_copydata(crp, crp->crp_digest_start, dlen,
+ expected);
+ if (timingsafe_bcmp(job->digest, expected, dlen) != 0)
+ crp->crp_etype = EBADMSG;
+ break;
+ case CSP_MODE_AEAD:
+ if ((crp->crp_op & CRYPTO_OP_ENCRYPT) != 0)
+ crypto_copyback(crp, crp->crp_digest_start,
+ dlen, job->digest);
+ break;
+ }
+ }
+
+ crypto_done(crp);
+ free(job, M_SEC);
+}
+
+static bool
+check_cipher(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ case CRYPTO_AES_ICM:
+ if (csp->csp_ivlen != AES_BLOCK_LEN)
+ return (false);
+ return (csp->csp_cipher_klen == 16 ||
+ csp->csp_cipher_klen == 24 ||
+ csp->csp_cipher_klen == 32);
+ case CRYPTO_AES_XTS:
+ if (csp->csp_ivlen != AES_XTS_IV_LEN)
+ return (false);
+ return (csp->csp_cipher_klen == 32 ||
+ csp->csp_cipher_klen == 64);
+ default:
+ return (false);
+ }
+}
+
+static bool
+check_aead(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ if (csp->csp_auth_mlen != 0 &&
+ csp->csp_auth_mlen != AES_GMAC_HASH_LEN)
+ return (false);
+ return (csp->csp_cipher_klen == 16 ||
+ csp->csp_cipher_klen == 24 ||
+ csp->csp_cipher_klen == 32);
+ case CRYPTO_AES_CCM_16:
+ return (csp->csp_cipher_klen == 16 ||
+ csp->csp_cipher_klen == 24 ||
+ csp->csp_cipher_klen == 32);
+ default:
+ return (false);
+ }
+}
+
+/*
+ * Map an opencrypto auth_alg to its SEC selector and digest length.
+ * skeylen is zero for a plain hash, which is what tells the two apart.
+ */
+static bool
+sec_hash_params(int auth_alg, uint32_t *alg, uint8_t *dlen, uint8_t *skeylen)
+{
+
+ switch (auth_alg) {
+ case CRYPTO_SHA1_HMAC:
+ *alg = ALG_SHA1; *dlen = 20; *skeylen = 40; return (true);
+ case CRYPTO_SHA2_224_HMAC:
+ *alg = ALG_SHA224; *dlen = 28; *skeylen = 64; return (true);
+ case CRYPTO_SHA2_256_HMAC:
+ *alg = ALG_SHA256; *dlen = 32; *skeylen = 64; return (true);
+ case CRYPTO_SHA2_384_HMAC:
+ *alg = ALG_SHA384; *dlen = 48; *skeylen = 128; return (true);
+ case CRYPTO_SHA2_512_HMAC:
+ *alg = ALG_SHA512; *dlen = 64; *skeylen = 128; return (true);
+ case CRYPTO_SHA1:
+ *alg = ALG_SHA1; *dlen = 20; *skeylen = 0; return (true);
+ case CRYPTO_SHA2_224:
+ *alg = ALG_SHA224; *dlen = 28; *skeylen = 0; return (true);
+ case CRYPTO_SHA2_256:
+ *alg = ALG_SHA256; *dlen = 32; *skeylen = 0; return (true);
+ case CRYPTO_SHA2_384:
+ *alg = ALG_SHA384; *dlen = 48; *skeylen = 0; return (true);
+ case CRYPTO_SHA2_512:
+ *alg = ALG_SHA512; *dlen = 64; *skeylen = 0; return (true);
+ }
+ return (false);
+}
+
+static bool
+check_digest(const struct crypto_session_params *csp)
+{
+ uint32_t alg;
+ uint8_t dlen, skeylen;
+
+ /* GMAC is AESA rather than MDHA, so it has its own constraints. */
+ if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) {
+ if (csp->csp_ivlen != AES_GCM_IV_LEN)
+ return (false);
+ if (csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
+ return (false);
+ return (csp->csp_auth_klen == 16 ||
+ csp->csp_auth_klen == 24 ||
+ csp->csp_auth_klen == 32);
+ }
+
+ if (!sec_hash_params(csp->csp_auth_alg, &alg, &dlen, &skeylen))
+ return (false);
+ /* Keyed variants require a key; plain hashes must not carry one. */
+ if ((skeylen != 0) != (csp->csp_auth_klen != 0))
+ return (false);
+ return (csp->csp_auth_mlen <= dlen);
+}
+
+static bool
+check_eta(const struct crypto_session_params *csp)
+{
+
+ /*
+ * ESN appends four bytes from crp_esn to the MAC input, which the
+ * descriptor has no way to splice in, so refuse rather than
+ * authenticate the wrong span.
+ */
+ if ((csp->csp_flags & CSP_F_ESN) != 0)
+ return (false);
+ /*
+ * XTS carries its tweak in the class 1 context and pairs with no
+ * MAC; its shared descriptor is shaped differently.
+ */
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+ return (false);
+ /* The MAC half has to be keyed; a bare hash authenticates nothing. */
+ if (csp->csp_auth_klen == 0)
+ return (false);
+ return (check_cipher(csp) && check_digest(csp));
+}
+
+/*
+ * Software split-key generator: computes the HMAC ipad/opad hash-state
+ * halves in software and packs them big-endian for SEC's Class 2 KEY
+ * register.
+ *
+ * Runs the CPU through one SHA block per pad (two total). Much cheaper than
+ * the round trip through the job ring for setup.
+ */
+static void
+sec_pack_state32(uint8_t *dst, const uint32_t *src, unsigned int nbytes)
+{
+ unsigned int i;
+
+ for (i = 0; i < nbytes; i += 4)
+ be32enc(dst + i, src[i / 4]);
+}
+
+static void
+sec_pack_state64(uint8_t *dst, const uint64_t *src, unsigned int nbytes)
+{
+ unsigned int i;
+
+ for (i = 0; i < nbytes; i += 8)
+ be64enc(dst + i, src[i / 8]);
+}
+
+static void
+sec_sw_gen_split_key(const struct crypto_session_params *csp,
+ uint8_t *out, size_t out_len)
+{
+ union authctx ictx, octx;
+ const struct auth_hash *axf;
+ uint8_t half;
+
+ axf = crypto_auth_hash(csp);
+ hmac_init_ipad(axf, csp->csp_auth_key, csp->csp_auth_klen, &ictx);
+ hmac_init_opad(axf, csp->csp_auth_key, csp->csp_auth_klen, &octx);
+
+ KASSERT(out_len % 2 == 0, ("split key len must be even"));
+ half = out_len / 2;
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_SHA1_HMAC:
+ sec_pack_state32(out, ictx.sha1ctx.h.b32, half);
+ sec_pack_state32(out + half, octx.sha1ctx.h.b32, half);
+ break;
+ case CRYPTO_SHA2_224_HMAC:
+ sec_pack_state32(out, ictx.sha224ctx.state, half);
+ sec_pack_state32(out + half, octx.sha224ctx.state, half);
+ break;
+ case CRYPTO_SHA2_256_HMAC:
+ sec_pack_state32(out, ictx.sha256ctx.state, half);
+ sec_pack_state32(out + half, octx.sha256ctx.state, half);
+ break;
+ case CRYPTO_SHA2_384_HMAC:
+ sec_pack_state64(out, ictx.sha384ctx.state, half);
+ sec_pack_state64(out + half, octx.sha384ctx.state, half);
+ break;
+ case CRYPTO_SHA2_512_HMAC:
+ sec_pack_state64(out, ictx.sha512ctx.state, half);
+ sec_pack_state64(out + half, octx.sha512ctx.state, half);
+ break;
+ }
+
+ explicit_bzero(&ictx, sizeof(ictx));
+ explicit_bzero(&octx, sizeof(octx));
+}
+
+/*
+ * Descriptor builder. Word 0 is the HEADER and is filled in last, since its
+ * length field is only known once the body has been emitted.
+ */
+struct sec_desc_builder {
+ uint32_t *desc;
+ unsigned int idx; /* next word to write */
+ unsigned int max;
+ int err;
+};
+
+static inline void
+sec_desc_init(struct sec_desc_builder *b, uint32_t *desc, unsigned int max)
+{
+
+ b->desc = desc;
+ b->idx = 1; /* reserve word 0 for the HEADER */
+ b->max = max;
+ b->err = 0;
+}
+
+static inline void
+sec_desc_word(struct sec_desc_builder *b, uint32_t w)
+{
+
+ if (b->err != 0)
+ return;
+ if (b->idx >= b->max) {
+ b->err = ENOSPC;
+ return;
+ }
+ b->desc[b->idx++] = w;
+}
+
+/* Emit a KEY command with the key inline after it. */
+static inline void
+sec_desc_key_imm(struct sec_desc_builder *b, uint32_t class,
+ const void *key, unsigned int keylen)
+{
+ unsigned int nwords = howmany(keylen, sizeof(uint32_t));
+
+ if (b->err != 0)
*** 1954 LINES SKIPPED ***