git: 9006a6ad1b30 - main - dwc(4): Support a major update of Synopsys IP.
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Wed, 30 Sep 2026 11:27:19 UTC
The branch main has been updated by br:
URL: https://cgit.FreeBSD.org/src/commit/?id=9006a6ad1b3089f7cb3c2e3fb9632d5ccbbfbbbc
commit 9006a6ad1b3089f7cb3c2e3fb9632d5ccbbfbbbc
Author: Ruslan Bukin <br@FreeBSD.org>
AuthorDate: 2026-09-30 10:18:04 +0000
Commit: Ruslan Bukin <br@FreeBSD.org>
CommitDate: 2026-09-30 11:25:25 +0000
dwc(4): Support a major update of Synopsys IP.
Support new versions 4.x and 5.x of Synopsys DesignWare Gigabit
Ethernet MAC, often referred to as DWC Quality-of-Service IP Core.
In particular:
- version 5_30 (0x53) that is found on stm32mp2 arm64 SoC
- version 5_40 (0x54) that is found on Spacemit K3 RISC-V RVA23 SoC
DWC Ethernet QoS introduced a new scalable multi-channel architecture
with up to eight TX/RX channels, enhanced DCB support, TCP segmentation
offload (TSO).
Thanks to manu@ for start splitting out the DMA code of the older
(dwc version 3.x) driver to separate files in 2023. This patch is
a continuation of that work to support new core and DMA engine.
Thanks to mmel@ for initial review. Initial support covers the same
interface capabilities as the version 3 (VLAN_MTU, HWCSUM, HWCSUM_IPV6).
Reviewed by: mmel
No objection: manu
Sponsored by: Innovate UK
Differential Revision: https://reviews.freebsd.org/D59191
---
sys/conf/files | 2 +
sys/dev/dwc/dwc1000_dma.c | 15 +
sys/dev/dwc/dwc1000_dma.h | 12 -
sys/dev/dwc/dwc4_core.c | 432 ++++++++++++++++++++++
sys/dev/dwc/dwc4_dma.c | 920 ++++++++++++++++++++++++++++++++++++++++++++++
sys/dev/dwc/dwc4_dma.h | 111 ++++++
sys/dev/dwc/dwc4_mtl.h | 124 +++++++
sys/dev/dwc/dwc4_reg.h | 242 ++++++++++++
sys/dev/dwc/if_dwc.c | 132 +++++--
sys/dev/dwc/if_dwcvar.h | 45 ++-
10 files changed, 1997 insertions(+), 38 deletions(-)
diff --git a/sys/conf/files b/sys/conf/files
index 9dc3d34f3bd6..7450d65bf8b7 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1633,6 +1633,8 @@ dev/dwc/if_dwc.c optional dwc fdt
dev/dwc/if_dwc_if.m optional dwc fdt
dev/dwc/dwc1000_core.c optional dwc fdt
dev/dwc/dwc1000_dma.c optional dwc fdt
+dev/dwc/dwc4_core.c optional dwc fdt
+dev/dwc/dwc4_dma.c optional dwc fdt
dev/efidev/efidev.c optional efirt efidev
dev/efidev/efirt.c optional efirt
dev/efidev/efirtc.c optional efirt efirtc
diff --git a/sys/dev/dwc/dwc1000_dma.c b/sys/dev/dwc/dwc1000_dma.c
index 6457503d2a7f..3189056b9c61 100644
--- a/sys/dev/dwc/dwc1000_dma.c
+++ b/sys/dev/dwc/dwc1000_dma.c
@@ -151,6 +151,21 @@
#define ERDESC1_RBS1_MASK 0x7ffff
#define ERDESC1_RBS1_SHIFT 0 /* Receive Buffer 1 Size */
+/*
+ * A hardware buffer descriptor. Rx and Tx buffers have the same descriptor
+ * layout, but the bits in the fields have different meanings.
+ */
+struct dwc_hwdesc
+{
+ uint32_t desc0;
+ uint32_t desc1;
+ uint32_t addr1; /* ptr to first buffer data */
+ uint32_t addr2; /* ptr to next descriptor / second buffer data*/
+};
+
+#define RX_DESC_SIZE (sizeof(struct dwc_hwdesc) * RX_DESC_COUNT)
+#define TX_DESC_SIZE (sizeof(struct dwc_hwdesc) * TX_DESC_COUNT)
+
/*
* The hardware imposes alignment restrictions on various objects involved in
* DMA transfers. These values are expressed in bytes (not bits).
diff --git a/sys/dev/dwc/dwc1000_dma.h b/sys/dev/dwc/dwc1000_dma.h
index f0d22a67617f..6554c6d6a65b 100644
--- a/sys/dev/dwc/dwc1000_dma.h
+++ b/sys/dev/dwc/dwc1000_dma.h
@@ -30,18 +30,6 @@
#ifndef __DWC1000_DMA_H__
#define __DWC1000_DMA_H__
-/*
- * A hardware buffer descriptor. Rx and Tx buffers have the same descriptor
- * layout, but the bits in the fields have different meanings.
- */
-struct dwc_hwdesc
-{
- uint32_t desc0;
- uint32_t desc1;
- uint32_t addr1; /* ptr to first buffer data */
- uint32_t addr2; /* ptr to next descriptor / second buffer data*/
-};
-
int dma1000_init(struct dwc_softc *sc);
void dma1000_free(struct dwc_softc *sc);
void dma1000_start(struct dwc_softc *sc);
diff --git a/sys/dev/dwc/dwc4_core.c b/sys/dev/dwc/dwc4_core.c
new file mode 100644
index 000000000000..57cbb541edda
--- /dev/null
+++ b/sys/dev/dwc/dwc4_core.c
@@ -0,0 +1,432 @@
+/*-
+ * Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/rman.h>
+#include <sys/socket.h>
+
+#include <net/if.h>
+#include <net/ethernet.h>
+#include <net/if_dl.h>
+#include <net/if_media.h>
+
+#include <machine/bus.h>
+
+#include <dev/clk/clk.h>
+#include <dev/hwreset/hwreset.h>
+
+#include <dev/mii/mii.h>
+#include <dev/mii/miivar.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+#include <dev/mii/mii_fdt.h>
+
+#include <dev/dwc/if_dwcvar.h>
+#include <dev/dwc/dwc4_reg.h>
+#include <dev/dwc/dwc4_dma.h>
+
+#include "if_dwc_if.h"
+
+struct dwc_hash_maddr_ctx {
+ struct dwc_softc *sc;
+ uint32_t hash[8];
+};
+
+#define STATS_HARVEST_INTERVAL 2
+
+/* Pause time field in the transmitted control frame */
+static int dwc_pause_time = 0xffff;
+TUNABLE_INT("hw.dwc.pause_time", &dwc_pause_time);
+
+/*
+ * MIIBUS functions
+ */
+
+int
+dwc4_miibus_read_reg(device_t dev, int phy, int reg)
+{
+ struct dwc_softc *sc;
+ uint32_t mii;
+ size_t cnt;
+ int rv = 0;
+
+ sc = device_get_softc(dev);
+
+ mii = ((phy << MACMDIOAR_PA_S) & MACMDIOAR_PA_M)
+ | ((reg << MACMDIOAR_RDA_S) & MACMDIOAR_RDA_M)
+ | (sc->mii_clk << MACMDIOAR_CR_S)
+ | MACMDIOAR_READ
+ | MACMDIOAR_GB; /* Busy flag */
+
+ WRITE4(sc, ETH_MACMDIOAR, mii);
+
+ for (cnt = 0; cnt < 1000; cnt++) {
+ if (!(READ4(sc, ETH_MACMDIOAR) & MACMDIOAR_GB)) {
+ rv = READ4(sc, ETH_MACMDIODR);
+ break;
+ }
+ DELAY(10);
+ }
+
+ return rv;
+}
+
+int
+dwc4_miibus_write_reg(device_t dev, int phy, int reg, int val)
+{
+ struct dwc_softc *sc;
+ uint32_t mii;
+ size_t cnt;
+
+ sc = device_get_softc(dev);
+
+ mii = ((phy << MACMDIOAR_PA_S) & MACMDIOAR_PA_M)
+ | ((reg << MACMDIOAR_RDA_S) & MACMDIOAR_RDA_M)
+ | (sc->mii_clk << MACMDIOAR_CR_S)
+ | MACMDIOAR_WRITE
+ | MACMDIOAR_GB; /* Busy flag */
+
+ WRITE4(sc, ETH_MACMDIODR, val);
+ WRITE4(sc, ETH_MACMDIOAR, mii);
+
+ for (cnt = 0; cnt < 1000; cnt++) {
+ if (!(READ4(sc, ETH_MACMDIOAR) & MACMDIOAR_GB)) {
+ break;
+ }
+ DELAY(10);
+ }
+
+ return (0);
+}
+
+void
+dwc4_miibus_statchg(device_t dev)
+{
+ struct dwc_softc *sc;
+ struct mii_data *mii;
+ uint32_t reg;
+
+ /*
+ * Called by the MII bus driver when the PHY establishes
+ * link to set the MAC interface registers.
+ */
+
+ sc = device_get_softc(dev);
+
+ DWC_ASSERT_LOCKED(sc);
+
+ mii = sc->mii_softc;
+
+ if (mii->mii_media_status & IFM_ACTIVE)
+ sc->link_is_up = true;
+ else
+ sc->link_is_up = false;
+
+ reg = READ4(sc, ETH_MACCR);
+ switch (IFM_SUBTYPE(mii->mii_media_active)) {
+ case IFM_1000_T:
+ case IFM_1000_SX:
+ reg &= ~(MACCR_FES | MACCR_PS);
+ break;
+ case IFM_100_TX:
+ reg |= (MACCR_FES | MACCR_PS);
+ break;
+ case IFM_10_T:
+ reg &= ~(MACCR_FES);
+ reg |= (MACCR_PS);
+ break;
+ case IFM_NONE:
+ sc->link_is_up = false;
+ return;
+ default:
+ sc->link_is_up = false;
+ device_printf(dev, "Unsupported media %u\n",
+ IFM_SUBTYPE(mii->mii_media_active));
+ return;
+ }
+ if ((IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) != 0)
+ reg |= (MACCR_DM);
+ else
+ reg &= ~(MACCR_DM);
+ WRITE4(sc, ETH_MACCR, reg);
+
+ IF_DWC_SET_SPEED(dev, IFM_SUBTYPE(mii->mii_media_active));
+
+}
+
+void
+dwc4_core_setup(struct dwc_softc *sc)
+{
+ uint32_t reg;
+
+ DWC_ASSERT_LOCKED(sc);
+
+ reg = READ4(sc, ETH_MACVR);
+ device_printf(sc->dev, "MAC Version: %x\n", reg);
+
+ /* Enable core */
+ reg = READ4(sc, ETH_MACCR);
+ reg |= (MACCR_JD | MACCR_ACS | MACCR_BE);
+ WRITE4(sc, ETH_MACCR, reg);
+}
+
+void
+dwc4_enable_mac(struct dwc_softc *sc, bool enable)
+{
+ uint32_t reg;
+
+ DWC_ASSERT_LOCKED(sc);
+
+ reg = READ4(sc, ETH_MACCR);
+ if (enable)
+ reg |= MACCR_TE | MACCR_RE;
+ else
+ reg &= ~(MACCR_TE | MACCR_RE);
+ WRITE4(sc, ETH_MACCR, reg);
+}
+
+void
+dwc4_enable_csum_offload(struct dwc_softc *sc)
+{
+ uint32_t reg;
+
+ DWC_ASSERT_LOCKED(sc);
+ reg = READ4(sc, ETH_MACCR);
+ if ((if_getcapenable(sc->ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) !=
+ 0)
+ reg |= MACCR_IPC;
+ else
+ reg &= ~MACCR_IPC;
+ WRITE4(sc, ETH_MACCR, reg);
+}
+
+static const uint8_t nibbletab[] = {
+ /* 0x0 0000 -> 0000 */ 0x0,
+ /* 0x1 0001 -> 1000 */ 0x8,
+ /* 0x2 0010 -> 0100 */ 0x4,
+ /* 0x3 0011 -> 1100 */ 0xc,
+ /* 0x4 0100 -> 0010 */ 0x2,
+ /* 0x5 0101 -> 1010 */ 0xa,
+ /* 0x6 0110 -> 0110 */ 0x6,
+ /* 0x7 0111 -> 1110 */ 0xe,
+ /* 0x8 1000 -> 0001 */ 0x1,
+ /* 0x9 1001 -> 1001 */ 0x9,
+ /* 0xa 1010 -> 0101 */ 0x5,
+ /* 0xb 1011 -> 1101 */ 0xd,
+ /* 0xc 1100 -> 0011 */ 0x3,
+ /* 0xd 1101 -> 1011 */ 0xb,
+ /* 0xe 1110 -> 0111 */ 0x7,
+ /* 0xf 1111 -> 1111 */ 0xf, };
+
+static uint8_t
+bitreverse(uint8_t x)
+{
+
+ return (nibbletab[x & 0xf] << 4) | nibbletab[x >> 4];
+}
+
+static u_int
+dwc_hash_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt)
+{
+ struct dwc_hash_maddr_ctx *ctx = arg;
+ uint32_t crc, hashbit, hashreg;
+ uint8_t val;
+
+ crc = ether_crc32_le(LLADDR(sdl), ETHER_ADDR_LEN);
+ /* Take lower 8 bits and reverse it */
+ val = bitreverse(~crc & 0xff);
+ /*
+ * TODO: There is probably a HW_FEATURES bit which isn't
+ * related to the extended descriptors that describe this
+ */
+ if (!ctx->sc->dma_ext_desc)
+ val >>= 2; /* Only need lower 6 bits */
+ hashreg = (val >> 5);
+ hashbit = (val & 31);
+ ctx->hash[hashreg] |= (1 << hashbit);
+
+ return (1);
+}
+
+void
+dwc4_setup_rxfilter(struct dwc_softc *sc)
+{
+ struct dwc_hash_maddr_ctx ctx;
+ if_t ifp;
+ uint8_t *eaddr;
+ uint32_t ffval, hi, lo;
+ int nhash, i;
+
+ DWC_ASSERT_LOCKED(sc);
+
+ ifp = sc->ifp;
+ /*
+ * TODO: There is probably a HW_FEATURES bit which isn't
+ * related to the extended descriptors that describe this
+ */
+ nhash = sc->dma_ext_desc == false ? 2 : 8;
+
+ /*
+ * Set the multicast (group) filter hash.
+ */
+ if ((if_getflags(ifp) & IFF_ALLMULTI) != 0) {
+ ffval = (MACPFR_PM);
+ for (i = 0; i < nhash; i++)
+ ctx.hash[i] = ~0;
+ } else {
+ ffval = (MACPFR_HMC);
+ for (i = 0; i < nhash; i++)
+ ctx.hash[i] = 0;
+ ctx.sc = sc;
+ if_foreach_llmaddr(ifp, dwc_hash_maddr, &ctx);
+ }
+
+ /*
+ * Set the individual address filter hash.
+ */
+ if ((if_getflags(ifp) & IFF_PROMISC) != 0)
+ ffval |= MACPFR_PR;
+
+ /*
+ * Set the primary address.
+ */
+ eaddr = if_getlladdr(ifp);
+ lo = eaddr[0] | (eaddr[1] << 8) | (eaddr[2] << 16) |
+ (eaddr[3] << 24);
+ hi = eaddr[4] | (eaddr[5] << 8);
+ WRITE4(sc, ETH_MACA0LR, lo);
+ WRITE4(sc, ETH_MACA0HR, hi);
+
+ WRITE4(sc, ETH_MACPFR, ffval);
+ if (!sc->dma_ext_desc) {
+ WRITE4(sc, ETH_MACHT0R, ctx.hash[0]);
+ WRITE4(sc, ETH_MACHT1R, ctx.hash[1]);
+ } else
+ panic("implement me: ext_desc");
+}
+
+void
+dwc4_get_hwaddr(struct dwc_softc *sc, uint8_t *hwaddr)
+{
+ uint32_t hi, lo, rnd;
+
+ /*
+ * Try to recover a MAC address from the running hardware. If there's
+ * something non-zero there, assume the bootloader did the right thing
+ * and just use it.
+ *
+ * Otherwise, set the address to a convenient locally assigned address,
+ * 'bsd' + random 24 low-order bits. 'b' is 0x62, which has the locally
+ * assigned bit set, and the broadcast/multicast bit clear.
+ */
+ lo = READ4(sc, ETH_MACA0LR);
+ hi = READ4(sc, ETH_MACA0HR) & 0xffff;
+ if ((lo != 0xffffffff) || (hi != 0xffff)) {
+ hwaddr[0] = (lo >> 0) & 0xff;
+ hwaddr[1] = (lo >> 8) & 0xff;
+ hwaddr[2] = (lo >> 16) & 0xff;
+ hwaddr[3] = (lo >> 24) & 0xff;
+ hwaddr[4] = (hi >> 0) & 0xff;
+ hwaddr[5] = (hi >> 8) & 0xff;
+ } else {
+ rnd = arc4random() & 0x00ffffff;
+ hwaddr[0] = 'b';
+ hwaddr[1] = 's';
+ hwaddr[2] = 'd';
+ hwaddr[3] = rnd >> 16;
+ hwaddr[4] = rnd >> 8;
+ hwaddr[5] = rnd >> 0;
+ }
+}
+
+/*
+ * Stats
+ */
+
+static void
+dwc4_clear_stats(struct dwc_softc *sc)
+{
+ uint32_t reg;
+
+ reg = READ4(sc, ETH_MMC_CONTROL);
+ reg |= MMC_CONTROL_CNTRST;
+ WRITE4(sc, ETH_MMC_CONTROL, reg);
+}
+
+void
+dwc4_harvest_stats(struct dwc_softc *sc)
+{
+ if_t ifp __unused;
+ int val;
+
+ /* We don't need to harvest too often. */
+ if (++sc->stats_harvest_count < STATS_HARVEST_INTERVAL)
+ return;
+
+ sc->stats_harvest_count = 0;
+ ifp = sc->ifp;
+
+ val = READ4(sc, ETH_RX_RECEIVE_ERROR);
+ if_inc_counter(ifp, IFCOUNTER_IERRORS, val);
+
+ val = READ4(sc, ETH_TX_PACKET_COUNT_GOOD_BAD);
+ val -= READ4(sc, ETH_TX_PACKET_COUNT_GOOD);
+ if_inc_counter(ifp, IFCOUNTER_OERRORS, val);
+
+ val = READ4(sc, ETH_TX_SINGLE_COLLISION_GOOD_PACKETS);
+ val += READ4(sc, ETH_TX_MULTIPLE_COLLISION_GOOD_PACKETS);
+ val += READ4(sc, ETH_TX_LATE_COLLISION_PACKETS);
+ val += READ4(sc, ETH_TX_EXCESSIVE_COLLISION_PACKETS);
+ if_inc_counter(ifp, IFCOUNTER_COLLISIONS, val);
+
+ dwc4_clear_stats(sc);
+}
+
+void
+dwc4_intr(struct dwc_softc *sc)
+{
+ uint32_t reg;
+
+ DWC_ASSERT_LOCKED(sc);
+
+ reg = READ4(sc, ETH_MACISR);
+ if (reg)
+ READ4(sc, ETH_MACPHYCSR);
+}
+
+void
+dwc4_intr_disable(struct dwc_softc *sc)
+{
+
+ WRITE4(sc, ETH_MACIER, 0);
+}
diff --git a/sys/dev/dwc/dwc4_dma.c b/sys/dev/dwc/dwc4_dma.c
new file mode 100644
index 000000000000..cdf2b0c073c1
--- /dev/null
+++ b/sys/dev/dwc/dwc4_dma.c
@@ -0,0 +1,920 @@
+/*-
+ * Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/rman.h>
+#include <sys/socket.h>
+
+#include <net/bpf.h>
+#include <net/if.h>
+#include <net/ethernet.h>
+#include <net/if_dl.h>
+#include <net/if_media.h>
+#include <net/if_types.h>
+#include <net/if_var.h>
+
+#include <machine/bus.h>
+
+#include <dev/clk/clk.h>
+#include <dev/hwreset/hwreset.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <dev/dwc/if_dwcvar.h>
+#include <dev/dwc/dwc4_reg.h>
+#include <dev/dwc/dwc4_dma.h>
+#include <dev/dwc/dwc4_mtl.h>
+
+#define WATCHDOG_TIMEOUT_SECS 5
+#define DMA_RESET_TIMEOUT 100
+
+/* RDES1 Write-back format */
+#define RDES1_IPCE (1 << 7) /* IP Payload Error;
+ * 16-bit IP payload checksum mismatch.
+ */
+#define RDES1_IPCB (1 << 6) /* Checksum offload engine is bypassed. */
+#define RDES1_IPHE (1 << 3) /*
+ * IP Header Error;
+ * 16-bit IP header checksum mismatch.
+ */
+/* RX Read format */
+#define RDES3_OWN (1 << 31) /* DMA owns the descriptor */
+#define RDES3_IOC (1 << 30) /* Interrupt Enabled on Completion */
+#define RDES3_BUF2V (1 << 25) /* Buffer 2 Address Valid */
+#define RDES3_BUF1V (1 << 24) /* Buffer 1 Address Valid */
+
+/* RX Write-back format */
+#define RDES3_FD (1 << 29) /* First Descriptor */
+#define RDES3_LD (1 << 28) /* Last Descriptor */
+#define RDES3_RE (1 << 20) /* Receive Error */
+#define RDES3_PL_S 0 /* Packet Length */
+#define RDES3_PL_M (0x7fff << RDES3_PL_S)
+#define RDES3_CE (1 << 24) /* CRC Error */
+#define RDES3_LT_S 16 /* Length/Type Field */
+#define RDES3_LT_M (0x3 << RDES3_LT_S)
+
+/* TX Read format */
+#define TDES2_IOC (1 << 31) /* Interrupt on completion */
+#define TDES2_TTSE (1 << 30) /* Transmit Timestamp Enable */
+#define TDES2_B2L_S 16 /* Buffer 2 Length */
+#define TDES2_B2L_M (0x3fff << TDES2_B2L_S)
+#define TDES2_VTIR_S 14 /* VLAN Tag Insertion or Replacement */
+#define TDES2_VTIR_M (0x3 << TDES2_VTIR_S)
+
+#define TDES3_OWN (1 << 31) /* the DMA owns the descriptor */
+#define TDES3_CTXT (1 << 30) /* Context Type */
+#define TDES3_FD (1 << 29) /* First Descriptor */
+#define TDES3_LD (1 << 28) /* Last Descriptor */
+#define TDES3_CPC_S 26
+#define TDES3_CPC_M (0x3 << TDES3_CPC_S)
+#define TDES3_CPC_CRC_PAD (0x0 << TDES3_CPC_S)
+#define TDES3_CPC_CRC (0x1 << TDES3_CPC_S)
+#define TDES3_CPC_CRC_DISABLE (0x2 << TDES3_CPC_S)
+#define TDES3_CPC_CRC_REPLACE (0x3 << TDES3_CPC_S)
+#define TDES3_SAIC_S 23
+#define TDES3_SAIC_M (0x7 << TDES3_SAIC_S)
+#define TDES3_TSE 18 /* TCP Segmentation Enable */
+#define TDES3_CIC_S 16
+#define TDES3_CIC_M (0x3 << TDES3_CIC_S)
+#define TDES3_CIC_HDR (0x1 << TDES3_CIC_S) /* Only IP header csum inserted */
+#define TDES3_CIC_FULL (0x2 << TDES3_CIC_S) /* IP header & payload */
+#define TDES3_CIC_FULLP (0x3 << TDES3_CIC_S) /* IP header, payload, pseudo-hdr*/
+#define TDES3_TPL_S 0 /* TCP Payload Length */
+#define TDES3_TPL_M (0x3ffff << TDES3_TPL_S)
+
+/*
+ * A hardware buffer descriptor. Rx and Tx buffers have the same descriptor
+ * layout, but the bits in the fields have different meanings.
+ */
+struct dwc_hwdesc
+{
+ uint32_t tdes0; /* Header or Buffer 1 Address[31:0] */
+ uint32_t tdes1; /* Buffer 2 Address [31:0] or Buffer 1 Address[63:32] */
+ uint32_t tdes2;
+ uint32_t tdes3;
+};
+
+#define RX_DESC_SIZE (sizeof(struct dwc_hwdesc) * RX_DESC_COUNT)
+#define TX_DESC_SIZE (sizeof(struct dwc_hwdesc) * TX_DESC_COUNT)
+
+/*
+ * The hardware imposes alignment restrictions on various objects involved in
+ * DMA transfers. These values are expressed in bytes (not bits).
+ */
+#define DWC_DESC_RING_ALIGN 2048
+
+static inline uint32_t
+next_txidx(struct dwc_softc *sc, uint32_t curidx)
+{
+
+ return ((curidx + 1) % TX_DESC_COUNT);
+}
+
+static inline uint32_t
+next_rxidx(struct dwc_softc *sc, uint32_t curidx)
+{
+
+ return ((curidx + 1) % RX_DESC_COUNT);
+}
+
+static void
+dwc_get1paddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
+{
+
+ if (error != 0)
+ return;
+ *(bus_addr_t *)arg = segs[0].ds_addr;
+}
+
+inline static void
+txdesc_clear(struct dwc_softc *sc, int idx)
+{
+
+ sc->tx_desccount--;
+ sc->txdesc_ring[idx].tdes0 = 0;
+ sc->txdesc_ring[idx].tdes1 = 0;
+ sc->txdesc_ring[idx].tdes2 = 0;
+ sc->txdesc_ring[idx].tdes3 = 0;
+}
+
+inline static void
+txdesc_setup(struct dwc_softc *sc, int idx, bus_addr_t paddr,
+ uint32_t len, uint32_t flags, bool first, bool last)
+{
+ uint32_t tdes0, tdes1, tdes2, tdes3;
+
+ if (!sc->dma_ext_desc) {
+ tdes0 = paddr;
+ tdes1 = paddr >> 32;
+ tdes2 = len | TDES2_TTSE;
+ tdes3 = flags;
+
+ if (first)
+ tdes3 |= TDES3_FD;
+ if (last) {
+ tdes2 |= TDES2_IOC;
+ tdes3 |= TDES3_LD;
+ }
+ } else {
+ tdes0 = 0;
+ tdes1 = 0;
+ tdes2 = 0;
+ tdes3 = 0;
+ }
+ ++sc->tx_desccount;
+ sc->txdesc_ring[idx].tdes0 = tdes0;
+ sc->txdesc_ring[idx].tdes1 = tdes1;
+ sc->txdesc_ring[idx].tdes2 = tdes2;
+ sc->txdesc_ring[idx].tdes3 = tdes3;
+
+ wmb();
+ sc->txdesc_ring[idx].tdes3 |= TDES3_OWN;
+ wmb();
+}
+
+inline static uint32_t
+rxdesc_setup(struct dwc_softc *sc, int idx, bus_addr_t paddr)
+{
+
+ sc->rxdesc_ring[idx].tdes0 = (uint32_t)paddr;
+ sc->rxdesc_ring[idx].tdes1 = (uint32_t)(paddr >> 32);
+ sc->rxdesc_ring[idx].tdes2 = 0;
+ sc->rxdesc_ring[idx].tdes3 = RDES3_IOC | RDES3_BUF1V;
+
+ wmb();
+ sc->rxdesc_ring[idx].tdes3 |= RDES3_OWN;
+ wmb();
+
+ return (0);
+}
+
+int
+dma4_setup_txbuf(struct dwc_softc *sc, int idx, struct mbuf **mp)
+{
+ struct bus_dma_segment segs[TX_MAP_MAX_SEGS];
+ int error, nsegs;
+ struct mbuf * m;
+ uint32_t flags;
+ uint32_t csum_flags;
+ int i;
+ int last;
+
+ flags = 0;
+ error = bus_dmamap_load_mbuf_sg(sc->txbuf_tag, sc->txbuf_map[idx].map,
+ *mp, segs, &nsegs, 0);
+ if (error == EFBIG) {
+ /*
+ * The map may be partially mapped from the first call.
+ * Make sure to reset it.
+ */
+ bus_dmamap_unload(sc->txbuf_tag, sc->txbuf_map[idx].map);
+ if ((m = m_defrag(*mp, M_NOWAIT)) == NULL)
+ return (ENOMEM);
+ *mp = m;
+ error = bus_dmamap_load_mbuf_sg(sc->txbuf_tag,
+ sc->txbuf_map[idx].map, *mp, segs, &nsegs, 0);
+ }
+ if (error != 0)
+ return (ENOMEM);
+
+ if (sc->tx_desccount + nsegs > TX_DESC_COUNT) {
+ bus_dmamap_unload(sc->txbuf_tag, sc->txbuf_map[idx].map);
+ return (ENOMEM);
+ }
+
+ m = *mp;
+
+ csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP6_TCP | CSUM_IP6_UDP;
+ if ((m->m_pkthdr.csum_flags & csum_flags) != 0) {
+ if (!sc->dma_ext_desc)
+ flags = TDES3_CIC_FULLP;
+ else
+ panic("implement me");
+ } else if ((m->m_pkthdr.csum_flags & CSUM_IP) != 0) {
+ if (!sc->dma_ext_desc)
+ flags = TDES3_CIC_HDR;
+ else
+ panic("implement me");
+ }
+
+ bus_dmamap_sync(sc->txbuf_tag, sc->txbuf_map[idx].map,
+ BUS_DMASYNC_PREWRITE);
+
+ sc->txbuf_map[idx].mbuf = m;
+
+ for (i = 0; i < nsegs; i++) {
+ txdesc_setup(sc, sc->tx_desc_head,
+ segs[i].ds_addr, segs[i].ds_len,
+ (i == 0) ? flags : 0, /* only first desc needs flags */
+ (i == 0),
+ (i == nsegs - 1));
+ last = sc->tx_desc_head;
+ sc->tx_desc_head = next_txidx(sc, sc->tx_desc_head);
+ }
+
+ sc->txbuf_map[idx].last_desc_idx = last;
+
+ wmb();
+
+ WRITE4(sc, ETH_DMACTXDTPR(0), sc->txdesc_ring_paddr +
+ sc->tx_desc_head * sizeof(struct dwc_hwdesc));
+
+ return (0);
+}
+
+static int
+dma4_setup_rxbuf(struct dwc_softc *sc, int idx, struct mbuf *m)
+{
+ struct bus_dma_segment seg;
+ int error, nsegs;
+
+ m_adj(m, ETHER_ALIGN);
+
+ error = bus_dmamap_load_mbuf_sg(sc->rxbuf_tag, sc->rxbuf_map[idx].map,
+ m, &seg, &nsegs, 0);
+ if (error != 0)
+ return (error);
+
+ KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
+
+ bus_dmamap_sync(sc->rxbuf_tag, sc->rxbuf_map[idx].map,
+ BUS_DMASYNC_PREREAD);
+
+ sc->rxbuf_map[idx].mbuf = m;
+ rxdesc_setup(sc, idx, seg.ds_addr);
+
+ return (0);
+}
+
+static struct mbuf *
+dwc_alloc_mbufcl(struct dwc_softc *sc)
+{
+ struct mbuf *m;
+
+ m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
+ if (m != NULL)
+ m->m_pkthdr.len = m->m_len = m->m_ext.ext_size;
+
+ return (m);
+}
+
+static struct mbuf *
+dwc_rxfinish_one(struct dwc_softc *sc, struct dwc_hwdesc *desc,
+ struct dwc_bufmap *map)
+{
+ if_t ifp;
+ struct mbuf *m, *m0;
+ int len;
+ uint32_t rdes1;
+ uint32_t rdes3;
+
+ m = map->mbuf;
+ ifp = sc->ifp;
+ rdes1 = desc->tdes1;
+ rdes3 = desc->tdes3;
+
+ if ((rdes3 & (RDES3_FD | RDES3_LD)) != (RDES3_FD | RDES3_LD)) {
+ /*
+ * Something very wrong happens. The whole packet should be
+ * received in one descriptor. Report problem.
+ */
+ device_printf(sc->dev,
+ "%s: RX descriptor without FIRST and LAST bit set: 0x%08X",
+ __func__, rdes3);
+ return (NULL);
+ }
+
+ len = (rdes3 & RDES3_PL_M) >> RDES3_PL_S;
+
+ /* Allocate new buffer */
+ m0 = dwc_alloc_mbufcl(sc);
+ if (m0 == NULL) {
+ device_printf(sc->dev, "no mbufs\n");
+
+ /* no new mbuf available, recycle old */
+ if_inc_counter(sc->ifp, IFCOUNTER_IQDROPS, 1);
+ return (NULL);
+ }
+ /* Do dmasync for newly received packet */
+ bus_dmamap_sync(sc->rxbuf_tag, map->map, BUS_DMASYNC_POSTREAD);
+ bus_dmamap_unload(sc->rxbuf_tag, map->map);
+
+ /* Received packet is valid, process it */
+ m->m_pkthdr.rcvif = ifp;
+ m->m_pkthdr.len = len;
+ m->m_len = len;
+ if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
+
+ if ((if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) != 0 &&
+ (rdes1 & RDES1_IPCB) == 0) {
+ m->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
+ if ((rdes1 & RDES1_IPCE) == 0)
+ m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
+ if ((rdes1 & RDES1_IPHE) == 0) {
+ m->m_pkthdr.csum_flags |=
+ CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
+ m->m_pkthdr.csum_data = 0xffff;
+ }
+ }
+
+ /* Remove trailing FCS */
+ m_adj(m, -ETHER_CRC_LEN);
+
+ DWC_UNLOCK(sc);
+ if_input(ifp, m);
+ DWC_LOCK(sc);
+ return (m0);
+}
+
+void
+dma4_txfinish_locked(struct dwc_softc *sc)
+{
+ struct dwc_bufmap *bmap;
+ struct dwc_hwdesc *desc;
+ if_t ifp;
+ int idx, last_idx;
+ bool map_finished;
+
+ DWC_ASSERT_LOCKED(sc);
+
+ ifp = sc->ifp;
+ /* check if all descriptors of the map are done */
+ while (sc->tx_map_tail != sc->tx_map_head) {
+ map_finished = true;
+ bmap = &sc->txbuf_map[sc->tx_map_tail];
+ idx = sc->tx_desc_tail;
+ last_idx = next_txidx(sc, bmap->last_desc_idx);
+ while (idx != last_idx) {
+ desc = &sc->txdesc_ring[idx];
+ if ((desc->tdes3 & TDES3_OWN) != 0) {
+ map_finished = false;
+ break;
+ }
+ idx = next_txidx(sc, idx);
+ }
+
+ if (!map_finished)
+ break;
+ bus_dmamap_sync(sc->txbuf_tag, bmap->map,
+ BUS_DMASYNC_POSTWRITE);
+ bus_dmamap_unload(sc->txbuf_tag, bmap->map);
+ m_freem(bmap->mbuf);
+ bmap->mbuf = NULL;
+ sc->tx_mapcount--;
*** 1319 LINES SKIPPED ***