git: e6890ee9e183 - stable/12 - if_dwc: Reorder functions and sort them by usage

From: Mitchell Horne <mhorne_at_FreeBSD.org>
Date: Thu, 11 Aug 2022 14:53:16 UTC
The branch stable/12 has been updated by mhorne:

URL: https://cgit.FreeBSD.org/src/commit/?id=e6890ee9e18342618c2a6952e36201b09b275647

commit e6890ee9e18342618c2a6952e36201b09b275647
Author:     Emmanuel Vadot <manu@FreeBSD.org>
AuthorDate: 2020-11-20 11:30:01 +0000
Commit:     Mitchell Horne <mhorne@FreeBSD.org>
CommitDate: 2022-08-11 14:52:02 +0000

    if_dwc: Reorder functions and sort them by usage
    
    No functional changes intended
    
    (cherry picked from commit 2a35d39179c20ff0822446f20dee70ffd1122dc7)
---
 sys/dev/dwc/if_dwc.c | 1452 ++++++++++++++++++++++++++------------------------
 1 file changed, 743 insertions(+), 709 deletions(-)

diff --git a/sys/dev/dwc/if_dwc.c b/sys/dev/dwc/if_dwc.c
index 8a0adfeea6cd..355c3489c85f 100644
--- a/sys/dev/dwc/if_dwc.c
+++ b/sys/dev/dwc/if_dwc.c
@@ -195,6 +195,12 @@ struct dwc_hwdesc
 	uint32_t addr2;		/* ptr to next descriptor / second buffer data*/
 };
 
+
+struct dwc_hash_maddr_ctx {
+	struct dwc_softc *sc;
+	uint32_t hash[8];
+};
+
 /*
  * The hardware imposes alignment restrictions on various objects involved in
  * DMA transfers.  These values are expressed in bytes (not bits).
@@ -216,344 +222,524 @@ static void dwc_enable_mac(struct dwc_softc *sc, bool enable);
 static void dwc_init_dma(struct dwc_softc *sc);
 static void dwc_stop_dma(struct dwc_softc *sc);
 
-static inline uint32_t
-next_rxidx(struct dwc_softc *sc, uint32_t curidx)
-{
+static void dwc_tick(void *arg);
 
-	return ((curidx + 1) % RX_DESC_COUNT);
-}
+/*
+ * MIIBUS functions
+ */
 
-static inline uint32_t
-next_txidx(struct dwc_softc *sc, uint32_t curidx)
+static int
+dwc_miibus_read_reg(device_t dev, int phy, int reg)
 {
+	struct dwc_softc *sc;
+	uint16_t mii;
+	size_t cnt;
+	int rv = 0;
 
-	return ((curidx + 1) % TX_DESC_COUNT);
-}
-
-static void
-dwc_get1paddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
-{
+	sc = device_get_softc(dev);
 
-	if (error != 0)
-		return;
-	*(bus_addr_t *)arg = segs[0].ds_addr;
-}
+	mii = ((phy & GMII_ADDRESS_PA_MASK) << GMII_ADDRESS_PA_SHIFT)
+	    | ((reg & GMII_ADDRESS_GR_MASK) << GMII_ADDRESS_GR_SHIFT)
+	    | (sc->mii_clk << GMII_ADDRESS_CR_SHIFT)
+	    | GMII_ADDRESS_GB; /* Busy flag */
 
-inline static void
-dwc_setup_txdesc(struct dwc_softc *sc, int idx, bus_addr_t paddr,
-    uint32_t len)
-{
-	uint32_t desc0, desc1;
+	WRITE4(sc, GMII_ADDRESS, mii);
 
-	/* Addr/len 0 means we're clearing the descriptor after xmit done. */
-	if (paddr == 0 || len == 0) {
-		desc0 = 0;
-		desc1 = 0;
-		--sc->txcount;
-	} else {
-		if (sc->mactype != DWC_GMAC_EXT_DESC) {
-			desc0 = 0;
-			desc1 = NTDESC1_TCH | NTDESC1_FS | NTDESC1_LS |
-			    NTDESC1_IC | len;
-		} else {
-			desc0 = ETDESC0_TCH | ETDESC0_FS | ETDESC0_LS |
-			    ETDESC0_IC;
-			desc1 = len;
+	for (cnt = 0; cnt < 1000; cnt++) {
+		if (!(READ4(sc, GMII_ADDRESS) & GMII_ADDRESS_GB)) {
+			rv = READ4(sc, GMII_DATA);
+			break;
 		}
-		++sc->txcount;
+		DELAY(10);
 	}
 
-	sc->txdesc_ring[idx].addr1 = (uint32_t)(paddr);
-	sc->txdesc_ring[idx].desc0 = desc0;
-	sc->txdesc_ring[idx].desc1 = desc1;
-
-	if (paddr && len) {
-		wmb();
-		sc->txdesc_ring[idx].desc0 |= TDESC0_OWN;
-		wmb();
-	}
+	return rv;
 }
 
 static int
-dwc_setup_txbuf(struct dwc_softc *sc, int idx, struct mbuf **mp)
+dwc_miibus_write_reg(device_t dev, int phy, int reg, int val)
 {
-	struct bus_dma_segment seg;
-	int error, nsegs;
-	struct mbuf * m;
-
-	if ((m = m_defrag(*mp, M_NOWAIT)) == NULL)
-		return (ENOMEM);
-	*mp = m;
-
-	error = bus_dmamap_load_mbuf_sg(sc->txbuf_tag, sc->txbuf_map[idx].map,
-	    m, &seg, &nsegs, 0);
-	if (error != 0) {
-		return (ENOMEM);
-	}
+	struct dwc_softc *sc;
+	uint16_t mii;
+	size_t cnt;
 
-	KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
+	sc = device_get_softc(dev);
 
-	bus_dmamap_sync(sc->txbuf_tag, sc->txbuf_map[idx].map,
-	    BUS_DMASYNC_PREWRITE);
+	mii = ((phy & GMII_ADDRESS_PA_MASK) << GMII_ADDRESS_PA_SHIFT)
+	    | ((reg & GMII_ADDRESS_GR_MASK) << GMII_ADDRESS_GR_SHIFT)
+	    | (sc->mii_clk << GMII_ADDRESS_CR_SHIFT)
+	    | GMII_ADDRESS_GB | GMII_ADDRESS_GW;
 
-	sc->txbuf_map[idx].mbuf = m;
+	WRITE4(sc, GMII_DATA, val);
+	WRITE4(sc, GMII_ADDRESS, mii);
 
-	dwc_setup_txdesc(sc, idx, seg.ds_addr, seg.ds_len);
+	for (cnt = 0; cnt < 1000; cnt++) {
+		if (!(READ4(sc, GMII_ADDRESS) & GMII_ADDRESS_GB)) {
+			break;
+                }
+		DELAY(10);
+	}
 
 	return (0);
 }
 
 static void
-dwc_txstart_locked(struct dwc_softc *sc)
+dwc_miibus_statchg(device_t dev)
 {
-	struct ifnet *ifp;
-	struct mbuf *m;
-	int enqueued;
-
-	DWC_ASSERT_LOCKED(sc);
+	struct dwc_softc *sc;
+	struct mii_data *mii;
+	uint32_t reg;
 
-	if (!sc->link_is_up)
-		return;
+	/*
+	 * Called by the MII bus driver when the PHY establishes
+	 * link to set the MAC interface registers.
+	 */
 
-	ifp = sc->ifp;
+	sc = device_get_softc(dev);
 
-	if (ifp->if_drv_flags & IFF_DRV_OACTIVE)
-		return;
+	DWC_ASSERT_LOCKED(sc);
 
-	enqueued = 0;
+	mii = sc->mii_softc;
 
-	for (;;) {
-		if (sc->txcount == (TX_DESC_COUNT - 1)) {
-			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
-			break;
-		}
+	if (mii->mii_media_status & IFM_ACTIVE)
+		sc->link_is_up = true;
+	else
+		sc->link_is_up = false;
 
-		IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
-		if (m == NULL)
-			break;
-		if (dwc_setup_txbuf(sc, sc->tx_idx_head, &m) != 0) {
-			 IFQ_DRV_PREPEND(&ifp->if_snd, m);
-			break;
-		}
-		BPF_MTAP(ifp, m);
-		sc->tx_idx_head = next_txidx(sc, sc->tx_idx_head);
-		++enqueued;
+	reg = READ4(sc, MAC_CONFIGURATION);
+	switch (IFM_SUBTYPE(mii->mii_media_active)) {
+	case IFM_1000_T:
+	case IFM_1000_SX:
+		reg &= ~(CONF_FES | CONF_PS);
+		break;
+	case IFM_100_TX:
+		reg |= (CONF_FES | CONF_PS);
+		break;
+	case IFM_10_T:
+		reg &= ~(CONF_FES);
+		reg |= (CONF_PS);
+		break;
+	case IFM_NONE:
+		sc->link_is_up = false;
+		return;
+	default:
+		sc->link_is_up = false;
+		device_printf(dev, "Unsupported media %u\n",
+		    IFM_SUBTYPE(mii->mii_media_active));
+		return;
 	}
+	if ((IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) != 0)
+		reg |= (CONF_DM);
+	else
+		reg &= ~(CONF_DM);
+	WRITE4(sc, MAC_CONFIGURATION, reg);
+
+	IF_DWC_SET_SPEED(dev, IFM_SUBTYPE(mii->mii_media_active));
 
-	if (enqueued != 0) {
-		WRITE4(sc, TRANSMIT_POLL_DEMAND, 0x1);
-		sc->tx_watchdog_count = WATCHDOG_TIMEOUT_SECS;
-	}
 }
 
+/*
+ * Media functions
+ */
+
 static void
-dwc_txstart(struct ifnet *ifp)
+dwc_media_status(struct ifnet * ifp, struct ifmediareq *ifmr)
 {
-	struct dwc_softc *sc = ifp->if_softc;
+	struct dwc_softc *sc;
+	struct mii_data *mii;
 
+	sc = ifp->if_softc;
+	mii = sc->mii_softc;
 	DWC_LOCK(sc);
-	dwc_txstart_locked(sc);
+	mii_pollstat(mii);
+	ifmr->ifm_active = mii->mii_media_active;
+	ifmr->ifm_status = mii->mii_media_status;
 	DWC_UNLOCK(sc);
 }
 
-static void
-dwc_stop_locked(struct dwc_softc *sc)
+static int
+dwc_media_change_locked(struct dwc_softc *sc)
 {
-	struct ifnet *ifp;
 
-	DWC_ASSERT_LOCKED(sc);
+	return (mii_mediachg(sc->mii_softc));
+}
 
-	ifp = sc->ifp;
-	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
-	sc->tx_watchdog_count = 0;
-	sc->stats_harvest_count = 0;
+static int
+dwc_media_change(struct ifnet * ifp)
+{
+	struct dwc_softc *sc;
+	int error;
 
-	callout_stop(&sc->dwc_callout);
+	sc = ifp->if_softc;
 
-	dwc_stop_dma(sc);
-	dwc_enable_mac(sc, false);
+	DWC_LOCK(sc);
+	error = dwc_media_change_locked(sc);
+	DWC_UNLOCK(sc);
+	return (error);
 }
 
-static void dwc_clear_stats(struct dwc_softc *sc)
+/*
+ * Core functions
+ */
+
+static const uint8_t nibbletab[] = {
+	/* 0x0 0000 -> 0000 */  0x0,
+	/* 0x1 0001 -> 1000 */  0x8,
+	/* 0x2 0010 -> 0100 */  0x4,
+	/* 0x3 0011 -> 1100 */  0xc,
+	/* 0x4 0100 -> 0010 */  0x2,
+	/* 0x5 0101 -> 1010 */  0xa,
+	/* 0x6 0110 -> 0110 */  0x6,
+	/* 0x7 0111 -> 1110 */  0xe,
+	/* 0x8 1000 -> 0001 */  0x1,
+	/* 0x9 1001 -> 1001 */  0x9,
+	/* 0xa 1010 -> 0101 */  0x5,
+	/* 0xb 1011 -> 1101 */  0xd,
+	/* 0xc 1100 -> 0011 */  0x3,
+	/* 0xd 1101 -> 1011 */  0xb,
+	/* 0xe 1110 -> 0111 */  0x7,
+	/* 0xf 1111 -> 1111 */  0xf, };
+
+static uint8_t
+bitreverse(uint8_t x)
 {
-	uint32_t reg;
 
-	reg = READ4(sc, MMC_CONTROL);
-	reg |= (MMC_CONTROL_CNTRST);
-	WRITE4(sc, MMC_CONTROL, reg);
+	return (nibbletab[x & 0xf] << 4) | nibbletab[x >> 4];
 }
 
-static void
-dwc_harvest_stats(struct dwc_softc *sc)
+static u_int
+dwc_hash_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt)
 {
-	struct ifnet *ifp;
-
-	/* We don't need to harvest too often. */
-	if (++sc->stats_harvest_count < STATS_HARVEST_INTERVAL)
-		return;
-
-	sc->stats_harvest_count = 0;
-	ifp = sc->ifp;
-
-	if_inc_counter(ifp, IFCOUNTER_IPACKETS, READ4(sc, RXFRAMECOUNT_GB));
-	if_inc_counter(ifp, IFCOUNTER_IMCASTS, READ4(sc, RXMULTICASTFRAMES_G));
-	if_inc_counter(ifp, IFCOUNTER_IERRORS,
-	    READ4(sc, RXOVERSIZE_G) + READ4(sc, RXUNDERSIZE_G) +
-	    READ4(sc, RXCRCERROR) + READ4(sc, RXALIGNMENTERROR) +
-	    READ4(sc, RXRUNTERROR) + READ4(sc, RXJABBERERROR) +
-	    READ4(sc, RXLENGTHERROR));
-
-	if_inc_counter(ifp, IFCOUNTER_OPACKETS, READ4(sc, TXFRAMECOUNT_G));
-	if_inc_counter(ifp, IFCOUNTER_OMCASTS, READ4(sc, TXMULTICASTFRAMES_G));
-	if_inc_counter(ifp, IFCOUNTER_OERRORS,
-	    READ4(sc, TXOVERSIZE_G) + READ4(sc, TXEXCESSDEF) +
-	    READ4(sc, TXCARRIERERR) + READ4(sc, TXUNDERFLOWERROR));
+	struct dwc_hash_maddr_ctx *ctx = arg;
+	uint32_t crc, hashbit, hashreg;
+	uint8_t val;
 
-	if_inc_counter(ifp, IFCOUNTER_COLLISIONS,
-	    READ4(sc, TXEXESSCOL) + READ4(sc, TXLATECOL));
+	crc = ether_crc32_le(LLADDR(sdl), ETHER_ADDR_LEN);
+	/* Take lower 8 bits and reverse it */
+	val = bitreverse(~crc & 0xff);
+	if (ctx->sc->mactype != DWC_GMAC_EXT_DESC)
+		val >>= 2; /* Only need lower 6 bits */
+	hashreg = (val >> 5);
+	hashbit = (val & 31);
+	ctx->hash[hashreg] |= (1 << hashbit);
 
-	dwc_clear_stats(sc);
+	return (1);
 }
 
 static void
-dwc_tick(void *arg)
+dwc_setup_rxfilter(struct dwc_softc *sc)
 {
-	struct dwc_softc *sc;
+	struct dwc_hash_maddr_ctx ctx;
 	struct ifnet *ifp;
-	int link_was_up;
-
-	sc = arg;
+	uint8_t *eaddr;
+	uint32_t ffval, hi, lo;
+	int nhash, i;
 
 	DWC_ASSERT_LOCKED(sc);
 
 	ifp = sc->ifp;
-
-	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
-	    return;
+	nhash = sc->mactype != DWC_GMAC_EXT_DESC ? 2 : 8;
 
 	/*
-	 * Typical tx watchdog.  If this fires it indicates that we enqueued
-	 * packets for output and never got a txdone interrupt for them.  Maybe
-	 * it's a missed interrupt somehow, just pretend we got one.
+	 * Set the multicast (group) filter hash.
 	 */
-	if (sc->tx_watchdog_count > 0) {
-		if (--sc->tx_watchdog_count == 0) {
-			dwc_txfinish_locked(sc);
-		}
+	if ((ifp->if_flags & IFF_ALLMULTI) != 0) {
+		ffval = (FRAME_FILTER_PM);
+		for (i = 0; i < nhash; i++)
+			ctx.hash[i] = ~0;
+	} else {
+		ffval = (FRAME_FILTER_HMC);
+		for (i = 0; i < nhash; i++)
+			ctx.hash[i] = 0;
+		ctx.sc = sc;
+		if_foreach_llmaddr(ifp, dwc_hash_maddr, &ctx);
 	}
 
-	/* Gather stats from hardware counters. */
-	dwc_harvest_stats(sc);
-
-	/* Check the media status. */
-	link_was_up = sc->link_is_up;
-	mii_tick(sc->mii_softc);
-	if (sc->link_is_up && !link_was_up)
-		dwc_txstart_locked(sc);
+	/*
+	 * Set the individual address filter hash.
+	 */
+	if (ifp->if_flags & IFF_PROMISC)
+		ffval |= (FRAME_FILTER_PR);
 
-	/* Schedule another check one second from now. */
-	callout_reset(&sc->dwc_callout, hz, dwc_tick, sc);
+	/*
+	 * Set the primary address.
+	 */
+	eaddr = IF_LLADDR(ifp);
+	lo = eaddr[0] | (eaddr[1] << 8) | (eaddr[2] << 16) |
+	    (eaddr[3] << 24);
+	hi = eaddr[4] | (eaddr[5] << 8);
+	WRITE4(sc, MAC_ADDRESS_LOW(0), lo);
+	WRITE4(sc, MAC_ADDRESS_HIGH(0), hi);
+	WRITE4(sc, MAC_FRAME_FILTER, ffval);
+	if (sc->mactype != DWC_GMAC_EXT_DESC) {
+		WRITE4(sc, GMAC_MAC_HTLOW, ctx.hash[0]);
+		WRITE4(sc, GMAC_MAC_HTHIGH, ctx.hash[1]);
+	} else {
+		for (i = 0; i < nhash; i++)
+			WRITE4(sc, HASH_TABLE_REG(i), ctx.hash[i]);
+	}
 }
 
 static void
-dwc_init_locked(struct dwc_softc *sc)
+dwc_setup_core(struct dwc_softc *sc)
 {
-	struct ifnet *ifp = sc->ifp;
+	uint32_t reg;
 
 	DWC_ASSERT_LOCKED(sc);
 
-	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
-		return;
-
-	dwc_setup_rxfilter(sc);
-	dwc_setup_core(sc);
-	dwc_enable_mac(sc, true);
-	dwc_init_dma(sc);
+	/* Enable core */
+	reg = READ4(sc, MAC_CONFIGURATION);
+	reg |= (CONF_JD | CONF_ACS | CONF_BE);
+	WRITE4(sc, MAC_CONFIGURATION, reg);
+}
 
-	if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
+static void
+dwc_enable_mac(struct dwc_softc *sc, bool enable)
+{
+	uint32_t reg;
 
-	/*
-	 * Call mii_mediachg() which will call back into dwc_miibus_statchg()
-	 * to set up the remaining config registers based on current media.
-	 */
-	mii_mediachg(sc->mii_softc);
-	callout_reset(&sc->dwc_callout, hz, dwc_tick, sc);
+	DWC_ASSERT_LOCKED(sc);
+	reg = READ4(sc, MAC_CONFIGURATION);
+	if (enable)
+		reg |= CONF_TE | CONF_RE;
+	else
+		reg &= ~(CONF_TE | CONF_RE);
+	WRITE4(sc, MAC_CONFIGURATION, reg);
 }
 
 static void
-dwc_init(void *if_softc)
+dwc_get_hwaddr(struct dwc_softc *sc, uint8_t *hwaddr)
 {
-	struct dwc_softc *sc = if_softc;
+	uint32_t hi, lo, rnd;
 
-	DWC_LOCK(sc);
-	dwc_init_locked(sc);
-	DWC_UNLOCK(sc);
+	/*
+	 * Try to recover a MAC address from the running hardware. If there's
+	 * something non-zero there, assume the bootloader did the right thing
+	 * and just use it.
+	 *
+	 * Otherwise, set the address to a convenient locally assigned address,
+	 * 'bsd' + random 24 low-order bits.  'b' is 0x62, which has the locally
+	 * assigned bit set, and the broadcast/multicast bit clear.
+	 */
+	lo = READ4(sc, MAC_ADDRESS_LOW(0));
+	hi = READ4(sc, MAC_ADDRESS_HIGH(0)) & 0xffff;
+	if ((lo != 0xffffffff) || (hi != 0xffff)) {
+		hwaddr[0] = (lo >>  0) & 0xff;
+		hwaddr[1] = (lo >>  8) & 0xff;
+		hwaddr[2] = (lo >> 16) & 0xff;
+		hwaddr[3] = (lo >> 24) & 0xff;
+		hwaddr[4] = (hi >>  0) & 0xff;
+		hwaddr[5] = (hi >>  8) & 0xff;
+	} else {
+		rnd = arc4random() & 0x00ffffff;
+		hwaddr[0] = 'b';
+		hwaddr[1] = 's';
+		hwaddr[2] = 'd';
+		hwaddr[3] = rnd >> 16;
+		hwaddr[4] = rnd >>  8;
+		hwaddr[5] = rnd >>  0;
+	}
 }
 
+/*
+ * DMA functions
+ */
 
-inline static uint32_t
-dwc_setup_rxdesc(struct dwc_softc *sc, int idx, bus_addr_t paddr)
+static void
+dwc_init_dma(struct dwc_softc *sc)
 {
-	uint32_t nidx;
+	uint32_t reg;
 
-	sc->rxdesc_ring[idx].addr1 = (uint32_t)paddr;
-	nidx = next_rxidx(sc, idx);
-	sc->rxdesc_ring[idx].addr2 = sc->rxdesc_ring_paddr +
-	    (nidx * sizeof(struct dwc_hwdesc));
-	if (sc->mactype != DWC_GMAC_EXT_DESC)
-		sc->rxdesc_ring[idx].desc1 = NRDESC1_RCH |
-		    MIN(MCLBYTES, NRDESC1_RBS1_MASK);
-	else
-		sc->rxdesc_ring[idx].desc1 = ERDESC1_RCH |
-		    MIN(MCLBYTES, ERDESC1_RBS1_MASK);
+	DWC_ASSERT_LOCKED(sc);
 
-	wmb();
-	sc->rxdesc_ring[idx].desc0 = RDESC0_OWN;
-	wmb();
-	return (nidx);
+	/* Initializa DMA and enable transmitters */
+	reg = READ4(sc, OPERATION_MODE);
+	reg |= (MODE_TSF | MODE_OSF | MODE_FUF);
+	reg &= ~(MODE_RSF);
+	reg |= (MODE_RTC_LEV32 << MODE_RTC_SHIFT);
+	WRITE4(sc, OPERATION_MODE, reg);
+
+	WRITE4(sc, INTERRUPT_ENABLE, INT_EN_DEFAULT);
+
+	/* Start DMA */
+	reg = READ4(sc, OPERATION_MODE);
+	reg |= (MODE_ST | MODE_SR);
+	WRITE4(sc, OPERATION_MODE, reg);
 }
 
-static int
-dwc_setup_rxbuf(struct dwc_softc *sc, int idx, struct mbuf *m)
+static void
+dwc_stop_dma(struct dwc_softc *sc)
 {
-	struct bus_dma_segment seg;
-	int error, nsegs;
+	uint32_t reg;
 
-	m_adj(m, ETHER_ALIGN);
+	DWC_ASSERT_LOCKED(sc);
 
-	error = bus_dmamap_load_mbuf_sg(sc->rxbuf_tag, sc->rxbuf_map[idx].map,
-	    m, &seg, &nsegs, 0);
-	if (error != 0)
-		return (error);
+	/* Stop DMA TX */
+	reg = READ4(sc, OPERATION_MODE);
+	reg &= ~(MODE_ST);
+	WRITE4(sc, OPERATION_MODE, reg);
 
-	KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
+	/* Flush TX */
+	reg = READ4(sc, OPERATION_MODE);
+	reg |= (MODE_FTF);
+	WRITE4(sc, OPERATION_MODE, reg);
 
-	bus_dmamap_sync(sc->rxbuf_tag, sc->rxbuf_map[idx].map,
-	    BUS_DMASYNC_PREREAD);
+	/* Stop DMA RX */
+	reg = READ4(sc, OPERATION_MODE);
+	reg &= ~(MODE_SR);
+	WRITE4(sc, OPERATION_MODE, reg);
+}
 
-	sc->rxbuf_map[idx].mbuf = m;
-	dwc_setup_rxdesc(sc, idx, seg.ds_addr);
+static inline uint32_t
+next_rxidx(struct dwc_softc *sc, uint32_t curidx)
+{
 
-	return (0);
+	return ((curidx + 1) % RX_DESC_COUNT);
 }
 
-static struct mbuf *
-dwc_alloc_mbufcl(struct dwc_softc *sc)
+static inline uint32_t
+next_txidx(struct dwc_softc *sc, uint32_t curidx)
 {
-	struct mbuf *m;
-
-	m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
-	if (m != NULL)
-		m->m_pkthdr.len = m->m_len = m->m_ext.ext_size;
 
-	return (m);
+	return ((curidx + 1) % TX_DESC_COUNT);
 }
 
-static struct mbuf *
-dwc_rxfinish_one(struct dwc_softc *sc, struct dwc_hwdesc *desc,
-    struct dwc_bufmap *map)
+static void
+dwc_get1paddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 {
-	struct ifnet *ifp;
+
+	if (error != 0)
+		return;
+	*(bus_addr_t *)arg = segs[0].ds_addr;
+}
+
+inline static void
+dwc_setup_txdesc(struct dwc_softc *sc, int idx, bus_addr_t paddr,
+    uint32_t len)
+{
+	uint32_t desc0, desc1;
+
+	/* Addr/len 0 means we're clearing the descriptor after xmit done. */
+	if (paddr == 0 || len == 0) {
+		desc0 = 0;
+		desc1 = 0;
+		--sc->txcount;
+	} else {
+		if (sc->mactype != DWC_GMAC_EXT_DESC) {
+			desc0 = 0;
+			desc1 = NTDESC1_TCH | NTDESC1_FS | NTDESC1_LS |
+			    NTDESC1_IC | len;
+		} else {
+			desc0 = ETDESC0_TCH | ETDESC0_FS | ETDESC0_LS |
+			    ETDESC0_IC;
+			desc1 = len;
+		}
+		++sc->txcount;
+	}
+
+	sc->txdesc_ring[idx].addr1 = (uint32_t)(paddr);
+	sc->txdesc_ring[idx].desc0 = desc0;
+	sc->txdesc_ring[idx].desc1 = desc1;
+
+	if (paddr && len) {
+		wmb();
+		sc->txdesc_ring[idx].desc0 |= TDESC0_OWN;
+		wmb();
+	}
+}
+
+static int
+dwc_setup_txbuf(struct dwc_softc *sc, int idx, struct mbuf **mp)
+{
+	struct bus_dma_segment seg;
+	int error, nsegs;
+	struct mbuf * m;
+
+	if ((m = m_defrag(*mp, M_NOWAIT)) == NULL)
+		return (ENOMEM);
+	*mp = m;
+
+	error = bus_dmamap_load_mbuf_sg(sc->txbuf_tag, sc->txbuf_map[idx].map,
+	    m, &seg, &nsegs, 0);
+	if (error != 0) {
+		return (ENOMEM);
+	}
+
+	KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
+
+	bus_dmamap_sync(sc->txbuf_tag, sc->txbuf_map[idx].map,
+	    BUS_DMASYNC_PREWRITE);
+
+	sc->txbuf_map[idx].mbuf = m;
+
+	dwc_setup_txdesc(sc, idx, seg.ds_addr, seg.ds_len);
+
+	return (0);
+}
+
+inline static uint32_t
+dwc_setup_rxdesc(struct dwc_softc *sc, int idx, bus_addr_t paddr)
+{
+	uint32_t nidx;
+
+	sc->rxdesc_ring[idx].addr1 = (uint32_t)paddr;
+	nidx = next_rxidx(sc, idx);
+	sc->rxdesc_ring[idx].addr2 = sc->rxdesc_ring_paddr +
+	    (nidx * sizeof(struct dwc_hwdesc));
+	if (sc->mactype != DWC_GMAC_EXT_DESC)
+		sc->rxdesc_ring[idx].desc1 = NRDESC1_RCH |
+		    MIN(MCLBYTES, NRDESC1_RBS1_MASK);
+	else
+		sc->rxdesc_ring[idx].desc1 = ERDESC1_RCH |
+		    MIN(MCLBYTES, ERDESC1_RBS1_MASK);
+
+	wmb();
+	sc->rxdesc_ring[idx].desc0 = RDESC0_OWN;
+	wmb();
+	return (nidx);
+}
+
+static int
+dwc_setup_rxbuf(struct dwc_softc *sc, int idx, struct mbuf *m)
+{
+	struct bus_dma_segment seg;
+	int error, nsegs;
+
+	m_adj(m, ETHER_ALIGN);
+
+	error = bus_dmamap_load_mbuf_sg(sc->rxbuf_tag, sc->rxbuf_map[idx].map,
+	    m, &seg, &nsegs, 0);
+	if (error != 0)
+		return (error);
+
+	KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
+
+	bus_dmamap_sync(sc->rxbuf_tag, sc->rxbuf_map[idx].map,
+	    BUS_DMASYNC_PREREAD);
+
+	sc->rxbuf_map[idx].mbuf = m;
+	dwc_setup_rxdesc(sc, idx, seg.ds_addr);
+
+	return (0);
+}
+
+static struct mbuf *
+dwc_alloc_mbufcl(struct dwc_softc *sc)
+{
+	struct mbuf *m;
+
+	m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
+	if (m != NULL)
+		m->m_pkthdr.len = m->m_len = m->m_ext.ext_size;
+
+	return (m);
+}
+
+static struct mbuf *
+dwc_rxfinish_one(struct dwc_softc *sc, struct dwc_hwdesc *desc,
+    struct dwc_bufmap *map)
+{
+	struct ifnet *ifp;
 	struct mbuf *m, *m0;
 	int len;
 	uint32_t rdesc0;
@@ -617,215 +803,276 @@ dwc_rxfinish_one(struct dwc_softc *sc, struct dwc_hwdesc *desc,
 	return (m0);
 }
 
-static void
-dwc_media_status(struct ifnet * ifp, struct ifmediareq *ifmr)
+static int
+setup_dma(struct dwc_softc *sc)
 {
-	struct dwc_softc *sc;
-	struct mii_data *mii;
+	struct mbuf *m;
+	int error;
+	int nidx;
+	int idx;
 
-	sc = ifp->if_softc;
-	mii = sc->mii_softc;
-	DWC_LOCK(sc);
-	mii_pollstat(mii);
-	ifmr->ifm_active = mii->mii_media_active;
-	ifmr->ifm_status = mii->mii_media_status;
-	DWC_UNLOCK(sc);
-}
+	/*
+	 * Set up TX descriptor ring, descriptors, and dma maps.
+	 */
+	error = bus_dma_tag_create(
+	    bus_get_dma_tag(sc->dev),	/* Parent tag. */
+	    DWC_DESC_RING_ALIGN, 0,	/* alignment, boundary */
+	    BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
+	    BUS_SPACE_MAXADDR,		/* highaddr */
+	    NULL, NULL,			/* filter, filterarg */
+	    TX_DESC_SIZE, 1, 		/* maxsize, nsegments */
+	    TX_DESC_SIZE,		/* maxsegsize */
+	    0,				/* flags */
+	    NULL, NULL,			/* lockfunc, lockarg */
+	    &sc->txdesc_tag);
+	if (error != 0) {
+		device_printf(sc->dev,
+		    "could not create TX ring DMA tag.\n");
+		goto out;
+	}
 
-static int
-dwc_media_change_locked(struct dwc_softc *sc)
-{
+	error = bus_dmamem_alloc(sc->txdesc_tag, (void**)&sc->txdesc_ring,
+	    BUS_DMA_COHERENT | BUS_DMA_WAITOK | BUS_DMA_ZERO,
+	    &sc->txdesc_map);
+	if (error != 0) {
+		device_printf(sc->dev,
+		    "could not allocate TX descriptor ring.\n");
+		goto out;
+	}
 
-	return (mii_mediachg(sc->mii_softc));
-}
+	error = bus_dmamap_load(sc->txdesc_tag, sc->txdesc_map,
+	    sc->txdesc_ring, TX_DESC_SIZE, dwc_get1paddr,
+	    &sc->txdesc_ring_paddr, 0);
+	if (error != 0) {
+		device_printf(sc->dev,
+		    "could not load TX descriptor ring map.\n");
+		goto out;
+	}
 
-static int
-dwc_media_change(struct ifnet * ifp)
-{
-	struct dwc_softc *sc;
-	int error;
+	for (idx = 0; idx < TX_DESC_COUNT; idx++) {
+		nidx = next_txidx(sc, idx);
+		sc->txdesc_ring[idx].addr2 = sc->txdesc_ring_paddr +
+		    (nidx * sizeof(struct dwc_hwdesc));
+	}
 
-	sc = ifp->if_softc;
+	error = bus_dma_tag_create(
+	    bus_get_dma_tag(sc->dev),	/* Parent tag. */
+	    1, 0,			/* alignment, boundary */
+	    BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
+	    BUS_SPACE_MAXADDR,		/* highaddr */
+	    NULL, NULL,			/* filter, filterarg */
+	    MCLBYTES, 1, 		/* maxsize, nsegments */
+	    MCLBYTES,			/* maxsegsize */
+	    0,				/* flags */
+	    NULL, NULL,			/* lockfunc, lockarg */
+	    &sc->txbuf_tag);
+	if (error != 0) {
+		device_printf(sc->dev,
+		    "could not create TX ring DMA tag.\n");
+		goto out;
+	}
 
-	DWC_LOCK(sc);
-	error = dwc_media_change_locked(sc);
-	DWC_UNLOCK(sc);
-	return (error);
-}
+	for (idx = 0; idx < TX_DESC_COUNT; idx++) {
+		error = bus_dmamap_create(sc->txbuf_tag, BUS_DMA_COHERENT,
+		    &sc->txbuf_map[idx].map);
+		if (error != 0) {
+			device_printf(sc->dev,
+			    "could not create TX buffer DMA map.\n");
+			goto out;
+		}
+		dwc_setup_txdesc(sc, idx, 0, 0);
+	}
 
-static const uint8_t nibbletab[] = {
-	/* 0x0 0000 -> 0000 */  0x0,
-	/* 0x1 0001 -> 1000 */  0x8,
-	/* 0x2 0010 -> 0100 */  0x4,
-	/* 0x3 0011 -> 1100 */  0xc,
-	/* 0x4 0100 -> 0010 */  0x2,
-	/* 0x5 0101 -> 1010 */  0xa,
-	/* 0x6 0110 -> 0110 */  0x6,
-	/* 0x7 0111 -> 1110 */  0xe,
-	/* 0x8 1000 -> 0001 */  0x1,
-	/* 0x9 1001 -> 1001 */  0x9,
-	/* 0xa 1010 -> 0101 */  0x5,
-	/* 0xb 1011 -> 1101 */  0xd,
-	/* 0xc 1100 -> 0011 */  0x3,
-	/* 0xd 1101 -> 1011 */  0xb,
-	/* 0xe 1110 -> 0111 */  0x7,
-	/* 0xf 1111 -> 1111 */  0xf, };
+	/*
+	 * Set up RX descriptor ring, descriptors, dma maps, and mbufs.
+	 */
+	error = bus_dma_tag_create(
+	    bus_get_dma_tag(sc->dev),	/* Parent tag. */
+	    DWC_DESC_RING_ALIGN, 0,	/* alignment, boundary */
+	    BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
+	    BUS_SPACE_MAXADDR,		/* highaddr */
+	    NULL, NULL,			/* filter, filterarg */
+	    RX_DESC_SIZE, 1, 		/* maxsize, nsegments */
+	    RX_DESC_SIZE,		/* maxsegsize */
+	    0,				/* flags */
+	    NULL, NULL,			/* lockfunc, lockarg */
+	    &sc->rxdesc_tag);
+	if (error != 0) {
+		device_printf(sc->dev,
+		    "could not create RX ring DMA tag.\n");
+		goto out;
+	}
 
-static uint8_t
-bitreverse(uint8_t x)
-{
+	error = bus_dmamem_alloc(sc->rxdesc_tag, (void **)&sc->rxdesc_ring,
+	    BUS_DMA_COHERENT | BUS_DMA_WAITOK | BUS_DMA_ZERO,
+	    &sc->rxdesc_map);
+	if (error != 0) {
+		device_printf(sc->dev,
+		    "could not allocate RX descriptor ring.\n");
+		goto out;
+	}
 
-	return (nibbletab[x & 0xf] << 4) | nibbletab[x >> 4];
-}
+	error = bus_dmamap_load(sc->rxdesc_tag, sc->rxdesc_map,
+	    sc->rxdesc_ring, RX_DESC_SIZE, dwc_get1paddr,
+	    &sc->rxdesc_ring_paddr, 0);
+	if (error != 0) {
+		device_printf(sc->dev,
+		    "could not load RX descriptor ring map.\n");
+		goto out;
+	}
 
-struct dwc_hash_maddr_ctx {
-	struct dwc_softc *sc;
-	uint32_t hash[8];
-};
+	error = bus_dma_tag_create(
+	    bus_get_dma_tag(sc->dev),	/* Parent tag. */
+	    1, 0,			/* alignment, boundary */
+	    BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
+	    BUS_SPACE_MAXADDR,		/* highaddr */
+	    NULL, NULL,			/* filter, filterarg */
+	    MCLBYTES, 1, 		/* maxsize, nsegments */
+	    MCLBYTES,			/* maxsegsize */
+	    0,				/* flags */
+	    NULL, NULL,			/* lockfunc, lockarg */
+	    &sc->rxbuf_tag);
+	if (error != 0) {
+		device_printf(sc->dev,
+		    "could not create RX buf DMA tag.\n");
+		goto out;
+	}
 
-static u_int
*** 705 LINES SKIPPED ***