svn commit: r199670 - head/sys/dev/bge

[Scott Long]

By definition, PCIe can't transfer across 4GB boundaries.  It's not a  
bug specific to Broadcom.  If you're loading dynamic buffers (i.e.  
mbufs), setting an appropriate boundary value in the tag will allow  
busdma to take care of this.  If you're allocating static buffers,  
busdma won't honor this.  But what you've done here is best anyways;  
control buffers that are going to be frequently transferred are best  
kept in the lower 4GB of the address space.  It simplifies PCIe  
handling, and it's significantly faster on PCI/PCI-X.  So I'd suggest  
making this the rule rather than the exception in the driver.

Scott

On Nov 22, 2009, at 1:50 PM, Pyun YongHyeon wrote:

> Author: yongari
> Date: Sun Nov 22 20:50:27 2009
> New Revision: 199670
> URL: http://svn.freebsd.org/changeset/base/199670
>
> Log:
>  Fix two long standing bugs on bge(4). Most pre BCM5755 controllers
>  have a DMA bug when buffer address crosses a multiple of the 4GB
>  boundary(e.g. 4GB, 8GB, 12GB etc). Limit DMA address to be within
>  4GB address for these controllers. The second DMA bug limits DMA
>  address to be within 40bit address space. This bug applies to
>  BCM5714 and BCM5715 and 5708(bce(4) controller). This is not
>  actually a MAC controller bug but an issue with the embedded PCIe
>  to PCI-X bridge in the device. So for BCM5714/BCM5715 controllers
>  also limit the DMA address to be within 40bit address space.
>  Special thanks to davidch@ who gave me detailed errata information.
>  I think this change will fix long standing bge(4) instability
>  issues on systems with more than 4GB memory.
>
>  Reviewed by:	davidch
>
> Modified:
>  head/sys/dev/bge/if_bge.c
>  head/sys/dev/bge/if_bgereg.h
>
> Modified: head/sys/dev/bge/if_bge.c
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> ======================================================================
> --- head/sys/dev/bge/if_bge.c	Sun Nov 22 20:45:15 2009	(r199669)
> +++ head/sys/dev/bge/if_bge.c	Sun Nov 22 20:50:27 2009	(r199670)
> @@ -2104,15 +2104,21 @@ bge_dma_alloc(device_t dev)
> {
> 	struct bge_dmamap_arg ctx;
> 	struct bge_softc *sc;
> +	bus_addr_t lowaddr;
> 	int i, error;
>
> 	sc = device_get_softc(dev);
>
> +	lowaddr = BUS_SPACE_MAXADDR;
> +	if ((sc->bge_flags & BGE_FLAG_40BIT_BUG) != 0)
> +		lowaddr = BGE_DMA_MAXADDR;
> +	if ((sc->bge_flags & BGE_FLAG_4G_BNDRY_BUG) != 0)
> +		lowaddr = BUS_SPACE_MAXADDR_32BIT;
> 	/*
> 	 * Allocate the parent bus DMA tag appropriate for PCI.
> 	 */
> 	error = bus_dma_tag_create(bus_get_dma_tag(sc->bge_dev),
> -	    1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,	NULL,
> +	    1, 0, lowaddr, BUS_SPACE_MAXADDR, NULL,
> 	    NULL, BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT,
> 	    0, NULL, NULL, &sc->bge_cdata.bge_parent_tag);
>
> @@ -2566,6 +2572,16 @@ bge_attach(device_t dev)
> 			sc->bge_flags |= BGE_FLAG_BER_BUG;
> 	}
>
> +	/*
> +	 * All controllers that are not 5755 or higher have 4GB
> +	 * boundary DMA bug.
> +	 * Whenever an address crosses a multiple of the 4GB boundary
> +	 * (including 4GB, 8Gb, 12Gb, etc.) and makes the transition
> +	 * from 0xX_FFFF_FFFF to 0x(X+1)_0000_0000 an internal DMA
> +	 * state machine will lockup and cause the device to hang.
> +	 */
> +	if (BGE_IS_5755_PLUS(sc) == 0)
> +		sc->bge_flags |= BGE_FLAG_4G_BNDRY_BUG;
>
> 	/*
> 	 * We could possibly check for BCOM_DEVICEID_BCM5788 in bge_probe()
> @@ -2729,6 +2745,13 @@ bge_attach(device_t dev)
> #ifdef DEVICE_POLLING
> 	ifp->if_capabilities |= IFCAP_POLLING;
> #endif
> +	/*
> +	 * The 40bit DMA bug applies to the 5714/5715 controllers and is
> +	 * not actually a MAC controller bug but an issue with the embedded
> +	 * PCIe to PCI-X bridge in the device. Use 40bit DMA workaround.
> +	 */
> +	if (BGE_IS_5714_FAMILY(sc) && (sc->bge_flags & BGE_FLAG_PCIX))
> +		sc->bge_flags |= BGE_FLAG_40BIT_BUG;
>
> 	/*
> 	 * 5700 B0 chips do not support checksumming correctly due
>
> Modified: head/sys/dev/bge/if_bgereg.h
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> = 
> ======================================================================
> --- head/sys/dev/bge/if_bgereg.h	Sun Nov 22 20:45:15 2009	(r199669)
> +++ head/sys/dev/bge/if_bgereg.h	Sun Nov 22 20:50:27 2009	(r199670)
> @@ -2484,6 +2484,13 @@ struct bge_gib {
> #define	BGE_NSEG_JUMBO	4
> #define	BGE_NSEG_NEW 32
>
> +/* Maximum DMA address for controllers that have 40bit DMA address  
> bug. */
> +#if (BUS_SPACE_MAXADDR < 0xFFFFFFFFFF)
> +#define	BGE_DMA_MAXADDR		BUS_SPACE_MAXADDR
> +#else
> +#define	BGE_DMA_MAXADDR		0xFFFFFFFFFF
> +#endif
> +
> /*
>  * Ring structures. Most of these reside in host memory and we tell
>  * the NIC where they are via the ring control blocks. The exceptions
> @@ -2600,6 +2607,8 @@ struct bge_softc {
> #define	BGE_FLAG_5714_FAMILY	0x00004000
> #define	BGE_FLAG_575X_PLUS	0x00008000
> #define	BGE_FLAG_5755_PLUS	0x00010000
> +#define	BGE_FLAG_40BIT_BUG	0x00020000
> +#define	BGE_FLAG_4G_BNDRY_BUG	0x00040000
> #define	BGE_FLAG_RX_ALIGNBUG	0x00100000
> #define	BGE_FLAG_NO_3LED	0x00200000
> #define	BGE_FLAG_ADC_BUG	0x00400000