svn commit: r213379 - head/sys/dev/usb/controller
Hans Petter Selasky
hselasky at FreeBSD.org
Sun Oct 3 08:12:17 UTC 2010
Author: hselasky
Date: Sun Oct 3 08:12:17 2010
New Revision: 213379
URL: http://svn.freebsd.org/changeset/base/213379
Log:
Commit initial version of new XHCI driver which was written from
scratch. This driver adds support for USB3.0 devices. The XHCI
interface is also backwards compatible to USB2.0 and USB1.0 and will
evntually replace the OHCI/UHCI and EHCI drivers.
There will be follow-up commits during the coming week to link the
driver into the default kernel build and add missing USB3.0
functionality in the USB core. Currently only the driver files are
committed.
Approved by: thompsa (mentor)
Added:
head/sys/dev/usb/controller/xhci.c (contents, props changed)
head/sys/dev/usb/controller/xhci.h (contents, props changed)
head/sys/dev/usb/controller/xhci_pci.c (contents, props changed)
head/sys/dev/usb/controller/xhcireg.h (contents, props changed)
Modified:
Directory Properties:
head/sys/dev/usb/controller/ (props changed)
Added: head/sys/dev/usb/controller/xhci.c
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/dev/usb/controller/xhci.c Sun Oct 3 08:12:17 2010 (r213379)
@@ -0,0 +1,3862 @@
+/*-
+ * Copyright (c) 2010 Hans Petter Selasky. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * USB eXtensible Host Controller Interface, a.k.a. USB 3.0 controller.
+ *
+ * The XHCI 1.0 spec can be found at
+ * http://www.intel.com/technology/usb/download/xHCI_Specification_for_USB.pdf
+ * and the USB 3.0 spec at
+ * http://www.usb.org/developers/docs/usb_30_spec_060910.zip
+ */
+
+/*
+ * A few words about the design implementation: This driver emulates
+ * the concept about TDs which is found in EHCI specification. This
+ * way we avoid too much diveration among USB drivers.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/stdint.h>
+#include <sys/stddef.h>
+#include <sys/param.h>
+#include <sys/queue.h>
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+#include <sys/linker_set.h>
+#include <sys/module.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/condvar.h>
+#include <sys/sysctl.h>
+#include <sys/sx.h>
+#include <sys/unistd.h>
+#include <sys/callout.h>
+#include <sys/malloc.h>
+#include <sys/priv.h>
+
+#include <dev/usb/usb.h>
+#include <dev/usb/usbdi.h>
+
+#define USB_DEBUG_VAR xhcidebug
+
+#include <dev/usb/usb_core.h>
+#include <dev/usb/usb_debug.h>
+#include <dev/usb/usb_busdma.h>
+#include <dev/usb/usb_process.h>
+#include <dev/usb/usb_transfer.h>
+#include <dev/usb/usb_device.h>
+#include <dev/usb/usb_hub.h>
+#include <dev/usb/usb_util.h>
+
+#include <dev/usb/usb_controller.h>
+#include <dev/usb/usb_bus.h>
+#include <dev/usb/controller/xhci.h>
+#include <dev/usb/controller/xhcireg.h>
+
+#define XHCI_BUS2SC(bus) \
+ ((struct xhci_softc *)(((uint8_t *)(bus)) - \
+ ((uint8_t *)&(((struct xhci_softc *)0)->sc_bus))))
+
+#ifdef USB_DEBUG
+static int xhcidebug = 0;
+
+SYSCTL_NODE(_hw_usb, OID_AUTO, xhci, CTLFLAG_RW, 0, "USB XHCI");
+SYSCTL_INT(_hw_usb_xhci, OID_AUTO, debug, CTLFLAG_RW,
+ &xhcidebug, 0, "Debug level");
+
+TUNABLE_INT("hw.usb.xhci.debug", &xhcidebug);
+
+#endif
+
+#define XHCI_INTR_ENDPT 1
+
+struct xhci_std_temp {
+ struct xhci_softc *sc;
+ struct usb_page_cache *pc;
+ struct xhci_td *td;
+ struct xhci_td *td_next;
+ uint32_t len;
+ uint32_t offset;
+ uint32_t max_packet_size;
+ uint32_t average;
+ uint16_t isoc_delta;
+ uint16_t isoc_frame;
+ uint8_t shortpkt;
+ uint8_t multishort;
+ uint8_t last_frame;
+ uint8_t trb_type;
+ uint8_t direction;
+ uint8_t tbc;
+ uint8_t tlbpc;
+ uint8_t step_td;
+};
+
+static void xhci_do_poll(struct usb_bus *);
+static void xhci_device_done(struct usb_xfer *, usb_error_t);
+static void xhci_root_intr(struct xhci_softc *);
+static void xhci_free_device_ext(struct usb_device *);
+static struct xhci_endpoint_ext *xhci_get_endpoint_ext(struct usb_device *,
+ struct usb_endpoint_descriptor *);
+static usb_proc_callback_t xhci_configure_msg;
+static usb_error_t xhci_configure_device(struct usb_device *);
+static usb_error_t xhci_configure_endpoint(struct usb_device *,
+ struct usb_endpoint_descriptor *, uint64_t, uint16_t,
+ uint8_t, uint8_t, uint8_t, uint16_t, uint16_t);
+static usb_error_t xhci_configure_mask(struct usb_device *,
+ uint32_t, uint8_t);
+static usb_error_t xhci_cmd_evaluate_ctx(struct xhci_softc *,
+ uint64_t, uint8_t);
+static void xhci_endpoint_doorbell(struct usb_xfer *);
+
+extern struct usb_bus_methods xhci_bus_methods;
+
+#ifdef USB_DEBUG
+static void
+xhci_dump_trb(struct xhci_trb *trb)
+{
+ DPRINTFN(5, "trb = %p\n", trb);
+ DPRINTFN(5, "qwTrb0 = 0x%016llx\n", (long long)le64toh(trb->qwTrb0));
+ DPRINTFN(5, "dwTrb2 = 0x%08x\n", le32toh(trb->dwTrb2));
+ DPRINTFN(5, "dwTrb3 = 0x%08x\n", le32toh(trb->dwTrb3));
+}
+
+static void
+xhci_dump_endpoint(struct xhci_endp_ctx *pep)
+{
+ DPRINTFN(5, "pep = %p\n", pep);
+ DPRINTFN(5, "dwEpCtx0=0x%08x\n", pep->dwEpCtx0);
+ DPRINTFN(5, "dwEpCtx1=0x%08x\n", pep->dwEpCtx1);
+ DPRINTFN(5, "qwEpCtx2=0x%016llx\n", (long long)pep->qwEpCtx2);
+ DPRINTFN(5, "dwEpCtx4=0x%08x\n", pep->dwEpCtx4);
+ DPRINTFN(5, "dwEpCtx5=0x%08x\n", pep->dwEpCtx5);
+ DPRINTFN(5, "dwEpCtx6=0x%08x\n", pep->dwEpCtx6);
+ DPRINTFN(5, "dwEpCtx7=0x%08x\n", pep->dwEpCtx7);
+}
+
+static void
+xhci_dump_device(struct xhci_slot_ctx *psl)
+{
+ DPRINTFN(5, "psl = %p\n", psl);
+ DPRINTFN(5, "dwSctx0=0x%08x\n", psl->dwSctx0);
+ DPRINTFN(5, "dwSctx1=0x%08x\n", psl->dwSctx1);
+ DPRINTFN(5, "dwSctx2=0x%08x\n", psl->dwSctx2);
+ DPRINTFN(5, "dwSctx3=0x%08x\n", psl->dwSctx3);
+}
+#endif
+
+static void
+xhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
+{
+ struct xhci_softc *sc = XHCI_BUS2SC(bus);
+ uint8_t i;
+
+ cb(bus, &sc->sc_hw.root_pc, &sc->sc_hw.root_pg,
+ sizeof(struct xhci_hw_root), XHCI_PAGE_SIZE);
+
+ cb(bus, &sc->sc_hw.ctx_pc, &sc->sc_hw.ctx_pg,
+ sizeof(struct xhci_dev_ctx_addr), XHCI_PAGE_SIZE);
+
+ for (i = 0; i != XHCI_MAX_SCRATCHPADS; i++) {
+ cb(bus, &sc->sc_hw.scratch_pc[i], &sc->sc_hw.scratch_pg[i],
+ XHCI_PAGE_SIZE, XHCI_PAGE_SIZE);
+ }
+}
+
+usb_error_t
+xhci_start_controller(struct xhci_softc *sc)
+{
+ struct usb_page_search buf_res;
+ struct xhci_hw_root *phwr;
+ struct xhci_dev_ctx_addr *pdctxa;
+ uint64_t addr;
+ uint32_t temp;
+ uint16_t i;
+
+ DPRINTF("\n");
+
+ sc->sc_capa_off = 0;
+ sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH);
+ sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0x1F;
+ sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3;
+
+ DPRINTF("CAPLENGTH=0x%x\n", sc->sc_oper_off);
+ DPRINTF("RUNTIMEOFFSET=0x%x\n", sc->sc_runt_off);
+ DPRINTF("DOOROFFSET=0x%x\n", sc->sc_door_off);
+
+ sc->sc_event_ccs = 1;
+ sc->sc_event_idx = 0;
+ sc->sc_command_ccs = 1;
+ sc->sc_command_idx = 0;
+
+ DPRINTF("xHCI version = 0x%04x\n", XREAD2(sc, capa, XHCI_HCIVERSION));
+
+ temp = XREAD4(sc, capa, XHCI_HCSPARAMS0);
+
+ DPRINTF("HCS0 = 0x%08x\n", temp);
+
+ if (XHCI_HCS0_CSZ(temp)) {
+ device_printf(sc->sc_bus.parent, "Driver does not "
+ "support 64-byte contexts.");
+ return (USB_ERR_IOERROR);
+ }
+
+ /* Reset controller */
+ XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_HCRST);
+
+ for (i = 0; i != 100; i++) {
+ usb_pause_mtx(NULL, hz / 1000);
+ temp = XREAD4(sc, oper, XHCI_USBCMD) &
+ (XHCI_CMD_HCRST | XHCI_STS_CNR);
+ if (!temp)
+ break;
+ }
+
+ if (temp) {
+ device_printf(sc->sc_bus.parent, "Controller "
+ "reset timeout.\n");
+ return (USB_ERR_IOERROR);
+ }
+
+ if (!(XREAD4(sc, oper, XHCI_PAGESIZE) & XHCI_PAGESIZE_4K)) {
+ device_printf(sc->sc_bus.parent, "Controller does "
+ "not support 4K page size.\n");
+ return (USB_ERR_IOERROR);
+ }
+
+ temp = XREAD4(sc, capa, XHCI_HCSPARAMS1);
+
+ i = XHCI_HCS1_N_PORTS(temp);
+
+ if (i == 0) {
+ device_printf(sc->sc_bus.parent, "Invalid number "
+ "of ports: %u\n", i);
+ return (USB_ERR_IOERROR);
+ }
+
+ sc->sc_noport = i;
+ sc->sc_noslot = XHCI_HCS1_DEVSLOT_MAX(temp);
+
+ if (sc->sc_noslot > XHCI_MAX_DEVICES)
+ sc->sc_noslot = XHCI_MAX_DEVICES;
+
+ /* setup number of device slots */
+
+ DPRINTF("CONFIG=0x%08x -> 0x%08x\n",
+ XREAD4(sc, oper, XHCI_CONFIG), sc->sc_noslot);
+
+ XWRITE4(sc, oper, XHCI_CONFIG, sc->sc_noslot);
+
+ DPRINTF("Max slots: %u\n", sc->sc_noslot);
+
+ temp = XREAD4(sc, capa, XHCI_HCSPARAMS2);
+
+ sc->sc_noscratch = XHCI_HCS2_SPB_MAX(temp);
+
+ if (sc->sc_noscratch > XHCI_MAX_SCRATCHPADS) {
+ device_printf(sc->sc_bus.parent, "XHCI request "
+ "too many scratchpads\n");
+ return (USB_ERR_NOMEM);
+ }
+
+ DPRINTF("Max scratch: %u\n", sc->sc_noscratch);
+
+ temp = XREAD4(sc, capa, XHCI_HCSPARAMS3);
+
+ sc->sc_exit_lat_max = XHCI_HCS3_U1_DEL(temp) +
+ XHCI_HCS3_U2_DEL(temp) + 250 /* us */;
+
+ temp = XREAD4(sc, oper, XHCI_USBSTS);
+
+ /* clear interrupts */
+ XWRITE4(sc, oper, XHCI_USBSTS, temp);
+ /* disable all device notifications */
+ XWRITE4(sc, oper, XHCI_DNCTRL, 0);
+
+ /* setup device context base address */
+ usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res);
+ pdctxa = buf_res.buffer;
+ memset(pdctxa, 0, sizeof(*pdctxa));
+
+ addr = buf_res.physaddr;
+ addr += (uintptr_t)&((struct xhci_dev_ctx_addr *)0)->qwSpBufPtr[0];
+
+ /* slot 0 points to the table of scratchpad pointers */
+ pdctxa->qwBaaDevCtxAddr[0] = htole64(addr);
+
+ for (i = 0; i != sc->sc_noscratch; i++) {
+ struct usb_page_search buf_scp;
+ usbd_get_page(&sc->sc_hw.scratch_pc[i], 0, &buf_scp);
+ pdctxa->qwSpBufPtr[i] = htole64((uint64_t)buf_scp.physaddr);
+ }
+
+ addr = buf_res.physaddr;
+
+ XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr);
+ XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32));
+ XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr);
+ XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32));
+
+ /* Setup event table size */
+
+ temp = XREAD4(sc, capa, XHCI_HCSPARAMS2);
+
+ DPRINTF("HCS2=0x%08x\n", temp);
+
+ temp = XHCI_HCS2_ERST_MAX(temp);
+ temp = 1U << temp;
+ if (temp > XHCI_MAX_RSEG)
+ temp = XHCI_MAX_RSEG;
+
+ sc->sc_erst_max = temp;
+
+ DPRINTF("ERSTSZ=0x%08x -> 0x%08x\n",
+ XREAD4(sc, runt, XHCI_ERSTSZ(0)), temp);
+
+ XWRITE4(sc, runt, XHCI_ERSTSZ(0), XHCI_ERSTS_SET(temp));
+
+ /* Setup interrupt rate */
+ XWRITE4(sc, runt, XHCI_IMOD(0), XHCI_IMOD_DEFAULT);
+
+ usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
+
+ phwr = buf_res.buffer;
+ addr = buf_res.physaddr;
+ addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[0];
+
+ /* reset hardware root structure */
+ memset(phwr, 0, sizeof(*phwr));
+
+ phwr->hwr_ring_seg[0].qwEvrsTablePtr = htole64(addr);
+ phwr->hwr_ring_seg[0].dwEvrsTableSize = htole32(XHCI_MAX_EVENTS);
+
+ DPRINTF("ERDP(0)=0x%016llx\n", (unsigned long long)addr);
+
+ XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr);
+ XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32));
+
+ addr = (uint64_t)buf_res.physaddr;
+
+ DPRINTF("ERSTBA(0)=0x%016llx\n", (unsigned long long)addr);
+
+ XWRITE4(sc, runt, XHCI_ERSTBA_LO(0), (uint32_t)addr);
+ XWRITE4(sc, runt, XHCI_ERSTBA_HI(0), (uint32_t)(addr >> 32));
+
+ /* Setup interrupter registers */
+
+ temp = XREAD4(sc, runt, XHCI_IMAN(0));
+ temp |= XHCI_IMAN_INTR_ENA;
+ XWRITE4(sc, runt, XHCI_IMAN(0), temp);
+
+ /* setup command ring control base address */
+ addr = buf_res.physaddr;
+ addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[0];
+
+ DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr);
+
+ XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS);
+ XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32));
+
+ phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr);
+
+ usb_bus_mem_flush_all(&sc->sc_bus, &xhci_iterate_hw_softc);
+
+ /* Go! */
+ XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_RS |
+ XHCI_CMD_INTE | XHCI_CMD_HSEE);
+
+ for (i = 0; i != 100; i++) {
+ usb_pause_mtx(NULL, hz / 1000);
+ temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH;
+ if (!temp)
+ break;
+ }
+ if (temp) {
+ XWRITE4(sc, oper, XHCI_USBCMD, 0);
+ device_printf(sc->sc_bus.parent, "Run timeout.\n");
+ return (USB_ERR_IOERROR);
+ }
+
+ /* catch any lost interrupts */
+ xhci_do_poll(&sc->sc_bus);
+
+ return (0);
+}
+
+usb_error_t
+xhci_halt_controller(struct xhci_softc *sc)
+{
+ uint32_t temp;
+ uint16_t i;
+
+ DPRINTF("\n");
+
+ sc->sc_capa_off = 0;
+ sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH);
+ sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0xF;
+ sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3;
+
+ /* Halt controller */
+ XWRITE4(sc, oper, XHCI_USBCMD, 0);
+
+ for (i = 0; i != 100; i++) {
+ usb_pause_mtx(NULL, hz / 1000);
+ temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH;
+ if (temp)
+ break;
+ }
+
+ if (!temp) {
+ device_printf(sc->sc_bus.parent, "Controller halt timeout.\n");
+ return (USB_ERR_IOERROR);
+ }
+ return (0);
+}
+
+usb_error_t
+xhci_init(struct xhci_softc *sc, device_t self)
+{
+ /* initialise some bus fields */
+ sc->sc_bus.parent = self;
+
+ /* set the bus revision */
+ sc->sc_bus.usbrev = USB_REV_3_0;
+
+ /* set up the bus struct */
+ sc->sc_bus.methods = &xhci_bus_methods;
+
+ /* setup devices array */
+ sc->sc_bus.devices = sc->sc_devices;
+ sc->sc_bus.devices_max = XHCI_MAX_DEVICES;
+
+ /* setup command queue mutex and condition varible */
+ cv_init(&sc->sc_cmd_cv, "CMDQ");
+ sx_init(&sc->sc_cmd_sx, "CMDQ lock");
+
+ /* get all DMA memory */
+ if (usb_bus_mem_alloc_all(&sc->sc_bus,
+ USB_GET_DMA_TAG(self), &xhci_iterate_hw_softc)) {
+ return (ENOMEM);
+ }
+
+ sc->sc_config_msg[0].hdr.pm_callback = &xhci_configure_msg;
+ sc->sc_config_msg[0].bus = &sc->sc_bus;
+ sc->sc_config_msg[1].hdr.pm_callback = &xhci_configure_msg;
+ sc->sc_config_msg[1].bus = &sc->sc_bus;
+
+ if (usb_proc_create(&sc->sc_config_proc,
+ &sc->sc_bus.bus_mtx, device_get_nameunit(self), USB_PRI_MED)) {
+ printf("WARNING: Creation of XHCI configure "
+ "callback process failed.\n");
+ }
+ return (0);
+}
+
+void
+xhci_uninit(struct xhci_softc *sc)
+{
+ usb_proc_free(&sc->sc_config_proc);
+
+ usb_bus_mem_free_all(&sc->sc_bus, &xhci_iterate_hw_softc);
+
+ cv_destroy(&sc->sc_cmd_cv);
+ sx_destroy(&sc->sc_cmd_sx);
+}
+
+void
+xhci_suspend(struct xhci_softc *sc)
+{
+ /* XXX TODO */
+}
+
+void
+xhci_resume(struct xhci_softc *sc)
+{
+ /* XXX TODO */
+}
+
+void
+xhci_shutdown(struct xhci_softc *sc)
+{
+ DPRINTF("Stopping the XHCI\n");
+
+ xhci_halt_controller(sc);
+}
+
+static usb_error_t
+xhci_generic_done_sub(struct usb_xfer *xfer)
+{
+ struct xhci_td *td;
+ struct xhci_td *td_alt_next;
+ uint32_t len;
+ uint8_t status;
+
+ td = xfer->td_transfer_cache;
+ td_alt_next = td->alt_next;
+
+ if (xfer->aframes != xfer->nframes)
+ usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
+
+ while (1) {
+
+ usb_pc_cpu_invalidate(td->page_cache);
+
+ status = td->status;
+ len = td->remainder;
+
+ DPRINTFN(4, "xfer=%p[%u/%u] rem=%u/%u status=%u\n",
+ xfer, (unsigned int)xfer->aframes,
+ (unsigned int)xfer->nframes,
+ (unsigned int)len, (unsigned int)td->len,
+ (unsigned int)status);
+
+ /*
+ * Verify the status length and
+ * add the length to "frlengths[]":
+ */
+ if (len > td->len) {
+ /* should not happen */
+ DPRINTF("Invalid status length, "
+ "0x%04x/0x%04x bytes\n", len, td->len);
+ status = XHCI_TRB_ERROR_LENGTH;
+ } else if (xfer->aframes != xfer->nframes) {
+ xfer->frlengths[xfer->aframes] += td->len - len;
+ }
+ /* Check for last transfer */
+ if (((void *)td) == xfer->td_transfer_last) {
+ td = NULL;
+ break;
+ }
+ /* Check for transfer error */
+ if (status != XHCI_TRB_ERROR_SHORT_PKT &&
+ status != XHCI_TRB_ERROR_SUCCESS) {
+ /* the transfer is finished */
+ td = NULL;
+ break;
+ }
+ /* Check for short transfer */
+ if (len > 0) {
+ if (xfer->flags_int.short_frames_ok ||
+ xfer->flags_int.isochronous_xfr ||
+ xfer->flags_int.control_xfr) {
+ /* follow alt next */
+ td = td->alt_next;
+ } else {
+ /* the transfer is finished */
+ td = NULL;
+ }
+ break;
+ }
+ td = td->obj_next;
+
+ if (td->alt_next != td_alt_next) {
+ /* this USB frame is complete */
+ break;
+ }
+ }
+
+ /* update transfer cache */
+
+ xfer->td_transfer_cache = td;
+
+ return ((status == XHCI_TRB_ERROR_STALL) ? USB_ERR_STALLED :
+ (status != XHCI_TRB_ERROR_SHORT_PKT &&
+ status != XHCI_TRB_ERROR_SUCCESS) ? USB_ERR_IOERROR :
+ USB_ERR_NORMAL_COMPLETION);
+}
+
+static void
+xhci_generic_done(struct usb_xfer *xfer)
+{
+ usb_error_t err = 0;
+
+ DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
+ xfer, xfer->endpoint);
+
+ /* reset scanner */
+
+ xfer->td_transfer_cache = xfer->td_transfer_first;
+
+ if (xfer->flags_int.control_xfr) {
+
+ if (xfer->flags_int.control_hdr)
+ err = xhci_generic_done_sub(xfer);
+
+ xfer->aframes = 1;
+
+ if (xfer->td_transfer_cache == NULL)
+ goto done;
+ }
+
+ while (xfer->aframes != xfer->nframes) {
+
+ err = xhci_generic_done_sub(xfer);
+ xfer->aframes++;
+
+ if (xfer->td_transfer_cache == NULL)
+ goto done;
+ }
+
+ if (xfer->flags_int.control_xfr &&
+ !xfer->flags_int.control_act)
+ err = xhci_generic_done_sub(xfer);
+done:
+ /* transfer is complete */
+ xhci_device_done(xfer, err);
+}
+
+static void
+xhci_activate_transfer(struct usb_xfer *xfer)
+{
+ struct xhci_td *td;
+
+ td = xfer->td_transfer_cache;
+
+ usb_pc_cpu_invalidate(td->page_cache);
+
+ if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) {
+
+ /* activate the transfer */
+
+ td->td_trb[0].dwTrb3 |= htole32(XHCI_TRB_3_CYCLE_BIT);
+ usb_pc_cpu_flush(td->page_cache);
+
+ xhci_endpoint_doorbell(xfer);
+ }
+}
+
+static void
+xhci_skip_transfer(struct usb_xfer *xfer)
+{
+ struct xhci_td *td;
+ struct xhci_td *td_last;
+
+ td = xfer->td_transfer_cache;
+ td_last = xfer->td_transfer_last;
+
+ td = td->alt_next;
+
+ usb_pc_cpu_invalidate(td->page_cache);
+
+ if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) {
+
+ usb_pc_cpu_invalidate(td_last->page_cache);
+
+ /* copy LINK TRB to current waiting location */
+
+ td->td_trb[0].qwTrb0 = td_last->td_trb[td_last->ntrb].qwTrb0;
+ td->td_trb[0].dwTrb2 = td_last->td_trb[td_last->ntrb].dwTrb2;
+ usb_pc_cpu_flush(td->page_cache);
+
+ td->td_trb[0].dwTrb3 = td_last->td_trb[td_last->ntrb].dwTrb3;
+ usb_pc_cpu_flush(td->page_cache);
+
+ xhci_endpoint_doorbell(xfer);
+ }
+}
+
+/*------------------------------------------------------------------------*
+ * xhci_check_transfer
+ *------------------------------------------------------------------------*/
+static void
+xhci_check_transfer(struct xhci_softc *sc, struct xhci_trb *trb)
+{
+ int64_t offset;
+ uint64_t td_event;
+ uint32_t temp;
+ uint32_t remainder;
+ uint8_t status;
+ uint8_t halted;
+ uint8_t epno;
+ uint8_t index;
+ uint8_t i;
+
+ /* decode TRB */
+ td_event = le64toh(trb->qwTrb0);
+ temp = le32toh(trb->dwTrb2);
+
+ remainder = XHCI_TRB_2_REM_GET(temp);
+ status = XHCI_TRB_2_ERROR_GET(temp);
+
+ temp = le32toh(trb->dwTrb3);
+ epno = XHCI_TRB_3_EP_GET(temp);
+ index = XHCI_TRB_3_SLOT_GET(temp);
+
+ /* check if error means halted */
+ halted = (status != XHCI_TRB_ERROR_SHORT_PKT &&
+ status != XHCI_TRB_ERROR_SUCCESS);
+
+ DPRINTF("slot=%u epno=%u remainder=%u status=%u\n",
+ index, epno, remainder, status);
+
+ if (index > sc->sc_noslot) {
+ DPRINTF("Invalid slot.\n");
+ return;
+ }
+
+ if ((epno == 0) || (epno >= XHCI_MAX_ENDPOINTS)) {
+ DPRINTF("Invalid endpoint.\n");
+ return;
+ }
+
+ /* try to find the USB transfer that generated the event */
+ for (i = 0; i != (XHCI_MAX_TRANSFERS - 1); i++) {
+ struct usb_xfer *xfer;
+ struct xhci_td *td;
+ struct xhci_endpoint_ext *pepext;
+
+ pepext = &sc->sc_hw.devs[index].endp[epno];
+
+ xfer = pepext->xfer[i];
+ if (xfer == NULL)
+ continue;
+
+ td = xfer->td_transfer_cache;
+
+ DPRINTFN(5, "Checking if 0x%016llx == (0x%016llx .. 0x%016llx)\n",
+ (long long)td_event,
+ (long long)td->td_self,
+ (long long)td->td_self + sizeof(td->td_trb));
+
+ /*
+ * NOTE: Some XHCI implementations might not trigger
+ * an event on the last LINK TRB so we need to
+ * consider both the last and second last event
+ * address as conditions for a successful transfer.
+ *
+ * NOTE: We assume that the XHCI will only trigger one
+ * event per chain of TRBs.
+ */
+
+ offset = td_event - td->td_self;
+
+ if (offset >= 0 &&
+ offset < sizeof(td->td_trb)) {
+
+ usb_pc_cpu_invalidate(td->page_cache);
+
+ /* compute rest of remainder, if any */
+ for (i = (offset / 16) + 1; i < td->ntrb; i++) {
+ temp = le32toh(td->td_trb[i].dwTrb2);
+ remainder += XHCI_TRB_2_BYTES_GET(temp);
+ }
+
+ DPRINTFN(5, "New remainder: %u\n", remainder);
+
+ /* clear isochronous transfer errors */
+ if (xfer->flags_int.isochronous_xfr) {
+ if (halted) {
+ halted = 0;
+ status = XHCI_TRB_ERROR_SUCCESS;
+ remainder = td->len;
+ }
+ }
+
+ /* "td->remainder" is verified later */
+ td->remainder = remainder;
+ td->status = status;
+
+ usb_pc_cpu_flush(td->page_cache);
+
+ /*
+ * 1) Last transfer descriptor makes the
+ * transfer done
+ */
+ if (((void *)td) == xfer->td_transfer_last) {
+ DPRINTF("TD is last\n");
+ xhci_generic_done(xfer);
+ break;
+ }
+
+ /*
+ * 2) Any kind of error makes the transfer
+ * done
+ */
+ if (halted) {
+ DPRINTF("TD has I/O error\n");
+ xhci_generic_done(xfer);
+ break;
+ }
+
+ /*
+ * 3) If there is no alternate next transfer,
+ * a short packet also makes the transfer done
+ */
+ if (td->remainder > 0) {
+ DPRINTF("TD has short pkt\n");
+ if (xfer->flags_int.short_frames_ok ||
+ xfer->flags_int.isochronous_xfr ||
+ xfer->flags_int.control_xfr) {
+ /* follow the alt next */
+ xfer->td_transfer_cache = td->alt_next;
+ xhci_activate_transfer(xfer);
+ break;
+ }
+ xhci_skip_transfer(xfer);
+ xhci_generic_done(xfer);
+ break;
+ }
+
+ /*
+ * 4) Transfer complete - go to next TD
+ */
+ DPRINTF("Following next TD\n");
+ xfer->td_transfer_cache = td->obj_next;
+ xhci_activate_transfer(xfer);
+ break; /* there should only be one match */
+ }
+ }
+}
+
+static void
+xhci_check_command(struct xhci_softc *sc, struct xhci_trb *trb)
+{
+ if (sc->sc_cmd_addr == trb->qwTrb0) {
+ DPRINTF("Received command event\n");
+ sc->sc_cmd_result[0] = trb->dwTrb2;
+ sc->sc_cmd_result[1] = trb->dwTrb3;
+ cv_signal(&sc->sc_cmd_cv);
+ }
+}
+
+static void
+xhci_interrupt_poll(struct xhci_softc *sc)
+{
+ struct usb_page_search buf_res;
+ struct xhci_hw_root *phwr;
+ uint64_t addr;
+ uint32_t temp;
+ uint16_t i;
+ uint8_t event;
+ uint8_t j;
+ uint8_t k;
+ uint8_t t;
+
+ usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
+
+ phwr = buf_res.buffer;
+
+ /* Receive any events */
+
+ usb_pc_cpu_invalidate(&sc->sc_hw.root_pc);
+
+ i = sc->sc_event_idx;
+ j = sc->sc_event_ccs;
+ t = 2;
+
+ while (1) {
+
+ temp = le32toh(phwr->hwr_events[i].dwTrb3);
+
+ k = (temp & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0;
+
+ if (j != k)
+ break;
+
+ event = XHCI_TRB_3_TYPE_GET(temp);
+
+ DPRINTFN(10, "event[%u] = %u (0x%016llx 0x%08lx 0x%08lx)\n",
+ i, event, (long long)le64toh(phwr->hwr_events[i].qwTrb0),
+ (long)le32toh(phwr->hwr_events[i].dwTrb2),
+ (long)le32toh(phwr->hwr_events[i].dwTrb3));
+
+ switch (event) {
+ case XHCI_TRB_EVENT_TRANSFER:
+ xhci_check_transfer(sc, &phwr->hwr_events[i]);
+ break;
+ case XHCI_TRB_EVENT_CMD_COMPLETE:
+ xhci_check_command(sc, &phwr->hwr_events[i]);
+ break;
+ default:
+ DPRINTF("Unhandled event = %u\n", event);
+ break;
+ }
+
+ i++;
+
+ if (i == XHCI_MAX_EVENTS) {
+ i = 0;
+ j ^= 1;
+
+ /* check for timeout */
+ if (!--t)
+ break;
+ }
+ }
+
+ sc->sc_event_idx = i;
+ sc->sc_event_ccs = j;
+
+ /*
+ * NOTE: The Event Ring Dequeue Pointer Register is 64-bit
+ * latched. That means to activate the register we need to
+ * write both the low and high double word of the 64-bit
+ * register.
+ */
+
+ addr = (uint32_t)buf_res.physaddr;
+ addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[i];
+
+ /* try to clear busy bit */
+ addr |= XHCI_ERDP_LO_BUSY;
+
+ XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr);
+ XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32));
+}
+
+static usb_error_t
+xhci_do_command(struct xhci_softc *sc, struct xhci_trb *trb,
+ uint16_t timeout_ms)
+{
+ struct usb_page_search buf_res;
+ struct xhci_hw_root *phwr;
+ uint64_t addr;
+ uint32_t temp;
+ uint8_t i;
+ uint8_t j;
+ int err;
+
+ XHCI_CMD_ASSERT_LOCKED(sc);
+
+ /* get hardware root structure */
+
+ usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
+
+ phwr = buf_res.buffer;
+
+ /* Queue command */
+
+ USB_BUS_LOCK(&sc->sc_bus);
+
+ i = sc->sc_command_idx;
+ j = sc->sc_command_ccs;
+
+ DPRINTFN(10, "command[%u] = %u (0x%016llx, 0x%08lx, 0x%08lx)\n",
+ i, XHCI_TRB_3_TYPE_GET(le32toh(trb->dwTrb3)),
+ (long long)le64toh(trb->qwTrb0),
+ (long)le32toh(trb->dwTrb2),
+ (long)le32toh(trb->dwTrb3));
+
+ phwr->hwr_commands[i].qwTrb0 = trb->qwTrb0;
+ phwr->hwr_commands[i].dwTrb2 = trb->dwTrb2;
+
+ usb_pc_cpu_flush(&sc->sc_hw.root_pc);
+
+ temp = trb->dwTrb3;
+
+ if (j)
+ temp |= htole32(XHCI_TRB_3_CYCLE_BIT);
+ else
+ temp &= ~htole32(XHCI_TRB_3_CYCLE_BIT);
+
+ temp &= ~htole32(XHCI_TRB_3_TC_BIT);
+
+ phwr->hwr_commands[i].dwTrb3 = temp;
+
+ usb_pc_cpu_flush(&sc->sc_hw.root_pc);
+
+ addr = buf_res.physaddr;
+ addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[i];
+
+ sc->sc_cmd_addr = htole64(addr);
+
+ i++;
+
+ if (i == (XHCI_MAX_COMMANDS - 1)) {
+
+ if (j) {
+ temp = htole32(XHCI_TRB_3_TC_BIT |
+ XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) |
+ XHCI_TRB_3_CYCLE_BIT);
+ } else {
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-head
mailing list