git: 9f90536c74b8 - main - apple_bce/vhci: add T2 virtual USB host controller
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Sun, 14 Jun 2026 20:57:24 UTC
The branch main has been updated by adrian:
URL: https://cgit.FreeBSD.org/src/commit/?id=9f90536c74b8172fc67cd977e5451f37a12462d5
commit 9f90536c74b8172fc67cd977e5451f37a12462d5
Author: Abdelkader Boudih <freebsd@seuros.com>
AuthorDate: 2026-06-14 20:54:28 +0000
Commit: Adrian Chadd <adrian@FreeBSD.org>
CommitDate: 2026-06-14 20:56:53 +0000
apple_bce/vhci: add T2 virtual USB host controller
Implements a VHCI driver on top of the BCE transport:
- Virtual USB bus registration via usb_controller
- Port discovery and device enumeration
- Control, interrupt, and bulk endpoint support
- Firmware event handling with taskqueue
- Suspend/resume via BCE mailbox
Provides keyboard, trackpad, and Touch Bar access on T2 Macs.
Tested-on: MacBookPro16,2 (A2251), Mac mini 8,1 (A1993)
Reviewed by: adrian
Differential Revision: https://reviews.freebsd.org/D57089
---
sys/conf/files.amd64 | 1 +
sys/dev/apple_bce/apple_bce.c | 87 +
sys/dev/apple_bce/apple_bce.h | 3 +-
sys/dev/apple_bce/apple_bce_mailbox.c | 5 +
sys/dev/apple_bce/apple_bce_vhci.c | 4821 +++++++++++++++++++++++++++++++
sys/dev/apple_bce/apple_bce_vhci.h | 251 ++
sys/dev/usb/controller/usb_controller.c | 1 +
sys/modules/apple_bce/Makefile | 2 +
8 files changed, 5170 insertions(+), 1 deletion(-)
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index 4d37266d1973..0b4a22eebc78 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -115,6 +115,7 @@ dev/amdgpio/amdgpio.c optional amdgpio
dev/apple_bce/apple_bce.c optional apple_bce pci
dev/apple_bce/apple_bce_mailbox.c optional apple_bce pci
dev/apple_bce/apple_bce_queue.c optional apple_bce pci
+dev/apple_bce/apple_bce_vhci.c optional apple_bce pci
dev/asmc/asmc.c optional asmc isa
dev/asmc/asmcmmio.c optional asmc isa
dev/axgbe/if_axgbe_pci.c optional axp
diff --git a/sys/dev/apple_bce/apple_bce.c b/sys/dev/apple_bce/apple_bce.c
index 8700eddbc0fe..2b0e6928ffcd 100644
--- a/sys/dev/apple_bce/apple_bce.c
+++ b/sys/dev/apple_bce/apple_bce.c
@@ -32,10 +32,13 @@
#include "apple_bce.h"
#include "apple_bce_mailbox.h"
#include "apple_bce_queue.h"
+#include "apple_bce_vhci.h"
static int apple_bce_probe(device_t dev);
static int apple_bce_attach(device_t dev);
static int apple_bce_detach(device_t dev);
+static int apple_bce_suspend(device_t dev);
+static int apple_bce_resume(device_t dev);
static void apple_bce_timestamp_cb(void *arg);
static void apple_bce_timestamp_init(struct apple_bce_softc *sc);
static void apple_bce_timestamp_start(struct apple_bce_softc *sc,
@@ -516,6 +519,12 @@ apple_bce_attach(device_t dev)
goto fail;
device_printf(dev, "Apple T2 BCE initialized\n");
+
+ /* Create VHCI child for virtual USB */
+ error = bce_vhci_attach(sc);
+ if (error != 0)
+ goto fail;
+
return (0);
fail:
@@ -531,6 +540,9 @@ apple_bce_detach(device_t dev)
{
struct apple_bce_softc *sc = device_get_softc(dev);
+ /* 0. Detach VHCI child first (before destroying parent resources) */
+ bce_vhci_detach(sc);
+
/* 1. Stop timestamp */
if (sc->sc_bar4 != NULL && mtx_initialized(&sc->sc_timestamp_lock))
apple_bce_timestamp_stop(sc);
@@ -616,10 +628,85 @@ apple_bce_detach(device_t dev)
return (0);
}
+static int
+apple_bce_suspend(device_t dev)
+{
+ struct apple_bce_softc *sc = device_get_softc(dev);
+ int error, restore_error;
+
+ apple_bce_timestamp_stop(sc);
+
+ error = bce_vhci_detach(sc);
+ if (error != 0) {
+ device_printf(dev, "failed to detach VHCI for suspend: %d\n",
+ error);
+ apple_bce_timestamp_start(sc, 0);
+ return (error);
+ }
+
+ error = bce_mailbox_send(&sc->sc_mbox,
+ BCE_MB_MSG(BCE_MB_SLEEP_NO_STATE, 0), NULL,
+ BCE_MBOX_TIMEOUT_MS);
+ if (error != 0) {
+ device_printf(dev,
+ "failed to send SLEEP_NO_STATE mailbox command: %d\n",
+ error);
+ restore_error = bce_vhci_attach(sc);
+ if (restore_error != 0) {
+ device_printf(dev,
+ "failed to reattach VHCI after suspend error: %d\n",
+ restore_error);
+ }
+ apple_bce_timestamp_start(sc, 0);
+ return (error);
+ }
+
+ return (0);
+}
+
+static int
+apple_bce_resume(device_t dev)
+{
+ struct apple_bce_softc *sc = device_get_softc(dev);
+ uint64_t reply;
+ int error;
+
+ error = bce_mailbox_send(&sc->sc_mbox,
+ BCE_MB_MSG(BCE_MB_RESTORE_NO_STATE, 0), &reply,
+ BCE_MBOX_TIMEOUT_MS);
+ if (error != 0) {
+ device_printf(dev,
+ "failed to send RESTORE_NO_STATE mailbox command: %d\n",
+ error);
+ return (error);
+ }
+
+ if (BCE_MB_TYPE(reply) != BCE_MB_RESTORE_NO_STATE) {
+ device_printf(dev,
+ "unexpected RESTORE_NO_STATE reply: type=%u val=0x%llx\n",
+ BCE_MB_TYPE(reply),
+ (unsigned long long)BCE_MB_VALUE(reply));
+ return (EINVAL);
+ }
+
+ error = bce_vhci_attach(sc);
+ if (error != 0) {
+ device_printf(dev, "failed to reattach VHCI after resume: %d\n",
+ error);
+ apple_bce_timestamp_start(sc, 0);
+ return (error);
+ }
+
+ apple_bce_timestamp_start(sc, 0);
+ return (0);
+}
+
static device_method_t apple_bce_methods[] = {
DEVMETHOD(device_probe, apple_bce_probe),
DEVMETHOD(device_attach, apple_bce_attach),
DEVMETHOD(device_detach, apple_bce_detach),
+ DEVMETHOD(device_suspend, apple_bce_suspend),
+ DEVMETHOD(device_resume, apple_bce_resume),
DEVMETHOD_END
};
diff --git a/sys/dev/apple_bce/apple_bce.h b/sys/dev/apple_bce/apple_bce.h
index 7c2198c7232b..0e1999e48d5b 100644
--- a/sys/dev/apple_bce/apple_bce.h
+++ b/sys/dev/apple_bce/apple_bce.h
@@ -27,7 +27,7 @@
#define BCE_PCI_DEVICE_T2 0x1801
#define BCE_MAX_QUEUE_COUNT 0x100
-#define BCE_MAX_CQ_COUNT 16 /* Max completion queues tracked */
+#define BCE_MAX_CQ_COUNT 64 /* Max completion queues tracked */
#define BCE_QUEUE_USER_MIN 2
#define BCE_QUEUE_USER_MAX (BCE_MAX_QUEUE_COUNT - 1)
#define BCE_CMD_SIZE 0x40
@@ -286,6 +286,7 @@ struct apple_bce_softc {
struct mtx sc_queues_lock;
struct bce_queue_cq *sc_cq_list[BCE_MAX_CQ_COUNT];
struct bce_queue_sq *sc_int_sq_list[BCE_MAX_QUEUE_COUNT];
+ device_t sc_vhci_dev;
};
/* Inline helpers */
diff --git a/sys/dev/apple_bce/apple_bce_mailbox.c b/sys/dev/apple_bce/apple_bce_mailbox.c
index abdb809b5c54..c19a01b7269c 100644
--- a/sys/dev/apple_bce/apple_bce_mailbox.c
+++ b/sys/dev/apple_bce/apple_bce_mailbox.c
@@ -57,6 +57,11 @@ bce_mailbox_send(struct bce_mailbox *mb, uint64_t msg, uint64_t *recv,
bus_write_4(mb->reg, BCE_REG_MBOX_OUT + 8, 0);
bus_write_4(mb->reg, BCE_REG_MBOX_OUT + 12, 0);
+ if (recv == NULL) {
+ atomic_store_int(&mb->status, 0);
+ return (0);
+ }
+
/* Wait for interrupt-driven reply */
if (sema_timedwait(&mb->mb_cmpl, hz * timeout_ms / 1000) != 0) {
/* Timeout -- reset to idle */
diff --git a/sys/dev/apple_bce/apple_bce_vhci.c b/sys/dev/apple_bce/apple_bce_vhci.c
new file mode 100644
index 000000000000..b8dbc9638b36
--- /dev/null
+++ b/sys/dev/apple_bce/apple_bce_vhci.c
@@ -0,0 +1,4821 @@
+/*-
+ * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Apple T2 BCE Virtual USB Host Controller Interface (VHCI).
+ */
+
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
+#include <sys/stdint.h>
+#include <sys/stddef.h>
+#include <sys/param.h>
+#include <sys/queue.h>
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+#include <sys/module.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/condvar.h>
+#include <sys/sysctl.h>
+#include <sys/sx.h>
+#include <sys/unistd.h>
+#include <sys/callout.h>
+#include <sys/malloc.h>
+#include <sys/priv.h>
+#endif
+
+#include <sys/sema.h>
+#include <sys/taskqueue.h>
+#include <sys/endian.h>
+#include <machine/bus.h>
+#include <machine/atomic.h>
+
+#include <dev/usb/usb.h>
+#include <dev/usb/usbdi.h>
+#include <dev/usb/usb_core.h>
+#include <dev/usb/usb_busdma.h>
+#include <dev/usb/usb_process.h>
+#include <dev/usb/usb_transfer.h>
+#include <dev/usb/usb_device.h>
+#include <dev/usb/usb_hub.h>
+#include <dev/usb/usb_util.h>
+#include <dev/usb/usb_controller.h>
+#include <dev/usb/usb_bus.h>
+
+#include "apple_bce.h"
+#include "apple_bce_queue.h"
+#include "apple_bce_vhci.h"
+
+/*
+ * VHCI softc, defined here because it depends on USB headers.
+ */
+struct bce_vhci_softc {
+ struct usb_bus sc_bus; /* Must be first */
+ struct usb_device *sc_devices[BCE_VHCI_MAX_DEVICES];
+ struct apple_bce_softc *sc_bce;
+ device_t sc_dev;
+
+ /* Controller state */
+ uint32_t sc_port_mask;
+ uint8_t sc_port_count;
+ int sc_started;
+
+ /* Port state */
+ uint32_t sc_port_status[BCE_VHCI_MAX_PORTS];
+ uint32_t sc_port_change[BCE_VHCI_MAX_PORTS];
+ uint8_t sc_port_power[BCE_VHCI_MAX_PORTS];
+
+ /* Hub scratch buffer (for descriptor/status responses) */
+ uint8_t sc_hub_idata[32];
+
+ /* Message queues (host -> device) */
+ struct bce_vhci_msg_queue msg_commands;
+ struct bce_vhci_msg_queue msg_system;
+ struct bce_vhci_msg_queue msg_isochronous;
+ struct bce_vhci_msg_queue msg_interrupt;
+ struct bce_vhci_msg_queue msg_asynchronous;
+
+ /* Event queues (device -> host), share a single CQ */
+ struct bce_queue_cq *ev_cq;
+ struct bce_vhci_evt_queue ev_commands;
+ struct bce_vhci_evt_queue ev_system;
+ struct bce_vhci_evt_queue ev_isochronous;
+ struct bce_vhci_evt_queue ev_interrupt;
+ struct bce_vhci_evt_queue ev_asynchronous;
+
+ /* Command execution (synchronous, wraps msg_commands) */
+ struct bce_vhci_cmd_queue cmd;
+
+ /* Queue ID bitmap (256 bits = BCE_MAX_QUEUE_COUNT) */
+ uint32_t sc_qid_bitmap[8];
+
+ /* Per-device state (indexed by firmware device ID) */
+ struct bce_vhci_device sc_devs[BCE_VHCI_MAX_DEVICES];
+ uint8_t sc_port_to_dev[BCE_VHCI_MAX_PORTS];
+
+ /* Deferred firmware event processing (from ev_commands) */
+ struct task sc_fwevt_task;
+ volatile int sc_detaching; /* Teardown guard */
+
+ /*
+ * Firmware event mailbox: ISR copies events here, task processes.
+ * Protected by sc_fwevt_lock. Ring of BCE_VHCI_EVT_PENDING entries.
+ */
+ struct mtx sc_fwevt_lock;
+#define BCE_VHCI_FWEVT_RING (BCE_VHCI_EVT_PENDING + 1)
+ struct {
+ struct bce_vhci_message msg;
+ int needs_reply;
+ } sc_fwevt_ring[BCE_VHCI_FWEVT_RING];
+ uint32_t sc_fwevt_prod;
+ uint32_t sc_fwevt_cons;
+
+ /* Spinlock for msg_asynchronous writes (ISR + taskqueue context) */
+ struct mtx sc_async_lock;
+
+ /* Deferred endpoint reset (cannot sleep in pipe_start) */
+ struct task sc_reset_task;
+
+ /* Deferred endpoint create (cannot sleep in pipe_start) */
+ struct task sc_create_task;
+
+ /* Deferred port status change (ISR cannot call cmd_execute) */
+ struct task sc_port_chg_task;
+ volatile uint32_t sc_port_chg_mask;
+};
+
+/* Command timeout (ticks) */
+#define BCE_VHCI_CMD_TIMEOUT_SHORT (hz * 2)
+#define BCE_VHCI_CMD_TIMEOUT_LONG (hz * 30)
+
+static usb_handle_req_t bce_vhci_roothub_exec;
+static void bce_vhci_endpoint_init(struct usb_device *udev,
+ struct usb_endpoint_descriptor *edesc, struct usb_endpoint *ep);
+static void bce_vhci_xfer_setup(struct usb_setup_params *parm);
+static void bce_vhci_xfer_unsetup(struct usb_xfer *xfer);
+static void bce_vhci_get_dma_delay(struct usb_device *udev, uint32_t *pus);
+
+static void bce_vhci_pipe_open(struct usb_xfer *xfer);
+static void bce_vhci_pipe_close(struct usb_xfer *xfer);
+static void bce_vhci_pipe_enter(struct usb_xfer *xfer);
+static void bce_vhci_pipe_start(struct usb_xfer *xfer);
+
+static int bce_vhci_probe(device_t dev);
+static int bce_vhci_attach_dev(device_t dev);
+static int bce_vhci_detach_dev(device_t dev);
+
+static int bce_vhci_alloc_qid(struct bce_vhci_softc *vhci);
+static void bce_vhci_free_qid(struct bce_vhci_softc *vhci, int qid);
+static int bce_vhci_create_queues(struct bce_vhci_softc *vhci);
+static void bce_vhci_destroy_queues(struct bce_vhci_softc *vhci);
+static int bce_vhci_start_controller(struct bce_vhci_softc *vhci);
+static void bce_vhci_msg_queue_completion(struct bce_queue_sq *sq);
+static void bce_vhci_ev_cmd_completion(struct bce_queue_sq *sq);
+static void bce_vhci_ev_system_completion(struct bce_queue_sq *sq);
+static void bce_vhci_ev_generic_completion(struct bce_queue_sq *sq);
+static void bce_vhci_cmd_deliver_completion(struct bce_vhci_softc *vhci,
+ struct bce_vhci_message *msg);
+static void bce_vhci_handle_port_status_change(struct bce_vhci_softc *vhci,
+ struct bce_vhci_message *msg);
+static void bce_vhci_evt_queue_submit_pending(struct bce_vhci_softc *vhci,
+ struct bce_vhci_evt_queue *eq, uint32_t count);
+
+static int bce_vhci_device_create(struct bce_vhci_softc *vhci, uint8_t port);
+static void bce_vhci_device_destroy(struct bce_vhci_softc *vhci, uint8_t port);
+static int bce_vhci_endpoint_create(struct bce_vhci_softc *vhci,
+ struct bce_vhci_device *dev, uint8_t ep_addr,
+ struct usb_endpoint_descriptor *edesc);
+static void bce_vhci_endpoint_destroy(struct bce_vhci_softc *vhci,
+ struct bce_vhci_device *dev, uint8_t ep_addr);
+static void bce_vhci_handle_transfer_request(struct bce_vhci_softc *vhci,
+ struct bce_vhci_message *msg);
+static void bce_vhci_complete_ctrl_locked(struct bce_vhci_softc *vhci,
+ struct bce_vhci_transfer_queue *tq, struct bce_vhci_message *msg);
+static void bce_vhci_handle_ctrl_status(struct bce_vhci_softc *vhci,
+ struct bce_vhci_message *msg);
+static uint16_t bce_vhci_handle_endpoint_req_state(struct bce_vhci_softc *vhci,
+ struct bce_vhci_message *msg);
+static uint16_t bce_vhci_handle_endpoint_set_state(struct bce_vhci_softc *vhci,
+ struct bce_vhci_message *msg);
+static void bce_vhci_fwevt_task(void *arg, int pending);
+static void bce_vhci_send_fw_event_reply(struct bce_vhci_softc *vhci,
+ struct bce_vhci_message *req, uint16_t status);
+static void bce_vhci_tq_completion(struct bce_queue_sq *sq);
+static void bce_vhci_reset_task(void *arg, int pending);
+static void bce_vhci_create_task(void *arg, int pending);
+static void bce_vhci_port_chg_task(void *arg, int pending);
+static int bce_vhci_cmd_execute(struct bce_vhci_softc *vhci,
+ struct bce_vhci_message *req, struct bce_vhci_message *reply,
+ int timeout_ticks);
+
+/*
+ * Convert USB endpoint address to tq[] index.
+ * ep0 (0x00) maps to index 0. For other endpoints, IN and OUT get
+ * separate slots: OUT 0x01 -> 1, IN 0x81 -> 2, OUT 0x02 -> 3, etc.
+ * Maximum index is 30 (ep 0x8F), fits in BCE_VHCI_MAX_ENDPOINTS (32).
+ */
+static inline uint8_t
+bce_vhci_ep_index(uint8_t ep_addr)
+{
+ uint8_t num;
+
+ num = ep_addr & 0x0F;
+ if (num == 0)
+ return (0);
+ return (num * 2 - ((ep_addr & 0x80) ? 0 : 1));
+}
+
+static const struct usb_bus_methods bce_vhci_bus_methods = {
+ .roothub_exec = bce_vhci_roothub_exec,
+ .endpoint_init = bce_vhci_endpoint_init,
+ .xfer_setup = bce_vhci_xfer_setup,
+ .xfer_unsetup = bce_vhci_xfer_unsetup,
+ .get_dma_delay = bce_vhci_get_dma_delay,
+};
+
+/*
+ * Generic pipe methods (all transfer types for now).
+ */
+static const struct usb_pipe_methods bce_vhci_pipe_methods = {
+ .open = bce_vhci_pipe_open,
+ .close = bce_vhci_pipe_close,
+ .enter = bce_vhci_pipe_enter,
+ .start = bce_vhci_pipe_start,
+};
+
+/*
+ * Device methods.
+ */
+static device_method_t bce_vhci_methods[] = {
+ DEVMETHOD(device_probe, bce_vhci_probe),
+ DEVMETHOD(device_attach, bce_vhci_attach_dev),
+ DEVMETHOD(device_detach, bce_vhci_detach_dev),
+ DEVMETHOD(device_suspend, bus_generic_suspend),
+ DEVMETHOD(device_resume, bus_generic_resume),
+ DEVMETHOD(device_shutdown, bus_generic_shutdown),
+
+ /* Bus interface for usbus child */
+ DEVMETHOD(bus_print_child, bus_generic_print_child),
+ DEVMETHOD_END
+};
+
+static driver_t bce_vhci_driver = {
+ .name = "bce_vhci",
+ .methods = bce_vhci_methods,
+ .size = sizeof(struct bce_vhci_softc),
+};
+
+DRIVER_MODULE(bce_vhci, apple_bce, bce_vhci_driver, 0, 0);
+MODULE_DEPEND(bce_vhci, usb, 1, 1, 1);
+
+/*
+ * Hub descriptor (USB 2.0 hub with per-port power switching)
+ */
+
+/* Hub descriptor built dynamically in roothub_exec (port count varies) */
+
+/* Device descriptor for the root hub */
+static const struct usb_device_descriptor bce_vhci_devd = {
+ .bLength = sizeof(struct usb_device_descriptor),
+ .bDescriptorType = UDESC_DEVICE,
+ .bcdUSB = { 0x00, 0x02 }, /* USB 2.0 */
+ .bDeviceClass = UDCLASS_HUB,
+ .bDeviceSubClass = UDSUBCLASS_HUB,
+ .bDeviceProtocol = UDPROTO_HSHUBSTT,
+ .bMaxPacketSize = 64,
+ .idVendor = { 0x6b, 0x10 }, /* Apple 0x106b */
+ .idProduct = { 0x01, 0x18 }, /* T2 BCE 0x1801 */
+ .bcdDevice = { 0x00, 0x01 }, /* 1.00 */
+ .iManufacturer = 1,
+ .iProduct = 2,
+ .bNumConfigurations = 1,
+};
+
+static const struct usb_device_qualifier bce_vhci_odevd = {
+ .bLength = sizeof(struct usb_device_qualifier),
+ .bDescriptorType = UDESC_DEVICE_QUALIFIER,
+ .bcdUSB = { 0x00, 0x02 },
+ .bDeviceClass = UDCLASS_HUB,
+ .bDeviceSubClass = UDSUBCLASS_HUB,
+ .bDeviceProtocol = UDPROTO_HSHUBSTT,
+ .bMaxPacketSize0 = 0,
+ .bNumConfigurations = 0,
+};
+
+/* Configuration descriptor + interface + endpoint */
+/* 9 + 9 + 7 = 25 bytes */
+static const uint8_t bce_vhci_confd[] = {
+ /* Configuration descriptor */
+ 0x09, 0x02, /* bLength, bDescriptorType */
+ 0x19, 0x00, /* wTotalLength = 25 */
+ 0x01, 0x01, 0x00, 0xC0, 0x00, /* nIntf, cfgVal, iCfg */
+ /* Interface descriptor */
+ 0x09, 0x04, /* bLength, bDescriptorType */
+ 0x00, 0x00, 0x01, 0x09, 0x00, 0x01, 0x00,
+ /* Endpoint descriptor (interrupt IN ep1) */
+ 0x07, 0x05, /* bLength, bDescriptorType */
+ 0x81, 0x03, 0x08, 0x00, 0xFF, /* addr, attr, maxPkt, interval */
+};
+
+struct bce_vhci_dma_cb_arg {
+ bus_addr_t addr;
+ int error;
+};
+
+static void
+bce_vhci_dma_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
+{
+ struct bce_vhci_dma_cb_arg *cb = arg;
+
+ cb->error = error;
+ if (error == 0)
+ cb->addr = segs[0].ds_addr;
+}
+
+/*
+ * Allocate a CQ + SQ pair and DMA message buffer for a host->device queue.
+ * Register it with firmware under the given name.
+ */
+static int
+bce_vhci_msg_queue_create(struct bce_vhci_softc *vhci,
+ struct bce_vhci_msg_queue *mq, const char *name, int cq_qid, int sq_qid,
+ bce_sq_completion_fn compl_fn, void *compl_arg)
+{
+ struct apple_bce_softc *sc = vhci->sc_bce;
+ struct bce_vhci_dma_cb_arg cb;
+ struct bce_queue_memcfg cfg;
+ uint32_t el_count = BCE_VHCI_MSG_QUEUE_EL;
+ uint32_t status;
+ int error, i;
+
+ memset(mq, 0, sizeof(*mq));
+ mq->el_count = el_count;
+
+ /* Allocate CQ */
+ mq->cq = bce_alloc_cq(sc, cq_qid, el_count);
+ if (mq->cq == NULL)
+ return (ENOMEM);
+
+ /* Register CQ with firmware via command path */
+ bce_get_cq_memcfg(mq->cq, &cfg);
+ /* CQ interrupt vector = 4 (DMA MSI) */
+ cfg.vector_or_cq = 4;
+ status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, NULL, 0);
+ if (status != 0) {
+ device_printf(vhci->sc_dev,
+ "failed to register CQ %d for %s: %u\n",
+ cq_qid, name, status);
+ error = EIO;
+ goto fail_cq;
+ }
+
+ /* Register CQ in parent's dispatch tables */
+ mtx_lock(&sc->sc_queues_lock);
+ sc->sc_queues[cq_qid] = mq->cq;
+ for (i = 0; i < BCE_MAX_CQ_COUNT; i++) {
+ if (sc->sc_cq_list[i] == NULL) {
+ sc->sc_cq_list[i] = mq->cq;
+ break;
+ }
+ }
+ if (i == BCE_MAX_CQ_COUNT) {
+ sc->sc_queues[cq_qid] = NULL;
+ mtx_unlock(&sc->sc_queues_lock);
+ device_printf(vhci->sc_dev,
+ "CQ list full for %s\n", name);
+ error = ENOSPC;
+ goto fail_cq_reg;
+ }
+ mtx_unlock(&sc->sc_queues_lock);
+
+ /* Allocate SQ (element size = bce_qe_submission = 32 bytes) */
+ mq->sq = bce_alloc_sq(sc, sq_qid,
+ sizeof(struct bce_qe_submission), el_count,
+ compl_fn, compl_arg);
+ if (mq->sq == NULL) {
+ error = ENOMEM;
+ goto fail_cq_reg;
+ }
+
+ /* Register SQ with firmware under the given name */
+ bce_get_sq_memcfg(mq->sq, mq->cq, &cfg);
+ status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, name, 1);
+ if (status != 0) {
+ device_printf(vhci->sc_dev,
+ "failed to register SQ %d (%s): %u\n",
+ sq_qid, name, status);
+ error = EIO;
+ goto fail_sq;
+ }
+
+ /* Register SQ in parent's dispatch tables */
+ mtx_lock(&sc->sc_queues_lock);
+ sc->sc_queues[sq_qid] = mq->sq;
+ sc->sc_int_sq_list[sq_qid] = mq->sq;
+ mtx_unlock(&sc->sc_queues_lock);
+
+ /* Allocate DMA-coherent message buffer */
+ error = bus_dma_tag_create(sc->sc_dma_tag,
+ 4, 0,
+ BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
+ NULL, NULL,
+ el_count * sizeof(struct bce_vhci_message), 1,
+ el_count * sizeof(struct bce_vhci_message),
+ BUS_DMA_WAITOK,
+ NULL, NULL,
+ &mq->dma_tag);
+ if (error != 0)
+ goto fail_sq_reg;
+
+ error = bus_dmamem_alloc(mq->dma_tag, (void **)&mq->data,
+ BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT,
+ &mq->dma_map);
+ if (error != 0)
+ goto fail_dma_tag;
+
+ error = bus_dmamap_load(mq->dma_tag, mq->dma_map, mq->data,
+ el_count * sizeof(struct bce_vhci_message),
+ bce_vhci_dma_cb, &cb, BUS_DMA_WAITOK);
+ if (error != 0 || cb.error != 0) {
+ error = error != 0 ? error : cb.error;
+ goto fail_dma_mem;
+ }
+ mq->dma_addr = cb.addr;
+
+ return (0);
+
+fail_dma_mem:
+ bus_dmamem_free(mq->dma_tag, mq->data, mq->dma_map);
+fail_dma_tag:
+ bus_dma_tag_destroy(mq->dma_tag);
+fail_sq_reg:
+ bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, sq_qid);
+ mtx_lock(&sc->sc_queues_lock);
+ sc->sc_queues[sq_qid] = NULL;
+ sc->sc_int_sq_list[sq_qid] = NULL;
+ mtx_unlock(&sc->sc_queues_lock);
+fail_sq:
+ bce_free_sq(sc, mq->sq);
+ mq->sq = NULL;
+fail_cq_reg:
+ bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, cq_qid);
+ mtx_lock(&sc->sc_queues_lock);
+ sc->sc_queues[cq_qid] = NULL;
+ for (i = 0; i < BCE_MAX_CQ_COUNT; i++) {
+ if (sc->sc_cq_list[i] == mq->cq) {
+ sc->sc_cq_list[i] = NULL;
+ break;
+ }
+ }
+ mtx_unlock(&sc->sc_queues_lock);
+fail_cq:
+ bce_free_cq(sc, mq->cq);
+ mq->cq = NULL;
+ return (error);
+}
+
+static void
+bce_vhci_msg_queue_destroy(struct bce_vhci_softc *vhci,
+ struct bce_vhci_msg_queue *mq)
+{
+ struct apple_bce_softc *sc = vhci->sc_bce;
+ int i;
+
+ if (mq->cq == NULL)
+ return;
+
+ /*
+ * Unregister and free SQ before releasing the DMA buffer it
+ * references
+ */
+ if (mq->sq != NULL) {
+ bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, mq->sq->qid);
+ mtx_lock(&sc->sc_queues_lock);
+ sc->sc_queues[mq->sq->qid] = NULL;
+ sc->sc_int_sq_list[mq->sq->qid] = NULL;
+ mtx_unlock(&sc->sc_queues_lock);
+ bce_free_sq(sc, mq->sq);
+ mq->sq = NULL;
+ }
+
+ /* Free DMA message buffer */
+ if (mq->data != NULL) {
+ bus_dmamap_unload(mq->dma_tag, mq->dma_map);
+ bus_dmamem_free(mq->dma_tag, mq->data, mq->dma_map);
+ bus_dma_tag_destroy(mq->dma_tag);
+ mq->data = NULL;
+ }
+
+ /* Unregister and free CQ */
+ bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, mq->cq->qid);
+ mtx_lock(&sc->sc_queues_lock);
+ sc->sc_queues[mq->cq->qid] = NULL;
+ for (i = 0; i < BCE_MAX_CQ_COUNT; i++) {
+ if (sc->sc_cq_list[i] == mq->cq) {
+ sc->sc_cq_list[i] = NULL;
+ break;
+ }
+ }
+ mtx_unlock(&sc->sc_queues_lock);
+ bce_free_cq(sc, mq->cq);
+ mq->cq = NULL;
+}
+
+/*
+ * Write a message to a host->device queue.
+ * Caller must have reserved a submission slot.
+ */
+static void
+bce_vhci_msg_queue_write(struct bce_vhci_softc *vhci,
+ struct bce_vhci_msg_queue *mq, struct bce_vhci_message *msg)
+{
+ struct bce_qe_submission *s;
+ uint32_t sidx;
+
+ sidx = mq->sq->tail;
+ s = bce_next_submission(mq->sq);
+
+ /* Copy message into DMA buffer slot and sync for device access */
+ mq->data[sidx] = *msg;
+ bus_dmamap_sync(mq->dma_tag, mq->dma_map, BUS_DMASYNC_PREWRITE);
+
+ /* Fill SQ entry pointing to the DMA buffer slot */
+ s->length = sizeof(struct bce_vhci_message);
+ s->addr = mq->dma_addr +
+ sidx * sizeof(struct bce_vhci_message);
+ s->segl_addr = 0;
+ s->segl_length = 0;
+
+ bce_submit_to_device(vhci->sc_bce, mq->sq);
+}
+
+/*
+ * Message queue completion: consume completions and free slots.
+ */
+static void
+bce_vhci_msg_queue_completion(struct bce_queue_sq *sq)
+{
+ struct bce_vhci_msg_queue *mq = sq->userdata;
+
+ while (sq->completion_cidx != sq->completion_tail) {
+ sq->completion_cidx =
+ (sq->completion_cidx + 1) % sq->el_count;
+ bce_notify_submission_complete(sq);
+ }
+ bus_dmamap_sync(mq->dma_tag, mq->dma_map, BUS_DMASYNC_POSTWRITE);
+}
+
+/*
+ * Allocate an SQ (paired with the shared ev_cq) and DMA buffer for a
+ * device->host event queue. Register with firmware and pre-submit
+ * receive buffers.
+ */
+static int
+bce_vhci_evt_queue_create(struct bce_vhci_softc *vhci,
+ struct bce_vhci_evt_queue *eq, const char *name, int sq_qid,
+ bce_sq_completion_fn compl_fn)
+{
+ struct apple_bce_softc *sc = vhci->sc_bce;
+ struct bce_vhci_dma_cb_arg cb;
+ struct bce_queue_memcfg cfg;
+ uint32_t el_count = BCE_VHCI_EVT_QUEUE_EL;
+ uint32_t status;
+ int error;
+
+ memset(eq, 0, sizeof(*eq));
+ eq->el_count = el_count;
+ eq->userdata = vhci;
+
+ /* Allocate SQ (shared CQ = vhci->ev_cq) */
+ eq->sq = bce_alloc_sq(sc, sq_qid,
+ sizeof(struct bce_qe_submission), el_count,
+ compl_fn, eq);
+ if (eq->sq == NULL)
+ return (ENOMEM);
+
+ /* Register SQ with firmware (direction = from device = 0) */
+ bce_get_sq_memcfg(eq->sq, vhci->ev_cq, &cfg);
+ status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, name, 0);
+ if (status != 0) {
+ device_printf(vhci->sc_dev,
+ "failed to register event SQ %d (%s): %u\n",
+ sq_qid, name, status);
+ error = EIO;
+ goto fail_sq;
+ }
+
+ /* Register SQ in dispatch tables */
+ mtx_lock(&sc->sc_queues_lock);
+ sc->sc_queues[sq_qid] = eq->sq;
+ sc->sc_int_sq_list[sq_qid] = eq->sq;
+ mtx_unlock(&sc->sc_queues_lock);
+
+ /* Allocate DMA-coherent receive buffer */
+ error = bus_dma_tag_create(sc->sc_dma_tag,
+ 4, 0,
+ BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
+ NULL, NULL,
+ el_count * sizeof(struct bce_vhci_message), 1,
+ el_count * sizeof(struct bce_vhci_message),
+ BUS_DMA_WAITOK,
+ NULL, NULL,
+ &eq->dma_tag);
+ if (error != 0)
+ goto fail_sq_reg;
+
+ error = bus_dmamem_alloc(eq->dma_tag, (void **)&eq->data,
+ BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT,
+ &eq->dma_map);
+ if (error != 0)
+ goto fail_dma_tag;
+
+ error = bus_dmamap_load(eq->dma_tag, eq->dma_map, eq->data,
+ el_count * sizeof(struct bce_vhci_message),
+ bce_vhci_dma_cb, &cb, BUS_DMA_WAITOK);
+ if (error != 0 || cb.error != 0) {
+ error = error != 0 ? error : cb.error;
+ goto fail_dma_mem;
+ }
+ eq->dma_addr = cb.addr;
+
+ /* Pre-submit receive buffers */
+ bce_vhci_evt_queue_submit_pending(vhci, eq, BCE_VHCI_EVT_PENDING);
+
+ return (0);
+
+fail_dma_mem:
+ bus_dmamem_free(eq->dma_tag, eq->data, eq->dma_map);
+fail_dma_tag:
+ bus_dma_tag_destroy(eq->dma_tag);
+fail_sq_reg:
+ bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, sq_qid);
+ mtx_lock(&sc->sc_queues_lock);
+ sc->sc_queues[sq_qid] = NULL;
+ sc->sc_int_sq_list[sq_qid] = NULL;
+ mtx_unlock(&sc->sc_queues_lock);
+fail_sq:
+ bce_free_sq(sc, eq->sq);
+ eq->sq = NULL;
+ return (error);
+}
+
+static void
+bce_vhci_evt_queue_destroy(struct bce_vhci_softc *vhci,
+ struct bce_vhci_evt_queue *eq)
+{
+ struct apple_bce_softc *sc = vhci->sc_bce;
+
+ if (eq->sq == NULL)
+ return;
+
+ /* Unregister SQ from dispatch tables FIRST to stop IRQ callbacks */
+ bce_cmd_flush_queue(sc->sc_cmd_cmdq, sc, eq->sq->qid);
+ bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, eq->sq->qid);
+ mtx_lock(&sc->sc_queues_lock);
+ sc->sc_queues[eq->sq->qid] = NULL;
+ sc->sc_int_sq_list[eq->sq->qid] = NULL;
+ mtx_unlock(&sc->sc_queues_lock);
+
+ /* Now safe to free DMA buffer; no IRQ can reference it */
+ if (eq->data != NULL) {
+ bus_dmamap_unload(eq->dma_tag, eq->dma_map);
+ bus_dmamem_free(eq->dma_tag, eq->data, eq->dma_map);
+ bus_dma_tag_destroy(eq->dma_tag);
+ eq->data = NULL;
+ }
+ bce_free_sq(sc, eq->sq);
+ eq->sq = NULL;
+}
+
+/*
+ * Submit empty receive buffers to an event queue so firmware can
+ * write messages into them.
+ */
+static void
+bce_vhci_evt_queue_submit_pending(struct bce_vhci_softc *vhci,
+ struct bce_vhci_evt_queue *eq, uint32_t count)
+{
+ struct bce_qe_submission *s;
+ uint32_t idx;
+
+ bus_dmamap_sync(eq->dma_tag, eq->dma_map, BUS_DMASYNC_PREREAD);
+
+ while (count-- > 0) {
+ if (bce_reserve_submission(eq->sq) != 0) {
+ device_printf(vhci->sc_dev,
+ "cannot reserve event submission\n");
+ break;
+ }
+ idx = eq->sq->tail;
+ s = bce_next_submission(eq->sq);
+ s->length = sizeof(struct bce_vhci_message);
+ s->addr = eq->dma_addr +
+ idx * sizeof(struct bce_vhci_message);
+ s->segl_addr = 0;
+ s->segl_length = 0;
+ }
+ bce_submit_to_device(vhci->sc_bce, eq->sq);
+}
+
+/*
+ * Enqueue a firmware event into sc_fwevt_ring for deferred processing.
+ * Called from ISR context; returns 0 on success, -1 if ring is full.
+ */
+static int
+bce_vhci_fwevt_enqueue(struct bce_vhci_softc *vhci,
+ struct bce_vhci_message *msg, int needs_reply)
+{
+ uint32_t next_prod;
+
+ mtx_lock_spin(&vhci->sc_fwevt_lock);
+ next_prod = (vhci->sc_fwevt_prod + 1) % BCE_VHCI_FWEVT_RING;
+ if (next_prod == vhci->sc_fwevt_cons) {
+ mtx_unlock_spin(&vhci->sc_fwevt_lock);
+ device_printf(vhci->sc_dev,
+ "fwevt ring full, dropping 0x%04x\n", msg->cmd);
+ return (-1);
+ }
+ vhci->sc_fwevt_ring[vhci->sc_fwevt_prod].msg = *msg;
+ vhci->sc_fwevt_ring[vhci->sc_fwevt_prod].needs_reply = needs_reply;
+ vhci->sc_fwevt_prod = next_prod;
+ mtx_unlock_spin(&vhci->sc_fwevt_lock);
+
+ if (vhci->sc_detaching == 0)
+ taskqueue_enqueue(taskqueue_thread, &vhci->sc_fwevt_task);
+ return (0);
+}
+
+/*
+ * Generic event queue completion: read messages and resubmit buffers.
+ * Used for system, isochronous, interrupt, and asynchronous event queues.
+ */
+static void
+bce_vhci_ev_generic_completion(struct bce_queue_sq *sq)
+{
+ struct bce_vhci_evt_queue *eq = sq->userdata;
+ struct bce_vhci_softc *vhci = eq->userdata;
+ struct bce_vhci_message *msg;
+ uint32_t cnt = 0;
+
+ bus_dmamap_sync(eq->dma_tag, eq->dma_map, BUS_DMASYNC_POSTREAD);
+
+ while (sq->completion_cidx != sq->completion_tail) {
+ struct bce_sq_completion_data *cd;
+
+ cd = &sq->completion_data[sq->completion_cidx];
+ if (cd->status == BCE_COMP_ABORTED) {
+ sq->completion_cidx =
+ (sq->completion_cidx + 1) % sq->el_count;
+ bce_notify_submission_complete(sq);
+ cnt++;
+ continue;
+ }
+
+ msg = &eq->data[sq->head];
+ /*
+ * Route events to appropriate handlers.
+ * Strip 0x4000 flag; firmware uses it as a
+ * variant marker.
+ */
+ if (msg->cmd & BCE_VHCI_CMD_REPLY_FLAG)
+ bce_vhci_cmd_deliver_completion(vhci, msg);
+ else {
+ uint16_t base_cmd = msg->cmd &
*** 4335 LINES SKIPPED ***