git: 48ac4113c88f - main - riscv: Incoming MSI Controller support.
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Mon, 02 Nov 2026 12:21:28 UTC
The branch main has been updated by br:
URL: https://cgit.FreeBSD.org/src/commit/?id=48ac4113c88fddcea619d0bf252b87c62132f214
commit 48ac4113c88fddcea619d0bf252b87c62132f214
Author: Ruslan Bukin <br@FreeBSD.org>
AuthorDate: 2026-11-02 11:43:07 +0000
Commit: Ruslan Bukin <br@FreeBSD.org>
CommitDate: 2026-11-02 12:18:44 +0000
riscv: Incoming MSI Controller support.
Support for Incoming Message Signalling Interrupt Controller (IMSIC)
defined by the RISC-V Advanced Interrupt Architecture (AIA) specification.
This is based on early work of Himanshu Chauhan on IMSIC support for
IPI delivery. This change, however, does not include IPI delivery support,
instead it provides MSI support for peripheral devices.
IMSIC is a core-local interrupt controller with non-global interrupts,
which means that interrupt identity is effectively (hart, interrupt ID).
When IMSIC present on hart, APLIC does not trap the CPU directly,
instead it delivers the interrupt to the IMSIC instance associated with
the target hart. Upon reception of interrupt, IMSIC on hart invokes APLIC
for handling. Note that PCIe devices deliver interrupt directly to IMSIC.
Apart from IMSIC support, this includes APLIC changes:
- Obtain the interrupt trigger type from FDT and configure the interrupt
accordingly (e.g. level or edge-triggered);
- Per the AIA requirement, level-triggered MSI interrupts must be
retriggered manually after being handled. See the comment in
aplic_post_ithread();
- Reorganize APLIC register definitions to have some structure.
Reviewed by: mhorne
Sponsored by: EPSRC ("UKRI3001: CHERI Research Centre")
Differential Revision: https://reviews.freebsd.org/D59695
---
sys/conf/files.riscv | 1 +
sys/riscv/include/imsic.h | 31 ++
sys/riscv/include/intr.h | 9 +-
sys/riscv/riscv/aplic.c | 481 ++++++++++++++++++---------
sys/riscv/riscv/imsic.c | 808 ++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 1168 insertions(+), 162 deletions(-)
diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv
index c5e6e5256112..0278b354eee6 100644
--- a/sys/conf/files.riscv
+++ b/sys/conf/files.riscv
@@ -71,6 +71,7 @@ riscv/riscv/exception.S standard
riscv/riscv/exec_machdep.c standard
riscv/riscv/fpe.c standard
riscv/riscv/gdb_machdep.c optional gdb
+riscv/riscv/imsic.c standard
riscv/riscv/intc.c standard
riscv/riscv/identcpu.c standard
riscv/riscv/locore.S standard no-obj
diff --git a/sys/riscv/include/imsic.h b/sys/riscv/include/imsic.h
new file mode 100644
index 000000000000..0bf368269470
--- /dev/null
+++ b/sys/riscv/include/imsic.h
@@ -0,0 +1,31 @@
+/*-
+ * Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_IMSIC_H_
+#define _MACHINE_IMSIC_H_
+
+int imsic_alloc_aplic_msi(device_t dev, int cpu, int aplic_irq, int *msi);
+
+#endif /* !_MACHINE_IMSIC_H_ */
diff --git a/sys/riscv/include/intr.h b/sys/riscv/include/intr.h
index da5d420f7e6d..cb8dafc884a7 100644
--- a/sys/riscv/include/intr.h
+++ b/sys/riscv/include/intr.h
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2015-2026 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -36,7 +36,12 @@
#define _MACHINE_INTR_MACHDEP_H_
#ifndef NIRQ
-#define NIRQ 1024
+/*
+ * Max interrupts per spec:
+ * APLIC is 1023 total
+ * IMSIC is 2047 per core
+ */
+#define NIRQ 8192
#endif
#ifndef LOCORE
diff --git a/sys/riscv/riscv/aplic.c b/sys/riscv/riscv/aplic.c
index 6550b7ba36d1..f8718266b3cb 100644
--- a/sys/riscv/riscv/aplic.c
+++ b/sys/riscv/riscv/aplic.c
@@ -1,6 +1,7 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
+ * Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
* Copyright (c) 2024 Himanshu Chauhan <hchauhan@thechauhan.dev>
*
* Redistribution and use in source and binary forms, with or without
@@ -34,15 +35,19 @@
#include <sys/proc.h>
#include <sys/rman.h>
#include <sys/smp.h>
+#include <sys/intr.h>
#include <machine/bus.h>
#include <machine/intr.h>
#include <machine/riscvreg.h>
+#include <machine/imsic.h>
#include <dev/ofw/openfirm.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
#include "pic_if.h"
#define APLIC_MAX_IRQS 1023
@@ -61,6 +66,8 @@ static pic_bind_intr_t aplic_bind_intr;
struct aplic_irqsrc {
struct intr_irqsrc isrc;
u_int irq;
+ u_int msi;
+ u_int type;
};
struct aplic_softc {
@@ -69,125 +76,136 @@ struct aplic_softc {
struct resource *irq_res;
void *ih;
struct aplic_irqsrc isrcs[APLIC_MAX_IRQS + 1];
- unsigned int hart_indices[MAXCPU];
+ u_int hart_indices[MAXCPU];
+ device_t parent;
+ struct intr_pic *pic;
int ndev;
+ bool has_imsic;
};
-#define APLIC_DOMAIN_CFG_IE (1UL << 8) /* Enable domain IRQs */
-#define APLIC_DOMAIN_CFG_DM (1UL << 2) /* IRQ delivery mode */
-#define APLIC_DOMAIN_CFG_BE (1UL << 0) /* Endianess */
-
-#define APLIC_MODE_DIRECT 0 /* Direct delivery mode */
-#define APLIC_MODE_MSI 1 /* MSI delivery mode */
-
-#define APLIC_SRC_CFG_DLGT (1UL << 10) /* Source delegation */
-#define APLIC_SRC_CFG_SM_SHIFT 0
-#define APLIC_SRC_CFG_SM_MASK (0x7UL << APLIC_SRC_CFG_SM_SHIFT)
-
-#define APLIC_SRC_CFG_SM_INACTIVE 0 /* APLIC inactive in domain */
-#define APLIC_SRC_CFG_SM_DETACHED 1 /* Detached from source wire */
-#define APLIC_SRC_CFG_SM_EDGE_RSE 4 /* Asserted on rising edge */
-#define APLIC_SRC_CFG_SM_EDGE_FLL 5 /* Asserted on falling edge */
-#define APLIC_SRC_CFG_SM_LVL_HI 6 /* Asserted when high */
-#define APLIC_SRC_CFG_SM_LVL_LO 7 /* Asserted when low */
-
/* Register offsets in APLIC configuration space */
-#define APLIC_DOMAIN_CFG 0x0000
-#define APLIC_SRC_CFG(_idx) (0x0004 + (((_idx) - 1) * 4))
-#define APLIC_TARGET(_idx) (0x3004 + (((_idx) - 1) * 4))
-#define APLIC_MMSIADDRCFG 0x1BC0
-#define APLIC_MMSIADDRCFGH 0x1BC4
-#define APLIC_SMSIADDRCFG 0x1BC8
-#define APLIC_SMSIADDRCFGH 0x1BCC
-#define APLIC_SETIPNUM 0x1CDC
-#define APLIC_CLRIPNUM 0x1DDC
-#define APLIC_SETIENUM 0x1EDC
-#define APLIC_CLRIENUM 0x1FDC
-#define APLIC_SETIPNUM_LE 0x2000
-#define APLIC_SETIPNUM_BE 0x2004
-#define APLIC_GENMSI 0x3000
-#define APLIC_IDC_BASE 0x4000
-
-#define APLIC_SETIE_BASE 0x1E00
-#define APLIC_CLRIE_BASE 0x1F00
+#define APLIC_DOMAIN_CFG 0x0000
+#define DOMAIN_CFG_IE (1UL << 8) /* Enable domain IRQs */
+#define DOMAIN_CFG_DM (1UL << 2) /* IRQ delivery mode */
+#define DOMAIN_CFG_BE (1UL << 0) /* Endianess */
+#define APLIC_SRC_CFG(_idx) (0x0004 + (((_idx) - 1) * 4))
+#define SRC_CFG_DLGT (1UL << 10) /* Source delegation */
+#define SRC_CFG_SM_SHIFT 0
+#define SRC_CFG_SM_MASK (0x7UL << SRC_CFG_SM_SHIFT)
+#define SRC_CFG_SM_INACTIVE 0 /* APLIC inactive in domain */
+#define SRC_CFG_SM_DETACHED 1 /* Detached from source wire */
+#define SRC_CFG_SM_EDGE_RSE 4 /* Asserted on rising edge */
+#define SRC_CFG_SM_EDGE_FLL 5 /* Asserted on falling edge */
+#define SRC_CFG_SM_LVL_HI 6 /* Asserted when high */
+#define SRC_CFG_SM_LVL_LO 7 /* Asserted when low */
+#define APLIC_TARGET(_idx) (0x3004 + (((_idx) - 1) * 4))
+#define TARGET_HART_ID_S 18
+#define TARGET_HART_ID_M (0x3fff << TARGET_HART_ID_S)
+/* Direct delivery mode. */
+#define TARGET_IPRIO_S 0
+#define TARGET_IPRIO_M (0xff << TARGET_IPRIO_S)
+/* MSI delivery mode. */
+#define TARGET_GUEST_ID_S 12
+#define TARGET_GUEST_ID_M (0x3f << TARGET_GUEST_ID_S)
+#define TARGET_EIID_S 0
+#define TARGET_EIID_M (0x7ff << TARGET_EIID_S)
+#define APLIC_MMSIADDRCFG 0x1BC0
+#define APLIC_MMSIADDRCFGH 0x1BC4
+#define APLIC_SMSIADDRCFG 0x1BC8
+#define APLIC_SMSIADDRCFGH 0x1BCC
+#define APLIC_SETIPNUM 0x1CDC
+#define APLIC_CLRIPNUM 0x1DDC
+#define APLIC_SETIE_BASE 0x1E00
+#define APLIC_SETIENUM 0x1EDC
+#define APLIC_CLRIE_BASE 0x1F00
+#define APLIC_CLRIENUM 0x1FDC
+#define APLIC_SETIPNUM_LE 0x2000
+#define APLIC_SETIPNUM_BE 0x2004
+#define APLIC_GENMSI 0x3000
+#define APLIC_IDC_BASE 0x4000
/* Interrupt delivery control structure */
+#define APLIC_IDC_SIZE 0x20
#define APLIC_IDC_IDELIVERY_OFFS 0x0000
#define APLIC_IDC_IFORCE_OFFS 0x0004
#define APLIC_IDC_ITHRESHOLD_OFFS 0x0008
#define APLIC_IDC_TOPI_OFFS 0x0018
#define APLIC_IDC_CLAIMI_OFFS 0x001C
-
-#define APLIC_IDC_SZ 0x20
-
-#define APLIC_IDC_IDELIVERY_DISABLE 0
-#define APLIC_IDC_IDELIVERY_ENABLE 1
-#define APLIC_IDC_ITHRESHOLD_DISABLE 0
-
-#define APLIC_IDC_CLAIMI_PRIO_MASK 0xff
-#define APLIC_IDC_CLAIMI_IRQ_SHIFT 16
-#define APLIC_IDC_CLAIMI_IRQ_MASK 0x3ff
-
-#define APLIC_IDC_CLAIMI_IRQ(_claimi) \
- (((_claimi) >> APLIC_IDC_CLAIMI_IRQ_SHIFT) \
- & APLIC_IDC_CLAIMI_IRQ_MASK)
-
-#define APLIC_IDC_CLAIMI_PRIO(_claimi) ((_claimi) & APLIC_IDC_CLAIMI_PRIO_MASK)
-
#define APLIC_IDC_REG(_sc, _cpu, _field) \
- (APLIC_IDC_BASE + APLIC_IDC_##_field##_OFFS + \
- ((_sc->hart_indices[_cpu]) * APLIC_IDC_SZ))
-
-#define APLIC_IDC_IDELIVERY(_sc, _cpu) \
- APLIC_IDC_REG(_sc, _cpu, IDELIVERY)
-
-#define APLIC_IDC_IFORCE(_sc, _cpu) \
- APLIC_IDC_REG(_sc, _cpu, IFORCE)
-
-#define APLIC_IDC_ITHRESHOLD(_sc, _cpu) \
- APLIC_IDC_REG(_sc, _cpu, ITHRESHOLD)
-
-#define APLIC_IDC_TOPI(_sc, _cpu) \
- APLIC_IDC_REG(_sc, _cpu, TOPI)
-
-#define APLIC_IDC_CLAIMI(_sc, _cpu) \
- APLIC_IDC_REG(_sc, _cpu, CLAIMI)
-
-#define APLIC_MK_IRQ_TARGET(_sc, _cpu, _prio) \
- (_sc->hart_indices[_cpu] << 18 | ((_prio) & 0xff))
+ APLIC_IDC_BASE + APLIC_IDC_##_field##_OFFS + \
+ ((_sc->hart_indices[_cpu]) * APLIC_IDC_SIZE)
+
+#define APLIC_IDC_IDELIVERY(_sc, _cpu) APLIC_IDC_REG(_sc, _cpu, IDELIVERY)
+#define IDELIVERY_DISABLE 0
+#define IDELIVERY_ENABLE 1
+#define APLIC_IDC_IFORCE(_sc, _cpu) APLIC_IDC_REG(_sc, _cpu, IFORCE)
+#define APLIC_IDC_ITHRESHOLD(_sc, _cpu) APLIC_IDC_REG(_sc, _cpu, ITHRESHOLD)
+#define ITHRESHOLD_DISABLE 0
+#define APLIC_IDC_TOPI(_sc, _cpu) APLIC_IDC_REG(_sc, _cpu, TOPI)
+#define APLIC_IDC_CLAIMI(_sc, _cpu) APLIC_IDC_REG(_sc, _cpu, CLAIMI)
+#define CLAIMI_PRIO_MASK 0xff
+#define CLAIMI_IRQ_SHIFT 16
+#define CLAIMI_IRQ_MASK 0x3ff
+#define CLAIMI_PRIO(_r) ((_r) & CLAIMI_PRIO_MASK)
+#define CLAIMI_IRQ(_r) (((_r) >> CLAIMI_IRQ_SHIFT) & CLAIMI_IRQ_MASK)
#define aplic_read(sc, reg) bus_read_4(sc->mem_res, (reg))
#define aplic_write(sc, reg, val) bus_write_4(sc->mem_res, (reg), (val))
static u_int aplic_irq_cpu;
-static inline int
+static inline uint32_t
riscv_cpu_to_hartid(int cpu)
{
- return pcpu_find(cpu)->pc_hart;
+ return (pcpu_find(cpu)->pc_hart);
}
static inline int
-riscv_hartid_to_cpu(int hartid)
+riscv_hartid_to_cpu(uint32_t hartid)
{
int cpu;
CPU_FOREACH(cpu) {
if (riscv_cpu_to_hartid(cpu) == hartid)
- return cpu;
+ return (cpu);
}
return (-1);
}
static int
-fdt_get_hartid(device_t dev, phandle_t aplic)
+fdt_get_parent_imsic(device_t dev)
+{
+ struct aplic_softc *sc;
+ phandle_t parent_xref, node;
+ int rc;
+
+ sc = device_get_softc(dev);
+ node = ofw_bus_get_node(dev);
+
+ rc = OF_getencprop(node, "msi-parent", &parent_xref,
+ sizeof(parent_xref));
+ if (rc <= 0) {
+ device_printf(dev, "can't read parent node property\n");
+ return (ENXIO);
+ }
+
+ sc->parent = OF_device_from_xref(parent_xref);
+ if (sc->parent == NULL) {
+ device_printf(dev, "can't find parent MSI controller\n");
+ return (ENXIO);
+ }
+
+ return (0);
+}
+
+static int
+fdt_get_hartid(device_t dev, phandle_t aplic, uint32_t *hartid)
{
- int hartid;
/* Check the interrupt controller layout. */
- if (OF_searchencprop(aplic, "#interrupt-cells", &hartid,
- sizeof(hartid)) == -1) {
+ if (OF_searchencprop(aplic, "#interrupt-cells", hartid,
+ sizeof(*hartid)) == -1) {
device_printf(dev,
"Could not find #interrupt-cells for phandle %u\n", aplic);
return (-1);
@@ -197,13 +215,13 @@ fdt_get_hartid(device_t dev, phandle_t aplic)
* The parent of the interrupt-controller is the CPU we are
* interested in, so search for its hart ID.
*/
- if (OF_searchencprop(OF_parent(aplic), "reg", (pcell_t *)&hartid,
- sizeof(hartid)) == -1) {
+ if (OF_searchencprop(OF_parent(aplic), "reg", (pcell_t *)hartid,
+ sizeof(*hartid)) == -1) {
device_printf(dev, "Could not find hartid\n");
return (-1);
}
- return (hartid);
+ return (0);
}
static inline void
@@ -219,6 +237,19 @@ aplic_irq_dispatch(struct aplic_softc *sc, u_int irq, u_int prio,
device_printf(sc->dev, "Stray irq %u detected\n", irq);
}
+static int
+aplic_imsic_intr(void *arg, unsigned long irq)
+{
+ struct aplic_softc *sc;
+ struct trapframe *tf;
+
+ sc = arg;
+ tf = curthread->td_intr_frame;
+ aplic_irq_dispatch(sc, irq, 0, tf);
+
+ return (FILTER_HANDLED);
+}
+
static int
aplic_intr(void *arg)
{
@@ -233,8 +264,8 @@ aplic_intr(void *arg)
/* Claim all pending interrupts. */
while ((claimi = aplic_read(sc, APLIC_IDC_CLAIMI(sc, cpu))) != 0) {
- prio = APLIC_IDC_CLAIMI_PRIO(claimi);
- irq = APLIC_IDC_CLAIMI_IRQ(claimi);
+ prio = CLAIMI_PRIO(claimi);
+ irq = CLAIMI_IRQ(claimi);
KASSERT((irq != 0), ("Invalid IRQ 0"));
@@ -277,6 +308,8 @@ aplic_map_intr(device_t dev, struct intr_map_data *data,
{
struct intr_map_data_fdt *daf;
struct aplic_softc *sc;
+ u_int type;
+ u_int irq;
sc = device_get_softc(dev);
@@ -289,7 +322,11 @@ aplic_map_intr(device_t dev, struct intr_map_data *data,
return (EINVAL);
}
- *isrcp = &sc->isrcs[daf->cells[0]].isrc;
+ irq = daf->cells[0];
+ type = daf->cells[1];
+
+ sc->isrcs[irq].type = type;
+ *isrcp = &sc->isrcs[irq].isrc;
return (0);
}
@@ -313,16 +350,112 @@ aplic_probe(device_t dev)
*/
static int
aplic_setup_direct_mode(device_t dev)
+{
+ struct aplic_softc *sc;
+ phandle_t node;
+ pcell_t *cells;
+ uint32_t hartid;
+ int nintr, idc;
+ int cpu, i;
+ int error;
+
+ error = ENXIO;
+
+ sc = device_get_softc(dev);
+ node = ofw_bus_get_node(dev);
+
+ device_printf(dev, "Configuring APLIC in Direct mode.\n");
+ /* Set APLIC in direct mode and enable all interrupts */
+ aplic_write(sc, APLIC_DOMAIN_CFG, DOMAIN_CFG_IE);
+
+ nintr = OF_getencprop_alloc_multi(node, "interrupts-extended",
+ sizeof(uint32_t), (void **)&cells);
+ if (nintr <= 0) {
+ device_printf(dev, "Could not read interrupts-extended\n");
+ goto fail;
+ }
+
+ /* interrupts-extended is a list of phandles and interrupt types. */
+ for (i = 0, idc = 0; i < nintr; i += 2, idc++) {
+ /* Skip M-mode external interrupts */
+ if (cells[i + 1] != IRQ_EXTERNAL_SUPERVISOR)
+ continue;
+
+ /* Get the hart ID from the CLIC's phandle. */
+ error = fdt_get_hartid(dev, OF_node_from_xref(cells[i]),
+ &hartid);
+ if (error != 0) {
+ OF_prop_free(cells);
+ goto fail;
+ }
+
+ /* Get the corresponding cpuid. */
+ cpu = riscv_hartid_to_cpu(hartid);
+ if (cpu == -1) {
+ device_printf(dev, "Invalid cpu for hart %d\n", hartid);
+ OF_prop_free(cells);
+ goto fail;
+ }
+
+ sc->hart_indices[cpu] = idc;
+ }
+
+ /* Turn off the interrupt delivery for all CPUs within or out domain */
+ CPU_FOREACH(cpu) {
+ aplic_write(sc, APLIC_IDC_IDELIVERY(sc, cpu),
+ IDELIVERY_DISABLE);
+ aplic_write(sc, APLIC_IDC_ITHRESHOLD(sc, cpu),
+ ITHRESHOLD_DISABLE);
+ }
+
+ error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK,
+ aplic_intr, NULL, sc, &sc->ih);
+ if (error != 0) {
+ device_printf(dev, "Unable to setup IRQ resource\n");
+ return (ENXIO);
+ }
+
+fail:
+ return (error);
+}
+
+static int
+aplic_setup_imsic_mode(device_t dev)
+{
+ struct aplic_softc *sc;
+ int error = ENXIO;
+
+ sc = device_get_softc(dev);
+
+ device_printf(dev, "Configuring APLIC with IMSIC.\n");
+
+ if ((error = fdt_get_parent_imsic(dev)) != 0)
+ return (error);
+
+ error = intr_pic_add_handler(sc->parent, sc->pic,
+ aplic_imsic_intr, sc, 1, APLIC_MAX_IRQS);
+ if (error != 0) {
+ device_printf(dev, "Failed to install child handler, "
+ "error %d\n", error);
+ return (error);
+ }
+
+ aplic_write(sc, APLIC_DOMAIN_CFG, DOMAIN_CFG_DM | DOMAIN_CFG_IE);
+
+ return (0);
+}
+
+static int
+aplic_setup(device_t dev)
{
struct aplic_irqsrc *isrcs;
struct aplic_softc *sc;
- struct intr_pic *pic;
const char *name;
phandle_t node, xref, iparent;
- pcell_t *cells, cell;
+ pcell_t cell;
int error = ENXIO;
u_int irq;
- int cpu, hartid, rid, i, nintr, idc;
+ int rid;
device_t rootdev;
sc = device_get_softc(dev);
@@ -351,10 +484,6 @@ aplic_setup_direct_mode(device_t dev)
return (error);
}
- /* Set APLIC in direct mode and enable all interrupts */
- aplic_write(sc, APLIC_DOMAIN_CFG,
- APLIC_MODE_DIRECT | APLIC_DOMAIN_CFG_IE);
-
/* Register the interrupt sources */
isrcs = sc->isrcs;
name = device_get_nameunit(sc->dev);
@@ -366,48 +495,7 @@ aplic_setup_direct_mode(device_t dev)
if (error != 0)
goto fail;
- aplic_write(sc, APLIC_SRC_CFG(irq),
- APLIC_SRC_CFG_SM_DETACHED);
- }
-
- nintr = OF_getencprop_alloc_multi(node, "interrupts-extended",
- sizeof(uint32_t), (void **)&cells);
- if (nintr <= 0) {
- device_printf(dev, "Could not read interrupts-extended\n");
- goto fail;
- }
-
- /* interrupts-extended is a list of phandles and interrupt types. */
- for (i = 0, idc = 0; i < nintr; i += 2, idc++) {
- /* Skip M-mode external interrupts */
- if (cells[i + 1] != IRQ_EXTERNAL_SUPERVISOR)
- continue;
-
- /* Get the hart ID from the CLIC's phandle. */
- hartid = fdt_get_hartid(dev, OF_node_from_xref(cells[i]));
- if (hartid < 0) {
- OF_prop_free(cells);
- goto fail;
- }
-
- /* Get the corresponding cpuid. */
- cpu = riscv_hartid_to_cpu(hartid);
- if (cpu < 0) {
- device_printf(dev, "Invalid cpu for hart %d\n", hartid);
- OF_prop_free(cells);
- goto fail;
- }
-
- sc->hart_indices[cpu] = idc;
- }
- OF_prop_free(cells);
-
- /* Turn off the interrupt delivery for all CPUs within or out domain */
- CPU_FOREACH(cpu) {
- aplic_write(sc, APLIC_IDC_IDELIVERY(sc, cpu),
- APLIC_IDC_IDELIVERY_DISABLE);
- aplic_write(sc, APLIC_IDC_ITHRESHOLD(sc, cpu),
- APLIC_IDC_ITHRESHOLD_DISABLE);
+ aplic_write(sc, APLIC_SRC_CFG(irq), SRC_CFG_SM_DETACHED);
}
rootdev = intr_irq_root_device(INTR_ROOT_IRQ);
@@ -429,20 +517,19 @@ aplic_setup_direct_mode(device_t dev)
return (ENXIO);
}
+ if (sc->has_imsic)
+ error = aplic_setup_imsic_mode(dev);
+ else
+ error = aplic_setup_direct_mode(dev);
+ if (error != 0)
+ goto fail;
+
xref = OF_xref_from_node(node);
- pic = intr_pic_register(sc->dev, xref);
- if (pic == NULL) {
+ sc->pic = intr_pic_register(sc->dev, xref);
+ if (sc->pic == NULL) {
error = ENXIO;
goto fail;
}
-
- error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK,
- aplic_intr, NULL, sc, &sc->ih);
- if (error != 0) {
- device_printf(dev, "Unable to setup IRQ resource\n");
- return (ENXIO);
- }
-
fail:
return (error);
}
@@ -450,15 +537,15 @@ fail:
static int
aplic_attach(device_t dev)
{
+ struct aplic_softc *sc;
int rc;
- /* APLIC with IMSIC on hart is not supported */
- if (ofw_bus_has_prop(dev, "msi-parent")) {
- device_printf(dev, "APLIC with IMSIC is unsupported\n");
- return (ENXIO);
- }
+ sc = device_get_softc(dev);
- rc = aplic_setup_direct_mode(dev);
+ if (ofw_bus_has_prop(dev, "msi-parent"))
+ sc->has_imsic = true;
+
+ rc = aplic_setup(dev);
return (rc);
}
@@ -466,13 +553,44 @@ aplic_attach(device_t dev)
static void
aplic_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
{
+
aplic_disable_intr(dev, isrc);
}
static void
aplic_post_ithread(device_t dev, struct intr_irqsrc *isrc)
{
+ struct aplic_softc *sc;
+ struct aplic_irqsrc *src;
+
+ sc = device_get_softc(dev);
+ src = (struct aplic_irqsrc *)isrc;
+
aplic_enable_intr(dev, isrc);
+
+ if (!src->msi)
+ return;
+
+ /*
+ * The section "4.9.2 Special consideration for level-sensitive
+ * interrupt sources" of the RISC-V AIA specification says:
+ *
+ * A second option is for the interrupt service routine to write the
+ * APLIC’s source identity number for the interrupt to the domain’s
+ * setipnum register just before exiting. This will cause the
+ * interrupt’s pending bit to be set to one again if the source is
+ * still asserting an interrupt, but not if the source is not asserting
+ * an interrupt.
+ */
+
+ switch (src->type) {
+ case IRQ_TYPE_LEVEL_HIGH:
+ case IRQ_TYPE_LEVEL_LOW:
+ aplic_write(sc, APLIC_SETIPNUM_LE, src->irq);
+ break;
+ default:
+ break;
+ }
}
static int
@@ -481,13 +599,38 @@ aplic_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res,
{
struct aplic_irqsrc *src;
struct aplic_softc *sc;
+ uint32_t reg;
CPU_ZERO(&isrc->isrc_cpu);
sc = device_get_softc(dev);
src = (struct aplic_irqsrc *)isrc;
- aplic_write(sc, APLIC_SRC_CFG(src->irq), APLIC_SRC_CFG_SM_EDGE_RSE);
+ if (bootverbose)
+ device_printf(dev, "configuring irq %d type %x\n", src->irq,
+ src->type);
+
+ switch (src->type) {
+ case IRQ_TYPE_LEVEL_HIGH:
+ reg = SRC_CFG_SM_LVL_HI;
+ break;
+ case IRQ_TYPE_LEVEL_LOW:
+ reg = SRC_CFG_SM_LVL_LO;
+ break;
+ case IRQ_TYPE_EDGE_RISING:
+ reg = SRC_CFG_SM_EDGE_RSE;
+ break;
+ case IRQ_TYPE_EDGE_FALLING:
+ reg = SRC_CFG_SM_EDGE_FLL;
+ break;
+ case IRQ_TYPE_EDGE_BOTH:
+ reg = SRC_CFG_SM_EDGE_FLL | SRC_CFG_SM_EDGE_RSE;
+ break;
+ default:
+ panic("unknown level");
+ };
+
+ aplic_write(sc, APLIC_SRC_CFG(src->irq), reg);
/*
* In uniprocessor system, bind_intr will not be called.
@@ -506,6 +649,7 @@ aplic_bind_intr(device_t dev, struct intr_irqsrc *isrc)
struct aplic_softc *sc;
struct aplic_irqsrc *src;
uint32_t cpu, hartid;
+ uint32_t reg;
sc = device_get_softc(dev);
src = (struct aplic_irqsrc *)isrc;
@@ -517,9 +661,8 @@ aplic_bind_intr(device_t dev, struct intr_irqsrc *isrc)
cpu = aplic_irq_cpu = intr_irq_next_cpu(aplic_irq_cpu,
&all_cpus);
CPU_SETOF(cpu, &isrc->isrc_cpu);
- } else {
+ } else
cpu = CPU_FFS(&isrc->isrc_cpu) - 1;
- }
hartid = riscv_cpu_to_hartid(cpu);
@@ -527,10 +670,28 @@ aplic_bind_intr(device_t dev, struct intr_irqsrc *isrc)
device_printf(dev, "Bind irq %d to cpu%d (hart %d)\n", src->irq,
cpu, hartid);
- aplic_write(sc, APLIC_TARGET(src->irq),
- APLIC_MK_IRQ_TARGET(sc, cpu, APLIC_INTR_DEF_PRIO));
- aplic_write(sc, APLIC_IDC_IDELIVERY(sc, cpu),
- APLIC_IDC_IDELIVERY_ENABLE);
+ if (sc->has_imsic) {
+ if (imsic_alloc_aplic_msi(sc->parent, cpu, src->irq,
+ &src->msi)) {
+ device_printf(dev, "MSI allocation for irq %d failed\n",
+ src->irq);
+ return (ENXIO);
+ }
+
+ if (bootverbose)
+ device_printf(dev, "Setting up IRQ %d on CPU %d "
+ "MSI ID %d\n", src->irq, cpu, src->msi);
+
+ reg = hartid << TARGET_HART_ID_S;
+ reg |= src->msi << TARGET_EIID_S;
+ aplic_write(sc, APLIC_TARGET(src->irq), reg);
+ } else {
+ reg = sc->hart_indices[cpu] << TARGET_HART_ID_S;
+ reg |= (APLIC_INTR_DEF_PRIO << TARGET_IPRIO_S) & TARGET_IPRIO_M;
+ aplic_write(sc, APLIC_TARGET(src->irq), reg);
+ aplic_write(sc, APLIC_IDC_IDELIVERY(sc, cpu), IDELIVERY_ENABLE);
+ }
+
aplic_enable_intr(dev, isrc);
return (0);
diff --git a/sys/riscv/riscv/imsic.c b/sys/riscv/riscv/imsic.c
new file mode 100644
index 000000000000..072c8572cd2e
--- /dev/null
+++ b/sys/riscv/riscv/imsic.c
@@ -0,0 +1,808 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2024 Himanshu Chauhan <hchauhan@thechauhan.dev>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/module.h>
+#include <sys/proc.h>
+#include <sys/rman.h>
+#include <sys/smp.h>
+#include <sys/mutex.h>
+#include <sys/intr.h>
+
+#include <machine/bus.h>
+#include <machine/intr.h>
+#include <machine/imsic.h>
+#include <machine/smp.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include "pic_if.h"
+#include "msi_if.h"
+
+#define IMSIC_MSI_PPN_SHIFT 12 /* Number of bits to shift for PPN. */
+#define IMSIC_PAGE_SHIFT 12 /* Interrupt file page shift. */
+
+#define IMSIC_EIDELIVERY 0x70
+#define EIDELIVERY_EN (1 << 0)
+#define IMSIC_EITHRESHOLD 0x72
+#define ITHRESHOLD_DIS 0
+#define IMSIC_EIP(n) (0x80 + ((n / 64) * 2))
+#define IMSIC_EIE(n) (0xC0 + ((n / 64) * 2))
+
+#define IMSIC_MAX_MSI 2048 /* Max MSI ID per core (per spec). */
+BITSET_DEFINE(msiset, IMSIC_MAX_MSI);
+#define IMSIC_FILL(p) BIT_FILL(IMSIC_MAX_MSI, p)
+#define IMSIC_ISEMPTY(p) BIT_EMPTY(IMSIC_MAX_MSI, p)
+#define IMSIC_FFS(p) BIT_FFS(IMSIC_MAX_MSI, p)
+#define IMSIC_SET(n, p) BIT_SET(IMSIC_MAX_MSI, n, p)
+#define IMSIC_CLR(n, p) BIT_CLR(IMSIC_MAX_MSI, n, p)
+#define IMSIC_LOCK(_sc, _cpu) mtx_lock_spin(&(_sc)->pcpu_data[_cpu].lock)
+#define IMSIC_UNLOCK(_sc, _cpu) mtx_unlock_spin(&(_sc)->pcpu_data[_cpu].lock)
+
+#define ISRC_TO_IRQ(_isrc) ((struct imsic_irq *)(_isrc))
+
+struct imsic_msi_config {
+ u_int lhxs; /* Low hart index shift */
+ u_int lhxw; /* Low hart index width */
+ u_int hhxs; /* High hart index shift */
+ u_int hhxw; /* High hart index width */
+ vm_paddr_t msi_base; /* Base address for PPN */
+ unsigned long msi_sz; /* Total MSI region size for all harts */
+};
+
+#ifdef SMP
+struct smp_action {
+ int irq;
+};
+#endif
+
+struct imsic_irq {
+ struct intr_irqsrc isrc;
+ int cpuid;
+ int msi;
+};
+
+struct imsic_pcpu {
+ struct mtx lock;
+ struct msiset allocated_msis;
+ struct imsic_irq *irqs[IMSIC_MAX_MSI];
+ uint16_t aplic_irq[IMSIC_MAX_MSI];
+};
+
+struct imsic_softc {
+ device_t dev;
+ struct intr_pic *pic;
+ struct resource *mem_res;
+ struct resource *irq_res;
+ struct imsic_pcpu *pcpu_data;
+ struct imsic_msi_config msi_config;
+ cpuset_t target_cpus;
+ int nr_parent_irqs;
+ int ndev;
+};
+
+static inline void
+imsic_csr_write(int reg, uint64_t val)
+{
+
+ csr_write(siselect, reg);
+ csr_write(sireg, val);
+}
+
+static inline uint64_t
+imsic_csr_read(int reg)
+{
+ uint64_t val;
+
+ csr_write(siselect, reg);
+ val = csr_read(sireg);
+
+ return (val);
+}
+
+static inline uint32_t
+riscv_cpu_to_hartid(int cpu)
+{
+
+ return (pcpu_find(cpu)->pc_hart);
+}
+
+static inline int
+riscv_hartid_to_cpu(uint32_t hartid)
+{
+ int cpu;
+
+ CPU_FOREACH(cpu) {
+ if (riscv_cpu_to_hartid(cpu) == hartid)
+ return (cpu);
+ }
+
+ return (-1);
+}
+
+static int
+fdt_get_hartid(device_t dev, phandle_t imsic, uint32_t *hartid)
+{
+
+ /* Check the interrupt controller layout. */
+ if (OF_searchencprop(imsic, "#interrupt-cells", hartid,
+ sizeof(*hartid)) == -1) {
+ device_printf(dev,
+ "Could not find #interrupt-cells for phandle %u\n", imsic);
+ return (-1);
+ }
+
*** 643 LINES SKIPPED ***