svn commit: r295972 - in head/sys: boot/fdt/dts/riscv conf riscv/conf riscv/htif riscv/include riscv/riscv

Ruslan Bukin br at FreeBSD.org
Wed Feb 24 16:50:37 UTC 2016


Author: br
Date: Wed Feb 24 16:50:34 2016
New Revision: 295972
URL: https://svnweb.freebsd.org/changeset/base/295972

Log:
  Add support for symmetric multiprocessing (SMP).
  
  Tested on Spike simulator with 2 and 16 cores (tlb enabled),
  so set MAXCPU to 16 at this time.
  
  This uses FDT data to get information about CPUs
  (code based on arm64 mp_machdep).
  
  Invalidate entire TLB cache as it is the only way yet.
  
  Sponsored by:	DARPA, AFRL
  Sponsored by:	HEIF5

Added:
  head/sys/riscv/riscv/mp_machdep.c   (contents, props changed)
Modified:
  head/sys/boot/fdt/dts/riscv/spike.dts
  head/sys/conf/files.riscv
  head/sys/riscv/conf/GENERIC
  head/sys/riscv/htif/htif.c
  head/sys/riscv/htif/htif_block.c
  head/sys/riscv/htif/htif_console.c
  head/sys/riscv/include/intr.h
  head/sys/riscv/include/param.h
  head/sys/riscv/include/pcpu.h
  head/sys/riscv/include/riscvreg.h
  head/sys/riscv/include/smp.h
  head/sys/riscv/riscv/cpufunc_asm.S
  head/sys/riscv/riscv/exception.S
  head/sys/riscv/riscv/genassym.c
  head/sys/riscv/riscv/intr_machdep.c
  head/sys/riscv/riscv/locore.S
  head/sys/riscv/riscv/machdep.c
  head/sys/riscv/riscv/pmap.c
  head/sys/riscv/riscv/swtch.S

Modified: head/sys/boot/fdt/dts/riscv/spike.dts
==============================================================================
--- head/sys/boot/fdt/dts/riscv/spike.dts	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/boot/fdt/dts/riscv/spike.dts	Wed Feb 24 16:50:34 2016	(r295972)
@@ -43,6 +43,23 @@
 	#size-cells = <1>;
 	#interrupt-cells = <1>;
 
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu at 0 {
+			device_type = "cpu";
+			compatible = "riscv,rv64i";
+			reg = <0x40002000>;
+		};
+
+		cpu at 1 {
+			device_type = "cpu";
+			compatible = "riscv,rv64i";
+			reg = <0x4000a000>;
+		};
+	};
+
 	aliases {
 		console0 = &console0;
 	};

Modified: head/sys/conf/files.riscv
==============================================================================
--- head/sys/conf/files.riscv	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/conf/files.riscv	Wed Feb 24 16:50:34 2016	(r295972)
@@ -1,6 +1,7 @@
 # $FreeBSD$
 crypto/blowfish/bf_enc.c	optional	crypto | ipsec
 crypto/des/des_enc.c		optional	crypto | ipsec | netsmb
+dev/ofw/ofw_cpu.c		optional	fdt
 kern/kern_clocksource.c		standard
 kern/subr_dummy_vdso_tc.c	standard
 libkern/bcmp.c			standard
@@ -29,8 +30,9 @@ riscv/riscv/intr_machdep.c	standard
 riscv/riscv/in_cksum.c		optional	inet | inet6
 riscv/riscv/identcpu.c		standard
 riscv/riscv/locore.S		standard	no-obj
-riscv/riscv/minidump_machdep.c	standard
 riscv/riscv/machdep.c		standard
+riscv/riscv/minidump_machdep.c	standard
+riscv/riscv/mp_machdep.c	optional	smp
 riscv/riscv/mem.c		standard
 riscv/riscv/nexus.c		standard
 riscv/riscv/pmap.c		standard

Modified: head/sys/riscv/conf/GENERIC
==============================================================================
--- head/sys/riscv/conf/GENERIC	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/riscv/conf/GENERIC	Wed Feb 24 16:50:34 2016	(r295972)
@@ -70,7 +70,13 @@ options 	MAC			# TrustedBSD MAC Framewor
 options 	RACCT			# Resource accounting framework
 options 	RACCT_DEFAULT_TO_DISABLED # Set kern.racct.enable=0 by default
 options 	RCTL			# Resource limits
-# options 	SMP
+options 	SMP
+
+# Uncomment for memory disk
+# options 	MD_ROOT
+# options 	MD_ROOT_SIZE=8192	# 8MB ram disk
+# makeoptions	MFS_IMAGE=/path/to/img
+# options 	ROOTDEVNAME=\"ufs:/dev/md0\"
 
 # Debugging support.  Always need this:
 # options 	KDB			# Enable kernel debugger support.

Modified: head/sys/riscv/htif/htif.c
==============================================================================
--- head/sys/riscv/htif/htif.c	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/riscv/htif/htif.c	Wed Feb 24 16:50:34 2016	(r295972)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015 Ruslan Bukin <br at bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br at bsdpad.com>
  * All rights reserved.
  *
  * Portions of this software were developed by SRI International and the
@@ -62,7 +62,7 @@ __FBSDID("$FreeBSD$");
 #include "htif.h"
 
 static struct resource_spec htif_spec[] = {
-	{ SYS_RES_IRQ,		0,	RF_ACTIVE },
+	{ SYS_RES_IRQ,		0,	RF_ACTIVE | RF_SHAREABLE},
 	{ -1, 0 }
 };
 
@@ -126,9 +126,9 @@ htif_intr(void *arg)
 
 	sc = arg;
 
-	htif_handle_entry(sc);
+	csr_clear(sip, SIP_SSIP);
 
-	csr_clear(sip, SIE_SSIE);
+	htif_handle_entry(sc);
 
 	return (FILTER_HANDLED);
 }

Modified: head/sys/riscv/htif/htif_block.c
==============================================================================
--- head/sys/riscv/htif/htif_block.c	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/riscv/htif/htif_block.c	Wed Feb 24 16:50:34 2016	(r295972)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015 Ruslan Bukin <br at bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br at bsdpad.com>
  * All rights reserved.
  *
  * Portions of this software were developed by SRI International and the
@@ -117,6 +117,9 @@ htif_blk_intr(void *arg, uint64_t entry)
 	if (sc->curtag == data) {
 		sc->cmd_done = 1;
 		wakeup(&sc->intr_chan);
+	} else {
+		device_printf(sc->dev, "Unexpected tag %d (should be %d)\n",
+		    data, sc->curtag);
 	}
 }
 
@@ -212,6 +215,8 @@ htif_blk_task(void *arg)
 		HTIF_BLK_UNLOCK(sc);
 
 		if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
+			HTIF_BLK_LOCK(sc);
+
 			req.offset = (bp->bio_pblkno * sc->disk->d_sectorsize);
 			req.size = bp->bio_bcount;
 			paddr = vtophys(bp->bio_data);
@@ -233,7 +238,6 @@ htif_blk_task(void *arg)
 			htif_command(cmd);
 
 			/* Wait for interrupt */
-			HTIF_BLK_LOCK(sc);
 			i = 0;
 			while (sc->cmd_done == 0) {
 				msleep(&sc->intr_chan, &sc->sc_mtx, PRIBIO, "intr", hz/2);

Modified: head/sys/riscv/htif/htif_console.c
==============================================================================
--- head/sys/riscv/htif/htif_console.c	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/riscv/htif/htif_console.c	Wed Feb 24 16:50:34 2016	(r295972)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015 Ruslan Bukin <br at bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br at bsdpad.com>
  * All rights reserved.
  *
  * Portions of this software were developed by SRI International and the
@@ -89,6 +89,7 @@ CONSOLE_DRIVER(riscv);
 #define	MAX_BURST_LEN		1
 #define	QUEUE_SIZE		256
 #define	CONSOLE_DEFAULT_ID	1ul
+#define	SPIN_IN_MACHINE_MODE	1
 
 struct queue_entry {
 	uint64_t data;
@@ -109,7 +110,12 @@ htif_putc(int c)
 	cmd |= (CONSOLE_DEFAULT_ID << HTIF_DEV_ID_SHIFT);
 	cmd |= c;
 
+#ifdef SPIN_IN_MACHINE_MODE
+	machine_command(ECALL_HTIF_LOWPUTC, cmd);
+#else
 	htif_command(cmd);
+#endif
+
 }
 
 static uint8_t
@@ -141,6 +147,7 @@ riscv_putc(int c)
 
 	htif_putc(c);
 
+#ifndef SPIN_IN_MACHINE_MODE
 	/* Wait for an interrupt */
 	__asm __volatile(
 		"li	%0, 1\n"	/* counter = 1 */
@@ -153,6 +160,7 @@ riscv_putc(int c)
 	"2:"
 		: "=&r"(counter), "=&r"(val) : "r"(cc)
 	);
+#endif
 }
 
 #ifdef EARLY_PRINTF

Modified: head/sys/riscv/include/intr.h
==============================================================================
--- head/sys/riscv/include/intr.h	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/riscv/include/intr.h	Wed Feb 24 16:50:34 2016	(r295972)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015 Ruslan Bukin <br at bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br at bsdpad.com>
  * All rights reserved.
  *
  * Portions of this software were developed by SRI International and the
@@ -51,4 +51,16 @@ typedef unsigned long * riscv_intrcnt_t;
 riscv_intrcnt_t riscv_intrcnt_create(const char *);
 void riscv_intrcnt_setname(riscv_intrcnt_t, const char *);
 
+#ifdef SMP
+void riscv_setup_ipihandler(driver_filter_t *);
+void riscv_unmask_ipi(void);
+#endif
+
+enum {
+	IRQ_SOFTWARE,
+	IRQ_TIMER,
+	IRQ_HTIF,
+	NIRQS
+};
+
 #endif /* !_MACHINE_INTR_MACHDEP_H_ */

Modified: head/sys/riscv/include/param.h
==============================================================================
--- head/sys/riscv/include/param.h	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/riscv/include/param.h	Wed Feb 24 16:50:34 2016	(r295972)
@@ -51,7 +51,7 @@
 
 #if defined(SMP) || defined(KLD_MODULE)
 #ifndef MAXCPU
-#define	MAXCPU		2
+#define	MAXCPU		16
 #endif
 #else
 #define	MAXCPU		1

Modified: head/sys/riscv/include/pcpu.h
==============================================================================
--- head/sys/riscv/include/pcpu.h	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/riscv/include/pcpu.h	Wed Feb 24 16:50:34 2016	(r295972)
@@ -1,7 +1,16 @@
 /*-
  * Copyright (c) 1999 Luoqi Chen <luoqi at freebsd.org>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br at bsdpad.com>
  * All rights reserved.
  *
+ * Portions of this software were developed by SRI International and the
+ * University of Cambridge Computer Laboratory under DARPA/AFRL contract
+ * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Portions of this software were developed by the University of Cambridge
+ * Computer Laboratory as part of the CTSRD Project, with support from the
+ * UK Higher Education Innovation Fund (HEIF).
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -36,7 +45,10 @@
 #define	ALT_STACK_SIZE	128
 
 #define	PCPU_MD_FIELDS							\
-	char __pad[129]
+	uint32_t pc_pending_ipis;	/* IPIs pending to this CPU */	\
+	uint64_t pc_sptbr;		/* L0 page table base (VA) */	\
+	uint64_t pc_reg;		/* CPU MMIO base (PA) */	\
+	char __pad[109]
 
 #ifdef _KERNEL
 

Modified: head/sys/riscv/include/riscvreg.h
==============================================================================
--- head/sys/riscv/include/riscvreg.h	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/riscv/include/riscvreg.h	Wed Feb 24 16:50:34 2016	(r295972)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015 Ruslan Bukin <br at bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br at bsdpad.com>
  * All rights reserved.
  *
  * Portions of this software were developed by SRI International and the
@@ -38,12 +38,16 @@
 #define	_MACHINE_RISCVREG_H_
 
 /* Machine mode requests */
-#define	ECALL_MTIMECMP		0x1
-#define	ECALL_CLEAR_PENDING	0x2
-#define	ECALL_HTIF_CMD		0x3
-#define	ECALL_HTIF_GET_ENTRY	0x4
-#define	ECALL_MCPUID_GET	0x5
-#define	ECALL_MIMPID_GET	0x6
+#define	ECALL_MTIMECMP		0x01
+#define	ECALL_CLEAR_PENDING	0x02
+#define	ECALL_HTIF_CMD		0x03
+#define	ECALL_HTIF_GET_ENTRY	0x04
+#define	ECALL_MCPUID_GET	0x05
+#define	ECALL_MIMPID_GET	0x06
+#define	ECALL_SEND_IPI		0x07
+#define	ECALL_CLEAR_IPI		0x08
+#define	ECALL_HTIF_LOWPUTC	0x09
+#define	ECALL_MIE_SET		0x10
 
 #define	EXCP_SHIFT			0
 #define	EXCP_MASK			(0xf << EXCP_SHIFT)
@@ -109,9 +113,14 @@
 #define	SIE_SSIE	(1 << 1)
 #define	SIE_STIE	(1 << 5)
 
-/* Note: sip register is not yet implement in Spike simulator */
+/* Note: sip register has no SIP_STIP bit in Spike simulator */
+#define	SIP_SSIP	(1 << 1)
 #define	SIP_STIP	(1 << 5)
 
+#define	NCSRS	4096
+#define	CSR_IPI	0x783
+#define	XLEN	8
+
 #define	CSR_ZIMM(val)							\
 	(__builtin_constant_p(val) && ((u_long)(val) < 32))
 

Modified: head/sys/riscv/include/smp.h
==============================================================================
--- head/sys/riscv/include/smp.h	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/riscv/include/smp.h	Wed Feb 24 16:50:34 2016	(r295972)
@@ -1 +1,55 @@
-/* $FreeBSD$ */
+/*-
+ * Copyright (c) 2016 Ruslan Bukin <br at bsdpad.com>
+ * All rights reserved.
+ *
+ * Portions of this software were developed by SRI International and the
+ * University of Cambridge Computer Laboratory under DARPA/AFRL contract
+ * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Portions of this software were developed by the University of Cambridge
+ * Computer Laboratory as part of the CTSRD Project, with support from the
+ * UK Higher Education Innovation Fund (HEIF).
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_SMP_H_
+#define	_MACHINE_SMP_H_
+
+#include <machine/pcb.h>
+
+#define	IPI_AST		(1 << 0)
+#define	IPI_PREEMPT	(1 << 1)
+#define	IPI_RENDEZVOUS	(1 << 2)
+#define	IPI_STOP	(1 << 3)
+#define	IPI_STOP_HARD	(1 << 4)
+#define	IPI_HARDCLOCK	(1 << 5)
+
+void ipi_all_but_self(u_int ipi);
+void ipi_cpu(int cpu, u_int ipi);
+void ipi_selected(cpuset_t cpus, u_int ipi);
+
+extern struct pcb stoppcbs[];
+
+#endif /* !_MACHINE_SMP_H_ */

Modified: head/sys/riscv/riscv/cpufunc_asm.S
==============================================================================
--- head/sys/riscv/riscv/cpufunc_asm.S	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/riscv/riscv/cpufunc_asm.S	Wed Feb 24 16:50:34 2016	(r295972)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015 Ruslan Bukin <br at bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br at bsdpad.com>
  * All rights reserved.
  *
  * Portions of this software were developed by SRI International and the
@@ -64,7 +64,7 @@ END(riscv_tlb_flushID_SE)
  * void riscv_dcache_wb_range(vm_offset_t, vm_size_t)
  */
 ENTRY(riscv_dcache_wb_range)
-	/* RISCVTODO */
+	sfence.vm
 	ret
 END(riscv_dcache_wb_range)
 
@@ -72,7 +72,7 @@ END(riscv_dcache_wb_range)
  * void riscv_dcache_wbinv_range(vm_offset_t, vm_size_t)
  */
 ENTRY(riscv_dcache_wbinv_range)
-	/* RISCVTODO */
+	sfence.vm
 	ret
 END(riscv_dcache_wbinv_range)
 
@@ -80,7 +80,7 @@ END(riscv_dcache_wbinv_range)
  * void riscv_dcache_inv_range(vm_offset_t, vm_size_t)
  */
 ENTRY(riscv_dcache_inv_range)
-	/* RISCVTODO */
+	sfence.vm
 	ret
 END(riscv_dcache_inv_range)
 
@@ -88,7 +88,8 @@ END(riscv_dcache_inv_range)
  * void riscv_idcache_wbinv_range(vm_offset_t, vm_size_t)
  */
 ENTRY(riscv_idcache_wbinv_range)
-	/* RISCVTODO */
+	fence.i
+	sfence.vm
 	ret
 END(riscv_idcache_wbinv_range)
 
@@ -96,6 +97,6 @@ END(riscv_idcache_wbinv_range)
  * void riscv_icache_sync_range(vm_offset_t, vm_size_t)
  */
 ENTRY(riscv_icache_sync_range)
-	/* RISCVTODO */
+	fence.i
 	ret
 END(riscv_icache_sync_range)

Modified: head/sys/riscv/riscv/exception.S
==============================================================================
--- head/sys/riscv/riscv/exception.S	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/riscv/riscv/exception.S	Wed Feb 24 16:50:34 2016	(r295972)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015 Ruslan Bukin <br at bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br at bsdpad.com>
  * All rights reserved.
  *
  * Portions of this software were developed by SRI International and the
@@ -229,6 +229,7 @@ bad_trap:
 	j bad_trap
 
 user_trap:
+	/* Save state */
 	csrrw	sp, mscratch, sp
 	addi	sp, sp, -64
 	sd	t0, (8 * 0)(sp)
@@ -282,7 +283,31 @@ machine_interrupt:
 	j	1b
 
 software_interrupt:
-	/* Redirect to supervisor */
+	li	t0, MIP_MSIP
+	csrc	mip, t0
+	li	t0, MIP_SSIP
+	csrs	mip, t0
+
+	/* If PRV1 is PRV_U (user) then serve the trap */
+	csrr	t0, mstatus
+	li	t1, (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT)
+	and	t0, t0, t1
+	beqz	t0, 1f
+
+	/*
+	 * If PRV1 is supervisor and interrupts were enabled,
+	 * then serve the trap.
+	 */
+	csrr	t0, mstatus
+	li	t1, (SR_IE1 | (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT))
+	and	t0, t0, t1
+	li	t1, (SR_IE1 | (MSTATUS_PRV_S << MSTATUS_PRV1_SHIFT))
+	beq	t0, t1, 1f
+
+	j	exit
+
+1:
+	/* Serve a trap in supervisor mode */
 	j	exit_mrts
 
 timer_interrupt:
@@ -294,17 +319,20 @@ timer_interrupt:
 	li	t0, MIP_MTIP
 	csrc	mip, t0
 
-	/* Post supervisor software interrupt */
+	/* Post supervisor timer interrupt */
 	li	t0, MIP_STIP
 	csrs	mip, t0
 
-	/* If PRV1 is PRV_U (user) then serve a trap */
+	/* If PRV1 is PRV_U (user) then serve the trap */
 	csrr	t0, mstatus
 	li	t1, (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT)
 	and	t0, t0, t1
 	beqz	t0, 1f
 
-	/* If PRV1 is supervisor and interrupts were enabled, then serve a trap */
+	/*
+	 * If PRV1 is supervisor and interrupts were enabled,
+	 * then serve the trap.
+	 */
 	csrr	t0, mstatus
 	li	t1, (SR_IE1 | (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT))
 	and	t0, t0, t1
@@ -332,25 +360,35 @@ htif_interrupt:
 	la	t0, console_intr
 	li	t1, 1
 	sd	t1, 0(t0)
-	j	3f
+
+	/* Check if there is any other pending event */
+	j	1b
 
 2:
 	/* Save entry */
-	la	t0, htif_ring_cursor
-	beqz	t0, 3f		/* not initialized */
-	ld	t0, 0(t0)	/* load struct */
-	sd	t5, 0(t0)	/* put entry */
+	la	t0, htif_ring
+	csrr	t1, mhartid
+	li	t4, (HTIF_RING_SIZE + 16)
+	mulw	t4, t4, t1
+	add	t0, t0, t4
+	li	t4, (HTIF_RING_SIZE)
+	add	t0, t0, t4	/* t0 == htif_ring_cursor */
+
+	ld	t1, 0(t0)	/* load ptr to cursor */
+	sd	t5, 0(t1)	/* put entry */
 	li	t4, 1
-	sd	t4, 8(t0)	/* mark used */
-	ld	t4, 16(t0)	/* take next */
+	sd	t4, 8(t1)	/* mark used */
+	ld	t4, 16(t1)	/* take next */
 	/* Update cursor */
-	la	t0, htif_ring_cursor
 	sd	t4, 0(t0)
 
 	/* Post supervisor software interrupt */
 	li	t0, MIP_SSIP
 	csrs	mip, t0
 
+	/* Check if there is any other pending event */
+	j	1b
+
 3:
 	j	exit
 
@@ -369,6 +407,18 @@ supervisor_call:
 	beq	t5, t4, mcpuid_get
 	li	t4, ECALL_MIMPID_GET
 	beq	t5, t4, mimpid_get
+	li	t4, ECALL_SEND_IPI
+	beq	t5, t4, send_ipi
+	li	t4, ECALL_CLEAR_IPI
+	beq	t5, t4, clear_ipi
+	li	t4, ECALL_HTIF_LOWPUTC
+	beq	t5, t4, htif_lowputc
+	li	t4, ECALL_MIE_SET
+	beq	t5, t4, mie_set
+	j	exit_next_instr
+
+mie_set:
+	csrs	mie, t6
 	j	exit_next_instr
 
 mcpuid_get:
@@ -379,28 +429,111 @@ mimpid_get:
 	csrr	t6, mimpid
 	j	exit_next_instr
 
+send_ipi:
+	/* CPU mmio base in t6 */
+	mv	t0, t6
+	li	t2, (CSR_IPI * XLEN)
+	add	t0, t0, t2	/* t0 = CSR_IPI */
+	li	t2, 1
+	sd	t2, 0(t0)
+	j	exit_next_instr
+
+clear_ipi:
+	/* Do only clear if there are no new entries in HTIF ring */
+	la	t0, htif_ring
+	csrr	t2, mhartid
+	li	t4, (HTIF_RING_SIZE + 16)
+	mulw	t4, t4, t2
+	add	t0, t0, t4
+	li	t4, (HTIF_RING_SIZE)
+	add	t0, t0, t4	/* t0  == ptr to htif_ring_cursor */
+	ld	t2, 8(t0)	/* load htif_ring_last */
+	ld	t2, 8(t2)	/* load used */
+	bnez	t2, 1f
+
+	/* Clear supervisor software interrupt pending bit */
+	li	t0, MIP_SSIP
+	csrc	mip, t0
+
+1:
+	j	exit_next_instr
+
 htif_get_entry:
+	/* Get a htif_ring for current core */
+	la	t0, htif_ring
+	csrr	t2, mhartid
+	li	t4, (HTIF_RING_SIZE + 16)
+	mulw	t4, t4, t2
+	add	t0, t0, t4
+	li	t4, (HTIF_RING_SIZE + 8)
+	add	t0, t0, t4	/* t0 == htif_ring_last */
+
+	/* Check for new entries */
 	li	t6, 0		/* preset return value */
-	la	t0, htif_ring_last
-	ld	t0, 0(t0)	/* load struct */
-	ld	t4, 8(t0)	/* get used */
-	beqz	t4, 1f
-	ld	t6, 0(t0)	/* get entry */
+	ld	t2, 0(t0)	/* load ptr to last */
+	ld	t4, 8(t2)	/* get used */
+	beqz	t4, 1f		/* No new entries. Exit */
+
+	/* Get one */
+	ld	t6, 0(t2)	/* get entry */
 	li	t4, 0
-	sd	t4, 8(t0)	/* mark free */
-	sd	t4, 0(t0)	/* free entry, just in case */
-	ld	t4, 16(t0)	/* take next */
-	la	t0, htif_ring_last
-	sd	t4, 0(t0)
+	sd	t4, 8(t2)	/* mark free */
+	sd	t4, 0(t2)	/* free entry, just in case */
+	ld	t4, 16(t2)	/* take next */
+	sd	t4, 0(t0)	/* update ptr to last */
 1:
 	/* Exit. Result is stored in t6 */
 	j	exit_next_instr
 
 htif_cmd:
+1:
 	mv	t0, t6
+	csrrw	t0, mtohost, t0
+	bnez	t0, 1b
+	j	exit_next_instr
+
+htif_lowputc:
 1:
+	mv	t0, t6
 	csrrw	t0, mtohost, t0
 	bnez	t0, 1b
+
+2:
+	li	t4, 0
+	csrrw	t5, mfromhost, t4
+	beqz	t5, 2b
+
+	/* Console PUT intr ? */
+	mv	t2, t5
+	srli	t2, t2, 48
+	li	t3, 0x0101
+	beq	t2, t3, 3f
+
+	/* Not a console PUT, so save entry */
+	la	t0, htif_ring
+	csrr	t2, mhartid
+	li	t4, (HTIF_RING_SIZE + 16)
+	mulw	t4, t4, t2
+	add	t0, t0, t4
+	li	t4, (HTIF_RING_SIZE)
+	add	t0, t0, t4	/* t0 == htif_ring_cursor */
+
+	ld	t2, 0(t0)	/* load ptr to cursor */
+	sd	t5, 0(t2)	/* put entry */
+	li	t4, 1
+	sd	t4, 8(t2)	/* mark used */
+	ld	t4, 16(t2)	/* take next */
+	/* Update cursor */
+	sd	t4, 0(t0)
+
+	/* Post supervisor software interrupt */
+	li	t0, MIP_SSIP
+	csrs	mip, t0
+
+	/* Wait for console intr again */
+	j	2b
+
+3:
 	j	exit_next_instr
 
 set_mtimecmp:

Modified: head/sys/riscv/riscv/genassym.c
==============================================================================
--- head/sys/riscv/riscv/genassym.c	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/riscv/riscv/genassym.c	Wed Feb 24 16:50:34 2016	(r295972)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015 Ruslan Bukin <br at bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br at bsdpad.com>
  * All rights reserved.
  *
  * Portions of this software were developed by SRI International and the
@@ -75,6 +75,7 @@ ASSYM(PCB_A, offsetof(struct pcb, pcb_a)
 ASSYM(SF_UC, offsetof(struct sigframe, sf_uc));
 
 ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb));
+ASSYM(PC_SPTBR, offsetof(struct pcpu, pc_sptbr));
 ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread));
 
 ASSYM(TD_PCB, offsetof(struct thread, td_pcb));

Modified: head/sys/riscv/riscv/intr_machdep.c
==============================================================================
--- head/sys/riscv/riscv/intr_machdep.c	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/riscv/riscv/intr_machdep.c	Wed Feb 24 16:50:34 2016	(r295972)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015 Ruslan Bukin <br at bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br at bsdpad.com>
  * All rights reserved.
  *
  * Portions of this software were developed by SRI International and the
@@ -38,7 +38,9 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
+#include <sys/cpuset.h>
 #include <sys/interrupt.h>
+#include <sys/smp.h>
 
 #include <machine/clock.h>
 #include <machine/cpu.h>
@@ -46,12 +48,9 @@ __FBSDID("$FreeBSD$");
 #include <machine/frame.h>
 #include <machine/intr.h>
 
-enum {
-	IRQ_SOFTWARE,
-	IRQ_TIMER,
-	IRQ_HTIF,
-	NIRQS
-};
+#ifdef SMP
+#include <machine/smp.h>
+#endif
 
 u_long intrcnt[NIRQS];
 size_t sintrcnt = sizeof(intrcnt);
@@ -159,8 +158,12 @@ riscv_setup_intr(const char *name, drive
 		riscv_unmask_irq((void*)(uintptr_t)irq);
 	}
 
-	intr_event_add_handler(event, name, filt, handler, arg,
+	error = intr_event_add_handler(event, name, filt, handler, arg,
 	    intr_priority(flags), flags, cookiep);
+	if (error) {
+		printf("Failed to setup intr: %d\n", irq);
+		return (error);
+	}
 
 	riscv_intrcnt_setname(riscv_intr_counters[irq],
 			     event->ie_fullname);
@@ -221,3 +224,74 @@ riscv_cpu_intr(struct trapframe *frame)
 
 	critical_exit();
 }
+
+#ifdef SMP
+void
+riscv_setup_ipihandler(driver_filter_t *filt)
+{
+
+	riscv_setup_intr("ipi", filt, NULL, NULL, IRQ_SOFTWARE,
+	    INTR_TYPE_MISC, NULL);
+}
+
+void
+riscv_unmask_ipi(void)
+{
+
+	csr_set(sie, SIE_SSIE);
+}
+
+/* Sending IPI */
+static void
+ipi_send(struct pcpu *pc, int ipi)
+{
+
+	CTR3(KTR_SMP, "%s: cpu=%d, ipi=%x", __func__, pc->pc_cpuid, ipi);
+
+	atomic_set_32(&pc->pc_pending_ipis, ipi);
+	machine_command(ECALL_SEND_IPI, pc->pc_reg);
+
+	CTR1(KTR_SMP, "%s: sent", __func__);
+}
+
+void
+ipi_all_but_self(u_int ipi)
+{
+	cpuset_t other_cpus;
+
+	other_cpus = all_cpus;
+	CPU_CLR(PCPU_GET(cpuid), &other_cpus);
+
+	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
+	ipi_selected(other_cpus, ipi);
+}
+
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+	cpuset_t cpus;
+
+	CPU_ZERO(&cpus);
+	CPU_SET(cpu, &cpus);
+
+	CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x\n", __func__, cpu, ipi);
+	ipi_send(cpuid_to_pcpu[cpu], ipi);
+}
+
+void
+ipi_selected(cpuset_t cpus, u_int ipi)
+{
+	struct pcpu *pc;
+
+	CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi);
+
+	STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
+		if (CPU_ISSET(pc->pc_cpuid, &cpus)) {
+			CTR3(KTR_SMP, "%s: pc: %p, ipi: %x\n", __func__, pc,
+			    ipi);
+			ipi_send(pc, ipi);
+		}
+	}
+}
+
+#endif

Modified: head/sys/riscv/riscv/locore.S
==============================================================================
--- head/sys/riscv/riscv/locore.S	Wed Feb 24 15:15:46 2016	(r295971)
+++ head/sys/riscv/riscv/locore.S	Wed Feb 24 16:50:34 2016	(r295972)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015 Ruslan Bukin <br at bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br at bsdpad.com>
  * All rights reserved.
  *
  * Portions of this software were developed by SRI International and the
@@ -43,8 +43,55 @@
 #include <machine/riscvreg.h>
 #include <machine/pte.h>
 
-#define	HTIF_RING_SIZE	(64)
-#define	HTIF_RING_LAST	(24 * (HTIF_RING_SIZE - 1))
+#define	HTIF_RING_NENTRIES	(512)
+#define	HTIF_RING_ENTRY_SZ	(24)
+#define	HTIF_RING_SIZE		(HTIF_RING_ENTRY_SZ * HTIF_RING_NENTRIES)
+#define	HW_STACK_SIZE		(96)
+
+/*
+ * Event queue for each CPU core:
+ *
+ * struct htif_ring {
+ *     uint64_t data;
+ *     uint64_t used;
+ *     uint64_t next;
+ * } htif_ring[HTIF_RING_NENTRIES];
+ * uint64_t htif_ring_cursor;
+ * uint64_t htif_ring_last;
+ */
+
+.macro build_ring
+	la	t0, htif_ring
+#ifdef SMP
+	csrr	a0, mhartid
+	li	s0, (HTIF_RING_SIZE + 16)
+	mulw	s0, a0, s0
+	add	t0, t0, s0
+#endif
+	li	t1, 0
+	sd	t1, 0(t0)	/* zero data */
+	sd	t1, 8(t0)	/* zero used */
+	mv	t2, t0
+	mv	t3, t0
+	li	t5, (HTIF_RING_SIZE - HTIF_RING_ENTRY_SZ)
+	li	t6, 0
+	add	t4, t0, t5
+1:
+	addi	t3, t3, 24	/* pointer to next */
+	beq	t3, t4, 2f	/* finish */
+	sd	t3, 16(t2)	/* store pointer */
+	addi	t2, t2, 24	/* next entry */
+	addi	t6, t6, 1	/* counter */
+	j	1b
+2:
+	sd	t0, 16(t3)	/* last -> first */
+
+	li	t2, (HTIF_RING_SIZE)
+	add	s0, t0, t2
+	sd	t0, 0(s0)	/* cursor */
+	sd	t0, 8(s0)	/* last */
+	/* finish building ring */
+.endm
 
 	.globl	kernbase
 	.set	kernbase, KERNBASE
@@ -74,33 +121,14 @@ mentry:
 	.align 8
 	.globl _start
 _start:
-	li	s11, KERNBASE
+	/* Direct secondary cores to mpentry */
+	csrr	a0, mhartid
+	bnez	a0, mpentry
 
-	/* Build ring */
-	la	t0, htif_ring
-	li	t1, 0
-	sd	t1, 0(t0)	/* zero data */
-	sd	t1, 8(t0)	/* zero used */
-	mv	t2, t0
-	mv	t3, t0
-	li	t5, HTIF_RING_LAST
-	li	t6, 0
-	add	t4, t0, t5
-1:
-	addi	t3, t3, 24	/* pointer to next */
-	beq	t3, t4, 2f	/* finish */
-	sd	t3, 16(t2)	/* store pointer */
-	addi	t2, t2, 24	/* next entry */
-	addi	t6, t6, 1	/* counter */
-	j	1b
-2:
-	sd	t0, 16(t3)	/* last -> first */
-	la	t1, htif_ring_cursor
-	sd	t0, 0(t1)
-	la	t1, htif_ring_last
-	sd	t0, 0(t1)
-	/* finish building ring */
+	/* Build event queue for current core */
+	build_ring
 
+	/* Setup machine-mode stack for CPU 0 */
 	la	t0, hardstack_end
 	csrw	mscratch, t0
 
@@ -129,7 +157,14 @@ _start:
 	li	a6, PTE_SIZE
 	mulw	a5, a5, a6
 	add	t0, s1, a5
+	/* Store it to pagetable_l0 for each cpu */
+	li	t1, MAXCPU
+	li	t2, PAGE_SIZE
+1:
 	sd	t6, 0(t0)
+	add	t0, t0, t2
+	addi	t1, t1, -1
+	bnez	t1, 1b
 
 	/* Level 1 */
 	la	s1, pagetable_l1
@@ -177,8 +212,16 @@ _start:
 		     (MSTATUS_PRV_U << MSTATUS_PRV2_SHIFT));
 	csrw	mstatus, s0
 
+	/*
+	 * Enable machine-mode software interrupts
+	 * so we can deliver IPI to this core.
+	 */
+	li	t0, MIE_MSIE
+	csrs	mie, t0
+
 	/* Exit from machine mode */
 	la	t0, .Lmmu_on
+	li	s11, KERNBASE
 	add	t0, t0, s11
 	csrw	mepc, t0
 	eret
@@ -213,24 +256,16 @@ initstack:
 	.space  (PAGE_SIZE * KSTACK_PAGES)
 initstack_end:
 hardstack:
-	.space  (PAGE_SIZE)
+	.space  (HW_STACK_SIZE * MAXCPU)
 hardstack_end:
 
 	.globl htif_ring
 htif_ring:
-	.space  (24 * 1024)
-
-	.globl htif_ring_cursor
-htif_ring_cursor:
-	.space  (8)
-
-	.globl htif_ring_last
-htif_ring_last:
-	.space  (8)
+	.space ((HTIF_RING_SIZE + 16) * MAXCPU)
 
 	.globl console_intr
 console_intr:
-	.space  (8)
+	.space (8)
 
 ENTRY(sigcode)
 	mv	a0, sp
@@ -259,7 +294,7 @@ szsigcode:
 	.align	12
 	.globl pagetable_l0
 pagetable_l0:
-	.space	PAGE_SIZE
+	.space (PAGE_SIZE * MAXCPU)
 pagetable_l1:
 	.space	PAGE_SIZE
 pagetable_l2:
@@ -270,4 +305,95 @@ pagetable_end:
 init_pt_va:
 	.quad pagetable_l2	/* XXX: Keep page tables VA */
 
+#ifndef SMP
+ENTRY(mpentry)
+1:
+	wfi
+	j	1b
+END(mpentry)
+#else
+/*
+ * mpentry(unsigned long)
+ *
+ * Called by a core when it is being brought online.
+ * The data in x0 is passed straight to init_secondary.
+ */
+ENTRY(mpentry)
+	/*

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***


More information about the svn-src-head mailing list