svn commit: r287961 - head/sys/arm64/arm64

[Konstantin Belousov]

Author: kib
Date: Fri Sep 18 17:09:59 2015
New Revision: 287961
URL: https://svnweb.freebsd.org/changeset/base/287961

Log:
  Do not execute exception handlers with disabled interrupts.
  
  We should not call vm_fault(), or send a signal, with interrupts
  disabled.  MI kernel code is not prepared for such environment, not to
  mention that this increases system latency, since code appears to be
  executing as being under spinlock.
  
  The FAR register for data aborts is read before the interrupts are
  enabled, to avoid its corruption due to nested exception or context
  switch.
  
  Add asserts, similar to the checks done by other architectures, about
  not taking page faults in non-sleepable contexts, rather than die with
  late and somewhat confusing witness diagnostic.
  
  Reviewed by:	andrew
  Sponsored by:	The FreeBSD Foundation
  Differential Revision:	https://reviews.freebsd.org/D3669

Modified:
  head/sys/arm64/arm64/trap.c

Modified: head/sys/arm64/arm64/trap.c
==============================================================================
--- head/sys/arm64/arm64/trap.c	Fri Sep 18 16:52:18 2015	(r287960)
+++ head/sys/arm64/arm64/trap.c	Fri Sep 18 17:09:59 2015	(r287961)
@@ -76,6 +76,7 @@ extern register_t fsu_intr_fault;
 void do_el1h_sync(struct trapframe *);
 void do_el0_sync(struct trapframe *);
 void do_el0_error(struct trapframe *);
+static void print_registers(struct trapframe *frame);
 
 int (*dtrace_invop_jump_addr)(struct trapframe *);
 
@@ -144,7 +145,7 @@ svc_handler(struct trapframe *frame)
 }
 
 static void
-data_abort(struct trapframe *frame, uint64_t esr, int lower)
+data_abort(struct trapframe *frame, uint64_t esr, uint64_t far, int lower)
 {
 	struct vm_map *map;
 	struct thread *td;
@@ -152,7 +153,6 @@ data_abort(struct trapframe *frame, uint
 	struct pcb *pcb;
 	vm_prot_t ftype;
 	vm_offset_t va;
-	uint64_t far;
 	int error, sig, ucode;
 
 	/*
@@ -181,17 +181,23 @@ data_abort(struct trapframe *frame, uint
 		return;
 	}
 
-	far = READ_SPECIALREG(far_el1);
-	p = td->td_proc;
+	KASSERT(td->td_md.md_spinlock_count == 0,
+	    ("data abort with spinlock held"));
+	if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK |
+	    WARN_GIANTOK, NULL, "Kernel page fault") != 0) {
+		print_registers(frame);
+		panic("data abort in critical section or under mutex");
+	}
 
+	p = td->td_proc;
 	if (lower)
-		map = &td->td_proc->p_vmspace->vm_map;
+		map = &p->p_vmspace->vm_map;
 	else {
 		/* The top bit tells us which range to use */
 		if ((far >> 63) == 1)
 			map = kernel_map;
 		else
-			map = &td->td_proc->p_vmspace->vm_map;
+			map = &p->p_vmspace->vm_map;
 	}
 
 	va = trunc_page(far);
@@ -246,7 +252,7 @@ void
 do_el1h_sync(struct trapframe *frame)
 {
 	uint32_t exception;
-	uint64_t esr;
+	uint64_t esr, far;
 
 	/* Read the esr register to get the exception details */
 	esr = READ_SPECIALREG(esr_el1);
@@ -281,7 +287,9 @@ do_el1h_sync(struct trapframe *frame)
 		print_registers(frame);
 		panic("VFP exception in the kernel");
 	case EXCP_DATA_ABORT:
-		data_abort(frame, esr, 0);
+		far = READ_SPECIALREG(far_el1);
+		intr_enable();
+		data_abort(frame, esr, far, 0);
 		break;
 	case EXCP_BRK:
 #ifdef KDTRACE_HOOKS
@@ -328,7 +336,7 @@ do_el0_sync(struct trapframe *frame)
 {
 	struct thread *td;
 	uint32_t exception;
-	uint64_t esr;
+	uint64_t esr, far;
 
 	/* Check we have a sane environment when entering from userland */
 	KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS,
@@ -337,6 +345,13 @@ do_el0_sync(struct trapframe *frame)
 
 	esr = READ_SPECIALREG(esr_el1);
 	exception = ESR_ELx_EXCEPTION(esr);
+	switch (exception) {
+	case EXCP_INSN_ABORT_L:
+	case EXCP_DATA_ABORT_L:
+	case EXCP_DATA_ABORT:
+		far = READ_SPECIALREG(far_el1);
+	}
+	intr_enable();
 
 	CTR4(KTR_TRAP,
 	    "do_el0_sync: curthread: %p, esr %lx, elr: %lx, frame: %p",
@@ -352,18 +367,12 @@ do_el0_sync(struct trapframe *frame)
 #endif
 		break;
 	case EXCP_SVC:
-		/*
-		 * Ensure the svc_handler is being run with interrupts enabled.
-		 * They will be automatically restored when returning from
-		 * exception handler.
-		 */
-		intr_enable();
 		svc_handler(frame);
 		break;
 	case EXCP_INSN_ABORT_L:
 	case EXCP_DATA_ABORT_L:
 	case EXCP_DATA_ABORT:
-		data_abort(frame, esr, 1);
+		data_abort(frame, esr, far, 1);
 		break;
 	case EXCP_UNKNOWN:
 		el0_excp_unknown(frame);