PERFORCE change 134576 for review
John Birrell
jb at FreeBSD.org
Thu Jan 31 19:51:50 PST 2008
http://perforce.freebsd.org/chv.cgi?CH=134576
Change 134576 by jb at jb_freebsd1 on 2008/02/01 03:50:58
IFdtrace. Exception handling for fbt. Timer callback for cyclic.
Trap handing for safety. And syscall probe mumbo.
Affected files ...
.. //depot/projects/dtrace7/src/sys/amd64/amd64/exception.S#3 edit
.. //depot/projects/dtrace7/src/sys/amd64/amd64/local_apic.c#2 edit
.. //depot/projects/dtrace7/src/sys/amd64/amd64/trap.c#4 edit
Differences ...
==== //depot/projects/dtrace7/src/sys/amd64/amd64/exception.S#3 (text+ko) ====
@@ -1,8 +1,12 @@
/*-
* Copyright (c) 1989, 1990 William F. Jolitz.
* Copyright (c) 1990 The Regents of the University of California.
+ * Copyright (c) 2007 The FreeBSD Foundation
* All rights reserved.
*
+ * Portions of this software were developed by A. Joseph Koshy under
+ * sponsorship from the FreeBSD Foundation and Google, Inc.
+ *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
@@ -27,11 +31,13 @@
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $FreeBSD: src/sys/amd64/amd64/exception.S,v 1.129.2.1 2007/11/21 16:38:54 jhb Exp $
+ * $FreeBSD: src/sys/amd64/amd64/exception.S,v 1.131 2007/12/07 08:20:15 jkoshy Exp $
*/
#include "opt_atpic.h"
#include "opt_compat.h"
+#include "opt_hwpmc_hooks.h"
+#include "opt_kdtrace.h"
#include <machine/asmacros.h>
#include <machine/psl.h>
@@ -39,7 +45,25 @@
#include "assym.s"
+#ifdef KDTRACE_HOOKS
+ .bss
+ .globl dtrace_invop_jump_addr
+ .align 8
+ .type dtrace_invop_jump_addr, @object
+ .size dtrace_invop_jump_addr, 8
+dtrace_invop_jump_addr:
+ .zero 8
+ .globl dtrace_invop_calltrap_addr
+ .align 8
+ .type dtrace_invop_calltrap_addr, @object
+ .size dtrace_invop_calltrap_addr, 8
+dtrace_invop_calltrap_addr:
+ .zero 8
+#endif
.text
+#ifdef HWPMC_HOOKS
+ ENTRY(start_exceptions)
+#endif
/*****************************************************************************/
/* Trap handling */
@@ -162,6 +186,30 @@
movq %r14,TF_R14(%rsp)
movq %r15,TF_R15(%rsp)
FAKE_MCOUNT(TF_RIP(%rsp))
+#ifdef KDTRACE_HOOKS
+ /*
+ * DTrace Function Boundary Trace (fbt) and Statically Defined
+ * Trace (sdt) probes are triggered by int3 (0xcc) which causes
+ * the #BP (T_BPTFLT) breakpoint interrupt. For all other trap
+ * types, just handle them in the usual way.
+ */
+ cmpq $T_BPTFLT,TF_TRAPNO(%rsp)
+ jne calltrap
+
+ /* Check if there is no DTrace hook registered. */
+ cmpq $0,dtrace_invop_jump_addr
+ je calltrap
+
+ /*
+ * Set our jump address for the jump back in the event that
+ * the breakpoint wasn't caused by DTrace at all.
+ */
+ movq $calltrap, dtrace_invop_calltrap_addr(%rip)
+
+ /* Jump to the code hooked in by DTrace. */
+ movq dtrace_invop_jump_addr, %rax
+ jmpq *dtrace_invop_jump_addr
+#endif
.globl calltrap
.type calltrap, at function
calltrap:
@@ -348,6 +396,9 @@
* execute the NMI handler with interrupts disabled to prevent a
* nested interrupt from executing an 'iretq' instruction and
* inadvertently taking the processor out of NMI mode.
+ *
+ * Third, the NMI handler runs on its own stack (tss_ist1), shared
+ * with the double fault handler.
*/
IDTVEC(nmi)
@@ -386,6 +437,61 @@
movq %rsp, %rdi
call trap
MEXITCOUNT
+#ifdef HWPMC_HOOKS
+ /*
+ * Check if the current trap was from user mode and if so
+ * whether the current thread needs a user call chain to be
+ * captured. We are still in NMI mode at this point.
+ */
+ testb $SEL_RPL_MASK,TF_CS(%rsp)
+ jz nocallchain
+ movq PCPU(CURTHREAD),%rax /* curthread present? */
+ orq %rax,%rax
+ jz nocallchain
+ testl $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */
+ jz nocallchain
+ /*
+ * A user callchain is to be captured, so:
+ * - Move execution to the regular kernel stack, to allow for
+ * nested NMI interrupts.
+ * - Take the processor out of "NMI" mode by faking an "iret".
+ * - Enable interrupts, so that copyin() can work.
+ */
+ movq %rsp,%rsi /* source stack pointer */
+ movq $TF_SIZE,%rcx
+ movq PCPU(RSP0),%rbx
+ subq %rcx,%rbx
+ movq %rbx,%rdi /* destination stack pointer */
+
+ shrq $3,%rcx /* trap frame size in long words */
+ cld
+ rep
+ movsq /* copy trapframe */
+
+ movl %ss,%eax
+ pushq %rax /* tf_ss */
+ pushq %rbx /* tf_rsp (on kernel stack) */
+ pushfq /* tf_rflags */
+ movl %cs,%eax
+ pushq %rax /* tf_cs */
+ pushq $outofnmi /* tf_rip */
+ iretq
+outofnmi:
+ /*
+ * At this point the processor has exited NMI mode and is running
+ * with interrupts turned off on the normal kernel stack.
+ * We turn interrupts back on, and take the usual 'doreti' exit
+ * path.
+ *
+ * If a pending NMI gets recognized at or after this point, it
+ * will cause a kernel callchain to be traced. Since this path
+ * is only taken for NMI interrupts from user space, our `swapgs'
+ * state is correct for taking the doreti path.
+ */
+ sti
+ jmp doreti
+nocallchain:
+#endif
testl %ebx,%ebx
jz nmi_restoreregs
swapgs
@@ -556,3 +662,6 @@
movq $0,TF_ADDR(%rsp)
FAKE_MCOUNT(TF_RIP(%rsp))
jmp calltrap
+#ifdef HWPMC_HOOKS
+ ENTRY(end_exceptions)
+#endif
==== //depot/projects/dtrace7/src/sys/amd64/amd64/local_apic.c#2 (text+ko) ====
@@ -32,9 +32,10 @@
*/
#include <sys/cdefs.h>
-__FBSDID("$FreeBSD: src/sys/amd64/amd64/local_apic.c,v 1.42.2.1 2007/11/08 20:09:15 jhb Exp $");
+__FBSDID("$FreeBSD: src/sys/amd64/amd64/local_apic.c,v 1.43 2007/10/27 13:34:53 jhb Exp $");
#include "opt_hwpmc_hooks.h"
+#include "opt_kdtrace.h"
#include "opt_ddb.h"
@@ -65,6 +66,11 @@
#include <ddb/ddb.h>
#endif
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+cyclic_clock_func_t lapic_cyclic_clock_func[MAXCPU];
+#endif
+
/* Sanity checks on IDT vectors. */
CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT);
CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS);
@@ -668,6 +674,17 @@
(*la->la_timer_count)++;
critical_enter();
+#ifdef KDTRACE_HOOKS
+ /*
+ * If the DTrace hooks are configured and a callback function
+ * has been registered, then call it to process the high speed
+ * timers.
+ */
+ int cpu = PCPU_GET(cpuid);
+ if (lapic_cyclic_clock_func[cpu] != NULL)
+ (*lapic_cyclic_clock_func[cpu])(frame);
+#endif
+
/* Fire hardclock at hz. */
la->la_hard_ticks += hz;
if (la->la_hard_ticks >= lapic_timer_hz) {
==== //depot/projects/dtrace7/src/sys/amd64/amd64/trap.c#4 (text+ko) ====
@@ -38,7 +38,7 @@
*/
#include <sys/cdefs.h>
-__FBSDID("$FreeBSD: src/sys/amd64/amd64/trap.c,v 1.319.2.2 2007/12/06 14:20:25 kib Exp $");
+__FBSDID("$FreeBSD: src/sys/amd64/amd64/trap.c,v 1.324 2007/12/07 08:20:15 jkoshy Exp $");
/*
* AMD64 Trap and System call handling
@@ -49,6 +49,7 @@
#include "opt_hwpmc_hooks.h"
#include "opt_isa.h"
#include "opt_kdb.h"
+#include "opt_kdtrace.h"
#include "opt_ktrace.h"
#include <sys/param.h>
@@ -94,6 +95,24 @@
#endif
#include <machine/tss.h>
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+
+/*
+ * This is a hook which is initialised by the dtrace module
+ * to handle traps which might occur during DTrace probe
+ * execution.
+ */
+dtrace_trap_func_t dtrace_trap_func;
+
+/*
+ * This is a hook which is initialised by the systrace module
+ * when it is loaded. This keeps the DTrace syscall provider
+ * implementation opaque.
+ */
+systrace_probe_func_t systrace_probe_func;
+#endif
+
extern void trap(struct trapframe *frame);
extern void syscall(struct trapframe *frame);
void dblfault_handler(struct trapframe *frame);
@@ -195,11 +214,28 @@
* the NMI was handled by it and we can return immediately.
*/
if (type == T_NMI && pmc_intr &&
- (*pmc_intr)(PCPU_GET(cpuid), (uintptr_t) frame->tf_rip,
- TRAPF_USERMODE(frame)))
+ (*pmc_intr)(PCPU_GET(cpuid), frame))
goto out;
#endif
+#ifdef KDTRACE_HOOKS
+ /*
+ * A trap can occur while DTrace executes a probe. Before
+ * executing the probe, DTrace blocks re-scheduling and sets
+ * a flag in it's per-cpu flags to indicate that it doesn't
+ * want to fault. On returning from the the probe, the no-fault
+ * flag is cleared and finally re-scheduling is enabled.
+ *
+ * If the DTrace kernel module has registered a trap handler,
+ * call it and if it returns non-zero, assume that it has
+ * handled the trap and modified the trap frame so that this
+ * function can return normally.
+ */
+ if (dtrace_trap_func != NULL)
+ if ((*dtrace_trap_func)(frame, type))
+ goto out;
+#endif
+
if ((frame->tf_rflags & PSL_I) == 0) {
/*
* Buggy application or kernel code has disabled
@@ -211,7 +247,7 @@
if (ISPL(frame->tf_cs) == SEL_UPL)
printf(
"pid %ld (%s): trap %d with interrupts disabled\n",
- (long)curproc->p_pid, curproc->p_comm, type);
+ (long)curproc->p_pid, curthread->td_name, type);
else if (type != T_NMI && type != T_BPTFLT &&
type != T_TRCTRAP) {
/*
@@ -708,8 +744,8 @@
printf("current process = ");
if (curproc) {
printf("%lu (%s)\n",
- (u_long)curproc->p_pid, curproc->p_comm ?
- curproc->p_comm : "");
+ (u_long)curproc->p_pid, curthread->td_name ?
+ curthread->td_name : "");
} else {
printf("Idle\n");
}
@@ -836,7 +872,7 @@
#endif
CTR4(KTR_SYSC, "syscall enter thread %p pid %d proc %s code %d", td,
- td->td_proc->p_pid, td->td_proc->p_comm, code);
+ td->td_proc->p_pid, td->td_name, code);
td->td_syscalls++;
@@ -848,9 +884,38 @@
PTRACESTOP_SC(p, td, S_PT_SCE);
+#ifdef KDTRACE_HOOKS
+ /*
+ * If the systrace module has registered it's probe
+ * callback and if there is a probe active for the
+ * syscall 'entry', process the probe.
+ */
+ if (systrace_probe_func != NULL && callp->sy_entry != 0)
+ (*systrace_probe_func)(callp->sy_entry, code, callp,
+ args);
+#endif
+
AUDIT_SYSCALL_ENTER(code, td);
error = (*callp->sy_call)(td, argp);
AUDIT_SYSCALL_EXIT(error, td);
+
+ /* Save the latest error return value. */
+ td->td_errno = error;
+
+#ifdef KDTRACE_HOOKS
+ /* Save the error return variable for DTrace to reference. */
+ args[0] = error;
+ args[1] = error;
+
+ /*
+ * If the systrace module has registered it's probe
+ * callback and if there is a probe active for the
+ * syscall 'return', process the probe.
+ */
+ if (systrace_probe_func != NULL && callp->sy_return != 0)
+ (*systrace_probe_func)(callp->sy_return, code, callp,
+ args);
+#endif
}
switch (error) {
@@ -918,7 +983,7 @@
userret(td, frame);
CTR4(KTR_SYSC, "syscall exit thread %p pid %d proc %s code %d", td,
- td->td_proc->p_pid, td->td_proc->p_comm, code);
+ td->td_proc->p_pid, td->td_name, code);
#ifdef KTRACE
if (KTRPOINT(td, KTR_SYSRET))
More information about the p4-projects
mailing list