svn commit: r194456 - user/gnn/fasttrap/sys/cddl/dev/fasttrap
George V. Neville-Neil
gnn at FreeBSD.org
Thu Jun 18 19:45:23 UTC 2009
Author: gnn
Date: Thu Jun 18 19:45:22 2009
New Revision: 194456
URL: http://svn.freebsd.org/changeset/base/194456
Log:
Import vendor version of fasttrap.c for modification.
Added:
user/gnn/fasttrap/sys/cddl/dev/fasttrap/
user/gnn/fasttrap/sys/cddl/dev/fasttrap/fasttrap.c
- copied unchanged from r194454, vendor-sys/opensolaris/dist/uts/common/dtrace/fasttrap.c
Copied: user/gnn/fasttrap/sys/cddl/dev/fasttrap/fasttrap.c (from r194454, vendor-sys/opensolaris/dist/uts/common/dtrace/fasttrap.c)
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/gnn/fasttrap/sys/cddl/dev/fasttrap/fasttrap.c Thu Jun 18 19:45:22 2009 (r194456, copy of r194454, vendor-sys/opensolaris/dist/uts/common/dtrace/fasttrap.c)
@@ -0,0 +1,2383 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#pragma ident "%Z%%M% %I% %E% SMI"
+
+#include <sys/atomic.h>
+#include <sys/errno.h>
+#include <sys/stat.h>
+#include <sys/modctl.h>
+#include <sys/conf.h>
+#include <sys/systm.h>
+#include <sys/ddi.h>
+#include <sys/sunddi.h>
+#include <sys/cpuvar.h>
+#include <sys/kmem.h>
+#include <sys/strsubr.h>
+#include <sys/fasttrap.h>
+#include <sys/fasttrap_impl.h>
+#include <sys/fasttrap_isa.h>
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+#include <sys/sysmacros.h>
+#include <sys/proc.h>
+#include <sys/priv.h>
+#include <sys/policy.h>
+#include <util/qsort.h>
+
+/*
+ * User-Land Trap-Based Tracing
+ * ----------------------------
+ *
+ * The fasttrap provider allows DTrace consumers to instrument any user-level
+ * instruction to gather data; this includes probes with semantic
+ * signifigance like entry and return as well as simple offsets into the
+ * function. While the specific techniques used are very ISA specific, the
+ * methodology is generalizable to any architecture.
+ *
+ *
+ * The General Methodology
+ * -----------------------
+ *
+ * With the primary goal of tracing every user-land instruction and the
+ * limitation that we can't trust user space so don't want to rely on much
+ * information there, we begin by replacing the instructions we want to trace
+ * with trap instructions. Each instruction we overwrite is saved into a hash
+ * table keyed by process ID and pc address. When we enter the kernel due to
+ * this trap instruction, we need the effects of the replaced instruction to
+ * appear to have occurred before we proceed with the user thread's
+ * execution.
+ *
+ * Each user level thread is represented by a ulwp_t structure which is
+ * always easily accessible through a register. The most basic way to produce
+ * the effects of the instruction we replaced is to copy that instruction out
+ * to a bit of scratch space reserved in the user thread's ulwp_t structure
+ * (a sort of kernel-private thread local storage), set the PC to that
+ * scratch space and single step. When we reenter the kernel after single
+ * stepping the instruction we must then adjust the PC to point to what would
+ * normally be the next instruction. Of course, special care must be taken
+ * for branches and jumps, but these represent such a small fraction of any
+ * instruction set that writing the code to emulate these in the kernel is
+ * not too difficult.
+ *
+ * Return probes may require several tracepoints to trace every return site,
+ * and, conversely, each tracepoint may activate several probes (the entry
+ * and offset 0 probes, for example). To solve this muliplexing problem,
+ * tracepoints contain lists of probes to activate and probes contain lists
+ * of tracepoints to enable. If a probe is activated, it adds its ID to
+ * existing tracepoints or creates new ones as necessary.
+ *
+ * Most probes are activated _before_ the instruction is executed, but return
+ * probes are activated _after_ the effects of the last instruction of the
+ * function are visible. Return probes must be fired _after_ we have
+ * single-stepped the instruction whereas all other probes are fired
+ * beforehand.
+ *
+ *
+ * Lock Ordering
+ * -------------
+ *
+ * The lock ordering below -- both internally and with respect to the DTrace
+ * framework -- is a little tricky and bears some explanation. Each provider
+ * has a lock (ftp_mtx) that protects its members including reference counts
+ * for enabled probes (ftp_rcount), consumers actively creating probes
+ * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
+ * from being freed. A provider is looked up by taking the bucket lock for the
+ * provider hash table, and is returned with its lock held. The provider lock
+ * may be taken in functions invoked by the DTrace framework, but may not be
+ * held while calling functions in the DTrace framework.
+ *
+ * To ensure consistency over multiple calls to the DTrace framework, the
+ * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
+ * not be taken when holding the provider lock as that would create a cyclic
+ * lock ordering. In situations where one would naturally take the provider
+ * lock and then the creation lock, we instead up a reference count to prevent
+ * the provider from disappearing, drop the provider lock, and acquire the
+ * creation lock.
+ *
+ * Briefly:
+ * bucket lock before provider lock
+ * DTrace before provider lock
+ * creation lock before DTrace
+ * never hold the provider lock and creation lock simultaneously
+ */
+
+static dev_info_t *fasttrap_devi;
+static dtrace_meta_provider_id_t fasttrap_meta_id;
+
+static timeout_id_t fasttrap_timeout;
+static kmutex_t fasttrap_cleanup_mtx;
+static uint_t fasttrap_cleanup_work;
+
+/*
+ * Generation count on modifications to the global tracepoint lookup table.
+ */
+static volatile uint64_t fasttrap_mod_gen;
+
+/*
+ * When the fasttrap provider is loaded, fasttrap_max is set to either
+ * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
+ * fasttrap.conf file. Each time a probe is created, fasttrap_total is
+ * incremented by the number of tracepoints that may be associated with that
+ * probe; fasttrap_total is capped at fasttrap_max.
+ */
+#define FASTTRAP_MAX_DEFAULT 250000
+static uint32_t fasttrap_max;
+static uint32_t fasttrap_total;
+
+
+#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000
+#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100
+#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100
+
+#define FASTTRAP_PID_NAME "pid"
+
+fasttrap_hash_t fasttrap_tpoints;
+static fasttrap_hash_t fasttrap_provs;
+static fasttrap_hash_t fasttrap_procs;
+
+static uint64_t fasttrap_pid_count; /* pid ref count */
+static kmutex_t fasttrap_count_mtx; /* lock on ref count */
+
+#define FASTTRAP_ENABLE_FAIL 1
+#define FASTTRAP_ENABLE_PARTIAL 2
+
+static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
+static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
+
+static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
+ const dtrace_pattr_t *);
+static void fasttrap_provider_retire(pid_t, const char *, int);
+static void fasttrap_provider_free(fasttrap_provider_t *);
+
+static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
+static void fasttrap_proc_release(fasttrap_proc_t *);
+
+#define FASTTRAP_PROVS_INDEX(pid, name) \
+ ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
+
+#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
+
+static int
+fasttrap_highbit(ulong_t i)
+{
+ int h = 1;
+
+ if (i == 0)
+ return (0);
+#ifdef _LP64
+ if (i & 0xffffffff00000000ul) {
+ h += 32; i >>= 32;
+ }
+#endif
+ if (i & 0xffff0000) {
+ h += 16; i >>= 16;
+ }
+ if (i & 0xff00) {
+ h += 8; i >>= 8;
+ }
+ if (i & 0xf0) {
+ h += 4; i >>= 4;
+ }
+ if (i & 0xc) {
+ h += 2; i >>= 2;
+ }
+ if (i & 0x2) {
+ h += 1;
+ }
+ return (h);
+}
+
+static uint_t
+fasttrap_hash_str(const char *p)
+{
+ unsigned int g;
+ uint_t hval = 0;
+
+ while (*p) {
+ hval = (hval << 4) + *p++;
+ if ((g = (hval & 0xf0000000)) != 0)
+ hval ^= g >> 24;
+ hval &= ~g;
+ }
+ return (hval);
+}
+
+void
+fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
+{
+ sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
+
+ sqp->sq_info.si_signo = SIGTRAP;
+ sqp->sq_info.si_code = TRAP_DTRACE;
+ sqp->sq_info.si_addr = (caddr_t)pc;
+
+ mutex_enter(&p->p_lock);
+ sigaddqa(p, t, sqp);
+ mutex_exit(&p->p_lock);
+
+ if (t != NULL)
+ aston(t);
+}
+
+/*
+ * This function ensures that no threads are actively using the memory
+ * associated with probes that were formerly live.
+ */
+static void
+fasttrap_mod_barrier(uint64_t gen)
+{
+ int i;
+
+ if (gen < fasttrap_mod_gen)
+ return;
+
+ fasttrap_mod_gen++;
+
+ for (i = 0; i < NCPU; i++) {
+ mutex_enter(&cpu_core[i].cpuc_pid_lock);
+ mutex_exit(&cpu_core[i].cpuc_pid_lock);
+ }
+}
+
+/*
+ * This is the timeout's callback for cleaning up the providers and their
+ * probes.
+ */
+/*ARGSUSED*/
+static void
+fasttrap_pid_cleanup_cb(void *data)
+{
+ fasttrap_provider_t **fpp, *fp;
+ fasttrap_bucket_t *bucket;
+ dtrace_provider_id_t provid;
+ int i, later;
+
+ static volatile int in = 0;
+ ASSERT(in == 0);
+ in = 1;
+
+ mutex_enter(&fasttrap_cleanup_mtx);
+ while (fasttrap_cleanup_work) {
+ fasttrap_cleanup_work = 0;
+ mutex_exit(&fasttrap_cleanup_mtx);
+
+ later = 0;
+
+ /*
+ * Iterate over all the providers trying to remove the marked
+ * ones. If a provider is marked but not retired, we just
+ * have to take a crack at removing it -- it's no big deal if
+ * we can't.
+ */
+ for (i = 0; i < fasttrap_provs.fth_nent; i++) {
+ bucket = &fasttrap_provs.fth_table[i];
+ mutex_enter(&bucket->ftb_mtx);
+ fpp = (fasttrap_provider_t **)&bucket->ftb_data;
+
+ while ((fp = *fpp) != NULL) {
+ if (!fp->ftp_marked) {
+ fpp = &fp->ftp_next;
+ continue;
+ }
+
+ mutex_enter(&fp->ftp_mtx);
+
+ /*
+ * If this provider has consumers actively
+ * creating probes (ftp_ccount) or is a USDT
+ * provider (ftp_mcount), we can't unregister
+ * or even condense.
+ */
+ if (fp->ftp_ccount != 0 ||
+ fp->ftp_mcount != 0) {
+ mutex_exit(&fp->ftp_mtx);
+ fp->ftp_marked = 0;
+ continue;
+ }
+
+ if (!fp->ftp_retired || fp->ftp_rcount != 0)
+ fp->ftp_marked = 0;
+
+ mutex_exit(&fp->ftp_mtx);
+
+ /*
+ * If we successfully unregister this
+ * provider we can remove it from the hash
+ * chain and free the memory. If our attempt
+ * to unregister fails and this is a retired
+ * provider, increment our flag to try again
+ * pretty soon. If we've consumed more than
+ * half of our total permitted number of
+ * probes call dtrace_condense() to try to
+ * clean out the unenabled probes.
+ */
+ provid = fp->ftp_provid;
+ if (dtrace_unregister(provid) != 0) {
+ if (fasttrap_total > fasttrap_max / 2)
+ (void) dtrace_condense(provid);
+ later += fp->ftp_marked;
+ fpp = &fp->ftp_next;
+ } else {
+ *fpp = fp->ftp_next;
+ fasttrap_provider_free(fp);
+ }
+ }
+ mutex_exit(&bucket->ftb_mtx);
+ }
+
+ mutex_enter(&fasttrap_cleanup_mtx);
+ }
+
+ ASSERT(fasttrap_timeout != 0);
+
+ /*
+ * If we were unable to remove a retired provider, try again after
+ * a second. This situation can occur in certain circumstances where
+ * providers cannot be unregistered even though they have no probes
+ * enabled because of an execution of dtrace -l or something similar.
+ * If the timeout has been disabled (set to 1 because we're trying
+ * to detach), we set fasttrap_cleanup_work to ensure that we'll
+ * get a chance to do that work if and when the timeout is reenabled
+ * (if detach fails).
+ */
+ if (later > 0 && fasttrap_timeout != (timeout_id_t)1)
+ fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, hz);
+ else if (later > 0)
+ fasttrap_cleanup_work = 1;
+ else
+ fasttrap_timeout = 0;
+
+ mutex_exit(&fasttrap_cleanup_mtx);
+ in = 0;
+}
+
+/*
+ * Activates the asynchronous cleanup mechanism.
+ */
+static void
+fasttrap_pid_cleanup(void)
+{
+ mutex_enter(&fasttrap_cleanup_mtx);
+ fasttrap_cleanup_work = 1;
+ if (fasttrap_timeout == 0)
+ fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1);
+ mutex_exit(&fasttrap_cleanup_mtx);
+}
+
+/*
+ * This is called from cfork() via dtrace_fasttrap_fork(). The child
+ * process's address space is (roughly) a copy of the parent process's so
+ * we have to remove all the instrumentation we had previously enabled in the
+ * parent.
+ */
+static void
+fasttrap_fork(proc_t *p, proc_t *cp)
+{
+ pid_t ppid = p->p_pid;
+ int i;
+
+ ASSERT(curproc == p);
+ ASSERT(p->p_proc_flag & P_PR_LOCK);
+ ASSERT(p->p_dtrace_count > 0);
+ ASSERT(cp->p_dtrace_count == 0);
+
+ /*
+ * This would be simpler and faster if we maintained per-process
+ * hash tables of enabled tracepoints. It could, however, potentially
+ * slow down execution of a tracepoint since we'd need to go
+ * through two levels of indirection. In the future, we should
+ * consider either maintaining per-process ancillary lists of
+ * enabled tracepoints or hanging a pointer to a per-process hash
+ * table of enabled tracepoints off the proc structure.
+ */
+
+ /*
+ * We don't have to worry about the child process disappearing
+ * because we're in fork().
+ */
+ mutex_enter(&cp->p_lock);
+ sprlock_proc(cp);
+ mutex_exit(&cp->p_lock);
+
+ /*
+ * Iterate over every tracepoint looking for ones that belong to the
+ * parent process, and remove each from the child process.
+ */
+ for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
+ fasttrap_tracepoint_t *tp;
+ fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
+
+ mutex_enter(&bucket->ftb_mtx);
+ for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
+ if (tp->ftt_pid == ppid &&
+ tp->ftt_proc->ftpc_acount != 0) {
+ int ret = fasttrap_tracepoint_remove(cp, tp);
+ ASSERT(ret == 0);
+
+ /*
+ * The count of active providers can only be
+ * decremented (i.e. to zero) during exec,
+ * exit, and removal of a meta provider so it
+ * should be impossible to drop the count
+ * mid-fork.
+ */
+ ASSERT(tp->ftt_proc->ftpc_acount != 0);
+ }
+ }
+ mutex_exit(&bucket->ftb_mtx);
+ }
+
+ mutex_enter(&cp->p_lock);
+ sprunlock(cp);
+}
+
+/*
+ * This is called from proc_exit() or from exec_common() if p_dtrace_probes
+ * is set on the proc structure to indicate that there is a pid provider
+ * associated with this process.
+ */
+static void
+fasttrap_exec_exit(proc_t *p)
+{
+ ASSERT(p == curproc);
+ ASSERT(MUTEX_HELD(&p->p_lock));
+
+ mutex_exit(&p->p_lock);
+
+ /*
+ * We clean up the pid provider for this process here; user-land
+ * static probes are handled by the meta-provider remove entry point.
+ */
+ fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
+
+ mutex_enter(&p->p_lock);
+}
+
+
+/*ARGSUSED*/
+static void
+fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc)
+{
+ /*
+ * There are no "default" pid probes.
+ */
+}
+
+static int
+fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
+{
+ fasttrap_tracepoint_t *tp, *new_tp = NULL;
+ fasttrap_bucket_t *bucket;
+ fasttrap_id_t *id;
+ pid_t pid;
+ uintptr_t pc;
+
+ ASSERT(index < probe->ftp_ntps);
+
+ pid = probe->ftp_pid;
+ pc = probe->ftp_tps[index].fit_tp->ftt_pc;
+ id = &probe->ftp_tps[index].fit_id;
+
+ ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
+
+ ASSERT(!(p->p_flag & SVFORK));
+
+ /*
+ * Before we make any modifications, make sure we've imposed a barrier
+ * on the generation in which this probe was last modified.
+ */
+ fasttrap_mod_barrier(probe->ftp_gen);
+
+ bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
+
+ /*
+ * If the tracepoint has already been enabled, just add our id to the
+ * list of interested probes. This may be our second time through
+ * this path in which case we'll have constructed the tracepoint we'd
+ * like to install. If we can't find a match, and have an allocated
+ * tracepoint ready to go, enable that one now.
+ *
+ * A tracepoint whose process is defunct is also considered defunct.
+ */
+again:
+ mutex_enter(&bucket->ftb_mtx);
+ for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
+ /*
+ * Note that it's safe to access the active count on the
+ * associated proc structure because we know that at least one
+ * provider (this one) will still be around throughout this
+ * operation.
+ */
+ if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
+ tp->ftt_proc->ftpc_acount == 0)
+ continue;
+
+ /*
+ * Now that we've found a matching tracepoint, it would be
+ * a decent idea to confirm that the tracepoint is still
+ * enabled and the trap instruction hasn't been overwritten.
+ * Since this is a little hairy, we'll punt for now.
+ */
+
+ /*
+ * This can't be the first interested probe. We don't have
+ * to worry about another thread being in the midst of
+ * deleting this tracepoint (which would be the only valid
+ * reason for a tracepoint to have no interested probes)
+ * since we're holding P_PR_LOCK for this process.
+ */
+ ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
+
+ switch (id->fti_ptype) {
+ case DTFTP_ENTRY:
+ case DTFTP_OFFSETS:
+ case DTFTP_IS_ENABLED:
+ id->fti_next = tp->ftt_ids;
+ membar_producer();
+ tp->ftt_ids = id;
+ membar_producer();
+ break;
+
+ case DTFTP_RETURN:
+ case DTFTP_POST_OFFSETS:
+ id->fti_next = tp->ftt_retids;
+ membar_producer();
+ tp->ftt_retids = id;
+ membar_producer();
+ break;
+
+ default:
+ ASSERT(0);
+ }
+
+ mutex_exit(&bucket->ftb_mtx);
+
+ if (new_tp != NULL) {
+ new_tp->ftt_ids = NULL;
+ new_tp->ftt_retids = NULL;
+ }
+
+ return (0);
+ }
+
+ /*
+ * If we have a good tracepoint ready to go, install it now while
+ * we have the lock held and no one can screw with us.
+ */
+ if (new_tp != NULL) {
+ int rc = 0;
+
+ new_tp->ftt_next = bucket->ftb_data;
+ membar_producer();
+ bucket->ftb_data = new_tp;
+ membar_producer();
+ mutex_exit(&bucket->ftb_mtx);
+
+ /*
+ * Activate the tracepoint in the ISA-specific manner.
+ * If this fails, we need to report the failure, but
+ * indicate that this tracepoint must still be disabled
+ * by calling fasttrap_tracepoint_disable().
+ */
+ if (fasttrap_tracepoint_install(p, new_tp) != 0)
+ rc = FASTTRAP_ENABLE_PARTIAL;
+
+ /*
+ * Increment the count of the number of tracepoints active in
+ * the victim process.
+ */
+ ASSERT(p->p_proc_flag & P_PR_LOCK);
+ p->p_dtrace_count++;
+
+ return (rc);
+ }
+
+ mutex_exit(&bucket->ftb_mtx);
+
+ /*
+ * Initialize the tracepoint that's been preallocated with the probe.
+ */
+ new_tp = probe->ftp_tps[index].fit_tp;
+
+ ASSERT(new_tp->ftt_pid == pid);
+ ASSERT(new_tp->ftt_pc == pc);
+ ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
+ ASSERT(new_tp->ftt_ids == NULL);
+ ASSERT(new_tp->ftt_retids == NULL);
+
+ switch (id->fti_ptype) {
+ case DTFTP_ENTRY:
+ case DTFTP_OFFSETS:
+ case DTFTP_IS_ENABLED:
+ id->fti_next = NULL;
+ new_tp->ftt_ids = id;
+ break;
+
+ case DTFTP_RETURN:
+ case DTFTP_POST_OFFSETS:
+ id->fti_next = NULL;
+ new_tp->ftt_retids = id;
+ break;
+
+ default:
+ ASSERT(0);
+ }
+
+ /*
+ * If the ISA-dependent initialization goes to plan, go back to the
+ * beginning and try to install this freshly made tracepoint.
+ */
+ if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
+ goto again;
+
+ new_tp->ftt_ids = NULL;
+ new_tp->ftt_retids = NULL;
+
+ return (FASTTRAP_ENABLE_FAIL);
+}
+
+static void
+fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
+{
+ fasttrap_bucket_t *bucket;
+ fasttrap_provider_t *provider = probe->ftp_prov;
+ fasttrap_tracepoint_t **pp, *tp;
+ fasttrap_id_t *id, **idp;
+ pid_t pid;
+ uintptr_t pc;
+
+ ASSERT(index < probe->ftp_ntps);
+
+ pid = probe->ftp_pid;
+ pc = probe->ftp_tps[index].fit_tp->ftt_pc;
+ id = &probe->ftp_tps[index].fit_id;
+
+ ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
+
+ /*
+ * Find the tracepoint and make sure that our id is one of the
+ * ones registered with it.
+ */
+ bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
+ mutex_enter(&bucket->ftb_mtx);
+ for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
+ if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
+ tp->ftt_proc == provider->ftp_proc)
+ break;
+ }
+
+ /*
+ * If we somehow lost this tracepoint, we're in a world of hurt.
+ */
+ ASSERT(tp != NULL);
+
+ switch (id->fti_ptype) {
+ case DTFTP_ENTRY:
+ case DTFTP_OFFSETS:
+ case DTFTP_IS_ENABLED:
+ ASSERT(tp->ftt_ids != NULL);
+ idp = &tp->ftt_ids;
+ break;
+
+ case DTFTP_RETURN:
+ case DTFTP_POST_OFFSETS:
+ ASSERT(tp->ftt_retids != NULL);
+ idp = &tp->ftt_retids;
+ break;
+
+ default:
+ ASSERT(0);
+ }
+
+ while ((*idp)->fti_probe != probe) {
+ idp = &(*idp)->fti_next;
+ ASSERT(*idp != NULL);
+ }
+
+ id = *idp;
+ *idp = id->fti_next;
+ membar_producer();
+
+ ASSERT(id->fti_probe == probe);
+
+ /*
+ * If there are other registered enablings of this tracepoint, we're
+ * all done, but if this was the last probe assocated with this
+ * this tracepoint, we need to remove and free it.
+ */
+ if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
+
+ /*
+ * If the current probe's tracepoint is in use, swap it
+ * for an unused tracepoint.
+ */
+ if (tp == probe->ftp_tps[index].fit_tp) {
+ fasttrap_probe_t *tmp_probe;
+ fasttrap_tracepoint_t **tmp_tp;
+ uint_t tmp_index;
+
+ if (tp->ftt_ids != NULL) {
+ tmp_probe = tp->ftt_ids->fti_probe;
+ /* LINTED - alignment */
+ tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
+ tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
+ } else {
+ tmp_probe = tp->ftt_retids->fti_probe;
+ /* LINTED - alignment */
+ tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
+ tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
+ }
+
+ ASSERT(*tmp_tp != NULL);
+ ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
+ ASSERT((*tmp_tp)->ftt_ids == NULL);
+ ASSERT((*tmp_tp)->ftt_retids == NULL);
+
+ probe->ftp_tps[index].fit_tp = *tmp_tp;
+ *tmp_tp = tp;
+ }
+
+ mutex_exit(&bucket->ftb_mtx);
+
+ /*
+ * Tag the modified probe with the generation in which it was
+ * changed.
+ */
+ probe->ftp_gen = fasttrap_mod_gen;
+ return;
+ }
+
+ mutex_exit(&bucket->ftb_mtx);
+
+ /*
+ * We can't safely remove the tracepoint from the set of active
+ * tracepoints until we've actually removed the fasttrap instruction
+ * from the process's text. We can, however, operate on this
+ * tracepoint secure in the knowledge that no other thread is going to
+ * be looking at it since we hold P_PR_LOCK on the process if it's
+ * live or we hold the provider lock on the process if it's dead and
+ * gone.
+ */
+
+ /*
+ * We only need to remove the actual instruction if we're looking
+ * at an existing process
+ */
+ if (p != NULL) {
+ /*
+ * If we fail to restore the instruction we need to kill
+ * this process since it's in a completely unrecoverable
+ * state.
+ */
+ if (fasttrap_tracepoint_remove(p, tp) != 0)
+ fasttrap_sigtrap(p, NULL, pc);
+
+ /*
+ * Decrement the count of the number of tracepoints active
+ * in the victim process.
+ */
+ ASSERT(p->p_proc_flag & P_PR_LOCK);
+ p->p_dtrace_count--;
+ }
+
+ /*
+ * Remove the probe from the hash table of active tracepoints.
+ */
+ mutex_enter(&bucket->ftb_mtx);
+ pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
+ ASSERT(*pp != NULL);
+ while (*pp != tp) {
+ pp = &(*pp)->ftt_next;
+ ASSERT(*pp != NULL);
+ }
+
+ *pp = tp->ftt_next;
+ membar_producer();
+
+ mutex_exit(&bucket->ftb_mtx);
+
+ /*
+ * Tag the modified probe with the generation in which it was changed.
+ */
+ probe->ftp_gen = fasttrap_mod_gen;
+}
+
+static void
+fasttrap_enable_callbacks(void)
+{
+ /*
+ * We don't have to play the rw lock game here because we're
+ * providing something rather than taking something away --
+ * we can be sure that no threads have tried to follow this
+ * function pointer yet.
+ */
+ mutex_enter(&fasttrap_count_mtx);
+ if (fasttrap_pid_count == 0) {
+ ASSERT(dtrace_pid_probe_ptr == NULL);
+ ASSERT(dtrace_return_probe_ptr == NULL);
+ dtrace_pid_probe_ptr = &fasttrap_pid_probe;
+ dtrace_return_probe_ptr = &fasttrap_return_probe;
+ }
+ ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
+ ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
+ fasttrap_pid_count++;
+ mutex_exit(&fasttrap_count_mtx);
+}
+
+static void
+fasttrap_disable_callbacks(void)
+{
+ ASSERT(MUTEX_HELD(&cpu_lock));
+
+ mutex_enter(&fasttrap_count_mtx);
+ ASSERT(fasttrap_pid_count > 0);
+ fasttrap_pid_count--;
+ if (fasttrap_pid_count == 0) {
+ cpu_t *cur, *cpu = CPU;
+
+ for (cur = cpu->cpu_next_onln; cur != cpu;
+ cur = cur->cpu_next_onln) {
+ rw_enter(&cur->cpu_ft_lock, RW_WRITER);
+ }
+
+ dtrace_pid_probe_ptr = NULL;
+ dtrace_return_probe_ptr = NULL;
+
+ for (cur = cpu->cpu_next_onln; cur != cpu;
+ cur = cur->cpu_next_onln) {
+ rw_exit(&cur->cpu_ft_lock);
+ }
+ }
+ mutex_exit(&fasttrap_count_mtx);
+}
+
+/*ARGSUSED*/
+static void
+fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
+{
+ fasttrap_probe_t *probe = parg;
+ proc_t *p;
+ int i, rc;
+
+ ASSERT(probe != NULL);
+ ASSERT(!probe->ftp_enabled);
+ ASSERT(id == probe->ftp_id);
+ ASSERT(MUTEX_HELD(&cpu_lock));
+
+ /*
+ * Increment the count of enabled probes on this probe's provider;
+ * the provider can't go away while the probe still exists. We
+ * must increment this even if we aren't able to properly enable
+ * this probe.
+ */
+ mutex_enter(&probe->ftp_prov->ftp_mtx);
+ probe->ftp_prov->ftp_rcount++;
+ mutex_exit(&probe->ftp_prov->ftp_mtx);
+
+ /*
+ * If this probe's provider is retired (meaning it was valid in a
+ * previously exec'ed incarnation of this address space), bail out. The
+ * provider can't go away while we're in this code path.
+ */
+ if (probe->ftp_prov->ftp_retired)
+ return;
+
+ /*
+ * If we can't find the process, it may be that we're in the context of
+ * a fork in which the traced process is being born and we're copying
+ * USDT probes. Otherwise, the process is gone so bail.
+ */
+ if ((p = sprlock(probe->ftp_pid)) == NULL) {
+ if ((curproc->p_flag & SFORKING) == 0)
+ return;
+
+ mutex_enter(&pidlock);
+ p = prfind(probe->ftp_pid);
+
+ /*
+ * Confirm that curproc is indeed forking the process in which
+ * we're trying to enable probes.
+ */
+ ASSERT(p != NULL);
+ ASSERT(p->p_parent == curproc);
+ ASSERT(p->p_stat == SIDL);
+
+ mutex_enter(&p->p_lock);
+ mutex_exit(&pidlock);
+
+ sprlock_proc(p);
+ }
+
+ ASSERT(!(p->p_flag & SVFORK));
+ mutex_exit(&p->p_lock);
+
+ /*
+ * We have to enable the trap entry point before any user threads have
+ * the chance to execute the trap instruction we're about to place
+ * in their process's text.
+ */
+ fasttrap_enable_callbacks();
+
+ /*
+ * Enable all the tracepoints and add this probe's id to each
+ * tracepoint's list of active probes.
+ */
+ for (i = 0; i < probe->ftp_ntps; i++) {
+ if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
+ /*
+ * If enabling the tracepoint failed completely,
+ * we don't have to disable it; if the failure
+ * was only partial we must disable it.
+ */
+ if (rc == FASTTRAP_ENABLE_FAIL)
+ i--;
+ else
+ ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
+
+ /*
+ * Back up and pull out all the tracepoints we've
+ * created so far for this probe.
+ */
+ while (i >= 0) {
+ fasttrap_tracepoint_disable(p, probe, i);
+ i--;
+ }
+
+ mutex_enter(&p->p_lock);
+ sprunlock(p);
+
+ /*
+ * Since we're not actually enabling this probe,
+ * drop our reference on the trap table entry.
+ */
+ fasttrap_disable_callbacks();
+ return;
+ }
+ }
+
+ mutex_enter(&p->p_lock);
+ sprunlock(p);
+
+ probe->ftp_enabled = 1;
+}
+
+/*ARGSUSED*/
+static void
+fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
+{
+ fasttrap_probe_t *probe = parg;
+ fasttrap_provider_t *provider = probe->ftp_prov;
+ proc_t *p;
+ int i, whack = 0;
+
+ ASSERT(id == probe->ftp_id);
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-user
mailing list