PERFORCE change 100004 for review
Kip Macy
kmacy at FreeBSD.org
Mon Jun 26 00:26:07 UTC 2006
http://perforce.freebsd.org/chv.cgi?CH=100004
Change 100004 by kmacy at kmacy_storage:sun4v_work_sleepq on 2006/06/26 00:24:59
hoist large functions out of line for mutex profiling this yields a ~8% speedup on sun4v
Affected files ...
.. //depot/projects/kmacy_sun4v/src/sys/kern/kern_mutex.c#19 edit
.. //depot/projects/kmacy_sun4v/src/sys/kern/subr_lock.c#3 edit
.. //depot/projects/kmacy_sun4v/src/sys/sys/lock_profile.h#7 edit
Differences ...
==== //depot/projects/kmacy_sun4v/src/sys/kern/kern_mutex.c#19 (text+ko) ====
@@ -257,13 +257,24 @@
if (v == 0)
return (0);
- bzero(mprof_buf, MPROF_HASH_SIZE);
+ bzero(mprof_buf, MPROF_HASH_SIZE*sizeof(*mprof_buf));
allocated_mprof_buf = 0;
return (0);
}
SYSCTL_PROC(_debug_mutex_prof, OID_AUTO, reset, CTLTYPE_INT | CTLFLAG_RW,
NULL, 0, reset_mutex_prof_stats, "I", "Reset mutex profiling statistics");
+
+static inline void mutex_profile_init(void)
+{
+ int i;
+ /* Initialize the mutex profiling locks */
+ for (i = 0; i < MPROF_LOCK_SIZE; i++) {
+ mtx_init(&mprof_locks[i], "mprof lock",
+ NULL, MTX_SPIN|MTX_QUIET|MTX_NOPROFILE);
+ }
+}
+
#endif
/*
@@ -521,22 +532,27 @@
_mtx_trylockn_spin(struct mtx *m, uintptr_t tid, int n, const char *file, int line)
{
uintptr_t _tid = (uintptr_t)(tid);
- int i, v;
+ int i, v, contested;
+ uint64_t waittime;
KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin,
("mtx_lock_spin() of sleep mutex %s @ %s:%d",
m->mtx_object.lo_name, file, line));
+ lock_profile_waitstart(&waittime);
for (i = 0, v = 0; i < n; i++) {
spinlock_enter();
v = _obtain_lock((m), _tid);
- if (!v)
+ if (!v) {
+ lock_profile_obtain_lock_failed(&m->mtx_object, &contested);
spinlock_exit();
- else {
+ } else {
WITNESS_LOCK(&m->mtx_object, LOP_EXCLUSIVE, file, line);
- break;
+ lock_profile_obtain_lock_success(&m->mtx_object, waittime, file, line);
+ return v;
}
}
+ lock_profile_update_wait(&m->mtx_object, waittime);
return v;
}
==== //depot/projects/kmacy_sun4v/src/sys/kern/subr_lock.c#3 (text+ko) ====
@@ -36,12 +36,14 @@
__FBSDID("$FreeBSD: src/sys/kern/subr_lock.c,v 1.4 2006/01/27 23:13:25 jhb Exp $");
#include "opt_ddb.h"
+#include "opt_mprof.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/ktr.h>
#include <sys/linker_set.h>
#include <sys/lock.h>
+#include <sys/lock_profile.h>
#ifdef DDB
#include <ddb/ddb.h>
@@ -55,6 +57,7 @@
&lock_class_sx,
&lock_class_rw,
};
+static u_int global_lock_id;
void
lock_init(struct lock_object *lock, struct lock_class *class, const char *name,
@@ -78,6 +81,7 @@
lock->lo_name = name;
lock->lo_type = type != NULL ? type : name;
lock->lo_flags |= flags | LO_INITIALIZED;
+ lock->lo_id = atomic_fetchadd_int(&global_lock_id, 1);
LOCK_LOG_INIT(lock, 0);
WITNESS_INIT(lock);
}
@@ -113,3 +117,151 @@
class->lc_ddb_show(lock);
}
#endif
+
+#ifdef MUTEX_PROFILING
+void _lock_profile_obtain_lock_success(struct lock_object *lo, uint64_t waittime, const char *file, int line)
+{
+ struct lock_profile_object *l = &lo->lo_profile_obj;
+
+ /* don't reset the timer when/if recursing */
+ if (l->lpo_acqtime == 0) {
+ l->lpo_filename = file;
+ l->lpo_lineno = line;
+ l->lpo_acqtime = rd(tick); /* substitute for more general TSC read */
+ if (waittime) {
+ if (l->lpo_acqtime > waittime)
+ l->lpo_waittime = l->lpo_acqtime - waittime;
+ }
+ }
+}
+
+void _lock_profile_update_wait(struct lock_object *lo, uint64_t waitstart)
+{
+ struct lock_profile_object *l = &lo->lo_profile_obj;
+
+ if (mutex_prof_enable && waitstart) {
+ uint64_t now, waittime;
+ struct mutex_prof *mpp;
+ u_int hash;
+ const char *p = l->lpo_filename;
+ int collision = 0;
+ now = rd(tick);
+ if (now < waitstart)
+ return;
+ waittime = now - waitstart;
+ hash = (l->lpo_namehash * 31 * 31 + (uintptr_t)p * 31 + l->lpo_lineno) & MPROF_HASH_MASK;
+
+ mpp = &mprof_buf[hash];
+ while (mpp->name != NULL) {
+ if (mpp->line == l->lpo_lineno &&
+ mpp->file == p &&
+ mpp->namehash == l->lpo_namehash)
+ break;
+ /* If the mprof_hash entry is allocated to someone else, try the next one */
+ collision = 1;
+ CTR4(KTR_SPARE1, "Hash collision, %s:%d %s(%x)", mpp->file, mpp->line, mpp->name, mpp->namehash);
+ hash = (hash + 1) & MPROF_HASH_MASK;
+ mpp = &mprof_buf[hash];
+ }
+ if (mpp->name == NULL) {
+ int buf;
+
+ buf = atomic_fetchadd_int(&allocated_mprof_buf, 1);
+ /* Just exit if we cannot get a trace buffer */
+ if (buf >= MPROF_HASH_SIZE) {
+ ++mutex_prof_rejected;
+ return;
+ }
+ mpp->file = p;
+ mpp->line = l->lpo_lineno;
+ mpp->name = lo->lo_name;
+ mpp->namehash = l->lpo_namehash;
+ if (collision)
+ ++mutex_prof_collisions;
+ /* We might have raced someone else but who cares, they'll try again next time */
+ ++mutex_prof_records;
+ }
+ MPROF_LOCK(hash);
+ mpp->cnt_wait += waittime;
+ MPROF_UNLOCK(hash);
+ }
+}
+
+void _lock_profile_release_lock(struct lock_object *lo)
+{
+ struct lock_profile_object *l = &lo->lo_profile_obj;
+
+ if (l->lpo_acqtime && !(lo->lo_flags & LO_NOPROFILE)) {
+ const char *unknown = "(unknown)";
+ u_int64_t acqtime, now, waittime;
+ struct mutex_prof *mpp;
+ u_int hash;
+ const char *p = l->lpo_filename;
+ int collision = 0;
+
+ now = rd(tick);
+ acqtime = l->lpo_acqtime;
+ waittime = l->lpo_waittime;
+ if (now <= acqtime)
+ return;
+ if (p == NULL || *p == '\0')
+ p = unknown;
+ hash = (l->lpo_namehash * 31 * 31 + (uintptr_t)p * 31 + l->lpo_lineno) & MPROF_HASH_MASK;
+ CTR5(KTR_SPARE1, "Hashing %s(%x) %s:%d to %d", l->lpo_name, l->lpo_namehash, p, l->lpo_lineno, hash);
+ mpp = &mprof_buf[hash];
+ while (mpp->name != NULL) {
+ if (mpp->line == l->lpo_lineno &&
+ mpp->file == p &&
+ mpp->namehash == l->lpo_namehash)
+ break;
+ /* If the mprof_hash entry is allocated to someone else, try the next one */
+ collision = 1;
+ CTR4(KTR_SPARE1, "Hash collision, %s:%d %s(%x)", mpp->file, mpp->line, mpp->name, mpp->namehash);
+ hash = (hash + 1) & MPROF_HASH_MASK;
+ mpp = &mprof_buf[hash];
+ }
+ if (mpp->name == NULL) {
+ int buf;
+
+ buf = atomic_fetchadd_int(&allocated_mprof_buf, 1);
+ /* Just exit if we cannot get a trace buffer */
+ if (buf >= MPROF_HASH_SIZE) {
+ ++mutex_prof_rejected;
+ return;
+ }
+ mpp->file = p;
+ mpp->line = l->lpo_lineno;
+ mpp->name = lo->lo_name;
+ mpp->namehash = l->lpo_namehash;
+ if (collision)
+ ++mutex_prof_collisions;
+ /* We might have raced someone else but who cares, they'll try again next time */
+ ++mutex_prof_records;
+ }
+ MPROF_LOCK(hash);
+ /*
+ * Record if the mutex has been held longer now than ever
+ * before.
+ */
+ if (now - acqtime > mpp->cnt_max)
+ mpp->cnt_max = now - acqtime;
+ mpp->cnt_tot += now - acqtime;
+ mpp->cnt_wait += waittime;
+ mpp->cnt_cur++;
+ /*
+ * There's a small race, really we should cmpxchg
+ * 0 with the current value, but that would bill
+ * the contention to the wrong lock instance if
+ * it followed this also.
+ */
+ mpp->cnt_contest_holding += l->lpo_contest_holding;
+ mpp->cnt_contest_locking += l->lpo_contest_locking;
+ MPROF_UNLOCK(hash);
+
+ }
+ l->lpo_acqtime = 0;
+ l->lpo_waittime = 0;
+ l->lpo_contest_locking = 0;
+ l->lpo_contest_holding = 0;
+}
+#endif
==== //depot/projects/kmacy_sun4v/src/sys/sys/lock_profile.h#7 (text+ko) ====
@@ -4,6 +4,7 @@
#ifdef MUTEX_PROFILING
#include <sys/stdint.h>
#include <sys/ktr.h>
+#include <sys/mutex.h>
#ifndef MPROF_HASH_SIZE
#define MPROF_HASH_SIZE 4096
@@ -40,6 +41,10 @@
extern int mutex_prof_rejected;
extern int mutex_prof_collisions;
+void _lock_profile_obtain_lock_success(struct lock_object *lo, uint64_t waittime, const char *file, int line);
+void _lock_profile_update_wait(struct lock_object *lo, uint64_t waitstart);
+void _lock_profile_release_lock(struct lock_object *lo);
+
static inline void lock_profile_init(struct lock_object *lo, const char *name) {
const char *p;
u_int hash = 0;
@@ -74,19 +79,10 @@
}
-static inline void mutex_profile_init(void)
-{
- int i;
- /* Initialize the mutex profiling locks */
- for (i = 0; i < MPROF_LOCK_SIZE; i++) {
- mtx_init(&mprof_locks[i], "mprof lock",
- NULL, MTX_SPIN|MTX_QUIET|MTX_NOPROFILE);
- }
-}
-
static inline void lock_profile_waitstart(uint64_t *waittime)
{
- *waittime = rd(tick);
+ if (mutex_prof_enable)
+ *waittime = rd(tick);
}
static inline void lock_profile_obtain_lock_failed(struct lock_object *lo, int *contested)
@@ -100,104 +96,31 @@
static inline void lock_profile_obtain_lock_success(struct lock_object *lo, uint64_t waittime, const char *file, int line)
{
- struct lock_profile_object *l = &lo->lo_profile_obj;
-
- /* don't reset the timer when/if recursing */
- if (mutex_prof_enable && l->lpo_acqtime == 0) {
- l->lpo_filename = file;
- l->lpo_lineno = line;
- l->lpo_acqtime = rd(tick); /* substitute for more general TSC read */
- if (waittime) {
- if (l->lpo_acqtime > waittime)
- l->lpo_waittime = l->lpo_acqtime - waittime;
- }
- }
+ if (mutex_prof_enable)
+ _lock_profile_obtain_lock_success(lo, waittime, file, line);
+}
+
+static inline void lock_profile_update_wait(struct lock_object *lo, uint64_t waitstart)
+{
+ if (mutex_prof_enable)
+ _lock_profile_update_wait(lo, waitstart);
}
-static inline void lock_profile_release_lock(struct lock_object *lo)
+static inline void lock_profile_release_lock(struct lock_object *lo)
{
struct lock_profile_object *l = &lo->lo_profile_obj;
-
- if (l->lpo_acqtime && !(lo->lo_flags & LO_NOPROFILE)) {
- const char *unknown = "(unknown)";
- struct mutex_prof *mpp;
- u_int64_t acqtime, now, waittime;
- u_int hash;
- const char *p = l->lpo_filename;
- int collision = 0;
-
- now = rd(tick);
- acqtime = l->lpo_acqtime;
- waittime = l->lpo_waittime;
- l->lpo_acqtime = 0;
- l->lpo_waittime = 0;
- if (now <= acqtime)
- return;
- if (p == NULL || *p == '\0')
- p = unknown;
- hash = (l->lpo_namehash * 31 * 31 + (uintptr_t)p * 31 + l->lpo_lineno) & MPROF_HASH_MASK;
- CTR5(KTR_SPARE1, "Hashing %s(%x) %s:%d to %d", l->lpo_name, l->lpo_namehash, p, l->lpo_lineno, hash);
- mpp = &mprof_buf[hash];
- while (mpp->name != NULL) {
- if (mpp->line == l->lpo_lineno &&
- mpp->file == p &&
- mpp->namehash == l->lpo_namehash)
- break;
- /* If the mprof_hash entry is allocated to someone else, try the next one */
- collision = 1;
- CTR4(KTR_SPARE1, "Hash collision, %s:%d %s(%x)", mpp->file, mpp->line, mpp->name, mpp->namehash);
- hash = (hash + 1) & MPROF_HASH_MASK;
- mpp = &mprof_buf[hash];
- }
- if (mpp->name == NULL) {
- int buf;
-
- buf = atomic_fetchadd_int(&allocated_mprof_buf, 1);
- /* Just exit if we cannot get a trace buffer */
- if (buf >= MPROF_HASH_SIZE) {
- ++mutex_prof_rejected;
- return;
- }
- mpp->file = p;
- mpp->line = l->lpo_lineno;
- mpp->name = lo->lo_name;
- mpp->namehash = l->lpo_namehash;
- if (collision)
- ++mutex_prof_collisions;
- /* We might have raced someone else but who cares, they'll try again next time */
- ++mutex_prof_records;
- }
- MPROF_LOCK(hash);
- /*
- * Record if the mutex has been held longer now than ever
- * before.
- */
- if (now - acqtime > mpp->cnt_max)
- mpp->cnt_max = now - acqtime;
- mpp->cnt_tot += now - acqtime;
- mpp->cnt_wait += waittime;
- mpp->cnt_cur++;
- /*
- * There's a small race, really we should cmpxchg
- * 0 with the current value, but that would bill
- * the contention to the wrong lock instance if
- * it followed this also.
- */
- mpp->cnt_contest_holding += l->lpo_contest_holding;
- l->lpo_contest_holding = 0;
- mpp->cnt_contest_locking += l->lpo_contest_locking;
- l->lpo_contest_locking = 0;
- MPROF_UNLOCK(hash);
- }
+ if (mutex_prof_enable || l->lpo_acqtime)
+ _lock_profile_release_lock(lo);
}
#else /* !MUTEX_PROFILING */
static inline void mutex_profile_init(void) {;}
static inline void lock_profile_init(struct lock_object *lo, const char *name) {;}
+static inline void lock_profile_destroy(struct lock_object *lo) {;}
static inline void lock_profile_obtain_lock_failed(struct lock_object *m, int *contested) {;}
static inline void lock_profile_obtain_lock_success(struct lock_object *m, const char *file, int line) {;}
+static inline void lock_profile_update_wait(struct lock_object *lo, uint64_t waitstart) {;}
static inline void lock_profile_release_lock(struct lock_object *m) {;}
-static inline void lock_profile_destroy(struct lock_object *lo) {;}
#endif /* !MUTEX_PROFILING */
#endif /* _SYS_LOCK_PROFILE_H_ */
More information about the p4-projects
mailing list