svn commit: r326106 - in head/sys: kern sys
Mateusz Guzik
mjg at FreeBSD.org
Wed Nov 22 21:51:19 UTC 2017
Author: mjg
Date: Wed Nov 22 21:51:17 2017
New Revision: 326106
URL: https://svnweb.freebsd.org/changeset/base/326106
Log:
locks: remove the file + line argument from internal primitives when not used
The pair is of use only in debug or LOCKPROF kernels, but was passed (zeroed)
for many locks even in production kernels.
While here whack the tid argument from wlock hard and xlock hard.
There is no kbi change of any sort - "external" primitives still accept the
pair.
Modified:
head/sys/kern/kern_mutex.c
head/sys/kern/kern_rwlock.c
head/sys/kern/kern_sx.c
head/sys/sys/lock.h
head/sys/sys/mutex.h
head/sys/sys/rwlock.h
head/sys/sys/sx.h
Modified: head/sys/kern/kern_mutex.c
==============================================================================
--- head/sys/kern/kern_mutex.c Wed Nov 22 21:24:47 2017 (r326105)
+++ head/sys/kern/kern_mutex.c Wed Nov 22 21:51:17 2017 (r326106)
@@ -380,9 +380,8 @@ __mtx_unlock_spin_flags(volatile uintptr_t *c, int opt
* is already owned, it will recursively acquire the lock.
*/
int
-_mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line)
+_mtx_trylock_flags_int(struct mtx *m, int opts LOCK_FILE_LINE_ARG_DEF)
{
- struct mtx *m;
struct thread *td;
uintptr_t tid, v;
#ifdef LOCK_PROFILING
@@ -397,8 +396,6 @@ _mtx_trylock_flags_(volatile uintptr_t *c, int opts, c
if (SCHEDULER_STOPPED_TD(td))
return (1);
- m = mtxlock2mtx(c);
-
KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(td),
("mtx_trylock() by idle thread %p on sleep mutex %s @ %s:%d",
curthread, m->lock_object.lo_name, file, line));
@@ -441,6 +438,15 @@ _mtx_trylock_flags_(volatile uintptr_t *c, int opts, c
}
return (rval);
+}
+
+int
+_mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line)
+{
+ struct mtx *m;
+
+ m = mtxlock2mtx(c);
+ return (_mtx_trylock_flags_int(m, opts LOCK_FILE_LINE_ARG));
}
/*
Modified: head/sys/kern/kern_rwlock.c
==============================================================================
--- head/sys/kern/kern_rwlock.c Wed Nov 22 21:24:47 2017 (r326105)
+++ head/sys/kern/kern_rwlock.c Wed Nov 22 21:51:17 2017 (r326106)
@@ -273,7 +273,7 @@ _rw_wlock_cookie(volatile uintptr_t *c, const char *fi
tid = (uintptr_t)curthread;
v = RW_UNLOCKED;
if (!_rw_write_lock_fetch(rw, &v, tid))
- _rw_wlock_hard(rw, v, tid, file, line);
+ _rw_wlock_hard(rw, v, file, line);
else
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw,
0, 0, file, line, LOCKSTAT_WRITER);
@@ -369,8 +369,8 @@ _rw_wunlock_cookie(volatile uintptr_t *c, const char *
RW_LOCK_READ || ((td)->td_rw_rlocks && (_rw) & RW_LOCK_READ))
static bool __always_inline
-__rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp,
- const char *file, int line)
+__rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp
+ LOCK_FILE_LINE_ARG_DEF)
{
/*
@@ -399,10 +399,9 @@ __rw_rlock_try(struct rwlock *rw, struct thread *td, u
}
static void __noinline
-__rw_rlock_hard(volatile uintptr_t *c, struct thread *td, uintptr_t v,
- const char *file, int line)
+__rw_rlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v
+ LOCK_FILE_LINE_ARG_DEF)
{
- struct rwlock *rw;
struct turnstile *ts;
#ifdef ADAPTIVE_RWLOCKS
volatile struct thread *owner;
@@ -434,7 +433,6 @@ __rw_rlock_hard(volatile uintptr_t *c, struct thread *
#elif defined(KDTRACE_HOOKS)
lock_delay_arg_init(&lda, NULL);
#endif
- rw = rwlock2rw(c);
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
@@ -454,7 +452,7 @@ __rw_rlock_hard(volatile uintptr_t *c, struct thread *
#endif
for (;;) {
- if (__rw_rlock_try(rw, td, &v, file, line))
+ if (__rw_rlock_try(rw, td, &v LOCK_FILE_LINE_ARG))
break;
#ifdef KDTRACE_HOOKS
lda.spin_cnt++;
@@ -612,14 +610,12 @@ __rw_rlock_hard(volatile uintptr_t *c, struct thread *
}
void
-__rw_rlock(volatile uintptr_t *c, const char *file, int line)
+__rw_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
{
- struct rwlock *rw;
struct thread *td;
uintptr_t v;
td = curthread;
- rw = rwlock2rw(c);
KASSERT(kdb_active != 0 || SCHEDULER_STOPPED_TD(td) ||
!TD_IS_IDLETHREAD(td),
@@ -634,14 +630,23 @@ __rw_rlock(volatile uintptr_t *c, const char *file, in
v = RW_READ_VALUE(rw);
if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(rw__acquire) ||
- !__rw_rlock_try(rw, td, &v, file, line)))
- __rw_rlock_hard(c, td, v, file, line);
+ !__rw_rlock_try(rw, td, &v LOCK_FILE_LINE_ARG)))
+ __rw_rlock_hard(rw, td, v LOCK_FILE_LINE_ARG);
LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);
WITNESS_LOCK(&rw->lock_object, 0, file, line);
TD_LOCKS_INC(curthread);
}
+void
+__rw_rlock(volatile uintptr_t *c, const char *file, int line)
+{
+ struct rwlock *rw;
+
+ rw = rwlock2rw(c);
+ __rw_rlock_int(rw LOCK_FILE_LINE_ARG);
+}
+
int
__rw_try_rlock(volatile uintptr_t *c, const char *file, int line)
{
@@ -724,18 +729,15 @@ __rw_runlock_try(struct rwlock *rw, struct thread *td,
}
static void __noinline
-__rw_runlock_hard(volatile uintptr_t *c, struct thread *td, uintptr_t v,
- const char *file, int line)
+__rw_runlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v
+ LOCK_FILE_LINE_ARG_DEF)
{
- struct rwlock *rw;
struct turnstile *ts;
uintptr_t x, queue;
if (SCHEDULER_STOPPED())
return;
- rw = rwlock2rw(c);
-
for (;;) {
if (__rw_runlock_try(rw, td, &v))
break;
@@ -799,17 +801,14 @@ __rw_runlock_hard(volatile uintptr_t *c, struct thread
}
void
-_rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
+_rw_runlock_cookie_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
{
- struct rwlock *rw;
struct thread *td;
uintptr_t v;
- rw = rwlock2rw(c);
-
KASSERT(rw->rw_lock != RW_DESTROYED,
("rw_runlock() of destroyed rwlock @ %s:%d", file, line));
- __rw_assert(c, RA_RLOCKED, file, line);
+ __rw_assert(&rw->rw_lock, RA_RLOCKED, file, line);
WITNESS_UNLOCK(&rw->lock_object, 0, file, line);
LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line);
@@ -818,20 +817,29 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *
if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(rw__release) ||
!__rw_runlock_try(rw, td, &v)))
- __rw_runlock_hard(c, td, v, file, line);
+ __rw_runlock_hard(rw, td, v LOCK_FILE_LINE_ARG);
TD_LOCKS_DEC(curthread);
}
+void
+_rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
+{
+ struct rwlock *rw;
+
+ rw = rwlock2rw(c);
+ _rw_runlock_cookie_int(rw LOCK_FILE_LINE_ARG);
+}
+
/*
* This function is called when we are unable to obtain a write lock on the
* first try. This means that at least one other thread holds either a
* read or write lock.
*/
void
-__rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
- const char *file, int line)
+__rw_wlock_hard(volatile uintptr_t *c, uintptr_t v LOCK_FILE_LINE_ARG_DEF)
{
+ uintptr_t tid;
struct rwlock *rw;
struct turnstile *ts;
#ifdef ADAPTIVE_RWLOCKS
@@ -857,6 +865,7 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, ui
int doing_lockprof;
#endif
+ tid = (uintptr_t)curthread;
if (SCHEDULER_STOPPED())
return;
@@ -1069,8 +1078,7 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, ui
* on this lock.
*/
void
-__rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
- int line)
+__rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid LOCK_FILE_LINE_ARG_DEF)
{
struct rwlock *rw;
struct turnstile *ts;
@@ -1145,9 +1153,8 @@ __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid
* lock. Returns true if the upgrade succeeded and false otherwise.
*/
int
-__rw_try_upgrade(volatile uintptr_t *c, const char *file, int line)
+__rw_try_upgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
{
- struct rwlock *rw;
uintptr_t v, x, tid;
struct turnstile *ts;
int success;
@@ -1155,11 +1162,9 @@ __rw_try_upgrade(volatile uintptr_t *c, const char *fi
if (SCHEDULER_STOPPED())
return (1);
- rw = rwlock2rw(c);
-
KASSERT(rw->rw_lock != RW_DESTROYED,
("rw_try_upgrade() of destroyed rwlock @ %s:%d", file, line));
- __rw_assert(c, RA_RLOCKED, file, line);
+ __rw_assert(&rw->rw_lock, RA_RLOCKED, file, line);
/*
* Attempt to switch from one reader to a writer. If there
@@ -1217,13 +1222,21 @@ __rw_try_upgrade(volatile uintptr_t *c, const char *fi
return (success);
}
+int
+__rw_try_upgrade(volatile uintptr_t *c, const char *file, int line)
+{
+ struct rwlock *rw;
+
+ rw = rwlock2rw(c);
+ return (__rw_try_upgrade_int(rw LOCK_FILE_LINE_ARG));
+}
+
/*
* Downgrade a write lock into a single read lock.
*/
void
-__rw_downgrade(volatile uintptr_t *c, const char *file, int line)
+__rw_downgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
{
- struct rwlock *rw;
struct turnstile *ts;
uintptr_t tid, v;
int rwait, wwait;
@@ -1231,11 +1244,9 @@ __rw_downgrade(volatile uintptr_t *c, const char *file
if (SCHEDULER_STOPPED())
return;
- rw = rwlock2rw(c);
-
KASSERT(rw->rw_lock != RW_DESTROYED,
("rw_downgrade() of destroyed rwlock @ %s:%d", file, line));
- __rw_assert(c, RA_WLOCKED | RA_NOTRECURSED, file, line);
+ __rw_assert(&rw->rw_lock, RA_WLOCKED | RA_NOTRECURSED, file, line);
#ifndef INVARIANTS
if (rw_recursed(rw))
panic("downgrade of a recursed lock");
@@ -1285,6 +1296,15 @@ out:
curthread->td_rw_rlocks++;
LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line);
LOCKSTAT_RECORD0(rw__downgrade, rw);
+}
+
+void
+__rw_downgrade(volatile uintptr_t *c, const char *file, int line)
+{
+ struct rwlock *rw;
+
+ rw = rwlock2rw(c);
+ __rw_downgrade_int(rw LOCK_FILE_LINE_ARG);
}
#ifdef INVARIANT_SUPPORT
Modified: head/sys/kern/kern_sx.c
==============================================================================
--- head/sys/kern/kern_sx.c Wed Nov 22 21:24:47 2017 (r326105)
+++ head/sys/kern/kern_sx.c Wed Nov 22 21:51:17 2017 (r326106)
@@ -259,7 +259,7 @@ sx_destroy(struct sx *sx)
}
int
-sx_try_slock_(struct sx *sx, const char *file, int line)
+sx_try_slock_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t x;
@@ -291,6 +291,13 @@ sx_try_slock_(struct sx *sx, const char *file, int lin
}
int
+sx_try_slock_(struct sx *sx, const char *file, int line)
+{
+
+ return (sx_try_slock_int(sx LOCK_FILE_LINE_ARG));
+}
+
+int
_sx_xlock(struct sx *sx, int opts, const char *file, int line)
{
uintptr_t tid, x;
@@ -307,7 +314,7 @@ _sx_xlock(struct sx *sx, int opts, const char *file, i
tid = (uintptr_t)curthread;
x = SX_LOCK_UNLOCKED;
if (!atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid))
- error = _sx_xlock_hard(sx, x, tid, opts, file, line);
+ error = _sx_xlock_hard(sx, x, opts LOCK_FILE_LINE_ARG);
else
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
0, 0, file, line, LOCKSTAT_WRITER);
@@ -322,7 +329,7 @@ _sx_xlock(struct sx *sx, int opts, const char *file, i
}
int
-sx_try_xlock_(struct sx *sx, const char *file, int line)
+sx_try_xlock_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF)
{
struct thread *td;
uintptr_t tid, x;
@@ -370,6 +377,13 @@ sx_try_xlock_(struct sx *sx, const char *file, int lin
return (rval);
}
+int
+sx_try_xlock_(struct sx *sx, const char *file, int line)
+{
+
+ return (sx_try_xlock_int(sx LOCK_FILE_LINE_ARG));
+}
+
void
_sx_xunlock(struct sx *sx, const char *file, int line)
{
@@ -394,7 +408,7 @@ _sx_xunlock(struct sx *sx, const char *file, int line)
* Return 1 if if the upgrade succeed, 0 otherwise.
*/
int
-sx_try_upgrade_(struct sx *sx, const char *file, int line)
+sx_try_upgrade_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t x;
int success;
@@ -423,11 +437,18 @@ sx_try_upgrade_(struct sx *sx, const char *file, int l
return (success);
}
+int
+sx_try_upgrade_(struct sx *sx, const char *file, int line)
+{
+
+ return (sx_try_upgrade_int(sx LOCK_FILE_LINE_ARG));
+}
+
/*
* Downgrade an unrecursed exclusive lock into a single shared lock.
*/
void
-sx_downgrade_(struct sx *sx, const char *file, int line)
+sx_downgrade_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t x;
int wakeup_swapper;
@@ -490,6 +511,13 @@ sx_downgrade_(struct sx *sx, const char *file, int lin
kick_proc0();
}
+void
+sx_downgrade_(struct sx *sx, const char *file, int line)
+{
+
+ sx_downgrade_int(sx LOCK_FILE_LINE_ARG);
+}
+
/*
* This function represents the so-called 'hard case' for sx_xlock
* operation. All 'easy case' failures are redirected to this. Note
@@ -497,10 +525,10 @@ sx_downgrade_(struct sx *sx, const char *file, int lin
* accessible from at least sx.h.
*/
int
-_sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t tid, int opts,
- const char *file, int line)
+_sx_xlock_hard(struct sx *sx, uintptr_t x, int opts LOCK_FILE_LINE_ARG_DEF)
{
GIANT_DECLARE;
+ uintptr_t tid;
#ifdef ADAPTIVE_SX
volatile struct thread *owner;
u_int i, n, spintries = 0;
@@ -523,6 +551,7 @@ _sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t t
#endif
int extra_work = 0;
+ tid = (uintptr_t)curthread;
if (SCHEDULER_STOPPED())
return (0);
@@ -770,7 +799,7 @@ _sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t t
* accessible from at least sx.h.
*/
void
-_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
+_sx_xunlock_hard(struct sx *sx, uintptr_t tid LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t x, setx;
int queue, wakeup_swapper;
@@ -835,7 +864,7 @@ _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const c
}
static bool __always_inline
-__sx_slock_try(struct sx *sx, uintptr_t *xp, const char *file, int line)
+__sx_slock_try(struct sx *sx, uintptr_t *xp LOCK_FILE_LINE_ARG_DEF)
{
/*
@@ -859,7 +888,7 @@ __sx_slock_try(struct sx *sx, uintptr_t *xp, const cha
}
static int __noinline
-_sx_slock_hard(struct sx *sx, int opts, const char *file, int line, uintptr_t x)
+_sx_slock_hard(struct sx *sx, int opts, uintptr_t x LOCK_FILE_LINE_ARG_DEF)
{
GIANT_DECLARE;
#ifdef ADAPTIVE_SX
@@ -914,7 +943,7 @@ _sx_slock_hard(struct sx *sx, int opts, const char *fi
* shared locks once there is an exclusive waiter.
*/
for (;;) {
- if (__sx_slock_try(sx, &x, file, line))
+ if (__sx_slock_try(sx, &x LOCK_FILE_LINE_ARG))
break;
#ifdef KDTRACE_HOOKS
lda.spin_cnt++;
@@ -1058,7 +1087,7 @@ _sx_slock_hard(struct sx *sx, int opts, const char *fi
}
int
-_sx_slock(struct sx *sx, int opts, const char *file, int line)
+_sx_slock_int(struct sx *sx, int opts LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t x;
int error;
@@ -1074,8 +1103,8 @@ _sx_slock(struct sx *sx, int opts, const char *file, i
error = 0;
x = SX_READ_VALUE(sx);
if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(sx__acquire) ||
- !__sx_slock_try(sx, &x, file, line)))
- error = _sx_slock_hard(sx, opts, file, line, x);
+ !__sx_slock_try(sx, &x LOCK_FILE_LINE_ARG)))
+ error = _sx_slock_hard(sx, opts, x LOCK_FILE_LINE_ARG);
if (error == 0) {
LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line);
WITNESS_LOCK(&sx->lock_object, 0, file, line);
@@ -1084,6 +1113,13 @@ _sx_slock(struct sx *sx, int opts, const char *file, i
return (error);
}
+int
+_sx_slock(struct sx *sx, int opts, const char *file, int line)
+{
+
+ return (_sx_slock_int(sx, opts LOCK_FILE_LINE_ARG));
+}
+
static bool __always_inline
_sx_sunlock_try(struct sx *sx, uintptr_t *xp)
{
@@ -1135,7 +1171,7 @@ _sx_sunlock_try(struct sx *sx, uintptr_t *xp)
}
static void __noinline
-_sx_sunlock_hard(struct sx *sx, uintptr_t x, const char *file, int line)
+_sx_sunlock_hard(struct sx *sx, uintptr_t x LOCK_FILE_LINE_ARG_DEF)
{
int wakeup_swapper;
@@ -1182,7 +1218,7 @@ _sx_sunlock_hard(struct sx *sx, uintptr_t x, const cha
}
void
-_sx_sunlock(struct sx *sx, const char *file, int line)
+_sx_sunlock_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t x;
@@ -1195,9 +1231,16 @@ _sx_sunlock(struct sx *sx, const char *file, int line)
x = SX_READ_VALUE(sx);
if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(sx__release) ||
!_sx_sunlock_try(sx, &x)))
- _sx_sunlock_hard(sx, x, file, line);
+ _sx_sunlock_hard(sx, x LOCK_FILE_LINE_ARG);
TD_LOCKS_DEC(curthread);
+}
+
+void
+_sx_sunlock(struct sx *sx, const char *file, int line)
+{
+
+ _sx_sunlock_int(sx LOCK_FILE_LINE_ARG);
}
#ifdef INVARIANT_SUPPORT
Modified: head/sys/sys/lock.h
==============================================================================
--- head/sys/sys/lock.h Wed Nov 22 21:24:47 2017 (r326105)
+++ head/sys/sys/lock.h Wed Nov 22 21:51:17 2017 (r326106)
@@ -137,9 +137,13 @@ struct lock_class {
* operations. Otherwise, use default values to avoid the unneeded bloat.
*/
#if LOCK_DEBUG > 0
+#define LOCK_FILE_LINE_ARG_DEF , const char *file, int line
+#define LOCK_FILE_LINE_ARG , file, line
#define LOCK_FILE __FILE__
#define LOCK_LINE __LINE__
#else
+#define LOCK_FILE_LINE_ARG_DEF
+#define LOCK_FILE_LINE_ARG
#define LOCK_FILE NULL
#define LOCK_LINE 0
#endif
Modified: head/sys/sys/mutex.h
==============================================================================
--- head/sys/sys/mutex.h Wed Nov 22 21:24:47 2017 (r326105)
+++ head/sys/sys/mutex.h Wed Nov 22 21:51:17 2017 (r326106)
@@ -91,6 +91,7 @@ void _mtx_init(volatile uintptr_t *c, const char *name
int opts);
void _mtx_destroy(volatile uintptr_t *c);
void mtx_sysinit(void *arg);
+int _mtx_trylock_flags_int(struct mtx *m, int opts LOCK_FILE_LINE_ARG_DEF);
int _mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file,
int line);
void mutex_init(void);
Modified: head/sys/sys/rwlock.h
==============================================================================
--- head/sys/sys/rwlock.h Wed Nov 22 21:24:47 2017 (r326105)
+++ head/sys/sys/rwlock.h Wed Nov 22 21:51:17 2017 (r326106)
@@ -105,7 +105,7 @@
\
if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__acquire) || \
!_rw_write_lock_fetch((rw), &_v, _tid))) \
- _rw_wlock_hard((rw), _v, _tid, (file), (line)); \
+ _rw_wlock_hard((rw), _v, (file), (line)); \
} while (0)
/* Release a write lock. */
@@ -128,16 +128,22 @@ void rw_sysinit(void *arg);
void rw_sysinit_flags(void *arg);
int _rw_wowned(const volatile uintptr_t *c);
void _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line);
+int __rw_try_wlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
int __rw_try_wlock(volatile uintptr_t *c, const char *file, int line);
void _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line);
+void __rw_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
void __rw_rlock(volatile uintptr_t *c, const char *file, int line);
+int __rw_try_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
int __rw_try_rlock(volatile uintptr_t *c, const char *file, int line);
+void _rw_runlock_cookie_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
void _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line);
-void __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
- const char *file, int line);
-void __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid,
- const char *file, int line);
+void __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v
+ LOCK_FILE_LINE_ARG_DEF);
+void __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t v
+ LOCK_FILE_LINE_ARG_DEF);
+int __rw_try_upgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
int __rw_try_upgrade(volatile uintptr_t *c, const char *file, int line);
+void __rw_downgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
void __rw_downgrade(volatile uintptr_t *c, const char *file, int line);
#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
void __rw_assert(const volatile uintptr_t *c, int what, const char *file,
@@ -163,20 +169,38 @@ void __rw_assert(const volatile uintptr_t *c, int what
__rw_try_wlock(&(rw)->rw_lock, f, l)
#define _rw_wunlock(rw, f, l) \
_rw_wunlock_cookie(&(rw)->rw_lock, f, l)
-#define _rw_rlock(rw, f, l) \
- __rw_rlock(&(rw)->rw_lock, f, l)
#define _rw_try_rlock(rw, f, l) \
__rw_try_rlock(&(rw)->rw_lock, f, l)
+#if LOCK_DEBUG > 0
+#define _rw_rlock(rw, f, l) \
+ __rw_rlock(&(rw)->rw_lock, f, l)
#define _rw_runlock(rw, f, l) \
_rw_runlock_cookie(&(rw)->rw_lock, f, l)
-#define _rw_wlock_hard(rw, v, t, f, l) \
- __rw_wlock_hard(&(rw)->rw_lock, v, t, f, l)
-#define _rw_wunlock_hard(rw, t, f, l) \
- __rw_wunlock_hard(&(rw)->rw_lock, t, f, l)
+#else
+#define _rw_rlock(rw, f, l) \
+ __rw_rlock_int((struct rwlock *)rw)
+#define _rw_runlock(rw, f, l) \
+ _rw_runlock_cookie_int((struct rwlock *)rw)
+#endif
+#if LOCK_DEBUG > 0
+#define _rw_wlock_hard(rw, v, f, l) \
+ __rw_wlock_hard(&(rw)->rw_lock, v, f, l)
+#define _rw_wunlock_hard(rw, v, f, l) \
+ __rw_wunlock_hard(&(rw)->rw_lock, v, f, l)
#define _rw_try_upgrade(rw, f, l) \
__rw_try_upgrade(&(rw)->rw_lock, f, l)
#define _rw_downgrade(rw, f, l) \
__rw_downgrade(&(rw)->rw_lock, f, l)
+#else
+#define _rw_wlock_hard(rw, v, f, l) \
+ __rw_wlock_hard(&(rw)->rw_lock, v)
+#define _rw_wunlock_hard(rw, v, f, l) \
+ __rw_wunlock_hard(&(rw)->rw_lock, v)
+#define _rw_try_upgrade(rw, f, l) \
+ __rw_try_upgrade_int(rw)
+#define _rw_downgrade(rw, f, l) \
+ __rw_downgrade_int(rw)
+#endif
#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
#define _rw_assert(rw, w, f, l) \
__rw_assert(&(rw)->rw_lock, w, f, l)
Modified: head/sys/sys/sx.h
==============================================================================
--- head/sys/sys/sx.h Wed Nov 22 21:24:47 2017 (r326105)
+++ head/sys/sys/sx.h Wed Nov 22 21:51:17 2017 (r326106)
@@ -101,18 +101,22 @@ void sx_sysinit(void *arg);
#define sx_init(sx, desc) sx_init_flags((sx), (desc), 0)
void sx_init_flags(struct sx *sx, const char *description, int opts);
void sx_destroy(struct sx *sx);
+int sx_try_slock_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF);
int sx_try_slock_(struct sx *sx, const char *file, int line);
+int sx_try_xlock_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF);
int sx_try_xlock_(struct sx *sx, const char *file, int line);
+int sx_try_upgrade_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF);
int sx_try_upgrade_(struct sx *sx, const char *file, int line);
+void sx_downgrade_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF);
void sx_downgrade_(struct sx *sx, const char *file, int line);
+int _sx_slock_int(struct sx *sx, int opts LOCK_FILE_LINE_ARG_DEF);
int _sx_slock(struct sx *sx, int opts, const char *file, int line);
int _sx_xlock(struct sx *sx, int opts, const char *file, int line);
+void _sx_sunlock_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF);
void _sx_sunlock(struct sx *sx, const char *file, int line);
void _sx_xunlock(struct sx *sx, const char *file, int line);
-int _sx_xlock_hard(struct sx *sx, uintptr_t v, uintptr_t tid, int opts,
- const char *file, int line);
-void _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int
- line);
+int _sx_xlock_hard(struct sx *sx, uintptr_t x, int opts LOCK_FILE_LINE_ARG_DEF);
+void _sx_xunlock_hard(struct sx *sx, uintptr_t tid LOCK_FILE_LINE_ARG_DEF);
#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
void _sx_assert(const struct sx *sx, int what, const char *file, int line);
#endif
@@ -157,7 +161,7 @@ __sx_xlock(struct sx *sx, struct thread *td, int opts,
if (__predict_false(LOCKSTAT_PROFILE_ENABLED(sx__acquire) ||
!atomic_fcmpset_acq_ptr(&sx->sx_lock, &v, tid)))
- error = _sx_xlock_hard(sx, v, tid, opts, file, line);
+ error = _sx_xlock_hard(sx, v, opts);
return (error);
}
@@ -170,7 +174,7 @@ __sx_xunlock(struct sx *sx, struct thread *td, const c
if (__predict_false(LOCKSTAT_PROFILE_ENABLED(sx__release) ||
!atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED)))
- _sx_xunlock_hard(sx, tid, file, line);
+ _sx_xunlock_hard(sx, tid);
}
#endif
@@ -195,6 +199,7 @@ __sx_xunlock(struct sx *sx, struct thread *td, const c
#define sx_xunlock_(sx, file, line) \
__sx_xunlock((sx), curthread, (file), (line))
#endif /* LOCK_DEBUG > 0 || SX_NOINLINE */
+#if (LOCK_DEBUG > 0)
#define sx_slock_(sx, file, line) \
(void)_sx_slock((sx), 0, (file), (line))
#define sx_slock_sig_(sx, file, line) \
@@ -205,6 +210,18 @@ __sx_xunlock(struct sx *sx, struct thread *td, const c
#define sx_try_xlock(sx) sx_try_xlock_((sx), LOCK_FILE, LOCK_LINE)
#define sx_try_upgrade(sx) sx_try_upgrade_((sx), LOCK_FILE, LOCK_LINE)
#define sx_downgrade(sx) sx_downgrade_((sx), LOCK_FILE, LOCK_LINE)
+#else
+#define sx_slock_(sx, file, line) \
+ (void)_sx_slock_int((sx), 0)
+#define sx_slock_sig_(sx, file, line) \
+ _sx_slock_int((sx), SX_INTERRUPTIBLE)
+#define sx_sunlock_(sx, file, line) \
+ _sx_sunlock_int((sx))
+#define sx_try_slock(sx) sx_try_slock_int((sx))
+#define sx_try_xlock(sx) sx_try_xlock_int((sx))
+#define sx_try_upgrade(sx) sx_try_upgrade_int((sx))
+#define sx_downgrade(sx) sx_downgrade_int((sx))
+#endif
#ifdef INVARIANTS
#define sx_assert_(sx, what, file, line) \
_sx_assert((sx), (what), (file), (line))
More information about the svn-src-head
mailing list