svn commit: r273785 - head/sys/kern
Konstantin Belousov
kib at FreeBSD.org
Tue Oct 28 15:30:34 UTC 2014
Author: kib
Date: Tue Oct 28 15:30:33 2014
New Revision: 273785
URL: https://svnweb.freebsd.org/changeset/base/273785
Log:
Convert kern_umtx.c to use fueword() and casueword().
Also fix some mishandling of suword(9) errors as errno, which resulted
in spurious ERESTART.
Sponsored by: The FreeBSD Foundation
Tested by: pho
MFC after: 3 weeks
Modified:
head/sys/kern/kern_umtx.c
Modified: head/sys/kern/kern_umtx.c
==============================================================================
--- head/sys/kern/kern_umtx.c Tue Oct 28 15:28:20 2014 (r273784)
+++ head/sys/kern/kern_umtx.c Tue Oct 28 15:30:33 2014 (r273785)
@@ -510,6 +510,15 @@ umtxq_unbusy(struct umtx_key *key)
wakeup_one(uc);
}
+static inline void
+umtxq_unbusy_unlocked(struct umtx_key *key)
+{
+
+ umtxq_lock(key);
+ umtxq_unbusy(key);
+ umtxq_unlock(key);
+}
+
static struct umtxq_queue *
umtxq_queue_lookup(struct umtx_key *key, int q)
{
@@ -847,6 +856,7 @@ do_wait(struct thread *td, void *addr, u
struct abs_timeout timo;
struct umtx_q *uq;
u_long tmp;
+ uint32_t tmp32;
int error = 0;
uq = td->td_umtxq;
@@ -860,18 +870,29 @@ do_wait(struct thread *td, void *addr, u
umtxq_lock(&uq->uq_key);
umtxq_insert(uq);
umtxq_unlock(&uq->uq_key);
- if (compat32 == 0)
- tmp = fuword(addr);
- else
- tmp = (unsigned int)fuword32(addr);
+ if (compat32 == 0) {
+ error = fueword(addr, &tmp);
+ if (error != 0)
+ error = EFAULT;
+ } else {
+ error = fueword32(addr, &tmp32);
+ if (error == 0)
+ tmp = tmp32;
+ else
+ error = EFAULT;
+ }
umtxq_lock(&uq->uq_key);
- if (tmp == id)
- error = umtxq_sleep(uq, "uwait", timeout == NULL ?
- NULL : &timo);
- if ((uq->uq_flags & UQF_UMTXQ) == 0)
- error = 0;
- else
+ if (error == 0) {
+ if (tmp == id)
+ error = umtxq_sleep(uq, "uwait", timeout == NULL ?
+ NULL : &timo);
+ if ((uq->uq_flags & UQF_UMTXQ) == 0)
+ error = 0;
+ else
+ umtxq_remove(uq);
+ } else if ((uq->uq_flags & UQF_UMTXQ) != 0) {
umtxq_remove(uq);
+ }
umtxq_unlock(&uq->uq_key);
umtx_key_release(&uq->uq_key);
if (error == ERESTART)
@@ -908,11 +929,11 @@ do_lock_normal(struct thread *td, struct
struct abs_timeout timo;
struct umtx_q *uq;
uint32_t owner, old, id;
- int error = 0;
+ int error, rv;
id = td->td_tid;
uq = td->td_umtxq;
-
+ error = 0;
if (timeout != NULL)
abs_timeout_init2(&timo, timeout);
@@ -921,7 +942,9 @@ do_lock_normal(struct thread *td, struct
* can fault on any access.
*/
for (;;) {
- owner = fuword32(__DEVOLATILE(void *, &m->m_owner));
+ rv = fueword32(__DEVOLATILE(void *, &m->m_owner), &owner);
+ if (rv == -1)
+ return (EFAULT);
if (mode == _UMUTEX_WAIT) {
if (owner == UMUTEX_UNOWNED || owner == UMUTEX_CONTESTED)
return (0);
@@ -929,31 +952,31 @@ do_lock_normal(struct thread *td, struct
/*
* Try the uncontested case. This should be done in userland.
*/
- owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
+ rv = casueword32(&m->m_owner, UMUTEX_UNOWNED,
+ &owner, id);
+ /* The address was invalid. */
+ if (rv == -1)
+ return (EFAULT);
/* The acquire succeeded. */
if (owner == UMUTEX_UNOWNED)
return (0);
- /* The address was invalid. */
- if (owner == -1)
- return (EFAULT);
-
/* If no one owns it but it is contested try to acquire it. */
if (owner == UMUTEX_CONTESTED) {
- owner = casuword32(&m->m_owner,
- UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
+ rv = casueword32(&m->m_owner,
+ UMUTEX_CONTESTED, &owner,
+ id | UMUTEX_CONTESTED);
+ /* The address was invalid. */
+ if (rv == -1)
+ return (EFAULT);
if (owner == UMUTEX_CONTESTED)
return (0);
- /* The address was invalid. */
- if (owner == -1)
- return (EFAULT);
-
- error = umtxq_check_susp(td);
- if (error != 0)
- return (error);
+ rv = umtxq_check_susp(td);
+ if (rv != 0)
+ return (rv);
/* If this failed the lock has changed, restart. */
continue;
@@ -985,10 +1008,11 @@ do_lock_normal(struct thread *td, struct
* either some one else has acquired the lock or it has been
* released.
*/
- old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED);
+ rv = casueword32(&m->m_owner, owner, &old,
+ owner | UMUTEX_CONTESTED);
/* The address was invalid. */
- if (old == -1) {
+ if (rv == -1) {
umtxq_lock(&uq->uq_key);
umtxq_remove(uq);
umtxq_unbusy(&uq->uq_key);
@@ -1033,16 +1057,16 @@ do_unlock_normal(struct thread *td, stru
/*
* Make sure we own this mtx.
*/
- owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
- if (owner == -1)
+ error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner);
+ if (error == -1)
return (EFAULT);
if ((owner & ~UMUTEX_CONTESTED) != id)
return (EPERM);
if ((owner & UMUTEX_CONTESTED) == 0) {
- old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED);
- if (old == -1)
+ error = casueword32(&m->m_owner, owner, &old, UMUTEX_UNOWNED);
+ if (error == -1)
return (EFAULT);
if (old == owner)
return (0);
@@ -1064,14 +1088,14 @@ do_unlock_normal(struct thread *td, stru
* there is zero or one thread only waiting for it.
* Otherwise, it must be marked as contested.
*/
- old = casuword32(&m->m_owner, owner,
- count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
+ error = casueword32(&m->m_owner, owner, &old,
+ count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
umtxq_lock(&key);
umtxq_signal(&key,1);
umtxq_unbusy(&key);
umtxq_unlock(&key);
umtx_key_release(&key);
- if (old == -1)
+ if (error == -1)
return (EFAULT);
if (old != owner)
return (EINVAL);
@@ -1091,14 +1115,16 @@ do_wake_umutex(struct thread *td, struct
int error;
int count;
- owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
- if (owner == -1)
+ error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner);
+ if (error == -1)
return (EFAULT);
if ((owner & ~UMUTEX_CONTESTED) != 0)
return (0);
- flags = fuword32(&m->m_flags);
+ error = fueword32(&m->m_flags, &flags);
+ if (error == -1)
+ return (EFAULT);
/* We should only ever be in here for contested locks */
if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
@@ -1110,16 +1136,20 @@ do_wake_umutex(struct thread *td, struct
count = umtxq_count(&key);
umtxq_unlock(&key);
- if (count <= 1)
- owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, UMUTEX_UNOWNED);
+ if (count <= 1) {
+ error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
+ UMUTEX_UNOWNED);
+ if (error == -1)
+ error = EFAULT;
+ }
umtxq_lock(&key);
- if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0)
+ if (error == 0 && count != 0 && (owner & ~UMUTEX_CONTESTED) == 0)
umtxq_signal(&key, 1);
umtxq_unbusy(&key);
umtxq_unlock(&key);
umtx_key_release(&key);
- return (0);
+ return (error);
}
/*
@@ -1162,41 +1192,49 @@ do_wake2_umutex(struct thread *td, struc
* any memory.
*/
if (count > 1) {
- owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
- while ((owner & UMUTEX_CONTESTED) ==0) {
- old = casuword32(&m->m_owner, owner,
- owner|UMUTEX_CONTESTED);
+ error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner),
+ &owner);
+ if (error == -1)
+ error = EFAULT;
+ while (error == 0 && (owner & UMUTEX_CONTESTED) == 0) {
+ error = casueword32(&m->m_owner, owner, &old,
+ owner | UMUTEX_CONTESTED);
+ if (error == -1) {
+ error = EFAULT;
+ break;
+ }
if (old == owner)
break;
owner = old;
- if (old == -1)
- break;
error = umtxq_check_susp(td);
if (error != 0)
break;
}
} else if (count == 1) {
- owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
- while ((owner & ~UMUTEX_CONTESTED) != 0 &&
+ error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner),
+ &owner);
+ if (error == -1)
+ error = EFAULT;
+ while (error == 0 && (owner & ~UMUTEX_CONTESTED) != 0 &&
(owner & UMUTEX_CONTESTED) == 0) {
- old = casuword32(&m->m_owner, owner,
- owner|UMUTEX_CONTESTED);
+ error = casueword32(&m->m_owner, owner, &old,
+ owner | UMUTEX_CONTESTED);
+ if (error == -1) {
+ error = EFAULT;
+ break;
+ }
if (old == owner)
break;
owner = old;
- if (old == -1)
- break;
error = umtxq_check_susp(td);
if (error != 0)
break;
}
}
umtxq_lock(&key);
- if (owner == -1) {
- error = EFAULT;
+ if (error == EFAULT) {
umtxq_signal(&key, INT_MAX);
- }
- else if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0)
+ } else if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0)
umtxq_signal(&key, 1);
umtxq_unbusy(&key);
umtxq_unlock(&key);
@@ -1576,7 +1614,7 @@ do_lock_pi(struct thread *td, struct umu
struct umtx_q *uq;
struct umtx_pi *pi, *new_pi;
uint32_t id, owner, old;
- int error;
+ int error, rv;
id = td->td_tid;
uq = td->td_umtxq;
@@ -1619,7 +1657,12 @@ do_lock_pi(struct thread *td, struct umu
/*
* Try the uncontested case. This should be done in userland.
*/
- owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
+ rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id);
+ /* The address was invalid. */
+ if (rv == -1) {
+ error = EFAULT;
+ break;
+ }
/* The acquire succeeded. */
if (owner == UMUTEX_UNOWNED) {
@@ -1627,16 +1670,15 @@ do_lock_pi(struct thread *td, struct umu
break;
}
- /* The address was invalid. */
- if (owner == -1) {
- error = EFAULT;
- break;
- }
-
/* If no one owns it but it is contested try to acquire it. */
if (owner == UMUTEX_CONTESTED) {
- owner = casuword32(&m->m_owner,
- UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
+ rv = casueword32(&m->m_owner,
+ UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED);
+ /* The address was invalid. */
+ if (rv == -1) {
+ error = EFAULT;
+ break;
+ }
if (owner == UMUTEX_CONTESTED) {
umtxq_lock(&uq->uq_key);
@@ -1647,12 +1689,6 @@ do_lock_pi(struct thread *td, struct umu
break;
}
- /* The address was invalid. */
- if (owner == -1) {
- error = EFAULT;
- break;
- }
-
error = umtxq_check_susp(td);
if (error != 0)
break;
@@ -1683,13 +1719,12 @@ do_lock_pi(struct thread *td, struct umu
* either some one else has acquired the lock or it has been
* released.
*/
- old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED);
+ rv = casueword32(&m->m_owner, owner, &old,
+ owner | UMUTEX_CONTESTED);
/* The address was invalid. */
- if (old == -1) {
- umtxq_lock(&uq->uq_key);
- umtxq_unbusy(&uq->uq_key);
- umtxq_unlock(&uq->uq_key);
+ if (rv == -1) {
+ umtxq_unbusy_unlocked(&uq->uq_key);
error = EFAULT;
break;
}
@@ -1741,8 +1776,8 @@ do_unlock_pi(struct thread *td, struct u
/*
* Make sure we own this mtx.
*/
- owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
- if (owner == -1)
+ error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner);
+ if (error == -1)
return (EFAULT);
if ((owner & ~UMUTEX_CONTESTED) != id)
@@ -1750,8 +1785,8 @@ do_unlock_pi(struct thread *td, struct u
/* This should be done in userland */
if ((owner & UMUTEX_CONTESTED) == 0) {
- old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED);
- if (old == -1)
+ error = casueword32(&m->m_owner, owner, &old, UMUTEX_UNOWNED);
+ if (error == -1)
return (EFAULT);
if (old == owner)
return (0);
@@ -1809,14 +1844,12 @@ do_unlock_pi(struct thread *td, struct u
* there is zero or one thread only waiting for it.
* Otherwise, it must be marked as contested.
*/
- old = casuword32(&m->m_owner, owner,
- count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
+ error = casueword32(&m->m_owner, owner, &old,
+ count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
- umtxq_lock(&key);
- umtxq_unbusy(&key);
- umtxq_unlock(&key);
+ umtxq_unbusy_unlocked(&key);
umtx_key_release(&key);
- if (old == -1)
+ if (error == -1)
return (EFAULT);
if (old != owner)
return (EINVAL);
@@ -1835,7 +1868,7 @@ do_lock_pp(struct thread *td, struct umu
struct umtx_pi *pi;
uint32_t ceiling;
uint32_t owner, id;
- int error, pri, old_inherited_pri, su;
+ int error, pri, old_inherited_pri, su, rv;
id = td->td_tid;
uq = td->td_umtxq;
@@ -1853,7 +1886,12 @@ do_lock_pp(struct thread *td, struct umu
umtxq_busy(&uq->uq_key);
umtxq_unlock(&uq->uq_key);
- ceiling = RTP_PRIO_MAX - fuword32(&m->m_ceilings[0]);
+ rv = fueword32(&m->m_ceilings[0], &ceiling);
+ if (rv == -1) {
+ error = EFAULT;
+ goto out;
+ }
+ ceiling = RTP_PRIO_MAX - ceiling;
if (ceiling > RTP_PRIO_MAX) {
error = EINVAL;
goto out;
@@ -1874,17 +1912,16 @@ do_lock_pp(struct thread *td, struct umu
}
mtx_unlock_spin(&umtx_lock);
- owner = casuword32(&m->m_owner,
- UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
-
- if (owner == UMUTEX_CONTESTED) {
- error = 0;
+ rv = casueword32(&m->m_owner,
+ UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED);
+ /* The address was invalid. */
+ if (rv == -1) {
+ error = EFAULT;
break;
}
- /* The address was invalid. */
- if (owner == -1) {
- error = EFAULT;
+ if (owner == UMUTEX_CONTESTED) {
+ error = 0;
break;
}
@@ -1946,9 +1983,7 @@ do_lock_pp(struct thread *td, struct umu
}
out:
- umtxq_lock(&uq->uq_key);
- umtxq_unbusy(&uq->uq_key);
- umtxq_unlock(&uq->uq_key);
+ umtxq_unbusy_unlocked(&uq->uq_key);
umtx_key_release(&uq->uq_key);
return (error);
}
@@ -1973,8 +2008,8 @@ do_unlock_pp(struct thread *td, struct u
/*
* Make sure we own this mtx.
*/
- owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
- if (owner == -1)
+ error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner);
+ if (error == -1)
return (EFAULT);
if ((owner & ~UMUTEX_CONTESTED) != id)
@@ -2047,9 +2082,11 @@ do_set_ceiling(struct thread *td, struct
uint32_t save_ceiling;
uint32_t owner, id;
uint32_t flags;
- int error;
+ int error, rv;
- flags = fuword32(&m->m_flags);
+ error = fueword32(&m->m_flags, &flags);
+ if (error == -1)
+ return (EFAULT);
if ((flags & UMUTEX_PRIO_PROTECT) == 0)
return (EINVAL);
if (ceiling > RTP_PRIO_MAX)
@@ -2064,10 +2101,18 @@ do_set_ceiling(struct thread *td, struct
umtxq_busy(&uq->uq_key);
umtxq_unlock(&uq->uq_key);
- save_ceiling = fuword32(&m->m_ceilings[0]);
+ rv = fueword32(&m->m_ceilings[0], &save_ceiling);
+ if (rv == -1) {
+ error = EFAULT;
+ break;
+ }
- owner = casuword32(&m->m_owner,
- UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
+ rv = casueword32(&m->m_owner,
+ UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED);
+ if (rv == -1) {
+ error = EFAULT;
+ break;
+ }
if (owner == UMUTEX_CONTESTED) {
suword32(&m->m_ceilings[0], ceiling);
@@ -2077,12 +2122,6 @@ do_set_ceiling(struct thread *td, struct
break;
}
- /* The address was invalid. */
- if (owner == -1) {
- error = EFAULT;
- break;
- }
-
if ((owner & ~UMUTEX_CONTESTED) == id) {
suword32(&m->m_ceilings[0], ceiling);
error = 0;
@@ -2129,8 +2168,8 @@ do_lock_umutex(struct thread *td, struct
uint32_t flags;
int error;
- flags = fuword32(&m->m_flags);
- if (flags == -1)
+ error = fueword32(&m->m_flags, &flags);
+ if (error == -1)
return (EFAULT);
switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
@@ -2164,9 +2203,10 @@ static int
do_unlock_umutex(struct thread *td, struct umutex *m)
{
uint32_t flags;
+ int error;
- flags = fuword32(&m->m_flags);
- if (flags == -1)
+ error = fueword32(&m->m_flags, &flags);
+ if (error == -1)
return (EFAULT);
switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
@@ -2187,21 +2227,27 @@ do_cv_wait(struct thread *td, struct uco
{
struct abs_timeout timo;
struct umtx_q *uq;
- uint32_t flags;
- uint32_t clockid;
+ uint32_t flags, clockid, hasw;
int error;
uq = td->td_umtxq;
- flags = fuword32(&cv->c_flags);
+ error = fueword32(&cv->c_flags, &flags);
+ if (error == -1)
+ return (EFAULT);
error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
if (error != 0)
return (error);
if ((wflags & CVWAIT_CLOCKID) != 0) {
- clockid = fuword32(&cv->c_clockid);
+ error = fueword32(&cv->c_clockid, &clockid);
+ if (error == -1) {
+ umtx_key_release(&uq->uq_key);
+ return (EFAULT);
+ }
if (clockid < CLOCK_REALTIME ||
clockid >= CLOCK_THREAD_CPUTIME_ID) {
/* hmm, only HW clock id will work. */
+ umtx_key_release(&uq->uq_key);
return (EINVAL);
}
} else {
@@ -2217,12 +2263,12 @@ do_cv_wait(struct thread *td, struct uco
* Set c_has_waiters to 1 before releasing user mutex, also
* don't modify cache line when unnecessary.
*/
- if (fuword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters)) == 0)
+ error = fueword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters),
+ &hasw);
+ if (error == 0 && hasw == 0)
suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 1);
- umtxq_lock(&uq->uq_key);
- umtxq_unbusy(&uq->uq_key);
- umtxq_unlock(&uq->uq_key);
+ umtxq_unbusy_unlocked(&uq->uq_key);
error = do_unlock_umutex(td, m);
@@ -2276,7 +2322,9 @@ do_cv_signal(struct thread *td, struct u
int error, cnt, nwake;
uint32_t flags;
- flags = fuword32(&cv->c_flags);
+ error = fueword32(&cv->c_flags, &flags);
+ if (error == -1)
+ return (EFAULT);
if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
return (error);
umtxq_lock(&key);
@@ -2287,6 +2335,8 @@ do_cv_signal(struct thread *td, struct u
umtxq_unlock(&key);
error = suword32(
__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0);
+ if (error == -1)
+ error = EFAULT;
umtxq_lock(&key);
}
umtxq_unbusy(&key);
@@ -2302,7 +2352,9 @@ do_cv_broadcast(struct thread *td, struc
int error;
uint32_t flags;
- flags = fuword32(&cv->c_flags);
+ error = fueword32(&cv->c_flags, &flags);
+ if (error == -1)
+ return (EFAULT);
if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
return (error);
@@ -2312,10 +2364,10 @@ do_cv_broadcast(struct thread *td, struc
umtxq_unlock(&key);
error = suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0);
+ if (error == -1)
+ error = EFAULT;
- umtxq_lock(&key);
- umtxq_unbusy(&key);
- umtxq_unlock(&key);
+ umtxq_unbusy_unlocked(&key);
umtx_key_release(&key);
return (error);
@@ -2329,10 +2381,12 @@ do_rw_rdlock(struct thread *td, struct u
uint32_t flags, wrflags;
int32_t state, oldstate;
int32_t blocked_readers;
- int error;
+ int error, rv;
uq = td->td_umtxq;
- flags = fuword32(&rwlock->rw_flags);
+ error = fueword32(&rwlock->rw_flags, &flags);
+ if (error == -1)
+ return (EFAULT);
error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
if (error != 0)
return (error);
@@ -2345,15 +2399,22 @@ do_rw_rdlock(struct thread *td, struct u
wrflags |= URWLOCK_WRITE_WAITERS;
for (;;) {
- state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+ rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state),
+ &state);
+ if (rv == -1) {
+ umtx_key_release(&uq->uq_key);
+ return (EFAULT);
+ }
+
/* try to lock it */
while (!(state & wrflags)) {
if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS)) {
umtx_key_release(&uq->uq_key);
return (EAGAIN);
}
- oldstate = casuword32(&rwlock->rw_state, state, state + 1);
- if (oldstate == -1) {
+ rv = casueword32(&rwlock->rw_state, state,
+ &oldstate, state + 1);
+ if (rv == -1) {
umtx_key_release(&uq->uq_key);
return (EFAULT);
}
@@ -2379,12 +2440,17 @@ do_rw_rdlock(struct thread *td, struct u
* re-read the state, in case it changed between the try-lock above
* and the check below
*/
- state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+ rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state),
+ &state);
+ if (rv == -1)
+ error = EFAULT;
/* set read contention bit */
- while ((state & wrflags) && !(state & URWLOCK_READ_WAITERS)) {
- oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_READ_WAITERS);
- if (oldstate == -1) {
+ while (error == 0 && (state & wrflags) &&
+ !(state & URWLOCK_READ_WAITERS)) {
+ rv = casueword32(&rwlock->rw_state, state,
+ &oldstate, state | URWLOCK_READ_WAITERS);
+ if (rv == -1) {
error = EFAULT;
break;
}
@@ -2396,17 +2462,13 @@ do_rw_rdlock(struct thread *td, struct u
break;
}
if (error != 0) {
- umtxq_lock(&uq->uq_key);
- umtxq_unbusy(&uq->uq_key);
- umtxq_unlock(&uq->uq_key);
+ umtxq_unbusy_unlocked(&uq->uq_key);
break;
}
/* state is changed while setting flags, restart */
if (!(state & wrflags)) {
- umtxq_lock(&uq->uq_key);
- umtxq_unbusy(&uq->uq_key);
- umtxq_unlock(&uq->uq_key);
+ umtxq_unbusy_unlocked(&uq->uq_key);
error = umtxq_check_susp(td);
if (error != 0)
break;
@@ -2415,7 +2477,13 @@ do_rw_rdlock(struct thread *td, struct u
sleep:
/* contention bit is set, before sleeping, increase read waiter count */
- blocked_readers = fuword32(&rwlock->rw_blocked_readers);
+ rv = fueword32(&rwlock->rw_blocked_readers,
+ &blocked_readers);
+ if (rv == -1) {
+ umtxq_unbusy_unlocked(&uq->uq_key);
+ error = EFAULT;
+ break;
+ }
suword32(&rwlock->rw_blocked_readers, blocked_readers+1);
while (state & wrflags) {
@@ -2431,18 +2499,32 @@ sleep:
umtxq_unlock(&uq->uq_key);
if (error)
break;
- state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+ rv = fueword32(__DEVOLATILE(int32_t *,
+ &rwlock->rw_state), &state);
+ if (rv == -1) {
+ error = EFAULT;
+ break;
+ }
}
/* decrease read waiter count, and may clear read contention bit */
- blocked_readers = fuword32(&rwlock->rw_blocked_readers);
+ rv = fueword32(&rwlock->rw_blocked_readers,
+ &blocked_readers);
+ if (rv == -1) {
+ umtxq_unbusy_unlocked(&uq->uq_key);
+ error = EFAULT;
+ break;
+ }
suword32(&rwlock->rw_blocked_readers, blocked_readers-1);
if (blocked_readers == 1) {
- state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
- for (;;) {
- oldstate = casuword32(&rwlock->rw_state, state,
- state & ~URWLOCK_READ_WAITERS);
- if (oldstate == -1) {
+ rv = fueword32(__DEVOLATILE(int32_t *,
+ &rwlock->rw_state), &state);
+ if (rv == -1)
+ error = EFAULT;
+ while (error == 0) {
+ rv = casueword32(&rwlock->rw_state, state,
+ &oldstate, state & ~URWLOCK_READ_WAITERS);
+ if (rv == -1) {
error = EFAULT;
break;
}
@@ -2450,14 +2532,10 @@ sleep:
break;
state = oldstate;
error = umtxq_check_susp(td);
- if (error != 0)
- break;
}
}
- umtxq_lock(&uq->uq_key);
- umtxq_unbusy(&uq->uq_key);
- umtxq_unlock(&uq->uq_key);
+ umtxq_unbusy_unlocked(&uq->uq_key);
if (error != 0)
break;
}
@@ -2476,10 +2554,12 @@ do_rw_wrlock(struct thread *td, struct u
int32_t state, oldstate;
int32_t blocked_writers;
int32_t blocked_readers;
- int error;
+ int error, rv;
uq = td->td_umtxq;
- flags = fuword32(&rwlock->rw_flags);
+ error = fueword32(&rwlock->rw_flags, &flags);
+ if (error == -1)
+ return (EFAULT);
error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
if (error != 0)
return (error);
@@ -2489,10 +2569,16 @@ do_rw_wrlock(struct thread *td, struct u
blocked_readers = 0;
for (;;) {
- state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+ rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state),
+ &state);
+ if (rv == -1) {
+ umtx_key_release(&uq->uq_key);
+ return (EFAULT);
+ }
while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
- oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_OWNER);
- if (oldstate == -1) {
+ rv = casueword32(&rwlock->rw_state, state,
+ &oldstate, state | URWLOCK_WRITE_OWNER);
+ if (rv == -1) {
umtx_key_release(&uq->uq_key);
return (EFAULT);
}
@@ -2528,12 +2614,17 @@ do_rw_wrlock(struct thread *td, struct u
* re-read the state, in case it changed between the try-lock above
* and the check below
*/
- state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+ rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state),
+ &state);
+ if (rv == -1)
+ error = EFAULT;
- while (((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) &&
- (state & URWLOCK_WRITE_WAITERS) == 0) {
- oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_WAITERS);
- if (oldstate == -1) {
+ while (error == 0 && ((state & URWLOCK_WRITE_OWNER) ||
+ URWLOCK_READER_COUNT(state) != 0) &&
+ (state & URWLOCK_WRITE_WAITERS) == 0) {
+ rv = casueword32(&rwlock->rw_state, state,
+ &oldstate, state | URWLOCK_WRITE_WAITERS);
+ if (rv == -1) {
error = EFAULT;
break;
}
@@ -2545,23 +2636,25 @@ do_rw_wrlock(struct thread *td, struct u
break;
}
if (error != 0) {
- umtxq_lock(&uq->uq_key);
- umtxq_unbusy(&uq->uq_key);
- umtxq_unlock(&uq->uq_key);
+ umtxq_unbusy_unlocked(&uq->uq_key);
break;
}
if (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
- umtxq_lock(&uq->uq_key);
- umtxq_unbusy(&uq->uq_key);
- umtxq_unlock(&uq->uq_key);
+ umtxq_unbusy_unlocked(&uq->uq_key);
error = umtxq_check_susp(td);
if (error != 0)
break;
continue;
}
sleep:
- blocked_writers = fuword32(&rwlock->rw_blocked_writers);
+ rv = fueword32(&rwlock->rw_blocked_writers,
+ &blocked_writers);
+ if (rv == -1) {
+ umtxq_unbusy_unlocked(&uq->uq_key);
+ error = EFAULT;
+ break;
+ }
suword32(&rwlock->rw_blocked_writers, blocked_writers+1);
while ((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) {
@@ -2577,17 +2670,34 @@ sleep:
umtxq_unlock(&uq->uq_key);
if (error)
break;
- state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+ rv = fueword32(__DEVOLATILE(int32_t *,
+ &rwlock->rw_state), &state);
+ if (rv == -1) {
+ error = EFAULT;
+ break;
+ }
}
- blocked_writers = fuword32(&rwlock->rw_blocked_writers);
+ rv = fueword32(&rwlock->rw_blocked_writers,
+ &blocked_writers);
+ if (rv == -1) {
+ umtxq_unbusy_unlocked(&uq->uq_key);
+ error = EFAULT;
+ break;
+ }
suword32(&rwlock->rw_blocked_writers, blocked_writers-1);
if (blocked_writers == 1) {
- state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+ rv = fueword32(__DEVOLATILE(int32_t *,
+ &rwlock->rw_state), &state);
+ if (rv == -1) {
+ umtxq_unbusy_unlocked(&uq->uq_key);
+ error = EFAULT;
+ break;
+ }
for (;;) {
- oldstate = casuword32(&rwlock->rw_state, state,
- state & ~URWLOCK_WRITE_WAITERS);
- if (oldstate == -1) {
+ rv = casueword32(&rwlock->rw_state, state,
+ &oldstate, state & ~URWLOCK_WRITE_WAITERS);
+ if (rv == -1) {
error = EFAULT;
break;
}
@@ -2603,13 +2713,17 @@ sleep:
if (error != 0)
break;
}
- blocked_readers = fuword32(&rwlock->rw_blocked_readers);
+ rv = fueword32(&rwlock->rw_blocked_readers,
+ &blocked_readers);
+ if (rv == -1) {
+ umtxq_unbusy_unlocked(&uq->uq_key);
+ error = EFAULT;
+ break;
+ }
} else
blocked_readers = 0;
- umtxq_lock(&uq->uq_key);
- umtxq_unbusy(&uq->uq_key);
- umtxq_unlock(&uq->uq_key);
+ umtxq_unbusy_unlocked(&uq->uq_key);
}
umtx_key_release(&uq->uq_key);
@@ -2624,20 +2738,26 @@ do_rw_unlock(struct thread *td, struct u
struct umtx_q *uq;
uint32_t flags;
int32_t state, oldstate;
- int error, q, count;
+ int error, rv, q, count;
uq = td->td_umtxq;
- flags = fuword32(&rwlock->rw_flags);
+ error = fueword32(&rwlock->rw_flags, &flags);
+ if (error == -1)
+ return (EFAULT);
error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
if (error != 0)
return (error);
- state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+ error = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state), &state);
+ if (error == -1) {
+ error = EFAULT;
+ goto out;
+ }
if (state & URWLOCK_WRITE_OWNER) {
for (;;) {
- oldstate = casuword32(&rwlock->rw_state, state,
- state & ~URWLOCK_WRITE_OWNER);
- if (oldstate == -1) {
+ rv = casueword32(&rwlock->rw_state, state,
+ &oldstate, state & ~URWLOCK_WRITE_OWNER);
+ if (rv == -1) {
error = EFAULT;
goto out;
}
@@ -2655,9 +2775,9 @@ do_rw_unlock(struct thread *td, struct u
}
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-head
mailing list