PERFORCE change 66889 for review
David Xu
davidxu at FreeBSD.org
Sun Dec 12 06:50:37 PST 2004
http://perforce.freebsd.org/chv.cgi?CH=66889
Change 66889 by davidxu at davidxu_tiger on 2004/12/12 14:49:47
Make umtx to be sharable between processes, based on this
change, it is possible to implement PTHREAD_PROCESS_SHARED
pthread_mutex and pthread_cond etcs in thread library, posix
requires PTHREAD_PROCESS_SHARED sync objects to be in shared
memory page if user wants it to be shared betwee processes.
Affected files ...
.. //depot/projects/davidxu_thread/src/sys/kern/kern_umtx.c#4 edit
Differences ...
==== //depot/projects/davidxu_thread/src/sys/kern/kern_umtx.c#4 (text+ko) ====
@@ -37,21 +37,31 @@
#include <sys/sysent.h>
#include <sys/systm.h>
#include <sys/sysproto.h>
+#include <sys/eventhandler.h>
#include <sys/thr.h>
#include <sys/umtx.h>
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/pmap.h>
+#include <vm/vm_map.h>
+#include <vm/vm_object.h>
+
+struct umtx_key {
+ vm_object_t uk_object;
+ vm_ooffset_t uk_offset;
+};
+
struct umtx_q {
LIST_ENTRY(umtx_q) uq_next; /* Linked list for the hash. */
- TAILQ_HEAD(, thread) uq_tdq; /* List of threads blocked here. */
- struct umtx *uq_umtx; /* Pointer key component. */
- pid_t uq_pid; /* Pid key component. */
- int uq_count; /* How many threads blocked. */
+ struct umtx_key uq_key;
+ struct thread *uq_thread;
};
LIST_HEAD(umtx_head, umtx_q);
struct umtxq_chain {
- struct mtx uc_lock; /* lock for this chain. */
- struct umtx_head uc_queues; /* List of sleep queues. */
+ struct mtx uc_lock; /* Lock for this chain. */
+ struct umtx_head uc_queue; /* List of sleep queues. */
};
#define GOLDEN_RATIO_PRIME 2654404609U
@@ -64,17 +74,17 @@
#define UMTX_CONTESTED LONG_MIN
static void umtx_init_chains(void *);
-static int umtxq_hash(struct thread *, struct umtx *);
-static void umtxq_lock(struct thread *td, struct umtx *key);
-static void umtxq_unlock(struct thread *td, struct umtx *key);
-static struct umtx_q *umtxq_lookup(struct thread *, struct umtx *);
-static struct umtx_q *umtxq_insert(struct thread *, struct umtx *);
-static int umtxq_count(struct thread *td, struct umtx *umtx);
-static int umtx_sleep(struct thread *td, struct umtx *umtx, int priority,
- const char *wmesg, int timo);
-static void umtx_signal(struct thread *td, struct umtx *umtx);
+static int umtxq_hash(struct umtx_key *);
+static struct mtx *umtxq_mtx(int chain);
+static void umtxq_lock(int chain);
+static void umtxq_unlock(int chain);
+static void umtxq_insert(int chain, struct umtx_q *);
+static int umtxq_count(struct umtx_key *);
+static void umtxq_signal(struct umtx_key *);
+static void fork_handler(void *arg, struct proc *p1, struct proc *p2,
+ int flags);
-SYSINIT(umtx, SI_SUB_LOCK, SI_ORDER_MIDDLE, umtx_init_chains, NULL);
+SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtx_init_chains, NULL);
static void
umtx_init_chains(void *arg __unused)
@@ -84,154 +94,119 @@
for (i = 0; i < UMTX_CHAINS; ++i) {
mtx_init(&umtxq_chains[i].uc_lock, "umtxq_lock", NULL,
MTX_DEF | MTX_DUPOK);
- LIST_INIT(&umtxq_chains[i].uc_queues);
+ LIST_INIT(&umtxq_chains[i].uc_queue);
+ }
+ EVENTHANDLER_REGISTER(process_fork, fork_handler, 0, 10000);
+}
+
+static void
+fork_handler(void *arg, struct proc *p1, struct proc *p2, int flags)
+{
+ struct thread *td;
+
+ PROC_LOCK(p1);
+ FOREACH_THREAD_IN_PROC(p1, td) {
+ if (td->td_flags & TDF_UMTXQ)
+ wakeup(td);
}
+ PROC_UNLOCK(p1);
}
static inline int
-umtxq_hash(struct thread *td, struct umtx *umtx)
+umtxq_hash(struct umtx_key *key)
{
- unsigned n = (uintptr_t)umtx + td->td_proc->p_pid;
+ unsigned n = (uintptr_t)key->uk_object + key->uk_offset;
return (((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS);
}
-static inline void
-umtxq_lock(struct thread *td, struct umtx *key)
+static inline struct mtx *
+umtxq_mtx(int chain)
{
- int chain = umtxq_hash(td, key);
- mtx_lock(&umtxq_chains[chain].uc_lock);
+ return (&umtxq_chains[chain].uc_lock);
}
static inline void
-umtxq_unlock(struct thread *td, struct umtx *key)
+umtxq_lock(int chain)
{
- int chain = umtxq_hash(td, key);
- mtx_unlock(&umtxq_chains[chain].uc_lock);
+ mtx_lock(umtxq_mtx(chain));
}
-static struct umtx_q *
-umtxq_lookup(struct thread *td, struct umtx *umtx)
+static inline void
+umtxq_unlock(int chain)
{
- struct umtx_head *head;
- struct umtx_q *uq;
- pid_t pid;
- int chain;
-
- chain = umtxq_hash(td, umtx);
- mtx_assert(&umtxq_chains[chain].uc_lock, MA_OWNED);
- pid = td->td_proc->p_pid;
- head = &umtxq_chains[chain].uc_queues;
- LIST_FOREACH(uq, head, uq_next) {
- if (uq->uq_pid == pid && uq->uq_umtx == umtx)
- return (uq);
- }
- return (NULL);
+ mtx_unlock(umtxq_mtx(chain));
}
/*
* Insert a thread onto the umtx queue.
*/
-static struct umtx_q *
-umtxq_insert(struct thread *td, struct umtx *umtx)
+static inline void
+umtxq_insert(int chain, struct umtx_q *uq)
{
struct umtx_head *head;
- struct umtx_q *uq, *ins = NULL;
- pid_t pid;
- int chain;
- chain = umtxq_hash(td, umtx);
- pid = td->td_proc->p_pid;
- if ((uq = umtxq_lookup(td, umtx)) == NULL) {
- umtxq_unlock(td, umtx);
- ins = malloc(sizeof(*uq), M_UMTX, M_ZERO | M_WAITOK);
- umtxq_lock(td, umtx);
-
- /*
- * Some one else could have succeeded while we were blocked
- * waiting on memory.
- */
- if ((uq = umtxq_lookup(td, umtx)) == NULL) {
- head = &umtxq_chains[chain].uc_queues;
- uq = ins;
- uq->uq_pid = pid;
- uq->uq_umtx = umtx;
- uq->uq_count = 0;
- LIST_INSERT_HEAD(head, uq, uq_next);
- TAILQ_INIT(&uq->uq_tdq);
- ins = NULL;
- }
- }
- TAILQ_INSERT_TAIL(&uq->uq_tdq, td, td_umtx);
- uq->uq_count++;
- if (ins) {
- umtxq_unlock(td, umtx);
- free(ins, M_UMTX);
- umtxq_lock(td, umtx);
- }
- return (uq);
+ head = &umtxq_chains[chain].uc_queue;
+ LIST_INSERT_HEAD(head, uq, uq_next);
+ mtx_lock_spin(&sched_lock);
+ uq->uq_thread->td_flags |= TDF_UMTXQ;
+ mtx_unlock_spin(&sched_lock);
}
/*
- * Remove thread from umtx queue, umtx chain lock is also
- * released.
+ * Remove thread from umtx queue released.
*/
-static void
-umtx_remove(struct umtx_q *uq, struct thread *td, struct umtx *umtx)
+static inline void
+umtxq_remove(struct umtx_q *uq)
{
- int chain;
-
- chain = umtxq_hash(td, umtx);
- mtx_assert(&umtxq_chains[chain].uc_lock, MA_OWNED);
- TAILQ_REMOVE(&uq->uq_tdq, td, td_umtx);
- uq->uq_count--;
- if (TAILQ_EMPTY(&uq->uq_tdq)) {
+ if (uq->uq_thread->td_flags & TDF_UMTXQ) {
LIST_REMOVE(uq, uq_next);
- umtxq_unlock(td, umtx);
- free(uq, M_UMTX);
- } else
- umtxq_unlock(td, umtx);
+ mtx_lock_spin(&sched_lock);
+ uq->uq_thread->td_flags &= ~TDF_UMTXQ;
+ mtx_unlock_spin(&sched_lock);
+ }
}
-static inline int
-umtxq_count(struct thread *td, struct umtx *umtx)
+static int
+umtxq_count(struct umtx_key *key)
{
struct umtx_q *uq;
- int count = 0;
+ struct umtx_head *head;
+ int chain, count = 0;
- umtxq_lock(td, umtx);
- if ((uq = umtxq_lookup(td, umtx)) != NULL)
- count = uq->uq_count;
- umtxq_unlock(td, umtx);
+ chain = umtxq_hash(key);
+ umtxq_lock(chain);
+ head = &umtxq_chains[chain].uc_queue;
+ LIST_FOREACH(uq, head, uq_next) {
+ if (uq->uq_key.uk_object == key->uk_object &&
+ uq->uq_key.uk_offset == key->uk_offset) {
+ if (++count > 1)
+ break;
+ }
+ }
+ umtxq_unlock(chain);
return (count);
}
-static inline int
-umtx_sleep(struct thread *td, struct umtx *umtx, int priority,
- const char *wmesg, int timo)
-{
- int chain;
-
- chain = umtxq_hash(td, umtx);
- mtx_assert(&umtxq_chains[chain].uc_lock, MA_OWNED);
- return (msleep(td, &umtxq_chains[chain].uc_lock, priority,
- wmesg, timo));
-}
-
static void
-umtx_signal(struct thread *td, struct umtx *umtx)
+umtxq_signal(struct umtx_key *key)
{
struct umtx_q *uq;
+ struct umtx_head *head;
struct thread *blocked = NULL;
+ int chain;
- umtxq_lock(td, umtx);
- if ((uq = umtxq_lookup(td, umtx)) != NULL) {
- if ((blocked = TAILQ_FIRST(&uq->uq_tdq)) != NULL) {
- mtx_lock_spin(&sched_lock);
- blocked->td_flags |= TDF_UMTXWAKEUP;
- mtx_unlock_spin(&sched_lock);
+ chain = umtxq_hash(key);
+ umtxq_lock(chain);
+ head = &umtxq_chains[chain].uc_queue;
+ LIST_FOREACH(uq, head, uq_next) {
+ if (uq->uq_key.uk_object == key->uk_object &&
+ uq->uq_key.uk_offset == key->uk_offset) {
+ blocked = uq->uq_thread;
+ umtxq_remove(uq);
+ break;
}
}
- umtxq_unlock(td, umtx);
+ umtxq_unlock(chain);
if (blocked != NULL)
wakeup(blocked);
}
@@ -240,19 +215,29 @@
_umtx_lock(struct thread *td, struct _umtx_lock_args *uap)
/* struct umtx *umtx */
{
- struct umtx_q *uq;
+ struct umtx_q uq;
struct umtx *umtx;
+ vm_map_t map;
+ vm_map_entry_t entry;
+ vm_object_t object;
+ vm_pindex_t pindex;
+ vm_prot_t prot;
+ boolean_t wired;
intptr_t owner;
intptr_t old;
+ int chain, page_off;
int error = 0;
- uq = NULL;
-
/*
* Care must be exercised when dealing with this structure. It
* can fault on any access.
*/
- umtx = uap->umtx;
+ umtx = uap->umtx;
+
+ page_off = ((unsigned long)umtx) % PAGE_SIZE;
+ /* Must not on page boundary. */
+ if (page_off + sizeof(void *) > PAGE_SIZE)
+ return (EINVAL);
for (;;) {
/*
@@ -292,9 +277,22 @@
if (error)
return (error);
- umtxq_lock(td, umtx);
- uq = umtxq_insert(td, umtx);
- umtxq_unlock(td, umtx);
+ map = &td->td_proc->p_vmspace->vm_map;
+ if (vm_map_lookup(&map, (vm_offset_t)umtx, VM_PROT_WRITE,
+ &entry, &object, &pindex, &prot, &wired) != KERN_SUCCESS) {
+ vm_map_lookup_done(map, entry);
+ return EFAULT;
+ }
+ vm_object_reference(object);
+ uq.uq_key.uk_object = object;
+ uq.uq_key.uk_offset = entry->offset + entry->start -
+ (vm_offset_t)umtx;
+ uq.uq_thread = td;
+ chain = umtxq_hash(&uq.uq_key);
+ umtxq_lock(chain);
+ umtxq_insert(chain, &uq);
+ umtxq_unlock(chain);
+ vm_map_lookup_done(map, entry);
/*
* Set the contested bit so that a release in user space
@@ -307,9 +305,10 @@
/* The address was invalid. */
if (old == -1) {
- umtxq_lock(td, umtx);
- umtx_remove(uq, td, umtx);
- /* unlocked by umtx_remove */
+ umtxq_lock(chain);
+ umtxq_remove(&uq);
+ umtxq_unlock(chain);
+ vm_object_deallocate(uq.uq_key.uk_object);
return (EFAULT);
}
@@ -318,31 +317,22 @@
* and we need to retry or we lost a race to the thread
* unlocking the umtx.
*/
- umtxq_lock(td, umtx);
- if (old == owner && (td->td_flags & TDF_UMTXWAKEUP) == 0)
- error = umtx_sleep(td, umtx, td->td_priority | PCATCH,
- "umtx", 0);
- else
+ umtxq_lock(chain);
+ if (old == owner && (td->td_flags & TDF_UMTXQ)) {
+ error = msleep(td, umtxq_mtx(chain),
+ td->td_priority | PCATCH | PDROP,
+ "umtx", 0);
+ if (td->td_flags & TDF_UMTXQ) {
+ umtxq_lock(chain);
+ umtxq_remove(&uq);
+ umtxq_unlock(chain);
+ }
+ } else {
error = 0;
- umtx_remove(uq, td, umtx);
- /* unlocked by umtx_remove */
-
- if (td->td_flags & TDF_UMTXWAKEUP) {
- /*
- * If we were resumed by umtxq_unlock, we should retry
- * to avoid a race.
- */
- mtx_lock_spin(&sched_lock);
- td->td_flags &= ~TDF_UMTXWAKEUP;
- mtx_unlock_spin(&sched_lock);
- continue;
+ umtxq_remove(&uq);
+ umtxq_unlock(chain);
}
-
- /*
- * If we caught a signal, exit immediately.
- */
- if (error)
- return (error);
+ vm_object_deallocate(uq.uq_key.uk_object);
}
return (0);
@@ -352,7 +342,14 @@
_umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap)
/* struct umtx *umtx */
{
+ struct umtx_q uq;
struct umtx *umtx;
+ vm_map_t map;
+ vm_map_entry_t entry;
+ vm_object_t object;
+ vm_pindex_t pindex;
+ vm_prot_t prot;
+ boolean_t wired;
intptr_t owner;
intptr_t old;
int count;
@@ -368,7 +365,7 @@
if ((owner = fuword(&umtx->u_owner)) == -1)
return (EFAULT);
- if ((owner & ~UMTX_CONTESTED) != td->td_tid)
+ if ((owner & ~UMTX_CONTESTED) != td->td_tid)
return (EPERM);
/* We should only ever be in here for contested locks */
@@ -386,15 +383,28 @@
if (old != owner)
return (EINVAL);
+ map = &td->td_proc->p_vmspace->vm_map;
+ if (vm_map_lookup(&map, (vm_offset_t)umtx, VM_PROT_WRITE,
+ &entry, &object, &pindex, &prot, &wired) != KERN_SUCCESS) {
+ vm_map_lookup_done(map, entry);
+ return EFAULT;
+ }
+ vm_object_reference(object);
+ uq.uq_key.uk_object = object;
+ uq.uq_key.uk_offset = entry->offset + entry->start - (vm_offset_t)umtx;
+ vm_map_lookup_done(map, entry);
+
/*
* At the point, a new thread can lock the umtx before we
* reach here, so contested bit will not be set, if there
* are two or more threads on wait queue, we should set
* contensted bit for them.
*/
- count = umtxq_count(td, umtx);
- if (count <= 0)
+ count = umtxq_count(&uq.uq_key);
+ if (count <= 0) {
+ vm_object_deallocate(object);
return (0);
+ }
/*
* If there is second thread waiting on umtx, set contested bit,
@@ -408,8 +418,10 @@
owner | UMTX_CONTESTED);
if (old == owner)
break;
- if (old == -1)
+ if (old == -1) {
+ vm_object_deallocate(object);
return (EFAULT);
+ }
owner = old;
}
/*
@@ -417,12 +429,15 @@
* to wake more threads, that thread will do it when it unlocks
* the umtx.
*/
- if ((owner & ~UMTX_CONTESTED) != 0)
+ if ((owner & ~UMTX_CONTESTED) != 0) {
+ vm_object_deallocate(object);
return (0);
+ }
}
/* Wake blocked thread. */
- umtx_signal(td, umtx);
+ umtxq_signal(&uq.uq_key);
+ vm_object_deallocate(object);
return (0);
}
More information about the p4-projects
mailing list