svn commit: r225840 - head/sys/vm
Konstantin Belousov
kib at FreeBSD.org
Wed Sep 28 14:57:51 UTC 2011
Author: kib
Date: Wed Sep 28 14:57:50 2011
New Revision: 225840
URL: http://svn.freebsd.org/changeset/base/225840
Log:
Use the trick of performing the atomic operation on the contained aligned
word to handle the dirty mask updates in vm_page_clear_dirty_mask().
Remove the vm page queue lock around vm_page_dirty() call in vm_fault_hold()
the sole purpose of which was to protect dirty on architectures which
does not provide short or byte-wide atomics.
Reviewed by: alc, attilio
Tested by: flo (sparc64)
MFC after: 2 weeks
Modified:
head/sys/vm/vm_fault.c
head/sys/vm/vm_page.c
head/sys/vm/vm_page.h
Modified: head/sys/vm/vm_fault.c
==============================================================================
--- head/sys/vm/vm_fault.c Wed Sep 28 14:52:25 2011 (r225839)
+++ head/sys/vm/vm_fault.c Wed Sep 28 14:57:50 2011 (r225840)
@@ -1090,18 +1090,10 @@ vm_fault_quick_hold_pages(vm_map_t map,
* performed through an unmanaged mapping or by a DMA
* operation.
*
- * The object lock is not held here. Therefore, like
- * a pmap operation, the page queues lock may be
- * required in order to call vm_page_dirty(). See
- * vm_page_clear_dirty_mask().
+ * The object lock is not held here.
+ * See vm_page_clear_dirty_mask().
*/
-#if defined(__amd64__) || defined(__i386__) || defined(__ia64__)
vm_page_dirty(*mp);
-#else
- vm_page_lock_queues();
- vm_page_dirty(*mp);
- vm_page_unlock_queues();
-#endif
}
}
if (pmap_failed) {
Modified: head/sys/vm/vm_page.c
==============================================================================
--- head/sys/vm/vm_page.c Wed Sep 28 14:52:25 2011 (r225839)
+++ head/sys/vm/vm_page.c Wed Sep 28 14:57:50 2011 (r225840)
@@ -745,9 +745,9 @@ vm_page_sleep(vm_page_t m, const char *m
*
* Set all bits in the page's dirty field.
*
- * The object containing the specified page must be locked if the call is
- * made from the machine-independent layer. If, however, the call is
- * made from the pmap layer, then the page queues lock may be required.
+ * The object containing the specified page must be locked if the
+ * call is made from the machine-independent layer.
+ *
* See vm_page_clear_dirty_mask().
*/
void
@@ -2339,44 +2339,53 @@ vm_page_set_valid(vm_page_t m, int base,
static __inline void
vm_page_clear_dirty_mask(vm_page_t m, int pagebits)
{
+ uintptr_t addr;
+#if PAGE_SIZE < 16384
+ int shift;
+#endif
/*
* If the object is locked and the page is neither VPO_BUSY nor
* PGA_WRITEABLE, then the page's dirty field cannot possibly be
- * set by a concurrent pmap operation.
+ * set by a concurrent pmap operation.
+ *
*/
VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0)
m->dirty &= ~pagebits;
else {
-#if defined(__amd64__) || defined(__i386__) || defined(__ia64__)
/*
- * On the aforementioned architectures, the page queues lock
- * is not required by the following read-modify-write
- * operation. The combination of the object's lock and an
- * atomic operation suffice. Moreover, the pmap layer on
- * these architectures can call vm_page_dirty() without
- * holding the page queues lock.
+ * The pmap layer can call vm_page_dirty() without
+ * holding a distinguished lock. The combination of
+ * the object's lock and an atomic operation suffice
+ * to guarantee consistency of the page dirty field.
+ *
+ * For PAGE_SIZE == 32768 case, compiler already
+ * properly aligns the dirty field, so no forcible
+ * alignment is needed. Only require existence of
+ * atomic_clear_64 when page size if 32768.
*/
-#if PAGE_SIZE == 4096
- atomic_clear_char(&m->dirty, pagebits);
-#elif PAGE_SIZE == 8192
- atomic_clear_short(&m->dirty, pagebits);
+ addr = (uintptr_t)&m->dirty;
+#if PAGE_SIZE == 32768
+#error pagebits too short
+ atomic_clear_64((uint64_t *)addr, pagebits);
#elif PAGE_SIZE == 16384
- atomic_clear_int(&m->dirty, pagebits);
-#else
-#error "PAGE_SIZE is not supported."
-#endif
-#else
+ atomic_clear_32((uint32_t *)addr, pagebits);
+#else /* PAGE_SIZE <= 8192 */
/*
- * Otherwise, the page queues lock is required to ensure that
- * a concurrent pmap operation does not set the page's dirty
- * field during the following read-modify-write operation.
+ * Use a trick to perform an 32bit atomic on the
+ * contained aligned word, to not depend on existence
+ * of the atomic_clear_{8, 16}.
*/
- vm_page_lock_queues();
- m->dirty &= ~pagebits;
- vm_page_unlock_queues();
+ shift = addr & (sizeof(uint32_t) - 1);
+#if BYTE_ORDER == BIG_ENDIAN
+ shift = (sizeof(uint32_t) - sizeof(m->dirty) - shift) * NBBY;
+#else
+ shift *= NBBY;
#endif
+ addr &= ~(sizeof(uint32_t) - 1);
+ atomic_clear_32((uint32_t *)addr, pagebits << shift);
+#endif /* PAGE_SIZE */
}
}
Modified: head/sys/vm/vm_page.h
==============================================================================
--- head/sys/vm/vm_page.h Wed Sep 28 14:52:25 2011 (r225839)
+++ head/sys/vm/vm_page.h Wed Sep 28 14:57:50 2011 (r225840)
@@ -94,21 +94,21 @@
* object that the page belongs to (O), the pool lock for the page (P),
* or the lock for either the free or paging queues (Q). If a field is
* annotated below with two of these locks, then holding either lock is
- * sufficient for read access, but both locks are required for write
+ * sufficient for read access, but both locks are required for write
* access.
*
- * In contrast, the synchronization of accesses to the page's dirty field
- * is machine dependent (M). In the machine-independent layer, the lock
- * on the object that the page belongs to must be held in order to
- * operate on the field. However, the pmap layer is permitted to set
- * all bits within the field without holding that lock. Therefore, if
- * the underlying architecture does not support atomic read-modify-write
- * operations on the field's type, then the machine-independent layer
- * must also hold the page queues lock when performing read-modify-write
- * operations and the pmap layer must hold the page queues lock when
- * setting the field. In the machine-independent layer, the
- * implementation of read-modify-write operations on the field is
- * encapsulated in vm_page_clear_dirty_mask().
+ * In contrast, the synchronization of accesses to the page's
+ * dirty field is machine dependent (M). In the
+ * machine-independent layer, the lock on the object that the
+ * page belongs to must be held in order to operate on the field.
+ * However, the pmap layer is permitted to set all bits within
+ * the field without holding that lock. If the underlying
+ * architecture does not support atomic read-modify-write
+ * operations on the field's type, then the machine-independent
+ * layer uses 32bit atomic on the aligned 32bit word that
+ * contains the dirty field. In the machine-independent layer,
+ * the implementation of read-modify-write operations on the
+ * field is encapsulated in vm_page_clear_dirty_mask().
*/
TAILQ_HEAD(pglist, vm_page);
More information about the svn-src-head
mailing list