svn commit: r269377 - in projects/lwref/sys: amd64/amd64 conf kern sys
Gleb Smirnoff
glebius at FreeBSD.org
Fri Aug 1 13:33:49 UTC 2014
Author: glebius
Date: Fri Aug 1 13:33:47 2014
New Revision: 269377
URL: http://svnweb.freebsd.org/changeset/base/269377
Log:
Drop into subversion the proof-of-concept unfinished and dirty
implementation of light weight reference counting mechanism.
Added:
projects/lwref/sys/amd64/amd64/lwref.S (contents, props changed)
projects/lwref/sys/kern/subr_lwref.c (contents, props changed)
projects/lwref/sys/sys/lwref.h (contents, props changed)
Modified:
projects/lwref/sys/amd64/amd64/apic_vector.S
projects/lwref/sys/conf/files
projects/lwref/sys/conf/files.amd64
projects/lwref/sys/kern/sched_ule.c
projects/lwref/sys/sys/sched.h
Modified: projects/lwref/sys/amd64/amd64/apic_vector.S
==============================================================================
--- projects/lwref/sys/amd64/amd64/apic_vector.S Fri Aug 1 12:42:37 2014 (r269376)
+++ projects/lwref/sys/amd64/amd64/apic_vector.S Fri Aug 1 13:33:47 2014 (r269377)
@@ -70,6 +70,8 @@ IDTVEC(vec_name) ; \
movq %rsp, %rsi ; \
movl %eax, %edi ; /* pass the IRQ */ \
call lapic_handle_intr ; \
+.globl vec_name ## _ret ; \
+vec_name ## _ret : ; \
1: ; \
MEXITCOUNT ; \
jmp doreti
@@ -107,6 +109,8 @@ IDTVEC(timerint)
FAKE_MCOUNT(TF_RIP(%rsp))
movq %rsp, %rdi
call lapic_handle_timer
+.globl timerint_ret
+timerint_ret:
MEXITCOUNT
jmp doreti
@@ -234,6 +238,8 @@ IDTVEC(ipi_intr_bitmap_handler)
FAKE_MCOUNT(TF_RIP(%rsp))
call ipi_bitmap_handler
+.globl ipi_intr_bitmap_handler_ret
+ipi_intr_bitmap_handler_ret:
MEXITCOUNT
jmp doreti
Added: projects/lwref/sys/amd64/amd64/lwref.S
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/lwref/sys/amd64/amd64/lwref.S Fri Aug 1 13:33:47 2014 (r269377)
@@ -0,0 +1,32 @@
+#include <machine/asmacros.h>
+
+/*
+ * void *lwref_acquire(lwref_t lwr, counter_u64_t *cp)
+ * {
+ * void *ptr;
+ *
+ * ptr = lwr->ptr;
+ * cp = &lwr->refs;
+ *
+ * counter_u64_add(*cp, 1);
+ *
+ * return (ptr);
+ * }
+ */
+
+ENTRY(lwref_acquire)
+ mov (%rdi), %rax
+ mov 0x8(%rdi), %rcx
+ mov %rcx, (%rsi)
+ mov $__pcpu, %rdx
+ sub %rdx, %rcx
+ mov $100000, %r10
+cycle:
+ sub $1, %r10
+ cmpq $0, %r10
+ jne cycle
+ addq $1, %gs:(%rcx)
+.globl lwref_acquire_ponr
+lwref_acquire_ponr:
+ ret
+END(lwref_acquire)
Modified: projects/lwref/sys/conf/files
==============================================================================
--- projects/lwref/sys/conf/files Fri Aug 1 12:42:37 2014 (r269376)
+++ projects/lwref/sys/conf/files Fri Aug 1 13:33:47 2014 (r269377)
@@ -2972,6 +2972,7 @@ kern/subr_kdb.c standard
kern/subr_kobj.c standard
kern/subr_lock.c standard
kern/subr_log.c standard
+kern/subr_lwref.c standard
kern/subr_mbpool.c optional libmbpool
kern/subr_mchain.c optional libmchain
kern/subr_module.c standard
Modified: projects/lwref/sys/conf/files.amd64
==============================================================================
--- projects/lwref/sys/conf/files.amd64 Fri Aug 1 12:42:37 2014 (r269376)
+++ projects/lwref/sys/conf/files.amd64 Fri Aug 1 13:33:47 2014 (r269377)
@@ -109,6 +109,7 @@ amd64/amd64/initcpu.c standard
amd64/amd64/io.c optional io
amd64/amd64/locore.S standard no-obj
amd64/amd64/xen-locore.S optional xenhvm
+amd64/amd64/lwref.S standard
amd64/amd64/machdep.c standard
amd64/amd64/mem.c optional mem
amd64/amd64/minidump_machdep.c standard
Modified: projects/lwref/sys/kern/sched_ule.c
==============================================================================
--- projects/lwref/sys/kern/sched_ule.c Fri Aug 1 12:42:37 2014 (r269376)
+++ projects/lwref/sys/kern/sched_ule.c Fri Aug 1 13:33:47 2014 (r269377)
@@ -2707,6 +2707,27 @@ sched_fork_exit(struct thread *td)
}
/*
+ * Apply a function to every thread on runqueue.
+ */
+void
+sched_foreach_on_runq(void(*func)(void *))
+{
+ struct tdq *tdq;
+ struct thread *td;
+
+ tdq = TDQ_SELF();
+
+ for (int i = 0; i < RQ_NQS; i++) {
+ TAILQ_FOREACH(td, &tdq->tdq_realtime.rq_queues[i], td_runq)
+ (func)(td);
+ TAILQ_FOREACH(td, &tdq->tdq_timeshare.rq_queues[i], td_runq)
+ (func)(td);
+ TAILQ_FOREACH(td, &tdq->tdq_idle.rq_queues[i], td_runq)
+ (func)(td);
+ }
+}
+
+/*
* Create on first use to catch odd startup conditons.
*/
char *
Added: projects/lwref/sys/kern/subr_lwref.c
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/lwref/sys/kern/subr_lwref.c Fri Aug 1 13:33:47 2014 (r269377)
@@ -0,0 +1,208 @@
+/*-
+ * Copyright (c) 2014 Gleb Smirnoff <glebius at FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/lock.h>
+#include <sys/malloc.h> /* XXXGL: M_TEMP */
+#include <sys/mutex.h>
+#include <sys/lwref.h>
+#include <sys/proc.h>
+#include <sys/sched.h>
+#include <sys/smp.h>
+#include <sys/pcpu.h>
+
+#include <machine/frame.h>
+#include <machine/pcb.h>
+
+struct lwref {
+ void *ptr;
+ counter_u64_t refcnt;
+ struct mtx mtx;
+};
+
+static void lwref_change_action(void *v);
+
+#ifdef INVARIANTS
+extern void Xrendezvous(void);
+extern void Xtimerint(void);
+#endif
+
+lwref_t
+lwref_alloc(void *ptr, int flags)
+{
+ lwref_t lwr;
+
+ lwr = malloc(sizeof(*lwr), M_TEMP, flags | M_ZERO);
+ if (lwr == NULL)
+ return (NULL);
+ lwr->refcnt = counter_u64_alloc(flags);
+ if (lwr->refcnt == NULL) {
+ free(lwr, M_TEMP);
+ return (NULL);
+ }
+ lwr->ptr = ptr;
+ mtx_init(&lwr->mtx, "lwref", NULL, MTX_DEF);
+
+ return (lwr);
+}
+
+struct lwref_change_ctx {
+ lwref_t lwr;
+ void *newptr;
+ counter_u64_t newcnt;
+ u_int cpu;
+ u_int oldcnt;
+};
+
+static void
+lwref_fixup_rip(register_t *rip, const char *p)
+{
+
+ if (*rip >= (register_t )lwref_acquire &&
+ *rip < (register_t )lwref_acquire_ponr) {
+ if (p)
+ printf("%s: %p\n", p, (void *)*rip);
+ *rip = (register_t )lwref_acquire;
+ }
+}
+
+static void
+lwref_fixup_td(void *arg)
+{
+ struct thread *td = arg;
+ register_t *rbp;
+
+ /*
+ * The timer interrupt trapframe is 3 functions deep:
+ * Xtimerint -> lapic_handle_timer -> mi_switch -> sched_switch,
+ * so in 99% this would work:
+ *
+ * tf = (struct trapframe *)
+ * ((register_t *)(***(void ****)(td->td_pcb->pcb_rbp)) + 2);
+ *
+ */
+ for (rbp = (register_t *)td->td_pcb->pcb_rbp;
+ rbp && rbp < (register_t *)*rbp;
+ rbp = (register_t *)*rbp) {
+ struct trapframe *tf;
+ register_t rip = (register_t )*(rbp + 1);
+
+ if (
+ rip == (register_t )timerint_ret ||
+ rip == (register_t )apic_isr1_ret ||
+ rip == (register_t )apic_isr2_ret ||
+ rip == (register_t )apic_isr3_ret ||
+ rip == (register_t )apic_isr4_ret ||
+ rip == (register_t )apic_isr5_ret ||
+ rip == (register_t )apic_isr6_ret ||
+ rip == (register_t )apic_isr7_ret ||
+ rip == (register_t )ipi_intr_bitmap_handler_ret
+ ) {
+ struct trapframe *tf;
+
+ tf = (struct trapframe *)(rbp + 2);
+ lwref_fixup_rip(&tf->tf_rip, __func__);
+ }
+
+ tf = (struct trapframe *)(rbp + 2);
+ if (tf->tf_rip > (register_t )lwref_acquire &&
+ tf->tf_rip < (register_t )lwref_acquire_ponr)
+ panic("lwref deteceted\n");
+ }
+}
+
+static void
+lwref_change_action(void *v)
+{
+ struct lwref_change_ctx *ctx = v;
+ lwref_t lwr = ctx->lwr;
+ struct trapframe *tf;
+
+ atomic_add_int(&ctx->oldcnt, *(uint64_t *)zpcpu_get(lwr->refcnt));
+
+ lwr->ptr = ctx->newptr;
+ lwr->refcnt = ctx->newcnt;
+
+ sched_foreach_on_runq(lwref_fixup_td);
+
+ if (ctx->cpu == curcpu)
+ /* We are not in IPI. */
+ return;
+
+ /*
+ * We are in IPI and we need to check the trap frame of
+ * the IPI, whether we interrupted lwref_acquire().
+ *
+ * We are two functions deep below the trap frame:
+ * Xrendezvous -> smp_rendezvous_action -> lwref_change_action
+ */
+ KASSERT(__builtin_return_address(0) > (void *)&smp_rendezvous_action &&
+ __builtin_return_address(0) <= (void *)((char *)&smp_rendezvous_action + 506) &&
+ __builtin_return_address(1) > (void *)&Xrendezvous &&
+ __builtin_return_address(1) <= (void *)((char *)&Xrendezvous + 201),
+ ("%p called via invalid path: 0 %p 1 %p", __func__,
+ __builtin_return_address(0), __builtin_return_address(1)));
+
+ /* After pushed %rbp and %rip begins the trap frame. */
+ tf = (struct trapframe *)
+ ((register_t *)__builtin_frame_address(1) + 2);
+ lwref_fixup_rip(&tf->tf_rip, __func__);
+}
+
+int
+lwref_change(lwref_t lwr, void *newptr, void (*freefn)(void *, void *),
+ void *freearg)
+{
+ struct lwref_change_ctx ctx;
+ counter_u64_t orefcnt;
+ void *optr;
+
+ ctx.newcnt = counter_u64_alloc(M_WAITOK); /* XXXGL */
+ ctx.oldcnt = 0;
+
+ mtx_lock(&lwr->mtx);
+ optr = lwr->ptr;
+ orefcnt = lwr->refcnt;
+ ctx.lwr = lwr;
+ ctx.newptr = newptr;
+ ctx.cpu = curcpu; /* XXXGL: race */
+ smp_rendezvous(smp_no_rendevous_barrier, lwref_change_action,
+ smp_no_rendevous_barrier, &ctx);
+ mtx_unlock(&lwr->mtx);
+
+ if (ctx.oldcnt == 0) {
+ (freefn)(freearg, optr);
+ counter_u64_free(orefcnt);
+ } else
+ printf("Leaking %p with cnt %p %u (%ju) refs\n",
+ optr, orefcnt, ctx.oldcnt, (uintmax_t )counter_u64_fetch(orefcnt));
+
+ return (0);
+}
Added: projects/lwref/sys/sys/lwref.h
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/lwref/sys/sys/lwref.h Fri Aug 1 13:33:47 2014 (r269377)
@@ -0,0 +1,63 @@
+/*-
+ * Copyright (c) 2014 Gleb Smirnoff <glebius at FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef __SYS_LWREF_H__
+#define __SYS_LWREF_H__
+
+#include <sys/counter.h>
+
+struct lwref;
+typedef struct lwref * lwref_t;
+
+lwref_t lwref_alloc(void *, int);
+int lwref_change(lwref_t, void *, void(*)(void *, void *), void *);
+
+/* asm */
+void *lwref_acquire(lwref_t, counter_u64_t *);
+extern char lwref_acquire_ponr[];
+
+extern char timerint_ret[];
+extern char apic_isr1_ret[];
+extern char apic_isr2_ret[];
+extern char apic_isr3_ret[];
+extern char apic_isr4_ret[];
+extern char apic_isr5_ret[];
+extern char apic_isr6_ret[];
+extern char apic_isr7_ret[];
+extern char ipi_intr_bitmap_handler_ret[];
+
+#ifdef INVARIANTS
+#define lwref_release(p, c) do { \
+ p = NULL; \
+ counter_u64_add(c, -1); \
+} while (0)
+#else
+#define lwref_release(p, c) counter_u64_add(c, -1)
+#endif
+
+#endif /* ! __SYS_LWREF_H__ */
Modified: projects/lwref/sys/sys/sched.h
==============================================================================
--- projects/lwref/sys/sys/sched.h Fri Aug 1 12:42:37 2014 (r269376)
+++ projects/lwref/sys/sys/sched.h Fri Aug 1 13:33:47 2014 (r269377)
@@ -147,6 +147,11 @@ char *sched_tdname(struct thread *td);
void sched_clear_tdname(struct thread *td);
#endif
+/*
+ * Used for lwref fixups.
+ */
+void sched_foreach_on_runq(void(*)(void *));
+
static __inline void
sched_pin(void)
{
More information about the svn-src-projects
mailing list