svn commit: r303654 - in user/alc/PQ_LAUNDRY/sys: ddb dev/cxgbe dev/cxgbe/iw_cxgbe dev/pci kern net ofed/drivers/infiniband/ulp/ipoib opencrypto sys
Mark Johnston
markj at FreeBSD.org
Tue Aug 2 02:32:02 UTC 2016
Author: markj
Date: Tue Aug 2 02:32:00 2016
New Revision: 303654
URL: https://svnweb.freebsd.org/changeset/base/303654
Log:
MFH r303653
Modified:
user/alc/PQ_LAUNDRY/sys/ddb/db_ps.c
user/alc/PQ_LAUNDRY/sys/dev/cxgbe/adapter.h
user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/cq.c
user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/qp.c
user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_main.c
user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_netmap.c
user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_sge.c
user/alc/PQ_LAUNDRY/sys/dev/pci/pci_pci.c
user/alc/PQ_LAUNDRY/sys/kern/kern_mutex.c
user/alc/PQ_LAUNDRY/sys/kern/kern_rwlock.c
user/alc/PQ_LAUNDRY/sys/kern/kern_sx.c
user/alc/PQ_LAUNDRY/sys/kern/subr_lock.c
user/alc/PQ_LAUNDRY/sys/net/route.h
user/alc/PQ_LAUNDRY/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c
user/alc/PQ_LAUNDRY/sys/opencrypto/xform_aes_icm.c
user/alc/PQ_LAUNDRY/sys/sys/lock.h
Directory Properties:
user/alc/PQ_LAUNDRY/ (props changed)
Modified: user/alc/PQ_LAUNDRY/sys/ddb/db_ps.c
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/ddb/db_ps.c Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/ddb/db_ps.c Tue Aug 2 02:32:00 2016 (r303654)
@@ -440,9 +440,16 @@ DB_SHOW_COMMAND(proc, db_show_proc)
p->p_leader);
if (p->p_sysent != NULL)
db_printf(" ABI: %s\n", p->p_sysent->sv_name);
- if (p->p_args != NULL)
- db_printf(" arguments: %.*s\n", (int)p->p_args->ar_length,
- p->p_args->ar_args);
+ if (p->p_args != NULL) {
+ db_printf(" arguments: ");
+ for (i = 0; i < (int)p->p_args->ar_length; i++) {
+ if (p->p_args->ar_args[i] == '\0')
+ db_printf(" ");
+ else
+ db_printf("%c", p->p_args->ar_args[i]);
+ }
+ db_printf("\n");
+ }
db_printf(" threads: %d\n", p->p_numthreads);
FOREACH_THREAD_IN_PROC(p, td) {
dumpthread(p, td, 1);
Modified: user/alc/PQ_LAUNDRY/sys/dev/cxgbe/adapter.h
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/dev/cxgbe/adapter.h Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/dev/cxgbe/adapter.h Tue Aug 2 02:32:00 2016 (r303654)
@@ -781,6 +781,8 @@ struct adapter {
struct sge_rxq *rxq;
struct sge_nm_rxq *nm_rxq;
} __aligned(CACHE_LINE_SIZE) *irq;
+ int sge_gts_reg;
+ int sge_kdoorbell_reg;
bus_dma_tag_t dmat; /* Parent DMA tag */
Modified: user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/cq.c
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/cq.c Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/cq.c Tue Aug 2 02:32:00 2016 (r303654)
@@ -172,7 +172,7 @@ create_cq(struct c4iw_rdev *rdev, struct
cq->gen = 1;
cq->gts = (void *)((unsigned long)rman_get_virtual(sc->regs_res) +
- MYPF_REG(SGE_PF_GTS));
+ sc->sge_gts_reg);
cq->rdev = rdev;
if (user) {
Modified: user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/qp.c
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/qp.c Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/qp.c Tue Aug 2 02:32:00 2016 (r303654)
@@ -182,9 +182,9 @@ static int create_qp(struct c4iw_rdev *r
pci_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr);
wq->db = (void *)((unsigned long)rman_get_virtual(sc->regs_res) +
- MYPF_REG(SGE_PF_KDOORBELL));
+ sc->sge_kdoorbell_reg);
wq->gts = (void *)((unsigned long)rman_get_virtual(rdev->adap->regs_res)
- + MYPF_REG(SGE_PF_GTS));
+ + sc->sge_gts_reg);
if (user) {
wq->sq.udb = (u64)((char*)rman_get_virtual(rdev->adap->udbs_res) +
(wq->sq.qid << rdev->qpshift));
Modified: user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_main.c
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_main.c Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_main.c Tue Aug 2 02:32:00 2016 (r303654)
@@ -697,6 +697,8 @@ t4_attach(device_t dev)
sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
}
+ sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS);
+ sc->sge_kdoorbell_reg = MYPF_REG(A_SGE_PF_KDOORBELL);
sc->traceq = -1;
mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
Modified: user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_netmap.c
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_netmap.c Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_netmap.c Tue Aug 2 02:32:00 2016 (r303654)
@@ -204,7 +204,7 @@ alloc_nm_rxq_hwq(struct vi_info *vi, str
}
}
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
+ t4_write_reg(sc, sc->sge_gts_reg,
V_INGRESSQID(nm_rxq->iq_cntxt_id) |
V_SEINTARM(V_QINTR_TIMER_IDX(holdoff_tmr_idx)));
@@ -364,7 +364,7 @@ cxgbe_netmap_on(struct adapter *sc, stru
MPASS((j & 7) == 0);
j /= 8; /* driver pidx to hardware pidx */
wmb();
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
+ t4_write_reg(sc, sc->sge_kdoorbell_reg,
nm_rxq->fl_db_val | V_PIDX(j));
atomic_cmpset_int(&irq->nm_state, NM_OFF, NM_ON);
@@ -537,7 +537,7 @@ ring_nm_txq_db(struct adapter *sc, struc
break;
case DOORBELL_KDB:
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
+ t4_write_reg(sc, sc->sge_kdoorbell_reg,
V_QID(nm_txq->cntxt_id) | V_PIDX(n));
break;
}
@@ -818,7 +818,7 @@ cxgbe_netmap_rxsync(struct netmap_kring
}
if (++dbinc == 8 && n >= 32) {
wmb();
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
+ t4_write_reg(sc, sc->sge_kdoorbell_reg,
nm_rxq->fl_db_val | V_PIDX(dbinc));
dbinc = 0;
}
@@ -827,7 +827,7 @@ cxgbe_netmap_rxsync(struct netmap_kring
if (dbinc > 0) {
wmb();
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
+ t4_write_reg(sc, sc->sge_kdoorbell_reg,
nm_rxq->fl_db_val | V_PIDX(dbinc));
}
}
@@ -981,14 +981,14 @@ t4_nm_intr(void *arg)
fl_credits /= 8;
IDXINCR(nm_rxq->fl_pidx, fl_credits * 8,
nm_rxq->fl_sidx);
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
+ t4_write_reg(sc, sc->sge_kdoorbell_reg,
nm_rxq->fl_db_val | V_PIDX(fl_credits));
fl_credits = fl_cidx & 7;
} else if (!black_hole) {
netmap_rx_irq(ifp, nm_rxq->nid, &work);
MPASS(work != 0);
}
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
+ t4_write_reg(sc, sc->sge_gts_reg,
V_CIDXINC(n) | V_INGRESSQID(nm_rxq->iq_cntxt_id) |
V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
n = 0;
@@ -999,12 +999,12 @@ t4_nm_intr(void *arg)
if (black_hole) {
fl_credits /= 8;
IDXINCR(nm_rxq->fl_pidx, fl_credits * 8, nm_rxq->fl_sidx);
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
+ t4_write_reg(sc, sc->sge_kdoorbell_reg,
nm_rxq->fl_db_val | V_PIDX(fl_credits));
} else
netmap_rx_irq(ifp, nm_rxq->nid, &work);
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(n) |
+ t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(n) |
V_INGRESSQID((u32)nm_rxq->iq_cntxt_id) |
V_SEINTARM(V_QINTR_TIMER_IDX(holdoff_tmr_idx)));
}
Modified: user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_sge.c
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_sge.c Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_sge.c Tue Aug 2 02:32:00 2016 (r303654)
@@ -1474,7 +1474,7 @@ service_iq(struct sge_iq *iq, int budget
d = &iq->desc[0];
}
if (__predict_false(++ndescs == limit)) {
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
+ t4_write_reg(sc, sc->sge_gts_reg,
V_CIDXINC(ndescs) |
V_INGRESSQID(iq->cntxt_id) |
V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
@@ -1529,7 +1529,7 @@ process_iql:
}
#endif
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
+ t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
if (iq->flags & IQ_HAS_FL) {
@@ -2793,7 +2793,7 @@ alloc_iq_fl(struct vi_info *vi, struct s
/* Enable IQ interrupts */
atomic_store_rel_int(&iq->state, IQS_IDLE);
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) |
+ t4_write_reg(sc, sc->sge_gts_reg, V_SEINTARM(iq->intr_params) |
V_INGRESSQID(iq->cntxt_id));
return (0);
@@ -3676,7 +3676,7 @@ ring_fl_db(struct adapter *sc, struct sg
if (fl->udb)
*fl->udb = htole32(v);
else
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v);
+ t4_write_reg(sc, sc->sge_kdoorbell_reg, v);
IDXINCR(fl->dbidx, n, fl->sidx);
}
@@ -4409,7 +4409,7 @@ ring_eq_db(struct adapter *sc, struct sg
break;
case DOORBELL_KDB:
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
+ t4_write_reg(sc, sc->sge_kdoorbell_reg,
V_QID(eq->cntxt_id) | V_PIDX(n));
break;
}
Modified: user/alc/PQ_LAUNDRY/sys/dev/pci/pci_pci.c
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/dev/pci/pci_pci.c Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/dev/pci/pci_pci.c Tue Aug 2 02:32:00 2016 (r303654)
@@ -932,6 +932,13 @@ pcib_probe_hotplug(struct pcib_softc *sc
sc->pcie_link_cap = pcie_read_config(dev, PCIER_LINK_CAP, 4);
sc->pcie_slot_cap = pcie_read_config(dev, PCIER_SLOT_CAP, 4);
+ /*
+ * XXX: Handling of slots with a power controller needs to be
+ * reexamined. Ignore hotplug on such slots for now.
+ */
+ if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_PCP)
+ return;
+
if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_HPC)
sc->flags |= PCIB_HOTPLUG;
}
Modified: user/alc/PQ_LAUNDRY/sys/kern/kern_mutex.c
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/kern/kern_mutex.c Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/kern/kern_mutex.c Tue Aug 2 02:32:00 2016 (r303654)
@@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$");
#include <sys/resourcevar.h>
#include <sys/sched.h>
#include <sys/sbuf.h>
+#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/turnstile.h>
#include <sys/vmmeter.h>
@@ -138,6 +139,37 @@ struct lock_class lock_class_mtx_spin =
#endif
};
+#ifdef ADAPTIVE_MUTEXES
+static SYSCTL_NODE(_debug, OID_AUTO, mtx, CTLFLAG_RD, NULL, "mtx debugging");
+
+static struct lock_delay_config mtx_delay = {
+ .initial = 1000,
+ .step = 500,
+ .min = 100,
+ .max = 5000,
+};
+
+SYSCTL_INT(_debug_mtx, OID_AUTO, delay_initial, CTLFLAG_RW, &mtx_delay.initial,
+ 0, "");
+SYSCTL_INT(_debug_mtx, OID_AUTO, delay_step, CTLFLAG_RW, &mtx_delay.step,
+ 0, "");
+SYSCTL_INT(_debug_mtx, OID_AUTO, delay_min, CTLFLAG_RW, &mtx_delay.min,
+ 0, "");
+SYSCTL_INT(_debug_mtx, OID_AUTO, delay_max, CTLFLAG_RW, &mtx_delay.max,
+ 0, "");
+
+static void
+mtx_delay_sysinit(void *dummy)
+{
+
+ mtx_delay.initial = mp_ncpus * 25;
+ mtx_delay.step = (mp_ncpus * 25) / 2;
+ mtx_delay.min = mp_ncpus * 5;
+ mtx_delay.max = mp_ncpus * 25 * 10;
+}
+LOCK_DELAY_SYSINIT(mtx_delay_sysinit);
+#endif
+
/*
* System-wide mutexes
*/
@@ -408,8 +440,10 @@ __mtx_lock_sleep(volatile uintptr_t *c,
int contested = 0;
uint64_t waittime = 0;
#endif
+#if defined(ADAPTIVE_MUTEXES) || defined(KDTRACE_HOOKS)
+ struct lock_delay_arg lda;
+#endif
#ifdef KDTRACE_HOOKS
- u_int spin_cnt = 0;
u_int sleep_cnt = 0;
int64_t sleep_time = 0;
int64_t all_time = 0;
@@ -418,6 +452,9 @@ __mtx_lock_sleep(volatile uintptr_t *c,
if (SCHEDULER_STOPPED())
return;
+#if defined(ADAPTIVE_MUTEXES) || defined(KDTRACE_HOOKS)
+ lock_delay_arg_init(&lda, &mtx_delay);
+#endif
m = mtxlock2mtx(c);
if (mtx_owned(m)) {
@@ -451,7 +488,7 @@ __mtx_lock_sleep(volatile uintptr_t *c,
if (m->mtx_lock == MTX_UNOWNED && _mtx_obtain_lock(m, tid))
break;
#ifdef KDTRACE_HOOKS
- spin_cnt++;
+ lda.spin_cnt++;
#endif
#ifdef ADAPTIVE_MUTEXES
/*
@@ -471,12 +508,8 @@ __mtx_lock_sleep(volatile uintptr_t *c,
"spinning", "lockname:\"%s\"",
m->lock_object.lo_name);
while (mtx_owner(m) == owner &&
- TD_IS_RUNNING(owner)) {
- cpu_spinwait();
-#ifdef KDTRACE_HOOKS
- spin_cnt++;
-#endif
- }
+ TD_IS_RUNNING(owner))
+ lock_delay(&lda);
KTR_STATE0(KTR_SCHED, "thread",
sched_tdname((struct thread *)tid),
"running");
@@ -570,7 +603,7 @@ __mtx_lock_sleep(volatile uintptr_t *c,
/*
* Only record the loops spinning and not sleeping.
*/
- if (spin_cnt > sleep_cnt)
+ if (lda.spin_cnt > sleep_cnt)
LOCKSTAT_RECORD1(adaptive__spin, m, all_time - sleep_time);
#endif
}
Modified: user/alc/PQ_LAUNDRY/sys/kern/kern_rwlock.c
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/kern/kern_rwlock.c Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/kern/kern_rwlock.c Tue Aug 2 02:32:00 2016 (r303654)
@@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$");
#include <sys/proc.h>
#include <sys/rwlock.h>
#include <sys/sched.h>
+#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/turnstile.h>
@@ -65,15 +66,6 @@ PMC_SOFT_DECLARE( , , lock, failed);
*/
#define rwlock2rw(c) (__containerof(c, struct rwlock, rw_lock))
-#ifdef ADAPTIVE_RWLOCKS
-static int rowner_retries = 10;
-static int rowner_loops = 10000;
-static SYSCTL_NODE(_debug, OID_AUTO, rwlock, CTLFLAG_RD, NULL,
- "rwlock debugging");
-SYSCTL_INT(_debug_rwlock, OID_AUTO, retry, CTLFLAG_RW, &rowner_retries, 0, "");
-SYSCTL_INT(_debug_rwlock, OID_AUTO, loops, CTLFLAG_RW, &rowner_loops, 0, "");
-#endif
-
#ifdef DDB
#include <ddb/ddb.h>
@@ -100,6 +92,42 @@ struct lock_class lock_class_rw = {
#endif
};
+#ifdef ADAPTIVE_RWLOCKS
+static int rowner_retries = 10;
+static int rowner_loops = 10000;
+static SYSCTL_NODE(_debug, OID_AUTO, rwlock, CTLFLAG_RD, NULL,
+ "rwlock debugging");
+SYSCTL_INT(_debug_rwlock, OID_AUTO, retry, CTLFLAG_RW, &rowner_retries, 0, "");
+SYSCTL_INT(_debug_rwlock, OID_AUTO, loops, CTLFLAG_RW, &rowner_loops, 0, "");
+
+static struct lock_delay_config rw_delay = {
+ .initial = 1000,
+ .step = 500,
+ .min = 100,
+ .max = 5000,
+};
+
+SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_initial, CTLFLAG_RW, &rw_delay.initial,
+ 0, "");
+SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_step, CTLFLAG_RW, &rw_delay.step,
+ 0, "");
+SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_min, CTLFLAG_RW, &rw_delay.min,
+ 0, "");
+SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_max, CTLFLAG_RW, &rw_delay.max,
+ 0, "");
+
+static void
+rw_delay_sysinit(void *dummy)
+{
+
+ rw_delay.initial = mp_ncpus * 25;
+ rw_delay.step = (mp_ncpus * 25) / 2;
+ rw_delay.min = mp_ncpus * 5;
+ rw_delay.max = mp_ncpus * 25 * 10;
+}
+LOCK_DELAY_SYSINIT(rw_delay_sysinit);
+#endif
+
/*
* Return a pointer to the owning thread if the lock is write-locked or
* NULL if the lock is unlocked or read-locked.
@@ -355,9 +383,11 @@ __rw_rlock(volatile uintptr_t *c, const
int contested = 0;
#endif
uintptr_t v;
+#if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS)
+ struct lock_delay_arg lda;
+#endif
#ifdef KDTRACE_HOOKS
uintptr_t state;
- u_int spin_cnt = 0;
u_int sleep_cnt = 0;
int64_t sleep_time = 0;
int64_t all_time = 0;
@@ -366,6 +396,9 @@ __rw_rlock(volatile uintptr_t *c, const
if (SCHEDULER_STOPPED())
return;
+#if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS)
+ lock_delay_arg_init(&lda, &rw_delay);
+#endif
rw = rwlock2rw(c);
KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
@@ -412,7 +445,7 @@ __rw_rlock(volatile uintptr_t *c, const
continue;
}
#ifdef KDTRACE_HOOKS
- spin_cnt++;
+ lda.spin_cnt++;
#endif
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
@@ -437,12 +470,8 @@ __rw_rlock(volatile uintptr_t *c, const
sched_tdname(curthread), "spinning",
"lockname:\"%s\"", rw->lock_object.lo_name);
while ((struct thread*)RW_OWNER(rw->rw_lock) ==
- owner && TD_IS_RUNNING(owner)) {
- cpu_spinwait();
-#ifdef KDTRACE_HOOKS
- spin_cnt++;
-#endif
- }
+ owner && TD_IS_RUNNING(owner))
+ lock_delay(&lda);
KTR_STATE0(KTR_SCHED, "thread",
sched_tdname(curthread), "running");
continue;
@@ -459,7 +488,7 @@ __rw_rlock(volatile uintptr_t *c, const
cpu_spinwait();
}
#ifdef KDTRACE_HOOKS
- spin_cnt += rowner_loops - i;
+ lda.spin_cnt += rowner_loops - i;
#endif
KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread),
"running");
@@ -552,7 +581,7 @@ __rw_rlock(volatile uintptr_t *c, const
(state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state));
/* Record only the loops spinning and not sleeping. */
- if (spin_cnt > sleep_cnt)
+ if (lda.spin_cnt > sleep_cnt)
LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time,
LOCKSTAT_READER, (state & RW_LOCK_READ) == 0,
(state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state));
@@ -740,9 +769,11 @@ __rw_wlock_hard(volatile uintptr_t *c, u
uint64_t waittime = 0;
int contested = 0;
#endif
+#if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS)
+ struct lock_delay_arg lda;
+#endif
#ifdef KDTRACE_HOOKS
uintptr_t state;
- u_int spin_cnt = 0;
u_int sleep_cnt = 0;
int64_t sleep_time = 0;
int64_t all_time = 0;
@@ -751,6 +782,9 @@ __rw_wlock_hard(volatile uintptr_t *c, u
if (SCHEDULER_STOPPED())
return;
+#if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS)
+ lock_delay_arg_init(&lda, &rw_delay);
+#endif
rw = rwlock2rw(c);
if (rw_wlocked(rw)) {
@@ -775,7 +809,7 @@ __rw_wlock_hard(volatile uintptr_t *c, u
if (rw->rw_lock == RW_UNLOCKED && _rw_write_lock(rw, tid))
break;
#ifdef KDTRACE_HOOKS
- spin_cnt++;
+ lda.spin_cnt++;
#endif
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
@@ -798,12 +832,8 @@ __rw_wlock_hard(volatile uintptr_t *c, u
"spinning", "lockname:\"%s\"",
rw->lock_object.lo_name);
while ((struct thread*)RW_OWNER(rw->rw_lock) == owner &&
- TD_IS_RUNNING(owner)) {
- cpu_spinwait();
-#ifdef KDTRACE_HOOKS
- spin_cnt++;
-#endif
- }
+ TD_IS_RUNNING(owner))
+ lock_delay(&lda);
KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread),
"running");
continue;
@@ -828,7 +858,7 @@ __rw_wlock_hard(volatile uintptr_t *c, u
KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread),
"running");
#ifdef KDTRACE_HOOKS
- spin_cnt += rowner_loops - i;
+ lda.spin_cnt += rowner_loops - i;
#endif
if (i != rowner_loops)
continue;
@@ -918,7 +948,7 @@ __rw_wlock_hard(volatile uintptr_t *c, u
(state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state));
/* Record only the loops spinning and not sleeping. */
- if (spin_cnt > sleep_cnt)
+ if (lda.spin_cnt > sleep_cnt)
LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time,
LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0,
(state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state));
Modified: user/alc/PQ_LAUNDRY/sys/kern/kern_sx.c
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/kern/kern_sx.c Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/kern/kern_sx.c Tue Aug 2 02:32:00 2016 (r303654)
@@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$");
#include <sys/sched.h>
#include <sys/sleepqueue.h>
#include <sys/sx.h>
+#include <sys/smp.h>
#include <sys/sysctl.h>
#if defined(SMP) && !defined(NO_ADAPTIVE_SX)
@@ -145,6 +146,33 @@ static u_int asx_loops = 10000;
static SYSCTL_NODE(_debug, OID_AUTO, sx, CTLFLAG_RD, NULL, "sxlock debugging");
SYSCTL_UINT(_debug_sx, OID_AUTO, retries, CTLFLAG_RW, &asx_retries, 0, "");
SYSCTL_UINT(_debug_sx, OID_AUTO, loops, CTLFLAG_RW, &asx_loops, 0, "");
+
+static struct lock_delay_config sx_delay = {
+ .initial = 1000,
+ .step = 500,
+ .min = 100,
+ .max = 5000,
+};
+
+SYSCTL_INT(_debug_sx, OID_AUTO, delay_initial, CTLFLAG_RW, &sx_delay.initial,
+ 0, "");
+SYSCTL_INT(_debug_sx, OID_AUTO, delay_step, CTLFLAG_RW, &sx_delay.step,
+ 0, "");
+SYSCTL_INT(_debug_sx, OID_AUTO, delay_min, CTLFLAG_RW, &sx_delay.min,
+ 0, "");
+SYSCTL_INT(_debug_sx, OID_AUTO, delay_max, CTLFLAG_RW, &sx_delay.max,
+ 0, "");
+
+static void
+sx_delay_sysinit(void *dummy)
+{
+
+ sx_delay.initial = mp_ncpus * 25;
+ sx_delay.step = (mp_ncpus * 25) / 2;
+ sx_delay.min = mp_ncpus * 5;
+ sx_delay.max = mp_ncpus * 25 * 10;
+}
+LOCK_DELAY_SYSINIT(sx_delay_sysinit);
#endif
void
@@ -513,9 +541,11 @@ _sx_xlock_hard(struct sx *sx, uintptr_t
int contested = 0;
#endif
int error = 0;
+#if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS)
+ struct lock_delay_arg lda;
+#endif
#ifdef KDTRACE_HOOKS
uintptr_t state;
- u_int spin_cnt = 0;
u_int sleep_cnt = 0;
int64_t sleep_time = 0;
int64_t all_time = 0;
@@ -524,6 +554,10 @@ _sx_xlock_hard(struct sx *sx, uintptr_t
if (SCHEDULER_STOPPED())
return (0);
+#if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS)
+ lock_delay_arg_init(&lda, &sx_delay);
+#endif
+
/* If we already hold an exclusive lock, then recurse. */
if (sx_xlocked(sx)) {
KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0,
@@ -549,7 +583,7 @@ _sx_xlock_hard(struct sx *sx, uintptr_t
atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid))
break;
#ifdef KDTRACE_HOOKS
- spin_cnt++;
+ lda.spin_cnt++;
#endif
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
@@ -578,12 +612,8 @@ _sx_xlock_hard(struct sx *sx, uintptr_t
sx->lock_object.lo_name);
GIANT_SAVE();
while (SX_OWNER(sx->sx_lock) == x &&
- TD_IS_RUNNING(owner)) {
- cpu_spinwait();
-#ifdef KDTRACE_HOOKS
- spin_cnt++;
-#endif
- }
+ TD_IS_RUNNING(owner))
+ lock_delay(&lda);
KTR_STATE0(KTR_SCHED, "thread",
sched_tdname(curthread), "running");
continue;
@@ -605,7 +635,7 @@ _sx_xlock_hard(struct sx *sx, uintptr_t
break;
cpu_spinwait();
#ifdef KDTRACE_HOOKS
- spin_cnt++;
+ lda.spin_cnt++;
#endif
}
KTR_STATE0(KTR_SCHED, "thread",
@@ -725,7 +755,7 @@ _sx_xlock_hard(struct sx *sx, uintptr_t
LOCKSTAT_RECORD4(sx__block, sx, sleep_time,
LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0,
(state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
- if (spin_cnt > sleep_cnt)
+ if (lda.spin_cnt > sleep_cnt)
LOCKSTAT_RECORD4(sx__spin, sx, all_time - sleep_time,
LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0,
(state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
@@ -818,9 +848,11 @@ _sx_slock_hard(struct sx *sx, int opts,
#endif
uintptr_t x;
int error = 0;
+#if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS)
+ struct lock_delay_arg lda;
+#endif
#ifdef KDTRACE_HOOKS
uintptr_t state;
- u_int spin_cnt = 0;
u_int sleep_cnt = 0;
int64_t sleep_time = 0;
int64_t all_time = 0;
@@ -829,6 +861,9 @@ _sx_slock_hard(struct sx *sx, int opts,
if (SCHEDULER_STOPPED())
return (0);
+#if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS)
+ lock_delay_arg_init(&lda, &sx_delay);
+#endif
#ifdef KDTRACE_HOOKS
state = sx->sx_lock;
all_time -= lockstat_nsecs(&sx->lock_object);
@@ -840,7 +875,7 @@ _sx_slock_hard(struct sx *sx, int opts,
*/
for (;;) {
#ifdef KDTRACE_HOOKS
- spin_cnt++;
+ lda.spin_cnt++;
#endif
x = sx->sx_lock;
@@ -888,12 +923,8 @@ _sx_slock_hard(struct sx *sx, int opts,
"lockname:\"%s\"", sx->lock_object.lo_name);
GIANT_SAVE();
while (SX_OWNER(sx->sx_lock) == x &&
- TD_IS_RUNNING(owner)) {
- cpu_spinwait();
-#ifdef KDTRACE_HOOKS
- spin_cnt++;
-#endif
- }
+ TD_IS_RUNNING(owner))
+ lock_delay(&lda);
KTR_STATE0(KTR_SCHED, "thread",
sched_tdname(curthread), "running");
continue;
@@ -989,7 +1020,7 @@ _sx_slock_hard(struct sx *sx, int opts,
LOCKSTAT_RECORD4(sx__block, sx, sleep_time,
LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0,
(state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
- if (spin_cnt > sleep_cnt)
+ if (lda.spin_cnt > sleep_cnt)
LOCKSTAT_RECORD4(sx__spin, sx, all_time - sleep_time,
LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0,
(state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
Modified: user/alc/PQ_LAUNDRY/sys/kern/subr_lock.c
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/kern/subr_lock.c Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/kern/subr_lock.c Tue Aug 2 02:32:00 2016 (r303654)
@@ -103,6 +103,34 @@ lock_destroy(struct lock_object *lock)
lock->lo_flags &= ~LO_INITIALIZED;
}
+void
+lock_delay(struct lock_delay_arg *la)
+{
+ u_int i, delay, backoff, min, max;
+ struct lock_delay_config *lc = la->config;
+
+ delay = la->delay;
+
+ if (delay == 0)
+ delay = lc->initial;
+ else {
+ delay += lc->step;
+ max = lc->max;
+ if (delay > max)
+ delay = max;
+ }
+
+ backoff = cpu_ticks() % delay;
+ min = lc->min;
+ if (backoff < min)
+ backoff = min;
+ for (i = 0; i < backoff; i++)
+ cpu_spinwait();
+
+ la->delay = delay;
+ la->spin_cnt += backoff;
+}
+
#ifdef DDB
DB_SHOW_COMMAND(lock, db_show_lock)
{
Modified: user/alc/PQ_LAUNDRY/sys/net/route.h
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/net/route.h Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/net/route.h Tue Aug 2 02:32:00 2016 (r303654)
@@ -360,7 +360,7 @@ struct rt_addrinfo {
|| (ifp)->if_link_state == LINK_STATE_UP)
#define RT_LOCK_INIT(_rt) \
- mtx_init(&(_rt)->rt_mtx, "rtentry", NULL, MTX_DEF | MTX_DUPOK)
+ mtx_init(&(_rt)->rt_mtx, "rtentry", NULL, MTX_DEF | MTX_DUPOK | MTX_NEW)
#define RT_LOCK(_rt) mtx_lock(&(_rt)->rt_mtx)
#define RT_UNLOCK(_rt) mtx_unlock(&(_rt)->rt_mtx)
#define RT_LOCK_DESTROY(_rt) mtx_destroy(&(_rt)->rt_mtx)
Modified: user/alc/PQ_LAUNDRY/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c Tue Aug 2 02:32:00 2016 (r303654)
@@ -660,7 +660,13 @@ ipoib_unicast_send(struct mbuf *mb, stru
new_path = 1;
}
if (path) {
- _IF_ENQUEUE(&path->queue, mb);
+ if (_IF_QLEN(&path->queue) < IPOIB_MAX_PATH_REC_QUEUE)
+ _IF_ENQUEUE(&path->queue, mb);
+ else {
+ if_inc_counter(priv->dev, IFCOUNTER_OERRORS, 1);
+ m_freem(mb);
+ }
+
if (!path->query && path_rec_start(priv, path)) {
spin_unlock_irqrestore(&priv->lock, flags);
if (new_path)
Modified: user/alc/PQ_LAUNDRY/sys/opencrypto/xform_aes_icm.c
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/opencrypto/xform_aes_icm.c Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/opencrypto/xform_aes_icm.c Tue Aug 2 02:32:00 2016 (r303654)
@@ -65,7 +65,7 @@ struct enc_xform enc_xform_aes_icm = {
aes_icm_crypt,
aes_icm_crypt,
aes_icm_setkey,
- rijndael128_zerokey,
+ aes_icm_zerokey,
aes_icm_reinit,
};
Modified: user/alc/PQ_LAUNDRY/sys/sys/lock.h
==============================================================================
--- user/alc/PQ_LAUNDRY/sys/sys/lock.h Tue Aug 2 02:29:48 2016 (r303653)
+++ user/alc/PQ_LAUNDRY/sys/sys/lock.h Tue Aug 2 02:32:00 2016 (r303654)
@@ -201,9 +201,33 @@ extern struct lock_class lock_class_lock
extern struct lock_class *lock_classes[];
+struct lock_delay_config {
+ u_int initial;
+ u_int step;
+ u_int min;
+ u_int max;
+};
+
+struct lock_delay_arg {
+ struct lock_delay_config *config;
+ u_int delay;
+ u_int spin_cnt;
+};
+
+static inline void
+lock_delay_arg_init(struct lock_delay_arg *la, struct lock_delay_config *lc) {
+ la->config = lc;
+ la->delay = 0;
+ la->spin_cnt = 0;
+}
+
+#define LOCK_DELAY_SYSINIT(func) \
+ SYSINIT(func##_ld, SI_SUB_LOCK, SI_ORDER_ANY, func, NULL)
+
void lock_init(struct lock_object *, struct lock_class *,
const char *, const char *, int);
void lock_destroy(struct lock_object *);
+void lock_delay(struct lock_delay_arg *);
void spinlock_enter(void);
void spinlock_exit(void);
void witness_init(struct lock_object *, const char *);
More information about the svn-src-user
mailing list