svn commit: r247319 - in projects/calloutng/sys: kern sys
Alexander Motin
mav at FreeBSD.org
Tue Feb 26 15:25:44 UTC 2013
Author: mav
Date: Tue Feb 26 15:25:43 2013
New Revision: 247319
URL: http://svnweb.freebsd.org/changeset/base/247319
Log:
Optimize callout_process() to use less variables and less conditions to
implement the same logic. Now it fits better into CPU registers, and
according to PMC significntly reduces number of resource stalls, reducing
consumed by it CPU time during usleep(1) benchmark by 30%.
Remove callout_new_inserted variable and call cpu_new_callout() function
directly. Since most architectures are using it and there is #define to
identify others, there is no point to keep this extra indirection.
Reviewed by: davide
Modified:
projects/calloutng/sys/kern/kern_clocksource.c
projects/calloutng/sys/kern/kern_timeout.c
projects/calloutng/sys/sys/callout.h
projects/calloutng/sys/sys/systm.h
Modified: projects/calloutng/sys/kern/kern_clocksource.c
==============================================================================
--- projects/calloutng/sys/kern/kern_clocksource.c Tue Feb 26 13:59:39 2013 (r247318)
+++ projects/calloutng/sys/kern/kern_clocksource.c Tue Feb 26 15:25:43 2013 (r247319)
@@ -73,8 +73,6 @@ static int round_freq(struct eventtimer
static sbintime_t getnextcpuevent(int idle);
static sbintime_t getnextevent(void);
static int handleevents(sbintime_t now, int fake);
-static void cpu_new_callout(int cpu, sbintime_t bt,
- sbintime_t bt_opt);
static struct mtx et_hw_mtx;
@@ -99,7 +97,7 @@ static sbintime_t timerperiod; /* Timer
static sbintime_t statperiod; /* statclock() events period. */
static sbintime_t profperiod; /* profclock() events period. */
static sbintime_t nexttick; /* Next global timer tick time. */
-static u_int busy = 0; /* Reconfiguration is in progress. */
+static u_int busy = 1; /* Reconfiguration is in progress. */
static int profiling = 0; /* Profiling events enabled. */
static char timername[32]; /* Wanted timer. */
@@ -527,6 +525,8 @@ configtimer(int start)
state->nexthard = next;
state->nextstat = next;
state->nextprof = next;
+ state->nextcall = next;
+ state->nextcallopt = next;
hardclock_sync(cpu);
}
busy = 0;
@@ -609,7 +609,6 @@ cpu_initclocks_bsp(void)
state->nextcall = -1;
state->nextcallopt = -1;
}
- callout_new_inserted = cpu_new_callout;
periodic = want_periodic;
/* Grab requested timer or the best of present. */
if (timername[0])
@@ -803,6 +802,9 @@ clocksource_cyc_set(const struct bintime
sbintime_t now, t;
struct pcpu_state *state;
+ /* Do not touch anything if somebody reconfiguring timers. */
+ if (busy)
+ return;
t = bintime2sbintime(*bt);
state = DPCPU_PTR(timerstate);
if (periodic)
@@ -831,12 +833,15 @@ clocksource_cyc_set(const struct bintime
}
#endif
-static void
+void
cpu_new_callout(int cpu, sbintime_t bt, sbintime_t bt_opt)
{
sbintime_t now;
struct pcpu_state *state;
+ /* Do not touch anything if somebody reconfiguring timers. */
+ if (busy)
+ return;
CTR6(KTR_SPARE2, "new co at %d: on %d at %d.%08x - %d.%08x",
curcpu, cpu, (int)(bt_opt >> 32), (u_int)(bt_opt & 0xffffffff),
(int)(bt >> 32), (u_int)(bt & 0xffffffff));
Modified: projects/calloutng/sys/kern/kern_timeout.c
==============================================================================
--- projects/calloutng/sys/kern/kern_timeout.c Tue Feb 26 13:59:39 2013 (r247318)
+++ projects/calloutng/sys/kern/kern_timeout.c Tue Feb 26 15:25:43 2013 (r247319)
@@ -179,7 +179,6 @@ struct callout_cpu cc_cpu;
(sizeof(time_t) == (sizeof(int64_t)) ? INT64_MAX : INT32_MAX)
static int timeout_cpu;
-void (*callout_new_inserted)(int cpu, sbintime_t bt, sbintime_t bt_opt) = NULL;
static void
softclock_call_cc(struct callout *c, struct callout_cpu *cc,
#ifdef CALLOUT_PROFILING
@@ -391,12 +390,11 @@ callout_process(sbintime_t now)
struct callout_tailq *sc;
sbintime_t first, last, max, tmp_max;
uint32_t lookahead;
- int firstb, lastb, nowb, need_softclock, exit_allowed, exit_wanted;
+ u_int firstb, lastb, nowb;
#ifdef CALLOUT_PROFILING
int depth_dir = 0, mpcalls_dir = 0, lockcalls_dir = 0;
#endif
- need_softclock = 0;
cc = CC_SELF();
mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET);
@@ -427,14 +425,10 @@ callout_process(sbintime_t now)
lastb = firstb - 1;
if (nowb - firstb >= callwheelsize)
nowb = firstb - 1;
- nowb &= callwheelmask;
- lastb &= callwheelmask;
- firstb &= callwheelmask;
/* Iterate callwheel from firstb to nowb and then up to lastb. */
- exit_allowed = exit_wanted = 0;
- for (;;) {
- sc = &cc->cc_callwheel[firstb];
+ do {
+ sc = &cc->cc_callwheel[firstb & callwheelmask];
tmp = TAILQ_FIRST(sc);
while (tmp != NULL) {
/* Run the callout if present time within allowed. */
@@ -462,7 +456,6 @@ callout_process(sbintime_t now)
TAILQ_INSERT_TAIL(&cc->cc_expireq,
tmp, c_links.tqe);
tmp->c_flags |= CALLOUT_PROCESSED;
- need_softclock = 1;
tmp = tmpn;
}
continue;
@@ -475,37 +468,31 @@ callout_process(sbintime_t now)
* time, so it cannot be aggregated.
*/
if (tmp->c_time > last) {
- exit_wanted = 1;
+ lastb = nowb;
goto next;
}
/* Update first and last time, respecting this event. */
if (tmp->c_time < first)
first = tmp->c_time;
- tmp_max = tmp->c_time;
- tmp_max += tmp->c_precision;
+ tmp_max = tmp->c_time + tmp->c_precision;
if (tmp_max < last)
last = tmp_max;
next:
tmp = TAILQ_NEXT(tmp, c_links.tqe);
}
- /* Stop if we looked far enough into the future. */
- if (firstb == lastb)
- break;
+ /* Proceed with the next bucket. */
+ firstb++;
/*
* Stop if we looked after present time and found
* some event we can't execute at now.
+ * Stop if we looked far enough into the future.
*/
- if (firstb == nowb)
- exit_allowed = 1;
- if (exit_allowed && exit_wanted)
- break;
- /* Proceed with the next bucket. */
- firstb = (firstb + 1) & callwheelmask;
- }
+ } while (firstb <= lastb);
cc->cc_exec_next_dir = NULL;
- if (callout_new_inserted != NULL)
- (*callout_new_inserted)(curcpu, last, first);
cc->cc_firstevent = last;
+#ifndef NO_EVENTTIMERS
+ cpu_new_callout(curcpu, last, first);
+#endif
#ifdef CALLOUT_PROFILING
avg_depth_dir += (depth_dir * 1000 - avg_depth_dir) >> 8;
avg_mpcalls_dir += (mpcalls_dir * 1000 - avg_mpcalls_dir) >> 8;
@@ -516,7 +503,7 @@ next:
* swi_sched acquires the thread lock, so we don't want to call it
* with cc_lock held; incorrect locking order.
*/
- if (need_softclock)
+ if (!TAILQ_EMPTY(&cc->cc_expireq))
swi_sched(cc->cc_cookie, 0);
}
@@ -549,7 +536,6 @@ callout_cc_add(struct callout *c, struct
sbintime_t sbt, sbintime_t precision, void (*func)(void *),
void *arg, int cpu, int flags)
{
- sbintime_t last;
int bucket;
CC_LOCK_ASSERT(cc);
@@ -568,16 +554,17 @@ callout_cc_add(struct callout *c, struct
c, (int)(c->c_precision >> 32),
(u_int)(c->c_precision & 0xffffffff));
TAILQ_INSERT_TAIL(&cc->cc_callwheel[bucket], c, c_links.tqe);
+#ifndef NO_EVENTTIMERS
/*
* Inform the eventtimers(4) subsystem there's a new callout
* that has been inserted, but only if really required.
*/
- last = c->c_time + c->c_precision;
- if (callout_new_inserted != NULL && ((last < cc->cc_firstevent) ||
- (cc->cc_firstevent == 0))) {
- cc->cc_firstevent = last;
- (*callout_new_inserted)(cpu, last, c->c_time);
+ sbt = c->c_time + c->c_precision;
+ if (sbt < cc->cc_firstevent || cc->cc_firstevent == 0) {
+ cc->cc_firstevent = sbt;
+ cpu_new_callout(cpu, sbt, c->c_time);
}
+#endif
}
static void
Modified: projects/calloutng/sys/sys/callout.h
==============================================================================
--- projects/calloutng/sys/sys/callout.h Tue Feb 26 13:59:39 2013 (r247318)
+++ projects/calloutng/sys/sys/callout.h Tue Feb 26 15:25:43 2013 (r247319)
@@ -97,7 +97,6 @@ int callout_schedule_on(struct callout *
#define callout_stop(c) _callout_stop_safe(c, 0)
int _callout_stop_safe(struct callout *, int);
void callout_process(sbintime_t now);
-extern void (*callout_new_inserted)(int cpu, sbintime_t bt, sbintime_t bt_opt);
#endif
Modified: projects/calloutng/sys/sys/systm.h
==============================================================================
--- projects/calloutng/sys/sys/systm.h Tue Feb 26 13:59:39 2013 (r247318)
+++ projects/calloutng/sys/sys/systm.h Tue Feb 26 15:25:43 2013 (r247319)
@@ -269,6 +269,7 @@ void cpu_startprofclock(void);
void cpu_stopprofclock(void);
sbintime_t cpu_idleclock(void);
void cpu_activeclock(void);
+void cpu_new_callout(int cpu, sbintime_t bt, sbintime_t bt_opt);
extern int cpu_can_deep_sleep;
extern int cpu_disable_deep_sleep;
More information about the svn-src-projects
mailing list