svn commit: r220433 - in head/sys: amd64/amd64
cddl/dev/dtrace/amd64 cddl/dev/dtrace/i386 compat/linprocfs
contrib/altq/altq dev/acpica i386/i386 i386/isa pc98/pc98
x86/cpufreq x86/isa x86/x86
Jung-uk Kim
jkim at FreeBSD.org
Thu Apr 7 23:28:28 UTC 2011
Author: jkim
Date: Thu Apr 7 23:28:28 2011
New Revision: 220433
URL: http://svn.freebsd.org/changeset/base/220433
Log:
Use atomic load & store for TSC frequency. It may be overkill for amd64 but
safer for i386 because it can be easily over 4 GHz now. More worse, it can
be easily changed by user with 'machdep.tsc_freq' tunable (directly) or
cpufreq(4) (indirectly). Note it is intentionally not used in performance
critical paths to avoid performance regression (but we should, in theory).
Alternatively, we may add "virtual TSC" with lower frequency if maximum
frequency overflows 32 bits (and ignore possible incoherency as we do now).
Modified:
head/sys/amd64/amd64/legacy.c
head/sys/amd64/amd64/machdep.c
head/sys/amd64/amd64/prof_machdep.c
head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c
head/sys/cddl/dev/dtrace/i386/dtrace_subr.c
head/sys/compat/linprocfs/linprocfs.c
head/sys/contrib/altq/altq/altq_subr.c
head/sys/dev/acpica/acpi_cpu.c
head/sys/i386/i386/legacy.c
head/sys/i386/i386/machdep.c
head/sys/i386/i386/perfmon.c
head/sys/i386/isa/prof_machdep.c
head/sys/pc98/pc98/machdep.c
head/sys/x86/cpufreq/est.c
head/sys/x86/isa/clock.c
head/sys/x86/x86/tsc.c
Modified: head/sys/amd64/amd64/legacy.c
==============================================================================
--- head/sys/amd64/amd64/legacy.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/amd64/amd64/legacy.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -321,7 +321,8 @@ cpu_read_ivar(device_t dev, device_t chi
break;
case CPU_IVAR_NOMINAL_MHZ:
if (tsc_is_invariant) {
- *result = (uintptr_t)(tsc_freq / 1000000);
+ *result = (uintptr_t)(atomic_load_acq_64(&tsc_freq) /
+ 1000000);
break;
}
/* FALLTHROUGH */
Modified: head/sys/amd64/amd64/machdep.c
==============================================================================
--- head/sys/amd64/amd64/machdep.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/amd64/amd64/machdep.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -546,18 +546,19 @@ int
cpu_est_clockrate(int cpu_id, uint64_t *rate)
{
register_t reg;
- uint64_t tsc1, tsc2;
+ uint64_t freq, tsc1, tsc2;
if (pcpu_find(cpu_id) == NULL || rate == NULL)
return (EINVAL);
+ freq = atomic_load_acq_64(&tsc_freq);
/* If TSC is P-state invariant, DELAY(9) based logic fails. */
- if (tsc_is_invariant && tsc_freq != 0)
+ if (tsc_is_invariant && freq != 0)
return (EOPNOTSUPP);
/* If we're booting, trust the rate calibrated moments ago. */
- if (cold && tsc_freq != 0) {
- *rate = tsc_freq;
+ if (cold && freq != 0) {
+ *rate = freq;
return (0);
}
@@ -586,7 +587,7 @@ cpu_est_clockrate(int cpu_id, uint64_t *
#endif
tsc2 -= tsc1;
- if (tsc_freq != 0) {
+ if (freq != 0) {
*rate = tsc2 * 1000;
return (0);
}
Modified: head/sys/amd64/amd64/prof_machdep.c
==============================================================================
--- head/sys/amd64/amd64/prof_machdep.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/amd64/amd64/prof_machdep.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -311,18 +311,22 @@ void
startguprof(gp)
struct gmonparam *gp;
{
+ uint64_t freq;
+
+ freq = atomic_load_acq_64(&tsc_freq);
if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED) {
- cputime_clock = CPUTIME_CLOCK_I8254;
- if (tsc_freq != 0 && mp_ncpus == 1)
+ if (freq != 0 && mp_ncpus == 1)
cputime_clock = CPUTIME_CLOCK_TSC;
+ else
+ cputime_clock = CPUTIME_CLOCK_I8254;
}
- gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT;
if (cputime_clock == CPUTIME_CLOCK_TSC) {
- gp->profrate = tsc_freq >> 1;
+ gp->profrate = freq >> 1;
cputime_prof_active = 1;
- }
+ } else
+ gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT;
#if defined(PERFMON) && defined(I586_PMC_GUPROF)
- else if (cputime_clock == CPUTIME_CLOCK_I586_PMC) {
+ if (cputime_clock == CPUTIME_CLOCK_I586_PMC) {
if (perfmon_avail() &&
perfmon_setup(0, cputime_clock_pmc_conf) == 0) {
if (perfmon_start(0) != 0)
Modified: head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c
==============================================================================
--- head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -403,7 +403,7 @@ dtrace_gethrtime_init(void *arg)
* Otherwise tick->time conversion will be inaccurate, but
* will preserve monotonic property of TSC.
*/
- tsc_f = tsc_freq;
+ tsc_f = atomic_load_acq_64(&tsc_freq);
/*
* The following line checks that nsec_scale calculated below
Modified: head/sys/cddl/dev/dtrace/i386/dtrace_subr.c
==============================================================================
--- head/sys/cddl/dev/dtrace/i386/dtrace_subr.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/cddl/dev/dtrace/i386/dtrace_subr.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -403,7 +403,7 @@ dtrace_gethrtime_init(void *arg)
* Otherwise tick->time conversion will be inaccurate, but
* will preserve monotonic property of TSC.
*/
- tsc_f = tsc_freq;
+ tsc_f = atomic_load_acq_64(&tsc_freq);
/*
* The following line checks that nsec_scale calculated below
Modified: head/sys/compat/linprocfs/linprocfs.c
==============================================================================
--- head/sys/compat/linprocfs/linprocfs.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/compat/linprocfs/linprocfs.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -221,6 +221,7 @@ linprocfs_docpuinfo(PFS_FILL_ARGS)
{
int hw_model[2];
char model[128];
+ uint64_t freq;
size_t size;
int class, fqmhz, fqkhz;
int i;
@@ -303,9 +304,10 @@ linprocfs_docpuinfo(PFS_FILL_ARGS)
if (cpu_feature & (1 << i))
sbuf_printf(sb, " %s", flags[i]);
sbuf_cat(sb, "\n");
- if (class >= 5) {
- fqmhz = (tsc_freq + 4999) / 1000000;
- fqkhz = ((tsc_freq + 4999) / 10000) % 100;
+ freq = atomic_load_acq_64(&tsc_freq);
+ if (freq != 0) {
+ fqmhz = (freq + 4999) / 1000000;
+ fqkhz = ((freq + 4999) / 10000) % 100;
sbuf_printf(sb,
"cpu MHz\t\t: %d.%02d\n"
"bogomips\t: %d.%02d\n",
Modified: head/sys/contrib/altq/altq/altq_subr.c
==============================================================================
--- head/sys/contrib/altq/altq/altq_subr.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/contrib/altq/altq/altq_subr.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -929,7 +929,8 @@ init_machclk_setup(void)
#if defined(__amd64__) || defined(__i386__)
/* check if TSC is available */
#ifdef __FreeBSD__
- if ((cpu_feature & CPUID_TSC) == 0 || tsc_freq == 0)
+ if ((cpu_feature & CPUID_TSC) == 0 ||
+ atomic_load_acq_64(&tsc_freq) == 0)
#else
if ((cpu_feature & CPUID_TSC) == 0)
#endif
@@ -964,7 +965,7 @@ init_machclk(void)
*/
#if defined(__amd64__) || defined(__i386__)
#ifdef __FreeBSD__
- machclk_freq = tsc_freq;
+ machclk_freq = atomic_load_acq_64(&tsc_freq);
#elif defined(__NetBSD__)
machclk_freq = (u_int32_t)cpu_tsc_freq;
#elif defined(__OpenBSD__) && (defined(I586_CPU) || defined(I686_CPU))
Modified: head/sys/dev/acpica/acpi_cpu.c
==============================================================================
--- head/sys/dev/acpica/acpi_cpu.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/dev/acpica/acpi_cpu.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -516,7 +516,7 @@ acpi_cpu_read_ivar(device_t dev, device_
#if defined(__amd64__) || defined(__i386__)
case CPU_IVAR_NOMINAL_MHZ:
if (tsc_is_invariant) {
- *result = (uintptr_t)(tsc_freq / 1000000);
+ *result = (uintptr_t)(atomic_load_acq_64(&tsc_freq) / 1000000);
break;
}
/* FALLTHROUGH */
Modified: head/sys/i386/i386/legacy.c
==============================================================================
--- head/sys/i386/i386/legacy.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/i386/i386/legacy.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -342,7 +342,8 @@ cpu_read_ivar(device_t dev, device_t chi
break;
case CPU_IVAR_NOMINAL_MHZ:
if (tsc_is_invariant) {
- *result = (uintptr_t)(tsc_freq / 1000000);
+ *result = (uintptr_t)(atomic_load_acq_64(&tsc_freq) /
+ 1000000);
break;
}
/* FALLTHROUGH */
Modified: head/sys/i386/i386/machdep.c
==============================================================================
--- head/sys/i386/i386/machdep.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/i386/i386/machdep.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -1137,20 +1137,21 @@ int
cpu_est_clockrate(int cpu_id, uint64_t *rate)
{
register_t reg;
- uint64_t tsc1, tsc2;
+ uint64_t freq, tsc1, tsc2;
if (pcpu_find(cpu_id) == NULL || rate == NULL)
return (EINVAL);
if ((cpu_feature & CPUID_TSC) == 0)
return (EOPNOTSUPP);
+ freq = atomic_load_acq_64(&tsc_freq);
/* If TSC is P-state invariant, DELAY(9) based logic fails. */
- if (tsc_is_invariant && tsc_freq != 0)
+ if (tsc_is_invariant && freq != 0)
return (EOPNOTSUPP);
/* If we're booting, trust the rate calibrated moments ago. */
- if (cold && tsc_freq != 0) {
- *rate = tsc_freq;
+ if (cold && freq != 0) {
+ *rate = freq;
return (0);
}
@@ -1179,7 +1180,7 @@ cpu_est_clockrate(int cpu_id, uint64_t *
#endif
tsc2 -= tsc1;
- if (tsc_freq != 0) {
+ if (freq != 0) {
*rate = tsc2 * 1000;
return (0);
}
Modified: head/sys/i386/i386/perfmon.c
==============================================================================
--- head/sys/i386/i386/perfmon.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/i386/i386/perfmon.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -336,6 +336,7 @@ perfmon_ioctl(struct cdev *dev, u_long c
struct pmc *pmc;
struct pmc_data *pmcd;
struct pmc_tstamp *pmct;
+ uint64_t freq;
int *ip;
int rv;
@@ -386,13 +387,14 @@ perfmon_ioctl(struct cdev *dev, u_long c
break;
case PMIOTSTAMP:
- if (!tsc_freq) {
+ freq = atomic_load_acq_64(&tsc_freq);
+ if (freq == 0) {
rv = ENOTTY;
break;
}
pmct = (struct pmc_tstamp *)param;
/* XXX interface loses precision. */
- pmct->pmct_rate = tsc_freq / 1000000;
+ pmct->pmct_rate = freq / 1000000;
pmct->pmct_value = rdtsc();
rv = 0;
break;
Modified: head/sys/i386/isa/prof_machdep.c
==============================================================================
--- head/sys/i386/isa/prof_machdep.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/i386/isa/prof_machdep.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -170,8 +170,8 @@ cputime()
{
u_int count;
int delta;
-#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) && \
- defined(PERFMON) && defined(I586_PMC_GUPROF)
+#if (defined(I586_CPU) || defined(I686_CPU)) && \
+ defined(PERFMON) && defined(I586_PMC_GUPROF) && !defined(SMP)
u_quad_t event_count;
#endif
u_char high, low;
@@ -286,21 +286,23 @@ void
startguprof(gp)
struct gmonparam *gp;
{
- if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED) {
- cputime_clock = CPUTIME_CLOCK_I8254;
#if defined(I586_CPU) || defined(I686_CPU)
- if (tsc_freq != 0 && mp_ncpus == 1)
+ uint64_t freq;
+
+ freq = atomic_load_acq_64(&tsc_freq);
+ if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED) {
+ if (freq != 0 && mp_ncpus == 1)
cputime_clock = CPUTIME_CLOCK_TSC;
-#endif
+ else
+ cputime_clock = CPUTIME_CLOCK_I8254;
}
- gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT;
-#if defined(I586_CPU) || defined(I686_CPU)
if (cputime_clock == CPUTIME_CLOCK_TSC) {
- gp->profrate = tsc_freq >> 1;
+ gp->profrate = freq >> 1;
cputime_prof_active = 1;
- }
+ } else
+ gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT;
#if defined(PERFMON) && defined(I586_PMC_GUPROF)
- else if (cputime_clock == CPUTIME_CLOCK_I586_PMC) {
+ if (cputime_clock == CPUTIME_CLOCK_I586_PMC) {
if (perfmon_avail() &&
perfmon_setup(0, cputime_clock_pmc_conf) == 0) {
if (perfmon_start(0) != 0)
@@ -325,6 +327,10 @@ startguprof(gp)
}
}
#endif /* PERFMON && I586_PMC_GUPROF */
+#else /* !(I586_CPU || I686_CPU) */
+ if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED)
+ cputime_clock = CPUTIME_CLOCK_I8254;
+ gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT;
#endif /* I586_CPU || I686_CPU */
cputime_bias = 0;
cputime();
Modified: head/sys/pc98/pc98/machdep.c
==============================================================================
--- head/sys/pc98/pc98/machdep.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/pc98/pc98/machdep.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -1072,16 +1072,17 @@ int
cpu_est_clockrate(int cpu_id, uint64_t *rate)
{
register_t reg;
- uint64_t tsc1, tsc2;
+ uint64_t freq, tsc1, tsc2;
if (pcpu_find(cpu_id) == NULL || rate == NULL)
return (EINVAL);
if ((cpu_feature & CPUID_TSC) == 0)
return (EOPNOTSUPP);
+ freq = atomic_load_acq_64(&tsc_freq);
/* If we're booting, trust the rate calibrated moments ago. */
- if (cold && tsc_freq != 0) {
- *rate = tsc_freq;
+ if (cold && freq != 0) {
+ *rate = freq;
return (0);
}
@@ -1110,7 +1111,7 @@ cpu_est_clockrate(int cpu_id, uint64_t *
#endif
tsc2 -= tsc1;
- if (tsc_freq != 0) {
+ if (freq != 0) {
*rate = tsc2 * 1000;
return (0);
}
Modified: head/sys/x86/cpufreq/est.c
==============================================================================
--- head/sys/x86/cpufreq/est.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/x86/cpufreq/est.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -1215,7 +1215,7 @@ est_msr_info(device_t dev, uint64_t msr,
return (EOPNOTSUPP);
/* Figure out the bus clock. */
- freq = tsc_freq / 1000000;
+ freq = atomic_load_acq_64(&tsc_freq) / 1000000;
id = msr >> 32;
bus = freq / (id >> 8);
device_printf(dev, "Guessed bus clock (high) of %d MHz\n", bus);
Modified: head/sys/x86/isa/clock.c
==============================================================================
--- head/sys/x86/isa/clock.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/x86/isa/clock.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -246,13 +246,13 @@ getit(void)
}
static __inline void
-delay_tsc(int n)
+delay_tsc(int n, uint64_t freq)
{
uint64_t start, end, now;
sched_pin();
start = rdtsc();
- end = start + (tsc_freq * n) / 1000000;
+ end = start + (freq * n) / 1000000;
do {
cpu_spinwait();
now = rdtsc();
@@ -290,6 +290,7 @@ void
DELAY(int n)
{
struct timecounter *tc;
+ uint64_t freq;
int delta, prev_tick, tick, ticks_left;
#ifdef DELAYDEBUG
@@ -298,8 +299,9 @@ DELAY(int n)
static int state = 0;
#endif
- if (tsc_freq != 0) {
- delay_tsc(n);
+ freq = atomic_load_acq_64(&tsc_freq);
+ if (freq != 0) {
+ delay_tsc(n, freq);
return;
}
tc = timecounter;
Modified: head/sys/x86/x86/tsc.c
==============================================================================
--- head/sys/x86/x86/tsc.c Thu Apr 7 23:14:33 2011 (r220432)
+++ head/sys/x86/x86/tsc.c Thu Apr 7 23:28:28 2011 (r220433)
@@ -245,14 +245,16 @@ tsc_freq_changing(void *arg, const struc
static void
tsc_freq_changed(void *arg, const struct cf_level *level, int status)
{
+ uint64_t freq;
/* If there was an error during the transition, don't do anything. */
if (tsc_disabled || status != 0)
return;
/* Total setting for this level gives the new frequency in MHz. */
- tsc_freq = (uint64_t)level->total_set.freq * 1000000;
- tsc_timecounter.tc_frequency = tsc_freq;
+ freq = (uint64_t)level->total_set.freq * 1000000;
+ atomic_store_rel_64(&tsc_freq, freq);
+ atomic_store_rel_64(&tsc_timecounter.tc_frequency, freq);
}
static int
@@ -261,13 +263,13 @@ sysctl_machdep_tsc_freq(SYSCTL_HANDLER_A
int error;
uint64_t freq;
- if (tsc_timecounter.tc_frequency == 0)
+ freq = atomic_load_acq_64(&tsc_freq);
+ if (freq == 0)
return (EOPNOTSUPP);
- freq = tsc_freq;
error = sysctl_handle_64(oidp, &freq, 0, req);
if (error == 0 && req->newptr != NULL) {
- tsc_freq = freq;
- tsc_timecounter.tc_frequency = tsc_freq;
+ atomic_store_rel_64(&tsc_freq, freq);
+ atomic_store_rel_64(&tsc_timecounter.tc_frequency, freq);
}
return (error);
}
More information about the svn-src-head
mailing list