From d11ca916b80e62ec1a3f9259912582ef099e9090 Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Tue, 21 Jan 2014 16:56:38 -0800 Subject: [PATCH] sched_clock: Backport high-resolution support sched_clock: Use seqcount instead of rolling our own We're going to increase the cyc value to 64 bits in the near future. Doing that is going to break the custom seqcount implementation in the sched_clock code because 64 bit numbers aren't guaranteed to be atomic. Replace the cyc_copy with a seqcount to avoid this problem. Cc: Russell King Acked-by: Will Deacon Signed-off-by: Stephen Boyd Signed-off-by: John Stultz Git-commit: 85c3d2dd15be4d577a37ffb8bbbd019fc8e3280a Git-repo: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git [imaund@codeaurora.org: resolve merge conflicts] Signed-off-by: Ian Maund Change-Id: Ic30515ef65e5d3ec008f03ae020a9599e7c01fb2 sched_clock: Use an hrtimer instead of timer In the next patch we're going to increase the number of bits that the generic sched_clock can handle to be greater than 32. With more than 32 bits the wraparound time can be larger than what can fit into the units that msecs_to_jiffies takes (unsigned int). Luckily, the wraparound is initially calculated in nanoseconds which we can easily use with hrtimers, so switch to using an hrtimer. Change-Id: Id6059fae75863ddd3972f4dcac1cf7b803b09ac9 Cc: Russell King Signed-off-by: Stephen Boyd [jstultz: Fixup hrtimer intitialization order issue] Signed-off-by: John Stultz Git-commit: a08ca5d1089da03724f96fa0870c64968e66765b Git-repo: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git Signed-off-by: Ian Maund sched_clock: Add support for >32 bit sched_clock The ARM architected system counter has at least 56 usable bits. Add support for counters with more than 32 bits to the generic sched_clock implementation so we can increase the time between wakeups due to dealing with wrap-around on these devices while benefiting from the irqtime accounting and suspend/resume handling that the generic sched_clock code already has. On my system using 56 bits over 32 bits changes the wraparound time from a few minutes to an hour. For faster running counters (GHz range) this is even more important because we may not be able to execute the timer in time to deal with the wraparound if only 32 bits are used. We choose a maxsec value of 3600 seconds because we assume no system will go idle for more than an hour. In the future we may need to increase this value. Note: All users should switch over to the 64-bit read function so we can remove setup_sched_clock() in favor of sched_clock_register(). Change-Id: I81a4b7102db5316bd31d1fa54e2d801f1ee133c1 Cc: Russell King Signed-off-by: Stephen Boyd Signed-off-by: John Stultz Git-commit: e7e3ff1bfe9c42ee31172e9afdc0383a9e595e29 Git-repo: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git Signed-off-by: Ian Maund sched_clock: Avoid corrupting hrtimer tree during suspend During suspend we call sched_clock_poll() to update the epoch and accumulated time and reprogram the sched_clock_timer to fire before the next wrap-around time. Unfortunately, sched_clock_poll() doesn't restart the timer, instead it relies on the hrtimer layer to do that and during suspend we aren't calling that function from the hrtimer layer. Instead, we're reprogramming the expires time while the hrtimer is enqueued, which can cause the hrtimer tree to be corrupted. Furthermore, we restart the timer during suspend but we update the epoch during resume which seems counter-intuitive. Let's fix this by saving the accumulated state and canceling the timer during suspend. On resume we can update the epoch and restart the timer similar to what we would do if we were starting the clock for the first time. Change-Id: Iee2a1cca42e5b681347ea0607e9af420a63892d7 CRs-Fixed: 696826 Fixes: a08ca5d1089d "sched_clock: Use an hrtimer instead of timer" Signed-off-by: Stephen Boyd sched_clock: Make ARM's sched_clock generic for all architectures Nothing about the sched_clock implementation in the ARM port is specific to the architecture. Generalize the code so that other architectures can use it by selecting GENERIC_SCHED_CLOCK. Change-Id: I1e846fd1fb87ec1217ea84f1e2e02596c9eaa96b Signed-off-by: Stephen Boyd [jstultz: Merge minor collisions with other patches in my tree] Signed-off-by: John Stultz Change-Id: Id4ff982d3b5dfdc2fa172f3bdbddc6021134c932 Signed-off-by: Kevin F. Haggerty --- arch/arm/Kconfig | 1 + arch/arm/common/timer-sp.c | 2 +- arch/arm/kernel/Makefile | 2 +- arch/arm/kernel/arch_timer.c | 6 +- arch/arm/kernel/time.c | 3 +- arch/arm/mach-davinci/time.c | 2 +- arch/arm/mach-integrator/integrator_ap.c | 1 - arch/arm/mach-ixp4xx/common.c | 2 +- arch/arm/mach-mmp/time.c | 2 +- arch/arm/mach-msm/timer.c | 2 +- arch/arm/mach-omap1/time.c | 2 +- arch/arm/mach-omap2/timer.c | 3 +- arch/arm/mach-pxa/time.c | 2 +- arch/arm/mach-sa1100/time.c | 2 +- arch/arm/mach-u300/timer.c | 2 +- arch/arm/plat-iop/time.c | 2 +- arch/arm/plat-omap/counter_32k.c | 3 +- arch/arm/plat-orion/time.c | 2 +- arch/arm/plat-versatile/sched-clock.c | 2 +- drivers/clocksource/clksrc-dbx500-prcmu.c | 3 +- .../asm => include/linux}/sched_clock.h | 13 +- init/Kconfig | 3 + init/main.c | 2 + kernel/time/Makefile | 1 + kernel/time/sched_clock.c | 225 ++++++++++++++++++ 25 files changed, 264 insertions(+), 26 deletions(-) rename {arch/arm/include/asm => include/linux}/sched_clock.h (57%) create mode 100644 kernel/time/sched_clock.c diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 18ca1d5422a..8bee5df34f5 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -10,6 +10,7 @@ config ARM select SYS_SUPPORTS_APM_EMULATION select GENERIC_ATOMIC64 if (CPU_V6 || !CPU_32v6K || !AEABI) select HAVE_OPROFILE if (HAVE_PERF_EVENTS) + select GENERIC_SCHED_CLOCK select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL select HAVE_ARCH_KGDB select HAVE_KPROBES if !XIP_KERNEL diff --git a/arch/arm/common/timer-sp.c b/arch/arm/common/timer-sp.c index df13a3ffff3..0b04bf2f1bc 100644 --- a/arch/arm/common/timer-sp.c +++ b/arch/arm/common/timer-sp.c @@ -25,8 +25,8 @@ #include #include #include +#include -#include #include static long __init sp804_get_clock_rate(const char *name) diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index fc00a23f940..3b0b4df13c9 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -16,7 +16,7 @@ CFLAGS_REMOVE_return_address.o = -pg # Object file lists. obj-y := elf.o entry-armv.o entry-common.o irq.o opcodes.o \ - process.o ptrace.o return_address.o sched_clock.o \ + process.o ptrace.o return_address.o \ setup.o signal.o stacktrace.o sys_arm.o time.o traps.o obj-$(CONFIG_DEPRECATED_PARAM_STRUCT) += compat.o diff --git a/arch/arm/kernel/arch_timer.c b/arch/arm/kernel/arch_timer.c index 7ac034e904c..bb1aba3667c 100644 --- a/arch/arm/kernel/arch_timer.c +++ b/arch/arm/kernel/arch_timer.c @@ -25,12 +25,12 @@ #include #include #include +#include #include #include #include #include -#include #include #include @@ -377,7 +377,7 @@ static u32 arch_counter_get_cntvct32(void) return (u32)(cntvct & (u32)~0); } -static u32 notrace arch_timer_update_sched_clock(void) +static u64 notrace arch_timer_update_sched_clock(void) { return arch_counter_get_cntvct32(); } @@ -403,7 +403,7 @@ static void __init arch_timer_counter_init(void) { clocksource_register_hz(&clocksource_counter, arch_timer_rate); - setup_sched_clock(arch_timer_update_sched_clock, 32, arch_timer_rate); + sched_clock_register(arch_timer_update_sched_clock, 32, arch_timer_rate); /* Use the architected timer for the delay loop. */ arch_delay_timer.read_current_timer = &arch_timer_read_current_timer; diff --git a/arch/arm/kernel/time.c b/arch/arm/kernel/time.c index fe31b22f18f..c88115b5171 100644 --- a/arch/arm/kernel/time.c +++ b/arch/arm/kernel/time.c @@ -24,10 +24,10 @@ #include #include #include +#include #include #include -#include #include #include #include @@ -147,6 +147,5 @@ void __init time_init(void) { system_timer = machine_desc->timer; system_timer->init(); - sched_clock_postinit(); } diff --git a/arch/arm/mach-davinci/time.c b/arch/arm/mach-davinci/time.c index 75da315b658..2b5cd2c5f0e 100644 --- a/arch/arm/mach-davinci/time.c +++ b/arch/arm/mach-davinci/time.c @@ -18,8 +18,8 @@ #include #include #include +#include -#include #include #include diff --git a/arch/arm/mach-integrator/integrator_ap.c b/arch/arm/mach-integrator/integrator_ap.c index 871f148ffd7..821b1983340 100644 --- a/arch/arm/mach-integrator/integrator_ap.c +++ b/arch/arm/mach-integrator/integrator_ap.c @@ -41,7 +41,6 @@ #include #include /* HZ */ #include -#include #include #include diff --git a/arch/arm/mach-ixp4xx/common.c b/arch/arm/mach-ixp4xx/common.c index a9f80943d01..fa82651d1ae 100644 --- a/arch/arm/mach-ixp4xx/common.c +++ b/arch/arm/mach-ixp4xx/common.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -37,7 +38,6 @@ #include #include #include -#include #include #include diff --git a/arch/arm/mach-mmp/time.c b/arch/arm/mach-mmp/time.c index 71fc4ee4602..17e6198897f 100644 --- a/arch/arm/mach-mmp/time.c +++ b/arch/arm/mach-mmp/time.c @@ -25,8 +25,8 @@ #include #include +#include -#include #include #include #include diff --git a/arch/arm/mach-msm/timer.c b/arch/arm/mach-msm/timer.c index 1ea0f2dc621..c2c2415d7b5 100644 --- a/arch/arm/mach-msm/timer.c +++ b/arch/arm/mach-msm/timer.c @@ -24,11 +24,11 @@ #include #include #include +#include #include #include #include -#include #include #include #include diff --git a/arch/arm/mach-omap1/time.c b/arch/arm/mach-omap1/time.c index 4d8dd9a1b04..c1ec0c62565 100644 --- a/arch/arm/mach-omap1/time.c +++ b/arch/arm/mach-omap1/time.c @@ -43,10 +43,10 @@ #include #include #include +#include #include #include -#include #include #include diff --git a/arch/arm/mach-omap2/timer.c b/arch/arm/mach-omap2/timer.c index c512bac69ec..7d8264f121b 100644 --- a/arch/arm/mach-omap2/timer.c +++ b/arch/arm/mach-omap2/timer.c @@ -36,11 +36,12 @@ #include #include #include +#include #include #include #include -#include + #include "common.h" #include #include diff --git a/arch/arm/mach-pxa/time.c b/arch/arm/mach-pxa/time.c index 3d6c9bd90de..aa977ca20c6 100644 --- a/arch/arm/mach-pxa/time.c +++ b/arch/arm/mach-pxa/time.c @@ -16,11 +16,11 @@ #include #include #include +#include #include #include #include -#include #include #include diff --git a/arch/arm/mach-sa1100/time.c b/arch/arm/mach-sa1100/time.c index 6af26e8d55e..38525514643 100644 --- a/arch/arm/mach-sa1100/time.c +++ b/arch/arm/mach-sa1100/time.c @@ -14,9 +14,9 @@ #include #include #include +#include #include -#include #include #include diff --git a/arch/arm/mach-u300/timer.c b/arch/arm/mach-u300/timer.c index bc1c7897e82..cb62d7c591c 100644 --- a/arch/arm/mach-u300/timer.c +++ b/arch/arm/mach-u300/timer.c @@ -17,11 +17,11 @@ #include #include #include +#include #include /* Generic stuff */ -#include #include #include #include diff --git a/arch/arm/plat-iop/time.c b/arch/arm/plat-iop/time.c index cbfbbe46178..a174c12f6b7 100644 --- a/arch/arm/plat-iop/time.c +++ b/arch/arm/plat-iop/time.c @@ -22,9 +22,9 @@ #include #include #include +#include #include #include -#include #include #include #include diff --git a/arch/arm/plat-omap/counter_32k.c b/arch/arm/plat-omap/counter_32k.c index fd433e7a25f..ebc3032511a 100644 --- a/arch/arm/plat-omap/counter_32k.c +++ b/arch/arm/plat-omap/counter_32k.c @@ -18,8 +18,9 @@ #include #include #include +#include -#include +#include #include #include diff --git a/arch/arm/plat-orion/time.c b/arch/arm/plat-orion/time.c index 1ed8d1397fc..10845507e23 100644 --- a/arch/arm/plat-orion/time.c +++ b/arch/arm/plat-orion/time.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include /* * MBus bridge block registers. diff --git a/arch/arm/plat-versatile/sched-clock.c b/arch/arm/plat-versatile/sched-clock.c index b33b74c8723..51b109e3b6c 100644 --- a/arch/arm/plat-versatile/sched-clock.c +++ b/arch/arm/plat-versatile/sched-clock.c @@ -20,8 +20,8 @@ */ #include #include +#include -#include #include static void __iomem *ctr; diff --git a/drivers/clocksource/clksrc-dbx500-prcmu.c b/drivers/clocksource/clksrc-dbx500-prcmu.c index c26c369eb9e..c4ef2810f4a 100644 --- a/drivers/clocksource/clksrc-dbx500-prcmu.c +++ b/drivers/clocksource/clksrc-dbx500-prcmu.c @@ -14,8 +14,7 @@ */ #include #include - -#include +#include #include #include diff --git a/arch/arm/include/asm/sched_clock.h b/include/linux/sched_clock.h similarity index 57% rename from arch/arm/include/asm/sched_clock.h rename to include/linux/sched_clock.h index e3f75726343..ec5b3d0e402 100644 --- a/arch/arm/include/asm/sched_clock.h +++ b/include/linux/sched_clock.h @@ -5,10 +5,17 @@ * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ -#ifndef ASM_SCHED_CLOCK -#define ASM_SCHED_CLOCK +#ifndef LINUX_SCHED_CLOCK +#define LINUX_SCHED_CLOCK +#ifdef CONFIG_GENERIC_SCHED_CLOCK extern void sched_clock_postinit(void); -extern void setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate); +#else +static inline void sched_clock_postinit(void) { } +#endif + +extern void sched_clock_register(u64 (*read)(void), int bits, + unsigned long rate); #endif + diff --git a/init/Kconfig b/init/Kconfig index d5817a29456..b5ab259e32a 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -582,6 +582,9 @@ config LOG_BUF_SHIFT config HAVE_UNSTABLE_SCHED_CLOCK bool +config GENERIC_SCHED_CLOCK + bool + menuconfig CGROUPS boolean "Control Group support" depends on EVENTFD diff --git a/init/main.c b/init/main.c index 1865764561f..8c8e18d4d3a 100644 --- a/init/main.c +++ b/init/main.c @@ -72,6 +72,7 @@ #include #endif #include +#include #include #include @@ -708,6 +709,7 @@ asmlinkage void __init start_kernel(void) softirq_init(); timekeeping_init(); time_init(); + sched_clock_postinit(); profile_init(); call_function_init(); if (!irqs_disabled()) diff --git a/kernel/time/Makefile b/kernel/time/Makefile index cae2ad7491b..07609113236 100644 --- a/kernel/time/Makefile +++ b/kernel/time/Makefile @@ -4,6 +4,7 @@ obj-y += timeconv.o posix-clock.o #alarmtimer.o obj-$(CONFIG_GENERIC_CLOCKEVENTS_BUILD) += clockevents.o obj-$(CONFIG_GENERIC_CLOCKEVENTS) += tick-common.o obj-$(CONFIG_GENERIC_CLOCKEVENTS_BROADCAST) += tick-broadcast.o +obj-$(CONFIG_GENERIC_SCHED_CLOCK) += sched_clock.o obj-$(CONFIG_TICK_ONESHOT) += tick-oneshot.o obj-$(CONFIG_TICK_ONESHOT) += tick-sched.o obj-$(CONFIG_TIMER_STATS) += timer_stats.o diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c new file mode 100644 index 00000000000..610af0d97ab --- /dev/null +++ b/kernel/time/sched_clock.c @@ -0,0 +1,225 @@ +/* + * sched_clock.c: support for extending counters to full 64-bit ns counter + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct clock_data { + ktime_t wrap_kt; + u64 epoch_ns; + u64 epoch_cyc; + seqcount_t seq; + unsigned long rate; + u32 mult; + u32 shift; + bool suspended; +}; + +static struct hrtimer sched_clock_timer; +static int irqtime = -1; +static int initialized; + +core_param(irqtime, irqtime, int, 0400); + +static struct clock_data cd = { + .mult = NSEC_PER_SEC / HZ, +}; + +static u64 __read_mostly sched_clock_mask; + +static u64 notrace jiffy_sched_clock_read(void) +{ + /* + * We don't need to use get_jiffies_64 on 32-bit arches here + * because we register with BITS_PER_LONG + */ + return (u64)(jiffies - INITIAL_JIFFIES); +} + +static u64 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read; + +static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift) +{ + return (cyc * mult) >> shift; +} + +unsigned long long notrace sched_clock(void) +{ + u64 epoch_ns; + u64 epoch_cyc; + u64 cyc; + unsigned long seq; + + if (cd.suspended) + return cd.epoch_ns; + + do { + seq = read_seqcount_begin(&cd.seq); + epoch_cyc = cd.epoch_cyc; + epoch_ns = cd.epoch_ns; + } while (read_seqcount_retry(&cd.seq, seq)); + + cyc = read_sched_clock(); + cyc = (cyc - epoch_cyc) & sched_clock_mask; + return epoch_ns + cyc_to_ns(cyc, cd.mult, cd.shift); +} + +/* + * Atomically update the sched_clock epoch. + */ +static void notrace update_sched_clock(void) +{ + unsigned long flags; + u64 cyc; + u64 ns; + + cyc = read_sched_clock(); + ns = cd.epoch_ns + + cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, + cd.mult, cd.shift); + + raw_local_irq_save(flags); + write_seqcount_begin(&cd.seq); + cd.epoch_ns = ns; + cd.epoch_cyc = cyc; + write_seqcount_end(&cd.seq); + raw_local_irq_restore(flags); +} + +static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt) +{ + update_sched_clock(); + hrtimer_forward_now(hrt, cd.wrap_kt); + return HRTIMER_RESTART; +} + +void __init sched_clock_register(u64 (*read)(void), int bits, + unsigned long rate) +{ + u64 res, wrap, new_mask, new_epoch, cyc, ns; + u32 new_mult, new_shift; + ktime_t new_wrap_kt; + unsigned long r; + char r_unit; + + if (cd.rate > rate) + return; + + WARN_ON(!irqs_disabled()); + + /* calculate the mult/shift to convert counter ticks to ns. */ + clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600); + + new_mask = CLOCKSOURCE_MASK(bits); + + /* calculate how many ns until we wrap */ + wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask); + new_wrap_kt = ns_to_ktime(wrap - (wrap >> 3)); + + /* update epoch for new counter and update epoch_ns from old counter*/ + new_epoch = read(); + cyc = read_sched_clock(); + ns = cd.epoch_ns + cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, + cd.mult, cd.shift); + + write_seqcount_begin(&cd.seq); + read_sched_clock = read; + sched_clock_mask = new_mask; + cd.rate = rate; + cd.wrap_kt = new_wrap_kt; + cd.mult = new_mult; + cd.shift = new_shift; + cd.epoch_cyc = new_epoch; + cd.epoch_ns = ns; + write_seqcount_end(&cd.seq); + + r = rate; + if (r >= 4000000) { + r /= 1000000; + r_unit = 'M'; + } else if (r >= 1000) { + r /= 1000; + r_unit = 'k'; + } else + r_unit = ' '; + + /* calculate the ns resolution of this counter */ + res = cyc_to_ns(1ULL, new_mult, new_shift); + + pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n", + bits, r, r_unit, res, wrap); + + /* Enable IRQ time accounting if we have a fast enough sched_clock */ + if (irqtime > 0 || (irqtime == -1 && rate >= 1000000)) + enable_sched_clock_irqtime(); + + pr_debug("Registered %pF as sched_clock source\n", read); +} + +int sched_clock_initialized(void) +{ + return initialized; +} + +void __init sched_clock_postinit(void) +{ + /* + * If no sched_clock function has been provided at that point, + * make it the final one one. + */ + if (read_sched_clock == jiffy_sched_clock_read) + sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ); + + update_sched_clock(); + + /* + * Start the timer to keep sched_clock() properly updated and + * sets the initial epoch. + */ + hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + sched_clock_timer.function = sched_clock_poll; + hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); + + initialized = 1; +} + +static int sched_clock_suspend(void) +{ + update_sched_clock(); + hrtimer_cancel(&sched_clock_timer); + cd.suspended = true; + return 0; +} + +static void sched_clock_resume(void) +{ + cd.epoch_cyc = read_sched_clock(); + hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); + cd.suspended = false; +} + +static struct syscore_ops sched_clock_ops = { + .suspend = sched_clock_suspend, + .resume = sched_clock_resume, +}; + +static int __init sched_clock_syscore_init(void) +{ + register_syscore_ops(&sched_clock_ops); + return 0; +} +device_initcall(sched_clock_syscore_init);