diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 18ca1d5422a..8bee5df34f5 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -10,6 +10,7 @@ config ARM select SYS_SUPPORTS_APM_EMULATION select GENERIC_ATOMIC64 if (CPU_V6 || !CPU_32v6K || !AEABI) select HAVE_OPROFILE if (HAVE_PERF_EVENTS) + select GENERIC_SCHED_CLOCK select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL select HAVE_ARCH_KGDB select HAVE_KPROBES if !XIP_KERNEL diff --git a/arch/arm/common/timer-sp.c b/arch/arm/common/timer-sp.c index df13a3ffff3..0b04bf2f1bc 100644 --- a/arch/arm/common/timer-sp.c +++ b/arch/arm/common/timer-sp.c @@ -25,8 +25,8 @@ #include #include #include +#include -#include #include static long __init sp804_get_clock_rate(const char *name) diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index fc00a23f940..3b0b4df13c9 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -16,7 +16,7 @@ CFLAGS_REMOVE_return_address.o = -pg # Object file lists. obj-y := elf.o entry-armv.o entry-common.o irq.o opcodes.o \ - process.o ptrace.o return_address.o sched_clock.o \ + process.o ptrace.o return_address.o \ setup.o signal.o stacktrace.o sys_arm.o time.o traps.o obj-$(CONFIG_DEPRECATED_PARAM_STRUCT) += compat.o diff --git a/arch/arm/kernel/arch_timer.c b/arch/arm/kernel/arch_timer.c index 7ac034e904c..bb1aba3667c 100644 --- a/arch/arm/kernel/arch_timer.c +++ b/arch/arm/kernel/arch_timer.c @@ -25,12 +25,12 @@ #include #include #include +#include #include #include #include #include -#include #include #include @@ -377,7 +377,7 @@ static u32 arch_counter_get_cntvct32(void) return (u32)(cntvct & (u32)~0); } -static u32 notrace arch_timer_update_sched_clock(void) +static u64 notrace arch_timer_update_sched_clock(void) { return arch_counter_get_cntvct32(); } @@ -403,7 +403,7 @@ static void __init arch_timer_counter_init(void) { clocksource_register_hz(&clocksource_counter, arch_timer_rate); - setup_sched_clock(arch_timer_update_sched_clock, 32, arch_timer_rate); + sched_clock_register(arch_timer_update_sched_clock, 32, arch_timer_rate); /* Use the architected timer for the delay loop. */ arch_delay_timer.read_current_timer = &arch_timer_read_current_timer; diff --git a/arch/arm/kernel/time.c b/arch/arm/kernel/time.c index fe31b22f18f..c88115b5171 100644 --- a/arch/arm/kernel/time.c +++ b/arch/arm/kernel/time.c @@ -24,10 +24,10 @@ #include #include #include +#include #include #include -#include #include #include #include @@ -147,6 +147,5 @@ void __init time_init(void) { system_timer = machine_desc->timer; system_timer->init(); - sched_clock_postinit(); } diff --git a/arch/arm/mach-davinci/time.c b/arch/arm/mach-davinci/time.c index 75da315b658..2b5cd2c5f0e 100644 --- a/arch/arm/mach-davinci/time.c +++ b/arch/arm/mach-davinci/time.c @@ -18,8 +18,8 @@ #include #include #include +#include -#include #include #include diff --git a/arch/arm/mach-integrator/integrator_ap.c b/arch/arm/mach-integrator/integrator_ap.c index 871f148ffd7..821b1983340 100644 --- a/arch/arm/mach-integrator/integrator_ap.c +++ b/arch/arm/mach-integrator/integrator_ap.c @@ -41,7 +41,6 @@ #include #include /* HZ */ #include -#include #include #include diff --git a/arch/arm/mach-ixp4xx/common.c b/arch/arm/mach-ixp4xx/common.c index a9f80943d01..fa82651d1ae 100644 --- a/arch/arm/mach-ixp4xx/common.c +++ b/arch/arm/mach-ixp4xx/common.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -37,7 +38,6 @@ #include #include #include -#include #include #include diff --git a/arch/arm/mach-mmp/time.c b/arch/arm/mach-mmp/time.c index 71fc4ee4602..17e6198897f 100644 --- a/arch/arm/mach-mmp/time.c +++ b/arch/arm/mach-mmp/time.c @@ -25,8 +25,8 @@ #include #include +#include -#include #include #include #include diff --git a/arch/arm/mach-msm/timer.c b/arch/arm/mach-msm/timer.c index 1ea0f2dc621..c2c2415d7b5 100644 --- a/arch/arm/mach-msm/timer.c +++ b/arch/arm/mach-msm/timer.c @@ -24,11 +24,11 @@ #include #include #include +#include #include #include #include -#include #include #include #include diff --git a/arch/arm/mach-omap1/time.c b/arch/arm/mach-omap1/time.c index 4d8dd9a1b04..c1ec0c62565 100644 --- a/arch/arm/mach-omap1/time.c +++ b/arch/arm/mach-omap1/time.c @@ -43,10 +43,10 @@ #include #include #include +#include #include #include -#include #include #include diff --git a/arch/arm/mach-omap2/timer.c b/arch/arm/mach-omap2/timer.c index c512bac69ec..7d8264f121b 100644 --- a/arch/arm/mach-omap2/timer.c +++ b/arch/arm/mach-omap2/timer.c @@ -36,11 +36,12 @@ #include #include #include +#include #include #include #include -#include + #include "common.h" #include #include diff --git a/arch/arm/mach-pxa/time.c b/arch/arm/mach-pxa/time.c index 3d6c9bd90de..aa977ca20c6 100644 --- a/arch/arm/mach-pxa/time.c +++ b/arch/arm/mach-pxa/time.c @@ -16,11 +16,11 @@ #include #include #include +#include #include #include #include -#include #include #include diff --git a/arch/arm/mach-sa1100/time.c b/arch/arm/mach-sa1100/time.c index 6af26e8d55e..38525514643 100644 --- a/arch/arm/mach-sa1100/time.c +++ b/arch/arm/mach-sa1100/time.c @@ -14,9 +14,9 @@ #include #include #include +#include #include -#include #include #include diff --git a/arch/arm/mach-u300/timer.c b/arch/arm/mach-u300/timer.c index bc1c7897e82..cb62d7c591c 100644 --- a/arch/arm/mach-u300/timer.c +++ b/arch/arm/mach-u300/timer.c @@ -17,11 +17,11 @@ #include #include #include +#include #include /* Generic stuff */ -#include #include #include #include diff --git a/arch/arm/plat-iop/time.c b/arch/arm/plat-iop/time.c index cbfbbe46178..a174c12f6b7 100644 --- a/arch/arm/plat-iop/time.c +++ b/arch/arm/plat-iop/time.c @@ -22,9 +22,9 @@ #include #include #include +#include #include #include -#include #include #include #include diff --git a/arch/arm/plat-omap/counter_32k.c b/arch/arm/plat-omap/counter_32k.c index fd433e7a25f..ebc3032511a 100644 --- a/arch/arm/plat-omap/counter_32k.c +++ b/arch/arm/plat-omap/counter_32k.c @@ -18,8 +18,9 @@ #include #include #include +#include -#include +#include #include #include diff --git a/arch/arm/plat-orion/time.c b/arch/arm/plat-orion/time.c index 1ed8d1397fc..10845507e23 100644 --- a/arch/arm/plat-orion/time.c +++ b/arch/arm/plat-orion/time.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include /* * MBus bridge block registers. diff --git a/arch/arm/plat-versatile/sched-clock.c b/arch/arm/plat-versatile/sched-clock.c index b33b74c8723..51b109e3b6c 100644 --- a/arch/arm/plat-versatile/sched-clock.c +++ b/arch/arm/plat-versatile/sched-clock.c @@ -20,8 +20,8 @@ */ #include #include +#include -#include #include static void __iomem *ctr; diff --git a/drivers/clocksource/clksrc-dbx500-prcmu.c b/drivers/clocksource/clksrc-dbx500-prcmu.c index c26c369eb9e..c4ef2810f4a 100644 --- a/drivers/clocksource/clksrc-dbx500-prcmu.c +++ b/drivers/clocksource/clksrc-dbx500-prcmu.c @@ -14,8 +14,7 @@ */ #include #include - -#include +#include #include #include diff --git a/arch/arm/include/asm/sched_clock.h b/include/linux/sched_clock.h similarity index 57% rename from arch/arm/include/asm/sched_clock.h rename to include/linux/sched_clock.h index e3f75726343..ec5b3d0e402 100644 --- a/arch/arm/include/asm/sched_clock.h +++ b/include/linux/sched_clock.h @@ -5,10 +5,17 @@ * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ -#ifndef ASM_SCHED_CLOCK -#define ASM_SCHED_CLOCK +#ifndef LINUX_SCHED_CLOCK +#define LINUX_SCHED_CLOCK +#ifdef CONFIG_GENERIC_SCHED_CLOCK extern void sched_clock_postinit(void); -extern void setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate); +#else +static inline void sched_clock_postinit(void) { } +#endif + +extern void sched_clock_register(u64 (*read)(void), int bits, + unsigned long rate); #endif + diff --git a/init/Kconfig b/init/Kconfig index d5817a29456..b5ab259e32a 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -582,6 +582,9 @@ config LOG_BUF_SHIFT config HAVE_UNSTABLE_SCHED_CLOCK bool +config GENERIC_SCHED_CLOCK + bool + menuconfig CGROUPS boolean "Control Group support" depends on EVENTFD diff --git a/init/main.c b/init/main.c index 1865764561f..8c8e18d4d3a 100644 --- a/init/main.c +++ b/init/main.c @@ -72,6 +72,7 @@ #include #endif #include +#include #include #include @@ -708,6 +709,7 @@ asmlinkage void __init start_kernel(void) softirq_init(); timekeeping_init(); time_init(); + sched_clock_postinit(); profile_init(); call_function_init(); if (!irqs_disabled()) diff --git a/kernel/time/Makefile b/kernel/time/Makefile index cae2ad7491b..07609113236 100644 --- a/kernel/time/Makefile +++ b/kernel/time/Makefile @@ -4,6 +4,7 @@ obj-y += timeconv.o posix-clock.o #alarmtimer.o obj-$(CONFIG_GENERIC_CLOCKEVENTS_BUILD) += clockevents.o obj-$(CONFIG_GENERIC_CLOCKEVENTS) += tick-common.o obj-$(CONFIG_GENERIC_CLOCKEVENTS_BROADCAST) += tick-broadcast.o +obj-$(CONFIG_GENERIC_SCHED_CLOCK) += sched_clock.o obj-$(CONFIG_TICK_ONESHOT) += tick-oneshot.o obj-$(CONFIG_TICK_ONESHOT) += tick-sched.o obj-$(CONFIG_TIMER_STATS) += timer_stats.o diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c new file mode 100644 index 00000000000..610af0d97ab --- /dev/null +++ b/kernel/time/sched_clock.c @@ -0,0 +1,225 @@ +/* + * sched_clock.c: support for extending counters to full 64-bit ns counter + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct clock_data { + ktime_t wrap_kt; + u64 epoch_ns; + u64 epoch_cyc; + seqcount_t seq; + unsigned long rate; + u32 mult; + u32 shift; + bool suspended; +}; + +static struct hrtimer sched_clock_timer; +static int irqtime = -1; +static int initialized; + +core_param(irqtime, irqtime, int, 0400); + +static struct clock_data cd = { + .mult = NSEC_PER_SEC / HZ, +}; + +static u64 __read_mostly sched_clock_mask; + +static u64 notrace jiffy_sched_clock_read(void) +{ + /* + * We don't need to use get_jiffies_64 on 32-bit arches here + * because we register with BITS_PER_LONG + */ + return (u64)(jiffies - INITIAL_JIFFIES); +} + +static u64 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read; + +static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift) +{ + return (cyc * mult) >> shift; +} + +unsigned long long notrace sched_clock(void) +{ + u64 epoch_ns; + u64 epoch_cyc; + u64 cyc; + unsigned long seq; + + if (cd.suspended) + return cd.epoch_ns; + + do { + seq = read_seqcount_begin(&cd.seq); + epoch_cyc = cd.epoch_cyc; + epoch_ns = cd.epoch_ns; + } while (read_seqcount_retry(&cd.seq, seq)); + + cyc = read_sched_clock(); + cyc = (cyc - epoch_cyc) & sched_clock_mask; + return epoch_ns + cyc_to_ns(cyc, cd.mult, cd.shift); +} + +/* + * Atomically update the sched_clock epoch. + */ +static void notrace update_sched_clock(void) +{ + unsigned long flags; + u64 cyc; + u64 ns; + + cyc = read_sched_clock(); + ns = cd.epoch_ns + + cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, + cd.mult, cd.shift); + + raw_local_irq_save(flags); + write_seqcount_begin(&cd.seq); + cd.epoch_ns = ns; + cd.epoch_cyc = cyc; + write_seqcount_end(&cd.seq); + raw_local_irq_restore(flags); +} + +static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt) +{ + update_sched_clock(); + hrtimer_forward_now(hrt, cd.wrap_kt); + return HRTIMER_RESTART; +} + +void __init sched_clock_register(u64 (*read)(void), int bits, + unsigned long rate) +{ + u64 res, wrap, new_mask, new_epoch, cyc, ns; + u32 new_mult, new_shift; + ktime_t new_wrap_kt; + unsigned long r; + char r_unit; + + if (cd.rate > rate) + return; + + WARN_ON(!irqs_disabled()); + + /* calculate the mult/shift to convert counter ticks to ns. */ + clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600); + + new_mask = CLOCKSOURCE_MASK(bits); + + /* calculate how many ns until we wrap */ + wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask); + new_wrap_kt = ns_to_ktime(wrap - (wrap >> 3)); + + /* update epoch for new counter and update epoch_ns from old counter*/ + new_epoch = read(); + cyc = read_sched_clock(); + ns = cd.epoch_ns + cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, + cd.mult, cd.shift); + + write_seqcount_begin(&cd.seq); + read_sched_clock = read; + sched_clock_mask = new_mask; + cd.rate = rate; + cd.wrap_kt = new_wrap_kt; + cd.mult = new_mult; + cd.shift = new_shift; + cd.epoch_cyc = new_epoch; + cd.epoch_ns = ns; + write_seqcount_end(&cd.seq); + + r = rate; + if (r >= 4000000) { + r /= 1000000; + r_unit = 'M'; + } else if (r >= 1000) { + r /= 1000; + r_unit = 'k'; + } else + r_unit = ' '; + + /* calculate the ns resolution of this counter */ + res = cyc_to_ns(1ULL, new_mult, new_shift); + + pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n", + bits, r, r_unit, res, wrap); + + /* Enable IRQ time accounting if we have a fast enough sched_clock */ + if (irqtime > 0 || (irqtime == -1 && rate >= 1000000)) + enable_sched_clock_irqtime(); + + pr_debug("Registered %pF as sched_clock source\n", read); +} + +int sched_clock_initialized(void) +{ + return initialized; +} + +void __init sched_clock_postinit(void) +{ + /* + * If no sched_clock function has been provided at that point, + * make it the final one one. + */ + if (read_sched_clock == jiffy_sched_clock_read) + sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ); + + update_sched_clock(); + + /* + * Start the timer to keep sched_clock() properly updated and + * sets the initial epoch. + */ + hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + sched_clock_timer.function = sched_clock_poll; + hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); + + initialized = 1; +} + +static int sched_clock_suspend(void) +{ + update_sched_clock(); + hrtimer_cancel(&sched_clock_timer); + cd.suspended = true; + return 0; +} + +static void sched_clock_resume(void) +{ + cd.epoch_cyc = read_sched_clock(); + hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); + cd.suspended = false; +} + +static struct syscore_ops sched_clock_ops = { + .suspend = sched_clock_suspend, + .resume = sched_clock_resume, +}; + +static int __init sched_clock_syscore_init(void) +{ + register_syscore_ops(&sched_clock_ops); + return 0; +} +device_initcall(sched_clock_syscore_init);