diff options
author | Stephen Rothwell <sfr@canb.auug.org.au> | 2008-06-26 10:57:00 +1000 |
---|---|---|
committer | Stephen Rothwell <sfr@canb.auug.org.au> | 2008-06-26 10:57:00 +1000 |
commit | a34b11f8e2e07b144fb9c6a3e281867ff5ce1a96 (patch) | |
tree | fa3a7015a80fcf0e01f1b4075cfca25da9dd7562 | |
parent | 71f48c60649258aa6113886d44de5c14f81a29c3 (diff) | |
parent | 4e8ae82889528fec6317fd6c8c7c5d4fe1b91e16 (diff) |
Merge commit 'tip-core/auto-core-next'
45 files changed, 536 insertions, 249 deletions
diff --git a/Documentation/cputopology.txt b/Documentation/cputopology.txt index b61cb9564023..bd699da24666 100644 --- a/Documentation/cputopology.txt +++ b/Documentation/cputopology.txt @@ -14,9 +14,8 @@ represent the thread siblings to cpu X in the same physical package; To implement it in an architecture-neutral way, a new source file, drivers/base/topology.c, is to export the 4 attributes. -If one architecture wants to support this feature, it just needs to -implement 4 defines, typically in file include/asm-XXX/topology.h. -The 4 defines are: +For an architecture to support this feature, it must define some of +these macros in include/asm-XXX/topology.h: #define topology_physical_package_id(cpu) #define topology_core_id(cpu) #define topology_thread_siblings(cpu) @@ -25,17 +24,10 @@ The 4 defines are: The type of **_id is int. The type of siblings is cpumask_t. -To be consistent on all architectures, the 4 attributes should have -default values if their values are unavailable. Below is the rule. -1) physical_package_id: If cpu has no physical package id, -1 is the -default value. -2) core_id: If cpu doesn't support multi-core, its core id is 0. -3) thread_siblings: Just include itself, if the cpu doesn't support -HT/multi-thread. -4) core_siblings: Just include itself, if the cpu doesn't support -multi-core and HT/Multi-thread. - -So be careful when declaring the 4 defines in include/asm-XXX/topology.h. - -If an attribute isn't defined on an architecture, it won't be exported. - +To be consistent on all architectures, include/linux/topology.h +provides default definitions for any of the above macros that are +not defined by include/asm-XXX/topology.h: +1) physical_package_id: -1 +2) core_id: 0 +3) thread_siblings: just the given CPU +4) core_siblings: just the given CPU diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index e07c432c731f..042588fa12e5 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -1971,6 +1971,9 @@ and is between 256 and 4096 characters. It is defined in the file snd-ymfpci= [HW,ALSA] + softlockup_panic= + [KNL] Should the soft-lockup detector generate panics. + sonypi.*= [HW] Sony Programmable I/O Control Device driver See Documentation/sonypi.txt diff --git a/arch/avr32/kernel/vmlinux.lds.S b/arch/avr32/kernel/vmlinux.lds.S index 481cfd40c053..bc932c9b4272 100644 --- a/arch/avr32/kernel/vmlinux.lds.S +++ b/arch/avr32/kernel/vmlinux.lds.S @@ -93,8 +93,6 @@ SECTIONS __stop___ex_table = .; } - BUG_TABLE - RODATA . = ALIGN(THREAD_SIZE); diff --git a/arch/parisc/kernel/vmlinux.lds.S b/arch/parisc/kernel/vmlinux.lds.S index 2e516b871752..1a3b6ccd3620 100644 --- a/arch/parisc/kernel/vmlinux.lds.S +++ b/arch/parisc/kernel/vmlinux.lds.S @@ -67,7 +67,6 @@ SECTIONS _etext = .; RODATA - BUG_TABLE /* writeable */ /* Make sure this is page aligned so diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S index 0c3000bf8d75..53d57d17a894 100644 --- a/arch/powerpc/kernel/vmlinux.lds.S +++ b/arch/powerpc/kernel/vmlinux.lds.S @@ -64,8 +64,6 @@ SECTIONS NOTES - BUG_TABLE - /* * Init sections discarded at runtime */ diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index b4607155e8d0..76c1e60c92f3 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -40,7 +40,6 @@ SECTIONS _etext = .; /* End of text section */ NOTES :text :note - BUG_TABLE :text RODATA diff --git a/arch/sh/kernel/vmlinux_32.lds.S b/arch/sh/kernel/vmlinux_32.lds.S index c7113786ecd4..7b4b82bd1156 100644 --- a/arch/sh/kernel/vmlinux_32.lds.S +++ b/arch/sh/kernel/vmlinux_32.lds.S @@ -44,7 +44,6 @@ SECTIONS _etext = .; /* End of text section */ - BUG_TABLE NOTES RO_DATA(PAGE_SIZE) diff --git a/arch/sh/kernel/vmlinux_64.lds.S b/arch/sh/kernel/vmlinux_64.lds.S index d1e177009a41..33fa46451406 100644 --- a/arch/sh/kernel/vmlinux_64.lds.S +++ b/arch/sh/kernel/vmlinux_64.lds.S @@ -65,7 +65,6 @@ SECTIONS _etext = .; /* End of text section */ - BUG_TABLE NOTES RO_DATA(PAGE_SIZE) diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index d0463a946247..b2f54fafb8bc 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -21,7 +21,7 @@ #include "cpu.h" -DEFINE_PER_CPU(struct gdt_page, gdt_page) = { .gdt = { +DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = { [GDT_ENTRY_KERNEL_CS] = { { { 0x0000ffff, 0x00cf9a00 } } }, [GDT_ENTRY_KERNEL_DS] = { { { 0x0000ffff, 0x00cf9200 } } }, [GDT_ENTRY_DEFAULT_USER_CS] = { { { 0x0000ffff, 0x00cffa00 } } }, diff --git a/arch/x86/kernel/early_printk.c b/arch/x86/kernel/early_printk.c index 643fd861b724..ff9e7350da54 100644 --- a/arch/x86/kernel/early_printk.c +++ b/arch/x86/kernel/early_printk.c @@ -196,7 +196,7 @@ static struct console simnow_console = { static struct console *early_console = &early_vga_console; static int early_console_initialized; -void early_printk(const char *fmt, ...) +asmlinkage void early_printk(const char *fmt, ...) { char buf[512]; int n; diff --git a/arch/x86/kernel/vmlinux_32.lds.S b/arch/x86/kernel/vmlinux_32.lds.S index ce5ed083a1e9..21c86f6d0172 100644 --- a/arch/x86/kernel/vmlinux_32.lds.S +++ b/arch/x86/kernel/vmlinux_32.lds.S @@ -49,16 +49,14 @@ SECTIONS _etext = .; /* End of text section */ } :text = 0x9090 + NOTES :text :note + . = ALIGN(16); /* Exception table */ __ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) { __start___ex_table = .; *(__ex_table) __stop___ex_table = .; - } - - NOTES :text :note - - BUG_TABLE :text + } :text = 0x9090 . = ALIGN(4); .tracedata : AT(ADDR(.tracedata) - LOAD_OFFSET) { @@ -189,6 +187,7 @@ SECTIONS . = ALIGN(PAGE_SIZE); .data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { __per_cpu_start = .; + *(.data.percpu.page_aligned) *(.data.percpu) *(.data.percpu.shared_aligned) __per_cpu_end = .; diff --git a/arch/x86/kernel/vmlinux_64.lds.S b/arch/x86/kernel/vmlinux_64.lds.S index fad3674b06a5..d123747af1e4 100644 --- a/arch/x86/kernel/vmlinux_64.lds.S +++ b/arch/x86/kernel/vmlinux_64.lds.S @@ -19,7 +19,7 @@ PHDRS { data PT_LOAD FLAGS(7); /* RWE */ user PT_LOAD FLAGS(7); /* RWE */ data.init PT_LOAD FLAGS(7); /* RWE */ - note PT_NOTE FLAGS(4); /* R__ */ + note PT_NOTE FLAGS(0); /* ___ */ } SECTIONS { @@ -40,16 +40,14 @@ SECTIONS _etext = .; /* End of text section */ } :text = 0x9090 + NOTES :text :note + . = ALIGN(16); /* Exception table */ __ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) { __start___ex_table = .; *(__ex_table) __stop___ex_table = .; - } - - NOTES :text :note - - BUG_TABLE :text + } :text = 0x9090 RODATA diff --git a/block/blk-core.c b/block/blk-core.c index 1905aaba49fb..af094ff48357 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -2045,7 +2045,7 @@ int __init blk_dev_init(void) for_each_possible_cpu(i) INIT_LIST_HEAD(&per_cpu(blk_cpu_done, i)); - open_softirq(BLOCK_SOFTIRQ, blk_done_softirq, NULL); + open_softirq(BLOCK_SOFTIRQ, blk_done_softirq); register_hotcpu_notifier(&blk_cpu_notifier); return 0; diff --git a/drivers/base/topology.c b/drivers/base/topology.c index fdf4044d2e74..f0cb27011930 100644 --- a/drivers/base/topology.c +++ b/drivers/base/topology.c @@ -72,47 +72,27 @@ static inline ssize_t show_##name##_list(struct sys_device *dev, char *buf) \ #define define_siblings_show_func(name) \ define_siblings_show_map(name); define_siblings_show_list(name) -#ifdef topology_physical_package_id define_id_show_func(physical_package_id); define_one_ro(physical_package_id); -#define ref_physical_package_id_attr &attr_physical_package_id.attr, -#else -#define ref_physical_package_id_attr -#endif -#ifdef topology_core_id define_id_show_func(core_id); define_one_ro(core_id); -#define ref_core_id_attr &attr_core_id.attr, -#else -#define ref_core_id_attr -#endif -#ifdef topology_thread_siblings define_siblings_show_func(thread_siblings); define_one_ro(thread_siblings); define_one_ro(thread_siblings_list); -#define ref_thread_siblings_attr \ - &attr_thread_siblings.attr, &attr_thread_siblings_list.attr, -#else -#define ref_thread_siblings_attr -#endif -#ifdef topology_core_siblings define_siblings_show_func(core_siblings); define_one_ro(core_siblings); define_one_ro(core_siblings_list); -#define ref_core_siblings_attr \ - &attr_core_siblings.attr, &attr_core_siblings_list.attr, -#else -#define ref_core_siblings_attr -#endif static struct attribute *default_attrs[] = { - ref_physical_package_id_attr - ref_core_id_attr - ref_thread_siblings_attr - ref_core_siblings_attr + &attr_physical_package_id.attr, + &attr_core_id.attr, + &attr_thread_siblings.attr, + &attr_thread_siblings_list.attr, + &attr_core_siblings.attr, + &attr_core_siblings_list.attr, NULL }; diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h index b0e63c672ebd..36d188f47c98 100644 --- a/include/asm-generic/percpu.h +++ b/include/asm-generic/percpu.h @@ -45,7 +45,12 @@ extern unsigned long __per_cpu_offset[NR_CPUS]; * Only S390 provides its own means of moving the pointer. */ #ifndef SHIFT_PERCPU_PTR -#define SHIFT_PERCPU_PTR(__p, __offset) RELOC_HIDE((__p), (__offset)) +# ifdef CONFIG_HAVE_ZERO_BASED_PER_CPU +# define SHIFT_PERCPU_PTR(__p, __offset) \ + ((__typeof(__p))(((void *)(__p)) + (__offset))) +# else +# define SHIFT_PERCPU_PTR(__p, __offset) RELOC_HIDE((__p), (__offset)) +# endif /* CONFIG_HAVE_ZERO_BASED_PER_CPU */ #endif /* @@ -70,6 +75,8 @@ extern void setup_per_cpu_areas(void); #define per_cpu(var, cpu) (*((void)(cpu), &per_cpu_var(var))) #define __get_cpu_var(var) per_cpu_var(var) #define __raw_get_cpu_var(var) per_cpu_var(var) +#define SHIFT_PERCPU_PTR(__p, __offset) (__p) +#define per_cpu_offset(x) 0L #endif /* SMP */ diff --git a/include/asm-generic/sections.h b/include/asm-generic/sections.h index 8feeae1f2369..f697620c3e21 100644 --- a/include/asm-generic/sections.h +++ b/include/asm-generic/sections.h @@ -9,7 +9,17 @@ extern char __bss_start[], __bss_stop[]; extern char __init_begin[], __init_end[]; extern char _sinittext[], _einittext[]; extern char _end[]; +#ifdef CONFIG_HAVE_ZERO_BASED_PER_CPU +extern char __per_cpu_load[]; +extern char ____per_cpu_size[]; +#define __per_cpu_size ((unsigned long)&____per_cpu_size) +#define __per_cpu_start ((char *)0) +#define __per_cpu_end ((char *)__per_cpu_size) +#else extern char __per_cpu_start[], __per_cpu_end[]; +#define __per_cpu_load __per_cpu_start +#define __per_cpu_size (__per_cpu_end - __per_cpu_start) +#endif extern char __kprobes_text_start[], __kprobes_text_end[]; extern char __initdata_begin[], __initdata_end[]; extern char __start_rodata[], __end_rodata[]; diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index f054778e916c..b3d2a3868775 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -67,6 +67,8 @@ *(.rodata1) \ } \ \ + BUG_TABLE \ + \ /* PCI quirks */ \ .pci_fixup : AT(ADDR(.pci_fixup) - LOAD_OFFSET) { \ VMLINUX_SYMBOL(__start_pci_fixups_early) = .; \ @@ -310,6 +312,7 @@ .stab.indexstr 0 : { *(.stab.indexstr) } \ .comment 0 : { *(.comment) } +#ifdef CONFIG_GENERIC_BUG #define BUG_TABLE \ . = ALIGN(8); \ __bug_table : AT(ADDR(__bug_table) - LOAD_OFFSET) { \ @@ -317,6 +320,9 @@ *(__bug_table) \ __stop___bug_table = .; \ } +#else +#define BUG_TABLE +#endif #define NOTES \ .notes : AT(ADDR(.notes) - LOAD_OFFSET) { \ @@ -344,11 +350,28 @@ *(.initcall7.init) \ *(.initcall7s.init) +#ifdef CONFIG_HAVE_ZERO_BASED_PER_CPU +#define PERCPU(align) \ + . = ALIGN(align); \ + percpu : { } :percpu \ + __per_cpu_load = .; \ + .data.percpu 0 : AT(__per_cpu_load - LOAD_OFFSET) { \ + *(.data.percpu.first) \ + *(.data.percpu.shared_aligned) \ + *(.data.percpu) \ + *(.data.percpu.page_aligned) \ + ____per_cpu_size = .; \ + } \ + . = __per_cpu_load + ____per_cpu_size; \ + data : { } :data +#else #define PERCPU(align) \ . = ALIGN(align); \ __per_cpu_start = .; \ .data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { \ + *(.data.percpu.page_aligned) \ *(.data.percpu) \ *(.data.percpu.shared_aligned) \ } \ __per_cpu_end = .; +#endif diff --git a/include/asm-x86/percpu.h b/include/asm-x86/percpu.h index 736fc3bb8e1e..d85e5aaca7b1 100644 --- a/include/asm-x86/percpu.h +++ b/include/asm-x86/percpu.h @@ -108,6 +108,11 @@ do { \ : "+m" (var) \ : "ri" ((T__)val)); \ break; \ + case 8: \ + asm(op "q %1,"__percpu_seg"%0" \ + : "+m" (var) \ + : "ri" ((T__)val)); \ + break; \ default: __bad_percpu_size(); \ } \ } while (0) @@ -131,16 +136,84 @@ do { \ : "=r" (ret__) \ : "m" (var)); \ break; \ + case 8: \ + asm(op "q "__percpu_seg"%1,%0" \ + : "=r" (ret__) \ + : "m" (var)); \ + break; \ default: __bad_percpu_size(); \ } \ ret__; \ }) -#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var) -#define x86_write_percpu(var, val) percpu_to_op("mov", per_cpu__##var, val) -#define x86_add_percpu(var, val) percpu_to_op("add", per_cpu__##var, val) -#define x86_sub_percpu(var, val) percpu_to_op("sub", per_cpu__##var, val) -#define x86_or_percpu(var, val) percpu_to_op("or", per_cpu__##var, val) +#define percpu_addr_op(op, var) \ +({ \ + switch (sizeof(var)) { \ + case 1: \ + asm(op "b "__percpu_seg"%0" \ + : : "m"(var)); \ + break; \ + case 2: \ + asm(op "w "__percpu_seg"%0" \ + : : "m"(var)); \ + break; \ + case 4: \ + asm(op "l "__percpu_seg"%0" \ + : : "m"(var)); \ + break; \ + case 8: \ + asm(op "q "__percpu_seg"%0" \ + : : "m"(var)); \ + break; \ + default: __bad_percpu_size(); \ + } \ +}) + +#define percpu_cmpxchg_op(var, old, new) \ +({ \ + typeof(var) prev; \ + switch (sizeof(var)) { \ + case 1: \ + asm("cmpxchgb %b1, "__percpu_seg"%2" \ + : "=a"(prev) \ + : "q"(new), "m"(var), "0"(old) \ + : "memory"); \ + break; \ + case 2: \ + asm("cmpxchgw %w1, "__percpu_seg"%2" \ + : "=a"(prev) \ + : "r"(new), "m"(var), "0"(old) \ + : "memory"); \ + break; \ + case 4: \ + asm("cmpxchgl %k1, "__percpu_seg"%2" \ + : "=a"(prev) \ + : "r"(new), "m"(var), "0"(old) \ + : "memory"); \ + break; \ + case 8: \ + asm("cmpxchgq %1, "__percpu_seg"%2" \ + : "=a"(prev) \ + : "r"(new), "m"(var), "0"(old) \ + : "memory"); \ + break; \ + default: \ + __bad_percpu_size(); \ + } \ + return prev; \ +}) + +#define x86_read_percpu(var) percpu_from_op("mov", per_cpu_var(var)) +#define x86_write_percpu(var, val) percpu_to_op("mov", per_cpu_var(var), val) +#define x86_add_percpu(var, val) percpu_to_op("add", per_cpu_var(var), val) +#define x86_sub_percpu(var, val) percpu_to_op("sub", per_cpu_var(var), val) +#define x86_inc_percpu(var) percpu_addr_op("inc", per_cpu_var(var)) +#define x86_dec_percpu(var) percpu_addr_op("dec", per_cpu_var(var)) +#define x86_or_percpu(var, val) percpu_to_op("or", per_cpu_var(var), val) +#define x86_xchg_percpu(var, val) percpu_to_op("xchg", per_cpu_var(var), val) +#define x86_cmpxchg_percpu(var, old, new) \ + percpu_cmpxchg_op(per_cpu_var(var), old, new) + #endif /* !__ASSEMBLY__ */ #endif /* !CONFIG_X86_64 */ #endif /* _ASM_X86_PERCPU_H_ */ diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index f1fc7470d26c..44e499d989ed 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -285,17 +285,19 @@ enum struct softirq_action { void (*action)(struct softirq_action *); - void *data; }; asmlinkage void do_softirq(void); asmlinkage void __do_softirq(void); -extern void open_softirq(int nr, void (*action)(struct softirq_action*), void *data); +extern void open_softirq(int nr, void (*action)(struct softirq_action *)); extern void softirq_init(void); #define __raise_softirq_irqoff(nr) do { or_softirq_pending(1UL << (nr)); } while (0) extern void raise_softirq_irqoff(unsigned int nr); extern void raise_softirq(unsigned int nr); +#ifdef CONFIG_SOFTLOCKUP_SOFTIRQ_DEBUG +extern void *get_last_softirq_action(int cpu); +#endif /* Tasklets --- multithreaded analogue of BHs. diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 792bf0aa779b..4cb8d3df414e 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -184,9 +184,6 @@ asmlinkage int vprintk(const char *fmt, va_list args) __attribute__ ((format (printf, 1, 0))); asmlinkage int printk(const char * fmt, ...) __attribute__ ((format (printf, 1, 2))) __cold; -extern int log_buf_get_len(void); -extern int log_buf_read(int idx); -extern int log_buf_copy(char *dest, int idx, int len); extern int printk_ratelimit_jiffies; extern int printk_ratelimit_burst; @@ -202,9 +199,6 @@ static inline int vprintk(const char *s, va_list args) { return 0; } static inline int printk(const char *s, ...) __attribute__ ((format (printf, 1, 2))); static inline int __cold printk(const char *s, ...) { return 0; } -static inline int log_buf_get_len(void) { return 0; } -static inline int log_buf_read(int idx) { return 0; } -static inline int log_buf_copy(char *dest, int idx, int len) { return 0; } static inline int printk_ratelimit(void) { return 0; } static inline int __printk_ratelimit(int ratelimit_jiffies, \ int ratelimit_burst) { return 0; } @@ -213,7 +207,7 @@ static inline bool printk_timed_ratelimit(unsigned long *caller_jiffies, \ { return false; } #endif -extern void __attribute__((format(printf, 1, 2))) +extern void asmlinkage __attribute__((format(printf, 1, 2))) early_printk(const char *fmt, ...); unsigned long int_sqrt(unsigned long); diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index 4c4d236ded18..2486eb4edbf1 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -182,6 +182,9 @@ struct lock_list { * We record lock dependency chains, so that we can cache them: */ struct lock_chain { + u8 irq_context; + u8 depth; + u16 base; struct list_head entry; u64 chain_key; }; @@ -276,14 +279,6 @@ extern void lockdep_init_map(struct lockdep_map *lock, const char *name, (lock)->dep_map.key, sub) /* - * To initialize a lockdep_map statically use this macro. - * Note that _name must not be NULL. - */ -#define STATIC_LOCKDEP_MAP_INIT(_name, _key) \ - { .name = (_name), .key = (void *)(_key), } - - -/* * Acquire a lock. * * Values for "read": diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 4cdd393e71e1..3d1a7f6cc738 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -23,13 +23,35 @@ __attribute__((__section__(SHARED_ALIGNED_SECTION))) \ PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name \ ____cacheline_aligned_in_smp + +#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \ + __attribute__((__section__(".data.percpu.page_aligned"))) \ + PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name + +#ifdef CONFIG_HAVE_ZERO_BASED_PER_CPU +#define DEFINE_PER_CPU_FIRST(type, name) \ + __attribute__((__section__(".data.percpu.first"))) \ + PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name #else +#define DEFINE_PER_CPU_FIRST(type, name) \ + DEFINE_PER_CPU(type, name) +#endif + +#else /* !CONFIG_SMP */ + #define DEFINE_PER_CPU(type, name) \ PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ DEFINE_PER_CPU(type, name) -#endif + +#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \ + DEFINE_PER_CPU(type, name) + +#define DEFINE_PER_CPU_FIRST(type, name) \ + DEFINE_PER_CPU(type, name) + +#endif /* !CONFIG_SMP */ #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var) #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var) diff --git a/include/linux/sched.h b/include/linux/sched.h index c5d3f847ca8d..69760a379b6d 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -294,10 +294,11 @@ extern void softlockup_tick(void); extern void spawn_softlockup_task(void); extern void touch_softlockup_watchdog(void); extern void touch_all_softlockup_watchdogs(void); -extern unsigned long softlockup_thresh; +extern unsigned int softlockup_panic; extern unsigned long sysctl_hung_task_check_count; extern unsigned long sysctl_hung_task_timeout_secs; extern unsigned long sysctl_hung_task_warnings; +extern int softlockup_thresh; #else static inline void softlockup_tick(void) { diff --git a/include/linux/topology.h b/include/linux/topology.h index 24f3d2282e11..2158fc0d5a56 100644 --- a/include/linux/topology.h +++ b/include/linux/topology.h @@ -179,4 +179,17 @@ void arch_update_cpu_topology(void); #endif #endif /* CONFIG_NUMA */ +#ifndef topology_physical_package_id +#define topology_physical_package_id(cpu) ((void)(cpu), -1) +#endif +#ifndef topology_core_id +#define topology_core_id(cpu) ((void)(cpu), 0) +#endif +#ifndef topology_thread_siblings +#define topology_thread_siblings(cpu) cpumask_of_cpu(cpu) +#endif +#ifndef topology_core_siblings +#define topology_core_siblings(cpu) cpumask_of_cpu(cpu) +#endif + #endif /* _LINUX_TOPOLOGY_H */ diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 421be5fe5cc7..861b4088092a 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -1669,7 +1669,7 @@ void __init hrtimers_init(void) (void *)(long)smp_processor_id()); register_cpu_notifier(&hrtimers_nb); #ifdef CONFIG_HIGH_RES_TIMERS - open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq, NULL); + open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq); #endif } diff --git a/kernel/lockdep.c b/kernel/lockdep.c index 81a4e4a3f087..30bfc9d1164f 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c @@ -609,8 +609,8 @@ static int static_obj(void *obj) * percpu var? */ for_each_possible_cpu(i) { - start = (unsigned long) &__per_cpu_start + per_cpu_offset(i); - end = (unsigned long) &__per_cpu_start + PERCPU_ENOUGH_ROOM + start = (unsigned long) __per_cpu_start + per_cpu_offset(i); + end = (unsigned long) __per_cpu_start + PERCPU_ENOUGH_ROOM + per_cpu_offset(i); if ((addr >= start) && (addr < end)) @@ -1458,7 +1458,14 @@ out_bug: } unsigned long nr_lock_chains; -static struct lock_chain lock_chains[MAX_LOCKDEP_CHAINS]; +struct lock_chain lock_chains[MAX_LOCKDEP_CHAINS]; +int nr_chain_hlocks; +static u16 chain_hlocks[MAX_LOCKDEP_CHAIN_HLOCKS]; + +struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i) +{ + return lock_classes + chain_hlocks[chain->base + i]; +} /* * Look up a dependency chain. If the key is not present yet then @@ -1466,10 +1473,15 @@ static struct lock_chain lock_chains[MAX_LOCKDEP_CHAINS]; * validated. If the key is already hashed, return 0. * (On return with 1 graph_lock is held.) */ -static inline int lookup_chain_cache(u64 chain_key, struct lock_class *class) +static inline int lookup_chain_cache(struct task_struct *curr, + struct held_lock *hlock, + u64 chain_key) { + struct lock_class *class = hlock->class; struct list_head *hash_head = chainhashentry(chain_key); struct lock_chain *chain; + struct held_lock *hlock_curr, *hlock_next; + int i, j, n, cn; if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) return 0; @@ -1517,6 +1529,32 @@ cache_hit: } chain = lock_chains + nr_lock_chains++; chain->chain_key = chain_key; + chain->irq_context = hlock->irq_context; + /* Find the first held_lock of current chain */ + hlock_next = hlock; + for (i = curr->lockdep_depth - 1; i >= 0; i--) { + hlock_curr = curr->held_locks + i; + if (hlock_curr->irq_context != hlock_next->irq_context) + break; + hlock_next = hlock; + } + i++; + chain->depth = curr->lockdep_depth + 1 - i; + cn = nr_chain_hlocks; + while (cn + chain->depth <= MAX_LOCKDEP_CHAIN_HLOCKS) { + n = cmpxchg(&nr_chain_hlocks, cn, cn + chain->depth); + if (n == cn) + break; + cn = n; + } + if (likely(cn + chain->depth <= MAX_LOCKDEP_CHAIN_HLOCKS)) { + chain->base = cn; + for (j = 0; j < chain->depth - 1; j++, i++) { + int lock_id = curr->held_locks[i].class - lock_classes; + chain_hlocks[chain->base + j] = lock_id; + } + chain_hlocks[chain->base + j] = class - lock_classes; + } list_add_tail_rcu(&chain->entry, hash_head); debug_atomic_inc(&chain_lookup_misses); inc_chains(); @@ -1538,7 +1576,7 @@ static int validate_chain(struct task_struct *curr, struct lockdep_map *lock, * graph_lock for us) */ if (!hlock->trylock && (hlock->check == 2) && - lookup_chain_cache(chain_key, hlock->class)) { + lookup_chain_cache(curr, hlock, chain_key)) { /* * Check whether last held lock: * diff --git a/kernel/lockdep_internals.h b/kernel/lockdep_internals.h index 8ce09bc4613d..c3600a091a28 100644 --- a/kernel/lockdep_internals.h +++ b/kernel/lockdep_internals.h @@ -23,6 +23,8 @@ #define MAX_LOCKDEP_CHAINS_BITS 14 #define MAX_LOCKDEP_CHAINS (1UL << MAX_LOCKDEP_CHAINS_BITS) +#define MAX_LOCKDEP_CHAIN_HLOCKS (MAX_LOCKDEP_CHAINS*5) + /* * Stack-trace: tightly packed array of stack backtrace * addresses. Protected by the hash_lock. @@ -30,15 +32,19 @@ #define MAX_STACK_TRACE_ENTRIES 262144UL extern struct list_head all_lock_classes; +extern struct lock_chain lock_chains[]; extern void get_usage_chars(struct lock_class *class, char *c1, char *c2, char *c3, char *c4); extern const char * __get_key_name(struct lockdep_subclass_key *key, char *str); +struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i); + extern unsigned long nr_lock_classes; extern unsigned long nr_list_entries; extern unsigned long nr_lock_chains; +extern int nr_chain_hlocks; extern unsigned long nr_stack_trace_entries; extern unsigned int nr_hardirq_chains; diff --git a/kernel/lockdep_proc.c b/kernel/lockdep_proc.c index dc5d29648d85..9b0e940e2545 100644 --- a/kernel/lockdep_proc.c +++ b/kernel/lockdep_proc.c @@ -139,7 +139,7 @@ static int l_show(struct seq_file *m, void *v) list_for_each_entry(entry, &class->locks_after, entry) { if (entry->distance == 1) { - seq_printf(m, " -> [%p] ", entry->class); + seq_printf(m, " -> [%p] ", entry->class->key); print_name(m, entry->class); seq_puts(m, "\n"); } @@ -178,6 +178,95 @@ static const struct file_operations proc_lockdep_operations = { .release = seq_release, }; +#ifdef CONFIG_PROVE_LOCKING +static void *lc_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct lock_chain *chain; + + (*pos)++; + + if (v == SEQ_START_TOKEN) + chain = m->private; + else { + chain = v; + + if (*pos < nr_lock_chains) + chain = lock_chains + *pos; + else + chain = NULL; + } + + return chain; +} + +static void *lc_start(struct seq_file *m, loff_t *pos) +{ + if (*pos == 0) + return SEQ_START_TOKEN; + + if (*pos < nr_lock_chains) + return lock_chains + *pos; + + return NULL; +} + +static void lc_stop(struct seq_file *m, void *v) +{ +} + +static int lc_show(struct seq_file *m, void *v) +{ + struct lock_chain *chain = v; + struct lock_class *class; + int i; + + if (v == SEQ_START_TOKEN) { + seq_printf(m, "all lock chains:\n"); + return 0; + } + + seq_printf(m, "irq_context: %d\n", chain->irq_context); + + for (i = 0; i < chain->depth; i++) { + class = lock_chain_get_class(chain, i); + seq_printf(m, "[%p] ", class->key); + print_name(m, class); + seq_puts(m, "\n"); + } + seq_puts(m, "\n"); + + return 0; +} + +static const struct seq_operations lockdep_chains_ops = { + .start = lc_start, + .next = lc_next, + .stop = lc_stop, + .show = lc_show, +}; + +static int lockdep_chains_open(struct inode *inode, struct file *file) +{ + int res = seq_open(file, &lockdep_chains_ops); + if (!res) { + struct seq_file *m = file->private_data; + + if (nr_lock_chains) + m->private = lock_chains; + else + m->private = NULL; + } + return res; +} + +static const struct file_operations proc_lockdep_chains_operations = { + .open = lockdep_chains_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; +#endif /* CONFIG_PROVE_LOCKING */ + static void lockdep_stats_debug_show(struct seq_file *m) { #ifdef CONFIG_DEBUG_LOCKDEP @@ -294,6 +383,8 @@ static int lockdep_stats_show(struct seq_file *m, void *v) #ifdef CONFIG_PROVE_LOCKING seq_printf(m, " dependency chains: %11lu [max: %lu]\n", nr_lock_chains, MAX_LOCKDEP_CHAINS); + seq_printf(m, " dependency chain hlocks: %11d [max: %lu]\n", + nr_chain_hlocks, MAX_LOCKDEP_CHAIN_HLOCKS); #endif #ifdef CONFIG_TRACE_IRQFLAGS @@ -661,6 +752,10 @@ static const struct file_operations proc_lock_stat_operations = { static int __init lockdep_proc_init(void) { proc_create("lockdep", S_IRUSR, NULL, &proc_lockdep_operations); +#ifdef CONFIG_PROVE_LOCKING + proc_create("lockdep_chains", S_IRUSR, NULL, + &proc_lockdep_chains_operations); +#endif proc_create("lockdep_stats", S_IRUSR, NULL, &proc_lockdep_stats_operations); diff --git a/kernel/module.c b/kernel/module.c index 5f80478b746d..f525015dd65d 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -44,6 +44,7 @@ #include <linux/unwind.h> #include <asm/uaccess.h> #include <asm/cacheflush.h> +#include <asm/sections.h> #include <linux/license.h> #include <asm/sections.h> @@ -365,7 +366,7 @@ static void *percpu_modalloc(unsigned long size, unsigned long align, align = PAGE_SIZE; } - ptr = __per_cpu_start; + ptr = __per_cpu_load; for (i = 0; i < pcpu_num_used; ptr += block_size(pcpu_size[i]), i++) { /* Extra for alignment requirement. */ extra = ALIGN((unsigned long)ptr, align) - (unsigned long)ptr; @@ -400,7 +401,7 @@ static void *percpu_modalloc(unsigned long size, unsigned long align, static void percpu_modfree(void *freeme) { unsigned int i; - void *ptr = __per_cpu_start + block_size(pcpu_size[0]); + void *ptr = __per_cpu_load + block_size(pcpu_size[0]); /* First entry is core kernel percpu data. */ for (i = 1; i < pcpu_num_used; ptr += block_size(pcpu_size[i]), i++) { @@ -451,7 +452,7 @@ static int percpu_modinit(void) pcpu_size = kmalloc(sizeof(pcpu_size[0]) * pcpu_num_allocated, GFP_KERNEL); /* Static in-kernel percpu data (used). */ - pcpu_size[0] = -(__per_cpu_end-__per_cpu_start); + pcpu_size[0] = -__per_cpu_size; /* Free room. */ pcpu_size[1] = PERCPU_ENOUGH_ROOM + pcpu_size[0]; if (pcpu_size[1] < 0) { diff --git a/kernel/mutex-debug.c b/kernel/mutex-debug.c index 3aaa06c561de..1d94160eb532 100644 --- a/kernel/mutex-debug.c +++ b/kernel/mutex-debug.c @@ -79,8 +79,8 @@ void debug_mutex_unlock(struct mutex *lock) if (unlikely(!debug_locks)) return; - DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info()); DEBUG_LOCKS_WARN_ON(lock->magic != lock); + DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info()); DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next); DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info()); } diff --git a/kernel/mutex.c b/kernel/mutex.c index d046a345d365..bcdc9ac8ef60 100644 --- a/kernel/mutex.c +++ b/kernel/mutex.c @@ -165,10 +165,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, * got a signal? (This code gets eliminated in the * TASK_UNINTERRUPTIBLE case.) */ - if (unlikely((state == TASK_INTERRUPTIBLE && - signal_pending(task)) || - (state == TASK_KILLABLE && - fatal_signal_pending(task)))) { + if (unlikely(signal_pending_state(state, task))) { mutex_remove_waiter(lock, &waiter, task_thread_info(task)); mutex_release(&lock->dep_map, 1, ip); diff --git a/kernel/printk.c b/kernel/printk.c index 8fb01c32aa3b..de1a4f4470c3 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -38,7 +38,7 @@ /* * Architectures can override it: */ -void __attribute__((weak)) early_printk(const char *fmt, ...) +void asmlinkage __attribute__((weak)) early_printk(const char *fmt, ...) { } @@ -75,6 +75,8 @@ EXPORT_SYMBOL(oops_in_progress); static DECLARE_MUTEX(console_sem); static DECLARE_MUTEX(secondary_console_sem); struct console *console_drivers; +EXPORT_SYMBOL_GPL(console_drivers); + /* * This is used for debugging the mess that is the VT code by * keeping track if we have the console semaphore held. It's @@ -231,7 +233,7 @@ static inline void boot_delay_msec(void) /* * Return the number of unread characters in the log buffer. */ -int log_buf_get_len(void) +static int log_buf_get_len(void) { return logged_chars; } @@ -268,19 +270,6 @@ int log_buf_copy(char *dest, int idx, int len) } /* - * Extract a single character from the log buffer. - */ -int log_buf_read(int idx) -{ - char ret; - - if (log_buf_copy(&ret, idx, 1) == 1) - return ret; - else - return -1; -} - -/* * Commands to do_syslog: * * 0 -- Close the log. Currently a NOP. @@ -665,18 +654,17 @@ static int acquire_console_semaphore_for_printk(unsigned int cpu) spin_unlock(&logbuf_lock); return retval; } - -const char printk_recursion_bug_msg [] = - KERN_CRIT "BUG: recent printk recursion!\n"; -static int printk_recursion_bug; +static const char recursion_bug_msg [] = + KERN_CRIT "BUG: recent printk recursion!\n"; +static int recursion_bug; + static int new_text_line = 1; +static char printk_buf[1024]; asmlinkage int vprintk(const char *fmt, va_list args) { - static int log_level_unknown = 1; - static char printk_buf[1024]; - - unsigned long flags; int printed_len = 0; + int current_log_level = default_message_loglevel; + unsigned long flags; int this_cpu; char *p; @@ -699,7 +687,7 @@ asmlinkage int vprintk(const char *fmt, va_list args) * it can be printed at the next appropriate moment: */ if (!oops_in_progress) { - printk_recursion_bug = 1; + recursion_bug = 1; goto out_restore_irqs; } zap_locks(); @@ -709,70 +697,62 @@ asmlinkage int vprintk(const char *fmt, va_list args) spin_lock(&logbuf_lock); printk_cpu = this_cpu; - if (printk_recursion_bug) { - printk_recursion_bug = 0; - strcpy(printk_buf, printk_recursion_bug_msg); - printed_len = sizeof(printk_recursion_bug_msg); + if (recursion_bug) { + recursion_bug = 0; + strcpy(printk_buf, recursion_bug_msg); + printed_len = sizeof(recursion_bug_msg); } /* Emit the output into the temporary buffer */ printed_len += vscnprintf(printk_buf + printed_len, sizeof(printk_buf) - printed_len, fmt, args); + /* * Copy the output into log_buf. If the caller didn't provide * appropriate log level tags, we insert them here */ for (p = printk_buf; *p; p++) { - if (log_level_unknown) { - /* log_level_unknown signals the start of a new line */ + if (new_text_line) { + /* If a token, set current_log_level and skip over */ + if (p[0] == '<' && p[1] >= '0' && p[1] <= '7' && + p[2] == '>') { + current_log_level = p[1] - '0'; + p += 3; + printed_len -= 3; + } + + /* Always output the token */ + emit_log_char('<'); + emit_log_char(current_log_level + '0'); + emit_log_char('>'); + printed_len += 3; + new_text_line = 0; + if (printk_time) { - int loglev_char; + /* Follow the token with the time */ char tbuf[50], *tp; unsigned tlen; unsigned long long t; unsigned long nanosec_rem; - /* - * force the log level token to be - * before the time output. - */ - if (p[0] == '<' && p[1] >='0' && - p[1] <= '7' && p[2] == '>') { - loglev_char = p[1]; - p += 3; - printed_len -= 3; - } else { - loglev_char = default_message_loglevel - + '0'; - } t = cpu_clock(printk_cpu); nanosec_rem = do_div(t, 1000000000); - tlen = sprintf(tbuf, - "<%c>[%5lu.%06lu] ", - loglev_char, - (unsigned long)t, - nanosec_rem/1000); + tlen = sprintf(tbuf, "[%5lu.%06lu] ", + (unsigned long) t, + nanosec_rem / 1000); for (tp = tbuf; tp < tbuf + tlen; tp++) emit_log_char(*tp); printed_len += tlen; - } else { - if (p[0] != '<' || p[1] < '0' || - p[1] > '7' || p[2] != '>') { - emit_log_char('<'); - emit_log_char(default_message_loglevel - + '0'); - emit_log_char('>'); - printed_len += 3; - } } - log_level_unknown = 0; + if (!*p) break; } + emit_log_char(*p); if (*p == '\n') - log_level_unknown = 1; + new_text_line = 1; } /* @@ -1172,8 +1152,11 @@ void register_console(struct console *console) console->index = 0; if (console->setup == NULL || console->setup(console, NULL) == 0) { - console->flags |= CON_ENABLED | CON_CONSDEV; - preferred_console = 0; + console->flags |= CON_ENABLED; + if (console->device) { + console->flags |= CON_CONSDEV; + preferred_console = 0; + } } } diff --git a/kernel/rcuclassic.c b/kernel/rcuclassic.c index f4ffbd0f306f..f6e01f3ae9c6 100644 --- a/kernel/rcuclassic.c +++ b/kernel/rcuclassic.c @@ -529,7 +529,7 @@ static void __cpuinit rcu_online_cpu(int cpu) rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp); rcu_init_percpu_data(cpu, &rcu_bh_ctrlblk, bh_rdp); - open_softirq(RCU_SOFTIRQ, rcu_process_callbacks, NULL); + open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); } static int __cpuinit rcu_cpu_notify(struct notifier_block *self, diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c index 5e02b7740702..902dce221a63 100644 --- a/kernel/rcupreempt.c +++ b/kernel/rcupreempt.c @@ -1123,7 +1123,7 @@ void __init __rcu_init(void) for_each_online_cpu(cpu) rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long) cpu); - open_softirq(RCU_SOFTIRQ, rcu_process_callbacks, NULL); + open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); } /* diff --git a/kernel/sched.c b/kernel/sched.c index 3aaa5c8cb421..56958359d20c 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -7832,7 +7832,7 @@ void __init sched_init(void) #endif #ifdef CONFIG_SMP - open_softirq(SCHED_SOFTIRQ, run_rebalance_domains, NULL); + open_softirq(SCHED_SOFTIRQ, run_rebalance_domains); #endif #ifdef CONFIG_RT_MUTEXES diff --git a/kernel/softirq.c b/kernel/softirq.c index 36e061740047..1eb09bf17c29 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -131,23 +131,17 @@ void _local_bh_enable(void) EXPORT_SYMBOL(_local_bh_enable); -void local_bh_enable(void) +static inline void _local_bh_enable_ip(unsigned long ip) { + WARN_ON_ONCE(in_irq() || irqs_disabled()); #ifdef CONFIG_TRACE_IRQFLAGS - unsigned long flags; - - WARN_ON_ONCE(in_irq()); -#endif - WARN_ON_ONCE(irqs_disabled()); - -#ifdef CONFIG_TRACE_IRQFLAGS - local_irq_save(flags); + local_irq_disable(); #endif /* * Are softirqs going to be turned on now: */ if (softirq_count() == SOFTIRQ_OFFSET) - trace_softirqs_on((unsigned long)__builtin_return_address(0)); + trace_softirqs_on(ip); /* * Keep preemption disabled until we are done with * softirq processing: @@ -159,42 +153,31 @@ void local_bh_enable(void) dec_preempt_count(); #ifdef CONFIG_TRACE_IRQFLAGS - local_irq_restore(flags); + local_irq_enable(); #endif preempt_check_resched(); } + +void local_bh_enable(void) +{ + _local_bh_enable_ip((unsigned long)__builtin_return_address(0)); +} EXPORT_SYMBOL(local_bh_enable); void local_bh_enable_ip(unsigned long ip) { -#ifdef CONFIG_TRACE_IRQFLAGS - unsigned long flags; - - WARN_ON_ONCE(in_irq()); - - local_irq_save(flags); -#endif - /* - * Are softirqs going to be turned on now: - */ - if (softirq_count() == SOFTIRQ_OFFSET) - trace_softirqs_on(ip); - /* - * Keep preemption disabled until we are done with - * softirq processing: - */ - sub_preempt_count(SOFTIRQ_OFFSET - 1); + _local_bh_enable_ip(ip); +} +EXPORT_SYMBOL(local_bh_enable_ip); - if (unlikely(!in_interrupt() && local_softirq_pending())) - do_softirq(); +#ifdef CONFIG_SOFTLOCKUP_SOFTIRQ_DEBUG +static DEFINE_PER_CPU(void *, last_softirq_action); - dec_preempt_count(); -#ifdef CONFIG_TRACE_IRQFLAGS - local_irq_restore(flags); -#endif - preempt_check_resched(); +void *get_last_softirq_action(int cpu) +{ + return per_cpu(last_softirq_action, cpu); } -EXPORT_SYMBOL(local_bh_enable_ip); +#endif /* * We restart softirq processing MAX_SOFTIRQ_RESTART times, @@ -231,6 +214,10 @@ restart: do { if (pending & 1) { +#ifdef CONFIG_SOFTLOCKUP_SOFTIRQ_DEBUG + per_cpu(last_softirq_action, cpu) = h->action; +#endif + h->action(h); rcu_bh_qsctr_inc(cpu); } @@ -347,9 +334,8 @@ void raise_softirq(unsigned int nr) local_irq_restore(flags); } -void open_softirq(int nr, void (*action)(struct softirq_action*), void *data) +void open_softirq(int nr, void (*action)(struct softirq_action *)) { - softirq_vec[nr].data = data; softirq_vec[nr].action = action; } @@ -360,10 +346,8 @@ struct tasklet_head struct tasklet_struct **tail; }; -/* Some compilers disobey section attribute on statics when not - initialized -- RR */ -static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL }; -static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL }; +static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec); +static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec); void __tasklet_schedule(struct tasklet_struct *t) { @@ -503,8 +487,8 @@ void __init softirq_init(void) &per_cpu(tasklet_hi_vec, cpu).head; } - open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL); - open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL); + open_softirq(TASKLET_SOFTIRQ, tasklet_action); + open_softirq(HI_SOFTIRQ, tasklet_hi_action); } static int ksoftirqd(void * __bind_cpu) diff --git a/kernel/softlockup.c b/kernel/softlockup.c index c828c2339cc9..2f95ca7bc692 100644 --- a/kernel/softlockup.c +++ b/kernel/softlockup.c @@ -10,8 +10,10 @@ #include <linux/cpu.h> #include <linux/nmi.h> #include <linux/init.h> +#include <linux/interrupt.h> #include <linux/delay.h> #include <linux/freezer.h> +#include <linux/kallsyms.h> #include <linux/kthread.h> #include <linux/notifier.h> #include <linux/module.h> @@ -25,7 +27,22 @@ static DEFINE_PER_CPU(unsigned long, print_timestamp); static DEFINE_PER_CPU(struct task_struct *, watchdog_task); static int __read_mostly did_panic; -unsigned long __read_mostly softlockup_thresh = 60; +int __read_mostly softlockup_thresh = 60; + +/* + * Should we panic (and reboot, if panic_timeout= is set) when a + * soft-lockup occurs: + */ +unsigned int __read_mostly softlockup_panic = + CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE; + +static int __init softlockup_panic_setup(char *str) +{ + softlockup_panic = simple_strtoul(str, NULL, 0); + + return 1; +} +__setup("softlockup_panic=", softlockup_panic_setup); static int softlock_panic(struct notifier_block *this, unsigned long event, void *ptr) @@ -84,6 +101,14 @@ void softlockup_tick(void) struct pt_regs *regs = get_irq_regs(); unsigned long now; + /* Is detection switched off? */ + if (!per_cpu(watchdog_task, this_cpu) || softlockup_thresh <= 0) { + /* Be sure we don't false trigger if switched back on */ + if (touch_timestamp) + per_cpu(touch_timestamp, this_cpu) = 0; + return; + } + if (touch_timestamp == 0) { __touch_softlockup_watchdog(); return; @@ -94,7 +119,7 @@ void softlockup_tick(void) /* report at most once a second */ if ((print_timestamp >= touch_timestamp && print_timestamp < (touch_timestamp + 1)) || - did_panic || !per_cpu(watchdog_task, this_cpu)) { + did_panic) { return; } @@ -120,11 +145,19 @@ void softlockup_tick(void) printk(KERN_ERR "BUG: soft lockup - CPU#%d stuck for %lus! [%s:%d]\n", this_cpu, now - touch_timestamp, current->comm, task_pid_nr(current)); + print_modules(); +#ifdef CONFIG_SOFTLOCKUP_SOFTIRQ_DEBUG + print_symbol(KERN_ERR "Last softirq was %s\n", + (unsigned long) get_last_softirq_action(this_cpu)); +#endif if (regs) show_regs(regs); else dump_stack(); spin_unlock(&print_lock); + + if (softlockup_panic) + panic("softlockup: hung tasks"); } /* @@ -177,6 +210,9 @@ static void check_hung_task(struct task_struct *t, unsigned long now) t->last_switch_timestamp = now; touch_nmi_watchdog(); + + if (softlockup_panic) + panic("softlockup: blocked tasks"); } /* diff --git a/kernel/stacktrace.c b/kernel/stacktrace.c index b71816e47a30..7eaea9d02a52 100644 --- a/kernel/stacktrace.c +++ b/kernel/stacktrace.c @@ -11,14 +11,14 @@ void print_stack_trace(struct stack_trace *trace, int spaces) { - int i, j; + int i; - for (i = 0; i < trace->nr_entries; i++) { - unsigned long ip = trace->entries[i]; + if (WARN_ON(!trace->entries)) + return; - for (j = 0; j < spaces + 1; j++) - printk(" "); - print_ip_sym(ip); + for (i = 0; i < trace->nr_entries; i++) { + printk("%*c", 1 + spaces, ' '); + print_ip_sym(trace->entries[i]); } } diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 29116652dca8..a829dc8d7a9e 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -84,12 +84,13 @@ extern int latencytop_enabled; extern int sysctl_nr_open_min, sysctl_nr_open_max; /* Constants used for minimum and maximum */ -#if defined(CONFIG_DETECT_SOFTLOCKUP) || defined(CONFIG_HIGHMEM) +#if defined(CONFIG_HIGHMEM) || defined(CONFIG_DETECT_SOFTLOCKUP) static int one = 1; #endif #ifdef CONFIG_DETECT_SOFTLOCKUP static int sixty = 60; +static int neg_one = -1; #endif #ifdef CONFIG_MMU @@ -729,13 +730,24 @@ static struct ctl_table kern_table[] = { #ifdef CONFIG_DETECT_SOFTLOCKUP { .ctl_name = CTL_UNNUMBERED, + .procname = "softlockup_panic", + .data = &softlockup_panic, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_doulongvec_minmax, + .strategy = &sysctl_intvec, + .extra1 = &zero, + .extra2 = &one, + }, + { + .ctl_name = CTL_UNNUMBERED, .procname = "softlockup_thresh", .data = &softlockup_thresh, - .maxlen = sizeof(unsigned long), + .maxlen = sizeof(int), .mode = 0644, - .proc_handler = &proc_doulongvec_minmax, + .proc_handler = &proc_dointvec_minmax, .strategy = &sysctl_intvec, - .extra1 = &one, + .extra1 = &neg_one, .extra2 = &sixty, }, { diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index b854a895591e..28abad66fc8e 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -133,8 +133,6 @@ void tick_nohz_update_jiffies(void) if (!ts->tick_stopped) return; - touch_softlockup_watchdog(); - cpu_clear(cpu, nohz_cpu_mask); now = ktime_get(); ts->idle_waketime = now; @@ -142,6 +140,8 @@ void tick_nohz_update_jiffies(void) local_irq_save(flags); tick_do_update_jiffies64(now); local_irq_restore(flags); + + touch_softlockup_watchdog(); } void tick_nohz_stop_idle(int cpu) diff --git a/kernel/timer.c b/kernel/timer.c index ceacc6626572..b4da888497fa 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -1502,7 +1502,7 @@ void __init init_timers(void) BUG_ON(err == NOTIFY_BAD); register_cpu_notifier(&timers_nb); - open_softirq(TIMER_SOFTIRQ, run_timer_softirq, NULL); + open_softirq(TIMER_SOFTIRQ, run_timer_softirq); } /** diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 8f9e6b6d1799..3a7d277b2436 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -150,7 +150,7 @@ config DETECT_SOFTLOCKUP help Say Y here to enable the kernel to detect "soft lockups", which are bugs that cause the kernel to loop in kernel - mode for more than 10 seconds, without giving other tasks a + mode for more than 60 seconds, without giving other tasks a chance to run. When a soft-lockup is detected, the kernel will print the @@ -162,6 +162,40 @@ config DETECT_SOFTLOCKUP can be detected via the NMI-watchdog, on platforms that support it.) +config BOOTPARAM_SOFTLOCKUP_PANIC + bool "Panic (Reboot) On Soft Lockups" + depends on DETECT_SOFTLOCKUP + help + Say Y here to enable the kernel to panic on "soft lockups", + which are bugs that cause the kernel to loop in kernel + mode for more than 60 seconds, without giving other tasks a + chance to run. + + The panic can be used in combination with panic_timeout, + to cause the system to reboot automatically after a + lockup has been detected. This feature is useful for + high-availability systems that have uptime guarantees and + where a lockup must be resolved ASAP. + + Say N if unsure. + +config BOOTPARAM_SOFTLOCKUP_PANIC_VALUE + int + depends on DETECT_SOFTLOCKUP + range 0 1 + default 0 if !BOOTPARAM_SOFTLOCKUP_PANIC + default 1 if BOOTPARAM_SOFTLOCKUP_PANIC + +config SOFTLOCKUP_SOFTIRQ_DEBUG + bool "Debug softirq lockups" + depends on DETECT_SOFTLOCKUP + default n + help + If a softlockup happens in a softirq, the softlockup + stack trace is utterly unhelpful; it will only show the + stack up to __do_softirq(), since this is where interrupts + are reenabled. + config SCHED_DEBUG bool "Collect scheduler debugging info" depends on DEBUG_KERNEL && PROC_FS @@ -422,7 +456,6 @@ config DEBUG_LOCKING_API_SELFTESTS config STACKTRACE bool - depends on DEBUG_KERNEL depends on STACKTRACE_SUPPORT config DEBUG_KOBJECT diff --git a/lib/debugobjects.c b/lib/debugobjects.c index a76a5e122ae1..85b18d79be89 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -68,6 +68,7 @@ static int fill_pool(void) { gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN; struct debug_obj *new; + unsigned long flags; if (likely(obj_pool_free >= ODEBUG_POOL_MIN_LEVEL)) return obj_pool_free; @@ -81,10 +82,10 @@ static int fill_pool(void) if (!new) return obj_pool_free; - spin_lock(&pool_lock); + spin_lock_irqsave(&pool_lock, flags); hlist_add_head(&new->node, &obj_pool); obj_pool_free++; - spin_unlock(&pool_lock); + spin_unlock_irqrestore(&pool_lock, flags); } return obj_pool_free; } @@ -110,16 +111,13 @@ static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b) } /* - * Allocate a new object. If the pool is empty and no refill possible, - * switch off the debugger. + * Allocate a new object. If the pool is empty, switch off the debugger. */ static struct debug_obj * alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr) { struct debug_obj *obj = NULL; - int retry = 0; -repeat: spin_lock(&pool_lock); if (obj_pool.first) { obj = hlist_entry(obj_pool.first, typeof(*obj), node); @@ -141,9 +139,6 @@ repeat: } spin_unlock(&pool_lock); - if (fill_pool() && !obj && !retry++) - goto repeat; - return obj; } @@ -261,6 +256,8 @@ __debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack) struct debug_obj *obj; unsigned long flags; + fill_pool(); + db = get_bucket((unsigned long) addr); spin_lock_irqsave(&db->lock, flags); diff --git a/mm/filemap.c b/mm/filemap.c index 1e6a7d34874f..72a953682912 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1000,8 +1000,9 @@ page_ok: page_not_up_to_date: /* Get exclusive access to the page ... */ - if (lock_page_killable(page)) - goto readpage_eio; + error = lock_page_killable(page); + if (unlikely(error)) + goto readpage_error; /* Did it get truncated before we got the lock? */ if (!page->mapping) { @@ -1029,8 +1030,9 @@ readpage: } if (!PageUptodate(page)) { - if (lock_page_killable(page)) - goto readpage_eio; + error = lock_page_killable(page); + if (unlikely(error)) + goto readpage_error; if (!PageUptodate(page)) { if (page->mapping == NULL) { /* @@ -1042,15 +1044,14 @@ readpage: } unlock_page(page); shrink_readahead_size_eio(filp, ra); - goto readpage_eio; + error = -EIO; + goto readpage_error; } unlock_page(page); } goto page_ok; -readpage_eio: - error = -EIO; readpage_error: /* UHHUH! A synchronous read error occurred. Report it */ desc->error = error; diff --git a/net/core/dev.c b/net/core/dev.c index c421a1f8f0b9..48f1e0413130 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4585,8 +4585,8 @@ static int __init net_dev_init(void) dev_boot_phase = 0; - open_softirq(NET_TX_SOFTIRQ, net_tx_action, NULL); - open_softirq(NET_RX_SOFTIRQ, net_rx_action, NULL); + open_softirq(NET_TX_SOFTIRQ, net_tx_action); + open_softirq(NET_RX_SOFTIRQ, net_rx_action); hotcpu_notifier(dev_cpu_callback, 0); dst_init(); |