diff options
author | Kent Overstreet <koverstreet@google.com> | 2013-03-12 17:56:23 -0700 |
---|---|---|
committer | Kent Overstreet <koverstreet@google.com> | 2013-03-12 17:56:23 -0700 |
commit | 95ce3ef1de1fc2ac6952e795ed74390e8192cf50 (patch) | |
tree | 8b50c89a3706d8ce4b75f877544a119d8d48b664 | |
parent | 19f949f52599ba7c3f67a5897ac6be14bfcb1200 (diff) |
irqs_disabled counterirq_stuff
-rw-r--r-- | arch/x86/include/asm/irqflags.h | 14 | ||||
-rw-r--r-- | arch/x86/kernel/cpu/common.c | 3 | ||||
-rw-r--r-- | arch/x86/kernel/entry_64.S | 2 | ||||
-rw-r--r-- | include/linux/irqflags.h | 148 |
4 files changed, 88 insertions, 79 deletions
diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h index bba3cf88e624..9a39472f11de 100644 --- a/arch/x86/include/asm/irqflags.h +++ b/arch/x86/include/asm/irqflags.h @@ -191,11 +191,17 @@ static inline int arch_irqs_disabled(void) #endif #ifdef CONFIG_TRACE_IRQFLAGS -# define TRACE_IRQS_ON call trace_hardirqs_on_thunk; -# define TRACE_IRQS_OFF call trace_hardirqs_off_thunk; +# define TRACE_IRQS_ON \ + call trace_hardirqs_on_thunk; \ + movl $0,PER_CPU_VAR(irq_disable_count); +# define TRACE_IRQS_OFF \ + movl $1,PER_CPU_VAR(irq_disable_count); \ + call trace_hardirqs_off_thunk; #else -# define TRACE_IRQS_ON -# define TRACE_IRQS_OFF +# define TRACE_IRQS_ON \ + movl $0,PER_CPU_VAR(irq_disable_count); +# define TRACE_IRQS_OFF \ + movl $1,PER_CPU_VAR(irq_disable_count); #endif #ifdef CONFIG_DEBUG_LOCK_ALLOC # define LOCKDEP_SYS_EXIT ARCH_LOCKDEP_SYS_EXIT diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 9c3ab43a6954..a05b9590b238 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1093,6 +1093,9 @@ DEFINE_PER_CPU(char *, irq_stack_ptr) = DEFINE_PER_CPU(unsigned int, irq_count) = -1; +DEFINE_PER_CPU(unsigned, irq_disable_count); +EXPORT_PER_CPU_SYMBOL(irq_disable_count); + DEFINE_PER_CPU(struct task_struct *, fpu_owner_task); /* diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index cb3c591339aa..45c52ed7864d 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -289,9 +289,9 @@ ENDPROC(native_usergs_sysret64) #ifdef CONFIG_TRACE_IRQFLAGS bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */ jnc 1f - TRACE_IRQS_ON 1: #endif + TRACE_IRQS_ON .endm /* diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h index d176d658fe25..abbd51ae391c 100644 --- a/include/linux/irqflags.h +++ b/include/linux/irqflags.h @@ -13,6 +13,7 @@ #include <linux/typecheck.h> #include <asm/irqflags.h> +#include <asm/percpu.h> #ifdef CONFIG_TRACE_IRQFLAGS extern void trace_softirqs_on(unsigned long ip); @@ -53,98 +54,97 @@ # define start_critical_timings() do { } while (0) #endif +DECLARE_PER_CPU(unsigned, irq_disable_count); + /* * Wrap the arch provided IRQ routines to provide appropriate checks. */ -#define raw_local_irq_disable() arch_local_irq_disable() -#define raw_local_irq_enable() arch_local_irq_enable() +#define raw_local_irq_disable() \ +do { \ + arch_local_irq_disable(); \ + __raw_get_cpu_var(irq_disable_count) = 1; \ +} while (0) + +#define raw_local_irq_enable() \ +do { \ + __raw_get_cpu_var(irq_disable_count) = 0; \ + arch_local_irq_enable(); \ +} while (0) + #define raw_local_irq_save(flags) \ - do { \ - typecheck(unsigned long, flags); \ - flags = arch_local_irq_save(); \ - } while (0) +do { \ + typecheck(unsigned long, flags); \ + \ + arch_local_irq_disable(); \ + flags = __raw_get_cpu_var(irq_disable_count)++; \ +} while (0) + +#if 1 #define raw_local_irq_restore(flags) \ - do { \ - typecheck(unsigned long, flags); \ - arch_local_irq_restore(flags); \ - } while (0) +do { \ + typecheck(unsigned long, flags); \ + \ + __raw_get_cpu_var(irq_disable_count) = flags; \ + if (!flags) \ + arch_local_irq_enable(); \ +} while (0) +#else +#define raw_local_irq_restore(flags) \ +do { \ + typecheck(unsigned long, flags); \ + \ + if (!--__raw_get_cpu_var(irq_disable_count)) \ + arch_local_irq_enable(); \ +} while (0) +#endif + #define raw_local_save_flags(flags) \ - do { \ - typecheck(unsigned long, flags); \ - flags = arch_local_save_flags(); \ - } while (0) +do { \ + typecheck(unsigned long, flags); \ + flags = __raw_get_cpu_var(irq_disable_count); \ +} while (0) + #define raw_irqs_disabled_flags(flags) \ - ({ \ - typecheck(unsigned long, flags); \ - arch_irqs_disabled_flags(flags); \ - }) -#define raw_irqs_disabled() (arch_irqs_disabled()) -#define raw_safe_halt() arch_safe_halt() +({ \ + typecheck(unsigned long, flags); \ + flags != 0; \ +}) + +#define raw_irqs_disabled() (__raw_get_cpu_var(irq_disable_count) != 0) /* * The local_irq_*() APIs are equal to the raw_local_irq*() * if !TRACE_IRQFLAGS. */ -#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT #define local_irq_enable() \ do { trace_hardirqs_on(); raw_local_irq_enable(); } while (0) #define local_irq_disable() \ do { raw_local_irq_disable(); trace_hardirqs_off(); } while (0) -#define local_irq_save(flags) \ - do { \ - raw_local_irq_save(flags); \ - trace_hardirqs_off(); \ - } while (0) +#define local_irq_save(flags) \ +do { \ + raw_local_irq_save(flags); \ + trace_hardirqs_off(); \ +} while (0) #define local_irq_restore(flags) \ - do { \ - if (raw_irqs_disabled_flags(flags)) { \ - raw_local_irq_restore(flags); \ - trace_hardirqs_off(); \ - } else { \ - trace_hardirqs_on(); \ - raw_local_irq_restore(flags); \ - } \ - } while (0) -#define local_save_flags(flags) \ - do { \ - raw_local_save_flags(flags); \ - } while (0) - -#define irqs_disabled_flags(flags) \ - ({ \ - raw_irqs_disabled_flags(flags); \ - }) - -#define irqs_disabled() \ - ({ \ - unsigned long _flags; \ - raw_local_save_flags(_flags); \ - raw_irqs_disabled_flags(_flags); \ - }) - -#define safe_halt() \ - do { \ - trace_hardirqs_on(); \ - raw_safe_halt(); \ - } while (0) - - -#else /* !CONFIG_TRACE_IRQFLAGS_SUPPORT */ - -#define local_irq_enable() do { raw_local_irq_enable(); } while (0) -#define local_irq_disable() do { raw_local_irq_disable(); } while (0) -#define local_irq_save(flags) \ - do { \ - raw_local_irq_save(flags); \ - } while (0) -#define local_irq_restore(flags) do { raw_local_irq_restore(flags); } while (0) -#define local_save_flags(flags) do { raw_local_save_flags(flags); } while (0) -#define irqs_disabled() (raw_irqs_disabled()) -#define irqs_disabled_flags(flags) (raw_irqs_disabled_flags(flags)) -#define safe_halt() do { raw_safe_halt(); } while (0) - -#endif /* CONFIG_TRACE_IRQFLAGS_SUPPORT */ +do { \ + if (raw_irqs_disabled_flags(flags)) { \ + raw_local_irq_restore(flags); \ + trace_hardirqs_off(); \ + } else { \ + trace_hardirqs_on(); \ + raw_local_irq_restore(flags); \ + } \ +} while (0) + +#define local_save_flags(flags) raw_local_save_flags(flags) +#define irqs_disabled() raw_irqs_disabled() +#define irqs_disabled_flags(flags) raw_irqs_disabled_flags(flags) +#define safe_halt() \ +do { \ + trace_hardirqs_on(); \ + arch_safe_halt(); \ +} while (0) #endif |