summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKent Overstreet <koverstreet@google.com>2013-03-12 17:56:23 -0700
committerKent Overstreet <koverstreet@google.com>2013-03-12 17:56:23 -0700
commit95ce3ef1de1fc2ac6952e795ed74390e8192cf50 (patch)
tree8b50c89a3706d8ce4b75f877544a119d8d48b664
parent19f949f52599ba7c3f67a5897ac6be14bfcb1200 (diff)
irqs_disabled counterirq_stuff
-rw-r--r--arch/x86/include/asm/irqflags.h14
-rw-r--r--arch/x86/kernel/cpu/common.c3
-rw-r--r--arch/x86/kernel/entry_64.S2
-rw-r--r--include/linux/irqflags.h148
4 files changed, 88 insertions, 79 deletions
diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h
index bba3cf88e624..9a39472f11de 100644
--- a/arch/x86/include/asm/irqflags.h
+++ b/arch/x86/include/asm/irqflags.h
@@ -191,11 +191,17 @@ static inline int arch_irqs_disabled(void)
#endif
#ifdef CONFIG_TRACE_IRQFLAGS
-# define TRACE_IRQS_ON call trace_hardirqs_on_thunk;
-# define TRACE_IRQS_OFF call trace_hardirqs_off_thunk;
+# define TRACE_IRQS_ON \
+ call trace_hardirqs_on_thunk; \
+ movl $0,PER_CPU_VAR(irq_disable_count);
+# define TRACE_IRQS_OFF \
+ movl $1,PER_CPU_VAR(irq_disable_count); \
+ call trace_hardirqs_off_thunk;
#else
-# define TRACE_IRQS_ON
-# define TRACE_IRQS_OFF
+# define TRACE_IRQS_ON \
+ movl $0,PER_CPU_VAR(irq_disable_count);
+# define TRACE_IRQS_OFF \
+ movl $1,PER_CPU_VAR(irq_disable_count);
#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
# define LOCKDEP_SYS_EXIT ARCH_LOCKDEP_SYS_EXIT
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 9c3ab43a6954..a05b9590b238 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1093,6 +1093,9 @@ DEFINE_PER_CPU(char *, irq_stack_ptr) =
DEFINE_PER_CPU(unsigned int, irq_count) = -1;
+DEFINE_PER_CPU(unsigned, irq_disable_count);
+EXPORT_PER_CPU_SYMBOL(irq_disable_count);
+
DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
/*
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index cb3c591339aa..45c52ed7864d 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -289,9 +289,9 @@ ENDPROC(native_usergs_sysret64)
#ifdef CONFIG_TRACE_IRQFLAGS
bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
jnc 1f
- TRACE_IRQS_ON
1:
#endif
+ TRACE_IRQS_ON
.endm
/*
diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h
index d176d658fe25..abbd51ae391c 100644
--- a/include/linux/irqflags.h
+++ b/include/linux/irqflags.h
@@ -13,6 +13,7 @@
#include <linux/typecheck.h>
#include <asm/irqflags.h>
+#include <asm/percpu.h>
#ifdef CONFIG_TRACE_IRQFLAGS
extern void trace_softirqs_on(unsigned long ip);
@@ -53,98 +54,97 @@
# define start_critical_timings() do { } while (0)
#endif
+DECLARE_PER_CPU(unsigned, irq_disable_count);
+
/*
* Wrap the arch provided IRQ routines to provide appropriate checks.
*/
-#define raw_local_irq_disable() arch_local_irq_disable()
-#define raw_local_irq_enable() arch_local_irq_enable()
+#define raw_local_irq_disable() \
+do { \
+ arch_local_irq_disable(); \
+ __raw_get_cpu_var(irq_disable_count) = 1; \
+} while (0)
+
+#define raw_local_irq_enable() \
+do { \
+ __raw_get_cpu_var(irq_disable_count) = 0; \
+ arch_local_irq_enable(); \
+} while (0)
+
#define raw_local_irq_save(flags) \
- do { \
- typecheck(unsigned long, flags); \
- flags = arch_local_irq_save(); \
- } while (0)
+do { \
+ typecheck(unsigned long, flags); \
+ \
+ arch_local_irq_disable(); \
+ flags = __raw_get_cpu_var(irq_disable_count)++; \
+} while (0)
+
+#if 1
#define raw_local_irq_restore(flags) \
- do { \
- typecheck(unsigned long, flags); \
- arch_local_irq_restore(flags); \
- } while (0)
+do { \
+ typecheck(unsigned long, flags); \
+ \
+ __raw_get_cpu_var(irq_disable_count) = flags; \
+ if (!flags) \
+ arch_local_irq_enable(); \
+} while (0)
+#else
+#define raw_local_irq_restore(flags) \
+do { \
+ typecheck(unsigned long, flags); \
+ \
+ if (!--__raw_get_cpu_var(irq_disable_count)) \
+ arch_local_irq_enable(); \
+} while (0)
+#endif
+
#define raw_local_save_flags(flags) \
- do { \
- typecheck(unsigned long, flags); \
- flags = arch_local_save_flags(); \
- } while (0)
+do { \
+ typecheck(unsigned long, flags); \
+ flags = __raw_get_cpu_var(irq_disable_count); \
+} while (0)
+
#define raw_irqs_disabled_flags(flags) \
- ({ \
- typecheck(unsigned long, flags); \
- arch_irqs_disabled_flags(flags); \
- })
-#define raw_irqs_disabled() (arch_irqs_disabled())
-#define raw_safe_halt() arch_safe_halt()
+({ \
+ typecheck(unsigned long, flags); \
+ flags != 0; \
+})
+
+#define raw_irqs_disabled() (__raw_get_cpu_var(irq_disable_count) != 0)
/*
* The local_irq_*() APIs are equal to the raw_local_irq*()
* if !TRACE_IRQFLAGS.
*/
-#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
#define local_irq_enable() \
do { trace_hardirqs_on(); raw_local_irq_enable(); } while (0)
#define local_irq_disable() \
do { raw_local_irq_disable(); trace_hardirqs_off(); } while (0)
-#define local_irq_save(flags) \
- do { \
- raw_local_irq_save(flags); \
- trace_hardirqs_off(); \
- } while (0)
+#define local_irq_save(flags) \
+do { \
+ raw_local_irq_save(flags); \
+ trace_hardirqs_off(); \
+} while (0)
#define local_irq_restore(flags) \
- do { \
- if (raw_irqs_disabled_flags(flags)) { \
- raw_local_irq_restore(flags); \
- trace_hardirqs_off(); \
- } else { \
- trace_hardirqs_on(); \
- raw_local_irq_restore(flags); \
- } \
- } while (0)
-#define local_save_flags(flags) \
- do { \
- raw_local_save_flags(flags); \
- } while (0)
-
-#define irqs_disabled_flags(flags) \
- ({ \
- raw_irqs_disabled_flags(flags); \
- })
-
-#define irqs_disabled() \
- ({ \
- unsigned long _flags; \
- raw_local_save_flags(_flags); \
- raw_irqs_disabled_flags(_flags); \
- })
-
-#define safe_halt() \
- do { \
- trace_hardirqs_on(); \
- raw_safe_halt(); \
- } while (0)
-
-
-#else /* !CONFIG_TRACE_IRQFLAGS_SUPPORT */
-
-#define local_irq_enable() do { raw_local_irq_enable(); } while (0)
-#define local_irq_disable() do { raw_local_irq_disable(); } while (0)
-#define local_irq_save(flags) \
- do { \
- raw_local_irq_save(flags); \
- } while (0)
-#define local_irq_restore(flags) do { raw_local_irq_restore(flags); } while (0)
-#define local_save_flags(flags) do { raw_local_save_flags(flags); } while (0)
-#define irqs_disabled() (raw_irqs_disabled())
-#define irqs_disabled_flags(flags) (raw_irqs_disabled_flags(flags))
-#define safe_halt() do { raw_safe_halt(); } while (0)
-
-#endif /* CONFIG_TRACE_IRQFLAGS_SUPPORT */
+do { \
+ if (raw_irqs_disabled_flags(flags)) { \
+ raw_local_irq_restore(flags); \
+ trace_hardirqs_off(); \
+ } else { \
+ trace_hardirqs_on(); \
+ raw_local_irq_restore(flags); \
+ } \
+} while (0)
+
+#define local_save_flags(flags) raw_local_save_flags(flags)
+#define irqs_disabled() raw_irqs_disabled()
+#define irqs_disabled_flags(flags) raw_irqs_disabled_flags(flags)
+#define safe_halt() \
+do { \
+ trace_hardirqs_on(); \
+ arch_safe_halt(); \
+} while (0)
#endif