irqs_disabled counterirq_stuff

4 files changed, 88 insertions, 79 deletions

diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h
index bba3cf88e624..9a39472f11de 100644
--- a/arch/x86/include/asm/irqflags.h
+++ b/arch/x86/include/asm/irqflags.h
@@ -191,11 +191,17 @@ static inline int arch_irqs_disabled(void)
 #endif
 
 #ifdef CONFIG_TRACE_IRQFLAGS
-#  define TRACE_IRQS_ON		call trace_hardirqs_on_thunk;
-#  define TRACE_IRQS_OFF	call trace_hardirqs_off_thunk;
+#  define TRACE_IRQS_ON				\
+	call trace_hardirqs_on_thunk;		\
+	movl $0,PER_CPU_VAR(irq_disable_count);
+#  define TRACE_IRQS_OFF			\
+	movl $1,PER_CPU_VAR(irq_disable_count);	\
+	call trace_hardirqs_off_thunk;
 #else
-#  define TRACE_IRQS_ON
-#  define TRACE_IRQS_OFF
+#  define TRACE_IRQS_ON				\
+	movl $0,PER_CPU_VAR(irq_disable_count);
+#  define TRACE_IRQS_OFF			\
+	movl $1,PER_CPU_VAR(irq_disable_count);
 #endif
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 #  define LOCKDEP_SYS_EXIT	ARCH_LOCKDEP_SYS_EXIT
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 9c3ab43a6954..a05b9590b238 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1093,6 +1093,9 @@ DEFINE_PER_CPU(char *, irq_stack_ptr) =
 
 DEFINE_PER_CPU(unsigned int, irq_count) = -1;
 
+DEFINE_PER_CPU(unsigned, irq_disable_count);
+EXPORT_PER_CPU_SYMBOL(irq_disable_count);
+
 DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
 
 /*
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index cb3c591339aa..45c52ed7864d 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -289,9 +289,9 @@ ENDPROC(native_usergs_sysret64)
 #ifdef CONFIG_TRACE_IRQFLAGS
 	bt   $9,EFLAGS-\offset(%rsp)	/* interrupts off? */
 	jnc  1f
-	TRACE_IRQS_ON
 1:
 #endif
+	TRACE_IRQS_ON
 .endm
 
 /*
diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h
index d176d658fe25..abbd51ae391c 100644
--- a/include/linux/irqflags.h
+++ b/include/linux/irqflags.h
@@ -13,6 +13,7 @@
 
 #include <linux/typecheck.h>
 #include <asm/irqflags.h>
+#include <asm/percpu.h>
 
 #ifdef CONFIG_TRACE_IRQFLAGS
   extern void trace_softirqs_on(unsigned long ip);
@@ -53,98 +54,97 @@
 # define start_critical_timings() do { } while (0)
 #endif
 
+DECLARE_PER_CPU(unsigned, irq_disable_count);
+
 /*
  * Wrap the arch provided IRQ routines to provide appropriate checks.
  */
-#define raw_local_irq_disable()		arch_local_irq_disable()
-#define raw_local_irq_enable()		arch_local_irq_enable()
+#define raw_local_irq_disable()				\
+do {							\
+	arch_local_irq_disable();			\
+	__raw_get_cpu_var(irq_disable_count) = 1;	\
+} while (0)
+
+#define raw_local_irq_enable()				\
+do {							\
+	__raw_get_cpu_var(irq_disable_count) = 0;	\
+	arch_local_irq_enable();			\
+} while (0)
+
 #define raw_local_irq_save(flags)			\
-	do {						\
-		typecheck(unsigned long, flags);	\
-		flags = arch_local_irq_save();		\
-	} while (0)
+do {							\
+	typecheck(unsigned long, flags);		\
+							\
+	arch_local_irq_disable();			\
+	flags = __raw_get_cpu_var(irq_disable_count)++;	\
+} while (0)
+
+#if 1
 #define raw_local_irq_restore(flags)			\
-	do {						\
-		typecheck(unsigned long, flags);	\
-		arch_local_irq_restore(flags);		\
-	} while (0)
+do {							\
+	typecheck(unsigned long, flags);		\
+							\
+	__raw_get_cpu_var(irq_disable_count) = flags;	\
+	if (!flags)					\
+		arch_local_irq_enable();		\
+} while (0)
+#else
+#define raw_local_irq_restore(flags)			\
+do {							\
+	typecheck(unsigned long, flags);		\
+							\
+	if (!--__raw_get_cpu_var(irq_disable_count))	\
+		arch_local_irq_enable();		\
+} while (0)
+#endif
+
 #define raw_local_save_flags(flags)			\
-	do {						\
-		typecheck(unsigned long, flags);	\
-		flags = arch_local_save_flags();	\
-	} while (0)
+do {							\
+	typecheck(unsigned long, flags);		\
+	flags = __raw_get_cpu_var(irq_disable_count);	\
+} while (0)
+
 #define raw_irqs_disabled_flags(flags)			\
-	({						\
-		typecheck(unsigned long, flags);	\
-		arch_irqs_disabled_flags(flags);	\
-	})
-#define raw_irqs_disabled()		(arch_irqs_disabled())
-#define raw_safe_halt()			arch_safe_halt()
+({							\
+	typecheck(unsigned long, flags);		\
+	flags != 0;					\
+})
+
+#define raw_irqs_disabled()	(__raw_get_cpu_var(irq_disable_count) != 0)
 
 /*
  * The local_irq_*() APIs are equal to the raw_local_irq*()
  * if !TRACE_IRQFLAGS.
  */
-#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
 #define local_irq_enable() \
 	do { trace_hardirqs_on(); raw_local_irq_enable(); } while (0)
 #define local_irq_disable() \
 	do { raw_local_irq_disable(); trace_hardirqs_off(); } while (0)
-#define local_irq_save(flags)				\
-	do {						\
-		raw_local_irq_save(flags);		\
-		trace_hardirqs_off();			\
-	} while (0)
 
+#define local_irq_save(flags)				\
+do {							\
+	raw_local_irq_save(flags);			\
+	trace_hardirqs_off();				\
+} while (0)
 
 #define local_irq_restore(flags)			\
-	do {						\
-		if (raw_irqs_disabled_flags(flags)) {	\
-			raw_local_irq_restore(flags);	\
-			trace_hardirqs_off();		\
-		} else {				\
-			trace_hardirqs_on();		\
-			raw_local_irq_restore(flags);	\
-		}					\
-	} while (0)
-#define local_save_flags(flags)				\
-	do {						\
-		raw_local_save_flags(flags);		\
-	} while (0)
-
-#define irqs_disabled_flags(flags)			\
-	({						\
-		raw_irqs_disabled_flags(flags);		\
-	})
-
-#define irqs_disabled()					\
-	({						\
-		unsigned long _flags;			\
-		raw_local_save_flags(_flags);		\
-		raw_irqs_disabled_flags(_flags);	\
-	})
-
-#define safe_halt()				\
-	do {					\
-		trace_hardirqs_on();		\
-		raw_safe_halt();		\
-	} while (0)
-
-
-#else /* !CONFIG_TRACE_IRQFLAGS_SUPPORT */
-
-#define local_irq_enable()	do { raw_local_irq_enable(); } while (0)
-#define local_irq_disable()	do { raw_local_irq_disable(); } while (0)
-#define local_irq_save(flags)					\
-	do {							\
-		raw_local_irq_save(flags);			\
-	} while (0)
-#define local_irq_restore(flags) do { raw_local_irq_restore(flags); } while (0)
-#define local_save_flags(flags)	do { raw_local_save_flags(flags); } while (0)
-#define irqs_disabled()		(raw_irqs_disabled())
-#define irqs_disabled_flags(flags) (raw_irqs_disabled_flags(flags))
-#define safe_halt()		do { raw_safe_halt(); } while (0)
-
-#endif /* CONFIG_TRACE_IRQFLAGS_SUPPORT */
+do {							\
+	if (raw_irqs_disabled_flags(flags)) {		\
+		raw_local_irq_restore(flags);		\
+		trace_hardirqs_off();			\
+	} else {					\
+		trace_hardirqs_on();			\
+		raw_local_irq_restore(flags);		\
+	}						\
+} while (0)
+
+#define local_save_flags(flags)		raw_local_save_flags(flags)
+#define irqs_disabled()			raw_irqs_disabled()
+#define irqs_disabled_flags(flags)	raw_irqs_disabled_flags(flags)
+#define safe_halt()					\
+do {							\
+	trace_hardirqs_on();				\
+	arch_safe_halt();				\
+} while (0)
 
 #endif