From dee081bf8f824cabeb7c7495367d5dad0a444eb1 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 19 Dec 2019 14:22:31 +0000 Subject: READ_ONCE: Drop pointer qualifiers when reading from scalar types Passing a volatile-qualified pointer to READ_ONCE() is an absolute trainwreck for code generation: the use of 'typeof()' to define a temporary variable inside the macro means that the final evaluation in macro scope ends up forcing a read back from the stack. When stack protector is enabled (the default for arm64, at least), this causes the compiler to vomit up all sorts of junk. Unfortunately, dropping pointer qualifiers inside the macro poses quite a challenge, especially since the pointed-to type is permitted to be an aggregate, and this is relied upon by mm/ code accessing things like 'pmd_t'. Based on numerous hacks and discussions on the mailing list, this is the best I've managed to come up with. Introduce '__unqual_scalar_typeof()' which takes an expression and, if the expression is an optionally qualified 8, 16, 32 or 64-bit scalar type, evaluates to the unqualified type. Other input types, including aggregates, remain unchanged. Hopefully READ_ONCE() on volatile aggregate pointers isn't something we do on a fast-path. Cc: Peter Zijlstra Cc: Arnd Bergmann Suggested-by: Linus Torvalds Reported-by: Michael Ellerman Signed-off-by: Will Deacon --- include/linux/compiler_types.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'include/linux/compiler_types.h') diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index e970f97a7fcb..233066c92f6f 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -210,6 +210,27 @@ struct ftrace_likely_data { /* Are two types/vars the same type (ignoring qualifiers)? */ #define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b)) +/* + * __unqual_scalar_typeof(x) - Declare an unqualified scalar type, leaving + * non-scalar types unchanged. + * + * We build this out of a couple of helper macros in a vain attempt to + * help you keep your lunch down while reading it. + */ +#define __pick_scalar_type(x, type, otherwise) \ + __builtin_choose_expr(__same_type(x, type), (type)0, otherwise) + +#define __pick_integer_type(x, type, otherwise) \ + __pick_scalar_type(x, unsigned type, \ + __pick_scalar_type(x, signed type, otherwise)) + +#define __unqual_scalar_typeof(x) typeof( \ + __pick_integer_type(x, char, \ + __pick_integer_type(x, short, \ + __pick_integer_type(x, int, \ + __pick_integer_type(x, long, \ + __pick_integer_type(x, long long, x)))))) + /* Is this type a native word size -- useful for atomic operations */ #define __native_word(t) \ (sizeof(t) == sizeof(char) || sizeof(t) == sizeof(short) || \ -- cgit v1.2.3 From 8d4beed7bbc71666de2630b79899c8852c3bf5cd Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 5 Jun 2020 11:05:51 +0100 Subject: compiler-types.h: Include naked type in __pick_integer_type() match __pick_integer_type() checks whether the type of its first argument is compatible with an explicitly signed or unsigned integer type, returning the compatible type if it exists. Unfortunately, 'char' is neither compatible with 'signed char' nor 'unsigned char', so add a check against the naked type to allow the __unqual_scalar_typeof() macro to strip qualifiers from char types without an explicit signedness. Reported-by: Rasmus Villemoes Signed-off-by: Will Deacon --- include/linux/compiler_types.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'include/linux/compiler_types.h') diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 233066c92f6f..6ed0612bc143 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -220,9 +220,14 @@ struct ftrace_likely_data { #define __pick_scalar_type(x, type, otherwise) \ __builtin_choose_expr(__same_type(x, type), (type)0, otherwise) +/* + * 'char' is not type-compatible with either 'signed char' or 'unsigned char', + * so we include the naked type here as well as the signed/unsigned variants. + */ #define __pick_integer_type(x, type, otherwise) \ - __pick_scalar_type(x, unsigned type, \ - __pick_scalar_type(x, signed type, otherwise)) + __pick_scalar_type(x, type, \ + __pick_scalar_type(x, unsigned type, \ + __pick_scalar_type(x, signed type, otherwise))) #define __unqual_scalar_typeof(x) typeof( \ __pick_integer_type(x, char, \ -- cgit v1.2.3 From 1fd76043ecb04b8567a76b106db09ac778e1e2b8 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Wed, 27 May 2020 12:32:36 +0200 Subject: compiler_types.h: Optimize __unqual_scalar_typeof compilation time If the compiler supports C11's _Generic, use it to speed up compilation times of __unqual_scalar_typeof(). GCC version 4.9 or later and all supported versions of Clang support the feature (the oldest supported compiler that doesn't support _Generic is GCC 4.8, for which we use the slower alternative). The non-_Generic variant relies on multiple expansions of __pick_integer_type -> __pick_scalar_type -> __builtin_choose_expr, which increases pre-processed code size, and can cause compile times to increase in files with numerous expansions of READ_ONCE(), or other users of __unqual_scalar_typeof(). Summary of compile-time benchmarking done by Arnd Bergmann: clang-11 gcc-9 this patch 0.78 0.91 ideal 0.76 0.86 See https://lkml.kernel.org/r/CAK8P3a3UYQeXhiufUevz=rwe09WM_vSTCd9W+KvJHJcOeQyWVA@mail.gmail.com Further compile-testing done with: gcc 4.8, 4.9, 5.5, 6.4, 7.5, 8.4; clang 9, 10. Reported-by: Arnd Bergmann Signed-off-by: Marco Elver Signed-off-by: Borislav Petkov Acked-by: Peter Zijlstra Tested-by: Arnd Bergmann Link: https://lkml.kernel.org/r/20200527103236.148700-1-elver@google.com Link: https://lkml.kernel.org/r/CAK8P3a0RJtbVi1JMsfik=jkHCNFv+DJn_FeDg-YLW+ueQW3tNg@mail.gmail.com [will: tweak new macros to make them a bit more readable] Signed-off-by: Will Deacon --- include/linux/compiler_types.h | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) (limited to 'include/linux/compiler_types.h') diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 6ed0612bc143..31416b60eabf 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -213,7 +213,9 @@ struct ftrace_likely_data { /* * __unqual_scalar_typeof(x) - Declare an unqualified scalar type, leaving * non-scalar types unchanged. - * + */ +#if defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 40900 +/* * We build this out of a couple of helper macros in a vain attempt to * help you keep your lunch down while reading it. */ @@ -235,6 +237,25 @@ struct ftrace_likely_data { __pick_integer_type(x, int, \ __pick_integer_type(x, long, \ __pick_integer_type(x, long long, x)))))) +#else +/* + * If supported, prefer C11 _Generic for better compile-times. As above, 'char' + * is not type-compatible with 'signed char', and we define a separate case. + */ +#define __scalar_type_to_expr_cases(type) \ + unsigned type: (unsigned type)0, \ + signed type: (signed type)0 + +#define __unqual_scalar_typeof(x) typeof( \ + _Generic((x), \ + char: (char)0, \ + __scalar_type_to_expr_cases(char), \ + __scalar_type_to_expr_cases(short), \ + __scalar_type_to_expr_cases(int), \ + __scalar_type_to_expr_cases(long), \ + __scalar_type_to_expr_cases(long long), \ + default: (x))) +#endif /* Is this type a native word size -- useful for atomic operations */ #define __native_word(t) \ -- cgit v1.2.3 From b398ace5d2ea0b7f00d9f1ce23c647e289c206ca Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Thu, 28 May 2020 09:43:13 +0200 Subject: compiler_types.h: Use unoptimized __unqual_scalar_typeof for sparse If the file is being checked with sparse, use the unoptimized version of __unqual_scalar_typeof(), since sparse does not support _Generic. Reported-by: kbuild test robot Signed-off-by: Marco Elver Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/202005280727.lXn1VnTw%lkp@intel.com Signed-off-by: Will Deacon --- include/linux/compiler_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux/compiler_types.h') diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 31416b60eabf..cd73e3857a87 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -214,7 +214,7 @@ struct ftrace_likely_data { * __unqual_scalar_typeof(x) - Declare an unqualified scalar type, leaving * non-scalar types unchanged. */ -#if defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 40900 +#if (defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 40900) || defined(__CHECKER__) /* * We build this out of a couple of helper macros in a vain attempt to * help you keep your lunch down while reading it. -- cgit v1.2.3