diff options
Diffstat (limited to 'mm/kfence')
| -rw-r--r-- | mm/kfence/core.c | 106 | ||||
| -rw-r--r-- | mm/kfence/kfence.h | 10 | ||||
| -rw-r--r-- | mm/kfence/kfence_test.c | 22 | ||||
| -rw-r--r-- | mm/kfence/report.c | 2 |
4 files changed, 81 insertions, 59 deletions
diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 79c94ee55f97..dad3c0eb70a0 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -297,20 +297,13 @@ metadata_update_state(struct kfence_metadata *meta, enum kfence_object_state nex WRITE_ONCE(meta->state, next); } -/* Write canary byte to @addr. */ -static inline bool set_canary_byte(u8 *addr) -{ - *addr = KFENCE_CANARY_PATTERN(addr); - return true; -} - /* Check canary byte at @addr. */ static inline bool check_canary_byte(u8 *addr) { struct kfence_metadata *meta; unsigned long flags; - if (likely(*addr == KFENCE_CANARY_PATTERN(addr))) + if (likely(*addr == KFENCE_CANARY_PATTERN_U8(addr))) return true; atomic_long_inc(&counters[KFENCE_COUNTER_BUGS]); @@ -323,15 +316,31 @@ static inline bool check_canary_byte(u8 *addr) return false; } -/* __always_inline this to ensure we won't do an indirect call to fn. */ -static __always_inline void for_each_canary(const struct kfence_metadata *meta, bool (*fn)(u8 *)) +static inline void set_canary(const struct kfence_metadata *meta) { const unsigned long pageaddr = ALIGN_DOWN(meta->addr, PAGE_SIZE); - unsigned long addr; + unsigned long addr = pageaddr; /* - * We'll iterate over each canary byte per-side until fn() returns - * false. However, we'll still iterate over the canary bytes to the + * The canary may be written to part of the object memory, but it does + * not affect it. The user should initialize the object before using it. + */ + for (; addr < meta->addr; addr += sizeof(u64)) + *((u64 *)addr) = KFENCE_CANARY_PATTERN_U64; + + addr = ALIGN_DOWN(meta->addr + meta->size, sizeof(u64)); + for (; addr - pageaddr < PAGE_SIZE; addr += sizeof(u64)) + *((u64 *)addr) = KFENCE_CANARY_PATTERN_U64; +} + +static inline void check_canary(const struct kfence_metadata *meta) +{ + const unsigned long pageaddr = ALIGN_DOWN(meta->addr, PAGE_SIZE); + unsigned long addr = pageaddr; + + /* + * We'll iterate over each canary byte per-side until a corrupted byte + * is found. However, we'll still iterate over the canary bytes to the * right of the object even if there was an error in the canary bytes to * the left of the object. Specifically, if check_canary_byte() * generates an error, showing both sides might give more clues as to @@ -339,16 +348,35 @@ static __always_inline void for_each_canary(const struct kfence_metadata *meta, */ /* Apply to left of object. */ - for (addr = pageaddr; addr < meta->addr; addr++) { - if (!fn((u8 *)addr)) + for (; meta->addr - addr >= sizeof(u64); addr += sizeof(u64)) { + if (unlikely(*((u64 *)addr) != KFENCE_CANARY_PATTERN_U64)) break; } - /* Apply to right of object. */ - for (addr = meta->addr + meta->size; addr < pageaddr + PAGE_SIZE; addr++) { - if (!fn((u8 *)addr)) + /* + * If the canary is corrupted in a certain 64 bytes, or the canary + * memory cannot be completely covered by multiple consecutive 64 bytes, + * it needs to be checked one by one. + */ + for (; addr < meta->addr; addr++) { + if (unlikely(!check_canary_byte((u8 *)addr))) break; } + + /* Apply to right of object. */ + for (addr = meta->addr + meta->size; addr % sizeof(u64) != 0; addr++) { + if (unlikely(!check_canary_byte((u8 *)addr))) + return; + } + for (; addr - pageaddr < PAGE_SIZE; addr += sizeof(u64)) { + if (unlikely(*((u64 *)addr) != KFENCE_CANARY_PATTERN_U64)) { + + for (; addr - pageaddr < PAGE_SIZE; addr++) { + if (!check_canary_byte((u8 *)addr)) + return; + } + } + } } static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t gfp, @@ -434,7 +462,7 @@ static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t g #endif /* Memory initialization. */ - for_each_canary(meta, set_canary_byte); + set_canary(meta); /* * We check slab_want_init_on_alloc() ourselves, rather than letting @@ -495,7 +523,7 @@ static void kfence_guarded_free(void *addr, struct kfence_metadata *meta, bool z alloc_covered_add(meta->alloc_stack_hash, -1); /* Check canary bytes for memory corruption. */ - for_each_canary(meta, check_canary_byte); + check_canary(meta); /* * Clear memory if init-on-free is set. While we protect the page, the @@ -556,15 +584,11 @@ static unsigned long kfence_init_pool(void) * enters __slab_free() slow-path. */ for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) { - struct slab *slab = page_slab(&pages[i]); + struct slab *slab = page_slab(nth_page(pages, i)); if (!i || (i % 2)) continue; - /* Verify we do not have a compound head page. */ - if (WARN_ON(compound_head(&pages[i]) != &pages[i])) - return addr; - __folio_set_slab(slab_folio(slab)); #ifdef CONFIG_MEMCG slab->memcg_data = (unsigned long)&kfence_metadata[i / 2 - 1].objcg | @@ -597,12 +621,26 @@ static unsigned long kfence_init_pool(void) /* Protect the right redzone. */ if (unlikely(!kfence_protect(addr + PAGE_SIZE))) - return addr; + goto reset_slab; addr += 2 * PAGE_SIZE; } return 0; + +reset_slab: + for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) { + struct slab *slab = page_slab(nth_page(pages, i)); + + if (!i || (i % 2)) + continue; +#ifdef CONFIG_MEMCG + slab->memcg_data = 0; +#endif + __folio_clear_slab(slab_folio(slab)); + } + + return addr; } static bool __init kfence_init_pool_early(void) @@ -632,16 +670,6 @@ static bool __init kfence_init_pool_early(void) * fails for the first page, and therefore expect addr==__kfence_pool in * most failure cases. */ - for (char *p = (char *)addr; p < __kfence_pool + KFENCE_POOL_SIZE; p += PAGE_SIZE) { - struct slab *slab = virt_to_slab(p); - - if (!slab) - continue; -#ifdef CONFIG_MEMCG - slab->memcg_data = 0; -#endif - __folio_clear_slab(slab_folio(slab)); - } memblock_free_late(__pa(addr), KFENCE_POOL_SIZE - (addr - (unsigned long)__kfence_pool)); __kfence_pool = NULL; return false; @@ -751,7 +779,7 @@ static void kfence_check_all_canary(void) struct kfence_metadata *meta = &kfence_metadata[i]; if (meta->state == KFENCE_OBJECT_ALLOCATED) - for_each_canary(meta, check_canary_byte); + check_canary(meta); } } @@ -818,6 +846,10 @@ void __init kfence_alloc_pool(void) if (!kfence_sample_interval) return; + /* if the pool has already been initialized by arch, skip the below. */ + if (__kfence_pool) + return; + __kfence_pool = memblock_alloc(KFENCE_POOL_SIZE, PAGE_SIZE); if (!__kfence_pool) diff --git a/mm/kfence/kfence.h b/mm/kfence/kfence.h index 600f2e2431d6..2aafc46a4aaf 100644 --- a/mm/kfence/kfence.h +++ b/mm/kfence/kfence.h @@ -21,7 +21,15 @@ * lower 3 bits of the address, to detect memory corruptions with higher * probability, where similar constants are used. */ -#define KFENCE_CANARY_PATTERN(addr) ((u8)0xaa ^ (u8)((unsigned long)(addr) & 0x7)) +#define KFENCE_CANARY_PATTERN_U8(addr) ((u8)0xaa ^ (u8)((unsigned long)(addr) & 0x7)) + +/* + * Define a continuous 8-byte canary starting from a multiple of 8. The canary + * of each byte is only related to the lowest three bits of its address, so the + * canary of every 8 bytes is the same. 64-bit memory can be filled and checked + * at a time instead of byte by byte to improve performance. + */ +#define KFENCE_CANARY_PATTERN_U64 ((u64)0xaaaaaaaaaaaaaaaa ^ (u64)(0x0706050403020100)) /* Maximum stack depth for reports. */ #define KFENCE_STACK_DEPTH 64 diff --git a/mm/kfence/kfence_test.c b/mm/kfence/kfence_test.c index b5d66a69200d..6aee19a79236 100644 --- a/mm/kfence/kfence_test.c +++ b/mm/kfence/kfence_test.c @@ -825,33 +825,15 @@ static void test_exit(struct kunit *test) test_cache_destroy(); } -static void register_tracepoints(struct tracepoint *tp, void *ignore) -{ - check_trace_callback_type_console(probe_console); - if (!strcmp(tp->name, "console")) - WARN_ON(tracepoint_probe_register(tp, probe_console, NULL)); -} - -static void unregister_tracepoints(struct tracepoint *tp, void *ignore) -{ - if (!strcmp(tp->name, "console")) - tracepoint_probe_unregister(tp, probe_console, NULL); -} - static int kfence_suite_init(struct kunit_suite *suite) { - /* - * Because we want to be able to build the test as a module, we need to - * iterate through all known tracepoints, since the static registration - * won't work here. - */ - for_each_kernel_tracepoint(register_tracepoints, NULL); + register_trace_console(probe_console, NULL); return 0; } static void kfence_suite_exit(struct kunit_suite *suite) { - for_each_kernel_tracepoint(unregister_tracepoints, NULL); + unregister_trace_console(probe_console, NULL); tracepoint_synchronize_unregister(); } diff --git a/mm/kfence/report.c b/mm/kfence/report.c index 60205f1257ef..197430a5be4a 100644 --- a/mm/kfence/report.c +++ b/mm/kfence/report.c @@ -168,7 +168,7 @@ static void print_diff_canary(unsigned long address, size_t bytes_to_show, pr_cont("["); for (cur = (const u8 *)address; cur < end; cur++) { - if (*cur == KFENCE_CANARY_PATTERN(cur)) + if (*cur == KFENCE_CANARY_PATTERN_U8(cur)) pr_cont(" ."); else if (no_hash_pointers) pr_cont(" 0x%02x", *cur); |