diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/Kconfig.debug | 100 | ||||
-rw-r--r-- | lib/Makefile | 2 | ||||
-rw-r--r-- | lib/fdt_addresses.c | 2 | ||||
-rw-r--r-- | lib/iov_iter.c | 3 | ||||
-rw-r--r-- | lib/livepatch/test_klp_shadow_vars.c | 119 | ||||
-rw-r--r-- | lib/sbitmap.c | 2 | ||||
-rw-r--r-- | lib/strncpy_from_user.c | 14 | ||||
-rw-r--r-- | lib/strnlen_user.c | 14 | ||||
-rw-r--r-- | lib/test_xarray.c | 78 | ||||
-rw-r--r-- | lib/vdso/gettimeofday.c | 1 | ||||
-rw-r--r-- | lib/xarray.c | 41 |
11 files changed, 234 insertions, 142 deletions
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index d1842fe756d5..5ffe144c9794 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1483,6 +1483,55 @@ config PROVIDE_OHCI1394_DMA_INIT See Documentation/debugging-via-ohci1394.txt for more information. +source "samples/Kconfig" + +config ARCH_HAS_DEVMEM_IS_ALLOWED + bool + +config STRICT_DEVMEM + bool "Filter access to /dev/mem" + depends on MMU && DEVMEM + depends on ARCH_HAS_DEVMEM_IS_ALLOWED + default y if PPC || X86 || ARM64 + help + If this option is disabled, you allow userspace (root) access to all + of memory, including kernel and userspace memory. Accidental + access to this is obviously disastrous, but specific access can + be used by people debugging the kernel. Note that with PAT support + enabled, even in this case there are restrictions on /dev/mem + use due to the cache aliasing requirements. + + If this option is switched on, and IO_STRICT_DEVMEM=n, the /dev/mem + file only allows userspace access to PCI space and the BIOS code and + data regions. This is sufficient for dosemu and X and all common + users of /dev/mem. + + If in doubt, say Y. + +config IO_STRICT_DEVMEM + bool "Filter I/O access to /dev/mem" + depends on STRICT_DEVMEM + help + If this option is disabled, you allow userspace (root) access to all + io-memory regardless of whether a driver is actively using that + range. Accidental access to this is obviously disastrous, but + specific access can be used by people debugging kernel drivers. + + If this option is switched on, the /dev/mem file only allows + userspace access to *idle* io-memory ranges (see /proc/iomem) This + may break traditional users of /dev/mem (dosemu, legacy X, etc...) + if the driver using a given range cannot be disabled. + + If in doubt, say Y. + +menu "$(SRCARCH) Debugging" + +source "arch/$(SRCARCH)/Kconfig.debug" + +endmenu + +menu "Kernel Testing and Coverage" + source "lib/kunit/Kconfig" config NOTIFIER_ERROR_INJECTION @@ -1643,10 +1692,6 @@ config FAULT_INJECTION_STACKTRACE_FILTER help Provide stacktrace filter for fault-injection capabilities -endmenu # "Kernel Testing and Coverage" - -menu "Kernel Testing and Coverage" - config ARCH_HAS_KCOV bool help @@ -2130,52 +2175,7 @@ config MEMTEST memtest=17, mean do 17 test patterns. If you are unsure how to answer this question, answer N. -source "samples/Kconfig" - -config ARCH_HAS_DEVMEM_IS_ALLOWED - bool - -config STRICT_DEVMEM - bool "Filter access to /dev/mem" - depends on MMU && DEVMEM - depends on ARCH_HAS_DEVMEM_IS_ALLOWED - default y if PPC || X86 || ARM64 - ---help--- - If this option is disabled, you allow userspace (root) access to all - of memory, including kernel and userspace memory. Accidental - access to this is obviously disastrous, but specific access can - be used by people debugging the kernel. Note that with PAT support - enabled, even in this case there are restrictions on /dev/mem - use due to the cache aliasing requirements. - - If this option is switched on, and IO_STRICT_DEVMEM=n, the /dev/mem - file only allows userspace access to PCI space and the BIOS code and - data regions. This is sufficient for dosemu and X and all common - users of /dev/mem. - - If in doubt, say Y. -config IO_STRICT_DEVMEM - bool "Filter I/O access to /dev/mem" - depends on STRICT_DEVMEM - ---help--- - If this option is disabled, you allow userspace (root) access to all - io-memory regardless of whether a driver is actively using that - range. Accidental access to this is obviously disastrous, but - specific access can be used by people debugging kernel drivers. - - If this option is switched on, the /dev/mem file only allows - userspace access to *idle* io-memory ranges (see /proc/iomem) This - may break traditional users of /dev/mem (dosemu, legacy X, etc...) - if the driver using a given range cannot be disabled. - - If in doubt, say Y. - -menu "$(SRCARCH) Debugging" - -source "arch/$(SRCARCH)/Kconfig.debug" - -endmenu config HYPERV_TESTING bool "Microsoft Hyper-V driver testing" @@ -2184,4 +2184,6 @@ config HYPERV_TESTING help Select this option to enable Hyper-V vmbus testing. +endmenu # "Kernel Testing and Coverage" + endmenu # Kernel hacking diff --git a/lib/Makefile b/lib/Makefile index 93217d44237f..c20b1debe9b4 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -223,7 +223,7 @@ KASAN_SANITIZE_stackdepot.o := n KCOV_INSTRUMENT_stackdepot.o := n libfdt_files = fdt.o fdt_ro.o fdt_wip.o fdt_rw.o fdt_sw.o fdt_strerror.o \ - fdt_empty_tree.o + fdt_empty_tree.o fdt_addresses.o $(foreach file, $(libfdt_files), \ $(eval CFLAGS_$(file) = -I $(srctree)/scripts/dtc/libfdt)) lib-$(CONFIG_LIBFDT) += $(libfdt_files) diff --git a/lib/fdt_addresses.c b/lib/fdt_addresses.c new file mode 100644 index 000000000000..23610bcf390b --- /dev/null +++ b/lib/fdt_addresses.c @@ -0,0 +1,2 @@ +#include <linux/libfdt_env.h> +#include "../scripts/dtc/libfdt/fdt_addresses.c" diff --git a/lib/iov_iter.c b/lib/iov_iter.c index fb29c02c6a3c..51595bf3af85 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1222,11 +1222,12 @@ EXPORT_SYMBOL(iov_iter_discard); unsigned long iov_iter_alignment(const struct iov_iter *i) { - unsigned int p_mask = i->pipe->ring_size - 1; unsigned long res = 0; size_t size = i->count; if (unlikely(iov_iter_is_pipe(i))) { + unsigned int p_mask = i->pipe->ring_size - 1; + if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask])) return size | i->iov_offset; return size; diff --git a/lib/livepatch/test_klp_shadow_vars.c b/lib/livepatch/test_klp_shadow_vars.c index fe5c413efe96..f0b5a1d24e55 100644 --- a/lib/livepatch/test_klp_shadow_vars.c +++ b/lib/livepatch/test_klp_shadow_vars.c @@ -60,36 +60,43 @@ static int ptr_id(void *ptr) */ static void *shadow_get(void *obj, unsigned long id) { - void *ret = klp_shadow_get(obj, id); + int **sv; + sv = klp_shadow_get(obj, id); pr_info("klp_%s(obj=PTR%d, id=0x%lx) = PTR%d\n", - __func__, ptr_id(obj), id, ptr_id(ret)); + __func__, ptr_id(obj), id, ptr_id(sv)); - return ret; + return sv; } static void *shadow_alloc(void *obj, unsigned long id, size_t size, gfp_t gfp_flags, klp_shadow_ctor_t ctor, void *ctor_data) { - void *ret = klp_shadow_alloc(obj, id, size, gfp_flags, ctor, - ctor_data); + int **var = ctor_data; + int **sv; + + sv = klp_shadow_alloc(obj, id, size, gfp_flags, ctor, var); pr_info("klp_%s(obj=PTR%d, id=0x%lx, size=%zx, gfp_flags=%pGg), ctor=PTR%d, ctor_data=PTR%d = PTR%d\n", __func__, ptr_id(obj), id, size, &gfp_flags, ptr_id(ctor), - ptr_id(ctor_data), ptr_id(ret)); - return ret; + ptr_id(*var), ptr_id(sv)); + + return sv; } static void *shadow_get_or_alloc(void *obj, unsigned long id, size_t size, gfp_t gfp_flags, klp_shadow_ctor_t ctor, void *ctor_data) { - void *ret = klp_shadow_get_or_alloc(obj, id, size, gfp_flags, ctor, - ctor_data); + int **var = ctor_data; + int **sv; + + sv = klp_shadow_get_or_alloc(obj, id, size, gfp_flags, ctor, var); pr_info("klp_%s(obj=PTR%d, id=0x%lx, size=%zx, gfp_flags=%pGg), ctor=PTR%d, ctor_data=PTR%d = PTR%d\n", __func__, ptr_id(obj), id, size, &gfp_flags, ptr_id(ctor), - ptr_id(ctor_data), ptr_id(ret)); - return ret; + ptr_id(*var), ptr_id(sv)); + + return sv; } static void shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor) @@ -110,58 +117,70 @@ static void shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor) /* Shadow variable constructor - remember simple pointer data */ static int shadow_ctor(void *obj, void *shadow_data, void *ctor_data) { - int **shadow_int = shadow_data; - *shadow_int = ctor_data; + int **sv = shadow_data; + int **var = ctor_data; + + if (!var) + return -EINVAL; + + *sv = *var; pr_info("%s: PTR%d -> PTR%d\n", - __func__, ptr_id(shadow_int), ptr_id(ctor_data)); + __func__, ptr_id(sv), ptr_id(*var)); return 0; } static void shadow_dtor(void *obj, void *shadow_data) { + int **sv = shadow_data; + pr_info("%s(obj=PTR%d, shadow_data=PTR%d)\n", - __func__, ptr_id(obj), ptr_id(shadow_data)); + __func__, ptr_id(obj), ptr_id(sv)); } static int test_klp_shadow_vars_init(void) { void *obj = THIS_MODULE; int id = 0x1234; - size_t size = sizeof(int *); gfp_t gfp_flags = GFP_KERNEL; int var1, var2, var3, var4; + int *pv1, *pv2, *pv3, *pv4; int **sv1, **sv2, **sv3, **sv4; - void *ret; + int **sv; + + pv1 = &var1; + pv2 = &var2; + pv3 = &var3; + pv4 = &var4; ptr_id(NULL); - ptr_id(&var1); - ptr_id(&var2); - ptr_id(&var3); - ptr_id(&var4); + ptr_id(pv1); + ptr_id(pv2); + ptr_id(pv3); + ptr_id(pv4); /* * With an empty shadow variable hash table, expect not to find * any matches. */ - ret = shadow_get(obj, id); - if (!ret) + sv = shadow_get(obj, id); + if (!sv) pr_info(" got expected NULL result\n"); /* * Allocate a few shadow variables with different <obj> and <id>. */ - sv1 = shadow_alloc(obj, id, size, gfp_flags, shadow_ctor, &var1); + sv1 = shadow_alloc(obj, id, sizeof(pv1), gfp_flags, shadow_ctor, &pv1); if (!sv1) return -ENOMEM; - sv2 = shadow_alloc(obj + 1, id, size, gfp_flags, shadow_ctor, &var2); + sv2 = shadow_alloc(obj + 1, id, sizeof(pv2), gfp_flags, shadow_ctor, &pv2); if (!sv2) return -ENOMEM; - sv3 = shadow_alloc(obj, id + 1, size, gfp_flags, shadow_ctor, &var3); + sv3 = shadow_alloc(obj, id + 1, sizeof(pv3), gfp_flags, shadow_ctor, &pv3); if (!sv3) return -ENOMEM; @@ -169,23 +188,23 @@ static int test_klp_shadow_vars_init(void) * Verify we can find our new shadow variables and that they point * to expected data. */ - ret = shadow_get(obj, id); - if (!ret) + sv = shadow_get(obj, id); + if (!sv) return -EINVAL; - if (ret == sv1 && *sv1 == &var1) + if (sv == sv1 && *sv1 == pv1) pr_info(" got expected PTR%d -> PTR%d result\n", ptr_id(sv1), ptr_id(*sv1)); - ret = shadow_get(obj + 1, id); - if (!ret) + sv = shadow_get(obj + 1, id); + if (!sv) return -EINVAL; - if (ret == sv2 && *sv2 == &var2) + if (sv == sv2 && *sv2 == pv2) pr_info(" got expected PTR%d -> PTR%d result\n", ptr_id(sv2), ptr_id(*sv2)); - ret = shadow_get(obj, id + 1); - if (!ret) + sv = shadow_get(obj, id + 1); + if (!sv) return -EINVAL; - if (ret == sv3 && *sv3 == &var3) + if (sv == sv3 && *sv3 == pv3) pr_info(" got expected PTR%d -> PTR%d result\n", ptr_id(sv3), ptr_id(*sv3)); @@ -193,14 +212,14 @@ static int test_klp_shadow_vars_init(void) * Allocate or get a few more, this time with the same <obj>, <id>. * The second invocation should return the same shadow var. */ - sv4 = shadow_get_or_alloc(obj + 2, id, size, gfp_flags, shadow_ctor, &var4); + sv4 = shadow_get_or_alloc(obj + 2, id, sizeof(pv4), gfp_flags, shadow_ctor, &pv4); if (!sv4) return -ENOMEM; - ret = shadow_get_or_alloc(obj + 2, id, size, gfp_flags, shadow_ctor, &var4); - if (!ret) + sv = shadow_get_or_alloc(obj + 2, id, sizeof(pv4), gfp_flags, shadow_ctor, &pv4); + if (!sv) return -EINVAL; - if (ret == sv4 && *sv4 == &var4) + if (sv == sv4 && *sv4 == pv4) pr_info(" got expected PTR%d -> PTR%d result\n", ptr_id(sv4), ptr_id(*sv4)); @@ -209,27 +228,27 @@ static int test_klp_shadow_vars_init(void) * longer find them. */ shadow_free(obj, id, shadow_dtor); /* sv1 */ - ret = shadow_get(obj, id); - if (!ret) + sv = shadow_get(obj, id); + if (!sv) pr_info(" got expected NULL result\n"); shadow_free(obj + 1, id, shadow_dtor); /* sv2 */ - ret = shadow_get(obj + 1, id); - if (!ret) + sv = shadow_get(obj + 1, id); + if (!sv) pr_info(" got expected NULL result\n"); shadow_free(obj + 2, id, shadow_dtor); /* sv4 */ - ret = shadow_get(obj + 2, id); - if (!ret) + sv = shadow_get(obj + 2, id); + if (!sv) pr_info(" got expected NULL result\n"); /* * We should still find an <id+1> variable. */ - ret = shadow_get(obj, id + 1); - if (!ret) + sv = shadow_get(obj, id + 1); + if (!sv) return -EINVAL; - if (ret == sv3 && *sv3 == &var3) + if (sv == sv3 && *sv3 == pv3) pr_info(" got expected PTR%d -> PTR%d result\n", ptr_id(sv3), ptr_id(*sv3)); @@ -237,8 +256,8 @@ static int test_klp_shadow_vars_init(void) * Free all the <id+1> variables, too. */ shadow_free_all(id + 1, shadow_dtor); /* sv3 */ - ret = shadow_get(obj, id); - if (!ret) + sv = shadow_get(obj, id); + if (!sv) pr_info(" shadow_get() got expected NULL result\n"); diff --git a/lib/sbitmap.c b/lib/sbitmap.c index 33feec8989f1..af88d1346dd7 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -650,8 +650,8 @@ void sbitmap_add_wait_queue(struct sbitmap_queue *sbq, if (!sbq_wait->sbq) { sbq_wait->sbq = sbq; atomic_inc(&sbq->ws_active); + add_wait_queue(&ws->wait, &sbq_wait->wait); } - add_wait_queue(&ws->wait, &sbq_wait->wait); } EXPORT_SYMBOL_GPL(sbitmap_add_wait_queue); diff --git a/lib/strncpy_from_user.c b/lib/strncpy_from_user.c index dccb95af6003..706020b06617 100644 --- a/lib/strncpy_from_user.c +++ b/lib/strncpy_from_user.c @@ -30,13 +30,6 @@ static inline long do_strncpy_from_user(char *dst, const char __user *src, const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS; unsigned long res = 0; - /* - * Truncate 'max' to the user-specified limit, so that - * we only have one limit we need to check in the loop - */ - if (max > count) - max = count; - if (IS_UNALIGNED(src, dst)) goto byte_at_a_time; @@ -114,6 +107,13 @@ long strncpy_from_user(char *dst, const char __user *src, long count) unsigned long max = max_addr - src_addr; long retval; + /* + * Truncate 'max' to the user-specified limit, so that + * we only have one limit we need to check in the loop + */ + if (max > count) + max = count; + kasan_check_write(dst, count); check_object_size(dst, count, false); if (user_access_begin(src, max)) { diff --git a/lib/strnlen_user.c b/lib/strnlen_user.c index 6c0005d5dd5c..41670d4a5816 100644 --- a/lib/strnlen_user.c +++ b/lib/strnlen_user.c @@ -27,13 +27,6 @@ static inline long do_strnlen_user(const char __user *src, unsigned long count, unsigned long c; /* - * Truncate 'max' to the user-specified limit, so that - * we only have one limit we need to check in the loop - */ - if (max > count) - max = count; - - /* * Do everything aligned. But that means that we * need to also expand the maximum.. */ @@ -109,6 +102,13 @@ long strnlen_user(const char __user *str, long count) unsigned long max = max_addr - src_addr; long retval; + /* + * Truncate 'max' to the user-specified limit, so that + * we only have one limit we need to check in the loop + */ + if (max > count) + max = count; + if (user_access_begin(str, max)) { retval = do_strnlen_user(str, count, max); user_access_end(); diff --git a/lib/test_xarray.c b/lib/test_xarray.c index 7df4f7f395bf..55c14e8c8859 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -2,6 +2,7 @@ /* * test_xarray.c: Test the XArray API * Copyright (c) 2017-2018 Microsoft Corporation + * Copyright (c) 2019-2020 Oracle * Author: Matthew Wilcox <willy@infradead.org> */ @@ -902,28 +903,34 @@ static noinline void check_store_iter(struct xarray *xa) XA_BUG_ON(xa, !xa_empty(xa)); } -static noinline void check_multi_find(struct xarray *xa) +static noinline void check_multi_find_1(struct xarray *xa, unsigned order) { #ifdef CONFIG_XARRAY_MULTI + unsigned long multi = 3 << order; + unsigned long next = 4 << order; unsigned long index; - xa_store_order(xa, 12, 2, xa_mk_value(12), GFP_KERNEL); - XA_BUG_ON(xa, xa_store_index(xa, 16, GFP_KERNEL) != NULL); + xa_store_order(xa, multi, order, xa_mk_value(multi), GFP_KERNEL); + XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL) != NULL); + XA_BUG_ON(xa, xa_store_index(xa, next + 1, GFP_KERNEL) != NULL); index = 0; XA_BUG_ON(xa, xa_find(xa, &index, ULONG_MAX, XA_PRESENT) != - xa_mk_value(12)); - XA_BUG_ON(xa, index != 12); - index = 13; + xa_mk_value(multi)); + XA_BUG_ON(xa, index != multi); + index = multi + 1; XA_BUG_ON(xa, xa_find(xa, &index, ULONG_MAX, XA_PRESENT) != - xa_mk_value(12)); - XA_BUG_ON(xa, (index < 12) || (index >= 16)); + xa_mk_value(multi)); + XA_BUG_ON(xa, (index < multi) || (index >= next)); XA_BUG_ON(xa, xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT) != - xa_mk_value(16)); - XA_BUG_ON(xa, index != 16); - - xa_erase_index(xa, 12); - xa_erase_index(xa, 16); + xa_mk_value(next)); + XA_BUG_ON(xa, index != next); + XA_BUG_ON(xa, xa_find_after(xa, &index, next, XA_PRESENT) != NULL); + XA_BUG_ON(xa, index != next); + + xa_erase_index(xa, multi); + xa_erase_index(xa, next); + xa_erase_index(xa, next + 1); XA_BUG_ON(xa, !xa_empty(xa)); #endif } @@ -1046,12 +1053,33 @@ static noinline void check_find_3(struct xarray *xa) xa_destroy(xa); } +static noinline void check_find_4(struct xarray *xa) +{ + unsigned long index = 0; + void *entry; + + xa_store_index(xa, ULONG_MAX, GFP_KERNEL); + + entry = xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT); + XA_BUG_ON(xa, entry != xa_mk_index(ULONG_MAX)); + + entry = xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT); + XA_BUG_ON(xa, entry); + + xa_erase_index(xa, ULONG_MAX); +} + static noinline void check_find(struct xarray *xa) { + unsigned i; + check_find_1(xa); check_find_2(xa); check_find_3(xa); - check_multi_find(xa); + check_find_4(xa); + + for (i = 2; i < 10; i++) + check_multi_find_1(xa, i); check_multi_find_2(xa); } @@ -1132,6 +1160,27 @@ static noinline void check_move_tiny(struct xarray *xa) XA_BUG_ON(xa, !xa_empty(xa)); } +static noinline void check_move_max(struct xarray *xa) +{ + XA_STATE(xas, xa, 0); + + xa_store_index(xa, ULONG_MAX, GFP_KERNEL); + rcu_read_lock(); + XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != xa_mk_index(ULONG_MAX)); + XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != NULL); + rcu_read_unlock(); + + xas_set(&xas, 0); + rcu_read_lock(); + XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != xa_mk_index(ULONG_MAX)); + xas_pause(&xas); + XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != NULL); + rcu_read_unlock(); + + xa_erase_index(xa, ULONG_MAX); + XA_BUG_ON(xa, !xa_empty(xa)); +} + static noinline void check_move_small(struct xarray *xa, unsigned long idx) { XA_STATE(xas, xa, 0); @@ -1240,6 +1289,7 @@ static noinline void check_move(struct xarray *xa) xa_destroy(xa); check_move_tiny(xa); + check_move_max(xa); for (i = 0; i < 16; i++) check_move_small(xa, 1UL << i); diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index 9ecfd3b547ba..42bd8ab955fa 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -221,6 +221,7 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) return 0; } +static __maybe_unused int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) { int ret = __cvdso_clock_getres_common(clock, res); diff --git a/lib/xarray.c b/lib/xarray.c index 1237c213f52b..1d9fab7db8da 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1,7 +1,8 @@ // SPDX-License-Identifier: GPL-2.0+ /* * XArray implementation - * Copyright (c) 2017 Microsoft Corporation + * Copyright (c) 2017-2018 Microsoft Corporation + * Copyright (c) 2018-2020 Oracle * Author: Matthew Wilcox <willy@infradead.org> */ @@ -967,6 +968,7 @@ void xas_pause(struct xa_state *xas) if (xas_invalid(xas)) return; + xas->xa_node = XAS_RESTART; if (node) { unsigned int offset = xas->xa_offset; while (++offset < XA_CHUNK_SIZE) { @@ -974,10 +976,11 @@ void xas_pause(struct xa_state *xas) break; } xas->xa_index += (offset - xas->xa_offset) << node->shift; + if (xas->xa_index == 0) + xas->xa_node = XAS_BOUNDS; } else { xas->xa_index++; } - xas->xa_node = XAS_RESTART; } EXPORT_SYMBOL_GPL(xas_pause); @@ -1079,13 +1082,15 @@ void *xas_find(struct xa_state *xas, unsigned long max) { void *entry; - if (xas_error(xas)) + if (xas_error(xas) || xas->xa_node == XAS_BOUNDS) return NULL; + if (xas->xa_index > max) + return set_bounds(xas); if (!xas->xa_node) { xas->xa_index = 1; return set_bounds(xas); - } else if (xas_top(xas->xa_node)) { + } else if (xas->xa_node == XAS_RESTART) { entry = xas_load(xas); if (entry || xas_not_node(xas->xa_node)) return entry; @@ -1150,6 +1155,8 @@ void *xas_find_marked(struct xa_state *xas, unsigned long max, xa_mark_t mark) if (xas_error(xas)) return NULL; + if (xas->xa_index > max) + goto max; if (!xas->xa_node) { xas->xa_index = 1; @@ -1824,6 +1831,17 @@ void *xa_find(struct xarray *xa, unsigned long *indexp, } EXPORT_SYMBOL(xa_find); +static bool xas_sibling(struct xa_state *xas) +{ + struct xa_node *node = xas->xa_node; + unsigned long mask; + + if (!node) + return false; + mask = (XA_CHUNK_SIZE << node->shift) - 1; + return (xas->xa_index & mask) > (xas->xa_offset << node->shift); +} + /** * xa_find_after() - Search the XArray for a present entry. * @xa: XArray. @@ -1847,21 +1865,20 @@ void *xa_find_after(struct xarray *xa, unsigned long *indexp, XA_STATE(xas, xa, *indexp + 1); void *entry; + if (xas.xa_index == 0) + return NULL; + rcu_read_lock(); for (;;) { if ((__force unsigned int)filter < XA_MAX_MARKS) entry = xas_find_marked(&xas, max, filter); else entry = xas_find(&xas, max); - if (xas.xa_node == XAS_BOUNDS) + + if (xas_invalid(&xas)) break; - if (xas.xa_shift) { - if (xas.xa_index & ((1UL << xas.xa_shift) - 1)) - continue; - } else { - if (xas.xa_offset < (xas.xa_index & XA_CHUNK_MASK)) - continue; - } + if (xas_sibling(&xas)) + continue; if (!xas_retry(&xas, entry)) break; } |