diff options
author | Stephen Rothwell <sfr@canb.auug.org.au> | 2009-10-06 15:02:30 +1100 |
---|---|---|
committer | Stephen Rothwell <sfr@canb.auug.org.au> | 2009-10-06 15:02:30 +1100 |
commit | a36d5708db41bc6d762e8ca200094340e0dffd55 (patch) | |
tree | 582b12b632ba89d58dd6a7d20be2739f60e560f1 | |
parent | 675317260aaa10fa14356e726f1e8cb3c2c2ee1a (diff) | |
parent | 3cbeae2f737fb8a02fca462bd7dd720378bdc11b (diff) |
Merge remote branch 'percpu/for-next'
30 files changed, 767 insertions, 552 deletions
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 1ee596cd942f..2d7f56a98e0f 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -87,9 +87,6 @@ config GENERIC_TIME_VSYSCALL bool default y -config HAVE_LEGACY_PER_CPU_AREA - def_bool y - config HAVE_SETUP_PER_CPU_AREA def_bool y diff --git a/arch/ia64/include/asm/meminit.h b/arch/ia64/include/asm/meminit.h index 688a812c017d..61c7b1750b16 100644 --- a/arch/ia64/include/asm/meminit.h +++ b/arch/ia64/include/asm/meminit.h @@ -61,7 +61,7 @@ extern int register_active_ranges(u64 start, u64 len, int nid); #ifdef CONFIG_VIRTUAL_MEM_MAP # define LARGE_GAP 0x40000000 /* Use virtual mem map if hole is > than this */ - extern unsigned long vmalloc_end; + extern unsigned long VMALLOC_END; extern struct page *vmem_map; extern int find_largest_hole(u64 start, u64 end, void *arg); extern int create_mem_map_page_table(u64 start, u64 end, void *arg); diff --git a/arch/ia64/include/asm/pgtable.h b/arch/ia64/include/asm/pgtable.h index 8840a690d1e7..69bf13857a9f 100644 --- a/arch/ia64/include/asm/pgtable.h +++ b/arch/ia64/include/asm/pgtable.h @@ -228,8 +228,7 @@ ia64_phys_addr_valid (unsigned long addr) #define VMALLOC_START (RGN_BASE(RGN_GATE) + 0x200000000UL) #ifdef CONFIG_VIRTUAL_MEM_MAP # define VMALLOC_END_INIT (RGN_BASE(RGN_GATE) + (1UL << (4*PAGE_SHIFT - 9))) -# define VMALLOC_END vmalloc_end - extern unsigned long vmalloc_end; +extern unsigned long VMALLOC_END; #else #if defined(CONFIG_SPARSEMEM) && defined(CONFIG_SPARSEMEM_VMEMMAP) /* SPARSEMEM_VMEMMAP uses half of vmalloc... */ diff --git a/arch/ia64/kernel/acpi.c b/arch/ia64/kernel/acpi.c index baec6f00f7f3..40574ae11401 100644 --- a/arch/ia64/kernel/acpi.c +++ b/arch/ia64/kernel/acpi.c @@ -702,11 +702,23 @@ int __init early_acpi_boot_init(void) printk(KERN_ERR PREFIX "Error parsing MADT - no LAPIC entries\n"); +#ifdef CONFIG_SMP + if (available_cpus == 0) { + printk(KERN_INFO "ACPI: Found 0 CPUS; assuming 1\n"); + printk(KERN_INFO "CPU 0 (0x%04x)", hard_smp_processor_id()); + smp_boot_data.cpu_phys_id[available_cpus] = + hard_smp_processor_id(); + available_cpus = 1; /* We've got at least one of these, no? */ + } + smp_boot_data.cpu_count = available_cpus; +#endif + /* Make boot-up look pretty */ + printk(KERN_INFO "%d CPUs available, %d CPUs total\n", available_cpus, + total_cpus); + return 0; } - - int __init acpi_boot_init(void) { @@ -769,18 +781,8 @@ int __init acpi_boot_init(void) if (acpi_table_parse(ACPI_SIG_FADT, acpi_parse_fadt)) printk(KERN_ERR PREFIX "Can't find FADT\n"); +#ifdef CONFIG_ACPI_NUMA #ifdef CONFIG_SMP - if (available_cpus == 0) { - printk(KERN_INFO "ACPI: Found 0 CPUS; assuming 1\n"); - printk(KERN_INFO "CPU 0 (0x%04x)", hard_smp_processor_id()); - smp_boot_data.cpu_phys_id[available_cpus] = - hard_smp_processor_id(); - available_cpus = 1; /* We've got at least one of these, no? */ - } - smp_boot_data.cpu_count = available_cpus; - - smp_build_cpu_map(); -# ifdef CONFIG_ACPI_NUMA if (srat_num_cpus == 0) { int cpu, i = 1; for (cpu = 0; cpu < smp_boot_data.cpu_count; cpu++) @@ -789,14 +791,9 @@ int __init acpi_boot_init(void) node_cpuid[i++].phys_id = smp_boot_data.cpu_phys_id[cpu]; } -# endif #endif -#ifdef CONFIG_ACPI_NUMA build_cpu_to_node_map(); #endif - /* Make boot-up look pretty */ - printk(KERN_INFO "%d CPUs available, %d CPUs total\n", available_cpus, - total_cpus); return 0; } diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c index 1de86c96801d..bc1ef4ae828c 100644 --- a/arch/ia64/kernel/setup.c +++ b/arch/ia64/kernel/setup.c @@ -566,19 +566,18 @@ setup_arch (char **cmdline_p) early_acpi_boot_init(); # ifdef CONFIG_ACPI_NUMA acpi_numa_init(); -#ifdef CONFIG_ACPI_HOTPLUG_CPU +# ifdef CONFIG_ACPI_HOTPLUG_CPU prefill_possible_map(); -#endif +# endif per_cpu_scan_finalize((cpus_weight(early_cpu_possible_map) == 0 ? 32 : cpus_weight(early_cpu_possible_map)), additional_cpus > 0 ? additional_cpus : 0); # endif -#else -# ifdef CONFIG_SMP - smp_build_cpu_map(); /* happens, e.g., with the Ski simulator */ -# endif #endif /* CONFIG_APCI_BOOT */ +#ifdef CONFIG_SMP + smp_build_cpu_map(); +#endif find_memory(); /* process SAL system table: */ @@ -856,18 +855,6 @@ identify_cpu (struct cpuinfo_ia64 *c) } /* - * In UP configuration, setup_per_cpu_areas() is defined in - * include/linux/percpu.h - */ -#ifdef CONFIG_SMP -void __init -setup_per_cpu_areas (void) -{ - /* start_kernel() requires this... */ -} -#endif - -/* * Do the following calculations: * * 1. the max. cache line size. diff --git a/arch/ia64/kernel/vmlinux.lds.S b/arch/ia64/kernel/vmlinux.lds.S index 0a0c77b2c988..1295ba327f6f 100644 --- a/arch/ia64/kernel/vmlinux.lds.S +++ b/arch/ia64/kernel/vmlinux.lds.S @@ -166,6 +166,12 @@ SECTIONS } #endif +#ifdef CONFIG_SMP + . = ALIGN(PERCPU_PAGE_SIZE); + __cpu0_per_cpu = .; + . = . + PERCPU_PAGE_SIZE; /* cpu0 per-cpu space */ +#endif + . = ALIGN(PAGE_SIZE); __init_end = .; @@ -198,11 +204,6 @@ SECTIONS data : { } :data .data : AT(ADDR(.data) - LOAD_OFFSET) { -#ifdef CONFIG_SMP - . = ALIGN(PERCPU_PAGE_SIZE); - __cpu0_per_cpu = .; - . = . + PERCPU_PAGE_SIZE; /* cpu0 per-cpu space */ -#endif INIT_TASK_DATA(PAGE_SIZE) CACHELINE_ALIGNED_DATA(SMP_CACHE_BYTES) READ_MOSTLY_DATA(SMP_CACHE_BYTES) diff --git a/arch/ia64/mm/contig.c b/arch/ia64/mm/contig.c index 2f724d2bf299..54bf54059811 100644 --- a/arch/ia64/mm/contig.c +++ b/arch/ia64/mm/contig.c @@ -154,38 +154,99 @@ static void *cpu_data; void * __cpuinit per_cpu_init (void) { - int cpu; - static int first_time=1; + static bool first_time = true; + void *cpu0_data = __cpu0_per_cpu; + unsigned int cpu; + + if (!first_time) + goto skip; + first_time = false; /* - * get_free_pages() cannot be used before cpu_init() done. BSP - * allocates "NR_CPUS" pages for all CPUs to avoid that AP calls - * get_zeroed_page(). + * get_free_pages() cannot be used before cpu_init() done. + * BSP allocates PERCPU_PAGE_SIZE bytes for all possible CPUs + * to avoid that AP calls get_zeroed_page(). */ - if (first_time) { - void *cpu0_data = __cpu0_per_cpu; + for_each_possible_cpu(cpu) { + void *src = cpu == 0 ? cpu0_data : __phys_per_cpu_start; - first_time=0; + memcpy(cpu_data, src, __per_cpu_end - __per_cpu_start); + __per_cpu_offset[cpu] = (char *)cpu_data - __per_cpu_start; + per_cpu(local_per_cpu_offset, cpu) = __per_cpu_offset[cpu]; - __per_cpu_offset[0] = (char *) cpu0_data - __per_cpu_start; - per_cpu(local_per_cpu_offset, 0) = __per_cpu_offset[0]; + /* + * percpu area for cpu0 is moved from the __init area + * which is setup by head.S and used till this point. + * Update ar.k3. This move is ensures that percpu + * area for cpu0 is on the correct node and its + * virtual address isn't insanely far from other + * percpu areas which is important for congruent + * percpu allocator. + */ + if (cpu == 0) + ia64_set_kr(IA64_KR_PER_CPU_DATA, __pa(cpu_data) - + (unsigned long)__per_cpu_start); - for (cpu = 1; cpu < NR_CPUS; cpu++) { - memcpy(cpu_data, __phys_per_cpu_start, __per_cpu_end - __per_cpu_start); - __per_cpu_offset[cpu] = (char *) cpu_data - __per_cpu_start; - cpu_data += PERCPU_PAGE_SIZE; - per_cpu(local_per_cpu_offset, cpu) = __per_cpu_offset[cpu]; - } + cpu_data += PERCPU_PAGE_SIZE; } +skip: return __per_cpu_start + __per_cpu_offset[smp_processor_id()]; } static inline void alloc_per_cpu_data(void) { - cpu_data = __alloc_bootmem(PERCPU_PAGE_SIZE * NR_CPUS-1, + cpu_data = __alloc_bootmem(PERCPU_PAGE_SIZE * num_possible_cpus(), PERCPU_PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); } + +/** + * setup_per_cpu_areas - setup percpu areas + * + * Arch code has already allocated and initialized percpu areas. All + * this function has to do is to teach the determined layout to the + * dynamic percpu allocator, which happens to be more complex than + * creating whole new ones using helpers. + */ +void __init +setup_per_cpu_areas(void) +{ + struct pcpu_alloc_info *ai; + struct pcpu_group_info *gi; + unsigned int cpu; + ssize_t static_size, reserved_size, dyn_size; + int rc; + + ai = pcpu_alloc_alloc_info(1, num_possible_cpus()); + if (!ai) + panic("failed to allocate pcpu_alloc_info"); + gi = &ai->groups[0]; + + /* units are assigned consecutively to possible cpus */ + for_each_possible_cpu(cpu) + gi->cpu_map[gi->nr_units++] = cpu; + + /* set parameters */ + static_size = __per_cpu_end - __per_cpu_start; + reserved_size = PERCPU_MODULE_RESERVE; + dyn_size = PERCPU_PAGE_SIZE - static_size - reserved_size; + if (dyn_size < 0) + panic("percpu area overflow static=%zd reserved=%zd\n", + static_size, reserved_size); + + ai->static_size = static_size; + ai->reserved_size = reserved_size; + ai->dyn_size = dyn_size; + ai->unit_size = PERCPU_PAGE_SIZE; + ai->atom_size = PAGE_SIZE; + ai->alloc_size = PERCPU_PAGE_SIZE; + + rc = pcpu_setup_first_chunk(ai, __per_cpu_start + __per_cpu_offset[0]); + if (rc) + panic("failed to setup percpu area (err=%d)", rc); + + pcpu_free_alloc_info(ai); +} #else #define alloc_per_cpu_data() do { } while (0) #endif /* CONFIG_SMP */ @@ -270,8 +331,8 @@ paging_init (void) map_size = PAGE_ALIGN(ALIGN(max_low_pfn, MAX_ORDER_NR_PAGES) * sizeof(struct page)); - vmalloc_end -= map_size; - vmem_map = (struct page *) vmalloc_end; + VMALLOC_END -= map_size; + vmem_map = (struct page *) VMALLOC_END; efi_memmap_walk(create_mem_map_page_table, NULL); /* diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c index d85ba98d9008..40e4c1fbf76b 100644 --- a/arch/ia64/mm/discontig.c +++ b/arch/ia64/mm/discontig.c @@ -143,22 +143,120 @@ static void *per_cpu_node_setup(void *cpu_data, int node) int cpu; for_each_possible_early_cpu(cpu) { - if (cpu == 0) { - void *cpu0_data = __cpu0_per_cpu; - __per_cpu_offset[cpu] = (char*)cpu0_data - - __per_cpu_start; - } else if (node == node_cpuid[cpu].nid) { - memcpy(__va(cpu_data), __phys_per_cpu_start, - __per_cpu_end - __per_cpu_start); - __per_cpu_offset[cpu] = (char*)__va(cpu_data) - - __per_cpu_start; - cpu_data += PERCPU_PAGE_SIZE; - } + void *src = cpu == 0 ? __cpu0_per_cpu : __phys_per_cpu_start; + + if (node != node_cpuid[cpu].nid) + continue; + + memcpy(__va(cpu_data), src, __per_cpu_end - __per_cpu_start); + __per_cpu_offset[cpu] = (char *)__va(cpu_data) - + __per_cpu_start; + + /* + * percpu area for cpu0 is moved from the __init area + * which is setup by head.S and used till this point. + * Update ar.k3. This move is ensures that percpu + * area for cpu0 is on the correct node and its + * virtual address isn't insanely far from other + * percpu areas which is important for congruent + * percpu allocator. + */ + if (cpu == 0) + ia64_set_kr(IA64_KR_PER_CPU_DATA, + (unsigned long)cpu_data - + (unsigned long)__per_cpu_start); + + cpu_data += PERCPU_PAGE_SIZE; } #endif return cpu_data; } +#ifdef CONFIG_SMP +/** + * setup_per_cpu_areas - setup percpu areas + * + * Arch code has already allocated and initialized percpu areas. All + * this function has to do is to teach the determined layout to the + * dynamic percpu allocator, which happens to be more complex than + * creating whole new ones using helpers. + */ +void __init setup_per_cpu_areas(void) +{ + struct pcpu_alloc_info *ai; + struct pcpu_group_info *uninitialized_var(gi); + unsigned int *cpu_map; + void *base; + unsigned long base_offset; + unsigned int cpu; + ssize_t static_size, reserved_size, dyn_size; + int node, prev_node, unit, nr_units, rc; + + ai = pcpu_alloc_alloc_info(MAX_NUMNODES, nr_cpu_ids); + if (!ai) + panic("failed to allocate pcpu_alloc_info"); + cpu_map = ai->groups[0].cpu_map; + + /* determine base */ + base = (void *)ULONG_MAX; + for_each_possible_cpu(cpu) + base = min(base, + (void *)(__per_cpu_offset[cpu] + __per_cpu_start)); + base_offset = (void *)__per_cpu_start - base; + + /* build cpu_map, units are grouped by node */ + unit = 0; + for_each_node(node) + for_each_possible_cpu(cpu) + if (node == node_cpuid[cpu].nid) + cpu_map[unit++] = cpu; + nr_units = unit; + + /* set basic parameters */ + static_size = __per_cpu_end - __per_cpu_start; + reserved_size = PERCPU_MODULE_RESERVE; + dyn_size = PERCPU_PAGE_SIZE - static_size - reserved_size; + if (dyn_size < 0) + panic("percpu area overflow static=%zd reserved=%zd\n", + static_size, reserved_size); + + ai->static_size = static_size; + ai->reserved_size = reserved_size; + ai->dyn_size = dyn_size; + ai->unit_size = PERCPU_PAGE_SIZE; + ai->atom_size = PAGE_SIZE; + ai->alloc_size = PERCPU_PAGE_SIZE; + + /* + * CPUs are put into groups according to node. Walk cpu_map + * and create new groups at node boundaries. + */ + prev_node = -1; + ai->nr_groups = 0; + for (unit = 0; unit < nr_units; unit++) { + cpu = cpu_map[unit]; + node = node_cpuid[cpu].nid; + + if (node == prev_node) { + gi->nr_units++; + continue; + } + prev_node = node; + + gi = &ai->groups[ai->nr_groups++]; + gi->nr_units = 1; + gi->base_offset = __per_cpu_offset[cpu] + base_offset; + gi->cpu_map = &cpu_map[unit]; + } + + rc = pcpu_setup_first_chunk(ai, base); + if (rc) + panic("failed to setup percpu area (err=%d)", rc); + + pcpu_free_alloc_info(ai); +} +#endif + /** * fill_pernode - initialize pernode data. * @node: the node id. @@ -666,9 +764,9 @@ void __init paging_init(void) sparse_init(); #ifdef CONFIG_VIRTUAL_MEM_MAP - vmalloc_end -= PAGE_ALIGN(ALIGN(max_low_pfn, MAX_ORDER_NR_PAGES) * + VMALLOC_END -= PAGE_ALIGN(ALIGN(max_low_pfn, MAX_ORDER_NR_PAGES) * sizeof(struct page)); - vmem_map = (struct page *) vmalloc_end; + vmem_map = (struct page *) VMALLOC_END; efi_memmap_walk(create_mem_map_page_table, NULL); printk("Virtual mem_map starts at 0x%p\n", vmem_map); #endif diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index 1857766a63c1..b9609c69343a 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -44,8 +44,8 @@ extern void ia64_tlb_init (void); unsigned long MAX_DMA_ADDRESS = PAGE_OFFSET + 0x100000000UL; #ifdef CONFIG_VIRTUAL_MEM_MAP -unsigned long vmalloc_end = VMALLOC_END_INIT; -EXPORT_SYMBOL(vmalloc_end); +unsigned long VMALLOC_END = VMALLOC_END_INIT; +EXPORT_SYMBOL(VMALLOC_END); struct page *vmem_map; EXPORT_SYMBOL(vmem_map); #endif diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index b65a36defeb7..8b5ec19bdef4 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -153,6 +153,84 @@ do { \ #define percpu_or(var, val) percpu_to_op("or", per_cpu__##var, val) #define percpu_xor(var, val) percpu_to_op("xor", per_cpu__##var, val) +#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define __this_cpu_read_4(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) + +#define __this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) +#define __this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) +#define __this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) +#define __this_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val) +#define __this_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val) +#define __this_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val) +#define __this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) +#define __this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) +#define __this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) +#define __this_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val) +#define __this_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val) +#define __this_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val) +#define __this_cpu_xor_1(pcp, val) percpu_to_op("xor", (pcp), val) +#define __this_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val) +#define __this_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val) + +#define this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define this_cpu_read_4(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) +#define this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) +#define this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) +#define this_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val) +#define this_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val) +#define this_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val) +#define this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) +#define this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) +#define this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) +#define this_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val) +#define this_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val) +#define this_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val) +#define this_cpu_xor_1(pcp, val) percpu_to_op("xor", (pcp), val) +#define this_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val) +#define this_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val) + +#define irqsafe_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val) +#define irqsafe_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val) +#define irqsafe_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val) +#define irqsafe_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) +#define irqsafe_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) +#define irqsafe_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) +#define irqsafe_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val) +#define irqsafe_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val) +#define irqsafe_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val) +#define irqsafe_cpu_xor_1(pcp, val) percpu_to_op("xor", (pcp), val) +#define irqsafe_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val) +#define irqsafe_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val) + +/* + * Per cpu atomic 64 bit operations are only available under 64 bit. + * 32 bit must fall back to generic operations. + */ +#ifdef CONFIG_X86_64 +#define __this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define __this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) +#define __this_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val) +#define __this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) +#define __this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) +#define __this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) + +#define this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) +#define this_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val) +#define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) +#define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) +#define this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) + +#define irqsafe_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val) +#define irqsafe_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) +#define irqsafe_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) +#define irqsafe_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) + +#endif + /* This is not atomic against other CPUs -- CPU preemption needs to be off */ #define x86_test_and_clear_bit_percpu(bit, var) \ ({ \ diff --git a/crypto/cryptd.c b/crypto/cryptd.c index 35335825a4ef..3d7fe8306e2a 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -99,7 +99,7 @@ static int cryptd_enqueue_request(struct cryptd_queue *queue, struct cryptd_cpu_queue *cpu_queue; cpu = get_cpu(); - cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); + cpu_queue = this_cpu_ptr(queue->cpu_queue); err = crypto_enqueue_request(&cpu_queue->queue, request); queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); put_cpu(); diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index bd0b248de2cf..51d7480d3a92 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -326,14 +326,7 @@ arch_initcall(dma_channel_table_init); */ struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type) { - struct dma_chan *chan; - int cpu; - - cpu = get_cpu(); - chan = per_cpu_ptr(channel_table[tx_type], cpu)->chan; - put_cpu(); - - return chan; + return this_cpu_read(channel_table[tx_type]->chan); } EXPORT_SYMBOL(dma_find_channel); @@ -847,7 +840,6 @@ dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest, struct dma_async_tx_descriptor *tx; dma_addr_t dma_dest, dma_src; dma_cookie_t cookie; - int cpu; unsigned long flags; dma_src = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE); @@ -866,10 +858,10 @@ dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest, tx->callback = NULL; cookie = tx->tx_submit(tx); - cpu = get_cpu(); - per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; - per_cpu_ptr(chan->local, cpu)->memcpy_count++; - put_cpu(); + preempt_disable(); + __this_cpu_add(chan->local->bytes_transferred, len); + __this_cpu_inc(chan->local->memcpy_count); + preempt_enable(); return cookie; } @@ -896,7 +888,6 @@ dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page, struct dma_async_tx_descriptor *tx; dma_addr_t dma_dest, dma_src; dma_cookie_t cookie; - int cpu; unsigned long flags; dma_src = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE); @@ -913,10 +904,10 @@ dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page, tx->callback = NULL; cookie = tx->tx_submit(tx); - cpu = get_cpu(); - per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; - per_cpu_ptr(chan->local, cpu)->memcpy_count++; - put_cpu(); + preempt_disable(); + __this_cpu_add(chan->local->bytes_transferred, len); + __this_cpu_inc(chan->local->memcpy_count); + preempt_enable(); return cookie; } @@ -945,7 +936,6 @@ dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg, struct dma_async_tx_descriptor *tx; dma_addr_t dma_dest, dma_src; dma_cookie_t cookie; - int cpu; unsigned long flags; dma_src = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE); @@ -963,10 +953,10 @@ dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg, tx->callback = NULL; cookie = tx->tx_submit(tx); - cpu = get_cpu(); - per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; - per_cpu_ptr(chan->local, cpu)->memcpy_count++; - put_cpu(); + preempt_disable(); + __this_cpu_add(chan->local->bytes_transferred, len); + __this_cpu_inc(chan->local->memcpy_count); + preempt_enable(); return cookie; } diff --git a/drivers/infiniband/hw/ehca/ehca_irq.c b/drivers/infiniband/hw/ehca/ehca_irq.c index 4b89b791be6a..42be0b15084b 100644 --- a/drivers/infiniband/hw/ehca/ehca_irq.c +++ b/drivers/infiniband/hw/ehca/ehca_irq.c @@ -826,8 +826,7 @@ static void __cpuinit take_over_work(struct ehca_comp_pool *pool, int cpu) cq = list_entry(cct->cq_list.next, struct ehca_cq, entry); list_del(&cq->entry); - __queue_comp_task(cq, per_cpu_ptr(pool->cpu_comp_tasks, - smp_processor_id())); + __queue_comp_task(cq, this_cpu_ptr(pool->cpu_comp_tasks)); } spin_unlock_irqrestore(&cct->task_lock, flags_cct); diff --git a/drivers/net/chelsio/sge.c b/drivers/net/chelsio/sge.c index 8c658cf6f62f..109d2783e4d8 100644 --- a/drivers/net/chelsio/sge.c +++ b/drivers/net/chelsio/sge.c @@ -1378,7 +1378,7 @@ static void sge_rx(struct sge *sge, struct freelQ *fl, unsigned int len) } __skb_pull(skb, sizeof(*p)); - st = per_cpu_ptr(sge->port_stats[p->iff], smp_processor_id()); + st = this_cpu_ptr(sge->port_stats[p->iff]); skb->protocol = eth_type_trans(skb, adapter->port[p->iff].dev); if ((adapter->flags & RX_CSUM_ENABLED) && p->csum == 0xffff && @@ -1780,8 +1780,7 @@ netdev_tx_t t1_start_xmit(struct sk_buff *skb, struct net_device *dev) { struct adapter *adapter = dev->ml_priv; struct sge *sge = adapter->sge; - struct sge_port_stats *st = per_cpu_ptr(sge->port_stats[dev->if_port], - smp_processor_id()); + struct sge_port_stats *st = this_cpu_ptr(sge->port_stats[dev->if_port]); struct cpl_tx_pkt *cpl; struct sk_buff *orig_skb = skb; int ret; diff --git a/drivers/net/loopback.c b/drivers/net/loopback.c index 1bc654a73c47..8ebeb76a373d 100644 --- a/drivers/net/loopback.c +++ b/drivers/net/loopback.c @@ -81,7 +81,7 @@ static netdev_tx_t loopback_xmit(struct sk_buff *skb, /* it's OK to use per_cpu_ptr() because BHs are off */ pcpu_lstats = dev->ml_priv; - lb_stats = per_cpu_ptr(pcpu_lstats, smp_processor_id()); + lb_stats = this_cpu_ptr(pcpu_lstats); len = skb->len; if (likely(netif_rx(skb) == NET_RX_SUCCESS)) { diff --git a/drivers/net/veth.c b/drivers/net/veth.c index ade5b344f75d..0c4a81124257 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -153,7 +153,7 @@ static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev) struct net_device *rcv = NULL; struct veth_priv *priv, *rcv_priv; struct veth_net_stats *stats, *rcv_stats; - int length, cpu; + int length; skb_orphan(skb); @@ -161,9 +161,8 @@ static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev) rcv = priv->peer; rcv_priv = netdev_priv(rcv); - cpu = smp_processor_id(); - stats = per_cpu_ptr(priv->stats, cpu); - rcv_stats = per_cpu_ptr(rcv_priv->stats, cpu); + stats = this_cpu_ptr(priv->stats); + rcv_stats = this_cpu_ptr(rcv_priv->stats); if (!(rcv->flags & IFF_UP)) goto tx_drop; diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index bba12824defa..d527fd384582 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -3932,7 +3932,7 @@ static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) * per cpu locality group is to reduce the contention between block * request from multiple CPUs. */ - ac->ac_lg = per_cpu_ptr(sbi->s_locality_groups, raw_smp_processor_id()); + ac->ac_lg = __this_cpu_ptr(sbi->s_locality_groups); /* we're going to use group allocation */ ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; diff --git a/fs/nfs/iostat.h b/fs/nfs/iostat.h index ceda50aad73c..46d779abafd3 100644 --- a/fs/nfs/iostat.h +++ b/fs/nfs/iostat.h @@ -25,13 +25,7 @@ struct nfs_iostats { static inline void nfs_inc_server_stats(const struct nfs_server *server, enum nfs_stat_eventcounters stat) { - struct nfs_iostats *iostats; - int cpu; - - cpu = get_cpu(); - iostats = per_cpu_ptr(server->io_stats, cpu); - iostats->events[stat]++; - put_cpu(); + this_cpu_inc(server->io_stats->events[stat]); } static inline void nfs_inc_stats(const struct inode *inode, @@ -44,13 +38,7 @@ static inline void nfs_add_server_stats(const struct nfs_server *server, enum nfs_stat_bytecounters stat, unsigned long addend) { - struct nfs_iostats *iostats; - int cpu; - - cpu = get_cpu(); - iostats = per_cpu_ptr(server->io_stats, cpu); - iostats->bytes[stat] += addend; - put_cpu(); + this_cpu_add(server->io_stats->bytes[stat], addend); } static inline void nfs_add_stats(const struct inode *inode, @@ -65,13 +53,7 @@ static inline void nfs_add_fscache_stats(struct inode *inode, enum nfs_stat_fscachecounters stat, unsigned long addend) { - struct nfs_iostats *iostats; - int cpu; - - cpu = get_cpu(); - iostats = per_cpu_ptr(NFS_SERVER(inode)->io_stats, cpu); - iostats->fscache[stat] += addend; - put_cpu(); + this_cpu_add(NFS_SERVER(inode)->io_stats->fscache[stat], addend); } #endif diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index 8b6c9e807efb..ccafe8ef7ad5 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -2389,12 +2389,12 @@ xfs_icsb_modify_counters( { xfs_icsb_cnts_t *icsbp; long long lcounter; /* long counter for 64 bit fields */ - int cpu, ret = 0; + int ret = 0; might_sleep(); again: - cpu = get_cpu(); - icsbp = (xfs_icsb_cnts_t *)per_cpu_ptr(mp->m_sb_cnts, cpu); + preempt_disable(); + icsbp = this_cpu_ptr(mp->m_sb_cnts); /* * if the counter is disabled, go to slow path @@ -2438,11 +2438,11 @@ again: break; } xfs_icsb_unlock_cntr(icsbp); - put_cpu(); + preempt_enable(); return 0; slow_path: - put_cpu(); + preempt_enable(); /* * serialise with a mutex so we don't burn lots of cpu on @@ -2490,7 +2490,7 @@ slow_path: balance_counter: xfs_icsb_unlock_cntr(icsbp); - put_cpu(); + preempt_enable(); /* * We may have multiple threads here if multiple per-cpu diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h index 90079c373f1c..8087b90d4673 100644 --- a/include/asm-generic/percpu.h +++ b/include/asm-generic/percpu.h @@ -56,6 +56,9 @@ extern unsigned long __per_cpu_offset[NR_CPUS]; #define __raw_get_cpu_var(var) \ (*SHIFT_PERCPU_PTR(&per_cpu_var(var), __my_cpu_offset)) +#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset) +#define __this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset) + #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA extern void setup_per_cpu_areas(void); @@ -66,6 +69,8 @@ extern void setup_per_cpu_areas(void); #define per_cpu(var, cpu) (*((void)(cpu), &per_cpu_var(var))) #define __get_cpu_var(var) per_cpu_var(var) #define __raw_get_cpu_var(var) per_cpu_var(var) +#define this_cpu_ptr(ptr) per_cpu_ptr(ptr, 0) +#define __this_cpu_ptr(ptr) this_cpu_ptr(ptr) #endif /* SMP */ diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 878836ca999c..3d9ba92b104f 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -34,8 +34,6 @@ #ifdef CONFIG_SMP -#ifndef CONFIG_HAVE_LEGACY_PER_CPU_AREA - /* minimum unit size, also is the maximum supported allocation size */ #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(64 << 10) @@ -130,28 +128,6 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size, #define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))) extern void *__alloc_reserved_percpu(size_t size, size_t align); - -#else /* CONFIG_HAVE_LEGACY_PER_CPU_AREA */ - -struct percpu_data { - void *ptrs[1]; -}; - -/* pointer disguising messes up the kmemleak objects tracking */ -#ifndef CONFIG_DEBUG_KMEMLEAK -#define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata) -#else -#define __percpu_disguise(pdata) (struct percpu_data *)(pdata) -#endif - -#define per_cpu_ptr(ptr, cpu) \ -({ \ - struct percpu_data *__p = __percpu_disguise(ptr); \ - (__typeof__(ptr))__p->ptrs[(cpu)]; \ -}) - -#endif /* CONFIG_HAVE_LEGACY_PER_CPU_AREA */ - extern void *__alloc_percpu(size_t size, size_t align); extern void free_percpu(void *__pdata); @@ -243,4 +219,404 @@ do { \ # define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=) #endif +/* + * Branching function to split up a function into a set of functions that + * are called for different scalar sizes of the objects handled. + */ + +extern void __bad_size_call_parameter(void); + +#define __size_call_return(stem, variable) \ +({ typeof(variable) ret__; \ + switch(sizeof(variable)) { \ + case 1: ret__ = stem##1(variable);break; \ + case 2: ret__ = stem##2(variable);break; \ + case 4: ret__ = stem##4(variable);break; \ + case 8: ret__ = stem##8(variable);break; \ + default: \ + __bad_size_call_parameter();break; \ + } \ + ret__; \ +}) + +#define __size_call(stem, variable, ...) \ +do { \ + switch(sizeof(variable)) { \ + case 1: stem##1(variable, __VA_ARGS__);break; \ + case 2: stem##2(variable, __VA_ARGS__);break; \ + case 4: stem##4(variable, __VA_ARGS__);break; \ + case 8: stem##8(variable, __VA_ARGS__);break; \ + default: \ + __bad_size_call_parameter();break; \ + } \ +} while (0) + +/* + * Optimized manipulation for memory allocated through the per cpu + * allocator or for addresses of per cpu variables (can be determined + * using per_cpu_var(xx). + * + * These operation guarantee exclusivity of access for other operations + * on the *same* processor. The assumption is that per cpu data is only + * accessed by a single processor instance (the current one). + * + * The first group is used for accesses that must be done in a + * preemption safe way since we know that the context is not preempt + * safe. Interrupts may occur. If the interrupt modifies the variable + * too then RMW actions will not be reliable. + * + * The arch code can provide optimized functions in two ways: + * + * 1. Override the function completely. F.e. define this_cpu_add(). + * The arch must then ensure that the various scalar format passed + * are handled correctly. + * + * 2. Provide functions for certain scalar sizes. F.e. provide + * this_cpu_add_2() to provide per cpu atomic operations for 2 byte + * sized RMW actions. If arch code does not provide operations for + * a scalar size then the fallback in the generic code will be + * used. + */ + +#define _this_cpu_generic_read(pcp) \ +({ typeof(pcp) ret__; \ + preempt_disable(); \ + ret__ = *this_cpu_ptr(&(pcp)); \ + preempt_enable(); \ + ret__; \ +}) + +#ifndef this_cpu_read +# ifndef this_cpu_read_1 +# define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp) +# endif +# ifndef this_cpu_read_2 +# define this_cpu_read_2(pcp) _this_cpu_generic_read(pcp) +# endif +# ifndef this_cpu_read_4 +# define this_cpu_read_4(pcp) _this_cpu_generic_read(pcp) +# endif +# ifndef this_cpu_read_8 +# define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp) +# endif +# define this_cpu_read(pcp) __size_call_return(this_cpu_read_, (pcp)) +#endif + +#define _this_cpu_generic_to_op(pcp, val, op) \ +do { \ + preempt_disable(); \ + *__this_cpu_ptr(&pcp) op val; \ + preempt_enable(); \ +} while (0) + +#ifndef this_cpu_write +# ifndef this_cpu_write_1 +# define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) +# endif +# ifndef this_cpu_write_2 +# define this_cpu_write_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) +# endif +# ifndef this_cpu_write_4 +# define this_cpu_write_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) +# endif +# ifndef this_cpu_write_8 +# define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) +# endif +# define this_cpu_write(pcp, val) __size_call(this_cpu_write_, (pcp), (val)) +#endif + +#ifndef this_cpu_add +# ifndef this_cpu_add_1 +# define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) +# endif +# ifndef this_cpu_add_2 +# define this_cpu_add_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) +# endif +# ifndef this_cpu_add_4 +# define this_cpu_add_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) +# endif +# ifndef this_cpu_add_8 +# define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) +# endif +# define this_cpu_add(pcp, val) __size_call(this_cpu_add_, (pcp), (val)) +#endif + +#ifndef this_cpu_sub +# define this_cpu_sub(pcp, val) this_cpu_add((pcp), -(val)) +#endif + +#ifndef this_cpu_inc +# define this_cpu_inc(pcp) this_cpu_add((pcp), 1) +#endif + +#ifndef this_cpu_dec +# define this_cpu_dec(pcp) this_cpu_sub((pcp), 1) +#endif + +#ifndef this_cpu_and +# ifndef this_cpu_and_1 +# define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) +# endif +# ifndef this_cpu_and_2 +# define this_cpu_and_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) +# endif +# ifndef this_cpu_and_4 +# define this_cpu_and_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) +# endif +# ifndef this_cpu_and_8 +# define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) +# endif +# define this_cpu_and(pcp, val) __size_call(this_cpu_and_, (pcp), (val)) +#endif + +#ifndef this_cpu_or +# ifndef this_cpu_or_1 +# define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) +# endif +# ifndef this_cpu_or_2 +# define this_cpu_or_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) +# endif +# ifndef this_cpu_or_4 +# define this_cpu_or_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) +# endif +# ifndef this_cpu_or_8 +# define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) +# endif +# define this_cpu_or(pcp, val) __size_call(this_cpu_or_, (pcp), (val)) +#endif + +#ifndef this_cpu_xor +# ifndef this_cpu_xor_1 +# define this_cpu_xor_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=) +# endif +# ifndef this_cpu_xor_2 +# define this_cpu_xor_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=) +# endif +# ifndef this_cpu_xor_4 +# define this_cpu_xor_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=) +# endif +# ifndef this_cpu_xor_8 +# define this_cpu_xor_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=) +# endif +# define this_cpu_xor(pcp, val) __size_call(this_cpu_or_, (pcp), (val)) +#endif + +/* + * Generic percpu operations that do not require preemption handling. + * Either we do not care about races or the caller has the + * responsibility of handling preemptions issues. Arch code can still + * override these instructions since the arch per cpu code may be more + * efficient and may actually get race freeness for free (that is the + * case for x86 for example). + * + * If there is no other protection through preempt disable and/or + * disabling interupts then one of these RMW operations can show unexpected + * behavior because the execution thread was rescheduled on another processor + * or an interrupt occurred and the same percpu variable was modified from + * the interrupt context. + */ +#ifndef __this_cpu_read +# ifndef __this_cpu_read_1 +# define __this_cpu_read_1(pcp) (*__this_cpu_ptr(&(pcp))) +# endif +# ifndef __this_cpu_read_2 +# define __this_cpu_read_2(pcp) (*__this_cpu_ptr(&(pcp))) +# endif +# ifndef __this_cpu_read_4 +# define __this_cpu_read_4(pcp) (*__this_cpu_ptr(&(pcp))) +# endif +# ifndef __this_cpu_read_8 +# define __this_cpu_read_8(pcp) (*__this_cpu_ptr(&(pcp))) +# endif +# define __this_cpu_read(pcp) __size_call_return(__this_cpu_read_, (pcp)) +#endif + +#define __this_cpu_generic_to_op(pcp, val, op) \ +do { \ + *__this_cpu_ptr(&(pcp)) op val; \ +} while (0) + +#ifndef __this_cpu_write +# ifndef __this_cpu_write_1 +# define __this_cpu_write_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), =) +# endif +# ifndef __this_cpu_write_2 +# define __this_cpu_write_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), =) +# endif +# ifndef __this_cpu_write_4 +# define __this_cpu_write_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), =) +# endif +# ifndef __this_cpu_write_8 +# define __this_cpu_write_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), =) +# endif +# define __this_cpu_write(pcp, val) __size_call(__this_cpu_write_, (pcp), (val)) +#endif + +#ifndef __this_cpu_add +# ifndef __this_cpu_add_1 +# define __this_cpu_add_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) +# endif +# ifndef __this_cpu_add_2 +# define __this_cpu_add_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) +# endif +# ifndef __this_cpu_add_4 +# define __this_cpu_add_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) +# endif +# ifndef __this_cpu_add_8 +# define __this_cpu_add_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) +# endif +# define __this_cpu_add(pcp, val) __size_call(__this_cpu_add_, (pcp), (val)) +#endif + +#ifndef __this_cpu_sub +# define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(val)) +#endif + +#ifndef __this_cpu_inc +# define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1) +#endif + +#ifndef __this_cpu_dec +# define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1) +#endif + +#ifndef __this_cpu_and +# ifndef __this_cpu_and_1 +# define __this_cpu_and_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) +# endif +# ifndef __this_cpu_and_2 +# define __this_cpu_and_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) +# endif +# ifndef __this_cpu_and_4 +# define __this_cpu_and_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) +# endif +# ifndef __this_cpu_and_8 +# define __this_cpu_and_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) +# endif +# define __this_cpu_and(pcp, val) __size_call(__this_cpu_and_, (pcp), (val)) +#endif + +#ifndef __this_cpu_or +# ifndef __this_cpu_or_1 +# define __this_cpu_or_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) +# endif +# ifndef __this_cpu_or_2 +# define __this_cpu_or_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) +# endif +# ifndef __this_cpu_or_4 +# define __this_cpu_or_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) +# endif +# ifndef __this_cpu_or_8 +# define __this_cpu_or_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) +# endif +# define __this_cpu_or(pcp, val) __size_call(__this_cpu_or_, (pcp), (val)) +#endif + +#ifndef __this_cpu_xor +# ifndef __this_cpu_xor_1 +# define __this_cpu_xor_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=) +# endif +# ifndef __this_cpu_xor_2 +# define __this_cpu_xor_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=) +# endif +# ifndef __this_cpu_xor_4 +# define __this_cpu_xor_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=) +# endif +# ifndef __this_cpu_xor_8 +# define __this_cpu_xor_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=) +# endif +# define __this_cpu_xor(pcp, val) __size_call(__this_cpu_xor_, (pcp), (val)) +#endif + +/* + * IRQ safe versions of the per cpu RMW operations. Note that these operations + * are *not* safe against modification of the same variable from another + * processors (which one gets when using regular atomic operations) + . They are guaranteed to be atomic vs. local interrupts and + * preemption only. + */ +#define irqsafe_cpu_generic_to_op(pcp, val, op) \ +do { \ + unsigned long flags; \ + local_irq_save(flags); \ + *__this_cpu_ptr(&(pcp)) op val; \ + local_irq_restore(flags); \ +} while (0) + +#ifndef irqsafe_cpu_add +# ifndef irqsafe_cpu_add_1 +# define irqsafe_cpu_add_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=) +# endif +# ifndef irqsafe_cpu_add_2 +# define irqsafe_cpu_add_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=) +# endif +# ifndef irqsafe_cpu_add_4 +# define irqsafe_cpu_add_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=) +# endif +# ifndef irqsafe_cpu_add_8 +# define irqsafe_cpu_add_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=) +# endif +# define irqsafe_cpu_add(pcp, val) __size_call(irqsafe_cpu_add_, (pcp), (val)) +#endif + +#ifndef irqsafe_cpu_sub +# define irqsafe_cpu_sub(pcp, val) irqsafe_cpu_add((pcp), -(val)) +#endif + +#ifndef irqsafe_cpu_inc +# define irqsafe_cpu_inc(pcp) irqsafe_cpu_add((pcp), 1) +#endif + +#ifndef irqsafe_cpu_dec +# define irqsafe_cpu_dec(pcp) irqsafe_cpu_sub((pcp), 1) +#endif + +#ifndef irqsafe_cpu_and +# ifndef irqsafe_cpu_and_1 +# define irqsafe_cpu_and_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=) +# endif +# ifndef irqsafe_cpu_and_2 +# define irqsafe_cpu_and_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=) +# endif +# ifndef irqsafe_cpu_and_4 +# define irqsafe_cpu_and_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=) +# endif +# ifndef irqsafe_cpu_and_8 +# define irqsafe_cpu_and_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=) +# endif +# define irqsafe_cpu_and(pcp, val) __size_call(irqsafe_cpu_and_, (val)) +#endif + +#ifndef irqsafe_cpu_or +# ifndef irqsafe_cpu_or_1 +# define irqsafe_cpu_or_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=) +# endif +# ifndef irqsafe_cpu_or_2 +# define irqsafe_cpu_or_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=) +# endif +# ifndef irqsafe_cpu_or_4 +# define irqsafe_cpu_or_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=) +# endif +# ifndef irqsafe_cpu_or_8 +# define irqsafe_cpu_or_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=) +# endif +# define irqsafe_cpu_or(pcp, val) __size_call(irqsafe_cpu_or_, (val)) +#endif + +#ifndef irqsafe_cpu_xor +# ifndef irqsafe_cpu_xor_1 +# define irqsafe_cpu_xor_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=) +# endif +# ifndef irqsafe_cpu_xor_2 +# define irqsafe_cpu_xor_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=) +# endif +# ifndef irqsafe_cpu_xor_4 +# define irqsafe_cpu_xor_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=) +# endif +# ifndef irqsafe_cpu_xor_8 +# define irqsafe_cpu_xor_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=) +# endif +# define irqsafe_cpu_xor(pcp, val) __size_call(irqsafe_cpu_xor_, (val)) +#endif + #endif /* __LINUX_PERCPU_H */ diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h index 2d0f222388a8..d85889710f9b 100644 --- a/include/linux/vmstat.h +++ b/include/linux/vmstat.h @@ -76,24 +76,22 @@ DECLARE_PER_CPU(struct vm_event_state, vm_event_states); static inline void __count_vm_event(enum vm_event_item item) { - __get_cpu_var(vm_event_states).event[item]++; + __this_cpu_inc(per_cpu_var(vm_event_states).event[item]); } static inline void count_vm_event(enum vm_event_item item) { - get_cpu_var(vm_event_states).event[item]++; - put_cpu(); + this_cpu_inc(per_cpu_var(vm_event_states).event[item]); } static inline void __count_vm_events(enum vm_event_item item, long delta) { - __get_cpu_var(vm_event_states).event[item] += delta; + __this_cpu_add(per_cpu_var(vm_event_states).event[item], delta); } static inline void count_vm_events(enum vm_event_item item, long delta) { - get_cpu_var(vm_event_states).event[item] += delta; - put_cpu(); + this_cpu_add(per_cpu_var(vm_event_states).event[item], delta); } extern void all_vm_events(unsigned long *); diff --git a/include/net/neighbour.h b/include/net/neighbour.h index 3817fda82a80..f28403ff7648 100644 --- a/include/net/neighbour.h +++ b/include/net/neighbour.h @@ -90,12 +90,7 @@ struct neigh_statistics unsigned long unres_discards; /* number of unresolved drops */ }; -#define NEIGH_CACHE_STAT_INC(tbl, field) \ - do { \ - preempt_disable(); \ - (per_cpu_ptr((tbl)->stats, smp_processor_id())->field)++; \ - preempt_enable(); \ - } while (0) +#define NEIGH_CACHE_STAT_INC(tbl, field) this_cpu_inc((tbl)->stats->field) struct neighbour { diff --git a/include/net/netfilter/nf_conntrack.h b/include/net/netfilter/nf_conntrack.h index cbdd6284996d..dde549779e42 100644 --- a/include/net/netfilter/nf_conntrack.h +++ b/include/net/netfilter/nf_conntrack.h @@ -295,11 +295,11 @@ extern unsigned int nf_conntrack_htable_size; extern unsigned int nf_conntrack_max; #define NF_CT_STAT_INC(net, count) \ - (per_cpu_ptr((net)->ct.stat, raw_smp_processor_id())->count++) + __this_cpu_inc((net)->ct.stat->count) #define NF_CT_STAT_INC_ATOMIC(net, count) \ do { \ local_bh_disable(); \ - per_cpu_ptr((net)->ct.stat, raw_smp_processor_id())->count++; \ + __this_cpu_inc((net)->ct.stat->count); \ local_bh_enable(); \ } while (0) diff --git a/include/net/snmp.h b/include/net/snmp.h index 8c842e06bec8..f0d756f2ac99 100644 --- a/include/net/snmp.h +++ b/include/net/snmp.h @@ -136,45 +136,31 @@ struct linux_xfrm_mib { #define SNMP_STAT_BHPTR(name) (name[0]) #define SNMP_STAT_USRPTR(name) (name[1]) -#define SNMP_INC_STATS_BH(mib, field) \ - (per_cpu_ptr(mib[0], raw_smp_processor_id())->mibs[field]++) -#define SNMP_INC_STATS_USER(mib, field) \ - do { \ - per_cpu_ptr(mib[1], get_cpu())->mibs[field]++; \ - put_cpu(); \ - } while (0) -#define SNMP_INC_STATS(mib, field) \ - do { \ - per_cpu_ptr(mib[!in_softirq()], get_cpu())->mibs[field]++; \ - put_cpu(); \ - } while (0) -#define SNMP_DEC_STATS(mib, field) \ - do { \ - per_cpu_ptr(mib[!in_softirq()], get_cpu())->mibs[field]--; \ - put_cpu(); \ - } while (0) -#define SNMP_ADD_STATS(mib, field, addend) \ - do { \ - per_cpu_ptr(mib[!in_softirq()], get_cpu())->mibs[field] += addend; \ - put_cpu(); \ - } while (0) -#define SNMP_ADD_STATS_BH(mib, field, addend) \ - (per_cpu_ptr(mib[0], raw_smp_processor_id())->mibs[field] += addend) -#define SNMP_ADD_STATS_USER(mib, field, addend) \ - do { \ - per_cpu_ptr(mib[1], get_cpu())->mibs[field] += addend; \ - put_cpu(); \ - } while (0) +#define SNMP_INC_STATS_BH(mib, field) \ + __this_cpu_inc(mib[0]->mibs[field]) +#define SNMP_INC_STATS_USER(mib, field) \ + this_cpu_inc(mib[1]->mibs[field]) +#define SNMP_INC_STATS(mib, field) \ + this_cpu_inc(mib[!in_softirq()]->mibs[field]) +#define SNMP_DEC_STATS(mib, field) \ + this_cpu_dec(mib[!in_softirq()]->mibs[field]) +#define SNMP_ADD_STATS_BH(mib, field, addend) \ + __this_cpu_add(mib[0]->mibs[field], addend) +#define SNMP_ADD_STATS_USER(mib, field, addend) \ + this_cpu_add(mib[1]->mibs[field], addend) #define SNMP_UPD_PO_STATS(mib, basefield, addend) \ do { \ - __typeof__(mib[0]) ptr = per_cpu_ptr(mib[!in_softirq()], get_cpu());\ + __typeof__(mib[0]) ptr; \ + preempt_disable(); \ + ptr = this_cpu_ptr((mib)[!in_softirq()]); \ ptr->mibs[basefield##PKTS]++; \ ptr->mibs[basefield##OCTETS] += addend;\ - put_cpu(); \ + preempt_enable(); \ } while (0) #define SNMP_UPD_PO_STATS_BH(mib, basefield, addend) \ do { \ - __typeof__(mib[0]) ptr = per_cpu_ptr(mib[!in_softirq()], raw_smp_processor_id());\ + __typeof__(mib[0]) ptr = \ + __this_cpu_ptr((mib)[!in_softirq()]); \ ptr->mibs[basefield##PKTS]++; \ ptr->mibs[basefield##OCTETS] += addend;\ } while (0) diff --git a/kernel/module.c b/kernel/module.c index 8b7d8805819d..64787cddeb5e 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -370,8 +370,6 @@ EXPORT_SYMBOL_GPL(find_module); #ifdef CONFIG_SMP -#ifndef CONFIG_HAVE_LEGACY_PER_CPU_AREA - static void *percpu_modalloc(unsigned long size, unsigned long align, const char *name) { @@ -395,154 +393,6 @@ static void percpu_modfree(void *freeme) free_percpu(freeme); } -#else /* ... CONFIG_HAVE_LEGACY_PER_CPU_AREA */ - -/* Number of blocks used and allocated. */ -static unsigned int pcpu_num_used, pcpu_num_allocated; -/* Size of each block. -ve means used. */ -static int *pcpu_size; - -static int split_block(unsigned int i, unsigned short size) -{ - /* Reallocation required? */ - if (pcpu_num_used + 1 > pcpu_num_allocated) { - int *new; - - new = krealloc(pcpu_size, sizeof(new[0])*pcpu_num_allocated*2, - GFP_KERNEL); - if (!new) - return 0; - - pcpu_num_allocated *= 2; - pcpu_size = new; - } - - /* Insert a new subblock */ - memmove(&pcpu_size[i+1], &pcpu_size[i], - sizeof(pcpu_size[0]) * (pcpu_num_used - i)); - pcpu_num_used++; - - pcpu_size[i+1] -= size; - pcpu_size[i] = size; - return 1; -} - -static inline unsigned int block_size(int val) -{ - if (val < 0) - return -val; - return val; -} - -static void *percpu_modalloc(unsigned long size, unsigned long align, - const char *name) -{ - unsigned long extra; - unsigned int i; - void *ptr; - int cpu; - - if (align > PAGE_SIZE) { - printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n", - name, align, PAGE_SIZE); - align = PAGE_SIZE; - } - - ptr = __per_cpu_start; - for (i = 0; i < pcpu_num_used; ptr += block_size(pcpu_size[i]), i++) { - /* Extra for alignment requirement. */ - extra = ALIGN((unsigned long)ptr, align) - (unsigned long)ptr; - BUG_ON(i == 0 && extra != 0); - - if (pcpu_size[i] < 0 || pcpu_size[i] < extra + size) - continue; - - /* Transfer extra to previous block. */ - if (pcpu_size[i-1] < 0) - pcpu_size[i-1] -= extra; - else - pcpu_size[i-1] += extra; - pcpu_size[i] -= extra; - ptr += extra; - - /* Split block if warranted */ - if (pcpu_size[i] - size > sizeof(unsigned long)) - if (!split_block(i, size)) - return NULL; - - /* add the per-cpu scanning areas */ - for_each_possible_cpu(cpu) - kmemleak_alloc(ptr + per_cpu_offset(cpu), size, 0, - GFP_KERNEL); - - /* Mark allocated */ - pcpu_size[i] = -pcpu_size[i]; - return ptr; - } - - printk(KERN_WARNING "Could not allocate %lu bytes percpu data\n", - size); - return NULL; -} - -static void percpu_modfree(void *freeme) -{ - unsigned int i; - void *ptr = __per_cpu_start + block_size(pcpu_size[0]); - int cpu; - - /* First entry is core kernel percpu data. */ - for (i = 1; i < pcpu_num_used; ptr += block_size(pcpu_size[i]), i++) { - if (ptr == freeme) { - pcpu_size[i] = -pcpu_size[i]; - goto free; - } - } - BUG(); - - free: - /* remove the per-cpu scanning areas */ - for_each_possible_cpu(cpu) - kmemleak_free(freeme + per_cpu_offset(cpu)); - - /* Merge with previous? */ - if (pcpu_size[i-1] >= 0) { - pcpu_size[i-1] += pcpu_size[i]; - pcpu_num_used--; - memmove(&pcpu_size[i], &pcpu_size[i+1], - (pcpu_num_used - i) * sizeof(pcpu_size[0])); - i--; - } - /* Merge with next? */ - if (i+1 < pcpu_num_used && pcpu_size[i+1] >= 0) { - pcpu_size[i] += pcpu_size[i+1]; - pcpu_num_used--; - memmove(&pcpu_size[i+1], &pcpu_size[i+2], - (pcpu_num_used - (i+1)) * sizeof(pcpu_size[0])); - } -} - -static int percpu_modinit(void) -{ - pcpu_num_used = 2; - pcpu_num_allocated = 2; - pcpu_size = kmalloc(sizeof(pcpu_size[0]) * pcpu_num_allocated, - GFP_KERNEL); - /* Static in-kernel percpu data (used). */ - pcpu_size[0] = -(__per_cpu_end-__per_cpu_start); - /* Free room. */ - pcpu_size[1] = PERCPU_ENOUGH_ROOM + pcpu_size[0]; - if (pcpu_size[1] < 0) { - printk(KERN_ERR "No per-cpu room for modules.\n"); - pcpu_num_used = 1; - } - - return 0; -} -__initcall(percpu_modinit); - -#endif /* CONFIG_HAVE_LEGACY_PER_CPU_AREA */ - static unsigned int find_pcpusec(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, const char *secstrings) diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c index 697c0a0229d4..5595214358f3 100644 --- a/kernel/rcutorture.c +++ b/kernel/rcutorture.c @@ -729,13 +729,13 @@ static void rcu_torture_timer(unsigned long unused) /* Should not happen, but... */ pipe_count = RCU_TORTURE_PIPE_LEN; } - ++__get_cpu_var(rcu_torture_count)[pipe_count]; + __this_cpu_inc(per_cpu_var(rcu_torture_count)[pipe_count]); completed = cur_ops->completed() - completed; if (completed > RCU_TORTURE_PIPE_LEN) { /* Should not happen, but... */ completed = RCU_TORTURE_PIPE_LEN; } - ++__get_cpu_var(rcu_torture_batch)[completed]; + __this_cpu_inc(per_cpu_var(rcu_torture_batch)[completed]); preempt_enable(); cur_ops->readunlock(idx); } @@ -784,13 +784,13 @@ rcu_torture_reader(void *arg) /* Should not happen, but... */ pipe_count = RCU_TORTURE_PIPE_LEN; } - ++__get_cpu_var(rcu_torture_count)[pipe_count]; + __this_cpu_inc(per_cpu_var(rcu_torture_count)[pipe_count]); completed = cur_ops->completed() - completed; if (completed > RCU_TORTURE_PIPE_LEN) { /* Should not happen, but... */ completed = RCU_TORTURE_PIPE_LEN; } - ++__get_cpu_var(rcu_torture_batch)[completed]; + __this_cpu_inc(per_cpu_var(rcu_torture_batch)[completed]); preempt_enable(); cur_ops->readunlock(idx); schedule(); diff --git a/mm/Makefile b/mm/Makefile index e3980c2126ec..1f6e10c384cc 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -35,11 +35,7 @@ obj-$(CONFIG_FAILSLAB) += failslab.o obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o obj-$(CONFIG_FS_XIP) += filemap_xip.o obj-$(CONFIG_MIGRATION) += migrate.o -ifndef CONFIG_HAVE_LEGACY_PER_CPU_AREA obj-$(CONFIG_SMP) += percpu.o -else -obj-$(CONFIG_SMP) += allocpercpu.o -endif obj-$(CONFIG_QUICKLIST) += quicklist.o obj-$(CONFIG_CGROUP_MEM_RES_CTLR) += memcontrol.o page_cgroup.o obj-$(CONFIG_MEMORY_FAILURE) += memory-failure.o diff --git a/mm/allocpercpu.c b/mm/allocpercpu.c deleted file mode 100644 index df34ceae0c67..000000000000 --- a/mm/allocpercpu.c +++ /dev/null @@ -1,177 +0,0 @@ -/* - * linux/mm/allocpercpu.c - * - * Separated from slab.c August 11, 2006 Christoph Lameter - */ -#include <linux/mm.h> -#include <linux/module.h> -#include <linux/bootmem.h> -#include <asm/sections.h> - -#ifndef cache_line_size -#define cache_line_size() L1_CACHE_BYTES -#endif - -/** - * percpu_depopulate - depopulate per-cpu data for given cpu - * @__pdata: per-cpu data to depopulate - * @cpu: depopulate per-cpu data for this cpu - * - * Depopulating per-cpu data for a cpu going offline would be a typical - * use case. You need to register a cpu hotplug handler for that purpose. - */ -static void percpu_depopulate(void *__pdata, int cpu) -{ - struct percpu_data *pdata = __percpu_disguise(__pdata); - - kfree(pdata->ptrs[cpu]); - pdata->ptrs[cpu] = NULL; -} - -/** - * percpu_depopulate_mask - depopulate per-cpu data for some cpu's - * @__pdata: per-cpu data to depopulate - * @mask: depopulate per-cpu data for cpu's selected through mask bits - */ -static void __percpu_depopulate_mask(void *__pdata, const cpumask_t *mask) -{ - int cpu; - for_each_cpu_mask_nr(cpu, *mask) - percpu_depopulate(__pdata, cpu); -} - -#define percpu_depopulate_mask(__pdata, mask) \ - __percpu_depopulate_mask((__pdata), &(mask)) - -/** - * percpu_populate - populate per-cpu data for given cpu - * @__pdata: per-cpu data to populate further - * @size: size of per-cpu object - * @gfp: may sleep or not etc. - * @cpu: populate per-data for this cpu - * - * Populating per-cpu data for a cpu coming online would be a typical - * use case. You need to register a cpu hotplug handler for that purpose. - * Per-cpu object is populated with zeroed buffer. - */ -static void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu) -{ - struct percpu_data *pdata = __percpu_disguise(__pdata); - int node = cpu_to_node(cpu); - - /* - * We should make sure each CPU gets private memory. - */ - size = roundup(size, cache_line_size()); - - BUG_ON(pdata->ptrs[cpu]); - if (node_online(node)) - pdata->ptrs[cpu] = kmalloc_node(size, gfp|__GFP_ZERO, node); - else - pdata->ptrs[cpu] = kzalloc(size, gfp); - return pdata->ptrs[cpu]; -} - -/** - * percpu_populate_mask - populate per-cpu data for more cpu's - * @__pdata: per-cpu data to populate further - * @size: size of per-cpu object - * @gfp: may sleep or not etc. - * @mask: populate per-cpu data for cpu's selected through mask bits - * - * Per-cpu objects are populated with zeroed buffers. - */ -static int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp, - cpumask_t *mask) -{ - cpumask_t populated; - int cpu; - - cpus_clear(populated); - for_each_cpu_mask_nr(cpu, *mask) - if (unlikely(!percpu_populate(__pdata, size, gfp, cpu))) { - __percpu_depopulate_mask(__pdata, &populated); - return -ENOMEM; - } else - cpu_set(cpu, populated); - return 0; -} - -#define percpu_populate_mask(__pdata, size, gfp, mask) \ - __percpu_populate_mask((__pdata), (size), (gfp), &(mask)) - -/** - * alloc_percpu - initial setup of per-cpu data - * @size: size of per-cpu object - * @align: alignment - * - * Allocate dynamic percpu area. Percpu objects are populated with - * zeroed buffers. - */ -void *__alloc_percpu(size_t size, size_t align) -{ - /* - * We allocate whole cache lines to avoid false sharing - */ - size_t sz = roundup(nr_cpu_ids * sizeof(void *), cache_line_size()); - void *pdata = kzalloc(sz, GFP_KERNEL); - void *__pdata = __percpu_disguise(pdata); - - /* - * Can't easily make larger alignment work with kmalloc. WARN - * on it. Larger alignment should only be used for module - * percpu sections on SMP for which this path isn't used. - */ - WARN_ON_ONCE(align > SMP_CACHE_BYTES); - - if (unlikely(!pdata)) - return NULL; - if (likely(!__percpu_populate_mask(__pdata, size, GFP_KERNEL, - &cpu_possible_map))) - return __pdata; - kfree(pdata); - return NULL; -} -EXPORT_SYMBOL_GPL(__alloc_percpu); - -/** - * free_percpu - final cleanup of per-cpu data - * @__pdata: object to clean up - * - * We simply clean up any per-cpu object left. No need for the client to - * track and specify through a bis mask which per-cpu objects are to free. - */ -void free_percpu(void *__pdata) -{ - if (unlikely(!__pdata)) - return; - __percpu_depopulate_mask(__pdata, cpu_possible_mask); - kfree(__percpu_disguise(__pdata)); -} -EXPORT_SYMBOL_GPL(free_percpu); - -/* - * Generic percpu area setup. - */ -#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA -unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; - -EXPORT_SYMBOL(__per_cpu_offset); - -void __init setup_per_cpu_areas(void) -{ - unsigned long size, i; - char *ptr; - unsigned long nr_possible_cpus = num_possible_cpus(); - - /* Copy section for each CPU (we discard the original) */ - size = ALIGN(PERCPU_ENOUGH_ROOM, PAGE_SIZE); - ptr = alloc_bootmem_pages(size * nr_possible_cpus); - - for_each_possible_cpu(i) { - __per_cpu_offset[i] = ptr - __per_cpu_start; - memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start); - ptr += size; - } -} -#endif /* CONFIG_HAVE_SETUP_PER_CPU_AREA */ diff --git a/mm/percpu.c b/mm/percpu.c index 4a048abad043..ec158bb5f86d 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -46,8 +46,6 @@ * * To use this allocator, arch code should do the followings. * - * - drop CONFIG_HAVE_LEGACY_PER_CPU_AREA - * * - define __addr_to_pcpu_ptr() and __pcpu_ptr_to_addr() to translate * regular address to percpu pointer and back if they need to be * different from the default @@ -1870,13 +1868,14 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, ssize_t dyn_size, max_distance = 0; for (group = 0; group < ai->nr_groups; group++) { ai->groups[group].base_offset = areas[group] - base; - max_distance = max(max_distance, ai->groups[group].base_offset); + max_distance = max_t(size_t, max_distance, + ai->groups[group].base_offset); } max_distance += ai->unit_size; /* warn if maximum distance is further than 75% of vmalloc space */ if (max_distance > (VMALLOC_END - VMALLOC_START) * 3 / 4) { - pr_warning("PERCPU: max_distance=0x%lx too large for vmalloc " + pr_warning("PERCPU: max_distance=0x%zx too large for vmalloc " "space 0x%lx\n", max_distance, VMALLOC_END - VMALLOC_START); #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK |