From 126b3fcdecd350cad9700908d0ad845084e26a31 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 2 Oct 2009 13:28:55 +0900 Subject: ia64: don't alias VMALLOC_END to vmalloc_end If CONFIG_VIRTUAL_MEM_MAP is enabled, ia64 defines macro VMALLOC_END as unsigned long variable vmalloc_end which is adjusted to prepare room for vmemmap. This becomes probnlematic if a local variables vmalloc_end is defined in some function (not very unlikely) and VMALLOC_END is used in the function - the function thinks its referencing the global VMALLOC_END value but would be referencing its own local vmalloc_end variable. There's no reason VMALLOC_END should be a macro. Just define it as an unsigned long variable if CONFIG_VIRTUAL_MEM_MAP is set to avoid nasty surprises. Signed-off-by: Tejun Heo Acked-by: Tony Luck Cc: Fenghua Yu Cc: linux-ia64 Cc: Christoph Lameter --- arch/ia64/include/asm/meminit.h | 2 +- arch/ia64/include/asm/pgtable.h | 3 +-- arch/ia64/mm/contig.c | 4 ++-- arch/ia64/mm/discontig.c | 4 ++-- arch/ia64/mm/init.c | 4 ++-- 5 files changed, 8 insertions(+), 9 deletions(-) (limited to 'arch') diff --git a/arch/ia64/include/asm/meminit.h b/arch/ia64/include/asm/meminit.h index 688a812c017d..61c7b1750b16 100644 --- a/arch/ia64/include/asm/meminit.h +++ b/arch/ia64/include/asm/meminit.h @@ -61,7 +61,7 @@ extern int register_active_ranges(u64 start, u64 len, int nid); #ifdef CONFIG_VIRTUAL_MEM_MAP # define LARGE_GAP 0x40000000 /* Use virtual mem map if hole is > than this */ - extern unsigned long vmalloc_end; + extern unsigned long VMALLOC_END; extern struct page *vmem_map; extern int find_largest_hole(u64 start, u64 end, void *arg); extern int create_mem_map_page_table(u64 start, u64 end, void *arg); diff --git a/arch/ia64/include/asm/pgtable.h b/arch/ia64/include/asm/pgtable.h index 8840a690d1e7..69bf13857a9f 100644 --- a/arch/ia64/include/asm/pgtable.h +++ b/arch/ia64/include/asm/pgtable.h @@ -228,8 +228,7 @@ ia64_phys_addr_valid (unsigned long addr) #define VMALLOC_START (RGN_BASE(RGN_GATE) + 0x200000000UL) #ifdef CONFIG_VIRTUAL_MEM_MAP # define VMALLOC_END_INIT (RGN_BASE(RGN_GATE) + (1UL << (4*PAGE_SHIFT - 9))) -# define VMALLOC_END vmalloc_end - extern unsigned long vmalloc_end; +extern unsigned long VMALLOC_END; #else #if defined(CONFIG_SPARSEMEM) && defined(CONFIG_SPARSEMEM_VMEMMAP) /* SPARSEMEM_VMEMMAP uses half of vmalloc... */ diff --git a/arch/ia64/mm/contig.c b/arch/ia64/mm/contig.c index 2f724d2bf299..1341437c1b26 100644 --- a/arch/ia64/mm/contig.c +++ b/arch/ia64/mm/contig.c @@ -270,8 +270,8 @@ paging_init (void) map_size = PAGE_ALIGN(ALIGN(max_low_pfn, MAX_ORDER_NR_PAGES) * sizeof(struct page)); - vmalloc_end -= map_size; - vmem_map = (struct page *) vmalloc_end; + VMALLOC_END -= map_size; + vmem_map = (struct page *) VMALLOC_END; efi_memmap_walk(create_mem_map_page_table, NULL); /* diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c index d85ba98d9008..9f24b3c6dc71 100644 --- a/arch/ia64/mm/discontig.c +++ b/arch/ia64/mm/discontig.c @@ -666,9 +666,9 @@ void __init paging_init(void) sparse_init(); #ifdef CONFIG_VIRTUAL_MEM_MAP - vmalloc_end -= PAGE_ALIGN(ALIGN(max_low_pfn, MAX_ORDER_NR_PAGES) * + VMALLOC_END -= PAGE_ALIGN(ALIGN(max_low_pfn, MAX_ORDER_NR_PAGES) * sizeof(struct page)); - vmem_map = (struct page *) vmalloc_end; + vmem_map = (struct page *) VMALLOC_END; efi_memmap_walk(create_mem_map_page_table, NULL); printk("Virtual mem_map starts at 0x%p\n", vmem_map); #endif diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index 1857766a63c1..b9609c69343a 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -44,8 +44,8 @@ extern void ia64_tlb_init (void); unsigned long MAX_DMA_ADDRESS = PAGE_OFFSET + 0x100000000UL; #ifdef CONFIG_VIRTUAL_MEM_MAP -unsigned long vmalloc_end = VMALLOC_END_INIT; -EXPORT_SYMBOL(vmalloc_end); +unsigned long VMALLOC_END = VMALLOC_END_INIT; +EXPORT_SYMBOL(VMALLOC_END); struct page *vmem_map; EXPORT_SYMBOL(vmem_map); #endif -- cgit v1.2.3 From 12cda817779ce5381a9a4ba8d464abe17c50a9e2 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 2 Oct 2009 13:28:56 +0900 Subject: ia64: initialize cpu maps early All information necessary to initialize cpu possible and present maps are available once early_acpi_boot_init() is complete. Reorganize setup_arch() and acpi init functions such that, * CPU information is printed after LAPIC entries are parsed in early_acpi_boot_init(). * smp_build_cpu_map() is called by setup_arch() instead of acpi functions. * smp_build_cpu_map() is called once all CPU related information is available before memory is initialized. This is primarily to allow find_memory() to use cpu maps but is also a general cleanup. Please note that with this change, the somewhat ad-hoc early_cpu_possible_map defined and used for NUMA configurations is probably unnecessary. Something to clean up another day. Signed-off-by: Tejun Heo Acked-by: Tony Luck Cc: Fenghua Yu Cc: linux-ia64 --- arch/ia64/kernel/acpi.c | 33 +++++++++++++++------------------ arch/ia64/kernel/setup.c | 11 +++++------ 2 files changed, 20 insertions(+), 24 deletions(-) (limited to 'arch') diff --git a/arch/ia64/kernel/acpi.c b/arch/ia64/kernel/acpi.c index baec6f00f7f3..40574ae11401 100644 --- a/arch/ia64/kernel/acpi.c +++ b/arch/ia64/kernel/acpi.c @@ -702,11 +702,23 @@ int __init early_acpi_boot_init(void) printk(KERN_ERR PREFIX "Error parsing MADT - no LAPIC entries\n"); +#ifdef CONFIG_SMP + if (available_cpus == 0) { + printk(KERN_INFO "ACPI: Found 0 CPUS; assuming 1\n"); + printk(KERN_INFO "CPU 0 (0x%04x)", hard_smp_processor_id()); + smp_boot_data.cpu_phys_id[available_cpus] = + hard_smp_processor_id(); + available_cpus = 1; /* We've got at least one of these, no? */ + } + smp_boot_data.cpu_count = available_cpus; +#endif + /* Make boot-up look pretty */ + printk(KERN_INFO "%d CPUs available, %d CPUs total\n", available_cpus, + total_cpus); + return 0; } - - int __init acpi_boot_init(void) { @@ -769,18 +781,8 @@ int __init acpi_boot_init(void) if (acpi_table_parse(ACPI_SIG_FADT, acpi_parse_fadt)) printk(KERN_ERR PREFIX "Can't find FADT\n"); +#ifdef CONFIG_ACPI_NUMA #ifdef CONFIG_SMP - if (available_cpus == 0) { - printk(KERN_INFO "ACPI: Found 0 CPUS; assuming 1\n"); - printk(KERN_INFO "CPU 0 (0x%04x)", hard_smp_processor_id()); - smp_boot_data.cpu_phys_id[available_cpus] = - hard_smp_processor_id(); - available_cpus = 1; /* We've got at least one of these, no? */ - } - smp_boot_data.cpu_count = available_cpus; - - smp_build_cpu_map(); -# ifdef CONFIG_ACPI_NUMA if (srat_num_cpus == 0) { int cpu, i = 1; for (cpu = 0; cpu < smp_boot_data.cpu_count; cpu++) @@ -789,14 +791,9 @@ int __init acpi_boot_init(void) node_cpuid[i++].phys_id = smp_boot_data.cpu_phys_id[cpu]; } -# endif #endif -#ifdef CONFIG_ACPI_NUMA build_cpu_to_node_map(); #endif - /* Make boot-up look pretty */ - printk(KERN_INFO "%d CPUs available, %d CPUs total\n", available_cpus, - total_cpus); return 0; } diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c index 1de86c96801d..5d77c1e1c0ce 100644 --- a/arch/ia64/kernel/setup.c +++ b/arch/ia64/kernel/setup.c @@ -566,19 +566,18 @@ setup_arch (char **cmdline_p) early_acpi_boot_init(); # ifdef CONFIG_ACPI_NUMA acpi_numa_init(); -#ifdef CONFIG_ACPI_HOTPLUG_CPU +# ifdef CONFIG_ACPI_HOTPLUG_CPU prefill_possible_map(); -#endif +# endif per_cpu_scan_finalize((cpus_weight(early_cpu_possible_map) == 0 ? 32 : cpus_weight(early_cpu_possible_map)), additional_cpus > 0 ? additional_cpus : 0); # endif -#else -# ifdef CONFIG_SMP - smp_build_cpu_map(); /* happens, e.g., with the Ski simulator */ -# endif #endif /* CONFIG_APCI_BOOT */ +#ifdef CONFIG_SMP + smp_build_cpu_map(); +#endif find_memory(); /* process SAL system table: */ -- cgit v1.2.3 From 36886478f59ec0fdc24a8877c572b92f8d416aba Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 2 Oct 2009 13:28:56 +0900 Subject: ia64: allocate percpu area for cpu0 like percpu areas for other cpus cpu0 used special percpu area reserved by the linker, __cpu0_per_cpu, which is set up early in boot by head.S. However, this doesn't guarantee that the area will be on the same node as cpu0 and the percpu area for cpu0 ends up very far away from percpu areas for other cpus which cause problems for congruent percpu allocator. This patch makes percpu area initialization allocate percpu area for cpu0 like any other cpus and copy it from __cpu0_per_cpu which now resides in the __init area. This means that for cpu0, percpu area is first setup at __cpu0_per_cpu early by head.S and then moved to an area in the linear mapping during memory initialization and it's not allowed to take a pointer to percpu variables between head.S and memory initialization. Signed-off-by: Tejun Heo Acked-by: Tony Luck Cc: Fenghua Yu Cc: linux-ia64 --- arch/ia64/kernel/vmlinux.lds.S | 11 ++++++----- arch/ia64/mm/contig.c | 41 +++++++++++++++++++++++++++-------------- arch/ia64/mm/discontig.c | 35 ++++++++++++++++++++++++----------- 3 files changed, 57 insertions(+), 30 deletions(-) (limited to 'arch') diff --git a/arch/ia64/kernel/vmlinux.lds.S b/arch/ia64/kernel/vmlinux.lds.S index 0a0c77b2c988..1295ba327f6f 100644 --- a/arch/ia64/kernel/vmlinux.lds.S +++ b/arch/ia64/kernel/vmlinux.lds.S @@ -166,6 +166,12 @@ SECTIONS } #endif +#ifdef CONFIG_SMP + . = ALIGN(PERCPU_PAGE_SIZE); + __cpu0_per_cpu = .; + . = . + PERCPU_PAGE_SIZE; /* cpu0 per-cpu space */ +#endif + . = ALIGN(PAGE_SIZE); __init_end = .; @@ -198,11 +204,6 @@ SECTIONS data : { } :data .data : AT(ADDR(.data) - LOAD_OFFSET) { -#ifdef CONFIG_SMP - . = ALIGN(PERCPU_PAGE_SIZE); - __cpu0_per_cpu = .; - . = . + PERCPU_PAGE_SIZE; /* cpu0 per-cpu space */ -#endif INIT_TASK_DATA(PAGE_SIZE) CACHELINE_ALIGNED_DATA(SMP_CACHE_BYTES) READ_MOSTLY_DATA(SMP_CACHE_BYTES) diff --git a/arch/ia64/mm/contig.c b/arch/ia64/mm/contig.c index 1341437c1b26..351da0a06cd0 100644 --- a/arch/ia64/mm/contig.c +++ b/arch/ia64/mm/contig.c @@ -154,36 +154,49 @@ static void *cpu_data; void * __cpuinit per_cpu_init (void) { - int cpu; - static int first_time=1; + static bool first_time = true; + void *cpu0_data = __cpu0_per_cpu; + unsigned int cpu; + + if (!first_time) + goto skip; + first_time = false; /* * get_free_pages() cannot be used before cpu_init() done. BSP * allocates "NR_CPUS" pages for all CPUs to avoid that AP calls * get_zeroed_page(). */ - if (first_time) { - void *cpu0_data = __cpu0_per_cpu; + for (cpu = 0; cpu < NR_CPUS; cpu++) { + void *src = cpu == 0 ? cpu0_data : __phys_per_cpu_start; - first_time=0; + memcpy(cpu_data, src, __per_cpu_end - __per_cpu_start); + __per_cpu_offset[cpu] = (char *)cpu_data - __per_cpu_start; + per_cpu(local_per_cpu_offset, cpu) = __per_cpu_offset[cpu]; - __per_cpu_offset[0] = (char *) cpu0_data - __per_cpu_start; - per_cpu(local_per_cpu_offset, 0) = __per_cpu_offset[0]; + /* + * percpu area for cpu0 is moved from the __init area + * which is setup by head.S and used till this point. + * Update ar.k3. This move is ensures that percpu + * area for cpu0 is on the correct node and its + * virtual address isn't insanely far from other + * percpu areas which is important for congruent + * percpu allocator. + */ + if (cpu == 0) + ia64_set_kr(IA64_KR_PER_CPU_DATA, __pa(cpu_data) - + (unsigned long)__per_cpu_start); - for (cpu = 1; cpu < NR_CPUS; cpu++) { - memcpy(cpu_data, __phys_per_cpu_start, __per_cpu_end - __per_cpu_start); - __per_cpu_offset[cpu] = (char *) cpu_data - __per_cpu_start; - cpu_data += PERCPU_PAGE_SIZE; - per_cpu(local_per_cpu_offset, cpu) = __per_cpu_offset[cpu]; - } + cpu_data += PERCPU_PAGE_SIZE; } +skip: return __per_cpu_start + __per_cpu_offset[smp_processor_id()]; } static inline void alloc_per_cpu_data(void) { - cpu_data = __alloc_bootmem(PERCPU_PAGE_SIZE * NR_CPUS-1, + cpu_data = __alloc_bootmem(PERCPU_PAGE_SIZE * NR_CPUS, PERCPU_PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); } #else diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c index 9f24b3c6dc71..200282b92981 100644 --- a/arch/ia64/mm/discontig.c +++ b/arch/ia64/mm/discontig.c @@ -143,17 +143,30 @@ static void *per_cpu_node_setup(void *cpu_data, int node) int cpu; for_each_possible_early_cpu(cpu) { - if (cpu == 0) { - void *cpu0_data = __cpu0_per_cpu; - __per_cpu_offset[cpu] = (char*)cpu0_data - - __per_cpu_start; - } else if (node == node_cpuid[cpu].nid) { - memcpy(__va(cpu_data), __phys_per_cpu_start, - __per_cpu_end - __per_cpu_start); - __per_cpu_offset[cpu] = (char*)__va(cpu_data) - - __per_cpu_start; - cpu_data += PERCPU_PAGE_SIZE; - } + void *src = cpu == 0 ? __cpu0_per_cpu : __phys_per_cpu_start; + + if (node != node_cpuid[cpu].nid) + continue; + + memcpy(__va(cpu_data), src, __per_cpu_end - __per_cpu_start); + __per_cpu_offset[cpu] = (char *)__va(cpu_data) - + __per_cpu_start; + + /* + * percpu area for cpu0 is moved from the __init area + * which is setup by head.S and used till this point. + * Update ar.k3. This move is ensures that percpu + * area for cpu0 is on the correct node and its + * virtual address isn't insanely far from other + * percpu areas which is important for congruent + * percpu allocator. + */ + if (cpu == 0) + ia64_set_kr(IA64_KR_PER_CPU_DATA, + (unsigned long)cpu_data - + (unsigned long)__per_cpu_start); + + cpu_data += PERCPU_PAGE_SIZE; } #endif return cpu_data; -- cgit v1.2.3 From 52594762a39dfb6338c9d0906ca21dd9ae9453be Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 2 Oct 2009 13:28:56 +0900 Subject: ia64: convert to dynamic percpu allocator Unlike other archs, ia64 reserves space for percpu areas during early memory initialization. These areas occupy a contiguous region indexed by cpu number on contiguous memory model or are grouped by node on discontiguous memory model. As allocation and initialization are done by the arch code, all that setup_per_cpu_areas() needs to do is communicating the determined layout to the percpu allocator. This patch implements setup_per_cpu_areas() for both contig and discontig memory models and drops HAVE_LEGACY_PER_CPU_AREA. Please note that for contig model, the allocation itself is modified only to allocate for possible cpus instead of NR_CPUS. As dynamic percpu allocator can handle non-direct mapping, there's no reason to allocate memory for cpus which aren't possible. Signed-off-by: Tejun Heo Acked-by: Tony Luck Cc: Fenghua Yu Cc: linux-ia64 --- arch/ia64/Kconfig | 3 -- arch/ia64/kernel/setup.c | 12 ------- arch/ia64/mm/contig.c | 58 ++++++++++++++++++++++++++++++--- arch/ia64/mm/discontig.c | 85 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 138 insertions(+), 20 deletions(-) (limited to 'arch') diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 1ee596cd942f..2d7f56a98e0f 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -87,9 +87,6 @@ config GENERIC_TIME_VSYSCALL bool default y -config HAVE_LEGACY_PER_CPU_AREA - def_bool y - config HAVE_SETUP_PER_CPU_AREA def_bool y diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c index 5d77c1e1c0ce..bc1ef4ae828c 100644 --- a/arch/ia64/kernel/setup.c +++ b/arch/ia64/kernel/setup.c @@ -854,18 +854,6 @@ identify_cpu (struct cpuinfo_ia64 *c) c->unimpl_pa_mask = ~((1L<<63) | ((1L << phys_addr_size) - 1)); } -/* - * In UP configuration, setup_per_cpu_areas() is defined in - * include/linux/percpu.h - */ -#ifdef CONFIG_SMP -void __init -setup_per_cpu_areas (void) -{ - /* start_kernel() requires this... */ -} -#endif - /* * Do the following calculations: * diff --git a/arch/ia64/mm/contig.c b/arch/ia64/mm/contig.c index 351da0a06cd0..54bf54059811 100644 --- a/arch/ia64/mm/contig.c +++ b/arch/ia64/mm/contig.c @@ -163,11 +163,11 @@ per_cpu_init (void) first_time = false; /* - * get_free_pages() cannot be used before cpu_init() done. BSP - * allocates "NR_CPUS" pages for all CPUs to avoid that AP calls - * get_zeroed_page(). + * get_free_pages() cannot be used before cpu_init() done. + * BSP allocates PERCPU_PAGE_SIZE bytes for all possible CPUs + * to avoid that AP calls get_zeroed_page(). */ - for (cpu = 0; cpu < NR_CPUS; cpu++) { + for_each_possible_cpu(cpu) { void *src = cpu == 0 ? cpu0_data : __phys_per_cpu_start; memcpy(cpu_data, src, __per_cpu_end - __per_cpu_start); @@ -196,9 +196,57 @@ skip: static inline void alloc_per_cpu_data(void) { - cpu_data = __alloc_bootmem(PERCPU_PAGE_SIZE * NR_CPUS, + cpu_data = __alloc_bootmem(PERCPU_PAGE_SIZE * num_possible_cpus(), PERCPU_PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); } + +/** + * setup_per_cpu_areas - setup percpu areas + * + * Arch code has already allocated and initialized percpu areas. All + * this function has to do is to teach the determined layout to the + * dynamic percpu allocator, which happens to be more complex than + * creating whole new ones using helpers. + */ +void __init +setup_per_cpu_areas(void) +{ + struct pcpu_alloc_info *ai; + struct pcpu_group_info *gi; + unsigned int cpu; + ssize_t static_size, reserved_size, dyn_size; + int rc; + + ai = pcpu_alloc_alloc_info(1, num_possible_cpus()); + if (!ai) + panic("failed to allocate pcpu_alloc_info"); + gi = &ai->groups[0]; + + /* units are assigned consecutively to possible cpus */ + for_each_possible_cpu(cpu) + gi->cpu_map[gi->nr_units++] = cpu; + + /* set parameters */ + static_size = __per_cpu_end - __per_cpu_start; + reserved_size = PERCPU_MODULE_RESERVE; + dyn_size = PERCPU_PAGE_SIZE - static_size - reserved_size; + if (dyn_size < 0) + panic("percpu area overflow static=%zd reserved=%zd\n", + static_size, reserved_size); + + ai->static_size = static_size; + ai->reserved_size = reserved_size; + ai->dyn_size = dyn_size; + ai->unit_size = PERCPU_PAGE_SIZE; + ai->atom_size = PAGE_SIZE; + ai->alloc_size = PERCPU_PAGE_SIZE; + + rc = pcpu_setup_first_chunk(ai, __per_cpu_start + __per_cpu_offset[0]); + if (rc) + panic("failed to setup percpu area (err=%d)", rc); + + pcpu_free_alloc_info(ai); +} #else #define alloc_per_cpu_data() do { } while (0) #endif /* CONFIG_SMP */ diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c index 200282b92981..40e4c1fbf76b 100644 --- a/arch/ia64/mm/discontig.c +++ b/arch/ia64/mm/discontig.c @@ -172,6 +172,91 @@ static void *per_cpu_node_setup(void *cpu_data, int node) return cpu_data; } +#ifdef CONFIG_SMP +/** + * setup_per_cpu_areas - setup percpu areas + * + * Arch code has already allocated and initialized percpu areas. All + * this function has to do is to teach the determined layout to the + * dynamic percpu allocator, which happens to be more complex than + * creating whole new ones using helpers. + */ +void __init setup_per_cpu_areas(void) +{ + struct pcpu_alloc_info *ai; + struct pcpu_group_info *uninitialized_var(gi); + unsigned int *cpu_map; + void *base; + unsigned long base_offset; + unsigned int cpu; + ssize_t static_size, reserved_size, dyn_size; + int node, prev_node, unit, nr_units, rc; + + ai = pcpu_alloc_alloc_info(MAX_NUMNODES, nr_cpu_ids); + if (!ai) + panic("failed to allocate pcpu_alloc_info"); + cpu_map = ai->groups[0].cpu_map; + + /* determine base */ + base = (void *)ULONG_MAX; + for_each_possible_cpu(cpu) + base = min(base, + (void *)(__per_cpu_offset[cpu] + __per_cpu_start)); + base_offset = (void *)__per_cpu_start - base; + + /* build cpu_map, units are grouped by node */ + unit = 0; + for_each_node(node) + for_each_possible_cpu(cpu) + if (node == node_cpuid[cpu].nid) + cpu_map[unit++] = cpu; + nr_units = unit; + + /* set basic parameters */ + static_size = __per_cpu_end - __per_cpu_start; + reserved_size = PERCPU_MODULE_RESERVE; + dyn_size = PERCPU_PAGE_SIZE - static_size - reserved_size; + if (dyn_size < 0) + panic("percpu area overflow static=%zd reserved=%zd\n", + static_size, reserved_size); + + ai->static_size = static_size; + ai->reserved_size = reserved_size; + ai->dyn_size = dyn_size; + ai->unit_size = PERCPU_PAGE_SIZE; + ai->atom_size = PAGE_SIZE; + ai->alloc_size = PERCPU_PAGE_SIZE; + + /* + * CPUs are put into groups according to node. Walk cpu_map + * and create new groups at node boundaries. + */ + prev_node = -1; + ai->nr_groups = 0; + for (unit = 0; unit < nr_units; unit++) { + cpu = cpu_map[unit]; + node = node_cpuid[cpu].nid; + + if (node == prev_node) { + gi->nr_units++; + continue; + } + prev_node = node; + + gi = &ai->groups[ai->nr_groups++]; + gi->nr_units = 1; + gi->base_offset = __per_cpu_offset[cpu] + base_offset; + gi->cpu_map = &cpu_map[unit]; + } + + rc = pcpu_setup_first_chunk(ai, base); + if (rc) + panic("failed to setup percpu area (err=%d)", rc); + + pcpu_free_alloc_info(ai); +} +#endif + /** * fill_pernode - initialize pernode data. * @node: the node id. -- cgit v1.2.3 From 30ed1a79f5bf271d33e782afee3323582dcc621e Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Sat, 3 Oct 2009 19:48:22 +0900 Subject: this_cpu: Implement X86 optimized this_cpu operations Basically the existing percpu ops can be used for this_cpu variants that allow operations also on dynamically allocated percpu data. However, we do not pass a reference to a percpu variable in. Instead a dynamically or statically allocated percpu variable is provided. Preempt, the non preempt and the irqsafe operations generate the same code. It will always be possible to have the requires per cpu atomicness in a single RMW instruction with segment override on x86. 64 bit this_cpu operations are not supported on 32 bit. Signed-off-by: Christoph Lameter Signed-off-by: Tejun Heo --- arch/x86/include/asm/percpu.h | 78 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) (limited to 'arch') diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index b65a36defeb7..8b5ec19bdef4 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -153,6 +153,84 @@ do { \ #define percpu_or(var, val) percpu_to_op("or", per_cpu__##var, val) #define percpu_xor(var, val) percpu_to_op("xor", per_cpu__##var, val) +#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define __this_cpu_read_4(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) + +#define __this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) +#define __this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) +#define __this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) +#define __this_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val) +#define __this_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val) +#define __this_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val) +#define __this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) +#define __this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) +#define __this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) +#define __this_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val) +#define __this_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val) +#define __this_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val) +#define __this_cpu_xor_1(pcp, val) percpu_to_op("xor", (pcp), val) +#define __this_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val) +#define __this_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val) + +#define this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define this_cpu_read_4(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) +#define this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) +#define this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) +#define this_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val) +#define this_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val) +#define this_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val) +#define this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) +#define this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) +#define this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) +#define this_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val) +#define this_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val) +#define this_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val) +#define this_cpu_xor_1(pcp, val) percpu_to_op("xor", (pcp), val) +#define this_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val) +#define this_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val) + +#define irqsafe_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val) +#define irqsafe_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val) +#define irqsafe_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val) +#define irqsafe_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) +#define irqsafe_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) +#define irqsafe_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) +#define irqsafe_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val) +#define irqsafe_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val) +#define irqsafe_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val) +#define irqsafe_cpu_xor_1(pcp, val) percpu_to_op("xor", (pcp), val) +#define irqsafe_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val) +#define irqsafe_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val) + +/* + * Per cpu atomic 64 bit operations are only available under 64 bit. + * 32 bit must fall back to generic operations. + */ +#ifdef CONFIG_X86_64 +#define __this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define __this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) +#define __this_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val) +#define __this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) +#define __this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) +#define __this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) + +#define this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) +#define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) +#define this_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val) +#define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) +#define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) +#define this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) + +#define irqsafe_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val) +#define irqsafe_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) +#define irqsafe_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) +#define irqsafe_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val) + +#endif + /* This is not atomic against other CPUs -- CPU preemption needs to be off */ #define x86_test_and_clear_bit_percpu(bit, var) \ ({ \ -- cgit v1.2.3 From 494f6a9e12f5137d355d3ce3f5789ef148b642bc Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 7 Oct 2009 19:04:29 -0400 Subject: this_cpu: Use this_cpu_xx in nmi handling this_cpu_inc/dec reduces the number of instructions needed. Signed-off-by: Christoph Lameter Signed-off-by: Tejun Heo --- arch/sparc/kernel/nmi.c | 8 ++++---- arch/x86/kernel/apic/nmi.c | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) (limited to 'arch') diff --git a/arch/sparc/kernel/nmi.c b/arch/sparc/kernel/nmi.c index b129611590a4..f30f4a1ead23 100644 --- a/arch/sparc/kernel/nmi.c +++ b/arch/sparc/kernel/nmi.c @@ -47,7 +47,7 @@ static DEFINE_PER_CPU(short, wd_enabled); static int endflag __initdata; static DEFINE_PER_CPU(unsigned int, last_irq_sum); -static DEFINE_PER_CPU(local_t, alert_counter); +static DEFINE_PER_CPU(long, alert_counter); static DEFINE_PER_CPU(int, nmi_touch); void touch_nmi_watchdog(void) @@ -112,13 +112,13 @@ notrace __kprobes void perfctr_irq(int irq, struct pt_regs *regs) touched = 1; } if (!touched && __get_cpu_var(last_irq_sum) == sum) { - local_inc(&__get_cpu_var(alert_counter)); - if (local_read(&__get_cpu_var(alert_counter)) == 30 * nmi_hz) + __this_cpu_inc(per_cpu_var(alert_counter)); + if (__this_cpu_read(per_cpu_var(alert_counter)) == 30 * nmi_hz) die_nmi("BUG: NMI Watchdog detected LOCKUP", regs, panic_on_timeout); } else { __get_cpu_var(last_irq_sum) = sum; - local_set(&__get_cpu_var(alert_counter), 0); + __this_cpu_write(per_cpu_var(alert_counter), 0); } if (__get_cpu_var(wd_enabled)) { write_pic(picl_value(nmi_hz)); diff --git a/arch/x86/kernel/apic/nmi.c b/arch/x86/kernel/apic/nmi.c index 7ff61d6a188a..e631cc4416f7 100644 --- a/arch/x86/kernel/apic/nmi.c +++ b/arch/x86/kernel/apic/nmi.c @@ -360,7 +360,7 @@ void stop_apic_nmi_watchdog(void *unused) */ static DEFINE_PER_CPU(unsigned, last_irq_sum); -static DEFINE_PER_CPU(local_t, alert_counter); +static DEFINE_PER_CPU(long, alert_counter); static DEFINE_PER_CPU(int, nmi_touch); void touch_nmi_watchdog(void) @@ -437,8 +437,8 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason) * Ayiee, looks like this CPU is stuck ... * wait a few IRQs (5 seconds) before doing the oops ... */ - local_inc(&__get_cpu_var(alert_counter)); - if (local_read(&__get_cpu_var(alert_counter)) == 5 * nmi_hz) + __this_cpu_inc(per_cpu_var(alert_counter)); + if (__this_cpu_read(per_cpu_var(alert_counter)) == 5 * nmi_hz) /* * die_nmi will return ONLY if NOTIFY_STOP happens.. */ @@ -446,7 +446,7 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason) regs, panic_on_timeout); } else { __get_cpu_var(last_irq_sum) = sum; - local_set(&__get_cpu_var(alert_counter), 0); + __this_cpu_write(per_cpu_var(alert_counter), 0); } /* see if the nmi watchdog went off */ -- cgit v1.2.3 From 0f5e4816dbf38ce9488e611ca2296925c1e90d5e Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 29 Oct 2009 22:34:12 +0900 Subject: percpu: remove some sparse warnings Make the following changes to remove some sparse warnings. * Make DEFINE_PER_CPU_SECTION() declare __pcpu_unique_* before defining it. * Annotate pcpu_extend_area_map() that it is entered with pcpu_lock held, releases it and then reacquires it. * Make percpu related macros use unique nested variable names. * While at it, add pcpu prefix to __size_call[_return]() macros as to-be-implemented sparse annotations will add percpu specific stuff to these macros. Signed-off-by: Tejun Heo Reviewed-by: Christoph Lameter Cc: Rusty Russell --- arch/x86/include/asm/percpu.h | 26 +++++++++++------------ include/linux/percpu-defs.h | 1 + include/linux/percpu.h | 48 +++++++++++++++++++++---------------------- mm/percpu.c | 1 + 4 files changed, 39 insertions(+), 37 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index 8b5ec19bdef4..0c44196b78ac 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -74,31 +74,31 @@ extern void __bad_percpu_size(void); #define percpu_to_op(op, var, val) \ do { \ - typedef typeof(var) T__; \ + typedef typeof(var) pto_T__; \ if (0) { \ - T__ tmp__; \ - tmp__ = (val); \ + pto_T__ pto_tmp__; \ + pto_tmp__ = (val); \ } \ switch (sizeof(var)) { \ case 1: \ asm(op "b %1,"__percpu_arg(0) \ : "+m" (var) \ - : "qi" ((T__)(val))); \ + : "qi" ((pto_T__)(val))); \ break; \ case 2: \ asm(op "w %1,"__percpu_arg(0) \ : "+m" (var) \ - : "ri" ((T__)(val))); \ + : "ri" ((pto_T__)(val))); \ break; \ case 4: \ asm(op "l %1,"__percpu_arg(0) \ : "+m" (var) \ - : "ri" ((T__)(val))); \ + : "ri" ((pto_T__)(val))); \ break; \ case 8: \ asm(op "q %1,"__percpu_arg(0) \ : "+m" (var) \ - : "re" ((T__)(val))); \ + : "re" ((pto_T__)(val))); \ break; \ default: __bad_percpu_size(); \ } \ @@ -106,31 +106,31 @@ do { \ #define percpu_from_op(op, var, constraint) \ ({ \ - typeof(var) ret__; \ + typeof(var) pfo_ret__; \ switch (sizeof(var)) { \ case 1: \ asm(op "b "__percpu_arg(1)",%0" \ - : "=q" (ret__) \ + : "=q" (pfo_ret__) \ : constraint); \ break; \ case 2: \ asm(op "w "__percpu_arg(1)",%0" \ - : "=r" (ret__) \ + : "=r" (pfo_ret__) \ : constraint); \ break; \ case 4: \ asm(op "l "__percpu_arg(1)",%0" \ - : "=r" (ret__) \ + : "=r" (pfo_ret__) \ : constraint); \ break; \ case 8: \ asm(op "q "__percpu_arg(1)",%0" \ - : "=r" (ret__) \ + : "=r" (pfo_ret__) \ : constraint); \ break; \ default: __bad_percpu_size(); \ } \ - ret__; \ + pfo_ret__; \ }) /* diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index 9bd03193ecd4..5a5d6ce4bd55 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -60,6 +60,7 @@ #define DEFINE_PER_CPU_SECTION(type, name, sec) \ __PCPU_DUMMY_ATTRS char __pcpu_scope_##name; \ + extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \ __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \ __PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES __weak \ __typeof__(type) per_cpu__##name diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 519d6876590f..522f421ec213 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -226,20 +226,20 @@ do { \ extern void __bad_size_call_parameter(void); -#define __size_call_return(stem, variable) \ -({ typeof(variable) ret__; \ +#define __pcpu_size_call_return(stem, variable) \ +({ typeof(variable) pscr_ret__; \ switch(sizeof(variable)) { \ - case 1: ret__ = stem##1(variable);break; \ - case 2: ret__ = stem##2(variable);break; \ - case 4: ret__ = stem##4(variable);break; \ - case 8: ret__ = stem##8(variable);break; \ + case 1: pscr_ret__ = stem##1(variable);break; \ + case 2: pscr_ret__ = stem##2(variable);break; \ + case 4: pscr_ret__ = stem##4(variable);break; \ + case 8: pscr_ret__ = stem##8(variable);break; \ default: \ __bad_size_call_parameter();break; \ } \ - ret__; \ + pscr_ret__; \ }) -#define __size_call(stem, variable, ...) \ +#define __pcpu_size_call(stem, variable, ...) \ do { \ switch(sizeof(variable)) { \ case 1: stem##1(variable, __VA_ARGS__);break; \ @@ -299,7 +299,7 @@ do { \ # ifndef this_cpu_read_8 # define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp) # endif -# define this_cpu_read(pcp) __size_call_return(this_cpu_read_, (pcp)) +# define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp)) #endif #define _this_cpu_generic_to_op(pcp, val, op) \ @@ -322,7 +322,7 @@ do { \ # ifndef this_cpu_write_8 # define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) # endif -# define this_cpu_write(pcp, val) __size_call(this_cpu_write_, (pcp), (val)) +# define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, (pcp), (val)) #endif #ifndef this_cpu_add @@ -338,7 +338,7 @@ do { \ # ifndef this_cpu_add_8 # define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) # endif -# define this_cpu_add(pcp, val) __size_call(this_cpu_add_, (pcp), (val)) +# define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, (pcp), (val)) #endif #ifndef this_cpu_sub @@ -366,7 +366,7 @@ do { \ # ifndef this_cpu_and_8 # define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) # endif -# define this_cpu_and(pcp, val) __size_call(this_cpu_and_, (pcp), (val)) +# define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, (pcp), (val)) #endif #ifndef this_cpu_or @@ -382,7 +382,7 @@ do { \ # ifndef this_cpu_or_8 # define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) # endif -# define this_cpu_or(pcp, val) __size_call(this_cpu_or_, (pcp), (val)) +# define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val)) #endif #ifndef this_cpu_xor @@ -398,7 +398,7 @@ do { \ # ifndef this_cpu_xor_8 # define this_cpu_xor_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=) # endif -# define this_cpu_xor(pcp, val) __size_call(this_cpu_or_, (pcp), (val)) +# define this_cpu_xor(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val)) #endif /* @@ -428,7 +428,7 @@ do { \ # ifndef __this_cpu_read_8 # define __this_cpu_read_8(pcp) (*__this_cpu_ptr(&(pcp))) # endif -# define __this_cpu_read(pcp) __size_call_return(__this_cpu_read_, (pcp)) +# define __this_cpu_read(pcp) __pcpu_size_call_return(__this_cpu_read_, (pcp)) #endif #define __this_cpu_generic_to_op(pcp, val, op) \ @@ -449,7 +449,7 @@ do { \ # ifndef __this_cpu_write_8 # define __this_cpu_write_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), =) # endif -# define __this_cpu_write(pcp, val) __size_call(__this_cpu_write_, (pcp), (val)) +# define __this_cpu_write(pcp, val) __pcpu_size_call(__this_cpu_write_, (pcp), (val)) #endif #ifndef __this_cpu_add @@ -465,7 +465,7 @@ do { \ # ifndef __this_cpu_add_8 # define __this_cpu_add_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) # endif -# define __this_cpu_add(pcp, val) __size_call(__this_cpu_add_, (pcp), (val)) +# define __this_cpu_add(pcp, val) __pcpu_size_call(__this_cpu_add_, (pcp), (val)) #endif #ifndef __this_cpu_sub @@ -493,7 +493,7 @@ do { \ # ifndef __this_cpu_and_8 # define __this_cpu_and_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) # endif -# define __this_cpu_and(pcp, val) __size_call(__this_cpu_and_, (pcp), (val)) +# define __this_cpu_and(pcp, val) __pcpu_size_call(__this_cpu_and_, (pcp), (val)) #endif #ifndef __this_cpu_or @@ -509,7 +509,7 @@ do { \ # ifndef __this_cpu_or_8 # define __this_cpu_or_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) # endif -# define __this_cpu_or(pcp, val) __size_call(__this_cpu_or_, (pcp), (val)) +# define __this_cpu_or(pcp, val) __pcpu_size_call(__this_cpu_or_, (pcp), (val)) #endif #ifndef __this_cpu_xor @@ -525,7 +525,7 @@ do { \ # ifndef __this_cpu_xor_8 # define __this_cpu_xor_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=) # endif -# define __this_cpu_xor(pcp, val) __size_call(__this_cpu_xor_, (pcp), (val)) +# define __this_cpu_xor(pcp, val) __pcpu_size_call(__this_cpu_xor_, (pcp), (val)) #endif /* @@ -556,7 +556,7 @@ do { \ # ifndef irqsafe_cpu_add_8 # define irqsafe_cpu_add_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=) # endif -# define irqsafe_cpu_add(pcp, val) __size_call(irqsafe_cpu_add_, (pcp), (val)) +# define irqsafe_cpu_add(pcp, val) __pcpu_size_call(irqsafe_cpu_add_, (pcp), (val)) #endif #ifndef irqsafe_cpu_sub @@ -584,7 +584,7 @@ do { \ # ifndef irqsafe_cpu_and_8 # define irqsafe_cpu_and_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=) # endif -# define irqsafe_cpu_and(pcp, val) __size_call(irqsafe_cpu_and_, (val)) +# define irqsafe_cpu_and(pcp, val) __pcpu_size_call(irqsafe_cpu_and_, (val)) #endif #ifndef irqsafe_cpu_or @@ -600,7 +600,7 @@ do { \ # ifndef irqsafe_cpu_or_8 # define irqsafe_cpu_or_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=) # endif -# define irqsafe_cpu_or(pcp, val) __size_call(irqsafe_cpu_or_, (val)) +# define irqsafe_cpu_or(pcp, val) __pcpu_size_call(irqsafe_cpu_or_, (val)) #endif #ifndef irqsafe_cpu_xor @@ -616,7 +616,7 @@ do { \ # ifndef irqsafe_cpu_xor_8 # define irqsafe_cpu_xor_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=) # endif -# define irqsafe_cpu_xor(pcp, val) __size_call(irqsafe_cpu_xor_, (val)) +# define irqsafe_cpu_xor(pcp, val) __pcpu_size_call(irqsafe_cpu_xor_, (val)) #endif #endif /* __LINUX_PERCPU_H */ diff --git a/mm/percpu.c b/mm/percpu.c index ec158bb5f86d..e2e80fc78601 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -365,6 +365,7 @@ static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr) * 0 if noop, 1 if successfully extended, -errno on failure. */ static int pcpu_extend_area_map(struct pcpu_chunk *chunk) + __releases(lock) __acquires(lock) { int new_alloc; int *new; -- cgit v1.2.3 From f16250669d78a32bdfb27cec4d791e85141e11e2 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 29 Oct 2009 22:34:13 +0900 Subject: percpu: make percpu symbols in cpufreq unique This patch updates percpu related symbols in cpufreq such that percpu symbols are unique and don't clash with local symbols. This serves two purposes of decreasing the possibility of global percpu symbol collision and allowing dropping per_cpu__ prefix from percpu symbols. * drivers/cpufreq/cpufreq.c: s/policy_cpu/cpufreq_policy_cpu/ * drivers/cpufreq/freq_table.c: s/show_table/cpufreq_show_table/ * arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c: s/drv_data/acfreq_data/ s/old_perf/acfreq_old_perf/ Partly based on Rusty Russell's "alloc_percpu: rename percpu vars which cause name clashes" patch. Signed-off-by: Tejun Heo Cc: Rusty Russell --- arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c | 28 ++++++++++++++-------------- drivers/cpufreq/cpufreq.c | 16 ++++++++-------- drivers/cpufreq/freq_table.c | 12 ++++++------ 3 files changed, 28 insertions(+), 28 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c index 7d5c3b0ea8da..43eb3465dda7 100644 --- a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c +++ b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c @@ -68,9 +68,9 @@ struct acpi_cpufreq_data { unsigned int cpu_feature; }; -static DEFINE_PER_CPU(struct acpi_cpufreq_data *, drv_data); +static DEFINE_PER_CPU(struct acpi_cpufreq_data *, acfreq_data); -static DEFINE_PER_CPU(struct aperfmperf, old_perf); +static DEFINE_PER_CPU(struct aperfmperf, acfreq_old_perf); /* acpi_perf_data is a pointer to percpu data. */ static struct acpi_processor_performance *acpi_perf_data; @@ -214,14 +214,14 @@ static u32 get_cur_val(const struct cpumask *mask) if (unlikely(cpumask_empty(mask))) return 0; - switch (per_cpu(drv_data, cpumask_first(mask))->cpu_feature) { + switch (per_cpu(acfreq_data, cpumask_first(mask))->cpu_feature) { case SYSTEM_INTEL_MSR_CAPABLE: cmd.type = SYSTEM_INTEL_MSR_CAPABLE; cmd.addr.msr.reg = MSR_IA32_PERF_STATUS; break; case SYSTEM_IO_CAPABLE: cmd.type = SYSTEM_IO_CAPABLE; - perf = per_cpu(drv_data, cpumask_first(mask))->acpi_data; + perf = per_cpu(acfreq_data, cpumask_first(mask))->acpi_data; cmd.addr.io.port = perf->control_register.address; cmd.addr.io.bit_width = perf->control_register.bit_width; break; @@ -268,8 +268,8 @@ static unsigned int get_measured_perf(struct cpufreq_policy *policy, if (smp_call_function_single(cpu, read_measured_perf_ctrs, &perf, 1)) return 0; - ratio = calc_aperfmperf_ratio(&per_cpu(old_perf, cpu), &perf); - per_cpu(old_perf, cpu) = perf; + ratio = calc_aperfmperf_ratio(&per_cpu(acfreq_old_perf, cpu), &perf); + per_cpu(acfreq_old_perf, cpu) = perf; retval = (policy->cpuinfo.max_freq * ratio) >> APERFMPERF_SHIFT; @@ -278,7 +278,7 @@ static unsigned int get_measured_perf(struct cpufreq_policy *policy, static unsigned int get_cur_freq_on_cpu(unsigned int cpu) { - struct acpi_cpufreq_data *data = per_cpu(drv_data, cpu); + struct acpi_cpufreq_data *data = per_cpu(acfreq_data, cpu); unsigned int freq; unsigned int cached_freq; @@ -322,7 +322,7 @@ static unsigned int check_freqs(const struct cpumask *mask, unsigned int freq, static int acpi_cpufreq_target(struct cpufreq_policy *policy, unsigned int target_freq, unsigned int relation) { - struct acpi_cpufreq_data *data = per_cpu(drv_data, policy->cpu); + struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); struct acpi_processor_performance *perf; struct cpufreq_freqs freqs; struct drv_cmd cmd; @@ -416,7 +416,7 @@ out: static int acpi_cpufreq_verify(struct cpufreq_policy *policy) { - struct acpi_cpufreq_data *data = per_cpu(drv_data, policy->cpu); + struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); dprintk("acpi_cpufreq_verify\n"); @@ -563,7 +563,7 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) return -ENOMEM; data->acpi_data = per_cpu_ptr(acpi_perf_data, cpu); - per_cpu(drv_data, cpu) = data; + per_cpu(acfreq_data, cpu) = data; if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) acpi_cpufreq_driver.flags |= CPUFREQ_CONST_LOOPS; @@ -714,20 +714,20 @@ err_unreg: acpi_processor_unregister_performance(perf, cpu); err_free: kfree(data); - per_cpu(drv_data, cpu) = NULL; + per_cpu(acfreq_data, cpu) = NULL; return result; } static int acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy) { - struct acpi_cpufreq_data *data = per_cpu(drv_data, policy->cpu); + struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); dprintk("acpi_cpufreq_cpu_exit\n"); if (data) { cpufreq_frequency_table_put_attr(policy->cpu); - per_cpu(drv_data, policy->cpu) = NULL; + per_cpu(acfreq_data, policy->cpu) = NULL; acpi_processor_unregister_performance(data->acpi_data, policy->cpu); kfree(data); @@ -738,7 +738,7 @@ static int acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy) static int acpi_cpufreq_resume(struct cpufreq_policy *policy) { - struct acpi_cpufreq_data *data = per_cpu(drv_data, policy->cpu); + struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); dprintk("acpi_cpufreq_resume\n"); diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 3938c7817095..af93a8175c5e 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -64,14 +64,14 @@ static DEFINE_SPINLOCK(cpufreq_driver_lock); * - Lock should not be held across * __cpufreq_governor(data, CPUFREQ_GOV_STOP); */ -static DEFINE_PER_CPU(int, policy_cpu); +static DEFINE_PER_CPU(int, cpufreq_policy_cpu); static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem); #define lock_policy_rwsem(mode, cpu) \ int lock_policy_rwsem_##mode \ (int cpu) \ { \ - int policy_cpu = per_cpu(policy_cpu, cpu); \ + int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu); \ BUG_ON(policy_cpu == -1); \ down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu)); \ if (unlikely(!cpu_online(cpu))) { \ @@ -90,7 +90,7 @@ EXPORT_SYMBOL_GPL(lock_policy_rwsem_write); void unlock_policy_rwsem_read(int cpu) { - int policy_cpu = per_cpu(policy_cpu, cpu); + int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu); BUG_ON(policy_cpu == -1); up_read(&per_cpu(cpu_policy_rwsem, policy_cpu)); } @@ -98,7 +98,7 @@ EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read); void unlock_policy_rwsem_write(int cpu) { - int policy_cpu = per_cpu(policy_cpu, cpu); + int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu); BUG_ON(policy_cpu == -1); up_write(&per_cpu(cpu_policy_rwsem, policy_cpu)); } @@ -799,7 +799,7 @@ int cpufreq_add_dev_policy(unsigned int cpu, struct cpufreq_policy *policy, /* Set proper policy_cpu */ unlock_policy_rwsem_write(cpu); - per_cpu(policy_cpu, cpu) = managed_policy->cpu; + per_cpu(cpufreq_policy_cpu, cpu) = managed_policy->cpu; if (lock_policy_rwsem_write(cpu) < 0) { /* Should not go through policy unlock path */ @@ -906,7 +906,7 @@ int cpufreq_add_dev_interface(unsigned int cpu, struct cpufreq_policy *policy, if (!cpu_online(j)) continue; per_cpu(cpufreq_cpu_data, j) = policy; - per_cpu(policy_cpu, j) = policy->cpu; + per_cpu(cpufreq_policy_cpu, j) = policy->cpu; } spin_unlock_irqrestore(&cpufreq_driver_lock, flags); @@ -991,7 +991,7 @@ static int cpufreq_add_dev(struct sys_device *sys_dev) cpumask_copy(policy->cpus, cpumask_of(cpu)); /* Initially set CPU itself as the policy_cpu */ - per_cpu(policy_cpu, cpu) = cpu; + per_cpu(cpufreq_policy_cpu, cpu) = cpu; ret = (lock_policy_rwsem_write(cpu) < 0); WARN_ON(ret); @@ -1946,7 +1946,7 @@ static int __init cpufreq_core_init(void) int cpu; for_each_possible_cpu(cpu) { - per_cpu(policy_cpu, cpu) = -1; + per_cpu(cpufreq_policy_cpu, cpu) = -1; init_rwsem(&per_cpu(cpu_policy_rwsem, cpu)); } diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c index a9bd3a05a684..05432216e224 100644 --- a/drivers/cpufreq/freq_table.c +++ b/drivers/cpufreq/freq_table.c @@ -174,7 +174,7 @@ int cpufreq_frequency_table_target(struct cpufreq_policy *policy, } EXPORT_SYMBOL_GPL(cpufreq_frequency_table_target); -static DEFINE_PER_CPU(struct cpufreq_frequency_table *, show_table); +static DEFINE_PER_CPU(struct cpufreq_frequency_table *, cpufreq_show_table); /** * show_available_freqs - show available frequencies for the specified CPU */ @@ -185,10 +185,10 @@ static ssize_t show_available_freqs(struct cpufreq_policy *policy, char *buf) ssize_t count = 0; struct cpufreq_frequency_table *table; - if (!per_cpu(show_table, cpu)) + if (!per_cpu(cpufreq_show_table, cpu)) return -ENODEV; - table = per_cpu(show_table, cpu); + table = per_cpu(cpufreq_show_table, cpu); for (i = 0; (table[i].frequency != CPUFREQ_TABLE_END); i++) { if (table[i].frequency == CPUFREQ_ENTRY_INVALID) @@ -217,20 +217,20 @@ void cpufreq_frequency_table_get_attr(struct cpufreq_frequency_table *table, unsigned int cpu) { dprintk("setting show_table for cpu %u to %p\n", cpu, table); - per_cpu(show_table, cpu) = table; + per_cpu(cpufreq_show_table, cpu) = table; } EXPORT_SYMBOL_GPL(cpufreq_frequency_table_get_attr); void cpufreq_frequency_table_put_attr(unsigned int cpu) { dprintk("clearing show_table for cpu %u\n", cpu); - per_cpu(show_table, cpu) = NULL; + per_cpu(cpufreq_show_table, cpu) = NULL; } EXPORT_SYMBOL_GPL(cpufreq_frequency_table_put_attr); struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu) { - return per_cpu(show_table, cpu); + return per_cpu(cpufreq_show_table, cpu); } EXPORT_SYMBOL_GPL(cpufreq_frequency_get_table); -- cgit v1.2.3 From c6e22f9e3e99cc221fe01a0cacf94a9da8a59c31 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 29 Oct 2009 22:34:13 +0900 Subject: percpu: make percpu symbols in xen unique This patch updates percpu related symbols in xen such that percpu symbols are unique and don't clash with local symbols. This serves two purposes of decreasing the possibility of global percpu symbol collision and allowing dropping per_cpu__ prefix from percpu symbols. * arch/x86/xen/smp.c, arch/x86/xen/time.c, arch/ia64/xen/irq_xen.c: add xen_ prefix to percpu variables * arch/ia64/xen/time.c: add xen_ prefix to percpu variables, drop processed_ prefix and make them static Partly based on Rusty Russell's "alloc_percpu: rename percpu vars which cause name clashes" patch. Signed-off-by: Tejun Heo Cc: Rusty Russell Cc: Jeremy Fitzhardinge Cc: Chris Wright --- arch/ia64/xen/irq_xen.c | 131 +++++++++++++++++++++++++----------------------- arch/ia64/xen/time.c | 22 ++++---- arch/x86/xen/smp.c | 41 +++++++-------- arch/x86/xen/time.c | 24 ++++----- 4 files changed, 111 insertions(+), 107 deletions(-) (limited to 'arch') diff --git a/arch/ia64/xen/irq_xen.c b/arch/ia64/xen/irq_xen.c index f042e192d2fe..a3fb7cf9ae1d 100644 --- a/arch/ia64/xen/irq_xen.c +++ b/arch/ia64/xen/irq_xen.c @@ -63,19 +63,19 @@ xen_free_irq_vector(int vector) } -static DEFINE_PER_CPU(int, timer_irq) = -1; -static DEFINE_PER_CPU(int, ipi_irq) = -1; -static DEFINE_PER_CPU(int, resched_irq) = -1; -static DEFINE_PER_CPU(int, cmc_irq) = -1; -static DEFINE_PER_CPU(int, cmcp_irq) = -1; -static DEFINE_PER_CPU(int, cpep_irq) = -1; +static DEFINE_PER_CPU(int, xen_timer_irq) = -1; +static DEFINE_PER_CPU(int, xen_ipi_irq) = -1; +static DEFINE_PER_CPU(int, xen_resched_irq) = -1; +static DEFINE_PER_CPU(int, xen_cmc_irq) = -1; +static DEFINE_PER_CPU(int, xen_cmcp_irq) = -1; +static DEFINE_PER_CPU(int, xen_cpep_irq) = -1; #define NAME_SIZE 15 -static DEFINE_PER_CPU(char[NAME_SIZE], timer_name); -static DEFINE_PER_CPU(char[NAME_SIZE], ipi_name); -static DEFINE_PER_CPU(char[NAME_SIZE], resched_name); -static DEFINE_PER_CPU(char[NAME_SIZE], cmc_name); -static DEFINE_PER_CPU(char[NAME_SIZE], cmcp_name); -static DEFINE_PER_CPU(char[NAME_SIZE], cpep_name); +static DEFINE_PER_CPU(char[NAME_SIZE], xen_timer_name); +static DEFINE_PER_CPU(char[NAME_SIZE], xen_ipi_name); +static DEFINE_PER_CPU(char[NAME_SIZE], xen_resched_name); +static DEFINE_PER_CPU(char[NAME_SIZE], xen_cmc_name); +static DEFINE_PER_CPU(char[NAME_SIZE], xen_cmcp_name); +static DEFINE_PER_CPU(char[NAME_SIZE], xen_cpep_name); #undef NAME_SIZE struct saved_irq { @@ -144,64 +144,64 @@ __xen_register_percpu_irq(unsigned int cpu, unsigned int vec, if (xen_slab_ready) { switch (vec) { case IA64_TIMER_VECTOR: - snprintf(per_cpu(timer_name, cpu), - sizeof(per_cpu(timer_name, cpu)), + snprintf(per_cpu(xen_timer_name, cpu), + sizeof(per_cpu(xen_timer_name, cpu)), "%s%d", action->name, cpu); irq = bind_virq_to_irqhandler(VIRQ_ITC, cpu, action->handler, action->flags, - per_cpu(timer_name, cpu), action->dev_id); - per_cpu(timer_irq, cpu) = irq; + per_cpu(xen_timer_name, cpu), action->dev_id); + per_cpu(xen_timer_irq, cpu) = irq; break; case IA64_IPI_RESCHEDULE: - snprintf(per_cpu(resched_name, cpu), - sizeof(per_cpu(resched_name, cpu)), + snprintf(per_cpu(xen_resched_name, cpu), + sizeof(per_cpu(xen_resched_name, cpu)), "%s%d", action->name, cpu); irq = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR, cpu, action->handler, action->flags, - per_cpu(resched_name, cpu), action->dev_id); - per_cpu(resched_irq, cpu) = irq; + per_cpu(xen_resched_name, cpu), action->dev_id); + per_cpu(xen_resched_irq, cpu) = irq; break; case IA64_IPI_VECTOR: - snprintf(per_cpu(ipi_name, cpu), - sizeof(per_cpu(ipi_name, cpu)), + snprintf(per_cpu(xen_ipi_name, cpu), + sizeof(per_cpu(xen_ipi_name, cpu)), "%s%d", action->name, cpu); irq = bind_ipi_to_irqhandler(XEN_IPI_VECTOR, cpu, action->handler, action->flags, - per_cpu(ipi_name, cpu), action->dev_id); - per_cpu(ipi_irq, cpu) = irq; + per_cpu(xen_ipi_name, cpu), action->dev_id); + per_cpu(xen_ipi_irq, cpu) = irq; break; case IA64_CMC_VECTOR: - snprintf(per_cpu(cmc_name, cpu), - sizeof(per_cpu(cmc_name, cpu)), + snprintf(per_cpu(xen_cmc_name, cpu), + sizeof(per_cpu(xen_cmc_name, cpu)), "%s%d", action->name, cpu); irq = bind_virq_to_irqhandler(VIRQ_MCA_CMC, cpu, - action->handler, - action->flags, - per_cpu(cmc_name, cpu), - action->dev_id); - per_cpu(cmc_irq, cpu) = irq; + action->handler, + action->flags, + per_cpu(xen_cmc_name, cpu), + action->dev_id); + per_cpu(xen_cmc_irq, cpu) = irq; break; case IA64_CMCP_VECTOR: - snprintf(per_cpu(cmcp_name, cpu), - sizeof(per_cpu(cmcp_name, cpu)), + snprintf(per_cpu(xen_cmcp_name, cpu), + sizeof(per_cpu(xen_cmcp_name, cpu)), "%s%d", action->name, cpu); irq = bind_ipi_to_irqhandler(XEN_CMCP_VECTOR, cpu, - action->handler, - action->flags, - per_cpu(cmcp_name, cpu), - action->dev_id); - per_cpu(cmcp_irq, cpu) = irq; + action->handler, + action->flags, + per_cpu(xen_cmcp_name, cpu), + action->dev_id); + per_cpu(xen_cmcp_irq, cpu) = irq; break; case IA64_CPEP_VECTOR: - snprintf(per_cpu(cpep_name, cpu), - sizeof(per_cpu(cpep_name, cpu)), + snprintf(per_cpu(xen_cpep_name, cpu), + sizeof(per_cpu(xen_cpep_name, cpu)), "%s%d", action->name, cpu); irq = bind_ipi_to_irqhandler(XEN_CPEP_VECTOR, cpu, - action->handler, - action->flags, - per_cpu(cpep_name, cpu), - action->dev_id); - per_cpu(cpep_irq, cpu) = irq; + action->handler, + action->flags, + per_cpu(xen_cpep_name, cpu), + action->dev_id); + per_cpu(xen_cpep_irq, cpu) = irq; break; case IA64_CPE_VECTOR: case IA64_MCA_RENDEZ_VECTOR: @@ -275,30 +275,33 @@ unbind_evtchn_callback(struct notifier_block *nfb, if (action == CPU_DEAD) { /* Unregister evtchn. */ - if (per_cpu(cpep_irq, cpu) >= 0) { - unbind_from_irqhandler(per_cpu(cpep_irq, cpu), NULL); - per_cpu(cpep_irq, cpu) = -1; + if (per_cpu(xen_cpep_irq, cpu) >= 0) { + unbind_from_irqhandler(per_cpu(xen_cpep_irq, cpu), + NULL); + per_cpu(xen_cpep_irq, cpu) = -1; } - if (per_cpu(cmcp_irq, cpu) >= 0) { - unbind_from_irqhandler(per_cpu(cmcp_irq, cpu), NULL); - per_cpu(cmcp_irq, cpu) = -1; + if (per_cpu(xen_cmcp_irq, cpu) >= 0) { + unbind_from_irqhandler(per_cpu(xen_cmcp_irq, cpu), + NULL); + per_cpu(xen_cmcp_irq, cpu) = -1; } - if (per_cpu(cmc_irq, cpu) >= 0) { - unbind_from_irqhandler(per_cpu(cmc_irq, cpu), NULL); - per_cpu(cmc_irq, cpu) = -1; + if (per_cpu(xen_cmc_irq, cpu) >= 0) { + unbind_from_irqhandler(per_cpu(xen_cmc_irq, cpu), NULL); + per_cpu(xen_cmc_irq, cpu) = -1; } - if (per_cpu(ipi_irq, cpu) >= 0) { - unbind_from_irqhandler(per_cpu(ipi_irq, cpu), NULL); - per_cpu(ipi_irq, cpu) = -1; + if (per_cpu(xen_ipi_irq, cpu) >= 0) { + unbind_from_irqhandler(per_cpu(xen_ipi_irq, cpu), NULL); + per_cpu(xen_ipi_irq, cpu) = -1; } - if (per_cpu(resched_irq, cpu) >= 0) { - unbind_from_irqhandler(per_cpu(resched_irq, cpu), - NULL); - per_cpu(resched_irq, cpu) = -1; + if (per_cpu(xen_resched_irq, cpu) >= 0) { + unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu), + NULL); + per_cpu(xen_resched_irq, cpu) = -1; } - if (per_cpu(timer_irq, cpu) >= 0) { - unbind_from_irqhandler(per_cpu(timer_irq, cpu), NULL); - per_cpu(timer_irq, cpu) = -1; + if (per_cpu(xen_timer_irq, cpu) >= 0) { + unbind_from_irqhandler(per_cpu(xen_timer_irq, cpu), + NULL); + per_cpu(xen_timer_irq, cpu) = -1; } } return NOTIFY_OK; diff --git a/arch/ia64/xen/time.c b/arch/ia64/xen/time.c index dbeadb9c8e20..c1c544513e8d 100644 --- a/arch/ia64/xen/time.c +++ b/arch/ia64/xen/time.c @@ -34,15 +34,15 @@ #include "../kernel/fsyscall_gtod_data.h" -DEFINE_PER_CPU(struct vcpu_runstate_info, runstate); -DEFINE_PER_CPU(unsigned long, processed_stolen_time); -DEFINE_PER_CPU(unsigned long, processed_blocked_time); +static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate); +static DEFINE_PER_CPU(unsigned long, xen_stolen_time); +static DEFINE_PER_CPU(unsigned long, xen_blocked_time); /* taken from i386/kernel/time-xen.c */ static void xen_init_missing_ticks_accounting(int cpu) { struct vcpu_register_runstate_memory_area area; - struct vcpu_runstate_info *runstate = &per_cpu(runstate, cpu); + struct vcpu_runstate_info *runstate = &per_cpu(xen_runstate, cpu); int rc; memset(runstate, 0, sizeof(*runstate)); @@ -52,8 +52,8 @@ static void xen_init_missing_ticks_accounting(int cpu) &area); WARN_ON(rc && rc != -ENOSYS); - per_cpu(processed_blocked_time, cpu) = runstate->time[RUNSTATE_blocked]; - per_cpu(processed_stolen_time, cpu) = runstate->time[RUNSTATE_runnable] + per_cpu(xen_blocked_time, cpu) = runstate->time[RUNSTATE_blocked]; + per_cpu(xen_stolen_time, cpu) = runstate->time[RUNSTATE_runnable] + runstate->time[RUNSTATE_offline]; } @@ -68,7 +68,7 @@ static void get_runstate_snapshot(struct vcpu_runstate_info *res) BUG_ON(preemptible()); - state = &__get_cpu_var(runstate); + state = &__get_cpu_var(xen_runstate); /* * The runstate info is always updated by the hypervisor on @@ -103,12 +103,12 @@ consider_steal_time(unsigned long new_itm) * This function just checks and reject this effect. */ if (!time_after_eq(runstate.time[RUNSTATE_blocked], - per_cpu(processed_blocked_time, cpu))) + per_cpu(xen_blocked_time, cpu))) blocked = 0; if (!time_after_eq(runstate.time[RUNSTATE_runnable] + runstate.time[RUNSTATE_offline], - per_cpu(processed_stolen_time, cpu))) + per_cpu(xen_stolen_time, cpu))) stolen = 0; if (!time_after(delta_itm + new_itm, ia64_get_itc())) @@ -147,8 +147,8 @@ consider_steal_time(unsigned long new_itm) } else { local_cpu_data->itm_next = delta_itm + new_itm; } - per_cpu(processed_stolen_time, cpu) += NS_PER_TICK * stolen; - per_cpu(processed_blocked_time, cpu) += NS_PER_TICK * blocked; + per_cpu(xen_stolen_time, cpu) += NS_PER_TICK * stolen; + per_cpu(xen_blocked_time, cpu) += NS_PER_TICK * blocked; } return delta_itm; } diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c index fe03eeed7b48..1167d9830f5f 100644 --- a/arch/x86/xen/smp.c +++ b/arch/x86/xen/smp.c @@ -35,10 +35,10 @@ cpumask_var_t xen_cpu_initialized_map; -static DEFINE_PER_CPU(int, resched_irq); -static DEFINE_PER_CPU(int, callfunc_irq); -static DEFINE_PER_CPU(int, callfuncsingle_irq); -static DEFINE_PER_CPU(int, debug_irq) = -1; +static DEFINE_PER_CPU(int, xen_resched_irq); +static DEFINE_PER_CPU(int, xen_callfunc_irq); +static DEFINE_PER_CPU(int, xen_callfuncsingle_irq); +static DEFINE_PER_CPU(int, xen_debug_irq) = -1; static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id); static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id); @@ -103,7 +103,7 @@ static int xen_smp_intr_init(unsigned int cpu) NULL); if (rc < 0) goto fail; - per_cpu(resched_irq, cpu) = rc; + per_cpu(xen_resched_irq, cpu) = rc; callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu); rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR, @@ -114,7 +114,7 @@ static int xen_smp_intr_init(unsigned int cpu) NULL); if (rc < 0) goto fail; - per_cpu(callfunc_irq, cpu) = rc; + per_cpu(xen_callfunc_irq, cpu) = rc; debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu); rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt, @@ -122,7 +122,7 @@ static int xen_smp_intr_init(unsigned int cpu) debug_name, NULL); if (rc < 0) goto fail; - per_cpu(debug_irq, cpu) = rc; + per_cpu(xen_debug_irq, cpu) = rc; callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu); rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR, @@ -133,19 +133,20 @@ static int xen_smp_intr_init(unsigned int cpu) NULL); if (rc < 0) goto fail; - per_cpu(callfuncsingle_irq, cpu) = rc; + per_cpu(xen_callfuncsingle_irq, cpu) = rc; return 0; fail: - if (per_cpu(resched_irq, cpu) >= 0) - unbind_from_irqhandler(per_cpu(resched_irq, cpu), NULL); - if (per_cpu(callfunc_irq, cpu) >= 0) - unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL); - if (per_cpu(debug_irq, cpu) >= 0) - unbind_from_irqhandler(per_cpu(debug_irq, cpu), NULL); - if (per_cpu(callfuncsingle_irq, cpu) >= 0) - unbind_from_irqhandler(per_cpu(callfuncsingle_irq, cpu), NULL); + if (per_cpu(xen_resched_irq, cpu) >= 0) + unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu), NULL); + if (per_cpu(xen_callfunc_irq, cpu) >= 0) + unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu), NULL); + if (per_cpu(xen_debug_irq, cpu) >= 0) + unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu), NULL); + if (per_cpu(xen_callfuncsingle_irq, cpu) >= 0) + unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu), + NULL); return rc; } @@ -348,10 +349,10 @@ static void xen_cpu_die(unsigned int cpu) current->state = TASK_UNINTERRUPTIBLE; schedule_timeout(HZ/10); } - unbind_from_irqhandler(per_cpu(resched_irq, cpu), NULL); - unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL); - unbind_from_irqhandler(per_cpu(debug_irq, cpu), NULL); - unbind_from_irqhandler(per_cpu(callfuncsingle_irq, cpu), NULL); + unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu), NULL); + unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu), NULL); + unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu), NULL); + unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu), NULL); xen_uninit_lock_cpu(cpu); xen_teardown_timer(cpu); diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c index 0a5aa44299a5..26e37b787ad3 100644 --- a/arch/x86/xen/time.c +++ b/arch/x86/xen/time.c @@ -31,14 +31,14 @@ #define NS_PER_TICK (1000000000LL / HZ) /* runstate info updated by Xen */ -static DEFINE_PER_CPU(struct vcpu_runstate_info, runstate); +static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate); /* snapshots of runstate info */ -static DEFINE_PER_CPU(struct vcpu_runstate_info, runstate_snapshot); +static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate_snapshot); /* unused ns of stolen and blocked time */ -static DEFINE_PER_CPU(u64, residual_stolen); -static DEFINE_PER_CPU(u64, residual_blocked); +static DEFINE_PER_CPU(u64, xen_residual_stolen); +static DEFINE_PER_CPU(u64, xen_residual_blocked); /* return an consistent snapshot of 64-bit time/counter value */ static u64 get64(const u64 *p) @@ -79,7 +79,7 @@ static void get_runstate_snapshot(struct vcpu_runstate_info *res) BUG_ON(preemptible()); - state = &__get_cpu_var(runstate); + state = &__get_cpu_var(xen_runstate); /* * The runstate info is always updated by the hypervisor on @@ -97,14 +97,14 @@ static void get_runstate_snapshot(struct vcpu_runstate_info *res) /* return true when a vcpu could run but has no real cpu to run on */ bool xen_vcpu_stolen(int vcpu) { - return per_cpu(runstate, vcpu).state == RUNSTATE_runnable; + return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable; } static void setup_runstate_info(int cpu) { struct vcpu_register_runstate_memory_area area; - area.addr.v = &per_cpu(runstate, cpu); + area.addr.v = &per_cpu(xen_runstate, cpu); if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area, cpu, &area)) @@ -122,7 +122,7 @@ static void do_stolen_accounting(void) WARN_ON(state.state != RUNSTATE_running); - snap = &__get_cpu_var(runstate_snapshot); + snap = &__get_cpu_var(xen_runstate_snapshot); /* work out how much time the VCPU has not been runn*ing* */ blocked = state.time[RUNSTATE_blocked] - snap->time[RUNSTATE_blocked]; @@ -133,24 +133,24 @@ static void do_stolen_accounting(void) /* Add the appropriate number of ticks of stolen time, including any left-overs from last time. */ - stolen = runnable + offline + __get_cpu_var(residual_stolen); + stolen = runnable + offline + __get_cpu_var(xen_residual_stolen); if (stolen < 0) stolen = 0; ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen); - __get_cpu_var(residual_stolen) = stolen; + __get_cpu_var(xen_residual_stolen) = stolen; account_steal_ticks(ticks); /* Add the appropriate number of ticks of blocked time, including any left-overs from last time. */ - blocked += __get_cpu_var(residual_blocked); + blocked += __get_cpu_var(xen_residual_blocked); if (blocked < 0) blocked = 0; ticks = iter_div_u64_rem(blocked, NS_PER_TICK, &blocked); - __get_cpu_var(residual_blocked) = blocked; + __get_cpu_var(xen_residual_blocked) = blocked; account_idle_ticks(ticks); } -- cgit v1.2.3 From 0fe1e009541e925adc1748a605d8b66188e4b2ab Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 29 Oct 2009 22:34:14 +0900 Subject: percpu: make percpu symbols in x86 unique This patch updates percpu related symbols in x86 such that percpu symbols are unique and don't clash with local symbols. This serves two purposes of decreasing the possibility of global percpu symbol collision and allowing dropping per_cpu__ prefix from percpu symbols. * arch/x86/kernel/cpu/common.c: rename local variable to avoid collision * arch/x86/kvm/svm.c: s/svm_data/sd/ for local variables to avoid collision * arch/x86/kernel/cpu/cpu_debug.c: s/cpu_arr/cpud_arr/ s/priv_arr/cpud_priv_arr/ s/cpu_priv_count/cpud_priv_count/ * arch/x86/kernel/cpu/intel_cacheinfo.c: s/cpuid4_info/ici_cpuid4_info/ s/cache_kobject/ici_cache_kobject/ s/index_kobject/ici_index_kobject/ * arch/x86/kernel/ds.c: s/cpu_context/cpu_ds_context/ Partly based on Rusty Russell's "alloc_percpu: rename percpu vars which cause name clashes" patch. Signed-off-by: Tejun Heo Acked-by: (kvm) Avi Kivity Cc: Rusty Russell Cc: Thomas Gleixner Cc: Ingo Molnar Cc: H. Peter Anvin Cc: Marcelo Tosatti Cc: x86@kernel.org --- arch/x86/kernel/cpu/common.c | 8 ++--- arch/x86/kernel/cpu/cpu_debug.c | 30 ++++++++--------- arch/x86/kernel/cpu/intel_cacheinfo.c | 54 +++++++++++++++--------------- arch/x86/kernel/ds.c | 4 +-- arch/x86/kvm/svm.c | 63 +++++++++++++++++------------------ 5 files changed, 79 insertions(+), 80 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index cc25c2b4a567..3192f22f2fdd 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1093,7 +1093,7 @@ static void clear_all_debug_regs(void) void __cpuinit cpu_init(void) { - struct orig_ist *orig_ist; + struct orig_ist *oist; struct task_struct *me; struct tss_struct *t; unsigned long v; @@ -1102,7 +1102,7 @@ void __cpuinit cpu_init(void) cpu = stack_smp_processor_id(); t = &per_cpu(init_tss, cpu); - orig_ist = &per_cpu(orig_ist, cpu); + oist = &per_cpu(orig_ist, cpu); #ifdef CONFIG_NUMA if (cpu != 0 && percpu_read(node_number) == 0 && @@ -1143,12 +1143,12 @@ void __cpuinit cpu_init(void) /* * set up and load the per-CPU TSS */ - if (!orig_ist->ist[0]) { + if (!oist->ist[0]) { char *estacks = per_cpu(exception_stacks, cpu); for (v = 0; v < N_EXCEPTION_STACKS; v++) { estacks += exception_stack_sizes[v]; - orig_ist->ist[v] = t->x86_tss.ist[v] = + oist->ist[v] = t->x86_tss.ist[v] = (unsigned long)estacks; } } diff --git a/arch/x86/kernel/cpu/cpu_debug.c b/arch/x86/kernel/cpu/cpu_debug.c index dca325c03999..b368cd862997 100644 --- a/arch/x86/kernel/cpu/cpu_debug.c +++ b/arch/x86/kernel/cpu/cpu_debug.c @@ -30,9 +30,9 @@ #include #include -static DEFINE_PER_CPU(struct cpu_cpuX_base [CPU_REG_ALL_BIT], cpu_arr); -static DEFINE_PER_CPU(struct cpu_private * [MAX_CPU_FILES], priv_arr); -static DEFINE_PER_CPU(int, cpu_priv_count); +static DEFINE_PER_CPU(struct cpu_cpuX_base [CPU_REG_ALL_BIT], cpud_arr); +static DEFINE_PER_CPU(struct cpu_private * [MAX_CPU_FILES], cpud_priv_arr); +static DEFINE_PER_CPU(int, cpud_priv_count); static DEFINE_MUTEX(cpu_debug_lock); @@ -531,7 +531,7 @@ static int cpu_create_file(unsigned cpu, unsigned type, unsigned reg, /* Already intialized */ if (file == CPU_INDEX_BIT) - if (per_cpu(cpu_arr[type].init, cpu)) + if (per_cpu(cpud_arr[type].init, cpu)) return 0; priv = kzalloc(sizeof(*priv), GFP_KERNEL); @@ -543,8 +543,8 @@ static int cpu_create_file(unsigned cpu, unsigned type, unsigned reg, priv->reg = reg; priv->file = file; mutex_lock(&cpu_debug_lock); - per_cpu(priv_arr[type], cpu) = priv; - per_cpu(cpu_priv_count, cpu)++; + per_cpu(cpud_priv_arr[type], cpu) = priv; + per_cpu(cpud_priv_count, cpu)++; mutex_unlock(&cpu_debug_lock); if (file) @@ -552,10 +552,10 @@ static int cpu_create_file(unsigned cpu, unsigned type, unsigned reg, dentry, (void *)priv, &cpu_fops); else { debugfs_create_file(cpu_base[type].name, S_IRUGO, - per_cpu(cpu_arr[type].dentry, cpu), + per_cpu(cpud_arr[type].dentry, cpu), (void *)priv, &cpu_fops); mutex_lock(&cpu_debug_lock); - per_cpu(cpu_arr[type].init, cpu) = 1; + per_cpu(cpud_arr[type].init, cpu) = 1; mutex_unlock(&cpu_debug_lock); } @@ -615,7 +615,7 @@ static int cpu_init_allreg(unsigned cpu, struct dentry *dentry) if (!is_typeflag_valid(cpu, cpu_base[type].flag)) continue; cpu_dentry = debugfs_create_dir(cpu_base[type].name, dentry); - per_cpu(cpu_arr[type].dentry, cpu) = cpu_dentry; + per_cpu(cpud_arr[type].dentry, cpu) = cpu_dentry; if (type < CPU_TSS_BIT) err = cpu_init_msr(cpu, type, cpu_dentry); @@ -647,11 +647,11 @@ static int cpu_init_cpu(void) err = cpu_init_allreg(cpu, cpu_dentry); pr_info("cpu%d(%d) debug files %d\n", - cpu, nr_cpu_ids, per_cpu(cpu_priv_count, cpu)); - if (per_cpu(cpu_priv_count, cpu) > MAX_CPU_FILES) { + cpu, nr_cpu_ids, per_cpu(cpud_priv_count, cpu)); + if (per_cpu(cpud_priv_count, cpu) > MAX_CPU_FILES) { pr_err("Register files count %d exceeds limit %d\n", - per_cpu(cpu_priv_count, cpu), MAX_CPU_FILES); - per_cpu(cpu_priv_count, cpu) = MAX_CPU_FILES; + per_cpu(cpud_priv_count, cpu), MAX_CPU_FILES); + per_cpu(cpud_priv_count, cpu) = MAX_CPU_FILES; err = -ENFILE; } if (err) @@ -676,8 +676,8 @@ static void __exit cpu_debug_exit(void) debugfs_remove_recursive(cpu_debugfs_dir); for (cpu = 0; cpu < nr_cpu_ids; cpu++) - for (i = 0; i < per_cpu(cpu_priv_count, cpu); i++) - kfree(per_cpu(priv_arr[i], cpu)); + for (i = 0; i < per_cpu(cpud_priv_count, cpu); i++) + kfree(per_cpu(cpud_priv_arr[i], cpu)); } module_init(cpu_debug_init); diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c index 804c40e2bc3e..f5ccb4fa5a5d 100644 --- a/arch/x86/kernel/cpu/intel_cacheinfo.c +++ b/arch/x86/kernel/cpu/intel_cacheinfo.c @@ -512,8 +512,8 @@ unsigned int __cpuinit init_intel_cacheinfo(struct cpuinfo_x86 *c) #ifdef CONFIG_SYSFS /* pointer to _cpuid4_info array (for each cache leaf) */ -static DEFINE_PER_CPU(struct _cpuid4_info *, cpuid4_info); -#define CPUID4_INFO_IDX(x, y) (&((per_cpu(cpuid4_info, x))[y])) +static DEFINE_PER_CPU(struct _cpuid4_info *, ici_cpuid4_info); +#define CPUID4_INFO_IDX(x, y) (&((per_cpu(ici_cpuid4_info, x))[y])) #ifdef CONFIG_SMP static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index) @@ -526,7 +526,7 @@ static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index) if ((index == 3) && (c->x86_vendor == X86_VENDOR_AMD)) { struct cpuinfo_x86 *d; for_each_online_cpu(i) { - if (!per_cpu(cpuid4_info, i)) + if (!per_cpu(ici_cpuid4_info, i)) continue; d = &cpu_data(i); this_leaf = CPUID4_INFO_IDX(i, index); @@ -548,7 +548,7 @@ static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index) c->apicid >> index_msb) { cpumask_set_cpu(i, to_cpumask(this_leaf->shared_cpu_map)); - if (i != cpu && per_cpu(cpuid4_info, i)) { + if (i != cpu && per_cpu(ici_cpuid4_info, i)) { sibling_leaf = CPUID4_INFO_IDX(i, index); cpumask_set_cpu(cpu, to_cpumask( @@ -587,8 +587,8 @@ static void __cpuinit free_cache_attributes(unsigned int cpu) for (i = 0; i < num_cache_leaves; i++) cache_remove_shared_cpu_map(cpu, i); - kfree(per_cpu(cpuid4_info, cpu)); - per_cpu(cpuid4_info, cpu) = NULL; + kfree(per_cpu(ici_cpuid4_info, cpu)); + per_cpu(ici_cpuid4_info, cpu) = NULL; } static int @@ -627,15 +627,15 @@ static int __cpuinit detect_cache_attributes(unsigned int cpu) if (num_cache_leaves == 0) return -ENOENT; - per_cpu(cpuid4_info, cpu) = kzalloc( + per_cpu(ici_cpuid4_info, cpu) = kzalloc( sizeof(struct _cpuid4_info) * num_cache_leaves, GFP_KERNEL); - if (per_cpu(cpuid4_info, cpu) == NULL) + if (per_cpu(ici_cpuid4_info, cpu) == NULL) return -ENOMEM; smp_call_function_single(cpu, get_cpu_leaves, &retval, true); if (retval) { - kfree(per_cpu(cpuid4_info, cpu)); - per_cpu(cpuid4_info, cpu) = NULL; + kfree(per_cpu(ici_cpuid4_info, cpu)); + per_cpu(ici_cpuid4_info, cpu) = NULL; } return retval; @@ -647,7 +647,7 @@ static int __cpuinit detect_cache_attributes(unsigned int cpu) extern struct sysdev_class cpu_sysdev_class; /* from drivers/base/cpu.c */ /* pointer to kobject for cpuX/cache */ -static DEFINE_PER_CPU(struct kobject *, cache_kobject); +static DEFINE_PER_CPU(struct kobject *, ici_cache_kobject); struct _index_kobject { struct kobject kobj; @@ -656,8 +656,8 @@ struct _index_kobject { }; /* pointer to array of kobjects for cpuX/cache/indexY */ -static DEFINE_PER_CPU(struct _index_kobject *, index_kobject); -#define INDEX_KOBJECT_PTR(x, y) (&((per_cpu(index_kobject, x))[y])) +static DEFINE_PER_CPU(struct _index_kobject *, ici_index_kobject); +#define INDEX_KOBJECT_PTR(x, y) (&((per_cpu(ici_index_kobject, x))[y])) #define show_one_plus(file_name, object, val) \ static ssize_t show_##file_name \ @@ -876,10 +876,10 @@ static struct kobj_type ktype_percpu_entry = { static void __cpuinit cpuid4_cache_sysfs_exit(unsigned int cpu) { - kfree(per_cpu(cache_kobject, cpu)); - kfree(per_cpu(index_kobject, cpu)); - per_cpu(cache_kobject, cpu) = NULL; - per_cpu(index_kobject, cpu) = NULL; + kfree(per_cpu(ici_cache_kobject, cpu)); + kfree(per_cpu(ici_index_kobject, cpu)); + per_cpu(ici_cache_kobject, cpu) = NULL; + per_cpu(ici_index_kobject, cpu) = NULL; free_cache_attributes(cpu); } @@ -895,14 +895,14 @@ static int __cpuinit cpuid4_cache_sysfs_init(unsigned int cpu) return err; /* Allocate all required memory */ - per_cpu(cache_kobject, cpu) = + per_cpu(ici_cache_kobject, cpu) = kzalloc(sizeof(struct kobject), GFP_KERNEL); - if (unlikely(per_cpu(cache_kobject, cpu) == NULL)) + if (unlikely(per_cpu(ici_cache_kobject, cpu) == NULL)) goto err_out; - per_cpu(index_kobject, cpu) = kzalloc( + per_cpu(ici_index_kobject, cpu) = kzalloc( sizeof(struct _index_kobject) * num_cache_leaves, GFP_KERNEL); - if (unlikely(per_cpu(index_kobject, cpu) == NULL)) + if (unlikely(per_cpu(ici_index_kobject, cpu) == NULL)) goto err_out; return 0; @@ -926,7 +926,7 @@ static int __cpuinit cache_add_dev(struct sys_device * sys_dev) if (unlikely(retval < 0)) return retval; - retval = kobject_init_and_add(per_cpu(cache_kobject, cpu), + retval = kobject_init_and_add(per_cpu(ici_cache_kobject, cpu), &ktype_percpu_entry, &sys_dev->kobj, "%s", "cache"); if (retval < 0) { @@ -940,12 +940,12 @@ static int __cpuinit cache_add_dev(struct sys_device * sys_dev) this_object->index = i; retval = kobject_init_and_add(&(this_object->kobj), &ktype_cache, - per_cpu(cache_kobject, cpu), + per_cpu(ici_cache_kobject, cpu), "index%1lu", i); if (unlikely(retval)) { for (j = 0; j < i; j++) kobject_put(&(INDEX_KOBJECT_PTR(cpu, j)->kobj)); - kobject_put(per_cpu(cache_kobject, cpu)); + kobject_put(per_cpu(ici_cache_kobject, cpu)); cpuid4_cache_sysfs_exit(cpu); return retval; } @@ -953,7 +953,7 @@ static int __cpuinit cache_add_dev(struct sys_device * sys_dev) } cpumask_set_cpu(cpu, to_cpumask(cache_dev_map)); - kobject_uevent(per_cpu(cache_kobject, cpu), KOBJ_ADD); + kobject_uevent(per_cpu(ici_cache_kobject, cpu), KOBJ_ADD); return 0; } @@ -962,7 +962,7 @@ static void __cpuinit cache_remove_dev(struct sys_device * sys_dev) unsigned int cpu = sys_dev->id; unsigned long i; - if (per_cpu(cpuid4_info, cpu) == NULL) + if (per_cpu(ici_cpuid4_info, cpu) == NULL) return; if (!cpumask_test_cpu(cpu, to_cpumask(cache_dev_map))) return; @@ -970,7 +970,7 @@ static void __cpuinit cache_remove_dev(struct sys_device * sys_dev) for (i = 0; i < num_cache_leaves; i++) kobject_put(&(INDEX_KOBJECT_PTR(cpu, i)->kobj)); - kobject_put(per_cpu(cache_kobject, cpu)); + kobject_put(per_cpu(ici_cache_kobject, cpu)); cpuid4_cache_sysfs_exit(cpu); } diff --git a/arch/x86/kernel/ds.c b/arch/x86/kernel/ds.c index ef42a038f1a6..1c47390dd0e5 100644 --- a/arch/x86/kernel/ds.c +++ b/arch/x86/kernel/ds.c @@ -265,13 +265,13 @@ struct ds_context { int cpu; }; -static DEFINE_PER_CPU(struct ds_context *, cpu_context); +static DEFINE_PER_CPU(struct ds_context *, cpu_ds_context); static struct ds_context *ds_get_context(struct task_struct *task, int cpu) { struct ds_context **p_context = - (task ? &task->thread.ds_ctx : &per_cpu(cpu_context, cpu)); + (task ? &task->thread.ds_ctx : &per_cpu(cpu_ds_context, cpu)); struct ds_context *context = NULL; struct ds_context *new_context = NULL; diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index 944cc9c04b3c..6c79a14a3b6f 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c @@ -319,7 +319,7 @@ static void svm_hardware_disable(void *garbage) static void svm_hardware_enable(void *garbage) { - struct svm_cpu_data *svm_data; + struct svm_cpu_data *sd; uint64_t efer; struct descriptor_table gdt_descr; struct desc_struct *gdt; @@ -329,62 +329,61 @@ static void svm_hardware_enable(void *garbage) printk(KERN_ERR "svm_cpu_init: err EOPNOTSUPP on %d\n", me); return; } - svm_data = per_cpu(svm_data, me); + sd = per_cpu(svm_data, me); - if (!svm_data) { + if (!sd) { printk(KERN_ERR "svm_cpu_init: svm_data is NULL on %d\n", me); return; } - svm_data->asid_generation = 1; - svm_data->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; - svm_data->next_asid = svm_data->max_asid + 1; + sd->asid_generation = 1; + sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; + sd->next_asid = sd->max_asid + 1; kvm_get_gdt(&gdt_descr); gdt = (struct desc_struct *)gdt_descr.base; - svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); + sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); rdmsrl(MSR_EFER, efer); wrmsrl(MSR_EFER, efer | EFER_SVME); wrmsrl(MSR_VM_HSAVE_PA, - page_to_pfn(svm_data->save_area) << PAGE_SHIFT); + page_to_pfn(sd->save_area) << PAGE_SHIFT); } static void svm_cpu_uninit(int cpu) { - struct svm_cpu_data *svm_data - = per_cpu(svm_data, raw_smp_processor_id()); + struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id()); - if (!svm_data) + if (!sd) return; per_cpu(svm_data, raw_smp_processor_id()) = NULL; - __free_page(svm_data->save_area); - kfree(svm_data); + __free_page(sd->save_area); + kfree(sd); } static int svm_cpu_init(int cpu) { - struct svm_cpu_data *svm_data; + struct svm_cpu_data *sd; int r; - svm_data = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL); - if (!svm_data) + sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL); + if (!sd) return -ENOMEM; - svm_data->cpu = cpu; - svm_data->save_area = alloc_page(GFP_KERNEL); + sd->cpu = cpu; + sd->save_area = alloc_page(GFP_KERNEL); r = -ENOMEM; - if (!svm_data->save_area) + if (!sd->save_area) goto err_1; - per_cpu(svm_data, cpu) = svm_data; + per_cpu(svm_data, cpu) = sd; return 0; err_1: - kfree(svm_data); + kfree(sd); return r; } @@ -1094,16 +1093,16 @@ static void save_host_msrs(struct kvm_vcpu *vcpu) #endif } -static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *svm_data) +static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) { - if (svm_data->next_asid > svm_data->max_asid) { - ++svm_data->asid_generation; - svm_data->next_asid = 1; + if (sd->next_asid > sd->max_asid) { + ++sd->asid_generation; + sd->next_asid = 1; svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; } - svm->asid_generation = svm_data->asid_generation; - svm->vmcb->control.asid = svm_data->next_asid++; + svm->asid_generation = sd->asid_generation; + svm->vmcb->control.asid = sd->next_asid++; } static unsigned long svm_get_dr(struct kvm_vcpu *vcpu, int dr) @@ -2377,8 +2376,8 @@ static void reload_tss(struct kvm_vcpu *vcpu) { int cpu = raw_smp_processor_id(); - struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu); - svm_data->tss_desc->type = 9; /* available 32/64-bit TSS */ + struct svm_cpu_data *sd = per_cpu(svm_data, cpu); + sd->tss_desc->type = 9; /* available 32/64-bit TSS */ load_TR_desc(); } @@ -2386,12 +2385,12 @@ static void pre_svm_run(struct vcpu_svm *svm) { int cpu = raw_smp_processor_id(); - struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu); + struct svm_cpu_data *sd = per_cpu(svm_data, cpu); svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; /* FIXME: handle wraparound of asid_generation */ - if (svm->asid_generation != svm_data->asid_generation) - new_asid(svm, svm_data); + if (svm->asid_generation != sd->asid_generation) + new_asid(svm, sd); } static void svm_inject_nmi(struct kvm_vcpu *vcpu) -- cgit v1.2.3 From 6b7487fc6517736a6e32ccc0f8b46109c1b998ec Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 29 Oct 2009 22:34:14 +0900 Subject: percpu: make percpu symbols in powerpc unique This patch updates percpu related symbols in powerpc such that percpu symbols are unique and don't clash with local symbols. This serves two purposes of decreasing the possibility of global percpu symbol collision and allowing dropping per_cpu__ prefix from percpu symbols. * arch/powerpc/kernel/perf_callchain.c: s/callchain/cpu_perf_callchain/ * arch/powerpc/kernel/setup-common.c: s/pvr/cpu_pvr/ * arch/powerpc/platforms/pseries/dtl.c: s/dtl/cpu_dtl/ * arch/powerpc/platforms/cell/interrupt.c: s/iic/cpu_iic/ Partly based on Rusty Russell's "alloc_percpu: rename percpu vars which cause name clashes" patch. Signed-off-by: Tejun Heo Acked-by: Arnd Bergmann Acked-by: Benjamin Herrenschmidt Cc: Rusty Russell Cc: Paul Mackerras Cc: linuxppc-dev@ozlabs.org --- arch/powerpc/include/asm/smp.h | 2 +- arch/powerpc/kernel/perf_callchain.c | 4 ++-- arch/powerpc/kernel/setup-common.c | 4 ++-- arch/powerpc/kernel/smp.c | 2 +- arch/powerpc/platforms/cell/interrupt.c | 14 +++++++------- arch/powerpc/platforms/pseries/dtl.c | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) (limited to 'arch') diff --git a/arch/powerpc/include/asm/smp.h b/arch/powerpc/include/asm/smp.h index d9ea8d39c342..1d3b270d3083 100644 --- a/arch/powerpc/include/asm/smp.h +++ b/arch/powerpc/include/asm/smp.h @@ -37,7 +37,7 @@ extern void cpu_die(void); extern void smp_send_debugger_break(int cpu); extern void smp_message_recv(int); -DECLARE_PER_CPU(unsigned int, pvr); +DECLARE_PER_CPU(unsigned int, cpu_pvr); #ifdef CONFIG_HOTPLUG_CPU extern void fixup_irqs(cpumask_t map); diff --git a/arch/powerpc/kernel/perf_callchain.c b/arch/powerpc/kernel/perf_callchain.c index 0a03cf70d247..fe59c44f9b5b 100644 --- a/arch/powerpc/kernel/perf_callchain.c +++ b/arch/powerpc/kernel/perf_callchain.c @@ -497,11 +497,11 @@ static void perf_callchain_user_32(struct pt_regs *regs, * Since we can't get PMU interrupts inside a PMU interrupt handler, * we don't need separate irq and nmi entries here. */ -static DEFINE_PER_CPU(struct perf_callchain_entry, callchain); +static DEFINE_PER_CPU(struct perf_callchain_entry, cpu_perf_callchain); struct perf_callchain_entry *perf_callchain(struct pt_regs *regs) { - struct perf_callchain_entry *entry = &__get_cpu_var(callchain); + struct perf_callchain_entry *entry = &__get_cpu_var(cpu_perf_callchain); entry->nr = 0; diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c index 4271f7a655a3..aa5aeb947bc5 100644 --- a/arch/powerpc/kernel/setup-common.c +++ b/arch/powerpc/kernel/setup-common.c @@ -157,7 +157,7 @@ extern u32 cpu_temp_both(unsigned long cpu); #endif /* CONFIG_TAU */ #ifdef CONFIG_SMP -DEFINE_PER_CPU(unsigned int, pvr); +DEFINE_PER_CPU(unsigned int, cpu_pvr); #endif static int show_cpuinfo(struct seq_file *m, void *v) @@ -209,7 +209,7 @@ static int show_cpuinfo(struct seq_file *m, void *v) } #ifdef CONFIG_SMP - pvr = per_cpu(pvr, cpu_id); + pvr = per_cpu(cpu_pvr, cpu_id); #else pvr = mfspr(SPRN_PVR); #endif diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 9b86a74d2815..2ebb48410976 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -232,7 +232,7 @@ struct thread_info *current_set[NR_CPUS]; static void __devinit smp_store_cpu_info(int id) { - per_cpu(pvr, id) = mfspr(SPRN_PVR); + per_cpu(cpu_pvr, id) = mfspr(SPRN_PVR); } static void __init smp_create_idle(unsigned int cpu) diff --git a/arch/powerpc/platforms/cell/interrupt.c b/arch/powerpc/platforms/cell/interrupt.c index 882e47080e74..54bad901e4c9 100644 --- a/arch/powerpc/platforms/cell/interrupt.c +++ b/arch/powerpc/platforms/cell/interrupt.c @@ -54,7 +54,7 @@ struct iic { struct device_node *node; }; -static DEFINE_PER_CPU(struct iic, iic); +static DEFINE_PER_CPU(struct iic, cpu_iic); #define IIC_NODE_COUNT 2 static struct irq_host *iic_host; @@ -82,7 +82,7 @@ static void iic_unmask(unsigned int irq) static void iic_eoi(unsigned int irq) { - struct iic *iic = &__get_cpu_var(iic); + struct iic *iic = &__get_cpu_var(cpu_iic); out_be64(&iic->regs->prio, iic->eoi_stack[--iic->eoi_ptr]); BUG_ON(iic->eoi_ptr < 0); } @@ -146,7 +146,7 @@ static unsigned int iic_get_irq(void) struct iic *iic; unsigned int virq; - iic = &__get_cpu_var(iic); + iic = &__get_cpu_var(cpu_iic); *(unsigned long *) &pending = in_be64((u64 __iomem *) &iic->regs->pending_destr); if (!(pending.flags & CBE_IIC_IRQ_VALID)) @@ -161,12 +161,12 @@ static unsigned int iic_get_irq(void) void iic_setup_cpu(void) { - out_be64(&__get_cpu_var(iic).regs->prio, 0xff); + out_be64(&__get_cpu_var(cpu_iic).regs->prio, 0xff); } u8 iic_get_target_id(int cpu) { - return per_cpu(iic, cpu).target_id; + return per_cpu(cpu_iic, cpu).target_id; } EXPORT_SYMBOL_GPL(iic_get_target_id); @@ -181,7 +181,7 @@ static inline int iic_ipi_to_irq(int ipi) void iic_cause_IPI(int cpu, int mesg) { - out_be64(&per_cpu(iic, cpu).regs->generate, (0xf - mesg) << 4); + out_be64(&per_cpu(cpu_iic, cpu).regs->generate, (0xf - mesg) << 4); } struct irq_host *iic_get_irq_host(int node) @@ -348,7 +348,7 @@ static void __init init_one_iic(unsigned int hw_cpu, unsigned long addr, /* XXX FIXME: should locate the linux CPU number from the HW cpu * number properly. We are lucky for now */ - struct iic *iic = &per_cpu(iic, hw_cpu); + struct iic *iic = &per_cpu(cpu_iic, hw_cpu); iic->regs = ioremap(addr, sizeof(struct cbe_iic_thread_regs)); BUG_ON(iic->regs == NULL); diff --git a/arch/powerpc/platforms/pseries/dtl.c b/arch/powerpc/platforms/pseries/dtl.c index 937a544a236d..c5f3116b6ca5 100644 --- a/arch/powerpc/platforms/pseries/dtl.c +++ b/arch/powerpc/platforms/pseries/dtl.c @@ -54,7 +54,7 @@ struct dtl { int buf_entries; u64 last_idx; }; -static DEFINE_PER_CPU(struct dtl, dtl); +static DEFINE_PER_CPU(struct dtl, cpu_dtl); /* * Dispatch trace log event mask: @@ -261,7 +261,7 @@ static int dtl_init(void) /* set up the per-cpu log structures */ for_each_possible_cpu(i) { - struct dtl *dtl = &per_cpu(dtl, i); + struct dtl *dtl = &per_cpu(cpu_dtl, i); dtl->cpu = i; rc = dtl_setup_file(dtl); -- cgit v1.2.3 From 877105cc49f6e6ad32e3d63a214e8f537c0339ef Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 29 Oct 2009 22:34:14 +0900 Subject: percpu: make percpu symbols in ia64 unique This patch updates percpu related symbols in ia64 such that percpu symbols are unique and don't clash with local symbols. This serves two purposes of decreasing the possibility of global percpu symbol collision and allowing dropping per_cpu__ prefix from percpu symbols. * arch/ia64/kernel/setup.c: s/cpu_info/ia64_cpu_info/ Partly based on Rusty Russell's "alloc_percpu: rename percpu vars which cause name clashes" patch. Signed-off-by: Tejun Heo Reviewed-by: Christoph Lameter Cc: Rusty Russell Cc: Tony Luck Cc: Fenghua Yu Cc: linux-ia64@vger.kernel.org --- arch/ia64/include/asm/processor.h | 6 +++--- arch/ia64/kernel/head.S | 4 ++-- arch/ia64/kernel/ia64_ksyms.c | 2 +- arch/ia64/kernel/mca_asm.S | 2 +- arch/ia64/kernel/relocate_kernel.S | 2 +- arch/ia64/kernel/setup.c | 4 ++-- arch/ia64/mm/discontig.c | 5 +++-- arch/ia64/sn/kernel/sn2/sn2_smp.c | 8 ++++---- 8 files changed, 17 insertions(+), 16 deletions(-) (limited to 'arch') diff --git a/arch/ia64/include/asm/processor.h b/arch/ia64/include/asm/processor.h index 3eaeedf1aef2..7fa90f73f6be 100644 --- a/arch/ia64/include/asm/processor.h +++ b/arch/ia64/include/asm/processor.h @@ -229,7 +229,7 @@ struct cpuinfo_ia64 { #endif }; -DECLARE_PER_CPU(struct cpuinfo_ia64, cpu_info); +DECLARE_PER_CPU(struct cpuinfo_ia64, ia64_cpu_info); /* * The "local" data variable. It refers to the per-CPU data of the currently executing @@ -237,8 +237,8 @@ DECLARE_PER_CPU(struct cpuinfo_ia64, cpu_info); * Do not use the address of local_cpu_data, since it will be different from * cpu_data(smp_processor_id())! */ -#define local_cpu_data (&__ia64_per_cpu_var(cpu_info)) -#define cpu_data(cpu) (&per_cpu(cpu_info, cpu)) +#define local_cpu_data (&__ia64_per_cpu_var(ia64_cpu_info)) +#define cpu_data(cpu) (&per_cpu(ia64_cpu_info, cpu)) extern void print_cpu_info (struct cpuinfo_ia64 *); diff --git a/arch/ia64/kernel/head.S b/arch/ia64/kernel/head.S index 696eff28a0c4..17a9fba38930 100644 --- a/arch/ia64/kernel/head.S +++ b/arch/ia64/kernel/head.S @@ -1051,7 +1051,7 @@ END(ia64_delay_loop) * intermediate precision so that we can produce a full 64-bit result. */ GLOBAL_ENTRY(ia64_native_sched_clock) - addl r8=THIS_CPU(cpu_info) + IA64_CPUINFO_NSEC_PER_CYC_OFFSET,r0 + addl r8=THIS_CPU(ia64_cpu_info) + IA64_CPUINFO_NSEC_PER_CYC_OFFSET,r0 mov.m r9=ar.itc // fetch cycle-counter (35 cyc) ;; ldf8 f8=[r8] @@ -1077,7 +1077,7 @@ sched_clock = ia64_native_sched_clock #ifdef CONFIG_VIRT_CPU_ACCOUNTING GLOBAL_ENTRY(cycle_to_cputime) alloc r16=ar.pfs,1,0,0,0 - addl r8=THIS_CPU(cpu_info) + IA64_CPUINFO_NSEC_PER_CYC_OFFSET,r0 + addl r8=THIS_CPU(ia64_cpu_info) + IA64_CPUINFO_NSEC_PER_CYC_OFFSET,r0 ;; ldf8 f8=[r8] ;; diff --git a/arch/ia64/kernel/ia64_ksyms.c b/arch/ia64/kernel/ia64_ksyms.c index 14d39e300627..461b99902bf6 100644 --- a/arch/ia64/kernel/ia64_ksyms.c +++ b/arch/ia64/kernel/ia64_ksyms.c @@ -30,7 +30,7 @@ EXPORT_SYMBOL(max_low_pfn); /* defined by bootmem.c, but not exported by generic #endif #include -EXPORT_SYMBOL(per_cpu__cpu_info); +EXPORT_SYMBOL(per_cpu__ia64_cpu_info); #ifdef CONFIG_SMP EXPORT_SYMBOL(per_cpu__local_per_cpu_offset); #endif diff --git a/arch/ia64/kernel/mca_asm.S b/arch/ia64/kernel/mca_asm.S index 7461d2573d41..d5bdf9de36b6 100644 --- a/arch/ia64/kernel/mca_asm.S +++ b/arch/ia64/kernel/mca_asm.S @@ -59,7 +59,7 @@ ia64_do_tlb_purge: #define O(member) IA64_CPUINFO_##member##_OFFSET - GET_THIS_PADDR(r2, cpu_info) // load phys addr of cpu_info into r2 + GET_THIS_PADDR(r2, ia64_cpu_info) // load phys addr of cpu_info into r2 ;; addl r17=O(PTCE_STRIDE),r2 addl r2=O(PTCE_BASE),r2 diff --git a/arch/ia64/kernel/relocate_kernel.S b/arch/ia64/kernel/relocate_kernel.S index 32f6fc131fbe..c370e02f0061 100644 --- a/arch/ia64/kernel/relocate_kernel.S +++ b/arch/ia64/kernel/relocate_kernel.S @@ -61,7 +61,7 @@ GLOBAL_ENTRY(relocate_new_kernel) // purge all TC entries #define O(member) IA64_CPUINFO_##member##_OFFSET - GET_THIS_PADDR(r2, cpu_info) // load phys addr of cpu_info into r2 + GET_THIS_PADDR(r2, ia64_cpu_info) // load phys addr of cpu_info into r2 ;; addl r17=O(PTCE_STRIDE),r2 addl r2=O(PTCE_BASE),r2 diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c index bc1ef4ae828c..a1ea87919777 100644 --- a/arch/ia64/kernel/setup.c +++ b/arch/ia64/kernel/setup.c @@ -74,7 +74,7 @@ unsigned long __per_cpu_offset[NR_CPUS]; EXPORT_SYMBOL(__per_cpu_offset); #endif -DEFINE_PER_CPU(struct cpuinfo_ia64, cpu_info); +DEFINE_PER_CPU(struct cpuinfo_ia64, ia64_cpu_info); DEFINE_PER_CPU(unsigned long, local_per_cpu_offset); unsigned long ia64_cycles_per_usec; struct ia64_boot_param *ia64_boot_param; @@ -967,7 +967,7 @@ cpu_init (void) * depends on the data returned by identify_cpu(). We break the dependency by * accessing cpu_data() through the canonical per-CPU address. */ - cpu_info = cpu_data + ((char *) &__ia64_per_cpu_var(cpu_info) - __per_cpu_start); + cpu_info = cpu_data + ((char *) &__ia64_per_cpu_var(ia64_cpu_info) - __per_cpu_start); identify_cpu(cpu_info); #ifdef CONFIG_MCKINLEY diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c index 40e4c1fbf76b..19c4b2195dce 100644 --- a/arch/ia64/mm/discontig.c +++ b/arch/ia64/mm/discontig.c @@ -450,7 +450,8 @@ static void __init initialize_pernode_data(void) /* Set the node_data pointer for each per-cpu struct */ for_each_possible_early_cpu(cpu) { node = node_cpuid[cpu].nid; - per_cpu(cpu_info, cpu).node_data = mem_data[node].node_data; + per_cpu(ia64_cpu_info, cpu).node_data = + mem_data[node].node_data; } #else { @@ -458,7 +459,7 @@ static void __init initialize_pernode_data(void) cpu = 0; node = node_cpuid[cpu].nid; cpu0_cpu_info = (struct cpuinfo_ia64 *)(__phys_per_cpu_start + - ((char *)&per_cpu__cpu_info - __per_cpu_start)); + ((char *)&per_cpu__ia64_cpu_info - __per_cpu_start)); cpu0_cpu_info->node_data = mem_data[node].node_data; } #endif /* CONFIG_SMP */ diff --git a/arch/ia64/sn/kernel/sn2/sn2_smp.c b/arch/ia64/sn/kernel/sn2/sn2_smp.c index 1176506b2bae..e884ba4e031d 100644 --- a/arch/ia64/sn/kernel/sn2/sn2_smp.c +++ b/arch/ia64/sn/kernel/sn2/sn2_smp.c @@ -496,13 +496,13 @@ static int sn2_ptc_seq_show(struct seq_file *file, void *data) seq_printf(file, "cpu %d %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld\n", cpu, stat->ptc_l, stat->change_rid, stat->shub_ptc_flushes, stat->nodes_flushed, stat->deadlocks, - 1000 * stat->lock_itc_clocks / per_cpu(cpu_info, cpu).cyc_per_usec, - 1000 * stat->shub_itc_clocks / per_cpu(cpu_info, cpu).cyc_per_usec, - 1000 * stat->shub_itc_clocks_max / per_cpu(cpu_info, cpu).cyc_per_usec, + 1000 * stat->lock_itc_clocks / per_cpu(ia64_cpu_info, cpu).cyc_per_usec, + 1000 * stat->shub_itc_clocks / per_cpu(ia64_cpu_info, cpu).cyc_per_usec, + 1000 * stat->shub_itc_clocks_max / per_cpu(ia64_cpu_info, cpu).cyc_per_usec, stat->shub_ptc_flushes_not_my_mm, stat->deadlocks2, stat->shub_ipi_flushes, - 1000 * stat->shub_ipi_flushes_itc_clocks / per_cpu(cpu_info, cpu).cyc_per_usec); + 1000 * stat->shub_ipi_flushes_itc_clocks / per_cpu(ia64_cpu_info, cpu).cyc_per_usec); } return 0; } -- cgit v1.2.3 From 390dfd95c5df1ab3921dd388d11b2aee332c3f2c Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 29 Oct 2009 22:34:14 +0900 Subject: percpu: make misc percpu symbols unique This patch updates misc percpu related symbols such that percpu symbols are unique and don't clash with local symbols. This serves two purposes of decreasing the possibility of global percpu symbol collision and allowing dropping per_cpu__ prefix from percpu symbols. * drivers/crypto/padlock-aes.c: s/last_cword/paes_last_cword/ * drivers/lguest/x86/core.c: s/last_cpu/lg_last_cpu/ * drivers/s390/net/netiucv.c: rename the variable used in a macro to avoid clashing with percpu symbol * arch/mn10300/kernel/kprobes.c: replace current_ prefix with cur_ for static variables. Please note that percpu symbol current_kprobe can't be changed as it's used by generic code. Partly based on Rusty Russell's "alloc_percpu: rename percpu vars which cause name clashes" patch. Signed-off-by: Tejun Heo Reviewed-by: Christoph Lameter Cc: Rusty Russell Cc: Herbert Xu Cc: Chuck Ebbert Cc: David Howells Cc: Koichi Yasutake Cc: Ananth N Mavinakayanahalli Cc: Anil S Keshavamurthy Cc: David S. Miller Cc: Masami Hiramatsu Cc: Martin Schwidefsky Cc: Heiko Carstens Cc: linux390@de.ibm.com --- arch/mn10300/kernel/kprobes.c | 61 +++++++++++++++++++++---------------------- drivers/crypto/padlock-aes.c | 12 ++++----- drivers/lguest/x86/core.c | 6 ++--- drivers/s390/net/netiucv.c | 8 +++--- 4 files changed, 42 insertions(+), 45 deletions(-) (limited to 'arch') diff --git a/arch/mn10300/kernel/kprobes.c b/arch/mn10300/kernel/kprobes.c index dacafab00eb2..67e6389d625a 100644 --- a/arch/mn10300/kernel/kprobes.c +++ b/arch/mn10300/kernel/kprobes.c @@ -31,13 +31,13 @@ const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist); #define KPROBE_HIT_ACTIVE 0x00000001 #define KPROBE_HIT_SS 0x00000002 -static struct kprobe *current_kprobe; -static unsigned long current_kprobe_orig_pc; -static unsigned long current_kprobe_next_pc; -static int current_kprobe_ss_flags; +static struct kprobe *cur_kprobe; +static unsigned long cur_kprobe_orig_pc; +static unsigned long cur_kprobe_next_pc; +static int cur_kprobe_ss_flags; static unsigned long kprobe_status; -static kprobe_opcode_t current_kprobe_ss_buf[MAX_INSN_SIZE + 2]; -static unsigned long current_kprobe_bp_addr; +static kprobe_opcode_t cur_kprobe_ss_buf[MAX_INSN_SIZE + 2]; +static unsigned long cur_kprobe_bp_addr; DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; @@ -399,26 +399,25 @@ void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs) { unsigned long nextpc; - current_kprobe_orig_pc = regs->pc; - memcpy(current_kprobe_ss_buf, &p->ainsn.insn[0], MAX_INSN_SIZE); - regs->pc = (unsigned long) current_kprobe_ss_buf; + cur_kprobe_orig_pc = regs->pc; + memcpy(cur_kprobe_ss_buf, &p->ainsn.insn[0], MAX_INSN_SIZE); + regs->pc = (unsigned long) cur_kprobe_ss_buf; - nextpc = find_nextpc(regs, ¤t_kprobe_ss_flags); - if (current_kprobe_ss_flags & SINGLESTEP_PCREL) - current_kprobe_next_pc = - current_kprobe_orig_pc + (nextpc - regs->pc); + nextpc = find_nextpc(regs, &cur_kprobe_ss_flags); + if (cur_kprobe_ss_flags & SINGLESTEP_PCREL) + cur_kprobe_next_pc = cur_kprobe_orig_pc + (nextpc - regs->pc); else - current_kprobe_next_pc = nextpc; + cur_kprobe_next_pc = nextpc; /* branching instructions need special handling */ - if (current_kprobe_ss_flags & SINGLESTEP_BRANCH) + if (cur_kprobe_ss_flags & SINGLESTEP_BRANCH) nextpc = singlestep_branch_setup(regs); - current_kprobe_bp_addr = nextpc; + cur_kprobe_bp_addr = nextpc; *(u8 *) nextpc = BREAKPOINT_INSTRUCTION; - mn10300_dcache_flush_range2((unsigned) current_kprobe_ss_buf, - sizeof(current_kprobe_ss_buf)); + mn10300_dcache_flush_range2((unsigned) cur_kprobe_ss_buf, + sizeof(cur_kprobe_ss_buf)); mn10300_icache_inv(); } @@ -440,7 +439,7 @@ static inline int __kprobes kprobe_handler(struct pt_regs *regs) disarm_kprobe(p, regs); ret = 1; } else { - p = current_kprobe; + p = cur_kprobe; if (p->break_handler && p->break_handler(p, regs)) goto ss_probe; } @@ -464,7 +463,7 @@ static inline int __kprobes kprobe_handler(struct pt_regs *regs) } kprobe_status = KPROBE_HIT_ACTIVE; - current_kprobe = p; + cur_kprobe = p; if (p->pre_handler(p, regs)) { /* handler has already set things up, so skip ss setup */ return 1; @@ -491,8 +490,8 @@ no_kprobe: static void __kprobes resume_execution(struct kprobe *p, struct pt_regs *regs) { /* we may need to fixup regs/stack after singlestepping a call insn */ - if (current_kprobe_ss_flags & SINGLESTEP_BRANCH) { - regs->pc = current_kprobe_orig_pc; + if (cur_kprobe_ss_flags & SINGLESTEP_BRANCH) { + regs->pc = cur_kprobe_orig_pc; switch (p->ainsn.insn[0]) { case 0xcd: /* CALL (d16,PC) */ *(unsigned *) regs->sp = regs->mdr = regs->pc + 5; @@ -523,8 +522,8 @@ static void __kprobes resume_execution(struct kprobe *p, struct pt_regs *regs) } } - regs->pc = current_kprobe_next_pc; - current_kprobe_bp_addr = 0; + regs->pc = cur_kprobe_next_pc; + cur_kprobe_bp_addr = 0; } static inline int __kprobes post_kprobe_handler(struct pt_regs *regs) @@ -532,10 +531,10 @@ static inline int __kprobes post_kprobe_handler(struct pt_regs *regs) if (!kprobe_running()) return 0; - if (current_kprobe->post_handler) - current_kprobe->post_handler(current_kprobe, regs, 0); + if (cur_kprobe->post_handler) + cur_kprobe->post_handler(cur_kprobe, regs, 0); - resume_execution(current_kprobe, regs); + resume_execution(cur_kprobe, regs); reset_current_kprobe(); preempt_enable_no_resched(); return 1; @@ -545,12 +544,12 @@ static inline int __kprobes post_kprobe_handler(struct pt_regs *regs) static inline int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) { - if (current_kprobe->fault_handler && - current_kprobe->fault_handler(current_kprobe, regs, trapnr)) + if (cur_kprobe->fault_handler && + cur_kprobe->fault_handler(cur_kprobe, regs, trapnr)) return 1; if (kprobe_status & KPROBE_HIT_SS) { - resume_execution(current_kprobe, regs); + resume_execution(cur_kprobe, regs); reset_current_kprobe(); preempt_enable_no_resched(); } @@ -567,7 +566,7 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, switch (val) { case DIE_BREAKPOINT: - if (current_kprobe_bp_addr != args->regs->pc) { + if (cur_kprobe_bp_addr != args->regs->pc) { if (kprobe_handler(args->regs)) return NOTIFY_STOP; } else { diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c index a9952b1236b0..721d004a0235 100644 --- a/drivers/crypto/padlock-aes.c +++ b/drivers/crypto/padlock-aes.c @@ -64,7 +64,7 @@ struct aes_ctx { u32 *D; }; -static DEFINE_PER_CPU(struct cword *, last_cword); +static DEFINE_PER_CPU(struct cword *, paes_last_cword); /* Tells whether the ACE is capable to generate the extended key for a given key_len. */ @@ -152,9 +152,9 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, ok: for_each_online_cpu(cpu) - if (&ctx->cword.encrypt == per_cpu(last_cword, cpu) || - &ctx->cword.decrypt == per_cpu(last_cword, cpu)) - per_cpu(last_cword, cpu) = NULL; + if (&ctx->cword.encrypt == per_cpu(paes_last_cword, cpu) || + &ctx->cword.decrypt == per_cpu(paes_last_cword, cpu)) + per_cpu(paes_last_cword, cpu) = NULL; return 0; } @@ -166,7 +166,7 @@ static inline void padlock_reset_key(struct cword *cword) { int cpu = raw_smp_processor_id(); - if (cword != per_cpu(last_cword, cpu)) + if (cword != per_cpu(paes_last_cword, cpu)) #ifndef CONFIG_X86_64 asm volatile ("pushfl; popfl"); #else @@ -176,7 +176,7 @@ static inline void padlock_reset_key(struct cword *cword) static inline void padlock_store_cword(struct cword *cword) { - per_cpu(last_cword, raw_smp_processor_id()) = cword; + per_cpu(paes_last_cword, raw_smp_processor_id()) = cword; } /* diff --git a/drivers/lguest/x86/core.c b/drivers/lguest/x86/core.c index 6ae388849a3b..fb2b7ef7868e 100644 --- a/drivers/lguest/x86/core.c +++ b/drivers/lguest/x86/core.c @@ -69,7 +69,7 @@ static struct lguest_pages *lguest_pages(unsigned int cpu) (SWITCHER_ADDR + SHARED_SWITCHER_PAGES*PAGE_SIZE))[cpu]); } -static DEFINE_PER_CPU(struct lg_cpu *, last_cpu); +static DEFINE_PER_CPU(struct lg_cpu *, lg_last_cpu); /*S:010 * We approach the Switcher. @@ -90,8 +90,8 @@ static void copy_in_guest_info(struct lg_cpu *cpu, struct lguest_pages *pages) * meanwhile). If that's not the case, we pretend everything in the * Guest has changed. */ - if (__get_cpu_var(last_cpu) != cpu || cpu->last_pages != pages) { - __get_cpu_var(last_cpu) = cpu; + if (__get_cpu_var(lg_last_cpu) != cpu || cpu->last_pages != pages) { + __get_cpu_var(lg_last_cpu) = cpu; cpu->last_pages = pages; cpu->changed = CHANGED_ALL; } diff --git a/drivers/s390/net/netiucv.c b/drivers/s390/net/netiucv.c index c84eadd3602a..14e61441ba0b 100644 --- a/drivers/s390/net/netiucv.c +++ b/drivers/s390/net/netiucv.c @@ -113,11 +113,9 @@ static inline int iucv_dbf_passes(debug_info_t *dbf_grp, int level) #define IUCV_DBF_TEXT_(name, level, text...) \ do { \ if (iucv_dbf_passes(iucv_dbf_##name, level)) { \ - char* iucv_dbf_txt_buf = \ - get_cpu_var(iucv_dbf_txt_buf); \ - sprintf(iucv_dbf_txt_buf, text); \ - debug_text_event(iucv_dbf_##name, level, \ - iucv_dbf_txt_buf); \ + char* __buf = get_cpu_var(iucv_dbf_txt_buf); \ + sprintf(__buf, text); \ + debug_text_event(iucv_dbf_##name, level, __buf); \ put_cpu_var(iucv_dbf_txt_buf); \ } \ } while (0) -- cgit v1.2.3 From dd17c8f72993f9461e9c19250e3f155d6d99df22 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 29 Oct 2009 22:34:15 +0900 Subject: percpu: remove per_cpu__ prefix. Now that the return from alloc_percpu is compatible with the address of per-cpu vars, it makes sense to hand around the address of per-cpu variables. To make this sane, we remove the per_cpu__ prefix we used created to stop people accidentally using these vars directly. Now we have sparse, we can use that (next patch). tj: * Updated to convert stuff which were missed by or added after the original patch. * Kill per_cpu_var() macro. Signed-off-by: Rusty Russell Signed-off-by: Tejun Heo Reviewed-by: Christoph Lameter --- arch/blackfin/mach-common/entry.S | 4 ++-- arch/cris/arch-v10/kernel/entry.S | 2 +- arch/cris/arch-v32/mm/mmu.S | 2 +- arch/ia64/include/asm/percpu.h | 4 ++-- arch/ia64/kernel/ia64_ksyms.c | 4 ++-- arch/ia64/mm/discontig.c | 2 +- arch/microblaze/include/asm/entry.h | 2 +- arch/parisc/lib/fixup.S | 8 +++---- arch/powerpc/platforms/pseries/hvCall.S | 2 +- arch/sparc/kernel/nmi.c | 6 +++--- arch/sparc/kernel/rtrap_64.S | 8 +++---- arch/x86/include/asm/percpu.h | 37 +++++++++++++++------------------ arch/x86/include/asm/system.h | 8 +++---- arch/x86/kernel/apic/nmi.c | 6 +++--- arch/x86/kernel/head_32.S | 6 +++--- arch/x86/kernel/vmlinux.lds.S | 4 ++-- arch/x86/xen/xen-asm_32.S | 4 ++-- include/asm-generic/percpu.h | 12 +++++------ include/linux/percpu-defs.h | 18 ++++++---------- include/linux/percpu.h | 5 ++--- include/linux/vmstat.h | 8 +++---- kernel/rcutorture.c | 8 +++---- kernel/trace/trace.c | 6 +++--- kernel/trace/trace_functions_graph.c | 4 ++-- 24 files changed, 80 insertions(+), 90 deletions(-) (limited to 'arch') diff --git a/arch/blackfin/mach-common/entry.S b/arch/blackfin/mach-common/entry.S index 1e7cac23e25f..a3ea7e9fe43b 100644 --- a/arch/blackfin/mach-common/entry.S +++ b/arch/blackfin/mach-common/entry.S @@ -835,8 +835,8 @@ ENDPROC(_resume) ENTRY(_ret_from_exception) #ifdef CONFIG_IPIPE - p2.l = _per_cpu__ipipe_percpu_domain; - p2.h = _per_cpu__ipipe_percpu_domain; + p2.l = _ipipe_percpu_domain; + p2.h = _ipipe_percpu_domain; r0.l = _ipipe_root; r0.h = _ipipe_root; r2 = [p2]; diff --git a/arch/cris/arch-v10/kernel/entry.S b/arch/cris/arch-v10/kernel/entry.S index 2c18d08cd913..c52bef39e250 100644 --- a/arch/cris/arch-v10/kernel/entry.S +++ b/arch/cris/arch-v10/kernel/entry.S @@ -358,7 +358,7 @@ mmu_bus_fault: 1: btstq 12, $r1 ; Refill? bpl 2f lsrq 24, $r1 ; Get PGD index (bit 24-31) - move.d [per_cpu__current_pgd], $r0 ; PGD for the current process + move.d [current_pgd], $r0 ; PGD for the current process move.d [$r0+$r1.d], $r0 ; Get PMD beq 2f nop diff --git a/arch/cris/arch-v32/mm/mmu.S b/arch/cris/arch-v32/mm/mmu.S index 2238d154bde3..f125d912e140 100644 --- a/arch/cris/arch-v32/mm/mmu.S +++ b/arch/cris/arch-v32/mm/mmu.S @@ -115,7 +115,7 @@ #ifdef CONFIG_SMP move $s7, $acr ; PGD #else - move.d per_cpu__current_pgd, $acr ; PGD + move.d current_pgd, $acr ; PGD #endif ; Look up PMD in PGD lsrq 24, $r0 ; Get PMD index into PGD (bit 24-31) diff --git a/arch/ia64/include/asm/percpu.h b/arch/ia64/include/asm/percpu.h index 30cf46534dd2..f7c00a5e0e2b 100644 --- a/arch/ia64/include/asm/percpu.h +++ b/arch/ia64/include/asm/percpu.h @@ -9,7 +9,7 @@ #define PERCPU_ENOUGH_ROOM PERCPU_PAGE_SIZE #ifdef __ASSEMBLY__ -# define THIS_CPU(var) (per_cpu__##var) /* use this to mark accesses to per-CPU variables... */ +# define THIS_CPU(var) (var) /* use this to mark accesses to per-CPU variables... */ #else /* !__ASSEMBLY__ */ @@ -39,7 +39,7 @@ extern void *per_cpu_init(void); * On the positive side, using __ia64_per_cpu_var() instead of __get_cpu_var() is slightly * more efficient. */ -#define __ia64_per_cpu_var(var) per_cpu__##var +#define __ia64_per_cpu_var(var) var #include diff --git a/arch/ia64/kernel/ia64_ksyms.c b/arch/ia64/kernel/ia64_ksyms.c index 461b99902bf6..7f4a0ed24152 100644 --- a/arch/ia64/kernel/ia64_ksyms.c +++ b/arch/ia64/kernel/ia64_ksyms.c @@ -30,9 +30,9 @@ EXPORT_SYMBOL(max_low_pfn); /* defined by bootmem.c, but not exported by generic #endif #include -EXPORT_SYMBOL(per_cpu__ia64_cpu_info); +EXPORT_SYMBOL(ia64_cpu_info); #ifdef CONFIG_SMP -EXPORT_SYMBOL(per_cpu__local_per_cpu_offset); +EXPORT_SYMBOL(local_per_cpu_offset); #endif #include diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c index 19c4b2195dce..8d586d1e2515 100644 --- a/arch/ia64/mm/discontig.c +++ b/arch/ia64/mm/discontig.c @@ -459,7 +459,7 @@ static void __init initialize_pernode_data(void) cpu = 0; node = node_cpuid[cpu].nid; cpu0_cpu_info = (struct cpuinfo_ia64 *)(__phys_per_cpu_start + - ((char *)&per_cpu__ia64_cpu_info - __per_cpu_start)); + ((char *)&ia64_cpu_info - __per_cpu_start)); cpu0_cpu_info->node_data = mem_data[node].node_data; } #endif /* CONFIG_SMP */ diff --git a/arch/microblaze/include/asm/entry.h b/arch/microblaze/include/asm/entry.h index 61abbd232640..ec89f2ad0fe1 100644 --- a/arch/microblaze/include/asm/entry.h +++ b/arch/microblaze/include/asm/entry.h @@ -21,7 +21,7 @@ * places */ -#define PER_CPU(var) per_cpu__##var +#define PER_CPU(var) var # ifndef __ASSEMBLY__ DECLARE_PER_CPU(unsigned int, KSP); /* Saved kernel stack pointer */ diff --git a/arch/parisc/lib/fixup.S b/arch/parisc/lib/fixup.S index d172d4245cdc..f8c45cc2947d 100644 --- a/arch/parisc/lib/fixup.S +++ b/arch/parisc/lib/fixup.S @@ -36,8 +36,8 @@ #endif /* t2 = &__per_cpu_offset[smp_processor_id()]; */ LDREGX \t2(\t1),\t2 - addil LT%per_cpu__exception_data,%r27 - LDREG RT%per_cpu__exception_data(%r1),\t1 + addil LT%exception_data,%r27 + LDREG RT%exception_data(%r1),\t1 /* t1 = &__get_cpu_var(exception_data) */ add,l \t1,\t2,\t1 /* t1 = t1->fault_ip */ @@ -46,8 +46,8 @@ #else .macro get_fault_ip t1 t2 /* t1 = &__get_cpu_var(exception_data) */ - addil LT%per_cpu__exception_data,%r27 - LDREG RT%per_cpu__exception_data(%r1),\t2 + addil LT%exception_data,%r27 + LDREG RT%exception_data(%r1),\t2 /* t1 = t2->fault_ip */ LDREG EXCDATA_IP(\t2), \t1 .endm diff --git a/arch/powerpc/platforms/pseries/hvCall.S b/arch/powerpc/platforms/pseries/hvCall.S index c1427b3634ec..580f789cae7f 100644 --- a/arch/powerpc/platforms/pseries/hvCall.S +++ b/arch/powerpc/platforms/pseries/hvCall.S @@ -55,7 +55,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_PURR); \ /* calculate address of stat structure r4 = opcode */ \ srdi r4,r4,2; /* index into array */ \ mulli r4,r4,HCALL_STAT_SIZE; \ - LOAD_REG_ADDR(r7, per_cpu__hcall_stats); \ + LOAD_REG_ADDR(r7, hcall_stats); \ add r4,r4,r7; \ ld r7,PACA_DATA_OFFSET(r13); /* per cpu offset */ \ add r4,r4,r7; \ diff --git a/arch/sparc/kernel/nmi.c b/arch/sparc/kernel/nmi.c index f30f4a1ead23..2ad288ff99a4 100644 --- a/arch/sparc/kernel/nmi.c +++ b/arch/sparc/kernel/nmi.c @@ -112,13 +112,13 @@ notrace __kprobes void perfctr_irq(int irq, struct pt_regs *regs) touched = 1; } if (!touched && __get_cpu_var(last_irq_sum) == sum) { - __this_cpu_inc(per_cpu_var(alert_counter)); - if (__this_cpu_read(per_cpu_var(alert_counter)) == 30 * nmi_hz) + __this_cpu_inc(alert_counter); + if (__this_cpu_read(alert_counter) == 30 * nmi_hz) die_nmi("BUG: NMI Watchdog detected LOCKUP", regs, panic_on_timeout); } else { __get_cpu_var(last_irq_sum) = sum; - __this_cpu_write(per_cpu_var(alert_counter), 0); + __this_cpu_write(alert_counter, 0); } if (__get_cpu_var(wd_enabled)) { write_pic(picl_value(nmi_hz)); diff --git a/arch/sparc/kernel/rtrap_64.S b/arch/sparc/kernel/rtrap_64.S index fd3cee4d117c..1ddec403f512 100644 --- a/arch/sparc/kernel/rtrap_64.S +++ b/arch/sparc/kernel/rtrap_64.S @@ -149,11 +149,11 @@ rtrap_nmi: ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 rtrap_irq: rtrap: #ifndef CONFIG_SMP - sethi %hi(per_cpu____cpu_data), %l0 - lduw [%l0 + %lo(per_cpu____cpu_data)], %l1 + sethi %hi(__cpu_data), %l0 + lduw [%l0 + %lo(__cpu_data)], %l1 #else - sethi %hi(per_cpu____cpu_data), %l0 - or %l0, %lo(per_cpu____cpu_data), %l0 + sethi %hi(__cpu_data), %l0 + or %l0, %lo(__cpu_data), %l0 lduw [%l0 + %g5], %l1 #endif cmp %l1, 0 diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index 0c44196b78ac..4c170ccc72ed 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -25,19 +25,18 @@ */ #ifdef CONFIG_SMP #define PER_CPU(var, reg) \ - __percpu_mov_op %__percpu_seg:per_cpu__this_cpu_off, reg; \ - lea per_cpu__##var(reg), reg -#define PER_CPU_VAR(var) %__percpu_seg:per_cpu__##var + __percpu_mov_op %__percpu_seg:this_cpu_off, reg; \ + lea var(reg), reg +#define PER_CPU_VAR(var) %__percpu_seg:var #else /* ! SMP */ -#define PER_CPU(var, reg) \ - __percpu_mov_op $per_cpu__##var, reg -#define PER_CPU_VAR(var) per_cpu__##var +#define PER_CPU(var, reg) __percpu_mov_op $var, reg +#define PER_CPU_VAR(var) var #endif /* SMP */ #ifdef CONFIG_X86_64_SMP #define INIT_PER_CPU_VAR(var) init_per_cpu__##var #else -#define INIT_PER_CPU_VAR(var) per_cpu__##var +#define INIT_PER_CPU_VAR(var) var #endif #else /* ...!ASSEMBLY */ @@ -60,12 +59,12 @@ * There also must be an entry in vmlinux_64.lds.S */ #define DECLARE_INIT_PER_CPU(var) \ - extern typeof(per_cpu_var(var)) init_per_cpu_var(var) + extern typeof(var) init_per_cpu_var(var) #ifdef CONFIG_X86_64_SMP #define init_per_cpu_var(var) init_per_cpu__##var #else -#define init_per_cpu_var(var) per_cpu_var(var) +#define init_per_cpu_var(var) var #endif /* For arch-specific code, we can use direct single-insn ops (they @@ -142,16 +141,14 @@ do { \ * per-thread variables implemented as per-cpu variables and thus * stable for the duration of the respective task. */ -#define percpu_read(var) percpu_from_op("mov", per_cpu__##var, \ - "m" (per_cpu__##var)) -#define percpu_read_stable(var) percpu_from_op("mov", per_cpu__##var, \ - "p" (&per_cpu__##var)) -#define percpu_write(var, val) percpu_to_op("mov", per_cpu__##var, val) -#define percpu_add(var, val) percpu_to_op("add", per_cpu__##var, val) -#define percpu_sub(var, val) percpu_to_op("sub", per_cpu__##var, val) -#define percpu_and(var, val) percpu_to_op("and", per_cpu__##var, val) -#define percpu_or(var, val) percpu_to_op("or", per_cpu__##var, val) -#define percpu_xor(var, val) percpu_to_op("xor", per_cpu__##var, val) +#define percpu_read(var) percpu_from_op("mov", var, "m" (var)) +#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var))) +#define percpu_write(var, val) percpu_to_op("mov", var, val) +#define percpu_add(var, val) percpu_to_op("add", var, val) +#define percpu_sub(var, val) percpu_to_op("sub", var, val) +#define percpu_and(var, val) percpu_to_op("and", var, val) +#define percpu_or(var, val) percpu_to_op("or", var, val) +#define percpu_xor(var, val) percpu_to_op("xor", var, val) #define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) #define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) @@ -236,7 +233,7 @@ do { \ ({ \ int old__; \ asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0" \ - : "=r" (old__), "+m" (per_cpu__##var) \ + : "=r" (old__), "+m" (var) \ : "dIr" (bit)); \ old__; \ }) diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h index f08f97374892..de10c19d9558 100644 --- a/arch/x86/include/asm/system.h +++ b/arch/x86/include/asm/system.h @@ -31,7 +31,7 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p, "movl %P[task_canary](%[next]), %%ebx\n\t" \ "movl %%ebx, "__percpu_arg([stack_canary])"\n\t" #define __switch_canary_oparam \ - , [stack_canary] "=m" (per_cpu_var(stack_canary.canary)) + , [stack_canary] "=m" (stack_canary.canary) #define __switch_canary_iparam \ , [task_canary] "i" (offsetof(struct task_struct, stack_canary)) #else /* CC_STACKPROTECTOR */ @@ -113,7 +113,7 @@ do { \ "movq %P[task_canary](%%rsi),%%r8\n\t" \ "movq %%r8,"__percpu_arg([gs_canary])"\n\t" #define __switch_canary_oparam \ - , [gs_canary] "=m" (per_cpu_var(irq_stack_union.stack_canary)) + , [gs_canary] "=m" (irq_stack_union.stack_canary) #define __switch_canary_iparam \ , [task_canary] "i" (offsetof(struct task_struct, stack_canary)) #else /* CC_STACKPROTECTOR */ @@ -134,7 +134,7 @@ do { \ __switch_canary \ "movq %P[thread_info](%%rsi),%%r8\n\t" \ "movq %%rax,%%rdi\n\t" \ - "testl %[_tif_fork],%P[ti_flags](%%r8)\n\t" \ + "testl %[_tif_fork],%P[ti_flags](%%r8)\n\t" \ "jnz ret_from_fork\n\t" \ RESTORE_CONTEXT \ : "=a" (last) \ @@ -144,7 +144,7 @@ do { \ [ti_flags] "i" (offsetof(struct thread_info, flags)), \ [_tif_fork] "i" (_TIF_FORK), \ [thread_info] "i" (offsetof(struct task_struct, stack)), \ - [current_task] "m" (per_cpu_var(current_task)) \ + [current_task] "m" (current_task) \ __switch_canary_iparam \ : "memory", "cc" __EXTRA_CLOBBER) #endif diff --git a/arch/x86/kernel/apic/nmi.c b/arch/x86/kernel/apic/nmi.c index e631cc4416f7..45404379d173 100644 --- a/arch/x86/kernel/apic/nmi.c +++ b/arch/x86/kernel/apic/nmi.c @@ -437,8 +437,8 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason) * Ayiee, looks like this CPU is stuck ... * wait a few IRQs (5 seconds) before doing the oops ... */ - __this_cpu_inc(per_cpu_var(alert_counter)); - if (__this_cpu_read(per_cpu_var(alert_counter)) == 5 * nmi_hz) + __this_cpu_inc(alert_counter); + if (__this_cpu_read(alert_counter) == 5 * nmi_hz) /* * die_nmi will return ONLY if NOTIFY_STOP happens.. */ @@ -446,7 +446,7 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason) regs, panic_on_timeout); } else { __get_cpu_var(last_irq_sum) = sum; - __this_cpu_write(per_cpu_var(alert_counter), 0); + __this_cpu_write(alert_counter, 0); } /* see if the nmi watchdog went off */ diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S index 050c278481b1..fd39eaf83b84 100644 --- a/arch/x86/kernel/head_32.S +++ b/arch/x86/kernel/head_32.S @@ -438,8 +438,8 @@ is386: movl $2,%ecx # set MP */ cmpb $0,ready jne 1f - movl $per_cpu__gdt_page,%eax - movl $per_cpu__stack_canary,%ecx + movl $gdt_page,%eax + movl $stack_canary,%ecx movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax) shrl $16, %ecx movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax) @@ -702,7 +702,7 @@ idt_descr: .word 0 # 32 bit align gdt_desc.address ENTRY(early_gdt_descr) .word GDT_ENTRIES*8-1 - .long per_cpu__gdt_page /* Overwritten for secondary CPUs */ + .long gdt_page /* Overwritten for secondary CPUs */ /* * The boot_gdt must mirror the equivalent in setup.S and is diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 92929fb3f9fa..ecb92717c412 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -312,7 +312,7 @@ SECTIONS * Per-cpu symbols which need to be offset from __per_cpu_load * for the boot processor. */ -#define INIT_PER_CPU(x) init_per_cpu__##x = per_cpu__##x + __per_cpu_load +#define INIT_PER_CPU(x) init_per_cpu__##x = x + __per_cpu_load INIT_PER_CPU(gdt_page); INIT_PER_CPU(irq_stack_union); @@ -323,7 +323,7 @@ INIT_PER_CPU(irq_stack_union); "kernel image bigger than KERNEL_IMAGE_SIZE"); #ifdef CONFIG_SMP -. = ASSERT((per_cpu__irq_stack_union == 0), +. = ASSERT((irq_stack_union == 0), "irq_stack_union is not at start of per-cpu area"); #endif diff --git a/arch/x86/xen/xen-asm_32.S b/arch/x86/xen/xen-asm_32.S index 88e15deb8b82..22a2093b5862 100644 --- a/arch/x86/xen/xen-asm_32.S +++ b/arch/x86/xen/xen-asm_32.S @@ -90,9 +90,9 @@ ENTRY(xen_iret) GET_THREAD_INFO(%eax) movl TI_cpu(%eax), %eax movl __per_cpu_offset(,%eax,4), %eax - mov per_cpu__xen_vcpu(%eax), %eax + mov xen_vcpu(%eax), %eax #else - movl per_cpu__xen_vcpu, %eax + movl xen_vcpu, %eax #endif /* check IF state we're restoring */ diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h index 8087b90d4673..ca6f0491412b 100644 --- a/include/asm-generic/percpu.h +++ b/include/asm-generic/percpu.h @@ -50,11 +50,11 @@ extern unsigned long __per_cpu_offset[NR_CPUS]; * offset. */ #define per_cpu(var, cpu) \ - (*SHIFT_PERCPU_PTR(&per_cpu_var(var), per_cpu_offset(cpu))) + (*SHIFT_PERCPU_PTR(&(var), per_cpu_offset(cpu))) #define __get_cpu_var(var) \ - (*SHIFT_PERCPU_PTR(&per_cpu_var(var), my_cpu_offset)) + (*SHIFT_PERCPU_PTR(&(var), my_cpu_offset)) #define __raw_get_cpu_var(var) \ - (*SHIFT_PERCPU_PTR(&per_cpu_var(var), __my_cpu_offset)) + (*SHIFT_PERCPU_PTR(&(var), __my_cpu_offset)) #define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset) #define __this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset) @@ -66,9 +66,9 @@ extern void setup_per_cpu_areas(void); #else /* ! SMP */ -#define per_cpu(var, cpu) (*((void)(cpu), &per_cpu_var(var))) -#define __get_cpu_var(var) per_cpu_var(var) -#define __raw_get_cpu_var(var) per_cpu_var(var) +#define per_cpu(var, cpu) (*((void)(cpu), &(var))) +#define __get_cpu_var(var) (var) +#define __raw_get_cpu_var(var) (var) #define this_cpu_ptr(ptr) per_cpu_ptr(ptr, 0) #define __this_cpu_ptr(ptr) this_cpu_ptr(ptr) diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index 5a5d6ce4bd55..ee99f6c2cdcd 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -1,12 +1,6 @@ #ifndef _LINUX_PERCPU_DEFS_H #define _LINUX_PERCPU_DEFS_H -/* - * Determine the real variable name from the name visible in the - * kernel sources. - */ -#define per_cpu_var(var) per_cpu__##var - /* * Base implementations of per-CPU variable declarations and definitions, where * the section in which the variable is to be placed is provided by the @@ -56,24 +50,24 @@ */ #define DECLARE_PER_CPU_SECTION(type, name, sec) \ extern __PCPU_DUMMY_ATTRS char __pcpu_scope_##name; \ - extern __PCPU_ATTRS(sec) __typeof__(type) per_cpu__##name + extern __PCPU_ATTRS(sec) __typeof__(type) name #define DEFINE_PER_CPU_SECTION(type, name, sec) \ __PCPU_DUMMY_ATTRS char __pcpu_scope_##name; \ extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \ __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \ __PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES __weak \ - __typeof__(type) per_cpu__##name + __typeof__(type) name #else /* * Normal declaration and definition macros. */ #define DECLARE_PER_CPU_SECTION(type, name, sec) \ - extern __PCPU_ATTRS(sec) __typeof__(type) per_cpu__##name + extern __PCPU_ATTRS(sec) __typeof__(type) name #define DEFINE_PER_CPU_SECTION(type, name, sec) \ __PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES \ - __typeof__(type) per_cpu__##name + __typeof__(type) name #endif /* @@ -137,8 +131,8 @@ /* * Intermodule exports for per-CPU variables. */ -#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var) -#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var) +#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(var) +#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(var) #endif /* _LINUX_PERCPU_DEFS_H */ diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 522f421ec213..e12410e55e05 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -182,7 +182,7 @@ static inline void *pcpu_lpage_remapped(void *kaddr) #ifndef percpu_read # define percpu_read(var) \ ({ \ - typeof(per_cpu_var(var)) __tmp_var__; \ + typeof(var) __tmp_var__; \ __tmp_var__ = get_cpu_var(var); \ put_cpu_var(var); \ __tmp_var__; \ @@ -253,8 +253,7 @@ do { \ /* * Optimized manipulation for memory allocated through the per cpu - * allocator or for addresses of per cpu variables (can be determined - * using per_cpu_var(xx). + * allocator or for addresses of per cpu variables. * * These operation guarantee exclusivity of access for other operations * on the *same* processor. The assumption is that per cpu data is only diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h index d85889710f9b..3e489fda11a1 100644 --- a/include/linux/vmstat.h +++ b/include/linux/vmstat.h @@ -76,22 +76,22 @@ DECLARE_PER_CPU(struct vm_event_state, vm_event_states); static inline void __count_vm_event(enum vm_event_item item) { - __this_cpu_inc(per_cpu_var(vm_event_states).event[item]); + __this_cpu_inc(vm_event_states.event[item]); } static inline void count_vm_event(enum vm_event_item item) { - this_cpu_inc(per_cpu_var(vm_event_states).event[item]); + this_cpu_inc(vm_event_states.event[item]); } static inline void __count_vm_events(enum vm_event_item item, long delta) { - __this_cpu_add(per_cpu_var(vm_event_states).event[item], delta); + __this_cpu_add(vm_event_states.event[item], delta); } static inline void count_vm_events(enum vm_event_item item, long delta) { - this_cpu_add(per_cpu_var(vm_event_states).event[item], delta); + this_cpu_add(vm_event_states.event[item], delta); } extern void all_vm_events(unsigned long *); diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c index 178967b6434e..e339ab349121 100644 --- a/kernel/rcutorture.c +++ b/kernel/rcutorture.c @@ -731,13 +731,13 @@ static void rcu_torture_timer(unsigned long unused) /* Should not happen, but... */ pipe_count = RCU_TORTURE_PIPE_LEN; } - __this_cpu_inc(per_cpu_var(rcu_torture_count)[pipe_count]); + __this_cpu_inc(rcu_torture_count[pipe_count]); completed = cur_ops->completed() - completed; if (completed > RCU_TORTURE_PIPE_LEN) { /* Should not happen, but... */ completed = RCU_TORTURE_PIPE_LEN; } - __this_cpu_inc(per_cpu_var(rcu_torture_batch)[completed]); + __this_cpu_inc(rcu_torture_batch[completed]); preempt_enable(); cur_ops->readunlock(idx); } @@ -786,13 +786,13 @@ rcu_torture_reader(void *arg) /* Should not happen, but... */ pipe_count = RCU_TORTURE_PIPE_LEN; } - __this_cpu_inc(per_cpu_var(rcu_torture_count)[pipe_count]); + __this_cpu_inc(rcu_torture_count[pipe_count]); completed = cur_ops->completed() - completed; if (completed > RCU_TORTURE_PIPE_LEN) { /* Should not happen, but... */ completed = RCU_TORTURE_PIPE_LEN; } - __this_cpu_inc(per_cpu_var(rcu_torture_batch)[completed]); + __this_cpu_inc(rcu_torture_batch[completed]); preempt_enable(); cur_ops->readunlock(idx); schedule(); diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 85a5ed70b5b2..b808177af816 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -91,12 +91,12 @@ DEFINE_PER_CPU(int, ftrace_cpu_disabled); static inline void ftrace_disable_cpu(void) { preempt_disable(); - __this_cpu_inc(per_cpu_var(ftrace_cpu_disabled)); + __this_cpu_inc(ftrace_cpu_disabled); } static inline void ftrace_enable_cpu(void) { - __this_cpu_dec(per_cpu_var(ftrace_cpu_disabled)); + __this_cpu_dec(ftrace_cpu_disabled); preempt_enable(); } @@ -1085,7 +1085,7 @@ trace_function(struct trace_array *tr, struct ftrace_entry *entry; /* If we are reading the ring buffer, don't trace */ - if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled)))) + if (unlikely(__this_cpu_read(ftrace_cpu_disabled))) return; event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index 90a6daa10962..8614e3241ff8 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -176,7 +176,7 @@ static int __trace_graph_entry(struct trace_array *tr, struct ring_buffer *buffer = tr->buffer; struct ftrace_graph_ent_entry *entry; - if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled)))) + if (unlikely(__this_cpu_read(ftrace_cpu_disabled))) return 0; event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT, @@ -240,7 +240,7 @@ static void __trace_graph_return(struct trace_array *tr, struct ring_buffer *buffer = tr->buffer; struct ftrace_graph_ret_entry *entry; - if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled)))) + if (unlikely(__this_cpu_read(ftrace_cpu_disabled))) return; event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET, -- cgit v1.2.3