diff options
Diffstat (limited to 'arch/arm/kernel')
-rw-r--r-- | arch/arm/kernel/Makefile | 2 | ||||
-rw-r--r-- | arch/arm/kernel/entry-armv.S | 12 | ||||
-rw-r--r-- | arch/arm/kernel/fiq.c | 10 | ||||
-rw-r--r-- | arch/arm/kernel/head.S | 57 | ||||
-rw-r--r-- | arch/arm/kernel/irq.c | 23 | ||||
-rw-r--r-- | arch/arm/kernel/machine_kexec.c | 30 | ||||
-rw-r--r-- | arch/arm/kernel/module.c | 109 | ||||
-rw-r--r-- | arch/arm/kernel/relocate_kernel.S | 2 | ||||
-rw-r--r-- | arch/arm/kernel/setup.c | 16 | ||||
-rw-r--r-- | arch/arm/kernel/smp.c | 243 | ||||
-rw-r--r-- | arch/arm/kernel/smp_twd.c | 5 | ||||
-rw-r--r-- | arch/arm/kernel/swp_emulate.c | 267 | ||||
-rw-r--r-- | arch/arm/kernel/traps.c | 14 | ||||
-rw-r--r-- | arch/arm/kernel/vmlinux.lds.S | 1 |
14 files changed, 578 insertions, 213 deletions
diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index 5b9b268f4fbb..4036a5926465 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -42,6 +42,8 @@ obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_ARM_UNWIND) += unwind.o obj-$(CONFIG_HAVE_TCM) += tcm.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o +obj-$(CONFIG_SWP_EMULATE) += swp_emulate.o +CFLAGS_swp_emulate.o := -Wa,-march=armv7-a obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o obj-$(CONFIG_CRUNCH) += crunch.o crunch-bits.o diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index c09e3573c5de..25c9b52448fa 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -48,7 +48,7 @@ */ ALT_SMP(test_for_ipi r0, r6, r5, lr) ALT_UP_B(9997f) - movne r0, sp + movne r1, sp adrne lr, BSYM(1b) bne do_IPI @@ -735,7 +735,7 @@ ENTRY(__switch_to) THUMB( stmia ip!, {r4 - sl, fp} ) @ Store most regs on stack THUMB( str sp, [ip], #4 ) THUMB( str lr, [ip], #4 ) -#ifdef CONFIG_MMU +#ifdef CONFIG_CPU_USE_DOMAINS ldr r6, [r2, #TI_CPU_DOMAIN] #endif set_tls r3, r4, r5 @@ -744,7 +744,7 @@ ENTRY(__switch_to) ldr r8, =__stack_chk_guard ldr r7, [r7, #TSK_STACK_CANARY] #endif -#ifdef CONFIG_MMU +#ifdef CONFIG_CPU_USE_DOMAINS mcr p15, 0, r6, c3, c0, 0 @ Set domain register #endif mov r5, r0 @@ -842,7 +842,7 @@ __kuser_helper_start: */ __kuser_memory_barrier: @ 0xffff0fa0 - smp_dmb + smp_dmb arm usr_ret lr .align 5 @@ -911,7 +911,7 @@ __kuser_cmpxchg: @ 0xffff0fc0 * A special ghost syscall is used for that (see traps.c). */ stmfd sp!, {r7, lr} - ldr r7, =1f @ it's 20 bits + ldr r7, 1f @ it's 20 bits swi __ARM_NR_cmpxchg ldmfd sp!, {r7, pc} 1: .word __ARM_NR_cmpxchg @@ -959,7 +959,7 @@ kuser_cmpxchg_fixup: #else - smp_dmb + smp_dmb arm 1: ldrex r3, [r2] subs r3, r3, r0 strexeq r3, r1, [r2] diff --git a/arch/arm/kernel/fiq.c b/arch/arm/kernel/fiq.c index 6ff7919613d7..e72dc34eea1c 100644 --- a/arch/arm/kernel/fiq.c +++ b/arch/arm/kernel/fiq.c @@ -45,6 +45,7 @@ #include <asm/fiq.h> #include <asm/irq.h> #include <asm/system.h> +#include <asm/traps.h> static unsigned long no_fiq_insn; @@ -67,17 +68,22 @@ static struct fiq_handler default_owner = { static struct fiq_handler *current_fiq = &default_owner; -int show_fiq_list(struct seq_file *p, void *v) +int show_fiq_list(struct seq_file *p, int prec) { if (current_fiq != &default_owner) - seq_printf(p, "FIQ: %s\n", current_fiq->name); + seq_printf(p, "%*s: %s\n", prec, "FIQ", + current_fiq->name); return 0; } void set_fiq_handler(void *start, unsigned int length) { +#if defined(CONFIG_CPU_USE_DOMAINS) memcpy((void *)0xffff001c, start, length); +#else + memcpy(vectors_page + 0x1c, start, length); +#endif flush_icache_range(0xffff001c, 0xffff001c + length); if (!vectors_high()) flush_icache_range(0x1c, 0x1c + length); diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index dd6b369ac69c..f17d9a09e8fb 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -85,10 +85,17 @@ ENTRY(stext) mrc p15, 0, r9, c0, c0 @ get processor id bl __lookup_processor_type @ r5=procinfo r9=cpuid movs r10, r5 @ invalid processor (r5=0)? + THUMB( it eq ) @ force fixup-able long branch encoding beq __error_p @ yes, error 'p' bl __lookup_machine_type @ r5=machinfo movs r8, r5 @ invalid machine (r5=0)? + THUMB( it eq ) @ force fixup-able long branch encoding beq __error_a @ yes, error 'a' + + /* + * r1 = machine no, r2 = atags, + * r8 = machinfo, r9 = cpuid, r10 = procinfo + */ bl __vet_atags #ifdef CONFIG_SMP_ON_UP bl __fixup_smp @@ -262,6 +269,7 @@ __create_page_tables: mov pc, lr ENDPROC(__create_page_tables) .ltorg + .align __enable_mmu_loc: .long . .long __enable_mmu @@ -282,6 +290,7 @@ ENTRY(secondary_startup) bl __lookup_processor_type movs r10, r5 @ invalid processor? moveq r0, #'p' @ yes, error 'p' + THUMB( it eq ) @ force fixup-able long branch encoding beq __error_p /* @@ -308,6 +317,8 @@ ENTRY(__secondary_switched) b secondary_start_kernel ENDPROC(__secondary_switched) + .align + .type __secondary_data, %object __secondary_data: .long . @@ -381,19 +392,19 @@ ENDPROC(__turn_mmu_on) #ifdef CONFIG_SMP_ON_UP __fixup_smp: - mov r7, #0x00070000 - orr r6, r7, #0xff000000 @ mask 0xff070000 - orr r7, r7, #0x41000000 @ val 0x41070000 - and r0, r9, r6 - teq r0, r7 @ ARM CPU and ARMv6/v7? + mov r4, #0x00070000 + orr r3, r4, #0xff000000 @ mask 0xff070000 + orr r4, r4, #0x41000000 @ val 0x41070000 + and r0, r9, r3 + teq r0, r4 @ ARM CPU and ARMv6/v7? bne __fixup_smp_on_up @ no, assume UP - orr r6, r6, #0x0000ff00 - orr r6, r6, #0x000000f0 @ mask 0xff07fff0 - orr r7, r7, #0x0000b000 - orr r7, r7, #0x00000020 @ val 0x4107b020 - and r0, r9, r6 - teq r0, r7 @ ARM 11MPCore? + orr r3, r3, #0x0000ff00 + orr r3, r3, #0x000000f0 @ mask 0xff07fff0 + orr r4, r4, #0x0000b000 + orr r4, r4, #0x00000020 @ val 0x4107b020 + and r0, r9, r3 + teq r0, r4 @ ARM 11MPCore? moveq pc, lr @ yes, assume SMP mrc p15, 0, r0, c0, c0, 5 @ read MPIDR @@ -402,17 +413,25 @@ __fixup_smp: __fixup_smp_on_up: adr r0, 1f - ldmia r0, {r3, r6, r7} + ldmia r0, {r3 - r5} sub r3, r0, r3 - add r6, r6, r3 - add r7, r7, r3 -2: cmp r6, r7 - ldmia r6!, {r0, r4} - strlo r4, [r0, r3] - blo 2b - mov pc, lr + add r4, r4, r3 + add r5, r5, r3 +2: cmp r4, r5 + movhs pc, lr + ldmia r4!, {r0, r6} + ARM( str r6, [r0, r3] ) + THUMB( add r0, r0, r3 ) +#ifdef __ARMEB__ + THUMB( mov r6, r6, ror #16 ) @ Convert word order for big-endian. +#endif + THUMB( strh r6, [r0], #2 ) @ For Thumb-2, store as two halfwords + THUMB( mov r6, r6, lsr #16 ) @ to be robust against misaligned r3. + THUMB( strh r6, [r0] ) + b 2b ENDPROC(__fixup_smp) + .align 1: .word . .word __smpalt_begin .word __smpalt_end diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index 36ad3be4692a..6276f01df9e4 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -57,11 +57,20 @@ int show_interrupts(struct seq_file *p, void *v) struct irq_desc *desc; struct irqaction * action; unsigned long flags; + int prec, n; + + for (prec = 3, n = 1000; prec < 10 && n <= nr_irqs; prec++) + n *= 10; + +#ifdef CONFIG_SMP + if (prec < 4) + prec = 4; +#endif if (i == 0) { char cpuname[12]; - seq_printf(p, " "); + seq_printf(p, "%*s ", prec, ""); for_each_present_cpu(cpu) { sprintf(cpuname, "CPU%d", cpu); seq_printf(p, " %10s", cpuname); @@ -76,7 +85,7 @@ int show_interrupts(struct seq_file *p, void *v) if (!action) goto unlock; - seq_printf(p, "%3d: ", i); + seq_printf(p, "%*d: ", prec, i); for_each_present_cpu(cpu) seq_printf(p, "%10u ", kstat_irqs_cpu(i, cpu)); seq_printf(p, " %10s", desc->chip->name ? : "-"); @@ -89,13 +98,15 @@ unlock: raw_spin_unlock_irqrestore(&desc->lock, flags); } else if (i == nr_irqs) { #ifdef CONFIG_FIQ - show_fiq_list(p, v); + show_fiq_list(p, prec); #endif #ifdef CONFIG_SMP - show_ipi_list(p); - show_local_irqs(p); + show_ipi_list(p, prec); +#endif +#ifdef CONFIG_LOCAL_TIMERS + show_local_irqs(p, prec); #endif - seq_printf(p, "Err: %10lu\n", irq_err_count); + seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count); } return 0; } diff --git a/arch/arm/kernel/machine_kexec.c b/arch/arm/kernel/machine_kexec.c index 3a8fd5140d7a..30ead135ff5f 100644 --- a/arch/arm/kernel/machine_kexec.c +++ b/arch/arm/kernel/machine_kexec.c @@ -23,6 +23,8 @@ extern unsigned long kexec_indirection_page; extern unsigned long kexec_mach_type; extern unsigned long kexec_boot_atags; +static atomic_t waiting_for_crash_ipi; + /* * Provide a dummy crash_notes definition while crash dump arrives to arm. * This prevents breakage of crash_notes attribute in kernel/ksysfs.c. @@ -37,9 +39,37 @@ void machine_kexec_cleanup(struct kimage *image) { } +void machine_crash_nonpanic_core(void *unused) +{ + struct pt_regs regs; + + crash_setup_regs(®s, NULL); + printk(KERN_DEBUG "CPU %u will stop doing anything useful since another CPU has crashed\n", + smp_processor_id()); + crash_save_cpu(®s, smp_processor_id()); + flush_cache_all(); + + atomic_dec(&waiting_for_crash_ipi); + while (1) + cpu_relax(); +} + void machine_crash_shutdown(struct pt_regs *regs) { + unsigned long msecs; + local_irq_disable(); + + atomic_set(&waiting_for_crash_ipi, num_online_cpus() - 1); + smp_call_function(machine_crash_nonpanic_core, NULL, false); + msecs = 1000; /* Wait at most a second for the other cpus to stop */ + while ((atomic_read(&waiting_for_crash_ipi) > 0) && msecs) { + mdelay(1); + msecs--; + } + if (atomic_read(&waiting_for_crash_ipi) > 0) + printk(KERN_WARNING "Non-crashing CPUs did not react to IPI\n"); + crash_save_cpu(regs, smp_processor_id()); printk(KERN_INFO "Loading crashdump kernel...\n"); diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index d9bd786ce23d..0c1bb68ff4a8 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -67,35 +67,6 @@ int module_frob_arch_sections(Elf_Ehdr *hdr, char *secstrings, struct module *mod) { -#ifdef CONFIG_ARM_UNWIND - Elf_Shdr *s, *sechdrs_end = sechdrs + hdr->e_shnum; - struct arm_unwind_mapping *maps = mod->arch.map; - - for (s = sechdrs; s < sechdrs_end; s++) { - char const *secname = secstrings + s->sh_name; - - if (strcmp(".ARM.exidx.init.text", secname) == 0) - maps[ARM_SEC_INIT].unw_sec = s; - else if (strcmp(".ARM.exidx.devinit.text", secname) == 0) - maps[ARM_SEC_DEVINIT].unw_sec = s; - else if (strcmp(".ARM.exidx", secname) == 0) - maps[ARM_SEC_CORE].unw_sec = s; - else if (strcmp(".ARM.exidx.exit.text", secname) == 0) - maps[ARM_SEC_EXIT].unw_sec = s; - else if (strcmp(".ARM.exidx.devexit.text", secname) == 0) - maps[ARM_SEC_DEVEXIT].unw_sec = s; - else if (strcmp(".init.text", secname) == 0) - maps[ARM_SEC_INIT].sec_text = s; - else if (strcmp(".devinit.text", secname) == 0) - maps[ARM_SEC_DEVINIT].sec_text = s; - else if (strcmp(".text", secname) == 0) - maps[ARM_SEC_CORE].sec_text = s; - else if (strcmp(".exit.text", secname) == 0) - maps[ARM_SEC_EXIT].sec_text = s; - else if (strcmp(".devexit.text", secname) == 0) - maps[ARM_SEC_DEVEXIT].sec_text = s; - } -#endif return 0; } @@ -300,41 +271,69 @@ apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, return -ENOEXEC; } -#ifdef CONFIG_ARM_UNWIND -static void register_unwind_tables(struct module *mod) +struct mod_unwind_map { + const Elf_Shdr *unw_sec; + const Elf_Shdr *txt_sec; +}; + +int module_finalize(const Elf32_Ehdr *hdr, const Elf_Shdr *sechdrs, + struct module *mod) { +#ifdef CONFIG_ARM_UNWIND + const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; + const Elf_Shdr *s, *sechdrs_end = sechdrs + hdr->e_shnum; + struct mod_unwind_map maps[ARM_SEC_MAX]; int i; - for (i = 0; i < ARM_SEC_MAX; ++i) { - struct arm_unwind_mapping *map = &mod->arch.map[i]; - if (map->unw_sec && map->sec_text) - map->unwind = unwind_table_add(map->unw_sec->sh_addr, - map->unw_sec->sh_size, - map->sec_text->sh_addr, - map->sec_text->sh_size); + + memset(maps, 0, sizeof(maps)); + + for (s = sechdrs; s < sechdrs_end; s++) { + const char *secname = secstrs + s->sh_name; + + if (!(s->sh_flags & SHF_ALLOC)) + continue; + + if (strcmp(".ARM.exidx.init.text", secname) == 0) + maps[ARM_SEC_INIT].unw_sec = s; + else if (strcmp(".ARM.exidx.devinit.text", secname) == 0) + maps[ARM_SEC_DEVINIT].unw_sec = s; + else if (strcmp(".ARM.exidx", secname) == 0) + maps[ARM_SEC_CORE].unw_sec = s; + else if (strcmp(".ARM.exidx.exit.text", secname) == 0) + maps[ARM_SEC_EXIT].unw_sec = s; + else if (strcmp(".ARM.exidx.devexit.text", secname) == 0) + maps[ARM_SEC_DEVEXIT].unw_sec = s; + else if (strcmp(".init.text", secname) == 0) + maps[ARM_SEC_INIT].txt_sec = s; + else if (strcmp(".devinit.text", secname) == 0) + maps[ARM_SEC_DEVINIT].txt_sec = s; + else if (strcmp(".text", secname) == 0) + maps[ARM_SEC_CORE].txt_sec = s; + else if (strcmp(".exit.text", secname) == 0) + maps[ARM_SEC_EXIT].txt_sec = s; + else if (strcmp(".devexit.text", secname) == 0) + maps[ARM_SEC_DEVEXIT].txt_sec = s; } -} -static void unregister_unwind_tables(struct module *mod) -{ - int i = ARM_SEC_MAX; - while (--i >= 0) - unwind_table_del(mod->arch.map[i].unwind); -} -#else -static inline void register_unwind_tables(struct module *mod) { } -static inline void unregister_unwind_tables(struct module *mod) { } + for (i = 0; i < ARM_SEC_MAX; i++) + if (maps[i].unw_sec && maps[i].txt_sec) + mod->arch.unwind[i] = + unwind_table_add(maps[i].unw_sec->sh_addr, + maps[i].unw_sec->sh_size, + maps[i].txt_sec->sh_addr, + maps[i].txt_sec->sh_size); #endif - -int -module_finalize(const Elf32_Ehdr *hdr, const Elf_Shdr *sechdrs, - struct module *module) -{ - register_unwind_tables(module); return 0; } void module_arch_cleanup(struct module *mod) { - unregister_unwind_tables(mod); +#ifdef CONFIG_ARM_UNWIND + int i; + + for (i = 0; i < ARM_SEC_MAX; i++) + if (mod->arch.unwind[i]) + unwind_table_del(mod->arch.unwind[i]); +#endif } diff --git a/arch/arm/kernel/relocate_kernel.S b/arch/arm/kernel/relocate_kernel.S index fd26f8d65151..9cf4cbf8f95b 100644 --- a/arch/arm/kernel/relocate_kernel.S +++ b/arch/arm/kernel/relocate_kernel.S @@ -59,6 +59,8 @@ relocate_new_kernel: ldr r2,kexec_boot_atags mov pc,lr + .align + .globl kexec_start_address kexec_start_address: .long 0x0 diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 336f14e0e5c2..8075e592f902 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -75,9 +75,9 @@ extern void reboot_setup(char *str); unsigned int processor_id; EXPORT_SYMBOL(processor_id); -unsigned int __machine_arch_type; +unsigned int __machine_arch_type __read_mostly; EXPORT_SYMBOL(__machine_arch_type); -unsigned int cacheid; +unsigned int cacheid __read_mostly; EXPORT_SYMBOL(cacheid); unsigned int __atags_pointer __initdata; @@ -91,24 +91,24 @@ EXPORT_SYMBOL(system_serial_low); unsigned int system_serial_high; EXPORT_SYMBOL(system_serial_high); -unsigned int elf_hwcap; +unsigned int elf_hwcap __read_mostly; EXPORT_SYMBOL(elf_hwcap); #ifdef MULTI_CPU -struct processor processor; +struct processor processor __read_mostly; #endif #ifdef MULTI_TLB -struct cpu_tlb_fns cpu_tlb; +struct cpu_tlb_fns cpu_tlb __read_mostly; #endif #ifdef MULTI_USER -struct cpu_user_fns cpu_user; +struct cpu_user_fns cpu_user __read_mostly; #endif #ifdef MULTI_CACHE -struct cpu_cache_fns cpu_cache; +struct cpu_cache_fns cpu_cache __read_mostly; #endif #ifdef CONFIG_OUTER_CACHE -struct outer_cache_fns outer_cache; +struct outer_cache_fns outer_cache __read_mostly; EXPORT_SYMBOL(outer_cache); #endif diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 73cef403352a..ed598264e508 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -24,6 +24,7 @@ #include <linux/irq.h> #include <linux/percpu.h> #include <linux/clockchips.h> +#include <linux/completion.h> #include <asm/atomic.h> #include <asm/cacheflush.h> @@ -46,22 +47,8 @@ */ struct secondary_data secondary_data; -/* - * structures for inter-processor calls - * - A collection of single bit ipi messages. - */ -struct ipi_data { - spinlock_t lock; - unsigned long ipi_count; - unsigned long bits; -}; - -static DEFINE_PER_CPU(struct ipi_data, ipi_data) = { - .lock = SPIN_LOCK_UNLOCKED, -}; - enum ipi_msg_type { - IPI_TIMER, + IPI_TIMER = 2, IPI_RESCHEDULE, IPI_CALL_FUNC, IPI_CALL_FUNC_SINGLE, @@ -216,12 +203,20 @@ int __cpu_disable(void) return 0; } +static DECLARE_COMPLETION(cpu_died); + /* * called on the thread which is asking for a CPU to be shutdown - * waits until shutdown has completed, or it is timed out. */ void __cpu_die(unsigned int cpu) { + if (!wait_for_completion_timeout(&cpu_died, 5000)) { + pr_err("CPU%u: cpu didn't die\n", cpu); + return; + } + printk(KERN_NOTICE "CPU%u: shutdown\n", cpu); + if (!platform_cpu_kill(cpu)) printk("CPU%u: unable to kill\n", cpu); } @@ -238,12 +233,17 @@ void __ref cpu_die(void) { unsigned int cpu = smp_processor_id(); - local_irq_disable(); idle_task_exit(); + local_irq_disable(); + mb(); + + /* Tell __cpu_die() that this CPU is now safe to dispose of */ + complete(&cpu_died); + /* * actual CPU shutdown procedure is at least platform (if not - * CPU) specific + * CPU) specific. */ platform_cpu_die(cpu); @@ -260,6 +260,17 @@ void __ref cpu_die(void) #endif /* CONFIG_HOTPLUG_CPU */ /* + * Called by both boot and secondaries to move global data into + * per-processor storage. + */ +static void __cpuinit smp_store_cpu_info(unsigned int cpuid) +{ + struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid); + + cpu_info->loops_per_jiffy = loops_per_jiffy; +} + +/* * This is the secondary CPU boot entry. We're using this CPUs * idle thread stack, but a set of temporary page tables. */ @@ -284,6 +295,7 @@ asmlinkage void __cpuinit secondary_start_kernel(void) cpu_init(); preempt_disable(); + trace_hardirqs_off(); /* * Give the platform a chance to do its own initialisation. @@ -317,17 +329,6 @@ asmlinkage void __cpuinit secondary_start_kernel(void) cpu_idle(); } -/* - * Called by both boot and secondaries to move global data into - * per-processor storage. - */ -void __cpuinit smp_store_cpu_info(unsigned int cpuid) -{ - struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid); - - cpu_info->loops_per_jiffy = loops_per_jiffy; -} - void __init smp_cpus_done(unsigned int max_cpus) { int cpu; @@ -350,27 +351,39 @@ void __init smp_prepare_boot_cpu(void) per_cpu(cpu_data, cpu).idle = current; } -static void send_ipi_message(const struct cpumask *mask, enum ipi_msg_type msg) +void __init smp_prepare_cpus(unsigned int max_cpus) { - unsigned long flags; - unsigned int cpu; + unsigned int ncores = num_possible_cpus(); + + smp_store_cpu_info(smp_processor_id()); - local_irq_save(flags); + /* + * are we trying to boot more cores than exist? + */ + if (max_cpus > ncores) + max_cpus = ncores; - for_each_cpu(cpu, mask) { - struct ipi_data *ipi = &per_cpu(ipi_data, cpu); + if (max_cpus > 1) { + /* + * Enable the local timer or broadcast device for the + * boot CPU, but only if we have more than one CPU. + */ + percpu_timer_setup(); - spin_lock(&ipi->lock); - ipi->bits |= 1 << msg; - spin_unlock(&ipi->lock); + /* + * Initialise the SCU if there are more than one CPU + * and let them know where to start. + */ + platform_smp_prepare_cpus(max_cpus); } +} +static void send_ipi_message(const struct cpumask *mask, enum ipi_msg_type msg) +{ /* * Call the platform specific cross-CPU call function. */ - smp_cross_call(mask); - - local_irq_restore(flags); + smp_cross_call(mask, msg); } void arch_send_call_function_ipi_mask(const struct cpumask *mask) @@ -383,28 +396,43 @@ void arch_send_call_function_single_ipi(int cpu) send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE); } -void show_ipi_list(struct seq_file *p) +static const char *ipi_types[NR_IPI] = { +#define S(x,s) [x - IPI_TIMER] = s + S(IPI_TIMER, "Timer broadcast interrupts"), + S(IPI_RESCHEDULE, "Rescheduling interrupts"), + S(IPI_CALL_FUNC, "Function call interrupts"), + S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"), + S(IPI_CPU_STOP, "CPU stop interrupts"), +}; + +void show_ipi_list(struct seq_file *p, int prec) { - unsigned int cpu; + unsigned int cpu, i; - seq_puts(p, "IPI:"); + for (i = 0; i < NR_IPI; i++) { + seq_printf(p, "%*s%u: ", prec - 1, "IPI", i); - for_each_present_cpu(cpu) - seq_printf(p, " %10lu", per_cpu(ipi_data, cpu).ipi_count); + for_each_present_cpu(cpu) + seq_printf(p, "%10u ", + __get_irq_stat(cpu, ipi_irqs[i])); - seq_putc(p, '\n'); + seq_printf(p, " %s\n", ipi_types[i]); + } } -void show_local_irqs(struct seq_file *p) +u64 smp_irq_stat_cpu(unsigned int cpu) { - unsigned int cpu; + u64 sum = 0; + int i; - seq_printf(p, "LOC: "); + for (i = 0; i < NR_IPI; i++) + sum += __get_irq_stat(cpu, ipi_irqs[i]); - for_each_present_cpu(cpu) - seq_printf(p, "%10u ", irq_stat[cpu].local_timer_irqs); +#ifdef CONFIG_LOCAL_TIMERS + sum += __get_irq_stat(cpu, local_timer_irqs); +#endif - seq_putc(p, '\n'); + return sum; } /* @@ -427,12 +455,24 @@ asmlinkage void __exception do_local_timer(struct pt_regs *regs) int cpu = smp_processor_id(); if (local_timer_ack()) { - irq_stat[cpu].local_timer_irqs++; + __inc_irq_stat(cpu, local_timer_irqs); ipi_timer(); } set_irq_regs(old_regs); } + +void show_local_irqs(struct seq_file *p, int prec) +{ + unsigned int cpu; + + seq_printf(p, "%*s: ", prec, "LOC"); + + for_each_present_cpu(cpu) + seq_printf(p, "%10u ", __get_irq_stat(cpu, local_timer_irqs)); + + seq_printf(p, " Local timer interrupts\n"); +} #endif #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST @@ -501,71 +541,44 @@ static void ipi_cpu_stop(unsigned int cpu) /* * Main handler for inter-processor interrupts - * - * For ARM, the ipimask now only identifies a single - * category of IPI (Bit 1 IPIs have been replaced by a - * different mechanism): - * - * Bit 0 - Inter-processor function call */ -asmlinkage void __exception do_IPI(struct pt_regs *regs) +asmlinkage void __exception do_IPI(int ipinr, struct pt_regs *regs) { unsigned int cpu = smp_processor_id(); - struct ipi_data *ipi = &per_cpu(ipi_data, cpu); struct pt_regs *old_regs = set_irq_regs(regs); - ipi->ipi_count++; - - for (;;) { - unsigned long msgs; - - spin_lock(&ipi->lock); - msgs = ipi->bits; - ipi->bits = 0; - spin_unlock(&ipi->lock); + if (ipinr >= IPI_TIMER && ipinr < IPI_TIMER + NR_IPI) + __inc_irq_stat(cpu, ipi_irqs[ipinr - IPI_TIMER]); - if (!msgs) - break; - - do { - unsigned nextmsg; + switch (ipinr) { + case IPI_TIMER: + ipi_timer(); + break; - nextmsg = msgs & -msgs; - msgs &= ~nextmsg; - nextmsg = ffz(~nextmsg); + case IPI_RESCHEDULE: + /* + * nothing more to do - eveything is + * done on the interrupt return path + */ + break; - switch (nextmsg) { - case IPI_TIMER: - ipi_timer(); - break; + case IPI_CALL_FUNC: + generic_smp_call_function_interrupt(); + break; - case IPI_RESCHEDULE: - /* - * nothing more to do - eveything is - * done on the interrupt return path - */ - break; + case IPI_CALL_FUNC_SINGLE: + generic_smp_call_function_single_interrupt(); + break; - case IPI_CALL_FUNC: - generic_smp_call_function_interrupt(); - break; + case IPI_CPU_STOP: + ipi_cpu_stop(cpu); + break; - case IPI_CALL_FUNC_SINGLE: - generic_smp_call_function_single_interrupt(); - break; - - case IPI_CPU_STOP: - ipi_cpu_stop(cpu); - break; - - default: - printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%x\n", - cpu, nextmsg); - break; - } - } while (msgs); + default: + printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%x\n", + cpu, ipinr); + break; } - set_irq_regs(old_regs); } @@ -576,10 +589,22 @@ void smp_send_reschedule(int cpu) void smp_send_stop(void) { - cpumask_t mask = cpu_online_map; - cpu_clear(smp_processor_id(), mask); - if (!cpus_empty(mask)) + unsigned long timeout; + + if (num_online_cpus() > 1) { + cpumask_t mask = cpu_online_map; + cpu_clear(smp_processor_id(), mask); + send_ipi_message(&mask, IPI_CPU_STOP); + } + + /* Wait up to one second for other CPUs to stop */ + timeout = USEC_PER_SEC; + while (num_online_cpus() > 1 && timeout--) + udelay(1); + + if (num_online_cpus() > 1) + pr_warning("SMP: failed to stop secondary CPUs\n"); } /* diff --git a/arch/arm/kernel/smp_twd.c b/arch/arm/kernel/smp_twd.c index 35882fbf37f9..948d14944260 100644 --- a/arch/arm/kernel/smp_twd.c +++ b/arch/arm/kernel/smp_twd.c @@ -143,10 +143,7 @@ void __cpuinit twd_timer_setup(struct clock_event_device *clk) clk->min_delta_ns = clockevent_delta2ns(0xf, clk); /* Make sure our local interrupt controller has this enabled */ - local_irq_save(flags); - irq_to_desc(clk->irq)->status |= IRQ_NOPROBE; - get_irq_chip(clk->irq)->unmask(clk->irq); - local_irq_restore(flags); + gic_enable_ppi(clk->irq); clockevents_register_device(clk); } diff --git a/arch/arm/kernel/swp_emulate.c b/arch/arm/kernel/swp_emulate.c new file mode 100644 index 000000000000..7a5760922914 --- /dev/null +++ b/arch/arm/kernel/swp_emulate.c @@ -0,0 +1,267 @@ +/* + * linux/arch/arm/kernel/swp_emulate.c + * + * Copyright (C) 2009 ARM Limited + * __user_* functions adapted from include/asm/uaccess.h + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * Implements emulation of the SWP/SWPB instructions using load-exclusive and + * store-exclusive for processors that have them disabled (or future ones that + * might not implement them). + * + * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>] + * Where: Rt = destination + * Rt2 = source + * Rn = address + */ + +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/proc_fs.h> +#include <linux/sched.h> +#include <linux/syscalls.h> +#include <linux/perf_event.h> + +#include <asm/traps.h> +#include <asm/uaccess.h> + +/* + * Error-checking SWP macros implemented using ldrex{b}/strex{b} + */ +#define __user_swpX_asm(data, addr, res, temp, B) \ + __asm__ __volatile__( \ + " mov %2, %1\n" \ + "0: ldrex"B" %1, [%3]\n" \ + "1: strex"B" %0, %2, [%3]\n" \ + " cmp %0, #0\n" \ + " movne %0, %4\n" \ + "2:\n" \ + " .section .fixup,\"ax\"\n" \ + " .align 2\n" \ + "3: mov %0, %5\n" \ + " b 2b\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + " .align 3\n" \ + " .long 0b, 3b\n" \ + " .long 1b, 3b\n" \ + " .previous" \ + : "=&r" (res), "+r" (data), "=&r" (temp) \ + : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT) \ + : "cc", "memory") + +#define __user_swp_asm(data, addr, res, temp) \ + __user_swpX_asm(data, addr, res, temp, "") +#define __user_swpb_asm(data, addr, res, temp) \ + __user_swpX_asm(data, addr, res, temp, "b") + +/* + * Macros/defines for extracting register numbers from instruction. + */ +#define EXTRACT_REG_NUM(instruction, offset) \ + (((instruction) & (0xf << (offset))) >> (offset)) +#define RN_OFFSET 16 +#define RT_OFFSET 12 +#define RT2_OFFSET 0 +/* + * Bit 22 of the instruction encoding distinguishes between + * the SWP and SWPB variants (bit set means SWPB). + */ +#define TYPE_SWPB (1 << 22) + +static unsigned long swpcounter; +static unsigned long swpbcounter; +static unsigned long abtcounter; +static pid_t previous_pid; + +#ifdef CONFIG_PROC_FS +static int proc_read_status(char *page, char **start, off_t off, int count, + int *eof, void *data) +{ + char *p = page; + int len; + + p += sprintf(p, "Emulated SWP:\t\t%lu\n", swpcounter); + p += sprintf(p, "Emulated SWPB:\t\t%lu\n", swpbcounter); + p += sprintf(p, "Aborted SWP{B}:\t\t%lu\n", abtcounter); + if (previous_pid != 0) + p += sprintf(p, "Last process:\t\t%d\n", previous_pid); + + len = (p - page) - off; + if (len < 0) + len = 0; + + *eof = (len <= count) ? 1 : 0; + *start = page + off; + + return len; +} +#endif + +/* + * Set up process info to signal segmentation fault - called on access error. + */ +static void set_segfault(struct pt_regs *regs, unsigned long addr) +{ + siginfo_t info; + + if (find_vma(current->mm, addr) == NULL) + info.si_code = SEGV_MAPERR; + else + info.si_code = SEGV_ACCERR; + + info.si_signo = SIGSEGV; + info.si_errno = 0; + info.si_addr = (void *) instruction_pointer(regs); + + pr_debug("SWP{B} emulation: access caused memory abort!\n"); + arm_notify_die("Illegal memory access", regs, &info, 0, 0); + + abtcounter++; +} + +static int emulate_swpX(unsigned int address, unsigned int *data, + unsigned int type) +{ + unsigned int res = 0; + + if ((type != TYPE_SWPB) && (address & 0x3)) { + /* SWP to unaligned address not permitted */ + pr_debug("SWP instruction on unaligned pointer!\n"); + return -EFAULT; + } + + while (1) { + unsigned long temp; + + /* + * Barrier required between accessing protected resource and + * releasing a lock for it. Legacy code might not have done + * this, and we cannot determine that this is not the case + * being emulated, so insert always. + */ + smp_mb(); + + if (type == TYPE_SWPB) + __user_swpb_asm(*data, address, res, temp); + else + __user_swp_asm(*data, address, res, temp); + + if (likely(res != -EAGAIN) || signal_pending(current)) + break; + + cond_resched(); + } + + if (res == 0) { + /* + * Barrier also required between aquiring a lock for a + * protected resource and accessing the resource. Inserted for + * same reason as above. + */ + smp_mb(); + + if (type == TYPE_SWPB) + swpbcounter++; + else + swpcounter++; + } + + return res; +} + +/* + * swp_handler logs the id of calling process, dissects the instruction, sanity + * checks the memory location, calls emulate_swpX for the actual operation and + * deals with fixup/error handling before returning + */ +static int swp_handler(struct pt_regs *regs, unsigned int instr) +{ + unsigned int address, destreg, data, type; + unsigned int res = 0; + + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, regs->ARM_pc); + + if (current->pid != previous_pid) { + pr_debug("\"%s\" (%ld) uses deprecated SWP{B} instruction\n", + current->comm, (unsigned long)current->pid); + previous_pid = current->pid; + } + + address = regs->uregs[EXTRACT_REG_NUM(instr, RN_OFFSET)]; + data = regs->uregs[EXTRACT_REG_NUM(instr, RT2_OFFSET)]; + destreg = EXTRACT_REG_NUM(instr, RT_OFFSET); + + type = instr & TYPE_SWPB; + + pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n", + EXTRACT_REG_NUM(instr, RN_OFFSET), address, + destreg, EXTRACT_REG_NUM(instr, RT2_OFFSET), data); + + /* Check access in reasonable access range for both SWP and SWPB */ + if (!access_ok(VERIFY_WRITE, (address & ~3), 4)) { + pr_debug("SWP{B} emulation: access to %p not allowed!\n", + (void *)address); + res = -EFAULT; + } else { + res = emulate_swpX(address, &data, type); + } + + if (res == 0) { + /* + * On successful emulation, revert the adjustment to the PC + * made in kernel/traps.c in order to resume execution at the + * instruction following the SWP{B}. + */ + regs->ARM_pc += 4; + regs->uregs[destreg] = data; + } else if (res == -EFAULT) { + /* + * Memory errors do not mean emulation failed. + * Set up signal info to return SEGV, then return OK + */ + set_segfault(regs, address); + } + + return 0; +} + +/* + * Only emulate SWP/SWPB executed in ARM state/User mode. + * The kernel must be SWP free and SWP{B} does not exist in Thumb/ThumbEE. + */ +static struct undef_hook swp_hook = { + .instr_mask = 0x0fb00ff0, + .instr_val = 0x01000090, + .cpsr_mask = MODE_MASK | PSR_T_BIT | PSR_J_BIT, + .cpsr_val = USR_MODE, + .fn = swp_handler +}; + +/* + * Register handler and create status file in /proc/cpu + * Invoked as late_initcall, since not needed before init spawned. + */ +static int __init swp_emulation_init(void) +{ +#ifdef CONFIG_PROC_FS + struct proc_dir_entry *res; + + res = create_proc_entry("cpu/swp_emulation", S_IRUGO, NULL); + + if (!res) + return -ENOMEM; + + res->read_proc = proc_read_status; +#endif /* CONFIG_PROC_FS */ + + printk(KERN_NOTICE "Registering SWP/SWPB emulation handler\n"); + register_undef_hook(&swp_hook); + + return 0; +} + +late_initcall(swp_emulation_init); diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 354cd4ce400a..ee57640ba2bb 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -37,6 +37,8 @@ static const char *handler[]= { "prefetch abort", "data abort", "address exception", "interrupt" }; +void *vectors_page; + #ifdef CONFIG_DEBUG_USER unsigned int user_debug; @@ -756,7 +758,11 @@ static void __init kuser_get_tls_init(unsigned long vectors) void __init early_trap_init(void) { +#if defined(CONFIG_CPU_USE_DOMAINS) unsigned long vectors = CONFIG_VECTORS_BASE; +#else + unsigned long vectors = (unsigned long)vectors_page; +#endif extern char __stubs_start[], __stubs_end[]; extern char __vectors_start[], __vectors_end[]; extern char __kuser_helper_start[], __kuser_helper_end[]; @@ -780,10 +786,10 @@ void __init early_trap_init(void) * Copy signal return handlers into the vector page, and * set sigreturn to be a pointer to these. */ - memcpy((void *)KERN_SIGRETURN_CODE, sigreturn_codes, - sizeof(sigreturn_codes)); - memcpy((void *)KERN_RESTART_CODE, syscall_restart_code, - sizeof(syscall_restart_code)); + memcpy((void *)(vectors + KERN_SIGRETURN_CODE - CONFIG_VECTORS_BASE), + sigreturn_codes, sizeof(sigreturn_codes)); + memcpy((void *)(vectors + KERN_RESTART_CODE - CONFIG_VECTORS_BASE), + syscall_restart_code, sizeof(syscall_restart_code)); flush_icache_range(vectors, vectors + PAGE_SIZE); modify_domain(DOMAIN_USER, DOMAIN_CLIENT); diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S index cead8893b46b..1581f6d18cca 100644 --- a/arch/arm/kernel/vmlinux.lds.S +++ b/arch/arm/kernel/vmlinux.lds.S @@ -167,6 +167,7 @@ SECTIONS NOSAVE_DATA CACHELINE_ALIGNED_DATA(32) + READ_MOSTLY_DATA(32) /* * The exception fixup table (might need resorting at runtime) |