From 14778d9072e53d2171f66ffd9657daff41acfaed Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 21 Mar 2006 02:29:39 -0800 Subject: [SPARC]: Respect vm_page_prot in io_remap_page_range(). Make sure the callers do a pgprot_noncached() on vma->vm_page_prot. Pointed out by Hugh Dickens. Signed-off-by: David S. Miller --- arch/sparc64/kernel/pci.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/sparc64/kernel') diff --git a/arch/sparc64/kernel/pci.c b/arch/sparc64/kernel/pci.c index 95ffa9418620..dfccff29e182 100644 --- a/arch/sparc64/kernel/pci.c +++ b/arch/sparc64/kernel/pci.c @@ -656,6 +656,7 @@ int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, __pci_mmap_set_flags(dev, vma, mmap_state); __pci_mmap_set_pgprot(dev, vma, mmap_state); + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); ret = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, vma->vm_end - vma->vm_start, @@ -663,7 +664,6 @@ int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, if (ret) return ret; - vma->vm_flags |= VM_IO; return 0; } -- cgit v1.2.3 From dcc1e8dd88d4bc55e32a26dad7633d20ffe606d2 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Wed, 22 Mar 2006 00:49:59 -0800 Subject: [SPARC64]: Add a secondary TSB for hugepage mappings. Signed-off-by: David S. Miller --- arch/sparc64/Kconfig | 4 +- arch/sparc64/kernel/sun4v_tlb_miss.S | 39 +++--- arch/sparc64/kernel/traps.c | 21 +++- arch/sparc64/kernel/tsb.S | 210 +++++++++++++++++++++++-------- arch/sparc64/mm/fault.c | 15 ++- arch/sparc64/mm/hugetlbpage.c | 28 +++-- arch/sparc64/mm/init.c | 21 +++- arch/sparc64/mm/tsb.c | 234 ++++++++++++++++++++++------------- include/asm-sparc64/cpudata.h | 5 +- include/asm-sparc64/mmu.h | 29 ++++- include/asm-sparc64/mmu_context.h | 21 ++-- include/asm-sparc64/page.h | 34 ++--- include/asm-sparc64/pgtable.h | 2 + 13 files changed, 462 insertions(+), 201 deletions(-) (limited to 'arch/sparc64/kernel') diff --git a/arch/sparc64/Kconfig b/arch/sparc64/Kconfig index c3685b314d71..267afddf63cf 100644 --- a/arch/sparc64/Kconfig +++ b/arch/sparc64/Kconfig @@ -175,11 +175,11 @@ config HUGETLB_PAGE_SIZE_4MB bool "4MB" config HUGETLB_PAGE_SIZE_512K - depends on !SPARC64_PAGE_SIZE_4MB + depends on !SPARC64_PAGE_SIZE_4MB && !SPARC64_PAGE_SIZE_512KB bool "512K" config HUGETLB_PAGE_SIZE_64K - depends on !SPARC64_PAGE_SIZE_4MB && !SPARC64_PAGE_SIZE_512KB + depends on !SPARC64_PAGE_SIZE_4MB && !SPARC64_PAGE_SIZE_512KB && !SPARC64_PAGE_SIZE_64K bool "64K" endchoice diff --git a/arch/sparc64/kernel/sun4v_tlb_miss.S b/arch/sparc64/kernel/sun4v_tlb_miss.S index ab23ddb7116e..b731881224e8 100644 --- a/arch/sparc64/kernel/sun4v_tlb_miss.S +++ b/arch/sparc64/kernel/sun4v_tlb_miss.S @@ -29,15 +29,15 @@ * * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL; * tsb_base = tsb_reg & ~0x7UL; - * tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask); + * tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask); * tsb_ptr = tsb_base + (tsb_index * 16); */ -#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, TMP1, TMP2) \ +#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \ and TSB_PTR, 0x7, TMP1; \ mov 512, TMP2; \ andn TSB_PTR, 0x7, TSB_PTR; \ sllx TMP2, TMP1, TMP2; \ - srlx VADDR, PAGE_SHIFT, TMP1; \ + srlx VADDR, HASH_SHIFT, TMP1; \ sub TMP2, 1, TMP2; \ and TMP1, TMP2, TMP1; \ sllx TMP1, 4, TMP1; \ @@ -53,7 +53,7 @@ sun4v_itlb_miss: LOAD_ITLB_INFO(%g2, %g4, %g5) COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v) - COMPUTE_TSB_PTR(%g1, %g4, %g3, %g7) + COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7) /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2 @@ -99,7 +99,7 @@ sun4v_dtlb_miss: LOAD_DTLB_INFO(%g2, %g4, %g5) COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v) - COMPUTE_TSB_PTR(%g1, %g4, %g3, %g7) + COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7) /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2 @@ -171,21 +171,26 @@ sun4v_dtsb_miss: /* fallthrough */ - /* Create TSB pointer into %g1. This is something like: - * - * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL; - * tsb_base = tsb_reg & ~0x7UL; - * tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask); - * tsb_ptr = tsb_base + (tsb_index * 16); - */ sun4v_tsb_miss_common: - COMPUTE_TSB_PTR(%g1, %g4, %g5, %g7) + COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7) - /* Branch directly to page table lookup. We have SCRATCHPAD_MMU_MISS - * still in %g2, so it's quite trivial to get at the PGD PHYS value - * so we can preload it into %g7. - */ sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2 + +#ifdef CONFIG_HUGETLB_PAGE + mov SCRATCHPAD_UTSBREG2, %g5 + ldxa [%g5] ASI_SCRATCHPAD, %g5 + cmp %g5, -1 + be,pt %xcc, 80f + nop + COMPUTE_TSB_PTR(%g5, %g4, HPAGE_SHIFT, %g2, %g7) + + /* That clobbered %g2, reload it. */ + ldxa [%g0] ASI_SCRATCHPAD, %g2 + sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2 + +80: stx %g5, [%g2 + TRAP_PER_CPU_TSB_HUGE_TEMP] +#endif + ba,pt %xcc, tsb_miss_page_table_walk_sun4v_fastpath ldx [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7 diff --git a/arch/sparc64/kernel/traps.c b/arch/sparc64/kernel/traps.c index 7f7dba0ca96a..df612e4f75f9 100644 --- a/arch/sparc64/kernel/traps.c +++ b/arch/sparc64/kernel/traps.c @@ -2482,6 +2482,7 @@ void init_cur_cpu_trap(struct thread_info *t) extern void thread_info_offsets_are_bolixed_dave(void); extern void trap_per_cpu_offsets_are_bolixed_dave(void); +extern void tsb_config_offsets_are_bolixed_dave(void); /* Only invoked on boot processor. */ void __init trap_init(void) @@ -2535,9 +2536,27 @@ void __init trap_init(void) (TRAP_PER_CPU_CPU_MONDO_BLOCK_PA != offsetof(struct trap_per_cpu, cpu_mondo_block_pa)) || (TRAP_PER_CPU_CPU_LIST_PA != - offsetof(struct trap_per_cpu, cpu_list_pa))) + offsetof(struct trap_per_cpu, cpu_list_pa)) || + (TRAP_PER_CPU_TSB_HUGE != + offsetof(struct trap_per_cpu, tsb_huge)) || + (TRAP_PER_CPU_TSB_HUGE_TEMP != + offsetof(struct trap_per_cpu, tsb_huge_temp))) trap_per_cpu_offsets_are_bolixed_dave(); + if ((TSB_CONFIG_TSB != + offsetof(struct tsb_config, tsb)) || + (TSB_CONFIG_RSS_LIMIT != + offsetof(struct tsb_config, tsb_rss_limit)) || + (TSB_CONFIG_NENTRIES != + offsetof(struct tsb_config, tsb_nentries)) || + (TSB_CONFIG_REG_VAL != + offsetof(struct tsb_config, tsb_reg_val)) || + (TSB_CONFIG_MAP_VADDR != + offsetof(struct tsb_config, tsb_map_vaddr)) || + (TSB_CONFIG_MAP_PTE != + offsetof(struct tsb_config, tsb_map_pte))) + tsb_config_offsets_are_bolixed_dave(); + /* Attach to the address space of init_task. On SMP we * do this in smp.c:smp_callin for other cpus. */ diff --git a/arch/sparc64/kernel/tsb.S b/arch/sparc64/kernel/tsb.S index 118baea44f69..a0c8ba58920b 100644 --- a/arch/sparc64/kernel/tsb.S +++ b/arch/sparc64/kernel/tsb.S @@ -3,8 +3,13 @@ * Copyright (C) 2006 David S. Miller */ +#include + #include #include +#include +#include +#include .text .align 32 @@ -34,34 +39,124 @@ tsb_miss_itlb: ldxa [%g4] ASI_IMMU, %g4 /* At this point we have: - * %g1 -- TSB entry address + * %g1 -- PAGE_SIZE TSB entry address * %g3 -- FAULT_CODE_{D,I}TLB * %g4 -- missing virtual address * %g6 -- TAG TARGET (vaddr >> 22) */ tsb_miss_page_table_walk: - TRAP_LOAD_PGD_PHYS(%g7, %g5) + TRAP_LOAD_TRAP_BLOCK(%g7, %g5) - /* And now we have the PGD base physical address in %g7. */ -tsb_miss_page_table_walk_sun4v_fastpath: - USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault) + /* Before committing to a full page table walk, + * check the huge page TSB. + */ +#ifdef CONFIG_HUGETLB_PAGE + +661: ldx [%g7 + TRAP_PER_CPU_TSB_HUGE], %g5 + nop + .section .sun4v_2insn_patch, "ax" + .word 661b + mov SCRATCHPAD_UTSBREG2, %g5 + ldxa [%g5] ASI_SCRATCHPAD, %g5 + .previous + + cmp %g5, -1 + be,pt %xcc, 80f + nop + + /* We need an aligned pair of registers containing 2 values + * which can be easily rematerialized. %g6 and %g7 foot the + * bill just nicely. We'll save %g6 away into %g2 for the + * huge page TSB TAG comparison. + * + * Perform a huge page TSB lookup. + */ + mov %g6, %g2 + and %g5, 0x7, %g6 + mov 512, %g7 + andn %g5, 0x7, %g5 + sllx %g7, %g6, %g7 + srlx %g4, HPAGE_SHIFT, %g6 + sub %g7, 1, %g7 + and %g6, %g7, %g6 + sllx %g6, 4, %g6 + add %g5, %g6, %g5 + + TSB_LOAD_QUAD(%g5, %g6) + cmp %g6, %g2 + be,a,pt %xcc, tsb_tlb_reload + mov %g7, %g5 + + /* No match, remember the huge page TSB entry address, + * and restore %g6 and %g7. + */ + TRAP_LOAD_TRAP_BLOCK(%g7, %g6) + srlx %g4, 22, %g6 +80: stx %g5, [%g7 + TRAP_PER_CPU_TSB_HUGE_TEMP] + +#endif + + ldx [%g7 + TRAP_PER_CPU_PGD_PADDR], %g7 /* At this point we have: * %g1 -- TSB entry address * %g3 -- FAULT_CODE_{D,I}TLB - * %g5 -- physical address of PTE in Linux page tables + * %g4 -- missing virtual address * %g6 -- TAG TARGET (vaddr >> 22) + * %g7 -- page table physical address + * + * We know that both the base PAGE_SIZE TSB and the HPAGE_SIZE + * TSB both lack a matching entry. */ -tsb_reload: - TSB_LOCK_TAG(%g1, %g2, %g7) +tsb_miss_page_table_walk_sun4v_fastpath: + USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault) /* Load and check PTE. */ ldxa [%g5] ASI_PHYS_USE_EC, %g5 - mov 1, %g7 - sllx %g7, TSB_TAG_INVALID_BIT, %g7 - brgez,a,pn %g5, tsb_do_fault - TSB_STORE(%g1, %g7) + brgez,pn %g5, tsb_do_fault + nop + +#ifdef CONFIG_HUGETLB_PAGE +661: sethi %uhi(_PAGE_SZALL_4U), %g7 + sllx %g7, 32, %g7 + .section .sun4v_2insn_patch, "ax" + .word 661b + mov _PAGE_SZALL_4V, %g7 + nop + .previous + + and %g5, %g7, %g2 + +661: sethi %uhi(_PAGE_SZHUGE_4U), %g7 + sllx %g7, 32, %g7 + .section .sun4v_2insn_patch, "ax" + .word 661b + mov _PAGE_SZHUGE_4V, %g7 + nop + .previous + + cmp %g2, %g7 + bne,pt %xcc, 60f + nop + + /* It is a huge page, use huge page TSB entry address we + * calculated above. + */ + TRAP_LOAD_TRAP_BLOCK(%g7, %g2) + ldx [%g7 + TRAP_PER_CPU_TSB_HUGE_TEMP], %g2 + cmp %g2, -1 + movne %xcc, %g2, %g1 +60: +#endif + /* At this point we have: + * %g1 -- TSB entry address + * %g3 -- FAULT_CODE_{D,I}TLB + * %g5 -- valid PTE + * %g6 -- TAG TARGET (vaddr >> 22) + */ +tsb_reload: + TSB_LOCK_TAG(%g1, %g2, %g7) TSB_WRITE(%g1, %g5, %g6) /* Finally, load TLB and return from trap. */ @@ -240,10 +335,9 @@ tsb_flush: * schedule() time. * * %o0: page table physical address - * %o1: TSB register value - * %o2: TSB virtual address - * %o3: TSB mapping locked PTE - * %o4: Hypervisor TSB descriptor physical address + * %o1: TSB base config pointer + * %o2: TSB huge config pointer, or NULL if none + * %o3: Hypervisor TSB descriptor physical address * * We have to run this whole thing with interrupts * disabled so that the current cpu doesn't change @@ -253,63 +347,79 @@ tsb_flush: .globl __tsb_context_switch .type __tsb_context_switch,#function __tsb_context_switch: - rdpr %pstate, %o5 - wrpr %o5, PSTATE_IE, %pstate + rdpr %pstate, %g1 + wrpr %g1, PSTATE_IE, %pstate + + TRAP_LOAD_TRAP_BLOCK(%g2, %g3) - ldub [%g6 + TI_CPU], %g1 - sethi %hi(trap_block), %g2 - sllx %g1, TRAP_BLOCK_SZ_SHIFT, %g1 - or %g2, %lo(trap_block), %g2 - add %g2, %g1, %g2 stx %o0, [%g2 + TRAP_PER_CPU_PGD_PADDR] - sethi %hi(tlb_type), %g1 - lduw [%g1 + %lo(tlb_type)], %g1 - cmp %g1, 3 - bne,pt %icc, 1f + ldx [%o1 + TSB_CONFIG_REG_VAL], %o0 + brz,pt %o2, 1f + mov -1, %g3 + + ldx [%o2 + TSB_CONFIG_REG_VAL], %g3 + +1: stx %g3, [%g2 + TRAP_PER_CPU_TSB_HUGE] + + sethi %hi(tlb_type), %g2 + lduw [%g2 + %lo(tlb_type)], %g2 + cmp %g2, 3 + bne,pt %icc, 50f nop /* Hypervisor TSB switch. */ - mov SCRATCHPAD_UTSBREG1, %g1 - stxa %o1, [%g1] ASI_SCRATCHPAD - mov -1, %g2 - mov SCRATCHPAD_UTSBREG2, %g1 - stxa %g2, [%g1] ASI_SCRATCHPAD - - /* Save away %o5's %pstate, we have to use %o5 for - * the hypervisor call. - */ - mov %o5, %g1 + mov SCRATCHPAD_UTSBREG1, %o5 + stxa %o0, [%o5] ASI_SCRATCHPAD + mov SCRATCHPAD_UTSBREG2, %o5 + stxa %g3, [%o5] ASI_SCRATCHPAD + + mov 2, %o0 + cmp %g3, -1 + move %xcc, 1, %o0 mov HV_FAST_MMU_TSB_CTXNON0, %o5 - mov 1, %o0 - mov %o4, %o1 + mov %o3, %o1 ta HV_FAST_TRAP - /* Finish up and restore %o5. */ + /* Finish up. */ ba,pt %xcc, 9f - mov %g1, %o5 + nop /* SUN4U TSB switch. */ -1: mov TSB_REG, %g1 - stxa %o1, [%g1] ASI_DMMU +50: mov TSB_REG, %o5 + stxa %o0, [%o5] ASI_DMMU membar #Sync - stxa %o1, [%g1] ASI_IMMU + stxa %o0, [%o5] ASI_IMMU membar #Sync -2: brz %o2, 9f - nop +2: ldx [%o1 + TSB_CONFIG_MAP_VADDR], %o4 + brz %o4, 9f + ldx [%o1 + TSB_CONFIG_MAP_PTE], %o5 sethi %hi(sparc64_highest_unlocked_tlb_ent), %g2 - mov TLB_TAG_ACCESS, %g1 + mov TLB_TAG_ACCESS, %g3 lduw [%g2 + %lo(sparc64_highest_unlocked_tlb_ent)], %g2 - stxa %o2, [%g1] ASI_DMMU + stxa %o4, [%g3] ASI_DMMU membar #Sync sllx %g2, 3, %g2 - stxa %o3, [%g2] ASI_DTLB_DATA_ACCESS + stxa %o5, [%g2] ASI_DTLB_DATA_ACCESS + membar #Sync + + brz,pt %o2, 9f + nop + + ldx [%o2 + TSB_CONFIG_MAP_VADDR], %o4 + ldx [%o2 + TSB_CONFIG_MAP_PTE], %o5 + mov TLB_TAG_ACCESS, %g3 + stxa %o4, [%g3] ASI_DMMU + membar #Sync + sub %g2, (1 << 3), %g2 + stxa %o5, [%g2] ASI_DTLB_DATA_ACCESS membar #Sync + 9: - wrpr %o5, %pstate + wrpr %g1, %pstate retl nop diff --git a/arch/sparc64/mm/fault.c b/arch/sparc64/mm/fault.c index 63b6cc0cd5d5..d21ff3230c02 100644 --- a/arch/sparc64/mm/fault.c +++ b/arch/sparc64/mm/fault.c @@ -410,9 +410,18 @@ good_area: up_read(&mm->mmap_sem); mm_rss = get_mm_rss(mm); - if (unlikely(mm_rss >= mm->context.tsb_rss_limit)) - tsb_grow(mm, mm_rss); - +#ifdef CONFIG_HUGETLB_PAGE + mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE / PAGE_SIZE)); +#endif + if (unlikely(mm_rss >= + mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit)) + tsb_grow(mm, MM_TSB_BASE, mm_rss); +#ifdef CONFIG_HUGETLB_PAGE + mm_rss = mm->context.huge_pte_count; + if (unlikely(mm_rss >= + mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) + tsb_grow(mm, MM_TSB_HUGE, mm_rss); +#endif return; /* diff --git a/arch/sparc64/mm/hugetlbpage.c b/arch/sparc64/mm/hugetlbpage.c index a7a24869d045..0a1d4cd24cda 100644 --- a/arch/sparc64/mm/hugetlbpage.c +++ b/arch/sparc64/mm/hugetlbpage.c @@ -199,13 +199,11 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) pte_t *pte = NULL; pgd = pgd_offset(mm, addr); - if (pgd) { - pud = pud_offset(pgd, addr); - if (pud) { - pmd = pmd_alloc(mm, pud, addr); - if (pmd) - pte = pte_alloc_map(mm, pmd, addr); - } + pud = pud_alloc(mm, pgd, addr); + if (pud) { + pmd = pmd_alloc(mm, pud, addr); + if (pmd) + pte = pte_alloc_map(mm, pmd, addr); } return pte; } @@ -231,13 +229,14 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) return pte; } -#define mk_pte_huge(entry) do { pte_val(entry) |= _PAGE_SZHUGE; } while (0) - void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t entry) { int i; + if (!pte_present(*ptep) && pte_present(entry)) + mm->context.huge_pte_count++; + for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { set_pte_at(mm, addr, ptep, entry); ptep++; @@ -253,6 +252,8 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, int i; entry = *ptep; + if (pte_present(entry)) + mm->context.huge_pte_count--; for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { pte_clear(mm, addr, ptep); @@ -302,6 +303,15 @@ static void context_reload(void *__data) void hugetlb_prefault_arch_hook(struct mm_struct *mm) { + struct tsb_config *tp = &mm->context.tsb_block[MM_TSB_HUGE]; + + if (likely(tp->tsb != NULL)) + return; + + tsb_grow(mm, MM_TSB_HUGE, 0); + tsb_context_switch(mm); + smp_tsb_sync(mm); + /* On UltraSPARC-III+ and later, configure the second half of * the Data-TLB for huge pages. */ diff --git a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c index c2b556106fc1..16d231703d6a 100644 --- a/arch/sparc64/mm/init.c +++ b/arch/sparc64/mm/init.c @@ -283,6 +283,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t p struct mm_struct *mm; struct tsb *tsb; unsigned long tag, flags; + unsigned long tsb_index, tsb_hash_shift; if (tlb_type != hypervisor) { unsigned long pfn = pte_pfn(pte); @@ -312,10 +313,26 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t p mm = vma->vm_mm; + tsb_index = MM_TSB_BASE; + tsb_hash_shift = PAGE_SHIFT; + spin_lock_irqsave(&mm->context.lock, flags); - tsb = &mm->context.tsb[(address >> PAGE_SHIFT) & - (mm->context.tsb_nentries - 1UL)]; +#ifdef CONFIG_HUGETLB_PAGE + if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) { + if ((tlb_type == hypervisor && + (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) || + (tlb_type != hypervisor && + (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U)) { + tsb_index = MM_TSB_HUGE; + tsb_hash_shift = HPAGE_SHIFT; + } + } +#endif + + tsb = mm->context.tsb_block[tsb_index].tsb; + tsb += ((address >> tsb_hash_shift) & + (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL)); tag = (address >> 22UL); tsb_insert(tsb, tag, pte_val(pte)); diff --git a/arch/sparc64/mm/tsb.c b/arch/sparc64/mm/tsb.c index b2064e2a44d6..beaa02810f0e 100644 --- a/arch/sparc64/mm/tsb.c +++ b/arch/sparc64/mm/tsb.c @@ -15,9 +15,9 @@ extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; -static inline unsigned long tsb_hash(unsigned long vaddr, unsigned long nentries) +static inline unsigned long tsb_hash(unsigned long vaddr, unsigned long hash_shift, unsigned long nentries) { - vaddr >>= PAGE_SHIFT; + vaddr >>= hash_shift; return vaddr & (nentries - 1); } @@ -36,7 +36,8 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end) unsigned long v; for (v = start; v < end; v += PAGE_SIZE) { - unsigned long hash = tsb_hash(v, KERNEL_TSB_NENTRIES); + unsigned long hash = tsb_hash(v, PAGE_SHIFT, + KERNEL_TSB_NENTRIES); struct tsb *ent = &swapper_tsb[hash]; if (tag_compare(ent->tag, v)) { @@ -46,49 +47,91 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end) } } -void flush_tsb_user(struct mmu_gather *mp) +static void __flush_tsb_one(struct mmu_gather *mp, unsigned long hash_shift, unsigned long tsb, unsigned long nentries) { - struct mm_struct *mm = mp->mm; - unsigned long nentries, base, flags; - struct tsb *tsb; - int i; - - spin_lock_irqsave(&mm->context.lock, flags); - - tsb = mm->context.tsb; - nentries = mm->context.tsb_nentries; + unsigned long i; - if (tlb_type == cheetah_plus || tlb_type == hypervisor) - base = __pa(tsb); - else - base = (unsigned long) tsb; - for (i = 0; i < mp->tlb_nr; i++) { unsigned long v = mp->vaddrs[i]; unsigned long tag, ent, hash; v &= ~0x1UL; - hash = tsb_hash(v, nentries); - ent = base + (hash * sizeof(struct tsb)); + hash = tsb_hash(v, hash_shift, nentries); + ent = tsb + (hash * sizeof(struct tsb)); tag = (v >> 22UL); tsb_flush(ent, tag); } +} + +void flush_tsb_user(struct mmu_gather *mp) +{ + struct mm_struct *mm = mp->mm; + unsigned long nentries, base, flags; + + spin_lock_irqsave(&mm->context.lock, flags); + base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; + nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; + if (tlb_type == cheetah_plus || tlb_type == hypervisor) + base = __pa(base); + __flush_tsb_one(mp, PAGE_SHIFT, base, nentries); + +#ifdef CONFIG_HUGETLB_PAGE + if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { + base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; + nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; + if (tlb_type == cheetah_plus || tlb_type == hypervisor) + base = __pa(base); + __flush_tsb_one(mp, HPAGE_SHIFT, base, nentries); + } +#endif spin_unlock_irqrestore(&mm->context.lock, flags); } -static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_bytes) +#if defined(CONFIG_SPARC64_PAGE_SIZE_8KB) +#define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_8K +#define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_8K +#elif defined(CONFIG_SPARC64_PAGE_SIZE_64KB) +#define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_64K +#define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_64K +#elif defined(CONFIG_SPARC64_PAGE_SIZE_512KB) +#define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_512K +#define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_512K +#elif defined(CONFIG_SPARC64_PAGE_SIZE_4MB) +#define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_4MB +#define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_4MB +#else +#error Broken base page size setting... +#endif + +#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE_SIZE_64K) +#define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_64K +#define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_64K +#elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K) +#define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_512K +#define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_512K +#elif defined(CONFIG_HUGETLB_PAGE_SIZE_4MB) +#define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_4MB +#define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_4MB +#else +#error Broken huge page size setting... +#endif +#endif + +static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsigned long tsb_bytes) { unsigned long tsb_reg, base, tsb_paddr; unsigned long page_sz, tte; - mm->context.tsb_nentries = tsb_bytes / sizeof(struct tsb); + mm->context.tsb_block[tsb_idx].tsb_nentries = + tsb_bytes / sizeof(struct tsb); base = TSBMAP_BASE; tte = pgprot_val(PAGE_KERNEL_LOCKED); - tsb_paddr = __pa(mm->context.tsb); + tsb_paddr = __pa(mm->context.tsb_block[tsb_idx].tsb); BUG_ON(tsb_paddr & (tsb_bytes - 1UL)); /* Use the smallest page size that can map the whole TSB @@ -147,61 +190,49 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_bytes) /* Physical mapping, no locked TLB entry for TSB. */ tsb_reg |= tsb_paddr; - mm->context.tsb_reg_val = tsb_reg; - mm->context.tsb_map_vaddr = 0; - mm->context.tsb_map_pte = 0; + mm->context.tsb_block[tsb_idx].tsb_reg_val = tsb_reg; + mm->context.tsb_block[tsb_idx].tsb_map_vaddr = 0; + mm->context.tsb_block[tsb_idx].tsb_map_pte = 0; } else { tsb_reg |= base; tsb_reg |= (tsb_paddr & (page_sz - 1UL)); tte |= (tsb_paddr & ~(page_sz - 1UL)); - mm->context.tsb_reg_val = tsb_reg; - mm->context.tsb_map_vaddr = base; - mm->context.tsb_map_pte = tte; + mm->context.tsb_block[tsb_idx].tsb_reg_val = tsb_reg; + mm->context.tsb_block[tsb_idx].tsb_map_vaddr = base; + mm->context.tsb_block[tsb_idx].tsb_map_pte = tte; } /* Setup the Hypervisor TSB descriptor. */ if (tlb_type == hypervisor) { - struct hv_tsb_descr *hp = &mm->context.tsb_descr; + struct hv_tsb_descr *hp = &mm->context.tsb_descr[tsb_idx]; - switch (PAGE_SIZE) { - case 8192: - default: - hp->pgsz_idx = HV_PGSZ_IDX_8K; + switch (tsb_idx) { + case MM_TSB_BASE: + hp->pgsz_idx = HV_PGSZ_IDX_BASE; break; - - case 64 * 1024: - hp->pgsz_idx = HV_PGSZ_IDX_64K; - break; - - case 512 * 1024: - hp->pgsz_idx = HV_PGSZ_IDX_512K; - break; - - case 4 * 1024 * 1024: - hp->pgsz_idx = HV_PGSZ_IDX_4MB; +#ifdef CONFIG_HUGETLB_PAGE + case MM_TSB_HUGE: + hp->pgsz_idx = HV_PGSZ_IDX_HUGE; break; +#endif + default: + BUG(); }; hp->assoc = 1; hp->num_ttes = tsb_bytes / 16; hp->ctx_idx = 0; - switch (PAGE_SIZE) { - case 8192: - default: - hp->pgsz_mask = HV_PGSZ_MASK_8K; - break; - - case 64 * 1024: - hp->pgsz_mask = HV_PGSZ_MASK_64K; - break; - - case 512 * 1024: - hp->pgsz_mask = HV_PGSZ_MASK_512K; + switch (tsb_idx) { + case MM_TSB_BASE: + hp->pgsz_mask = HV_PGSZ_MASK_BASE; break; - - case 4 * 1024 * 1024: - hp->pgsz_mask = HV_PGSZ_MASK_4MB; +#ifdef CONFIG_HUGETLB_PAGE + case MM_TSB_HUGE: + hp->pgsz_mask = HV_PGSZ_MASK_HUGE; break; +#endif + default: + BUG(); }; hp->tsb_base = tsb_paddr; hp->resv = 0; @@ -241,11 +272,11 @@ void __init tsb_cache_init(void) } } -/* When the RSS of an address space exceeds mm->context.tsb_rss_limit, - * do_sparc64_fault() invokes this routine to try and grow the TSB. +/* When the RSS of an address space exceeds tsb_rss_limit for a TSB, + * do_sparc64_fault() invokes this routine to try and grow it. * * When we reach the maximum TSB size supported, we stick ~0UL into - * mm->context.tsb_rss_limit so the grow checks in update_mmu_cache() + * tsb_rss_limit for that TSB so the grow checks in do_sparc64_fault() * will not trigger any longer. * * The TSB can be anywhere from 8K to 1MB in size, in increasing powers @@ -257,7 +288,7 @@ void __init tsb_cache_init(void) * the number of entries that the current TSB can hold at once. Currently, * we trigger when the RSS hits 3/4 of the TSB capacity. */ -void tsb_grow(struct mm_struct *mm, unsigned long rss) +void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long rss) { unsigned long max_tsb_size = 1 * 1024 * 1024; unsigned long new_size, old_size, flags; @@ -297,7 +328,8 @@ retry_tsb_alloc: * down to a 0-order allocation and force no TSB * growing for this address space. */ - if (mm->context.tsb == NULL && new_cache_index > 0) { + if (mm->context.tsb_block[tsb_index].tsb == NULL && + new_cache_index > 0) { new_cache_index = 0; new_size = 8192; new_rss_limit = ~0UL; @@ -307,8 +339,8 @@ retry_tsb_alloc: /* If we failed on a TSB grow, we are under serious * memory pressure so don't try to grow any more. */ - if (mm->context.tsb != NULL) - mm->context.tsb_rss_limit = ~0UL; + if (mm->context.tsb_block[tsb_index].tsb != NULL) + mm->context.tsb_block[tsb_index].tsb_rss_limit = ~0UL; return; } @@ -339,23 +371,26 @@ retry_tsb_alloc: */ spin_lock_irqsave(&mm->context.lock, flags); - old_tsb = mm->context.tsb; - old_cache_index = (mm->context.tsb_reg_val & 0x7UL); - old_size = mm->context.tsb_nentries * sizeof(struct tsb); + old_tsb = mm->context.tsb_block[tsb_index].tsb; + old_cache_index = + (mm->context.tsb_block[tsb_index].tsb_reg_val & 0x7UL); + old_size = (mm->context.tsb_block[tsb_index].tsb_nentries * + sizeof(struct tsb)); /* Handle multiple threads trying to grow the TSB at the same time. * One will get in here first, and bump the size and the RSS limit. * The others will get in here next and hit this check. */ - if (unlikely(old_tsb && (rss < mm->context.tsb_rss_limit))) { + if (unlikely(old_tsb && + (rss < mm->context.tsb_block[tsb_index].tsb_rss_limit))) { spin_unlock_irqrestore(&mm->context.lock, flags); kmem_cache_free(tsb_caches[new_cache_index], new_tsb); return; } - mm->context.tsb_rss_limit = new_rss_limit; + mm->context.tsb_block[tsb_index].tsb_rss_limit = new_rss_limit; if (old_tsb) { extern void copy_tsb(unsigned long old_tsb_base, @@ -372,8 +407,8 @@ retry_tsb_alloc: copy_tsb(old_tsb_base, old_size, new_tsb_base, new_size); } - mm->context.tsb = new_tsb; - setup_tsb_params(mm, new_size); + mm->context.tsb_block[tsb_index].tsb = new_tsb; + setup_tsb_params(mm, tsb_index, new_size); spin_unlock_irqrestore(&mm->context.lock, flags); @@ -394,40 +429,65 @@ retry_tsb_alloc: int init_new_context(struct task_struct *tsk, struct mm_struct *mm) { +#ifdef CONFIG_HUGETLB_PAGE + unsigned long huge_pte_count; +#endif + unsigned int i; + spin_lock_init(&mm->context.lock); mm->context.sparc64_ctx_val = 0UL; +#ifdef CONFIG_HUGETLB_PAGE + /* We reset it to zero because the fork() page copying + * will re-increment the counters as the parent PTEs are + * copied into the child address space. + */ + huge_pte_count = mm->context.huge_pte_count; + mm->context.huge_pte_count = 0; +#endif + /* copy_mm() copies over the parent's mm_struct before calling * us, so we need to zero out the TSB pointer or else tsb_grow() * will be confused and think there is an older TSB to free up. */ - mm->context.tsb = NULL; + for (i = 0; i < MM_NUM_TSBS; i++) + mm->context.tsb_block[i].tsb = NULL; /* If this is fork, inherit the parent's TSB size. We would * grow it to that size on the first page fault anyways. */ - tsb_grow(mm, get_mm_rss(mm)); + tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm)); - if (unlikely(!mm->context.tsb)) +#ifdef CONFIG_HUGETLB_PAGE + if (unlikely(huge_pte_count)) + tsb_grow(mm, MM_TSB_HUGE, huge_pte_count); +#endif + + if (unlikely(!mm->context.tsb_block[MM_TSB_BASE].tsb)) return -ENOMEM; return 0; } -void destroy_context(struct mm_struct *mm) +static void tsb_destroy_one(struct tsb_config *tp) { - unsigned long flags, cache_index; + unsigned long cache_index; - cache_index = (mm->context.tsb_reg_val & 0x7UL); - kmem_cache_free(tsb_caches[cache_index], mm->context.tsb); + if (!tp->tsb) + return; + cache_index = tp->tsb_reg_val & 0x7UL; + kmem_cache_free(tsb_caches[cache_index], tp->tsb); + tp->tsb = NULL; + tp->tsb_reg_val = 0UL; +} - /* We can remove these later, but for now it's useful - * to catch any bogus post-destroy_context() references - * to the TSB. - */ - mm->context.tsb = NULL; - mm->context.tsb_reg_val = 0UL; +void destroy_context(struct mm_struct *mm) +{ + unsigned long flags, i; + + for (i = 0; i < MM_NUM_TSBS; i++) + tsb_destroy_one(&mm->context.tsb_block[i]); spin_lock_irqsave(&ctx_alloc_lock, flags); diff --git a/include/asm-sparc64/cpudata.h b/include/asm-sparc64/cpudata.h index c66a81bbc84d..9d6a6dbaf126 100644 --- a/include/asm-sparc64/cpudata.h +++ b/include/asm-sparc64/cpudata.h @@ -71,7 +71,8 @@ struct trap_per_cpu { /* Dcache line 7: Physical addresses of CPU send mondo block and CPU list. */ unsigned long cpu_mondo_block_pa; unsigned long cpu_list_pa; - unsigned long __pad1[2]; + unsigned long tsb_huge; + unsigned long tsb_huge_temp; /* Dcache line 8: Unused, needed to keep trap_block a power-of-2 in size. */ unsigned long __pad2[4]; @@ -116,6 +117,8 @@ extern struct sun4v_2insn_patch_entry __sun4v_2insn_patch, #define TRAP_PER_CPU_FAULT_INFO 0x40 #define TRAP_PER_CPU_CPU_MONDO_BLOCK_PA 0xc0 #define TRAP_PER_CPU_CPU_LIST_PA 0xc8 +#define TRAP_PER_CPU_TSB_HUGE 0xd0 +#define TRAP_PER_CPU_TSB_HUGE_TEMP 0xd8 #define TRAP_BLOCK_SZ_SHIFT 8 diff --git a/include/asm-sparc64/mmu.h b/include/asm-sparc64/mmu.h index 230ba678d3b0..2d4f2ea9568a 100644 --- a/include/asm-sparc64/mmu.h +++ b/include/asm-sparc64/mmu.h @@ -90,18 +90,39 @@ extern void __tsb_insert(unsigned long ent, unsigned long tag, unsigned long pte extern void tsb_flush(unsigned long ent, unsigned long tag); extern void tsb_init(struct tsb *tsb, unsigned long size); -typedef struct { - spinlock_t lock; - unsigned long sparc64_ctx_val; +struct tsb_config { struct tsb *tsb; unsigned long tsb_rss_limit; unsigned long tsb_nentries; unsigned long tsb_reg_val; unsigned long tsb_map_vaddr; unsigned long tsb_map_pte; - struct hv_tsb_descr tsb_descr; +}; + +#define MM_TSB_BASE 0 + +#ifdef CONFIG_HUGETLB_PAGE +#define MM_TSB_HUGE 1 +#define MM_NUM_TSBS 2 +#else +#define MM_NUM_TSBS 1 +#endif + +typedef struct { + spinlock_t lock; + unsigned long sparc64_ctx_val; + unsigned long huge_pte_count; + struct tsb_config tsb_block[MM_NUM_TSBS]; + struct hv_tsb_descr tsb_descr[MM_NUM_TSBS]; } mm_context_t; #endif /* !__ASSEMBLY__ */ +#define TSB_CONFIG_TSB 0x00 +#define TSB_CONFIG_RSS_LIMIT 0x08 +#define TSB_CONFIG_NENTRIES 0x10 +#define TSB_CONFIG_REG_VAL 0x18 +#define TSB_CONFIG_MAP_VADDR 0x20 +#define TSB_CONFIG_MAP_PTE 0x28 + #endif /* __MMU_H */ diff --git a/include/asm-sparc64/mmu_context.h b/include/asm-sparc64/mmu_context.h index e7974321d052..2337eb487719 100644 --- a/include/asm-sparc64/mmu_context.h +++ b/include/asm-sparc64/mmu_context.h @@ -29,20 +29,25 @@ extern int init_new_context(struct task_struct *tsk, struct mm_struct *mm); extern void destroy_context(struct mm_struct *mm); extern void __tsb_context_switch(unsigned long pgd_pa, - unsigned long tsb_reg, - unsigned long tsb_vaddr, - unsigned long tsb_pte, + struct tsb_config *tsb_base, + struct tsb_config *tsb_huge, unsigned long tsb_descr_pa); static inline void tsb_context_switch(struct mm_struct *mm) { - __tsb_context_switch(__pa(mm->pgd), mm->context.tsb_reg_val, - mm->context.tsb_map_vaddr, - mm->context.tsb_map_pte, - __pa(&mm->context.tsb_descr)); + __tsb_context_switch(__pa(mm->pgd), + &mm->context.tsb_block[0], +#ifdef CONFIG_HUGETLB_PAGE + (mm->context.tsb_block[1].tsb ? + &mm->context.tsb_block[1] : + NULL) +#else + NULL +#endif + , __pa(&mm->context.tsb_descr[0])); } -extern void tsb_grow(struct mm_struct *mm, unsigned long mm_rss); +extern void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long mm_rss); #ifdef CONFIG_SMP extern void smp_tsb_sync(struct mm_struct *mm); #else diff --git a/include/asm-sparc64/page.h b/include/asm-sparc64/page.h index fcb2812265f4..66fe4ac59fd6 100644 --- a/include/asm-sparc64/page.h +++ b/include/asm-sparc64/page.h @@ -30,6 +30,23 @@ #ifdef __KERNEL__ +#if defined(CONFIG_HUGETLB_PAGE_SIZE_4MB) +#define HPAGE_SHIFT 22 +#elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K) +#define HPAGE_SHIFT 19 +#elif defined(CONFIG_HUGETLB_PAGE_SIZE_64K) +#define HPAGE_SHIFT 16 +#endif + +#ifdef CONFIG_HUGETLB_PAGE +#define HPAGE_SIZE (_AC(1,UL) << HPAGE_SHIFT) +#define HPAGE_MASK (~(HPAGE_SIZE - 1UL)) +#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) +#define ARCH_HAS_SETCLEAR_HUGE_PTE +#define ARCH_HAS_HUGETLB_PREFAULT_HOOK +#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA +#endif + #ifndef __ASSEMBLY__ extern void _clear_page(void *page); @@ -90,23 +107,6 @@ typedef unsigned long pgprot_t; #endif /* (STRICT_MM_TYPECHECKS) */ -#if defined(CONFIG_HUGETLB_PAGE_SIZE_4MB) -#define HPAGE_SHIFT 22 -#elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K) -#define HPAGE_SHIFT 19 -#elif defined(CONFIG_HUGETLB_PAGE_SIZE_64K) -#define HPAGE_SHIFT 16 -#endif - -#ifdef CONFIG_HUGETLB_PAGE -#define HPAGE_SIZE (_AC(1,UL) << HPAGE_SHIFT) -#define HPAGE_MASK (~(HPAGE_SIZE - 1UL)) -#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) -#define ARCH_HAS_SETCLEAR_HUGE_PTE -#define ARCH_HAS_HUGETLB_PREFAULT_HOOK -#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA -#endif - #define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \ (_AC(0x0000000070000000,UL)) : \ (_AC(0xfffff80000000000,UL) + (1UL << 32UL))) diff --git a/include/asm-sparc64/pgtable.h b/include/asm-sparc64/pgtable.h index ed4124edf837..c44e7466534e 100644 --- a/include/asm-sparc64/pgtable.h +++ b/include/asm-sparc64/pgtable.h @@ -105,6 +105,7 @@ #define _PAGE_RES1_4U _AC(0x0002000000000000,UL) /* Reserved */ #define _PAGE_SZ32MB_4U _AC(0x0001000000000000,UL) /* (Panther) 32MB page */ #define _PAGE_SZ256MB_4U _AC(0x2001000000000000,UL) /* (Panther) 256MB page */ +#define _PAGE_SZALL_4U _AC(0x6001000000000000,UL) /* All pgsz bits */ #define _PAGE_SN_4U _AC(0x0000800000000000,UL) /* (Cheetah) Snoop */ #define _PAGE_RES2_4U _AC(0x0000780000000000,UL) /* Reserved */ #define _PAGE_PADDR_4U _AC(0x000007FFFFFFE000,UL) /* (Cheetah) pa[42:13] */ @@ -150,6 +151,7 @@ #define _PAGE_SZ512K_4V _AC(0x0000000000000002,UL) /* 512K Page */ #define _PAGE_SZ64K_4V _AC(0x0000000000000001,UL) /* 64K Page */ #define _PAGE_SZ8K_4V _AC(0x0000000000000000,UL) /* 8K Page */ +#define _PAGE_SZALL_4V _AC(0x0000000000000007,UL) /* All pgsz bits */ #if PAGE_SHIFT == 13 #define _PAGE_SZBITS_4U _PAGE_SZ8K_4U -- cgit v1.2.3 From 394e3902c55e667945f6f1c2bdbc59842cce70f7 Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Thu, 23 Mar 2006 03:01:05 -0800 Subject: [PATCH] more for_each_cpu() conversions When we stop allocating percpu memory for not-possible CPUs we must not touch the percpu data for not-possible CPUs at all. The correct way of doing this is to test cpu_possible() or to use for_each_cpu(). This patch is a kernel-wide sweep of all instances of NR_CPUS. I found very few instances of this bug, if any. But the patch converts lots of open-coded test to use the preferred helper macros. Cc: Mikael Starvik Cc: David Howells Acked-by: Kyle McMartin Cc: Anton Blanchard Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Martin Schwidefsky Cc: Heiko Carstens Cc: Paul Mundt Cc: "David S. Miller" Cc: William Lee Irwin III Cc: Andi Kleen Cc: Christian Zankel Cc: Philippe Elie Cc: Nathan Scott Cc: Jens Axboe Cc: Eric Dumazet Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/cris/kernel/irq.c | 10 ++++------ arch/frv/kernel/irq.c | 10 ++++------ arch/i386/kernel/cpu/cpufreq/powernow-k8.c | 4 +--- arch/i386/kernel/io_apic.c | 22 +++++++++------------- arch/i386/kernel/nmi.c | 4 ++-- arch/i386/oprofile/nmi_int.c | 7 ++----- arch/m32r/kernel/irq.c | 10 ++++------ arch/mips/kernel/irq.c | 10 ++++------ arch/mips/kernel/smp.c | 4 ++-- arch/mips/sgi-ip27/ip27-irq.c | 5 +---- arch/parisc/kernel/smp.c | 25 ++++++++++--------------- arch/powerpc/kernel/irq.c | 5 ++--- arch/powerpc/kernel/setup-common.c | 5 ++--- arch/powerpc/kernel/setup_32.c | 5 ++--- arch/powerpc/platforms/powermac/smp.c | 4 +--- arch/ppc/kernel/setup.c | 10 ++++------ arch/s390/kernel/smp.c | 4 +--- arch/sh/kernel/irq.c | 5 ++--- arch/sh/kernel/setup.c | 5 ++--- arch/sh64/kernel/irq.c | 5 ++--- arch/sparc/kernel/irq.c | 5 ++--- arch/sparc/kernel/smp.c | 24 ++++++++++-------------- arch/sparc/kernel/sun4d_irq.c | 8 +++----- arch/sparc/kernel/sun4d_smp.c | 8 +++----- arch/sparc/kernel/sun4m_smp.c | 6 ++---- arch/sparc64/kernel/irq.c | 4 +--- arch/sparc64/kernel/smp.c | 30 ++++++++++++------------------ arch/x86_64/kernel/irq.c | 21 ++++++++------------- arch/x86_64/kernel/nmi.c | 4 +--- arch/xtensa/kernel/irq.c | 15 ++++++--------- drivers/net/loopback.c | 4 +--- drivers/oprofile/cpu_buffer.c | 3 +-- fs/xfs/linux-2.6/xfs_stats.c | 7 ++----- fs/xfs/linux-2.6/xfs_sysctl.c | 3 +-- include/asm-alpha/mmu_context.h | 5 ++--- include/asm-alpha/topology.h | 4 ++-- include/asm-generic/percpu.h | 7 +++---- include/asm-powerpc/percpu.h | 7 +++---- include/asm-s390/percpu.h | 7 +++---- include/asm-sparc64/percpu.h | 7 +++---- include/asm-x86_64/percpu.h | 7 +++---- include/linux/genhd.h | 14 ++++---------- 42 files changed, 137 insertions(+), 222 deletions(-) (limited to 'arch/sparc64/kernel') diff --git a/arch/cris/kernel/irq.c b/arch/cris/kernel/irq.c index 30deaf1b728a..b504def3e346 100644 --- a/arch/cris/kernel/irq.c +++ b/arch/cris/kernel/irq.c @@ -52,9 +52,8 @@ int show_interrupts(struct seq_file *p, void *v) if (i == 0) { seq_printf(p, " "); - for (j=0; jtypename); seq_printf(p, " %s", action->name); diff --git a/arch/frv/kernel/irq.c b/arch/frv/kernel/irq.c index 27ab4c30aac6..11fa326a8f62 100644 --- a/arch/frv/kernel/irq.c +++ b/arch/frv/kernel/irq.c @@ -75,9 +75,8 @@ int show_interrupts(struct seq_file *p, void *v) switch (i) { case 0: seq_printf(p, " "); - for (j = 0; j < NR_CPUS; j++) - if (cpu_online(j)) - seq_printf(p, "CPU%d ",j); + for_each_online_cpu(j) + seq_printf(p, "CPU%d ",j); seq_putc(p, '\n'); break; @@ -100,9 +99,8 @@ int show_interrupts(struct seq_file *p, void *v) #ifndef CONFIG_SMP seq_printf(p, "%10u ", kstat_irqs(i)); #else - for (j = 0; j < NR_CPUS; j++) - if (cpu_online(j)) - seq_printf(p, "%10u ", kstat_cpu(j).irqs[i - 1]); + for_each_online_cpu(j) + seq_printf(p, "%10u ", kstat_cpu(j).irqs[i - 1]); #endif level = group->sources[ix]->level - frv_irq_levels; diff --git a/arch/i386/kernel/cpu/cpufreq/powernow-k8.c b/arch/i386/kernel/cpu/cpufreq/powernow-k8.c index e11a09207ec8..3d5110b65cc3 100644 --- a/arch/i386/kernel/cpu/cpufreq/powernow-k8.c +++ b/arch/i386/kernel/cpu/cpufreq/powernow-k8.c @@ -1145,9 +1145,7 @@ static int __cpuinit powernowk8_init(void) { unsigned int i, supported_cpus = 0; - for (i=0; i CPU_IRQ(i)) { @@ -441,9 +439,7 @@ tryanothercpu: */ tmp_cpu_irq = 0; tmp_loaded = -1; - for (i = 0; i < NR_CPUS; i++) { - if (!cpu_online(i)) - continue; + for_each_online_cpu(i) { if (i != CPU_TO_PACKAGEINDEX(i)) continue; if (max_cpu_irq <= CPU_IRQ(i)) @@ -619,9 +615,7 @@ static int __init balanced_irq_init(void) if (smp_num_siblings > 1 && !cpus_empty(tmp)) physical_balance = 1; - for (i = 0; i < NR_CPUS; i++) { - if (!cpu_online(i)) - continue; + for_each_online_cpu(i) { irq_cpu_data[i].irq_delta = kmalloc(sizeof(unsigned long) * NR_IRQS, GFP_KERNEL); irq_cpu_data[i].last_irq = kmalloc(sizeof(unsigned long) * NR_IRQS, GFP_KERNEL); if (irq_cpu_data[i].irq_delta == NULL || irq_cpu_data[i].last_irq == NULL) { @@ -638,9 +632,11 @@ static int __init balanced_irq_init(void) else printk(KERN_ERR "balanced_irq_init: failed to spawn balanced_irq"); failed: - for (i = 0; i < NR_CPUS; i++) { + for_each_cpu(i) { kfree(irq_cpu_data[i].irq_delta); + irq_cpu_data[i].irq_delta = NULL; kfree(irq_cpu_data[i].last_irq); + irq_cpu_data[i].last_irq = NULL; } return 0; } diff --git a/arch/i386/kernel/nmi.c b/arch/i386/kernel/nmi.c index 1db34effdd8d..9074818b9473 100644 --- a/arch/i386/kernel/nmi.c +++ b/arch/i386/kernel/nmi.c @@ -143,7 +143,7 @@ static int __init check_nmi_watchdog(void) local_irq_enable(); mdelay((10*1000)/nmi_hz); // wait 10 ticks - for (cpu = 0; cpu < NR_CPUS; cpu++) { + for_each_cpu(cpu) { #ifdef CONFIG_SMP /* Check cpu_callin_map here because that is set after the timer is started. */ @@ -510,7 +510,7 @@ void touch_nmi_watchdog (void) * Just reset the alert counters, (other CPUs might be * spinning on locks we hold): */ - for (i = 0; i < NR_CPUS; i++) + for_each_cpu(i) alert_counter[i] = 0; /* diff --git a/arch/i386/oprofile/nmi_int.c b/arch/i386/oprofile/nmi_int.c index 0493e8b8ec49..1accce50c2c7 100644 --- a/arch/i386/oprofile/nmi_int.c +++ b/arch/i386/oprofile/nmi_int.c @@ -122,7 +122,7 @@ static void nmi_save_registers(void * dummy) static void free_msrs(void) { int i; - for (i = 0; i < NR_CPUS; ++i) { + for_each_cpu(i) { kfree(cpu_msrs[i].counters); cpu_msrs[i].counters = NULL; kfree(cpu_msrs[i].controls); @@ -138,10 +138,7 @@ static int allocate_msrs(void) size_t counters_size = sizeof(struct op_msr) * model->num_counters; int i; - for (i = 0; i < NR_CPUS; ++i) { - if (!cpu_online(i)) - continue; - + for_each_online_cpu(i) { cpu_msrs[i].counters = kmalloc(counters_size, GFP_KERNEL); if (!cpu_msrs[i].counters) { success = 0; diff --git a/arch/m32r/kernel/irq.c b/arch/m32r/kernel/irq.c index 1ce63926a3c0..a4634b06f675 100644 --- a/arch/m32r/kernel/irq.c +++ b/arch/m32r/kernel/irq.c @@ -37,9 +37,8 @@ int show_interrupts(struct seq_file *p, void *v) if (i == 0) { seq_printf(p, " "); - for (j=0; jtypename); seq_printf(p, " %s", action->name); diff --git a/arch/mips/kernel/irq.c b/arch/mips/kernel/irq.c index 7d93992e462c..3dd76b3d2967 100644 --- a/arch/mips/kernel/irq.c +++ b/arch/mips/kernel/irq.c @@ -68,9 +68,8 @@ int show_interrupts(struct seq_file *p, void *v) if (i == 0) { seq_printf(p, " "); - for (j=0; jtypename); seq_printf(p, " %s", action->name); diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c index 06ed90752424..78d171bfa331 100644 --- a/arch/mips/kernel/smp.c +++ b/arch/mips/kernel/smp.c @@ -167,8 +167,8 @@ int smp_call_function (void (*func) (void *info), void *info, int retry, mb(); /* Send a message to all other CPUs and wait for them to respond */ - for (i = 0; i < NR_CPUS; i++) - if (cpu_online(i) && i != cpu) + for_each_online_cpu(i) + if (i != cpu) core_send_ipi(i, SMP_CALL_FUNCTION); /* Wait for response */ diff --git a/arch/mips/sgi-ip27/ip27-irq.c b/arch/mips/sgi-ip27/ip27-irq.c index 73e5e52781d8..2854ac4c9be1 100644 --- a/arch/mips/sgi-ip27/ip27-irq.c +++ b/arch/mips/sgi-ip27/ip27-irq.c @@ -88,12 +88,9 @@ static inline int find_level(cpuid_t *cpunum, int irq) { int cpu, i; - for (cpu = 0; cpu <= NR_CPUS; cpu++) { + for_each_online_cpu(cpu) { struct slice_data *si = cpu_data[cpu].data; - if (!cpu_online(cpu)) - continue; - for (i = BASE_PCI_IRQ; i < LEVELS_PER_SLICE; i++) if (si->level_to_irq[i] == irq) { *cpunum = cpu; diff --git a/arch/parisc/kernel/smp.c b/arch/parisc/kernel/smp.c index 25564b7ca6bb..d6ac1c60a471 100644 --- a/arch/parisc/kernel/smp.c +++ b/arch/parisc/kernel/smp.c @@ -298,8 +298,8 @@ send_IPI_allbutself(enum ipi_message_type op) { int i; - for (i = 0; i < NR_CPUS; i++) { - if (cpu_online(i) && i != smp_processor_id()) + for_each_online_cpu(i) { + if (i != smp_processor_id()) send_IPI_single(i, op); } } @@ -643,14 +643,13 @@ int sys_cpus(int argc, char **argv) if ( argc == 1 ){ #ifdef DUMP_MORE_STATE - for(i=0; iflags & SA_INTERRUPT) ? '+' : ' ', diff --git a/arch/sparc/kernel/sun4d_smp.c b/arch/sparc/kernel/sun4d_smp.c index 4219dd2ce3a2..41bb9596be48 100644 --- a/arch/sparc/kernel/sun4d_smp.c +++ b/arch/sparc/kernel/sun4d_smp.c @@ -249,11 +249,9 @@ void __init smp4d_boot_cpus(void) } else { unsigned long bogosum = 0; - for(i = 0; i < NR_CPUS; i++) { - if (cpu_isset(i, cpu_present_map)) { - bogosum += cpu_data(i).udelay_val; - smp_highest_cpu = i; - } + for_each_present_cpu(i) { + bogosum += cpu_data(i).udelay_val; + smp_highest_cpu = i; } SMP_PRINTK(("Total of %d Processors activated (%lu.%02lu BogoMIPS).\n", cpucount + 1, bogosum/(500000/HZ), (bogosum/(5000/HZ))%100)); printk("Total of %d Processors activated (%lu.%02lu BogoMIPS).\n", diff --git a/arch/sparc/kernel/sun4m_smp.c b/arch/sparc/kernel/sun4m_smp.c index fbbd8a474c4c..1dde312eebda 100644 --- a/arch/sparc/kernel/sun4m_smp.c +++ b/arch/sparc/kernel/sun4m_smp.c @@ -218,10 +218,8 @@ void __init smp4m_boot_cpus(void) cpu_present_map = cpumask_of_cpu(smp_processor_id()); } else { unsigned long bogosum = 0; - for(i = 0; i < NR_CPUS; i++) { - if (cpu_isset(i, cpu_present_map)) - bogosum += cpu_data(i).udelay_val; - } + for_each_present_cpu(i) + bogosum += cpu_data(i).udelay_val; printk("Total of %d Processors activated (%lu.%02lu BogoMIPS).\n", cpucount + 1, bogosum/(500000/HZ), diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c index 8c93ba655b33..e505a4125e35 100644 --- a/arch/sparc64/kernel/irq.c +++ b/arch/sparc64/kernel/irq.c @@ -117,9 +117,7 @@ int show_interrupts(struct seq_file *p, void *v) #ifndef CONFIG_SMP seq_printf(p, "%10u ", kstat_irqs(i)); #else - for (j = 0; j < NR_CPUS; j++) { - if (!cpu_online(j)) - continue; + for_each_online_cpu(j) { seq_printf(p, "%10u ", kstat_cpu(j).irqs[i]); } diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c index 373a701c90a5..1b6e2ade1008 100644 --- a/arch/sparc64/kernel/smp.c +++ b/arch/sparc64/kernel/smp.c @@ -57,25 +57,21 @@ void smp_info(struct seq_file *m) int i; seq_printf(m, "State:\n"); - for (i = 0; i < NR_CPUS; i++) { - if (cpu_online(i)) - seq_printf(m, - "CPU%d:\t\tonline\n", i); - } + for_each_online_cpu(i) + seq_printf(m, "CPU%d:\t\tonline\n", i); } void smp_bogo(struct seq_file *m) { int i; - for (i = 0; i < NR_CPUS; i++) - if (cpu_online(i)) - seq_printf(m, - "Cpu%dBogo\t: %lu.%02lu\n" - "Cpu%dClkTck\t: %016lx\n", - i, cpu_data(i).udelay_val / (500000/HZ), - (cpu_data(i).udelay_val / (5000/HZ)) % 100, - i, cpu_data(i).clock_tick); + for_each_online_cpu(i) + seq_printf(m, + "Cpu%dBogo\t: %lu.%02lu\n" + "Cpu%dClkTck\t: %016lx\n", + i, cpu_data(i).udelay_val / (500000/HZ), + (cpu_data(i).udelay_val / (5000/HZ)) % 100, + i, cpu_data(i).clock_tick); } void __init smp_store_cpu_info(int id) @@ -1282,7 +1278,7 @@ int setup_profiling_timer(unsigned int multiplier) return -EINVAL; spin_lock_irqsave(&prof_setup_lock, flags); - for (i = 0; i < NR_CPUS; i++) + for_each_cpu(i) prof_multiplier(i) = multiplier; current_tick_offset = (timer_tick_offset / multiplier); spin_unlock_irqrestore(&prof_setup_lock, flags); @@ -1384,10 +1380,8 @@ void __init smp_cpus_done(unsigned int max_cpus) unsigned long bogosum = 0; int i; - for (i = 0; i < NR_CPUS; i++) { - if (cpu_online(i)) - bogosum += cpu_data(i).udelay_val; - } + for_each_online_cpu(i) + bogosum += cpu_data(i).udelay_val; printk("Total of %ld processors activated " "(%lu.%02lu BogoMIPS).\n", (long) num_online_cpus(), diff --git a/arch/x86_64/kernel/irq.c b/arch/x86_64/kernel/irq.c index 30d2a1e545fe..d8bd0b345b1e 100644 --- a/arch/x86_64/kernel/irq.c +++ b/arch/x86_64/kernel/irq.c @@ -38,9 +38,8 @@ int show_interrupts(struct seq_file *p, void *v) if (i == 0) { seq_printf(p, " "); - for (j=0; jtypename); @@ -68,15 +65,13 @@ skip: spin_unlock_irqrestore(&irq_desc[i].lock, flags); } else if (i == NR_IRQS) { seq_printf(p, "NMI: "); - for (j = 0; j < NR_CPUS; j++) - if (cpu_online(j)) - seq_printf(p, "%10u ", cpu_pda(j)->__nmi_count); + for_each_online_cpu(j) + seq_printf(p, "%10u ", cpu_pda(j)->__nmi_count); seq_putc(p, '\n'); #ifdef CONFIG_X86_LOCAL_APIC seq_printf(p, "LOC: "); - for (j = 0; j < NR_CPUS; j++) - if (cpu_online(j)) - seq_printf(p, "%10u ", cpu_pda(j)->apic_timer_irqs); + for_each_online_cpu(j) + seq_printf(p, "%10u ", cpu_pda(j)->apic_timer_irqs); seq_putc(p, '\n'); #endif seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count)); diff --git a/arch/x86_64/kernel/nmi.c b/arch/x86_64/kernel/nmi.c index 5bf17e41cd2d..66c009e10bac 100644 --- a/arch/x86_64/kernel/nmi.c +++ b/arch/x86_64/kernel/nmi.c @@ -162,9 +162,7 @@ int __init check_nmi_watchdog (void) local_irq_enable(); mdelay((10*1000)/nmi_hz); // wait 10 ticks - for (cpu = 0; cpu < NR_CPUS; cpu++) { - if (!cpu_online(cpu)) - continue; + for_each_online_cpu(cpu) { if (cpu_pda(cpu)->__nmi_count - counts[cpu] <= 5) { endflag = 1; printk("CPU#%d: NMI appears to be stuck (%d->%d)!\n", diff --git a/arch/xtensa/kernel/irq.c b/arch/xtensa/kernel/irq.c index 4cbf6d91571f..51f9bed455fa 100644 --- a/arch/xtensa/kernel/irq.c +++ b/arch/xtensa/kernel/irq.c @@ -83,9 +83,8 @@ int show_interrupts(struct seq_file *p, void *v) if (i == 0) { seq_printf(p, " "); - for (j=0; jtypename); seq_printf(p, " %s", action->name); @@ -113,9 +111,8 @@ skip: spin_unlock_irqrestore(&irq_desc[i].lock, flags); } else if (i == NR_IRQS) { seq_printf(p, "NMI: "); - for (j = 0; j < NR_CPUS; j++) - if (cpu_online(j)) - seq_printf(p, "%10u ", nmi_count(j)); + for_each_online_cpu(j) + seq_printf(p, "%10u ", nmi_count(j)); seq_putc(p, '\n'); seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count)); } diff --git a/drivers/net/loopback.c b/drivers/net/loopback.c index 690a1aae0b34..0c13795dca38 100644 --- a/drivers/net/loopback.c +++ b/drivers/net/loopback.c @@ -172,11 +172,9 @@ static struct net_device_stats *get_stats(struct net_device *dev) memset(stats, 0, sizeof(struct net_device_stats)); - for (i=0; i < NR_CPUS; i++) { + for_each_cpu(i) { struct net_device_stats *lb_stats; - if (!cpu_possible(i)) - continue; lb_stats = &per_cpu(loopback_stats, i); stats->rx_bytes += lb_stats->rx_bytes; stats->tx_bytes += lb_stats->tx_bytes; diff --git a/drivers/oprofile/cpu_buffer.c b/drivers/oprofile/cpu_buffer.c index 78193e4bbdb5..330d3869b41e 100644 --- a/drivers/oprofile/cpu_buffer.c +++ b/drivers/oprofile/cpu_buffer.c @@ -38,9 +38,8 @@ void free_cpu_buffers(void) { int i; - for_each_online_cpu(i) { + for_each_online_cpu(i) vfree(cpu_buffer[i].buffer); - } } int alloc_cpu_buffers(void) diff --git a/fs/xfs/linux-2.6/xfs_stats.c b/fs/xfs/linux-2.6/xfs_stats.c index 8955720a2c6b..713e6a7505d0 100644 --- a/fs/xfs/linux-2.6/xfs_stats.c +++ b/fs/xfs/linux-2.6/xfs_stats.c @@ -62,18 +62,15 @@ xfs_read_xfsstats( while (j < xstats[i].endpoint) { val = 0; /* sum over all cpus */ - for (c = 0; c < NR_CPUS; c++) { - if (!cpu_possible(c)) continue; + for_each_cpu(c) val += *(((__u32*)&per_cpu(xfsstats, c) + j)); - } len += sprintf(buffer + len, " %u", val); j++; } buffer[len++] = '\n'; } /* extra precision counters */ - for (i = 0; i < NR_CPUS; i++) { - if (!cpu_possible(i)) continue; + for_each_cpu(i) { xs_xstrat_bytes += per_cpu(xfsstats, i).xs_xstrat_bytes; xs_write_bytes += per_cpu(xfsstats, i).xs_write_bytes; xs_read_bytes += per_cpu(xfsstats, i).xs_read_bytes; diff --git a/fs/xfs/linux-2.6/xfs_sysctl.c b/fs/xfs/linux-2.6/xfs_sysctl.c index a02564972420..7079cc837210 100644 --- a/fs/xfs/linux-2.6/xfs_sysctl.c +++ b/fs/xfs/linux-2.6/xfs_sysctl.c @@ -38,8 +38,7 @@ xfs_stats_clear_proc_handler( if (!ret && write && *valp) { printk("XFS Clearing xfsstats\n"); - for (c = 0; c < NR_CPUS; c++) { - if (!cpu_possible(c)) continue; + for_each_cpu(c) { preempt_disable(); /* save vn_active, it's a universal truth! */ vn_active = per_cpu(xfsstats, c).vn_active; diff --git a/include/asm-alpha/mmu_context.h b/include/asm-alpha/mmu_context.h index 6f92482cc96c..0c017fc181c1 100644 --- a/include/asm-alpha/mmu_context.h +++ b/include/asm-alpha/mmu_context.h @@ -231,9 +231,8 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) { int i; - for (i = 0; i < NR_CPUS; i++) - if (cpu_online(i)) - mm->context[i] = 0; + for_each_online_cpu(i) + mm->context[i] = 0; if (tsk != current) task_thread_info(tsk)->pcb.ptbr = ((unsigned long)mm->pgd - IDENT_ADDR) >> PAGE_SHIFT; diff --git a/include/asm-alpha/topology.h b/include/asm-alpha/topology.h index eb740e280d9c..420ccde6b916 100644 --- a/include/asm-alpha/topology.h +++ b/include/asm-alpha/topology.h @@ -27,8 +27,8 @@ static inline cpumask_t node_to_cpumask(int node) cpumask_t node_cpu_mask = CPU_MASK_NONE; int cpu; - for(cpu = 0; cpu < NR_CPUS; cpu++) { - if (cpu_online(cpu) && (cpu_to_node(cpu) == node)) + for_each_online_cpu(cpu) { + if (cpu_to_node(cpu) == node) cpu_set(cpu, node_cpu_mask); } diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h index 9044aeb37828..78cf45547e31 100644 --- a/include/asm-generic/percpu.h +++ b/include/asm-generic/percpu.h @@ -19,10 +19,9 @@ extern unsigned long __per_cpu_offset[NR_CPUS]; #define percpu_modcopy(pcpudst, src, size) \ do { \ unsigned int __i; \ - for (__i = 0; __i < NR_CPUS; __i++) \ - if (cpu_possible(__i)) \ - memcpy((pcpudst)+__per_cpu_offset[__i], \ - (src), (size)); \ + for_each_cpu(__i) \ + memcpy((pcpudst)+__per_cpu_offset[__i], \ + (src), (size)); \ } while (0) #else /* ! SMP */ diff --git a/include/asm-powerpc/percpu.h b/include/asm-powerpc/percpu.h index e31922c50e53..464301cd0d03 100644 --- a/include/asm-powerpc/percpu.h +++ b/include/asm-powerpc/percpu.h @@ -27,10 +27,9 @@ #define percpu_modcopy(pcpudst, src, size) \ do { \ unsigned int __i; \ - for (__i = 0; __i < NR_CPUS; __i++) \ - if (cpu_possible(__i)) \ - memcpy((pcpudst)+__per_cpu_offset(__i), \ - (src), (size)); \ + for_each_cpu(__i) \ + memcpy((pcpudst)+__per_cpu_offset(__i), \ + (src), (size)); \ } while (0) extern void setup_per_cpu_areas(void); diff --git a/include/asm-s390/percpu.h b/include/asm-s390/percpu.h index 123fcaca295e..e10ed87094f0 100644 --- a/include/asm-s390/percpu.h +++ b/include/asm-s390/percpu.h @@ -46,10 +46,9 @@ extern unsigned long __per_cpu_offset[NR_CPUS]; #define percpu_modcopy(pcpudst, src, size) \ do { \ unsigned int __i; \ - for (__i = 0; __i < NR_CPUS; __i++) \ - if (cpu_possible(__i)) \ - memcpy((pcpudst)+__per_cpu_offset[__i], \ - (src), (size)); \ + for_each_cpu(__i) \ + memcpy((pcpudst)+__per_cpu_offset[__i], \ + (src), (size)); \ } while (0) #else /* ! SMP */ diff --git a/include/asm-sparc64/percpu.h b/include/asm-sparc64/percpu.h index aea4e51e7cd1..82032e159a76 100644 --- a/include/asm-sparc64/percpu.h +++ b/include/asm-sparc64/percpu.h @@ -26,10 +26,9 @@ register unsigned long __local_per_cpu_offset asm("g5"); #define percpu_modcopy(pcpudst, src, size) \ do { \ unsigned int __i; \ - for (__i = 0; __i < NR_CPUS; __i++) \ - if (cpu_possible(__i)) \ - memcpy((pcpudst)+__per_cpu_offset(__i), \ - (src), (size)); \ + for_each_cpu(__i) \ + memcpy((pcpudst)+__per_cpu_offset(__i), \ + (src), (size)); \ } while (0) #else /* ! SMP */ diff --git a/include/asm-x86_64/percpu.h b/include/asm-x86_64/percpu.h index 29a6b0408f75..4405b4adeaba 100644 --- a/include/asm-x86_64/percpu.h +++ b/include/asm-x86_64/percpu.h @@ -26,10 +26,9 @@ #define percpu_modcopy(pcpudst, src, size) \ do { \ unsigned int __i; \ - for (__i = 0; __i < NR_CPUS; __i++) \ - if (cpu_possible(__i)) \ - memcpy((pcpudst)+__per_cpu_offset(__i), \ - (src), (size)); \ + for_each_cpu(__i) \ + memcpy((pcpudst)+__per_cpu_offset(__i), \ + (src), (size)); \ } while (0) extern void setup_per_cpu_areas(void); diff --git a/include/linux/genhd.h b/include/linux/genhd.h index eef5ccdcd731..fd647fde5ec1 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -149,22 +149,16 @@ struct disk_attribute { ({ \ typeof(gendiskp->dkstats->field) res = 0; \ int i; \ - for (i=0; i < NR_CPUS; i++) { \ - if (!cpu_possible(i)) \ - continue; \ + for_each_cpu(i) \ res += per_cpu_ptr(gendiskp->dkstats, i)->field; \ - } \ res; \ }) static inline void disk_stat_set_all(struct gendisk *gendiskp, int value) { int i; - for (i=0; i < NR_CPUS; i++) { - if (cpu_possible(i)) { - memset(per_cpu_ptr(gendiskp->dkstats, i), value, - sizeof (struct disk_stats)); - } - } + for_each_cpu(i) + memset(per_cpu_ptr(gendiskp->dkstats, i), value, + sizeof (struct disk_stats)); } #else -- cgit v1.2.3