From e6f72d3abafd50984decc2833c706e717f5ba04e Mon Sep 17 00:00:00 2001 From: David Daney Date: Wed, 20 May 2009 11:40:58 -0700 Subject: MIPS: Replace some magic numbers with symbolic values in tlbex.c The logic used to split the r4000 refill handler is liberally sprinkled with magic numbers. We attempt to explain what they are and normalize them against a new symbolic value (MIPS64_REFILL_INSNS). CC: David VomLehn Reviewed-by: Paul Gortmaker Signed-off-by: David Daney Signed-off-by: Ralf Baechle --- arch/mips/mm/tlbex.c | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) (limited to 'arch/mips/mm/tlbex.c') diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 0615b62efd6d..678e63398461 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -649,6 +649,14 @@ static void __cpuinit build_update_entries(u32 **p, unsigned int tmp, #endif } +/* + * For a 64-bit kernel, we are using the 64-bit XTLB refill exception + * because EXL == 0. If we wrap, we can also use the 32 instruction + * slots before the XTLB refill exception handler which belong to the + * unused TLB refill exception. + */ +#define MIPS64_REFILL_INSNS 32 + static void __cpuinit build_r4000_tlb_refill_handler(void) { u32 *p = tlb_handler; @@ -702,9 +710,10 @@ static void __cpuinit build_r4000_tlb_refill_handler(void) if ((p - tlb_handler) > 64) panic("TLB refill handler space exceeded"); #else - if (((p - tlb_handler) > 63) - || (((p - tlb_handler) > 61) - && uasm_insn_has_bdelay(relocs, tlb_handler + 29))) + if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) + || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) + && uasm_insn_has_bdelay(relocs, + tlb_handler + MIPS64_REFILL_INSNS - 3))) panic("TLB refill handler space exceeded"); #endif @@ -717,16 +726,24 @@ static void __cpuinit build_r4000_tlb_refill_handler(void) uasm_copy_handler(relocs, labels, tlb_handler, p, f); final_len = p - tlb_handler; #else /* CONFIG_64BIT */ - f = final_handler + 32; - if ((p - tlb_handler) <= 32) { + f = final_handler + MIPS64_REFILL_INSNS; + if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { /* Just copy the handler. */ uasm_copy_handler(relocs, labels, tlb_handler, p, f); final_len = p - tlb_handler; } else { - u32 *split = tlb_handler + 30; + /* + * Split two instructions before the end. One for the + * branch and one for the instruction in the delay + * slot. + */ + u32 *split = tlb_handler + MIPS64_REFILL_INSNS - 2; /* - * Find the split point. + * Find the split point. If the branch would fall in + * a delay slot, we must back up an additional + * instruction so that it is no longer in a delay + * slot. */ if (uasm_insn_has_bdelay(relocs, split - 1)) split--; @@ -749,7 +766,8 @@ static void __cpuinit build_r4000_tlb_refill_handler(void) /* Copy the rest of the handler. */ uasm_copy_handler(relocs, labels, split, p, final_handler); - final_len = (f - (final_handler + 32)) + (p - split); + final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) + + (p - split); } #endif /* CONFIG_64BIT */ -- cgit v1.2.3 From 95affdda9bfba0ac17025d48c622e1f30964e316 Mon Sep 17 00:00:00 2001 From: David Daney Date: Wed, 20 May 2009 11:40:59 -0700 Subject: MIPS: Fold the TLB refill at the vmalloc path if possible. Try to fold the 64-bit TLB refill handler opportunistically at the beginning of the vmalloc path so as to avoid splitting execution flow in half and wasting cycles for a branch required at that point then. Resort to doing the split if either of the newly created parts would not fit into its designated slot. Original-patch-by: Maciej W. Rozycki Signed-off-by: Maciej W. Rozycki Signed-off-by: David Daney Signed-off-by: Ralf Baechle --- arch/mips/mm/tlbex.c | 73 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 24 deletions(-) (limited to 'arch/mips/mm/tlbex.c') diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 678e63398461..d9a18b2b7f82 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -6,7 +6,7 @@ * Synthesize TLB refill handlers at runtime. * * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer - * Copyright (C) 2005, 2007 Maciej W. Rozycki + * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) * * ... and the days got worse and worse and now you see @@ -19,6 +19,7 @@ * (Condolences to Napoleon XIV) */ +#include #include #include #include @@ -732,36 +733,60 @@ static void __cpuinit build_r4000_tlb_refill_handler(void) uasm_copy_handler(relocs, labels, tlb_handler, p, f); final_len = p - tlb_handler; } else { - /* - * Split two instructions before the end. One for the - * branch and one for the instruction in the delay - * slot. - */ - u32 *split = tlb_handler + MIPS64_REFILL_INSNS - 2; +#ifdef MODULE_START + const enum label_id ls = label_module_alloc; +#else + const enum label_id ls = label_vmalloc; +#endif + u32 *split; + int ov = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++) + ; + BUG_ON(i == ARRAY_SIZE(labels)); + split = labels[i].addr; /* - * Find the split point. If the branch would fall in - * a delay slot, we must back up an additional - * instruction so that it is no longer in a delay - * slot. + * See if we have overflown one way or the other. */ - if (uasm_insn_has_bdelay(relocs, split - 1)) - split--; - + if (split > tlb_handler + MIPS64_REFILL_INSNS || + split < p - MIPS64_REFILL_INSNS) + ov = 1; + + if (ov) { + /* + * Split two instructions before the end. One + * for the branch and one for the instruction + * in the delay slot. + */ + split = tlb_handler + MIPS64_REFILL_INSNS - 2; + + /* + * If the branch would fall in a delay slot, + * we must back up an additional instruction + * so that it is no longer in a delay slot. + */ + if (uasm_insn_has_bdelay(relocs, split - 1)) + split--; + } /* Copy first part of the handler. */ uasm_copy_handler(relocs, labels, tlb_handler, split, f); f += split - tlb_handler; - /* Insert branch. */ - uasm_l_split(&l, final_handler); - uasm_il_b(&f, &r, label_split); - if (uasm_insn_has_bdelay(relocs, split)) - uasm_i_nop(&f); - else { - uasm_copy_handler(relocs, labels, split, split + 1, f); - uasm_move_labels(labels, f, f + 1, -1); - f++; - split++; + if (ov) { + /* Insert branch. */ + uasm_l_split(&l, final_handler); + uasm_il_b(&f, &r, label_split); + if (uasm_insn_has_bdelay(relocs, split)) + uasm_i_nop(&f); + else { + uasm_copy_handler(relocs, labels, + split, split + 1, f); + uasm_move_labels(labels, f, f + 1, -1); + f++; + split++; + } } /* Copy the rest of the handler. */ -- cgit v1.2.3 From 41f0e4d041aa30507a34998c29d0b7ac0bede277 Mon Sep 17 00:00:00 2001 From: David Daney Date: Tue, 12 May 2009 12:41:53 -0700 Subject: MIPS: Allow R2 CPUs to turn off generation of 'ehb' instructions. Some CPUs do not need ehb instructions after writing CP0 registers. By allowing ehb generation to be overridden in cpu-feature-overrides.h, we can save a few instructions in the TLB handler hot paths. Signed-off-by: David Daney Signed-off-by: Ralf Baechle --- arch/mips/include/asm/cpu-features.h | 4 ++++ arch/mips/mm/tlbex.c | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'arch/mips/mm/tlbex.c') diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h index c0047f861337..1cba4b2ffd1e 100644 --- a/arch/mips/include/asm/cpu-features.h +++ b/arch/mips/include/asm/cpu-features.h @@ -147,6 +147,10 @@ #define cpu_has_mips_r (cpu_has_mips32r1 | cpu_has_mips32r2 | \ cpu_has_mips64r1 | cpu_has_mips64r2) +#ifndef cpu_has_mips_r2_exec_hazard +#define cpu_has_mips_r2_exec_hazard cpu_has_mips_r2 +#endif + /* * MIPS32, MIPS64, VR5500, IDT32332, IDT32334 and maybe a few other * pre-MIPS32/MIPS53 processors have CLO, CLZ. For 64-bit kernels diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index d9a18b2b7f82..0e34faaadb5c 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -259,7 +259,8 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, } if (cpu_has_mips_r2) { - uasm_i_ehb(p); + if (cpu_has_mips_r2_exec_hazard) + uasm_i_ehb(p); tlbw(p); return; } -- cgit v1.2.3 From faed5288af0f05aa61ac1e8d47306d855a2868f0 Mon Sep 17 00:00:00 2001 From: David Daney Date: Tue, 12 May 2009 12:41:55 -0700 Subject: MIPS: Remove dead case label. CPU_CAVIUM_OCTEON is mips_r2 which is handled before the switch. This label in the switch statement is dead code, so we remove it. Signed-off-by: David Daney Reviewed by: David VomLehn Signed-off-by: Ralf Baechle --- arch/mips/mm/tlbex.c | 1 - 1 file changed, 1 deletion(-) (limited to 'arch/mips/mm/tlbex.c') diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 0e34faaadb5c..2104aa0fa3eb 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -312,7 +312,6 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, case CPU_BCM3302: case CPU_BCM4710: case CPU_LOONGSON2: - case CPU_CAVIUM_OCTEON: case CPU_R5500: if (m4kc_tlbp_war()) uasm_i_nop(p); -- cgit v1.2.3 From bd1437e49d80fa3369ffbea9e73cde7f6d69e550 Mon Sep 17 00:00:00 2001 From: David Daney Date: Fri, 8 May 2009 15:10:50 -0700 Subject: MIPS: Remove unused parameters from iPTE_LW. The l parameter to iPTE_LW() is unused. Remove it and from some of its callers as well. Signed-off-by: David Daney Signed-off-by: Ralf Baechle --- arch/mips/mm/tlbex.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'arch/mips/mm/tlbex.c') diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 2104aa0fa3eb..62fbd0d89aeb 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -825,7 +825,7 @@ u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned; u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned; static void __cpuinit -iPTE_LW(u32 **p, struct uasm_label **l, unsigned int pte, unsigned int ptr) +iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) { #ifdef CONFIG_SMP # ifdef CONFIG_64BIT_PHYS_ADDR @@ -905,13 +905,13 @@ iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, * with it's original value. */ static void __cpuinit -build_pte_present(u32 **p, struct uasm_label **l, struct uasm_reloc **r, +build_pte_present(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, enum label_id lid) { uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); uasm_il_bnez(p, r, pte, lid); - iPTE_LW(p, l, pte, ptr); + iPTE_LW(p, pte, ptr); } /* Make PTE valid, store result in PTR. */ @@ -929,13 +929,13 @@ build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, * restore PTE with value from PTR when done. */ static void __cpuinit -build_pte_writable(u32 **p, struct uasm_label **l, struct uasm_reloc **r, +build_pte_writable(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, enum label_id lid) { uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); uasm_il_bnez(p, r, pte, lid); - iPTE_LW(p, l, pte, ptr); + iPTE_LW(p, pte, ptr); } /* Make PTE writable, update software status bits as well, then store @@ -956,12 +956,12 @@ build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, * restore PTE with value from PTR when done. */ static void __cpuinit -build_pte_modifiable(u32 **p, struct uasm_label **l, struct uasm_reloc **r, +build_pte_modifiable(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, enum label_id lid) { uasm_i_andi(p, pte, pte, _PAGE_WRITE); uasm_il_beqz(p, r, pte, lid); - iPTE_LW(p, l, pte, ptr); + iPTE_LW(p, pte, ptr); } /* @@ -1037,7 +1037,7 @@ static void __cpuinit build_r3000_tlb_load_handler(void) memset(relocs, 0, sizeof(relocs)); build_r3000_tlbchange_handler_head(&p, K0, K1); - build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl); + build_pte_present(&p, &r, K0, K1, label_nopage_tlbl); uasm_i_nop(&p); /* load delay */ build_make_valid(&p, &r, K0, K1); build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); @@ -1067,7 +1067,7 @@ static void __cpuinit build_r3000_tlb_store_handler(void) memset(relocs, 0, sizeof(relocs)); build_r3000_tlbchange_handler_head(&p, K0, K1); - build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs); + build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs); uasm_i_nop(&p); /* load delay */ build_make_write(&p, &r, K0, K1); build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); @@ -1097,7 +1097,7 @@ static void __cpuinit build_r3000_tlb_modify_handler(void) memset(relocs, 0, sizeof(relocs)); build_r3000_tlbchange_handler_head(&p, K0, K1); - build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm); + build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm); uasm_i_nop(&p); /* load delay */ build_make_write(&p, &r, K0, K1); build_r3000_pte_reload_tlbwi(&p, K0, K1); @@ -1139,7 +1139,7 @@ build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, #ifdef CONFIG_SMP uasm_l_smp_pgtable_change(l, *p); #endif - iPTE_LW(p, l, pte, ptr); /* get even pte */ + iPTE_LW(p, pte, ptr); /* get even pte */ if (!m4kc_tlbp_war()) build_tlb_probe_entry(p); } @@ -1181,7 +1181,7 @@ static void __cpuinit build_r4000_tlb_load_handler(void) } build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); - build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl); + build_pte_present(&p, &r, K0, K1, label_nopage_tlbl); if (m4kc_tlbp_war()) build_tlb_probe_entry(&p); build_make_valid(&p, &r, K0, K1); @@ -1212,7 +1212,7 @@ static void __cpuinit build_r4000_tlb_store_handler(void) memset(relocs, 0, sizeof(relocs)); build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); - build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs); + build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs); if (m4kc_tlbp_war()) build_tlb_probe_entry(&p); build_make_write(&p, &r, K0, K1); @@ -1243,7 +1243,7 @@ static void __cpuinit build_r4000_tlb_modify_handler(void) memset(relocs, 0, sizeof(relocs)); build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); - build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm); + build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm); if (m4kc_tlbp_war()) build_tlb_probe_entry(&p); /* Present and writable bits set, set accessed and dirty bits. */ -- cgit v1.2.3 From fd062c847a8cea2821347d7e18165dfa658f7dce Mon Sep 17 00:00:00 2001 From: David Daney Date: Wed, 27 May 2009 17:47:44 -0700 Subject: MIPS: TLB support for hugetlbfs. The TLB handlers need to check for huge pages and give them special handling. Huge pages consist of two contiguous sub-pages of physical memory. * Loading entrylo0 and entrylo1 need to be handled specially. * The page mask must be set for huge pages and then restored after writing the TLB entries. * The PTE for huge pages resides in the PMD, we halt traversal of the tables there. Signed-off-by: David Daney Signed-off-by: Ralf Baechle --- arch/mips/mm/tlb-r4k.c | 43 +++++++++---- arch/mips/mm/tlbex.c | 165 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 196 insertions(+), 12 deletions(-) (limited to 'arch/mips/mm/tlbex.c') diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c index 892be426787c..f60fe513eb60 100644 --- a/arch/mips/mm/tlb-r4k.c +++ b/arch/mips/mm/tlb-r4k.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -295,21 +296,41 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) pudp = pud_offset(pgdp, address); pmdp = pmd_offset(pudp, address); idx = read_c0_index(); - ptep = pte_offset_map(pmdp, address); +#ifdef CONFIG_HUGETLB_PAGE + /* this could be a huge page */ + if (pmd_huge(*pmdp)) { + unsigned long lo; + write_c0_pagemask(PM_HUGE_MASK); + ptep = (pte_t *)pmdp; + lo = pte_val(*ptep) >> 6; + write_c0_entrylo0(lo); + write_c0_entrylo1(lo + (HPAGE_SIZE >> 7)); + + mtc0_tlbw_hazard(); + if (idx < 0) + tlb_write_random(); + else + tlb_write_indexed(); + write_c0_pagemask(PM_DEFAULT_MASK); + } else +#endif + { + ptep = pte_offset_map(pmdp, address); #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32) - write_c0_entrylo0(ptep->pte_high); - ptep++; - write_c0_entrylo1(ptep->pte_high); + write_c0_entrylo0(ptep->pte_high); + ptep++; + write_c0_entrylo1(ptep->pte_high); #else - write_c0_entrylo0(pte_val(*ptep++) >> 6); - write_c0_entrylo1(pte_val(*ptep) >> 6); + write_c0_entrylo0(pte_val(*ptep++) >> 6); + write_c0_entrylo1(pte_val(*ptep) >> 6); #endif - mtc0_tlbw_hazard(); - if (idx < 0) - tlb_write_random(); - else - tlb_write_indexed(); + mtc0_tlbw_hazard(); + if (idx < 0) + tlb_write_random(); + else + tlb_write_indexed(); + } tlbw_use_hazard(); FLUSH_ITLB_VM(vma); EXIT_CRITICAL(flags); diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 62fbd0d89aeb..8f606ead826e 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -8,6 +8,7 @@ * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) + * Copyright (C) 2008, 2009 Cavium Networks, Inc. * * ... and the days got worse and worse and now you see * I've gone completly out of my mind. @@ -83,6 +84,9 @@ enum label_id { label_nopage_tlbm, label_smp_pgtable_change, label_r3000_write_probe_fail, +#ifdef CONFIG_HUGETLB_PAGE + label_tlb_huge_update, +#endif }; UASM_L_LA(_second_part) @@ -99,6 +103,9 @@ UASM_L_LA(_nopage_tlbs) UASM_L_LA(_nopage_tlbm) UASM_L_LA(_smp_pgtable_change) UASM_L_LA(_r3000_write_probe_fail) +#ifdef CONFIG_HUGETLB_PAGE +UASM_L_LA(_tlb_huge_update) +#endif /* * For debug purposes. @@ -126,6 +133,7 @@ static inline void dump_handler(const u32 *handler, int count) #define C0_TCBIND 2, 2 #define C0_ENTRYLO1 3, 0 #define C0_CONTEXT 4, 0 +#define C0_PAGEMASK 5, 0 #define C0_BADVADDR 8, 0 #define C0_ENTRYHI 10, 0 #define C0_EPC 14, 0 @@ -383,6 +391,98 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, } } +#ifdef CONFIG_HUGETLB_PAGE +static __cpuinit void build_huge_tlb_write_entry(u32 **p, + struct uasm_label **l, + struct uasm_reloc **r, + unsigned int tmp, + enum tlb_write_entry wmode) +{ + /* Set huge page tlb entry size */ + uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); + uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); + uasm_i_mtc0(p, tmp, C0_PAGEMASK); + + build_tlb_write_entry(p, l, r, wmode); + + /* Reset default page size */ + if (PM_DEFAULT_MASK >> 16) { + uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); + uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); + uasm_il_b(p, r, label_leave); + uasm_i_mtc0(p, tmp, C0_PAGEMASK); + } else if (PM_DEFAULT_MASK) { + uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); + uasm_il_b(p, r, label_leave); + uasm_i_mtc0(p, tmp, C0_PAGEMASK); + } else { + uasm_il_b(p, r, label_leave); + uasm_i_mtc0(p, 0, C0_PAGEMASK); + } +} + +/* + * Check if Huge PTE is present, if so then jump to LABEL. + */ +static void __cpuinit +build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, + unsigned int pmd, int lid) +{ + UASM_i_LW(p, tmp, 0, pmd); + uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); + uasm_il_bnez(p, r, tmp, lid); +} + +static __cpuinit void build_huge_update_entries(u32 **p, + unsigned int pte, + unsigned int tmp) +{ + int small_sequence; + + /* + * A huge PTE describes an area the size of the + * configured huge page size. This is twice the + * of the large TLB entry size we intend to use. + * A TLB entry half the size of the configured + * huge page size is configured into entrylo0 + * and entrylo1 to cover the contiguous huge PTE + * address space. + */ + small_sequence = (HPAGE_SIZE >> 7) < 0x10000; + + /* We can clobber tmp. It isn't used after this.*/ + if (!small_sequence) + uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); + + UASM_i_SRL(p, pte, pte, 6); /* convert to entrylo */ + uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* load it */ + /* convert to entrylo1 */ + if (small_sequence) + UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); + else + UASM_i_ADDU(p, pte, pte, tmp); + + uasm_i_mtc0(p, pte, C0_ENTRYLO1); /* load it */ +} + +static __cpuinit void build_huge_handler_tail(u32 **p, + struct uasm_reloc **r, + struct uasm_label **l, + unsigned int pte, + unsigned int ptr) +{ +#ifdef CONFIG_SMP + UASM_i_SC(p, pte, 0, ptr); + uasm_il_beqz(p, r, pte, label_tlb_huge_update); + UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ +#else + UASM_i_SW(p, pte, 0, ptr); +#endif + build_huge_update_entries(p, pte, ptr); + build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed); +} +#endif /* CONFIG_HUGETLB_PAGE */ + #ifdef CONFIG_64BIT /* * TMP and PTR are scratch. @@ -689,12 +789,23 @@ static void __cpuinit build_r4000_tlb_refill_handler(void) build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ #endif +#ifdef CONFIG_HUGETLB_PAGE + build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); +#endif + build_get_ptep(&p, K0, K1); build_update_entries(&p, K0, K1); build_tlb_write_entry(&p, &l, &r, tlb_random); uasm_l_leave(&l, p); uasm_i_eret(&p); /* return from trap */ +#ifdef CONFIG_HUGETLB_PAGE + uasm_l_tlb_huge_update(&l, p); + UASM_i_LW(&p, K0, 0, K1); + build_huge_update_entries(&p, K0, K1); + build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random); +#endif + #ifdef CONFIG_64BIT build_get_pgd_vmalloc64(&p, &l, &r, K0, K1); #endif @@ -733,7 +844,9 @@ static void __cpuinit build_r4000_tlb_refill_handler(void) uasm_copy_handler(relocs, labels, tlb_handler, p, f); final_len = p - tlb_handler; } else { -#ifdef MODULE_START +#if defined(CONFIG_HUGETLB_PAGE) + const enum label_id ls = label_tlb_huge_update; +#elif defined(MODULE_START) const enum label_id ls = label_module_alloc; #else const enum label_id ls = label_vmalloc; @@ -1130,6 +1243,15 @@ build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, build_get_pgde32(p, pte, ptr); /* get pgd in ptr */ #endif +#ifdef CONFIG_HUGETLB_PAGE + /* + * For huge tlb entries, pmd doesn't contain an address but + * instead contains the tlb pte. Check the PAGE_HUGE bit and + * see if we need to jump to huge tlb processing. + */ + build_is_huge_pte(p, r, pte, ptr, label_tlb_huge_update); +#endif + UASM_i_MFC0(p, pte, C0_BADVADDR); UASM_i_LW(p, ptr, 0, ptr); UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); @@ -1187,6 +1309,19 @@ static void __cpuinit build_r4000_tlb_load_handler(void) build_make_valid(&p, &r, K0, K1); build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); +#ifdef CONFIG_HUGETLB_PAGE + /* + * This is the entry point when build_r4000_tlbchange_handler_head + * spots a huge page. + */ + uasm_l_tlb_huge_update(&l, p); + iPTE_LW(&p, K0, K1); + build_pte_present(&p, &r, K0, K1, label_nopage_tlbl); + build_tlb_probe_entry(&p); + uasm_i_ori(&p, K0, K0, (_PAGE_ACCESSED | _PAGE_VALID)); + build_huge_handler_tail(&p, &r, &l, K0, K1); +#endif + uasm_l_nopage_tlbl(&l, p); uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); uasm_i_nop(&p); @@ -1218,6 +1353,20 @@ static void __cpuinit build_r4000_tlb_store_handler(void) build_make_write(&p, &r, K0, K1); build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); +#ifdef CONFIG_HUGETLB_PAGE + /* + * This is the entry point when + * build_r4000_tlbchange_handler_head spots a huge page. + */ + uasm_l_tlb_huge_update(&l, p); + iPTE_LW(&p, K0, K1); + build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs); + build_tlb_probe_entry(&p); + uasm_i_ori(&p, K0, K0, + _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); + build_huge_handler_tail(&p, &r, &l, K0, K1); +#endif + uasm_l_nopage_tlbs(&l, p); uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); uasm_i_nop(&p); @@ -1250,6 +1399,20 @@ static void __cpuinit build_r4000_tlb_modify_handler(void) build_make_write(&p, &r, K0, K1); build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); +#ifdef CONFIG_HUGETLB_PAGE + /* + * This is the entry point when + * build_r4000_tlbchange_handler_head spots a huge page. + */ + uasm_l_tlb_huge_update(&l, p); + iPTE_LW(&p, K0, K1); + build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm); + build_tlb_probe_entry(&p); + uasm_i_ori(&p, K0, K0, + _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); + build_huge_handler_tail(&p, &r, &l, K0, K1); +#endif + uasm_l_nopage_tlbm(&l, p); uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); uasm_i_nop(&p); -- cgit v1.2.3 From 631330f5847b3f8a7ea67d689e9f7c56833ccaa6 Mon Sep 17 00:00:00 2001 From: Ralf Baechle Date: Fri, 19 Jun 2009 14:05:26 +0100 Subject: MIPS: Build fix - include into all smp_processor_id() users. Some of the were relying into smp.h being dragged in by another header which of course is fragile. uses smp_processor_id() only in macros and including smp.h there leads to an include loop, so don't change cpu-info.h. Signed-off-by: Ralf Baechle --- arch/mips/cavium-octeon/octeon-irq.c | 2 +- arch/mips/cavium-octeon/setup.c | 1 + arch/mips/include/asm/bugs.h | 1 + arch/mips/include/asm/irq.h | 1 + arch/mips/include/asm/mmu_context.h | 1 + arch/mips/include/asm/smp.h | 1 + arch/mips/include/asm/sn/addrs.h | 1 + arch/mips/jazz/irq.c | 1 + arch/mips/kernel/cevt-bcm1480.c | 1 + arch/mips/kernel/cevt-r4k.c | 1 + arch/mips/kernel/cevt-sb1250.c | 1 + arch/mips/kernel/cevt-smtc.c | 1 + arch/mips/kernel/cpu-probe.c | 1 + arch/mips/kernel/i8253.c | 1 + arch/mips/kernel/irq-gic.c | 1 + arch/mips/kernel/kgdb.c | 1 + arch/mips/kernel/smp-cmp.c | 1 + arch/mips/kernel/smp.c | 1 + arch/mips/kernel/smtc.c | 1 + arch/mips/mm/c-octeon.c | 1 + arch/mips/mm/c-r3k.c | 1 + arch/mips/mm/c-r4k.c | 1 + arch/mips/mm/c-tx39.c | 1 + arch/mips/mm/highmem.c | 1 + arch/mips/mm/init.c | 1 + arch/mips/mm/page.c | 1 + arch/mips/mm/tlb-r3k.c | 1 + arch/mips/mm/tlb-r4k.c | 1 + arch/mips/mm/tlb-r8k.c | 1 + arch/mips/mm/tlbex.c | 1 + arch/mips/mti-malta/malta-int.c | 1 + arch/mips/pci/pci-ip27.c | 1 + arch/mips/pmc-sierra/yosemite/smp.c | 1 + arch/mips/sgi-ip27/ip27-init.c | 1 + arch/mips/sgi-ip27/ip27-irq.c | 1 + arch/mips/sgi-ip27/ip27-timer.c | 1 + arch/mips/sgi-ip27/ip27-xtalk.c | 1 + arch/mips/sibyte/bcm1480/irq.c | 1 + arch/mips/sni/time.c | 1 + 39 files changed, 39 insertions(+), 1 deletion(-) (limited to 'arch/mips/mm/tlbex.c') diff --git a/arch/mips/cavium-octeon/octeon-irq.c b/arch/mips/cavium-octeon/octeon-irq.c index 8dfa009e0070..3090ee37e109 100644 --- a/arch/mips/cavium-octeon/octeon-irq.c +++ b/arch/mips/cavium-octeon/octeon-irq.c @@ -7,7 +7,7 @@ */ #include #include -#include +#include #include #include diff --git a/arch/mips/cavium-octeon/setup.c b/arch/mips/cavium-octeon/setup.c index 5f4e49ba4713..da559249cc2f 100644 --- a/arch/mips/cavium-octeon/setup.c +++ b/arch/mips/cavium-octeon/setup.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include /* for memset */ #include diff --git a/arch/mips/include/asm/bugs.h b/arch/mips/include/asm/bugs.h index 9dc10df32078..b160a706795d 100644 --- a/arch/mips/include/asm/bugs.h +++ b/arch/mips/include/asm/bugs.h @@ -11,6 +11,7 @@ #include #include +#include #include #include diff --git a/arch/mips/include/asm/irq.h b/arch/mips/include/asm/irq.h index 4f1eed107b08..09b08d05ff72 100644 --- a/arch/mips/include/asm/irq.h +++ b/arch/mips/include/asm/irq.h @@ -10,6 +10,7 @@ #define _ASM_IRQ_H #include +#include #include diff --git a/arch/mips/include/asm/mmu_context.h b/arch/mips/include/asm/mmu_context.h index d7f3eb03ad12..d3bea88d8744 100644 --- a/arch/mips/include/asm/mmu_context.h +++ b/arch/mips/include/asm/mmu_context.h @@ -13,6 +13,7 @@ #include #include +#include #include #include #include diff --git a/arch/mips/include/asm/smp.h b/arch/mips/include/asm/smp.h index 40e5ef1d4d26..2f83fa8631db 100644 --- a/arch/mips/include/asm/smp.h +++ b/arch/mips/include/asm/smp.h @@ -13,6 +13,7 @@ #include #include +#include #include #include diff --git a/arch/mips/include/asm/sn/addrs.h b/arch/mips/include/asm/sn/addrs.h index 3a56d90abfa6..2367b56dcdef 100644 --- a/arch/mips/include/asm/sn/addrs.h +++ b/arch/mips/include/asm/sn/addrs.h @@ -11,6 +11,7 @@ #ifndef __ASSEMBLY__ +#include #include #endif /* !__ASSEMBLY__ */ diff --git a/arch/mips/jazz/irq.c b/arch/mips/jazz/irq.c index d9b6a5b5399d..7fd170d007e7 100644 --- a/arch/mips/jazz/irq.c +++ b/arch/mips/jazz/irq.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include diff --git a/arch/mips/kernel/cevt-bcm1480.c b/arch/mips/kernel/cevt-bcm1480.c index a5182a207696..e02f79b1eb51 100644 --- a/arch/mips/kernel/cevt-bcm1480.c +++ b/arch/mips/kernel/cevt-bcm1480.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include diff --git a/arch/mips/kernel/cevt-r4k.c b/arch/mips/kernel/cevt-r4k.c index 0015e442572b..2652362ce047 100644 --- a/arch/mips/kernel/cevt-r4k.c +++ b/arch/mips/kernel/cevt-r4k.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include diff --git a/arch/mips/kernel/cevt-sb1250.c b/arch/mips/kernel/cevt-sb1250.c index 340f53e5c6b1..ac5903d1b20e 100644 --- a/arch/mips/kernel/cevt-sb1250.c +++ b/arch/mips/kernel/cevt-sb1250.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include diff --git a/arch/mips/kernel/cevt-smtc.c b/arch/mips/kernel/cevt-smtc.c index df6f5bc60572..98bd7de75778 100644 --- a/arch/mips/kernel/cevt-smtc.c +++ b/arch/mips/kernel/cevt-smtc.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index b13b8eb30596..1abe9905c9c1 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include diff --git a/arch/mips/kernel/i8253.c b/arch/mips/kernel/i8253.c index ed20e7fe65e3..f7d8d5d0ddbf 100644 --- a/arch/mips/kernel/i8253.c +++ b/arch/mips/kernel/i8253.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include diff --git a/arch/mips/kernel/irq-gic.c b/arch/mips/kernel/irq-gic.c index 3f43c2e3aa5a..39000f103f2c 100644 --- a/arch/mips/kernel/irq-gic.c +++ b/arch/mips/kernel/irq-gic.c @@ -2,6 +2,7 @@ #include #include +#include #include #include diff --git a/arch/mips/kernel/kgdb.c b/arch/mips/kernel/kgdb.c index 6e152c80cd4a..50c9bb880667 100644 --- a/arch/mips/kernel/kgdb.c +++ b/arch/mips/kernel/kgdb.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/kernel/smp-cmp.c b/arch/mips/kernel/smp-cmp.c index f27beca4b26d..653be061b9ec 100644 --- a/arch/mips/kernel/smp-cmp.c +++ b/arch/mips/kernel/smp-cmp.c @@ -20,6 +20,7 @@ #include #include +#include #include #include #include diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c index c937506a03aa..58f4679bbd43 100644 --- a/arch/mips/kernel/smp.c +++ b/arch/mips/kernel/smp.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/kernel/smtc.c b/arch/mips/kernel/smtc.c index 37d51cd124e9..8a0626cbb108 100644 --- a/arch/mips/kernel/smtc.c +++ b/arch/mips/kernel/smtc.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/mm/c-octeon.c b/arch/mips/mm/c-octeon.c index 44d01a0a8490..b165cdcb2818 100644 --- a/arch/mips/mm/c-octeon.c +++ b/arch/mips/mm/c-octeon.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/mm/c-r3k.c b/arch/mips/mm/c-r3k.c index 5500c20c79ae..54e5f7b9f440 100644 --- a/arch/mips/mm/c-r3k.c +++ b/arch/mips/mm/c-r3k.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index 71fe4cb778cd..6721ee2b1e8b 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/mm/c-tx39.c b/arch/mips/mm/c-tx39.c index f7c8f9ce39c1..6515b4418714 100644 --- a/arch/mips/mm/c-tx39.c +++ b/arch/mips/mm/c-tx39.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include diff --git a/arch/mips/mm/highmem.c b/arch/mips/mm/highmem.c index 2b1309b2580a..e274fda329f4 100644 --- a/arch/mips/mm/highmem.c +++ b/arch/mips/mm/highmem.c @@ -1,5 +1,6 @@ #include #include +#include #include #include diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c index c5511294a9ee..0e820508ff23 100644 --- a/arch/mips/mm/init.c +++ b/arch/mips/mm/init.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/mm/page.c b/arch/mips/mm/page.c index 48060c635acd..f5c73754d664 100644 --- a/arch/mips/mm/page.c +++ b/arch/mips/mm/page.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/mm/tlb-r3k.c b/arch/mips/mm/tlb-r3k.c index 1c0048a6f5cf..0f5ab236ab69 100644 --- a/arch/mips/mm/tlb-r3k.c +++ b/arch/mips/mm/tlb-r3k.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c index f60fe513eb60..cee502caf398 100644 --- a/arch/mips/mm/tlb-r4k.c +++ b/arch/mips/mm/tlb-r4k.c @@ -10,6 +10,7 @@ */ #include #include +#include #include #include diff --git a/arch/mips/mm/tlb-r8k.c b/arch/mips/mm/tlb-r8k.c index 4ec95cc2df2f..2b82f23df1a1 100644 --- a/arch/mips/mm/tlb-r8k.c +++ b/arch/mips/mm/tlb-r8k.c @@ -10,6 +10,7 @@ */ #include #include +#include #include #include diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 8f606ead826e..9a17bf8395df 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include diff --git a/arch/mips/mti-malta/malta-int.c b/arch/mips/mti-malta/malta-int.c index ea176113fea9..b4eaf137e4a7 100644 --- a/arch/mips/mti-malta/malta-int.c +++ b/arch/mips/mti-malta/malta-int.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/pci/pci-ip27.c b/arch/mips/pci/pci-ip27.c index dda6f2058665..a0e726eb039a 100644 --- a/arch/mips/pci/pci-ip27.c +++ b/arch/mips/pci/pci-ip27.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/pmc-sierra/yosemite/smp.c b/arch/mips/pmc-sierra/yosemite/smp.c index f78c29b68d77..8ace27716232 100644 --- a/arch/mips/pmc-sierra/yosemite/smp.c +++ b/arch/mips/pmc-sierra/yosemite/smp.c @@ -1,5 +1,6 @@ #include #include +#include #include #include diff --git a/arch/mips/sgi-ip27/ip27-init.c b/arch/mips/sgi-ip27/ip27-init.c index 4a500e8cd3cc..51d3a4f2d7e1 100644 --- a/arch/mips/sgi-ip27/ip27-init.c +++ b/arch/mips/sgi-ip27/ip27-init.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/sgi-ip27/ip27-irq.c b/arch/mips/sgi-ip27/ip27-irq.c index 1bb692a3b319..c1c8e40d65d6 100644 --- a/arch/mips/sgi-ip27/ip27-irq.c +++ b/arch/mips/sgi-ip27/ip27-irq.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/sgi-ip27/ip27-timer.c b/arch/mips/sgi-ip27/ip27-timer.c index f10a7cd64f7e..6d0e59ffba2e 100644 --- a/arch/mips/sgi-ip27/ip27-timer.c +++ b/arch/mips/sgi-ip27/ip27-timer.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/sgi-ip27/ip27-xtalk.c b/arch/mips/sgi-ip27/ip27-xtalk.c index 6ae64e8dfc40..5e871e75a8d9 100644 --- a/arch/mips/sgi-ip27/ip27-xtalk.c +++ b/arch/mips/sgi-ip27/ip27-xtalk.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include diff --git a/arch/mips/sibyte/bcm1480/irq.c b/arch/mips/sibyte/bcm1480/irq.c index 690de06bde90..ba59839a021e 100644 --- a/arch/mips/sibyte/bcm1480/irq.c +++ b/arch/mips/sibyte/bcm1480/irq.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/sni/time.c b/arch/mips/sni/time.c index 69f5f88711cc..0d9ec1a5c24a 100644 --- a/arch/mips/sni/time.c +++ b/arch/mips/sni/time.c @@ -1,5 +1,6 @@ #include #include +#include #include #include -- cgit v1.2.3