diff options
Diffstat (limited to 'arch/sparc')
92 files changed, 2486 insertions, 456 deletions
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 68ac5c7cd982..58243b0d21c0 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -31,7 +31,8 @@ config SPARC select ARCH_WANT_IPC_PARSE_VERSION select GENERIC_PCI_IOMAP select HAVE_NMI_WATCHDOG if SPARC64 - select HAVE_CBPF_JIT + select HAVE_CBPF_JIT if SPARC32 + select HAVE_EBPF_JIT if SPARC64 select HAVE_DEBUG_BUGVERBOSE select GENERIC_SMP_IDLE_THREAD select GENERIC_CLOCKEVENTS @@ -42,8 +43,7 @@ config SPARC select OLD_SIGSUSPEND select ARCH_HAS_SG_CHAIN select CPU_NO_EFFICIENT_FFS - select HAVE_ARCH_HARDENED_USERCOPY - select PROVE_LOCKING_SMALL if PROVE_LOCKING + select LOCKDEP_SMALL if LOCKDEP select ARCH_WANT_RELAX_ORDER config SPARC32 @@ -82,6 +82,7 @@ config SPARC64 select HAVE_ARCH_AUDITSYSCALL select ARCH_SUPPORTS_ATOMIC_RMW select HAVE_NMI + select HAVE_REGS_AND_STACK_ACCESS_API config ARCH_DEFCONFIG string diff --git a/arch/sparc/include/asm/dma-mapping.h b/arch/sparc/include/asm/dma-mapping.h index 1180ae254154..69cc627779f2 100644 --- a/arch/sparc/include/asm/dma-mapping.h +++ b/arch/sparc/include/asm/dma-mapping.h @@ -18,20 +18,20 @@ static inline void dma_cache_sync(struct device *dev, void *vaddr, size_t size, */ } -extern struct dma_map_ops *dma_ops; -extern struct dma_map_ops *leon_dma_ops; -extern struct dma_map_ops pci32_dma_ops; +extern const struct dma_map_ops *dma_ops; +extern const struct dma_map_ops *leon_dma_ops; +extern const struct dma_map_ops pci32_dma_ops; extern struct bus_type pci_bus_type; -static inline struct dma_map_ops *get_dma_ops(struct device *dev) +static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) { #ifdef CONFIG_SPARC_LEON if (sparc_cpu_model == sparc_leon) return leon_dma_ops; #endif #if defined(CONFIG_SPARC32) && defined(CONFIG_PCI) - if (dev->bus == &pci_bus_type) + if (bus == &pci_bus_type) return &pci32_dma_ops; #endif return dma_ops; diff --git a/arch/sparc/include/asm/kprobes.h b/arch/sparc/include/asm/kprobes.h index a145d798e112..49f8402035d7 100644 --- a/arch/sparc/include/asm/kprobes.h +++ b/arch/sparc/include/asm/kprobes.h @@ -1,13 +1,17 @@ #ifndef _SPARC64_KPROBES_H #define _SPARC64_KPROBES_H +#include <asm-generic/kprobes.h> + +#define BREAKPOINT_INSTRUCTION 0x91d02070 /* ta 0x70 */ +#define BREAKPOINT_INSTRUCTION_2 0x91d02071 /* ta 0x71 */ + +#ifdef CONFIG_KPROBES #include <linux/types.h> #include <linux/percpu.h> typedef u32 kprobe_opcode_t; -#define BREAKPOINT_INSTRUCTION 0x91d02070 /* ta 0x70 */ -#define BREAKPOINT_INSTRUCTION_2 0x91d02071 /* ta 0x71 */ #define MAX_INSN_SIZE 2 #define kretprobe_blacklist_size 0 @@ -48,4 +52,6 @@ int kprobe_exceptions_notify(struct notifier_block *self, int kprobe_fault_handler(struct pt_regs *regs, int trapnr); asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, struct pt_regs *regs); + +#endif /* CONFIG_KPROBES */ #endif /* _SPARC64_KPROBES_H */ diff --git a/arch/sparc/include/asm/mmu_context_64.h b/arch/sparc/include/asm/mmu_context_64.h index d0317993e947..22fede6eba11 100644 --- a/arch/sparc/include/asm/mmu_context_64.h +++ b/arch/sparc/include/asm/mmu_context_64.h @@ -6,6 +6,8 @@ #ifndef __ASSEMBLY__ #include <linux/spinlock.h> +#include <linux/mm_types.h> + #include <asm/spitfire.h> #include <asm-generic/mm_hooks.h> diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h index c1263fc390db..5961b2d8398a 100644 --- a/arch/sparc/include/asm/page_64.h +++ b/arch/sparc/include/asm/page_64.h @@ -17,7 +17,9 @@ #define HPAGE_SHIFT 23 #define REAL_HPAGE_SHIFT 22 - +#define HPAGE_2GB_SHIFT 31 +#define HPAGE_256MB_SHIFT 28 +#define HPAGE_64K_SHIFT 16 #define REAL_HPAGE_SIZE (_AC(1,UL) << REAL_HPAGE_SHIFT) #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) @@ -26,6 +28,7 @@ #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) #define HAVE_ARCH_HUGETLB_UNMAPPED_AREA #define REAL_HPAGE_PER_HPAGE (_AC(1,UL) << (HPAGE_SHIFT - REAL_HPAGE_SHIFT)) +#define HUGE_MAX_HSTATE 4 #endif #ifndef __ASSEMBLY__ diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index 314b66851348..6fbd931f0570 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -12,6 +12,7 @@ * the SpitFire page tables. */ +#include <asm-generic/5level-fixup.h> #include <linux/compiler.h> #include <linux/const.h> #include <asm/types.h> @@ -375,7 +376,10 @@ static inline pgprot_t pgprot_noncached(pgprot_t prot) #define pgprot_noncached pgprot_noncached #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) -static inline unsigned long __pte_huge_mask(void) +extern pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, + struct page *page, int writable); +#define arch_make_huge_pte arch_make_huge_pte +static inline unsigned long __pte_default_huge_mask(void) { unsigned long mask; @@ -395,12 +399,14 @@ static inline unsigned long __pte_huge_mask(void) static inline pte_t pte_mkhuge(pte_t pte) { - return __pte(pte_val(pte) | _PAGE_PMD_HUGE | __pte_huge_mask()); + return __pte(pte_val(pte) | __pte_default_huge_mask()); } -static inline bool is_hugetlb_pte(pte_t pte) +static inline bool is_default_hugetlb_pte(pte_t pte) { - return !!(pte_val(pte) & __pte_huge_mask()); + unsigned long mask = __pte_default_huge_mask(); + + return (pte_val(pte) & mask) == mask; } static inline bool is_hugetlb_pmd(pmd_t pmd) @@ -673,26 +679,27 @@ static inline unsigned long pmd_pfn(pmd_t pmd) return pte_pfn(pte); } -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -static inline unsigned long pmd_dirty(pmd_t pmd) +#define __HAVE_ARCH_PMD_WRITE +static inline unsigned long pmd_write(pmd_t pmd) { pte_t pte = __pte(pmd_val(pmd)); - return pte_dirty(pte); + return pte_write(pte); } -static inline unsigned long pmd_young(pmd_t pmd) +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline unsigned long pmd_dirty(pmd_t pmd) { pte_t pte = __pte(pmd_val(pmd)); - return pte_young(pte); + return pte_dirty(pte); } -static inline unsigned long pmd_write(pmd_t pmd) +static inline unsigned long pmd_young(pmd_t pmd) { pte_t pte = __pte(pmd_val(pmd)); - return pte_write(pte); + return pte_young(pte); } static inline unsigned long pmd_trans_huge(pmd_t pmd) @@ -873,12 +880,17 @@ static inline unsigned long pud_pfn(pud_t pud) #define pte_offset_map pte_index #define pte_unmap(pte) do { } while (0) +/* We cannot include <linux/mm_types.h> at this point yet: */ +extern struct mm_struct init_mm; + /* Actual page table PTE updates. */ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, - pte_t *ptep, pte_t orig, int fullmm); + pte_t *ptep, pte_t orig, int fullmm, + unsigned int hugepage_shift); static void maybe_tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, - pte_t *ptep, pte_t orig, int fullmm) + pte_t *ptep, pte_t orig, int fullmm, + unsigned int hugepage_shift) { /* It is more efficient to let flush_tlb_kernel_range() * handle init_mm tlb flushes. @@ -887,7 +899,7 @@ static void maybe_tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, * and SUN4V pte layout, so this inline test is fine. */ if (likely(mm != &init_mm) && pte_accessible(mm, orig)) - tlb_batch_add(mm, vaddr, ptep, orig, fullmm); + tlb_batch_add(mm, vaddr, ptep, orig, fullmm, hugepage_shift); } #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR @@ -906,7 +918,7 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t orig = *ptep; *ptep = pte; - maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm); + maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm, PAGE_SHIFT); } #define set_pte_at(mm,addr,ptep,pte) \ diff --git a/arch/sparc/include/asm/processor_32.h b/arch/sparc/include/asm/processor_32.h index 365d4cb267b4..dd27159819eb 100644 --- a/arch/sparc/include/asm/processor_32.h +++ b/arch/sparc/include/asm/processor_32.h @@ -18,12 +18,6 @@ #include <asm/signal.h> #include <asm/page.h> -/* - * The sparc has no problems with write protection - */ -#define wp_works_ok 1 -#define wp_works_ok__is_a_macro /* for versions in ksyms.c */ - /* Whee, this is STACK_TOP + PAGE_SIZE and the lowest kernel address too... * That one page is used to protect kernel from intruders, so that * we can make our access_ok test faster diff --git a/arch/sparc/include/asm/processor_64.h b/arch/sparc/include/asm/processor_64.h index 6448cfc8292f..b58ee9018433 100644 --- a/arch/sparc/include/asm/processor_64.h +++ b/arch/sparc/include/asm/processor_64.h @@ -18,10 +18,6 @@ #include <asm/ptrace.h> #include <asm/page.h> -/* The sparc has no problems with write protection */ -#define wp_works_ok 1 -#define wp_works_ok__is_a_macro /* for versions in ksyms.c */ - /* * User lives in his very own context, and cannot reference us. Note * that TASK_SIZE is a misnomer, it really gives maximum user virtual diff --git a/arch/sparc/include/asm/ptrace.h b/arch/sparc/include/asm/ptrace.h index ca57f08bd3db..d73428e4333c 100644 --- a/arch/sparc/include/asm/ptrace.h +++ b/arch/sparc/include/asm/ptrace.h @@ -83,7 +83,8 @@ unsigned long profile_pc(struct pt_regs *); #define MAX_REG_OFFSET (offsetof(struct pt_regs, magic)) -extern int regs_query_register_offset(const char *name); +int regs_query_register_offset(const char *name); +unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n); /** * regs_get_register() - get register value from its offset diff --git a/arch/sparc/include/asm/setup.h b/arch/sparc/include/asm/setup.h index 29d64b1758ed..478bf6bb4598 100644 --- a/arch/sparc/include/asm/setup.h +++ b/arch/sparc/include/asm/setup.h @@ -59,8 +59,11 @@ extern atomic_t dcpage_flushes; extern atomic_t dcpage_flushes_xcall; extern int sysctl_tsb_ratio; -#endif +#ifdef CONFIG_SERIAL_SUNHV +void sunhv_migrate_hvcons_irq(int cpu); +#endif +#endif void sun_do_break(void); extern int stop_a_enabled; extern int scons_pwroff; diff --git a/arch/sparc/include/asm/switch_to_32.h b/arch/sparc/include/asm/switch_to_32.h index 16f10374feb3..475dd4158ae4 100644 --- a/arch/sparc/include/asm/switch_to_32.h +++ b/arch/sparc/include/asm/switch_to_32.h @@ -9,7 +9,7 @@ extern struct thread_info *current_set[NR_CPUS]; * Flush windows so that the VM switch which follows * would not pull the stack from under us. * - * SWITCH_ENTER and SWITH_DO_LAZY_FPU do not work yet (e.g. SMP does not work) + * SWITCH_ENTER and SWITCH_DO_LAZY_FPU do not work yet (e.g. SMP does not work) * XXX WTF is the above comment? Found in late teen 2.4.x. */ #ifdef CONFIG_SMP diff --git a/arch/sparc/include/asm/tlbflush_64.h b/arch/sparc/include/asm/tlbflush_64.h index a8e192e90700..54be88a6774c 100644 --- a/arch/sparc/include/asm/tlbflush_64.h +++ b/arch/sparc/include/asm/tlbflush_64.h @@ -8,7 +8,7 @@ #define TLB_BATCH_NR 192 struct tlb_batch { - bool huge; + unsigned int hugepage_shift; struct mm_struct *mm; unsigned long tlb_nr; unsigned long active; @@ -17,7 +17,8 @@ struct tlb_batch { void flush_tsb_kernel_range(unsigned long start, unsigned long end); void flush_tsb_user(struct tlb_batch *tb); -void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, bool huge); +void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, + unsigned int hugepage_shift); /* TLB flush operations. */ diff --git a/arch/sparc/include/asm/topology_64.h b/arch/sparc/include/asm/topology_64.h index 225543000122..ad5293f89680 100644 --- a/arch/sparc/include/asm/topology_64.h +++ b/arch/sparc/include/asm/topology_64.h @@ -4,7 +4,6 @@ #ifdef CONFIG_NUMA #include <asm/mmzone.h> -#include <asm/cpudata.h> static inline int cpu_to_node(int cpu) { @@ -42,6 +41,9 @@ int __node_distance(int, int); #endif /* !(CONFIG_NUMA) */ #ifdef CONFIG_SMP + +#include <asm/cpudata.h> + #define topology_physical_package_id(cpu) (cpu_data(cpu).proc_id) #define topology_core_id(cpu) (cpu_data(cpu).core_id) #define topology_core_cpumask(cpu) (&cpu_core_sib_map[cpu]) diff --git a/arch/sparc/include/asm/uaccess.h b/arch/sparc/include/asm/uaccess.h index bd56c28fff9f..9e068bf9060a 100644 --- a/arch/sparc/include/asm/uaccess.h +++ b/arch/sparc/include/asm/uaccess.h @@ -7,7 +7,7 @@ #endif #define user_addr_max() \ - (segment_eq(get_fs(), USER_DS) ? TASK_SIZE : ~0UL) + (uaccess_kernel() ? ~0UL : TASK_SIZE) long strncpy_from_user(char *dest, const char __user *src, long count); diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h index ea55f86d7ccd..12ebee2d97c7 100644 --- a/arch/sparc/include/asm/uaccess_32.h +++ b/arch/sparc/include/asm/uaccess_32.h @@ -7,14 +7,8 @@ #ifndef _ASM_UACCESS_H #define _ASM_UACCESS_H -#ifdef __KERNEL__ #include <linux/compiler.h> -#include <linux/sched.h> #include <linux/string.h> -#include <linux/errno.h> -#endif - -#ifndef __ASSEMBLY__ #include <asm/processor.h> @@ -30,9 +24,6 @@ #define KERNEL_DS ((mm_segment_t) { 0 }) #define USER_DS ((mm_segment_t) { -1 }) -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #define get_ds() (KERNEL_DS) #define get_fs() (current->thread.current_ds) #define set_fs(val) ((current->thread.current_ds) = (val)) @@ -45,7 +36,7 @@ * large size and address near to PAGE_OFFSET - a fault will break his intentions. */ #define __user_ok(addr, size) ({ (void)(size); (addr) < STACK_TOP; }) -#define __kernel_ok (segment_eq(get_fs(), KERNEL_DS)) +#define __kernel_ok (uaccess_kernel()) #define __access_ok(addr, size) (__user_ok((addr) & get_fs().seg, (size))) #define access_ok(type, addr, size) \ ({ (void)(type); __access_ok((unsigned long)(addr), size); }) @@ -80,8 +71,6 @@ struct exception_table_entry /* Returns 0 if exception not found and fixup otherwise. */ unsigned long search_extables_range(unsigned long addr, unsigned long *g2); -void __ret_efault(void); - /* Uh, these should become the main single-value transfer routines.. * They automatically use the right size if we just have the right * pointer type.. @@ -246,39 +235,18 @@ int __get_user_bad(void); unsigned long __copy_user(void __user *to, const void __user *from, unsigned long size); -static inline unsigned long copy_to_user(void __user *to, const void *from, unsigned long n) -{ - if (n && __access_ok((unsigned long) to, n)) { - check_object_size(from, n, true); - return __copy_user(to, (__force void __user *) from, n); - } else - return n; -} - -static inline unsigned long __copy_to_user(void __user *to, const void *from, unsigned long n) +static inline unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n) { - check_object_size(from, n, true); return __copy_user(to, (__force void __user *) from, n); } -static inline unsigned long copy_from_user(void *to, const void __user *from, unsigned long n) -{ - if (n && __access_ok((unsigned long) from, n)) { - check_object_size(to, n, false); - return __copy_user((__force void __user *) to, from, n); - } else { - memset(to, 0, n); - return n; - } -} - -static inline unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n) +static inline unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n) { return __copy_user((__force void __user *) to, from, n); } -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER static inline unsigned long __clear_user(void __user *addr, unsigned long size) { @@ -312,6 +280,4 @@ static inline unsigned long clear_user(void __user *addr, unsigned long n) __must_check long strlen_user(const char __user *str); __must_check long strnlen_user(const char __user *str, long n); -#endif /* __ASSEMBLY__ */ - #endif /* _ASM_UACCESS_H */ diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h index 5373136c412b..6096d671aa63 100644 --- a/arch/sparc/include/asm/uaccess_64.h +++ b/arch/sparc/include/asm/uaccess_64.h @@ -5,18 +5,12 @@ * User space memory access functions */ -#ifdef __KERNEL__ -#include <linux/errno.h> #include <linux/compiler.h> #include <linux/string.h> -#include <linux/thread_info.h> #include <asm/asi.h> #include <asm/spitfire.h> #include <asm-generic/uaccess-unaligned.h> #include <asm/extable_64.h> -#endif - -#ifndef __ASSEMBLY__ #include <asm/processor.h> @@ -36,9 +30,6 @@ #define KERNEL_DS ((mm_segment_t) { ASI_P }) #define USER_DS ((mm_segment_t) { ASI_AIUS }) /* har har har */ -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #define get_fs() ((mm_segment_t){(current_thread_info()->current_ds)}) #define get_ds() (KERNEL_DS) @@ -185,39 +176,19 @@ __asm__ __volatile__( \ int __get_user_bad(void); -unsigned long __must_check ___copy_from_user(void *to, +unsigned long __must_check raw_copy_from_user(void *to, const void __user *from, unsigned long size); -static inline unsigned long __must_check -copy_from_user(void *to, const void __user *from, unsigned long size) -{ - check_object_size(to, size, false); - return ___copy_from_user(to, from, size); -} -#define __copy_from_user copy_from_user - -unsigned long __must_check ___copy_to_user(void __user *to, +unsigned long __must_check raw_copy_to_user(void __user *to, const void *from, unsigned long size); -static inline unsigned long __must_check -copy_to_user(void __user *to, const void *from, unsigned long size) -{ - check_object_size(from, size, true); +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER - return ___copy_to_user(to, from, size); -} -#define __copy_to_user copy_to_user - -unsigned long __must_check ___copy_in_user(void __user *to, +unsigned long __must_check raw_copy_in_user(void __user *to, const void __user *from, unsigned long size); -static inline unsigned long __must_check -copy_in_user(void __user *to, void __user *from, unsigned long size) -{ - return ___copy_in_user(to, from, size); -} -#define __copy_in_user copy_in_user unsigned long __must_check __clear_user(void __user *, unsigned long); @@ -226,14 +197,9 @@ unsigned long __must_check __clear_user(void __user *, unsigned long); __must_check long strlen_user(const char __user *str); __must_check long strnlen_user(const char __user *str, long n); -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user - struct pt_regs; unsigned long compute_effective_address(struct pt_regs *, unsigned int insn, unsigned int rd); -#endif /* __ASSEMBLY__ */ - #endif /* _ASM_UACCESS_H */ diff --git a/arch/sparc/include/asm/uprobes.h b/arch/sparc/include/asm/uprobes.h index f87aae5a908e..36196c17aff8 100644 --- a/arch/sparc/include/asm/uprobes.h +++ b/arch/sparc/include/asm/uprobes.h @@ -42,8 +42,8 @@ struct arch_uprobe { }; struct arch_uprobe_task { - u32 saved_tpc; - u32 saved_tnpc; + u64 saved_tpc; + u64 saved_tnpc; }; struct task_struct; diff --git a/arch/sparc/include/uapi/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h index a25dc32f5d6a..3f4ad19d9ec7 100644 --- a/arch/sparc/include/uapi/asm/socket.h +++ b/arch/sparc/include/uapi/asm/socket.h @@ -88,6 +88,12 @@ #define SCM_TIMESTAMPING_OPT_STATS 0x0038 +#define SO_MEMINFO 0x0039 + +#define SO_INCOMING_NAPI_ID 0x003a + +#define SO_COOKIE 0x003b + /* Security levels - as per NRL IPv6 - don't actually do anything */ #define SO_SECURITY_AUTHENTICATION 0x5001 #define SO_SECURITY_ENCRYPTION_TRANSPORT 0x5002 diff --git a/arch/sparc/include/uapi/asm/unistd.h b/arch/sparc/include/uapi/asm/unistd.h index 36eee8132c22..ae77df75bffa 100644 --- a/arch/sparc/include/uapi/asm/unistd.h +++ b/arch/sparc/include/uapi/asm/unistd.h @@ -425,8 +425,9 @@ #define __NR_copy_file_range 357 #define __NR_preadv2 358 #define __NR_pwritev2 359 +#define __NR_statx 360 -#define NR_syscalls 360 +#define NR_syscalls 361 /* Bitmask values returned from kern_features system call. */ #define KERN_FEATURE_MIXED_MODE_STACK 0x00000001 @@ -442,4 +443,9 @@ #define __IGNORE_getresgid #endif +/* Sparc doesn't have protection keys. */ +#define __IGNORE_pkey_mprotect +#define __IGNORE_pkey_alloc +#define __IGNORE_pkey_free + #endif /* _UAPI_SPARC_UNISTD_H */ diff --git a/arch/sparc/kernel/asm-offsets.c b/arch/sparc/kernel/asm-offsets.c index f76389a32342..3f09e1c83f58 100644 --- a/arch/sparc/kernel/asm-offsets.c +++ b/arch/sparc/kernel/asm-offsets.c @@ -11,6 +11,7 @@ */ #include <linux/sched.h> +#include <linux/mm_types.h> // #include <linux/mm.h> #include <linux/kbuild.h> diff --git a/arch/sparc/kernel/ds.c b/arch/sparc/kernel/ds.c index f87a55d77094..b542cc7c8d94 100644 --- a/arch/sparc/kernel/ds.c +++ b/arch/sparc/kernel/ds.c @@ -9,6 +9,7 @@ #include <linux/string.h> #include <linux/slab.h> #include <linux/sched.h> +#include <linux/sched/clock.h> #include <linux/delay.h> #include <linux/mutex.h> #include <linux/kthread.h> diff --git a/arch/sparc/kernel/head_32.S b/arch/sparc/kernel/head_32.S index 7bb317b87dde..7274e43ff9be 100644 --- a/arch/sparc/kernel/head_32.S +++ b/arch/sparc/kernel/head_32.S @@ -809,10 +809,3 @@ lvl14_save: .word 0 .word 0 .word t_irq14 - - .section ".fixup",#alloc,#execinstr - .globl __ret_efault -__ret_efault: - ret - restore %g0, -EFAULT, %o0 -EXPORT_SYMBOL(__ret_efault) diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S index 6aa3da152c20..44101196d02b 100644 --- a/arch/sparc/kernel/head_64.S +++ b/arch/sparc/kernel/head_64.S @@ -96,6 +96,7 @@ sparc64_boot: andn %g1, PSTATE_AM, %g1 wrpr %g1, 0x0, %pstate ba,a,pt %xcc, 1f + nop .globl prom_finddev_name, prom_chosen_path, prom_root_node .globl prom_getprop_name, prom_mmu_name, prom_peer_name @@ -613,6 +614,7 @@ niagara_tlb_fixup: nop ba,a,pt %xcc, 80f + nop niagara4_patch: call niagara4_patch_copyops nop @@ -622,6 +624,7 @@ niagara4_patch: nop ba,a,pt %xcc, 80f + nop niagara2_patch: call niagara2_patch_copyops @@ -632,6 +635,7 @@ niagara2_patch: nop ba,a,pt %xcc, 80f + nop niagara_patch: call niagara_patch_copyops diff --git a/arch/sparc/kernel/iommu.c b/arch/sparc/kernel/iommu.c index 9df997995f6b..c63ba99ca551 100644 --- a/arch/sparc/kernel/iommu.c +++ b/arch/sparc/kernel/iommu.c @@ -741,7 +741,7 @@ static void dma_4u_sync_sg_for_cpu(struct device *dev, spin_unlock_irqrestore(&iommu->lock, flags); } -static struct dma_map_ops sun4u_dma_ops = { +static const struct dma_map_ops sun4u_dma_ops = { .alloc = dma_4u_alloc_coherent, .free = dma_4u_free_coherent, .map_page = dma_4u_map_page, @@ -752,7 +752,7 @@ static struct dma_map_ops sun4u_dma_ops = { .sync_sg_for_cpu = dma_4u_sync_sg_for_cpu, }; -struct dma_map_ops *dma_ops = &sun4u_dma_ops; +const struct dma_map_ops *dma_ops = &sun4u_dma_ops; EXPORT_SYMBOL(dma_ops); int dma_supported(struct device *dev, u64 device_mask) diff --git a/arch/sparc/kernel/ioport.c b/arch/sparc/kernel/ioport.c index 6ffaec44931a..cf20033a1458 100644 --- a/arch/sparc/kernel/ioport.c +++ b/arch/sparc/kernel/ioport.c @@ -401,7 +401,7 @@ static void sbus_sync_sg_for_device(struct device *dev, struct scatterlist *sg, BUG(); } -static struct dma_map_ops sbus_dma_ops = { +static const struct dma_map_ops sbus_dma_ops = { .alloc = sbus_alloc_coherent, .free = sbus_free_coherent, .map_page = sbus_map_page, @@ -637,7 +637,7 @@ static void pci32_sync_sg_for_device(struct device *device, struct scatterlist * } } -struct dma_map_ops pci32_dma_ops = { +const struct dma_map_ops pci32_dma_ops = { .alloc = pci32_alloc_coherent, .free = pci32_free_coherent, .map_page = pci32_map_page, @@ -652,10 +652,10 @@ struct dma_map_ops pci32_dma_ops = { EXPORT_SYMBOL(pci32_dma_ops); /* leon re-uses pci32_dma_ops */ -struct dma_map_ops *leon_dma_ops = &pci32_dma_ops; +const struct dma_map_ops *leon_dma_ops = &pci32_dma_ops; EXPORT_SYMBOL(leon_dma_ops); -struct dma_map_ops *dma_ops = &sbus_dma_ops; +const struct dma_map_ops *dma_ops = &sbus_dma_ops; EXPORT_SYMBOL(dma_ops); diff --git a/arch/sparc/kernel/led.c b/arch/sparc/kernel/led.c index 3ae36f36e758..44a3ed93c214 100644 --- a/arch/sparc/kernel/led.c +++ b/arch/sparc/kernel/led.c @@ -8,6 +8,7 @@ #include <linux/jiffies.h> #include <linux/timer.h> #include <linux/uaccess.h> +#include <linux/sched/loadavg.h> #include <asm/auxio.h> diff --git a/arch/sparc/kernel/leon_smp.c b/arch/sparc/kernel/leon_smp.c index 71e16f2241c2..db7acf27bea2 100644 --- a/arch/sparc/kernel/leon_smp.c +++ b/arch/sparc/kernel/leon_smp.c @@ -9,7 +9,7 @@ #include <asm/head.h> #include <linux/kernel.h> -#include <linux/sched.h> +#include <linux/sched/mm.h> #include <linux/threads.h> #include <linux/smp.h> #include <linux/interrupt.h> @@ -93,7 +93,7 @@ void leon_cpu_pre_online(void *arg) : "memory" /* paranoid */); /* Attach to the address space of init_task. */ - atomic_inc(&init_mm.mm_count); + mmgrab(&init_mm); current->active_mm = &init_mm; while (!cpumask_test_cpu(cpuid, &smp_commenced_mask)) diff --git a/arch/sparc/kernel/misctrap.S b/arch/sparc/kernel/misctrap.S index 34b4933900bf..9276d2f0dd86 100644 --- a/arch/sparc/kernel/misctrap.S +++ b/arch/sparc/kernel/misctrap.S @@ -82,6 +82,7 @@ do_stdfmna: call handle_stdfmna add %sp, PTREGS_OFF, %o0 ba,a,pt %xcc, rtrap + nop .size do_stdfmna,.-do_stdfmna .type breakpoint_trap,#function diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c index f4daccd12bf5..68bec7c97cb8 100644 --- a/arch/sparc/kernel/pci_sun4v.c +++ b/arch/sparc/kernel/pci_sun4v.c @@ -669,7 +669,7 @@ static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, local_irq_restore(flags); } -static struct dma_map_ops sun4v_dma_ops = { +static const struct dma_map_ops sun4v_dma_ops = { .alloc = dma_4v_alloc_coherent, .free = dma_4v_free_coherent, .map_page = dma_4v_map_page, diff --git a/arch/sparc/kernel/process_32.c b/arch/sparc/kernel/process_32.c index 48ffc3e7d1dd..b6dac8e980f0 100644 --- a/arch/sparc/kernel/process_32.c +++ b/arch/sparc/kernel/process_32.c @@ -14,6 +14,9 @@ #include <linux/errno.h> #include <linux/module.h> #include <linux/sched.h> +#include <linux/sched/debug.h> +#include <linux/sched/task.h> +#include <linux/sched/task_stack.h> #include <linux/kernel.h> #include <linux/mm.h> #include <linux/stddef.h> diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index d249ca10b203..1badc493e62e 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c @@ -14,6 +14,9 @@ #include <linux/errno.h> #include <linux/export.h> #include <linux/sched.h> +#include <linux/sched/debug.h> +#include <linux/sched/task.h> +#include <linux/sched/task_stack.h> #include <linux/kernel.h> #include <linux/mm.h> #include <linux/fs.h> diff --git a/arch/sparc/kernel/ptrace_64.c b/arch/sparc/kernel/ptrace_64.c index 901063c1cf7e..e1d965e90e16 100644 --- a/arch/sparc/kernel/ptrace_64.c +++ b/arch/sparc/kernel/ptrace_64.c @@ -12,6 +12,7 @@ #include <linux/kernel.h> #include <linux/sched.h> +#include <linux/sched/task_stack.h> #include <linux/mm.h> #include <linux/errno.h> #include <linux/export.h> @@ -350,7 +351,7 @@ static int genregs64_set(struct task_struct *target, } if (!ret) { - unsigned long y; + unsigned long y = regs->y; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &y, @@ -1161,3 +1162,39 @@ int regs_query_register_offset(const char *name) return roff->offset; return -EINVAL; } + +/** + * regs_within_kernel_stack() - check the address in the stack + * @regs: pt_regs which contains kernel stack pointer. + * @addr: address which is checked. + * + * regs_within_kernel_stack() checks @addr is within the kernel stack page(s). + * If @addr is within the kernel stack, it returns true. If not, returns false. + */ +static inline int regs_within_kernel_stack(struct pt_regs *regs, + unsigned long addr) +{ + unsigned long ksp = kernel_stack_pointer(regs) + STACK_BIAS; + return ((addr & ~(THREAD_SIZE - 1)) == + (ksp & ~(THREAD_SIZE - 1))); +} + +/** + * regs_get_kernel_stack_nth() - get Nth entry of the stack + * @regs: pt_regs which contains kernel stack pointer. + * @n: stack entry number. + * + * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which + * is specified by @regs. If the @n th entry is NOT in the kernel stack, + * this returns 0. + */ +unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n) +{ + unsigned long ksp = kernel_stack_pointer(regs) + STACK_BIAS; + unsigned long *addr = (unsigned long *)ksp; + addr += n; + if (regs_within_kernel_stack(regs, (unsigned long)addr)) + return *addr; + else + return 0; +} diff --git a/arch/sparc/kernel/rtrap_64.S b/arch/sparc/kernel/rtrap_64.S index 216948ca4382..709a82ebd294 100644 --- a/arch/sparc/kernel/rtrap_64.S +++ b/arch/sparc/kernel/rtrap_64.S @@ -237,6 +237,7 @@ rt_continue: ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1 bne,pt %xcc, user_rtt_fill_32bit wrpr %g1, %cwp ba,a,pt %xcc, user_rtt_fill_64bit + nop user_rtt_fill_fixup_dax: ba,pt %xcc, user_rtt_fill_fixup_common diff --git a/arch/sparc/kernel/setup_32.c b/arch/sparc/kernel/setup_32.c index c4e65cb3280f..6f06058c5ae7 100644 --- a/arch/sparc/kernel/setup_32.c +++ b/arch/sparc/kernel/setup_32.c @@ -82,7 +82,7 @@ static void prom_sync_me(void) "nop\n\t" : : "r" (&trapbase)); prom_printf("PROM SYNC COMMAND...\n"); - show_free_areas(0); + show_free_areas(0, NULL); if (!is_idle_task(current)) { local_irq_enable(); sys_sync(); diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index 0ce347f8e4cc..b3bc0ac757cc 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c @@ -5,7 +5,8 @@ #include <linux/export.h> #include <linux/kernel.h> -#include <linux/sched.h> +#include <linux/sched/mm.h> +#include <linux/sched/hotplug.h> #include <linux/mm.h> #include <linux/pagemap.h> #include <linux/threads.h> @@ -122,7 +123,7 @@ void smp_callin(void) current_thread_info()->new_child = 0; /* Attach to the address space of init_task. */ - atomic_inc(&init_mm.mm_count); + mmgrab(&init_mm); current->active_mm = &init_mm; /* inform the notifiers about the new cpu */ @@ -1443,6 +1444,7 @@ void __irq_entry smp_receive_signal_client(int irq, struct pt_regs *regs) static void stop_this_cpu(void *dummy) { + set_cpu_online(smp_processor_id(), false); prom_stopself(); } @@ -1451,9 +1453,15 @@ void smp_send_stop(void) int cpu; if (tlb_type == hypervisor) { + int this_cpu = smp_processor_id(); +#ifdef CONFIG_SERIAL_SUNHV + sunhv_migrate_hvcons_irq(this_cpu); +#endif for_each_online_cpu(cpu) { - if (cpu == smp_processor_id()) + if (cpu == this_cpu) continue; + + set_cpu_online(cpu, false); #ifdef CONFIG_SUN_LDOMS if (ldom_domaining_enabled) { unsigned long hv_err; diff --git a/arch/sparc/kernel/spiterrs.S b/arch/sparc/kernel/spiterrs.S index 4a73009f66a5..d7e540842809 100644 --- a/arch/sparc/kernel/spiterrs.S +++ b/arch/sparc/kernel/spiterrs.S @@ -86,6 +86,7 @@ __spitfire_cee_trap_continue: rd %pc, %g7 ba,a,pt %xcc, 2f + nop 1: ba,pt %xcc, etrap_irq rd %pc, %g7 diff --git a/arch/sparc/kernel/stacktrace.c b/arch/sparc/kernel/stacktrace.c index e78386a0029f..be4c14cccc05 100644 --- a/arch/sparc/kernel/stacktrace.c +++ b/arch/sparc/kernel/stacktrace.c @@ -1,4 +1,5 @@ #include <linux/sched.h> +#include <linux/sched/debug.h> #include <linux/stacktrace.h> #include <linux/thread_info.h> #include <linux/ftrace.h> diff --git a/arch/sparc/kernel/sun4d_smp.c b/arch/sparc/kernel/sun4d_smp.c index 9d98e5002a09..af93b50e3ce4 100644 --- a/arch/sparc/kernel/sun4d_smp.c +++ b/arch/sparc/kernel/sun4d_smp.c @@ -10,7 +10,7 @@ #include <linux/interrupt.h> #include <linux/profile.h> #include <linux/delay.h> -#include <linux/sched.h> +#include <linux/sched/mm.h> #include <linux/cpu.h> #include <asm/cacheflush.h> @@ -93,7 +93,7 @@ void sun4d_cpu_pre_online(void *arg) show_leds(cpuid); /* Attach to the address space of init_task. */ - atomic_inc(&init_mm.mm_count); + mmgrab(&init_mm); current->active_mm = &init_mm; local_ops->cache_all(); diff --git a/arch/sparc/kernel/sun4m_irq.c b/arch/sparc/kernel/sun4m_irq.c index da737c712fa8..aa84da0b2d30 100644 --- a/arch/sparc/kernel/sun4m_irq.c +++ b/arch/sparc/kernel/sun4m_irq.c @@ -10,6 +10,7 @@ */ #include <linux/slab.h> +#include <linux/sched/debug.h> #include <asm/timer.h> #include <asm/traps.h> diff --git a/arch/sparc/kernel/sun4m_smp.c b/arch/sparc/kernel/sun4m_smp.c index 278c40abce82..5547fcb1d72d 100644 --- a/arch/sparc/kernel/sun4m_smp.c +++ b/arch/sparc/kernel/sun4m_smp.c @@ -8,7 +8,7 @@ #include <linux/interrupt.h> #include <linux/profile.h> #include <linux/delay.h> -#include <linux/sched.h> +#include <linux/sched/mm.h> #include <linux/cpu.h> #include <asm/cacheflush.h> @@ -59,7 +59,7 @@ void sun4m_cpu_pre_online(void *arg) : "memory" /* paranoid */); /* Attach to the address space of init_task. */ - atomic_inc(&init_mm.mm_count); + mmgrab(&init_mm); current->active_mm = &init_mm; while (!cpumask_test_cpu(cpuid, &smp_commenced_mask)) diff --git a/arch/sparc/kernel/sun4v_tlb_miss.S b/arch/sparc/kernel/sun4v_tlb_miss.S index 6179e19bc9b9..c19f352f46c7 100644 --- a/arch/sparc/kernel/sun4v_tlb_miss.S +++ b/arch/sparc/kernel/sun4v_tlb_miss.S @@ -352,6 +352,7 @@ sun4v_mna: call sun4v_do_mna add %sp, PTREGS_OFF, %o0 ba,a,pt %xcc, rtrap + nop /* Privileged Action. */ sun4v_privact: diff --git a/arch/sparc/kernel/sys_sparc_32.c b/arch/sparc/kernel/sys_sparc_32.c index fb7b185ee941..7aecb239626d 100644 --- a/arch/sparc/kernel/sys_sparc_32.c +++ b/arch/sparc/kernel/sys_sparc_32.c @@ -7,7 +7,9 @@ #include <linux/errno.h> #include <linux/types.h> -#include <linux/sched.h> +#include <linux/sched/signal.h> +#include <linux/sched/mm.h> +#include <linux/sched/debug.h> #include <linux/mm.h> #include <linux/fs.h> #include <linux/file.h> diff --git a/arch/sparc/kernel/sys_sparc_64.c b/arch/sparc/kernel/sys_sparc_64.c index 884c70331345..ef4520efc813 100644 --- a/arch/sparc/kernel/sys_sparc_64.c +++ b/arch/sparc/kernel/sys_sparc_64.c @@ -7,7 +7,9 @@ #include <linux/errno.h> #include <linux/types.h> -#include <linux/sched.h> +#include <linux/sched/signal.h> +#include <linux/sched/mm.h> +#include <linux/sched/debug.h> #include <linux/fs.h> #include <linux/file.h> #include <linux/mm.h> diff --git a/arch/sparc/kernel/sysfs.c b/arch/sparc/kernel/sysfs.c index 4808b6d23455..5fd352b759af 100644 --- a/arch/sparc/kernel/sysfs.c +++ b/arch/sparc/kernel/sysfs.c @@ -98,27 +98,7 @@ static struct attribute_group mmu_stat_group = { .name = "mmu_stats", }; -/* XXX convert to rusty's on_one_cpu */ -static unsigned long run_on_cpu(unsigned long cpu, - unsigned long (*func)(unsigned long), - unsigned long arg) -{ - cpumask_t old_affinity; - unsigned long ret; - - cpumask_copy(&old_affinity, tsk_cpus_allowed(current)); - /* should return -EINVAL to userspace */ - if (set_cpus_allowed_ptr(current, cpumask_of(cpu))) - return 0; - - ret = func(arg); - - set_cpus_allowed_ptr(current, &old_affinity); - - return ret; -} - -static unsigned long read_mmustat_enable(unsigned long junk) +static long read_mmustat_enable(void *data __maybe_unused) { unsigned long ra = 0; @@ -127,11 +107,11 @@ static unsigned long read_mmustat_enable(unsigned long junk) return ra != 0; } -static unsigned long write_mmustat_enable(unsigned long val) +static long write_mmustat_enable(void *data) { - unsigned long ra, orig_ra; + unsigned long ra, orig_ra, *val = data; - if (val) + if (*val) ra = __pa(&per_cpu(mmu_stats, smp_processor_id())); else ra = 0UL; @@ -142,7 +122,8 @@ static unsigned long write_mmustat_enable(unsigned long val) static ssize_t show_mmustat_enable(struct device *s, struct device_attribute *attr, char *buf) { - unsigned long val = run_on_cpu(s->id, read_mmustat_enable, 0); + long val = work_on_cpu(s->id, read_mmustat_enable, NULL); + return sprintf(buf, "%lx\n", val); } @@ -150,13 +131,15 @@ static ssize_t store_mmustat_enable(struct device *s, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long val, err; - int ret = sscanf(buf, "%lu", &val); + unsigned long val; + long err; + int ret; + ret = sscanf(buf, "%lu", &val); if (ret != 1) return -EINVAL; - err = run_on_cpu(s->id, write_mmustat_enable, val); + err = work_on_cpu(s->id, write_mmustat_enable, &val); if (err) return -EIO; diff --git a/arch/sparc/kernel/systbls_32.S b/arch/sparc/kernel/systbls_32.S index eac7f0db5c8c..5253e895b81b 100644 --- a/arch/sparc/kernel/systbls_32.S +++ b/arch/sparc/kernel/systbls_32.S @@ -89,3 +89,4 @@ sys_call_table: /*345*/ .long sys_renameat2, sys_seccomp, sys_getrandom, sys_memfd_create, sys_bpf /*350*/ .long sys_execveat, sys_membarrier, sys_userfaultfd, sys_bind, sys_listen /*355*/ .long sys_setsockopt, sys_mlock2, sys_copy_file_range, sys_preadv2, sys_pwritev2 +/*360*/ .long sys_statx diff --git a/arch/sparc/kernel/systbls_64.S b/arch/sparc/kernel/systbls_64.S index b0f17ff2ddba..82339f6be0b2 100644 --- a/arch/sparc/kernel/systbls_64.S +++ b/arch/sparc/kernel/systbls_64.S @@ -90,6 +90,7 @@ sys_call_table32: .word sys32_renameat2, sys_seccomp, sys_getrandom, sys_memfd_create, sys_bpf /*350*/ .word sys32_execveat, sys_membarrier, sys_userfaultfd, sys_bind, sys_listen .word compat_sys_setsockopt, sys_mlock2, sys_copy_file_range, compat_sys_preadv2, compat_sys_pwritev2 +/*360*/ .word sys_statx #endif /* CONFIG_COMPAT */ @@ -171,3 +172,4 @@ sys_call_table: .word sys_renameat2, sys_seccomp, sys_getrandom, sys_memfd_create, sys_bpf /*350*/ .word sys64_execveat, sys_membarrier, sys_userfaultfd, sys_bind, sys_listen .word sys_setsockopt, sys_mlock2, sys_copy_file_range, sys_preadv2, sys_pwritev2 +/*360*/ .word sys_statx diff --git a/arch/sparc/kernel/time_32.c b/arch/sparc/kernel/time_32.c index 244062bdaa56..9f575dfc2e41 100644 --- a/arch/sparc/kernel/time_32.c +++ b/arch/sparc/kernel/time_32.c @@ -228,7 +228,9 @@ void register_percpu_ce(int cpu) ce->mult = div_sc(sparc_config.clock_rate, NSEC_PER_SEC, ce->shift); ce->max_delta_ns = clockevent_delta2ns(sparc_config.clock_rate, ce); + ce->max_delta_ticks = (unsigned long)sparc_config.clock_rate; ce->min_delta_ns = clockevent_delta2ns(100, ce); + ce->min_delta_ticks = 100; clockevents_register_device(ce); } diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c index 12a6d3555cb8..98d05de8da66 100644 --- a/arch/sparc/kernel/time_64.c +++ b/arch/sparc/kernel/time_64.c @@ -796,8 +796,10 @@ void __init time_init(void) sparc64_clockevent.max_delta_ns = clockevent_delta2ns(0x7fffffffffffffffUL, &sparc64_clockevent); + sparc64_clockevent.max_delta_ticks = 0x7fffffffffffffffUL; sparc64_clockevent.min_delta_ns = clockevent_delta2ns(0xF, &sparc64_clockevent); + sparc64_clockevent.min_delta_ticks = 0xF; printk("clockevent: mult[%x] shift[%d]\n", sparc64_clockevent.mult, sparc64_clockevent.shift); diff --git a/arch/sparc/kernel/traps_32.c b/arch/sparc/kernel/traps_32.c index 4f21df7d4f13..466d4aed06c7 100644 --- a/arch/sparc/kernel/traps_32.c +++ b/arch/sparc/kernel/traps_32.c @@ -9,7 +9,9 @@ * I hate traps on the sparc, grrr... */ -#include <linux/sched.h> /* for jiffies */ +#include <linux/sched/mm.h> +#include <linux/sched/debug.h> +#include <linux/mm_types.h> #include <linux/kernel.h> #include <linux/signal.h> #include <linux/smp.h> @@ -448,7 +450,7 @@ void trap_init(void) thread_info_offsets_are_bolixed_pete(); /* Attach to the address space of init_task. */ - atomic_inc(&init_mm.mm_count); + mmgrab(&init_mm); current->active_mm = &init_mm; /* NOTE: Other cpus have this done as they are started diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c index dfc97a47c9a0..196ee5eb4d48 100644 --- a/arch/sparc/kernel/traps_64.c +++ b/arch/sparc/kernel/traps_64.c @@ -9,7 +9,8 @@ */ #include <linux/extable.h> -#include <linux/sched.h> +#include <linux/sched/mm.h> +#include <linux/sched/debug.h> #include <linux/linkage.h> #include <linux/kernel.h> #include <linux/signal.h> @@ -2837,6 +2838,6 @@ void __init trap_init(void) /* Attach to the address space of init_task. On SMP we * do this in smp.c:smp_callin for other cpus. */ - atomic_inc(&init_mm.mm_count); + mmgrab(&init_mm); current->active_mm = &init_mm; } diff --git a/arch/sparc/kernel/tsb.S b/arch/sparc/kernel/tsb.S index d568c8207af7..10689cfd0ad4 100644 --- a/arch/sparc/kernel/tsb.S +++ b/arch/sparc/kernel/tsb.S @@ -117,26 +117,11 @@ tsb_miss_page_table_walk_sun4v_fastpath: /* Valid PTE is now in %g5. */ #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) -661: sethi %uhi(_PAGE_SZALL_4U), %g7 + sethi %uhi(_PAGE_PMD_HUGE), %g7 sllx %g7, 32, %g7 - .section .sun4v_2insn_patch, "ax" - .word 661b - mov _PAGE_SZALL_4V, %g7 - nop - .previous - - and %g5, %g7, %g2 - -661: sethi %uhi(_PAGE_SZHUGE_4U), %g7 - sllx %g7, 32, %g7 - .section .sun4v_2insn_patch, "ax" - .word 661b - mov _PAGE_SZHUGE_4V, %g7 - nop - .previous - cmp %g2, %g7 - bne,pt %xcc, 60f + andcc %g5, %g7, %g0 + be,pt %xcc, 60f nop /* It is a huge page, use huge page TSB entry address we diff --git a/arch/sparc/kernel/unaligned_32.c b/arch/sparc/kernel/unaligned_32.c index d20d4e3fd129..8367dce5f41b 100644 --- a/arch/sparc/kernel/unaligned_32.c +++ b/arch/sparc/kernel/unaligned_32.c @@ -8,7 +8,7 @@ #include <linux/kernel.h> -#include <linux/sched.h> +#include <linux/sched/signal.h> #include <linux/mm.h> #include <asm/ptrace.h> #include <asm/processor.h> diff --git a/arch/sparc/kernel/urtt_fill.S b/arch/sparc/kernel/urtt_fill.S index 5604a2b051d4..364af3250646 100644 --- a/arch/sparc/kernel/urtt_fill.S +++ b/arch/sparc/kernel/urtt_fill.S @@ -92,6 +92,7 @@ user_rtt_fill_fixup_common: call sun4v_data_access_exception nop ba,a,pt %xcc, rtrap + nop 1: call spitfire_data_access_exception nop diff --git a/arch/sparc/kernel/viohs.c b/arch/sparc/kernel/viohs.c index 526fcb5d8ce9..b30b30ab3ddd 100644 --- a/arch/sparc/kernel/viohs.c +++ b/arch/sparc/kernel/viohs.c @@ -8,6 +8,7 @@ #include <linux/string.h> #include <linux/delay.h> #include <linux/sched.h> +#include <linux/sched/clock.h> #include <linux/slab.h> #include <asm/ldc.h> diff --git a/arch/sparc/kernel/visemul.c b/arch/sparc/kernel/visemul.c index c4ac58e483a4..8f35eea2103a 100644 --- a/arch/sparc/kernel/visemul.c +++ b/arch/sparc/kernel/visemul.c @@ -30,7 +30,7 @@ /* 001001011 - two 32-bit merges */ #define FPMERGE_OPF 0x04b -/* 000110001 - 8-by-16-bit partitoned product */ +/* 000110001 - 8-by-16-bit partitioned product */ #define FMUL8x16_OPF 0x031 /* 000110011 - 8-by-16-bit upper alpha partitioned product */ diff --git a/arch/sparc/kernel/winfixup.S b/arch/sparc/kernel/winfixup.S index 855019a8590e..1ee173cc3c39 100644 --- a/arch/sparc/kernel/winfixup.S +++ b/arch/sparc/kernel/winfixup.S @@ -152,6 +152,8 @@ fill_fixup_dax: call sun4v_data_access_exception nop ba,a,pt %xcc, rtrap + nop 1: call spitfire_data_access_exception nop ba,a,pt %xcc, rtrap + nop diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S index 69a439fa2fc1..8aa16ef113f2 100644 --- a/arch/sparc/lib/GENcopy_from_user.S +++ b/arch/sparc/lib/GENcopy_from_user.S @@ -23,7 +23,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S index 9947427ce354..311c8fa5e98e 100644 --- a/arch/sparc/lib/GENcopy_to_user.S +++ b/arch/sparc/lib/GENcopy_to_user.S @@ -27,7 +27,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/GENpatch.S b/arch/sparc/lib/GENpatch.S index fab9e89f16bd..95e2f1f9e477 100644 --- a/arch/sparc/lib/GENpatch.S +++ b/arch/sparc/lib/GENpatch.S @@ -26,8 +26,8 @@ .type generic_patch_copyops,#function generic_patch_copyops: GEN_DO_PATCH(memcpy, GENmemcpy) - GEN_DO_PATCH(___copy_from_user, GENcopy_from_user) - GEN_DO_PATCH(___copy_to_user, GENcopy_to_user) + GEN_DO_PATCH(raw_copy_from_user, GENcopy_from_user) + GEN_DO_PATCH(raw_copy_to_user, GENcopy_to_user) retl nop .size generic_patch_copyops,.-generic_patch_copyops diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S index b79a6998d87c..0d8a018118c2 100644 --- a/arch/sparc/lib/NG2copy_from_user.S +++ b/arch/sparc/lib/NG2copy_from_user.S @@ -36,7 +36,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S index dcec55f254ab..a7a0ea0d8a0b 100644 --- a/arch/sparc/lib/NG2copy_to_user.S +++ b/arch/sparc/lib/NG2copy_to_user.S @@ -45,7 +45,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S index c629dbd121b6..64dcd6cdb606 100644 --- a/arch/sparc/lib/NG2memcpy.S +++ b/arch/sparc/lib/NG2memcpy.S @@ -326,11 +326,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ blu 170f nop ba,a,pt %xcc, 180f + nop 4: /* 32 <= low bits < 48 */ blu 150f nop ba,a,pt %xcc, 160f + nop 5: /* 0 < low bits < 32 */ blu,a 6f cmp %g2, 8 @@ -338,6 +340,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ blu 130f nop ba,a,pt %xcc, 140f + nop 6: /* 0 < low bits < 16 */ bgeu 120f nop @@ -475,6 +478,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ brz,pt %o2, 85f sub %o0, %o1, GLOBAL_SPARE ba,a,pt %XCC, 90f + nop .align 64 75: /* 16 < len <= 64 */ diff --git a/arch/sparc/lib/NG2patch.S b/arch/sparc/lib/NG2patch.S index 28c36f06a6d1..56ccc19adde8 100644 --- a/arch/sparc/lib/NG2patch.S +++ b/arch/sparc/lib/NG2patch.S @@ -26,8 +26,8 @@ .type niagara2_patch_copyops,#function niagara2_patch_copyops: NG_DO_PATCH(memcpy, NG2memcpy) - NG_DO_PATCH(___copy_from_user, NG2copy_from_user) - NG_DO_PATCH(___copy_to_user, NG2copy_to_user) + NG_DO_PATCH(raw_copy_from_user, NG2copy_from_user) + NG_DO_PATCH(raw_copy_to_user, NG2copy_to_user) retl nop .size niagara2_patch_copyops,.-niagara2_patch_copyops diff --git a/arch/sparc/lib/NG4copy_from_user.S b/arch/sparc/lib/NG4copy_from_user.S index 16a286c1a528..5bb506bd61fa 100644 --- a/arch/sparc/lib/NG4copy_from_user.S +++ b/arch/sparc/lib/NG4copy_from_user.S @@ -31,7 +31,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NG4copy_to_user.S b/arch/sparc/lib/NG4copy_to_user.S index 6b0276ffc858..a82d4d45fc1c 100644 --- a/arch/sparc/lib/NG4copy_to_user.S +++ b/arch/sparc/lib/NG4copy_to_user.S @@ -40,7 +40,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NG4memcpy.S b/arch/sparc/lib/NG4memcpy.S index 75bb93b1437f..78ea962edcbe 100644 --- a/arch/sparc/lib/NG4memcpy.S +++ b/arch/sparc/lib/NG4memcpy.S @@ -530,4 +530,5 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ bne,pt %icc, 1b EX_ST(STORE(stb, %g1, %o0 - 0x01), NG4_retl_o2_plus_1) ba,a,pt %icc, .Lexit + nop .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/NG4memset.S b/arch/sparc/lib/NG4memset.S index 41da4bdd95cb..7c0c81f18837 100644 --- a/arch/sparc/lib/NG4memset.S +++ b/arch/sparc/lib/NG4memset.S @@ -102,4 +102,5 @@ NG4bzero: bne,pt %icc, 1b add %o0, 0x30, %o0 ba,a,pt %icc, .Lpostloop + nop .size NG4bzero,.-NG4bzero diff --git a/arch/sparc/lib/NG4patch.S b/arch/sparc/lib/NG4patch.S index a114cbcf2a48..3cc0f8cc95df 100644 --- a/arch/sparc/lib/NG4patch.S +++ b/arch/sparc/lib/NG4patch.S @@ -26,8 +26,8 @@ .type niagara4_patch_copyops,#function niagara4_patch_copyops: NG_DO_PATCH(memcpy, NG4memcpy) - NG_DO_PATCH(___copy_from_user, NG4copy_from_user) - NG_DO_PATCH(___copy_to_user, NG4copy_to_user) + NG_DO_PATCH(raw_copy_from_user, NG4copy_from_user) + NG_DO_PATCH(raw_copy_to_user, NG4copy_to_user) retl nop .size niagara4_patch_copyops,.-niagara4_patch_copyops diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S index 9cd42fcbc781..2333b6f3e824 100644 --- a/arch/sparc/lib/NGcopy_from_user.S +++ b/arch/sparc/lib/NGcopy_from_user.S @@ -25,7 +25,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S index 5c358afd464e..07ba20bc4ea4 100644 --- a/arch/sparc/lib/NGcopy_to_user.S +++ b/arch/sparc/lib/NGcopy_to_user.S @@ -28,7 +28,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NGmemcpy.S b/arch/sparc/lib/NGmemcpy.S index d88c4ed50a00..cd654a719b27 100644 --- a/arch/sparc/lib/NGmemcpy.S +++ b/arch/sparc/lib/NGmemcpy.S @@ -394,6 +394,7 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ brz,pt %i2, 85f sub %o0, %i1, %i3 ba,a,pt %XCC, 90f + nop .align 64 70: /* 16 < len <= 64 */ diff --git a/arch/sparc/lib/NGpatch.S b/arch/sparc/lib/NGpatch.S index 3b0674fc3366..62ccda7e7b38 100644 --- a/arch/sparc/lib/NGpatch.S +++ b/arch/sparc/lib/NGpatch.S @@ -26,8 +26,8 @@ .type niagara_patch_copyops,#function niagara_patch_copyops: NG_DO_PATCH(memcpy, NGmemcpy) - NG_DO_PATCH(___copy_from_user, NGcopy_from_user) - NG_DO_PATCH(___copy_to_user, NGcopy_to_user) + NG_DO_PATCH(raw_copy_from_user, NGcopy_from_user) + NG_DO_PATCH(raw_copy_to_user, NGcopy_to_user) retl nop .size niagara_patch_copyops,.-niagara_patch_copyops diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S index bb6ff73229e3..9a6e68a9bf4a 100644 --- a/arch/sparc/lib/U1copy_from_user.S +++ b/arch/sparc/lib/U1copy_from_user.S @@ -19,7 +19,7 @@ .text; \ .align 4; -#define FUNC_NAME ___copy_from_user +#define FUNC_NAME raw_copy_from_user #define LOAD(type,addr,dest) type##a [addr] %asi, dest #define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_AIUS, dest #define EX_RETVAL(x) 0 @@ -31,7 +31,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop; \ #include "U1memcpy.S" diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S index ed92ce739558..d7b28491eddf 100644 --- a/arch/sparc/lib/U1copy_to_user.S +++ b/arch/sparc/lib/U1copy_to_user.S @@ -19,7 +19,7 @@ .text; \ .align 4; -#define FUNC_NAME ___copy_to_user +#define FUNC_NAME raw_copy_to_user #define STORE(type,src,addr) type##a src, [addr] ASI_AIUS #define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS #define EX_RETVAL(x) 0 @@ -31,7 +31,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop; \ #include "U1memcpy.S" diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S index c4ee858e352a..f48fb87fe9f2 100644 --- a/arch/sparc/lib/U3copy_to_user.S +++ b/arch/sparc/lib/U3copy_to_user.S @@ -31,7 +31,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop; \ #include "U3memcpy.S" diff --git a/arch/sparc/lib/U3patch.S b/arch/sparc/lib/U3patch.S index ecc302619a6e..91cd6539b6e1 100644 --- a/arch/sparc/lib/U3patch.S +++ b/arch/sparc/lib/U3patch.S @@ -26,8 +26,8 @@ .type cheetah_patch_copyops,#function cheetah_patch_copyops: ULTRA3_DO_PATCH(memcpy, U3memcpy) - ULTRA3_DO_PATCH(___copy_from_user, U3copy_from_user) - ULTRA3_DO_PATCH(___copy_to_user, U3copy_to_user) + ULTRA3_DO_PATCH(raw_copy_from_user, U3copy_from_user) + ULTRA3_DO_PATCH(raw_copy_to_user, U3copy_to_user) retl nop .size cheetah_patch_copyops,.-cheetah_patch_copyops diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S index 0252b218de45..1b73bb80aeb0 100644 --- a/arch/sparc/lib/copy_in_user.S +++ b/arch/sparc/lib/copy_in_user.S @@ -44,7 +44,7 @@ __retl_o2_plus_1: * to copy register windows around during thread cloning. */ -ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ +ENTRY(raw_copy_in_user) /* %o0=dst, %o1=src, %o2=len */ cmp %o2, 0 be,pn %XCC, 85f or %o0, %o1, %o3 @@ -105,5 +105,5 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ add %o0, 1, %o0 retl clr %o0 -ENDPROC(___copy_in_user) -EXPORT_SYMBOL(___copy_in_user) +ENDPROC(raw_copy_in_user) +EXPORT_SYMBOL(raw_copy_in_user) diff --git a/arch/sparc/lib/copy_user.S b/arch/sparc/lib/copy_user.S index cea644dc67a6..bc243ee807cc 100644 --- a/arch/sparc/lib/copy_user.S +++ b/arch/sparc/lib/copy_user.S @@ -364,21 +364,7 @@ short_aligned_end: 97: mov %o2, %g3 fixupretl: - sethi %hi(PAGE_OFFSET), %g1 - cmp %o0, %g1 - blu 1f - cmp %o1, %g1 - bgeu 1f - ld [%g6 + TI_PREEMPT], %g1 - cmp %g1, 0 - bne 1f - nop - save %sp, -64, %sp - mov %i0, %o0 - call __bzero - mov %g3, %o1 - restore -1: retl + retl mov %g3, %o0 /* exception routine sets %g2 to (broken_insn - first_insn)>>2 */ diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index 643c149a3151..b84c4dd14954 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -10,6 +10,7 @@ #include <linux/string.h> #include <linux/types.h> #include <linux/sched.h> +#include <linux/sched/debug.h> #include <linux/ptrace.h> #include <linux/mman.h> #include <linux/signal.h> diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c index 988acc8b1b80..7c29d38e6b99 100644 --- a/arch/sparc/mm/hugetlbpage.c +++ b/arch/sparc/mm/hugetlbpage.c @@ -6,6 +6,7 @@ #include <linux/fs.h> #include <linux/mm.h> +#include <linux/sched/mm.h> #include <linux/hugetlb.h> #include <linux/pagemap.h> #include <linux/sysctl.h> @@ -28,6 +29,7 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp, unsigned long pgoff, unsigned long flags) { + struct hstate *h = hstate_file(filp); unsigned long task_size = TASK_SIZE; struct vm_unmapped_area_info info; @@ -38,7 +40,7 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp, info.length = len; info.low_limit = TASK_UNMAPPED_BASE; info.high_limit = min(task_size, VA_EXCLUDE_START); - info.align_mask = PAGE_MASK & ~HPAGE_MASK; + info.align_mask = PAGE_MASK & ~huge_page_mask(h); info.align_offset = 0; addr = vm_unmapped_area(&info); @@ -58,6 +60,7 @@ hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, const unsigned long pgoff, const unsigned long flags) { + struct hstate *h = hstate_file(filp); struct mm_struct *mm = current->mm; unsigned long addr = addr0; struct vm_unmapped_area_info info; @@ -69,7 +72,7 @@ hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, info.length = len; info.low_limit = PAGE_SIZE; info.high_limit = mm->mmap_base; - info.align_mask = PAGE_MASK & ~HPAGE_MASK; + info.align_mask = PAGE_MASK & ~huge_page_mask(h); info.align_offset = 0; addr = vm_unmapped_area(&info); @@ -94,6 +97,7 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { + struct hstate *h = hstate_file(file); struct mm_struct *mm = current->mm; struct vm_area_struct *vma; unsigned long task_size = TASK_SIZE; @@ -101,7 +105,7 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr, if (test_thread_flag(TIF_32BIT)) task_size = STACK_TOP32; - if (len & ~HPAGE_MASK) + if (len & ~huge_page_mask(h)) return -EINVAL; if (len > task_size) return -ENOMEM; @@ -113,7 +117,7 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr, } if (addr) { - addr = ALIGN(addr, HPAGE_SIZE); + addr = ALIGN(addr, huge_page_size(h)); vma = find_vma(mm, addr); if (task_size - len >= addr && (!vma || addr + len <= vma->vm_start)) @@ -127,17 +131,148 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr, pgoff, flags); } +static pte_t sun4u_hugepage_shift_to_tte(pte_t entry, unsigned int shift) +{ + return entry; +} + +static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, unsigned int shift) +{ + unsigned long hugepage_size = _PAGE_SZ4MB_4V; + + pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V; + + switch (shift) { + case HPAGE_2GB_SHIFT: + hugepage_size = _PAGE_SZ2GB_4V; + pte_val(entry) |= _PAGE_PMD_HUGE; + break; + case HPAGE_256MB_SHIFT: + hugepage_size = _PAGE_SZ256MB_4V; + pte_val(entry) |= _PAGE_PMD_HUGE; + break; + case HPAGE_SHIFT: + pte_val(entry) |= _PAGE_PMD_HUGE; + break; + case HPAGE_64K_SHIFT: + hugepage_size = _PAGE_SZ64K_4V; + break; + default: + WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift); + } + + pte_val(entry) = pte_val(entry) | hugepage_size; + return entry; +} + +static pte_t hugepage_shift_to_tte(pte_t entry, unsigned int shift) +{ + if (tlb_type == hypervisor) + return sun4v_hugepage_shift_to_tte(entry, shift); + else + return sun4u_hugepage_shift_to_tte(entry, shift); +} + +pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, + struct page *page, int writeable) +{ + unsigned int shift = huge_page_shift(hstate_vma(vma)); + + return hugepage_shift_to_tte(entry, shift); +} + +static unsigned int sun4v_huge_tte_to_shift(pte_t entry) +{ + unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4V; + unsigned int shift; + + switch (tte_szbits) { + case _PAGE_SZ2GB_4V: + shift = HPAGE_2GB_SHIFT; + break; + case _PAGE_SZ256MB_4V: + shift = HPAGE_256MB_SHIFT; + break; + case _PAGE_SZ4MB_4V: + shift = REAL_HPAGE_SHIFT; + break; + case _PAGE_SZ64K_4V: + shift = HPAGE_64K_SHIFT; + break; + default: + shift = PAGE_SHIFT; + break; + } + return shift; +} + +static unsigned int sun4u_huge_tte_to_shift(pte_t entry) +{ + unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4U; + unsigned int shift; + + switch (tte_szbits) { + case _PAGE_SZ256MB_4U: + shift = HPAGE_256MB_SHIFT; + break; + case _PAGE_SZ4MB_4U: + shift = REAL_HPAGE_SHIFT; + break; + case _PAGE_SZ64K_4U: + shift = HPAGE_64K_SHIFT; + break; + default: + shift = PAGE_SHIFT; + break; + } + return shift; +} + +static unsigned int huge_tte_to_shift(pte_t entry) +{ + unsigned long shift; + + if (tlb_type == hypervisor) + shift = sun4v_huge_tte_to_shift(entry); + else + shift = sun4u_huge_tte_to_shift(entry); + + if (shift == PAGE_SHIFT) + WARN_ONCE(1, "tto_to_shift: invalid hugepage tte=0x%lx\n", + pte_val(entry)); + + return shift; +} + +static unsigned long huge_tte_to_size(pte_t pte) +{ + unsigned long size = 1UL << huge_tte_to_shift(pte); + + if (size == REAL_HPAGE_SIZE) + size = HPAGE_SIZE; + return size; +} + pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) { pgd_t *pgd; pud_t *pud; + pmd_t *pmd; pte_t *pte = NULL; pgd = pgd_offset(mm, addr); pud = pud_alloc(mm, pgd, addr); - if (pud) - pte = (pte_t *)pmd_alloc(mm, pud, addr); + if (pud) { + pmd = pmd_alloc(mm, pud, addr); + if (!pmd) + return NULL; + + if (sz >= PMD_SIZE) + pte = (pte_t *)pmd; + else + pte = pte_alloc_map(mm, pmd, addr); + } return pte; } @@ -146,49 +281,83 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) { pgd_t *pgd; pud_t *pud; + pmd_t *pmd; pte_t *pte = NULL; pgd = pgd_offset(mm, addr); if (!pgd_none(*pgd)) { pud = pud_offset(pgd, addr); - if (!pud_none(*pud)) - pte = (pte_t *)pmd_offset(pud, addr); + if (!pud_none(*pud)) { + pmd = pmd_offset(pud, addr); + if (!pmd_none(*pmd)) { + if (is_hugetlb_pmd(*pmd)) + pte = (pte_t *)pmd; + else + pte = pte_offset_map(pmd, addr); + } + } } + return pte; } void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t entry) { + unsigned int i, nptes, orig_shift, shift; + unsigned long size; pte_t orig; + size = huge_tte_to_size(entry); + shift = size >= HPAGE_SIZE ? PMD_SHIFT : PAGE_SHIFT; + nptes = size >> shift; + if (!pte_present(*ptep) && pte_present(entry)) - mm->context.hugetlb_pte_count++; + mm->context.hugetlb_pte_count += nptes; - addr &= HPAGE_MASK; + addr &= ~(size - 1); orig = *ptep; - *ptep = entry; + orig_shift = pte_none(orig) ? PAGE_SHIFT : huge_tte_to_shift(orig); + + for (i = 0; i < nptes; i++) + ptep[i] = __pte(pte_val(entry) + (i << shift)); - /* Issue TLB flush at REAL_HPAGE_SIZE boundaries */ - maybe_tlb_batch_add(mm, addr, ptep, orig, 0); - maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0); + maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift); + /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */ + if (size == HPAGE_SIZE) + maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0, + orig_shift); } pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { + unsigned int i, nptes, hugepage_shift; + unsigned long size; pte_t entry; entry = *ptep; + size = huge_tte_to_size(entry); + if (size >= HPAGE_SIZE) + nptes = size >> PMD_SHIFT; + else + nptes = size >> PAGE_SHIFT; + + hugepage_shift = pte_none(entry) ? PAGE_SHIFT : + huge_tte_to_shift(entry); + if (pte_present(entry)) - mm->context.hugetlb_pte_count--; + mm->context.hugetlb_pte_count -= nptes; - addr &= HPAGE_MASK; - *ptep = __pte(0UL); + addr &= ~(size - 1); + for (i = 0; i < nptes; i++) + ptep[i] = __pte(0UL); - /* Issue TLB flush at REAL_HPAGE_SIZE boundaries */ - maybe_tlb_batch_add(mm, addr, ptep, entry, 0); - maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0); + maybe_tlb_batch_add(mm, addr, ptep, entry, 0, hugepage_shift); + /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */ + if (size == HPAGE_SIZE) + maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0, + hugepage_shift); return entry; } @@ -292,6 +461,22 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, pgd_t *pgd; unsigned long next; + addr &= PMD_MASK; + if (addr < floor) { + addr += PMD_SIZE; + if (!addr) + return; + } + if (ceiling) { + ceiling &= PMD_MASK; + if (!ceiling) + return; + } + if (end - 1 > ceiling - 1) + end -= PMD_SIZE; + if (addr > end - 1) + return; + pgd = pgd_offset(tlb->mm, addr); do { next = pgd_addr_end(addr, end); diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c index eb8287155279..c6afe98de4d9 100644 --- a/arch/sparc/mm/init_32.c +++ b/arch/sparc/mm/init_32.c @@ -55,17 +55,6 @@ extern unsigned int sparc_ramdisk_size; unsigned long highstart_pfn, highend_pfn; -void show_mem(unsigned int filter) -{ - printk("Mem-info:\n"); - show_free_areas(filter); - printk("Free swap: %6ldkB\n", - get_nr_swap_pages() << (PAGE_SHIFT-10)); - printk("%ld pages of RAM\n", totalram_pages); - printk("%ld free pages\n", nr_free_pages()); -} - - unsigned long last_valid_pfn; unsigned long calc_highpages(void) diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index 5d2f91511c60..0cda653ae007 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c @@ -324,6 +324,54 @@ static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_inde tsb_insert(tsb, tag, tte); } +#ifdef CONFIG_HUGETLB_PAGE +static int __init setup_hugepagesz(char *string) +{ + unsigned long long hugepage_size; + unsigned int hugepage_shift; + unsigned short hv_pgsz_idx; + unsigned int hv_pgsz_mask; + int rc = 0; + + hugepage_size = memparse(string, &string); + hugepage_shift = ilog2(hugepage_size); + + switch (hugepage_shift) { + case HPAGE_2GB_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_2GB; + hv_pgsz_idx = HV_PGSZ_IDX_2GB; + break; + case HPAGE_256MB_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_256MB; + hv_pgsz_idx = HV_PGSZ_IDX_256MB; + break; + case HPAGE_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_4MB; + hv_pgsz_idx = HV_PGSZ_IDX_4MB; + break; + case HPAGE_64K_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_64K; + hv_pgsz_idx = HV_PGSZ_IDX_64K; + break; + default: + hv_pgsz_mask = 0; + } + + if ((hv_pgsz_mask & cpu_pgsz_mask) == 0U) { + pr_warn("hugepagesz=%llu not supported by MMU.\n", + hugepage_size); + goto out; + } + + hugetlb_add_hstate(hugepage_shift - PAGE_SHIFT); + rc = 1; + +out: + return rc; +} +__setup("hugepagesz=", setup_hugepagesz); +#endif /* CONFIG_HUGETLB_PAGE */ + void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) { struct mm_struct *mm; @@ -347,7 +395,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t * #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) if ((mm->context.hugetlb_pte_count || mm->context.thp_pte_count) && - is_hugetlb_pte(pte)) { + is_hugetlb_pmd(__pmd(pte_val(pte)))) { /* We are fabricating 8MB pages using 4MB real hw pages. */ pte_val(pte) |= (address & (1UL << REAL_HPAGE_SHIFT)); __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT, @@ -785,13 +833,23 @@ static void __init find_ramdisk(unsigned long phys_base) struct node_mem_mask { unsigned long mask; - unsigned long val; + unsigned long match; }; static struct node_mem_mask node_masks[MAX_NUMNODES]; static int num_node_masks; #ifdef CONFIG_NEED_MULTIPLE_NODES +struct mdesc_mlgroup { + u64 node; + u64 latency; + u64 match; + u64 mask; +}; + +static struct mdesc_mlgroup *mlgroups; +static int num_mlgroups; + int numa_cpu_lookup_table[NR_CPUS]; cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES]; @@ -802,78 +860,129 @@ struct mdesc_mblock { }; static struct mdesc_mblock *mblocks; static int num_mblocks; -static int find_numa_node_for_addr(unsigned long pa, - struct node_mem_mask *pnode_mask); -static unsigned long __init ra_to_pa(unsigned long addr) +static struct mdesc_mblock * __init addr_to_mblock(unsigned long addr) { + struct mdesc_mblock *m = NULL; int i; for (i = 0; i < num_mblocks; i++) { - struct mdesc_mblock *m = &mblocks[i]; + m = &mblocks[i]; if (addr >= m->base && addr < (m->base + m->size)) { - addr += m->offset; break; } } - return addr; + + return m; } -static int __init find_node(unsigned long addr) +static u64 __init memblock_nid_range_sun4u(u64 start, u64 end, int *nid) { - static bool search_mdesc = true; - static struct node_mem_mask last_mem_mask = { ~0UL, ~0UL }; - static int last_index; - int i; + int prev_nid, new_nid; - addr = ra_to_pa(addr); - for (i = 0; i < num_node_masks; i++) { - struct node_mem_mask *p = &node_masks[i]; + prev_nid = -1; + for ( ; start < end; start += PAGE_SIZE) { + for (new_nid = 0; new_nid < num_node_masks; new_nid++) { + struct node_mem_mask *p = &node_masks[new_nid]; - if ((addr & p->mask) == p->val) - return i; - } - /* The following condition has been observed on LDOM guests because - * node_masks only contains the best latency mask and value. - * LDOM guest's mdesc can contain a single latency group to - * cover multiple address range. Print warning message only if the - * address cannot be found in node_masks nor mdesc. - */ - if ((search_mdesc) && - ((addr & last_mem_mask.mask) != last_mem_mask.val)) { - /* find the available node in the mdesc */ - last_index = find_numa_node_for_addr(addr, &last_mem_mask); - numadbg("find_node: latency group for address 0x%lx is %d\n", - addr, last_index); - if ((last_index < 0) || (last_index >= num_node_masks)) { - /* WARN_ONCE() and use default group 0 */ - WARN_ONCE(1, "find_node: A physical address doesn't match a NUMA node rule. Some physical memory will be owned by node 0."); - search_mdesc = false; - last_index = 0; + if ((start & p->mask) == p->match) { + if (prev_nid == -1) + prev_nid = new_nid; + break; + } } + + if (new_nid == num_node_masks) { + prev_nid = 0; + WARN_ONCE(1, "addr[%Lx] doesn't match a NUMA node rule. Some memory will be owned by node 0.", + start); + break; + } + + if (prev_nid != new_nid) + break; } + *nid = prev_nid; - return last_index; + return start > end ? end : start; } static u64 __init memblock_nid_range(u64 start, u64 end, int *nid) { - *nid = find_node(start); - start += PAGE_SIZE; - while (start < end) { - int n = find_node(start); + u64 ret_end, pa_start, m_mask, m_match, m_end; + struct mdesc_mblock *mblock; + int _nid, i; + + if (tlb_type != hypervisor) + return memblock_nid_range_sun4u(start, end, nid); + + mblock = addr_to_mblock(start); + if (!mblock) { + WARN_ONCE(1, "memblock_nid_range: Can't find mblock addr[%Lx]", + start); + + _nid = 0; + ret_end = end; + goto done; + } + + pa_start = start + mblock->offset; + m_match = 0; + m_mask = 0; - if (n != *nid) + for (_nid = 0; _nid < num_node_masks; _nid++) { + struct node_mem_mask *const m = &node_masks[_nid]; + + if ((pa_start & m->mask) == m->match) { + m_match = m->match; + m_mask = m->mask; break; - start += PAGE_SIZE; + } + } + + if (num_node_masks == _nid) { + /* We could not find NUMA group, so default to 0, but lets + * search for latency group, so we could calculate the correct + * end address that we return + */ + _nid = 0; + + for (i = 0; i < num_mlgroups; i++) { + struct mdesc_mlgroup *const m = &mlgroups[i]; + + if ((pa_start & m->mask) == m->match) { + m_match = m->match; + m_mask = m->mask; + break; + } + } + + if (i == num_mlgroups) { + WARN_ONCE(1, "memblock_nid_range: Can't find latency group addr[%Lx]", + start); + + ret_end = end; + goto done; + } } - if (start > end) - start = end; + /* + * Each latency group has match and mask, and each memory block has an + * offset. An address belongs to a latency group if its address matches + * the following formula: ((addr + offset) & mask) == match + * It is, however, slow to check every single page if it matches a + * particular latency group. As optimization we calculate end value by + * using bit arithmetics. + */ + m_end = m_match + (1ul << __ffs(m_mask)) - mblock->offset; + m_end += pa_start & ~((1ul << fls64(m_mask)) - 1); + ret_end = m_end > end ? end : m_end; - return start; +done: + *nid = _nid; + return ret_end; } #endif @@ -914,7 +1023,8 @@ static void init_node_masks_nonnuma(void) numadbg("Initializing tables for non-numa.\n"); - node_masks[0].mask = node_masks[0].val = 0; + node_masks[0].mask = 0; + node_masks[0].match = 0; num_node_masks = 1; #ifdef CONFIG_NEED_MULTIPLE_NODES @@ -932,15 +1042,6 @@ EXPORT_SYMBOL(numa_cpu_lookup_table); EXPORT_SYMBOL(numa_cpumask_lookup_table); EXPORT_SYMBOL(node_data); -struct mdesc_mlgroup { - u64 node; - u64 latency; - u64 match; - u64 mask; -}; -static struct mdesc_mlgroup *mlgroups; -static int num_mlgroups; - static int scan_pio_for_cfg_handle(struct mdesc_handle *md, u64 pio, u32 cfg_handle) { @@ -1029,6 +1130,10 @@ int of_node_to_nid(struct device_node *dp) static void __init add_node_ranges(void) { struct memblock_region *reg; + unsigned long prev_max; + +memblock_resized: + prev_max = memblock.memory.max; for_each_memblock(memory, reg) { unsigned long size = reg->size; @@ -1048,6 +1153,8 @@ static void __init add_node_ranges(void) memblock_set_node(start, this_end - start, &memblock.memory, nid); + if (memblock.memory.max != prev_max) + goto memblock_resized; start = this_end; } } @@ -1182,41 +1289,6 @@ int __node_distance(int from, int to) return numa_latency[from][to]; } -static int find_numa_node_for_addr(unsigned long pa, - struct node_mem_mask *pnode_mask) -{ - struct mdesc_handle *md = mdesc_grab(); - u64 node, arc; - int i = 0; - - node = mdesc_node_by_name(md, MDESC_NODE_NULL, "latency-groups"); - if (node == MDESC_NODE_NULL) - goto out; - - mdesc_for_each_node_by_name(md, node, "group") { - mdesc_for_each_arc(arc, md, node, MDESC_ARC_TYPE_FWD) { - u64 target = mdesc_arc_target(md, arc); - struct mdesc_mlgroup *m = find_mlgroup(target); - - if (!m) - continue; - if ((pa & m->mask) == m->match) { - if (pnode_mask) { - pnode_mask->mask = m->mask; - pnode_mask->val = m->match; - } - mdesc_release(md); - return i; - } - } - i++; - } - -out: - mdesc_release(md); - return -1; -} - static int __init find_best_numa_node_for_mlgroup(struct mdesc_mlgroup *grp) { int i; @@ -1224,7 +1296,7 @@ static int __init find_best_numa_node_for_mlgroup(struct mdesc_mlgroup *grp) for (i = 0; i < MAX_NUMNODES; i++) { struct node_mem_mask *n = &node_masks[i]; - if ((grp->mask == n->mask) && (grp->match == n->val)) + if ((grp->mask == n->mask) && (grp->match == n->match)) break; } return i; @@ -1279,10 +1351,10 @@ static int __init numa_attach_mlgroup(struct mdesc_handle *md, u64 grp, n = &node_masks[num_node_masks++]; n->mask = candidate->mask; - n->val = candidate->match; + n->match = candidate->match; - numadbg("NUMA NODE[%d]: mask[%lx] val[%lx] (latency[%llx])\n", - index, n->mask, n->val, candidate->latency); + numadbg("NUMA NODE[%d]: mask[%lx] match[%lx] (latency[%llx])\n", + index, n->mask, n->match, candidate->latency); return 0; } @@ -1379,7 +1451,7 @@ static int __init numa_parse_jbus(void) numa_cpu_lookup_table[cpu] = index; cpumask_copy(&numa_cpumask_lookup_table[index], cpumask_of(cpu)); node_masks[index].mask = ~((1UL << 36UL) - 1UL); - node_masks[index].val = cpu << 36UL; + node_masks[index].match = cpu << 36UL; index++; } @@ -1495,7 +1567,7 @@ bool kern_addr_valid(unsigned long addr) if ((long)addr < 0L) { unsigned long pa = __pa(addr); - if ((addr >> max_phys_bits) != 0UL) + if ((pa >> max_phys_bits) != 0UL) return false; return pfn_valid(pa >> PAGE_SHIFT); diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index c7f2a5295b3a..8e76ebba2986 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -54,6 +54,7 @@ enum mbus_module srmmu_modtype; static unsigned int hwbug_bitmask; int vac_cache_size; +EXPORT_SYMBOL(vac_cache_size); int vac_line_size; extern struct resource sparc_iomap; @@ -1444,7 +1445,7 @@ static void poke_viking(void) srmmu_set_mmureg(mreg); } -static struct sparc32_cachetlb_ops viking_ops = { +static struct sparc32_cachetlb_ops viking_ops __ro_after_init = { .cache_all = viking_flush_cache_all, .cache_mm = viking_flush_cache_mm, .cache_page = viking_flush_cache_page, @@ -1475,7 +1476,7 @@ static struct sparc32_cachetlb_ops viking_ops = { * flushes going at once will require SMP locking anyways so there's * no real value in trying any harder than this. */ -static struct sparc32_cachetlb_ops viking_sun4d_smp_ops = { +static struct sparc32_cachetlb_ops viking_sun4d_smp_ops __ro_after_init = { .cache_all = viking_flush_cache_all, .cache_mm = viking_flush_cache_mm, .cache_page = viking_flush_cache_page, @@ -1759,7 +1760,7 @@ static void smp_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr) local_ops->sig_insns(mm, insn_addr); } -static struct sparc32_cachetlb_ops smp_cachetlb_ops = { +static struct sparc32_cachetlb_ops smp_cachetlb_ops __ro_after_init = { .cache_all = smp_flush_cache_all, .cache_mm = smp_flush_cache_mm, .cache_page = smp_flush_cache_page, diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c index c56a195c9071..ee8066c3d96c 100644 --- a/arch/sparc/mm/tlb.c +++ b/arch/sparc/mm/tlb.c @@ -67,7 +67,7 @@ void arch_leave_lazy_mmu_mode(void) } static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr, - bool exec, bool huge) + bool exec, unsigned int hugepage_shift) { struct tlb_batch *tb = &get_cpu_var(tlb_batch); unsigned long nr; @@ -84,19 +84,19 @@ static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr, } if (!tb->active) { - flush_tsb_user_page(mm, vaddr, huge); + flush_tsb_user_page(mm, vaddr, hugepage_shift); global_flush_tlb_page(mm, vaddr); goto out; } if (nr == 0) { tb->mm = mm; - tb->huge = huge; + tb->hugepage_shift = hugepage_shift; } - if (tb->huge != huge) { + if (tb->hugepage_shift != hugepage_shift) { flush_tlb_pending(); - tb->huge = huge; + tb->hugepage_shift = hugepage_shift; nr = 0; } @@ -110,10 +110,9 @@ out: } void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, - pte_t *ptep, pte_t orig, int fullmm) + pte_t *ptep, pte_t orig, int fullmm, + unsigned int hugepage_shift) { - bool huge = is_hugetlb_pte(orig); - if (tlb_type != hypervisor && pte_dirty(orig)) { unsigned long paddr, pfn = pte_pfn(orig); @@ -139,7 +138,7 @@ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, no_cache_flush: if (!fullmm) - tlb_batch_add_one(mm, vaddr, pte_exec(orig), huge); + tlb_batch_add_one(mm, vaddr, pte_exec(orig), hugepage_shift); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -155,7 +154,7 @@ static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr, if (pte_val(*pte) & _PAGE_VALID) { bool exec = pte_exec(*pte); - tlb_batch_add_one(mm, vaddr, exec, false); + tlb_batch_add_one(mm, vaddr, exec, PAGE_SHIFT); } pte++; vaddr += PAGE_SIZE; @@ -210,9 +209,9 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, pte_t orig_pte = __pte(pmd_val(orig)); bool exec = pte_exec(orig_pte); - tlb_batch_add_one(mm, addr, exec, true); + tlb_batch_add_one(mm, addr, exec, REAL_HPAGE_SHIFT); tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec, - true); + REAL_HPAGE_SHIFT); } else { tlb_batch_pmd_scan(mm, addr, orig); } diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c index e20fbbafb0b0..bedf08b22a47 100644 --- a/arch/sparc/mm/tsb.c +++ b/arch/sparc/mm/tsb.c @@ -6,6 +6,8 @@ #include <linux/kernel.h> #include <linux/preempt.h> #include <linux/slab.h> +#include <linux/mm_types.h> + #include <asm/page.h> #include <asm/pgtable.h> #include <asm/mmu_context.h> @@ -86,6 +88,33 @@ static void __flush_tsb_one(struct tlb_batch *tb, unsigned long hash_shift, __flush_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift, nentries); } +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +static void __flush_huge_tsb_one_entry(unsigned long tsb, unsigned long v, + unsigned long hash_shift, + unsigned long nentries, + unsigned int hugepage_shift) +{ + unsigned int hpage_entries; + unsigned int i; + + hpage_entries = 1 << (hugepage_shift - hash_shift); + for (i = 0; i < hpage_entries; i++) + __flush_tsb_one_entry(tsb, v + (i << hash_shift), hash_shift, + nentries); +} + +static void __flush_huge_tsb_one(struct tlb_batch *tb, unsigned long hash_shift, + unsigned long tsb, unsigned long nentries, + unsigned int hugepage_shift) +{ + unsigned long i; + + for (i = 0; i < tb->tlb_nr; i++) + __flush_huge_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift, + nentries, hugepage_shift); +} +#endif + void flush_tsb_user(struct tlb_batch *tb) { struct mm_struct *mm = tb->mm; @@ -93,45 +122,61 @@ void flush_tsb_user(struct tlb_batch *tb) spin_lock_irqsave(&mm->context.lock, flags); - if (!tb->huge) { + if (tb->hugepage_shift < REAL_HPAGE_SHIFT) { base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) base = __pa(base); - __flush_tsb_one(tb, PAGE_SHIFT, base, nentries); + if (tb->hugepage_shift == PAGE_SHIFT) + __flush_tsb_one(tb, PAGE_SHIFT, base, nentries); +#if defined(CONFIG_HUGETLB_PAGE) + else + __flush_huge_tsb_one(tb, PAGE_SHIFT, base, nentries, + tb->hugepage_shift); +#endif } #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - if (tb->huge && mm->context.tsb_block[MM_TSB_HUGE].tsb) { + else if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) base = __pa(base); - __flush_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries); + __flush_huge_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries, + tb->hugepage_shift); } #endif spin_unlock_irqrestore(&mm->context.lock, flags); } -void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, bool huge) +void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, + unsigned int hugepage_shift) { unsigned long nentries, base, flags; spin_lock_irqsave(&mm->context.lock, flags); - if (!huge) { + if (hugepage_shift < REAL_HPAGE_SHIFT) { base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) base = __pa(base); - __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries); + if (hugepage_shift == PAGE_SHIFT) + __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, + nentries); +#if defined(CONFIG_HUGETLB_PAGE) + else + __flush_huge_tsb_one_entry(base, vaddr, PAGE_SHIFT, + nentries, hugepage_shift); +#endif } #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - if (huge && mm->context.tsb_block[MM_TSB_HUGE].tsb) { + else if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) base = __pa(base); - __flush_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT, nentries); + __flush_huge_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT, + nentries, hugepage_shift); } #endif spin_unlock_irqrestore(&mm->context.lock, flags); diff --git a/arch/sparc/net/Makefile b/arch/sparc/net/Makefile index 1306a58ac541..76fa8e95b721 100644 --- a/arch/sparc/net/Makefile +++ b/arch/sparc/net/Makefile @@ -1,4 +1,4 @@ # # Arch-specific network modules # -obj-$(CONFIG_BPF_JIT) += bpf_jit_asm.o bpf_jit_comp.o +obj-$(CONFIG_BPF_JIT) += bpf_jit_asm_$(BITS).o bpf_jit_comp_$(BITS).o diff --git a/arch/sparc/net/bpf_jit.h b/arch/sparc/net/bpf_jit_32.h index 33d6b375ff12..d5c069bff5f9 100644 --- a/arch/sparc/net/bpf_jit.h +++ b/arch/sparc/net/bpf_jit_32.h @@ -39,7 +39,7 @@ #define r_TMP2 G2 #define r_OFF G3 -/* assembly code in arch/sparc/net/bpf_jit_asm.S */ +/* assembly code in arch/sparc/net/bpf_jit_asm_32.S */ extern u32 bpf_jit_load_word[]; extern u32 bpf_jit_load_half[]; extern u32 bpf_jit_load_byte[]; diff --git a/arch/sparc/net/bpf_jit_64.h b/arch/sparc/net/bpf_jit_64.h new file mode 100644 index 000000000000..74abd45796ea --- /dev/null +++ b/arch/sparc/net/bpf_jit_64.h @@ -0,0 +1,66 @@ +#ifndef _BPF_JIT_H +#define _BPF_JIT_H + +#ifndef __ASSEMBLER__ +#define G0 0x00 +#define G1 0x01 +#define G2 0x02 +#define G3 0x03 +#define G6 0x06 +#define G7 0x07 +#define O0 0x08 +#define O1 0x09 +#define O2 0x0a +#define O3 0x0b +#define O4 0x0c +#define O5 0x0d +#define SP 0x0e +#define O7 0x0f +#define L0 0x10 +#define L1 0x11 +#define L2 0x12 +#define L3 0x13 +#define L4 0x14 +#define L5 0x15 +#define L6 0x16 +#define L7 0x17 +#define I0 0x18 +#define I1 0x19 +#define I2 0x1a +#define I3 0x1b +#define I4 0x1c +#define I5 0x1d +#define FP 0x1e +#define I7 0x1f + +#define r_SKB L0 +#define r_HEADLEN L4 +#define r_SKB_DATA L5 +#define r_TMP G1 +#define r_TMP2 G3 + +/* assembly code in arch/sparc/net/bpf_jit_asm_64.S */ +extern u32 bpf_jit_load_word[]; +extern u32 bpf_jit_load_half[]; +extern u32 bpf_jit_load_byte[]; +extern u32 bpf_jit_load_byte_msh[]; +extern u32 bpf_jit_load_word_positive_offset[]; +extern u32 bpf_jit_load_half_positive_offset[]; +extern u32 bpf_jit_load_byte_positive_offset[]; +extern u32 bpf_jit_load_byte_msh_positive_offset[]; +extern u32 bpf_jit_load_word_negative_offset[]; +extern u32 bpf_jit_load_half_negative_offset[]; +extern u32 bpf_jit_load_byte_negative_offset[]; +extern u32 bpf_jit_load_byte_msh_negative_offset[]; + +#else +#define r_RESULT %o0 +#define r_SKB %o0 +#define r_OFF %o1 +#define r_HEADLEN %l4 +#define r_SKB_DATA %l5 +#define r_TMP %g1 +#define r_TMP2 %g3 +#endif + +#endif /* _BPF_JIT_H */ diff --git a/arch/sparc/net/bpf_jit_asm.S b/arch/sparc/net/bpf_jit_asm_32.S index 8c83f4b8eb15..dcc402f5738a 100644 --- a/arch/sparc/net/bpf_jit_asm.S +++ b/arch/sparc/net/bpf_jit_asm_32.S @@ -1,18 +1,11 @@ #include <asm/ptrace.h> -#include "bpf_jit.h" +#include "bpf_jit_32.h" -#ifdef CONFIG_SPARC64 -#define SAVE_SZ 176 -#define SCRATCH_OFF STACK_BIAS + 128 -#define BE_PTR(label) be,pn %xcc, label -#define SIGN_EXTEND(reg) sra reg, 0, reg -#else #define SAVE_SZ 96 #define SCRATCH_OFF 72 #define BE_PTR(label) be label #define SIGN_EXTEND(reg) -#endif #define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */ diff --git a/arch/sparc/net/bpf_jit_asm_64.S b/arch/sparc/net/bpf_jit_asm_64.S new file mode 100644 index 000000000000..3b3f14655f81 --- /dev/null +++ b/arch/sparc/net/bpf_jit_asm_64.S @@ -0,0 +1,161 @@ +#include <asm/ptrace.h> + +#include "bpf_jit_64.h" + +#define SAVE_SZ 176 +#define SCRATCH_OFF STACK_BIAS + 128 +#define BE_PTR(label) be,pn %xcc, label +#define SIGN_EXTEND(reg) sra reg, 0, reg + +#define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */ + + .text + .globl bpf_jit_load_word +bpf_jit_load_word: + cmp r_OFF, 0 + bl bpf_slow_path_word_neg + nop + .globl bpf_jit_load_word_positive_offset +bpf_jit_load_word_positive_offset: + sub r_HEADLEN, r_OFF, r_TMP + cmp r_TMP, 3 + ble bpf_slow_path_word + add r_SKB_DATA, r_OFF, r_TMP + andcc r_TMP, 3, %g0 + bne load_word_unaligned + nop + retl + ld [r_TMP], r_RESULT +load_word_unaligned: + ldub [r_TMP + 0x0], r_OFF + ldub [r_TMP + 0x1], r_TMP2 + sll r_OFF, 8, r_OFF + or r_OFF, r_TMP2, r_OFF + ldub [r_TMP + 0x2], r_TMP2 + sll r_OFF, 8, r_OFF + or r_OFF, r_TMP2, r_OFF + ldub [r_TMP + 0x3], r_TMP2 + sll r_OFF, 8, r_OFF + retl + or r_OFF, r_TMP2, r_RESULT + + .globl bpf_jit_load_half +bpf_jit_load_half: + cmp r_OFF, 0 + bl bpf_slow_path_half_neg + nop + .globl bpf_jit_load_half_positive_offset +bpf_jit_load_half_positive_offset: + sub r_HEADLEN, r_OFF, r_TMP + cmp r_TMP, 1 + ble bpf_slow_path_half + add r_SKB_DATA, r_OFF, r_TMP + andcc r_TMP, 1, %g0 + bne load_half_unaligned + nop + retl + lduh [r_TMP], r_RESULT +load_half_unaligned: + ldub [r_TMP + 0x0], r_OFF + ldub [r_TMP + 0x1], r_TMP2 + sll r_OFF, 8, r_OFF + retl + or r_OFF, r_TMP2, r_RESULT + + .globl bpf_jit_load_byte +bpf_jit_load_byte: + cmp r_OFF, 0 + bl bpf_slow_path_byte_neg + nop + .globl bpf_jit_load_byte_positive_offset +bpf_jit_load_byte_positive_offset: + cmp r_OFF, r_HEADLEN + bge bpf_slow_path_byte + nop + retl + ldub [r_SKB_DATA + r_OFF], r_RESULT + +#define bpf_slow_path_common(LEN) \ + save %sp, -SAVE_SZ, %sp; \ + mov %i0, %o0; \ + mov %i1, %o1; \ + add %fp, SCRATCH_OFF, %o2; \ + call skb_copy_bits; \ + mov (LEN), %o3; \ + cmp %o0, 0; \ + restore; + +bpf_slow_path_word: + bpf_slow_path_common(4) + bl bpf_error + ld [%sp + SCRATCH_OFF], r_RESULT + retl + nop +bpf_slow_path_half: + bpf_slow_path_common(2) + bl bpf_error + lduh [%sp + SCRATCH_OFF], r_RESULT + retl + nop +bpf_slow_path_byte: + bpf_slow_path_common(1) + bl bpf_error + ldub [%sp + SCRATCH_OFF], r_RESULT + retl + nop + +#define bpf_negative_common(LEN) \ + save %sp, -SAVE_SZ, %sp; \ + mov %i0, %o0; \ + mov %i1, %o1; \ + SIGN_EXTEND(%o1); \ + call bpf_internal_load_pointer_neg_helper; \ + mov (LEN), %o2; \ + mov %o0, r_TMP; \ + cmp %o0, 0; \ + BE_PTR(bpf_error); \ + restore; + +bpf_slow_path_word_neg: + sethi %hi(SKF_MAX_NEG_OFF), r_TMP + cmp r_OFF, r_TMP + bl bpf_error + nop + .globl bpf_jit_load_word_negative_offset +bpf_jit_load_word_negative_offset: + bpf_negative_common(4) + andcc r_TMP, 3, %g0 + bne load_word_unaligned + nop + retl + ld [r_TMP], r_RESULT + +bpf_slow_path_half_neg: + sethi %hi(SKF_MAX_NEG_OFF), r_TMP + cmp r_OFF, r_TMP + bl bpf_error + nop + .globl bpf_jit_load_half_negative_offset +bpf_jit_load_half_negative_offset: + bpf_negative_common(2) + andcc r_TMP, 1, %g0 + bne load_half_unaligned + nop + retl + lduh [r_TMP], r_RESULT + +bpf_slow_path_byte_neg: + sethi %hi(SKF_MAX_NEG_OFF), r_TMP + cmp r_OFF, r_TMP + bl bpf_error + nop + .globl bpf_jit_load_byte_negative_offset +bpf_jit_load_byte_negative_offset: + bpf_negative_common(1) + retl + ldub [r_TMP], r_RESULT + +bpf_error: + /* Make the JIT program itself return zero. */ + ret + restore %g0, %g0, %o0 diff --git a/arch/sparc/net/bpf_jit_comp.c b/arch/sparc/net/bpf_jit_comp_32.c index a6d9204a6a0b..d193748548e2 100644 --- a/arch/sparc/net/bpf_jit_comp.c +++ b/arch/sparc/net/bpf_jit_comp_32.c @@ -8,7 +8,7 @@ #include <asm/cacheflush.h> #include <asm/ptrace.h> -#include "bpf_jit.h" +#include "bpf_jit_32.h" int bpf_jit_enable __read_mostly; @@ -17,24 +17,6 @@ static inline bool is_simm13(unsigned int value) return value + 0x1000 < 0x2000; } -static void bpf_flush_icache(void *start_, void *end_) -{ -#ifdef CONFIG_SPARC64 - /* Cheetah's I-cache is fully coherent. */ - if (tlb_type == spitfire) { - unsigned long start = (unsigned long) start_; - unsigned long end = (unsigned long) end_; - - start &= ~7UL; - end = (end + 7UL) & ~7UL; - while (start < end) { - flushi(start); - start += 32; - } - } -#endif -} - #define SEEN_DATAREF 1 /* might call external helpers */ #define SEEN_XREG 2 /* ebx is used */ #define SEEN_MEM 4 /* use mem[] for temporary storage */ @@ -82,11 +64,7 @@ static void bpf_flush_icache(void *start_, void *end_) #define BE (F2(0, 2) | CONDE) #define BNE (F2(0, 2) | CONDNE) -#ifdef CONFIG_SPARC64 -#define BE_PTR (F2(0, 1) | CONDE | (2 << 20)) -#else #define BE_PTR BE -#endif #define SETHI(K, REG) \ (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff)) @@ -116,13 +94,8 @@ static void bpf_flush_icache(void *start_, void *end_) #define LD64 F3(3, 0x0b) #define ST32 F3(3, 0x04) -#ifdef CONFIG_SPARC64 -#define LDPTR LD64 -#define BASE_STACKFRAME 176 -#else #define LDPTR LD32 #define BASE_STACKFRAME 96 -#endif #define LD32I (LD32 | IMMED) #define LD8I (LD8 | IMMED) @@ -234,11 +207,7 @@ do { BUILD_BUG_ON(FIELD_SIZEOF(STRUCT, FIELD) != sizeof(u8)); \ __emit_load8(BASE, STRUCT, FIELD, DEST); \ } while (0) -#ifdef CONFIG_SPARC64 -#define BIAS (STACK_BIAS - 4) -#else #define BIAS (-4) -#endif #define emit_ldmem(OFF, DEST) \ do { *prog++ = LD32I | RS1(SP) | S13(BIAS - (OFF)) | RD(DEST); \ @@ -249,13 +218,8 @@ do { *prog++ = ST32I | RS1(SP) | S13(BIAS - (OFF)) | RD(SRC); \ } while (0) #ifdef CONFIG_SMP -#ifdef CONFIG_SPARC64 -#define emit_load_cpu(REG) \ - emit_load16(G6, struct thread_info, cpu, REG) -#else #define emit_load_cpu(REG) \ emit_load32(G6, struct thread_info, cpu, REG) -#endif #else #define emit_load_cpu(REG) emit_clear(REG) #endif @@ -486,7 +450,6 @@ void bpf_jit_compile(struct bpf_prog *fp) if (K == 1) break; emit_write_y(G0); -#ifdef CONFIG_SPARC32 /* The Sparc v8 architecture requires * three instructions between a %y * register write and the first use. @@ -494,31 +457,21 @@ void bpf_jit_compile(struct bpf_prog *fp) emit_nop(); emit_nop(); emit_nop(); -#endif emit_alu_K(DIV, K); break; case BPF_ALU | BPF_DIV | BPF_X: /* A /= X; */ emit_cmpi(r_X, 0); if (pc_ret0 > 0) { t_offset = addrs[pc_ret0 - 1]; -#ifdef CONFIG_SPARC32 emit_branch(BE, t_offset + 20); -#else - emit_branch(BE, t_offset + 8); -#endif emit_nop(); /* delay slot */ } else { emit_branch_off(BNE, 16); emit_nop(); -#ifdef CONFIG_SPARC32 emit_jump(cleanup_addr + 20); -#else - emit_jump(cleanup_addr + 8); -#endif emit_clear(r_A); } emit_write_y(G0); -#ifdef CONFIG_SPARC32 /* The Sparc v8 architecture requires * three instructions between a %y * register write and the first use. @@ -526,7 +479,6 @@ void bpf_jit_compile(struct bpf_prog *fp) emit_nop(); emit_nop(); emit_nop(); -#endif emit_alu_X(DIV); break; case BPF_ALU | BPF_NEG: @@ -797,7 +749,6 @@ cond_branch: f_offset = addrs[i + filter[i].jf]; bpf_jit_dump(flen, proglen, pass + 1, image); if (image) { - bpf_flush_icache(image, image + proglen); fp->bpf_func = (void *)image; fp->jited = 1; } diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c new file mode 100644 index 000000000000..21de77419f48 --- /dev/null +++ b/arch/sparc/net/bpf_jit_comp_64.c @@ -0,0 +1,1566 @@ +#include <linux/moduleloader.h> +#include <linux/workqueue.h> +#include <linux/netdevice.h> +#include <linux/filter.h> +#include <linux/bpf.h> +#include <linux/cache.h> +#include <linux/if_vlan.h> + +#include <asm/cacheflush.h> +#include <asm/ptrace.h> + +#include "bpf_jit_64.h" + +int bpf_jit_enable __read_mostly; + +static inline bool is_simm13(unsigned int value) +{ + return value + 0x1000 < 0x2000; +} + +static inline bool is_simm10(unsigned int value) +{ + return value + 0x200 < 0x400; +} + +static inline bool is_simm5(unsigned int value) +{ + return value + 0x10 < 0x20; +} + +static inline bool is_sethi(unsigned int value) +{ + return (value & ~0x3fffff) == 0; +} + +static void bpf_flush_icache(void *start_, void *end_) +{ + /* Cheetah's I-cache is fully coherent. */ + if (tlb_type == spitfire) { + unsigned long start = (unsigned long) start_; + unsigned long end = (unsigned long) end_; + + start &= ~7UL; + end = (end + 7UL) & ~7UL; + while (start < end) { + flushi(start); + start += 32; + } + } +} + +#define SEEN_DATAREF 1 /* might call external helpers */ +#define SEEN_XREG 2 /* ebx is used */ +#define SEEN_MEM 4 /* use mem[] for temporary storage */ + +#define S13(X) ((X) & 0x1fff) +#define S5(X) ((X) & 0x1f) +#define IMMED 0x00002000 +#define RD(X) ((X) << 25) +#define RS1(X) ((X) << 14) +#define RS2(X) ((X)) +#define OP(X) ((X) << 30) +#define OP2(X) ((X) << 22) +#define OP3(X) ((X) << 19) +#define COND(X) (((X) & 0xf) << 25) +#define CBCOND(X) (((X) & 0x1f) << 25) +#define F1(X) OP(X) +#define F2(X, Y) (OP(X) | OP2(Y)) +#define F3(X, Y) (OP(X) | OP3(Y)) +#define ASI(X) (((X) & 0xff) << 5) + +#define CONDN COND(0x0) +#define CONDE COND(0x1) +#define CONDLE COND(0x2) +#define CONDL COND(0x3) +#define CONDLEU COND(0x4) +#define CONDCS COND(0x5) +#define CONDNEG COND(0x6) +#define CONDVC COND(0x7) +#define CONDA COND(0x8) +#define CONDNE COND(0x9) +#define CONDG COND(0xa) +#define CONDGE COND(0xb) +#define CONDGU COND(0xc) +#define CONDCC COND(0xd) +#define CONDPOS COND(0xe) +#define CONDVS COND(0xf) + +#define CONDGEU CONDCC +#define CONDLU CONDCS + +#define WDISP22(X) (((X) >> 2) & 0x3fffff) +#define WDISP19(X) (((X) >> 2) & 0x7ffff) + +/* The 10-bit branch displacement for CBCOND is split into two fields */ +static u32 WDISP10(u32 off) +{ + u32 ret = ((off >> 2) & 0xff) << 5; + + ret |= ((off >> (2 + 8)) & 0x03) << 19; + + return ret; +} + +#define CBCONDE CBCOND(0x09) +#define CBCONDLE CBCOND(0x0a) +#define CBCONDL CBCOND(0x0b) +#define CBCONDLEU CBCOND(0x0c) +#define CBCONDCS CBCOND(0x0d) +#define CBCONDN CBCOND(0x0e) +#define CBCONDVS CBCOND(0x0f) +#define CBCONDNE CBCOND(0x19) +#define CBCONDG CBCOND(0x1a) +#define CBCONDGE CBCOND(0x1b) +#define CBCONDGU CBCOND(0x1c) +#define CBCONDCC CBCOND(0x1d) +#define CBCONDPOS CBCOND(0x1e) +#define CBCONDVC CBCOND(0x1f) + +#define CBCONDGEU CBCONDCC +#define CBCONDLU CBCONDCS + +#define ANNUL (1 << 29) +#define XCC (1 << 21) + +#define BRANCH (F2(0, 1) | XCC) +#define CBCOND_OP (F2(0, 3) | XCC) + +#define BA (BRANCH | CONDA) +#define BG (BRANCH | CONDG) +#define BGU (BRANCH | CONDGU) +#define BLEU (BRANCH | CONDLEU) +#define BGE (BRANCH | CONDGE) +#define BGEU (BRANCH | CONDGEU) +#define BLU (BRANCH | CONDLU) +#define BE (BRANCH | CONDE) +#define BNE (BRANCH | CONDNE) + +#define SETHI(K, REG) \ + (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff)) +#define OR_LO(K, REG) \ + (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG)) + +#define ADD F3(2, 0x00) +#define AND F3(2, 0x01) +#define ANDCC F3(2, 0x11) +#define OR F3(2, 0x02) +#define XOR F3(2, 0x03) +#define SUB F3(2, 0x04) +#define SUBCC F3(2, 0x14) +#define MUL F3(2, 0x0a) +#define MULX F3(2, 0x09) +#define UDIVX F3(2, 0x0d) +#define DIV F3(2, 0x0e) +#define SLL F3(2, 0x25) +#define SLLX (F3(2, 0x25)|(1<<12)) +#define SRA F3(2, 0x27) +#define SRAX (F3(2, 0x27)|(1<<12)) +#define SRL F3(2, 0x26) +#define SRLX (F3(2, 0x26)|(1<<12)) +#define JMPL F3(2, 0x38) +#define SAVE F3(2, 0x3c) +#define RESTORE F3(2, 0x3d) +#define CALL F1(1) +#define BR F2(0, 0x01) +#define RD_Y F3(2, 0x28) +#define WR_Y F3(2, 0x30) + +#define LD32 F3(3, 0x00) +#define LD8 F3(3, 0x01) +#define LD16 F3(3, 0x02) +#define LD64 F3(3, 0x0b) +#define LD64A F3(3, 0x1b) +#define ST8 F3(3, 0x05) +#define ST16 F3(3, 0x06) +#define ST32 F3(3, 0x04) +#define ST64 F3(3, 0x0e) + +#define CAS F3(3, 0x3c) +#define CASX F3(3, 0x3e) + +#define LDPTR LD64 +#define BASE_STACKFRAME 176 + +#define LD32I (LD32 | IMMED) +#define LD8I (LD8 | IMMED) +#define LD16I (LD16 | IMMED) +#define LD64I (LD64 | IMMED) +#define LDPTRI (LDPTR | IMMED) +#define ST32I (ST32 | IMMED) + +struct jit_ctx { + struct bpf_prog *prog; + unsigned int *offset; + int idx; + int epilogue_offset; + bool tmp_1_used; + bool tmp_2_used; + bool tmp_3_used; + bool saw_ld_abs_ind; + bool saw_frame_pointer; + bool saw_call; + bool saw_tail_call; + u32 *image; +}; + +#define TMP_REG_1 (MAX_BPF_JIT_REG + 0) +#define TMP_REG_2 (MAX_BPF_JIT_REG + 1) +#define SKB_HLEN_REG (MAX_BPF_JIT_REG + 2) +#define SKB_DATA_REG (MAX_BPF_JIT_REG + 3) +#define TMP_REG_3 (MAX_BPF_JIT_REG + 4) + +/* Map BPF registers to SPARC registers */ +static const int bpf2sparc[] = { + /* return value from in-kernel function, and exit value from eBPF */ + [BPF_REG_0] = O5, + + /* arguments from eBPF program to in-kernel function */ + [BPF_REG_1] = O0, + [BPF_REG_2] = O1, + [BPF_REG_3] = O2, + [BPF_REG_4] = O3, + [BPF_REG_5] = O4, + + /* callee saved registers that in-kernel function will preserve */ + [BPF_REG_6] = L0, + [BPF_REG_7] = L1, + [BPF_REG_8] = L2, + [BPF_REG_9] = L3, + + /* read-only frame pointer to access stack */ + [BPF_REG_FP] = L6, + + [BPF_REG_AX] = G7, + + /* temporary register for internal BPF JIT */ + [TMP_REG_1] = G1, + [TMP_REG_2] = G2, + [TMP_REG_3] = G3, + + [SKB_HLEN_REG] = L4, + [SKB_DATA_REG] = L5, +}; + +static void emit(const u32 insn, struct jit_ctx *ctx) +{ + if (ctx->image != NULL) + ctx->image[ctx->idx] = insn; + + ctx->idx++; +} + +static void emit_call(u32 *func, struct jit_ctx *ctx) +{ + if (ctx->image != NULL) { + void *here = &ctx->image[ctx->idx]; + unsigned int off; + + off = (void *)func - here; + ctx->image[ctx->idx] = CALL | ((off >> 2) & 0x3fffffff); + } + ctx->idx++; +} + +static void emit_nop(struct jit_ctx *ctx) +{ + emit(SETHI(0, G0), ctx); +} + +static void emit_reg_move(u32 from, u32 to, struct jit_ctx *ctx) +{ + emit(OR | RS1(G0) | RS2(from) | RD(to), ctx); +} + +/* Emit 32-bit constant, zero extended. */ +static void emit_set_const(s32 K, u32 reg, struct jit_ctx *ctx) +{ + emit(SETHI(K, reg), ctx); + emit(OR_LO(K, reg), ctx); +} + +/* Emit 32-bit constant, sign extended. */ +static void emit_set_const_sext(s32 K, u32 reg, struct jit_ctx *ctx) +{ + if (K >= 0) { + emit(SETHI(K, reg), ctx); + emit(OR_LO(K, reg), ctx); + } else { + u32 hbits = ~(u32) K; + u32 lbits = -0x400 | (u32) K; + + emit(SETHI(hbits, reg), ctx); + emit(XOR | IMMED | RS1(reg) | S13(lbits) | RD(reg), ctx); + } +} + +static void emit_alu(u32 opcode, u32 src, u32 dst, struct jit_ctx *ctx) +{ + emit(opcode | RS1(dst) | RS2(src) | RD(dst), ctx); +} + +static void emit_alu3(u32 opcode, u32 a, u32 b, u32 c, struct jit_ctx *ctx) +{ + emit(opcode | RS1(a) | RS2(b) | RD(c), ctx); +} + +static void emit_alu_K(unsigned int opcode, unsigned int dst, unsigned int imm, + struct jit_ctx *ctx) +{ + bool small_immed = is_simm13(imm); + unsigned int insn = opcode; + + insn |= RS1(dst) | RD(dst); + if (small_immed) { + emit(insn | IMMED | S13(imm), ctx); + } else { + unsigned int tmp = bpf2sparc[TMP_REG_1]; + + ctx->tmp_1_used = true; + + emit_set_const_sext(imm, tmp, ctx); + emit(insn | RS2(tmp), ctx); + } +} + +static void emit_alu3_K(unsigned int opcode, unsigned int src, unsigned int imm, + unsigned int dst, struct jit_ctx *ctx) +{ + bool small_immed = is_simm13(imm); + unsigned int insn = opcode; + + insn |= RS1(src) | RD(dst); + if (small_immed) { + emit(insn | IMMED | S13(imm), ctx); + } else { + unsigned int tmp = bpf2sparc[TMP_REG_1]; + + ctx->tmp_1_used = true; + + emit_set_const_sext(imm, tmp, ctx); + emit(insn | RS2(tmp), ctx); + } +} + +static void emit_loadimm32(s32 K, unsigned int dest, struct jit_ctx *ctx) +{ + if (K >= 0 && is_simm13(K)) { + /* or %g0, K, DEST */ + emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx); + } else { + emit_set_const(K, dest, ctx); + } +} + +static void emit_loadimm(s32 K, unsigned int dest, struct jit_ctx *ctx) +{ + if (is_simm13(K)) { + /* or %g0, K, DEST */ + emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx); + } else { + emit_set_const(K, dest, ctx); + } +} + +static void emit_loadimm_sext(s32 K, unsigned int dest, struct jit_ctx *ctx) +{ + if (is_simm13(K)) { + /* or %g0, K, DEST */ + emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx); + } else { + emit_set_const_sext(K, dest, ctx); + } +} + +static void analyze_64bit_constant(u32 high_bits, u32 low_bits, + int *hbsp, int *lbsp, int *abbasp) +{ + int lowest_bit_set, highest_bit_set, all_bits_between_are_set; + int i; + + lowest_bit_set = highest_bit_set = -1; + i = 0; + do { + if ((lowest_bit_set == -1) && ((low_bits >> i) & 1)) + lowest_bit_set = i; + if ((highest_bit_set == -1) && ((high_bits >> (32 - i - 1)) & 1)) + highest_bit_set = (64 - i - 1); + } while (++i < 32 && (highest_bit_set == -1 || + lowest_bit_set == -1)); + if (i == 32) { + i = 0; + do { + if (lowest_bit_set == -1 && ((high_bits >> i) & 1)) + lowest_bit_set = i + 32; + if (highest_bit_set == -1 && + ((low_bits >> (32 - i - 1)) & 1)) + highest_bit_set = 32 - i - 1; + } while (++i < 32 && (highest_bit_set == -1 || + lowest_bit_set == -1)); + } + + all_bits_between_are_set = 1; + for (i = lowest_bit_set; i <= highest_bit_set; i++) { + if (i < 32) { + if ((low_bits & (1 << i)) != 0) + continue; + } else { + if ((high_bits & (1 << (i - 32))) != 0) + continue; + } + all_bits_between_are_set = 0; + break; + } + *hbsp = highest_bit_set; + *lbsp = lowest_bit_set; + *abbasp = all_bits_between_are_set; +} + +static unsigned long create_simple_focus_bits(unsigned long high_bits, + unsigned long low_bits, + int lowest_bit_set, int shift) +{ + long hi, lo; + + if (lowest_bit_set < 32) { + lo = (low_bits >> lowest_bit_set) << shift; + hi = ((high_bits << (32 - lowest_bit_set)) << shift); + } else { + lo = 0; + hi = ((high_bits >> (lowest_bit_set - 32)) << shift); + } + return hi | lo; +} + +static bool const64_is_2insns(unsigned long high_bits, + unsigned long low_bits) +{ + int highest_bit_set, lowest_bit_set, all_bits_between_are_set; + + if (high_bits == 0 || high_bits == 0xffffffff) + return true; + + analyze_64bit_constant(high_bits, low_bits, + &highest_bit_set, &lowest_bit_set, + &all_bits_between_are_set); + + if ((highest_bit_set == 63 || lowest_bit_set == 0) && + all_bits_between_are_set != 0) + return true; + + if (highest_bit_set - lowest_bit_set < 21) + return true; + + return false; +} + +static void sparc_emit_set_const64_quick2(unsigned long high_bits, + unsigned long low_imm, + unsigned int dest, + int shift_count, struct jit_ctx *ctx) +{ + emit_loadimm32(high_bits, dest, ctx); + + /* Now shift it up into place. */ + emit_alu_K(SLLX, dest, shift_count, ctx); + + /* If there is a low immediate part piece, finish up by + * putting that in as well. + */ + if (low_imm != 0) + emit(OR | IMMED | RS1(dest) | S13(low_imm) | RD(dest), ctx); +} + +static void emit_loadimm64(u64 K, unsigned int dest, struct jit_ctx *ctx) +{ + int all_bits_between_are_set, lowest_bit_set, highest_bit_set; + unsigned int tmp = bpf2sparc[TMP_REG_1]; + u32 low_bits = (K & 0xffffffff); + u32 high_bits = (K >> 32); + + /* These two tests also take care of all of the one + * instruction cases. + */ + if (high_bits == 0xffffffff && (low_bits & 0x80000000)) + return emit_loadimm_sext(K, dest, ctx); + if (high_bits == 0x00000000) + return emit_loadimm32(K, dest, ctx); + + analyze_64bit_constant(high_bits, low_bits, &highest_bit_set, + &lowest_bit_set, &all_bits_between_are_set); + + /* 1) mov -1, %reg + * sllx %reg, shift, %reg + * 2) mov -1, %reg + * srlx %reg, shift, %reg + * 3) mov some_small_const, %reg + * sllx %reg, shift, %reg + */ + if (((highest_bit_set == 63 || lowest_bit_set == 0) && + all_bits_between_are_set != 0) || + ((highest_bit_set - lowest_bit_set) < 12)) { + int shift = lowest_bit_set; + long the_const = -1; + + if ((highest_bit_set != 63 && lowest_bit_set != 0) || + all_bits_between_are_set == 0) { + the_const = + create_simple_focus_bits(high_bits, low_bits, + lowest_bit_set, 0); + } else if (lowest_bit_set == 0) + shift = -(63 - highest_bit_set); + + emit(OR | IMMED | RS1(G0) | S13(the_const) | RD(dest), ctx); + if (shift > 0) + emit_alu_K(SLLX, dest, shift, ctx); + else if (shift < 0) + emit_alu_K(SRLX, dest, -shift, ctx); + + return; + } + + /* Now a range of 22 or less bits set somewhere. + * 1) sethi %hi(focus_bits), %reg + * sllx %reg, shift, %reg + * 2) sethi %hi(focus_bits), %reg + * srlx %reg, shift, %reg + */ + if ((highest_bit_set - lowest_bit_set) < 21) { + unsigned long focus_bits = + create_simple_focus_bits(high_bits, low_bits, + lowest_bit_set, 10); + + emit(SETHI(focus_bits, dest), ctx); + + /* If lowest_bit_set == 10 then a sethi alone could + * have done it. + */ + if (lowest_bit_set < 10) + emit_alu_K(SRLX, dest, 10 - lowest_bit_set, ctx); + else if (lowest_bit_set > 10) + emit_alu_K(SLLX, dest, lowest_bit_set - 10, ctx); + return; + } + + /* Ok, now 3 instruction sequences. */ + if (low_bits == 0) { + emit_loadimm32(high_bits, dest, ctx); + emit_alu_K(SLLX, dest, 32, ctx); + return; + } + + /* We may be able to do something quick + * when the constant is negated, so try that. + */ + if (const64_is_2insns((~high_bits) & 0xffffffff, + (~low_bits) & 0xfffffc00)) { + /* NOTE: The trailing bits get XOR'd so we need the + * non-negated bits, not the negated ones. + */ + unsigned long trailing_bits = low_bits & 0x3ff; + + if ((((~high_bits) & 0xffffffff) == 0 && + ((~low_bits) & 0x80000000) == 0) || + (((~high_bits) & 0xffffffff) == 0xffffffff && + ((~low_bits) & 0x80000000) != 0)) { + unsigned long fast_int = (~low_bits & 0xffffffff); + + if ((is_sethi(fast_int) && + (~high_bits & 0xffffffff) == 0)) { + emit(SETHI(fast_int, dest), ctx); + } else if (is_simm13(fast_int)) { + emit(OR | IMMED | RS1(G0) | S13(fast_int) | RD(dest), ctx); + } else { + emit_loadimm64(fast_int, dest, ctx); + } + } else { + u64 n = ((~low_bits) & 0xfffffc00) | + (((unsigned long)((~high_bits) & 0xffffffff))<<32); + emit_loadimm64(n, dest, ctx); + } + + low_bits = -0x400 | trailing_bits; + + emit(XOR | IMMED | RS1(dest) | S13(low_bits) | RD(dest), ctx); + return; + } + + /* 1) sethi %hi(xxx), %reg + * or %reg, %lo(xxx), %reg + * sllx %reg, yyy, %reg + */ + if ((highest_bit_set - lowest_bit_set) < 32) { + unsigned long focus_bits = + create_simple_focus_bits(high_bits, low_bits, + lowest_bit_set, 0); + + /* So what we know is that the set bits straddle the + * middle of the 64-bit word. + */ + sparc_emit_set_const64_quick2(focus_bits, 0, dest, + lowest_bit_set, ctx); + return; + } + + /* 1) sethi %hi(high_bits), %reg + * or %reg, %lo(high_bits), %reg + * sllx %reg, 32, %reg + * or %reg, low_bits, %reg + */ + if (is_simm13(low_bits) && ((int)low_bits > 0)) { + sparc_emit_set_const64_quick2(high_bits, low_bits, + dest, 32, ctx); + return; + } + + /* Oh well, we tried... Do a full 64-bit decomposition. */ + ctx->tmp_1_used = true; + + emit_loadimm32(high_bits, tmp, ctx); + emit_loadimm32(low_bits, dest, ctx); + emit_alu_K(SLLX, tmp, 32, ctx); + emit(OR | RS1(dest) | RS2(tmp) | RD(dest), ctx); +} + +static void emit_branch(unsigned int br_opc, unsigned int from_idx, unsigned int to_idx, + struct jit_ctx *ctx) +{ + unsigned int off = to_idx - from_idx; + + if (br_opc & XCC) + emit(br_opc | WDISP19(off << 2), ctx); + else + emit(br_opc | WDISP22(off << 2), ctx); +} + +static void emit_cbcond(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx, + const u8 dst, const u8 src, struct jit_ctx *ctx) +{ + unsigned int off = to_idx - from_idx; + + emit(cb_opc | WDISP10(off << 2) | RS1(dst) | RS2(src), ctx); +} + +static void emit_cbcondi(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx, + const u8 dst, s32 imm, struct jit_ctx *ctx) +{ + unsigned int off = to_idx - from_idx; + + emit(cb_opc | IMMED | WDISP10(off << 2) | RS1(dst) | S5(imm), ctx); +} + +#define emit_read_y(REG, CTX) emit(RD_Y | RD(REG), CTX) +#define emit_write_y(REG, CTX) emit(WR_Y | IMMED | RS1(REG) | S13(0), CTX) + +#define emit_cmp(R1, R2, CTX) \ + emit(SUBCC | RS1(R1) | RS2(R2) | RD(G0), CTX) + +#define emit_cmpi(R1, IMM, CTX) \ + emit(SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX) + +#define emit_btst(R1, R2, CTX) \ + emit(ANDCC | RS1(R1) | RS2(R2) | RD(G0), CTX) + +#define emit_btsti(R1, IMM, CTX) \ + emit(ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX) + +static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src, + const s32 imm, bool is_imm, int branch_dst, + struct jit_ctx *ctx) +{ + bool use_cbcond = (sparc64_elf_hwcap & AV_SPARC_CBCOND) != 0; + const u8 tmp = bpf2sparc[TMP_REG_1]; + + branch_dst = ctx->offset[branch_dst]; + + if (!is_simm10(branch_dst - ctx->idx) || + BPF_OP(code) == BPF_JSET) + use_cbcond = false; + + if (is_imm) { + bool fits = true; + + if (use_cbcond) { + if (!is_simm5(imm)) + fits = false; + } else if (!is_simm13(imm)) { + fits = false; + } + if (!fits) { + ctx->tmp_1_used = true; + emit_loadimm_sext(imm, tmp, ctx); + src = tmp; + is_imm = false; + } + } + + if (!use_cbcond) { + u32 br_opcode; + + if (BPF_OP(code) == BPF_JSET) { + if (is_imm) + emit_btsti(dst, imm, ctx); + else + emit_btst(dst, src, ctx); + } else { + if (is_imm) + emit_cmpi(dst, imm, ctx); + else + emit_cmp(dst, src, ctx); + } + switch (BPF_OP(code)) { + case BPF_JEQ: + br_opcode = BE; + break; + case BPF_JGT: + br_opcode = BGU; + break; + case BPF_JGE: + br_opcode = BGEU; + break; + case BPF_JSET: + case BPF_JNE: + br_opcode = BNE; + break; + case BPF_JSGT: + br_opcode = BG; + break; + case BPF_JSGE: + br_opcode = BGE; + break; + default: + /* Make sure we dont leak kernel information to the + * user. + */ + return -EFAULT; + } + emit_branch(br_opcode, ctx->idx, branch_dst, ctx); + emit_nop(ctx); + } else { + u32 cbcond_opcode; + + switch (BPF_OP(code)) { + case BPF_JEQ: + cbcond_opcode = CBCONDE; + break; + case BPF_JGT: + cbcond_opcode = CBCONDGU; + break; + case BPF_JGE: + cbcond_opcode = CBCONDGEU; + break; + case BPF_JNE: + cbcond_opcode = CBCONDNE; + break; + case BPF_JSGT: + cbcond_opcode = CBCONDG; + break; + case BPF_JSGE: + cbcond_opcode = CBCONDGE; + break; + default: + /* Make sure we dont leak kernel information to the + * user. + */ + return -EFAULT; + } + cbcond_opcode |= CBCOND_OP; + if (is_imm) + emit_cbcondi(cbcond_opcode, ctx->idx, branch_dst, + dst, imm, ctx); + else + emit_cbcond(cbcond_opcode, ctx->idx, branch_dst, + dst, src, ctx); + } + return 0; +} + +static void load_skb_regs(struct jit_ctx *ctx, u8 r_skb) +{ + const u8 r_headlen = bpf2sparc[SKB_HLEN_REG]; + const u8 r_data = bpf2sparc[SKB_DATA_REG]; + const u8 r_tmp = bpf2sparc[TMP_REG_1]; + unsigned int off; + + off = offsetof(struct sk_buff, len); + emit(LD32I | RS1(r_skb) | S13(off) | RD(r_headlen), ctx); + + off = offsetof(struct sk_buff, data_len); + emit(LD32I | RS1(r_skb) | S13(off) | RD(r_tmp), ctx); + + emit(SUB | RS1(r_headlen) | RS2(r_tmp) | RD(r_headlen), ctx); + + off = offsetof(struct sk_buff, data); + emit(LDPTRI | RS1(r_skb) | S13(off) | RD(r_data), ctx); +} + +/* Just skip the save instruction and the ctx register move. */ +#define BPF_TAILCALL_PROLOGUE_SKIP 16 +#define BPF_TAILCALL_CNT_SP_OFF (STACK_BIAS + 128) + +static void build_prologue(struct jit_ctx *ctx) +{ + s32 stack_needed = BASE_STACKFRAME; + + if (ctx->saw_frame_pointer || ctx->saw_tail_call) + stack_needed += MAX_BPF_STACK; + + if (ctx->saw_tail_call) + stack_needed += 8; + + /* save %sp, -176, %sp */ + emit(SAVE | IMMED | RS1(SP) | S13(-stack_needed) | RD(SP), ctx); + + /* tail_call_cnt = 0 */ + if (ctx->saw_tail_call) { + u32 off = BPF_TAILCALL_CNT_SP_OFF; + + emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(G0), ctx); + } else { + emit_nop(ctx); + } + if (ctx->saw_frame_pointer) { + const u8 vfp = bpf2sparc[BPF_REG_FP]; + + emit(ADD | IMMED | RS1(FP) | S13(STACK_BIAS) | RD(vfp), ctx); + } + + emit_reg_move(I0, O0, ctx); + /* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */ + + if (ctx->saw_ld_abs_ind) + load_skb_regs(ctx, bpf2sparc[BPF_REG_1]); +} + +static void build_epilogue(struct jit_ctx *ctx) +{ + ctx->epilogue_offset = ctx->idx; + + /* ret (jmpl %i7 + 8, %g0) */ + emit(JMPL | IMMED | RS1(I7) | S13(8) | RD(G0), ctx); + + /* restore %i5, %g0, %o0 */ + emit(RESTORE | RS1(bpf2sparc[BPF_REG_0]) | RS2(G0) | RD(O0), ctx); +} + +static void emit_tail_call(struct jit_ctx *ctx) +{ + const u8 bpf_array = bpf2sparc[BPF_REG_2]; + const u8 bpf_index = bpf2sparc[BPF_REG_3]; + const u8 tmp = bpf2sparc[TMP_REG_1]; + u32 off; + + ctx->saw_tail_call = true; + + off = offsetof(struct bpf_array, map.max_entries); + emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx); + emit_cmp(bpf_index, tmp, ctx); +#define OFFSET1 17 + emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET1, ctx); + emit_nop(ctx); + + off = BPF_TAILCALL_CNT_SP_OFF; + emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx); + emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx); +#define OFFSET2 13 + emit_branch(BGU, ctx->idx, ctx->idx + OFFSET2, ctx); + emit_nop(ctx); + + emit_alu_K(ADD, tmp, 1, ctx); + off = BPF_TAILCALL_CNT_SP_OFF; + emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx); + + emit_alu3_K(SLL, bpf_index, 3, tmp, ctx); + emit_alu(ADD, bpf_array, tmp, ctx); + off = offsetof(struct bpf_array, ptrs); + emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx); + + emit_cmpi(tmp, 0, ctx); +#define OFFSET3 5 + emit_branch(BE, ctx->idx, ctx->idx + OFFSET3, ctx); + emit_nop(ctx); + + off = offsetof(struct bpf_prog, bpf_func); + emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx); + + off = BPF_TAILCALL_PROLOGUE_SKIP; + emit(JMPL | IMMED | RS1(tmp) | S13(off) | RD(G0), ctx); + emit_nop(ctx); +} + +static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) +{ + const u8 code = insn->code; + const u8 dst = bpf2sparc[insn->dst_reg]; + const u8 src = bpf2sparc[insn->src_reg]; + const int i = insn - ctx->prog->insnsi; + const s16 off = insn->off; + const s32 imm = insn->imm; + u32 *func; + + if (insn->src_reg == BPF_REG_FP) + ctx->saw_frame_pointer = true; + + switch (code) { + /* dst = src */ + case BPF_ALU | BPF_MOV | BPF_X: + emit_alu3_K(SRL, src, 0, dst, ctx); + break; + case BPF_ALU64 | BPF_MOV | BPF_X: + emit_reg_move(src, dst, ctx); + break; + /* dst = dst OP src */ + case BPF_ALU | BPF_ADD | BPF_X: + case BPF_ALU64 | BPF_ADD | BPF_X: + emit_alu(ADD, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_SUB | BPF_X: + case BPF_ALU64 | BPF_SUB | BPF_X: + emit_alu(SUB, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_AND | BPF_X: + case BPF_ALU64 | BPF_AND | BPF_X: + emit_alu(AND, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_OR | BPF_X: + case BPF_ALU64 | BPF_OR | BPF_X: + emit_alu(OR, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_XOR | BPF_X: + case BPF_ALU64 | BPF_XOR | BPF_X: + emit_alu(XOR, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_MUL | BPF_X: + emit_alu(MUL, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_MUL | BPF_X: + emit_alu(MULX, src, dst, ctx); + break; + case BPF_ALU | BPF_DIV | BPF_X: + emit_cmp(src, G0, ctx); + emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx); + emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx); + + emit_write_y(G0, ctx); + emit_alu(DIV, src, dst, ctx); + break; + + case BPF_ALU64 | BPF_DIV | BPF_X: + emit_cmp(src, G0, ctx); + emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx); + emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx); + + emit_alu(UDIVX, src, dst, ctx); + break; + + case BPF_ALU | BPF_MOD | BPF_X: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + + ctx->tmp_1_used = true; + + emit_cmp(src, G0, ctx); + emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx); + emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx); + + emit_write_y(G0, ctx); + emit_alu3(DIV, dst, src, tmp, ctx); + emit_alu3(MULX, tmp, src, tmp, ctx); + emit_alu3(SUB, dst, tmp, dst, ctx); + goto do_alu32_trunc; + } + case BPF_ALU64 | BPF_MOD | BPF_X: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + + ctx->tmp_1_used = true; + + emit_cmp(src, G0, ctx); + emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx); + emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx); + + emit_alu3(UDIVX, dst, src, tmp, ctx); + emit_alu3(MULX, tmp, src, tmp, ctx); + emit_alu3(SUB, dst, tmp, dst, ctx); + break; + } + case BPF_ALU | BPF_LSH | BPF_X: + emit_alu(SLL, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_LSH | BPF_X: + emit_alu(SLLX, src, dst, ctx); + break; + case BPF_ALU | BPF_RSH | BPF_X: + emit_alu(SRL, src, dst, ctx); + break; + case BPF_ALU64 | BPF_RSH | BPF_X: + emit_alu(SRLX, src, dst, ctx); + break; + case BPF_ALU | BPF_ARSH | BPF_X: + emit_alu(SRA, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_ARSH | BPF_X: + emit_alu(SRAX, src, dst, ctx); + break; + + /* dst = -dst */ + case BPF_ALU | BPF_NEG: + case BPF_ALU64 | BPF_NEG: + emit(SUB | RS1(0) | RS2(dst) | RD(dst), ctx); + goto do_alu32_trunc; + + case BPF_ALU | BPF_END | BPF_FROM_BE: + switch (imm) { + case 16: + emit_alu_K(SLL, dst, 16, ctx); + emit_alu_K(SRL, dst, 16, ctx); + break; + case 32: + emit_alu_K(SRL, dst, 0, ctx); + break; + case 64: + /* nop */ + break; + + } + break; + + /* dst = BSWAP##imm(dst) */ + case BPF_ALU | BPF_END | BPF_FROM_LE: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + const u8 tmp2 = bpf2sparc[TMP_REG_2]; + + ctx->tmp_1_used = true; + switch (imm) { + case 16: + emit_alu3_K(AND, dst, 0xff, tmp, ctx); + emit_alu3_K(SRL, dst, 8, dst, ctx); + emit_alu3_K(AND, dst, 0xff, dst, ctx); + emit_alu3_K(SLL, tmp, 8, tmp, ctx); + emit_alu(OR, tmp, dst, ctx); + break; + + case 32: + ctx->tmp_2_used = true; + emit_alu3_K(SRL, dst, 24, tmp, ctx); /* tmp = dst >> 24 */ + emit_alu3_K(SRL, dst, 16, tmp2, ctx); /* tmp2 = dst >> 16 */ + emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */ + emit_alu3_K(SLL, tmp2, 8, tmp2, ctx); /* tmp2 = tmp2 << 8 */ + emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */ + emit_alu3_K(SRL, dst, 8, tmp2, ctx); /* tmp2 = dst >> 8 */ + emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */ + emit_alu3_K(SLL, tmp2, 16, tmp2, ctx); /* tmp2 = tmp2 << 16 */ + emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */ + emit_alu3_K(AND, dst, 0xff, dst, ctx); /* dst = dst & 0xff */ + emit_alu3_K(SLL, dst, 24, dst, ctx); /* dst = dst << 24 */ + emit_alu(OR, tmp, dst, ctx); /* dst = dst | tmp */ + break; + + case 64: + emit_alu3_K(ADD, SP, STACK_BIAS + 128, tmp, ctx); + emit(ST64 | RS1(tmp) | RS2(G0) | RD(dst), ctx); + emit(LD64A | ASI(ASI_PL) | RS1(tmp) | RS2(G0) | RD(dst), ctx); + break; + } + break; + } + /* dst = imm */ + case BPF_ALU | BPF_MOV | BPF_K: + emit_loadimm32(imm, dst, ctx); + break; + case BPF_ALU64 | BPF_MOV | BPF_K: + emit_loadimm_sext(imm, dst, ctx); + break; + /* dst = dst OP imm */ + case BPF_ALU | BPF_ADD | BPF_K: + case BPF_ALU64 | BPF_ADD | BPF_K: + emit_alu_K(ADD, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_SUB | BPF_K: + case BPF_ALU64 | BPF_SUB | BPF_K: + emit_alu_K(SUB, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_AND | BPF_K: + case BPF_ALU64 | BPF_AND | BPF_K: + emit_alu_K(AND, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_OR | BPF_K: + case BPF_ALU64 | BPF_OR | BPF_K: + emit_alu_K(OR, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_XOR | BPF_K: + case BPF_ALU64 | BPF_XOR | BPF_K: + emit_alu_K(XOR, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_MUL | BPF_K: + emit_alu_K(MUL, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_MUL | BPF_K: + emit_alu_K(MULX, dst, imm, ctx); + break; + case BPF_ALU | BPF_DIV | BPF_K: + if (imm == 0) + return -EINVAL; + + emit_write_y(G0, ctx); + emit_alu_K(DIV, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_DIV | BPF_K: + if (imm == 0) + return -EINVAL; + + emit_alu_K(UDIVX, dst, imm, ctx); + break; + case BPF_ALU64 | BPF_MOD | BPF_K: + case BPF_ALU | BPF_MOD | BPF_K: { + const u8 tmp = bpf2sparc[TMP_REG_2]; + unsigned int div; + + if (imm == 0) + return -EINVAL; + + div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV; + + ctx->tmp_2_used = true; + + if (BPF_CLASS(code) != BPF_ALU64) + emit_write_y(G0, ctx); + if (is_simm13(imm)) { + emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx); + emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx); + emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx); + } else { + const u8 tmp1 = bpf2sparc[TMP_REG_1]; + + ctx->tmp_1_used = true; + + emit_set_const_sext(imm, tmp1, ctx); + emit(div | RS1(dst) | RS2(tmp1) | RD(tmp), ctx); + emit(MULX | RS1(tmp) | RS2(tmp1) | RD(tmp), ctx); + emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx); + } + goto do_alu32_trunc; + } + case BPF_ALU | BPF_LSH | BPF_K: + emit_alu_K(SLL, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_LSH | BPF_K: + emit_alu_K(SLLX, dst, imm, ctx); + break; + case BPF_ALU | BPF_RSH | BPF_K: + emit_alu_K(SRL, dst, imm, ctx); + break; + case BPF_ALU64 | BPF_RSH | BPF_K: + emit_alu_K(SRLX, dst, imm, ctx); + break; + case BPF_ALU | BPF_ARSH | BPF_K: + emit_alu_K(SRA, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_ARSH | BPF_K: + emit_alu_K(SRAX, dst, imm, ctx); + break; + + do_alu32_trunc: + if (BPF_CLASS(code) == BPF_ALU) + emit_alu_K(SRL, dst, 0, ctx); + break; + + /* JUMP off */ + case BPF_JMP | BPF_JA: + emit_branch(BA, ctx->idx, ctx->offset[i + off], ctx); + emit_nop(ctx); + break; + /* IF (dst COND src) JUMP off */ + case BPF_JMP | BPF_JEQ | BPF_X: + case BPF_JMP | BPF_JGT | BPF_X: + case BPF_JMP | BPF_JGE | BPF_X: + case BPF_JMP | BPF_JNE | BPF_X: + case BPF_JMP | BPF_JSGT | BPF_X: + case BPF_JMP | BPF_JSGE | BPF_X: + case BPF_JMP | BPF_JSET | BPF_X: { + int err; + + err = emit_compare_and_branch(code, dst, src, 0, false, i + off, ctx); + if (err) + return err; + break; + } + /* IF (dst COND imm) JUMP off */ + case BPF_JMP | BPF_JEQ | BPF_K: + case BPF_JMP | BPF_JGT | BPF_K: + case BPF_JMP | BPF_JGE | BPF_K: + case BPF_JMP | BPF_JNE | BPF_K: + case BPF_JMP | BPF_JSGT | BPF_K: + case BPF_JMP | BPF_JSGE | BPF_K: + case BPF_JMP | BPF_JSET | BPF_K: { + int err; + + err = emit_compare_and_branch(code, dst, 0, imm, true, i + off, ctx); + if (err) + return err; + break; + } + + /* function call */ + case BPF_JMP | BPF_CALL: + { + u8 *func = ((u8 *)__bpf_call_base) + imm; + + ctx->saw_call = true; + + emit_call((u32 *)func, ctx); + emit_nop(ctx); + + emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx); + + if (bpf_helper_changes_pkt_data(func) && ctx->saw_ld_abs_ind) + load_skb_regs(ctx, bpf2sparc[BPF_REG_6]); + break; + } + + /* tail call */ + case BPF_JMP | BPF_CALL |BPF_X: + emit_tail_call(ctx); + break; + + /* function return */ + case BPF_JMP | BPF_EXIT: + /* Optimization: when last instruction is EXIT, + simply fallthrough to epilogue. */ + if (i == ctx->prog->len - 1) + break; + emit_branch(BA, ctx->idx, ctx->epilogue_offset, ctx); + emit_nop(ctx); + break; + + /* dst = imm64 */ + case BPF_LD | BPF_IMM | BPF_DW: + { + const struct bpf_insn insn1 = insn[1]; + u64 imm64; + + imm64 = (u64)insn1.imm << 32 | (u32)imm; + emit_loadimm64(imm64, dst, ctx); + + return 1; + } + + /* LDX: dst = *(size *)(src + off) */ + case BPF_LDX | BPF_MEM | BPF_W: + case BPF_LDX | BPF_MEM | BPF_H: + case BPF_LDX | BPF_MEM | BPF_B: + case BPF_LDX | BPF_MEM | BPF_DW: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + u32 opcode = 0, rs2; + + ctx->tmp_1_used = true; + switch (BPF_SIZE(code)) { + case BPF_W: + opcode = LD32; + break; + case BPF_H: + opcode = LD16; + break; + case BPF_B: + opcode = LD8; + break; + case BPF_DW: + opcode = LD64; + break; + } + + if (is_simm13(off)) { + opcode |= IMMED; + rs2 = S13(off); + } else { + emit_loadimm(off, tmp, ctx); + rs2 = RS2(tmp); + } + emit(opcode | RS1(src) | rs2 | RD(dst), ctx); + break; + } + /* ST: *(size *)(dst + off) = imm */ + case BPF_ST | BPF_MEM | BPF_W: + case BPF_ST | BPF_MEM | BPF_H: + case BPF_ST | BPF_MEM | BPF_B: + case BPF_ST | BPF_MEM | BPF_DW: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + const u8 tmp2 = bpf2sparc[TMP_REG_2]; + u32 opcode = 0, rs2; + + ctx->tmp_2_used = true; + emit_loadimm(imm, tmp2, ctx); + + switch (BPF_SIZE(code)) { + case BPF_W: + opcode = ST32; + break; + case BPF_H: + opcode = ST16; + break; + case BPF_B: + opcode = ST8; + break; + case BPF_DW: + opcode = ST64; + break; + } + + if (is_simm13(off)) { + opcode |= IMMED; + rs2 = S13(off); + } else { + ctx->tmp_1_used = true; + emit_loadimm(off, tmp, ctx); + rs2 = RS2(tmp); + } + emit(opcode | RS1(dst) | rs2 | RD(tmp2), ctx); + break; + } + + /* STX: *(size *)(dst + off) = src */ + case BPF_STX | BPF_MEM | BPF_W: + case BPF_STX | BPF_MEM | BPF_H: + case BPF_STX | BPF_MEM | BPF_B: + case BPF_STX | BPF_MEM | BPF_DW: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + u32 opcode = 0, rs2; + + switch (BPF_SIZE(code)) { + case BPF_W: + opcode = ST32; + break; + case BPF_H: + opcode = ST16; + break; + case BPF_B: + opcode = ST8; + break; + case BPF_DW: + opcode = ST64; + break; + } + if (is_simm13(off)) { + opcode |= IMMED; + rs2 = S13(off); + } else { + ctx->tmp_1_used = true; + emit_loadimm(off, tmp, ctx); + rs2 = RS2(tmp); + } + emit(opcode | RS1(dst) | rs2 | RD(src), ctx); + break; + } + + /* STX XADD: lock *(u32 *)(dst + off) += src */ + case BPF_STX | BPF_XADD | BPF_W: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + const u8 tmp2 = bpf2sparc[TMP_REG_2]; + const u8 tmp3 = bpf2sparc[TMP_REG_3]; + + ctx->tmp_1_used = true; + ctx->tmp_2_used = true; + ctx->tmp_3_used = true; + emit_loadimm(off, tmp, ctx); + emit_alu3(ADD, dst, tmp, tmp, ctx); + + emit(LD32 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx); + emit_alu3(ADD, tmp2, src, tmp3, ctx); + emit(CAS | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx); + emit_cmp(tmp2, tmp3, ctx); + emit_branch(BNE, 4, 0, ctx); + emit_nop(ctx); + break; + } + /* STX XADD: lock *(u64 *)(dst + off) += src */ + case BPF_STX | BPF_XADD | BPF_DW: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + const u8 tmp2 = bpf2sparc[TMP_REG_2]; + const u8 tmp3 = bpf2sparc[TMP_REG_3]; + + ctx->tmp_1_used = true; + ctx->tmp_2_used = true; + ctx->tmp_3_used = true; + emit_loadimm(off, tmp, ctx); + emit_alu3(ADD, dst, tmp, tmp, ctx); + + emit(LD64 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx); + emit_alu3(ADD, tmp2, src, tmp3, ctx); + emit(CASX | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx); + emit_cmp(tmp2, tmp3, ctx); + emit_branch(BNE, 4, 0, ctx); + emit_nop(ctx); + break; + } +#define CHOOSE_LOAD_FUNC(K, func) \ + ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset) + + /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */ + case BPF_LD | BPF_ABS | BPF_W: + func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_word); + goto common_load; + case BPF_LD | BPF_ABS | BPF_H: + func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_half); + goto common_load; + case BPF_LD | BPF_ABS | BPF_B: + func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_byte); + goto common_load; + /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */ + case BPF_LD | BPF_IND | BPF_W: + func = bpf_jit_load_word; + goto common_load; + case BPF_LD | BPF_IND | BPF_H: + func = bpf_jit_load_half; + goto common_load; + + case BPF_LD | BPF_IND | BPF_B: + func = bpf_jit_load_byte; + common_load: + ctx->saw_ld_abs_ind = true; + + emit_reg_move(bpf2sparc[BPF_REG_6], O0, ctx); + emit_loadimm(imm, O1, ctx); + + if (BPF_MODE(code) == BPF_IND) + emit_alu(ADD, src, O1, ctx); + + emit_call(func, ctx); + emit_alu_K(SRA, O1, 0, ctx); + + emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx); + break; + + default: + pr_err_once("unknown opcode %02x\n", code); + return -EINVAL; + } + + return 0; +} + +static int build_body(struct jit_ctx *ctx) +{ + const struct bpf_prog *prog = ctx->prog; + int i; + + for (i = 0; i < prog->len; i++) { + const struct bpf_insn *insn = &prog->insnsi[i]; + int ret; + + ret = build_insn(insn, ctx); + + if (ret > 0) { + i++; + ctx->offset[i] = ctx->idx; + continue; + } + ctx->offset[i] = ctx->idx; + if (ret) + return ret; + } + return 0; +} + +static void jit_fill_hole(void *area, unsigned int size) +{ + u32 *ptr; + /* We are guaranteed to have aligned memory. */ + for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) + *ptr++ = 0x91d02005; /* ta 5 */ +} + +struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +{ + struct bpf_prog *tmp, *orig_prog = prog; + struct bpf_binary_header *header; + bool tmp_blinded = false; + struct jit_ctx ctx; + u32 image_size; + u8 *image_ptr; + int pass; + + if (!bpf_jit_enable) + return orig_prog; + + tmp = bpf_jit_blind_constants(prog); + /* If blinding was requested and we failed during blinding, + * we must fall back to the interpreter. + */ + if (IS_ERR(tmp)) + return orig_prog; + if (tmp != prog) { + tmp_blinded = true; + prog = tmp; + } + + memset(&ctx, 0, sizeof(ctx)); + ctx.prog = prog; + + ctx.offset = kcalloc(prog->len, sizeof(unsigned int), GFP_KERNEL); + if (ctx.offset == NULL) { + prog = orig_prog; + goto out; + } + + /* Fake pass to detect features used, and get an accurate assessment + * of what the final image size will be. + */ + if (build_body(&ctx)) { + prog = orig_prog; + goto out_off; + } + build_prologue(&ctx); + build_epilogue(&ctx); + + /* Now we know the actual image size. */ + image_size = sizeof(u32) * ctx.idx; + header = bpf_jit_binary_alloc(image_size, &image_ptr, + sizeof(u32), jit_fill_hole); + if (header == NULL) { + prog = orig_prog; + goto out_off; + } + + ctx.image = (u32 *)image_ptr; + + for (pass = 1; pass < 3; pass++) { + ctx.idx = 0; + + build_prologue(&ctx); + + if (build_body(&ctx)) { + bpf_jit_binary_free(header); + prog = orig_prog; + goto out_off; + } + + build_epilogue(&ctx); + + if (bpf_jit_enable > 1) + pr_info("Pass %d: shrink = %d, seen = [%c%c%c%c%c%c%c]\n", pass, + image_size - (ctx.idx * 4), + ctx.tmp_1_used ? '1' : ' ', + ctx.tmp_2_used ? '2' : ' ', + ctx.tmp_3_used ? '3' : ' ', + ctx.saw_ld_abs_ind ? 'L' : ' ', + ctx.saw_frame_pointer ? 'F' : ' ', + ctx.saw_call ? 'C' : ' ', + ctx.saw_tail_call ? 'T' : ' '); + } + + if (bpf_jit_enable > 1) + bpf_jit_dump(prog->len, image_size, pass, ctx.image); + + bpf_flush_icache(header, (u8 *)header + (header->pages * PAGE_SIZE)); + + bpf_jit_binary_lock_ro(header); + + prog->bpf_func = (void *)ctx.image; + prog->jited = 1; + +out_off: + kfree(ctx.offset); +out: + if (tmp_blinded) + bpf_jit_prog_release_other(prog, prog == orig_prog ? + tmp : orig_prog); + return prog; +} |