diff options
Diffstat (limited to 'arch/s390/mm/pgtable.c')
-rw-r--r-- | arch/s390/mm/pgtable.c | 103 |
1 files changed, 85 insertions, 18 deletions
diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index d7cfd57815fb..23b44ef46379 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -53,8 +53,10 @@ static void __crst_table_upgrade(void *arg) { struct mm_struct *mm = arg; - if (current->active_mm == mm) - update_user_asce(mm, 1); + if (current->active_mm == mm) { + clear_user_asce(); + set_user_asce(mm); + } __tlb_flush_local(); } @@ -108,7 +110,7 @@ void crst_table_downgrade(struct mm_struct *mm, unsigned long limit) pgd_t *pgd; if (current->active_mm == mm) { - clear_user_asce(mm, 1); + clear_user_asce(); __tlb_flush_mm(mm); } while (mm->context.asce_limit > limit) { @@ -134,7 +136,7 @@ void crst_table_downgrade(struct mm_struct *mm, unsigned long limit) crst_table_free(mm, (unsigned long *) pgd); } if (current->active_mm == mm) - update_user_asce(mm, 1); + set_user_asce(mm); } #endif @@ -832,6 +834,7 @@ void gmap_do_ipte_notify(struct mm_struct *mm, pte_t *pte) } spin_unlock(&gmap_notifier_lock); } +EXPORT_SYMBOL_GPL(gmap_do_ipte_notify); static inline int page_table_with_pgste(struct page *page) { @@ -864,8 +867,7 @@ static inline unsigned long *page_table_alloc_pgste(struct mm_struct *mm, atomic_set(&page->_mapcount, 0); table = (unsigned long *) page_to_phys(page); clear_table(table, _PAGE_INVALID, PAGE_SIZE/2); - clear_table(table + PTRS_PER_PTE, PGSTE_HR_BIT | PGSTE_HC_BIT, - PAGE_SIZE/2); + clear_table(table + PTRS_PER_PTE, 0, PAGE_SIZE/2); return table; } @@ -883,8 +885,8 @@ static inline void page_table_free_pgste(unsigned long *table) __free_page(page); } -static inline unsigned long page_table_reset_pte(struct mm_struct *mm, - pmd_t *pmd, unsigned long addr, unsigned long end) +static inline unsigned long page_table_reset_pte(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, unsigned long end, bool init_skey) { pte_t *start_pte, *pte; spinlock_t *ptl; @@ -895,6 +897,22 @@ static inline unsigned long page_table_reset_pte(struct mm_struct *mm, do { pgste = pgste_get_lock(pte); pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK; + if (init_skey) { + unsigned long address; + + pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT | + PGSTE_GR_BIT | PGSTE_GC_BIT); + + /* skip invalid and not writable pages */ + if (pte_val(*pte) & _PAGE_INVALID || + !(pte_val(*pte) & _PAGE_WRITE)) { + pgste_set_unlock(pte, pgste); + continue; + } + + address = pte_val(*pte) & PAGE_MASK; + page_set_storage_key(address, PAGE_DEFAULT_KEY, 1); + } pgste_set_unlock(pte, pgste); } while (pte++, addr += PAGE_SIZE, addr != end); pte_unmap_unlock(start_pte, ptl); @@ -902,8 +920,8 @@ static inline unsigned long page_table_reset_pte(struct mm_struct *mm, return addr; } -static inline unsigned long page_table_reset_pmd(struct mm_struct *mm, - pud_t *pud, unsigned long addr, unsigned long end) +static inline unsigned long page_table_reset_pmd(struct mm_struct *mm, pud_t *pud, + unsigned long addr, unsigned long end, bool init_skey) { unsigned long next; pmd_t *pmd; @@ -913,14 +931,14 @@ static inline unsigned long page_table_reset_pmd(struct mm_struct *mm, next = pmd_addr_end(addr, end); if (pmd_none_or_clear_bad(pmd)) continue; - next = page_table_reset_pte(mm, pmd, addr, next); + next = page_table_reset_pte(mm, pmd, addr, next, init_skey); } while (pmd++, addr = next, addr != end); return addr; } -static inline unsigned long page_table_reset_pud(struct mm_struct *mm, - pgd_t *pgd, unsigned long addr, unsigned long end) +static inline unsigned long page_table_reset_pud(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr, unsigned long end, bool init_skey) { unsigned long next; pud_t *pud; @@ -930,14 +948,14 @@ static inline unsigned long page_table_reset_pud(struct mm_struct *mm, next = pud_addr_end(addr, end); if (pud_none_or_clear_bad(pud)) continue; - next = page_table_reset_pmd(mm, pud, addr, next); + next = page_table_reset_pmd(mm, pud, addr, next, init_skey); } while (pud++, addr = next, addr != end); return addr; } -void page_table_reset_pgste(struct mm_struct *mm, - unsigned long start, unsigned long end) +void page_table_reset_pgste(struct mm_struct *mm, unsigned long start, + unsigned long end, bool init_skey) { unsigned long addr, next; pgd_t *pgd; @@ -949,7 +967,7 @@ void page_table_reset_pgste(struct mm_struct *mm, next = pgd_addr_end(addr, end); if (pgd_none_or_clear_bad(pgd)) continue; - next = page_table_reset_pud(mm, pgd, addr, next); + next = page_table_reset_pud(mm, pgd, addr, next, init_skey); } while (pgd++, addr = next, addr != end); up_read(&mm->mmap_sem); } @@ -989,7 +1007,7 @@ int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, /* changing the guest storage key is considered a change of the page */ if ((pgste_val(new) ^ pgste_val(old)) & (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT)) - pgste_val(new) |= PGSTE_HC_BIT; + pgste_val(new) |= PGSTE_UC_BIT; pgste_set_unlock(ptep, new); pte_unmap_unlock(*ptep, ptl); @@ -1011,6 +1029,11 @@ static inline unsigned long *page_table_alloc_pgste(struct mm_struct *mm, return NULL; } +void page_table_reset_pgste(struct mm_struct *mm, unsigned long start, + unsigned long end, bool init_skey) +{ +} + static inline void page_table_free_pgste(unsigned long *table) { } @@ -1357,6 +1380,50 @@ int s390_enable_sie(void) } EXPORT_SYMBOL_GPL(s390_enable_sie); +/* + * Enable storage key handling from now on and initialize the storage + * keys with the default key. + */ +void s390_enable_skey(void) +{ + /* + * To avoid races between multiple vcpus, ending in calling + * page_table_reset twice or more, + * the page_table_lock is taken for serialization. + */ + spin_lock(¤t->mm->page_table_lock); + if (mm_use_skey(current->mm)) { + spin_unlock(¤t->mm->page_table_lock); + return; + } + + current->mm->context.use_skey = 1; + spin_unlock(¤t->mm->page_table_lock); + page_table_reset_pgste(current->mm, 0, TASK_SIZE, true); +} +EXPORT_SYMBOL_GPL(s390_enable_skey); + +/* + * Test and reset if a guest page is dirty + */ +bool gmap_test_and_clear_dirty(unsigned long address, struct gmap *gmap) +{ + pte_t *pte; + spinlock_t *ptl; + bool dirty = false; + + pte = get_locked_pte(gmap->mm, address, &ptl); + if (unlikely(!pte)) + return false; + + if (ptep_test_and_clear_user_dirty(gmap->mm, address, pte)) + dirty = true; + + spin_unlock(ptl); + return dirty; +} +EXPORT_SYMBOL_GPL(gmap_test_and_clear_dirty); + #ifdef CONFIG_TRANSPARENT_HUGEPAGE int pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) |