386 static inline int mm_has_pgste(struct mm_struct *mm)  in mm_has_pgste()  argument
389 	if (unlikely(mm->context.has_pgste))  in mm_has_pgste()
395 static inline int mm_alloc_pgste(struct mm_struct *mm)  in mm_alloc_pgste()  argument
398 	if (unlikely(mm->context.alloc_pgste))  in mm_alloc_pgste()
409 static inline int mm_use_skey(struct mm_struct *mm)  in mm_use_skey()  argument
412 	if (mm->context.use_skey)  in mm_use_skey()
629 				       struct mm_struct *mm)  in pgste_update_all()  argument
634 	if (!mm_use_skey(mm) || pte_val(*ptep) & _PAGE_INVALID)  in pgste_update_all()
650 				 struct mm_struct *mm)  in pgste_set_key()  argument
656 	if (!mm_use_skey(mm) || pte_val(entry) & _PAGE_INVALID)  in pgste_set_key()
706 	struct mm_struct *mm;  member
726 struct gmap *gmap_alloc(struct mm_struct *mm, unsigned long limit);
747 static inline pgste_t pgste_ipte_notify(struct mm_struct *mm,  in pgste_ipte_notify()  argument
754 		gmap_do_ipte_notify(mm, addr, ptep);  in pgste_ipte_notify()
765 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,  in set_pte_at()  argument
770 	if (mm_has_pgste(mm)) {  in set_pte_at()
773 		pgste_set_key(ptep, pgste, entry, mm);  in set_pte_at()
827 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)  in pte_clear()  argument
946 static inline void ptep_flush_direct(struct mm_struct *mm,  in ptep_flush_direct()  argument
953 	active = (mm == current->active_mm) ? 1 : 0;  in ptep_flush_direct()
954 	count = atomic_add_return(0x10000, &mm->context.attach_count);  in ptep_flush_direct()
956 	    cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))  in ptep_flush_direct()
960 	atomic_sub(0x10000, &mm->context.attach_count);  in ptep_flush_direct()
963 static inline void ptep_flush_lazy(struct mm_struct *mm,  in ptep_flush_lazy()  argument
970 	active = (mm == current->active_mm) ? 1 : 0;  in ptep_flush_lazy()
971 	count = atomic_add_return(0x10000, &mm->context.attach_count);  in ptep_flush_lazy()
974 		mm->context.flush_mm = 1;  in ptep_flush_lazy()
977 	atomic_sub(0x10000, &mm->context.attach_count);  in ptep_flush_lazy()
983 static inline int ptep_test_and_clear_user_dirty(struct mm_struct *mm,  in ptep_test_and_clear_user_dirty()  argument
991 	if (!mm_has_pgste(mm))  in ptep_test_and_clear_user_dirty()
998 		pgste = pgste_ipte_notify(mm, addr, ptep, pgste);  in ptep_test_and_clear_user_dirty()
1059 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,  in ptep_get_and_clear()  argument
1065 	if (mm_has_pgste(mm)) {  in ptep_get_and_clear()
1067 		pgste = pgste_ipte_notify(mm, address, ptep, pgste);  in ptep_get_and_clear()
1071 	ptep_flush_lazy(mm, address, ptep);  in ptep_get_and_clear()
1074 	if (mm_has_pgste(mm)) {  in ptep_get_and_clear()
1075 		pgste = pgste_update_all(&pte, pgste, mm);  in ptep_get_and_clear()
1082 static inline pte_t ptep_modify_prot_start(struct mm_struct *mm,  in ptep_modify_prot_start()  argument
1089 	if (mm_has_pgste(mm)) {  in ptep_modify_prot_start()
1091 		pgste_ipte_notify(mm, address, ptep, pgste);  in ptep_modify_prot_start()
1095 	ptep_flush_lazy(mm, address, ptep);  in ptep_modify_prot_start()
1097 	if (mm_has_pgste(mm)) {  in ptep_modify_prot_start()
1098 		pgste = pgste_update_all(&pte, pgste, mm);  in ptep_modify_prot_start()
1104 static inline void ptep_modify_prot_commit(struct mm_struct *mm,  in ptep_modify_prot_commit()  argument
1110 	if (mm_has_pgste(mm)) {  in ptep_modify_prot_commit()
1112 		pgste_set_key(ptep, pgste, pte, mm);  in ptep_modify_prot_commit()
1153 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,  in ptep_get_and_clear_full()  argument
1160 	if (!full && mm_has_pgste(mm)) {  in ptep_get_and_clear_full()
1162 		pgste = pgste_ipte_notify(mm, address, ptep, pgste);  in ptep_get_and_clear_full()
1167 		ptep_flush_lazy(mm, address, ptep);  in ptep_get_and_clear_full()
1170 	if (!full && mm_has_pgste(mm)) {  in ptep_get_and_clear_full()
1171 		pgste = pgste_update_all(&pte, pgste, mm);  in ptep_get_and_clear_full()
1178 static inline pte_t ptep_set_wrprotect(struct mm_struct *mm,  in ptep_set_wrprotect()  argument
1185 		if (mm_has_pgste(mm)) {  in ptep_set_wrprotect()
1187 			pgste = pgste_ipte_notify(mm, address, ptep, pgste);  in ptep_set_wrprotect()
1190 		ptep_flush_lazy(mm, address, ptep);  in ptep_set_wrprotect()
1193 		if (mm_has_pgste(mm)) {  in ptep_set_wrprotect()
1253 #define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))  argument
1420 static inline void pmdp_flush_direct(struct mm_struct *mm,  in pmdp_flush_direct()  argument
1431 	active = (mm == current->active_mm) ? 1 : 0;  in pmdp_flush_direct()
1432 	count = atomic_add_return(0x10000, &mm->context.attach_count);  in pmdp_flush_direct()
1434 	    cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))  in pmdp_flush_direct()
1438 	atomic_sub(0x10000, &mm->context.attach_count);  in pmdp_flush_direct()
1441 static inline void pmdp_flush_lazy(struct mm_struct *mm,  in pmdp_flush_lazy()  argument
1448 	active = (mm == current->active_mm) ? 1 : 0;  in pmdp_flush_lazy()
1449 	count = atomic_add_return(0x10000, &mm->context.attach_count);  in pmdp_flush_lazy()
1452 		mm->context.flush_mm = 1;  in pmdp_flush_lazy()
1457 	atomic_sub(0x10000, &mm->context.attach_count);  in pmdp_flush_lazy()
1463 extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
1467 extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
1475 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,  in set_pmd_at()  argument
1502 static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,  in pmdp_get_and_clear()  argument
1507 	pmdp_flush_direct(mm, address, pmdp);  in pmdp_get_and_clear()
1513 static inline pmd_t pmdp_get_and_clear_full(struct mm_struct *mm,  in pmdp_get_and_clear_full()  argument
1520 		pmdp_flush_lazy(mm, address, pmdp);  in pmdp_get_and_clear_full()
1540 static inline void pmdp_set_wrprotect(struct mm_struct *mm,  in pmdp_set_wrprotect()  argument
1546 		pmdp_flush_direct(mm, address, pmdp);  in pmdp_set_wrprotect()
1547 		set_pmd_at(mm, address, pmdp, pmd_wrprotect(pmd));  in pmdp_set_wrprotect()
1623 extern void s390_reset_cmma(struct mm_struct *mm);