Lines Matching refs:mm
390 static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) in dup_mmap() argument
400 uprobe_dup_mmap(oldmm, mm); in dup_mmap()
404 down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING); in dup_mmap()
407 RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm)); in dup_mmap()
409 mm->total_vm = oldmm->total_vm; in dup_mmap()
410 mm->shared_vm = oldmm->shared_vm; in dup_mmap()
411 mm->exec_vm = oldmm->exec_vm; in dup_mmap()
412 mm->stack_vm = oldmm->stack_vm; in dup_mmap()
414 rb_link = &mm->mm_rb.rb_node; in dup_mmap()
416 pprev = &mm->mmap; in dup_mmap()
417 retval = ksm_fork(mm, oldmm); in dup_mmap()
420 retval = khugepaged_fork(mm, oldmm); in dup_mmap()
429 vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file, in dup_mmap()
449 tmp->vm_mm = mm; in dup_mmap()
489 __vma_link_rb(mm, tmp, rb_link, rb_parent); in dup_mmap()
493 mm->map_count++; in dup_mmap()
494 retval = copy_page_range(mm, oldmm, mpnt); in dup_mmap()
503 arch_dup_mmap(oldmm, mm); in dup_mmap()
506 up_write(&mm->mmap_sem); in dup_mmap()
521 static inline int mm_alloc_pgd(struct mm_struct *mm) in mm_alloc_pgd() argument
523 mm->pgd = pgd_alloc(mm); in mm_alloc_pgd()
524 if (unlikely(!mm->pgd)) in mm_alloc_pgd()
529 static inline void mm_free_pgd(struct mm_struct *mm) in mm_free_pgd() argument
531 pgd_free(mm, mm->pgd); in mm_free_pgd()
534 static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) in dup_mmap() argument
537 RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm)); in dup_mmap()
541 #define mm_alloc_pgd(mm) (0) argument
542 #define mm_free_pgd(mm) argument
548 #define free_mm(mm) (kmem_cache_free(mm_cachep, (mm))) argument
564 static void mm_init_aio(struct mm_struct *mm) in mm_init_aio() argument
567 spin_lock_init(&mm->ioctx_lock); in mm_init_aio()
568 mm->ioctx_table = NULL; in mm_init_aio()
572 static void mm_init_owner(struct mm_struct *mm, struct task_struct *p) in mm_init_owner() argument
575 mm->owner = p; in mm_init_owner()
579 static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p) in mm_init() argument
581 mm->mmap = NULL; in mm_init()
582 mm->mm_rb = RB_ROOT; in mm_init()
583 mm->vmacache_seqnum = 0; in mm_init()
584 atomic_set(&mm->mm_users, 1); in mm_init()
585 atomic_set(&mm->mm_count, 1); in mm_init()
586 init_rwsem(&mm->mmap_sem); in mm_init()
587 INIT_LIST_HEAD(&mm->mmlist); in mm_init()
588 mm->core_state = NULL; in mm_init()
589 atomic_long_set(&mm->nr_ptes, 0); in mm_init()
590 mm_nr_pmds_init(mm); in mm_init()
591 mm->map_count = 0; in mm_init()
592 mm->locked_vm = 0; in mm_init()
593 mm->pinned_vm = 0; in mm_init()
594 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); in mm_init()
595 spin_lock_init(&mm->page_table_lock); in mm_init()
596 mm_init_cpumask(mm); in mm_init()
597 mm_init_aio(mm); in mm_init()
598 mm_init_owner(mm, p); in mm_init()
599 mmu_notifier_mm_init(mm); in mm_init()
600 clear_tlb_flush_pending(mm); in mm_init()
602 mm->pmd_huge_pte = NULL; in mm_init()
605 if (current->mm) { in mm_init()
606 mm->flags = current->mm->flags & MMF_INIT_MASK; in mm_init()
607 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK; in mm_init()
609 mm->flags = default_dump_filter; in mm_init()
610 mm->def_flags = 0; in mm_init()
613 if (mm_alloc_pgd(mm)) in mm_init()
616 if (init_new_context(p, mm)) in mm_init()
619 return mm; in mm_init()
622 mm_free_pgd(mm); in mm_init()
624 free_mm(mm); in mm_init()
628 static void check_mm(struct mm_struct *mm) in check_mm() argument
633 long x = atomic_long_read(&mm->rss_stat.count[i]); in check_mm()
637 "mm:%p idx:%d val:%ld\n", mm, i, x); in check_mm()
640 if (atomic_long_read(&mm->nr_ptes)) in check_mm()
642 atomic_long_read(&mm->nr_ptes)); in check_mm()
643 if (mm_nr_pmds(mm)) in check_mm()
645 mm_nr_pmds(mm)); in check_mm()
648 VM_BUG_ON_MM(mm->pmd_huge_pte, mm); in check_mm()
657 struct mm_struct *mm; in mm_alloc() local
659 mm = allocate_mm(); in mm_alloc()
660 if (!mm) in mm_alloc()
663 memset(mm, 0, sizeof(*mm)); in mm_alloc()
664 return mm_init(mm, current); in mm_alloc()
672 void __mmdrop(struct mm_struct *mm) in __mmdrop() argument
674 BUG_ON(mm == &init_mm); in __mmdrop()
675 mm_free_pgd(mm); in __mmdrop()
676 destroy_context(mm); in __mmdrop()
677 mmu_notifier_mm_destroy(mm); in __mmdrop()
678 check_mm(mm); in __mmdrop()
679 free_mm(mm); in __mmdrop()
686 void mmput(struct mm_struct *mm) in mmput() argument
690 if (atomic_dec_and_test(&mm->mm_users)) { in mmput()
691 uprobe_clear_state(mm); in mmput()
692 exit_aio(mm); in mmput()
693 ksm_exit(mm); in mmput()
694 khugepaged_exit(mm); /* must run before exit_mmap */ in mmput()
695 exit_mmap(mm); in mmput()
696 set_mm_exe_file(mm, NULL); in mmput()
697 if (!list_empty(&mm->mmlist)) { in mmput()
699 list_del(&mm->mmlist); in mmput()
702 if (mm->binfmt) in mmput()
703 module_put(mm->binfmt->module); in mmput()
704 mmdrop(mm); in mmput()
720 void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) in set_mm_exe_file() argument
729 old_exe_file = rcu_dereference_raw(mm->exe_file); in set_mm_exe_file()
733 rcu_assign_pointer(mm->exe_file, new_exe_file); in set_mm_exe_file()
744 struct file *get_mm_exe_file(struct mm_struct *mm) in get_mm_exe_file() argument
749 exe_file = rcu_dereference(mm->exe_file); in get_mm_exe_file()
768 struct mm_struct *mm; in get_task_mm() local
771 mm = task->mm; in get_task_mm()
772 if (mm) { in get_task_mm()
774 mm = NULL; in get_task_mm()
776 atomic_inc(&mm->mm_users); in get_task_mm()
779 return mm; in get_task_mm()
785 struct mm_struct *mm; in mm_access() local
792 mm = get_task_mm(task); in mm_access()
793 if (mm && mm != current->mm && in mm_access()
795 mmput(mm); in mm_access()
796 mm = ERR_PTR(-EACCES); in mm_access()
800 return mm; in mm_access()
848 void mm_release(struct task_struct *tsk, struct mm_struct *mm) in mm_release() argument
869 deactivate_mm(tsk, mm); in mm_release()
880 atomic_read(&mm->mm_users) > 1) { in mm_release()
906 struct mm_struct *mm, *oldmm = current->mm; in dup_mm() local
909 mm = allocate_mm(); in dup_mm()
910 if (!mm) in dup_mm()
913 memcpy(mm, oldmm, sizeof(*mm)); in dup_mm()
915 if (!mm_init(mm, tsk)) in dup_mm()
918 err = dup_mmap(mm, oldmm); in dup_mm()
922 mm->hiwater_rss = get_mm_rss(mm); in dup_mm()
923 mm->hiwater_vm = mm->total_vm; in dup_mm()
925 if (mm->binfmt && !try_module_get(mm->binfmt->module)) in dup_mm()
928 return mm; in dup_mm()
932 mm->binfmt = NULL; in dup_mm()
933 mmput(mm); in dup_mm()
941 struct mm_struct *mm, *oldmm; in copy_mm() local
950 tsk->mm = NULL; in copy_mm()
958 oldmm = current->mm; in copy_mm()
967 mm = oldmm; in copy_mm()
972 mm = dup_mm(tsk); in copy_mm()
973 if (!mm) in copy_mm()
977 tsk->mm = mm; in copy_mm()
978 tsk->active_mm = mm; in copy_mm()
1615 if (p->mm) in copy_process()
1616 mmput(p->mm); in copy_process()