diff --git a/include/linux/fs.h b/include/linux/fs.h index c30fc311f60..c4b8f1020e5 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1096,6 +1096,7 @@ struct file_handle { }; #define get_file(x) atomic_long_inc(&(x)->f_count) +#define get_file_rcu(x) atomic_long_inc_not_zero(&(x)->f_count) #define fput_atomic(x) atomic_long_add_unless(&(x)->f_count, -1, 1) #define file_count(x) atomic_long_read(&(x)->f_count) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index cb33f1aa9e7..edc6d502370 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -382,7 +382,7 @@ struct mm_struct { #endif /* store ref to file /proc//exe symlink points to */ - struct file *exe_file; + struct file __rcu *exe_file; unsigned long num_exe_file_vmas; #ifdef CONFIG_MMU_NOTIFIER struct mmu_notifier_mm *mmu_notifier_mm; diff --git a/kernel/fork.c b/kernel/fork.c index 10841ae1f8c..49ed634cba6 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -354,6 +354,9 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) */ down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING); + /* No ordering required: file already has been exposed. */ + RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm)); + mm->locked_vm = 0; mm->mmap = NULL; mm->mmap_cache = NULL; @@ -484,7 +487,13 @@ static inline void mm_free_pgd(struct mm_struct *mm) pgd_free(mm, mm->pgd); } #else -#define dup_mmap(mm, oldmm) (0) +static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) +{ + down_write(&oldmm->mmap_sem); + RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm)); + up_write(&oldmm->mmap_sem); + return 0; +} #define mm_alloc_pgd(mm) (0) #define mm_free_pgd(mm) #endif /* CONFIG_MMU */ @@ -640,37 +649,47 @@ void removed_exe_file_vma(struct mm_struct *mm) } +/** + * set_mm_exe_file - change a reference to the mm's executable file + * + * This changes mm's executable file (shown as symlink /proc/[pid]/exe). + * + * Main users are mmput(), sys_execve() and sys_prctl(PR_SET_MM_MAP/EXE_FILE). + * Callers prevent concurrent invocations: in mmput() nobody alive left, + * in execve task is single-threaded, prctl holds mmap_sem exclusively. + */ void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) { + struct file *old_exe_file = rcu_dereference_protected(mm->exe_file, + !atomic_read(&mm->mm_users) || current->in_execve || + lockdep_is_held(&mm->mmap_sem)); + if (new_exe_file) get_file(new_exe_file); - if (mm->exe_file) - fput(mm->exe_file); - mm->exe_file = new_exe_file; + rcu_assign_pointer(mm->exe_file, new_exe_file); + if (old_exe_file) + fput(old_exe_file); mm->num_exe_file_vmas = 0; } +/** + * get_mm_exe_file - acquire a reference to the mm's executable file + * + * Returns %NULL if mm has no associated executable file. + * User must release file via fput(). + */ struct file *get_mm_exe_file(struct mm_struct *mm) { struct file *exe_file; - /* We need mmap_sem to protect against races with removal of - * VM_EXECUTABLE vmas */ - down_read(&mm->mmap_sem); - exe_file = mm->exe_file; - if (exe_file) - get_file(exe_file); - up_read(&mm->mmap_sem); + rcu_read_lock(); + exe_file = rcu_dereference(mm->exe_file); + if (exe_file && !get_file_rcu(exe_file)) + exe_file = NULL; + rcu_read_unlock(); return exe_file; } -static void dup_mm_exe_file(struct mm_struct *oldmm, struct mm_struct *newmm) -{ - /* It's safe to write the exe_file pointer without exe_file_lock because - * this is called during fork when the task is not yet in /proc */ - newmm->exe_file = get_mm_exe_file(oldmm); -} - /** * get_task_mm - acquire a reference to the task's mm * @@ -837,8 +856,6 @@ struct mm_struct *dup_mm(struct task_struct *tsk) if (init_new_context(tsk, mm)) goto fail_nocontext; - dup_mm_exe_file(oldmm, mm); - err = dup_mmap(mm, oldmm); if (err) goto free_pt;