diff options
Diffstat (limited to 'fs/file_table.c')
-rw-r--r-- | fs/file_table.c | 208 |
1 files changed, 83 insertions, 125 deletions
diff --git a/fs/file_table.c b/fs/file_table.c index a305d9e2d1b2..b44e4c559786 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -23,6 +23,8 @@ #include <linux/lglock.h> #include <linux/percpu_counter.h> #include <linux/percpu.h> +#include <linux/hardirq.h> +#include <linux/task_work.h> #include <linux/ima.h> #include <linux/atomic.h> @@ -34,14 +36,14 @@ struct files_stat_struct files_stat = { .max_files = NR_FILE }; -DEFINE_LGLOCK(files_lglock); +DEFINE_STATIC_LGLOCK(files_lglock); /* SLAB cache for file structures */ static struct kmem_cache *filp_cachep __read_mostly; static struct percpu_counter nr_files __cacheline_aligned_in_smp; -static inline void file_free_rcu(struct rcu_head *head) +static void file_free_rcu(struct rcu_head *head) { struct file *f = container_of(head, struct file, f_u.fu_rcuhead); @@ -92,8 +94,8 @@ int proc_nr_files(ctl_table *table, int write, #endif /* Find an unused file structure and return a pointer to it. - * Returns NULL, if there are no more free file structures or - * we run out of memory. + * Returns an error pointer if some error happend e.g. we over file + * structures limit, run out of memory or operation is not permitted. * * Be very careful using this. You are responsible for * getting write access to any mount that you might assign @@ -105,7 +107,8 @@ struct file *get_empty_filp(void) { const struct cred *cred = current_cred(); static long old_max; - struct file * f; + struct file *f; + int error; /* * Privileged users can go above max_files @@ -120,13 +123,16 @@ struct file *get_empty_filp(void) } f = kmem_cache_zalloc(filp_cachep, GFP_KERNEL); - if (f == NULL) - goto fail; + if (unlikely(!f)) + return ERR_PTR(-ENOMEM); percpu_counter_inc(&nr_files); f->f_cred = get_cred(cred); - if (security_file_alloc(f)) - goto fail_sec; + error = security_file_alloc(f); + if (unlikely(error)) { + file_free(f); + return ERR_PTR(error); + } INIT_LIST_HEAD(&f->f_u.fu_list); atomic_long_set(&f->f_count, 1); @@ -142,12 +148,7 @@ over: pr_info("VFS: file-max limit %lu reached\n", get_max_files()); old_max = get_nr_files(); } - goto fail; - -fail_sec: - file_free(f); -fail: - return NULL; + return ERR_PTR(-ENFILE); } /** @@ -171,10 +172,11 @@ struct file *alloc_file(struct path *path, fmode_t mode, struct file *file; file = get_empty_filp(); - if (!file) - return NULL; + if (IS_ERR(file)) + return file; file->f_path = *path; + file->f_inode = path->dentry->d_inode; file->f_mapping = path->dentry->d_inode->i_mapping; file->f_mode = mode; file->f_op = fop; @@ -215,7 +217,7 @@ static void drop_file_write_access(struct file *file) return; if (file_check_writeable(file) != 0) return; - mnt_drop_write(mnt); + __mnt_drop_write(mnt); file_release_write(file); } @@ -225,7 +227,7 @@ static void __fput(struct file *file) { struct dentry *dentry = file->f_path.dentry; struct vfsmount *mnt = file->f_path.mnt; - struct inode *inode = dentry->d_inode; + struct inode *inode = file->f_inode; might_sleep(); @@ -241,142 +243,105 @@ static void __fput(struct file *file) if (file->f_op && file->f_op->fasync) file->f_op->fasync(-1, file, 0); } + ima_file_free(file); if (file->f_op && file->f_op->release) file->f_op->release(inode, file); security_file_free(file); - ima_file_free(file); if (unlikely(S_ISCHR(inode->i_mode) && inode->i_cdev != NULL && !(file->f_mode & FMODE_PATH))) { cdev_put(inode->i_cdev); } fops_put(file->f_op); put_pid(file->f_owner.pid); - file_sb_list_del(file); if ((file->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ) i_readcount_dec(inode); if (file->f_mode & FMODE_WRITE) drop_file_write_access(file); file->f_path.dentry = NULL; file->f_path.mnt = NULL; + file->f_inode = NULL; file_free(file); dput(dentry); mntput(mnt); } -void fput(struct file *file) +static LLIST_HEAD(delayed_fput_list); +static void delayed_fput(struct work_struct *unused) { - if (atomic_long_dec_and_test(&file->f_count)) - __fput(file); -} + struct llist_node *node = llist_del_all(&delayed_fput_list); + struct llist_node *next; -EXPORT_SYMBOL(fput); - -struct file *fget(unsigned int fd) -{ - struct file *file; - struct files_struct *files = current->files; - - rcu_read_lock(); - file = fcheck_files(files, fd); - if (file) { - /* File object ref couldn't be taken */ - if (file->f_mode & FMODE_PATH || - !atomic_long_inc_not_zero(&file->f_count)) - file = NULL; + for (; node; node = next) { + next = llist_next(node); + __fput(llist_entry(node, struct file, f_u.fu_llist)); } - rcu_read_unlock(); - - return file; } -EXPORT_SYMBOL(fget); - -struct file *fget_raw(unsigned int fd) +static void ____fput(struct callback_head *work) { - struct file *file; - struct files_struct *files = current->files; - - rcu_read_lock(); - file = fcheck_files(files, fd); - if (file) { - /* File object ref couldn't be taken */ - if (!atomic_long_inc_not_zero(&file->f_count)) - file = NULL; - } - rcu_read_unlock(); - - return file; + __fput(container_of(work, struct file, f_u.fu_rcuhead)); } -EXPORT_SYMBOL(fget_raw); - /* - * Lightweight file lookup - no refcnt increment if fd table isn't shared. - * - * You can use this instead of fget if you satisfy all of the following - * conditions: - * 1) You must call fput_light before exiting the syscall and returning control - * to userspace (i.e. you cannot remember the returned struct file * after - * returning to userspace). - * 2) You must not call filp_close on the returned struct file * in between - * calls to fget_light and fput_light. - * 3) You must not clone the current task in between the calls to fget_light - * and fput_light. - * - * The fput_needed flag returned by fget_light should be passed to the - * corresponding fput_light. + * If kernel thread really needs to have the final fput() it has done + * to complete, call this. The only user right now is the boot - we + * *do* need to make sure our writes to binaries on initramfs has + * not left us with opened struct file waiting for __fput() - execve() + * won't work without that. Please, don't add more callers without + * very good reasons; in particular, never call that with locks + * held and never call that from a thread that might need to do + * some work on any kind of umount. */ -struct file *fget_light(unsigned int fd, int *fput_needed) +void flush_delayed_fput(void) { - struct file *file; - struct files_struct *files = current->files; - - *fput_needed = 0; - if (atomic_read(&files->count) == 1) { - file = fcheck_files(files, fd); - if (file && (file->f_mode & FMODE_PATH)) - file = NULL; - } else { - rcu_read_lock(); - file = fcheck_files(files, fd); - if (file) { - if (!(file->f_mode & FMODE_PATH) && - atomic_long_inc_not_zero(&file->f_count)) - *fput_needed = 1; - else - /* Didn't get the reference, someone's freed */ - file = NULL; - } - rcu_read_unlock(); - } - - return file; + delayed_fput(NULL); } -struct file *fget_raw_light(unsigned int fd, int *fput_needed) +static DECLARE_WORK(delayed_fput_work, delayed_fput); + +void fput(struct file *file) { - struct file *file; - struct files_struct *files = current->files; - - *fput_needed = 0; - if (atomic_read(&files->count) == 1) { - file = fcheck_files(files, fd); - } else { - rcu_read_lock(); - file = fcheck_files(files, fd); - if (file) { - if (atomic_long_inc_not_zero(&file->f_count)) - *fput_needed = 1; - else - /* Didn't get the reference, someone's freed */ - file = NULL; + if (atomic_long_dec_and_test(&file->f_count)) { + struct task_struct *task = current; + + file_sb_list_del(file); + if (likely(!in_interrupt() && !(task->flags & PF_KTHREAD))) { + init_task_work(&file->f_u.fu_rcuhead, ____fput); + if (!task_work_add(task, &file->f_u.fu_rcuhead, true)) + return; + /* + * After this task has run exit_task_work(), + * task_work_add() will fail. free_ipc_ns()-> + * shm_destroy() can do this. Fall through to delayed + * fput to avoid leaking *file. + */ } - rcu_read_unlock(); + + if (llist_add(&file->f_u.fu_llist, &delayed_fput_list)) + schedule_work(&delayed_fput_work); } +} - return file; +/* + * synchronous analog of fput(); for kernel threads that might be needed + * in some umount() (and thus can't use flush_delayed_fput() without + * risking deadlocks), need to wait for completion of __fput() and know + * for this specific struct file it won't involve anything that would + * need them. Use only if you really need it - at the very least, + * don't blindly convert fput() by kernel thread to that. + */ +void __fput_sync(struct file *file) +{ + if (atomic_long_dec_and_test(&file->f_count)) { + struct task_struct *task = current; + file_sb_list_del(file); + BUG_ON(!(task->flags & PF_KTHREAD)); + __fput(file); + } } +EXPORT_SYMBOL(fput); + void put_filp(struct file *file) { if (atomic_long_dec_and_test(&file->f_count)) { @@ -483,11 +448,9 @@ void mark_files_ro(struct super_block *sb) { struct file *f; -retry: lg_global_lock(&files_lglock); do_file_list_for_each_entry(sb, f) { - struct vfsmount *mnt; - if (!S_ISREG(f->f_path.dentry->d_inode->i_mode)) + if (!S_ISREG(file_inode(f)->i_mode)) continue; if (!file_count(f)) continue; @@ -498,13 +461,8 @@ retry: spin_unlock(&f->f_lock); if (file_check_writeable(f) != 0) continue; + __mnt_drop_write(f->f_path.mnt); file_release_write(f); - mnt = mntget(f->f_path.mnt); - /* This can sleep, so we can't hold the spinlock. */ - lg_global_unlock(&files_lglock); - mnt_drop_write(mnt); - mntput(mnt); - goto retry; } while_file_list_for_each_entry; lg_global_unlock(&files_lglock); } |