// SPDX-License-Identifier: GPL-2.0 /* * I/O and data path helper functionality. * * Borrowed from NFS Copyright (c) 2016 Trond Myklebust */ #include #include #include "internal.h" struct netfs_wb_waiter { struct list_head link; /* Link in ictx->wb_queue */ struct task_struct *waiter; /* Waiter task; cleared when lock granted */ }; /* * inode_dio_wait_interruptible - wait for outstanding DIO requests to finish * @inode: inode to wait for * * Waits for all pending direct I/O requests to finish so that we can * proceed with a truncate or equivalent operation. * * Must be called under a lock that serializes taking new references * to i_dio_count, usually by inode->i_mutex. */ static int netfs_inode_dio_wait_interruptible(struct inode *inode) { if (inode_dio_finished(inode)) return 0; inode_dio_wait_interruptible(inode); return !inode_dio_finished(inode) ? -ERESTARTSYS : 0; } /* Call with exclusively locked inode->i_rwsem */ static int netfs_block_o_direct(struct netfs_inode *ictx) { if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) return 0; clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags); return netfs_inode_dio_wait_interruptible(&ictx->inode); } /** * netfs_start_io_read - declare the file is being used for buffered reads * @inode: file inode * * Declare that a buffered read operation is about to start, and ensure * that we block all direct I/O. * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is unset, * and holds a shared lock on inode->i_rwsem to ensure that the flag * cannot be changed. * In practice, this means that buffered read operations are allowed to * execute in parallel, thanks to the shared lock, whereas direct I/O * operations need to wait to grab an exclusive lock in order to set * NETFS_ICTX_ODIRECT. * Note that buffered writes and truncates both take a write lock on * inode->i_rwsem, meaning that those are serialised w.r.t. the reads. */ int netfs_start_io_read(struct inode *inode) __acquires(inode->i_rwsem) { struct netfs_inode *ictx = netfs_inode(inode); /* Be an optimist! */ if (down_read_interruptible(&inode->i_rwsem) < 0) return -ERESTARTSYS; if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) == 0) return 0; up_read(&inode->i_rwsem); /* Slow path.... */ if (down_write_killable(&inode->i_rwsem) < 0) return -ERESTARTSYS; if (netfs_block_o_direct(ictx) < 0) { up_write(&inode->i_rwsem); return -ERESTARTSYS; } downgrade_write(&inode->i_rwsem); return 0; } EXPORT_SYMBOL(netfs_start_io_read); /** * netfs_end_io_read - declare that the buffered read operation is done * @inode: file inode * * Declare that a buffered read operation is done, and release the shared * lock on inode->i_rwsem. */ void netfs_end_io_read(struct inode *inode) __releases(inode->i_rwsem) { up_read(&inode->i_rwsem); } EXPORT_SYMBOL(netfs_end_io_read); /** * netfs_start_io_write - declare the file is being used for buffered writes * @inode: file inode * * Declare that a buffered read operation is about to start, and ensure * that we block all direct I/O. */ int netfs_start_io_write(struct inode *inode) __acquires(inode->i_rwsem) { struct netfs_inode *ictx = netfs_inode(inode); if (down_write_killable(&inode->i_rwsem) < 0) return -ERESTARTSYS; if (netfs_block_o_direct(ictx) < 0) { up_write(&inode->i_rwsem); return -ERESTARTSYS; } downgrade_write(&inode->i_rwsem); return 0; } EXPORT_SYMBOL(netfs_start_io_write); /** * netfs_end_io_write - declare that the buffered write operation is done * @inode: file inode * * Declare that a buffered write operation is done, and release the * lock on inode->i_rwsem. */ void netfs_end_io_write(struct inode *inode) __releases(inode->i_rwsem) { up_read(&inode->i_rwsem); } EXPORT_SYMBOL(netfs_end_io_write); /* Call with exclusively locked inode->i_rwsem */ static int netfs_block_buffered(struct inode *inode) { struct netfs_inode *ictx = netfs_inode(inode); int ret; if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) { set_bit(NETFS_ICTX_ODIRECT, &ictx->flags); if (inode->i_mapping->nrpages != 0) { unmap_mapping_range(inode->i_mapping, 0, 0, 0); ret = filemap_fdatawait(inode->i_mapping); if (ret < 0) { clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags); return ret; } } } return 0; } /** * netfs_start_io_direct - declare the file is being used for direct i/o * @inode: file inode * * Declare that a direct I/O operation is about to start, and ensure * that we block all buffered I/O. * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is set, * and holds a shared lock on inode->i_rwsem to ensure that the flag * cannot be changed. * In practice, this means that direct I/O operations are allowed to * execute in parallel, thanks to the shared lock, whereas buffered I/O * operations need to wait to grab an exclusive lock in order to clear * NETFS_ICTX_ODIRECT. * Note that buffered writes and truncates both take a write lock on * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT. */ int netfs_start_io_direct(struct inode *inode) __acquires(inode->i_rwsem) { struct netfs_inode *ictx = netfs_inode(inode); int ret; /* Be an optimist! */ if (down_read_interruptible(&inode->i_rwsem) < 0) return -ERESTARTSYS; if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) != 0) return 0; up_read(&inode->i_rwsem); /* Slow path.... */ if (down_write_killable(&inode->i_rwsem) < 0) return -ERESTARTSYS; ret = netfs_block_buffered(inode); if (ret < 0) { up_write(&inode->i_rwsem); return ret; } downgrade_write(&inode->i_rwsem); return 0; } EXPORT_SYMBOL(netfs_start_io_direct); /** * netfs_end_io_direct - declare that the direct i/o operation is done * @inode: file inode * * Declare that a direct I/O operation is done, and release the shared * lock on inode->i_rwsem. */ void netfs_end_io_direct(struct inode *inode) __releases(inode->i_rwsem) { up_read(&inode->i_rwsem); } EXPORT_SYMBOL(netfs_end_io_direct); /* * Wait to have exclusive access to writeback. */ static bool netfs_wb_begin_wait(struct netfs_inode *ictx) { struct netfs_wb_waiter waiter = {}; struct task_struct *tsk = current; bool got = false; spin_lock(&ictx->lock); if (test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags)) { get_task_struct(tsk); waiter.waiter = tsk; list_add_tail(&waiter.link, &ictx->wb_queue); } else { got = true; } spin_unlock(&ictx->lock); if (!got) { for (;;) { set_current_state(TASK_UNINTERRUPTIBLE); /* Read waiter before accessing inode state. */ if (smp_load_acquire(&waiter.waiter) == NULL) break; schedule(); } } __set_current_state(TASK_RUNNING); return true; } /** * netfs_wb_begin - Begin writeback, waiting if need be * @ictx: The inode to get writeback access on * @nowait: Return failure immediately rather than waiting if true * * Begin writeback to an inode, waiting for exclusive access if @nowait is * false. This prevents collection from being done out of order with respect * to the issuance of write subrequests. * * Note that writeback may be ended in a different process (e.g. the collection * function on a workqueue) than started it. * * Return: True if can proceed, false if denied. */ bool netfs_wb_begin(struct netfs_inode *ictx, bool nowait) { if (!test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags)) return true; if (nowait) { netfs_stat(&netfs_n_wb_lock_skip); return false; } netfs_stat(&netfs_n_wb_lock_wait); return netfs_wb_begin_wait(ictx); } EXPORT_SYMBOL(netfs_wb_begin); /* netfs_wb_end - End writeback * @ictx: The inode we have writeback access to * * End writeback access on an inode, waking up the next writeback request. */ void netfs_wb_end(struct netfs_inode *ictx) { struct netfs_wb_waiter *waiter; struct task_struct *tsk; WARN_ON_ONCE(!test_bit(NETFS_ICTX_WB_LOCK, &ictx->flags)); spin_lock(&ictx->lock); waiter = list_first_entry_or_null(&ictx->wb_queue, struct netfs_wb_waiter, link); if (waiter) { list_del(&waiter->link); tsk = waiter->waiter; /* Write inode state before clearing waiter. */ smp_store_release(&waiter->waiter, NULL); wake_up_process(tsk); put_task_struct(tsk); } else { clear_bit_unlock(NETFS_ICTX_WB_LOCK, &ictx->flags); } spin_unlock(&ictx->lock); } EXPORT_SYMBOL(netfs_wb_end);