summaryrefslogtreecommitdiff
path: root/io_uring
diff options
context:
space:
mode:
Diffstat (limited to 'io_uring')
-rw-r--r--io_uring/eventfd.c8
-rw-r--r--io_uring/eventfd.h2
-rw-r--r--io_uring/futex.c21
-rw-r--r--io_uring/io_uring.c7
-rw-r--r--io_uring/poll.c23
-rw-r--r--io_uring/rsrc.c31
-rw-r--r--io_uring/waitid.c2
7 files changed, 46 insertions, 48 deletions
diff --git a/io_uring/eventfd.c b/io_uring/eventfd.c
index ab789e1ebe91..7a57dd88304a 100644
--- a/io_uring/eventfd.c
+++ b/io_uring/eventfd.c
@@ -50,9 +50,9 @@ static void io_eventfd_do_signal(struct rcu_head *rcu)
/*
* Returns true if the caller should put the ev_fd reference, false if not.
*/
-static bool __io_eventfd_signal(struct io_ev_fd *ev_fd)
+static bool __io_eventfd_signal(struct io_ev_fd *ev_fd, bool defer)
{
- if (eventfd_signal_allowed()) {
+ if (!defer && eventfd_signal_allowed()) {
eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);
return true;
}
@@ -72,7 +72,7 @@ static bool io_eventfd_trigger(struct io_ev_fd *ev_fd)
return !ev_fd->eventfd_async || io_wq_current_is_worker();
}
-void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event)
+void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer)
{
bool skip = false;
struct io_ev_fd *ev_fd;
@@ -112,7 +112,7 @@ void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event)
spin_unlock(&ctx->completion_lock);
}
- if (skip || __io_eventfd_signal(ev_fd))
+ if (skip || __io_eventfd_signal(ev_fd, defer))
io_eventfd_put(ev_fd);
}
diff --git a/io_uring/eventfd.h b/io_uring/eventfd.h
index e2f1985c2cf9..86c3c69eba1d 100644
--- a/io_uring/eventfd.h
+++ b/io_uring/eventfd.h
@@ -4,4 +4,4 @@ int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,
unsigned int eventfd_async);
int io_eventfd_unregister(struct io_ring_ctx *ctx);
-void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event);
+void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer);
diff --git a/io_uring/futex.c b/io_uring/futex.c
index 57a17c694221..3e92c2a63960 100644
--- a/io_uring/futex.c
+++ b/io_uring/futex.c
@@ -149,14 +149,16 @@ int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
+ struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
int ret;
ret = io_futex_prep(req, sqe);
if (unlikely(ret))
return ret;
- /* Mark as inflight, so file exit cancelation will find it */
- io_req_track_inflight(req);
+ /* inflight tracking only needed for mm private hash */
+ if (!(iof->futex_flags & FLAGS_SHARED))
+ io_req_track_inflight(req);
return 0;
}
@@ -172,13 +174,14 @@ static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q)
io_req_set_res(req, 0, 0);
req->io_task_work.func = io_futexv_complete;
- io_req_task_work_add(req);
+ __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE);
}
int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
struct futex_vector *futexv;
+ unsigned int i;
int ret;
/* No flags or mask supported for waitv */
@@ -202,8 +205,14 @@ int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return ret;
}
- /* Mark as inflight, so file exit cancelation will find it */
- io_req_track_inflight(req);
+ /* inflight tracking only needed for mm private hash */
+ for (i = 0; i < iof->futex_nr; i++) {
+ if (!(futexv[i].w.flags & FLAGS_SHARED)) {
+ io_req_track_inflight(req);
+ break;
+ }
+ }
+
iof->futexv_owned = 0;
iof->futexv_unqueued = 0;
req->flags |= REQ_F_ASYNC_DATA;
@@ -221,7 +230,7 @@ static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q)
io_req_set_res(req, 0, 0);
req->io_task_work.func = io_futex_complete;
- io_req_task_work_add(req);
+ __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE);
}
int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags)
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 7f398c4a3a6e..9784463fb718 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -578,7 +578,7 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
if (ctx->off_timeout_used)
io_flush_timeouts(ctx);
if (ctx->has_evfd)
- io_eventfd_signal(ctx, true);
+ io_eventfd_signal(ctx, true, false);
}
static inline void __io_cq_lock(struct io_ring_ctx *ctx)
@@ -1312,7 +1312,7 @@ static void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
if (!head) {
io_ctx_mark_taskrun(ctx);
if (ctx->has_evfd)
- io_eventfd_signal(ctx, false);
+ io_eventfd_signal(ctx, false, flags & IOU_F_TWQ_IN_WAKE);
}
nr_wait = atomic_read(&ctx->cq_wait_nr);
@@ -3867,8 +3867,7 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
static_branch_deferred_inc(&io_key_has_sqarray);
if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&
- !(ctx->flags & IORING_SETUP_IOPOLL) &&
- !(ctx->flags & IORING_SETUP_SQPOLL))
+ !(ctx->flags & IORING_SETUP_IOPOLL))
ctx->task_complete = true;
if (ctx->task_complete || (ctx->flags & IORING_SETUP_IOPOLL))
diff --git a/io_uring/poll.c b/io_uring/poll.c
index a5e78747e63a..6745bbc96328 100644
--- a/io_uring/poll.c
+++ b/io_uring/poll.c
@@ -190,9 +190,9 @@ enum {
IOU_POLL_REQUEUE = 4,
};
-static void __io_poll_execute(struct io_kiocb *req, int mask)
+static void __io_poll_execute(struct io_kiocb *req, int mask, unsigned tw_flags)
{
- unsigned flags = 0;
+ unsigned flags = tw_flags;
io_req_set_res(req, mask, 0);
req->io_task_work.func = io_poll_task_func;
@@ -200,14 +200,15 @@ static void __io_poll_execute(struct io_kiocb *req, int mask)
trace_io_uring_task_add(req, mask);
if (!(req->flags & REQ_F_POLL_NO_LAZY))
- flags = IOU_F_TWQ_LAZY_WAKE;
+ flags |= IOU_F_TWQ_LAZY_WAKE;
__io_req_task_work_add(req, flags);
}
-static inline void io_poll_execute(struct io_kiocb *req, int res)
+static inline void io_poll_execute(struct io_kiocb *req, int res,
+ unsigned tw_flags)
{
if (io_poll_get_ownership(req))
- __io_poll_execute(req, res);
+ __io_poll_execute(req, res, tw_flags);
}
/*
@@ -323,7 +324,7 @@ void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw)
if (ret == IOU_POLL_NO_ACTION) {
return;
} else if (ret == IOU_POLL_REQUEUE) {
- __io_poll_execute(req, 0);
+ __io_poll_execute(req, 0, 0);
return;
}
io_poll_remove_entries(req);
@@ -362,7 +363,7 @@ static void io_poll_cancel_req(struct io_kiocb *req)
{
io_poll_mark_cancelled(req);
/* kick tw, which should complete the request */
- io_poll_execute(req, 0);
+ io_poll_execute(req, 0, 0);
}
#define IO_ASYNC_POLL_COMMON (EPOLLONESHOT | EPOLLPRI)
@@ -371,7 +372,7 @@ static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll)
{
io_poll_mark_cancelled(req);
/* we have to kick tw in case it's not already */
- io_poll_execute(req, 0);
+ io_poll_execute(req, 0, IOU_F_TWQ_IN_WAKE);
/*
* If the waitqueue is being freed early but someone is already
@@ -426,7 +427,7 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
else
req->flags &= ~REQ_F_SINGLE_POLL;
}
- __io_poll_execute(req, mask);
+ __io_poll_execute(req, mask, IOU_F_TWQ_IN_WAKE);
}
return 1;
}
@@ -614,7 +615,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req,
if (mask && (poll->events & EPOLLET) &&
io_poll_can_finish_inline(req, ipt)) {
- __io_poll_execute(req, mask);
+ __io_poll_execute(req, mask, 0);
return 0;
}
io_napi_add(req);
@@ -625,7 +626,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req,
* poll was waken up, queue up a tw, it'll deal with it.
*/
if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1)
- __io_poll_execute(req, 0);
+ __io_poll_execute(req, 0, 0);
}
return 0;
}
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index b6a070abbf99..8d28c6bfebb1 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -94,20 +94,6 @@ int io_validate_user_buf_range(u64 uaddr, u64 ulen)
return 0;
}
-static int io_buffer_validate(struct iovec *iov)
-{
- /*
- * Don't impose further limits on the size and buffer
- * constraints here, we'll -EINVAL later when IO is
- * submitted if they are wrong.
- */
- if (!iov->iov_base)
- return iov->iov_len ? -EFAULT : 0;
-
- return io_validate_user_buf_range((unsigned long)iov->iov_base,
- iov->iov_len);
-}
-
static void io_release_ubuf(void *priv)
{
struct io_mapped_ubuf *imu = priv;
@@ -317,9 +303,6 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
err = -EFAULT;
break;
}
- err = io_buffer_validate(iov);
- if (err)
- break;
node = io_sqe_buffer_register(ctx, iov, &last_hpage);
if (IS_ERR(node)) {
err = PTR_ERR(node);
@@ -788,8 +771,17 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
struct io_imu_folio_data data;
bool coalesced = false;
- if (!iov->iov_base)
+ if (!iov->iov_base) {
+ if (iov->iov_len)
+ return ERR_PTR(-EFAULT);
+ /* remove the buffer without installing a new one */
return NULL;
+ }
+
+ ret = io_validate_user_buf_range((unsigned long)iov->iov_base,
+ iov->iov_len);
+ if (ret)
+ return ERR_PTR(ret);
node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
if (!node)
@@ -895,9 +887,6 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
ret = PTR_ERR(iov);
break;
}
- ret = io_buffer_validate(iov);
- if (ret)
- break;
if (ctx->compat)
arg += sizeof(struct compat_iovec);
else
diff --git a/io_uring/waitid.c b/io_uring/waitid.c
index 921b4de3a31c..97cce0bd2d50 100644
--- a/io_uring/waitid.c
+++ b/io_uring/waitid.c
@@ -237,7 +237,7 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode,
return 1;
req->io_task_work.func = io_waitid_cb;
- io_req_task_work_add(req);
+ __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE);
return 1;
}
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-04 11:05:59 +0000