admin/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2025-12-07 11:56:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'for-6.19/io_uring-20251201' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux

Browse Source 
Pull io_uring updates from Jens Axboe:

 - Unify how task_work cancelations are detected, placing it in the
   task_work running state rather than needing to check the task state

 - Series cleaning up and moving the cancelation code to where it
   belongs, in cancel.c

 - Cleanup of waitid and futex argument handling

 - Add support for mixed sized SQEs. 6.18 added support for mixed sized
   CQEs, improving flexibility and efficiency of workloads that need big
   CQEs. This adds similar support for SQEs, where the occasional need
   for a 128b SQE doesn't necessitate having all SQEs be 128b in size

 - Introduce zcrx and SQ/CQ layout queries. The former returns what zcrx
   features are available. And both return the ring size information to
   help with allocation size calculation for user provided rings like
   IORING_SETUP_NO_MMAP and IORING_MEM_REGION_TYPE_USER

 - Zcrx updates for 6.19. It includes a bunch of small patches,
   IORING_REGISTER_ZCRX_CTRL and RQ flushing and David's work on sharing
   zcrx b/w multiple io_uring instances

 - Series cleaning up ring initializations, notable deduplicating ring
   size and offset calculations. It also moves most of the checking
   before doing any allocations, making the code simpler

 - Add support for getsockname and getpeername, which is mostly a
   trivial hookup after a bit of refactoring on the networking side

 - Various fixes and cleanups

* tag 'for-6.19/io_uring-20251201' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: (68 commits)
  io_uring: Introduce getsockname io_uring cmd
  socket: Split out a getsockname helper for io_uring
  socket: Unify getsockname and getpeername implementation
  io_uring/query: drop unused io_handle_query_entry() ctx arg
  io_uring/kbuf: remove obsolete buf_nr_pages and update comments
  io_uring/register: use correct location for io_rings_layout
  io_uring/zcrx: share an ifq between rings
  io_uring/zcrx: add io_fill_zcrx_offsets()
  io_uring/zcrx: export zcrx via a file
  io_uring/zcrx: move io_zcrx_scrub() and dependencies up
  io_uring/zcrx: count zcrx users
  io_uring/zcrx: add sync refill queue flushing
  io_uring/zcrx: introduce IORING_REGISTER_ZCRX_CTRL
  io_uring/zcrx: elide passing msg flags
  io_uring/zcrx: use folio_nr_pages() instead of shift operation
  io_uring/zcrx: convert to use netmem_desc
  io_uring/query: introduce rings info query
  io_uring/query: introduce zcrx query
  io_uring: move cq/sq user offset init around
  io_uring: pre-calculate scq layout
  ...
This commit is contained in:
Linus Torvalds
2025-12-03 18:58:57 -08:00
parent 8f7aa3d3c7 5d24321e4c
commit 0abcfd8983
46 changed files with 1308 additions and 865 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
block/ioctl.c
View File

@@ -769,14 +769,16 @@ struct blk_iou_cmd {
bool nowait;
};
static void blk_cmd_complete(struct io_uring_cmd *cmd, unsigned int issue_flags)
static void blk_cmd_complete(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
struct blk_iou_cmd *bic = io_uring_cmd_to_pdu(cmd, struct blk_iou_cmd);
if (bic->res == -EAGAIN && bic->nowait)
io_uring_cmd_issue_blocking(cmd);
else
io_uring_cmd_done(cmd, bic->res, issue_flags);
io_uring_cmd_done(cmd, bic->res,
IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
}
static void bio_cmd_bio_end_io(struct bio *bio)

22
drivers/block/ublk_drv.c
View File

@@ -1302,10 +1302,9 @@ static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req,
return true;
}
static void ublk_dispatch_req(struct ublk_queue *ubq,
struct request *req,
unsigned int issue_flags)
static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req)
{
unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
int tag = req->tag;
struct ublk_io *io = &ubq->ios[tag];
@@ -1348,13 +1347,13 @@ static void ublk_dispatch_req(struct ublk_queue *ubq,
ublk_complete_io_cmd(io, req, UBLK_IO_RES_OK, issue_flags);
}
static void ublk_cmd_tw_cb(struct io_uring_cmd *cmd,
unsigned int issue_flags)
static void ublk_cmd_tw_cb(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
struct ublk_queue *ubq = pdu->ubq;
ublk_dispatch_req(ubq, pdu->req, issue_flags);
ublk_dispatch_req(ubq, pdu->req);
}
static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq)
@@ -1366,9 +1365,9 @@ static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq)
io_uring_cmd_complete_in_task(cmd, ublk_cmd_tw_cb);
}
static void ublk_cmd_list_tw_cb(struct io_uring_cmd *cmd,
unsigned int issue_flags)
static void ublk_cmd_list_tw_cb(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
struct request *rq = pdu->req_list;
struct request *next;
@@ -1376,7 +1375,7 @@ static void ublk_cmd_list_tw_cb(struct io_uring_cmd *cmd,
do {
next = rq->rq_next;
rq->rq_next = NULL;
ublk_dispatch_req(rq->mq_hctx->driver_data, rq, issue_flags);
ublk_dispatch_req(rq->mq_hctx->driver_data, rq);
rq = next;
} while (rq);
}
@@ -2523,9 +2522,10 @@ fail_put:
return NULL;
}
static void ublk_ch_uring_cmd_cb(struct io_uring_cmd *cmd,
unsigned int issue_flags)
static void ublk_ch_uring_cmd_cb(struct io_tw_req tw_req, io_tw_token_t tw)
{
unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
int ret = ublk_ch_uring_cmd_local(cmd, issue_flags);
if (ret != -EIOCBQUEUED)

7
drivers/nvme/host/ioctl.c
View File

@@ -398,14 +398,15 @@ static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu(
return io_uring_cmd_to_pdu(ioucmd, struct nvme_uring_cmd_pdu);
}
static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd,
unsigned issue_flags)
static void nvme_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_uring_cmd *ioucmd = io_uring_cmd_from_tw(tw_req);
struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
if (pdu->bio)
blk_rq_unmap_user(pdu->bio);
io_uring_cmd_done32(ioucmd, pdu->status, pdu->result, issue_flags);
io_uring_cmd_done32(ioucmd, pdu->status, pdu->result,
IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
}
static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,

5
fs/btrfs/ioctl.c
View File

@@ -4632,8 +4632,9 @@ struct io_btrfs_cmd {
struct btrfs_uring_priv *priv;
};
static void btrfs_uring_read_finished(struct io_uring_cmd *cmd, unsigned int issue_flags)
static void btrfs_uring_read_finished(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
struct io_btrfs_cmd *bc = io_uring_cmd_to_pdu(cmd, struct io_btrfs_cmd);
struct btrfs_uring_priv *priv = bc->priv;
struct btrfs_inode *inode = BTRFS_I(file_inode(priv->iocb.ki_filp));
@@ -4678,7 +4679,7 @@ out:
btrfs_unlock_extent(io_tree, priv->start, priv->lockend, &priv->cached_state);
btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
io_uring_cmd_done(cmd, ret, issue_flags);
io_uring_cmd_done(cmd, ret, IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
add_rchar(current, ret);
for (index = 0; index < priv->nr_pages; index++)

7
fs/fuse/dev_uring.c
View File

@@ -1209,14 +1209,15 @@ static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd,
* User buffers are not mapped yet - the application does not have permission
* to write to it - this has to be executed in ring task context.
*/
static void fuse_uring_send_in_task(struct io_uring_cmd *cmd,
unsigned int issue_flags)
static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw)
{
unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd);
struct fuse_ring_queue *queue = ent->queue;
int err;
if (!(issue_flags & IO_URING_F_TASK_DEAD)) {
if (!tw.cancel) {
err = fuse_uring_prepare_send(ent, ent->fuse_req);
if (err) {
fuse_uring_next_fuse_req(ent, queue, issue_flags);

22
include/linux/io_uring/cmd.h
View File

@@ -11,17 +11,13 @@
/* io_uring_cmd is being issued again */
#define IORING_URING_CMD_REISSUE (1U << 31)
typedef void (*io_uring_cmd_tw_t)(struct io_uring_cmd *cmd,
unsigned issue_flags);
struct io_uring_cmd {
struct file *file;
const struct io_uring_sqe *sqe;
/* callback to defer completions to task context */
io_uring_cmd_tw_t task_work_cb;
u32 cmd_op;
u32 flags;
u8 pdu[32]; /* available inline for free use */
u8 unused[8];
};
static inline const void *io_uring_sqe_cmd(const struct io_uring_sqe *sqe)
@@ -60,7 +56,7 @@ void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, u64 res2,
unsigned issue_flags, bool is_cqe32);
void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
io_uring_cmd_tw_t task_work_cb,
io_req_tw_func_t task_work_cb,
unsigned flags);
/*
@@ -109,7 +105,7 @@ static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret,
{
}
static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
io_uring_cmd_tw_t task_work_cb, unsigned flags)
io_req_tw_func_t task_work_cb, unsigned flags)
{
}
static inline void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
@@ -132,15 +128,23 @@ static inline bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd,
}
#endif
static inline struct io_uring_cmd *io_uring_cmd_from_tw(struct io_tw_req tw_req)
{
return io_kiocb_to_cmd(tw_req.req, struct io_uring_cmd);
}
/* task_work executor checks the deferred list completion */
#define IO_URING_CMD_TASK_WORK_ISSUE_FLAGS IO_URING_F_COMPLETE_DEFER
/* users must follow the IOU_F_TWQ_LAZY_WAKE semantics */
static inline void io_uring_cmd_do_in_task_lazy(struct io_uring_cmd *ioucmd,
io_uring_cmd_tw_t task_work_cb)
io_req_tw_func_t task_work_cb)
{
__io_uring_cmd_do_in_task(ioucmd, task_work_cb, IOU_F_TWQ_LAZY_WAKE);
}
static inline void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd,
io_uring_cmd_tw_t task_work_cb)
io_req_tw_func_t task_work_cb)
{
__io_uring_cmd_do_in_task(ioucmd, task_work_cb, 0);
}

12
include/linux/io_uring_types.h
View File

@@ -39,7 +39,6 @@ enum io_uring_cmd_flags {
/* set when uring wants to cancel a previously issued command */
IO_URING_F_CANCEL = (1 << 11),
IO_URING_F_COMPAT = (1 << 12),
IO_URING_F_TASK_DEAD = (1 << 13),
};
struct io_wq_work_node {
@@ -328,8 +327,8 @@ struct io_ring_ctx {
/*
* Modifications are protected by ->uring_lock and ->mmap_lock.
* The flags, buf_pages and buf_nr_pages fields should be stable
* once published.
* The buffer list's io mapped region should be stable once
* published.
*/
struct xarray io_bl_xa;
@@ -474,6 +473,7 @@ struct io_ring_ctx {
* ONLY core io_uring.c should instantiate this struct.
*/
struct io_tw_state {
bool cancel;
};
/* Alias to use in code that doesn't instantiate struct io_tw_state */
typedef struct io_tw_state io_tw_token_t;
@@ -614,7 +614,11 @@ enum {
REQ_F_SQE_COPIED = IO_REQ_FLAG(REQ_F_SQE_COPIED_BIT),
};
typedef void (*io_req_tw_func_t)(struct io_kiocb *req, io_tw_token_t tw);
struct io_tw_req {
struct io_kiocb *req;
};
typedef void (*io_req_tw_func_t)(struct io_tw_req tw_req, io_tw_token_t tw);
struct io_task_work {
struct llist_node node;

1
include/linux/netdevice.h
View File

@@ -3417,6 +3417,7 @@ struct net_device *dev_get_by_index(struct net *net, int ifindex);
struct net_device *__dev_get_by_index(struct net *net, int ifindex);
struct net_device *netdev_get_by_index(struct net *net, int ifindex,
netdevice_tracker *tracker, gfp_t gfp);
struct net_device *netdev_get_by_index_lock(struct net *net, int ifindex);
struct net_device *netdev_get_by_name(struct net *net, const char *name,
netdevice_tracker *tracker, gfp_t gfp);
struct net_device *netdev_get_by_flags_rcu(struct net *net, netdevice_tracker *tracker,

6
include/linux/socket.h
View File

@@ -468,10 +468,10 @@ extern int __sys_connect(int fd, struct sockaddr __user *uservaddr,
int addrlen);
extern int __sys_listen(int fd, int backlog);
extern int __sys_listen_socket(struct socket *sock, int backlog);
extern int do_getsockname(struct socket *sock, int peer,
struct sockaddr __user *usockaddr, int __user *usockaddr_len);
extern int __sys_getsockname(int fd, struct sockaddr __user *usockaddr,
int __user *usockaddr_len);
extern int __sys_getpeername(int fd, struct sockaddr __user *usockaddr,
int __user *usockaddr_len);
int __user *usockaddr_len, int peer);
extern int __sys_socketpair(int family, int type, int protocol,
int __user *usockvec);
extern int __sys_shutdown_sock(struct socket *sock, int how);

43
include/uapi/linux/io_uring.h
View File

@@ -231,6 +231,12 @@ enum io_uring_sqe_flags_bit {
*/
#define IORING_SETUP_CQE_MIXED (1U << 18)
/*
* Allow both 64b and 128b SQEs. If a 128b SQE is posted, it will have
* a 128b opcode.
*/
#define IORING_SETUP_SQE_MIXED (1U << 19)
enum io_uring_op {
IORING_OP_NOP,
IORING_OP_READV,
@@ -295,6 +301,8 @@ enum io_uring_op {
IORING_OP_READV_FIXED,
IORING_OP_WRITEV_FIXED,
IORING_OP_PIPE,
IORING_OP_NOP128,
IORING_OP_URING_CMD128,
/* this goes last, obviously */
IORING_OP_LAST,
@@ -689,6 +697,9 @@ enum io_uring_register_op {
/* query various aspects of io_uring, see linux/io_uring/query.h */
IORING_REGISTER_QUERY = 35,
/* auxiliary zcrx configuration, see enum zcrx_ctrl_op */
IORING_REGISTER_ZCRX_CTRL = 36,
/* this goes last */
IORING_REGISTER_LAST,
@@ -998,6 +1009,7 @@ enum io_uring_socket_op {
SOCKET_URING_OP_GETSOCKOPT,
SOCKET_URING_OP_SETSOCKOPT,
SOCKET_URING_OP_TX_TIMESTAMP,
SOCKET_URING_OP_GETSOCKNAME,
};
/*
@@ -1052,6 +1064,10 @@ struct io_uring_zcrx_area_reg {
__u64 __resv2[2];
};
enum zcrx_reg_flags {
ZCRX_REG_IMPORT = 1,
};
/*
* Argument for IORING_REGISTER_ZCRX_IFQ
*/
@@ -1070,6 +1086,33 @@ struct io_uring_zcrx_ifq_reg {
__u64 __resv[3];
};
enum zcrx_ctrl_op {
ZCRX_CTRL_FLUSH_RQ,
ZCRX_CTRL_EXPORT,
__ZCRX_CTRL_LAST,
};
struct zcrx_ctrl_flush_rq {
__u64 __resv[6];
};
struct zcrx_ctrl_export {
__u32 zcrx_fd;
__u32 __resv1[11];
};
struct zcrx_ctrl {
__u32 zcrx_id;
__u32 op; /* see enum zcrx_ctrl_op */
__u64 __resv[2];
union {
struct zcrx_ctrl_export zc_export;
struct zcrx_ctrl_flush_rq zc_flush;
};
};
#ifdef __cplusplus
}
#endif

24
include/uapi/linux/io_uring/query.h
View File

@@ -18,6 +18,8 @@ struct io_uring_query_hdr {
enum {
IO_URING_QUERY_OPCODES = 0,
IO_URING_QUERY_ZCRX = 1,
IO_URING_QUERY_SCQ = 2,
__IO_URING_QUERY_MAX,
};
@@ -41,4 +43,26 @@ struct io_uring_query_opcode {
__u32 __pad;
};
struct io_uring_query_zcrx {
/* Bitmask of supported ZCRX_REG_* flags, */
__u64 register_flags;
/* Bitmask of all supported IORING_ZCRX_AREA_* flags */
__u64 area_flags;
/* The number of supported ZCRX_CTRL_* opcodes */
__u32 nr_ctrl_opcodes;
__u32 __resv1;
/* The refill ring header size */
__u32 rq_hdr_size;
/* The alignment for the header */
__u32 rq_hdr_alignment;
__u64 __resv2;
};
struct io_uring_query_scq {
/* The SQ/CQ rings header size */
__u64 hdr_size;
/* The alignment for the header */
__u64 hdr_alignment;
};
#endif

270
io_uring/cancel.c
View File

@@ -14,6 +14,8 @@
#include "filetable.h"
#include "io_uring.h"
#include "tctx.h"
#include "sqpoll.h"
#include "uring_cmd.h"
#include "poll.h"
#include "timeout.h"
#include "waitid.h"
@@ -384,3 +386,271 @@ int io_cancel_remove(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
io_ring_submit_unlock(ctx, issue_flags);
return nr ?: -ENOENT;
}
static bool io_match_linked(struct io_kiocb *head)
{
struct io_kiocb *req;
io_for_each_link(req, head) {
if (req->flags & REQ_F_INFLIGHT)
return true;
}
return false;
}
/*
* As io_match_task() but protected against racing with linked timeouts.
* User must not hold timeout_lock.
*/
bool io_match_task_safe(struct io_kiocb *head, struct io_uring_task *tctx,
bool cancel_all)
{
bool matched;
if (tctx && head->tctx != tctx)
return false;
if (cancel_all)
return true;
if (head->flags & REQ_F_LINK_TIMEOUT) {
struct io_ring_ctx *ctx = head->ctx;
/* protect against races with linked timeouts */
raw_spin_lock_irq(&ctx->timeout_lock);
matched = io_match_linked(head);
raw_spin_unlock_irq(&ctx->timeout_lock);
} else {
matched = io_match_linked(head);
}
return matched;
}
void __io_uring_cancel(bool cancel_all)
{
io_uring_unreg_ringfd();
io_uring_cancel_generic(cancel_all, NULL);
}
struct io_task_cancel {
struct io_uring_task *tctx;
bool all;
};
static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
{
struct io_kiocb *req = container_of(work, struct io_kiocb, work);
struct io_task_cancel *cancel = data;
return io_match_task_safe(req, cancel->tctx, cancel->all);
}
static __cold bool io_cancel_defer_files(struct io_ring_ctx *ctx,
struct io_uring_task *tctx,
bool cancel_all)
{
struct io_defer_entry *de;
LIST_HEAD(list);
list_for_each_entry_reverse(de, &ctx->defer_list, list) {
if (io_match_task_safe(de->req, tctx, cancel_all)) {
list_cut_position(&list, &ctx->defer_list, &de->list);
break;
}
}
if (list_empty(&list))
return false;
while (!list_empty(&list)) {
de = list_first_entry(&list, struct io_defer_entry, list);
list_del_init(&de->list);
ctx->nr_drained -= io_linked_nr(de->req);
io_req_task_queue_fail(de->req, -ECANCELED);
kfree(de);
}
return true;
}
__cold bool io_cancel_ctx_cb(struct io_wq_work *work, void *data)
{
struct io_kiocb *req = container_of(work, struct io_kiocb, work);
return req->ctx == data;
}
static __cold bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)
{
struct io_tctx_node *node;
enum io_wq_cancel cret;
bool ret = false;
mutex_lock(&ctx->uring_lock);
list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
struct io_uring_task *tctx = node->task->io_uring;
/*
* io_wq will stay alive while we hold uring_lock, because it's
* killed after ctx nodes, which requires to take the lock.
*/
if (!tctx || !tctx->io_wq)
continue;
cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true);
ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
}
mutex_unlock(&ctx->uring_lock);
return ret;
}
__cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
struct io_uring_task *tctx,
bool cancel_all, bool is_sqpoll_thread)
{
struct io_task_cancel cancel = { .tctx = tctx, .all = cancel_all, };
enum io_wq_cancel cret;
bool ret = false;
/* set it so io_req_local_work_add() would wake us up */
if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
atomic_set(&ctx->cq_wait_nr, 1);
smp_mb();
}
/* failed during ring init, it couldn't have issued any requests */
if (!ctx->rings)
return false;
if (!tctx) {
ret |= io_uring_try_cancel_iowq(ctx);
} else if (tctx->io_wq) {
/*
* Cancels requests of all rings, not only @ctx, but
* it's fine as the task is in exit/exec.
*/
cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
&cancel, true);
ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
}
/* SQPOLL thread does its own polling */
if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
is_sqpoll_thread) {
while (!wq_list_empty(&ctx->iopoll_list)) {
io_iopoll_try_reap_events(ctx);
ret = true;
cond_resched();
}
}
if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&
io_allowed_defer_tw_run(ctx))
ret |= io_run_local_work(ctx, INT_MAX, INT_MAX) > 0;
mutex_lock(&ctx->uring_lock);
ret |= io_cancel_defer_files(ctx, tctx, cancel_all);
ret |= io_poll_remove_all(ctx, tctx, cancel_all);
ret |= io_waitid_remove_all(ctx, tctx, cancel_all);
ret |= io_futex_remove_all(ctx, tctx, cancel_all);
ret |= io_uring_try_cancel_uring_cmd(ctx, tctx, cancel_all);
mutex_unlock(&ctx->uring_lock);
ret |= io_kill_timeouts(ctx, tctx, cancel_all);
if (tctx)
ret |= io_run_task_work() > 0;
else
ret |= flush_delayed_work(&ctx->fallback_work);
return ret;
}
static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)
{
if (tracked)
return atomic_read(&tctx->inflight_tracked);
return percpu_counter_sum(&tctx->inflight);
}
/*
* Find any io_uring ctx that this task has registered or done IO on, and cancel
* requests. @sqd should be not-null IFF it's an SQPOLL thread cancellation.
*/
__cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
{
struct io_uring_task *tctx = current->io_uring;
struct io_ring_ctx *ctx;
struct io_tctx_node *node;
unsigned long index;
s64 inflight;
DEFINE_WAIT(wait);
WARN_ON_ONCE(sqd && sqpoll_task_locked(sqd) != current);
if (!current->io_uring)
return;
if (tctx->io_wq)
io_wq_exit_start(tctx->io_wq);
atomic_inc(&tctx->in_cancel);
do {
bool loop = false;
io_uring_drop_tctx_refs(current);
if (!tctx_inflight(tctx, !cancel_all))
break;
/* read completions before cancelations */
inflight = tctx_inflight(tctx, false);
if (!inflight)
break;
if (!sqd) {
xa_for_each(&tctx->xa, index, node) {
/* sqpoll task will cancel all its requests */
if (node->ctx->sq_data)
continue;
loop |= io_uring_try_cancel_requests(node->ctx,
current->io_uring,
cancel_all,
false);
}
} else {
list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
loop |= io_uring_try_cancel_requests(ctx,
current->io_uring,
cancel_all,
true);
}
if (loop) {
cond_resched();
continue;
}
prepare_to_wait(&tctx->wait, &wait, TASK_INTERRUPTIBLE);
io_run_task_work();
io_uring_drop_tctx_refs(current);
xa_for_each(&tctx->xa, index, node) {
if (io_local_work_pending(node->ctx)) {
WARN_ON_ONCE(node->ctx->submitter_task &&
node->ctx->submitter_task != current);
goto end_wait;
}
}
/*
* If we've seen completions, retry without waiting. This
* avoids a race where a completion comes in before we did
* prepare_to_wait().
*/
if (inflight == tctx_inflight(tctx, !cancel_all))
schedule();
end_wait:
finish_wait(&tctx->wait, &wait);
} while (1);
io_uring_clean_tctx(tctx);
if (cancel_all) {
/*
* We shouldn't run task_works after cancel, so just leave
* ->in_cancel set for normal exit.
*/
atomic_dec(&tctx->in_cancel);
/* for exec all current's requests should be gone, kill tctx */
__io_uring_free(current);
}
}

8
io_uring/cancel.h
View File

@@ -23,14 +23,20 @@ int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd,
int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg);
bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd);
bool io_match_task_safe(struct io_kiocb *head, struct io_uring_task *tctx,
bool cancel_all);
bool io_cancel_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
struct hlist_head *list, bool cancel_all,
bool (*cancel)(struct io_kiocb *));
int io_cancel_remove(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
unsigned int issue_flags, struct hlist_head *list,
bool (*cancel)(struct io_kiocb *));
__cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
struct io_uring_task *tctx,
bool cancel_all, bool is_sqpoll_thread);
__cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);
__cold bool io_cancel_ctx_cb(struct io_wq_work *work, void *data);
static inline bool io_cancel_match_sequence(struct io_kiocb *req, int sequence)
{

22
io_uring/cmd_net.c
View File

@@ -132,6 +132,26 @@ static int io_uring_cmd_timestamp(struct socket *sock,
return -EAGAIN;
}
static int io_uring_cmd_getsockname(struct socket *sock,
struct io_uring_cmd *cmd,
unsigned int issue_flags)
{
const struct io_uring_sqe *sqe = cmd->sqe;
struct sockaddr __user *uaddr;
unsigned int peer;
int __user *ulen;
if (sqe->ioprio || sqe->__pad1 || sqe->len || sqe->rw_flags)
return -EINVAL;
uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr));
ulen = u64_to_user_ptr(sqe->addr3);
peer = READ_ONCE(sqe->optlen);
if (peer > 1)
return -EINVAL;
return do_getsockname(sock, peer, uaddr, ulen);
}
int io_uring_cmd_sock(struct io_uring_cmd *cmd, unsigned int issue_flags)
{
struct socket *sock = cmd->file->private_data;
@@ -159,6 +179,8 @@ int io_uring_cmd_sock(struct io_uring_cmd *cmd, unsigned int issue_flags)
return io_uring_cmd_setsockopt(sock, cmd, issue_flags);
case SOCKET_URING_OP_TX_TIMESTAMP:
return io_uring_cmd_timestamp(sock, cmd, issue_flags);
case SOCKET_URING_OP_GETSOCKNAME:
return io_uring_cmd_getsockname(sock, cmd, issue_flags);
default:
return -EOPNOTSUPP;
}

37
io_uring/fdinfo.c
View File

@@ -5,6 +5,7 @@
#include <linux/file.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/nospec.h>
#include <linux/io_uring.h>
#include <uapi/linux/io_uring.h>
@@ -14,6 +15,7 @@
#include "fdinfo.h"
#include "cancel.h"
#include "rsrc.h"
#include "opdef.h"
#ifdef CONFIG_NET_RX_BUSY_POLL
static __cold void common_tracking_show_fdinfo(struct io_ring_ctx *ctx,
@@ -93,21 +95,46 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
unsigned int entry = i + sq_head;
struct io_uring_sqe *sqe;
unsigned int sq_idx;
bool sqe128 = false;
u8 opcode;
if (ctx->flags & IORING_SETUP_NO_SQARRAY)
break;
sq_idx = READ_ONCE(ctx->sq_array[entry & sq_mask]);
sq_idx = entry & sq_mask;
else
sq_idx = READ_ONCE(ctx->sq_array[entry & sq_mask]);
if (sq_idx > sq_mask)
continue;
sqe = &ctx->sq_sqes[sq_idx << sq_shift];
opcode = READ_ONCE(sqe->opcode);
if (opcode >= IORING_OP_LAST)
continue;
opcode = array_index_nospec(opcode, IORING_OP_LAST);
if (sq_shift) {
sqe128 = true;
} else if (io_issue_defs[opcode].is_128) {
if (!(ctx->flags & IORING_SETUP_SQE_MIXED)) {
seq_printf(m,
"%5u: invalid sqe, 128B entry on non-mixed sq\n",
sq_idx);
break;
}
if ((++sq_head & sq_mask) == 0) {
seq_printf(m,
"%5u: corrupted sqe, wrapping 128B entry\n",
sq_idx);
break;
}
sqe128 = true;
}
seq_printf(m, "%5u: opcode:%s, fd:%d, flags:%x, off:%llu, "
"addr:0x%llx, rw_flags:0x%x, buf_index:%d "
"user_data:%llu",
sq_idx, io_uring_get_opcode(sqe->opcode), sqe->fd,
sq_idx, io_uring_get_opcode(opcode), sqe->fd,
sqe->flags, (unsigned long long) sqe->off,
(unsigned long long) sqe->addr, sqe->rw_flags,
sqe->buf_index, sqe->user_data);
if (sq_shift) {
if (sqe128) {
u64 *sqeb = (void *) (sqe + 1);
int size = sizeof(struct io_uring_sqe) / sizeof(u64);
int j;
@@ -128,7 +155,7 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
cqe = &r->cqes[(cq_head & cq_mask)];
if (cqe->flags & IORING_CQE_F_32 || ctx->flags & IORING_SETUP_CQE32)
cqe32 = true;
seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x",
seq_printf(m, "%5u: user_data:%llu, res:%d, flags:%x",
cq_head & cq_mask, cqe->user_data, cqe->res,
cqe->flags);
if (cqe32)

57
io_uring/futex.c
View File

@@ -17,7 +17,6 @@ struct io_futex {
void __user *uaddr;
unsigned long futex_val;
unsigned long futex_mask;
unsigned long futexv_owned;
u32 futex_flags;
unsigned int futex_nr;
bool futexv_unqueued;
@@ -28,6 +27,11 @@ struct io_futex_data {
struct io_kiocb *req;
};
struct io_futexv_data {
unsigned long owned;
struct futex_vector futexv[];
};
#define IO_FUTEX_ALLOC_CACHE_MAX 32
bool io_futex_cache_init(struct io_ring_ctx *ctx)
@@ -41,45 +45,46 @@ void io_futex_cache_free(struct io_ring_ctx *ctx)
io_alloc_cache_free(&ctx->futex_cache, kfree);
}
static void __io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)
static void __io_futex_complete(struct io_tw_req tw_req, io_tw_token_t tw)
{
hlist_del_init(&req->hash_node);
io_req_task_complete(req, tw);
hlist_del_init(&tw_req.req->hash_node);
io_req_task_complete(tw_req, tw);
}
static void io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)
static void io_futex_complete(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_kiocb *req = tw_req.req;
struct io_ring_ctx *ctx = req->ctx;
io_tw_lock(ctx, tw);
io_cache_free(&ctx->futex_cache, req->async_data);
io_req_async_data_clear(req, 0);
__io_futex_complete(req, tw);
__io_futex_complete(tw_req, tw);
}
static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw)
static void io_futexv_complete(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_kiocb *req = tw_req.req;
struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
struct futex_vector *futexv = req->async_data;
struct io_futexv_data *ifd = req->async_data;
io_tw_lock(req->ctx, tw);
if (!iof->futexv_unqueued) {
int res;
res = futex_unqueue_multiple(futexv, iof->futex_nr);
res = futex_unqueue_multiple(ifd->futexv, iof->futex_nr);
if (res != -1)
io_req_set_res(req, res, 0);
}
io_req_async_data_free(req);
__io_futex_complete(req, tw);
__io_futex_complete(tw_req, tw);
}
static bool io_futexv_claim(struct io_futex *iof)
static bool io_futexv_claim(struct io_futexv_data *ifd)
{
if (test_bit(0, &iof->futexv_owned) ||
test_and_set_bit_lock(0, &iof->futexv_owned))
if (test_bit(0, &ifd->owned) || test_and_set_bit_lock(0, &ifd->owned))
return false;
return true;
}
@@ -94,9 +99,9 @@ static bool __io_futex_cancel(struct io_kiocb *req)
return false;
req->io_task_work.func = io_futex_complete;
} else {
struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
struct io_futexv_data *ifd = req->async_data;
if (!io_futexv_claim(iof))
if (!io_futexv_claim(ifd))
return false;
req->io_task_work.func = io_futexv_complete;
}
@@ -152,9 +157,9 @@ int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q)
{
struct io_kiocb *req = q->wake_data;
struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
struct io_futexv_data *ifd = req->async_data;
if (!io_futexv_claim(iof))
if (!io_futexv_claim(ifd))
return;
if (unlikely(!__futex_wake_mark(q)))
return;
@@ -167,7 +172,7 @@ static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q)
int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
struct futex_vector *futexv;
struct io_futexv_data *ifd;
int ret;
/* No flags or mask supported for waitv */
@@ -180,23 +185,23 @@ int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
if (!iof->futex_nr || iof->futex_nr > FUTEX_WAITV_MAX)
return -EINVAL;
futexv = kcalloc(iof->futex_nr, sizeof(*futexv), GFP_KERNEL);
if (!futexv)
ifd = kzalloc(struct_size_t(struct io_futexv_data, futexv, iof->futex_nr),
GFP_KERNEL);
if (!ifd)
return -ENOMEM;
ret = futex_parse_waitv(futexv, iof->uaddr, iof->futex_nr,
ret = futex_parse_waitv(ifd->futexv, iof->uaddr, iof->futex_nr,
io_futex_wakev_fn, req);
if (ret) {
kfree(futexv);
kfree(ifd);
return ret;
}
/* Mark as inflight, so file exit cancelation will find it */
io_req_track_inflight(req);
iof->futexv_owned = 0;
iof->futexv_unqueued = 0;
req->flags |= REQ_F_ASYNC_DATA;
req->async_data = futexv;
req->async_data = ifd;
return 0;
}
@@ -216,13 +221,13 @@ static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q)
int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags)
{
struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
struct futex_vector *futexv = req->async_data;
struct io_futexv_data *ifd = req->async_data;
struct io_ring_ctx *ctx = req->ctx;
int ret, woken = -1;
io_ring_submit_lock(ctx, issue_flags);
ret = futex_wait_multiple_setup(futexv, iof->futex_nr, &woken);
ret = futex_wait_multiple_setup(ifd->futexv, iof->futex_nr, &woken);
/*
* Error case, ret is < 0. Mark the request as failed.

549
io_uring/io_uring.c
View File

@@ -124,11 +124,6 @@
#define IO_REQ_ALLOC_BATCH 8
#define IO_LOCAL_TW_DEFAULT_MAX 20
struct io_defer_entry {
struct list_head list;
struct io_kiocb *req;
};
/* requests with any of those set should undergo io_disarm_next() */
#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)
@@ -140,11 +135,6 @@ struct io_defer_entry {
/* Forced wake up if there is a waiter regardless of ->cq_wait_nr */
#define IO_CQ_WAKE_FORCE (IO_CQ_WAKE_INIT >> 1)
static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
struct io_uring_task *tctx,
bool cancel_all,
bool is_sqpoll_thread);
static void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags);
static void __io_req_caches_free(struct io_ring_ctx *ctx);
@@ -207,44 +197,6 @@ static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx)
return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head);
}
static bool io_match_linked(struct io_kiocb *head)
{
struct io_kiocb *req;
io_for_each_link(req, head) {
if (req->flags & REQ_F_INFLIGHT)
return true;
}
return false;
}
/*
* As io_match_task() but protected against racing with linked timeouts.
* User must not hold timeout_lock.
*/
bool io_match_task_safe(struct io_kiocb *head, struct io_uring_task *tctx,
bool cancel_all)
{
bool matched;
if (tctx && head->tctx != tctx)
return false;
if (cancel_all)
return true;
if (head->flags & REQ_F_LINK_TIMEOUT) {
struct io_ring_ctx *ctx = head->ctx;
/* protect against races with linked timeouts */
raw_spin_lock_irq(&ctx->timeout_lock);
matched = io_match_linked(head);
raw_spin_unlock_irq(&ctx->timeout_lock);
} else {
matched = io_match_linked(head);
}
return matched;
}
static inline void req_fail_link_node(struct io_kiocb *req, int res)
{
req_set_fail(req);
@@ -265,6 +217,20 @@ static __cold void io_ring_ctx_ref_free(struct percpu_ref *ref)
complete(&ctx->ref_comp);
}
/*
* Terminate the request if either of these conditions are true:
*
* 1) It's being executed by the original task, but that task is marked
* with PF_EXITING as it's exiting.
* 2) PF_KTHREAD is set, in which case the invoker of the task_work is
* our fallback task_work.
* 3) The ring has been closed and is going away.
*/
static inline bool io_should_terminate_tw(struct io_ring_ctx *ctx)
{
return (current->flags & (PF_EXITING | PF_KTHREAD)) || percpu_ref_is_dying(&ctx->refs);
}
static __cold void io_fallback_req_func(struct work_struct *work)
{
struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
@@ -275,8 +241,9 @@ static __cold void io_fallback_req_func(struct work_struct *work)
percpu_ref_get(&ctx->refs);
mutex_lock(&ctx->uring_lock);
ts.cancel = io_should_terminate_tw(ctx);
llist_for_each_entry_safe(req, tmp, node, io_task_work.node)
req->io_task_work.func(req, ts);
req->io_task_work.func((struct io_tw_req){req}, ts);
io_submit_flush_completions(ctx);
mutex_unlock(&ctx->uring_lock);
percpu_ref_put(&ctx->refs);
@@ -524,9 +491,9 @@ static void io_queue_iowq(struct io_kiocb *req)
io_wq_enqueue(tctx->io_wq, &req->work);
}
static void io_req_queue_iowq_tw(struct io_kiocb *req, io_tw_token_t tw)
static void io_req_queue_iowq_tw(struct io_tw_req tw_req, io_tw_token_t tw)
{
io_queue_iowq(req);
io_queue_iowq(tw_req.req);
}
void io_req_queue_iowq(struct io_kiocb *req)
@@ -535,7 +502,7 @@ void io_req_queue_iowq(struct io_kiocb *req)
io_req_task_work_add(req);
}
static unsigned io_linked_nr(struct io_kiocb *req)
unsigned io_linked_nr(struct io_kiocb *req)
{
struct io_kiocb *tmp;
unsigned nr = 0;
@@ -706,7 +673,7 @@ void io_task_refs_refill(struct io_uring_task *tctx)
tctx->cached_refs += refill;
}
static __cold void io_uring_drop_tctx_refs(struct task_struct *task)
__cold void io_uring_drop_tctx_refs(struct task_struct *task)
{
struct io_uring_task *tctx = task->io_uring;
unsigned int refs = tctx->cached_refs;
@@ -917,7 +884,7 @@ bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags
}
/*
* Must be called from inline task_work so we now a flush will happen later,
* Must be called from inline task_work so we know a flush will happen later,
* and obviously with ctx->uring_lock held (tw always has that).
*/
void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
@@ -1149,10 +1116,11 @@ struct llist_node *io_handle_tw_list(struct llist_node *node,
ctx = req->ctx;
mutex_lock(&ctx->uring_lock);
percpu_ref_get(&ctx->refs);
ts.cancel = io_should_terminate_tw(ctx);
}
INDIRECT_CALL_2(req->io_task_work.func,
io_poll_task_func, io_req_rw_complete,
req, ts);
(struct io_tw_req){req}, ts);
node = next;
(*count)++;
if (unlikely(need_resched())) {
@@ -1207,11 +1175,6 @@ struct llist_node *tctx_task_work_run(struct io_uring_task *tctx,
{
struct llist_node *node;
if (unlikely(current->flags & PF_EXITING)) {
io_fallback_tw(tctx, true);
return NULL;
}
node = llist_del_all(&tctx->task_list);
if (node) {
node = llist_reverse_order(node);
@@ -1248,7 +1211,7 @@ static void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
BUILD_BUG_ON(IO_CQ_WAKE_FORCE <= IORING_MAX_CQ_ENTRIES);
/*
* We don't know how many reuqests is there in the link and whether
* We don't know how many requests there are in the link and whether
* they can even be queued lazily, fall back to non-lazy.
*/
if (req->flags & IO_REQ_LINK_FLAGS)
@@ -1380,7 +1343,7 @@ static int __io_run_local_work_loop(struct llist_node **node,
io_task_work.node);
INDIRECT_CALL_2(req->io_task_work.func,
io_poll_task_func, io_req_rw_complete,
req, tw);
(struct io_tw_req){req}, tw);
*node = next;
if (++ret >= events)
break;
@@ -1401,6 +1364,7 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw,
if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
again:
tw.cancel = io_should_terminate_tw(ctx);
min_events -= ret;
ret = __io_run_local_work_loop(&ctx->retry_llist.first, tw, max_events);
if (ctx->retry_llist.first)
@@ -1437,8 +1401,7 @@ static inline int io_run_local_work_locked(struct io_ring_ctx *ctx,
max(IO_LOCAL_TW_DEFAULT_MAX, min_events));
}
static int io_run_local_work(struct io_ring_ctx *ctx, int min_events,
int max_events)
int io_run_local_work(struct io_ring_ctx *ctx, int min_events, int max_events)
{
struct io_tw_state ts = {};
int ret;
@@ -1449,18 +1412,21 @@ static int io_run_local_work(struct io_ring_ctx *ctx, int min_events,
return ret;
}
static void io_req_task_cancel(struct io_kiocb *req, io_tw_token_t tw)
static void io_req_task_cancel(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_kiocb *req = tw_req.req;
io_tw_lock(req->ctx, tw);
io_req_defer_failed(req, req->cqe.res);
}
void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw)
void io_req_task_submit(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_kiocb *req = tw_req.req;
struct io_ring_ctx *ctx = req->ctx;
io_tw_lock(ctx, tw);
if (unlikely(io_should_terminate_tw(ctx)))
if (unlikely(tw.cancel))
io_req_defer_failed(req, -EFAULT);
else if (req->flags & REQ_F_FORCE_ASYNC)
io_queue_iowq(req);
@@ -1589,7 +1555,7 @@ static unsigned io_cqring_events(struct io_ring_ctx *ctx)
* We can't just wait for polled events to come to us, we have to actively
* find and complete them.
*/
static __cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
__cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
{
if (!(ctx->flags & IORING_SETUP_IOPOLL))
return;
@@ -1692,9 +1658,9 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned int min_events)
return 0;
}
void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw)
void io_req_task_complete(struct io_tw_req tw_req, io_tw_token_t tw)
{
io_req_complete_defer(req);
io_req_complete_defer(tw_req.req);
}
/*
@@ -2153,7 +2119,7 @@ static __cold int io_init_fail_req(struct io_kiocb *req, int err)
}
static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
const struct io_uring_sqe *sqe)
const struct io_uring_sqe *sqe, unsigned int *left)
__must_hold(&ctx->uring_lock)
{
const struct io_issue_def *def;
@@ -2179,6 +2145,24 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
opcode = array_index_nospec(opcode, IORING_OP_LAST);
def = &io_issue_defs[opcode];
if (def->is_128 && !(ctx->flags & IORING_SETUP_SQE128)) {
/*
* A 128b op on a non-128b SQ requires mixed SQE support as
* well as 2 contiguous entries.
*/
if (!(ctx->flags & IORING_SETUP_SQE_MIXED) || *left < 2 ||
!(ctx->cached_sq_head & (ctx->sq_entries - 1)))
return io_init_fail_req(req, -EINVAL);
/*
* A 128b operation on a mixed SQ uses two entries, so we have
* to increment the head and cached refs, and decrement what's
* left.
*/
current->io_uring->cached_refs++;
ctx->cached_sq_head++;
(*left)--;
}
if (unlikely(sqe_flags & ~SQE_COMMON_FLAGS)) {
/* enforce forwards compatibility on users */
if (sqe_flags & ~SQE_VALID_FLAGS)
@@ -2288,13 +2272,13 @@ static __cold int io_submit_fail_init(const struct io_uring_sqe *sqe,
}
static inline int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
const struct io_uring_sqe *sqe)
const struct io_uring_sqe *sqe, unsigned int *left)
__must_hold(&ctx->uring_lock)
{
struct io_submit_link *link = &ctx->submit_state.link;
int ret;
ret = io_init_req(ctx, req, sqe);
ret = io_init_req(ctx, req, sqe, left);
if (unlikely(ret))
return io_submit_fail_init(sqe, req, ret);
@@ -2424,10 +2408,11 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
unsigned int left;
int ret;
entries = min(nr, entries);
if (unlikely(!entries))
return 0;
/* make sure SQ entry isn't read before tail */
ret = left = min(nr, entries);
ret = left = entries;
io_get_task_refs(left);
io_submit_state_start(&ctx->submit_state, left);
@@ -2446,7 +2431,7 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
* Continue submitting even for sqe failure if the
* ring was setup with IORING_SETUP_SUBMIT_ALL
*/
if (unlikely(io_submit_sqe(ctx, req, sqe)) &&
if (unlikely(io_submit_sqe(ctx, req, sqe, &left)) &&
!(ctx->flags & IORING_SETUP_SUBMIT_ALL)) {
left--;
break;
@@ -2768,51 +2753,63 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
static void io_rings_free(struct io_ring_ctx *ctx)
{
io_free_region(ctx, &ctx->sq_region);
io_free_region(ctx, &ctx->ring_region);
io_free_region(ctx->user, &ctx->sq_region);
io_free_region(ctx->user, &ctx->ring_region);
ctx->rings = NULL;
ctx->sq_sqes = NULL;
}
unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
unsigned int cq_entries, size_t *sq_offset)
static int rings_size(unsigned int flags, unsigned int sq_entries,
unsigned int cq_entries, struct io_rings_layout *rl)
{
struct io_rings *rings;
size_t off, sq_array_size;
size_t sqe_size;
size_t off;
off = struct_size(rings, cqes, cq_entries);
if (off == SIZE_MAX)
return SIZE_MAX;
if (flags & IORING_SETUP_CQE32) {
if (check_shl_overflow(off, 1, &off))
return SIZE_MAX;
}
if (flags & IORING_SETUP_CQE_MIXED) {
if (cq_entries < 2)
return SIZE_MAX;
return -EOVERFLOW;
}
if (flags & IORING_SETUP_SQE_MIXED) {
if (sq_entries < 2)
return -EOVERFLOW;
}
rl->sq_array_offset = SIZE_MAX;
sqe_size = sizeof(struct io_uring_sqe);
if (flags & IORING_SETUP_SQE128)
sqe_size *= 2;
rl->sq_size = array_size(sqe_size, sq_entries);
if (rl->sq_size == SIZE_MAX)
return -EOVERFLOW;
off = struct_size(rings, cqes, cq_entries);
if (flags & IORING_SETUP_CQE32)
off = size_mul(off, 2);
if (off == SIZE_MAX)
return -EOVERFLOW;
#ifdef CONFIG_SMP
off = ALIGN(off, SMP_CACHE_BYTES);
if (off == 0)
return SIZE_MAX;
return -EOVERFLOW;
#endif
if (flags & IORING_SETUP_NO_SQARRAY) {
*sq_offset = SIZE_MAX;
return off;
if (!(flags & IORING_SETUP_NO_SQARRAY)) {
size_t sq_array_size;
rl->sq_array_offset = off;
sq_array_size = array_size(sizeof(u32), sq_entries);
off = size_add(off, sq_array_size);
if (off == SIZE_MAX)
return -EOVERFLOW;
}
*sq_offset = off;
sq_array_size = array_size(sizeof(u32), sq_entries);
if (sq_array_size == SIZE_MAX)
return SIZE_MAX;
if (check_add_overflow(off, sq_array_size, &off))
return SIZE_MAX;
return off;
rl->rings_size = off;
return 0;
}
static __cold void __io_req_caches_free(struct io_ring_ctx *ctx)
@@ -2850,7 +2847,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
io_eventfd_unregister(ctx);
io_free_alloc_caches(ctx);
io_destroy_buffers(ctx);
io_free_region(ctx, &ctx->param_region);
io_free_region(ctx->user, &ctx->param_region);
mutex_unlock(&ctx->uring_lock);
if (ctx->sq_creds)
put_cred(ctx->sq_creds);
@@ -2980,13 +2977,6 @@ static __cold void io_tctx_exit_cb(struct callback_head *cb)
complete(&work->completion);
}
static __cold bool io_cancel_ctx_cb(struct io_wq_work *work, void *data)
{
struct io_kiocb *req = container_of(work, struct io_kiocb, work);
return req->ctx == data;
}
static __cold void io_ring_exit_work(struct work_struct *work)
{
struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
@@ -3008,11 +2998,6 @@ static __cold void io_ring_exit_work(struct work_struct *work)
io_cqring_overflow_kill(ctx);
mutex_unlock(&ctx->uring_lock);
}
if (!xa_empty(&ctx->zcrx_ctxs)) {
mutex_lock(&ctx->uring_lock);
io_shutdown_zcrx_ifqs(ctx);
mutex_unlock(&ctx->uring_lock);
}
if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
io_move_task_work_from_local(ctx);
@@ -3120,230 +3105,6 @@ static int io_uring_release(struct inode *inode, struct file *file)
return 0;
}
struct io_task_cancel {
struct io_uring_task *tctx;
bool all;
};
static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
{
struct io_kiocb *req = container_of(work, struct io_kiocb, work);
struct io_task_cancel *cancel = data;
return io_match_task_safe(req, cancel->tctx, cancel->all);
}
static __cold bool io_cancel_defer_files(struct io_ring_ctx *ctx,
struct io_uring_task *tctx,
bool cancel_all)
{
struct io_defer_entry *de;
LIST_HEAD(list);
list_for_each_entry_reverse(de, &ctx->defer_list, list) {
if (io_match_task_safe(de->req, tctx, cancel_all)) {
list_cut_position(&list, &ctx->defer_list, &de->list);
break;
}
}
if (list_empty(&list))
return false;
while (!list_empty(&list)) {
de = list_first_entry(&list, struct io_defer_entry, list);
list_del_init(&de->list);
ctx->nr_drained -= io_linked_nr(de->req);
io_req_task_queue_fail(de->req, -ECANCELED);
kfree(de);
}
return true;
}
static __cold bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)
{
struct io_tctx_node *node;
enum io_wq_cancel cret;
bool ret = false;
mutex_lock(&ctx->uring_lock);
list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
struct io_uring_task *tctx = node->task->io_uring;
/*
* io_wq will stay alive while we hold uring_lock, because it's
* killed after ctx nodes, which requires to take the lock.
*/
if (!tctx || !tctx->io_wq)
continue;
cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true);
ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
}
mutex_unlock(&ctx->uring_lock);
return ret;
}
static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
struct io_uring_task *tctx,
bool cancel_all,
bool is_sqpoll_thread)
{
struct io_task_cancel cancel = { .tctx = tctx, .all = cancel_all, };
enum io_wq_cancel cret;
bool ret = false;
/* set it so io_req_local_work_add() would wake us up */
if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
atomic_set(&ctx->cq_wait_nr, 1);
smp_mb();
}
/* failed during ring init, it couldn't have issued any requests */
if (!ctx->rings)
return false;
if (!tctx) {
ret |= io_uring_try_cancel_iowq(ctx);
} else if (tctx->io_wq) {
/*
* Cancels requests of all rings, not only @ctx, but
* it's fine as the task is in exit/exec.
*/
cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
&cancel, true);
ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
}
/* SQPOLL thread does its own polling */
if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
is_sqpoll_thread) {
while (!wq_list_empty(&ctx->iopoll_list)) {
io_iopoll_try_reap_events(ctx);
ret = true;
cond_resched();
}
}
if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&
io_allowed_defer_tw_run(ctx))
ret |= io_run_local_work(ctx, INT_MAX, INT_MAX) > 0;
mutex_lock(&ctx->uring_lock);
ret |= io_cancel_defer_files(ctx, tctx, cancel_all);
ret |= io_poll_remove_all(ctx, tctx, cancel_all);
ret |= io_waitid_remove_all(ctx, tctx, cancel_all);
ret |= io_futex_remove_all(ctx, tctx, cancel_all);
ret |= io_uring_try_cancel_uring_cmd(ctx, tctx, cancel_all);
mutex_unlock(&ctx->uring_lock);
ret |= io_kill_timeouts(ctx, tctx, cancel_all);
if (tctx)
ret |= io_run_task_work() > 0;
else
ret |= flush_delayed_work(&ctx->fallback_work);
return ret;
}
static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)
{
if (tracked)
return atomic_read(&tctx->inflight_tracked);
return percpu_counter_sum(&tctx->inflight);
}
/*
* Find any io_uring ctx that this task has registered or done IO on, and cancel
* requests. @sqd should be not-null IFF it's an SQPOLL thread cancellation.
*/
__cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
{
struct io_uring_task *tctx = current->io_uring;
struct io_ring_ctx *ctx;
struct io_tctx_node *node;
unsigned long index;
s64 inflight;
DEFINE_WAIT(wait);
WARN_ON_ONCE(sqd && sqpoll_task_locked(sqd) != current);
if (!current->io_uring)
return;
if (tctx->io_wq)
io_wq_exit_start(tctx->io_wq);
atomic_inc(&tctx->in_cancel);
do {
bool loop = false;
io_uring_drop_tctx_refs(current);
if (!tctx_inflight(tctx, !cancel_all))
break;
/* read completions before cancelations */
inflight = tctx_inflight(tctx, false);
if (!inflight)
break;
if (!sqd) {
xa_for_each(&tctx->xa, index, node) {
/* sqpoll task will cancel all its requests */
if (node->ctx->sq_data)
continue;
loop |= io_uring_try_cancel_requests(node->ctx,
current->io_uring,
cancel_all,
false);
}
} else {
list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
loop |= io_uring_try_cancel_requests(ctx,
current->io_uring,
cancel_all,
true);
}
if (loop) {
cond_resched();
continue;
}
prepare_to_wait(&tctx->wait, &wait, TASK_INTERRUPTIBLE);
io_run_task_work();
io_uring_drop_tctx_refs(current);
xa_for_each(&tctx->xa, index, node) {
if (io_local_work_pending(node->ctx)) {
WARN_ON_ONCE(node->ctx->submitter_task &&
node->ctx->submitter_task != current);
goto end_wait;
}
}
/*
* If we've seen completions, retry without waiting. This
* avoids a race where a completion comes in before we did
* prepare_to_wait().
*/
if (inflight == tctx_inflight(tctx, !cancel_all))
schedule();
end_wait:
finish_wait(&tctx->wait, &wait);
} while (1);
io_uring_clean_tctx(tctx);
if (cancel_all) {
/*
* We shouldn't run task_works after cancel, so just leave
* ->in_cancel set for normal exit.
*/
atomic_dec(&tctx->in_cancel);
/* for exec all current's requests should be gone, kill tctx */
__io_uring_free(current);
}
}
void __io_uring_cancel(bool cancel_all)
{
io_uring_unreg_ringfd();
io_uring_cancel_generic(cancel_all, NULL);
}
static struct io_uring_reg_wait *io_get_ext_arg_reg(struct io_ring_ctx *ctx,
const struct io_uring_getevents_arg __user *uarg)
{
@@ -3594,24 +3355,20 @@ bool io_is_uring_fops(struct file *file)
}
static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
struct io_uring_params *p)
struct io_ctx_config *config)
{
struct io_uring_params *p = &config->p;
struct io_rings_layout *rl = &config->layout;
struct io_uring_region_desc rd;
struct io_rings *rings;
size_t size, sq_array_offset;
int ret;
/* make sure these are sane, as we already accounted them */
ctx->sq_entries = p->sq_entries;
ctx->cq_entries = p->cq_entries;
size = rings_size(ctx->flags, p->sq_entries, p->cq_entries,
&sq_array_offset);
if (size == SIZE_MAX)
return -EOVERFLOW;
memset(&rd, 0, sizeof(rd));
rd.size = PAGE_ALIGN(size);
rd.size = PAGE_ALIGN(rl->rings_size);
if (ctx->flags & IORING_SETUP_NO_MMAP) {
rd.user_addr = p->cq_off.user_addr;
rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -3620,25 +3377,11 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
if (ret)
return ret;
ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
rings->sq_ring_mask = p->sq_entries - 1;
rings->cq_ring_mask = p->cq_entries - 1;
rings->sq_ring_entries = p->sq_entries;
rings->cq_ring_entries = p->cq_entries;
if (p->flags & IORING_SETUP_SQE128)
size = array_size(2 * sizeof(struct io_uring_sqe), p->sq_entries);
else
size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
if (size == SIZE_MAX) {
io_rings_free(ctx);
return -EOVERFLOW;
}
ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);
memset(&rd, 0, sizeof(rd));
rd.size = PAGE_ALIGN(size);
rd.size = PAGE_ALIGN(rl->sq_size);
if (ctx->flags & IORING_SETUP_NO_MMAP) {
rd.user_addr = p->sq_off.user_addr;
rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -3649,6 +3392,12 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
return ret;
}
ctx->sq_sqes = io_region_get_ptr(&ctx->sq_region);
memset(rings, 0, sizeof(*rings));
WRITE_ONCE(rings->sq_ring_mask, ctx->sq_entries - 1);
WRITE_ONCE(rings->cq_ring_mask, ctx->cq_entries - 1);
WRITE_ONCE(rings->sq_ring_entries, ctx->sq_entries);
WRITE_ONCE(rings->cq_ring_entries, ctx->cq_entries);
return 0;
}
@@ -3679,6 +3428,9 @@ static int io_uring_sanitise_params(struct io_uring_params *p)
{
unsigned flags = p->flags;
if (flags & ~IORING_SETUP_FLAGS)
return -EINVAL;
/* There is no way to mmap rings without a real fd */
if ((flags & IORING_SETUP_REGISTERED_FD_ONLY) &&
!(flags & IORING_SETUP_NO_MMAP))
@@ -3717,12 +3469,21 @@ static int io_uring_sanitise_params(struct io_uring_params *p)
if ((flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED)) ==
(IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED))
return -EINVAL;
/*
* Nonsensical to ask for SQE128 and mixed SQE support, it's not
* supported to post 64b SQEs on a ring setup with SQE128.
*/
if ((flags & (IORING_SETUP_SQE128|IORING_SETUP_SQE_MIXED)) ==
(IORING_SETUP_SQE128|IORING_SETUP_SQE_MIXED))
return -EINVAL;
return 0;
}
int io_uring_fill_params(unsigned entries, struct io_uring_params *p)
static int io_uring_fill_params(struct io_uring_params *p)
{
unsigned entries = p->sq_entries;
if (!entries)
return -EINVAL;
if (entries > IORING_MAX_ENTRIES) {
@@ -3760,6 +3521,27 @@ int io_uring_fill_params(unsigned entries, struct io_uring_params *p)
p->cq_entries = 2 * p->sq_entries;
}
return 0;
}
int io_prepare_config(struct io_ctx_config *config)
{
struct io_uring_params *p = &config->p;
int ret;
ret = io_uring_sanitise_params(p);
if (ret)
return ret;
ret = io_uring_fill_params(p);
if (ret)
return ret;
ret = rings_size(p->flags, p->sq_entries, p->cq_entries,
&config->layout);
if (ret)
return ret;
p->sq_off.head = offsetof(struct io_rings, sq.head);
p->sq_off.tail = offsetof(struct io_rings, sq.tail);
p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
@@ -3780,26 +3562,24 @@ int io_uring_fill_params(unsigned entries, struct io_uring_params *p)
p->cq_off.resv1 = 0;
if (!(p->flags & IORING_SETUP_NO_MMAP))
p->cq_off.user_addr = 0;
if (!(p->flags & IORING_SETUP_NO_SQARRAY))
p->sq_off.array = config->layout.sq_array_offset;
return 0;
}
static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
struct io_uring_params __user *params)
static __cold int io_uring_create(struct io_ctx_config *config)
{
struct io_uring_params *p = &config->p;
struct io_ring_ctx *ctx;
struct io_uring_task *tctx;
struct file *file;
int ret;
ret = io_uring_sanitise_params(p);
ret = io_prepare_config(config);
if (ret)
return ret;
ret = io_uring_fill_params(entries, p);
if (unlikely(ret))
return ret;
ctx = io_ring_ctx_alloc(p);
if (!ctx)
return -ENOMEM;
@@ -3857,20 +3637,17 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
mmgrab(current->mm);
ctx->mm_account = current->mm;
ret = io_allocate_scq_urings(ctx, p);
ret = io_allocate_scq_urings(ctx, config);
if (ret)
goto err;
if (!(p->flags & IORING_SETUP_NO_SQARRAY))
p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
ret = io_sq_offload_create(ctx, p);
if (ret)
goto err;
p->features = IORING_FEAT_FLAGS;
if (copy_to_user(params, p, sizeof(*p))) {
if (copy_to_user(config->uptr, p, sizeof(*p))) {
ret = -EFAULT;
goto err;
}
@@ -3923,19 +3700,19 @@ err_fput:
*/
static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
{
struct io_uring_params p;
int i;
struct io_ctx_config config;
if (copy_from_user(&p, params, sizeof(p)))
memset(&config, 0, sizeof(config));
if (copy_from_user(&config.p, params, sizeof(config.p)))
return -EFAULT;
for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
if (p.resv[i])
return -EINVAL;
}
if (p.flags & ~IORING_SETUP_FLAGS)
if (!mem_is_zero(&config.p.resv, sizeof(config.p.resv)))
return -EINVAL;
return io_uring_create(entries, &p, params);
config.p.sq_entries = entries;
config.uptr = params;
return io_uring_create(&config);
}
static inline int io_uring_allowed(void)

63
io_uring/io_uring.h
View File

@@ -17,6 +17,20 @@
#include <trace/events/io_uring.h>
#endif
struct io_rings_layout {
/* size of CQ + headers + SQ offset array */
size_t rings_size;
size_t sq_size;
size_t sq_array_offset;
};
struct io_ctx_config {
struct io_uring_params p;
struct io_rings_layout layout;
struct io_uring_params __user *uptr;
};
#define IORING_FEAT_FLAGS (IORING_FEAT_SINGLE_MMAP |\
IORING_FEAT_NODROP |\
IORING_FEAT_SUBMIT_STABLE |\
@@ -54,7 +68,8 @@
IORING_SETUP_REGISTERED_FD_ONLY |\
IORING_SETUP_NO_SQARRAY |\
IORING_SETUP_HYBRID_IOPOLL |\
IORING_SETUP_CQE_MIXED)
IORING_SETUP_CQE_MIXED |\
IORING_SETUP_SQE_MIXED)
#define IORING_ENTER_FLAGS (IORING_ENTER_GETEVENTS |\
IORING_ENTER_SQ_WAKEUP |\
@@ -95,6 +110,11 @@ enum {
IOU_REQUEUE = -3072,
};
struct io_defer_entry {
struct list_head list;
struct io_kiocb *req;
};
struct io_wait_queue {
struct wait_queue_entry wq;
struct io_ring_ctx *ctx;
@@ -128,11 +148,11 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
#define IORING_MAX_ENTRIES 32768
#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
unsigned int cq_entries, size_t *sq_offset);
int io_uring_fill_params(unsigned entries, struct io_uring_params *p);
int io_prepare_config(struct io_ctx_config *config);
bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);
int io_run_task_work_sig(struct io_ring_ctx *ctx);
int io_run_local_work(struct io_ring_ctx *ctx, int min_events, int max_events);
void io_req_defer_failed(struct io_kiocb *req, s32 res);
bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags);
void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags);
@@ -140,6 +160,7 @@ bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags);
bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe src_cqe[2]);
void __io_commit_cqring_flush(struct io_ring_ctx *ctx);
unsigned io_linked_nr(struct io_kiocb *req);
void io_req_track_inflight(struct io_kiocb *req);
struct file *io_file_get_normal(struct io_kiocb *req, int fd);
struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
@@ -148,13 +169,13 @@ struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
void __io_req_task_work_add(struct io_kiocb *req, unsigned flags);
void io_req_task_work_add_remote(struct io_kiocb *req, unsigned flags);
void io_req_task_queue(struct io_kiocb *req);
void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw);
void io_req_task_complete(struct io_tw_req tw_req, io_tw_token_t tw);
void io_req_task_queue_fail(struct io_kiocb *req, int ret);
void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw);
void io_req_task_submit(struct io_tw_req tw_req, io_tw_token_t tw);
struct llist_node *io_handle_tw_list(struct llist_node *node, unsigned int *count, unsigned int max_entries);
struct llist_node *tctx_task_work_run(struct io_uring_task *tctx, unsigned int max_entries, unsigned int *count);
void tctx_task_work(struct callback_head *cb);
__cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);
__cold void io_uring_drop_tctx_refs(struct task_struct *task);
int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file,
int start, int end);
@@ -163,6 +184,7 @@ void io_req_queue_iowq(struct io_kiocb *req);
int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw);
int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr);
int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin);
__cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx);
void __io_submit_flush_completions(struct io_ring_ctx *ctx);
struct io_wq_work *io_wq_free_work(struct io_wq_work *work);
@@ -173,9 +195,6 @@ void io_queue_next(struct io_kiocb *req);
void io_task_refs_refill(struct io_uring_task *tctx);
bool __io_alloc_req_refill(struct io_ring_ctx *ctx);
bool io_match_task_safe(struct io_kiocb *head, struct io_uring_task *tctx,
bool cancel_all);
void io_activate_pollwq(struct io_ring_ctx *ctx);
static inline void io_lockdep_assert_cq_locked(struct io_ring_ctx *ctx)
@@ -558,19 +577,6 @@ static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx)
ctx->submitter_task == current);
}
/*
* Terminate the request if either of these conditions are true:
*
* 1) It's being executed by the original task, but that task is marked
* with PF_EXITING as it's exiting.
* 2) PF_KTHREAD is set, in which case the invoker of the task_work is
* our fallback task_work.
*/
static inline bool io_should_terminate_tw(struct io_ring_ctx *ctx)
{
return (current->flags & (PF_KTHREAD | PF_EXITING)) || percpu_ref_is_dying(&ctx->refs);
}
static inline void io_req_queue_tw_complete(struct io_kiocb *req, s32 res)
{
io_req_set_res(req, res, 0);
@@ -578,17 +584,6 @@ static inline void io_req_queue_tw_complete(struct io_kiocb *req, s32 res)
io_req_task_work_add(req);
}
/*
* IORING_SETUP_SQE128 contexts allocate twice the normal SQE size for each
* slot.
*/
static inline size_t uring_sqe_size(struct io_ring_ctx *ctx)
{
if (ctx->flags & IORING_SETUP_SQE128)
return 2 * sizeof(struct io_uring_sqe);
return sizeof(struct io_uring_sqe);
}
static inline bool io_file_can_poll(struct io_kiocb *req)
{
if (req->flags & REQ_F_CAN_POLL)

6
io_uring/kbuf.c
View File

@@ -428,7 +428,7 @@ static int io_remove_buffers_legacy(struct io_ring_ctx *ctx,
static void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl)
{
if (bl->flags & IOBL_BUF_RING)
io_free_region(ctx, &bl->region);
io_free_region(ctx->user, &bl->region);
else
io_remove_buffers_legacy(ctx, bl, -1U);
@@ -641,7 +641,7 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg)
rd.user_addr = reg.ring_addr;
rd.flags |= IORING_MEM_REGION_TYPE_USER;
}
ret = io_create_region_mmap_safe(ctx, &bl->region, &rd, mmap_offset);
ret = io_create_region(ctx, &bl->region, &rd, mmap_offset);
if (ret)
goto fail;
br = io_region_get_ptr(&bl->region);
@@ -672,7 +672,7 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg)
io_buffer_add_list(ctx, bl, reg.bgid);
return 0;
fail:
io_free_region(ctx, &bl->region);
io_free_region(ctx->user, &bl->region);
kfree(bl);
return ret;
}

5
io_uring/kbuf.h
View File

@@ -14,8 +14,8 @@ enum {
struct io_buffer_list {
/*
* If ->buf_nr_pages is set, then buf_pages/buf_ring are used. If not,
* then these are classic provided buffers and ->buf_list is used.
* If the IOBL_BUF_RING flag is set, then buf_ring is used. If not, then
* these are classic provided buffers and ->buf_list is used.
*/
union {
struct list_head buf_list;
@@ -27,7 +27,6 @@ struct io_buffer_list {
__u16 bgid;
/* below is for ring provided buffers */
__u16 buf_nr_pages;
__u16 nr_entries;
__u16 head;
__u16 mask;

59
io_uring/memmap.c
View File

@@ -15,26 +15,26 @@
#include "rsrc.h"
#include "zcrx.h"
static void *io_mem_alloc_compound(struct page **pages, int nr_pages,
size_t size, gfp_t gfp)
static bool io_mem_alloc_compound(struct page **pages, int nr_pages,
size_t size, gfp_t gfp)
{
struct page *page;
int i, order;
order = get_order(size);
if (order > MAX_PAGE_ORDER)
return ERR_PTR(-ENOMEM);
return false;
else if (order)
gfp |= __GFP_COMP;
page = alloc_pages(gfp, order);
if (!page)
return ERR_PTR(-ENOMEM);
return false;
for (i = 0; i < nr_pages; i++)
pages[i] = page + i;
return page_address(page);
return true;
}
struct page **io_pin_pages(unsigned long uaddr, unsigned long len, int *npages)
@@ -88,7 +88,7 @@ enum {
IO_REGION_F_SINGLE_REF = 4,
};
void io_free_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr)
void io_free_region(struct user_struct *user, struct io_mapped_region *mr)
{
if (mr->pages) {
long nr_refs = mr->nr_pages;
@@ -105,8 +105,8 @@ void io_free_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr)
}
if ((mr->flags & IO_REGION_F_VMAP) && mr->ptr)
vunmap(mr->ptr);
if (mr->nr_pages && ctx->user)
__io_unaccount_mem(ctx->user, mr->nr_pages);
if (mr->nr_pages && user)
__io_unaccount_mem(user, mr->nr_pages);
memset(mr, 0, sizeof(*mr));
}
@@ -131,11 +131,10 @@ static int io_region_init_ptr(struct io_mapped_region *mr)
return 0;
}
static int io_region_pin_pages(struct io_ring_ctx *ctx,
struct io_mapped_region *mr,
struct io_uring_region_desc *reg)
static int io_region_pin_pages(struct io_mapped_region *mr,
struct io_uring_region_desc *reg)
{
unsigned long size = (size_t) mr->nr_pages << PAGE_SHIFT;
size_t size = io_region_size(mr);
struct page **pages;
int nr_pages;
@@ -150,23 +149,20 @@ static int io_region_pin_pages(struct io_ring_ctx *ctx,
return 0;
}
static int io_region_allocate_pages(struct io_ring_ctx *ctx,
struct io_mapped_region *mr,
static int io_region_allocate_pages(struct io_mapped_region *mr,
struct io_uring_region_desc *reg,
unsigned long mmap_offset)
{
gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN;
size_t size = (size_t) mr->nr_pages << PAGE_SHIFT;
size_t size = io_region_size(mr);
unsigned long nr_allocated;
struct page **pages;
void *p;
pages = kvmalloc_array(mr->nr_pages, sizeof(*pages), gfp);
if (!pages)
return -ENOMEM;
p = io_mem_alloc_compound(pages, mr->nr_pages, size, gfp);
if (!IS_ERR(p)) {
if (io_mem_alloc_compound(pages, mr->nr_pages, size, gfp)) {
mr->flags |= IO_REGION_F_SINGLE_REF;
goto done;
}
@@ -219,9 +215,9 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
mr->nr_pages = nr_pages;
if (reg->flags & IORING_MEM_REGION_TYPE_USER)
ret = io_region_pin_pages(ctx, mr, reg);
ret = io_region_pin_pages(mr, reg);
else
ret = io_region_allocate_pages(ctx, mr, reg, mmap_offset);
ret = io_region_allocate_pages(mr, reg, mmap_offset);
if (ret)
goto out_free;
@@ -230,31 +226,10 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
goto out_free;
return 0;
out_free:
io_free_region(ctx, mr);
io_free_region(ctx->user, mr);
return ret;
}
int io_create_region_mmap_safe(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
struct io_uring_region_desc *reg,
unsigned long mmap_offset)
{
struct io_mapped_region tmp_mr;
int ret;
memcpy(&tmp_mr, mr, sizeof(tmp_mr));
ret = io_create_region(ctx, &tmp_mr, reg, mmap_offset);
if (ret)
return ret;
/*
* Once published mmap can find it without holding only the ->mmap_lock
* and not ->uring_lock.
*/
guard(mutex)(&ctx->mmap_lock);
memcpy(mr, &tmp_mr, sizeof(tmp_mr));
return 0;
}
static struct io_mapped_region *io_mmap_get_region(struct io_ring_ctx *ctx,
loff_t pgoff)
{

24
io_uring/memmap.h
View File

@@ -16,16 +16,11 @@ unsigned long io_uring_get_unmapped_area(struct file *file, unsigned long addr,
unsigned long flags);
int io_uring_mmap(struct file *file, struct vm_area_struct *vma);
void io_free_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr);
void io_free_region(struct user_struct *user, struct io_mapped_region *mr);
int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
struct io_uring_region_desc *reg,
unsigned long mmap_offset);
int io_create_region_mmap_safe(struct io_ring_ctx *ctx,
struct io_mapped_region *mr,
struct io_uring_region_desc *reg,
unsigned long mmap_offset);
static inline void *io_region_get_ptr(struct io_mapped_region *mr)
{
return mr->ptr;
@@ -36,4 +31,21 @@ static inline bool io_region_is_set(struct io_mapped_region *mr)
return !!mr->nr_pages;
}
static inline void io_region_publish(struct io_ring_ctx *ctx,
struct io_mapped_region *src_region,
struct io_mapped_region *dst_region)
{
/*
* Once published mmap can find it without holding only the ->mmap_lock
* and not ->uring_lock.
*/
guard(mutex)(&ctx->mmap_lock);
*dst_region = *src_region;
}
static inline size_t io_region_size(struct io_mapped_region *mr)
{
return (size_t) mr->nr_pages << PAGE_SHIFT;
}
#endif

3
io_uring/msg_ring.c
View File

@@ -70,8 +70,9 @@ static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx)
return target_ctx->task_complete;
}
static void io_msg_tw_complete(struct io_kiocb *req, io_tw_token_t tw)
static void io_msg_tw_complete(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_kiocb *req = tw_req.req;
struct io_ring_ctx *ctx = req->ctx;
io_add_aux_cqe(ctx, req->cqe.user_data, req->cqe.res, req->cqe.flags);

7
io_uring/net.c
View File

@@ -110,7 +110,6 @@ enum sr_retry_flags {
struct io_recvzc {
struct file *file;
unsigned msg_flags;
u16 flags;
u32 len;
struct io_zcrx_ifq *ifq;
@@ -1253,8 +1252,7 @@ int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
zc->len = READ_ONCE(sqe->len);
zc->flags = READ_ONCE(sqe->ioprio);
zc->msg_flags = READ_ONCE(sqe->msg_flags);
if (zc->msg_flags)
if (READ_ONCE(sqe->msg_flags))
return -EINVAL;
if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT))
return -EINVAL;
@@ -1283,8 +1281,7 @@ int io_recvzc(struct io_kiocb *req, unsigned int issue_flags)
return -ENOTSOCK;
len = zc->len;
ret = io_zcrx_recv(req, zc->ifq, sock, zc->msg_flags | MSG_DONTWAIT,
issue_flags, &zc->len);
ret = io_zcrx_recv(req, zc->ifq, sock, 0, issue_flags, &zc->len);
if (len && zc->len == 0) {
io_req_set_res(req, 0, 0);

7
io_uring/notif.c
View File

@@ -11,8 +11,9 @@
static const struct ubuf_info_ops io_ubuf_ops;
static void io_notif_tw_complete(struct io_kiocb *notif, io_tw_token_t tw)
static void io_notif_tw_complete(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_kiocb *notif = tw_req.req;
struct io_notif_data *nd = io_notif_to_data(notif);
struct io_ring_ctx *ctx = notif->ctx;
@@ -34,7 +35,7 @@ static void io_notif_tw_complete(struct io_kiocb *notif, io_tw_token_t tw)
}
nd = nd->next;
io_req_task_complete(notif, tw);
io_req_task_complete((struct io_tw_req){notif}, tw);
} while (nd);
}
@@ -92,7 +93,7 @@ static int io_link_skb(struct sk_buff *skb, struct ubuf_info *uarg)
prev_nd = container_of(prev_uarg, struct io_notif_data, uarg);
prev_notif = cmd_to_io_kiocb(prev_nd);
/* make sure all noifications can be finished in the same task_work */
/* make sure all notifications can be finished in the same task_work */
if (unlikely(notif->ctx != prev_notif->ctx ||
notif->tctx != prev_notif->tctx))
return -EEXIST;

26
io_uring/opdef.c
View File

@@ -575,6 +575,24 @@ const struct io_issue_def io_issue_defs[] = {
.prep = io_pipe_prep,
.issue = io_pipe,
},
[IORING_OP_NOP128] = {
.audit_skip = 1,
.iopoll = 1,
.is_128 = 1,
.prep = io_nop_prep,
.issue = io_nop,
},
[IORING_OP_URING_CMD128] = {
.buffer_select = 1,
.needs_file = 1,
.plug = 1,
.iopoll = 1,
.iopoll_queue = 1,
.is_128 = 1,
.async_size = sizeof(struct io_async_cmd),
.prep = io_uring_cmd_prep,
.issue = io_uring_cmd,
},
};
const struct io_cold_def io_cold_defs[] = {
@@ -825,6 +843,14 @@ const struct io_cold_def io_cold_defs[] = {
[IORING_OP_PIPE] = {
.name = "PIPE",
},
[IORING_OP_NOP128] = {
.name = "NOP128",
},
[IORING_OP_URING_CMD128] = {
.name = "URING_CMD128",
.sqe_copy = io_uring_cmd_sqe_copy,
.cleanup = io_uring_cmd_cleanup,
},
};
const char *io_uring_get_opcode(u8 opcode)

2
io_uring/opdef.h
View File

@@ -27,6 +27,8 @@ struct io_issue_def {
unsigned iopoll_queue : 1;
/* vectored opcode, set if 1) vectored, and 2) handler needs to know */
unsigned vectored : 1;
/* set to 1 if this opcode uses 128b sqes in a mixed sq */
unsigned is_128 : 1;
/* size of async data needed, if any */
unsigned short async_size;

13
io_uring/poll.c
View File

@@ -224,7 +224,7 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
{
int v;
if (unlikely(io_should_terminate_tw(req->ctx)))
if (unlikely(tw.cancel))
return -ECANCELED;
do {
@@ -310,8 +310,9 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
return IOU_POLL_NO_ACTION;
}
void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw)
void io_poll_task_func(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_kiocb *req = tw_req.req;
int ret;
ret = io_poll_check_events(req, tw);
@@ -332,7 +333,7 @@ void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw)
poll = io_kiocb_to_cmd(req, struct io_poll);
req->cqe.res = mangle_poll(req->cqe.res & poll->events);
} else if (ret == IOU_POLL_REISSUE) {
io_req_task_submit(req, tw);
io_req_task_submit(tw_req, tw);
return;
} else if (ret != IOU_POLL_REMOVE_POLL_USE_RES) {
req->cqe.res = ret;
@@ -340,14 +341,14 @@ void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw)
}
io_req_set_res(req, req->cqe.res, 0);
io_req_task_complete(req, tw);
io_req_task_complete(tw_req, tw);
} else {
io_tw_lock(req->ctx, tw);
if (ret == IOU_POLL_REMOVE_POLL_USE_RES)
io_req_task_complete(req, tw);
io_req_task_complete(tw_req, tw);
else if (ret == IOU_POLL_DONE || ret == IOU_POLL_REISSUE)
io_req_task_submit(req, tw);
io_req_task_submit(tw_req, tw);
else
io_req_defer_failed(req, ret);
}

2
io_uring/poll.h
View File

@@ -46,4 +46,4 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags);
bool io_poll_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
bool cancel_all);
void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw);
void io_poll_task_func(struct io_tw_req tw_req, io_tw_token_t tw);

55
io_uring/query.c
View File

@@ -4,15 +4,20 @@
#include "query.h"
#include "io_uring.h"
#include "zcrx.h"
#define IO_MAX_QUERY_SIZE (sizeof(struct io_uring_query_opcode))
union io_query_data {
struct io_uring_query_opcode opcodes;
struct io_uring_query_zcrx zcrx;
struct io_uring_query_scq scq;
};
#define IO_MAX_QUERY_SIZE sizeof(union io_query_data)
#define IO_MAX_QUERY_ENTRIES 1000
static ssize_t io_query_ops(void *data)
static ssize_t io_query_ops(union io_query_data *data)
{
struct io_uring_query_opcode *e = data;
BUILD_BUG_ON(sizeof(*e) > IO_MAX_QUERY_SIZE);
struct io_uring_query_opcode *e = &data->opcodes;
e->nr_request_opcodes = IORING_OP_LAST;
e->nr_register_opcodes = IORING_REGISTER_LAST;
@@ -25,8 +30,30 @@ static ssize_t io_query_ops(void *data)
return sizeof(*e);
}
static int io_handle_query_entry(struct io_ring_ctx *ctx,
void *data, void __user *uhdr,
static ssize_t io_query_zcrx(union io_query_data *data)
{
struct io_uring_query_zcrx *e = &data->zcrx;
e->register_flags = ZCRX_REG_IMPORT;
e->area_flags = IORING_ZCRX_AREA_DMABUF;
e->nr_ctrl_opcodes = __ZCRX_CTRL_LAST;
e->rq_hdr_size = sizeof(struct io_uring);
e->rq_hdr_alignment = L1_CACHE_BYTES;
e->__resv1 = 0;
e->__resv2 = 0;
return sizeof(*e);
}
static ssize_t io_query_scq(union io_query_data *data)
{
struct io_uring_query_scq *e = &data->scq;
e->hdr_size = sizeof(struct io_rings);
e->hdr_alignment = SMP_CACHE_BYTES;
return sizeof(*e);
}
static int io_handle_query_entry(union io_query_data *data, void __user *uhdr,
u64 *next_entry)
{
struct io_uring_query_hdr hdr;
@@ -53,6 +80,12 @@ static int io_handle_query_entry(struct io_ring_ctx *ctx,
case IO_URING_QUERY_OPCODES:
ret = io_query_ops(data);
break;
case IO_URING_QUERY_ZCRX:
ret = io_query_zcrx(data);
break;
case IO_URING_QUERY_SCQ:
ret = io_query_scq(data);
break;
}
if (ret >= 0) {
@@ -73,13 +106,13 @@ out:
return 0;
}
int io_query(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
int io_query(void __user *arg, unsigned nr_args)
{
char entry_buffer[IO_MAX_QUERY_SIZE];
union io_query_data entry_buffer;
void __user *uhdr = arg;
int ret, nr = 0;
memset(entry_buffer, 0, sizeof(entry_buffer));
memset(&entry_buffer, 0, sizeof(entry_buffer));
if (nr_args)
return -EINVAL;
@@ -87,7 +120,7 @@ int io_query(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
while (uhdr) {
u64 next_hdr;
ret = io_handle_query_entry(ctx, entry_buffer, uhdr, &next_hdr);
ret = io_handle_query_entry(&entry_buffer, uhdr, &next_hdr);
if (ret)
return ret;
uhdr = u64_to_user_ptr(next_hdr);

2
io_uring/query.h
View File

@@ -4,6 +4,6 @@
#include <linux/io_uring_types.h>
int io_query(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args);
int io_query(void __user *arg, unsigned nr_args);
#endif

105
io_uring/register.c
View File

@@ -379,11 +379,10 @@ struct io_ring_ctx_rings {
};
static void io_register_free_rings(struct io_ring_ctx *ctx,
struct io_uring_params *p,
struct io_ring_ctx_rings *r)
{
io_free_region(ctx, &r->sq_region);
io_free_region(ctx, &r->ring_region);
io_free_region(ctx->user, &r->sq_region);
io_free_region(ctx->user, &r->ring_region);
}
#define swap_old(ctx, o, n, field) \
@@ -395,48 +394,45 @@ static void io_register_free_rings(struct io_ring_ctx *ctx,
#define RESIZE_FLAGS (IORING_SETUP_CQSIZE | IORING_SETUP_CLAMP)
#define COPY_FLAGS (IORING_SETUP_NO_SQARRAY | IORING_SETUP_SQE128 | \
IORING_SETUP_CQE32 | IORING_SETUP_NO_MMAP | \
IORING_SETUP_CQE_MIXED)
IORING_SETUP_CQE_MIXED | IORING_SETUP_SQE_MIXED)
static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
{
struct io_ctx_config config;
struct io_uring_region_desc rd;
struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;
size_t size, sq_array_offset;
unsigned i, tail, old_head;
struct io_uring_params p;
struct io_uring_params *p = &config.p;
struct io_rings_layout *rl = &config.layout;
int ret;
memset(&config, 0, sizeof(config));
/* limited to DEFER_TASKRUN for now */
if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
return -EINVAL;
if (copy_from_user(&p, arg, sizeof(p)))
if (copy_from_user(p, arg, sizeof(*p)))
return -EFAULT;
if (p.flags & ~RESIZE_FLAGS)
if (p->flags & ~RESIZE_FLAGS)
return -EINVAL;
/* properties that are always inherited */
p.flags |= (ctx->flags & COPY_FLAGS);
p->flags |= (ctx->flags & COPY_FLAGS);
ret = io_uring_fill_params(p.sq_entries, &p);
ret = io_prepare_config(&config);
if (unlikely(ret))
return ret;
size = rings_size(p.flags, p.sq_entries, p.cq_entries,
&sq_array_offset);
if (size == SIZE_MAX)
return -EOVERFLOW;
memset(&rd, 0, sizeof(rd));
rd.size = PAGE_ALIGN(size);
if (p.flags & IORING_SETUP_NO_MMAP) {
rd.user_addr = p.cq_off.user_addr;
rd.size = PAGE_ALIGN(rl->rings_size);
if (p->flags & IORING_SETUP_NO_MMAP) {
rd.user_addr = p->cq_off.user_addr;
rd.flags |= IORING_MEM_REGION_TYPE_USER;
}
ret = io_create_region_mmap_safe(ctx, &n.ring_region, &rd, IORING_OFF_CQ_RING);
if (ret) {
io_register_free_rings(ctx, &p, &n);
ret = io_create_region(ctx, &n.ring_region, &rd, IORING_OFF_CQ_RING);
if (ret)
return ret;
}
n.rings = io_region_get_ptr(&n.ring_region);
/*
@@ -447,34 +443,25 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
* intent... Use read/write once helpers from here on to indicate the
* shared nature of it.
*/
WRITE_ONCE(n.rings->sq_ring_mask, p.sq_entries - 1);
WRITE_ONCE(n.rings->cq_ring_mask, p.cq_entries - 1);
WRITE_ONCE(n.rings->sq_ring_entries, p.sq_entries);
WRITE_ONCE(n.rings->cq_ring_entries, p.cq_entries);
WRITE_ONCE(n.rings->sq_ring_mask, p->sq_entries - 1);
WRITE_ONCE(n.rings->cq_ring_mask, p->cq_entries - 1);
WRITE_ONCE(n.rings->sq_ring_entries, p->sq_entries);
WRITE_ONCE(n.rings->cq_ring_entries, p->cq_entries);
if (copy_to_user(arg, &p, sizeof(p))) {
io_register_free_rings(ctx, &p, &n);
if (copy_to_user(arg, p, sizeof(*p))) {
io_register_free_rings(ctx, &n);
return -EFAULT;
}
if (p.flags & IORING_SETUP_SQE128)
size = array_size(2 * sizeof(struct io_uring_sqe), p.sq_entries);
else
size = array_size(sizeof(struct io_uring_sqe), p.sq_entries);
if (size == SIZE_MAX) {
io_register_free_rings(ctx, &p, &n);
return -EOVERFLOW;
}
memset(&rd, 0, sizeof(rd));
rd.size = PAGE_ALIGN(size);
if (p.flags & IORING_SETUP_NO_MMAP) {
rd.user_addr = p.sq_off.user_addr;
rd.size = PAGE_ALIGN(rl->sq_size);
if (p->flags & IORING_SETUP_NO_MMAP) {
rd.user_addr = p->sq_off.user_addr;
rd.flags |= IORING_MEM_REGION_TYPE_USER;
}
ret = io_create_region_mmap_safe(ctx, &n.sq_region, &rd, IORING_OFF_SQES);
ret = io_create_region(ctx, &n.sq_region, &rd, IORING_OFF_SQES);
if (ret) {
io_register_free_rings(ctx, &p, &n);
io_register_free_rings(ctx, &n);
return ret;
}
n.sq_sqes = io_region_get_ptr(&n.sq_region);
@@ -510,11 +497,11 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
*/
tail = READ_ONCE(o.rings->sq.tail);
old_head = READ_ONCE(o.rings->sq.head);
if (tail - old_head > p.sq_entries)
if (tail - old_head > p->sq_entries)
goto overflow;
for (i = old_head; i < tail; i++) {
unsigned src_head = i & (ctx->sq_entries - 1);
unsigned dst_head = i & (p.sq_entries - 1);
unsigned dst_head = i & (p->sq_entries - 1);
n.sq_sqes[dst_head] = o.sq_sqes[src_head];
}
@@ -523,7 +510,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
tail = READ_ONCE(o.rings->cq.tail);
old_head = READ_ONCE(o.rings->cq.head);
if (tail - old_head > p.cq_entries) {
if (tail - old_head > p->cq_entries) {
overflow:
/* restore old rings, and return -EOVERFLOW via cleanup path */
ctx->rings = o.rings;
@@ -534,7 +521,7 @@ overflow:
}
for (i = old_head; i < tail; i++) {
unsigned src_head = i & (ctx->cq_entries - 1);
unsigned dst_head = i & (p.cq_entries - 1);
unsigned dst_head = i & (p->cq_entries - 1);
n.rings->cqes[dst_head] = o.rings->cqes[src_head];
}
@@ -550,10 +537,10 @@ overflow:
/* all done, store old pointers and assign new ones */
if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
ctx->sq_array = (u32 *)((char *)n.rings + sq_array_offset);
ctx->sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset);
ctx->sq_entries = p.sq_entries;
ctx->cq_entries = p.cq_entries;
ctx->sq_entries = p->sq_entries;
ctx->cq_entries = p->cq_entries;
ctx->rings = n.rings;
ctx->sq_sqes = n.sq_sqes;
@@ -564,7 +551,7 @@ overflow:
out:
spin_unlock(&ctx->completion_lock);
mutex_unlock(&ctx->mmap_lock);
io_register_free_rings(ctx, &p, to_free);
io_register_free_rings(ctx, to_free);
if (ctx->sq_data)
io_sq_thread_unpark(ctx->sq_data);
@@ -578,6 +565,7 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg)
struct io_uring_mem_region_reg reg;
struct io_uring_region_desc __user *rd_uptr;
struct io_uring_region_desc rd;
struct io_mapped_region region = {};
int ret;
if (io_region_is_set(&ctx->param_region))
@@ -601,20 +589,20 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg)
!(ctx->flags & IORING_SETUP_R_DISABLED))
return -EINVAL;
ret = io_create_region_mmap_safe(ctx, &ctx->param_region, &rd,
IORING_MAP_OFF_PARAM_REGION);
ret = io_create_region(ctx, &region, &rd, IORING_MAP_OFF_PARAM_REGION);
if (ret)
return ret;
if (copy_to_user(rd_uptr, &rd, sizeof(rd))) {
guard(mutex)(&ctx->mmap_lock);
io_free_region(ctx, &ctx->param_region);
io_free_region(ctx->user, &region);
return -EFAULT;
}
if (reg.flags & IORING_MEM_REGION_REG_WAIT_ARG) {
ctx->cq_wait_arg = io_region_get_ptr(&ctx->param_region);
ctx->cq_wait_arg = io_region_get_ptr(&region);
ctx->cq_wait_size = rd.size;
}
io_region_publish(ctx, &region, &ctx->param_region);
return 0;
}
@@ -825,7 +813,10 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
ret = io_register_mem_region(ctx, arg);
break;
case IORING_REGISTER_QUERY:
ret = io_query(ctx, arg, nr_args);
ret = io_query(arg, nr_args);
break;
case IORING_REGISTER_ZCRX_CTRL:
ret = io_zcrx_ctrl(ctx, arg, nr_args);
break;
default:
ret = -EINVAL;
@@ -897,7 +888,7 @@ static int io_uring_register_blind(unsigned int opcode, void __user *arg,
case IORING_REGISTER_SEND_MSG_RING:
return io_uring_register_send_msg_ring(arg, nr_args);
case IORING_REGISTER_QUERY:
return io_query(NULL, arg, nr_args);
return io_query(arg, nr_args);
}
return -EINVAL;
}

30
io_uring/rsrc.c
View File

@@ -56,27 +56,29 @@ int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
return 0;
}
void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
void io_unaccount_mem(struct user_struct *user, struct mm_struct *mm_account,
unsigned long nr_pages)
{
if (ctx->user)
__io_unaccount_mem(ctx->user, nr_pages);
if (user)
__io_unaccount_mem(user, nr_pages);
if (ctx->mm_account)
atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm);
if (mm_account)
atomic64_sub(nr_pages, &mm_account->pinned_vm);
}
int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
int io_account_mem(struct user_struct *user, struct mm_struct *mm_account,
unsigned long nr_pages)
{
int ret;
if (ctx->user) {
ret = __io_account_mem(ctx->user, nr_pages);
if (user) {
ret = __io_account_mem(user, nr_pages);
if (ret)
return ret;
}
if (ctx->mm_account)
atomic64_add(nr_pages, &ctx->mm_account->pinned_vm);
if (mm_account)
atomic64_add(nr_pages, &mm_account->pinned_vm);
return 0;
}
@@ -145,7 +147,7 @@ static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu)
}
if (imu->acct_pages)
io_unaccount_mem(ctx, imu->acct_pages);
io_unaccount_mem(ctx->user, ctx->mm_account, imu->acct_pages);
imu->release(imu->priv);
io_free_imu(ctx, imu);
}
@@ -454,7 +456,7 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req,
return -ENXIO;
for (done = 0; done < up->nr_args; done++) {
if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
if (get_user(fd, &fds[done])) {
ret = -EFAULT;
break;
}
@@ -468,7 +470,7 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req,
IORING_FILE_INDEX_ALLOC);
if (ret < 0)
break;
if (copy_to_user(&fds[done], &ret, sizeof(ret))) {
if (put_user(ret, &fds[done])) {
__io_close_fixed(req->ctx, issue_flags, ret);
ret = -EFAULT;
break;
@@ -684,7 +686,7 @@ static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
if (!imu->acct_pages)
return 0;
ret = io_account_mem(ctx, imu->acct_pages);
ret = io_account_mem(ctx->user, ctx->mm_account, imu->acct_pages);
if (ret)
imu->acct_pages = 0;
return ret;

6
io_uring/rsrc.h
View File

@@ -120,8 +120,10 @@ int io_files_update(struct io_kiocb *req, unsigned int issue_flags);
int io_files_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
int __io_account_mem(struct user_struct *user, unsigned long nr_pages);
int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages);
void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages);
int io_account_mem(struct user_struct *user, struct mm_struct *mm_account,
unsigned long nr_pages);
void io_unaccount_mem(struct user_struct *user, struct mm_struct *mm_account,
unsigned long nr_pages);
static inline void __io_unaccount_mem(struct user_struct *user,
unsigned long nr_pages)

12
io_uring/rw.c
View File

@@ -186,7 +186,7 @@ static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags)
* This is really a bug in the core code that does this, any issue
* path should assume that a successful (or -EIOCBQUEUED) return can
* mean that the underlying data can be gone at any time. But that
* should be fixed seperately, and then this check could be killed.
* should be fixed separately, and then this check could be killed.
*/
if (!(req->flags & (REQ_F_REISSUE | REQ_F_REFCOUNT))) {
req->flags &= ~REQ_F_NEED_CLEANUP;
@@ -348,7 +348,7 @@ static int io_prep_rwv(struct io_kiocb *req, const struct io_uring_sqe *sqe,
/*
* Have to do this validation here, as this is in io_read() rw->len
* might have chanaged due to buffer selection
* might have changed due to buffer selection
*/
return io_iov_buffer_select_prep(req);
}
@@ -566,15 +566,17 @@ static inline int io_fixup_rw_res(struct io_kiocb *req, long res)
return res;
}
void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw)
void io_req_rw_complete(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_kiocb *req = tw_req.req;
io_req_io_end(req);
if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING))
req->cqe.flags |= io_put_kbuf(req, req->cqe.res, NULL);
io_req_rw_cleanup(req, 0);
io_req_task_complete(req, tw);
io_req_task_complete(tw_req, tw);
}
static void io_complete_rw(struct kiocb *kiocb, long res)
@@ -1010,7 +1012,7 @@ static int __io_read(struct io_kiocb *req, struct io_br_sel *sel,
iov_iter_restore(&io->iter, &io->iter_state);
} while (ret > 0);
done:
/* it's faster to check here then delegate to kfree */
/* it's faster to check here than delegate to kfree */
return ret;
}

2
io_uring/rw.h
View File

@@ -46,7 +46,7 @@ int io_read_fixed(struct io_kiocb *req, unsigned int issue_flags);
int io_write_fixed(struct io_kiocb *req, unsigned int issue_flags);
void io_readv_writev_cleanup(struct io_kiocb *req);
void io_rw_fail(struct io_kiocb *req);
void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw);
void io_req_rw_complete(struct io_tw_req tw_req, io_tw_token_t tw);
int io_read_mshot_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags);
void io_rw_cache_free(const void *entry);

18
io_uring/slist.h
View File

@@ -67,24 +67,6 @@ static inline void wq_list_cut(struct io_wq_work_list *list,
last->next = NULL;
}
static inline void __wq_list_splice(struct io_wq_work_list *list,
struct io_wq_work_node *to)
{
list->last->next = to->next;
to->next = list->first;
INIT_WQ_LIST(list);
}
static inline bool wq_list_splice(struct io_wq_work_list *list,
struct io_wq_work_node *to)
{
if (!wq_list_empty(list)) {
__wq_list_splice(list, to);
return true;
}
return false;
}
static inline void wq_stack_add_head(struct io_wq_work_node *node,
struct io_wq_work_node *stack)
{

1
io_uring/sqpoll.c
View File

@@ -19,6 +19,7 @@
#include "io_uring.h"
#include "tctx.h"
#include "napi.h"
#include "cancel.h"
#include "sqpoll.h"
#define IORING_SQPOLL_CAP_ENTRIES_VALUE 8

20
io_uring/timeout.c
View File

@@ -68,8 +68,9 @@ static inline bool io_timeout_finish(struct io_timeout *timeout,
static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer);
static void io_timeout_complete(struct io_kiocb *req, io_tw_token_t tw)
static void io_timeout_complete(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_kiocb *req = tw_req.req;
struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
struct io_timeout_data *data = req->async_data;
struct io_ring_ctx *ctx = req->ctx;
@@ -85,7 +86,7 @@ static void io_timeout_complete(struct io_kiocb *req, io_tw_token_t tw)
}
}
io_req_task_complete(req, tw);
io_req_task_complete(tw_req, tw);
}
static __cold bool io_flush_killed_timeouts(struct list_head *list, int err)
@@ -157,8 +158,10 @@ __cold void io_flush_timeouts(struct io_ring_ctx *ctx)
io_flush_killed_timeouts(&list, 0);
}
static void io_req_tw_fail_links(struct io_kiocb *link, io_tw_token_t tw)
static void io_req_tw_fail_links(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_kiocb *link = tw_req.req;
io_tw_lock(link->ctx, tw);
while (link) {
struct io_kiocb *nxt = link->link;
@@ -168,7 +171,7 @@ static void io_req_tw_fail_links(struct io_kiocb *link, io_tw_token_t tw)
res = link->cqe.res;
link->link = NULL;
io_req_set_res(link, res, 0);
io_req_task_complete(link, tw);
io_req_task_complete((struct io_tw_req){link}, tw);
link = nxt;
}
}
@@ -317,14 +320,15 @@ int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
return 0;
}
static void io_req_task_link_timeout(struct io_kiocb *req, io_tw_token_t tw)
static void io_req_task_link_timeout(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_kiocb *req = tw_req.req;
struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
struct io_kiocb *prev = timeout->prev;
int ret;
if (prev) {
if (!io_should_terminate_tw(req->ctx)) {
if (!tw.cancel) {
struct io_cancel_data cd = {
.ctx = req->ctx,
.data = prev->cqe.user_data,
@@ -335,11 +339,11 @@ static void io_req_task_link_timeout(struct io_kiocb *req, io_tw_token_t tw)
ret = -ECANCELED;
}
io_req_set_res(req, ret ?: -ETIME, 0);
io_req_task_complete(req, tw);
io_req_task_complete(tw_req, tw);
io_put_req(prev);
} else {
io_req_set_res(req, -ETIME, 0);
io_req_task_complete(req, tw);
io_req_task_complete(tw_req, tw);
}
}

34
io_uring/uring_cmd.c
View File

@@ -113,20 +113,8 @@ void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
}
EXPORT_SYMBOL_GPL(io_uring_cmd_mark_cancelable);
static void io_uring_cmd_work(struct io_kiocb *req, io_tw_token_t tw)
{
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
unsigned int flags = IO_URING_F_COMPLETE_DEFER;
if (io_should_terminate_tw(req->ctx))
flags |= IO_URING_F_TASK_DEAD;
/* task_work executor checks the deffered list completion */
ioucmd->task_work_cb(ioucmd, flags);
}
void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
io_uring_cmd_tw_t task_work_cb,
io_req_tw_func_t task_work_cb,
unsigned flags)
{
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
@@ -134,8 +122,7 @@ void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT))
return;
ioucmd->task_work_cb = task_work_cb;
req->io_task_work.func = io_uring_cmd_work;
req->io_task_work.func = task_work_cb;
__io_req_task_work_add(req, flags);
}
EXPORT_SYMBOL_GPL(__io_uring_cmd_do_in_task);
@@ -216,6 +203,18 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return 0;
}
/*
* IORING_SETUP_SQE128 contexts allocate twice the normal SQE size for each
* slot.
*/
static inline size_t uring_sqe_size(struct io_kiocb *req)
{
if (req->ctx->flags & IORING_SETUP_SQE128 ||
req->opcode == IORING_OP_URING_CMD128)
return 2 * sizeof(struct io_uring_sqe);
return sizeof(struct io_uring_sqe);
}
void io_uring_cmd_sqe_copy(struct io_kiocb *req)
{
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
@@ -224,7 +223,7 @@ void io_uring_cmd_sqe_copy(struct io_kiocb *req)
/* Should not happen, as REQ_F_SQE_COPIED covers this */
if (WARN_ON_ONCE(ioucmd->sqe == ac->sqes))
return;
memcpy(ac->sqes, ioucmd->sqe, uring_sqe_size(req->ctx));
memcpy(ac->sqes, ioucmd->sqe, uring_sqe_size(req));
ioucmd->sqe = ac->sqes;
}
@@ -242,7 +241,8 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
if (ret)
return ret;
if (ctx->flags & IORING_SETUP_SQE128)
if (ctx->flags & IORING_SETUP_SQE128 ||
req->opcode == IORING_OP_URING_CMD128)
issue_flags |= IO_URING_F_SQE128;
if (ctx->flags & (IORING_SETUP_CQE32 | IORING_SETUP_CQE_MIXED))
issue_flags |= IO_URING_F_CQE32;

48
io_uring/waitid.c
View File

@@ -16,7 +16,7 @@
#include "waitid.h"
#include "../kernel/exit.h"
static void io_waitid_cb(struct io_kiocb *req, io_tw_token_t tw);
static void io_waitid_cb(struct io_tw_req tw_req, io_tw_token_t tw);
#define IO_WAITID_CANCEL_FLAG BIT(31)
#define IO_WAITID_REF_MASK GENMASK(30, 0)
@@ -109,6 +109,22 @@ static int io_waitid_finish(struct io_kiocb *req, int ret)
return ret;
}
static void io_waitid_remove_wq(struct io_kiocb *req)
{
struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
struct wait_queue_head *head;
head = READ_ONCE(iw->head);
if (head) {
struct io_waitid_async *iwa = req->async_data;
iw->head = NULL;
spin_lock_irq(&head->lock);
list_del_init(&iwa->wo.child_wait.entry);
spin_unlock_irq(&head->lock);
}
}
static void io_waitid_complete(struct io_kiocb *req, int ret)
{
struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
@@ -119,6 +135,7 @@ static void io_waitid_complete(struct io_kiocb *req, int ret)
lockdep_assert_held(&req->ctx->uring_lock);
hlist_del_init(&req->hash_node);
io_waitid_remove_wq(req);
ret = io_waitid_finish(req, ret);
if (ret < 0)
@@ -129,7 +146,8 @@ static void io_waitid_complete(struct io_kiocb *req, int ret)
static bool __io_waitid_cancel(struct io_kiocb *req)
{
struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
struct io_waitid_async *iwa = req->async_data;
lockdep_assert_held(&req->ctx->uring_lock);
/*
* Mark us canceled regardless of ownership. This will prevent a
@@ -141,9 +159,6 @@ static bool __io_waitid_cancel(struct io_kiocb *req)
if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK)
return false;
spin_lock_irq(&iw->head->lock);
list_del_init(&iwa->wo.child_wait.entry);
spin_unlock_irq(&iw->head->lock);
io_waitid_complete(req, -ECANCELED);
io_req_queue_tw_complete(req, -ECANCELED);
return true;
@@ -164,23 +179,24 @@ bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
static inline bool io_waitid_drop_issue_ref(struct io_kiocb *req)
{
struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
struct io_waitid_async *iwa = req->async_data;
if (!atomic_sub_return(1, &iw->refs))
return false;
io_waitid_remove_wq(req);
/*
* Wakeup triggered, racing with us. It was prevented from
* completing because of that, queue up the tw to do that.
*/
req->io_task_work.func = io_waitid_cb;
io_req_task_work_add(req);
remove_wait_queue(iw->head, &iwa->wo.child_wait);
return true;
}
static void io_waitid_cb(struct io_kiocb *req, io_tw_token_t tw)
static void io_waitid_cb(struct io_tw_req tw_req, io_tw_token_t tw)
{
struct io_kiocb *req = tw_req.req;
struct io_waitid_async *iwa = req->async_data;
struct io_ring_ctx *ctx = req->ctx;
int ret;
@@ -209,13 +225,12 @@ static void io_waitid_cb(struct io_kiocb *req, io_tw_token_t tw)
io_waitid_drop_issue_ref(req);
return;
}
remove_wait_queue(iw->head, &iwa->wo.child_wait);
/* fall through to complete, will kill waitqueue */
}
}
io_waitid_complete(req, ret);
io_req_task_complete(req, tw);
io_req_task_complete(tw_req, tw);
}
static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode,
@@ -231,6 +246,7 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode,
return 0;
list_del_init(&wait->entry);
iw->head = NULL;
/* cancel is in progress */
if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK)
@@ -257,6 +273,7 @@ int io_waitid_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
iw->which = READ_ONCE(sqe->len);
iw->upid = READ_ONCE(sqe->fd);
iw->options = READ_ONCE(sqe->file_index);
iw->head = NULL;
iw->infop = u64_to_user_ptr(READ_ONCE(sqe->addr2));
return 0;
}
@@ -287,11 +304,16 @@ int io_waitid(struct io_kiocb *req, unsigned int issue_flags)
* callback.
*/
io_ring_submit_lock(ctx, issue_flags);
/*
* iw->head is valid under the ring lock, and as long as the request
* is on the waitid_list where cancelations may find it.
*/
iw->head = &current->signal->wait_chldexit;
hlist_add_head(&req->hash_node, &ctx->waitid_list);
init_waitqueue_func_entry(&iwa->wo.child_wait, io_waitid_wait);
iwa->wo.child_wait.private = req->tctx->task;
iw->head = &current->signal->wait_chldexit;
add_wait_queue(iw->head, &iwa->wo.child_wait);
ret = __do_wait(&iwa->wo);
@@ -314,7 +336,7 @@ int io_waitid(struct io_kiocb *req, unsigned int issue_flags)
}
hlist_del_init(&req->hash_node);
remove_wait_queue(iw->head, &iwa->wo.child_wait);
io_waitid_remove_wq(req);
ret = io_waitid_finish(req, ret);
io_ring_submit_unlock(ctx, issue_flags);

421
io_uring/zcrx.c
View File

@@ -8,6 +8,7 @@
#include <linux/netdevice.h>
#include <linux/rtnetlink.h>
#include <linux/skbuff_ref.h>
#include <linux/anon_inodes.h>
#include <net/page_pool/helpers.h>
#include <net/page_pool/memory_provider.h>
@@ -170,7 +171,7 @@ static unsigned long io_count_account_pages(struct page **pages, unsigned nr_pag
if (folio == last_folio)
continue;
last_folio = folio;
res += 1UL << folio_order(folio);
res += folio_nr_pages(folio);
}
return res;
}
@@ -200,7 +201,7 @@ static int io_import_umem(struct io_zcrx_ifq *ifq,
}
mem->account_pages = io_count_account_pages(pages, nr_pages);
ret = io_account_mem(ifq->ctx, mem->account_pages);
ret = io_account_mem(ifq->user, ifq->mm_account, mem->account_pages);
if (ret < 0)
mem->account_pages = 0;
@@ -344,7 +345,15 @@ static void io_zcrx_get_niov_uref(struct net_iov *niov)
atomic_inc(io_get_user_counter(niov));
}
static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq,
static void io_fill_zcrx_offsets(struct io_uring_zcrx_offsets *offsets)
{
offsets->head = offsetof(struct io_uring, head);
offsets->tail = offsetof(struct io_uring, tail);
offsets->rqes = ALIGN(sizeof(struct io_uring), L1_CACHE_BYTES);
}
static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx,
struct io_zcrx_ifq *ifq,
struct io_uring_zcrx_ifq_reg *reg,
struct io_uring_region_desc *rd,
u32 id)
@@ -354,7 +363,8 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq,
void *ptr;
int ret;
off = ALIGN(sizeof(struct io_uring), L1_CACHE_BYTES);
io_fill_zcrx_offsets(&reg->offsets);
off = reg->offsets.rqes;
size = off + sizeof(struct io_uring_zcrx_rqe) * reg->rq_entries;
if (size > rd->size)
return -EINVAL;
@@ -362,7 +372,7 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq,
mmap_offset = IORING_MAP_OFF_ZCRX_REGION;
mmap_offset += id << IORING_OFF_PBUF_SHIFT;
ret = io_create_region(ifq->ctx, &ifq->region, rd, mmap_offset);
ret = io_create_region(ctx, &ifq->region, rd, mmap_offset);
if (ret < 0)
return ret;
@@ -370,26 +380,25 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq,
ifq->rq_ring = (struct io_uring *)ptr;
ifq->rqes = (struct io_uring_zcrx_rqe *)(ptr + off);
reg->offsets.head = offsetof(struct io_uring, head);
reg->offsets.tail = offsetof(struct io_uring, tail);
reg->offsets.rqes = off;
return 0;
}
static void io_free_rbuf_ring(struct io_zcrx_ifq *ifq)
{
io_free_region(ifq->ctx, &ifq->region);
io_free_region(ifq->user, &ifq->region);
ifq->rq_ring = NULL;
ifq->rqes = NULL;
}
static void io_zcrx_free_area(struct io_zcrx_area *area)
static void io_zcrx_free_area(struct io_zcrx_ifq *ifq,
struct io_zcrx_area *area)
{
io_zcrx_unmap_area(area->ifq, area);
io_zcrx_unmap_area(ifq, area);
io_release_area_mem(&area->mem);
if (area->mem.account_pages)
io_unaccount_mem(area->ifq->ctx, area->mem.account_pages);
io_unaccount_mem(ifq->user, ifq->mm_account,
area->mem.account_pages);
kvfree(area->freelist);
kvfree(area->nia.niovs);
@@ -463,7 +472,7 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,
return 0;
err:
if (area)
io_zcrx_free_area(area);
io_zcrx_free_area(ifq, area);
return ret;
}
@@ -476,9 +485,10 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx)
return NULL;
ifq->if_rxq = -1;
ifq->ctx = ctx;
spin_lock_init(&ifq->rq_lock);
mutex_init(&ifq->pp_lock);
refcount_set(&ifq->refs, 1);
refcount_set(&ifq->user_refs, 1);
return ifq;
}
@@ -522,7 +532,10 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq)
io_close_queue(ifq);
if (ifq->area)
io_zcrx_free_area(ifq->area);
io_zcrx_free_area(ifq, ifq->area);
free_uid(ifq->user);
if (ifq->mm_account)
mmdrop(ifq->mm_account);
if (ifq->dev)
put_device(ifq->dev);
@@ -531,6 +544,63 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq)
kfree(ifq);
}
static void io_put_zcrx_ifq(struct io_zcrx_ifq *ifq)
{
if (refcount_dec_and_test(&ifq->refs))
io_zcrx_ifq_free(ifq);
}
static void io_zcrx_return_niov_freelist(struct net_iov *niov)
{
struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
spin_lock_bh(&area->freelist_lock);
area->freelist[area->free_count++] = net_iov_idx(niov);
spin_unlock_bh(&area->freelist_lock);
}
static void io_zcrx_return_niov(struct net_iov *niov)
{
netmem_ref netmem = net_iov_to_netmem(niov);
if (!niov->desc.pp) {
/* copy fallback allocated niovs */
io_zcrx_return_niov_freelist(niov);
return;
}
page_pool_put_unrefed_netmem(niov->desc.pp, netmem, -1, false);
}
static void io_zcrx_scrub(struct io_zcrx_ifq *ifq)
{
struct io_zcrx_area *area = ifq->area;
int i;
if (!area)
return;
/* Reclaim back all buffers given to the user space. */
for (i = 0; i < area->nia.num_niovs; i++) {
struct net_iov *niov = &area->nia.niovs[i];
int nr;
if (!atomic_read(io_get_user_counter(niov)))
continue;
nr = atomic_xchg(io_get_user_counter(niov), 0);
if (nr && !page_pool_unref_netmem(net_iov_to_netmem(niov), nr))
io_zcrx_return_niov(niov);
}
}
static void zcrx_unregister(struct io_zcrx_ifq *ifq)
{
if (refcount_dec_and_test(&ifq->user_refs)) {
io_close_queue(ifq);
io_zcrx_scrub(ifq);
}
io_put_zcrx_ifq(ifq);
}
struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
unsigned int id)
{
@@ -541,6 +611,112 @@ struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
return ifq ? &ifq->region : NULL;
}
static int zcrx_box_release(struct inode *inode, struct file *file)
{
struct io_zcrx_ifq *ifq = file->private_data;
if (WARN_ON_ONCE(!ifq))
return -EFAULT;
zcrx_unregister(ifq);
return 0;
}
static const struct file_operations zcrx_box_fops = {
.owner = THIS_MODULE,
.release = zcrx_box_release,
};
static int zcrx_export(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq,
struct zcrx_ctrl *ctrl, void __user *arg)
{
struct zcrx_ctrl_export *ce = &ctrl->zc_export;
struct file *file;
int fd = -1;
if (!mem_is_zero(ce, sizeof(*ce)))
return -EINVAL;
fd = get_unused_fd_flags(O_CLOEXEC);
if (fd < 0)
return fd;
ce->zcrx_fd = fd;
if (copy_to_user(arg, ctrl, sizeof(*ctrl))) {
put_unused_fd(fd);
return -EFAULT;
}
refcount_inc(&ifq->refs);
refcount_inc(&ifq->user_refs);
file = anon_inode_create_getfile("[zcrx]", &zcrx_box_fops,
ifq, O_CLOEXEC, NULL);
if (IS_ERR(file)) {
put_unused_fd(fd);
zcrx_unregister(ifq);
return PTR_ERR(file);
}
fd_install(fd, file);
return 0;
}
static int import_zcrx(struct io_ring_ctx *ctx,
struct io_uring_zcrx_ifq_reg __user *arg,
struct io_uring_zcrx_ifq_reg *reg)
{
struct io_zcrx_ifq *ifq;
struct file *file;
int fd, ret;
u32 id;
if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
return -EINVAL;
if (!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED)))
return -EINVAL;
if (reg->if_rxq || reg->rq_entries || reg->area_ptr || reg->region_ptr)
return -EINVAL;
fd = reg->if_idx;
CLASS(fd, f)(fd);
if (fd_empty(f))
return -EBADF;
file = fd_file(f);
if (file->f_op != &zcrx_box_fops || !file->private_data)
return -EBADF;
ifq = file->private_data;
refcount_inc(&ifq->refs);
refcount_inc(&ifq->user_refs);
scoped_guard(mutex, &ctx->mmap_lock) {
ret = xa_alloc(&ctx->zcrx_ctxs, &id, NULL, xa_limit_31b, GFP_KERNEL);
if (ret)
goto err;
}
reg->zcrx_id = id;
io_fill_zcrx_offsets(&reg->offsets);
if (copy_to_user(arg, reg, sizeof(*reg))) {
ret = -EFAULT;
goto err_xa_erase;
}
scoped_guard(mutex, &ctx->mmap_lock) {
ret = -ENOMEM;
if (xa_store(&ctx->zcrx_ctxs, id, ifq, GFP_KERNEL))
goto err_xa_erase;
}
return 0;
err_xa_erase:
scoped_guard(mutex, &ctx->mmap_lock)
xa_erase(&ctx->zcrx_ctxs, id);
err:
zcrx_unregister(ifq);
return ret;
}
int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
struct io_uring_zcrx_ifq_reg __user *arg)
{
@@ -566,11 +742,13 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
return -EINVAL;
if (copy_from_user(&reg, arg, sizeof(reg)))
return -EFAULT;
if (copy_from_user(&rd, u64_to_user_ptr(reg.region_ptr), sizeof(rd)))
return -EFAULT;
if (!mem_is_zero(&reg.__resv, sizeof(reg.__resv)) ||
reg.__resv2 || reg.zcrx_id)
return -EINVAL;
if (reg.flags & ZCRX_REG_IMPORT)
return import_zcrx(ctx, arg, &reg);
if (copy_from_user(&rd, u64_to_user_ptr(reg.region_ptr), sizeof(rd)))
return -EFAULT;
if (reg.if_rxq == -1 || !reg.rq_entries || reg.flags)
return -EINVAL;
if (reg.rq_entries > IO_RQ_MAX_ENTRIES) {
@@ -586,6 +764,15 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
ifq = io_zcrx_ifq_alloc(ctx);
if (!ifq)
return -ENOMEM;
if (ctx->user) {
get_uid(ctx->user);
ifq->user = ctx->user;
}
if (ctx->mm_account) {
mmgrab(ctx->mm_account);
ifq->mm_account = ctx->mm_account;
}
ifq->rq_entries = reg.rq_entries;
scoped_guard(mutex, &ctx->mmap_lock) {
@@ -595,33 +782,34 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
goto ifq_free;
}
ret = io_allocate_rbuf_ring(ifq, &reg, &rd, id);
ret = io_allocate_rbuf_ring(ctx, ifq, &reg, &rd, id);
if (ret)
goto err;
ifq->netdev = netdev_get_by_index(current->nsproxy->net_ns, reg.if_idx,
&ifq->netdev_tracker, GFP_KERNEL);
ifq->netdev = netdev_get_by_index_lock(current->nsproxy->net_ns, reg.if_idx);
if (!ifq->netdev) {
ret = -ENODEV;
goto err;
}
netdev_hold(ifq->netdev, &ifq->netdev_tracker, GFP_KERNEL);
ifq->dev = netdev_queue_get_dma_dev(ifq->netdev, reg.if_rxq);
if (!ifq->dev) {
ret = -EOPNOTSUPP;
goto err;
goto netdev_put_unlock;
}
get_device(ifq->dev);
ret = io_zcrx_create_area(ifq, &area);
if (ret)
goto err;
goto netdev_put_unlock;
mp_param.mp_ops = &io_uring_pp_zc_ops;
mp_param.mp_priv = ifq;
ret = net_mp_open_rxq(ifq->netdev, reg.if_rxq, &mp_param);
ret = __net_mp_open_rxq(ifq->netdev, reg.if_rxq, &mp_param, NULL);
if (ret)
goto err;
goto netdev_put_unlock;
netdev_unlock(ifq->netdev);
ifq->if_rxq = reg.if_rxq;
reg.zcrx_id = id;
@@ -640,6 +828,9 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
goto err;
}
return 0;
netdev_put_unlock:
netdev_put(ifq->netdev, &ifq->netdev_tracker);
netdev_unlock(ifq->netdev);
err:
scoped_guard(mutex, &ctx->mmap_lock)
xa_erase(&ctx->zcrx_ctxs, id);
@@ -648,6 +839,16 @@ ifq_free:
return ret;
}
static struct net_iov *__io_zcrx_get_free_niov(struct io_zcrx_area *area)
{
unsigned niov_idx;
lockdep_assert_held(&area->freelist_lock);
niov_idx = area->freelist[--area->free_count];
return &area->nia.niovs[niov_idx];
}
void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx)
{
struct io_zcrx_ifq *ifq;
@@ -664,77 +865,12 @@ void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx)
}
if (!ifq)
break;
io_zcrx_ifq_free(ifq);
zcrx_unregister(ifq);
}
xa_destroy(&ctx->zcrx_ctxs);
}
static struct net_iov *__io_zcrx_get_free_niov(struct io_zcrx_area *area)
{
unsigned niov_idx;
lockdep_assert_held(&area->freelist_lock);
niov_idx = area->freelist[--area->free_count];
return &area->nia.niovs[niov_idx];
}
static void io_zcrx_return_niov_freelist(struct net_iov *niov)
{
struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
spin_lock_bh(&area->freelist_lock);
area->freelist[area->free_count++] = net_iov_idx(niov);
spin_unlock_bh(&area->freelist_lock);
}
static void io_zcrx_return_niov(struct net_iov *niov)
{
netmem_ref netmem = net_iov_to_netmem(niov);
if (!niov->pp) {
/* copy fallback allocated niovs */
io_zcrx_return_niov_freelist(niov);
return;
}
page_pool_put_unrefed_netmem(niov->pp, netmem, -1, false);
}
static void io_zcrx_scrub(struct io_zcrx_ifq *ifq)
{
struct io_zcrx_area *area = ifq->area;
int i;
if (!area)
return;
/* Reclaim back all buffers given to the user space. */
for (i = 0; i < area->nia.num_niovs; i++) {
struct net_iov *niov = &area->nia.niovs[i];
int nr;
if (!atomic_read(io_get_user_counter(niov)))
continue;
nr = atomic_xchg(io_get_user_counter(niov), 0);
if (nr && !page_pool_unref_netmem(net_iov_to_netmem(niov), nr))
io_zcrx_return_niov(niov);
}
}
void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx)
{
struct io_zcrx_ifq *ifq;
unsigned long index;
lockdep_assert_held(&ctx->uring_lock);
xa_for_each(&ctx->zcrx_ctxs, index, ifq) {
io_zcrx_scrub(ifq);
io_close_queue(ifq);
}
}
static inline u32 io_zcrx_rqring_entries(struct io_zcrx_ifq *ifq)
{
u32 entries;
@@ -800,7 +936,7 @@ static void io_zcrx_ring_refill(struct page_pool *pp,
if (!page_pool_unref_and_test(netmem))
continue;
if (unlikely(niov->pp != pp)) {
if (unlikely(niov->desc.pp != pp)) {
io_zcrx_return_niov(niov);
continue;
}
@@ -880,15 +1016,13 @@ static int io_pp_zc_init(struct page_pool *pp)
if (ret)
return ret;
percpu_ref_get(&ifq->ctx->refs);
refcount_inc(&ifq->refs);
return 0;
}
static void io_pp_zc_destroy(struct page_pool *pp)
{
struct io_zcrx_ifq *ifq = io_pp_to_ifq(pp);
percpu_ref_put(&ifq->ctx->refs);
io_put_zcrx_ifq(io_pp_to_ifq(pp));
}
static int io_pp_nl_fill(void *mp_priv, struct sk_buff *rsp,
@@ -928,6 +1062,97 @@ static const struct memory_provider_ops io_uring_pp_zc_ops = {
.uninstall = io_pp_uninstall,
};
static unsigned zcrx_parse_rq(netmem_ref *netmem_array, unsigned nr,
struct io_zcrx_ifq *zcrx)
{
unsigned int mask = zcrx->rq_entries - 1;
unsigned int i;
guard(spinlock_bh)(&zcrx->rq_lock);
nr = min(nr, io_zcrx_rqring_entries(zcrx));
for (i = 0; i < nr; i++) {
struct io_uring_zcrx_rqe *rqe = io_zcrx_get_rqe(zcrx, mask);
struct net_iov *niov;
if (!io_parse_rqe(rqe, zcrx, &niov))
break;
netmem_array[i] = net_iov_to_netmem(niov);
}
smp_store_release(&zcrx->rq_ring->head, zcrx->cached_rq_head);
return i;
}
#define ZCRX_FLUSH_BATCH 32
static void zcrx_return_buffers(netmem_ref *netmems, unsigned nr)
{
unsigned i;
for (i = 0; i < nr; i++) {
netmem_ref netmem = netmems[i];
struct net_iov *niov = netmem_to_net_iov(netmem);
if (!io_zcrx_put_niov_uref(niov))
continue;
if (!page_pool_unref_and_test(netmem))
continue;
io_zcrx_return_niov(niov);
}
}
static int zcrx_flush_rq(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx,
struct zcrx_ctrl *ctrl)
{
struct zcrx_ctrl_flush_rq *frq = &ctrl->zc_flush;
netmem_ref netmems[ZCRX_FLUSH_BATCH];
unsigned total = 0;
unsigned nr;
if (!mem_is_zero(&frq->__resv, sizeof(frq->__resv)))
return -EINVAL;
do {
nr = zcrx_parse_rq(netmems, ZCRX_FLUSH_BATCH, zcrx);
zcrx_return_buffers(netmems, nr);
total += nr;
if (fatal_signal_pending(current))
break;
cond_resched();
} while (nr == ZCRX_FLUSH_BATCH && total < zcrx->rq_entries);
return 0;
}
int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
{
struct zcrx_ctrl ctrl;
struct io_zcrx_ifq *zcrx;
if (nr_args)
return -EINVAL;
if (copy_from_user(&ctrl, arg, sizeof(ctrl)))
return -EFAULT;
if (!mem_is_zero(&ctrl.__resv, sizeof(ctrl.__resv)))
return -EFAULT;
zcrx = xa_load(&ctx->zcrx_ctxs, ctrl.zcrx_id);
if (!zcrx)
return -ENXIO;
switch (ctrl.op) {
case ZCRX_CTRL_FLUSH_RQ:
return zcrx_flush_rq(ctx, zcrx, &ctrl);
case ZCRX_CTRL_EXPORT:
return zcrx_export(ctx, zcrx, &ctrl, arg);
}
return -EOPNOTSUPP;
}
static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov,
struct io_zcrx_ifq *ifq, int off, int len)
{
@@ -1069,13 +1294,15 @@ static int io_zcrx_recv_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
const skb_frag_t *frag, int off, int len)
{
struct net_iov *niov;
struct page_pool *pp;
if (unlikely(!skb_frag_is_net_iov(frag)))
return io_zcrx_copy_frag(req, ifq, frag, off, len);
niov = netmem_to_net_iov(frag->netmem);
if (!niov->pp || niov->pp->mp_ops != &io_uring_pp_zc_ops ||
io_pp_to_ifq(niov->pp) != ifq)
pp = niov->desc.pp;
if (!pp || pp->mp_ops != &io_uring_pp_zc_ops || io_pp_to_ifq(pp) != ifq)
return -EFAULT;
if (!io_zcrx_queue_cqe(req, niov, ifq, off + skb_frag_off(frag), len))

16
io_uring/zcrx.h
View File

@@ -39,9 +39,10 @@ struct io_zcrx_area {
};
struct io_zcrx_ifq {
struct io_ring_ctx *ctx;
struct io_zcrx_area *area;
unsigned niov_shift;
struct user_struct *user;
struct mm_struct *mm_account;
spinlock_t rq_lock ____cacheline_aligned_in_smp;
struct io_uring *rq_ring;
@@ -53,6 +54,9 @@ struct io_zcrx_ifq {
struct device *dev;
struct net_device *netdev;
netdevice_tracker netdev_tracker;
refcount_t refs;
/* counts userspace facing users like io_uring */
refcount_t user_refs;
/*
* Page pool and net configuration lock, can be taken deeper in the
@@ -63,10 +67,10 @@ struct io_zcrx_ifq {
};
#if defined(CONFIG_IO_URING_ZCRX)
int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_arg);
int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
struct io_uring_zcrx_ifq_reg __user *arg);
void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx);
void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx);
int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
struct socket *sock, unsigned int flags,
unsigned issue_flags, unsigned int *len);
@@ -81,9 +85,6 @@ static inline int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
static inline void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx)
{
}
static inline void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx)
{
}
static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
struct socket *sock, unsigned int flags,
unsigned issue_flags, unsigned int *len)
@@ -95,6 +96,11 @@ static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ct
{
return NULL;
}
static inline int io_zcrx_ctrl(struct io_ring_ctx *ctx,
void __user *arg, unsigned nr_arg)
{
return -EOPNOTSUPP;
}
#endif
int io_recvzc(struct io_kiocb *req, unsigned int issue_flags);

4
net/compat.c
View File

@@ -460,10 +460,10 @@ COMPAT_SYSCALL_DEFINE2(socketcall, int, call, u32 __user *, args)
ret = __sys_accept4(a0, compat_ptr(a1), compat_ptr(a[2]), 0);
break;
case SYS_GETSOCKNAME:
ret = __sys_getsockname(a0, compat_ptr(a1), compat_ptr(a[2]));
ret = __sys_getsockname(a0, compat_ptr(a1), compat_ptr(a[2]), 0);
break;
case SYS_GETPEERNAME:
ret = __sys_getpeername(a0, compat_ptr(a1), compat_ptr(a[2]));
ret = __sys_getsockname(a0, compat_ptr(a1), compat_ptr(a[2]), 1);
break;
case SYS_SOCKETPAIR:
ret = __sys_socketpair(a0, a1, a[2], compat_ptr(a[3]));

1
net/core/dev.h
View File

@@ -29,7 +29,6 @@ struct napi_struct *
netdev_napi_by_id_lock(struct net *net, unsigned int napi_id);
struct net_device *dev_get_by_napi_id(unsigned int napi_id);
struct net_device *netdev_get_by_index_lock(struct net *net, int ifindex);
struct net_device *__netdev_put_lock(struct net_device *dev, struct net *net);
struct net_device *
netdev_xa_find_lock(struct net *net, struct net_device *dev,

83
net/socket.c
View File

@@ -2105,78 +2105,53 @@ SYSCALL_DEFINE3(connect, int, fd, struct sockaddr __user *, uservaddr,
return __sys_connect(fd, uservaddr, addrlen);
}
/*
* Get the local address ('name') of a socket object. Move the obtained
* name to user space.
*/
int do_getsockname(struct socket *sock, int peer,
struct sockaddr __user *usockaddr, int __user *usockaddr_len)
{
struct sockaddr_storage address;
int err;
if (peer)
err = security_socket_getpeername(sock);
else
err = security_socket_getsockname(sock);
if (err)
return err;
err = READ_ONCE(sock->ops)->getname(sock, (struct sockaddr *)&address, peer);
if (err < 0)
return err;
/* "err" is actually length in this case */
return move_addr_to_user(&address, err, usockaddr, usockaddr_len);
}
/*
* Get the remote or local address ('name') of a socket object. Move the
* obtained name to user space.
*/
int __sys_getsockname(int fd, struct sockaddr __user *usockaddr,
int __user *usockaddr_len)
int __user *usockaddr_len, int peer)
{
struct socket *sock;
struct sockaddr_storage address;
CLASS(fd, f)(fd);
int err;
if (fd_empty(f))
return -EBADF;
sock = sock_from_file(fd_file(f));
if (unlikely(!sock))
return -ENOTSOCK;
err = security_socket_getsockname(sock);
if (err)
return err;
err = READ_ONCE(sock->ops)->getname(sock, (struct sockaddr *)&address, 0);
if (err < 0)
return err;
/* "err" is actually length in this case */
return move_addr_to_user(&address, err, usockaddr, usockaddr_len);
return do_getsockname(sock, peer, usockaddr, usockaddr_len);
}
SYSCALL_DEFINE3(getsockname, int, fd, struct sockaddr __user *, usockaddr,
int __user *, usockaddr_len)
{
return __sys_getsockname(fd, usockaddr, usockaddr_len);
}
/*
* Get the remote address ('name') of a socket object. Move the obtained
* name to user space.
*/
int __sys_getpeername(int fd, struct sockaddr __user *usockaddr,
int __user *usockaddr_len)
{
struct socket *sock;
struct sockaddr_storage address;
CLASS(fd, f)(fd);
int err;
if (fd_empty(f))
return -EBADF;
sock = sock_from_file(fd_file(f));
if (unlikely(!sock))
return -ENOTSOCK;
err = security_socket_getpeername(sock);
if (err)
return err;
err = READ_ONCE(sock->ops)->getname(sock, (struct sockaddr *)&address, 1);
if (err < 0)
return err;
/* "err" is actually length in this case */
return move_addr_to_user(&address, err, usockaddr, usockaddr_len);
return __sys_getsockname(fd, usockaddr, usockaddr_len, 0);
}
SYSCALL_DEFINE3(getpeername, int, fd, struct sockaddr __user *, usockaddr,
int __user *, usockaddr_len)
{
return __sys_getpeername(fd, usockaddr, usockaddr_len);
return __sys_getsockname(fd, usockaddr, usockaddr_len, 1);
}
/*
@@ -3140,12 +3115,12 @@ SYSCALL_DEFINE2(socketcall, int, call, unsigned long __user *, args)
case SYS_GETSOCKNAME:
err =
__sys_getsockname(a0, (struct sockaddr __user *)a1,
(int __user *)a[2]);
(int __user *)a[2], 0);
break;
case SYS_GETPEERNAME:
err =
__sys_getpeername(a0, (struct sockaddr __user *)a1,
(int __user *)a[2]);
__sys_getsockname(a0, (struct sockaddr __user *)a1,
(int __user *)a[2], 1);
break;
case SYS_SOCKETPAIR:
err = __sys_socketpair(a0, a1, a[2], (int __user *)a[3]);