1 #include <sys/types.h>
2 #include <sys/stat.h>
3 #include <sys/mman.h>
4 #include <unistd.h>
5 #include <errno.h>
6 #include <string.h>
7
8 #include "liburing.h"
9 #include "barrier.h"
10
11 static int __io_uring_get_cqe(struct io_uring *ring,
12 struct io_uring_cqe **cqe_ptr, int wait)
13 {
14 struct io_uring_cq *cq = &ring->cq;
15 const unsigned mask = *cq->kring_mask;
16 unsigned head;
17 int ret;
18
19 *cqe_ptr = NULL;
20 head = *cq->khead;
21 do {
22 /*
23 * It's necessary to use a read_barrier() before reading
24 * the CQ tail, since the kernel updates it locklessly. The
25 * kernel has the matching store barrier for the update. The
26 * kernel also ensures that previous stores to CQEs are ordered
27 * with the tail update.
28 */
29 read_barrier();
30 if (head != *cq->ktail) {
31 *cqe_ptr = &cq->cqes[head & mask];
32 break;
33 }
34 if (!wait)
35 break;
36 ret = io_uring_enter(ring->ring_fd, 0, 1,
37 IORING_ENTER_GETEVENTS, NULL);
38 if (ret < 0)
39 return -errno;
40 } while (1);
41
42 return 0;
43 }
44
45 /*
46 * Return an IO completion, if one is readily available. Returns 0 with
47 * cqe_ptr filled in on success, -errno on failure.
48 */
49 int io_uring_peek_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr)
50 {
51 return __io_uring_get_cqe(ring, cqe_ptr, 0);
52 }
53
54 /*
55 * Return an IO completion, waiting for it if necessary. Returns 0 with
56 * cqe_ptr filled in on success, -errno on failure.
57 */
58 int io_uring_wait_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr)
59 {
60 return __io_uring_get_cqe(ring, cqe_ptr, 1);
61 }
62
63 /*
64 * Submit sqes acquired from io_uring_get_sqe() to the kernel.
65 *
66 * Returns number of sqes submitted
67 */
68 int io_uring_submit(struct io_uring *ring)
69 {
70 struct io_uring_sq *sq = &ring->sq;
71 const unsigned mask = *sq->kring_mask;
72 unsigned ktail, ktail_next, submitted, to_submit;
73 int ret;
74
75 /*
76 * If we have pending IO in the kring, submit it first. We need a
77 * read barrier here to match the kernels store barrier when updating
78 * the SQ head.
79 */
80 read_barrier();
81 if (*sq->khead != *sq->ktail) {
82 submitted = *sq->kring_entries;
83 goto submit;
84 }
85
86 if (sq->sqe_head == sq->sqe_tail)
87 return 0;
88
89 /*
90 * Fill in sqes that we have queued up, adding them to the kernel ring
91 */
92 submitted = 0;
93 ktail = ktail_next = *sq->ktail;
94 to_submit = sq->sqe_tail - sq->sqe_head;
95 while (to_submit--) {
96 ktail_next++;
97 read_barrier();
98
99 sq->array[ktail & mask] = sq->sqe_head & mask;
100 ktail = ktail_next;
101
102 sq->sqe_head++;
103 submitted++;
104 }
105
106 if (!submitted)
107 return 0;
108
109 if (*sq->ktail != ktail) {
110 /*
111 * First write barrier ensures that the SQE stores are updated
112 * with the tail update. This is needed so that the kernel
113 * will never see a tail update without the preceeding sQE
114 * stores being done.
115 */
116 write_barrier();
117 *sq->ktail = ktail;
118 /*
119 * The kernel has the matching read barrier for reading the
120 * SQ tail.
121 */
122 write_barrier();
123 }
124
125 submit:
126 ret = io_uring_enter(ring->ring_fd, submitted, 0,
127 IORING_ENTER_GETEVENTS, NULL);
128 if (ret < 0)
129 return -errno;
130
131 return ret;
132 }
133
134 /*
135 * Return an sqe to fill. Application must later call io_uring_submit()
136 * when it's ready to tell the kernel about it. The caller may call this
137 * function multiple times before calling io_uring_submit().
138 *
139 * Returns a vacant sqe, or NULL if we're full.
140 */
141 struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)
142 {
143 struct io_uring_sq *sq = &ring->sq;
144 unsigned next = sq->sqe_tail + 1;
145 struct io_uring_sqe *sqe;
146
147 /*
148 * All sqes are used
149 */
150 if (next - sq->sqe_head > *sq->kring_entries)
151 return NULL;
152
153 sqe = &sq->sqes[sq->sqe_tail & *sq->kring_mask];
154 sq->sqe_tail = next;
155 return sqe;
156 }