1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Intel MIC Platform Software Stack (MPSS)
4 *
5 * Copyright(c) 2014 Intel Corporation.
6 *
7 * Intel SCIF driver.
8 */
9 #include <linux/circ_buf.h>
10 #include <linux/types.h>
11 #include <linux/io.h>
12 #include <linux/errno.h>
13
14 #include "scif_rb.h"
15
16 #define scif_rb_ring_cnt(head, tail, size) CIRC_CNT(head, tail, size)
17 #define scif_rb_ring_space(head, tail, size) CIRC_SPACE(head, tail, size)
18
19 /**
20 * scif_rb_init - Initializes the ring buffer
21 * @rb: ring buffer
22 * @read_ptr: A pointer to the read offset
23 * @write_ptr: A pointer to the write offset
24 * @rb_base: A pointer to the base of the ring buffer
25 * @size: The size of the ring buffer in powers of two
26 */
27 void scif_rb_init(struct scif_rb *rb, u32 *read_ptr, u32 *write_ptr,
28 void *rb_base, u8 size)
29 {
30 rb->rb_base = rb_base;
31 rb->size = (1 << size);
32 rb->read_ptr = read_ptr;
33 rb->write_ptr = write_ptr;
34 rb->current_read_offset = *read_ptr;
35 rb->current_write_offset = *write_ptr;
36 }
37
38 /* Copies a message to the ring buffer -- handles the wrap around case */
39 static void memcpy_torb(struct scif_rb *rb, void *header,
40 void *msg, u32 size)
41 {
42 u32 size1, size2;
43
44 if (header + size >= rb->rb_base + rb->size) {
45 /* Need to call two copies if it wraps around */
46 size1 = (u32)(rb->rb_base + rb->size - header);
47 size2 = size - size1;
48 memcpy_toio((void __iomem __force *)header, msg, size1);
49 memcpy_toio((void __iomem __force *)rb->rb_base,
50 msg + size1, size2);
51 } else {
52 memcpy_toio((void __iomem __force *)header, msg, size);
53 }
54 }
55
56 /* Copies a message from the ring buffer -- handles the wrap around case */
57 static void memcpy_fromrb(struct scif_rb *rb, void *header,
58 void *msg, u32 size)
59 {
60 u32 size1, size2;
61
62 if (header + size >= rb->rb_base + rb->size) {
63 /* Need to call two copies if it wraps around */
64 size1 = (u32)(rb->rb_base + rb->size - header);
65 size2 = size - size1;
66 memcpy_fromio(msg, (void __iomem __force *)header, size1);
67 memcpy_fromio(msg + size1,
68 (void __iomem __force *)rb->rb_base, size2);
69 } else {
70 memcpy_fromio(msg, (void __iomem __force *)header, size);
71 }
72 }
73
74 /**
75 * scif_rb_space - Query space available for writing to the RB
76 * @rb: ring buffer
77 *
78 * Return: size available for writing to RB in bytes.
79 */
80 u32 scif_rb_space(struct scif_rb *rb)
81 {
82 rb->current_read_offset = *rb->read_ptr;
83 /*
84 * Update from the HW read pointer only once the peer has exposed the
85 * new empty slot. This barrier is paired with the memory barrier
86 * scif_rb_update_read_ptr()
87 */
88 mb();
89 return scif_rb_ring_space(rb->current_write_offset,
90 rb->current_read_offset, rb->size);
91 }
92
93 /**
94 * scif_rb_write - Write a message to the RB
95 * @rb: ring buffer
96 * @msg: buffer to send the message. Must be at least size bytes long
97 * @size: the size (in bytes) to be copied to the RB
98 *
99 * This API does not block if there isn't enough space in the RB.
100 * Returns: 0 on success or -ENOMEM on failure
101 */
102 int scif_rb_write(struct scif_rb *rb, void *msg, u32 size)
103 {
104 void *header;
105
106 if (scif_rb_space(rb) < size)
107 return -ENOMEM;
108 header = rb->rb_base + rb->current_write_offset;
109 memcpy_torb(rb, header, msg, size);
110 /*
111 * Wait until scif_rb_commit(). Update the local ring
112 * buffer data, not the shared data until commit.
113 */
114 rb->current_write_offset =
115 (rb->current_write_offset + size) & (rb->size - 1);
116 return 0;
117 }
118
119 /**
120 * scif_rb_commit - To submit the message to let the peer fetch it
121 * @rb: ring buffer
122 */
123 void scif_rb_commit(struct scif_rb *rb)
124 {
125 /*
126 * We must ensure ordering between the all the data committed
127 * previously before we expose the new message to the peer by
128 * updating the write_ptr. This write barrier is paired with
129 * the read barrier in scif_rb_count(..)
130 */
131 wmb();
132 WRITE_ONCE(*rb->write_ptr, rb->current_write_offset);
133 #ifdef CONFIG_INTEL_MIC_CARD
134 /*
135 * X100 Si bug: For the case where a Core is performing an EXT_WR
136 * followed by a Doorbell Write, the Core must perform two EXT_WR to the
137 * same address with the same data before it does the Doorbell Write.
138 * This way, if ordering is violated for the Interrupt Message, it will
139 * fall just behind the first Posted associated with the first EXT_WR.
140 */
141 WRITE_ONCE(*rb->write_ptr, rb->current_write_offset);
142 #endif
143 }
144
145 /**
146 * scif_rb_get - To get next message from the ring buffer
147 * @rb: ring buffer
148 * @size: Number of bytes to be read
149 *
150 * Return: NULL if no bytes to be read from the ring buffer, otherwise the
151 * pointer to the next byte
152 */
153 static void *scif_rb_get(struct scif_rb *rb, u32 size)
154 {
155 void *header = NULL;
156
157 if (scif_rb_count(rb, size) >= size)
158 header = rb->rb_base + rb->current_read_offset;
159 return header;
160 }
161
162 /*
163 * scif_rb_get_next - Read from ring buffer.
164 * @rb: ring buffer
165 * @msg: buffer to hold the message. Must be at least size bytes long
166 * @size: Number of bytes to be read
167 *
168 * Return: number of bytes read if available bytes are >= size, otherwise
169 * returns zero.
170 */
171 u32 scif_rb_get_next(struct scif_rb *rb, void *msg, u32 size)
172 {
173 void *header = NULL;
174 int read_size = 0;
175
176 header = scif_rb_get(rb, size);
177 if (header) {
178 u32 next_cmd_offset =
179 (rb->current_read_offset + size) & (rb->size - 1);
180
181 read_size = size;
182 rb->current_read_offset = next_cmd_offset;
183 memcpy_fromrb(rb, header, msg, size);
184 }
185 return read_size;
186 }
187
188 /**
189 * scif_rb_update_read_ptr
190 * @rb: ring buffer
191 */
192 void scif_rb_update_read_ptr(struct scif_rb *rb)
193 {
194 u32 new_offset;
195
196 new_offset = rb->current_read_offset;
197 /*
198 * We must ensure ordering between the all the data committed or read
199 * previously before we expose the empty slot to the peer by updating
200 * the read_ptr. This barrier is paired with the memory barrier in
201 * scif_rb_space(..)
202 */
203 mb();
204 WRITE_ONCE(*rb->read_ptr, new_offset);
205 #ifdef CONFIG_INTEL_MIC_CARD
206 /*
207 * X100 Si Bug: For the case where a Core is performing an EXT_WR
208 * followed by a Doorbell Write, the Core must perform two EXT_WR to the
209 * same address with the same data before it does the Doorbell Write.
210 * This way, if ordering is violated for the Interrupt Message, it will
211 * fall just behind the first Posted associated with the first EXT_WR.
212 */
213 WRITE_ONCE(*rb->read_ptr, new_offset);
214 #endif
215 }
216
217 /**
218 * scif_rb_count
219 * @rb: ring buffer
220 * @size: Number of bytes expected to be read
221 *
222 * Return: number of bytes that can be read from the RB
223 */
224 u32 scif_rb_count(struct scif_rb *rb, u32 size)
225 {
226 if (scif_rb_ring_cnt(rb->current_write_offset,
227 rb->current_read_offset,
228 rb->size) < size) {
229 rb->current_write_offset = *rb->write_ptr;
230 /*
231 * Update from the HW write pointer if empty only once the peer
232 * has exposed the new message. This read barrier is paired
233 * with the write barrier in scif_rb_commit(..)
234 */
235 smp_rmb();
236 }
237 return scif_rb_ring_cnt(rb->current_write_offset,
238 rb->current_read_offset,
239 rb->size);
240 }