This source file includes following definitions.
- iowait_schedule
- iowait_tid_schedule
- iowait_sdma_drain
- iowait_sdma_pending
- iowait_sdma_inc
- iowait_sdma_add
- iowait_sdma_dec
- iowait_pio_drain
- iowait_pio_pending
- iowait_pio_inc
- iowait_pio_dec
- iowait_drain_wakeup
- iowait_get_txhead
- iowait_get_desc
- iowait_get_all_desc
- iowait_update_priority
- iowait_update_all_priority
- iowait_init_priority
- iowait_get_priority
- iowait_queue
- iowait_starve_clear
- iowait_packet_queued
- iowait_inc_wait_count
- iowait_get_tid_work
- iowait_get_ib_work
- iowait_ioww_to_iow
1 #ifndef _HFI1_IOWAIT_H
2 #define _HFI1_IOWAIT_H
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50 #include <linux/list.h>
51 #include <linux/workqueue.h>
52 #include <linux/wait.h>
53 #include <linux/sched.h>
54
55 #include "sdma_txreq.h"
56
57
58
59
60
61 typedef void (*restart_t)(struct work_struct *work);
62
63 #define IOWAIT_PENDING_IB 0x0
64 #define IOWAIT_PENDING_TID 0x1
65
66
67
68
69
70
71
72 #define IOWAIT_SES 2
73 #define IOWAIT_IB_SE 0
74 #define IOWAIT_TID_SE 1
75
76 struct sdma_txreq;
77 struct sdma_engine;
78
79
80
81
82
83
84
85
86
87
88
89
90 struct iowait;
91 struct iowait_work {
92 struct work_struct iowork;
93 struct list_head tx_head;
94 struct iowait *iow;
95 };
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139 struct iowait {
140 struct list_head list;
141 int (*sleep)(
142 struct sdma_engine *sde,
143 struct iowait_work *wait,
144 struct sdma_txreq *tx,
145 uint seq,
146 bool pkts_sent
147 );
148 void (*wakeup)(struct iowait *wait, int reason);
149 void (*sdma_drained)(struct iowait *wait);
150 void (*init_priority)(struct iowait *wait);
151 seqlock_t *lock;
152 wait_queue_head_t wait_dma;
153 wait_queue_head_t wait_pio;
154 atomic_t sdma_busy;
155 atomic_t pio_busy;
156 u32 count;
157 u32 tx_limit;
158 u32 tx_count;
159 u8 starved_cnt;
160 u8 priority;
161 unsigned long flags;
162 struct iowait_work wait[IOWAIT_SES];
163 };
164
165 #define SDMA_AVAIL_REASON 0
166
167 void iowait_set_flag(struct iowait *wait, u32 flag);
168 bool iowait_flag_set(struct iowait *wait, u32 flag);
169 void iowait_clear_flag(struct iowait *wait, u32 flag);
170
171 void iowait_init(struct iowait *wait, u32 tx_limit,
172 void (*func)(struct work_struct *work),
173 void (*tidfunc)(struct work_struct *work),
174 int (*sleep)(struct sdma_engine *sde,
175 struct iowait_work *wait,
176 struct sdma_txreq *tx,
177 uint seq,
178 bool pkts_sent),
179 void (*wakeup)(struct iowait *wait, int reason),
180 void (*sdma_drained)(struct iowait *wait),
181 void (*init_priority)(struct iowait *wait));
182
183
184
185
186
187
188
189 static inline bool iowait_schedule(struct iowait *wait,
190 struct workqueue_struct *wq, int cpu)
191 {
192 return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_IB_SE].iowork);
193 }
194
195
196
197
198
199
200
201 static inline bool iowait_tid_schedule(struct iowait *wait,
202 struct workqueue_struct *wq, int cpu)
203 {
204 return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_TID_SE].iowork);
205 }
206
207
208
209
210
211
212
213
214
215 static inline void iowait_sdma_drain(struct iowait *wait)
216 {
217 wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy));
218 }
219
220
221
222
223
224
225
226 static inline int iowait_sdma_pending(struct iowait *wait)
227 {
228 return atomic_read(&wait->sdma_busy);
229 }
230
231
232
233
234
235 static inline void iowait_sdma_inc(struct iowait *wait)
236 {
237 atomic_inc(&wait->sdma_busy);
238 }
239
240
241
242
243
244 static inline void iowait_sdma_add(struct iowait *wait, int count)
245 {
246 atomic_add(count, &wait->sdma_busy);
247 }
248
249
250
251
252
253 static inline int iowait_sdma_dec(struct iowait *wait)
254 {
255 if (!wait)
256 return 0;
257 return atomic_dec_and_test(&wait->sdma_busy);
258 }
259
260
261
262
263
264
265
266
267
268 static inline void iowait_pio_drain(struct iowait *wait)
269 {
270 wait_event_timeout(wait->wait_pio,
271 !atomic_read(&wait->pio_busy),
272 HZ);
273 }
274
275
276
277
278
279
280
281 static inline int iowait_pio_pending(struct iowait *wait)
282 {
283 return atomic_read(&wait->pio_busy);
284 }
285
286
287
288
289
290 static inline void iowait_pio_inc(struct iowait *wait)
291 {
292 atomic_inc(&wait->pio_busy);
293 }
294
295
296
297
298
299 static inline int iowait_pio_dec(struct iowait *wait)
300 {
301 if (!wait)
302 return 0;
303 return atomic_dec_and_test(&wait->pio_busy);
304 }
305
306
307
308
309
310
311
312
313 static inline void iowait_drain_wakeup(struct iowait *wait)
314 {
315 wake_up(&wait->wait_dma);
316 wake_up(&wait->wait_pio);
317 if (wait->sdma_drained)
318 wait->sdma_drained(wait);
319 }
320
321
322
323
324
325
326 static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait)
327 {
328 struct sdma_txreq *tx = NULL;
329
330 if (!list_empty(&wait->tx_head)) {
331 tx = list_first_entry(
332 &wait->tx_head,
333 struct sdma_txreq,
334 list);
335 list_del_init(&tx->list);
336 }
337 return tx;
338 }
339
340 static inline u16 iowait_get_desc(struct iowait_work *w)
341 {
342 u16 num_desc = 0;
343 struct sdma_txreq *tx = NULL;
344
345 if (!list_empty(&w->tx_head)) {
346 tx = list_first_entry(&w->tx_head, struct sdma_txreq,
347 list);
348 num_desc = tx->num_desc;
349 if (tx->flags & SDMA_TXREQ_F_VIP)
350 w->iow->priority++;
351 }
352 return num_desc;
353 }
354
355 static inline u32 iowait_get_all_desc(struct iowait *w)
356 {
357 u32 num_desc = 0;
358
359 num_desc = iowait_get_desc(&w->wait[IOWAIT_IB_SE]);
360 num_desc += iowait_get_desc(&w->wait[IOWAIT_TID_SE]);
361 return num_desc;
362 }
363
364 static inline void iowait_update_priority(struct iowait_work *w)
365 {
366 struct sdma_txreq *tx = NULL;
367
368 if (!list_empty(&w->tx_head)) {
369 tx = list_first_entry(&w->tx_head, struct sdma_txreq,
370 list);
371 if (tx->flags & SDMA_TXREQ_F_VIP)
372 w->iow->priority++;
373 }
374 }
375
376 static inline void iowait_update_all_priority(struct iowait *w)
377 {
378 iowait_update_priority(&w->wait[IOWAIT_IB_SE]);
379 iowait_update_priority(&w->wait[IOWAIT_TID_SE]);
380 }
381
382 static inline void iowait_init_priority(struct iowait *w)
383 {
384 w->priority = 0;
385 if (w->init_priority)
386 w->init_priority(w);
387 }
388
389 static inline void iowait_get_priority(struct iowait *w)
390 {
391 iowait_init_priority(w);
392 iowait_update_all_priority(w);
393 }
394
395
396
397
398
399
400
401
402
403
404
405 static inline void iowait_queue(bool pkts_sent, struct iowait *w,
406 struct list_head *wait_head)
407 {
408
409
410
411
412
413
414 if (pkts_sent)
415 w->starved_cnt = 0;
416 else
417 w->starved_cnt++;
418
419 if (w->priority > 0 || !pkts_sent)
420 list_add(&w->list, wait_head);
421 else
422 list_add_tail(&w->list, wait_head);
423 }
424
425
426
427
428
429
430
431
432
433 static inline void iowait_starve_clear(bool pkts_sent, struct iowait *w)
434 {
435 if (pkts_sent)
436 w->starved_cnt = 0;
437 }
438
439
440 uint iowait_priority_update_top(struct iowait *w,
441 struct iowait *top,
442 uint idx, uint top_idx);
443
444
445
446
447
448 static inline bool iowait_packet_queued(struct iowait_work *wait)
449 {
450 return !list_empty(&wait->tx_head);
451 }
452
453
454
455
456
457
458 static inline void iowait_inc_wait_count(struct iowait_work *w, u16 n)
459 {
460 if (!w)
461 return;
462 w->iow->tx_count++;
463 w->iow->count += n;
464 }
465
466
467
468
469
470 static inline struct iowait_work *iowait_get_tid_work(struct iowait *w)
471 {
472 return &w->wait[IOWAIT_TID_SE];
473 }
474
475
476
477
478
479 static inline struct iowait_work *iowait_get_ib_work(struct iowait *w)
480 {
481 return &w->wait[IOWAIT_IB_SE];
482 }
483
484
485
486
487
488 static inline struct iowait *iowait_ioww_to_iow(struct iowait_work *w)
489 {
490 if (likely(w))
491 return w->iow;
492 return NULL;
493 }
494
495 void iowait_cancel_work(struct iowait *w);
496 int iowait_set_work_flag(struct iowait_work *w);
497
498 #endif