105 static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq)  in vnic_rq_desc_avail()  argument
108 	return rq->ring.desc_avail;  in vnic_rq_desc_avail()
111 static inline unsigned int vnic_rq_desc_used(struct vnic_rq *rq)  in vnic_rq_desc_used()  argument
114 	return rq->ring.desc_count - rq->ring.desc_avail - 1;  in vnic_rq_desc_used()
117 static inline void *vnic_rq_next_desc(struct vnic_rq *rq)  in vnic_rq_next_desc()  argument
119 	return rq->to_use->desc;  in vnic_rq_next_desc()
122 static inline unsigned int vnic_rq_next_index(struct vnic_rq *rq)  in vnic_rq_next_index()  argument
124 	return rq->to_use->index;  in vnic_rq_next_index()
127 static inline void vnic_rq_post(struct vnic_rq *rq,  in vnic_rq_post()  argument
132 	struct vnic_rq_buf *buf = rq->to_use;  in vnic_rq_post()
141 	rq->to_use = buf;  in vnic_rq_post()
142 	rq->ring.desc_avail--;  in vnic_rq_post()
158 		iowrite32(buf->index, &rq->ctrl->posted_index);  in vnic_rq_post()
162 static inline void vnic_rq_return_descs(struct vnic_rq *rq, unsigned int count)  in vnic_rq_return_descs()  argument
164 	rq->ring.desc_avail += count;  in vnic_rq_return_descs()
172 static inline void vnic_rq_service(struct vnic_rq *rq,  in vnic_rq_service()  argument
174 	int desc_return, void (*buf_service)(struct vnic_rq *rq,  in vnic_rq_service()  argument
181 	buf = rq->to_clean;  in vnic_rq_service()
186 		(*buf_service)(rq, cq_desc, buf, skipped, opaque);  in vnic_rq_service()
189 			rq->ring.desc_avail++;  in vnic_rq_service()
191 		rq->to_clean = buf->next;  in vnic_rq_service()
196 		buf = rq->to_clean;  in vnic_rq_service()
200 static inline int vnic_rq_fill(struct vnic_rq *rq,  in vnic_rq_fill()  argument
201 	int (*buf_fill)(struct vnic_rq *rq))  in vnic_rq_fill()  argument
205 	while (vnic_rq_desc_avail(rq) > 0) {  in vnic_rq_fill()
207 		err = (*buf_fill)(rq);  in vnic_rq_fill()
216 static inline void enic_busy_poll_init_lock(struct vnic_rq *rq)  in enic_busy_poll_init_lock()  argument
218 	spin_lock_init(&rq->bpoll_lock);  in enic_busy_poll_init_lock()
219 	rq->bpoll_state = ENIC_POLL_STATE_IDLE;  in enic_busy_poll_init_lock()
222 static inline bool enic_poll_lock_napi(struct vnic_rq *rq)  in enic_poll_lock_napi()  argument
226 	spin_lock(&rq->bpoll_lock);  in enic_poll_lock_napi()
227 	if (rq->bpoll_state & ENIC_POLL_LOCKED) {  in enic_poll_lock_napi()
228 		WARN_ON(rq->bpoll_state & ENIC_POLL_STATE_NAPI);  in enic_poll_lock_napi()
229 		rq->bpoll_state |= ENIC_POLL_STATE_NAPI_YIELD;  in enic_poll_lock_napi()
232 		rq->bpoll_state = ENIC_POLL_STATE_NAPI;  in enic_poll_lock_napi()
234 	spin_unlock(&rq->bpoll_lock);  in enic_poll_lock_napi()
239 static inline bool enic_poll_unlock_napi(struct vnic_rq *rq)  in enic_poll_unlock_napi()  argument
243 	spin_lock(&rq->bpoll_lock);  in enic_poll_unlock_napi()
244 	WARN_ON(rq->bpoll_state &  in enic_poll_unlock_napi()
246 	if (rq->bpoll_state & ENIC_POLL_STATE_POLL_YIELD)  in enic_poll_unlock_napi()
248 	rq->bpoll_state = ENIC_POLL_STATE_IDLE;  in enic_poll_unlock_napi()
249 	spin_unlock(&rq->bpoll_lock);  in enic_poll_unlock_napi()
254 static inline bool enic_poll_lock_poll(struct vnic_rq *rq)  in enic_poll_lock_poll()  argument
258 	spin_lock_bh(&rq->bpoll_lock);  in enic_poll_lock_poll()
259 	if (rq->bpoll_state & ENIC_POLL_LOCKED) {  in enic_poll_lock_poll()
260 		rq->bpoll_state |= ENIC_POLL_STATE_POLL_YIELD;  in enic_poll_lock_poll()
263 		rq->bpoll_state |= ENIC_POLL_STATE_POLL;  in enic_poll_lock_poll()
265 	spin_unlock_bh(&rq->bpoll_lock);  in enic_poll_lock_poll()
270 static inline bool enic_poll_unlock_poll(struct vnic_rq *rq)  in enic_poll_unlock_poll()  argument
274 	spin_lock_bh(&rq->bpoll_lock);  in enic_poll_unlock_poll()
275 	WARN_ON(rq->bpoll_state & ENIC_POLL_STATE_NAPI);  in enic_poll_unlock_poll()
276 	if (rq->bpoll_state & ENIC_POLL_STATE_POLL_YIELD)  in enic_poll_unlock_poll()
278 	rq->bpoll_state = ENIC_POLL_STATE_IDLE;  in enic_poll_unlock_poll()
279 	spin_unlock_bh(&rq->bpoll_lock);  in enic_poll_unlock_poll()
284 static inline bool enic_poll_busy_polling(struct vnic_rq *rq)  in enic_poll_busy_polling()  argument
286 	WARN_ON(!(rq->bpoll_state & ENIC_POLL_LOCKED));  in enic_poll_busy_polling()
287 	return rq->bpoll_state & ENIC_POLL_USER_PEND;  in enic_poll_busy_polling()
292 static inline void enic_busy_poll_init_lock(struct vnic_rq *rq)  in enic_busy_poll_init_lock()  argument
296 static inline bool enic_poll_lock_napi(struct vnic_rq *rq)  in enic_poll_lock_napi()  argument
301 static inline bool enic_poll_unlock_napi(struct vnic_rq *rq)  in enic_poll_unlock_napi()  argument
306 static inline bool enic_poll_lock_poll(struct vnic_rq *rq)  in enic_poll_lock_poll()  argument
311 static inline bool enic_poll_unlock_poll(struct vnic_rq *rq)  in enic_poll_unlock_poll()  argument
316 static inline bool enic_poll_ll_polling(struct vnic_rq *rq)  in enic_poll_ll_polling()  argument
322 void vnic_rq_free(struct vnic_rq *rq);
323 int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index,
325 void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index,
328 unsigned int vnic_rq_error_status(struct vnic_rq *rq);
329 void vnic_rq_enable(struct vnic_rq *rq);
330 int vnic_rq_disable(struct vnic_rq *rq);
331 void vnic_rq_clean(struct vnic_rq *rq,
332 	void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf));