1/* 2 * Copyright 2008-2010 Cisco Systems, Inc. All rights reserved. 3 * Copyright 2007 Nuova Systems, Inc. All rights reserved. 4 * 5 * This program is free software; you may redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; version 2 of the License. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 10 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 11 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 12 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 13 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 14 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 15 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 16 * SOFTWARE. 17 * 18 */ 19 20#ifndef _VNIC_RQ_H_ 21#define _VNIC_RQ_H_ 22 23#include <linux/pci.h> 24 25#include "vnic_dev.h" 26#include "vnic_cq.h" 27 28/* Receive queue control */ 29struct vnic_rq_ctrl { 30 u64 ring_base; /* 0x00 */ 31 u32 ring_size; /* 0x08 */ 32 u32 pad0; 33 u32 posted_index; /* 0x10 */ 34 u32 pad1; 35 u32 cq_index; /* 0x18 */ 36 u32 pad2; 37 u32 enable; /* 0x20 */ 38 u32 pad3; 39 u32 running; /* 0x28 */ 40 u32 pad4; 41 u32 fetch_index; /* 0x30 */ 42 u32 pad5; 43 u32 error_interrupt_enable; /* 0x38 */ 44 u32 pad6; 45 u32 error_interrupt_offset; /* 0x40 */ 46 u32 pad7; 47 u32 error_status; /* 0x48 */ 48 u32 pad8; 49 u32 dropped_packet_count; /* 0x50 */ 50 u32 pad9; 51 u32 dropped_packet_count_rc; /* 0x58 */ 52 u32 pad10; 53}; 54 55/* Break the vnic_rq_buf allocations into blocks of 32/64 entries */ 56#define VNIC_RQ_BUF_MIN_BLK_ENTRIES 32 57#define VNIC_RQ_BUF_DFLT_BLK_ENTRIES 64 58#define VNIC_RQ_BUF_BLK_ENTRIES(entries) \ 59 ((unsigned int)((entries < VNIC_RQ_BUF_DFLT_BLK_ENTRIES) ? \ 60 VNIC_RQ_BUF_MIN_BLK_ENTRIES : VNIC_RQ_BUF_DFLT_BLK_ENTRIES)) 61#define VNIC_RQ_BUF_BLK_SZ(entries) \ 62 (VNIC_RQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_rq_buf)) 63#define VNIC_RQ_BUF_BLKS_NEEDED(entries) \ 64 DIV_ROUND_UP(entries, VNIC_RQ_BUF_BLK_ENTRIES(entries)) 65#define VNIC_RQ_BUF_BLKS_MAX VNIC_RQ_BUF_BLKS_NEEDED(4096) 66 67struct vnic_rq_buf { 68 struct vnic_rq_buf *next; 69 dma_addr_t dma_addr; 70 void *os_buf; 71 unsigned int os_buf_index; 72 unsigned int len; 73 unsigned int index; 74 void *desc; 75 uint64_t wr_id; 76}; 77 78struct vnic_rq { 79 unsigned int index; 80 struct vnic_dev *vdev; 81 struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */ 82 struct vnic_dev_ring ring; 83 struct vnic_rq_buf *bufs[VNIC_RQ_BUF_BLKS_MAX]; 84 struct vnic_rq_buf *to_use; 85 struct vnic_rq_buf *to_clean; 86 void *os_buf_head; 87 unsigned int pkts_outstanding; 88#ifdef CONFIG_NET_RX_BUSY_POLL 89#define ENIC_POLL_STATE_IDLE 0 90#define ENIC_POLL_STATE_NAPI (1 << 0) /* NAPI owns this poll */ 91#define ENIC_POLL_STATE_POLL (1 << 1) /* poll owns this poll */ 92#define ENIC_POLL_STATE_NAPI_YIELD (1 << 2) /* NAPI yielded this poll */ 93#define ENIC_POLL_STATE_POLL_YIELD (1 << 3) /* poll yielded this poll */ 94#define ENIC_POLL_YIELD (ENIC_POLL_STATE_NAPI_YIELD | \ 95 ENIC_POLL_STATE_POLL_YIELD) 96#define ENIC_POLL_LOCKED (ENIC_POLL_STATE_NAPI | \ 97 ENIC_POLL_STATE_POLL) 98#define ENIC_POLL_USER_PEND (ENIC_POLL_STATE_POLL | \ 99 ENIC_POLL_STATE_POLL_YIELD) 100 unsigned int bpoll_state; 101 spinlock_t bpoll_lock; 102#endif /* CONFIG_NET_RX_BUSY_POLL */ 103}; 104 105static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq) 106{ 107 /* how many does SW own? */ 108 return rq->ring.desc_avail; 109} 110 111static inline unsigned int vnic_rq_desc_used(struct vnic_rq *rq) 112{ 113 /* how many does HW own? */ 114 return rq->ring.desc_count - rq->ring.desc_avail - 1; 115} 116 117static inline void *vnic_rq_next_desc(struct vnic_rq *rq) 118{ 119 return rq->to_use->desc; 120} 121 122static inline unsigned int vnic_rq_next_index(struct vnic_rq *rq) 123{ 124 return rq->to_use->index; 125} 126 127static inline void vnic_rq_post(struct vnic_rq *rq, 128 void *os_buf, unsigned int os_buf_index, 129 dma_addr_t dma_addr, unsigned int len, 130 uint64_t wrid) 131{ 132 struct vnic_rq_buf *buf = rq->to_use; 133 134 buf->os_buf = os_buf; 135 buf->os_buf_index = os_buf_index; 136 buf->dma_addr = dma_addr; 137 buf->len = len; 138 buf->wr_id = wrid; 139 140 buf = buf->next; 141 rq->to_use = buf; 142 rq->ring.desc_avail--; 143 144 /* Move the posted_index every nth descriptor 145 */ 146 147#ifndef VNIC_RQ_RETURN_RATE 148#define VNIC_RQ_RETURN_RATE 0xf /* keep 2^n - 1 */ 149#endif 150 151 if ((buf->index & VNIC_RQ_RETURN_RATE) == 0) { 152 /* Adding write memory barrier prevents compiler and/or CPU 153 * reordering, thus avoiding descriptor posting before 154 * descriptor is initialized. Otherwise, hardware can read 155 * stale descriptor fields. 156 */ 157 wmb(); 158 iowrite32(buf->index, &rq->ctrl->posted_index); 159 } 160} 161 162static inline void vnic_rq_return_descs(struct vnic_rq *rq, unsigned int count) 163{ 164 rq->ring.desc_avail += count; 165} 166 167enum desc_return_options { 168 VNIC_RQ_RETURN_DESC, 169 VNIC_RQ_DEFER_RETURN_DESC, 170}; 171 172static inline void vnic_rq_service(struct vnic_rq *rq, 173 struct cq_desc *cq_desc, u16 completed_index, 174 int desc_return, void (*buf_service)(struct vnic_rq *rq, 175 struct cq_desc *cq_desc, struct vnic_rq_buf *buf, 176 int skipped, void *opaque), void *opaque) 177{ 178 struct vnic_rq_buf *buf; 179 int skipped; 180 181 buf = rq->to_clean; 182 while (1) { 183 184 skipped = (buf->index != completed_index); 185 186 (*buf_service)(rq, cq_desc, buf, skipped, opaque); 187 188 if (desc_return == VNIC_RQ_RETURN_DESC) 189 rq->ring.desc_avail++; 190 191 rq->to_clean = buf->next; 192 193 if (!skipped) 194 break; 195 196 buf = rq->to_clean; 197 } 198} 199 200static inline int vnic_rq_fill(struct vnic_rq *rq, 201 int (*buf_fill)(struct vnic_rq *rq)) 202{ 203 int err; 204 205 while (vnic_rq_desc_avail(rq) > 0) { 206 207 err = (*buf_fill)(rq); 208 if (err) 209 return err; 210 } 211 212 return 0; 213} 214 215#ifdef CONFIG_NET_RX_BUSY_POLL 216static inline void enic_busy_poll_init_lock(struct vnic_rq *rq) 217{ 218 spin_lock_init(&rq->bpoll_lock); 219 rq->bpoll_state = ENIC_POLL_STATE_IDLE; 220} 221 222static inline bool enic_poll_lock_napi(struct vnic_rq *rq) 223{ 224 bool rc = true; 225 226 spin_lock(&rq->bpoll_lock); 227 if (rq->bpoll_state & ENIC_POLL_LOCKED) { 228 WARN_ON(rq->bpoll_state & ENIC_POLL_STATE_NAPI); 229 rq->bpoll_state |= ENIC_POLL_STATE_NAPI_YIELD; 230 rc = false; 231 } else { 232 rq->bpoll_state = ENIC_POLL_STATE_NAPI; 233 } 234 spin_unlock(&rq->bpoll_lock); 235 236 return rc; 237} 238 239static inline bool enic_poll_unlock_napi(struct vnic_rq *rq) 240{ 241 bool rc = false; 242 243 spin_lock(&rq->bpoll_lock); 244 WARN_ON(rq->bpoll_state & 245 (ENIC_POLL_STATE_POLL | ENIC_POLL_STATE_NAPI_YIELD)); 246 if (rq->bpoll_state & ENIC_POLL_STATE_POLL_YIELD) 247 rc = true; 248 rq->bpoll_state = ENIC_POLL_STATE_IDLE; 249 spin_unlock(&rq->bpoll_lock); 250 251 return rc; 252} 253 254static inline bool enic_poll_lock_poll(struct vnic_rq *rq) 255{ 256 bool rc = true; 257 258 spin_lock_bh(&rq->bpoll_lock); 259 if (rq->bpoll_state & ENIC_POLL_LOCKED) { 260 rq->bpoll_state |= ENIC_POLL_STATE_POLL_YIELD; 261 rc = false; 262 } else { 263 rq->bpoll_state |= ENIC_POLL_STATE_POLL; 264 } 265 spin_unlock_bh(&rq->bpoll_lock); 266 267 return rc; 268} 269 270static inline bool enic_poll_unlock_poll(struct vnic_rq *rq) 271{ 272 bool rc = false; 273 274 spin_lock_bh(&rq->bpoll_lock); 275 WARN_ON(rq->bpoll_state & ENIC_POLL_STATE_NAPI); 276 if (rq->bpoll_state & ENIC_POLL_STATE_POLL_YIELD) 277 rc = true; 278 rq->bpoll_state = ENIC_POLL_STATE_IDLE; 279 spin_unlock_bh(&rq->bpoll_lock); 280 281 return rc; 282} 283 284static inline bool enic_poll_busy_polling(struct vnic_rq *rq) 285{ 286 WARN_ON(!(rq->bpoll_state & ENIC_POLL_LOCKED)); 287 return rq->bpoll_state & ENIC_POLL_USER_PEND; 288} 289 290#else 291 292static inline void enic_busy_poll_init_lock(struct vnic_rq *rq) 293{ 294} 295 296static inline bool enic_poll_lock_napi(struct vnic_rq *rq) 297{ 298 return true; 299} 300 301static inline bool enic_poll_unlock_napi(struct vnic_rq *rq) 302{ 303 return false; 304} 305 306static inline bool enic_poll_lock_poll(struct vnic_rq *rq) 307{ 308 return false; 309} 310 311static inline bool enic_poll_unlock_poll(struct vnic_rq *rq) 312{ 313 return false; 314} 315 316static inline bool enic_poll_ll_polling(struct vnic_rq *rq) 317{ 318 return false; 319} 320#endif /* CONFIG_NET_RX_BUSY_POLL */ 321 322void vnic_rq_free(struct vnic_rq *rq); 323int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, 324 unsigned int desc_count, unsigned int desc_size); 325void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, 326 unsigned int error_interrupt_enable, 327 unsigned int error_interrupt_offset); 328unsigned int vnic_rq_error_status(struct vnic_rq *rq); 329void vnic_rq_enable(struct vnic_rq *rq); 330int vnic_rq_disable(struct vnic_rq *rq); 331void vnic_rq_clean(struct vnic_rq *rq, 332 void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)); 333 334#endif /* _VNIC_RQ_H_ */ 335