1/* 2 * Copyright (c) 2012, 2013 Intel Corporation. All rights reserved. 3 * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved. 4 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35#include <rdma/ib_mad.h> 36#include <rdma/ib_user_verbs.h> 37#include <linux/io.h> 38#include <linux/module.h> 39#include <linux/utsname.h> 40#include <linux/rculist.h> 41#include <linux/mm.h> 42#include <linux/random.h> 43#include <linux/vmalloc.h> 44 45#include "qib.h" 46#include "qib_common.h" 47 48static unsigned int ib_qib_qp_table_size = 256; 49module_param_named(qp_table_size, ib_qib_qp_table_size, uint, S_IRUGO); 50MODULE_PARM_DESC(qp_table_size, "QP table size"); 51 52unsigned int ib_qib_lkey_table_size = 16; 53module_param_named(lkey_table_size, ib_qib_lkey_table_size, uint, 54 S_IRUGO); 55MODULE_PARM_DESC(lkey_table_size, 56 "LKEY table size in bits (2^n, 1 <= n <= 23)"); 57 58static unsigned int ib_qib_max_pds = 0xFFFF; 59module_param_named(max_pds, ib_qib_max_pds, uint, S_IRUGO); 60MODULE_PARM_DESC(max_pds, 61 "Maximum number of protection domains to support"); 62 63static unsigned int ib_qib_max_ahs = 0xFFFF; 64module_param_named(max_ahs, ib_qib_max_ahs, uint, S_IRUGO); 65MODULE_PARM_DESC(max_ahs, "Maximum number of address handles to support"); 66 67unsigned int ib_qib_max_cqes = 0x2FFFF; 68module_param_named(max_cqes, ib_qib_max_cqes, uint, S_IRUGO); 69MODULE_PARM_DESC(max_cqes, 70 "Maximum number of completion queue entries to support"); 71 72unsigned int ib_qib_max_cqs = 0x1FFFF; 73module_param_named(max_cqs, ib_qib_max_cqs, uint, S_IRUGO); 74MODULE_PARM_DESC(max_cqs, "Maximum number of completion queues to support"); 75 76unsigned int ib_qib_max_qp_wrs = 0x3FFF; 77module_param_named(max_qp_wrs, ib_qib_max_qp_wrs, uint, S_IRUGO); 78MODULE_PARM_DESC(max_qp_wrs, "Maximum number of QP WRs to support"); 79 80unsigned int ib_qib_max_qps = 16384; 81module_param_named(max_qps, ib_qib_max_qps, uint, S_IRUGO); 82MODULE_PARM_DESC(max_qps, "Maximum number of QPs to support"); 83 84unsigned int ib_qib_max_sges = 0x60; 85module_param_named(max_sges, ib_qib_max_sges, uint, S_IRUGO); 86MODULE_PARM_DESC(max_sges, "Maximum number of SGEs to support"); 87 88unsigned int ib_qib_max_mcast_grps = 16384; 89module_param_named(max_mcast_grps, ib_qib_max_mcast_grps, uint, S_IRUGO); 90MODULE_PARM_DESC(max_mcast_grps, 91 "Maximum number of multicast groups to support"); 92 93unsigned int ib_qib_max_mcast_qp_attached = 16; 94module_param_named(max_mcast_qp_attached, ib_qib_max_mcast_qp_attached, 95 uint, S_IRUGO); 96MODULE_PARM_DESC(max_mcast_qp_attached, 97 "Maximum number of attached QPs to support"); 98 99unsigned int ib_qib_max_srqs = 1024; 100module_param_named(max_srqs, ib_qib_max_srqs, uint, S_IRUGO); 101MODULE_PARM_DESC(max_srqs, "Maximum number of SRQs to support"); 102 103unsigned int ib_qib_max_srq_sges = 128; 104module_param_named(max_srq_sges, ib_qib_max_srq_sges, uint, S_IRUGO); 105MODULE_PARM_DESC(max_srq_sges, "Maximum number of SRQ SGEs to support"); 106 107unsigned int ib_qib_max_srq_wrs = 0x1FFFF; 108module_param_named(max_srq_wrs, ib_qib_max_srq_wrs, uint, S_IRUGO); 109MODULE_PARM_DESC(max_srq_wrs, "Maximum number of SRQ WRs support"); 110 111static unsigned int ib_qib_disable_sma; 112module_param_named(disable_sma, ib_qib_disable_sma, uint, S_IWUSR | S_IRUGO); 113MODULE_PARM_DESC(disable_sma, "Disable the SMA"); 114 115/* 116 * Note that it is OK to post send work requests in the SQE and ERR 117 * states; qib_do_send() will process them and generate error 118 * completions as per IB 1.2 C10-96. 119 */ 120const int ib_qib_state_ops[IB_QPS_ERR + 1] = { 121 [IB_QPS_RESET] = 0, 122 [IB_QPS_INIT] = QIB_POST_RECV_OK, 123 [IB_QPS_RTR] = QIB_POST_RECV_OK | QIB_PROCESS_RECV_OK, 124 [IB_QPS_RTS] = QIB_POST_RECV_OK | QIB_PROCESS_RECV_OK | 125 QIB_POST_SEND_OK | QIB_PROCESS_SEND_OK | 126 QIB_PROCESS_NEXT_SEND_OK, 127 [IB_QPS_SQD] = QIB_POST_RECV_OK | QIB_PROCESS_RECV_OK | 128 QIB_POST_SEND_OK | QIB_PROCESS_SEND_OK, 129 [IB_QPS_SQE] = QIB_POST_RECV_OK | QIB_PROCESS_RECV_OK | 130 QIB_POST_SEND_OK | QIB_FLUSH_SEND, 131 [IB_QPS_ERR] = QIB_POST_RECV_OK | QIB_FLUSH_RECV | 132 QIB_POST_SEND_OK | QIB_FLUSH_SEND, 133}; 134 135struct qib_ucontext { 136 struct ib_ucontext ibucontext; 137}; 138 139static inline struct qib_ucontext *to_iucontext(struct ib_ucontext 140 *ibucontext) 141{ 142 return container_of(ibucontext, struct qib_ucontext, ibucontext); 143} 144 145/* 146 * Translate ib_wr_opcode into ib_wc_opcode. 147 */ 148const enum ib_wc_opcode ib_qib_wc_opcode[] = { 149 [IB_WR_RDMA_WRITE] = IB_WC_RDMA_WRITE, 150 [IB_WR_RDMA_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE, 151 [IB_WR_SEND] = IB_WC_SEND, 152 [IB_WR_SEND_WITH_IMM] = IB_WC_SEND, 153 [IB_WR_RDMA_READ] = IB_WC_RDMA_READ, 154 [IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP, 155 [IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD 156}; 157 158/* 159 * System image GUID. 160 */ 161__be64 ib_qib_sys_image_guid; 162 163/** 164 * qib_copy_sge - copy data to SGE memory 165 * @ss: the SGE state 166 * @data: the data to copy 167 * @length: the length of the data 168 */ 169void qib_copy_sge(struct qib_sge_state *ss, void *data, u32 length, int release) 170{ 171 struct qib_sge *sge = &ss->sge; 172 173 while (length) { 174 u32 len = sge->length; 175 176 if (len > length) 177 len = length; 178 if (len > sge->sge_length) 179 len = sge->sge_length; 180 BUG_ON(len == 0); 181 memcpy(sge->vaddr, data, len); 182 sge->vaddr += len; 183 sge->length -= len; 184 sge->sge_length -= len; 185 if (sge->sge_length == 0) { 186 if (release) 187 qib_put_mr(sge->mr); 188 if (--ss->num_sge) 189 *sge = *ss->sg_list++; 190 } else if (sge->length == 0 && sge->mr->lkey) { 191 if (++sge->n >= QIB_SEGSZ) { 192 if (++sge->m >= sge->mr->mapsz) 193 break; 194 sge->n = 0; 195 } 196 sge->vaddr = 197 sge->mr->map[sge->m]->segs[sge->n].vaddr; 198 sge->length = 199 sge->mr->map[sge->m]->segs[sge->n].length; 200 } 201 data += len; 202 length -= len; 203 } 204} 205 206/** 207 * qib_skip_sge - skip over SGE memory - XXX almost dup of prev func 208 * @ss: the SGE state 209 * @length: the number of bytes to skip 210 */ 211void qib_skip_sge(struct qib_sge_state *ss, u32 length, int release) 212{ 213 struct qib_sge *sge = &ss->sge; 214 215 while (length) { 216 u32 len = sge->length; 217 218 if (len > length) 219 len = length; 220 if (len > sge->sge_length) 221 len = sge->sge_length; 222 BUG_ON(len == 0); 223 sge->vaddr += len; 224 sge->length -= len; 225 sge->sge_length -= len; 226 if (sge->sge_length == 0) { 227 if (release) 228 qib_put_mr(sge->mr); 229 if (--ss->num_sge) 230 *sge = *ss->sg_list++; 231 } else if (sge->length == 0 && sge->mr->lkey) { 232 if (++sge->n >= QIB_SEGSZ) { 233 if (++sge->m >= sge->mr->mapsz) 234 break; 235 sge->n = 0; 236 } 237 sge->vaddr = 238 sge->mr->map[sge->m]->segs[sge->n].vaddr; 239 sge->length = 240 sge->mr->map[sge->m]->segs[sge->n].length; 241 } 242 length -= len; 243 } 244} 245 246/* 247 * Count the number of DMA descriptors needed to send length bytes of data. 248 * Don't modify the qib_sge_state to get the count. 249 * Return zero if any of the segments is not aligned. 250 */ 251static u32 qib_count_sge(struct qib_sge_state *ss, u32 length) 252{ 253 struct qib_sge *sg_list = ss->sg_list; 254 struct qib_sge sge = ss->sge; 255 u8 num_sge = ss->num_sge; 256 u32 ndesc = 1; /* count the header */ 257 258 while (length) { 259 u32 len = sge.length; 260 261 if (len > length) 262 len = length; 263 if (len > sge.sge_length) 264 len = sge.sge_length; 265 BUG_ON(len == 0); 266 if (((long) sge.vaddr & (sizeof(u32) - 1)) || 267 (len != length && (len & (sizeof(u32) - 1)))) { 268 ndesc = 0; 269 break; 270 } 271 ndesc++; 272 sge.vaddr += len; 273 sge.length -= len; 274 sge.sge_length -= len; 275 if (sge.sge_length == 0) { 276 if (--num_sge) 277 sge = *sg_list++; 278 } else if (sge.length == 0 && sge.mr->lkey) { 279 if (++sge.n >= QIB_SEGSZ) { 280 if (++sge.m >= sge.mr->mapsz) 281 break; 282 sge.n = 0; 283 } 284 sge.vaddr = 285 sge.mr->map[sge.m]->segs[sge.n].vaddr; 286 sge.length = 287 sge.mr->map[sge.m]->segs[sge.n].length; 288 } 289 length -= len; 290 } 291 return ndesc; 292} 293 294/* 295 * Copy from the SGEs to the data buffer. 296 */ 297static void qib_copy_from_sge(void *data, struct qib_sge_state *ss, u32 length) 298{ 299 struct qib_sge *sge = &ss->sge; 300 301 while (length) { 302 u32 len = sge->length; 303 304 if (len > length) 305 len = length; 306 if (len > sge->sge_length) 307 len = sge->sge_length; 308 BUG_ON(len == 0); 309 memcpy(data, sge->vaddr, len); 310 sge->vaddr += len; 311 sge->length -= len; 312 sge->sge_length -= len; 313 if (sge->sge_length == 0) { 314 if (--ss->num_sge) 315 *sge = *ss->sg_list++; 316 } else if (sge->length == 0 && sge->mr->lkey) { 317 if (++sge->n >= QIB_SEGSZ) { 318 if (++sge->m >= sge->mr->mapsz) 319 break; 320 sge->n = 0; 321 } 322 sge->vaddr = 323 sge->mr->map[sge->m]->segs[sge->n].vaddr; 324 sge->length = 325 sge->mr->map[sge->m]->segs[sge->n].length; 326 } 327 data += len; 328 length -= len; 329 } 330} 331 332/** 333 * qib_post_one_send - post one RC, UC, or UD send work request 334 * @qp: the QP to post on 335 * @wr: the work request to send 336 */ 337static int qib_post_one_send(struct qib_qp *qp, struct ib_send_wr *wr, 338 int *scheduled) 339{ 340 struct qib_swqe *wqe; 341 u32 next; 342 int i; 343 int j; 344 int acc; 345 int ret; 346 unsigned long flags; 347 struct qib_lkey_table *rkt; 348 struct qib_pd *pd; 349 350 spin_lock_irqsave(&qp->s_lock, flags); 351 352 /* Check that state is OK to post send. */ 353 if (unlikely(!(ib_qib_state_ops[qp->state] & QIB_POST_SEND_OK))) 354 goto bail_inval; 355 356 /* IB spec says that num_sge == 0 is OK. */ 357 if (wr->num_sge > qp->s_max_sge) 358 goto bail_inval; 359 360 /* 361 * Don't allow RDMA reads or atomic operations on UC or 362 * undefined operations. 363 * Make sure buffer is large enough to hold the result for atomics. 364 */ 365 if (wr->opcode == IB_WR_REG_MR) { 366 if (qib_reg_mr(qp, reg_wr(wr))) 367 goto bail_inval; 368 } else if (qp->ibqp.qp_type == IB_QPT_UC) { 369 if ((unsigned) wr->opcode >= IB_WR_RDMA_READ) 370 goto bail_inval; 371 } else if (qp->ibqp.qp_type != IB_QPT_RC) { 372 /* Check IB_QPT_SMI, IB_QPT_GSI, IB_QPT_UD opcode */ 373 if (wr->opcode != IB_WR_SEND && 374 wr->opcode != IB_WR_SEND_WITH_IMM) 375 goto bail_inval; 376 /* Check UD destination address PD */ 377 if (qp->ibqp.pd != ud_wr(wr)->ah->pd) 378 goto bail_inval; 379 } else if ((unsigned) wr->opcode > IB_WR_ATOMIC_FETCH_AND_ADD) 380 goto bail_inval; 381 else if (wr->opcode >= IB_WR_ATOMIC_CMP_AND_SWP && 382 (wr->num_sge == 0 || 383 wr->sg_list[0].length < sizeof(u64) || 384 wr->sg_list[0].addr & (sizeof(u64) - 1))) 385 goto bail_inval; 386 else if (wr->opcode >= IB_WR_RDMA_READ && !qp->s_max_rd_atomic) 387 goto bail_inval; 388 389 next = qp->s_head + 1; 390 if (next >= qp->s_size) 391 next = 0; 392 if (next == qp->s_last) { 393 ret = -ENOMEM; 394 goto bail; 395 } 396 397 rkt = &to_idev(qp->ibqp.device)->lk_table; 398 pd = to_ipd(qp->ibqp.pd); 399 wqe = get_swqe_ptr(qp, qp->s_head); 400 401 if (qp->ibqp.qp_type != IB_QPT_UC && 402 qp->ibqp.qp_type != IB_QPT_RC) 403 memcpy(&wqe->ud_wr, ud_wr(wr), sizeof(wqe->ud_wr)); 404 else if (wr->opcode == IB_WR_REG_MR) 405 memcpy(&wqe->reg_wr, reg_wr(wr), 406 sizeof(wqe->reg_wr)); 407 else if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 408 wr->opcode == IB_WR_RDMA_WRITE || 409 wr->opcode == IB_WR_RDMA_READ) 410 memcpy(&wqe->rdma_wr, rdma_wr(wr), sizeof(wqe->rdma_wr)); 411 else if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || 412 wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) 413 memcpy(&wqe->atomic_wr, atomic_wr(wr), sizeof(wqe->atomic_wr)); 414 else 415 memcpy(&wqe->wr, wr, sizeof(wqe->wr)); 416 417 wqe->length = 0; 418 j = 0; 419 if (wr->num_sge) { 420 acc = wr->opcode >= IB_WR_RDMA_READ ? 421 IB_ACCESS_LOCAL_WRITE : 0; 422 for (i = 0; i < wr->num_sge; i++) { 423 u32 length = wr->sg_list[i].length; 424 int ok; 425 426 if (length == 0) 427 continue; 428 ok = qib_lkey_ok(rkt, pd, &wqe->sg_list[j], 429 &wr->sg_list[i], acc); 430 if (!ok) 431 goto bail_inval_free; 432 wqe->length += length; 433 j++; 434 } 435 wqe->wr.num_sge = j; 436 } 437 if (qp->ibqp.qp_type == IB_QPT_UC || 438 qp->ibqp.qp_type == IB_QPT_RC) { 439 if (wqe->length > 0x80000000U) 440 goto bail_inval_free; 441 } else if (wqe->length > (dd_from_ibdev(qp->ibqp.device)->pport + 442 qp->port_num - 1)->ibmtu) 443 goto bail_inval_free; 444 else 445 atomic_inc(&to_iah(ud_wr(wr)->ah)->refcount); 446 wqe->ssn = qp->s_ssn++; 447 qp->s_head = next; 448 449 ret = 0; 450 goto bail; 451 452bail_inval_free: 453 while (j) { 454 struct qib_sge *sge = &wqe->sg_list[--j]; 455 456 qib_put_mr(sge->mr); 457 } 458bail_inval: 459 ret = -EINVAL; 460bail: 461 if (!ret && !wr->next && 462 !qib_sdma_empty( 463 dd_from_ibdev(qp->ibqp.device)->pport + qp->port_num - 1)) { 464 qib_schedule_send(qp); 465 *scheduled = 1; 466 } 467 spin_unlock_irqrestore(&qp->s_lock, flags); 468 return ret; 469} 470 471/** 472 * qib_post_send - post a send on a QP 473 * @ibqp: the QP to post the send on 474 * @wr: the list of work requests to post 475 * @bad_wr: the first bad WR is put here 476 * 477 * This may be called from interrupt context. 478 */ 479static int qib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 480 struct ib_send_wr **bad_wr) 481{ 482 struct qib_qp *qp = to_iqp(ibqp); 483 int err = 0; 484 int scheduled = 0; 485 486 for (; wr; wr = wr->next) { 487 err = qib_post_one_send(qp, wr, &scheduled); 488 if (err) { 489 *bad_wr = wr; 490 goto bail; 491 } 492 } 493 494 /* Try to do the send work in the caller's context. */ 495 if (!scheduled) 496 qib_do_send(&qp->s_work); 497 498bail: 499 return err; 500} 501 502/** 503 * qib_post_receive - post a receive on a QP 504 * @ibqp: the QP to post the receive on 505 * @wr: the WR to post 506 * @bad_wr: the first bad WR is put here 507 * 508 * This may be called from interrupt context. 509 */ 510static int qib_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, 511 struct ib_recv_wr **bad_wr) 512{ 513 struct qib_qp *qp = to_iqp(ibqp); 514 struct qib_rwq *wq = qp->r_rq.wq; 515 unsigned long flags; 516 int ret; 517 518 /* Check that state is OK to post receive. */ 519 if (!(ib_qib_state_ops[qp->state] & QIB_POST_RECV_OK) || !wq) { 520 *bad_wr = wr; 521 ret = -EINVAL; 522 goto bail; 523 } 524 525 for (; wr; wr = wr->next) { 526 struct qib_rwqe *wqe; 527 u32 next; 528 int i; 529 530 if ((unsigned) wr->num_sge > qp->r_rq.max_sge) { 531 *bad_wr = wr; 532 ret = -EINVAL; 533 goto bail; 534 } 535 536 spin_lock_irqsave(&qp->r_rq.lock, flags); 537 next = wq->head + 1; 538 if (next >= qp->r_rq.size) 539 next = 0; 540 if (next == wq->tail) { 541 spin_unlock_irqrestore(&qp->r_rq.lock, flags); 542 *bad_wr = wr; 543 ret = -ENOMEM; 544 goto bail; 545 } 546 547 wqe = get_rwqe_ptr(&qp->r_rq, wq->head); 548 wqe->wr_id = wr->wr_id; 549 wqe->num_sge = wr->num_sge; 550 for (i = 0; i < wr->num_sge; i++) 551 wqe->sg_list[i] = wr->sg_list[i]; 552 /* Make sure queue entry is written before the head index. */ 553 smp_wmb(); 554 wq->head = next; 555 spin_unlock_irqrestore(&qp->r_rq.lock, flags); 556 } 557 ret = 0; 558 559bail: 560 return ret; 561} 562 563/** 564 * qib_qp_rcv - processing an incoming packet on a QP 565 * @rcd: the context pointer 566 * @hdr: the packet header 567 * @has_grh: true if the packet has a GRH 568 * @data: the packet data 569 * @tlen: the packet length 570 * @qp: the QP the packet came on 571 * 572 * This is called from qib_ib_rcv() to process an incoming packet 573 * for the given QP. 574 * Called at interrupt level. 575 */ 576static void qib_qp_rcv(struct qib_ctxtdata *rcd, struct qib_ib_header *hdr, 577 int has_grh, void *data, u32 tlen, struct qib_qp *qp) 578{ 579 struct qib_ibport *ibp = &rcd->ppd->ibport_data; 580 581 spin_lock(&qp->r_lock); 582 583 /* Check for valid receive state. */ 584 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) { 585 ibp->n_pkt_drops++; 586 goto unlock; 587 } 588 589 switch (qp->ibqp.qp_type) { 590 case IB_QPT_SMI: 591 case IB_QPT_GSI: 592 if (ib_qib_disable_sma) 593 break; 594 /* FALLTHROUGH */ 595 case IB_QPT_UD: 596 qib_ud_rcv(ibp, hdr, has_grh, data, tlen, qp); 597 break; 598 599 case IB_QPT_RC: 600 qib_rc_rcv(rcd, hdr, has_grh, data, tlen, qp); 601 break; 602 603 case IB_QPT_UC: 604 qib_uc_rcv(ibp, hdr, has_grh, data, tlen, qp); 605 break; 606 607 default: 608 break; 609 } 610 611unlock: 612 spin_unlock(&qp->r_lock); 613} 614 615/** 616 * qib_ib_rcv - process an incoming packet 617 * @rcd: the context pointer 618 * @rhdr: the header of the packet 619 * @data: the packet payload 620 * @tlen: the packet length 621 * 622 * This is called from qib_kreceive() to process an incoming packet at 623 * interrupt level. Tlen is the length of the header + data + CRC in bytes. 624 */ 625void qib_ib_rcv(struct qib_ctxtdata *rcd, void *rhdr, void *data, u32 tlen) 626{ 627 struct qib_pportdata *ppd = rcd->ppd; 628 struct qib_ibport *ibp = &ppd->ibport_data; 629 struct qib_ib_header *hdr = rhdr; 630 struct qib_other_headers *ohdr; 631 struct qib_qp *qp; 632 u32 qp_num; 633 int lnh; 634 u8 opcode; 635 u16 lid; 636 637 /* 24 == LRH+BTH+CRC */ 638 if (unlikely(tlen < 24)) 639 goto drop; 640 641 /* Check for a valid destination LID (see ch. 7.11.1). */ 642 lid = be16_to_cpu(hdr->lrh[1]); 643 if (lid < QIB_MULTICAST_LID_BASE) { 644 lid &= ~((1 << ppd->lmc) - 1); 645 if (unlikely(lid != ppd->lid)) 646 goto drop; 647 } 648 649 /* Check for GRH */ 650 lnh = be16_to_cpu(hdr->lrh[0]) & 3; 651 if (lnh == QIB_LRH_BTH) 652 ohdr = &hdr->u.oth; 653 else if (lnh == QIB_LRH_GRH) { 654 u32 vtf; 655 656 ohdr = &hdr->u.l.oth; 657 if (hdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR) 658 goto drop; 659 vtf = be32_to_cpu(hdr->u.l.grh.version_tclass_flow); 660 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION) 661 goto drop; 662 } else 663 goto drop; 664 665 opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0x7f; 666#ifdef CONFIG_DEBUG_FS 667 rcd->opstats->stats[opcode].n_bytes += tlen; 668 rcd->opstats->stats[opcode].n_packets++; 669#endif 670 671 /* Get the destination QP number. */ 672 qp_num = be32_to_cpu(ohdr->bth[1]) & QIB_QPN_MASK; 673 if (qp_num == QIB_MULTICAST_QPN) { 674 struct qib_mcast *mcast; 675 struct qib_mcast_qp *p; 676 677 if (lnh != QIB_LRH_GRH) 678 goto drop; 679 mcast = qib_mcast_find(ibp, &hdr->u.l.grh.dgid); 680 if (mcast == NULL) 681 goto drop; 682 this_cpu_inc(ibp->pmastats->n_multicast_rcv); 683 list_for_each_entry_rcu(p, &mcast->qp_list, list) 684 qib_qp_rcv(rcd, hdr, 1, data, tlen, p->qp); 685 /* 686 * Notify qib_multicast_detach() if it is waiting for us 687 * to finish. 688 */ 689 if (atomic_dec_return(&mcast->refcount) <= 1) 690 wake_up(&mcast->wait); 691 } else { 692 if (rcd->lookaside_qp) { 693 if (rcd->lookaside_qpn != qp_num) { 694 if (atomic_dec_and_test( 695 &rcd->lookaside_qp->refcount)) 696 wake_up( 697 &rcd->lookaside_qp->wait); 698 rcd->lookaside_qp = NULL; 699 } 700 } 701 if (!rcd->lookaside_qp) { 702 qp = qib_lookup_qpn(ibp, qp_num); 703 if (!qp) 704 goto drop; 705 rcd->lookaside_qp = qp; 706 rcd->lookaside_qpn = qp_num; 707 } else 708 qp = rcd->lookaside_qp; 709 this_cpu_inc(ibp->pmastats->n_unicast_rcv); 710 qib_qp_rcv(rcd, hdr, lnh == QIB_LRH_GRH, data, tlen, qp); 711 } 712 return; 713 714drop: 715 ibp->n_pkt_drops++; 716} 717 718/* 719 * This is called from a timer to check for QPs 720 * which need kernel memory in order to send a packet. 721 */ 722static void mem_timer(unsigned long data) 723{ 724 struct qib_ibdev *dev = (struct qib_ibdev *) data; 725 struct list_head *list = &dev->memwait; 726 struct qib_qp *qp = NULL; 727 unsigned long flags; 728 729 spin_lock_irqsave(&dev->pending_lock, flags); 730 if (!list_empty(list)) { 731 qp = list_entry(list->next, struct qib_qp, iowait); 732 list_del_init(&qp->iowait); 733 atomic_inc(&qp->refcount); 734 if (!list_empty(list)) 735 mod_timer(&dev->mem_timer, jiffies + 1); 736 } 737 spin_unlock_irqrestore(&dev->pending_lock, flags); 738 739 if (qp) { 740 spin_lock_irqsave(&qp->s_lock, flags); 741 if (qp->s_flags & QIB_S_WAIT_KMEM) { 742 qp->s_flags &= ~QIB_S_WAIT_KMEM; 743 qib_schedule_send(qp); 744 } 745 spin_unlock_irqrestore(&qp->s_lock, flags); 746 if (atomic_dec_and_test(&qp->refcount)) 747 wake_up(&qp->wait); 748 } 749} 750 751static void update_sge(struct qib_sge_state *ss, u32 length) 752{ 753 struct qib_sge *sge = &ss->sge; 754 755 sge->vaddr += length; 756 sge->length -= length; 757 sge->sge_length -= length; 758 if (sge->sge_length == 0) { 759 if (--ss->num_sge) 760 *sge = *ss->sg_list++; 761 } else if (sge->length == 0 && sge->mr->lkey) { 762 if (++sge->n >= QIB_SEGSZ) { 763 if (++sge->m >= sge->mr->mapsz) 764 return; 765 sge->n = 0; 766 } 767 sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr; 768 sge->length = sge->mr->map[sge->m]->segs[sge->n].length; 769 } 770} 771 772#ifdef __LITTLE_ENDIAN 773static inline u32 get_upper_bits(u32 data, u32 shift) 774{ 775 return data >> shift; 776} 777 778static inline u32 set_upper_bits(u32 data, u32 shift) 779{ 780 return data << shift; 781} 782 783static inline u32 clear_upper_bytes(u32 data, u32 n, u32 off) 784{ 785 data <<= ((sizeof(u32) - n) * BITS_PER_BYTE); 786 data >>= ((sizeof(u32) - n - off) * BITS_PER_BYTE); 787 return data; 788} 789#else 790static inline u32 get_upper_bits(u32 data, u32 shift) 791{ 792 return data << shift; 793} 794 795static inline u32 set_upper_bits(u32 data, u32 shift) 796{ 797 return data >> shift; 798} 799 800static inline u32 clear_upper_bytes(u32 data, u32 n, u32 off) 801{ 802 data >>= ((sizeof(u32) - n) * BITS_PER_BYTE); 803 data <<= ((sizeof(u32) - n - off) * BITS_PER_BYTE); 804 return data; 805} 806#endif 807 808static void copy_io(u32 __iomem *piobuf, struct qib_sge_state *ss, 809 u32 length, unsigned flush_wc) 810{ 811 u32 extra = 0; 812 u32 data = 0; 813 u32 last; 814 815 while (1) { 816 u32 len = ss->sge.length; 817 u32 off; 818 819 if (len > length) 820 len = length; 821 if (len > ss->sge.sge_length) 822 len = ss->sge.sge_length; 823 BUG_ON(len == 0); 824 /* If the source address is not aligned, try to align it. */ 825 off = (unsigned long)ss->sge.vaddr & (sizeof(u32) - 1); 826 if (off) { 827 u32 *addr = (u32 *)((unsigned long)ss->sge.vaddr & 828 ~(sizeof(u32) - 1)); 829 u32 v = get_upper_bits(*addr, off * BITS_PER_BYTE); 830 u32 y; 831 832 y = sizeof(u32) - off; 833 if (len > y) 834 len = y; 835 if (len + extra >= sizeof(u32)) { 836 data |= set_upper_bits(v, extra * 837 BITS_PER_BYTE); 838 len = sizeof(u32) - extra; 839 if (len == length) { 840 last = data; 841 break; 842 } 843 __raw_writel(data, piobuf); 844 piobuf++; 845 extra = 0; 846 data = 0; 847 } else { 848 /* Clear unused upper bytes */ 849 data |= clear_upper_bytes(v, len, extra); 850 if (len == length) { 851 last = data; 852 break; 853 } 854 extra += len; 855 } 856 } else if (extra) { 857 /* Source address is aligned. */ 858 u32 *addr = (u32 *) ss->sge.vaddr; 859 int shift = extra * BITS_PER_BYTE; 860 int ushift = 32 - shift; 861 u32 l = len; 862 863 while (l >= sizeof(u32)) { 864 u32 v = *addr; 865 866 data |= set_upper_bits(v, shift); 867 __raw_writel(data, piobuf); 868 data = get_upper_bits(v, ushift); 869 piobuf++; 870 addr++; 871 l -= sizeof(u32); 872 } 873 /* 874 * We still have 'extra' number of bytes leftover. 875 */ 876 if (l) { 877 u32 v = *addr; 878 879 if (l + extra >= sizeof(u32)) { 880 data |= set_upper_bits(v, shift); 881 len -= l + extra - sizeof(u32); 882 if (len == length) { 883 last = data; 884 break; 885 } 886 __raw_writel(data, piobuf); 887 piobuf++; 888 extra = 0; 889 data = 0; 890 } else { 891 /* Clear unused upper bytes */ 892 data |= clear_upper_bytes(v, l, extra); 893 if (len == length) { 894 last = data; 895 break; 896 } 897 extra += l; 898 } 899 } else if (len == length) { 900 last = data; 901 break; 902 } 903 } else if (len == length) { 904 u32 w; 905 906 /* 907 * Need to round up for the last dword in the 908 * packet. 909 */ 910 w = (len + 3) >> 2; 911 qib_pio_copy(piobuf, ss->sge.vaddr, w - 1); 912 piobuf += w - 1; 913 last = ((u32 *) ss->sge.vaddr)[w - 1]; 914 break; 915 } else { 916 u32 w = len >> 2; 917 918 qib_pio_copy(piobuf, ss->sge.vaddr, w); 919 piobuf += w; 920 921 extra = len & (sizeof(u32) - 1); 922 if (extra) { 923 u32 v = ((u32 *) ss->sge.vaddr)[w]; 924 925 /* Clear unused upper bytes */ 926 data = clear_upper_bytes(v, extra, 0); 927 } 928 } 929 update_sge(ss, len); 930 length -= len; 931 } 932 /* Update address before sending packet. */ 933 update_sge(ss, length); 934 if (flush_wc) { 935 /* must flush early everything before trigger word */ 936 qib_flush_wc(); 937 __raw_writel(last, piobuf); 938 /* be sure trigger word is written */ 939 qib_flush_wc(); 940 } else 941 __raw_writel(last, piobuf); 942} 943 944static noinline struct qib_verbs_txreq *__get_txreq(struct qib_ibdev *dev, 945 struct qib_qp *qp) 946{ 947 struct qib_verbs_txreq *tx; 948 unsigned long flags; 949 950 spin_lock_irqsave(&qp->s_lock, flags); 951 spin_lock(&dev->pending_lock); 952 953 if (!list_empty(&dev->txreq_free)) { 954 struct list_head *l = dev->txreq_free.next; 955 956 list_del(l); 957 spin_unlock(&dev->pending_lock); 958 spin_unlock_irqrestore(&qp->s_lock, flags); 959 tx = list_entry(l, struct qib_verbs_txreq, txreq.list); 960 } else { 961 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK && 962 list_empty(&qp->iowait)) { 963 dev->n_txwait++; 964 qp->s_flags |= QIB_S_WAIT_TX; 965 list_add_tail(&qp->iowait, &dev->txwait); 966 } 967 qp->s_flags &= ~QIB_S_BUSY; 968 spin_unlock(&dev->pending_lock); 969 spin_unlock_irqrestore(&qp->s_lock, flags); 970 tx = ERR_PTR(-EBUSY); 971 } 972 return tx; 973} 974 975static inline struct qib_verbs_txreq *get_txreq(struct qib_ibdev *dev, 976 struct qib_qp *qp) 977{ 978 struct qib_verbs_txreq *tx; 979 unsigned long flags; 980 981 spin_lock_irqsave(&dev->pending_lock, flags); 982 /* assume the list non empty */ 983 if (likely(!list_empty(&dev->txreq_free))) { 984 struct list_head *l = dev->txreq_free.next; 985 986 list_del(l); 987 spin_unlock_irqrestore(&dev->pending_lock, flags); 988 tx = list_entry(l, struct qib_verbs_txreq, txreq.list); 989 } else { 990 /* call slow path to get the extra lock */ 991 spin_unlock_irqrestore(&dev->pending_lock, flags); 992 tx = __get_txreq(dev, qp); 993 } 994 return tx; 995} 996 997void qib_put_txreq(struct qib_verbs_txreq *tx) 998{ 999 struct qib_ibdev *dev; 1000 struct qib_qp *qp; 1001 unsigned long flags; 1002 1003 qp = tx->qp; 1004 dev = to_idev(qp->ibqp.device); 1005 1006 if (atomic_dec_and_test(&qp->refcount)) 1007 wake_up(&qp->wait); 1008 if (tx->mr) { 1009 qib_put_mr(tx->mr); 1010 tx->mr = NULL; 1011 } 1012 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_FREEBUF) { 1013 tx->txreq.flags &= ~QIB_SDMA_TXREQ_F_FREEBUF; 1014 dma_unmap_single(&dd_from_dev(dev)->pcidev->dev, 1015 tx->txreq.addr, tx->hdr_dwords << 2, 1016 DMA_TO_DEVICE); 1017 kfree(tx->align_buf); 1018 } 1019 1020 spin_lock_irqsave(&dev->pending_lock, flags); 1021 1022 /* Put struct back on free list */ 1023 list_add(&tx->txreq.list, &dev->txreq_free); 1024 1025 if (!list_empty(&dev->txwait)) { 1026 /* Wake up first QP wanting a free struct */ 1027 qp = list_entry(dev->txwait.next, struct qib_qp, iowait); 1028 list_del_init(&qp->iowait); 1029 atomic_inc(&qp->refcount); 1030 spin_unlock_irqrestore(&dev->pending_lock, flags); 1031 1032 spin_lock_irqsave(&qp->s_lock, flags); 1033 if (qp->s_flags & QIB_S_WAIT_TX) { 1034 qp->s_flags &= ~QIB_S_WAIT_TX; 1035 qib_schedule_send(qp); 1036 } 1037 spin_unlock_irqrestore(&qp->s_lock, flags); 1038 1039 if (atomic_dec_and_test(&qp->refcount)) 1040 wake_up(&qp->wait); 1041 } else 1042 spin_unlock_irqrestore(&dev->pending_lock, flags); 1043} 1044 1045/* 1046 * This is called when there are send DMA descriptors that might be 1047 * available. 1048 * 1049 * This is called with ppd->sdma_lock held. 1050 */ 1051void qib_verbs_sdma_desc_avail(struct qib_pportdata *ppd, unsigned avail) 1052{ 1053 struct qib_qp *qp, *nqp; 1054 struct qib_qp *qps[20]; 1055 struct qib_ibdev *dev; 1056 unsigned i, n; 1057 1058 n = 0; 1059 dev = &ppd->dd->verbs_dev; 1060 spin_lock(&dev->pending_lock); 1061 1062 /* Search wait list for first QP wanting DMA descriptors. */ 1063 list_for_each_entry_safe(qp, nqp, &dev->dmawait, iowait) { 1064 if (qp->port_num != ppd->port) 1065 continue; 1066 if (n == ARRAY_SIZE(qps)) 1067 break; 1068 if (qp->s_tx->txreq.sg_count > avail) 1069 break; 1070 avail -= qp->s_tx->txreq.sg_count; 1071 list_del_init(&qp->iowait); 1072 atomic_inc(&qp->refcount); 1073 qps[n++] = qp; 1074 } 1075 1076 spin_unlock(&dev->pending_lock); 1077 1078 for (i = 0; i < n; i++) { 1079 qp = qps[i]; 1080 spin_lock(&qp->s_lock); 1081 if (qp->s_flags & QIB_S_WAIT_DMA_DESC) { 1082 qp->s_flags &= ~QIB_S_WAIT_DMA_DESC; 1083 qib_schedule_send(qp); 1084 } 1085 spin_unlock(&qp->s_lock); 1086 if (atomic_dec_and_test(&qp->refcount)) 1087 wake_up(&qp->wait); 1088 } 1089} 1090 1091/* 1092 * This is called with ppd->sdma_lock held. 1093 */ 1094static void sdma_complete(struct qib_sdma_txreq *cookie, int status) 1095{ 1096 struct qib_verbs_txreq *tx = 1097 container_of(cookie, struct qib_verbs_txreq, txreq); 1098 struct qib_qp *qp = tx->qp; 1099 1100 spin_lock(&qp->s_lock); 1101 if (tx->wqe) 1102 qib_send_complete(qp, tx->wqe, IB_WC_SUCCESS); 1103 else if (qp->ibqp.qp_type == IB_QPT_RC) { 1104 struct qib_ib_header *hdr; 1105 1106 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_FREEBUF) 1107 hdr = &tx->align_buf->hdr; 1108 else { 1109 struct qib_ibdev *dev = to_idev(qp->ibqp.device); 1110 1111 hdr = &dev->pio_hdrs[tx->hdr_inx].hdr; 1112 } 1113 qib_rc_send_complete(qp, hdr); 1114 } 1115 if (atomic_dec_and_test(&qp->s_dma_busy)) { 1116 if (qp->state == IB_QPS_RESET) 1117 wake_up(&qp->wait_dma); 1118 else if (qp->s_flags & QIB_S_WAIT_DMA) { 1119 qp->s_flags &= ~QIB_S_WAIT_DMA; 1120 qib_schedule_send(qp); 1121 } 1122 } 1123 spin_unlock(&qp->s_lock); 1124 1125 qib_put_txreq(tx); 1126} 1127 1128static int wait_kmem(struct qib_ibdev *dev, struct qib_qp *qp) 1129{ 1130 unsigned long flags; 1131 int ret = 0; 1132 1133 spin_lock_irqsave(&qp->s_lock, flags); 1134 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) { 1135 spin_lock(&dev->pending_lock); 1136 if (list_empty(&qp->iowait)) { 1137 if (list_empty(&dev->memwait)) 1138 mod_timer(&dev->mem_timer, jiffies + 1); 1139 qp->s_flags |= QIB_S_WAIT_KMEM; 1140 list_add_tail(&qp->iowait, &dev->memwait); 1141 } 1142 spin_unlock(&dev->pending_lock); 1143 qp->s_flags &= ~QIB_S_BUSY; 1144 ret = -EBUSY; 1145 } 1146 spin_unlock_irqrestore(&qp->s_lock, flags); 1147 1148 return ret; 1149} 1150 1151static int qib_verbs_send_dma(struct qib_qp *qp, struct qib_ib_header *hdr, 1152 u32 hdrwords, struct qib_sge_state *ss, u32 len, 1153 u32 plen, u32 dwords) 1154{ 1155 struct qib_ibdev *dev = to_idev(qp->ibqp.device); 1156 struct qib_devdata *dd = dd_from_dev(dev); 1157 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 1158 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 1159 struct qib_verbs_txreq *tx; 1160 struct qib_pio_header *phdr; 1161 u32 control; 1162 u32 ndesc; 1163 int ret; 1164 1165 tx = qp->s_tx; 1166 if (tx) { 1167 qp->s_tx = NULL; 1168 /* resend previously constructed packet */ 1169 ret = qib_sdma_verbs_send(ppd, tx->ss, tx->dwords, tx); 1170 goto bail; 1171 } 1172 1173 tx = get_txreq(dev, qp); 1174 if (IS_ERR(tx)) 1175 goto bail_tx; 1176 1177 control = dd->f_setpbc_control(ppd, plen, qp->s_srate, 1178 be16_to_cpu(hdr->lrh[0]) >> 12); 1179 tx->qp = qp; 1180 atomic_inc(&qp->refcount); 1181 tx->wqe = qp->s_wqe; 1182 tx->mr = qp->s_rdma_mr; 1183 if (qp->s_rdma_mr) 1184 qp->s_rdma_mr = NULL; 1185 tx->txreq.callback = sdma_complete; 1186 if (dd->flags & QIB_HAS_SDMA_TIMEOUT) 1187 tx->txreq.flags = QIB_SDMA_TXREQ_F_HEADTOHOST; 1188 else 1189 tx->txreq.flags = QIB_SDMA_TXREQ_F_INTREQ; 1190 if (plen + 1 > dd->piosize2kmax_dwords) 1191 tx->txreq.flags |= QIB_SDMA_TXREQ_F_USELARGEBUF; 1192 1193 if (len) { 1194 /* 1195 * Don't try to DMA if it takes more descriptors than 1196 * the queue holds. 1197 */ 1198 ndesc = qib_count_sge(ss, len); 1199 if (ndesc >= ppd->sdma_descq_cnt) 1200 ndesc = 0; 1201 } else 1202 ndesc = 1; 1203 if (ndesc) { 1204 phdr = &dev->pio_hdrs[tx->hdr_inx]; 1205 phdr->pbc[0] = cpu_to_le32(plen); 1206 phdr->pbc[1] = cpu_to_le32(control); 1207 memcpy(&phdr->hdr, hdr, hdrwords << 2); 1208 tx->txreq.flags |= QIB_SDMA_TXREQ_F_FREEDESC; 1209 tx->txreq.sg_count = ndesc; 1210 tx->txreq.addr = dev->pio_hdrs_phys + 1211 tx->hdr_inx * sizeof(struct qib_pio_header); 1212 tx->hdr_dwords = hdrwords + 2; /* add PBC length */ 1213 ret = qib_sdma_verbs_send(ppd, ss, dwords, tx); 1214 goto bail; 1215 } 1216 1217 /* Allocate a buffer and copy the header and payload to it. */ 1218 tx->hdr_dwords = plen + 1; 1219 phdr = kmalloc(tx->hdr_dwords << 2, GFP_ATOMIC); 1220 if (!phdr) 1221 goto err_tx; 1222 phdr->pbc[0] = cpu_to_le32(plen); 1223 phdr->pbc[1] = cpu_to_le32(control); 1224 memcpy(&phdr->hdr, hdr, hdrwords << 2); 1225 qib_copy_from_sge((u32 *) &phdr->hdr + hdrwords, ss, len); 1226 1227 tx->txreq.addr = dma_map_single(&dd->pcidev->dev, phdr, 1228 tx->hdr_dwords << 2, DMA_TO_DEVICE); 1229 if (dma_mapping_error(&dd->pcidev->dev, tx->txreq.addr)) 1230 goto map_err; 1231 tx->align_buf = phdr; 1232 tx->txreq.flags |= QIB_SDMA_TXREQ_F_FREEBUF; 1233 tx->txreq.sg_count = 1; 1234 ret = qib_sdma_verbs_send(ppd, NULL, 0, tx); 1235 goto unaligned; 1236 1237map_err: 1238 kfree(phdr); 1239err_tx: 1240 qib_put_txreq(tx); 1241 ret = wait_kmem(dev, qp); 1242unaligned: 1243 ibp->n_unaligned++; 1244bail: 1245 return ret; 1246bail_tx: 1247 ret = PTR_ERR(tx); 1248 goto bail; 1249} 1250 1251/* 1252 * If we are now in the error state, return zero to flush the 1253 * send work request. 1254 */ 1255static int no_bufs_available(struct qib_qp *qp) 1256{ 1257 struct qib_ibdev *dev = to_idev(qp->ibqp.device); 1258 struct qib_devdata *dd; 1259 unsigned long flags; 1260 int ret = 0; 1261 1262 /* 1263 * Note that as soon as want_buffer() is called and 1264 * possibly before it returns, qib_ib_piobufavail() 1265 * could be called. Therefore, put QP on the I/O wait list before 1266 * enabling the PIO avail interrupt. 1267 */ 1268 spin_lock_irqsave(&qp->s_lock, flags); 1269 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) { 1270 spin_lock(&dev->pending_lock); 1271 if (list_empty(&qp->iowait)) { 1272 dev->n_piowait++; 1273 qp->s_flags |= QIB_S_WAIT_PIO; 1274 list_add_tail(&qp->iowait, &dev->piowait); 1275 dd = dd_from_dev(dev); 1276 dd->f_wantpiobuf_intr(dd, 1); 1277 } 1278 spin_unlock(&dev->pending_lock); 1279 qp->s_flags &= ~QIB_S_BUSY; 1280 ret = -EBUSY; 1281 } 1282 spin_unlock_irqrestore(&qp->s_lock, flags); 1283 return ret; 1284} 1285 1286static int qib_verbs_send_pio(struct qib_qp *qp, struct qib_ib_header *ibhdr, 1287 u32 hdrwords, struct qib_sge_state *ss, u32 len, 1288 u32 plen, u32 dwords) 1289{ 1290 struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); 1291 struct qib_pportdata *ppd = dd->pport + qp->port_num - 1; 1292 u32 *hdr = (u32 *) ibhdr; 1293 u32 __iomem *piobuf_orig; 1294 u32 __iomem *piobuf; 1295 u64 pbc; 1296 unsigned long flags; 1297 unsigned flush_wc; 1298 u32 control; 1299 u32 pbufn; 1300 1301 control = dd->f_setpbc_control(ppd, plen, qp->s_srate, 1302 be16_to_cpu(ibhdr->lrh[0]) >> 12); 1303 pbc = ((u64) control << 32) | plen; 1304 piobuf = dd->f_getsendbuf(ppd, pbc, &pbufn); 1305 if (unlikely(piobuf == NULL)) 1306 return no_bufs_available(qp); 1307 1308 /* 1309 * Write the pbc. 1310 * We have to flush after the PBC for correctness on some cpus 1311 * or WC buffer can be written out of order. 1312 */ 1313 writeq(pbc, piobuf); 1314 piobuf_orig = piobuf; 1315 piobuf += 2; 1316 1317 flush_wc = dd->flags & QIB_PIO_FLUSH_WC; 1318 if (len == 0) { 1319 /* 1320 * If there is just the header portion, must flush before 1321 * writing last word of header for correctness, and after 1322 * the last header word (trigger word). 1323 */ 1324 if (flush_wc) { 1325 qib_flush_wc(); 1326 qib_pio_copy(piobuf, hdr, hdrwords - 1); 1327 qib_flush_wc(); 1328 __raw_writel(hdr[hdrwords - 1], piobuf + hdrwords - 1); 1329 qib_flush_wc(); 1330 } else 1331 qib_pio_copy(piobuf, hdr, hdrwords); 1332 goto done; 1333 } 1334 1335 if (flush_wc) 1336 qib_flush_wc(); 1337 qib_pio_copy(piobuf, hdr, hdrwords); 1338 piobuf += hdrwords; 1339 1340 /* The common case is aligned and contained in one segment. */ 1341 if (likely(ss->num_sge == 1 && len <= ss->sge.length && 1342 !((unsigned long)ss->sge.vaddr & (sizeof(u32) - 1)))) { 1343 u32 *addr = (u32 *) ss->sge.vaddr; 1344 1345 /* Update address before sending packet. */ 1346 update_sge(ss, len); 1347 if (flush_wc) { 1348 qib_pio_copy(piobuf, addr, dwords - 1); 1349 /* must flush early everything before trigger word */ 1350 qib_flush_wc(); 1351 __raw_writel(addr[dwords - 1], piobuf + dwords - 1); 1352 /* be sure trigger word is written */ 1353 qib_flush_wc(); 1354 } else 1355 qib_pio_copy(piobuf, addr, dwords); 1356 goto done; 1357 } 1358 copy_io(piobuf, ss, len, flush_wc); 1359done: 1360 if (dd->flags & QIB_USE_SPCL_TRIG) { 1361 u32 spcl_off = (pbufn >= dd->piobcnt2k) ? 2047 : 1023; 1362 1363 qib_flush_wc(); 1364 __raw_writel(0xaebecede, piobuf_orig + spcl_off); 1365 } 1366 qib_sendbuf_done(dd, pbufn); 1367 if (qp->s_rdma_mr) { 1368 qib_put_mr(qp->s_rdma_mr); 1369 qp->s_rdma_mr = NULL; 1370 } 1371 if (qp->s_wqe) { 1372 spin_lock_irqsave(&qp->s_lock, flags); 1373 qib_send_complete(qp, qp->s_wqe, IB_WC_SUCCESS); 1374 spin_unlock_irqrestore(&qp->s_lock, flags); 1375 } else if (qp->ibqp.qp_type == IB_QPT_RC) { 1376 spin_lock_irqsave(&qp->s_lock, flags); 1377 qib_rc_send_complete(qp, ibhdr); 1378 spin_unlock_irqrestore(&qp->s_lock, flags); 1379 } 1380 return 0; 1381} 1382 1383/** 1384 * qib_verbs_send - send a packet 1385 * @qp: the QP to send on 1386 * @hdr: the packet header 1387 * @hdrwords: the number of 32-bit words in the header 1388 * @ss: the SGE to send 1389 * @len: the length of the packet in bytes 1390 * 1391 * Return zero if packet is sent or queued OK. 1392 * Return non-zero and clear qp->s_flags QIB_S_BUSY otherwise. 1393 */ 1394int qib_verbs_send(struct qib_qp *qp, struct qib_ib_header *hdr, 1395 u32 hdrwords, struct qib_sge_state *ss, u32 len) 1396{ 1397 struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); 1398 u32 plen; 1399 int ret; 1400 u32 dwords = (len + 3) >> 2; 1401 1402 /* 1403 * Calculate the send buffer trigger address. 1404 * The +1 counts for the pbc control dword following the pbc length. 1405 */ 1406 plen = hdrwords + dwords + 1; 1407 1408 /* 1409 * VL15 packets (IB_QPT_SMI) will always use PIO, so we 1410 * can defer SDMA restart until link goes ACTIVE without 1411 * worrying about just how we got there. 1412 */ 1413 if (qp->ibqp.qp_type == IB_QPT_SMI || 1414 !(dd->flags & QIB_HAS_SEND_DMA)) 1415 ret = qib_verbs_send_pio(qp, hdr, hdrwords, ss, len, 1416 plen, dwords); 1417 else 1418 ret = qib_verbs_send_dma(qp, hdr, hdrwords, ss, len, 1419 plen, dwords); 1420 1421 return ret; 1422} 1423 1424int qib_snapshot_counters(struct qib_pportdata *ppd, u64 *swords, 1425 u64 *rwords, u64 *spkts, u64 *rpkts, 1426 u64 *xmit_wait) 1427{ 1428 int ret; 1429 struct qib_devdata *dd = ppd->dd; 1430 1431 if (!(dd->flags & QIB_PRESENT)) { 1432 /* no hardware, freeze, etc. */ 1433 ret = -EINVAL; 1434 goto bail; 1435 } 1436 *swords = dd->f_portcntr(ppd, QIBPORTCNTR_WORDSEND); 1437 *rwords = dd->f_portcntr(ppd, QIBPORTCNTR_WORDRCV); 1438 *spkts = dd->f_portcntr(ppd, QIBPORTCNTR_PKTSEND); 1439 *rpkts = dd->f_portcntr(ppd, QIBPORTCNTR_PKTRCV); 1440 *xmit_wait = dd->f_portcntr(ppd, QIBPORTCNTR_SENDSTALL); 1441 1442 ret = 0; 1443 1444bail: 1445 return ret; 1446} 1447 1448/** 1449 * qib_get_counters - get various chip counters 1450 * @dd: the qlogic_ib device 1451 * @cntrs: counters are placed here 1452 * 1453 * Return the counters needed by recv_pma_get_portcounters(). 1454 */ 1455int qib_get_counters(struct qib_pportdata *ppd, 1456 struct qib_verbs_counters *cntrs) 1457{ 1458 int ret; 1459 1460 if (!(ppd->dd->flags & QIB_PRESENT)) { 1461 /* no hardware, freeze, etc. */ 1462 ret = -EINVAL; 1463 goto bail; 1464 } 1465 cntrs->symbol_error_counter = 1466 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_IBSYMBOLERR); 1467 cntrs->link_error_recovery_counter = 1468 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_IBLINKERRRECOV); 1469 /* 1470 * The link downed counter counts when the other side downs the 1471 * connection. We add in the number of times we downed the link 1472 * due to local link integrity errors to compensate. 1473 */ 1474 cntrs->link_downed_counter = 1475 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_IBLINKDOWN); 1476 cntrs->port_rcv_errors = 1477 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RXDROPPKT) + 1478 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RCVOVFL) + 1479 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERR_RLEN) + 1480 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_INVALIDRLEN) + 1481 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRLINK) + 1482 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRICRC) + 1483 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRVCRC) + 1484 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRLPCRC) + 1485 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_BADFORMAT); 1486 cntrs->port_rcv_errors += 1487 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RXLOCALPHYERR); 1488 cntrs->port_rcv_errors += 1489 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RXVLERR); 1490 cntrs->port_rcv_remphys_errors = 1491 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RCVEBP); 1492 cntrs->port_xmit_discards = 1493 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_UNSUPVL); 1494 cntrs->port_xmit_data = ppd->dd->f_portcntr(ppd, 1495 QIBPORTCNTR_WORDSEND); 1496 cntrs->port_rcv_data = ppd->dd->f_portcntr(ppd, 1497 QIBPORTCNTR_WORDRCV); 1498 cntrs->port_xmit_packets = ppd->dd->f_portcntr(ppd, 1499 QIBPORTCNTR_PKTSEND); 1500 cntrs->port_rcv_packets = ppd->dd->f_portcntr(ppd, 1501 QIBPORTCNTR_PKTRCV); 1502 cntrs->local_link_integrity_errors = 1503 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_LLI); 1504 cntrs->excessive_buffer_overrun_errors = 1505 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_EXCESSBUFOVFL); 1506 cntrs->vl15_dropped = 1507 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_VL15PKTDROP); 1508 1509 ret = 0; 1510 1511bail: 1512 return ret; 1513} 1514 1515/** 1516 * qib_ib_piobufavail - callback when a PIO buffer is available 1517 * @dd: the device pointer 1518 * 1519 * This is called from qib_intr() at interrupt level when a PIO buffer is 1520 * available after qib_verbs_send() returned an error that no buffers were 1521 * available. Disable the interrupt if there are no more QPs waiting. 1522 */ 1523void qib_ib_piobufavail(struct qib_devdata *dd) 1524{ 1525 struct qib_ibdev *dev = &dd->verbs_dev; 1526 struct list_head *list; 1527 struct qib_qp *qps[5]; 1528 struct qib_qp *qp; 1529 unsigned long flags; 1530 unsigned i, n; 1531 1532 list = &dev->piowait; 1533 n = 0; 1534 1535 /* 1536 * Note: checking that the piowait list is empty and clearing 1537 * the buffer available interrupt needs to be atomic or we 1538 * could end up with QPs on the wait list with the interrupt 1539 * disabled. 1540 */ 1541 spin_lock_irqsave(&dev->pending_lock, flags); 1542 while (!list_empty(list)) { 1543 if (n == ARRAY_SIZE(qps)) 1544 goto full; 1545 qp = list_entry(list->next, struct qib_qp, iowait); 1546 list_del_init(&qp->iowait); 1547 atomic_inc(&qp->refcount); 1548 qps[n++] = qp; 1549 } 1550 dd->f_wantpiobuf_intr(dd, 0); 1551full: 1552 spin_unlock_irqrestore(&dev->pending_lock, flags); 1553 1554 for (i = 0; i < n; i++) { 1555 qp = qps[i]; 1556 1557 spin_lock_irqsave(&qp->s_lock, flags); 1558 if (qp->s_flags & QIB_S_WAIT_PIO) { 1559 qp->s_flags &= ~QIB_S_WAIT_PIO; 1560 qib_schedule_send(qp); 1561 } 1562 spin_unlock_irqrestore(&qp->s_lock, flags); 1563 1564 /* Notify qib_destroy_qp() if it is waiting. */ 1565 if (atomic_dec_and_test(&qp->refcount)) 1566 wake_up(&qp->wait); 1567 } 1568} 1569 1570static int qib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, 1571 struct ib_udata *uhw) 1572{ 1573 struct qib_devdata *dd = dd_from_ibdev(ibdev); 1574 struct qib_ibdev *dev = to_idev(ibdev); 1575 1576 if (uhw->inlen || uhw->outlen) 1577 return -EINVAL; 1578 memset(props, 0, sizeof(*props)); 1579 1580 props->device_cap_flags = IB_DEVICE_BAD_PKEY_CNTR | 1581 IB_DEVICE_BAD_QKEY_CNTR | IB_DEVICE_SHUTDOWN_PORT | 1582 IB_DEVICE_SYS_IMAGE_GUID | IB_DEVICE_RC_RNR_NAK_GEN | 1583 IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_SRQ_RESIZE; 1584 props->page_size_cap = PAGE_SIZE; 1585 props->vendor_id = 1586 QIB_SRC_OUI_1 << 16 | QIB_SRC_OUI_2 << 8 | QIB_SRC_OUI_3; 1587 props->vendor_part_id = dd->deviceid; 1588 props->hw_ver = dd->minrev; 1589 props->sys_image_guid = ib_qib_sys_image_guid; 1590 props->max_mr_size = ~0ULL; 1591 props->max_qp = ib_qib_max_qps; 1592 props->max_qp_wr = ib_qib_max_qp_wrs; 1593 props->max_sge = ib_qib_max_sges; 1594 props->max_sge_rd = ib_qib_max_sges; 1595 props->max_cq = ib_qib_max_cqs; 1596 props->max_ah = ib_qib_max_ahs; 1597 props->max_cqe = ib_qib_max_cqes; 1598 props->max_mr = dev->lk_table.max; 1599 props->max_fmr = dev->lk_table.max; 1600 props->max_map_per_fmr = 32767; 1601 props->max_pd = ib_qib_max_pds; 1602 props->max_qp_rd_atom = QIB_MAX_RDMA_ATOMIC; 1603 props->max_qp_init_rd_atom = 255; 1604 /* props->max_res_rd_atom */ 1605 props->max_srq = ib_qib_max_srqs; 1606 props->max_srq_wr = ib_qib_max_srq_wrs; 1607 props->max_srq_sge = ib_qib_max_srq_sges; 1608 /* props->local_ca_ack_delay */ 1609 props->atomic_cap = IB_ATOMIC_GLOB; 1610 props->max_pkeys = qib_get_npkeys(dd); 1611 props->max_mcast_grp = ib_qib_max_mcast_grps; 1612 props->max_mcast_qp_attach = ib_qib_max_mcast_qp_attached; 1613 props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * 1614 props->max_mcast_grp; 1615 1616 return 0; 1617} 1618 1619static int qib_query_port(struct ib_device *ibdev, u8 port, 1620 struct ib_port_attr *props) 1621{ 1622 struct qib_devdata *dd = dd_from_ibdev(ibdev); 1623 struct qib_ibport *ibp = to_iport(ibdev, port); 1624 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 1625 enum ib_mtu mtu; 1626 u16 lid = ppd->lid; 1627 1628 memset(props, 0, sizeof(*props)); 1629 props->lid = lid ? lid : be16_to_cpu(IB_LID_PERMISSIVE); 1630 props->lmc = ppd->lmc; 1631 props->sm_lid = ibp->sm_lid; 1632 props->sm_sl = ibp->sm_sl; 1633 props->state = dd->f_iblink_state(ppd->lastibcstat); 1634 props->phys_state = dd->f_ibphys_portstate(ppd->lastibcstat); 1635 props->port_cap_flags = ibp->port_cap_flags; 1636 props->gid_tbl_len = QIB_GUIDS_PER_PORT; 1637 props->max_msg_sz = 0x80000000; 1638 props->pkey_tbl_len = qib_get_npkeys(dd); 1639 props->bad_pkey_cntr = ibp->pkey_violations; 1640 props->qkey_viol_cntr = ibp->qkey_violations; 1641 props->active_width = ppd->link_width_active; 1642 /* See rate_show() */ 1643 props->active_speed = ppd->link_speed_active; 1644 props->max_vl_num = qib_num_vls(ppd->vls_supported); 1645 props->init_type_reply = 0; 1646 1647 props->max_mtu = qib_ibmtu ? qib_ibmtu : IB_MTU_4096; 1648 switch (ppd->ibmtu) { 1649 case 4096: 1650 mtu = IB_MTU_4096; 1651 break; 1652 case 2048: 1653 mtu = IB_MTU_2048; 1654 break; 1655 case 1024: 1656 mtu = IB_MTU_1024; 1657 break; 1658 case 512: 1659 mtu = IB_MTU_512; 1660 break; 1661 case 256: 1662 mtu = IB_MTU_256; 1663 break; 1664 default: 1665 mtu = IB_MTU_2048; 1666 } 1667 props->active_mtu = mtu; 1668 props->subnet_timeout = ibp->subnet_timeout; 1669 1670 return 0; 1671} 1672 1673static int qib_modify_device(struct ib_device *device, 1674 int device_modify_mask, 1675 struct ib_device_modify *device_modify) 1676{ 1677 struct qib_devdata *dd = dd_from_ibdev(device); 1678 unsigned i; 1679 int ret; 1680 1681 if (device_modify_mask & ~(IB_DEVICE_MODIFY_SYS_IMAGE_GUID | 1682 IB_DEVICE_MODIFY_NODE_DESC)) { 1683 ret = -EOPNOTSUPP; 1684 goto bail; 1685 } 1686 1687 if (device_modify_mask & IB_DEVICE_MODIFY_NODE_DESC) { 1688 memcpy(device->node_desc, device_modify->node_desc, 64); 1689 for (i = 0; i < dd->num_pports; i++) { 1690 struct qib_ibport *ibp = &dd->pport[i].ibport_data; 1691 1692 qib_node_desc_chg(ibp); 1693 } 1694 } 1695 1696 if (device_modify_mask & IB_DEVICE_MODIFY_SYS_IMAGE_GUID) { 1697 ib_qib_sys_image_guid = 1698 cpu_to_be64(device_modify->sys_image_guid); 1699 for (i = 0; i < dd->num_pports; i++) { 1700 struct qib_ibport *ibp = &dd->pport[i].ibport_data; 1701 1702 qib_sys_guid_chg(ibp); 1703 } 1704 } 1705 1706 ret = 0; 1707 1708bail: 1709 return ret; 1710} 1711 1712static int qib_modify_port(struct ib_device *ibdev, u8 port, 1713 int port_modify_mask, struct ib_port_modify *props) 1714{ 1715 struct qib_ibport *ibp = to_iport(ibdev, port); 1716 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 1717 1718 ibp->port_cap_flags |= props->set_port_cap_mask; 1719 ibp->port_cap_flags &= ~props->clr_port_cap_mask; 1720 if (props->set_port_cap_mask || props->clr_port_cap_mask) 1721 qib_cap_mask_chg(ibp); 1722 if (port_modify_mask & IB_PORT_SHUTDOWN) 1723 qib_set_linkstate(ppd, QIB_IB_LINKDOWN); 1724 if (port_modify_mask & IB_PORT_RESET_QKEY_CNTR) 1725 ibp->qkey_violations = 0; 1726 return 0; 1727} 1728 1729static int qib_query_gid(struct ib_device *ibdev, u8 port, 1730 int index, union ib_gid *gid) 1731{ 1732 struct qib_devdata *dd = dd_from_ibdev(ibdev); 1733 int ret = 0; 1734 1735 if (!port || port > dd->num_pports) 1736 ret = -EINVAL; 1737 else { 1738 struct qib_ibport *ibp = to_iport(ibdev, port); 1739 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 1740 1741 gid->global.subnet_prefix = ibp->gid_prefix; 1742 if (index == 0) 1743 gid->global.interface_id = ppd->guid; 1744 else if (index < QIB_GUIDS_PER_PORT) 1745 gid->global.interface_id = ibp->guids[index - 1]; 1746 else 1747 ret = -EINVAL; 1748 } 1749 1750 return ret; 1751} 1752 1753static struct ib_pd *qib_alloc_pd(struct ib_device *ibdev, 1754 struct ib_ucontext *context, 1755 struct ib_udata *udata) 1756{ 1757 struct qib_ibdev *dev = to_idev(ibdev); 1758 struct qib_pd *pd; 1759 struct ib_pd *ret; 1760 1761 /* 1762 * This is actually totally arbitrary. Some correctness tests 1763 * assume there's a maximum number of PDs that can be allocated. 1764 * We don't actually have this limit, but we fail the test if 1765 * we allow allocations of more than we report for this value. 1766 */ 1767 1768 pd = kmalloc(sizeof(*pd), GFP_KERNEL); 1769 if (!pd) { 1770 ret = ERR_PTR(-ENOMEM); 1771 goto bail; 1772 } 1773 1774 spin_lock(&dev->n_pds_lock); 1775 if (dev->n_pds_allocated == ib_qib_max_pds) { 1776 spin_unlock(&dev->n_pds_lock); 1777 kfree(pd); 1778 ret = ERR_PTR(-ENOMEM); 1779 goto bail; 1780 } 1781 1782 dev->n_pds_allocated++; 1783 spin_unlock(&dev->n_pds_lock); 1784 1785 /* ib_alloc_pd() will initialize pd->ibpd. */ 1786 pd->user = udata != NULL; 1787 1788 ret = &pd->ibpd; 1789 1790bail: 1791 return ret; 1792} 1793 1794static int qib_dealloc_pd(struct ib_pd *ibpd) 1795{ 1796 struct qib_pd *pd = to_ipd(ibpd); 1797 struct qib_ibdev *dev = to_idev(ibpd->device); 1798 1799 spin_lock(&dev->n_pds_lock); 1800 dev->n_pds_allocated--; 1801 spin_unlock(&dev->n_pds_lock); 1802 1803 kfree(pd); 1804 1805 return 0; 1806} 1807 1808int qib_check_ah(struct ib_device *ibdev, struct ib_ah_attr *ah_attr) 1809{ 1810 /* A multicast address requires a GRH (see ch. 8.4.1). */ 1811 if (ah_attr->dlid >= QIB_MULTICAST_LID_BASE && 1812 ah_attr->dlid != QIB_PERMISSIVE_LID && 1813 !(ah_attr->ah_flags & IB_AH_GRH)) 1814 goto bail; 1815 if ((ah_attr->ah_flags & IB_AH_GRH) && 1816 ah_attr->grh.sgid_index >= QIB_GUIDS_PER_PORT) 1817 goto bail; 1818 if (ah_attr->dlid == 0) 1819 goto bail; 1820 if (ah_attr->port_num < 1 || 1821 ah_attr->port_num > ibdev->phys_port_cnt) 1822 goto bail; 1823 if (ah_attr->static_rate != IB_RATE_PORT_CURRENT && 1824 ib_rate_to_mult(ah_attr->static_rate) < 0) 1825 goto bail; 1826 if (ah_attr->sl > 15) 1827 goto bail; 1828 return 0; 1829bail: 1830 return -EINVAL; 1831} 1832 1833/** 1834 * qib_create_ah - create an address handle 1835 * @pd: the protection domain 1836 * @ah_attr: the attributes of the AH 1837 * 1838 * This may be called from interrupt context. 1839 */ 1840static struct ib_ah *qib_create_ah(struct ib_pd *pd, 1841 struct ib_ah_attr *ah_attr) 1842{ 1843 struct qib_ah *ah; 1844 struct ib_ah *ret; 1845 struct qib_ibdev *dev = to_idev(pd->device); 1846 unsigned long flags; 1847 1848 if (qib_check_ah(pd->device, ah_attr)) { 1849 ret = ERR_PTR(-EINVAL); 1850 goto bail; 1851 } 1852 1853 ah = kmalloc(sizeof(*ah), GFP_ATOMIC); 1854 if (!ah) { 1855 ret = ERR_PTR(-ENOMEM); 1856 goto bail; 1857 } 1858 1859 spin_lock_irqsave(&dev->n_ahs_lock, flags); 1860 if (dev->n_ahs_allocated == ib_qib_max_ahs) { 1861 spin_unlock_irqrestore(&dev->n_ahs_lock, flags); 1862 kfree(ah); 1863 ret = ERR_PTR(-ENOMEM); 1864 goto bail; 1865 } 1866 1867 dev->n_ahs_allocated++; 1868 spin_unlock_irqrestore(&dev->n_ahs_lock, flags); 1869 1870 /* ib_create_ah() will initialize ah->ibah. */ 1871 ah->attr = *ah_attr; 1872 atomic_set(&ah->refcount, 0); 1873 1874 ret = &ah->ibah; 1875 1876bail: 1877 return ret; 1878} 1879 1880struct ib_ah *qib_create_qp0_ah(struct qib_ibport *ibp, u16 dlid) 1881{ 1882 struct ib_ah_attr attr; 1883 struct ib_ah *ah = ERR_PTR(-EINVAL); 1884 struct qib_qp *qp0; 1885 1886 memset(&attr, 0, sizeof(attr)); 1887 attr.dlid = dlid; 1888 attr.port_num = ppd_from_ibp(ibp)->port; 1889 rcu_read_lock(); 1890 qp0 = rcu_dereference(ibp->qp0); 1891 if (qp0) 1892 ah = ib_create_ah(qp0->ibqp.pd, &attr); 1893 rcu_read_unlock(); 1894 return ah; 1895} 1896 1897/** 1898 * qib_destroy_ah - destroy an address handle 1899 * @ibah: the AH to destroy 1900 * 1901 * This may be called from interrupt context. 1902 */ 1903static int qib_destroy_ah(struct ib_ah *ibah) 1904{ 1905 struct qib_ibdev *dev = to_idev(ibah->device); 1906 struct qib_ah *ah = to_iah(ibah); 1907 unsigned long flags; 1908 1909 if (atomic_read(&ah->refcount) != 0) 1910 return -EBUSY; 1911 1912 spin_lock_irqsave(&dev->n_ahs_lock, flags); 1913 dev->n_ahs_allocated--; 1914 spin_unlock_irqrestore(&dev->n_ahs_lock, flags); 1915 1916 kfree(ah); 1917 1918 return 0; 1919} 1920 1921static int qib_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr) 1922{ 1923 struct qib_ah *ah = to_iah(ibah); 1924 1925 if (qib_check_ah(ibah->device, ah_attr)) 1926 return -EINVAL; 1927 1928 ah->attr = *ah_attr; 1929 1930 return 0; 1931} 1932 1933static int qib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr) 1934{ 1935 struct qib_ah *ah = to_iah(ibah); 1936 1937 *ah_attr = ah->attr; 1938 1939 return 0; 1940} 1941 1942/** 1943 * qib_get_npkeys - return the size of the PKEY table for context 0 1944 * @dd: the qlogic_ib device 1945 */ 1946unsigned qib_get_npkeys(struct qib_devdata *dd) 1947{ 1948 return ARRAY_SIZE(dd->rcd[0]->pkeys); 1949} 1950 1951/* 1952 * Return the indexed PKEY from the port PKEY table. 1953 * No need to validate rcd[ctxt]; the port is setup if we are here. 1954 */ 1955unsigned qib_get_pkey(struct qib_ibport *ibp, unsigned index) 1956{ 1957 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 1958 struct qib_devdata *dd = ppd->dd; 1959 unsigned ctxt = ppd->hw_pidx; 1960 unsigned ret; 1961 1962 /* dd->rcd null if mini_init or some init failures */ 1963 if (!dd->rcd || index >= ARRAY_SIZE(dd->rcd[ctxt]->pkeys)) 1964 ret = 0; 1965 else 1966 ret = dd->rcd[ctxt]->pkeys[index]; 1967 1968 return ret; 1969} 1970 1971static int qib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, 1972 u16 *pkey) 1973{ 1974 struct qib_devdata *dd = dd_from_ibdev(ibdev); 1975 int ret; 1976 1977 if (index >= qib_get_npkeys(dd)) { 1978 ret = -EINVAL; 1979 goto bail; 1980 } 1981 1982 *pkey = qib_get_pkey(to_iport(ibdev, port), index); 1983 ret = 0; 1984 1985bail: 1986 return ret; 1987} 1988 1989/** 1990 * qib_alloc_ucontext - allocate a ucontest 1991 * @ibdev: the infiniband device 1992 * @udata: not used by the QLogic_IB driver 1993 */ 1994 1995static struct ib_ucontext *qib_alloc_ucontext(struct ib_device *ibdev, 1996 struct ib_udata *udata) 1997{ 1998 struct qib_ucontext *context; 1999 struct ib_ucontext *ret; 2000 2001 context = kmalloc(sizeof(*context), GFP_KERNEL); 2002 if (!context) { 2003 ret = ERR_PTR(-ENOMEM); 2004 goto bail; 2005 } 2006 2007 ret = &context->ibucontext; 2008 2009bail: 2010 return ret; 2011} 2012 2013static int qib_dealloc_ucontext(struct ib_ucontext *context) 2014{ 2015 kfree(to_iucontext(context)); 2016 return 0; 2017} 2018 2019static void init_ibport(struct qib_pportdata *ppd) 2020{ 2021 struct qib_verbs_counters cntrs; 2022 struct qib_ibport *ibp = &ppd->ibport_data; 2023 2024 spin_lock_init(&ibp->lock); 2025 /* Set the prefix to the default value (see ch. 4.1.1) */ 2026 ibp->gid_prefix = IB_DEFAULT_GID_PREFIX; 2027 ibp->sm_lid = be16_to_cpu(IB_LID_PERMISSIVE); 2028 ibp->port_cap_flags = IB_PORT_SYS_IMAGE_GUID_SUP | 2029 IB_PORT_CLIENT_REG_SUP | IB_PORT_SL_MAP_SUP | 2030 IB_PORT_TRAP_SUP | IB_PORT_AUTO_MIGR_SUP | 2031 IB_PORT_DR_NOTICE_SUP | IB_PORT_CAP_MASK_NOTICE_SUP | 2032 IB_PORT_OTHER_LOCAL_CHANGES_SUP; 2033 if (ppd->dd->flags & QIB_HAS_LINK_LATENCY) 2034 ibp->port_cap_flags |= IB_PORT_LINK_LATENCY_SUP; 2035 ibp->pma_counter_select[0] = IB_PMA_PORT_XMIT_DATA; 2036 ibp->pma_counter_select[1] = IB_PMA_PORT_RCV_DATA; 2037 ibp->pma_counter_select[2] = IB_PMA_PORT_XMIT_PKTS; 2038 ibp->pma_counter_select[3] = IB_PMA_PORT_RCV_PKTS; 2039 ibp->pma_counter_select[4] = IB_PMA_PORT_XMIT_WAIT; 2040 2041 /* Snapshot current HW counters to "clear" them. */ 2042 qib_get_counters(ppd, &cntrs); 2043 ibp->z_symbol_error_counter = cntrs.symbol_error_counter; 2044 ibp->z_link_error_recovery_counter = 2045 cntrs.link_error_recovery_counter; 2046 ibp->z_link_downed_counter = cntrs.link_downed_counter; 2047 ibp->z_port_rcv_errors = cntrs.port_rcv_errors; 2048 ibp->z_port_rcv_remphys_errors = cntrs.port_rcv_remphys_errors; 2049 ibp->z_port_xmit_discards = cntrs.port_xmit_discards; 2050 ibp->z_port_xmit_data = cntrs.port_xmit_data; 2051 ibp->z_port_rcv_data = cntrs.port_rcv_data; 2052 ibp->z_port_xmit_packets = cntrs.port_xmit_packets; 2053 ibp->z_port_rcv_packets = cntrs.port_rcv_packets; 2054 ibp->z_local_link_integrity_errors = 2055 cntrs.local_link_integrity_errors; 2056 ibp->z_excessive_buffer_overrun_errors = 2057 cntrs.excessive_buffer_overrun_errors; 2058 ibp->z_vl15_dropped = cntrs.vl15_dropped; 2059 RCU_INIT_POINTER(ibp->qp0, NULL); 2060 RCU_INIT_POINTER(ibp->qp1, NULL); 2061} 2062 2063static int qib_port_immutable(struct ib_device *ibdev, u8 port_num, 2064 struct ib_port_immutable *immutable) 2065{ 2066 struct ib_port_attr attr; 2067 int err; 2068 2069 err = qib_query_port(ibdev, port_num, &attr); 2070 if (err) 2071 return err; 2072 2073 immutable->pkey_tbl_len = attr.pkey_tbl_len; 2074 immutable->gid_tbl_len = attr.gid_tbl_len; 2075 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB; 2076 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 2077 2078 return 0; 2079} 2080 2081/** 2082 * qib_register_ib_device - register our device with the infiniband core 2083 * @dd: the device data structure 2084 * Return the allocated qib_ibdev pointer or NULL on error. 2085 */ 2086int qib_register_ib_device(struct qib_devdata *dd) 2087{ 2088 struct qib_ibdev *dev = &dd->verbs_dev; 2089 struct ib_device *ibdev = &dev->ibdev; 2090 struct qib_pportdata *ppd = dd->pport; 2091 unsigned i, lk_tab_size; 2092 int ret; 2093 2094 dev->qp_table_size = ib_qib_qp_table_size; 2095 get_random_bytes(&dev->qp_rnd, sizeof(dev->qp_rnd)); 2096 dev->qp_table = kmalloc_array( 2097 dev->qp_table_size, 2098 sizeof(*dev->qp_table), 2099 GFP_KERNEL); 2100 if (!dev->qp_table) { 2101 ret = -ENOMEM; 2102 goto err_qpt; 2103 } 2104 for (i = 0; i < dev->qp_table_size; i++) 2105 RCU_INIT_POINTER(dev->qp_table[i], NULL); 2106 2107 for (i = 0; i < dd->num_pports; i++) 2108 init_ibport(ppd + i); 2109 2110 /* Only need to initialize non-zero fields. */ 2111 spin_lock_init(&dev->qpt_lock); 2112 spin_lock_init(&dev->n_pds_lock); 2113 spin_lock_init(&dev->n_ahs_lock); 2114 spin_lock_init(&dev->n_cqs_lock); 2115 spin_lock_init(&dev->n_qps_lock); 2116 spin_lock_init(&dev->n_srqs_lock); 2117 spin_lock_init(&dev->n_mcast_grps_lock); 2118 init_timer(&dev->mem_timer); 2119 dev->mem_timer.function = mem_timer; 2120 dev->mem_timer.data = (unsigned long) dev; 2121 2122 qib_init_qpn_table(dd, &dev->qpn_table); 2123 2124 /* 2125 * The top ib_qib_lkey_table_size bits are used to index the 2126 * table. The lower 8 bits can be owned by the user (copied from 2127 * the LKEY). The remaining bits act as a generation number or tag. 2128 */ 2129 spin_lock_init(&dev->lk_table.lock); 2130 /* insure generation is at least 4 bits see keys.c */ 2131 if (ib_qib_lkey_table_size > MAX_LKEY_TABLE_BITS) { 2132 qib_dev_warn(dd, "lkey bits %u too large, reduced to %u\n", 2133 ib_qib_lkey_table_size, MAX_LKEY_TABLE_BITS); 2134 ib_qib_lkey_table_size = MAX_LKEY_TABLE_BITS; 2135 } 2136 dev->lk_table.max = 1 << ib_qib_lkey_table_size; 2137 lk_tab_size = dev->lk_table.max * sizeof(*dev->lk_table.table); 2138 dev->lk_table.table = (struct qib_mregion __rcu **) 2139 vmalloc(lk_tab_size); 2140 if (dev->lk_table.table == NULL) { 2141 ret = -ENOMEM; 2142 goto err_lk; 2143 } 2144 RCU_INIT_POINTER(dev->dma_mr, NULL); 2145 for (i = 0; i < dev->lk_table.max; i++) 2146 RCU_INIT_POINTER(dev->lk_table.table[i], NULL); 2147 INIT_LIST_HEAD(&dev->pending_mmaps); 2148 spin_lock_init(&dev->pending_lock); 2149 dev->mmap_offset = PAGE_SIZE; 2150 spin_lock_init(&dev->mmap_offset_lock); 2151 INIT_LIST_HEAD(&dev->piowait); 2152 INIT_LIST_HEAD(&dev->dmawait); 2153 INIT_LIST_HEAD(&dev->txwait); 2154 INIT_LIST_HEAD(&dev->memwait); 2155 INIT_LIST_HEAD(&dev->txreq_free); 2156 2157 if (ppd->sdma_descq_cnt) { 2158 dev->pio_hdrs = dma_alloc_coherent(&dd->pcidev->dev, 2159 ppd->sdma_descq_cnt * 2160 sizeof(struct qib_pio_header), 2161 &dev->pio_hdrs_phys, 2162 GFP_KERNEL); 2163 if (!dev->pio_hdrs) { 2164 ret = -ENOMEM; 2165 goto err_hdrs; 2166 } 2167 } 2168 2169 for (i = 0; i < ppd->sdma_descq_cnt; i++) { 2170 struct qib_verbs_txreq *tx; 2171 2172 tx = kzalloc(sizeof(*tx), GFP_KERNEL); 2173 if (!tx) { 2174 ret = -ENOMEM; 2175 goto err_tx; 2176 } 2177 tx->hdr_inx = i; 2178 list_add(&tx->txreq.list, &dev->txreq_free); 2179 } 2180 2181 /* 2182 * The system image GUID is supposed to be the same for all 2183 * IB HCAs in a single system but since there can be other 2184 * device types in the system, we can't be sure this is unique. 2185 */ 2186 if (!ib_qib_sys_image_guid) 2187 ib_qib_sys_image_guid = ppd->guid; 2188 2189 strlcpy(ibdev->name, "qib%d", IB_DEVICE_NAME_MAX); 2190 ibdev->owner = THIS_MODULE; 2191 ibdev->node_guid = ppd->guid; 2192 ibdev->uverbs_abi_ver = QIB_UVERBS_ABI_VERSION; 2193 ibdev->uverbs_cmd_mask = 2194 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | 2195 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | 2196 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | 2197 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | 2198 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | 2199 (1ull << IB_USER_VERBS_CMD_CREATE_AH) | 2200 (1ull << IB_USER_VERBS_CMD_MODIFY_AH) | 2201 (1ull << IB_USER_VERBS_CMD_QUERY_AH) | 2202 (1ull << IB_USER_VERBS_CMD_DESTROY_AH) | 2203 (1ull << IB_USER_VERBS_CMD_REG_MR) | 2204 (1ull << IB_USER_VERBS_CMD_DEREG_MR) | 2205 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | 2206 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | 2207 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | 2208 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | 2209 (1ull << IB_USER_VERBS_CMD_POLL_CQ) | 2210 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | 2211 (1ull << IB_USER_VERBS_CMD_CREATE_QP) | 2212 (1ull << IB_USER_VERBS_CMD_QUERY_QP) | 2213 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | 2214 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | 2215 (1ull << IB_USER_VERBS_CMD_POST_SEND) | 2216 (1ull << IB_USER_VERBS_CMD_POST_RECV) | 2217 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) | 2218 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) | 2219 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | 2220 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | 2221 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | 2222 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | 2223 (1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV); 2224 ibdev->node_type = RDMA_NODE_IB_CA; 2225 ibdev->phys_port_cnt = dd->num_pports; 2226 ibdev->num_comp_vectors = 1; 2227 ibdev->dma_device = &dd->pcidev->dev; 2228 ibdev->query_device = qib_query_device; 2229 ibdev->modify_device = qib_modify_device; 2230 ibdev->query_port = qib_query_port; 2231 ibdev->modify_port = qib_modify_port; 2232 ibdev->query_pkey = qib_query_pkey; 2233 ibdev->query_gid = qib_query_gid; 2234 ibdev->alloc_ucontext = qib_alloc_ucontext; 2235 ibdev->dealloc_ucontext = qib_dealloc_ucontext; 2236 ibdev->alloc_pd = qib_alloc_pd; 2237 ibdev->dealloc_pd = qib_dealloc_pd; 2238 ibdev->create_ah = qib_create_ah; 2239 ibdev->destroy_ah = qib_destroy_ah; 2240 ibdev->modify_ah = qib_modify_ah; 2241 ibdev->query_ah = qib_query_ah; 2242 ibdev->create_srq = qib_create_srq; 2243 ibdev->modify_srq = qib_modify_srq; 2244 ibdev->query_srq = qib_query_srq; 2245 ibdev->destroy_srq = qib_destroy_srq; 2246 ibdev->create_qp = qib_create_qp; 2247 ibdev->modify_qp = qib_modify_qp; 2248 ibdev->query_qp = qib_query_qp; 2249 ibdev->destroy_qp = qib_destroy_qp; 2250 ibdev->post_send = qib_post_send; 2251 ibdev->post_recv = qib_post_receive; 2252 ibdev->post_srq_recv = qib_post_srq_receive; 2253 ibdev->create_cq = qib_create_cq; 2254 ibdev->destroy_cq = qib_destroy_cq; 2255 ibdev->resize_cq = qib_resize_cq; 2256 ibdev->poll_cq = qib_poll_cq; 2257 ibdev->req_notify_cq = qib_req_notify_cq; 2258 ibdev->get_dma_mr = qib_get_dma_mr; 2259 ibdev->reg_phys_mr = qib_reg_phys_mr; 2260 ibdev->reg_user_mr = qib_reg_user_mr; 2261 ibdev->dereg_mr = qib_dereg_mr; 2262 ibdev->alloc_mr = qib_alloc_mr; 2263 ibdev->map_mr_sg = qib_map_mr_sg; 2264 ibdev->alloc_fmr = qib_alloc_fmr; 2265 ibdev->map_phys_fmr = qib_map_phys_fmr; 2266 ibdev->unmap_fmr = qib_unmap_fmr; 2267 ibdev->dealloc_fmr = qib_dealloc_fmr; 2268 ibdev->attach_mcast = qib_multicast_attach; 2269 ibdev->detach_mcast = qib_multicast_detach; 2270 ibdev->process_mad = qib_process_mad; 2271 ibdev->mmap = qib_mmap; 2272 ibdev->dma_ops = &qib_dma_mapping_ops; 2273 ibdev->get_port_immutable = qib_port_immutable; 2274 2275 snprintf(ibdev->node_desc, sizeof(ibdev->node_desc), 2276 "Intel Infiniband HCA %s", init_utsname()->nodename); 2277 2278 ret = ib_register_device(ibdev, qib_create_port_files); 2279 if (ret) 2280 goto err_reg; 2281 2282 ret = qib_create_agents(dev); 2283 if (ret) 2284 goto err_agents; 2285 2286 ret = qib_verbs_register_sysfs(dd); 2287 if (ret) 2288 goto err_class; 2289 2290 goto bail; 2291 2292err_class: 2293 qib_free_agents(dev); 2294err_agents: 2295 ib_unregister_device(ibdev); 2296err_reg: 2297err_tx: 2298 while (!list_empty(&dev->txreq_free)) { 2299 struct list_head *l = dev->txreq_free.next; 2300 struct qib_verbs_txreq *tx; 2301 2302 list_del(l); 2303 tx = list_entry(l, struct qib_verbs_txreq, txreq.list); 2304 kfree(tx); 2305 } 2306 if (ppd->sdma_descq_cnt) 2307 dma_free_coherent(&dd->pcidev->dev, 2308 ppd->sdma_descq_cnt * 2309 sizeof(struct qib_pio_header), 2310 dev->pio_hdrs, dev->pio_hdrs_phys); 2311err_hdrs: 2312 vfree(dev->lk_table.table); 2313err_lk: 2314 kfree(dev->qp_table); 2315err_qpt: 2316 qib_dev_err(dd, "cannot register verbs: %d!\n", -ret); 2317bail: 2318 return ret; 2319} 2320 2321void qib_unregister_ib_device(struct qib_devdata *dd) 2322{ 2323 struct qib_ibdev *dev = &dd->verbs_dev; 2324 struct ib_device *ibdev = &dev->ibdev; 2325 u32 qps_inuse; 2326 unsigned lk_tab_size; 2327 2328 qib_verbs_unregister_sysfs(dd); 2329 2330 qib_free_agents(dev); 2331 2332 ib_unregister_device(ibdev); 2333 2334 if (!list_empty(&dev->piowait)) 2335 qib_dev_err(dd, "piowait list not empty!\n"); 2336 if (!list_empty(&dev->dmawait)) 2337 qib_dev_err(dd, "dmawait list not empty!\n"); 2338 if (!list_empty(&dev->txwait)) 2339 qib_dev_err(dd, "txwait list not empty!\n"); 2340 if (!list_empty(&dev->memwait)) 2341 qib_dev_err(dd, "memwait list not empty!\n"); 2342 if (dev->dma_mr) 2343 qib_dev_err(dd, "DMA MR not NULL!\n"); 2344 2345 qps_inuse = qib_free_all_qps(dd); 2346 if (qps_inuse) 2347 qib_dev_err(dd, "QP memory leak! %u still in use\n", 2348 qps_inuse); 2349 2350 del_timer_sync(&dev->mem_timer); 2351 qib_free_qpn_table(&dev->qpn_table); 2352 while (!list_empty(&dev->txreq_free)) { 2353 struct list_head *l = dev->txreq_free.next; 2354 struct qib_verbs_txreq *tx; 2355 2356 list_del(l); 2357 tx = list_entry(l, struct qib_verbs_txreq, txreq.list); 2358 kfree(tx); 2359 } 2360 if (dd->pport->sdma_descq_cnt) 2361 dma_free_coherent(&dd->pcidev->dev, 2362 dd->pport->sdma_descq_cnt * 2363 sizeof(struct qib_pio_header), 2364 dev->pio_hdrs, dev->pio_hdrs_phys); 2365 lk_tab_size = dev->lk_table.max * sizeof(*dev->lk_table.table); 2366 vfree(dev->lk_table.table); 2367 kfree(dev->qp_table); 2368} 2369 2370/* 2371 * This must be called with s_lock held. 2372 */ 2373void qib_schedule_send(struct qib_qp *qp) 2374{ 2375 if (qib_send_ok(qp)) { 2376 struct qib_ibport *ibp = 2377 to_iport(qp->ibqp.device, qp->port_num); 2378 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 2379 2380 queue_work(ppd->qib_wq, &qp->s_work); 2381 } 2382} 2383