1/* 2 * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34#include <linux/module.h> 35#include <linux/moduleparam.h> 36#include <linux/random.h> 37#include <linux/highmem.h> 38#include <linux/slab.h> 39#include <asm/byteorder.h> 40 41#include <rdma/ib_verbs.h> 42#include <rdma/iw_cm.h> 43#include <rdma/ib_user_verbs.h> 44 45#include "nes.h" 46 47#include <rdma/ib_umem.h> 48 49atomic_t mod_qp_timouts; 50atomic_t qps_created; 51atomic_t sw_qps_destroyed; 52 53static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev); 54static int nes_dereg_mr(struct ib_mr *ib_mr); 55 56/** 57 * nes_alloc_mw 58 */ 59static struct ib_mw *nes_alloc_mw(struct ib_pd *ibpd, enum ib_mw_type type) 60{ 61 struct nes_pd *nespd = to_nespd(ibpd); 62 struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); 63 struct nes_device *nesdev = nesvnic->nesdev; 64 struct nes_adapter *nesadapter = nesdev->nesadapter; 65 struct nes_cqp_request *cqp_request; 66 struct nes_mr *nesmr; 67 struct ib_mw *ibmw; 68 struct nes_hw_cqp_wqe *cqp_wqe; 69 int ret; 70 u32 stag; 71 u32 stag_index = 0; 72 u32 next_stag_index = 0; 73 u32 driver_key = 0; 74 u8 stag_key = 0; 75 76 if (type != IB_MW_TYPE_1) 77 return ERR_PTR(-EINVAL); 78 79 get_random_bytes(&next_stag_index, sizeof(next_stag_index)); 80 stag_key = (u8)next_stag_index; 81 82 driver_key = 0; 83 84 next_stag_index >>= 8; 85 next_stag_index %= nesadapter->max_mr; 86 87 ret = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, 88 nesadapter->max_mr, &stag_index, &next_stag_index, NES_RESOURCE_MW); 89 if (ret) { 90 return ERR_PTR(ret); 91 } 92 93 nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); 94 if (!nesmr) { 95 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 96 return ERR_PTR(-ENOMEM); 97 } 98 99 stag = stag_index << 8; 100 stag |= driver_key; 101 stag += (u32)stag_key; 102 103 nes_debug(NES_DBG_MR, "Registering STag 0x%08X, index = 0x%08X\n", 104 stag, stag_index); 105 106 /* Register the region with the adapter */ 107 cqp_request = nes_get_cqp_request(nesdev); 108 if (cqp_request == NULL) { 109 kfree(nesmr); 110 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 111 return ERR_PTR(-ENOMEM); 112 } 113 114 cqp_request->waiting = 1; 115 cqp_wqe = &cqp_request->cqp_wqe; 116 117 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = 118 cpu_to_le32( NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_RIGHTS_REMOTE_READ | 119 NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_VA_TO | 120 NES_CQP_STAG_REM_ACC_EN); 121 122 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 123 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX, (nespd->pd_id & 0x00007fff)); 124 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag); 125 126 atomic_set(&cqp_request->refcount, 2); 127 nes_post_cqp_request(nesdev, cqp_request); 128 129 /* Wait for CQP */ 130 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), 131 NES_EVENT_TIMEOUT); 132 nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u," 133 " CQP Major:Minor codes = 0x%04X:0x%04X.\n", 134 stag, ret, cqp_request->major_code, cqp_request->minor_code); 135 if ((!ret) || (cqp_request->major_code)) { 136 nes_put_cqp_request(nesdev, cqp_request); 137 kfree(nesmr); 138 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 139 if (!ret) { 140 return ERR_PTR(-ETIME); 141 } else { 142 return ERR_PTR(-ENOMEM); 143 } 144 } 145 nes_put_cqp_request(nesdev, cqp_request); 146 147 nesmr->ibmw.rkey = stag; 148 nesmr->mode = IWNES_MEMREG_TYPE_MW; 149 ibmw = &nesmr->ibmw; 150 nesmr->pbl_4k = 0; 151 nesmr->pbls_used = 0; 152 153 return ibmw; 154} 155 156 157/** 158 * nes_dealloc_mw 159 */ 160static int nes_dealloc_mw(struct ib_mw *ibmw) 161{ 162 struct nes_mr *nesmr = to_nesmw(ibmw); 163 struct nes_vnic *nesvnic = to_nesvnic(ibmw->device); 164 struct nes_device *nesdev = nesvnic->nesdev; 165 struct nes_adapter *nesadapter = nesdev->nesadapter; 166 struct nes_hw_cqp_wqe *cqp_wqe; 167 struct nes_cqp_request *cqp_request; 168 int err = 0; 169 int ret; 170 171 /* Deallocate the window with the adapter */ 172 cqp_request = nes_get_cqp_request(nesdev); 173 if (cqp_request == NULL) { 174 nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); 175 return -ENOMEM; 176 } 177 cqp_request->waiting = 1; 178 cqp_wqe = &cqp_request->cqp_wqe; 179 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 180 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, NES_CQP_DEALLOCATE_STAG); 181 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, ibmw->rkey); 182 183 atomic_set(&cqp_request->refcount, 2); 184 nes_post_cqp_request(nesdev, cqp_request); 185 186 /* Wait for CQP */ 187 nes_debug(NES_DBG_MR, "Waiting for deallocate STag 0x%08X to complete.\n", 188 ibmw->rkey); 189 ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), 190 NES_EVENT_TIMEOUT); 191 nes_debug(NES_DBG_MR, "Deallocate STag completed, wait_event_timeout ret = %u," 192 " CQP Major:Minor codes = 0x%04X:0x%04X.\n", 193 ret, cqp_request->major_code, cqp_request->minor_code); 194 if (!ret) 195 err = -ETIME; 196 else if (cqp_request->major_code) 197 err = -EIO; 198 199 nes_put_cqp_request(nesdev, cqp_request); 200 201 nes_free_resource(nesadapter, nesadapter->allocated_mrs, 202 (ibmw->rkey & 0x0fffff00) >> 8); 203 kfree(nesmr); 204 205 return err; 206} 207 208 209/** 210 * nes_bind_mw 211 */ 212static int nes_bind_mw(struct ib_qp *ibqp, struct ib_mw *ibmw, 213 struct ib_mw_bind *ibmw_bind) 214{ 215 u64 u64temp; 216 struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); 217 struct nes_device *nesdev = nesvnic->nesdev; 218 /* struct nes_mr *nesmr = to_nesmw(ibmw); */ 219 struct nes_qp *nesqp = to_nesqp(ibqp); 220 struct nes_hw_qp_wqe *wqe; 221 unsigned long flags = 0; 222 u32 head; 223 u32 wqe_misc = 0; 224 u32 qsize; 225 226 if (nesqp->ibqp_state > IB_QPS_RTS) 227 return -EINVAL; 228 229 spin_lock_irqsave(&nesqp->lock, flags); 230 231 head = nesqp->hwqp.sq_head; 232 qsize = nesqp->hwqp.sq_tail; 233 234 /* Check for SQ overflow */ 235 if (((head + (2 * qsize) - nesqp->hwqp.sq_tail) % qsize) == (qsize - 1)) { 236 spin_unlock_irqrestore(&nesqp->lock, flags); 237 return -ENOMEM; 238 } 239 240 wqe = &nesqp->hwqp.sq_vbase[head]; 241 /* nes_debug(NES_DBG_MR, "processing sq wqe at %p, head = %u.\n", wqe, head); */ 242 nes_fill_init_qp_wqe(wqe, nesqp, head); 243 u64temp = ibmw_bind->wr_id; 244 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX, u64temp); 245 wqe_misc = NES_IWARP_SQ_OP_BIND; 246 247 wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE; 248 249 if (ibmw_bind->send_flags & IB_SEND_SIGNALED) 250 wqe_misc |= NES_IWARP_SQ_WQE_SIGNALED_COMPL; 251 252 if (ibmw_bind->bind_info.mw_access_flags & IB_ACCESS_REMOTE_WRITE) 253 wqe_misc |= NES_CQP_STAG_RIGHTS_REMOTE_WRITE; 254 if (ibmw_bind->bind_info.mw_access_flags & IB_ACCESS_REMOTE_READ) 255 wqe_misc |= NES_CQP_STAG_RIGHTS_REMOTE_READ; 256 257 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_MISC_IDX, wqe_misc); 258 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_MR_IDX, 259 ibmw_bind->bind_info.mr->lkey); 260 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_MW_IDX, ibmw->rkey); 261 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_LENGTH_LOW_IDX, 262 ibmw_bind->bind_info.length); 263 wqe->wqe_words[NES_IWARP_SQ_BIND_WQE_LENGTH_HIGH_IDX] = 0; 264 u64temp = (u64)ibmw_bind->bind_info.addr; 265 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_VA_FBO_LOW_IDX, u64temp); 266 267 head++; 268 if (head >= qsize) 269 head = 0; 270 271 nesqp->hwqp.sq_head = head; 272 barrier(); 273 274 nes_write32(nesdev->regs+NES_WQE_ALLOC, 275 (1 << 24) | 0x00800000 | nesqp->hwqp.qp_id); 276 277 spin_unlock_irqrestore(&nesqp->lock, flags); 278 279 return 0; 280} 281 282 283/* 284 * nes_alloc_fast_mr 285 */ 286static int alloc_fast_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd, 287 u32 stag, u32 page_count) 288{ 289 struct nes_hw_cqp_wqe *cqp_wqe; 290 struct nes_cqp_request *cqp_request; 291 unsigned long flags; 292 int ret; 293 struct nes_adapter *nesadapter = nesdev->nesadapter; 294 u32 opcode = 0; 295 u16 major_code; 296 u64 region_length = page_count * PAGE_SIZE; 297 298 299 cqp_request = nes_get_cqp_request(nesdev); 300 if (cqp_request == NULL) { 301 nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); 302 return -ENOMEM; 303 } 304 nes_debug(NES_DBG_MR, "alloc_fast_reg_mr: page_count = %d, " 305 "region_length = %llu\n", 306 page_count, region_length); 307 cqp_request->waiting = 1; 308 cqp_wqe = &cqp_request->cqp_wqe; 309 310 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 311 if (nesadapter->free_4kpbl > 0) { 312 nesadapter->free_4kpbl--; 313 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 314 } else { 315 /* No 4kpbl's available: */ 316 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 317 nes_debug(NES_DBG_MR, "Out of Pbls\n"); 318 nes_free_cqp_request(nesdev, cqp_request); 319 return -ENOMEM; 320 } 321 322 opcode = NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_MR | 323 NES_CQP_STAG_PBL_BLK_SIZE | NES_CQP_STAG_VA_TO | 324 NES_CQP_STAG_REM_ACC_EN; 325 /* 326 * The current OFED API does not support the zero based TO option. 327 * If added then need to changed the NES_CQP_STAG_VA* option. Also, 328 * the API does not support that ability to have the MR set for local 329 * access only when created and not allow the SQ op to override. Given 330 * this the remote enable must be set here. 331 */ 332 333 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 334 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); 335 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, 1); 336 337 cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] = 338 cpu_to_le32((u32)(region_length >> 8) & 0xff000000); 339 cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] |= 340 cpu_to_le32(nespd->pd_id & 0x00007fff); 341 342 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag); 343 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_VA_LOW_IDX, 0); 344 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_LOW_IDX, 0); 345 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, 0); 346 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (page_count * 8)); 347 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE); 348 barrier(); 349 350 atomic_set(&cqp_request->refcount, 2); 351 nes_post_cqp_request(nesdev, cqp_request); 352 353 /* Wait for CQP */ 354 ret = wait_event_timeout(cqp_request->waitq, 355 (0 != cqp_request->request_done), 356 NES_EVENT_TIMEOUT); 357 358 nes_debug(NES_DBG_MR, "Allocate STag 0x%08X completed, " 359 "wait_event_timeout ret = %u, CQP Major:Minor codes = " 360 "0x%04X:0x%04X.\n", stag, ret, cqp_request->major_code, 361 cqp_request->minor_code); 362 major_code = cqp_request->major_code; 363 nes_put_cqp_request(nesdev, cqp_request); 364 365 if (!ret || major_code) { 366 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 367 nesadapter->free_4kpbl++; 368 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 369 } 370 371 if (!ret) 372 return -ETIME; 373 else if (major_code) 374 return -EIO; 375 return 0; 376} 377 378/* 379 * nes_alloc_mr 380 */ 381static struct ib_mr *nes_alloc_mr(struct ib_pd *ibpd, 382 enum ib_mr_type mr_type, 383 u32 max_num_sg) 384{ 385 struct nes_pd *nespd = to_nespd(ibpd); 386 struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); 387 struct nes_device *nesdev = nesvnic->nesdev; 388 struct nes_adapter *nesadapter = nesdev->nesadapter; 389 390 u32 next_stag_index; 391 u8 stag_key = 0; 392 u32 driver_key = 0; 393 int err = 0; 394 u32 stag_index = 0; 395 struct nes_mr *nesmr; 396 u32 stag; 397 int ret; 398 struct ib_mr *ibmr; 399 400 if (mr_type != IB_MR_TYPE_MEM_REG) 401 return ERR_PTR(-EINVAL); 402 403 if (max_num_sg > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64))) 404 return ERR_PTR(-E2BIG); 405 406/* 407 * Note: Set to always use a fixed length single page entry PBL. This is to allow 408 * for the fast_reg_mr operation to always know the size of the PBL. 409 */ 410 if (max_num_sg > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64))) 411 return ERR_PTR(-E2BIG); 412 413 get_random_bytes(&next_stag_index, sizeof(next_stag_index)); 414 stag_key = (u8)next_stag_index; 415 next_stag_index >>= 8; 416 next_stag_index %= nesadapter->max_mr; 417 418 err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, 419 nesadapter->max_mr, &stag_index, 420 &next_stag_index, NES_RESOURCE_FAST_MR); 421 if (err) 422 return ERR_PTR(err); 423 424 nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); 425 if (!nesmr) { 426 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 427 return ERR_PTR(-ENOMEM); 428 } 429 430 stag = stag_index << 8; 431 stag |= driver_key; 432 stag += (u32)stag_key; 433 434 nes_debug(NES_DBG_MR, "Allocating STag 0x%08X index = 0x%08X\n", 435 stag, stag_index); 436 437 ret = alloc_fast_reg_mr(nesdev, nespd, stag, max_num_sg); 438 439 if (ret == 0) { 440 nesmr->ibmr.rkey = stag; 441 nesmr->ibmr.lkey = stag; 442 nesmr->mode = IWNES_MEMREG_TYPE_FMEM; 443 ibmr = &nesmr->ibmr; 444 } else { 445 kfree(nesmr); 446 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 447 return ERR_PTR(-ENOMEM); 448 } 449 450 nesmr->pages = pci_alloc_consistent(nesdev->pcidev, 451 max_num_sg * sizeof(u64), 452 &nesmr->paddr); 453 if (!nesmr->paddr) 454 goto err; 455 456 nesmr->max_pages = max_num_sg; 457 458 return ibmr; 459 460err: 461 nes_dereg_mr(ibmr); 462 463 return ERR_PTR(-ENOMEM); 464} 465 466static int nes_set_page(struct ib_mr *ibmr, u64 addr) 467{ 468 struct nes_mr *nesmr = to_nesmr(ibmr); 469 470 if (unlikely(nesmr->npages == nesmr->max_pages)) 471 return -ENOMEM; 472 473 nesmr->pages[nesmr->npages++] = cpu_to_le64(addr); 474 475 return 0; 476} 477 478static int nes_map_mr_sg(struct ib_mr *ibmr, 479 struct scatterlist *sg, 480 int sg_nents) 481{ 482 struct nes_mr *nesmr = to_nesmr(ibmr); 483 484 nesmr->npages = 0; 485 486 return ib_sg_to_pages(ibmr, sg, sg_nents, nes_set_page); 487} 488 489/** 490 * nes_query_device 491 */ 492static int nes_query_device(struct ib_device *ibdev, struct ib_device_attr *props, 493 struct ib_udata *uhw) 494{ 495 struct nes_vnic *nesvnic = to_nesvnic(ibdev); 496 struct nes_device *nesdev = nesvnic->nesdev; 497 struct nes_ib_device *nesibdev = nesvnic->nesibdev; 498 499 if (uhw->inlen || uhw->outlen) 500 return -EINVAL; 501 502 memset(props, 0, sizeof(*props)); 503 memcpy(&props->sys_image_guid, nesvnic->netdev->dev_addr, 6); 504 505 props->fw_ver = nesdev->nesadapter->firmware_version; 506 props->device_cap_flags = nesdev->nesadapter->device_cap_flags; 507 props->vendor_id = nesdev->nesadapter->vendor_id; 508 props->vendor_part_id = nesdev->nesadapter->vendor_part_id; 509 props->hw_ver = nesdev->nesadapter->hw_rev; 510 props->max_mr_size = 0x80000000; 511 props->max_qp = nesibdev->max_qp; 512 props->max_qp_wr = nesdev->nesadapter->max_qp_wr - 2; 513 props->max_sge = nesdev->nesadapter->max_sge; 514 props->max_cq = nesibdev->max_cq; 515 props->max_cqe = nesdev->nesadapter->max_cqe; 516 props->max_mr = nesibdev->max_mr; 517 props->max_mw = nesibdev->max_mr; 518 props->max_pd = nesibdev->max_pd; 519 props->max_sge_rd = 1; 520 switch (nesdev->nesadapter->max_irrq_wr) { 521 case 0: 522 props->max_qp_rd_atom = 2; 523 break; 524 case 1: 525 props->max_qp_rd_atom = 8; 526 break; 527 case 2: 528 props->max_qp_rd_atom = 32; 529 break; 530 case 3: 531 props->max_qp_rd_atom = 64; 532 break; 533 default: 534 props->max_qp_rd_atom = 0; 535 } 536 props->max_qp_init_rd_atom = props->max_qp_rd_atom; 537 props->atomic_cap = IB_ATOMIC_NONE; 538 props->max_map_per_fmr = 1; 539 540 return 0; 541} 542 543 544/** 545 * nes_query_port 546 */ 547static int nes_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props) 548{ 549 struct nes_vnic *nesvnic = to_nesvnic(ibdev); 550 struct net_device *netdev = nesvnic->netdev; 551 552 memset(props, 0, sizeof(*props)); 553 554 props->max_mtu = IB_MTU_4096; 555 556 if (netdev->mtu >= 4096) 557 props->active_mtu = IB_MTU_4096; 558 else if (netdev->mtu >= 2048) 559 props->active_mtu = IB_MTU_2048; 560 else if (netdev->mtu >= 1024) 561 props->active_mtu = IB_MTU_1024; 562 else if (netdev->mtu >= 512) 563 props->active_mtu = IB_MTU_512; 564 else 565 props->active_mtu = IB_MTU_256; 566 567 props->lid = 1; 568 props->lmc = 0; 569 props->sm_lid = 0; 570 props->sm_sl = 0; 571 if (netif_queue_stopped(netdev)) 572 props->state = IB_PORT_DOWN; 573 else if (nesvnic->linkup) 574 props->state = IB_PORT_ACTIVE; 575 else 576 props->state = IB_PORT_DOWN; 577 props->phys_state = 0; 578 props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 579 IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP; 580 props->gid_tbl_len = 1; 581 props->pkey_tbl_len = 1; 582 props->qkey_viol_cntr = 0; 583 props->active_width = IB_WIDTH_4X; 584 props->active_speed = IB_SPEED_SDR; 585 props->max_msg_sz = 0x80000000; 586 587 return 0; 588} 589 590/** 591 * nes_query_pkey 592 */ 593static int nes_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 594{ 595 *pkey = 0; 596 return 0; 597} 598 599 600/** 601 * nes_query_gid 602 */ 603static int nes_query_gid(struct ib_device *ibdev, u8 port, 604 int index, union ib_gid *gid) 605{ 606 struct nes_vnic *nesvnic = to_nesvnic(ibdev); 607 608 memset(&(gid->raw[0]), 0, sizeof(gid->raw)); 609 memcpy(&(gid->raw[0]), nesvnic->netdev->dev_addr, 6); 610 611 return 0; 612} 613 614 615/** 616 * nes_alloc_ucontext - Allocate the user context data structure. This keeps track 617 * of all objects associated with a particular user-mode client. 618 */ 619static struct ib_ucontext *nes_alloc_ucontext(struct ib_device *ibdev, 620 struct ib_udata *udata) 621{ 622 struct nes_vnic *nesvnic = to_nesvnic(ibdev); 623 struct nes_device *nesdev = nesvnic->nesdev; 624 struct nes_adapter *nesadapter = nesdev->nesadapter; 625 struct nes_alloc_ucontext_req req; 626 struct nes_alloc_ucontext_resp uresp; 627 struct nes_ucontext *nes_ucontext; 628 struct nes_ib_device *nesibdev = nesvnic->nesibdev; 629 630 631 if (ib_copy_from_udata(&req, udata, sizeof(struct nes_alloc_ucontext_req))) { 632 printk(KERN_ERR PFX "Invalid structure size on allocate user context.\n"); 633 return ERR_PTR(-EINVAL); 634 } 635 636 if (req.userspace_ver != NES_ABI_USERSPACE_VER) { 637 printk(KERN_ERR PFX "Invalid userspace driver version detected. Detected version %d, should be %d\n", 638 req.userspace_ver, NES_ABI_USERSPACE_VER); 639 return ERR_PTR(-EINVAL); 640 } 641 642 643 memset(&uresp, 0, sizeof uresp); 644 645 uresp.max_qps = nesibdev->max_qp; 646 uresp.max_pds = nesibdev->max_pd; 647 uresp.wq_size = nesdev->nesadapter->max_qp_wr * 2; 648 uresp.virtwq = nesadapter->virtwq; 649 uresp.kernel_ver = NES_ABI_KERNEL_VER; 650 651 nes_ucontext = kzalloc(sizeof *nes_ucontext, GFP_KERNEL); 652 if (!nes_ucontext) 653 return ERR_PTR(-ENOMEM); 654 655 nes_ucontext->nesdev = nesdev; 656 nes_ucontext->mmap_wq_offset = uresp.max_pds; 657 nes_ucontext->mmap_cq_offset = nes_ucontext->mmap_wq_offset + 658 ((sizeof(struct nes_hw_qp_wqe) * uresp.max_qps * 2) + PAGE_SIZE-1) / 659 PAGE_SIZE; 660 661 662 if (ib_copy_to_udata(udata, &uresp, sizeof uresp)) { 663 kfree(nes_ucontext); 664 return ERR_PTR(-EFAULT); 665 } 666 667 INIT_LIST_HEAD(&nes_ucontext->cq_reg_mem_list); 668 INIT_LIST_HEAD(&nes_ucontext->qp_reg_mem_list); 669 atomic_set(&nes_ucontext->usecnt, 1); 670 return &nes_ucontext->ibucontext; 671} 672 673 674/** 675 * nes_dealloc_ucontext 676 */ 677static int nes_dealloc_ucontext(struct ib_ucontext *context) 678{ 679 /* struct nes_vnic *nesvnic = to_nesvnic(context->device); */ 680 /* struct nes_device *nesdev = nesvnic->nesdev; */ 681 struct nes_ucontext *nes_ucontext = to_nesucontext(context); 682 683 if (!atomic_dec_and_test(&nes_ucontext->usecnt)) 684 return 0; 685 kfree(nes_ucontext); 686 return 0; 687} 688 689 690/** 691 * nes_mmap 692 */ 693static int nes_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 694{ 695 unsigned long index; 696 struct nes_vnic *nesvnic = to_nesvnic(context->device); 697 struct nes_device *nesdev = nesvnic->nesdev; 698 /* struct nes_adapter *nesadapter = nesdev->nesadapter; */ 699 struct nes_ucontext *nes_ucontext; 700 struct nes_qp *nesqp; 701 702 nes_ucontext = to_nesucontext(context); 703 704 705 if (vma->vm_pgoff >= nes_ucontext->mmap_wq_offset) { 706 index = (vma->vm_pgoff - nes_ucontext->mmap_wq_offset) * PAGE_SIZE; 707 index /= ((sizeof(struct nes_hw_qp_wqe) * nesdev->nesadapter->max_qp_wr * 2) + 708 PAGE_SIZE-1) & (~(PAGE_SIZE-1)); 709 if (!test_bit(index, nes_ucontext->allocated_wqs)) { 710 nes_debug(NES_DBG_MMAP, "wq %lu not allocated\n", index); 711 return -EFAULT; 712 } 713 nesqp = nes_ucontext->mmap_nesqp[index]; 714 if (nesqp == NULL) { 715 nes_debug(NES_DBG_MMAP, "wq %lu has a NULL QP base.\n", index); 716 return -EFAULT; 717 } 718 if (remap_pfn_range(vma, vma->vm_start, 719 virt_to_phys(nesqp->hwqp.sq_vbase) >> PAGE_SHIFT, 720 vma->vm_end - vma->vm_start, 721 vma->vm_page_prot)) { 722 nes_debug(NES_DBG_MMAP, "remap_pfn_range failed.\n"); 723 return -EAGAIN; 724 } 725 vma->vm_private_data = nesqp; 726 return 0; 727 } else { 728 index = vma->vm_pgoff; 729 if (!test_bit(index, nes_ucontext->allocated_doorbells)) 730 return -EFAULT; 731 732 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 733 if (io_remap_pfn_range(vma, vma->vm_start, 734 (nesdev->doorbell_start + 735 ((nes_ucontext->mmap_db_index[index] - nesdev->base_doorbell_index) * 4096)) 736 >> PAGE_SHIFT, PAGE_SIZE, vma->vm_page_prot)) 737 return -EAGAIN; 738 vma->vm_private_data = nes_ucontext; 739 return 0; 740 } 741 742 return -ENOSYS; 743} 744 745 746/** 747 * nes_alloc_pd 748 */ 749static struct ib_pd *nes_alloc_pd(struct ib_device *ibdev, 750 struct ib_ucontext *context, struct ib_udata *udata) 751{ 752 struct nes_pd *nespd; 753 struct nes_vnic *nesvnic = to_nesvnic(ibdev); 754 struct nes_device *nesdev = nesvnic->nesdev; 755 struct nes_adapter *nesadapter = nesdev->nesadapter; 756 struct nes_ucontext *nesucontext; 757 struct nes_alloc_pd_resp uresp; 758 u32 pd_num = 0; 759 int err; 760 761 nes_debug(NES_DBG_PD, "nesvnic=%p, netdev=%p %s, ibdev=%p, context=%p, netdev refcnt=%u\n", 762 nesvnic, nesdev->netdev[0], nesdev->netdev[0]->name, ibdev, context, 763 netdev_refcnt_read(nesvnic->netdev)); 764 765 err = nes_alloc_resource(nesadapter, nesadapter->allocated_pds, 766 nesadapter->max_pd, &pd_num, &nesadapter->next_pd, NES_RESOURCE_PD); 767 if (err) { 768 return ERR_PTR(err); 769 } 770 771 nespd = kzalloc(sizeof (struct nes_pd), GFP_KERNEL); 772 if (!nespd) { 773 nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num); 774 return ERR_PTR(-ENOMEM); 775 } 776 777 nes_debug(NES_DBG_PD, "Allocating PD (%p) for ib device %s\n", 778 nespd, nesvnic->nesibdev->ibdev.name); 779 780 nespd->pd_id = (pd_num << (PAGE_SHIFT-12)) + nesadapter->base_pd; 781 782 if (context) { 783 nesucontext = to_nesucontext(context); 784 nespd->mmap_db_index = find_next_zero_bit(nesucontext->allocated_doorbells, 785 NES_MAX_USER_DB_REGIONS, nesucontext->first_free_db); 786 nes_debug(NES_DBG_PD, "find_first_zero_biton doorbells returned %u, mapping pd_id %u.\n", 787 nespd->mmap_db_index, nespd->pd_id); 788 if (nespd->mmap_db_index >= NES_MAX_USER_DB_REGIONS) { 789 nes_debug(NES_DBG_PD, "mmap_db_index > MAX\n"); 790 nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num); 791 kfree(nespd); 792 return ERR_PTR(-ENOMEM); 793 } 794 795 uresp.pd_id = nespd->pd_id; 796 uresp.mmap_db_index = nespd->mmap_db_index; 797 if (ib_copy_to_udata(udata, &uresp, sizeof (struct nes_alloc_pd_resp))) { 798 nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num); 799 kfree(nespd); 800 return ERR_PTR(-EFAULT); 801 } 802 803 set_bit(nespd->mmap_db_index, nesucontext->allocated_doorbells); 804 nesucontext->mmap_db_index[nespd->mmap_db_index] = nespd->pd_id; 805 nesucontext->first_free_db = nespd->mmap_db_index + 1; 806 } 807 808 nes_debug(NES_DBG_PD, "PD%u structure located @%p.\n", nespd->pd_id, nespd); 809 return &nespd->ibpd; 810} 811 812 813/** 814 * nes_dealloc_pd 815 */ 816static int nes_dealloc_pd(struct ib_pd *ibpd) 817{ 818 struct nes_ucontext *nesucontext; 819 struct nes_pd *nespd = to_nespd(ibpd); 820 struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); 821 struct nes_device *nesdev = nesvnic->nesdev; 822 struct nes_adapter *nesadapter = nesdev->nesadapter; 823 824 if ((ibpd->uobject) && (ibpd->uobject->context)) { 825 nesucontext = to_nesucontext(ibpd->uobject->context); 826 nes_debug(NES_DBG_PD, "Clearing bit %u from allocated doorbells\n", 827 nespd->mmap_db_index); 828 clear_bit(nespd->mmap_db_index, nesucontext->allocated_doorbells); 829 nesucontext->mmap_db_index[nespd->mmap_db_index] = 0; 830 if (nesucontext->first_free_db > nespd->mmap_db_index) { 831 nesucontext->first_free_db = nespd->mmap_db_index; 832 } 833 } 834 835 nes_debug(NES_DBG_PD, "Deallocating PD%u structure located @%p.\n", 836 nespd->pd_id, nespd); 837 nes_free_resource(nesadapter, nesadapter->allocated_pds, 838 (nespd->pd_id-nesadapter->base_pd)>>(PAGE_SHIFT-12)); 839 kfree(nespd); 840 841 return 0; 842} 843 844 845/** 846 * nes_create_ah 847 */ 848static struct ib_ah *nes_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr) 849{ 850 return ERR_PTR(-ENOSYS); 851} 852 853 854/** 855 * nes_destroy_ah 856 */ 857static int nes_destroy_ah(struct ib_ah *ah) 858{ 859 return -ENOSYS; 860} 861 862 863/** 864 * nes_get_encoded_size 865 */ 866static inline u8 nes_get_encoded_size(int *size) 867{ 868 u8 encoded_size = 0; 869 if (*size <= 32) { 870 *size = 32; 871 encoded_size = 1; 872 } else if (*size <= 128) { 873 *size = 128; 874 encoded_size = 2; 875 } else if (*size <= 512) { 876 *size = 512; 877 encoded_size = 3; 878 } 879 return (encoded_size); 880} 881 882 883 884/** 885 * nes_setup_virt_qp 886 */ 887static int nes_setup_virt_qp(struct nes_qp *nesqp, struct nes_pbl *nespbl, 888 struct nes_vnic *nesvnic, int sq_size, int rq_size) 889{ 890 unsigned long flags; 891 void *mem; 892 __le64 *pbl = NULL; 893 __le64 *tpbl; 894 __le64 *pblbuffer; 895 struct nes_device *nesdev = nesvnic->nesdev; 896 struct nes_adapter *nesadapter = nesdev->nesadapter; 897 u32 pbl_entries; 898 u8 rq_pbl_entries; 899 u8 sq_pbl_entries; 900 901 pbl_entries = nespbl->pbl_size >> 3; 902 nes_debug(NES_DBG_QP, "Userspace PBL, pbl_size=%u, pbl_entries = %d pbl_vbase=%p, pbl_pbase=%lx\n", 903 nespbl->pbl_size, pbl_entries, 904 (void *)nespbl->pbl_vbase, 905 (unsigned long) nespbl->pbl_pbase); 906 pbl = (__le64 *) nespbl->pbl_vbase; /* points to first pbl entry */ 907 /* now lets set the sq_vbase as well as rq_vbase addrs we will assign */ 908 /* the first pbl to be fro the rq_vbase... */ 909 rq_pbl_entries = (rq_size * sizeof(struct nes_hw_qp_wqe)) >> 12; 910 sq_pbl_entries = (sq_size * sizeof(struct nes_hw_qp_wqe)) >> 12; 911 nesqp->hwqp.sq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1]))) << 32); 912 if (!nespbl->page) { 913 nes_debug(NES_DBG_QP, "QP nespbl->page is NULL \n"); 914 kfree(nespbl); 915 return -ENOMEM; 916 } 917 918 nesqp->hwqp.sq_vbase = kmap(nespbl->page); 919 nesqp->page = nespbl->page; 920 if (!nesqp->hwqp.sq_vbase) { 921 nes_debug(NES_DBG_QP, "QP sq_vbase kmap failed\n"); 922 kfree(nespbl); 923 return -ENOMEM; 924 } 925 926 /* Now to get to sq.. we need to calculate how many */ 927 /* PBL entries were used by the rq.. */ 928 pbl += sq_pbl_entries; 929 nesqp->hwqp.rq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1]))) << 32); 930 /* nesqp->hwqp.rq_vbase = bus_to_virt(*pbl); */ 931 /*nesqp->hwqp.rq_vbase = phys_to_virt(*pbl); */ 932 933 nes_debug(NES_DBG_QP, "QP sq_vbase= %p sq_pbase=%lx rq_vbase=%p rq_pbase=%lx\n", 934 nesqp->hwqp.sq_vbase, (unsigned long) nesqp->hwqp.sq_pbase, 935 nesqp->hwqp.rq_vbase, (unsigned long) nesqp->hwqp.rq_pbase); 936 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 937 if (!nesadapter->free_256pbl) { 938 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, 939 nespbl->pbl_pbase); 940 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 941 kunmap(nesqp->page); 942 kfree(nespbl); 943 return -ENOMEM; 944 } 945 nesadapter->free_256pbl--; 946 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 947 948 nesqp->pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 256, &nesqp->pbl_pbase); 949 pblbuffer = nesqp->pbl_vbase; 950 if (!nesqp->pbl_vbase) { 951 /* memory allocated during nes_reg_user_mr() */ 952 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, 953 nespbl->pbl_pbase); 954 kfree(nespbl); 955 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 956 nesadapter->free_256pbl++; 957 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 958 kunmap(nesqp->page); 959 return -ENOMEM; 960 } 961 memset(nesqp->pbl_vbase, 0, 256); 962 /* fill in the page address in the pbl buffer.. */ 963 tpbl = pblbuffer + 16; 964 pbl = (__le64 *)nespbl->pbl_vbase; 965 while (sq_pbl_entries--) 966 *tpbl++ = *pbl++; 967 tpbl = pblbuffer; 968 while (rq_pbl_entries--) 969 *tpbl++ = *pbl++; 970 971 /* done with memory allocated during nes_reg_user_mr() */ 972 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, 973 nespbl->pbl_pbase); 974 kfree(nespbl); 975 976 nesqp->qp_mem_size = 977 max((u32)sizeof(struct nes_qp_context), ((u32)256)) + 256; /* this is Q2 */ 978 /* Round up to a multiple of a page */ 979 nesqp->qp_mem_size += PAGE_SIZE - 1; 980 nesqp->qp_mem_size &= ~(PAGE_SIZE - 1); 981 982 mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size, 983 &nesqp->hwqp.q2_pbase); 984 985 if (!mem) { 986 pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase); 987 nesqp->pbl_vbase = NULL; 988 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 989 nesadapter->free_256pbl++; 990 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 991 kunmap(nesqp->page); 992 return -ENOMEM; 993 } 994 nesqp->sq_kmapped = 1; 995 nesqp->hwqp.q2_vbase = mem; 996 mem += 256; 997 memset(nesqp->hwqp.q2_vbase, 0, 256); 998 nesqp->nesqp_context = mem; 999 memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context)); 1000 nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256; 1001 1002 return 0; 1003} 1004 1005 1006/** 1007 * nes_setup_mmap_qp 1008 */ 1009static int nes_setup_mmap_qp(struct nes_qp *nesqp, struct nes_vnic *nesvnic, 1010 int sq_size, int rq_size) 1011{ 1012 void *mem; 1013 struct nes_device *nesdev = nesvnic->nesdev; 1014 1015 nesqp->qp_mem_size = (sizeof(struct nes_hw_qp_wqe) * sq_size) + 1016 (sizeof(struct nes_hw_qp_wqe) * rq_size) + 1017 max((u32)sizeof(struct nes_qp_context), ((u32)256)) + 1018 256; /* this is Q2 */ 1019 /* Round up to a multiple of a page */ 1020 nesqp->qp_mem_size += PAGE_SIZE - 1; 1021 nesqp->qp_mem_size &= ~(PAGE_SIZE - 1); 1022 1023 mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size, 1024 &nesqp->hwqp.sq_pbase); 1025 if (!mem) 1026 return -ENOMEM; 1027 nes_debug(NES_DBG_QP, "PCI consistent memory for " 1028 "host descriptor rings located @ %p (pa = 0x%08lX.) size = %u.\n", 1029 mem, (unsigned long)nesqp->hwqp.sq_pbase, nesqp->qp_mem_size); 1030 1031 memset(mem, 0, nesqp->qp_mem_size); 1032 1033 nesqp->hwqp.sq_vbase = mem; 1034 mem += sizeof(struct nes_hw_qp_wqe) * sq_size; 1035 1036 nesqp->hwqp.rq_vbase = mem; 1037 nesqp->hwqp.rq_pbase = nesqp->hwqp.sq_pbase + 1038 sizeof(struct nes_hw_qp_wqe) * sq_size; 1039 mem += sizeof(struct nes_hw_qp_wqe) * rq_size; 1040 1041 nesqp->hwqp.q2_vbase = mem; 1042 nesqp->hwqp.q2_pbase = nesqp->hwqp.rq_pbase + 1043 sizeof(struct nes_hw_qp_wqe) * rq_size; 1044 mem += 256; 1045 memset(nesqp->hwqp.q2_vbase, 0, 256); 1046 1047 nesqp->nesqp_context = mem; 1048 nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256; 1049 memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context)); 1050 return 0; 1051} 1052 1053 1054/** 1055 * nes_free_qp_mem() is to free up the qp's pci_alloc_consistent() memory. 1056 */ 1057static inline void nes_free_qp_mem(struct nes_device *nesdev, 1058 struct nes_qp *nesqp, int virt_wqs) 1059{ 1060 unsigned long flags; 1061 struct nes_adapter *nesadapter = nesdev->nesadapter; 1062 if (!virt_wqs) { 1063 pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, 1064 nesqp->hwqp.sq_vbase, nesqp->hwqp.sq_pbase); 1065 }else { 1066 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 1067 nesadapter->free_256pbl++; 1068 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1069 pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase); 1070 pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase ); 1071 nesqp->pbl_vbase = NULL; 1072 if (nesqp->sq_kmapped) { 1073 nesqp->sq_kmapped = 0; 1074 kunmap(nesqp->page); 1075 } 1076 } 1077} 1078 1079 1080/** 1081 * nes_create_qp 1082 */ 1083static struct ib_qp *nes_create_qp(struct ib_pd *ibpd, 1084 struct ib_qp_init_attr *init_attr, struct ib_udata *udata) 1085{ 1086 u64 u64temp= 0; 1087 u64 u64nesqp = 0; 1088 struct nes_pd *nespd = to_nespd(ibpd); 1089 struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); 1090 struct nes_device *nesdev = nesvnic->nesdev; 1091 struct nes_adapter *nesadapter = nesdev->nesadapter; 1092 struct nes_qp *nesqp; 1093 struct nes_cq *nescq; 1094 struct nes_ucontext *nes_ucontext; 1095 struct nes_hw_cqp_wqe *cqp_wqe; 1096 struct nes_cqp_request *cqp_request; 1097 struct nes_create_qp_req req; 1098 struct nes_create_qp_resp uresp; 1099 struct nes_pbl *nespbl = NULL; 1100 u32 qp_num = 0; 1101 u32 opcode = 0; 1102 /* u32 counter = 0; */ 1103 void *mem; 1104 unsigned long flags; 1105 int ret; 1106 int err; 1107 int virt_wqs = 0; 1108 int sq_size; 1109 int rq_size; 1110 u8 sq_encoded_size; 1111 u8 rq_encoded_size; 1112 /* int counter; */ 1113 1114 if (init_attr->create_flags) 1115 return ERR_PTR(-EINVAL); 1116 1117 atomic_inc(&qps_created); 1118 switch (init_attr->qp_type) { 1119 case IB_QPT_RC: 1120 if (nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) { 1121 init_attr->cap.max_inline_data = 0; 1122 } else { 1123 init_attr->cap.max_inline_data = 64; 1124 } 1125 sq_size = init_attr->cap.max_send_wr; 1126 rq_size = init_attr->cap.max_recv_wr; 1127 1128 /* check if the encoded sizes are OK or not... */ 1129 sq_encoded_size = nes_get_encoded_size(&sq_size); 1130 rq_encoded_size = nes_get_encoded_size(&rq_size); 1131 1132 if ((!sq_encoded_size) || (!rq_encoded_size)) { 1133 nes_debug(NES_DBG_QP, "ERROR bad rq (%u) or sq (%u) size\n", 1134 rq_size, sq_size); 1135 return ERR_PTR(-EINVAL); 1136 } 1137 1138 init_attr->cap.max_send_wr = sq_size -2; 1139 init_attr->cap.max_recv_wr = rq_size -1; 1140 nes_debug(NES_DBG_QP, "RQ size=%u, SQ Size=%u\n", rq_size, sq_size); 1141 1142 ret = nes_alloc_resource(nesadapter, nesadapter->allocated_qps, 1143 nesadapter->max_qp, &qp_num, &nesadapter->next_qp, NES_RESOURCE_QP); 1144 if (ret) { 1145 return ERR_PTR(ret); 1146 } 1147 1148 /* Need 512 (actually now 1024) byte alignment on this structure */ 1149 mem = kzalloc(sizeof(*nesqp)+NES_SW_CONTEXT_ALIGN-1, GFP_KERNEL); 1150 if (!mem) { 1151 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1152 nes_debug(NES_DBG_QP, "Unable to allocate QP\n"); 1153 return ERR_PTR(-ENOMEM); 1154 } 1155 u64nesqp = (unsigned long)mem; 1156 u64nesqp += ((u64)NES_SW_CONTEXT_ALIGN) - 1; 1157 u64temp = ((u64)NES_SW_CONTEXT_ALIGN) - 1; 1158 u64nesqp &= ~u64temp; 1159 nesqp = (struct nes_qp *)(unsigned long)u64nesqp; 1160 /* nes_debug(NES_DBG_QP, "nesqp=%p, allocated buffer=%p. Rounded to closest %u\n", 1161 nesqp, mem, NES_SW_CONTEXT_ALIGN); */ 1162 nesqp->allocated_buffer = mem; 1163 1164 if (udata) { 1165 if (ib_copy_from_udata(&req, udata, sizeof(struct nes_create_qp_req))) { 1166 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1167 kfree(nesqp->allocated_buffer); 1168 nes_debug(NES_DBG_QP, "ib_copy_from_udata() Failed \n"); 1169 return ERR_PTR(-EFAULT); 1170 } 1171 if (req.user_wqe_buffers) { 1172 virt_wqs = 1; 1173 } 1174 if (req.user_qp_buffer) 1175 nesqp->nesuqp_addr = req.user_qp_buffer; 1176 if ((ibpd->uobject) && (ibpd->uobject->context)) { 1177 nesqp->user_mode = 1; 1178 nes_ucontext = to_nesucontext(ibpd->uobject->context); 1179 if (virt_wqs) { 1180 err = 1; 1181 list_for_each_entry(nespbl, &nes_ucontext->qp_reg_mem_list, list) { 1182 if (nespbl->user_base == (unsigned long )req.user_wqe_buffers) { 1183 list_del(&nespbl->list); 1184 err = 0; 1185 nes_debug(NES_DBG_QP, "Found PBL for virtual QP. nespbl=%p. user_base=0x%lx\n", 1186 nespbl, nespbl->user_base); 1187 break; 1188 } 1189 } 1190 if (err) { 1191 nes_debug(NES_DBG_QP, "Didn't Find PBL for virtual QP. address = %llx.\n", 1192 (long long unsigned int)req.user_wqe_buffers); 1193 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1194 kfree(nesqp->allocated_buffer); 1195 return ERR_PTR(-EFAULT); 1196 } 1197 } 1198 1199 nes_ucontext = to_nesucontext(ibpd->uobject->context); 1200 nesqp->mmap_sq_db_index = 1201 find_next_zero_bit(nes_ucontext->allocated_wqs, 1202 NES_MAX_USER_WQ_REGIONS, nes_ucontext->first_free_wq); 1203 /* nes_debug(NES_DBG_QP, "find_first_zero_biton wqs returned %u\n", 1204 nespd->mmap_db_index); */ 1205 if (nesqp->mmap_sq_db_index >= NES_MAX_USER_WQ_REGIONS) { 1206 nes_debug(NES_DBG_QP, 1207 "db index > max user regions, failing create QP\n"); 1208 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1209 if (virt_wqs) { 1210 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, 1211 nespbl->pbl_pbase); 1212 kfree(nespbl); 1213 } 1214 kfree(nesqp->allocated_buffer); 1215 return ERR_PTR(-ENOMEM); 1216 } 1217 set_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs); 1218 nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = nesqp; 1219 nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index + 1; 1220 } else { 1221 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1222 kfree(nesqp->allocated_buffer); 1223 return ERR_PTR(-EFAULT); 1224 } 1225 } 1226 err = (!virt_wqs) ? nes_setup_mmap_qp(nesqp, nesvnic, sq_size, rq_size) : 1227 nes_setup_virt_qp(nesqp, nespbl, nesvnic, sq_size, rq_size); 1228 if (err) { 1229 nes_debug(NES_DBG_QP, 1230 "error geting qp mem code = %d\n", err); 1231 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1232 kfree(nesqp->allocated_buffer); 1233 return ERR_PTR(-ENOMEM); 1234 } 1235 1236 nesqp->hwqp.sq_size = sq_size; 1237 nesqp->hwqp.sq_encoded_size = sq_encoded_size; 1238 nesqp->hwqp.sq_head = 1; 1239 nesqp->hwqp.rq_size = rq_size; 1240 nesqp->hwqp.rq_encoded_size = rq_encoded_size; 1241 /* nes_debug(NES_DBG_QP, "nesqp->nesqp_context_pbase = %p\n", 1242 (void *)nesqp->nesqp_context_pbase); 1243 */ 1244 nesqp->hwqp.qp_id = qp_num; 1245 nesqp->ibqp.qp_num = nesqp->hwqp.qp_id; 1246 nesqp->nespd = nespd; 1247 1248 nescq = to_nescq(init_attr->send_cq); 1249 nesqp->nesscq = nescq; 1250 nescq = to_nescq(init_attr->recv_cq); 1251 nesqp->nesrcq = nescq; 1252 1253 nesqp->nesqp_context->misc |= cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 1254 NES_QPCONTEXT_MISC_PCI_FCN_SHIFT); 1255 nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.rq_encoded_size << 1256 NES_QPCONTEXT_MISC_RQ_SIZE_SHIFT); 1257 nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.sq_encoded_size << 1258 NES_QPCONTEXT_MISC_SQ_SIZE_SHIFT); 1259 if (!udata) { 1260 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_PRIV_EN); 1261 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_FAST_REGISTER_EN); 1262 } 1263 nesqp->nesqp_context->cqs = cpu_to_le32(nesqp->nesscq->hw_cq.cq_number + 1264 ((u32)nesqp->nesrcq->hw_cq.cq_number << 16)); 1265 u64temp = (u64)nesqp->hwqp.sq_pbase; 1266 nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp); 1267 nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); 1268 1269 1270 if (!virt_wqs) { 1271 u64temp = (u64)nesqp->hwqp.sq_pbase; 1272 nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp); 1273 nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); 1274 u64temp = (u64)nesqp->hwqp.rq_pbase; 1275 nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp); 1276 nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); 1277 } else { 1278 u64temp = (u64)nesqp->pbl_pbase; 1279 nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp); 1280 nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); 1281 } 1282 1283 /* nes_debug(NES_DBG_QP, "next_qp_nic_index=%u, using nic_index=%d\n", 1284 nesvnic->next_qp_nic_index, 1285 nesvnic->qp_nic_index[nesvnic->next_qp_nic_index]); */ 1286 spin_lock_irqsave(&nesdev->cqp.lock, flags); 1287 nesqp->nesqp_context->misc2 |= cpu_to_le32( 1288 (u32)nesvnic->qp_nic_index[nesvnic->next_qp_nic_index] << 1289 NES_QPCONTEXT_MISC2_NIC_INDEX_SHIFT); 1290 nesvnic->next_qp_nic_index++; 1291 if ((nesvnic->next_qp_nic_index > 3) || 1292 (nesvnic->qp_nic_index[nesvnic->next_qp_nic_index] == 0xf)) { 1293 nesvnic->next_qp_nic_index = 0; 1294 } 1295 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 1296 1297 nesqp->nesqp_context->pd_index_wscale |= cpu_to_le32((u32)nesqp->nespd->pd_id << 16); 1298 u64temp = (u64)nesqp->hwqp.q2_pbase; 1299 nesqp->nesqp_context->q2_addr_low = cpu_to_le32((u32)u64temp); 1300 nesqp->nesqp_context->q2_addr_high = cpu_to_le32((u32)(u64temp >> 32)); 1301 nesqp->nesqp_context->aeq_token_low = cpu_to_le32((u32)((unsigned long)(nesqp))); 1302 nesqp->nesqp_context->aeq_token_high = cpu_to_le32((u32)(upper_32_bits((unsigned long)(nesqp)))); 1303 nesqp->nesqp_context->ird_ord_sizes = cpu_to_le32(NES_QPCONTEXT_ORDIRD_ALSMM | 1304 NES_QPCONTEXT_ORDIRD_AAH | 1305 ((((u32)nesadapter->max_irrq_wr) << 1306 NES_QPCONTEXT_ORDIRD_IRDSIZE_SHIFT) & NES_QPCONTEXT_ORDIRD_IRDSIZE_MASK)); 1307 if (disable_mpa_crc) { 1308 nes_debug(NES_DBG_QP, "Disabling MPA crc checking due to module option.\n"); 1309 nesqp->nesqp_context->ird_ord_sizes |= cpu_to_le32(NES_QPCONTEXT_ORDIRD_RNMC); 1310 } 1311 1312 1313 /* Create the QP */ 1314 cqp_request = nes_get_cqp_request(nesdev); 1315 if (cqp_request == NULL) { 1316 nes_debug(NES_DBG_QP, "Failed to get a cqp_request\n"); 1317 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1318 nes_free_qp_mem(nesdev, nesqp,virt_wqs); 1319 kfree(nesqp->allocated_buffer); 1320 return ERR_PTR(-ENOMEM); 1321 } 1322 cqp_request->waiting = 1; 1323 cqp_wqe = &cqp_request->cqp_wqe; 1324 1325 if (!virt_wqs) { 1326 opcode = NES_CQP_CREATE_QP | NES_CQP_QP_TYPE_IWARP | 1327 NES_CQP_QP_IWARP_STATE_IDLE; 1328 } else { 1329 opcode = NES_CQP_CREATE_QP | NES_CQP_QP_TYPE_IWARP | NES_CQP_QP_VIRT_WQS | 1330 NES_CQP_QP_IWARP_STATE_IDLE; 1331 } 1332 opcode |= NES_CQP_QP_CQS_VALID; 1333 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 1334 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); 1335 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id); 1336 1337 u64temp = (u64)nesqp->nesqp_context_pbase; 1338 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp); 1339 1340 atomic_set(&cqp_request->refcount, 2); 1341 nes_post_cqp_request(nesdev, cqp_request); 1342 1343 /* Wait for CQP */ 1344 nes_debug(NES_DBG_QP, "Waiting for create iWARP QP%u to complete.\n", 1345 nesqp->hwqp.qp_id); 1346 ret = wait_event_timeout(cqp_request->waitq, 1347 (cqp_request->request_done != 0), NES_EVENT_TIMEOUT); 1348 nes_debug(NES_DBG_QP, "Create iwarp QP%u completed, wait_event_timeout ret=%u," 1349 " nesdev->cqp_head = %u, nesdev->cqp.sq_tail = %u," 1350 " CQP Major:Minor codes = 0x%04X:0x%04X.\n", 1351 nesqp->hwqp.qp_id, ret, nesdev->cqp.sq_head, nesdev->cqp.sq_tail, 1352 cqp_request->major_code, cqp_request->minor_code); 1353 if ((!ret) || (cqp_request->major_code)) { 1354 nes_put_cqp_request(nesdev, cqp_request); 1355 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1356 nes_free_qp_mem(nesdev, nesqp,virt_wqs); 1357 kfree(nesqp->allocated_buffer); 1358 if (!ret) { 1359 return ERR_PTR(-ETIME); 1360 } else { 1361 return ERR_PTR(-EIO); 1362 } 1363 } 1364 1365 nes_put_cqp_request(nesdev, cqp_request); 1366 1367 if (ibpd->uobject) { 1368 uresp.mmap_sq_db_index = nesqp->mmap_sq_db_index; 1369 uresp.mmap_rq_db_index = 0; 1370 uresp.actual_sq_size = sq_size; 1371 uresp.actual_rq_size = rq_size; 1372 uresp.qp_id = nesqp->hwqp.qp_id; 1373 uresp.nes_drv_opt = nes_drv_opt; 1374 if (ib_copy_to_udata(udata, &uresp, sizeof uresp)) { 1375 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1376 nes_free_qp_mem(nesdev, nesqp,virt_wqs); 1377 kfree(nesqp->allocated_buffer); 1378 return ERR_PTR(-EFAULT); 1379 } 1380 } 1381 1382 nes_debug(NES_DBG_QP, "QP%u structure located @%p.Size = %u.\n", 1383 nesqp->hwqp.qp_id, nesqp, (u32)sizeof(*nesqp)); 1384 spin_lock_init(&nesqp->lock); 1385 nes_add_ref(&nesqp->ibqp); 1386 break; 1387 default: 1388 nes_debug(NES_DBG_QP, "Invalid QP type: %d\n", init_attr->qp_type); 1389 return ERR_PTR(-EINVAL); 1390 } 1391 1392 nesqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR); 1393 init_timer(&nesqp->terminate_timer); 1394 nesqp->terminate_timer.function = nes_terminate_timeout; 1395 nesqp->terminate_timer.data = (unsigned long)nesqp; 1396 1397 /* update the QP table */ 1398 nesdev->nesadapter->qp_table[nesqp->hwqp.qp_id-NES_FIRST_QPN] = nesqp; 1399 nes_debug(NES_DBG_QP, "netdev refcnt=%u\n", 1400 netdev_refcnt_read(nesvnic->netdev)); 1401 1402 return &nesqp->ibqp; 1403} 1404 1405/** 1406 * nes_clean_cq 1407 */ 1408static void nes_clean_cq(struct nes_qp *nesqp, struct nes_cq *nescq) 1409{ 1410 u32 cq_head; 1411 u32 lo; 1412 u32 hi; 1413 u64 u64temp; 1414 unsigned long flags = 0; 1415 1416 spin_lock_irqsave(&nescq->lock, flags); 1417 1418 cq_head = nescq->hw_cq.cq_head; 1419 while (le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) { 1420 rmb(); 1421 lo = le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); 1422 hi = le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]); 1423 u64temp = (((u64)hi) << 32) | ((u64)lo); 1424 u64temp &= ~(NES_SW_CONTEXT_ALIGN-1); 1425 if (u64temp == (u64)(unsigned long)nesqp) { 1426 /* Zero the context value so cqe will be ignored */ 1427 nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] = 0; 1428 nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX] = 0; 1429 } 1430 1431 if (++cq_head >= nescq->hw_cq.cq_size) 1432 cq_head = 0; 1433 } 1434 1435 spin_unlock_irqrestore(&nescq->lock, flags); 1436} 1437 1438 1439/** 1440 * nes_destroy_qp 1441 */ 1442static int nes_destroy_qp(struct ib_qp *ibqp) 1443{ 1444 struct nes_qp *nesqp = to_nesqp(ibqp); 1445 struct nes_ucontext *nes_ucontext; 1446 struct ib_qp_attr attr; 1447 struct iw_cm_id *cm_id; 1448 struct iw_cm_event cm_event; 1449 int ret = 0; 1450 1451 atomic_inc(&sw_qps_destroyed); 1452 nesqp->destroyed = 1; 1453 1454 /* Blow away the connection if it exists. */ 1455 if (nesqp->ibqp_state >= IB_QPS_INIT && nesqp->ibqp_state <= IB_QPS_RTS) { 1456 /* if (nesqp->ibqp_state == IB_QPS_RTS) { */ 1457 attr.qp_state = IB_QPS_ERR; 1458 nes_modify_qp(&nesqp->ibqp, &attr, IB_QP_STATE, NULL); 1459 } 1460 1461 if (((nesqp->ibqp_state == IB_QPS_INIT) || 1462 (nesqp->ibqp_state == IB_QPS_RTR)) && (nesqp->cm_id)) { 1463 cm_id = nesqp->cm_id; 1464 cm_event.event = IW_CM_EVENT_CONNECT_REPLY; 1465 cm_event.status = -ETIMEDOUT; 1466 cm_event.local_addr = cm_id->local_addr; 1467 cm_event.remote_addr = cm_id->remote_addr; 1468 cm_event.private_data = NULL; 1469 cm_event.private_data_len = 0; 1470 1471 nes_debug(NES_DBG_QP, "Generating a CM Timeout Event for " 1472 "QP%u. cm_id = %p, refcount = %u. \n", 1473 nesqp->hwqp.qp_id, cm_id, atomic_read(&nesqp->refcount)); 1474 1475 cm_id->rem_ref(cm_id); 1476 ret = cm_id->event_handler(cm_id, &cm_event); 1477 if (ret) 1478 nes_debug(NES_DBG_QP, "OFA CM event_handler returned, ret=%d\n", ret); 1479 } 1480 1481 if (nesqp->user_mode) { 1482 if ((ibqp->uobject)&&(ibqp->uobject->context)) { 1483 nes_ucontext = to_nesucontext(ibqp->uobject->context); 1484 clear_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs); 1485 nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = NULL; 1486 if (nes_ucontext->first_free_wq > nesqp->mmap_sq_db_index) { 1487 nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index; 1488 } 1489 } 1490 if (nesqp->pbl_pbase && nesqp->sq_kmapped) { 1491 nesqp->sq_kmapped = 0; 1492 kunmap(nesqp->page); 1493 } 1494 } else { 1495 /* Clean any pending completions from the cq(s) */ 1496 if (nesqp->nesscq) 1497 nes_clean_cq(nesqp, nesqp->nesscq); 1498 1499 if ((nesqp->nesrcq) && (nesqp->nesrcq != nesqp->nesscq)) 1500 nes_clean_cq(nesqp, nesqp->nesrcq); 1501 } 1502 nes_rem_ref(&nesqp->ibqp); 1503 return 0; 1504} 1505 1506 1507/** 1508 * nes_create_cq 1509 */ 1510static struct ib_cq *nes_create_cq(struct ib_device *ibdev, 1511 const struct ib_cq_init_attr *attr, 1512 struct ib_ucontext *context, 1513 struct ib_udata *udata) 1514{ 1515 int entries = attr->cqe; 1516 u64 u64temp; 1517 struct nes_vnic *nesvnic = to_nesvnic(ibdev); 1518 struct nes_device *nesdev = nesvnic->nesdev; 1519 struct nes_adapter *nesadapter = nesdev->nesadapter; 1520 struct nes_cq *nescq; 1521 struct nes_ucontext *nes_ucontext = NULL; 1522 struct nes_cqp_request *cqp_request; 1523 void *mem = NULL; 1524 struct nes_hw_cqp_wqe *cqp_wqe; 1525 struct nes_pbl *nespbl = NULL; 1526 struct nes_create_cq_req req; 1527 struct nes_create_cq_resp resp; 1528 u32 cq_num = 0; 1529 u32 opcode = 0; 1530 u32 pbl_entries = 1; 1531 int err; 1532 unsigned long flags; 1533 int ret; 1534 1535 if (attr->flags) 1536 return ERR_PTR(-EINVAL); 1537 1538 if (entries > nesadapter->max_cqe) 1539 return ERR_PTR(-EINVAL); 1540 1541 err = nes_alloc_resource(nesadapter, nesadapter->allocated_cqs, 1542 nesadapter->max_cq, &cq_num, &nesadapter->next_cq, NES_RESOURCE_CQ); 1543 if (err) { 1544 return ERR_PTR(err); 1545 } 1546 1547 nescq = kzalloc(sizeof(struct nes_cq), GFP_KERNEL); 1548 if (!nescq) { 1549 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1550 nes_debug(NES_DBG_CQ, "Unable to allocate nes_cq struct\n"); 1551 return ERR_PTR(-ENOMEM); 1552 } 1553 1554 nescq->hw_cq.cq_size = max(entries + 1, 5); 1555 nescq->hw_cq.cq_number = cq_num; 1556 nescq->ibcq.cqe = nescq->hw_cq.cq_size - 1; 1557 1558 1559 if (context) { 1560 nes_ucontext = to_nesucontext(context); 1561 if (ib_copy_from_udata(&req, udata, sizeof (struct nes_create_cq_req))) { 1562 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1563 kfree(nescq); 1564 return ERR_PTR(-EFAULT); 1565 } 1566 nesvnic->mcrq_ucontext = nes_ucontext; 1567 nes_ucontext->mcrqf = req.mcrqf; 1568 if (nes_ucontext->mcrqf) { 1569 if (nes_ucontext->mcrqf & 0x80000000) 1570 nescq->hw_cq.cq_number = nesvnic->nic.qp_id + 28 + 2 * ((nes_ucontext->mcrqf & 0xf) - 1); 1571 else if (nes_ucontext->mcrqf & 0x40000000) 1572 nescq->hw_cq.cq_number = nes_ucontext->mcrqf & 0xffff; 1573 else 1574 nescq->hw_cq.cq_number = nesvnic->mcrq_qp_id + nes_ucontext->mcrqf-1; 1575 nescq->mcrqf = nes_ucontext->mcrqf; 1576 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1577 } 1578 nes_debug(NES_DBG_CQ, "CQ Virtual Address = %08lX, size = %u.\n", 1579 (unsigned long)req.user_cq_buffer, entries); 1580 err = 1; 1581 list_for_each_entry(nespbl, &nes_ucontext->cq_reg_mem_list, list) { 1582 if (nespbl->user_base == (unsigned long )req.user_cq_buffer) { 1583 list_del(&nespbl->list); 1584 err = 0; 1585 nes_debug(NES_DBG_CQ, "Found PBL for virtual CQ. nespbl=%p.\n", 1586 nespbl); 1587 break; 1588 } 1589 } 1590 if (err) { 1591 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1592 kfree(nescq); 1593 return ERR_PTR(-EFAULT); 1594 } 1595 1596 pbl_entries = nespbl->pbl_size >> 3; 1597 nescq->cq_mem_size = 0; 1598 } else { 1599 nescq->cq_mem_size = nescq->hw_cq.cq_size * sizeof(struct nes_hw_cqe); 1600 nes_debug(NES_DBG_CQ, "Attempting to allocate pci memory (%u entries, %u bytes) for CQ%u.\n", 1601 entries, nescq->cq_mem_size, nescq->hw_cq.cq_number); 1602 1603 /* allocate the physical buffer space */ 1604 mem = pci_zalloc_consistent(nesdev->pcidev, nescq->cq_mem_size, 1605 &nescq->hw_cq.cq_pbase); 1606 if (!mem) { 1607 printk(KERN_ERR PFX "Unable to allocate pci memory for cq\n"); 1608 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1609 kfree(nescq); 1610 return ERR_PTR(-ENOMEM); 1611 } 1612 1613 nescq->hw_cq.cq_vbase = mem; 1614 nescq->hw_cq.cq_head = 0; 1615 nes_debug(NES_DBG_CQ, "CQ%u virtual address @ %p, phys = 0x%08X\n", 1616 nescq->hw_cq.cq_number, nescq->hw_cq.cq_vbase, 1617 (u32)nescq->hw_cq.cq_pbase); 1618 } 1619 1620 nescq->hw_cq.ce_handler = nes_iwarp_ce_handler; 1621 spin_lock_init(&nescq->lock); 1622 1623 /* send CreateCQ request to CQP */ 1624 cqp_request = nes_get_cqp_request(nesdev); 1625 if (cqp_request == NULL) { 1626 nes_debug(NES_DBG_CQ, "Failed to get a cqp_request.\n"); 1627 if (!context) 1628 pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, 1629 nescq->hw_cq.cq_pbase); 1630 else { 1631 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, 1632 nespbl->pbl_vbase, nespbl->pbl_pbase); 1633 kfree(nespbl); 1634 } 1635 1636 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1637 kfree(nescq); 1638 return ERR_PTR(-ENOMEM); 1639 } 1640 cqp_request->waiting = 1; 1641 cqp_wqe = &cqp_request->cqp_wqe; 1642 1643 opcode = NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID | 1644 NES_CQP_CQ_CHK_OVERFLOW | 1645 NES_CQP_CQ_CEQE_MASK | ((u32)nescq->hw_cq.cq_size << 16); 1646 1647 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 1648 1649 if (pbl_entries != 1) { 1650 if (pbl_entries > 32) { 1651 /* use 4k pbl */ 1652 nes_debug(NES_DBG_CQ, "pbl_entries=%u, use a 4k PBL\n", pbl_entries); 1653 if (nesadapter->free_4kpbl == 0) { 1654 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1655 nes_free_cqp_request(nesdev, cqp_request); 1656 if (!context) 1657 pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, 1658 nescq->hw_cq.cq_pbase); 1659 else { 1660 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, 1661 nespbl->pbl_vbase, nespbl->pbl_pbase); 1662 kfree(nespbl); 1663 } 1664 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1665 kfree(nescq); 1666 return ERR_PTR(-ENOMEM); 1667 } else { 1668 opcode |= (NES_CQP_CQ_VIRT | NES_CQP_CQ_4KB_CHUNK); 1669 nescq->virtual_cq = 2; 1670 nesadapter->free_4kpbl--; 1671 } 1672 } else { 1673 /* use 256 byte pbl */ 1674 nes_debug(NES_DBG_CQ, "pbl_entries=%u, use a 256 byte PBL\n", pbl_entries); 1675 if (nesadapter->free_256pbl == 0) { 1676 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1677 nes_free_cqp_request(nesdev, cqp_request); 1678 if (!context) 1679 pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, 1680 nescq->hw_cq.cq_pbase); 1681 else { 1682 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, 1683 nespbl->pbl_vbase, nespbl->pbl_pbase); 1684 kfree(nespbl); 1685 } 1686 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1687 kfree(nescq); 1688 return ERR_PTR(-ENOMEM); 1689 } else { 1690 opcode |= NES_CQP_CQ_VIRT; 1691 nescq->virtual_cq = 1; 1692 nesadapter->free_256pbl--; 1693 } 1694 } 1695 } 1696 1697 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1698 1699 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 1700 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); 1701 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, 1702 (nescq->hw_cq.cq_number | ((u32)nesdev->ceq_index << 16))); 1703 1704 if (context) { 1705 if (pbl_entries != 1) 1706 u64temp = (u64)nespbl->pbl_pbase; 1707 else 1708 u64temp = le64_to_cpu(nespbl->pbl_vbase[0]); 1709 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX, 1710 nes_ucontext->mmap_db_index[0]); 1711 } else { 1712 u64temp = (u64)nescq->hw_cq.cq_pbase; 1713 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0; 1714 } 1715 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); 1716 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0; 1717 u64temp = (u64)(unsigned long)&nescq->hw_cq; 1718 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = 1719 cpu_to_le32((u32)(u64temp >> 1)); 1720 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 1721 cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF); 1722 1723 atomic_set(&cqp_request->refcount, 2); 1724 nes_post_cqp_request(nesdev, cqp_request); 1725 1726 /* Wait for CQP */ 1727 nes_debug(NES_DBG_CQ, "Waiting for create iWARP CQ%u to complete.\n", 1728 nescq->hw_cq.cq_number); 1729 ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), 1730 NES_EVENT_TIMEOUT * 2); 1731 nes_debug(NES_DBG_CQ, "Create iWARP CQ%u completed, wait_event_timeout ret = %d.\n", 1732 nescq->hw_cq.cq_number, ret); 1733 if ((!ret) || (cqp_request->major_code)) { 1734 nes_put_cqp_request(nesdev, cqp_request); 1735 if (!context) 1736 pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, 1737 nescq->hw_cq.cq_pbase); 1738 else { 1739 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, 1740 nespbl->pbl_vbase, nespbl->pbl_pbase); 1741 kfree(nespbl); 1742 } 1743 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1744 kfree(nescq); 1745 return ERR_PTR(-EIO); 1746 } 1747 nes_put_cqp_request(nesdev, cqp_request); 1748 1749 if (context) { 1750 /* free the nespbl */ 1751 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, 1752 nespbl->pbl_pbase); 1753 kfree(nespbl); 1754 resp.cq_id = nescq->hw_cq.cq_number; 1755 resp.cq_size = nescq->hw_cq.cq_size; 1756 resp.mmap_db_index = 0; 1757 if (ib_copy_to_udata(udata, &resp, sizeof resp - sizeof resp.reserved)) { 1758 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1759 kfree(nescq); 1760 return ERR_PTR(-EFAULT); 1761 } 1762 } 1763 1764 return &nescq->ibcq; 1765} 1766 1767 1768/** 1769 * nes_destroy_cq 1770 */ 1771static int nes_destroy_cq(struct ib_cq *ib_cq) 1772{ 1773 struct nes_cq *nescq; 1774 struct nes_device *nesdev; 1775 struct nes_vnic *nesvnic; 1776 struct nes_adapter *nesadapter; 1777 struct nes_hw_cqp_wqe *cqp_wqe; 1778 struct nes_cqp_request *cqp_request; 1779 unsigned long flags; 1780 u32 opcode = 0; 1781 int ret; 1782 1783 if (ib_cq == NULL) 1784 return 0; 1785 1786 nescq = to_nescq(ib_cq); 1787 nesvnic = to_nesvnic(ib_cq->device); 1788 nesdev = nesvnic->nesdev; 1789 nesadapter = nesdev->nesadapter; 1790 1791 nes_debug(NES_DBG_CQ, "Destroy CQ%u\n", nescq->hw_cq.cq_number); 1792 1793 /* Send DestroyCQ request to CQP */ 1794 cqp_request = nes_get_cqp_request(nesdev); 1795 if (cqp_request == NULL) { 1796 nes_debug(NES_DBG_CQ, "Failed to get a cqp_request.\n"); 1797 return -ENOMEM; 1798 } 1799 cqp_request->waiting = 1; 1800 cqp_wqe = &cqp_request->cqp_wqe; 1801 opcode = NES_CQP_DESTROY_CQ | (nescq->hw_cq.cq_size << 16); 1802 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 1803 if (nescq->virtual_cq == 1) { 1804 nesadapter->free_256pbl++; 1805 if (nesadapter->free_256pbl > nesadapter->max_256pbl) { 1806 printk(KERN_ERR PFX "%s: free 256B PBLs(%u) has exceeded the max(%u)\n", 1807 __func__, nesadapter->free_256pbl, nesadapter->max_256pbl); 1808 } 1809 } else if (nescq->virtual_cq == 2) { 1810 nesadapter->free_4kpbl++; 1811 if (nesadapter->free_4kpbl > nesadapter->max_4kpbl) { 1812 printk(KERN_ERR PFX "%s: free 4K PBLs(%u) has exceeded the max(%u)\n", 1813 __func__, nesadapter->free_4kpbl, nesadapter->max_4kpbl); 1814 } 1815 opcode |= NES_CQP_CQ_4KB_CHUNK; 1816 } 1817 1818 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1819 1820 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 1821 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); 1822 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, 1823 (nescq->hw_cq.cq_number | ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 16))); 1824 if (!nescq->mcrqf) 1825 nes_free_resource(nesadapter, nesadapter->allocated_cqs, nescq->hw_cq.cq_number); 1826 1827 atomic_set(&cqp_request->refcount, 2); 1828 nes_post_cqp_request(nesdev, cqp_request); 1829 1830 /* Wait for CQP */ 1831 nes_debug(NES_DBG_CQ, "Waiting for destroy iWARP CQ%u to complete.\n", 1832 nescq->hw_cq.cq_number); 1833 ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), 1834 NES_EVENT_TIMEOUT); 1835 nes_debug(NES_DBG_CQ, "Destroy iWARP CQ%u completed, wait_event_timeout ret = %u," 1836 " CQP Major:Minor codes = 0x%04X:0x%04X.\n", 1837 nescq->hw_cq.cq_number, ret, cqp_request->major_code, 1838 cqp_request->minor_code); 1839 if (!ret) { 1840 nes_debug(NES_DBG_CQ, "iWARP CQ%u destroy timeout expired\n", 1841 nescq->hw_cq.cq_number); 1842 ret = -ETIME; 1843 } else if (cqp_request->major_code) { 1844 nes_debug(NES_DBG_CQ, "iWARP CQ%u destroy failed\n", 1845 nescq->hw_cq.cq_number); 1846 ret = -EIO; 1847 } else { 1848 ret = 0; 1849 } 1850 nes_put_cqp_request(nesdev, cqp_request); 1851 1852 if (nescq->cq_mem_size) 1853 pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, 1854 nescq->hw_cq.cq_vbase, nescq->hw_cq.cq_pbase); 1855 kfree(nescq); 1856 1857 return ret; 1858} 1859 1860/** 1861 * root_256 1862 */ 1863static u32 root_256(struct nes_device *nesdev, 1864 struct nes_root_vpbl *root_vpbl, 1865 struct nes_root_vpbl *new_root, 1866 u16 pbl_count_4k) 1867{ 1868 u64 leaf_pbl; 1869 int i, j, k; 1870 1871 if (pbl_count_4k == 1) { 1872 new_root->pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 1873 512, &new_root->pbl_pbase); 1874 1875 if (new_root->pbl_vbase == NULL) 1876 return 0; 1877 1878 leaf_pbl = (u64)root_vpbl->pbl_pbase; 1879 for (i = 0; i < 16; i++) { 1880 new_root->pbl_vbase[i].pa_low = 1881 cpu_to_le32((u32)leaf_pbl); 1882 new_root->pbl_vbase[i].pa_high = 1883 cpu_to_le32((u32)((((u64)leaf_pbl) >> 32))); 1884 leaf_pbl += 256; 1885 } 1886 } else { 1887 for (i = 3; i >= 0; i--) { 1888 j = i * 16; 1889 root_vpbl->pbl_vbase[j] = root_vpbl->pbl_vbase[i]; 1890 leaf_pbl = le32_to_cpu(root_vpbl->pbl_vbase[j].pa_low) + 1891 (((u64)le32_to_cpu(root_vpbl->pbl_vbase[j].pa_high)) 1892 << 32); 1893 for (k = 1; k < 16; k++) { 1894 leaf_pbl += 256; 1895 root_vpbl->pbl_vbase[j + k].pa_low = 1896 cpu_to_le32((u32)leaf_pbl); 1897 root_vpbl->pbl_vbase[j + k].pa_high = 1898 cpu_to_le32((u32)((((u64)leaf_pbl) >> 32))); 1899 } 1900 } 1901 } 1902 1903 return 1; 1904} 1905 1906 1907/** 1908 * nes_reg_mr 1909 */ 1910static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd, 1911 u32 stag, u64 region_length, struct nes_root_vpbl *root_vpbl, 1912 dma_addr_t single_buffer, u16 pbl_count_4k, 1913 u16 residual_page_count_4k, int acc, u64 *iova_start, 1914 u16 *actual_pbl_cnt, u8 *used_4k_pbls) 1915{ 1916 struct nes_hw_cqp_wqe *cqp_wqe; 1917 struct nes_cqp_request *cqp_request; 1918 unsigned long flags; 1919 int ret; 1920 struct nes_adapter *nesadapter = nesdev->nesadapter; 1921 uint pg_cnt = 0; 1922 u16 pbl_count_256 = 0; 1923 u16 pbl_count = 0; 1924 u8 use_256_pbls = 0; 1925 u8 use_4k_pbls = 0; 1926 u16 use_two_level = (pbl_count_4k > 1) ? 1 : 0; 1927 struct nes_root_vpbl new_root = { 0, NULL, NULL }; 1928 u32 opcode = 0; 1929 u16 major_code; 1930 1931 /* Register the region with the adapter */ 1932 cqp_request = nes_get_cqp_request(nesdev); 1933 if (cqp_request == NULL) { 1934 nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); 1935 return -ENOMEM; 1936 } 1937 cqp_request->waiting = 1; 1938 cqp_wqe = &cqp_request->cqp_wqe; 1939 1940 if (pbl_count_4k) { 1941 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 1942 1943 pg_cnt = ((pbl_count_4k - 1) * 512) + residual_page_count_4k; 1944 pbl_count_256 = (pg_cnt + 31) / 32; 1945 if (pg_cnt <= 32) { 1946 if (pbl_count_256 <= nesadapter->free_256pbl) 1947 use_256_pbls = 1; 1948 else if (pbl_count_4k <= nesadapter->free_4kpbl) 1949 use_4k_pbls = 1; 1950 } else if (pg_cnt <= 2048) { 1951 if (((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) && 1952 (nesadapter->free_4kpbl > (nesadapter->max_4kpbl >> 1))) { 1953 use_4k_pbls = 1; 1954 } else if ((pbl_count_256 + 1) <= nesadapter->free_256pbl) { 1955 use_256_pbls = 1; 1956 use_two_level = 1; 1957 } else if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) { 1958 use_4k_pbls = 1; 1959 } 1960 } else { 1961 if ((pbl_count_4k + 1) <= nesadapter->free_4kpbl) 1962 use_4k_pbls = 1; 1963 } 1964 1965 if (use_256_pbls) { 1966 pbl_count = pbl_count_256; 1967 nesadapter->free_256pbl -= pbl_count + use_two_level; 1968 } else if (use_4k_pbls) { 1969 pbl_count = pbl_count_4k; 1970 nesadapter->free_4kpbl -= pbl_count + use_two_level; 1971 } else { 1972 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1973 nes_debug(NES_DBG_MR, "Out of Pbls\n"); 1974 nes_free_cqp_request(nesdev, cqp_request); 1975 return -ENOMEM; 1976 } 1977 1978 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1979 } 1980 1981 if (use_256_pbls && use_two_level) { 1982 if (root_256(nesdev, root_vpbl, &new_root, pbl_count_4k) == 1) { 1983 if (new_root.pbl_pbase != 0) 1984 root_vpbl = &new_root; 1985 } else { 1986 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 1987 nesadapter->free_256pbl += pbl_count_256 + use_two_level; 1988 use_256_pbls = 0; 1989 1990 if (pbl_count_4k == 1) 1991 use_two_level = 0; 1992 pbl_count = pbl_count_4k; 1993 1994 if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) { 1995 nesadapter->free_4kpbl -= pbl_count + use_two_level; 1996 use_4k_pbls = 1; 1997 } 1998 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1999 2000 if (use_4k_pbls == 0) 2001 return -ENOMEM; 2002 } 2003 } 2004 2005 opcode = NES_CQP_REGISTER_STAG | NES_CQP_STAG_RIGHTS_LOCAL_READ | 2006 NES_CQP_STAG_VA_TO | NES_CQP_STAG_MR; 2007 if (acc & IB_ACCESS_LOCAL_WRITE) 2008 opcode |= NES_CQP_STAG_RIGHTS_LOCAL_WRITE; 2009 if (acc & IB_ACCESS_REMOTE_WRITE) 2010 opcode |= NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_REM_ACC_EN; 2011 if (acc & IB_ACCESS_REMOTE_READ) 2012 opcode |= NES_CQP_STAG_RIGHTS_REMOTE_READ | NES_CQP_STAG_REM_ACC_EN; 2013 if (acc & IB_ACCESS_MW_BIND) 2014 opcode |= NES_CQP_STAG_RIGHTS_WINDOW_BIND | NES_CQP_STAG_REM_ACC_EN; 2015 2016 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 2017 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); 2018 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_VA_LOW_IDX, *iova_start); 2019 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_LOW_IDX, region_length); 2020 2021 cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] = 2022 cpu_to_le32((u32)(region_length >> 8) & 0xff000000); 2023 cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] |= 2024 cpu_to_le32(nespd->pd_id & 0x00007fff); 2025 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag); 2026 2027 if (pbl_count == 0) { 2028 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, single_buffer); 2029 } else { 2030 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, root_vpbl->pbl_pbase); 2031 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, pbl_count); 2032 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (pg_cnt * 8)); 2033 2034 if (use_4k_pbls) 2035 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE); 2036 } 2037 barrier(); 2038 2039 atomic_set(&cqp_request->refcount, 2); 2040 nes_post_cqp_request(nesdev, cqp_request); 2041 2042 /* Wait for CQP */ 2043 ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), 2044 NES_EVENT_TIMEOUT); 2045 nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u," 2046 " CQP Major:Minor codes = 0x%04X:0x%04X.\n", 2047 stag, ret, cqp_request->major_code, cqp_request->minor_code); 2048 major_code = cqp_request->major_code; 2049 nes_put_cqp_request(nesdev, cqp_request); 2050 2051 if ((!ret || major_code) && pbl_count != 0) { 2052 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 2053 if (use_256_pbls) 2054 nesadapter->free_256pbl += pbl_count + use_two_level; 2055 else if (use_4k_pbls) 2056 nesadapter->free_4kpbl += pbl_count + use_two_level; 2057 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 2058 } 2059 if (new_root.pbl_pbase) 2060 pci_free_consistent(nesdev->pcidev, 512, new_root.pbl_vbase, 2061 new_root.pbl_pbase); 2062 2063 if (!ret) 2064 return -ETIME; 2065 else if (major_code) 2066 return -EIO; 2067 2068 *actual_pbl_cnt = pbl_count + use_two_level; 2069 *used_4k_pbls = use_4k_pbls; 2070 return 0; 2071} 2072 2073 2074/** 2075 * nes_reg_phys_mr 2076 */ 2077static struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd, 2078 struct ib_phys_buf *buffer_list, int num_phys_buf, int acc, 2079 u64 * iova_start) 2080{ 2081 u64 region_length; 2082 struct nes_pd *nespd = to_nespd(ib_pd); 2083 struct nes_vnic *nesvnic = to_nesvnic(ib_pd->device); 2084 struct nes_device *nesdev = nesvnic->nesdev; 2085 struct nes_adapter *nesadapter = nesdev->nesadapter; 2086 struct nes_mr *nesmr; 2087 struct ib_mr *ibmr; 2088 struct nes_vpbl vpbl; 2089 struct nes_root_vpbl root_vpbl; 2090 u32 stag; 2091 u32 i; 2092 unsigned long mask; 2093 u32 stag_index = 0; 2094 u32 next_stag_index = 0; 2095 u32 driver_key = 0; 2096 u32 root_pbl_index = 0; 2097 u32 cur_pbl_index = 0; 2098 int err = 0; 2099 int ret = 0; 2100 u16 pbl_count = 0; 2101 u8 single_page = 1; 2102 u8 stag_key = 0; 2103 2104 region_length = 0; 2105 vpbl.pbl_vbase = NULL; 2106 root_vpbl.pbl_vbase = NULL; 2107 root_vpbl.pbl_pbase = 0; 2108 2109 get_random_bytes(&next_stag_index, sizeof(next_stag_index)); 2110 stag_key = (u8)next_stag_index; 2111 2112 driver_key = 0; 2113 2114 next_stag_index >>= 8; 2115 next_stag_index %= nesadapter->max_mr; 2116 if (num_phys_buf > (1024*512)) { 2117 return ERR_PTR(-E2BIG); 2118 } 2119 2120 if ((buffer_list[0].addr ^ *iova_start) & ~PAGE_MASK) 2121 return ERR_PTR(-EINVAL); 2122 2123 err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, nesadapter->max_mr, 2124 &stag_index, &next_stag_index, NES_RESOURCE_PHYS_MR); 2125 if (err) { 2126 return ERR_PTR(err); 2127 } 2128 2129 nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); 2130 if (!nesmr) { 2131 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2132 return ERR_PTR(-ENOMEM); 2133 } 2134 2135 for (i = 0; i < num_phys_buf; i++) { 2136 2137 if ((i & 0x01FF) == 0) { 2138 if (root_pbl_index == 1) { 2139 /* Allocate the root PBL */ 2140 root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 8192, 2141 &root_vpbl.pbl_pbase); 2142 nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n", 2143 root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase); 2144 if (!root_vpbl.pbl_vbase) { 2145 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, 2146 vpbl.pbl_pbase); 2147 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2148 kfree(nesmr); 2149 return ERR_PTR(-ENOMEM); 2150 } 2151 root_vpbl.leaf_vpbl = kzalloc(sizeof(*root_vpbl.leaf_vpbl)*1024, GFP_KERNEL); 2152 if (!root_vpbl.leaf_vpbl) { 2153 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase, 2154 root_vpbl.pbl_pbase); 2155 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, 2156 vpbl.pbl_pbase); 2157 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2158 kfree(nesmr); 2159 return ERR_PTR(-ENOMEM); 2160 } 2161 root_vpbl.pbl_vbase[0].pa_low = cpu_to_le32((u32)vpbl.pbl_pbase); 2162 root_vpbl.pbl_vbase[0].pa_high = 2163 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32))); 2164 root_vpbl.leaf_vpbl[0] = vpbl; 2165 } 2166 /* Allocate a 4K buffer for the PBL */ 2167 vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096, 2168 &vpbl.pbl_pbase); 2169 nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%016lX\n", 2170 vpbl.pbl_vbase, (unsigned long)vpbl.pbl_pbase); 2171 if (!vpbl.pbl_vbase) { 2172 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2173 ibmr = ERR_PTR(-ENOMEM); 2174 kfree(nesmr); 2175 goto reg_phys_err; 2176 } 2177 /* Fill in the root table */ 2178 if (1 <= root_pbl_index) { 2179 root_vpbl.pbl_vbase[root_pbl_index].pa_low = 2180 cpu_to_le32((u32)vpbl.pbl_pbase); 2181 root_vpbl.pbl_vbase[root_pbl_index].pa_high = 2182 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32))); 2183 root_vpbl.leaf_vpbl[root_pbl_index] = vpbl; 2184 } 2185 root_pbl_index++; 2186 cur_pbl_index = 0; 2187 } 2188 2189 mask = !buffer_list[i].size; 2190 if (i != 0) 2191 mask |= buffer_list[i].addr; 2192 if (i != num_phys_buf - 1) 2193 mask |= buffer_list[i].addr + buffer_list[i].size; 2194 2195 if (mask & ~PAGE_MASK) { 2196 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2197 nes_debug(NES_DBG_MR, "Invalid buffer addr or size\n"); 2198 ibmr = ERR_PTR(-EINVAL); 2199 kfree(nesmr); 2200 goto reg_phys_err; 2201 } 2202 2203 region_length += buffer_list[i].size; 2204 if ((i != 0) && (single_page)) { 2205 if ((buffer_list[i-1].addr+PAGE_SIZE) != buffer_list[i].addr) 2206 single_page = 0; 2207 } 2208 vpbl.pbl_vbase[cur_pbl_index].pa_low = cpu_to_le32((u32)buffer_list[i].addr & PAGE_MASK); 2209 vpbl.pbl_vbase[cur_pbl_index++].pa_high = 2210 cpu_to_le32((u32)((((u64)buffer_list[i].addr) >> 32))); 2211 } 2212 2213 stag = stag_index << 8; 2214 stag |= driver_key; 2215 stag += (u32)stag_key; 2216 2217 nes_debug(NES_DBG_MR, "Registering STag 0x%08X, VA = 0x%016lX," 2218 " length = 0x%016lX, index = 0x%08X\n", 2219 stag, (unsigned long)*iova_start, (unsigned long)region_length, stag_index); 2220 2221 /* Make the leaf PBL the root if only one PBL */ 2222 if (root_pbl_index == 1) { 2223 root_vpbl.pbl_pbase = vpbl.pbl_pbase; 2224 } 2225 2226 if (single_page) { 2227 pbl_count = 0; 2228 } else { 2229 pbl_count = root_pbl_index; 2230 } 2231 ret = nes_reg_mr(nesdev, nespd, stag, region_length, &root_vpbl, 2232 buffer_list[0].addr, pbl_count, (u16)cur_pbl_index, acc, iova_start, 2233 &nesmr->pbls_used, &nesmr->pbl_4k); 2234 2235 if (ret == 0) { 2236 nesmr->ibmr.rkey = stag; 2237 nesmr->ibmr.lkey = stag; 2238 nesmr->mode = IWNES_MEMREG_TYPE_MEM; 2239 ibmr = &nesmr->ibmr; 2240 } else { 2241 kfree(nesmr); 2242 ibmr = ERR_PTR(-ENOMEM); 2243 } 2244 2245 reg_phys_err: 2246 /* free the resources */ 2247 if (root_pbl_index == 1) { 2248 /* single PBL case */ 2249 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, vpbl.pbl_pbase); 2250 } else { 2251 for (i=0; i<root_pbl_index; i++) { 2252 pci_free_consistent(nesdev->pcidev, 4096, root_vpbl.leaf_vpbl[i].pbl_vbase, 2253 root_vpbl.leaf_vpbl[i].pbl_pbase); 2254 } 2255 kfree(root_vpbl.leaf_vpbl); 2256 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase, 2257 root_vpbl.pbl_pbase); 2258 } 2259 2260 return ibmr; 2261} 2262 2263 2264/** 2265 * nes_get_dma_mr 2266 */ 2267static struct ib_mr *nes_get_dma_mr(struct ib_pd *pd, int acc) 2268{ 2269 struct ib_phys_buf bl; 2270 u64 kva = 0; 2271 2272 nes_debug(NES_DBG_MR, "\n"); 2273 2274 bl.size = (u64)0xffffffffffULL; 2275 bl.addr = 0; 2276 return nes_reg_phys_mr(pd, &bl, 1, acc, &kva); 2277} 2278 2279 2280/** 2281 * nes_reg_user_mr 2282 */ 2283static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 2284 u64 virt, int acc, struct ib_udata *udata) 2285{ 2286 u64 iova_start; 2287 __le64 *pbl; 2288 u64 region_length; 2289 dma_addr_t last_dma_addr = 0; 2290 dma_addr_t first_dma_addr = 0; 2291 struct nes_pd *nespd = to_nespd(pd); 2292 struct nes_vnic *nesvnic = to_nesvnic(pd->device); 2293 struct nes_device *nesdev = nesvnic->nesdev; 2294 struct nes_adapter *nesadapter = nesdev->nesadapter; 2295 struct ib_mr *ibmr = ERR_PTR(-EINVAL); 2296 struct scatterlist *sg; 2297 struct nes_ucontext *nes_ucontext; 2298 struct nes_pbl *nespbl; 2299 struct nes_mr *nesmr; 2300 struct ib_umem *region; 2301 struct nes_mem_reg_req req; 2302 struct nes_vpbl vpbl; 2303 struct nes_root_vpbl root_vpbl; 2304 int entry, page_index; 2305 int page_count = 0; 2306 int err, pbl_depth = 0; 2307 int chunk_pages; 2308 int ret; 2309 u32 stag; 2310 u32 stag_index = 0; 2311 u32 next_stag_index; 2312 u32 driver_key; 2313 u32 root_pbl_index = 0; 2314 u32 cur_pbl_index = 0; 2315 u32 skip_pages; 2316 u16 pbl_count; 2317 u8 single_page = 1; 2318 u8 stag_key; 2319 int first_page = 1; 2320 2321 region = ib_umem_get(pd->uobject->context, start, length, acc, 0); 2322 if (IS_ERR(region)) { 2323 return (struct ib_mr *)region; 2324 } 2325 2326 nes_debug(NES_DBG_MR, "User base = 0x%lX, Virt base = 0x%lX, length = %u," 2327 " offset = %u, page size = %u.\n", 2328 (unsigned long int)start, (unsigned long int)virt, (u32)length, 2329 ib_umem_offset(region), region->page_size); 2330 2331 skip_pages = ((u32)ib_umem_offset(region)) >> 12; 2332 2333 if (ib_copy_from_udata(&req, udata, sizeof(req))) { 2334 ib_umem_release(region); 2335 return ERR_PTR(-EFAULT); 2336 } 2337 nes_debug(NES_DBG_MR, "Memory Registration type = %08X.\n", req.reg_type); 2338 2339 switch (req.reg_type) { 2340 case IWNES_MEMREG_TYPE_MEM: 2341 pbl_depth = 0; 2342 region_length = 0; 2343 vpbl.pbl_vbase = NULL; 2344 root_vpbl.pbl_vbase = NULL; 2345 root_vpbl.pbl_pbase = 0; 2346 2347 get_random_bytes(&next_stag_index, sizeof(next_stag_index)); 2348 stag_key = (u8)next_stag_index; 2349 2350 driver_key = next_stag_index & 0x70000000; 2351 2352 next_stag_index >>= 8; 2353 next_stag_index %= nesadapter->max_mr; 2354 2355 err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, 2356 nesadapter->max_mr, &stag_index, &next_stag_index, NES_RESOURCE_USER_MR); 2357 if (err) { 2358 ib_umem_release(region); 2359 return ERR_PTR(err); 2360 } 2361 2362 nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); 2363 if (!nesmr) { 2364 ib_umem_release(region); 2365 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2366 return ERR_PTR(-ENOMEM); 2367 } 2368 nesmr->region = region; 2369 2370 for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) { 2371 if (sg_dma_address(sg) & ~PAGE_MASK) { 2372 ib_umem_release(region); 2373 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2374 nes_debug(NES_DBG_MR, "Unaligned Memory Buffer: 0x%x\n", 2375 (unsigned int) sg_dma_address(sg)); 2376 ibmr = ERR_PTR(-EINVAL); 2377 kfree(nesmr); 2378 goto reg_user_mr_err; 2379 } 2380 2381 if (!sg_dma_len(sg)) { 2382 ib_umem_release(region); 2383 nes_free_resource(nesadapter, nesadapter->allocated_mrs, 2384 stag_index); 2385 nes_debug(NES_DBG_MR, "Invalid Buffer Size\n"); 2386 ibmr = ERR_PTR(-EINVAL); 2387 kfree(nesmr); 2388 goto reg_user_mr_err; 2389 } 2390 2391 region_length += sg_dma_len(sg); 2392 chunk_pages = sg_dma_len(sg) >> 12; 2393 region_length -= skip_pages << 12; 2394 for (page_index = skip_pages; page_index < chunk_pages; page_index++) { 2395 skip_pages = 0; 2396 if ((page_count != 0) && (page_count << 12) - (ib_umem_offset(region) & (4096 - 1)) >= region->length) 2397 goto enough_pages; 2398 if ((page_count&0x01FF) == 0) { 2399 if (page_count >= 1024 * 512) { 2400 ib_umem_release(region); 2401 nes_free_resource(nesadapter, 2402 nesadapter->allocated_mrs, stag_index); 2403 kfree(nesmr); 2404 ibmr = ERR_PTR(-E2BIG); 2405 goto reg_user_mr_err; 2406 } 2407 if (root_pbl_index == 1) { 2408 root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 2409 8192, &root_vpbl.pbl_pbase); 2410 nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n", 2411 root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase); 2412 if (!root_vpbl.pbl_vbase) { 2413 ib_umem_release(region); 2414 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, 2415 vpbl.pbl_pbase); 2416 nes_free_resource(nesadapter, nesadapter->allocated_mrs, 2417 stag_index); 2418 kfree(nesmr); 2419 ibmr = ERR_PTR(-ENOMEM); 2420 goto reg_user_mr_err; 2421 } 2422 root_vpbl.leaf_vpbl = kzalloc(sizeof(*root_vpbl.leaf_vpbl)*1024, 2423 GFP_KERNEL); 2424 if (!root_vpbl.leaf_vpbl) { 2425 ib_umem_release(region); 2426 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase, 2427 root_vpbl.pbl_pbase); 2428 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, 2429 vpbl.pbl_pbase); 2430 nes_free_resource(nesadapter, nesadapter->allocated_mrs, 2431 stag_index); 2432 kfree(nesmr); 2433 ibmr = ERR_PTR(-ENOMEM); 2434 goto reg_user_mr_err; 2435 } 2436 root_vpbl.pbl_vbase[0].pa_low = 2437 cpu_to_le32((u32)vpbl.pbl_pbase); 2438 root_vpbl.pbl_vbase[0].pa_high = 2439 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32))); 2440 root_vpbl.leaf_vpbl[0] = vpbl; 2441 } 2442 vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096, 2443 &vpbl.pbl_pbase); 2444 nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%08X\n", 2445 vpbl.pbl_vbase, (unsigned int)vpbl.pbl_pbase); 2446 if (!vpbl.pbl_vbase) { 2447 ib_umem_release(region); 2448 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2449 ibmr = ERR_PTR(-ENOMEM); 2450 kfree(nesmr); 2451 goto reg_user_mr_err; 2452 } 2453 if (1 <= root_pbl_index) { 2454 root_vpbl.pbl_vbase[root_pbl_index].pa_low = 2455 cpu_to_le32((u32)vpbl.pbl_pbase); 2456 root_vpbl.pbl_vbase[root_pbl_index].pa_high = 2457 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase)>>32))); 2458 root_vpbl.leaf_vpbl[root_pbl_index] = vpbl; 2459 } 2460 root_pbl_index++; 2461 cur_pbl_index = 0; 2462 } 2463 if (single_page) { 2464 if (page_count != 0) { 2465 if ((last_dma_addr+4096) != 2466 (sg_dma_address(sg)+ 2467 (page_index*4096))) 2468 single_page = 0; 2469 last_dma_addr = sg_dma_address(sg)+ 2470 (page_index*4096); 2471 } else { 2472 first_dma_addr = sg_dma_address(sg)+ 2473 (page_index*4096); 2474 last_dma_addr = first_dma_addr; 2475 } 2476 } 2477 2478 vpbl.pbl_vbase[cur_pbl_index].pa_low = 2479 cpu_to_le32((u32)(sg_dma_address(sg)+ 2480 (page_index*4096))); 2481 vpbl.pbl_vbase[cur_pbl_index].pa_high = 2482 cpu_to_le32((u32)((((u64)(sg_dma_address(sg)+ 2483 (page_index*4096))) >> 32))); 2484 cur_pbl_index++; 2485 page_count++; 2486 } 2487 } 2488 2489 enough_pages: 2490 nes_debug(NES_DBG_MR, "calculating stag, stag_index=0x%08x, driver_key=0x%08x," 2491 " stag_key=0x%08x\n", 2492 stag_index, driver_key, stag_key); 2493 stag = stag_index << 8; 2494 stag |= driver_key; 2495 stag += (u32)stag_key; 2496 2497 iova_start = virt; 2498 /* Make the leaf PBL the root if only one PBL */ 2499 if (root_pbl_index == 1) { 2500 root_vpbl.pbl_pbase = vpbl.pbl_pbase; 2501 } 2502 2503 if (single_page) { 2504 pbl_count = 0; 2505 } else { 2506 pbl_count = root_pbl_index; 2507 first_dma_addr = 0; 2508 } 2509 nes_debug(NES_DBG_MR, "Registering STag 0x%08X, VA = 0x%08X, length = 0x%08X," 2510 " index = 0x%08X, region->length=0x%08llx, pbl_count = %u\n", 2511 stag, (unsigned int)iova_start, 2512 (unsigned int)region_length, stag_index, 2513 (unsigned long long)region->length, pbl_count); 2514 ret = nes_reg_mr(nesdev, nespd, stag, region->length, &root_vpbl, 2515 first_dma_addr, pbl_count, (u16)cur_pbl_index, acc, 2516 &iova_start, &nesmr->pbls_used, &nesmr->pbl_4k); 2517 2518 nes_debug(NES_DBG_MR, "ret=%d\n", ret); 2519 2520 if (ret == 0) { 2521 nesmr->ibmr.rkey = stag; 2522 nesmr->ibmr.lkey = stag; 2523 nesmr->mode = IWNES_MEMREG_TYPE_MEM; 2524 ibmr = &nesmr->ibmr; 2525 } else { 2526 ib_umem_release(region); 2527 kfree(nesmr); 2528 ibmr = ERR_PTR(-ENOMEM); 2529 } 2530 2531 reg_user_mr_err: 2532 /* free the resources */ 2533 if (root_pbl_index == 1) { 2534 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, 2535 vpbl.pbl_pbase); 2536 } else { 2537 for (page_index=0; page_index<root_pbl_index; page_index++) { 2538 pci_free_consistent(nesdev->pcidev, 4096, 2539 root_vpbl.leaf_vpbl[page_index].pbl_vbase, 2540 root_vpbl.leaf_vpbl[page_index].pbl_pbase); 2541 } 2542 kfree(root_vpbl.leaf_vpbl); 2543 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase, 2544 root_vpbl.pbl_pbase); 2545 } 2546 2547 nes_debug(NES_DBG_MR, "Leaving, ibmr=%p", ibmr); 2548 2549 return ibmr; 2550 case IWNES_MEMREG_TYPE_QP: 2551 case IWNES_MEMREG_TYPE_CQ: 2552 if (!region->length) { 2553 nes_debug(NES_DBG_MR, "Unable to register zero length region for CQ\n"); 2554 ib_umem_release(region); 2555 return ERR_PTR(-EINVAL); 2556 } 2557 nespbl = kzalloc(sizeof(*nespbl), GFP_KERNEL); 2558 if (!nespbl) { 2559 nes_debug(NES_DBG_MR, "Unable to allocate PBL\n"); 2560 ib_umem_release(region); 2561 return ERR_PTR(-ENOMEM); 2562 } 2563 nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); 2564 if (!nesmr) { 2565 ib_umem_release(region); 2566 kfree(nespbl); 2567 nes_debug(NES_DBG_MR, "Unable to allocate nesmr\n"); 2568 return ERR_PTR(-ENOMEM); 2569 } 2570 nesmr->region = region; 2571 nes_ucontext = to_nesucontext(pd->uobject->context); 2572 pbl_depth = region->length >> 12; 2573 pbl_depth += (region->length & (4096-1)) ? 1 : 0; 2574 nespbl->pbl_size = pbl_depth*sizeof(u64); 2575 if (req.reg_type == IWNES_MEMREG_TYPE_QP) { 2576 nes_debug(NES_DBG_MR, "Attempting to allocate QP PBL memory"); 2577 } else { 2578 nes_debug(NES_DBG_MR, "Attempting to allocate CP PBL memory"); 2579 } 2580 2581 nes_debug(NES_DBG_MR, " %u bytes, %u entries.\n", 2582 nespbl->pbl_size, pbl_depth); 2583 pbl = pci_alloc_consistent(nesdev->pcidev, nespbl->pbl_size, 2584 &nespbl->pbl_pbase); 2585 if (!pbl) { 2586 ib_umem_release(region); 2587 kfree(nesmr); 2588 kfree(nespbl); 2589 nes_debug(NES_DBG_MR, "Unable to allocate PBL memory\n"); 2590 return ERR_PTR(-ENOMEM); 2591 } 2592 2593 nespbl->pbl_vbase = (u64 *)pbl; 2594 nespbl->user_base = start; 2595 nes_debug(NES_DBG_MR, "Allocated PBL memory, %u bytes, pbl_pbase=%lx," 2596 " pbl_vbase=%p user_base=0x%lx\n", 2597 nespbl->pbl_size, (unsigned long) nespbl->pbl_pbase, 2598 (void *) nespbl->pbl_vbase, nespbl->user_base); 2599 2600 for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) { 2601 chunk_pages = sg_dma_len(sg) >> 12; 2602 chunk_pages += (sg_dma_len(sg) & (4096-1)) ? 1 : 0; 2603 if (first_page) { 2604 nespbl->page = sg_page(sg); 2605 first_page = 0; 2606 } 2607 2608 for (page_index = 0; page_index < chunk_pages; page_index++) { 2609 ((__le32 *)pbl)[0] = cpu_to_le32((u32) 2610 (sg_dma_address(sg)+ 2611 (page_index*4096))); 2612 ((__le32 *)pbl)[1] = cpu_to_le32(((u64) 2613 (sg_dma_address(sg)+ 2614 (page_index*4096)))>>32); 2615 nes_debug(NES_DBG_MR, "pbl=%p, *pbl=0x%016llx, 0x%08x%08x\n", pbl, 2616 (unsigned long long)*pbl, 2617 le32_to_cpu(((__le32 *)pbl)[1]), le32_to_cpu(((__le32 *)pbl)[0])); 2618 pbl++; 2619 } 2620 } 2621 2622 if (req.reg_type == IWNES_MEMREG_TYPE_QP) { 2623 list_add_tail(&nespbl->list, &nes_ucontext->qp_reg_mem_list); 2624 } else { 2625 list_add_tail(&nespbl->list, &nes_ucontext->cq_reg_mem_list); 2626 } 2627 nesmr->ibmr.rkey = -1; 2628 nesmr->ibmr.lkey = -1; 2629 nesmr->mode = req.reg_type; 2630 return &nesmr->ibmr; 2631 } 2632 2633 ib_umem_release(region); 2634 return ERR_PTR(-ENOSYS); 2635} 2636 2637 2638/** 2639 * nes_dereg_mr 2640 */ 2641static int nes_dereg_mr(struct ib_mr *ib_mr) 2642{ 2643 struct nes_mr *nesmr = to_nesmr(ib_mr); 2644 struct nes_vnic *nesvnic = to_nesvnic(ib_mr->device); 2645 struct nes_device *nesdev = nesvnic->nesdev; 2646 struct nes_adapter *nesadapter = nesdev->nesadapter; 2647 struct nes_hw_cqp_wqe *cqp_wqe; 2648 struct nes_cqp_request *cqp_request; 2649 unsigned long flags; 2650 int ret; 2651 u16 major_code; 2652 u16 minor_code; 2653 2654 2655 if (nesmr->pages) 2656 pci_free_consistent(nesdev->pcidev, 2657 nesmr->max_pages * sizeof(u64), 2658 nesmr->pages, 2659 nesmr->paddr); 2660 2661 if (nesmr->region) { 2662 ib_umem_release(nesmr->region); 2663 } 2664 if (nesmr->mode != IWNES_MEMREG_TYPE_MEM) { 2665 kfree(nesmr); 2666 return 0; 2667 } 2668 2669 /* Deallocate the region with the adapter */ 2670 2671 cqp_request = nes_get_cqp_request(nesdev); 2672 if (cqp_request == NULL) { 2673 nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); 2674 return -ENOMEM; 2675 } 2676 cqp_request->waiting = 1; 2677 cqp_wqe = &cqp_request->cqp_wqe; 2678 2679 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 2680 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, 2681 NES_CQP_DEALLOCATE_STAG | NES_CQP_STAG_VA_TO | 2682 NES_CQP_STAG_DEALLOC_PBLS | NES_CQP_STAG_MR); 2683 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, ib_mr->rkey); 2684 2685 atomic_set(&cqp_request->refcount, 2); 2686 nes_post_cqp_request(nesdev, cqp_request); 2687 2688 /* Wait for CQP */ 2689 nes_debug(NES_DBG_MR, "Waiting for deallocate STag 0x%08X completed\n", ib_mr->rkey); 2690 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), 2691 NES_EVENT_TIMEOUT); 2692 nes_debug(NES_DBG_MR, "Deallocate STag 0x%08X completed, wait_event_timeout ret = %u," 2693 " CQP Major:Minor codes = 0x%04X:0x%04X\n", 2694 ib_mr->rkey, ret, cqp_request->major_code, cqp_request->minor_code); 2695 2696 major_code = cqp_request->major_code; 2697 minor_code = cqp_request->minor_code; 2698 2699 nes_put_cqp_request(nesdev, cqp_request); 2700 2701 if (!ret) { 2702 nes_debug(NES_DBG_MR, "Timeout waiting to destroy STag," 2703 " ib_mr=%p, rkey = 0x%08X\n", 2704 ib_mr, ib_mr->rkey); 2705 return -ETIME; 2706 } else if (major_code) { 2707 nes_debug(NES_DBG_MR, "Error (0x%04X:0x%04X) while attempting" 2708 " to destroy STag, ib_mr=%p, rkey = 0x%08X\n", 2709 major_code, minor_code, ib_mr, ib_mr->rkey); 2710 return -EIO; 2711 } 2712 2713 if (nesmr->pbls_used != 0) { 2714 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 2715 if (nesmr->pbl_4k) { 2716 nesadapter->free_4kpbl += nesmr->pbls_used; 2717 if (nesadapter->free_4kpbl > nesadapter->max_4kpbl) 2718 printk(KERN_ERR PFX "free 4KB PBLs(%u) has " 2719 "exceeded the max(%u)\n", 2720 nesadapter->free_4kpbl, 2721 nesadapter->max_4kpbl); 2722 } else { 2723 nesadapter->free_256pbl += nesmr->pbls_used; 2724 if (nesadapter->free_256pbl > nesadapter->max_256pbl) 2725 printk(KERN_ERR PFX "free 256B PBLs(%u) has " 2726 "exceeded the max(%u)\n", 2727 nesadapter->free_256pbl, 2728 nesadapter->max_256pbl); 2729 } 2730 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 2731 } 2732 nes_free_resource(nesadapter, nesadapter->allocated_mrs, 2733 (ib_mr->rkey & 0x0fffff00) >> 8); 2734 2735 kfree(nesmr); 2736 2737 return 0; 2738} 2739 2740 2741/** 2742 * show_rev 2743 */ 2744static ssize_t show_rev(struct device *dev, struct device_attribute *attr, 2745 char *buf) 2746{ 2747 struct nes_ib_device *nesibdev = 2748 container_of(dev, struct nes_ib_device, ibdev.dev); 2749 struct nes_vnic *nesvnic = nesibdev->nesvnic; 2750 2751 nes_debug(NES_DBG_INIT, "\n"); 2752 return sprintf(buf, "%x\n", nesvnic->nesdev->nesadapter->hw_rev); 2753} 2754 2755 2756/** 2757 * show_fw_ver 2758 */ 2759static ssize_t show_fw_ver(struct device *dev, struct device_attribute *attr, 2760 char *buf) 2761{ 2762 struct nes_ib_device *nesibdev = 2763 container_of(dev, struct nes_ib_device, ibdev.dev); 2764 struct nes_vnic *nesvnic = nesibdev->nesvnic; 2765 2766 nes_debug(NES_DBG_INIT, "\n"); 2767 return sprintf(buf, "%u.%u\n", 2768 (nesvnic->nesdev->nesadapter->firmware_version >> 16), 2769 (nesvnic->nesdev->nesadapter->firmware_version & 0x000000ff)); 2770} 2771 2772 2773/** 2774 * show_hca 2775 */ 2776static ssize_t show_hca(struct device *dev, struct device_attribute *attr, 2777 char *buf) 2778{ 2779 nes_debug(NES_DBG_INIT, "\n"); 2780 return sprintf(buf, "NES020\n"); 2781} 2782 2783 2784/** 2785 * show_board 2786 */ 2787static ssize_t show_board(struct device *dev, struct device_attribute *attr, 2788 char *buf) 2789{ 2790 nes_debug(NES_DBG_INIT, "\n"); 2791 return sprintf(buf, "%.*s\n", 32, "NES020 Board ID"); 2792} 2793 2794 2795static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL); 2796static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL); 2797static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL); 2798static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL); 2799 2800static struct device_attribute *nes_dev_attributes[] = { 2801 &dev_attr_hw_rev, 2802 &dev_attr_fw_ver, 2803 &dev_attr_hca_type, 2804 &dev_attr_board_id 2805}; 2806 2807 2808/** 2809 * nes_query_qp 2810 */ 2811static int nes_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2812 int attr_mask, struct ib_qp_init_attr *init_attr) 2813{ 2814 struct nes_qp *nesqp = to_nesqp(ibqp); 2815 2816 nes_debug(NES_DBG_QP, "\n"); 2817 2818 attr->qp_access_flags = 0; 2819 attr->cap.max_send_wr = nesqp->hwqp.sq_size; 2820 attr->cap.max_recv_wr = nesqp->hwqp.rq_size; 2821 attr->cap.max_recv_sge = 1; 2822 if (nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) 2823 attr->cap.max_inline_data = 0; 2824 else 2825 attr->cap.max_inline_data = 64; 2826 2827 init_attr->event_handler = nesqp->ibqp.event_handler; 2828 init_attr->qp_context = nesqp->ibqp.qp_context; 2829 init_attr->send_cq = nesqp->ibqp.send_cq; 2830 init_attr->recv_cq = nesqp->ibqp.recv_cq; 2831 init_attr->srq = nesqp->ibqp.srq; 2832 init_attr->cap = attr->cap; 2833 2834 return 0; 2835} 2836 2837 2838/** 2839 * nes_hw_modify_qp 2840 */ 2841int nes_hw_modify_qp(struct nes_device *nesdev, struct nes_qp *nesqp, 2842 u32 next_iwarp_state, u32 termlen, u32 wait_completion) 2843{ 2844 struct nes_hw_cqp_wqe *cqp_wqe; 2845 /* struct iw_cm_id *cm_id = nesqp->cm_id; */ 2846 /* struct iw_cm_event cm_event; */ 2847 struct nes_cqp_request *cqp_request; 2848 int ret; 2849 u16 major_code; 2850 2851 nes_debug(NES_DBG_MOD_QP, "QP%u, refcount=%d\n", 2852 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount)); 2853 2854 cqp_request = nes_get_cqp_request(nesdev); 2855 if (cqp_request == NULL) { 2856 nes_debug(NES_DBG_MOD_QP, "Failed to get a cqp_request.\n"); 2857 return -ENOMEM; 2858 } 2859 if (wait_completion) { 2860 cqp_request->waiting = 1; 2861 } else { 2862 cqp_request->waiting = 0; 2863 } 2864 cqp_wqe = &cqp_request->cqp_wqe; 2865 2866 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, 2867 NES_CQP_MODIFY_QP | NES_CQP_QP_TYPE_IWARP | next_iwarp_state); 2868 nes_debug(NES_DBG_MOD_QP, "using next_iwarp_state=%08x, wqe_words=%08x\n", 2869 next_iwarp_state, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])); 2870 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 2871 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id); 2872 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, (u64)nesqp->nesqp_context_pbase); 2873 2874 /* If sending a terminate message, fill in the length (in words) */ 2875 if (((next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK) == NES_CQP_QP_IWARP_STATE_TERMINATE) && 2876 !(next_iwarp_state & NES_CQP_QP_TERM_DONT_SEND_TERM_MSG)) { 2877 termlen = ((termlen + 3) >> 2) << NES_CQP_OP_TERMLEN_SHIFT; 2878 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_NEW_MSS_IDX, termlen); 2879 } 2880 2881 atomic_set(&cqp_request->refcount, 2); 2882 nes_post_cqp_request(nesdev, cqp_request); 2883 2884 /* Wait for CQP */ 2885 if (wait_completion) { 2886 /* nes_debug(NES_DBG_MOD_QP, "Waiting for modify iWARP QP%u to complete.\n", 2887 nesqp->hwqp.qp_id); */ 2888 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), 2889 NES_EVENT_TIMEOUT); 2890 nes_debug(NES_DBG_MOD_QP, "Modify iwarp QP%u completed, wait_event_timeout ret=%u, " 2891 "CQP Major:Minor codes = 0x%04X:0x%04X.\n", 2892 nesqp->hwqp.qp_id, ret, cqp_request->major_code, cqp_request->minor_code); 2893 major_code = cqp_request->major_code; 2894 if (major_code) { 2895 nes_debug(NES_DBG_MOD_QP, "Modify iwarp QP%u failed" 2896 "CQP Major:Minor codes = 0x%04X:0x%04X, intended next state = 0x%08X.\n", 2897 nesqp->hwqp.qp_id, cqp_request->major_code, 2898 cqp_request->minor_code, next_iwarp_state); 2899 } 2900 2901 nes_put_cqp_request(nesdev, cqp_request); 2902 2903 if (!ret) 2904 return -ETIME; 2905 else if (major_code) 2906 return -EIO; 2907 else 2908 return 0; 2909 } else { 2910 return 0; 2911 } 2912} 2913 2914 2915/** 2916 * nes_modify_qp 2917 */ 2918int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2919 int attr_mask, struct ib_udata *udata) 2920{ 2921 struct nes_qp *nesqp = to_nesqp(ibqp); 2922 struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); 2923 struct nes_device *nesdev = nesvnic->nesdev; 2924 /* u32 cqp_head; */ 2925 /* u32 counter; */ 2926 u32 next_iwarp_state = 0; 2927 int err; 2928 unsigned long qplockflags; 2929 int ret; 2930 u16 original_last_aeq; 2931 u8 issue_modify_qp = 0; 2932 u8 dont_wait = 0; 2933 2934 nes_debug(NES_DBG_MOD_QP, "QP%u: QP State=%u, cur QP State=%u," 2935 " iwarp_state=0x%X, refcount=%d\n", 2936 nesqp->hwqp.qp_id, attr->qp_state, nesqp->ibqp_state, 2937 nesqp->iwarp_state, atomic_read(&nesqp->refcount)); 2938 2939 spin_lock_irqsave(&nesqp->lock, qplockflags); 2940 2941 nes_debug(NES_DBG_MOD_QP, "QP%u: hw_iwarp_state=0x%X, hw_tcp_state=0x%X," 2942 " QP Access Flags=0x%X, attr_mask = 0x%0x\n", 2943 nesqp->hwqp.qp_id, nesqp->hw_iwarp_state, 2944 nesqp->hw_tcp_state, attr->qp_access_flags, attr_mask); 2945 2946 if (attr_mask & IB_QP_STATE) { 2947 switch (attr->qp_state) { 2948 case IB_QPS_INIT: 2949 nes_debug(NES_DBG_MOD_QP, "QP%u: new state = init\n", 2950 nesqp->hwqp.qp_id); 2951 if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_IDLE) { 2952 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 2953 return -EINVAL; 2954 } 2955 next_iwarp_state = NES_CQP_QP_IWARP_STATE_IDLE; 2956 issue_modify_qp = 1; 2957 break; 2958 case IB_QPS_RTR: 2959 nes_debug(NES_DBG_MOD_QP, "QP%u: new state = rtr\n", 2960 nesqp->hwqp.qp_id); 2961 if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_IDLE) { 2962 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 2963 return -EINVAL; 2964 } 2965 next_iwarp_state = NES_CQP_QP_IWARP_STATE_IDLE; 2966 issue_modify_qp = 1; 2967 break; 2968 case IB_QPS_RTS: 2969 nes_debug(NES_DBG_MOD_QP, "QP%u: new state = rts\n", 2970 nesqp->hwqp.qp_id); 2971 if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_RTS) { 2972 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 2973 return -EINVAL; 2974 } 2975 if (nesqp->cm_id == NULL) { 2976 nes_debug(NES_DBG_MOD_QP, "QP%u: Failing attempt to move QP to RTS without a CM_ID. \n", 2977 nesqp->hwqp.qp_id ); 2978 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 2979 return -EINVAL; 2980 } 2981 next_iwarp_state = NES_CQP_QP_IWARP_STATE_RTS; 2982 if (nesqp->iwarp_state != NES_CQP_QP_IWARP_STATE_RTS) 2983 next_iwarp_state |= NES_CQP_QP_CONTEXT_VALID | 2984 NES_CQP_QP_ARP_VALID | NES_CQP_QP_ORD_VALID; 2985 issue_modify_qp = 1; 2986 nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_ESTABLISHED; 2987 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_RTS; 2988 nesqp->hte_added = 1; 2989 break; 2990 case IB_QPS_SQD: 2991 issue_modify_qp = 1; 2992 nes_debug(NES_DBG_MOD_QP, "QP%u: new state=closing. SQ head=%u, SQ tail=%u\n", 2993 nesqp->hwqp.qp_id, nesqp->hwqp.sq_head, nesqp->hwqp.sq_tail); 2994 if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_CLOSING) { 2995 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 2996 return 0; 2997 } else { 2998 if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_CLOSING) { 2999 nes_debug(NES_DBG_MOD_QP, "QP%u: State change to closing" 3000 " ignored due to current iWARP state\n", 3001 nesqp->hwqp.qp_id); 3002 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3003 return -EINVAL; 3004 } 3005 if (nesqp->hw_iwarp_state != NES_AEQE_IWARP_STATE_RTS) { 3006 nes_debug(NES_DBG_MOD_QP, "QP%u: State change to closing" 3007 " already done based on hw state.\n", 3008 nesqp->hwqp.qp_id); 3009 issue_modify_qp = 0; 3010 } 3011 switch (nesqp->hw_iwarp_state) { 3012 case NES_AEQE_IWARP_STATE_CLOSING: 3013 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING; 3014 break; 3015 case NES_AEQE_IWARP_STATE_TERMINATE: 3016 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE; 3017 break; 3018 case NES_AEQE_IWARP_STATE_ERROR: 3019 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR; 3020 break; 3021 default: 3022 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING; 3023 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING; 3024 break; 3025 } 3026 } 3027 break; 3028 case IB_QPS_SQE: 3029 nes_debug(NES_DBG_MOD_QP, "QP%u: new state = terminate\n", 3030 nesqp->hwqp.qp_id); 3031 if (nesqp->iwarp_state>=(u32)NES_CQP_QP_IWARP_STATE_TERMINATE) { 3032 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3033 return -EINVAL; 3034 } 3035 /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */ 3036 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE; 3037 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE; 3038 issue_modify_qp = 1; 3039 break; 3040 case IB_QPS_ERR: 3041 case IB_QPS_RESET: 3042 if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_ERROR) { 3043 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3044 return -EINVAL; 3045 } 3046 nes_debug(NES_DBG_MOD_QP, "QP%u: new state = error\n", 3047 nesqp->hwqp.qp_id); 3048 if (nesqp->term_flags) 3049 del_timer(&nesqp->terminate_timer); 3050 3051 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR; 3052 /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */ 3053 if (nesqp->hte_added) { 3054 nes_debug(NES_DBG_MOD_QP, "set CQP_QP_DEL_HTE\n"); 3055 next_iwarp_state |= NES_CQP_QP_DEL_HTE; 3056 nesqp->hte_added = 0; 3057 } 3058 if ((nesqp->hw_tcp_state > NES_AEQE_TCP_STATE_CLOSED) && 3059 (nesdev->iw_status) && 3060 (nesqp->hw_tcp_state != NES_AEQE_TCP_STATE_TIME_WAIT)) { 3061 next_iwarp_state |= NES_CQP_QP_RESET; 3062 } else { 3063 nes_debug(NES_DBG_MOD_QP, "QP%u NOT setting NES_CQP_QP_RESET since TCP state = %u\n", 3064 nesqp->hwqp.qp_id, nesqp->hw_tcp_state); 3065 dont_wait = 1; 3066 } 3067 issue_modify_qp = 1; 3068 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR; 3069 break; 3070 default: 3071 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3072 return -EINVAL; 3073 break; 3074 } 3075 3076 nesqp->ibqp_state = attr->qp_state; 3077 nesqp->iwarp_state = next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK; 3078 nes_debug(NES_DBG_MOD_QP, "Change nesqp->iwarp_state=%08x\n", 3079 nesqp->iwarp_state); 3080 } 3081 3082 if (attr_mask & IB_QP_ACCESS_FLAGS) { 3083 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) { 3084 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN | 3085 NES_QPCONTEXT_MISC_RDMA_READ_EN); 3086 issue_modify_qp = 1; 3087 } 3088 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) { 3089 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN); 3090 issue_modify_qp = 1; 3091 } 3092 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) { 3093 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_READ_EN); 3094 issue_modify_qp = 1; 3095 } 3096 if (attr->qp_access_flags & IB_ACCESS_MW_BIND) { 3097 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_WBIND_EN); 3098 issue_modify_qp = 1; 3099 } 3100 3101 if (nesqp->user_mode) { 3102 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN | 3103 NES_QPCONTEXT_MISC_RDMA_READ_EN); 3104 issue_modify_qp = 1; 3105 } 3106 } 3107 3108 original_last_aeq = nesqp->last_aeq; 3109 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3110 3111 nes_debug(NES_DBG_MOD_QP, "issue_modify_qp=%u\n", issue_modify_qp); 3112 3113 ret = 0; 3114 3115 3116 if (issue_modify_qp) { 3117 nes_debug(NES_DBG_MOD_QP, "call nes_hw_modify_qp\n"); 3118 ret = nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 1); 3119 if (ret) 3120 nes_debug(NES_DBG_MOD_QP, "nes_hw_modify_qp (next_iwarp_state = 0x%08X)" 3121 " failed for QP%u.\n", 3122 next_iwarp_state, nesqp->hwqp.qp_id); 3123 3124 } 3125 3126 if ((issue_modify_qp) && (nesqp->ibqp_state > IB_QPS_RTS)) { 3127 nes_debug(NES_DBG_MOD_QP, "QP%u Issued ModifyQP refcount (%d)," 3128 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", 3129 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), 3130 original_last_aeq, nesqp->last_aeq); 3131 if (!ret || original_last_aeq != NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) { 3132 if (dont_wait) { 3133 if (nesqp->cm_id && nesqp->hw_tcp_state != 0) { 3134 nes_debug(NES_DBG_MOD_QP, "QP%u Queuing fake disconnect for QP refcount (%d)," 3135 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", 3136 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), 3137 original_last_aeq, nesqp->last_aeq); 3138 /* this one is for the cm_disconnect thread */ 3139 spin_lock_irqsave(&nesqp->lock, qplockflags); 3140 nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED; 3141 nesqp->last_aeq = NES_AEQE_AEID_RESET_SENT; 3142 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3143 nes_cm_disconn(nesqp); 3144 } else { 3145 nes_debug(NES_DBG_MOD_QP, "QP%u No fake disconnect, QP refcount=%d\n", 3146 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount)); 3147 } 3148 } else { 3149 spin_lock_irqsave(&nesqp->lock, qplockflags); 3150 if (nesqp->cm_id) { 3151 /* These two are for the timer thread */ 3152 if (atomic_inc_return(&nesqp->close_timer_started) == 1) { 3153 nesqp->cm_id->add_ref(nesqp->cm_id); 3154 nes_debug(NES_DBG_MOD_QP, "QP%u Not decrementing QP refcount (%d)," 3155 " need ae to finish up, original_last_aeq = 0x%04X." 3156 " last_aeq = 0x%04X, scheduling timer.\n", 3157 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), 3158 original_last_aeq, nesqp->last_aeq); 3159 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *) nesqp, NES_TIMER_TYPE_CLOSE, 1, 0); 3160 } 3161 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3162 } else { 3163 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3164 nes_debug(NES_DBG_MOD_QP, "QP%u Not decrementing QP refcount (%d)," 3165 " need ae to finish up, original_last_aeq = 0x%04X." 3166 " last_aeq = 0x%04X.\n", 3167 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), 3168 original_last_aeq, nesqp->last_aeq); 3169 } 3170 } 3171 } else { 3172 nes_debug(NES_DBG_MOD_QP, "QP%u Decrementing QP refcount (%d), No ae to finish up," 3173 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", 3174 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), 3175 original_last_aeq, nesqp->last_aeq); 3176 } 3177 } else { 3178 nes_debug(NES_DBG_MOD_QP, "QP%u Decrementing QP refcount (%d), No ae to finish up," 3179 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", 3180 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), 3181 original_last_aeq, nesqp->last_aeq); 3182 } 3183 3184 err = 0; 3185 3186 nes_debug(NES_DBG_MOD_QP, "QP%u Leaving, refcount=%d\n", 3187 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount)); 3188 3189 return err; 3190} 3191 3192 3193/** 3194 * nes_muticast_attach 3195 */ 3196static int nes_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) 3197{ 3198 nes_debug(NES_DBG_INIT, "\n"); 3199 return -ENOSYS; 3200} 3201 3202 3203/** 3204 * nes_multicast_detach 3205 */ 3206static int nes_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) 3207{ 3208 nes_debug(NES_DBG_INIT, "\n"); 3209 return -ENOSYS; 3210} 3211 3212 3213/** 3214 * nes_process_mad 3215 */ 3216static int nes_process_mad(struct ib_device *ibdev, int mad_flags, 3217 u8 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, 3218 const struct ib_mad_hdr *in, size_t in_mad_size, 3219 struct ib_mad_hdr *out, size_t *out_mad_size, 3220 u16 *out_mad_pkey_index) 3221{ 3222 nes_debug(NES_DBG_INIT, "\n"); 3223 return -ENOSYS; 3224} 3225 3226static inline void 3227fill_wqe_sg_send(struct nes_hw_qp_wqe *wqe, struct ib_send_wr *ib_wr, u32 uselkey) 3228{ 3229 int sge_index; 3230 int total_payload_length = 0; 3231 for (sge_index = 0; sge_index < ib_wr->num_sge; sge_index++) { 3232 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX+(sge_index*4), 3233 ib_wr->sg_list[sge_index].addr); 3234 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_LENGTH0_IDX + (sge_index*4), 3235 ib_wr->sg_list[sge_index].length); 3236 if (uselkey) 3237 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX + (sge_index*4), 3238 (ib_wr->sg_list[sge_index].lkey)); 3239 else 3240 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX + (sge_index*4), 0); 3241 3242 total_payload_length += ib_wr->sg_list[sge_index].length; 3243 } 3244 nes_debug(NES_DBG_IW_TX, "UC UC UC, sending total_payload_length=%u \n", 3245 total_payload_length); 3246 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX, 3247 total_payload_length); 3248} 3249 3250/** 3251 * nes_post_send 3252 */ 3253static int nes_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr, 3254 struct ib_send_wr **bad_wr) 3255{ 3256 u64 u64temp; 3257 unsigned long flags = 0; 3258 struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); 3259 struct nes_device *nesdev = nesvnic->nesdev; 3260 struct nes_qp *nesqp = to_nesqp(ibqp); 3261 struct nes_hw_qp_wqe *wqe; 3262 int err = 0; 3263 u32 qsize = nesqp->hwqp.sq_size; 3264 u32 head; 3265 u32 wqe_misc = 0; 3266 u32 wqe_count = 0; 3267 u32 counter; 3268 3269 if (nesqp->ibqp_state > IB_QPS_RTS) { 3270 err = -EINVAL; 3271 goto out; 3272 } 3273 3274 spin_lock_irqsave(&nesqp->lock, flags); 3275 3276 head = nesqp->hwqp.sq_head; 3277 3278 while (ib_wr) { 3279 /* Check for QP error */ 3280 if (nesqp->term_flags) { 3281 err = -EINVAL; 3282 break; 3283 } 3284 3285 /* Check for SQ overflow */ 3286 if (((head + (2 * qsize) - nesqp->hwqp.sq_tail) % qsize) == (qsize - 1)) { 3287 err = -ENOMEM; 3288 break; 3289 } 3290 3291 wqe = &nesqp->hwqp.sq_vbase[head]; 3292 /* nes_debug(NES_DBG_IW_TX, "processing sq wqe for QP%u at %p, head = %u.\n", 3293 nesqp->hwqp.qp_id, wqe, head); */ 3294 nes_fill_init_qp_wqe(wqe, nesqp, head); 3295 u64temp = (u64)(ib_wr->wr_id); 3296 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX, 3297 u64temp); 3298 switch (ib_wr->opcode) { 3299 case IB_WR_SEND: 3300 case IB_WR_SEND_WITH_INV: 3301 if (IB_WR_SEND == ib_wr->opcode) { 3302 if (ib_wr->send_flags & IB_SEND_SOLICITED) 3303 wqe_misc = NES_IWARP_SQ_OP_SENDSE; 3304 else 3305 wqe_misc = NES_IWARP_SQ_OP_SEND; 3306 } else { 3307 if (ib_wr->send_flags & IB_SEND_SOLICITED) 3308 wqe_misc = NES_IWARP_SQ_OP_SENDSEINV; 3309 else 3310 wqe_misc = NES_IWARP_SQ_OP_SENDINV; 3311 3312 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX, 3313 ib_wr->ex.invalidate_rkey); 3314 } 3315 3316 if (ib_wr->num_sge > nesdev->nesadapter->max_sge) { 3317 err = -EINVAL; 3318 break; 3319 } 3320 3321 if (ib_wr->send_flags & IB_SEND_FENCE) 3322 wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE; 3323 3324 if ((ib_wr->send_flags & IB_SEND_INLINE) && 3325 ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) && 3326 (ib_wr->sg_list[0].length <= 64)) { 3327 memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX], 3328 (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length); 3329 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX, 3330 ib_wr->sg_list[0].length); 3331 wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA; 3332 } else { 3333 fill_wqe_sg_send(wqe, ib_wr, 1); 3334 } 3335 3336 break; 3337 case IB_WR_RDMA_WRITE: 3338 wqe_misc = NES_IWARP_SQ_OP_RDMAW; 3339 if (ib_wr->num_sge > nesdev->nesadapter->max_sge) { 3340 nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=%u\n", 3341 ib_wr->num_sge, nesdev->nesadapter->max_sge); 3342 err = -EINVAL; 3343 break; 3344 } 3345 3346 if (ib_wr->send_flags & IB_SEND_FENCE) 3347 wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE; 3348 3349 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX, 3350 rdma_wr(ib_wr)->rkey); 3351 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX, 3352 rdma_wr(ib_wr)->remote_addr); 3353 3354 if ((ib_wr->send_flags & IB_SEND_INLINE) && 3355 ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) && 3356 (ib_wr->sg_list[0].length <= 64)) { 3357 memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX], 3358 (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length); 3359 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX, 3360 ib_wr->sg_list[0].length); 3361 wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA; 3362 } else { 3363 fill_wqe_sg_send(wqe, ib_wr, 1); 3364 } 3365 3366 wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] = 3367 wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]; 3368 break; 3369 case IB_WR_RDMA_READ: 3370 case IB_WR_RDMA_READ_WITH_INV: 3371 /* iWARP only supports 1 sge for RDMA reads */ 3372 if (ib_wr->num_sge > 1) { 3373 nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=1\n", 3374 ib_wr->num_sge); 3375 err = -EINVAL; 3376 break; 3377 } 3378 if (ib_wr->opcode == IB_WR_RDMA_READ) { 3379 wqe_misc = NES_IWARP_SQ_OP_RDMAR; 3380 } else { 3381 wqe_misc = NES_IWARP_SQ_OP_RDMAR_LOCINV; 3382 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX, 3383 ib_wr->ex.invalidate_rkey); 3384 } 3385 3386 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX, 3387 rdma_wr(ib_wr)->remote_addr); 3388 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX, 3389 rdma_wr(ib_wr)->rkey); 3390 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX, 3391 ib_wr->sg_list->length); 3392 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX, 3393 ib_wr->sg_list->addr); 3394 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX, 3395 ib_wr->sg_list->lkey); 3396 break; 3397 case IB_WR_LOCAL_INV: 3398 wqe_misc = NES_IWARP_SQ_OP_LOCINV; 3399 set_wqe_32bit_value(wqe->wqe_words, 3400 NES_IWARP_SQ_LOCINV_WQE_INV_STAG_IDX, 3401 ib_wr->ex.invalidate_rkey); 3402 break; 3403 case IB_WR_REG_MR: 3404 { 3405 struct nes_mr *mr = to_nesmr(reg_wr(ib_wr)->mr); 3406 int page_shift = ilog2(reg_wr(ib_wr)->mr->page_size); 3407 int flags = reg_wr(ib_wr)->access; 3408 3409 if (mr->npages > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64))) { 3410 nes_debug(NES_DBG_IW_TX, "SQ_FMR: bad page_list_len\n"); 3411 err = -EINVAL; 3412 break; 3413 } 3414 wqe_misc = NES_IWARP_SQ_OP_FAST_REG; 3415 set_wqe_64bit_value(wqe->wqe_words, 3416 NES_IWARP_SQ_FMR_WQE_VA_FBO_LOW_IDX, 3417 mr->ibmr.iova); 3418 set_wqe_32bit_value(wqe->wqe_words, 3419 NES_IWARP_SQ_FMR_WQE_LENGTH_LOW_IDX, 3420 mr->ibmr.length); 3421 set_wqe_32bit_value(wqe->wqe_words, 3422 NES_IWARP_SQ_FMR_WQE_LENGTH_HIGH_IDX, 0); 3423 set_wqe_32bit_value(wqe->wqe_words, 3424 NES_IWARP_SQ_FMR_WQE_MR_STAG_IDX, 3425 reg_wr(ib_wr)->key); 3426 3427 if (page_shift == 12) { 3428 wqe_misc |= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_4K; 3429 } else if (page_shift == 21) { 3430 wqe_misc |= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_2M; 3431 } else { 3432 nes_debug(NES_DBG_IW_TX, "Invalid page shift," 3433 " ib_wr=%u, max=1\n", ib_wr->num_sge); 3434 err = -EINVAL; 3435 break; 3436 } 3437 3438 /* Set access_flags */ 3439 wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_READ; 3440 if (flags & IB_ACCESS_LOCAL_WRITE) 3441 wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_WRITE; 3442 3443 if (flags & IB_ACCESS_REMOTE_WRITE) 3444 wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_WRITE; 3445 3446 if (flags & IB_ACCESS_REMOTE_READ) 3447 wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_READ; 3448 3449 if (flags & IB_ACCESS_MW_BIND) 3450 wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_WINDOW_BIND; 3451 3452 /* Fill in PBL info: */ 3453 set_wqe_64bit_value(wqe->wqe_words, 3454 NES_IWARP_SQ_FMR_WQE_PBL_ADDR_LOW_IDX, 3455 mr->paddr); 3456 3457 set_wqe_32bit_value(wqe->wqe_words, 3458 NES_IWARP_SQ_FMR_WQE_PBL_LENGTH_IDX, 3459 mr->npages * 8); 3460 3461 nes_debug(NES_DBG_IW_TX, "SQ_REG_MR: iova_start: %llx, " 3462 "length: %d, rkey: %0x, pgl_paddr: %llx, " 3463 "page_list_len: %u, wqe_misc: %x\n", 3464 (unsigned long long) mr->ibmr.iova, 3465 mr->ibmr.length, 3466 reg_wr(ib_wr)->key, 3467 (unsigned long long) mr->paddr, 3468 mr->npages, 3469 wqe_misc); 3470 break; 3471 } 3472 default: 3473 /* error */ 3474 err = -EINVAL; 3475 break; 3476 } 3477 3478 if (err) 3479 break; 3480 3481 if ((ib_wr->send_flags & IB_SEND_SIGNALED) || nesqp->sig_all) 3482 wqe_misc |= NES_IWARP_SQ_WQE_SIGNALED_COMPL; 3483 3484 wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = cpu_to_le32(wqe_misc); 3485 3486 ib_wr = ib_wr->next; 3487 head++; 3488 wqe_count++; 3489 if (head >= qsize) 3490 head = 0; 3491 3492 } 3493 3494 nesqp->hwqp.sq_head = head; 3495 barrier(); 3496 while (wqe_count) { 3497 counter = min(wqe_count, ((u32)255)); 3498 wqe_count -= counter; 3499 nes_write32(nesdev->regs + NES_WQE_ALLOC, 3500 (counter << 24) | 0x00800000 | nesqp->hwqp.qp_id); 3501 } 3502 3503 spin_unlock_irqrestore(&nesqp->lock, flags); 3504 3505out: 3506 if (err) 3507 *bad_wr = ib_wr; 3508 return err; 3509} 3510 3511 3512/** 3513 * nes_post_recv 3514 */ 3515static int nes_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr, 3516 struct ib_recv_wr **bad_wr) 3517{ 3518 u64 u64temp; 3519 unsigned long flags = 0; 3520 struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); 3521 struct nes_device *nesdev = nesvnic->nesdev; 3522 struct nes_qp *nesqp = to_nesqp(ibqp); 3523 struct nes_hw_qp_wqe *wqe; 3524 int err = 0; 3525 int sge_index; 3526 u32 qsize = nesqp->hwqp.rq_size; 3527 u32 head; 3528 u32 wqe_count = 0; 3529 u32 counter; 3530 u32 total_payload_length; 3531 3532 if (nesqp->ibqp_state > IB_QPS_RTS) { 3533 err = -EINVAL; 3534 goto out; 3535 } 3536 3537 spin_lock_irqsave(&nesqp->lock, flags); 3538 3539 head = nesqp->hwqp.rq_head; 3540 3541 while (ib_wr) { 3542 /* Check for QP error */ 3543 if (nesqp->term_flags) { 3544 err = -EINVAL; 3545 break; 3546 } 3547 3548 if (ib_wr->num_sge > nesdev->nesadapter->max_sge) { 3549 err = -EINVAL; 3550 break; 3551 } 3552 /* Check for RQ overflow */ 3553 if (((head + (2 * qsize) - nesqp->hwqp.rq_tail) % qsize) == (qsize - 1)) { 3554 err = -ENOMEM; 3555 break; 3556 } 3557 3558 nes_debug(NES_DBG_IW_RX, "ibwr sge count = %u.\n", ib_wr->num_sge); 3559 wqe = &nesqp->hwqp.rq_vbase[head]; 3560 3561 /* nes_debug(NES_DBG_IW_RX, "QP%u:processing rq wqe at %p, head = %u.\n", 3562 nesqp->hwqp.qp_id, wqe, head); */ 3563 nes_fill_init_qp_wqe(wqe, nesqp, head); 3564 u64temp = (u64)(ib_wr->wr_id); 3565 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX, 3566 u64temp); 3567 total_payload_length = 0; 3568 for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) { 3569 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_FRAG0_LOW_IDX+(sge_index*4), 3570 ib_wr->sg_list[sge_index].addr); 3571 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_LENGTH0_IDX+(sge_index*4), 3572 ib_wr->sg_list[sge_index].length); 3573 set_wqe_32bit_value(wqe->wqe_words,NES_IWARP_RQ_WQE_STAG0_IDX+(sge_index*4), 3574 ib_wr->sg_list[sge_index].lkey); 3575 3576 total_payload_length += ib_wr->sg_list[sge_index].length; 3577 } 3578 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX, 3579 total_payload_length); 3580 3581 ib_wr = ib_wr->next; 3582 head++; 3583 wqe_count++; 3584 if (head >= qsize) 3585 head = 0; 3586 } 3587 3588 nesqp->hwqp.rq_head = head; 3589 barrier(); 3590 while (wqe_count) { 3591 counter = min(wqe_count, ((u32)255)); 3592 wqe_count -= counter; 3593 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter<<24) | nesqp->hwqp.qp_id); 3594 } 3595 3596 spin_unlock_irqrestore(&nesqp->lock, flags); 3597 3598out: 3599 if (err) 3600 *bad_wr = ib_wr; 3601 return err; 3602} 3603 3604 3605/** 3606 * nes_poll_cq 3607 */ 3608static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) 3609{ 3610 u64 u64temp; 3611 u64 wrid; 3612 unsigned long flags = 0; 3613 struct nes_vnic *nesvnic = to_nesvnic(ibcq->device); 3614 struct nes_device *nesdev = nesvnic->nesdev; 3615 struct nes_cq *nescq = to_nescq(ibcq); 3616 struct nes_qp *nesqp; 3617 struct nes_hw_cqe cqe; 3618 u32 head; 3619 u32 wq_tail = 0; 3620 u32 cq_size; 3621 u32 cqe_count = 0; 3622 u32 wqe_index; 3623 u32 u32temp; 3624 u32 move_cq_head = 1; 3625 u32 err_code; 3626 3627 nes_debug(NES_DBG_CQ, "\n"); 3628 3629 spin_lock_irqsave(&nescq->lock, flags); 3630 3631 head = nescq->hw_cq.cq_head; 3632 cq_size = nescq->hw_cq.cq_size; 3633 3634 while (cqe_count < num_entries) { 3635 if ((le32_to_cpu(nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & 3636 NES_CQE_VALID) == 0) 3637 break; 3638 3639 /* 3640 * Make sure we read CQ entry contents *after* 3641 * we've checked the valid bit. 3642 */ 3643 rmb(); 3644 3645 cqe = nescq->hw_cq.cq_vbase[head]; 3646 u32temp = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); 3647 wqe_index = u32temp & (nesdev->nesadapter->max_qp_wr - 1); 3648 u32temp &= ~(NES_SW_CONTEXT_ALIGN-1); 3649 /* parse CQE, get completion context from WQE (either rq or sq) */ 3650 u64temp = (((u64)(le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32) | 3651 ((u64)u32temp); 3652 3653 if (u64temp) { 3654 nesqp = (struct nes_qp *)(unsigned long)u64temp; 3655 memset(entry, 0, sizeof *entry); 3656 if (cqe.cqe_words[NES_CQE_ERROR_CODE_IDX] == 0) { 3657 entry->status = IB_WC_SUCCESS; 3658 } else { 3659 err_code = le32_to_cpu(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]); 3660 if (NES_IWARP_CQE_MAJOR_DRV == (err_code >> 16)) { 3661 entry->status = err_code & 0x0000ffff; 3662 3663 /* The rest of the cqe's will be marked as flushed */ 3664 nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX] = 3665 cpu_to_le32((NES_IWARP_CQE_MAJOR_FLUSH << 16) | 3666 NES_IWARP_CQE_MINOR_FLUSH); 3667 } else 3668 entry->status = IB_WC_WR_FLUSH_ERR; 3669 } 3670 3671 entry->qp = &nesqp->ibqp; 3672 entry->src_qp = nesqp->hwqp.qp_id; 3673 3674 if (le32_to_cpu(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) { 3675 if (nesqp->skip_lsmm) { 3676 nesqp->skip_lsmm = 0; 3677 nesqp->hwqp.sq_tail++; 3678 } 3679 3680 /* Working on a SQ Completion*/ 3681 wrid = (((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index]. 3682 wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) | 3683 ((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index]. 3684 wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX]))); 3685 entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index]. 3686 wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]); 3687 3688 switch (le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index]. 3689 wqe_words[NES_IWARP_SQ_WQE_MISC_IDX]) & 0x3f) { 3690 case NES_IWARP_SQ_OP_RDMAW: 3691 nes_debug(NES_DBG_CQ, "Operation = RDMA WRITE.\n"); 3692 entry->opcode = IB_WC_RDMA_WRITE; 3693 break; 3694 case NES_IWARP_SQ_OP_RDMAR: 3695 nes_debug(NES_DBG_CQ, "Operation = RDMA READ.\n"); 3696 entry->opcode = IB_WC_RDMA_READ; 3697 entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index]. 3698 wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX]); 3699 break; 3700 case NES_IWARP_SQ_OP_SENDINV: 3701 case NES_IWARP_SQ_OP_SENDSEINV: 3702 case NES_IWARP_SQ_OP_SEND: 3703 case NES_IWARP_SQ_OP_SENDSE: 3704 nes_debug(NES_DBG_CQ, "Operation = Send.\n"); 3705 entry->opcode = IB_WC_SEND; 3706 break; 3707 case NES_IWARP_SQ_OP_LOCINV: 3708 entry->opcode = IB_WC_LOCAL_INV; 3709 break; 3710 case NES_IWARP_SQ_OP_FAST_REG: 3711 entry->opcode = IB_WC_REG_MR; 3712 break; 3713 } 3714 3715 nesqp->hwqp.sq_tail = (wqe_index+1)&(nesqp->hwqp.sq_size - 1); 3716 if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.sq_tail != nesqp->hwqp.sq_head)) { 3717 move_cq_head = 0; 3718 wq_tail = nesqp->hwqp.sq_tail; 3719 } 3720 } else { 3721 /* Working on a RQ Completion*/ 3722 entry->byte_len = le32_to_cpu(cqe.cqe_words[NES_CQE_PAYLOAD_LENGTH_IDX]); 3723 wrid = ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX]))) | 3724 ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32); 3725 entry->opcode = IB_WC_RECV; 3726 3727 nesqp->hwqp.rq_tail = (wqe_index+1)&(nesqp->hwqp.rq_size - 1); 3728 if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.rq_tail != nesqp->hwqp.rq_head)) { 3729 move_cq_head = 0; 3730 wq_tail = nesqp->hwqp.rq_tail; 3731 } 3732 } 3733 3734 entry->wr_id = wrid; 3735 entry++; 3736 cqe_count++; 3737 } 3738 3739 if (move_cq_head) { 3740 nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0; 3741 if (++head >= cq_size) 3742 head = 0; 3743 nescq->polled_completions++; 3744 3745 if ((nescq->polled_completions > (cq_size / 2)) || 3746 (nescq->polled_completions == 255)) { 3747 nes_debug(NES_DBG_CQ, "CQ%u Issuing CQE Allocate since more than half of cqes" 3748 " are pending %u of %u.\n", 3749 nescq->hw_cq.cq_number, nescq->polled_completions, cq_size); 3750 nes_write32(nesdev->regs+NES_CQE_ALLOC, 3751 nescq->hw_cq.cq_number | (nescq->polled_completions << 16)); 3752 nescq->polled_completions = 0; 3753 } 3754 } else { 3755 /* Update the wqe index and set status to flush */ 3756 wqe_index = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); 3757 wqe_index = (wqe_index & (~(nesdev->nesadapter->max_qp_wr - 1))) | wq_tail; 3758 nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] = 3759 cpu_to_le32(wqe_index); 3760 move_cq_head = 1; /* ready for next pass */ 3761 } 3762 } 3763 3764 if (nescq->polled_completions) { 3765 nes_write32(nesdev->regs+NES_CQE_ALLOC, 3766 nescq->hw_cq.cq_number | (nescq->polled_completions << 16)); 3767 nescq->polled_completions = 0; 3768 } 3769 3770 nescq->hw_cq.cq_head = head; 3771 nes_debug(NES_DBG_CQ, "Reporting %u completions for CQ%u.\n", 3772 cqe_count, nescq->hw_cq.cq_number); 3773 3774 spin_unlock_irqrestore(&nescq->lock, flags); 3775 3776 return cqe_count; 3777} 3778 3779 3780/** 3781 * nes_req_notify_cq 3782 */ 3783static int nes_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags) 3784 { 3785 struct nes_vnic *nesvnic = to_nesvnic(ibcq->device); 3786 struct nes_device *nesdev = nesvnic->nesdev; 3787 struct nes_cq *nescq = to_nescq(ibcq); 3788 u32 cq_arm; 3789 3790 nes_debug(NES_DBG_CQ, "Requesting notification for CQ%u.\n", 3791 nescq->hw_cq.cq_number); 3792 3793 cq_arm = nescq->hw_cq.cq_number; 3794 if ((notify_flags & IB_CQ_SOLICITED_MASK) == IB_CQ_NEXT_COMP) 3795 cq_arm |= NES_CQE_ALLOC_NOTIFY_NEXT; 3796 else if ((notify_flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED) 3797 cq_arm |= NES_CQE_ALLOC_NOTIFY_SE; 3798 else 3799 return -EINVAL; 3800 3801 nes_write32(nesdev->regs+NES_CQE_ALLOC, cq_arm); 3802 nes_read32(nesdev->regs+NES_CQE_ALLOC); 3803 3804 return 0; 3805} 3806 3807static int nes_port_immutable(struct ib_device *ibdev, u8 port_num, 3808 struct ib_port_immutable *immutable) 3809{ 3810 struct ib_port_attr attr; 3811 int err; 3812 3813 err = nes_query_port(ibdev, port_num, &attr); 3814 if (err) 3815 return err; 3816 3817 immutable->pkey_tbl_len = attr.pkey_tbl_len; 3818 immutable->gid_tbl_len = attr.gid_tbl_len; 3819 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 3820 3821 return 0; 3822} 3823 3824/** 3825 * nes_init_ofa_device 3826 */ 3827struct nes_ib_device *nes_init_ofa_device(struct net_device *netdev) 3828{ 3829 struct nes_ib_device *nesibdev; 3830 struct nes_vnic *nesvnic = netdev_priv(netdev); 3831 struct nes_device *nesdev = nesvnic->nesdev; 3832 3833 nesibdev = (struct nes_ib_device *)ib_alloc_device(sizeof(struct nes_ib_device)); 3834 if (nesibdev == NULL) { 3835 return NULL; 3836 } 3837 strlcpy(nesibdev->ibdev.name, "nes%d", IB_DEVICE_NAME_MAX); 3838 nesibdev->ibdev.owner = THIS_MODULE; 3839 3840 nesibdev->ibdev.node_type = RDMA_NODE_RNIC; 3841 memset(&nesibdev->ibdev.node_guid, 0, sizeof(nesibdev->ibdev.node_guid)); 3842 memcpy(&nesibdev->ibdev.node_guid, netdev->dev_addr, 6); 3843 3844 nesibdev->ibdev.uverbs_cmd_mask = 3845 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | 3846 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | 3847 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | 3848 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | 3849 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | 3850 (1ull << IB_USER_VERBS_CMD_REG_MR) | 3851 (1ull << IB_USER_VERBS_CMD_DEREG_MR) | 3852 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | 3853 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | 3854 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | 3855 (1ull << IB_USER_VERBS_CMD_CREATE_AH) | 3856 (1ull << IB_USER_VERBS_CMD_DESTROY_AH) | 3857 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | 3858 (1ull << IB_USER_VERBS_CMD_CREATE_QP) | 3859 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | 3860 (1ull << IB_USER_VERBS_CMD_POLL_CQ) | 3861 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | 3862 (1ull << IB_USER_VERBS_CMD_ALLOC_MW) | 3863 (1ull << IB_USER_VERBS_CMD_BIND_MW) | 3864 (1ull << IB_USER_VERBS_CMD_DEALLOC_MW) | 3865 (1ull << IB_USER_VERBS_CMD_POST_RECV) | 3866 (1ull << IB_USER_VERBS_CMD_POST_SEND); 3867 3868 nesibdev->ibdev.phys_port_cnt = 1; 3869 nesibdev->ibdev.num_comp_vectors = 1; 3870 nesibdev->ibdev.dma_device = &nesdev->pcidev->dev; 3871 nesibdev->ibdev.dev.parent = &nesdev->pcidev->dev; 3872 nesibdev->ibdev.query_device = nes_query_device; 3873 nesibdev->ibdev.query_port = nes_query_port; 3874 nesibdev->ibdev.query_pkey = nes_query_pkey; 3875 nesibdev->ibdev.query_gid = nes_query_gid; 3876 nesibdev->ibdev.alloc_ucontext = nes_alloc_ucontext; 3877 nesibdev->ibdev.dealloc_ucontext = nes_dealloc_ucontext; 3878 nesibdev->ibdev.mmap = nes_mmap; 3879 nesibdev->ibdev.alloc_pd = nes_alloc_pd; 3880 nesibdev->ibdev.dealloc_pd = nes_dealloc_pd; 3881 nesibdev->ibdev.create_ah = nes_create_ah; 3882 nesibdev->ibdev.destroy_ah = nes_destroy_ah; 3883 nesibdev->ibdev.create_qp = nes_create_qp; 3884 nesibdev->ibdev.modify_qp = nes_modify_qp; 3885 nesibdev->ibdev.query_qp = nes_query_qp; 3886 nesibdev->ibdev.destroy_qp = nes_destroy_qp; 3887 nesibdev->ibdev.create_cq = nes_create_cq; 3888 nesibdev->ibdev.destroy_cq = nes_destroy_cq; 3889 nesibdev->ibdev.poll_cq = nes_poll_cq; 3890 nesibdev->ibdev.get_dma_mr = nes_get_dma_mr; 3891 nesibdev->ibdev.reg_phys_mr = nes_reg_phys_mr; 3892 nesibdev->ibdev.reg_user_mr = nes_reg_user_mr; 3893 nesibdev->ibdev.dereg_mr = nes_dereg_mr; 3894 nesibdev->ibdev.alloc_mw = nes_alloc_mw; 3895 nesibdev->ibdev.dealloc_mw = nes_dealloc_mw; 3896 nesibdev->ibdev.bind_mw = nes_bind_mw; 3897 3898 nesibdev->ibdev.alloc_mr = nes_alloc_mr; 3899 nesibdev->ibdev.map_mr_sg = nes_map_mr_sg; 3900 3901 nesibdev->ibdev.attach_mcast = nes_multicast_attach; 3902 nesibdev->ibdev.detach_mcast = nes_multicast_detach; 3903 nesibdev->ibdev.process_mad = nes_process_mad; 3904 3905 nesibdev->ibdev.req_notify_cq = nes_req_notify_cq; 3906 nesibdev->ibdev.post_send = nes_post_send; 3907 nesibdev->ibdev.post_recv = nes_post_recv; 3908 3909 nesibdev->ibdev.iwcm = kzalloc(sizeof(*nesibdev->ibdev.iwcm), GFP_KERNEL); 3910 if (nesibdev->ibdev.iwcm == NULL) { 3911 ib_dealloc_device(&nesibdev->ibdev); 3912 return NULL; 3913 } 3914 nesibdev->ibdev.iwcm->add_ref = nes_add_ref; 3915 nesibdev->ibdev.iwcm->rem_ref = nes_rem_ref; 3916 nesibdev->ibdev.iwcm->get_qp = nes_get_qp; 3917 nesibdev->ibdev.iwcm->connect = nes_connect; 3918 nesibdev->ibdev.iwcm->accept = nes_accept; 3919 nesibdev->ibdev.iwcm->reject = nes_reject; 3920 nesibdev->ibdev.iwcm->create_listen = nes_create_listen; 3921 nesibdev->ibdev.iwcm->destroy_listen = nes_destroy_listen; 3922 nesibdev->ibdev.get_port_immutable = nes_port_immutable; 3923 3924 return nesibdev; 3925} 3926 3927 3928/** 3929 * nes_handle_delayed_event 3930 */ 3931static void nes_handle_delayed_event(unsigned long data) 3932{ 3933 struct nes_vnic *nesvnic = (void *) data; 3934 3935 if (nesvnic->delayed_event != nesvnic->last_dispatched_event) { 3936 struct ib_event event; 3937 3938 event.device = &nesvnic->nesibdev->ibdev; 3939 if (!event.device) 3940 goto stop_timer; 3941 event.event = nesvnic->delayed_event; 3942 event.element.port_num = nesvnic->logical_port + 1; 3943 ib_dispatch_event(&event); 3944 } 3945 3946stop_timer: 3947 nesvnic->event_timer.function = NULL; 3948} 3949 3950 3951void nes_port_ibevent(struct nes_vnic *nesvnic) 3952{ 3953 struct nes_ib_device *nesibdev = nesvnic->nesibdev; 3954 struct nes_device *nesdev = nesvnic->nesdev; 3955 struct ib_event event; 3956 event.device = &nesibdev->ibdev; 3957 event.element.port_num = nesvnic->logical_port + 1; 3958 event.event = nesdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; 3959 3960 if (!nesvnic->event_timer.function) { 3961 ib_dispatch_event(&event); 3962 nesvnic->last_dispatched_event = event.event; 3963 nesvnic->event_timer.function = nes_handle_delayed_event; 3964 nesvnic->event_timer.data = (unsigned long) nesvnic; 3965 nesvnic->event_timer.expires = jiffies + NES_EVENT_DELAY; 3966 add_timer(&nesvnic->event_timer); 3967 } else { 3968 mod_timer(&nesvnic->event_timer, jiffies + NES_EVENT_DELAY); 3969 } 3970 nesvnic->delayed_event = event.event; 3971} 3972 3973 3974/** 3975 * nes_destroy_ofa_device 3976 */ 3977void nes_destroy_ofa_device(struct nes_ib_device *nesibdev) 3978{ 3979 if (nesibdev == NULL) 3980 return; 3981 3982 nes_unregister_ofa_device(nesibdev); 3983 3984 kfree(nesibdev->ibdev.iwcm); 3985 ib_dealloc_device(&nesibdev->ibdev); 3986} 3987 3988 3989/** 3990 * nes_register_ofa_device 3991 */ 3992int nes_register_ofa_device(struct nes_ib_device *nesibdev) 3993{ 3994 struct nes_vnic *nesvnic = nesibdev->nesvnic; 3995 struct nes_device *nesdev = nesvnic->nesdev; 3996 struct nes_adapter *nesadapter = nesdev->nesadapter; 3997 int i, ret; 3998 3999 ret = ib_register_device(&nesvnic->nesibdev->ibdev, NULL); 4000 if (ret) { 4001 return ret; 4002 } 4003 4004 /* Get the resources allocated to this device */ 4005 nesibdev->max_cq = (nesadapter->max_cq-NES_FIRST_QPN) / nesadapter->port_count; 4006 nesibdev->max_mr = nesadapter->max_mr / nesadapter->port_count; 4007 nesibdev->max_qp = (nesadapter->max_qp-NES_FIRST_QPN) / nesadapter->port_count; 4008 nesibdev->max_pd = nesadapter->max_pd / nesadapter->port_count; 4009 4010 for (i = 0; i < ARRAY_SIZE(nes_dev_attributes); ++i) { 4011 ret = device_create_file(&nesibdev->ibdev.dev, nes_dev_attributes[i]); 4012 if (ret) { 4013 while (i > 0) { 4014 i--; 4015 device_remove_file(&nesibdev->ibdev.dev, 4016 nes_dev_attributes[i]); 4017 } 4018 ib_unregister_device(&nesibdev->ibdev); 4019 return ret; 4020 } 4021 } 4022 4023 nesvnic->of_device_registered = 1; 4024 4025 return 0; 4026} 4027 4028 4029/** 4030 * nes_unregister_ofa_device 4031 */ 4032static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev) 4033{ 4034 struct nes_vnic *nesvnic = nesibdev->nesvnic; 4035 int i; 4036 4037 for (i = 0; i < ARRAY_SIZE(nes_dev_attributes); ++i) { 4038 device_remove_file(&nesibdev->ibdev.dev, nes_dev_attributes[i]); 4039 } 4040 4041 if (nesvnic->of_device_registered) { 4042 ib_unregister_device(&nesibdev->ibdev); 4043 } 4044 4045 nesvnic->of_device_registered = 0; 4046} 4047