1/* 2 * Copyright (c) 2012, 2013 Intel Corporation. All rights reserved. 3 * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved. 4 * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35#include <linux/pci.h> 36#include <linux/netdevice.h> 37#include <linux/vmalloc.h> 38#include <linux/delay.h> 39#include <linux/idr.h> 40#include <linux/module.h> 41#include <linux/printk.h> 42#ifdef CONFIG_INFINIBAND_QIB_DCA 43#include <linux/dca.h> 44#endif 45 46#include "qib.h" 47#include "qib_common.h" 48#include "qib_mad.h" 49#ifdef CONFIG_DEBUG_FS 50#include "qib_debugfs.h" 51#include "qib_verbs.h" 52#endif 53 54#undef pr_fmt 55#define pr_fmt(fmt) QIB_DRV_NAME ": " fmt 56 57/* 58 * min buffers we want to have per context, after driver 59 */ 60#define QIB_MIN_USER_CTXT_BUFCNT 7 61 62#define QLOGIC_IB_R_SOFTWARE_MASK 0xFF 63#define QLOGIC_IB_R_SOFTWARE_SHIFT 24 64#define QLOGIC_IB_R_EMULATOR_MASK (1ULL<<62) 65 66/* 67 * Number of ctxts we are configured to use (to allow for more pio 68 * buffers per ctxt, etc.) Zero means use chip value. 69 */ 70ushort qib_cfgctxts; 71module_param_named(cfgctxts, qib_cfgctxts, ushort, S_IRUGO); 72MODULE_PARM_DESC(cfgctxts, "Set max number of contexts to use"); 73 74unsigned qib_numa_aware; 75module_param_named(numa_aware, qib_numa_aware, uint, S_IRUGO); 76MODULE_PARM_DESC(numa_aware, 77 "0 -> PSM allocation close to HCA, 1 -> PSM allocation local to process"); 78 79/* 80 * If set, do not write to any regs if avoidable, hack to allow 81 * check for deranged default register values. 82 */ 83ushort qib_mini_init; 84module_param_named(mini_init, qib_mini_init, ushort, S_IRUGO); 85MODULE_PARM_DESC(mini_init, "If set, do minimal diag init"); 86 87unsigned qib_n_krcv_queues; 88module_param_named(krcvqs, qib_n_krcv_queues, uint, S_IRUGO); 89MODULE_PARM_DESC(krcvqs, "number of kernel receive queues per IB port"); 90 91unsigned qib_cc_table_size; 92module_param_named(cc_table_size, qib_cc_table_size, uint, S_IRUGO); 93MODULE_PARM_DESC(cc_table_size, "Congestion control table entries 0 (CCA disabled - default), min = 128, max = 1984"); 94 95static void verify_interrupt(unsigned long); 96 97static struct idr qib_unit_table; 98u32 qib_cpulist_count; 99unsigned long *qib_cpulist; 100 101/* set number of contexts we'll actually use */ 102void qib_set_ctxtcnt(struct qib_devdata *dd) 103{ 104 if (!qib_cfgctxts) { 105 dd->cfgctxts = dd->first_user_ctxt + num_online_cpus(); 106 if (dd->cfgctxts > dd->ctxtcnt) 107 dd->cfgctxts = dd->ctxtcnt; 108 } else if (qib_cfgctxts < dd->num_pports) 109 dd->cfgctxts = dd->ctxtcnt; 110 else if (qib_cfgctxts <= dd->ctxtcnt) 111 dd->cfgctxts = qib_cfgctxts; 112 else 113 dd->cfgctxts = dd->ctxtcnt; 114 dd->freectxts = (dd->first_user_ctxt > dd->cfgctxts) ? 0 : 115 dd->cfgctxts - dd->first_user_ctxt; 116} 117 118/* 119 * Common code for creating the receive context array. 120 */ 121int qib_create_ctxts(struct qib_devdata *dd) 122{ 123 unsigned i; 124 int local_node_id = pcibus_to_node(dd->pcidev->bus); 125 126 if (local_node_id < 0) 127 local_node_id = numa_node_id(); 128 dd->assigned_node_id = local_node_id; 129 130 /* 131 * Allocate full ctxtcnt array, rather than just cfgctxts, because 132 * cleanup iterates across all possible ctxts. 133 */ 134 dd->rcd = kcalloc(dd->ctxtcnt, sizeof(*dd->rcd), GFP_KERNEL); 135 if (!dd->rcd) { 136 qib_dev_err(dd, 137 "Unable to allocate ctxtdata array, failing\n"); 138 return -ENOMEM; 139 } 140 141 /* create (one or more) kctxt */ 142 for (i = 0; i < dd->first_user_ctxt; ++i) { 143 struct qib_pportdata *ppd; 144 struct qib_ctxtdata *rcd; 145 146 if (dd->skip_kctxt_mask & (1 << i)) 147 continue; 148 149 ppd = dd->pport + (i % dd->num_pports); 150 151 rcd = qib_create_ctxtdata(ppd, i, dd->assigned_node_id); 152 if (!rcd) { 153 qib_dev_err(dd, 154 "Unable to allocate ctxtdata for Kernel ctxt, failing\n"); 155 kfree(dd->rcd); 156 dd->rcd = NULL; 157 return -ENOMEM; 158 } 159 rcd->pkeys[0] = QIB_DEFAULT_P_KEY; 160 rcd->seq_cnt = 1; 161 } 162 return 0; 163} 164 165/* 166 * Common code for user and kernel context setup. 167 */ 168struct qib_ctxtdata *qib_create_ctxtdata(struct qib_pportdata *ppd, u32 ctxt, 169 int node_id) 170{ 171 struct qib_devdata *dd = ppd->dd; 172 struct qib_ctxtdata *rcd; 173 174 rcd = kzalloc_node(sizeof(*rcd), GFP_KERNEL, node_id); 175 if (rcd) { 176 INIT_LIST_HEAD(&rcd->qp_wait_list); 177 rcd->node_id = node_id; 178 rcd->ppd = ppd; 179 rcd->dd = dd; 180 rcd->cnt = 1; 181 rcd->ctxt = ctxt; 182 dd->rcd[ctxt] = rcd; 183#ifdef CONFIG_DEBUG_FS 184 if (ctxt < dd->first_user_ctxt) { /* N/A for PSM contexts */ 185 rcd->opstats = kzalloc_node(sizeof(*rcd->opstats), 186 GFP_KERNEL, node_id); 187 if (!rcd->opstats) { 188 kfree(rcd); 189 qib_dev_err(dd, 190 "Unable to allocate per ctxt stats buffer\n"); 191 return NULL; 192 } 193 } 194#endif 195 dd->f_init_ctxt(rcd); 196 197 /* 198 * To avoid wasting a lot of memory, we allocate 32KB chunks 199 * of physically contiguous memory, advance through it until 200 * used up and then allocate more. Of course, we need 201 * memory to store those extra pointers, now. 32KB seems to 202 * be the most that is "safe" under memory pressure 203 * (creating large files and then copying them over 204 * NFS while doing lots of MPI jobs). The OOM killer can 205 * get invoked, even though we say we can sleep and this can 206 * cause significant system problems.... 207 */ 208 rcd->rcvegrbuf_size = 0x8000; 209 rcd->rcvegrbufs_perchunk = 210 rcd->rcvegrbuf_size / dd->rcvegrbufsize; 211 rcd->rcvegrbuf_chunks = (rcd->rcvegrcnt + 212 rcd->rcvegrbufs_perchunk - 1) / 213 rcd->rcvegrbufs_perchunk; 214 BUG_ON(!is_power_of_2(rcd->rcvegrbufs_perchunk)); 215 rcd->rcvegrbufs_perchunk_shift = 216 ilog2(rcd->rcvegrbufs_perchunk); 217 } 218 return rcd; 219} 220 221/* 222 * Common code for initializing the physical port structure. 223 */ 224int qib_init_pportdata(struct qib_pportdata *ppd, struct qib_devdata *dd, 225 u8 hw_pidx, u8 port) 226{ 227 int size; 228 229 ppd->dd = dd; 230 ppd->hw_pidx = hw_pidx; 231 ppd->port = port; /* IB port number, not index */ 232 233 spin_lock_init(&ppd->sdma_lock); 234 spin_lock_init(&ppd->lflags_lock); 235 spin_lock_init(&ppd->cc_shadow_lock); 236 init_waitqueue_head(&ppd->state_wait); 237 238 init_timer(&ppd->symerr_clear_timer); 239 ppd->symerr_clear_timer.function = qib_clear_symerror_on_linkup; 240 ppd->symerr_clear_timer.data = (unsigned long)ppd; 241 242 ppd->qib_wq = NULL; 243 ppd->ibport_data.pmastats = 244 alloc_percpu(struct qib_pma_counters); 245 if (!ppd->ibport_data.pmastats) 246 return -ENOMEM; 247 248 if (qib_cc_table_size < IB_CCT_MIN_ENTRIES) 249 goto bail; 250 251 ppd->cc_supported_table_entries = min(max_t(int, qib_cc_table_size, 252 IB_CCT_MIN_ENTRIES), IB_CCT_ENTRIES*IB_CC_TABLE_CAP_DEFAULT); 253 254 ppd->cc_max_table_entries = 255 ppd->cc_supported_table_entries/IB_CCT_ENTRIES; 256 257 size = IB_CC_TABLE_CAP_DEFAULT * sizeof(struct ib_cc_table_entry) 258 * IB_CCT_ENTRIES; 259 ppd->ccti_entries = kzalloc(size, GFP_KERNEL); 260 if (!ppd->ccti_entries) { 261 qib_dev_err(dd, 262 "failed to allocate congestion control table for port %d!\n", 263 port); 264 goto bail; 265 } 266 267 size = IB_CC_CCS_ENTRIES * sizeof(struct ib_cc_congestion_entry); 268 ppd->congestion_entries = kzalloc(size, GFP_KERNEL); 269 if (!ppd->congestion_entries) { 270 qib_dev_err(dd, 271 "failed to allocate congestion setting list for port %d!\n", 272 port); 273 goto bail_1; 274 } 275 276 size = sizeof(struct cc_table_shadow); 277 ppd->ccti_entries_shadow = kzalloc(size, GFP_KERNEL); 278 if (!ppd->ccti_entries_shadow) { 279 qib_dev_err(dd, 280 "failed to allocate shadow ccti list for port %d!\n", 281 port); 282 goto bail_2; 283 } 284 285 size = sizeof(struct ib_cc_congestion_setting_attr); 286 ppd->congestion_entries_shadow = kzalloc(size, GFP_KERNEL); 287 if (!ppd->congestion_entries_shadow) { 288 qib_dev_err(dd, 289 "failed to allocate shadow congestion setting list for port %d!\n", 290 port); 291 goto bail_3; 292 } 293 294 return 0; 295 296bail_3: 297 kfree(ppd->ccti_entries_shadow); 298 ppd->ccti_entries_shadow = NULL; 299bail_2: 300 kfree(ppd->congestion_entries); 301 ppd->congestion_entries = NULL; 302bail_1: 303 kfree(ppd->ccti_entries); 304 ppd->ccti_entries = NULL; 305bail: 306 /* User is intentionally disabling the congestion control agent */ 307 if (!qib_cc_table_size) 308 return 0; 309 310 if (qib_cc_table_size < IB_CCT_MIN_ENTRIES) { 311 qib_cc_table_size = 0; 312 qib_dev_err(dd, 313 "Congestion Control table size %d less than minimum %d for port %d\n", 314 qib_cc_table_size, IB_CCT_MIN_ENTRIES, port); 315 } 316 317 qib_dev_err(dd, "Congestion Control Agent disabled for port %d\n", 318 port); 319 return 0; 320} 321 322static int init_pioavailregs(struct qib_devdata *dd) 323{ 324 int ret, pidx; 325 u64 *status_page; 326 327 dd->pioavailregs_dma = dma_alloc_coherent( 328 &dd->pcidev->dev, PAGE_SIZE, &dd->pioavailregs_phys, 329 GFP_KERNEL); 330 if (!dd->pioavailregs_dma) { 331 qib_dev_err(dd, 332 "failed to allocate PIOavail reg area in memory\n"); 333 ret = -ENOMEM; 334 goto done; 335 } 336 337 /* 338 * We really want L2 cache aligned, but for current CPUs of 339 * interest, they are the same. 340 */ 341 status_page = (u64 *) 342 ((char *) dd->pioavailregs_dma + 343 ((2 * L1_CACHE_BYTES + 344 dd->pioavregs * sizeof(u64)) & ~L1_CACHE_BYTES)); 345 /* device status comes first, for backwards compatibility */ 346 dd->devstatusp = status_page; 347 *status_page++ = 0; 348 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 349 dd->pport[pidx].statusp = status_page; 350 *status_page++ = 0; 351 } 352 353 /* 354 * Setup buffer to hold freeze and other messages, accessible to 355 * apps, following statusp. This is per-unit, not per port. 356 */ 357 dd->freezemsg = (char *) status_page; 358 *dd->freezemsg = 0; 359 /* length of msg buffer is "whatever is left" */ 360 ret = (char *) status_page - (char *) dd->pioavailregs_dma; 361 dd->freezelen = PAGE_SIZE - ret; 362 363 ret = 0; 364 365done: 366 return ret; 367} 368 369/** 370 * init_shadow_tids - allocate the shadow TID array 371 * @dd: the qlogic_ib device 372 * 373 * allocate the shadow TID array, so we can qib_munlock previous 374 * entries. It may make more sense to move the pageshadow to the 375 * ctxt data structure, so we only allocate memory for ctxts actually 376 * in use, since we at 8k per ctxt, now. 377 * We don't want failures here to prevent use of the driver/chip, 378 * so no return value. 379 */ 380static void init_shadow_tids(struct qib_devdata *dd) 381{ 382 struct page **pages; 383 dma_addr_t *addrs; 384 385 pages = vzalloc(dd->cfgctxts * dd->rcvtidcnt * sizeof(struct page *)); 386 if (!pages) { 387 qib_dev_err(dd, 388 "failed to allocate shadow page * array, no expected sends!\n"); 389 goto bail; 390 } 391 392 addrs = vzalloc(dd->cfgctxts * dd->rcvtidcnt * sizeof(dma_addr_t)); 393 if (!addrs) { 394 qib_dev_err(dd, 395 "failed to allocate shadow dma handle array, no expected sends!\n"); 396 goto bail_free; 397 } 398 399 dd->pageshadow = pages; 400 dd->physshadow = addrs; 401 return; 402 403bail_free: 404 vfree(pages); 405bail: 406 dd->pageshadow = NULL; 407} 408 409/* 410 * Do initialization for device that is only needed on 411 * first detect, not on resets. 412 */ 413static int loadtime_init(struct qib_devdata *dd) 414{ 415 int ret = 0; 416 417 if (((dd->revision >> QLOGIC_IB_R_SOFTWARE_SHIFT) & 418 QLOGIC_IB_R_SOFTWARE_MASK) != QIB_CHIP_SWVERSION) { 419 qib_dev_err(dd, 420 "Driver only handles version %d, chip swversion is %d (%llx), failng\n", 421 QIB_CHIP_SWVERSION, 422 (int)(dd->revision >> 423 QLOGIC_IB_R_SOFTWARE_SHIFT) & 424 QLOGIC_IB_R_SOFTWARE_MASK, 425 (unsigned long long) dd->revision); 426 ret = -ENOSYS; 427 goto done; 428 } 429 430 if (dd->revision & QLOGIC_IB_R_EMULATOR_MASK) 431 qib_devinfo(dd->pcidev, "%s", dd->boardversion); 432 433 spin_lock_init(&dd->pioavail_lock); 434 spin_lock_init(&dd->sendctrl_lock); 435 spin_lock_init(&dd->uctxt_lock); 436 spin_lock_init(&dd->qib_diag_trans_lock); 437 spin_lock_init(&dd->eep_st_lock); 438 mutex_init(&dd->eep_lock); 439 440 if (qib_mini_init) 441 goto done; 442 443 ret = init_pioavailregs(dd); 444 init_shadow_tids(dd); 445 446 qib_get_eeprom_info(dd); 447 448 /* setup time (don't start yet) to verify we got interrupt */ 449 init_timer(&dd->intrchk_timer); 450 dd->intrchk_timer.function = verify_interrupt; 451 dd->intrchk_timer.data = (unsigned long) dd; 452 453 ret = qib_cq_init(dd); 454done: 455 return ret; 456} 457 458/** 459 * init_after_reset - re-initialize after a reset 460 * @dd: the qlogic_ib device 461 * 462 * sanity check at least some of the values after reset, and 463 * ensure no receive or transmit (explicitly, in case reset 464 * failed 465 */ 466static int init_after_reset(struct qib_devdata *dd) 467{ 468 int i; 469 470 /* 471 * Ensure chip does no sends or receives, tail updates, or 472 * pioavail updates while we re-initialize. This is mostly 473 * for the driver data structures, not chip registers. 474 */ 475 for (i = 0; i < dd->num_pports; ++i) { 476 /* 477 * ctxt == -1 means "all contexts". Only really safe for 478 * _dis_abling things, as here. 479 */ 480 dd->f_rcvctrl(dd->pport + i, QIB_RCVCTRL_CTXT_DIS | 481 QIB_RCVCTRL_INTRAVAIL_DIS | 482 QIB_RCVCTRL_TAILUPD_DIS, -1); 483 /* Redundant across ports for some, but no big deal. */ 484 dd->f_sendctrl(dd->pport + i, QIB_SENDCTRL_SEND_DIS | 485 QIB_SENDCTRL_AVAIL_DIS); 486 } 487 488 return 0; 489} 490 491static void enable_chip(struct qib_devdata *dd) 492{ 493 u64 rcvmask; 494 int i; 495 496 /* 497 * Enable PIO send, and update of PIOavail regs to memory. 498 */ 499 for (i = 0; i < dd->num_pports; ++i) 500 dd->f_sendctrl(dd->pport + i, QIB_SENDCTRL_SEND_ENB | 501 QIB_SENDCTRL_AVAIL_ENB); 502 /* 503 * Enable kernel ctxts' receive and receive interrupt. 504 * Other ctxts done as user opens and inits them. 505 */ 506 rcvmask = QIB_RCVCTRL_CTXT_ENB | QIB_RCVCTRL_INTRAVAIL_ENB; 507 rcvmask |= (dd->flags & QIB_NODMA_RTAIL) ? 508 QIB_RCVCTRL_TAILUPD_DIS : QIB_RCVCTRL_TAILUPD_ENB; 509 for (i = 0; dd->rcd && i < dd->first_user_ctxt; ++i) { 510 struct qib_ctxtdata *rcd = dd->rcd[i]; 511 512 if (rcd) 513 dd->f_rcvctrl(rcd->ppd, rcvmask, i); 514 } 515} 516 517static void verify_interrupt(unsigned long opaque) 518{ 519 struct qib_devdata *dd = (struct qib_devdata *) opaque; 520 u64 int_counter; 521 522 if (!dd) 523 return; /* being torn down */ 524 525 /* 526 * If we don't have a lid or any interrupts, let the user know and 527 * don't bother checking again. 528 */ 529 int_counter = qib_int_counter(dd) - dd->z_int_counter; 530 if (int_counter == 0) { 531 if (!dd->f_intr_fallback(dd)) 532 dev_err(&dd->pcidev->dev, 533 "No interrupts detected, not usable.\n"); 534 else /* re-arm the timer to see if fallback works */ 535 mod_timer(&dd->intrchk_timer, jiffies + HZ/2); 536 } 537} 538 539static void init_piobuf_state(struct qib_devdata *dd) 540{ 541 int i, pidx; 542 u32 uctxts; 543 544 /* 545 * Ensure all buffers are free, and fifos empty. Buffers 546 * are common, so only do once for port 0. 547 * 548 * After enable and qib_chg_pioavailkernel so we can safely 549 * enable pioavail updates and PIOENABLE. After this, packets 550 * are ready and able to go out. 551 */ 552 dd->f_sendctrl(dd->pport, QIB_SENDCTRL_DISARM_ALL); 553 for (pidx = 0; pidx < dd->num_pports; ++pidx) 554 dd->f_sendctrl(dd->pport + pidx, QIB_SENDCTRL_FLUSH); 555 556 /* 557 * If not all sendbufs are used, add the one to each of the lower 558 * numbered contexts. pbufsctxt and lastctxt_piobuf are 559 * calculated in chip-specific code because it may cause some 560 * chip-specific adjustments to be made. 561 */ 562 uctxts = dd->cfgctxts - dd->first_user_ctxt; 563 dd->ctxts_extrabuf = dd->pbufsctxt ? 564 dd->lastctxt_piobuf - (dd->pbufsctxt * uctxts) : 0; 565 566 /* 567 * Set up the shadow copies of the piobufavail registers, 568 * which we compare against the chip registers for now, and 569 * the in memory DMA'ed copies of the registers. 570 * By now pioavail updates to memory should have occurred, so 571 * copy them into our working/shadow registers; this is in 572 * case something went wrong with abort, but mostly to get the 573 * initial values of the generation bit correct. 574 */ 575 for (i = 0; i < dd->pioavregs; i++) { 576 __le64 tmp; 577 578 tmp = dd->pioavailregs_dma[i]; 579 /* 580 * Don't need to worry about pioavailkernel here 581 * because we will call qib_chg_pioavailkernel() later 582 * in initialization, to busy out buffers as needed. 583 */ 584 dd->pioavailshadow[i] = le64_to_cpu(tmp); 585 } 586 while (i < ARRAY_SIZE(dd->pioavailshadow)) 587 dd->pioavailshadow[i++] = 0; /* for debugging sanity */ 588 589 /* after pioavailshadow is setup */ 590 qib_chg_pioavailkernel(dd, 0, dd->piobcnt2k + dd->piobcnt4k, 591 TXCHK_CHG_TYPE_KERN, NULL); 592 dd->f_initvl15_bufs(dd); 593} 594 595/** 596 * qib_create_workqueues - create per port workqueues 597 * @dd: the qlogic_ib device 598 */ 599static int qib_create_workqueues(struct qib_devdata *dd) 600{ 601 int pidx; 602 struct qib_pportdata *ppd; 603 604 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 605 ppd = dd->pport + pidx; 606 if (!ppd->qib_wq) { 607 char wq_name[8]; /* 3 + 2 + 1 + 1 + 1 */ 608 609 snprintf(wq_name, sizeof(wq_name), "qib%d_%d", 610 dd->unit, pidx); 611 ppd->qib_wq = 612 create_singlethread_workqueue(wq_name); 613 if (!ppd->qib_wq) 614 goto wq_error; 615 } 616 } 617 return 0; 618wq_error: 619 pr_err("create_singlethread_workqueue failed for port %d\n", 620 pidx + 1); 621 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 622 ppd = dd->pport + pidx; 623 if (ppd->qib_wq) { 624 destroy_workqueue(ppd->qib_wq); 625 ppd->qib_wq = NULL; 626 } 627 } 628 return -ENOMEM; 629} 630 631static void qib_free_pportdata(struct qib_pportdata *ppd) 632{ 633 free_percpu(ppd->ibport_data.pmastats); 634 ppd->ibport_data.pmastats = NULL; 635} 636 637/** 638 * qib_init - do the actual initialization sequence on the chip 639 * @dd: the qlogic_ib device 640 * @reinit: reinitializing, so don't allocate new memory 641 * 642 * Do the actual initialization sequence on the chip. This is done 643 * both from the init routine called from the PCI infrastructure, and 644 * when we reset the chip, or detect that it was reset internally, 645 * or it's administratively re-enabled. 646 * 647 * Memory allocation here and in called routines is only done in 648 * the first case (reinit == 0). We have to be careful, because even 649 * without memory allocation, we need to re-write all the chip registers 650 * TIDs, etc. after the reset or enable has completed. 651 */ 652int qib_init(struct qib_devdata *dd, int reinit) 653{ 654 int ret = 0, pidx, lastfail = 0; 655 u32 portok = 0; 656 unsigned i; 657 struct qib_ctxtdata *rcd; 658 struct qib_pportdata *ppd; 659 unsigned long flags; 660 661 /* Set linkstate to unknown, so we can watch for a transition. */ 662 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 663 ppd = dd->pport + pidx; 664 spin_lock_irqsave(&ppd->lflags_lock, flags); 665 ppd->lflags &= ~(QIBL_LINKACTIVE | QIBL_LINKARMED | 666 QIBL_LINKDOWN | QIBL_LINKINIT | 667 QIBL_LINKV); 668 spin_unlock_irqrestore(&ppd->lflags_lock, flags); 669 } 670 671 if (reinit) 672 ret = init_after_reset(dd); 673 else 674 ret = loadtime_init(dd); 675 if (ret) 676 goto done; 677 678 /* Bypass most chip-init, to get to device creation */ 679 if (qib_mini_init) 680 return 0; 681 682 ret = dd->f_late_initreg(dd); 683 if (ret) 684 goto done; 685 686 /* dd->rcd can be NULL if early init failed */ 687 for (i = 0; dd->rcd && i < dd->first_user_ctxt; ++i) { 688 /* 689 * Set up the (kernel) rcvhdr queue and egr TIDs. If doing 690 * re-init, the simplest way to handle this is to free 691 * existing, and re-allocate. 692 * Need to re-create rest of ctxt 0 ctxtdata as well. 693 */ 694 rcd = dd->rcd[i]; 695 if (!rcd) 696 continue; 697 698 lastfail = qib_create_rcvhdrq(dd, rcd); 699 if (!lastfail) 700 lastfail = qib_setup_eagerbufs(rcd); 701 if (lastfail) { 702 qib_dev_err(dd, 703 "failed to allocate kernel ctxt's rcvhdrq and/or egr bufs\n"); 704 continue; 705 } 706 } 707 708 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 709 int mtu; 710 711 if (lastfail) 712 ret = lastfail; 713 ppd = dd->pport + pidx; 714 mtu = ib_mtu_enum_to_int(qib_ibmtu); 715 if (mtu == -1) { 716 mtu = QIB_DEFAULT_MTU; 717 qib_ibmtu = 0; /* don't leave invalid value */ 718 } 719 /* set max we can ever have for this driver load */ 720 ppd->init_ibmaxlen = min(mtu > 2048 ? 721 dd->piosize4k : dd->piosize2k, 722 dd->rcvegrbufsize + 723 (dd->rcvhdrentsize << 2)); 724 /* 725 * Have to initialize ibmaxlen, but this will normally 726 * change immediately in qib_set_mtu(). 727 */ 728 ppd->ibmaxlen = ppd->init_ibmaxlen; 729 qib_set_mtu(ppd, mtu); 730 731 spin_lock_irqsave(&ppd->lflags_lock, flags); 732 ppd->lflags |= QIBL_IB_LINK_DISABLED; 733 spin_unlock_irqrestore(&ppd->lflags_lock, flags); 734 735 lastfail = dd->f_bringup_serdes(ppd); 736 if (lastfail) { 737 qib_devinfo(dd->pcidev, 738 "Failed to bringup IB port %u\n", ppd->port); 739 lastfail = -ENETDOWN; 740 continue; 741 } 742 743 portok++; 744 } 745 746 if (!portok) { 747 /* none of the ports initialized */ 748 if (!ret && lastfail) 749 ret = lastfail; 750 else if (!ret) 751 ret = -ENETDOWN; 752 /* but continue on, so we can debug cause */ 753 } 754 755 enable_chip(dd); 756 757 init_piobuf_state(dd); 758 759done: 760 if (!ret) { 761 /* chip is OK for user apps; mark it as initialized */ 762 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 763 ppd = dd->pport + pidx; 764 /* 765 * Set status even if port serdes is not initialized 766 * so that diags will work. 767 */ 768 *ppd->statusp |= QIB_STATUS_CHIP_PRESENT | 769 QIB_STATUS_INITTED; 770 if (!ppd->link_speed_enabled) 771 continue; 772 if (dd->flags & QIB_HAS_SEND_DMA) 773 ret = qib_setup_sdma(ppd); 774 init_timer(&ppd->hol_timer); 775 ppd->hol_timer.function = qib_hol_event; 776 ppd->hol_timer.data = (unsigned long)ppd; 777 ppd->hol_state = QIB_HOL_UP; 778 } 779 780 /* now we can enable all interrupts from the chip */ 781 dd->f_set_intr_state(dd, 1); 782 783 /* 784 * Setup to verify we get an interrupt, and fallback 785 * to an alternate if necessary and possible. 786 */ 787 mod_timer(&dd->intrchk_timer, jiffies + HZ/2); 788 /* start stats retrieval timer */ 789 mod_timer(&dd->stats_timer, jiffies + HZ * ACTIVITY_TIMER); 790 } 791 792 /* if ret is non-zero, we probably should do some cleanup here... */ 793 return ret; 794} 795 796/* 797 * These next two routines are placeholders in case we don't have per-arch 798 * code for controlling write combining. If explicit control of write 799 * combining is not available, performance will probably be awful. 800 */ 801 802int __attribute__((weak)) qib_enable_wc(struct qib_devdata *dd) 803{ 804 return -EOPNOTSUPP; 805} 806 807void __attribute__((weak)) qib_disable_wc(struct qib_devdata *dd) 808{ 809} 810 811static inline struct qib_devdata *__qib_lookup(int unit) 812{ 813 return idr_find(&qib_unit_table, unit); 814} 815 816struct qib_devdata *qib_lookup(int unit) 817{ 818 struct qib_devdata *dd; 819 unsigned long flags; 820 821 spin_lock_irqsave(&qib_devs_lock, flags); 822 dd = __qib_lookup(unit); 823 spin_unlock_irqrestore(&qib_devs_lock, flags); 824 825 return dd; 826} 827 828/* 829 * Stop the timers during unit shutdown, or after an error late 830 * in initialization. 831 */ 832static void qib_stop_timers(struct qib_devdata *dd) 833{ 834 struct qib_pportdata *ppd; 835 int pidx; 836 837 if (dd->stats_timer.data) { 838 del_timer_sync(&dd->stats_timer); 839 dd->stats_timer.data = 0; 840 } 841 if (dd->intrchk_timer.data) { 842 del_timer_sync(&dd->intrchk_timer); 843 dd->intrchk_timer.data = 0; 844 } 845 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 846 ppd = dd->pport + pidx; 847 if (ppd->hol_timer.data) 848 del_timer_sync(&ppd->hol_timer); 849 if (ppd->led_override_timer.data) { 850 del_timer_sync(&ppd->led_override_timer); 851 atomic_set(&ppd->led_override_timer_active, 0); 852 } 853 if (ppd->symerr_clear_timer.data) 854 del_timer_sync(&ppd->symerr_clear_timer); 855 } 856} 857 858/** 859 * qib_shutdown_device - shut down a device 860 * @dd: the qlogic_ib device 861 * 862 * This is called to make the device quiet when we are about to 863 * unload the driver, and also when the device is administratively 864 * disabled. It does not free any data structures. 865 * Everything it does has to be setup again by qib_init(dd, 1) 866 */ 867static void qib_shutdown_device(struct qib_devdata *dd) 868{ 869 struct qib_pportdata *ppd; 870 unsigned pidx; 871 872 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 873 ppd = dd->pport + pidx; 874 875 spin_lock_irq(&ppd->lflags_lock); 876 ppd->lflags &= ~(QIBL_LINKDOWN | QIBL_LINKINIT | 877 QIBL_LINKARMED | QIBL_LINKACTIVE | 878 QIBL_LINKV); 879 spin_unlock_irq(&ppd->lflags_lock); 880 *ppd->statusp &= ~(QIB_STATUS_IB_CONF | QIB_STATUS_IB_READY); 881 } 882 dd->flags &= ~QIB_INITTED; 883 884 /* mask interrupts, but not errors */ 885 dd->f_set_intr_state(dd, 0); 886 887 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 888 ppd = dd->pport + pidx; 889 dd->f_rcvctrl(ppd, QIB_RCVCTRL_TAILUPD_DIS | 890 QIB_RCVCTRL_CTXT_DIS | 891 QIB_RCVCTRL_INTRAVAIL_DIS | 892 QIB_RCVCTRL_PKEY_ENB, -1); 893 /* 894 * Gracefully stop all sends allowing any in progress to 895 * trickle out first. 896 */ 897 dd->f_sendctrl(ppd, QIB_SENDCTRL_CLEAR); 898 } 899 900 /* 901 * Enough for anything that's going to trickle out to have actually 902 * done so. 903 */ 904 udelay(20); 905 906 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 907 ppd = dd->pport + pidx; 908 dd->f_setextled(ppd, 0); /* make sure LEDs are off */ 909 910 if (dd->flags & QIB_HAS_SEND_DMA) 911 qib_teardown_sdma(ppd); 912 913 dd->f_sendctrl(ppd, QIB_SENDCTRL_AVAIL_DIS | 914 QIB_SENDCTRL_SEND_DIS); 915 /* 916 * Clear SerdesEnable. 917 * We can't count on interrupts since we are stopping. 918 */ 919 dd->f_quiet_serdes(ppd); 920 921 if (ppd->qib_wq) { 922 destroy_workqueue(ppd->qib_wq); 923 ppd->qib_wq = NULL; 924 } 925 qib_free_pportdata(ppd); 926 } 927 928} 929 930/** 931 * qib_free_ctxtdata - free a context's allocated data 932 * @dd: the qlogic_ib device 933 * @rcd: the ctxtdata structure 934 * 935 * free up any allocated data for a context 936 * This should not touch anything that would affect a simultaneous 937 * re-allocation of context data, because it is called after qib_mutex 938 * is released (and can be called from reinit as well). 939 * It should never change any chip state, or global driver state. 940 */ 941void qib_free_ctxtdata(struct qib_devdata *dd, struct qib_ctxtdata *rcd) 942{ 943 if (!rcd) 944 return; 945 946 if (rcd->rcvhdrq) { 947 dma_free_coherent(&dd->pcidev->dev, rcd->rcvhdrq_size, 948 rcd->rcvhdrq, rcd->rcvhdrq_phys); 949 rcd->rcvhdrq = NULL; 950 if (rcd->rcvhdrtail_kvaddr) { 951 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE, 952 rcd->rcvhdrtail_kvaddr, 953 rcd->rcvhdrqtailaddr_phys); 954 rcd->rcvhdrtail_kvaddr = NULL; 955 } 956 } 957 if (rcd->rcvegrbuf) { 958 unsigned e; 959 960 for (e = 0; e < rcd->rcvegrbuf_chunks; e++) { 961 void *base = rcd->rcvegrbuf[e]; 962 size_t size = rcd->rcvegrbuf_size; 963 964 dma_free_coherent(&dd->pcidev->dev, size, 965 base, rcd->rcvegrbuf_phys[e]); 966 } 967 kfree(rcd->rcvegrbuf); 968 rcd->rcvegrbuf = NULL; 969 kfree(rcd->rcvegrbuf_phys); 970 rcd->rcvegrbuf_phys = NULL; 971 rcd->rcvegrbuf_chunks = 0; 972 } 973 974 kfree(rcd->tid_pg_list); 975 vfree(rcd->user_event_mask); 976 vfree(rcd->subctxt_uregbase); 977 vfree(rcd->subctxt_rcvegrbuf); 978 vfree(rcd->subctxt_rcvhdr_base); 979#ifdef CONFIG_DEBUG_FS 980 kfree(rcd->opstats); 981 rcd->opstats = NULL; 982#endif 983 kfree(rcd); 984} 985 986/* 987 * Perform a PIO buffer bandwidth write test, to verify proper system 988 * configuration. Even when all the setup calls work, occasionally 989 * BIOS or other issues can prevent write combining from working, or 990 * can cause other bandwidth problems to the chip. 991 * 992 * This test simply writes the same buffer over and over again, and 993 * measures close to the peak bandwidth to the chip (not testing 994 * data bandwidth to the wire). On chips that use an address-based 995 * trigger to send packets to the wire, this is easy. On chips that 996 * use a count to trigger, we want to make sure that the packet doesn't 997 * go out on the wire, or trigger flow control checks. 998 */ 999static void qib_verify_pioperf(struct qib_devdata *dd) 1000{ 1001 u32 pbnum, cnt, lcnt; 1002 u32 __iomem *piobuf; 1003 u32 *addr; 1004 u64 msecs, emsecs; 1005 1006 piobuf = dd->f_getsendbuf(dd->pport, 0ULL, &pbnum); 1007 if (!piobuf) { 1008 qib_devinfo(dd->pcidev, 1009 "No PIObufs for checking perf, skipping\n"); 1010 return; 1011 } 1012 1013 /* 1014 * Enough to give us a reasonable test, less than piobuf size, and 1015 * likely multiple of store buffer length. 1016 */ 1017 cnt = 1024; 1018 1019 addr = vmalloc(cnt); 1020 if (!addr) { 1021 qib_devinfo(dd->pcidev, 1022 "Couldn't get memory for checking PIO perf, skipping\n"); 1023 goto done; 1024 } 1025 1026 preempt_disable(); /* we want reasonably accurate elapsed time */ 1027 msecs = 1 + jiffies_to_msecs(jiffies); 1028 for (lcnt = 0; lcnt < 10000U; lcnt++) { 1029 /* wait until we cross msec boundary */ 1030 if (jiffies_to_msecs(jiffies) >= msecs) 1031 break; 1032 udelay(1); 1033 } 1034 1035 dd->f_set_armlaunch(dd, 0); 1036 1037 /* 1038 * length 0, no dwords actually sent 1039 */ 1040 writeq(0, piobuf); 1041 qib_flush_wc(); 1042 1043 /* 1044 * This is only roughly accurate, since even with preempt we 1045 * still take interrupts that could take a while. Running for 1046 * >= 5 msec seems to get us "close enough" to accurate values. 1047 */ 1048 msecs = jiffies_to_msecs(jiffies); 1049 for (emsecs = lcnt = 0; emsecs <= 5UL; lcnt++) { 1050 qib_pio_copy(piobuf + 64, addr, cnt >> 2); 1051 emsecs = jiffies_to_msecs(jiffies) - msecs; 1052 } 1053 1054 /* 1 GiB/sec, slightly over IB SDR line rate */ 1055 if (lcnt < (emsecs * 1024U)) 1056 qib_dev_err(dd, 1057 "Performance problem: bandwidth to PIO buffers is only %u MiB/sec\n", 1058 lcnt / (u32) emsecs); 1059 1060 preempt_enable(); 1061 1062 vfree(addr); 1063 1064done: 1065 /* disarm piobuf, so it's available again */ 1066 dd->f_sendctrl(dd->pport, QIB_SENDCTRL_DISARM_BUF(pbnum)); 1067 qib_sendbuf_done(dd, pbnum); 1068 dd->f_set_armlaunch(dd, 1); 1069} 1070 1071void qib_free_devdata(struct qib_devdata *dd) 1072{ 1073 unsigned long flags; 1074 1075 spin_lock_irqsave(&qib_devs_lock, flags); 1076 idr_remove(&qib_unit_table, dd->unit); 1077 list_del(&dd->list); 1078 spin_unlock_irqrestore(&qib_devs_lock, flags); 1079 1080#ifdef CONFIG_DEBUG_FS 1081 qib_dbg_ibdev_exit(&dd->verbs_dev); 1082#endif 1083 free_percpu(dd->int_counter); 1084 ib_dealloc_device(&dd->verbs_dev.ibdev); 1085} 1086 1087u64 qib_int_counter(struct qib_devdata *dd) 1088{ 1089 int cpu; 1090 u64 int_counter = 0; 1091 1092 for_each_possible_cpu(cpu) 1093 int_counter += *per_cpu_ptr(dd->int_counter, cpu); 1094 return int_counter; 1095} 1096 1097u64 qib_sps_ints(void) 1098{ 1099 unsigned long flags; 1100 struct qib_devdata *dd; 1101 u64 sps_ints = 0; 1102 1103 spin_lock_irqsave(&qib_devs_lock, flags); 1104 list_for_each_entry(dd, &qib_dev_list, list) { 1105 sps_ints += qib_int_counter(dd); 1106 } 1107 spin_unlock_irqrestore(&qib_devs_lock, flags); 1108 return sps_ints; 1109} 1110 1111/* 1112 * Allocate our primary per-unit data structure. Must be done via verbs 1113 * allocator, because the verbs cleanup process both does cleanup and 1114 * free of the data structure. 1115 * "extra" is for chip-specific data. 1116 * 1117 * Use the idr mechanism to get a unit number for this unit. 1118 */ 1119struct qib_devdata *qib_alloc_devdata(struct pci_dev *pdev, size_t extra) 1120{ 1121 unsigned long flags; 1122 struct qib_devdata *dd; 1123 int ret; 1124 1125 dd = (struct qib_devdata *) ib_alloc_device(sizeof(*dd) + extra); 1126 if (!dd) 1127 return ERR_PTR(-ENOMEM); 1128 1129 INIT_LIST_HEAD(&dd->list); 1130 1131 idr_preload(GFP_KERNEL); 1132 spin_lock_irqsave(&qib_devs_lock, flags); 1133 1134 ret = idr_alloc(&qib_unit_table, dd, 0, 0, GFP_NOWAIT); 1135 if (ret >= 0) { 1136 dd->unit = ret; 1137 list_add(&dd->list, &qib_dev_list); 1138 } 1139 1140 spin_unlock_irqrestore(&qib_devs_lock, flags); 1141 idr_preload_end(); 1142 1143 if (ret < 0) { 1144 qib_early_err(&pdev->dev, 1145 "Could not allocate unit ID: error %d\n", -ret); 1146 goto bail; 1147 } 1148 dd->int_counter = alloc_percpu(u64); 1149 if (!dd->int_counter) { 1150 ret = -ENOMEM; 1151 qib_early_err(&pdev->dev, 1152 "Could not allocate per-cpu int_counter\n"); 1153 goto bail; 1154 } 1155 1156 if (!qib_cpulist_count) { 1157 u32 count = num_online_cpus(); 1158 1159 qib_cpulist = kzalloc(BITS_TO_LONGS(count) * 1160 sizeof(long), GFP_KERNEL); 1161 if (qib_cpulist) 1162 qib_cpulist_count = count; 1163 else 1164 qib_early_err(&pdev->dev, 1165 "Could not alloc cpulist info, cpu affinity might be wrong\n"); 1166 } 1167#ifdef CONFIG_DEBUG_FS 1168 qib_dbg_ibdev_init(&dd->verbs_dev); 1169#endif 1170 return dd; 1171bail: 1172 if (!list_empty(&dd->list)) 1173 list_del_init(&dd->list); 1174 ib_dealloc_device(&dd->verbs_dev.ibdev); 1175 return ERR_PTR(ret); 1176} 1177 1178/* 1179 * Called from freeze mode handlers, and from PCI error 1180 * reporting code. Should be paranoid about state of 1181 * system and data structures. 1182 */ 1183void qib_disable_after_error(struct qib_devdata *dd) 1184{ 1185 if (dd->flags & QIB_INITTED) { 1186 u32 pidx; 1187 1188 dd->flags &= ~QIB_INITTED; 1189 if (dd->pport) 1190 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 1191 struct qib_pportdata *ppd; 1192 1193 ppd = dd->pport + pidx; 1194 if (dd->flags & QIB_PRESENT) { 1195 qib_set_linkstate(ppd, 1196 QIB_IB_LINKDOWN_DISABLE); 1197 dd->f_setextled(ppd, 0); 1198 } 1199 *ppd->statusp &= ~QIB_STATUS_IB_READY; 1200 } 1201 } 1202 1203 /* 1204 * Mark as having had an error for driver, and also 1205 * for /sys and status word mapped to user programs. 1206 * This marks unit as not usable, until reset. 1207 */ 1208 if (dd->devstatusp) 1209 *dd->devstatusp |= QIB_STATUS_HWERROR; 1210} 1211 1212static void qib_remove_one(struct pci_dev *); 1213static int qib_init_one(struct pci_dev *, const struct pci_device_id *); 1214 1215#define DRIVER_LOAD_MSG "Intel " QIB_DRV_NAME " loaded: " 1216#define PFX QIB_DRV_NAME ": " 1217 1218static const struct pci_device_id qib_pci_tbl[] = { 1219 { PCI_DEVICE(PCI_VENDOR_ID_PATHSCALE, PCI_DEVICE_ID_QLOGIC_IB_6120) }, 1220 { PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_IB_7220) }, 1221 { PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_IB_7322) }, 1222 { 0, } 1223}; 1224 1225MODULE_DEVICE_TABLE(pci, qib_pci_tbl); 1226 1227static struct pci_driver qib_driver = { 1228 .name = QIB_DRV_NAME, 1229 .probe = qib_init_one, 1230 .remove = qib_remove_one, 1231 .id_table = qib_pci_tbl, 1232 .err_handler = &qib_pci_err_handler, 1233}; 1234 1235#ifdef CONFIG_INFINIBAND_QIB_DCA 1236 1237static int qib_notify_dca(struct notifier_block *, unsigned long, void *); 1238static struct notifier_block dca_notifier = { 1239 .notifier_call = qib_notify_dca, 1240 .next = NULL, 1241 .priority = 0 1242}; 1243 1244static int qib_notify_dca_device(struct device *device, void *data) 1245{ 1246 struct qib_devdata *dd = dev_get_drvdata(device); 1247 unsigned long event = *(unsigned long *)data; 1248 1249 return dd->f_notify_dca(dd, event); 1250} 1251 1252static int qib_notify_dca(struct notifier_block *nb, unsigned long event, 1253 void *p) 1254{ 1255 int rval; 1256 1257 rval = driver_for_each_device(&qib_driver.driver, NULL, 1258 &event, qib_notify_dca_device); 1259 return rval ? NOTIFY_BAD : NOTIFY_DONE; 1260} 1261 1262#endif 1263 1264/* 1265 * Do all the generic driver unit- and chip-independent memory 1266 * allocation and initialization. 1267 */ 1268static int __init qib_ib_init(void) 1269{ 1270 int ret; 1271 1272 ret = qib_dev_init(); 1273 if (ret) 1274 goto bail; 1275 1276 /* 1277 * These must be called before the driver is registered with 1278 * the PCI subsystem. 1279 */ 1280 idr_init(&qib_unit_table); 1281 1282#ifdef CONFIG_INFINIBAND_QIB_DCA 1283 dca_register_notify(&dca_notifier); 1284#endif 1285#ifdef CONFIG_DEBUG_FS 1286 qib_dbg_init(); 1287#endif 1288 ret = pci_register_driver(&qib_driver); 1289 if (ret < 0) { 1290 pr_err("Unable to register driver: error %d\n", -ret); 1291 goto bail_dev; 1292 } 1293 1294 /* not fatal if it doesn't work */ 1295 if (qib_init_qibfs()) 1296 pr_err("Unable to register ipathfs\n"); 1297 goto bail; /* all OK */ 1298 1299bail_dev: 1300#ifdef CONFIG_INFINIBAND_QIB_DCA 1301 dca_unregister_notify(&dca_notifier); 1302#endif 1303#ifdef CONFIG_DEBUG_FS 1304 qib_dbg_exit(); 1305#endif 1306 idr_destroy(&qib_unit_table); 1307 qib_dev_cleanup(); 1308bail: 1309 return ret; 1310} 1311 1312module_init(qib_ib_init); 1313 1314/* 1315 * Do the non-unit driver cleanup, memory free, etc. at unload. 1316 */ 1317static void __exit qib_ib_cleanup(void) 1318{ 1319 int ret; 1320 1321 ret = qib_exit_qibfs(); 1322 if (ret) 1323 pr_err( 1324 "Unable to cleanup counter filesystem: error %d\n", 1325 -ret); 1326 1327#ifdef CONFIG_INFINIBAND_QIB_DCA 1328 dca_unregister_notify(&dca_notifier); 1329#endif 1330 pci_unregister_driver(&qib_driver); 1331#ifdef CONFIG_DEBUG_FS 1332 qib_dbg_exit(); 1333#endif 1334 1335 qib_cpulist_count = 0; 1336 kfree(qib_cpulist); 1337 1338 idr_destroy(&qib_unit_table); 1339 qib_dev_cleanup(); 1340} 1341 1342module_exit(qib_ib_cleanup); 1343 1344/* this can only be called after a successful initialization */ 1345static void cleanup_device_data(struct qib_devdata *dd) 1346{ 1347 int ctxt; 1348 int pidx; 1349 struct qib_ctxtdata **tmp; 1350 unsigned long flags; 1351 1352 /* users can't do anything more with chip */ 1353 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 1354 if (dd->pport[pidx].statusp) 1355 *dd->pport[pidx].statusp &= ~QIB_STATUS_CHIP_PRESENT; 1356 1357 spin_lock(&dd->pport[pidx].cc_shadow_lock); 1358 1359 kfree(dd->pport[pidx].congestion_entries); 1360 dd->pport[pidx].congestion_entries = NULL; 1361 kfree(dd->pport[pidx].ccti_entries); 1362 dd->pport[pidx].ccti_entries = NULL; 1363 kfree(dd->pport[pidx].ccti_entries_shadow); 1364 dd->pport[pidx].ccti_entries_shadow = NULL; 1365 kfree(dd->pport[pidx].congestion_entries_shadow); 1366 dd->pport[pidx].congestion_entries_shadow = NULL; 1367 1368 spin_unlock(&dd->pport[pidx].cc_shadow_lock); 1369 } 1370 1371 qib_disable_wc(dd); 1372 1373 if (dd->pioavailregs_dma) { 1374 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE, 1375 (void *) dd->pioavailregs_dma, 1376 dd->pioavailregs_phys); 1377 dd->pioavailregs_dma = NULL; 1378 } 1379 1380 if (dd->pageshadow) { 1381 struct page **tmpp = dd->pageshadow; 1382 dma_addr_t *tmpd = dd->physshadow; 1383 int i; 1384 1385 for (ctxt = 0; ctxt < dd->cfgctxts; ctxt++) { 1386 int ctxt_tidbase = ctxt * dd->rcvtidcnt; 1387 int maxtid = ctxt_tidbase + dd->rcvtidcnt; 1388 1389 for (i = ctxt_tidbase; i < maxtid; i++) { 1390 if (!tmpp[i]) 1391 continue; 1392 pci_unmap_page(dd->pcidev, tmpd[i], 1393 PAGE_SIZE, PCI_DMA_FROMDEVICE); 1394 qib_release_user_pages(&tmpp[i], 1); 1395 tmpp[i] = NULL; 1396 } 1397 } 1398 1399 dd->pageshadow = NULL; 1400 vfree(tmpp); 1401 dd->physshadow = NULL; 1402 vfree(tmpd); 1403 } 1404 1405 /* 1406 * Free any resources still in use (usually just kernel contexts) 1407 * at unload; we do for ctxtcnt, because that's what we allocate. 1408 * We acquire lock to be really paranoid that rcd isn't being 1409 * accessed from some interrupt-related code (that should not happen, 1410 * but best to be sure). 1411 */ 1412 spin_lock_irqsave(&dd->uctxt_lock, flags); 1413 tmp = dd->rcd; 1414 dd->rcd = NULL; 1415 spin_unlock_irqrestore(&dd->uctxt_lock, flags); 1416 for (ctxt = 0; tmp && ctxt < dd->ctxtcnt; ctxt++) { 1417 struct qib_ctxtdata *rcd = tmp[ctxt]; 1418 1419 tmp[ctxt] = NULL; /* debugging paranoia */ 1420 qib_free_ctxtdata(dd, rcd); 1421 } 1422 kfree(tmp); 1423 kfree(dd->boardname); 1424 qib_cq_exit(dd); 1425} 1426 1427/* 1428 * Clean up on unit shutdown, or error during unit load after 1429 * successful initialization. 1430 */ 1431static void qib_postinit_cleanup(struct qib_devdata *dd) 1432{ 1433 /* 1434 * Clean up chip-specific stuff. 1435 * We check for NULL here, because it's outside 1436 * the kregbase check, and we need to call it 1437 * after the free_irq. Thus it's possible that 1438 * the function pointers were never initialized. 1439 */ 1440 if (dd->f_cleanup) 1441 dd->f_cleanup(dd); 1442 1443 qib_pcie_ddcleanup(dd); 1444 1445 cleanup_device_data(dd); 1446 1447 qib_free_devdata(dd); 1448} 1449 1450static int qib_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 1451{ 1452 int ret, j, pidx, initfail; 1453 struct qib_devdata *dd = NULL; 1454 1455 ret = qib_pcie_init(pdev, ent); 1456 if (ret) 1457 goto bail; 1458 1459 /* 1460 * Do device-specific initialiation, function table setup, dd 1461 * allocation, etc. 1462 */ 1463 switch (ent->device) { 1464 case PCI_DEVICE_ID_QLOGIC_IB_6120: 1465#ifdef CONFIG_PCI_MSI 1466 dd = qib_init_iba6120_funcs(pdev, ent); 1467#else 1468 qib_early_err(&pdev->dev, 1469 "Intel PCIE device 0x%x cannot work if CONFIG_PCI_MSI is not enabled\n", 1470 ent->device); 1471 dd = ERR_PTR(-ENODEV); 1472#endif 1473 break; 1474 1475 case PCI_DEVICE_ID_QLOGIC_IB_7220: 1476 dd = qib_init_iba7220_funcs(pdev, ent); 1477 break; 1478 1479 case PCI_DEVICE_ID_QLOGIC_IB_7322: 1480 dd = qib_init_iba7322_funcs(pdev, ent); 1481 break; 1482 1483 default: 1484 qib_early_err(&pdev->dev, 1485 "Failing on unknown Intel deviceid 0x%x\n", 1486 ent->device); 1487 ret = -ENODEV; 1488 } 1489 1490 if (IS_ERR(dd)) 1491 ret = PTR_ERR(dd); 1492 if (ret) 1493 goto bail; /* error already printed */ 1494 1495 ret = qib_create_workqueues(dd); 1496 if (ret) 1497 goto bail; 1498 1499 /* do the generic initialization */ 1500 initfail = qib_init(dd, 0); 1501 1502 ret = qib_register_ib_device(dd); 1503 1504 /* 1505 * Now ready for use. this should be cleared whenever we 1506 * detect a reset, or initiate one. If earlier failure, 1507 * we still create devices, so diags, etc. can be used 1508 * to determine cause of problem. 1509 */ 1510 if (!qib_mini_init && !initfail && !ret) 1511 dd->flags |= QIB_INITTED; 1512 1513 j = qib_device_create(dd); 1514 if (j) 1515 qib_dev_err(dd, "Failed to create /dev devices: %d\n", -j); 1516 j = qibfs_add(dd); 1517 if (j) 1518 qib_dev_err(dd, "Failed filesystem setup for counters: %d\n", 1519 -j); 1520 1521 if (qib_mini_init || initfail || ret) { 1522 qib_stop_timers(dd); 1523 flush_workqueue(ib_wq); 1524 for (pidx = 0; pidx < dd->num_pports; ++pidx) 1525 dd->f_quiet_serdes(dd->pport + pidx); 1526 if (qib_mini_init) 1527 goto bail; 1528 if (!j) { 1529 (void) qibfs_remove(dd); 1530 qib_device_remove(dd); 1531 } 1532 if (!ret) 1533 qib_unregister_ib_device(dd); 1534 qib_postinit_cleanup(dd); 1535 if (initfail) 1536 ret = initfail; 1537 goto bail; 1538 } 1539 1540 ret = qib_enable_wc(dd); 1541 if (ret) { 1542 qib_dev_err(dd, 1543 "Write combining not enabled (err %d): performance may be poor\n", 1544 -ret); 1545 ret = 0; 1546 } 1547 1548 qib_verify_pioperf(dd); 1549bail: 1550 return ret; 1551} 1552 1553static void qib_remove_one(struct pci_dev *pdev) 1554{ 1555 struct qib_devdata *dd = pci_get_drvdata(pdev); 1556 int ret; 1557 1558 /* unregister from IB core */ 1559 qib_unregister_ib_device(dd); 1560 1561 /* 1562 * Disable the IB link, disable interrupts on the device, 1563 * clear dma engines, etc. 1564 */ 1565 if (!qib_mini_init) 1566 qib_shutdown_device(dd); 1567 1568 qib_stop_timers(dd); 1569 1570 /* wait until all of our (qsfp) queue_work() calls complete */ 1571 flush_workqueue(ib_wq); 1572 1573 ret = qibfs_remove(dd); 1574 if (ret) 1575 qib_dev_err(dd, "Failed counters filesystem cleanup: %d\n", 1576 -ret); 1577 1578 qib_device_remove(dd); 1579 1580 qib_postinit_cleanup(dd); 1581} 1582 1583/** 1584 * qib_create_rcvhdrq - create a receive header queue 1585 * @dd: the qlogic_ib device 1586 * @rcd: the context data 1587 * 1588 * This must be contiguous memory (from an i/o perspective), and must be 1589 * DMA'able (which means for some systems, it will go through an IOMMU, 1590 * or be forced into a low address range). 1591 */ 1592int qib_create_rcvhdrq(struct qib_devdata *dd, struct qib_ctxtdata *rcd) 1593{ 1594 unsigned amt; 1595 int old_node_id; 1596 1597 if (!rcd->rcvhdrq) { 1598 dma_addr_t phys_hdrqtail; 1599 gfp_t gfp_flags; 1600 1601 amt = ALIGN(dd->rcvhdrcnt * dd->rcvhdrentsize * 1602 sizeof(u32), PAGE_SIZE); 1603 gfp_flags = (rcd->ctxt >= dd->first_user_ctxt) ? 1604 GFP_USER : GFP_KERNEL; 1605 1606 old_node_id = dev_to_node(&dd->pcidev->dev); 1607 set_dev_node(&dd->pcidev->dev, rcd->node_id); 1608 rcd->rcvhdrq = dma_alloc_coherent( 1609 &dd->pcidev->dev, amt, &rcd->rcvhdrq_phys, 1610 gfp_flags | __GFP_COMP); 1611 set_dev_node(&dd->pcidev->dev, old_node_id); 1612 1613 if (!rcd->rcvhdrq) { 1614 qib_dev_err(dd, 1615 "attempt to allocate %d bytes for ctxt %u rcvhdrq failed\n", 1616 amt, rcd->ctxt); 1617 goto bail; 1618 } 1619 1620 if (rcd->ctxt >= dd->first_user_ctxt) { 1621 rcd->user_event_mask = vmalloc_user(PAGE_SIZE); 1622 if (!rcd->user_event_mask) 1623 goto bail_free_hdrq; 1624 } 1625 1626 if (!(dd->flags & QIB_NODMA_RTAIL)) { 1627 set_dev_node(&dd->pcidev->dev, rcd->node_id); 1628 rcd->rcvhdrtail_kvaddr = dma_alloc_coherent( 1629 &dd->pcidev->dev, PAGE_SIZE, &phys_hdrqtail, 1630 gfp_flags); 1631 set_dev_node(&dd->pcidev->dev, old_node_id); 1632 if (!rcd->rcvhdrtail_kvaddr) 1633 goto bail_free; 1634 rcd->rcvhdrqtailaddr_phys = phys_hdrqtail; 1635 } 1636 1637 rcd->rcvhdrq_size = amt; 1638 } 1639 1640 /* clear for security and sanity on each use */ 1641 memset(rcd->rcvhdrq, 0, rcd->rcvhdrq_size); 1642 if (rcd->rcvhdrtail_kvaddr) 1643 memset(rcd->rcvhdrtail_kvaddr, 0, PAGE_SIZE); 1644 return 0; 1645 1646bail_free: 1647 qib_dev_err(dd, 1648 "attempt to allocate 1 page for ctxt %u rcvhdrqtailaddr failed\n", 1649 rcd->ctxt); 1650 vfree(rcd->user_event_mask); 1651 rcd->user_event_mask = NULL; 1652bail_free_hdrq: 1653 dma_free_coherent(&dd->pcidev->dev, amt, rcd->rcvhdrq, 1654 rcd->rcvhdrq_phys); 1655 rcd->rcvhdrq = NULL; 1656bail: 1657 return -ENOMEM; 1658} 1659 1660/** 1661 * allocate eager buffers, both kernel and user contexts. 1662 * @rcd: the context we are setting up. 1663 * 1664 * Allocate the eager TID buffers and program them into hip. 1665 * They are no longer completely contiguous, we do multiple allocation 1666 * calls. Otherwise we get the OOM code involved, by asking for too 1667 * much per call, with disastrous results on some kernels. 1668 */ 1669int qib_setup_eagerbufs(struct qib_ctxtdata *rcd) 1670{ 1671 struct qib_devdata *dd = rcd->dd; 1672 unsigned e, egrcnt, egrperchunk, chunk, egrsize, egroff; 1673 size_t size; 1674 gfp_t gfp_flags; 1675 int old_node_id; 1676 1677 /* 1678 * GFP_USER, but without GFP_FS, so buffer cache can be 1679 * coalesced (we hope); otherwise, even at order 4, 1680 * heavy filesystem activity makes these fail, and we can 1681 * use compound pages. 1682 */ 1683 gfp_flags = __GFP_WAIT | __GFP_IO | __GFP_COMP; 1684 1685 egrcnt = rcd->rcvegrcnt; 1686 egroff = rcd->rcvegr_tid_base; 1687 egrsize = dd->rcvegrbufsize; 1688 1689 chunk = rcd->rcvegrbuf_chunks; 1690 egrperchunk = rcd->rcvegrbufs_perchunk; 1691 size = rcd->rcvegrbuf_size; 1692 if (!rcd->rcvegrbuf) { 1693 rcd->rcvegrbuf = 1694 kzalloc_node(chunk * sizeof(rcd->rcvegrbuf[0]), 1695 GFP_KERNEL, rcd->node_id); 1696 if (!rcd->rcvegrbuf) 1697 goto bail; 1698 } 1699 if (!rcd->rcvegrbuf_phys) { 1700 rcd->rcvegrbuf_phys = 1701 kmalloc_node(chunk * sizeof(rcd->rcvegrbuf_phys[0]), 1702 GFP_KERNEL, rcd->node_id); 1703 if (!rcd->rcvegrbuf_phys) 1704 goto bail_rcvegrbuf; 1705 } 1706 for (e = 0; e < rcd->rcvegrbuf_chunks; e++) { 1707 if (rcd->rcvegrbuf[e]) 1708 continue; 1709 1710 old_node_id = dev_to_node(&dd->pcidev->dev); 1711 set_dev_node(&dd->pcidev->dev, rcd->node_id); 1712 rcd->rcvegrbuf[e] = 1713 dma_alloc_coherent(&dd->pcidev->dev, size, 1714 &rcd->rcvegrbuf_phys[e], 1715 gfp_flags); 1716 set_dev_node(&dd->pcidev->dev, old_node_id); 1717 if (!rcd->rcvegrbuf[e]) 1718 goto bail_rcvegrbuf_phys; 1719 } 1720 1721 rcd->rcvegr_phys = rcd->rcvegrbuf_phys[0]; 1722 1723 for (e = chunk = 0; chunk < rcd->rcvegrbuf_chunks; chunk++) { 1724 dma_addr_t pa = rcd->rcvegrbuf_phys[chunk]; 1725 unsigned i; 1726 1727 /* clear for security and sanity on each use */ 1728 memset(rcd->rcvegrbuf[chunk], 0, size); 1729 1730 for (i = 0; e < egrcnt && i < egrperchunk; e++, i++) { 1731 dd->f_put_tid(dd, e + egroff + 1732 (u64 __iomem *) 1733 ((char __iomem *) 1734 dd->kregbase + 1735 dd->rcvegrbase), 1736 RCVHQ_RCV_TYPE_EAGER, pa); 1737 pa += egrsize; 1738 } 1739 cond_resched(); /* don't hog the cpu */ 1740 } 1741 1742 return 0; 1743 1744bail_rcvegrbuf_phys: 1745 for (e = 0; e < rcd->rcvegrbuf_chunks && rcd->rcvegrbuf[e]; e++) 1746 dma_free_coherent(&dd->pcidev->dev, size, 1747 rcd->rcvegrbuf[e], rcd->rcvegrbuf_phys[e]); 1748 kfree(rcd->rcvegrbuf_phys); 1749 rcd->rcvegrbuf_phys = NULL; 1750bail_rcvegrbuf: 1751 kfree(rcd->rcvegrbuf); 1752 rcd->rcvegrbuf = NULL; 1753bail: 1754 return -ENOMEM; 1755} 1756 1757/* 1758 * Note: Changes to this routine should be mirrored 1759 * for the diagnostics routine qib_remap_ioaddr32(). 1760 * There is also related code for VL15 buffers in qib_init_7322_variables(). 1761 * The teardown code that unmaps is in qib_pcie_ddcleanup() 1762 */ 1763int init_chip_wc_pat(struct qib_devdata *dd, u32 vl15buflen) 1764{ 1765 u64 __iomem *qib_kregbase = NULL; 1766 void __iomem *qib_piobase = NULL; 1767 u64 __iomem *qib_userbase = NULL; 1768 u64 qib_kreglen; 1769 u64 qib_pio2koffset = dd->piobufbase & 0xffffffff; 1770 u64 qib_pio4koffset = dd->piobufbase >> 32; 1771 u64 qib_pio2klen = dd->piobcnt2k * dd->palign; 1772 u64 qib_pio4klen = dd->piobcnt4k * dd->align4k; 1773 u64 qib_physaddr = dd->physaddr; 1774 u64 qib_piolen; 1775 u64 qib_userlen = 0; 1776 1777 /* 1778 * Free the old mapping because the kernel will try to reuse the 1779 * old mapping and not create a new mapping with the 1780 * write combining attribute. 1781 */ 1782 iounmap(dd->kregbase); 1783 dd->kregbase = NULL; 1784 1785 /* 1786 * Assumes chip address space looks like: 1787 * - kregs + sregs + cregs + uregs (in any order) 1788 * - piobufs (2K and 4K bufs in either order) 1789 * or: 1790 * - kregs + sregs + cregs (in any order) 1791 * - piobufs (2K and 4K bufs in either order) 1792 * - uregs 1793 */ 1794 if (dd->piobcnt4k == 0) { 1795 qib_kreglen = qib_pio2koffset; 1796 qib_piolen = qib_pio2klen; 1797 } else if (qib_pio2koffset < qib_pio4koffset) { 1798 qib_kreglen = qib_pio2koffset; 1799 qib_piolen = qib_pio4koffset + qib_pio4klen - qib_kreglen; 1800 } else { 1801 qib_kreglen = qib_pio4koffset; 1802 qib_piolen = qib_pio2koffset + qib_pio2klen - qib_kreglen; 1803 } 1804 qib_piolen += vl15buflen; 1805 /* Map just the configured ports (not all hw ports) */ 1806 if (dd->uregbase > qib_kreglen) 1807 qib_userlen = dd->ureg_align * dd->cfgctxts; 1808 1809 /* Sanity checks passed, now create the new mappings */ 1810 qib_kregbase = ioremap_nocache(qib_physaddr, qib_kreglen); 1811 if (!qib_kregbase) 1812 goto bail; 1813 1814 qib_piobase = ioremap_wc(qib_physaddr + qib_kreglen, qib_piolen); 1815 if (!qib_piobase) 1816 goto bail_kregbase; 1817 1818 if (qib_userlen) { 1819 qib_userbase = ioremap_nocache(qib_physaddr + dd->uregbase, 1820 qib_userlen); 1821 if (!qib_userbase) 1822 goto bail_piobase; 1823 } 1824 1825 dd->kregbase = qib_kregbase; 1826 dd->kregend = (u64 __iomem *) 1827 ((char __iomem *) qib_kregbase + qib_kreglen); 1828 dd->piobase = qib_piobase; 1829 dd->pio2kbase = (void __iomem *) 1830 (((char __iomem *) dd->piobase) + 1831 qib_pio2koffset - qib_kreglen); 1832 if (dd->piobcnt4k) 1833 dd->pio4kbase = (void __iomem *) 1834 (((char __iomem *) dd->piobase) + 1835 qib_pio4koffset - qib_kreglen); 1836 if (qib_userlen) 1837 /* ureg will now be accessed relative to dd->userbase */ 1838 dd->userbase = qib_userbase; 1839 return 0; 1840 1841bail_piobase: 1842 iounmap(qib_piobase); 1843bail_kregbase: 1844 iounmap(qib_kregbase); 1845bail: 1846 return -ENOMEM; 1847} 1848