1/* 2 * Intel MIC Platform Software Stack (MPSS) 3 * 4 * Copyright(c) 2013 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License, version 2, as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 * General Public License for more details. 14 * 15 * The full GNU General Public License is included in this distribution in 16 * the file called "COPYING". 17 * 18 * Intel MIC Host driver. 19 * 20 */ 21#include <linux/pci.h> 22#include <linux/interrupt.h> 23 24#include "../common/mic_dev.h" 25#include "mic_device.h" 26 27static irqreturn_t mic_thread_fn(int irq, void *dev) 28{ 29 struct mic_device *mdev = dev; 30 struct mic_intr_info *intr_info = mdev->intr_info; 31 struct mic_irq_info *irq_info = &mdev->irq_info; 32 struct mic_intr_cb *intr_cb; 33 struct pci_dev *pdev = mdev->pdev; 34 int i; 35 36 spin_lock(&irq_info->mic_thread_lock); 37 for (i = intr_info->intr_start_idx[MIC_INTR_DB]; 38 i < intr_info->intr_len[MIC_INTR_DB]; i++) 39 if (test_and_clear_bit(i, &irq_info->mask)) { 40 list_for_each_entry(intr_cb, &irq_info->cb_list[i], 41 list) 42 if (intr_cb->thread_fn) 43 intr_cb->thread_fn(pdev->irq, 44 intr_cb->data); 45 } 46 spin_unlock(&irq_info->mic_thread_lock); 47 return IRQ_HANDLED; 48} 49/** 50 * mic_interrupt - Generic interrupt handler for 51 * MSI and INTx based interrupts. 52 */ 53static irqreturn_t mic_interrupt(int irq, void *dev) 54{ 55 struct mic_device *mdev = dev; 56 struct mic_intr_info *intr_info = mdev->intr_info; 57 struct mic_irq_info *irq_info = &mdev->irq_info; 58 struct mic_intr_cb *intr_cb; 59 struct pci_dev *pdev = mdev->pdev; 60 u32 mask; 61 int i; 62 63 mask = mdev->ops->ack_interrupt(mdev); 64 if (!mask) 65 return IRQ_NONE; 66 67 spin_lock(&irq_info->mic_intr_lock); 68 for (i = intr_info->intr_start_idx[MIC_INTR_DB]; 69 i < intr_info->intr_len[MIC_INTR_DB]; i++) 70 if (mask & BIT(i)) { 71 list_for_each_entry(intr_cb, &irq_info->cb_list[i], 72 list) 73 if (intr_cb->handler) 74 intr_cb->handler(pdev->irq, 75 intr_cb->data); 76 set_bit(i, &irq_info->mask); 77 } 78 spin_unlock(&irq_info->mic_intr_lock); 79 return IRQ_WAKE_THREAD; 80} 81 82/* Return the interrupt offset from the index. Index is 0 based. */ 83static u16 mic_map_src_to_offset(struct mic_device *mdev, 84 int intr_src, enum mic_intr_type type) 85{ 86 if (type >= MIC_NUM_INTR_TYPES) 87 return MIC_NUM_OFFSETS; 88 if (intr_src >= mdev->intr_info->intr_len[type]) 89 return MIC_NUM_OFFSETS; 90 91 return mdev->intr_info->intr_start_idx[type] + intr_src; 92} 93 94/* Return next available msix_entry. */ 95static struct msix_entry *mic_get_available_vector(struct mic_device *mdev) 96{ 97 int i; 98 struct mic_irq_info *info = &mdev->irq_info; 99 100 for (i = 0; i < info->num_vectors; i++) 101 if (!info->mic_msi_map[i]) 102 return &info->msix_entries[i]; 103 return NULL; 104} 105 106/** 107 * mic_register_intr_callback - Register a callback handler for the 108 * given source id. 109 * 110 * @mdev: pointer to the mic_device instance 111 * @idx: The source id to be registered. 112 * @handler: The function to be called when the source id receives 113 * the interrupt. 114 * @thread_fn: thread fn. corresponding to the handler 115 * @data: Private data of the requester. 116 * Return the callback structure that was registered or an 117 * appropriate error on failure. 118 */ 119static struct mic_intr_cb *mic_register_intr_callback(struct mic_device *mdev, 120 u8 idx, irq_handler_t handler, irq_handler_t thread_fn, 121 void *data) 122{ 123 struct mic_intr_cb *intr_cb; 124 unsigned long flags; 125 int rc; 126 intr_cb = kmalloc(sizeof(*intr_cb), GFP_KERNEL); 127 128 if (!intr_cb) 129 return ERR_PTR(-ENOMEM); 130 131 intr_cb->handler = handler; 132 intr_cb->thread_fn = thread_fn; 133 intr_cb->data = data; 134 intr_cb->cb_id = ida_simple_get(&mdev->irq_info.cb_ida, 135 0, 0, GFP_KERNEL); 136 if (intr_cb->cb_id < 0) { 137 rc = intr_cb->cb_id; 138 goto ida_fail; 139 } 140 141 spin_lock(&mdev->irq_info.mic_thread_lock); 142 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags); 143 list_add_tail(&intr_cb->list, &mdev->irq_info.cb_list[idx]); 144 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags); 145 spin_unlock(&mdev->irq_info.mic_thread_lock); 146 147 return intr_cb; 148ida_fail: 149 kfree(intr_cb); 150 return ERR_PTR(rc); 151} 152 153/** 154 * mic_unregister_intr_callback - Unregister the callback handler 155 * identified by its callback id. 156 * 157 * @mdev: pointer to the mic_device instance 158 * @idx: The callback structure id to be unregistered. 159 * Return the source id that was unregistered or MIC_NUM_OFFSETS if no 160 * such callback handler was found. 161 */ 162static u8 mic_unregister_intr_callback(struct mic_device *mdev, u32 idx) 163{ 164 struct list_head *pos, *tmp; 165 struct mic_intr_cb *intr_cb; 166 unsigned long flags; 167 int i; 168 169 spin_lock(&mdev->irq_info.mic_thread_lock); 170 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags); 171 for (i = 0; i < MIC_NUM_OFFSETS; i++) { 172 list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) { 173 intr_cb = list_entry(pos, struct mic_intr_cb, list); 174 if (intr_cb->cb_id == idx) { 175 list_del(pos); 176 ida_simple_remove(&mdev->irq_info.cb_ida, 177 intr_cb->cb_id); 178 kfree(intr_cb); 179 spin_unlock_irqrestore( 180 &mdev->irq_info.mic_intr_lock, flags); 181 spin_unlock(&mdev->irq_info.mic_thread_lock); 182 return i; 183 } 184 } 185 } 186 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags); 187 spin_unlock(&mdev->irq_info.mic_thread_lock); 188 return MIC_NUM_OFFSETS; 189} 190 191/** 192 * mic_setup_msix - Initializes MSIx interrupts. 193 * 194 * @mdev: pointer to mic_device instance 195 * 196 * 197 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 198 */ 199static int mic_setup_msix(struct mic_device *mdev, struct pci_dev *pdev) 200{ 201 int rc, i; 202 int entry_size = sizeof(*mdev->irq_info.msix_entries); 203 204 mdev->irq_info.msix_entries = kmalloc_array(MIC_MIN_MSIX, 205 entry_size, GFP_KERNEL); 206 if (!mdev->irq_info.msix_entries) { 207 rc = -ENOMEM; 208 goto err_nomem1; 209 } 210 211 for (i = 0; i < MIC_MIN_MSIX; i++) 212 mdev->irq_info.msix_entries[i].entry = i; 213 214 rc = pci_enable_msix_exact(pdev, mdev->irq_info.msix_entries, 215 MIC_MIN_MSIX); 216 if (rc) { 217 dev_dbg(&pdev->dev, "Error enabling MSIx. rc = %d\n", rc); 218 goto err_enable_msix; 219 } 220 221 mdev->irq_info.num_vectors = MIC_MIN_MSIX; 222 mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) * 223 mdev->irq_info.num_vectors), GFP_KERNEL); 224 225 if (!mdev->irq_info.mic_msi_map) { 226 rc = -ENOMEM; 227 goto err_nomem2; 228 } 229 230 dev_dbg(&mdev->pdev->dev, 231 "%d MSIx irqs setup\n", mdev->irq_info.num_vectors); 232 return 0; 233err_nomem2: 234 pci_disable_msix(pdev); 235err_enable_msix: 236 kfree(mdev->irq_info.msix_entries); 237err_nomem1: 238 mdev->irq_info.num_vectors = 0; 239 return rc; 240} 241 242/** 243 * mic_setup_callbacks - Initialize data structures needed 244 * to handle callbacks. 245 * 246 * @mdev: pointer to mic_device instance 247 */ 248static int mic_setup_callbacks(struct mic_device *mdev) 249{ 250 int i; 251 252 mdev->irq_info.cb_list = kmalloc_array(MIC_NUM_OFFSETS, 253 sizeof(*mdev->irq_info.cb_list), 254 GFP_KERNEL); 255 if (!mdev->irq_info.cb_list) 256 return -ENOMEM; 257 258 for (i = 0; i < MIC_NUM_OFFSETS; i++) 259 INIT_LIST_HEAD(&mdev->irq_info.cb_list[i]); 260 ida_init(&mdev->irq_info.cb_ida); 261 spin_lock_init(&mdev->irq_info.mic_intr_lock); 262 spin_lock_init(&mdev->irq_info.mic_thread_lock); 263 return 0; 264} 265 266/** 267 * mic_release_callbacks - Uninitialize data structures needed 268 * to handle callbacks. 269 * 270 * @mdev: pointer to mic_device instance 271 */ 272static void mic_release_callbacks(struct mic_device *mdev) 273{ 274 unsigned long flags; 275 struct list_head *pos, *tmp; 276 struct mic_intr_cb *intr_cb; 277 int i; 278 279 spin_lock(&mdev->irq_info.mic_thread_lock); 280 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags); 281 for (i = 0; i < MIC_NUM_OFFSETS; i++) { 282 if (list_empty(&mdev->irq_info.cb_list[i])) 283 break; 284 285 list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) { 286 intr_cb = list_entry(pos, struct mic_intr_cb, list); 287 list_del(pos); 288 ida_simple_remove(&mdev->irq_info.cb_ida, 289 intr_cb->cb_id); 290 kfree(intr_cb); 291 } 292 } 293 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags); 294 spin_unlock(&mdev->irq_info.mic_thread_lock); 295 ida_destroy(&mdev->irq_info.cb_ida); 296 kfree(mdev->irq_info.cb_list); 297} 298 299/** 300 * mic_setup_msi - Initializes MSI interrupts. 301 * 302 * @mdev: pointer to mic_device instance 303 * @pdev: PCI device structure 304 * 305 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 306 */ 307static int mic_setup_msi(struct mic_device *mdev, struct pci_dev *pdev) 308{ 309 int rc; 310 311 rc = pci_enable_msi(pdev); 312 if (rc) { 313 dev_dbg(&pdev->dev, "Error enabling MSI. rc = %d\n", rc); 314 return rc; 315 } 316 317 mdev->irq_info.num_vectors = 1; 318 mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) * 319 mdev->irq_info.num_vectors), GFP_KERNEL); 320 321 if (!mdev->irq_info.mic_msi_map) { 322 rc = -ENOMEM; 323 goto err_nomem1; 324 } 325 326 rc = mic_setup_callbacks(mdev); 327 if (rc) { 328 dev_err(&pdev->dev, "Error setting up callbacks\n"); 329 goto err_nomem2; 330 } 331 332 rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn, 333 0, "mic-msi", mdev); 334 if (rc) { 335 dev_err(&pdev->dev, "Error allocating MSI interrupt\n"); 336 goto err_irq_req_fail; 337 } 338 339 dev_dbg(&pdev->dev, "%d MSI irqs setup\n", mdev->irq_info.num_vectors); 340 return 0; 341err_irq_req_fail: 342 mic_release_callbacks(mdev); 343err_nomem2: 344 kfree(mdev->irq_info.mic_msi_map); 345err_nomem1: 346 pci_disable_msi(pdev); 347 mdev->irq_info.num_vectors = 0; 348 return rc; 349} 350 351/** 352 * mic_setup_intx - Initializes legacy interrupts. 353 * 354 * @mdev: pointer to mic_device instance 355 * @pdev: PCI device structure 356 * 357 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 358 */ 359static int mic_setup_intx(struct mic_device *mdev, struct pci_dev *pdev) 360{ 361 int rc; 362 363 /* Enable intx */ 364 pci_intx(pdev, 1); 365 rc = mic_setup_callbacks(mdev); 366 if (rc) { 367 dev_err(&pdev->dev, "Error setting up callbacks\n"); 368 goto err_nomem; 369 } 370 371 rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn, 372 IRQF_SHARED, "mic-intx", mdev); 373 if (rc) 374 goto err; 375 376 dev_dbg(&pdev->dev, "intx irq setup\n"); 377 return 0; 378err: 379 mic_release_callbacks(mdev); 380err_nomem: 381 return rc; 382} 383 384/** 385 * mic_next_db - Retrieve the next doorbell interrupt source id. 386 * The id is picked sequentially from the available pool of 387 * doorlbell ids. 388 * 389 * @mdev: pointer to the mic_device instance. 390 * 391 * Returns the next doorbell interrupt source. 392 */ 393int mic_next_db(struct mic_device *mdev) 394{ 395 int next_db; 396 397 next_db = mdev->irq_info.next_avail_src % 398 mdev->intr_info->intr_len[MIC_INTR_DB]; 399 mdev->irq_info.next_avail_src++; 400 return next_db; 401} 402 403#define COOKIE_ID_SHIFT 16 404#define GET_ENTRY(cookie) ((cookie) & 0xFFFF) 405#define GET_OFFSET(cookie) ((cookie) >> COOKIE_ID_SHIFT) 406#define MK_COOKIE(x, y) ((x) | (y) << COOKIE_ID_SHIFT) 407 408/** 409 * mic_request_threaded_irq - request an irq. mic_mutex needs 410 * to be held before calling this function. 411 * 412 * @mdev: pointer to mic_device instance 413 * @handler: The callback function that handles the interrupt. 414 * The function needs to call ack_interrupts 415 * (mdev->ops->ack_interrupt(mdev)) when handling the interrupts. 416 * @thread_fn: thread fn required by request_threaded_irq. 417 * @name: The ASCII name of the callee requesting the irq. 418 * @data: private data that is returned back when calling the 419 * function handler. 420 * @intr_src: The source id of the requester. Its the doorbell id 421 * for Doorbell interrupts and DMA channel id for DMA interrupts. 422 * @type: The type of interrupt. Values defined in mic_intr_type 423 * 424 * returns: The cookie that is transparent to the caller. Passed 425 * back when calling mic_free_irq. An appropriate error code 426 * is returned on failure. Caller needs to use IS_ERR(return_val) 427 * to check for failure and PTR_ERR(return_val) to obtained the 428 * error code. 429 * 430 */ 431struct mic_irq * 432mic_request_threaded_irq(struct mic_device *mdev, 433 irq_handler_t handler, irq_handler_t thread_fn, 434 const char *name, void *data, int intr_src, 435 enum mic_intr_type type) 436{ 437 u16 offset; 438 int rc = 0; 439 struct msix_entry *msix = NULL; 440 unsigned long cookie = 0; 441 u16 entry; 442 struct mic_intr_cb *intr_cb; 443 struct pci_dev *pdev = mdev->pdev; 444 445 offset = mic_map_src_to_offset(mdev, intr_src, type); 446 if (offset >= MIC_NUM_OFFSETS) { 447 dev_err(&mdev->pdev->dev, 448 "Error mapping index %d to a valid source id.\n", 449 intr_src); 450 rc = -EINVAL; 451 goto err; 452 } 453 454 if (mdev->irq_info.num_vectors > 1) { 455 msix = mic_get_available_vector(mdev); 456 if (!msix) { 457 dev_err(&mdev->pdev->dev, 458 "No MSIx vectors available for use.\n"); 459 rc = -ENOSPC; 460 goto err; 461 } 462 463 rc = request_threaded_irq(msix->vector, handler, thread_fn, 464 0, name, data); 465 if (rc) { 466 dev_dbg(&mdev->pdev->dev, 467 "request irq failed rc = %d\n", rc); 468 goto err; 469 } 470 entry = msix->entry; 471 mdev->irq_info.mic_msi_map[entry] |= BIT(offset); 472 mdev->intr_ops->program_msi_to_src_map(mdev, 473 entry, offset, true); 474 cookie = MK_COOKIE(entry, offset); 475 dev_dbg(&mdev->pdev->dev, "irq: %d assigned for src: %d\n", 476 msix->vector, intr_src); 477 } else { 478 intr_cb = mic_register_intr_callback(mdev, offset, handler, 479 thread_fn, data); 480 if (IS_ERR(intr_cb)) { 481 dev_err(&mdev->pdev->dev, 482 "No available callback entries for use\n"); 483 rc = PTR_ERR(intr_cb); 484 goto err; 485 } 486 487 entry = 0; 488 if (pci_dev_msi_enabled(pdev)) { 489 mdev->irq_info.mic_msi_map[entry] |= (1 << offset); 490 mdev->intr_ops->program_msi_to_src_map(mdev, 491 entry, offset, true); 492 } 493 cookie = MK_COOKIE(entry, intr_cb->cb_id); 494 dev_dbg(&mdev->pdev->dev, "callback %d registered for src: %d\n", 495 intr_cb->cb_id, intr_src); 496 } 497 return (struct mic_irq *)cookie; 498err: 499 return ERR_PTR(rc); 500} 501 502/** 503 * mic_free_irq - free irq. mic_mutex 504 * needs to be held before calling this function. 505 * 506 * @mdev: pointer to mic_device instance 507 * @cookie: cookie obtained during a successful call to mic_request_threaded_irq 508 * @data: private data specified by the calling function during the 509 * mic_request_threaded_irq 510 * 511 * returns: none. 512 */ 513void mic_free_irq(struct mic_device *mdev, 514 struct mic_irq *cookie, void *data) 515{ 516 u32 offset; 517 u32 entry; 518 u8 src_id; 519 unsigned int irq; 520 struct pci_dev *pdev = mdev->pdev; 521 522 entry = GET_ENTRY((unsigned long)cookie); 523 offset = GET_OFFSET((unsigned long)cookie); 524 if (mdev->irq_info.num_vectors > 1) { 525 if (entry >= mdev->irq_info.num_vectors) { 526 dev_warn(&mdev->pdev->dev, 527 "entry %d should be < num_irq %d\n", 528 entry, mdev->irq_info.num_vectors); 529 return; 530 } 531 irq = mdev->irq_info.msix_entries[entry].vector; 532 free_irq(irq, data); 533 mdev->irq_info.mic_msi_map[entry] &= ~(BIT(offset)); 534 mdev->intr_ops->program_msi_to_src_map(mdev, 535 entry, offset, false); 536 537 dev_dbg(&mdev->pdev->dev, "irq: %d freed\n", irq); 538 } else { 539 irq = pdev->irq; 540 src_id = mic_unregister_intr_callback(mdev, offset); 541 if (src_id >= MIC_NUM_OFFSETS) { 542 dev_warn(&mdev->pdev->dev, "Error unregistering callback\n"); 543 return; 544 } 545 if (pci_dev_msi_enabled(pdev)) { 546 mdev->irq_info.mic_msi_map[entry] &= ~(BIT(src_id)); 547 mdev->intr_ops->program_msi_to_src_map(mdev, 548 entry, src_id, false); 549 } 550 dev_dbg(&mdev->pdev->dev, "callback %d unregistered for src: %d\n", 551 offset, src_id); 552 } 553} 554 555/** 556 * mic_setup_interrupts - Initializes interrupts. 557 * 558 * @mdev: pointer to mic_device instance 559 * @pdev: PCI device structure 560 * 561 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 562 */ 563int mic_setup_interrupts(struct mic_device *mdev, struct pci_dev *pdev) 564{ 565 int rc; 566 567 rc = mic_setup_msix(mdev, pdev); 568 if (!rc) 569 goto done; 570 571 rc = mic_setup_msi(mdev, pdev); 572 if (!rc) 573 goto done; 574 575 rc = mic_setup_intx(mdev, pdev); 576 if (rc) { 577 dev_err(&mdev->pdev->dev, "no usable interrupts\n"); 578 return rc; 579 } 580done: 581 mdev->intr_ops->enable_interrupts(mdev); 582 return 0; 583} 584 585/** 586 * mic_free_interrupts - Frees interrupts setup by mic_setup_interrupts 587 * 588 * @mdev: pointer to mic_device instance 589 * @pdev: PCI device structure 590 * 591 * returns none. 592 */ 593void mic_free_interrupts(struct mic_device *mdev, struct pci_dev *pdev) 594{ 595 int i; 596 597 mdev->intr_ops->disable_interrupts(mdev); 598 if (mdev->irq_info.num_vectors > 1) { 599 for (i = 0; i < mdev->irq_info.num_vectors; i++) { 600 if (mdev->irq_info.mic_msi_map[i]) 601 dev_warn(&pdev->dev, "irq %d may still be in use.\n", 602 mdev->irq_info.msix_entries[i].vector); 603 } 604 kfree(mdev->irq_info.mic_msi_map); 605 kfree(mdev->irq_info.msix_entries); 606 pci_disable_msix(pdev); 607 } else { 608 if (pci_dev_msi_enabled(pdev)) { 609 free_irq(pdev->irq, mdev); 610 kfree(mdev->irq_info.mic_msi_map); 611 pci_disable_msi(pdev); 612 } else { 613 free_irq(pdev->irq, mdev); 614 } 615 mic_release_callbacks(mdev); 616 } 617} 618 619/** 620 * mic_intr_restore - Restore MIC interrupt registers. 621 * 622 * @mdev: pointer to mic_device instance. 623 * 624 * Restore the interrupt registers to values previously 625 * stored in the SW data structures. mic_mutex needs to 626 * be held before calling this function. 627 * 628 * returns None. 629 */ 630void mic_intr_restore(struct mic_device *mdev) 631{ 632 int entry, offset; 633 struct pci_dev *pdev = mdev->pdev; 634 635 if (!pci_dev_msi_enabled(pdev)) 636 return; 637 638 for (entry = 0; entry < mdev->irq_info.num_vectors; entry++) { 639 for (offset = 0; offset < MIC_NUM_OFFSETS; offset++) { 640 if (mdev->irq_info.mic_msi_map[entry] & BIT(offset)) 641 mdev->intr_ops->program_msi_to_src_map(mdev, 642 entry, offset, true); 643 } 644 } 645} 646