1/* 2 * Intel MIC Platform Software Stack (MPSS) 3 * 4 * Copyright(c) 2013 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License, version 2, as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 * General Public License for more details. 14 * 15 * The full GNU General Public License is included in this distribution in 16 * the file called "COPYING". 17 * 18 * Intel MIC Host driver. 19 * 20 */ 21#include <linux/pci.h> 22#include <linux/interrupt.h> 23 24#include "../common/mic_dev.h" 25#include "mic_device.h" 26 27static irqreturn_t mic_thread_fn(int irq, void *dev) 28{ 29 struct mic_device *mdev = dev; 30 struct mic_intr_info *intr_info = mdev->intr_info; 31 struct mic_irq_info *irq_info = &mdev->irq_info; 32 struct mic_intr_cb *intr_cb; 33 struct pci_dev *pdev = container_of(mdev->sdev->parent, 34 struct pci_dev, dev); 35 int i; 36 37 spin_lock(&irq_info->mic_thread_lock); 38 for (i = intr_info->intr_start_idx[MIC_INTR_DB]; 39 i < intr_info->intr_len[MIC_INTR_DB]; i++) 40 if (test_and_clear_bit(i, &irq_info->mask)) { 41 list_for_each_entry(intr_cb, &irq_info->cb_list[i], 42 list) 43 if (intr_cb->thread_fn) 44 intr_cb->thread_fn(pdev->irq, 45 intr_cb->data); 46 } 47 spin_unlock(&irq_info->mic_thread_lock); 48 return IRQ_HANDLED; 49} 50/** 51 * mic_interrupt - Generic interrupt handler for 52 * MSI and INTx based interrupts. 53 */ 54static irqreturn_t mic_interrupt(int irq, void *dev) 55{ 56 struct mic_device *mdev = dev; 57 struct mic_intr_info *intr_info = mdev->intr_info; 58 struct mic_irq_info *irq_info = &mdev->irq_info; 59 struct mic_intr_cb *intr_cb; 60 struct pci_dev *pdev = container_of(mdev->sdev->parent, 61 struct pci_dev, dev); 62 u32 mask; 63 int i; 64 65 mask = mdev->ops->ack_interrupt(mdev); 66 if (!mask) 67 return IRQ_NONE; 68 69 spin_lock(&irq_info->mic_intr_lock); 70 for (i = intr_info->intr_start_idx[MIC_INTR_DB]; 71 i < intr_info->intr_len[MIC_INTR_DB]; i++) 72 if (mask & BIT(i)) { 73 list_for_each_entry(intr_cb, &irq_info->cb_list[i], 74 list) 75 if (intr_cb->handler) 76 intr_cb->handler(pdev->irq, 77 intr_cb->data); 78 set_bit(i, &irq_info->mask); 79 } 80 spin_unlock(&irq_info->mic_intr_lock); 81 return IRQ_WAKE_THREAD; 82} 83 84/* Return the interrupt offset from the index. Index is 0 based. */ 85static u16 mic_map_src_to_offset(struct mic_device *mdev, 86 int intr_src, enum mic_intr_type type) 87{ 88 if (type >= MIC_NUM_INTR_TYPES) 89 return MIC_NUM_OFFSETS; 90 if (intr_src >= mdev->intr_info->intr_len[type]) 91 return MIC_NUM_OFFSETS; 92 93 return mdev->intr_info->intr_start_idx[type] + intr_src; 94} 95 96/* Return next available msix_entry. */ 97static struct msix_entry *mic_get_available_vector(struct mic_device *mdev) 98{ 99 int i; 100 struct mic_irq_info *info = &mdev->irq_info; 101 102 for (i = 0; i < info->num_vectors; i++) 103 if (!info->mic_msi_map[i]) 104 return &info->msix_entries[i]; 105 return NULL; 106} 107 108/** 109 * mic_register_intr_callback - Register a callback handler for the 110 * given source id. 111 * 112 * @mdev: pointer to the mic_device instance 113 * @idx: The source id to be registered. 114 * @handler: The function to be called when the source id receives 115 * the interrupt. 116 * @thread_fn: thread fn. corresponding to the handler 117 * @data: Private data of the requester. 118 * Return the callback structure that was registered or an 119 * appropriate error on failure. 120 */ 121static struct mic_intr_cb *mic_register_intr_callback(struct mic_device *mdev, 122 u8 idx, irq_handler_t handler, irq_handler_t thread_fn, 123 void *data) 124{ 125 struct mic_intr_cb *intr_cb; 126 unsigned long flags; 127 int rc; 128 intr_cb = kmalloc(sizeof(*intr_cb), GFP_KERNEL); 129 130 if (!intr_cb) 131 return ERR_PTR(-ENOMEM); 132 133 intr_cb->handler = handler; 134 intr_cb->thread_fn = thread_fn; 135 intr_cb->data = data; 136 intr_cb->cb_id = ida_simple_get(&mdev->irq_info.cb_ida, 137 0, 0, GFP_KERNEL); 138 if (intr_cb->cb_id < 0) { 139 rc = intr_cb->cb_id; 140 goto ida_fail; 141 } 142 143 spin_lock(&mdev->irq_info.mic_thread_lock); 144 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags); 145 list_add_tail(&intr_cb->list, &mdev->irq_info.cb_list[idx]); 146 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags); 147 spin_unlock(&mdev->irq_info.mic_thread_lock); 148 149 return intr_cb; 150ida_fail: 151 kfree(intr_cb); 152 return ERR_PTR(rc); 153} 154 155/** 156 * mic_unregister_intr_callback - Unregister the callback handler 157 * identified by its callback id. 158 * 159 * @mdev: pointer to the mic_device instance 160 * @idx: The callback structure id to be unregistered. 161 * Return the source id that was unregistered or MIC_NUM_OFFSETS if no 162 * such callback handler was found. 163 */ 164static u8 mic_unregister_intr_callback(struct mic_device *mdev, u32 idx) 165{ 166 struct list_head *pos, *tmp; 167 struct mic_intr_cb *intr_cb; 168 unsigned long flags; 169 int i; 170 171 spin_lock(&mdev->irq_info.mic_thread_lock); 172 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags); 173 for (i = 0; i < MIC_NUM_OFFSETS; i++) { 174 list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) { 175 intr_cb = list_entry(pos, struct mic_intr_cb, list); 176 if (intr_cb->cb_id == idx) { 177 list_del(pos); 178 ida_simple_remove(&mdev->irq_info.cb_ida, 179 intr_cb->cb_id); 180 kfree(intr_cb); 181 spin_unlock_irqrestore( 182 &mdev->irq_info.mic_intr_lock, flags); 183 spin_unlock(&mdev->irq_info.mic_thread_lock); 184 return i; 185 } 186 } 187 } 188 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags); 189 spin_unlock(&mdev->irq_info.mic_thread_lock); 190 return MIC_NUM_OFFSETS; 191} 192 193/** 194 * mic_setup_msix - Initializes MSIx interrupts. 195 * 196 * @mdev: pointer to mic_device instance 197 * 198 * 199 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 200 */ 201static int mic_setup_msix(struct mic_device *mdev, struct pci_dev *pdev) 202{ 203 int rc, i; 204 int entry_size = sizeof(*mdev->irq_info.msix_entries); 205 206 mdev->irq_info.msix_entries = kmalloc_array(MIC_MIN_MSIX, 207 entry_size, GFP_KERNEL); 208 if (!mdev->irq_info.msix_entries) { 209 rc = -ENOMEM; 210 goto err_nomem1; 211 } 212 213 for (i = 0; i < MIC_MIN_MSIX; i++) 214 mdev->irq_info.msix_entries[i].entry = i; 215 216 rc = pci_enable_msix_exact(pdev, mdev->irq_info.msix_entries, 217 MIC_MIN_MSIX); 218 if (rc) { 219 dev_dbg(&pdev->dev, "Error enabling MSIx. rc = %d\n", rc); 220 goto err_enable_msix; 221 } 222 223 mdev->irq_info.num_vectors = MIC_MIN_MSIX; 224 mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) * 225 mdev->irq_info.num_vectors), GFP_KERNEL); 226 227 if (!mdev->irq_info.mic_msi_map) { 228 rc = -ENOMEM; 229 goto err_nomem2; 230 } 231 232 dev_dbg(mdev->sdev->parent, 233 "%d MSIx irqs setup\n", mdev->irq_info.num_vectors); 234 return 0; 235err_nomem2: 236 pci_disable_msix(pdev); 237err_enable_msix: 238 kfree(mdev->irq_info.msix_entries); 239err_nomem1: 240 mdev->irq_info.num_vectors = 0; 241 return rc; 242} 243 244/** 245 * mic_setup_callbacks - Initialize data structures needed 246 * to handle callbacks. 247 * 248 * @mdev: pointer to mic_device instance 249 */ 250static int mic_setup_callbacks(struct mic_device *mdev) 251{ 252 int i; 253 254 mdev->irq_info.cb_list = kmalloc_array(MIC_NUM_OFFSETS, 255 sizeof(*mdev->irq_info.cb_list), 256 GFP_KERNEL); 257 if (!mdev->irq_info.cb_list) 258 return -ENOMEM; 259 260 for (i = 0; i < MIC_NUM_OFFSETS; i++) 261 INIT_LIST_HEAD(&mdev->irq_info.cb_list[i]); 262 ida_init(&mdev->irq_info.cb_ida); 263 spin_lock_init(&mdev->irq_info.mic_intr_lock); 264 spin_lock_init(&mdev->irq_info.mic_thread_lock); 265 return 0; 266} 267 268/** 269 * mic_release_callbacks - Uninitialize data structures needed 270 * to handle callbacks. 271 * 272 * @mdev: pointer to mic_device instance 273 */ 274static void mic_release_callbacks(struct mic_device *mdev) 275{ 276 unsigned long flags; 277 struct list_head *pos, *tmp; 278 struct mic_intr_cb *intr_cb; 279 int i; 280 281 spin_lock(&mdev->irq_info.mic_thread_lock); 282 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags); 283 for (i = 0; i < MIC_NUM_OFFSETS; i++) { 284 285 if (list_empty(&mdev->irq_info.cb_list[i])) 286 break; 287 288 list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) { 289 intr_cb = list_entry(pos, struct mic_intr_cb, list); 290 list_del(pos); 291 ida_simple_remove(&mdev->irq_info.cb_ida, 292 intr_cb->cb_id); 293 kfree(intr_cb); 294 } 295 } 296 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags); 297 spin_unlock(&mdev->irq_info.mic_thread_lock); 298 ida_destroy(&mdev->irq_info.cb_ida); 299 kfree(mdev->irq_info.cb_list); 300} 301 302/** 303 * mic_setup_msi - Initializes MSI interrupts. 304 * 305 * @mdev: pointer to mic_device instance 306 * @pdev: PCI device structure 307 * 308 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 309 */ 310static int mic_setup_msi(struct mic_device *mdev, struct pci_dev *pdev) 311{ 312 int rc; 313 314 rc = pci_enable_msi(pdev); 315 if (rc) { 316 dev_dbg(&pdev->dev, "Error enabling MSI. rc = %d\n", rc); 317 return rc; 318 } 319 320 mdev->irq_info.num_vectors = 1; 321 mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) * 322 mdev->irq_info.num_vectors), GFP_KERNEL); 323 324 if (!mdev->irq_info.mic_msi_map) { 325 rc = -ENOMEM; 326 goto err_nomem1; 327 } 328 329 rc = mic_setup_callbacks(mdev); 330 if (rc) { 331 dev_err(&pdev->dev, "Error setting up callbacks\n"); 332 goto err_nomem2; 333 } 334 335 rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn, 336 0, "mic-msi", mdev); 337 if (rc) { 338 dev_err(&pdev->dev, "Error allocating MSI interrupt\n"); 339 goto err_irq_req_fail; 340 } 341 342 dev_dbg(&pdev->dev, "%d MSI irqs setup\n", mdev->irq_info.num_vectors); 343 return 0; 344err_irq_req_fail: 345 mic_release_callbacks(mdev); 346err_nomem2: 347 kfree(mdev->irq_info.mic_msi_map); 348err_nomem1: 349 pci_disable_msi(pdev); 350 mdev->irq_info.num_vectors = 0; 351 return rc; 352} 353 354/** 355 * mic_setup_intx - Initializes legacy interrupts. 356 * 357 * @mdev: pointer to mic_device instance 358 * @pdev: PCI device structure 359 * 360 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 361 */ 362static int mic_setup_intx(struct mic_device *mdev, struct pci_dev *pdev) 363{ 364 int rc; 365 366 /* Enable intx */ 367 pci_intx(pdev, 1); 368 rc = mic_setup_callbacks(mdev); 369 if (rc) { 370 dev_err(&pdev->dev, "Error setting up callbacks\n"); 371 goto err_nomem; 372 } 373 374 rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn, 375 IRQF_SHARED, "mic-intx", mdev); 376 if (rc) 377 goto err; 378 379 dev_dbg(&pdev->dev, "intx irq setup\n"); 380 return 0; 381err: 382 mic_release_callbacks(mdev); 383err_nomem: 384 return rc; 385} 386 387/** 388 * mic_next_db - Retrieve the next doorbell interrupt source id. 389 * The id is picked sequentially from the available pool of 390 * doorlbell ids. 391 * 392 * @mdev: pointer to the mic_device instance. 393 * 394 * Returns the next doorbell interrupt source. 395 */ 396int mic_next_db(struct mic_device *mdev) 397{ 398 int next_db; 399 400 next_db = mdev->irq_info.next_avail_src % 401 mdev->intr_info->intr_len[MIC_INTR_DB]; 402 mdev->irq_info.next_avail_src++; 403 return next_db; 404} 405 406#define COOKIE_ID_SHIFT 16 407#define GET_ENTRY(cookie) ((cookie) & 0xFFFF) 408#define GET_OFFSET(cookie) ((cookie) >> COOKIE_ID_SHIFT) 409#define MK_COOKIE(x, y) ((x) | (y) << COOKIE_ID_SHIFT) 410 411/** 412 * mic_request_threaded_irq - request an irq. mic_mutex needs 413 * to be held before calling this function. 414 * 415 * @mdev: pointer to mic_device instance 416 * @handler: The callback function that handles the interrupt. 417 * The function needs to call ack_interrupts 418 * (mdev->ops->ack_interrupt(mdev)) when handling the interrupts. 419 * @thread_fn: thread fn required by request_threaded_irq. 420 * @name: The ASCII name of the callee requesting the irq. 421 * @data: private data that is returned back when calling the 422 * function handler. 423 * @intr_src: The source id of the requester. Its the doorbell id 424 * for Doorbell interrupts and DMA channel id for DMA interrupts. 425 * @type: The type of interrupt. Values defined in mic_intr_type 426 * 427 * returns: The cookie that is transparent to the caller. Passed 428 * back when calling mic_free_irq. An appropriate error code 429 * is returned on failure. Caller needs to use IS_ERR(return_val) 430 * to check for failure and PTR_ERR(return_val) to obtained the 431 * error code. 432 * 433 */ 434struct mic_irq * 435mic_request_threaded_irq(struct mic_device *mdev, 436 irq_handler_t handler, irq_handler_t thread_fn, 437 const char *name, void *data, int intr_src, 438 enum mic_intr_type type) 439{ 440 u16 offset; 441 int rc = 0; 442 struct msix_entry *msix = NULL; 443 unsigned long cookie = 0; 444 u16 entry; 445 struct mic_intr_cb *intr_cb; 446 struct pci_dev *pdev = container_of(mdev->sdev->parent, 447 struct pci_dev, dev); 448 449 offset = mic_map_src_to_offset(mdev, intr_src, type); 450 if (offset >= MIC_NUM_OFFSETS) { 451 dev_err(mdev->sdev->parent, 452 "Error mapping index %d to a valid source id.\n", 453 intr_src); 454 rc = -EINVAL; 455 goto err; 456 } 457 458 if (mdev->irq_info.num_vectors > 1) { 459 msix = mic_get_available_vector(mdev); 460 if (!msix) { 461 dev_err(mdev->sdev->parent, 462 "No MSIx vectors available for use.\n"); 463 rc = -ENOSPC; 464 goto err; 465 } 466 467 rc = request_threaded_irq(msix->vector, handler, thread_fn, 468 0, name, data); 469 if (rc) { 470 dev_dbg(mdev->sdev->parent, 471 "request irq failed rc = %d\n", rc); 472 goto err; 473 } 474 entry = msix->entry; 475 mdev->irq_info.mic_msi_map[entry] |= BIT(offset); 476 mdev->intr_ops->program_msi_to_src_map(mdev, 477 entry, offset, true); 478 cookie = MK_COOKIE(entry, offset); 479 dev_dbg(mdev->sdev->parent, "irq: %d assigned for src: %d\n", 480 msix->vector, intr_src); 481 } else { 482 intr_cb = mic_register_intr_callback(mdev, offset, handler, 483 thread_fn, data); 484 if (IS_ERR(intr_cb)) { 485 dev_err(mdev->sdev->parent, 486 "No available callback entries for use\n"); 487 rc = PTR_ERR(intr_cb); 488 goto err; 489 } 490 491 entry = 0; 492 if (pci_dev_msi_enabled(pdev)) { 493 mdev->irq_info.mic_msi_map[entry] |= (1 << offset); 494 mdev->intr_ops->program_msi_to_src_map(mdev, 495 entry, offset, true); 496 } 497 cookie = MK_COOKIE(entry, intr_cb->cb_id); 498 dev_dbg(mdev->sdev->parent, "callback %d registered for src: %d\n", 499 intr_cb->cb_id, intr_src); 500 } 501 return (struct mic_irq *)cookie; 502err: 503 return ERR_PTR(rc); 504} 505 506/** 507 * mic_free_irq - free irq. mic_mutex 508 * needs to be held before calling this function. 509 * 510 * @mdev: pointer to mic_device instance 511 * @cookie: cookie obtained during a successful call to mic_request_threaded_irq 512 * @data: private data specified by the calling function during the 513 * mic_request_threaded_irq 514 * 515 * returns: none. 516 */ 517void mic_free_irq(struct mic_device *mdev, 518 struct mic_irq *cookie, void *data) 519{ 520 u32 offset; 521 u32 entry; 522 u8 src_id; 523 unsigned int irq; 524 struct pci_dev *pdev = container_of(mdev->sdev->parent, 525 struct pci_dev, dev); 526 527 entry = GET_ENTRY((unsigned long)cookie); 528 offset = GET_OFFSET((unsigned long)cookie); 529 if (mdev->irq_info.num_vectors > 1) { 530 if (entry >= mdev->irq_info.num_vectors) { 531 dev_warn(mdev->sdev->parent, 532 "entry %d should be < num_irq %d\n", 533 entry, mdev->irq_info.num_vectors); 534 return; 535 } 536 irq = mdev->irq_info.msix_entries[entry].vector; 537 free_irq(irq, data); 538 mdev->irq_info.mic_msi_map[entry] &= ~(BIT(offset)); 539 mdev->intr_ops->program_msi_to_src_map(mdev, 540 entry, offset, false); 541 542 dev_dbg(mdev->sdev->parent, "irq: %d freed\n", irq); 543 } else { 544 irq = pdev->irq; 545 src_id = mic_unregister_intr_callback(mdev, offset); 546 if (src_id >= MIC_NUM_OFFSETS) { 547 dev_warn(mdev->sdev->parent, "Error unregistering callback\n"); 548 return; 549 } 550 if (pci_dev_msi_enabled(pdev)) { 551 mdev->irq_info.mic_msi_map[entry] &= ~(BIT(src_id)); 552 mdev->intr_ops->program_msi_to_src_map(mdev, 553 entry, src_id, false); 554 } 555 dev_dbg(mdev->sdev->parent, "callback %d unregistered for src: %d\n", 556 offset, src_id); 557 } 558} 559 560/** 561 * mic_setup_interrupts - Initializes interrupts. 562 * 563 * @mdev: pointer to mic_device instance 564 * @pdev: PCI device structure 565 * 566 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 567 */ 568int mic_setup_interrupts(struct mic_device *mdev, struct pci_dev *pdev) 569{ 570 int rc; 571 572 rc = mic_setup_msix(mdev, pdev); 573 if (!rc) 574 goto done; 575 576 rc = mic_setup_msi(mdev, pdev); 577 if (!rc) 578 goto done; 579 580 rc = mic_setup_intx(mdev, pdev); 581 if (rc) { 582 dev_err(mdev->sdev->parent, "no usable interrupts\n"); 583 return rc; 584 } 585done: 586 mdev->intr_ops->enable_interrupts(mdev); 587 return 0; 588} 589 590/** 591 * mic_free_interrupts - Frees interrupts setup by mic_setup_interrupts 592 * 593 * @mdev: pointer to mic_device instance 594 * @pdev: PCI device structure 595 * 596 * returns none. 597 */ 598void mic_free_interrupts(struct mic_device *mdev, struct pci_dev *pdev) 599{ 600 int i; 601 602 mdev->intr_ops->disable_interrupts(mdev); 603 if (mdev->irq_info.num_vectors > 1) { 604 for (i = 0; i < mdev->irq_info.num_vectors; i++) { 605 if (mdev->irq_info.mic_msi_map[i]) 606 dev_warn(&pdev->dev, "irq %d may still be in use.\n", 607 mdev->irq_info.msix_entries[i].vector); 608 } 609 kfree(mdev->irq_info.mic_msi_map); 610 kfree(mdev->irq_info.msix_entries); 611 pci_disable_msix(pdev); 612 } else { 613 if (pci_dev_msi_enabled(pdev)) { 614 free_irq(pdev->irq, mdev); 615 kfree(mdev->irq_info.mic_msi_map); 616 pci_disable_msi(pdev); 617 } else { 618 free_irq(pdev->irq, mdev); 619 } 620 mic_release_callbacks(mdev); 621 } 622} 623 624/** 625 * mic_intr_restore - Restore MIC interrupt registers. 626 * 627 * @mdev: pointer to mic_device instance. 628 * 629 * Restore the interrupt registers to values previously 630 * stored in the SW data structures. mic_mutex needs to 631 * be held before calling this function. 632 * 633 * returns None. 634 */ 635void mic_intr_restore(struct mic_device *mdev) 636{ 637 int entry, offset; 638 struct pci_dev *pdev = container_of(mdev->sdev->parent, 639 struct pci_dev, dev); 640 641 if (!pci_dev_msi_enabled(pdev)) 642 return; 643 644 for (entry = 0; entry < mdev->irq_info.num_vectors; entry++) { 645 for (offset = 0; offset < MIC_NUM_OFFSETS; offset++) { 646 if (mdev->irq_info.mic_msi_map[entry] & BIT(offset)) 647 mdev->intr_ops->program_msi_to_src_map(mdev, 648 entry, offset, true); 649 } 650 } 651} 652