29 	struct mic_device *mdev = dev;  in mic_thread_fn()  local
30 	struct mic_intr_info *intr_info = mdev->intr_info;  in mic_thread_fn()
31 	struct mic_irq_info *irq_info = &mdev->irq_info;  in mic_thread_fn()
33 	struct pci_dev *pdev = mdev->pdev;  in mic_thread_fn()
55 	struct mic_device *mdev = dev;  in mic_interrupt()  local
56 	struct mic_intr_info *intr_info = mdev->intr_info;  in mic_interrupt()
57 	struct mic_irq_info *irq_info = &mdev->irq_info;  in mic_interrupt()
59 	struct pci_dev *pdev = mdev->pdev;  in mic_interrupt()
63 	mask = mdev->ops->ack_interrupt(mdev);  in mic_interrupt()
83 static u16 mic_map_src_to_offset(struct mic_device *mdev,  in mic_map_src_to_offset()  argument
88 	if (intr_src >= mdev->intr_info->intr_len[type])  in mic_map_src_to_offset()
91 	return mdev->intr_info->intr_start_idx[type] + intr_src;  in mic_map_src_to_offset()
95 static struct msix_entry *mic_get_available_vector(struct mic_device *mdev)  in mic_get_available_vector()  argument
98 	struct mic_irq_info *info = &mdev->irq_info;  in mic_get_available_vector()
119 static struct mic_intr_cb *mic_register_intr_callback(struct mic_device *mdev,  in mic_register_intr_callback()  argument
134 	intr_cb->cb_id = ida_simple_get(&mdev->irq_info.cb_ida,  in mic_register_intr_callback()
141 	spin_lock(&mdev->irq_info.mic_thread_lock);  in mic_register_intr_callback()
142 	spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);  in mic_register_intr_callback()
143 	list_add_tail(&intr_cb->list, &mdev->irq_info.cb_list[idx]);  in mic_register_intr_callback()
144 	spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);  in mic_register_intr_callback()
145 	spin_unlock(&mdev->irq_info.mic_thread_lock);  in mic_register_intr_callback()
162 static u8 mic_unregister_intr_callback(struct mic_device *mdev, u32 idx)  in mic_unregister_intr_callback()  argument
169 	spin_lock(&mdev->irq_info.mic_thread_lock);  in mic_unregister_intr_callback()
170 	spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);  in mic_unregister_intr_callback()
172 		list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {  in mic_unregister_intr_callback()
176 				ida_simple_remove(&mdev->irq_info.cb_ida,  in mic_unregister_intr_callback()
180 					&mdev->irq_info.mic_intr_lock, flags);  in mic_unregister_intr_callback()
181 				spin_unlock(&mdev->irq_info.mic_thread_lock);  in mic_unregister_intr_callback()
186 	spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);  in mic_unregister_intr_callback()
187 	spin_unlock(&mdev->irq_info.mic_thread_lock);  in mic_unregister_intr_callback()
199 static int mic_setup_msix(struct mic_device *mdev, struct pci_dev *pdev)  in mic_setup_msix()  argument
202 	int entry_size = sizeof(*mdev->irq_info.msix_entries);  in mic_setup_msix()
204 	mdev->irq_info.msix_entries = kmalloc_array(MIC_MIN_MSIX,  in mic_setup_msix()
206 	if (!mdev->irq_info.msix_entries) {  in mic_setup_msix()
212 		mdev->irq_info.msix_entries[i].entry = i;  in mic_setup_msix()
214 	rc = pci_enable_msix_exact(pdev, mdev->irq_info.msix_entries,  in mic_setup_msix()
221 	mdev->irq_info.num_vectors = MIC_MIN_MSIX;  in mic_setup_msix()
222 	mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *  in mic_setup_msix()
223 		mdev->irq_info.num_vectors), GFP_KERNEL);  in mic_setup_msix()
225 	if (!mdev->irq_info.mic_msi_map) {  in mic_setup_msix()
230 	dev_dbg(&mdev->pdev->dev,  in mic_setup_msix()
231 		"%d MSIx irqs setup\n", mdev->irq_info.num_vectors);  in mic_setup_msix()
236 	kfree(mdev->irq_info.msix_entries);  in mic_setup_msix()
238 	mdev->irq_info.num_vectors = 0;  in mic_setup_msix()
248 static int mic_setup_callbacks(struct mic_device *mdev)  in mic_setup_callbacks()  argument
252 	mdev->irq_info.cb_list = kmalloc_array(MIC_NUM_OFFSETS,  in mic_setup_callbacks()
253 					       sizeof(*mdev->irq_info.cb_list),  in mic_setup_callbacks()
255 	if (!mdev->irq_info.cb_list)  in mic_setup_callbacks()
259 		INIT_LIST_HEAD(&mdev->irq_info.cb_list[i]);  in mic_setup_callbacks()
260 	ida_init(&mdev->irq_info.cb_ida);  in mic_setup_callbacks()
261 	spin_lock_init(&mdev->irq_info.mic_intr_lock);  in mic_setup_callbacks()
262 	spin_lock_init(&mdev->irq_info.mic_thread_lock);  in mic_setup_callbacks()
272 static void mic_release_callbacks(struct mic_device *mdev)  in mic_release_callbacks()  argument
279 	spin_lock(&mdev->irq_info.mic_thread_lock);  in mic_release_callbacks()
280 	spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);  in mic_release_callbacks()
282 		if (list_empty(&mdev->irq_info.cb_list[i]))  in mic_release_callbacks()
285 		list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {  in mic_release_callbacks()
288 			ida_simple_remove(&mdev->irq_info.cb_ida,  in mic_release_callbacks()
293 	spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);  in mic_release_callbacks()
294 	spin_unlock(&mdev->irq_info.mic_thread_lock);  in mic_release_callbacks()
295 	ida_destroy(&mdev->irq_info.cb_ida);  in mic_release_callbacks()
296 	kfree(mdev->irq_info.cb_list);  in mic_release_callbacks()
307 static int mic_setup_msi(struct mic_device *mdev, struct pci_dev *pdev)  in mic_setup_msi()  argument
317 	mdev->irq_info.num_vectors = 1;  in mic_setup_msi()
318 	mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *  in mic_setup_msi()
319 		mdev->irq_info.num_vectors), GFP_KERNEL);  in mic_setup_msi()
321 	if (!mdev->irq_info.mic_msi_map) {  in mic_setup_msi()
326 	rc = mic_setup_callbacks(mdev);  in mic_setup_msi()
333 				  0, "mic-msi", mdev);  in mic_setup_msi()
339 	dev_dbg(&pdev->dev, "%d MSI irqs setup\n", mdev->irq_info.num_vectors);  in mic_setup_msi()
342 	mic_release_callbacks(mdev);  in mic_setup_msi()
344 	kfree(mdev->irq_info.mic_msi_map);  in mic_setup_msi()
347 	mdev->irq_info.num_vectors = 0;  in mic_setup_msi()
359 static int mic_setup_intx(struct mic_device *mdev, struct pci_dev *pdev)  in mic_setup_intx()  argument
365 	rc = mic_setup_callbacks(mdev);  in mic_setup_intx()
372 				  IRQF_SHARED, "mic-intx", mdev);  in mic_setup_intx()
379 	mic_release_callbacks(mdev);  in mic_setup_intx()
393 int mic_next_db(struct mic_device *mdev)  in mic_next_db()  argument
397 	next_db = mdev->irq_info.next_avail_src %  in mic_next_db()
398 		mdev->intr_info->intr_len[MIC_INTR_DB];  in mic_next_db()
399 	mdev->irq_info.next_avail_src++;  in mic_next_db()
432 mic_request_threaded_irq(struct mic_device *mdev,  in mic_request_threaded_irq()  argument
443 	struct pci_dev *pdev = mdev->pdev;  in mic_request_threaded_irq()
445 	offset = mic_map_src_to_offset(mdev, intr_src, type);  in mic_request_threaded_irq()
447 		dev_err(&mdev->pdev->dev,  in mic_request_threaded_irq()
454 	if (mdev->irq_info.num_vectors > 1) {  in mic_request_threaded_irq()
455 		msix = mic_get_available_vector(mdev);  in mic_request_threaded_irq()
457 			dev_err(&mdev->pdev->dev,  in mic_request_threaded_irq()
466 			dev_dbg(&mdev->pdev->dev,  in mic_request_threaded_irq()
471 		mdev->irq_info.mic_msi_map[entry] |= BIT(offset);  in mic_request_threaded_irq()
472 		mdev->intr_ops->program_msi_to_src_map(mdev,  in mic_request_threaded_irq()
475 		dev_dbg(&mdev->pdev->dev, "irq: %d assigned for src: %d\n",  in mic_request_threaded_irq()
478 		intr_cb = mic_register_intr_callback(mdev, offset, handler,  in mic_request_threaded_irq()
481 			dev_err(&mdev->pdev->dev,  in mic_request_threaded_irq()
489 			mdev->irq_info.mic_msi_map[entry] |= (1 << offset);  in mic_request_threaded_irq()
490 			mdev->intr_ops->program_msi_to_src_map(mdev,  in mic_request_threaded_irq()
494 		dev_dbg(&mdev->pdev->dev, "callback %d registered for src: %d\n",  in mic_request_threaded_irq()
513 void mic_free_irq(struct mic_device *mdev,  in mic_free_irq()  argument
520 	struct pci_dev *pdev = mdev->pdev;  in mic_free_irq()
524 	if (mdev->irq_info.num_vectors > 1) {  in mic_free_irq()
525 		if (entry >= mdev->irq_info.num_vectors) {  in mic_free_irq()
526 			dev_warn(&mdev->pdev->dev,  in mic_free_irq()
528 				entry, mdev->irq_info.num_vectors);  in mic_free_irq()
531 		irq = mdev->irq_info.msix_entries[entry].vector;  in mic_free_irq()
533 		mdev->irq_info.mic_msi_map[entry] &= ~(BIT(offset));  in mic_free_irq()
534 		mdev->intr_ops->program_msi_to_src_map(mdev,  in mic_free_irq()
537 		dev_dbg(&mdev->pdev->dev, "irq: %d freed\n", irq);  in mic_free_irq()
540 		src_id = mic_unregister_intr_callback(mdev, offset);  in mic_free_irq()
542 			dev_warn(&mdev->pdev->dev, "Error unregistering callback\n");  in mic_free_irq()
546 			mdev->irq_info.mic_msi_map[entry] &= ~(BIT(src_id));  in mic_free_irq()
547 			mdev->intr_ops->program_msi_to_src_map(mdev,  in mic_free_irq()
550 		dev_dbg(&mdev->pdev->dev, "callback %d unregistered for src: %d\n",  in mic_free_irq()
563 int mic_setup_interrupts(struct mic_device *mdev, struct pci_dev *pdev)  in mic_setup_interrupts()  argument
567 	rc = mic_setup_msix(mdev, pdev);  in mic_setup_interrupts()
571 	rc = mic_setup_msi(mdev, pdev);  in mic_setup_interrupts()
575 	rc = mic_setup_intx(mdev, pdev);  in mic_setup_interrupts()
577 		dev_err(&mdev->pdev->dev, "no usable interrupts\n");  in mic_setup_interrupts()
581 	mdev->intr_ops->enable_interrupts(mdev);  in mic_setup_interrupts()
593 void mic_free_interrupts(struct mic_device *mdev, struct pci_dev *pdev)  in mic_free_interrupts()  argument
597 	mdev->intr_ops->disable_interrupts(mdev);  in mic_free_interrupts()
598 	if (mdev->irq_info.num_vectors > 1) {  in mic_free_interrupts()
599 		for (i = 0; i < mdev->irq_info.num_vectors; i++) {  in mic_free_interrupts()
600 			if (mdev->irq_info.mic_msi_map[i])  in mic_free_interrupts()
602 					 mdev->irq_info.msix_entries[i].vector);  in mic_free_interrupts()
604 		kfree(mdev->irq_info.mic_msi_map);  in mic_free_interrupts()
605 		kfree(mdev->irq_info.msix_entries);  in mic_free_interrupts()
609 			free_irq(pdev->irq, mdev);  in mic_free_interrupts()
610 			kfree(mdev->irq_info.mic_msi_map);  in mic_free_interrupts()
613 			free_irq(pdev->irq, mdev);  in mic_free_interrupts()
615 		mic_release_callbacks(mdev);  in mic_free_interrupts()
630 void mic_intr_restore(struct mic_device *mdev)  in mic_intr_restore()  argument
633 	struct pci_dev *pdev = mdev->pdev;  in mic_intr_restore()
638 	for (entry = 0; entry < mdev->irq_info.num_vectors; entry++) {  in mic_intr_restore()
640 			if (mdev->irq_info.mic_msi_map[entry] & BIT(offset))  in mic_intr_restore()
641 				mdev->intr_ops->program_msi_to_src_map(mdev,  in mic_intr_restore()