92 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_pcie_free_fw_monitor()  local
94 	if (!trans_pcie->fw_mon_page)  in iwl_pcie_free_fw_monitor()
97 	dma_unmap_page(trans->dev, trans_pcie->fw_mon_phys,  in iwl_pcie_free_fw_monitor()
98 		       trans_pcie->fw_mon_size, DMA_FROM_DEVICE);  in iwl_pcie_free_fw_monitor()
99 	__free_pages(trans_pcie->fw_mon_page,  in iwl_pcie_free_fw_monitor()
100 		     get_order(trans_pcie->fw_mon_size));  in iwl_pcie_free_fw_monitor()
101 	trans_pcie->fw_mon_page = NULL;  in iwl_pcie_free_fw_monitor()
102 	trans_pcie->fw_mon_phys = 0;  in iwl_pcie_free_fw_monitor()
103 	trans_pcie->fw_mon_size = 0;  in iwl_pcie_free_fw_monitor()
108 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_pcie_alloc_fw_monitor()  local
126 	if (trans_pcie->fw_mon_page) {  in iwl_pcie_alloc_fw_monitor()
127 		dma_sync_single_for_device(trans->dev, trans_pcie->fw_mon_phys,  in iwl_pcie_alloc_fw_monitor()
128 					   trans_pcie->fw_mon_size,  in iwl_pcie_alloc_fw_monitor()
166 	trans_pcie->fw_mon_page = page;  in iwl_pcie_alloc_fw_monitor()
167 	trans_pcie->fw_mon_phys = phys;  in iwl_pcie_alloc_fw_monitor()
168 	trans_pcie->fw_mon_size = size;  in iwl_pcie_alloc_fw_monitor()
205 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_pcie_apm_config()  local
217 	pcie_capability_read_word(trans_pcie->pci_dev, PCI_EXP_LNKCTL, &lctl);  in iwl_pcie_apm_config()
224 	pcie_capability_read_word(trans_pcie->pci_dev, PCI_EXP_DEVCTL2, &cap);  in iwl_pcie_apm_config()
521 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_pcie_nic_init()  local
524 	spin_lock(&trans_pcie->irq_lock);  in iwl_pcie_nic_init()
527 	spin_unlock(&trans_pcie->irq_lock);  in iwl_pcie_nic_init()
617 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_pcie_load_firmware_chunk()  local
620 	trans_pcie->ucode_write_complete = false;  in iwl_pcie_load_firmware_chunk()
651 	ret = wait_event_timeout(trans_pcie->ucode_write_waitq,  in iwl_pcie_load_firmware_chunk()
652 				 trans_pcie->ucode_write_complete, 5 * HZ);  in iwl_pcie_load_firmware_chunk()
870 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_pcie_apply_destination()  local
926 	if (dest->monitor_mode == EXTERNAL_MODE && trans_pcie->fw_mon_size) {  in iwl_pcie_apply_destination()
928 			       trans_pcie->fw_mon_phys >> dest->base_shift);  in iwl_pcie_apply_destination()
931 				       (trans_pcie->fw_mon_phys +  in iwl_pcie_apply_destination()
932 					trans_pcie->fw_mon_size - 256) >>  in iwl_pcie_apply_destination()
936 				       (trans_pcie->fw_mon_phys +  in iwl_pcie_apply_destination()
937 					trans_pcie->fw_mon_size) >>  in iwl_pcie_apply_destination()
945 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_pcie_load_given_ucode()  local
975 		if (trans_pcie->fw_mon_size) {  in iwl_pcie_load_given_ucode()
977 				       trans_pcie->fw_mon_phys >> 4);  in iwl_pcie_load_given_ucode()
979 				       (trans_pcie->fw_mon_phys +  in iwl_pcie_load_given_ucode()
980 					trans_pcie->fw_mon_size) >> 4);  in iwl_pcie_load_given_ucode()
1027 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_start_fw()  local
1031 	mutex_lock(&trans_pcie->mutex);  in iwl_trans_pcie_start_fw()
1034 	if (trans_pcie->is_down) {  in iwl_trans_pcie_start_fw()
1090 	mutex_unlock(&trans_pcie->mutex);  in iwl_trans_pcie_start_fw()
1102 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in _iwl_trans_pcie_stop_device()  local
1105 	lockdep_assert_held(&trans_pcie->mutex);  in _iwl_trans_pcie_stop_device()
1107 	if (trans_pcie->is_down)  in _iwl_trans_pcie_stop_device()
1110 	trans_pcie->is_down = true;  in _iwl_trans_pcie_stop_device()
1115 	spin_lock(&trans_pcie->irq_lock);  in _iwl_trans_pcie_stop_device()
1117 	spin_unlock(&trans_pcie->irq_lock);  in _iwl_trans_pcie_stop_device()
1160 	spin_lock(&trans_pcie->irq_lock);  in _iwl_trans_pcie_stop_device()
1162 	spin_unlock(&trans_pcie->irq_lock);  in _iwl_trans_pcie_stop_device()
1203 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_stop_device()  local
1205 	mutex_lock(&trans_pcie->mutex);  in iwl_trans_pcie_stop_device()
1207 	mutex_unlock(&trans_pcie->mutex);  in iwl_trans_pcie_stop_device()
1212 	struct iwl_trans_pcie __maybe_unused *trans_pcie =  in iwl_trans_pcie_rf_kill()  local
1215 	lockdep_assert_held(&trans_pcie->mutex);  in iwl_trans_pcie_rf_kill()
1223 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_d3_suspend()  local
1242 	synchronize_irq(trans_pcie->pci_dev->irq);  in iwl_trans_pcie_d3_suspend()
1323 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in _iwl_trans_pcie_start_hw()  local
1327 	lockdep_assert_held(&trans_pcie->mutex);  in _iwl_trans_pcie_start_hw()
1346 	trans_pcie->is_down = false;  in _iwl_trans_pcie_start_hw()
1361 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_start_hw()  local
1364 	mutex_lock(&trans_pcie->mutex);  in iwl_trans_pcie_start_hw()
1366 	mutex_unlock(&trans_pcie->mutex);  in iwl_trans_pcie_start_hw()
1373 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_op_mode_leave()  local
1375 	mutex_lock(&trans_pcie->mutex);  in iwl_trans_pcie_op_mode_leave()
1378 	spin_lock(&trans_pcie->irq_lock);  in iwl_trans_pcie_op_mode_leave()
1380 	spin_unlock(&trans_pcie->irq_lock);  in iwl_trans_pcie_op_mode_leave()
1384 	spin_lock(&trans_pcie->irq_lock);  in iwl_trans_pcie_op_mode_leave()
1386 	spin_unlock(&trans_pcie->irq_lock);  in iwl_trans_pcie_op_mode_leave()
1390 	mutex_unlock(&trans_pcie->mutex);  in iwl_trans_pcie_op_mode_leave()
1392 	synchronize_irq(trans_pcie->pci_dev->irq);  in iwl_trans_pcie_op_mode_leave()
1434 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_configure()  local
1436 	trans_pcie->cmd_queue = trans_cfg->cmd_queue;  in iwl_trans_pcie_configure()
1437 	trans_pcie->cmd_fifo = trans_cfg->cmd_fifo;  in iwl_trans_pcie_configure()
1438 	trans_pcie->cmd_q_wdg_timeout = trans_cfg->cmd_q_wdg_timeout;  in iwl_trans_pcie_configure()
1440 		trans_pcie->n_no_reclaim_cmds = 0;  in iwl_trans_pcie_configure()
1442 		trans_pcie->n_no_reclaim_cmds = trans_cfg->n_no_reclaim_cmds;  in iwl_trans_pcie_configure()
1443 	if (trans_pcie->n_no_reclaim_cmds)  in iwl_trans_pcie_configure()
1444 		memcpy(trans_pcie->no_reclaim_cmds, trans_cfg->no_reclaim_cmds,  in iwl_trans_pcie_configure()
1445 		       trans_pcie->n_no_reclaim_cmds * sizeof(u8));  in iwl_trans_pcie_configure()
1447 	trans_pcie->rx_buf_size_8k = trans_cfg->rx_buf_size_8k;  in iwl_trans_pcie_configure()
1448 	if (trans_pcie->rx_buf_size_8k)  in iwl_trans_pcie_configure()
1449 		trans_pcie->rx_page_order = get_order(8 * 1024);  in iwl_trans_pcie_configure()
1451 		trans_pcie->rx_page_order = get_order(4 * 1024);  in iwl_trans_pcie_configure()
1453 	trans_pcie->wide_cmd_header = trans_cfg->wide_cmd_header;  in iwl_trans_pcie_configure()
1454 	trans_pcie->command_names = trans_cfg->command_names;  in iwl_trans_pcie_configure()
1455 	trans_pcie->bc_table_dword = trans_cfg->bc_table_dword;  in iwl_trans_pcie_configure()
1456 	trans_pcie->scd_set_active = trans_cfg->scd_set_active;  in iwl_trans_pcie_configure()
1459 	trans_pcie->ref_count = 1;  in iwl_trans_pcie_configure()
1466 	if (!trans_pcie->napi.poll) {  in iwl_trans_pcie_configure()
1467 		init_dummy_netdev(&trans_pcie->napi_dev);  in iwl_trans_pcie_configure()
1468 		netif_napi_add(&trans_pcie->napi_dev, &trans_pcie->napi,  in iwl_trans_pcie_configure()
1475 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_free()  local
1477 	synchronize_irq(trans_pcie->pci_dev->irq);  in iwl_trans_pcie_free()
1482 	free_irq(trans_pcie->pci_dev->irq, trans);  in iwl_trans_pcie_free()
1485 	pci_disable_msi(trans_pcie->pci_dev);  in iwl_trans_pcie_free()
1486 	iounmap(trans_pcie->hw_base);  in iwl_trans_pcie_free()
1487 	pci_release_regions(trans_pcie->pci_dev);  in iwl_trans_pcie_free()
1488 	pci_disable_device(trans_pcie->pci_dev);  in iwl_trans_pcie_free()
1490 	if (trans_pcie->napi.poll)  in iwl_trans_pcie_free()
1491 		netif_napi_del(&trans_pcie->napi);  in iwl_trans_pcie_free()
1510 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_grab_nic_access()  local
1512 	spin_lock_irqsave(&trans_pcie->reg_lock, *flags);  in iwl_trans_pcie_grab_nic_access()
1514 	if (trans_pcie->cmd_hold_nic_awake)  in iwl_trans_pcie_grab_nic_access()
1553 			spin_unlock_irqrestore(&trans_pcie->reg_lock, *flags);  in iwl_trans_pcie_grab_nic_access()
1563 	__release(&trans_pcie->reg_lock);  in iwl_trans_pcie_grab_nic_access()
1570 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_release_nic_access()  local
1572 	lockdep_assert_held(&trans_pcie->reg_lock);  in iwl_trans_pcie_release_nic_access()
1578 	__acquire(&trans_pcie->reg_lock);  in iwl_trans_pcie_release_nic_access()
1580 	if (trans_pcie->cmd_hold_nic_awake)  in iwl_trans_pcie_release_nic_access()
1593 	spin_unlock_irqrestore(&trans_pcie->reg_lock, *flags);  in iwl_trans_pcie_release_nic_access()
1637 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_freeze_txq_timer()  local
1641 		struct iwl_txq *txq = &trans_pcie->txq[queue];  in iwl_trans_pcie_freeze_txq_timer()
1691 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_wait_txq_empty()  local
1704 		if (cnt == trans_pcie->cmd_queue)  in iwl_trans_pcie_wait_txq_empty()
1706 		if (!test_bit(cnt, trans_pcie->queue_used))  in iwl_trans_pcie_wait_txq_empty()
1712 		txq = &trans_pcie->txq[cnt];  in iwl_trans_pcie_wait_txq_empty()
1743 	scd_sram_addr = trans_pcie->scd_base_addr +  in iwl_trans_pcie_wait_txq_empty()
1758 			iwl_trans_read_mem32(trans, trans_pcie->scd_base_addr +  in iwl_trans_pcie_wait_txq_empty()
1780 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_set_bits_mask()  local
1783 	spin_lock_irqsave(&trans_pcie->reg_lock, flags);  in iwl_trans_pcie_set_bits_mask()
1785 	spin_unlock_irqrestore(&trans_pcie->reg_lock, flags);  in iwl_trans_pcie_set_bits_mask()
1790 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_ref()  local
1796 	spin_lock_irqsave(&trans_pcie->ref_lock, flags);  in iwl_trans_pcie_ref()
1797 	IWL_DEBUG_RPM(trans, "ref_counter: %d\n", trans_pcie->ref_count);  in iwl_trans_pcie_ref()
1798 	trans_pcie->ref_count++;  in iwl_trans_pcie_ref()
1799 	spin_unlock_irqrestore(&trans_pcie->ref_lock, flags);  in iwl_trans_pcie_ref()
1804 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_unref()  local
1810 	spin_lock_irqsave(&trans_pcie->ref_lock, flags);  in iwl_trans_pcie_unref()
1811 	IWL_DEBUG_RPM(trans, "ref_counter: %d\n", trans_pcie->ref_count);  in iwl_trans_pcie_unref()
1812 	if (WARN_ON_ONCE(trans_pcie->ref_count == 0)) {  in iwl_trans_pcie_unref()
1813 		spin_unlock_irqrestore(&trans_pcie->ref_lock, flags);  in iwl_trans_pcie_unref()
1816 	trans_pcie->ref_count--;  in iwl_trans_pcie_unref()
1817 	spin_unlock_irqrestore(&trans_pcie->ref_lock, flags);  in iwl_trans_pcie_unref()
1929 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_dbgfs_tx_queue_read()  local
1940 	if (!trans_pcie->txq)  in iwl_dbgfs_tx_queue_read()
1948 		txq = &trans_pcie->txq[cnt];  in iwl_dbgfs_tx_queue_read()
1953 				!!test_bit(cnt, trans_pcie->queue_used),  in iwl_dbgfs_tx_queue_read()
1954 				 !!test_bit(cnt, trans_pcie->queue_stopped),  in iwl_dbgfs_tx_queue_read()
1956 				 (cnt == trans_pcie->cmd_queue ? " HCMD" : ""));  in iwl_dbgfs_tx_queue_read()
1968 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_dbgfs_rx_queue_read()  local
1969 	struct iwl_rxq *rxq = &trans_pcie->rxq;  in iwl_dbgfs_rx_queue_read()
1999 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_dbgfs_interrupt_read()  local
2000 	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;  in iwl_dbgfs_interrupt_read()
2057 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_dbgfs_interrupt_write()  local
2058 	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;  in iwl_dbgfs_interrupt_write()
2300 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_dump_rbs()  local
2301 	int max_len = PAGE_SIZE << trans_pcie->rx_page_order;  in iwl_trans_pcie_dump_rbs()
2302 	struct iwl_rxq *rxq = &trans_pcie->rxq;  in iwl_trans_pcie_dump_rbs()
2411 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_dump_monitor()  local
2414 	if ((trans_pcie->fw_mon_page &&  in iwl_trans_pcie_dump_monitor()
2442 		if (trans_pcie->fw_mon_page) {  in iwl_trans_pcie_dump_monitor()
2450 						trans_pcie->fw_mon_phys,  in iwl_trans_pcie_dump_monitor()
2451 						trans_pcie->fw_mon_size,  in iwl_trans_pcie_dump_monitor()
2454 			       page_address(trans_pcie->fw_mon_page),  in iwl_trans_pcie_dump_monitor()
2455 			       trans_pcie->fw_mon_size);  in iwl_trans_pcie_dump_monitor()
2457 			monitor_len = trans_pcie->fw_mon_size;  in iwl_trans_pcie_dump_monitor()
2488 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_dump_data()  local
2490 	struct iwl_txq *cmdq = &trans_pcie->txq[trans_pcie->cmd_queue];  in iwl_trans_pcie_dump_data()
2506 	if (trans_pcie->fw_mon_page) {  in iwl_trans_pcie_dump_data()
2508 		       trans_pcie->fw_mon_size;  in iwl_trans_pcie_dump_data()
2509 		monitor_len = trans_pcie->fw_mon_size;  in iwl_trans_pcie_dump_data()
2563 				      trans_pcie->rxq.rb_stts->closed_rb_num))  in iwl_trans_pcie_dump_data()
2565 		num_rbs = (num_rbs - trans_pcie->rxq.read) & RX_QUEUE_MASK;  in iwl_trans_pcie_dump_data()
2568 				  (PAGE_SIZE << trans_pcie->rx_page_order));  in iwl_trans_pcie_dump_data()
2663 	struct iwl_trans_pcie *trans_pcie;  in iwl_trans_pcie_alloc()  local
2675 	trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);  in iwl_trans_pcie_alloc()
2677 	trans_pcie->trans = trans;  in iwl_trans_pcie_alloc()
2678 	spin_lock_init(&trans_pcie->irq_lock);  in iwl_trans_pcie_alloc()
2679 	spin_lock_init(&trans_pcie->reg_lock);  in iwl_trans_pcie_alloc()
2680 	spin_lock_init(&trans_pcie->ref_lock);  in iwl_trans_pcie_alloc()
2681 	mutex_init(&trans_pcie->mutex);  in iwl_trans_pcie_alloc()
2682 	init_waitqueue_head(&trans_pcie->ucode_write_waitq);  in iwl_trans_pcie_alloc()
2722 	trans_pcie->hw_base = pci_ioremap_bar(pdev, 0);  in iwl_trans_pcie_alloc()
2723 	if (!trans_pcie->hw_base) {  in iwl_trans_pcie_alloc()
2734 	trans_pcie->pci_dev = pdev;  in iwl_trans_pcie_alloc()
2804 	init_waitqueue_head(&trans_pcie->wait_command_queue);  in iwl_trans_pcie_alloc()
2818 	trans_pcie->inta_mask = CSR_INI_SET_MASK;  in iwl_trans_pcie_alloc()