154 pvscsi_dev(const struct pvscsi_adapter *adapter)  in pvscsi_dev()  argument
156 	return &(adapter->dev->dev);  in pvscsi_dev()
160 pvscsi_find_context(const struct pvscsi_adapter *adapter, struct scsi_cmnd *cmd)  in pvscsi_find_context()  argument
164 	end = &adapter->cmd_map[adapter->req_depth];  in pvscsi_find_context()
165 	for (ctx = adapter->cmd_map; ctx < end; ctx++)  in pvscsi_find_context()
173 pvscsi_acquire_context(struct pvscsi_adapter *adapter, struct scsi_cmnd *cmd)  in pvscsi_acquire_context()  argument
177 	if (list_empty(&adapter->cmd_pool))  in pvscsi_acquire_context()
180 	ctx = list_first_entry(&adapter->cmd_pool, struct pvscsi_ctx, list);  in pvscsi_acquire_context()
187 static void pvscsi_release_context(struct pvscsi_adapter *adapter,  in pvscsi_release_context()  argument
192 	list_add(&ctx->list, &adapter->cmd_pool);  in pvscsi_release_context()
200 static u64 pvscsi_map_context(const struct pvscsi_adapter *adapter,  in pvscsi_map_context()  argument
203 	return ctx - adapter->cmd_map + 1;  in pvscsi_map_context()
207 pvscsi_get_context(const struct pvscsi_adapter *adapter, u64 context)  in pvscsi_get_context()  argument
209 	return &adapter->cmd_map[context - 1];  in pvscsi_get_context()
212 static void pvscsi_reg_write(const struct pvscsi_adapter *adapter,  in pvscsi_reg_write()  argument
215 	writel(val, adapter->mmioBase + offset);  in pvscsi_reg_write()
218 static u32 pvscsi_reg_read(const struct pvscsi_adapter *adapter, u32 offset)  in pvscsi_reg_read()  argument
220 	return readl(adapter->mmioBase + offset);  in pvscsi_reg_read()
223 static u32 pvscsi_read_intr_status(const struct pvscsi_adapter *adapter)  in pvscsi_read_intr_status()  argument
225 	return pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_INTR_STATUS);  in pvscsi_read_intr_status()
228 static void pvscsi_write_intr_status(const struct pvscsi_adapter *adapter,  in pvscsi_write_intr_status()  argument
231 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_STATUS, val);  in pvscsi_write_intr_status()
234 static void pvscsi_unmask_intr(const struct pvscsi_adapter *adapter)  in pvscsi_unmask_intr()  argument
239 	if (adapter->use_msg)  in pvscsi_unmask_intr()
242 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_MASK, intr_bits);  in pvscsi_unmask_intr()
245 static void pvscsi_mask_intr(const struct pvscsi_adapter *adapter)  in pvscsi_mask_intr()  argument
247 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_MASK, 0);  in pvscsi_mask_intr()
250 static void pvscsi_write_cmd_desc(const struct pvscsi_adapter *adapter,  in pvscsi_write_cmd_desc()  argument
257 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND, cmd);  in pvscsi_write_cmd_desc()
259 		pvscsi_reg_write(adapter,  in pvscsi_write_cmd_desc()
263 static void pvscsi_abort_cmd(const struct pvscsi_adapter *adapter,  in pvscsi_abort_cmd()  argument
269 	cmd.context = pvscsi_map_context(adapter, ctx);  in pvscsi_abort_cmd()
271 	pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_ABORT_CMD, &cmd, sizeof(cmd));  in pvscsi_abort_cmd()
274 static void pvscsi_kick_rw_io(const struct pvscsi_adapter *adapter)  in pvscsi_kick_rw_io()  argument
276 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_KICK_RW_IO, 0);  in pvscsi_kick_rw_io()
279 static void pvscsi_process_request_ring(const struct pvscsi_adapter *adapter)  in pvscsi_process_request_ring()  argument
281 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_KICK_NON_RW_IO, 0);  in pvscsi_process_request_ring()
292 static void pvscsi_kick_io(const struct pvscsi_adapter *adapter,  in pvscsi_kick_io()  argument
296 		struct PVSCSIRingsState *s = adapter->rings_state;  in pvscsi_kick_io()
298 		if (!adapter->use_req_threshold ||  in pvscsi_kick_io()
300 			pvscsi_kick_rw_io(adapter);  in pvscsi_kick_io()
302 		pvscsi_process_request_ring(adapter);  in pvscsi_kick_io()
306 static void ll_adapter_reset(const struct pvscsi_adapter *adapter)  in ll_adapter_reset()  argument
308 	dev_dbg(pvscsi_dev(adapter), "Adapter Reset on %p\n", adapter);  in ll_adapter_reset()
310 	pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_ADAPTER_RESET, NULL, 0);  in ll_adapter_reset()
313 static void ll_bus_reset(const struct pvscsi_adapter *adapter)  in ll_bus_reset()  argument
315 	dev_dbg(pvscsi_dev(adapter), "Resetting bus on %p\n", adapter);  in ll_bus_reset()
317 	pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_RESET_BUS, NULL, 0);  in ll_bus_reset()
320 static void ll_device_reset(const struct pvscsi_adapter *adapter, u32 target)  in ll_device_reset()  argument
324 	dev_dbg(pvscsi_dev(adapter), "Resetting device: target=%u\n", target);  in ll_device_reset()
328 	pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_RESET_DEVICE,  in ll_device_reset()
352 static void pvscsi_map_buffers(struct pvscsi_adapter *adapter,  in pvscsi_map_buffers()  argument
373 			ctx->sglPA = pci_map_single(adapter->dev, ctx->sgl,  in pvscsi_map_buffers()
383 		ctx->dataPA = pci_map_single(adapter->dev, sg, bufflen,  in pvscsi_map_buffers()
389 static void pvscsi_unmap_buffers(const struct pvscsi_adapter *adapter,  in pvscsi_unmap_buffers()  argument
404 				pci_unmap_single(adapter->dev, ctx->sglPA,  in pvscsi_unmap_buffers()
409 			pci_unmap_single(adapter->dev, ctx->dataPA, bufflen,  in pvscsi_unmap_buffers()
413 		pci_unmap_single(adapter->dev, ctx->sensePA,  in pvscsi_unmap_buffers()
417 static int pvscsi_allocate_rings(struct pvscsi_adapter *adapter)  in pvscsi_allocate_rings()  argument
419 	adapter->rings_state = pci_alloc_consistent(adapter->dev, PAGE_SIZE,  in pvscsi_allocate_rings()
420 						    &adapter->ringStatePA);  in pvscsi_allocate_rings()
421 	if (!adapter->rings_state)  in pvscsi_allocate_rings()
424 	adapter->req_pages = min(PVSCSI_MAX_NUM_PAGES_REQ_RING,  in pvscsi_allocate_rings()
426 	adapter->req_depth = adapter->req_pages  in pvscsi_allocate_rings()
428 	adapter->req_ring = pci_alloc_consistent(adapter->dev,  in pvscsi_allocate_rings()
429 						 adapter->req_pages * PAGE_SIZE,  in pvscsi_allocate_rings()
430 						 &adapter->reqRingPA);  in pvscsi_allocate_rings()
431 	if (!adapter->req_ring)  in pvscsi_allocate_rings()
434 	adapter->cmp_pages = min(PVSCSI_MAX_NUM_PAGES_CMP_RING,  in pvscsi_allocate_rings()
436 	adapter->cmp_ring = pci_alloc_consistent(adapter->dev,  in pvscsi_allocate_rings()
437 						 adapter->cmp_pages * PAGE_SIZE,  in pvscsi_allocate_rings()
438 						 &adapter->cmpRingPA);  in pvscsi_allocate_rings()
439 	if (!adapter->cmp_ring)  in pvscsi_allocate_rings()
442 	BUG_ON(!IS_ALIGNED(adapter->ringStatePA, PAGE_SIZE));  in pvscsi_allocate_rings()
443 	BUG_ON(!IS_ALIGNED(adapter->reqRingPA, PAGE_SIZE));  in pvscsi_allocate_rings()
444 	BUG_ON(!IS_ALIGNED(adapter->cmpRingPA, PAGE_SIZE));  in pvscsi_allocate_rings()
446 	if (!adapter->use_msg)  in pvscsi_allocate_rings()
449 	adapter->msg_pages = min(PVSCSI_MAX_NUM_PAGES_MSG_RING,  in pvscsi_allocate_rings()
451 	adapter->msg_ring = pci_alloc_consistent(adapter->dev,  in pvscsi_allocate_rings()
452 						 adapter->msg_pages * PAGE_SIZE,  in pvscsi_allocate_rings()
453 						 &adapter->msgRingPA);  in pvscsi_allocate_rings()
454 	if (!adapter->msg_ring)  in pvscsi_allocate_rings()
456 	BUG_ON(!IS_ALIGNED(adapter->msgRingPA, PAGE_SIZE));  in pvscsi_allocate_rings()
461 static void pvscsi_setup_all_rings(const struct pvscsi_adapter *adapter)  in pvscsi_setup_all_rings()  argument
467 	cmd.ringsStatePPN   = adapter->ringStatePA >> PAGE_SHIFT;  in pvscsi_setup_all_rings()
468 	cmd.reqRingNumPages = adapter->req_pages;  in pvscsi_setup_all_rings()
469 	cmd.cmpRingNumPages = adapter->cmp_pages;  in pvscsi_setup_all_rings()
471 	base = adapter->reqRingPA;  in pvscsi_setup_all_rings()
472 	for (i = 0; i < adapter->req_pages; i++) {  in pvscsi_setup_all_rings()
477 	base = adapter->cmpRingPA;  in pvscsi_setup_all_rings()
478 	for (i = 0; i < adapter->cmp_pages; i++) {  in pvscsi_setup_all_rings()
483 	memset(adapter->rings_state, 0, PAGE_SIZE);  in pvscsi_setup_all_rings()
484 	memset(adapter->req_ring, 0, adapter->req_pages * PAGE_SIZE);  in pvscsi_setup_all_rings()
485 	memset(adapter->cmp_ring, 0, adapter->cmp_pages * PAGE_SIZE);  in pvscsi_setup_all_rings()
487 	pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_SETUP_RINGS,  in pvscsi_setup_all_rings()
490 	if (adapter->use_msg) {  in pvscsi_setup_all_rings()
493 		cmd_msg.numPages = adapter->msg_pages;  in pvscsi_setup_all_rings()
495 		base = adapter->msgRingPA;  in pvscsi_setup_all_rings()
496 		for (i = 0; i < adapter->msg_pages; i++) {  in pvscsi_setup_all_rings()
500 		memset(adapter->msg_ring, 0, adapter->msg_pages * PAGE_SIZE);  in pvscsi_setup_all_rings()
502 		pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_SETUP_MSG_RING,  in pvscsi_setup_all_rings()
518 static void pvscsi_complete_request(struct pvscsi_adapter *adapter,  in pvscsi_complete_request()  argument
527 	ctx = pvscsi_get_context(adapter, e->context);  in pvscsi_complete_request()
530 	pvscsi_unmap_buffers(adapter, ctx);  in pvscsi_complete_request()
531 	pvscsi_release_context(adapter, ctx);  in pvscsi_complete_request()
624 static void pvscsi_process_completion_ring(struct pvscsi_adapter *adapter)  in pvscsi_process_completion_ring()  argument
626 	struct PVSCSIRingsState *s = adapter->rings_state;  in pvscsi_process_completion_ring()
627 	struct PVSCSIRingCmpDesc *ring = adapter->cmp_ring;  in pvscsi_process_completion_ring()
640 		pvscsi_complete_request(adapter, e);  in pvscsi_process_completion_ring()
655 static int pvscsi_queue_ring(struct pvscsi_adapter *adapter,  in pvscsi_queue_ring()  argument
663 	s = adapter->rings_state;  in pvscsi_queue_ring()
682 	e = adapter->req_ring + (s->reqProdIdx & MASK(req_entries));  in pvscsi_queue_ring()
690 		ctx->sensePA = pci_map_single(adapter->dev, cmd->sense_buffer,  in pvscsi_queue_ring()
714 	pvscsi_map_buffers(adapter, ctx, cmd, e);  in pvscsi_queue_ring()
716 	e->context = pvscsi_map_context(adapter, ctx);  in pvscsi_queue_ring()
728 	struct pvscsi_adapter *adapter = shost_priv(host);  in pvscsi_queue_lck()  local
732 	spin_lock_irqsave(&adapter->hw_lock, flags);  in pvscsi_queue_lck()
734 	ctx = pvscsi_acquire_context(adapter, cmd);  in pvscsi_queue_lck()
735 	if (!ctx || pvscsi_queue_ring(adapter, ctx, cmd) != 0) {  in pvscsi_queue_lck()
737 			pvscsi_release_context(adapter, ctx);  in pvscsi_queue_lck()
738 		spin_unlock_irqrestore(&adapter->hw_lock, flags);  in pvscsi_queue_lck()
747 	spin_unlock_irqrestore(&adapter->hw_lock, flags);  in pvscsi_queue_lck()
749 	pvscsi_kick_io(adapter, cmd->cmnd[0]);  in pvscsi_queue_lck()
758 	struct pvscsi_adapter *adapter = shost_priv(cmd->device->host);  in DEF_SCSI_QCMD()  local
765 		    adapter->host->host_no, cmd);  in DEF_SCSI_QCMD()
767 	spin_lock_irqsave(&adapter->hw_lock, flags);  in DEF_SCSI_QCMD()
773 	pvscsi_process_completion_ring(adapter);  in DEF_SCSI_QCMD()
779 	ctx = pvscsi_find_context(adapter, cmd);  in DEF_SCSI_QCMD()
791 	pvscsi_abort_cmd(adapter, ctx);  in DEF_SCSI_QCMD()
792 	spin_unlock_irqrestore(&adapter->hw_lock, flags);  in DEF_SCSI_QCMD()
795 	spin_lock_irqsave(&adapter->hw_lock, flags);  in DEF_SCSI_QCMD()
817 	spin_unlock_irqrestore(&adapter->hw_lock, flags);  in DEF_SCSI_QCMD()
827 static void pvscsi_reset_all(struct pvscsi_adapter *adapter)  in pvscsi_reset_all()  argument
831 	for (i = 0; i < adapter->req_depth; i++) {  in pvscsi_reset_all()
832 		struct pvscsi_ctx *ctx = &adapter->cmd_map[i];  in pvscsi_reset_all()
837 			pvscsi_unmap_buffers(adapter, ctx);  in pvscsi_reset_all()
838 			pvscsi_release_context(adapter, ctx);  in pvscsi_reset_all()
848 	struct pvscsi_adapter *adapter = shost_priv(host);  in pvscsi_host_reset()  local
854 	spin_lock_irqsave(&adapter->hw_lock, flags);  in pvscsi_host_reset()
856 	use_msg = adapter->use_msg;  in pvscsi_host_reset()
859 		adapter->use_msg = 0;  in pvscsi_host_reset()
860 		spin_unlock_irqrestore(&adapter->hw_lock, flags);  in pvscsi_host_reset()
866 		flush_workqueue(adapter->workqueue);  in pvscsi_host_reset()
867 		spin_lock_irqsave(&adapter->hw_lock, flags);  in pvscsi_host_reset()
876 	pvscsi_process_request_ring(adapter);  in pvscsi_host_reset()
878 	ll_adapter_reset(adapter);  in pvscsi_host_reset()
887 	pvscsi_process_completion_ring(adapter);  in pvscsi_host_reset()
889 	pvscsi_reset_all(adapter);  in pvscsi_host_reset()
890 	adapter->use_msg = use_msg;  in pvscsi_host_reset()
891 	pvscsi_setup_all_rings(adapter);  in pvscsi_host_reset()
892 	pvscsi_unmask_intr(adapter);  in pvscsi_host_reset()
894 	spin_unlock_irqrestore(&adapter->hw_lock, flags);  in pvscsi_host_reset()
902 	struct pvscsi_adapter *adapter = shost_priv(host);  in pvscsi_bus_reset()  local
913 	spin_lock_irqsave(&adapter->hw_lock, flags);  in pvscsi_bus_reset()
915 	pvscsi_process_request_ring(adapter);  in pvscsi_bus_reset()
916 	ll_bus_reset(adapter);  in pvscsi_bus_reset()
917 	pvscsi_process_completion_ring(adapter);  in pvscsi_bus_reset()
919 	spin_unlock_irqrestore(&adapter->hw_lock, flags);  in pvscsi_bus_reset()
927 	struct pvscsi_adapter *adapter = shost_priv(host);  in pvscsi_device_reset()  local
938 	spin_lock_irqsave(&adapter->hw_lock, flags);  in pvscsi_device_reset()
940 	pvscsi_process_request_ring(adapter);  in pvscsi_device_reset()
941 	ll_device_reset(adapter, cmd->device->id);  in pvscsi_device_reset()
942 	pvscsi_process_completion_ring(adapter);  in pvscsi_device_reset()
944 	spin_unlock_irqrestore(&adapter->hw_lock, flags);  in pvscsi_device_reset()
953 	struct pvscsi_adapter *adapter = shost_priv(host);  in pvscsi_info()  local
957 		"%u/%u/%u pages, cmd_per_lun=%u", adapter->rev,  in pvscsi_info()
958 		adapter->req_pages, adapter->cmp_pages, adapter->msg_pages,  in pvscsi_info()
982 static void pvscsi_process_msg(const struct pvscsi_adapter *adapter,  in pvscsi_process_msg()  argument
985 	struct PVSCSIRingsState *s = adapter->rings_state;  in pvscsi_process_msg()
986 	struct Scsi_Host *host = adapter->host;  in pvscsi_process_msg()
1011 			scsi_add_device(adapter->host, desc->bus,  in pvscsi_process_msg()
1040 static int pvscsi_msg_pending(const struct pvscsi_adapter *adapter)  in pvscsi_msg_pending()  argument
1042 	struct PVSCSIRingsState *s = adapter->rings_state;  in pvscsi_msg_pending()
1047 static void pvscsi_process_msg_ring(const struct pvscsi_adapter *adapter)  in pvscsi_process_msg_ring()  argument
1049 	struct PVSCSIRingsState *s = adapter->rings_state;  in pvscsi_process_msg_ring()
1050 	struct PVSCSIRingMsgDesc *ring = adapter->msg_ring;  in pvscsi_process_msg_ring()
1053 	while (pvscsi_msg_pending(adapter)) {  in pvscsi_process_msg_ring()
1058 		pvscsi_process_msg(adapter, e);  in pvscsi_process_msg_ring()
1066 	struct pvscsi_adapter *adapter;  in pvscsi_msg_workqueue_handler()  local
1068 	adapter = container_of(data, struct pvscsi_adapter, work);  in pvscsi_msg_workqueue_handler()
1070 	pvscsi_process_msg_ring(adapter);  in pvscsi_msg_workqueue_handler()
1073 static int pvscsi_setup_msg_workqueue(struct pvscsi_adapter *adapter)  in pvscsi_setup_msg_workqueue()  argument
1080 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND,  in pvscsi_setup_msg_workqueue()
1083 	if (pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_COMMAND_STATUS) == -1)  in pvscsi_setup_msg_workqueue()
1087 		 "vmw_pvscsi_wq_%u", adapter->host->host_no);  in pvscsi_setup_msg_workqueue()
1089 	adapter->workqueue = create_singlethread_workqueue(name);  in pvscsi_setup_msg_workqueue()
1090 	if (!adapter->workqueue) {  in pvscsi_setup_msg_workqueue()
1094 	INIT_WORK(&adapter->work, pvscsi_msg_workqueue_handler);  in pvscsi_setup_msg_workqueue()
1099 static bool pvscsi_setup_req_threshold(struct pvscsi_adapter *adapter,  in pvscsi_setup_req_threshold()  argument
1107 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND,  in pvscsi_setup_req_threshold()
1109 	val = pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_COMMAND_STATUS);  in pvscsi_setup_req_threshold()
1119 		pvscsi_write_cmd_desc(adapter,  in pvscsi_setup_req_threshold()
1122 		return pvscsi_reg_read(adapter,  in pvscsi_setup_req_threshold()
1129 	struct pvscsi_adapter *adapter = devp;  in pvscsi_isr()  local
1132 	if (adapter->use_msi || adapter->use_msix)  in pvscsi_isr()
1135 		u32 val = pvscsi_read_intr_status(adapter);  in pvscsi_isr()
1144 		spin_lock_irqsave(&adapter->hw_lock, flags);  in pvscsi_isr()
1146 		pvscsi_process_completion_ring(adapter);  in pvscsi_isr()
1147 		if (adapter->use_msg && pvscsi_msg_pending(adapter))  in pvscsi_isr()
1148 			queue_work(adapter->workqueue, &adapter->work);  in pvscsi_isr()
1150 		spin_unlock_irqrestore(&adapter->hw_lock, flags);  in pvscsi_isr()
1156 static void pvscsi_free_sgls(const struct pvscsi_adapter *adapter)  in pvscsi_free_sgls()  argument
1158 	struct pvscsi_ctx *ctx = adapter->cmd_map;  in pvscsi_free_sgls()
1161 	for (i = 0; i < adapter->req_depth; ++i, ++ctx)  in pvscsi_free_sgls()
1165 static int pvscsi_setup_msix(const struct pvscsi_adapter *adapter,  in pvscsi_setup_msix()  argument
1171 	ret = pci_enable_msix_exact(adapter->dev, &entry, 1);  in pvscsi_setup_msix()
1180 static void pvscsi_shutdown_intr(struct pvscsi_adapter *adapter)  in pvscsi_shutdown_intr()  argument
1182 	if (adapter->irq) {  in pvscsi_shutdown_intr()
1183 		free_irq(adapter->irq, adapter);  in pvscsi_shutdown_intr()
1184 		adapter->irq = 0;  in pvscsi_shutdown_intr()
1186 	if (adapter->use_msi) {  in pvscsi_shutdown_intr()
1187 		pci_disable_msi(adapter->dev);  in pvscsi_shutdown_intr()
1188 		adapter->use_msi = 0;  in pvscsi_shutdown_intr()
1189 	} else if (adapter->use_msix) {  in pvscsi_shutdown_intr()
1190 		pci_disable_msix(adapter->dev);  in pvscsi_shutdown_intr()
1191 		adapter->use_msix = 0;  in pvscsi_shutdown_intr()
1195 static void pvscsi_release_resources(struct pvscsi_adapter *adapter)  in pvscsi_release_resources()  argument
1197 	pvscsi_shutdown_intr(adapter);  in pvscsi_release_resources()
1199 	if (adapter->workqueue)  in pvscsi_release_resources()
1200 		destroy_workqueue(adapter->workqueue);  in pvscsi_release_resources()
1202 	if (adapter->mmioBase)  in pvscsi_release_resources()
1203 		pci_iounmap(adapter->dev, adapter->mmioBase);  in pvscsi_release_resources()
1205 	pci_release_regions(adapter->dev);  in pvscsi_release_resources()
1207 	if (adapter->cmd_map) {  in pvscsi_release_resources()
1208 		pvscsi_free_sgls(adapter);  in pvscsi_release_resources()
1209 		kfree(adapter->cmd_map);  in pvscsi_release_resources()
1212 	if (adapter->rings_state)  in pvscsi_release_resources()
1213 		pci_free_consistent(adapter->dev, PAGE_SIZE,  in pvscsi_release_resources()
1214 				    adapter->rings_state, adapter->ringStatePA);  in pvscsi_release_resources()
1216 	if (adapter->req_ring)  in pvscsi_release_resources()
1217 		pci_free_consistent(adapter->dev,  in pvscsi_release_resources()
1218 				    adapter->req_pages * PAGE_SIZE,  in pvscsi_release_resources()
1219 				    adapter->req_ring, adapter->reqRingPA);  in pvscsi_release_resources()
1221 	if (adapter->cmp_ring)  in pvscsi_release_resources()
1222 		pci_free_consistent(adapter->dev,  in pvscsi_release_resources()
1223 				    adapter->cmp_pages * PAGE_SIZE,  in pvscsi_release_resources()
1224 				    adapter->cmp_ring, adapter->cmpRingPA);  in pvscsi_release_resources()
1226 	if (adapter->msg_ring)  in pvscsi_release_resources()
1227 		pci_free_consistent(adapter->dev,  in pvscsi_release_resources()
1228 				    adapter->msg_pages * PAGE_SIZE,  in pvscsi_release_resources()
1229 				    adapter->msg_ring, adapter->msgRingPA);  in pvscsi_release_resources()
1245 static int pvscsi_allocate_sg(struct pvscsi_adapter *adapter)  in pvscsi_allocate_sg()  argument
1250 	ctx = adapter->cmd_map;  in pvscsi_allocate_sg()
1253 	for (i = 0; i < adapter->req_depth; ++i, ++ctx) {  in pvscsi_allocate_sg()
1276 static u32 pvscsi_get_max_targets(struct pvscsi_adapter *adapter)  in pvscsi_get_max_targets()  argument
1285 	dev = pvscsi_dev(adapter);  in pvscsi_get_max_targets()
1286 	config_page = pci_alloc_consistent(adapter->dev, PAGE_SIZE,  in pvscsi_get_max_targets()
1310 	pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_CONFIG, &cmd, sizeof cmd);  in pvscsi_get_max_targets()
1321 	pci_free_consistent(adapter->dev, PAGE_SIZE, config_page, configPagePA);  in pvscsi_get_max_targets()
1328 	struct pvscsi_adapter *adapter;  in pvscsi_probe()  local
1357 	adapter = &adapter_temp;  in pvscsi_probe()
1358 	memset(adapter, 0, sizeof(*adapter));  in pvscsi_probe()
1359 	adapter->dev  = pdev;  in pvscsi_probe()
1360 	adapter->rev = pdev->revision;  in pvscsi_probe()
1383 	adapter->mmioBase = pci_iomap(pdev, i, PVSCSI_MEM_SPACE_SIZE);  in pvscsi_probe()
1385 	if (!adapter->mmioBase) {  in pvscsi_probe()
1398 	max_id = pvscsi_get_max_targets(adapter);  in pvscsi_probe()
1427 	adapter = shost_priv(host);  in pvscsi_probe()
1428 	memset(adapter, 0, sizeof(*adapter));  in pvscsi_probe()
1429 	adapter->dev  = pdev;  in pvscsi_probe()
1430 	adapter->host = host;  in pvscsi_probe()
1434 	adapter->rev = adapter_temp.rev;  in pvscsi_probe()
1435 	adapter->mmioBase = adapter_temp.mmioBase;  in pvscsi_probe()
1437 	spin_lock_init(&adapter->hw_lock);  in pvscsi_probe()
1445 	ll_adapter_reset(adapter);  in pvscsi_probe()
1447 	adapter->use_msg = pvscsi_setup_msg_workqueue(adapter);  in pvscsi_probe()
1449 	error = pvscsi_allocate_rings(adapter);  in pvscsi_probe()
1459 	pvscsi_setup_all_rings(adapter);  in pvscsi_probe()
1461 	adapter->cmd_map = kcalloc(adapter->req_depth,  in pvscsi_probe()
1463 	if (!adapter->cmd_map) {  in pvscsi_probe()
1469 	INIT_LIST_HEAD(&adapter->cmd_pool);  in pvscsi_probe()
1470 	for (i = 0; i < adapter->req_depth; i++) {  in pvscsi_probe()
1471 		struct pvscsi_ctx *ctx = adapter->cmd_map + i;  in pvscsi_probe()
1472 		list_add(&ctx->list, &adapter->cmd_pool);  in pvscsi_probe()
1475 	error = pvscsi_allocate_sg(adapter);  in pvscsi_probe()
1482 	    pvscsi_setup_msix(adapter, &adapter->irq) == 0) {  in pvscsi_probe()
1484 		adapter->use_msix = 1;  in pvscsi_probe()
1487 		adapter->use_msi = 1;  in pvscsi_probe()
1488 		adapter->irq = pdev->irq;  in pvscsi_probe()
1491 		adapter->irq = pdev->irq;  in pvscsi_probe()
1495 	adapter->use_req_threshold = pvscsi_setup_req_threshold(adapter, true);  in pvscsi_probe()
1497 	       adapter->use_req_threshold ? "en" : "dis");  in pvscsi_probe()
1499 	error = request_irq(adapter->irq, pvscsi_isr, flags,  in pvscsi_probe()
1500 			    "vmw_pvscsi", adapter);  in pvscsi_probe()
1504 		adapter->irq = 0;  in pvscsi_probe()
1516 		 adapter->rev, host->host_no);  in pvscsi_probe()
1518 	pvscsi_unmask_intr(adapter);  in pvscsi_probe()
1525 	ll_adapter_reset(adapter);  in pvscsi_probe()
1527 	pvscsi_release_resources(adapter);  in pvscsi_probe()
1535 	pvscsi_release_resources(adapter);  in pvscsi_probe()
1539 static void __pvscsi_shutdown(struct pvscsi_adapter *adapter)  in __pvscsi_shutdown()  argument
1541 	pvscsi_mask_intr(adapter);  in __pvscsi_shutdown()
1543 	if (adapter->workqueue)  in __pvscsi_shutdown()
1544 		flush_workqueue(adapter->workqueue);  in __pvscsi_shutdown()
1546 	pvscsi_shutdown_intr(adapter);  in __pvscsi_shutdown()
1548 	pvscsi_process_request_ring(adapter);  in __pvscsi_shutdown()
1549 	pvscsi_process_completion_ring(adapter);  in __pvscsi_shutdown()
1550 	ll_adapter_reset(adapter);  in __pvscsi_shutdown()
1556 	struct pvscsi_adapter *adapter = shost_priv(host);  in pvscsi_shutdown()  local
1558 	__pvscsi_shutdown(adapter);  in pvscsi_shutdown()
1564 	struct pvscsi_adapter *adapter = shost_priv(host);  in pvscsi_remove()  local
1568 	__pvscsi_shutdown(adapter);  in pvscsi_remove()
1569 	pvscsi_release_resources(adapter);  in pvscsi_remove()