Lines Matching refs:dd
124 static int mtip_block_initialize(struct driver_data *dd);
152 struct driver_data *dd = pci_get_drvdata(pdev); in mtip_check_surprise_removal() local
154 if (dd->sr) in mtip_check_surprise_removal()
160 dd->sr = true; in mtip_check_surprise_removal()
161 if (dd->queue) in mtip_check_surprise_removal()
162 set_bit(QUEUE_FLAG_DEAD, &dd->queue->queue_flags); in mtip_check_surprise_removal()
164 dev_warn(&dd->pdev->dev, in mtip_check_surprise_removal()
166 if (dd->port) { in mtip_check_surprise_removal()
167 set_bit(MTIP_PF_SR_CLEANUP_BIT, &dd->port->flags); in mtip_check_surprise_removal()
168 wake_up_interruptible(&dd->port->svc_wait); in mtip_check_surprise_removal()
170 dev_warn(&dd->pdev->dev, in mtip_check_surprise_removal()
178 static struct mtip_cmd *mtip_get_int_command(struct driver_data *dd) in mtip_get_int_command() argument
182 rq = blk_mq_alloc_request(dd->queue, 0, __GFP_WAIT, true); in mtip_get_int_command()
186 static void mtip_put_int_command(struct driver_data *dd, struct mtip_cmd *cmd) in mtip_put_int_command() argument
194 static struct request *mtip_rq_from_tag(struct driver_data *dd, in mtip_rq_from_tag() argument
197 struct blk_mq_hw_ctx *hctx = dd->queue->queue_hw_ctx[0]; in mtip_rq_from_tag()
202 static struct mtip_cmd *mtip_cmd_from_tag(struct driver_data *dd, in mtip_cmd_from_tag() argument
205 struct request *rq = mtip_rq_from_tag(dd, tag); in mtip_cmd_from_tag()
231 struct driver_data *dd = port->dd; in mtip_async_complete() local
234 if (unlikely(!dd) || unlikely(!port)) in mtip_async_complete()
238 dev_warn(&port->dd->pdev->dev, in mtip_async_complete()
243 dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents, cmd->direction); in mtip_async_complete()
245 rq = mtip_rq_from_tag(dd, tag); in mtip_async_complete()
262 static int mtip_hba_reset(struct driver_data *dd) in mtip_hba_reset() argument
267 writel(HOST_RESET, dd->mmio + HOST_CTL); in mtip_hba_reset()
270 readl(dd->mmio + HOST_CTL); in mtip_hba_reset()
276 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) in mtip_hba_reset()
279 } while ((readl(dd->mmio + HOST_CTL) & HOST_RESET) in mtip_hba_reset()
282 if (readl(dd->mmio + HOST_CTL) & HOST_RESET) in mtip_hba_reset()
418 if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) { in mtip_init_port()
433 for (i = 0; i < port->dd->slot_groups; i++) in mtip_init_port()
440 writel(readl(port->dd->mmio + HOST_IRQ_STAT), in mtip_init_port()
441 port->dd->mmio + HOST_IRQ_STAT); in mtip_init_port()
468 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
476 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
479 if (mtip_hba_reset(port->dd)) in mtip_restart_port()
480 dev_err(&port->dd->pdev->dev, in mtip_restart_port()
487 dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n"); in mtip_restart_port()
499 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
513 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
517 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
525 static int mtip_device_reset(struct driver_data *dd) in mtip_device_reset() argument
529 if (mtip_check_surprise_removal(dd->pdev)) in mtip_device_reset()
532 if (mtip_hba_reset(dd) < 0) in mtip_device_reset()
536 mtip_init_port(dd->port); in mtip_device_reset()
537 mtip_start_port(dd->port); in mtip_device_reset()
540 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_device_reset()
541 dd->mmio + HOST_CTL); in mtip_device_reset()
548 static void print_tags(struct driver_data *dd, in print_tags() argument
560 dev_warn(&dd->pdev->dev, in print_tags()
584 dev_warn(&port->dd->pdev->dev, in mtip_completion()
607 static void mtip_handle_tfe(struct driver_data *dd) in mtip_handle_tfe() argument
620 dev_warn(&dd->pdev->dev, "Taskfile error\n"); in mtip_handle_tfe()
622 port = dd->port; in mtip_handle_tfe()
628 cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); in mtip_handle_tfe()
642 for (group = 0; group < dd->slot_groups; group++) { in mtip_handle_tfe()
645 dev_warn(&dd->pdev->dev, "g=%u, comp=%x\n", group, completed); in mtip_handle_tfe()
660 cmd = mtip_cmd_from_tag(dd, tag); in mtip_handle_tfe()
666 dev_err(&port->dd->pdev->dev, in mtip_handle_tfe()
669 if (mtip_check_surprise_removal(dd->pdev)) { in mtip_handle_tfe()
677 print_tags(dd, "completed (TFE)", tagaccum, cmd_cnt); in mtip_handle_tfe()
684 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_handle_tfe()
685 dd->port->log_buf, in mtip_handle_tfe()
686 dd->port->log_buf_dma, 1); in mtip_handle_tfe()
688 dev_warn(&dd->pdev->dev, in mtip_handle_tfe()
692 buf = (unsigned char *)dd->port->log_buf; in mtip_handle_tfe()
694 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
696 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); in mtip_handle_tfe()
701 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
703 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); in mtip_handle_tfe()
708 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); in mtip_handle_tfe()
709 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
720 for (group = 0; group < dd->slot_groups; group++) { in mtip_handle_tfe()
724 cmd = mtip_cmd_from_tag(dd, tag); in mtip_handle_tfe()
736 dev_warn(&dd->pdev->dev, in mtip_handle_tfe()
766 dev_warn(&port->dd->pdev->dev, in mtip_handle_tfe()
772 dev_warn(&port->dd->pdev->dev, in mtip_handle_tfe()
777 print_tags(dd, "reissued (TFE)", tagaccum, cmd_cnt); in mtip_handle_tfe()
791 struct driver_data *dd = port->dd; in mtip_workq_sdbfx() local
811 command = mtip_cmd_from_tag(dd, tag); in mtip_workq_sdbfx()
815 dev_dbg(&dd->pdev->dev, in mtip_workq_sdbfx()
820 dd->pdev)) { in mtip_workq_sdbfx()
829 if (atomic_dec_return(&dd->irq_workers_active) == 0) in mtip_workq_sdbfx()
830 writel(0xffffffff, dd->mmio + HOST_IRQ_STAT); in mtip_workq_sdbfx()
836 static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat) in mtip_process_legacy() argument
838 struct mtip_port *port = dd->port; in mtip_process_legacy()
839 struct mtip_cmd *cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); in mtip_process_legacy()
856 static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat) in mtip_process_errors() argument
860 dev_warn(&dd->pdev->dev, in mtip_process_errors()
862 writel((1 << 26), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
866 dev_warn(&dd->pdev->dev, in mtip_process_errors()
868 writel((1 << 16), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
872 dev_warn(&dd->pdev->dev, in mtip_process_errors()
875 if (mtip_check_surprise_removal(dd->pdev)) in mtip_process_errors()
879 set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags); in mtip_process_errors()
880 wake_up_interruptible(&dd->port->svc_wait); in mtip_process_errors()
886 struct driver_data *dd = (struct driver_data *) data; in mtip_handle_irq() local
887 struct mtip_port *port = dd->port; in mtip_handle_irq()
893 hba_stat = readl(dd->mmio + HOST_IRQ_STAT); in mtip_handle_irq()
900 mtip_check_surprise_removal(dd->pdev); in mtip_handle_irq()
908 WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0); in mtip_handle_irq()
913 twork = &dd->work[i]; in mtip_handle_irq()
919 atomic_set(&dd->irq_workers_active, workers); in mtip_handle_irq()
922 twork = &dd->work[i]; in mtip_handle_irq()
926 dd->isr_workq, in mtip_handle_irq()
930 if (likely(dd->work[0].completed)) in mtip_handle_irq()
932 dd->work[0].completed); in mtip_handle_irq()
944 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { in mtip_handle_irq()
949 &dd->dd_flag)) in mtip_handle_irq()
952 mtip_process_errors(dd, port_stat & PORT_IRQ_ERR); in mtip_handle_irq()
956 mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY); in mtip_handle_irq()
961 writel(hba_stat, dd->mmio + HOST_IRQ_STAT); in mtip_handle_irq()
978 struct driver_data *dd = instance; in mtip_irq_handler() local
980 return mtip_handle_irq(dd); in mtip_irq_handler()
1014 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
1015 clear_bit(MTIP_DDF_REBUILD_FAILED_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
1040 blk_mq_stop_hw_queues(port->dd->queue); in mtip_quiesce_io()
1051 if (mtip_check_surprise_removal(port->dd->pdev)) in mtip_quiesce_io()
1053 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_quiesce_io()
1061 for (n = 1; n < port->dd->slot_groups; n++) in mtip_quiesce_io()
1068 blk_mq_start_stopped_hw_queues(port->dd->queue, true); in mtip_quiesce_io()
1071 blk_mq_start_stopped_hw_queues(port->dd->queue, true); in mtip_quiesce_io()
1105 struct driver_data *dd = port->dd; in mtip_exec_internal_command() local
1111 dev_err(&dd->pdev->dev, "SG buffer is not 8 byte aligned\n"); in mtip_exec_internal_command()
1115 int_cmd = mtip_get_int_command(dd); in mtip_exec_internal_command()
1128 dev_warn(&dd->pdev->dev, in mtip_exec_internal_command()
1130 mtip_put_int_command(dd, int_cmd); in mtip_exec_internal_command()
1182 dev_err(&dd->pdev->dev, in mtip_exec_internal_command()
1189 dev_err(&dd->pdev->dev, in mtip_exec_internal_command()
1193 dev_err(&dd->pdev->dev, in mtip_exec_internal_command()
1197 if (mtip_check_surprise_removal(dd->pdev) || in mtip_exec_internal_command()
1199 &dd->dd_flag)) { in mtip_exec_internal_command()
1200 dev_err(&dd->pdev->dev, in mtip_exec_internal_command()
1206 mtip_device_reset(dd); /* recover from timeout issue */ in mtip_exec_internal_command()
1218 if (mtip_check_surprise_removal(dd->pdev)) { in mtip_exec_internal_command()
1224 &dd->dd_flag)) { in mtip_exec_internal_command()
1233 dev_err(&dd->pdev->dev, in mtip_exec_internal_command()
1236 mtip_device_reset(dd); in mtip_exec_internal_command()
1241 hba_stat = readl(dd->mmio + HOST_IRQ_STAT); in mtip_exec_internal_command()
1244 dd->mmio + HOST_IRQ_STAT); in mtip_exec_internal_command()
1253 if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { in mtip_exec_internal_command()
1254 mtip_device_reset(dd); in mtip_exec_internal_command()
1260 mtip_put_int_command(dd, int_cmd); in mtip_exec_internal_command()
1291 static void mtip_set_timeout(struct driver_data *dd, in mtip_set_timeout() argument
1302 *timeout = ((*(dd->port->identify + 90) * 2) * 60000); in mtip_set_timeout()
1304 *timeout = ((*(dd->port->identify + 89) * 2) * 60000); in mtip_set_timeout()
1346 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_get_identify()
1394 set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_get_identify()
1396 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_get_identify()
1401 port->dd->trim_supp = true; in mtip_get_identify()
1404 port->dd->trim_supp = false; in mtip_get_identify()
1445 mtip_set_timeout(port->dd, &fis, &timeout, 0); in mtip_standby_immediate()
1459 dev_warn(&port->dd->pdev->dev, in mtip_standby_immediate()
1560 dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n"); in mtip_get_smart_attr()
1564 dev_warn(&port->dd->pdev->dev, "SMART not supported\n"); in mtip_get_smart_attr()
1568 dev_warn(&port->dd->pdev->dev, "SMART not enabled\n"); in mtip_get_smart_attr()
1575 dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n"); in mtip_get_smart_attr()
1587 dev_warn(&port->dd->pdev->dev, in mtip_get_smart_attr()
1607 static int mtip_send_trim(struct driver_data *dd, unsigned int lba, in mtip_send_trim() argument
1616 if (!len || dd->trim_supp == false) in mtip_send_trim()
1629 buf = dmam_alloc_coherent(&dd->pdev->dev, ATA_SECT_SIZE, &dma_addr, in mtip_send_trim()
1657 if (mtip_exec_internal_command(dd->port, in mtip_send_trim()
1667 dmam_free_coherent(&dd->pdev->dev, ATA_SECT_SIZE, buf, dma_addr); in mtip_send_trim()
1681 static bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors) in mtip_hw_get_capacity() argument
1683 struct mtip_port *port = dd->port; in mtip_hw_get_capacity()
1712 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1716 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1720 dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf); in mtip_dump_identify()
1722 dev_info(&port->dd->pdev->dev, "Security: %04x %s\n", in mtip_dump_identify()
1726 if (mtip_hw_get_capacity(port->dd, §ors)) in mtip_dump_identify()
1727 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1732 pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid); in mtip_dump_identify()
1744 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1757 static inline void fill_command_sg(struct driver_data *dd, in fill_command_sg() argument
1771 dev_err(&dd->pdev->dev, in fill_command_sg()
1808 mtip_set_timeout(port->dd, &fis, &to, 0); in exec_drive_task()
1874 buf = dmam_alloc_coherent(&port->dd->pdev->dev, in exec_drive_command()
1879 dev_err(&port->dd->pdev->dev, in exec_drive_command()
1900 mtip_set_timeout(port->dd, &fis, &to, 0); in exec_drive_command()
1953 dmam_free_coherent(&port->dd->pdev->dev, in exec_drive_command()
2009 static int exec_drive_taskfile(struct driver_data *dd, in exec_drive_taskfile() argument
2050 outbuf_dma = pci_map_single(dd->pdev, in exec_drive_taskfile()
2072 inbuf_dma = pci_map_single(dd->pdev, in exec_drive_taskfile()
2086 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
2089 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
2092 reply = (dd->port->rxfis + RX_FIS_D2H_REG); in exec_drive_taskfile()
2134 dev_warn(&dd->pdev->dev, in exec_drive_taskfile()
2162 mtip_set_timeout(dd, &fis, &timeout, erasemode); in exec_drive_taskfile()
2171 if (mtip_exec_internal_command(dd->port, in exec_drive_taskfile()
2183 task_file_data = readl(dd->port->mmio+PORT_TFDATA); in exec_drive_taskfile()
2186 reply = dd->port->rxfis + RX_FIS_PIO_SETUP; in exec_drive_taskfile()
2189 reply = dd->port->rxfis + RX_FIS_D2H_REG; in exec_drive_taskfile()
2195 pci_unmap_single(dd->pdev, inbuf_dma, in exec_drive_taskfile()
2198 pci_unmap_single(dd->pdev, outbuf_dma, in exec_drive_taskfile()
2246 pci_unmap_single(dd->pdev, inbuf_dma, in exec_drive_taskfile()
2249 pci_unmap_single(dd->pdev, outbuf_dma, in exec_drive_taskfile()
2274 static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd, in mtip_hw_ioctl() argument
2280 if (copy_to_user((void __user *)arg, dd->port->identify, in mtip_hw_ioctl()
2296 if (exec_drive_command(dd->port, in mtip_hw_ioctl()
2320 if (exec_drive_task(dd->port, drive_command)) in mtip_hw_ioctl()
2341 ret = exec_drive_taskfile(dd, (void __user *) arg, in mtip_hw_ioctl()
2378 static void mtip_hw_submit_io(struct driver_data *dd, struct request *rq, in mtip_hw_submit_io() argument
2383 struct mtip_port *port = dd->port; in mtip_hw_submit_io()
2389 nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir); in mtip_hw_submit_io()
2423 fill_command_sg(dd, command, nents); in mtip_hw_submit_io()
2438 command->comp_data = dd; in mtip_hw_submit_io()
2470 struct driver_data *dd = dev_to_disk(dev)->private_data; in mtip_hw_show_status() local
2473 if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag)) in mtip_hw_show_status()
2475 else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag)) in mtip_hw_show_status()
2490 struct driver_data *dd, *tmp; in show_device_status() local
2497 list_for_each_entry_safe(dd, tmp, &online_list, online_list) { in show_device_status()
2498 if (dd->pdev) { in show_device_status()
2499 if (dd->port && in show_device_status()
2500 dd->port->identify && in show_device_status()
2501 dd->port->identify_valid) { in show_device_status()
2503 (char *) (dd->port->identify + 10), 21); in show_device_status()
2504 status = *(dd->port->identify + 141); in show_device_status()
2510 if (dd->port && in show_device_status()
2511 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { in show_device_status()
2514 dev_name(&dd->pdev->dev), in show_device_status()
2520 dev_name(&dd->pdev->dev), in show_device_status()
2527 list_for_each_entry_safe(dd, tmp, &removing_list, remove_list) { in show_device_status()
2528 if (dd->pdev) { in show_device_status()
2529 if (dd->port && in show_device_status()
2530 dd->port->identify && in show_device_status()
2531 dd->port->identify_valid) { in show_device_status()
2533 (char *) (dd->port->identify+10), 21); in show_device_status()
2534 status = *(dd->port->identify + 141); in show_device_status()
2540 if (dd->port && in show_device_status()
2541 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { in show_device_status()
2544 dev_name(&dd->pdev->dev), in show_device_status()
2550 dev_name(&dd->pdev->dev), in show_device_status()
2563 struct driver_data *dd = (struct driver_data *)f->private_data; in mtip_hw_read_device_status() local
2573 dev_err(&dd->pdev->dev, in mtip_hw_read_device_status()
2592 struct driver_data *dd = (struct driver_data *)f->private_data; in mtip_hw_read_registers() local
2603 dev_err(&dd->pdev->dev, in mtip_hw_read_registers()
2610 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2612 readl(dd->port->s_active[n])); in mtip_hw_read_registers()
2617 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2619 readl(dd->port->cmd_issue[n])); in mtip_hw_read_registers()
2624 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2626 readl(dd->port->completed[n])); in mtip_hw_read_registers()
2630 readl(dd->port->mmio + PORT_IRQ_STAT)); in mtip_hw_read_registers()
2632 readl(dd->mmio + HOST_IRQ_STAT)); in mtip_hw_read_registers()
2637 for (n = dd->slot_groups-1; n >= 0; n--) { in mtip_hw_read_registers()
2640 dd->port->allocated[n/2] >> (32*(n&1)); in mtip_hw_read_registers()
2642 group_allocated = dd->port->allocated[n]; in mtip_hw_read_registers()
2649 for (n = dd->slot_groups-1; n >= 0; n--) { in mtip_hw_read_registers()
2652 dd->port->cmds_to_issue[n/2] >> (32*(n&1)); in mtip_hw_read_registers()
2654 group_allocated = dd->port->cmds_to_issue[n]; in mtip_hw_read_registers()
2671 struct driver_data *dd = (struct driver_data *)f->private_data; in mtip_hw_read_flags() local
2681 dev_err(&dd->pdev->dev, in mtip_hw_read_flags()
2687 dd->port->flags); in mtip_hw_read_flags()
2689 dd->dd_flag); in mtip_hw_read_flags()
2731 static int mtip_hw_sysfs_init(struct driver_data *dd, struct kobject *kobj) in mtip_hw_sysfs_init() argument
2733 if (!kobj || !dd) in mtip_hw_sysfs_init()
2737 dev_warn(&dd->pdev->dev, in mtip_hw_sysfs_init()
2752 static int mtip_hw_sysfs_exit(struct driver_data *dd, struct kobject *kobj) in mtip_hw_sysfs_exit() argument
2754 if (!kobj || !dd) in mtip_hw_sysfs_exit()
2762 static int mtip_hw_debugfs_init(struct driver_data *dd) in mtip_hw_debugfs_init() argument
2767 dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent); in mtip_hw_debugfs_init()
2768 if (IS_ERR_OR_NULL(dd->dfs_node)) { in mtip_hw_debugfs_init()
2769 dev_warn(&dd->pdev->dev, in mtip_hw_debugfs_init()
2771 dd->disk->disk_name); in mtip_hw_debugfs_init()
2772 dd->dfs_node = NULL; in mtip_hw_debugfs_init()
2776 debugfs_create_file("flags", S_IRUGO, dd->dfs_node, dd, in mtip_hw_debugfs_init()
2778 debugfs_create_file("registers", S_IRUGO, dd->dfs_node, dd, in mtip_hw_debugfs_init()
2784 static void mtip_hw_debugfs_exit(struct driver_data *dd) in mtip_hw_debugfs_exit() argument
2786 if (dd->dfs_node) in mtip_hw_debugfs_exit()
2787 debugfs_remove_recursive(dd->dfs_node); in mtip_hw_debugfs_exit()
2798 static inline void hba_setup(struct driver_data *dd) in hba_setup() argument
2801 hwdata = readl(dd->mmio + HOST_HSORG); in hba_setup()
2807 dd->mmio + HOST_HSORG); in hba_setup()
2810 static int mtip_device_unaligned_constrained(struct driver_data *dd) in mtip_device_unaligned_constrained() argument
2812 return (dd->pdev->device == P420M_DEVICE_ID ? 1 : 0); in mtip_device_unaligned_constrained()
2825 static void mtip_detect_product(struct driver_data *dd) in mtip_detect_product() argument
2837 hwdata = readl(dd->mmio + HOST_HSORG); in mtip_detect_product()
2839 dd->product_type = MTIP_PRODUCT_UNKNOWN; in mtip_detect_product()
2840 dd->slot_groups = 1; in mtip_detect_product()
2843 dd->product_type = MTIP_PRODUCT_ASICFPGA; in mtip_detect_product()
2846 dev_info(&dd->pdev->dev, in mtip_detect_product()
2854 dev_warn(&dd->pdev->dev, in mtip_detect_product()
2859 dd->slot_groups = slotgroups; in mtip_detect_product()
2863 dev_warn(&dd->pdev->dev, "Unrecognized product id\n"); in mtip_detect_product()
2875 static int mtip_ftl_rebuild_poll(struct driver_data *dd) in mtip_ftl_rebuild_poll() argument
2879 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2887 &dd->dd_flag))) in mtip_ftl_rebuild_poll()
2889 if (mtip_check_surprise_removal(dd->pdev)) in mtip_ftl_rebuild_poll()
2892 if (mtip_get_identify(dd->port, NULL) < 0) in mtip_ftl_rebuild_poll()
2895 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_ftl_rebuild_poll()
2900 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2906 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2909 mtip_block_initialize(dd); in mtip_ftl_rebuild_poll()
2916 dev_err(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2933 struct driver_data *dd = (struct driver_data *)data; in mtip_service_thread() local
2935 unsigned int num_cmd_slots = dd->slot_groups * 32; in mtip_service_thread()
2936 struct mtip_port *port = dd->port; in mtip_service_thread()
2962 &dd->dd_flag))) in mtip_service_thread()
2968 mtip_handle_tfe(dd); in mtip_service_thread()
3007 if (mtip_ftl_rebuild_poll(dd) < 0) in mtip_service_thread()
3009 &dd->dd_flag); in mtip_service_thread()
3016 if (test_bit(MTIP_DDF_REMOVE_DONE_BIT, &dd->dd_flag)) in mtip_service_thread()
3034 static void mtip_dma_free(struct driver_data *dd) in mtip_dma_free() argument
3036 struct mtip_port *port = dd->port; in mtip_dma_free()
3039 dmam_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, in mtip_dma_free()
3043 dmam_free_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, in mtip_dma_free()
3056 static int mtip_dma_alloc(struct driver_data *dd) in mtip_dma_alloc() argument
3058 struct mtip_port *port = dd->port; in mtip_dma_alloc()
3062 dmam_alloc_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, in mtip_dma_alloc()
3070 dmam_alloc_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, in mtip_dma_alloc()
3073 dmam_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, in mtip_dma_alloc()
3094 static int mtip_hw_get_identify(struct driver_data *dd) in mtip_hw_get_identify() argument
3100 if (mtip_get_identify(dd->port, NULL) < 0) in mtip_hw_get_identify()
3103 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_hw_get_identify()
3105 set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags); in mtip_hw_get_identify()
3108 mtip_dump_identify(dd->port); in mtip_hw_get_identify()
3111 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_hw_get_identify()
3112 dd->port->log_buf, in mtip_hw_get_identify()
3113 dd->port->log_buf_dma, 1); in mtip_hw_get_identify()
3115 dev_warn(&dd->pdev->dev, in mtip_hw_get_identify()
3119 buf = (unsigned char *)dd->port->log_buf; in mtip_hw_get_identify()
3121 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
3123 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); in mtip_hw_get_identify()
3126 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
3128 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); in mtip_hw_get_identify()
3131 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
3133 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); in mtip_hw_get_identify()
3139 if (mtip_get_smart_attr(dd->port, 242, &attr242)) in mtip_hw_get_identify()
3140 dev_warn(&dd->pdev->dev, in mtip_hw_get_identify()
3143 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
3158 static int mtip_hw_init(struct driver_data *dd) in mtip_hw_init() argument
3165 dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR]; in mtip_hw_init()
3167 mtip_detect_product(dd); in mtip_hw_init()
3168 if (dd->product_type == MTIP_PRODUCT_UNKNOWN) { in mtip_hw_init()
3172 num_command_slots = dd->slot_groups * 32; in mtip_hw_init()
3174 hba_setup(dd); in mtip_hw_init()
3176 dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL, in mtip_hw_init()
3177 dd->numa_node); in mtip_hw_init()
3178 if (!dd->port) { in mtip_hw_init()
3179 dev_err(&dd->pdev->dev, in mtip_hw_init()
3186 dd->work[i].port = dd->port; in mtip_hw_init()
3189 if (mtip_device_unaligned_constrained(dd)) in mtip_hw_init()
3190 dd->unal_qdepth = MTIP_MAX_UNALIGNED_SLOTS; in mtip_hw_init()
3192 dd->unal_qdepth = 0; in mtip_hw_init()
3194 sema_init(&dd->port->cmd_slot_unal, dd->unal_qdepth); in mtip_hw_init()
3198 spin_lock_init(&dd->port->cmd_issue_lock[i]); in mtip_hw_init()
3201 dd->port->mmio = dd->mmio + PORT_OFFSET; in mtip_hw_init()
3202 dd->port->dd = dd; in mtip_hw_init()
3205 rv = mtip_dma_alloc(dd); in mtip_hw_init()
3210 for (i = 0; i < dd->slot_groups; i++) { in mtip_hw_init()
3211 dd->port->s_active[i] = in mtip_hw_init()
3212 dd->port->mmio + i*0x80 + PORT_SCR_ACT; in mtip_hw_init()
3213 dd->port->cmd_issue[i] = in mtip_hw_init()
3214 dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE; in mtip_hw_init()
3215 dd->port->completed[i] = in mtip_hw_init()
3216 dd->port->mmio + i*0x80 + PORT_SDBV; in mtip_hw_init()
3221 while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) && in mtip_hw_init()
3225 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { in mtip_hw_init()
3227 dev_warn(&dd->pdev->dev, in mtip_hw_init()
3233 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) { in mtip_hw_init()
3235 dev_warn(&dd->pdev->dev, in mtip_hw_init()
3243 if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) { in mtip_hw_init()
3244 if (mtip_hba_reset(dd) < 0) { in mtip_hw_init()
3245 dev_err(&dd->pdev->dev, in mtip_hw_init()
3252 writel(readl(dd->mmio + HOST_IRQ_STAT), in mtip_hw_init()
3253 dd->mmio + HOST_IRQ_STAT); in mtip_hw_init()
3256 mtip_init_port(dd->port); in mtip_hw_init()
3257 mtip_start_port(dd->port); in mtip_hw_init()
3260 rv = devm_request_irq(&dd->pdev->dev, in mtip_hw_init()
3261 dd->pdev->irq, in mtip_hw_init()
3264 dev_driver_string(&dd->pdev->dev), in mtip_hw_init()
3265 dd); in mtip_hw_init()
3268 dev_err(&dd->pdev->dev, in mtip_hw_init()
3269 "Unable to allocate IRQ %d\n", dd->pdev->irq); in mtip_hw_init()
3272 irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding)); in mtip_hw_init()
3275 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_hw_init()
3276 dd->mmio + HOST_CTL); in mtip_hw_init()
3278 init_waitqueue_head(&dd->port->svc_wait); in mtip_hw_init()
3280 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { in mtip_hw_init()
3289 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_init()
3290 dd->mmio + HOST_CTL); in mtip_hw_init()
3293 irq_set_affinity_hint(dd->pdev->irq, NULL); in mtip_hw_init()
3294 devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd); in mtip_hw_init()
3297 mtip_deinit_port(dd->port); in mtip_hw_init()
3298 mtip_dma_free(dd); in mtip_hw_init()
3302 kfree(dd->port); in mtip_hw_init()
3307 static int mtip_standby_drive(struct driver_data *dd) in mtip_standby_drive() argument
3311 if (dd->sr || !dd->port) in mtip_standby_drive()
3317 if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) && in mtip_standby_drive()
3318 !test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag) && in mtip_standby_drive()
3319 !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) { in mtip_standby_drive()
3320 rv = mtip_standby_immediate(dd->port); in mtip_standby_drive()
3322 dev_warn(&dd->pdev->dev, in mtip_standby_drive()
3336 static int mtip_hw_exit(struct driver_data *dd) in mtip_hw_exit() argument
3342 if (!dd->sr) { in mtip_hw_exit()
3344 mtip_deinit_port(dd->port); in mtip_hw_exit()
3347 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_exit()
3348 dd->mmio + HOST_CTL); in mtip_hw_exit()
3352 irq_set_affinity_hint(dd->pdev->irq, NULL); in mtip_hw_exit()
3353 devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd); in mtip_hw_exit()
3357 mtip_dma_free(dd); in mtip_hw_exit()
3360 kfree(dd->port); in mtip_hw_exit()
3361 dd->port = NULL; in mtip_hw_exit()
3377 static int mtip_hw_shutdown(struct driver_data *dd) in mtip_hw_shutdown() argument
3383 mtip_standby_drive(dd); in mtip_hw_shutdown()
3400 static int mtip_hw_suspend(struct driver_data *dd) in mtip_hw_suspend() argument
3406 if (mtip_standby_drive(dd) != 0) { in mtip_hw_suspend()
3407 dev_err(&dd->pdev->dev, in mtip_hw_suspend()
3413 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_suspend()
3414 dd->mmio + HOST_CTL); in mtip_hw_suspend()
3415 mtip_deinit_port(dd->port); in mtip_hw_suspend()
3432 static int mtip_hw_resume(struct driver_data *dd) in mtip_hw_resume() argument
3435 hba_setup(dd); in mtip_hw_resume()
3438 if (mtip_hba_reset(dd) != 0) { in mtip_hw_resume()
3439 dev_err(&dd->pdev->dev, in mtip_hw_resume()
3448 mtip_init_port(dd->port); in mtip_hw_resume()
3449 mtip_start_port(dd->port); in mtip_hw_resume()
3452 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_hw_resume()
3453 dd->mmio + HOST_CTL); in mtip_hw_resume()
3507 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_ioctl() local
3512 if (!dd) in mtip_block_ioctl()
3515 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) in mtip_block_ioctl()
3522 return mtip_hw_ioctl(dd, cmd, arg); in mtip_block_ioctl()
3545 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_compat_ioctl() local
3550 if (!dd) in mtip_block_compat_ioctl()
3553 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) in mtip_block_compat_ioctl()
3582 ret = exec_drive_taskfile(dd, (void __user *) arg, in mtip_block_compat_ioctl()
3599 return mtip_hw_ioctl(dd, cmd, arg); in mtip_block_compat_ioctl()
3625 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_getgeo() local
3628 if (!dd) in mtip_block_getgeo()
3631 if (!(mtip_hw_get_capacity(dd, &capacity))) { in mtip_block_getgeo()
3632 dev_warn(&dd->pdev->dev, in mtip_block_getgeo()
3646 struct driver_data *dd; in mtip_block_open() local
3649 dd = (struct driver_data *) dev->bd_disk->private_data; in mtip_block_open()
3651 if (dd) { in mtip_block_open()
3653 &dd->dd_flag)) { in mtip_block_open()
3696 struct driver_data *dd = hctx->queue->queuedata; in mtip_submit_request() local
3700 if (unlikely(dd->dd_flag & MTIP_DDF_STOP_IO)) { in mtip_submit_request()
3702 &dd->dd_flag))) { in mtip_submit_request()
3705 if (unlikely(test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))) { in mtip_submit_request()
3709 &dd->dd_flag) && in mtip_submit_request()
3713 if (unlikely(test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag) || in mtip_submit_request()
3714 test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag))) in mtip_submit_request()
3721 err = mtip_send_trim(dd, blk_rq_pos(rq), blk_rq_sectors(rq)); in mtip_submit_request()
3730 mtip_hw_submit_io(dd, rq, cmd, nents, hctx); in mtip_submit_request()
3737 struct driver_data *dd = hctx->queue->queuedata; in mtip_check_unal_depth() local
3740 if (rq_data_dir(rq) == READ || !dd->unal_qdepth) in mtip_check_unal_depth()
3752 if (cmd->unaligned && down_trylock(&dd->port->cmd_slot_unal)) in mtip_check_unal_depth()
3780 struct driver_data *dd = data; in mtip_free_cmd() local
3786 dmam_free_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, in mtip_free_cmd()
3793 struct driver_data *dd = data; in mtip_init_cmd() local
3795 u32 host_cap_64 = readl(dd->mmio + HOST_CAP) & HOST_CAP_64; in mtip_init_cmd()
3797 cmd->command = dmam_alloc_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, in mtip_init_cmd()
3805 cmd->command_header = dd->port->command_list + in mtip_init_cmd()
3807 cmd->command_header_dma = dd->port->command_list_dma + in mtip_init_cmd()
3837 static int mtip_block_initialize(struct driver_data *dd) in mtip_block_initialize() argument
3845 if (dd->disk) in mtip_block_initialize()
3848 if (mtip_hw_init(dd)) { in mtip_block_initialize()
3853 dd->disk = alloc_disk_node(MTIP_MAX_MINORS, dd->numa_node); in mtip_block_initialize()
3854 if (dd->disk == NULL) { in mtip_block_initialize()
3855 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3876 dd->disk->disk_name, in mtip_block_initialize()
3881 dd->disk->driverfs_dev = &dd->pdev->dev; in mtip_block_initialize()
3882 dd->disk->major = dd->major; in mtip_block_initialize()
3883 dd->disk->first_minor = dd->instance * MTIP_MAX_MINORS; in mtip_block_initialize()
3884 dd->disk->fops = &mtip_block_ops; in mtip_block_initialize()
3885 dd->disk->private_data = dd; in mtip_block_initialize()
3886 dd->index = index; in mtip_block_initialize()
3888 mtip_hw_debugfs_init(dd); in mtip_block_initialize()
3891 memset(&dd->tags, 0, sizeof(dd->tags)); in mtip_block_initialize()
3892 dd->tags.ops = &mtip_mq_ops; in mtip_block_initialize()
3893 dd->tags.nr_hw_queues = 1; in mtip_block_initialize()
3894 dd->tags.queue_depth = MTIP_MAX_COMMAND_SLOTS; in mtip_block_initialize()
3895 dd->tags.reserved_tags = 1; in mtip_block_initialize()
3896 dd->tags.cmd_size = sizeof(struct mtip_cmd); in mtip_block_initialize()
3897 dd->tags.numa_node = dd->numa_node; in mtip_block_initialize()
3898 dd->tags.flags = BLK_MQ_F_SHOULD_MERGE; in mtip_block_initialize()
3899 dd->tags.driver_data = dd; in mtip_block_initialize()
3901 rv = blk_mq_alloc_tag_set(&dd->tags); in mtip_block_initialize()
3903 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3909 dd->queue = blk_mq_init_queue(&dd->tags); in mtip_block_initialize()
3910 if (IS_ERR(dd->queue)) { in mtip_block_initialize()
3911 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3917 dd->disk->queue = dd->queue; in mtip_block_initialize()
3918 dd->queue->queuedata = dd; in mtip_block_initialize()
3921 wait_for_rebuild = mtip_hw_get_identify(dd); in mtip_block_initialize()
3923 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3937 set_bit(QUEUE_FLAG_NONROT, &dd->queue->queue_flags); in mtip_block_initialize()
3938 clear_bit(QUEUE_FLAG_ADD_RANDOM, &dd->queue->queue_flags); in mtip_block_initialize()
3939 blk_queue_max_segments(dd->queue, MTIP_MAX_SG); in mtip_block_initialize()
3940 blk_queue_physical_block_size(dd->queue, 4096); in mtip_block_initialize()
3941 blk_queue_max_hw_sectors(dd->queue, 0xffff); in mtip_block_initialize()
3942 blk_queue_max_segment_size(dd->queue, 0x400000); in mtip_block_initialize()
3943 blk_queue_io_min(dd->queue, 4096); in mtip_block_initialize()
3944 blk_queue_bounce_limit(dd->queue, dd->pdev->dma_mask); in mtip_block_initialize()
3950 blk_queue_flush(dd->queue, 0); in mtip_block_initialize()
3953 if (dd->trim_supp == true) { in mtip_block_initialize()
3954 set_bit(QUEUE_FLAG_DISCARD, &dd->queue->queue_flags); in mtip_block_initialize()
3955 dd->queue->limits.discard_granularity = 4096; in mtip_block_initialize()
3956 blk_queue_max_discard_sectors(dd->queue, in mtip_block_initialize()
3958 dd->queue->limits.discard_zeroes_data = 0; in mtip_block_initialize()
3962 if (!(mtip_hw_get_capacity(dd, &capacity))) { in mtip_block_initialize()
3963 dev_warn(&dd->pdev->dev, in mtip_block_initialize()
3968 set_capacity(dd->disk, capacity); in mtip_block_initialize()
3971 add_disk(dd->disk); in mtip_block_initialize()
3973 dd->bdev = bdget_disk(dd->disk, 0); in mtip_block_initialize()
3978 kobj = kobject_get(&disk_to_dev(dd->disk)->kobj); in mtip_block_initialize()
3980 mtip_hw_sysfs_init(dd, kobj); in mtip_block_initialize()
3984 if (dd->mtip_svc_handler) { in mtip_block_initialize()
3985 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); in mtip_block_initialize()
3991 dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread, in mtip_block_initialize()
3992 dd, dd->numa_node, "%s", in mtip_block_initialize()
3995 if (IS_ERR(dd->mtip_svc_handler)) { in mtip_block_initialize()
3996 dev_err(&dd->pdev->dev, "service thread failed to start\n"); in mtip_block_initialize()
3997 dd->mtip_svc_handler = NULL; in mtip_block_initialize()
4001 wake_up_process(dd->mtip_svc_handler); in mtip_block_initialize()
4008 bdput(dd->bdev); in mtip_block_initialize()
4009 dd->bdev = NULL; in mtip_block_initialize()
4012 del_gendisk(dd->disk); in mtip_block_initialize()
4016 blk_cleanup_queue(dd->queue); in mtip_block_initialize()
4017 blk_mq_free_tag_set(&dd->tags); in mtip_block_initialize()
4019 mtip_hw_debugfs_exit(dd); in mtip_block_initialize()
4026 put_disk(dd->disk); in mtip_block_initialize()
4029 mtip_hw_exit(dd); /* De-initialize the protocol layer. */ in mtip_block_initialize()
4045 static int mtip_block_remove(struct driver_data *dd) in mtip_block_remove() argument
4049 mtip_hw_debugfs_exit(dd); in mtip_block_remove()
4051 if (dd->mtip_svc_handler) { in mtip_block_remove()
4052 set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags); in mtip_block_remove()
4053 wake_up_interruptible(&dd->port->svc_wait); in mtip_block_remove()
4054 kthread_stop(dd->mtip_svc_handler); in mtip_block_remove()
4058 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) { in mtip_block_remove()
4059 kobj = kobject_get(&disk_to_dev(dd->disk)->kobj); in mtip_block_remove()
4061 mtip_hw_sysfs_exit(dd, kobj); in mtip_block_remove()
4066 if (!dd->sr) in mtip_block_remove()
4067 mtip_standby_drive(dd); in mtip_block_remove()
4069 dev_info(&dd->pdev->dev, "device %s surprise removal\n", in mtip_block_remove()
4070 dd->disk->disk_name); in mtip_block_remove()
4076 if (dd->bdev) { in mtip_block_remove()
4077 bdput(dd->bdev); in mtip_block_remove()
4078 dd->bdev = NULL; in mtip_block_remove()
4080 if (dd->disk) { in mtip_block_remove()
4081 del_gendisk(dd->disk); in mtip_block_remove()
4082 if (dd->disk->queue) { in mtip_block_remove()
4083 blk_cleanup_queue(dd->queue); in mtip_block_remove()
4084 blk_mq_free_tag_set(&dd->tags); in mtip_block_remove()
4085 dd->queue = NULL; in mtip_block_remove()
4087 put_disk(dd->disk); in mtip_block_remove()
4089 dd->disk = NULL; in mtip_block_remove()
4092 ida_remove(&rssd_index_ida, dd->index); in mtip_block_remove()
4096 mtip_hw_exit(dd); in mtip_block_remove()
4113 static int mtip_block_shutdown(struct driver_data *dd) in mtip_block_shutdown() argument
4115 mtip_hw_shutdown(dd); in mtip_block_shutdown()
4118 if (dd->disk) { in mtip_block_shutdown()
4119 dev_info(&dd->pdev->dev, in mtip_block_shutdown()
4120 "Shutting down %s ...\n", dd->disk->disk_name); in mtip_block_shutdown()
4122 del_gendisk(dd->disk); in mtip_block_shutdown()
4123 if (dd->disk->queue) { in mtip_block_shutdown()
4124 blk_cleanup_queue(dd->queue); in mtip_block_shutdown()
4125 blk_mq_free_tag_set(&dd->tags); in mtip_block_shutdown()
4127 put_disk(dd->disk); in mtip_block_shutdown()
4128 dd->disk = NULL; in mtip_block_shutdown()
4129 dd->queue = NULL; in mtip_block_shutdown()
4133 ida_remove(&rssd_index_ida, dd->index); in mtip_block_shutdown()
4138 static int mtip_block_suspend(struct driver_data *dd) in mtip_block_suspend() argument
4140 dev_info(&dd->pdev->dev, in mtip_block_suspend()
4141 "Suspending %s ...\n", dd->disk->disk_name); in mtip_block_suspend()
4142 mtip_hw_suspend(dd); in mtip_block_suspend()
4146 static int mtip_block_resume(struct driver_data *dd) in mtip_block_resume() argument
4148 dev_info(&dd->pdev->dev, "Resuming %s ...\n", in mtip_block_resume()
4149 dd->disk->disk_name); in mtip_block_resume()
4150 mtip_hw_resume(dd); in mtip_block_resume()
4204 static void mtip_disable_link_opts(struct driver_data *dd, struct pci_dev *pdev) in mtip_disable_link_opts() argument
4216 dev_info(&dd->pdev->dev, in mtip_disable_link_opts()
4228 static void mtip_fix_ero_nosnoop(struct driver_data *dd, struct pci_dev *pdev) in mtip_fix_ero_nosnoop() argument
4237 mtip_disable_link_opts(dd, pdev->bus->self); in mtip_fix_ero_nosnoop()
4248 mtip_disable_link_opts(dd, in mtip_fix_ero_nosnoop()
4268 struct driver_data *dd = NULL; in mtip_pci_probe() local
4288 dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node); in mtip_pci_probe()
4289 if (dd == NULL) { in mtip_pci_probe()
4296 pci_set_drvdata(pdev, dd); in mtip_pci_probe()
4326 dd->major = mtip_major; in mtip_pci_probe()
4327 dd->instance = instance; in mtip_pci_probe()
4328 dd->pdev = pdev; in mtip_pci_probe()
4329 dd->numa_node = my_node; in mtip_pci_probe()
4331 INIT_LIST_HEAD(&dd->online_list); in mtip_pci_probe()
4332 INIT_LIST_HEAD(&dd->remove_list); in mtip_pci_probe()
4334 memset(dd->workq_name, 0, 32); in mtip_pci_probe()
4335 snprintf(dd->workq_name, 31, "mtipq%d", dd->instance); in mtip_pci_probe()
4337 dd->isr_workq = create_workqueue(dd->workq_name); in mtip_pci_probe()
4338 if (!dd->isr_workq) { in mtip_pci_probe()
4339 dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance); in mtip_pci_probe()
4346 node_mask = cpumask_of_node(dd->numa_node); in mtip_pci_probe()
4355 dd->numa_node, in mtip_pci_probe()
4357 nr_cpus_node(dd->numa_node), in mtip_pci_probe()
4362 dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
4364 cpu_to_node(dd->isr_binding), dd->isr_binding); in mtip_pci_probe()
4367 dd->work[0].cpu_binding = dd->isr_binding; in mtip_pci_probe()
4368 dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
4369 dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
4370 dd->work[3].cpu_binding = dd->work[0].cpu_binding; in mtip_pci_probe()
4371 dd->work[4].cpu_binding = dd->work[1].cpu_binding; in mtip_pci_probe()
4372 dd->work[5].cpu_binding = dd->work[2].cpu_binding; in mtip_pci_probe()
4373 dd->work[6].cpu_binding = dd->work[2].cpu_binding; in mtip_pci_probe()
4374 dd->work[7].cpu_binding = dd->work[1].cpu_binding; in mtip_pci_probe()
4380 if (dd->work[i].cpu_binding == cpu) { in mtip_pci_probe()
4389 INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0); in mtip_pci_probe()
4390 INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1); in mtip_pci_probe()
4391 INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2); in mtip_pci_probe()
4392 INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3); in mtip_pci_probe()
4393 INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4); in mtip_pci_probe()
4394 INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5); in mtip_pci_probe()
4395 INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6); in mtip_pci_probe()
4396 INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7); in mtip_pci_probe()
4406 mtip_fix_ero_nosnoop(dd, pdev); in mtip_pci_probe()
4409 rv = mtip_block_initialize(dd); in mtip_pci_probe()
4422 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); in mtip_pci_probe()
4428 list_add(&dd->online_list, &online_list); in mtip_pci_probe()
4437 if (dd->isr_workq) { in mtip_pci_probe()
4438 flush_workqueue(dd->isr_workq); in mtip_pci_probe()
4439 destroy_workqueue(dd->isr_workq); in mtip_pci_probe()
4440 drop_cpu(dd->work[0].cpu_binding); in mtip_pci_probe()
4441 drop_cpu(dd->work[1].cpu_binding); in mtip_pci_probe()
4442 drop_cpu(dd->work[2].cpu_binding); in mtip_pci_probe()
4448 kfree(dd); in mtip_pci_probe()
4464 struct driver_data *dd = pci_get_drvdata(pdev); in mtip_pci_remove() local
4467 set_bit(MTIP_DDF_REMOVAL_BIT, &dd->dd_flag); in mtip_pci_remove()
4470 list_del_init(&dd->online_list); in mtip_pci_remove()
4471 list_add(&dd->remove_list, &removing_list); in mtip_pci_remove()
4475 synchronize_irq(dd->pdev->irq); in mtip_pci_remove()
4481 } while (atomic_read(&dd->irq_workers_active) != 0 && in mtip_pci_remove()
4484 fsync_bdev(dd->bdev); in mtip_pci_remove()
4486 if (atomic_read(&dd->irq_workers_active) != 0) { in mtip_pci_remove()
4487 dev_warn(&dd->pdev->dev, in mtip_pci_remove()
4491 if (dd->sr) in mtip_pci_remove()
4492 blk_mq_stop_hw_queues(dd->queue); in mtip_pci_remove()
4494 set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); in mtip_pci_remove()
4497 mtip_block_remove(dd); in mtip_pci_remove()
4499 if (dd->isr_workq) { in mtip_pci_remove()
4500 flush_workqueue(dd->isr_workq); in mtip_pci_remove()
4501 destroy_workqueue(dd->isr_workq); in mtip_pci_remove()
4502 drop_cpu(dd->work[0].cpu_binding); in mtip_pci_remove()
4503 drop_cpu(dd->work[1].cpu_binding); in mtip_pci_remove()
4504 drop_cpu(dd->work[2].cpu_binding); in mtip_pci_remove()
4510 list_del_init(&dd->remove_list); in mtip_pci_remove()
4513 kfree(dd); in mtip_pci_remove()
4514 set_bit(MTIP_DDF_REMOVE_DONE_BIT, &dd->dd_flag); in mtip_pci_remove()
4530 struct driver_data *dd = pci_get_drvdata(pdev); in mtip_pci_suspend() local
4532 if (!dd) { in mtip_pci_suspend()
4538 set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); in mtip_pci_suspend()
4541 rv = mtip_block_suspend(dd); in mtip_pci_suspend()
4571 struct driver_data *dd; in mtip_pci_resume() local
4573 dd = pci_get_drvdata(pdev); in mtip_pci_resume()
4574 if (!dd) { in mtip_pci_resume()
4599 rv = mtip_block_resume(dd); in mtip_pci_resume()
4604 clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); in mtip_pci_resume()
4617 struct driver_data *dd = pci_get_drvdata(pdev); in mtip_pci_shutdown() local
4618 if (dd) in mtip_pci_shutdown()
4619 mtip_block_shutdown(dd); in mtip_pci_shutdown()