Lines Matching refs:port
228 static void mtip_async_complete(struct mtip_port *port, in mtip_async_complete() argument
231 struct driver_data *dd = port->dd; in mtip_async_complete()
234 if (unlikely(!dd) || unlikely(!port)) in mtip_async_complete()
238 dev_warn(&port->dd->pdev->dev, in mtip_async_complete()
297 static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag) in mtip_issue_ncq_command() argument
302 spin_lock(&port->cmd_issue_lock[group]); in mtip_issue_ncq_command()
304 port->s_active[MTIP_TAG_INDEX(tag)]); in mtip_issue_ncq_command()
306 port->cmd_issue[MTIP_TAG_INDEX(tag)]); in mtip_issue_ncq_command()
307 spin_unlock(&port->cmd_issue_lock[group]); in mtip_issue_ncq_command()
319 static int mtip_enable_fis(struct mtip_port *port, int enable) in mtip_enable_fis() argument
324 tmp = readl(port->mmio + PORT_CMD); in mtip_enable_fis()
326 writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD); in mtip_enable_fis()
328 writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD); in mtip_enable_fis()
331 readl(port->mmio + PORT_CMD); in mtip_enable_fis()
345 static int mtip_enable_engine(struct mtip_port *port, int enable) in mtip_enable_engine() argument
350 tmp = readl(port->mmio + PORT_CMD); in mtip_enable_engine()
352 writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD); in mtip_enable_engine()
354 writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD); in mtip_enable_engine()
356 readl(port->mmio + PORT_CMD); in mtip_enable_engine()
366 static inline void mtip_start_port(struct mtip_port *port) in mtip_start_port() argument
369 mtip_enable_fis(port, 1); in mtip_start_port()
372 mtip_enable_engine(port, 1); in mtip_start_port()
384 static inline void mtip_deinit_port(struct mtip_port *port) in mtip_deinit_port() argument
387 writel(0, port->mmio + PORT_IRQ_MASK); in mtip_deinit_port()
390 mtip_enable_engine(port, 0); in mtip_deinit_port()
393 mtip_enable_fis(port, 0); in mtip_deinit_port()
409 static void mtip_init_port(struct mtip_port *port) in mtip_init_port() argument
412 mtip_deinit_port(port); in mtip_init_port()
415 if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) { in mtip_init_port()
416 writel((port->command_list_dma >> 16) >> 16, in mtip_init_port()
417 port->mmio + PORT_LST_ADDR_HI); in mtip_init_port()
418 writel((port->rxfis_dma >> 16) >> 16, in mtip_init_port()
419 port->mmio + PORT_FIS_ADDR_HI); in mtip_init_port()
422 writel(port->command_list_dma & 0xFFFFFFFF, in mtip_init_port()
423 port->mmio + PORT_LST_ADDR); in mtip_init_port()
424 writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR); in mtip_init_port()
427 writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR); in mtip_init_port()
430 for (i = 0; i < port->dd->slot_groups; i++) in mtip_init_port()
431 writel(0xFFFFFFFF, port->completed[i]); in mtip_init_port()
434 writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT); in mtip_init_port()
437 writel(readl(port->dd->mmio + HOST_IRQ_STAT), in mtip_init_port()
438 port->dd->mmio + HOST_IRQ_STAT); in mtip_init_port()
441 writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK); in mtip_init_port()
452 static void mtip_restart_port(struct mtip_port *port) in mtip_restart_port() argument
457 mtip_enable_engine(port, 0); in mtip_restart_port()
461 while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) in mtip_restart_port()
465 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
472 if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) { in mtip_restart_port()
473 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
476 if (mtip_hba_reset(port->dd)) in mtip_restart_port()
477 dev_err(&port->dd->pdev->dev, in mtip_restart_port()
484 dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n"); in mtip_restart_port()
487 writel(readl(port->mmio + PORT_SCR_CTL) | in mtip_restart_port()
488 1, port->mmio + PORT_SCR_CTL); in mtip_restart_port()
489 readl(port->mmio + PORT_SCR_CTL); in mtip_restart_port()
496 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
500 writel(readl(port->mmio + PORT_SCR_CTL) & ~1, in mtip_restart_port()
501 port->mmio + PORT_SCR_CTL); in mtip_restart_port()
502 readl(port->mmio + PORT_SCR_CTL); in mtip_restart_port()
506 while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) in mtip_restart_port()
510 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
513 if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) in mtip_restart_port()
514 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
517 mtip_init_port(port); in mtip_restart_port()
518 mtip_start_port(port); in mtip_restart_port()
533 mtip_init_port(dd->port); in mtip_device_reset()
534 mtip_start_port(dd->port); in mtip_device_reset()
576 static void mtip_completion(struct mtip_port *port, in mtip_completion() argument
581 dev_warn(&port->dd->pdev->dev, in mtip_completion()
589 static void mtip_null_completion(struct mtip_port *port, in mtip_null_completion() argument
594 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer,
596 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id,
609 struct mtip_port *port; in mtip_handle_tfe() local
621 port = dd->port; in mtip_handle_tfe()
623 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags)) { in mtip_handle_tfe()
628 cmd->comp_func(port, MTIP_TAG_INTERNAL, in mtip_handle_tfe()
639 completed = readl(port->completed[group]); in mtip_handle_tfe()
644 writel(completed, port->completed[group]); in mtip_handle_tfe()
660 cmd->comp_func(port, tag, cmd, 0); in mtip_handle_tfe()
662 dev_err(&port->dd->pdev->dev, in mtip_handle_tfe()
677 mtip_restart_port(port); in mtip_handle_tfe()
680 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_handle_tfe()
681 dd->port->log_buf, in mtip_handle_tfe()
682 dd->port->log_buf_dma, 1); in mtip_handle_tfe()
688 buf = (unsigned char *)dd->port->log_buf; in mtip_handle_tfe()
740 cmd->comp_func(port, tag, in mtip_handle_tfe()
756 mtip_issue_ncq_command(port, tag); in mtip_handle_tfe()
762 dev_warn(&port->dd->pdev->dev, in mtip_handle_tfe()
766 cmd->comp_func(port, tag, cmd, PORT_IRQ_TF_ERR); in mtip_handle_tfe()
768 dev_warn(&port->dd->pdev->dev, in mtip_handle_tfe()
779 static inline void mtip_workq_sdbfx(struct mtip_port *port, int group, in mtip_workq_sdbfx() argument
782 struct driver_data *dd = port->dd; in mtip_workq_sdbfx()
791 writel(completed, port->completed[group]); in mtip_workq_sdbfx()
804 command->comp_func(port, tag, command, 0); in mtip_workq_sdbfx()
829 struct mtip_port *port = dd->port; in mtip_process_legacy() local
832 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && in mtip_process_legacy()
833 (cmd != NULL) && !(readl(port->cmd_issue[MTIP_TAG_INTERNAL]) in mtip_process_legacy()
836 cmd->comp_func(port, MTIP_TAG_INTERNAL, cmd, 0); in mtip_process_legacy()
853 writel((1 << 26), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
859 writel((1 << 16), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
870 set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags); in mtip_process_errors()
871 wake_up_interruptible(&dd->port->svc_wait); in mtip_process_errors()
878 struct mtip_port *port = dd->port; in mtip_handle_irq() local
889 port_stat = readl(port->mmio + PORT_IRQ_STAT); in mtip_handle_irq()
894 writel(port_stat, port->mmio + PORT_IRQ_STAT); in mtip_handle_irq()
905 twork->completed = readl(port->completed[i]); in mtip_handle_irq()
922 mtip_workq_sdbfx(port, 0, in mtip_handle_irq()
974 static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag) in mtip_issue_non_ncq_command() argument
977 port->cmd_issue[MTIP_TAG_INDEX(tag)]); in mtip_issue_non_ncq_command()
980 static bool mtip_pause_ncq(struct mtip_port *port, in mtip_pause_ncq() argument
986 reply = port->rxfis + RX_FIS_D2H_REG; in mtip_pause_ncq()
987 task_file_data = readl(port->mmio+PORT_TFDATA); in mtip_pause_ncq()
993 port->ic_pause_timer = jiffies; in mtip_pause_ncq()
997 set_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); in mtip_pause_ncq()
998 port->ic_pause_timer = jiffies; in mtip_pause_ncq()
1004 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
1005 clear_bit(MTIP_DDF_REBUILD_FAILED_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
1007 mtip_restart_port(port); in mtip_pause_ncq()
1008 clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); in mtip_pause_ncq()
1026 static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout, in mtip_quiesce_io() argument
1033 blk_mq_stop_hw_queues(port->dd->queue); in mtip_quiesce_io()
1037 if (test_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags) && in mtip_quiesce_io()
1038 test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags) && in mtip_quiesce_io()
1051 if (mtip_check_surprise_removal(port->dd->pdev)) in mtip_quiesce_io()
1058 active = readl(port->s_active[0]) & 0xFFFFFFFE; in mtip_quiesce_io()
1059 for (n = 1; n < port->dd->slot_groups; n++) in mtip_quiesce_io()
1060 active |= readl(port->s_active[n]); in mtip_quiesce_io()
1066 blk_mq_start_stopped_hw_queues(port->dd->queue, true); in mtip_quiesce_io()
1069 blk_mq_start_stopped_hw_queues(port->dd->queue, true); in mtip_quiesce_io()
1091 static int mtip_exec_internal_command(struct mtip_port *port, in mtip_exec_internal_command() argument
1103 struct driver_data *dd = port->dd; in mtip_exec_internal_command()
1119 set_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
1122 set_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
1124 clear_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
1129 if (mtip_quiesce_io(port, in mtip_exec_internal_command()
1134 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
1135 wake_up_interruptible(&port->svc_wait); in mtip_exec_internal_command()
1176 mtip_issue_non_ncq_command(port, MTIP_TAG_INTERNAL); in mtip_exec_internal_command()
1218 while ((readl(port->cmd_issue[MTIP_TAG_INTERNAL]) in mtip_exec_internal_command()
1231 port_stat = readl(port->mmio + PORT_IRQ_STAT); in mtip_exec_internal_command()
1243 writel(port_stat, port->mmio + PORT_IRQ_STAT); in mtip_exec_internal_command()
1253 if (readl(port->cmd_issue[MTIP_TAG_INTERNAL]) in mtip_exec_internal_command()
1264 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
1265 if (rv >= 0 && mtip_pause_ncq(port, fis)) { in mtip_exec_internal_command()
1269 wake_up_interruptible(&port->svc_wait); in mtip_exec_internal_command()
1305 *timeout = ((*(dd->port->identify + 90) * 2) * 60000); in mtip_set_timeout()
1307 *timeout = ((*(dd->port->identify + 89) * 2) * 60000); in mtip_set_timeout()
1344 static int mtip_get_identify(struct mtip_port *port, void __user *user_buffer) in mtip_get_identify() argument
1349 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_get_identify()
1359 port->identify_valid = 0; in mtip_get_identify()
1362 memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS); in mtip_get_identify()
1365 if (mtip_exec_internal_command(port, in mtip_get_identify()
1368 port->identify_dma, in mtip_get_identify()
1384 ata_swap_string(port->identify + 27, 40); /* model string*/ in mtip_get_identify()
1385 ata_swap_string(port->identify + 23, 8); /* firmware string*/ in mtip_get_identify()
1386 ata_swap_string(port->identify + 10, 20); /* serial# string*/ in mtip_get_identify()
1391 port->identify[i] = le16_to_cpu(port->identify[i]); in mtip_get_identify()
1396 if (port->identify[128] & 0x4) in mtip_get_identify()
1397 set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_get_identify()
1399 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_get_identify()
1403 if (port->identify[69] & (1 << 14) && port->identify[69] & (1 << 5)) in mtip_get_identify()
1404 port->dd->trim_supp = true; in mtip_get_identify()
1407 port->dd->trim_supp = false; in mtip_get_identify()
1410 port->identify_valid = 1; in mtip_get_identify()
1415 port->identify, in mtip_get_identify()
1435 static int mtip_standby_immediate(struct mtip_port *port) in mtip_standby_immediate() argument
1448 mtip_set_timeout(port->dd, &fis, &timeout, 0); in mtip_standby_immediate()
1451 rv = mtip_exec_internal_command(port, in mtip_standby_immediate()
1462 dev_warn(&port->dd->pdev->dev, in mtip_standby_immediate()
1480 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer, in mtip_read_log_page() argument
1497 return mtip_exec_internal_command(port, in mtip_read_log_page()
1517 static int mtip_get_smart_data(struct mtip_port *port, u8 *buffer, in mtip_get_smart_data() argument
1532 return mtip_exec_internal_command(port, in mtip_get_smart_data()
1553 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id, in mtip_get_smart_attr() argument
1562 if (!port->identify_valid) { in mtip_get_smart_attr()
1563 dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n"); in mtip_get_smart_attr()
1566 if (!(port->identify[82] & 0x1)) { in mtip_get_smart_attr()
1567 dev_warn(&port->dd->pdev->dev, "SMART not supported\n"); in mtip_get_smart_attr()
1570 if (!(port->identify[85] & 0x1)) { in mtip_get_smart_attr()
1571 dev_warn(&port->dd->pdev->dev, "SMART not enabled\n"); in mtip_get_smart_attr()
1575 memset(port->smart_buf, 0, ATA_SECT_SIZE); in mtip_get_smart_attr()
1576 rv = mtip_get_smart_data(port, port->smart_buf, port->smart_buf_dma); in mtip_get_smart_attr()
1578 dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n"); in mtip_get_smart_attr()
1582 pattr = (struct smart_attr *)(port->smart_buf + 2); in mtip_get_smart_attr()
1590 dev_warn(&port->dd->pdev->dev, in mtip_get_smart_attr()
1660 if (mtip_exec_internal_command(dd->port, in mtip_send_trim()
1686 struct mtip_port *port = dd->port; in mtip_hw_get_capacity() local
1688 raw0 = port->identify[100]; in mtip_hw_get_capacity()
1689 raw1 = port->identify[101]; in mtip_hw_get_capacity()
1690 raw2 = port->identify[102]; in mtip_hw_get_capacity()
1691 raw3 = port->identify[103]; in mtip_hw_get_capacity()
1694 return (bool) !!port->identify_valid; in mtip_hw_get_capacity()
1705 static void mtip_dump_identify(struct mtip_port *port) in mtip_dump_identify() argument
1711 if (!port->identify_valid) in mtip_dump_identify()
1714 strlcpy(cbuf, (char *)(port->identify+10), 21); in mtip_dump_identify()
1715 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1718 strlcpy(cbuf, (char *)(port->identify+23), 9); in mtip_dump_identify()
1719 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1722 strlcpy(cbuf, (char *)(port->identify+27), 41); in mtip_dump_identify()
1723 dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf); in mtip_dump_identify()
1725 dev_info(&port->dd->pdev->dev, "Security: %04x %s\n", in mtip_dump_identify()
1726 port->identify[128], in mtip_dump_identify()
1727 port->identify[128] & 0x4 ? "(LOCKED)" : ""); in mtip_dump_identify()
1729 if (mtip_hw_get_capacity(port->dd, §ors)) in mtip_dump_identify()
1730 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1735 pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid); in mtip_dump_identify()
1747 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1793 static int exec_drive_task(struct mtip_port *port, u8 *command) in exec_drive_task() argument
1796 struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG); in exec_drive_task()
1811 mtip_set_timeout(port->dd, &fis, &to, 0); in exec_drive_task()
1824 if (mtip_exec_internal_command(port, in exec_drive_task()
1863 static int exec_drive_command(struct mtip_port *port, u8 *command, in exec_drive_command() argument
1877 buf = dmam_alloc_coherent(&port->dd->pdev->dev, in exec_drive_command()
1882 dev_err(&port->dd->pdev->dev, in exec_drive_command()
1903 mtip_set_timeout(port->dd, &fis, &to, 0); in exec_drive_command()
1906 reply = (port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_command()
1908 reply = (port->rxfis + RX_FIS_D2H_REG); in exec_drive_command()
1920 if (mtip_exec_internal_command(port, in exec_drive_command()
1956 dmam_free_coherent(&port->dd->pdev->dev, in exec_drive_command()
2089 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
2092 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
2095 reply = (dd->port->rxfis + RX_FIS_D2H_REG); in exec_drive_taskfile()
2174 if (mtip_exec_internal_command(dd->port, in exec_drive_taskfile()
2186 task_file_data = readl(dd->port->mmio+PORT_TFDATA); in exec_drive_taskfile()
2189 reply = dd->port->rxfis + RX_FIS_PIO_SETUP; in exec_drive_taskfile()
2192 reply = dd->port->rxfis + RX_FIS_D2H_REG; in exec_drive_taskfile()
2283 if (copy_to_user((void __user *)arg, dd->port->identify, in mtip_hw_ioctl()
2299 if (exec_drive_command(dd->port, in mtip_hw_ioctl()
2323 if (exec_drive_task(dd->port, drive_command)) in mtip_hw_ioctl()
2386 struct mtip_port *port = dd->port; in mtip_hw_submit_io() local
2394 prefetch(&port->flags); in mtip_hw_submit_io()
2449 if (unlikely(port->flags & MTIP_PF_PAUSE_IO)) { in mtip_hw_submit_io()
2450 set_bit(rq->tag, port->cmds_to_issue); in mtip_hw_submit_io()
2451 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); in mtip_hw_submit_io()
2456 mtip_issue_ncq_command(port, rq->tag); in mtip_hw_submit_io()
2502 if (dd->port && in show_device_status()
2503 dd->port->identify && in show_device_status()
2504 dd->port->identify_valid) { in show_device_status()
2506 (char *) (dd->port->identify + 10), 21); in show_device_status()
2507 status = *(dd->port->identify + 141); in show_device_status()
2513 if (dd->port && in show_device_status()
2514 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { in show_device_status()
2532 if (dd->port && in show_device_status()
2533 dd->port->identify && in show_device_status()
2534 dd->port->identify_valid) { in show_device_status()
2536 (char *) (dd->port->identify+10), 21); in show_device_status()
2537 status = *(dd->port->identify + 141); in show_device_status()
2543 if (dd->port && in show_device_status()
2544 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { in show_device_status()
2615 readl(dd->port->s_active[n])); in mtip_hw_read_registers()
2622 readl(dd->port->cmd_issue[n])); in mtip_hw_read_registers()
2629 readl(dd->port->completed[n])); in mtip_hw_read_registers()
2633 readl(dd->port->mmio + PORT_IRQ_STAT)); in mtip_hw_read_registers()
2643 dd->port->cmds_to_issue[n/2] >> (32*(n&1)); in mtip_hw_read_registers()
2645 group_allocated = dd->port->cmds_to_issue[n]; in mtip_hw_read_registers()
2678 dd->port->flags); in mtip_hw_read_flags()
2883 if (mtip_get_identify(dd->port, NULL) < 0) in mtip_ftl_rebuild_poll()
2886 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_ftl_rebuild_poll()
2922 up(&dd->port->cmd_slot_unal); in mtip_softirq_done_fn()
2934 clear_bit(req->tag, dd->port->cmds_to_issue); in mtip_abort_cmd()
2944 set_bit(req->tag, dd->port->cmds_to_issue); in mtip_queue_cmd()
2962 struct mtip_port *port = dd->port; in mtip_service_thread() local
2966 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) in mtip_service_thread()
2968 clear_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); in mtip_service_thread()
2974 wait_event_interruptible(port->svc_wait, (port->flags) && in mtip_service_thread()
2975 (port->flags & MTIP_PF_SVC_THD_WORK)); in mtip_service_thread()
2978 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) in mtip_service_thread()
2985 set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); in mtip_service_thread()
2989 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) { in mtip_service_thread()
2991 clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); in mtip_service_thread()
2994 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) in mtip_service_thread()
2997 if (test_bit(MTIP_PF_TO_ACTIVE_BIT, &port->flags)) { in mtip_service_thread()
3014 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &dd->port->flags); in mtip_service_thread()
3020 clear_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags); in mtip_service_thread()
3023 if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { in mtip_service_thread()
3029 slot = find_next_bit(port->cmds_to_issue, in mtip_service_thread()
3046 mtip_issue_ncq_command(port, slot); in mtip_service_thread()
3048 clear_bit(slot, port->cmds_to_issue); in mtip_service_thread()
3051 clear_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); in mtip_service_thread()
3054 if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) { in mtip_service_thread()
3056 clear_bit(MTIP_PF_REBUILD_BIT, &port->flags); in mtip_service_thread()
3074 struct mtip_port *port = dd->port; in mtip_dma_free() local
3076 if (port->block1) in mtip_dma_free()
3078 port->block1, port->block1_dma); in mtip_dma_free()
3080 if (port->command_list) { in mtip_dma_free()
3082 port->command_list, port->command_list_dma); in mtip_dma_free()
3096 struct mtip_port *port = dd->port; in mtip_dma_alloc() local
3099 port->block1 = in mtip_dma_alloc()
3101 &port->block1_dma, GFP_KERNEL); in mtip_dma_alloc()
3102 if (!port->block1) in mtip_dma_alloc()
3104 memset(port->block1, 0, BLOCK_DMA_ALLOC_SZ); in mtip_dma_alloc()
3107 port->command_list = in mtip_dma_alloc()
3109 &port->command_list_dma, GFP_KERNEL); in mtip_dma_alloc()
3110 if (!port->command_list) { in mtip_dma_alloc()
3112 port->block1, port->block1_dma); in mtip_dma_alloc()
3113 port->block1 = NULL; in mtip_dma_alloc()
3114 port->block1_dma = 0; in mtip_dma_alloc()
3117 memset(port->command_list, 0, AHCI_CMD_TBL_SZ); in mtip_dma_alloc()
3120 port->rxfis = port->block1 + AHCI_RX_FIS_OFFSET; in mtip_dma_alloc()
3121 port->rxfis_dma = port->block1_dma + AHCI_RX_FIS_OFFSET; in mtip_dma_alloc()
3122 port->identify = port->block1 + AHCI_IDFY_OFFSET; in mtip_dma_alloc()
3123 port->identify_dma = port->block1_dma + AHCI_IDFY_OFFSET; in mtip_dma_alloc()
3124 port->log_buf = port->block1 + AHCI_SECTBUF_OFFSET; in mtip_dma_alloc()
3125 port->log_buf_dma = port->block1_dma + AHCI_SECTBUF_OFFSET; in mtip_dma_alloc()
3126 port->smart_buf = port->block1 + AHCI_SMARTBUF_OFFSET; in mtip_dma_alloc()
3127 port->smart_buf_dma = port->block1_dma + AHCI_SMARTBUF_OFFSET; in mtip_dma_alloc()
3138 if (mtip_get_identify(dd->port, NULL) < 0) in mtip_hw_get_identify()
3141 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_hw_get_identify()
3143 set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags); in mtip_hw_get_identify()
3146 mtip_dump_identify(dd->port); in mtip_hw_get_identify()
3149 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_hw_get_identify()
3150 dd->port->log_buf, in mtip_hw_get_identify()
3151 dd->port->log_buf_dma, 1); in mtip_hw_get_identify()
3157 buf = (unsigned char *)dd->port->log_buf; in mtip_hw_get_identify()
3177 if (mtip_get_smart_attr(dd->port, 242, &attr242)) in mtip_hw_get_identify()
3214 dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL, in mtip_hw_init()
3216 if (!dd->port) { in mtip_hw_init()
3224 dd->work[i].port = dd->port; in mtip_hw_init()
3232 sema_init(&dd->port->cmd_slot_unal, dd->unal_qdepth); in mtip_hw_init()
3236 spin_lock_init(&dd->port->cmd_issue_lock[i]); in mtip_hw_init()
3239 dd->port->mmio = dd->mmio + PORT_OFFSET; in mtip_hw_init()
3240 dd->port->dd = dd; in mtip_hw_init()
3249 dd->port->s_active[i] = in mtip_hw_init()
3250 dd->port->mmio + i*0x80 + PORT_SCR_ACT; in mtip_hw_init()
3251 dd->port->cmd_issue[i] = in mtip_hw_init()
3252 dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE; in mtip_hw_init()
3253 dd->port->completed[i] = in mtip_hw_init()
3254 dd->port->mmio + i*0x80 + PORT_SDBV; in mtip_hw_init()
3259 while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) && in mtip_hw_init()
3294 mtip_init_port(dd->port); in mtip_hw_init()
3295 mtip_start_port(dd->port); in mtip_hw_init()
3316 init_waitqueue_head(&dd->port->svc_wait); in mtip_hw_init()
3335 mtip_deinit_port(dd->port); in mtip_hw_init()
3340 kfree(dd->port); in mtip_hw_init()
3349 if (dd->sr || !dd->port) in mtip_standby_drive()
3355 if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) && in mtip_standby_drive()
3358 rv = mtip_standby_immediate(dd->port); in mtip_standby_drive()
3378 mtip_deinit_port(dd->port); in mtip_hw_exit()
3394 kfree(dd->port); in mtip_hw_exit()
3395 dd->port = NULL; in mtip_hw_exit()
3449 mtip_deinit_port(dd->port); in mtip_hw_suspend()
3482 mtip_init_port(dd->port); in mtip_hw_resume()
3483 mtip_start_port(dd->port); in mtip_hw_resume()
3719 if (unlikely(test_bit(MTIP_PF_SE_ACTIVE_BIT, &dd->port->flags))) { in is_se_active()
3720 if (dd->port->ic_pause_timer) { in is_se_active()
3721 unsigned long to = dd->port->ic_pause_timer + in is_se_active()
3725 &dd->port->flags); in is_se_active()
3727 dd->port->ic_pause_timer = 0; in is_se_active()
3728 wake_up_interruptible(&dd->port->svc_wait); in is_se_active()
3809 if (cmd->unaligned && down_trylock(&dd->port->cmd_slot_unal)) in mtip_check_unal_depth()
3870 cmd->command_header = dd->port->command_list + in mtip_init_cmd()
3872 cmd->command_header_dma = dd->port->command_list_dma + in mtip_init_cmd()
3893 if (test_bit(req->tag, dd->port->cmds_to_issue)) in mtip_cmd_timeout()
3896 if (test_and_set_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags)) in mtip_cmd_timeout()
3899 wake_up_interruptible(&dd->port->svc_wait); in mtip_cmd_timeout()
4130 else if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &dd->port->flags)) { in mtip_no_dev_cleanup()
4134 cmd->comp_func(dd->port, MTIP_TAG_INTERNAL, in mtip_no_dev_cleanup()
4156 set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags); in mtip_block_remove()
4157 wake_up_interruptible(&dd->port->svc_wait); in mtip_block_remove()
4175 if (!mtip_quiesce_io(dd->port, MTIP_QUIESCE_IO_TIMEOUT_MS, in mtip_block_remove()