Lines Matching refs:port
166 if (dd->port) { in mtip_check_surprise_removal()
167 set_bit(MTIP_PF_SR_CLEANUP_BIT, &dd->port->flags); in mtip_check_surprise_removal()
168 wake_up_interruptible(&dd->port->svc_wait); in mtip_check_surprise_removal()
228 static void mtip_async_complete(struct mtip_port *port, in mtip_async_complete() argument
231 struct driver_data *dd = port->dd; in mtip_async_complete()
234 if (unlikely(!dd) || unlikely(!port)) in mtip_async_complete()
238 dev_warn(&port->dd->pdev->dev, in mtip_async_complete()
248 up(&port->cmd_slot_unal); in mtip_async_complete()
300 static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag) in mtip_issue_ncq_command() argument
305 spin_lock(&port->cmd_issue_lock[group]); in mtip_issue_ncq_command()
307 port->s_active[MTIP_TAG_INDEX(tag)]); in mtip_issue_ncq_command()
309 port->cmd_issue[MTIP_TAG_INDEX(tag)]); in mtip_issue_ncq_command()
310 spin_unlock(&port->cmd_issue_lock[group]); in mtip_issue_ncq_command()
322 static int mtip_enable_fis(struct mtip_port *port, int enable) in mtip_enable_fis() argument
327 tmp = readl(port->mmio + PORT_CMD); in mtip_enable_fis()
329 writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD); in mtip_enable_fis()
331 writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD); in mtip_enable_fis()
334 readl(port->mmio + PORT_CMD); in mtip_enable_fis()
348 static int mtip_enable_engine(struct mtip_port *port, int enable) in mtip_enable_engine() argument
353 tmp = readl(port->mmio + PORT_CMD); in mtip_enable_engine()
355 writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD); in mtip_enable_engine()
357 writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD); in mtip_enable_engine()
359 readl(port->mmio + PORT_CMD); in mtip_enable_engine()
369 static inline void mtip_start_port(struct mtip_port *port) in mtip_start_port() argument
372 mtip_enable_fis(port, 1); in mtip_start_port()
375 mtip_enable_engine(port, 1); in mtip_start_port()
387 static inline void mtip_deinit_port(struct mtip_port *port) in mtip_deinit_port() argument
390 writel(0, port->mmio + PORT_IRQ_MASK); in mtip_deinit_port()
393 mtip_enable_engine(port, 0); in mtip_deinit_port()
396 mtip_enable_fis(port, 0); in mtip_deinit_port()
412 static void mtip_init_port(struct mtip_port *port) in mtip_init_port() argument
415 mtip_deinit_port(port); in mtip_init_port()
418 if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) { in mtip_init_port()
419 writel((port->command_list_dma >> 16) >> 16, in mtip_init_port()
420 port->mmio + PORT_LST_ADDR_HI); in mtip_init_port()
421 writel((port->rxfis_dma >> 16) >> 16, in mtip_init_port()
422 port->mmio + PORT_FIS_ADDR_HI); in mtip_init_port()
425 writel(port->command_list_dma & 0xFFFFFFFF, in mtip_init_port()
426 port->mmio + PORT_LST_ADDR); in mtip_init_port()
427 writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR); in mtip_init_port()
430 writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR); in mtip_init_port()
433 for (i = 0; i < port->dd->slot_groups; i++) in mtip_init_port()
434 writel(0xFFFFFFFF, port->completed[i]); in mtip_init_port()
437 writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT); in mtip_init_port()
440 writel(readl(port->dd->mmio + HOST_IRQ_STAT), in mtip_init_port()
441 port->dd->mmio + HOST_IRQ_STAT); in mtip_init_port()
444 writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK); in mtip_init_port()
455 static void mtip_restart_port(struct mtip_port *port) in mtip_restart_port() argument
460 mtip_enable_engine(port, 0); in mtip_restart_port()
464 while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) in mtip_restart_port()
468 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
475 if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) { in mtip_restart_port()
476 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
479 if (mtip_hba_reset(port->dd)) in mtip_restart_port()
480 dev_err(&port->dd->pdev->dev, in mtip_restart_port()
487 dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n"); in mtip_restart_port()
490 writel(readl(port->mmio + PORT_SCR_CTL) | in mtip_restart_port()
491 1, port->mmio + PORT_SCR_CTL); in mtip_restart_port()
492 readl(port->mmio + PORT_SCR_CTL); in mtip_restart_port()
499 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
503 writel(readl(port->mmio + PORT_SCR_CTL) & ~1, in mtip_restart_port()
504 port->mmio + PORT_SCR_CTL); in mtip_restart_port()
505 readl(port->mmio + PORT_SCR_CTL); in mtip_restart_port()
509 while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) in mtip_restart_port()
513 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
516 if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) in mtip_restart_port()
517 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
520 mtip_init_port(port); in mtip_restart_port()
521 mtip_start_port(port); in mtip_restart_port()
536 mtip_init_port(dd->port); in mtip_device_reset()
537 mtip_start_port(dd->port); in mtip_device_reset()
579 static void mtip_completion(struct mtip_port *port, in mtip_completion() argument
584 dev_warn(&port->dd->pdev->dev, in mtip_completion()
590 static void mtip_null_completion(struct mtip_port *port, in mtip_null_completion() argument
595 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer,
597 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id,
610 struct mtip_port *port; in mtip_handle_tfe() local
622 port = dd->port; in mtip_handle_tfe()
624 set_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); in mtip_handle_tfe()
626 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && in mtip_handle_tfe()
627 test_bit(MTIP_TAG_INTERNAL, port->allocated)) { in mtip_handle_tfe()
632 cmd->comp_func(port, MTIP_TAG_INTERNAL, in mtip_handle_tfe()
643 completed = readl(port->completed[group]); in mtip_handle_tfe()
648 writel(completed, port->completed[group]); in mtip_handle_tfe()
664 cmd->comp_func(port, tag, cmd, 0); in mtip_handle_tfe()
666 dev_err(&port->dd->pdev->dev, in mtip_handle_tfe()
681 mtip_restart_port(port); in mtip_handle_tfe()
684 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_handle_tfe()
685 dd->port->log_buf, in mtip_handle_tfe()
686 dd->port->log_buf_dma, 1); in mtip_handle_tfe()
692 buf = (unsigned char *)dd->port->log_buf; in mtip_handle_tfe()
744 cmd->comp_func(port, tag, in mtip_handle_tfe()
760 mtip_issue_ncq_command(port, tag); in mtip_handle_tfe()
766 dev_warn(&port->dd->pdev->dev, in mtip_handle_tfe()
770 cmd->comp_func(port, tag, cmd, PORT_IRQ_TF_ERR); in mtip_handle_tfe()
772 dev_warn(&port->dd->pdev->dev, in mtip_handle_tfe()
781 clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); in mtip_handle_tfe()
782 wake_up_interruptible(&port->svc_wait); in mtip_handle_tfe()
788 static inline void mtip_workq_sdbfx(struct mtip_port *port, int group, in mtip_workq_sdbfx() argument
791 struct driver_data *dd = port->dd; in mtip_workq_sdbfx()
800 writel(completed, port->completed[group]); in mtip_workq_sdbfx()
813 command->comp_func(port, tag, command, 0); in mtip_workq_sdbfx()
838 struct mtip_port *port = dd->port; in mtip_process_legacy() local
841 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && in mtip_process_legacy()
842 (cmd != NULL) && !(readl(port->cmd_issue[MTIP_TAG_INTERNAL]) in mtip_process_legacy()
845 cmd->comp_func(port, MTIP_TAG_INTERNAL, cmd, 0); in mtip_process_legacy()
862 writel((1 << 26), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
868 writel((1 << 16), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
879 set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags); in mtip_process_errors()
880 wake_up_interruptible(&dd->port->svc_wait); in mtip_process_errors()
887 struct mtip_port *port = dd->port; in mtip_handle_irq() local
898 port_stat = readl(port->mmio + PORT_IRQ_STAT); in mtip_handle_irq()
903 writel(port_stat, port->mmio + PORT_IRQ_STAT); in mtip_handle_irq()
914 twork->completed = readl(port->completed[i]); in mtip_handle_irq()
931 mtip_workq_sdbfx(port, 0, in mtip_handle_irq()
983 static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag) in mtip_issue_non_ncq_command() argument
986 port->cmd_issue[MTIP_TAG_INDEX(tag)]); in mtip_issue_non_ncq_command()
989 static bool mtip_pause_ncq(struct mtip_port *port, in mtip_pause_ncq() argument
995 reply = port->rxfis + RX_FIS_D2H_REG; in mtip_pause_ncq()
996 task_file_data = readl(port->mmio+PORT_TFDATA); in mtip_pause_ncq()
1002 set_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); in mtip_pause_ncq()
1003 port->ic_pause_timer = jiffies; in mtip_pause_ncq()
1007 set_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); in mtip_pause_ncq()
1008 port->ic_pause_timer = jiffies; in mtip_pause_ncq()
1014 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
1015 clear_bit(MTIP_DDF_REBUILD_FAILED_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
1017 mtip_restart_port(port); in mtip_pause_ncq()
1034 static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout) in mtip_quiesce_io() argument
1040 blk_mq_stop_hw_queues(port->dd->queue); in mtip_quiesce_io()
1044 if (test_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags) && in mtip_quiesce_io()
1045 test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { in mtip_quiesce_io()
1051 if (mtip_check_surprise_removal(port->dd->pdev)) in mtip_quiesce_io()
1053 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_quiesce_io()
1060 active = readl(port->s_active[0]) & 0xFFFFFFFE; in mtip_quiesce_io()
1061 for (n = 1; n < port->dd->slot_groups; n++) in mtip_quiesce_io()
1062 active |= readl(port->s_active[n]); in mtip_quiesce_io()
1068 blk_mq_start_stopped_hw_queues(port->dd->queue, true); in mtip_quiesce_io()
1071 blk_mq_start_stopped_hw_queues(port->dd->queue, true); in mtip_quiesce_io()
1093 static int mtip_exec_internal_command(struct mtip_port *port, in mtip_exec_internal_command() argument
1105 struct driver_data *dd = port->dd; in mtip_exec_internal_command()
1117 set_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
1118 port->ic_pause_timer = 0; in mtip_exec_internal_command()
1120 clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
1121 clear_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
1126 if (mtip_quiesce_io(port, in mtip_exec_internal_command()
1131 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
1132 wake_up_interruptible(&port->svc_wait); in mtip_exec_internal_command()
1173 mtip_issue_non_ncq_command(port, MTIP_TAG_INTERNAL); in mtip_exec_internal_command()
1215 while ((readl(port->cmd_issue[MTIP_TAG_INTERNAL]) in mtip_exec_internal_command()
1228 port_stat = readl(port->mmio + PORT_IRQ_STAT); in mtip_exec_internal_command()
1240 writel(port_stat, port->mmio + PORT_IRQ_STAT); in mtip_exec_internal_command()
1250 if (readl(port->cmd_issue[MTIP_TAG_INTERNAL]) in mtip_exec_internal_command()
1261 if (rv >= 0 && mtip_pause_ncq(port, fis)) { in mtip_exec_internal_command()
1265 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
1266 wake_up_interruptible(&port->svc_wait); in mtip_exec_internal_command()
1302 *timeout = ((*(dd->port->identify + 90) * 2) * 60000); in mtip_set_timeout()
1304 *timeout = ((*(dd->port->identify + 89) * 2) * 60000); in mtip_set_timeout()
1341 static int mtip_get_identify(struct mtip_port *port, void __user *user_buffer) in mtip_get_identify() argument
1346 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_get_identify()
1356 port->identify_valid = 0; in mtip_get_identify()
1359 memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS); in mtip_get_identify()
1362 if (mtip_exec_internal_command(port, in mtip_get_identify()
1365 port->identify_dma, in mtip_get_identify()
1381 ata_swap_string(port->identify + 27, 40); /* model string*/ in mtip_get_identify()
1382 ata_swap_string(port->identify + 23, 8); /* firmware string*/ in mtip_get_identify()
1383 ata_swap_string(port->identify + 10, 20); /* serial# string*/ in mtip_get_identify()
1388 port->identify[i] = le16_to_cpu(port->identify[i]); in mtip_get_identify()
1393 if (port->identify[128] & 0x4) in mtip_get_identify()
1394 set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_get_identify()
1396 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_get_identify()
1400 if (port->identify[69] & (1 << 14) && port->identify[69] & (1 << 5)) in mtip_get_identify()
1401 port->dd->trim_supp = true; in mtip_get_identify()
1404 port->dd->trim_supp = false; in mtip_get_identify()
1407 port->identify_valid = 1; in mtip_get_identify()
1412 port->identify, in mtip_get_identify()
1432 static int mtip_standby_immediate(struct mtip_port *port) in mtip_standby_immediate() argument
1445 mtip_set_timeout(port->dd, &fis, &timeout, 0); in mtip_standby_immediate()
1448 rv = mtip_exec_internal_command(port, in mtip_standby_immediate()
1459 dev_warn(&port->dd->pdev->dev, in mtip_standby_immediate()
1477 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer, in mtip_read_log_page() argument
1494 return mtip_exec_internal_command(port, in mtip_read_log_page()
1514 static int mtip_get_smart_data(struct mtip_port *port, u8 *buffer, in mtip_get_smart_data() argument
1529 return mtip_exec_internal_command(port, in mtip_get_smart_data()
1550 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id, in mtip_get_smart_attr() argument
1559 if (!port->identify_valid) { in mtip_get_smart_attr()
1560 dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n"); in mtip_get_smart_attr()
1563 if (!(port->identify[82] & 0x1)) { in mtip_get_smart_attr()
1564 dev_warn(&port->dd->pdev->dev, "SMART not supported\n"); in mtip_get_smart_attr()
1567 if (!(port->identify[85] & 0x1)) { in mtip_get_smart_attr()
1568 dev_warn(&port->dd->pdev->dev, "SMART not enabled\n"); in mtip_get_smart_attr()
1572 memset(port->smart_buf, 0, ATA_SECT_SIZE); in mtip_get_smart_attr()
1573 rv = mtip_get_smart_data(port, port->smart_buf, port->smart_buf_dma); in mtip_get_smart_attr()
1575 dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n"); in mtip_get_smart_attr()
1579 pattr = (struct smart_attr *)(port->smart_buf + 2); in mtip_get_smart_attr()
1587 dev_warn(&port->dd->pdev->dev, in mtip_get_smart_attr()
1657 if (mtip_exec_internal_command(dd->port, in mtip_send_trim()
1683 struct mtip_port *port = dd->port; in mtip_hw_get_capacity() local
1685 raw0 = port->identify[100]; in mtip_hw_get_capacity()
1686 raw1 = port->identify[101]; in mtip_hw_get_capacity()
1687 raw2 = port->identify[102]; in mtip_hw_get_capacity()
1688 raw3 = port->identify[103]; in mtip_hw_get_capacity()
1691 return (bool) !!port->identify_valid; in mtip_hw_get_capacity()
1702 static void mtip_dump_identify(struct mtip_port *port) in mtip_dump_identify() argument
1708 if (!port->identify_valid) in mtip_dump_identify()
1711 strlcpy(cbuf, (char *)(port->identify+10), 21); in mtip_dump_identify()
1712 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1715 strlcpy(cbuf, (char *)(port->identify+23), 9); in mtip_dump_identify()
1716 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1719 strlcpy(cbuf, (char *)(port->identify+27), 41); in mtip_dump_identify()
1720 dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf); in mtip_dump_identify()
1722 dev_info(&port->dd->pdev->dev, "Security: %04x %s\n", in mtip_dump_identify()
1723 port->identify[128], in mtip_dump_identify()
1724 port->identify[128] & 0x4 ? "(LOCKED)" : ""); in mtip_dump_identify()
1726 if (mtip_hw_get_capacity(port->dd, §ors)) in mtip_dump_identify()
1727 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1732 pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid); in mtip_dump_identify()
1744 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1790 static int exec_drive_task(struct mtip_port *port, u8 *command) in exec_drive_task() argument
1793 struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG); in exec_drive_task()
1808 mtip_set_timeout(port->dd, &fis, &to, 0); in exec_drive_task()
1821 if (mtip_exec_internal_command(port, in exec_drive_task()
1860 static int exec_drive_command(struct mtip_port *port, u8 *command, in exec_drive_command() argument
1874 buf = dmam_alloc_coherent(&port->dd->pdev->dev, in exec_drive_command()
1879 dev_err(&port->dd->pdev->dev, in exec_drive_command()
1900 mtip_set_timeout(port->dd, &fis, &to, 0); in exec_drive_command()
1903 reply = (port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_command()
1905 reply = (port->rxfis + RX_FIS_D2H_REG); in exec_drive_command()
1917 if (mtip_exec_internal_command(port, in exec_drive_command()
1953 dmam_free_coherent(&port->dd->pdev->dev, in exec_drive_command()
2086 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
2089 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
2092 reply = (dd->port->rxfis + RX_FIS_D2H_REG); in exec_drive_taskfile()
2171 if (mtip_exec_internal_command(dd->port, in exec_drive_taskfile()
2183 task_file_data = readl(dd->port->mmio+PORT_TFDATA); in exec_drive_taskfile()
2186 reply = dd->port->rxfis + RX_FIS_PIO_SETUP; in exec_drive_taskfile()
2189 reply = dd->port->rxfis + RX_FIS_D2H_REG; in exec_drive_taskfile()
2280 if (copy_to_user((void __user *)arg, dd->port->identify, in mtip_hw_ioctl()
2296 if (exec_drive_command(dd->port, in mtip_hw_ioctl()
2320 if (exec_drive_task(dd->port, drive_command)) in mtip_hw_ioctl()
2383 struct mtip_port *port = dd->port; in mtip_hw_submit_io() local
2391 prefetch(&port->flags); in mtip_hw_submit_io()
2446 if (unlikely(port->flags & MTIP_PF_PAUSE_IO)) { in mtip_hw_submit_io()
2447 set_bit(rq->tag, port->cmds_to_issue); in mtip_hw_submit_io()
2448 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); in mtip_hw_submit_io()
2453 mtip_issue_ncq_command(port, rq->tag); in mtip_hw_submit_io()
2499 if (dd->port && in show_device_status()
2500 dd->port->identify && in show_device_status()
2501 dd->port->identify_valid) { in show_device_status()
2503 (char *) (dd->port->identify + 10), 21); in show_device_status()
2504 status = *(dd->port->identify + 141); in show_device_status()
2510 if (dd->port && in show_device_status()
2511 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { in show_device_status()
2529 if (dd->port && in show_device_status()
2530 dd->port->identify && in show_device_status()
2531 dd->port->identify_valid) { in show_device_status()
2533 (char *) (dd->port->identify+10), 21); in show_device_status()
2534 status = *(dd->port->identify + 141); in show_device_status()
2540 if (dd->port && in show_device_status()
2541 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { in show_device_status()
2612 readl(dd->port->s_active[n])); in mtip_hw_read_registers()
2619 readl(dd->port->cmd_issue[n])); in mtip_hw_read_registers()
2626 readl(dd->port->completed[n])); in mtip_hw_read_registers()
2630 readl(dd->port->mmio + PORT_IRQ_STAT)); in mtip_hw_read_registers()
2640 dd->port->allocated[n/2] >> (32*(n&1)); in mtip_hw_read_registers()
2642 group_allocated = dd->port->allocated[n]; in mtip_hw_read_registers()
2652 dd->port->cmds_to_issue[n/2] >> (32*(n&1)); in mtip_hw_read_registers()
2654 group_allocated = dd->port->cmds_to_issue[n]; in mtip_hw_read_registers()
2687 dd->port->flags); in mtip_hw_read_flags()
2892 if (mtip_get_identify(dd->port, NULL) < 0) in mtip_ftl_rebuild_poll()
2895 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_ftl_rebuild_poll()
2936 struct mtip_port *port = dd->port; in mtip_service_thread() local
2940 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) in mtip_service_thread()
2942 clear_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); in mtip_service_thread()
2948 wait_event_interruptible(port->svc_wait, (port->flags) && in mtip_service_thread()
2949 !(port->flags & MTIP_PF_PAUSE_IO)); in mtip_service_thread()
2951 set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); in mtip_service_thread()
2954 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) in mtip_service_thread()
2958 if (test_bit(MTIP_PF_SR_CLEANUP_BIT, &port->flags)) in mtip_service_thread()
2967 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) { in mtip_service_thread()
2969 clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); in mtip_service_thread()
2972 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) in mtip_service_thread()
2975 if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { in mtip_service_thread()
2981 slot = find_next_bit(port->cmds_to_issue, in mtip_service_thread()
2998 mtip_issue_ncq_command(port, slot); in mtip_service_thread()
3000 clear_bit(slot, port->cmds_to_issue); in mtip_service_thread()
3003 clear_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); in mtip_service_thread()
3006 if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) { in mtip_service_thread()
3010 clear_bit(MTIP_PF_REBUILD_BIT, &port->flags); in mtip_service_thread()
3036 struct mtip_port *port = dd->port; in mtip_dma_free() local
3038 if (port->block1) in mtip_dma_free()
3040 port->block1, port->block1_dma); in mtip_dma_free()
3042 if (port->command_list) { in mtip_dma_free()
3044 port->command_list, port->command_list_dma); in mtip_dma_free()
3058 struct mtip_port *port = dd->port; in mtip_dma_alloc() local
3061 port->block1 = in mtip_dma_alloc()
3063 &port->block1_dma, GFP_KERNEL); in mtip_dma_alloc()
3064 if (!port->block1) in mtip_dma_alloc()
3066 memset(port->block1, 0, BLOCK_DMA_ALLOC_SZ); in mtip_dma_alloc()
3069 port->command_list = in mtip_dma_alloc()
3071 &port->command_list_dma, GFP_KERNEL); in mtip_dma_alloc()
3072 if (!port->command_list) { in mtip_dma_alloc()
3074 port->block1, port->block1_dma); in mtip_dma_alloc()
3075 port->block1 = NULL; in mtip_dma_alloc()
3076 port->block1_dma = 0; in mtip_dma_alloc()
3079 memset(port->command_list, 0, AHCI_CMD_TBL_SZ); in mtip_dma_alloc()
3082 port->rxfis = port->block1 + AHCI_RX_FIS_OFFSET; in mtip_dma_alloc()
3083 port->rxfis_dma = port->block1_dma + AHCI_RX_FIS_OFFSET; in mtip_dma_alloc()
3084 port->identify = port->block1 + AHCI_IDFY_OFFSET; in mtip_dma_alloc()
3085 port->identify_dma = port->block1_dma + AHCI_IDFY_OFFSET; in mtip_dma_alloc()
3086 port->log_buf = port->block1 + AHCI_SECTBUF_OFFSET; in mtip_dma_alloc()
3087 port->log_buf_dma = port->block1_dma + AHCI_SECTBUF_OFFSET; in mtip_dma_alloc()
3088 port->smart_buf = port->block1 + AHCI_SMARTBUF_OFFSET; in mtip_dma_alloc()
3089 port->smart_buf_dma = port->block1_dma + AHCI_SMARTBUF_OFFSET; in mtip_dma_alloc()
3100 if (mtip_get_identify(dd->port, NULL) < 0) in mtip_hw_get_identify()
3103 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_hw_get_identify()
3105 set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags); in mtip_hw_get_identify()
3108 mtip_dump_identify(dd->port); in mtip_hw_get_identify()
3111 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_hw_get_identify()
3112 dd->port->log_buf, in mtip_hw_get_identify()
3113 dd->port->log_buf_dma, 1); in mtip_hw_get_identify()
3119 buf = (unsigned char *)dd->port->log_buf; in mtip_hw_get_identify()
3139 if (mtip_get_smart_attr(dd->port, 242, &attr242)) in mtip_hw_get_identify()
3176 dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL, in mtip_hw_init()
3178 if (!dd->port) { in mtip_hw_init()
3186 dd->work[i].port = dd->port; in mtip_hw_init()
3194 sema_init(&dd->port->cmd_slot_unal, dd->unal_qdepth); in mtip_hw_init()
3198 spin_lock_init(&dd->port->cmd_issue_lock[i]); in mtip_hw_init()
3201 dd->port->mmio = dd->mmio + PORT_OFFSET; in mtip_hw_init()
3202 dd->port->dd = dd; in mtip_hw_init()
3211 dd->port->s_active[i] = in mtip_hw_init()
3212 dd->port->mmio + i*0x80 + PORT_SCR_ACT; in mtip_hw_init()
3213 dd->port->cmd_issue[i] = in mtip_hw_init()
3214 dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE; in mtip_hw_init()
3215 dd->port->completed[i] = in mtip_hw_init()
3216 dd->port->mmio + i*0x80 + PORT_SDBV; in mtip_hw_init()
3221 while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) && in mtip_hw_init()
3256 mtip_init_port(dd->port); in mtip_hw_init()
3257 mtip_start_port(dd->port); in mtip_hw_init()
3278 init_waitqueue_head(&dd->port->svc_wait); in mtip_hw_init()
3297 mtip_deinit_port(dd->port); in mtip_hw_init()
3302 kfree(dd->port); in mtip_hw_init()
3311 if (dd->sr || !dd->port) in mtip_standby_drive()
3317 if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) && in mtip_standby_drive()
3320 rv = mtip_standby_immediate(dd->port); in mtip_standby_drive()
3344 mtip_deinit_port(dd->port); in mtip_hw_exit()
3360 kfree(dd->port); in mtip_hw_exit()
3361 dd->port = NULL; in mtip_hw_exit()
3415 mtip_deinit_port(dd->port); in mtip_hw_suspend()
3448 mtip_init_port(dd->port); in mtip_hw_resume()
3449 mtip_start_port(dd->port); in mtip_hw_resume()
3752 if (cmd->unaligned && down_trylock(&dd->port->cmd_slot_unal)) in mtip_check_unal_depth()
3805 cmd->command_header = dd->port->command_list + in mtip_init_cmd()
3807 cmd->command_header_dma = dd->port->command_list_dma + in mtip_init_cmd()
4052 set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags); in mtip_block_remove()
4053 wake_up_interruptible(&dd->port->svc_wait); in mtip_block_remove()