root/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. hisi_sas_read32
  2. hisi_sas_write32
  3. hisi_sas_phy_write32
  4. hisi_sas_phy_read32
  5. init_reg_v3_hw
  6. config_phy_opt_mode_v3_hw
  7. config_id_frame_v3_hw
  8. setup_itct_v3_hw
  9. clear_itct_v3_hw
  10. dereg_device_v3_hw
  11. reset_hw_v3_hw
  12. hw_init_v3_hw
  13. enable_phy_v3_hw
  14. disable_phy_v3_hw
  15. start_phy_v3_hw
  16. phy_hard_reset_v3_hw
  17. phy_get_max_linkrate_v3_hw
  18. phys_init_v3_hw
  19. sl_notify_ssp_v3_hw
  20. get_wideport_bitmap_v3_hw
  21. start_delivery_v3_hw
  22. prep_prd_sge_v3_hw
  23. prep_prd_sge_dif_v3_hw
  24. get_prot_chk_msk_v3_hw
  25. fill_prot_v3_hw
  26. prep_ssp_v3_hw
  27. prep_smp_v3_hw
  28. prep_ata_v3_hw
  29. prep_abort_v3_hw
  30. phy_up_v3_hw
  31. phy_down_v3_hw
  32. phy_bcast_v3_hw
  33. int_phy_up_down_bcast_v3_hw
  34. handle_chl_int1_v3_hw
  35. phy_get_events_v3_hw
  36. handle_chl_int2_v3_hw
  37. handle_chl_int0_v3_hw
  38. int_chnl_int_v3_hw
  39. multi_bit_ecc_error_process_v3_hw
  40. fatal_ecc_int_v3_hw
  41. fatal_axi_int_v3_hw
  42. slot_err_v3_hw
  43. slot_complete_v3_hw
  44. cq_tasklet_v3_hw
  45. cq_interrupt_v3_hw
  46. setup_reply_map_v3_hw
  47. interrupt_init_v3_hw
  48. hisi_sas_v3_init
  49. phy_set_linkrate_v3_hw
  50. interrupt_disable_v3_hw
  51. get_phys_state_v3_hw
  52. disable_host_v3_hw
  53. soft_reset_v3_hw
  54. write_gpio_v3_hw
  55. wait_cmds_complete_timeout_v3_hw
  56. intr_conv_v3_hw_show
  57. config_intr_coal_v3_hw
  58. intr_coal_ticks_v3_hw_show
  59. intr_coal_ticks_v3_hw_store
  60. intr_coal_count_v3_hw_show
  61. intr_coal_count_v3_hw_store
  62. debugfs_snapshot_prepare_v3_hw
  63. debugfs_snapshot_restore_v3_hw
  64. read_iost_itct_cache_v3_hw
  65. hisi_sas_bist_test_prep_v3_hw
  66. hisi_sas_bist_test_restore_v3_hw
  67. debugfs_set_bist_v3_hw
  68. hisi_sas_shost_alloc_pci
  69. hisi_sas_v3_probe
  70. hisi_sas_v3_destroy_irqs
  71. hisi_sas_v3_remove
  72. hisi_sas_reset_prepare_v3_hw
  73. hisi_sas_reset_done_v3_hw
  74. hisi_sas_v3_suspend
  75. hisi_sas_v3_resume

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (c) 2017 Hisilicon Limited.
   4  */
   5 
   6 #include "hisi_sas.h"
   7 #define DRV_NAME "hisi_sas_v3_hw"
   8 
   9 /* global registers need init */
  10 #define DLVRY_QUEUE_ENABLE              0x0
  11 #define IOST_BASE_ADDR_LO               0x8
  12 #define IOST_BASE_ADDR_HI               0xc
  13 #define ITCT_BASE_ADDR_LO               0x10
  14 #define ITCT_BASE_ADDR_HI               0x14
  15 #define IO_BROKEN_MSG_ADDR_LO           0x18
  16 #define IO_BROKEN_MSG_ADDR_HI           0x1c
  17 #define PHY_CONTEXT                     0x20
  18 #define PHY_STATE                       0x24
  19 #define PHY_PORT_NUM_MA                 0x28
  20 #define PHY_CONN_RATE                   0x30
  21 #define ITCT_CLR                        0x44
  22 #define ITCT_CLR_EN_OFF                 16
  23 #define ITCT_CLR_EN_MSK                 (0x1 << ITCT_CLR_EN_OFF)
  24 #define ITCT_DEV_OFF                    0
  25 #define ITCT_DEV_MSK                    (0x7ff << ITCT_DEV_OFF)
  26 #define SAS_AXI_USER3                   0x50
  27 #define IO_SATA_BROKEN_MSG_ADDR_LO      0x58
  28 #define IO_SATA_BROKEN_MSG_ADDR_HI      0x5c
  29 #define SATA_INITI_D2H_STORE_ADDR_LO    0x60
  30 #define SATA_INITI_D2H_STORE_ADDR_HI    0x64
  31 #define CFG_MAX_TAG                     0x68
  32 #define HGC_SAS_TX_OPEN_FAIL_RETRY_CTRL 0x84
  33 #define HGC_SAS_TXFAIL_RETRY_CTRL       0x88
  34 #define HGC_GET_ITV_TIME                0x90
  35 #define DEVICE_MSG_WORK_MODE            0x94
  36 #define OPENA_WT_CONTI_TIME             0x9c
  37 #define I_T_NEXUS_LOSS_TIME             0xa0
  38 #define MAX_CON_TIME_LIMIT_TIME         0xa4
  39 #define BUS_INACTIVE_LIMIT_TIME         0xa8
  40 #define REJECT_TO_OPEN_LIMIT_TIME       0xac
  41 #define CQ_INT_CONVERGE_EN              0xb0
  42 #define CFG_AGING_TIME                  0xbc
  43 #define HGC_DFX_CFG2                    0xc0
  44 #define CFG_ABT_SET_QUERY_IPTT  0xd4
  45 #define CFG_SET_ABORTED_IPTT_OFF        0
  46 #define CFG_SET_ABORTED_IPTT_MSK        (0xfff << CFG_SET_ABORTED_IPTT_OFF)
  47 #define CFG_SET_ABORTED_EN_OFF  12
  48 #define CFG_ABT_SET_IPTT_DONE   0xd8
  49 #define CFG_ABT_SET_IPTT_DONE_OFF       0
  50 #define HGC_IOMB_PROC1_STATUS   0x104
  51 #define HGC_LM_DFX_STATUS2              0x128
  52 #define HGC_LM_DFX_STATUS2_IOSTLIST_OFF         0
  53 #define HGC_LM_DFX_STATUS2_IOSTLIST_MSK (0xfff << \
  54                                          HGC_LM_DFX_STATUS2_IOSTLIST_OFF)
  55 #define HGC_LM_DFX_STATUS2_ITCTLIST_OFF         12
  56 #define HGC_LM_DFX_STATUS2_ITCTLIST_MSK (0x7ff << \
  57                                          HGC_LM_DFX_STATUS2_ITCTLIST_OFF)
  58 #define HGC_CQE_ECC_ADDR                0x13c
  59 #define HGC_CQE_ECC_1B_ADDR_OFF 0
  60 #define HGC_CQE_ECC_1B_ADDR_MSK (0x3f << HGC_CQE_ECC_1B_ADDR_OFF)
  61 #define HGC_CQE_ECC_MB_ADDR_OFF 8
  62 #define HGC_CQE_ECC_MB_ADDR_MSK (0x3f << HGC_CQE_ECC_MB_ADDR_OFF)
  63 #define HGC_IOST_ECC_ADDR               0x140
  64 #define HGC_IOST_ECC_1B_ADDR_OFF        0
  65 #define HGC_IOST_ECC_1B_ADDR_MSK        (0x3ff << HGC_IOST_ECC_1B_ADDR_OFF)
  66 #define HGC_IOST_ECC_MB_ADDR_OFF        16
  67 #define HGC_IOST_ECC_MB_ADDR_MSK        (0x3ff << HGC_IOST_ECC_MB_ADDR_OFF)
  68 #define HGC_DQE_ECC_ADDR                0x144
  69 #define HGC_DQE_ECC_1B_ADDR_OFF 0
  70 #define HGC_DQE_ECC_1B_ADDR_MSK (0xfff << HGC_DQE_ECC_1B_ADDR_OFF)
  71 #define HGC_DQE_ECC_MB_ADDR_OFF 16
  72 #define HGC_DQE_ECC_MB_ADDR_MSK (0xfff << HGC_DQE_ECC_MB_ADDR_OFF)
  73 #define CHNL_INT_STATUS                 0x148
  74 #define TAB_DFX                         0x14c
  75 #define HGC_ITCT_ECC_ADDR               0x150
  76 #define HGC_ITCT_ECC_1B_ADDR_OFF                0
  77 #define HGC_ITCT_ECC_1B_ADDR_MSK                (0x3ff << \
  78                                                  HGC_ITCT_ECC_1B_ADDR_OFF)
  79 #define HGC_ITCT_ECC_MB_ADDR_OFF                16
  80 #define HGC_ITCT_ECC_MB_ADDR_MSK                (0x3ff << \
  81                                                  HGC_ITCT_ECC_MB_ADDR_OFF)
  82 #define HGC_AXI_FIFO_ERR_INFO  0x154
  83 #define AXI_ERR_INFO_OFF               0
  84 #define AXI_ERR_INFO_MSK               (0xff << AXI_ERR_INFO_OFF)
  85 #define FIFO_ERR_INFO_OFF              8
  86 #define FIFO_ERR_INFO_MSK              (0xff << FIFO_ERR_INFO_OFF)
  87 #define TAB_RD_TYPE                     0x15c
  88 #define INT_COAL_EN                     0x19c
  89 #define OQ_INT_COAL_TIME                0x1a0
  90 #define OQ_INT_COAL_CNT                 0x1a4
  91 #define ENT_INT_COAL_TIME               0x1a8
  92 #define ENT_INT_COAL_CNT                0x1ac
  93 #define OQ_INT_SRC                      0x1b0
  94 #define OQ_INT_SRC_MSK                  0x1b4
  95 #define ENT_INT_SRC1                    0x1b8
  96 #define ENT_INT_SRC1_D2H_FIS_CH0_OFF    0
  97 #define ENT_INT_SRC1_D2H_FIS_CH0_MSK    (0x1 << ENT_INT_SRC1_D2H_FIS_CH0_OFF)
  98 #define ENT_INT_SRC1_D2H_FIS_CH1_OFF    8
  99 #define ENT_INT_SRC1_D2H_FIS_CH1_MSK    (0x1 << ENT_INT_SRC1_D2H_FIS_CH1_OFF)
 100 #define ENT_INT_SRC2                    0x1bc
 101 #define ENT_INT_SRC3                    0x1c0
 102 #define ENT_INT_SRC3_WP_DEPTH_OFF               8
 103 #define ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF      9
 104 #define ENT_INT_SRC3_RP_DEPTH_OFF               10
 105 #define ENT_INT_SRC3_AXI_OFF                    11
 106 #define ENT_INT_SRC3_FIFO_OFF                   12
 107 #define ENT_INT_SRC3_LM_OFF                             14
 108 #define ENT_INT_SRC3_ITC_INT_OFF        15
 109 #define ENT_INT_SRC3_ITC_INT_MSK        (0x1 << ENT_INT_SRC3_ITC_INT_OFF)
 110 #define ENT_INT_SRC3_ABT_OFF            16
 111 #define ENT_INT_SRC3_DQE_POISON_OFF     18
 112 #define ENT_INT_SRC3_IOST_POISON_OFF    19
 113 #define ENT_INT_SRC3_ITCT_POISON_OFF    20
 114 #define ENT_INT_SRC3_ITCT_NCQ_POISON_OFF        21
 115 #define ENT_INT_SRC_MSK1                0x1c4
 116 #define ENT_INT_SRC_MSK2                0x1c8
 117 #define ENT_INT_SRC_MSK3                0x1cc
 118 #define ENT_INT_SRC_MSK3_ENT95_MSK_OFF  31
 119 #define CHNL_PHYUPDOWN_INT_MSK          0x1d0
 120 #define CHNL_ENT_INT_MSK                        0x1d4
 121 #define HGC_COM_INT_MSK                         0x1d8
 122 #define ENT_INT_SRC_MSK3_ENT95_MSK_MSK  (0x1 << ENT_INT_SRC_MSK3_ENT95_MSK_OFF)
 123 #define SAS_ECC_INTR                    0x1e8
 124 #define SAS_ECC_INTR_DQE_ECC_1B_OFF             0
 125 #define SAS_ECC_INTR_DQE_ECC_MB_OFF             1
 126 #define SAS_ECC_INTR_IOST_ECC_1B_OFF    2
 127 #define SAS_ECC_INTR_IOST_ECC_MB_OFF    3
 128 #define SAS_ECC_INTR_ITCT_ECC_1B_OFF    4
 129 #define SAS_ECC_INTR_ITCT_ECC_MB_OFF    5
 130 #define SAS_ECC_INTR_ITCTLIST_ECC_1B_OFF        6
 131 #define SAS_ECC_INTR_ITCTLIST_ECC_MB_OFF        7
 132 #define SAS_ECC_INTR_IOSTLIST_ECC_1B_OFF        8
 133 #define SAS_ECC_INTR_IOSTLIST_ECC_MB_OFF        9
 134 #define SAS_ECC_INTR_CQE_ECC_1B_OFF             10
 135 #define SAS_ECC_INTR_CQE_ECC_MB_OFF             11
 136 #define SAS_ECC_INTR_NCQ_MEM0_ECC_1B_OFF        12
 137 #define SAS_ECC_INTR_NCQ_MEM0_ECC_MB_OFF        13
 138 #define SAS_ECC_INTR_NCQ_MEM1_ECC_1B_OFF        14
 139 #define SAS_ECC_INTR_NCQ_MEM1_ECC_MB_OFF        15
 140 #define SAS_ECC_INTR_NCQ_MEM2_ECC_1B_OFF        16
 141 #define SAS_ECC_INTR_NCQ_MEM2_ECC_MB_OFF        17
 142 #define SAS_ECC_INTR_NCQ_MEM3_ECC_1B_OFF        18
 143 #define SAS_ECC_INTR_NCQ_MEM3_ECC_MB_OFF        19
 144 #define SAS_ECC_INTR_OOO_RAM_ECC_1B_OFF         20
 145 #define SAS_ECC_INTR_OOO_RAM_ECC_MB_OFF         21
 146 #define SAS_ECC_INTR_MSK                0x1ec
 147 #define HGC_ERR_STAT_EN                 0x238
 148 #define CQE_SEND_CNT                    0x248
 149 #define DLVRY_Q_0_BASE_ADDR_LO          0x260
 150 #define DLVRY_Q_0_BASE_ADDR_HI          0x264
 151 #define DLVRY_Q_0_DEPTH                 0x268
 152 #define DLVRY_Q_0_WR_PTR                0x26c
 153 #define DLVRY_Q_0_RD_PTR                0x270
 154 #define HYPER_STREAM_ID_EN_CFG          0xc80
 155 #define OQ0_INT_SRC_MSK                 0xc90
 156 #define COMPL_Q_0_BASE_ADDR_LO          0x4e0
 157 #define COMPL_Q_0_BASE_ADDR_HI          0x4e4
 158 #define COMPL_Q_0_DEPTH                 0x4e8
 159 #define COMPL_Q_0_WR_PTR                0x4ec
 160 #define COMPL_Q_0_RD_PTR                0x4f0
 161 #define HGC_RXM_DFX_STATUS14            0xae8
 162 #define HGC_RXM_DFX_STATUS14_MEM0_OFF   0
 163 #define HGC_RXM_DFX_STATUS14_MEM0_MSK   (0x1ff << \
 164                                          HGC_RXM_DFX_STATUS14_MEM0_OFF)
 165 #define HGC_RXM_DFX_STATUS14_MEM1_OFF   9
 166 #define HGC_RXM_DFX_STATUS14_MEM1_MSK   (0x1ff << \
 167                                          HGC_RXM_DFX_STATUS14_MEM1_OFF)
 168 #define HGC_RXM_DFX_STATUS14_MEM2_OFF   18
 169 #define HGC_RXM_DFX_STATUS14_MEM2_MSK   (0x1ff << \
 170                                          HGC_RXM_DFX_STATUS14_MEM2_OFF)
 171 #define HGC_RXM_DFX_STATUS15            0xaec
 172 #define HGC_RXM_DFX_STATUS15_MEM3_OFF   0
 173 #define HGC_RXM_DFX_STATUS15_MEM3_MSK   (0x1ff << \
 174                                          HGC_RXM_DFX_STATUS15_MEM3_OFF)
 175 #define AWQOS_AWCACHE_CFG       0xc84
 176 #define ARQOS_ARCACHE_CFG       0xc88
 177 #define HILINK_ERR_DFX          0xe04
 178 #define SAS_GPIO_CFG_0          0x1000
 179 #define SAS_GPIO_CFG_1          0x1004
 180 #define SAS_GPIO_TX_0_1 0x1040
 181 #define SAS_CFG_DRIVE_VLD       0x1070
 182 
 183 /* phy registers requiring init */
 184 #define PORT_BASE                       (0x2000)
 185 #define PHY_CFG                         (PORT_BASE + 0x0)
 186 #define HARD_PHY_LINKRATE               (PORT_BASE + 0x4)
 187 #define PHY_CFG_ENA_OFF                 0
 188 #define PHY_CFG_ENA_MSK                 (0x1 << PHY_CFG_ENA_OFF)
 189 #define PHY_CFG_DC_OPT_OFF              2
 190 #define PHY_CFG_DC_OPT_MSK              (0x1 << PHY_CFG_DC_OPT_OFF)
 191 #define PHY_CFG_PHY_RST_OFF             3
 192 #define PHY_CFG_PHY_RST_MSK             (0x1 << PHY_CFG_PHY_RST_OFF)
 193 #define PROG_PHY_LINK_RATE              (PORT_BASE + 0x8)
 194 #define CFG_PROG_PHY_LINK_RATE_OFF      8
 195 #define CFG_PROG_PHY_LINK_RATE_MSK      (0xf << CFG_PROG_PHY_LINK_RATE_OFF)
 196 #define PHY_CTRL                        (PORT_BASE + 0x14)
 197 #define PHY_CTRL_RESET_OFF              0
 198 #define PHY_CTRL_RESET_MSK              (0x1 << PHY_CTRL_RESET_OFF)
 199 #define CMD_HDR_PIR_OFF                 8
 200 #define CMD_HDR_PIR_MSK                 (0x1 << CMD_HDR_PIR_OFF)
 201 #define SERDES_CFG                      (PORT_BASE + 0x1c)
 202 #define CFG_ALOS_CHK_DISABLE_OFF        9
 203 #define CFG_ALOS_CHK_DISABLE_MSK        (0x1 << CFG_ALOS_CHK_DISABLE_OFF)
 204 #define SAS_PHY_BIST_CTRL               (PORT_BASE + 0x2c)
 205 #define CFG_BIST_MODE_SEL_OFF           0
 206 #define CFG_BIST_MODE_SEL_MSK           (0xf << CFG_BIST_MODE_SEL_OFF)
 207 #define CFG_LOOP_TEST_MODE_OFF          14
 208 #define CFG_LOOP_TEST_MODE_MSK          (0x3 << CFG_LOOP_TEST_MODE_OFF)
 209 #define CFG_RX_BIST_EN_OFF              16
 210 #define CFG_RX_BIST_EN_MSK              (0x1 << CFG_RX_BIST_EN_OFF)
 211 #define CFG_TX_BIST_EN_OFF              17
 212 #define CFG_TX_BIST_EN_MSK              (0x1 << CFG_TX_BIST_EN_OFF)
 213 #define CFG_BIST_TEST_OFF               18
 214 #define CFG_BIST_TEST_MSK               (0x1 << CFG_BIST_TEST_OFF)
 215 #define SAS_PHY_BIST_CODE               (PORT_BASE + 0x30)
 216 #define SAS_PHY_BIST_CODE1              (PORT_BASE + 0x34)
 217 #define SAS_BIST_ERR_CNT                (PORT_BASE + 0x38)
 218 #define SL_CFG                          (PORT_BASE + 0x84)
 219 #define AIP_LIMIT                       (PORT_BASE + 0x90)
 220 #define SL_CONTROL                      (PORT_BASE + 0x94)
 221 #define SL_CONTROL_NOTIFY_EN_OFF        0
 222 #define SL_CONTROL_NOTIFY_EN_MSK        (0x1 << SL_CONTROL_NOTIFY_EN_OFF)
 223 #define SL_CTA_OFF              17
 224 #define SL_CTA_MSK              (0x1 << SL_CTA_OFF)
 225 #define RX_PRIMS_STATUS                 (PORT_BASE + 0x98)
 226 #define RX_BCAST_CHG_OFF                1
 227 #define RX_BCAST_CHG_MSK                (0x1 << RX_BCAST_CHG_OFF)
 228 #define TX_ID_DWORD0                    (PORT_BASE + 0x9c)
 229 #define TX_ID_DWORD1                    (PORT_BASE + 0xa0)
 230 #define TX_ID_DWORD2                    (PORT_BASE + 0xa4)
 231 #define TX_ID_DWORD3                    (PORT_BASE + 0xa8)
 232 #define TX_ID_DWORD4                    (PORT_BASE + 0xaC)
 233 #define TX_ID_DWORD5                    (PORT_BASE + 0xb0)
 234 #define TX_ID_DWORD6                    (PORT_BASE + 0xb4)
 235 #define TXID_AUTO                               (PORT_BASE + 0xb8)
 236 #define CT3_OFF         1
 237 #define CT3_MSK         (0x1 << CT3_OFF)
 238 #define TX_HARDRST_OFF          2
 239 #define TX_HARDRST_MSK          (0x1 << TX_HARDRST_OFF)
 240 #define RX_IDAF_DWORD0                  (PORT_BASE + 0xc4)
 241 #define RXOP_CHECK_CFG_H                (PORT_BASE + 0xfc)
 242 #define STP_LINK_TIMER                  (PORT_BASE + 0x120)
 243 #define STP_LINK_TIMEOUT_STATE          (PORT_BASE + 0x124)
 244 #define CON_CFG_DRIVER                  (PORT_BASE + 0x130)
 245 #define SAS_SSP_CON_TIMER_CFG           (PORT_BASE + 0x134)
 246 #define SAS_SMP_CON_TIMER_CFG           (PORT_BASE + 0x138)
 247 #define SAS_STP_CON_TIMER_CFG           (PORT_BASE + 0x13c)
 248 #define CHL_INT0                        (PORT_BASE + 0x1b4)
 249 #define CHL_INT0_HOTPLUG_TOUT_OFF       0
 250 #define CHL_INT0_HOTPLUG_TOUT_MSK       (0x1 << CHL_INT0_HOTPLUG_TOUT_OFF)
 251 #define CHL_INT0_SL_RX_BCST_ACK_OFF     1
 252 #define CHL_INT0_SL_RX_BCST_ACK_MSK     (0x1 << CHL_INT0_SL_RX_BCST_ACK_OFF)
 253 #define CHL_INT0_SL_PHY_ENABLE_OFF      2
 254 #define CHL_INT0_SL_PHY_ENABLE_MSK      (0x1 << CHL_INT0_SL_PHY_ENABLE_OFF)
 255 #define CHL_INT0_NOT_RDY_OFF            4
 256 #define CHL_INT0_NOT_RDY_MSK            (0x1 << CHL_INT0_NOT_RDY_OFF)
 257 #define CHL_INT0_PHY_RDY_OFF            5
 258 #define CHL_INT0_PHY_RDY_MSK            (0x1 << CHL_INT0_PHY_RDY_OFF)
 259 #define CHL_INT1                        (PORT_BASE + 0x1b8)
 260 #define CHL_INT1_DMAC_TX_ECC_MB_ERR_OFF 15
 261 #define CHL_INT1_DMAC_TX_ECC_1B_ERR_OFF 16
 262 #define CHL_INT1_DMAC_RX_ECC_MB_ERR_OFF 17
 263 #define CHL_INT1_DMAC_RX_ECC_1B_ERR_OFF 18
 264 #define CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF 19
 265 #define CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF 20
 266 #define CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF 21
 267 #define CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF 22
 268 #define CHL_INT1_DMAC_TX_FIFO_ERR_OFF   23
 269 #define CHL_INT1_DMAC_RX_FIFO_ERR_OFF   24
 270 #define CHL_INT1_DMAC_TX_AXI_RUSER_ERR_OFF      26
 271 #define CHL_INT1_DMAC_RX_AXI_RUSER_ERR_OFF      27
 272 #define CHL_INT2                        (PORT_BASE + 0x1bc)
 273 #define CHL_INT2_SL_IDAF_TOUT_CONF_OFF  0
 274 #define CHL_INT2_RX_DISP_ERR_OFF        28
 275 #define CHL_INT2_RX_CODE_ERR_OFF        29
 276 #define CHL_INT2_RX_INVLD_DW_OFF        30
 277 #define CHL_INT2_STP_LINK_TIMEOUT_OFF   31
 278 #define CHL_INT0_MSK                    (PORT_BASE + 0x1c0)
 279 #define CHL_INT1_MSK                    (PORT_BASE + 0x1c4)
 280 #define CHL_INT2_MSK                    (PORT_BASE + 0x1c8)
 281 #define SAS_EC_INT_COAL_TIME            (PORT_BASE + 0x1cc)
 282 #define CHL_INT_COAL_EN                 (PORT_BASE + 0x1d0)
 283 #define SAS_RX_TRAIN_TIMER              (PORT_BASE + 0x2a4)
 284 #define PHY_CTRL_RDY_MSK                (PORT_BASE + 0x2b0)
 285 #define PHYCTRL_NOT_RDY_MSK             (PORT_BASE + 0x2b4)
 286 #define PHYCTRL_DWS_RESET_MSK           (PORT_BASE + 0x2b8)
 287 #define PHYCTRL_PHY_ENA_MSK             (PORT_BASE + 0x2bc)
 288 #define SL_RX_BCAST_CHK_MSK             (PORT_BASE + 0x2c0)
 289 #define PHYCTRL_OOB_RESTART_MSK         (PORT_BASE + 0x2c4)
 290 #define DMA_TX_STATUS                   (PORT_BASE + 0x2d0)
 291 #define DMA_TX_STATUS_BUSY_OFF          0
 292 #define DMA_TX_STATUS_BUSY_MSK          (0x1 << DMA_TX_STATUS_BUSY_OFF)
 293 #define DMA_RX_STATUS                   (PORT_BASE + 0x2e8)
 294 #define DMA_RX_STATUS_BUSY_OFF          0
 295 #define DMA_RX_STATUS_BUSY_MSK          (0x1 << DMA_RX_STATUS_BUSY_OFF)
 296 
 297 #define COARSETUNE_TIME                 (PORT_BASE + 0x304)
 298 #define ERR_CNT_DWS_LOST                (PORT_BASE + 0x380)
 299 #define ERR_CNT_RESET_PROB              (PORT_BASE + 0x384)
 300 #define ERR_CNT_INVLD_DW                (PORT_BASE + 0x390)
 301 #define ERR_CNT_CODE_ERR                (PORT_BASE + 0x394)
 302 #define ERR_CNT_DISP_ERR                (PORT_BASE + 0x398)
 303 
 304 #define DEFAULT_ITCT_HW         2048 /* reset value, not reprogrammed */
 305 #if (HISI_SAS_MAX_DEVICES > DEFAULT_ITCT_HW)
 306 #error Max ITCT exceeded
 307 #endif
 308 
 309 #define AXI_MASTER_CFG_BASE             (0x5000)
 310 #define AM_CTRL_GLOBAL                  (0x0)
 311 #define AM_CTRL_SHUTDOWN_REQ_OFF        0
 312 #define AM_CTRL_SHUTDOWN_REQ_MSK        (0x1 << AM_CTRL_SHUTDOWN_REQ_OFF)
 313 #define AM_CURR_TRANS_RETURN    (0x150)
 314 
 315 #define AM_CFG_MAX_TRANS                (0x5010)
 316 #define AM_CFG_SINGLE_PORT_MAX_TRANS    (0x5014)
 317 #define AXI_CFG                                 (0x5100)
 318 #define AM_ROB_ECC_ERR_ADDR             (0x510c)
 319 #define AM_ROB_ECC_ERR_ADDR_OFF 0
 320 #define AM_ROB_ECC_ERR_ADDR_MSK 0xffffffff
 321 
 322 /* RAS registers need init */
 323 #define RAS_BASE                (0x6000)
 324 #define SAS_RAS_INTR0                   (RAS_BASE)
 325 #define SAS_RAS_INTR1                   (RAS_BASE + 0x04)
 326 #define SAS_RAS_INTR0_MASK              (RAS_BASE + 0x08)
 327 #define SAS_RAS_INTR1_MASK              (RAS_BASE + 0x0c)
 328 #define CFG_SAS_RAS_INTR_MASK           (RAS_BASE + 0x1c)
 329 #define SAS_RAS_INTR2                   (RAS_BASE + 0x20)
 330 #define SAS_RAS_INTR2_MASK              (RAS_BASE + 0x24)
 331 
 332 /* HW dma structures */
 333 /* Delivery queue header */
 334 /* dw0 */
 335 #define CMD_HDR_ABORT_FLAG_OFF          0
 336 #define CMD_HDR_ABORT_FLAG_MSK          (0x3 << CMD_HDR_ABORT_FLAG_OFF)
 337 #define CMD_HDR_ABORT_DEVICE_TYPE_OFF   2
 338 #define CMD_HDR_ABORT_DEVICE_TYPE_MSK   (0x1 << CMD_HDR_ABORT_DEVICE_TYPE_OFF)
 339 #define CMD_HDR_RESP_REPORT_OFF         5
 340 #define CMD_HDR_RESP_REPORT_MSK         (0x1 << CMD_HDR_RESP_REPORT_OFF)
 341 #define CMD_HDR_TLR_CTRL_OFF            6
 342 #define CMD_HDR_TLR_CTRL_MSK            (0x3 << CMD_HDR_TLR_CTRL_OFF)
 343 #define CMD_HDR_PORT_OFF                18
 344 #define CMD_HDR_PORT_MSK                (0xf << CMD_HDR_PORT_OFF)
 345 #define CMD_HDR_PRIORITY_OFF            27
 346 #define CMD_HDR_PRIORITY_MSK            (0x1 << CMD_HDR_PRIORITY_OFF)
 347 #define CMD_HDR_CMD_OFF                 29
 348 #define CMD_HDR_CMD_MSK                 (0x7 << CMD_HDR_CMD_OFF)
 349 /* dw1 */
 350 #define CMD_HDR_UNCON_CMD_OFF   3
 351 #define CMD_HDR_DIR_OFF                 5
 352 #define CMD_HDR_DIR_MSK                 (0x3 << CMD_HDR_DIR_OFF)
 353 #define CMD_HDR_RESET_OFF               7
 354 #define CMD_HDR_RESET_MSK               (0x1 << CMD_HDR_RESET_OFF)
 355 #define CMD_HDR_VDTL_OFF                10
 356 #define CMD_HDR_VDTL_MSK                (0x1 << CMD_HDR_VDTL_OFF)
 357 #define CMD_HDR_FRAME_TYPE_OFF          11
 358 #define CMD_HDR_FRAME_TYPE_MSK          (0x1f << CMD_HDR_FRAME_TYPE_OFF)
 359 #define CMD_HDR_DEV_ID_OFF              16
 360 #define CMD_HDR_DEV_ID_MSK              (0xffff << CMD_HDR_DEV_ID_OFF)
 361 /* dw2 */
 362 #define CMD_HDR_CFL_OFF                 0
 363 #define CMD_HDR_CFL_MSK                 (0x1ff << CMD_HDR_CFL_OFF)
 364 #define CMD_HDR_NCQ_TAG_OFF             10
 365 #define CMD_HDR_NCQ_TAG_MSK             (0x1f << CMD_HDR_NCQ_TAG_OFF)
 366 #define CMD_HDR_MRFL_OFF                15
 367 #define CMD_HDR_MRFL_MSK                (0x1ff << CMD_HDR_MRFL_OFF)
 368 #define CMD_HDR_SG_MOD_OFF              24
 369 #define CMD_HDR_SG_MOD_MSK              (0x3 << CMD_HDR_SG_MOD_OFF)
 370 /* dw3 */
 371 #define CMD_HDR_IPTT_OFF                0
 372 #define CMD_HDR_IPTT_MSK                (0xffff << CMD_HDR_IPTT_OFF)
 373 /* dw6 */
 374 #define CMD_HDR_DIF_SGL_LEN_OFF         0
 375 #define CMD_HDR_DIF_SGL_LEN_MSK         (0xffff << CMD_HDR_DIF_SGL_LEN_OFF)
 376 #define CMD_HDR_DATA_SGL_LEN_OFF        16
 377 #define CMD_HDR_DATA_SGL_LEN_MSK        (0xffff << CMD_HDR_DATA_SGL_LEN_OFF)
 378 /* dw7 */
 379 #define CMD_HDR_ADDR_MODE_SEL_OFF               15
 380 #define CMD_HDR_ADDR_MODE_SEL_MSK               (1 << CMD_HDR_ADDR_MODE_SEL_OFF)
 381 #define CMD_HDR_ABORT_IPTT_OFF          16
 382 #define CMD_HDR_ABORT_IPTT_MSK          (0xffff << CMD_HDR_ABORT_IPTT_OFF)
 383 
 384 /* Completion header */
 385 /* dw0 */
 386 #define CMPLT_HDR_CMPLT_OFF             0
 387 #define CMPLT_HDR_CMPLT_MSK             (0x3 << CMPLT_HDR_CMPLT_OFF)
 388 #define CMPLT_HDR_ERROR_PHASE_OFF   2
 389 #define CMPLT_HDR_ERROR_PHASE_MSK   (0xff << CMPLT_HDR_ERROR_PHASE_OFF)
 390 #define CMPLT_HDR_RSPNS_XFRD_OFF        10
 391 #define CMPLT_HDR_RSPNS_XFRD_MSK        (0x1 << CMPLT_HDR_RSPNS_XFRD_OFF)
 392 #define CMPLT_HDR_ERX_OFF               12
 393 #define CMPLT_HDR_ERX_MSK               (0x1 << CMPLT_HDR_ERX_OFF)
 394 #define CMPLT_HDR_ABORT_STAT_OFF        13
 395 #define CMPLT_HDR_ABORT_STAT_MSK        (0x7 << CMPLT_HDR_ABORT_STAT_OFF)
 396 /* abort_stat */
 397 #define STAT_IO_NOT_VALID               0x1
 398 #define STAT_IO_NO_DEVICE               0x2
 399 #define STAT_IO_COMPLETE                0x3
 400 #define STAT_IO_ABORTED                 0x4
 401 /* dw1 */
 402 #define CMPLT_HDR_IPTT_OFF              0
 403 #define CMPLT_HDR_IPTT_MSK              (0xffff << CMPLT_HDR_IPTT_OFF)
 404 #define CMPLT_HDR_DEV_ID_OFF            16
 405 #define CMPLT_HDR_DEV_ID_MSK            (0xffff << CMPLT_HDR_DEV_ID_OFF)
 406 /* dw3 */
 407 #define CMPLT_HDR_IO_IN_TARGET_OFF      17
 408 #define CMPLT_HDR_IO_IN_TARGET_MSK      (0x1 << CMPLT_HDR_IO_IN_TARGET_OFF)
 409 
 410 /* ITCT header */
 411 /* qw0 */
 412 #define ITCT_HDR_DEV_TYPE_OFF           0
 413 #define ITCT_HDR_DEV_TYPE_MSK           (0x3 << ITCT_HDR_DEV_TYPE_OFF)
 414 #define ITCT_HDR_VALID_OFF              2
 415 #define ITCT_HDR_VALID_MSK              (0x1 << ITCT_HDR_VALID_OFF)
 416 #define ITCT_HDR_MCR_OFF                5
 417 #define ITCT_HDR_MCR_MSK                (0xf << ITCT_HDR_MCR_OFF)
 418 #define ITCT_HDR_VLN_OFF                9
 419 #define ITCT_HDR_VLN_MSK                (0xf << ITCT_HDR_VLN_OFF)
 420 #define ITCT_HDR_SMP_TIMEOUT_OFF        16
 421 #define ITCT_HDR_AWT_CONTINUE_OFF       25
 422 #define ITCT_HDR_PORT_ID_OFF            28
 423 #define ITCT_HDR_PORT_ID_MSK            (0xf << ITCT_HDR_PORT_ID_OFF)
 424 /* qw2 */
 425 #define ITCT_HDR_INLT_OFF               0
 426 #define ITCT_HDR_INLT_MSK               (0xffffULL << ITCT_HDR_INLT_OFF)
 427 #define ITCT_HDR_RTOLT_OFF              48
 428 #define ITCT_HDR_RTOLT_MSK              (0xffffULL << ITCT_HDR_RTOLT_OFF)
 429 
 430 struct hisi_sas_protect_iu_v3_hw {
 431         u32 dw0;
 432         u32 lbrtcv;
 433         u32 lbrtgv;
 434         u32 dw3;
 435         u32 dw4;
 436         u32 dw5;
 437         u32 rsv;
 438 };
 439 
 440 struct hisi_sas_complete_v3_hdr {
 441         __le32 dw0;
 442         __le32 dw1;
 443         __le32 act;
 444         __le32 dw3;
 445 };
 446 
 447 struct hisi_sas_err_record_v3 {
 448         /* dw0 */
 449         __le32 trans_tx_fail_type;
 450 
 451         /* dw1 */
 452         __le32 trans_rx_fail_type;
 453 
 454         /* dw2 */
 455         __le16 dma_tx_err_type;
 456         __le16 sipc_rx_err_type;
 457 
 458         /* dw3 */
 459         __le32 dma_rx_err_type;
 460 };
 461 
 462 #define RX_DATA_LEN_UNDERFLOW_OFF       6
 463 #define RX_DATA_LEN_UNDERFLOW_MSK       (1 << RX_DATA_LEN_UNDERFLOW_OFF)
 464 
 465 #define HISI_SAS_COMMAND_ENTRIES_V3_HW 4096
 466 #define HISI_SAS_MSI_COUNT_V3_HW 32
 467 
 468 #define DIR_NO_DATA 0
 469 #define DIR_TO_INI 1
 470 #define DIR_TO_DEVICE 2
 471 #define DIR_RESERVED 3
 472 
 473 #define FIS_CMD_IS_UNCONSTRAINED(fis) \
 474         ((fis.command == ATA_CMD_READ_LOG_EXT) || \
 475         (fis.command == ATA_CMD_READ_LOG_DMA_EXT) || \
 476         ((fis.command == ATA_CMD_DEV_RESET) && \
 477         ((fis.control & ATA_SRST) != 0)))
 478 
 479 #define T10_INSRT_EN_OFF    0
 480 #define T10_INSRT_EN_MSK    (1 << T10_INSRT_EN_OFF)
 481 #define T10_RMV_EN_OFF      1
 482 #define T10_RMV_EN_MSK      (1 << T10_RMV_EN_OFF)
 483 #define T10_RPLC_EN_OFF     2
 484 #define T10_RPLC_EN_MSK     (1 << T10_RPLC_EN_OFF)
 485 #define T10_CHK_EN_OFF      3
 486 #define T10_CHK_EN_MSK      (1 << T10_CHK_EN_OFF)
 487 #define INCR_LBRT_OFF       5
 488 #define INCR_LBRT_MSK       (1 << INCR_LBRT_OFF)
 489 #define USR_DATA_BLOCK_SZ_OFF   20
 490 #define USR_DATA_BLOCK_SZ_MSK   (0x3 << USR_DATA_BLOCK_SZ_OFF)
 491 #define T10_CHK_MSK_OFF     16
 492 #define T10_CHK_REF_TAG_MSK (0xf0 << T10_CHK_MSK_OFF)
 493 #define T10_CHK_APP_TAG_MSK (0xc << T10_CHK_MSK_OFF)
 494 
 495 #define BASE_VECTORS_V3_HW  16
 496 #define MIN_AFFINE_VECTORS_V3_HW  (BASE_VECTORS_V3_HW + 1)
 497 
 498 enum {
 499         DSM_FUNC_ERR_HANDLE_MSI = 0,
 500 };
 501 
 502 static bool hisi_sas_intr_conv;
 503 MODULE_PARM_DESC(intr_conv, "interrupt converge enable (0-1)");
 504 
 505 /* permit overriding the host protection capabilities mask (EEDP/T10 PI) */
 506 static int prot_mask;
 507 module_param(prot_mask, int, 0);
 508 MODULE_PARM_DESC(prot_mask, " host protection capabilities mask, def=0x0 ");
 509 
 510 static bool auto_affine_msi_experimental;
 511 module_param(auto_affine_msi_experimental, bool, 0444);
 512 MODULE_PARM_DESC(auto_affine_msi_experimental, "Enable auto-affinity of MSI IRQs as experimental:\n"
 513                  "default is off");
 514 
 515 static u32 hisi_sas_read32(struct hisi_hba *hisi_hba, u32 off)
 516 {
 517         void __iomem *regs = hisi_hba->regs + off;
 518 
 519         return readl(regs);
 520 }
 521 
 522 static void hisi_sas_write32(struct hisi_hba *hisi_hba, u32 off, u32 val)
 523 {
 524         void __iomem *regs = hisi_hba->regs + off;
 525 
 526         writel(val, regs);
 527 }
 528 
 529 static void hisi_sas_phy_write32(struct hisi_hba *hisi_hba, int phy_no,
 530                                  u32 off, u32 val)
 531 {
 532         void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
 533 
 534         writel(val, regs);
 535 }
 536 
 537 static u32 hisi_sas_phy_read32(struct hisi_hba *hisi_hba,
 538                                       int phy_no, u32 off)
 539 {
 540         void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
 541 
 542         return readl(regs);
 543 }
 544 
 545 #define hisi_sas_read32_poll_timeout(off, val, cond, delay_us,          \
 546                                      timeout_us)                        \
 547 ({                                                                      \
 548         void __iomem *regs = hisi_hba->regs + off;                      \
 549         readl_poll_timeout(regs, val, cond, delay_us, timeout_us);      \
 550 })
 551 
 552 #define hisi_sas_read32_poll_timeout_atomic(off, val, cond, delay_us,   \
 553                                             timeout_us)                 \
 554 ({                                                                      \
 555         void __iomem *regs = hisi_hba->regs + off;                      \
 556         readl_poll_timeout_atomic(regs, val, cond, delay_us, timeout_us);\
 557 })
 558 
 559 static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 560 {
 561         int i;
 562 
 563         /* Global registers init */
 564         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE,
 565                          (u32)((1ULL << hisi_hba->queue_count) - 1));
 566         hisi_sas_write32(hisi_hba, SAS_AXI_USER3, 0);
 567         hisi_sas_write32(hisi_hba, CFG_MAX_TAG, 0xfff0400);
 568         hisi_sas_write32(hisi_hba, HGC_SAS_TXFAIL_RETRY_CTRL, 0x108);
 569         hisi_sas_write32(hisi_hba, CFG_AGING_TIME, 0x1);
 570         hisi_sas_write32(hisi_hba, INT_COAL_EN, 0x1);
 571         hisi_sas_write32(hisi_hba, OQ_INT_COAL_TIME, 0x1);
 572         hisi_sas_write32(hisi_hba, OQ_INT_COAL_CNT, 0x1);
 573         hisi_sas_write32(hisi_hba, CQ_INT_CONVERGE_EN,
 574                          hisi_sas_intr_conv);
 575         hisi_sas_write32(hisi_hba, OQ_INT_SRC, 0xffff);
 576         hisi_sas_write32(hisi_hba, ENT_INT_SRC1, 0xffffffff);
 577         hisi_sas_write32(hisi_hba, ENT_INT_SRC2, 0xffffffff);
 578         hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 0xffffffff);
 579         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xfefefefe);
 580         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xfefefefe);
 581         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffc220ff);
 582         hisi_sas_write32(hisi_hba, CHNL_PHYUPDOWN_INT_MSK, 0x0);
 583         hisi_sas_write32(hisi_hba, CHNL_ENT_INT_MSK, 0x0);
 584         hisi_sas_write32(hisi_hba, HGC_COM_INT_MSK, 0x0);
 585         hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0x155555);
 586         hisi_sas_write32(hisi_hba, AWQOS_AWCACHE_CFG, 0xf0f0);
 587         hisi_sas_write32(hisi_hba, ARQOS_ARCACHE_CFG, 0xf0f0);
 588         for (i = 0; i < hisi_hba->queue_count; i++)
 589                 hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK+0x4*i, 0);
 590 
 591         hisi_sas_write32(hisi_hba, HYPER_STREAM_ID_EN_CFG, 1);
 592 
 593         for (i = 0; i < hisi_hba->n_phy; i++) {
 594                 struct hisi_sas_phy *phy = &hisi_hba->phy[i];
 595                 struct asd_sas_phy *sas_phy = &phy->sas_phy;
 596                 u32 prog_phy_link_rate = 0x800;
 597 
 598                 if (!sas_phy->phy || (sas_phy->phy->maximum_linkrate <
 599                                 SAS_LINK_RATE_1_5_GBPS)) {
 600                         prog_phy_link_rate = 0x855;
 601                 } else {
 602                         enum sas_linkrate max = sas_phy->phy->maximum_linkrate;
 603 
 604                         prog_phy_link_rate =
 605                                 hisi_sas_get_prog_phy_linkrate_mask(max) |
 606                                 0x800;
 607                 }
 608                 hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE,
 609                         prog_phy_link_rate);
 610                 hisi_sas_phy_write32(hisi_hba, i, SERDES_CFG, 0xffc00);
 611                 hisi_sas_phy_write32(hisi_hba, i, SAS_RX_TRAIN_TIMER, 0x13e80);
 612                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT0, 0xffffffff);
 613                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff);
 614                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xffffffff);
 615                 hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000);
 616                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xf2057fff);
 617                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffbfe);
 618                 hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL_RDY_MSK, 0x0);
 619                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0);
 620                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_DWS_RESET_MSK, 0x0);
 621                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x0);
 622                 hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x0);
 623                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_OOB_RESTART_MSK, 0x1);
 624                 hisi_sas_phy_write32(hisi_hba, i, STP_LINK_TIMER, 0x7f7a120);
 625                 hisi_sas_phy_write32(hisi_hba, i, CON_CFG_DRIVER, 0x2a0a01);
 626                 hisi_sas_phy_write32(hisi_hba, i, SAS_SSP_CON_TIMER_CFG, 0x32);
 627                 hisi_sas_phy_write32(hisi_hba, i, SAS_EC_INT_COAL_TIME,
 628                                      0x30f4240);
 629                 /* used for 12G negotiate */
 630                 hisi_sas_phy_write32(hisi_hba, i, COARSETUNE_TIME, 0x1e);
 631                 hisi_sas_phy_write32(hisi_hba, i, AIP_LIMIT, 0x2ffff);
 632         }
 633 
 634         for (i = 0; i < hisi_hba->queue_count; i++) {
 635                 /* Delivery queue */
 636                 hisi_sas_write32(hisi_hba,
 637                                  DLVRY_Q_0_BASE_ADDR_HI + (i * 0x14),
 638                                  upper_32_bits(hisi_hba->cmd_hdr_dma[i]));
 639 
 640                 hisi_sas_write32(hisi_hba, DLVRY_Q_0_BASE_ADDR_LO + (i * 0x14),
 641                                  lower_32_bits(hisi_hba->cmd_hdr_dma[i]));
 642 
 643                 hisi_sas_write32(hisi_hba, DLVRY_Q_0_DEPTH + (i * 0x14),
 644                                  HISI_SAS_QUEUE_SLOTS);
 645 
 646                 /* Completion queue */
 647                 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_HI + (i * 0x14),
 648                                  upper_32_bits(hisi_hba->complete_hdr_dma[i]));
 649 
 650                 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_LO + (i * 0x14),
 651                                  lower_32_bits(hisi_hba->complete_hdr_dma[i]));
 652 
 653                 hisi_sas_write32(hisi_hba, COMPL_Q_0_DEPTH + (i * 0x14),
 654                                  HISI_SAS_QUEUE_SLOTS);
 655         }
 656 
 657         /* itct */
 658         hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_LO,
 659                          lower_32_bits(hisi_hba->itct_dma));
 660 
 661         hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_HI,
 662                          upper_32_bits(hisi_hba->itct_dma));
 663 
 664         /* iost */
 665         hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_LO,
 666                          lower_32_bits(hisi_hba->iost_dma));
 667 
 668         hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_HI,
 669                          upper_32_bits(hisi_hba->iost_dma));
 670 
 671         /* breakpoint */
 672         hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_LO,
 673                          lower_32_bits(hisi_hba->breakpoint_dma));
 674 
 675         hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_HI,
 676                          upper_32_bits(hisi_hba->breakpoint_dma));
 677 
 678         /* SATA broken msg */
 679         hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_LO,
 680                          lower_32_bits(hisi_hba->sata_breakpoint_dma));
 681 
 682         hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_HI,
 683                          upper_32_bits(hisi_hba->sata_breakpoint_dma));
 684 
 685         /* SATA initial fis */
 686         hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_LO,
 687                          lower_32_bits(hisi_hba->initial_fis_dma));
 688 
 689         hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_HI,
 690                          upper_32_bits(hisi_hba->initial_fis_dma));
 691 
 692         /* RAS registers init */
 693         hisi_sas_write32(hisi_hba, SAS_RAS_INTR0_MASK, 0x0);
 694         hisi_sas_write32(hisi_hba, SAS_RAS_INTR1_MASK, 0x0);
 695         hisi_sas_write32(hisi_hba, SAS_RAS_INTR2_MASK, 0x0);
 696         hisi_sas_write32(hisi_hba, CFG_SAS_RAS_INTR_MASK, 0x0);
 697 
 698         /* LED registers init */
 699         hisi_sas_write32(hisi_hba, SAS_CFG_DRIVE_VLD, 0x80000ff);
 700         hisi_sas_write32(hisi_hba, SAS_GPIO_TX_0_1, 0x80808080);
 701         hisi_sas_write32(hisi_hba, SAS_GPIO_TX_0_1 + 0x4, 0x80808080);
 702         /* Configure blink generator rate A to 1Hz and B to 4Hz */
 703         hisi_sas_write32(hisi_hba, SAS_GPIO_CFG_1, 0x121700);
 704         hisi_sas_write32(hisi_hba, SAS_GPIO_CFG_0, 0x800000);
 705 }
 706 
 707 static void config_phy_opt_mode_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 708 {
 709         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 710 
 711         cfg &= ~PHY_CFG_DC_OPT_MSK;
 712         cfg |= 1 << PHY_CFG_DC_OPT_OFF;
 713         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 714 }
 715 
 716 static void config_id_frame_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 717 {
 718         struct sas_identify_frame identify_frame;
 719         u32 *identify_buffer;
 720 
 721         memset(&identify_frame, 0, sizeof(identify_frame));
 722         identify_frame.dev_type = SAS_END_DEVICE;
 723         identify_frame.frame_type = 0;
 724         identify_frame._un1 = 1;
 725         identify_frame.initiator_bits = SAS_PROTOCOL_ALL;
 726         identify_frame.target_bits = SAS_PROTOCOL_NONE;
 727         memcpy(&identify_frame._un4_11[0], hisi_hba->sas_addr, SAS_ADDR_SIZE);
 728         memcpy(&identify_frame.sas_addr[0], hisi_hba->sas_addr, SAS_ADDR_SIZE);
 729         identify_frame.phy_id = phy_no;
 730         identify_buffer = (u32 *)(&identify_frame);
 731 
 732         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD0,
 733                         __swab32(identify_buffer[0]));
 734         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD1,
 735                         __swab32(identify_buffer[1]));
 736         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD2,
 737                         __swab32(identify_buffer[2]));
 738         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD3,
 739                         __swab32(identify_buffer[3]));
 740         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD4,
 741                         __swab32(identify_buffer[4]));
 742         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD5,
 743                         __swab32(identify_buffer[5]));
 744 }
 745 
 746 static void setup_itct_v3_hw(struct hisi_hba *hisi_hba,
 747                              struct hisi_sas_device *sas_dev)
 748 {
 749         struct domain_device *device = sas_dev->sas_device;
 750         struct device *dev = hisi_hba->dev;
 751         u64 qw0, device_id = sas_dev->device_id;
 752         struct hisi_sas_itct *itct = &hisi_hba->itct[device_id];
 753         struct domain_device *parent_dev = device->parent;
 754         struct asd_sas_port *sas_port = device->port;
 755         struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
 756         u64 sas_addr;
 757 
 758         memset(itct, 0, sizeof(*itct));
 759 
 760         /* qw0 */
 761         qw0 = 0;
 762         switch (sas_dev->dev_type) {
 763         case SAS_END_DEVICE:
 764         case SAS_EDGE_EXPANDER_DEVICE:
 765         case SAS_FANOUT_EXPANDER_DEVICE:
 766                 qw0 = HISI_SAS_DEV_TYPE_SSP << ITCT_HDR_DEV_TYPE_OFF;
 767                 break;
 768         case SAS_SATA_DEV:
 769         case SAS_SATA_PENDING:
 770                 if (parent_dev && dev_is_expander(parent_dev->dev_type))
 771                         qw0 = HISI_SAS_DEV_TYPE_STP << ITCT_HDR_DEV_TYPE_OFF;
 772                 else
 773                         qw0 = HISI_SAS_DEV_TYPE_SATA << ITCT_HDR_DEV_TYPE_OFF;
 774                 break;
 775         default:
 776                 dev_warn(dev, "setup itct: unsupported dev type (%d)\n",
 777                          sas_dev->dev_type);
 778         }
 779 
 780         qw0 |= ((1 << ITCT_HDR_VALID_OFF) |
 781                 (device->linkrate << ITCT_HDR_MCR_OFF) |
 782                 (1 << ITCT_HDR_VLN_OFF) |
 783                 (0xfa << ITCT_HDR_SMP_TIMEOUT_OFF) |
 784                 (1 << ITCT_HDR_AWT_CONTINUE_OFF) |
 785                 (port->id << ITCT_HDR_PORT_ID_OFF));
 786         itct->qw0 = cpu_to_le64(qw0);
 787 
 788         /* qw1 */
 789         memcpy(&sas_addr, device->sas_addr, SAS_ADDR_SIZE);
 790         itct->sas_addr = cpu_to_le64(__swab64(sas_addr));
 791 
 792         /* qw2 */
 793         if (!dev_is_sata(device))
 794                 itct->qw2 = cpu_to_le64((5000ULL << ITCT_HDR_INLT_OFF) |
 795                                         (0x1ULL << ITCT_HDR_RTOLT_OFF));
 796 }
 797 
 798 static void clear_itct_v3_hw(struct hisi_hba *hisi_hba,
 799                               struct hisi_sas_device *sas_dev)
 800 {
 801         DECLARE_COMPLETION_ONSTACK(completion);
 802         u64 dev_id = sas_dev->device_id;
 803         struct hisi_sas_itct *itct = &hisi_hba->itct[dev_id];
 804         u32 reg_val = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
 805 
 806         sas_dev->completion = &completion;
 807 
 808         /* clear the itct interrupt state */
 809         if (ENT_INT_SRC3_ITC_INT_MSK & reg_val)
 810                 hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
 811                                  ENT_INT_SRC3_ITC_INT_MSK);
 812 
 813         /* clear the itct table */
 814         reg_val = ITCT_CLR_EN_MSK | (dev_id & ITCT_DEV_MSK);
 815         hisi_sas_write32(hisi_hba, ITCT_CLR, reg_val);
 816 
 817         wait_for_completion(sas_dev->completion);
 818         memset(itct, 0, sizeof(struct hisi_sas_itct));
 819 }
 820 
 821 static void dereg_device_v3_hw(struct hisi_hba *hisi_hba,
 822                                 struct domain_device *device)
 823 {
 824         struct hisi_sas_slot *slot, *slot2;
 825         struct hisi_sas_device *sas_dev = device->lldd_dev;
 826         u32 cfg_abt_set_query_iptt;
 827 
 828         cfg_abt_set_query_iptt = hisi_sas_read32(hisi_hba,
 829                 CFG_ABT_SET_QUERY_IPTT);
 830         list_for_each_entry_safe(slot, slot2, &sas_dev->list, entry) {
 831                 cfg_abt_set_query_iptt &= ~CFG_SET_ABORTED_IPTT_MSK;
 832                 cfg_abt_set_query_iptt |= (1 << CFG_SET_ABORTED_EN_OFF) |
 833                         (slot->idx << CFG_SET_ABORTED_IPTT_OFF);
 834                 hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
 835                         cfg_abt_set_query_iptt);
 836         }
 837         cfg_abt_set_query_iptt &= ~(1 << CFG_SET_ABORTED_EN_OFF);
 838         hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
 839                 cfg_abt_set_query_iptt);
 840         hisi_sas_write32(hisi_hba, CFG_ABT_SET_IPTT_DONE,
 841                                         1 << CFG_ABT_SET_IPTT_DONE_OFF);
 842 }
 843 
 844 static int reset_hw_v3_hw(struct hisi_hba *hisi_hba)
 845 {
 846         struct device *dev = hisi_hba->dev;
 847         int ret;
 848         u32 val;
 849 
 850         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0);
 851 
 852         /* Disable all of the PHYs */
 853         hisi_sas_stop_phys(hisi_hba);
 854         udelay(50);
 855 
 856         /* Ensure axi bus idle */
 857         ret = hisi_sas_read32_poll_timeout(AXI_CFG, val, !val,
 858                                            20000, 1000000);
 859         if (ret) {
 860                 dev_err(dev, "axi bus is not idle, ret = %d!\n", ret);
 861                 return -EIO;
 862         }
 863 
 864         if (ACPI_HANDLE(dev)) {
 865                 acpi_status s;
 866 
 867                 s = acpi_evaluate_object(ACPI_HANDLE(dev), "_RST", NULL, NULL);
 868                 if (ACPI_FAILURE(s)) {
 869                         dev_err(dev, "Reset failed\n");
 870                         return -EIO;
 871                 }
 872         } else {
 873                 dev_err(dev, "no reset method!\n");
 874                 return -EINVAL;
 875         }
 876 
 877         return 0;
 878 }
 879 
 880 static int hw_init_v3_hw(struct hisi_hba *hisi_hba)
 881 {
 882         struct device *dev = hisi_hba->dev;
 883         union acpi_object *obj;
 884         guid_t guid;
 885         int rc;
 886 
 887         rc = reset_hw_v3_hw(hisi_hba);
 888         if (rc) {
 889                 dev_err(dev, "hisi_sas_reset_hw failed, rc=%d", rc);
 890                 return rc;
 891         }
 892 
 893         msleep(100);
 894         init_reg_v3_hw(hisi_hba);
 895 
 896         if (guid_parse("D5918B4B-37AE-4E10-A99F-E5E8A6EF4C1F", &guid)) {
 897                 dev_err(dev, "Parse GUID failed\n");
 898                 return -EINVAL;
 899         }
 900 
 901         /* Switch over to MSI handling , from PCI AER default */
 902         obj = acpi_evaluate_dsm(ACPI_HANDLE(dev), &guid, 0,
 903                                 DSM_FUNC_ERR_HANDLE_MSI, NULL);
 904         if (!obj)
 905                 dev_warn(dev, "Switch over to MSI handling failed\n");
 906         else
 907                 ACPI_FREE(obj);
 908 
 909         return 0;
 910 }
 911 
 912 static void enable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 913 {
 914         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 915 
 916         cfg |= PHY_CFG_ENA_MSK;
 917         cfg &= ~PHY_CFG_PHY_RST_MSK;
 918         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 919 }
 920 
 921 static void disable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 922 {
 923         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 924         u32 irq_msk = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2_MSK);
 925         static const u32 msk = BIT(CHL_INT2_RX_DISP_ERR_OFF) |
 926                                BIT(CHL_INT2_RX_CODE_ERR_OFF) |
 927                                BIT(CHL_INT2_RX_INVLD_DW_OFF);
 928         u32 state;
 929 
 930         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2_MSK, msk | irq_msk);
 931 
 932         cfg &= ~PHY_CFG_ENA_MSK;
 933         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 934 
 935         mdelay(50);
 936 
 937         state = hisi_sas_read32(hisi_hba, PHY_STATE);
 938         if (state & BIT(phy_no)) {
 939                 cfg |= PHY_CFG_PHY_RST_MSK;
 940                 hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 941         }
 942 
 943         udelay(1);
 944 
 945         hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_INVLD_DW);
 946         hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DISP_ERR);
 947         hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_CODE_ERR);
 948 
 949         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2, msk);
 950         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2_MSK, irq_msk);
 951 }
 952 
 953 static void start_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 954 {
 955         config_id_frame_v3_hw(hisi_hba, phy_no);
 956         config_phy_opt_mode_v3_hw(hisi_hba, phy_no);
 957         enable_phy_v3_hw(hisi_hba, phy_no);
 958 }
 959 
 960 static void phy_hard_reset_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 961 {
 962         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 963         u32 txid_auto;
 964 
 965         hisi_sas_phy_enable(hisi_hba, phy_no, 0);
 966         if (phy->identify.device_type == SAS_END_DEVICE) {
 967                 txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
 968                 hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
 969                                         txid_auto | TX_HARDRST_MSK);
 970         }
 971         msleep(100);
 972         hisi_sas_phy_enable(hisi_hba, phy_no, 1);
 973 }
 974 
 975 static enum sas_linkrate phy_get_max_linkrate_v3_hw(void)
 976 {
 977         return SAS_LINK_RATE_12_0_GBPS;
 978 }
 979 
 980 static void phys_init_v3_hw(struct hisi_hba *hisi_hba)
 981 {
 982         int i;
 983 
 984         for (i = 0; i < hisi_hba->n_phy; i++) {
 985                 struct hisi_sas_phy *phy = &hisi_hba->phy[i];
 986                 struct asd_sas_phy *sas_phy = &phy->sas_phy;
 987 
 988                 if (!sas_phy->phy->enabled)
 989                         continue;
 990 
 991                 hisi_sas_phy_enable(hisi_hba, i, 1);
 992         }
 993 }
 994 
 995 static void sl_notify_ssp_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 996 {
 997         u32 sl_control;
 998 
 999         sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
1000         sl_control |= SL_CONTROL_NOTIFY_EN_MSK;
1001         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
1002         msleep(1);
1003         sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
1004         sl_control &= ~SL_CONTROL_NOTIFY_EN_MSK;
1005         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
1006 }
1007 
1008 static int get_wideport_bitmap_v3_hw(struct hisi_hba *hisi_hba, int port_id)
1009 {
1010         int i, bitmap = 0;
1011         u32 phy_port_num_ma = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
1012         u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1013 
1014         for (i = 0; i < hisi_hba->n_phy; i++)
1015                 if (phy_state & BIT(i))
1016                         if (((phy_port_num_ma >> (i * 4)) & 0xf) == port_id)
1017                                 bitmap |= BIT(i);
1018 
1019         return bitmap;
1020 }
1021 
1022 static void start_delivery_v3_hw(struct hisi_sas_dq *dq)
1023 {
1024         struct hisi_hba *hisi_hba = dq->hisi_hba;
1025         struct hisi_sas_slot *s, *s1, *s2 = NULL;
1026         int dlvry_queue = dq->id;
1027         int wp;
1028 
1029         list_for_each_entry_safe(s, s1, &dq->list, delivery) {
1030                 if (!s->ready)
1031                         break;
1032                 s2 = s;
1033                 list_del(&s->delivery);
1034         }
1035 
1036         if (!s2)
1037                 return;
1038 
1039         /*
1040          * Ensure that memories for slots built on other CPUs is observed.
1041          */
1042         smp_rmb();
1043         wp = (s2->dlvry_queue_slot + 1) % HISI_SAS_QUEUE_SLOTS;
1044 
1045         hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14), wp);
1046 }
1047 
1048 static void prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba,
1049                               struct hisi_sas_slot *slot,
1050                               struct hisi_sas_cmd_hdr *hdr,
1051                               struct scatterlist *scatter,
1052                               int n_elem)
1053 {
1054         struct hisi_sas_sge_page *sge_page = hisi_sas_sge_addr_mem(slot);
1055         struct scatterlist *sg;
1056         int i;
1057 
1058         for_each_sg(scatter, sg, n_elem, i) {
1059                 struct hisi_sas_sge *entry = &sge_page->sge[i];
1060 
1061                 entry->addr = cpu_to_le64(sg_dma_address(sg));
1062                 entry->page_ctrl_0 = entry->page_ctrl_1 = 0;
1063                 entry->data_len = cpu_to_le32(sg_dma_len(sg));
1064                 entry->data_off = 0;
1065         }
1066 
1067         hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot));
1068 
1069         hdr->sg_len |= cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF);
1070 }
1071 
1072 static void prep_prd_sge_dif_v3_hw(struct hisi_hba *hisi_hba,
1073                                    struct hisi_sas_slot *slot,
1074                                    struct hisi_sas_cmd_hdr *hdr,
1075                                    struct scatterlist *scatter,
1076                                    int n_elem)
1077 {
1078         struct hisi_sas_sge_dif_page *sge_dif_page;
1079         struct scatterlist *sg;
1080         int i;
1081 
1082         sge_dif_page = hisi_sas_sge_dif_addr_mem(slot);
1083 
1084         for_each_sg(scatter, sg, n_elem, i) {
1085                 struct hisi_sas_sge *entry = &sge_dif_page->sge[i];
1086 
1087                 entry->addr = cpu_to_le64(sg_dma_address(sg));
1088                 entry->page_ctrl_0 = 0;
1089                 entry->page_ctrl_1 = 0;
1090                 entry->data_len = cpu_to_le32(sg_dma_len(sg));
1091                 entry->data_off = 0;
1092         }
1093 
1094         hdr->dif_prd_table_addr =
1095                 cpu_to_le64(hisi_sas_sge_dif_addr_dma(slot));
1096 
1097         hdr->sg_len |= cpu_to_le32(n_elem << CMD_HDR_DIF_SGL_LEN_OFF);
1098 }
1099 
1100 static u32 get_prot_chk_msk_v3_hw(struct scsi_cmnd *scsi_cmnd)
1101 {
1102         unsigned char prot_flags = scsi_cmnd->prot_flags;
1103 
1104         if (prot_flags & SCSI_PROT_REF_CHECK)
1105                 return T10_CHK_APP_TAG_MSK;
1106         return T10_CHK_REF_TAG_MSK | T10_CHK_APP_TAG_MSK;
1107 }
1108 
1109 static void fill_prot_v3_hw(struct scsi_cmnd *scsi_cmnd,
1110                             struct hisi_sas_protect_iu_v3_hw *prot)
1111 {
1112         unsigned char prot_op = scsi_get_prot_op(scsi_cmnd);
1113         unsigned int interval = scsi_prot_interval(scsi_cmnd);
1114         u32 lbrt_chk_val = t10_pi_ref_tag(scsi_cmnd->request);
1115 
1116         switch (prot_op) {
1117         case SCSI_PROT_READ_INSERT:
1118                 prot->dw0 |= T10_INSRT_EN_MSK;
1119                 prot->lbrtgv = lbrt_chk_val;
1120                 break;
1121         case SCSI_PROT_READ_STRIP:
1122                 prot->dw0 |= (T10_RMV_EN_MSK | T10_CHK_EN_MSK);
1123                 prot->lbrtcv = lbrt_chk_val;
1124                 prot->dw4 |= get_prot_chk_msk_v3_hw(scsi_cmnd);
1125                 break;
1126         case SCSI_PROT_READ_PASS:
1127                 prot->dw0 |= T10_CHK_EN_MSK;
1128                 prot->lbrtcv = lbrt_chk_val;
1129                 prot->dw4 |= get_prot_chk_msk_v3_hw(scsi_cmnd);
1130                 break;
1131         case SCSI_PROT_WRITE_INSERT:
1132                 prot->dw0 |= T10_INSRT_EN_MSK;
1133                 prot->lbrtgv = lbrt_chk_val;
1134                 break;
1135         case SCSI_PROT_WRITE_STRIP:
1136                 prot->dw0 |= (T10_RMV_EN_MSK | T10_CHK_EN_MSK);
1137                 prot->lbrtcv = lbrt_chk_val;
1138                 break;
1139         case SCSI_PROT_WRITE_PASS:
1140                 prot->dw0 |= T10_CHK_EN_MSK;
1141                 prot->lbrtcv = lbrt_chk_val;
1142                 prot->dw4 |= get_prot_chk_msk_v3_hw(scsi_cmnd);
1143                 break;
1144         default:
1145                 WARN(1, "prot_op(0x%x) is not valid\n", prot_op);
1146                 break;
1147         }
1148 
1149         switch (interval) {
1150         case 512:
1151                 break;
1152         case 4096:
1153                 prot->dw0 |= (0x1 << USR_DATA_BLOCK_SZ_OFF);
1154                 break;
1155         case 520:
1156                 prot->dw0 |= (0x2 << USR_DATA_BLOCK_SZ_OFF);
1157                 break;
1158         default:
1159                 WARN(1, "protection interval (0x%x) invalid\n",
1160                      interval);
1161                 break;
1162         }
1163 
1164         prot->dw0 |= INCR_LBRT_MSK;
1165 }
1166 
1167 static void prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
1168                           struct hisi_sas_slot *slot)
1169 {
1170         struct sas_task *task = slot->task;
1171         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1172         struct domain_device *device = task->dev;
1173         struct hisi_sas_device *sas_dev = device->lldd_dev;
1174         struct hisi_sas_port *port = slot->port;
1175         struct sas_ssp_task *ssp_task = &task->ssp_task;
1176         struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
1177         struct hisi_sas_tmf_task *tmf = slot->tmf;
1178         int has_data = 0, priority = !!tmf;
1179         unsigned char prot_op;
1180         u8 *buf_cmd;
1181         u32 dw1 = 0, dw2 = 0, len = 0;
1182 
1183         hdr->dw0 = cpu_to_le32((1 << CMD_HDR_RESP_REPORT_OFF) |
1184                                (2 << CMD_HDR_TLR_CTRL_OFF) |
1185                                (port->id << CMD_HDR_PORT_OFF) |
1186                                (priority << CMD_HDR_PRIORITY_OFF) |
1187                                (1 << CMD_HDR_CMD_OFF)); /* ssp */
1188 
1189         dw1 = 1 << CMD_HDR_VDTL_OFF;
1190         if (tmf) {
1191                 dw1 |= 2 << CMD_HDR_FRAME_TYPE_OFF;
1192                 dw1 |= DIR_NO_DATA << CMD_HDR_DIR_OFF;
1193         } else {
1194                 prot_op = scsi_get_prot_op(scsi_cmnd);
1195                 dw1 |= 1 << CMD_HDR_FRAME_TYPE_OFF;
1196                 switch (scsi_cmnd->sc_data_direction) {
1197                 case DMA_TO_DEVICE:
1198                         has_data = 1;
1199                         dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
1200                         break;
1201                 case DMA_FROM_DEVICE:
1202                         has_data = 1;
1203                         dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
1204                         break;
1205                 default:
1206                         dw1 &= ~CMD_HDR_DIR_MSK;
1207                 }
1208         }
1209 
1210         /* map itct entry */
1211         dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
1212 
1213         dw2 = (((sizeof(struct ssp_command_iu) + sizeof(struct ssp_frame_hdr)
1214               + 3) / 4) << CMD_HDR_CFL_OFF) |
1215               ((HISI_SAS_MAX_SSP_RESP_SZ / 4) << CMD_HDR_MRFL_OFF) |
1216               (2 << CMD_HDR_SG_MOD_OFF);
1217         hdr->dw2 = cpu_to_le32(dw2);
1218         hdr->transfer_tags = cpu_to_le32(slot->idx);
1219 
1220         if (has_data) {
1221                 prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
1222                                    slot->n_elem);
1223 
1224                 if (scsi_prot_sg_count(scsi_cmnd))
1225                         prep_prd_sge_dif_v3_hw(hisi_hba, slot, hdr,
1226                                                scsi_prot_sglist(scsi_cmnd),
1227                                                slot->n_elem_dif);
1228         }
1229 
1230         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
1231         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1232 
1233         buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot) +
1234                 sizeof(struct ssp_frame_hdr);
1235 
1236         memcpy(buf_cmd, &task->ssp_task.LUN, 8);
1237         if (!tmf) {
1238                 buf_cmd[9] = ssp_task->task_attr | (ssp_task->task_prio << 3);
1239                 memcpy(buf_cmd + 12, scsi_cmnd->cmnd, scsi_cmnd->cmd_len);
1240         } else {
1241                 buf_cmd[10] = tmf->tmf;
1242                 switch (tmf->tmf) {
1243                 case TMF_ABORT_TASK:
1244                 case TMF_QUERY_TASK:
1245                         buf_cmd[12] =
1246                                 (tmf->tag_of_task_to_be_managed >> 8) & 0xff;
1247                         buf_cmd[13] =
1248                                 tmf->tag_of_task_to_be_managed & 0xff;
1249                         break;
1250                 default:
1251                         break;
1252                 }
1253         }
1254 
1255         if (has_data && (prot_op != SCSI_PROT_NORMAL)) {
1256                 struct hisi_sas_protect_iu_v3_hw prot;
1257                 u8 *buf_cmd_prot;
1258 
1259                 hdr->dw7 |= cpu_to_le32(1 << CMD_HDR_ADDR_MODE_SEL_OFF);
1260                 dw1 |= CMD_HDR_PIR_MSK;
1261                 buf_cmd_prot = hisi_sas_cmd_hdr_addr_mem(slot) +
1262                                sizeof(struct ssp_frame_hdr) +
1263                                sizeof(struct ssp_command_iu);
1264 
1265                 memset(&prot, 0, sizeof(struct hisi_sas_protect_iu_v3_hw));
1266                 fill_prot_v3_hw(scsi_cmnd, &prot);
1267                 memcpy(buf_cmd_prot, &prot,
1268                        sizeof(struct hisi_sas_protect_iu_v3_hw));
1269                 /*
1270                  * For READ, we need length of info read to memory, while for
1271                  * WRITE we need length of data written to the disk.
1272                  */
1273                 if (prot_op == SCSI_PROT_WRITE_INSERT ||
1274                     prot_op == SCSI_PROT_READ_INSERT ||
1275                     prot_op == SCSI_PROT_WRITE_PASS ||
1276                     prot_op == SCSI_PROT_READ_PASS) {
1277                         unsigned int interval = scsi_prot_interval(scsi_cmnd);
1278                         unsigned int ilog2_interval = ilog2(interval);
1279 
1280                         len = (task->total_xfer_len >> ilog2_interval) * 8;
1281                 }
1282         }
1283 
1284         hdr->dw1 = cpu_to_le32(dw1);
1285 
1286         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len + len);
1287 }
1288 
1289 static void prep_smp_v3_hw(struct hisi_hba *hisi_hba,
1290                           struct hisi_sas_slot *slot)
1291 {
1292         struct sas_task *task = slot->task;
1293         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1294         struct domain_device *device = task->dev;
1295         struct hisi_sas_port *port = slot->port;
1296         struct scatterlist *sg_req;
1297         struct hisi_sas_device *sas_dev = device->lldd_dev;
1298         dma_addr_t req_dma_addr;
1299         unsigned int req_len;
1300 
1301         /* req */
1302         sg_req = &task->smp_task.smp_req;
1303         req_len = sg_dma_len(sg_req);
1304         req_dma_addr = sg_dma_address(sg_req);
1305 
1306         /* create header */
1307         /* dw0 */
1308         hdr->dw0 = cpu_to_le32((port->id << CMD_HDR_PORT_OFF) |
1309                                (1 << CMD_HDR_PRIORITY_OFF) | /* high pri */
1310                                (2 << CMD_HDR_CMD_OFF)); /* smp */
1311 
1312         /* map itct entry */
1313         hdr->dw1 = cpu_to_le32((sas_dev->device_id << CMD_HDR_DEV_ID_OFF) |
1314                                (1 << CMD_HDR_FRAME_TYPE_OFF) |
1315                                (DIR_NO_DATA << CMD_HDR_DIR_OFF));
1316 
1317         /* dw2 */
1318         hdr->dw2 = cpu_to_le32((((req_len - 4) / 4) << CMD_HDR_CFL_OFF) |
1319                                (HISI_SAS_MAX_SMP_RESP_SZ / 4 <<
1320                                CMD_HDR_MRFL_OFF));
1321 
1322         hdr->transfer_tags = cpu_to_le32(slot->idx << CMD_HDR_IPTT_OFF);
1323 
1324         hdr->cmd_table_addr = cpu_to_le64(req_dma_addr);
1325         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1326 
1327 }
1328 
1329 static void prep_ata_v3_hw(struct hisi_hba *hisi_hba,
1330                           struct hisi_sas_slot *slot)
1331 {
1332         struct sas_task *task = slot->task;
1333         struct domain_device *device = task->dev;
1334         struct domain_device *parent_dev = device->parent;
1335         struct hisi_sas_device *sas_dev = device->lldd_dev;
1336         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1337         struct asd_sas_port *sas_port = device->port;
1338         struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
1339         u8 *buf_cmd;
1340         int has_data = 0, hdr_tag = 0;
1341         u32 dw1 = 0, dw2 = 0;
1342 
1343         hdr->dw0 = cpu_to_le32(port->id << CMD_HDR_PORT_OFF);
1344         if (parent_dev && dev_is_expander(parent_dev->dev_type))
1345                 hdr->dw0 |= cpu_to_le32(3 << CMD_HDR_CMD_OFF);
1346         else
1347                 hdr->dw0 |= cpu_to_le32(4U << CMD_HDR_CMD_OFF);
1348 
1349         switch (task->data_dir) {
1350         case DMA_TO_DEVICE:
1351                 has_data = 1;
1352                 dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
1353                 break;
1354         case DMA_FROM_DEVICE:
1355                 has_data = 1;
1356                 dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
1357                 break;
1358         default:
1359                 dw1 &= ~CMD_HDR_DIR_MSK;
1360         }
1361 
1362         if ((task->ata_task.fis.command == ATA_CMD_DEV_RESET) &&
1363                         (task->ata_task.fis.control & ATA_SRST))
1364                 dw1 |= 1 << CMD_HDR_RESET_OFF;
1365 
1366         dw1 |= (hisi_sas_get_ata_protocol(
1367                 &task->ata_task.fis, task->data_dir))
1368                 << CMD_HDR_FRAME_TYPE_OFF;
1369         dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
1370 
1371         if (FIS_CMD_IS_UNCONSTRAINED(task->ata_task.fis))
1372                 dw1 |= 1 << CMD_HDR_UNCON_CMD_OFF;
1373 
1374         hdr->dw1 = cpu_to_le32(dw1);
1375 
1376         /* dw2 */
1377         if (task->ata_task.use_ncq) {
1378                 struct ata_queued_cmd *qc = task->uldd_task;
1379 
1380                 hdr_tag = qc->tag;
1381                 task->ata_task.fis.sector_count |= (u8) (hdr_tag << 3);
1382                 dw2 |= hdr_tag << CMD_HDR_NCQ_TAG_OFF;
1383         }
1384 
1385         dw2 |= (HISI_SAS_MAX_STP_RESP_SZ / 4) << CMD_HDR_CFL_OFF |
1386                         2 << CMD_HDR_SG_MOD_OFF;
1387         hdr->dw2 = cpu_to_le32(dw2);
1388 
1389         /* dw3 */
1390         hdr->transfer_tags = cpu_to_le32(slot->idx);
1391 
1392         if (has_data)
1393                 prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
1394                                         slot->n_elem);
1395 
1396         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
1397         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
1398         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1399 
1400         buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot);
1401 
1402         if (likely(!task->ata_task.device_control_reg_update))
1403                 task->ata_task.fis.flags |= 0x80; /* C=1: update ATA cmd reg */
1404         /* fill in command FIS */
1405         memcpy(buf_cmd, &task->ata_task.fis, sizeof(struct host_to_dev_fis));
1406 }
1407 
1408 static void prep_abort_v3_hw(struct hisi_hba *hisi_hba,
1409                 struct hisi_sas_slot *slot,
1410                 int device_id, int abort_flag, int tag_to_abort)
1411 {
1412         struct sas_task *task = slot->task;
1413         struct domain_device *dev = task->dev;
1414         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1415         struct hisi_sas_port *port = slot->port;
1416 
1417         /* dw0 */
1418         hdr->dw0 = cpu_to_le32((5U << CMD_HDR_CMD_OFF) | /*abort*/
1419                                (port->id << CMD_HDR_PORT_OFF) |
1420                                    (dev_is_sata(dev)
1421                                         << CMD_HDR_ABORT_DEVICE_TYPE_OFF) |
1422                                         (abort_flag
1423                                          << CMD_HDR_ABORT_FLAG_OFF));
1424 
1425         /* dw1 */
1426         hdr->dw1 = cpu_to_le32(device_id
1427                         << CMD_HDR_DEV_ID_OFF);
1428 
1429         /* dw7 */
1430         hdr->dw7 = cpu_to_le32(tag_to_abort << CMD_HDR_ABORT_IPTT_OFF);
1431         hdr->transfer_tags = cpu_to_le32(slot->idx);
1432 
1433 }
1434 
1435 static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1436 {
1437         int i;
1438         irqreturn_t res;
1439         u32 context, port_id, link_rate;
1440         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1441         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1442         struct device *dev = hisi_hba->dev;
1443         unsigned long flags;
1444 
1445         del_timer(&phy->timer);
1446         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 1);
1447 
1448         port_id = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
1449         port_id = (port_id >> (4 * phy_no)) & 0xf;
1450         link_rate = hisi_sas_read32(hisi_hba, PHY_CONN_RATE);
1451         link_rate = (link_rate >> (phy_no * 4)) & 0xf;
1452 
1453         if (port_id == 0xf) {
1454                 dev_err(dev, "phyup: phy%d invalid portid\n", phy_no);
1455                 res = IRQ_NONE;
1456                 goto end;
1457         }
1458         sas_phy->linkrate = link_rate;
1459         phy->phy_type &= ~(PORT_TYPE_SAS | PORT_TYPE_SATA);
1460 
1461         /* Check for SATA dev */
1462         context = hisi_sas_read32(hisi_hba, PHY_CONTEXT);
1463         if (context & (1 << phy_no)) {
1464                 struct hisi_sas_initial_fis *initial_fis;
1465                 struct dev_to_host_fis *fis;
1466                 u8 attached_sas_addr[SAS_ADDR_SIZE] = {0};
1467                 struct Scsi_Host *shost = hisi_hba->shost;
1468 
1469                 dev_info(dev, "phyup: phy%d link_rate=%d(sata)\n", phy_no, link_rate);
1470                 initial_fis = &hisi_hba->initial_fis[phy_no];
1471                 fis = &initial_fis->fis;
1472 
1473                 /* check ERR bit of Status Register */
1474                 if (fis->status & ATA_ERR) {
1475                         dev_warn(dev, "sata int: phy%d FIS status: 0x%x\n",
1476                                  phy_no, fis->status);
1477                         hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1478                         res = IRQ_NONE;
1479                         goto end;
1480                 }
1481 
1482                 sas_phy->oob_mode = SATA_OOB_MODE;
1483                 attached_sas_addr[0] = 0x50;
1484                 attached_sas_addr[6] = shost->host_no;
1485                 attached_sas_addr[7] = phy_no;
1486                 memcpy(sas_phy->attached_sas_addr,
1487                        attached_sas_addr,
1488                        SAS_ADDR_SIZE);
1489                 memcpy(sas_phy->frame_rcvd, fis,
1490                        sizeof(struct dev_to_host_fis));
1491                 phy->phy_type |= PORT_TYPE_SATA;
1492                 phy->identify.device_type = SAS_SATA_DEV;
1493                 phy->frame_rcvd_size = sizeof(struct dev_to_host_fis);
1494                 phy->identify.target_port_protocols = SAS_PROTOCOL_SATA;
1495         } else {
1496                 u32 *frame_rcvd = (u32 *)sas_phy->frame_rcvd;
1497                 struct sas_identify_frame *id =
1498                         (struct sas_identify_frame *)frame_rcvd;
1499 
1500                 dev_info(dev, "phyup: phy%d link_rate=%d\n", phy_no, link_rate);
1501                 for (i = 0; i < 6; i++) {
1502                         u32 idaf = hisi_sas_phy_read32(hisi_hba, phy_no,
1503                                                RX_IDAF_DWORD0 + (i * 4));
1504                         frame_rcvd[i] = __swab32(idaf);
1505                 }
1506                 sas_phy->oob_mode = SAS_OOB_MODE;
1507                 memcpy(sas_phy->attached_sas_addr,
1508                        &id->sas_addr,
1509                        SAS_ADDR_SIZE);
1510                 phy->phy_type |= PORT_TYPE_SAS;
1511                 phy->identify.device_type = id->dev_type;
1512                 phy->frame_rcvd_size = sizeof(struct sas_identify_frame);
1513                 if (phy->identify.device_type == SAS_END_DEVICE)
1514                         phy->identify.target_port_protocols =
1515                                 SAS_PROTOCOL_SSP;
1516                 else if (phy->identify.device_type != SAS_PHY_UNUSED)
1517                         phy->identify.target_port_protocols =
1518                                 SAS_PROTOCOL_SMP;
1519         }
1520 
1521         phy->port_id = port_id;
1522         phy->phy_attached = 1;
1523         hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP);
1524         res = IRQ_HANDLED;
1525         spin_lock_irqsave(&phy->lock, flags);
1526         if (phy->reset_completion) {
1527                 phy->in_reset = 0;
1528                 complete(phy->reset_completion);
1529         }
1530         spin_unlock_irqrestore(&phy->lock, flags);
1531 end:
1532         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1533                              CHL_INT0_SL_PHY_ENABLE_MSK);
1534         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 0);
1535 
1536         return res;
1537 }
1538 
1539 static irqreturn_t phy_down_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1540 {
1541         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1542         u32 phy_state, sl_ctrl, txid_auto;
1543         struct device *dev = hisi_hba->dev;
1544 
1545         del_timer(&phy->timer);
1546         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 1);
1547 
1548         phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1549         dev_info(dev, "phydown: phy%d phy_state=0x%x\n", phy_no, phy_state);
1550         hisi_sas_phy_down(hisi_hba, phy_no, (phy_state & 1 << phy_no) ? 1 : 0);
1551 
1552         sl_ctrl = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
1553         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL,
1554                                                 sl_ctrl&(~SL_CTA_MSK));
1555 
1556         txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
1557         hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
1558                                                 txid_auto | CT3_MSK);
1559 
1560         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_NOT_RDY_MSK);
1561         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 0);
1562 
1563         return IRQ_HANDLED;
1564 }
1565 
1566 static irqreturn_t phy_bcast_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1567 {
1568         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1569         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1570         struct sas_ha_struct *sas_ha = &hisi_hba->sha;
1571         u32 bcast_status;
1572 
1573         hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 1);
1574         bcast_status = hisi_sas_phy_read32(hisi_hba, phy_no, RX_PRIMS_STATUS);
1575         if ((bcast_status & RX_BCAST_CHG_MSK) &&
1576             !test_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags))
1577                 sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD);
1578         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1579                              CHL_INT0_SL_RX_BCST_ACK_MSK);
1580         hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 0);
1581 
1582         return IRQ_HANDLED;
1583 }
1584 
1585 static irqreturn_t int_phy_up_down_bcast_v3_hw(int irq_no, void *p)
1586 {
1587         struct hisi_hba *hisi_hba = p;
1588         u32 irq_msk;
1589         int phy_no = 0;
1590         irqreturn_t res = IRQ_NONE;
1591 
1592         irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1593                                 & 0x11111111;
1594         while (irq_msk) {
1595                 if (irq_msk  & 1) {
1596                         u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no,
1597                                                             CHL_INT0);
1598                         u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1599                         int rdy = phy_state & (1 << phy_no);
1600 
1601                         if (rdy) {
1602                                 if (irq_value & CHL_INT0_SL_PHY_ENABLE_MSK)
1603                                         /* phy up */
1604                                         if (phy_up_v3_hw(phy_no, hisi_hba)
1605                                                         == IRQ_HANDLED)
1606                                                 res = IRQ_HANDLED;
1607                                 if (irq_value & CHL_INT0_SL_RX_BCST_ACK_MSK)
1608                                         /* phy bcast */
1609                                         if (phy_bcast_v3_hw(phy_no, hisi_hba)
1610                                                         == IRQ_HANDLED)
1611                                                 res = IRQ_HANDLED;
1612                         } else {
1613                                 if (irq_value & CHL_INT0_NOT_RDY_MSK)
1614                                         /* phy down */
1615                                         if (phy_down_v3_hw(phy_no, hisi_hba)
1616                                                         == IRQ_HANDLED)
1617                                                 res = IRQ_HANDLED;
1618                         }
1619                 }
1620                 irq_msk >>= 4;
1621                 phy_no++;
1622         }
1623 
1624         return res;
1625 }
1626 
1627 static const struct hisi_sas_hw_error port_axi_error[] = {
1628         {
1629                 .irq_msk = BIT(CHL_INT1_DMAC_TX_ECC_MB_ERR_OFF),
1630                 .msg = "dmac_tx_ecc_bad_err",
1631         },
1632         {
1633                 .irq_msk = BIT(CHL_INT1_DMAC_RX_ECC_MB_ERR_OFF),
1634                 .msg = "dmac_rx_ecc_bad_err",
1635         },
1636         {
1637                 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF),
1638                 .msg = "dma_tx_axi_wr_err",
1639         },
1640         {
1641                 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF),
1642                 .msg = "dma_tx_axi_rd_err",
1643         },
1644         {
1645                 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF),
1646                 .msg = "dma_rx_axi_wr_err",
1647         },
1648         {
1649                 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF),
1650                 .msg = "dma_rx_axi_rd_err",
1651         },
1652         {
1653                 .irq_msk = BIT(CHL_INT1_DMAC_TX_FIFO_ERR_OFF),
1654                 .msg = "dma_tx_fifo_err",
1655         },
1656         {
1657                 .irq_msk = BIT(CHL_INT1_DMAC_RX_FIFO_ERR_OFF),
1658                 .msg = "dma_rx_fifo_err",
1659         },
1660         {
1661                 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_RUSER_ERR_OFF),
1662                 .msg = "dma_tx_axi_ruser_err",
1663         },
1664         {
1665                 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_RUSER_ERR_OFF),
1666                 .msg = "dma_rx_axi_ruser_err",
1667         },
1668 };
1669 
1670 static void handle_chl_int1_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1671 {
1672         u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT1);
1673         u32 irq_msk = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT1_MSK);
1674         struct device *dev = hisi_hba->dev;
1675         int i;
1676 
1677         irq_value &= ~irq_msk;
1678         if (!irq_value)
1679                 return;
1680 
1681         for (i = 0; i < ARRAY_SIZE(port_axi_error); i++) {
1682                 const struct hisi_sas_hw_error *error = &port_axi_error[i];
1683 
1684                 if (!(irq_value & error->irq_msk))
1685                         continue;
1686 
1687                 dev_err(dev, "%s error (phy%d 0x%x) found!\n",
1688                         error->msg, phy_no, irq_value);
1689                 queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1690         }
1691 
1692         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT1, irq_value);
1693 }
1694 
1695 static void phy_get_events_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1696 {
1697         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1698         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1699         struct sas_phy *sphy = sas_phy->phy;
1700         unsigned long flags;
1701         u32 reg_value;
1702 
1703         spin_lock_irqsave(&phy->lock, flags);
1704 
1705         /* loss dword sync */
1706         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DWS_LOST);
1707         sphy->loss_of_dword_sync_count += reg_value;
1708 
1709         /* phy reset problem */
1710         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_RESET_PROB);
1711         sphy->phy_reset_problem_count += reg_value;
1712 
1713         /* invalid dword */
1714         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_INVLD_DW);
1715         sphy->invalid_dword_count += reg_value;
1716 
1717         /* disparity err */
1718         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DISP_ERR);
1719         sphy->running_disparity_error_count += reg_value;
1720 
1721         /* code violation error */
1722         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_CODE_ERR);
1723         phy->code_violation_err_count += reg_value;
1724 
1725         spin_unlock_irqrestore(&phy->lock, flags);
1726 }
1727 
1728 static void handle_chl_int2_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1729 {
1730         u32 irq_msk = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2_MSK);
1731         u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2);
1732         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1733         struct pci_dev *pci_dev = hisi_hba->pci_dev;
1734         struct device *dev = hisi_hba->dev;
1735         static const u32 msk = BIT(CHL_INT2_RX_DISP_ERR_OFF) |
1736                         BIT(CHL_INT2_RX_CODE_ERR_OFF) |
1737                         BIT(CHL_INT2_RX_INVLD_DW_OFF);
1738 
1739         irq_value &= ~irq_msk;
1740         if (!irq_value)
1741                 return;
1742 
1743         if (irq_value & BIT(CHL_INT2_SL_IDAF_TOUT_CONF_OFF)) {
1744                 dev_warn(dev, "phy%d identify timeout\n", phy_no);
1745                 hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1746         }
1747 
1748         if (irq_value & BIT(CHL_INT2_STP_LINK_TIMEOUT_OFF)) {
1749                 u32 reg_value = hisi_sas_phy_read32(hisi_hba, phy_no,
1750                                 STP_LINK_TIMEOUT_STATE);
1751 
1752                 dev_warn(dev, "phy%d stp link timeout (0x%x)\n",
1753                          phy_no, reg_value);
1754                 if (reg_value & BIT(4))
1755                         hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1756         }
1757 
1758         if (pci_dev->revision > 0x20 && (irq_value & msk)) {
1759                 struct asd_sas_phy *sas_phy = &phy->sas_phy;
1760                 struct sas_phy *sphy = sas_phy->phy;
1761 
1762                 phy_get_events_v3_hw(hisi_hba, phy_no);
1763 
1764                 if (irq_value & BIT(CHL_INT2_RX_INVLD_DW_OFF))
1765                         dev_info(dev, "phy%d invalid dword cnt:   %u\n", phy_no,
1766                                  sphy->invalid_dword_count);
1767 
1768                 if (irq_value & BIT(CHL_INT2_RX_CODE_ERR_OFF))
1769                         dev_info(dev, "phy%d code violation cnt:  %u\n", phy_no,
1770                                  phy->code_violation_err_count);
1771 
1772                 if (irq_value & BIT(CHL_INT2_RX_DISP_ERR_OFF))
1773                         dev_info(dev, "phy%d disparity error cnt: %u\n", phy_no,
1774                                  sphy->running_disparity_error_count);
1775         }
1776 
1777         if ((irq_value & BIT(CHL_INT2_RX_INVLD_DW_OFF)) &&
1778             (pci_dev->revision == 0x20)) {
1779                 u32 reg_value;
1780                 int rc;
1781 
1782                 rc = hisi_sas_read32_poll_timeout_atomic(
1783                                 HILINK_ERR_DFX, reg_value,
1784                                 !((reg_value >> 8) & BIT(phy_no)),
1785                                 1000, 10000);
1786                 if (rc)
1787                         hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1788         }
1789 
1790         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2, irq_value);
1791 }
1792 
1793 static void handle_chl_int0_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1794 {
1795         u32 irq_value0 = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT0);
1796 
1797         if (irq_value0 & CHL_INT0_PHY_RDY_MSK)
1798                 hisi_sas_phy_oob_ready(hisi_hba, phy_no);
1799 
1800         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1801                              irq_value0 & (~CHL_INT0_SL_RX_BCST_ACK_MSK)
1802                              & (~CHL_INT0_SL_PHY_ENABLE_MSK)
1803                              & (~CHL_INT0_NOT_RDY_MSK));
1804 }
1805 
1806 static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
1807 {
1808         struct hisi_hba *hisi_hba = p;
1809         u32 irq_msk;
1810         int phy_no = 0;
1811 
1812         irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1813                                 & 0xeeeeeeee;
1814 
1815         while (irq_msk) {
1816                 if (irq_msk & (2 << (phy_no * 4)))
1817                         handle_chl_int0_v3_hw(hisi_hba, phy_no);
1818 
1819                 if (irq_msk & (4 << (phy_no * 4)))
1820                         handle_chl_int1_v3_hw(hisi_hba, phy_no);
1821 
1822                 if (irq_msk & (8 << (phy_no * 4)))
1823                         handle_chl_int2_v3_hw(hisi_hba, phy_no);
1824 
1825                 irq_msk &= ~(0xe << (phy_no * 4));
1826                 phy_no++;
1827         }
1828 
1829         return IRQ_HANDLED;
1830 }
1831 
1832 static const struct hisi_sas_hw_error multi_bit_ecc_errors[] = {
1833         {
1834                 .irq_msk = BIT(SAS_ECC_INTR_DQE_ECC_MB_OFF),
1835                 .msk = HGC_DQE_ECC_MB_ADDR_MSK,
1836                 .shift = HGC_DQE_ECC_MB_ADDR_OFF,
1837                 .msg = "hgc_dqe_eccbad_intr",
1838                 .reg = HGC_DQE_ECC_ADDR,
1839         },
1840         {
1841                 .irq_msk = BIT(SAS_ECC_INTR_IOST_ECC_MB_OFF),
1842                 .msk = HGC_IOST_ECC_MB_ADDR_MSK,
1843                 .shift = HGC_IOST_ECC_MB_ADDR_OFF,
1844                 .msg = "hgc_iost_eccbad_intr",
1845                 .reg = HGC_IOST_ECC_ADDR,
1846         },
1847         {
1848                 .irq_msk = BIT(SAS_ECC_INTR_ITCT_ECC_MB_OFF),
1849                 .msk = HGC_ITCT_ECC_MB_ADDR_MSK,
1850                 .shift = HGC_ITCT_ECC_MB_ADDR_OFF,
1851                 .msg = "hgc_itct_eccbad_intr",
1852                 .reg = HGC_ITCT_ECC_ADDR,
1853         },
1854         {
1855                 .irq_msk = BIT(SAS_ECC_INTR_IOSTLIST_ECC_MB_OFF),
1856                 .msk = HGC_LM_DFX_STATUS2_IOSTLIST_MSK,
1857                 .shift = HGC_LM_DFX_STATUS2_IOSTLIST_OFF,
1858                 .msg = "hgc_iostl_eccbad_intr",
1859                 .reg = HGC_LM_DFX_STATUS2,
1860         },
1861         {
1862                 .irq_msk = BIT(SAS_ECC_INTR_ITCTLIST_ECC_MB_OFF),
1863                 .msk = HGC_LM_DFX_STATUS2_ITCTLIST_MSK,
1864                 .shift = HGC_LM_DFX_STATUS2_ITCTLIST_OFF,
1865                 .msg = "hgc_itctl_eccbad_intr",
1866                 .reg = HGC_LM_DFX_STATUS2,
1867         },
1868         {
1869                 .irq_msk = BIT(SAS_ECC_INTR_CQE_ECC_MB_OFF),
1870                 .msk = HGC_CQE_ECC_MB_ADDR_MSK,
1871                 .shift = HGC_CQE_ECC_MB_ADDR_OFF,
1872                 .msg = "hgc_cqe_eccbad_intr",
1873                 .reg = HGC_CQE_ECC_ADDR,
1874         },
1875         {
1876                 .irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM0_ECC_MB_OFF),
1877                 .msk = HGC_RXM_DFX_STATUS14_MEM0_MSK,
1878                 .shift = HGC_RXM_DFX_STATUS14_MEM0_OFF,
1879                 .msg = "rxm_mem0_eccbad_intr",
1880                 .reg = HGC_RXM_DFX_STATUS14,
1881         },
1882         {
1883                 .irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM1_ECC_MB_OFF),
1884                 .msk = HGC_RXM_DFX_STATUS14_MEM1_MSK,
1885                 .shift = HGC_RXM_DFX_STATUS14_MEM1_OFF,
1886                 .msg = "rxm_mem1_eccbad_intr",
1887                 .reg = HGC_RXM_DFX_STATUS14,
1888         },
1889         {
1890                 .irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM2_ECC_MB_OFF),
1891                 .msk = HGC_RXM_DFX_STATUS14_MEM2_MSK,
1892                 .shift = HGC_RXM_DFX_STATUS14_MEM2_OFF,
1893                 .msg = "rxm_mem2_eccbad_intr",
1894                 .reg = HGC_RXM_DFX_STATUS14,
1895         },
1896         {
1897                 .irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM3_ECC_MB_OFF),
1898                 .msk = HGC_RXM_DFX_STATUS15_MEM3_MSK,
1899                 .shift = HGC_RXM_DFX_STATUS15_MEM3_OFF,
1900                 .msg = "rxm_mem3_eccbad_intr",
1901                 .reg = HGC_RXM_DFX_STATUS15,
1902         },
1903         {
1904                 .irq_msk = BIT(SAS_ECC_INTR_OOO_RAM_ECC_MB_OFF),
1905                 .msk = AM_ROB_ECC_ERR_ADDR_MSK,
1906                 .shift = AM_ROB_ECC_ERR_ADDR_OFF,
1907                 .msg = "ooo_ram_eccbad_intr",
1908                 .reg = AM_ROB_ECC_ERR_ADDR,
1909         },
1910 };
1911 
1912 static void multi_bit_ecc_error_process_v3_hw(struct hisi_hba *hisi_hba,
1913                                               u32 irq_value)
1914 {
1915         struct device *dev = hisi_hba->dev;
1916         const struct hisi_sas_hw_error *ecc_error;
1917         u32 val;
1918         int i;
1919 
1920         for (i = 0; i < ARRAY_SIZE(multi_bit_ecc_errors); i++) {
1921                 ecc_error = &multi_bit_ecc_errors[i];
1922                 if (irq_value & ecc_error->irq_msk) {
1923                         val = hisi_sas_read32(hisi_hba, ecc_error->reg);
1924                         val &= ecc_error->msk;
1925                         val >>= ecc_error->shift;
1926                         dev_err(dev, "%s (0x%x) found: mem addr is 0x%08X\n",
1927                                 ecc_error->msg, irq_value, val);
1928                         queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1929                 }
1930         }
1931 }
1932 
1933 static void fatal_ecc_int_v3_hw(struct hisi_hba *hisi_hba)
1934 {
1935         u32 irq_value, irq_msk;
1936 
1937         irq_msk = hisi_sas_read32(hisi_hba, SAS_ECC_INTR_MSK);
1938         hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xffffffff);
1939 
1940         irq_value = hisi_sas_read32(hisi_hba, SAS_ECC_INTR);
1941         if (irq_value)
1942                 multi_bit_ecc_error_process_v3_hw(hisi_hba, irq_value);
1943 
1944         hisi_sas_write32(hisi_hba, SAS_ECC_INTR, irq_value);
1945         hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, irq_msk);
1946 }
1947 
1948 static const struct hisi_sas_hw_error axi_error[] = {
1949         { .msk = BIT(0), .msg = "IOST_AXI_W_ERR" },
1950         { .msk = BIT(1), .msg = "IOST_AXI_R_ERR" },
1951         { .msk = BIT(2), .msg = "ITCT_AXI_W_ERR" },
1952         { .msk = BIT(3), .msg = "ITCT_AXI_R_ERR" },
1953         { .msk = BIT(4), .msg = "SATA_AXI_W_ERR" },
1954         { .msk = BIT(5), .msg = "SATA_AXI_R_ERR" },
1955         { .msk = BIT(6), .msg = "DQE_AXI_R_ERR" },
1956         { .msk = BIT(7), .msg = "CQE_AXI_W_ERR" },
1957         {}
1958 };
1959 
1960 static const struct hisi_sas_hw_error fifo_error[] = {
1961         { .msk = BIT(8),  .msg = "CQE_WINFO_FIFO" },
1962         { .msk = BIT(9),  .msg = "CQE_MSG_FIFIO" },
1963         { .msk = BIT(10), .msg = "GETDQE_FIFO" },
1964         { .msk = BIT(11), .msg = "CMDP_FIFO" },
1965         { .msk = BIT(12), .msg = "AWTCTRL_FIFO" },
1966         {}
1967 };
1968 
1969 static const struct hisi_sas_hw_error fatal_axi_error[] = {
1970         {
1971                 .irq_msk = BIT(ENT_INT_SRC3_WP_DEPTH_OFF),
1972                 .msg = "write pointer and depth",
1973         },
1974         {
1975                 .irq_msk = BIT(ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF),
1976                 .msg = "iptt no match slot",
1977         },
1978         {
1979                 .irq_msk = BIT(ENT_INT_SRC3_RP_DEPTH_OFF),
1980                 .msg = "read pointer and depth",
1981         },
1982         {
1983                 .irq_msk = BIT(ENT_INT_SRC3_AXI_OFF),
1984                 .reg = HGC_AXI_FIFO_ERR_INFO,
1985                 .sub = axi_error,
1986         },
1987         {
1988                 .irq_msk = BIT(ENT_INT_SRC3_FIFO_OFF),
1989                 .reg = HGC_AXI_FIFO_ERR_INFO,
1990                 .sub = fifo_error,
1991         },
1992         {
1993                 .irq_msk = BIT(ENT_INT_SRC3_LM_OFF),
1994                 .msg = "LM add/fetch list",
1995         },
1996         {
1997                 .irq_msk = BIT(ENT_INT_SRC3_ABT_OFF),
1998                 .msg = "SAS_HGC_ABT fetch LM list",
1999         },
2000         {
2001                 .irq_msk = BIT(ENT_INT_SRC3_DQE_POISON_OFF),
2002                 .msg = "read dqe poison",
2003         },
2004         {
2005                 .irq_msk = BIT(ENT_INT_SRC3_IOST_POISON_OFF),
2006                 .msg = "read iost poison",
2007         },
2008         {
2009                 .irq_msk = BIT(ENT_INT_SRC3_ITCT_POISON_OFF),
2010                 .msg = "read itct poison",
2011         },
2012         {
2013                 .irq_msk = BIT(ENT_INT_SRC3_ITCT_NCQ_POISON_OFF),
2014                 .msg = "read itct ncq poison",
2015         },
2016 
2017 };
2018 
2019 static irqreturn_t fatal_axi_int_v3_hw(int irq_no, void *p)
2020 {
2021         u32 irq_value, irq_msk;
2022         struct hisi_hba *hisi_hba = p;
2023         struct device *dev = hisi_hba->dev;
2024         struct pci_dev *pdev = hisi_hba->pci_dev;
2025         int i;
2026 
2027         irq_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3);
2028         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk | 0x1df00);
2029 
2030         irq_value = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
2031         irq_value &= ~irq_msk;
2032 
2033         for (i = 0; i < ARRAY_SIZE(fatal_axi_error); i++) {
2034                 const struct hisi_sas_hw_error *error = &fatal_axi_error[i];
2035 
2036                 if (!(irq_value & error->irq_msk))
2037                         continue;
2038 
2039                 if (error->sub) {
2040                         const struct hisi_sas_hw_error *sub = error->sub;
2041                         u32 err_value = hisi_sas_read32(hisi_hba, error->reg);
2042 
2043                         for (; sub->msk || sub->msg; sub++) {
2044                                 if (!(err_value & sub->msk))
2045                                         continue;
2046 
2047                                 dev_err(dev, "%s error (0x%x) found!\n",
2048                                         sub->msg, irq_value);
2049                                 queue_work(hisi_hba->wq, &hisi_hba->rst_work);
2050                         }
2051                 } else {
2052                         dev_err(dev, "%s error (0x%x) found!\n",
2053                                 error->msg, irq_value);
2054                         queue_work(hisi_hba->wq, &hisi_hba->rst_work);
2055                 }
2056 
2057                 if (pdev->revision < 0x21) {
2058                         u32 reg_val;
2059 
2060                         reg_val = hisi_sas_read32(hisi_hba,
2061                                                   AXI_MASTER_CFG_BASE +
2062                                                   AM_CTRL_GLOBAL);
2063                         reg_val |= AM_CTRL_SHUTDOWN_REQ_MSK;
2064                         hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE +
2065                                          AM_CTRL_GLOBAL, reg_val);
2066                 }
2067         }
2068 
2069         fatal_ecc_int_v3_hw(hisi_hba);
2070 
2071         if (irq_value & BIT(ENT_INT_SRC3_ITC_INT_OFF)) {
2072                 u32 reg_val = hisi_sas_read32(hisi_hba, ITCT_CLR);
2073                 u32 dev_id = reg_val & ITCT_DEV_MSK;
2074                 struct hisi_sas_device *sas_dev =
2075                                 &hisi_hba->devices[dev_id];
2076 
2077                 hisi_sas_write32(hisi_hba, ITCT_CLR, 0);
2078                 dev_dbg(dev, "clear ITCT ok\n");
2079                 complete(sas_dev->completion);
2080         }
2081 
2082         hisi_sas_write32(hisi_hba, ENT_INT_SRC3, irq_value & 0x1df00);
2083         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk);
2084 
2085         return IRQ_HANDLED;
2086 }
2087 
2088 static void
2089 slot_err_v3_hw(struct hisi_hba *hisi_hba, struct sas_task *task,
2090                struct hisi_sas_slot *slot)
2091 {
2092         struct task_status_struct *ts = &task->task_status;
2093         struct hisi_sas_complete_v3_hdr *complete_queue =
2094                         hisi_hba->complete_hdr[slot->cmplt_queue];
2095         struct hisi_sas_complete_v3_hdr *complete_hdr =
2096                         &complete_queue[slot->cmplt_queue_slot];
2097         struct hisi_sas_err_record_v3 *record =
2098                         hisi_sas_status_buf_addr_mem(slot);
2099         u32 dma_rx_err_type = le32_to_cpu(record->dma_rx_err_type);
2100         u32 trans_tx_fail_type = le32_to_cpu(record->trans_tx_fail_type);
2101         u32 dw3 = le32_to_cpu(complete_hdr->dw3);
2102 
2103         switch (task->task_proto) {
2104         case SAS_PROTOCOL_SSP:
2105                 if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
2106                         ts->residual = trans_tx_fail_type;
2107                         ts->stat = SAS_DATA_UNDERRUN;
2108                 } else if (dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) {
2109                         ts->stat = SAS_QUEUE_FULL;
2110                         slot->abort = 1;
2111                 } else {
2112                         ts->stat = SAS_OPEN_REJECT;
2113                         ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
2114                 }
2115                 break;
2116         case SAS_PROTOCOL_SATA:
2117         case SAS_PROTOCOL_STP:
2118         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
2119                 if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
2120                         ts->residual = trans_tx_fail_type;
2121                         ts->stat = SAS_DATA_UNDERRUN;
2122                 } else if (dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) {
2123                         ts->stat = SAS_PHY_DOWN;
2124                         slot->abort = 1;
2125                 } else {
2126                         ts->stat = SAS_OPEN_REJECT;
2127                         ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
2128                 }
2129                 hisi_sas_sata_done(task, slot);
2130                 break;
2131         case SAS_PROTOCOL_SMP:
2132                 ts->stat = SAM_STAT_CHECK_CONDITION;
2133                 break;
2134         default:
2135                 break;
2136         }
2137 }
2138 
2139 static int
2140 slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
2141 {
2142         struct sas_task *task = slot->task;
2143         struct hisi_sas_device *sas_dev;
2144         struct device *dev = hisi_hba->dev;
2145         struct task_status_struct *ts;
2146         struct domain_device *device;
2147         struct sas_ha_struct *ha;
2148         enum exec_status sts;
2149         struct hisi_sas_complete_v3_hdr *complete_queue =
2150                         hisi_hba->complete_hdr[slot->cmplt_queue];
2151         struct hisi_sas_complete_v3_hdr *complete_hdr =
2152                         &complete_queue[slot->cmplt_queue_slot];
2153         unsigned long flags;
2154         bool is_internal = slot->is_internal;
2155         u32 dw0, dw1, dw3;
2156 
2157         if (unlikely(!task || !task->lldd_task || !task->dev))
2158                 return -EINVAL;
2159 
2160         ts = &task->task_status;
2161         device = task->dev;
2162         ha = device->port->ha;
2163         sas_dev = device->lldd_dev;
2164 
2165         spin_lock_irqsave(&task->task_state_lock, flags);
2166         task->task_state_flags &=
2167                 ~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR);
2168         spin_unlock_irqrestore(&task->task_state_lock, flags);
2169 
2170         memset(ts, 0, sizeof(*ts));
2171         ts->resp = SAS_TASK_COMPLETE;
2172 
2173         if (unlikely(!sas_dev)) {
2174                 dev_dbg(dev, "slot complete: port has not device\n");
2175                 ts->stat = SAS_PHY_DOWN;
2176                 goto out;
2177         }
2178 
2179         dw0 = le32_to_cpu(complete_hdr->dw0);
2180         dw1 = le32_to_cpu(complete_hdr->dw1);
2181         dw3 = le32_to_cpu(complete_hdr->dw3);
2182 
2183         /*
2184          * Use SAS+TMF status codes
2185          */
2186         switch ((dw0 & CMPLT_HDR_ABORT_STAT_MSK) >> CMPLT_HDR_ABORT_STAT_OFF) {
2187         case STAT_IO_ABORTED:
2188                 /* this IO has been aborted by abort command */
2189                 ts->stat = SAS_ABORTED_TASK;
2190                 goto out;
2191         case STAT_IO_COMPLETE:
2192                 /* internal abort command complete */
2193                 ts->stat = TMF_RESP_FUNC_SUCC;
2194                 goto out;
2195         case STAT_IO_NO_DEVICE:
2196                 ts->stat = TMF_RESP_FUNC_COMPLETE;
2197                 goto out;
2198         case STAT_IO_NOT_VALID:
2199                 /*
2200                  * abort single IO, the controller can't find the IO
2201                  */
2202                 ts->stat = TMF_RESP_FUNC_FAILED;
2203                 goto out;
2204         default:
2205                 break;
2206         }
2207 
2208         /* check for erroneous completion */
2209         if ((dw0 & CMPLT_HDR_CMPLT_MSK) == 0x3) {
2210                 u32 *error_info = hisi_sas_status_buf_addr_mem(slot);
2211 
2212                 slot_err_v3_hw(hisi_hba, task, slot);
2213                 if (ts->stat != SAS_DATA_UNDERRUN)
2214                         dev_info(dev, "erroneous completion iptt=%d task=%pK dev id=%d CQ hdr: 0x%x 0x%x 0x%x 0x%x Error info: 0x%x 0x%x 0x%x 0x%x\n",
2215                                  slot->idx, task, sas_dev->device_id,
2216                                  dw0, dw1, complete_hdr->act, dw3,
2217                                  error_info[0], error_info[1],
2218                                  error_info[2], error_info[3]);
2219                 if (unlikely(slot->abort))
2220                         return ts->stat;
2221                 goto out;
2222         }
2223 
2224         switch (task->task_proto) {
2225         case SAS_PROTOCOL_SSP: {
2226                 struct ssp_response_iu *iu =
2227                         hisi_sas_status_buf_addr_mem(slot) +
2228                         sizeof(struct hisi_sas_err_record);
2229 
2230                 sas_ssp_task_response(dev, task, iu);
2231                 break;
2232         }
2233         case SAS_PROTOCOL_SMP: {
2234                 struct scatterlist *sg_resp = &task->smp_task.smp_resp;
2235                 void *to = page_address(sg_page(sg_resp));
2236 
2237                 ts->stat = SAM_STAT_GOOD;
2238 
2239                 dma_unmap_sg(dev, &task->smp_task.smp_req, 1,
2240                              DMA_TO_DEVICE);
2241                 memcpy(to + sg_resp->offset,
2242                         hisi_sas_status_buf_addr_mem(slot) +
2243                        sizeof(struct hisi_sas_err_record),
2244                        sg_resp->length);
2245                 break;
2246         }
2247         case SAS_PROTOCOL_SATA:
2248         case SAS_PROTOCOL_STP:
2249         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
2250                 ts->stat = SAM_STAT_GOOD;
2251                 hisi_sas_sata_done(task, slot);
2252                 break;
2253         default:
2254                 ts->stat = SAM_STAT_CHECK_CONDITION;
2255                 break;
2256         }
2257 
2258         if (!slot->port->port_attached) {
2259                 dev_warn(dev, "slot complete: port %d has removed\n",
2260                         slot->port->sas_port.id);
2261                 ts->stat = SAS_PHY_DOWN;
2262         }
2263 
2264 out:
2265         sts = ts->stat;
2266         spin_lock_irqsave(&task->task_state_lock, flags);
2267         if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
2268                 spin_unlock_irqrestore(&task->task_state_lock, flags);
2269                 dev_info(dev, "slot complete: task(%pK) aborted\n", task);
2270                 return SAS_ABORTED_TASK;
2271         }
2272         task->task_state_flags |= SAS_TASK_STATE_DONE;
2273         spin_unlock_irqrestore(&task->task_state_lock, flags);
2274         hisi_sas_slot_task_free(hisi_hba, task, slot);
2275 
2276         if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) {
2277                 spin_lock_irqsave(&device->done_lock, flags);
2278                 if (test_bit(SAS_HA_FROZEN, &ha->state)) {
2279                         spin_unlock_irqrestore(&device->done_lock, flags);
2280                         dev_info(dev, "slot complete: task(%pK) ignored\n ",
2281                                  task);
2282                         return sts;
2283                 }
2284                 spin_unlock_irqrestore(&device->done_lock, flags);
2285         }
2286 
2287         if (task->task_done)
2288                 task->task_done(task);
2289 
2290         return sts;
2291 }
2292 
2293 static void cq_tasklet_v3_hw(unsigned long val)
2294 {
2295         struct hisi_sas_cq *cq = (struct hisi_sas_cq *)val;
2296         struct hisi_hba *hisi_hba = cq->hisi_hba;
2297         struct hisi_sas_slot *slot;
2298         struct hisi_sas_complete_v3_hdr *complete_queue;
2299         u32 rd_point = cq->rd_point, wr_point;
2300         int queue = cq->id;
2301 
2302         complete_queue = hisi_hba->complete_hdr[queue];
2303 
2304         wr_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_WR_PTR +
2305                                    (0x14 * queue));
2306 
2307         while (rd_point != wr_point) {
2308                 struct hisi_sas_complete_v3_hdr *complete_hdr;
2309                 struct device *dev = hisi_hba->dev;
2310                 u32 dw1;
2311                 int iptt;
2312 
2313                 complete_hdr = &complete_queue[rd_point];
2314                 dw1 = le32_to_cpu(complete_hdr->dw1);
2315 
2316                 iptt = dw1 & CMPLT_HDR_IPTT_MSK;
2317                 if (likely(iptt < HISI_SAS_COMMAND_ENTRIES_V3_HW)) {
2318                         slot = &hisi_hba->slot_info[iptt];
2319                         slot->cmplt_queue_slot = rd_point;
2320                         slot->cmplt_queue = queue;
2321                         slot_complete_v3_hw(hisi_hba, slot);
2322                 } else
2323                         dev_err(dev, "IPTT %d is invalid, discard it.\n", iptt);
2324 
2325                 if (++rd_point >= HISI_SAS_QUEUE_SLOTS)
2326                         rd_point = 0;
2327         }
2328 
2329         /* update rd_point */
2330         cq->rd_point = rd_point;
2331         hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point);
2332 }
2333 
2334 static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p)
2335 {
2336         struct hisi_sas_cq *cq = p;
2337         struct hisi_hba *hisi_hba = cq->hisi_hba;
2338         int queue = cq->id;
2339 
2340         hisi_sas_write32(hisi_hba, OQ_INT_SRC, 1 << queue);
2341 
2342         tasklet_schedule(&cq->tasklet);
2343 
2344         return IRQ_HANDLED;
2345 }
2346 
2347 static void setup_reply_map_v3_hw(struct hisi_hba *hisi_hba, int nvecs)
2348 {
2349         const struct cpumask *mask;
2350         int queue, cpu;
2351 
2352         for (queue = 0; queue < nvecs; queue++) {
2353                 struct hisi_sas_cq *cq = &hisi_hba->cq[queue];
2354 
2355                 mask = pci_irq_get_affinity(hisi_hba->pci_dev, queue +
2356                                             BASE_VECTORS_V3_HW);
2357                 if (!mask)
2358                         goto fallback;
2359                 cq->pci_irq_mask = mask;
2360                 for_each_cpu(cpu, mask)
2361                         hisi_hba->reply_map[cpu] = queue;
2362         }
2363         return;
2364 
2365 fallback:
2366         for_each_possible_cpu(cpu)
2367                 hisi_hba->reply_map[cpu] = cpu % hisi_hba->queue_count;
2368         /* Don't clean all CQ masks */
2369 }
2370 
2371 static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba)
2372 {
2373         struct device *dev = hisi_hba->dev;
2374         struct pci_dev *pdev = hisi_hba->pci_dev;
2375         int vectors, rc, i;
2376         int max_msi = HISI_SAS_MSI_COUNT_V3_HW, min_msi;
2377 
2378         if (auto_affine_msi_experimental) {
2379                 struct irq_affinity desc = {
2380                         .pre_vectors = BASE_VECTORS_V3_HW,
2381                 };
2382 
2383                 min_msi = MIN_AFFINE_VECTORS_V3_HW;
2384 
2385                 hisi_hba->reply_map = devm_kcalloc(dev, nr_cpu_ids,
2386                                                    sizeof(unsigned int),
2387                                                    GFP_KERNEL);
2388                 if (!hisi_hba->reply_map)
2389                         return -ENOMEM;
2390                 vectors = pci_alloc_irq_vectors_affinity(hisi_hba->pci_dev,
2391                                                          min_msi, max_msi,
2392                                                          PCI_IRQ_MSI |
2393                                                          PCI_IRQ_AFFINITY,
2394                                                          &desc);
2395                 if (vectors < 0)
2396                         return -ENOENT;
2397                 setup_reply_map_v3_hw(hisi_hba, vectors - BASE_VECTORS_V3_HW);
2398         } else {
2399                 min_msi = max_msi;
2400                 vectors = pci_alloc_irq_vectors(hisi_hba->pci_dev, min_msi,
2401                                                 max_msi, PCI_IRQ_MSI);
2402                 if (vectors < 0)
2403                         return vectors;
2404         }
2405 
2406         hisi_hba->cq_nvecs = vectors - BASE_VECTORS_V3_HW;
2407 
2408         rc = devm_request_irq(dev, pci_irq_vector(pdev, 1),
2409                               int_phy_up_down_bcast_v3_hw, 0,
2410                               DRV_NAME " phy", hisi_hba);
2411         if (rc) {
2412                 dev_err(dev, "could not request phy interrupt, rc=%d\n", rc);
2413                 rc = -ENOENT;
2414                 goto free_irq_vectors;
2415         }
2416 
2417         rc = devm_request_irq(dev, pci_irq_vector(pdev, 2),
2418                               int_chnl_int_v3_hw, 0,
2419                               DRV_NAME " channel", hisi_hba);
2420         if (rc) {
2421                 dev_err(dev, "could not request chnl interrupt, rc=%d\n", rc);
2422                 rc = -ENOENT;
2423                 goto free_irq_vectors;
2424         }
2425 
2426         rc = devm_request_irq(dev, pci_irq_vector(pdev, 11),
2427                               fatal_axi_int_v3_hw, 0,
2428                               DRV_NAME " fatal", hisi_hba);
2429         if (rc) {
2430                 dev_err(dev, "could not request fatal interrupt, rc=%d\n", rc);
2431                 rc = -ENOENT;
2432                 goto free_irq_vectors;
2433         }
2434 
2435         /* Init tasklets for cq only */
2436         for (i = 0; i < hisi_hba->cq_nvecs; i++) {
2437                 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
2438                 struct tasklet_struct *t = &cq->tasklet;
2439                 int nr = hisi_sas_intr_conv ? 16 : 16 + i;
2440                 unsigned long irqflags = hisi_sas_intr_conv ? IRQF_SHARED : 0;
2441 
2442                 rc = devm_request_irq(dev, pci_irq_vector(pdev, nr),
2443                                       cq_interrupt_v3_hw, irqflags,
2444                                       DRV_NAME " cq", cq);
2445                 if (rc) {
2446                         dev_err(dev, "could not request cq%d interrupt, rc=%d\n",
2447                                 i, rc);
2448                         rc = -ENOENT;
2449                         goto free_irq_vectors;
2450                 }
2451 
2452                 tasklet_init(t, cq_tasklet_v3_hw, (unsigned long)cq);
2453         }
2454 
2455         return 0;
2456 
2457 free_irq_vectors:
2458         pci_free_irq_vectors(pdev);
2459         return rc;
2460 }
2461 
2462 static int hisi_sas_v3_init(struct hisi_hba *hisi_hba)
2463 {
2464         int rc;
2465 
2466         rc = hw_init_v3_hw(hisi_hba);
2467         if (rc)
2468                 return rc;
2469 
2470         rc = interrupt_init_v3_hw(hisi_hba);
2471         if (rc)
2472                 return rc;
2473 
2474         return 0;
2475 }
2476 
2477 static void phy_set_linkrate_v3_hw(struct hisi_hba *hisi_hba, int phy_no,
2478                 struct sas_phy_linkrates *r)
2479 {
2480         enum sas_linkrate max = r->maximum_linkrate;
2481         u32 prog_phy_link_rate = 0x800;
2482 
2483         prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max);
2484         hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
2485                              prog_phy_link_rate);
2486 }
2487 
2488 static void interrupt_disable_v3_hw(struct hisi_hba *hisi_hba)
2489 {
2490         struct pci_dev *pdev = hisi_hba->pci_dev;
2491         int i;
2492 
2493         synchronize_irq(pci_irq_vector(pdev, 1));
2494         synchronize_irq(pci_irq_vector(pdev, 2));
2495         synchronize_irq(pci_irq_vector(pdev, 11));
2496         for (i = 0; i < hisi_hba->queue_count; i++) {
2497                 hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, 0x1);
2498                 synchronize_irq(pci_irq_vector(pdev, i + 16));
2499         }
2500 
2501         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xffffffff);
2502         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xffffffff);
2503         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffffffff);
2504         hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xffffffff);
2505 
2506         for (i = 0; i < hisi_hba->n_phy; i++) {
2507                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xffffffff);
2508                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffffff);
2509                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x1);
2510                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x1);
2511                 hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x1);
2512         }
2513 }
2514 
2515 static u32 get_phys_state_v3_hw(struct hisi_hba *hisi_hba)
2516 {
2517         return hisi_sas_read32(hisi_hba, PHY_STATE);
2518 }
2519 
2520 static int disable_host_v3_hw(struct hisi_hba *hisi_hba)
2521 {
2522         struct device *dev = hisi_hba->dev;
2523         u32 status, reg_val;
2524         int rc;
2525 
2526         interrupt_disable_v3_hw(hisi_hba);
2527         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0x0);
2528         hisi_sas_kill_tasklets(hisi_hba);
2529 
2530         hisi_sas_stop_phys(hisi_hba);
2531 
2532         mdelay(10);
2533 
2534         reg_val = hisi_sas_read32(hisi_hba, AXI_MASTER_CFG_BASE +
2535                                   AM_CTRL_GLOBAL);
2536         reg_val |= AM_CTRL_SHUTDOWN_REQ_MSK;
2537         hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE +
2538                          AM_CTRL_GLOBAL, reg_val);
2539 
2540         /* wait until bus idle */
2541         rc = hisi_sas_read32_poll_timeout(AXI_MASTER_CFG_BASE +
2542                                           AM_CURR_TRANS_RETURN, status,
2543                                           status == 0x3, 10, 100);
2544         if (rc) {
2545                 dev_err(dev, "axi bus is not idle, rc=%d\n", rc);
2546                 return rc;
2547         }
2548 
2549         return 0;
2550 }
2551 
2552 static int soft_reset_v3_hw(struct hisi_hba *hisi_hba)
2553 {
2554         struct device *dev = hisi_hba->dev;
2555         int rc;
2556 
2557         rc = disable_host_v3_hw(hisi_hba);
2558         if (rc) {
2559                 dev_err(dev, "soft reset: disable host failed rc=%d\n", rc);
2560                 return rc;
2561         }
2562 
2563         hisi_sas_init_mem(hisi_hba);
2564 
2565         return hw_init_v3_hw(hisi_hba);
2566 }
2567 
2568 static int write_gpio_v3_hw(struct hisi_hba *hisi_hba, u8 reg_type,
2569                         u8 reg_index, u8 reg_count, u8 *write_data)
2570 {
2571         struct device *dev = hisi_hba->dev;
2572         u32 *data = (u32 *)write_data;
2573         int i;
2574 
2575         switch (reg_type) {
2576         case SAS_GPIO_REG_TX:
2577                 if ((reg_index + reg_count) > ((hisi_hba->n_phy + 3) / 4)) {
2578                         dev_err(dev, "write gpio: invalid reg range[%d, %d]\n",
2579                                 reg_index, reg_index + reg_count - 1);
2580                         return -EINVAL;
2581                 }
2582 
2583                 for (i = 0; i < reg_count; i++)
2584                         hisi_sas_write32(hisi_hba,
2585                                          SAS_GPIO_TX_0_1 + (reg_index + i) * 4,
2586                                          data[i]);
2587                 break;
2588         default:
2589                 dev_err(dev, "write gpio: unsupported or bad reg type %d\n",
2590                         reg_type);
2591                 return -EINVAL;
2592         }
2593 
2594         return 0;
2595 }
2596 
2597 static void wait_cmds_complete_timeout_v3_hw(struct hisi_hba *hisi_hba,
2598                                              int delay_ms, int timeout_ms)
2599 {
2600         struct device *dev = hisi_hba->dev;
2601         int entries, entries_old = 0, time;
2602 
2603         for (time = 0; time < timeout_ms; time += delay_ms) {
2604                 entries = hisi_sas_read32(hisi_hba, CQE_SEND_CNT);
2605                 if (entries == entries_old)
2606                         break;
2607 
2608                 entries_old = entries;
2609                 msleep(delay_ms);
2610         }
2611 
2612         if (time >= timeout_ms) {
2613                 dev_dbg(dev, "Wait commands complete timeout!\n");
2614                 return;
2615         }
2616 
2617         dev_dbg(dev, "wait commands complete %dms\n", time);
2618 }
2619 
2620 static ssize_t intr_conv_v3_hw_show(struct device *dev,
2621                                     struct device_attribute *attr, char *buf)
2622 {
2623         return scnprintf(buf, PAGE_SIZE, "%u\n", hisi_sas_intr_conv);
2624 }
2625 static DEVICE_ATTR_RO(intr_conv_v3_hw);
2626 
2627 static void config_intr_coal_v3_hw(struct hisi_hba *hisi_hba)
2628 {
2629         /* config those registers between enable and disable PHYs */
2630         hisi_sas_stop_phys(hisi_hba);
2631 
2632         if (hisi_hba->intr_coal_ticks == 0 ||
2633             hisi_hba->intr_coal_count == 0) {
2634                 hisi_sas_write32(hisi_hba, INT_COAL_EN, 0x1);
2635                 hisi_sas_write32(hisi_hba, OQ_INT_COAL_TIME, 0x1);
2636                 hisi_sas_write32(hisi_hba, OQ_INT_COAL_CNT, 0x1);
2637         } else {
2638                 hisi_sas_write32(hisi_hba, INT_COAL_EN, 0x3);
2639                 hisi_sas_write32(hisi_hba, OQ_INT_COAL_TIME,
2640                                  hisi_hba->intr_coal_ticks);
2641                 hisi_sas_write32(hisi_hba, OQ_INT_COAL_CNT,
2642                                  hisi_hba->intr_coal_count);
2643         }
2644         phys_init_v3_hw(hisi_hba);
2645 }
2646 
2647 static ssize_t intr_coal_ticks_v3_hw_show(struct device *dev,
2648                                           struct device_attribute *attr,
2649                                           char *buf)
2650 {
2651         struct Scsi_Host *shost = class_to_shost(dev);
2652         struct hisi_hba *hisi_hba = shost_priv(shost);
2653 
2654         return scnprintf(buf, PAGE_SIZE, "%u\n",
2655                          hisi_hba->intr_coal_ticks);
2656 }
2657 
2658 static ssize_t intr_coal_ticks_v3_hw_store(struct device *dev,
2659                                            struct device_attribute *attr,
2660                                            const char *buf, size_t count)
2661 {
2662         struct Scsi_Host *shost = class_to_shost(dev);
2663         struct hisi_hba *hisi_hba = shost_priv(shost);
2664         u32 intr_coal_ticks;
2665         int ret;
2666 
2667         ret = kstrtou32(buf, 10, &intr_coal_ticks);
2668         if (ret) {
2669                 dev_err(dev, "Input data of interrupt coalesce unmatch\n");
2670                 return -EINVAL;
2671         }
2672 
2673         if (intr_coal_ticks >= BIT(24)) {
2674                 dev_err(dev, "intr_coal_ticks must be less than 2^24!\n");
2675                 return -EINVAL;
2676         }
2677 
2678         hisi_hba->intr_coal_ticks = intr_coal_ticks;
2679 
2680         config_intr_coal_v3_hw(hisi_hba);
2681 
2682         return count;
2683 }
2684 static DEVICE_ATTR_RW(intr_coal_ticks_v3_hw);
2685 
2686 static ssize_t intr_coal_count_v3_hw_show(struct device *dev,
2687                                           struct device_attribute
2688                                           *attr, char *buf)
2689 {
2690         struct Scsi_Host *shost = class_to_shost(dev);
2691         struct hisi_hba *hisi_hba = shost_priv(shost);
2692 
2693         return scnprintf(buf, PAGE_SIZE, "%u\n",
2694                          hisi_hba->intr_coal_count);
2695 }
2696 
2697 static ssize_t intr_coal_count_v3_hw_store(struct device *dev,
2698                 struct device_attribute
2699                 *attr, const char *buf, size_t count)
2700 {
2701         struct Scsi_Host *shost = class_to_shost(dev);
2702         struct hisi_hba *hisi_hba = shost_priv(shost);
2703         u32 intr_coal_count;
2704         int ret;
2705 
2706         ret = kstrtou32(buf, 10, &intr_coal_count);
2707         if (ret) {
2708                 dev_err(dev, "Input data of interrupt coalesce unmatch\n");
2709                 return -EINVAL;
2710         }
2711 
2712         if (intr_coal_count >= BIT(8)) {
2713                 dev_err(dev, "intr_coal_count must be less than 2^8!\n");
2714                 return -EINVAL;
2715         }
2716 
2717         hisi_hba->intr_coal_count = intr_coal_count;
2718 
2719         config_intr_coal_v3_hw(hisi_hba);
2720 
2721         return count;
2722 }
2723 static DEVICE_ATTR_RW(intr_coal_count_v3_hw);
2724 
2725 static struct device_attribute *host_attrs_v3_hw[] = {
2726         &dev_attr_phy_event_threshold,
2727         &dev_attr_intr_conv_v3_hw,
2728         &dev_attr_intr_coal_ticks_v3_hw,
2729         &dev_attr_intr_coal_count_v3_hw,
2730         NULL
2731 };
2732 
2733 static const struct hisi_sas_debugfs_reg_lu debugfs_port_reg_lu[] = {
2734         HISI_SAS_DEBUGFS_REG(PHY_CFG),
2735         HISI_SAS_DEBUGFS_REG(HARD_PHY_LINKRATE),
2736         HISI_SAS_DEBUGFS_REG(PROG_PHY_LINK_RATE),
2737         HISI_SAS_DEBUGFS_REG(PHY_CTRL),
2738         HISI_SAS_DEBUGFS_REG(SL_CFG),
2739         HISI_SAS_DEBUGFS_REG(AIP_LIMIT),
2740         HISI_SAS_DEBUGFS_REG(SL_CONTROL),
2741         HISI_SAS_DEBUGFS_REG(RX_PRIMS_STATUS),
2742         HISI_SAS_DEBUGFS_REG(TX_ID_DWORD0),
2743         HISI_SAS_DEBUGFS_REG(TX_ID_DWORD1),
2744         HISI_SAS_DEBUGFS_REG(TX_ID_DWORD2),
2745         HISI_SAS_DEBUGFS_REG(TX_ID_DWORD3),
2746         HISI_SAS_DEBUGFS_REG(TX_ID_DWORD4),
2747         HISI_SAS_DEBUGFS_REG(TX_ID_DWORD5),
2748         HISI_SAS_DEBUGFS_REG(TX_ID_DWORD6),
2749         HISI_SAS_DEBUGFS_REG(TXID_AUTO),
2750         HISI_SAS_DEBUGFS_REG(RX_IDAF_DWORD0),
2751         HISI_SAS_DEBUGFS_REG(RXOP_CHECK_CFG_H),
2752         HISI_SAS_DEBUGFS_REG(STP_LINK_TIMER),
2753         HISI_SAS_DEBUGFS_REG(STP_LINK_TIMEOUT_STATE),
2754         HISI_SAS_DEBUGFS_REG(CON_CFG_DRIVER),
2755         HISI_SAS_DEBUGFS_REG(SAS_SSP_CON_TIMER_CFG),
2756         HISI_SAS_DEBUGFS_REG(SAS_SMP_CON_TIMER_CFG),
2757         HISI_SAS_DEBUGFS_REG(SAS_STP_CON_TIMER_CFG),
2758         HISI_SAS_DEBUGFS_REG(CHL_INT0),
2759         HISI_SAS_DEBUGFS_REG(CHL_INT1),
2760         HISI_SAS_DEBUGFS_REG(CHL_INT2),
2761         HISI_SAS_DEBUGFS_REG(CHL_INT0_MSK),
2762         HISI_SAS_DEBUGFS_REG(CHL_INT1_MSK),
2763         HISI_SAS_DEBUGFS_REG(CHL_INT2_MSK),
2764         HISI_SAS_DEBUGFS_REG(SAS_EC_INT_COAL_TIME),
2765         HISI_SAS_DEBUGFS_REG(CHL_INT_COAL_EN),
2766         HISI_SAS_DEBUGFS_REG(SAS_RX_TRAIN_TIMER),
2767         HISI_SAS_DEBUGFS_REG(PHY_CTRL_RDY_MSK),
2768         HISI_SAS_DEBUGFS_REG(PHYCTRL_NOT_RDY_MSK),
2769         HISI_SAS_DEBUGFS_REG(PHYCTRL_DWS_RESET_MSK),
2770         HISI_SAS_DEBUGFS_REG(PHYCTRL_PHY_ENA_MSK),
2771         HISI_SAS_DEBUGFS_REG(SL_RX_BCAST_CHK_MSK),
2772         HISI_SAS_DEBUGFS_REG(PHYCTRL_OOB_RESTART_MSK),
2773         HISI_SAS_DEBUGFS_REG(DMA_TX_STATUS),
2774         HISI_SAS_DEBUGFS_REG(DMA_RX_STATUS),
2775         HISI_SAS_DEBUGFS_REG(COARSETUNE_TIME),
2776         HISI_SAS_DEBUGFS_REG(ERR_CNT_DWS_LOST),
2777         HISI_SAS_DEBUGFS_REG(ERR_CNT_RESET_PROB),
2778         HISI_SAS_DEBUGFS_REG(ERR_CNT_INVLD_DW),
2779         HISI_SAS_DEBUGFS_REG(ERR_CNT_CODE_ERR),
2780         HISI_SAS_DEBUGFS_REG(ERR_CNT_DISP_ERR),
2781         {}
2782 };
2783 
2784 static const struct hisi_sas_debugfs_reg debugfs_port_reg = {
2785         .lu = debugfs_port_reg_lu,
2786         .count = 0x100,
2787         .base_off = PORT_BASE,
2788         .read_port_reg = hisi_sas_phy_read32,
2789 };
2790 
2791 static const struct hisi_sas_debugfs_reg_lu debugfs_global_reg_lu[] = {
2792         HISI_SAS_DEBUGFS_REG(DLVRY_QUEUE_ENABLE),
2793         HISI_SAS_DEBUGFS_REG(PHY_CONTEXT),
2794         HISI_SAS_DEBUGFS_REG(PHY_STATE),
2795         HISI_SAS_DEBUGFS_REG(PHY_PORT_NUM_MA),
2796         HISI_SAS_DEBUGFS_REG(PHY_CONN_RATE),
2797         HISI_SAS_DEBUGFS_REG(ITCT_CLR),
2798         HISI_SAS_DEBUGFS_REG(IO_SATA_BROKEN_MSG_ADDR_LO),
2799         HISI_SAS_DEBUGFS_REG(IO_SATA_BROKEN_MSG_ADDR_HI),
2800         HISI_SAS_DEBUGFS_REG(SATA_INITI_D2H_STORE_ADDR_LO),
2801         HISI_SAS_DEBUGFS_REG(SATA_INITI_D2H_STORE_ADDR_HI),
2802         HISI_SAS_DEBUGFS_REG(CFG_MAX_TAG),
2803         HISI_SAS_DEBUGFS_REG(HGC_SAS_TX_OPEN_FAIL_RETRY_CTRL),
2804         HISI_SAS_DEBUGFS_REG(HGC_SAS_TXFAIL_RETRY_CTRL),
2805         HISI_SAS_DEBUGFS_REG(HGC_GET_ITV_TIME),
2806         HISI_SAS_DEBUGFS_REG(DEVICE_MSG_WORK_MODE),
2807         HISI_SAS_DEBUGFS_REG(OPENA_WT_CONTI_TIME),
2808         HISI_SAS_DEBUGFS_REG(I_T_NEXUS_LOSS_TIME),
2809         HISI_SAS_DEBUGFS_REG(MAX_CON_TIME_LIMIT_TIME),
2810         HISI_SAS_DEBUGFS_REG(BUS_INACTIVE_LIMIT_TIME),
2811         HISI_SAS_DEBUGFS_REG(REJECT_TO_OPEN_LIMIT_TIME),
2812         HISI_SAS_DEBUGFS_REG(CQ_INT_CONVERGE_EN),
2813         HISI_SAS_DEBUGFS_REG(CFG_AGING_TIME),
2814         HISI_SAS_DEBUGFS_REG(HGC_DFX_CFG2),
2815         HISI_SAS_DEBUGFS_REG(CFG_ABT_SET_QUERY_IPTT),
2816         HISI_SAS_DEBUGFS_REG(CFG_ABT_SET_IPTT_DONE),
2817         HISI_SAS_DEBUGFS_REG(HGC_IOMB_PROC1_STATUS),
2818         HISI_SAS_DEBUGFS_REG(CHNL_INT_STATUS),
2819         HISI_SAS_DEBUGFS_REG(HGC_AXI_FIFO_ERR_INFO),
2820         HISI_SAS_DEBUGFS_REG(INT_COAL_EN),
2821         HISI_SAS_DEBUGFS_REG(OQ_INT_COAL_TIME),
2822         HISI_SAS_DEBUGFS_REG(OQ_INT_COAL_CNT),
2823         HISI_SAS_DEBUGFS_REG(ENT_INT_COAL_TIME),
2824         HISI_SAS_DEBUGFS_REG(ENT_INT_COAL_CNT),
2825         HISI_SAS_DEBUGFS_REG(OQ_INT_SRC),
2826         HISI_SAS_DEBUGFS_REG(OQ_INT_SRC_MSK),
2827         HISI_SAS_DEBUGFS_REG(ENT_INT_SRC1),
2828         HISI_SAS_DEBUGFS_REG(ENT_INT_SRC2),
2829         HISI_SAS_DEBUGFS_REG(ENT_INT_SRC3),
2830         HISI_SAS_DEBUGFS_REG(ENT_INT_SRC_MSK1),
2831         HISI_SAS_DEBUGFS_REG(ENT_INT_SRC_MSK2),
2832         HISI_SAS_DEBUGFS_REG(ENT_INT_SRC_MSK3),
2833         HISI_SAS_DEBUGFS_REG(CHNL_PHYUPDOWN_INT_MSK),
2834         HISI_SAS_DEBUGFS_REG(CHNL_ENT_INT_MSK),
2835         HISI_SAS_DEBUGFS_REG(HGC_COM_INT_MSK),
2836         HISI_SAS_DEBUGFS_REG(SAS_ECC_INTR),
2837         HISI_SAS_DEBUGFS_REG(SAS_ECC_INTR_MSK),
2838         HISI_SAS_DEBUGFS_REG(HGC_ERR_STAT_EN),
2839         HISI_SAS_DEBUGFS_REG(CQE_SEND_CNT),
2840         HISI_SAS_DEBUGFS_REG(DLVRY_Q_0_DEPTH),
2841         HISI_SAS_DEBUGFS_REG(DLVRY_Q_0_WR_PTR),
2842         HISI_SAS_DEBUGFS_REG(DLVRY_Q_0_RD_PTR),
2843         HISI_SAS_DEBUGFS_REG(HYPER_STREAM_ID_EN_CFG),
2844         HISI_SAS_DEBUGFS_REG(OQ0_INT_SRC_MSK),
2845         HISI_SAS_DEBUGFS_REG(COMPL_Q_0_DEPTH),
2846         HISI_SAS_DEBUGFS_REG(COMPL_Q_0_WR_PTR),
2847         HISI_SAS_DEBUGFS_REG(COMPL_Q_0_RD_PTR),
2848         HISI_SAS_DEBUGFS_REG(AWQOS_AWCACHE_CFG),
2849         HISI_SAS_DEBUGFS_REG(ARQOS_ARCACHE_CFG),
2850         HISI_SAS_DEBUGFS_REG(HILINK_ERR_DFX),
2851         HISI_SAS_DEBUGFS_REG(SAS_GPIO_CFG_0),
2852         HISI_SAS_DEBUGFS_REG(SAS_GPIO_CFG_1),
2853         HISI_SAS_DEBUGFS_REG(SAS_GPIO_TX_0_1),
2854         HISI_SAS_DEBUGFS_REG(SAS_CFG_DRIVE_VLD),
2855         {}
2856 };
2857 
2858 static const struct hisi_sas_debugfs_reg debugfs_global_reg = {
2859         .lu = debugfs_global_reg_lu,
2860         .count = 0x800,
2861         .read_global_reg = hisi_sas_read32,
2862 };
2863 
2864 static const struct hisi_sas_debugfs_reg_lu debugfs_axi_reg_lu[] = {
2865         HISI_SAS_DEBUGFS_REG(AM_CFG_MAX_TRANS),
2866         HISI_SAS_DEBUGFS_REG(AM_CFG_SINGLE_PORT_MAX_TRANS),
2867         HISI_SAS_DEBUGFS_REG(AXI_CFG),
2868         HISI_SAS_DEBUGFS_REG(AM_ROB_ECC_ERR_ADDR),
2869         {}
2870 };
2871 
2872 static const struct hisi_sas_debugfs_reg debugfs_axi_reg = {
2873         .lu = debugfs_axi_reg_lu,
2874         .count = 0x61,
2875         .base_off = AXI_MASTER_CFG_BASE,
2876         .read_global_reg = hisi_sas_read32,
2877 };
2878 
2879 static const struct hisi_sas_debugfs_reg_lu debugfs_ras_reg_lu[] = {
2880         HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR1),
2881         HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR0_MASK),
2882         HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR1_MASK),
2883         HISI_SAS_DEBUGFS_REG(CFG_SAS_RAS_INTR_MASK),
2884         HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR2),
2885         HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR2_MASK),
2886         {}
2887 };
2888 
2889 static const struct hisi_sas_debugfs_reg debugfs_ras_reg = {
2890         .lu = debugfs_ras_reg_lu,
2891         .count = 0x10,
2892         .base_off = RAS_BASE,
2893         .read_global_reg = hisi_sas_read32,
2894 };
2895 
2896 static void debugfs_snapshot_prepare_v3_hw(struct hisi_hba *hisi_hba)
2897 {
2898         set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2899 
2900         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0);
2901 
2902         wait_cmds_complete_timeout_v3_hw(hisi_hba, 100, 5000);
2903 
2904         hisi_sas_kill_tasklets(hisi_hba);
2905 }
2906 
2907 static void debugfs_snapshot_restore_v3_hw(struct hisi_hba *hisi_hba)
2908 {
2909         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE,
2910                          (u32)((1ULL << hisi_hba->queue_count) - 1));
2911 
2912         clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2913 }
2914 
2915 static void read_iost_itct_cache_v3_hw(struct hisi_hba *hisi_hba,
2916                                        enum hisi_sas_debugfs_cache_type type,
2917                                        u32 *cache)
2918 {
2919         u32 cache_dw_size = HISI_SAS_IOST_ITCT_CACHE_DW_SZ *
2920                             HISI_SAS_IOST_ITCT_CACHE_NUM;
2921         u32 *buf = cache;
2922         u32 i, val;
2923 
2924         hisi_sas_write32(hisi_hba, TAB_RD_TYPE, type);
2925 
2926         for (i = 0; i < HISI_SAS_IOST_ITCT_CACHE_DW_SZ; i++) {
2927                 val = hisi_sas_read32(hisi_hba, TAB_DFX);
2928                 if (val == 0xffffffff)
2929                         break;
2930         }
2931 
2932         if (val != 0xffffffff) {
2933                 pr_err("Issue occur when reading IOST/ITCT cache!\n");
2934                 return;
2935         }
2936 
2937         memset(buf, 0, cache_dw_size * 4);
2938         buf[0] = val;
2939 
2940         for (i = 1; i < cache_dw_size; i++)
2941                 buf[i] = hisi_sas_read32(hisi_hba, TAB_DFX);
2942 }
2943 
2944 static void hisi_sas_bist_test_prep_v3_hw(struct hisi_hba *hisi_hba)
2945 {
2946         u32 reg_val;
2947         int phy_id = hisi_hba->debugfs_bist_phy_no;
2948 
2949         /* disable PHY */
2950         hisi_sas_phy_enable(hisi_hba, phy_id, 0);
2951 
2952         /* disable ALOS */
2953         reg_val = hisi_sas_phy_read32(hisi_hba, phy_id, SERDES_CFG);
2954         reg_val |= CFG_ALOS_CHK_DISABLE_MSK;
2955         hisi_sas_phy_write32(hisi_hba, phy_id, SERDES_CFG, reg_val);
2956 }
2957 
2958 static void hisi_sas_bist_test_restore_v3_hw(struct hisi_hba *hisi_hba)
2959 {
2960         u32 reg_val;
2961         int phy_id = hisi_hba->debugfs_bist_phy_no;
2962 
2963         /* disable loopback */
2964         reg_val = hisi_sas_phy_read32(hisi_hba, phy_id, SAS_PHY_BIST_CTRL);
2965         reg_val &= ~(CFG_RX_BIST_EN_MSK | CFG_TX_BIST_EN_MSK |
2966                      CFG_BIST_TEST_MSK);
2967         hisi_sas_phy_write32(hisi_hba, phy_id, SAS_PHY_BIST_CTRL, reg_val);
2968 
2969         /* enable ALOS */
2970         reg_val = hisi_sas_phy_read32(hisi_hba, phy_id, SERDES_CFG);
2971         reg_val &= ~CFG_ALOS_CHK_DISABLE_MSK;
2972         hisi_sas_phy_write32(hisi_hba, phy_id, SERDES_CFG, reg_val);
2973 
2974         /* restore the linkrate */
2975         reg_val = hisi_sas_phy_read32(hisi_hba, phy_id, PROG_PHY_LINK_RATE);
2976         /* init OOB link rate as 1.5 Gbits */
2977         reg_val &= ~CFG_PROG_PHY_LINK_RATE_MSK;
2978         reg_val |= (0x8 << CFG_PROG_PHY_LINK_RATE_OFF);
2979         hisi_sas_phy_write32(hisi_hba, phy_id, PROG_PHY_LINK_RATE, reg_val);
2980 
2981         /* enable PHY */
2982         hisi_sas_phy_enable(hisi_hba, phy_id, 1);
2983 }
2984 
2985 #define SAS_PHY_BIST_CODE_INIT  0x1
2986 #define SAS_PHY_BIST_CODE1_INIT 0X80
2987 static int debugfs_set_bist_v3_hw(struct hisi_hba *hisi_hba, bool enable)
2988 {
2989         u32 reg_val, mode_tmp;
2990         u32 linkrate = hisi_hba->debugfs_bist_linkrate;
2991         u32 phy_id = hisi_hba->debugfs_bist_phy_no;
2992         u32 code_mode = hisi_hba->debugfs_bist_code_mode;
2993         u32 path_mode = hisi_hba->debugfs_bist_mode;
2994         struct device *dev = hisi_hba->dev;
2995 
2996         dev_info(dev, "BIST info:linkrate=%d phy_id=%d code_mode=%d path_mode=%d\n",
2997                  linkrate, phy_id, code_mode, path_mode);
2998         mode_tmp = path_mode ? 2 : 1;
2999         if (enable) {
3000                 /* some preparations before bist test */
3001                 hisi_sas_bist_test_prep_v3_hw(hisi_hba);
3002 
3003                 /* set linkrate of bit test*/
3004                 reg_val = hisi_sas_phy_read32(hisi_hba, phy_id,
3005                                               PROG_PHY_LINK_RATE);
3006                 reg_val &= ~CFG_PROG_PHY_LINK_RATE_MSK;
3007                 reg_val |= (linkrate << CFG_PROG_PHY_LINK_RATE_OFF);
3008                 hisi_sas_phy_write32(hisi_hba, phy_id,
3009                                      PROG_PHY_LINK_RATE, reg_val);
3010 
3011                 /* set code mode of bit test */
3012                 reg_val = hisi_sas_phy_read32(hisi_hba, phy_id,
3013                                               SAS_PHY_BIST_CTRL);
3014                 reg_val &= ~(CFG_BIST_MODE_SEL_MSK |
3015                                 CFG_LOOP_TEST_MODE_MSK |
3016                                 CFG_RX_BIST_EN_MSK |
3017                                 CFG_TX_BIST_EN_MSK |
3018                                 CFG_BIST_TEST_MSK);
3019                 reg_val |= ((code_mode << CFG_BIST_MODE_SEL_OFF) |
3020                             (mode_tmp << CFG_LOOP_TEST_MODE_OFF) |
3021                             CFG_BIST_TEST_MSK);
3022                 hisi_sas_phy_write32(hisi_hba, phy_id,
3023                                      SAS_PHY_BIST_CTRL, reg_val);
3024 
3025                 /* set the bist init value */
3026                 hisi_sas_phy_write32(hisi_hba, phy_id,
3027                                      SAS_PHY_BIST_CODE,
3028                                      SAS_PHY_BIST_CODE_INIT);
3029                 hisi_sas_phy_write32(hisi_hba, phy_id,
3030                                      SAS_PHY_BIST_CODE1,
3031                                      SAS_PHY_BIST_CODE1_INIT);
3032 
3033                 mdelay(100);
3034                 reg_val |= (CFG_RX_BIST_EN_MSK | CFG_TX_BIST_EN_MSK);
3035                 hisi_sas_phy_write32(hisi_hba, phy_id,
3036                                      SAS_PHY_BIST_CTRL, reg_val);
3037 
3038                 /* clear error bit */
3039                 mdelay(100);
3040                 hisi_sas_phy_read32(hisi_hba, phy_id, SAS_BIST_ERR_CNT);
3041         } else {
3042                 /* disable bist test and recover it */
3043                 hisi_hba->debugfs_bist_cnt += hisi_sas_phy_read32(hisi_hba,
3044                                 phy_id, SAS_BIST_ERR_CNT);
3045                 hisi_sas_bist_test_restore_v3_hw(hisi_hba);
3046         }
3047 
3048         return 0;
3049 }
3050 
3051 static struct scsi_host_template sht_v3_hw = {
3052         .name                   = DRV_NAME,
3053         .module                 = THIS_MODULE,
3054         .queuecommand           = sas_queuecommand,
3055         .target_alloc           = sas_target_alloc,
3056         .slave_configure        = hisi_sas_slave_configure,
3057         .scan_finished          = hisi_sas_scan_finished,
3058         .scan_start             = hisi_sas_scan_start,
3059         .change_queue_depth     = sas_change_queue_depth,
3060         .bios_param             = sas_bios_param,
3061         .this_id                = -1,
3062         .sg_tablesize           = HISI_SAS_SGE_PAGE_CNT,
3063         .sg_prot_tablesize      = HISI_SAS_SGE_PAGE_CNT,
3064         .max_sectors            = SCSI_DEFAULT_MAX_SECTORS,
3065         .eh_device_reset_handler = sas_eh_device_reset_handler,
3066         .eh_target_reset_handler = sas_eh_target_reset_handler,
3067         .target_destroy         = sas_target_destroy,
3068         .ioctl                  = sas_ioctl,
3069         .shost_attrs            = host_attrs_v3_hw,
3070         .tag_alloc_policy       = BLK_TAG_ALLOC_RR,
3071         .host_reset             = hisi_sas_host_reset,
3072 };
3073 
3074 static const struct hisi_sas_hw hisi_sas_v3_hw = {
3075         .hw_init = hisi_sas_v3_init,
3076         .setup_itct = setup_itct_v3_hw,
3077         .get_wideport_bitmap = get_wideport_bitmap_v3_hw,
3078         .complete_hdr_size = sizeof(struct hisi_sas_complete_v3_hdr),
3079         .clear_itct = clear_itct_v3_hw,
3080         .sl_notify_ssp = sl_notify_ssp_v3_hw,
3081         .prep_ssp = prep_ssp_v3_hw,
3082         .prep_smp = prep_smp_v3_hw,
3083         .prep_stp = prep_ata_v3_hw,
3084         .prep_abort = prep_abort_v3_hw,
3085         .start_delivery = start_delivery_v3_hw,
3086         .phys_init = phys_init_v3_hw,
3087         .phy_start = start_phy_v3_hw,
3088         .phy_disable = disable_phy_v3_hw,
3089         .phy_hard_reset = phy_hard_reset_v3_hw,
3090         .phy_get_max_linkrate = phy_get_max_linkrate_v3_hw,
3091         .phy_set_linkrate = phy_set_linkrate_v3_hw,
3092         .dereg_device = dereg_device_v3_hw,
3093         .soft_reset = soft_reset_v3_hw,
3094         .get_phys_state = get_phys_state_v3_hw,
3095         .get_events = phy_get_events_v3_hw,
3096         .write_gpio = write_gpio_v3_hw,
3097         .wait_cmds_complete_timeout = wait_cmds_complete_timeout_v3_hw,
3098         .debugfs_reg_array[DEBUGFS_GLOBAL] = &debugfs_global_reg,
3099         .debugfs_reg_array[DEBUGFS_AXI] = &debugfs_axi_reg,
3100         .debugfs_reg_array[DEBUGFS_RAS] = &debugfs_ras_reg,
3101         .debugfs_reg_port = &debugfs_port_reg,
3102         .snapshot_prepare = debugfs_snapshot_prepare_v3_hw,
3103         .snapshot_restore = debugfs_snapshot_restore_v3_hw,
3104         .read_iost_itct_cache = read_iost_itct_cache_v3_hw,
3105         .set_bist = debugfs_set_bist_v3_hw,
3106 };
3107 
3108 static struct Scsi_Host *
3109 hisi_sas_shost_alloc_pci(struct pci_dev *pdev)
3110 {
3111         struct Scsi_Host *shost;
3112         struct hisi_hba *hisi_hba;
3113         struct device *dev = &pdev->dev;
3114 
3115         shost = scsi_host_alloc(&sht_v3_hw, sizeof(*hisi_hba));
3116         if (!shost) {
3117                 dev_err(dev, "shost alloc failed\n");
3118                 return NULL;
3119         }
3120         hisi_hba = shost_priv(shost);
3121 
3122         INIT_WORK(&hisi_hba->rst_work, hisi_sas_rst_work_handler);
3123         INIT_WORK(&hisi_hba->debugfs_work, hisi_sas_debugfs_work_handler);
3124         hisi_hba->hw = &hisi_sas_v3_hw;
3125         hisi_hba->pci_dev = pdev;
3126         hisi_hba->dev = dev;
3127         hisi_hba->shost = shost;
3128         SHOST_TO_SAS_HA(shost) = &hisi_hba->sha;
3129 
3130         if (prot_mask & ~HISI_SAS_PROT_MASK)
3131                 dev_err(dev, "unsupported protection mask 0x%x, using default (0x0)\n",
3132                         prot_mask);
3133         else
3134                 hisi_hba->prot_mask = prot_mask;
3135 
3136         if (hisi_sas_get_fw_info(hisi_hba) < 0)
3137                 goto err_out;
3138 
3139         if (hisi_sas_alloc(hisi_hba)) {
3140                 hisi_sas_free(hisi_hba);
3141                 goto err_out;
3142         }
3143 
3144         return shost;
3145 err_out:
3146         scsi_host_put(shost);
3147         dev_err(dev, "shost alloc failed\n");
3148         return NULL;
3149 }
3150 
3151 static int
3152 hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id)
3153 {
3154         struct Scsi_Host *shost;
3155         struct hisi_hba *hisi_hba;
3156         struct device *dev = &pdev->dev;
3157         struct asd_sas_phy **arr_phy;
3158         struct asd_sas_port **arr_port;
3159         struct sas_ha_struct *sha;
3160         int rc, phy_nr, port_nr, i;
3161 
3162         rc = pci_enable_device(pdev);
3163         if (rc)
3164                 goto err_out;
3165 
3166         pci_set_master(pdev);
3167 
3168         rc = pci_request_regions(pdev, DRV_NAME);
3169         if (rc)
3170                 goto err_out_disable_device;
3171 
3172         rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3173         if (rc)
3174                 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3175         if (rc) {
3176                 dev_err(dev, "No usable DMA addressing method\n");
3177                 rc = -ENODEV;
3178                 goto err_out_regions;
3179         }
3180 
3181         shost = hisi_sas_shost_alloc_pci(pdev);
3182         if (!shost) {
3183                 rc = -ENOMEM;
3184                 goto err_out_regions;
3185         }
3186 
3187         sha = SHOST_TO_SAS_HA(shost);
3188         hisi_hba = shost_priv(shost);
3189         dev_set_drvdata(dev, sha);
3190 
3191         hisi_hba->regs = pcim_iomap(pdev, 5, 0);
3192         if (!hisi_hba->regs) {
3193                 dev_err(dev, "cannot map register\n");
3194                 rc = -ENOMEM;
3195                 goto err_out_ha;
3196         }
3197 
3198         phy_nr = port_nr = hisi_hba->n_phy;
3199 
3200         arr_phy = devm_kcalloc(dev, phy_nr, sizeof(void *), GFP_KERNEL);
3201         arr_port = devm_kcalloc(dev, port_nr, sizeof(void *), GFP_KERNEL);
3202         if (!arr_phy || !arr_port) {
3203                 rc = -ENOMEM;
3204                 goto err_out_ha;
3205         }
3206 
3207         sha->sas_phy = arr_phy;
3208         sha->sas_port = arr_port;
3209         sha->core.shost = shost;
3210         sha->lldd_ha = hisi_hba;
3211 
3212         shost->transportt = hisi_sas_stt;
3213         shost->max_id = HISI_SAS_MAX_DEVICES;
3214         shost->max_lun = ~0;
3215         shost->max_channel = 1;
3216         shost->max_cmd_len = 16;
3217         shost->can_queue = HISI_SAS_UNRESERVED_IPTT;
3218         shost->cmd_per_lun = HISI_SAS_UNRESERVED_IPTT;
3219 
3220         sha->sas_ha_name = DRV_NAME;
3221         sha->dev = dev;
3222         sha->lldd_module = THIS_MODULE;
3223         sha->sas_addr = &hisi_hba->sas_addr[0];
3224         sha->num_phys = hisi_hba->n_phy;
3225 
3226         for (i = 0; i < hisi_hba->n_phy; i++) {
3227                 sha->sas_phy[i] = &hisi_hba->phy[i].sas_phy;
3228                 sha->sas_port[i] = &hisi_hba->port[i].sas_port;
3229         }
3230 
3231         if (hisi_hba->prot_mask) {
3232                 dev_info(dev, "Registering for DIF/DIX prot_mask=0x%x\n",
3233                          prot_mask);
3234                 scsi_host_set_prot(hisi_hba->shost, prot_mask);
3235                 if (hisi_hba->prot_mask & HISI_SAS_DIX_PROT_MASK)
3236                         scsi_host_set_guard(hisi_hba->shost,
3237                                             SHOST_DIX_GUARD_CRC);
3238         }
3239 
3240         if (hisi_sas_debugfs_enable)
3241                 hisi_sas_debugfs_init(hisi_hba);
3242 
3243         rc = scsi_add_host(shost, dev);
3244         if (rc)
3245                 goto err_out_ha;
3246 
3247         rc = sas_register_ha(sha);
3248         if (rc)
3249                 goto err_out_register_ha;
3250 
3251         rc = hisi_hba->hw->hw_init(hisi_hba);
3252         if (rc)
3253                 goto err_out_register_ha;
3254 
3255         scsi_scan_host(shost);
3256 
3257         return 0;
3258 
3259 err_out_register_ha:
3260         scsi_remove_host(shost);
3261 err_out_ha:
3262         hisi_sas_debugfs_exit(hisi_hba);
3263         scsi_host_put(shost);
3264 err_out_regions:
3265         pci_release_regions(pdev);
3266 err_out_disable_device:
3267         pci_disable_device(pdev);
3268 err_out:
3269         return rc;
3270 }
3271 
3272 static void
3273 hisi_sas_v3_destroy_irqs(struct pci_dev *pdev, struct hisi_hba *hisi_hba)
3274 {
3275         int i;
3276 
3277         free_irq(pci_irq_vector(pdev, 1), hisi_hba);
3278         free_irq(pci_irq_vector(pdev, 2), hisi_hba);
3279         free_irq(pci_irq_vector(pdev, 11), hisi_hba);
3280         for (i = 0; i < hisi_hba->cq_nvecs; i++) {
3281                 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
3282                 int nr = hisi_sas_intr_conv ? 16 : 16 + i;
3283 
3284                 free_irq(pci_irq_vector(pdev, nr), cq);
3285         }
3286         pci_free_irq_vectors(pdev);
3287 }
3288 
3289 static void hisi_sas_v3_remove(struct pci_dev *pdev)
3290 {
3291         struct device *dev = &pdev->dev;
3292         struct sas_ha_struct *sha = dev_get_drvdata(dev);
3293         struct hisi_hba *hisi_hba = sha->lldd_ha;
3294         struct Scsi_Host *shost = sha->core.shost;
3295 
3296         hisi_sas_debugfs_exit(hisi_hba);
3297 
3298         if (timer_pending(&hisi_hba->timer))
3299                 del_timer(&hisi_hba->timer);
3300 
3301         sas_unregister_ha(sha);
3302         sas_remove_host(sha->core.shost);
3303 
3304         hisi_sas_v3_destroy_irqs(pdev, hisi_hba);
3305         hisi_sas_kill_tasklets(hisi_hba);
3306         pci_release_regions(pdev);
3307         pci_disable_device(pdev);
3308         hisi_sas_free(hisi_hba);
3309         scsi_host_put(shost);
3310 }
3311 
3312 static void hisi_sas_reset_prepare_v3_hw(struct pci_dev *pdev)
3313 {
3314         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
3315         struct hisi_hba *hisi_hba = sha->lldd_ha;
3316         struct device *dev = hisi_hba->dev;
3317         int rc;
3318 
3319         dev_info(dev, "FLR prepare\n");
3320         set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
3321         hisi_sas_controller_reset_prepare(hisi_hba);
3322 
3323         rc = disable_host_v3_hw(hisi_hba);
3324         if (rc)
3325                 dev_err(dev, "FLR: disable host failed rc=%d\n", rc);
3326 }
3327 
3328 static void hisi_sas_reset_done_v3_hw(struct pci_dev *pdev)
3329 {
3330         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
3331         struct hisi_hba *hisi_hba = sha->lldd_ha;
3332         struct device *dev = hisi_hba->dev;
3333         int rc;
3334 
3335         hisi_sas_init_mem(hisi_hba);
3336 
3337         rc = hw_init_v3_hw(hisi_hba);
3338         if (rc) {
3339                 dev_err(dev, "FLR: hw init failed rc=%d\n", rc);
3340                 return;
3341         }
3342 
3343         hisi_sas_controller_reset_done(hisi_hba);
3344         dev_info(dev, "FLR done\n");
3345 }
3346 
3347 enum {
3348         /* instances of the controller */
3349         hip08,
3350 };
3351 
3352 static int hisi_sas_v3_suspend(struct pci_dev *pdev, pm_message_t state)
3353 {
3354         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
3355         struct hisi_hba *hisi_hba = sha->lldd_ha;
3356         struct device *dev = hisi_hba->dev;
3357         struct Scsi_Host *shost = hisi_hba->shost;
3358         pci_power_t device_state;
3359         int rc;
3360 
3361         if (!pdev->pm_cap) {
3362                 dev_err(dev, "PCI PM not supported\n");
3363                 return -ENODEV;
3364         }
3365 
3366         if (test_and_set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags))
3367                 return -1;
3368 
3369         scsi_block_requests(shost);
3370         set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
3371         flush_workqueue(hisi_hba->wq);
3372 
3373         rc = disable_host_v3_hw(hisi_hba);
3374         if (rc) {
3375                 dev_err(dev, "PM suspend: disable host failed rc=%d\n", rc);
3376                 clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
3377                 clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
3378                 scsi_unblock_requests(shost);
3379                 return rc;
3380         }
3381 
3382         hisi_sas_init_mem(hisi_hba);
3383 
3384         device_state = pci_choose_state(pdev, state);
3385         dev_warn(dev, "entering operating state [D%d]\n",
3386                         device_state);
3387         pci_save_state(pdev);
3388         pci_disable_device(pdev);
3389         pci_set_power_state(pdev, device_state);
3390 
3391         hisi_sas_release_tasks(hisi_hba);
3392 
3393         sas_suspend_ha(sha);
3394         return 0;
3395 }
3396 
3397 static int hisi_sas_v3_resume(struct pci_dev *pdev)
3398 {
3399         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
3400         struct hisi_hba *hisi_hba = sha->lldd_ha;
3401         struct Scsi_Host *shost = hisi_hba->shost;
3402         struct device *dev = hisi_hba->dev;
3403         unsigned int rc;
3404         pci_power_t device_state = pdev->current_state;
3405 
3406         dev_warn(dev, "resuming from operating state [D%d]\n",
3407                  device_state);
3408         pci_set_power_state(pdev, PCI_D0);
3409         pci_enable_wake(pdev, PCI_D0, 0);
3410         pci_restore_state(pdev);
3411         rc = pci_enable_device(pdev);
3412         if (rc) {
3413                 dev_err(dev, "enable device failed during resume (%d)\n", rc);
3414                 return rc;
3415         }
3416 
3417         pci_set_master(pdev);
3418         scsi_unblock_requests(shost);
3419         clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
3420 
3421         sas_prep_resume_ha(sha);
3422         rc = hw_init_v3_hw(hisi_hba);
3423         if (rc) {
3424                 scsi_remove_host(shost);
3425                 pci_disable_device(pdev);
3426                 return rc;
3427         }
3428         hisi_hba->hw->phys_init(hisi_hba);
3429         sas_resume_ha(sha);
3430         clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
3431 
3432         return 0;
3433 }
3434 
3435 static const struct pci_device_id sas_v3_pci_table[] = {
3436         { PCI_VDEVICE(HUAWEI, 0xa230), hip08 },
3437         {}
3438 };
3439 MODULE_DEVICE_TABLE(pci, sas_v3_pci_table);
3440 
3441 static const struct pci_error_handlers hisi_sas_err_handler = {
3442         .reset_prepare  = hisi_sas_reset_prepare_v3_hw,
3443         .reset_done     = hisi_sas_reset_done_v3_hw,
3444 };
3445 
3446 static struct pci_driver sas_v3_pci_driver = {
3447         .name           = DRV_NAME,
3448         .id_table       = sas_v3_pci_table,
3449         .probe          = hisi_sas_v3_probe,
3450         .remove         = hisi_sas_v3_remove,
3451         .suspend        = hisi_sas_v3_suspend,
3452         .resume         = hisi_sas_v3_resume,
3453         .err_handler    = &hisi_sas_err_handler,
3454 };
3455 
3456 module_pci_driver(sas_v3_pci_driver);
3457 module_param_named(intr_conv, hisi_sas_intr_conv, bool, 0444);
3458 
3459 MODULE_LICENSE("GPL");
3460 MODULE_AUTHOR("John Garry <john.garry@huawei.com>");
3461 MODULE_DESCRIPTION("HISILICON SAS controller v3 hw driver based on pci device");
3462 MODULE_ALIAS("pci:" DRV_NAME);

/* [<][>][^][v][top][bottom][index][help] */