hr_dev 45 drivers/infiniband/hw/hns/hns_roce_ah.c struct hns_roce_dev *hr_dev = to_hr_dev(ibah->device); hr_dev 47 drivers/infiniband/hw/hns/hns_roce_ah.c struct device *dev = hr_dev->dev; hr_dev 160 drivers/infiniband/hw/hns/hns_roce_alloc.c void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size, hr_dev 164 drivers/infiniband/hw/hns/hns_roce_alloc.c struct device *dev = hr_dev->dev; hr_dev 178 drivers/infiniband/hw/hns/hns_roce_alloc.c int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, hr_dev 183 drivers/infiniband/hw/hns/hns_roce_alloc.c struct device *dev = hr_dev->dev; hr_dev 236 drivers/infiniband/hw/hns/hns_roce_alloc.c hns_roce_buf_free(hr_dev, size, buf); hr_dev 240 drivers/infiniband/hw/hns/hns_roce_alloc.c int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, hr_dev 248 drivers/infiniband/hw/hns/hns_roce_alloc.c dev_err(hr_dev->dev, hr_dev 265 drivers/infiniband/hw/hns/hns_roce_alloc.c int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, hr_dev 275 drivers/infiniband/hw/hns/hns_roce_alloc.c dev_err(hr_dev->dev, "invalid page shift %d!\n", page_shift); hr_dev 338 drivers/infiniband/hw/hns/hns_roce_alloc.c void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev) hr_dev 340 drivers/infiniband/hw/hns/hns_roce_alloc.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) hr_dev 341 drivers/infiniband/hw/hns/hns_roce_alloc.c hns_roce_cleanup_srq_table(hr_dev); hr_dev 342 drivers/infiniband/hw/hns/hns_roce_alloc.c hns_roce_cleanup_qp_table(hr_dev); hr_dev 343 drivers/infiniband/hw/hns/hns_roce_alloc.c hns_roce_cleanup_cq_table(hr_dev); hr_dev 344 drivers/infiniband/hw/hns/hns_roce_alloc.c hns_roce_cleanup_mr_table(hr_dev); hr_dev 345 drivers/infiniband/hw/hns/hns_roce_alloc.c hns_roce_cleanup_pd_table(hr_dev); hr_dev 346 drivers/infiniband/hw/hns/hns_roce_alloc.c hns_roce_cleanup_uar_table(hr_dev); hr_dev 43 drivers/infiniband/hw/hns/hns_roce_cmd.c static int hns_roce_cmd_mbox_post_hw(struct hns_roce_dev *hr_dev, u64 in_param, hr_dev 48 drivers/infiniband/hw/hns/hns_roce_cmd.c struct hns_roce_cmdq *cmd = &hr_dev->cmd; hr_dev 52 drivers/infiniband/hw/hns/hns_roce_cmd.c ret = hr_dev->hw->post_mbox(hr_dev, in_param, out_param, in_modifier, hr_dev 60 drivers/infiniband/hw/hns/hns_roce_cmd.c static int __hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param, hr_dev 65 drivers/infiniband/hw/hns/hns_roce_cmd.c struct device *dev = hr_dev->dev; hr_dev 68 drivers/infiniband/hw/hns/hns_roce_cmd.c ret = hns_roce_cmd_mbox_post_hw(hr_dev, in_param, out_param, hr_dev 76 drivers/infiniband/hw/hns/hns_roce_cmd.c return hr_dev->hw->chk_mbox(hr_dev, timeout); hr_dev 79 drivers/infiniband/hw/hns/hns_roce_cmd.c static int hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param, hr_dev 85 drivers/infiniband/hw/hns/hns_roce_cmd.c down(&hr_dev->cmd.poll_sem); hr_dev 86 drivers/infiniband/hw/hns/hns_roce_cmd.c ret = __hns_roce_cmd_mbox_poll(hr_dev, in_param, out_param, in_modifier, hr_dev 88 drivers/infiniband/hw/hns/hns_roce_cmd.c up(&hr_dev->cmd.poll_sem); hr_dev 93 drivers/infiniband/hw/hns/hns_roce_cmd.c void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status, hr_dev 97 drivers/infiniband/hw/hns/hns_roce_cmd.c *context = &hr_dev->cmd.context[token & hr_dev->cmd.token_mask]; hr_dev 108 drivers/infiniband/hw/hns/hns_roce_cmd.c static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, hr_dev 113 drivers/infiniband/hw/hns/hns_roce_cmd.c struct hns_roce_cmdq *cmd = &hr_dev->cmd; hr_dev 115 drivers/infiniband/hw/hns/hns_roce_cmd.c struct device *dev = hr_dev->dev; hr_dev 127 drivers/infiniband/hw/hns/hns_roce_cmd.c ret = hns_roce_cmd_mbox_post_hw(hr_dev, in_param, out_param, hr_dev 160 drivers/infiniband/hw/hns/hns_roce_cmd.c static int hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, hr_dev 166 drivers/infiniband/hw/hns/hns_roce_cmd.c down(&hr_dev->cmd.event_sem); hr_dev 167 drivers/infiniband/hw/hns/hns_roce_cmd.c ret = __hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param, hr_dev 169 drivers/infiniband/hw/hns/hns_roce_cmd.c up(&hr_dev->cmd.event_sem); hr_dev 174 drivers/infiniband/hw/hns/hns_roce_cmd.c int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param, hr_dev 180 drivers/infiniband/hw/hns/hns_roce_cmd.c if (hr_dev->hw->rst_prc_mbox) { hr_dev 181 drivers/infiniband/hw/hns/hns_roce_cmd.c ret = hr_dev->hw->rst_prc_mbox(hr_dev); hr_dev 188 drivers/infiniband/hw/hns/hns_roce_cmd.c if (hr_dev->cmd.use_events) hr_dev 189 drivers/infiniband/hw/hns/hns_roce_cmd.c ret = hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param, hr_dev 193 drivers/infiniband/hw/hns/hns_roce_cmd.c ret = hns_roce_cmd_mbox_poll(hr_dev, in_param, out_param, hr_dev 200 drivers/infiniband/hw/hns/hns_roce_cmd.c if (ret && (hr_dev->hw->rst_prc_mbox && hr_dev 201 drivers/infiniband/hw/hns/hns_roce_cmd.c hr_dev->hw->rst_prc_mbox(hr_dev) == CMD_RST_PRC_SUCCESS)) hr_dev 207 drivers/infiniband/hw/hns/hns_roce_cmd.c int hns_roce_cmd_init(struct hns_roce_dev *hr_dev) hr_dev 209 drivers/infiniband/hw/hns/hns_roce_cmd.c struct device *dev = hr_dev->dev; hr_dev 211 drivers/infiniband/hw/hns/hns_roce_cmd.c mutex_init(&hr_dev->cmd.hcr_mutex); hr_dev 212 drivers/infiniband/hw/hns/hns_roce_cmd.c sema_init(&hr_dev->cmd.poll_sem, 1); hr_dev 213 drivers/infiniband/hw/hns/hns_roce_cmd.c hr_dev->cmd.use_events = 0; hr_dev 214 drivers/infiniband/hw/hns/hns_roce_cmd.c hr_dev->cmd.max_cmds = CMD_MAX_NUM; hr_dev 215 drivers/infiniband/hw/hns/hns_roce_cmd.c hr_dev->cmd.pool = dma_pool_create("hns_roce_cmd", dev, hr_dev 218 drivers/infiniband/hw/hns/hns_roce_cmd.c if (!hr_dev->cmd.pool) hr_dev 224 drivers/infiniband/hw/hns/hns_roce_cmd.c void hns_roce_cmd_cleanup(struct hns_roce_dev *hr_dev) hr_dev 226 drivers/infiniband/hw/hns/hns_roce_cmd.c dma_pool_destroy(hr_dev->cmd.pool); hr_dev 229 drivers/infiniband/hw/hns/hns_roce_cmd.c int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev) hr_dev 231 drivers/infiniband/hw/hns/hns_roce_cmd.c struct hns_roce_cmdq *hr_cmd = &hr_dev->cmd; hr_dev 257 drivers/infiniband/hw/hns/hns_roce_cmd.c void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev) hr_dev 259 drivers/infiniband/hw/hns/hns_roce_cmd.c struct hns_roce_cmdq *hr_cmd = &hr_dev->cmd; hr_dev 266 drivers/infiniband/hw/hns/hns_roce_cmd.c *hns_roce_alloc_cmd_mailbox(struct hns_roce_dev *hr_dev) hr_dev 274 drivers/infiniband/hw/hns/hns_roce_cmd.c mailbox->buf = dma_pool_alloc(hr_dev->cmd.pool, GFP_KERNEL, hr_dev 284 drivers/infiniband/hw/hns/hns_roce_cmd.c void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev, hr_dev 290 drivers/infiniband/hw/hns/hns_roce_cmd.c dma_pool_free(hr_dev->cmd.pool, mailbox->buf, mailbox->dma); hr_dev 142 drivers/infiniband/hw/hns/hns_roce_cmd.h int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param, hr_dev 147 drivers/infiniband/hw/hns/hns_roce_cmd.h *hns_roce_alloc_cmd_mailbox(struct hns_roce_dev *hr_dev); hr_dev 148 drivers/infiniband/hw/hns/hns_roce_cmd.h void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev, hr_dev 52 drivers/infiniband/hw/hns/hns_roce_cq.c struct hns_roce_dev *hr_dev; hr_dev 57 drivers/infiniband/hw/hns/hns_roce_cq.c hr_dev = to_hr_dev(ibcq->device); hr_dev 62 drivers/infiniband/hw/hns/hns_roce_cq.c dev_err(hr_dev->dev, hr_dev 84 drivers/infiniband/hw/hns/hns_roce_cq.c static int hns_roce_cq_alloc(struct hns_roce_dev *hr_dev, int nent, hr_dev 91 drivers/infiniband/hw/hns/hns_roce_cq.c struct device *dev = hr_dev->dev; hr_dev 96 drivers/infiniband/hw/hns/hns_roce_cq.c cq_table = &hr_dev->cq_table; hr_dev 99 drivers/infiniband/hw/hns/hns_roce_cq.c if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) hr_dev 100 drivers/infiniband/hw/hns/hns_roce_cq.c mtt_table = &hr_dev->mr_table.mtt_cqe_table; hr_dev 102 drivers/infiniband/hw/hns/hns_roce_cq.c mtt_table = &hr_dev->mr_table.mtt_table; hr_dev 104 drivers/infiniband/hw/hns/hns_roce_cq.c mtts = hns_roce_table_find(hr_dev, mtt_table, hr_dev 111 drivers/infiniband/hw/hns/hns_roce_cq.c if (vector >= hr_dev->caps.num_comp_vectors) { hr_dev 124 drivers/infiniband/hw/hns/hns_roce_cq.c ret = hns_roce_table_get(hr_dev, &cq_table->table, hr_cq->cqn); hr_dev 137 drivers/infiniband/hw/hns/hns_roce_cq.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 143 drivers/infiniband/hw/hns/hns_roce_cq.c hr_dev->hw->write_cqc(hr_dev, hr_cq, mailbox->buf, mtts, dma_handle, hr_dev 147 drivers/infiniband/hw/hns/hns_roce_cq.c ret = hns_roce_sw2hw_cq(hr_dev, mailbox, hr_cq->cqn); hr_dev 148 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 166 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); hr_dev 182 drivers/infiniband/hw/hns/hns_roce_cq.c void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) hr_dev 184 drivers/infiniband/hw/hns/hns_roce_cq.c struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; hr_dev 185 drivers/infiniband/hw/hns/hns_roce_cq.c struct device *dev = hr_dev->dev; hr_dev 188 drivers/infiniband/hw/hns/hns_roce_cq.c ret = hns_roce_hw2sw_cq(hr_dev, NULL, hr_cq->cqn); hr_dev 196 drivers/infiniband/hw/hns/hns_roce_cq.c synchronize_irq(hr_dev->eq_table.eq[hr_cq->vector].irq); hr_dev 203 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); hr_dev 207 drivers/infiniband/hw/hns/hns_roce_cq.c static int hns_roce_ib_get_cq_umem(struct hns_roce_dev *hr_dev, hr_dev 216 drivers/infiniband/hw/hns/hns_roce_cq.c *umem = ib_umem_get(udata, buf_addr, cqe * hr_dev->caps.cq_entry_sz, hr_dev 221 drivers/infiniband/hw/hns/hns_roce_cq.c if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) hr_dev 226 drivers/infiniband/hw/hns/hns_roce_cq.c if (hr_dev->caps.cqe_buf_pg_sz) { hr_dev 228 drivers/infiniband/hw/hns/hns_roce_cq.c (1 << hr_dev->caps.cqe_buf_pg_sz) - 1) / hr_dev 229 drivers/infiniband/hw/hns/hns_roce_cq.c (1 << hr_dev->caps.cqe_buf_pg_sz); hr_dev 230 drivers/infiniband/hw/hns/hns_roce_cq.c page_shift = PAGE_SHIFT + hr_dev->caps.cqe_buf_pg_sz; hr_dev 231 drivers/infiniband/hw/hns/hns_roce_cq.c ret = hns_roce_mtt_init(hr_dev, npages, page_shift, hr_dev 234 drivers/infiniband/hw/hns/hns_roce_cq.c ret = hns_roce_mtt_init(hr_dev, ib_umem_page_count(*umem), hr_dev 240 drivers/infiniband/hw/hns/hns_roce_cq.c ret = hns_roce_ib_umem_write_mtt(hr_dev, &buf->hr_mtt, *umem); hr_dev 247 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_mtt_cleanup(hr_dev, &buf->hr_mtt); hr_dev 254 drivers/infiniband/hw/hns/hns_roce_cq.c static int hns_roce_ib_alloc_cq_buf(struct hns_roce_dev *hr_dev, hr_dev 258 drivers/infiniband/hw/hns/hns_roce_cq.c u32 page_shift = PAGE_SHIFT + hr_dev->caps.cqe_buf_pg_sz; hr_dev 260 drivers/infiniband/hw/hns/hns_roce_cq.c ret = hns_roce_buf_alloc(hr_dev, nent * hr_dev->caps.cq_entry_sz, hr_dev 266 drivers/infiniband/hw/hns/hns_roce_cq.c if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) hr_dev 271 drivers/infiniband/hw/hns/hns_roce_cq.c ret = hns_roce_mtt_init(hr_dev, buf->hr_buf.npages, hr_dev 276 drivers/infiniband/hw/hns/hns_roce_cq.c ret = hns_roce_buf_write_mtt(hr_dev, &buf->hr_mtt, &buf->hr_buf); hr_dev 283 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_mtt_cleanup(hr_dev, &buf->hr_mtt); hr_dev 286 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_buf_free(hr_dev, nent * hr_dev->caps.cq_entry_sz, hr_dev 292 drivers/infiniband/hw/hns/hns_roce_cq.c static void hns_roce_ib_free_cq_buf(struct hns_roce_dev *hr_dev, hr_dev 295 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_buf_free(hr_dev, (cqe + 1) * hr_dev->caps.cq_entry_sz, hr_dev 299 drivers/infiniband/hw/hns/hns_roce_cq.c static int create_user_cq(struct hns_roce_dev *hr_dev, hr_dev 306 drivers/infiniband/hw/hns/hns_roce_cq.c struct device *dev = hr_dev->dev; hr_dev 317 drivers/infiniband/hw/hns/hns_roce_cq.c ret = hns_roce_ib_get_cq_umem(hr_dev, udata, &hr_cq->hr_buf, hr_dev 325 drivers/infiniband/hw/hns/hns_roce_cq.c if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && hr_dev 340 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); hr_dev 346 drivers/infiniband/hw/hns/hns_roce_cq.c static int create_kernel_cq(struct hns_roce_dev *hr_dev, hr_dev 349 drivers/infiniband/hw/hns/hns_roce_cq.c struct device *dev = hr_dev->dev; hr_dev 353 drivers/infiniband/hw/hns/hns_roce_cq.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) { hr_dev 354 drivers/infiniband/hw/hns/hns_roce_cq.c ret = hns_roce_alloc_db(hr_dev, &hr_cq->db, 1); hr_dev 364 drivers/infiniband/hw/hns/hns_roce_cq.c ret = hns_roce_ib_alloc_cq_buf(hr_dev, &hr_cq->hr_buf, cq_entries); hr_dev 370 drivers/infiniband/hw/hns/hns_roce_cq.c uar = &hr_dev->priv_uar; hr_dev 371 drivers/infiniband/hw/hns/hns_roce_cq.c hr_cq->cq_db_l = hr_dev->reg_base + hr_dev->odb_offset + hr_dev 377 drivers/infiniband/hw/hns/hns_roce_cq.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) hr_dev 378 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_free_db(hr_dev, &hr_cq->db); hr_dev 383 drivers/infiniband/hw/hns/hns_roce_cq.c static void destroy_user_cq(struct hns_roce_dev *hr_dev, hr_dev 391 drivers/infiniband/hw/hns/hns_roce_cq.c if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && hr_dev 395 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); hr_dev 399 drivers/infiniband/hw/hns/hns_roce_cq.c static void destroy_kernel_cq(struct hns_roce_dev *hr_dev, hr_dev 402 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); hr_dev 403 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf, hr_cq->ib_cq.cqe); hr_dev 405 drivers/infiniband/hw/hns/hns_roce_cq.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) hr_dev 406 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_free_db(hr_dev, &hr_cq->db); hr_dev 413 drivers/infiniband/hw/hns/hns_roce_cq.c struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); hr_dev 414 drivers/infiniband/hw/hns/hns_roce_cq.c struct device *dev = hr_dev->dev; hr_dev 421 drivers/infiniband/hw/hns/hns_roce_cq.c if (cq_entries < 1 || cq_entries > hr_dev->caps.max_cqes) { hr_dev 423 drivers/infiniband/hw/hns/hns_roce_cq.c cq_entries, hr_dev->caps.max_cqes); hr_dev 427 drivers/infiniband/hw/hns/hns_roce_cq.c if (hr_dev->caps.min_cqes) hr_dev 428 drivers/infiniband/hw/hns/hns_roce_cq.c cq_entries = max(cq_entries, hr_dev->caps.min_cqes); hr_dev 435 drivers/infiniband/hw/hns/hns_roce_cq.c ret = create_user_cq(hr_dev, hr_cq, udata, &resp, cq_entries); hr_dev 441 drivers/infiniband/hw/hns/hns_roce_cq.c ret = create_kernel_cq(hr_dev, hr_cq, cq_entries); hr_dev 449 drivers/infiniband/hw/hns/hns_roce_cq.c ret = hns_roce_cq_alloc(hr_dev, cq_entries, &hr_cq->hr_buf.hr_mtt, hr_dev 480 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_free_cq(hr_dev, hr_cq); hr_dev 484 drivers/infiniband/hw/hns/hns_roce_cq.c destroy_user_cq(hr_dev, hr_cq, udata, &resp); hr_dev 486 drivers/infiniband/hw/hns/hns_roce_cq.c destroy_kernel_cq(hr_dev, hr_cq); hr_dev 494 drivers/infiniband/hw/hns/hns_roce_cq.c struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); hr_dev 497 drivers/infiniband/hw/hns/hns_roce_cq.c if (hr_dev->hw->destroy_cq) { hr_dev 498 drivers/infiniband/hw/hns/hns_roce_cq.c hr_dev->hw->destroy_cq(ib_cq, udata); hr_dev 502 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_free_cq(hr_dev, hr_cq); hr_dev 503 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); hr_dev 515 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf, ib_cq->cqe); hr_dev 516 drivers/infiniband/hw/hns/hns_roce_cq.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) hr_dev 517 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_free_db(hr_dev, &hr_cq->db); hr_dev 521 drivers/infiniband/hw/hns/hns_roce_cq.c void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn) hr_dev 523 drivers/infiniband/hw/hns/hns_roce_cq.c struct device *dev = hr_dev->dev; hr_dev 526 drivers/infiniband/hw/hns/hns_roce_cq.c cq = xa_load(&hr_dev->cq_table.array, cqn & (hr_dev->caps.num_cqs - 1)); hr_dev 536 drivers/infiniband/hw/hns/hns_roce_cq.c void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) hr_dev 538 drivers/infiniband/hw/hns/hns_roce_cq.c struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; hr_dev 539 drivers/infiniband/hw/hns/hns_roce_cq.c struct device *dev = hr_dev->dev; hr_dev 542 drivers/infiniband/hw/hns/hns_roce_cq.c cq = xa_load(&cq_table->array, cqn & (hr_dev->caps.num_cqs - 1)); hr_dev 557 drivers/infiniband/hw/hns/hns_roce_cq.c int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev) hr_dev 559 drivers/infiniband/hw/hns/hns_roce_cq.c struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; hr_dev 563 drivers/infiniband/hw/hns/hns_roce_cq.c return hns_roce_bitmap_init(&cq_table->bitmap, hr_dev->caps.num_cqs, hr_dev 564 drivers/infiniband/hw/hns/hns_roce_cq.c hr_dev->caps.num_cqs - 1, hr_dev 565 drivers/infiniband/hw/hns/hns_roce_cq.c hr_dev->caps.reserved_cqs, 0); hr_dev 568 drivers/infiniband/hw/hns/hns_roce_cq.c void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev) hr_dev 570 drivers/infiniband/hw/hns/hns_roce_cq.c hns_roce_bitmap_cleanup(&hr_dev->cq_table.bitmap); hr_dev 125 drivers/infiniband/hw/hns/hns_roce_db.c int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db, hr_dev 131 drivers/infiniband/hw/hns/hns_roce_db.c mutex_lock(&hr_dev->pgdir_mutex); hr_dev 133 drivers/infiniband/hw/hns/hns_roce_db.c list_for_each_entry(pgdir, &hr_dev->pgdir_list, list) hr_dev 137 drivers/infiniband/hw/hns/hns_roce_db.c pgdir = hns_roce_alloc_db_pgdir(hr_dev->dev); hr_dev 143 drivers/infiniband/hw/hns/hns_roce_db.c list_add(&pgdir->list, &hr_dev->pgdir_list); hr_dev 149 drivers/infiniband/hw/hns/hns_roce_db.c mutex_unlock(&hr_dev->pgdir_mutex); hr_dev 154 drivers/infiniband/hw/hns/hns_roce_db.c void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db) hr_dev 159 drivers/infiniband/hw/hns/hns_roce_db.c mutex_lock(&hr_dev->pgdir_mutex); hr_dev 174 drivers/infiniband/hw/hns/hns_roce_db.c dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page, hr_dev 180 drivers/infiniband/hw/hns/hns_roce_db.c mutex_unlock(&hr_dev->pgdir_mutex); hr_dev 754 drivers/infiniband/hw/hns/hns_roce_device.h struct hns_roce_dev *hr_dev; hr_dev 915 drivers/infiniband/hw/hns/hns_roce_device.h struct hns_roce_dev *hr_dev; hr_dev 924 drivers/infiniband/hw/hns/hns_roce_device.h int (*query_cqc_info)(struct hns_roce_dev *hr_dev, u32 cqn, hr_dev 929 drivers/infiniband/hw/hns/hns_roce_device.h int (*reset)(struct hns_roce_dev *hr_dev, bool enable); hr_dev 930 drivers/infiniband/hw/hns/hns_roce_device.h int (*cmq_init)(struct hns_roce_dev *hr_dev); hr_dev 931 drivers/infiniband/hw/hns/hns_roce_device.h void (*cmq_exit)(struct hns_roce_dev *hr_dev); hr_dev 932 drivers/infiniband/hw/hns/hns_roce_device.h int (*hw_profile)(struct hns_roce_dev *hr_dev); hr_dev 933 drivers/infiniband/hw/hns/hns_roce_device.h int (*hw_init)(struct hns_roce_dev *hr_dev); hr_dev 934 drivers/infiniband/hw/hns/hns_roce_device.h void (*hw_exit)(struct hns_roce_dev *hr_dev); hr_dev 935 drivers/infiniband/hw/hns/hns_roce_device.h int (*post_mbox)(struct hns_roce_dev *hr_dev, u64 in_param, hr_dev 938 drivers/infiniband/hw/hns/hns_roce_device.h int (*chk_mbox)(struct hns_roce_dev *hr_dev, unsigned long timeout); hr_dev 939 drivers/infiniband/hw/hns/hns_roce_device.h int (*rst_prc_mbox)(struct hns_roce_dev *hr_dev); hr_dev 940 drivers/infiniband/hw/hns/hns_roce_device.h int (*set_gid)(struct hns_roce_dev *hr_dev, u8 port, int gid_index, hr_dev 942 drivers/infiniband/hw/hns/hns_roce_device.h int (*set_mac)(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr); hr_dev 943 drivers/infiniband/hw/hns/hns_roce_device.h void (*set_mtu)(struct hns_roce_dev *hr_dev, u8 phy_port, hr_dev 947 drivers/infiniband/hw/hns/hns_roce_device.h int (*rereg_write_mtpt)(struct hns_roce_dev *hr_dev, hr_dev 953 drivers/infiniband/hw/hns/hns_roce_device.h void (*write_cqc)(struct hns_roce_dev *hr_dev, hr_dev 956 drivers/infiniband/hw/hns/hns_roce_device.h int (*set_hem)(struct hns_roce_dev *hr_dev, hr_dev 958 drivers/infiniband/hw/hns/hns_roce_device.h int (*clear_hem)(struct hns_roce_dev *hr_dev, hr_dev 967 drivers/infiniband/hw/hns/hns_roce_device.h int (*qp_flow_control_init)(struct hns_roce_dev *hr_dev, hr_dev 975 drivers/infiniband/hw/hns/hns_roce_device.h int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, hr_dev 979 drivers/infiniband/hw/hns/hns_roce_device.h int (*init_eq)(struct hns_roce_dev *hr_dev); hr_dev 980 drivers/infiniband/hw/hns/hns_roce_device.h void (*cleanup_eq)(struct hns_roce_dev *hr_dev); hr_dev 981 drivers/infiniband/hw/hns/hns_roce_device.h void (*write_srqc)(struct hns_roce_dev *hr_dev, hr_dev 1105 drivers/infiniband/hw/hns/hns_roce_device.h *__hns_roce_qp_lookup(struct hns_roce_dev *hr_dev, u32 qpn) hr_dev 1107 drivers/infiniband/hw/hns/hns_roce_device.h return xa_load(&hr_dev->qp_table_xa, qpn & (hr_dev->caps.num_qps - 1)); hr_dev 1126 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_cmd_init(struct hns_roce_dev *hr_dev); hr_dev 1127 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_cmd_cleanup(struct hns_roce_dev *hr_dev); hr_dev 1128 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status, hr_dev 1130 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev); hr_dev 1131 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev); hr_dev 1133 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift, hr_dev 1135 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, hr_dev 1137 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev, hr_dev 1142 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_mtr_attach(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, hr_dev 1145 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_mtr_cleanup(struct hns_roce_dev *hr_dev, hr_dev 1150 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, hr_dev 1153 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev); hr_dev 1154 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev); hr_dev 1155 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_init_eq_table(struct hns_roce_dev *hr_dev); hr_dev 1156 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev); hr_dev 1157 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev); hr_dev 1158 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev); hr_dev 1160 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_cleanup_pd_table(struct hns_roce_dev *hr_dev); hr_dev 1161 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev); hr_dev 1162 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_cleanup_eq_table(struct hns_roce_dev *hr_dev); hr_dev 1163 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev); hr_dev 1164 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev); hr_dev 1165 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_cleanup_srq_table(struct hns_roce_dev *hr_dev); hr_dev 1173 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev); hr_dev 1200 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev, hr_dev 1209 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size, hr_dev 1211 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, hr_dev 1214 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev, hr_dev 1223 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, hr_dev 1225 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, hr_dev 1252 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp); hr_dev 1253 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp); hr_dev 1254 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn, hr_dev 1264 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq); hr_dev 1271 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db, hr_dev 1273 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db); hr_dev 1275 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn); hr_dev 1276 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type); hr_dev 1277 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type); hr_dev 1278 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type); hr_dev 1279 drivers/infiniband/hw/hns/hns_roce_device.h int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index); hr_dev 1280 drivers/infiniband/hw/hns/hns_roce_device.h int hns_roce_init(struct hns_roce_dev *hr_dev); hr_dev 1281 drivers/infiniband/hw/hns/hns_roce_device.h void hns_roce_exit(struct hns_roce_dev *hr_dev); hr_dev 42 drivers/infiniband/hw/hns/hns_roce_hem.c bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) hr_dev 48 drivers/infiniband/hw/hns/hns_roce_hem.c hop_num = hr_dev->caps.qpc_hop_num; hr_dev 51 drivers/infiniband/hw/hns/hns_roce_hem.c hop_num = hr_dev->caps.mpt_hop_num; hr_dev 54 drivers/infiniband/hw/hns/hns_roce_hem.c hop_num = hr_dev->caps.cqc_hop_num; hr_dev 57 drivers/infiniband/hw/hns/hns_roce_hem.c hop_num = hr_dev->caps.srqc_hop_num; hr_dev 60 drivers/infiniband/hw/hns/hns_roce_hem.c hop_num = hr_dev->caps.sccc_hop_num; hr_dev 63 drivers/infiniband/hw/hns/hns_roce_hem.c hop_num = hr_dev->caps.qpc_timer_hop_num; hr_dev 66 drivers/infiniband/hw/hns/hns_roce_hem.c hop_num = hr_dev->caps.cqc_timer_hop_num; hr_dev 69 drivers/infiniband/hw/hns/hns_roce_hem.c hop_num = hr_dev->caps.cqe_hop_num; hr_dev 72 drivers/infiniband/hw/hns/hns_roce_hem.c hop_num = hr_dev->caps.mtt_hop_num; hr_dev 75 drivers/infiniband/hw/hns/hns_roce_hem.c hop_num = hr_dev->caps.srqwqe_hop_num; hr_dev 78 drivers/infiniband/hw/hns/hns_roce_hem.c hop_num = hr_dev->caps.idx_hop_num; hr_dev 123 drivers/infiniband/hw/hns/hns_roce_hem.c static int get_hem_table_config(struct hns_roce_dev *hr_dev, hr_dev 127 drivers/infiniband/hw/hns/hns_roce_hem.c struct device *dev = hr_dev->dev; hr_dev 131 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz hr_dev 133 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz hr_dev 135 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->ba_l0_num = hr_dev->caps.qpc_bt_num; hr_dev 136 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->hop_num = hr_dev->caps.qpc_hop_num; hr_dev 139 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz hr_dev 141 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz hr_dev 143 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->ba_l0_num = hr_dev->caps.mpt_bt_num; hr_dev 144 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->hop_num = hr_dev->caps.mpt_hop_num; hr_dev 147 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz hr_dev 149 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz hr_dev 151 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->ba_l0_num = hr_dev->caps.cqc_bt_num; hr_dev 152 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->hop_num = hr_dev->caps.cqc_hop_num; hr_dev 155 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz hr_dev 157 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz hr_dev 159 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->ba_l0_num = hr_dev->caps.sccc_bt_num; hr_dev 160 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->hop_num = hr_dev->caps.sccc_hop_num; hr_dev 163 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz hr_dev 165 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz hr_dev 167 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num; hr_dev 168 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->hop_num = hr_dev->caps.qpc_timer_hop_num; hr_dev 171 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz hr_dev 173 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz hr_dev 175 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num; hr_dev 176 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->hop_num = hr_dev->caps.cqc_timer_hop_num; hr_dev 179 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz hr_dev 181 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz hr_dev 183 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->ba_l0_num = hr_dev->caps.srqc_bt_num; hr_dev 184 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->hop_num = hr_dev->caps.srqc_hop_num; hr_dev 187 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz hr_dev 189 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz hr_dev 192 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->hop_num = hr_dev->caps.mtt_hop_num; hr_dev 195 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz hr_dev 197 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz hr_dev 200 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->hop_num = hr_dev->caps.cqe_hop_num; hr_dev 203 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->buf_chunk_size = 1 << (hr_dev->caps.srqwqe_buf_pg_sz hr_dev 205 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->bt_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz hr_dev 208 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->hop_num = hr_dev->caps.srqwqe_hop_num; hr_dev 211 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->buf_chunk_size = 1 << (hr_dev->caps.idx_buf_pg_sz hr_dev 213 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->bt_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz hr_dev 216 drivers/infiniband/hw/hns/hns_roce_hem.c mhop->hop_num = hr_dev->caps.idx_hop_num; hr_dev 227 drivers/infiniband/hw/hns/hns_roce_hem.c int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, hr_dev 231 drivers/infiniband/hw/hns/hns_roce_hem.c struct device *dev = hr_dev->dev; hr_dev 237 drivers/infiniband/hw/hns/hns_roce_hem.c if (get_hem_table_config(hr_dev, mhop, table->type)) hr_dev 277 drivers/infiniband/hw/hns/hns_roce_hem.c static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, hr_dev 322 drivers/infiniband/hw/hns/hns_roce_hem.c buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order, hr_dev 338 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_free_hem(hr_dev, hem); hr_dev 342 drivers/infiniband/hw/hns/hns_roce_hem.c void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem) hr_dev 352 drivers/infiniband/hw/hns/hns_roce_hem.c dma_free_coherent(hr_dev->dev, hr_dev 362 drivers/infiniband/hw/hns/hns_roce_hem.c static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, hr_dev 365 drivers/infiniband/hw/hns/hns_roce_hem.c spinlock_t *lock = &hr_dev->bt_cmd_lock; hr_dev 366 drivers/infiniband/hw/hns/hns_roce_hem.c struct device *dev = hr_dev->dev; hr_dev 405 drivers/infiniband/hw/hns/hns_roce_hem.c bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; hr_dev 430 drivers/infiniband/hw/hns/hns_roce_hem.c hr_dev->reg_base + ROCEE_BT_CMD_L_REG); hr_dev 437 drivers/infiniband/hw/hns/hns_roce_hem.c static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, hr_dev 441 drivers/infiniband/hw/hns/hns_roce_hem.c struct device *dev = hr_dev->dev; hr_dev 460 drivers/infiniband/hw/hns/hns_roce_hem.c ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); hr_dev 519 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { hr_dev 543 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { hr_dev 555 drivers/infiniband/hw/hns/hns_roce_hem.c table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev, hr_dev 583 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { hr_dev 614 drivers/infiniband/hw/hns/hns_roce_hem.c int hns_roce_table_get(struct hns_roce_dev *hr_dev, hr_dev 617 drivers/infiniband/hw/hns/hns_roce_hem.c struct device *dev = hr_dev->dev; hr_dev 621 drivers/infiniband/hw/hns/hns_roce_hem.c if (hns_roce_check_whether_mhop(hr_dev, table->type)) hr_dev 622 drivers/infiniband/hw/hns/hns_roce_hem.c return hns_roce_table_mhop_get(hr_dev, table, obj); hr_dev 634 drivers/infiniband/hw/hns/hns_roce_hem.c table->hem[i] = hns_roce_alloc_hem(hr_dev, hr_dev 645 drivers/infiniband/hw/hns/hns_roce_hem.c if (hns_roce_set_hem(hr_dev, table, obj)) { hr_dev 646 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_free_hem(hr_dev, table->hem[i]); hr_dev 659 drivers/infiniband/hw/hns/hns_roce_hem.c static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, hr_dev 664 drivers/infiniband/hw/hns/hns_roce_hem.c struct device *dev = hr_dev->dev; hr_dev 676 drivers/infiniband/hw/hns/hns_roce_hem.c ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); hr_dev 711 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) hr_dev 714 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2)) hr_dev 718 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) hr_dev 726 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_free_hem(hr_dev, table->hem[hem_idx]); hr_dev 734 drivers/infiniband/hw/hns/hns_roce_hem.c hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) hr_dev 747 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) hr_dev 758 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->hw->clear_hem(hr_dev, table, obj, hr_dev 773 drivers/infiniband/hw/hns/hns_roce_hem.c void hns_roce_table_put(struct hns_roce_dev *hr_dev, hr_dev 776 drivers/infiniband/hw/hns/hns_roce_hem.c struct device *dev = hr_dev->dev; hr_dev 779 drivers/infiniband/hw/hns/hns_roce_hem.c if (hns_roce_check_whether_mhop(hr_dev, table->type)) { hr_dev 780 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_table_mhop_put(hr_dev, table, obj, 1); hr_dev 791 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) hr_dev 794 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_free_hem(hr_dev, table->hem[i]); hr_dev 801 drivers/infiniband/hw/hns/hns_roce_hem.c void *hns_roce_table_find(struct hns_roce_dev *hr_dev, hr_dev 822 drivers/infiniband/hw/hns/hns_roce_hem.c if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { hr_dev 830 drivers/infiniband/hw/hns/hns_roce_hem.c if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop)) hr_dev 874 drivers/infiniband/hw/hns/hns_roce_hem.c int hns_roce_table_get_range(struct hns_roce_dev *hr_dev, hr_dev 883 drivers/infiniband/hw/hns/hns_roce_hem.c if (hns_roce_check_whether_mhop(hr_dev, table->type)) { hr_dev 884 drivers/infiniband/hw/hns/hns_roce_hem.c ret = hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); hr_dev 892 drivers/infiniband/hw/hns/hns_roce_hem.c ret = hns_roce_table_get(hr_dev, table, i); hr_dev 902 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_table_put(hr_dev, table, i); hr_dev 907 drivers/infiniband/hw/hns/hns_roce_hem.c void hns_roce_table_put_range(struct hns_roce_dev *hr_dev, hr_dev 915 drivers/infiniband/hw/hns/hns_roce_hem.c if (hns_roce_check_whether_mhop(hr_dev, table->type)) { hr_dev 916 drivers/infiniband/hw/hns/hns_roce_hem.c if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) hr_dev 922 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_table_put(hr_dev, table, i); hr_dev 925 drivers/infiniband/hw/hns/hns_roce_hem.c int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, hr_dev 933 drivers/infiniband/hw/hns/hns_roce_hem.c if (!hns_roce_check_whether_mhop(hr_dev, type)) { hr_dev 934 drivers/infiniband/hw/hns/hns_roce_hem.c table->table_chunk_size = hr_dev->caps.chunk_sz; hr_dev 949 drivers/infiniband/hw/hns/hns_roce_hem.c if (get_hem_table_config(hr_dev, &mhop, type)) hr_dev 1031 drivers/infiniband/hw/hns/hns_roce_hem.c static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev, hr_dev 1039 drivers/infiniband/hw/hns/hns_roce_hem.c if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) hr_dev 1047 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_table_mhop_put(hr_dev, table, obj, 0); hr_dev 1062 drivers/infiniband/hw/hns/hns_roce_hem.c void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, hr_dev 1065 drivers/infiniband/hw/hns/hns_roce_hem.c struct device *dev = hr_dev->dev; hr_dev 1068 drivers/infiniband/hw/hns/hns_roce_hem.c if (hns_roce_check_whether_mhop(hr_dev, table->type)) { hr_dev 1069 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_cleanup_mhop_hem_table(hr_dev, table); hr_dev 1075 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->hw->clear_hem(hr_dev, table, hr_dev 1079 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_free_hem(hr_dev, table->hem[i]); hr_dev 1085 drivers/infiniband/hw/hns/hns_roce_hem.c void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) hr_dev 1087 drivers/infiniband/hw/hns/hns_roce_hem.c if ((hr_dev->caps.num_idx_segs)) hr_dev 1088 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_cleanup_hem_table(hr_dev, hr_dev 1089 drivers/infiniband/hw/hns/hns_roce_hem.c &hr_dev->mr_table.mtt_idx_table); hr_dev 1090 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->caps.num_srqwqe_segs) hr_dev 1091 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_cleanup_hem_table(hr_dev, hr_dev 1092 drivers/infiniband/hw/hns/hns_roce_hem.c &hr_dev->mr_table.mtt_srqwqe_table); hr_dev 1093 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->caps.srqc_entry_sz) hr_dev 1094 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_cleanup_hem_table(hr_dev, hr_dev 1095 drivers/infiniband/hw/hns/hns_roce_hem.c &hr_dev->srq_table.table); hr_dev 1096 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); hr_dev 1097 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->caps.qpc_timer_entry_sz) hr_dev 1098 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_cleanup_hem_table(hr_dev, hr_dev 1099 drivers/infiniband/hw/hns/hns_roce_hem.c &hr_dev->qpc_timer_table); hr_dev 1100 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->caps.cqc_timer_entry_sz) hr_dev 1101 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_cleanup_hem_table(hr_dev, hr_dev 1102 drivers/infiniband/hw/hns/hns_roce_hem.c &hr_dev->cqc_timer_table); hr_dev 1103 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->caps.sccc_entry_sz) hr_dev 1104 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_cleanup_hem_table(hr_dev, hr_dev 1105 drivers/infiniband/hw/hns/hns_roce_hem.c &hr_dev->qp_table.sccc_table); hr_dev 1106 drivers/infiniband/hw/hns/hns_roce_hem.c if (hr_dev->caps.trrl_entry_sz) hr_dev 1107 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_cleanup_hem_table(hr_dev, hr_dev 1108 drivers/infiniband/hw/hns/hns_roce_hem.c &hr_dev->qp_table.trrl_table); hr_dev 1109 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); hr_dev 1110 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); hr_dev 1111 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); hr_dev 1112 drivers/infiniband/hw/hns/hns_roce_hem.c if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) hr_dev 1113 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_cleanup_hem_table(hr_dev, hr_dev 1114 drivers/infiniband/hw/hns/hns_roce_hem.c &hr_dev->mr_table.mtt_cqe_table); hr_dev 1115 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table); hr_dev 1128 drivers/infiniband/hw/hns/hns_roce_hem.c static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev, hr_dev 1140 drivers/infiniband/hw/hns/hns_roce_hem.c hem->addr = dma_alloc_coherent(hr_dev->dev, hr_dev 1158 drivers/infiniband/hw/hns/hns_roce_hem.c static void hem_list_free_item(struct hns_roce_dev *hr_dev, hr_dev 1162 drivers/infiniband/hw/hns/hns_roce_hem.c dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN, hr_dev 1167 drivers/infiniband/hw/hns/hns_roce_hem.c static void hem_list_free_all(struct hns_roce_dev *hr_dev, hr_dev 1174 drivers/infiniband/hw/hns/hns_roce_hem.c hem_list_free_item(hr_dev, hem, exist_bt); hr_dev 1178 drivers/infiniband/hw/hns/hns_roce_hem.c static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr, hr_dev 1185 drivers/infiniband/hw/hns/hns_roce_hem.c static void hem_list_assign_bt(struct hns_roce_dev *hr_dev, hr_dev 1288 drivers/infiniband/hw/hns/hns_roce_hem.c static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, hr_dev 1309 drivers/infiniband/hw/hns/hns_roce_hem.c dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum); hr_dev 1314 drivers/infiniband/hw/hns/hns_roce_hem.c dev_err(hr_dev->dev, "invalid offset %d,min %d!\n", hr_dev 1340 drivers/infiniband/hw/hns/hns_roce_hem.c cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit, hr_dev 1355 drivers/infiniband/hw/hns/hns_roce_hem.c hem_list_link_bt(hr_dev, pre->addr + step, hr_dev 1368 drivers/infiniband/hw/hns/hns_roce_hem.c hem_list_free_all(hr_dev, &temp_list[level], true); hr_dev 1373 drivers/infiniband/hw/hns/hns_roce_hem.c static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, hr_dev 1400 drivers/infiniband/hw/hns/hns_roce_hem.c root_hem = hem_list_alloc_item(hr_dev, total, r->offset + r->count - 1, hr_dev 1426 drivers/infiniband/hw/hns/hns_roce_hem.c hem = hem_list_alloc_item(hr_dev, r->offset, hr_dev 1433 drivers/infiniband/hw/hns/hns_roce_hem.c hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); hr_dev 1447 drivers/infiniband/hw/hns/hns_roce_hem.c hem_list_link_bt(hr_dev, cpu_base + offset, hr_dev 1463 drivers/infiniband/hw/hns/hns_roce_hem.c hem_list_free_all(hr_dev, &temp_list[i], false); hr_dev 1465 drivers/infiniband/hw/hns/hns_roce_hem.c hem_list_free_all(hr_dev, &temp_root, true); hr_dev 1471 drivers/infiniband/hw/hns/hns_roce_hem.c int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, hr_dev 1483 drivers/infiniband/hw/hns/hns_roce_hem.c dev_err(hr_dev->dev, "invalid region region_cnt %d!\n", hr_dev 1496 drivers/infiniband/hw/hns/hns_roce_hem.c ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs, hr_dev 1500 drivers/infiniband/hw/hns/hns_roce_hem.c dev_err(hr_dev->dev, hr_dev 1507 drivers/infiniband/hw/hns/hns_roce_hem.c ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions, hr_dev 1510 drivers/infiniband/hw/hns/hns_roce_hem.c dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret); hr_dev 1515 drivers/infiniband/hw/hns/hns_roce_hem.c hns_roce_hem_list_release(hr_dev, hem_list); hr_dev 1520 drivers/infiniband/hw/hns/hns_roce_hem.c void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, hr_dev 1527 drivers/infiniband/hw/hns/hns_roce_hem.c hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j], hr_dev 1530 drivers/infiniband/hw/hns/hns_roce_hem.c hem_list_free_all(hr_dev, &hem_list->root_bt, true); hr_dev 1549 drivers/infiniband/hw/hns/hns_roce_hem.c void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, hr_dev 110 drivers/infiniband/hw/hns/hns_roce_hem.h void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem); hr_dev 111 drivers/infiniband/hw/hns/hns_roce_hem.h int hns_roce_table_get(struct hns_roce_dev *hr_dev, hr_dev 113 drivers/infiniband/hw/hns/hns_roce_hem.h void hns_roce_table_put(struct hns_roce_dev *hr_dev, hr_dev 115 drivers/infiniband/hw/hns/hns_roce_hem.h void *hns_roce_table_find(struct hns_roce_dev *hr_dev, hr_dev 118 drivers/infiniband/hw/hns/hns_roce_hem.h int hns_roce_table_get_range(struct hns_roce_dev *hr_dev, hr_dev 121 drivers/infiniband/hw/hns/hns_roce_hem.h void hns_roce_table_put_range(struct hns_roce_dev *hr_dev, hr_dev 124 drivers/infiniband/hw/hns/hns_roce_hem.h int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, hr_dev 128 drivers/infiniband/hw/hns/hns_roce_hem.h void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, hr_dev 130 drivers/infiniband/hw/hns/hns_roce_hem.h void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev); hr_dev 131 drivers/infiniband/hw/hns/hns_roce_hem.h int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, hr_dev 134 drivers/infiniband/hw/hns/hns_roce_hem.h bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type); hr_dev 140 drivers/infiniband/hw/hns/hns_roce_hem.h int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, hr_dev 144 drivers/infiniband/hw/hns/hns_roce_hem.h void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, hr_dev 146 drivers/infiniband/hw/hns/hns_roce_hem.h void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, hr_dev 65 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 71 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 141 drivers/infiniband/hw/hns/hns_roce_hw_v1.c smac = (u8 *)hr_dev->dev_addr[qp->port]; hr_dev 187 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_get_gid_index(hr_dev, qp->phy_port, hr_dev 288 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->caps.max_sq_inline) { hr_dev 293 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->caps.max_sq_inline); hr_dev 352 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 353 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 445 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_set_db_event_mode(struct hns_roce_dev *hr_dev, hr_dev 451 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, ROCEE_GLB_CFG_REG); hr_dev 456 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_GLB_CFG_REG, val); hr_dev 459 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_set_db_ext_mode(struct hns_roce_dev *hr_dev, u32 sdb_mode, hr_dev 466 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, ROCEE_GLB_CFG_REG); hr_dev 471 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_GLB_CFG_REG, val); hr_dev 474 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_set_sdb(struct hns_roce_dev *hr_dev, u32 sdb_alept, hr_dev 481 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, ROCEE_DB_SQ_WL_REG); hr_dev 488 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_DB_SQ_WL_REG, val); hr_dev 491 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_set_odb(struct hns_roce_dev *hr_dev, u32 odb_alept, hr_dev 498 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, ROCEE_DB_OTHERS_WL_REG); hr_dev 505 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_DB_OTHERS_WL_REG, val); hr_dev 508 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept, hr_dev 511 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 518 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 522 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_EMPTY_REG, ext_sdb_alept); hr_dev 523 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_REG, ext_sdb_alful); hr_dev 527 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_EXT_DB_SQ_REG, (u32)(sdb_dma_addr >> 12)); hr_dev 530 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, ROCEE_EXT_DB_SQ_H_REG); hr_dev 543 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_EXT_DB_SQ_H_REG, val); hr_dev 550 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_set_odb_ext(struct hns_roce_dev *hr_dev, u32 ext_odb_alept, hr_dev 553 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 560 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 564 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG, ext_odb_alept); hr_dev 565 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_REG, ext_odb_alful); hr_dev 569 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_EXT_DB_OTH_REG, (u32)(odb_dma_addr >> 12)); hr_dev 572 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, ROCEE_EXT_DB_OTH_H_REG); hr_dev 581 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_EXT_DB_OTH_H_REG, val); hr_dev 588 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_db_ext_init(struct hns_roce_dev *hr_dev, u32 sdb_ext_mod, hr_dev 591 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 598 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 623 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_set_sdb_ext(hr_dev, HNS_ROCE_V1_EXT_SDB_ALEPT, hr_dev 626 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_set_sdb(hr_dev, HNS_ROCE_V1_SDB_ALEPT, hr_dev 647 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_set_odb_ext(hr_dev, HNS_ROCE_V1_EXT_ODB_ALEPT, hr_dev 650 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_set_odb(hr_dev, HNS_ROCE_V1_ODB_ALEPT, hr_dev 653 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_set_db_ext_mode(hr_dev, sdb_ext_mod, odb_ext_mod); hr_dev 676 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static struct hns_roce_qp *hns_roce_v1_create_lp_qp(struct hns_roce_dev *hr_dev, hr_dev 679 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 698 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) hr_dev 700 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_caps *caps = &hr_dev->caps; hr_dev 701 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 720 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 727 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ibdev = &hr_dev->ib_dev; hr_dev 738 drivers/infiniband/hw/hns/hns_roce_hw_v1.c free_mr->mr_free_cq->ib_cq.device = &hr_dev->ib_dev; hr_dev 757 drivers/infiniband/hw/hns/hns_roce_hw_v1.c free_mr->mr_free_pd->ibpd.device = &hr_dev->ib_dev; hr_dev 786 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (hr_dev->iboe.phy_port[j] == phy_port) { hr_dev 796 drivers/infiniband/hw/hns/hns_roce_hw_v1.c free_mr->mr_free_qp[i] = hns_roce_v1_create_lp_qp(hr_dev, pd); hr_dev 807 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_qp->ibqp.device = &hr_dev->ib_dev; hr_dev 820 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->dev_addr[port], hr_dev 824 drivers/infiniband/hw/hns/hns_roce_hw_v1.c memcpy(&dgid.raw[8], hr_dev->dev_addr[port], 3); hr_dev 825 drivers/infiniband/hw/hns/hns_roce_hw_v1.c memcpy(&dgid.raw[13], hr_dev->dev_addr[port] + 3, 3); hr_dev 831 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask, hr_dev 838 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, IB_QP_DEST_QPN, hr_dev 845 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask, hr_dev 874 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev) hr_dev 876 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 883 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 903 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_db_init(struct hns_roce_dev *hr_dev) hr_dev 905 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 914 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 929 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hns_roce_db_ext_init(hr_dev, sdb_ext_mod, odb_ext_mod); hr_dev 935 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_set_db_event_mode(hr_dev, sdb_evt_mod, odb_evt_mod); hr_dev 943 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev; hr_dev 947 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev = to_hr_dev(lp_qp_work->ib_dev); hr_dev 949 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_v1_release_lp_qp(hr_dev); hr_dev 951 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (hns_roce_v1_rsv_lp_qp(hr_dev)) hr_dev 952 drivers/infiniband/hw/hns/hns_roce_hw_v1.c dev_err(&hr_dev->pdev->dev, "create reserver qp failed\n"); hr_dev 960 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev) hr_dev 962 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 969 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 979 drivers/infiniband/hw/hns/hns_roce_hw_v1.c lp_qp_work->ib_dev = &(hr_dev->ib_dev); hr_dev 1004 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device); hr_dev 1005 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 1034 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev; hr_dev 1046 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev = to_hr_dev(mr_work->ib_dev); hr_dev 1047 drivers/infiniband/hw/hns/hns_roce_hw_v1.c dev = &hr_dev->pdev->dev; hr_dev 1049 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 1097 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev, hr_dev 1100 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 1110 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 1114 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mr->key) hr_dev 1115 drivers/infiniband/hw/hns/hns_roce_hw_v1.c & (hr_dev->caps.num_mtpts - 1))) hr_dev 1127 drivers/infiniband/hw/hns/hns_roce_hw_v1.c mr_work->ib_dev = &(hr_dev->ib_dev); hr_dev 1159 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, hr_dev 1169 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_db_free(struct hns_roce_dev *hr_dev) hr_dev 1171 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 1175 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 1195 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_raq_init(struct hns_roce_dev *hr_dev) hr_dev 1204 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 1206 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 1222 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_EXT_RAQ_REG, raq->e_raq_buf->map >> 12); hr_dev 1226 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, ROCEE_EXT_RAQ_H_REG); hr_dev 1239 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_EXT_RAQ_H_REG, val); hr_dev 1243 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, ROCEE_RAQ_WL_REG); hr_dev 1249 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_RAQ_WL_REG, val); hr_dev 1253 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG); hr_dev 1267 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG, val); hr_dev 1271 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, ROCEE_GLB_CFG_REG); hr_dev 1275 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_GLB_CFG_REG, val); hr_dev 1285 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_raq_free(struct hns_roce_dev *hr_dev) hr_dev 1287 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 1291 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 1299 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_port_enable(struct hns_roce_dev *hr_dev, int enable_flag) hr_dev 1305 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, ROCEE_GLB_CFG_REG); hr_dev 1312 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_GLB_CFG_REG, val); hr_dev 1314 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, ROCEE_GLB_CFG_REG); hr_dev 1320 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_GLB_CFG_REG, val); hr_dev 1324 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_bt_init(struct hns_roce_dev *hr_dev) hr_dev 1326 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 1330 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 1367 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_bt_free(struct hns_roce_dev *hr_dev) hr_dev 1369 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 1372 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 1384 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_tptr_init(struct hns_roce_dev *hr_dev) hr_dev 1386 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 1390 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 1404 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->tptr_dma_addr = tptr_buf->map; hr_dev 1405 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->tptr_size = HNS_ROCE_V1_TPTR_BUF_SIZE; hr_dev 1410 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_tptr_free(struct hns_roce_dev *hr_dev) hr_dev 1412 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 1416 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 1423 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_free_mr_init(struct hns_roce_dev *hr_dev) hr_dev 1425 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 1430 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 1439 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hns_roce_v1_rsv_lp_qp(hr_dev); hr_dev 1449 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_free_mr_free(struct hns_roce_dev *hr_dev) hr_dev 1454 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 1460 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_v1_release_lp_qp(hr_dev); hr_dev 1469 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset) hr_dev 1472 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 1512 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev) hr_dev 1515 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_caps *caps = &hr_dev->caps; hr_dev 1517 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->vendor_id = roce_read(hr_dev, ROCEE_VENDOR_ID_REG); hr_dev 1518 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->vendor_part_id = roce_read(hr_dev, ROCEE_VENDOR_PART_ID_REG); hr_dev 1519 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->sys_image_guid = roce_read(hr_dev, ROCEE_SYS_IMAGE_GUID_L_REG) | hr_dev 1520 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ((u64)roce_read(hr_dev, hr_dev 1522 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->hw_rev = HNS_ROCE_HW_VER1; hr_dev 1575 drivers/infiniband/hw/hns/hns_roce_hw_v1.c caps->local_ca_ack_delay = roce_read(hr_dev, ROCEE_ACK_DELAY_REG); hr_dev 1581 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_init(struct hns_roce_dev *hr_dev) hr_dev 1586 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 1589 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG1_REG); hr_dev 1597 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_DMAE_USER_CFG1_REG, val); hr_dev 1599 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG2_REG); hr_dev 1607 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hns_roce_db_init(hr_dev); hr_dev 1613 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hns_roce_raq_init(hr_dev); hr_dev 1619 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hns_roce_bt_init(hr_dev); hr_dev 1625 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hns_roce_tptr_init(hr_dev); hr_dev 1631 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hns_roce_free_mr_init(hr_dev); hr_dev 1637 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_UP); hr_dev 1642 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_tptr_free(hr_dev); hr_dev 1645 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_bt_free(hr_dev); hr_dev 1648 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_raq_free(hr_dev); hr_dev 1651 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_db_free(hr_dev); hr_dev 1655 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_v1_exit(struct hns_roce_dev *hr_dev) hr_dev 1657 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN); hr_dev 1658 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_free_mr_free(hr_dev); hr_dev 1659 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_tptr_free(hr_dev); hr_dev 1660 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_bt_free(hr_dev); hr_dev 1661 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_raq_free(hr_dev); hr_dev 1662 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_db_free(hr_dev); hr_dev 1665 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_cmd_pending(struct hns_roce_dev *hr_dev) hr_dev 1667 drivers/infiniband/hw/hns/hns_roce_hw_v1.c u32 status = readl(hr_dev->reg_base + ROCEE_MB6_REG); hr_dev 1672 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param, hr_dev 1676 drivers/infiniband/hw/hns/hns_roce_hw_v1.c u32 __iomem *hcr = (u32 __iomem *)(hr_dev->reg_base + ROCEE_MB1_REG); hr_dev 1682 drivers/infiniband/hw/hns/hns_roce_hw_v1.c while (hns_roce_v1_cmd_pending(hr_dev)) { hr_dev 1684 drivers/infiniband/hw/hns/hns_roce_hw_v1.c dev_err(hr_dev->dev, "jiffies=%d end=%d\n", hr_dev 1713 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_chk_mbox(struct hns_roce_dev *hr_dev, hr_dev 1716 drivers/infiniband/hw/hns/hns_roce_hw_v1.c u8 __iomem *hcr = hr_dev->reg_base + ROCEE_MB1_REG; hr_dev 1721 drivers/infiniband/hw/hns/hns_roce_hw_v1.c while (hns_roce_v1_cmd_pending(hr_dev) && time_before(jiffies, end)) hr_dev 1724 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (hns_roce_v1_cmd_pending(hr_dev)) { hr_dev 1725 drivers/infiniband/hw/hns/hns_roce_hw_v1.c dev_err(hr_dev->dev, "[cmd_poll]hw run cmd TIMEDOUT!\n"); hr_dev 1732 drivers/infiniband/hw/hns/hns_roce_hw_v1.c dev_err(hr_dev->dev, "mailbox status 0x%x!\n", status); hr_dev 1739 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, hr_dev 1747 drivers/infiniband/hw/hns/hns_roce_hw_v1.c gid_idx = hns_get_gid_index(hr_dev, port, gid_index); hr_dev 1749 drivers/infiniband/hw/hns/hns_roce_hw_v1.c spin_lock_irqsave(&hr_dev->iboe.lock, flags); hr_dev 1752 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_L_0_REG + hr_dev 1756 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_ML_0_REG + hr_dev 1760 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_MH_0_REG + hr_dev 1764 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_H_0_REG + hr_dev 1767 drivers/infiniband/hw/hns/hns_roce_hw_v1.c spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); hr_dev 1772 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, hr_dev 1787 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (hr_dev->hw->dereg_mr) { hr_dev 1790 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hns_roce_v1_recreate_lp_qp(hr_dev); hr_dev 1797 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_raw_write(reg_smac_l, hr_dev->reg_base + ROCEE_SMAC_L_0_REG + hr_dev 1800 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, hr_dev 1808 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET, hr_dev 1814 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_v1_set_mtu(struct hns_roce_dev *hr_dev, u8 phy_port, hr_dev 1820 drivers/infiniband/hw/hns/hns_roce_hw_v1.c val = roce_read(hr_dev, hr_dev 1826 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET, hr_dev 2070 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev, hr_dev 2081 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 2197 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); hr_dev 2198 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 2225 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_qp = __hns_roce_qp_lookup(hr_dev, qpn); hr_dev 2420 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev, hr_dev 2424 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 2432 drivers/infiniband/hw/hns/hns_roce_hw_v1.c priv = (struct hns_roce_v1_priv *)hr_dev->priv; hr_dev 2457 drivers/infiniband/hw/hns/hns_roce_hw_v1.c spin_lock_irqsave(&hr_dev->bt_cmd_lock, flags); hr_dev 2459 drivers/infiniband/hw/hns/hns_roce_hw_v1.c bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; hr_dev 2465 drivers/infiniband/hw/hns/hns_roce_hw_v1.c spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, hr_dev 2479 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_write64_k(bt_cmd_val, hr_dev->reg_base + ROCEE_BT_CMD_L_REG); hr_dev 2481 drivers/infiniband/hw/hns/hns_roce_hw_v1.c spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, flags); hr_dev 2486 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev, hr_dev 2533 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 2545 drivers/infiniband/hw/hns/hns_roce_hw_v1.c return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2, hr_dev 2550 drivers/infiniband/hw/hns/hns_roce_hw_v1.c return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2, hr_dev 2554 drivers/infiniband/hw/hns/hns_roce_hw_v1.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 2560 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0, hr_dev 2564 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 2572 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 2575 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 2588 drivers/infiniband/hw/hns/hns_roce_hw_v1.c mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table, hr_dev 2664 drivers/infiniband/hw/hns/hns_roce_hw_v1.c addr = (u32 __iomem *)(hr_dev->reg_base + hr_dev 2681 drivers/infiniband/hw/hns/hns_roce_hw_v1.c reg_val = roce_read(hr_dev, ROCEE_QP1C_CFG0_0_REG + hr_dev 2687 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_QP1C_CFG0_0_REG + hr_dev 2716 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 2718 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 2738 drivers/infiniband/hw/hns/hns_roce_hw_v1.c mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table, hr_dev 2746 drivers/infiniband/hw/hns/hns_roce_hw_v1.c mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table, hr_dev 2928 drivers/infiniband/hw/hns/hns_roce_hw_v1.c smac = (u8 *)hr_dev->dev_addr[port]; hr_dev 2931 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->loop_idc == 0x1) hr_dev 2954 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_get_gid_index(hr_dev, hr_dev 3261 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hns_roce_v1_qp_modify(hr_dev, &hr_qp->mtt, hr_dev 3287 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_qp->rq.db_reg_l = hr_dev->reg_base + hr_dev 3288 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->odb_offset + hr_dev 3289 drivers/infiniband/hw/hns/hns_roce_hw_v1.c DB_REG_OFFSET * hr_dev->priv_uar.index; hr_dev 3301 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; hr_dev 3355 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_query_qpc(struct hns_roce_dev *hr_dev, hr_dev 3362 drivers/infiniband/hw/hns/hns_roce_hw_v1.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 3366 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0, hr_dev 3372 drivers/infiniband/hw/hns/hns_roce_hw_v1.c dev_err(&hr_dev->pdev->dev, "QUERY QP cmd process error\n"); hr_dev 3374 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 3383 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 3397 drivers/infiniband/hw/hns/hns_roce_hw_v1.c context.qp1c_bytes_4 = cpu_to_le32(roce_read(hr_dev, addr)); hr_dev 3398 drivers/infiniband/hw/hns/hns_roce_hw_v1.c context.sq_rq_bt_l = cpu_to_le32(roce_read(hr_dev, addr + 1)); hr_dev 3399 drivers/infiniband/hw/hns/hns_roce_hw_v1.c context.qp1c_bytes_12 = cpu_to_le32(roce_read(hr_dev, addr + 2)); hr_dev 3400 drivers/infiniband/hw/hns/hns_roce_hw_v1.c context.qp1c_bytes_16 = cpu_to_le32(roce_read(hr_dev, addr + 3)); hr_dev 3401 drivers/infiniband/hw/hns/hns_roce_hw_v1.c context.qp1c_bytes_20 = cpu_to_le32(roce_read(hr_dev, addr + 4)); hr_dev 3402 drivers/infiniband/hw/hns/hns_roce_hw_v1.c context.cur_rq_wqe_ba_l = cpu_to_le32(roce_read(hr_dev, addr + 5)); hr_dev 3403 drivers/infiniband/hw/hns/hns_roce_hw_v1.c context.qp1c_bytes_28 = cpu_to_le32(roce_read(hr_dev, addr + 6)); hr_dev 3404 drivers/infiniband/hw/hns/hns_roce_hw_v1.c context.qp1c_bytes_32 = cpu_to_le32(roce_read(hr_dev, addr + 7)); hr_dev 3405 drivers/infiniband/hw/hns/hns_roce_hw_v1.c context.cur_sq_wqe_ba_l = cpu_to_le32(roce_read(hr_dev, addr + 8)); hr_dev 3406 drivers/infiniband/hw/hns/hns_roce_hw_v1.c context.qp1c_bytes_40 = cpu_to_le32(roce_read(hr_dev, addr + 9)); hr_dev 3453 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 3455 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 3475 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hns_roce_v1_query_qpc(hr_dev, hr_qp, context); hr_dev 3604 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 3625 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_qp_remove(hr_dev, hr_qp); hr_dev 3626 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_qp_free(hr_dev, hr_qp); hr_dev 3630 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1); hr_dev 3632 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt); hr_dev 3639 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); hr_dev 3651 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); hr_dev 3653 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 3659 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_free_cq(hr_dev, hr_cq); hr_dev 3665 drivers/infiniband/hw/hns/hns_roce_hw_v1.c cqe_cnt_ori = roce_read(hr_dev, ROCEE_SCAEP_WR_CQE_CNT); hr_dev 3667 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (roce_read(hr_dev, ROCEE_CAEP_CQE_WCMD_EMPTY) & hr_dev 3671 drivers/infiniband/hw/hns/hns_roce_hw_v1.c cqe_cnt_cur = roce_read(hr_dev, ROCEE_SCAEP_WR_CQE_CNT); hr_dev 3684 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); hr_dev 3689 drivers/infiniband/hw/hns/hns_roce_hw_v1.c cq_buf_size = (ibcq->cqe + 1) * hr_dev->caps.cq_entry_sz; hr_dev 3690 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_buf_free(hr_dev, cq_buf_size, &hr_cq->hr_buf.hr_buf); hr_dev 3700 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_v1_wq_catas_err_handle(struct hns_roce_dev *hr_dev, hr_dev 3703 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 3734 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_v1_local_wq_access_err_handle(struct hns_roce_dev *hr_dev, hr_dev 3738 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 3769 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_v1_qp_err_handle(struct hns_roce_dev *hr_dev, hr_dev 3773 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 3792 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_v1_wq_catas_err_handle(hr_dev, aeqe, qpn); hr_dev 3795 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_v1_local_wq_access_err_handle(hr_dev, aeqe, qpn); hr_dev 3801 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_qp_event(hr_dev, qpn, event_type); hr_dev 3804 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_v1_cq_err_handle(struct hns_roce_dev *hr_dev, hr_dev 3808 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 3829 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_cq_event(hr_dev, cqn, event_type); hr_dev 3832 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_v1_db_overflow_handle(struct hns_roce_dev *hr_dev, hr_dev 3835 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 3880 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_aeq_int(struct hns_roce_dev *hr_dev, hr_dev 3883 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 3919 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_v1_qp_err_handle(hr_dev, aeqe, event_type); hr_dev 3929 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_v1_cq_err_handle(hr_dev, aeqe, event_type); hr_dev 3935 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_cmd_event(hr_dev, hr_dev 3942 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_v1_db_overflow_handle(hr_dev, aeqe); hr_dev 3959 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (eq->cons_index > 2 * hr_dev->caps.aeqe_depth - 1) { hr_dev 3989 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_ceq_int(struct hns_roce_dev *hr_dev, hr_dev 4006 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_cq_completion(hr_dev, cqn); hr_dev 4012 drivers/infiniband/hw/hns/hns_roce_hw_v1.c EQ_DEPTH_COEFF * hr_dev->caps.ceqe_depth - 1) { hr_dev 4013 drivers/infiniband/hw/hns/hns_roce_hw_v1.c dev_warn(&eq->hr_dev->pdev->dev, hr_dev 4027 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev = eq->hr_dev; hr_dev 4032 drivers/infiniband/hw/hns/hns_roce_hw_v1.c int_work = hns_roce_v1_ceq_int(hr_dev, eq); hr_dev 4035 drivers/infiniband/hw/hns/hns_roce_hw_v1.c int_work = hns_roce_v1_aeq_int(hr_dev, eq); hr_dev 4042 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev = dev_id; hr_dev 4043 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 4059 drivers/infiniband/hw/hns/hns_roce_hw_v1.c aeshift_val = roce_read(hr_dev, ROCEE_CAEP_AEQC_AEQE_SHIFT_REG); hr_dev 4068 drivers/infiniband/hw/hns/hns_roce_hw_v1.c caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG); hr_dev 4073 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val); hr_dev 4076 drivers/infiniband/hw/hns/hns_roce_hw_v1.c caepaest_val = roce_read(hr_dev, ROCEE_CAEP_AE_ST_REG); hr_dev 4080 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_CAEP_AE_ST_REG, caepaest_val); hr_dev 4083 drivers/infiniband/hw/hns/hns_roce_hw_v1.c caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG); hr_dev 4088 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val); hr_dev 4092 drivers/infiniband/hw/hns/hns_roce_hw_v1.c for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) { hr_dev 4093 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ceshift_val = roce_read(hr_dev, ROCEE_CAEP_CEQC_SHIFT_0_REG + hr_dev 4103 drivers/infiniband/hw/hns/hns_roce_hw_v1.c cemask_val = roce_read(hr_dev, hr_dev 4111 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG + hr_dev 4115 drivers/infiniband/hw/hns/hns_roce_hw_v1.c cealmovf_val = roce_read(hr_dev, hr_dev 4123 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_CAEP_CEQ_ALM_OVF_0_REG + hr_dev 4127 drivers/infiniband/hw/hns/hns_roce_hw_v1.c cemask_val = roce_read(hr_dev, hr_dev 4135 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG + hr_dev 4142 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_read(hr_dev, ROCEE_ECC_UCERR_ALM0_REG), hr_dev 4143 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_read(hr_dev, ROCEE_ECC_UCERR_ALM1_REG), hr_dev 4144 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_read(hr_dev, ROCEE_ECC_UCERR_ALM2_REG)); hr_dev 4147 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_read(hr_dev, ROCEE_ECC_CERR_ALM0_REG), hr_dev 4148 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_read(hr_dev, ROCEE_ECC_CERR_ALM1_REG), hr_dev 4149 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_read(hr_dev, ROCEE_ECC_CERR_ALM2_REG)); hr_dev 4154 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_v1_int_mask_enable(struct hns_roce_dev *hr_dev) hr_dev 4162 drivers/infiniband/hw/hns/hns_roce_hw_v1.c aemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG); hr_dev 4168 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, aemask_val); hr_dev 4171 drivers/infiniband/hw/hns/hns_roce_hw_v1.c for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) { hr_dev 4173 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG + hr_dev 4178 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_v1_free_eq(struct hns_roce_dev *hr_dev, hr_dev 4189 drivers/infiniband/hw/hns/hns_roce_hw_v1.c dma_free_coherent(&hr_dev->pdev->dev, HNS_ROCE_BA_SIZE, hr_dev 4195 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_v1_enable_eq(struct hns_roce_dev *hr_dev, int eq_num, hr_dev 4198 drivers/infiniband/hw/hns/hns_roce_hw_v1.c void __iomem *eqc = hr_dev->eq_table.eqc_base[eq_num]; hr_dev 4220 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_create_eq(struct hns_roce_dev *hr_dev, hr_dev 4223 drivers/infiniband/hw/hns/hns_roce_hw_v1.c void __iomem *eqc = hr_dev->eq_table.eqc_base[eq->eqn]; hr_dev 4224 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 4305 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_v1_init_eq_table(struct hns_roce_dev *hr_dev) hr_dev 4307 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; hr_dev 4308 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 4315 drivers/infiniband/hw/hns/hns_roce_hw_v1.c eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; hr_dev 4316 drivers/infiniband/hw/hns/hns_roce_hw_v1.c irq_num = eq_num + hr_dev->caps.num_other_vectors; hr_dev 4331 drivers/infiniband/hw/hns/hns_roce_hw_v1.c eq->hr_dev = hr_dev; hr_dev 4333 drivers/infiniband/hw/hns/hns_roce_hw_v1.c eq->irq = hr_dev->irq[i]; hr_dev 4336 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (i < hr_dev->caps.num_comp_vectors) { hr_dev 4338 drivers/infiniband/hw/hns/hns_roce_hw_v1.c eq_table->eqc_base[i] = hr_dev->reg_base + hr_dev 4342 drivers/infiniband/hw/hns/hns_roce_hw_v1.c eq->doorbell = hr_dev->reg_base + hr_dev 4345 drivers/infiniband/hw/hns/hns_roce_hw_v1.c eq->entries = hr_dev->caps.ceqe_depth; hr_dev 4350 drivers/infiniband/hw/hns/hns_roce_hw_v1.c eq_table->eqc_base[i] = hr_dev->reg_base + hr_dev 4353 drivers/infiniband/hw/hns/hns_roce_hw_v1.c eq->doorbell = hr_dev->reg_base + hr_dev 4355 drivers/infiniband/hw/hns/hns_roce_hw_v1.c eq->entries = hr_dev->caps.aeqe_depth; hr_dev 4362 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_v1_int_mask_enable(hr_dev); hr_dev 4365 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_CAEP_CE_INTERVAL_CFG_REG, hr_dev 4369 drivers/infiniband/hw/hns/hns_roce_hw_v1.c roce_write(hr_dev, ROCEE_CAEP_CE_BURST_NUM_CFG_REG, hr_dev 4373 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hns_roce_v1_create_eq(hr_dev, &eq_table->eq[i]); hr_dev 4382 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = request_irq(hr_dev->irq[j], hr_dev 4384 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->irq_names[j], hr_dev 4387 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = request_irq(hr_dev->irq[j], hr_dev 4389 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->irq_names[j], hr_dev); hr_dev 4398 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_v1_enable_eq(hr_dev, i, EQ_ENABLE); hr_dev 4404 drivers/infiniband/hw/hns/hns_roce_hw_v1.c free_irq(hr_dev->irq[j], &eq_table->eq[j]); hr_dev 4408 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_v1_free_eq(hr_dev, &eq_table->eq[i]); hr_dev 4418 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static void hns_roce_v1_cleanup_eq_table(struct hns_roce_dev *hr_dev) hr_dev 4420 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; hr_dev 4425 drivers/infiniband/hw/hns/hns_roce_hw_v1.c eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; hr_dev 4426 drivers/infiniband/hw/hns/hns_roce_hw_v1.c irq_num = eq_num + hr_dev->caps.num_other_vectors; hr_dev 4429 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_v1_enable_eq(hr_dev, i, EQ_DISABLE); hr_dev 4431 drivers/infiniband/hw/hns/hns_roce_hw_v1.c free_irq(hr_dev->irq[i], &eq_table->eq[i]); hr_dev 4433 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_v1_free_eq(hr_dev, &eq_table->eq[i]); hr_dev 4436 drivers/infiniband/hw/hns/hns_roce_hw_v1.c free_irq(hr_dev->irq[i], hr_dev); hr_dev 4503 drivers/infiniband/hw/hns/hns_roce_hw_v1.c static int hns_roce_get_cfg(struct hns_roce_dev *hr_dev) hr_dev 4505 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct device *dev = &hr_dev->pdev->dev; hr_dev 4523 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->hw = (const struct hns_roce_hw *)of_id->data; hr_dev 4524 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (!hr_dev->hw) { hr_dev 4536 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->hw = (const struct hns_roce_hw *) acpi_id->driver_data; hr_dev 4537 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (!hr_dev->hw) { hr_dev 4547 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->reg_base = devm_platform_ioremap_resource(hr_dev->pdev, 0); hr_dev 4548 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (IS_ERR(hr_dev->reg_base)) hr_dev 4549 drivers/infiniband/hw/hns/hns_roce_hw_v1.c return PTR_ERR(hr_dev->reg_base); hr_dev 4553 drivers/infiniband/hw/hns/hns_roce_hw_v1.c (u8 *)&hr_dev->ib_dev.node_guid, hr_dev 4586 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->iboe.netdevs[port_cnt] = netdev; hr_dev 4587 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->iboe.phy_port[port_cnt] = phy_port; hr_dev 4602 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->caps.num_ports = port_cnt; hr_dev 4605 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->cmd_mod = 1; hr_dev 4606 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->loop_idc = 0; hr_dev 4607 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG; hr_dev 4608 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->odb_offset = ROCEE_DB_OTHERS_L_0_REG; hr_dev 4612 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->irq_names, hr_dev 4621 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->irq[i] = platform_get_irq(hr_dev->pdev, i); hr_dev 4622 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (hr_dev->irq[i] <= 0) hr_dev 4638 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev; hr_dev 4641 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev = ib_alloc_device(hns_roce_dev, ib_dev); hr_dev 4642 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (!hr_dev) hr_dev 4645 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->priv = kzalloc(sizeof(struct hns_roce_v1_priv), GFP_KERNEL); hr_dev 4646 drivers/infiniband/hw/hns/hns_roce_hw_v1.c if (!hr_dev->priv) { hr_dev 4651 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->pdev = pdev; hr_dev 4652 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hr_dev->dev = dev; hr_dev 4653 drivers/infiniband/hw/hns/hns_roce_hw_v1.c platform_set_drvdata(pdev, hr_dev); hr_dev 4662 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hns_roce_get_cfg(hr_dev); hr_dev 4668 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ret = hns_roce_init(hr_dev); hr_dev 4677 drivers/infiniband/hw/hns/hns_roce_hw_v1.c kfree(hr_dev->priv); hr_dev 4680 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ib_dealloc_device(&hr_dev->ib_dev); hr_dev 4691 drivers/infiniband/hw/hns/hns_roce_hw_v1.c struct hns_roce_dev *hr_dev = platform_get_drvdata(pdev); hr_dev 4693 drivers/infiniband/hw/hns/hns_roce_hw_v1.c hns_roce_exit(hr_dev); hr_dev 4694 drivers/infiniband/hw/hns/hns_roce_hw_v1.c kfree(hr_dev->priv); hr_dev 4695 drivers/infiniband/hw/hns/hns_roce_hw_v1.c ib_dealloc_device(&hr_dev->ib_dev); hr_dev 165 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 173 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.max_sq_inline) { hr_dev 175 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "inline len(1-%d)=%d, illegal", hr_dev 176 drivers/infiniband/hw/hns/hns_roce_hw_v2.c rc_sq_wqe->msg_len, hr_dev->caps.max_sq_inline); hr_dev 182 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Not support inline data!\n"); hr_dev 237 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 243 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 333 drivers/infiniband/hw/hns/hns_roce_hw_v2.c smac = (u8 *)hr_dev->dev_addr[qp->port]; hr_dev 429 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_get_gid_index(hr_dev, qp->phy_port, hr_dev 594 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l); hr_dev 621 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 625 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 676 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) { hr_dev 717 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev, hr_dev 730 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->is_reset = true; hr_dev 731 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->dis_db = true; hr_dev 740 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev, hr_dev 744 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; hr_dev 757 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->dis_db = true; hr_dev 759 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->is_reset = true; hr_dev 761 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT || hr_dev 768 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev) hr_dev 770 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; hr_dev 778 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->dis_db = true; hr_dev 779 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt) hr_dev 780 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->is_reset = true; hr_dev 785 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev) hr_dev 787 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; hr_dev 796 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (hr_dev->is_reset) hr_dev 812 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (reset_cnt != hr_dev->reset_cnt) hr_dev 813 drivers/infiniband/hw/hns/hns_roce_hw_v2.c return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage, hr_dev 816 drivers/infiniband/hw/hns/hns_roce_hw_v2.c return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage, hr_dev 819 drivers/infiniband/hw/hns/hns_roce_hw_v2.c return hns_roce_v2_cmd_sw_resetting(hr_dev); hr_dev 833 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev, hr_dev 842 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size, hr_dev 844 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) { hr_dev 854 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev, hr_dev 857 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dma_unmap_single(hr_dev->dev, ring->desc_dma_addr, hr_dev 865 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type) hr_dev 867 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; hr_dev 875 drivers/infiniband/hw/hns/hns_roce_hw_v2.c return hns_roce_alloc_cmq_desc(hr_dev, ring); hr_dev 878 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type) hr_dev 880 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; hr_dev 886 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma); hr_dev 887 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG, hr_dev 889 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG, hr_dev 891 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0); hr_dev 892 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0); hr_dev 894 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma); hr_dev 895 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG, hr_dev 897 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG, hr_dev 899 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0); hr_dev 900 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0); hr_dev 904 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev) hr_dev 906 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; hr_dev 921 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ); hr_dev 923 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret); hr_dev 928 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ); hr_dev 930 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret); hr_dev 935 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ); hr_dev 938 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ); hr_dev 943 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq); hr_dev 948 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev) hr_dev 950 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; hr_dev 952 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq); hr_dev 953 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq); hr_dev 970 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev) hr_dev 972 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; hr_dev 973 drivers/infiniband/hw/hns/hns_roce_hw_v2.c u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG); hr_dev 978 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev) hr_dev 980 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; hr_dev 988 drivers/infiniband/hw/hns/hns_roce_hw_v2.c head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG); hr_dev 1002 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev, hr_dev 1005 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; hr_dev 1031 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_dbg(hr_dev->dev, "set cmq desc:\n"); hr_dev 1039 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, csq->next_to_use); hr_dev 1047 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (hns_roce_cmq_csq_done(hr_dev)) hr_dev 1054 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (hns_roce_cmq_csq_done(hr_dev)) { hr_dev 1061 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_dbg(hr_dev->dev, "Get cmq desc:\n"); hr_dev 1079 drivers/infiniband/hw/hns/hns_roce_hw_v2.c handle = hns_roce_cmq_csq_clean(hr_dev); hr_dev 1081 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_warn(hr_dev->dev, "Cleaned %d, need to clean %d\n", hr_dev 1089 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev, hr_dev 1095 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_v2_rst_process_cmd(hr_dev); hr_dev 1101 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = __hns_roce_cmq_send(hr_dev, desc, num); hr_dev 1103 drivers/infiniband/hw/hns/hns_roce_hw_v2.c retval = hns_roce_v2_rst_process_cmd(hr_dev); hr_dev 1113 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev) hr_dev 1120 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmq_send(hr_dev, &desc, 1); hr_dev 1125 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version); hr_dev 1126 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->vendor_id = hr_dev->pci_dev->vendor; hr_dev 1131 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static bool hns_roce_func_clr_chk_rst(struct hns_roce_dev *hr_dev) hr_dev 1133 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; hr_dev 1144 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (reset_cnt != hr_dev->reset_cnt || hw_resetting || sw_resetting) hr_dev 1150 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_func_clr_rst_prc(struct hns_roce_dev *hr_dev, int retval, hr_dev 1153 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; hr_dev 1167 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (reset_cnt != hr_dev->reset_cnt) { hr_dev 1168 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->dis_db = true; hr_dev 1169 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->is_reset = true; hr_dev 1170 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_info(hr_dev->dev, "Func clear success after reset.\n"); hr_dev 1172 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->dis_db = true; hr_dev 1174 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_warn(hr_dev->dev, hr_dev 1179 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->is_reset = true; hr_dev 1180 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_info(hr_dev->dev, hr_dev 1188 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_warn(hr_dev->dev, "Func clear failed.\n"); hr_dev 1190 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->dis_db = true; hr_dev 1192 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_warn(hr_dev->dev, hr_dev 1197 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->reset_cnt) { hr_dev 1198 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->is_reset = true; hr_dev 1199 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_info(hr_dev->dev, hr_dev 1207 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_warn(hr_dev->dev, "Func clear failed because of unfinished sw reset\n"); hr_dev 1210 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_warn(hr_dev->dev, hr_dev 1213 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_warn(hr_dev->dev, "Func clear failed.\n"); hr_dev 1216 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) hr_dev 1224 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (hns_roce_func_clr_chk_rst(hr_dev)) hr_dev 1230 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmq_send(hr_dev, &desc, 1); hr_dev 1233 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n", hr_dev 1241 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (hns_roce_func_clr_chk_rst(hr_dev)) hr_dev 1249 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmq_send(hr_dev, &desc, 1); hr_dev 1254 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->is_reset = true; hr_dev 1260 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Func clear fail.\n"); hr_dev 1261 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_func_clr_rst_prc(hr_dev, ret, fclr_write_fail_flag); hr_dev 1264 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev) hr_dev 1271 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmq_send(hr_dev, &desc, 1); hr_dev 1276 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver)); hr_dev 1281 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev) hr_dev 1298 drivers/infiniband/hw/hns/hns_roce_hw_v2.c return hns_roce_cmq_send(hr_dev, &desc, 1); hr_dev 1301 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev) hr_dev 1319 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmq_send(hr_dev, desc, 2); hr_dev 1326 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.qpc_bt_num = roce_get_field(req_a->qpc_bt_idx_num, hr_dev 1329 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.srqc_bt_num = roce_get_field(req_a->srqc_bt_idx_num, hr_dev 1332 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.cqc_bt_num = roce_get_field(req_a->cqc_bt_idx_num, hr_dev 1335 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.mpt_bt_num = roce_get_field(req_a->mpt_bt_idx_num, hr_dev 1339 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num, hr_dev 1342 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.sccc_bt_num = roce_get_field(req_b->sccc_bt_idx_num, hr_dev 1349 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev) hr_dev 1366 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmq_send(hr_dev, desc, 2); hr_dev 1372 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.qpc_timer_bt_num = hr_dev 1376 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.cqc_timer_bt_num = hr_dev 1384 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, hr_dev 1398 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmq_send(hr_dev, &desc, 1); hr_dev 1408 drivers/infiniband/hw/hns/hns_roce_hw_v2.c return hns_roce_cmq_send(hr_dev, &desc, 1); hr_dev 1411 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev) hr_dev 1506 drivers/infiniband/hw/hns/hns_roce_hw_v2.c return hns_roce_cmq_send(hr_dev, desc, 2); hr_dev 1509 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev) hr_dev 1511 drivers/infiniband/hw/hns/hns_roce_hw_v2.c u8 srqc_hop_num = hr_dev->caps.srqc_hop_num; hr_dev 1512 drivers/infiniband/hw/hns/hns_roce_hw_v2.c u8 qpc_hop_num = hr_dev->caps.qpc_hop_num; hr_dev 1513 drivers/infiniband/hw/hns/hns_roce_hw_v2.c u8 cqc_hop_num = hr_dev->caps.cqc_hop_num; hr_dev 1514 drivers/infiniband/hw/hns/hns_roce_hw_v2.c u8 mpt_hop_num = hr_dev->caps.mpt_hop_num; hr_dev 1515 drivers/infiniband/hw/hns/hns_roce_hw_v2.c u8 sccc_hop_num = hr_dev->caps.sccc_hop_num; hr_dev 1525 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.qpc_ba_pg_sz + PG_SHIFT_OFFSET); hr_dev 1528 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.qpc_buf_pg_sz + PG_SHIFT_OFFSET); hr_dev 1535 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.srqc_ba_pg_sz + PG_SHIFT_OFFSET); hr_dev 1538 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.srqc_buf_pg_sz + PG_SHIFT_OFFSET); hr_dev 1545 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.cqc_ba_pg_sz + PG_SHIFT_OFFSET); hr_dev 1548 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.cqc_buf_pg_sz + PG_SHIFT_OFFSET); hr_dev 1555 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.mpt_ba_pg_sz + PG_SHIFT_OFFSET); hr_dev 1558 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.mpt_buf_pg_sz + PG_SHIFT_OFFSET); hr_dev 1566 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.sccc_ba_pg_sz + PG_SHIFT_OFFSET); hr_dev 1570 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.sccc_buf_pg_sz + PG_SHIFT_OFFSET); hr_dev 1577 drivers/infiniband/hw/hns/hns_roce_hw_v2.c return hns_roce_cmq_send(hr_dev, &desc, 1); hr_dev 1580 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) hr_dev 1582 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_caps *caps = &hr_dev->caps; hr_dev 1585 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmq_query_hw_info(hr_dev); hr_dev 1587 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n", hr_dev 1592 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_query_fw_ver(hr_dev); hr_dev 1594 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n", hr_dev 1599 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_config_global_param(hr_dev); hr_dev 1601 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n", hr_dev 1607 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_query_pf_resource(hr_dev); hr_dev 1609 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n", hr_dev 1614 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (hr_dev->pci_dev->revision == 0x21) { hr_dev 1615 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_query_pf_timer_resource(hr_dev); hr_dev 1617 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, hr_dev 1624 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_alloc_vf_resource(hr_dev); hr_dev 1626 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n", hr_dev 1631 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (hr_dev->pci_dev->revision == 0x21) { hr_dev 1632 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_set_vf_switch_param(hr_dev, 0); hr_dev 1634 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, hr_dev 1641 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->vendor_part_id = hr_dev->pci_dev->device; hr_dev 1642 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid); hr_dev 1732 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (hr_dev->pci_dev->revision == 0x21) hr_dev 1747 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (hr_dev->pci_dev->revision == 0x21) { hr_dev 1769 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_v2_set_bt(hr_dev); hr_dev 1771 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n", hr_dev 1777 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev, hr_dev 1785 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = hr_dev->priv; hr_dev 1856 drivers/infiniband/hw/hns/hns_roce_hw_v2.c return hns_roce_cmq_send(hr_dev, desc, 2); hr_dev 1859 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev, hr_dev 1862 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = hr_dev->priv; hr_dev 1865 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 1878 drivers/infiniband/hw/hns/hns_roce_hw_v2.c buf_chk_sz = 1 << (hr_dev->caps.tsq_buf_pg_sz + PAGE_SHIFT); hr_dev 1879 drivers/infiniband/hw/hns/hns_roce_hw_v2.c pg_num_a = hr_dev->caps.num_qps * 8 / buf_chk_sz; hr_dev 1880 drivers/infiniband/hw/hns/hns_roce_hw_v2.c pg_num_b = hr_dev->caps.sl_num * 4 + 2; hr_dev 1884 drivers/infiniband/hw/hns/hns_roce_hw_v2.c buf_chk_sz = 1 << (hr_dev->caps.tpq_buf_pg_sz + PAGE_SHIFT); hr_dev 1885 drivers/infiniband/hw/hns/hns_roce_hw_v2.c pg_num_a = hr_dev->caps.num_cqs * 4 / buf_chk_sz; hr_dev 1926 drivers/infiniband/hw/hns/hns_roce_hw_v2.c return hns_roce_config_link_table(hr_dev, type); hr_dev 1943 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev, hr_dev 1946 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 1963 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_init(struct hns_roce_dev *hr_dev) hr_dev 1965 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = hr_dev->priv; hr_dev 1970 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE); hr_dev 1972 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "TSQ init failed, ret = %d.\n", ret); hr_dev 1976 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE); hr_dev 1978 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "TPQ init failed, ret = %d.\n", ret); hr_dev 1983 drivers/infiniband/hw/hns/hns_roce_hw_v2.c for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num; hr_dev 1985 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table, hr_dev 1988 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "QPC Timer get failed\n"); hr_dev 1994 drivers/infiniband/hw/hns/hns_roce_hw_v2.c for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num; hr_dev 1996 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table, hr_dev 1999 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "CQC Timer get failed\n"); hr_dev 2008 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i); hr_dev 2012 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i); hr_dev 2014 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_link_table(hr_dev, &priv->tpq); hr_dev 2017 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_link_table(hr_dev, &priv->tsq); hr_dev 2022 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev) hr_dev 2024 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = hr_dev->priv; hr_dev 2026 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (hr_dev->pci_dev->revision == 0x21) hr_dev 2027 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_function_clear(hr_dev); hr_dev 2029 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_link_table(hr_dev, &priv->tpq); hr_dev 2030 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_link_table(hr_dev, &priv->tsq); hr_dev 2033 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev) hr_dev 2042 drivers/infiniband/hw/hns/hns_roce_hw_v2.c status = hns_roce_cmq_send(hr_dev, &desc, 1); hr_dev 2049 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev) hr_dev 2051 drivers/infiniband/hw/hns/hns_roce_hw_v2.c u32 status = hns_roce_query_mbox_status(hr_dev); hr_dev 2056 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev) hr_dev 2058 drivers/infiniband/hw/hns/hns_roce_hw_v2.c u32 status = hns_roce_query_mbox_status(hr_dev); hr_dev 2063 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param, hr_dev 2079 drivers/infiniband/hw/hns/hns_roce_hw_v2.c return hns_roce_cmq_send(hr_dev, &desc, 1); hr_dev 2082 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param, hr_dev 2086 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 2091 drivers/infiniband/hw/hns/hns_roce_hw_v2.c while (hns_roce_v2_cmd_pending(hr_dev)) { hr_dev 2100 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier, hr_dev 2108 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev, hr_dev 2111 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 2116 drivers/infiniband/hw/hns/hns_roce_hw_v2.c while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end)) hr_dev 2119 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (hns_roce_v2_cmd_pending(hr_dev)) { hr_dev 2124 drivers/infiniband/hw/hns/hns_roce_hw_v2.c status = hns_roce_v2_cmd_complete(hr_dev); hr_dev 2136 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_config_sgid_table(struct hns_roce_dev *hr_dev, hr_dev 2166 drivers/infiniband/hw/hns/hns_roce_hw_v2.c return hns_roce_cmq_send(hr_dev, &desc, 1); hr_dev 2169 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port, hr_dev 2189 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_config_sgid_table(hr_dev, gid_index, gid, sgid_type); hr_dev 2191 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Configure sgid table failed(%d)!\n", ret); hr_dev 2196 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, hr_dev 2219 drivers/infiniband/hw/hns/hns_roce_hw_v2.c return hns_roce_cmq_send(hr_dev, &desc, 1); hr_dev 2322 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev, hr_dev 2554 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, hr_dev 2584 drivers/infiniband/hw/hns/hns_roce_hw_v2.c V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num == hr_dev 2585 drivers/infiniband/hw/hns/hns_roce_hw_v2.c HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num); hr_dev 2595 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.cqe_ba_pg_sz + PG_SHIFT_OFFSET); hr_dev 2599 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.cqe_buf_pg_sz + PG_SHIFT_OFFSET); hr_dev 2629 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); hr_dev 2655 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_write64(hr_dev, doorbell, hr_cq->cq_db_l); hr_dev 2698 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev; hr_dev 2727 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev = to_hr_dev(hr_cq->ib_cq.device); hr_dev 2728 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_qp = __hns_roce_qp_lookup(hr_dev, qpn); hr_dev 2730 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "CQ %06lx with entry for unknown QPN %06x\n", hr_dev 2997 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type, hr_dev 3028 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_warn(hr_dev->dev, hr_dev 3036 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, hr_dev 3054 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (!hns_roce_check_whether_mhop(hr_dev, table->type)) hr_dev 3057 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); hr_dev 3074 drivers/infiniband/hw/hns/hns_roce_hw_v2.c op = get_op_for_set_hem(hr_dev, table->type, step_idx); hr_dev 3078 drivers/infiniband/hw/hns/hns_roce_hw_v2.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 3092 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, hr_dev 3103 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj, hr_dev 3107 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 3111 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev, hr_dev 3115 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 3120 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (!hns_roce_check_whether_mhop(hr_dev, table->type)) hr_dev 3153 drivers/infiniband/hw/hns/hns_roce_hw_v2.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 3158 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op, hr_dev 3161 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 3165 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev, hr_dev 3174 drivers/infiniband/hw/hns/hns_roce_hw_v2.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 3180 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0, hr_dev 3184 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 3262 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 3340 drivers/infiniband/hw/hns/hns_roce_hw_v2.c (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) ? 1 : 0); hr_dev 3641 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static bool check_wqe_rq_mtt_count(struct hns_roce_dev *hr_dev, hr_dev 3645 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 3672 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 3674 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 3689 drivers/infiniband/hw/hns/hns_roce_hw_v2.c page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); hr_dev 3690 drivers/infiniband/hw/hns/hns_roce_hw_v2.c count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_dev 3694 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (!check_wqe_rq_mtt_count(hr_dev, hr_qp, count, page_size)) hr_dev 3698 drivers/infiniband/hw/hns/hns_roce_hw_v2.c mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table, hr_dev 3706 drivers/infiniband/hw/hns/hns_roce_hw_v2.c mtts_3 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table, hr_dev 3735 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.wqe_sq_hop_num == HNS_ROCE_HOP_NUM_0 ? hr_dev 3736 drivers/infiniband/hw/hns/hns_roce_hw_v2.c 0 : hr_dev->caps.wqe_sq_hop_num); hr_dev 3745 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.wqe_sge_hop_num : 0); hr_dev 3753 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.wqe_rq_hop_num == HNS_ROCE_HOP_NUM_0 ? hr_dev 3754 drivers/infiniband/hw/hns/hns_roce_hw_v2.c 0 : hr_dev->caps.wqe_rq_hop_num); hr_dev 3770 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.mtt_buf_pg_sz + PG_SHIFT_OFFSET); hr_dev 3827 drivers/infiniband/hw/hns/hns_roce_hw_v2.c smac = (u8 *)hr_dev->dev_addr[port]; hr_dev 3830 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->loop_idc == 0x1) { hr_dev 3847 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_get_gid_index(hr_dev, port_num - 1, hr_dev 3917 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 3919 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 3926 drivers/infiniband/hw/hns/hns_roce_hw_v2.c count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL); hr_dev 3933 drivers/infiniband/hw/hns/hns_roce_hw_v2.c page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); hr_dev 3934 drivers/infiniband/hw/hns/hns_roce_hw_v2.c count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_dev 4063 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 4075 drivers/infiniband/hw/hns/hns_roce_hw_v2.c is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) && hr_dev 4105 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) { hr_dev 4106 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "sgid_index(%u) too large. max is %d\n", hr_dev 4107 drivers/infiniband/hw/hns/hns_roce_hw_v2.c grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]); hr_dev 4112 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "ah attr is not RDMA roce type\n"); hr_dev 4135 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (hr_dev->pci_dev->revision == 0x21 && is_udp) hr_dev 4166 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 4190 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Illegal state for QP!\n"); hr_dev 4205 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 4225 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_warn(hr_dev->dev, hr_dev 4371 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 4381 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; hr_dev 4390 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 4395 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 4450 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_v2_qp_modify(hr_dev, cur_state, new_state, ctx, hr_qp); hr_dev 4495 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, hr_dev 4502 drivers/infiniband/hw/hns/hns_roce_hw_v2.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 4506 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0, hr_dev 4510 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "QUERY QP cmd process error\n"); hr_dev 4517 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 4525 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 4528 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 4544 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, &context); hr_dev 4649 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, hr_dev 4654 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct ib_device *ibdev = &hr_dev->ib_dev; hr_dev 4677 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_qp_remove(hr_dev, hr_qp); hr_dev 4681 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_qp_free(hr_dev, hr_qp); hr_dev 4687 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1); hr_dev 4689 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr); hr_dev 4706 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); hr_dev 4708 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_db(hr_dev, &hr_qp->rdb); hr_dev 4712 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && hr_dev 4723 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 4727 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata); hr_dev 4729 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ibdev_err(&hr_dev->ib_dev, "Destroy qp 0x%06lx failed(%d)\n", hr_dev 4740 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev, hr_dev 4748 drivers/infiniband/hw/hns/hns_roce_hw_v2.c mutex_lock(&hr_dev->qp_table.scc_mutex); hr_dev 4752 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmq_send(hr_dev, &desc, 1); hr_dev 4754 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Reset SCC ctx failed(%d)\n", ret); hr_dev 4762 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmq_send(hr_dev, &desc, 1); hr_dev 4764 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Clear SCC ctx failed(%d)\n", ret); hr_dev 4773 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmq_send(hr_dev, &desc, 1); hr_dev 4775 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Query clr cmq failed(%d)\n", ret); hr_dev 4785 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Query SCC clr done flag overtime.\n"); hr_dev 4789 drivers/infiniband/hw/hns/hns_roce_hw_v2.c mutex_unlock(&hr_dev->qp_table.scc_mutex); hr_dev 4795 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(cq->device); hr_dev 4802 drivers/infiniband/hw/hns/hns_roce_hw_v2.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 4824 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 1, hr_dev 4827 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 4829 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "MODIFY CQ Failed to cmd mailbox.\n"); hr_dev 4834 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_set_qps_to_err(struct hns_roce_dev *hr_dev, u32 qpn) hr_dev 4841 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_qp = __hns_roce_qp_lookup(hr_dev, qpn); hr_dev 4843 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_warn(hr_dev->dev, "no hr_qp can be found!\n"); hr_dev 4853 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_warn(hr_dev->dev, "flush cqe is unsupported in userspace!\n"); hr_dev 4863 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "failed to modify qp %d to err state.\n", hr_dev 4871 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = irq_work->hr_dev->dev; hr_dev 4890 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_set_qps_to_err(irq_work->hr_dev, qpn); hr_dev 4895 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_set_qps_to_err(irq_work->hr_dev, qpn); hr_dev 4900 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_set_qps_to_err(irq_work->hr_dev, qpn); hr_dev 4930 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev, hr_dev 4941 drivers/infiniband/hw/hns/hns_roce_hw_v2.c irq_work->hr_dev = hr_dev; hr_dev 4946 drivers/infiniband/hw/hns/hns_roce_hw_v2.c queue_work(hr_dev->irq_workq, &(irq_work->work)); hr_dev 4951 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = eq->hr_dev; hr_dev 4978 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_write64(hr_dev, doorbell, eq->doorbell); hr_dev 5023 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev, hr_dev 5026 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 5066 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_qp_event(hr_dev, qpn, event_type); hr_dev 5070 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_srq_event(hr_dev, srqn, event_type); hr_dev 5074 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_cq_event(hr_dev, cqn, event_type); hr_dev 5079 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_cmd_event(hr_dev, hr_dev 5102 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_v2_init_irq_work(hr_dev, eq, qpn, cqn); hr_dev 5153 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev, hr_dev 5156 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 5171 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_cq_completion(hr_dev, cqn); hr_dev 5192 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = eq->hr_dev; hr_dev 5197 drivers/infiniband/hw/hns/hns_roce_hw_v2.c int_work = hns_roce_v2_ceq_int(hr_dev, eq); hr_dev 5200 drivers/infiniband/hw/hns/hns_roce_hw_v2.c int_work = hns_roce_v2_aeq_int(hr_dev, eq); hr_dev 5207 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = dev_id; hr_dev 5208 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 5214 drivers/infiniband/hw/hns/hns_roce_hw_v2.c int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG); hr_dev 5215 drivers/infiniband/hw/hns/hns_roce_hw_v2.c int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG); hr_dev 5218 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct pci_dev *pdev = hr_dev->pci_dev; hr_dev 5225 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); hr_dev 5235 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); hr_dev 5242 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); hr_dev 5245 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); hr_dev 5252 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); hr_dev 5255 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); hr_dev 5264 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev, hr_dev 5271 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG + hr_dev 5275 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, hr_dev 5277 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, hr_dev 5281 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG + hr_dev 5285 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, hr_dev 5287 drivers/infiniband/hw/hns/hns_roce_hw_v2.c roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, hr_dev 5292 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn) hr_dev 5294 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 5297 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (eqn < hr_dev->caps.num_comp_vectors) hr_dev 5298 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M, hr_dev 5302 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M, hr_dev 5309 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev, hr_dev 5312 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 5322 drivers/infiniband/hw/hns/hns_roce_hw_v2.c mhop_num = hr_dev->caps.eqe_hop_num; hr_dev 5323 drivers/infiniband/hw/hns/hns_roce_hw_v2.c buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT); hr_dev 5324 drivers/infiniband/hw/hns/hns_roce_hw_v2.c bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT); hr_dev 5378 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_v2_free_eq(struct hns_roce_dev *hr_dev, hr_dev 5385 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (hr_dev->caps.eqe_hop_num) { hr_dev 5386 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_mhop_free_eq(hr_dev, eq); hr_dev 5390 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dma_free_coherent(hr_dev->dev, buf_chk_sz, eq->buf_list->buf, hr_dev 5395 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev, hr_dev 5405 drivers/infiniband/hw/hns/hns_roce_hw_v2.c eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG; hr_dev 5406 drivers/infiniband/hw/hns/hns_roce_hw_v2.c eq->hop_num = hr_dev->caps.eqe_hop_num; hr_dev 5411 drivers/infiniband/hw/hns/hns_roce_hw_v2.c eq->eqe_ba_pg_sz = hr_dev->caps.eqe_ba_pg_sz; hr_dev 5412 drivers/infiniband/hw/hns/hns_roce_hw_v2.c eq->eqe_buf_pg_sz = hr_dev->caps.eqe_buf_pg_sz; hr_dev 5544 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev, hr_dev 5547 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 5563 drivers/infiniband/hw/hns/hns_roce_hw_v2.c mhop_num = hr_dev->caps.eqe_hop_num; hr_dev 5564 drivers/infiniband/hw/hns/hns_roce_hw_v2.c buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT); hr_dev 5565 drivers/infiniband/hw/hns/hns_roce_hw_v2.c bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT); hr_dev 5751 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev, hr_dev 5755 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 5761 drivers/infiniband/hw/hns/hns_roce_hw_v2.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 5765 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (!hr_dev->caps.eqe_hop_num) { hr_dev 5766 drivers/infiniband/hw/hns/hns_roce_hw_v2.c buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT); hr_dev 5784 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_mhop_alloc_eq(hr_dev, eq); hr_dev 5791 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_config_eqc(hr_dev, eq, mailbox->buf); hr_dev 5793 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0, hr_dev 5800 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 5805 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (!hr_dev->caps.eqe_hop_num) hr_dev 5809 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_mhop_free_eq(hr_dev, eq); hr_dev 5817 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 5822 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num, hr_dev 5825 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; hr_dev 5830 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN, hr_dev 5832 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (!hr_dev->irq_names[i]) { hr_dev 5840 drivers/infiniband/hw/hns/hns_roce_hw_v2.c snprintf((char *)hr_dev->irq_names[j], hr_dev 5844 drivers/infiniband/hw/hns/hns_roce_hw_v2.c snprintf((char *)hr_dev->irq_names[j], hr_dev 5849 drivers/infiniband/hw/hns/hns_roce_hw_v2.c snprintf((char *)hr_dev->irq_names[j], hr_dev 5855 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = request_irq(hr_dev->irq[j], hr_dev 5857 drivers/infiniband/hw/hns/hns_roce_hw_v2.c 0, hr_dev->irq_names[j], hr_dev); hr_dev 5862 drivers/infiniband/hw/hns/hns_roce_hw_v2.c 0, hr_dev->irq_names[j + aeq_num], hr_dev 5867 drivers/infiniband/hw/hns/hns_roce_hw_v2.c 0, hr_dev->irq_names[j - comp_num], hr_dev 5870 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Request irq error!\n"); hr_dev 5880 drivers/infiniband/hw/hns/hns_roce_hw_v2.c free_irq(hr_dev->irq[j], hr_dev); hr_dev 5887 drivers/infiniband/hw/hns/hns_roce_hw_v2.c kfree(hr_dev->irq_names[i]); hr_dev 5892 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev) hr_dev 5898 drivers/infiniband/hw/hns/hns_roce_hw_v2.c eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; hr_dev 5899 drivers/infiniband/hw/hns/hns_roce_hw_v2.c irq_num = eq_num + hr_dev->caps.num_other_vectors; hr_dev 5901 drivers/infiniband/hw/hns/hns_roce_hw_v2.c for (i = 0; i < hr_dev->caps.num_other_vectors; i++) hr_dev 5902 drivers/infiniband/hw/hns/hns_roce_hw_v2.c free_irq(hr_dev->irq[i], hr_dev); hr_dev 5905 drivers/infiniband/hw/hns/hns_roce_hw_v2.c free_irq(hr_dev->eq_table.eq[i].irq, &hr_dev->eq_table.eq[i]); hr_dev 5908 drivers/infiniband/hw/hns/hns_roce_hw_v2.c kfree(hr_dev->irq_names[i]); hr_dev 5911 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev) hr_dev 5913 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; hr_dev 5914 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct device *dev = hr_dev->dev; hr_dev 5925 drivers/infiniband/hw/hns/hns_roce_hw_v2.c other_num = hr_dev->caps.num_other_vectors; hr_dev 5926 drivers/infiniband/hw/hns/hns_roce_hw_v2.c comp_num = hr_dev->caps.num_comp_vectors; hr_dev 5927 drivers/infiniband/hw/hns/hns_roce_hw_v2.c aeq_num = hr_dev->caps.num_aeq_vectors; hr_dev 5939 drivers/infiniband/hw/hns/hns_roce_hw_v2.c eq->hr_dev = hr_dev; hr_dev 5945 drivers/infiniband/hw/hns/hns_roce_hw_v2.c eq->entries = hr_dev->caps.ceqe_depth; hr_dev 5947 drivers/infiniband/hw/hns/hns_roce_hw_v2.c eq->irq = hr_dev->irq[i + other_num + aeq_num]; hr_dev 5954 drivers/infiniband/hw/hns/hns_roce_hw_v2.c eq->entries = hr_dev->caps.aeqe_depth; hr_dev 5956 drivers/infiniband/hw/hns/hns_roce_hw_v2.c eq->irq = hr_dev->irq[i - comp_num + other_num]; hr_dev 5961 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd); hr_dev 5969 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE); hr_dev 5971 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num, hr_dev 5978 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->irq_workq = hr_dev 5980 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (!hr_dev->irq_workq) { hr_dev 5989 drivers/infiniband/hw/hns/hns_roce_hw_v2.c __hns_roce_free_irq(hr_dev); hr_dev 5992 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE); hr_dev 5996 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]); hr_dev 6002 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev) hr_dev 6004 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; hr_dev 6008 drivers/infiniband/hw/hns/hns_roce_hw_v2.c eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; hr_dev 6011 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE); hr_dev 6013 drivers/infiniband/hw/hns/hns_roce_hw_v2.c __hns_roce_free_irq(hr_dev); hr_dev 6016 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_v2_destroy_eqc(hr_dev, i); hr_dev 6018 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]); hr_dev 6023 drivers/infiniband/hw/hns/hns_roce_hw_v2.c flush_workqueue(hr_dev->irq_workq); hr_dev 6024 drivers/infiniband/hw/hns/hns_roce_hw_v2.c destroy_workqueue(hr_dev->irq_workq); hr_dev 6027 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev, hr_dev 6044 drivers/infiniband/hw/hns/hns_roce_hw_v2.c (hr_dev->caps.srqwqe_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev 6045 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.srqwqe_hop_num)); hr_dev 6087 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.idx_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev 6088 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.idx_hop_num); hr_dev 6093 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.idx_ba_pg_sz + PG_SHIFT_OFFSET); hr_dev 6097 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.idx_buf_pg_sz + PG_SHIFT_OFFSET); hr_dev 6111 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.srqwqe_ba_pg_sz + PG_SHIFT_OFFSET); hr_dev 6115 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.srqwqe_buf_pg_sz + PG_SHIFT_OFFSET); hr_dev 6126 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); hr_dev 6137 drivers/infiniband/hw/hns/hns_roce_hw_v2.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 6153 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0, hr_dev 6156 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 6158 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, hr_dev 6169 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); hr_dev 6176 drivers/infiniband/hw/hns/hns_roce_hw_v2.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 6181 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0, hr_dev 6185 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "QUERY SRQ cmd process error\n"); hr_dev 6200 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 6233 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); hr_dev 6303 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l); hr_dev 6381 drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev, hr_dev 6384 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_v2_priv *priv = hr_dev->priv; hr_dev 6387 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->hw = &hns_roce_hw_v2; hr_dev 6388 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->dfx = &hns_roce_dfx_hw_v2; hr_dev 6389 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG; hr_dev 6390 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->odb_offset = hr_dev->sdb_offset; hr_dev 6393 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->reg_base = handle->rinfo.roce_io_base; hr_dev 6394 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->caps.num_ports = 1; hr_dev 6395 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->iboe.netdevs[0] = handle->rinfo.netdev; hr_dev 6396 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->iboe.phy_port[0] = 0; hr_dev 6398 drivers/infiniband/hw/hns/hns_roce_hw_v2.c addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid, hr_dev 6399 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->iboe.netdevs[0]->dev_addr); hr_dev 6402 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->irq[i] = pci_irq_vector(handle->pdev, hr_dev 6406 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->cmd_mod = 1; hr_dev 6407 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->loop_idc = 0; hr_dev 6409 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle); hr_dev 6417 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev; hr_dev 6420 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev = ib_alloc_device(hns_roce_dev, ib_dev); hr_dev 6421 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (!hr_dev) hr_dev 6424 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL); hr_dev 6425 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (!hr_dev->priv) { hr_dev 6430 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->pci_dev = handle->pdev; hr_dev 6431 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->dev = &handle->pdev->dev; hr_dev 6433 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_hw_v2_get_cfg(hr_dev, handle); hr_dev 6435 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "Get Configuration failed!\n"); hr_dev 6439 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ret = hns_roce_init(hr_dev); hr_dev 6441 drivers/infiniband/hw/hns/hns_roce_hw_v2.c dev_err(hr_dev->dev, "RoCE Engine init failed!\n"); hr_dev 6445 drivers/infiniband/hw/hns/hns_roce_hw_v2.c handle->priv = hr_dev; hr_dev 6450 drivers/infiniband/hw/hns/hns_roce_hw_v2.c kfree(hr_dev->priv); hr_dev 6453 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ib_dealloc_device(&hr_dev->ib_dev); hr_dev 6461 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle->priv; hr_dev 6463 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (!hr_dev) hr_dev 6467 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hns_roce_exit(hr_dev); hr_dev 6468 drivers/infiniband/hw/hns/hns_roce_hw_v2.c kfree(hr_dev->priv); hr_dev 6469 drivers/infiniband/hw/hns/hns_roce_hw_v2.c ib_dealloc_device(&hr_dev->ib_dev); hr_dev 6527 drivers/infiniband/hw/hns/hns_roce_hw_v2.c struct hns_roce_dev *hr_dev; hr_dev 6538 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev = (struct hns_roce_dev *)handle->priv; hr_dev 6539 drivers/infiniband/hw/hns/hns_roce_hw_v2.c if (!hr_dev) hr_dev 6542 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->is_reset = true; hr_dev 6543 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->active = false; hr_dev 6544 drivers/infiniband/hw/hns/hns_roce_hw_v2.c hr_dev->dis_db = true; hr_dev 6547 drivers/infiniband/hw/hns/hns_roce_hw_v2.c event.device = &hr_dev->ib_dev; hr_dev 1824 drivers/infiniband/hw/hns/hns_roce_hw_v2.h int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn, hr_dev 1827 drivers/infiniband/hw/hns/hns_roce_hw_v2.h static inline void hns_roce_write64(struct hns_roce_dev *hr_dev, __le32 val[2], hr_dev 1830 drivers/infiniband/hw/hns/hns_roce_hw_v2.h struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; hr_dev 1834 drivers/infiniband/hw/hns/hns_roce_hw_v2.h if (!hr_dev->dis_db && !ops->get_hw_reset_stat(handle)) hr_dev 9 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn, hr_dev 16 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 21 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, cqn, 0, hr_dev 25 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c dev_err(hr_dev->dev, "QUERY cqc cmd process error\n"); hr_dev 32 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 56 drivers/infiniband/hw/hns/hns_roce_main.c int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index) hr_dev 58 drivers/infiniband/hw/hns/hns_roce_main.c return gid_index * hr_dev->caps.num_ports + port; hr_dev 61 drivers/infiniband/hw/hns/hns_roce_main.c static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr) hr_dev 66 drivers/infiniband/hw/hns/hns_roce_main.c if (!memcmp(hr_dev->dev_addr[port], addr, ETH_ALEN)) hr_dev 70 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->dev_addr[port][i] = addr[i]; hr_dev 72 drivers/infiniband/hw/hns/hns_roce_main.c phy_port = hr_dev->iboe.phy_port[port]; hr_dev 73 drivers/infiniband/hw/hns/hns_roce_main.c return hr_dev->hw->set_mac(hr_dev, phy_port, addr); hr_dev 78 drivers/infiniband/hw/hns/hns_roce_main.c struct hns_roce_dev *hr_dev = to_hr_dev(attr->device); hr_dev 82 drivers/infiniband/hw/hns/hns_roce_main.c if (port >= hr_dev->caps.num_ports) hr_dev 85 drivers/infiniband/hw/hns/hns_roce_main.c ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, &attr->gid, attr); hr_dev 92 drivers/infiniband/hw/hns/hns_roce_main.c struct hns_roce_dev *hr_dev = to_hr_dev(attr->device); hr_dev 97 drivers/infiniband/hw/hns/hns_roce_main.c if (port >= hr_dev->caps.num_ports) hr_dev 100 drivers/infiniband/hw/hns/hns_roce_main.c ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, &zgid, &zattr); hr_dev 105 drivers/infiniband/hw/hns/hns_roce_main.c static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port, hr_dev 108 drivers/infiniband/hw/hns/hns_roce_main.c struct device *dev = hr_dev->dev; hr_dev 112 drivers/infiniband/hw/hns/hns_roce_main.c netdev = hr_dev->iboe.netdevs[port]; hr_dev 123 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_set_mac(hr_dev, port, netdev->dev_addr); hr_dev 143 drivers/infiniband/hw/hns/hns_roce_main.c struct hns_roce_dev *hr_dev = NULL; hr_dev 147 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev = container_of(self, struct hns_roce_dev, iboe.nb); hr_dev 148 drivers/infiniband/hw/hns/hns_roce_main.c iboe = &hr_dev->iboe; hr_dev 150 drivers/infiniband/hw/hns/hns_roce_main.c for (port = 0; port < hr_dev->caps.num_ports; port++) { hr_dev 152 drivers/infiniband/hw/hns/hns_roce_main.c ret = handle_en_event(hr_dev, port, event); hr_dev 162 drivers/infiniband/hw/hns/hns_roce_main.c static int hns_roce_setup_mtu_mac(struct hns_roce_dev *hr_dev) hr_dev 167 drivers/infiniband/hw/hns/hns_roce_main.c for (i = 0; i < hr_dev->caps.num_ports; i++) { hr_dev 168 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->hw->set_mtu) hr_dev 169 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->hw->set_mtu(hr_dev, hr_dev->iboe.phy_port[i], hr_dev 170 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.max_mtu); hr_dev 171 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_set_mac(hr_dev, i, hr_dev 172 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->iboe.netdevs[i]->dev_addr); hr_dev 184 drivers/infiniband/hw/hns/hns_roce_main.c struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); hr_dev 188 drivers/infiniband/hw/hns/hns_roce_main.c props->fw_ver = hr_dev->caps.fw_ver; hr_dev 189 drivers/infiniband/hw/hns/hns_roce_main.c props->sys_image_guid = cpu_to_be64(hr_dev->sys_image_guid); hr_dev 191 drivers/infiniband/hw/hns/hns_roce_main.c props->page_size_cap = hr_dev->caps.page_size_cap; hr_dev 192 drivers/infiniband/hw/hns/hns_roce_main.c props->vendor_id = hr_dev->vendor_id; hr_dev 193 drivers/infiniband/hw/hns/hns_roce_main.c props->vendor_part_id = hr_dev->vendor_part_id; hr_dev 194 drivers/infiniband/hw/hns/hns_roce_main.c props->hw_ver = hr_dev->hw_rev; hr_dev 195 drivers/infiniband/hw/hns/hns_roce_main.c props->max_qp = hr_dev->caps.num_qps; hr_dev 196 drivers/infiniband/hw/hns/hns_roce_main.c props->max_qp_wr = hr_dev->caps.max_wqes; hr_dev 199 drivers/infiniband/hw/hns/hns_roce_main.c props->max_send_sge = hr_dev->caps.max_sq_sg; hr_dev 200 drivers/infiniband/hw/hns/hns_roce_main.c props->max_recv_sge = hr_dev->caps.max_rq_sg; hr_dev 202 drivers/infiniband/hw/hns/hns_roce_main.c props->max_cq = hr_dev->caps.num_cqs; hr_dev 203 drivers/infiniband/hw/hns/hns_roce_main.c props->max_cqe = hr_dev->caps.max_cqes; hr_dev 204 drivers/infiniband/hw/hns/hns_roce_main.c props->max_mr = hr_dev->caps.num_mtpts; hr_dev 205 drivers/infiniband/hw/hns/hns_roce_main.c props->max_pd = hr_dev->caps.num_pds; hr_dev 206 drivers/infiniband/hw/hns/hns_roce_main.c props->max_qp_rd_atom = hr_dev->caps.max_qp_dest_rdma; hr_dev 207 drivers/infiniband/hw/hns/hns_roce_main.c props->max_qp_init_rd_atom = hr_dev->caps.max_qp_init_rdma; hr_dev 208 drivers/infiniband/hw/hns/hns_roce_main.c props->atomic_cap = hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_ATOMIC ? hr_dev 211 drivers/infiniband/hw/hns/hns_roce_main.c props->local_ca_ack_delay = hr_dev->caps.local_ca_ack_delay; hr_dev 212 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) { hr_dev 213 drivers/infiniband/hw/hns/hns_roce_main.c props->max_srq = hr_dev->caps.max_srqs; hr_dev 214 drivers/infiniband/hw/hns/hns_roce_main.c props->max_srq_wr = hr_dev->caps.max_srq_wrs; hr_dev 215 drivers/infiniband/hw/hns/hns_roce_main.c props->max_srq_sge = hr_dev->caps.max_srq_sges; hr_dev 218 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_FRMR) { hr_dev 229 drivers/infiniband/hw/hns/hns_roce_main.c struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); hr_dev 230 drivers/infiniband/hw/hns/hns_roce_main.c struct device *dev = hr_dev->dev; hr_dev 241 drivers/infiniband/hw/hns/hns_roce_main.c props->max_mtu = hr_dev->caps.max_mtu; hr_dev 242 drivers/infiniband/hw/hns/hns_roce_main.c props->gid_tbl_len = hr_dev->caps.gid_table_len[port]; hr_dev 251 drivers/infiniband/hw/hns/hns_roce_main.c spin_lock_irqsave(&hr_dev->iboe.lock, flags); hr_dev 253 drivers/infiniband/hw/hns/hns_roce_main.c net_dev = hr_dev->iboe.netdevs[port]; hr_dev 255 drivers/infiniband/hw/hns/hns_roce_main.c spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); hr_dev 268 drivers/infiniband/hw/hns/hns_roce_main.c spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); hr_dev 316 drivers/infiniband/hw/hns/hns_roce_main.c struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device); hr_dev 318 drivers/infiniband/hw/hns/hns_roce_main.c if (!hr_dev->active) hr_dev 321 drivers/infiniband/hw/hns/hns_roce_main.c resp.qp_tab_size = hr_dev->caps.num_qps; hr_dev 323 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_uar_alloc(hr_dev, &context->uar); hr_dev 327 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) { hr_dev 339 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_uar_free(hr_dev, &context->uar); hr_dev 355 drivers/infiniband/hw/hns/hns_roce_main.c struct hns_roce_dev *hr_dev = to_hr_dev(context->device); hr_dev 366 drivers/infiniband/hw/hns/hns_roce_main.c if (!hr_dev->tptr_dma_addr || !hr_dev->tptr_size) hr_dev 373 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->tptr_dma_addr >> PAGE_SHIFT, hr_dev 374 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->tptr_size, hr_dev 407 drivers/infiniband/hw/hns/hns_roce_main.c static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev) hr_dev 409 drivers/infiniband/hw/hns/hns_roce_main.c struct hns_roce_ib_iboe *iboe = &hr_dev->iboe; hr_dev 411 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->active = false; hr_dev 413 drivers/infiniband/hw/hns/hns_roce_main.c ib_unregister_device(&hr_dev->ib_dev); hr_dev 476 drivers/infiniband/hw/hns/hns_roce_main.c static int hns_roce_register_device(struct hns_roce_dev *hr_dev) hr_dev 481 drivers/infiniband/hw/hns/hns_roce_main.c struct device *dev = hr_dev->dev; hr_dev 484 drivers/infiniband/hw/hns/hns_roce_main.c iboe = &hr_dev->iboe; hr_dev 487 drivers/infiniband/hw/hns/hns_roce_main.c ib_dev = &hr_dev->ib_dev; hr_dev 492 drivers/infiniband/hw/hns/hns_roce_main.c ib_dev->phys_port_cnt = hr_dev->caps.num_ports; hr_dev 493 drivers/infiniband/hw/hns/hns_roce_main.c ib_dev->local_dma_lkey = hr_dev->caps.reserved_lkey; hr_dev 494 drivers/infiniband/hw/hns/hns_roce_main.c ib_dev->num_comp_vectors = hr_dev->caps.num_comp_vectors; hr_dev 514 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_REREG_MR) { hr_dev 520 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_MW) { hr_dev 528 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_FRMR) hr_dev 532 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) { hr_dev 540 drivers/infiniband/hw/hns/hns_roce_main.c ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_srq_ops); hr_dev 543 drivers/infiniband/hw/hns/hns_roce_main.c ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_ops); hr_dev 545 drivers/infiniband/hw/hns/hns_roce_main.c for (i = 0; i < hr_dev->caps.num_ports; i++) { hr_dev 546 drivers/infiniband/hw/hns/hns_roce_main.c if (!hr_dev->iboe.netdevs[i]) hr_dev 549 drivers/infiniband/hw/hns/hns_roce_main.c ret = ib_device_set_netdev(ib_dev, hr_dev->iboe.netdevs[i], hr_dev 560 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_setup_mtu_mac(hr_dev); hr_dev 573 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->active = true; hr_dev 582 drivers/infiniband/hw/hns/hns_roce_main.c static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) hr_dev 585 drivers/infiniband/hw/hns/hns_roce_main.c struct device *dev = hr_dev->dev; hr_dev 587 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_hem_table(hr_dev, &hr_dev->mr_table.mtt_table, hr_dev 588 drivers/infiniband/hw/hns/hns_roce_main.c HEM_TYPE_MTT, hr_dev->caps.mtt_entry_sz, hr_dev 589 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.num_mtt_segs, 1); hr_dev 595 drivers/infiniband/hw/hns/hns_roce_main.c if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) { hr_dev 596 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_hem_table(hr_dev, hr_dev 597 drivers/infiniband/hw/hns/hns_roce_main.c &hr_dev->mr_table.mtt_cqe_table, hr_dev 598 drivers/infiniband/hw/hns/hns_roce_main.c HEM_TYPE_CQE, hr_dev->caps.mtt_entry_sz, hr_dev 599 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.num_cqe_segs, 1); hr_dev 606 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table, hr_dev 607 drivers/infiniband/hw/hns/hns_roce_main.c HEM_TYPE_MTPT, hr_dev->caps.mtpt_entry_sz, hr_dev 608 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.num_mtpts, 1); hr_dev 614 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.qp_table, hr_dev 615 drivers/infiniband/hw/hns/hns_roce_main.c HEM_TYPE_QPC, hr_dev->caps.qpc_entry_sz, hr_dev 616 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.num_qps, 1); hr_dev 622 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.irrl_table, hr_dev 624 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.irrl_entry_sz * hr_dev 625 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.max_qp_init_rdma, hr_dev 626 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.num_qps, 1); hr_dev 632 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.trrl_entry_sz) { hr_dev 633 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_hem_table(hr_dev, hr_dev 634 drivers/infiniband/hw/hns/hns_roce_main.c &hr_dev->qp_table.trrl_table, hr_dev 636 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.trrl_entry_sz * hr_dev 637 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.max_qp_dest_rdma, hr_dev 638 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.num_qps, 1); hr_dev 646 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_hem_table(hr_dev, &hr_dev->cq_table.table, hr_dev 647 drivers/infiniband/hw/hns/hns_roce_main.c HEM_TYPE_CQC, hr_dev->caps.cqc_entry_sz, hr_dev 648 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.num_cqs, 1); hr_dev 654 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.srqc_entry_sz) { hr_dev 655 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_hem_table(hr_dev, &hr_dev->srq_table.table, hr_dev 657 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.srqc_entry_sz, hr_dev 658 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.num_srqs, 1); hr_dev 666 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.num_srqwqe_segs) { hr_dev 667 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_hem_table(hr_dev, hr_dev 668 drivers/infiniband/hw/hns/hns_roce_main.c &hr_dev->mr_table.mtt_srqwqe_table, hr_dev 670 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.mtt_entry_sz, hr_dev 671 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.num_srqwqe_segs, 1); hr_dev 679 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.num_idx_segs) { hr_dev 680 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_hem_table(hr_dev, hr_dev 681 drivers/infiniband/hw/hns/hns_roce_main.c &hr_dev->mr_table.mtt_idx_table, hr_dev 683 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.idx_entry_sz, hr_dev 684 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.num_idx_segs, 1); hr_dev 692 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.sccc_entry_sz) { hr_dev 693 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_hem_table(hr_dev, hr_dev 694 drivers/infiniband/hw/hns/hns_roce_main.c &hr_dev->qp_table.sccc_table, hr_dev 696 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.sccc_entry_sz, hr_dev 697 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.num_qps, 1); hr_dev 705 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.qpc_timer_entry_sz) { hr_dev 706 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_hem_table(hr_dev, hr_dev 707 drivers/infiniband/hw/hns/hns_roce_main.c &hr_dev->qpc_timer_table, hr_dev 709 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.qpc_timer_entry_sz, hr_dev 710 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.num_qpc_timer, 1); hr_dev 718 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.cqc_timer_entry_sz) { hr_dev 719 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_hem_table(hr_dev, hr_dev 720 drivers/infiniband/hw/hns/hns_roce_main.c &hr_dev->cqc_timer_table, hr_dev 722 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.cqc_timer_entry_sz, hr_dev 723 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->caps.num_cqc_timer, 1); hr_dev 734 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.qpc_timer_entry_sz) hr_dev 735 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_hem_table(hr_dev, hr_dev 736 drivers/infiniband/hw/hns/hns_roce_main.c &hr_dev->qpc_timer_table); hr_dev 739 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.sccc_entry_sz) hr_dev 740 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_hem_table(hr_dev, hr_dev 741 drivers/infiniband/hw/hns/hns_roce_main.c &hr_dev->qp_table.sccc_table); hr_dev 744 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.num_idx_segs) hr_dev 745 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_hem_table(hr_dev, hr_dev 746 drivers/infiniband/hw/hns/hns_roce_main.c &hr_dev->mr_table.mtt_idx_table); hr_dev 749 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.num_srqwqe_segs) hr_dev 750 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_hem_table(hr_dev, hr_dev 751 drivers/infiniband/hw/hns/hns_roce_main.c &hr_dev->mr_table.mtt_srqwqe_table); hr_dev 754 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.srqc_entry_sz) hr_dev 755 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_hem_table(hr_dev, &hr_dev->srq_table.table); hr_dev 758 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); hr_dev 761 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.trrl_entry_sz) hr_dev 762 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_hem_table(hr_dev, hr_dev 763 drivers/infiniband/hw/hns/hns_roce_main.c &hr_dev->qp_table.trrl_table); hr_dev 766 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); hr_dev 769 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); hr_dev 772 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); hr_dev 775 drivers/infiniband/hw/hns/hns_roce_main.c if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) hr_dev 776 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_hem_table(hr_dev, hr_dev 777 drivers/infiniband/hw/hns/hns_roce_main.c &hr_dev->mr_table.mtt_cqe_table); hr_dev 780 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table); hr_dev 790 drivers/infiniband/hw/hns/hns_roce_main.c static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) hr_dev 793 drivers/infiniband/hw/hns/hns_roce_main.c struct device *dev = hr_dev->dev; hr_dev 795 drivers/infiniband/hw/hns/hns_roce_main.c spin_lock_init(&hr_dev->sm_lock); hr_dev 796 drivers/infiniband/hw/hns/hns_roce_main.c spin_lock_init(&hr_dev->bt_cmd_lock); hr_dev 798 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) { hr_dev 799 drivers/infiniband/hw/hns/hns_roce_main.c INIT_LIST_HEAD(&hr_dev->pgdir_list); hr_dev 800 drivers/infiniband/hw/hns/hns_roce_main.c mutex_init(&hr_dev->pgdir_mutex); hr_dev 803 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_uar_table(hr_dev); hr_dev 809 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_uar_alloc(hr_dev, &hr_dev->priv_uar); hr_dev 815 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_pd_table(hr_dev); hr_dev 821 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_mr_table(hr_dev); hr_dev 827 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_cq_table(hr_dev); hr_dev 833 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_qp_table(hr_dev); hr_dev 839 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) { hr_dev 840 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_srq_table(hr_dev); hr_dev 851 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) hr_dev 852 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_qp_table(hr_dev); hr_dev 855 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_cq_table(hr_dev); hr_dev 858 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_mr_table(hr_dev); hr_dev 861 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_pd_table(hr_dev); hr_dev 864 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_uar_free(hr_dev, &hr_dev->priv_uar); hr_dev 867 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_uar_table(hr_dev); hr_dev 871 drivers/infiniband/hw/hns/hns_roce_main.c int hns_roce_init(struct hns_roce_dev *hr_dev) hr_dev 874 drivers/infiniband/hw/hns/hns_roce_main.c struct device *dev = hr_dev->dev; hr_dev 876 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->hw->reset) { hr_dev 877 drivers/infiniband/hw/hns/hns_roce_main.c ret = hr_dev->hw->reset(hr_dev, true); hr_dev 883 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->is_reset = false; hr_dev 885 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->hw->cmq_init) { hr_dev 886 drivers/infiniband/hw/hns/hns_roce_main.c ret = hr_dev->hw->cmq_init(hr_dev); hr_dev 893 drivers/infiniband/hw/hns/hns_roce_main.c ret = hr_dev->hw->hw_profile(hr_dev); hr_dev 899 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_cmd_init(hr_dev); hr_dev 906 drivers/infiniband/hw/hns/hns_roce_main.c ret = hr_dev->hw->init_eq(hr_dev); hr_dev 912 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->cmd_mod) { hr_dev 913 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_cmd_use_events(hr_dev); hr_dev 917 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cmd_use_polling(hr_dev); hr_dev 921 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_init_hem(hr_dev); hr_dev 927 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_setup_hca(hr_dev); hr_dev 933 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->hw->hw_init) { hr_dev 934 drivers/infiniband/hw/hns/hns_roce_main.c ret = hr_dev->hw->hw_init(hr_dev); hr_dev 941 drivers/infiniband/hw/hns/hns_roce_main.c ret = hns_roce_register_device(hr_dev); hr_dev 948 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->hw->hw_exit) hr_dev 949 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->hw->hw_exit(hr_dev); hr_dev 952 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_bitmap(hr_dev); hr_dev 955 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_hem(hr_dev); hr_dev 958 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->cmd_mod) hr_dev 959 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cmd_use_polling(hr_dev); hr_dev 960 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->hw->cleanup_eq(hr_dev); hr_dev 963 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cmd_cleanup(hr_dev); hr_dev 966 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->hw->cmq_exit) hr_dev 967 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->hw->cmq_exit(hr_dev); hr_dev 970 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->hw->reset) { hr_dev 971 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->hw->reset(hr_dev, false)) hr_dev 978 drivers/infiniband/hw/hns/hns_roce_main.c void hns_roce_exit(struct hns_roce_dev *hr_dev) hr_dev 980 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_unregister_device(hr_dev); hr_dev 982 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->hw->hw_exit) hr_dev 983 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->hw->hw_exit(hr_dev); hr_dev 984 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_bitmap(hr_dev); hr_dev 985 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cleanup_hem(hr_dev); hr_dev 987 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->cmd_mod) hr_dev 988 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cmd_use_polling(hr_dev); hr_dev 990 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->hw->cleanup_eq(hr_dev); hr_dev 991 drivers/infiniband/hw/hns/hns_roce_main.c hns_roce_cmd_cleanup(hr_dev); hr_dev 992 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->hw->cmq_exit) hr_dev 993 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->hw->cmq_exit(hr_dev); hr_dev 994 drivers/infiniband/hw/hns/hns_roce_main.c if (hr_dev->hw->reset) hr_dev 995 drivers/infiniband/hw/hns/hns_roce_main.c hr_dev->hw->reset(hr_dev, false); hr_dev 51 drivers/infiniband/hw/hns/hns_roce_mr.c static int hns_roce_sw2hw_mpt(struct hns_roce_dev *hr_dev, hr_dev 55 drivers/infiniband/hw/hns/hns_roce_mr.c return hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, mpt_index, 0, hr_dev 60 drivers/infiniband/hw/hns/hns_roce_mr.c int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev, hr_dev 64 drivers/infiniband/hw/hns/hns_roce_mr.c return hns_roce_cmd_mbox(hr_dev, 0, mailbox ? mailbox->dma : 0, hr_dev 177 drivers/infiniband/hw/hns/hns_roce_mr.c static int hns_roce_alloc_mtt_range(struct hns_roce_dev *hr_dev, int order, hr_dev 180 drivers/infiniband/hw/hns/hns_roce_mr.c struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; hr_dev 203 drivers/infiniband/hw/hns/hns_roce_mr.c dev_err(hr_dev->dev, "Unsupport MTT table type: %d\n", hr_dev 212 drivers/infiniband/hw/hns/hns_roce_mr.c if (hns_roce_table_get_range(hr_dev, table, *seg, hr_dev 221 drivers/infiniband/hw/hns/hns_roce_mr.c int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift, hr_dev 243 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_alloc_mtt_range(hr_dev, mtt->order, &mtt->first_seg, hr_dev 251 drivers/infiniband/hw/hns/hns_roce_mr.c void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt) hr_dev 253 drivers/infiniband/hw/hns/hns_roce_mr.c struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; hr_dev 262 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_table_put_range(hr_dev, &mr_table->mtt_table, hr_dev 269 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_table_put_range(hr_dev, &mr_table->mtt_cqe_table, hr_dev 276 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_table_put_range(hr_dev, &mr_table->mtt_srqwqe_table, hr_dev 283 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_table_put_range(hr_dev, &mr_table->mtt_idx_table, hr_dev 288 drivers/infiniband/hw/hns/hns_roce_mr.c dev_err(hr_dev->dev, hr_dev 295 drivers/infiniband/hw/hns/hns_roce_mr.c static void hns_roce_loop_free(struct hns_roce_dev *hr_dev, hr_dev 299 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 305 drivers/infiniband/hw/hns/hns_roce_mr.c pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); hr_dev 306 drivers/infiniband/hw/hns/hns_roce_mr.c mhop_num = hr_dev->caps.pbl_hop_num; hr_dev 350 drivers/infiniband/hw/hns/hns_roce_mr.c static int pbl_1hop_alloc(struct hns_roce_dev *hr_dev, int npages, hr_dev 353 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 369 drivers/infiniband/hw/hns/hns_roce_mr.c mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz; hr_dev 370 drivers/infiniband/hw/hns/hns_roce_mr.c mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz; hr_dev 376 drivers/infiniband/hw/hns/hns_roce_mr.c static int pbl_2hop_alloc(struct hns_roce_dev *hr_dev, int npages, hr_dev 379 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 400 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_loop_free(hr_dev, mr, 1, i, 0); hr_dev 416 drivers/infiniband/hw/hns/hns_roce_mr.c static int pbl_3hop_alloc(struct hns_roce_dev *hr_dev, int npages, hr_dev 419 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 449 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_loop_free(hr_dev, mr, 1, i, 0); hr_dev 470 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_loop_free(hr_dev, mr, 2, i, j); hr_dev 507 drivers/infiniband/hw/hns/hns_roce_mr.c static int hns_roce_mhop_alloc(struct hns_roce_dev *hr_dev, int npages, hr_dev 510 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 514 drivers/infiniband/hw/hns/hns_roce_mr.c mhop_num = (mr->type == MR_TYPE_FRMR ? 1 : hr_dev->caps.pbl_hop_num); hr_dev 515 drivers/infiniband/hw/hns/hns_roce_mr.c pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); hr_dev 521 drivers/infiniband/hw/hns/hns_roce_mr.c return pbl_1hop_alloc(hr_dev, npages, mr, pbl_bt_sz); hr_dev 542 drivers/infiniband/hw/hns/hns_roce_mr.c if (pbl_2hop_alloc(hr_dev, npages, mr, pbl_bt_sz)) hr_dev 547 drivers/infiniband/hw/hns/hns_roce_mr.c if (pbl_3hop_alloc(hr_dev, npages, mr, pbl_bt_sz)) hr_dev 554 drivers/infiniband/hw/hns/hns_roce_mr.c mr->pbl_hop_num = hr_dev->caps.pbl_hop_num; hr_dev 555 drivers/infiniband/hw/hns/hns_roce_mr.c mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz; hr_dev 556 drivers/infiniband/hw/hns/hns_roce_mr.c mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz; hr_dev 571 drivers/infiniband/hw/hns/hns_roce_mr.c static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova, hr_dev 575 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 580 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index); hr_dev 602 drivers/infiniband/hw/hns/hns_roce_mr.c if (!hr_dev->caps.pbl_hop_num) { hr_dev 610 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_mhop_alloc(hr_dev, npages, mr); hr_dev 617 drivers/infiniband/hw/hns/hns_roce_mr.c static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev, hr_dev 620 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 629 drivers/infiniband/hw/hns/hns_roce_mr.c pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); hr_dev 630 drivers/infiniband/hw/hns/hns_roce_mr.c mhop_num = (mr->type == MR_TYPE_FRMR) ? 1 : hr_dev->caps.pbl_hop_num; hr_dev 702 drivers/infiniband/hw/hns/hns_roce_mr.c static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, hr_dev 705 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 710 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mr->key) hr_dev 711 drivers/infiniband/hw/hns/hns_roce_mr.c & (hr_dev->caps.num_mtpts - 1)); hr_dev 720 drivers/infiniband/hw/hns/hns_roce_mr.c if (!hr_dev->caps.pbl_hop_num) hr_dev 725 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_mhop_free(hr_dev, mr); hr_dev 729 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table, hr_dev 732 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, hr_dev 736 drivers/infiniband/hw/hns/hns_roce_mr.c static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev, hr_dev 741 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 743 drivers/infiniband/hw/hns/hns_roce_mr.c struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; hr_dev 746 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx); hr_dev 751 drivers/infiniband/hw/hns/hns_roce_mr.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 758 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hr_dev->hw->write_mtpt(mailbox->buf, mr, mtpt_idx); hr_dev 760 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hr_dev->hw->frmr_write_mtpt(mailbox->buf, mr); hr_dev 766 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_sw2hw_mpt(hr_dev, mailbox, hr_dev 767 drivers/infiniband/hw/hns/hns_roce_mr.c mtpt_idx & (hr_dev->caps.num_mtpts - 1)); hr_dev 774 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 779 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 782 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx); hr_dev 786 drivers/infiniband/hw/hns/hns_roce_mr.c static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev, hr_dev 798 drivers/infiniband/hw/hns/hns_roce_mr.c table = &hr_dev->mr_table.mtt_table; hr_dev 799 drivers/infiniband/hw/hns/hns_roce_mr.c bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT); hr_dev 802 drivers/infiniband/hw/hns/hns_roce_mr.c table = &hr_dev->mr_table.mtt_cqe_table; hr_dev 803 drivers/infiniband/hw/hns/hns_roce_mr.c bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT); hr_dev 806 drivers/infiniband/hw/hns/hns_roce_mr.c table = &hr_dev->mr_table.mtt_srqwqe_table; hr_dev 807 drivers/infiniband/hw/hns/hns_roce_mr.c bt_page_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT); hr_dev 810 drivers/infiniband/hw/hns/hns_roce_mr.c table = &hr_dev->mr_table.mtt_idx_table; hr_dev 811 drivers/infiniband/hw/hns/hns_roce_mr.c bt_page_size = 1 << (hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT); hr_dev 825 drivers/infiniband/hw/hns/hns_roce_mr.c mtts = hns_roce_table_find(hr_dev, table, hr_dev 834 drivers/infiniband/hw/hns/hns_roce_mr.c if (!hr_dev->caps.mtt_hop_num) hr_dev 843 drivers/infiniband/hw/hns/hns_roce_mr.c static int hns_roce_write_mtt(struct hns_roce_dev *hr_dev, hr_dev 856 drivers/infiniband/hw/hns/hns_roce_mr.c bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT); hr_dev 859 drivers/infiniband/hw/hns/hns_roce_mr.c bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT); hr_dev 862 drivers/infiniband/hw/hns/hns_roce_mr.c bt_page_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT); hr_dev 865 drivers/infiniband/hw/hns/hns_roce_mr.c bt_page_size = 1 << (hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT); hr_dev 868 drivers/infiniband/hw/hns/hns_roce_mr.c dev_err(hr_dev->dev, hr_dev 877 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_write_mtt_chunk(hr_dev, mtt, start_index, chunk, hr_dev 890 drivers/infiniband/hw/hns/hns_roce_mr.c int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev, hr_dev 908 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_write_mtt(hr_dev, mtt, 0, buf->npages, page_list); hr_dev 915 drivers/infiniband/hw/hns/hns_roce_mr.c int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev) hr_dev 917 drivers/infiniband/hw/hns/hns_roce_mr.c struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; hr_dev 921 drivers/infiniband/hw/hns/hns_roce_mr.c hr_dev->caps.num_mtpts, hr_dev 922 drivers/infiniband/hw/hns/hns_roce_mr.c hr_dev->caps.num_mtpts - 1, hr_dev 923 drivers/infiniband/hw/hns/hns_roce_mr.c hr_dev->caps.reserved_mrws, 0); hr_dev 928 drivers/infiniband/hw/hns/hns_roce_mr.c ilog2(hr_dev->caps.num_mtt_segs)); hr_dev 932 drivers/infiniband/hw/hns/hns_roce_mr.c if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) { hr_dev 934 drivers/infiniband/hw/hns/hns_roce_mr.c ilog2(hr_dev->caps.num_cqe_segs)); hr_dev 939 drivers/infiniband/hw/hns/hns_roce_mr.c if (hr_dev->caps.num_srqwqe_segs) { hr_dev 941 drivers/infiniband/hw/hns/hns_roce_mr.c ilog2(hr_dev->caps.num_srqwqe_segs)); hr_dev 946 drivers/infiniband/hw/hns/hns_roce_mr.c if (hr_dev->caps.num_idx_segs) { hr_dev 948 drivers/infiniband/hw/hns/hns_roce_mr.c ilog2(hr_dev->caps.num_idx_segs)); hr_dev 956 drivers/infiniband/hw/hns/hns_roce_mr.c if (hr_dev->caps.num_srqwqe_segs) hr_dev 960 drivers/infiniband/hw/hns/hns_roce_mr.c if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) hr_dev 971 drivers/infiniband/hw/hns/hns_roce_mr.c void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev) hr_dev 973 drivers/infiniband/hw/hns/hns_roce_mr.c struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; hr_dev 975 drivers/infiniband/hw/hns/hns_roce_mr.c if (hr_dev->caps.num_idx_segs) hr_dev 977 drivers/infiniband/hw/hns/hns_roce_mr.c if (hr_dev->caps.num_srqwqe_segs) hr_dev 980 drivers/infiniband/hw/hns/hns_roce_mr.c if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) hr_dev 1019 drivers/infiniband/hw/hns/hns_roce_mr.c int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev, hr_dev 1022 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 1035 drivers/infiniband/hw/hns/hns_roce_mr.c order = hr_dev->caps.mtt_ba_pg_sz; hr_dev 1038 drivers/infiniband/hw/hns/hns_roce_mr.c order = hr_dev->caps.cqe_ba_pg_sz; hr_dev 1041 drivers/infiniband/hw/hns/hns_roce_mr.c order = hr_dev->caps.srqwqe_ba_pg_sz; hr_dev 1044 drivers/infiniband/hw/hns/hns_roce_mr.c order = hr_dev->caps.idx_ba_pg_sz; hr_dev 1074 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages); hr_dev 1083 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages); hr_dev 1090 drivers/infiniband/hw/hns/hns_roce_mr.c static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev, hr_dev 1099 drivers/infiniband/hw/hns/hns_roce_mr.c if (hr_dev->caps.pbl_hop_num == HNS_ROCE_HOP_NUM_0) hr_dev 1102 drivers/infiniband/hw/hns/hns_roce_mr.c pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); hr_dev 1105 drivers/infiniband/hw/hns/hns_roce_mr.c if (!hr_dev->caps.pbl_hop_num) { hr_dev 1108 drivers/infiniband/hw/hns/hns_roce_mr.c } else if (hr_dev->caps.pbl_hop_num == 1) { hr_dev 1111 drivers/infiniband/hw/hns/hns_roce_mr.c if (hr_dev->caps.pbl_hop_num == 2) hr_dev 1113 drivers/infiniband/hw/hns/hns_roce_mr.c else if (hr_dev->caps.pbl_hop_num == 3) hr_dev 1134 drivers/infiniband/hw/hns/hns_roce_mr.c struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); hr_dev 1135 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 1154 drivers/infiniband/hw/hns/hns_roce_mr.c if (!hr_dev->caps.pbl_hop_num) { hr_dev 1165 drivers/infiniband/hw/hns/hns_roce_mr.c bt_size = (1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT)) / hr_dev 1167 drivers/infiniband/hw/hns/hns_roce_mr.c for (i = 0; i < hr_dev->caps.pbl_hop_num; i++) hr_dev 1180 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, virt_addr, length, hr_dev 1185 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem); hr_dev 1189 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_mr_enable(hr_dev, mr); hr_dev 1198 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_mr_free(hr_dev, mr); hr_dev 1214 drivers/infiniband/hw/hns/hns_roce_mr.c struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device); hr_dev 1216 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 1223 drivers/infiniband/hw/hns/hns_roce_mr.c if (hr_dev->caps.pbl_hop_num) hr_dev 1224 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_mhop_free(hr_dev, mr); hr_dev 1239 drivers/infiniband/hw/hns/hns_roce_mr.c if (hr_dev->caps.pbl_hop_num) { hr_dev 1240 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_mhop_alloc(hr_dev, npages, mr); hr_dev 1253 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, pdn, hr_dev 1260 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem); hr_dev 1265 drivers/infiniband/hw/hns/hns_roce_mr.c if (hr_dev->caps.pbl_hop_num) hr_dev 1266 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_mhop_free(hr_dev, mr); hr_dev 1289 drivers/infiniband/hw/hns/hns_roce_mr.c struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device); hr_dev 1292 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 1300 drivers/infiniband/hw/hns/hns_roce_mr.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 1304 drivers/infiniband/hw/hns/hns_roce_mr.c mtpt_idx = key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1); hr_dev 1305 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, mtpt_idx, 0, hr_dev 1311 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_hw2sw_mpt(hr_dev, NULL, mtpt_idx); hr_dev 1328 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, pdn, hr_dev 1335 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_sw2hw_mpt(hr_dev, mailbox, mtpt_idx); hr_dev 1346 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 1351 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 1358 drivers/infiniband/hw/hns/hns_roce_mr.c struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device); hr_dev 1362 drivers/infiniband/hw/hns/hns_roce_mr.c if (hr_dev->hw->dereg_mr) { hr_dev 1363 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hr_dev->hw->dereg_mr(hr_dev, mr, udata); hr_dev 1365 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_mr_free(hr_dev, mr); hr_dev 1377 drivers/infiniband/hw/hns/hns_roce_mr.c struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); hr_dev 1378 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 1384 drivers/infiniband/hw/hns/hns_roce_mr.c page_size = 1 << (hr_dev->caps.pbl_buf_pg_sz + PAGE_SHIFT); hr_dev 1403 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, 0, length, hr_dev 1408 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_mr_enable(hr_dev, mr); hr_dev 1444 drivers/infiniband/hw/hns/hns_roce_mr.c static void hns_roce_mw_free(struct hns_roce_dev *hr_dev, hr_dev 1447 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 1451 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mw->rkey) hr_dev 1452 drivers/infiniband/hw/hns/hns_roce_mr.c & (hr_dev->caps.num_mtpts - 1)); hr_dev 1456 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table, hr_dev 1460 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, hr_dev 1464 drivers/infiniband/hw/hns/hns_roce_mr.c static int hns_roce_mw_enable(struct hns_roce_dev *hr_dev, hr_dev 1467 drivers/infiniband/hw/hns/hns_roce_mr.c struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; hr_dev 1469 drivers/infiniband/hw/hns/hns_roce_mr.c struct device *dev = hr_dev->dev; hr_dev 1474 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx); hr_dev 1478 drivers/infiniband/hw/hns/hns_roce_mr.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 1484 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hr_dev->hw->mw_write_mtpt(mailbox->buf, mw); hr_dev 1490 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_sw2hw_mpt(hr_dev, mailbox, hr_dev 1491 drivers/infiniband/hw/hns/hns_roce_mr.c mtpt_idx & (hr_dev->caps.num_mtpts - 1)); hr_dev 1499 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 1504 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 1507 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx); hr_dev 1515 drivers/infiniband/hw/hns/hns_roce_mr.c struct hns_roce_dev *hr_dev = to_hr_dev(ib_pd->device); hr_dev 1525 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index); hr_dev 1534 drivers/infiniband/hw/hns/hns_roce_mr.c mw->pbl_hop_num = hr_dev->caps.pbl_hop_num; hr_dev 1535 drivers/infiniband/hw/hns/hns_roce_mr.c mw->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz; hr_dev 1536 drivers/infiniband/hw/hns/hns_roce_mr.c mw->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz; hr_dev 1538 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_mw_enable(hr_dev, mw); hr_dev 1545 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_mw_free(hr_dev, mw); hr_dev 1555 drivers/infiniband/hw/hns/hns_roce_mr.c struct hns_roce_dev *hr_dev = to_hr_dev(ibmw->device); hr_dev 1558 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_mw_free(hr_dev, mw); hr_dev 1571 drivers/infiniband/hw/hns/hns_roce_mr.c void hns_roce_mtr_cleanup(struct hns_roce_dev *hr_dev, hr_dev 1574 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_hem_list_release(hr_dev, &mtr->hem_list); hr_dev 1577 drivers/infiniband/hw/hns/hns_roce_mr.c static int hns_roce_write_mtr(struct hns_roce_dev *hr_dev, hr_dev 1592 drivers/infiniband/hw/hns/hns_roce_mr.c mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list, hr_dev 1599 drivers/infiniband/hw/hns/hns_roce_mr.c if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) hr_dev 1612 drivers/infiniband/hw/hns/hns_roce_mr.c int hns_roce_mtr_attach(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, hr_dev 1620 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_hem_list_request(hr_dev, &mtr->hem_list, regions, hr_dev 1627 drivers/infiniband/hw/hns/hns_roce_mr.c ret = hns_roce_write_mtr(hr_dev, mtr, bufs[i], r); hr_dev 1629 drivers/infiniband/hw/hns/hns_roce_mr.c dev_err(hr_dev->dev, hr_dev 1639 drivers/infiniband/hw/hns/hns_roce_mr.c hns_roce_hem_list_release(hr_dev, &mtr->hem_list); hr_dev 1644 drivers/infiniband/hw/hns/hns_roce_mr.c int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, hr_dev 1660 drivers/infiniband/hw/hns/hns_roce_mr.c addr = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list, hr_dev 38 drivers/infiniband/hw/hns/hns_roce_pd.c static int hns_roce_pd_alloc(struct hns_roce_dev *hr_dev, unsigned long *pdn) hr_dev 40 drivers/infiniband/hw/hns/hns_roce_pd.c return hns_roce_bitmap_alloc(&hr_dev->pd_bitmap, pdn) ? -ENOMEM : 0; hr_dev 43 drivers/infiniband/hw/hns/hns_roce_pd.c static void hns_roce_pd_free(struct hns_roce_dev *hr_dev, unsigned long pdn) hr_dev 45 drivers/infiniband/hw/hns/hns_roce_pd.c hns_roce_bitmap_free(&hr_dev->pd_bitmap, pdn, BITMAP_NO_RR); hr_dev 48 drivers/infiniband/hw/hns/hns_roce_pd.c int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev) hr_dev 50 drivers/infiniband/hw/hns/hns_roce_pd.c return hns_roce_bitmap_init(&hr_dev->pd_bitmap, hr_dev->caps.num_pds, hr_dev 51 drivers/infiniband/hw/hns/hns_roce_pd.c hr_dev->caps.num_pds - 1, hr_dev 52 drivers/infiniband/hw/hns/hns_roce_pd.c hr_dev->caps.reserved_pds, 0); hr_dev 55 drivers/infiniband/hw/hns/hns_roce_pd.c void hns_roce_cleanup_pd_table(struct hns_roce_dev *hr_dev) hr_dev 57 drivers/infiniband/hw/hns/hns_roce_pd.c hns_roce_bitmap_cleanup(&hr_dev->pd_bitmap); hr_dev 63 drivers/infiniband/hw/hns/hns_roce_pd.c struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); hr_dev 64 drivers/infiniband/hw/hns/hns_roce_pd.c struct device *dev = hr_dev->dev; hr_dev 92 drivers/infiniband/hw/hns/hns_roce_pd.c int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar) hr_dev 98 drivers/infiniband/hw/hns/hns_roce_pd.c ret = hns_roce_bitmap_alloc(&hr_dev->uar_table.bitmap, &uar->logic_idx); hr_dev 102 drivers/infiniband/hw/hns/hns_roce_pd.c if (uar->logic_idx > 0 && hr_dev->caps.phy_num_uars > 1) hr_dev 104 drivers/infiniband/hw/hns/hns_roce_pd.c (hr_dev->caps.phy_num_uars - 1) + 1; hr_dev 108 drivers/infiniband/hw/hns/hns_roce_pd.c if (!dev_is_pci(hr_dev->dev)) { hr_dev 109 drivers/infiniband/hw/hns/hns_roce_pd.c res = platform_get_resource(hr_dev->pdev, IORESOURCE_MEM, 0); hr_dev 111 drivers/infiniband/hw/hns/hns_roce_pd.c dev_err(&hr_dev->pdev->dev, "memory resource not found!\n"); hr_dev 116 drivers/infiniband/hw/hns/hns_roce_pd.c uar->pfn = ((pci_resource_start(hr_dev->pci_dev, 2)) hr_dev 123 drivers/infiniband/hw/hns/hns_roce_pd.c void hns_roce_uar_free(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar) hr_dev 125 drivers/infiniband/hw/hns/hns_roce_pd.c hns_roce_bitmap_free(&hr_dev->uar_table.bitmap, uar->logic_idx, hr_dev 129 drivers/infiniband/hw/hns/hns_roce_pd.c int hns_roce_init_uar_table(struct hns_roce_dev *hr_dev) hr_dev 131 drivers/infiniband/hw/hns/hns_roce_pd.c return hns_roce_bitmap_init(&hr_dev->uar_table.bitmap, hr_dev 132 drivers/infiniband/hw/hns/hns_roce_pd.c hr_dev->caps.num_uars, hr_dev 133 drivers/infiniband/hw/hns/hns_roce_pd.c hr_dev->caps.num_uars - 1, hr_dev 134 drivers/infiniband/hw/hns/hns_roce_pd.c hr_dev->caps.reserved_uars, 0); hr_dev 137 drivers/infiniband/hw/hns/hns_roce_pd.c void hns_roce_cleanup_uar_table(struct hns_roce_dev *hr_dev) hr_dev 139 drivers/infiniband/hw/hns/hns_roce_pd.c hns_roce_bitmap_cleanup(&hr_dev->uar_table.bitmap); hr_dev 46 drivers/infiniband/hw/hns/hns_roce_qp.c void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) hr_dev 48 drivers/infiniband/hw/hns/hns_roce_qp.c struct device *dev = hr_dev->dev; hr_dev 51 drivers/infiniband/hw/hns/hns_roce_qp.c xa_lock(&hr_dev->qp_table_xa); hr_dev 52 drivers/infiniband/hw/hns/hns_roce_qp.c qp = __hns_roce_qp_lookup(hr_dev, qpn); hr_dev 55 drivers/infiniband/hw/hns/hns_roce_qp.c xa_unlock(&hr_dev->qp_table_xa); hr_dev 111 drivers/infiniband/hw/hns/hns_roce_qp.c static int hns_roce_reserve_range_qp(struct hns_roce_dev *hr_dev, int cnt, hr_dev 114 drivers/infiniband/hw/hns/hns_roce_qp.c struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; hr_dev 142 drivers/infiniband/hw/hns/hns_roce_qp.c static int hns_roce_gsi_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn, hr_dev 145 drivers/infiniband/hw/hns/hns_roce_qp.c struct xarray *xa = &hr_dev->qp_table_xa; hr_dev 155 drivers/infiniband/hw/hns/hns_roce_qp.c ret = xa_err(xa_store_irq(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1), hr_dev 158 drivers/infiniband/hw/hns/hns_roce_qp.c dev_err(hr_dev->dev, "QPC xa_store failed\n"); hr_dev 163 drivers/infiniband/hw/hns/hns_roce_qp.c static int hns_roce_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn, hr_dev 166 drivers/infiniband/hw/hns/hns_roce_qp.c struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; hr_dev 167 drivers/infiniband/hw/hns/hns_roce_qp.c struct device *dev = hr_dev->dev; hr_dev 176 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn); hr_dev 183 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn); hr_dev 189 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.trrl_entry_sz) { hr_dev 191 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table, hr_dev 199 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.sccc_entry_sz) { hr_dev 201 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table, hr_dev 209 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp); hr_dev 216 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.sccc_entry_sz) hr_dev 217 drivers/infiniband/hw/hns/hns_roce_qp.c hns_roce_table_put(hr_dev, &qp_table->sccc_table, hr_dev 221 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.trrl_entry_sz) hr_dev 222 drivers/infiniband/hw/hns/hns_roce_qp.c hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); hr_dev 225 drivers/infiniband/hw/hns/hns_roce_qp.c hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); hr_dev 228 drivers/infiniband/hw/hns/hns_roce_qp.c hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn); hr_dev 234 drivers/infiniband/hw/hns/hns_roce_qp.c void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) hr_dev 236 drivers/infiniband/hw/hns/hns_roce_qp.c struct xarray *xa = &hr_dev->qp_table_xa; hr_dev 240 drivers/infiniband/hw/hns/hns_roce_qp.c __xa_erase(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1)); hr_dev 244 drivers/infiniband/hw/hns/hns_roce_qp.c void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) hr_dev 246 drivers/infiniband/hw/hns/hns_roce_qp.c struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; hr_dev 253 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.trrl_entry_sz) hr_dev 254 drivers/infiniband/hw/hns/hns_roce_qp.c hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_dev 256 drivers/infiniband/hw/hns/hns_roce_qp.c hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); hr_dev 260 drivers/infiniband/hw/hns/hns_roce_qp.c void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn, hr_dev 263 drivers/infiniband/hw/hns/hns_roce_qp.c struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; hr_dev 265 drivers/infiniband/hw/hns/hns_roce_qp.c if (base_qpn < hr_dev->caps.reserved_qps) hr_dev 271 drivers/infiniband/hw/hns/hns_roce_qp.c static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev, hr_dev 275 drivers/infiniband/hw/hns/hns_roce_qp.c struct device *dev = hr_dev->dev; hr_dev 279 drivers/infiniband/hw/hns/hns_roce_qp.c if (cap->max_recv_wr > hr_dev->caps.max_wqes || hr_dev 280 drivers/infiniband/hw/hns/hns_roce_qp.c cap->max_recv_sge > hr_dev->caps.max_rq_sg) { hr_dev 298 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.min_wqes) hr_dev 299 drivers/infiniband/hw/hns/hns_roce_qp.c max_cnt = max(cap->max_recv_wr, hr_dev->caps.min_wqes); hr_dev 305 drivers/infiniband/hw/hns/hns_roce_qp.c if ((u32)hr_qp->rq.wqe_cnt > hr_dev->caps.max_wqes) { hr_dev 312 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.max_rq_sg <= 2) hr_dev 314 drivers/infiniband/hw/hns/hns_roce_qp.c ilog2(hr_dev->caps.max_rq_desc_sz); hr_dev 317 drivers/infiniband/hw/hns/hns_roce_qp.c ilog2(hr_dev->caps.max_rq_desc_sz hr_dev 327 drivers/infiniband/hw/hns/hns_roce_qp.c static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev, hr_dev 331 drivers/infiniband/hw/hns/hns_roce_qp.c u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz); hr_dev 337 drivers/infiniband/hw/hns/hns_roce_qp.c ibdev_err(&hr_dev->ib_dev, "check SQ size error!\n"); hr_dev 341 drivers/infiniband/hw/hns/hns_roce_qp.c if (cap->max_send_sge > hr_dev->caps.max_sq_sg) { hr_dev 342 drivers/infiniband/hw/hns/hns_roce_qp.c ibdev_err(&hr_dev->ib_dev, "SQ sge error! max_send_sge=%d\n", hr_dev 350 drivers/infiniband/hw/hns/hns_roce_qp.c static int hns_roce_set_user_sq_size(struct hns_roce_dev *hr_dev, hr_dev 361 drivers/infiniband/hw/hns/hns_roce_qp.c hr_qp->sq.wqe_cnt > hr_dev->caps.max_wqes) hr_dev 364 drivers/infiniband/hw/hns/hns_roce_qp.c ret = check_sq_size_with_integrity(hr_dev, cap, ucmd); hr_dev 366 drivers/infiniband/hw/hns/hns_roce_qp.c ibdev_err(&hr_dev->ib_dev, "Sanity check sq size failed\n"); hr_dev 373 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.max_sq_sg <= 2) hr_dev 382 drivers/infiniband/hw/hns/hns_roce_qp.c if ((hr_qp->sq.max_gs > 2) && (hr_dev->pci_dev->revision == 0x20)) { hr_dev 383 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_qp->sge.sge_cnt > hr_dev->caps.max_extend_sg) { hr_dev 384 drivers/infiniband/hw/hns/hns_roce_qp.c dev_err(hr_dev->dev, hr_dev 395 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.max_sq_sg <= 2) { hr_dev 405 drivers/infiniband/hw/hns/hns_roce_qp.c page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); hr_dev 436 drivers/infiniband/hw/hns/hns_roce_qp.c static int split_wqe_buf_region(struct hns_roce_dev *hr_dev, hr_dev 464 drivers/infiniband/hw/hns/hns_roce_qp.c hr_dev->caps.wqe_sq_hop_num, hr_dev 476 drivers/infiniband/hw/hns/hns_roce_qp.c hr_dev->caps.wqe_sge_hop_num, hr_dev 488 drivers/infiniband/hw/hns/hns_roce_qp.c hr_dev->caps.wqe_rq_hop_num, hr_dev 497 drivers/infiniband/hw/hns/hns_roce_qp.c static int calc_wqe_bt_page_shift(struct hns_roce_dev *hr_dev, hr_dev 505 drivers/infiniband/hw/hns/hns_roce_qp.c bt_pg_shift = PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz; hr_dev 521 drivers/infiniband/hw/hns/hns_roce_qp.c static int set_extend_sge_param(struct hns_roce_dev *hr_dev, hr_dev 524 drivers/infiniband/hw/hns/hns_roce_qp.c struct device *dev = hr_dev->dev; hr_dev 533 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.max_sq_sg > 2 && hr_qp->ibqp.qp_type == IB_QPT_GSI) { hr_dev 539 drivers/infiniband/hw/hns/hns_roce_qp.c if ((hr_qp->sq.max_gs > 2) && hr_dev->pci_dev->revision == 0x20) { hr_dev 540 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_qp->sge.sge_cnt > hr_dev->caps.max_extend_sg) { hr_dev 550 drivers/infiniband/hw/hns/hns_roce_qp.c static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev, hr_dev 554 drivers/infiniband/hw/hns/hns_roce_qp.c struct device *dev = hr_dev->dev; hr_dev 560 drivers/infiniband/hw/hns/hns_roce_qp.c if (cap->max_send_wr > hr_dev->caps.max_wqes || hr_dev 561 drivers/infiniband/hw/hns/hns_roce_qp.c cap->max_send_sge > hr_dev->caps.max_sq_sg || hr_dev 562 drivers/infiniband/hw/hns/hns_roce_qp.c cap->max_inline_data > hr_dev->caps.max_sq_inline) { hr_dev 567 drivers/infiniband/hw/hns/hns_roce_qp.c hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz); hr_dev 569 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.min_wqes) hr_dev 570 drivers/infiniband/hw/hns/hns_roce_qp.c max_cnt = max(cap->max_send_wr, hr_dev->caps.min_wqes); hr_dev 575 drivers/infiniband/hw/hns/hns_roce_qp.c if ((u32)hr_qp->sq.wqe_cnt > hr_dev->caps.max_wqes) { hr_dev 582 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.max_sq_sg <= 2) hr_dev 587 drivers/infiniband/hw/hns/hns_roce_qp.c ret = set_extend_sge_param(hr_dev, hr_qp); hr_dev 594 drivers/infiniband/hw/hns/hns_roce_qp.c page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); hr_dev 599 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.max_sq_sg > 2 && hr_qp->sge.sge_cnt) { hr_dev 684 drivers/infiniband/hw/hns/hns_roce_qp.c static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, hr_dev 691 drivers/infiniband/hw/hns/hns_roce_qp.c struct device *dev = hr_dev->dev; hr_dev 716 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_set_rq_size(hr_dev, &init_attr->cap, udata, hr_dev 723 drivers/infiniband/hw/hns/hns_roce_qp.c if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && hr_dev 732 drivers/infiniband/hw/hns/hns_roce_qp.c page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; hr_dev 740 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, hr_dev 754 drivers/infiniband/hw/hns/hns_roce_qp.c hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp, hr_dev 766 drivers/infiniband/hw/hns/hns_roce_qp.c buf_count = hns_roce_get_umem_bufs(hr_dev, hr_dev 778 drivers/infiniband/hw/hns/hns_roce_qp.c if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) && hr_dev 794 drivers/infiniband/hw/hns/hns_roce_qp.c if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && hr_dev 823 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_set_kernel_sq_size(hr_dev, &init_attr->cap, hr_dev 831 drivers/infiniband/hw/hns/hns_roce_qp.c hr_qp->sq.db_reg_l = hr_dev->reg_base + hr_dev->sdb_offset + hr_dev 832 drivers/infiniband/hw/hns/hns_roce_qp.c DB_REG_OFFSET * hr_dev->priv_uar.index; hr_dev 833 drivers/infiniband/hw/hns/hns_roce_qp.c hr_qp->rq.db_reg_l = hr_dev->reg_base + hr_dev->odb_offset + hr_dev 834 drivers/infiniband/hw/hns/hns_roce_qp.c DB_REG_OFFSET * hr_dev->priv_uar.index; hr_dev 836 drivers/infiniband/hw/hns/hns_roce_qp.c if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && hr_dev 838 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0); hr_dev 848 drivers/infiniband/hw/hns/hns_roce_qp.c if (hns_roce_buf_alloc(hr_dev, hr_qp->buff_size, hr_dev 855 drivers/infiniband/hw/hns/hns_roce_qp.c hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp, hr_dev 867 drivers/infiniband/hw/hns/hns_roce_qp.c buf_count = hns_roce_get_kmem_bufs(hr_dev, hr_dev 900 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_reserve_range_qp(hr_dev, 1, 1, &qpn); hr_dev 907 drivers/infiniband/hw/hns/hns_roce_qp.c hr_qp->wqe_bt_pg_shift = calc_wqe_bt_page_shift(hr_dev, hr_qp->regions, hr_dev 911 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_mtr_attach(hr_dev, &hr_qp->mtr, buf_list, hr_dev 919 drivers/infiniband/hw/hns/hns_roce_qp.c hr_dev->hw_rev == HNS_ROCE_HW_VER1) { hr_dev 921 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp); hr_dev 927 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_qp_alloc(hr_dev, qpn, hr_qp); hr_dev 946 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { hr_dev 947 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp); hr_dev 959 drivers/infiniband/hw/hns/hns_roce_qp.c hr_dev->hw_rev == HNS_ROCE_HW_VER1) hr_dev 960 drivers/infiniband/hw/hns/hns_roce_qp.c hns_roce_qp_remove(hr_dev, hr_qp); hr_dev 962 drivers/infiniband/hw/hns/hns_roce_qp.c hns_roce_qp_free(hr_dev, hr_qp); hr_dev 966 drivers/infiniband/hw/hns/hns_roce_qp.c hns_roce_release_range_qp(hr_dev, qpn, 1); hr_dev 969 drivers/infiniband/hw/hns/hns_roce_qp.c hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr); hr_dev 973 drivers/infiniband/hw/hns/hns_roce_qp.c if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && hr_dev 984 drivers/infiniband/hw/hns/hns_roce_qp.c if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) && hr_dev 999 drivers/infiniband/hw/hns/hns_roce_qp.c hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); hr_dev 1004 drivers/infiniband/hw/hns/hns_roce_qp.c (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB)) hr_dev 1005 drivers/infiniband/hw/hns/hns_roce_qp.c hns_roce_free_db(hr_dev, &hr_qp->rdb); hr_dev 1008 drivers/infiniband/hw/hns/hns_roce_qp.c if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && hr_dev 1020 drivers/infiniband/hw/hns/hns_roce_qp.c struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); hr_dev 1021 drivers/infiniband/hw/hns/hns_roce_qp.c struct ib_device *ibdev = &hr_dev->ib_dev; hr_dev 1032 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, 0, hr_dev 1058 drivers/infiniband/hw/hns/hns_roce_qp.c hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; hr_dev 1061 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.max_sq_sg <= 2) hr_dev 1063 drivers/infiniband/hw/hns/hns_roce_qp.c hr_dev->iboe.phy_port[hr_qp->port]; hr_dev 1067 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_dev 1105 drivers/infiniband/hw/hns/hns_roce_qp.c static int check_mtu_validate(struct hns_roce_dev *hr_dev, hr_dev 1113 drivers/infiniband/hw/hns/hns_roce_qp.c active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu); hr_dev 1115 drivers/infiniband/hw/hns/hns_roce_qp.c if ((hr_dev->caps.max_mtu >= IB_MTU_2048 && hr_dev 1116 drivers/infiniband/hw/hns/hns_roce_qp.c attr->path_mtu > hr_dev->caps.max_mtu) || hr_dev 1118 drivers/infiniband/hw/hns/hns_roce_qp.c ibdev_err(&hr_dev->ib_dev, hr_dev 1130 drivers/infiniband/hw/hns/hns_roce_qp.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 1135 drivers/infiniband/hw/hns/hns_roce_qp.c (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) { hr_dev 1136 drivers/infiniband/hw/hns/hns_roce_qp.c ibdev_err(&hr_dev->ib_dev, hr_dev 1144 drivers/infiniband/hw/hns/hns_roce_qp.c if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) { hr_dev 1145 drivers/infiniband/hw/hns/hns_roce_qp.c ibdev_err(&hr_dev->ib_dev, hr_dev 1153 drivers/infiniband/hw/hns/hns_roce_qp.c attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) { hr_dev 1154 drivers/infiniband/hw/hns/hns_roce_qp.c ibdev_err(&hr_dev->ib_dev, hr_dev 1161 drivers/infiniband/hw/hns/hns_roce_qp.c attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) { hr_dev 1162 drivers/infiniband/hw/hns/hns_roce_qp.c ibdev_err(&hr_dev->ib_dev, hr_dev 1169 drivers/infiniband/hw/hns/hns_roce_qp.c return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask); hr_dev 1177 drivers/infiniband/hw/hns/hns_roce_qp.c struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); hr_dev 1196 drivers/infiniband/hw/hns/hns_roce_qp.c ibdev_warn(&hr_dev->ib_dev, hr_dev 1204 drivers/infiniband/hw/hns/hns_roce_qp.c ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n"); hr_dev 1213 drivers/infiniband/hw/hns/hns_roce_qp.c if (hr_dev->caps.min_wqes) { hr_dev 1215 drivers/infiniband/hw/hns/hns_roce_qp.c ibdev_err(&hr_dev->ib_dev, hr_dev 1225 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state, hr_dev 1305 drivers/infiniband/hw/hns/hns_roce_qp.c int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) hr_dev 1307 drivers/infiniband/hw/hns/hns_roce_qp.c struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; hr_dev 1313 drivers/infiniband/hw/hns/hns_roce_qp.c xa_init(&hr_dev->qp_table_xa); hr_dev 1315 drivers/infiniband/hw/hns/hns_roce_qp.c reserved_from_bot = hr_dev->caps.reserved_qps; hr_dev 1317 drivers/infiniband/hw/hns/hns_roce_qp.c ret = hns_roce_bitmap_init(&qp_table->bitmap, hr_dev->caps.num_qps, hr_dev 1318 drivers/infiniband/hw/hns/hns_roce_qp.c hr_dev->caps.num_qps - 1, reserved_from_bot, hr_dev 1321 drivers/infiniband/hw/hns/hns_roce_qp.c dev_err(hr_dev->dev, "qp bitmap init failed!error=%d\n", hr_dev 1329 drivers/infiniband/hw/hns/hns_roce_qp.c void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev) hr_dev 1331 drivers/infiniband/hw/hns/hns_roce_qp.c hns_roce_bitmap_cleanup(&hr_dev->qp_table.bitmap); hr_dev 83 drivers/infiniband/hw/hns/hns_roce_restrack.c struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); hr_dev 89 drivers/infiniband/hw/hns/hns_roce_restrack.c if (!hr_dev->dfx->query_cqc_info) hr_dev 96 drivers/infiniband/hw/hns/hns_roce_restrack.c ret = hr_dev->dfx->query_cqc_info(hr_dev, hr_cq->cqn, (int *)context); hr_dev 12 drivers/infiniband/hw/hns/hns_roce_srq.c void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type) hr_dev 14 drivers/infiniband/hw/hns/hns_roce_srq.c struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; hr_dev 18 drivers/infiniband/hw/hns/hns_roce_srq.c srq = xa_load(&srq_table->xa, srqn & (hr_dev->caps.num_srqs - 1)); hr_dev 24 drivers/infiniband/hw/hns/hns_roce_srq.c dev_warn(hr_dev->dev, "Async event for bogus SRQ %08x\n", srqn); hr_dev 37 drivers/infiniband/hw/hns/hns_roce_srq.c struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device); hr_dev 52 drivers/infiniband/hw/hns/hns_roce_srq.c dev_err(hr_dev->dev, hr_dev 80 drivers/infiniband/hw/hns/hns_roce_srq.c static int hns_roce_srq_alloc(struct hns_roce_dev *hr_dev, u32 pdn, u32 cqn, hr_dev 84 drivers/infiniband/hw/hns/hns_roce_srq.c struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; hr_dev 93 drivers/infiniband/hw/hns/hns_roce_srq.c mtts_wqe = hns_roce_table_find(hr_dev, hr_dev 94 drivers/infiniband/hw/hns/hns_roce_srq.c &hr_dev->mr_table.mtt_srqwqe_table, hr_dev 98 drivers/infiniband/hw/hns/hns_roce_srq.c dev_err(hr_dev->dev, hr_dev 104 drivers/infiniband/hw/hns/hns_roce_srq.c mtts_idx = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_idx_table, hr_dev 108 drivers/infiniband/hw/hns/hns_roce_srq.c dev_err(hr_dev->dev, hr_dev 115 drivers/infiniband/hw/hns/hns_roce_srq.c dev_err(hr_dev->dev, "SRQ alloc.Failed to alloc index.\n"); hr_dev 119 drivers/infiniband/hw/hns/hns_roce_srq.c ret = hns_roce_table_get(hr_dev, &srq_table->table, srq->srqn); hr_dev 127 drivers/infiniband/hw/hns/hns_roce_srq.c mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); hr_dev 133 drivers/infiniband/hw/hns/hns_roce_srq.c hr_dev->hw->write_srqc(hr_dev, srq, pdn, xrcd, cqn, mailbox->buf, hr_dev 137 drivers/infiniband/hw/hns/hns_roce_srq.c ret = hns_roce_sw2hw_srq(hr_dev, mailbox, srq->srqn); hr_dev 138 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_free_cmd_mailbox(hr_dev, mailbox); hr_dev 150 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn); hr_dev 157 drivers/infiniband/hw/hns/hns_roce_srq.c static void hns_roce_srq_free(struct hns_roce_dev *hr_dev, hr_dev 160 drivers/infiniband/hw/hns/hns_roce_srq.c struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; hr_dev 163 drivers/infiniband/hw/hns/hns_roce_srq.c ret = hns_roce_hw2sw_srq(hr_dev, NULL, srq->srqn); hr_dev 165 drivers/infiniband/hw/hns/hns_roce_srq.c dev_err(hr_dev->dev, "HW2SW_SRQ failed (%d) for CQN %06lx\n", hr_dev 174 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn); hr_dev 181 drivers/infiniband/hw/hns/hns_roce_srq.c struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device); hr_dev 195 drivers/infiniband/hw/hns/hns_roce_srq.c (1 << hr_dev->caps.srqwqe_buf_pg_sz) - 1) / hr_dev 196 drivers/infiniband/hw/hns/hns_roce_srq.c (1 << hr_dev->caps.srqwqe_buf_pg_sz); hr_dev 197 drivers/infiniband/hw/hns/hns_roce_srq.c buf->page_shift = PAGE_SHIFT + hr_dev->caps.srqwqe_buf_pg_sz; hr_dev 198 drivers/infiniband/hw/hns/hns_roce_srq.c ret = hns_roce_mtt_init(hr_dev, buf->npages, buf->page_shift, hr_dev 203 drivers/infiniband/hw/hns/hns_roce_srq.c ret = hns_roce_ib_umem_write_mtt(hr_dev, &srq->mtt, srq->umem); hr_dev 211 drivers/infiniband/hw/hns/hns_roce_srq.c dev_err(hr_dev->dev, "ib_umem_get error for index queue\n"); hr_dev 218 drivers/infiniband/hw/hns/hns_roce_srq.c 1 << hr_dev->caps.idx_buf_pg_sz); hr_dev 219 drivers/infiniband/hw/hns/hns_roce_srq.c buf->page_shift = PAGE_SHIFT + hr_dev->caps.idx_buf_pg_sz; hr_dev 220 drivers/infiniband/hw/hns/hns_roce_srq.c ret = hns_roce_mtt_init(hr_dev, buf->npages, buf->page_shift, hr_dev 223 drivers/infiniband/hw/hns/hns_roce_srq.c dev_err(hr_dev->dev, "hns_roce_mtt_init error for idx que\n"); hr_dev 227 drivers/infiniband/hw/hns/hns_roce_srq.c ret = hns_roce_ib_umem_write_mtt(hr_dev, &srq->idx_que.mtt, hr_dev 230 drivers/infiniband/hw/hns/hns_roce_srq.c dev_err(hr_dev->dev, hr_dev 238 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt); hr_dev 244 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_mtt_cleanup(hr_dev, &srq->mtt); hr_dev 255 drivers/infiniband/hw/hns/hns_roce_srq.c struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); hr_dev 264 drivers/infiniband/hw/hns/hns_roce_srq.c if (hns_roce_buf_alloc(hr_dev, idx_que->buf_size, (1 << page_shift) * 2, hr_dev 275 drivers/infiniband/hw/hns/hns_roce_srq.c struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device); hr_dev 276 drivers/infiniband/hw/hns/hns_roce_srq.c u32 page_shift = PAGE_SHIFT + hr_dev->caps.srqwqe_buf_pg_sz; hr_dev 279 drivers/infiniband/hw/hns/hns_roce_srq.c if (hns_roce_buf_alloc(hr_dev, srq_buf_size, (1 << page_shift) * 2, hr_dev 286 drivers/infiniband/hw/hns/hns_roce_srq.c ret = hns_roce_mtt_init(hr_dev, srq->buf.npages, srq->buf.page_shift, hr_dev 291 drivers/infiniband/hw/hns/hns_roce_srq.c ret = hns_roce_buf_write_mtt(hr_dev, &srq->mtt, &srq->buf); hr_dev 295 drivers/infiniband/hw/hns/hns_roce_srq.c page_shift = PAGE_SHIFT + hr_dev->caps.idx_buf_pg_sz; hr_dev 298 drivers/infiniband/hw/hns/hns_roce_srq.c dev_err(hr_dev->dev, "Create idx queue fail(%d)!\n", ret); hr_dev 303 drivers/infiniband/hw/hns/hns_roce_srq.c ret = hns_roce_mtt_init(hr_dev, srq->idx_que.idx_buf.npages, hr_dev 310 drivers/infiniband/hw/hns/hns_roce_srq.c ret = hns_roce_buf_write_mtt(hr_dev, &srq->idx_que.mtt, hr_dev 324 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt); hr_dev 327 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_buf_free(hr_dev, srq->idx_que.buf_size, hr_dev 332 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_mtt_cleanup(hr_dev, &srq->mtt); hr_dev 335 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_buf_free(hr_dev, srq_buf_size, &srq->buf); hr_dev 340 drivers/infiniband/hw/hns/hns_roce_srq.c static void destroy_user_srq(struct hns_roce_dev *hr_dev, hr_dev 343 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt); hr_dev 345 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_mtt_cleanup(hr_dev, &srq->mtt); hr_dev 349 drivers/infiniband/hw/hns/hns_roce_srq.c static void destroy_kernel_srq(struct hns_roce_dev *hr_dev, hr_dev 353 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt); hr_dev 354 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_buf_free(hr_dev, srq->idx_que.buf_size, &srq->idx_que.idx_buf); hr_dev 356 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_mtt_cleanup(hr_dev, &srq->mtt); hr_dev 357 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_buf_free(hr_dev, srq_buf_size, &srq->buf); hr_dev 364 drivers/infiniband/hw/hns/hns_roce_srq.c struct hns_roce_dev *hr_dev = to_hr_dev(ib_srq->device); hr_dev 373 drivers/infiniband/hw/hns/hns_roce_srq.c if (srq_init_attr->attr.max_wr >= hr_dev->caps.max_srq_wrs || hr_dev 374 drivers/infiniband/hw/hns/hns_roce_srq.c srq_init_attr->attr.max_sge > hr_dev->caps.max_srq_sges) hr_dev 397 drivers/infiniband/hw/hns/hns_roce_srq.c dev_err(hr_dev->dev, "Create user srq failed\n"); hr_dev 403 drivers/infiniband/hw/hns/hns_roce_srq.c dev_err(hr_dev->dev, "Create kernel srq failed\n"); hr_dev 411 drivers/infiniband/hw/hns/hns_roce_srq.c srq->db_reg_l = hr_dev->reg_base + SRQ_DB_REG; hr_dev 413 drivers/infiniband/hw/hns/hns_roce_srq.c ret = hns_roce_srq_alloc(hr_dev, to_hr_pd(ib_srq->pd)->pdn, cqn, 0, hr_dev 432 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_srq_free(hr_dev, srq); hr_dev 436 drivers/infiniband/hw/hns/hns_roce_srq.c destroy_user_srq(hr_dev, srq); hr_dev 438 drivers/infiniband/hw/hns/hns_roce_srq.c destroy_kernel_srq(hr_dev, srq, srq_buf_size); hr_dev 446 drivers/infiniband/hw/hns/hns_roce_srq.c struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); hr_dev 449 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_srq_free(hr_dev, srq); hr_dev 450 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_mtt_cleanup(hr_dev, &srq->mtt); hr_dev 453 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt); hr_dev 456 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_buf_free(hr_dev, srq->max << srq->wqe_shift, hr_dev 463 drivers/infiniband/hw/hns/hns_roce_srq.c int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev) hr_dev 465 drivers/infiniband/hw/hns/hns_roce_srq.c struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; hr_dev 469 drivers/infiniband/hw/hns/hns_roce_srq.c return hns_roce_bitmap_init(&srq_table->bitmap, hr_dev->caps.num_srqs, hr_dev 470 drivers/infiniband/hw/hns/hns_roce_srq.c hr_dev->caps.num_srqs - 1, hr_dev 471 drivers/infiniband/hw/hns/hns_roce_srq.c hr_dev->caps.reserved_srqs, 0); hr_dev 474 drivers/infiniband/hw/hns/hns_roce_srq.c void hns_roce_cleanup_srq_table(struct hns_roce_dev *hr_dev) hr_dev 476 drivers/infiniband/hw/hns/hns_roce_srq.c hns_roce_bitmap_cleanup(&hr_dev->srq_table.bitmap);