This source file includes following definitions.
- qat_hal_set_live_ctx
- qat_hal_rd_ae_csr
- qat_hal_wr_ae_csr
- qat_hal_get_wakeup_event
- qat_hal_wait_cycles
- qat_hal_set_ae_ctx_mode
- qat_hal_set_ae_nn_mode
- qat_hal_set_ae_lm_mode
- qat_hal_get_reg_addr
- qat_hal_reset
- qat_hal_wr_indr_csr
- qat_hal_rd_indr_csr
- qat_hal_put_sig_event
- qat_hal_put_wakeup_event
- qat_hal_check_ae_alive
- qat_hal_check_ae_active
- qat_hal_reset_timestamp
- qat_hal_init_esram
- qat_hal_clr_reset
- qat_hal_disable_ctx
- qat_hal_parity_64bit
- qat_hal_set_uword_ecc
- qat_hal_wr_uwords
- qat_hal_enable_ctx
- qat_hal_clear_xfer
- qat_hal_clear_gpr
- qat_hal_init
- qat_hal_deinit
- qat_hal_start
- qat_hal_stop
- qat_hal_set_pc
- qat_hal_get_uwords
- qat_hal_wr_umem
- qat_hal_exec_micro_inst
- qat_hal_rd_rel_reg
- qat_hal_wr_rel_reg
- qat_hal_get_ins_num
- qat_hal_concat_micro_code
- qat_hal_exec_micro_init_lm
- qat_hal_batch_wr_lm
- qat_hal_put_rel_rd_xfer
- qat_hal_put_rel_wr_xfer
- qat_hal_put_rel_nn
- qat_hal_convert_abs_to_rel
- qat_hal_init_gpr
- qat_hal_init_wr_xfer
- qat_hal_init_rd_xfer
- qat_hal_init_nn
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47 #include <linux/slab.h>
48 #include <linux/delay.h>
49
50 #include "adf_accel_devices.h"
51 #include "adf_common_drv.h"
52 #include "icp_qat_hal.h"
53 #include "icp_qat_uclo.h"
54
55 #define BAD_REGADDR 0xffff
56 #define MAX_RETRY_TIMES 10000
57 #define INIT_CTX_ARB_VALUE 0x0
58 #define INIT_CTX_ENABLE_VALUE 0x0
59 #define INIT_PC_VALUE 0x0
60 #define INIT_WAKEUP_EVENTS_VALUE 0x1
61 #define INIT_SIG_EVENTS_VALUE 0x1
62 #define INIT_CCENABLE_VALUE 0x2000
63 #define RST_CSR_QAT_LSB 20
64 #define RST_CSR_AE_LSB 0
65 #define MC_TIMESTAMP_ENABLE (0x1 << 7)
66
67 #define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \
68 (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \
69 (~(1 << CE_REG_PAR_ERR_BITPOS)))
70 #define INSERT_IMMED_GPRA_CONST(inst, const_val) \
71 (inst = ((inst & 0xFFFF00C03FFull) | \
72 ((((const_val) << 12) & 0x0FF00000ull) | \
73 (((const_val) << 10) & 0x0003FC00ull))))
74 #define INSERT_IMMED_GPRB_CONST(inst, const_val) \
75 (inst = ((inst & 0xFFFF00FFF00ull) | \
76 ((((const_val) << 12) & 0x0FF00000ull) | \
77 (((const_val) << 0) & 0x000000FFull))))
78
79 #define AE(handle, ae) handle->hal_handle->aes[ae]
80
81 static const uint64_t inst_4b[] = {
82 0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull,
83 0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
84 0x0A021000000ull
85 };
86
87 static const uint64_t inst[] = {
88 0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull,
89 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
90 0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull,
91 0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
92 0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull,
93 0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull,
94 0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull,
95 0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull,
96 0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull,
97 0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull,
98 0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull,
99 0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull,
100 0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull,
101 0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull,
102 0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull,
103 0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull,
104 0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull,
105 0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull,
106 0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull,
107 0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull,
108 0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull,
109 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,
110 };
111
112 void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
113 unsigned char ae, unsigned int ctx_mask)
114 {
115 AE(handle, ae).live_ctx_mask = ctx_mask;
116 }
117
118 #define CSR_RETRY_TIMES 500
119 static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,
120 unsigned char ae, unsigned int csr)
121 {
122 unsigned int iterations = CSR_RETRY_TIMES;
123 int value;
124
125 do {
126 value = GET_AE_CSR(handle, ae, csr);
127 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
128 return value;
129 } while (iterations--);
130
131 pr_err("QAT: Read CSR timeout\n");
132 return 0;
133 }
134
135 static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,
136 unsigned char ae, unsigned int csr,
137 unsigned int value)
138 {
139 unsigned int iterations = CSR_RETRY_TIMES;
140
141 do {
142 SET_AE_CSR(handle, ae, csr, value);
143 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
144 return 0;
145 } while (iterations--);
146
147 pr_err("QAT: Write CSR Timeout\n");
148 return -EFAULT;
149 }
150
151 static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,
152 unsigned char ae, unsigned char ctx,
153 unsigned int *events)
154 {
155 unsigned int cur_ctx;
156
157 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
158 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
159 *events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT);
160 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
161 }
162
163 static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,
164 unsigned char ae, unsigned int cycles,
165 int chk_inactive)
166 {
167 unsigned int base_cnt = 0, cur_cnt = 0;
168 unsigned int csr = (1 << ACS_ABO_BITPOS);
169 int times = MAX_RETRY_TIMES;
170 int elapsed_cycles = 0;
171
172 base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
173 base_cnt &= 0xffff;
174 while ((int)cycles > elapsed_cycles && times--) {
175 if (chk_inactive)
176 csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
177
178 cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
179 cur_cnt &= 0xffff;
180 elapsed_cycles = cur_cnt - base_cnt;
181
182 if (elapsed_cycles < 0)
183 elapsed_cycles += 0x10000;
184
185
186 if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS)))
187 return 0;
188 }
189 if (times < 0) {
190 pr_err("QAT: wait_num_cycles time out\n");
191 return -EFAULT;
192 }
193 return 0;
194 }
195
196 #define CLR_BIT(wrd, bit) (wrd & ~(1 << bit))
197 #define SET_BIT(wrd, bit) (wrd | 1 << bit)
198
199 int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
200 unsigned char ae, unsigned char mode)
201 {
202 unsigned int csr, new_csr;
203
204 if ((mode != 4) && (mode != 8)) {
205 pr_err("QAT: bad ctx mode=%d\n", mode);
206 return -EINVAL;
207 }
208
209
210 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
211 csr = IGNORE_W1C_MASK & csr;
212 new_csr = (mode == 4) ?
213 SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :
214 CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS);
215 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
216 return 0;
217 }
218
219 int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
220 unsigned char ae, unsigned char mode)
221 {
222 unsigned int csr, new_csr;
223
224 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
225 csr &= IGNORE_W1C_MASK;
226
227 new_csr = (mode) ?
228 SET_BIT(csr, CE_NN_MODE_BITPOS) :
229 CLR_BIT(csr, CE_NN_MODE_BITPOS);
230
231 if (new_csr != csr)
232 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
233
234 return 0;
235 }
236
237 int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
238 unsigned char ae, enum icp_qat_uof_regtype lm_type,
239 unsigned char mode)
240 {
241 unsigned int csr, new_csr;
242
243 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
244 csr &= IGNORE_W1C_MASK;
245 switch (lm_type) {
246 case ICP_LMEM0:
247 new_csr = (mode) ?
248 SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) :
249 CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS);
250 break;
251 case ICP_LMEM1:
252 new_csr = (mode) ?
253 SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) :
254 CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS);
255 break;
256 default:
257 pr_err("QAT: lmType = 0x%x\n", lm_type);
258 return -EINVAL;
259 }
260
261 if (new_csr != csr)
262 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
263 return 0;
264 }
265
266 static unsigned short qat_hal_get_reg_addr(unsigned int type,
267 unsigned short reg_num)
268 {
269 unsigned short reg_addr;
270
271 switch (type) {
272 case ICP_GPA_ABS:
273 case ICP_GPB_ABS:
274 reg_addr = 0x80 | (reg_num & 0x7f);
275 break;
276 case ICP_GPA_REL:
277 case ICP_GPB_REL:
278 reg_addr = reg_num & 0x1f;
279 break;
280 case ICP_SR_RD_REL:
281 case ICP_SR_WR_REL:
282 case ICP_SR_REL:
283 reg_addr = 0x180 | (reg_num & 0x1f);
284 break;
285 case ICP_SR_ABS:
286 reg_addr = 0x140 | ((reg_num & 0x3) << 1);
287 break;
288 case ICP_DR_RD_REL:
289 case ICP_DR_WR_REL:
290 case ICP_DR_REL:
291 reg_addr = 0x1c0 | (reg_num & 0x1f);
292 break;
293 case ICP_DR_ABS:
294 reg_addr = 0x100 | ((reg_num & 0x3) << 1);
295 break;
296 case ICP_NEIGH_REL:
297 reg_addr = 0x280 | (reg_num & 0x1f);
298 break;
299 case ICP_LMEM0:
300 reg_addr = 0x200;
301 break;
302 case ICP_LMEM1:
303 reg_addr = 0x220;
304 break;
305 case ICP_NO_DEST:
306 reg_addr = 0x300 | (reg_num & 0xff);
307 break;
308 default:
309 reg_addr = BAD_REGADDR;
310 break;
311 }
312 return reg_addr;
313 }
314
315 void qat_hal_reset(struct icp_qat_fw_loader_handle *handle)
316 {
317 unsigned int ae_reset_csr;
318
319 ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
320 ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB;
321 ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB;
322 SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
323 }
324
325 static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
326 unsigned char ae, unsigned int ctx_mask,
327 unsigned int ae_csr, unsigned int csr_val)
328 {
329 unsigned int ctx, cur_ctx;
330
331 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
332
333 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
334 if (!(ctx_mask & (1 << ctx)))
335 continue;
336 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
337 qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val);
338 }
339
340 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
341 }
342
343 static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
344 unsigned char ae, unsigned char ctx,
345 unsigned int ae_csr)
346 {
347 unsigned int cur_ctx, csr_val;
348
349 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
350 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
351 csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr);
352 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
353
354 return csr_val;
355 }
356
357 static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
358 unsigned char ae, unsigned int ctx_mask,
359 unsigned int events)
360 {
361 unsigned int ctx, cur_ctx;
362
363 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
364 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
365 if (!(ctx_mask & (1 << ctx)))
366 continue;
367 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
368 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events);
369 }
370 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
371 }
372
373 static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,
374 unsigned char ae, unsigned int ctx_mask,
375 unsigned int events)
376 {
377 unsigned int ctx, cur_ctx;
378
379 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
380 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
381 if (!(ctx_mask & (1 << ctx)))
382 continue;
383 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
384 qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT,
385 events);
386 }
387 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
388 }
389
390 static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
391 {
392 unsigned int base_cnt, cur_cnt;
393 unsigned char ae;
394 int times = MAX_RETRY_TIMES;
395
396 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
397 base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
398 base_cnt &= 0xffff;
399
400 do {
401 cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
402 cur_cnt &= 0xffff;
403 } while (times-- && (cur_cnt == base_cnt));
404
405 if (times < 0) {
406 pr_err("QAT: AE%d is inactive!!\n", ae);
407 return -EFAULT;
408 }
409 }
410
411 return 0;
412 }
413
414 int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle,
415 unsigned int ae)
416 {
417 unsigned int enable = 0, active = 0;
418
419 enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
420 active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
421 if ((enable & (0xff << CE_ENABLE_BITPOS)) ||
422 (active & (1 << ACS_ABO_BITPOS)))
423 return 1;
424 else
425 return 0;
426 }
427
428 static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)
429 {
430 unsigned int misc_ctl;
431 unsigned char ae;
432
433
434 misc_ctl = GET_GLB_CSR(handle, MISC_CONTROL);
435 if (misc_ctl & MC_TIMESTAMP_ENABLE)
436 SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl &
437 (~MC_TIMESTAMP_ENABLE));
438
439 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
440 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0);
441 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0);
442 }
443
444 SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE);
445 }
446
447 #define ESRAM_AUTO_TINIT BIT(2)
448 #define ESRAM_AUTO_TINIT_DONE BIT(3)
449 #define ESRAM_AUTO_INIT_USED_CYCLES (1640)
450 #define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C
451 static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
452 {
453 void __iomem *csr_addr =
454 (void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v +
455 ESRAM_AUTO_INIT_CSR_OFFSET);
456 unsigned int csr_val;
457 int times = 30;
458
459 if (handle->pci_dev->device != ADF_DH895XCC_PCI_DEVICE_ID)
460 return 0;
461
462 csr_val = ADF_CSR_RD(csr_addr, 0);
463 if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))
464 return 0;
465
466 csr_val = ADF_CSR_RD(csr_addr, 0);
467 csr_val |= ESRAM_AUTO_TINIT;
468 ADF_CSR_WR(csr_addr, 0, csr_val);
469
470 do {
471 qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);
472 csr_val = ADF_CSR_RD(csr_addr, 0);
473 } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);
474 if ((times < 0)) {
475 pr_err("QAT: Fail to init eSram!\n");
476 return -EFAULT;
477 }
478 return 0;
479 }
480
481 #define SHRAM_INIT_CYCLES 2060
482 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)
483 {
484 unsigned int ae_reset_csr;
485 unsigned char ae;
486 unsigned int clk_csr;
487 unsigned int times = 100;
488 unsigned int csr;
489
490
491 ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
492 ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB);
493 ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB);
494 do {
495 SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
496 if (!(times--))
497 goto out_err;
498 csr = GET_GLB_CSR(handle, ICP_RESET);
499 } while ((handle->hal_handle->ae_mask |
500 (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr);
501
502 clk_csr = GET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE);
503 clk_csr |= handle->hal_handle->ae_mask << 0;
504 clk_csr |= handle->hal_handle->slice_mask << 20;
505 SET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr);
506 if (qat_hal_check_ae_alive(handle))
507 goto out_err;
508
509
510 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
511 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
512 INIT_CTX_ENABLE_VALUE);
513 qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX,
514 CTX_STS_INDIRECT,
515 handle->hal_handle->upc_mask &
516 INIT_PC_VALUE);
517 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
518 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
519 qat_hal_put_wakeup_event(handle, ae,
520 ICP_QAT_UCLO_AE_ALL_CTX,
521 INIT_WAKEUP_EVENTS_VALUE);
522 qat_hal_put_sig_event(handle, ae,
523 ICP_QAT_UCLO_AE_ALL_CTX,
524 INIT_SIG_EVENTS_VALUE);
525 }
526 if (qat_hal_init_esram(handle))
527 goto out_err;
528 if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0))
529 goto out_err;
530 qat_hal_reset_timestamp(handle);
531
532 return 0;
533 out_err:
534 pr_err("QAT: failed to get device out of reset\n");
535 return -EFAULT;
536 }
537
538 static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,
539 unsigned char ae, unsigned int ctx_mask)
540 {
541 unsigned int ctx;
542
543 ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
544 ctx &= IGNORE_W1C_MASK &
545 (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));
546 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
547 }
548
549 static uint64_t qat_hal_parity_64bit(uint64_t word)
550 {
551 word ^= word >> 1;
552 word ^= word >> 2;
553 word ^= word >> 4;
554 word ^= word >> 8;
555 word ^= word >> 16;
556 word ^= word >> 32;
557 return word & 1;
558 }
559
560 static uint64_t qat_hal_set_uword_ecc(uint64_t uword)
561 {
562 uint64_t bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL,
563 bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL,
564 bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL,
565 bit6_mask = 0xdaf69a46910ULL;
566
567
568 uword &= ~(0x7fULL << 0x2C);
569 uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C;
570 uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D;
571 uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E;
572 uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F;
573 uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30;
574 uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31;
575 uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32;
576 return uword;
577 }
578
579 void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
580 unsigned char ae, unsigned int uaddr,
581 unsigned int words_num, uint64_t *uword)
582 {
583 unsigned int ustore_addr;
584 unsigned int i;
585
586 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
587 uaddr |= UA_ECS;
588 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
589 for (i = 0; i < words_num; i++) {
590 unsigned int uwrd_lo, uwrd_hi;
591 uint64_t tmp;
592
593 tmp = qat_hal_set_uword_ecc(uword[i]);
594 uwrd_lo = (unsigned int)(tmp & 0xffffffff);
595 uwrd_hi = (unsigned int)(tmp >> 0x20);
596 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
597 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
598 }
599 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
600 }
601
602 static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,
603 unsigned char ae, unsigned int ctx_mask)
604 {
605 unsigned int ctx;
606
607 ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
608 ctx &= IGNORE_W1C_MASK;
609 ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;
610 ctx |= (ctx_mask << CE_ENABLE_BITPOS);
611 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
612 }
613
614 static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle)
615 {
616 unsigned char ae;
617 unsigned short reg;
618
619 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
620 for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) {
621 qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS,
622 reg, 0);
623 qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS,
624 reg, 0);
625 }
626 }
627 }
628
629 static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
630 {
631 unsigned char ae;
632 unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX;
633 int times = MAX_RETRY_TIMES;
634 unsigned int csr_val = 0;
635 unsigned int savctx = 0;
636 int ret = 0;
637
638 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
639 csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
640 csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);
641 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
642 csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
643 csr_val &= IGNORE_W1C_MASK;
644 csr_val |= CE_NN_MODE;
645 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);
646 qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst),
647 (uint64_t *)inst);
648 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
649 handle->hal_handle->upc_mask &
650 INIT_PC_VALUE);
651 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
652 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);
653 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);
654 qat_hal_wr_indr_csr(handle, ae, ctx_mask,
655 CTX_SIG_EVENTS_INDIRECT, 0);
656 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
657 qat_hal_enable_ctx(handle, ae, ctx_mask);
658 }
659 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
660
661 do {
662 ret = qat_hal_wait_cycles(handle, ae, 20, 1);
663 } while (ret && times--);
664
665 if (times < 0) {
666 pr_err("QAT: clear GPR of AE %d failed", ae);
667 return -EINVAL;
668 }
669 qat_hal_disable_ctx(handle, ae, ctx_mask);
670 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
671 savctx & ACS_ACNO);
672 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
673 INIT_CTX_ENABLE_VALUE);
674 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
675 handle->hal_handle->upc_mask &
676 INIT_PC_VALUE);
677 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
678 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
679 qat_hal_put_wakeup_event(handle, ae, ctx_mask,
680 INIT_WAKEUP_EVENTS_VALUE);
681 qat_hal_put_sig_event(handle, ae, ctx_mask,
682 INIT_SIG_EVENTS_VALUE);
683 }
684 return 0;
685 }
686
687 #define ICP_QAT_AE_OFFSET 0x20000
688 #define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000)
689 #define LOCAL_TO_XFER_REG_OFFSET 0x800
690 #define ICP_QAT_EP_OFFSET 0x3a000
691 int qat_hal_init(struct adf_accel_dev *accel_dev)
692 {
693 unsigned char ae;
694 unsigned int max_en_ae_id = 0;
695 struct icp_qat_fw_loader_handle *handle;
696 struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
697 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
698 struct adf_bar *misc_bar =
699 &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)];
700 struct adf_bar *sram_bar;
701
702 handle = kzalloc(sizeof(*handle), GFP_KERNEL);
703 if (!handle)
704 return -ENOMEM;
705
706 handle->hal_cap_g_ctl_csr_addr_v =
707 (void __iomem *)((uintptr_t)misc_bar->virt_addr +
708 ICP_QAT_CAP_OFFSET);
709 handle->hal_cap_ae_xfer_csr_addr_v =
710 (void __iomem *)((uintptr_t)misc_bar->virt_addr +
711 ICP_QAT_AE_OFFSET);
712 handle->hal_ep_csr_addr_v =
713 (void __iomem *)((uintptr_t)misc_bar->virt_addr +
714 ICP_QAT_EP_OFFSET);
715 handle->hal_cap_ae_local_csr_addr_v =
716 (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v +
717 LOCAL_TO_XFER_REG_OFFSET);
718 handle->pci_dev = pci_info->pci_dev;
719 if (handle->pci_dev->device == ADF_DH895XCC_PCI_DEVICE_ID) {
720 sram_bar =
721 &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];
722 handle->hal_sram_addr_v = sram_bar->virt_addr;
723 }
724 handle->fw_auth = (handle->pci_dev->device ==
725 ADF_DH895XCC_PCI_DEVICE_ID) ? false : true;
726 handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);
727 if (!handle->hal_handle)
728 goto out_hal_handle;
729 handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
730 handle->hal_handle->ae_mask = hw_data->ae_mask;
731 handle->hal_handle->slice_mask = hw_data->accel_mask;
732
733 handle->hal_handle->upc_mask = 0x1ffff;
734 handle->hal_handle->max_ustore = 0x4000;
735 for (ae = 0; ae < ICP_QAT_UCLO_MAX_AE; ae++) {
736 if (!(hw_data->ae_mask & (1 << ae)))
737 continue;
738 handle->hal_handle->aes[ae].free_addr = 0;
739 handle->hal_handle->aes[ae].free_size =
740 handle->hal_handle->max_ustore;
741 handle->hal_handle->aes[ae].ustore_size =
742 handle->hal_handle->max_ustore;
743 handle->hal_handle->aes[ae].live_ctx_mask =
744 ICP_QAT_UCLO_AE_ALL_CTX;
745 max_en_ae_id = ae;
746 }
747 handle->hal_handle->ae_max_num = max_en_ae_id + 1;
748
749 if (qat_hal_clr_reset(handle)) {
750 dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n");
751 goto out_err;
752 }
753 qat_hal_clear_xfer(handle);
754 if (!handle->fw_auth) {
755 if (qat_hal_clear_gpr(handle))
756 goto out_err;
757 }
758
759
760 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
761 unsigned int csr_val = 0;
762
763 csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE);
764 csr_val |= 0x1;
765 qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);
766 }
767 accel_dev->fw_loader->fw_loader = handle;
768 return 0;
769
770 out_err:
771 kfree(handle->hal_handle);
772 out_hal_handle:
773 kfree(handle);
774 return -EFAULT;
775 }
776
777 void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)
778 {
779 if (!handle)
780 return;
781 kfree(handle->hal_handle);
782 kfree(handle);
783 }
784
785 void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
786 unsigned int ctx_mask)
787 {
788 int retry = 0;
789 unsigned int fcu_sts = 0;
790
791 if (handle->fw_auth) {
792 SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_START);
793 do {
794 msleep(FW_AUTH_WAIT_PERIOD);
795 fcu_sts = GET_CAP_CSR(handle, FCU_STATUS);
796 if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1))
797 return;
798 } while (retry++ < FW_AUTH_MAX_RETRY);
799 pr_err("QAT: start error (AE 0x%x FCU_STS = 0x%x)\n", ae,
800 fcu_sts);
801 } else {
802 qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) &
803 ICP_QAT_UCLO_AE_ALL_CTX, 0x10000);
804 qat_hal_enable_ctx(handle, ae, ctx_mask);
805 }
806 }
807
808 void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
809 unsigned int ctx_mask)
810 {
811 if (!handle->fw_auth)
812 qat_hal_disable_ctx(handle, ae, ctx_mask);
813 }
814
815 void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,
816 unsigned char ae, unsigned int ctx_mask, unsigned int upc)
817 {
818 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
819 handle->hal_handle->upc_mask & upc);
820 }
821
822 static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,
823 unsigned char ae, unsigned int uaddr,
824 unsigned int words_num, uint64_t *uword)
825 {
826 unsigned int i, uwrd_lo, uwrd_hi;
827 unsigned int ustore_addr, misc_control;
828
829 misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
830 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL,
831 misc_control & 0xfffffffb);
832 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
833 uaddr |= UA_ECS;
834 for (i = 0; i < words_num; i++) {
835 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
836 uaddr++;
837 uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER);
838 uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER);
839 uword[i] = uwrd_hi;
840 uword[i] = (uword[i] << 0x20) | uwrd_lo;
841 }
842 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control);
843 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
844 }
845
846 void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
847 unsigned char ae, unsigned int uaddr,
848 unsigned int words_num, unsigned int *data)
849 {
850 unsigned int i, ustore_addr;
851
852 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
853 uaddr |= UA_ECS;
854 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
855 for (i = 0; i < words_num; i++) {
856 unsigned int uwrd_lo, uwrd_hi, tmp;
857
858 uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) |
859 ((data[i] & 0xff00) << 2) |
860 (0x3 << 8) | (data[i] & 0xff);
861 uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28);
862 uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8;
863 tmp = ((data[i] >> 0x10) & 0xffff);
864 uwrd_hi |= (hweight32(tmp) & 0x1) << 9;
865 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
866 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
867 }
868 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
869 }
870
871 #define MAX_EXEC_INST 100
872 static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
873 unsigned char ae, unsigned char ctx,
874 uint64_t *micro_inst, unsigned int inst_num,
875 int code_off, unsigned int max_cycle,
876 unsigned int *endpc)
877 {
878 uint64_t savuwords[MAX_EXEC_INST];
879 unsigned int ind_lm_addr0, ind_lm_addr1;
880 unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1;
881 unsigned int ind_cnt_sig;
882 unsigned int ind_sig, act_sig;
883 unsigned int csr_val = 0, newcsr_val;
884 unsigned int savctx;
885 unsigned int savcc, wakeup_events, savpc;
886 unsigned int ctxarb_ctl, ctx_enables;
887
888 if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) {
889 pr_err("QAT: invalid instruction num %d\n", inst_num);
890 return -EINVAL;
891 }
892
893 ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT);
894 ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT);
895 ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx,
896 INDIRECT_LM_ADDR_0_BYTE_INDEX);
897 ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx,
898 INDIRECT_LM_ADDR_1_BYTE_INDEX);
899 if (inst_num <= MAX_EXEC_INST)
900 qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);
901 qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);
902 savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT);
903 savpc = (savpc & handle->hal_handle->upc_mask) >> 0;
904 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
905 ctx_enables &= IGNORE_W1C_MASK;
906 savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE);
907 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
908 ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
909 ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
910 FUTURE_COUNT_SIGNAL_INDIRECT);
911 ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
912 CTX_SIG_EVENTS_INDIRECT);
913 act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE);
914
915 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
916 qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);
917 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0);
918 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO);
919 if (code_off)
920 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff);
921 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY);
922 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0);
923 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
924 qat_hal_enable_ctx(handle, ae, (1 << ctx));
925
926 if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0)
927 return -EFAULT;
928 if (endpc) {
929 unsigned int ctx_status;
930
931 ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx,
932 CTX_STS_INDIRECT);
933 *endpc = ctx_status & handle->hal_handle->upc_mask;
934 }
935
936 qat_hal_disable_ctx(handle, ae, (1 << ctx));
937 if (inst_num <= MAX_EXEC_INST)
938 qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords);
939 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);
940 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT,
941 handle->hal_handle->upc_mask & savpc);
942 csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
943 newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
944 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
945 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);
946 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO);
947 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl);
948 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
949 LM_ADDR_0_INDIRECT, ind_lm_addr0);
950 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
951 LM_ADDR_1_INDIRECT, ind_lm_addr1);
952 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
953 INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0);
954 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
955 INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1);
956 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
957 FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig);
958 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
959 CTX_SIG_EVENTS_INDIRECT, ind_sig);
960 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);
961 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
962
963 return 0;
964 }
965
966 static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
967 unsigned char ae, unsigned char ctx,
968 enum icp_qat_uof_regtype reg_type,
969 unsigned short reg_num, unsigned int *data)
970 {
971 unsigned int savctx, uaddr, uwrd_lo, uwrd_hi;
972 unsigned int ctxarb_cntl, ustore_addr, ctx_enables;
973 unsigned short reg_addr;
974 int status = 0;
975 uint64_t insts, savuword;
976
977 reg_addr = qat_hal_get_reg_addr(reg_type, reg_num);
978 if (reg_addr == BAD_REGADDR) {
979 pr_err("QAT: bad regaddr=0x%x\n", reg_addr);
980 return -EINVAL;
981 }
982 switch (reg_type) {
983 case ICP_GPA_REL:
984 insts = 0xA070000000ull | (reg_addr & 0x3ff);
985 break;
986 default:
987 insts = (uint64_t)0xA030000000ull | ((reg_addr & 0x3ff) << 10);
988 break;
989 }
990 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
991 ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
992 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
993 ctx_enables &= IGNORE_W1C_MASK;
994 if (ctx != (savctx & ACS_ACNO))
995 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
996 ctx & ACS_ACNO);
997 qat_hal_get_uwords(handle, ae, 0, 1, &savuword);
998 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
999 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
1000 uaddr = UA_ECS;
1001 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1002 insts = qat_hal_set_uword_ecc(insts);
1003 uwrd_lo = (unsigned int)(insts & 0xffffffff);
1004 uwrd_hi = (unsigned int)(insts >> 0x20);
1005 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
1006 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
1007 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1008
1009 qat_hal_wait_cycles(handle, ae, 0x8, 0);
1010
1011
1012
1013
1014
1015 *data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT);
1016 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
1017 qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);
1018 if (ctx != (savctx & ACS_ACNO))
1019 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
1020 savctx & ACS_ACNO);
1021 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl);
1022 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1023
1024 return status;
1025 }
1026
1027 static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,
1028 unsigned char ae, unsigned char ctx,
1029 enum icp_qat_uof_regtype reg_type,
1030 unsigned short reg_num, unsigned int data)
1031 {
1032 unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo;
1033 uint64_t insts[] = {
1034 0x0F440000000ull,
1035 0x0F040000000ull,
1036 0x0F0000C0300ull,
1037 0x0E000010000ull
1038 };
1039 const int num_inst = ARRAY_SIZE(insts), code_off = 1;
1040 const int imm_w1 = 0, imm_w0 = 1;
1041
1042 dest_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1043 if (dest_addr == BAD_REGADDR) {
1044 pr_err("QAT: bad destAddr=0x%x\n", dest_addr);
1045 return -EINVAL;
1046 }
1047
1048 data16lo = 0xffff & data;
1049 data16hi = 0xffff & (data >> 0x10);
1050 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1051 (0xff & data16hi));
1052 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1053 (0xff & data16lo));
1054 switch (reg_type) {
1055 case ICP_GPA_REL:
1056 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1057 ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1058 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1059 ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1060 break;
1061 default:
1062 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1063 ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1064
1065 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1066 ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1067 break;
1068 }
1069
1070 return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst,
1071 code_off, num_inst * 0x5, NULL);
1072 }
1073
1074 int qat_hal_get_ins_num(void)
1075 {
1076 return ARRAY_SIZE(inst_4b);
1077 }
1078
1079 static int qat_hal_concat_micro_code(uint64_t *micro_inst,
1080 unsigned int inst_num, unsigned int size,
1081 unsigned int addr, unsigned int *value)
1082 {
1083 int i;
1084 unsigned int cur_value;
1085 const uint64_t *inst_arr;
1086 int fixup_offset;
1087 int usize = 0;
1088 int orig_num;
1089
1090 orig_num = inst_num;
1091 cur_value = value[0];
1092 inst_arr = inst_4b;
1093 usize = ARRAY_SIZE(inst_4b);
1094 fixup_offset = inst_num;
1095 for (i = 0; i < usize; i++)
1096 micro_inst[inst_num++] = inst_arr[i];
1097 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr));
1098 fixup_offset++;
1099 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0);
1100 fixup_offset++;
1101 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0));
1102 fixup_offset++;
1103 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10));
1104
1105 return inst_num - orig_num;
1106 }
1107
1108 static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,
1109 unsigned char ae, unsigned char ctx,
1110 int *pfirst_exec, uint64_t *micro_inst,
1111 unsigned int inst_num)
1112 {
1113 int stat = 0;
1114 unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0;
1115 unsigned int gprb0 = 0, gprb1 = 0;
1116
1117 if (*pfirst_exec) {
1118 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0);
1119 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1);
1120 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2);
1121 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0);
1122 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1);
1123 *pfirst_exec = 0;
1124 }
1125 stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1,
1126 inst_num * 0x5, NULL);
1127 if (stat != 0)
1128 return -EFAULT;
1129 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0);
1130 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1);
1131 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2);
1132 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0);
1133 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1);
1134
1135 return 0;
1136 }
1137
1138 int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,
1139 unsigned char ae,
1140 struct icp_qat_uof_batch_init *lm_init_header)
1141 {
1142 struct icp_qat_uof_batch_init *plm_init;
1143 uint64_t *micro_inst_arry;
1144 int micro_inst_num;
1145 int alloc_inst_size;
1146 int first_exec = 1;
1147 int stat = 0;
1148
1149 plm_init = lm_init_header->next;
1150 alloc_inst_size = lm_init_header->size;
1151 if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)
1152 alloc_inst_size = handle->hal_handle->max_ustore;
1153 micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(uint64_t),
1154 GFP_KERNEL);
1155 if (!micro_inst_arry)
1156 return -ENOMEM;
1157 micro_inst_num = 0;
1158 while (plm_init) {
1159 unsigned int addr, *value, size;
1160
1161 ae = plm_init->ae;
1162 addr = plm_init->addr;
1163 value = plm_init->value;
1164 size = plm_init->size;
1165 micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry,
1166 micro_inst_num,
1167 size, addr, value);
1168 plm_init = plm_init->next;
1169 }
1170
1171 if (micro_inst_arry && (micro_inst_num > 0)) {
1172 micro_inst_arry[micro_inst_num++] = 0x0E000010000ull;
1173 stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec,
1174 micro_inst_arry,
1175 micro_inst_num);
1176 }
1177 kfree(micro_inst_arry);
1178 return stat;
1179 }
1180
1181 static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1182 unsigned char ae, unsigned char ctx,
1183 enum icp_qat_uof_regtype reg_type,
1184 unsigned short reg_num, unsigned int val)
1185 {
1186 int status = 0;
1187 unsigned int reg_addr;
1188 unsigned int ctx_enables;
1189 unsigned short mask;
1190 unsigned short dr_offset = 0x10;
1191
1192 status = ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1193 if (CE_INUSE_CONTEXTS & ctx_enables) {
1194 if (ctx & 0x1) {
1195 pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);
1196 return -EINVAL;
1197 }
1198 mask = 0x1f;
1199 dr_offset = 0x20;
1200 } else {
1201 mask = 0x0f;
1202 }
1203 if (reg_num & ~mask)
1204 return -EINVAL;
1205 reg_addr = reg_num + (ctx << 0x5);
1206 switch (reg_type) {
1207 case ICP_SR_RD_REL:
1208 case ICP_SR_REL:
1209 SET_AE_XFER(handle, ae, reg_addr, val);
1210 break;
1211 case ICP_DR_RD_REL:
1212 case ICP_DR_REL:
1213 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);
1214 break;
1215 default:
1216 status = -EINVAL;
1217 break;
1218 }
1219 return status;
1220 }
1221
1222 static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1223 unsigned char ae, unsigned char ctx,
1224 enum icp_qat_uof_regtype reg_type,
1225 unsigned short reg_num, unsigned int data)
1226 {
1227 unsigned int gprval, ctx_enables;
1228 unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi,
1229 data16low;
1230 unsigned short reg_mask;
1231 int status = 0;
1232 uint64_t micro_inst[] = {
1233 0x0F440000000ull,
1234 0x0F040000000ull,
1235 0x0A000000000ull,
1236 0x0F0000C0300ull,
1237 0x0E000010000ull
1238 };
1239 const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;
1240 const unsigned short gprnum = 0, dly = num_inst * 0x5;
1241
1242 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1243 if (CE_INUSE_CONTEXTS & ctx_enables) {
1244 if (ctx & 0x1) {
1245 pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);
1246 return -EINVAL;
1247 }
1248 reg_mask = (unsigned short)~0x1f;
1249 } else {
1250 reg_mask = (unsigned short)~0xf;
1251 }
1252 if (reg_num & reg_mask)
1253 return -EINVAL;
1254 xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1255 if (xfr_addr == BAD_REGADDR) {
1256 pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);
1257 return -EINVAL;
1258 }
1259 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);
1260 gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum);
1261 data16low = 0xffff & data;
1262 data16hi = 0xffff & (data >> 0x10);
1263 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1264 (unsigned short)(0xff & data16hi));
1265 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1266 (unsigned short)(0xff & data16low));
1267 micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) |
1268 ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1269 micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) |
1270 ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1271 micro_inst[0x2] = micro_inst[0x2] |
1272 ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10);
1273 status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst,
1274 code_off, dly, NULL);
1275 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval);
1276 return status;
1277 }
1278
1279 static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,
1280 unsigned char ae, unsigned char ctx,
1281 unsigned short nn, unsigned int val)
1282 {
1283 unsigned int ctx_enables;
1284 int stat = 0;
1285
1286 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1287 ctx_enables &= IGNORE_W1C_MASK;
1288 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);
1289
1290 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val);
1291 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1292 return stat;
1293 }
1294
1295 static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle
1296 *handle, unsigned char ae,
1297 unsigned short absreg_num,
1298 unsigned short *relreg,
1299 unsigned char *ctx)
1300 {
1301 unsigned int ctx_enables;
1302
1303 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1304 if (ctx_enables & CE_INUSE_CONTEXTS) {
1305
1306 *relreg = absreg_num & 0x1F;
1307 *ctx = (absreg_num >> 0x4) & 0x6;
1308 } else {
1309
1310 *relreg = absreg_num & 0x0F;
1311 *ctx = (absreg_num >> 0x4) & 0x7;
1312 }
1313 return 0;
1314 }
1315
1316 int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,
1317 unsigned char ae, unsigned char ctx_mask,
1318 enum icp_qat_uof_regtype reg_type,
1319 unsigned short reg_num, unsigned int regdata)
1320 {
1321 int stat = 0;
1322 unsigned short reg;
1323 unsigned char ctx = 0;
1324 enum icp_qat_uof_regtype type;
1325
1326 if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG)
1327 return -EINVAL;
1328
1329 do {
1330 if (ctx_mask == 0) {
1331 qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®,
1332 &ctx);
1333 type = reg_type - 1;
1334 } else {
1335 reg = reg_num;
1336 type = reg_type;
1337 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1338 continue;
1339 }
1340 stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata);
1341 if (stat) {
1342 pr_err("QAT: write gpr fail\n");
1343 return -EINVAL;
1344 }
1345 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1346
1347 return 0;
1348 }
1349
1350 int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1351 unsigned char ae, unsigned char ctx_mask,
1352 enum icp_qat_uof_regtype reg_type,
1353 unsigned short reg_num, unsigned int regdata)
1354 {
1355 int stat = 0;
1356 unsigned short reg;
1357 unsigned char ctx = 0;
1358 enum icp_qat_uof_regtype type;
1359
1360 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1361 return -EINVAL;
1362
1363 do {
1364 if (ctx_mask == 0) {
1365 qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®,
1366 &ctx);
1367 type = reg_type - 3;
1368 } else {
1369 reg = reg_num;
1370 type = reg_type;
1371 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1372 continue;
1373 }
1374 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg,
1375 regdata);
1376 if (stat) {
1377 pr_err("QAT: write wr xfer fail\n");
1378 return -EINVAL;
1379 }
1380 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1381
1382 return 0;
1383 }
1384
1385 int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1386 unsigned char ae, unsigned char ctx_mask,
1387 enum icp_qat_uof_regtype reg_type,
1388 unsigned short reg_num, unsigned int regdata)
1389 {
1390 int stat = 0;
1391 unsigned short reg;
1392 unsigned char ctx = 0;
1393 enum icp_qat_uof_regtype type;
1394
1395 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1396 return -EINVAL;
1397
1398 do {
1399 if (ctx_mask == 0) {
1400 qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®,
1401 &ctx);
1402 type = reg_type - 3;
1403 } else {
1404 reg = reg_num;
1405 type = reg_type;
1406 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1407 continue;
1408 }
1409 stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg,
1410 regdata);
1411 if (stat) {
1412 pr_err("QAT: write rd xfer fail\n");
1413 return -EINVAL;
1414 }
1415 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1416
1417 return 0;
1418 }
1419
1420 int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,
1421 unsigned char ae, unsigned char ctx_mask,
1422 unsigned short reg_num, unsigned int regdata)
1423 {
1424 int stat = 0;
1425 unsigned char ctx;
1426
1427 if (ctx_mask == 0)
1428 return -EINVAL;
1429
1430 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
1431 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1432 continue;
1433 stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata);
1434 if (stat) {
1435 pr_err("QAT: write neigh error\n");
1436 return -EINVAL;
1437 }
1438 }
1439
1440 return 0;
1441 }