nfp_prog 22 drivers/net/ethernet/netronome/nfp/bpf/jit.c #define nfp_for_each_insn_walk2(nfp_prog, pos, next) \ nfp_prog 23 drivers/net/ethernet/netronome/nfp/bpf/jit.c for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \ nfp_prog 25 drivers/net/ethernet/netronome/nfp/bpf/jit.c &(nfp_prog)->insns != &pos->l && \ nfp_prog 26 drivers/net/ethernet/netronome/nfp/bpf/jit.c &(nfp_prog)->insns != &next->l; \ nfp_prog 30 drivers/net/ethernet/netronome/nfp/bpf/jit.c #define nfp_for_each_insn_walk3(nfp_prog, pos, next, next2) \ nfp_prog 31 drivers/net/ethernet/netronome/nfp/bpf/jit.c for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \ nfp_prog 34 drivers/net/ethernet/netronome/nfp/bpf/jit.c &(nfp_prog)->insns != &pos->l && \ nfp_prog 35 drivers/net/ethernet/netronome/nfp/bpf/jit.c &(nfp_prog)->insns != &next->l && \ nfp_prog 36 drivers/net/ethernet/netronome/nfp/bpf/jit.c &(nfp_prog)->insns != &next2->l; \ nfp_prog 42 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 44 drivers/net/ethernet/netronome/nfp/bpf/jit.c return meta->l.prev != &nfp_prog->insns; nfp_prog 47 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn) nfp_prog 49 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (nfp_prog->__prog_alloc_len / sizeof(u64) == nfp_prog->prog_len) { nfp_prog 51 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->prog_len); nfp_prog 52 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = -ENOSPC; nfp_prog 56 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->prog[nfp_prog->prog_len] = insn; nfp_prog 57 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->prog_len++; nfp_prog 60 drivers/net/ethernet/netronome/nfp/bpf/jit.c static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog) nfp_prog 62 drivers/net/ethernet/netronome/nfp/bpf/jit.c return nfp_prog->prog_len; nfp_prog 66 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog_confirm_current_offset(struct nfp_prog *nfp_prog, unsigned int off) nfp_prog 72 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (nfp_prog->error) nfp_prog 74 drivers/net/ethernet/netronome/nfp/bpf/jit.c return !WARN_ON_ONCE(nfp_prog_current_offset(nfp_prog) != off); nfp_prog 79 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, nfp_prog 96 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog_push(nfp_prog, insn); nfp_prog 100 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd_any(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, nfp_prog 108 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = err; nfp_prog 113 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = -EFAULT; nfp_prog 118 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = -EFAULT; nfp_prog 122 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_cmd(nfp_prog, op, mode, xfer, reg.areg, reg.breg, size, ctx, nfp_prog 127 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, nfp_prog 130 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, ctx, false); nfp_prog 134 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd_indir(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, nfp_prog 137 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, ctx, true); nfp_prog 141 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip, nfp_prog 158 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog_push(nfp_prog, insn); nfp_prog 162 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_relo(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer, nfp_prog 167 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = -EFAULT; nfp_prog 171 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_br(nfp_prog, mask, nfp_prog 175 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->prog[nfp_prog->prog_len - 1] |= nfp_prog 180 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer) nfp_prog 182 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_relo(nfp_prog, mask, addr, defer, RELO_BR_REL); nfp_prog 186 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_br_bit(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 addr, u8 defer, nfp_prog 204 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog_push(nfp_prog, insn); nfp_prog 208 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_bit_relo(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr, nfp_prog 222 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = err; nfp_prog 226 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_br_bit(nfp_prog, reg.areg, reg.breg, addr, defer, set, nfp_prog 229 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->prog[nfp_prog->prog_len - 1] |= nfp_prog 234 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_bset(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr, u8 defer) nfp_prog 236 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_bit_relo(nfp_prog, src, bit, addr, defer, true, RELO_BR_REL); nfp_prog 240 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_br_alu(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi, nfp_prog 253 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog_push(nfp_prog, insn); nfp_prog 256 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void emit_rtn(struct nfp_prog *nfp_prog, swreg base, u8 defer) nfp_prog 263 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = err; nfp_prog 267 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_br_alu(nfp_prog, reg.areg, reg.breg, 0, defer, reg.dst_lmextn, nfp_prog 272 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi, nfp_prog 290 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog_push(nfp_prog, insn); nfp_prog 294 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_immed(struct nfp_prog *nfp_prog, swreg dst, u16 imm, nfp_prog 301 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = -EFAULT; nfp_prog 307 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = err; nfp_prog 312 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_immed(nfp_prog, nfp_prog 319 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_shf(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab, nfp_prog 327 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = -EFAULT; nfp_prog 359 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog_push(nfp_prog, insn); nfp_prog 363 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(struct nfp_prog *nfp_prog, swreg dst, nfp_prog 371 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = err; nfp_prog 375 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_shf(nfp_prog, reg.dst, reg.dst_ab, sc, shift, nfp_prog 381 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf_indir(struct nfp_prog *nfp_prog, swreg dst, nfp_prog 386 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = -EFAULT; nfp_prog 390 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, dst, lreg, op, rreg, sc, 0); nfp_prog 394 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_alu(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab, nfp_prog 411 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog_push(nfp_prog, insn); nfp_prog 415 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(struct nfp_prog *nfp_prog, swreg dst, nfp_prog 423 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = err; nfp_prog 427 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_alu(nfp_prog, reg.dst, reg.dst_ab, nfp_prog 433 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_mul(struct nfp_prog *nfp_prog, enum alu_dst_ab dst_ab, u16 areg, nfp_prog 450 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog_push(nfp_prog, insn); nfp_prog 454 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_mul(struct nfp_prog *nfp_prog, swreg lreg, enum mul_type type, nfp_prog 462 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = -EINVAL; nfp_prog 478 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = err; nfp_prog 482 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_mul(nfp_prog, reg.dst_ab, areg, type, step, reg.breg, reg.swap, nfp_prog 487 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_ld_field(struct nfp_prog *nfp_prog, enum shf_sc sc, nfp_prog 507 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog_push(nfp_prog, insn); nfp_prog 511 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field_any(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src, nfp_prog 520 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = err; nfp_prog 524 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_ld_field(nfp_prog, sc, reg.areg, bmask, reg.breg, shift, nfp_prog 530 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src, nfp_prog 533 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field_any(nfp_prog, dst, bmask, src, sc, shift, false); nfp_prog 537 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_lcsr(struct nfp_prog *nfp_prog, u16 areg, u16 breg, bool wr, u16 addr, nfp_prog 550 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog_push(nfp_prog, insn); nfp_prog 553 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void emit_csr_wr(struct nfp_prog *nfp_prog, swreg src, u16 addr) nfp_prog 570 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = err; nfp_prog 574 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_lcsr(nfp_prog, reg.areg, reg.breg, true, addr, nfp_prog 579 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void __emit_csr_rd(struct nfp_prog *nfp_prog, u16 addr) nfp_prog 581 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_lcsr(nfp_prog, 0, 0, false, addr, false, false); nfp_prog 584 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void emit_nop(struct nfp_prog *nfp_prog) nfp_prog 586 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_immed(nfp_prog, UR_REG_IMM, UR_REG_IMM, 0, 0, 0, 0, 0, 0, 0); nfp_prog 608 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void wrp_immed(struct nfp_prog *nfp_prog, swreg dst, u32 imm) nfp_prog 614 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, false, shift); nfp_prog 616 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, true, shift); nfp_prog 618 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_immed(nfp_prog, dst, imm & 0xffff, IMMED_WIDTH_ALL, nfp_prog 620 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_immed(nfp_prog, dst, imm >> 16, IMMED_WIDTH_WORD, nfp_prog 626 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst) nfp_prog 629 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(dst + 1), 0); nfp_prog 633 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed_relo(struct nfp_prog *nfp_prog, swreg dst, u32 imm, nfp_prog 638 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->error = -EFAULT; nfp_prog 641 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_immed(nfp_prog, dst, imm, IMMED_WIDTH_ALL, false, IMMED_SHIFT_0B); nfp_prog 643 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->prog[nfp_prog->prog_len - 1] |= nfp_prog 651 drivers/net/ethernet/netronome/nfp/bpf/jit.c static swreg ur_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg) nfp_prog 656 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, tmp_reg, imm); nfp_prog 664 drivers/net/ethernet/netronome/nfp/bpf/jit.c static swreg re_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg) nfp_prog 669 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, tmp_reg, imm); nfp_prog 673 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void wrp_nops(struct nfp_prog *nfp_prog, unsigned int count) nfp_prog 676 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_nop(nfp_prog); nfp_prog 679 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src) nfp_prog 681 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src); nfp_prog 684 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void wrp_reg_mov(struct nfp_prog *nfp_prog, u16 dst, u16 src) nfp_prog 686 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_both(dst), reg_b(src)); nfp_prog 693 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_subpart(struct nfp_prog *nfp_prog, swreg dst, swreg src, u8 field_len, nfp_prog 699 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field_any(nfp_prog, dst, mask, src, sc, offset * 8, true); nfp_prog 706 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_or_subpart(struct nfp_prog *nfp_prog, swreg dst, swreg src, nfp_prog 712 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field(nfp_prog, dst, mask, src, sc, 32 - offset * 8); nfp_prog 716 drivers/net/ethernet/netronome/nfp/bpf/jit.c addr40_offset(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset, nfp_prog 725 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(src_gpr), ALU_OP_ADD, offset); nfp_prog 726 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, imm_b(nfp_prog), reg_b(src_gpr + 1), ALU_OP_ADD_C, nfp_prog 728 drivers/net/ethernet/netronome/nfp/bpf/jit.c *rega = imm_a(nfp_prog); nfp_prog 729 drivers/net/ethernet/netronome/nfp/bpf/jit.c *regb = imm_b(nfp_prog); nfp_prog 733 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int nfp_cpp_memcpy(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 742 drivers/net/ethernet/netronome/nfp/bpf/jit.c off = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); nfp_prog 748 drivers/net/ethernet/netronome/nfp/bpf/jit.c addr40_offset(nfp_prog, meta->insn.src_reg * 2, off, &src_base, nfp_prog 753 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_none(), nfp_prog 757 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP, nfp_prog 763 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_xfer(i), reg_xfer(i)); nfp_prog 765 drivers/net/ethernet/netronome/nfp/bpf/jit.c off = re_load_imm_any(nfp_prog, meta->paired_st->off, imm_b(nfp_prog)); nfp_prog 769 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, nfp_prog 774 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, nfp_prog 779 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_none(), nfp_prog 781 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd_indir(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, nfp_prog 786 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_none(), nfp_prog 788 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, nfp_prog 795 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, nfp_prog 799 drivers/net/ethernet/netronome/nfp/bpf/jit.c off = re_load_imm_any(nfp_prog, meta->paired_st->off + 32, nfp_prog 800 drivers/net/ethernet/netronome/nfp/bpf/jit.c imm_b(nfp_prog)); nfp_prog 801 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 8, nfp_prog 810 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_none(), nfp_prog 812 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, nfp_prog 816 drivers/net/ethernet/netronome/nfp/bpf/jit.c off = re_load_imm_any(nfp_prog, new_off, imm_b(nfp_prog)); nfp_prog 817 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, nfp_prog 841 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2), nfp_prog 846 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2), nfp_prog 850 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2), nfp_prog 854 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2), nfp_prog 856 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), nfp_prog 862 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); nfp_prog 868 drivers/net/ethernet/netronome/nfp/bpf/jit.c data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, swreg offset, nfp_prog 880 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd(nfp_prog, CMD_TGT_READ8, CMD_MODE_32b, 0, nfp_prog 881 drivers/net/ethernet/netronome/nfp/bpf/jit.c pptr_reg(nfp_prog), offset, sz - 1, CMD_CTX_SWAP); nfp_prog 885 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst_gpr), reg_none(), SHF_OP_NONE, nfp_prog 889 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i)); nfp_prog 892 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, dst_gpr); nfp_prog 898 drivers/net/ethernet/netronome/nfp/bpf/jit.c data_ld_host_order(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 911 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd(nfp_prog, CMD_TGT_READ32_SWAP, mode, 0, nfp_prog 916 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field_any(nfp_prog, reg_both(dst_gpr), mask, nfp_prog 920 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i)); nfp_prog 923 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, dst_gpr); nfp_prog 929 drivers/net/ethernet/netronome/nfp/bpf/jit.c data_ld_host_order_addr32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 932 drivers/net/ethernet/netronome/nfp/bpf/jit.c return data_ld_host_order(nfp_prog, meta, dst_gpr, reg_a(src_gpr), nfp_prog 937 drivers/net/ethernet/netronome/nfp/bpf/jit.c data_ld_host_order_addr40(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 942 drivers/net/ethernet/netronome/nfp/bpf/jit.c addr40_offset(nfp_prog, src_gpr, offset, ®a, ®b); nfp_prog 944 drivers/net/ethernet/netronome/nfp/bpf/jit.c return data_ld_host_order(nfp_prog, meta, dst_gpr, rega, regb, nfp_prog 949 drivers/net/ethernet/netronome/nfp/bpf/jit.c construct_data_ind_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 955 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = ur_load_imm_any(nfp_prog, offset, imm_b(nfp_prog)); nfp_prog 956 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, imm_both(nfp_prog), reg_a(src), ALU_OP_ADD, tmp_reg); nfp_prog 959 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, imm_a(nfp_prog), nfp_prog 960 drivers/net/ethernet/netronome/nfp/bpf/jit.c imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size)); nfp_prog 961 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), nfp_prog 962 drivers/net/ethernet/netronome/nfp/bpf/jit.c plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog)); nfp_prog 963 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT); nfp_prog 966 drivers/net/ethernet/netronome/nfp/bpf/jit.c return data_ld(nfp_prog, meta, imm_b(nfp_prog), 0, size); nfp_prog 970 drivers/net/ethernet/netronome/nfp/bpf/jit.c construct_data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 976 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog)); nfp_prog 977 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg); nfp_prog 978 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT); nfp_prog 981 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog)); nfp_prog 982 drivers/net/ethernet/netronome/nfp/bpf/jit.c return data_ld(nfp_prog, meta, tmp_reg, 0, size); nfp_prog 986 drivers/net/ethernet/netronome/nfp/bpf/jit.c data_stx_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset, nfp_prog 992 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_xfer(i), reg_a(src_gpr + i)); nfp_prog 994 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, nfp_prog 1001 drivers/net/ethernet/netronome/nfp/bpf/jit.c data_st_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset, nfp_prog 1004 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_xfer(0), imm); nfp_prog 1006 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_xfer(1), imm >> 32); nfp_prog 1008 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, nfp_prog 1015 drivers/net/ethernet/netronome/nfp/bpf/jit.c (*lmem_step)(struct nfp_prog *nfp_prog, u8 gpr, u8 gpr_byte, s32 off, nfp_prog 1020 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_lmem_load(struct nfp_prog *nfp_prog, u8 dst, u8 dst_byte, s32 off, nfp_prog 1038 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_both(dst), nfp_prog 1070 drivers/net/ethernet/netronome/nfp/bpf/jit.c reg = imm_a(nfp_prog); nfp_prog 1077 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg, reg_lm(0, idx)); nfp_prog 1080 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field_any(nfp_prog, reg_both(dst), mask, reg, sc, shf, new_gpr); nfp_prog 1083 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3)); nfp_prog 1089 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_lmem_store(struct nfp_prog *nfp_prog, u8 src, u8 src_byte, s32 off, nfp_prog 1107 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, nfp_prog 1140 drivers/net/ethernet/netronome/nfp/bpf/jit.c reg = imm_a(nfp_prog); nfp_prog 1145 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg, reg_lm(0, idx)); nfp_prog 1148 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field(nfp_prog, reg, mask, reg_b(src), sc, shf); nfp_prog 1152 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_lm(0, idx), reg); nfp_prog 1154 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3)); nfp_prog 1161 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 1165 drivers/net/ethernet/netronome/nfp/bpf/jit.c s32 off = nfp_prog->stack_frame_depth + meta->insn.off + ptr_off; nfp_prog 1180 drivers/net/ethernet/netronome/nfp/bpf/jit.c stack_off_reg = ur_load_imm_any(nfp_prog, meta->insn.off, nfp_prog 1181 drivers/net/ethernet/netronome/nfp/bpf/jit.c stack_imm(nfp_prog)); nfp_prog 1183 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, imm_b(nfp_prog), nfp_prog 1197 drivers/net/ethernet/netronome/nfp/bpf/jit.c stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 32), nfp_prog 1198 drivers/net/ethernet/netronome/nfp/bpf/jit.c stack_imm(nfp_prog)); nfp_prog 1199 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, imm_b(nfp_prog), nfp_prog 1200 drivers/net/ethernet/netronome/nfp/bpf/jit.c stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg); nfp_prog 1204 drivers/net/ethernet/netronome/nfp/bpf/jit.c stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 4), nfp_prog 1205 drivers/net/ethernet/netronome/nfp/bpf/jit.c stack_imm(nfp_prog)); nfp_prog 1207 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, imm_b(nfp_prog), nfp_prog 1208 drivers/net/ethernet/netronome/nfp/bpf/jit.c stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg); nfp_prog 1218 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_csr_wr(nfp_prog, imm_b(nfp_prog), NFP_CSR_ACT_LM_ADDR3); nfp_prog 1224 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_nops(nfp_prog, nop_cnt); nfp_prog 1228 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, gpr); nfp_prog 1243 drivers/net/ethernet/netronome/nfp/bpf/jit.c ret = step(nfp_prog, gpr, gpr_byte, off, slice_size, nfp_prog 1265 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu_imm(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u32 imm) nfp_prog 1271 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(dst), 0); nfp_prog 1277 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(dst), ~0U); nfp_prog 1283 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_both(dst), reg_none(), nfp_prog 1289 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog)); nfp_prog 1290 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, tmp_reg); nfp_prog 1294 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 1305 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, imm & ~0U); nfp_prog 1306 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, alu_op, imm >> 32); nfp_prog 1312 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu64_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 1317 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src)); nfp_prog 1318 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_both(dst + 1), nfp_prog 1325 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 1331 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu_imm(nfp_prog, dst, alu_op, insn->imm); nfp_prog 1332 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, dst); nfp_prog 1338 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 1343 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src)); nfp_prog 1344 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, dst); nfp_prog 1350 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_test_reg_one(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u8 src, nfp_prog 1353 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(dst), alu_op, reg_b(src)); nfp_prog 1354 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, br_mask, off, 0); nfp_prog 1358 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 1363 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_test_reg_one(nfp_prog, insn->dst_reg * 2, alu_op, nfp_prog 1366 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op, nfp_prog 1400 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1416 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); nfp_prog 1418 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(reg), alu_op, tmp_reg); nfp_prog 1420 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), tmp_reg, alu_op, reg_a(reg)); nfp_prog 1423 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); nfp_prog 1425 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), nfp_prog 1428 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), nfp_prog 1432 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, code->br_mask, insn->off, 0); nfp_prog 1437 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1456 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg)); nfp_prog 1458 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), nfp_prog 1460 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, code->br_mask, insn->off, 0); nfp_prog 1465 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void wrp_end32(struct nfp_prog *nfp_prog, swreg reg_in, u8 gpr_out) nfp_prog 1467 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field(nfp_prog, reg_both(gpr_out), 0xf, reg_in, nfp_prog 1469 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field(nfp_prog, reg_both(gpr_out), 0x5, reg_a(gpr_out), nfp_prog 1474 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mul_u32(struct nfp_prog *nfp_prog, swreg dst_hi, swreg dst_lo, swreg lreg, nfp_prog 1477 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_mul(nfp_prog, lreg, MUL_TYPE_START, MUL_STEP_NONE, rreg); nfp_prog 1478 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_1, rreg); nfp_prog 1479 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_2, rreg); nfp_prog 1480 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_3, rreg); nfp_prog 1481 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_4, rreg); nfp_prog 1482 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_mul(nfp_prog, dst_lo, MUL_TYPE_STEP_32x32, MUL_LAST, reg_none()); nfp_prog 1484 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_mul(nfp_prog, dst_hi, MUL_TYPE_STEP_32x32, MUL_LAST_2, nfp_prog 1487 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, dst_hi, 0); nfp_prog 1491 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mul_u16(struct nfp_prog *nfp_prog, swreg dst_hi, swreg dst_lo, swreg lreg, nfp_prog 1494 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_mul(nfp_prog, lreg, MUL_TYPE_START, MUL_STEP_NONE, rreg); nfp_prog 1495 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_16x16, MUL_STEP_1, rreg); nfp_prog 1496 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_16x16, MUL_STEP_2, rreg); nfp_prog 1497 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_mul(nfp_prog, dst_lo, MUL_TYPE_STEP_16x16, MUL_LAST, reg_none()); nfp_prog 1501 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mul(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 1520 drivers/net/ethernet/netronome/nfp/bpf/jit.c multiplier = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog)); nfp_prog 1524 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mul_u32(nfp_prog, dst_hi, dst_lo, multiplicand, multiplier, nfp_prog 1527 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mul_u16(nfp_prog, dst_hi, dst_lo, multiplicand, multiplier); nfp_prog 1532 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int wrp_div_imm(struct nfp_prog *nfp_prog, u8 dst, u64 imm) nfp_prog 1540 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, dst_both, 0); nfp_prog 1556 drivers/net/ethernet/netronome/nfp/bpf/jit.c swreg tmp_b = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog)); nfp_prog 1558 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), dst_a, ALU_OP_SUB, tmp_b); nfp_prog 1559 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, imm_a(nfp_prog), 0); nfp_prog 1560 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, dst_both, imm_a(nfp_prog), ALU_OP_ADD_C, nfp_prog 1573 drivers/net/ethernet/netronome/nfp/bpf/jit.c magic = ur_load_imm_any(nfp_prog, rvalue.m, imm_b(nfp_prog)); nfp_prog 1575 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b, nfp_prog 1578 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mul_u32(nfp_prog, imm_both(nfp_prog), reg_none(), dst_a, nfp_prog 1580 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, dst_both, dst_a, ALU_OP_SUB, nfp_prog 1581 drivers/net/ethernet/netronome/nfp/bpf/jit.c imm_b(nfp_prog)); nfp_prog 1582 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b, nfp_prog 1584 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, dst_both, dst_a, ALU_OP_ADD, nfp_prog 1585 drivers/net/ethernet/netronome/nfp/bpf/jit.c imm_b(nfp_prog)); nfp_prog 1586 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b, nfp_prog 1590 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, nfp_prog 1592 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mul_u32(nfp_prog, dst_both, reg_none(), dst_a, magic, true); nfp_prog 1593 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, nfp_prog 1600 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1602 drivers/net/ethernet/netronome/nfp/bpf/jit.c swreg tmp = imm_a(nfp_prog), tmp_len = imm_b(nfp_prog); nfp_prog 1606 drivers/net/ethernet/netronome/nfp/bpf/jit.c adjust_head = &nfp_prog->bpf->adjust_head; nfp_prog 1609 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (nfp_prog->adjust_head_location != UINT_MAX) { nfp_prog 1610 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (WARN_ON_ONCE(nfp_prog->adjust_head_location != meta->n)) nfp_prog 1613 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, pptr_reg(nfp_prog), nfp_prog 1614 drivers/net/ethernet/netronome/nfp/bpf/jit.c reg_a(2 * 2), ALU_OP_ADD, pptr_reg(nfp_prog)); nfp_prog 1615 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, plen_reg(nfp_prog), nfp_prog 1616 drivers/net/ethernet/netronome/nfp/bpf/jit.c plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); nfp_prog 1617 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, pv_len(nfp_prog), nfp_prog 1618 drivers/net/ethernet/netronome/nfp/bpf/jit.c pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); nfp_prog 1620 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(0), 0); nfp_prog 1621 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(1), 0); nfp_prog 1630 drivers/net/ethernet/netronome/nfp/bpf/jit.c ret_einval = nfp_prog_current_offset(nfp_prog) + 14; nfp_prog 1634 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, tmp, nfp_prog 1635 drivers/net/ethernet/netronome/nfp/bpf/jit.c reg_a(2 * 2), ALU_OP_ADD_2B, pptr_reg(nfp_prog)); nfp_prog 1638 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), nfp_prog 1640 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BLO, ret_einval, 0); nfp_prog 1641 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), nfp_prog 1643 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BLO, ret_einval, 0); nfp_prog 1646 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, tmp_len, nfp_prog 1647 drivers/net/ethernet/netronome/nfp/bpf/jit.c plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); nfp_prog 1648 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), nfp_prog 1650 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BMI, ret_einval, 0); nfp_prog 1653 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(0), 0); nfp_prog 1654 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(1), 0); nfp_prog 1657 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field(nfp_prog, pptr_reg(nfp_prog), 0x3, tmp, SHF_SC_NONE, 0); nfp_prog 1660 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_UNC, end, 2); nfp_prog 1662 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, plen_reg(nfp_prog), nfp_prog 1663 drivers/net/ethernet/netronome/nfp/bpf/jit.c plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); nfp_prog 1664 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, pv_len(nfp_prog), nfp_prog 1665 drivers/net/ethernet/netronome/nfp/bpf/jit.c pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); nfp_prog 1668 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, ret_einval)) nfp_prog 1671 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(0), -22); nfp_prog 1672 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(1), ~0); nfp_prog 1674 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, end)) nfp_prog 1680 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int adjust_tail(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1685 drivers/net/ethernet/netronome/nfp/bpf/jit.c BUILD_BUG_ON(plen_reg(nfp_prog) != reg_b(STATIC_REG_PKT_LEN)); nfp_prog 1687 drivers/net/ethernet/netronome/nfp/bpf/jit.c plen = imm_a(nfp_prog); nfp_prog 1690 drivers/net/ethernet/netronome/nfp/bpf/jit.c ret_einval = nfp_prog_current_offset(nfp_prog) + 9; nfp_prog 1691 drivers/net/ethernet/netronome/nfp/bpf/jit.c end = nfp_prog_current_offset(nfp_prog) + 11; nfp_prog 1694 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, plen, plen_reg(nfp_prog), ALU_OP_ADD, delta); nfp_prog 1698 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BCC, ret_einval, 0); nfp_prog 1701 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), plen, ALU_OP_SUB, reg_imm(ETH_HLEN)); nfp_prog 1702 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BMI, ret_einval, 0); nfp_prog 1704 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, plen_reg(nfp_prog), nfp_prog 1705 drivers/net/ethernet/netronome/nfp/bpf/jit.c plen_reg(nfp_prog), ALU_OP_ADD, delta); nfp_prog 1706 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, pv_len(nfp_prog), nfp_prog 1707 drivers/net/ethernet/netronome/nfp/bpf/jit.c pv_len(nfp_prog), ALU_OP_ADD, delta); nfp_prog 1709 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_UNC, end, 2); nfp_prog 1710 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(0), 0); nfp_prog 1711 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(1), 0); nfp_prog 1713 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, ret_einval)) nfp_prog 1716 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(0), -22); nfp_prog 1717 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(1), ~0); nfp_prog 1719 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, end)) nfp_prog 1726 drivers/net/ethernet/netronome/nfp/bpf/jit.c map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1733 drivers/net/ethernet/netronome/nfp/bpf/jit.c lm_off = nfp_prog->stack_frame_depth; nfp_prog 1739 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_csr_wr(nfp_prog, reg_b(2 * 2), NFP_CSR_ACT_LM_ADDR0); nfp_prog 1741 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_csr_wr(nfp_prog, reg_b(3 * 2), NFP_CSR_ACT_LM_ADDR2); nfp_prog 1743 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id, nfp_prog 1745 drivers/net/ethernet/netronome/nfp/bpf/jit.c ret_tgt = nfp_prog_current_offset(nfp_prog) + 2; nfp_prog 1748 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_a(0), reg_a(2)); nfp_prog 1751 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL); nfp_prog 1753 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt)) nfp_prog 1760 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_csr_wr(nfp_prog, stack_reg(nfp_prog), NFP_CSR_ACT_LM_ADDR0); nfp_prog 1761 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_nops(nfp_prog, 3); nfp_prog 1767 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_get_prandom_u32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1769 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_csr_rd(nfp_prog, NFP_CSR_PSEUDO_RND_NUM); nfp_prog 1771 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_immed(nfp_prog, reg_both(0), 0, nfp_prog 1773 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_immed(nfp_prog, reg_both(1), 0, nfp_prog 1779 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_perf_event_output(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1784 drivers/net/ethernet/netronome/nfp/bpf/jit.c ptr_type = ur_load_imm_any(nfp_prog, meta->arg1.type, imm_a(nfp_prog)); nfp_prog 1786 drivers/net/ethernet/netronome/nfp/bpf/jit.c ret_tgt = nfp_prog_current_offset(nfp_prog) + 3; nfp_prog 1788 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id, nfp_prog 1792 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_a(1), ptr_type); nfp_prog 1795 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL); nfp_prog 1797 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt)) nfp_prog 1804 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_queue_select(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1808 drivers/net/ethernet/netronome/nfp/bpf/jit.c jmp_tgt = nfp_prog_current_offset(nfp_prog) + 5; nfp_prog 1811 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(meta->insn.src_reg * 2), nfp_prog 1813 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BEQ, jmp_tgt, 2); nfp_prog 1816 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, pv_qsel_set(nfp_prog), nfp_prog 1817 drivers/net/ethernet/netronome/nfp/bpf/jit.c pv_qsel_set(nfp_prog), SHF_OP_OR, reg_imm(1), nfp_prog 1819 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field(nfp_prog, nfp_prog 1820 drivers/net/ethernet/netronome/nfp/bpf/jit.c pv_qsel_val(nfp_prog), 0x1, reg_b(meta->insn.src_reg * 2), nfp_prog 1825 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field(nfp_prog, nfp_prog 1826 drivers/net/ethernet/netronome/nfp/bpf/jit.c pv_qsel_val(nfp_prog), 0x1, reg_imm(NFP_NET_RXR_MAX), nfp_prog 1829 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, jmp_tgt)) nfp_prog 1836 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1845 drivers/net/ethernet/netronome/nfp/bpf/jit.c stack_depth_reg = ur_load_imm_any(nfp_prog, nfp_prog 1846 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->stack_frame_depth, nfp_prog 1847 drivers/net/ethernet/netronome/nfp/bpf/jit.c stack_imm(nfp_prog)); nfp_prog 1848 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_both(dst), stack_reg(nfp_prog), nfp_prog 1850 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(dst + 1), 0); nfp_prog 1852 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_mov(nfp_prog, dst, src); nfp_prog 1853 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_mov(nfp_prog, dst + 1, src + 1); nfp_prog 1859 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mov_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1863 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2), imm & ~0U); nfp_prog 1864 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), imm >> 32); nfp_prog 1869 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int xor_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1871 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu64_reg(nfp_prog, meta, ALU_OP_XOR); nfp_prog 1874 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int xor_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1876 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu64_imm(nfp_prog, meta, ALU_OP_XOR, !meta->insn.imm); nfp_prog 1879 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int and_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1881 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu64_reg(nfp_prog, meta, ALU_OP_AND); nfp_prog 1884 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int and_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1886 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu64_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm); nfp_prog 1889 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int or_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1891 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu64_reg(nfp_prog, meta, ALU_OP_OR); nfp_prog 1894 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int or_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1896 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu64_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm); nfp_prog 1899 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int add_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1903 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), nfp_prog 1906 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), nfp_prog 1913 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int add_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1918 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_ADD, imm & ~0U); nfp_prog 1919 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_ADD_C, imm >> 32); nfp_prog 1924 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int sub_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1928 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), nfp_prog 1931 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), nfp_prog 1938 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int sub_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1943 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_SUB, imm & ~0U); nfp_prog 1944 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_SUB_C, imm >> 32); nfp_prog 1949 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mul_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1951 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_mul(nfp_prog, meta, true, true); nfp_prog 1954 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mul_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1956 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_mul(nfp_prog, meta, true, false); nfp_prog 1959 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int div_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1963 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_div_imm(nfp_prog, insn->dst_reg * 2, insn->imm); nfp_prog 1966 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int div_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1971 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_div_imm(nfp_prog, meta->insn.dst_reg * 2, meta->umin_src); nfp_prog 1974 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int neg_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 1978 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), reg_imm(0), nfp_prog 1980 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), reg_imm(0), nfp_prog 1996 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int __shl_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt) nfp_prog 2002 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst + 1), reg_a(dst + 1), nfp_prog 2005 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, nfp_prog 2008 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_mov(nfp_prog, dst + 1, dst); nfp_prog 2009 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(dst), 0); nfp_prog 2011 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, nfp_prog 2013 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(dst), 0); nfp_prog 2019 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2024 drivers/net/ethernet/netronome/nfp/bpf/jit.c return __shl_imm64(nfp_prog, dst, insn->imm); nfp_prog 2027 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shl_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src) nfp_prog 2029 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, imm_both(nfp_prog), reg_imm(32), ALU_OP_SUB, nfp_prog 2031 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), imm_a(nfp_prog), ALU_OP_OR, reg_imm(0)); nfp_prog 2032 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_a(dst + 1), SHF_OP_NONE, nfp_prog 2037 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shl_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src) nfp_prog 2039 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); nfp_prog 2040 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, nfp_prog 2044 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shl_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src) nfp_prog 2046 drivers/net/ethernet/netronome/nfp/bpf/jit.c shl_reg64_lt32_high(nfp_prog, dst, src); nfp_prog 2047 drivers/net/ethernet/netronome/nfp/bpf/jit.c shl_reg64_lt32_low(nfp_prog, dst, src); nfp_prog 2050 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shl_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src) nfp_prog 2052 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); nfp_prog 2053 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, nfp_prog 2055 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(dst), 0); nfp_prog 2058 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shl_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2068 drivers/net/ethernet/netronome/nfp/bpf/jit.c return __shl_imm64(nfp_prog, dst, umin); nfp_prog 2072 drivers/net/ethernet/netronome/nfp/bpf/jit.c shl_reg64_lt32(nfp_prog, dst, src); nfp_prog 2074 drivers/net/ethernet/netronome/nfp/bpf/jit.c shl_reg64_ge32(nfp_prog, dst, src); nfp_prog 2081 drivers/net/ethernet/netronome/nfp/bpf/jit.c label_ge32 = nfp_prog_current_offset(nfp_prog) + 7; nfp_prog 2082 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0); nfp_prog 2084 drivers/net/ethernet/netronome/nfp/bpf/jit.c shl_reg64_lt32_high(nfp_prog, dst, src); nfp_prog 2085 drivers/net/ethernet/netronome/nfp/bpf/jit.c label_end = nfp_prog_current_offset(nfp_prog) + 6; nfp_prog 2086 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_UNC, label_end, 2); nfp_prog 2088 drivers/net/ethernet/netronome/nfp/bpf/jit.c shl_reg64_lt32_low(nfp_prog, dst, src); nfp_prog 2090 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32)) nfp_prog 2092 drivers/net/ethernet/netronome/nfp/bpf/jit.c shl_reg64_ge32(nfp_prog, dst, src); nfp_prog 2094 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, label_end)) nfp_prog 2111 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int __shr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt) nfp_prog 2117 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE, nfp_prog 2119 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, nfp_prog 2122 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_mov(nfp_prog, dst, dst + 1); nfp_prog 2123 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(dst + 1), 0); nfp_prog 2125 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, nfp_prog 2127 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(dst + 1), 0); nfp_prog 2133 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2138 drivers/net/ethernet/netronome/nfp/bpf/jit.c return __shr_imm64(nfp_prog, dst, insn->imm); nfp_prog 2142 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src) nfp_prog 2144 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); nfp_prog 2145 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, nfp_prog 2149 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src) nfp_prog 2151 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); nfp_prog 2152 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf_indir(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE, nfp_prog 2156 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src) nfp_prog 2158 drivers/net/ethernet/netronome/nfp/bpf/jit.c shr_reg64_lt32_low(nfp_prog, dst, src); nfp_prog 2159 drivers/net/ethernet/netronome/nfp/bpf/jit.c shr_reg64_lt32_high(nfp_prog, dst, src); nfp_prog 2162 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src) nfp_prog 2164 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); nfp_prog 2165 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, nfp_prog 2167 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(dst + 1), 0); nfp_prog 2170 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2180 drivers/net/ethernet/netronome/nfp/bpf/jit.c return __shr_imm64(nfp_prog, dst, umin); nfp_prog 2184 drivers/net/ethernet/netronome/nfp/bpf/jit.c shr_reg64_lt32(nfp_prog, dst, src); nfp_prog 2186 drivers/net/ethernet/netronome/nfp/bpf/jit.c shr_reg64_ge32(nfp_prog, dst, src); nfp_prog 2193 drivers/net/ethernet/netronome/nfp/bpf/jit.c label_ge32 = nfp_prog_current_offset(nfp_prog) + 6; nfp_prog 2194 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0); nfp_prog 2195 drivers/net/ethernet/netronome/nfp/bpf/jit.c shr_reg64_lt32_low(nfp_prog, dst, src); nfp_prog 2196 drivers/net/ethernet/netronome/nfp/bpf/jit.c label_end = nfp_prog_current_offset(nfp_prog) + 6; nfp_prog 2197 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_UNC, label_end, 2); nfp_prog 2199 drivers/net/ethernet/netronome/nfp/bpf/jit.c shr_reg64_lt32_high(nfp_prog, dst, src); nfp_prog 2201 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32)) nfp_prog 2203 drivers/net/ethernet/netronome/nfp/bpf/jit.c shr_reg64_ge32(nfp_prog, dst, src); nfp_prog 2205 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, label_end)) nfp_prog 2215 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int __ashr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt) nfp_prog 2221 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE, nfp_prog 2224 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR, nfp_prog 2226 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, nfp_prog 2230 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_mov(nfp_prog, dst, dst + 1); nfp_prog 2231 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, nfp_prog 2234 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR, nfp_prog 2236 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, nfp_prog 2238 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, nfp_prog 2245 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int ashr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2250 drivers/net/ethernet/netronome/nfp/bpf/jit.c return __ashr_imm64(nfp_prog, dst, insn->imm); nfp_prog 2253 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void ashr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src) nfp_prog 2258 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1)); nfp_prog 2259 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, nfp_prog 2263 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void ashr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src) nfp_prog 2268 drivers/net/ethernet/netronome/nfp/bpf/jit.c return shr_reg64_lt32_low(nfp_prog, dst, src); nfp_prog 2271 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void ashr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src) nfp_prog 2273 drivers/net/ethernet/netronome/nfp/bpf/jit.c ashr_reg64_lt32_low(nfp_prog, dst, src); nfp_prog 2274 drivers/net/ethernet/netronome/nfp/bpf/jit.c ashr_reg64_lt32_high(nfp_prog, dst, src); nfp_prog 2277 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void ashr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src) nfp_prog 2279 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1)); nfp_prog 2280 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, nfp_prog 2282 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, nfp_prog 2287 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int ashr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2297 drivers/net/ethernet/netronome/nfp/bpf/jit.c return __ashr_imm64(nfp_prog, dst, umin); nfp_prog 2301 drivers/net/ethernet/netronome/nfp/bpf/jit.c ashr_reg64_lt32(nfp_prog, dst, src); nfp_prog 2303 drivers/net/ethernet/netronome/nfp/bpf/jit.c ashr_reg64_ge32(nfp_prog, dst, src); nfp_prog 2307 drivers/net/ethernet/netronome/nfp/bpf/jit.c label_ge32 = nfp_prog_current_offset(nfp_prog) + 6; nfp_prog 2308 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0); nfp_prog 2309 drivers/net/ethernet/netronome/nfp/bpf/jit.c ashr_reg64_lt32_low(nfp_prog, dst, src); nfp_prog 2310 drivers/net/ethernet/netronome/nfp/bpf/jit.c label_end = nfp_prog_current_offset(nfp_prog) + 6; nfp_prog 2311 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_UNC, label_end, 2); nfp_prog 2313 drivers/net/ethernet/netronome/nfp/bpf/jit.c ashr_reg64_lt32_high(nfp_prog, dst, src); nfp_prog 2315 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32)) nfp_prog 2317 drivers/net/ethernet/netronome/nfp/bpf/jit.c ashr_reg64_ge32(nfp_prog, dst, src); nfp_prog 2319 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, label_end)) nfp_prog 2326 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mov_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2330 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_mov(nfp_prog, insn->dst_reg * 2, insn->src_reg * 2); nfp_prog 2331 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); nfp_prog 2336 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mov_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2340 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm); nfp_prog 2341 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); nfp_prog 2346 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int xor_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2348 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu32_reg(nfp_prog, meta, ALU_OP_XOR); nfp_prog 2351 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int xor_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2353 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu32_imm(nfp_prog, meta, ALU_OP_XOR); nfp_prog 2356 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int and_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2358 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu32_reg(nfp_prog, meta, ALU_OP_AND); nfp_prog 2361 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int and_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2363 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu32_imm(nfp_prog, meta, ALU_OP_AND); nfp_prog 2366 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int or_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2368 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu32_reg(nfp_prog, meta, ALU_OP_OR); nfp_prog 2371 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int or_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2373 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu32_imm(nfp_prog, meta, ALU_OP_OR); nfp_prog 2376 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int add_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2378 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu32_reg(nfp_prog, meta, ALU_OP_ADD); nfp_prog 2381 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int add_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2383 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu32_imm(nfp_prog, meta, ALU_OP_ADD); nfp_prog 2386 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int sub_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2388 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu32_reg(nfp_prog, meta, ALU_OP_SUB); nfp_prog 2391 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int sub_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2393 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_alu32_imm(nfp_prog, meta, ALU_OP_SUB); nfp_prog 2396 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mul_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2398 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_mul(nfp_prog, meta, false, true); nfp_prog 2401 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mul_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2403 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_mul(nfp_prog, meta, false, false); nfp_prog 2406 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int div_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2408 drivers/net/ethernet/netronome/nfp/bpf/jit.c return div_reg64(nfp_prog, meta); nfp_prog 2411 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int div_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2413 drivers/net/ethernet/netronome/nfp/bpf/jit.c return div_imm64(nfp_prog, meta); nfp_prog 2416 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2420 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_both(dst), reg_imm(0), ALU_OP_SUB, reg_b(dst)); nfp_prog 2421 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, dst); nfp_prog 2427 drivers/net/ethernet/netronome/nfp/bpf/jit.c __ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst, nfp_prog 2432 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(dst), ALU_OP_OR, nfp_prog 2434 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, nfp_prog 2437 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, dst); nfp_prog 2442 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int ashr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2452 drivers/net/ethernet/netronome/nfp/bpf/jit.c return __ashr_imm(nfp_prog, meta, dst, umin); nfp_prog 2458 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst)); nfp_prog 2459 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, nfp_prog 2461 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, dst); nfp_prog 2466 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2471 drivers/net/ethernet/netronome/nfp/bpf/jit.c return __ashr_imm(nfp_prog, meta, dst, insn->imm); nfp_prog 2475 drivers/net/ethernet/netronome/nfp/bpf/jit.c __shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst, nfp_prog 2479 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, nfp_prog 2481 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, dst); nfp_prog 2485 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2490 drivers/net/ethernet/netronome/nfp/bpf/jit.c return __shr_imm(nfp_prog, meta, dst, insn->imm); nfp_prog 2493 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2503 drivers/net/ethernet/netronome/nfp/bpf/jit.c return __shr_imm(nfp_prog, meta, dst, umin); nfp_prog 2506 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); nfp_prog 2507 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, nfp_prog 2509 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, dst); nfp_prog 2514 drivers/net/ethernet/netronome/nfp/bpf/jit.c __shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst, nfp_prog 2518 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, nfp_prog 2520 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, dst); nfp_prog 2524 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2529 drivers/net/ethernet/netronome/nfp/bpf/jit.c return __shl_imm(nfp_prog, meta, dst, insn->imm); nfp_prog 2532 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shl_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2542 drivers/net/ethernet/netronome/nfp/bpf/jit.c return __shl_imm(nfp_prog, meta, dst, umin); nfp_prog 2545 drivers/net/ethernet/netronome/nfp/bpf/jit.c shl_reg64_lt32_low(nfp_prog, dst, src); nfp_prog 2546 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, dst); nfp_prog 2550 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int end_reg32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2557 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field(nfp_prog, reg_both(gpr), 0x9, reg_b(gpr), nfp_prog 2559 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field(nfp_prog, reg_both(gpr), 0xe, reg_a(gpr), nfp_prog 2562 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(gpr + 1), 0); nfp_prog 2565 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_end32(nfp_prog, reg_a(gpr), gpr); nfp_prog 2566 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(gpr + 1), 0); nfp_prog 2569 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, imm_a(nfp_prog), reg_b(gpr + 1)); nfp_prog 2571 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_end32(nfp_prog, reg_a(gpr), gpr + 1); nfp_prog 2572 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_end32(nfp_prog, imm_a(nfp_prog), gpr); nfp_prog 2579 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int imm_ld8_part2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2589 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(dst), imm_lo); nfp_prog 2593 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_both(dst + 1), reg_a(dst)); nfp_prog 2595 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(dst + 1), imm_hi); nfp_prog 2600 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2606 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2608 drivers/net/ethernet/netronome/nfp/bpf/jit.c return construct_data_ld(nfp_prog, meta, meta->insn.imm, 1); nfp_prog 2611 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2613 drivers/net/ethernet/netronome/nfp/bpf/jit.c return construct_data_ld(nfp_prog, meta, meta->insn.imm, 2); nfp_prog 2616 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2618 drivers/net/ethernet/netronome/nfp/bpf/jit.c return construct_data_ld(nfp_prog, meta, meta->insn.imm, 4); nfp_prog 2621 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2623 drivers/net/ethernet/netronome/nfp/bpf/jit.c return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm, nfp_prog 2627 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2629 drivers/net/ethernet/netronome/nfp/bpf/jit.c return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm, nfp_prog 2633 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2635 drivers/net/ethernet/netronome/nfp/bpf/jit.c return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm, nfp_prog 2640 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 2643 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_op_stack(nfp_prog, meta, size, ptr_off, nfp_prog 2648 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 2657 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, dst, plen_reg(nfp_prog)); nfp_prog 2662 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); nfp_prog 2667 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, dst, nfp_prog 2668 drivers/net/ethernet/netronome/nfp/bpf/jit.c plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); nfp_prog 2674 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); nfp_prog 2679 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 2688 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); nfp_prog 2693 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, dst, nfp_prog 2694 drivers/net/ethernet/netronome/nfp/bpf/jit.c plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); nfp_prog 2700 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); nfp_prog 2706 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 2711 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); nfp_prog 2713 drivers/net/ethernet/netronome/nfp/bpf/jit.c return data_ld_host_order_addr32(nfp_prog, meta, meta->insn.src_reg * 2, nfp_prog 2718 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx_emem(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 2723 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); nfp_prog 2725 drivers/net/ethernet/netronome/nfp/bpf/jit.c return data_ld_host_order_addr40(nfp_prog, meta, meta->insn.src_reg * 2, nfp_prog 2730 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx_data_init_pktcache(struct nfp_prog *nfp_prog, nfp_prog 2739 drivers/net/ethernet/netronome/nfp/bpf/jit.c off = re_load_imm_any(nfp_prog, range_start, imm_b(nfp_prog)); nfp_prog 2747 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_none(), nfp_prog 2751 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0, src_base, nfp_prog 2756 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx_data_from_pktcache_unaligned(struct nfp_prog *nfp_prog, nfp_prog 2783 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_subpart(nfp_prog, dst_lo, src_lo, len_lo, off); nfp_prog 2786 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, dst_gpr); nfp_prog 2793 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_or_subpart(nfp_prog, dst_lo, src_mid, len_mid, len_lo); nfp_prog 2794 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, dst_gpr); nfp_prog 2798 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_or_subpart(nfp_prog, dst_lo, src_mid, nfp_prog 2800 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_subpart(nfp_prog, dst_hi, src_mid, len_lo, nfp_prog 2802 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_or_subpart(nfp_prog, dst_hi, src_hi, REG_WIDTH - len_lo, nfp_prog 2810 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx_data_from_pktcache_aligned(struct nfp_prog *nfp_prog, nfp_prog 2824 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_subpart(nfp_prog, dst_lo, src_lo, size, 0); nfp_prog 2825 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, dst_gpr); nfp_prog 2827 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, dst_lo, src_lo); nfp_prog 2828 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(nfp_prog, meta, dst_gpr); nfp_prog 2832 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, dst_lo, src_lo); nfp_prog 2833 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, dst_hi, src_hi); nfp_prog 2840 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx_data_from_pktcache(struct nfp_prog *nfp_prog, nfp_prog 2846 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_ldx_data_from_pktcache_aligned(nfp_prog, meta, size); nfp_prog 2848 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_ldx_data_from_pktcache_unaligned(nfp_prog, meta, size); nfp_prog 2852 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 2856 drivers/net/ethernet/netronome/nfp/bpf/jit.c return nfp_cpp_memcpy(nfp_prog, meta); nfp_prog 2859 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (nfp_prog->type == BPF_PROG_TYPE_XDP) nfp_prog 2860 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_ldx_xdp(nfp_prog, meta, size); nfp_prog 2862 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_ldx_skb(nfp_prog, meta, size); nfp_prog 2868 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx_data_init_pktcache(nfp_prog, meta); nfp_prog 2870 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_ldx_data_from_pktcache(nfp_prog, meta, size); nfp_prog 2872 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_ldx_data(nfp_prog, meta, size); nfp_prog 2877 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_ldx_stack(nfp_prog, meta, size, nfp_prog 2881 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_ldx_emem(nfp_prog, meta, size); nfp_prog 2886 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_ldx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2888 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_ldx(nfp_prog, meta, 1); nfp_prog 2891 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_ldx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2893 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_ldx(nfp_prog, meta, 2); nfp_prog 2896 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2898 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_ldx(nfp_prog, meta, 4); nfp_prog 2901 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_ldx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2903 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_ldx(nfp_prog, meta, 8); nfp_prog 2907 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_st_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 2913 drivers/net/ethernet/netronome/nfp/bpf/jit.c off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); nfp_prog 2915 drivers/net/ethernet/netronome/nfp/bpf/jit.c return data_st_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, nfp_prog 2919 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_st(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 2923 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_st_data(nfp_prog, meta, size); nfp_prog 2928 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_st1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2930 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_st(nfp_prog, meta, 1); nfp_prog 2933 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_st2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2935 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_st(nfp_prog, meta, 2); nfp_prog 2938 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_st4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2940 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_st(nfp_prog, meta, 4); nfp_prog 2943 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_st8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2945 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_st(nfp_prog, meta, 8); nfp_prog 2949 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_stx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 2954 drivers/net/ethernet/netronome/nfp/bpf/jit.c off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); nfp_prog 2956 drivers/net/ethernet/netronome/nfp/bpf/jit.c return data_stx_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, nfp_prog 2961 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 2964 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_op_stack(nfp_prog, meta, size, ptr_off, nfp_prog 2969 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_stx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2973 drivers/net/ethernet/netronome/nfp/bpf/jit.c return nfp_queue_select(nfp_prog, meta); nfp_prog 2981 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 2985 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_stx_data(nfp_prog, meta, size); nfp_prog 2988 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_stx_stack(nfp_prog, meta, size, nfp_prog 2994 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_stx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 2996 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_stx(nfp_prog, meta, 1); nfp_prog 2999 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3001 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_stx(nfp_prog, meta, 2); nfp_prog 3004 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3007 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (nfp_prog->type == BPF_PROG_TYPE_XDP) nfp_prog 3008 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_stx_xdp(nfp_prog, meta); nfp_prog 3009 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_stx(nfp_prog, meta, 4); nfp_prog 3012 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_stx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3014 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_stx(nfp_prog, meta, 8); nfp_prog 3018 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_xadd(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, bool is64) nfp_prog 3025 drivers/net/ethernet/netronome/nfp/bpf/jit.c off = ur_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); nfp_prog 3032 drivers/net/ethernet/netronome/nfp/bpf/jit.c out = nfp_prog_current_offset(nfp_prog); nfp_prog 3033 drivers/net/ethernet/netronome/nfp/bpf/jit.c full_add = nfp_prog_current_offset(nfp_prog); nfp_prog 3052 drivers/net/ethernet/netronome/nfp/bpf/jit.c swreg max_imm = imm_a(nfp_prog); nfp_prog 3054 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, max_imm, 0xffff); nfp_prog 3055 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), nfp_prog 3057 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), nfp_prog 3059 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BLO, full_add, meta->insn.off ? 2 : 0); nfp_prog 3068 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, imma_a(nfp_prog), nfp_prog 3070 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, imma_b(nfp_prog), nfp_prog 3072 drivers/net/ethernet/netronome/nfp/bpf/jit.c addra = imma_a(nfp_prog); nfp_prog 3073 drivers/net/ethernet/netronome/nfp/bpf/jit.c addrb = imma_b(nfp_prog); nfp_prog 3078 drivers/net/ethernet/netronome/nfp/bpf/jit.c swreg prev_alu = imm_a(nfp_prog); nfp_prog 3080 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, prev_alu, nfp_prog 3084 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_or_subpart(nfp_prog, prev_alu, reg_b(src_gpr), 2, 2); nfp_prog 3085 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd_indir(nfp_prog, CMD_TGT_ADD_IMM, CMD_MODE_40b_BA, 0, nfp_prog 3089 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_UNC, out, 0); nfp_prog 3092 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, full_add)) nfp_prog 3097 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd(nfp_prog, CMD_TGT_ADD, CMD_MODE_40b_BA, 0, nfp_prog 3101 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_xfer(0), reg_a(src_gpr)); nfp_prog 3103 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_xfer(1), reg_a(src_gpr + 1)); nfp_prog 3106 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, out)) nfp_prog 3112 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_xadd4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3114 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_xadd(nfp_prog, meta, false); nfp_prog 3117 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_xadd8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3119 drivers/net/ethernet/netronome/nfp/bpf/jit.c return mem_xadd(nfp_prog, meta, true); nfp_prog 3122 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3124 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_UNC, meta->insn.off, 0); nfp_prog 3129 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3139 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); nfp_prog 3140 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, imm_a(nfp_prog), nfp_prog 3142 drivers/net/ethernet/netronome/nfp/bpf/jit.c or1 = imm_a(nfp_prog); nfp_prog 3146 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); nfp_prog 3147 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, imm_b(nfp_prog), nfp_prog 3149 drivers/net/ethernet/netronome/nfp/bpf/jit.c or2 = imm_b(nfp_prog); nfp_prog 3152 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), or1, ALU_OP_OR, or2); nfp_prog 3153 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BEQ, insn->off, 0); nfp_prog 3158 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jeq32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3163 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = ur_load_imm_any(nfp_prog, insn->imm, imm_b(nfp_prog)); nfp_prog 3164 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), nfp_prog 3166 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BEQ, insn->off, 0); nfp_prog 3171 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3178 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); nfp_prog 3179 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, imm_b(nfp_prog), nfp_prog 3185 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), nfp_prog 3186 drivers/net/ethernet/netronome/nfp/bpf/jit.c reg_a(dst_gpr + 1), ALU_OP_OR, imm_b(nfp_prog)); nfp_prog 3188 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BNE, insn->off, 0); nfp_prog 3193 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3202 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_none(), ALU_OP_NONE, nfp_prog 3205 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2), nfp_prog 3207 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BNE, insn->off, 0); nfp_prog 3211 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); nfp_prog 3212 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), nfp_prog 3214 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BNE, insn->off, 0); nfp_prog 3219 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); nfp_prog 3220 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), nfp_prog 3222 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BNE, insn->off, 0); nfp_prog 3227 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3231 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(insn->dst_reg * 2), nfp_prog 3234 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, imm_b(nfp_prog), nfp_prog 3237 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), imm_a(nfp_prog), ALU_OP_OR, nfp_prog 3238 drivers/net/ethernet/netronome/nfp/bpf/jit.c imm_b(nfp_prog)); nfp_prog 3240 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BEQ, insn->off, 0); nfp_prog 3245 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3247 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE); nfp_prog 3250 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3252 drivers/net/ethernet/netronome/nfp/bpf/jit.c return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE); nfp_prog 3256 drivers/net/ethernet/netronome/nfp/bpf/jit.c bpf_to_bpf_call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3261 drivers/net/ethernet/netronome/nfp/bpf/jit.c stack_depth = round_up(nfp_prog->stack_frame_depth, STACK_FRAME_ALIGN); nfp_prog 3266 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = ur_load_imm_any(nfp_prog, stack_depth, nfp_prog 3267 drivers/net/ethernet/netronome/nfp/bpf/jit.c stack_imm(nfp_prog)); nfp_prog 3268 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, stack_reg(nfp_prog), nfp_prog 3269 drivers/net/ethernet/netronome/nfp/bpf/jit.c stack_reg(nfp_prog), ALU_OP_ADD, tmp_reg); nfp_prog 3270 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_csr_wr(nfp_prog, stack_reg(nfp_prog), nfp_prog 3309 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (nfp_prog->subprog[meta->jmp_dst->subprog_idx].needs_reg_push) { nfp_prog 3310 drivers/net/ethernet/netronome/nfp/bpf/jit.c ret_tgt = nfp_prog_current_offset(nfp_prog) + 3; nfp_prog 3311 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, nfp_prog 3313 drivers/net/ethernet/netronome/nfp/bpf/jit.c offset_br = nfp_prog_current_offset(nfp_prog); nfp_prog 3314 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed_relo(nfp_prog, imm_b(nfp_prog), 0, RELO_IMMED_REL); nfp_prog 3316 drivers/net/ethernet/netronome/nfp/bpf/jit.c ret_tgt = nfp_prog_current_offset(nfp_prog) + 2; nfp_prog 3317 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_UNC, meta->insn.imm, 1); nfp_prog 3318 drivers/net/ethernet/netronome/nfp/bpf/jit.c offset_br = nfp_prog_current_offset(nfp_prog); nfp_prog 3320 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed_relo(nfp_prog, ret_reg(nfp_prog), ret_tgt, RELO_IMMED_REL); nfp_prog 3322 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt)) nfp_prog 3326 drivers/net/ethernet/netronome/nfp/bpf/jit.c tmp_reg = ur_load_imm_any(nfp_prog, stack_depth, nfp_prog 3327 drivers/net/ethernet/netronome/nfp/bpf/jit.c stack_imm(nfp_prog)); nfp_prog 3328 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, stack_reg(nfp_prog), nfp_prog 3329 drivers/net/ethernet/netronome/nfp/bpf/jit.c stack_reg(nfp_prog), ALU_OP_SUB, tmp_reg); nfp_prog 3330 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_csr_wr(nfp_prog, stack_reg(nfp_prog), nfp_prog 3332 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_nops(nfp_prog, 3); nfp_prog 3335 drivers/net/ethernet/netronome/nfp/bpf/jit.c meta->num_insns_after_br = nfp_prog_current_offset(nfp_prog); nfp_prog 3341 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int helper_call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3345 drivers/net/ethernet/netronome/nfp/bpf/jit.c return adjust_head(nfp_prog, meta); nfp_prog 3347 drivers/net/ethernet/netronome/nfp/bpf/jit.c return adjust_tail(nfp_prog, meta); nfp_prog 3351 drivers/net/ethernet/netronome/nfp/bpf/jit.c return map_call_stack_common(nfp_prog, meta); nfp_prog 3353 drivers/net/ethernet/netronome/nfp/bpf/jit.c return nfp_get_prandom_u32(nfp_prog, meta); nfp_prog 3355 drivers/net/ethernet/netronome/nfp/bpf/jit.c return nfp_perf_event_output(nfp_prog, meta); nfp_prog 3362 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3365 drivers/net/ethernet/netronome/nfp/bpf/jit.c return bpf_to_bpf_call(nfp_prog, meta); nfp_prog 3367 drivers/net/ethernet/netronome/nfp/bpf/jit.c return helper_call(nfp_prog, meta); nfp_prog 3375 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3377 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 0, RELO_BR_GO_OUT); nfp_prog 3383 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_subprog_epilogue(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3385 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (nfp_prog->subprog[meta->subprog_idx].needs_reg_push) { nfp_prog 3391 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 1, nfp_prog 3394 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, ret_reg(nfp_prog), reg_lm(0, 0)); nfp_prog 3397 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, ret_reg(nfp_prog), reg_lm(0, 0)); nfp_prog 3401 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_rtn(nfp_prog, ret_reg(nfp_prog), 0); nfp_prog 3407 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jmp_exit(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3410 drivers/net/ethernet/netronome/nfp/bpf/jit.c return goto_out(nfp_prog, meta); nfp_prog 3412 drivers/net/ethernet/netronome/nfp/bpf/jit.c return nfp_subprog_epilogue(nfp_prog, meta); nfp_prog 3535 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_fixup_immed_relo(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 3538 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (immed_get_value(nfp_prog->prog[br_idx + 1])) { nfp_prog 3543 drivers/net/ethernet/netronome/nfp/bpf/jit.c immed_set_value(&nfp_prog->prog[br_idx + 1], jmp_dst->off); nfp_prog 3548 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int nfp_fixup_branches(struct nfp_prog *nfp_prog) nfp_prog 3554 drivers/net/ethernet/netronome/nfp/bpf/jit.c list_for_each_entry(meta, &nfp_prog->insns, l) { nfp_prog 3565 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (list_is_last(&meta->l, &nfp_prog->insns)) nfp_prog 3566 drivers/net/ethernet/netronome/nfp/bpf/jit.c br_idx = nfp_prog->last_bpf_off; nfp_prog 3578 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_is_br(nfp_prog->prog[br_idx])) { nfp_prog 3580 drivers/net/ethernet/netronome/nfp/bpf/jit.c br_idx, meta->insn.code, nfp_prog->prog[br_idx]); nfp_prog 3588 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) != nfp_prog 3605 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->subprog[jmp_dst->subprog_idx].needs_reg_push) { nfp_prog 3606 drivers/net/ethernet/netronome/nfp/bpf/jit.c err = nfp_fixup_immed_relo(nfp_prog, meta, nfp_prog 3612 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) != nfp_prog 3617 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_is_br(nfp_prog->prog[idx])) nfp_prog 3619 drivers/net/ethernet/netronome/nfp/bpf/jit.c br_set_offset(&nfp_prog->prog[idx], jmp_dst->off); nfp_prog 3626 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_intro(struct nfp_prog *nfp_prog) nfp_prog 3628 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, plen_reg(nfp_prog), GENMASK(13, 0)); nfp_prog 3629 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, plen_reg(nfp_prog), nfp_prog 3630 drivers/net/ethernet/netronome/nfp/bpf/jit.c plen_reg(nfp_prog), ALU_OP_AND, pv_len(nfp_prog)); nfp_prog 3634 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_subprog_prologue(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3637 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_lm(0, 0), ret_reg(nfp_prog)); nfp_prog 3641 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_start_subprog(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) nfp_prog 3643 drivers/net/ethernet/netronome/nfp/bpf/jit.c unsigned int depth = nfp_prog->subprog[meta->subprog_idx].stack_depth; nfp_prog 3645 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->stack_frame_depth = round_up(depth, 4); nfp_prog 3646 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_subprog_prologue(nfp_prog, meta); nfp_prog 3654 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_outro_tc_da(struct nfp_prog *nfp_prog) nfp_prog 3669 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); nfp_prog 3671 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); nfp_prog 3673 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); nfp_prog 3674 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16); nfp_prog 3677 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); nfp_prog 3680 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_imm(7), ALU_OP_SUB, reg_b(0)); nfp_prog 3681 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0); nfp_prog 3682 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); nfp_prog 3684 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_b(2), 0x41221211); nfp_prog 3685 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_b(3), 0x41001211); nfp_prog 3687 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_a(1), nfp_prog 3690 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); nfp_prog 3691 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_a(2), nfp_prog 3694 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); nfp_prog 3695 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_b(2), nfp_prog 3698 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); nfp_prog 3700 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_b(2), nfp_prog 3702 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16); nfp_prog 3705 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_outro_xdp(struct nfp_prog *nfp_prog) nfp_prog 3715 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); nfp_prog 3717 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); nfp_prog 3719 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); nfp_prog 3720 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16); nfp_prog 3723 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); nfp_prog 3726 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_imm(3), ALU_OP_SUB, reg_b(0)); nfp_prog 3727 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0); nfp_prog 3729 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed(nfp_prog, reg_b(2), 0x44112282); nfp_prog 3731 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_a(1), nfp_prog 3734 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); nfp_prog 3735 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(nfp_prog, reg_b(2), nfp_prog 3738 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); nfp_prog 3740 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); nfp_prog 3741 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16); nfp_prog 3744 drivers/net/ethernet/netronome/nfp/bpf/jit.c static bool nfp_prog_needs_callee_reg_save(struct nfp_prog *nfp_prog) nfp_prog 3748 drivers/net/ethernet/netronome/nfp/bpf/jit.c for (idx = 1; idx < nfp_prog->subprog_cnt; idx++) nfp_prog 3749 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (nfp_prog->subprog[idx].needs_reg_push) nfp_prog 3755 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_push_callee_registers(struct nfp_prog *nfp_prog) nfp_prog 3762 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->tgt_call_push_regs = nfp_prog_current_offset(nfp_prog); nfp_prog 3770 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_lm(0, 1 + idx), reg_b(adj)); nfp_prog 3774 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_rtn(nfp_prog, imm_b(nfp_prog), 3); nfp_prog 3776 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_lm(0, 1 + idx + 1), reg_b(adj + 1)); nfp_prog 3780 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_pop_callee_registers(struct nfp_prog *nfp_prog) nfp_prog 3787 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->tgt_call_pop_regs = nfp_prog_current_offset(nfp_prog); nfp_prog 3795 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_both(adj), reg_lm(0, 1 + idx)); nfp_prog 3799 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_rtn(nfp_prog, ret_reg(nfp_prog), 3); nfp_prog 3801 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mov(nfp_prog, reg_both(adj + 1), reg_lm(0, 1 + idx + 1)); nfp_prog 3805 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_outro(struct nfp_prog *nfp_prog) nfp_prog 3807 drivers/net/ethernet/netronome/nfp/bpf/jit.c switch (nfp_prog->type) { nfp_prog 3809 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_outro_tc_da(nfp_prog); nfp_prog 3812 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_outro_xdp(nfp_prog); nfp_prog 3818 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog_needs_callee_reg_save(nfp_prog)) nfp_prog 3821 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_push_callee_registers(nfp_prog); nfp_prog 3822 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_pop_callee_registers(nfp_prog); nfp_prog 3825 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int nfp_translate(struct nfp_prog *nfp_prog) nfp_prog 3831 drivers/net/ethernet/netronome/nfp/bpf/jit.c depth = nfp_prog->subprog[0].stack_depth; nfp_prog 3832 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->stack_frame_depth = round_up(depth, 4); nfp_prog 3834 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_intro(nfp_prog); nfp_prog 3835 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (nfp_prog->error) nfp_prog 3836 drivers/net/ethernet/netronome/nfp/bpf/jit.c return nfp_prog->error; nfp_prog 3838 drivers/net/ethernet/netronome/nfp/bpf/jit.c list_for_each_entry(meta, &nfp_prog->insns, l) { nfp_prog 3841 drivers/net/ethernet/netronome/nfp/bpf/jit.c meta->off = nfp_prog_current_offset(nfp_prog); nfp_prog 3844 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_start_subprog(nfp_prog, meta); nfp_prog 3845 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (nfp_prog->error) nfp_prog 3846 drivers/net/ethernet/netronome/nfp/bpf/jit.c return nfp_prog->error; nfp_prog 3850 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->n_translated++; nfp_prog 3854 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (nfp_meta_has_prev(nfp_prog, meta) && nfp_prog 3859 drivers/net/ethernet/netronome/nfp/bpf/jit.c err = cb(nfp_prog, meta); nfp_prog 3862 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (nfp_prog->error) nfp_prog 3863 drivers/net/ethernet/netronome/nfp/bpf/jit.c return nfp_prog->error; nfp_prog 3865 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->n_translated++; nfp_prog 3868 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->last_bpf_off = nfp_prog_current_offset(nfp_prog) - 1; nfp_prog 3870 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_outro(nfp_prog); nfp_prog 3871 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (nfp_prog->error) nfp_prog 3872 drivers/net/ethernet/netronome/nfp/bpf/jit.c return nfp_prog->error; nfp_prog 3874 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_nops(nfp_prog, NFP_USTORE_PREFETCH_WINDOW); nfp_prog 3875 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (nfp_prog->error) nfp_prog 3876 drivers/net/ethernet/netronome/nfp/bpf/jit.c return nfp_prog->error; nfp_prog 3878 drivers/net/ethernet/netronome/nfp/bpf/jit.c return nfp_fixup_branches(nfp_prog); nfp_prog 3882 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog) nfp_prog 3886 drivers/net/ethernet/netronome/nfp/bpf/jit.c list_for_each_entry(meta, &nfp_prog->insns, l) { nfp_prog 3908 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_bpf_opt_neg_add_sub(struct nfp_prog *nfp_prog) nfp_prog 3912 drivers/net/ethernet/netronome/nfp/bpf/jit.c list_for_each_entry(meta, &nfp_prog->insns, l) { nfp_prog 3952 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog) nfp_prog 3961 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { nfp_prog 3991 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_bpf_opt_ld_shift(struct nfp_prog *nfp_prog) nfp_prog 3995 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_for_each_insn_walk3(nfp_prog, meta1, meta2, meta3) { nfp_prog 4180 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_bpf_opt_ldst_gather(struct nfp_prog *nfp_prog) nfp_prog 4189 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { nfp_prog 4270 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_bpf_opt_pkt_cache(struct nfp_prog *nfp_prog) nfp_prog 4279 drivers/net/ethernet/netronome/nfp/bpf/jit.c list_for_each_entry(meta, &nfp_prog->insns, l) { nfp_prog 4368 drivers/net/ethernet/netronome/nfp/bpf/jit.c list_for_each_entry(meta, &nfp_prog->insns, l) { nfp_prog 4384 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int nfp_bpf_optimize(struct nfp_prog *nfp_prog) nfp_prog 4386 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_bpf_opt_reg_init(nfp_prog); nfp_prog 4388 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_bpf_opt_neg_add_sub(nfp_prog); nfp_prog 4389 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_bpf_opt_ld_mask(nfp_prog); nfp_prog 4390 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_bpf_opt_ld_shift(nfp_prog); nfp_prog 4391 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_bpf_opt_ldst_gather(nfp_prog); nfp_prog 4392 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_bpf_opt_pkt_cache(nfp_prog); nfp_prog 4397 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int nfp_bpf_replace_map_ptrs(struct nfp_prog *nfp_prog) nfp_prog 4404 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { nfp_prog 4447 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_bpf_prog_trim(struct nfp_prog *nfp_prog) nfp_prog 4451 drivers/net/ethernet/netronome/nfp/bpf/jit.c prog = kvmalloc_array(nfp_prog->prog_len, sizeof(u64), GFP_KERNEL); nfp_prog 4455 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->__prog_alloc_len = nfp_prog->prog_len * sizeof(u64); nfp_prog 4456 drivers/net/ethernet/netronome/nfp/bpf/jit.c memcpy(prog, nfp_prog->prog, nfp_prog->__prog_alloc_len); nfp_prog 4457 drivers/net/ethernet/netronome/nfp/bpf/jit.c kvfree(nfp_prog->prog); nfp_prog 4458 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->prog = prog; nfp_prog 4461 drivers/net/ethernet/netronome/nfp/bpf/jit.c int nfp_bpf_jit(struct nfp_prog *nfp_prog) nfp_prog 4465 drivers/net/ethernet/netronome/nfp/bpf/jit.c ret = nfp_bpf_replace_map_ptrs(nfp_prog); nfp_prog 4469 drivers/net/ethernet/netronome/nfp/bpf/jit.c ret = nfp_bpf_optimize(nfp_prog); nfp_prog 4473 drivers/net/ethernet/netronome/nfp/bpf/jit.c ret = nfp_translate(nfp_prog); nfp_prog 4476 drivers/net/ethernet/netronome/nfp/bpf/jit.c ret, nfp_prog->n_translated); nfp_prog 4480 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_bpf_prog_trim(nfp_prog); nfp_prog 4485 drivers/net/ethernet/netronome/nfp/bpf/jit.c void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog) nfp_prog 4490 drivers/net/ethernet/netronome/nfp/bpf/jit.c list_for_each_entry(meta, &nfp_prog->insns, l) { nfp_prog 4513 drivers/net/ethernet/netronome/nfp/bpf/jit.c dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_idx); nfp_prog 4528 drivers/net/ethernet/netronome/nfp/bpf/jit.c void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv) nfp_prog 4534 drivers/net/ethernet/netronome/nfp/bpf/jit.c prog = kmemdup(nfp_prog->prog, nfp_prog->prog_len * sizeof(u64), nfp_prog 4539 drivers/net/ethernet/netronome/nfp/bpf/jit.c for (i = 0; i < nfp_prog->prog_len; i++) { nfp_prog 4553 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->tgt_out + bv->start_off); nfp_prog 4557 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog->tgt_abort + bv->start_off); nfp_prog 4560 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog->tgt_call_push_regs) { nfp_prog 4565 drivers/net/ethernet/netronome/nfp/bpf/jit.c off = nfp_prog->tgt_call_push_regs + bv->start_off; nfp_prog 4569 drivers/net/ethernet/netronome/nfp/bpf/jit.c if (!nfp_prog->tgt_call_pop_regs) { nfp_prog 4574 drivers/net/ethernet/netronome/nfp/bpf/jit.c off = nfp_prog->tgt_call_pop_regs + bv->start_off; nfp_prog 4585 drivers/net/ethernet/netronome/nfp/bpf/jit.c val = nfp_prog->bpf->helpers.map_lookup; nfp_prog 4588 drivers/net/ethernet/netronome/nfp/bpf/jit.c val = nfp_prog->bpf->helpers.map_update; nfp_prog 4591 drivers/net/ethernet/netronome/nfp/bpf/jit.c val = nfp_prog->bpf->helpers.map_delete; nfp_prog 4594 drivers/net/ethernet/netronome/nfp/bpf/jit.c val = nfp_prog->bpf->helpers.perf_event_output; nfp_prog 4612 drivers/net/ethernet/netronome/nfp/bpf/jit.c err = nfp_bpf_ustore_calc(prog, nfp_prog->prog_len); nfp_prog 233 drivers/net/ethernet/netronome/nfp/bpf/main.h struct nfp_prog; nfp_prog 235 drivers/net/ethernet/netronome/nfp/bpf/main.h typedef int (*instr_cb_t)(struct nfp_prog *, struct nfp_insn_meta *); nfp_prog 237 drivers/net/ethernet/netronome/nfp/bpf/main.h #define nfp_prog_first_meta(nfp_prog) \ nfp_prog 238 drivers/net/ethernet/netronome/nfp/bpf/main.h list_first_entry(&(nfp_prog)->insns, struct nfp_insn_meta, l) nfp_prog 239 drivers/net/ethernet/netronome/nfp/bpf/main.h #define nfp_prog_last_meta(nfp_prog) \ nfp_prog 240 drivers/net/ethernet/netronome/nfp/bpf/main.h list_last_entry(&(nfp_prog)->insns, struct nfp_insn_meta, l) nfp_prog 560 drivers/net/ethernet/netronome/nfp/bpf/main.h void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog); nfp_prog 561 drivers/net/ethernet/netronome/nfp/bpf/main.h int nfp_bpf_jit(struct nfp_prog *prog); nfp_prog 584 drivers/net/ethernet/netronome/nfp/bpf/main.h nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 587 drivers/net/ethernet/netronome/nfp/bpf/main.h void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv); nfp_prog 31 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_map_ptr_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, nfp_prog 41 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog->map_records[nfp_prog->map_records_cnt++] = record; nfp_prog 68 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog->map_records[nfp_prog->map_records_cnt++] = record; nfp_prog 80 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_map_ptrs_forget(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog) nfp_prog 85 drivers/net/ethernet/netronome/nfp/bpf/offload.c for (i = 0; i < nfp_prog->map_records_cnt; i++) { nfp_prog 86 drivers/net/ethernet/netronome/nfp/bpf/offload.c if (--nfp_prog->map_records[i]->count) { nfp_prog 87 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog->map_records[i] = NULL; nfp_prog 92 drivers/net/ethernet/netronome/nfp/bpf/offload.c &nfp_prog->map_records[i]->l, nfp_prog 100 drivers/net/ethernet/netronome/nfp/bpf/offload.c for (i = 0; i < nfp_prog->map_records_cnt; i++) nfp_prog 101 drivers/net/ethernet/netronome/nfp/bpf/offload.c if (nfp_prog->map_records[i]) { nfp_prog 102 drivers/net/ethernet/netronome/nfp/bpf/offload.c bpf_map_put(nfp_prog->map_records[i]->ptr); nfp_prog 103 drivers/net/ethernet/netronome/nfp/bpf/offload.c kfree(nfp_prog->map_records[i]); nfp_prog 107 drivers/net/ethernet/netronome/nfp/bpf/offload.c kfree(nfp_prog->map_records); nfp_prog 108 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog->map_records = NULL; nfp_prog 109 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog->map_records_cnt = 0; nfp_prog 113 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_map_ptrs_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, nfp_prog 126 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog->map_records = kmalloc_array(cnt, nfp_prog 127 drivers/net/ethernet/netronome/nfp/bpf/offload.c sizeof(nfp_prog->map_records[0]), nfp_prog 129 drivers/net/ethernet/netronome/nfp/bpf/offload.c if (!nfp_prog->map_records) nfp_prog 134 drivers/net/ethernet/netronome/nfp/bpf/offload.c err = nfp_map_ptr_record(bpf, nfp_prog, nfp_prog 137 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_map_ptrs_forget(bpf, nfp_prog); nfp_prog 141 drivers/net/ethernet/netronome/nfp/bpf/offload.c WARN_ON(cnt != nfp_prog->map_records_cnt); nfp_prog 147 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog, nfp_prog 165 drivers/net/ethernet/netronome/nfp/bpf/offload.c list_add_tail(&meta->l, &nfp_prog->insns); nfp_prog 167 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog->n_insns = cnt; nfp_prog 169 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_bpf_jit_prepare(nfp_prog); nfp_prog 174 drivers/net/ethernet/netronome/nfp/bpf/offload.c static void nfp_prog_free(struct nfp_prog *nfp_prog) nfp_prog 178 drivers/net/ethernet/netronome/nfp/bpf/offload.c kfree(nfp_prog->subprog); nfp_prog 180 drivers/net/ethernet/netronome/nfp/bpf/offload.c list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) { nfp_prog 184 drivers/net/ethernet/netronome/nfp/bpf/offload.c kfree(nfp_prog); nfp_prog 189 drivers/net/ethernet/netronome/nfp/bpf/offload.c struct nfp_prog *nfp_prog; nfp_prog 192 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL); nfp_prog 193 drivers/net/ethernet/netronome/nfp/bpf/offload.c if (!nfp_prog) nfp_prog 195 drivers/net/ethernet/netronome/nfp/bpf/offload.c prog->aux->offload->dev_priv = nfp_prog; nfp_prog 197 drivers/net/ethernet/netronome/nfp/bpf/offload.c INIT_LIST_HEAD(&nfp_prog->insns); nfp_prog 198 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog->type = prog->type; nfp_prog 199 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog->bpf = bpf_offload_dev_priv(prog->aux->offload->offdev); nfp_prog 201 drivers/net/ethernet/netronome/nfp/bpf/offload.c ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len); nfp_prog 205 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog); nfp_prog 210 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog_free(nfp_prog); nfp_prog 218 drivers/net/ethernet/netronome/nfp/bpf/offload.c struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; nfp_prog 227 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog->__prog_alloc_len = max_instr * sizeof(u64); nfp_prog 229 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL); nfp_prog 230 drivers/net/ethernet/netronome/nfp/bpf/offload.c if (!nfp_prog->prog) nfp_prog 233 drivers/net/ethernet/netronome/nfp/bpf/offload.c err = nfp_bpf_jit(nfp_prog); nfp_prog 237 drivers/net/ethernet/netronome/nfp/bpf/offload.c prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64); nfp_prog 238 drivers/net/ethernet/netronome/nfp/bpf/offload.c prog->aux->offload->jited_image = nfp_prog->prog; nfp_prog 240 drivers/net/ethernet/netronome/nfp/bpf/offload.c return nfp_map_ptrs_record(nfp_prog->bpf, nfp_prog, prog); nfp_prog 245 drivers/net/ethernet/netronome/nfp/bpf/offload.c struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; nfp_prog 247 drivers/net/ethernet/netronome/nfp/bpf/offload.c kvfree(nfp_prog->prog); nfp_prog 248 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_map_ptrs_forget(nfp_prog->bpf, nfp_prog); nfp_prog 249 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog_free(nfp_prog); nfp_prog 484 drivers/net/ethernet/netronome/nfp/bpf/offload.c struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; nfp_prog 498 drivers/net/ethernet/netronome/nfp/bpf/offload.c if (nfp_prog->stack_size > max_stack) { nfp_prog 504 drivers/net/ethernet/netronome/nfp/bpf/offload.c if (nfp_prog->prog_len > max_prog_len) { nfp_prog 509 drivers/net/ethernet/netronome/nfp/bpf/offload.c img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv); nfp_prog 514 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog->prog_len * sizeof(u64), nfp_prog 521 drivers/net/ethernet/netronome/nfp/bpf/offload.c nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len); nfp_prog 530 drivers/net/ethernet/netronome/nfp/bpf/offload.c dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64), nfp_prog 20 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 28 drivers/net/ethernet/netronome/nfp/bpf/verifier.c if (min(forward, backward) > nfp_prog->n_insns - insn_idx - 1) { nfp_prog 29 drivers/net/ethernet/netronome/nfp/bpf/verifier.c backward = nfp_prog->n_insns - insn_idx - 1; nfp_prog 30 drivers/net/ethernet/netronome/nfp/bpf/verifier.c meta = nfp_prog_last_meta(nfp_prog); nfp_prog 34 drivers/net/ethernet/netronome/nfp/bpf/verifier.c meta = nfp_prog_first_meta(nfp_prog); nfp_prog 48 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_record_adjust_head(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, nfp_prog 69 drivers/net/ethernet/netronome/nfp/bpf/verifier.c if (nfp_prog->adjust_head_location) { nfp_prog 71 drivers/net/ethernet/netronome/nfp/bpf/verifier.c if (nfp_prog->adjust_head_location != meta->n) nfp_prog 80 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_prog->adjust_head_location = location; nfp_prog 170 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_check_helper_call(struct nfp_prog *nfp_prog, nfp_prog 177 drivers/net/ethernet/netronome/nfp/bpf/verifier.c struct nfp_app_bpf *bpf = nfp_prog->bpf; nfp_prog 191 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_record_adjust_head(bpf, nfp_prog, meta, reg2); nfp_prog 284 drivers/net/ethernet/netronome/nfp/bpf/verifier.c dev_warn_once(&nfp_prog->bpf->app->pf->pdev->dev, nfp_prog 311 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_check_exit(struct nfp_prog *nfp_prog, nfp_prog 317 drivers/net/ethernet/netronome/nfp/bpf/verifier.c if (nfp_prog->type == BPF_PROG_TYPE_XDP) nfp_prog 330 drivers/net/ethernet/netronome/nfp/bpf/verifier.c if (nfp_prog->type == BPF_PROG_TYPE_SCHED_CLS && nfp_prog 343 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_check_stack_access(struct nfp_prog *nfp_prog, nfp_prog 451 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_check_ptr(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 466 drivers/net/ethernet/netronome/nfp/bpf/verifier.c err = nfp_bpf_check_stack_access(nfp_prog, meta, reg, env); nfp_prog 502 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_check_store(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 508 drivers/net/ethernet/netronome/nfp/bpf/verifier.c if (nfp_prog->type == BPF_PROG_TYPE_XDP) { nfp_prog 512 drivers/net/ethernet/netronome/nfp/bpf/verifier.c if (nfp_prog->bpf->queue_select) nfp_prog 522 drivers/net/ethernet/netronome/nfp/bpf/verifier.c return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.dst_reg); nfp_prog 526 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_check_xadd(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 547 drivers/net/ethernet/netronome/nfp/bpf/verifier.c return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.dst_reg); nfp_prog 551 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_check_alu(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, nfp_prog 629 drivers/net/ethernet/netronome/nfp/bpf/verifier.c struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv; nfp_prog 630 drivers/net/ethernet/netronome/nfp/bpf/verifier.c struct nfp_insn_meta *meta = nfp_prog->verifier_meta; nfp_prog 632 drivers/net/ethernet/netronome/nfp/bpf/verifier.c meta = nfp_bpf_goto_meta(nfp_prog, meta, insn_idx); nfp_prog 633 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_prog->verifier_meta = meta; nfp_prog 648 drivers/net/ethernet/netronome/nfp/bpf/verifier.c return nfp_bpf_check_helper_call(nfp_prog, env, meta); nfp_prog 650 drivers/net/ethernet/netronome/nfp/bpf/verifier.c return nfp_bpf_check_exit(nfp_prog, env); nfp_prog 653 drivers/net/ethernet/netronome/nfp/bpf/verifier.c return nfp_bpf_check_ptr(nfp_prog, meta, env, nfp_prog 656 drivers/net/ethernet/netronome/nfp/bpf/verifier.c return nfp_bpf_check_store(nfp_prog, meta, env); nfp_prog 659 drivers/net/ethernet/netronome/nfp/bpf/verifier.c return nfp_bpf_check_xadd(nfp_prog, meta, env); nfp_prog 662 drivers/net/ethernet/netronome/nfp/bpf/verifier.c return nfp_bpf_check_alu(nfp_prog, meta, env); nfp_prog 669 drivers/net/ethernet/netronome/nfp/bpf/verifier.c struct nfp_prog *nfp_prog) nfp_prog 674 drivers/net/ethernet/netronome/nfp/bpf/verifier.c list_for_each_entry(meta, &nfp_prog->insns, l) { nfp_prog 681 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_prog->subprog[index].needs_reg_push = 1; nfp_prog 684 drivers/net/ethernet/netronome/nfp/bpf/verifier.c if (index + 1 != nfp_prog->subprog_cnt) { nfp_prog 686 drivers/net/ethernet/netronome/nfp/bpf/verifier.c index + 1, nfp_prog->subprog_cnt); nfp_prog 693 drivers/net/ethernet/netronome/nfp/bpf/verifier.c static unsigned int nfp_bpf_get_stack_usage(struct nfp_prog *nfp_prog) nfp_prog 695 drivers/net/ethernet/netronome/nfp/bpf/verifier.c struct nfp_insn_meta *meta = nfp_prog_first_meta(nfp_prog); nfp_prog 709 drivers/net/ethernet/netronome/nfp/bpf/verifier.c frame_depths[frame] = nfp_prog->subprog[idx].stack_depth; nfp_prog 715 drivers/net/ethernet/netronome/nfp/bpf/verifier.c for (; meta != nfp_prog_last_meta(nfp_prog) && meta->subprog_idx == idx; nfp_prog 727 drivers/net/ethernet/netronome/nfp/bpf/verifier.c meta = nfp_bpf_goto_meta(nfp_prog, meta, nfp_prog 747 drivers/net/ethernet/netronome/nfp/bpf/verifier.c static void nfp_bpf_insn_flag_zext(struct nfp_prog *nfp_prog, nfp_prog 752 drivers/net/ethernet/netronome/nfp/bpf/verifier.c list_for_each_entry(meta, &nfp_prog->insns, l) { nfp_prog 761 drivers/net/ethernet/netronome/nfp/bpf/verifier.c struct nfp_prog *nfp_prog; nfp_prog 766 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_prog = env->prog->aux->offload->dev_priv; nfp_prog 767 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_prog->subprog_cnt = env->subprog_cnt; nfp_prog 768 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_prog->subprog = kcalloc(nfp_prog->subprog_cnt, nfp_prog 769 drivers/net/ethernet/netronome/nfp/bpf/verifier.c sizeof(nfp_prog->subprog[0]), GFP_KERNEL); nfp_prog 770 drivers/net/ethernet/netronome/nfp/bpf/verifier.c if (!nfp_prog->subprog) nfp_prog 773 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_assign_subprog_idx_and_regs(env, nfp_prog); nfp_prog 776 drivers/net/ethernet/netronome/nfp/bpf/verifier.c for (i = 0; i < nfp_prog->subprog_cnt; i++) { nfp_prog 777 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_prog->subprog[i].stack_depth = info[i].stack_depth; nfp_prog 783 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_prog->subprog[i].stack_depth += REG_WIDTH; nfp_prog 785 drivers/net/ethernet/netronome/nfp/bpf/verifier.c if (nfp_prog->subprog[i].needs_reg_push) nfp_prog 786 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_prog->subprog[i].stack_depth += BPF_REG_SIZE * 4; nfp_prog 791 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_prog->stack_size = nfp_bpf_get_stack_usage(nfp_prog); nfp_prog 792 drivers/net/ethernet/netronome/nfp/bpf/verifier.c if (nfp_prog->stack_size > max_stack) { nfp_prog 794 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_prog->stack_size, max_stack); nfp_prog 798 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_insn_flag_zext(nfp_prog, env->insn_aux_data); nfp_prog 805 drivers/net/ethernet/netronome/nfp/bpf/verifier.c struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv; nfp_prog 807 drivers/net/ethernet/netronome/nfp/bpf/verifier.c struct nfp_insn_meta *meta = nfp_prog->verifier_meta; nfp_prog 809 drivers/net/ethernet/netronome/nfp/bpf/verifier.c meta = nfp_bpf_goto_meta(nfp_prog, meta, aux_data[off].orig_idx); nfp_prog 810 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_prog->verifier_meta = meta; nfp_prog 838 drivers/net/ethernet/netronome/nfp/bpf/verifier.c struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv; nfp_prog 840 drivers/net/ethernet/netronome/nfp/bpf/verifier.c struct nfp_insn_meta *meta = nfp_prog->verifier_meta; nfp_prog 843 drivers/net/ethernet/netronome/nfp/bpf/verifier.c meta = nfp_bpf_goto_meta(nfp_prog, meta, aux_data[off].orig_idx); nfp_prog 846 drivers/net/ethernet/netronome/nfp/bpf/verifier.c if (WARN_ON_ONCE(&meta->l == &nfp_prog->insns))