nfp_prog           22 drivers/net/ethernet/netronome/nfp/bpf/jit.c #define nfp_for_each_insn_walk2(nfp_prog, pos, next)			\
nfp_prog           23 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
nfp_prog           25 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	     &(nfp_prog)->insns != &pos->l &&			\
nfp_prog           26 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	     &(nfp_prog)->insns != &next->l;			\
nfp_prog           30 drivers/net/ethernet/netronome/nfp/bpf/jit.c #define nfp_for_each_insn_walk3(nfp_prog, pos, next, next2)		\
nfp_prog           31 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
nfp_prog           34 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	     &(nfp_prog)->insns != &pos->l &&			\
nfp_prog           35 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	     &(nfp_prog)->insns != &next->l &&			\
nfp_prog           36 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	     &(nfp_prog)->insns != &next2->l;			\
nfp_prog           42 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog           44 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return meta->l.prev != &nfp_prog->insns;
nfp_prog           47 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn)
nfp_prog           49 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (nfp_prog->__prog_alloc_len / sizeof(u64) == nfp_prog->prog_len) {
nfp_prog           51 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			nfp_prog->prog_len);
nfp_prog           52 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = -ENOSPC;
nfp_prog           56 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->prog[nfp_prog->prog_len] = insn;
nfp_prog           57 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->prog_len++;
nfp_prog           60 drivers/net/ethernet/netronome/nfp/bpf/jit.c static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog)
nfp_prog           62 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return nfp_prog->prog_len;
nfp_prog           66 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_prog_confirm_current_offset(struct nfp_prog *nfp_prog, unsigned int off)
nfp_prog           72 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (nfp_prog->error)
nfp_prog           74 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return !WARN_ON_ONCE(nfp_prog_current_offset(nfp_prog) != off);
nfp_prog           79 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
nfp_prog           96 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog_push(nfp_prog, insn);
nfp_prog          100 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd_any(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer,
nfp_prog          108 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = err;
nfp_prog          113 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = -EFAULT;
nfp_prog          118 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = -EFAULT;
nfp_prog          122 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	__emit_cmd(nfp_prog, op, mode, xfer, reg.areg, reg.breg, size, ctx,
nfp_prog          127 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer,
nfp_prog          130 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, ctx, false);
nfp_prog          134 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_cmd_indir(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer,
nfp_prog          137 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, ctx, true);
nfp_prog          141 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip,
nfp_prog          158 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog_push(nfp_prog, insn);
nfp_prog          162 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_relo(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer,
nfp_prog          167 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = -EFAULT;
nfp_prog          171 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	__emit_br(nfp_prog, mask,
nfp_prog          175 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->prog[nfp_prog->prog_len - 1] |=
nfp_prog          180 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer)
nfp_prog          182 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br_relo(nfp_prog, mask, addr, defer, RELO_BR_REL);
nfp_prog          186 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_br_bit(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 addr, u8 defer,
nfp_prog          204 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog_push(nfp_prog, insn);
nfp_prog          208 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_bit_relo(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr,
nfp_prog          222 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = err;
nfp_prog          226 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	__emit_br_bit(nfp_prog, reg.areg, reg.breg, addr, defer, set,
nfp_prog          229 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->prog[nfp_prog->prog_len - 1] |=
nfp_prog          234 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_br_bset(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr, u8 defer)
nfp_prog          236 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br_bit_relo(nfp_prog, src, bit, addr, defer, true, RELO_BR_REL);
nfp_prog          240 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_br_alu(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi,
nfp_prog          253 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog_push(nfp_prog, insn);
nfp_prog          256 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void emit_rtn(struct nfp_prog *nfp_prog, swreg base, u8 defer)
nfp_prog          263 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = err;
nfp_prog          267 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	__emit_br_alu(nfp_prog, reg.areg, reg.breg, 0, defer, reg.dst_lmextn,
nfp_prog          272 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi,
nfp_prog          290 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog_push(nfp_prog, insn);
nfp_prog          294 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_immed(struct nfp_prog *nfp_prog, swreg dst, u16 imm,
nfp_prog          301 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = -EFAULT;
nfp_prog          307 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = err;
nfp_prog          312 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	__emit_immed(nfp_prog,
nfp_prog          319 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_shf(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab,
nfp_prog          327 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = -EFAULT;
nfp_prog          359 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog_push(nfp_prog, insn);
nfp_prog          363 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf(struct nfp_prog *nfp_prog, swreg dst,
nfp_prog          371 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = err;
nfp_prog          375 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	__emit_shf(nfp_prog, reg.dst, reg.dst_ab, sc, shift,
nfp_prog          381 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_shf_indir(struct nfp_prog *nfp_prog, swreg dst,
nfp_prog          386 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = -EFAULT;
nfp_prog          390 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf(nfp_prog, dst, lreg, op, rreg, sc, 0);
nfp_prog          394 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_alu(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab,
nfp_prog          411 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog_push(nfp_prog, insn);
nfp_prog          415 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_alu(struct nfp_prog *nfp_prog, swreg dst,
nfp_prog          423 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = err;
nfp_prog          427 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	__emit_alu(nfp_prog, reg.dst, reg.dst_ab,
nfp_prog          433 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_mul(struct nfp_prog *nfp_prog, enum alu_dst_ab dst_ab, u16 areg,
nfp_prog          450 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog_push(nfp_prog, insn);
nfp_prog          454 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_mul(struct nfp_prog *nfp_prog, swreg lreg, enum mul_type type,
nfp_prog          462 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = -EINVAL;
nfp_prog          478 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = err;
nfp_prog          482 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	__emit_mul(nfp_prog, reg.dst_ab, areg, type, step, reg.breg, reg.swap,
nfp_prog          487 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_ld_field(struct nfp_prog *nfp_prog, enum shf_sc sc,
nfp_prog          507 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog_push(nfp_prog, insn);
nfp_prog          511 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field_any(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src,
nfp_prog          520 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = err;
nfp_prog          524 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	__emit_ld_field(nfp_prog, sc, reg.areg, bmask, reg.breg, shift,
nfp_prog          530 drivers/net/ethernet/netronome/nfp/bpf/jit.c emit_ld_field(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src,
nfp_prog          533 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_ld_field_any(nfp_prog, dst, bmask, src, sc, shift, false);
nfp_prog          537 drivers/net/ethernet/netronome/nfp/bpf/jit.c __emit_lcsr(struct nfp_prog *nfp_prog, u16 areg, u16 breg, bool wr, u16 addr,
nfp_prog          550 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog_push(nfp_prog, insn);
nfp_prog          553 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void emit_csr_wr(struct nfp_prog *nfp_prog, swreg src, u16 addr)
nfp_prog          570 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = err;
nfp_prog          574 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	__emit_lcsr(nfp_prog, reg.areg, reg.breg, true, addr,
nfp_prog          579 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void __emit_csr_rd(struct nfp_prog *nfp_prog, u16 addr)
nfp_prog          581 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	__emit_lcsr(nfp_prog, 0, 0, false, addr, false, false);
nfp_prog          584 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void emit_nop(struct nfp_prog *nfp_prog)
nfp_prog          586 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	__emit_immed(nfp_prog, UR_REG_IMM, UR_REG_IMM, 0, 0, 0, 0, 0, 0, 0);
nfp_prog          608 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void wrp_immed(struct nfp_prog *nfp_prog, swreg dst, u32 imm)
nfp_prog          614 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, false, shift);
nfp_prog          616 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, true, shift);
nfp_prog          618 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_immed(nfp_prog, dst, imm & 0xffff, IMMED_WIDTH_ALL,
nfp_prog          620 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_immed(nfp_prog, dst, imm >> 16, IMMED_WIDTH_WORD,
nfp_prog          626 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_zext(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst)
nfp_prog          629 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_both(dst + 1), 0);
nfp_prog          633 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_immed_relo(struct nfp_prog *nfp_prog, swreg dst, u32 imm,
nfp_prog          638 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->error = -EFAULT;
nfp_prog          641 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_immed(nfp_prog, dst, imm, IMMED_WIDTH_ALL, false, IMMED_SHIFT_0B);
nfp_prog          643 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->prog[nfp_prog->prog_len - 1] |=
nfp_prog          651 drivers/net/ethernet/netronome/nfp/bpf/jit.c static swreg ur_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg)
nfp_prog          656 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, tmp_reg, imm);
nfp_prog          664 drivers/net/ethernet/netronome/nfp/bpf/jit.c static swreg re_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg)
nfp_prog          669 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, tmp_reg, imm);
nfp_prog          673 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void wrp_nops(struct nfp_prog *nfp_prog, unsigned int count)
nfp_prog          676 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_nop(nfp_prog);
nfp_prog          679 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src)
nfp_prog          681 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src);
nfp_prog          684 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void wrp_reg_mov(struct nfp_prog *nfp_prog, u16 dst, u16 src)
nfp_prog          686 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_mov(nfp_prog, reg_both(dst), reg_b(src));
nfp_prog          693 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_subpart(struct nfp_prog *nfp_prog, swreg dst, swreg src, u8 field_len,
nfp_prog          699 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_ld_field_any(nfp_prog, dst, mask, src, sc, offset * 8, true);
nfp_prog          706 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_reg_or_subpart(struct nfp_prog *nfp_prog, swreg dst, swreg src,
nfp_prog          712 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_ld_field(nfp_prog, dst, mask, src, sc, 32 - offset * 8);
nfp_prog          716 drivers/net/ethernet/netronome/nfp/bpf/jit.c addr40_offset(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset,
nfp_prog          725 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(src_gpr), ALU_OP_ADD, offset);
nfp_prog          726 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, imm_b(nfp_prog), reg_b(src_gpr + 1), ALU_OP_ADD_C,
nfp_prog          728 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	*rega = imm_a(nfp_prog);
nfp_prog          729 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	*regb = imm_b(nfp_prog);
nfp_prog          733 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int nfp_cpp_memcpy(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog          742 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	off = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
nfp_prog          748 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		addr40_offset(nfp_prog, meta->insn.src_reg * 2, off, &src_base,
nfp_prog          753 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_none(),
nfp_prog          757 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP,
nfp_prog          763 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, reg_xfer(i), reg_xfer(i));
nfp_prog          765 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	off = re_load_imm_any(nfp_prog, meta->paired_st->off, imm_b(nfp_prog));
nfp_prog          769 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
nfp_prog          774 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
nfp_prog          779 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_none(),
nfp_prog          781 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_cmd_indir(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
nfp_prog          786 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_none(),
nfp_prog          788 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
nfp_prog          795 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
nfp_prog          799 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		off = re_load_imm_any(nfp_prog, meta->paired_st->off + 32,
nfp_prog          800 drivers/net/ethernet/netronome/nfp/bpf/jit.c 				      imm_b(nfp_prog));
nfp_prog          801 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 8,
nfp_prog          810 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_none(),
nfp_prog          812 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
nfp_prog          816 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		off = re_load_imm_any(nfp_prog, new_off, imm_b(nfp_prog));
nfp_prog          817 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b,
nfp_prog          841 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2),
nfp_prog          846 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2),
nfp_prog          850 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2),
nfp_prog          854 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2),
nfp_prog          856 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1),
nfp_prog          862 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
nfp_prog          868 drivers/net/ethernet/netronome/nfp/bpf/jit.c data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, swreg offset,
nfp_prog          880 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_cmd(nfp_prog, CMD_TGT_READ8, CMD_MODE_32b, 0,
nfp_prog          881 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		 pptr_reg(nfp_prog), offset, sz - 1, CMD_CTX_SWAP);
nfp_prog          885 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, reg_both(dst_gpr), reg_none(), SHF_OP_NONE,
nfp_prog          889 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i));
nfp_prog          892 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_zext(nfp_prog, meta, dst_gpr);
nfp_prog          898 drivers/net/ethernet/netronome/nfp/bpf/jit.c data_ld_host_order(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog          911 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_cmd(nfp_prog, CMD_TGT_READ32_SWAP, mode, 0,
nfp_prog          916 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_ld_field_any(nfp_prog, reg_both(dst_gpr), mask,
nfp_prog          920 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i));
nfp_prog          923 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_zext(nfp_prog, meta, dst_gpr);
nfp_prog          929 drivers/net/ethernet/netronome/nfp/bpf/jit.c data_ld_host_order_addr32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog          932 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return data_ld_host_order(nfp_prog, meta, dst_gpr, reg_a(src_gpr),
nfp_prog          937 drivers/net/ethernet/netronome/nfp/bpf/jit.c data_ld_host_order_addr40(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog          942 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	addr40_offset(nfp_prog, src_gpr, offset, &rega, &regb);
nfp_prog          944 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return data_ld_host_order(nfp_prog, meta, dst_gpr, rega, regb,
nfp_prog          949 drivers/net/ethernet/netronome/nfp/bpf/jit.c construct_data_ind_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog          955 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	tmp_reg = ur_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
nfp_prog          956 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, imm_both(nfp_prog), reg_a(src), ALU_OP_ADD, tmp_reg);
nfp_prog          959 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, imm_a(nfp_prog),
nfp_prog          960 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		 imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size));
nfp_prog          961 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(),
nfp_prog          962 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		 plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog));
nfp_prog          963 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT);
nfp_prog          966 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return data_ld(nfp_prog, meta, imm_b(nfp_prog), 0, size);
nfp_prog          970 drivers/net/ethernet/netronome/nfp/bpf/jit.c construct_data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog          976 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog));
nfp_prog          977 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg);
nfp_prog          978 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT);
nfp_prog          981 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
nfp_prog          982 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return data_ld(nfp_prog, meta, tmp_reg, 0, size);
nfp_prog          986 drivers/net/ethernet/netronome/nfp/bpf/jit.c data_stx_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset,
nfp_prog          992 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, reg_xfer(i), reg_a(src_gpr + i));
nfp_prog          994 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
nfp_prog         1001 drivers/net/ethernet/netronome/nfp/bpf/jit.c data_st_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset,
nfp_prog         1004 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_xfer(0), imm);
nfp_prog         1006 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_xfer(1), imm >> 32);
nfp_prog         1008 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
nfp_prog         1015 drivers/net/ethernet/netronome/nfp/bpf/jit.c (*lmem_step)(struct nfp_prog *nfp_prog, u8 gpr, u8 gpr_byte, s32 off,
nfp_prog         1020 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_lmem_load(struct nfp_prog *nfp_prog, u8 dst, u8 dst_byte, s32 off,
nfp_prog         1038 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, reg_both(dst),
nfp_prog         1070 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		reg = imm_a(nfp_prog);
nfp_prog         1077 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			wrp_mov(nfp_prog, reg, reg_lm(0, idx));
nfp_prog         1080 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_ld_field_any(nfp_prog, reg_both(dst), mask, reg, sc, shf, new_gpr);
nfp_prog         1083 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3));
nfp_prog         1089 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_lmem_store(struct nfp_prog *nfp_prog, u8 src, u8 src_byte, s32 off,
nfp_prog         1107 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog,
nfp_prog         1140 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		reg = imm_a(nfp_prog);
nfp_prog         1145 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			wrp_mov(nfp_prog, reg, reg_lm(0, idx));
nfp_prog         1148 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_ld_field(nfp_prog, reg, mask, reg_b(src), sc, shf);
nfp_prog         1152 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			wrp_mov(nfp_prog, reg_lm(0, idx), reg);
nfp_prog         1154 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3));
nfp_prog         1161 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         1165 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	s32 off = nfp_prog->stack_frame_depth + meta->insn.off + ptr_off;
nfp_prog         1180 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		stack_off_reg = ur_load_imm_any(nfp_prog, meta->insn.off,
nfp_prog         1181 drivers/net/ethernet/netronome/nfp/bpf/jit.c 						stack_imm(nfp_prog));
nfp_prog         1183 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, imm_b(nfp_prog),
nfp_prog         1197 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 32),
nfp_prog         1198 drivers/net/ethernet/netronome/nfp/bpf/jit.c 						stack_imm(nfp_prog));
nfp_prog         1199 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, imm_b(nfp_prog),
nfp_prog         1200 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg);
nfp_prog         1204 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 4),
nfp_prog         1205 drivers/net/ethernet/netronome/nfp/bpf/jit.c 						stack_imm(nfp_prog));
nfp_prog         1207 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, imm_b(nfp_prog),
nfp_prog         1208 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg);
nfp_prog         1218 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_csr_wr(nfp_prog, imm_b(nfp_prog), NFP_CSR_ACT_LM_ADDR3);
nfp_prog         1224 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_nops(nfp_prog, nop_cnt);
nfp_prog         1228 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_zext(nfp_prog, meta, gpr);
nfp_prog         1243 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		ret = step(nfp_prog, gpr, gpr_byte, off, slice_size,
nfp_prog         1265 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu_imm(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u32 imm)
nfp_prog         1271 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			wrp_immed(nfp_prog, reg_both(dst), 0);
nfp_prog         1277 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			wrp_immed(nfp_prog, reg_both(dst), ~0U);
nfp_prog         1283 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			emit_alu(nfp_prog, reg_both(dst), reg_none(),
nfp_prog         1289 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	tmp_reg = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog));
nfp_prog         1290 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, tmp_reg);
nfp_prog         1294 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         1305 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, imm & ~0U);
nfp_prog         1306 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, alu_op, imm >> 32);
nfp_prog         1312 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu64_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         1317 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src));
nfp_prog         1318 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_both(dst + 1),
nfp_prog         1325 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         1331 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_alu_imm(nfp_prog, dst, alu_op, insn->imm);
nfp_prog         1332 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_zext(nfp_prog, meta, dst);
nfp_prog         1338 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         1343 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src));
nfp_prog         1344 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_zext(nfp_prog, meta, dst);
nfp_prog         1350 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_test_reg_one(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u8 src,
nfp_prog         1353 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_a(dst), alu_op, reg_b(src));
nfp_prog         1354 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, br_mask, off, 0);
nfp_prog         1358 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         1363 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_test_reg_one(nfp_prog, insn->dst_reg * 2, alu_op,
nfp_prog         1366 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op,
nfp_prog         1400 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1416 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
nfp_prog         1418 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, reg_none(), reg_a(reg), alu_op, tmp_reg);
nfp_prog         1420 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, reg_none(), tmp_reg, alu_op, reg_a(reg));
nfp_prog         1423 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
nfp_prog         1425 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			emit_alu(nfp_prog, reg_none(),
nfp_prog         1428 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			emit_alu(nfp_prog, reg_none(),
nfp_prog         1432 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, code->br_mask, insn->off, 0);
nfp_prog         1437 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1456 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg));
nfp_prog         1458 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, reg_none(),
nfp_prog         1460 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, code->br_mask, insn->off, 0);
nfp_prog         1465 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void wrp_end32(struct nfp_prog *nfp_prog, swreg reg_in, u8 gpr_out)
nfp_prog         1467 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_ld_field(nfp_prog, reg_both(gpr_out), 0xf, reg_in,
nfp_prog         1469 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_ld_field(nfp_prog, reg_both(gpr_out), 0x5, reg_a(gpr_out),
nfp_prog         1474 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mul_u32(struct nfp_prog *nfp_prog, swreg dst_hi, swreg dst_lo, swreg lreg,
nfp_prog         1477 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_mul(nfp_prog, lreg, MUL_TYPE_START, MUL_STEP_NONE, rreg);
nfp_prog         1478 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_1, rreg);
nfp_prog         1479 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_2, rreg);
nfp_prog         1480 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_3, rreg);
nfp_prog         1481 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_4, rreg);
nfp_prog         1482 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_mul(nfp_prog, dst_lo, MUL_TYPE_STEP_32x32, MUL_LAST, reg_none());
nfp_prog         1484 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_mul(nfp_prog, dst_hi, MUL_TYPE_STEP_32x32, MUL_LAST_2,
nfp_prog         1487 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, dst_hi, 0);
nfp_prog         1491 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mul_u16(struct nfp_prog *nfp_prog, swreg dst_hi, swreg dst_lo, swreg lreg,
nfp_prog         1494 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_mul(nfp_prog, lreg, MUL_TYPE_START, MUL_STEP_NONE, rreg);
nfp_prog         1495 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_16x16, MUL_STEP_1, rreg);
nfp_prog         1496 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_16x16, MUL_STEP_2, rreg);
nfp_prog         1497 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_mul(nfp_prog, dst_lo, MUL_TYPE_STEP_16x16, MUL_LAST, reg_none());
nfp_prog         1501 drivers/net/ethernet/netronome/nfp/bpf/jit.c wrp_mul(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         1520 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		multiplier = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog));
nfp_prog         1524 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mul_u32(nfp_prog, dst_hi, dst_lo, multiplicand, multiplier,
nfp_prog         1527 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mul_u16(nfp_prog, dst_hi, dst_lo, multiplicand, multiplier);
nfp_prog         1532 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int wrp_div_imm(struct nfp_prog *nfp_prog, u8 dst, u64 imm)
nfp_prog         1540 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, dst_both, 0);
nfp_prog         1556 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		swreg tmp_b = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog));
nfp_prog         1558 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, reg_none(), dst_a, ALU_OP_SUB, tmp_b);
nfp_prog         1559 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, imm_a(nfp_prog), 0);
nfp_prog         1560 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, dst_both, imm_a(nfp_prog), ALU_OP_ADD_C,
nfp_prog         1573 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	magic = ur_load_imm_any(nfp_prog, rvalue.m, imm_b(nfp_prog));
nfp_prog         1575 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b,
nfp_prog         1578 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mul_u32(nfp_prog, imm_both(nfp_prog), reg_none(), dst_a,
nfp_prog         1580 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, dst_both, dst_a, ALU_OP_SUB,
nfp_prog         1581 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			 imm_b(nfp_prog));
nfp_prog         1582 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b,
nfp_prog         1584 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, dst_both, dst_a, ALU_OP_ADD,
nfp_prog         1585 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			 imm_b(nfp_prog));
nfp_prog         1586 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b,
nfp_prog         1590 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE,
nfp_prog         1592 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mul_u32(nfp_prog, dst_both, reg_none(), dst_a, magic, true);
nfp_prog         1593 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE,
nfp_prog         1600 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1602 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	swreg tmp = imm_a(nfp_prog), tmp_len = imm_b(nfp_prog);
nfp_prog         1606 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	adjust_head = &nfp_prog->bpf->adjust_head;
nfp_prog         1609 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (nfp_prog->adjust_head_location != UINT_MAX) {
nfp_prog         1610 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (WARN_ON_ONCE(nfp_prog->adjust_head_location != meta->n))
nfp_prog         1613 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, pptr_reg(nfp_prog),
nfp_prog         1614 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			 reg_a(2 * 2), ALU_OP_ADD, pptr_reg(nfp_prog));
nfp_prog         1615 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, plen_reg(nfp_prog),
nfp_prog         1616 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
nfp_prog         1617 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, pv_len(nfp_prog),
nfp_prog         1618 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			 pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
nfp_prog         1620 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_both(0), 0);
nfp_prog         1621 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_both(1), 0);
nfp_prog         1630 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	ret_einval = nfp_prog_current_offset(nfp_prog) + 14;
nfp_prog         1634 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, tmp,
nfp_prog         1635 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		 reg_a(2 * 2), ALU_OP_ADD_2B, pptr_reg(nfp_prog));
nfp_prog         1638 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(),
nfp_prog         1640 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_BLO, ret_einval, 0);
nfp_prog         1641 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(),
nfp_prog         1643 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_BLO, ret_einval, 0);
nfp_prog         1646 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, tmp_len,
nfp_prog         1647 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
nfp_prog         1648 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(),
nfp_prog         1650 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_BMI, ret_einval, 0);
nfp_prog         1653 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(0), 0);
nfp_prog         1654 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(1), 0);
nfp_prog         1657 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_ld_field(nfp_prog, pptr_reg(nfp_prog), 0x3, tmp, SHF_SC_NONE, 0);
nfp_prog         1660 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_UNC, end, 2);
nfp_prog         1662 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, plen_reg(nfp_prog),
nfp_prog         1663 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
nfp_prog         1664 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, pv_len(nfp_prog),
nfp_prog         1665 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		 pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
nfp_prog         1668 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (!nfp_prog_confirm_current_offset(nfp_prog, ret_einval))
nfp_prog         1671 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(0), -22);
nfp_prog         1672 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(1), ~0);
nfp_prog         1674 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (!nfp_prog_confirm_current_offset(nfp_prog, end))
nfp_prog         1680 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int adjust_tail(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1685 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	BUILD_BUG_ON(plen_reg(nfp_prog) != reg_b(STATIC_REG_PKT_LEN));
nfp_prog         1687 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	plen = imm_a(nfp_prog);
nfp_prog         1690 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	ret_einval = nfp_prog_current_offset(nfp_prog) + 9;
nfp_prog         1691 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	end = nfp_prog_current_offset(nfp_prog) + 11;
nfp_prog         1694 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, plen, plen_reg(nfp_prog), ALU_OP_ADD, delta);
nfp_prog         1698 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_BCC, ret_einval, 0);
nfp_prog         1701 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), plen, ALU_OP_SUB, reg_imm(ETH_HLEN));
nfp_prog         1702 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_BMI, ret_einval, 0);
nfp_prog         1704 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, plen_reg(nfp_prog),
nfp_prog         1705 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		 plen_reg(nfp_prog), ALU_OP_ADD, delta);
nfp_prog         1706 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, pv_len(nfp_prog),
nfp_prog         1707 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		 pv_len(nfp_prog), ALU_OP_ADD, delta);
nfp_prog         1709 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_UNC, end, 2);
nfp_prog         1710 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(0), 0);
nfp_prog         1711 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(1), 0);
nfp_prog         1713 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (!nfp_prog_confirm_current_offset(nfp_prog, ret_einval))
nfp_prog         1716 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(0), -22);
nfp_prog         1717 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(1), ~0);
nfp_prog         1719 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (!nfp_prog_confirm_current_offset(nfp_prog, end))
nfp_prog         1726 drivers/net/ethernet/netronome/nfp/bpf/jit.c map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1733 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	lm_off = nfp_prog->stack_frame_depth;
nfp_prog         1739 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_csr_wr(nfp_prog, reg_b(2 * 2), NFP_CSR_ACT_LM_ADDR0);
nfp_prog         1741 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_csr_wr(nfp_prog, reg_b(3 * 2), NFP_CSR_ACT_LM_ADDR2);
nfp_prog         1743 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id,
nfp_prog         1745 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	ret_tgt = nfp_prog_current_offset(nfp_prog) + 2;
nfp_prog         1748 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_mov(nfp_prog, reg_a(0), reg_a(2));
nfp_prog         1751 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL);
nfp_prog         1753 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt))
nfp_prog         1760 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_csr_wr(nfp_prog, stack_reg(nfp_prog), NFP_CSR_ACT_LM_ADDR0);
nfp_prog         1761 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_nops(nfp_prog, 3);
nfp_prog         1767 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_get_prandom_u32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1769 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	__emit_csr_rd(nfp_prog, NFP_CSR_PSEUDO_RND_NUM);
nfp_prog         1771 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_immed(nfp_prog, reg_both(0), 0,
nfp_prog         1773 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_immed(nfp_prog, reg_both(1), 0,
nfp_prog         1779 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_perf_event_output(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1784 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	ptr_type = ur_load_imm_any(nfp_prog, meta->arg1.type, imm_a(nfp_prog));
nfp_prog         1786 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	ret_tgt = nfp_prog_current_offset(nfp_prog) + 3;
nfp_prog         1788 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id,
nfp_prog         1792 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_mov(nfp_prog, reg_a(1), ptr_type);
nfp_prog         1795 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL);
nfp_prog         1797 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt))
nfp_prog         1804 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_queue_select(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1808 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	jmp_tgt = nfp_prog_current_offset(nfp_prog) + 5;
nfp_prog         1811 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_a(meta->insn.src_reg * 2),
nfp_prog         1813 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_BEQ, jmp_tgt, 2);
nfp_prog         1816 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf(nfp_prog, pv_qsel_set(nfp_prog),
nfp_prog         1817 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		 pv_qsel_set(nfp_prog), SHF_OP_OR, reg_imm(1),
nfp_prog         1819 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_ld_field(nfp_prog,
nfp_prog         1820 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		      pv_qsel_val(nfp_prog), 0x1, reg_b(meta->insn.src_reg * 2),
nfp_prog         1825 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_ld_field(nfp_prog,
nfp_prog         1826 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		      pv_qsel_val(nfp_prog), 0x1, reg_imm(NFP_NET_RXR_MAX),
nfp_prog         1829 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (!nfp_prog_confirm_current_offset(nfp_prog, jmp_tgt))
nfp_prog         1836 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1845 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		stack_depth_reg = ur_load_imm_any(nfp_prog,
nfp_prog         1846 drivers/net/ethernet/netronome/nfp/bpf/jit.c 						  nfp_prog->stack_frame_depth,
nfp_prog         1847 drivers/net/ethernet/netronome/nfp/bpf/jit.c 						  stack_imm(nfp_prog));
nfp_prog         1848 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, reg_both(dst), stack_reg(nfp_prog),
nfp_prog         1850 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_both(dst + 1), 0);
nfp_prog         1852 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_reg_mov(nfp_prog, dst, src);
nfp_prog         1853 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_reg_mov(nfp_prog, dst + 1, src + 1);
nfp_prog         1859 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mov_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1863 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2), imm & ~0U);
nfp_prog         1864 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), imm >> 32);
nfp_prog         1869 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int xor_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1871 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu64_reg(nfp_prog, meta, ALU_OP_XOR);
nfp_prog         1874 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int xor_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1876 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu64_imm(nfp_prog, meta, ALU_OP_XOR, !meta->insn.imm);
nfp_prog         1879 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int and_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1881 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu64_reg(nfp_prog, meta, ALU_OP_AND);
nfp_prog         1884 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int and_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1886 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu64_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm);
nfp_prog         1889 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int or_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1891 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu64_reg(nfp_prog, meta, ALU_OP_OR);
nfp_prog         1894 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int or_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1896 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu64_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm);
nfp_prog         1899 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int add_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1903 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2),
nfp_prog         1906 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1),
nfp_prog         1913 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int add_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1918 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_ADD, imm & ~0U);
nfp_prog         1919 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_ADD_C, imm >> 32);
nfp_prog         1924 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int sub_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1928 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2),
nfp_prog         1931 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1),
nfp_prog         1938 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int sub_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1943 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_SUB, imm & ~0U);
nfp_prog         1944 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_SUB_C, imm >> 32);
nfp_prog         1949 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mul_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1951 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_mul(nfp_prog, meta, true, true);
nfp_prog         1954 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mul_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1956 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_mul(nfp_prog, meta, true, false);
nfp_prog         1959 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int div_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1963 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_div_imm(nfp_prog, insn->dst_reg * 2, insn->imm);
nfp_prog         1966 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int div_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1971 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_div_imm(nfp_prog, meta->insn.dst_reg * 2, meta->umin_src);
nfp_prog         1974 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int neg_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         1978 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), reg_imm(0),
nfp_prog         1980 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), reg_imm(0),
nfp_prog         1996 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int __shl_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
nfp_prog         2002 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, reg_both(dst + 1), reg_a(dst + 1),
nfp_prog         2005 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
nfp_prog         2008 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_reg_mov(nfp_prog, dst + 1, dst);
nfp_prog         2009 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_both(dst), 0);
nfp_prog         2011 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
nfp_prog         2013 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_both(dst), 0);
nfp_prog         2019 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2024 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return __shl_imm64(nfp_prog, dst, insn->imm);
nfp_prog         2027 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shl_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src)
nfp_prog         2029 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, imm_both(nfp_prog), reg_imm(32), ALU_OP_SUB,
nfp_prog         2031 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), imm_a(nfp_prog), ALU_OP_OR, reg_imm(0));
nfp_prog         2032 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_a(dst + 1), SHF_OP_NONE,
nfp_prog         2037 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shl_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src)
nfp_prog         2039 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
nfp_prog         2040 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
nfp_prog         2044 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shl_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
nfp_prog         2046 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	shl_reg64_lt32_high(nfp_prog, dst, src);
nfp_prog         2047 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	shl_reg64_lt32_low(nfp_prog, dst, src);
nfp_prog         2050 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shl_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
nfp_prog         2052 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
nfp_prog         2053 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
nfp_prog         2055 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(dst), 0);
nfp_prog         2058 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shl_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2068 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return __shl_imm64(nfp_prog, dst, umin);
nfp_prog         2072 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		shl_reg64_lt32(nfp_prog, dst, src);
nfp_prog         2074 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		shl_reg64_ge32(nfp_prog, dst, src);
nfp_prog         2081 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		label_ge32 = nfp_prog_current_offset(nfp_prog) + 7;
nfp_prog         2082 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0);
nfp_prog         2084 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		shl_reg64_lt32_high(nfp_prog, dst, src);
nfp_prog         2085 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		label_end = nfp_prog_current_offset(nfp_prog) + 6;
nfp_prog         2086 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_br(nfp_prog, BR_UNC, label_end, 2);
nfp_prog         2088 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		shl_reg64_lt32_low(nfp_prog, dst, src);
nfp_prog         2090 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32))
nfp_prog         2092 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		shl_reg64_ge32(nfp_prog, dst, src);
nfp_prog         2094 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (!nfp_prog_confirm_current_offset(nfp_prog, label_end))
nfp_prog         2111 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int __shr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
nfp_prog         2117 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE,
nfp_prog         2119 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
nfp_prog         2122 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_reg_mov(nfp_prog, dst, dst + 1);
nfp_prog         2123 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_both(dst + 1), 0);
nfp_prog         2125 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
nfp_prog         2127 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_both(dst + 1), 0);
nfp_prog         2133 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2138 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return __shr_imm64(nfp_prog, dst, insn->imm);
nfp_prog         2142 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src)
nfp_prog         2144 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
nfp_prog         2145 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
nfp_prog         2149 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src)
nfp_prog         2151 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
nfp_prog         2152 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf_indir(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE,
nfp_prog         2156 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
nfp_prog         2158 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	shr_reg64_lt32_low(nfp_prog, dst, src);
nfp_prog         2159 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	shr_reg64_lt32_high(nfp_prog, dst, src);
nfp_prog         2162 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void shr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
nfp_prog         2164 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
nfp_prog         2165 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
nfp_prog         2167 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(dst + 1), 0);
nfp_prog         2170 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2180 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return __shr_imm64(nfp_prog, dst, umin);
nfp_prog         2184 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		shr_reg64_lt32(nfp_prog, dst, src);
nfp_prog         2186 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		shr_reg64_ge32(nfp_prog, dst, src);
nfp_prog         2193 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		label_ge32 = nfp_prog_current_offset(nfp_prog) + 6;
nfp_prog         2194 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0);
nfp_prog         2195 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		shr_reg64_lt32_low(nfp_prog, dst, src);
nfp_prog         2196 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		label_end = nfp_prog_current_offset(nfp_prog) + 6;
nfp_prog         2197 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_br(nfp_prog, BR_UNC, label_end, 2);
nfp_prog         2199 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		shr_reg64_lt32_high(nfp_prog, dst, src);
nfp_prog         2201 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32))
nfp_prog         2203 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		shr_reg64_ge32(nfp_prog, dst, src);
nfp_prog         2205 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (!nfp_prog_confirm_current_offset(nfp_prog, label_end))
nfp_prog         2215 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int __ashr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
nfp_prog         2221 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE,
nfp_prog         2224 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR,
nfp_prog         2226 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
nfp_prog         2230 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_reg_mov(nfp_prog, dst, dst + 1);
nfp_prog         2231 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
nfp_prog         2234 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR,
nfp_prog         2236 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
nfp_prog         2238 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
nfp_prog         2245 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int ashr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2250 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return __ashr_imm64(nfp_prog, dst, insn->imm);
nfp_prog         2253 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void ashr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src)
nfp_prog         2258 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1));
nfp_prog         2259 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
nfp_prog         2263 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void ashr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src)
nfp_prog         2268 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return shr_reg64_lt32_low(nfp_prog, dst, src);
nfp_prog         2271 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void ashr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
nfp_prog         2273 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	ashr_reg64_lt32_low(nfp_prog, dst, src);
nfp_prog         2274 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	ashr_reg64_lt32_high(nfp_prog, dst, src);
nfp_prog         2277 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void ashr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
nfp_prog         2279 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1));
nfp_prog         2280 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
nfp_prog         2282 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
nfp_prog         2287 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int ashr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2297 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return __ashr_imm64(nfp_prog, dst, umin);
nfp_prog         2301 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		ashr_reg64_lt32(nfp_prog, dst, src);
nfp_prog         2303 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		ashr_reg64_ge32(nfp_prog, dst, src);
nfp_prog         2307 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		label_ge32 = nfp_prog_current_offset(nfp_prog) + 6;
nfp_prog         2308 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0);
nfp_prog         2309 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		ashr_reg64_lt32_low(nfp_prog, dst, src);
nfp_prog         2310 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		label_end = nfp_prog_current_offset(nfp_prog) + 6;
nfp_prog         2311 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_br(nfp_prog, BR_UNC, label_end, 2);
nfp_prog         2313 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		ashr_reg64_lt32_high(nfp_prog, dst, src);
nfp_prog         2315 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32))
nfp_prog         2317 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		ashr_reg64_ge32(nfp_prog, dst, src);
nfp_prog         2319 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (!nfp_prog_confirm_current_offset(nfp_prog, label_end))
nfp_prog         2326 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mov_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2330 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_reg_mov(nfp_prog, insn->dst_reg * 2,  insn->src_reg * 2);
nfp_prog         2331 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
nfp_prog         2336 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mov_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2340 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm);
nfp_prog         2341 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
nfp_prog         2346 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int xor_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2348 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_XOR);
nfp_prog         2351 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int xor_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2353 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_XOR);
nfp_prog         2356 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int and_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2358 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_AND);
nfp_prog         2361 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int and_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2363 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_AND);
nfp_prog         2366 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int or_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2368 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_OR);
nfp_prog         2371 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int or_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2373 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_OR);
nfp_prog         2376 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int add_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2378 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_ADD);
nfp_prog         2381 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int add_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2383 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_ADD);
nfp_prog         2386 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int sub_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2388 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_SUB);
nfp_prog         2391 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int sub_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2393 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_SUB);
nfp_prog         2396 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mul_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2398 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_mul(nfp_prog, meta, false, true);
nfp_prog         2401 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mul_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2403 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_mul(nfp_prog, meta, false, false);
nfp_prog         2406 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int div_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2408 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return div_reg64(nfp_prog, meta);
nfp_prog         2411 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int div_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2413 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return div_imm64(nfp_prog, meta);
nfp_prog         2416 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2420 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_both(dst), reg_imm(0), ALU_OP_SUB, reg_b(dst));
nfp_prog         2421 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_zext(nfp_prog, meta, dst);
nfp_prog         2427 drivers/net/ethernet/netronome/nfp/bpf/jit.c __ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
nfp_prog         2432 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, reg_none(), reg_a(dst), ALU_OP_OR,
nfp_prog         2434 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
nfp_prog         2437 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_zext(nfp_prog, meta, dst);
nfp_prog         2442 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int ashr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2452 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return __ashr_imm(nfp_prog, meta, dst, umin);
nfp_prog         2458 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst));
nfp_prog         2459 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
nfp_prog         2461 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_zext(nfp_prog, meta, dst);
nfp_prog         2466 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2471 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return __ashr_imm(nfp_prog, meta, dst, insn->imm);
nfp_prog         2475 drivers/net/ethernet/netronome/nfp/bpf/jit.c __shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
nfp_prog         2479 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
nfp_prog         2481 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_zext(nfp_prog, meta, dst);
nfp_prog         2485 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2490 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return __shr_imm(nfp_prog, meta, dst, insn->imm);
nfp_prog         2493 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2503 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return __shr_imm(nfp_prog, meta, dst, umin);
nfp_prog         2506 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
nfp_prog         2507 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
nfp_prog         2509 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_zext(nfp_prog, meta, dst);
nfp_prog         2514 drivers/net/ethernet/netronome/nfp/bpf/jit.c __shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
nfp_prog         2518 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
nfp_prog         2520 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_zext(nfp_prog, meta, dst);
nfp_prog         2524 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2529 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return __shl_imm(nfp_prog, meta, dst, insn->imm);
nfp_prog         2532 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int shl_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2542 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return __shl_imm(nfp_prog, meta, dst, umin);
nfp_prog         2545 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	shl_reg64_lt32_low(nfp_prog, dst, src);
nfp_prog         2546 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_zext(nfp_prog, meta, dst);
nfp_prog         2550 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int end_reg32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2557 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_ld_field(nfp_prog, reg_both(gpr), 0x9, reg_b(gpr),
nfp_prog         2559 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_ld_field(nfp_prog, reg_both(gpr), 0xe, reg_a(gpr),
nfp_prog         2562 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_both(gpr + 1), 0);
nfp_prog         2565 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_end32(nfp_prog, reg_a(gpr), gpr);
nfp_prog         2566 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_both(gpr + 1), 0);
nfp_prog         2569 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, imm_a(nfp_prog), reg_b(gpr + 1));
nfp_prog         2571 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_end32(nfp_prog, reg_a(gpr), gpr + 1);
nfp_prog         2572 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_end32(nfp_prog, imm_a(nfp_prog), gpr);
nfp_prog         2579 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int imm_ld8_part2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2589 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(dst), imm_lo);
nfp_prog         2593 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, reg_both(dst + 1), reg_a(dst));
nfp_prog         2595 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_both(dst + 1), imm_hi);
nfp_prog         2600 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2606 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2608 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return construct_data_ld(nfp_prog, meta, meta->insn.imm, 1);
nfp_prog         2611 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2613 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return construct_data_ld(nfp_prog, meta, meta->insn.imm, 2);
nfp_prog         2616 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2618 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return construct_data_ld(nfp_prog, meta, meta->insn.imm, 4);
nfp_prog         2621 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2623 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
nfp_prog         2627 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2629 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
nfp_prog         2633 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2635 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
nfp_prog         2640 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         2643 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_op_stack(nfp_prog, meta, size, ptr_off,
nfp_prog         2648 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         2657 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, dst, plen_reg(nfp_prog));
nfp_prog         2662 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog));
nfp_prog         2667 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, dst,
nfp_prog         2668 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog));
nfp_prog         2674 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
nfp_prog         2679 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         2688 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog));
nfp_prog         2693 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, dst,
nfp_prog         2694 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog));
nfp_prog         2700 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
nfp_prog         2706 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         2711 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
nfp_prog         2713 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return data_ld_host_order_addr32(nfp_prog, meta, meta->insn.src_reg * 2,
nfp_prog         2718 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx_emem(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         2723 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
nfp_prog         2725 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return data_ld_host_order_addr40(nfp_prog, meta, meta->insn.src_reg * 2,
nfp_prog         2730 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx_data_init_pktcache(struct nfp_prog *nfp_prog,
nfp_prog         2739 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	off = re_load_imm_any(nfp_prog, range_start, imm_b(nfp_prog));
nfp_prog         2747 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, reg_none(),
nfp_prog         2751 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0, src_base,
nfp_prog         2756 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx_data_from_pktcache_unaligned(struct nfp_prog *nfp_prog,
nfp_prog         2783 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_reg_subpart(nfp_prog, dst_lo, src_lo, len_lo, off);
nfp_prog         2786 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_zext(nfp_prog, meta, dst_gpr);
nfp_prog         2793 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_reg_or_subpart(nfp_prog, dst_lo, src_mid, len_mid, len_lo);
nfp_prog         2794 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_zext(nfp_prog, meta, dst_gpr);
nfp_prog         2798 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_reg_or_subpart(nfp_prog, dst_lo, src_mid,
nfp_prog         2800 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_reg_subpart(nfp_prog, dst_hi, src_mid, len_lo,
nfp_prog         2802 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_reg_or_subpart(nfp_prog, dst_hi, src_hi, REG_WIDTH - len_lo,
nfp_prog         2810 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx_data_from_pktcache_aligned(struct nfp_prog *nfp_prog,
nfp_prog         2824 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_reg_subpart(nfp_prog, dst_lo, src_lo, size, 0);
nfp_prog         2825 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_zext(nfp_prog, meta, dst_gpr);
nfp_prog         2827 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, dst_lo, src_lo);
nfp_prog         2828 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_zext(nfp_prog, meta, dst_gpr);
nfp_prog         2832 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, dst_lo, src_lo);
nfp_prog         2833 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, dst_hi, src_hi);
nfp_prog         2840 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx_data_from_pktcache(struct nfp_prog *nfp_prog,
nfp_prog         2846 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return mem_ldx_data_from_pktcache_aligned(nfp_prog, meta, size);
nfp_prog         2848 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_ldx_data_from_pktcache_unaligned(nfp_prog, meta, size);
nfp_prog         2852 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         2856 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return nfp_cpp_memcpy(nfp_prog, meta);
nfp_prog         2859 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (nfp_prog->type == BPF_PROG_TYPE_XDP)
nfp_prog         2860 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			return mem_ldx_xdp(nfp_prog, meta, size);
nfp_prog         2862 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			return mem_ldx_skb(nfp_prog, meta, size);
nfp_prog         2868 drivers/net/ethernet/netronome/nfp/bpf/jit.c 				mem_ldx_data_init_pktcache(nfp_prog, meta);
nfp_prog         2870 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			return mem_ldx_data_from_pktcache(nfp_prog, meta, size);
nfp_prog         2872 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			return mem_ldx_data(nfp_prog, meta, size);
nfp_prog         2877 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return mem_ldx_stack(nfp_prog, meta, size,
nfp_prog         2881 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return mem_ldx_emem(nfp_prog, meta, size);
nfp_prog         2886 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_ldx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2888 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_ldx(nfp_prog, meta, 1);
nfp_prog         2891 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_ldx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2893 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_ldx(nfp_prog, meta, 2);
nfp_prog         2896 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2898 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_ldx(nfp_prog, meta, 4);
nfp_prog         2901 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_ldx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2903 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_ldx(nfp_prog, meta, 8);
nfp_prog         2907 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_st_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         2913 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
nfp_prog         2915 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return data_st_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg,
nfp_prog         2919 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_st(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         2923 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return mem_st_data(nfp_prog, meta, size);
nfp_prog         2928 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_st1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2930 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_st(nfp_prog, meta, 1);
nfp_prog         2933 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_st2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2935 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_st(nfp_prog, meta, 2);
nfp_prog         2938 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_st4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2940 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_st(nfp_prog, meta, 4);
nfp_prog         2943 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_st8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2945 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_st(nfp_prog, meta, 8);
nfp_prog         2949 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_stx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         2954 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
nfp_prog         2956 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return data_stx_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg,
nfp_prog         2961 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         2964 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_op_stack(nfp_prog, meta, size, ptr_off,
nfp_prog         2969 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_stx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2973 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return nfp_queue_select(nfp_prog, meta);
nfp_prog         2981 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         2985 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return mem_stx_data(nfp_prog, meta, size);
nfp_prog         2988 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return mem_stx_stack(nfp_prog, meta, size,
nfp_prog         2994 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_stx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         2996 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_stx(nfp_prog, meta, 1);
nfp_prog         2999 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3001 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_stx(nfp_prog, meta, 2);
nfp_prog         3004 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3007 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (nfp_prog->type == BPF_PROG_TYPE_XDP)
nfp_prog         3008 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			return mem_stx_xdp(nfp_prog, meta);
nfp_prog         3009 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_stx(nfp_prog, meta, 4);
nfp_prog         3012 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_stx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3014 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_stx(nfp_prog, meta, 8);
nfp_prog         3018 drivers/net/ethernet/netronome/nfp/bpf/jit.c mem_xadd(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, bool is64)
nfp_prog         3025 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	off = ur_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
nfp_prog         3032 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	out = nfp_prog_current_offset(nfp_prog);
nfp_prog         3033 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	full_add = nfp_prog_current_offset(nfp_prog);
nfp_prog         3052 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		swreg max_imm = imm_a(nfp_prog);
nfp_prog         3054 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, max_imm, 0xffff);
nfp_prog         3055 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, reg_none(),
nfp_prog         3057 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, reg_none(),
nfp_prog         3059 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_br(nfp_prog, BR_BLO, full_add, meta->insn.off ? 2 : 0);
nfp_prog         3068 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, imma_a(nfp_prog),
nfp_prog         3070 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, imma_b(nfp_prog),
nfp_prog         3072 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		addra = imma_a(nfp_prog);
nfp_prog         3073 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		addrb = imma_b(nfp_prog);
nfp_prog         3078 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		swreg prev_alu = imm_a(nfp_prog);
nfp_prog         3080 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed(nfp_prog, prev_alu,
nfp_prog         3084 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_reg_or_subpart(nfp_prog, prev_alu, reg_b(src_gpr), 2, 2);
nfp_prog         3085 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_cmd_indir(nfp_prog, CMD_TGT_ADD_IMM, CMD_MODE_40b_BA, 0,
nfp_prog         3089 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			emit_br(nfp_prog, BR_UNC, out, 0);
nfp_prog         3092 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (!nfp_prog_confirm_current_offset(nfp_prog, full_add))
nfp_prog         3097 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_cmd(nfp_prog, CMD_TGT_ADD, CMD_MODE_40b_BA, 0,
nfp_prog         3101 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, reg_xfer(0), reg_a(src_gpr));
nfp_prog         3103 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			wrp_mov(nfp_prog, reg_xfer(1), reg_a(src_gpr + 1));
nfp_prog         3106 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (!nfp_prog_confirm_current_offset(nfp_prog, out))
nfp_prog         3112 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_xadd4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3114 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_xadd(nfp_prog, meta, false);
nfp_prog         3117 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int mem_xadd8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3119 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return mem_xadd(nfp_prog, meta, true);
nfp_prog         3122 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3124 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_UNC, meta->insn.off, 0);
nfp_prog         3129 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3139 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
nfp_prog         3140 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, imm_a(nfp_prog),
nfp_prog         3142 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		or1 = imm_a(nfp_prog);
nfp_prog         3146 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
nfp_prog         3147 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, imm_b(nfp_prog),
nfp_prog         3149 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		or2 = imm_b(nfp_prog);
nfp_prog         3152 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), or1, ALU_OP_OR, or2);
nfp_prog         3153 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_BEQ, insn->off, 0);
nfp_prog         3158 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jeq32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3163 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	tmp_reg = ur_load_imm_any(nfp_prog, insn->imm, imm_b(nfp_prog));
nfp_prog         3164 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(),
nfp_prog         3166 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_BEQ, insn->off, 0);
nfp_prog         3171 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3178 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
nfp_prog         3179 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, imm_b(nfp_prog),
nfp_prog         3185 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, reg_none(),
nfp_prog         3186 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			 reg_a(dst_gpr + 1), ALU_OP_OR, imm_b(nfp_prog));
nfp_prog         3188 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_BNE, insn->off, 0);
nfp_prog         3193 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3202 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			emit_alu(nfp_prog, reg_none(), reg_none(), ALU_OP_NONE,
nfp_prog         3205 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2),
nfp_prog         3207 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_br(nfp_prog, BR_BNE, insn->off, 0);
nfp_prog         3211 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
nfp_prog         3212 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(),
nfp_prog         3214 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_BNE, insn->off, 0);
nfp_prog         3219 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
nfp_prog         3220 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(),
nfp_prog         3222 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_BNE, insn->off, 0);
nfp_prog         3227 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3231 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(insn->dst_reg * 2),
nfp_prog         3234 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, imm_b(nfp_prog),
nfp_prog         3237 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, reg_none(), imm_a(nfp_prog), ALU_OP_OR,
nfp_prog         3238 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			 imm_b(nfp_prog));
nfp_prog         3240 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_BEQ, insn->off, 0);
nfp_prog         3245 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3247 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE);
nfp_prog         3250 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3252 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE);
nfp_prog         3256 drivers/net/ethernet/netronome/nfp/bpf/jit.c bpf_to_bpf_call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3261 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	stack_depth = round_up(nfp_prog->stack_frame_depth, STACK_FRAME_ALIGN);
nfp_prog         3266 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		tmp_reg = ur_load_imm_any(nfp_prog, stack_depth,
nfp_prog         3267 drivers/net/ethernet/netronome/nfp/bpf/jit.c 					  stack_imm(nfp_prog));
nfp_prog         3268 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, stack_reg(nfp_prog),
nfp_prog         3269 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			 stack_reg(nfp_prog), ALU_OP_ADD, tmp_reg);
nfp_prog         3270 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_csr_wr(nfp_prog, stack_reg(nfp_prog),
nfp_prog         3309 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (nfp_prog->subprog[meta->jmp_dst->subprog_idx].needs_reg_push) {
nfp_prog         3310 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		ret_tgt = nfp_prog_current_offset(nfp_prog) + 3;
nfp_prog         3311 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2,
nfp_prog         3313 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		offset_br = nfp_prog_current_offset(nfp_prog);
nfp_prog         3314 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_immed_relo(nfp_prog, imm_b(nfp_prog), 0, RELO_IMMED_REL);
nfp_prog         3316 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		ret_tgt = nfp_prog_current_offset(nfp_prog) + 2;
nfp_prog         3317 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_br(nfp_prog, BR_UNC, meta->insn.imm, 1);
nfp_prog         3318 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		offset_br = nfp_prog_current_offset(nfp_prog);
nfp_prog         3320 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed_relo(nfp_prog, ret_reg(nfp_prog), ret_tgt, RELO_IMMED_REL);
nfp_prog         3322 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt))
nfp_prog         3326 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		tmp_reg = ur_load_imm_any(nfp_prog, stack_depth,
nfp_prog         3327 drivers/net/ethernet/netronome/nfp/bpf/jit.c 					  stack_imm(nfp_prog));
nfp_prog         3328 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_alu(nfp_prog, stack_reg(nfp_prog),
nfp_prog         3329 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			 stack_reg(nfp_prog), ALU_OP_SUB, tmp_reg);
nfp_prog         3330 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_csr_wr(nfp_prog, stack_reg(nfp_prog),
nfp_prog         3332 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_nops(nfp_prog, 3);
nfp_prog         3335 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	meta->num_insns_after_br = nfp_prog_current_offset(nfp_prog);
nfp_prog         3341 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int helper_call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3345 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return adjust_head(nfp_prog, meta);
nfp_prog         3347 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return adjust_tail(nfp_prog, meta);
nfp_prog         3351 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return map_call_stack_common(nfp_prog, meta);
nfp_prog         3353 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return nfp_get_prandom_u32(nfp_prog, meta);
nfp_prog         3355 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return nfp_perf_event_output(nfp_prog, meta);
nfp_prog         3362 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3365 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return bpf_to_bpf_call(nfp_prog, meta);
nfp_prog         3367 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return helper_call(nfp_prog, meta);
nfp_prog         3375 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3377 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 0, RELO_BR_GO_OUT);
nfp_prog         3383 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_subprog_epilogue(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3385 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (nfp_prog->subprog[meta->subprog_idx].needs_reg_push) {
nfp_prog         3391 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 1,
nfp_prog         3394 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, ret_reg(nfp_prog), reg_lm(0, 0));
nfp_prog         3397 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, ret_reg(nfp_prog), reg_lm(0, 0));
nfp_prog         3401 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		emit_rtn(nfp_prog, ret_reg(nfp_prog), 0);
nfp_prog         3407 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int jmp_exit(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3410 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return goto_out(nfp_prog, meta);
nfp_prog         3412 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return nfp_subprog_epilogue(nfp_prog, meta);
nfp_prog         3535 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_fixup_immed_relo(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog         3538 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (immed_get_value(nfp_prog->prog[br_idx + 1])) {
nfp_prog         3543 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	immed_set_value(&nfp_prog->prog[br_idx + 1], jmp_dst->off);
nfp_prog         3548 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int nfp_fixup_branches(struct nfp_prog *nfp_prog)
nfp_prog         3554 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	list_for_each_entry(meta, &nfp_prog->insns, l) {
nfp_prog         3565 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (list_is_last(&meta->l, &nfp_prog->insns))
nfp_prog         3566 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			br_idx = nfp_prog->last_bpf_off;
nfp_prog         3578 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (!nfp_is_br(nfp_prog->prog[br_idx])) {
nfp_prog         3580 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			       br_idx, meta->insn.code, nfp_prog->prog[br_idx]);
nfp_prog         3588 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) !=
nfp_prog         3605 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		    nfp_prog->subprog[jmp_dst->subprog_idx].needs_reg_push) {
nfp_prog         3606 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			err = nfp_fixup_immed_relo(nfp_prog, meta,
nfp_prog         3612 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) !=
nfp_prog         3617 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			if (!nfp_is_br(nfp_prog->prog[idx]))
nfp_prog         3619 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			br_set_offset(&nfp_prog->prog[idx], jmp_dst->off);
nfp_prog         3626 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_intro(struct nfp_prog *nfp_prog)
nfp_prog         3628 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, plen_reg(nfp_prog), GENMASK(13, 0));
nfp_prog         3629 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, plen_reg(nfp_prog),
nfp_prog         3630 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		 plen_reg(nfp_prog), ALU_OP_AND, pv_len(nfp_prog));
nfp_prog         3634 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_subprog_prologue(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3637 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_mov(nfp_prog, reg_lm(0, 0), ret_reg(nfp_prog));
nfp_prog         3641 drivers/net/ethernet/netronome/nfp/bpf/jit.c nfp_start_subprog(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
nfp_prog         3643 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	unsigned int depth = nfp_prog->subprog[meta->subprog_idx].stack_depth;
nfp_prog         3645 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->stack_frame_depth = round_up(depth, 4);
nfp_prog         3646 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_subprog_prologue(nfp_prog, meta);
nfp_prog         3654 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_outro_tc_da(struct nfp_prog *nfp_prog)
nfp_prog         3669 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
nfp_prog         3671 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
nfp_prog         3673 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
nfp_prog         3674 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16);
nfp_prog         3677 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog);
nfp_prog         3680 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_imm(7), ALU_OP_SUB, reg_b(0));
nfp_prog         3681 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0);
nfp_prog         3682 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
nfp_prog         3684 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_b(2), 0x41221211);
nfp_prog         3685 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_b(3), 0x41001211);
nfp_prog         3687 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf(nfp_prog, reg_a(1),
nfp_prog         3690 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
nfp_prog         3691 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf(nfp_prog, reg_a(2),
nfp_prog         3694 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
nfp_prog         3695 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf(nfp_prog, reg_b(2),
nfp_prog         3698 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
nfp_prog         3700 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf(nfp_prog, reg_b(2),
nfp_prog         3702 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
nfp_prog         3705 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_outro_xdp(struct nfp_prog *nfp_prog)
nfp_prog         3715 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
nfp_prog         3717 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
nfp_prog         3719 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
nfp_prog         3720 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16);
nfp_prog         3723 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog);
nfp_prog         3726 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_imm(3), ALU_OP_SUB, reg_b(0));
nfp_prog         3727 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0);
nfp_prog         3729 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_immed(nfp_prog, reg_b(2), 0x44112282);
nfp_prog         3731 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf(nfp_prog, reg_a(1),
nfp_prog         3734 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
nfp_prog         3735 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_shf(nfp_prog, reg_b(2),
nfp_prog         3738 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
nfp_prog         3740 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
nfp_prog         3741 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
nfp_prog         3744 drivers/net/ethernet/netronome/nfp/bpf/jit.c static bool nfp_prog_needs_callee_reg_save(struct nfp_prog *nfp_prog)
nfp_prog         3748 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	for (idx = 1; idx < nfp_prog->subprog_cnt; idx++)
nfp_prog         3749 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (nfp_prog->subprog[idx].needs_reg_push)
nfp_prog         3755 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_push_callee_registers(struct nfp_prog *nfp_prog)
nfp_prog         3762 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->tgt_call_push_regs = nfp_prog_current_offset(nfp_prog);
nfp_prog         3770 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, reg_lm(0, 1 + idx), reg_b(adj));
nfp_prog         3774 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			emit_rtn(nfp_prog, imm_b(nfp_prog), 3);
nfp_prog         3776 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, reg_lm(0, 1 + idx + 1), reg_b(adj + 1));
nfp_prog         3780 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_pop_callee_registers(struct nfp_prog *nfp_prog)
nfp_prog         3787 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->tgt_call_pop_regs = nfp_prog_current_offset(nfp_prog);
nfp_prog         3795 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, reg_both(adj), reg_lm(0, 1 + idx));
nfp_prog         3799 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			emit_rtn(nfp_prog, ret_reg(nfp_prog), 3);
nfp_prog         3801 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		wrp_mov(nfp_prog, reg_both(adj + 1), reg_lm(0, 1 + idx + 1));
nfp_prog         3805 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_outro(struct nfp_prog *nfp_prog)
nfp_prog         3807 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	switch (nfp_prog->type) {
nfp_prog         3809 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_outro_tc_da(nfp_prog);
nfp_prog         3812 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_outro_xdp(nfp_prog);
nfp_prog         3818 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (!nfp_prog_needs_callee_reg_save(nfp_prog))
nfp_prog         3821 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_push_callee_registers(nfp_prog);
nfp_prog         3822 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_pop_callee_registers(nfp_prog);
nfp_prog         3825 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int nfp_translate(struct nfp_prog *nfp_prog)
nfp_prog         3831 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	depth = nfp_prog->subprog[0].stack_depth;
nfp_prog         3832 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->stack_frame_depth = round_up(depth, 4);
nfp_prog         3834 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_intro(nfp_prog);
nfp_prog         3835 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (nfp_prog->error)
nfp_prog         3836 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return nfp_prog->error;
nfp_prog         3838 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	list_for_each_entry(meta, &nfp_prog->insns, l) {
nfp_prog         3841 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		meta->off = nfp_prog_current_offset(nfp_prog);
nfp_prog         3844 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			nfp_start_subprog(nfp_prog, meta);
nfp_prog         3845 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			if (nfp_prog->error)
nfp_prog         3846 drivers/net/ethernet/netronome/nfp/bpf/jit.c 				return nfp_prog->error;
nfp_prog         3850 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			nfp_prog->n_translated++;
nfp_prog         3854 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (nfp_meta_has_prev(nfp_prog, meta) &&
nfp_prog         3859 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		err = cb(nfp_prog, meta);
nfp_prog         3862 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		if (nfp_prog->error)
nfp_prog         3863 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			return nfp_prog->error;
nfp_prog         3865 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		nfp_prog->n_translated++;
nfp_prog         3868 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->last_bpf_off = nfp_prog_current_offset(nfp_prog) - 1;
nfp_prog         3870 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_outro(nfp_prog);
nfp_prog         3871 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (nfp_prog->error)
nfp_prog         3872 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return nfp_prog->error;
nfp_prog         3874 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	wrp_nops(nfp_prog, NFP_USTORE_PREFETCH_WINDOW);
nfp_prog         3875 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	if (nfp_prog->error)
nfp_prog         3876 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		return nfp_prog->error;
nfp_prog         3878 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	return nfp_fixup_branches(nfp_prog);
nfp_prog         3882 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog)
nfp_prog         3886 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	list_for_each_entry(meta, &nfp_prog->insns, l) {
nfp_prog         3908 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_bpf_opt_neg_add_sub(struct nfp_prog *nfp_prog)
nfp_prog         3912 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	list_for_each_entry(meta, &nfp_prog->insns, l) {
nfp_prog         3952 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog)
nfp_prog         3961 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {
nfp_prog         3991 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_bpf_opt_ld_shift(struct nfp_prog *nfp_prog)
nfp_prog         3995 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_for_each_insn_walk3(nfp_prog, meta1, meta2, meta3) {
nfp_prog         4180 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_bpf_opt_ldst_gather(struct nfp_prog *nfp_prog)
nfp_prog         4189 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {
nfp_prog         4270 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_bpf_opt_pkt_cache(struct nfp_prog *nfp_prog)
nfp_prog         4279 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	list_for_each_entry(meta, &nfp_prog->insns, l) {
nfp_prog         4368 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	list_for_each_entry(meta, &nfp_prog->insns, l) {
nfp_prog         4384 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int nfp_bpf_optimize(struct nfp_prog *nfp_prog)
nfp_prog         4386 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_bpf_opt_reg_init(nfp_prog);
nfp_prog         4388 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_bpf_opt_neg_add_sub(nfp_prog);
nfp_prog         4389 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_bpf_opt_ld_mask(nfp_prog);
nfp_prog         4390 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_bpf_opt_ld_shift(nfp_prog);
nfp_prog         4391 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_bpf_opt_ldst_gather(nfp_prog);
nfp_prog         4392 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_bpf_opt_pkt_cache(nfp_prog);
nfp_prog         4397 drivers/net/ethernet/netronome/nfp/bpf/jit.c static int nfp_bpf_replace_map_ptrs(struct nfp_prog *nfp_prog)
nfp_prog         4404 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {
nfp_prog         4447 drivers/net/ethernet/netronome/nfp/bpf/jit.c static void nfp_bpf_prog_trim(struct nfp_prog *nfp_prog)
nfp_prog         4451 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	prog = kvmalloc_array(nfp_prog->prog_len, sizeof(u64), GFP_KERNEL);
nfp_prog         4455 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->__prog_alloc_len = nfp_prog->prog_len * sizeof(u64);
nfp_prog         4456 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	memcpy(prog, nfp_prog->prog, nfp_prog->__prog_alloc_len);
nfp_prog         4457 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	kvfree(nfp_prog->prog);
nfp_prog         4458 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_prog->prog = prog;
nfp_prog         4461 drivers/net/ethernet/netronome/nfp/bpf/jit.c int nfp_bpf_jit(struct nfp_prog *nfp_prog)
nfp_prog         4465 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	ret = nfp_bpf_replace_map_ptrs(nfp_prog);
nfp_prog         4469 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	ret = nfp_bpf_optimize(nfp_prog);
nfp_prog         4473 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	ret = nfp_translate(nfp_prog);
nfp_prog         4476 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		       ret, nfp_prog->n_translated);
nfp_prog         4480 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	nfp_bpf_prog_trim(nfp_prog);
nfp_prog         4485 drivers/net/ethernet/netronome/nfp/bpf/jit.c void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog)
nfp_prog         4490 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	list_for_each_entry(meta, &nfp_prog->insns, l) {
nfp_prog         4513 drivers/net/ethernet/netronome/nfp/bpf/jit.c 		dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_idx);
nfp_prog         4528 drivers/net/ethernet/netronome/nfp/bpf/jit.c void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv)
nfp_prog         4534 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	prog = kmemdup(nfp_prog->prog, nfp_prog->prog_len * sizeof(u64),
nfp_prog         4539 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	for (i = 0; i < nfp_prog->prog_len; i++) {
nfp_prog         4553 drivers/net/ethernet/netronome/nfp/bpf/jit.c 				      nfp_prog->tgt_out + bv->start_off);
nfp_prog         4557 drivers/net/ethernet/netronome/nfp/bpf/jit.c 				      nfp_prog->tgt_abort + bv->start_off);
nfp_prog         4560 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			if (!nfp_prog->tgt_call_push_regs) {
nfp_prog         4565 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			off = nfp_prog->tgt_call_push_regs + bv->start_off;
nfp_prog         4569 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			if (!nfp_prog->tgt_call_pop_regs) {
nfp_prog         4574 drivers/net/ethernet/netronome/nfp/bpf/jit.c 			off = nfp_prog->tgt_call_pop_regs + bv->start_off;
nfp_prog         4585 drivers/net/ethernet/netronome/nfp/bpf/jit.c 				val = nfp_prog->bpf->helpers.map_lookup;
nfp_prog         4588 drivers/net/ethernet/netronome/nfp/bpf/jit.c 				val = nfp_prog->bpf->helpers.map_update;
nfp_prog         4591 drivers/net/ethernet/netronome/nfp/bpf/jit.c 				val = nfp_prog->bpf->helpers.map_delete;
nfp_prog         4594 drivers/net/ethernet/netronome/nfp/bpf/jit.c 				val = nfp_prog->bpf->helpers.perf_event_output;
nfp_prog         4612 drivers/net/ethernet/netronome/nfp/bpf/jit.c 	err = nfp_bpf_ustore_calc(prog, nfp_prog->prog_len);
nfp_prog          233 drivers/net/ethernet/netronome/nfp/bpf/main.h struct nfp_prog;
nfp_prog          235 drivers/net/ethernet/netronome/nfp/bpf/main.h typedef int (*instr_cb_t)(struct nfp_prog *, struct nfp_insn_meta *);
nfp_prog          237 drivers/net/ethernet/netronome/nfp/bpf/main.h #define nfp_prog_first_meta(nfp_prog)					\
nfp_prog          238 drivers/net/ethernet/netronome/nfp/bpf/main.h 	list_first_entry(&(nfp_prog)->insns, struct nfp_insn_meta, l)
nfp_prog          239 drivers/net/ethernet/netronome/nfp/bpf/main.h #define nfp_prog_last_meta(nfp_prog)					\
nfp_prog          240 drivers/net/ethernet/netronome/nfp/bpf/main.h 	list_last_entry(&(nfp_prog)->insns, struct nfp_insn_meta, l)
nfp_prog          560 drivers/net/ethernet/netronome/nfp/bpf/main.h void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog);
nfp_prog          561 drivers/net/ethernet/netronome/nfp/bpf/main.h int nfp_bpf_jit(struct nfp_prog *prog);
nfp_prog          584 drivers/net/ethernet/netronome/nfp/bpf/main.h nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog          587 drivers/net/ethernet/netronome/nfp/bpf/main.h void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv);
nfp_prog           31 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_map_ptr_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
nfp_prog           41 drivers/net/ethernet/netronome/nfp/bpf/offload.c 		nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
nfp_prog           68 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
nfp_prog           80 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_map_ptrs_forget(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog)
nfp_prog           85 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	for (i = 0; i < nfp_prog->map_records_cnt; i++) {
nfp_prog           86 drivers/net/ethernet/netronome/nfp/bpf/offload.c 		if (--nfp_prog->map_records[i]->count) {
nfp_prog           87 drivers/net/ethernet/netronome/nfp/bpf/offload.c 			nfp_prog->map_records[i] = NULL;
nfp_prog           92 drivers/net/ethernet/netronome/nfp/bpf/offload.c 					       &nfp_prog->map_records[i]->l,
nfp_prog          100 drivers/net/ethernet/netronome/nfp/bpf/offload.c 		for (i = 0; i < nfp_prog->map_records_cnt; i++)
nfp_prog          101 drivers/net/ethernet/netronome/nfp/bpf/offload.c 			if (nfp_prog->map_records[i]) {
nfp_prog          102 drivers/net/ethernet/netronome/nfp/bpf/offload.c 				bpf_map_put(nfp_prog->map_records[i]->ptr);
nfp_prog          103 drivers/net/ethernet/netronome/nfp/bpf/offload.c 				kfree(nfp_prog->map_records[i]);
nfp_prog          107 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	kfree(nfp_prog->map_records);
nfp_prog          108 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nfp_prog->map_records = NULL;
nfp_prog          109 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nfp_prog->map_records_cnt = 0;
nfp_prog          113 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_map_ptrs_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
nfp_prog          126 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nfp_prog->map_records = kmalloc_array(cnt,
nfp_prog          127 drivers/net/ethernet/netronome/nfp/bpf/offload.c 					      sizeof(nfp_prog->map_records[0]),
nfp_prog          129 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	if (!nfp_prog->map_records)
nfp_prog          134 drivers/net/ethernet/netronome/nfp/bpf/offload.c 			err = nfp_map_ptr_record(bpf, nfp_prog,
nfp_prog          137 drivers/net/ethernet/netronome/nfp/bpf/offload.c 				nfp_map_ptrs_forget(bpf, nfp_prog);
nfp_prog          141 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	WARN_ON(cnt != nfp_prog->map_records_cnt);
nfp_prog          147 drivers/net/ethernet/netronome/nfp/bpf/offload.c nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
nfp_prog          165 drivers/net/ethernet/netronome/nfp/bpf/offload.c 		list_add_tail(&meta->l, &nfp_prog->insns);
nfp_prog          167 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nfp_prog->n_insns = cnt;
nfp_prog          169 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nfp_bpf_jit_prepare(nfp_prog);
nfp_prog          174 drivers/net/ethernet/netronome/nfp/bpf/offload.c static void nfp_prog_free(struct nfp_prog *nfp_prog)
nfp_prog          178 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	kfree(nfp_prog->subprog);
nfp_prog          180 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) {
nfp_prog          184 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	kfree(nfp_prog);
nfp_prog          189 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	struct nfp_prog *nfp_prog;
nfp_prog          192 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
nfp_prog          193 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	if (!nfp_prog)
nfp_prog          195 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	prog->aux->offload->dev_priv = nfp_prog;
nfp_prog          197 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	INIT_LIST_HEAD(&nfp_prog->insns);
nfp_prog          198 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nfp_prog->type = prog->type;
nfp_prog          199 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nfp_prog->bpf = bpf_offload_dev_priv(prog->aux->offload->offdev);
nfp_prog          201 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len);
nfp_prog          205 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog);
nfp_prog          210 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nfp_prog_free(nfp_prog);
nfp_prog          218 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
nfp_prog          227 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nfp_prog->__prog_alloc_len = max_instr * sizeof(u64);
nfp_prog          229 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL);
nfp_prog          230 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	if (!nfp_prog->prog)
nfp_prog          233 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	err = nfp_bpf_jit(nfp_prog);
nfp_prog          237 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64);
nfp_prog          238 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	prog->aux->offload->jited_image = nfp_prog->prog;
nfp_prog          240 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	return nfp_map_ptrs_record(nfp_prog->bpf, nfp_prog, prog);
nfp_prog          245 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
nfp_prog          247 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	kvfree(nfp_prog->prog);
nfp_prog          248 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nfp_map_ptrs_forget(nfp_prog->bpf, nfp_prog);
nfp_prog          249 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nfp_prog_free(nfp_prog);
nfp_prog          484 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
nfp_prog          498 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	if (nfp_prog->stack_size > max_stack) {
nfp_prog          504 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	if (nfp_prog->prog_len > max_prog_len) {
nfp_prog          509 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv);
nfp_prog          514 drivers/net/ethernet/netronome/nfp/bpf/offload.c 				  nfp_prog->prog_len * sizeof(u64),
nfp_prog          521 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);
nfp_prog          530 drivers/net/ethernet/netronome/nfp/bpf/offload.c 	dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),
nfp_prog           20 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog           28 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	if (min(forward, backward) > nfp_prog->n_insns - insn_idx - 1) {
nfp_prog           29 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		backward = nfp_prog->n_insns - insn_idx - 1;
nfp_prog           30 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		meta = nfp_prog_last_meta(nfp_prog);
nfp_prog           34 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		meta = nfp_prog_first_meta(nfp_prog);
nfp_prog           48 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_record_adjust_head(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
nfp_prog           69 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	if (nfp_prog->adjust_head_location) {
nfp_prog           71 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		if (nfp_prog->adjust_head_location != meta->n)
nfp_prog           80 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	nfp_prog->adjust_head_location = location;
nfp_prog          170 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_check_helper_call(struct nfp_prog *nfp_prog,
nfp_prog          177 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	struct nfp_app_bpf *bpf = nfp_prog->bpf;
nfp_prog          191 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		nfp_record_adjust_head(bpf, nfp_prog, meta, reg2);
nfp_prog          284 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		dev_warn_once(&nfp_prog->bpf->app->pf->pdev->dev,
nfp_prog          311 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_check_exit(struct nfp_prog *nfp_prog,
nfp_prog          317 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	if (nfp_prog->type == BPF_PROG_TYPE_XDP)
nfp_prog          330 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	if (nfp_prog->type == BPF_PROG_TYPE_SCHED_CLS &&
nfp_prog          343 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_check_stack_access(struct nfp_prog *nfp_prog,
nfp_prog          451 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_check_ptr(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog          466 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		err = nfp_bpf_check_stack_access(nfp_prog, meta, reg, env);
nfp_prog          502 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_check_store(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog          508 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		if (nfp_prog->type == BPF_PROG_TYPE_XDP) {
nfp_prog          512 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 				if (nfp_prog->bpf->queue_select)
nfp_prog          522 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.dst_reg);
nfp_prog          526 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_check_xadd(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog          547 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.dst_reg);
nfp_prog          551 drivers/net/ethernet/netronome/nfp/bpf/verifier.c nfp_bpf_check_alu(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
nfp_prog          629 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;
nfp_prog          630 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	struct nfp_insn_meta *meta = nfp_prog->verifier_meta;
nfp_prog          632 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	meta = nfp_bpf_goto_meta(nfp_prog, meta, insn_idx);
nfp_prog          633 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	nfp_prog->verifier_meta = meta;
nfp_prog          648 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		return nfp_bpf_check_helper_call(nfp_prog, env, meta);
nfp_prog          650 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		return nfp_bpf_check_exit(nfp_prog, env);
nfp_prog          653 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		return nfp_bpf_check_ptr(nfp_prog, meta, env,
nfp_prog          656 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		return nfp_bpf_check_store(nfp_prog, meta, env);
nfp_prog          659 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		return nfp_bpf_check_xadd(nfp_prog, meta, env);
nfp_prog          662 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		return nfp_bpf_check_alu(nfp_prog, meta, env);
nfp_prog          669 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 				struct nfp_prog *nfp_prog)
nfp_prog          674 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	list_for_each_entry(meta, &nfp_prog->insns, l) {
nfp_prog          681 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 			nfp_prog->subprog[index].needs_reg_push = 1;
nfp_prog          684 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	if (index + 1 != nfp_prog->subprog_cnt) {
nfp_prog          686 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 			index + 1, nfp_prog->subprog_cnt);
nfp_prog          693 drivers/net/ethernet/netronome/nfp/bpf/verifier.c static unsigned int nfp_bpf_get_stack_usage(struct nfp_prog *nfp_prog)
nfp_prog          695 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	struct nfp_insn_meta *meta = nfp_prog_first_meta(nfp_prog);
nfp_prog          709 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	frame_depths[frame] = nfp_prog->subprog[idx].stack_depth;
nfp_prog          715 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	for (; meta != nfp_prog_last_meta(nfp_prog) && meta->subprog_idx == idx;
nfp_prog          727 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		meta = nfp_bpf_goto_meta(nfp_prog, meta,
nfp_prog          747 drivers/net/ethernet/netronome/nfp/bpf/verifier.c static void nfp_bpf_insn_flag_zext(struct nfp_prog *nfp_prog,
nfp_prog          752 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	list_for_each_entry(meta, &nfp_prog->insns, l) {
nfp_prog          761 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	struct nfp_prog *nfp_prog;
nfp_prog          766 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	nfp_prog = env->prog->aux->offload->dev_priv;
nfp_prog          767 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	nfp_prog->subprog_cnt = env->subprog_cnt;
nfp_prog          768 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	nfp_prog->subprog = kcalloc(nfp_prog->subprog_cnt,
nfp_prog          769 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 				    sizeof(nfp_prog->subprog[0]), GFP_KERNEL);
nfp_prog          770 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	if (!nfp_prog->subprog)
nfp_prog          773 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	nfp_assign_subprog_idx_and_regs(env, nfp_prog);
nfp_prog          776 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	for (i = 0; i < nfp_prog->subprog_cnt; i++) {
nfp_prog          777 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		nfp_prog->subprog[i].stack_depth = info[i].stack_depth;
nfp_prog          783 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		nfp_prog->subprog[i].stack_depth += REG_WIDTH;
nfp_prog          785 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		if (nfp_prog->subprog[i].needs_reg_push)
nfp_prog          786 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 			nfp_prog->subprog[i].stack_depth += BPF_REG_SIZE * 4;
nfp_prog          791 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	nfp_prog->stack_size = nfp_bpf_get_stack_usage(nfp_prog);
nfp_prog          792 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	if (nfp_prog->stack_size > max_stack) {
nfp_prog          794 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 			nfp_prog->stack_size, max_stack);
nfp_prog          798 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	nfp_bpf_insn_flag_zext(nfp_prog, env->insn_aux_data);
nfp_prog          805 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;
nfp_prog          807 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	struct nfp_insn_meta *meta = nfp_prog->verifier_meta;
nfp_prog          809 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	meta = nfp_bpf_goto_meta(nfp_prog, meta, aux_data[off].orig_idx);
nfp_prog          810 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	nfp_prog->verifier_meta = meta;
nfp_prog          838 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;
nfp_prog          840 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	struct nfp_insn_meta *meta = nfp_prog->verifier_meta;
nfp_prog          843 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 	meta = nfp_bpf_goto_meta(nfp_prog, meta, aux_data[off].orig_idx);
nfp_prog          846 drivers/net/ethernet/netronome/nfp/bpf/verifier.c 		if (WARN_ON_ONCE(&meta->l == &nfp_prog->insns))