arch/riscv/net/bpf_jit

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
bpf_to_rv_reg
seen_reg
mark_fp
mark_call
seen_call
mark_tail_call
seen_tail_call
rv_tail_call_reg
emit
rv_r_insn
rv_i_insn
rv_s_insn
rv_sb_insn
rv_u_insn
rv_uj_insn
rv_amo_insn
rv_addiw
rv_addi
rv_addw
rv_add
rv_subw
rv_sub
rv_and
rv_or
rv_xor
rv_mulw
rv_mul
rv_divuw
rv_divu
rv_remuw
rv_remu
rv_sllw
rv_sll
rv_srlw
rv_srl
rv_sraw
rv_sra
rv_lui
rv_slli
rv_andi
rv_ori
rv_xori
rv_slliw
rv_srliw
rv_srli
rv_sraiw
rv_srai
rv_jal
rv_jalr
rv_beq
rv_bltu
rv_bgeu
rv_bne
rv_blt
rv_bge
rv_sb
rv_sh
rv_sw
rv_sd
rv_lbu
rv_lhu
rv_lwu
rv_ld
rv_amoadd_w
rv_amoadd_d
is_12b_int
is_13b_int
is_21b_int
is_32b_int
is_12b_check
is_13b_check
is_21b_check
emit_imm
rv_offset
epilogue_offset
__build_epilogue
emit_zext_32
emit_bpf_tail_call
init_regs
rv_offset_check
emit_zext_32_rd_rs
emit_sext_32_rd_rs
emit_zext_32_rd_t1
emit_sext_32_rd
emit_insn
build_prologue
build_epilogue
build_body
bpf_fill_ill_insns
bpf_flush_icache
bpf_jit_needs_zext
bpf_int_jit_compile
   1 // SPDX-License-Identifier: GPL-2.0
   2 /* BPF JIT compiler for RV64G
   3  *
   4  * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com>
   5  *
   6  */
   7 
   8 #include <linux/bpf.h>
   9 #include <linux/filter.h>
  10 #include <asm/cacheflush.h>
  11 
  12 enum {
  13         RV_REG_ZERO =   0,      /* The constant value 0 */
  14         RV_REG_RA =     1,      /* Return address */
  15         RV_REG_SP =     2,      /* Stack pointer */
  16         RV_REG_GP =     3,      /* Global pointer */
  17         RV_REG_TP =     4,      /* Thread pointer */
  18         RV_REG_T0 =     5,      /* Temporaries */
  19         RV_REG_T1 =     6,
  20         RV_REG_T2 =     7,
  21         RV_REG_FP =     8,
  22         RV_REG_S1 =     9,      /* Saved registers */
  23         RV_REG_A0 =     10,     /* Function argument/return values */
  24         RV_REG_A1 =     11,     /* Function arguments */
  25         RV_REG_A2 =     12,
  26         RV_REG_A3 =     13,
  27         RV_REG_A4 =     14,
  28         RV_REG_A5 =     15,
  29         RV_REG_A6 =     16,
  30         RV_REG_A7 =     17,
  31         RV_REG_S2 =     18,     /* Saved registers */
  32         RV_REG_S3 =     19,
  33         RV_REG_S4 =     20,
  34         RV_REG_S5 =     21,
  35         RV_REG_S6 =     22,
  36         RV_REG_S7 =     23,
  37         RV_REG_S8 =     24,
  38         RV_REG_S9 =     25,
  39         RV_REG_S10 =    26,
  40         RV_REG_S11 =    27,
  41         RV_REG_T3 =     28,     /* Temporaries */
  42         RV_REG_T4 =     29,
  43         RV_REG_T5 =     30,
  44         RV_REG_T6 =     31,
  45 };
  46 
  47 #define RV_REG_TCC RV_REG_A6
  48 #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */
  49 
  50 static const int regmap[] = {
  51         [BPF_REG_0] =   RV_REG_A5,
  52         [BPF_REG_1] =   RV_REG_A0,
  53         [BPF_REG_2] =   RV_REG_A1,
  54         [BPF_REG_3] =   RV_REG_A2,
  55         [BPF_REG_4] =   RV_REG_A3,
  56         [BPF_REG_5] =   RV_REG_A4,
  57         [BPF_REG_6] =   RV_REG_S1,
  58         [BPF_REG_7] =   RV_REG_S2,
  59         [BPF_REG_8] =   RV_REG_S3,
  60         [BPF_REG_9] =   RV_REG_S4,
  61         [BPF_REG_FP] =  RV_REG_S5,
  62         [BPF_REG_AX] =  RV_REG_T0,
  63 };
  64 
  65 enum {
  66         RV_CTX_F_SEEN_TAIL_CALL =       0,
  67         RV_CTX_F_SEEN_CALL =            RV_REG_RA,
  68         RV_CTX_F_SEEN_S1 =              RV_REG_S1,
  69         RV_CTX_F_SEEN_S2 =              RV_REG_S2,
  70         RV_CTX_F_SEEN_S3 =              RV_REG_S3,
  71         RV_CTX_F_SEEN_S4 =              RV_REG_S4,
  72         RV_CTX_F_SEEN_S5 =              RV_REG_S5,
  73         RV_CTX_F_SEEN_S6 =              RV_REG_S6,
  74 };
  75 
  76 struct rv_jit_context {
  77         struct bpf_prog *prog;
  78         u32 *insns; /* RV insns */
  79         int ninsns;
  80         int epilogue_offset;
  81         int *offset; /* BPF to RV */
  82         unsigned long flags;
  83         int stack_size;
  84 };
  85 
  86 struct rv_jit_data {
  87         struct bpf_binary_header *header;
  88         u8 *image;
  89         struct rv_jit_context ctx;
  90 };
  91 
  92 static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx)
  93 {
  94         u8 reg = regmap[bpf_reg];
  95 
  96         switch (reg) {
  97         case RV_CTX_F_SEEN_S1:
  98         case RV_CTX_F_SEEN_S2:
  99         case RV_CTX_F_SEEN_S3:
 100         case RV_CTX_F_SEEN_S4:
 101         case RV_CTX_F_SEEN_S5:
 102         case RV_CTX_F_SEEN_S6:
 103                 __set_bit(reg, &ctx->flags);
 104         }
 105         return reg;
 106 };
 107 
 108 static bool seen_reg(int reg, struct rv_jit_context *ctx)
 109 {
 110         switch (reg) {
 111         case RV_CTX_F_SEEN_CALL:
 112         case RV_CTX_F_SEEN_S1:
 113         case RV_CTX_F_SEEN_S2:
 114         case RV_CTX_F_SEEN_S3:
 115         case RV_CTX_F_SEEN_S4:
 116         case RV_CTX_F_SEEN_S5:
 117         case RV_CTX_F_SEEN_S6:
 118                 return test_bit(reg, &ctx->flags);
 119         }
 120         return false;
 121 }
 122 
 123 static void mark_fp(struct rv_jit_context *ctx)
 124 {
 125         __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags);
 126 }
 127 
 128 static void mark_call(struct rv_jit_context *ctx)
 129 {
 130         __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
 131 }
 132 
 133 static bool seen_call(struct rv_jit_context *ctx)
 134 {
 135         return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
 136 }
 137 
 138 static void mark_tail_call(struct rv_jit_context *ctx)
 139 {
 140         __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
 141 }
 142 
 143 static bool seen_tail_call(struct rv_jit_context *ctx)
 144 {
 145         return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
 146 }
 147 
 148 static u8 rv_tail_call_reg(struct rv_jit_context *ctx)
 149 {
 150         mark_tail_call(ctx);
 151 
 152         if (seen_call(ctx)) {
 153                 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags);
 154                 return RV_REG_S6;
 155         }
 156         return RV_REG_A6;
 157 }
 158 
 159 static void emit(const u32 insn, struct rv_jit_context *ctx)
 160 {
 161         if (ctx->insns)
 162                 ctx->insns[ctx->ninsns] = insn;
 163 
 164         ctx->ninsns++;
 165 }
 166 
 167 static u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd, u8 opcode)
 168 {
 169         return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
 170                 (rd << 7) | opcode;
 171 }
 172 
 173 static u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode)
 174 {
 175         return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) |
 176                 opcode;
 177 }
 178 
 179 static u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
 180 {
 181         u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f;
 182 
 183         return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
 184                 (imm4_0 << 7) | opcode;
 185 }
 186 
 187 static u32 rv_sb_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
 188 {
 189         u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4);
 190         u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10);
 191 
 192         return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
 193                 (imm4_1 << 7) | opcode;
 194 }
 195 
 196 static u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode)
 197 {
 198         return (imm31_12 << 12) | (rd << 7) | opcode;
 199 }
 200 
 201 static u32 rv_uj_insn(u32 imm20_1, u8 rd, u8 opcode)
 202 {
 203         u32 imm;
 204 
 205         imm = (imm20_1 & 0x80000) |  ((imm20_1 & 0x3ff) << 9) |
 206               ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11);
 207 
 208         return (imm << 12) | (rd << 7) | opcode;
 209 }
 210 
 211 static u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1,
 212                        u8 funct3, u8 rd, u8 opcode)
 213 {
 214         u8 funct7 = (funct5 << 2) | (aq << 1) | rl;
 215 
 216         return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode);
 217 }
 218 
 219 static u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0)
 220 {
 221         return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b);
 222 }
 223 
 224 static u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0)
 225 {
 226         return rv_i_insn(imm11_0, rs1, 0, rd, 0x13);
 227 }
 228 
 229 static u32 rv_addw(u8 rd, u8 rs1, u8 rs2)
 230 {
 231         return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b);
 232 }
 233 
 234 static u32 rv_add(u8 rd, u8 rs1, u8 rs2)
 235 {
 236         return rv_r_insn(0, rs2, rs1, 0, rd, 0x33);
 237 }
 238 
 239 static u32 rv_subw(u8 rd, u8 rs1, u8 rs2)
 240 {
 241         return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b);
 242 }
 243 
 244 static u32 rv_sub(u8 rd, u8 rs1, u8 rs2)
 245 {
 246         return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33);
 247 }
 248 
 249 static u32 rv_and(u8 rd, u8 rs1, u8 rs2)
 250 {
 251         return rv_r_insn(0, rs2, rs1, 7, rd, 0x33);
 252 }
 253 
 254 static u32 rv_or(u8 rd, u8 rs1, u8 rs2)
 255 {
 256         return rv_r_insn(0, rs2, rs1, 6, rd, 0x33);
 257 }
 258 
 259 static u32 rv_xor(u8 rd, u8 rs1, u8 rs2)
 260 {
 261         return rv_r_insn(0, rs2, rs1, 4, rd, 0x33);
 262 }
 263 
 264 static u32 rv_mulw(u8 rd, u8 rs1, u8 rs2)
 265 {
 266         return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b);
 267 }
 268 
 269 static u32 rv_mul(u8 rd, u8 rs1, u8 rs2)
 270 {
 271         return rv_r_insn(1, rs2, rs1, 0, rd, 0x33);
 272 }
 273 
 274 static u32 rv_divuw(u8 rd, u8 rs1, u8 rs2)
 275 {
 276         return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b);
 277 }
 278 
 279 static u32 rv_divu(u8 rd, u8 rs1, u8 rs2)
 280 {
 281         return rv_r_insn(1, rs2, rs1, 5, rd, 0x33);
 282 }
 283 
 284 static u32 rv_remuw(u8 rd, u8 rs1, u8 rs2)
 285 {
 286         return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b);
 287 }
 288 
 289 static u32 rv_remu(u8 rd, u8 rs1, u8 rs2)
 290 {
 291         return rv_r_insn(1, rs2, rs1, 7, rd, 0x33);
 292 }
 293 
 294 static u32 rv_sllw(u8 rd, u8 rs1, u8 rs2)
 295 {
 296         return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b);
 297 }
 298 
 299 static u32 rv_sll(u8 rd, u8 rs1, u8 rs2)
 300 {
 301         return rv_r_insn(0, rs2, rs1, 1, rd, 0x33);
 302 }
 303 
 304 static u32 rv_srlw(u8 rd, u8 rs1, u8 rs2)
 305 {
 306         return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b);
 307 }
 308 
 309 static u32 rv_srl(u8 rd, u8 rs1, u8 rs2)
 310 {
 311         return rv_r_insn(0, rs2, rs1, 5, rd, 0x33);
 312 }
 313 
 314 static u32 rv_sraw(u8 rd, u8 rs1, u8 rs2)
 315 {
 316         return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b);
 317 }
 318 
 319 static u32 rv_sra(u8 rd, u8 rs1, u8 rs2)
 320 {
 321         return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33);
 322 }
 323 
 324 static u32 rv_lui(u8 rd, u32 imm31_12)
 325 {
 326         return rv_u_insn(imm31_12, rd, 0x37);
 327 }
 328 
 329 static u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0)
 330 {
 331         return rv_i_insn(imm11_0, rs1, 1, rd, 0x13);
 332 }
 333 
 334 static u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0)
 335 {
 336         return rv_i_insn(imm11_0, rs1, 7, rd, 0x13);
 337 }
 338 
 339 static u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0)
 340 {
 341         return rv_i_insn(imm11_0, rs1, 6, rd, 0x13);
 342 }
 343 
 344 static u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0)
 345 {
 346         return rv_i_insn(imm11_0, rs1, 4, rd, 0x13);
 347 }
 348 
 349 static u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0)
 350 {
 351         return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b);
 352 }
 353 
 354 static u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0)
 355 {
 356         return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b);
 357 }
 358 
 359 static u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0)
 360 {
 361         return rv_i_insn(imm11_0, rs1, 5, rd, 0x13);
 362 }
 363 
 364 static u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0)
 365 {
 366         return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b);
 367 }
 368 
 369 static u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0)
 370 {
 371         return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13);
 372 }
 373 
 374 static u32 rv_jal(u8 rd, u32 imm20_1)
 375 {
 376         return rv_uj_insn(imm20_1, rd, 0x6f);
 377 }
 378 
 379 static u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0)
 380 {
 381         return rv_i_insn(imm11_0, rs1, 0, rd, 0x67);
 382 }
 383 
 384 static u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1)
 385 {
 386         return rv_sb_insn(imm12_1, rs2, rs1, 0, 0x63);
 387 }
 388 
 389 static u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1)
 390 {
 391         return rv_sb_insn(imm12_1, rs2, rs1, 6, 0x63);
 392 }
 393 
 394 static u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1)
 395 {
 396         return rv_sb_insn(imm12_1, rs2, rs1, 7, 0x63);
 397 }
 398 
 399 static u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1)
 400 {
 401         return rv_sb_insn(imm12_1, rs2, rs1, 1, 0x63);
 402 }
 403 
 404 static u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1)
 405 {
 406         return rv_sb_insn(imm12_1, rs2, rs1, 4, 0x63);
 407 }
 408 
 409 static u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1)
 410 {
 411         return rv_sb_insn(imm12_1, rs2, rs1, 5, 0x63);
 412 }
 413 
 414 static u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2)
 415 {
 416         return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23);
 417 }
 418 
 419 static u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2)
 420 {
 421         return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23);
 422 }
 423 
 424 static u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2)
 425 {
 426         return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23);
 427 }
 428 
 429 static u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2)
 430 {
 431         return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23);
 432 }
 433 
 434 static u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1)
 435 {
 436         return rv_i_insn(imm11_0, rs1, 4, rd, 0x03);
 437 }
 438 
 439 static u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1)
 440 {
 441         return rv_i_insn(imm11_0, rs1, 5, rd, 0x03);
 442 }
 443 
 444 static u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1)
 445 {
 446         return rv_i_insn(imm11_0, rs1, 6, rd, 0x03);
 447 }
 448 
 449 static u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1)
 450 {
 451         return rv_i_insn(imm11_0, rs1, 3, rd, 0x03);
 452 }
 453 
 454 static u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 455 {
 456         return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f);
 457 }
 458 
 459 static u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 460 {
 461         return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);
 462 }
 463 
 464 static bool is_12b_int(s64 val)
 465 {
 466         return -(1 << 11) <= val && val < (1 << 11);
 467 }
 468 
 469 static bool is_13b_int(s64 val)
 470 {
 471         return -(1 << 12) <= val && val < (1 << 12);
 472 }
 473 
 474 static bool is_21b_int(s64 val)
 475 {
 476         return -(1L << 20) <= val && val < (1L << 20);
 477 }
 478 
 479 static bool is_32b_int(s64 val)
 480 {
 481         return -(1L << 31) <= val && val < (1L << 31);
 482 }
 483 
 484 static int is_12b_check(int off, int insn)
 485 {
 486         if (!is_12b_int(off)) {
 487                 pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n",
 488                        insn, (int)off);
 489                 return -1;
 490         }
 491         return 0;
 492 }
 493 
 494 static int is_13b_check(int off, int insn)
 495 {
 496         if (!is_13b_int(off)) {
 497                 pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n",
 498                        insn, (int)off);
 499                 return -1;
 500         }
 501         return 0;
 502 }
 503 
 504 static int is_21b_check(int off, int insn)
 505 {
 506         if (!is_21b_int(off)) {
 507                 pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n",
 508                        insn, (int)off);
 509                 return -1;
 510         }
 511         return 0;
 512 }
 513 
 514 static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx)
 515 {
 516         /* Note that the immediate from the add is sign-extended,
 517          * which means that we need to compensate this by adding 2^12,
 518          * when the 12th bit is set. A simpler way of doing this, and
 519          * getting rid of the check, is to just add 2**11 before the
 520          * shift. The "Loading a 32-Bit constant" example from the
 521          * "Computer Organization and Design, RISC-V edition" book by
 522          * Patterson/Hennessy highlights this fact.
 523          *
 524          * This also means that we need to process LSB to MSB.
 525          */
 526         s64 upper = (val + (1 << 11)) >> 12, lower = val & 0xfff;
 527         int shift;
 528 
 529         if (is_32b_int(val)) {
 530                 if (upper)
 531                         emit(rv_lui(rd, upper), ctx);
 532 
 533                 if (!upper) {
 534                         emit(rv_addi(rd, RV_REG_ZERO, lower), ctx);
 535                         return;
 536                 }
 537 
 538                 emit(rv_addiw(rd, rd, lower), ctx);
 539                 return;
 540         }
 541 
 542         shift = __ffs(upper);
 543         upper >>= shift;
 544         shift += 12;
 545 
 546         emit_imm(rd, upper, ctx);
 547 
 548         emit(rv_slli(rd, rd, shift), ctx);
 549         if (lower)
 550                 emit(rv_addi(rd, rd, lower), ctx);
 551 }
 552 
 553 static int rv_offset(int bpf_to, int bpf_from, struct rv_jit_context *ctx)
 554 {
 555         int from = ctx->offset[bpf_from] - 1, to = ctx->offset[bpf_to];
 556 
 557         return (to - from) << 2;
 558 }
 559 
 560 static int epilogue_offset(struct rv_jit_context *ctx)
 561 {
 562         int to = ctx->epilogue_offset, from = ctx->ninsns;
 563 
 564         return (to - from) << 2;
 565 }
 566 
 567 static void __build_epilogue(u8 reg, struct rv_jit_context *ctx)
 568 {
 569         int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8;
 570 
 571         if (seen_reg(RV_REG_RA, ctx)) {
 572                 emit(rv_ld(RV_REG_RA, store_offset, RV_REG_SP), ctx);
 573                 store_offset -= 8;
 574         }
 575         emit(rv_ld(RV_REG_FP, store_offset, RV_REG_SP), ctx);
 576         store_offset -= 8;
 577         if (seen_reg(RV_REG_S1, ctx)) {
 578                 emit(rv_ld(RV_REG_S1, store_offset, RV_REG_SP), ctx);
 579                 store_offset -= 8;
 580         }
 581         if (seen_reg(RV_REG_S2, ctx)) {
 582                 emit(rv_ld(RV_REG_S2, store_offset, RV_REG_SP), ctx);
 583                 store_offset -= 8;
 584         }
 585         if (seen_reg(RV_REG_S3, ctx)) {
 586                 emit(rv_ld(RV_REG_S3, store_offset, RV_REG_SP), ctx);
 587                 store_offset -= 8;
 588         }
 589         if (seen_reg(RV_REG_S4, ctx)) {
 590                 emit(rv_ld(RV_REG_S4, store_offset, RV_REG_SP), ctx);
 591                 store_offset -= 8;
 592         }
 593         if (seen_reg(RV_REG_S5, ctx)) {
 594                 emit(rv_ld(RV_REG_S5, store_offset, RV_REG_SP), ctx);
 595                 store_offset -= 8;
 596         }
 597         if (seen_reg(RV_REG_S6, ctx)) {
 598                 emit(rv_ld(RV_REG_S6, store_offset, RV_REG_SP), ctx);
 599                 store_offset -= 8;
 600         }
 601 
 602         emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx);
 603         /* Set return value. */
 604         if (reg == RV_REG_RA)
 605                 emit(rv_addi(RV_REG_A0, RV_REG_A5, 0), ctx);
 606         emit(rv_jalr(RV_REG_ZERO, reg, 0), ctx);
 607 }
 608 
 609 static void emit_zext_32(u8 reg, struct rv_jit_context *ctx)
 610 {
 611         emit(rv_slli(reg, reg, 32), ctx);
 612         emit(rv_srli(reg, reg, 32), ctx);
 613 }
 614 
 615 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
 616 {
 617         int tc_ninsn, off, start_insn = ctx->ninsns;
 618         u8 tcc = rv_tail_call_reg(ctx);
 619 
 620         /* a0: &ctx
 621          * a1: &array
 622          * a2: index
 623          *
 624          * if (index >= array->map.max_entries)
 625          *      goto out;
 626          */
 627         tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] :
 628                    ctx->offset[0];
 629         emit_zext_32(RV_REG_A2, ctx);
 630 
 631         off = offsetof(struct bpf_array, map.max_entries);
 632         if (is_12b_check(off, insn))
 633                 return -1;
 634         emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx);
 635         off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
 636         if (is_13b_check(off, insn))
 637                 return -1;
 638         emit(rv_bgeu(RV_REG_A2, RV_REG_T1, off >> 1), ctx);
 639 
 640         /* if (TCC-- < 0)
 641          *     goto out;
 642          */
 643         emit(rv_addi(RV_REG_T1, tcc, -1), ctx);
 644         off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
 645         if (is_13b_check(off, insn))
 646                 return -1;
 647         emit(rv_blt(tcc, RV_REG_ZERO, off >> 1), ctx);
 648 
 649         /* prog = array->ptrs[index];
 650          * if (!prog)
 651          *     goto out;
 652          */
 653         emit(rv_slli(RV_REG_T2, RV_REG_A2, 3), ctx);
 654         emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_A1), ctx);
 655         off = offsetof(struct bpf_array, ptrs);
 656         if (is_12b_check(off, insn))
 657                 return -1;
 658         emit(rv_ld(RV_REG_T2, off, RV_REG_T2), ctx);
 659         off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
 660         if (is_13b_check(off, insn))
 661                 return -1;
 662         emit(rv_beq(RV_REG_T2, RV_REG_ZERO, off >> 1), ctx);
 663 
 664         /* goto *(prog->bpf_func + 4); */
 665         off = offsetof(struct bpf_prog, bpf_func);
 666         if (is_12b_check(off, insn))
 667                 return -1;
 668         emit(rv_ld(RV_REG_T3, off, RV_REG_T2), ctx);
 669         emit(rv_addi(RV_REG_T3, RV_REG_T3, 4), ctx);
 670         emit(rv_addi(RV_REG_TCC, RV_REG_T1, 0), ctx);
 671         __build_epilogue(RV_REG_T3, ctx);
 672         return 0;
 673 }
 674 
 675 static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn,
 676                       struct rv_jit_context *ctx)
 677 {
 678         u8 code = insn->code;
 679 
 680         switch (code) {
 681         case BPF_JMP | BPF_JA:
 682         case BPF_JMP | BPF_CALL:
 683         case BPF_JMP | BPF_EXIT:
 684         case BPF_JMP | BPF_TAIL_CALL:
 685                 break;
 686         default:
 687                 *rd = bpf_to_rv_reg(insn->dst_reg, ctx);
 688         }
 689 
 690         if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) ||
 691             code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) ||
 692             code & BPF_LDX || code & BPF_STX)
 693                 *rs = bpf_to_rv_reg(insn->src_reg, ctx);
 694 }
 695 
 696 static int rv_offset_check(int *rvoff, s16 off, int insn,
 697                            struct rv_jit_context *ctx)
 698 {
 699         *rvoff = rv_offset(insn + off, insn, ctx);
 700         return is_13b_check(*rvoff, insn);
 701 }
 702 
 703 static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)
 704 {
 705         emit(rv_addi(RV_REG_T2, *rd, 0), ctx);
 706         emit_zext_32(RV_REG_T2, ctx);
 707         emit(rv_addi(RV_REG_T1, *rs, 0), ctx);
 708         emit_zext_32(RV_REG_T1, ctx);
 709         *rd = RV_REG_T2;
 710         *rs = RV_REG_T1;
 711 }
 712 
 713 static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)
 714 {
 715         emit(rv_addiw(RV_REG_T2, *rd, 0), ctx);
 716         emit(rv_addiw(RV_REG_T1, *rs, 0), ctx);
 717         *rd = RV_REG_T2;
 718         *rs = RV_REG_T1;
 719 }
 720 
 721 static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx)
 722 {
 723         emit(rv_addi(RV_REG_T2, *rd, 0), ctx);
 724         emit_zext_32(RV_REG_T2, ctx);
 725         emit_zext_32(RV_REG_T1, ctx);
 726         *rd = RV_REG_T2;
 727 }
 728 
 729 static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx)
 730 {
 731         emit(rv_addiw(RV_REG_T2, *rd, 0), ctx);
 732         *rd = RV_REG_T2;
 733 }
 734 
 735 static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
 736                      bool extra_pass)
 737 {
 738         bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 ||
 739                     BPF_CLASS(insn->code) == BPF_JMP;
 740         struct bpf_prog_aux *aux = ctx->prog->aux;
 741         int rvoff, i = insn - ctx->prog->insnsi;
 742         u8 rd = -1, rs = -1, code = insn->code;
 743         s16 off = insn->off;
 744         s32 imm = insn->imm;
 745 
 746         init_regs(&rd, &rs, insn, ctx);
 747 
 748         switch (code) {
 749         /* dst = src */
 750         case BPF_ALU | BPF_MOV | BPF_X:
 751         case BPF_ALU64 | BPF_MOV | BPF_X:
 752                 if (imm == 1) {
 753                         /* Special mov32 for zext */
 754                         emit_zext_32(rd, ctx);
 755                         break;
 756                 }
 757                 emit(is64 ? rv_addi(rd, rs, 0) : rv_addiw(rd, rs, 0), ctx);
 758                 if (!is64 && !aux->verifier_zext)
 759                         emit_zext_32(rd, ctx);
 760                 break;
 761 
 762         /* dst = dst OP src */
 763         case BPF_ALU | BPF_ADD | BPF_X:
 764         case BPF_ALU64 | BPF_ADD | BPF_X:
 765                 emit(is64 ? rv_add(rd, rd, rs) : rv_addw(rd, rd, rs), ctx);
 766                 if (!is64 && !aux->verifier_zext)
 767                         emit_zext_32(rd, ctx);
 768                 break;
 769         case BPF_ALU | BPF_SUB | BPF_X:
 770         case BPF_ALU64 | BPF_SUB | BPF_X:
 771                 emit(is64 ? rv_sub(rd, rd, rs) : rv_subw(rd, rd, rs), ctx);
 772                 if (!is64 && !aux->verifier_zext)
 773                         emit_zext_32(rd, ctx);
 774                 break;
 775         case BPF_ALU | BPF_AND | BPF_X:
 776         case BPF_ALU64 | BPF_AND | BPF_X:
 777                 emit(rv_and(rd, rd, rs), ctx);
 778                 if (!is64 && !aux->verifier_zext)
 779                         emit_zext_32(rd, ctx);
 780                 break;
 781         case BPF_ALU | BPF_OR | BPF_X:
 782         case BPF_ALU64 | BPF_OR | BPF_X:
 783                 emit(rv_or(rd, rd, rs), ctx);
 784                 if (!is64 && !aux->verifier_zext)
 785                         emit_zext_32(rd, ctx);
 786                 break;
 787         case BPF_ALU | BPF_XOR | BPF_X:
 788         case BPF_ALU64 | BPF_XOR | BPF_X:
 789                 emit(rv_xor(rd, rd, rs), ctx);
 790                 if (!is64 && !aux->verifier_zext)
 791                         emit_zext_32(rd, ctx);
 792                 break;
 793         case BPF_ALU | BPF_MUL | BPF_X:
 794         case BPF_ALU64 | BPF_MUL | BPF_X:
 795                 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx);
 796                 if (!is64 && !aux->verifier_zext)
 797                         emit_zext_32(rd, ctx);
 798                 break;
 799         case BPF_ALU | BPF_DIV | BPF_X:
 800         case BPF_ALU64 | BPF_DIV | BPF_X:
 801                 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx);
 802                 if (!is64 && !aux->verifier_zext)
 803                         emit_zext_32(rd, ctx);
 804                 break;
 805         case BPF_ALU | BPF_MOD | BPF_X:
 806         case BPF_ALU64 | BPF_MOD | BPF_X:
 807                 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx);
 808                 if (!is64 && !aux->verifier_zext)
 809                         emit_zext_32(rd, ctx);
 810                 break;
 811         case BPF_ALU | BPF_LSH | BPF_X:
 812         case BPF_ALU64 | BPF_LSH | BPF_X:
 813                 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx);
 814                 if (!is64)
 815                         emit_zext_32(rd, ctx);
 816                 break;
 817         case BPF_ALU | BPF_RSH | BPF_X:
 818         case BPF_ALU64 | BPF_RSH | BPF_X:
 819                 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx);
 820                 if (!is64 && !aux->verifier_zext)
 821                         emit_zext_32(rd, ctx);
 822                 break;
 823         case BPF_ALU | BPF_ARSH | BPF_X:
 824         case BPF_ALU64 | BPF_ARSH | BPF_X:
 825                 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx);
 826                 if (!is64 && !aux->verifier_zext)
 827                         emit_zext_32(rd, ctx);
 828                 break;
 829 
 830         /* dst = -dst */
 831         case BPF_ALU | BPF_NEG:
 832         case BPF_ALU64 | BPF_NEG:
 833                 emit(is64 ? rv_sub(rd, RV_REG_ZERO, rd) :
 834                      rv_subw(rd, RV_REG_ZERO, rd), ctx);
 835                 if (!is64 && !aux->verifier_zext)
 836                         emit_zext_32(rd, ctx);
 837                 break;
 838 
 839         /* dst = BSWAP##imm(dst) */
 840         case BPF_ALU | BPF_END | BPF_FROM_LE:
 841         {
 842                 int shift = 64 - imm;
 843 
 844                 emit(rv_slli(rd, rd, shift), ctx);
 845                 emit(rv_srli(rd, rd, shift), ctx);
 846                 break;
 847         }
 848         case BPF_ALU | BPF_END | BPF_FROM_BE:
 849                 emit(rv_addi(RV_REG_T2, RV_REG_ZERO, 0), ctx);
 850 
 851                 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
 852                 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
 853                 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
 854                 emit(rv_srli(rd, rd, 8), ctx);
 855                 if (imm == 16)
 856                         goto out_be;
 857 
 858                 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
 859                 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
 860                 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
 861                 emit(rv_srli(rd, rd, 8), ctx);
 862 
 863                 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
 864                 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
 865                 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
 866                 emit(rv_srli(rd, rd, 8), ctx);
 867                 if (imm == 32)
 868                         goto out_be;
 869 
 870                 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
 871                 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
 872                 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
 873                 emit(rv_srli(rd, rd, 8), ctx);
 874 
 875                 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
 876                 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
 877                 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
 878                 emit(rv_srli(rd, rd, 8), ctx);
 879 
 880                 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
 881                 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
 882                 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
 883                 emit(rv_srli(rd, rd, 8), ctx);
 884 
 885                 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
 886                 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
 887                 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
 888                 emit(rv_srli(rd, rd, 8), ctx);
 889 out_be:
 890                 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
 891                 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
 892 
 893                 emit(rv_addi(rd, RV_REG_T2, 0), ctx);
 894                 break;
 895 
 896         /* dst = imm */
 897         case BPF_ALU | BPF_MOV | BPF_K:
 898         case BPF_ALU64 | BPF_MOV | BPF_K:
 899                 emit_imm(rd, imm, ctx);
 900                 if (!is64 && !aux->verifier_zext)
 901                         emit_zext_32(rd, ctx);
 902                 break;
 903 
 904         /* dst = dst OP imm */
 905         case BPF_ALU | BPF_ADD | BPF_K:
 906         case BPF_ALU64 | BPF_ADD | BPF_K:
 907                 if (is_12b_int(imm)) {
 908                         emit(is64 ? rv_addi(rd, rd, imm) :
 909                              rv_addiw(rd, rd, imm), ctx);
 910                 } else {
 911                         emit_imm(RV_REG_T1, imm, ctx);
 912                         emit(is64 ? rv_add(rd, rd, RV_REG_T1) :
 913                              rv_addw(rd, rd, RV_REG_T1), ctx);
 914                 }
 915                 if (!is64 && !aux->verifier_zext)
 916                         emit_zext_32(rd, ctx);
 917                 break;
 918         case BPF_ALU | BPF_SUB | BPF_K:
 919         case BPF_ALU64 | BPF_SUB | BPF_K:
 920                 if (is_12b_int(-imm)) {
 921                         emit(is64 ? rv_addi(rd, rd, -imm) :
 922                              rv_addiw(rd, rd, -imm), ctx);
 923                 } else {
 924                         emit_imm(RV_REG_T1, imm, ctx);
 925                         emit(is64 ? rv_sub(rd, rd, RV_REG_T1) :
 926                              rv_subw(rd, rd, RV_REG_T1), ctx);
 927                 }
 928                 if (!is64 && !aux->verifier_zext)
 929                         emit_zext_32(rd, ctx);
 930                 break;
 931         case BPF_ALU | BPF_AND | BPF_K:
 932         case BPF_ALU64 | BPF_AND | BPF_K:
 933                 if (is_12b_int(imm)) {
 934                         emit(rv_andi(rd, rd, imm), ctx);
 935                 } else {
 936                         emit_imm(RV_REG_T1, imm, ctx);
 937                         emit(rv_and(rd, rd, RV_REG_T1), ctx);
 938                 }
 939                 if (!is64 && !aux->verifier_zext)
 940                         emit_zext_32(rd, ctx);
 941                 break;
 942         case BPF_ALU | BPF_OR | BPF_K:
 943         case BPF_ALU64 | BPF_OR | BPF_K:
 944                 if (is_12b_int(imm)) {
 945                         emit(rv_ori(rd, rd, imm), ctx);
 946                 } else {
 947                         emit_imm(RV_REG_T1, imm, ctx);
 948                         emit(rv_or(rd, rd, RV_REG_T1), ctx);
 949                 }
 950                 if (!is64 && !aux->verifier_zext)
 951                         emit_zext_32(rd, ctx);
 952                 break;
 953         case BPF_ALU | BPF_XOR | BPF_K:
 954         case BPF_ALU64 | BPF_XOR | BPF_K:
 955                 if (is_12b_int(imm)) {
 956                         emit(rv_xori(rd, rd, imm), ctx);
 957                 } else {
 958                         emit_imm(RV_REG_T1, imm, ctx);
 959                         emit(rv_xor(rd, rd, RV_REG_T1), ctx);
 960                 }
 961                 if (!is64 && !aux->verifier_zext)
 962                         emit_zext_32(rd, ctx);
 963                 break;
 964         case BPF_ALU | BPF_MUL | BPF_K:
 965         case BPF_ALU64 | BPF_MUL | BPF_K:
 966                 emit_imm(RV_REG_T1, imm, ctx);
 967                 emit(is64 ? rv_mul(rd, rd, RV_REG_T1) :
 968                      rv_mulw(rd, rd, RV_REG_T1), ctx);
 969                 if (!is64 && !aux->verifier_zext)
 970                         emit_zext_32(rd, ctx);
 971                 break;
 972         case BPF_ALU | BPF_DIV | BPF_K:
 973         case BPF_ALU64 | BPF_DIV | BPF_K:
 974                 emit_imm(RV_REG_T1, imm, ctx);
 975                 emit(is64 ? rv_divu(rd, rd, RV_REG_T1) :
 976                      rv_divuw(rd, rd, RV_REG_T1), ctx);
 977                 if (!is64 && !aux->verifier_zext)
 978                         emit_zext_32(rd, ctx);
 979                 break;
 980         case BPF_ALU | BPF_MOD | BPF_K:
 981         case BPF_ALU64 | BPF_MOD | BPF_K:
 982                 emit_imm(RV_REG_T1, imm, ctx);
 983                 emit(is64 ? rv_remu(rd, rd, RV_REG_T1) :
 984                      rv_remuw(rd, rd, RV_REG_T1), ctx);
 985                 if (!is64 && !aux->verifier_zext)
 986                         emit_zext_32(rd, ctx);
 987                 break;
 988         case BPF_ALU | BPF_LSH | BPF_K:
 989         case BPF_ALU64 | BPF_LSH | BPF_K:
 990                 emit(is64 ? rv_slli(rd, rd, imm) : rv_slliw(rd, rd, imm), ctx);
 991                 if (!is64)
 992                         emit_zext_32(rd, ctx);
 993                 break;
 994         case BPF_ALU | BPF_RSH | BPF_K:
 995         case BPF_ALU64 | BPF_RSH | BPF_K:
 996                 emit(is64 ? rv_srli(rd, rd, imm) : rv_srliw(rd, rd, imm), ctx);
 997                 if (!is64)
 998                         emit_zext_32(rd, ctx);
 999                 break;
1000         case BPF_ALU | BPF_ARSH | BPF_K:
1001         case BPF_ALU64 | BPF_ARSH | BPF_K:
1002                 emit(is64 ? rv_srai(rd, rd, imm) : rv_sraiw(rd, rd, imm), ctx);
1003                 if (!is64)
1004                         emit_zext_32(rd, ctx);
1005                 break;
1006 
1007         /* JUMP off */
1008         case BPF_JMP | BPF_JA:
1009                 rvoff = rv_offset(i + off, i, ctx);
1010                 if (!is_21b_int(rvoff)) {
1011                         pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n",
1012                                i, rvoff);
1013                         return -1;
1014                 }
1015 
1016                 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx);
1017                 break;
1018 
1019         /* IF (dst COND src) JUMP off */
1020         case BPF_JMP | BPF_JEQ | BPF_X:
1021         case BPF_JMP32 | BPF_JEQ | BPF_X:
1022                 if (rv_offset_check(&rvoff, off, i, ctx))
1023                         return -1;
1024                 if (!is64)
1025                         emit_zext_32_rd_rs(&rd, &rs, ctx);
1026                 emit(rv_beq(rd, rs, rvoff >> 1), ctx);
1027                 break;
1028         case BPF_JMP | BPF_JGT | BPF_X:
1029         case BPF_JMP32 | BPF_JGT | BPF_X:
1030                 if (rv_offset_check(&rvoff, off, i, ctx))
1031                         return -1;
1032                 if (!is64)
1033                         emit_zext_32_rd_rs(&rd, &rs, ctx);
1034                 emit(rv_bltu(rs, rd, rvoff >> 1), ctx);
1035                 break;
1036         case BPF_JMP | BPF_JLT | BPF_X:
1037         case BPF_JMP32 | BPF_JLT | BPF_X:
1038                 if (rv_offset_check(&rvoff, off, i, ctx))
1039                         return -1;
1040                 if (!is64)
1041                         emit_zext_32_rd_rs(&rd, &rs, ctx);
1042                 emit(rv_bltu(rd, rs, rvoff >> 1), ctx);
1043                 break;
1044         case BPF_JMP | BPF_JGE | BPF_X:
1045         case BPF_JMP32 | BPF_JGE | BPF_X:
1046                 if (rv_offset_check(&rvoff, off, i, ctx))
1047                         return -1;
1048                 if (!is64)
1049                         emit_zext_32_rd_rs(&rd, &rs, ctx);
1050                 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx);
1051                 break;
1052         case BPF_JMP | BPF_JLE | BPF_X:
1053         case BPF_JMP32 | BPF_JLE | BPF_X:
1054                 if (rv_offset_check(&rvoff, off, i, ctx))
1055                         return -1;
1056                 if (!is64)
1057                         emit_zext_32_rd_rs(&rd, &rs, ctx);
1058                 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx);
1059                 break;
1060         case BPF_JMP | BPF_JNE | BPF_X:
1061         case BPF_JMP32 | BPF_JNE | BPF_X:
1062                 if (rv_offset_check(&rvoff, off, i, ctx))
1063                         return -1;
1064                 if (!is64)
1065                         emit_zext_32_rd_rs(&rd, &rs, ctx);
1066                 emit(rv_bne(rd, rs, rvoff >> 1), ctx);
1067                 break;
1068         case BPF_JMP | BPF_JSGT | BPF_X:
1069         case BPF_JMP32 | BPF_JSGT | BPF_X:
1070                 if (rv_offset_check(&rvoff, off, i, ctx))
1071                         return -1;
1072                 if (!is64)
1073                         emit_sext_32_rd_rs(&rd, &rs, ctx);
1074                 emit(rv_blt(rs, rd, rvoff >> 1), ctx);
1075                 break;
1076         case BPF_JMP | BPF_JSLT | BPF_X:
1077         case BPF_JMP32 | BPF_JSLT | BPF_X:
1078                 if (rv_offset_check(&rvoff, off, i, ctx))
1079                         return -1;
1080                 if (!is64)
1081                         emit_sext_32_rd_rs(&rd, &rs, ctx);
1082                 emit(rv_blt(rd, rs, rvoff >> 1), ctx);
1083                 break;
1084         case BPF_JMP | BPF_JSGE | BPF_X:
1085         case BPF_JMP32 | BPF_JSGE | BPF_X:
1086                 if (rv_offset_check(&rvoff, off, i, ctx))
1087                         return -1;
1088                 if (!is64)
1089                         emit_sext_32_rd_rs(&rd, &rs, ctx);
1090                 emit(rv_bge(rd, rs, rvoff >> 1), ctx);
1091                 break;
1092         case BPF_JMP | BPF_JSLE | BPF_X:
1093         case BPF_JMP32 | BPF_JSLE | BPF_X:
1094                 if (rv_offset_check(&rvoff, off, i, ctx))
1095                         return -1;
1096                 if (!is64)
1097                         emit_sext_32_rd_rs(&rd, &rs, ctx);
1098                 emit(rv_bge(rs, rd, rvoff >> 1), ctx);
1099                 break;
1100         case BPF_JMP | BPF_JSET | BPF_X:
1101         case BPF_JMP32 | BPF_JSET | BPF_X:
1102                 if (rv_offset_check(&rvoff, off, i, ctx))
1103                         return -1;
1104                 if (!is64)
1105                         emit_zext_32_rd_rs(&rd, &rs, ctx);
1106                 emit(rv_and(RV_REG_T1, rd, rs), ctx);
1107                 emit(rv_bne(RV_REG_T1, RV_REG_ZERO, rvoff >> 1), ctx);
1108                 break;
1109 
1110         /* IF (dst COND imm) JUMP off */
1111         case BPF_JMP | BPF_JEQ | BPF_K:
1112         case BPF_JMP32 | BPF_JEQ | BPF_K:
1113                 if (rv_offset_check(&rvoff, off, i, ctx))
1114                         return -1;
1115                 emit_imm(RV_REG_T1, imm, ctx);
1116                 if (!is64)
1117                         emit_zext_32_rd_t1(&rd, ctx);
1118                 emit(rv_beq(rd, RV_REG_T1, rvoff >> 1), ctx);
1119                 break;
1120         case BPF_JMP | BPF_JGT | BPF_K:
1121         case BPF_JMP32 | BPF_JGT | BPF_K:
1122                 if (rv_offset_check(&rvoff, off, i, ctx))
1123                         return -1;
1124                 emit_imm(RV_REG_T1, imm, ctx);
1125                 if (!is64)
1126                         emit_zext_32_rd_t1(&rd, ctx);
1127                 emit(rv_bltu(RV_REG_T1, rd, rvoff >> 1), ctx);
1128                 break;
1129         case BPF_JMP | BPF_JLT | BPF_K:
1130         case BPF_JMP32 | BPF_JLT | BPF_K:
1131                 if (rv_offset_check(&rvoff, off, i, ctx))
1132                         return -1;
1133                 emit_imm(RV_REG_T1, imm, ctx);
1134                 if (!is64)
1135                         emit_zext_32_rd_t1(&rd, ctx);
1136                 emit(rv_bltu(rd, RV_REG_T1, rvoff >> 1), ctx);
1137                 break;
1138         case BPF_JMP | BPF_JGE | BPF_K:
1139         case BPF_JMP32 | BPF_JGE | BPF_K:
1140                 if (rv_offset_check(&rvoff, off, i, ctx))
1141                         return -1;
1142                 emit_imm(RV_REG_T1, imm, ctx);
1143                 if (!is64)
1144                         emit_zext_32_rd_t1(&rd, ctx);
1145                 emit(rv_bgeu(rd, RV_REG_T1, rvoff >> 1), ctx);
1146                 break;
1147         case BPF_JMP | BPF_JLE | BPF_K:
1148         case BPF_JMP32 | BPF_JLE | BPF_K:
1149                 if (rv_offset_check(&rvoff, off, i, ctx))
1150                         return -1;
1151                 emit_imm(RV_REG_T1, imm, ctx);
1152                 if (!is64)
1153                         emit_zext_32_rd_t1(&rd, ctx);
1154                 emit(rv_bgeu(RV_REG_T1, rd, rvoff >> 1), ctx);
1155                 break;
1156         case BPF_JMP | BPF_JNE | BPF_K:
1157         case BPF_JMP32 | BPF_JNE | BPF_K:
1158                 if (rv_offset_check(&rvoff, off, i, ctx))
1159                         return -1;
1160                 emit_imm(RV_REG_T1, imm, ctx);
1161                 if (!is64)
1162                         emit_zext_32_rd_t1(&rd, ctx);
1163                 emit(rv_bne(rd, RV_REG_T1, rvoff >> 1), ctx);
1164                 break;
1165         case BPF_JMP | BPF_JSGT | BPF_K:
1166         case BPF_JMP32 | BPF_JSGT | BPF_K:
1167                 if (rv_offset_check(&rvoff, off, i, ctx))
1168                         return -1;
1169                 emit_imm(RV_REG_T1, imm, ctx);
1170                 if (!is64)
1171                         emit_sext_32_rd(&rd, ctx);
1172                 emit(rv_blt(RV_REG_T1, rd, rvoff >> 1), ctx);
1173                 break;
1174         case BPF_JMP | BPF_JSLT | BPF_K:
1175         case BPF_JMP32 | BPF_JSLT | BPF_K:
1176                 if (rv_offset_check(&rvoff, off, i, ctx))
1177                         return -1;
1178                 emit_imm(RV_REG_T1, imm, ctx);
1179                 if (!is64)
1180                         emit_sext_32_rd(&rd, ctx);
1181                 emit(rv_blt(rd, RV_REG_T1, rvoff >> 1), ctx);
1182                 break;
1183         case BPF_JMP | BPF_JSGE | BPF_K:
1184         case BPF_JMP32 | BPF_JSGE | BPF_K:
1185                 if (rv_offset_check(&rvoff, off, i, ctx))
1186                         return -1;
1187                 emit_imm(RV_REG_T1, imm, ctx);
1188                 if (!is64)
1189                         emit_sext_32_rd(&rd, ctx);
1190                 emit(rv_bge(rd, RV_REG_T1, rvoff >> 1), ctx);
1191                 break;
1192         case BPF_JMP | BPF_JSLE | BPF_K:
1193         case BPF_JMP32 | BPF_JSLE | BPF_K:
1194                 if (rv_offset_check(&rvoff, off, i, ctx))
1195                         return -1;
1196                 emit_imm(RV_REG_T1, imm, ctx);
1197                 if (!is64)
1198                         emit_sext_32_rd(&rd, ctx);
1199                 emit(rv_bge(RV_REG_T1, rd, rvoff >> 1), ctx);
1200                 break;
1201         case BPF_JMP | BPF_JSET | BPF_K:
1202         case BPF_JMP32 | BPF_JSET | BPF_K:
1203                 if (rv_offset_check(&rvoff, off, i, ctx))
1204                         return -1;
1205                 emit_imm(RV_REG_T1, imm, ctx);
1206                 if (!is64)
1207                         emit_zext_32_rd_t1(&rd, ctx);
1208                 emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx);
1209                 emit(rv_bne(RV_REG_T1, RV_REG_ZERO, rvoff >> 1), ctx);
1210                 break;
1211 
1212         /* function call */
1213         case BPF_JMP | BPF_CALL:
1214         {
1215                 bool fixed;
1216                 int i, ret;
1217                 u64 addr;
1218 
1219                 mark_call(ctx);
1220                 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr,
1221                                             &fixed);
1222                 if (ret < 0)
1223                         return ret;
1224                 if (fixed) {
1225                         emit_imm(RV_REG_T1, addr, ctx);
1226                 } else {
1227                         i = ctx->ninsns;
1228                         emit_imm(RV_REG_T1, addr, ctx);
1229                         for (i = ctx->ninsns - i; i < 8; i++) {
1230                                 /* nop */
1231                                 emit(rv_addi(RV_REG_ZERO, RV_REG_ZERO, 0),
1232                                      ctx);
1233                         }
1234                 }
1235                 emit(rv_jalr(RV_REG_RA, RV_REG_T1, 0), ctx);
1236                 rd = bpf_to_rv_reg(BPF_REG_0, ctx);
1237                 emit(rv_addi(rd, RV_REG_A0, 0), ctx);
1238                 break;
1239         }
1240         /* tail call */
1241         case BPF_JMP | BPF_TAIL_CALL:
1242                 if (emit_bpf_tail_call(i, ctx))
1243                         return -1;
1244                 break;
1245 
1246         /* function return */
1247         case BPF_JMP | BPF_EXIT:
1248                 if (i == ctx->prog->len - 1)
1249                         break;
1250 
1251                 rvoff = epilogue_offset(ctx);
1252                 if (is_21b_check(rvoff, i))
1253                         return -1;
1254                 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx);
1255                 break;
1256 
1257         /* dst = imm64 */
1258         case BPF_LD | BPF_IMM | BPF_DW:
1259         {
1260                 struct bpf_insn insn1 = insn[1];
1261                 u64 imm64;
1262 
1263                 imm64 = (u64)insn1.imm << 32 | (u32)imm;
1264                 emit_imm(rd, imm64, ctx);
1265                 return 1;
1266         }
1267 
1268         /* LDX: dst = *(size *)(src + off) */
1269         case BPF_LDX | BPF_MEM | BPF_B:
1270                 if (is_12b_int(off)) {
1271                         emit(rv_lbu(rd, off, rs), ctx);
1272                         break;
1273                 }
1274 
1275                 emit_imm(RV_REG_T1, off, ctx);
1276                 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
1277                 emit(rv_lbu(rd, 0, RV_REG_T1), ctx);
1278                 if (insn_is_zext(&insn[1]))
1279                         return 1;
1280                 break;
1281         case BPF_LDX | BPF_MEM | BPF_H:
1282                 if (is_12b_int(off)) {
1283                         emit(rv_lhu(rd, off, rs), ctx);
1284                         break;
1285                 }
1286 
1287                 emit_imm(RV_REG_T1, off, ctx);
1288                 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
1289                 emit(rv_lhu(rd, 0, RV_REG_T1), ctx);
1290                 if (insn_is_zext(&insn[1]))
1291                         return 1;
1292                 break;
1293         case BPF_LDX | BPF_MEM | BPF_W:
1294                 if (is_12b_int(off)) {
1295                         emit(rv_lwu(rd, off, rs), ctx);
1296                         break;
1297                 }
1298 
1299                 emit_imm(RV_REG_T1, off, ctx);
1300                 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
1301                 emit(rv_lwu(rd, 0, RV_REG_T1), ctx);
1302                 if (insn_is_zext(&insn[1]))
1303                         return 1;
1304                 break;
1305         case BPF_LDX | BPF_MEM | BPF_DW:
1306                 if (is_12b_int(off)) {
1307                         emit(rv_ld(rd, off, rs), ctx);
1308                         break;
1309                 }
1310 
1311                 emit_imm(RV_REG_T1, off, ctx);
1312                 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
1313                 emit(rv_ld(rd, 0, RV_REG_T1), ctx);
1314                 break;
1315 
1316         /* ST: *(size *)(dst + off) = imm */
1317         case BPF_ST | BPF_MEM | BPF_B:
1318                 emit_imm(RV_REG_T1, imm, ctx);
1319                 if (is_12b_int(off)) {
1320                         emit(rv_sb(rd, off, RV_REG_T1), ctx);
1321                         break;
1322                 }
1323 
1324                 emit_imm(RV_REG_T2, off, ctx);
1325                 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
1326                 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx);
1327                 break;
1328 
1329         case BPF_ST | BPF_MEM | BPF_H:
1330                 emit_imm(RV_REG_T1, imm, ctx);
1331                 if (is_12b_int(off)) {
1332                         emit(rv_sh(rd, off, RV_REG_T1), ctx);
1333                         break;
1334                 }
1335 
1336                 emit_imm(RV_REG_T2, off, ctx);
1337                 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
1338                 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx);
1339                 break;
1340         case BPF_ST | BPF_MEM | BPF_W:
1341                 emit_imm(RV_REG_T1, imm, ctx);
1342                 if (is_12b_int(off)) {
1343                         emit(rv_sw(rd, off, RV_REG_T1), ctx);
1344                         break;
1345                 }
1346 
1347                 emit_imm(RV_REG_T2, off, ctx);
1348                 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
1349                 emit(rv_sw(RV_REG_T2, 0, RV_REG_T1), ctx);
1350                 break;
1351         case BPF_ST | BPF_MEM | BPF_DW:
1352                 emit_imm(RV_REG_T1, imm, ctx);
1353                 if (is_12b_int(off)) {
1354                         emit(rv_sd(rd, off, RV_REG_T1), ctx);
1355                         break;
1356                 }
1357 
1358                 emit_imm(RV_REG_T2, off, ctx);
1359                 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
1360                 emit(rv_sd(RV_REG_T2, 0, RV_REG_T1), ctx);
1361                 break;
1362 
1363         /* STX: *(size *)(dst + off) = src */
1364         case BPF_STX | BPF_MEM | BPF_B:
1365                 if (is_12b_int(off)) {
1366                         emit(rv_sb(rd, off, rs), ctx);
1367                         break;
1368                 }
1369 
1370                 emit_imm(RV_REG_T1, off, ctx);
1371                 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
1372                 emit(rv_sb(RV_REG_T1, 0, rs), ctx);
1373                 break;
1374         case BPF_STX | BPF_MEM | BPF_H:
1375                 if (is_12b_int(off)) {
1376                         emit(rv_sh(rd, off, rs), ctx);
1377                         break;
1378                 }
1379 
1380                 emit_imm(RV_REG_T1, off, ctx);
1381                 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
1382                 emit(rv_sh(RV_REG_T1, 0, rs), ctx);
1383                 break;
1384         case BPF_STX | BPF_MEM | BPF_W:
1385                 if (is_12b_int(off)) {
1386                         emit(rv_sw(rd, off, rs), ctx);
1387                         break;
1388                 }
1389 
1390                 emit_imm(RV_REG_T1, off, ctx);
1391                 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
1392                 emit(rv_sw(RV_REG_T1, 0, rs), ctx);
1393                 break;
1394         case BPF_STX | BPF_MEM | BPF_DW:
1395                 if (is_12b_int(off)) {
1396                         emit(rv_sd(rd, off, rs), ctx);
1397                         break;
1398                 }
1399 
1400                 emit_imm(RV_REG_T1, off, ctx);
1401                 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
1402                 emit(rv_sd(RV_REG_T1, 0, rs), ctx);
1403                 break;
1404         /* STX XADD: lock *(u32 *)(dst + off) += src */
1405         case BPF_STX | BPF_XADD | BPF_W:
1406         /* STX XADD: lock *(u64 *)(dst + off) += src */
1407         case BPF_STX | BPF_XADD | BPF_DW:
1408                 if (off) {
1409                         if (is_12b_int(off)) {
1410                                 emit(rv_addi(RV_REG_T1, rd, off), ctx);
1411                         } else {
1412                                 emit_imm(RV_REG_T1, off, ctx);
1413                                 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
1414                         }
1415 
1416                         rd = RV_REG_T1;
1417                 }
1418 
1419                 emit(BPF_SIZE(code) == BPF_W ?
1420                      rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0) :
1421                      rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0), ctx);
1422                 break;
1423         default:
1424                 pr_err("bpf-jit: unknown opcode %02x\n", code);
1425                 return -EINVAL;
1426         }
1427 
1428         return 0;
1429 }
1430 
1431 static void build_prologue(struct rv_jit_context *ctx)
1432 {
1433         int stack_adjust = 0, store_offset, bpf_stack_adjust;
1434 
1435         bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
1436         if (bpf_stack_adjust)
1437                 mark_fp(ctx);
1438 
1439         if (seen_reg(RV_REG_RA, ctx))
1440                 stack_adjust += 8;
1441         stack_adjust += 8; /* RV_REG_FP */
1442         if (seen_reg(RV_REG_S1, ctx))
1443                 stack_adjust += 8;
1444         if (seen_reg(RV_REG_S2, ctx))
1445                 stack_adjust += 8;
1446         if (seen_reg(RV_REG_S3, ctx))
1447                 stack_adjust += 8;
1448         if (seen_reg(RV_REG_S4, ctx))
1449                 stack_adjust += 8;
1450         if (seen_reg(RV_REG_S5, ctx))
1451                 stack_adjust += 8;
1452         if (seen_reg(RV_REG_S6, ctx))
1453                 stack_adjust += 8;
1454 
1455         stack_adjust = round_up(stack_adjust, 16);
1456         stack_adjust += bpf_stack_adjust;
1457 
1458         store_offset = stack_adjust - 8;
1459 
1460         /* First instruction is always setting the tail-call-counter
1461          * (TCC) register. This instruction is skipped for tail calls.
1462          */
1463         emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx);
1464 
1465         emit(rv_addi(RV_REG_SP, RV_REG_SP, -stack_adjust), ctx);
1466 
1467         if (seen_reg(RV_REG_RA, ctx)) {
1468                 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_RA), ctx);
1469                 store_offset -= 8;
1470         }
1471         emit(rv_sd(RV_REG_SP, store_offset, RV_REG_FP), ctx);
1472         store_offset -= 8;
1473         if (seen_reg(RV_REG_S1, ctx)) {
1474                 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S1), ctx);
1475                 store_offset -= 8;
1476         }
1477         if (seen_reg(RV_REG_S2, ctx)) {
1478                 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S2), ctx);
1479                 store_offset -= 8;
1480         }
1481         if (seen_reg(RV_REG_S3, ctx)) {
1482                 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S3), ctx);
1483                 store_offset -= 8;
1484         }
1485         if (seen_reg(RV_REG_S4, ctx)) {
1486                 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S4), ctx);
1487                 store_offset -= 8;
1488         }
1489         if (seen_reg(RV_REG_S5, ctx)) {
1490                 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S5), ctx);
1491                 store_offset -= 8;
1492         }
1493         if (seen_reg(RV_REG_S6, ctx)) {
1494                 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S6), ctx);
1495                 store_offset -= 8;
1496         }
1497 
1498         emit(rv_addi(RV_REG_FP, RV_REG_SP, stack_adjust), ctx);
1499 
1500         if (bpf_stack_adjust)
1501                 emit(rv_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust), ctx);
1502 
1503         /* Program contains calls and tail calls, so RV_REG_TCC need
1504          * to be saved across calls.
1505          */
1506         if (seen_tail_call(ctx) && seen_call(ctx))
1507                 emit(rv_addi(RV_REG_TCC_SAVED, RV_REG_TCC, 0), ctx);
1508 
1509         ctx->stack_size = stack_adjust;
1510 }
1511 
1512 static void build_epilogue(struct rv_jit_context *ctx)
1513 {
1514         __build_epilogue(RV_REG_RA, ctx);
1515 }
1516 
1517 static int build_body(struct rv_jit_context *ctx, bool extra_pass)
1518 {
1519         const struct bpf_prog *prog = ctx->prog;
1520         int i;
1521 
1522         for (i = 0; i < prog->len; i++) {
1523                 const struct bpf_insn *insn = &prog->insnsi[i];
1524                 int ret;
1525 
1526                 ret = emit_insn(insn, ctx, extra_pass);
1527                 if (ret > 0) {
1528                         i++;
1529                         if (ctx->insns == NULL)
1530                                 ctx->offset[i] = ctx->ninsns;
1531                         continue;
1532                 }
1533                 if (ctx->insns == NULL)
1534                         ctx->offset[i] = ctx->ninsns;
1535                 if (ret)
1536                         return ret;
1537         }
1538         return 0;
1539 }
1540 
1541 static void bpf_fill_ill_insns(void *area, unsigned int size)
1542 {
1543         memset(area, 0, size);
1544 }
1545 
1546 static void bpf_flush_icache(void *start, void *end)
1547 {
1548         flush_icache_range((unsigned long)start, (unsigned long)end);
1549 }
1550 
1551 bool bpf_jit_needs_zext(void)
1552 {
1553         return true;
1554 }
1555 
1556 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1557 {
1558         bool tmp_blinded = false, extra_pass = false;
1559         struct bpf_prog *tmp, *orig_prog = prog;
1560         struct rv_jit_data *jit_data;
1561         struct rv_jit_context *ctx;
1562         unsigned int image_size;
1563 
1564         if (!prog->jit_requested)
1565                 return orig_prog;
1566 
1567         tmp = bpf_jit_blind_constants(prog);
1568         if (IS_ERR(tmp))
1569                 return orig_prog;
1570         if (tmp != prog) {
1571                 tmp_blinded = true;
1572                 prog = tmp;
1573         }
1574 
1575         jit_data = prog->aux->jit_data;
1576         if (!jit_data) {
1577                 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1578                 if (!jit_data) {
1579                         prog = orig_prog;
1580                         goto out;
1581                 }
1582                 prog->aux->jit_data = jit_data;
1583         }
1584 
1585         ctx = &jit_data->ctx;
1586 
1587         if (ctx->offset) {
1588                 extra_pass = true;
1589                 image_size = sizeof(u32) * ctx->ninsns;
1590                 goto skip_init_ctx;
1591         }
1592 
1593         ctx->prog = prog;
1594         ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
1595         if (!ctx->offset) {
1596                 prog = orig_prog;
1597                 goto out_offset;
1598         }
1599 
1600         /* First pass generates the ctx->offset, but does not emit an image. */
1601         if (build_body(ctx, extra_pass)) {
1602                 prog = orig_prog;
1603                 goto out_offset;
1604         }
1605         build_prologue(ctx);
1606         ctx->epilogue_offset = ctx->ninsns;
1607         build_epilogue(ctx);
1608 
1609         /* Allocate image, now that we know the size. */
1610         image_size = sizeof(u32) * ctx->ninsns;
1611         jit_data->header = bpf_jit_binary_alloc(image_size, &jit_data->image,
1612                                                 sizeof(u32),
1613                                                 bpf_fill_ill_insns);
1614         if (!jit_data->header) {
1615                 prog = orig_prog;
1616                 goto out_offset;
1617         }
1618 
1619         /* Second, real pass, that acutally emits the image. */
1620         ctx->insns = (u32 *)jit_data->image;
1621 skip_init_ctx:
1622         ctx->ninsns = 0;
1623 
1624         build_prologue(ctx);
1625         if (build_body(ctx, extra_pass)) {
1626                 bpf_jit_binary_free(jit_data->header);
1627                 prog = orig_prog;
1628                 goto out_offset;
1629         }
1630         build_epilogue(ctx);
1631 
1632         if (bpf_jit_enable > 1)
1633                 bpf_jit_dump(prog->len, image_size, 2, ctx->insns);
1634 
1635         prog->bpf_func = (void *)ctx->insns;
1636         prog->jited = 1;
1637         prog->jited_len = image_size;
1638 
1639         bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
1640 
1641         if (!prog->is_func || extra_pass) {
1642 out_offset:
1643                 kfree(ctx->offset);
1644                 kfree(jit_data);
1645                 prog->aux->jit_data = NULL;
1646         }
1647 out:
1648         if (tmp_blinded)
1649                 bpf_jit_prog_release_other(prog, prog == orig_prog ?
1650                                            tmp : orig_prog);
1651         return prog;
1652 }
/* [<][>][^][v][top][bottom][index][help] */
root/arch/riscv/net/bpf_jit_comp.c

DEFINITIONS
