arch/sparc/net/bpf_jit_comp

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
is_simm13
is_simm10
is_simm5
is_sethi
bpf_flush_icache
WDISP10
emit
emit_call
emit_nop
emit_reg_move
emit_set_const
emit_set_const_sext
emit_alu
emit_alu3
emit_alu_K
emit_alu3_K
emit_loadimm32
emit_loadimm
emit_loadimm_sext
analyze_64bit_constant
create_simple_focus_bits
const64_is_2insns
sparc_emit_set_const64_quick2
emit_loadimm64
emit_branch
emit_cbcond
emit_cbcondi
emit_compare_and_branch
build_prologue
build_epilogue
emit_tail_call
build_insn
build_body
jit_fill_hole
bpf_jit_needs_zext
bpf_int_jit_compile
   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/moduleloader.h>
   3 #include <linux/workqueue.h>
   4 #include <linux/netdevice.h>
   5 #include <linux/filter.h>
   6 #include <linux/bpf.h>
   7 #include <linux/cache.h>
   8 #include <linux/if_vlan.h>
   9 
  10 #include <asm/cacheflush.h>
  11 #include <asm/ptrace.h>
  12 
  13 #include "bpf_jit_64.h"
  14 
  15 static inline bool is_simm13(unsigned int value)
  16 {
  17         return value + 0x1000 < 0x2000;
  18 }
  19 
  20 static inline bool is_simm10(unsigned int value)
  21 {
  22         return value + 0x200 < 0x400;
  23 }
  24 
  25 static inline bool is_simm5(unsigned int value)
  26 {
  27         return value + 0x10 < 0x20;
  28 }
  29 
  30 static inline bool is_sethi(unsigned int value)
  31 {
  32         return (value & ~0x3fffff) == 0;
  33 }
  34 
  35 static void bpf_flush_icache(void *start_, void *end_)
  36 {
  37         /* Cheetah's I-cache is fully coherent.  */
  38         if (tlb_type == spitfire) {
  39                 unsigned long start = (unsigned long) start_;
  40                 unsigned long end = (unsigned long) end_;
  41 
  42                 start &= ~7UL;
  43                 end = (end + 7UL) & ~7UL;
  44                 while (start < end) {
  45                         flushi(start);
  46                         start += 32;
  47                 }
  48         }
  49 }
  50 
  51 #define S13(X)          ((X) & 0x1fff)
  52 #define S5(X)           ((X) & 0x1f)
  53 #define IMMED           0x00002000
  54 #define RD(X)           ((X) << 25)
  55 #define RS1(X)          ((X) << 14)
  56 #define RS2(X)          ((X))
  57 #define OP(X)           ((X) << 30)
  58 #define OP2(X)          ((X) << 22)
  59 #define OP3(X)          ((X) << 19)
  60 #define COND(X)         (((X) & 0xf) << 25)
  61 #define CBCOND(X)       (((X) & 0x1f) << 25)
  62 #define F1(X)           OP(X)
  63 #define F2(X, Y)        (OP(X) | OP2(Y))
  64 #define F3(X, Y)        (OP(X) | OP3(Y))
  65 #define ASI(X)          (((X) & 0xff) << 5)
  66 
  67 #define CONDN           COND(0x0)
  68 #define CONDE           COND(0x1)
  69 #define CONDLE          COND(0x2)
  70 #define CONDL           COND(0x3)
  71 #define CONDLEU         COND(0x4)
  72 #define CONDCS          COND(0x5)
  73 #define CONDNEG         COND(0x6)
  74 #define CONDVC          COND(0x7)
  75 #define CONDA           COND(0x8)
  76 #define CONDNE          COND(0x9)
  77 #define CONDG           COND(0xa)
  78 #define CONDGE          COND(0xb)
  79 #define CONDGU          COND(0xc)
  80 #define CONDCC          COND(0xd)
  81 #define CONDPOS         COND(0xe)
  82 #define CONDVS          COND(0xf)
  83 
  84 #define CONDGEU         CONDCC
  85 #define CONDLU          CONDCS
  86 
  87 #define WDISP22(X)      (((X) >> 2) & 0x3fffff)
  88 #define WDISP19(X)      (((X) >> 2) & 0x7ffff)
  89 
  90 /* The 10-bit branch displacement for CBCOND is split into two fields */
  91 static u32 WDISP10(u32 off)
  92 {
  93         u32 ret = ((off >> 2) & 0xff) << 5;
  94 
  95         ret |= ((off >> (2 + 8)) & 0x03) << 19;
  96 
  97         return ret;
  98 }
  99 
 100 #define CBCONDE         CBCOND(0x09)
 101 #define CBCONDLE        CBCOND(0x0a)
 102 #define CBCONDL         CBCOND(0x0b)
 103 #define CBCONDLEU       CBCOND(0x0c)
 104 #define CBCONDCS        CBCOND(0x0d)
 105 #define CBCONDN         CBCOND(0x0e)
 106 #define CBCONDVS        CBCOND(0x0f)
 107 #define CBCONDNE        CBCOND(0x19)
 108 #define CBCONDG         CBCOND(0x1a)
 109 #define CBCONDGE        CBCOND(0x1b)
 110 #define CBCONDGU        CBCOND(0x1c)
 111 #define CBCONDCC        CBCOND(0x1d)
 112 #define CBCONDPOS       CBCOND(0x1e)
 113 #define CBCONDVC        CBCOND(0x1f)
 114 
 115 #define CBCONDGEU       CBCONDCC
 116 #define CBCONDLU        CBCONDCS
 117 
 118 #define ANNUL           (1 << 29)
 119 #define XCC             (1 << 21)
 120 
 121 #define BRANCH          (F2(0, 1) | XCC)
 122 #define CBCOND_OP       (F2(0, 3) | XCC)
 123 
 124 #define BA              (BRANCH | CONDA)
 125 #define BG              (BRANCH | CONDG)
 126 #define BL              (BRANCH | CONDL)
 127 #define BLE             (BRANCH | CONDLE)
 128 #define BGU             (BRANCH | CONDGU)
 129 #define BLEU            (BRANCH | CONDLEU)
 130 #define BGE             (BRANCH | CONDGE)
 131 #define BGEU            (BRANCH | CONDGEU)
 132 #define BLU             (BRANCH | CONDLU)
 133 #define BE              (BRANCH | CONDE)
 134 #define BNE             (BRANCH | CONDNE)
 135 
 136 #define SETHI(K, REG)   \
 137         (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff))
 138 #define OR_LO(K, REG)   \
 139         (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG))
 140 
 141 #define ADD             F3(2, 0x00)
 142 #define AND             F3(2, 0x01)
 143 #define ANDCC           F3(2, 0x11)
 144 #define OR              F3(2, 0x02)
 145 #define XOR             F3(2, 0x03)
 146 #define SUB             F3(2, 0x04)
 147 #define SUBCC           F3(2, 0x14)
 148 #define MUL             F3(2, 0x0a)
 149 #define MULX            F3(2, 0x09)
 150 #define UDIVX           F3(2, 0x0d)
 151 #define DIV             F3(2, 0x0e)
 152 #define SLL             F3(2, 0x25)
 153 #define SLLX            (F3(2, 0x25)|(1<<12))
 154 #define SRA             F3(2, 0x27)
 155 #define SRAX            (F3(2, 0x27)|(1<<12))
 156 #define SRL             F3(2, 0x26)
 157 #define SRLX            (F3(2, 0x26)|(1<<12))
 158 #define JMPL            F3(2, 0x38)
 159 #define SAVE            F3(2, 0x3c)
 160 #define RESTORE         F3(2, 0x3d)
 161 #define CALL            F1(1)
 162 #define BR              F2(0, 0x01)
 163 #define RD_Y            F3(2, 0x28)
 164 #define WR_Y            F3(2, 0x30)
 165 
 166 #define LD32            F3(3, 0x00)
 167 #define LD8             F3(3, 0x01)
 168 #define LD16            F3(3, 0x02)
 169 #define LD64            F3(3, 0x0b)
 170 #define LD64A           F3(3, 0x1b)
 171 #define ST8             F3(3, 0x05)
 172 #define ST16            F3(3, 0x06)
 173 #define ST32            F3(3, 0x04)
 174 #define ST64            F3(3, 0x0e)
 175 
 176 #define CAS             F3(3, 0x3c)
 177 #define CASX            F3(3, 0x3e)
 178 
 179 #define LDPTR           LD64
 180 #define BASE_STACKFRAME 176
 181 
 182 #define LD32I           (LD32 | IMMED)
 183 #define LD8I            (LD8 | IMMED)
 184 #define LD16I           (LD16 | IMMED)
 185 #define LD64I           (LD64 | IMMED)
 186 #define LDPTRI          (LDPTR | IMMED)
 187 #define ST32I           (ST32 | IMMED)
 188 
 189 struct jit_ctx {
 190         struct bpf_prog         *prog;
 191         unsigned int            *offset;
 192         int                     idx;
 193         int                     epilogue_offset;
 194         bool                    tmp_1_used;
 195         bool                    tmp_2_used;
 196         bool                    tmp_3_used;
 197         bool                    saw_frame_pointer;
 198         bool                    saw_call;
 199         bool                    saw_tail_call;
 200         u32                     *image;
 201 };
 202 
 203 #define TMP_REG_1       (MAX_BPF_JIT_REG + 0)
 204 #define TMP_REG_2       (MAX_BPF_JIT_REG + 1)
 205 #define TMP_REG_3       (MAX_BPF_JIT_REG + 2)
 206 
 207 /* Map BPF registers to SPARC registers */
 208 static const int bpf2sparc[] = {
 209         /* return value from in-kernel function, and exit value from eBPF */
 210         [BPF_REG_0] = O5,
 211 
 212         /* arguments from eBPF program to in-kernel function */
 213         [BPF_REG_1] = O0,
 214         [BPF_REG_2] = O1,
 215         [BPF_REG_3] = O2,
 216         [BPF_REG_4] = O3,
 217         [BPF_REG_5] = O4,
 218 
 219         /* callee saved registers that in-kernel function will preserve */
 220         [BPF_REG_6] = L0,
 221         [BPF_REG_7] = L1,
 222         [BPF_REG_8] = L2,
 223         [BPF_REG_9] = L3,
 224 
 225         /* read-only frame pointer to access stack */
 226         [BPF_REG_FP] = L6,
 227 
 228         [BPF_REG_AX] = G7,
 229 
 230         /* temporary register for internal BPF JIT */
 231         [TMP_REG_1] = G1,
 232         [TMP_REG_2] = G2,
 233         [TMP_REG_3] = G3,
 234 };
 235 
 236 static void emit(const u32 insn, struct jit_ctx *ctx)
 237 {
 238         if (ctx->image != NULL)
 239                 ctx->image[ctx->idx] = insn;
 240 
 241         ctx->idx++;
 242 }
 243 
 244 static void emit_call(u32 *func, struct jit_ctx *ctx)
 245 {
 246         if (ctx->image != NULL) {
 247                 void *here = &ctx->image[ctx->idx];
 248                 unsigned int off;
 249 
 250                 off = (void *)func - here;
 251                 ctx->image[ctx->idx] = CALL | ((off >> 2) & 0x3fffffff);
 252         }
 253         ctx->idx++;
 254 }
 255 
 256 static void emit_nop(struct jit_ctx *ctx)
 257 {
 258         emit(SETHI(0, G0), ctx);
 259 }
 260 
 261 static void emit_reg_move(u32 from, u32 to, struct jit_ctx *ctx)
 262 {
 263         emit(OR | RS1(G0) | RS2(from) | RD(to), ctx);
 264 }
 265 
 266 /* Emit 32-bit constant, zero extended. */
 267 static void emit_set_const(s32 K, u32 reg, struct jit_ctx *ctx)
 268 {
 269         emit(SETHI(K, reg), ctx);
 270         emit(OR_LO(K, reg), ctx);
 271 }
 272 
 273 /* Emit 32-bit constant, sign extended. */
 274 static void emit_set_const_sext(s32 K, u32 reg, struct jit_ctx *ctx)
 275 {
 276         if (K >= 0) {
 277                 emit(SETHI(K, reg), ctx);
 278                 emit(OR_LO(K, reg), ctx);
 279         } else {
 280                 u32 hbits = ~(u32) K;
 281                 u32 lbits = -0x400 | (u32) K;
 282 
 283                 emit(SETHI(hbits, reg), ctx);
 284                 emit(XOR | IMMED | RS1(reg) | S13(lbits) | RD(reg), ctx);
 285         }
 286 }
 287 
 288 static void emit_alu(u32 opcode, u32 src, u32 dst, struct jit_ctx *ctx)
 289 {
 290         emit(opcode | RS1(dst) | RS2(src) | RD(dst), ctx);
 291 }
 292 
 293 static void emit_alu3(u32 opcode, u32 a, u32 b, u32 c, struct jit_ctx *ctx)
 294 {
 295         emit(opcode | RS1(a) | RS2(b) | RD(c), ctx);
 296 }
 297 
 298 static void emit_alu_K(unsigned int opcode, unsigned int dst, unsigned int imm,
 299                        struct jit_ctx *ctx)
 300 {
 301         bool small_immed = is_simm13(imm);
 302         unsigned int insn = opcode;
 303 
 304         insn |= RS1(dst) | RD(dst);
 305         if (small_immed) {
 306                 emit(insn | IMMED | S13(imm), ctx);
 307         } else {
 308                 unsigned int tmp = bpf2sparc[TMP_REG_1];
 309 
 310                 ctx->tmp_1_used = true;
 311 
 312                 emit_set_const_sext(imm, tmp, ctx);
 313                 emit(insn | RS2(tmp), ctx);
 314         }
 315 }
 316 
 317 static void emit_alu3_K(unsigned int opcode, unsigned int src, unsigned int imm,
 318                         unsigned int dst, struct jit_ctx *ctx)
 319 {
 320         bool small_immed = is_simm13(imm);
 321         unsigned int insn = opcode;
 322 
 323         insn |= RS1(src) | RD(dst);
 324         if (small_immed) {
 325                 emit(insn | IMMED | S13(imm), ctx);
 326         } else {
 327                 unsigned int tmp = bpf2sparc[TMP_REG_1];
 328 
 329                 ctx->tmp_1_used = true;
 330 
 331                 emit_set_const_sext(imm, tmp, ctx);
 332                 emit(insn | RS2(tmp), ctx);
 333         }
 334 }
 335 
 336 static void emit_loadimm32(s32 K, unsigned int dest, struct jit_ctx *ctx)
 337 {
 338         if (K >= 0 && is_simm13(K)) {
 339                 /* or %g0, K, DEST */
 340                 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
 341         } else {
 342                 emit_set_const(K, dest, ctx);
 343         }
 344 }
 345 
 346 static void emit_loadimm(s32 K, unsigned int dest, struct jit_ctx *ctx)
 347 {
 348         if (is_simm13(K)) {
 349                 /* or %g0, K, DEST */
 350                 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
 351         } else {
 352                 emit_set_const(K, dest, ctx);
 353         }
 354 }
 355 
 356 static void emit_loadimm_sext(s32 K, unsigned int dest, struct jit_ctx *ctx)
 357 {
 358         if (is_simm13(K)) {
 359                 /* or %g0, K, DEST */
 360                 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
 361         } else {
 362                 emit_set_const_sext(K, dest, ctx);
 363         }
 364 }
 365 
 366 static void analyze_64bit_constant(u32 high_bits, u32 low_bits,
 367                                    int *hbsp, int *lbsp, int *abbasp)
 368 {
 369         int lowest_bit_set, highest_bit_set, all_bits_between_are_set;
 370         int i;
 371 
 372         lowest_bit_set = highest_bit_set = -1;
 373         i = 0;
 374         do {
 375                 if ((lowest_bit_set == -1) && ((low_bits >> i) & 1))
 376                         lowest_bit_set = i;
 377                 if ((highest_bit_set == -1) && ((high_bits >> (32 - i - 1)) & 1))
 378                         highest_bit_set = (64 - i - 1);
 379         }  while (++i < 32 && (highest_bit_set == -1 ||
 380                                lowest_bit_set == -1));
 381         if (i == 32) {
 382                 i = 0;
 383                 do {
 384                         if (lowest_bit_set == -1 && ((high_bits >> i) & 1))
 385                                 lowest_bit_set = i + 32;
 386                         if (highest_bit_set == -1 &&
 387                             ((low_bits >> (32 - i - 1)) & 1))
 388                                 highest_bit_set = 32 - i - 1;
 389                 } while (++i < 32 && (highest_bit_set == -1 ||
 390                                       lowest_bit_set == -1));
 391         }
 392 
 393         all_bits_between_are_set = 1;
 394         for (i = lowest_bit_set; i <= highest_bit_set; i++) {
 395                 if (i < 32) {
 396                         if ((low_bits & (1 << i)) != 0)
 397                                 continue;
 398                 } else {
 399                         if ((high_bits & (1 << (i - 32))) != 0)
 400                                 continue;
 401                 }
 402                 all_bits_between_are_set = 0;
 403                 break;
 404         }
 405         *hbsp = highest_bit_set;
 406         *lbsp = lowest_bit_set;
 407         *abbasp = all_bits_between_are_set;
 408 }
 409 
 410 static unsigned long create_simple_focus_bits(unsigned long high_bits,
 411                                               unsigned long low_bits,
 412                                               int lowest_bit_set, int shift)
 413 {
 414         long hi, lo;
 415 
 416         if (lowest_bit_set < 32) {
 417                 lo = (low_bits >> lowest_bit_set) << shift;
 418                 hi = ((high_bits << (32 - lowest_bit_set)) << shift);
 419         } else {
 420                 lo = 0;
 421                 hi = ((high_bits >> (lowest_bit_set - 32)) << shift);
 422         }
 423         return hi | lo;
 424 }
 425 
 426 static bool const64_is_2insns(unsigned long high_bits,
 427                               unsigned long low_bits)
 428 {
 429         int highest_bit_set, lowest_bit_set, all_bits_between_are_set;
 430 
 431         if (high_bits == 0 || high_bits == 0xffffffff)
 432                 return true;
 433 
 434         analyze_64bit_constant(high_bits, low_bits,
 435                                &highest_bit_set, &lowest_bit_set,
 436                                &all_bits_between_are_set);
 437 
 438         if ((highest_bit_set == 63 || lowest_bit_set == 0) &&
 439             all_bits_between_are_set != 0)
 440                 return true;
 441 
 442         if (highest_bit_set - lowest_bit_set < 21)
 443                 return true;
 444 
 445         return false;
 446 }
 447 
 448 static void sparc_emit_set_const64_quick2(unsigned long high_bits,
 449                                           unsigned long low_imm,
 450                                           unsigned int dest,
 451                                           int shift_count, struct jit_ctx *ctx)
 452 {
 453         emit_loadimm32(high_bits, dest, ctx);
 454 
 455         /* Now shift it up into place.  */
 456         emit_alu_K(SLLX, dest, shift_count, ctx);
 457 
 458         /* If there is a low immediate part piece, finish up by
 459          * putting that in as well.
 460          */
 461         if (low_imm != 0)
 462                 emit(OR | IMMED | RS1(dest) | S13(low_imm) | RD(dest), ctx);
 463 }
 464 
 465 static void emit_loadimm64(u64 K, unsigned int dest, struct jit_ctx *ctx)
 466 {
 467         int all_bits_between_are_set, lowest_bit_set, highest_bit_set;
 468         unsigned int tmp = bpf2sparc[TMP_REG_1];
 469         u32 low_bits = (K & 0xffffffff);
 470         u32 high_bits = (K >> 32);
 471 
 472         /* These two tests also take care of all of the one
 473          * instruction cases.
 474          */
 475         if (high_bits == 0xffffffff && (low_bits & 0x80000000))
 476                 return emit_loadimm_sext(K, dest, ctx);
 477         if (high_bits == 0x00000000)
 478                 return emit_loadimm32(K, dest, ctx);
 479 
 480         analyze_64bit_constant(high_bits, low_bits, &highest_bit_set,
 481                                &lowest_bit_set, &all_bits_between_are_set);
 482 
 483         /* 1) mov       -1, %reg
 484          *    sllx      %reg, shift, %reg
 485          * 2) mov       -1, %reg
 486          *    srlx      %reg, shift, %reg
 487          * 3) mov       some_small_const, %reg
 488          *    sllx      %reg, shift, %reg
 489          */
 490         if (((highest_bit_set == 63 || lowest_bit_set == 0) &&
 491              all_bits_between_are_set != 0) ||
 492             ((highest_bit_set - lowest_bit_set) < 12)) {
 493                 int shift = lowest_bit_set;
 494                 long the_const = -1;
 495 
 496                 if ((highest_bit_set != 63 && lowest_bit_set != 0) ||
 497                     all_bits_between_are_set == 0) {
 498                         the_const =
 499                                 create_simple_focus_bits(high_bits, low_bits,
 500                                                          lowest_bit_set, 0);
 501                 } else if (lowest_bit_set == 0)
 502                         shift = -(63 - highest_bit_set);
 503 
 504                 emit(OR | IMMED | RS1(G0) | S13(the_const) | RD(dest), ctx);
 505                 if (shift > 0)
 506                         emit_alu_K(SLLX, dest, shift, ctx);
 507                 else if (shift < 0)
 508                         emit_alu_K(SRLX, dest, -shift, ctx);
 509 
 510                 return;
 511         }
 512 
 513         /* Now a range of 22 or less bits set somewhere.
 514          * 1) sethi     %hi(focus_bits), %reg
 515          *    sllx      %reg, shift, %reg
 516          * 2) sethi     %hi(focus_bits), %reg
 517          *    srlx      %reg, shift, %reg
 518          */
 519         if ((highest_bit_set - lowest_bit_set) < 21) {
 520                 unsigned long focus_bits =
 521                         create_simple_focus_bits(high_bits, low_bits,
 522                                                  lowest_bit_set, 10);
 523 
 524                 emit(SETHI(focus_bits, dest), ctx);
 525 
 526                 /* If lowest_bit_set == 10 then a sethi alone could
 527                  * have done it.
 528                  */
 529                 if (lowest_bit_set < 10)
 530                         emit_alu_K(SRLX, dest, 10 - lowest_bit_set, ctx);
 531                 else if (lowest_bit_set > 10)
 532                         emit_alu_K(SLLX, dest, lowest_bit_set - 10, ctx);
 533                 return;
 534         }
 535 
 536         /* Ok, now 3 instruction sequences.  */
 537         if (low_bits == 0) {
 538                 emit_loadimm32(high_bits, dest, ctx);
 539                 emit_alu_K(SLLX, dest, 32, ctx);
 540                 return;
 541         }
 542 
 543         /* We may be able to do something quick
 544          * when the constant is negated, so try that.
 545          */
 546         if (const64_is_2insns((~high_bits) & 0xffffffff,
 547                               (~low_bits) & 0xfffffc00)) {
 548                 /* NOTE: The trailing bits get XOR'd so we need the
 549                  * non-negated bits, not the negated ones.
 550                  */
 551                 unsigned long trailing_bits = low_bits & 0x3ff;
 552 
 553                 if ((((~high_bits) & 0xffffffff) == 0 &&
 554                      ((~low_bits) & 0x80000000) == 0) ||
 555                     (((~high_bits) & 0xffffffff) == 0xffffffff &&
 556                      ((~low_bits) & 0x80000000) != 0)) {
 557                         unsigned long fast_int = (~low_bits & 0xffffffff);
 558 
 559                         if ((is_sethi(fast_int) &&
 560                              (~high_bits & 0xffffffff) == 0)) {
 561                                 emit(SETHI(fast_int, dest), ctx);
 562                         } else if (is_simm13(fast_int)) {
 563                                 emit(OR | IMMED | RS1(G0) | S13(fast_int) | RD(dest), ctx);
 564                         } else {
 565                                 emit_loadimm64(fast_int, dest, ctx);
 566                         }
 567                 } else {
 568                         u64 n = ((~low_bits) & 0xfffffc00) |
 569                                 (((unsigned long)((~high_bits) & 0xffffffff))<<32);
 570                         emit_loadimm64(n, dest, ctx);
 571                 }
 572 
 573                 low_bits = -0x400 | trailing_bits;
 574 
 575                 emit(XOR | IMMED | RS1(dest) | S13(low_bits) | RD(dest), ctx);
 576                 return;
 577         }
 578 
 579         /* 1) sethi     %hi(xxx), %reg
 580          *    or        %reg, %lo(xxx), %reg
 581          *    sllx      %reg, yyy, %reg
 582          */
 583         if ((highest_bit_set - lowest_bit_set) < 32) {
 584                 unsigned long focus_bits =
 585                         create_simple_focus_bits(high_bits, low_bits,
 586                                                  lowest_bit_set, 0);
 587 
 588                 /* So what we know is that the set bits straddle the
 589                  * middle of the 64-bit word.
 590                  */
 591                 sparc_emit_set_const64_quick2(focus_bits, 0, dest,
 592                                               lowest_bit_set, ctx);
 593                 return;
 594         }
 595 
 596         /* 1) sethi     %hi(high_bits), %reg
 597          *    or        %reg, %lo(high_bits), %reg
 598          *    sllx      %reg, 32, %reg
 599          *    or        %reg, low_bits, %reg
 600          */
 601         if (is_simm13(low_bits) && ((int)low_bits > 0)) {
 602                 sparc_emit_set_const64_quick2(high_bits, low_bits,
 603                                               dest, 32, ctx);
 604                 return;
 605         }
 606 
 607         /* Oh well, we tried... Do a full 64-bit decomposition.  */
 608         ctx->tmp_1_used = true;
 609 
 610         emit_loadimm32(high_bits, tmp, ctx);
 611         emit_loadimm32(low_bits, dest, ctx);
 612         emit_alu_K(SLLX, tmp, 32, ctx);
 613         emit(OR | RS1(dest) | RS2(tmp) | RD(dest), ctx);
 614 }
 615 
 616 static void emit_branch(unsigned int br_opc, unsigned int from_idx, unsigned int to_idx,
 617                         struct jit_ctx *ctx)
 618 {
 619         unsigned int off = to_idx - from_idx;
 620 
 621         if (br_opc & XCC)
 622                 emit(br_opc | WDISP19(off << 2), ctx);
 623         else
 624                 emit(br_opc | WDISP22(off << 2), ctx);
 625 }
 626 
 627 static void emit_cbcond(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
 628                         const u8 dst, const u8 src, struct jit_ctx *ctx)
 629 {
 630         unsigned int off = to_idx - from_idx;
 631 
 632         emit(cb_opc | WDISP10(off << 2) | RS1(dst) | RS2(src), ctx);
 633 }
 634 
 635 static void emit_cbcondi(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
 636                          const u8 dst, s32 imm, struct jit_ctx *ctx)
 637 {
 638         unsigned int off = to_idx - from_idx;
 639 
 640         emit(cb_opc | IMMED | WDISP10(off << 2) | RS1(dst) | S5(imm), ctx);
 641 }
 642 
 643 #define emit_read_y(REG, CTX)   emit(RD_Y | RD(REG), CTX)
 644 #define emit_write_y(REG, CTX)  emit(WR_Y | IMMED | RS1(REG) | S13(0), CTX)
 645 
 646 #define emit_cmp(R1, R2, CTX)                           \
 647         emit(SUBCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
 648 
 649 #define emit_cmpi(R1, IMM, CTX)                         \
 650         emit(SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
 651 
 652 #define emit_btst(R1, R2, CTX)                          \
 653         emit(ANDCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
 654 
 655 #define emit_btsti(R1, IMM, CTX)                        \
 656         emit(ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
 657 
 658 static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src,
 659                                    const s32 imm, bool is_imm, int branch_dst,
 660                                    struct jit_ctx *ctx)
 661 {
 662         bool use_cbcond = (sparc64_elf_hwcap & AV_SPARC_CBCOND) != 0;
 663         const u8 tmp = bpf2sparc[TMP_REG_1];
 664 
 665         branch_dst = ctx->offset[branch_dst];
 666 
 667         if (!is_simm10(branch_dst - ctx->idx) ||
 668             BPF_OP(code) == BPF_JSET)
 669                 use_cbcond = false;
 670 
 671         if (is_imm) {
 672                 bool fits = true;
 673 
 674                 if (use_cbcond) {
 675                         if (!is_simm5(imm))
 676                                 fits = false;
 677                 } else if (!is_simm13(imm)) {
 678                         fits = false;
 679                 }
 680                 if (!fits) {
 681                         ctx->tmp_1_used = true;
 682                         emit_loadimm_sext(imm, tmp, ctx);
 683                         src = tmp;
 684                         is_imm = false;
 685                 }
 686         }
 687 
 688         if (!use_cbcond) {
 689                 u32 br_opcode;
 690 
 691                 if (BPF_OP(code) == BPF_JSET) {
 692                         if (is_imm)
 693                                 emit_btsti(dst, imm, ctx);
 694                         else
 695                                 emit_btst(dst, src, ctx);
 696                 } else {
 697                         if (is_imm)
 698                                 emit_cmpi(dst, imm, ctx);
 699                         else
 700                                 emit_cmp(dst, src, ctx);
 701                 }
 702                 switch (BPF_OP(code)) {
 703                 case BPF_JEQ:
 704                         br_opcode = BE;
 705                         break;
 706                 case BPF_JGT:
 707                         br_opcode = BGU;
 708                         break;
 709                 case BPF_JLT:
 710                         br_opcode = BLU;
 711                         break;
 712                 case BPF_JGE:
 713                         br_opcode = BGEU;
 714                         break;
 715                 case BPF_JLE:
 716                         br_opcode = BLEU;
 717                         break;
 718                 case BPF_JSET:
 719                 case BPF_JNE:
 720                         br_opcode = BNE;
 721                         break;
 722                 case BPF_JSGT:
 723                         br_opcode = BG;
 724                         break;
 725                 case BPF_JSLT:
 726                         br_opcode = BL;
 727                         break;
 728                 case BPF_JSGE:
 729                         br_opcode = BGE;
 730                         break;
 731                 case BPF_JSLE:
 732                         br_opcode = BLE;
 733                         break;
 734                 default:
 735                         /* Make sure we dont leak kernel information to the
 736                          * user.
 737                          */
 738                         return -EFAULT;
 739                 }
 740                 emit_branch(br_opcode, ctx->idx, branch_dst, ctx);
 741                 emit_nop(ctx);
 742         } else {
 743                 u32 cbcond_opcode;
 744 
 745                 switch (BPF_OP(code)) {
 746                 case BPF_JEQ:
 747                         cbcond_opcode = CBCONDE;
 748                         break;
 749                 case BPF_JGT:
 750                         cbcond_opcode = CBCONDGU;
 751                         break;
 752                 case BPF_JLT:
 753                         cbcond_opcode = CBCONDLU;
 754                         break;
 755                 case BPF_JGE:
 756                         cbcond_opcode = CBCONDGEU;
 757                         break;
 758                 case BPF_JLE:
 759                         cbcond_opcode = CBCONDLEU;
 760                         break;
 761                 case BPF_JNE:
 762                         cbcond_opcode = CBCONDNE;
 763                         break;
 764                 case BPF_JSGT:
 765                         cbcond_opcode = CBCONDG;
 766                         break;
 767                 case BPF_JSLT:
 768                         cbcond_opcode = CBCONDL;
 769                         break;
 770                 case BPF_JSGE:
 771                         cbcond_opcode = CBCONDGE;
 772                         break;
 773                 case BPF_JSLE:
 774                         cbcond_opcode = CBCONDLE;
 775                         break;
 776                 default:
 777                         /* Make sure we dont leak kernel information to the
 778                          * user.
 779                          */
 780                         return -EFAULT;
 781                 }
 782                 cbcond_opcode |= CBCOND_OP;
 783                 if (is_imm)
 784                         emit_cbcondi(cbcond_opcode, ctx->idx, branch_dst,
 785                                      dst, imm, ctx);
 786                 else
 787                         emit_cbcond(cbcond_opcode, ctx->idx, branch_dst,
 788                                     dst, src, ctx);
 789         }
 790         return 0;
 791 }
 792 
 793 /* Just skip the save instruction and the ctx register move.  */
 794 #define BPF_TAILCALL_PROLOGUE_SKIP      32
 795 #define BPF_TAILCALL_CNT_SP_OFF         (STACK_BIAS + 128)
 796 
 797 static void build_prologue(struct jit_ctx *ctx)
 798 {
 799         s32 stack_needed = BASE_STACKFRAME;
 800 
 801         if (ctx->saw_frame_pointer || ctx->saw_tail_call) {
 802                 struct bpf_prog *prog = ctx->prog;
 803                 u32 stack_depth;
 804 
 805                 stack_depth = prog->aux->stack_depth;
 806                 stack_needed += round_up(stack_depth, 16);
 807         }
 808 
 809         if (ctx->saw_tail_call)
 810                 stack_needed += 8;
 811 
 812         /* save %sp, -176, %sp */
 813         emit(SAVE | IMMED | RS1(SP) | S13(-stack_needed) | RD(SP), ctx);
 814 
 815         /* tail_call_cnt = 0 */
 816         if (ctx->saw_tail_call) {
 817                 u32 off = BPF_TAILCALL_CNT_SP_OFF;
 818 
 819                 emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(G0), ctx);
 820         } else {
 821                 emit_nop(ctx);
 822         }
 823         if (ctx->saw_frame_pointer) {
 824                 const u8 vfp = bpf2sparc[BPF_REG_FP];
 825 
 826                 emit(ADD | IMMED | RS1(FP) | S13(STACK_BIAS) | RD(vfp), ctx);
 827         } else {
 828                 emit_nop(ctx);
 829         }
 830 
 831         emit_reg_move(I0, O0, ctx);
 832         emit_reg_move(I1, O1, ctx);
 833         emit_reg_move(I2, O2, ctx);
 834         emit_reg_move(I3, O3, ctx);
 835         emit_reg_move(I4, O4, ctx);
 836         /* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */
 837 }
 838 
 839 static void build_epilogue(struct jit_ctx *ctx)
 840 {
 841         ctx->epilogue_offset = ctx->idx;
 842 
 843         /* ret (jmpl %i7 + 8, %g0) */
 844         emit(JMPL | IMMED | RS1(I7) | S13(8) | RD(G0), ctx);
 845 
 846         /* restore %i5, %g0, %o0 */
 847         emit(RESTORE | RS1(bpf2sparc[BPF_REG_0]) | RS2(G0) | RD(O0), ctx);
 848 }
 849 
 850 static void emit_tail_call(struct jit_ctx *ctx)
 851 {
 852         const u8 bpf_array = bpf2sparc[BPF_REG_2];
 853         const u8 bpf_index = bpf2sparc[BPF_REG_3];
 854         const u8 tmp = bpf2sparc[TMP_REG_1];
 855         u32 off;
 856 
 857         ctx->saw_tail_call = true;
 858 
 859         off = offsetof(struct bpf_array, map.max_entries);
 860         emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx);
 861         emit_cmp(bpf_index, tmp, ctx);
 862 #define OFFSET1 17
 863         emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET1, ctx);
 864         emit_nop(ctx);
 865 
 866         off = BPF_TAILCALL_CNT_SP_OFF;
 867         emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
 868         emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx);
 869 #define OFFSET2 13
 870         emit_branch(BGU, ctx->idx, ctx->idx + OFFSET2, ctx);
 871         emit_nop(ctx);
 872 
 873         emit_alu_K(ADD, tmp, 1, ctx);
 874         off = BPF_TAILCALL_CNT_SP_OFF;
 875         emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
 876 
 877         emit_alu3_K(SLL, bpf_index, 3, tmp, ctx);
 878         emit_alu(ADD, bpf_array, tmp, ctx);
 879         off = offsetof(struct bpf_array, ptrs);
 880         emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
 881 
 882         emit_cmpi(tmp, 0, ctx);
 883 #define OFFSET3 5
 884         emit_branch(BE, ctx->idx, ctx->idx + OFFSET3, ctx);
 885         emit_nop(ctx);
 886 
 887         off = offsetof(struct bpf_prog, bpf_func);
 888         emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
 889 
 890         off = BPF_TAILCALL_PROLOGUE_SKIP;
 891         emit(JMPL | IMMED | RS1(tmp) | S13(off) | RD(G0), ctx);
 892         emit_nop(ctx);
 893 }
 894 
 895 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 896 {
 897         const u8 code = insn->code;
 898         const u8 dst = bpf2sparc[insn->dst_reg];
 899         const u8 src = bpf2sparc[insn->src_reg];
 900         const int i = insn - ctx->prog->insnsi;
 901         const s16 off = insn->off;
 902         const s32 imm = insn->imm;
 903 
 904         if (insn->src_reg == BPF_REG_FP)
 905                 ctx->saw_frame_pointer = true;
 906 
 907         switch (code) {
 908         /* dst = src */
 909         case BPF_ALU | BPF_MOV | BPF_X:
 910                 emit_alu3_K(SRL, src, 0, dst, ctx);
 911                 if (insn_is_zext(&insn[1]))
 912                         return 1;
 913                 break;
 914         case BPF_ALU64 | BPF_MOV | BPF_X:
 915                 emit_reg_move(src, dst, ctx);
 916                 break;
 917         /* dst = dst OP src */
 918         case BPF_ALU | BPF_ADD | BPF_X:
 919         case BPF_ALU64 | BPF_ADD | BPF_X:
 920                 emit_alu(ADD, src, dst, ctx);
 921                 goto do_alu32_trunc;
 922         case BPF_ALU | BPF_SUB | BPF_X:
 923         case BPF_ALU64 | BPF_SUB | BPF_X:
 924                 emit_alu(SUB, src, dst, ctx);
 925                 goto do_alu32_trunc;
 926         case BPF_ALU | BPF_AND | BPF_X:
 927         case BPF_ALU64 | BPF_AND | BPF_X:
 928                 emit_alu(AND, src, dst, ctx);
 929                 goto do_alu32_trunc;
 930         case BPF_ALU | BPF_OR | BPF_X:
 931         case BPF_ALU64 | BPF_OR | BPF_X:
 932                 emit_alu(OR, src, dst, ctx);
 933                 goto do_alu32_trunc;
 934         case BPF_ALU | BPF_XOR | BPF_X:
 935         case BPF_ALU64 | BPF_XOR | BPF_X:
 936                 emit_alu(XOR, src, dst, ctx);
 937                 goto do_alu32_trunc;
 938         case BPF_ALU | BPF_MUL | BPF_X:
 939                 emit_alu(MUL, src, dst, ctx);
 940                 goto do_alu32_trunc;
 941         case BPF_ALU64 | BPF_MUL | BPF_X:
 942                 emit_alu(MULX, src, dst, ctx);
 943                 break;
 944         case BPF_ALU | BPF_DIV | BPF_X:
 945                 emit_write_y(G0, ctx);
 946                 emit_alu(DIV, src, dst, ctx);
 947                 if (insn_is_zext(&insn[1]))
 948                         return 1;
 949                 break;
 950         case BPF_ALU64 | BPF_DIV | BPF_X:
 951                 emit_alu(UDIVX, src, dst, ctx);
 952                 break;
 953         case BPF_ALU | BPF_MOD | BPF_X: {
 954                 const u8 tmp = bpf2sparc[TMP_REG_1];
 955 
 956                 ctx->tmp_1_used = true;
 957 
 958                 emit_write_y(G0, ctx);
 959                 emit_alu3(DIV, dst, src, tmp, ctx);
 960                 emit_alu3(MULX, tmp, src, tmp, ctx);
 961                 emit_alu3(SUB, dst, tmp, dst, ctx);
 962                 goto do_alu32_trunc;
 963         }
 964         case BPF_ALU64 | BPF_MOD | BPF_X: {
 965                 const u8 tmp = bpf2sparc[TMP_REG_1];
 966 
 967                 ctx->tmp_1_used = true;
 968 
 969                 emit_alu3(UDIVX, dst, src, tmp, ctx);
 970                 emit_alu3(MULX, tmp, src, tmp, ctx);
 971                 emit_alu3(SUB, dst, tmp, dst, ctx);
 972                 break;
 973         }
 974         case BPF_ALU | BPF_LSH | BPF_X:
 975                 emit_alu(SLL, src, dst, ctx);
 976                 goto do_alu32_trunc;
 977         case BPF_ALU64 | BPF_LSH | BPF_X:
 978                 emit_alu(SLLX, src, dst, ctx);
 979                 break;
 980         case BPF_ALU | BPF_RSH | BPF_X:
 981                 emit_alu(SRL, src, dst, ctx);
 982                 if (insn_is_zext(&insn[1]))
 983                         return 1;
 984                 break;
 985         case BPF_ALU64 | BPF_RSH | BPF_X:
 986                 emit_alu(SRLX, src, dst, ctx);
 987                 break;
 988         case BPF_ALU | BPF_ARSH | BPF_X:
 989                 emit_alu(SRA, src, dst, ctx);
 990                 goto do_alu32_trunc;
 991         case BPF_ALU64 | BPF_ARSH | BPF_X:
 992                 emit_alu(SRAX, src, dst, ctx);
 993                 break;
 994 
 995         /* dst = -dst */
 996         case BPF_ALU | BPF_NEG:
 997         case BPF_ALU64 | BPF_NEG:
 998                 emit(SUB | RS1(0) | RS2(dst) | RD(dst), ctx);
 999                 goto do_alu32_trunc;
1000 
1001         case BPF_ALU | BPF_END | BPF_FROM_BE:
1002                 switch (imm) {
1003                 case 16:
1004                         emit_alu_K(SLL, dst, 16, ctx);
1005                         emit_alu_K(SRL, dst, 16, ctx);
1006                         if (insn_is_zext(&insn[1]))
1007                                 return 1;
1008                         break;
1009                 case 32:
1010                         if (!ctx->prog->aux->verifier_zext)
1011                                 emit_alu_K(SRL, dst, 0, ctx);
1012                         break;
1013                 case 64:
1014                         /* nop */
1015                         break;
1016 
1017                 }
1018                 break;
1019 
1020         /* dst = BSWAP##imm(dst) */
1021         case BPF_ALU | BPF_END | BPF_FROM_LE: {
1022                 const u8 tmp = bpf2sparc[TMP_REG_1];
1023                 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1024 
1025                 ctx->tmp_1_used = true;
1026                 switch (imm) {
1027                 case 16:
1028                         emit_alu3_K(AND, dst, 0xff, tmp, ctx);
1029                         emit_alu3_K(SRL, dst, 8, dst, ctx);
1030                         emit_alu3_K(AND, dst, 0xff, dst, ctx);
1031                         emit_alu3_K(SLL, tmp, 8, tmp, ctx);
1032                         emit_alu(OR, tmp, dst, ctx);
1033                         if (insn_is_zext(&insn[1]))
1034                                 return 1;
1035                         break;
1036 
1037                 case 32:
1038                         ctx->tmp_2_used = true;
1039                         emit_alu3_K(SRL, dst, 24, tmp, ctx);    /* tmp  = dst >> 24 */
1040                         emit_alu3_K(SRL, dst, 16, tmp2, ctx);   /* tmp2 = dst >> 16 */
1041                         emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1042                         emit_alu3_K(SLL, tmp2, 8, tmp2, ctx);   /* tmp2 = tmp2 << 8 */
1043                         emit_alu(OR, tmp2, tmp, ctx);           /* tmp  = tmp | tmp2 */
1044                         emit_alu3_K(SRL, dst, 8, tmp2, ctx);    /* tmp2 = dst >> 8 */
1045                         emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1046                         emit_alu3_K(SLL, tmp2, 16, tmp2, ctx);  /* tmp2 = tmp2 << 16 */
1047                         emit_alu(OR, tmp2, tmp, ctx);           /* tmp  = tmp | tmp2 */
1048                         emit_alu3_K(AND, dst, 0xff, dst, ctx);  /* dst  = dst & 0xff */
1049                         emit_alu3_K(SLL, dst, 24, dst, ctx);    /* dst  = dst << 24 */
1050                         emit_alu(OR, tmp, dst, ctx);            /* dst  = dst | tmp */
1051                         if (insn_is_zext(&insn[1]))
1052                                 return 1;
1053                         break;
1054 
1055                 case 64:
1056                         emit_alu3_K(ADD, SP, STACK_BIAS + 128, tmp, ctx);
1057                         emit(ST64 | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1058                         emit(LD64A | ASI(ASI_PL) | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1059                         break;
1060                 }
1061                 break;
1062         }
1063         /* dst = imm */
1064         case BPF_ALU | BPF_MOV | BPF_K:
1065                 emit_loadimm32(imm, dst, ctx);
1066                 if (insn_is_zext(&insn[1]))
1067                         return 1;
1068                 break;
1069         case BPF_ALU64 | BPF_MOV | BPF_K:
1070                 emit_loadimm_sext(imm, dst, ctx);
1071                 break;
1072         /* dst = dst OP imm */
1073         case BPF_ALU | BPF_ADD | BPF_K:
1074         case BPF_ALU64 | BPF_ADD | BPF_K:
1075                 emit_alu_K(ADD, dst, imm, ctx);
1076                 goto do_alu32_trunc;
1077         case BPF_ALU | BPF_SUB | BPF_K:
1078         case BPF_ALU64 | BPF_SUB | BPF_K:
1079                 emit_alu_K(SUB, dst, imm, ctx);
1080                 goto do_alu32_trunc;
1081         case BPF_ALU | BPF_AND | BPF_K:
1082         case BPF_ALU64 | BPF_AND | BPF_K:
1083                 emit_alu_K(AND, dst, imm, ctx);
1084                 goto do_alu32_trunc;
1085         case BPF_ALU | BPF_OR | BPF_K:
1086         case BPF_ALU64 | BPF_OR | BPF_K:
1087                 emit_alu_K(OR, dst, imm, ctx);
1088                 goto do_alu32_trunc;
1089         case BPF_ALU | BPF_XOR | BPF_K:
1090         case BPF_ALU64 | BPF_XOR | BPF_K:
1091                 emit_alu_K(XOR, dst, imm, ctx);
1092                 goto do_alu32_trunc;
1093         case BPF_ALU | BPF_MUL | BPF_K:
1094                 emit_alu_K(MUL, dst, imm, ctx);
1095                 goto do_alu32_trunc;
1096         case BPF_ALU64 | BPF_MUL | BPF_K:
1097                 emit_alu_K(MULX, dst, imm, ctx);
1098                 break;
1099         case BPF_ALU | BPF_DIV | BPF_K:
1100                 if (imm == 0)
1101                         return -EINVAL;
1102 
1103                 emit_write_y(G0, ctx);
1104                 emit_alu_K(DIV, dst, imm, ctx);
1105                 goto do_alu32_trunc;
1106         case BPF_ALU64 | BPF_DIV | BPF_K:
1107                 if (imm == 0)
1108                         return -EINVAL;
1109 
1110                 emit_alu_K(UDIVX, dst, imm, ctx);
1111                 break;
1112         case BPF_ALU64 | BPF_MOD | BPF_K:
1113         case BPF_ALU | BPF_MOD | BPF_K: {
1114                 const u8 tmp = bpf2sparc[TMP_REG_2];
1115                 unsigned int div;
1116 
1117                 if (imm == 0)
1118                         return -EINVAL;
1119 
1120                 div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV;
1121 
1122                 ctx->tmp_2_used = true;
1123 
1124                 if (BPF_CLASS(code) != BPF_ALU64)
1125                         emit_write_y(G0, ctx);
1126                 if (is_simm13(imm)) {
1127                         emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx);
1128                         emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx);
1129                         emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1130                 } else {
1131                         const u8 tmp1 = bpf2sparc[TMP_REG_1];
1132 
1133                         ctx->tmp_1_used = true;
1134 
1135                         emit_set_const_sext(imm, tmp1, ctx);
1136                         emit(div | RS1(dst) | RS2(tmp1) | RD(tmp), ctx);
1137                         emit(MULX | RS1(tmp) | RS2(tmp1) | RD(tmp), ctx);
1138                         emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1139                 }
1140                 goto do_alu32_trunc;
1141         }
1142         case BPF_ALU | BPF_LSH | BPF_K:
1143                 emit_alu_K(SLL, dst, imm, ctx);
1144                 goto do_alu32_trunc;
1145         case BPF_ALU64 | BPF_LSH | BPF_K:
1146                 emit_alu_K(SLLX, dst, imm, ctx);
1147                 break;
1148         case BPF_ALU | BPF_RSH | BPF_K:
1149                 emit_alu_K(SRL, dst, imm, ctx);
1150                 if (insn_is_zext(&insn[1]))
1151                         return 1;
1152                 break;
1153         case BPF_ALU64 | BPF_RSH | BPF_K:
1154                 emit_alu_K(SRLX, dst, imm, ctx);
1155                 break;
1156         case BPF_ALU | BPF_ARSH | BPF_K:
1157                 emit_alu_K(SRA, dst, imm, ctx);
1158                 goto do_alu32_trunc;
1159         case BPF_ALU64 | BPF_ARSH | BPF_K:
1160                 emit_alu_K(SRAX, dst, imm, ctx);
1161                 break;
1162 
1163         do_alu32_trunc:
1164                 if (BPF_CLASS(code) == BPF_ALU &&
1165                     !ctx->prog->aux->verifier_zext)
1166                         emit_alu_K(SRL, dst, 0, ctx);
1167                 break;
1168 
1169         /* JUMP off */
1170         case BPF_JMP | BPF_JA:
1171                 emit_branch(BA, ctx->idx, ctx->offset[i + off], ctx);
1172                 emit_nop(ctx);
1173                 break;
1174         /* IF (dst COND src) JUMP off */
1175         case BPF_JMP | BPF_JEQ | BPF_X:
1176         case BPF_JMP | BPF_JGT | BPF_X:
1177         case BPF_JMP | BPF_JLT | BPF_X:
1178         case BPF_JMP | BPF_JGE | BPF_X:
1179         case BPF_JMP | BPF_JLE | BPF_X:
1180         case BPF_JMP | BPF_JNE | BPF_X:
1181         case BPF_JMP | BPF_JSGT | BPF_X:
1182         case BPF_JMP | BPF_JSLT | BPF_X:
1183         case BPF_JMP | BPF_JSGE | BPF_X:
1184         case BPF_JMP | BPF_JSLE | BPF_X:
1185         case BPF_JMP | BPF_JSET | BPF_X: {
1186                 int err;
1187 
1188                 err = emit_compare_and_branch(code, dst, src, 0, false, i + off, ctx);
1189                 if (err)
1190                         return err;
1191                 break;
1192         }
1193         /* IF (dst COND imm) JUMP off */
1194         case BPF_JMP | BPF_JEQ | BPF_K:
1195         case BPF_JMP | BPF_JGT | BPF_K:
1196         case BPF_JMP | BPF_JLT | BPF_K:
1197         case BPF_JMP | BPF_JGE | BPF_K:
1198         case BPF_JMP | BPF_JLE | BPF_K:
1199         case BPF_JMP | BPF_JNE | BPF_K:
1200         case BPF_JMP | BPF_JSGT | BPF_K:
1201         case BPF_JMP | BPF_JSLT | BPF_K:
1202         case BPF_JMP | BPF_JSGE | BPF_K:
1203         case BPF_JMP | BPF_JSLE | BPF_K:
1204         case BPF_JMP | BPF_JSET | BPF_K: {
1205                 int err;
1206 
1207                 err = emit_compare_and_branch(code, dst, 0, imm, true, i + off, ctx);
1208                 if (err)
1209                         return err;
1210                 break;
1211         }
1212 
1213         /* function call */
1214         case BPF_JMP | BPF_CALL:
1215         {
1216                 u8 *func = ((u8 *)__bpf_call_base) + imm;
1217 
1218                 ctx->saw_call = true;
1219 
1220                 emit_call((u32 *)func, ctx);
1221                 emit_nop(ctx);
1222 
1223                 emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
1224                 break;
1225         }
1226 
1227         /* tail call */
1228         case BPF_JMP | BPF_TAIL_CALL:
1229                 emit_tail_call(ctx);
1230                 break;
1231 
1232         /* function return */
1233         case BPF_JMP | BPF_EXIT:
1234                 /* Optimization: when last instruction is EXIT,
1235                    simply fallthrough to epilogue. */
1236                 if (i == ctx->prog->len - 1)
1237                         break;
1238                 emit_branch(BA, ctx->idx, ctx->epilogue_offset, ctx);
1239                 emit_nop(ctx);
1240                 break;
1241 
1242         /* dst = imm64 */
1243         case BPF_LD | BPF_IMM | BPF_DW:
1244         {
1245                 const struct bpf_insn insn1 = insn[1];
1246                 u64 imm64;
1247 
1248                 imm64 = (u64)insn1.imm << 32 | (u32)imm;
1249                 emit_loadimm64(imm64, dst, ctx);
1250 
1251                 return 1;
1252         }
1253 
1254         /* LDX: dst = *(size *)(src + off) */
1255         case BPF_LDX | BPF_MEM | BPF_W:
1256         case BPF_LDX | BPF_MEM | BPF_H:
1257         case BPF_LDX | BPF_MEM | BPF_B:
1258         case BPF_LDX | BPF_MEM | BPF_DW: {
1259                 const u8 tmp = bpf2sparc[TMP_REG_1];
1260                 u32 opcode = 0, rs2;
1261 
1262                 ctx->tmp_1_used = true;
1263                 switch (BPF_SIZE(code)) {
1264                 case BPF_W:
1265                         opcode = LD32;
1266                         break;
1267                 case BPF_H:
1268                         opcode = LD16;
1269                         break;
1270                 case BPF_B:
1271                         opcode = LD8;
1272                         break;
1273                 case BPF_DW:
1274                         opcode = LD64;
1275                         break;
1276                 }
1277 
1278                 if (is_simm13(off)) {
1279                         opcode |= IMMED;
1280                         rs2 = S13(off);
1281                 } else {
1282                         emit_loadimm(off, tmp, ctx);
1283                         rs2 = RS2(tmp);
1284                 }
1285                 emit(opcode | RS1(src) | rs2 | RD(dst), ctx);
1286                 if (opcode != LD64 && insn_is_zext(&insn[1]))
1287                         return 1;
1288                 break;
1289         }
1290         /* ST: *(size *)(dst + off) = imm */
1291         case BPF_ST | BPF_MEM | BPF_W:
1292         case BPF_ST | BPF_MEM | BPF_H:
1293         case BPF_ST | BPF_MEM | BPF_B:
1294         case BPF_ST | BPF_MEM | BPF_DW: {
1295                 const u8 tmp = bpf2sparc[TMP_REG_1];
1296                 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1297                 u32 opcode = 0, rs2;
1298 
1299                 if (insn->dst_reg == BPF_REG_FP)
1300                         ctx->saw_frame_pointer = true;
1301 
1302                 ctx->tmp_2_used = true;
1303                 emit_loadimm(imm, tmp2, ctx);
1304 
1305                 switch (BPF_SIZE(code)) {
1306                 case BPF_W:
1307                         opcode = ST32;
1308                         break;
1309                 case BPF_H:
1310                         opcode = ST16;
1311                         break;
1312                 case BPF_B:
1313                         opcode = ST8;
1314                         break;
1315                 case BPF_DW:
1316                         opcode = ST64;
1317                         break;
1318                 }
1319 
1320                 if (is_simm13(off)) {
1321                         opcode |= IMMED;
1322                         rs2 = S13(off);
1323                 } else {
1324                         ctx->tmp_1_used = true;
1325                         emit_loadimm(off, tmp, ctx);
1326                         rs2 = RS2(tmp);
1327                 }
1328                 emit(opcode | RS1(dst) | rs2 | RD(tmp2), ctx);
1329                 break;
1330         }
1331 
1332         /* STX: *(size *)(dst + off) = src */
1333         case BPF_STX | BPF_MEM | BPF_W:
1334         case BPF_STX | BPF_MEM | BPF_H:
1335         case BPF_STX | BPF_MEM | BPF_B:
1336         case BPF_STX | BPF_MEM | BPF_DW: {
1337                 const u8 tmp = bpf2sparc[TMP_REG_1];
1338                 u32 opcode = 0, rs2;
1339 
1340                 if (insn->dst_reg == BPF_REG_FP)
1341                         ctx->saw_frame_pointer = true;
1342 
1343                 switch (BPF_SIZE(code)) {
1344                 case BPF_W:
1345                         opcode = ST32;
1346                         break;
1347                 case BPF_H:
1348                         opcode = ST16;
1349                         break;
1350                 case BPF_B:
1351                         opcode = ST8;
1352                         break;
1353                 case BPF_DW:
1354                         opcode = ST64;
1355                         break;
1356                 }
1357                 if (is_simm13(off)) {
1358                         opcode |= IMMED;
1359                         rs2 = S13(off);
1360                 } else {
1361                         ctx->tmp_1_used = true;
1362                         emit_loadimm(off, tmp, ctx);
1363                         rs2 = RS2(tmp);
1364                 }
1365                 emit(opcode | RS1(dst) | rs2 | RD(src), ctx);
1366                 break;
1367         }
1368 
1369         /* STX XADD: lock *(u32 *)(dst + off) += src */
1370         case BPF_STX | BPF_XADD | BPF_W: {
1371                 const u8 tmp = bpf2sparc[TMP_REG_1];
1372                 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1373                 const u8 tmp3 = bpf2sparc[TMP_REG_3];
1374 
1375                 if (insn->dst_reg == BPF_REG_FP)
1376                         ctx->saw_frame_pointer = true;
1377 
1378                 ctx->tmp_1_used = true;
1379                 ctx->tmp_2_used = true;
1380                 ctx->tmp_3_used = true;
1381                 emit_loadimm(off, tmp, ctx);
1382                 emit_alu3(ADD, dst, tmp, tmp, ctx);
1383 
1384                 emit(LD32 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1385                 emit_alu3(ADD, tmp2, src, tmp3, ctx);
1386                 emit(CAS | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1387                 emit_cmp(tmp2, tmp3, ctx);
1388                 emit_branch(BNE, 4, 0, ctx);
1389                 emit_nop(ctx);
1390                 break;
1391         }
1392         /* STX XADD: lock *(u64 *)(dst + off) += src */
1393         case BPF_STX | BPF_XADD | BPF_DW: {
1394                 const u8 tmp = bpf2sparc[TMP_REG_1];
1395                 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1396                 const u8 tmp3 = bpf2sparc[TMP_REG_3];
1397 
1398                 if (insn->dst_reg == BPF_REG_FP)
1399                         ctx->saw_frame_pointer = true;
1400 
1401                 ctx->tmp_1_used = true;
1402                 ctx->tmp_2_used = true;
1403                 ctx->tmp_3_used = true;
1404                 emit_loadimm(off, tmp, ctx);
1405                 emit_alu3(ADD, dst, tmp, tmp, ctx);
1406 
1407                 emit(LD64 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1408                 emit_alu3(ADD, tmp2, src, tmp3, ctx);
1409                 emit(CASX | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1410                 emit_cmp(tmp2, tmp3, ctx);
1411                 emit_branch(BNE, 4, 0, ctx);
1412                 emit_nop(ctx);
1413                 break;
1414         }
1415 
1416         default:
1417                 pr_err_once("unknown opcode %02x\n", code);
1418                 return -EINVAL;
1419         }
1420 
1421         return 0;
1422 }
1423 
1424 static int build_body(struct jit_ctx *ctx)
1425 {
1426         const struct bpf_prog *prog = ctx->prog;
1427         int i;
1428 
1429         for (i = 0; i < prog->len; i++) {
1430                 const struct bpf_insn *insn = &prog->insnsi[i];
1431                 int ret;
1432 
1433                 ret = build_insn(insn, ctx);
1434 
1435                 if (ret > 0) {
1436                         i++;
1437                         ctx->offset[i] = ctx->idx;
1438                         continue;
1439                 }
1440                 ctx->offset[i] = ctx->idx;
1441                 if (ret)
1442                         return ret;
1443         }
1444         return 0;
1445 }
1446 
1447 static void jit_fill_hole(void *area, unsigned int size)
1448 {
1449         u32 *ptr;
1450         /* We are guaranteed to have aligned memory. */
1451         for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
1452                 *ptr++ = 0x91d02005; /* ta 5 */
1453 }
1454 
1455 bool bpf_jit_needs_zext(void)
1456 {
1457         return true;
1458 }
1459 
1460 struct sparc64_jit_data {
1461         struct bpf_binary_header *header;
1462         u8 *image;
1463         struct jit_ctx ctx;
1464 };
1465 
1466 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1467 {
1468         struct bpf_prog *tmp, *orig_prog = prog;
1469         struct sparc64_jit_data *jit_data;
1470         struct bpf_binary_header *header;
1471         u32 prev_image_size, image_size;
1472         bool tmp_blinded = false;
1473         bool extra_pass = false;
1474         struct jit_ctx ctx;
1475         u8 *image_ptr;
1476         int pass, i;
1477 
1478         if (!prog->jit_requested)
1479                 return orig_prog;
1480 
1481         tmp = bpf_jit_blind_constants(prog);
1482         /* If blinding was requested and we failed during blinding,
1483          * we must fall back to the interpreter.
1484          */
1485         if (IS_ERR(tmp))
1486                 return orig_prog;
1487         if (tmp != prog) {
1488                 tmp_blinded = true;
1489                 prog = tmp;
1490         }
1491 
1492         jit_data = prog->aux->jit_data;
1493         if (!jit_data) {
1494                 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1495                 if (!jit_data) {
1496                         prog = orig_prog;
1497                         goto out;
1498                 }
1499                 prog->aux->jit_data = jit_data;
1500         }
1501         if (jit_data->ctx.offset) {
1502                 ctx = jit_data->ctx;
1503                 image_ptr = jit_data->image;
1504                 header = jit_data->header;
1505                 extra_pass = true;
1506                 image_size = sizeof(u32) * ctx.idx;
1507                 prev_image_size = image_size;
1508                 pass = 1;
1509                 goto skip_init_ctx;
1510         }
1511 
1512         memset(&ctx, 0, sizeof(ctx));
1513         ctx.prog = prog;
1514 
1515         ctx.offset = kmalloc_array(prog->len, sizeof(unsigned int), GFP_KERNEL);
1516         if (ctx.offset == NULL) {
1517                 prog = orig_prog;
1518                 goto out_off;
1519         }
1520 
1521         /* Longest sequence emitted is for bswap32, 12 instructions.  Pre-cook
1522          * the offset array so that we converge faster.
1523          */
1524         for (i = 0; i < prog->len; i++)
1525                 ctx.offset[i] = i * (12 * 4);
1526 
1527         prev_image_size = ~0U;
1528         for (pass = 1; pass < 40; pass++) {
1529                 ctx.idx = 0;
1530 
1531                 build_prologue(&ctx);
1532                 if (build_body(&ctx)) {
1533                         prog = orig_prog;
1534                         goto out_off;
1535                 }
1536                 build_epilogue(&ctx);
1537 
1538                 if (bpf_jit_enable > 1)
1539                         pr_info("Pass %d: size = %u, seen = [%c%c%c%c%c%c]\n", pass,
1540                                 ctx.idx * 4,
1541                                 ctx.tmp_1_used ? '1' : ' ',
1542                                 ctx.tmp_2_used ? '2' : ' ',
1543                                 ctx.tmp_3_used ? '3' : ' ',
1544                                 ctx.saw_frame_pointer ? 'F' : ' ',
1545                                 ctx.saw_call ? 'C' : ' ',
1546                                 ctx.saw_tail_call ? 'T' : ' ');
1547 
1548                 if (ctx.idx * 4 == prev_image_size)
1549                         break;
1550                 prev_image_size = ctx.idx * 4;
1551                 cond_resched();
1552         }
1553 
1554         /* Now we know the actual image size. */
1555         image_size = sizeof(u32) * ctx.idx;
1556         header = bpf_jit_binary_alloc(image_size, &image_ptr,
1557                                       sizeof(u32), jit_fill_hole);
1558         if (header == NULL) {
1559                 prog = orig_prog;
1560                 goto out_off;
1561         }
1562 
1563         ctx.image = (u32 *)image_ptr;
1564 skip_init_ctx:
1565         ctx.idx = 0;
1566 
1567         build_prologue(&ctx);
1568 
1569         if (build_body(&ctx)) {
1570                 bpf_jit_binary_free(header);
1571                 prog = orig_prog;
1572                 goto out_off;
1573         }
1574 
1575         build_epilogue(&ctx);
1576 
1577         if (ctx.idx * 4 != prev_image_size) {
1578                 pr_err("bpf_jit: Failed to converge, prev_size=%u size=%d\n",
1579                        prev_image_size, ctx.idx * 4);
1580                 bpf_jit_binary_free(header);
1581                 prog = orig_prog;
1582                 goto out_off;
1583         }
1584 
1585         if (bpf_jit_enable > 1)
1586                 bpf_jit_dump(prog->len, image_size, pass, ctx.image);
1587 
1588         bpf_flush_icache(header, (u8 *)header + (header->pages * PAGE_SIZE));
1589 
1590         if (!prog->is_func || extra_pass) {
1591                 bpf_jit_binary_lock_ro(header);
1592         } else {
1593                 jit_data->ctx = ctx;
1594                 jit_data->image = image_ptr;
1595                 jit_data->header = header;
1596         }
1597 
1598         prog->bpf_func = (void *)ctx.image;
1599         prog->jited = 1;
1600         prog->jited_len = image_size;
1601 
1602         if (!prog->is_func || extra_pass) {
1603                 bpf_prog_fill_jited_linfo(prog, ctx.offset);
1604 out_off:
1605                 kfree(ctx.offset);
1606                 kfree(jit_data);
1607                 prog->aux->jit_data = NULL;
1608         }
1609 out:
1610         if (tmp_blinded)
1611                 bpf_jit_prog_release_other(prog, prog == orig_prog ?
1612                                            tmp : orig_prog);
1613         return prog;
1614 }
/* [<][>][^][v][top][bottom][index][help] */
root/arch/sparc/net/bpf_jit_comp_64.c

DEFINITIONS
