1/* 2 * Copyright (C) 2013 Huawei Ltd. 3 * Author: Jiang Liu <liuj97@gmail.com> 4 * 5 * Copyright (C) 2014 Zi Shen Lim <zlim.lnx@gmail.com> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 18 */ 19#include <linux/bitops.h> 20#include <linux/bug.h> 21#include <linux/compiler.h> 22#include <linux/kernel.h> 23#include <linux/mm.h> 24#include <linux/smp.h> 25#include <linux/spinlock.h> 26#include <linux/stop_machine.h> 27#include <linux/types.h> 28#include <linux/uaccess.h> 29 30#include <asm/cacheflush.h> 31#include <asm/debug-monitors.h> 32#include <asm/fixmap.h> 33#include <asm/insn.h> 34 35#define AARCH64_INSN_SF_BIT BIT(31) 36#define AARCH64_INSN_N_BIT BIT(22) 37 38static int aarch64_insn_encoding_class[] = { 39 AARCH64_INSN_CLS_UNKNOWN, 40 AARCH64_INSN_CLS_UNKNOWN, 41 AARCH64_INSN_CLS_UNKNOWN, 42 AARCH64_INSN_CLS_UNKNOWN, 43 AARCH64_INSN_CLS_LDST, 44 AARCH64_INSN_CLS_DP_REG, 45 AARCH64_INSN_CLS_LDST, 46 AARCH64_INSN_CLS_DP_FPSIMD, 47 AARCH64_INSN_CLS_DP_IMM, 48 AARCH64_INSN_CLS_DP_IMM, 49 AARCH64_INSN_CLS_BR_SYS, 50 AARCH64_INSN_CLS_BR_SYS, 51 AARCH64_INSN_CLS_LDST, 52 AARCH64_INSN_CLS_DP_REG, 53 AARCH64_INSN_CLS_LDST, 54 AARCH64_INSN_CLS_DP_FPSIMD, 55}; 56 57enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn) 58{ 59 return aarch64_insn_encoding_class[(insn >> 25) & 0xf]; 60} 61 62/* NOP is an alias of HINT */ 63bool __kprobes aarch64_insn_is_nop(u32 insn) 64{ 65 if (!aarch64_insn_is_hint(insn)) 66 return false; 67 68 switch (insn & 0xFE0) { 69 case AARCH64_INSN_HINT_YIELD: 70 case AARCH64_INSN_HINT_WFE: 71 case AARCH64_INSN_HINT_WFI: 72 case AARCH64_INSN_HINT_SEV: 73 case AARCH64_INSN_HINT_SEVL: 74 return false; 75 default: 76 return true; 77 } 78} 79 80bool aarch64_insn_is_branch_imm(u32 insn) 81{ 82 return (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn) || 83 aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn) || 84 aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) || 85 aarch64_insn_is_bcond(insn)); 86} 87 88static DEFINE_RAW_SPINLOCK(patch_lock); 89 90static void __kprobes *patch_map(void *addr, int fixmap) 91{ 92 unsigned long uintaddr = (uintptr_t) addr; 93 bool module = !core_kernel_text(uintaddr); 94 struct page *page; 95 96 if (module && IS_ENABLED(CONFIG_DEBUG_SET_MODULE_RONX)) 97 page = vmalloc_to_page(addr); 98 else if (!module && IS_ENABLED(CONFIG_DEBUG_RODATA)) 99 page = virt_to_page(addr); 100 else 101 return addr; 102 103 BUG_ON(!page); 104 return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + 105 (uintaddr & ~PAGE_MASK)); 106} 107 108static void __kprobes patch_unmap(int fixmap) 109{ 110 clear_fixmap(fixmap); 111} 112/* 113 * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always 114 * little-endian. 115 */ 116int __kprobes aarch64_insn_read(void *addr, u32 *insnp) 117{ 118 int ret; 119 u32 val; 120 121 ret = probe_kernel_read(&val, addr, AARCH64_INSN_SIZE); 122 if (!ret) 123 *insnp = le32_to_cpu(val); 124 125 return ret; 126} 127 128static int __kprobes __aarch64_insn_write(void *addr, u32 insn) 129{ 130 void *waddr = addr; 131 unsigned long flags = 0; 132 int ret; 133 134 raw_spin_lock_irqsave(&patch_lock, flags); 135 waddr = patch_map(addr, FIX_TEXT_POKE0); 136 137 ret = probe_kernel_write(waddr, &insn, AARCH64_INSN_SIZE); 138 139 patch_unmap(FIX_TEXT_POKE0); 140 raw_spin_unlock_irqrestore(&patch_lock, flags); 141 142 return ret; 143} 144 145int __kprobes aarch64_insn_write(void *addr, u32 insn) 146{ 147 insn = cpu_to_le32(insn); 148 return __aarch64_insn_write(addr, insn); 149} 150 151static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn) 152{ 153 if (aarch64_get_insn_class(insn) != AARCH64_INSN_CLS_BR_SYS) 154 return false; 155 156 return aarch64_insn_is_b(insn) || 157 aarch64_insn_is_bl(insn) || 158 aarch64_insn_is_svc(insn) || 159 aarch64_insn_is_hvc(insn) || 160 aarch64_insn_is_smc(insn) || 161 aarch64_insn_is_brk(insn) || 162 aarch64_insn_is_nop(insn); 163} 164 165/* 166 * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a 167 * Section B2.6.5 "Concurrent modification and execution of instructions": 168 * Concurrent modification and execution of instructions can lead to the 169 * resulting instruction performing any behavior that can be achieved by 170 * executing any sequence of instructions that can be executed from the 171 * same Exception level, except where the instruction before modification 172 * and the instruction after modification is a B, BL, NOP, BKPT, SVC, HVC, 173 * or SMC instruction. 174 */ 175bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) 176{ 177 return __aarch64_insn_hotpatch_safe(old_insn) && 178 __aarch64_insn_hotpatch_safe(new_insn); 179} 180 181int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn) 182{ 183 u32 *tp = addr; 184 int ret; 185 186 /* A64 instructions must be word aligned */ 187 if ((uintptr_t)tp & 0x3) 188 return -EINVAL; 189 190 ret = aarch64_insn_write(tp, insn); 191 if (ret == 0) 192 flush_icache_range((uintptr_t)tp, 193 (uintptr_t)tp + AARCH64_INSN_SIZE); 194 195 return ret; 196} 197 198struct aarch64_insn_patch { 199 void **text_addrs; 200 u32 *new_insns; 201 int insn_cnt; 202 atomic_t cpu_count; 203}; 204 205static int __kprobes aarch64_insn_patch_text_cb(void *arg) 206{ 207 int i, ret = 0; 208 struct aarch64_insn_patch *pp = arg; 209 210 /* The first CPU becomes master */ 211 if (atomic_inc_return(&pp->cpu_count) == 1) { 212 for (i = 0; ret == 0 && i < pp->insn_cnt; i++) 213 ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i], 214 pp->new_insns[i]); 215 /* 216 * aarch64_insn_patch_text_nosync() calls flush_icache_range(), 217 * which ends with "dsb; isb" pair guaranteeing global 218 * visibility. 219 */ 220 /* Notify other processors with an additional increment. */ 221 atomic_inc(&pp->cpu_count); 222 } else { 223 while (atomic_read(&pp->cpu_count) <= num_online_cpus()) 224 cpu_relax(); 225 isb(); 226 } 227 228 return ret; 229} 230 231int __kprobes aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt) 232{ 233 struct aarch64_insn_patch patch = { 234 .text_addrs = addrs, 235 .new_insns = insns, 236 .insn_cnt = cnt, 237 .cpu_count = ATOMIC_INIT(0), 238 }; 239 240 if (cnt <= 0) 241 return -EINVAL; 242 243 return stop_machine(aarch64_insn_patch_text_cb, &patch, 244 cpu_online_mask); 245} 246 247int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt) 248{ 249 int ret; 250 u32 insn; 251 252 /* Unsafe to patch multiple instructions without synchronizaiton */ 253 if (cnt == 1) { 254 ret = aarch64_insn_read(addrs[0], &insn); 255 if (ret) 256 return ret; 257 258 if (aarch64_insn_hotpatch_safe(insn, insns[0])) { 259 /* 260 * ARMv8 architecture doesn't guarantee all CPUs see 261 * the new instruction after returning from function 262 * aarch64_insn_patch_text_nosync(). So send IPIs to 263 * all other CPUs to achieve instruction 264 * synchronization. 265 */ 266 ret = aarch64_insn_patch_text_nosync(addrs[0], insns[0]); 267 kick_all_cpus_sync(); 268 return ret; 269 } 270 } 271 272 return aarch64_insn_patch_text_sync(addrs, insns, cnt); 273} 274 275static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type, 276 u32 *maskp, int *shiftp) 277{ 278 u32 mask; 279 int shift; 280 281 switch (type) { 282 case AARCH64_INSN_IMM_26: 283 mask = BIT(26) - 1; 284 shift = 0; 285 break; 286 case AARCH64_INSN_IMM_19: 287 mask = BIT(19) - 1; 288 shift = 5; 289 break; 290 case AARCH64_INSN_IMM_16: 291 mask = BIT(16) - 1; 292 shift = 5; 293 break; 294 case AARCH64_INSN_IMM_14: 295 mask = BIT(14) - 1; 296 shift = 5; 297 break; 298 case AARCH64_INSN_IMM_12: 299 mask = BIT(12) - 1; 300 shift = 10; 301 break; 302 case AARCH64_INSN_IMM_9: 303 mask = BIT(9) - 1; 304 shift = 12; 305 break; 306 case AARCH64_INSN_IMM_7: 307 mask = BIT(7) - 1; 308 shift = 15; 309 break; 310 case AARCH64_INSN_IMM_6: 311 case AARCH64_INSN_IMM_S: 312 mask = BIT(6) - 1; 313 shift = 10; 314 break; 315 case AARCH64_INSN_IMM_R: 316 mask = BIT(6) - 1; 317 shift = 16; 318 break; 319 default: 320 return -EINVAL; 321 } 322 323 *maskp = mask; 324 *shiftp = shift; 325 326 return 0; 327} 328 329#define ADR_IMM_HILOSPLIT 2 330#define ADR_IMM_SIZE SZ_2M 331#define ADR_IMM_LOMASK ((1 << ADR_IMM_HILOSPLIT) - 1) 332#define ADR_IMM_HIMASK ((ADR_IMM_SIZE >> ADR_IMM_HILOSPLIT) - 1) 333#define ADR_IMM_LOSHIFT 29 334#define ADR_IMM_HISHIFT 5 335 336u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn) 337{ 338 u32 immlo, immhi, mask; 339 int shift; 340 341 switch (type) { 342 case AARCH64_INSN_IMM_ADR: 343 shift = 0; 344 immlo = (insn >> ADR_IMM_LOSHIFT) & ADR_IMM_LOMASK; 345 immhi = (insn >> ADR_IMM_HISHIFT) & ADR_IMM_HIMASK; 346 insn = (immhi << ADR_IMM_HILOSPLIT) | immlo; 347 mask = ADR_IMM_SIZE - 1; 348 break; 349 default: 350 if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) { 351 pr_err("aarch64_insn_decode_immediate: unknown immediate encoding %d\n", 352 type); 353 return 0; 354 } 355 } 356 357 return (insn >> shift) & mask; 358} 359 360u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, 361 u32 insn, u64 imm) 362{ 363 u32 immlo, immhi, mask; 364 int shift; 365 366 switch (type) { 367 case AARCH64_INSN_IMM_ADR: 368 shift = 0; 369 immlo = (imm & ADR_IMM_LOMASK) << ADR_IMM_LOSHIFT; 370 imm >>= ADR_IMM_HILOSPLIT; 371 immhi = (imm & ADR_IMM_HIMASK) << ADR_IMM_HISHIFT; 372 imm = immlo | immhi; 373 mask = ((ADR_IMM_LOMASK << ADR_IMM_LOSHIFT) | 374 (ADR_IMM_HIMASK << ADR_IMM_HISHIFT)); 375 break; 376 default: 377 if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) { 378 pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n", 379 type); 380 return 0; 381 } 382 } 383 384 /* Update the immediate field. */ 385 insn &= ~(mask << shift); 386 insn |= (imm & mask) << shift; 387 388 return insn; 389} 390 391static u32 aarch64_insn_encode_register(enum aarch64_insn_register_type type, 392 u32 insn, 393 enum aarch64_insn_register reg) 394{ 395 int shift; 396 397 if (reg < AARCH64_INSN_REG_0 || reg > AARCH64_INSN_REG_SP) { 398 pr_err("%s: unknown register encoding %d\n", __func__, reg); 399 return 0; 400 } 401 402 switch (type) { 403 case AARCH64_INSN_REGTYPE_RT: 404 case AARCH64_INSN_REGTYPE_RD: 405 shift = 0; 406 break; 407 case AARCH64_INSN_REGTYPE_RN: 408 shift = 5; 409 break; 410 case AARCH64_INSN_REGTYPE_RT2: 411 case AARCH64_INSN_REGTYPE_RA: 412 shift = 10; 413 break; 414 case AARCH64_INSN_REGTYPE_RM: 415 shift = 16; 416 break; 417 default: 418 pr_err("%s: unknown register type encoding %d\n", __func__, 419 type); 420 return 0; 421 } 422 423 insn &= ~(GENMASK(4, 0) << shift); 424 insn |= reg << shift; 425 426 return insn; 427} 428 429static u32 aarch64_insn_encode_ldst_size(enum aarch64_insn_size_type type, 430 u32 insn) 431{ 432 u32 size; 433 434 switch (type) { 435 case AARCH64_INSN_SIZE_8: 436 size = 0; 437 break; 438 case AARCH64_INSN_SIZE_16: 439 size = 1; 440 break; 441 case AARCH64_INSN_SIZE_32: 442 size = 2; 443 break; 444 case AARCH64_INSN_SIZE_64: 445 size = 3; 446 break; 447 default: 448 pr_err("%s: unknown size encoding %d\n", __func__, type); 449 return 0; 450 } 451 452 insn &= ~GENMASK(31, 30); 453 insn |= size << 30; 454 455 return insn; 456} 457 458static inline long branch_imm_common(unsigned long pc, unsigned long addr, 459 long range) 460{ 461 long offset; 462 463 /* 464 * PC: A 64-bit Program Counter holding the address of the current 465 * instruction. A64 instructions must be word-aligned. 466 */ 467 BUG_ON((pc & 0x3) || (addr & 0x3)); 468 469 offset = ((long)addr - (long)pc); 470 BUG_ON(offset < -range || offset >= range); 471 472 return offset; 473} 474 475u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr, 476 enum aarch64_insn_branch_type type) 477{ 478 u32 insn; 479 long offset; 480 481 /* 482 * B/BL support [-128M, 128M) offset 483 * ARM64 virtual address arrangement guarantees all kernel and module 484 * texts are within +/-128M. 485 */ 486 offset = branch_imm_common(pc, addr, SZ_128M); 487 488 switch (type) { 489 case AARCH64_INSN_BRANCH_LINK: 490 insn = aarch64_insn_get_bl_value(); 491 break; 492 case AARCH64_INSN_BRANCH_NOLINK: 493 insn = aarch64_insn_get_b_value(); 494 break; 495 default: 496 BUG_ON(1); 497 return AARCH64_BREAK_FAULT; 498 } 499 500 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn, 501 offset >> 2); 502} 503 504u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr, 505 enum aarch64_insn_register reg, 506 enum aarch64_insn_variant variant, 507 enum aarch64_insn_branch_type type) 508{ 509 u32 insn; 510 long offset; 511 512 offset = branch_imm_common(pc, addr, SZ_1M); 513 514 switch (type) { 515 case AARCH64_INSN_BRANCH_COMP_ZERO: 516 insn = aarch64_insn_get_cbz_value(); 517 break; 518 case AARCH64_INSN_BRANCH_COMP_NONZERO: 519 insn = aarch64_insn_get_cbnz_value(); 520 break; 521 default: 522 BUG_ON(1); 523 return AARCH64_BREAK_FAULT; 524 } 525 526 switch (variant) { 527 case AARCH64_INSN_VARIANT_32BIT: 528 break; 529 case AARCH64_INSN_VARIANT_64BIT: 530 insn |= AARCH64_INSN_SF_BIT; 531 break; 532 default: 533 BUG_ON(1); 534 return AARCH64_BREAK_FAULT; 535 } 536 537 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg); 538 539 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn, 540 offset >> 2); 541} 542 543u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr, 544 enum aarch64_insn_condition cond) 545{ 546 u32 insn; 547 long offset; 548 549 offset = branch_imm_common(pc, addr, SZ_1M); 550 551 insn = aarch64_insn_get_bcond_value(); 552 553 BUG_ON(cond < AARCH64_INSN_COND_EQ || cond > AARCH64_INSN_COND_AL); 554 insn |= cond; 555 556 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn, 557 offset >> 2); 558} 559 560u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_op op) 561{ 562 return aarch64_insn_get_hint_value() | op; 563} 564 565u32 __kprobes aarch64_insn_gen_nop(void) 566{ 567 return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP); 568} 569 570u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg, 571 enum aarch64_insn_branch_type type) 572{ 573 u32 insn; 574 575 switch (type) { 576 case AARCH64_INSN_BRANCH_NOLINK: 577 insn = aarch64_insn_get_br_value(); 578 break; 579 case AARCH64_INSN_BRANCH_LINK: 580 insn = aarch64_insn_get_blr_value(); 581 break; 582 case AARCH64_INSN_BRANCH_RETURN: 583 insn = aarch64_insn_get_ret_value(); 584 break; 585 default: 586 BUG_ON(1); 587 return AARCH64_BREAK_FAULT; 588 } 589 590 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, reg); 591} 592 593u32 aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg, 594 enum aarch64_insn_register base, 595 enum aarch64_insn_register offset, 596 enum aarch64_insn_size_type size, 597 enum aarch64_insn_ldst_type type) 598{ 599 u32 insn; 600 601 switch (type) { 602 case AARCH64_INSN_LDST_LOAD_REG_OFFSET: 603 insn = aarch64_insn_get_ldr_reg_value(); 604 break; 605 case AARCH64_INSN_LDST_STORE_REG_OFFSET: 606 insn = aarch64_insn_get_str_reg_value(); 607 break; 608 default: 609 BUG_ON(1); 610 return AARCH64_BREAK_FAULT; 611 } 612 613 insn = aarch64_insn_encode_ldst_size(size, insn); 614 615 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg); 616 617 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, 618 base); 619 620 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, 621 offset); 622} 623 624u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1, 625 enum aarch64_insn_register reg2, 626 enum aarch64_insn_register base, 627 int offset, 628 enum aarch64_insn_variant variant, 629 enum aarch64_insn_ldst_type type) 630{ 631 u32 insn; 632 int shift; 633 634 switch (type) { 635 case AARCH64_INSN_LDST_LOAD_PAIR_PRE_INDEX: 636 insn = aarch64_insn_get_ldp_pre_value(); 637 break; 638 case AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX: 639 insn = aarch64_insn_get_stp_pre_value(); 640 break; 641 case AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX: 642 insn = aarch64_insn_get_ldp_post_value(); 643 break; 644 case AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX: 645 insn = aarch64_insn_get_stp_post_value(); 646 break; 647 default: 648 BUG_ON(1); 649 return AARCH64_BREAK_FAULT; 650 } 651 652 switch (variant) { 653 case AARCH64_INSN_VARIANT_32BIT: 654 /* offset must be multiples of 4 in the range [-256, 252] */ 655 BUG_ON(offset & 0x3); 656 BUG_ON(offset < -256 || offset > 252); 657 shift = 2; 658 break; 659 case AARCH64_INSN_VARIANT_64BIT: 660 /* offset must be multiples of 8 in the range [-512, 504] */ 661 BUG_ON(offset & 0x7); 662 BUG_ON(offset < -512 || offset > 504); 663 shift = 3; 664 insn |= AARCH64_INSN_SF_BIT; 665 break; 666 default: 667 BUG_ON(1); 668 return AARCH64_BREAK_FAULT; 669 } 670 671 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, 672 reg1); 673 674 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn, 675 reg2); 676 677 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, 678 base); 679 680 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_7, insn, 681 offset >> shift); 682} 683 684u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst, 685 enum aarch64_insn_register src, 686 int imm, enum aarch64_insn_variant variant, 687 enum aarch64_insn_adsb_type type) 688{ 689 u32 insn; 690 691 switch (type) { 692 case AARCH64_INSN_ADSB_ADD: 693 insn = aarch64_insn_get_add_imm_value(); 694 break; 695 case AARCH64_INSN_ADSB_SUB: 696 insn = aarch64_insn_get_sub_imm_value(); 697 break; 698 case AARCH64_INSN_ADSB_ADD_SETFLAGS: 699 insn = aarch64_insn_get_adds_imm_value(); 700 break; 701 case AARCH64_INSN_ADSB_SUB_SETFLAGS: 702 insn = aarch64_insn_get_subs_imm_value(); 703 break; 704 default: 705 BUG_ON(1); 706 return AARCH64_BREAK_FAULT; 707 } 708 709 switch (variant) { 710 case AARCH64_INSN_VARIANT_32BIT: 711 break; 712 case AARCH64_INSN_VARIANT_64BIT: 713 insn |= AARCH64_INSN_SF_BIT; 714 break; 715 default: 716 BUG_ON(1); 717 return AARCH64_BREAK_FAULT; 718 } 719 720 BUG_ON(imm & ~(SZ_4K - 1)); 721 722 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 723 724 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); 725 726 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm); 727} 728 729u32 aarch64_insn_gen_bitfield(enum aarch64_insn_register dst, 730 enum aarch64_insn_register src, 731 int immr, int imms, 732 enum aarch64_insn_variant variant, 733 enum aarch64_insn_bitfield_type type) 734{ 735 u32 insn; 736 u32 mask; 737 738 switch (type) { 739 case AARCH64_INSN_BITFIELD_MOVE: 740 insn = aarch64_insn_get_bfm_value(); 741 break; 742 case AARCH64_INSN_BITFIELD_MOVE_UNSIGNED: 743 insn = aarch64_insn_get_ubfm_value(); 744 break; 745 case AARCH64_INSN_BITFIELD_MOVE_SIGNED: 746 insn = aarch64_insn_get_sbfm_value(); 747 break; 748 default: 749 BUG_ON(1); 750 return AARCH64_BREAK_FAULT; 751 } 752 753 switch (variant) { 754 case AARCH64_INSN_VARIANT_32BIT: 755 mask = GENMASK(4, 0); 756 break; 757 case AARCH64_INSN_VARIANT_64BIT: 758 insn |= AARCH64_INSN_SF_BIT | AARCH64_INSN_N_BIT; 759 mask = GENMASK(5, 0); 760 break; 761 default: 762 BUG_ON(1); 763 return AARCH64_BREAK_FAULT; 764 } 765 766 BUG_ON(immr & ~mask); 767 BUG_ON(imms & ~mask); 768 769 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 770 771 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); 772 773 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr); 774 775 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms); 776} 777 778u32 aarch64_insn_gen_movewide(enum aarch64_insn_register dst, 779 int imm, int shift, 780 enum aarch64_insn_variant variant, 781 enum aarch64_insn_movewide_type type) 782{ 783 u32 insn; 784 785 switch (type) { 786 case AARCH64_INSN_MOVEWIDE_ZERO: 787 insn = aarch64_insn_get_movz_value(); 788 break; 789 case AARCH64_INSN_MOVEWIDE_KEEP: 790 insn = aarch64_insn_get_movk_value(); 791 break; 792 case AARCH64_INSN_MOVEWIDE_INVERSE: 793 insn = aarch64_insn_get_movn_value(); 794 break; 795 default: 796 BUG_ON(1); 797 return AARCH64_BREAK_FAULT; 798 } 799 800 BUG_ON(imm & ~(SZ_64K - 1)); 801 802 switch (variant) { 803 case AARCH64_INSN_VARIANT_32BIT: 804 BUG_ON(shift != 0 && shift != 16); 805 break; 806 case AARCH64_INSN_VARIANT_64BIT: 807 insn |= AARCH64_INSN_SF_BIT; 808 BUG_ON(shift != 0 && shift != 16 && shift != 32 && 809 shift != 48); 810 break; 811 default: 812 BUG_ON(1); 813 return AARCH64_BREAK_FAULT; 814 } 815 816 insn |= (shift >> 4) << 21; 817 818 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 819 820 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm); 821} 822 823u32 aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst, 824 enum aarch64_insn_register src, 825 enum aarch64_insn_register reg, 826 int shift, 827 enum aarch64_insn_variant variant, 828 enum aarch64_insn_adsb_type type) 829{ 830 u32 insn; 831 832 switch (type) { 833 case AARCH64_INSN_ADSB_ADD: 834 insn = aarch64_insn_get_add_value(); 835 break; 836 case AARCH64_INSN_ADSB_SUB: 837 insn = aarch64_insn_get_sub_value(); 838 break; 839 case AARCH64_INSN_ADSB_ADD_SETFLAGS: 840 insn = aarch64_insn_get_adds_value(); 841 break; 842 case AARCH64_INSN_ADSB_SUB_SETFLAGS: 843 insn = aarch64_insn_get_subs_value(); 844 break; 845 default: 846 BUG_ON(1); 847 return AARCH64_BREAK_FAULT; 848 } 849 850 switch (variant) { 851 case AARCH64_INSN_VARIANT_32BIT: 852 BUG_ON(shift & ~(SZ_32 - 1)); 853 break; 854 case AARCH64_INSN_VARIANT_64BIT: 855 insn |= AARCH64_INSN_SF_BIT; 856 BUG_ON(shift & ~(SZ_64 - 1)); 857 break; 858 default: 859 BUG_ON(1); 860 return AARCH64_BREAK_FAULT; 861 } 862 863 864 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 865 866 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); 867 868 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg); 869 870 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift); 871} 872 873u32 aarch64_insn_gen_data1(enum aarch64_insn_register dst, 874 enum aarch64_insn_register src, 875 enum aarch64_insn_variant variant, 876 enum aarch64_insn_data1_type type) 877{ 878 u32 insn; 879 880 switch (type) { 881 case AARCH64_INSN_DATA1_REVERSE_16: 882 insn = aarch64_insn_get_rev16_value(); 883 break; 884 case AARCH64_INSN_DATA1_REVERSE_32: 885 insn = aarch64_insn_get_rev32_value(); 886 break; 887 case AARCH64_INSN_DATA1_REVERSE_64: 888 BUG_ON(variant != AARCH64_INSN_VARIANT_64BIT); 889 insn = aarch64_insn_get_rev64_value(); 890 break; 891 default: 892 BUG_ON(1); 893 return AARCH64_BREAK_FAULT; 894 } 895 896 switch (variant) { 897 case AARCH64_INSN_VARIANT_32BIT: 898 break; 899 case AARCH64_INSN_VARIANT_64BIT: 900 insn |= AARCH64_INSN_SF_BIT; 901 break; 902 default: 903 BUG_ON(1); 904 return AARCH64_BREAK_FAULT; 905 } 906 907 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 908 909 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); 910} 911 912u32 aarch64_insn_gen_data2(enum aarch64_insn_register dst, 913 enum aarch64_insn_register src, 914 enum aarch64_insn_register reg, 915 enum aarch64_insn_variant variant, 916 enum aarch64_insn_data2_type type) 917{ 918 u32 insn; 919 920 switch (type) { 921 case AARCH64_INSN_DATA2_UDIV: 922 insn = aarch64_insn_get_udiv_value(); 923 break; 924 case AARCH64_INSN_DATA2_SDIV: 925 insn = aarch64_insn_get_sdiv_value(); 926 break; 927 case AARCH64_INSN_DATA2_LSLV: 928 insn = aarch64_insn_get_lslv_value(); 929 break; 930 case AARCH64_INSN_DATA2_LSRV: 931 insn = aarch64_insn_get_lsrv_value(); 932 break; 933 case AARCH64_INSN_DATA2_ASRV: 934 insn = aarch64_insn_get_asrv_value(); 935 break; 936 case AARCH64_INSN_DATA2_RORV: 937 insn = aarch64_insn_get_rorv_value(); 938 break; 939 default: 940 BUG_ON(1); 941 return AARCH64_BREAK_FAULT; 942 } 943 944 switch (variant) { 945 case AARCH64_INSN_VARIANT_32BIT: 946 break; 947 case AARCH64_INSN_VARIANT_64BIT: 948 insn |= AARCH64_INSN_SF_BIT; 949 break; 950 default: 951 BUG_ON(1); 952 return AARCH64_BREAK_FAULT; 953 } 954 955 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 956 957 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); 958 959 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg); 960} 961 962u32 aarch64_insn_gen_data3(enum aarch64_insn_register dst, 963 enum aarch64_insn_register src, 964 enum aarch64_insn_register reg1, 965 enum aarch64_insn_register reg2, 966 enum aarch64_insn_variant variant, 967 enum aarch64_insn_data3_type type) 968{ 969 u32 insn; 970 971 switch (type) { 972 case AARCH64_INSN_DATA3_MADD: 973 insn = aarch64_insn_get_madd_value(); 974 break; 975 case AARCH64_INSN_DATA3_MSUB: 976 insn = aarch64_insn_get_msub_value(); 977 break; 978 default: 979 BUG_ON(1); 980 return AARCH64_BREAK_FAULT; 981 } 982 983 switch (variant) { 984 case AARCH64_INSN_VARIANT_32BIT: 985 break; 986 case AARCH64_INSN_VARIANT_64BIT: 987 insn |= AARCH64_INSN_SF_BIT; 988 break; 989 default: 990 BUG_ON(1); 991 return AARCH64_BREAK_FAULT; 992 } 993 994 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 995 996 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RA, insn, src); 997 998 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, 999 reg1); 1000 1001 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, 1002 reg2); 1003} 1004 1005u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst, 1006 enum aarch64_insn_register src, 1007 enum aarch64_insn_register reg, 1008 int shift, 1009 enum aarch64_insn_variant variant, 1010 enum aarch64_insn_logic_type type) 1011{ 1012 u32 insn; 1013 1014 switch (type) { 1015 case AARCH64_INSN_LOGIC_AND: 1016 insn = aarch64_insn_get_and_value(); 1017 break; 1018 case AARCH64_INSN_LOGIC_BIC: 1019 insn = aarch64_insn_get_bic_value(); 1020 break; 1021 case AARCH64_INSN_LOGIC_ORR: 1022 insn = aarch64_insn_get_orr_value(); 1023 break; 1024 case AARCH64_INSN_LOGIC_ORN: 1025 insn = aarch64_insn_get_orn_value(); 1026 break; 1027 case AARCH64_INSN_LOGIC_EOR: 1028 insn = aarch64_insn_get_eor_value(); 1029 break; 1030 case AARCH64_INSN_LOGIC_EON: 1031 insn = aarch64_insn_get_eon_value(); 1032 break; 1033 case AARCH64_INSN_LOGIC_AND_SETFLAGS: 1034 insn = aarch64_insn_get_ands_value(); 1035 break; 1036 case AARCH64_INSN_LOGIC_BIC_SETFLAGS: 1037 insn = aarch64_insn_get_bics_value(); 1038 break; 1039 default: 1040 BUG_ON(1); 1041 return AARCH64_BREAK_FAULT; 1042 } 1043 1044 switch (variant) { 1045 case AARCH64_INSN_VARIANT_32BIT: 1046 BUG_ON(shift & ~(SZ_32 - 1)); 1047 break; 1048 case AARCH64_INSN_VARIANT_64BIT: 1049 insn |= AARCH64_INSN_SF_BIT; 1050 BUG_ON(shift & ~(SZ_64 - 1)); 1051 break; 1052 default: 1053 BUG_ON(1); 1054 return AARCH64_BREAK_FAULT; 1055 } 1056 1057 1058 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 1059 1060 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); 1061 1062 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg); 1063 1064 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift); 1065} 1066 1067/* 1068 * Decode the imm field of a branch, and return the byte offset as a 1069 * signed value (so it can be used when computing a new branch 1070 * target). 1071 */ 1072s32 aarch64_get_branch_offset(u32 insn) 1073{ 1074 s32 imm; 1075 1076 if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) { 1077 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_26, insn); 1078 return (imm << 6) >> 4; 1079 } 1080 1081 if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) || 1082 aarch64_insn_is_bcond(insn)) { 1083 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_19, insn); 1084 return (imm << 13) >> 11; 1085 } 1086 1087 if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn)) { 1088 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_14, insn); 1089 return (imm << 18) >> 16; 1090 } 1091 1092 /* Unhandled instruction */ 1093 BUG(); 1094} 1095 1096/* 1097 * Encode the displacement of a branch in the imm field and return the 1098 * updated instruction. 1099 */ 1100u32 aarch64_set_branch_offset(u32 insn, s32 offset) 1101{ 1102 if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) 1103 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn, 1104 offset >> 2); 1105 1106 if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) || 1107 aarch64_insn_is_bcond(insn)) 1108 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn, 1109 offset >> 2); 1110 1111 if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn)) 1112 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_14, insn, 1113 offset >> 2); 1114 1115 /* Unhandled instruction */ 1116 BUG(); 1117} 1118 1119bool aarch32_insn_is_wide(u32 insn) 1120{ 1121 return insn >= 0xe800; 1122} 1123 1124/* 1125 * Macros/defines for extracting register numbers from instruction. 1126 */ 1127u32 aarch32_insn_extract_reg_num(u32 insn, int offset) 1128{ 1129 return (insn & (0xf << offset)) >> offset; 1130} 1131 1132#define OPC2_MASK 0x7 1133#define OPC2_OFFSET 5 1134u32 aarch32_insn_mcr_extract_opc2(u32 insn) 1135{ 1136 return (insn & (OPC2_MASK << OPC2_OFFSET)) >> OPC2_OFFSET; 1137} 1138 1139#define CRM_MASK 0xf 1140u32 aarch32_insn_mcr_extract_crm(u32 insn) 1141{ 1142 return insn & CRM_MASK; 1143} 1144