154 apply_imm64 (struct module *mod, struct insn *insn, uint64_t val)  in apply_imm64()  argument
158 		       mod->name, slot(insn));  in apply_imm64()
166 apply_imm60 (struct module *mod, struct insn *insn, uint64_t val)  in apply_imm60()  argument
170 		       mod->name, slot(insn));  in apply_imm60()
175 			mod->name, (long) val);  in apply_imm60()
183 apply_imm22 (struct module *mod, struct insn *insn, uint64_t val)  in apply_imm22()  argument
187 			mod->name, (long)val);  in apply_imm22()
198 apply_imm21b (struct module *mod, struct insn *insn, uint64_t val)  in apply_imm21b()  argument
202 			mod->name, (long)val);  in apply_imm21b()
233 patch_plt (struct module *mod, struct plt_entry *plt, long target_ip, unsigned long target_gp)  in patch_plt()  argument
235 	if (apply_imm64(mod, (struct insn *) (plt->bundle[0] + 2), target_gp)  in patch_plt()
236 	    && apply_imm60(mod, (struct insn *) (plt->bundle[1] + 2),  in patch_plt()
283 patch_plt (struct module *mod, struct plt_entry *plt, long target_ip, unsigned long target_gp)  in patch_plt()  argument
285 	if (apply_imm64(mod, (struct insn *) (plt->bundle[0] + 2), target_ip)  in patch_plt()
286 	    && apply_imm64(mod, (struct insn *) (plt->bundle[1] + 2), target_gp))  in patch_plt()
308 module_arch_freeing_init (struct module *mod)  in module_arch_freeing_init()  argument
310 	if (mod->arch.init_unw_table) {  in module_arch_freeing_init()
311 		unw_remove_unwind_table(mod->arch.init_unw_table);  in module_arch_freeing_init()
312 		mod->arch.init_unw_table = NULL;  in module_arch_freeing_init()
422 			   struct module *mod)  in module_frob_arch_sections()  argument
433 			mod->arch.core_plt = s;  in module_frob_arch_sections()
435 			mod->arch.init_plt = s;  in module_frob_arch_sections()
437 			mod->arch.got = s;  in module_frob_arch_sections()
439 			mod->arch.opd = s;  in module_frob_arch_sections()
441 			mod->arch.unwind = s;  in module_frob_arch_sections()
445 			mod->arch.paravirt_bundles = s;  in module_frob_arch_sections()
448 			mod->arch.paravirt_insts = s;  in module_frob_arch_sections()
451 	if (!mod->arch.core_plt || !mod->arch.init_plt || !mod->arch.got || !mod->arch.opd) {  in module_frob_arch_sections()
452 		printk(KERN_ERR "%s: sections missing\n", mod->name);  in module_frob_arch_sections()
472 	mod->arch.core_plt->sh_type = SHT_NOBITS;  in module_frob_arch_sections()
473 	mod->arch.core_plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;  in module_frob_arch_sections()
474 	mod->arch.core_plt->sh_addralign = 16;  in module_frob_arch_sections()
475 	mod->arch.core_plt->sh_size = core_plts * sizeof(struct plt_entry);  in module_frob_arch_sections()
476 	mod->arch.init_plt->sh_type = SHT_NOBITS;  in module_frob_arch_sections()
477 	mod->arch.init_plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;  in module_frob_arch_sections()
478 	mod->arch.init_plt->sh_addralign = 16;  in module_frob_arch_sections()
479 	mod->arch.init_plt->sh_size = init_plts * sizeof(struct plt_entry);  in module_frob_arch_sections()
480 	mod->arch.got->sh_type = SHT_NOBITS;  in module_frob_arch_sections()
481 	mod->arch.got->sh_flags = ARCH_SHF_SMALL | SHF_ALLOC;  in module_frob_arch_sections()
482 	mod->arch.got->sh_addralign = 8;  in module_frob_arch_sections()
483 	mod->arch.got->sh_size = gots * sizeof(struct got_entry);  in module_frob_arch_sections()
484 	mod->arch.opd->sh_type = SHT_NOBITS;  in module_frob_arch_sections()
485 	mod->arch.opd->sh_flags = SHF_ALLOC;  in module_frob_arch_sections()
486 	mod->arch.opd->sh_addralign = 8;  in module_frob_arch_sections()
487 	mod->arch.opd->sh_size = fdescs * sizeof(struct fdesc);  in module_frob_arch_sections()
489 	       __func__, mod->arch.core_plt->sh_size, mod->arch.init_plt->sh_size,  in module_frob_arch_sections()
490 	       mod->arch.got->sh_size, mod->arch.opd->sh_size);  in module_frob_arch_sections()
495 in_init (const struct module *mod, uint64_t addr)  in in_init()  argument
497 	return addr - (uint64_t) mod->module_init < mod->init_size;  in in_init()
501 in_core (const struct module *mod, uint64_t addr)  in in_core()  argument
503 	return addr - (uint64_t) mod->module_core < mod->core_size;  in in_core()
507 is_internal (const struct module *mod, uint64_t value)  in is_internal()  argument
509 	return in_init(mod, value) || in_core(mod, value);  in is_internal()
516 get_ltoff (struct module *mod, uint64_t value, int *okp)  in get_ltoff()  argument
523 	got = (void *) mod->arch.got->sh_addr;  in get_ltoff()
524 	for (e = got; e < got + mod->arch.next_got_entry; ++e)  in get_ltoff()
529 	BUG_ON(e >= (struct got_entry *) (mod->arch.got->sh_addr + mod->arch.got->sh_size));  in get_ltoff()
532 	++mod->arch.next_got_entry;  in get_ltoff()
534 	return (uint64_t) e - mod->arch.gp;  in get_ltoff()
538 gp_addressable (struct module *mod, uint64_t value)  in gp_addressable()  argument
540 	return value - mod->arch.gp + MAX_LTOFF/2 < MAX_LTOFF;  in gp_addressable()
545 get_plt (struct module *mod, const struct insn *insn, uint64_t value, int *okp)  in get_plt()  argument
553 	if (in_init(mod, (uint64_t) insn)) {  in get_plt()
554 		plt = (void *) mod->arch.init_plt->sh_addr;  in get_plt()
555 		plt_end = (void *) plt + mod->arch.init_plt->sh_size;  in get_plt()
557 		plt = (void *) mod->arch.core_plt->sh_addr;  in get_plt()
558 		plt_end = (void *) plt + mod->arch.core_plt->sh_size;  in get_plt()
573 	if (!patch_plt(mod, plt, target_ip, target_gp)) {  in get_plt()
591 get_fdesc (struct module *mod, uint64_t value, int *okp)  in get_fdesc()  argument
593 	struct fdesc *fdesc = (void *) mod->arch.opd->sh_addr;  in get_fdesc()
599 		printk(KERN_ERR "%s: fdesc for zero requested!\n", mod->name);  in get_fdesc()
603 	if (!is_internal(mod, value))  in get_fdesc()
614 		if ((uint64_t) ++fdesc >= mod->arch.opd->sh_addr + mod->arch.opd->sh_size)  in get_fdesc()
620 	fdesc->gp = mod->arch.gp;  in get_fdesc()
625 do_reloc (struct module *mod, uint8_t r_type, Elf64_Sym *sym, uint64_t addend,  in do_reloc()  argument
640 	      case RV_GPREL:	  val -= mod->arch.gp; break;  in do_reloc()
641 	      case RV_LTREL:	  val = get_ltoff(mod, val, &ok); break;  in do_reloc()
642 	      case RV_PLTREL:	  val = get_plt(mod, location, val, &ok); break;  in do_reloc()
643 	      case RV_FPTR:	  val = get_fdesc(mod, val, &ok); break;  in do_reloc()
645 	      case RV_LTREL_FPTR: val = get_ltoff(mod, get_fdesc(mod, val, &ok), &ok); break;  in do_reloc()
650 			if ((in_init(mod, val) && in_core(mod, (uint64_t)location)) ||  in do_reloc()
651 			    (in_core(mod, val) && in_init(mod, (uint64_t)location))) {  in do_reloc()
658 					val = get_fdesc(mod, val, &ok);  in do_reloc()
659 					val = get_plt(mod, location, val, &ok);  in do_reloc()
661 			} else if (!is_internal(mod, val))  in do_reloc()
662 				val = get_plt(mod, location, val, &ok);  in do_reloc()
686 		val -= (uint64_t) (in_init(mod, val) ? mod->module_init : mod->module_core);  in do_reloc()
696 			if (!is_internal(mod, val)) {  in do_reloc()
711 			val = get_fdesc(mod, get_plt(mod, location, val, &ok), &ok);  in do_reloc()
723 			if (gp_addressable(mod, val))  in do_reloc()
724 				val -= mod->arch.gp;  in do_reloc()
726 				val = get_ltoff(mod, val, &ok);  in do_reloc()
731 			if (gp_addressable(mod, val)) {  in do_reloc()
741 				       mod->name, reloc_name[r_type]);  in do_reloc()
744 				       mod->name, r_type);  in do_reloc()
756 		       mod->name, reloc_name[r_type] ? reloc_name[r_type] : "?");  in do_reloc()
760 		printk(KERN_ERR "%s: unknown reloc %x\n", mod->name, r_type);  in do_reloc()
771 	      case RF_INSN21B:	ok = apply_imm21b(mod, location, (int64_t) val / 16); break;  in do_reloc()
772 	      case RF_INSN22:	ok = apply_imm22(mod, location, val); break;  in do_reloc()
773 	      case RF_INSN64:	ok = apply_imm64(mod, location, val); break;  in do_reloc()
774 	      case RF_INSN60:	ok = apply_imm60(mod, location, (int64_t) val / 16); break;  in do_reloc()
783 		       mod->name, format, reloc_name[r_type] ? reloc_name[r_type] : "?");  in do_reloc()
788 		       mod->name, reloc_name[r_type] ? reloc_name[r_type] : "?", format);  in do_reloc()
796 		    unsigned int relsec, struct module *mod)  in apply_relocate_add()  argument
815 	if (!mod->arch.gp) {  in apply_relocate_add()
821 		if (mod->core_size > MAX_LTOFF)  in apply_relocate_add()
826 			gp = mod->core_size - MAX_LTOFF / 2;  in apply_relocate_add()
828 			gp = mod->core_size / 2;  in apply_relocate_add()
829 		gp = (uint64_t) mod->module_core + ((gp + 7) & -8);  in apply_relocate_add()
830 		mod->arch.gp = gp;  in apply_relocate_add()
835 		ret = do_reloc(mod, ELF64_R_TYPE(rela[i].r_info),  in apply_relocate_add()
853 register_unwind_table (struct module *mod)  in register_unwind_table()  argument
855 	struct unw_table_entry *start = (void *) mod->arch.unwind->sh_addr;  in register_unwind_table()
856 	struct unw_table_entry *end = start + mod->arch.unwind->sh_size / sizeof (*start);  in register_unwind_table()
862 		if (in_init(mod, e1->start_offset))  in register_unwind_table()
883 	if (in_init(mod, start->start_offset)) {  in register_unwind_table()
892 	       mod->name, mod->arch.gp, num_init, num_core);  in register_unwind_table()
898 		mod->arch.core_unw_table = unw_add_unwind_table(mod->name, 0, mod->arch.gp,  in register_unwind_table()
901 		       mod->arch.core_unw_table, core, core + num_core);  in register_unwind_table()
904 		mod->arch.init_unw_table = unw_add_unwind_table(mod->name, 0, mod->arch.gp,  in register_unwind_table()
907 		       mod->arch.init_unw_table, init, init + num_init);  in register_unwind_table()
912 module_finalize (const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *mod)  in module_finalize()  argument
914 	DEBUGP("%s: init: entry=%p\n", __func__, mod->init);  in module_finalize()
915 	if (mod->arch.unwind)  in module_finalize()
916 		register_unwind_table(mod);  in module_finalize()
918         if (mod->arch.paravirt_bundles) {  in module_finalize()
921                         mod->arch.paravirt_bundles->sh_addr;  in module_finalize()
924                         (mod->arch.paravirt_bundles->sh_addr +  in module_finalize()
925                          mod->arch.paravirt_bundles->sh_size);  in module_finalize()
929         if (mod->arch.paravirt_insts) {  in module_finalize()
932                         mod->arch.paravirt_insts->sh_addr;  in module_finalize()
935                         (mod->arch.paravirt_insts->sh_addr +  in module_finalize()
936                          mod->arch.paravirt_insts->sh_size);  in module_finalize()
945 module_arch_cleanup (struct module *mod)  in module_arch_cleanup()  argument
947 	if (mod->arch.init_unw_table)  in module_arch_cleanup()
948 		unw_remove_unwind_table(mod->arch.init_unw_table);  in module_arch_cleanup()
949 	if (mod->arch.core_unw_table)  in module_arch_cleanup()
950 		unw_remove_unwind_table(mod->arch.core_unw_table);  in module_arch_cleanup()