68 static int kexec_calculate_store_digests(struct kimage *image);
138 static int kimage_is_destination_range(struct kimage *image,
140 static struct page *kimage_alloc_page(struct kimage *image,
144 static int copy_user_segment_list(struct kimage *image,  in copy_user_segment_list()  argument
152 	image->nr_segments = nr_segments;  in copy_user_segment_list()
154 	ret = copy_from_user(image->segment, segments, segment_bytes);  in copy_user_segment_list()
161 static int sanity_check_segment_list(struct kimage *image)  in sanity_check_segment_list()  argument
164 	unsigned long nr_segments = image->nr_segments;  in sanity_check_segment_list()
183 		mstart = image->segment[i].mem;  in sanity_check_segment_list()
184 		mend   = mstart + image->segment[i].memsz;  in sanity_check_segment_list()
201 		mstart = image->segment[i].mem;  in sanity_check_segment_list()
202 		mend   = mstart + image->segment[i].memsz;  in sanity_check_segment_list()
205 			pstart = image->segment[j].mem;  in sanity_check_segment_list()
206 			pend   = pstart + image->segment[j].memsz;  in sanity_check_segment_list()
220 		if (image->segment[i].bufsz > image->segment[i].memsz)  in sanity_check_segment_list()
234 	if (image->type == KEXEC_TYPE_CRASH) {  in sanity_check_segment_list()
239 			mstart = image->segment[i].mem;  in sanity_check_segment_list()
240 			mend = mstart + image->segment[i].memsz - 1;  in sanity_check_segment_list()
253 	struct kimage *image;  in do_kimage_alloc_init()  local
256 	image = kzalloc(sizeof(*image), GFP_KERNEL);  in do_kimage_alloc_init()
257 	if (!image)  in do_kimage_alloc_init()
260 	image->head = 0;  in do_kimage_alloc_init()
261 	image->entry = &image->head;  in do_kimage_alloc_init()
262 	image->last_entry = &image->head;  in do_kimage_alloc_init()
263 	image->control_page = ~0; /* By default this does not apply */  in do_kimage_alloc_init()
264 	image->type = KEXEC_TYPE_DEFAULT;  in do_kimage_alloc_init()
267 	INIT_LIST_HEAD(&image->control_pages);  in do_kimage_alloc_init()
270 	INIT_LIST_HEAD(&image->dest_pages);  in do_kimage_alloc_init()
273 	INIT_LIST_HEAD(&image->unusable_pages);  in do_kimage_alloc_init()
275 	return image;  in do_kimage_alloc_init()
286 	struct kimage *image;  in kimage_alloc_init()  local
296 	image = do_kimage_alloc_init();  in kimage_alloc_init()
297 	if (!image)  in kimage_alloc_init()
300 	image->start = entry;  in kimage_alloc_init()
302 	ret = copy_user_segment_list(image, nr_segments, segments);  in kimage_alloc_init()
306 	ret = sanity_check_segment_list(image);  in kimage_alloc_init()
312 		image->control_page = crashk_res.start;  in kimage_alloc_init()
313 		image->type = KEXEC_TYPE_CRASH;  in kimage_alloc_init()
322 	image->control_code_page = kimage_alloc_control_pages(image,  in kimage_alloc_init()
324 	if (!image->control_code_page) {  in kimage_alloc_init()
330 		image->swap_page = kimage_alloc_control_pages(image, 0);  in kimage_alloc_init()
331 		if (!image->swap_page) {  in kimage_alloc_init()
337 	*rimage = image;  in kimage_alloc_init()
340 	kimage_free_page_list(&image->control_pages);  in kimage_alloc_init()
342 	kfree(image);  in kimage_alloc_init()
407 int __weak arch_kexec_kernel_image_probe(struct kimage *image, void *buf,  in arch_kexec_kernel_image_probe()  argument
413 void * __weak arch_kexec_kernel_image_load(struct kimage *image)  in arch_kexec_kernel_image_load()  argument
418 void __weak arch_kimage_file_post_load_cleanup(struct kimage *image)  in arch_kimage_file_post_load_cleanup()  argument
422 int __weak arch_kexec_kernel_verify_sig(struct kimage *image, void *buf,  in arch_kexec_kernel_verify_sig()  argument
451 static void kimage_file_post_load_cleanup(struct kimage *image)  in kimage_file_post_load_cleanup()  argument
453 	struct purgatory_info *pi = &image->purgatory_info;  in kimage_file_post_load_cleanup()
455 	vfree(image->kernel_buf);  in kimage_file_post_load_cleanup()
456 	image->kernel_buf = NULL;  in kimage_file_post_load_cleanup()
458 	vfree(image->initrd_buf);  in kimage_file_post_load_cleanup()
459 	image->initrd_buf = NULL;  in kimage_file_post_load_cleanup()
461 	kfree(image->cmdline_buf);  in kimage_file_post_load_cleanup()
462 	image->cmdline_buf = NULL;  in kimage_file_post_load_cleanup()
471 	arch_kimage_file_post_load_cleanup(image);  in kimage_file_post_load_cleanup()
478 	kfree(image->image_loader_data);  in kimage_file_post_load_cleanup()
479 	image->image_loader_data = NULL;  in kimage_file_post_load_cleanup()
487 kimage_file_prepare_segments(struct kimage *image, int kernel_fd, int initrd_fd,  in kimage_file_prepare_segments()  argument
494 	ret = copy_file_from_fd(kernel_fd, &image->kernel_buf,  in kimage_file_prepare_segments()
495 				&image->kernel_buf_len);  in kimage_file_prepare_segments()
500 	ret = arch_kexec_kernel_image_probe(image, image->kernel_buf,  in kimage_file_prepare_segments()
501 					    image->kernel_buf_len);  in kimage_file_prepare_segments()
507 	ret = arch_kexec_kernel_verify_sig(image, image->kernel_buf,  in kimage_file_prepare_segments()
508 					   image->kernel_buf_len);  in kimage_file_prepare_segments()
517 		ret = copy_file_from_fd(initrd_fd, &image->initrd_buf,  in kimage_file_prepare_segments()
518 					&image->initrd_buf_len);  in kimage_file_prepare_segments()
524 		image->cmdline_buf = kzalloc(cmdline_len, GFP_KERNEL);  in kimage_file_prepare_segments()
525 		if (!image->cmdline_buf) {  in kimage_file_prepare_segments()
530 		ret = copy_from_user(image->cmdline_buf, cmdline_ptr,  in kimage_file_prepare_segments()
537 		image->cmdline_buf_len = cmdline_len;  in kimage_file_prepare_segments()
540 		if (image->cmdline_buf[cmdline_len - 1] != '\0') {  in kimage_file_prepare_segments()
547 	ldata = arch_kexec_kernel_image_load(image);  in kimage_file_prepare_segments()
554 	image->image_loader_data = ldata;  in kimage_file_prepare_segments()
558 		kimage_file_post_load_cleanup(image);  in kimage_file_prepare_segments()
568 	struct kimage *image;  in kimage_file_alloc_init()  local
571 	image = do_kimage_alloc_init();  in kimage_file_alloc_init()
572 	if (!image)  in kimage_file_alloc_init()
575 	image->file_mode = 1;  in kimage_file_alloc_init()
579 		image->control_page = crashk_res.start;  in kimage_file_alloc_init()
580 		image->type = KEXEC_TYPE_CRASH;  in kimage_file_alloc_init()
583 	ret = kimage_file_prepare_segments(image, kernel_fd, initrd_fd,  in kimage_file_alloc_init()
588 	ret = sanity_check_segment_list(image);  in kimage_file_alloc_init()
593 	image->control_code_page = kimage_alloc_control_pages(image,  in kimage_file_alloc_init()
595 	if (!image->control_code_page) {  in kimage_file_alloc_init()
601 		image->swap_page = kimage_alloc_control_pages(image, 0);  in kimage_file_alloc_init()
602 		if (!image->swap_page) {  in kimage_file_alloc_init()
608 	*rimage = image;  in kimage_file_alloc_init()
611 	kimage_free_page_list(&image->control_pages);  in kimage_file_alloc_init()
613 	kimage_file_post_load_cleanup(image);  in kimage_file_alloc_init()
615 	kfree(image);  in kimage_file_alloc_init()
619 static inline void kimage_file_post_load_cleanup(struct kimage *image) { }  in kimage_file_post_load_cleanup()  argument
622 static int kimage_is_destination_range(struct kimage *image,  in kimage_is_destination_range()  argument
628 	for (i = 0; i < image->nr_segments; i++) {  in kimage_is_destination_range()
631 		mstart = image->segment[i].mem;  in kimage_is_destination_range()
632 		mend = mstart + image->segment[i].memsz;  in kimage_is_destination_range()
681 static struct page *kimage_alloc_normal_control_pages(struct kimage *image,  in kimage_alloc_normal_control_pages()  argument
718 			      kimage_is_destination_range(image, addr, eaddr)) {  in kimage_alloc_normal_control_pages()
726 		list_add(&pages->lru, &image->control_pages);  in kimage_alloc_normal_control_pages()
747 static struct page *kimage_alloc_crash_control_pages(struct kimage *image,  in kimage_alloc_crash_control_pages()  argument
776 	hole_start = (image->control_page + (size - 1)) & ~(size - 1);  in kimage_alloc_crash_control_pages()
784 		for (i = 0; i < image->nr_segments; i++) {  in kimage_alloc_crash_control_pages()
787 			mstart = image->segment[i].mem;  in kimage_alloc_crash_control_pages()
788 			mend   = mstart + image->segment[i].memsz - 1;  in kimage_alloc_crash_control_pages()
797 		if (i == image->nr_segments) {  in kimage_alloc_crash_control_pages()
803 		image->control_page = hole_end;  in kimage_alloc_crash_control_pages()
809 struct page *kimage_alloc_control_pages(struct kimage *image,  in kimage_alloc_control_pages()  argument
814 	switch (image->type) {  in kimage_alloc_control_pages()
816 		pages = kimage_alloc_normal_control_pages(image, order);  in kimage_alloc_control_pages()
819 		pages = kimage_alloc_crash_control_pages(image, order);  in kimage_alloc_control_pages()
826 static int kimage_add_entry(struct kimage *image, kimage_entry_t entry)  in kimage_add_entry()  argument
828 	if (*image->entry != 0)  in kimage_add_entry()
829 		image->entry++;  in kimage_add_entry()
831 	if (image->entry == image->last_entry) {  in kimage_add_entry()
835 		page = kimage_alloc_page(image, GFP_KERNEL, KIMAGE_NO_DEST);  in kimage_add_entry()
840 		*image->entry = virt_to_phys(ind_page) | IND_INDIRECTION;  in kimage_add_entry()
841 		image->entry = ind_page;  in kimage_add_entry()
842 		image->last_entry = ind_page +  in kimage_add_entry()
845 	*image->entry = entry;  in kimage_add_entry()
846 	image->entry++;  in kimage_add_entry()
847 	*image->entry = 0;  in kimage_add_entry()
852 static int kimage_set_destination(struct kimage *image,  in kimage_set_destination()  argument
858 	result = kimage_add_entry(image, destination | IND_DESTINATION);  in kimage_set_destination()
864 static int kimage_add_page(struct kimage *image, unsigned long page)  in kimage_add_page()  argument
869 	result = kimage_add_entry(image, page | IND_SOURCE);  in kimage_add_page()
875 static void kimage_free_extra_pages(struct kimage *image)  in kimage_free_extra_pages()  argument
878 	kimage_free_page_list(&image->dest_pages);  in kimage_free_extra_pages()
881 	kimage_free_page_list(&image->unusable_pages);  in kimage_free_extra_pages()
884 static void kimage_terminate(struct kimage *image)  in kimage_terminate()  argument
886 	if (*image->entry != 0)  in kimage_terminate()
887 		image->entry++;  in kimage_terminate()
889 	*image->entry = IND_DONE;  in kimage_terminate()
892 #define for_each_kimage_entry(image, ptr, entry) \  argument
893 	for (ptr = &image->head; (entry = *ptr) && !(entry & IND_DONE); \
905 static void kimage_free(struct kimage *image)  in kimage_free()  argument
910 	if (!image)  in kimage_free()
913 	kimage_free_extra_pages(image);  in kimage_free()
914 	for_each_kimage_entry(image, ptr, entry) {  in kimage_free()
931 	machine_kexec_cleanup(image);  in kimage_free()
934 	kimage_free_page_list(&image->control_pages);  in kimage_free()
940 	if (image->file_mode)  in kimage_free()
941 		kimage_file_post_load_cleanup(image);  in kimage_free()
943 	kfree(image);  in kimage_free()
946 static kimage_entry_t *kimage_dst_used(struct kimage *image,  in kimage_dst_used()  argument
952 	for_each_kimage_entry(image, ptr, entry) {  in kimage_dst_used()
965 static struct page *kimage_alloc_page(struct kimage *image,  in kimage_alloc_page()  argument
994 	list_for_each_entry(page, &image->dest_pages, lru) {  in kimage_alloc_page()
1012 			list_add(&page->lru, &image->unusable_pages);  in kimage_alloc_page()
1022 		if (!kimage_is_destination_range(image, addr,  in kimage_alloc_page()
1031 		old = kimage_dst_used(image, addr);  in kimage_alloc_page()
1058 			list_add(&page->lru, &image->dest_pages);  in kimage_alloc_page()
1065 static int kimage_load_normal_segment(struct kimage *image,  in kimage_load_normal_segment()  argument
1075 	if (image->file_mode)  in kimage_load_normal_segment()
1083 	result = kimage_set_destination(image, maddr);  in kimage_load_normal_segment()
1092 		page = kimage_alloc_page(image, GFP_HIGHUSER, maddr);  in kimage_load_normal_segment()
1097 		result = kimage_add_page(image, page_to_pfn(page)  in kimage_load_normal_segment()
1111 		if (image->file_mode)  in kimage_load_normal_segment()
1122 		if (image->file_mode)  in kimage_load_normal_segment()
1132 static int kimage_load_crash_segment(struct kimage *image,  in kimage_load_crash_segment()  argument
1146 	if (image->file_mode)  in kimage_load_crash_segment()
1174 		if (image->file_mode)  in kimage_load_crash_segment()
1186 		if (image->file_mode)  in kimage_load_crash_segment()
1196 static int kimage_load_segment(struct kimage *image,  in kimage_load_segment()  argument
1201 	switch (image->type) {  in kimage_load_segment()
1203 		result = kimage_load_normal_segment(image, segment);  in kimage_load_segment()
1206 		result = kimage_load_crash_segment(image, segment);  in kimage_load_segment()
1242 	struct kimage **dest_image, *image;  in SYSCALL_DEFINE4()  local
1267 	image = NULL;  in SYSCALL_DEFINE4()
1295 			result = kimage_alloc_init(&image, entry, nr_segments,  in SYSCALL_DEFINE4()
1301 			result = kimage_alloc_init(&image, entry, nr_segments,  in SYSCALL_DEFINE4()
1308 			image->preserve_context = 1;  in SYSCALL_DEFINE4()
1309 		result = machine_kexec_prepare(image);  in SYSCALL_DEFINE4()
1314 			result = kimage_load_segment(image, &image->segment[i]);  in SYSCALL_DEFINE4()
1318 		kimage_terminate(image);  in SYSCALL_DEFINE4()
1323 	image = xchg(dest_image, image);  in SYSCALL_DEFINE4()
1327 	kimage_free(image);  in SYSCALL_DEFINE4()
1389 	struct kimage **dest_image, *image;  in SYSCALL_DEFINE5()  local
1399 	image = NULL;  in SYSCALL_DEFINE5()
1419 	ret = kimage_file_alloc_init(&image, kernel_fd, initrd_fd, cmdline_ptr,  in SYSCALL_DEFINE5()
1424 	ret = machine_kexec_prepare(image);  in SYSCALL_DEFINE5()
1428 	ret = kexec_calculate_store_digests(image);  in SYSCALL_DEFINE5()
1432 	for (i = 0; i < image->nr_segments; i++) {  in SYSCALL_DEFINE5()
1435 		ksegment = &image->segment[i];  in SYSCALL_DEFINE5()
1440 		ret = kimage_load_segment(image, &image->segment[i]);  in SYSCALL_DEFINE5()
1445 	kimage_terminate(image);  in SYSCALL_DEFINE5()
1451 	kimage_file_post_load_cleanup(image);  in SYSCALL_DEFINE5()
1453 	image = xchg(dest_image, image);  in SYSCALL_DEFINE5()
1456 	kimage_free(image);  in SYSCALL_DEFINE5()
2020 	struct kimage *image = kbuf->image;  in locate_mem_hole_top_down()  local
2039 		if (kimage_is_destination_range(image, temp_start, temp_end)) {  in locate_mem_hole_top_down()
2058 	struct kimage *image = kbuf->image;  in locate_mem_hole_bottom_up()  local
2073 		if (kimage_is_destination_range(image, temp_start, temp_end)) {  in locate_mem_hole_bottom_up()
2114 int kexec_add_buffer(struct kimage *image, char *buffer, unsigned long bufsz,  in kexec_add_buffer()  argument
2125 	if (!image->file_mode)  in kexec_add_buffer()
2128 	if (image->nr_segments >= KEXEC_SEGMENT_MAX)  in kexec_add_buffer()
2138 	if (!list_empty(&image->control_pages)) {  in kexec_add_buffer()
2145 	kbuf->image = image;  in kexec_add_buffer()
2156 	if (image->type == KEXEC_TYPE_CRASH)  in kexec_add_buffer()
2170 	ksegment = &image->segment[image->nr_segments];  in kexec_add_buffer()
2175 	image->nr_segments++;  in kexec_add_buffer()
2181 static int kexec_calculate_store_digests(struct kimage *image)  in kexec_calculate_store_digests()  argument
2190 	struct purgatory_info *pi = &image->purgatory_info;  in kexec_calculate_store_digests()
2226 	for (j = i = 0; i < image->nr_segments; i++) {  in kexec_calculate_store_digests()
2229 		ksegment = &image->segment[i];  in kexec_calculate_store_digests()
2270 		ret = kexec_purgatory_get_set_symbol(image, "sha_regions",  in kexec_calculate_store_digests()
2275 		ret = kexec_purgatory_get_set_symbol(image, "sha256_digest",  in kexec_calculate_store_digests()
2294 static int __kexec_load_purgatory(struct kimage *image, unsigned long min,  in __kexec_load_purgatory()  argument
2297 	struct purgatory_info *pi = &image->purgatory_info;  in __kexec_load_purgatory()
2412 	ret = kexec_add_buffer(image, purgatory_buf, buf_sz, memsz,  in __kexec_load_purgatory()
2458 	image->start = entry;  in __kexec_load_purgatory()
2475 static int kexec_apply_relocations(struct kimage *image)  in kexec_apply_relocations()  argument
2478 	struct purgatory_info *pi = &image->purgatory_info;  in kexec_apply_relocations()
2531 int kexec_load_purgatory(struct kimage *image, unsigned long min,  in kexec_load_purgatory()  argument
2535 	struct purgatory_info *pi = &image->purgatory_info;  in kexec_load_purgatory()
2557 	ret = __kexec_load_purgatory(image, min, max, top_down);  in kexec_load_purgatory()
2561 	ret = kexec_apply_relocations(image);  in kexec_load_purgatory()
2621 void *kexec_purgatory_get_symbol_addr(struct kimage *image, const char *name)  in kexec_purgatory_get_symbol_addr()  argument
2623 	struct purgatory_info *pi = &image->purgatory_info;  in kexec_purgatory_get_symbol_addr()
2644 int kexec_purgatory_get_set_symbol(struct kimage *image, const char *name,  in kexec_purgatory_get_set_symbol()  argument
2649 	struct purgatory_info *pi = &image->purgatory_info;  in kexec_purgatory_get_set_symbol()