47 typedef int (*rproc_handle_resources_t)(struct rproc *rproc,
49 typedef int (*rproc_handle_resource_t)(struct rproc *rproc,
78 	struct rproc *rproc = token;  in rproc_iommu_fault()  local
82 	rproc_report_crash(rproc, RPROC_MMUFAULT);  in rproc_iommu_fault()
91 static int rproc_enable_iommu(struct rproc *rproc)  in rproc_enable_iommu()  argument
94 	struct device *dev = rproc->dev.parent;  in rproc_enable_iommu()
97 	if (!rproc->has_iommu) {  in rproc_enable_iommu()
108 	iommu_set_fault_handler(domain, rproc_iommu_fault, rproc);  in rproc_enable_iommu()
116 	rproc->domain = domain;  in rproc_enable_iommu()
125 static void rproc_disable_iommu(struct rproc *rproc)  in rproc_disable_iommu()  argument
127 	struct iommu_domain *domain = rproc->domain;  in rproc_disable_iommu()
128 	struct device *dev = rproc->dev.parent;  in rproc_disable_iommu()
156 void *rproc_da_to_va(struct rproc *rproc, u64 da, int len)  in rproc_da_to_va()  argument
161 	list_for_each_entry(carveout, &rproc->carveouts, node) {  in rproc_da_to_va()
183 	struct rproc *rproc = rvdev->rproc;  in rproc_alloc_vring()  local
184 	struct device *dev = &rproc->dev;  in rproc_alloc_vring()
209 	ret = idr_alloc(&rproc->notifyids, rvring, 0, 0, GFP_KERNEL);  in rproc_alloc_vring()
230 	rsc = (void *)rproc->table_ptr + rvdev->rsc_offset;  in rproc_alloc_vring()
239 	struct rproc *rproc = rvdev->rproc;  in rproc_parse_vring()  local
240 	struct device *dev = &rproc->dev;  in rproc_parse_vring()
270 	struct rproc *rproc = rvring->rvdev->rproc;  in rproc_free_vring()  local
274 	dma_free_coherent(rproc->dev.parent, size, rvring->va, rvring->dma);  in rproc_free_vring()
275 	idr_remove(&rproc->notifyids, rvring->notifyid);  in rproc_free_vring()
278 	rsc = (void *)rproc->table_ptr + rvring->rvdev->rsc_offset;  in rproc_free_vring()
310 static int rproc_handle_vdev(struct rproc *rproc, struct fw_rsc_vdev *rsc,  in rproc_handle_vdev()  argument
313 	struct device *dev = &rproc->dev;  in rproc_handle_vdev()
343 	rvdev->rproc = rproc;  in rproc_handle_vdev()
355 	list_add_tail(&rvdev->node, &rproc->rvdevs);  in rproc_handle_vdev()
387 static int rproc_handle_trace(struct rproc *rproc, struct fw_rsc_trace *rsc,  in rproc_handle_trace()  argument
391 	struct device *dev = &rproc->dev;  in rproc_handle_trace()
407 	ptr = rproc_da_to_va(rproc, rsc->da, rsc->len);  in rproc_handle_trace()
424 	snprintf(name, sizeof(name), "trace%d", rproc->num_traces);  in rproc_handle_trace()
427 	trace->priv = rproc_create_trace_file(name, rproc, trace);  in rproc_handle_trace()
434 	list_add_tail(&trace->node, &rproc->traces);  in rproc_handle_trace()
436 	rproc->num_traces++;  in rproc_handle_trace()
469 static int rproc_handle_devmem(struct rproc *rproc, struct fw_rsc_devmem *rsc,  in rproc_handle_devmem()  argument
473 	struct device *dev = &rproc->dev;  in rproc_handle_devmem()
477 	if (!rproc->domain)  in rproc_handle_devmem()
497 	ret = iommu_map(rproc->domain, rsc->da, rsc->pa, rsc->len, rsc->flags);  in rproc_handle_devmem()
512 	list_add_tail(&mapping->node, &rproc->mappings);  in rproc_handle_devmem()
542 static int rproc_handle_carveout(struct rproc *rproc,  in rproc_handle_carveout()  argument
548 	struct device *dev = &rproc->dev;  in rproc_handle_carveout()
600 	if (rproc->domain) {  in rproc_handle_carveout()
608 		ret = iommu_map(rproc->domain, rsc->da, dma, rsc->len,  in rproc_handle_carveout()
624 		list_add_tail(&mapping->node, &rproc->mappings);  in rproc_handle_carveout()
654 	list_add_tail(&carveout->node, &rproc->carveouts);  in rproc_handle_carveout()
667 static int rproc_count_vrings(struct rproc *rproc, struct fw_rsc_vdev *rsc,  in rproc_count_vrings()  argument
671 	rproc->max_notifyid += rsc->num_of_vrings;  in rproc_count_vrings()
696 static int rproc_handle_resources(struct rproc *rproc, int len,  in rproc_handle_resources()  argument
699 	struct device *dev = &rproc->dev;  in rproc_handle_resources()
703 	for (i = 0; i < rproc->table_ptr->num; i++) {  in rproc_handle_resources()
704 		int offset = rproc->table_ptr->offset[i];  in rproc_handle_resources()
705 		struct fw_rsc_hdr *hdr = (void *)rproc->table_ptr + offset;  in rproc_handle_resources()
726 		ret = handler(rproc, rsc, offset + sizeof(*hdr), avail);  in rproc_handle_resources()
741 static void rproc_resource_cleanup(struct rproc *rproc)  in rproc_resource_cleanup()  argument
744 	struct device *dev = &rproc->dev;  in rproc_resource_cleanup()
747 	list_for_each_entry_safe(entry, tmp, &rproc->traces, node) {  in rproc_resource_cleanup()
749 		rproc->num_traces--;  in rproc_resource_cleanup()
755 	list_for_each_entry_safe(entry, tmp, &rproc->mappings, node) {  in rproc_resource_cleanup()
758 		unmapped = iommu_unmap(rproc->domain, entry->da, entry->len);  in rproc_resource_cleanup()
770 	list_for_each_entry_safe(entry, tmp, &rproc->carveouts, node) {  in rproc_resource_cleanup()
780 static int rproc_fw_boot(struct rproc *rproc, const struct firmware *fw)  in rproc_fw_boot()  argument
782 	struct device *dev = &rproc->dev;  in rproc_fw_boot()
783 	const char *name = rproc->firmware;  in rproc_fw_boot()
787 	if (!rproc->table_ptr)  in rproc_fw_boot()
790 	ret = rproc_fw_sanity_check(rproc, fw);  in rproc_fw_boot()
800 	ret = rproc_enable_iommu(rproc);  in rproc_fw_boot()
806 	rproc->bootaddr = rproc_get_boot_addr(rproc, fw);  in rproc_fw_boot()
810 	table = rproc_find_rsc_table(rproc, fw, &tablesz);  in rproc_fw_boot()
816 	if (rproc->table_csum != crc32(0, table, tablesz)) {  in rproc_fw_boot()
822 	ret = rproc_handle_resources(rproc, tablesz, rproc_loading_handlers);  in rproc_fw_boot()
829 	ret = rproc_load_segments(rproc, fw);  in rproc_fw_boot()
842 	loaded_table = rproc_find_loaded_rsc_table(rproc, fw);  in rproc_fw_boot()
848 	memcpy(loaded_table, rproc->cached_table, tablesz);  in rproc_fw_boot()
851 	ret = rproc->ops->start(rproc);  in rproc_fw_boot()
853 		dev_err(dev, "can't start rproc %s: %d\n", rproc->name, ret);  in rproc_fw_boot()
862 	rproc->table_ptr = loaded_table;  in rproc_fw_boot()
864 	rproc->state = RPROC_RUNNING;  in rproc_fw_boot()
866 	dev_info(dev, "remote processor %s is now up\n", rproc->name);  in rproc_fw_boot()
871 	rproc_resource_cleanup(rproc);  in rproc_fw_boot()
872 	rproc_disable_iommu(rproc);  in rproc_fw_boot()
886 	struct rproc *rproc = context;  in rproc_fw_config_virtio()  local
890 	if (rproc_fw_sanity_check(rproc, fw) < 0)  in rproc_fw_config_virtio()
894 	table = rproc_find_rsc_table(rproc, fw,  &tablesz);  in rproc_fw_config_virtio()
898 	rproc->table_csum = crc32(0, table, tablesz);  in rproc_fw_config_virtio()
906 	rproc->cached_table = kmemdup(table, tablesz, GFP_KERNEL);  in rproc_fw_config_virtio()
907 	if (!rproc->cached_table)  in rproc_fw_config_virtio()
910 	rproc->table_ptr = rproc->cached_table;  in rproc_fw_config_virtio()
913 	rproc->max_notifyid = -1;  in rproc_fw_config_virtio()
914 	ret = rproc_handle_resources(rproc, tablesz, rproc_count_vrings_handler);  in rproc_fw_config_virtio()
919 	ret = rproc_handle_resources(rproc, tablesz, rproc_vdev_handler);  in rproc_fw_config_virtio()
924 	complete_all(&rproc->firmware_loading_complete);  in rproc_fw_config_virtio()
927 static int rproc_add_virtio_devices(struct rproc *rproc)  in rproc_add_virtio_devices()  argument
932 	init_completion(&rproc->firmware_loading_complete);  in rproc_add_virtio_devices()
943 				      rproc->firmware, &rproc->dev, GFP_KERNEL,  in rproc_add_virtio_devices()
944 				      rproc, rproc_fw_config_virtio);  in rproc_add_virtio_devices()
946 		dev_err(&rproc->dev, "request_firmware_nowait err: %d\n", ret);  in rproc_add_virtio_devices()
947 		complete_all(&rproc->firmware_loading_complete);  in rproc_add_virtio_devices()
963 int rproc_trigger_recovery(struct rproc *rproc)  in rproc_trigger_recovery()  argument
967 	dev_err(&rproc->dev, "recovering %s\n", rproc->name);  in rproc_trigger_recovery()
969 	init_completion(&rproc->crash_comp);  in rproc_trigger_recovery()
972 	list_for_each_entry_safe(rvdev, rvtmp, &rproc->rvdevs, node)  in rproc_trigger_recovery()
976 	wait_for_completion(&rproc->crash_comp);  in rproc_trigger_recovery()
979 	kfree(rproc->cached_table);  in rproc_trigger_recovery()
981 	return rproc_add_virtio_devices(rproc);  in rproc_trigger_recovery()
992 	struct rproc *rproc = container_of(work, struct rproc, crash_handler);  in rproc_crash_handler_work()  local
993 	struct device *dev = &rproc->dev;  in rproc_crash_handler_work()
997 	mutex_lock(&rproc->lock);  in rproc_crash_handler_work()
999 	if (rproc->state == RPROC_CRASHED || rproc->state == RPROC_OFFLINE) {  in rproc_crash_handler_work()
1001 		mutex_unlock(&rproc->lock);  in rproc_crash_handler_work()
1005 	rproc->state = RPROC_CRASHED;  in rproc_crash_handler_work()
1006 	dev_err(dev, "handling crash #%u in %s\n", ++rproc->crash_cnt,  in rproc_crash_handler_work()
1007 		rproc->name);  in rproc_crash_handler_work()
1009 	mutex_unlock(&rproc->lock);  in rproc_crash_handler_work()
1011 	if (!rproc->recovery_disabled)  in rproc_crash_handler_work()
1012 		rproc_trigger_recovery(rproc);  in rproc_crash_handler_work()
1026 int rproc_boot(struct rproc *rproc)  in rproc_boot()  argument
1032 	if (!rproc) {  in rproc_boot()
1037 	dev = &rproc->dev;  in rproc_boot()
1039 	ret = mutex_lock_interruptible(&rproc->lock);  in rproc_boot()
1041 		dev_err(dev, "can't lock rproc %s: %d\n", rproc->name, ret);  in rproc_boot()
1046 	if (!rproc->firmware) {  in rproc_boot()
1060 	if (atomic_inc_return(&rproc->power) > 1) {  in rproc_boot()
1065 	dev_info(dev, "powering up %s\n", rproc->name);  in rproc_boot()
1068 	ret = request_firmware(&firmware_p, rproc->firmware, dev);  in rproc_boot()
1074 	ret = rproc_fw_boot(rproc, firmware_p);  in rproc_boot()
1081 		atomic_dec(&rproc->power);  in rproc_boot()
1084 	mutex_unlock(&rproc->lock);  in rproc_boot()
1108 void rproc_shutdown(struct rproc *rproc)  in rproc_shutdown()  argument
1110 	struct device *dev = &rproc->dev;  in rproc_shutdown()
1113 	ret = mutex_lock_interruptible(&rproc->lock);  in rproc_shutdown()
1115 		dev_err(dev, "can't lock rproc %s: %d\n", rproc->name, ret);  in rproc_shutdown()
1120 	if (!atomic_dec_and_test(&rproc->power))  in rproc_shutdown()
1124 	ret = rproc->ops->stop(rproc);  in rproc_shutdown()
1126 		atomic_inc(&rproc->power);  in rproc_shutdown()
1132 	rproc_resource_cleanup(rproc);  in rproc_shutdown()
1134 	rproc_disable_iommu(rproc);  in rproc_shutdown()
1137 	rproc->table_ptr = rproc->cached_table;  in rproc_shutdown()
1140 	if (rproc->state == RPROC_CRASHED)  in rproc_shutdown()
1141 		complete_all(&rproc->crash_comp);  in rproc_shutdown()
1143 	rproc->state = RPROC_OFFLINE;  in rproc_shutdown()
1145 	dev_info(dev, "stopped remote processor %s\n", rproc->name);  in rproc_shutdown()
1148 	mutex_unlock(&rproc->lock);  in rproc_shutdown()
1174 int rproc_add(struct rproc *rproc)  in rproc_add()  argument
1176 	struct device *dev = &rproc->dev;  in rproc_add()
1183 	dev_info(dev, "%s is available\n", rproc->name);  in rproc_add()
1189 	rproc_create_debug_dir(rproc);  in rproc_add()
1191 	return rproc_add_virtio_devices(rproc);  in rproc_add()
1206 	struct rproc *rproc = container_of(dev, struct rproc, dev);  in rproc_type_release()  local
1208 	dev_info(&rproc->dev, "releasing %s\n", rproc->name);  in rproc_type_release()
1210 	rproc_delete_debug_dir(rproc);  in rproc_type_release()
1212 	idr_destroy(&rproc->notifyids);  in rproc_type_release()
1214 	if (rproc->index >= 0)  in rproc_type_release()
1215 		ida_simple_remove(&rproc_dev_index, rproc->index);  in rproc_type_release()
1217 	kfree(rproc);  in rproc_type_release()
1248 struct rproc *rproc_alloc(struct device *dev, const char *name,  in rproc_alloc()
1252 	struct rproc *rproc;  in rproc_alloc()  local
1270 	rproc = kzalloc(sizeof(struct rproc) + len + name_len, GFP_KERNEL);  in rproc_alloc()
1271 	if (!rproc) {  in rproc_alloc()
1277 		p = (char *)rproc + sizeof(struct rproc) + len;  in rproc_alloc()
1283 	rproc->firmware = p;  in rproc_alloc()
1284 	rproc->name = name;  in rproc_alloc()
1285 	rproc->ops = ops;  in rproc_alloc()
1286 	rproc->priv = &rproc[1];  in rproc_alloc()
1288 	device_initialize(&rproc->dev);  in rproc_alloc()
1289 	rproc->dev.parent = dev;  in rproc_alloc()
1290 	rproc->dev.type = &rproc_type;  in rproc_alloc()
1293 	rproc->index = ida_simple_get(&rproc_dev_index, 0, 0, GFP_KERNEL);  in rproc_alloc()
1294 	if (rproc->index < 0) {  in rproc_alloc()
1295 		dev_err(dev, "ida_simple_get failed: %d\n", rproc->index);  in rproc_alloc()
1296 		put_device(&rproc->dev);  in rproc_alloc()
1300 	dev_set_name(&rproc->dev, "remoteproc%d", rproc->index);  in rproc_alloc()
1302 	atomic_set(&rproc->power, 0);  in rproc_alloc()
1305 	rproc->fw_ops = &rproc_elf_fw_ops;  in rproc_alloc()
1307 	mutex_init(&rproc->lock);  in rproc_alloc()
1309 	idr_init(&rproc->notifyids);  in rproc_alloc()
1311 	INIT_LIST_HEAD(&rproc->carveouts);  in rproc_alloc()
1312 	INIT_LIST_HEAD(&rproc->mappings);  in rproc_alloc()
1313 	INIT_LIST_HEAD(&rproc->traces);  in rproc_alloc()
1314 	INIT_LIST_HEAD(&rproc->rvdevs);  in rproc_alloc()
1316 	INIT_WORK(&rproc->crash_handler, rproc_crash_handler_work);  in rproc_alloc()
1317 	init_completion(&rproc->crash_comp);  in rproc_alloc()
1319 	rproc->state = RPROC_OFFLINE;  in rproc_alloc()
1321 	return rproc;  in rproc_alloc()
1334 void rproc_put(struct rproc *rproc)  in rproc_put()  argument
1336 	put_device(&rproc->dev);  in rproc_put()
1355 int rproc_del(struct rproc *rproc)  in rproc_del()  argument
1359 	if (!rproc)  in rproc_del()
1363 	wait_for_completion(&rproc->firmware_loading_complete);  in rproc_del()
1366 	list_for_each_entry_safe(rvdev, tmp, &rproc->rvdevs, node)  in rproc_del()
1370 	kfree(rproc->cached_table);  in rproc_del()
1372 	device_del(&rproc->dev);  in rproc_del()
1389 void rproc_report_crash(struct rproc *rproc, enum rproc_crash_type type)  in rproc_report_crash()  argument
1391 	if (!rproc) {  in rproc_report_crash()
1396 	dev_err(&rproc->dev, "crash detected in %s: type %s\n",  in rproc_report_crash()
1397 		rproc->name, rproc_crash_to_string(type));  in rproc_report_crash()
1400 	schedule_work(&rproc->crash_handler);  in rproc_report_crash()