114 static void block_list_remove(struct sst_dsp *dsp,  in block_list_remove()  argument
126 				dev_err(dsp->dev,  in block_list_remove()
135 		list_move(&block->list, &dsp->free_block_list);  in block_list_remove()
136 		dev_dbg(dsp->dev, "block freed %d:%d at offset 0x%x\n",  in block_list_remove()
142 static int block_list_prepare(struct sst_dsp *dsp,  in block_list_prepare()  argument
154 				dev_err(dsp->dev,  in block_list_prepare()
206 	struct sst_dsp *dsp = (struct sst_dsp *)param;  in dma_chan_filter()  local
208 	return chan->device->dev == dsp->dma_dev;  in dma_chan_filter()
211 int sst_dsp_dma_get_channel(struct sst_dsp *dsp, int chan_id)  in sst_dsp_dma_get_channel()  argument
213 	struct sst_dma *dma = dsp->dma;  in sst_dsp_dma_get_channel()
222 	dma->ch = dma_request_channel(mask, dma_chan_filter, dsp);  in sst_dsp_dma_get_channel()
224 		dev_err(dsp->dev, "error: DMA request channel failed\n");  in sst_dsp_dma_get_channel()
236 		dev_err(dsp->dev, "error: unable to set DMA slave config %d\n",  in sst_dsp_dma_get_channel()
246 void sst_dsp_dma_put_channel(struct sst_dsp *dsp)  in sst_dsp_dma_put_channel()  argument
248 	struct sst_dma *dma = dsp->dma;  in sst_dsp_dma_put_channel()
328 struct sst_fw *sst_fw_new(struct sst_dsp *dsp,   in sst_fw_new()  argument
334 	if (!dsp->ops->parse_fw)  in sst_fw_new()
341 	sst_fw->dsp = dsp;  in sst_fw_new()
346 	sst_fw->dma_buf = dma_alloc_coherent(dsp->dma_dev, sst_fw->size,  in sst_fw_new()
349 		dev_err(dsp->dev, "error: DMA alloc failed\n");  in sst_fw_new()
357 	if (dsp->fw_use_dma) {  in sst_fw_new()
358 		err = sst_dsp_dma_get_channel(dsp, 0);  in sst_fw_new()
364 	err = dsp->ops->parse_fw(sst_fw);  in sst_fw_new()
366 		dev_err(dsp->dev, "error: parse fw failed %d\n", err);  in sst_fw_new()
370 	if (dsp->fw_use_dma)  in sst_fw_new()
371 		sst_dsp_dma_put_channel(dsp);  in sst_fw_new()
373 	mutex_lock(&dsp->mutex);  in sst_fw_new()
374 	list_add(&sst_fw->list, &dsp->fw_list);  in sst_fw_new()
375 	mutex_unlock(&dsp->mutex);  in sst_fw_new()
380 	if (dsp->fw_use_dma)  in sst_fw_new()
381 		sst_dsp_dma_put_channel(dsp);  in sst_fw_new()
383 	dma_free_coherent(dsp->dma_dev, sst_fw->size,  in sst_fw_new()
394 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_fw_reload()  local
397 	dev_dbg(dsp->dev, "reloading firmware\n");  in sst_fw_reload()
400 	ret = dsp->ops->parse_fw(sst_fw);  in sst_fw_reload()
402 		dev_err(dsp->dev, "error: parse fw failed %d\n", ret);  in sst_fw_reload()
410 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_fw_unload()  local
414 	dev_dbg(dsp->dev, "unloading firmware\n");  in sst_fw_unload()
416 	mutex_lock(&dsp->mutex);  in sst_fw_unload()
419 	list_for_each_entry_safe(module, mtmp, &dsp->module_list, list) {  in sst_fw_unload()
425 				block_list_remove(dsp, &runtime->block_list);  in sst_fw_unload()
431 			block_list_remove(dsp, &module->block_list);  in sst_fw_unload()
438 	block_list_remove(dsp, &dsp->scratch_block_list);  in sst_fw_unload()
440 	mutex_unlock(&dsp->mutex);  in sst_fw_unload()
447 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_fw_free()  local
449 	mutex_lock(&dsp->mutex);  in sst_fw_free()
451 	mutex_unlock(&dsp->mutex);  in sst_fw_free()
454 		dma_free_coherent(dsp->dma_dev, sst_fw->size, sst_fw->dma_buf,  in sst_fw_free()
461 void sst_fw_free_all(struct sst_dsp *dsp)  in sst_fw_free_all()  argument
465 	mutex_lock(&dsp->mutex);  in sst_fw_free_all()
466 	list_for_each_entry_safe(sst_fw, t, &dsp->fw_list, list) {  in sst_fw_free_all()
469 		dma_free_coherent(dsp->dev, sst_fw->size, sst_fw->dma_buf,  in sst_fw_free_all()
473 	mutex_unlock(&dsp->mutex);  in sst_fw_free_all()
481 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_module_new()  local
489 	sst_module->dsp = dsp;  in sst_module_new()
499 	mutex_lock(&dsp->mutex);  in sst_module_new()
500 	list_add(&sst_module->list, &dsp->module_list);  in sst_module_new()
501 	mutex_unlock(&dsp->mutex);  in sst_module_new()
510 	struct sst_dsp *dsp = sst_module->dsp;  in sst_module_free()  local
512 	mutex_lock(&dsp->mutex);  in sst_module_free()
514 	mutex_unlock(&dsp->mutex);  in sst_module_free()
523 	struct sst_dsp *dsp = module->dsp;  in sst_module_runtime_new()  local
531 	runtime->dsp = dsp;  in sst_module_runtime_new()
535 	mutex_lock(&dsp->mutex);  in sst_module_runtime_new()
537 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_new()
545 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_free()  local
547 	mutex_lock(&dsp->mutex);  in sst_module_runtime_free()
549 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_free()
555 static struct sst_mem_block *find_block(struct sst_dsp *dsp,  in find_block()  argument
560 	list_for_each_entry(block, &dsp->free_block_list, list) {  in find_block()
569 static int block_alloc_contiguous(struct sst_dsp *dsp,  in block_alloc_contiguous()  argument
579 		block = find_block(dsp, ba);  in block_alloc_contiguous()
581 			list_splice(&tmp, &dsp->free_block_list);  in block_alloc_contiguous()
602 		dev_dbg(dsp->dev, "block allocated %d:%d at offset 0x%x\n",  in block_alloc_contiguous()
606 	list_splice(&tmp, &dsp->used_block_list);  in block_alloc_contiguous()
611 static int block_alloc(struct sst_dsp *dsp, struct sst_block_allocator *ba,  in block_alloc()  argument
621 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in block_alloc()
633 		list_move(&block->list, &dsp->used_block_list);  in block_alloc()
634 		dev_dbg(dsp->dev, "block allocated %d:%d at offset 0x%x\n",  in block_alloc()
640 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in block_alloc()
652 			ret = block_alloc_contiguous(dsp, ba, block_list);  in block_alloc()
663 int sst_alloc_blocks(struct sst_dsp *dsp, struct sst_block_allocator *ba,  in sst_alloc_blocks()  argument
668 	dev_dbg(dsp->dev, "block request 0x%x bytes at offset 0x%x type %d\n",  in sst_alloc_blocks()
671 	mutex_lock(&dsp->mutex);  in sst_alloc_blocks()
673 	ret = block_alloc(dsp, ba, block_list);  in sst_alloc_blocks()
675 		dev_err(dsp->dev, "error: can't alloc blocks %d\n", ret);  in sst_alloc_blocks()
680 	ret = block_list_prepare(dsp, block_list);  in sst_alloc_blocks()
682 		dev_err(dsp->dev, "error: prepare failed\n");  in sst_alloc_blocks()
685 	mutex_unlock(&dsp->mutex);  in sst_alloc_blocks()
690 int sst_free_blocks(struct sst_dsp *dsp, struct list_head *block_list)  in sst_free_blocks()  argument
692 	mutex_lock(&dsp->mutex);  in sst_free_blocks()
693 	block_list_remove(dsp, block_list);  in sst_free_blocks()
694 	mutex_unlock(&dsp->mutex);  in sst_free_blocks()
700 static int block_alloc_fixed(struct sst_dsp *dsp, struct sst_block_allocator *ba,  in block_alloc_fixed()  argument
731 			err = block_alloc_contiguous(dsp, &ba_tmp, block_list);  in block_alloc_fixed()
741 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in block_alloc_fixed()
752 			list_move(&block->list, &dsp->used_block_list);  in block_alloc_fixed()
754 			dev_dbg(dsp->dev, "block allocated %d:%d at offset 0x%x\n",  in block_alloc_fixed()
763 			list_move(&block->list, &dsp->used_block_list);  in block_alloc_fixed()
769 			err = block_alloc_contiguous(dsp, &ba_tmp, block_list);  in block_alloc_fixed()
783 	struct sst_dsp *dsp = module->dsp;  in sst_module_alloc_blocks()  local
793 	dev_dbg(dsp->dev, "block request 0x%x bytes at offset 0x%x type %d\n",  in sst_module_alloc_blocks()
796 	mutex_lock(&dsp->mutex);  in sst_module_alloc_blocks()
799 	ret = block_alloc_fixed(dsp, &ba, &module->block_list);  in sst_module_alloc_blocks()
801 		dev_err(dsp->dev,  in sst_module_alloc_blocks()
804 		mutex_unlock(&dsp->mutex);  in sst_module_alloc_blocks()
809 	ret = block_list_prepare(dsp, &module->block_list);  in sst_module_alloc_blocks()
811 		dev_err(dsp->dev, "error: fw module prepare failed\n");  in sst_module_alloc_blocks()
816 	if (dsp->fw_use_dma) {  in sst_module_alloc_blocks()
817 		ret = sst_dsp_dma_copyto(dsp,  in sst_module_alloc_blocks()
818 			dsp->addr.lpe_base + module->offset,  in sst_module_alloc_blocks()
822 			dev_err(dsp->dev, "error: module copy failed\n");  in sst_module_alloc_blocks()
826 		sst_memcpy32(dsp->addr.lpe + module->offset, module->data,  in sst_module_alloc_blocks()
829 	mutex_unlock(&dsp->mutex);  in sst_module_alloc_blocks()
833 	block_list_remove(dsp, &module->block_list);  in sst_module_alloc_blocks()
834 	mutex_unlock(&dsp->mutex);  in sst_module_alloc_blocks()
842 	struct sst_dsp *dsp = module->dsp;  in sst_module_free_blocks()  local
844 	mutex_lock(&dsp->mutex);  in sst_module_free_blocks()
845 	block_list_remove(dsp, &module->block_list);  in sst_module_free_blocks()
846 	mutex_unlock(&dsp->mutex);  in sst_module_free_blocks()
854 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_alloc_blocks()  local
866 	mutex_lock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
873 		dev_dbg(dsp->dev, "persistent fixed block request 0x%x bytes type %d offset 0x%x\n",  in sst_module_runtime_alloc_blocks()
877 		ret = block_alloc_fixed(dsp, &ba, &runtime->block_list);  in sst_module_runtime_alloc_blocks()
880 		dev_dbg(dsp->dev, "persistent block request 0x%x bytes type %d\n",  in sst_module_runtime_alloc_blocks()
884 		ret = block_alloc(dsp, &ba, &runtime->block_list);  in sst_module_runtime_alloc_blocks()
887 		dev_err(dsp->dev,  in sst_module_runtime_alloc_blocks()
890 		mutex_unlock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
896 	ret = block_list_prepare(dsp, &runtime->block_list);  in sst_module_runtime_alloc_blocks()
898 		dev_err(dsp->dev, "error: runtime block prepare failed\n");  in sst_module_runtime_alloc_blocks()
902 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
906 	block_list_remove(dsp, &module->block_list);  in sst_module_runtime_alloc_blocks()
907 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
914 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_free_blocks()  local
916 	mutex_lock(&dsp->mutex);  in sst_module_runtime_free_blocks()
917 	block_list_remove(dsp, &runtime->block_list);  in sst_module_runtime_free_blocks()
918 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_free_blocks()
926 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_save()  local
930 	dev_dbg(dsp->dev, "saving runtime %d memory at 0x%x size 0x%x\n",  in sst_module_runtime_save()
934 	context->buffer = dma_alloc_coherent(dsp->dma_dev,  in sst_module_runtime_save()
938 		dev_err(dsp->dev, "error: DMA context alloc failed\n");  in sst_module_runtime_save()
942 	mutex_lock(&dsp->mutex);  in sst_module_runtime_save()
944 	if (dsp->fw_use_dma) {  in sst_module_runtime_save()
946 		ret = sst_dsp_dma_get_channel(dsp, 0);  in sst_module_runtime_save()
950 		ret = sst_dsp_dma_copyfrom(dsp, context->dma_buffer,  in sst_module_runtime_save()
951 			dsp->addr.lpe_base + runtime->persistent_offset,  in sst_module_runtime_save()
953 		sst_dsp_dma_put_channel(dsp);  in sst_module_runtime_save()
955 			dev_err(dsp->dev, "error: context copy failed\n");  in sst_module_runtime_save()
959 		sst_memcpy32(context->buffer, dsp->addr.lpe +  in sst_module_runtime_save()
964 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_save()
972 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_restore()  local
976 	dev_dbg(dsp->dev, "restoring runtime %d memory at 0x%x size 0x%x\n",  in sst_module_runtime_restore()
980 	mutex_lock(&dsp->mutex);  in sst_module_runtime_restore()
983 		dev_info(dsp->dev, "no context buffer need to restore!\n");  in sst_module_runtime_restore()
987 	if (dsp->fw_use_dma) {  in sst_module_runtime_restore()
989 		ret = sst_dsp_dma_get_channel(dsp, 0);  in sst_module_runtime_restore()
993 		ret = sst_dsp_dma_copyto(dsp,  in sst_module_runtime_restore()
994 			dsp->addr.lpe_base + runtime->persistent_offset,  in sst_module_runtime_restore()
996 		sst_dsp_dma_put_channel(dsp);  in sst_module_runtime_restore()
998 			dev_err(dsp->dev, "error: module copy failed\n");  in sst_module_runtime_restore()
1002 		sst_memcpy32(dsp->addr.lpe + runtime->persistent_offset,  in sst_module_runtime_restore()
1005 	dma_free_coherent(dsp->dma_dev, module->persistent_size,  in sst_module_runtime_restore()
1010 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_restore()
1016 struct sst_mem_block *sst_mem_block_register(struct sst_dsp *dsp, u32 offset,  in sst_mem_block_register()  argument
1030 	block->dsp = dsp;  in sst_mem_block_register()
1034 	mutex_lock(&dsp->mutex);  in sst_mem_block_register()
1035 	list_add(&block->list, &dsp->free_block_list);  in sst_mem_block_register()
1036 	mutex_unlock(&dsp->mutex);  in sst_mem_block_register()
1043 void sst_mem_block_unregister_all(struct sst_dsp *dsp)  in sst_mem_block_unregister_all()  argument
1047 	mutex_lock(&dsp->mutex);  in sst_mem_block_unregister_all()
1050 	list_for_each_entry_safe(block, tmp, &dsp->used_block_list, list) {  in sst_mem_block_unregister_all()
1056 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in sst_mem_block_unregister_all()
1061 	mutex_unlock(&dsp->mutex);  in sst_mem_block_unregister_all()
1066 int sst_block_alloc_scratch(struct sst_dsp *dsp)  in sst_block_alloc_scratch()  argument
1072 	mutex_lock(&dsp->mutex);  in sst_block_alloc_scratch()
1075 	dsp->scratch_size = 0;  in sst_block_alloc_scratch()
1076 	list_for_each_entry(module, &dsp->module_list, list) {  in sst_block_alloc_scratch()
1077 		dev_dbg(dsp->dev, "module %d scratch req 0x%x bytes\n",  in sst_block_alloc_scratch()
1079 		if (dsp->scratch_size < module->scratch_size)  in sst_block_alloc_scratch()
1080 			dsp->scratch_size = module->scratch_size;  in sst_block_alloc_scratch()
1083 	dev_dbg(dsp->dev, "scratch buffer required is 0x%x bytes\n",  in sst_block_alloc_scratch()
1084 		dsp->scratch_size);  in sst_block_alloc_scratch()
1086 	if (dsp->scratch_size == 0) {  in sst_block_alloc_scratch()
1087 		dev_info(dsp->dev, "no modules need scratch buffer\n");  in sst_block_alloc_scratch()
1088 		mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1093 	dev_dbg(dsp->dev, "allocating scratch blocks\n");  in sst_block_alloc_scratch()
1095 	ba.size = dsp->scratch_size;  in sst_block_alloc_scratch()
1099 	if (dsp->scratch_offset != 0) {  in sst_block_alloc_scratch()
1101 		dev_dbg(dsp->dev, "block request 0x%x bytes type %d at 0x%x\n",  in sst_block_alloc_scratch()
1104 		ba.offset = dsp->scratch_offset;  in sst_block_alloc_scratch()
1107 		ret = block_alloc_fixed(dsp, &ba, &dsp->scratch_block_list);  in sst_block_alloc_scratch()
1110 		dev_dbg(dsp->dev, "block request 0x%x bytes type %d\n",  in sst_block_alloc_scratch()
1114 		ret = block_alloc(dsp, &ba, &dsp->scratch_block_list);  in sst_block_alloc_scratch()
1117 		dev_err(dsp->dev, "error: can't alloc scratch blocks\n");  in sst_block_alloc_scratch()
1118 		mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1122 	ret = block_list_prepare(dsp, &dsp->scratch_block_list);  in sst_block_alloc_scratch()
1124 		dev_err(dsp->dev, "error: scratch block prepare failed\n");  in sst_block_alloc_scratch()
1125 		mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1130 	dsp->scratch_offset = ba.offset;  in sst_block_alloc_scratch()
1131 	mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1132 	return dsp->scratch_size;  in sst_block_alloc_scratch()
1137 void sst_block_free_scratch(struct sst_dsp *dsp)  in sst_block_free_scratch()  argument
1139 	mutex_lock(&dsp->mutex);  in sst_block_free_scratch()
1140 	block_list_remove(dsp, &dsp->scratch_block_list);  in sst_block_free_scratch()
1141 	mutex_unlock(&dsp->mutex);  in sst_block_free_scratch()
1146 struct sst_module *sst_module_get_from_id(struct sst_dsp *dsp, u32 id)  in sst_module_get_from_id()  argument
1150 	mutex_lock(&dsp->mutex);  in sst_module_get_from_id()
1152 	list_for_each_entry(module, &dsp->module_list, list) {  in sst_module_get_from_id()
1154 			mutex_unlock(&dsp->mutex);  in sst_module_get_from_id()
1159 	mutex_unlock(&dsp->mutex);  in sst_module_get_from_id()
1168 	struct sst_dsp *dsp = module->dsp;  in sst_module_runtime_get_from_id()  local
1170 	mutex_lock(&dsp->mutex);  in sst_module_runtime_get_from_id()
1174 			mutex_unlock(&dsp->mutex);  in sst_module_runtime_get_from_id()
1179 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_get_from_id()
1185 u32 sst_dsp_get_offset(struct sst_dsp *dsp, u32 offset,  in sst_dsp_get_offset()  argument
1190 		return offset - dsp->addr.iram_offset +  in sst_dsp_get_offset()
1191 			dsp->addr.dsp_iram_offset;  in sst_dsp_get_offset()
1193 		return offset - dsp->addr.dram_offset +  in sst_dsp_get_offset()
1194 			dsp->addr.dsp_dram_offset;  in sst_dsp_get_offset()