115 static void block_list_remove(struct sst_dsp *dsp,  in block_list_remove()  argument
127 				dev_err(dsp->dev,  in block_list_remove()
136 		list_move(&block->list, &dsp->free_block_list);  in block_list_remove()
137 		dev_dbg(dsp->dev, "block freed %d:%d at offset 0x%x\n",  in block_list_remove()
143 static int block_list_prepare(struct sst_dsp *dsp,  in block_list_prepare()  argument
155 				dev_err(dsp->dev,  in block_list_prepare()
212 	struct sst_dsp *dsp = (struct sst_dsp *)param;  in dma_chan_filter()  local
214 	return chan->device->dev == dsp->dma_dev;  in dma_chan_filter()
217 int sst_dsp_dma_get_channel(struct sst_dsp *dsp, int chan_id)  in sst_dsp_dma_get_channel()  argument
219 	struct sst_dma *dma = dsp->dma;  in sst_dsp_dma_get_channel()
228 	dma->ch = dma_request_channel(mask, dma_chan_filter, dsp);  in sst_dsp_dma_get_channel()
230 		dev_err(dsp->dev, "error: DMA request channel failed\n");  in sst_dsp_dma_get_channel()
242 		dev_err(dsp->dev, "error: unable to set DMA slave config %d\n",  in sst_dsp_dma_get_channel()
252 void sst_dsp_dma_put_channel(struct sst_dsp *dsp)  in sst_dsp_dma_put_channel()  argument
254 	struct sst_dma *dma = dsp->dma;  in sst_dsp_dma_put_channel()
334 struct sst_fw *sst_fw_new(struct sst_dsp *dsp,   in sst_fw_new()  argument
340 	if (!dsp->ops->parse_fw)  in sst_fw_new()
347 	sst_fw->dsp = dsp;  in sst_fw_new()
352 	sst_fw->dma_buf = dma_alloc_coherent(dsp->dma_dev, sst_fw->size,  in sst_fw_new()
355 		dev_err(dsp->dev, "error: DMA alloc failed\n");  in sst_fw_new()
363 	if (dsp->fw_use_dma) {  in sst_fw_new()
364 		err = sst_dsp_dma_get_channel(dsp, 0);  in sst_fw_new()
370 	err = dsp->ops->parse_fw(sst_fw);  in sst_fw_new()
372 		dev_err(dsp->dev, "error: parse fw failed %d\n", err);  in sst_fw_new()
376 	if (dsp->fw_use_dma)  in sst_fw_new()
377 		sst_dsp_dma_put_channel(dsp);  in sst_fw_new()
379 	mutex_lock(&dsp->mutex);  in sst_fw_new()
380 	list_add(&sst_fw->list, &dsp->fw_list);  in sst_fw_new()
381 	mutex_unlock(&dsp->mutex);  in sst_fw_new()
386 	if (dsp->fw_use_dma)  in sst_fw_new()
387 		sst_dsp_dma_put_channel(dsp);  in sst_fw_new()
389 	dma_free_coherent(dsp->dma_dev, sst_fw->size,  in sst_fw_new()
400 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_fw_reload()  local
403 	dev_dbg(dsp->dev, "reloading firmware\n");  in sst_fw_reload()
406 	ret = dsp->ops->parse_fw(sst_fw);  in sst_fw_reload()
408 		dev_err(dsp->dev, "error: parse fw failed %d\n", ret);  in sst_fw_reload()
416 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_fw_unload()  local
420 	dev_dbg(dsp->dev, "unloading firmware\n");  in sst_fw_unload()
422 	mutex_lock(&dsp->mutex);  in sst_fw_unload()
425 	list_for_each_entry_safe(module, mtmp, &dsp->module_list, list) {  in sst_fw_unload()
431 				block_list_remove(dsp, &runtime->block_list);  in sst_fw_unload()
437 			block_list_remove(dsp, &module->block_list);  in sst_fw_unload()
444 	block_list_remove(dsp, &dsp->scratch_block_list);  in sst_fw_unload()
446 	mutex_unlock(&dsp->mutex);  in sst_fw_unload()
453 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_fw_free()  local
455 	mutex_lock(&dsp->mutex);  in sst_fw_free()
457 	mutex_unlock(&dsp->mutex);  in sst_fw_free()
460 		dma_free_coherent(dsp->dma_dev, sst_fw->size, sst_fw->dma_buf,  in sst_fw_free()
467 void sst_fw_free_all(struct sst_dsp *dsp)  in sst_fw_free_all()  argument
471 	mutex_lock(&dsp->mutex);  in sst_fw_free_all()
472 	list_for_each_entry_safe(sst_fw, t, &dsp->fw_list, list) {  in sst_fw_free_all()
475 		dma_free_coherent(dsp->dev, sst_fw->size, sst_fw->dma_buf,  in sst_fw_free_all()
479 	mutex_unlock(&dsp->mutex);  in sst_fw_free_all()
487 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_module_new()  local
495 	sst_module->dsp = dsp;  in sst_module_new()
505 	mutex_lock(&dsp->mutex);  in sst_module_new()
506 	list_add(&sst_module->list, &dsp->module_list);  in sst_module_new()
507 	mutex_unlock(&dsp->mutex);  in sst_module_new()
516 	struct sst_dsp *dsp = sst_module->dsp;  in sst_module_free()  local
518 	mutex_lock(&dsp->mutex);  in sst_module_free()
520 	mutex_unlock(&dsp->mutex);  in sst_module_free()
529 	struct sst_dsp *dsp = module->dsp;  in sst_module_runtime_new()  local
537 	runtime->dsp = dsp;  in sst_module_runtime_new()
541 	mutex_lock(&dsp->mutex);  in sst_module_runtime_new()
543 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_new()
551 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_free()  local
553 	mutex_lock(&dsp->mutex);  in sst_module_runtime_free()
555 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_free()
561 static struct sst_mem_block *find_block(struct sst_dsp *dsp,  in find_block()  argument
566 	list_for_each_entry(block, &dsp->free_block_list, list) {  in find_block()
575 static int block_alloc_contiguous(struct sst_dsp *dsp,  in block_alloc_contiguous()  argument
585 		block = find_block(dsp, ba);  in block_alloc_contiguous()
587 			list_splice(&tmp, &dsp->free_block_list);  in block_alloc_contiguous()
608 		dev_dbg(dsp->dev, "block allocated %d:%d at offset 0x%x\n",  in block_alloc_contiguous()
612 	list_splice(&tmp, &dsp->used_block_list);  in block_alloc_contiguous()
617 static int block_alloc(struct sst_dsp *dsp, struct sst_block_allocator *ba,  in block_alloc()  argument
627 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in block_alloc()
639 		list_move(&block->list, &dsp->used_block_list);  in block_alloc()
640 		dev_dbg(dsp->dev, "block allocated %d:%d at offset 0x%x\n",  in block_alloc()
646 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in block_alloc()
658 			ret = block_alloc_contiguous(dsp, ba, block_list);  in block_alloc()
669 int sst_alloc_blocks(struct sst_dsp *dsp, struct sst_block_allocator *ba,  in sst_alloc_blocks()  argument
674 	dev_dbg(dsp->dev, "block request 0x%x bytes at offset 0x%x type %d\n",  in sst_alloc_blocks()
677 	mutex_lock(&dsp->mutex);  in sst_alloc_blocks()
679 	ret = block_alloc(dsp, ba, block_list);  in sst_alloc_blocks()
681 		dev_err(dsp->dev, "error: can't alloc blocks %d\n", ret);  in sst_alloc_blocks()
686 	ret = block_list_prepare(dsp, block_list);  in sst_alloc_blocks()
688 		dev_err(dsp->dev, "error: prepare failed\n");  in sst_alloc_blocks()
691 	mutex_unlock(&dsp->mutex);  in sst_alloc_blocks()
696 int sst_free_blocks(struct sst_dsp *dsp, struct list_head *block_list)  in sst_free_blocks()  argument
698 	mutex_lock(&dsp->mutex);  in sst_free_blocks()
699 	block_list_remove(dsp, block_list);  in sst_free_blocks()
700 	mutex_unlock(&dsp->mutex);  in sst_free_blocks()
706 static int block_alloc_fixed(struct sst_dsp *dsp, struct sst_block_allocator *ba,  in block_alloc_fixed()  argument
737 			err = block_alloc_contiguous(dsp, &ba_tmp, block_list);  in block_alloc_fixed()
747 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in block_alloc_fixed()
758 			list_move(&block->list, &dsp->used_block_list);  in block_alloc_fixed()
760 			dev_dbg(dsp->dev, "block allocated %d:%d at offset 0x%x\n",  in block_alloc_fixed()
769 			list_move(&block->list, &dsp->used_block_list);  in block_alloc_fixed()
775 			err = block_alloc_contiguous(dsp, &ba_tmp, block_list);  in block_alloc_fixed()
789 	struct sst_dsp *dsp = module->dsp;  in sst_module_alloc_blocks()  local
799 	dev_dbg(dsp->dev, "block request 0x%x bytes at offset 0x%x type %d\n",  in sst_module_alloc_blocks()
802 	mutex_lock(&dsp->mutex);  in sst_module_alloc_blocks()
805 	ret = block_alloc_fixed(dsp, &ba, &module->block_list);  in sst_module_alloc_blocks()
807 		dev_err(dsp->dev,  in sst_module_alloc_blocks()
810 		mutex_unlock(&dsp->mutex);  in sst_module_alloc_blocks()
815 	ret = block_list_prepare(dsp, &module->block_list);  in sst_module_alloc_blocks()
817 		dev_err(dsp->dev, "error: fw module prepare failed\n");  in sst_module_alloc_blocks()
822 	if (dsp->fw_use_dma) {  in sst_module_alloc_blocks()
823 		ret = sst_dsp_dma_copyto(dsp,  in sst_module_alloc_blocks()
824 			dsp->addr.lpe_base + module->offset,  in sst_module_alloc_blocks()
828 			dev_err(dsp->dev, "error: module copy failed\n");  in sst_module_alloc_blocks()
832 		sst_memcpy32(dsp->addr.lpe + module->offset, module->data,  in sst_module_alloc_blocks()
835 	mutex_unlock(&dsp->mutex);  in sst_module_alloc_blocks()
839 	block_list_remove(dsp, &module->block_list);  in sst_module_alloc_blocks()
840 	mutex_unlock(&dsp->mutex);  in sst_module_alloc_blocks()
848 	struct sst_dsp *dsp = module->dsp;  in sst_module_free_blocks()  local
850 	mutex_lock(&dsp->mutex);  in sst_module_free_blocks()
851 	block_list_remove(dsp, &module->block_list);  in sst_module_free_blocks()
852 	mutex_unlock(&dsp->mutex);  in sst_module_free_blocks()
860 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_alloc_blocks()  local
872 	mutex_lock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
879 		dev_dbg(dsp->dev, "persistent fixed block request 0x%x bytes type %d offset 0x%x\n",  in sst_module_runtime_alloc_blocks()
883 		ret = block_alloc_fixed(dsp, &ba, &runtime->block_list);  in sst_module_runtime_alloc_blocks()
886 		dev_dbg(dsp->dev, "persistent block request 0x%x bytes type %d\n",  in sst_module_runtime_alloc_blocks()
890 		ret = block_alloc(dsp, &ba, &runtime->block_list);  in sst_module_runtime_alloc_blocks()
893 		dev_err(dsp->dev,  in sst_module_runtime_alloc_blocks()
896 		mutex_unlock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
902 	ret = block_list_prepare(dsp, &runtime->block_list);  in sst_module_runtime_alloc_blocks()
904 		dev_err(dsp->dev, "error: runtime block prepare failed\n");  in sst_module_runtime_alloc_blocks()
908 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
912 	block_list_remove(dsp, &module->block_list);  in sst_module_runtime_alloc_blocks()
913 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
920 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_free_blocks()  local
922 	mutex_lock(&dsp->mutex);  in sst_module_runtime_free_blocks()
923 	block_list_remove(dsp, &runtime->block_list);  in sst_module_runtime_free_blocks()
924 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_free_blocks()
932 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_save()  local
936 	dev_dbg(dsp->dev, "saving runtime %d memory at 0x%x size 0x%x\n",  in sst_module_runtime_save()
940 	context->buffer = dma_alloc_coherent(dsp->dma_dev,  in sst_module_runtime_save()
944 		dev_err(dsp->dev, "error: DMA context alloc failed\n");  in sst_module_runtime_save()
948 	mutex_lock(&dsp->mutex);  in sst_module_runtime_save()
950 	if (dsp->fw_use_dma) {  in sst_module_runtime_save()
952 		ret = sst_dsp_dma_get_channel(dsp, 0);  in sst_module_runtime_save()
956 		ret = sst_dsp_dma_copyfrom(dsp, context->dma_buffer,  in sst_module_runtime_save()
957 			dsp->addr.lpe_base + runtime->persistent_offset,  in sst_module_runtime_save()
959 		sst_dsp_dma_put_channel(dsp);  in sst_module_runtime_save()
961 			dev_err(dsp->dev, "error: context copy failed\n");  in sst_module_runtime_save()
965 		sst_memcpy32(context->buffer, dsp->addr.lpe +  in sst_module_runtime_save()
970 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_save()
978 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_restore()  local
982 	dev_dbg(dsp->dev, "restoring runtime %d memory at 0x%x size 0x%x\n",  in sst_module_runtime_restore()
986 	mutex_lock(&dsp->mutex);  in sst_module_runtime_restore()
989 		dev_info(dsp->dev, "no context buffer need to restore!\n");  in sst_module_runtime_restore()
993 	if (dsp->fw_use_dma) {  in sst_module_runtime_restore()
995 		ret = sst_dsp_dma_get_channel(dsp, 0);  in sst_module_runtime_restore()
999 		ret = sst_dsp_dma_copyto(dsp,  in sst_module_runtime_restore()
1000 			dsp->addr.lpe_base + runtime->persistent_offset,  in sst_module_runtime_restore()
1002 		sst_dsp_dma_put_channel(dsp);  in sst_module_runtime_restore()
1004 			dev_err(dsp->dev, "error: module copy failed\n");  in sst_module_runtime_restore()
1008 		sst_memcpy32(dsp->addr.lpe + runtime->persistent_offset,  in sst_module_runtime_restore()
1011 	dma_free_coherent(dsp->dma_dev, module->persistent_size,  in sst_module_runtime_restore()
1016 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_restore()
1022 struct sst_mem_block *sst_mem_block_register(struct sst_dsp *dsp, u32 offset,  in sst_mem_block_register()  argument
1036 	block->dsp = dsp;  in sst_mem_block_register()
1040 	mutex_lock(&dsp->mutex);  in sst_mem_block_register()
1041 	list_add(&block->list, &dsp->free_block_list);  in sst_mem_block_register()
1042 	mutex_unlock(&dsp->mutex);  in sst_mem_block_register()
1049 void sst_mem_block_unregister_all(struct sst_dsp *dsp)  in sst_mem_block_unregister_all()  argument
1053 	mutex_lock(&dsp->mutex);  in sst_mem_block_unregister_all()
1056 	list_for_each_entry_safe(block, tmp, &dsp->used_block_list, list) {  in sst_mem_block_unregister_all()
1062 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in sst_mem_block_unregister_all()
1067 	mutex_unlock(&dsp->mutex);  in sst_mem_block_unregister_all()
1072 int sst_block_alloc_scratch(struct sst_dsp *dsp)  in sst_block_alloc_scratch()  argument
1078 	mutex_lock(&dsp->mutex);  in sst_block_alloc_scratch()
1081 	dsp->scratch_size = 0;  in sst_block_alloc_scratch()
1082 	list_for_each_entry(module, &dsp->module_list, list) {  in sst_block_alloc_scratch()
1083 		dev_dbg(dsp->dev, "module %d scratch req 0x%x bytes\n",  in sst_block_alloc_scratch()
1085 		if (dsp->scratch_size < module->scratch_size)  in sst_block_alloc_scratch()
1086 			dsp->scratch_size = module->scratch_size;  in sst_block_alloc_scratch()
1089 	dev_dbg(dsp->dev, "scratch buffer required is 0x%x bytes\n",  in sst_block_alloc_scratch()
1090 		dsp->scratch_size);  in sst_block_alloc_scratch()
1092 	if (dsp->scratch_size == 0) {  in sst_block_alloc_scratch()
1093 		dev_info(dsp->dev, "no modules need scratch buffer\n");  in sst_block_alloc_scratch()
1094 		mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1099 	dev_dbg(dsp->dev, "allocating scratch blocks\n");  in sst_block_alloc_scratch()
1101 	ba.size = dsp->scratch_size;  in sst_block_alloc_scratch()
1105 	if (dsp->scratch_offset != 0) {  in sst_block_alloc_scratch()
1107 		dev_dbg(dsp->dev, "block request 0x%x bytes type %d at 0x%x\n",  in sst_block_alloc_scratch()
1110 		ba.offset = dsp->scratch_offset;  in sst_block_alloc_scratch()
1113 		ret = block_alloc_fixed(dsp, &ba, &dsp->scratch_block_list);  in sst_block_alloc_scratch()
1116 		dev_dbg(dsp->dev, "block request 0x%x bytes type %d\n",  in sst_block_alloc_scratch()
1120 		ret = block_alloc(dsp, &ba, &dsp->scratch_block_list);  in sst_block_alloc_scratch()
1123 		dev_err(dsp->dev, "error: can't alloc scratch blocks\n");  in sst_block_alloc_scratch()
1124 		mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1128 	ret = block_list_prepare(dsp, &dsp->scratch_block_list);  in sst_block_alloc_scratch()
1130 		dev_err(dsp->dev, "error: scratch block prepare failed\n");  in sst_block_alloc_scratch()
1131 		mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1136 	dsp->scratch_offset = ba.offset;  in sst_block_alloc_scratch()
1137 	mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1138 	return dsp->scratch_size;  in sst_block_alloc_scratch()
1143 void sst_block_free_scratch(struct sst_dsp *dsp)  in sst_block_free_scratch()  argument
1145 	mutex_lock(&dsp->mutex);  in sst_block_free_scratch()
1146 	block_list_remove(dsp, &dsp->scratch_block_list);  in sst_block_free_scratch()
1147 	mutex_unlock(&dsp->mutex);  in sst_block_free_scratch()
1152 struct sst_module *sst_module_get_from_id(struct sst_dsp *dsp, u32 id)  in sst_module_get_from_id()  argument
1156 	mutex_lock(&dsp->mutex);  in sst_module_get_from_id()
1158 	list_for_each_entry(module, &dsp->module_list, list) {  in sst_module_get_from_id()
1160 			mutex_unlock(&dsp->mutex);  in sst_module_get_from_id()
1165 	mutex_unlock(&dsp->mutex);  in sst_module_get_from_id()
1174 	struct sst_dsp *dsp = module->dsp;  in sst_module_runtime_get_from_id()  local
1176 	mutex_lock(&dsp->mutex);  in sst_module_runtime_get_from_id()
1180 			mutex_unlock(&dsp->mutex);  in sst_module_runtime_get_from_id()
1185 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_get_from_id()
1191 u32 sst_dsp_get_offset(struct sst_dsp *dsp, u32 offset,  in sst_dsp_get_offset()  argument
1196 		return offset - dsp->addr.iram_offset +  in sst_dsp_get_offset()
1197 			dsp->addr.dsp_iram_offset;  in sst_dsp_get_offset()
1199 		return offset - dsp->addr.dram_offset +  in sst_dsp_get_offset()
1200 			dsp->addr.dsp_dram_offset;  in sst_dsp_get_offset()