67 static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t iova)  in get_phys_addr()  argument
69 	u32 win_cnt = dma_domain->win_cnt;  in get_phys_addr()
70 	struct dma_window *win_ptr = &dma_domain->win_arr[0];  in get_phys_addr()
73 	geom = &dma_domain->iommu_domain.geometry;  in get_phys_addr()
75 	if (!win_cnt || !dma_domain->geom_size) {  in get_phys_addr()
85 		subwin_size = dma_domain->geom_size >> ilog2(win_cnt);  in get_phys_addr()
88 		win_ptr = &dma_domain->win_arr[wnd];  in get_phys_addr()
97 static int map_subwins(int liodn, struct fsl_dma_domain *dma_domain)  in map_subwins()  argument
99 	struct dma_window *sub_win_ptr = &dma_domain->win_arr[0];  in map_subwins()
103 	for (i = 0; i < dma_domain->win_cnt; i++) {  in map_subwins()
107 			ret = pamu_config_spaace(liodn, dma_domain->win_cnt, i,  in map_subwins()
111 						 dma_domain->snoop_id,  in map_subwins()
112 						 dma_domain->stash_id,  in map_subwins()
127 static int map_win(int liodn, struct fsl_dma_domain *dma_domain)  in map_win()  argument
130 	struct dma_window *wnd = &dma_domain->win_arr[0];  in map_win()
131 	phys_addr_t wnd_addr = dma_domain->iommu_domain.geometry.aperture_start;  in map_win()
139 				 dma_domain->snoop_id, dma_domain->stash_id,  in map_win()
149 static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain)  in map_liodn()  argument
151 	if (dma_domain->win_cnt > 1)  in map_liodn()
152 		return map_subwins(liodn, dma_domain);  in map_liodn()
154 		return map_win(liodn, dma_domain);  in map_liodn()
158 static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr)  in update_liodn()  argument
161 	struct dma_window *wnd = &dma_domain->win_arr[wnd_nr];  in update_liodn()
165 	if (dma_domain->win_cnt > 1) {  in update_liodn()
166 		ret = pamu_config_spaace(liodn, dma_domain->win_cnt, wnd_nr,  in update_liodn()
170 					 dma_domain->snoop_id,  in update_liodn()
171 					 dma_domain->stash_id,  in update_liodn()
180 		wnd_addr = dma_domain->iommu_domain.geometry.aperture_start;  in update_liodn()
186 					 dma_domain->snoop_id, dma_domain->stash_id,  in update_liodn()
198 static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain,  in update_liodn_stash()  argument
205 	if (!dma_domain->win_arr) {  in update_liodn_stash()
212 	for (i = 0; i < dma_domain->win_cnt; i++) {  in update_liodn_stash()
229 			  struct fsl_dma_domain *dma_domain,  in pamu_set_liodn()  argument
247 	window_size = dma_domain->geom_size;  in pamu_set_liodn()
253 					 0, dma_domain->snoop_id,  in pamu_set_liodn()
254 					 dma_domain->stash_id, win_cnt, 0);  in pamu_set_liodn()
270 							 0, dma_domain->snoop_id,  in pamu_set_liodn()
271 							 dma_domain->stash_id,  in pamu_set_liodn()
341 static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain)  in detach_device()  argument
346 	spin_lock_irqsave(&dma_domain->domain_lock, flags);  in detach_device()
348 	list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) {  in detach_device()
350 			remove_device_ref(info, dma_domain->win_cnt);  in detach_device()
352 	spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in detach_device()
355 static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev)  in attach_device()  argument
366 	if (old_domain_info && old_domain_info->domain != dma_domain) {  in attach_device()
376 	info->domain = dma_domain;  in attach_device()
378 	list_add(&info->link, &dma_domain->devices);  in attach_device()
392 	struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);  in fsl_pamu_iova_to_phys()  local
398 	return get_phys_addr(dma_domain, iova);  in fsl_pamu_iova_to_phys()
408 	struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);  in fsl_pamu_domain_free()  local
411 	detach_device(NULL, dma_domain);  in fsl_pamu_domain_free()
413 	dma_domain->enabled = 0;  in fsl_pamu_domain_free()
414 	dma_domain->mapped = 0;  in fsl_pamu_domain_free()
416 	kmem_cache_free(fsl_pamu_domain_cache, dma_domain);  in fsl_pamu_domain_free()
421 	struct fsl_dma_domain *dma_domain;  in fsl_pamu_domain_alloc()  local
426 	dma_domain = iommu_alloc_dma_domain();  in fsl_pamu_domain_alloc()
427 	if (!dma_domain) {  in fsl_pamu_domain_alloc()
432 	dma_domain->iommu_domain. geometry.aperture_start = 0;  in fsl_pamu_domain_alloc()
433 	dma_domain->iommu_domain.geometry.aperture_end = (1ULL << 36) - 1;  in fsl_pamu_domain_alloc()
434 	dma_domain->iommu_domain.geometry.force_aperture = true;  in fsl_pamu_domain_alloc()
436 	return &dma_domain->iommu_domain;  in fsl_pamu_domain_alloc()
440 static int pamu_set_domain_geometry(struct fsl_dma_domain *dma_domain,  in pamu_set_domain_geometry()  argument
447 	list_for_each_entry(info, &dma_domain->devices, link) {  in pamu_set_domain_geometry()
448 		ret = pamu_set_liodn(info->liodn, info->dev, dma_domain,  in pamu_set_domain_geometry()
458 static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val)  in update_domain_stash()  argument
463 	list_for_each_entry(info, &dma_domain->devices, link) {  in update_domain_stash()
464 		ret = update_liodn_stash(info->liodn, dma_domain, val);  in update_domain_stash()
473 static int update_domain_mapping(struct fsl_dma_domain *dma_domain, u32 wnd_nr)  in update_domain_mapping()  argument
478 	list_for_each_entry(info, &dma_domain->devices, link) {  in update_domain_mapping()
479 		ret = update_liodn(info->liodn, dma_domain, wnd_nr);  in update_domain_mapping()
486 static int disable_domain_win(struct fsl_dma_domain *dma_domain, u32 wnd_nr)  in disable_domain_win()  argument
491 	list_for_each_entry(info, &dma_domain->devices, link) {  in disable_domain_win()
492 		if (dma_domain->win_cnt == 1 && dma_domain->enabled) {  in disable_domain_win()
495 				dma_domain->enabled = 0;  in disable_domain_win()
506 	struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);  in fsl_pamu_window_disable()  local
510 	spin_lock_irqsave(&dma_domain->domain_lock, flags);  in fsl_pamu_window_disable()
511 	if (!dma_domain->win_arr) {  in fsl_pamu_window_disable()
513 		spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in fsl_pamu_window_disable()
517 	if (wnd_nr >= dma_domain->win_cnt) {  in fsl_pamu_window_disable()
519 		spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in fsl_pamu_window_disable()
523 	if (dma_domain->win_arr[wnd_nr].valid) {  in fsl_pamu_window_disable()
524 		ret = disable_domain_win(dma_domain, wnd_nr);  in fsl_pamu_window_disable()
526 			dma_domain->win_arr[wnd_nr].valid = 0;  in fsl_pamu_window_disable()
527 			dma_domain->mapped--;  in fsl_pamu_window_disable()
531 	spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in fsl_pamu_window_disable()
537 	struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);  in fsl_pamu_window_enable()  local
549 	spin_lock_irqsave(&dma_domain->domain_lock, flags);  in fsl_pamu_window_enable()
550 	if (!dma_domain->win_arr) {  in fsl_pamu_window_enable()
552 		spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in fsl_pamu_window_enable()
556 	if (wnd_nr >= dma_domain->win_cnt) {  in fsl_pamu_window_enable()
558 		spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in fsl_pamu_window_enable()
562 	win_size = dma_domain->geom_size >> ilog2(dma_domain->win_cnt);  in fsl_pamu_window_enable()
565 		spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in fsl_pamu_window_enable()
569 	if (dma_domain->win_cnt == 1) {  in fsl_pamu_window_enable()
570 		if (dma_domain->enabled) {  in fsl_pamu_window_enable()
572 			spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in fsl_pamu_window_enable()
579 			spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in fsl_pamu_window_enable()
584 	wnd = &dma_domain->win_arr[wnd_nr];  in fsl_pamu_window_enable()
590 		ret = update_domain_mapping(dma_domain, wnd_nr);  in fsl_pamu_window_enable()
593 			dma_domain->mapped++;  in fsl_pamu_window_enable()
600 	spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in fsl_pamu_window_enable()
609 static int handle_attach_device(struct fsl_dma_domain *dma_domain,  in handle_attach_device()  argument
614 	struct iommu_domain *domain = &dma_domain->iommu_domain;  in handle_attach_device()
618 	spin_lock_irqsave(&dma_domain->domain_lock, flags);  in handle_attach_device()
628 		attach_device(dma_domain, liodn[i], dev);  in handle_attach_device()
634 		if (dma_domain->win_arr) {  in handle_attach_device()
635 			u32 win_cnt = dma_domain->win_cnt > 1 ? dma_domain->win_cnt : 0;  in handle_attach_device()
637 			ret = pamu_set_liodn(liodn[i], dev, dma_domain,  in handle_attach_device()
641 			if (dma_domain->mapped) {  in handle_attach_device()
646 				ret = map_liodn(liodn[i], dma_domain);  in handle_attach_device()
652 	spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in handle_attach_device()
660 	struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);  in fsl_pamu_attach_device()  local
685 		ret = handle_attach_device(dma_domain, dev, liodn, liodn_cnt);  in fsl_pamu_attach_device()
698 	struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);  in fsl_pamu_detach_device()  local
721 		detach_device(dev, dma_domain);  in fsl_pamu_detach_device()
730 	struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);  in configure_domain_geometry()  local
745 	spin_lock_irqsave(&dma_domain->domain_lock, flags);  in configure_domain_geometry()
746 	if (dma_domain->enabled) {  in configure_domain_geometry()
748 		spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in configure_domain_geometry()
755 	dma_domain->geom_size = geom_size;  in configure_domain_geometry()
757 	spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in configure_domain_geometry()
763 static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data)  in configure_domain_stash()  argument
769 	spin_lock_irqsave(&dma_domain->domain_lock, flags);  in configure_domain_stash()
771 	memcpy(&dma_domain->dma_stash, stash_attr,  in configure_domain_stash()
774 	dma_domain->stash_id = get_stash_id(stash_attr->cache,  in configure_domain_stash()
776 	if (dma_domain->stash_id == ~(u32)0) {  in configure_domain_stash()
778 		spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in configure_domain_stash()
782 	ret = update_domain_stash(dma_domain, dma_domain->stash_id);  in configure_domain_stash()
784 	spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in configure_domain_stash()
790 static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool enable)  in configure_domain_dma_state()  argument
796 	spin_lock_irqsave(&dma_domain->domain_lock, flags);  in configure_domain_dma_state()
798 	if (enable && !dma_domain->mapped) {  in configure_domain_dma_state()
800 		spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in configure_domain_dma_state()
804 	dma_domain->enabled = enable;  in configure_domain_dma_state()
805 	list_for_each_entry(info, &dma_domain->devices, link) {  in configure_domain_dma_state()
812 	spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in configure_domain_dma_state()
820 	struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);  in fsl_pamu_set_domain_attr()  local
828 		ret = configure_domain_stash(dma_domain, data);  in fsl_pamu_set_domain_attr()
831 		ret = configure_domain_dma_state(dma_domain, *(int *)data);  in fsl_pamu_set_domain_attr()
845 	struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);  in fsl_pamu_get_domain_attr()  local
850 		memcpy(data, &dma_domain->dma_stash,  in fsl_pamu_get_domain_attr()
854 		*(int *)data = dma_domain->enabled;  in fsl_pamu_get_domain_attr()
996 	struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);  in fsl_pamu_set_windows()  local
1000 	spin_lock_irqsave(&dma_domain->domain_lock, flags);  in fsl_pamu_set_windows()
1002 	if (dma_domain->enabled) {  in fsl_pamu_set_windows()
1004 		spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in fsl_pamu_set_windows()
1009 	if (!dma_domain->geom_size) {  in fsl_pamu_set_windows()
1011 		spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in fsl_pamu_set_windows()
1021 		spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in fsl_pamu_set_windows()
1025 	ret = pamu_set_domain_geometry(dma_domain, &domain->geometry,  in fsl_pamu_set_windows()
1028 		kfree(dma_domain->win_arr);  in fsl_pamu_set_windows()
1029 		dma_domain->win_arr = kcalloc(w_count,  in fsl_pamu_set_windows()
1030 					      sizeof(*dma_domain->win_arr),  in fsl_pamu_set_windows()
1032 		if (!dma_domain->win_arr) {  in fsl_pamu_set_windows()
1033 			spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in fsl_pamu_set_windows()
1036 		dma_domain->win_cnt = w_count;  in fsl_pamu_set_windows()
1038 	spin_unlock_irqrestore(&dma_domain->domain_lock, flags);  in fsl_pamu_set_windows()
1045 	struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);  in fsl_pamu_get_windows()  local
1047 	return dma_domain->win_cnt;  in fsl_pamu_get_windows()