99 static int unmap_grant_pages(struct grant_map *map, int offset, int pages);
107 	struct grant_map *map;  in gntdev_print_maps()  local
110 	list_for_each_entry(map, &priv->maps, next)  in gntdev_print_maps()
112 		       map->index, map->count,  in gntdev_print_maps()
113 		       map->index == text_index && text ? text : "");  in gntdev_print_maps()
117 static void gntdev_free_map(struct grant_map *map)  in gntdev_free_map()  argument
119 	if (map == NULL)  in gntdev_free_map()
122 	if (map->pages)  in gntdev_free_map()
123 		gnttab_free_pages(map->count, map->pages);  in gntdev_free_map()
124 	kfree(map->pages);  in gntdev_free_map()
125 	kfree(map->grants);  in gntdev_free_map()
126 	kfree(map->map_ops);  in gntdev_free_map()
127 	kfree(map->unmap_ops);  in gntdev_free_map()
128 	kfree(map->kmap_ops);  in gntdev_free_map()
129 	kfree(map->kunmap_ops);  in gntdev_free_map()
130 	kfree(map);  in gntdev_free_map()
179 	struct grant_map *map;  in gntdev_add_map()  local
181 	list_for_each_entry(map, &priv->maps, next) {  in gntdev_add_map()
182 		if (add->index + add->count < map->index) {  in gntdev_add_map()
183 			list_add_tail(&add->next, &map->next);  in gntdev_add_map()
186 		add->index = map->index + map->count;  in gntdev_add_map()
197 	struct grant_map *map;  in gntdev_find_map_index()  local
199 	list_for_each_entry(map, &priv->maps, next) {  in gntdev_find_map_index()
200 		if (map->index != index)  in gntdev_find_map_index()
202 		if (count && map->count != count)  in gntdev_find_map_index()
204 		return map;  in gntdev_find_map_index()
209 static void gntdev_put_map(struct gntdev_priv *priv, struct grant_map *map)  in gntdev_put_map()  argument
211 	if (!map)  in gntdev_put_map()
214 	if (!atomic_dec_and_test(&map->users))  in gntdev_put_map()
217 	atomic_sub(map->count, &pages_mapped);  in gntdev_put_map()
219 	if (map->notify.flags & UNMAP_NOTIFY_SEND_EVENT) {  in gntdev_put_map()
220 		notify_remote_via_evtchn(map->notify.event);  in gntdev_put_map()
221 		evtchn_put(map->notify.event);  in gntdev_put_map()
226 		list_del(&map->next);  in gntdev_put_map()
230 	if (map->pages && !use_ptemod)  in gntdev_put_map()
231 		unmap_grant_pages(map, 0, map->count);  in gntdev_put_map()
232 	gntdev_free_map(map);  in gntdev_put_map()
240 	struct grant_map *map = data;  in find_grant_ptes()  local
241 	unsigned int pgnr = (addr - map->vma->vm_start) >> PAGE_SHIFT;  in find_grant_ptes()
242 	int flags = map->flags | GNTMAP_application_map | GNTMAP_contains_pte;  in find_grant_ptes()
245 	BUG_ON(pgnr >= map->count);  in find_grant_ptes()
256 	gnttab_set_map_op(&map->map_ops[pgnr], pte_maddr, flags,  in find_grant_ptes()
257 			  map->grants[pgnr].ref,  in find_grant_ptes()
258 			  map->grants[pgnr].domid);  in find_grant_ptes()
259 	gnttab_set_unmap_op(&map->unmap_ops[pgnr], pte_maddr, flags,  in find_grant_ptes()
273 static int map_grant_pages(struct grant_map *map)  in map_grant_pages()  argument
279 		if (map->map_ops[0].handle != -1)  in map_grant_pages()
281 		for (i = 0; i < map->count; i++) {  in map_grant_pages()
283 				pfn_to_kaddr(page_to_pfn(map->pages[i]));  in map_grant_pages()
284 			gnttab_set_map_op(&map->map_ops[i], addr, map->flags,  in map_grant_pages()
285 				map->grants[i].ref,  in map_grant_pages()
286 				map->grants[i].domid);  in map_grant_pages()
287 			gnttab_set_unmap_op(&map->unmap_ops[i], addr,  in map_grant_pages()
288 				map->flags, -1 /* handle */);  in map_grant_pages()
297 		for (i = 0; i < map->count; i++) {  in map_grant_pages()
299 				pfn_to_kaddr(page_to_pfn(map->pages[i]));  in map_grant_pages()
300 			BUG_ON(PageHighMem(map->pages[i]));  in map_grant_pages()
302 			gnttab_set_map_op(&map->kmap_ops[i], address,  in map_grant_pages()
303 				map->flags | GNTMAP_host_map,  in map_grant_pages()
304 				map->grants[i].ref,  in map_grant_pages()
305 				map->grants[i].domid);  in map_grant_pages()
306 			gnttab_set_unmap_op(&map->kunmap_ops[i], address,  in map_grant_pages()
307 				map->flags | GNTMAP_host_map, -1);  in map_grant_pages()
311 	pr_debug("map %d+%d\n", map->index, map->count);  in map_grant_pages()
312 	err = gnttab_map_refs(map->map_ops, use_ptemod ? map->kmap_ops : NULL,  in map_grant_pages()
313 			map->pages, map->count);  in map_grant_pages()
317 	for (i = 0; i < map->count; i++) {  in map_grant_pages()
318 		if (map->map_ops[i].status) {  in map_grant_pages()
323 		map->unmap_ops[i].handle = map->map_ops[i].handle;  in map_grant_pages()
325 			map->kunmap_ops[i].handle = map->kmap_ops[i].handle;  in map_grant_pages()
330 static int __unmap_grant_pages(struct grant_map *map, int offset, int pages)  in __unmap_grant_pages()  argument
335 	if (map->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) {  in __unmap_grant_pages()
336 		int pgno = (map->notify.addr >> PAGE_SHIFT);  in __unmap_grant_pages()
339 			uint8_t *tmp = pfn_to_kaddr(page_to_pfn(map->pages[pgno]));  in __unmap_grant_pages()
340 			tmp[map->notify.addr & (PAGE_SIZE-1)] = 0;  in __unmap_grant_pages()
341 			map->notify.flags &= ~UNMAP_NOTIFY_CLEAR_BYTE;  in __unmap_grant_pages()
345 	unmap_data.unmap_ops = map->unmap_ops + offset;  in __unmap_grant_pages()
346 	unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL;  in __unmap_grant_pages()
347 	unmap_data.pages = map->pages + offset;  in __unmap_grant_pages()
355 		if (map->unmap_ops[offset+i].status)  in __unmap_grant_pages()
358 			map->unmap_ops[offset+i].handle,  in __unmap_grant_pages()
359 			map->unmap_ops[offset+i].status);  in __unmap_grant_pages()
360 		map->unmap_ops[offset+i].handle = -1;  in __unmap_grant_pages()
365 static int unmap_grant_pages(struct grant_map *map, int offset, int pages)  in unmap_grant_pages()  argument
369 	pr_debug("unmap %d+%d [%d+%d]\n", map->index, map->count, offset, pages);  in unmap_grant_pages()
375 		while (pages && map->unmap_ops[offset].handle == -1) {  in unmap_grant_pages()
381 			if (map->unmap_ops[offset+range].handle == -1) {  in unmap_grant_pages()
387 		err = __unmap_grant_pages(map, offset, range);  in unmap_grant_pages()
399 	struct grant_map *map = vma->vm_private_data;  in gntdev_vma_open()  local
402 	atomic_inc(&map->users);  in gntdev_vma_open()
407 	struct grant_map *map = vma->vm_private_data;  in gntdev_vma_close()  local
421 		map->vma = NULL;  in gntdev_vma_close()
425 	gntdev_put_map(priv, map);  in gntdev_vma_close()
431 	struct grant_map *map = vma->vm_private_data;  in gntdev_vma_find_special_page()  local
433 	return map->pages[(addr - map->pages_vm_start) >> PAGE_SHIFT];  in gntdev_vma_find_special_page()
444 static void unmap_if_in_range(struct grant_map *map,  in unmap_if_in_range()  argument
450 	if (!map->vma)  in unmap_if_in_range()
452 	if (map->vma->vm_start >= end)  in unmap_if_in_range()
454 	if (map->vma->vm_end <= start)  in unmap_if_in_range()
456 	mstart = max(start, map->vma->vm_start);  in unmap_if_in_range()
457 	mend   = min(end,   map->vma->vm_end);  in unmap_if_in_range()
459 			map->index, map->count,  in unmap_if_in_range()
460 			map->vma->vm_start, map->vma->vm_end,  in unmap_if_in_range()
462 	err = unmap_grant_pages(map,  in unmap_if_in_range()
463 				(mstart - map->vma->vm_start) >> PAGE_SHIFT,  in unmap_if_in_range()
473 	struct grant_map *map;  in mn_invl_range_start()  local
476 	list_for_each_entry(map, &priv->maps, next) {  in mn_invl_range_start()
477 		unmap_if_in_range(map, start, end);  in mn_invl_range_start()
479 	list_for_each_entry(map, &priv->freeable_maps, next) {  in mn_invl_range_start()
480 		unmap_if_in_range(map, start, end);  in mn_invl_range_start()
496 	struct grant_map *map;  in mn_release()  local
500 	list_for_each_entry(map, &priv->maps, next) {  in mn_release()
501 		if (!map->vma)  in mn_release()
504 				map->index, map->count,  in mn_release()
505 				map->vma->vm_start, map->vma->vm_end);  in mn_release()
506 		err = unmap_grant_pages(map, /* offset */ 0, map->count);  in mn_release()
509 	list_for_each_entry(map, &priv->freeable_maps, next) {  in mn_release()
510 		if (!map->vma)  in mn_release()
513 				map->index, map->count,  in mn_release()
514 				map->vma->vm_start, map->vma->vm_end);  in mn_release()
515 		err = unmap_grant_pages(map, /* offset */ 0, map->count);  in mn_release()
567 	struct grant_map *map;  in gntdev_release()  local
573 		map = list_entry(priv->maps.next, struct grant_map, next);  in gntdev_release()
574 		list_del(&map->next);  in gntdev_release()
575 		gntdev_put_map(NULL /* already removed */, map);  in gntdev_release()
590 	struct grant_map *map;  in gntdev_ioctl_map_grant_ref()  local
600 	map = gntdev_alloc_map(priv, op.count);  in gntdev_ioctl_map_grant_ref()
601 	if (!map)  in gntdev_ioctl_map_grant_ref()
606 		gntdev_put_map(NULL, map);  in gntdev_ioctl_map_grant_ref()
610 	if (copy_from_user(map->grants, &u->refs,  in gntdev_ioctl_map_grant_ref()
611 			   sizeof(map->grants[0]) * op.count) != 0) {  in gntdev_ioctl_map_grant_ref()
612 		gntdev_put_map(NULL, map);  in gntdev_ioctl_map_grant_ref()
617 	gntdev_add_map(priv, map);  in gntdev_ioctl_map_grant_ref()
618 	op.index = map->index << PAGE_SHIFT;  in gntdev_ioctl_map_grant_ref()
631 	struct grant_map *map;  in gntdev_ioctl_unmap_grant_ref()  local
639 	map = gntdev_find_map_index(priv, op.index >> PAGE_SHIFT, op.count);  in gntdev_ioctl_unmap_grant_ref()
640 	if (map) {  in gntdev_ioctl_unmap_grant_ref()
641 		list_del(&map->next);  in gntdev_ioctl_unmap_grant_ref()
643 			list_add_tail(&map->next, &priv->freeable_maps);  in gntdev_ioctl_unmap_grant_ref()
647 	if (map)  in gntdev_ioctl_unmap_grant_ref()
648 		gntdev_put_map(priv, map);  in gntdev_ioctl_unmap_grant_ref()
657 	struct grant_map *map;  in gntdev_ioctl_get_offset_for_vaddr()  local
669 	map = vma->vm_private_data;  in gntdev_ioctl_get_offset_for_vaddr()
670 	if (!map)  in gntdev_ioctl_get_offset_for_vaddr()
673 	op.offset = map->index << PAGE_SHIFT;  in gntdev_ioctl_get_offset_for_vaddr()
674 	op.count = map->count;  in gntdev_ioctl_get_offset_for_vaddr()
688 	struct grant_map *map;  in gntdev_ioctl_notify()  local
716 	list_for_each_entry(map, &priv->maps, next) {  in gntdev_ioctl_notify()
717 		uint64_t begin = map->index << PAGE_SHIFT;  in gntdev_ioctl_notify()
718 		uint64_t end = (map->index + map->count) << PAGE_SHIFT;  in gntdev_ioctl_notify()
727 			(map->flags & GNTMAP_readonly)) {  in gntdev_ioctl_notify()
732 	out_flags = map->notify.flags;  in gntdev_ioctl_notify()
733 	out_event = map->notify.event;  in gntdev_ioctl_notify()
735 	map->notify.flags = op.action;  in gntdev_ioctl_notify()
736 	map->notify.addr = op.index - (map->index << PAGE_SHIFT);  in gntdev_ioctl_notify()
737 	map->notify.event = op.event_channel_port;  in gntdev_ioctl_notify()
783 	struct grant_map *map;  in gntdev_mmap()  local
793 	map = gntdev_find_map_index(priv, index, count);  in gntdev_mmap()
794 	if (!map)  in gntdev_mmap()
796 	if (use_ptemod && map->vma)  in gntdev_mmap()
803 	atomic_inc(&map->users);  in gntdev_mmap()
812 	vma->vm_private_data = map;  in gntdev_mmap()
815 		map->vma = vma;  in gntdev_mmap()
817 	if (map->flags) {  in gntdev_mmap()
819 				(map->flags & GNTMAP_readonly))  in gntdev_mmap()
822 		map->flags = GNTMAP_host_map;  in gntdev_mmap()
824 			map->flags |= GNTMAP_readonly;  in gntdev_mmap()
832 					  find_grant_ptes, map);  in gntdev_mmap()
839 	err = map_grant_pages(map);  in gntdev_mmap()
846 				map->pages[i]);  in gntdev_mmap()
867 		map->pages_vm_start = vma->vm_start;  in gntdev_mmap()
880 		map->vma = NULL;  in gntdev_mmap()
881 	gntdev_put_map(priv, map);  in gntdev_mmap()