1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /**************************************************************************
3 *
4 * Copyright 2017 VMware, Inc., Palo Alto, CA., USA
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 **************************************************************************/
28
29 #include "vmwgfx_drv.h"
30
31 /*
32 * Template that implements find_first_diff() for a generic
33 * unsigned integer type. @size and return value are in bytes.
34 */
35 #define VMW_FIND_FIRST_DIFF(_type) \
36 static size_t vmw_find_first_diff_ ## _type \
37 (const _type * dst, const _type * src, size_t size)\
38 { \
39 size_t i; \
40 \
41 for (i = 0; i < size; i += sizeof(_type)) { \
42 if (*dst++ != *src++) \
43 break; \
44 } \
45 \
46 return i; \
47 }
48
49
50 /*
51 * Template that implements find_last_diff() for a generic
52 * unsigned integer type. Pointers point to the item following the
53 * *end* of the area to be examined. @size and return value are in
54 * bytes.
55 */
56 #define VMW_FIND_LAST_DIFF(_type) \
57 static ssize_t vmw_find_last_diff_ ## _type( \
58 const _type * dst, const _type * src, size_t size) \
59 { \
60 while (size) { \
61 if (*--dst != *--src) \
62 break; \
63 \
64 size -= sizeof(_type); \
65 } \
66 return size; \
67 }
68
69
70 /*
71 * Instantiate find diff functions for relevant unsigned integer sizes,
72 * assuming that wider integers are faster (including aligning) up to the
73 * architecture native width, which is assumed to be 32 bit unless
74 * CONFIG_64BIT is defined.
75 */
76 VMW_FIND_FIRST_DIFF(u8);
77 VMW_FIND_LAST_DIFF(u8);
78
79 VMW_FIND_FIRST_DIFF(u16);
80 VMW_FIND_LAST_DIFF(u16);
81
82 VMW_FIND_FIRST_DIFF(u32);
83 VMW_FIND_LAST_DIFF(u32);
84
85 #ifdef CONFIG_64BIT
86 VMW_FIND_FIRST_DIFF(u64);
87 VMW_FIND_LAST_DIFF(u64);
88 #endif
89
90
91 /* We use size aligned copies. This computes (addr - align(addr)) */
92 #define SPILL(_var, _type) ((unsigned long) _var & (sizeof(_type) - 1))
93
94
95 /*
96 * Template to compute find_first_diff() for a certain integer type
97 * including a head copy for alignment, and adjustment of parameters
98 * for tail find or increased resolution find using an unsigned integer find
99 * of smaller width. If finding is complete, and resolution is sufficient,
100 * the macro executes a return statement. Otherwise it falls through.
101 */
102 #define VMW_TRY_FIND_FIRST_DIFF(_type) \
103 do { \
104 unsigned int spill = SPILL(dst, _type); \
105 size_t diff_offs; \
106 \
107 if (spill && spill == SPILL(src, _type) && \
108 sizeof(_type) - spill <= size) { \
109 spill = sizeof(_type) - spill; \
110 diff_offs = vmw_find_first_diff_u8(dst, src, spill); \
111 if (diff_offs < spill) \
112 return round_down(offset + diff_offs, granularity); \
113 \
114 dst += spill; \
115 src += spill; \
116 size -= spill; \
117 offset += spill; \
118 spill = 0; \
119 } \
120 if (!spill && !SPILL(src, _type)) { \
121 size_t to_copy = size & ~(sizeof(_type) - 1); \
122 \
123 diff_offs = vmw_find_first_diff_ ## _type \
124 ((_type *) dst, (_type *) src, to_copy); \
125 if (diff_offs >= size || granularity == sizeof(_type)) \
126 return (offset + diff_offs); \
127 \
128 dst += diff_offs; \
129 src += diff_offs; \
130 size -= diff_offs; \
131 offset += diff_offs; \
132 } \
133 } while (0) \
134
135
136 /**
137 * vmw_find_first_diff - find the first difference between dst and src
138 *
139 * @dst: The destination address
140 * @src: The source address
141 * @size: Number of bytes to compare
142 * @granularity: The granularity needed for the return value in bytes.
143 * return: The offset from find start where the first difference was
144 * encountered in bytes. If no difference was found, the function returns
145 * a value >= @size.
146 */
147 static size_t vmw_find_first_diff(const u8 *dst, const u8 *src, size_t size,
148 size_t granularity)
149 {
150 size_t offset = 0;
151
152 /*
153 * Try finding with large integers if alignment allows, or we can
154 * fix it. Fall through if we need better resolution or alignment
155 * was bad.
156 */
157 #ifdef CONFIG_64BIT
158 VMW_TRY_FIND_FIRST_DIFF(u64);
159 #endif
160 VMW_TRY_FIND_FIRST_DIFF(u32);
161 VMW_TRY_FIND_FIRST_DIFF(u16);
162
163 return round_down(offset + vmw_find_first_diff_u8(dst, src, size),
164 granularity);
165 }
166
167
168 /*
169 * Template to compute find_last_diff() for a certain integer type
170 * including a tail copy for alignment, and adjustment of parameters
171 * for head find or increased resolution find using an unsigned integer find
172 * of smaller width. If finding is complete, and resolution is sufficient,
173 * the macro executes a return statement. Otherwise it falls through.
174 */
175 #define VMW_TRY_FIND_LAST_DIFF(_type) \
176 do { \
177 unsigned int spill = SPILL(dst, _type); \
178 ssize_t location; \
179 ssize_t diff_offs; \
180 \
181 if (spill && spill <= size && spill == SPILL(src, _type)) { \
182 diff_offs = vmw_find_last_diff_u8(dst, src, spill); \
183 if (diff_offs) { \
184 location = size - spill + diff_offs - 1; \
185 return round_down(location, granularity); \
186 } \
187 \
188 dst -= spill; \
189 src -= spill; \
190 size -= spill; \
191 spill = 0; \
192 } \
193 if (!spill && !SPILL(src, _type)) { \
194 size_t to_copy = round_down(size, sizeof(_type)); \
195 \
196 diff_offs = vmw_find_last_diff_ ## _type \
197 ((_type *) dst, (_type *) src, to_copy); \
198 location = size - to_copy + diff_offs - sizeof(_type); \
199 if (location < 0 || granularity == sizeof(_type)) \
200 return location; \
201 \
202 dst -= to_copy - diff_offs; \
203 src -= to_copy - diff_offs; \
204 size -= to_copy - diff_offs; \
205 } \
206 } while (0)
207
208
209 /**
210 * vmw_find_last_diff - find the last difference between dst and src
211 *
212 * @dst: The destination address
213 * @src: The source address
214 * @size: Number of bytes to compare
215 * @granularity: The granularity needed for the return value in bytes.
216 * return: The offset from find start where the last difference was
217 * encountered in bytes, or a negative value if no difference was found.
218 */
219 static ssize_t vmw_find_last_diff(const u8 *dst, const u8 *src, size_t size,
220 size_t granularity)
221 {
222 dst += size;
223 src += size;
224
225 #ifdef CONFIG_64BIT
226 VMW_TRY_FIND_LAST_DIFF(u64);
227 #endif
228 VMW_TRY_FIND_LAST_DIFF(u32);
229 VMW_TRY_FIND_LAST_DIFF(u16);
230
231 return round_down(vmw_find_last_diff_u8(dst, src, size) - 1,
232 granularity);
233 }
234
235
236 /**
237 * vmw_memcpy - A wrapper around kernel memcpy with allowing to plug it into a
238 * struct vmw_diff_cpy.
239 *
240 * @diff: The struct vmw_diff_cpy closure argument (unused).
241 * @dest: The copy destination.
242 * @src: The copy source.
243 * @n: Number of bytes to copy.
244 */
245 void vmw_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src, size_t n)
246 {
247 memcpy(dest, src, n);
248 }
249
250
251 /**
252 * vmw_adjust_rect - Adjust rectangle coordinates for newly found difference
253 *
254 * @diff: The struct vmw_diff_cpy used to track the modified bounding box.
255 * @diff_offs: The offset from @diff->line_offset where the difference was
256 * found.
257 */
258 static void vmw_adjust_rect(struct vmw_diff_cpy *diff, size_t diff_offs)
259 {
260 size_t offs = (diff_offs + diff->line_offset) / diff->cpp;
261 struct drm_rect *rect = &diff->rect;
262
263 rect->x1 = min_t(int, rect->x1, offs);
264 rect->x2 = max_t(int, rect->x2, offs + 1);
265 rect->y1 = min_t(int, rect->y1, diff->line);
266 rect->y2 = max_t(int, rect->y2, diff->line + 1);
267 }
268
269 /**
270 * vmw_diff_memcpy - memcpy that creates a bounding box of modified content.
271 *
272 * @diff: The struct vmw_diff_cpy used to track the modified bounding box.
273 * @dest: The copy destination.
274 * @src: The copy source.
275 * @n: Number of bytes to copy.
276 *
277 * In order to correctly track the modified content, the field @diff->line must
278 * be pre-loaded with the current line number, the field @diff->line_offset must
279 * be pre-loaded with the line offset in bytes where the copy starts, and
280 * finally the field @diff->cpp need to be preloaded with the number of bytes
281 * per unit in the horizontal direction of the area we're examining.
282 * Typically bytes per pixel.
283 * This is needed to know the needed granularity of the difference computing
284 * operations. A higher cpp generally leads to faster execution at the cost of
285 * bounding box width precision.
286 */
287 void vmw_diff_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src,
288 size_t n)
289 {
290 ssize_t csize, byte_len;
291
292 if (WARN_ON_ONCE(round_down(n, diff->cpp) != n))
293 return;
294
295 /* TODO: Possibly use a single vmw_find_first_diff per line? */
296 csize = vmw_find_first_diff(dest, src, n, diff->cpp);
297 if (csize < n) {
298 vmw_adjust_rect(diff, csize);
299 byte_len = diff->cpp;
300
301 /*
302 * Starting from where first difference was found, find
303 * location of last difference, and then copy.
304 */
305 diff->line_offset += csize;
306 dest += csize;
307 src += csize;
308 n -= csize;
309 csize = vmw_find_last_diff(dest, src, n, diff->cpp);
310 if (csize >= 0) {
311 byte_len += csize;
312 vmw_adjust_rect(diff, csize);
313 }
314 memcpy(dest, src, byte_len);
315 }
316 diff->line_offset += n;
317 }
318
319 /**
320 * struct vmw_bo_blit_line_data - Convenience argument to vmw_bo_cpu_blit_line
321 *
322 * @mapped_dst: Already mapped destination page index in @dst_pages.
323 * @dst_addr: Kernel virtual address of mapped destination page.
324 * @dst_pages: Array of destination bo pages.
325 * @dst_num_pages: Number of destination bo pages.
326 * @dst_prot: Destination bo page protection.
327 * @mapped_src: Already mapped source page index in @dst_pages.
328 * @src_addr: Kernel virtual address of mapped source page.
329 * @src_pages: Array of source bo pages.
330 * @src_num_pages: Number of source bo pages.
331 * @src_prot: Source bo page protection.
332 * @diff: Struct vmw_diff_cpy, in the end forwarded to the memcpy routine.
333 */
334 struct vmw_bo_blit_line_data {
335 u32 mapped_dst;
336 u8 *dst_addr;
337 struct page **dst_pages;
338 u32 dst_num_pages;
339 pgprot_t dst_prot;
340 u32 mapped_src;
341 u8 *src_addr;
342 struct page **src_pages;
343 u32 src_num_pages;
344 pgprot_t src_prot;
345 struct vmw_diff_cpy *diff;
346 };
347
348 /**
349 * vmw_bo_cpu_blit_line - Blit part of a line from one bo to another.
350 *
351 * @d: Blit data as described above.
352 * @dst_offset: Destination copy start offset from start of bo.
353 * @src_offset: Source copy start offset from start of bo.
354 * @bytes_to_copy: Number of bytes to copy in this line.
355 */
356 static int vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data *d,
357 u32 dst_offset,
358 u32 src_offset,
359 u32 bytes_to_copy)
360 {
361 struct vmw_diff_cpy *diff = d->diff;
362
363 while (bytes_to_copy) {
364 u32 copy_size = bytes_to_copy;
365 u32 dst_page = dst_offset >> PAGE_SHIFT;
366 u32 src_page = src_offset >> PAGE_SHIFT;
367 u32 dst_page_offset = dst_offset & ~PAGE_MASK;
368 u32 src_page_offset = src_offset & ~PAGE_MASK;
369 bool unmap_dst = d->dst_addr && dst_page != d->mapped_dst;
370 bool unmap_src = d->src_addr && (src_page != d->mapped_src ||
371 unmap_dst);
372
373 copy_size = min_t(u32, copy_size, PAGE_SIZE - dst_page_offset);
374 copy_size = min_t(u32, copy_size, PAGE_SIZE - src_page_offset);
375
376 if (unmap_src) {
377 ttm_kunmap_atomic_prot(d->src_addr, d->src_prot);
378 d->src_addr = NULL;
379 }
380
381 if (unmap_dst) {
382 ttm_kunmap_atomic_prot(d->dst_addr, d->dst_prot);
383 d->dst_addr = NULL;
384 }
385
386 if (!d->dst_addr) {
387 if (WARN_ON_ONCE(dst_page >= d->dst_num_pages))
388 return -EINVAL;
389
390 d->dst_addr =
391 ttm_kmap_atomic_prot(d->dst_pages[dst_page],
392 d->dst_prot);
393 if (!d->dst_addr)
394 return -ENOMEM;
395
396 d->mapped_dst = dst_page;
397 }
398
399 if (!d->src_addr) {
400 if (WARN_ON_ONCE(src_page >= d->src_num_pages))
401 return -EINVAL;
402
403 d->src_addr =
404 ttm_kmap_atomic_prot(d->src_pages[src_page],
405 d->src_prot);
406 if (!d->src_addr)
407 return -ENOMEM;
408
409 d->mapped_src = src_page;
410 }
411 diff->do_cpy(diff, d->dst_addr + dst_page_offset,
412 d->src_addr + src_page_offset, copy_size);
413
414 bytes_to_copy -= copy_size;
415 dst_offset += copy_size;
416 src_offset += copy_size;
417 }
418
419 return 0;
420 }
421
422 /**
423 * ttm_bo_cpu_blit - in-kernel cpu blit.
424 *
425 * @dst: Destination buffer object.
426 * @dst_offset: Destination offset of blit start in bytes.
427 * @dst_stride: Destination stride in bytes.
428 * @src: Source buffer object.
429 * @src_offset: Source offset of blit start in bytes.
430 * @src_stride: Source stride in bytes.
431 * @w: Width of blit.
432 * @h: Height of blit.
433 * return: Zero on success. Negative error value on failure. Will print out
434 * kernel warnings on caller bugs.
435 *
436 * Performs a CPU blit from one buffer object to another avoiding a full
437 * bo vmap which may exhaust- or fragment vmalloc space.
438 * On supported architectures (x86), we're using kmap_atomic which avoids
439 * cross-processor TLB- and cache flushes and may, on non-HIGHMEM systems
440 * reference already set-up mappings.
441 *
442 * Neither of the buffer objects may be placed in PCI memory
443 * (Fixed memory in TTM terminology) when using this function.
444 */
445 int vmw_bo_cpu_blit(struct ttm_buffer_object *dst,
446 u32 dst_offset, u32 dst_stride,
447 struct ttm_buffer_object *src,
448 u32 src_offset, u32 src_stride,
449 u32 w, u32 h,
450 struct vmw_diff_cpy *diff)
451 {
452 struct ttm_operation_ctx ctx = {
453 .interruptible = false,
454 .no_wait_gpu = false
455 };
456 u32 j, initial_line = dst_offset / dst_stride;
457 struct vmw_bo_blit_line_data d;
458 int ret = 0;
459
460 /* Buffer objects need to be either pinned or reserved: */
461 if (!(dst->mem.placement & TTM_PL_FLAG_NO_EVICT))
462 dma_resv_assert_held(dst->base.resv);
463 if (!(src->mem.placement & TTM_PL_FLAG_NO_EVICT))
464 dma_resv_assert_held(src->base.resv);
465
466 if (dst->ttm->state == tt_unpopulated) {
467 ret = dst->ttm->bdev->driver->ttm_tt_populate(dst->ttm, &ctx);
468 if (ret)
469 return ret;
470 }
471
472 if (src->ttm->state == tt_unpopulated) {
473 ret = src->ttm->bdev->driver->ttm_tt_populate(src->ttm, &ctx);
474 if (ret)
475 return ret;
476 }
477
478 d.mapped_dst = 0;
479 d.mapped_src = 0;
480 d.dst_addr = NULL;
481 d.src_addr = NULL;
482 d.dst_pages = dst->ttm->pages;
483 d.src_pages = src->ttm->pages;
484 d.dst_num_pages = dst->num_pages;
485 d.src_num_pages = src->num_pages;
486 d.dst_prot = ttm_io_prot(dst->mem.placement, PAGE_KERNEL);
487 d.src_prot = ttm_io_prot(src->mem.placement, PAGE_KERNEL);
488 d.diff = diff;
489
490 for (j = 0; j < h; ++j) {
491 diff->line = j + initial_line;
492 diff->line_offset = dst_offset % dst_stride;
493 ret = vmw_bo_cpu_blit_line(&d, dst_offset, src_offset, w);
494 if (ret)
495 goto out;
496
497 dst_offset += dst_stride;
498 src_offset += src_stride;
499 }
500 out:
501 if (d.src_addr)
502 ttm_kunmap_atomic_prot(d.src_addr, d.src_prot);
503 if (d.dst_addr)
504 ttm_kunmap_atomic_prot(d.dst_addr, d.dst_prot);
505
506 return ret;
507 }