root/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c

DEFINITIONS

1. gen8_canonical_addr
2. gen8_noncanonical_addr
3. eb_use_cmdparser
4. eb_create
5. eb_vma_misplaced
6. eb_pin_vma
7. __eb_unreserve_vma
8. eb_unreserve_vma
9. eb_validate_vma
10. eb_add_vma
11. use_cpu_reloc
12. eb_reserve_vma
13. eb_reserve
14. eb_batch_index
15. eb_select_context
16. eb_lookup_vmas
17. eb_get_vma
18. eb_release_vmas
19. eb_reset_vmas
20. eb_destroy
21. relocation_target
22. reloc_cache_init
23. unmask_page
24. unmask_flags
25. cache_to_ggtt
26. reloc_gpu_flush
27. reloc_cache_reset
28. reloc_kmap
29. reloc_iomap
30. reloc_vaddr
31. clflush_write32
32. reloc_move_to_gpu
33. __reloc_gpu_alloc
34. reloc_gpu
35. relocate_entry
36. eb_relocate_entry
37. eb_relocate_vma
38. eb_relocate_vma_slow
39. check_relocations
40. eb_copy_relocations
41. eb_prefault_relocations
42. eb_relocate_slow
43. eb_relocate
44. eb_move_to_gpu
45. i915_gem_check_execbuffer
46. i915_reset_gen7_sol_offsets
47. shadow_batch_pin
48. eb_parse
49. add_to_client
50. eb_submit
51. num_vcs_engines
52. gen8_dispatch_bsd_engine
53. eb_throttle
54. __eb_pin_context
55. __eb_unpin_context
56. __eb_pin_engine
57. eb_unpin_engine
58. eb_select_legacy_ring
59. eb_pin_engine
60. __free_fence_array
61. get_fence_array
62. put_fence_array
63. await_fence_array
64. signal_fence_array
65. i915_gem_do_execbuffer
66. eb_element_size
67. check_buffer_count
68. i915_gem_execbuffer_ioctl
69. i915_gem_execbuffer2_ioctl

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2008,2010 Intel Corporation
   5  */
   6 
   7 #include <linux/intel-iommu.h>
   8 #include <linux/dma-resv.h>
   9 #include <linux/sync_file.h>
  10 #include <linux/uaccess.h>
  11 
  12 #include <drm/drm_syncobj.h>
  13 #include <drm/i915_drm.h>
  14 
  15 #include "display/intel_frontbuffer.h"
  16 
  17 #include "gem/i915_gem_ioctls.h"
  18 #include "gt/intel_context.h"
  19 #include "gt/intel_engine_pool.h"
  20 #include "gt/intel_gt.h"
  21 #include "gt/intel_gt_pm.h"
  22 
  23 #include "i915_drv.h"
  24 #include "i915_gem_clflush.h"
  25 #include "i915_gem_context.h"
  26 #include "i915_gem_ioctls.h"
  27 #include "i915_trace.h"
  28 
  29 enum {
  30         FORCE_CPU_RELOC = 1,
  31         FORCE_GTT_RELOC,
  32         FORCE_GPU_RELOC,
  33 #define DBG_FORCE_RELOC 0 /* choose one of the above! */
  34 };
  35 
  36 #define __EXEC_OBJECT_HAS_REF           BIT(31)
  37 #define __EXEC_OBJECT_HAS_PIN           BIT(30)
  38 #define __EXEC_OBJECT_HAS_FENCE         BIT(29)
  39 #define __EXEC_OBJECT_NEEDS_MAP         BIT(28)
  40 #define __EXEC_OBJECT_NEEDS_BIAS        BIT(27)
  41 #define __EXEC_OBJECT_INTERNAL_FLAGS    (~0u << 27) /* all of the above */
  42 #define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE)
  43 
  44 #define __EXEC_HAS_RELOC        BIT(31)
  45 #define __EXEC_VALIDATED        BIT(30)
  46 #define __EXEC_INTERNAL_FLAGS   (~0u << 30)
  47 #define UPDATE                  PIN_OFFSET_FIXED
  48 
  49 #define BATCH_OFFSET_BIAS (256*1024)
  50 
  51 #define __I915_EXEC_ILLEGAL_FLAGS \
  52         (__I915_EXEC_UNKNOWN_FLAGS | \
  53          I915_EXEC_CONSTANTS_MASK  | \
  54          I915_EXEC_RESOURCE_STREAMER)
  55 
  56 /* Catch emission of unexpected errors for CI! */
  57 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
  58 #undef EINVAL
  59 #define EINVAL ({ \
  60         DRM_DEBUG_DRIVER("EINVAL at %s:%d\n", __func__, __LINE__); \
  61         22; \
  62 })
  63 #endif
  64 
  65 /**
  66  * DOC: User command execution
  67  *
  68  * Userspace submits commands to be executed on the GPU as an instruction
  69  * stream within a GEM object we call a batchbuffer. This instructions may
  70  * refer to other GEM objects containing auxiliary state such as kernels,
  71  * samplers, render targets and even secondary batchbuffers. Userspace does
  72  * not know where in the GPU memory these objects reside and so before the
  73  * batchbuffer is passed to the GPU for execution, those addresses in the
  74  * batchbuffer and auxiliary objects are updated. This is known as relocation,
  75  * or patching. To try and avoid having to relocate each object on the next
  76  * execution, userspace is told the location of those objects in this pass,
  77  * but this remains just a hint as the kernel may choose a new location for
  78  * any object in the future.
  79  *
  80  * At the level of talking to the hardware, submitting a batchbuffer for the
  81  * GPU to execute is to add content to a buffer from which the HW
  82  * command streamer is reading.
  83  *
  84  * 1. Add a command to load the HW context. For Logical Ring Contexts, i.e.
  85  *    Execlists, this command is not placed on the same buffer as the
  86  *    remaining items.
  87  *
  88  * 2. Add a command to invalidate caches to the buffer.
  89  *
  90  * 3. Add a batchbuffer start command to the buffer; the start command is
  91  *    essentially a token together with the GPU address of the batchbuffer
  92  *    to be executed.
  93  *
  94  * 4. Add a pipeline flush to the buffer.
  95  *
  96  * 5. Add a memory write command to the buffer to record when the GPU
  97  *    is done executing the batchbuffer. The memory write writes the
  98  *    global sequence number of the request, ``i915_request::global_seqno``;
  99  *    the i915 driver uses the current value in the register to determine
 100  *    if the GPU has completed the batchbuffer.
 101  *
 102  * 6. Add a user interrupt command to the buffer. This command instructs
 103  *    the GPU to issue an interrupt when the command, pipeline flush and
 104  *    memory write are completed.
 105  *
 106  * 7. Inform the hardware of the additional commands added to the buffer
 107  *    (by updating the tail pointer).
 108  *
 109  * Processing an execbuf ioctl is conceptually split up into a few phases.
 110  *
 111  * 1. Validation - Ensure all the pointers, handles and flags are valid.
 112  * 2. Reservation - Assign GPU address space for every object
 113  * 3. Relocation - Update any addresses to point to the final locations
 114  * 4. Serialisation - Order the request with respect to its dependencies
 115  * 5. Construction - Construct a request to execute the batchbuffer
 116  * 6. Submission (at some point in the future execution)
 117  *
 118  * Reserving resources for the execbuf is the most complicated phase. We
 119  * neither want to have to migrate the object in the address space, nor do
 120  * we want to have to update any relocations pointing to this object. Ideally,
 121  * we want to leave the object where it is and for all the existing relocations
 122  * to match. If the object is given a new address, or if userspace thinks the
 123  * object is elsewhere, we have to parse all the relocation entries and update
 124  * the addresses. Userspace can set the I915_EXEC_NORELOC flag to hint that
 125  * all the target addresses in all of its objects match the value in the
 126  * relocation entries and that they all match the presumed offsets given by the
 127  * list of execbuffer objects. Using this knowledge, we know that if we haven't
 128  * moved any buffers, all the relocation entries are valid and we can skip
 129  * the update. (If userspace is wrong, the likely outcome is an impromptu GPU
 130  * hang.) The requirement for using I915_EXEC_NO_RELOC are:
 131  *
 132  *      The addresses written in the objects must match the corresponding
 133  *      reloc.presumed_offset which in turn must match the corresponding
 134  *      execobject.offset.
 135  *
 136  *      Any render targets written to in the batch must be flagged with
 137  *      EXEC_OBJECT_WRITE.
 138  *
 139  *      To avoid stalling, execobject.offset should match the current
 140  *      address of that object within the active context.
 141  *
 142  * The reservation is done is multiple phases. First we try and keep any
 143  * object already bound in its current location - so as long as meets the
 144  * constraints imposed by the new execbuffer. Any object left unbound after the
 145  * first pass is then fitted into any available idle space. If an object does
 146  * not fit, all objects are removed from the reservation and the process rerun
 147  * after sorting the objects into a priority order (more difficult to fit
 148  * objects are tried first). Failing that, the entire VM is cleared and we try
 149  * to fit the execbuf once last time before concluding that it simply will not
 150  * fit.
 151  *
 152  * A small complication to all of this is that we allow userspace not only to
 153  * specify an alignment and a size for the object in the address space, but
 154  * we also allow userspace to specify the exact offset. This objects are
 155  * simpler to place (the location is known a priori) all we have to do is make
 156  * sure the space is available.
 157  *
 158  * Once all the objects are in place, patching up the buried pointers to point
 159  * to the final locations is a fairly simple job of walking over the relocation
 160  * entry arrays, looking up the right address and rewriting the value into
 161  * the object. Simple! ... The relocation entries are stored in user memory
 162  * and so to access them we have to copy them into a local buffer. That copy
 163  * has to avoid taking any pagefaults as they may lead back to a GEM object
 164  * requiring the struct_mutex (i.e. recursive deadlock). So once again we split
 165  * the relocation into multiple passes. First we try to do everything within an
 166  * atomic context (avoid the pagefaults) which requires that we never wait. If
 167  * we detect that we may wait, or if we need to fault, then we have to fallback
 168  * to a slower path. The slowpath has to drop the mutex. (Can you hear alarm
 169  * bells yet?) Dropping the mutex means that we lose all the state we have
 170  * built up so far for the execbuf and we must reset any global data. However,
 171  * we do leave the objects pinned in their final locations - which is a
 172  * potential issue for concurrent execbufs. Once we have left the mutex, we can
 173  * allocate and copy all the relocation entries into a large array at our
 174  * leisure, reacquire the mutex, reclaim all the objects and other state and
 175  * then proceed to update any incorrect addresses with the objects.
 176  *
 177  * As we process the relocation entries, we maintain a record of whether the
 178  * object is being written to. Using NORELOC, we expect userspace to provide
 179  * this information instead. We also check whether we can skip the relocation
 180  * by comparing the expected value inside the relocation entry with the target's
 181  * final address. If they differ, we have to map the current object and rewrite
 182  * the 4 or 8 byte pointer within.
 183  *
 184  * Serialising an execbuf is quite simple according to the rules of the GEM
 185  * ABI. Execution within each context is ordered by the order of submission.
 186  * Writes to any GEM object are in order of submission and are exclusive. Reads
 187  * from a GEM object are unordered with respect to other reads, but ordered by
 188  * writes. A write submitted after a read cannot occur before the read, and
 189  * similarly any read submitted after a write cannot occur before the write.
 190  * Writes are ordered between engines such that only one write occurs at any
 191  * time (completing any reads beforehand) - using semaphores where available
 192  * and CPU serialisation otherwise. Other GEM access obey the same rules, any
 193  * write (either via mmaps using set-domain, or via pwrite) must flush all GPU
 194  * reads before starting, and any read (either using set-domain or pread) must
 195  * flush all GPU writes before starting. (Note we only employ a barrier before,
 196  * we currently rely on userspace not concurrently starting a new execution
 197  * whilst reading or writing to an object. This may be an advantage or not
 198  * depending on how much you trust userspace not to shoot themselves in the
 199  * foot.) Serialisation may just result in the request being inserted into
 200  * a DAG awaiting its turn, but most simple is to wait on the CPU until
 201  * all dependencies are resolved.
 202  *
 203  * After all of that, is just a matter of closing the request and handing it to
 204  * the hardware (well, leaving it in a queue to be executed). However, we also
 205  * offer the ability for batchbuffers to be run with elevated privileges so
 206  * that they access otherwise hidden registers. (Used to adjust L3 cache etc.)
 207  * Before any batch is given extra privileges we first must check that it
 208  * contains no nefarious instructions, we check that each instruction is from
 209  * our whitelist and all registers are also from an allowed list. We first
 210  * copy the user's batchbuffer to a shadow (so that the user doesn't have
 211  * access to it, either by the CPU or GPU as we scan it) and then parse each
 212  * instruction. If everything is ok, we set a flag telling the hardware to run
 213  * the batchbuffer in trusted mode, otherwise the ioctl is rejected.
 214  */
 215 
 216 struct i915_execbuffer {
 217         struct drm_i915_private *i915; /** i915 backpointer */
 218         struct drm_file *file; /** per-file lookup tables and limits */
 219         struct drm_i915_gem_execbuffer2 *args; /** ioctl parameters */
 220         struct drm_i915_gem_exec_object2 *exec; /** ioctl execobj[] */
 221         struct i915_vma **vma;
 222         unsigned int *flags;
 223 
 224         struct intel_engine_cs *engine; /** engine to queue the request to */
 225         struct intel_context *context; /* logical state for the request */
 226         struct i915_gem_context *gem_context; /** caller's context */
 227 
 228         struct i915_request *request; /** our request to build */
 229         struct i915_vma *batch; /** identity of the batch obj/vma */
 230 
 231         /** actual size of execobj[] as we may extend it for the cmdparser */
 232         unsigned int buffer_count;
 233 
 234         /** list of vma not yet bound during reservation phase */
 235         struct list_head unbound;
 236 
 237         /** list of vma that have execobj.relocation_count */
 238         struct list_head relocs;
 239 
 240         /**
 241          * Track the most recently used object for relocations, as we
 242          * frequently have to perform multiple relocations within the same
 243          * obj/page
 244          */
 245         struct reloc_cache {
 246                 struct drm_mm_node node; /** temporary GTT binding */
 247                 unsigned long vaddr; /** Current kmap address */
 248                 unsigned long page; /** Currently mapped page index */
 249                 unsigned int gen; /** Cached value of INTEL_GEN */
 250                 bool use_64bit_reloc : 1;
 251                 bool has_llc : 1;
 252                 bool has_fence : 1;
 253                 bool needs_unfenced : 1;
 254 
 255                 struct i915_request *rq;
 256                 u32 *rq_cmd;
 257                 unsigned int rq_size;
 258         } reloc_cache;
 259 
 260         u64 invalid_flags; /** Set of execobj.flags that are invalid */
 261         u32 context_flags; /** Set of execobj.flags to insert from the ctx */
 262 
 263         u32 batch_start_offset; /** Location within object of batch */
 264         u32 batch_len; /** Length of batch within object */
 265         u32 batch_flags; /** Flags composed for emit_bb_start() */
 266 
 267         /**
 268          * Indicate either the size of the hastable used to resolve
 269          * relocation handles, or if negative that we are using a direct
 270          * index into the execobj[].
 271          */
 272         int lut_size;
 273         struct hlist_head *buckets; /** ht for relocation handles */
 274 };
 275 
 276 #define exec_entry(EB, VMA) (&(EB)->exec[(VMA)->exec_flags - (EB)->flags])
 277 
 278 /*
 279  * Used to convert any address to canonical form.
 280  * Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS,
 281  * MI_LOAD_REGISTER_MEM and others, see Broadwell PRM Vol2a) require the
 282  * addresses to be in a canonical form:
 283  * "GraphicsAddress[63:48] are ignored by the HW and assumed to be in correct
 284  * canonical form [63:48] == [47]."
 285  */
 286 #define GEN8_HIGH_ADDRESS_BIT 47
 287 static inline u64 gen8_canonical_addr(u64 address)
 288 {
 289         return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT);
 290 }
 291 
 292 static inline u64 gen8_noncanonical_addr(u64 address)
 293 {
 294         return address & GENMASK_ULL(GEN8_HIGH_ADDRESS_BIT, 0);
 295 }
 296 
 297 static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb)
 298 {
 299         return intel_engine_requires_cmd_parser(eb->engine) ||
 300                 (intel_engine_using_cmd_parser(eb->engine) &&
 301                  eb->args->batch_len);
 302 }
 303 
 304 static int eb_create(struct i915_execbuffer *eb)
 305 {
 306         if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) {
 307                 unsigned int size = 1 + ilog2(eb->buffer_count);
 308 
 309                 /*
 310                  * Without a 1:1 association between relocation handles and
 311                  * the execobject[] index, we instead create a hashtable.
 312                  * We size it dynamically based on available memory, starting
 313                  * first with 1:1 assocative hash and scaling back until
 314                  * the allocation succeeds.
 315                  *
 316                  * Later on we use a positive lut_size to indicate we are
 317                  * using this hashtable, and a negative value to indicate a
 318                  * direct lookup.
 319                  */
 320                 do {
 321                         gfp_t flags;
 322 
 323                         /* While we can still reduce the allocation size, don't
 324                          * raise a warning and allow the allocation to fail.
 325                          * On the last pass though, we want to try as hard
 326                          * as possible to perform the allocation and warn
 327                          * if it fails.
 328                          */
 329                         flags = GFP_KERNEL;
 330                         if (size > 1)
 331                                 flags |= __GFP_NORETRY | __GFP_NOWARN;
 332 
 333                         eb->buckets = kzalloc(sizeof(struct hlist_head) << size,
 334                                               flags);
 335                         if (eb->buckets)
 336                                 break;
 337                 } while (--size);
 338 
 339                 if (unlikely(!size))
 340                         return -ENOMEM;
 341 
 342                 eb->lut_size = size;
 343         } else {
 344                 eb->lut_size = -eb->buffer_count;
 345         }
 346 
 347         return 0;
 348 }
 349 
 350 static bool
 351 eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry,
 352                  const struct i915_vma *vma,
 353                  unsigned int flags)
 354 {
 355         if (vma->node.size < entry->pad_to_size)
 356                 return true;
 357 
 358         if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment))
 359                 return true;
 360 
 361         if (flags & EXEC_OBJECT_PINNED &&
 362             vma->node.start != entry->offset)
 363                 return true;
 364 
 365         if (flags & __EXEC_OBJECT_NEEDS_BIAS &&
 366             vma->node.start < BATCH_OFFSET_BIAS)
 367                 return true;
 368 
 369         if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) &&
 370             (vma->node.start + vma->node.size - 1) >> 32)
 371                 return true;
 372 
 373         if (flags & __EXEC_OBJECT_NEEDS_MAP &&
 374             !i915_vma_is_map_and_fenceable(vma))
 375                 return true;
 376 
 377         return false;
 378 }
 379 
 380 static inline bool
 381 eb_pin_vma(struct i915_execbuffer *eb,
 382            const struct drm_i915_gem_exec_object2 *entry,
 383            struct i915_vma *vma)
 384 {
 385         unsigned int exec_flags = *vma->exec_flags;
 386         u64 pin_flags;
 387 
 388         if (vma->node.size)
 389                 pin_flags = vma->node.start;
 390         else
 391                 pin_flags = entry->offset & PIN_OFFSET_MASK;
 392 
 393         pin_flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED;
 394         if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_GTT))
 395                 pin_flags |= PIN_GLOBAL;
 396 
 397         if (unlikely(i915_vma_pin(vma, 0, 0, pin_flags)))
 398                 return false;
 399 
 400         if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) {
 401                 if (unlikely(i915_vma_pin_fence(vma))) {
 402                         i915_vma_unpin(vma);
 403                         return false;
 404                 }
 405 
 406                 if (vma->fence)
 407                         exec_flags |= __EXEC_OBJECT_HAS_FENCE;
 408         }
 409 
 410         *vma->exec_flags = exec_flags | __EXEC_OBJECT_HAS_PIN;
 411         return !eb_vma_misplaced(entry, vma, exec_flags);
 412 }
 413 
 414 static inline void __eb_unreserve_vma(struct i915_vma *vma, unsigned int flags)
 415 {
 416         GEM_BUG_ON(!(flags & __EXEC_OBJECT_HAS_PIN));
 417 
 418         if (unlikely(flags & __EXEC_OBJECT_HAS_FENCE))
 419                 __i915_vma_unpin_fence(vma);
 420 
 421         __i915_vma_unpin(vma);
 422 }
 423 
 424 static inline void
 425 eb_unreserve_vma(struct i915_vma *vma, unsigned int *flags)
 426 {
 427         if (!(*flags & __EXEC_OBJECT_HAS_PIN))
 428                 return;
 429 
 430         __eb_unreserve_vma(vma, *flags);
 431         *flags &= ~__EXEC_OBJECT_RESERVED;
 432 }
 433 
 434 static int
 435 eb_validate_vma(struct i915_execbuffer *eb,
 436                 struct drm_i915_gem_exec_object2 *entry,
 437                 struct i915_vma *vma)
 438 {
 439         if (unlikely(entry->flags & eb->invalid_flags))
 440                 return -EINVAL;
 441 
 442         if (unlikely(entry->alignment &&
 443                      !is_power_of_2_u64(entry->alignment)))
 444                 return -EINVAL;
 445 
 446         /*
 447          * Offset can be used as input (EXEC_OBJECT_PINNED), reject
 448          * any non-page-aligned or non-canonical addresses.
 449          */
 450         if (unlikely(entry->flags & EXEC_OBJECT_PINNED &&
 451                      entry->offset != gen8_canonical_addr(entry->offset & I915_GTT_PAGE_MASK)))
 452                 return -EINVAL;
 453 
 454         /* pad_to_size was once a reserved field, so sanitize it */
 455         if (entry->flags & EXEC_OBJECT_PAD_TO_SIZE) {
 456                 if (unlikely(offset_in_page(entry->pad_to_size)))
 457                         return -EINVAL;
 458         } else {
 459                 entry->pad_to_size = 0;
 460         }
 461 
 462         if (unlikely(vma->exec_flags)) {
 463                 DRM_DEBUG("Object [handle %d, index %d] appears more than once in object list\n",
 464                           entry->handle, (int)(entry - eb->exec));
 465                 return -EINVAL;
 466         }
 467 
 468         /*
 469          * From drm_mm perspective address space is continuous,
 470          * so from this point we're always using non-canonical
 471          * form internally.
 472          */
 473         entry->offset = gen8_noncanonical_addr(entry->offset);
 474 
 475         if (!eb->reloc_cache.has_fence) {
 476                 entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
 477         } else {
 478                 if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE ||
 479                      eb->reloc_cache.needs_unfenced) &&
 480                     i915_gem_object_is_tiled(vma->obj))
 481                         entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP;
 482         }
 483 
 484         if (!(entry->flags & EXEC_OBJECT_PINNED))
 485                 entry->flags |= eb->context_flags;
 486 
 487         return 0;
 488 }
 489 
 490 static int
 491 eb_add_vma(struct i915_execbuffer *eb,
 492            unsigned int i, unsigned batch_idx,
 493            struct i915_vma *vma)
 494 {
 495         struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
 496         int err;
 497 
 498         GEM_BUG_ON(i915_vma_is_closed(vma));
 499 
 500         if (!(eb->args->flags & __EXEC_VALIDATED)) {
 501                 err = eb_validate_vma(eb, entry, vma);
 502                 if (unlikely(err))
 503                         return err;
 504         }
 505 
 506         if (eb->lut_size > 0) {
 507                 vma->exec_handle = entry->handle;
 508                 hlist_add_head(&vma->exec_node,
 509                                &eb->buckets[hash_32(entry->handle,
 510                                                     eb->lut_size)]);
 511         }
 512 
 513         if (entry->relocation_count)
 514                 list_add_tail(&vma->reloc_link, &eb->relocs);
 515 
 516         /*
 517          * Stash a pointer from the vma to execobj, so we can query its flags,
 518          * size, alignment etc as provided by the user. Also we stash a pointer
 519          * to the vma inside the execobj so that we can use a direct lookup
 520          * to find the right target VMA when doing relocations.
 521          */
 522         eb->vma[i] = vma;
 523         eb->flags[i] = entry->flags;
 524         vma->exec_flags = &eb->flags[i];
 525 
 526         /*
 527          * SNA is doing fancy tricks with compressing batch buffers, which leads
 528          * to negative relocation deltas. Usually that works out ok since the
 529          * relocate address is still positive, except when the batch is placed
 530          * very low in the GTT. Ensure this doesn't happen.
 531          *
 532          * Note that actual hangs have only been observed on gen7, but for
 533          * paranoia do it everywhere.
 534          */
 535         if (i == batch_idx) {
 536                 if (entry->relocation_count &&
 537                     !(eb->flags[i] & EXEC_OBJECT_PINNED))
 538                         eb->flags[i] |= __EXEC_OBJECT_NEEDS_BIAS;
 539                 if (eb->reloc_cache.has_fence)
 540                         eb->flags[i] |= EXEC_OBJECT_NEEDS_FENCE;
 541 
 542                 eb->batch = vma;
 543         }
 544 
 545         err = 0;
 546         if (eb_pin_vma(eb, entry, vma)) {
 547                 if (entry->offset != vma->node.start) {
 548                         entry->offset = vma->node.start | UPDATE;
 549                         eb->args->flags |= __EXEC_HAS_RELOC;
 550                 }
 551         } else {
 552                 eb_unreserve_vma(vma, vma->exec_flags);
 553 
 554                 list_add_tail(&vma->exec_link, &eb->unbound);
 555                 if (drm_mm_node_allocated(&vma->node))
 556                         err = i915_vma_unbind(vma);
 557                 if (unlikely(err))
 558                         vma->exec_flags = NULL;
 559         }
 560         return err;
 561 }
 562 
 563 static inline int use_cpu_reloc(const struct reloc_cache *cache,
 564                                 const struct drm_i915_gem_object *obj)
 565 {
 566         if (!i915_gem_object_has_struct_page(obj))
 567                 return false;
 568 
 569         if (DBG_FORCE_RELOC == FORCE_CPU_RELOC)
 570                 return true;
 571 
 572         if (DBG_FORCE_RELOC == FORCE_GTT_RELOC)
 573                 return false;
 574 
 575         return (cache->has_llc ||
 576                 obj->cache_dirty ||
 577                 obj->cache_level != I915_CACHE_NONE);
 578 }
 579 
 580 static int eb_reserve_vma(const struct i915_execbuffer *eb,
 581                           struct i915_vma *vma)
 582 {
 583         struct drm_i915_gem_exec_object2 *entry = exec_entry(eb, vma);
 584         unsigned int exec_flags = *vma->exec_flags;
 585         u64 pin_flags;
 586         int err;
 587 
 588         pin_flags = PIN_USER | PIN_NONBLOCK;
 589         if (exec_flags & EXEC_OBJECT_NEEDS_GTT)
 590                 pin_flags |= PIN_GLOBAL;
 591 
 592         /*
 593          * Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
 594          * limit address to the first 4GBs for unflagged objects.
 595          */
 596         if (!(exec_flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
 597                 pin_flags |= PIN_ZONE_4G;
 598 
 599         if (exec_flags & __EXEC_OBJECT_NEEDS_MAP)
 600                 pin_flags |= PIN_MAPPABLE;
 601 
 602         if (exec_flags & EXEC_OBJECT_PINNED) {
 603                 pin_flags |= entry->offset | PIN_OFFSET_FIXED;
 604                 pin_flags &= ~PIN_NONBLOCK; /* force overlapping checks */
 605         } else if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS) {
 606                 pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
 607         }
 608 
 609         err = i915_vma_pin(vma,
 610                            entry->pad_to_size, entry->alignment,
 611                            pin_flags);
 612         if (err)
 613                 return err;
 614 
 615         if (entry->offset != vma->node.start) {
 616                 entry->offset = vma->node.start | UPDATE;
 617                 eb->args->flags |= __EXEC_HAS_RELOC;
 618         }
 619 
 620         if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) {
 621                 err = i915_vma_pin_fence(vma);
 622                 if (unlikely(err)) {
 623                         i915_vma_unpin(vma);
 624                         return err;
 625                 }
 626 
 627                 if (vma->fence)
 628                         exec_flags |= __EXEC_OBJECT_HAS_FENCE;
 629         }
 630 
 631         *vma->exec_flags = exec_flags | __EXEC_OBJECT_HAS_PIN;
 632         GEM_BUG_ON(eb_vma_misplaced(entry, vma, exec_flags));
 633 
 634         return 0;
 635 }
 636 
 637 static int eb_reserve(struct i915_execbuffer *eb)
 638 {
 639         const unsigned int count = eb->buffer_count;
 640         struct list_head last;
 641         struct i915_vma *vma;
 642         unsigned int i, pass;
 643         int err;
 644 
 645         /*
 646          * Attempt to pin all of the buffers into the GTT.
 647          * This is done in 3 phases:
 648          *
 649          * 1a. Unbind all objects that do not match the GTT constraints for
 650          *     the execbuffer (fenceable, mappable, alignment etc).
 651          * 1b. Increment pin count for already bound objects.
 652          * 2.  Bind new objects.
 653          * 3.  Decrement pin count.
 654          *
 655          * This avoid unnecessary unbinding of later objects in order to make
 656          * room for the earlier objects *unless* we need to defragment.
 657          */
 658 
 659         pass = 0;
 660         err = 0;
 661         do {
 662                 list_for_each_entry(vma, &eb->unbound, exec_link) {
 663                         err = eb_reserve_vma(eb, vma);
 664                         if (err)
 665                                 break;
 666                 }
 667                 if (err != -ENOSPC)
 668                         return err;
 669 
 670                 /* Resort *all* the objects into priority order */
 671                 INIT_LIST_HEAD(&eb->unbound);
 672                 INIT_LIST_HEAD(&last);
 673                 for (i = 0; i < count; i++) {
 674                         unsigned int flags = eb->flags[i];
 675                         struct i915_vma *vma = eb->vma[i];
 676 
 677                         if (flags & EXEC_OBJECT_PINNED &&
 678                             flags & __EXEC_OBJECT_HAS_PIN)
 679                                 continue;
 680 
 681                         eb_unreserve_vma(vma, &eb->flags[i]);
 682 
 683                         if (flags & EXEC_OBJECT_PINNED)
 684                                 /* Pinned must have their slot */
 685                                 list_add(&vma->exec_link, &eb->unbound);
 686                         else if (flags & __EXEC_OBJECT_NEEDS_MAP)
 687                                 /* Map require the lowest 256MiB (aperture) */
 688                                 list_add_tail(&vma->exec_link, &eb->unbound);
 689                         else if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
 690                                 /* Prioritise 4GiB region for restricted bo */
 691                                 list_add(&vma->exec_link, &last);
 692                         else
 693                                 list_add_tail(&vma->exec_link, &last);
 694                 }
 695                 list_splice_tail(&last, &eb->unbound);
 696 
 697                 switch (pass++) {
 698                 case 0:
 699                         break;
 700 
 701                 case 1:
 702                         /* Too fragmented, unbind everything and retry */
 703                         err = i915_gem_evict_vm(eb->context->vm);
 704                         if (err)
 705                                 return err;
 706                         break;
 707 
 708                 default:
 709                         return -ENOSPC;
 710                 }
 711         } while (1);
 712 }
 713 
 714 static unsigned int eb_batch_index(const struct i915_execbuffer *eb)
 715 {
 716         if (eb->args->flags & I915_EXEC_BATCH_FIRST)
 717                 return 0;
 718         else
 719                 return eb->buffer_count - 1;
 720 }
 721 
 722 static int eb_select_context(struct i915_execbuffer *eb)
 723 {
 724         struct i915_gem_context *ctx;
 725 
 726         ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1);
 727         if (unlikely(!ctx))
 728                 return -ENOENT;
 729 
 730         eb->gem_context = ctx;
 731         if (ctx->vm)
 732                 eb->invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
 733 
 734         eb->context_flags = 0;
 735         if (test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags))
 736                 eb->context_flags |= __EXEC_OBJECT_NEEDS_BIAS;
 737 
 738         return 0;
 739 }
 740 
 741 static int eb_lookup_vmas(struct i915_execbuffer *eb)
 742 {
 743         struct radix_tree_root *handles_vma = &eb->gem_context->handles_vma;
 744         struct drm_i915_gem_object *obj;
 745         unsigned int i, batch;
 746         int err;
 747 
 748         if (unlikely(i915_gem_context_is_banned(eb->gem_context)))
 749                 return -EIO;
 750 
 751         INIT_LIST_HEAD(&eb->relocs);
 752         INIT_LIST_HEAD(&eb->unbound);
 753 
 754         batch = eb_batch_index(eb);
 755 
 756         mutex_lock(&eb->gem_context->mutex);
 757         if (unlikely(i915_gem_context_is_closed(eb->gem_context))) {
 758                 err = -ENOENT;
 759                 goto err_ctx;
 760         }
 761 
 762         for (i = 0; i < eb->buffer_count; i++) {
 763                 u32 handle = eb->exec[i].handle;
 764                 struct i915_lut_handle *lut;
 765                 struct i915_vma *vma;
 766 
 767                 vma = radix_tree_lookup(handles_vma, handle);
 768                 if (likely(vma))
 769                         goto add_vma;
 770 
 771                 obj = i915_gem_object_lookup(eb->file, handle);
 772                 if (unlikely(!obj)) {
 773                         err = -ENOENT;
 774                         goto err_vma;
 775                 }
 776 
 777                 vma = i915_vma_instance(obj, eb->context->vm, NULL);
 778                 if (IS_ERR(vma)) {
 779                         err = PTR_ERR(vma);
 780                         goto err_obj;
 781                 }
 782 
 783                 lut = i915_lut_handle_alloc();
 784                 if (unlikely(!lut)) {
 785                         err = -ENOMEM;
 786                         goto err_obj;
 787                 }
 788 
 789                 err = radix_tree_insert(handles_vma, handle, vma);
 790                 if (unlikely(err)) {
 791                         i915_lut_handle_free(lut);
 792                         goto err_obj;
 793                 }
 794 
 795                 /* transfer ref to lut */
 796                 if (!atomic_fetch_inc(&vma->open_count))
 797                         i915_vma_reopen(vma);
 798                 lut->handle = handle;
 799                 lut->ctx = eb->gem_context;
 800 
 801                 i915_gem_object_lock(obj);
 802                 list_add(&lut->obj_link, &obj->lut_list);
 803                 i915_gem_object_unlock(obj);
 804 
 805 add_vma:
 806                 err = eb_add_vma(eb, i, batch, vma);
 807                 if (unlikely(err))
 808                         goto err_vma;
 809 
 810                 GEM_BUG_ON(vma != eb->vma[i]);
 811                 GEM_BUG_ON(vma->exec_flags != &eb->flags[i]);
 812                 GEM_BUG_ON(drm_mm_node_allocated(&vma->node) &&
 813                            eb_vma_misplaced(&eb->exec[i], vma, eb->flags[i]));
 814         }
 815 
 816         mutex_unlock(&eb->gem_context->mutex);
 817 
 818         eb->args->flags |= __EXEC_VALIDATED;
 819         return eb_reserve(eb);
 820 
 821 err_obj:
 822         i915_gem_object_put(obj);
 823 err_vma:
 824         eb->vma[i] = NULL;
 825 err_ctx:
 826         mutex_unlock(&eb->gem_context->mutex);
 827         return err;
 828 }
 829 
 830 static struct i915_vma *
 831 eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle)
 832 {
 833         if (eb->lut_size < 0) {
 834                 if (handle >= -eb->lut_size)
 835                         return NULL;
 836                 return eb->vma[handle];
 837         } else {
 838                 struct hlist_head *head;
 839                 struct i915_vma *vma;
 840 
 841                 head = &eb->buckets[hash_32(handle, eb->lut_size)];
 842                 hlist_for_each_entry(vma, head, exec_node) {
 843                         if (vma->exec_handle == handle)
 844                                 return vma;
 845                 }
 846                 return NULL;
 847         }
 848 }
 849 
 850 static void eb_release_vmas(const struct i915_execbuffer *eb)
 851 {
 852         const unsigned int count = eb->buffer_count;
 853         unsigned int i;
 854 
 855         for (i = 0; i < count; i++) {
 856                 struct i915_vma *vma = eb->vma[i];
 857                 unsigned int flags = eb->flags[i];
 858 
 859                 if (!vma)
 860                         break;
 861 
 862                 GEM_BUG_ON(vma->exec_flags != &eb->flags[i]);
 863                 vma->exec_flags = NULL;
 864                 eb->vma[i] = NULL;
 865 
 866                 if (flags & __EXEC_OBJECT_HAS_PIN)
 867                         __eb_unreserve_vma(vma, flags);
 868 
 869                 if (flags & __EXEC_OBJECT_HAS_REF)
 870                         i915_vma_put(vma);
 871         }
 872 }
 873 
 874 static void eb_reset_vmas(const struct i915_execbuffer *eb)
 875 {
 876         eb_release_vmas(eb);
 877         if (eb->lut_size > 0)
 878                 memset(eb->buckets, 0,
 879                        sizeof(struct hlist_head) << eb->lut_size);
 880 }
 881 
 882 static void eb_destroy(const struct i915_execbuffer *eb)
 883 {
 884         GEM_BUG_ON(eb->reloc_cache.rq);
 885 
 886         if (eb->lut_size > 0)
 887                 kfree(eb->buckets);
 888 }
 889 
 890 static inline u64
 891 relocation_target(const struct drm_i915_gem_relocation_entry *reloc,
 892                   const struct i915_vma *target)
 893 {
 894         return gen8_canonical_addr((int)reloc->delta + target->node.start);
 895 }
 896 
 897 static void reloc_cache_init(struct reloc_cache *cache,
 898                              struct drm_i915_private *i915)
 899 {
 900         cache->page = -1;
 901         cache->vaddr = 0;
 902         /* Must be a variable in the struct to allow GCC to unroll. */
 903         cache->gen = INTEL_GEN(i915);
 904         cache->has_llc = HAS_LLC(i915);
 905         cache->use_64bit_reloc = HAS_64BIT_RELOC(i915);
 906         cache->has_fence = cache->gen < 4;
 907         cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment;
 908         cache->node.allocated = false;
 909         cache->rq = NULL;
 910         cache->rq_size = 0;
 911 }
 912 
 913 static inline void *unmask_page(unsigned long p)
 914 {
 915         return (void *)(uintptr_t)(p & PAGE_MASK);
 916 }
 917 
 918 static inline unsigned int unmask_flags(unsigned long p)
 919 {
 920         return p & ~PAGE_MASK;
 921 }
 922 
 923 #define KMAP 0x4 /* after CLFLUSH_FLAGS */
 924 
 925 static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache)
 926 {
 927         struct drm_i915_private *i915 =
 928                 container_of(cache, struct i915_execbuffer, reloc_cache)->i915;
 929         return &i915->ggtt;
 930 }
 931 
 932 static void reloc_gpu_flush(struct reloc_cache *cache)
 933 {
 934         struct drm_i915_gem_object *obj = cache->rq->batch->obj;
 935 
 936         GEM_BUG_ON(cache->rq_size >= obj->base.size / sizeof(u32));
 937         cache->rq_cmd[cache->rq_size] = MI_BATCH_BUFFER_END;
 938 
 939         __i915_gem_object_flush_map(obj, 0, sizeof(u32) * (cache->rq_size + 1));
 940         i915_gem_object_unpin_map(obj);
 941 
 942         intel_gt_chipset_flush(cache->rq->engine->gt);
 943 
 944         i915_request_add(cache->rq);
 945         cache->rq = NULL;
 946 }
 947 
 948 static void reloc_cache_reset(struct reloc_cache *cache)
 949 {
 950         void *vaddr;
 951 
 952         if (cache->rq)
 953                 reloc_gpu_flush(cache);
 954 
 955         if (!cache->vaddr)
 956                 return;
 957 
 958         vaddr = unmask_page(cache->vaddr);
 959         if (cache->vaddr & KMAP) {
 960                 if (cache->vaddr & CLFLUSH_AFTER)
 961                         mb();
 962 
 963                 kunmap_atomic(vaddr);
 964                 i915_gem_object_finish_access((struct drm_i915_gem_object *)cache->node.mm);
 965         } else {
 966                 struct i915_ggtt *ggtt = cache_to_ggtt(cache);
 967 
 968                 intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 969                 io_mapping_unmap_atomic((void __iomem *)vaddr);
 970 
 971                 if (cache->node.allocated) {
 972                         ggtt->vm.clear_range(&ggtt->vm,
 973                                              cache->node.start,
 974                                              cache->node.size);
 975                         drm_mm_remove_node(&cache->node);
 976                 } else {
 977                         i915_vma_unpin((struct i915_vma *)cache->node.mm);
 978                 }
 979         }
 980 
 981         cache->vaddr = 0;
 982         cache->page = -1;
 983 }
 984 
 985 static void *reloc_kmap(struct drm_i915_gem_object *obj,
 986                         struct reloc_cache *cache,
 987                         unsigned long page)
 988 {
 989         void *vaddr;
 990 
 991         if (cache->vaddr) {
 992                 kunmap_atomic(unmask_page(cache->vaddr));
 993         } else {
 994                 unsigned int flushes;
 995                 int err;
 996 
 997                 err = i915_gem_object_prepare_write(obj, &flushes);
 998                 if (err)
 999                         return ERR_PTR(err);
1000 
1001                 BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS);
1002                 BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & PAGE_MASK);
1003 
1004                 cache->vaddr = flushes | KMAP;
1005                 cache->node.mm = (void *)obj;
1006                 if (flushes)
1007                         mb();
1008         }
1009 
1010         vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj, page));
1011         cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr;
1012         cache->page = page;
1013 
1014         return vaddr;
1015 }
1016 
1017 static void *reloc_iomap(struct drm_i915_gem_object *obj,
1018                          struct reloc_cache *cache,
1019                          unsigned long page)
1020 {
1021         struct i915_ggtt *ggtt = cache_to_ggtt(cache);
1022         unsigned long offset;
1023         void *vaddr;
1024 
1025         if (cache->vaddr) {
1026                 intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1027                 io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr));
1028         } else {
1029                 struct i915_vma *vma;
1030                 int err;
1031 
1032                 if (i915_gem_object_is_tiled(obj))
1033                         return ERR_PTR(-EINVAL);
1034 
1035                 if (use_cpu_reloc(cache, obj))
1036                         return NULL;
1037 
1038                 i915_gem_object_lock(obj);
1039                 err = i915_gem_object_set_to_gtt_domain(obj, true);
1040                 i915_gem_object_unlock(obj);
1041                 if (err)
1042                         return ERR_PTR(err);
1043 
1044                 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1045                                                PIN_MAPPABLE |
1046                                                PIN_NONBLOCK /* NOWARN */ |
1047                                                PIN_NOEVICT);
1048                 if (IS_ERR(vma)) {
1049                         memset(&cache->node, 0, sizeof(cache->node));
1050                         err = drm_mm_insert_node_in_range
1051                                 (&ggtt->vm.mm, &cache->node,
1052                                  PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
1053                                  0, ggtt->mappable_end,
1054                                  DRM_MM_INSERT_LOW);
1055                         if (err) /* no inactive aperture space, use cpu reloc */
1056                                 return NULL;
1057                 } else {
1058                         cache->node.start = vma->node.start;
1059                         cache->node.mm = (void *)vma;
1060                 }
1061         }
1062 
1063         offset = cache->node.start;
1064         if (cache->node.allocated) {
1065                 ggtt->vm.insert_page(&ggtt->vm,
1066                                      i915_gem_object_get_dma_address(obj, page),
1067                                      offset, I915_CACHE_NONE, 0);
1068         } else {
1069                 offset += page << PAGE_SHIFT;
1070         }
1071 
1072         vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->iomap,
1073                                                          offset);
1074         cache->page = page;
1075         cache->vaddr = (unsigned long)vaddr;
1076 
1077         return vaddr;
1078 }
1079 
1080 static void *reloc_vaddr(struct drm_i915_gem_object *obj,
1081                          struct reloc_cache *cache,
1082                          unsigned long page)
1083 {
1084         void *vaddr;
1085 
1086         if (cache->page == page) {
1087                 vaddr = unmask_page(cache->vaddr);
1088         } else {
1089                 vaddr = NULL;
1090                 if ((cache->vaddr & KMAP) == 0)
1091                         vaddr = reloc_iomap(obj, cache, page);
1092                 if (!vaddr)
1093                         vaddr = reloc_kmap(obj, cache, page);
1094         }
1095 
1096         return vaddr;
1097 }
1098 
1099 static void clflush_write32(u32 *addr, u32 value, unsigned int flushes)
1100 {
1101         if (unlikely(flushes & (CLFLUSH_BEFORE | CLFLUSH_AFTER))) {
1102                 if (flushes & CLFLUSH_BEFORE) {
1103                         clflushopt(addr);
1104                         mb();
1105                 }
1106 
1107                 *addr = value;
1108 
1109                 /*
1110                  * Writes to the same cacheline are serialised by the CPU
1111                  * (including clflush). On the write path, we only require
1112                  * that it hits memory in an orderly fashion and place
1113                  * mb barriers at the start and end of the relocation phase
1114                  * to ensure ordering of clflush wrt to the system.
1115                  */
1116                 if (flushes & CLFLUSH_AFTER)
1117                         clflushopt(addr);
1118         } else
1119                 *addr = value;
1120 }
1121 
1122 static int reloc_move_to_gpu(struct i915_request *rq, struct i915_vma *vma)
1123 {
1124         struct drm_i915_gem_object *obj = vma->obj;
1125         int err;
1126 
1127         i915_vma_lock(vma);
1128 
1129         if (obj->cache_dirty & ~obj->cache_coherent)
1130                 i915_gem_clflush_object(obj, 0);
1131         obj->write_domain = 0;
1132 
1133         err = i915_request_await_object(rq, vma->obj, true);
1134         if (err == 0)
1135                 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
1136 
1137         i915_vma_unlock(vma);
1138 
1139         return err;
1140 }
1141 
1142 static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
1143                              struct i915_vma *vma,
1144                              unsigned int len)
1145 {
1146         struct reloc_cache *cache = &eb->reloc_cache;
1147         struct intel_engine_pool_node *pool;
1148         struct i915_request *rq;
1149         struct i915_vma *batch;
1150         u32 *cmd;
1151         int err;
1152 
1153         pool = intel_engine_pool_get(&eb->engine->pool, PAGE_SIZE);
1154         if (IS_ERR(pool))
1155                 return PTR_ERR(pool);
1156 
1157         cmd = i915_gem_object_pin_map(pool->obj,
1158                                       cache->has_llc ?
1159                                       I915_MAP_FORCE_WB :
1160                                       I915_MAP_FORCE_WC);
1161         if (IS_ERR(cmd)) {
1162                 err = PTR_ERR(cmd);
1163                 goto out_pool;
1164         }
1165 
1166         batch = i915_vma_instance(pool->obj, vma->vm, NULL);
1167         if (IS_ERR(batch)) {
1168                 err = PTR_ERR(batch);
1169                 goto err_unmap;
1170         }
1171 
1172         err = i915_vma_pin(batch, 0, 0, PIN_USER | PIN_NONBLOCK);
1173         if (err)
1174                 goto err_unmap;
1175 
1176         rq = i915_request_create(eb->context);
1177         if (IS_ERR(rq)) {
1178                 err = PTR_ERR(rq);
1179                 goto err_unpin;
1180         }
1181 
1182         err = intel_engine_pool_mark_active(pool, rq);
1183         if (err)
1184                 goto err_request;
1185 
1186         err = reloc_move_to_gpu(rq, vma);
1187         if (err)
1188                 goto err_request;
1189 
1190         err = eb->engine->emit_bb_start(rq,
1191                                         batch->node.start, PAGE_SIZE,
1192                                         cache->gen > 5 ? 0 : I915_DISPATCH_SECURE);
1193         if (err)
1194                 goto skip_request;
1195 
1196         i915_vma_lock(batch);
1197         err = i915_request_await_object(rq, batch->obj, false);
1198         if (err == 0)
1199                 err = i915_vma_move_to_active(batch, rq, 0);
1200         i915_vma_unlock(batch);
1201         if (err)
1202                 goto skip_request;
1203 
1204         rq->batch = batch;
1205         i915_vma_unpin(batch);
1206 
1207         cache->rq = rq;
1208         cache->rq_cmd = cmd;
1209         cache->rq_size = 0;
1210 
1211         /* Return with batch mapping (cmd) still pinned */
1212         goto out_pool;
1213 
1214 skip_request:
1215         i915_request_skip(rq, err);
1216 err_request:
1217         i915_request_add(rq);
1218 err_unpin:
1219         i915_vma_unpin(batch);
1220 err_unmap:
1221         i915_gem_object_unpin_map(pool->obj);
1222 out_pool:
1223         intel_engine_pool_put(pool);
1224         return err;
1225 }
1226 
1227 static u32 *reloc_gpu(struct i915_execbuffer *eb,
1228                       struct i915_vma *vma,
1229                       unsigned int len)
1230 {
1231         struct reloc_cache *cache = &eb->reloc_cache;
1232         u32 *cmd;
1233 
1234         if (cache->rq_size > PAGE_SIZE/sizeof(u32) - (len + 1))
1235                 reloc_gpu_flush(cache);
1236 
1237         if (unlikely(!cache->rq)) {
1238                 int err;
1239 
1240                 /* If we need to copy for the cmdparser, we will stall anyway */
1241                 if (eb_use_cmdparser(eb))
1242                         return ERR_PTR(-EWOULDBLOCK);
1243 
1244                 if (!intel_engine_can_store_dword(eb->engine))
1245                         return ERR_PTR(-ENODEV);
1246 
1247                 err = __reloc_gpu_alloc(eb, vma, len);
1248                 if (unlikely(err))
1249                         return ERR_PTR(err);
1250         }
1251 
1252         cmd = cache->rq_cmd + cache->rq_size;
1253         cache->rq_size += len;
1254 
1255         return cmd;
1256 }
1257 
1258 static u64
1259 relocate_entry(struct i915_vma *vma,
1260                const struct drm_i915_gem_relocation_entry *reloc,
1261                struct i915_execbuffer *eb,
1262                const struct i915_vma *target)
1263 {
1264         u64 offset = reloc->offset;
1265         u64 target_offset = relocation_target(reloc, target);
1266         bool wide = eb->reloc_cache.use_64bit_reloc;
1267         void *vaddr;
1268 
1269         if (!eb->reloc_cache.vaddr &&
1270             (DBG_FORCE_RELOC == FORCE_GPU_RELOC ||
1271              !dma_resv_test_signaled_rcu(vma->resv, true))) {
1272                 const unsigned int gen = eb->reloc_cache.gen;
1273                 unsigned int len;
1274                 u32 *batch;
1275                 u64 addr;
1276 
1277                 if (wide)
1278                         len = offset & 7 ? 8 : 5;
1279                 else if (gen >= 4)
1280                         len = 4;
1281                 else
1282                         len = 3;
1283 
1284                 batch = reloc_gpu(eb, vma, len);
1285                 if (IS_ERR(batch))
1286                         goto repeat;
1287 
1288                 addr = gen8_canonical_addr(vma->node.start + offset);
1289                 if (wide) {
1290                         if (offset & 7) {
1291                                 *batch++ = MI_STORE_DWORD_IMM_GEN4;
1292                                 *batch++ = lower_32_bits(addr);
1293                                 *batch++ = upper_32_bits(addr);
1294                                 *batch++ = lower_32_bits(target_offset);
1295 
1296                                 addr = gen8_canonical_addr(addr + 4);
1297 
1298                                 *batch++ = MI_STORE_DWORD_IMM_GEN4;
1299                                 *batch++ = lower_32_bits(addr);
1300                                 *batch++ = upper_32_bits(addr);
1301                                 *batch++ = upper_32_bits(target_offset);
1302                         } else {
1303                                 *batch++ = (MI_STORE_DWORD_IMM_GEN4 | (1 << 21)) + 1;
1304                                 *batch++ = lower_32_bits(addr);
1305                                 *batch++ = upper_32_bits(addr);
1306                                 *batch++ = lower_32_bits(target_offset);
1307                                 *batch++ = upper_32_bits(target_offset);
1308                         }
1309                 } else if (gen >= 6) {
1310                         *batch++ = MI_STORE_DWORD_IMM_GEN4;
1311                         *batch++ = 0;
1312                         *batch++ = addr;
1313                         *batch++ = target_offset;
1314                 } else if (gen >= 4) {
1315                         *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1316                         *batch++ = 0;
1317                         *batch++ = addr;
1318                         *batch++ = target_offset;
1319                 } else {
1320                         *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
1321                         *batch++ = addr;
1322                         *batch++ = target_offset;
1323                 }
1324 
1325                 goto out;
1326         }
1327 
1328 repeat:
1329         vaddr = reloc_vaddr(vma->obj, &eb->reloc_cache, offset >> PAGE_SHIFT);
1330         if (IS_ERR(vaddr))
1331                 return PTR_ERR(vaddr);
1332 
1333         clflush_write32(vaddr + offset_in_page(offset),
1334                         lower_32_bits(target_offset),
1335                         eb->reloc_cache.vaddr);
1336 
1337         if (wide) {
1338                 offset += sizeof(u32);
1339                 target_offset >>= 32;
1340                 wide = false;
1341                 goto repeat;
1342         }
1343 
1344 out:
1345         return target->node.start | UPDATE;
1346 }
1347 
1348 static u64
1349 eb_relocate_entry(struct i915_execbuffer *eb,
1350                   struct i915_vma *vma,
1351                   const struct drm_i915_gem_relocation_entry *reloc)
1352 {
1353         struct i915_vma *target;
1354         int err;
1355 
1356         /* we've already hold a reference to all valid objects */
1357         target = eb_get_vma(eb, reloc->target_handle);
1358         if (unlikely(!target))
1359                 return -ENOENT;
1360 
1361         /* Validate that the target is in a valid r/w GPU domain */
1362         if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
1363                 DRM_DEBUG("reloc with multiple write domains: "
1364                           "target %d offset %d "
1365                           "read %08x write %08x",
1366                           reloc->target_handle,
1367                           (int) reloc->offset,
1368                           reloc->read_domains,
1369                           reloc->write_domain);
1370                 return -EINVAL;
1371         }
1372         if (unlikely((reloc->write_domain | reloc->read_domains)
1373                      & ~I915_GEM_GPU_DOMAINS)) {
1374                 DRM_DEBUG("reloc with read/write non-GPU domains: "
1375                           "target %d offset %d "
1376                           "read %08x write %08x",
1377                           reloc->target_handle,
1378                           (int) reloc->offset,
1379                           reloc->read_domains,
1380                           reloc->write_domain);
1381                 return -EINVAL;
1382         }
1383 
1384         if (reloc->write_domain) {
1385                 *target->exec_flags |= EXEC_OBJECT_WRITE;
1386 
1387                 /*
1388                  * Sandybridge PPGTT errata: We need a global gtt mapping
1389                  * for MI and pipe_control writes because the gpu doesn't
1390                  * properly redirect them through the ppgtt for non_secure
1391                  * batchbuffers.
1392                  */
1393                 if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
1394                     IS_GEN(eb->i915, 6)) {
1395                         err = i915_vma_bind(target, target->obj->cache_level,
1396                                             PIN_GLOBAL);
1397                         if (WARN_ONCE(err,
1398                                       "Unexpected failure to bind target VMA!"))
1399                                 return err;
1400                 }
1401         }
1402 
1403         /*
1404          * If the relocation already has the right value in it, no
1405          * more work needs to be done.
1406          */
1407         if (!DBG_FORCE_RELOC &&
1408             gen8_canonical_addr(target->node.start) == reloc->presumed_offset)
1409                 return 0;
1410 
1411         /* Check that the relocation address is valid... */
1412         if (unlikely(reloc->offset >
1413                      vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) {
1414                 DRM_DEBUG("Relocation beyond object bounds: "
1415                           "target %d offset %d size %d.\n",
1416                           reloc->target_handle,
1417                           (int)reloc->offset,
1418                           (int)vma->size);
1419                 return -EINVAL;
1420         }
1421         if (unlikely(reloc->offset & 3)) {
1422                 DRM_DEBUG("Relocation not 4-byte aligned: "
1423                           "target %d offset %d.\n",
1424                           reloc->target_handle,
1425                           (int)reloc->offset);
1426                 return -EINVAL;
1427         }
1428 
1429         /*
1430          * If we write into the object, we need to force the synchronisation
1431          * barrier, either with an asynchronous clflush or if we executed the
1432          * patching using the GPU (though that should be serialised by the
1433          * timeline). To be completely sure, and since we are required to
1434          * do relocations we are already stalling, disable the user's opt
1435          * out of our synchronisation.
1436          */
1437         *vma->exec_flags &= ~EXEC_OBJECT_ASYNC;
1438 
1439         /* and update the user's relocation entry */
1440         return relocate_entry(vma, reloc, eb, target);
1441 }
1442 
1443 static int eb_relocate_vma(struct i915_execbuffer *eb, struct i915_vma *vma)
1444 {
1445 #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
1446         struct drm_i915_gem_relocation_entry stack[N_RELOC(512)];
1447         struct drm_i915_gem_relocation_entry __user *urelocs;
1448         const struct drm_i915_gem_exec_object2 *entry = exec_entry(eb, vma);
1449         unsigned int remain;
1450 
1451         urelocs = u64_to_user_ptr(entry->relocs_ptr);
1452         remain = entry->relocation_count;
1453         if (unlikely(remain > N_RELOC(ULONG_MAX)))
1454                 return -EINVAL;
1455 
1456         /*
1457          * We must check that the entire relocation array is safe
1458          * to read. However, if the array is not writable the user loses
1459          * the updated relocation values.
1460          */
1461         if (unlikely(!access_ok(urelocs, remain*sizeof(*urelocs))))
1462                 return -EFAULT;
1463 
1464         do {
1465                 struct drm_i915_gem_relocation_entry *r = stack;
1466                 unsigned int count =
1467                         min_t(unsigned int, remain, ARRAY_SIZE(stack));
1468                 unsigned int copied;
1469 
1470                 /*
1471                  * This is the fast path and we cannot handle a pagefault
1472                  * whilst holding the struct mutex lest the user pass in the
1473                  * relocations contained within a mmaped bo. For in such a case
1474                  * we, the page fault handler would call i915_gem_fault() and
1475                  * we would try to acquire the struct mutex again. Obviously
1476                  * this is bad and so lockdep complains vehemently.
1477                  */
1478                 pagefault_disable();
1479                 copied = __copy_from_user_inatomic(r, urelocs, count * sizeof(r[0]));
1480                 pagefault_enable();
1481                 if (unlikely(copied)) {
1482                         remain = -EFAULT;
1483                         goto out;
1484                 }
1485 
1486                 remain -= count;
1487                 do {
1488                         u64 offset = eb_relocate_entry(eb, vma, r);
1489 
1490                         if (likely(offset == 0)) {
1491                         } else if ((s64)offset < 0) {
1492                                 remain = (int)offset;
1493                                 goto out;
1494                         } else {
1495                                 /*
1496                                  * Note that reporting an error now
1497                                  * leaves everything in an inconsistent
1498                                  * state as we have *already* changed
1499                                  * the relocation value inside the
1500                                  * object. As we have not changed the
1501                                  * reloc.presumed_offset or will not
1502                                  * change the execobject.offset, on the
1503                                  * call we may not rewrite the value
1504                                  * inside the object, leaving it
1505                                  * dangling and causing a GPU hang. Unless
1506                                  * userspace dynamically rebuilds the
1507                                  * relocations on each execbuf rather than
1508                                  * presume a static tree.
1509                                  *
1510                                  * We did previously check if the relocations
1511                                  * were writable (access_ok), an error now
1512                                  * would be a strange race with mprotect,
1513                                  * having already demonstrated that we
1514                                  * can read from this userspace address.
1515                                  */
1516                                 offset = gen8_canonical_addr(offset & ~UPDATE);
1517                                 if (unlikely(__put_user(offset, &urelocs[r-stack].presumed_offset))) {
1518                                         remain = -EFAULT;
1519                                         goto out;
1520                                 }
1521                         }
1522                 } while (r++, --count);
1523                 urelocs += ARRAY_SIZE(stack);
1524         } while (remain);
1525 out:
1526         reloc_cache_reset(&eb->reloc_cache);
1527         return remain;
1528 }
1529 
1530 static int
1531 eb_relocate_vma_slow(struct i915_execbuffer *eb, struct i915_vma *vma)
1532 {
1533         const struct drm_i915_gem_exec_object2 *entry = exec_entry(eb, vma);
1534         struct drm_i915_gem_relocation_entry *relocs =
1535                 u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
1536         unsigned int i;
1537         int err;
1538 
1539         for (i = 0; i < entry->relocation_count; i++) {
1540                 u64 offset = eb_relocate_entry(eb, vma, &relocs[i]);
1541 
1542                 if ((s64)offset < 0) {
1543                         err = (int)offset;
1544                         goto err;
1545                 }
1546         }
1547         err = 0;
1548 err:
1549         reloc_cache_reset(&eb->reloc_cache);
1550         return err;
1551 }
1552 
1553 static int check_relocations(const struct drm_i915_gem_exec_object2 *entry)
1554 {
1555         const char __user *addr, *end;
1556         unsigned long size;
1557         char __maybe_unused c;
1558 
1559         size = entry->relocation_count;
1560         if (size == 0)
1561                 return 0;
1562 
1563         if (size > N_RELOC(ULONG_MAX))
1564                 return -EINVAL;
1565 
1566         addr = u64_to_user_ptr(entry->relocs_ptr);
1567         size *= sizeof(struct drm_i915_gem_relocation_entry);
1568         if (!access_ok(addr, size))
1569                 return -EFAULT;
1570 
1571         end = addr + size;
1572         for (; addr < end; addr += PAGE_SIZE) {
1573                 int err = __get_user(c, addr);
1574                 if (err)
1575                         return err;
1576         }
1577         return __get_user(c, end - 1);
1578 }
1579 
1580 static int eb_copy_relocations(const struct i915_execbuffer *eb)
1581 {
1582         struct drm_i915_gem_relocation_entry *relocs;
1583         const unsigned int count = eb->buffer_count;
1584         unsigned int i;
1585         int err;
1586 
1587         for (i = 0; i < count; i++) {
1588                 const unsigned int nreloc = eb->exec[i].relocation_count;
1589                 struct drm_i915_gem_relocation_entry __user *urelocs;
1590                 unsigned long size;
1591                 unsigned long copied;
1592 
1593                 if (nreloc == 0)
1594                         continue;
1595 
1596                 err = check_relocations(&eb->exec[i]);
1597                 if (err)
1598                         goto err;
1599 
1600                 urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr);
1601                 size = nreloc * sizeof(*relocs);
1602 
1603                 relocs = kvmalloc_array(size, 1, GFP_KERNEL);
1604                 if (!relocs) {
1605                         err = -ENOMEM;
1606                         goto err;
1607                 }
1608 
1609                 /* copy_from_user is limited to < 4GiB */
1610                 copied = 0;
1611                 do {
1612                         unsigned int len =
1613                                 min_t(u64, BIT_ULL(31), size - copied);
1614 
1615                         if (__copy_from_user((char *)relocs + copied,
1616                                              (char __user *)urelocs + copied,
1617                                              len))
1618                                 goto end;
1619 
1620                         copied += len;
1621                 } while (copied < size);
1622 
1623                 /*
1624                  * As we do not update the known relocation offsets after
1625                  * relocating (due to the complexities in lock handling),
1626                  * we need to mark them as invalid now so that we force the
1627                  * relocation processing next time. Just in case the target
1628                  * object is evicted and then rebound into its old
1629                  * presumed_offset before the next execbuffer - if that
1630                  * happened we would make the mistake of assuming that the
1631                  * relocations were valid.
1632                  */
1633                 if (!user_access_begin(urelocs, size))
1634                         goto end;
1635 
1636                 for (copied = 0; copied < nreloc; copied++)
1637                         unsafe_put_user(-1,
1638                                         &urelocs[copied].presumed_offset,
1639                                         end_user);
1640                 user_access_end();
1641 
1642                 eb->exec[i].relocs_ptr = (uintptr_t)relocs;
1643         }
1644 
1645         return 0;
1646 
1647 end_user:
1648         user_access_end();
1649 end:
1650         kvfree(relocs);
1651         err = -EFAULT;
1652 err:
1653         while (i--) {
1654                 relocs = u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr);
1655                 if (eb->exec[i].relocation_count)
1656                         kvfree(relocs);
1657         }
1658         return err;
1659 }
1660 
1661 static int eb_prefault_relocations(const struct i915_execbuffer *eb)
1662 {
1663         const unsigned int count = eb->buffer_count;
1664         unsigned int i;
1665 
1666         if (unlikely(i915_modparams.prefault_disable))
1667                 return 0;
1668 
1669         for (i = 0; i < count; i++) {
1670                 int err;
1671 
1672                 err = check_relocations(&eb->exec[i]);
1673                 if (err)
1674                         return err;
1675         }
1676 
1677         return 0;
1678 }
1679 
1680 static noinline int eb_relocate_slow(struct i915_execbuffer *eb)
1681 {
1682         struct drm_device *dev = &eb->i915->drm;
1683         bool have_copy = false;
1684         struct i915_vma *vma;
1685         int err = 0;
1686 
1687 repeat:
1688         if (signal_pending(current)) {
1689                 err = -ERESTARTSYS;
1690                 goto out;
1691         }
1692 
1693         /* We may process another execbuffer during the unlock... */
1694         eb_reset_vmas(eb);
1695         mutex_unlock(&dev->struct_mutex);
1696 
1697         /*
1698          * We take 3 passes through the slowpatch.
1699          *
1700          * 1 - we try to just prefault all the user relocation entries and
1701          * then attempt to reuse the atomic pagefault disabled fast path again.
1702          *
1703          * 2 - we copy the user entries to a local buffer here outside of the
1704          * local and allow ourselves to wait upon any rendering before
1705          * relocations
1706          *
1707          * 3 - we already have a local copy of the relocation entries, but
1708          * were interrupted (EAGAIN) whilst waiting for the objects, try again.
1709          */
1710         if (!err) {
1711                 err = eb_prefault_relocations(eb);
1712         } else if (!have_copy) {
1713                 err = eb_copy_relocations(eb);
1714                 have_copy = err == 0;
1715         } else {
1716                 cond_resched();
1717                 err = 0;
1718         }
1719         if (err) {
1720                 mutex_lock(&dev->struct_mutex);
1721                 goto out;
1722         }
1723 
1724         /* A frequent cause for EAGAIN are currently unavailable client pages */
1725         flush_workqueue(eb->i915->mm.userptr_wq);
1726 
1727         err = i915_mutex_lock_interruptible(dev);
1728         if (err) {
1729                 mutex_lock(&dev->struct_mutex);
1730                 goto out;
1731         }
1732 
1733         /* reacquire the objects */
1734         err = eb_lookup_vmas(eb);
1735         if (err)
1736                 goto err;
1737 
1738         GEM_BUG_ON(!eb->batch);
1739 
1740         list_for_each_entry(vma, &eb->relocs, reloc_link) {
1741                 if (!have_copy) {
1742                         pagefault_disable();
1743                         err = eb_relocate_vma(eb, vma);
1744                         pagefault_enable();
1745                         if (err)
1746                                 goto repeat;
1747                 } else {
1748                         err = eb_relocate_vma_slow(eb, vma);
1749                         if (err)
1750                                 goto err;
1751                 }
1752         }
1753 
1754         /*
1755          * Leave the user relocations as are, this is the painfully slow path,
1756          * and we want to avoid the complication of dropping the lock whilst
1757          * having buffers reserved in the aperture and so causing spurious
1758          * ENOSPC for random operations.
1759          */
1760 
1761 err:
1762         if (err == -EAGAIN)
1763                 goto repeat;
1764 
1765 out:
1766         if (have_copy) {
1767                 const unsigned int count = eb->buffer_count;
1768                 unsigned int i;
1769 
1770                 for (i = 0; i < count; i++) {
1771                         const struct drm_i915_gem_exec_object2 *entry =
1772                                 &eb->exec[i];
1773                         struct drm_i915_gem_relocation_entry *relocs;
1774 
1775                         if (!entry->relocation_count)
1776                                 continue;
1777 
1778                         relocs = u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
1779                         kvfree(relocs);
1780                 }
1781         }
1782 
1783         return err;
1784 }
1785 
1786 static int eb_relocate(struct i915_execbuffer *eb)
1787 {
1788         if (eb_lookup_vmas(eb))
1789                 goto slow;
1790 
1791         /* The objects are in their final locations, apply the relocations. */
1792         if (eb->args->flags & __EXEC_HAS_RELOC) {
1793                 struct i915_vma *vma;
1794 
1795                 list_for_each_entry(vma, &eb->relocs, reloc_link) {
1796                         if (eb_relocate_vma(eb, vma))
1797                                 goto slow;
1798                 }
1799         }
1800 
1801         return 0;
1802 
1803 slow:
1804         return eb_relocate_slow(eb);
1805 }
1806 
1807 static int eb_move_to_gpu(struct i915_execbuffer *eb)
1808 {
1809         const unsigned int count = eb->buffer_count;
1810         struct ww_acquire_ctx acquire;
1811         unsigned int i;
1812         int err = 0;
1813 
1814         ww_acquire_init(&acquire, &reservation_ww_class);
1815 
1816         for (i = 0; i < count; i++) {
1817                 struct i915_vma *vma = eb->vma[i];
1818 
1819                 err = ww_mutex_lock_interruptible(&vma->resv->lock, &acquire);
1820                 if (!err)
1821                         continue;
1822 
1823                 GEM_BUG_ON(err == -EALREADY); /* No duplicate vma */
1824 
1825                 if (err == -EDEADLK) {
1826                         GEM_BUG_ON(i == 0);
1827                         do {
1828                                 int j = i - 1;
1829 
1830                                 ww_mutex_unlock(&eb->vma[j]->resv->lock);
1831 
1832                                 swap(eb->flags[i], eb->flags[j]);
1833                                 swap(eb->vma[i],  eb->vma[j]);
1834                                 eb->vma[i]->exec_flags = &eb->flags[i];
1835                         } while (--i);
1836                         GEM_BUG_ON(vma != eb->vma[0]);
1837                         vma->exec_flags = &eb->flags[0];
1838 
1839                         err = ww_mutex_lock_slow_interruptible(&vma->resv->lock,
1840                                                                &acquire);
1841                 }
1842                 if (err)
1843                         break;
1844         }
1845         ww_acquire_done(&acquire);
1846 
1847         while (i--) {
1848                 unsigned int flags = eb->flags[i];
1849                 struct i915_vma *vma = eb->vma[i];
1850                 struct drm_i915_gem_object *obj = vma->obj;
1851 
1852                 assert_vma_held(vma);
1853 
1854                 if (flags & EXEC_OBJECT_CAPTURE) {
1855                         struct i915_capture_list *capture;
1856 
1857                         capture = kmalloc(sizeof(*capture), GFP_KERNEL);
1858                         if (capture) {
1859                                 capture->next = eb->request->capture_list;
1860                                 capture->vma = vma;
1861                                 eb->request->capture_list = capture;
1862                         }
1863                 }
1864 
1865                 /*
1866                  * If the GPU is not _reading_ through the CPU cache, we need
1867                  * to make sure that any writes (both previous GPU writes from
1868                  * before a change in snooping levels and normal CPU writes)
1869                  * caught in that cache are flushed to main memory.
1870                  *
1871                  * We want to say
1872                  *   obj->cache_dirty &&
1873                  *   !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)
1874                  * but gcc's optimiser doesn't handle that as well and emits
1875                  * two jumps instead of one. Maybe one day...
1876                  */
1877                 if (unlikely(obj->cache_dirty & ~obj->cache_coherent)) {
1878                         if (i915_gem_clflush_object(obj, 0))
1879                                 flags &= ~EXEC_OBJECT_ASYNC;
1880                 }
1881 
1882                 if (err == 0 && !(flags & EXEC_OBJECT_ASYNC)) {
1883                         err = i915_request_await_object
1884                                 (eb->request, obj, flags & EXEC_OBJECT_WRITE);
1885                 }
1886 
1887                 if (err == 0)
1888                         err = i915_vma_move_to_active(vma, eb->request, flags);
1889 
1890                 i915_vma_unlock(vma);
1891 
1892                 __eb_unreserve_vma(vma, flags);
1893                 vma->exec_flags = NULL;
1894 
1895                 if (unlikely(flags & __EXEC_OBJECT_HAS_REF))
1896                         i915_vma_put(vma);
1897         }
1898         ww_acquire_fini(&acquire);
1899 
1900         if (unlikely(err))
1901                 goto err_skip;
1902 
1903         eb->exec = NULL;
1904 
1905         /* Unconditionally flush any chipset caches (for streaming writes). */
1906         intel_gt_chipset_flush(eb->engine->gt);
1907         return 0;
1908 
1909 err_skip:
1910         i915_request_skip(eb->request, err);
1911         return err;
1912 }
1913 
1914 static bool i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
1915 {
1916         if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS)
1917                 return false;
1918 
1919         /* Kernel clipping was a DRI1 misfeature */
1920         if (!(exec->flags & I915_EXEC_FENCE_ARRAY)) {
1921                 if (exec->num_cliprects || exec->cliprects_ptr)
1922                         return false;
1923         }
1924 
1925         if (exec->DR4 == 0xffffffff) {
1926                 DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
1927                 exec->DR4 = 0;
1928         }
1929         if (exec->DR1 || exec->DR4)
1930                 return false;
1931 
1932         if ((exec->batch_start_offset | exec->batch_len) & 0x7)
1933                 return false;
1934 
1935         return true;
1936 }
1937 
1938 static int i915_reset_gen7_sol_offsets(struct i915_request *rq)
1939 {
1940         u32 *cs;
1941         int i;
1942 
1943         if (!IS_GEN(rq->i915, 7) || rq->engine->id != RCS0) {
1944                 DRM_DEBUG("sol reset is gen7/rcs only\n");
1945                 return -EINVAL;
1946         }
1947 
1948         cs = intel_ring_begin(rq, 4 * 2 + 2);
1949         if (IS_ERR(cs))
1950                 return PTR_ERR(cs);
1951 
1952         *cs++ = MI_LOAD_REGISTER_IMM(4);
1953         for (i = 0; i < 4; i++) {
1954                 *cs++ = i915_mmio_reg_offset(GEN7_SO_WRITE_OFFSET(i));
1955                 *cs++ = 0;
1956         }
1957         *cs++ = MI_NOOP;
1958         intel_ring_advance(rq, cs);
1959 
1960         return 0;
1961 }
1962 
1963 static struct i915_vma *
1964 shadow_batch_pin(struct i915_execbuffer *eb, struct drm_i915_gem_object *obj)
1965 {
1966         struct drm_i915_private *dev_priv = eb->i915;
1967         struct i915_vma * const vma = *eb->vma;
1968         struct i915_address_space *vm;
1969         u64 flags;
1970 
1971         /*
1972          * PPGTT backed shadow buffers must be mapped RO, to prevent
1973          * post-scan tampering
1974          */
1975         if (CMDPARSER_USES_GGTT(dev_priv)) {
1976                 flags = PIN_GLOBAL;
1977                 vm = &dev_priv->ggtt.vm;
1978         } else if (vma->vm->has_read_only) {
1979                 flags = PIN_USER;
1980                 vm = vma->vm;
1981                 i915_gem_object_set_readonly(obj);
1982         } else {
1983                 DRM_DEBUG("Cannot prevent post-scan tampering without RO capable vm\n");
1984                 return ERR_PTR(-EINVAL);
1985         }
1986 
1987         return i915_gem_object_pin(obj, vm, NULL, 0, 0, flags);
1988 }
1989 
1990 static struct i915_vma *eb_parse(struct i915_execbuffer *eb)
1991 {
1992         struct intel_engine_pool_node *pool;
1993         struct i915_vma *vma;
1994         u64 batch_start;
1995         u64 shadow_batch_start;
1996         int err;
1997 
1998         pool = intel_engine_pool_get(&eb->engine->pool, eb->batch_len);
1999         if (IS_ERR(pool))
2000                 return ERR_CAST(pool);
2001 
2002         vma = shadow_batch_pin(eb, pool->obj);
2003         if (IS_ERR(vma))
2004                 goto err;
2005 
2006         batch_start = gen8_canonical_addr(eb->batch->node.start) +
2007                       eb->batch_start_offset;
2008 
2009         shadow_batch_start = gen8_canonical_addr(vma->node.start);
2010 
2011         err = intel_engine_cmd_parser(eb->gem_context,
2012                                       eb->engine,
2013                                       eb->batch->obj,
2014                                       batch_start,
2015                                       eb->batch_start_offset,
2016                                       eb->batch_len,
2017                                       pool->obj,
2018                                       shadow_batch_start);
2019 
2020         if (err) {
2021                 i915_vma_unpin(vma);
2022 
2023                 /*
2024                  * Unsafe GGTT-backed buffers can still be submitted safely
2025                  * as non-secure.
2026                  * For PPGTT backing however, we have no choice but to forcibly
2027                  * reject unsafe buffers
2028                  */
2029                 if (CMDPARSER_USES_GGTT(eb->i915) && (err == -EACCES))
2030                         /* Execute original buffer non-secure */
2031                         vma = NULL;
2032                 else
2033                         vma = ERR_PTR(err);
2034                 goto err;
2035         }
2036 
2037         eb->vma[eb->buffer_count] = i915_vma_get(vma);
2038         eb->flags[eb->buffer_count] =
2039                 __EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_REF;
2040         vma->exec_flags = &eb->flags[eb->buffer_count];
2041         eb->buffer_count++;
2042 
2043         eb->batch_start_offset = 0;
2044         eb->batch = vma;
2045 
2046         if (CMDPARSER_USES_GGTT(eb->i915))
2047                 eb->batch_flags |= I915_DISPATCH_SECURE;
2048 
2049         /* eb->batch_len unchanged */
2050 
2051         vma->private = pool;
2052         return vma;
2053 
2054 err:
2055         intel_engine_pool_put(pool);
2056         return vma;
2057 }
2058 
2059 static void
2060 add_to_client(struct i915_request *rq, struct drm_file *file)
2061 {
2062         struct drm_i915_file_private *file_priv = file->driver_priv;
2063 
2064         rq->file_priv = file_priv;
2065 
2066         spin_lock(&file_priv->mm.lock);
2067         list_add_tail(&rq->client_link, &file_priv->mm.request_list);
2068         spin_unlock(&file_priv->mm.lock);
2069 }
2070 
2071 static int eb_submit(struct i915_execbuffer *eb)
2072 {
2073         int err;
2074 
2075         err = eb_move_to_gpu(eb);
2076         if (err)
2077                 return err;
2078 
2079         if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) {
2080                 err = i915_reset_gen7_sol_offsets(eb->request);
2081                 if (err)
2082                         return err;
2083         }
2084 
2085         /*
2086          * After we completed waiting for other engines (using HW semaphores)
2087          * then we can signal that this request/batch is ready to run. This
2088          * allows us to determine if the batch is still waiting on the GPU
2089          * or actually running by checking the breadcrumb.
2090          */
2091         if (eb->engine->emit_init_breadcrumb) {
2092                 err = eb->engine->emit_init_breadcrumb(eb->request);
2093                 if (err)
2094                         return err;
2095         }
2096 
2097         err = eb->engine->emit_bb_start(eb->request,
2098                                         eb->batch->node.start +
2099                                         eb->batch_start_offset,
2100                                         eb->batch_len,
2101                                         eb->batch_flags);
2102         if (err)
2103                 return err;
2104 
2105         return 0;
2106 }
2107 
2108 static int num_vcs_engines(const struct drm_i915_private *i915)
2109 {
2110         return hweight64(INTEL_INFO(i915)->engine_mask &
2111                          GENMASK_ULL(VCS0 + I915_MAX_VCS - 1, VCS0));
2112 }
2113 
2114 /*
2115  * Find one BSD ring to dispatch the corresponding BSD command.
2116  * The engine index is returned.
2117  */
2118 static unsigned int
2119 gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
2120                          struct drm_file *file)
2121 {
2122         struct drm_i915_file_private *file_priv = file->driver_priv;
2123 
2124         /* Check whether the file_priv has already selected one ring. */
2125         if ((int)file_priv->bsd_engine < 0)
2126                 file_priv->bsd_engine =
2127                         get_random_int() % num_vcs_engines(dev_priv);
2128 
2129         return file_priv->bsd_engine;
2130 }
2131 
2132 static const enum intel_engine_id user_ring_map[] = {
2133         [I915_EXEC_DEFAULT]     = RCS0,
2134         [I915_EXEC_RENDER]      = RCS0,
2135         [I915_EXEC_BLT]         = BCS0,
2136         [I915_EXEC_BSD]         = VCS0,
2137         [I915_EXEC_VEBOX]       = VECS0
2138 };
2139 
2140 static struct i915_request *eb_throttle(struct intel_context *ce)
2141 {
2142         struct intel_ring *ring = ce->ring;
2143         struct intel_timeline *tl = ce->timeline;
2144         struct i915_request *rq;
2145 
2146         /*
2147          * Completely unscientific finger-in-the-air estimates for suitable
2148          * maximum user request size (to avoid blocking) and then backoff.
2149          */
2150         if (intel_ring_update_space(ring) >= PAGE_SIZE)
2151                 return NULL;
2152 
2153         /*
2154          * Find a request that after waiting upon, there will be at least half
2155          * the ring available. The hysteresis allows us to compete for the
2156          * shared ring and should mean that we sleep less often prior to
2157          * claiming our resources, but not so long that the ring completely
2158          * drains before we can submit our next request.
2159          */
2160         list_for_each_entry(rq, &tl->requests, link) {
2161                 if (rq->ring != ring)
2162                         continue;
2163 
2164                 if (__intel_ring_space(rq->postfix,
2165                                        ring->emit, ring->size) > ring->size / 2)
2166                         break;
2167         }
2168         if (&rq->link == &tl->requests)
2169                 return NULL; /* weird, we will check again later for real */
2170 
2171         return i915_request_get(rq);
2172 }
2173 
2174 static int
2175 __eb_pin_context(struct i915_execbuffer *eb, struct intel_context *ce)
2176 {
2177         int err;
2178 
2179         if (likely(atomic_inc_not_zero(&ce->pin_count)))
2180                 return 0;
2181 
2182         err = mutex_lock_interruptible(&eb->i915->drm.struct_mutex);
2183         if (err)
2184                 return err;
2185 
2186         err = __intel_context_do_pin(ce);
2187         mutex_unlock(&eb->i915->drm.struct_mutex);
2188 
2189         return err;
2190 }
2191 
2192 static void
2193 __eb_unpin_context(struct i915_execbuffer *eb, struct intel_context *ce)
2194 {
2195         if (likely(atomic_add_unless(&ce->pin_count, -1, 1)))
2196                 return;
2197 
2198         mutex_lock(&eb->i915->drm.struct_mutex);
2199         intel_context_unpin(ce);
2200         mutex_unlock(&eb->i915->drm.struct_mutex);
2201 }
2202 
2203 static int __eb_pin_engine(struct i915_execbuffer *eb, struct intel_context *ce)
2204 {
2205         struct intel_timeline *tl;
2206         struct i915_request *rq;
2207         int err;
2208 
2209         /*
2210          * ABI: Before userspace accesses the GPU (e.g. execbuffer), report
2211          * EIO if the GPU is already wedged.
2212          */
2213         err = intel_gt_terminally_wedged(ce->engine->gt);
2214         if (err)
2215                 return err;
2216 
2217         /*
2218          * Pinning the contexts may generate requests in order to acquire
2219          * GGTT space, so do this first before we reserve a seqno for
2220          * ourselves.
2221          */
2222         err = __eb_pin_context(eb, ce);
2223         if (err)
2224                 return err;
2225 
2226         /*
2227          * Take a local wakeref for preparing to dispatch the execbuf as
2228          * we expect to access the hardware fairly frequently in the
2229          * process, and require the engine to be kept awake between accesses.
2230          * Upon dispatch, we acquire another prolonged wakeref that we hold
2231          * until the timeline is idle, which in turn releases the wakeref
2232          * taken on the engine, and the parent device.
2233          */
2234         tl = intel_context_timeline_lock(ce);
2235         if (IS_ERR(tl)) {
2236                 err = PTR_ERR(tl);
2237                 goto err_unpin;
2238         }
2239 
2240         intel_context_enter(ce);
2241         rq = eb_throttle(ce);
2242 
2243         intel_context_timeline_unlock(tl);
2244 
2245         if (rq) {
2246                 if (i915_request_wait(rq,
2247                                       I915_WAIT_INTERRUPTIBLE,
2248                                       MAX_SCHEDULE_TIMEOUT) < 0) {
2249                         i915_request_put(rq);
2250                         err = -EINTR;
2251                         goto err_exit;
2252                 }
2253 
2254                 i915_request_put(rq);
2255         }
2256 
2257         eb->engine = ce->engine;
2258         eb->context = ce;
2259         return 0;
2260 
2261 err_exit:
2262         mutex_lock(&tl->mutex);
2263         intel_context_exit(ce);
2264         intel_context_timeline_unlock(tl);
2265 err_unpin:
2266         __eb_unpin_context(eb, ce);
2267         return err;
2268 }
2269 
2270 static void eb_unpin_engine(struct i915_execbuffer *eb)
2271 {
2272         struct intel_context *ce = eb->context;
2273         struct intel_timeline *tl = ce->timeline;
2274 
2275         mutex_lock(&tl->mutex);
2276         intel_context_exit(ce);
2277         mutex_unlock(&tl->mutex);
2278 
2279         __eb_unpin_context(eb, ce);
2280 }
2281 
2282 static unsigned int
2283 eb_select_legacy_ring(struct i915_execbuffer *eb,
2284                       struct drm_file *file,
2285                       struct drm_i915_gem_execbuffer2 *args)
2286 {
2287         struct drm_i915_private *i915 = eb->i915;
2288         unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK;
2289 
2290         if (user_ring_id != I915_EXEC_BSD &&
2291             (args->flags & I915_EXEC_BSD_MASK)) {
2292                 DRM_DEBUG("execbuf with non bsd ring but with invalid "
2293                           "bsd dispatch flags: %d\n", (int)(args->flags));
2294                 return -1;
2295         }
2296 
2297         if (user_ring_id == I915_EXEC_BSD && num_vcs_engines(i915) > 1) {
2298                 unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;
2299 
2300                 if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
2301                         bsd_idx = gen8_dispatch_bsd_engine(i915, file);
2302                 } else if (bsd_idx >= I915_EXEC_BSD_RING1 &&
2303                            bsd_idx <= I915_EXEC_BSD_RING2) {
2304                         bsd_idx >>= I915_EXEC_BSD_SHIFT;
2305                         bsd_idx--;
2306                 } else {
2307                         DRM_DEBUG("execbuf with unknown bsd ring: %u\n",
2308                                   bsd_idx);
2309                         return -1;
2310                 }
2311 
2312                 return _VCS(bsd_idx);
2313         }
2314 
2315         if (user_ring_id >= ARRAY_SIZE(user_ring_map)) {
2316                 DRM_DEBUG("execbuf with unknown ring: %u\n", user_ring_id);
2317                 return -1;
2318         }
2319 
2320         return user_ring_map[user_ring_id];
2321 }
2322 
2323 static int
2324 eb_pin_engine(struct i915_execbuffer *eb,
2325               struct drm_file *file,
2326               struct drm_i915_gem_execbuffer2 *args)
2327 {
2328         struct intel_context *ce;
2329         unsigned int idx;
2330         int err;
2331 
2332         if (i915_gem_context_user_engines(eb->gem_context))
2333                 idx = args->flags & I915_EXEC_RING_MASK;
2334         else
2335                 idx = eb_select_legacy_ring(eb, file, args);
2336 
2337         ce = i915_gem_context_get_engine(eb->gem_context, idx);
2338         if (IS_ERR(ce))
2339                 return PTR_ERR(ce);
2340 
2341         err = __eb_pin_engine(eb, ce);
2342         intel_context_put(ce);
2343 
2344         return err;
2345 }
2346 
2347 static void
2348 __free_fence_array(struct drm_syncobj **fences, unsigned int n)
2349 {
2350         while (n--)
2351                 drm_syncobj_put(ptr_mask_bits(fences[n], 2));
2352         kvfree(fences);
2353 }
2354 
2355 static struct drm_syncobj **
2356 get_fence_array(struct drm_i915_gem_execbuffer2 *args,
2357                 struct drm_file *file)
2358 {
2359         const unsigned long nfences = args->num_cliprects;
2360         struct drm_i915_gem_exec_fence __user *user;
2361         struct drm_syncobj **fences;
2362         unsigned long n;
2363         int err;
2364 
2365         if (!(args->flags & I915_EXEC_FENCE_ARRAY))
2366                 return NULL;
2367 
2368         /* Check multiplication overflow for access_ok() and kvmalloc_array() */
2369         BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long));
2370         if (nfences > min_t(unsigned long,
2371                             ULONG_MAX / sizeof(*user),
2372                             SIZE_MAX / sizeof(*fences)))
2373                 return ERR_PTR(-EINVAL);
2374 
2375         user = u64_to_user_ptr(args->cliprects_ptr);
2376         if (!access_ok(user, nfences * sizeof(*user)))
2377                 return ERR_PTR(-EFAULT);
2378 
2379         fences = kvmalloc_array(nfences, sizeof(*fences),
2380                                 __GFP_NOWARN | GFP_KERNEL);
2381         if (!fences)
2382                 return ERR_PTR(-ENOMEM);
2383 
2384         for (n = 0; n < nfences; n++) {
2385                 struct drm_i915_gem_exec_fence fence;
2386                 struct drm_syncobj *syncobj;
2387 
2388                 if (__copy_from_user(&fence, user++, sizeof(fence))) {
2389                         err = -EFAULT;
2390                         goto err;
2391                 }
2392 
2393                 if (fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS) {
2394                         err = -EINVAL;
2395                         goto err;
2396                 }
2397 
2398                 syncobj = drm_syncobj_find(file, fence.handle);
2399                 if (!syncobj) {
2400                         DRM_DEBUG("Invalid syncobj handle provided\n");
2401                         err = -ENOENT;
2402                         goto err;
2403                 }
2404 
2405                 BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) &
2406                              ~__I915_EXEC_FENCE_UNKNOWN_FLAGS);
2407 
2408                 fences[n] = ptr_pack_bits(syncobj, fence.flags, 2);
2409         }
2410 
2411         return fences;
2412 
2413 err:
2414         __free_fence_array(fences, n);
2415         return ERR_PTR(err);
2416 }
2417 
2418 static void
2419 put_fence_array(struct drm_i915_gem_execbuffer2 *args,
2420                 struct drm_syncobj **fences)
2421 {
2422         if (fences)
2423                 __free_fence_array(fences, args->num_cliprects);
2424 }
2425 
2426 static int
2427 await_fence_array(struct i915_execbuffer *eb,
2428                   struct drm_syncobj **fences)
2429 {
2430         const unsigned int nfences = eb->args->num_cliprects;
2431         unsigned int n;
2432         int err;
2433 
2434         for (n = 0; n < nfences; n++) {
2435                 struct drm_syncobj *syncobj;
2436                 struct dma_fence *fence;
2437                 unsigned int flags;
2438 
2439                 syncobj = ptr_unpack_bits(fences[n], &flags, 2);
2440                 if (!(flags & I915_EXEC_FENCE_WAIT))
2441                         continue;
2442 
2443                 fence = drm_syncobj_fence_get(syncobj);
2444                 if (!fence)
2445                         return -EINVAL;
2446 
2447                 err = i915_request_await_dma_fence(eb->request, fence);
2448                 dma_fence_put(fence);
2449                 if (err < 0)
2450                         return err;
2451         }
2452 
2453         return 0;
2454 }
2455 
2456 static void
2457 signal_fence_array(struct i915_execbuffer *eb,
2458                    struct drm_syncobj **fences)
2459 {
2460         const unsigned int nfences = eb->args->num_cliprects;
2461         struct dma_fence * const fence = &eb->request->fence;
2462         unsigned int n;
2463 
2464         for (n = 0; n < nfences; n++) {
2465                 struct drm_syncobj *syncobj;
2466                 unsigned int flags;
2467 
2468                 syncobj = ptr_unpack_bits(fences[n], &flags, 2);
2469                 if (!(flags & I915_EXEC_FENCE_SIGNAL))
2470                         continue;
2471 
2472                 drm_syncobj_replace_fence(syncobj, fence);
2473         }
2474 }
2475 
2476 static int
2477 i915_gem_do_execbuffer(struct drm_device *dev,
2478                        struct drm_file *file,
2479                        struct drm_i915_gem_execbuffer2 *args,
2480                        struct drm_i915_gem_exec_object2 *exec,
2481                        struct drm_syncobj **fences)
2482 {
2483         struct drm_i915_private *i915 = to_i915(dev);
2484         struct i915_execbuffer eb;
2485         struct dma_fence *in_fence = NULL;
2486         struct dma_fence *exec_fence = NULL;
2487         struct sync_file *out_fence = NULL;
2488         int out_fence_fd = -1;
2489         int err;
2490 
2491         BUILD_BUG_ON(__EXEC_INTERNAL_FLAGS & ~__I915_EXEC_ILLEGAL_FLAGS);
2492         BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS &
2493                      ~__EXEC_OBJECT_UNKNOWN_FLAGS);
2494 
2495         eb.i915 = i915;
2496         eb.file = file;
2497         eb.args = args;
2498         if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC))
2499                 args->flags |= __EXEC_HAS_RELOC;
2500 
2501         eb.exec = exec;
2502         eb.vma = (struct i915_vma **)(exec + args->buffer_count + 1);
2503         eb.vma[0] = NULL;
2504         eb.flags = (unsigned int *)(eb.vma + args->buffer_count + 1);
2505 
2506         eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
2507         reloc_cache_init(&eb.reloc_cache, eb.i915);
2508 
2509         eb.buffer_count = args->buffer_count;
2510         eb.batch_start_offset = args->batch_start_offset;
2511         eb.batch_len = args->batch_len;
2512 
2513         eb.batch_flags = 0;
2514         if (args->flags & I915_EXEC_SECURE) {
2515                 if (INTEL_GEN(i915) >= 11)
2516                         return -ENODEV;
2517 
2518                 /* Return -EPERM to trigger fallback code on old binaries. */
2519                 if (!HAS_SECURE_BATCHES(i915))
2520                         return -EPERM;
2521 
2522                 if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
2523                         return -EPERM;
2524 
2525                 eb.batch_flags |= I915_DISPATCH_SECURE;
2526         }
2527         if (args->flags & I915_EXEC_IS_PINNED)
2528                 eb.batch_flags |= I915_DISPATCH_PINNED;
2529 
2530         if (args->flags & I915_EXEC_FENCE_IN) {
2531                 in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2));
2532                 if (!in_fence)
2533                         return -EINVAL;
2534         }
2535 
2536         if (args->flags & I915_EXEC_FENCE_SUBMIT) {
2537                 if (in_fence) {
2538                         err = -EINVAL;
2539                         goto err_in_fence;
2540                 }
2541 
2542                 exec_fence = sync_file_get_fence(lower_32_bits(args->rsvd2));
2543                 if (!exec_fence) {
2544                         err = -EINVAL;
2545                         goto err_in_fence;
2546                 }
2547         }
2548 
2549         if (args->flags & I915_EXEC_FENCE_OUT) {
2550                 out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
2551                 if (out_fence_fd < 0) {
2552                         err = out_fence_fd;
2553                         goto err_exec_fence;
2554                 }
2555         }
2556 
2557         err = eb_create(&eb);
2558         if (err)
2559                 goto err_out_fence;
2560 
2561         GEM_BUG_ON(!eb.lut_size);
2562 
2563         err = eb_select_context(&eb);
2564         if (unlikely(err))
2565                 goto err_destroy;
2566 
2567         err = eb_pin_engine(&eb, file, args);
2568         if (unlikely(err))
2569                 goto err_context;
2570 
2571         err = i915_mutex_lock_interruptible(dev);
2572         if (err)
2573                 goto err_engine;
2574 
2575         err = eb_relocate(&eb);
2576         if (err) {
2577                 /*
2578                  * If the user expects the execobject.offset and
2579                  * reloc.presumed_offset to be an exact match,
2580                  * as for using NO_RELOC, then we cannot update
2581                  * the execobject.offset until we have completed
2582                  * relocation.
2583                  */
2584                 args->flags &= ~__EXEC_HAS_RELOC;
2585                 goto err_vma;
2586         }
2587 
2588         if (unlikely(*eb.batch->exec_flags & EXEC_OBJECT_WRITE)) {
2589                 DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
2590                 err = -EINVAL;
2591                 goto err_vma;
2592         }
2593         if (eb.batch_start_offset > eb.batch->size ||
2594             eb.batch_len > eb.batch->size - eb.batch_start_offset) {
2595                 DRM_DEBUG("Attempting to use out-of-bounds batch\n");
2596                 err = -EINVAL;
2597                 goto err_vma;
2598         }
2599 
2600         if (eb.batch_len == 0)
2601                 eb.batch_len = eb.batch->size - eb.batch_start_offset;
2602 
2603         if (eb_use_cmdparser(&eb)) {
2604                 struct i915_vma *vma;
2605 
2606                 vma = eb_parse(&eb);
2607                 if (IS_ERR(vma)) {
2608                         err = PTR_ERR(vma);
2609                         goto err_vma;
2610                 }
2611         }
2612 
2613         /*
2614          * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
2615          * batch" bit. Hence we need to pin secure batches into the global gtt.
2616          * hsw should have this fixed, but bdw mucks it up again. */
2617         if (eb.batch_flags & I915_DISPATCH_SECURE) {
2618                 struct i915_vma *vma;
2619 
2620                 /*
2621                  * So on first glance it looks freaky that we pin the batch here
2622                  * outside of the reservation loop. But:
2623                  * - The batch is already pinned into the relevant ppgtt, so we
2624                  *   already have the backing storage fully allocated.
2625                  * - No other BO uses the global gtt (well contexts, but meh),
2626                  *   so we don't really have issues with multiple objects not
2627                  *   fitting due to fragmentation.
2628                  * So this is actually safe.
2629                  */
2630                 vma = i915_gem_object_ggtt_pin(eb.batch->obj, NULL, 0, 0, 0);
2631                 if (IS_ERR(vma)) {
2632                         err = PTR_ERR(vma);
2633                         goto err_vma;
2634                 }
2635 
2636                 eb.batch = vma;
2637         }
2638 
2639         /* All GPU relocation batches must be submitted prior to the user rq */
2640         GEM_BUG_ON(eb.reloc_cache.rq);
2641 
2642         /* Allocate a request for this batch buffer nice and early. */
2643         eb.request = i915_request_create(eb.context);
2644         if (IS_ERR(eb.request)) {
2645                 err = PTR_ERR(eb.request);
2646                 goto err_batch_unpin;
2647         }
2648 
2649         if (in_fence) {
2650                 err = i915_request_await_dma_fence(eb.request, in_fence);
2651                 if (err < 0)
2652                         goto err_request;
2653         }
2654 
2655         if (exec_fence) {
2656                 err = i915_request_await_execution(eb.request, exec_fence,
2657                                                    eb.engine->bond_execute);
2658                 if (err < 0)
2659                         goto err_request;
2660         }
2661 
2662         if (fences) {
2663                 err = await_fence_array(&eb, fences);
2664                 if (err)
2665                         goto err_request;
2666         }
2667 
2668         if (out_fence_fd != -1) {
2669                 out_fence = sync_file_create(&eb.request->fence);
2670                 if (!out_fence) {
2671                         err = -ENOMEM;
2672                         goto err_request;
2673                 }
2674         }
2675 
2676         /*
2677          * Whilst this request exists, batch_obj will be on the
2678          * active_list, and so will hold the active reference. Only when this
2679          * request is retired will the the batch_obj be moved onto the
2680          * inactive_list and lose its active reference. Hence we do not need
2681          * to explicitly hold another reference here.
2682          */
2683         eb.request->batch = eb.batch;
2684         if (eb.batch->private)
2685                 intel_engine_pool_mark_active(eb.batch->private, eb.request);
2686 
2687         trace_i915_request_queue(eb.request, eb.batch_flags);
2688         err = eb_submit(&eb);
2689 err_request:
2690         add_to_client(eb.request, file);
2691         i915_request_add(eb.request);
2692 
2693         if (fences)
2694                 signal_fence_array(&eb, fences);
2695 
2696         if (out_fence) {
2697                 if (err == 0) {
2698                         fd_install(out_fence_fd, out_fence->file);
2699                         args->rsvd2 &= GENMASK_ULL(31, 0); /* keep in-fence */
2700                         args->rsvd2 |= (u64)out_fence_fd << 32;
2701                         out_fence_fd = -1;
2702                 } else {
2703                         fput(out_fence->file);
2704                 }
2705         }
2706 
2707 err_batch_unpin:
2708         if (eb.batch_flags & I915_DISPATCH_SECURE)
2709                 i915_vma_unpin(eb.batch);
2710         if (eb.batch->private)
2711                 intel_engine_pool_put(eb.batch->private);
2712 err_vma:
2713         if (eb.exec)
2714                 eb_release_vmas(&eb);
2715         mutex_unlock(&dev->struct_mutex);
2716 err_engine:
2717         eb_unpin_engine(&eb);
2718 err_context:
2719         i915_gem_context_put(eb.gem_context);
2720 err_destroy:
2721         eb_destroy(&eb);
2722 err_out_fence:
2723         if (out_fence_fd != -1)
2724                 put_unused_fd(out_fence_fd);
2725 err_exec_fence:
2726         dma_fence_put(exec_fence);
2727 err_in_fence:
2728         dma_fence_put(in_fence);
2729         return err;
2730 }
2731 
2732 static size_t eb_element_size(void)
2733 {
2734         return (sizeof(struct drm_i915_gem_exec_object2) +
2735                 sizeof(struct i915_vma *) +
2736                 sizeof(unsigned int));
2737 }
2738 
2739 static bool check_buffer_count(size_t count)
2740 {
2741         const size_t sz = eb_element_size();
2742 
2743         /*
2744          * When using LUT_HANDLE, we impose a limit of INT_MAX for the lookup
2745          * array size (see eb_create()). Otherwise, we can accept an array as
2746          * large as can be addressed (though use large arrays at your peril)!
2747          */
2748 
2749         return !(count < 1 || count > INT_MAX || count > SIZE_MAX / sz - 1);
2750 }
2751 
2752 /*
2753  * Legacy execbuffer just creates an exec2 list from the original exec object
2754  * list array and passes it to the real function.
2755  */
2756 int
2757 i915_gem_execbuffer_ioctl(struct drm_device *dev, void *data,
2758                           struct drm_file *file)
2759 {
2760         struct drm_i915_gem_execbuffer *args = data;
2761         struct drm_i915_gem_execbuffer2 exec2;
2762         struct drm_i915_gem_exec_object *exec_list = NULL;
2763         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
2764         const size_t count = args->buffer_count;
2765         unsigned int i;
2766         int err;
2767 
2768         if (!check_buffer_count(count)) {
2769                 DRM_DEBUG("execbuf2 with %zd buffers\n", count);
2770                 return -EINVAL;
2771         }
2772 
2773         exec2.buffers_ptr = args->buffers_ptr;
2774         exec2.buffer_count = args->buffer_count;
2775         exec2.batch_start_offset = args->batch_start_offset;
2776         exec2.batch_len = args->batch_len;
2777         exec2.DR1 = args->DR1;
2778         exec2.DR4 = args->DR4;
2779         exec2.num_cliprects = args->num_cliprects;
2780         exec2.cliprects_ptr = args->cliprects_ptr;
2781         exec2.flags = I915_EXEC_RENDER;
2782         i915_execbuffer2_set_context_id(exec2, 0);
2783 
2784         if (!i915_gem_check_execbuffer(&exec2))
2785                 return -EINVAL;
2786 
2787         /* Copy in the exec list from userland */
2788         exec_list = kvmalloc_array(count, sizeof(*exec_list),
2789                                    __GFP_NOWARN | GFP_KERNEL);
2790         exec2_list = kvmalloc_array(count + 1, eb_element_size(),
2791                                     __GFP_NOWARN | GFP_KERNEL);
2792         if (exec_list == NULL || exec2_list == NULL) {
2793                 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
2794                           args->buffer_count);
2795                 kvfree(exec_list);
2796                 kvfree(exec2_list);
2797                 return -ENOMEM;
2798         }
2799         err = copy_from_user(exec_list,
2800                              u64_to_user_ptr(args->buffers_ptr),
2801                              sizeof(*exec_list) * count);
2802         if (err) {
2803                 DRM_DEBUG("copy %d exec entries failed %d\n",
2804                           args->buffer_count, err);
2805                 kvfree(exec_list);
2806                 kvfree(exec2_list);
2807                 return -EFAULT;
2808         }
2809 
2810         for (i = 0; i < args->buffer_count; i++) {
2811                 exec2_list[i].handle = exec_list[i].handle;
2812                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
2813                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
2814                 exec2_list[i].alignment = exec_list[i].alignment;
2815                 exec2_list[i].offset = exec_list[i].offset;
2816                 if (INTEL_GEN(to_i915(dev)) < 4)
2817                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
2818                 else
2819                         exec2_list[i].flags = 0;
2820         }
2821 
2822         err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list, NULL);
2823         if (exec2.flags & __EXEC_HAS_RELOC) {
2824                 struct drm_i915_gem_exec_object __user *user_exec_list =
2825                         u64_to_user_ptr(args->buffers_ptr);
2826 
2827                 /* Copy the new buffer offsets back to the user's exec list. */
2828                 for (i = 0; i < args->buffer_count; i++) {
2829                         if (!(exec2_list[i].offset & UPDATE))
2830                                 continue;
2831 
2832                         exec2_list[i].offset =
2833                                 gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
2834                         exec2_list[i].offset &= PIN_OFFSET_MASK;
2835                         if (__copy_to_user(&user_exec_list[i].offset,
2836                                            &exec2_list[i].offset,
2837                                            sizeof(user_exec_list[i].offset)))
2838                                 break;
2839                 }
2840         }
2841 
2842         kvfree(exec_list);
2843         kvfree(exec2_list);
2844         return err;
2845 }
2846 
2847 int
2848 i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data,
2849                            struct drm_file *file)
2850 {
2851         struct drm_i915_gem_execbuffer2 *args = data;
2852         struct drm_i915_gem_exec_object2 *exec2_list;
2853         struct drm_syncobj **fences = NULL;
2854         const size_t count = args->buffer_count;
2855         int err;
2856 
2857         if (!check_buffer_count(count)) {
2858                 DRM_DEBUG("execbuf2 with %zd buffers\n", count);
2859                 return -EINVAL;
2860         }
2861 
2862         if (!i915_gem_check_execbuffer(args))
2863                 return -EINVAL;
2864 
2865         /* Allocate an extra slot for use by the command parser */
2866         exec2_list = kvmalloc_array(count + 1, eb_element_size(),
2867                                     __GFP_NOWARN | GFP_KERNEL);
2868         if (exec2_list == NULL) {
2869                 DRM_DEBUG("Failed to allocate exec list for %zd buffers\n",
2870                           count);
2871                 return -ENOMEM;
2872         }
2873         if (copy_from_user(exec2_list,
2874                            u64_to_user_ptr(args->buffers_ptr),
2875                            sizeof(*exec2_list) * count)) {
2876                 DRM_DEBUG("copy %zd exec entries failed\n", count);
2877                 kvfree(exec2_list);
2878                 return -EFAULT;
2879         }
2880 
2881         if (args->flags & I915_EXEC_FENCE_ARRAY) {
2882                 fences = get_fence_array(args, file);
2883                 if (IS_ERR(fences)) {
2884                         kvfree(exec2_list);
2885                         return PTR_ERR(fences);
2886                 }
2887         }
2888 
2889         err = i915_gem_do_execbuffer(dev, file, args, exec2_list, fences);
2890 
2891         /*
2892          * Now that we have begun execution of the batchbuffer, we ignore
2893          * any new error after this point. Also given that we have already
2894          * updated the associated relocations, we try to write out the current
2895          * object locations irrespective of any error.
2896          */
2897         if (args->flags & __EXEC_HAS_RELOC) {
2898                 struct drm_i915_gem_exec_object2 __user *user_exec_list =
2899                         u64_to_user_ptr(args->buffers_ptr);
2900                 unsigned int i;
2901 
2902                 /* Copy the new buffer offsets back to the user's exec list. */
2903                 /*
2904                  * Note: count * sizeof(*user_exec_list) does not overflow,
2905                  * because we checked 'count' in check_buffer_count().
2906                  *
2907                  * And this range already got effectively checked earlier
2908                  * when we did the "copy_from_user()" above.
2909                  */
2910                 if (!user_access_begin(user_exec_list, count * sizeof(*user_exec_list)))
2911                         goto end;
2912 
2913                 for (i = 0; i < args->buffer_count; i++) {
2914                         if (!(exec2_list[i].offset & UPDATE))
2915                                 continue;
2916 
2917                         exec2_list[i].offset =
2918                                 gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
2919                         unsafe_put_user(exec2_list[i].offset,
2920                                         &user_exec_list[i].offset,
2921                                         end_user);
2922                 }
2923 end_user:
2924                 user_access_end();
2925 end:;
2926         }
2927 
2928         args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS;
2929         put_fence_array(args, fences);
2930         kvfree(exec2_list);
2931         return err;
2932 }