root/drivers/gpu/drm/v3d/v3d_sched.c

DEFINITIONS

1. to_v3d_job
2. to_bin_job
3. to_render_job
4. to_tfu_job
5. to_csd_job
6. v3d_job_free
7. v3d_job_dependency
8. v3d_bin_job_run
9. v3d_render_job_run
10. v3d_tfu_job_run
11. v3d_csd_job_run
12. v3d_cache_clean_job_run
13. v3d_gpu_reset_for_timeout
14. v3d_cl_job_timedout
15. v3d_bin_job_timedout
16. v3d_render_job_timedout
17. v3d_generic_job_timedout
18. v3d_csd_job_timedout
19. v3d_sched_init
20. v3d_sched_fini

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /* Copyright (C) 2018 Broadcom */
   3 
   4 /**
   5  * DOC: Broadcom V3D scheduling
   6  *
   7  * The shared DRM GPU scheduler is used to coordinate submitting jobs
   8  * to the hardware.  Each DRM fd (roughly a client process) gets its
   9  * own scheduler entity, which will process jobs in order.  The GPU
  10  * scheduler will round-robin between clients to submit the next job.
  11  *
  12  * For simplicity, and in order to keep latency low for interactive
  13  * jobs when bulk background jobs are queued up, we submit a new job
  14  * to the HW only when it has completed the last one, instead of
  15  * filling up the CT[01]Q FIFOs with jobs.  Similarly, we use
  16  * v3d_job_dependency() to manage the dependency between bin and
  17  * render, instead of having the clients submit jobs using the HW's
  18  * semaphores to interlock between them.
  19  */
  20 
  21 #include <linux/kthread.h>
  22 
  23 #include "v3d_drv.h"
  24 #include "v3d_regs.h"
  25 #include "v3d_trace.h"
  26 
  27 static struct v3d_job *
  28 to_v3d_job(struct drm_sched_job *sched_job)
  29 {
  30         return container_of(sched_job, struct v3d_job, base);
  31 }
  32 
  33 static struct v3d_bin_job *
  34 to_bin_job(struct drm_sched_job *sched_job)
  35 {
  36         return container_of(sched_job, struct v3d_bin_job, base.base);
  37 }
  38 
  39 static struct v3d_render_job *
  40 to_render_job(struct drm_sched_job *sched_job)
  41 {
  42         return container_of(sched_job, struct v3d_render_job, base.base);
  43 }
  44 
  45 static struct v3d_tfu_job *
  46 to_tfu_job(struct drm_sched_job *sched_job)
  47 {
  48         return container_of(sched_job, struct v3d_tfu_job, base.base);
  49 }
  50 
  51 static struct v3d_csd_job *
  52 to_csd_job(struct drm_sched_job *sched_job)
  53 {
  54         return container_of(sched_job, struct v3d_csd_job, base.base);
  55 }
  56 
  57 static void
  58 v3d_job_free(struct drm_sched_job *sched_job)
  59 {
  60         struct v3d_job *job = to_v3d_job(sched_job);
  61 
  62         drm_sched_job_cleanup(sched_job);
  63         v3d_job_put(job);
  64 }
  65 
  66 /**
  67  * Returns the fences that the job depends on, one by one.
  68  *
  69  * If placed in the scheduler's .dependency method, the corresponding
  70  * .run_job won't be called until all of them have been signaled.
  71  */
  72 static struct dma_fence *
  73 v3d_job_dependency(struct drm_sched_job *sched_job,
  74                    struct drm_sched_entity *s_entity)
  75 {
  76         struct v3d_job *job = to_v3d_job(sched_job);
  77 
  78         /* XXX: Wait on a fence for switching the GMP if necessary,
  79          * and then do so.
  80          */
  81 
  82         if (!xa_empty(&job->deps))
  83                 return xa_erase(&job->deps, job->last_dep++);
  84 
  85         return NULL;
  86 }
  87 
  88 static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
  89 {
  90         struct v3d_bin_job *job = to_bin_job(sched_job);
  91         struct v3d_dev *v3d = job->base.v3d;
  92         struct drm_device *dev = &v3d->drm;
  93         struct dma_fence *fence;
  94         unsigned long irqflags;
  95 
  96         if (unlikely(job->base.base.s_fence->finished.error))
  97                 return NULL;
  98 
  99         /* Lock required around bin_job update vs
 100          * v3d_overflow_mem_work().
 101          */
 102         spin_lock_irqsave(&v3d->job_lock, irqflags);
 103         v3d->bin_job = job;
 104         /* Clear out the overflow allocation, so we don't
 105          * reuse the overflow attached to a previous job.
 106          */
 107         V3D_CORE_WRITE(0, V3D_PTB_BPOS, 0);
 108         spin_unlock_irqrestore(&v3d->job_lock, irqflags);
 109 
 110         v3d_invalidate_caches(v3d);
 111 
 112         fence = v3d_fence_create(v3d, V3D_BIN);
 113         if (IS_ERR(fence))
 114                 return NULL;
 115 
 116         if (job->base.irq_fence)
 117                 dma_fence_put(job->base.irq_fence);
 118         job->base.irq_fence = dma_fence_get(fence);
 119 
 120         trace_v3d_submit_cl(dev, false, to_v3d_fence(fence)->seqno,
 121                             job->start, job->end);
 122 
 123         /* Set the current and end address of the control list.
 124          * Writing the end register is what starts the job.
 125          */
 126         if (job->qma) {
 127                 V3D_CORE_WRITE(0, V3D_CLE_CT0QMA, job->qma);
 128                 V3D_CORE_WRITE(0, V3D_CLE_CT0QMS, job->qms);
 129         }
 130         if (job->qts) {
 131                 V3D_CORE_WRITE(0, V3D_CLE_CT0QTS,
 132                                V3D_CLE_CT0QTS_ENABLE |
 133                                job->qts);
 134         }
 135         V3D_CORE_WRITE(0, V3D_CLE_CT0QBA, job->start);
 136         V3D_CORE_WRITE(0, V3D_CLE_CT0QEA, job->end);
 137 
 138         return fence;
 139 }
 140 
 141 static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
 142 {
 143         struct v3d_render_job *job = to_render_job(sched_job);
 144         struct v3d_dev *v3d = job->base.v3d;
 145         struct drm_device *dev = &v3d->drm;
 146         struct dma_fence *fence;
 147 
 148         if (unlikely(job->base.base.s_fence->finished.error))
 149                 return NULL;
 150 
 151         v3d->render_job = job;
 152 
 153         /* Can we avoid this flush?  We need to be careful of
 154          * scheduling, though -- imagine job0 rendering to texture and
 155          * job1 reading, and them being executed as bin0, bin1,
 156          * render0, render1, so that render1's flush at bin time
 157          * wasn't enough.
 158          */
 159         v3d_invalidate_caches(v3d);
 160 
 161         fence = v3d_fence_create(v3d, V3D_RENDER);
 162         if (IS_ERR(fence))
 163                 return NULL;
 164 
 165         if (job->base.irq_fence)
 166                 dma_fence_put(job->base.irq_fence);
 167         job->base.irq_fence = dma_fence_get(fence);
 168 
 169         trace_v3d_submit_cl(dev, true, to_v3d_fence(fence)->seqno,
 170                             job->start, job->end);
 171 
 172         /* XXX: Set the QCFG */
 173 
 174         /* Set the current and end address of the control list.
 175          * Writing the end register is what starts the job.
 176          */
 177         V3D_CORE_WRITE(0, V3D_CLE_CT1QBA, job->start);
 178         V3D_CORE_WRITE(0, V3D_CLE_CT1QEA, job->end);
 179 
 180         return fence;
 181 }
 182 
 183 static struct dma_fence *
 184 v3d_tfu_job_run(struct drm_sched_job *sched_job)
 185 {
 186         struct v3d_tfu_job *job = to_tfu_job(sched_job);
 187         struct v3d_dev *v3d = job->base.v3d;
 188         struct drm_device *dev = &v3d->drm;
 189         struct dma_fence *fence;
 190 
 191         fence = v3d_fence_create(v3d, V3D_TFU);
 192         if (IS_ERR(fence))
 193                 return NULL;
 194 
 195         v3d->tfu_job = job;
 196         if (job->base.irq_fence)
 197                 dma_fence_put(job->base.irq_fence);
 198         job->base.irq_fence = dma_fence_get(fence);
 199 
 200         trace_v3d_submit_tfu(dev, to_v3d_fence(fence)->seqno);
 201 
 202         V3D_WRITE(V3D_TFU_IIA, job->args.iia);
 203         V3D_WRITE(V3D_TFU_IIS, job->args.iis);
 204         V3D_WRITE(V3D_TFU_ICA, job->args.ica);
 205         V3D_WRITE(V3D_TFU_IUA, job->args.iua);
 206         V3D_WRITE(V3D_TFU_IOA, job->args.ioa);
 207         V3D_WRITE(V3D_TFU_IOS, job->args.ios);
 208         V3D_WRITE(V3D_TFU_COEF0, job->args.coef[0]);
 209         if (job->args.coef[0] & V3D_TFU_COEF0_USECOEF) {
 210                 V3D_WRITE(V3D_TFU_COEF1, job->args.coef[1]);
 211                 V3D_WRITE(V3D_TFU_COEF2, job->args.coef[2]);
 212                 V3D_WRITE(V3D_TFU_COEF3, job->args.coef[3]);
 213         }
 214         /* ICFG kicks off the job. */
 215         V3D_WRITE(V3D_TFU_ICFG, job->args.icfg | V3D_TFU_ICFG_IOC);
 216 
 217         return fence;
 218 }
 219 
 220 static struct dma_fence *
 221 v3d_csd_job_run(struct drm_sched_job *sched_job)
 222 {
 223         struct v3d_csd_job *job = to_csd_job(sched_job);
 224         struct v3d_dev *v3d = job->base.v3d;
 225         struct drm_device *dev = &v3d->drm;
 226         struct dma_fence *fence;
 227         int i;
 228 
 229         v3d->csd_job = job;
 230 
 231         v3d_invalidate_caches(v3d);
 232 
 233         fence = v3d_fence_create(v3d, V3D_CSD);
 234         if (IS_ERR(fence))
 235                 return NULL;
 236 
 237         if (job->base.irq_fence)
 238                 dma_fence_put(job->base.irq_fence);
 239         job->base.irq_fence = dma_fence_get(fence);
 240 
 241         trace_v3d_submit_csd(dev, to_v3d_fence(fence)->seqno);
 242 
 243         for (i = 1; i <= 6; i++)
 244                 V3D_CORE_WRITE(0, V3D_CSD_QUEUED_CFG0 + 4 * i, job->args.cfg[i]);
 245         /* CFG0 write kicks off the job. */
 246         V3D_CORE_WRITE(0, V3D_CSD_QUEUED_CFG0, job->args.cfg[0]);
 247 
 248         return fence;
 249 }
 250 
 251 static struct dma_fence *
 252 v3d_cache_clean_job_run(struct drm_sched_job *sched_job)
 253 {
 254         struct v3d_job *job = to_v3d_job(sched_job);
 255         struct v3d_dev *v3d = job->v3d;
 256 
 257         v3d_clean_caches(v3d);
 258 
 259         return NULL;
 260 }
 261 
 262 static void
 263 v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job)
 264 {
 265         enum v3d_queue q;
 266 
 267         mutex_lock(&v3d->reset_lock);
 268 
 269         /* block scheduler */
 270         for (q = 0; q < V3D_MAX_QUEUES; q++)
 271                 drm_sched_stop(&v3d->queue[q].sched, sched_job);
 272 
 273         if (sched_job)
 274                 drm_sched_increase_karma(sched_job);
 275 
 276         /* get the GPU back into the init state */
 277         v3d_reset(v3d);
 278 
 279         for (q = 0; q < V3D_MAX_QUEUES; q++)
 280                 drm_sched_resubmit_jobs(&v3d->queue[q].sched);
 281 
 282         /* Unblock schedulers and restart their jobs. */
 283         for (q = 0; q < V3D_MAX_QUEUES; q++) {
 284                 drm_sched_start(&v3d->queue[q].sched, true);
 285         }
 286 
 287         mutex_unlock(&v3d->reset_lock);
 288 }
 289 
 290 /* If the current address or return address have changed, then the GPU
 291  * has probably made progress and we should delay the reset.  This
 292  * could fail if the GPU got in an infinite loop in the CL, but that
 293  * is pretty unlikely outside of an i-g-t testcase.
 294  */
 295 static void
 296 v3d_cl_job_timedout(struct drm_sched_job *sched_job, enum v3d_queue q,
 297                     u32 *timedout_ctca, u32 *timedout_ctra)
 298 {
 299         struct v3d_job *job = to_v3d_job(sched_job);
 300         struct v3d_dev *v3d = job->v3d;
 301         u32 ctca = V3D_CORE_READ(0, V3D_CLE_CTNCA(q));
 302         u32 ctra = V3D_CORE_READ(0, V3D_CLE_CTNRA(q));
 303 
 304         if (*timedout_ctca != ctca || *timedout_ctra != ctra) {
 305                 *timedout_ctca = ctca;
 306                 *timedout_ctra = ctra;
 307                 return;
 308         }
 309 
 310         v3d_gpu_reset_for_timeout(v3d, sched_job);
 311 }
 312 
 313 static void
 314 v3d_bin_job_timedout(struct drm_sched_job *sched_job)
 315 {
 316         struct v3d_bin_job *job = to_bin_job(sched_job);
 317 
 318         v3d_cl_job_timedout(sched_job, V3D_BIN,
 319                             &job->timedout_ctca, &job->timedout_ctra);
 320 }
 321 
 322 static void
 323 v3d_render_job_timedout(struct drm_sched_job *sched_job)
 324 {
 325         struct v3d_render_job *job = to_render_job(sched_job);
 326 
 327         v3d_cl_job_timedout(sched_job, V3D_RENDER,
 328                             &job->timedout_ctca, &job->timedout_ctra);
 329 }
 330 
 331 static void
 332 v3d_generic_job_timedout(struct drm_sched_job *sched_job)
 333 {
 334         struct v3d_job *job = to_v3d_job(sched_job);
 335 
 336         v3d_gpu_reset_for_timeout(job->v3d, sched_job);
 337 }
 338 
 339 static void
 340 v3d_csd_job_timedout(struct drm_sched_job *sched_job)
 341 {
 342         struct v3d_csd_job *job = to_csd_job(sched_job);
 343         struct v3d_dev *v3d = job->base.v3d;
 344         u32 batches = V3D_CORE_READ(0, V3D_CSD_CURRENT_CFG4);
 345 
 346         /* If we've made progress, skip reset and let the timer get
 347          * rearmed.
 348          */
 349         if (job->timedout_batches != batches) {
 350                 job->timedout_batches = batches;
 351                 return;
 352         }
 353 
 354         v3d_gpu_reset_for_timeout(v3d, sched_job);
 355 }
 356 
 357 static const struct drm_sched_backend_ops v3d_bin_sched_ops = {
 358         .dependency = v3d_job_dependency,
 359         .run_job = v3d_bin_job_run,
 360         .timedout_job = v3d_bin_job_timedout,
 361         .free_job = v3d_job_free,
 362 };
 363 
 364 static const struct drm_sched_backend_ops v3d_render_sched_ops = {
 365         .dependency = v3d_job_dependency,
 366         .run_job = v3d_render_job_run,
 367         .timedout_job = v3d_render_job_timedout,
 368         .free_job = v3d_job_free,
 369 };
 370 
 371 static const struct drm_sched_backend_ops v3d_tfu_sched_ops = {
 372         .dependency = v3d_job_dependency,
 373         .run_job = v3d_tfu_job_run,
 374         .timedout_job = v3d_generic_job_timedout,
 375         .free_job = v3d_job_free,
 376 };
 377 
 378 static const struct drm_sched_backend_ops v3d_csd_sched_ops = {
 379         .dependency = v3d_job_dependency,
 380         .run_job = v3d_csd_job_run,
 381         .timedout_job = v3d_csd_job_timedout,
 382         .free_job = v3d_job_free
 383 };
 384 
 385 static const struct drm_sched_backend_ops v3d_cache_clean_sched_ops = {
 386         .dependency = v3d_job_dependency,
 387         .run_job = v3d_cache_clean_job_run,
 388         .timedout_job = v3d_generic_job_timedout,
 389         .free_job = v3d_job_free
 390 };
 391 
 392 int
 393 v3d_sched_init(struct v3d_dev *v3d)
 394 {
 395         int hw_jobs_limit = 1;
 396         int job_hang_limit = 0;
 397         int hang_limit_ms = 500;
 398         int ret;
 399 
 400         ret = drm_sched_init(&v3d->queue[V3D_BIN].sched,
 401                              &v3d_bin_sched_ops,
 402                              hw_jobs_limit, job_hang_limit,
 403                              msecs_to_jiffies(hang_limit_ms),
 404                              "v3d_bin");
 405         if (ret) {
 406                 dev_err(v3d->dev, "Failed to create bin scheduler: %d.", ret);
 407                 return ret;
 408         }
 409 
 410         ret = drm_sched_init(&v3d->queue[V3D_RENDER].sched,
 411                              &v3d_render_sched_ops,
 412                              hw_jobs_limit, job_hang_limit,
 413                              msecs_to_jiffies(hang_limit_ms),
 414                              "v3d_render");
 415         if (ret) {
 416                 dev_err(v3d->dev, "Failed to create render scheduler: %d.",
 417                         ret);
 418                 v3d_sched_fini(v3d);
 419                 return ret;
 420         }
 421 
 422         ret = drm_sched_init(&v3d->queue[V3D_TFU].sched,
 423                              &v3d_tfu_sched_ops,
 424                              hw_jobs_limit, job_hang_limit,
 425                              msecs_to_jiffies(hang_limit_ms),
 426                              "v3d_tfu");
 427         if (ret) {
 428                 dev_err(v3d->dev, "Failed to create TFU scheduler: %d.",
 429                         ret);
 430                 v3d_sched_fini(v3d);
 431                 return ret;
 432         }
 433 
 434         if (v3d_has_csd(v3d)) {
 435                 ret = drm_sched_init(&v3d->queue[V3D_CSD].sched,
 436                                      &v3d_csd_sched_ops,
 437                                      hw_jobs_limit, job_hang_limit,
 438                                      msecs_to_jiffies(hang_limit_ms),
 439                                      "v3d_csd");
 440                 if (ret) {
 441                         dev_err(v3d->dev, "Failed to create CSD scheduler: %d.",
 442                                 ret);
 443                         v3d_sched_fini(v3d);
 444                         return ret;
 445                 }
 446 
 447                 ret = drm_sched_init(&v3d->queue[V3D_CACHE_CLEAN].sched,
 448                                      &v3d_cache_clean_sched_ops,
 449                                      hw_jobs_limit, job_hang_limit,
 450                                      msecs_to_jiffies(hang_limit_ms),
 451                                      "v3d_cache_clean");
 452                 if (ret) {
 453                         dev_err(v3d->dev, "Failed to create CACHE_CLEAN scheduler: %d.",
 454                                 ret);
 455                         v3d_sched_fini(v3d);
 456                         return ret;
 457                 }
 458         }
 459 
 460         return 0;
 461 }
 462 
 463 void
 464 v3d_sched_fini(struct v3d_dev *v3d)
 465 {
 466         enum v3d_queue q;
 467 
 468         for (q = 0; q < V3D_MAX_QUEUES; q++) {
 469                 if (v3d->queue[q].sched.ready)
 470                         drm_sched_fini(&v3d->queue[q].sched);
 471         }
 472 }