root/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c

DEFINITIONS

1. irq_enable
2. irq_disable
3. __intel_breadcrumbs_disarm_irq
4. intel_engine_disarm_breadcrumbs
5. __request_completed
6. check_signal_order
7. __dma_fence_signal
8. __dma_fence_signal__timestamp
9. __dma_fence_signal__notify
10. intel_engine_breadcrumbs_irq
11. intel_engine_signal_breadcrumbs
12. signal_irq_work
13. __intel_breadcrumbs_arm_irq
14. intel_engine_init_breadcrumbs
15. intel_engine_reset_breadcrumbs
16. intel_engine_fini_breadcrumbs
17. i915_request_enable_breadcrumb
18. i915_request_cancel_breadcrumb
19. intel_engine_print_breadcrumbs

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24 
  25 #include <linux/kthread.h>
  26 #include <trace/events/dma_fence.h>
  27 #include <uapi/linux/sched/types.h>
  28 
  29 #include "i915_drv.h"
  30 #include "i915_trace.h"
  31 
  32 static void irq_enable(struct intel_engine_cs *engine)
  33 {
  34         if (!engine->irq_enable)
  35                 return;
  36 
  37         /* Caller disables interrupts */
  38         spin_lock(&engine->gt->irq_lock);
  39         engine->irq_enable(engine);
  40         spin_unlock(&engine->gt->irq_lock);
  41 }
  42 
  43 static void irq_disable(struct intel_engine_cs *engine)
  44 {
  45         if (!engine->irq_disable)
  46                 return;
  47 
  48         /* Caller disables interrupts */
  49         spin_lock(&engine->gt->irq_lock);
  50         engine->irq_disable(engine);
  51         spin_unlock(&engine->gt->irq_lock);
  52 }
  53 
  54 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
  55 {
  56         lockdep_assert_held(&b->irq_lock);
  57 
  58         GEM_BUG_ON(!b->irq_enabled);
  59         if (!--b->irq_enabled)
  60                 irq_disable(container_of(b,
  61                                          struct intel_engine_cs,
  62                                          breadcrumbs));
  63 
  64         b->irq_armed = false;
  65 }
  66 
  67 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
  68 {
  69         struct intel_breadcrumbs *b = &engine->breadcrumbs;
  70         unsigned long flags;
  71 
  72         if (!b->irq_armed)
  73                 return;
  74 
  75         spin_lock_irqsave(&b->irq_lock, flags);
  76         if (b->irq_armed)
  77                 __intel_breadcrumbs_disarm_irq(b);
  78         spin_unlock_irqrestore(&b->irq_lock, flags);
  79 }
  80 
  81 static inline bool __request_completed(const struct i915_request *rq)
  82 {
  83         return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);
  84 }
  85 
  86 __maybe_unused static bool
  87 check_signal_order(struct intel_context *ce, struct i915_request *rq)
  88 {
  89         if (!list_is_last(&rq->signal_link, &ce->signals) &&
  90             i915_seqno_passed(rq->fence.seqno,
  91                               list_next_entry(rq, signal_link)->fence.seqno))
  92                 return false;
  93 
  94         if (!list_is_first(&rq->signal_link, &ce->signals) &&
  95             i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno,
  96                               rq->fence.seqno))
  97                 return false;
  98 
  99         return true;
 100 }
 101 
 102 static bool
 103 __dma_fence_signal(struct dma_fence *fence)
 104 {
 105         return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
 106 }
 107 
 108 static void
 109 __dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp)
 110 {
 111         fence->timestamp = timestamp;
 112         set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
 113         trace_dma_fence_signaled(fence);
 114 }
 115 
 116 static void
 117 __dma_fence_signal__notify(struct dma_fence *fence,
 118                            const struct list_head *list)
 119 {
 120         struct dma_fence_cb *cur, *tmp;
 121 
 122         lockdep_assert_held(fence->lock);
 123         lockdep_assert_irqs_disabled();
 124 
 125         list_for_each_entry_safe(cur, tmp, list, node) {
 126                 INIT_LIST_HEAD(&cur->node);
 127                 cur->func(fence, cur);
 128         }
 129 }
 130 
 131 void intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine)
 132 {
 133         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 134         const ktime_t timestamp = ktime_get();
 135         struct intel_context *ce, *cn;
 136         struct list_head *pos, *next;
 137         LIST_HEAD(signal);
 138 
 139         spin_lock(&b->irq_lock);
 140 
 141         if (b->irq_armed && list_empty(&b->signalers))
 142                 __intel_breadcrumbs_disarm_irq(b);
 143 
 144         list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) {
 145                 GEM_BUG_ON(list_empty(&ce->signals));
 146 
 147                 list_for_each_safe(pos, next, &ce->signals) {
 148                         struct i915_request *rq =
 149                                 list_entry(pos, typeof(*rq), signal_link);
 150 
 151                         GEM_BUG_ON(!check_signal_order(ce, rq));
 152 
 153                         if (!__request_completed(rq))
 154                                 break;
 155 
 156                         GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL,
 157                                              &rq->fence.flags));
 158                         clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
 159 
 160                         if (!__dma_fence_signal(&rq->fence))
 161                                 continue;
 162 
 163                         /*
 164                          * Queue for execution after dropping the signaling
 165                          * spinlock as the callback chain may end up adding
 166                          * more signalers to the same context or engine.
 167                          */
 168                         i915_request_get(rq);
 169                         list_add_tail(&rq->signal_link, &signal);
 170                 }
 171 
 172                 /*
 173                  * We process the list deletion in bulk, only using a list_add
 174                  * (not list_move) above but keeping the status of
 175                  * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit.
 176                  */
 177                 if (!list_is_first(pos, &ce->signals)) {
 178                         /* Advance the list to the first incomplete request */
 179                         __list_del_many(&ce->signals, pos);
 180                         if (&ce->signals == pos) /* now empty */
 181                                 list_del_init(&ce->signal_link);
 182                 }
 183         }
 184 
 185         spin_unlock(&b->irq_lock);
 186 
 187         list_for_each_safe(pos, next, &signal) {
 188                 struct i915_request *rq =
 189                         list_entry(pos, typeof(*rq), signal_link);
 190                 struct list_head cb_list;
 191 
 192                 spin_lock(&rq->lock);
 193                 list_replace(&rq->fence.cb_list, &cb_list);
 194                 __dma_fence_signal__timestamp(&rq->fence, timestamp);
 195                 __dma_fence_signal__notify(&rq->fence, &cb_list);
 196                 spin_unlock(&rq->lock);
 197 
 198                 i915_request_put(rq);
 199         }
 200 }
 201 
 202 void intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine)
 203 {
 204         local_irq_disable();
 205         intel_engine_breadcrumbs_irq(engine);
 206         local_irq_enable();
 207 }
 208 
 209 static void signal_irq_work(struct irq_work *work)
 210 {
 211         struct intel_engine_cs *engine =
 212                 container_of(work, typeof(*engine), breadcrumbs.irq_work);
 213 
 214         intel_engine_breadcrumbs_irq(engine);
 215 }
 216 
 217 static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
 218 {
 219         struct intel_engine_cs *engine =
 220                 container_of(b, struct intel_engine_cs, breadcrumbs);
 221 
 222         lockdep_assert_held(&b->irq_lock);
 223         if (b->irq_armed)
 224                 return;
 225 
 226         /*
 227          * The breadcrumb irq will be disarmed on the interrupt after the
 228          * waiters are signaled. This gives us a single interrupt window in
 229          * which we can add a new waiter and avoid the cost of re-enabling
 230          * the irq.
 231          */
 232         b->irq_armed = true;
 233 
 234         /*
 235          * Since we are waiting on a request, the GPU should be busy
 236          * and should have its own rpm reference. This is tracked
 237          * by i915->gt.awake, we can forgo holding our own wakref
 238          * for the interrupt as before i915->gt.awake is released (when
 239          * the driver is idle) we disarm the breadcrumbs.
 240          */
 241 
 242         if (!b->irq_enabled++)
 243                 irq_enable(engine);
 244 }
 245 
 246 void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
 247 {
 248         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 249 
 250         spin_lock_init(&b->irq_lock);
 251         INIT_LIST_HEAD(&b->signalers);
 252 
 253         init_irq_work(&b->irq_work, signal_irq_work);
 254 }
 255 
 256 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
 257 {
 258         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 259         unsigned long flags;
 260 
 261         spin_lock_irqsave(&b->irq_lock, flags);
 262 
 263         if (b->irq_enabled)
 264                 irq_enable(engine);
 265         else
 266                 irq_disable(engine);
 267 
 268         spin_unlock_irqrestore(&b->irq_lock, flags);
 269 }
 270 
 271 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
 272 {
 273 }
 274 
 275 bool i915_request_enable_breadcrumb(struct i915_request *rq)
 276 {
 277         lockdep_assert_held(&rq->lock);
 278         lockdep_assert_irqs_disabled();
 279 
 280         if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) {
 281                 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
 282                 struct intel_context *ce = rq->hw_context;
 283                 struct list_head *pos;
 284 
 285                 spin_lock(&b->irq_lock);
 286                 GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags));
 287 
 288                 __intel_breadcrumbs_arm_irq(b);
 289 
 290                 /*
 291                  * We keep the seqno in retirement order, so we can break
 292                  * inside intel_engine_breadcrumbs_irq as soon as we've passed
 293                  * the last completed request (or seen a request that hasn't
 294                  * event started). We could iterate the timeline->requests list,
 295                  * but keeping a separate signalers_list has the advantage of
 296                  * hopefully being much smaller than the full list and so
 297                  * provides faster iteration and detection when there are no
 298                  * more interrupts required for this context.
 299                  *
 300                  * We typically expect to add new signalers in order, so we
 301                  * start looking for our insertion point from the tail of
 302                  * the list.
 303                  */
 304                 list_for_each_prev(pos, &ce->signals) {
 305                         struct i915_request *it =
 306                                 list_entry(pos, typeof(*it), signal_link);
 307 
 308                         if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno))
 309                                 break;
 310                 }
 311                 list_add(&rq->signal_link, pos);
 312                 if (pos == &ce->signals) /* catch transitions from empty list */
 313                         list_move_tail(&ce->signal_link, &b->signalers);
 314                 GEM_BUG_ON(!check_signal_order(ce, rq));
 315 
 316                 set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
 317                 spin_unlock(&b->irq_lock);
 318         }
 319 
 320         return !__request_completed(rq);
 321 }
 322 
 323 void i915_request_cancel_breadcrumb(struct i915_request *rq)
 324 {
 325         struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
 326 
 327         lockdep_assert_held(&rq->lock);
 328         lockdep_assert_irqs_disabled();
 329 
 330         /*
 331          * We must wait for b->irq_lock so that we know the interrupt handler
 332          * has released its reference to the intel_context and has completed
 333          * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if
 334          * required).
 335          */
 336         spin_lock(&b->irq_lock);
 337         if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) {
 338                 struct intel_context *ce = rq->hw_context;
 339 
 340                 list_del(&rq->signal_link);
 341                 if (list_empty(&ce->signals))
 342                         list_del_init(&ce->signal_link);
 343 
 344                 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
 345         }
 346         spin_unlock(&b->irq_lock);
 347 }
 348 
 349 void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine,
 350                                     struct drm_printer *p)
 351 {
 352         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 353         struct intel_context *ce;
 354         struct i915_request *rq;
 355 
 356         if (list_empty(&b->signalers))
 357                 return;
 358 
 359         drm_printf(p, "Signals:\n");
 360 
 361         spin_lock_irq(&b->irq_lock);
 362         list_for_each_entry(ce, &b->signalers, signal_link) {
 363                 list_for_each_entry(rq, &ce->signals, signal_link) {
 364                         drm_printf(p, "\t[%llx:%llx%s] @ %dms\n",
 365                                    rq->fence.context, rq->fence.seqno,
 366                                    i915_request_completed(rq) ? "!" :
 367                                    i915_request_started(rq) ? "*" :
 368                                    "",
 369                                    jiffies_to_msecs(jiffies - rq->emitted_jiffies));
 370                 }
 371         }
 372         spin_unlock_irq(&b->irq_lock);
 373 }