drivers/gpu/drm/drm

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
store_vblank
drm_max_vblank_count
drm_vblank_no_hw_counter
__get_vblank_counter
drm_reset_vblank_timestamp
drm_update_vblank_count
drm_vblank_count
drm_crtc_accurate_vblank_count
__disable_vblank
drm_vblank_disable_and_save
vblank_disable_fn
drm_vblank_cleanup
drm_vblank_init
drm_crtc_vblank_waitqueue
drm_calc_timestamping_constants
drm_calc_vbltimestamp_from_scanoutpos
drm_get_last_vbltimestamp
drm_crtc_vblank_count
drm_vblank_count_and_time
drm_crtc_vblank_count_and_time
send_vblank_event
drm_crtc_arm_vblank_event
drm_crtc_send_vblank_event
__enable_vblank
drm_vblank_enable
drm_vblank_get
drm_crtc_vblank_get
drm_vblank_put
drm_crtc_vblank_put
drm_wait_one_vblank
drm_crtc_wait_one_vblank
drm_crtc_vblank_off
drm_crtc_vblank_reset
drm_crtc_set_max_vblank_count
drm_crtc_vblank_on
drm_vblank_restore
drm_crtc_vblank_restore
drm_legacy_vblank_pre_modeset
drm_legacy_vblank_post_modeset
drm_legacy_modeset_ctl_ioctl
vblank_passed
drm_queue_vblank_event
drm_wait_vblank_is_query
widen_32_to_64
drm_wait_vblank_reply
drm_wait_vblank_ioctl
drm_handle_vblank_events
drm_handle_vblank
drm_crtc_handle_vblank
drm_crtc_get_sequence_ioctl
drm_crtc_queue_sequence_ioctl
   1 /*
   2  * drm_irq.c IRQ and vblank support
   3  *
   4  * \author Rickard E. (Rik) Faith <faith@valinux.com>
   5  * \author Gareth Hughes <gareth@valinux.com>
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the "Software"),
   9  * to deal in the Software without restriction, including without limitation
  10  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11  * and/or sell copies of the Software, and to permit persons to whom the
  12  * Software is furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the next
  15  * paragraph) shall be included in all copies or substantial portions of the
  16  * Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21  * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  22  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  23  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  24  * OTHER DEALINGS IN THE SOFTWARE.
  25  */
  26 
  27 #include <linux/export.h>
  28 #include <linux/moduleparam.h>
  29 
  30 #include <drm/drm_crtc.h>
  31 #include <drm/drm_drv.h>
  32 #include <drm/drm_framebuffer.h>
  33 #include <drm/drm_print.h>
  34 #include <drm/drm_vblank.h>
  35 
  36 #include "drm_internal.h"
  37 #include "drm_trace.h"
  38 
  39 /**
  40  * DOC: vblank handling
  41  *
  42  * Vertical blanking plays a major role in graphics rendering. To achieve
  43  * tear-free display, users must synchronize page flips and/or rendering to
  44  * vertical blanking. The DRM API offers ioctls to perform page flips
  45  * synchronized to vertical blanking and wait for vertical blanking.
  46  *
  47  * The DRM core handles most of the vertical blanking management logic, which
  48  * involves filtering out spurious interrupts, keeping race-free blanking
  49  * counters, coping with counter wrap-around and resets and keeping use counts.
  50  * It relies on the driver to generate vertical blanking interrupts and
  51  * optionally provide a hardware vertical blanking counter.
  52  *
  53  * Drivers must initialize the vertical blanking handling core with a call to
  54  * drm_vblank_init(). Minimally, a driver needs to implement
  55  * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
  56  * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank
  57  * support.
  58  *
  59  * Vertical blanking interrupts can be enabled by the DRM core or by drivers
  60  * themselves (for instance to handle page flipping operations).  The DRM core
  61  * maintains a vertical blanking use count to ensure that the interrupts are not
  62  * disabled while a user still needs them. To increment the use count, drivers
  63  * call drm_crtc_vblank_get() and release the vblank reference again with
  64  * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
  65  * guaranteed to be enabled.
  66  *
  67  * On many hardware disabling the vblank interrupt cannot be done in a race-free
  68  * manner, see &drm_driver.vblank_disable_immediate and
  69  * &drm_driver.max_vblank_count. In that case the vblank core only disables the
  70  * vblanks after a timer has expired, which can be configured through the
  71  * ``vblankoffdelay`` module parameter.
  72  */
  73 
  74 /* Retry timestamp calculation up to 3 times to satisfy
  75  * drm_timestamp_precision before giving up.
  76  */
  77 #define DRM_TIMESTAMP_MAXRETRIES 3
  78 
  79 /* Threshold in nanoseconds for detection of redundant
  80  * vblank irq in drm_handle_vblank(). 1 msec should be ok.
  81  */
  82 #define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
  83 
  84 static bool
  85 drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
  86                           ktime_t *tvblank, bool in_vblank_irq);
  87 
  88 static unsigned int drm_timestamp_precision = 20;  /* Default to 20 usecs. */
  89 
  90 static int drm_vblank_offdelay = 5000;    /* Default to 5000 msecs. */
  91 
  92 module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
  93 module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
  94 MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
  95 MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
  96 
  97 static void store_vblank(struct drm_device *dev, unsigned int pipe,
  98                          u32 vblank_count_inc,
  99                          ktime_t t_vblank, u32 last)
 100 {
 101         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 102 
 103         assert_spin_locked(&dev->vblank_time_lock);
 104 
 105         vblank->last = last;
 106 
 107         write_seqlock(&vblank->seqlock);
 108         vblank->time = t_vblank;
 109         vblank->count += vblank_count_inc;
 110         write_sequnlock(&vblank->seqlock);
 111 }
 112 
 113 static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
 114 {
 115         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 116 
 117         return vblank->max_vblank_count ?: dev->max_vblank_count;
 118 }
 119 
 120 /*
 121  * "No hw counter" fallback implementation of .get_vblank_counter() hook,
 122  * if there is no useable hardware frame counter available.
 123  */
 124 static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
 125 {
 126         WARN_ON_ONCE(drm_max_vblank_count(dev, pipe) != 0);
 127         return 0;
 128 }
 129 
 130 static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 131 {
 132         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 133                 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
 134 
 135                 if (WARN_ON(!crtc))
 136                         return 0;
 137 
 138                 if (crtc->funcs->get_vblank_counter)
 139                         return crtc->funcs->get_vblank_counter(crtc);
 140         }
 141 
 142         if (dev->driver->get_vblank_counter)
 143                 return dev->driver->get_vblank_counter(dev, pipe);
 144 
 145         return drm_vblank_no_hw_counter(dev, pipe);
 146 }
 147 
 148 /*
 149  * Reset the stored timestamp for the current vblank count to correspond
 150  * to the last vblank occurred.
 151  *
 152  * Only to be called from drm_crtc_vblank_on().
 153  *
 154  * Note: caller must hold &drm_device.vbl_lock since this reads & writes
 155  * device vblank fields.
 156  */
 157 static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
 158 {
 159         u32 cur_vblank;
 160         bool rc;
 161         ktime_t t_vblank;
 162         int count = DRM_TIMESTAMP_MAXRETRIES;
 163 
 164         spin_lock(&dev->vblank_time_lock);
 165 
 166         /*
 167          * sample the current counter to avoid random jumps
 168          * when drm_vblank_enable() applies the diff
 169          */
 170         do {
 171                 cur_vblank = __get_vblank_counter(dev, pipe);
 172                 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
 173         } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
 174 
 175         /*
 176          * Only reinitialize corresponding vblank timestamp if high-precision query
 177          * available and didn't fail. Otherwise reinitialize delayed at next vblank
 178          * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
 179          */
 180         if (!rc)
 181                 t_vblank = 0;
 182 
 183         /*
 184          * +1 to make sure user will never see the same
 185          * vblank counter value before and after a modeset
 186          */
 187         store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
 188 
 189         spin_unlock(&dev->vblank_time_lock);
 190 }
 191 
 192 /*
 193  * Call back into the driver to update the appropriate vblank counter
 194  * (specified by @pipe).  Deal with wraparound, if it occurred, and
 195  * update the last read value so we can deal with wraparound on the next
 196  * call if necessary.
 197  *
 198  * Only necessary when going from off->on, to account for frames we
 199  * didn't get an interrupt for.
 200  *
 201  * Note: caller must hold &drm_device.vbl_lock since this reads & writes
 202  * device vblank fields.
 203  */
 204 static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
 205                                     bool in_vblank_irq)
 206 {
 207         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 208         u32 cur_vblank, diff;
 209         bool rc;
 210         ktime_t t_vblank;
 211         int count = DRM_TIMESTAMP_MAXRETRIES;
 212         int framedur_ns = vblank->framedur_ns;
 213         u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
 214 
 215         /*
 216          * Interrupts were disabled prior to this call, so deal with counter
 217          * wrap if needed.
 218          * NOTE!  It's possible we lost a full dev->max_vblank_count + 1 events
 219          * here if the register is small or we had vblank interrupts off for
 220          * a long time.
 221          *
 222          * We repeat the hardware vblank counter & timestamp query until
 223          * we get consistent results. This to prevent races between gpu
 224          * updating its hardware counter while we are retrieving the
 225          * corresponding vblank timestamp.
 226          */
 227         do {
 228                 cur_vblank = __get_vblank_counter(dev, pipe);
 229                 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
 230         } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
 231 
 232         if (max_vblank_count) {
 233                 /* trust the hw counter when it's around */
 234                 diff = (cur_vblank - vblank->last) & max_vblank_count;
 235         } else if (rc && framedur_ns) {
 236                 u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
 237 
 238                 /*
 239                  * Figure out how many vblanks we've missed based
 240                  * on the difference in the timestamps and the
 241                  * frame/field duration.
 242                  */
 243 
 244                 DRM_DEBUG_VBL("crtc %u: Calculating number of vblanks."
 245                               " diff_ns = %lld, framedur_ns = %d)\n",
 246                               pipe, (long long) diff_ns, framedur_ns);
 247 
 248                 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
 249 
 250                 if (diff == 0 && in_vblank_irq)
 251                         DRM_DEBUG_VBL("crtc %u: Redundant vblirq ignored\n",
 252                                       pipe);
 253         } else {
 254                 /* some kind of default for drivers w/o accurate vbl timestamping */
 255                 diff = in_vblank_irq ? 1 : 0;
 256         }
 257 
 258         /*
 259          * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
 260          * interval? If so then vblank irqs keep running and it will likely
 261          * happen that the hardware vblank counter is not trustworthy as it
 262          * might reset at some point in that interval and vblank timestamps
 263          * are not trustworthy either in that interval. Iow. this can result
 264          * in a bogus diff >> 1 which must be avoided as it would cause
 265          * random large forward jumps of the software vblank counter.
 266          */
 267         if (diff > 1 && (vblank->inmodeset & 0x2)) {
 268                 DRM_DEBUG_VBL("clamping vblank bump to 1 on crtc %u: diffr=%u"
 269                               " due to pre-modeset.\n", pipe, diff);
 270                 diff = 1;
 271         }
 272 
 273         DRM_DEBUG_VBL("updating vblank count on crtc %u:"
 274                       " current=%llu, diff=%u, hw=%u hw_last=%u\n",
 275                       pipe, vblank->count, diff, cur_vblank, vblank->last);
 276 
 277         if (diff == 0) {
 278                 WARN_ON_ONCE(cur_vblank != vblank->last);
 279                 return;
 280         }
 281 
 282         /*
 283          * Only reinitialize corresponding vblank timestamp if high-precision query
 284          * available and didn't fail, or we were called from the vblank interrupt.
 285          * Otherwise reinitialize delayed at next vblank interrupt and assign 0
 286          * for now, to mark the vblanktimestamp as invalid.
 287          */
 288         if (!rc && !in_vblank_irq)
 289                 t_vblank = 0;
 290 
 291         store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
 292 }
 293 
 294 static u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
 295 {
 296         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 297 
 298         if (WARN_ON(pipe >= dev->num_crtcs))
 299                 return 0;
 300 
 301         return vblank->count;
 302 }
 303 
 304 /**
 305  * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
 306  * @crtc: which counter to retrieve
 307  *
 308  * This function is similar to drm_crtc_vblank_count() but this function
 309  * interpolates to handle a race with vblank interrupts using the high precision
 310  * timestamping support.
 311  *
 312  * This is mostly useful for hardware that can obtain the scanout position, but
 313  * doesn't have a hardware frame counter.
 314  */
 315 u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
 316 {
 317         struct drm_device *dev = crtc->dev;
 318         unsigned int pipe = drm_crtc_index(crtc);
 319         u64 vblank;
 320         unsigned long flags;
 321 
 322         WARN_ONCE(drm_debug & DRM_UT_VBL && !dev->driver->get_vblank_timestamp,
 323                   "This function requires support for accurate vblank timestamps.");
 324 
 325         spin_lock_irqsave(&dev->vblank_time_lock, flags);
 326 
 327         drm_update_vblank_count(dev, pipe, false);
 328         vblank = drm_vblank_count(dev, pipe);
 329 
 330         spin_unlock_irqrestore(&dev->vblank_time_lock, flags);
 331 
 332         return vblank;
 333 }
 334 EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
 335 
 336 static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
 337 {
 338         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 339                 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
 340 
 341                 if (WARN_ON(!crtc))
 342                         return;
 343 
 344                 if (crtc->funcs->disable_vblank) {
 345                         crtc->funcs->disable_vblank(crtc);
 346                         return;
 347                 }
 348         }
 349 
 350         dev->driver->disable_vblank(dev, pipe);
 351 }
 352 
 353 /*
 354  * Disable vblank irq's on crtc, make sure that last vblank count
 355  * of hardware and corresponding consistent software vblank counter
 356  * are preserved, even if there are any spurious vblank irq's after
 357  * disable.
 358  */
 359 void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
 360 {
 361         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 362         unsigned long irqflags;
 363 
 364         assert_spin_locked(&dev->vbl_lock);
 365 
 366         /* Prevent vblank irq processing while disabling vblank irqs,
 367          * so no updates of timestamps or count can happen after we've
 368          * disabled. Needed to prevent races in case of delayed irq's.
 369          */
 370         spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
 371 
 372         /*
 373          * Update vblank count and disable vblank interrupts only if the
 374          * interrupts were enabled. This avoids calling the ->disable_vblank()
 375          * operation in atomic context with the hardware potentially runtime
 376          * suspended.
 377          */
 378         if (!vblank->enabled)
 379                 goto out;
 380 
 381         /*
 382          * Update the count and timestamp to maintain the
 383          * appearance that the counter has been ticking all along until
 384          * this time. This makes the count account for the entire time
 385          * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
 386          */
 387         drm_update_vblank_count(dev, pipe, false);
 388         __disable_vblank(dev, pipe);
 389         vblank->enabled = false;
 390 
 391 out:
 392         spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
 393 }
 394 
 395 static void vblank_disable_fn(struct timer_list *t)
 396 {
 397         struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);
 398         struct drm_device *dev = vblank->dev;
 399         unsigned int pipe = vblank->pipe;
 400         unsigned long irqflags;
 401 
 402         spin_lock_irqsave(&dev->vbl_lock, irqflags);
 403         if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
 404                 DRM_DEBUG("disabling vblank on crtc %u\n", pipe);
 405                 drm_vblank_disable_and_save(dev, pipe);
 406         }
 407         spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 408 }
 409 
 410 void drm_vblank_cleanup(struct drm_device *dev)
 411 {
 412         unsigned int pipe;
 413 
 414         /* Bail if the driver didn't call drm_vblank_init() */
 415         if (dev->num_crtcs == 0)
 416                 return;
 417 
 418         for (pipe = 0; pipe < dev->num_crtcs; pipe++) {
 419                 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 420 
 421                 WARN_ON(READ_ONCE(vblank->enabled) &&
 422                         drm_core_check_feature(dev, DRIVER_MODESET));
 423 
 424                 del_timer_sync(&vblank->disable_timer);
 425         }
 426 
 427         kfree(dev->vblank);
 428 
 429         dev->num_crtcs = 0;
 430 }
 431 
 432 /**
 433  * drm_vblank_init - initialize vblank support
 434  * @dev: DRM device
 435  * @num_crtcs: number of CRTCs supported by @dev
 436  *
 437  * This function initializes vblank support for @num_crtcs display pipelines.
 438  * Cleanup is handled by the DRM core, or through calling drm_dev_fini() for
 439  * drivers with a &drm_driver.release callback.
 440  *
 441  * Returns:
 442  * Zero on success or a negative error code on failure.
 443  */
 444 int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
 445 {
 446         int ret = -ENOMEM;
 447         unsigned int i;
 448 
 449         spin_lock_init(&dev->vbl_lock);
 450         spin_lock_init(&dev->vblank_time_lock);
 451 
 452         dev->num_crtcs = num_crtcs;
 453 
 454         dev->vblank = kcalloc(num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
 455         if (!dev->vblank)
 456                 goto err;
 457 
 458         for (i = 0; i < num_crtcs; i++) {
 459                 struct drm_vblank_crtc *vblank = &dev->vblank[i];
 460 
 461                 vblank->dev = dev;
 462                 vblank->pipe = i;
 463                 init_waitqueue_head(&vblank->queue);
 464                 timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
 465                 seqlock_init(&vblank->seqlock);
 466         }
 467 
 468         DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n");
 469 
 470         /* Driver specific high-precision vblank timestamping supported? */
 471         if (dev->driver->get_vblank_timestamp)
 472                 DRM_INFO("Driver supports precise vblank timestamp query.\n");
 473         else
 474                 DRM_INFO("No driver support for vblank timestamp query.\n");
 475 
 476         /* Must have precise timestamping for reliable vblank instant disable */
 477         if (dev->vblank_disable_immediate && !dev->driver->get_vblank_timestamp) {
 478                 dev->vblank_disable_immediate = false;
 479                 DRM_INFO("Setting vblank_disable_immediate to false because "
 480                          "get_vblank_timestamp == NULL\n");
 481         }
 482 
 483         return 0;
 484 
 485 err:
 486         dev->num_crtcs = 0;
 487         return ret;
 488 }
 489 EXPORT_SYMBOL(drm_vblank_init);
 490 
 491 /**
 492  * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
 493  * @crtc: which CRTC's vblank waitqueue to retrieve
 494  *
 495  * This function returns a pointer to the vblank waitqueue for the CRTC.
 496  * Drivers can use this to implement vblank waits using wait_event() and related
 497  * functions.
 498  */
 499 wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
 500 {
 501         return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
 502 }
 503 EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
 504 
 505 
 506 /**
 507  * drm_calc_timestamping_constants - calculate vblank timestamp constants
 508  * @crtc: drm_crtc whose timestamp constants should be updated.
 509  * @mode: display mode containing the scanout timings
 510  *
 511  * Calculate and store various constants which are later needed by vblank and
 512  * swap-completion timestamping, e.g, by
 513  * drm_calc_vbltimestamp_from_scanoutpos(). They are derived from CRTC's true
 514  * scanout timing, so they take things like panel scaling or other adjustments
 515  * into account.
 516  */
 517 void drm_calc_timestamping_constants(struct drm_crtc *crtc,
 518                                      const struct drm_display_mode *mode)
 519 {
 520         struct drm_device *dev = crtc->dev;
 521         unsigned int pipe = drm_crtc_index(crtc);
 522         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 523         int linedur_ns = 0, framedur_ns = 0;
 524         int dotclock = mode->crtc_clock;
 525 
 526         if (!dev->num_crtcs)
 527                 return;
 528 
 529         if (WARN_ON(pipe >= dev->num_crtcs))
 530                 return;
 531 
 532         /* Valid dotclock? */
 533         if (dotclock > 0) {
 534                 int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
 535 
 536                 /*
 537                  * Convert scanline length in pixels and video
 538                  * dot clock to line duration and frame duration
 539                  * in nanoseconds:
 540                  */
 541                 linedur_ns  = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
 542                 framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
 543 
 544                 /*
 545                  * Fields of interlaced scanout modes are only half a frame duration.
 546                  */
 547                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 548                         framedur_ns /= 2;
 549         } else
 550                 DRM_ERROR("crtc %u: Can't calculate constants, dotclock = 0!\n",
 551                           crtc->base.id);
 552 
 553         vblank->linedur_ns  = linedur_ns;
 554         vblank->framedur_ns = framedur_ns;
 555         vblank->hwmode = *mode;
 556 
 557         DRM_DEBUG("crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
 558                   crtc->base.id, mode->crtc_htotal,
 559                   mode->crtc_vtotal, mode->crtc_vdisplay);
 560         DRM_DEBUG("crtc %u: clock %d kHz framedur %d linedur %d\n",
 561                   crtc->base.id, dotclock, framedur_ns, linedur_ns);
 562 }
 563 EXPORT_SYMBOL(drm_calc_timestamping_constants);
 564 
 565 /**
 566  * drm_calc_vbltimestamp_from_scanoutpos - precise vblank timestamp helper
 567  * @dev: DRM device
 568  * @pipe: index of CRTC whose vblank timestamp to retrieve
 569  * @max_error: Desired maximum allowable error in timestamps (nanosecs)
 570  *             On return contains true maximum error of timestamp
 571  * @vblank_time: Pointer to time which should receive the timestamp
 572  * @in_vblank_irq:
 573  *     True when called from drm_crtc_handle_vblank().  Some drivers
 574  *     need to apply some workarounds for gpu-specific vblank irq quirks
 575  *     if flag is set.
 576  *
 577  * Implements calculation of exact vblank timestamps from given drm_display_mode
 578  * timings and current video scanout position of a CRTC. This can be directly
 579  * used as the &drm_driver.get_vblank_timestamp implementation of a kms driver
 580  * if &drm_driver.get_scanout_position is implemented.
 581  *
 582  * The current implementation only handles standard video modes. For double scan
 583  * and interlaced modes the driver is supposed to adjust the hardware mode
 584  * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
 585  * match the scanout position reported.
 586  *
 587  * Note that atomic drivers must call drm_calc_timestamping_constants() before
 588  * enabling a CRTC. The atomic helpers already take care of that in
 589  * drm_atomic_helper_update_legacy_modeset_state().
 590  *
 591  * Returns:
 592  *
 593  * Returns true on success, and false on failure, i.e. when no accurate
 594  * timestamp could be acquired.
 595  */
 596 bool drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev,
 597                                            unsigned int pipe,
 598                                            int *max_error,
 599                                            ktime_t *vblank_time,
 600                                            bool in_vblank_irq)
 601 {
 602         struct timespec64 ts_etime, ts_vblank_time;
 603         ktime_t stime, etime;
 604         bool vbl_status;
 605         struct drm_crtc *crtc;
 606         const struct drm_display_mode *mode;
 607         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 608         int vpos, hpos, i;
 609         int delta_ns, duration_ns;
 610 
 611         if (!drm_core_check_feature(dev, DRIVER_MODESET))
 612                 return false;
 613 
 614         crtc = drm_crtc_from_index(dev, pipe);
 615 
 616         if (pipe >= dev->num_crtcs || !crtc) {
 617                 DRM_ERROR("Invalid crtc %u\n", pipe);
 618                 return false;
 619         }
 620 
 621         /* Scanout position query not supported? Should not happen. */
 622         if (!dev->driver->get_scanout_position) {
 623                 DRM_ERROR("Called from driver w/o get_scanout_position()!?\n");
 624                 return false;
 625         }
 626 
 627         if (drm_drv_uses_atomic_modeset(dev))
 628                 mode = &vblank->hwmode;
 629         else
 630                 mode = &crtc->hwmode;
 631 
 632         /* If mode timing undefined, just return as no-op:
 633          * Happens during initial modesetting of a crtc.
 634          */
 635         if (mode->crtc_clock == 0) {
 636                 DRM_DEBUG("crtc %u: Noop due to uninitialized mode.\n", pipe);
 637                 WARN_ON_ONCE(drm_drv_uses_atomic_modeset(dev));
 638 
 639                 return false;
 640         }
 641 
 642         /* Get current scanout position with system timestamp.
 643          * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
 644          * if single query takes longer than max_error nanoseconds.
 645          *
 646          * This guarantees a tight bound on maximum error if
 647          * code gets preempted or delayed for some reason.
 648          */
 649         for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
 650                 /*
 651                  * Get vertical and horizontal scanout position vpos, hpos,
 652                  * and bounding timestamps stime, etime, pre/post query.
 653                  */
 654                 vbl_status = dev->driver->get_scanout_position(dev, pipe,
 655                                                                in_vblank_irq,
 656                                                                &vpos, &hpos,
 657                                                                &stime, &etime,
 658                                                                mode);
 659 
 660                 /* Return as no-op if scanout query unsupported or failed. */
 661                 if (!vbl_status) {
 662                         DRM_DEBUG("crtc %u : scanoutpos query failed.\n",
 663                                   pipe);
 664                         return false;
 665                 }
 666 
 667                 /* Compute uncertainty in timestamp of scanout position query. */
 668                 duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
 669 
 670                 /* Accept result with <  max_error nsecs timing uncertainty. */
 671                 if (duration_ns <= *max_error)
 672                         break;
 673         }
 674 
 675         /* Noisy system timing? */
 676         if (i == DRM_TIMESTAMP_MAXRETRIES) {
 677                 DRM_DEBUG("crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
 678                           pipe, duration_ns/1000, *max_error/1000, i);
 679         }
 680 
 681         /* Return upper bound of timestamp precision error. */
 682         *max_error = duration_ns;
 683 
 684         /* Convert scanout position into elapsed time at raw_time query
 685          * since start of scanout at first display scanline. delta_ns
 686          * can be negative if start of scanout hasn't happened yet.
 687          */
 688         delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
 689                            mode->crtc_clock);
 690 
 691         /* Subtract time delta from raw timestamp to get final
 692          * vblank_time timestamp for end of vblank.
 693          */
 694         *vblank_time = ktime_sub_ns(etime, delta_ns);
 695 
 696         if ((drm_debug & DRM_UT_VBL) == 0)
 697                 return true;
 698 
 699         ts_etime = ktime_to_timespec64(etime);
 700         ts_vblank_time = ktime_to_timespec64(*vblank_time);
 701 
 702         DRM_DEBUG_VBL("crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
 703                       pipe, hpos, vpos,
 704                       (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,
 705                       (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,
 706                       duration_ns / 1000, i);
 707 
 708         return true;
 709 }
 710 EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos);
 711 
 712 /**
 713  * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
 714  *                             vblank interval
 715  * @dev: DRM device
 716  * @pipe: index of CRTC whose vblank timestamp to retrieve
 717  * @tvblank: Pointer to target time which should receive the timestamp
 718  * @in_vblank_irq:
 719  *     True when called from drm_crtc_handle_vblank().  Some drivers
 720  *     need to apply some workarounds for gpu-specific vblank irq quirks
 721  *     if flag is set.
 722  *
 723  * Fetches the system timestamp corresponding to the time of the most recent
 724  * vblank interval on specified CRTC. May call into kms-driver to
 725  * compute the timestamp with a high-precision GPU specific method.
 726  *
 727  * Returns zero if timestamp originates from uncorrected do_gettimeofday()
 728  * call, i.e., it isn't very precisely locked to the true vblank.
 729  *
 730  * Returns:
 731  * True if timestamp is considered to be very precise, false otherwise.
 732  */
 733 static bool
 734 drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
 735                           ktime_t *tvblank, bool in_vblank_irq)
 736 {
 737         bool ret = false;
 738 
 739         /* Define requested maximum error on timestamps (nanoseconds). */
 740         int max_error = (int) drm_timestamp_precision * 1000;
 741 
 742         /* Query driver if possible and precision timestamping enabled. */
 743         if (dev->driver->get_vblank_timestamp && (max_error > 0))
 744                 ret = dev->driver->get_vblank_timestamp(dev, pipe, &max_error,
 745                                                         tvblank, in_vblank_irq);
 746 
 747         /* GPU high precision timestamp query unsupported or failed.
 748          * Return current monotonic/gettimeofday timestamp as best estimate.
 749          */
 750         if (!ret)
 751                 *tvblank = ktime_get();
 752 
 753         return ret;
 754 }
 755 
 756 /**
 757  * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
 758  * @crtc: which counter to retrieve
 759  *
 760  * Fetches the "cooked" vblank count value that represents the number of
 761  * vblank events since the system was booted, including lost events due to
 762  * modesetting activity. Note that this timer isn't correct against a racing
 763  * vblank interrupt (since it only reports the software vblank counter), see
 764  * drm_crtc_accurate_vblank_count() for such use-cases.
 765  *
 766  * Returns:
 767  * The software vblank counter.
 768  */
 769 u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
 770 {
 771         return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
 772 }
 773 EXPORT_SYMBOL(drm_crtc_vblank_count);
 774 
 775 /**
 776  * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
 777  *     system timestamp corresponding to that vblank counter value.
 778  * @dev: DRM device
 779  * @pipe: index of CRTC whose counter to retrieve
 780  * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
 781  *
 782  * Fetches the "cooked" vblank count value that represents the number of
 783  * vblank events since the system was booted, including lost events due to
 784  * modesetting activity. Returns corresponding system timestamp of the time
 785  * of the vblank interval that corresponds to the current vblank counter value.
 786  *
 787  * This is the legacy version of drm_crtc_vblank_count_and_time().
 788  */
 789 static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
 790                                      ktime_t *vblanktime)
 791 {
 792         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 793         u64 vblank_count;
 794         unsigned int seq;
 795 
 796         if (WARN_ON(pipe >= dev->num_crtcs)) {
 797                 *vblanktime = 0;
 798                 return 0;
 799         }
 800 
 801         do {
 802                 seq = read_seqbegin(&vblank->seqlock);
 803                 vblank_count = vblank->count;
 804                 *vblanktime = vblank->time;
 805         } while (read_seqretry(&vblank->seqlock, seq));
 806 
 807         return vblank_count;
 808 }
 809 
 810 /**
 811  * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
 812  *     and the system timestamp corresponding to that vblank counter value
 813  * @crtc: which counter to retrieve
 814  * @vblanktime: Pointer to time to receive the vblank timestamp.
 815  *
 816  * Fetches the "cooked" vblank count value that represents the number of
 817  * vblank events since the system was booted, including lost events due to
 818  * modesetting activity. Returns corresponding system timestamp of the time
 819  * of the vblank interval that corresponds to the current vblank counter value.
 820  */
 821 u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
 822                                    ktime_t *vblanktime)
 823 {
 824         return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
 825                                          vblanktime);
 826 }
 827 EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
 828 
 829 static void send_vblank_event(struct drm_device *dev,
 830                 struct drm_pending_vblank_event *e,
 831                 u64 seq, ktime_t now)
 832 {
 833         struct timespec64 tv;
 834 
 835         switch (e->event.base.type) {
 836         case DRM_EVENT_VBLANK:
 837         case DRM_EVENT_FLIP_COMPLETE:
 838                 tv = ktime_to_timespec64(now);
 839                 e->event.vbl.sequence = seq;
 840                 /*
 841                  * e->event is a user space structure, with hardcoded unsigned
 842                  * 32-bit seconds/microseconds. This is safe as we always use
 843                  * monotonic timestamps since linux-4.15
 844                  */
 845                 e->event.vbl.tv_sec = tv.tv_sec;
 846                 e->event.vbl.tv_usec = tv.tv_nsec / 1000;
 847                 break;
 848         case DRM_EVENT_CRTC_SEQUENCE:
 849                 if (seq)
 850                         e->event.seq.sequence = seq;
 851                 e->event.seq.time_ns = ktime_to_ns(now);
 852                 break;
 853         }
 854         trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);
 855         drm_send_event_locked(dev, &e->base);
 856 }
 857 
 858 /**
 859  * drm_crtc_arm_vblank_event - arm vblank event after pageflip
 860  * @crtc: the source CRTC of the vblank event
 861  * @e: the event to send
 862  *
 863  * A lot of drivers need to generate vblank events for the very next vblank
 864  * interrupt. For example when the page flip interrupt happens when the page
 865  * flip gets armed, but not when it actually executes within the next vblank
 866  * period. This helper function implements exactly the required vblank arming
 867  * behaviour.
 868  *
 869  * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
 870  * atomic commit must ensure that the next vblank happens at exactly the same
 871  * time as the atomic commit is committed to the hardware. This function itself
 872  * does **not** protect against the next vblank interrupt racing with either this
 873  * function call or the atomic commit operation. A possible sequence could be:
 874  *
 875  * 1. Driver commits new hardware state into vblank-synchronized registers.
 876  * 2. A vblank happens, committing the hardware state. Also the corresponding
 877  *    vblank interrupt is fired off and fully processed by the interrupt
 878  *    handler.
 879  * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
 880  * 4. The event is only send out for the next vblank, which is wrong.
 881  *
 882  * An equivalent race can happen when the driver calls
 883  * drm_crtc_arm_vblank_event() before writing out the new hardware state.
 884  *
 885  * The only way to make this work safely is to prevent the vblank from firing
 886  * (and the hardware from committing anything else) until the entire atomic
 887  * commit sequence has run to completion. If the hardware does not have such a
 888  * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
 889  * Instead drivers need to manually send out the event from their interrupt
 890  * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
 891  * possible race with the hardware committing the atomic update.
 892  *
 893  * Caller must hold a vblank reference for the event @e acquired by a
 894  * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.
 895  */
 896 void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
 897                                struct drm_pending_vblank_event *e)
 898 {
 899         struct drm_device *dev = crtc->dev;
 900         unsigned int pipe = drm_crtc_index(crtc);
 901 
 902         assert_spin_locked(&dev->event_lock);
 903 
 904         e->pipe = pipe;
 905         e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
 906         list_add_tail(&e->base.link, &dev->vblank_event_list);
 907 }
 908 EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
 909 
 910 /**
 911  * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
 912  * @crtc: the source CRTC of the vblank event
 913  * @e: the event to send
 914  *
 915  * Updates sequence # and timestamp on event for the most recently processed
 916  * vblank, and sends it to userspace.  Caller must hold event lock.
 917  *
 918  * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
 919  * situation, especially to send out events for atomic commit operations.
 920  */
 921 void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
 922                                 struct drm_pending_vblank_event *e)
 923 {
 924         struct drm_device *dev = crtc->dev;
 925         u64 seq;
 926         unsigned int pipe = drm_crtc_index(crtc);
 927         ktime_t now;
 928 
 929         if (dev->num_crtcs > 0) {
 930                 seq = drm_vblank_count_and_time(dev, pipe, &now);
 931         } else {
 932                 seq = 0;
 933 
 934                 now = ktime_get();
 935         }
 936         e->pipe = pipe;
 937         send_vblank_event(dev, e, seq, now);
 938 }
 939 EXPORT_SYMBOL(drm_crtc_send_vblank_event);
 940 
 941 static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
 942 {
 943         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 944                 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
 945 
 946                 if (WARN_ON(!crtc))
 947                         return 0;
 948 
 949                 if (crtc->funcs->enable_vblank)
 950                         return crtc->funcs->enable_vblank(crtc);
 951         }
 952 
 953         return dev->driver->enable_vblank(dev, pipe);
 954 }
 955 
 956 static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
 957 {
 958         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 959         int ret = 0;
 960 
 961         assert_spin_locked(&dev->vbl_lock);
 962 
 963         spin_lock(&dev->vblank_time_lock);
 964 
 965         if (!vblank->enabled) {
 966                 /*
 967                  * Enable vblank irqs under vblank_time_lock protection.
 968                  * All vblank count & timestamp updates are held off
 969                  * until we are done reinitializing master counter and
 970                  * timestamps. Filtercode in drm_handle_vblank() will
 971                  * prevent double-accounting of same vblank interval.
 972                  */
 973                 ret = __enable_vblank(dev, pipe);
 974                 DRM_DEBUG("enabling vblank on crtc %u, ret: %d\n", pipe, ret);
 975                 if (ret) {
 976                         atomic_dec(&vblank->refcount);
 977                 } else {
 978                         drm_update_vblank_count(dev, pipe, 0);
 979                         /* drm_update_vblank_count() includes a wmb so we just
 980                          * need to ensure that the compiler emits the write
 981                          * to mark the vblank as enabled after the call
 982                          * to drm_update_vblank_count().
 983                          */
 984                         WRITE_ONCE(vblank->enabled, true);
 985                 }
 986         }
 987 
 988         spin_unlock(&dev->vblank_time_lock);
 989 
 990         return ret;
 991 }
 992 
 993 static int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
 994 {
 995         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 996         unsigned long irqflags;
 997         int ret = 0;
 998 
 999         if (!dev->num_crtcs)
1000                 return -EINVAL;
1001 
1002         if (WARN_ON(pipe >= dev->num_crtcs))
1003                 return -EINVAL;
1004 
1005         spin_lock_irqsave(&dev->vbl_lock, irqflags);
1006         /* Going from 0->1 means we have to enable interrupts again */
1007         if (atomic_add_return(1, &vblank->refcount) == 1) {
1008                 ret = drm_vblank_enable(dev, pipe);
1009         } else {
1010                 if (!vblank->enabled) {
1011                         atomic_dec(&vblank->refcount);
1012                         ret = -EINVAL;
1013                 }
1014         }
1015         spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1016 
1017         return ret;
1018 }
1019 
1020 /**
1021  * drm_crtc_vblank_get - get a reference count on vblank events
1022  * @crtc: which CRTC to own
1023  *
1024  * Acquire a reference count on vblank events to avoid having them disabled
1025  * while in use.
1026  *
1027  * Returns:
1028  * Zero on success or a negative error code on failure.
1029  */
1030 int drm_crtc_vblank_get(struct drm_crtc *crtc)
1031 {
1032         return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
1033 }
1034 EXPORT_SYMBOL(drm_crtc_vblank_get);
1035 
1036 static void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1037 {
1038         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1039 
1040         if (WARN_ON(pipe >= dev->num_crtcs))
1041                 return;
1042 
1043         if (WARN_ON(atomic_read(&vblank->refcount) == 0))
1044                 return;
1045 
1046         /* Last user schedules interrupt disable */
1047         if (atomic_dec_and_test(&vblank->refcount)) {
1048                 if (drm_vblank_offdelay == 0)
1049                         return;
1050                 else if (drm_vblank_offdelay < 0)
1051                         vblank_disable_fn(&vblank->disable_timer);
1052                 else if (!dev->vblank_disable_immediate)
1053                         mod_timer(&vblank->disable_timer,
1054                                   jiffies + ((drm_vblank_offdelay * HZ)/1000));
1055         }
1056 }
1057 
1058 /**
1059  * drm_crtc_vblank_put - give up ownership of vblank events
1060  * @crtc: which counter to give up
1061  *
1062  * Release ownership of a given vblank counter, turning off interrupts
1063  * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
1064  */
1065 void drm_crtc_vblank_put(struct drm_crtc *crtc)
1066 {
1067         drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
1068 }
1069 EXPORT_SYMBOL(drm_crtc_vblank_put);
1070 
1071 /**
1072  * drm_wait_one_vblank - wait for one vblank
1073  * @dev: DRM device
1074  * @pipe: CRTC index
1075  *
1076  * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1077  * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1078  * due to lack of driver support or because the crtc is off.
1079  *
1080  * This is the legacy version of drm_crtc_wait_one_vblank().
1081  */
1082 void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1083 {
1084         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1085         int ret;
1086         u64 last;
1087 
1088         if (WARN_ON(pipe >= dev->num_crtcs))
1089                 return;
1090 
1091         ret = drm_vblank_get(dev, pipe);
1092         if (WARN(ret, "vblank not available on crtc %i, ret=%i\n", pipe, ret))
1093                 return;
1094 
1095         last = drm_vblank_count(dev, pipe);
1096 
1097         ret = wait_event_timeout(vblank->queue,
1098                                  last != drm_vblank_count(dev, pipe),
1099                                  msecs_to_jiffies(100));
1100 
1101         WARN(ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1102 
1103         drm_vblank_put(dev, pipe);
1104 }
1105 EXPORT_SYMBOL(drm_wait_one_vblank);
1106 
1107 /**
1108  * drm_crtc_wait_one_vblank - wait for one vblank
1109  * @crtc: DRM crtc
1110  *
1111  * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1112  * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1113  * due to lack of driver support or because the crtc is off.
1114  */
1115 void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1116 {
1117         drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1118 }
1119 EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1120 
1121 /**
1122  * drm_crtc_vblank_off - disable vblank events on a CRTC
1123  * @crtc: CRTC in question
1124  *
1125  * Drivers can use this function to shut down the vblank interrupt handling when
1126  * disabling a crtc. This function ensures that the latest vblank frame count is
1127  * stored so that drm_vblank_on can restore it again.
1128  *
1129  * Drivers must use this function when the hardware vblank counter can get
1130  * reset, e.g. when suspending or disabling the @crtc in general.
1131  */
1132 void drm_crtc_vblank_off(struct drm_crtc *crtc)
1133 {
1134         struct drm_device *dev = crtc->dev;
1135         unsigned int pipe = drm_crtc_index(crtc);
1136         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1137         struct drm_pending_vblank_event *e, *t;
1138 
1139         ktime_t now;
1140         unsigned long irqflags;
1141         u64 seq;
1142 
1143         if (WARN_ON(pipe >= dev->num_crtcs))
1144                 return;
1145 
1146         spin_lock_irqsave(&dev->event_lock, irqflags);
1147 
1148         spin_lock(&dev->vbl_lock);
1149         DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1150                       pipe, vblank->enabled, vblank->inmodeset);
1151 
1152         /* Avoid redundant vblank disables without previous
1153          * drm_crtc_vblank_on(). */
1154         if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
1155                 drm_vblank_disable_and_save(dev, pipe);
1156 
1157         wake_up(&vblank->queue);
1158 
1159         /*
1160          * Prevent subsequent drm_vblank_get() from re-enabling
1161          * the vblank interrupt by bumping the refcount.
1162          */
1163         if (!vblank->inmodeset) {
1164                 atomic_inc(&vblank->refcount);
1165                 vblank->inmodeset = 1;
1166         }
1167         spin_unlock(&dev->vbl_lock);
1168 
1169         /* Send any queued vblank events, lest the natives grow disquiet */
1170         seq = drm_vblank_count_and_time(dev, pipe, &now);
1171 
1172         list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1173                 if (e->pipe != pipe)
1174                         continue;
1175                 DRM_DEBUG("Sending premature vblank event on disable: "
1176                           "wanted %llu, current %llu\n",
1177                           e->sequence, seq);
1178                 list_del(&e->base.link);
1179                 drm_vblank_put(dev, pipe);
1180                 send_vblank_event(dev, e, seq, now);
1181         }
1182         spin_unlock_irqrestore(&dev->event_lock, irqflags);
1183 
1184         /* Will be reset by the modeset helpers when re-enabling the crtc by
1185          * calling drm_calc_timestamping_constants(). */
1186         vblank->hwmode.crtc_clock = 0;
1187 }
1188 EXPORT_SYMBOL(drm_crtc_vblank_off);
1189 
1190 /**
1191  * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1192  * @crtc: CRTC in question
1193  *
1194  * Drivers can use this function to reset the vblank state to off at load time.
1195  * Drivers should use this together with the drm_crtc_vblank_off() and
1196  * drm_crtc_vblank_on() functions. The difference compared to
1197  * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1198  * and hence doesn't need to call any driver hooks.
1199  *
1200  * This is useful for recovering driver state e.g. on driver load, or on resume.
1201  */
1202 void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1203 {
1204         struct drm_device *dev = crtc->dev;
1205         unsigned long irqflags;
1206         unsigned int pipe = drm_crtc_index(crtc);
1207         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1208 
1209         spin_lock_irqsave(&dev->vbl_lock, irqflags);
1210         /*
1211          * Prevent subsequent drm_vblank_get() from enabling the vblank
1212          * interrupt by bumping the refcount.
1213          */
1214         if (!vblank->inmodeset) {
1215                 atomic_inc(&vblank->refcount);
1216                 vblank->inmodeset = 1;
1217         }
1218         spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1219 
1220         WARN_ON(!list_empty(&dev->vblank_event_list));
1221 }
1222 EXPORT_SYMBOL(drm_crtc_vblank_reset);
1223 
1224 /**
1225  * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value
1226  * @crtc: CRTC in question
1227  * @max_vblank_count: max hardware vblank counter value
1228  *
1229  * Update the maximum hardware vblank counter value for @crtc
1230  * at runtime. Useful for hardware where the operation of the
1231  * hardware vblank counter depends on the currently active
1232  * display configuration.
1233  *
1234  * For example, if the hardware vblank counter does not work
1235  * when a specific connector is active the maximum can be set
1236  * to zero. And when that specific connector isn't active the
1237  * maximum can again be set to the appropriate non-zero value.
1238  *
1239  * If used, must be called before drm_vblank_on().
1240  */
1241 void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
1242                                    u32 max_vblank_count)
1243 {
1244         struct drm_device *dev = crtc->dev;
1245         unsigned int pipe = drm_crtc_index(crtc);
1246         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1247 
1248         WARN_ON(dev->max_vblank_count);
1249         WARN_ON(!READ_ONCE(vblank->inmodeset));
1250 
1251         vblank->max_vblank_count = max_vblank_count;
1252 }
1253 EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);
1254 
1255 /**
1256  * drm_crtc_vblank_on - enable vblank events on a CRTC
1257  * @crtc: CRTC in question
1258  *
1259  * This functions restores the vblank interrupt state captured with
1260  * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
1261  * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
1262  * unbalanced and so can also be unconditionally called in driver load code to
1263  * reflect the current hardware state of the crtc.
1264  */
1265 void drm_crtc_vblank_on(struct drm_crtc *crtc)
1266 {
1267         struct drm_device *dev = crtc->dev;
1268         unsigned int pipe = drm_crtc_index(crtc);
1269         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1270         unsigned long irqflags;
1271 
1272         if (WARN_ON(pipe >= dev->num_crtcs))
1273                 return;
1274 
1275         spin_lock_irqsave(&dev->vbl_lock, irqflags);
1276         DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1277                       pipe, vblank->enabled, vblank->inmodeset);
1278 
1279         /* Drop our private "prevent drm_vblank_get" refcount */
1280         if (vblank->inmodeset) {
1281                 atomic_dec(&vblank->refcount);
1282                 vblank->inmodeset = 0;
1283         }
1284 
1285         drm_reset_vblank_timestamp(dev, pipe);
1286 
1287         /*
1288          * re-enable interrupts if there are users left, or the
1289          * user wishes vblank interrupts to be enabled all the time.
1290          */
1291         if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)
1292                 WARN_ON(drm_vblank_enable(dev, pipe));
1293         spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1294 }
1295 EXPORT_SYMBOL(drm_crtc_vblank_on);
1296 
1297 /**
1298  * drm_vblank_restore - estimate missed vblanks and update vblank count.
1299  * @dev: DRM device
1300  * @pipe: CRTC index
1301  *
1302  * Power manamement features can cause frame counter resets between vblank
1303  * disable and enable. Drivers can use this function in their
1304  * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1305  * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1306  * vblank counter.
1307  *
1308  * This function is the legacy version of drm_crtc_vblank_restore().
1309  */
1310 void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
1311 {
1312         ktime_t t_vblank;
1313         struct drm_vblank_crtc *vblank;
1314         int framedur_ns;
1315         u64 diff_ns;
1316         u32 cur_vblank, diff = 1;
1317         int count = DRM_TIMESTAMP_MAXRETRIES;
1318 
1319         if (WARN_ON(pipe >= dev->num_crtcs))
1320                 return;
1321 
1322         assert_spin_locked(&dev->vbl_lock);
1323         assert_spin_locked(&dev->vblank_time_lock);
1324 
1325         vblank = &dev->vblank[pipe];
1326         WARN_ONCE((drm_debug & DRM_UT_VBL) && !vblank->framedur_ns,
1327                   "Cannot compute missed vblanks without frame duration\n");
1328         framedur_ns = vblank->framedur_ns;
1329 
1330         do {
1331                 cur_vblank = __get_vblank_counter(dev, pipe);
1332                 drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
1333         } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
1334 
1335         diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
1336         if (framedur_ns)
1337                 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
1338 
1339 
1340         DRM_DEBUG_VBL("missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",
1341                       diff, diff_ns, framedur_ns, cur_vblank - vblank->last);
1342         store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
1343 }
1344 EXPORT_SYMBOL(drm_vblank_restore);
1345 
1346 /**
1347  * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
1348  * @crtc: CRTC in question
1349  *
1350  * Power manamement features can cause frame counter resets between vblank
1351  * disable and enable. Drivers can use this function in their
1352  * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1353  * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1354  * vblank counter.
1355  */
1356 void drm_crtc_vblank_restore(struct drm_crtc *crtc)
1357 {
1358         drm_vblank_restore(crtc->dev, drm_crtc_index(crtc));
1359 }
1360 EXPORT_SYMBOL(drm_crtc_vblank_restore);
1361 
1362 static void drm_legacy_vblank_pre_modeset(struct drm_device *dev,
1363                                           unsigned int pipe)
1364 {
1365         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1366 
1367         /* vblank is not initialized (IRQ not installed ?), or has been freed */
1368         if (!dev->num_crtcs)
1369                 return;
1370 
1371         if (WARN_ON(pipe >= dev->num_crtcs))
1372                 return;
1373 
1374         /*
1375          * To avoid all the problems that might happen if interrupts
1376          * were enabled/disabled around or between these calls, we just
1377          * have the kernel take a reference on the CRTC (just once though
1378          * to avoid corrupting the count if multiple, mismatch calls occur),
1379          * so that interrupts remain enabled in the interim.
1380          */
1381         if (!vblank->inmodeset) {
1382                 vblank->inmodeset = 0x1;
1383                 if (drm_vblank_get(dev, pipe) == 0)
1384                         vblank->inmodeset |= 0x2;
1385         }
1386 }
1387 
1388 static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
1389                                            unsigned int pipe)
1390 {
1391         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1392         unsigned long irqflags;
1393 
1394         /* vblank is not initialized (IRQ not installed ?), or has been freed */
1395         if (!dev->num_crtcs)
1396                 return;
1397 
1398         if (WARN_ON(pipe >= dev->num_crtcs))
1399                 return;
1400 
1401         if (vblank->inmodeset) {
1402                 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1403                 drm_reset_vblank_timestamp(dev, pipe);
1404                 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1405 
1406                 if (vblank->inmodeset & 0x2)
1407                         drm_vblank_put(dev, pipe);
1408 
1409                 vblank->inmodeset = 0;
1410         }
1411 }
1412 
1413 int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
1414                                  struct drm_file *file_priv)
1415 {
1416         struct drm_modeset_ctl *modeset = data;
1417         unsigned int pipe;
1418 
1419         /* If drm_vblank_init() hasn't been called yet, just no-op */
1420         if (!dev->num_crtcs)
1421                 return 0;
1422 
1423         /* KMS drivers handle this internally */
1424         if (!drm_core_check_feature(dev, DRIVER_LEGACY))
1425                 return 0;
1426 
1427         pipe = modeset->crtc;
1428         if (pipe >= dev->num_crtcs)
1429                 return -EINVAL;
1430 
1431         switch (modeset->cmd) {
1432         case _DRM_PRE_MODESET:
1433                 drm_legacy_vblank_pre_modeset(dev, pipe);
1434                 break;
1435         case _DRM_POST_MODESET:
1436                 drm_legacy_vblank_post_modeset(dev, pipe);
1437                 break;
1438         default:
1439                 return -EINVAL;
1440         }
1441 
1442         return 0;
1443 }
1444 
1445 static inline bool vblank_passed(u64 seq, u64 ref)
1446 {
1447         return (seq - ref) <= (1 << 23);
1448 }
1449 
1450 static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1451                                   u64 req_seq,
1452                                   union drm_wait_vblank *vblwait,
1453                                   struct drm_file *file_priv)
1454 {
1455         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1456         struct drm_pending_vblank_event *e;
1457         ktime_t now;
1458         unsigned long flags;
1459         u64 seq;
1460         int ret;
1461 
1462         e = kzalloc(sizeof(*e), GFP_KERNEL);
1463         if (e == NULL) {
1464                 ret = -ENOMEM;
1465                 goto err_put;
1466         }
1467 
1468         e->pipe = pipe;
1469         e->event.base.type = DRM_EVENT_VBLANK;
1470         e->event.base.length = sizeof(e->event.vbl);
1471         e->event.vbl.user_data = vblwait->request.signal;
1472         e->event.vbl.crtc_id = 0;
1473         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1474                 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1475                 if (crtc)
1476                         e->event.vbl.crtc_id = crtc->base.id;
1477         }
1478 
1479         spin_lock_irqsave(&dev->event_lock, flags);
1480 
1481         /*
1482          * drm_crtc_vblank_off() might have been called after we called
1483          * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1484          * vblank disable, so no need for further locking.  The reference from
1485          * drm_vblank_get() protects against vblank disable from another source.
1486          */
1487         if (!READ_ONCE(vblank->enabled)) {
1488                 ret = -EINVAL;
1489                 goto err_unlock;
1490         }
1491 
1492         ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1493                                             &e->event.base);
1494 
1495         if (ret)
1496                 goto err_unlock;
1497 
1498         seq = drm_vblank_count_and_time(dev, pipe, &now);
1499 
1500         DRM_DEBUG("event on vblank count %llu, current %llu, crtc %u\n",
1501                   req_seq, seq, pipe);
1502 
1503         trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
1504 
1505         e->sequence = req_seq;
1506         if (vblank_passed(seq, req_seq)) {
1507                 drm_vblank_put(dev, pipe);
1508                 send_vblank_event(dev, e, seq, now);
1509                 vblwait->reply.sequence = seq;
1510         } else {
1511                 /* drm_handle_vblank_events will call drm_vblank_put */
1512                 list_add_tail(&e->base.link, &dev->vblank_event_list);
1513                 vblwait->reply.sequence = req_seq;
1514         }
1515 
1516         spin_unlock_irqrestore(&dev->event_lock, flags);
1517 
1518         return 0;
1519 
1520 err_unlock:
1521         spin_unlock_irqrestore(&dev->event_lock, flags);
1522         kfree(e);
1523 err_put:
1524         drm_vblank_put(dev, pipe);
1525         return ret;
1526 }
1527 
1528 static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1529 {
1530         if (vblwait->request.sequence)
1531                 return false;
1532 
1533         return _DRM_VBLANK_RELATIVE ==
1534                 (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
1535                                           _DRM_VBLANK_EVENT |
1536                                           _DRM_VBLANK_NEXTONMISS));
1537 }
1538 
1539 /*
1540  * Widen a 32-bit param to 64-bits.
1541  *
1542  * \param narrow 32-bit value (missing upper 32 bits)
1543  * \param near 64-bit value that should be 'close' to near
1544  *
1545  * This function returns a 64-bit value using the lower 32-bits from
1546  * 'narrow' and constructing the upper 32-bits so that the result is
1547  * as close as possible to 'near'.
1548  */
1549 
1550 static u64 widen_32_to_64(u32 narrow, u64 near)
1551 {
1552         return near + (s32) (narrow - near);
1553 }
1554 
1555 static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1556                                   struct drm_wait_vblank_reply *reply)
1557 {
1558         ktime_t now;
1559         struct timespec64 ts;
1560 
1561         /*
1562          * drm_wait_vblank_reply is a UAPI structure that uses 'long'
1563          * to store the seconds. This is safe as we always use monotonic
1564          * timestamps since linux-4.15.
1565          */
1566         reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1567         ts = ktime_to_timespec64(now);
1568         reply->tval_sec = (u32)ts.tv_sec;
1569         reply->tval_usec = ts.tv_nsec / 1000;
1570 }
1571 
1572 int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1573                           struct drm_file *file_priv)
1574 {
1575         struct drm_crtc *crtc;
1576         struct drm_vblank_crtc *vblank;
1577         union drm_wait_vblank *vblwait = data;
1578         int ret;
1579         u64 req_seq, seq;
1580         unsigned int pipe_index;
1581         unsigned int flags, pipe, high_pipe;
1582 
1583         if (!dev->irq_enabled)
1584                 return -EOPNOTSUPP;
1585 
1586         if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1587                 return -EINVAL;
1588 
1589         if (vblwait->request.type &
1590             ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1591               _DRM_VBLANK_HIGH_CRTC_MASK)) {
1592                 DRM_DEBUG("Unsupported type value 0x%x, supported mask 0x%x\n",
1593                           vblwait->request.type,
1594                           (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1595                            _DRM_VBLANK_HIGH_CRTC_MASK));
1596                 return -EINVAL;
1597         }
1598 
1599         flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1600         high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1601         if (high_pipe)
1602                 pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1603         else
1604                 pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1605 
1606         /* Convert lease-relative crtc index into global crtc index */
1607         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1608                 pipe = 0;
1609                 drm_for_each_crtc(crtc, dev) {
1610                         if (drm_lease_held(file_priv, crtc->base.id)) {
1611                                 if (pipe_index == 0)
1612                                         break;
1613                                 pipe_index--;
1614                         }
1615                         pipe++;
1616                 }
1617         } else {
1618                 pipe = pipe_index;
1619         }
1620 
1621         if (pipe >= dev->num_crtcs)
1622                 return -EINVAL;
1623 
1624         vblank = &dev->vblank[pipe];
1625 
1626         /* If the counter is currently enabled and accurate, short-circuit
1627          * queries to return the cached timestamp of the last vblank.
1628          */
1629         if (dev->vblank_disable_immediate &&
1630             drm_wait_vblank_is_query(vblwait) &&
1631             READ_ONCE(vblank->enabled)) {
1632                 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1633                 return 0;
1634         }
1635 
1636         ret = drm_vblank_get(dev, pipe);
1637         if (ret) {
1638                 DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1639                 return ret;
1640         }
1641         seq = drm_vblank_count(dev, pipe);
1642 
1643         switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1644         case _DRM_VBLANK_RELATIVE:
1645                 req_seq = seq + vblwait->request.sequence;
1646                 vblwait->request.sequence = req_seq;
1647                 vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1648                 break;
1649         case _DRM_VBLANK_ABSOLUTE:
1650                 req_seq = widen_32_to_64(vblwait->request.sequence, seq);
1651                 break;
1652         default:
1653                 ret = -EINVAL;
1654                 goto done;
1655         }
1656 
1657         if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1658             vblank_passed(seq, req_seq)) {
1659                 req_seq = seq + 1;
1660                 vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1661                 vblwait->request.sequence = req_seq;
1662         }
1663 
1664         if (flags & _DRM_VBLANK_EVENT) {
1665                 /* must hold on to the vblank ref until the event fires
1666                  * drm_vblank_put will be called asynchronously
1667                  */
1668                 return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1669         }
1670 
1671         if (req_seq != seq) {
1672                 int wait;
1673 
1674                 DRM_DEBUG("waiting on vblank count %llu, crtc %u\n",
1675                           req_seq, pipe);
1676                 wait = wait_event_interruptible_timeout(vblank->queue,
1677                         vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||
1678                                       !READ_ONCE(vblank->enabled),
1679                         msecs_to_jiffies(3000));
1680 
1681                 switch (wait) {
1682                 case 0:
1683                         /* timeout */
1684                         ret = -EBUSY;
1685                         break;
1686                 case -ERESTARTSYS:
1687                         /* interrupted by signal */
1688                         ret = -EINTR;
1689                         break;
1690                 default:
1691                         ret = 0;
1692                         break;
1693                 }
1694         }
1695 
1696         if (ret != -EINTR) {
1697                 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1698 
1699                 DRM_DEBUG("crtc %d returning %u to client\n",
1700                           pipe, vblwait->reply.sequence);
1701         } else {
1702                 DRM_DEBUG("crtc %d vblank wait interrupted by signal\n", pipe);
1703         }
1704 
1705 done:
1706         drm_vblank_put(dev, pipe);
1707         return ret;
1708 }
1709 
1710 static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1711 {
1712         struct drm_pending_vblank_event *e, *t;
1713         ktime_t now;
1714         u64 seq;
1715 
1716         assert_spin_locked(&dev->event_lock);
1717 
1718         seq = drm_vblank_count_and_time(dev, pipe, &now);
1719 
1720         list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1721                 if (e->pipe != pipe)
1722                         continue;
1723                 if (!vblank_passed(seq, e->sequence))
1724                         continue;
1725 
1726                 DRM_DEBUG("vblank event on %llu, current %llu\n",
1727                           e->sequence, seq);
1728 
1729                 list_del(&e->base.link);
1730                 drm_vblank_put(dev, pipe);
1731                 send_vblank_event(dev, e, seq, now);
1732         }
1733 
1734         trace_drm_vblank_event(pipe, seq);
1735 }
1736 
1737 /**
1738  * drm_handle_vblank - handle a vblank event
1739  * @dev: DRM device
1740  * @pipe: index of CRTC where this event occurred
1741  *
1742  * Drivers should call this routine in their vblank interrupt handlers to
1743  * update the vblank counter and send any signals that may be pending.
1744  *
1745  * This is the legacy version of drm_crtc_handle_vblank().
1746  */
1747 bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1748 {
1749         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1750         unsigned long irqflags;
1751         bool disable_irq;
1752 
1753         if (WARN_ON_ONCE(!dev->num_crtcs))
1754                 return false;
1755 
1756         if (WARN_ON(pipe >= dev->num_crtcs))
1757                 return false;
1758 
1759         spin_lock_irqsave(&dev->event_lock, irqflags);
1760 
1761         /* Need timestamp lock to prevent concurrent execution with
1762          * vblank enable/disable, as this would cause inconsistent
1763          * or corrupted timestamps and vblank counts.
1764          */
1765         spin_lock(&dev->vblank_time_lock);
1766 
1767         /* Vblank irq handling disabled. Nothing to do. */
1768         if (!vblank->enabled) {
1769                 spin_unlock(&dev->vblank_time_lock);
1770                 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1771                 return false;
1772         }
1773 
1774         drm_update_vblank_count(dev, pipe, true);
1775 
1776         spin_unlock(&dev->vblank_time_lock);
1777 
1778         wake_up(&vblank->queue);
1779 
1780         /* With instant-off, we defer disabling the interrupt until after
1781          * we finish processing the following vblank after all events have
1782          * been signaled. The disable has to be last (after
1783          * drm_handle_vblank_events) so that the timestamp is always accurate.
1784          */
1785         disable_irq = (dev->vblank_disable_immediate &&
1786                        drm_vblank_offdelay > 0 &&
1787                        !atomic_read(&vblank->refcount));
1788 
1789         drm_handle_vblank_events(dev, pipe);
1790 
1791         spin_unlock_irqrestore(&dev->event_lock, irqflags);
1792 
1793         if (disable_irq)
1794                 vblank_disable_fn(&vblank->disable_timer);
1795 
1796         return true;
1797 }
1798 EXPORT_SYMBOL(drm_handle_vblank);
1799 
1800 /**
1801  * drm_crtc_handle_vblank - handle a vblank event
1802  * @crtc: where this event occurred
1803  *
1804  * Drivers should call this routine in their vblank interrupt handlers to
1805  * update the vblank counter and send any signals that may be pending.
1806  *
1807  * This is the native KMS version of drm_handle_vblank().
1808  *
1809  * Returns:
1810  * True if the event was successfully handled, false on failure.
1811  */
1812 bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
1813 {
1814         return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
1815 }
1816 EXPORT_SYMBOL(drm_crtc_handle_vblank);
1817 
1818 /*
1819  * Get crtc VBLANK count.
1820  *
1821  * \param dev DRM device
1822  * \param data user arguement, pointing to a drm_crtc_get_sequence structure.
1823  * \param file_priv drm file private for the user's open file descriptor
1824  */
1825 
1826 int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
1827                                 struct drm_file *file_priv)
1828 {
1829         struct drm_crtc *crtc;
1830         struct drm_vblank_crtc *vblank;
1831         int pipe;
1832         struct drm_crtc_get_sequence *get_seq = data;
1833         ktime_t now;
1834         bool vblank_enabled;
1835         int ret;
1836 
1837         if (!drm_core_check_feature(dev, DRIVER_MODESET))
1838                 return -EOPNOTSUPP;
1839 
1840         if (!dev->irq_enabled)
1841                 return -EOPNOTSUPP;
1842 
1843         crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);
1844         if (!crtc)
1845                 return -ENOENT;
1846 
1847         pipe = drm_crtc_index(crtc);
1848 
1849         vblank = &dev->vblank[pipe];
1850         vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);
1851 
1852         if (!vblank_enabled) {
1853                 ret = drm_crtc_vblank_get(crtc);
1854                 if (ret) {
1855                         DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1856                         return ret;
1857                 }
1858         }
1859         drm_modeset_lock(&crtc->mutex, NULL);
1860         if (crtc->state)
1861                 get_seq->active = crtc->state->enable;
1862         else
1863                 get_seq->active = crtc->enabled;
1864         drm_modeset_unlock(&crtc->mutex);
1865         get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1866         get_seq->sequence_ns = ktime_to_ns(now);
1867         if (!vblank_enabled)
1868                 drm_crtc_vblank_put(crtc);
1869         return 0;
1870 }
1871 
1872 /*
1873  * Queue a event for VBLANK sequence
1874  *
1875  * \param dev DRM device
1876  * \param data user arguement, pointing to a drm_crtc_queue_sequence structure.
1877  * \param file_priv drm file private for the user's open file descriptor
1878  */
1879 
1880 int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
1881                                   struct drm_file *file_priv)
1882 {
1883         struct drm_crtc *crtc;
1884         struct drm_vblank_crtc *vblank;
1885         int pipe;
1886         struct drm_crtc_queue_sequence *queue_seq = data;
1887         ktime_t now;
1888         struct drm_pending_vblank_event *e;
1889         u32 flags;
1890         u64 seq;
1891         u64 req_seq;
1892         int ret;
1893         unsigned long spin_flags;
1894 
1895         if (!drm_core_check_feature(dev, DRIVER_MODESET))
1896                 return -EOPNOTSUPP;
1897 
1898         if (!dev->irq_enabled)
1899                 return -EOPNOTSUPP;
1900 
1901         crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);
1902         if (!crtc)
1903                 return -ENOENT;
1904 
1905         flags = queue_seq->flags;
1906         /* Check valid flag bits */
1907         if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
1908                       DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
1909                 return -EINVAL;
1910 
1911         pipe = drm_crtc_index(crtc);
1912 
1913         vblank = &dev->vblank[pipe];
1914 
1915         e = kzalloc(sizeof(*e), GFP_KERNEL);
1916         if (e == NULL)
1917                 return -ENOMEM;
1918 
1919         ret = drm_crtc_vblank_get(crtc);
1920         if (ret) {
1921                 DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1922                 goto err_free;
1923         }
1924 
1925         seq = drm_vblank_count_and_time(dev, pipe, &now);
1926         req_seq = queue_seq->sequence;
1927 
1928         if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
1929                 req_seq += seq;
1930 
1931         if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && vblank_passed(seq, req_seq))
1932                 req_seq = seq + 1;
1933 
1934         e->pipe = pipe;
1935         e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
1936         e->event.base.length = sizeof(e->event.seq);
1937         e->event.seq.user_data = queue_seq->user_data;
1938 
1939         spin_lock_irqsave(&dev->event_lock, spin_flags);
1940 
1941         /*
1942          * drm_crtc_vblank_off() might have been called after we called
1943          * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1944          * vblank disable, so no need for further locking.  The reference from
1945          * drm_crtc_vblank_get() protects against vblank disable from another source.
1946          */
1947         if (!READ_ONCE(vblank->enabled)) {
1948                 ret = -EINVAL;
1949                 goto err_unlock;
1950         }
1951 
1952         ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1953                                             &e->event.base);
1954 
1955         if (ret)
1956                 goto err_unlock;
1957 
1958         e->sequence = req_seq;
1959 
1960         if (vblank_passed(seq, req_seq)) {
1961                 drm_crtc_vblank_put(crtc);
1962                 send_vblank_event(dev, e, seq, now);
1963                 queue_seq->sequence = seq;
1964         } else {
1965                 /* drm_handle_vblank_events will call drm_vblank_put */
1966                 list_add_tail(&e->base.link, &dev->vblank_event_list);
1967                 queue_seq->sequence = req_seq;
1968         }
1969 
1970         spin_unlock_irqrestore(&dev->event_lock, spin_flags);
1971         return 0;
1972 
1973 err_unlock:
1974         spin_unlock_irqrestore(&dev->event_lock, spin_flags);
1975         drm_crtc_vblank_put(crtc);
1976 err_free:
1977         kfree(e);
1978         return ret;
1979 }
/* [<][>][^][v][top][bottom][index][help] */
root/drivers/gpu/drm/drm_vblank.c

DEFINITIONS
