drivers/gpu/drm/i915/gem/selftests/huge

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
get_largest_page_size
huge_pages_free_pages
get_huge_pages
put_huge_pages
huge_pages_object
fake_get_huge_pages
fake_get_huge_pages_single
fake_free_huge_pages
fake_put_huge_pages
fake_huge_pages_object
igt_check_page_sizes
igt_mock_exhaust_device_supported_pages
igt_mock_ppgtt_misaligned_dma
close_object_list
igt_mock_ppgtt_huge_fill
igt_mock_ppgtt_64K
gpu_write
cpu_check
__igt_write_huge
igt_write_huge
igt_ppgtt_exhaust_huge
igt_ppgtt_internal_huge
igt_can_allocate_thp
igt_ppgtt_gemfs_huge
igt_ppgtt_pin_update
igt_tmpfs_fallback
igt_shrink_thp
i915_gem_huge_page_mock_selftests
i915_gem_huge_page_live_selftests
   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2017 Intel Corporation
   5  */
   6 
   7 #include <linux/prime_numbers.h>
   8 
   9 #include "i915_selftest.h"
  10 
  11 #include "gem/i915_gem_pm.h"
  12 
  13 #include "gt/intel_gt.h"
  14 
  15 #include "igt_gem_utils.h"
  16 #include "mock_context.h"
  17 
  18 #include "selftests/mock_drm.h"
  19 #include "selftests/mock_gem_device.h"
  20 #include "selftests/i915_random.h"
  21 
  22 static const unsigned int page_sizes[] = {
  23         I915_GTT_PAGE_SIZE_2M,
  24         I915_GTT_PAGE_SIZE_64K,
  25         I915_GTT_PAGE_SIZE_4K,
  26 };
  27 
  28 static unsigned int get_largest_page_size(struct drm_i915_private *i915,
  29                                           u64 rem)
  30 {
  31         int i;
  32 
  33         for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
  34                 unsigned int page_size = page_sizes[i];
  35 
  36                 if (HAS_PAGE_SIZES(i915, page_size) && rem >= page_size)
  37                         return page_size;
  38         }
  39 
  40         return 0;
  41 }
  42 
  43 static void huge_pages_free_pages(struct sg_table *st)
  44 {
  45         struct scatterlist *sg;
  46 
  47         for (sg = st->sgl; sg; sg = __sg_next(sg)) {
  48                 if (sg_page(sg))
  49                         __free_pages(sg_page(sg), get_order(sg->length));
  50         }
  51 
  52         sg_free_table(st);
  53         kfree(st);
  54 }
  55 
  56 static int get_huge_pages(struct drm_i915_gem_object *obj)
  57 {
  58 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
  59         unsigned int page_mask = obj->mm.page_mask;
  60         struct sg_table *st;
  61         struct scatterlist *sg;
  62         unsigned int sg_page_sizes;
  63         u64 rem;
  64 
  65         st = kmalloc(sizeof(*st), GFP);
  66         if (!st)
  67                 return -ENOMEM;
  68 
  69         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
  70                 kfree(st);
  71                 return -ENOMEM;
  72         }
  73 
  74         rem = obj->base.size;
  75         sg = st->sgl;
  76         st->nents = 0;
  77         sg_page_sizes = 0;
  78 
  79         /*
  80          * Our goal here is simple, we want to greedily fill the object from
  81          * largest to smallest page-size, while ensuring that we use *every*
  82          * page-size as per the given page-mask.
  83          */
  84         do {
  85                 unsigned int bit = ilog2(page_mask);
  86                 unsigned int page_size = BIT(bit);
  87                 int order = get_order(page_size);
  88 
  89                 do {
  90                         struct page *page;
  91 
  92                         GEM_BUG_ON(order >= MAX_ORDER);
  93                         page = alloc_pages(GFP | __GFP_ZERO, order);
  94                         if (!page)
  95                                 goto err;
  96 
  97                         sg_set_page(sg, page, page_size, 0);
  98                         sg_page_sizes |= page_size;
  99                         st->nents++;
 100 
 101                         rem -= page_size;
 102                         if (!rem) {
 103                                 sg_mark_end(sg);
 104                                 break;
 105                         }
 106 
 107                         sg = __sg_next(sg);
 108                 } while ((rem - ((page_size-1) & page_mask)) >= page_size);
 109 
 110                 page_mask &= (page_size-1);
 111         } while (page_mask);
 112 
 113         if (i915_gem_gtt_prepare_pages(obj, st))
 114                 goto err;
 115 
 116         obj->mm.madv = I915_MADV_DONTNEED;
 117 
 118         GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask);
 119         __i915_gem_object_set_pages(obj, st, sg_page_sizes);
 120 
 121         return 0;
 122 
 123 err:
 124         sg_set_page(sg, NULL, 0, 0);
 125         sg_mark_end(sg);
 126         huge_pages_free_pages(st);
 127 
 128         return -ENOMEM;
 129 }
 130 
 131 static void put_huge_pages(struct drm_i915_gem_object *obj,
 132                            struct sg_table *pages)
 133 {
 134         i915_gem_gtt_finish_pages(obj, pages);
 135         huge_pages_free_pages(pages);
 136 
 137         obj->mm.dirty = false;
 138         obj->mm.madv = I915_MADV_WILLNEED;
 139 }
 140 
 141 static const struct drm_i915_gem_object_ops huge_page_ops = {
 142         .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
 143                  I915_GEM_OBJECT_IS_SHRINKABLE,
 144         .get_pages = get_huge_pages,
 145         .put_pages = put_huge_pages,
 146 };
 147 
 148 static struct drm_i915_gem_object *
 149 huge_pages_object(struct drm_i915_private *i915,
 150                   u64 size,
 151                   unsigned int page_mask)
 152 {
 153         struct drm_i915_gem_object *obj;
 154 
 155         GEM_BUG_ON(!size);
 156         GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask))));
 157 
 158         if (size >> PAGE_SHIFT > INT_MAX)
 159                 return ERR_PTR(-E2BIG);
 160 
 161         if (overflows_type(size, obj->base.size))
 162                 return ERR_PTR(-E2BIG);
 163 
 164         obj = i915_gem_object_alloc();
 165         if (!obj)
 166                 return ERR_PTR(-ENOMEM);
 167 
 168         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 169         i915_gem_object_init(obj, &huge_page_ops);
 170 
 171         obj->write_domain = I915_GEM_DOMAIN_CPU;
 172         obj->read_domains = I915_GEM_DOMAIN_CPU;
 173         obj->cache_level = I915_CACHE_NONE;
 174 
 175         obj->mm.page_mask = page_mask;
 176 
 177         return obj;
 178 }
 179 
 180 static int fake_get_huge_pages(struct drm_i915_gem_object *obj)
 181 {
 182         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 183         const u64 max_len = rounddown_pow_of_two(UINT_MAX);
 184         struct sg_table *st;
 185         struct scatterlist *sg;
 186         unsigned int sg_page_sizes;
 187         u64 rem;
 188 
 189         st = kmalloc(sizeof(*st), GFP);
 190         if (!st)
 191                 return -ENOMEM;
 192 
 193         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
 194                 kfree(st);
 195                 return -ENOMEM;
 196         }
 197 
 198         /* Use optimal page sized chunks to fill in the sg table */
 199         rem = obj->base.size;
 200         sg = st->sgl;
 201         st->nents = 0;
 202         sg_page_sizes = 0;
 203         do {
 204                 unsigned int page_size = get_largest_page_size(i915, rem);
 205                 unsigned int len = min(page_size * div_u64(rem, page_size),
 206                                        max_len);
 207 
 208                 GEM_BUG_ON(!page_size);
 209 
 210                 sg->offset = 0;
 211                 sg->length = len;
 212                 sg_dma_len(sg) = len;
 213                 sg_dma_address(sg) = page_size;
 214 
 215                 sg_page_sizes |= len;
 216 
 217                 st->nents++;
 218 
 219                 rem -= len;
 220                 if (!rem) {
 221                         sg_mark_end(sg);
 222                         break;
 223                 }
 224 
 225                 sg = sg_next(sg);
 226         } while (1);
 227 
 228         i915_sg_trim(st);
 229 
 230         obj->mm.madv = I915_MADV_DONTNEED;
 231 
 232         __i915_gem_object_set_pages(obj, st, sg_page_sizes);
 233 
 234         return 0;
 235 }
 236 
 237 static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj)
 238 {
 239         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 240         struct sg_table *st;
 241         struct scatterlist *sg;
 242         unsigned int page_size;
 243 
 244         st = kmalloc(sizeof(*st), GFP);
 245         if (!st)
 246                 return -ENOMEM;
 247 
 248         if (sg_alloc_table(st, 1, GFP)) {
 249                 kfree(st);
 250                 return -ENOMEM;
 251         }
 252 
 253         sg = st->sgl;
 254         st->nents = 1;
 255 
 256         page_size = get_largest_page_size(i915, obj->base.size);
 257         GEM_BUG_ON(!page_size);
 258 
 259         sg->offset = 0;
 260         sg->length = obj->base.size;
 261         sg_dma_len(sg) = obj->base.size;
 262         sg_dma_address(sg) = page_size;
 263 
 264         obj->mm.madv = I915_MADV_DONTNEED;
 265 
 266         __i915_gem_object_set_pages(obj, st, sg->length);
 267 
 268         return 0;
 269 #undef GFP
 270 }
 271 
 272 static void fake_free_huge_pages(struct drm_i915_gem_object *obj,
 273                                  struct sg_table *pages)
 274 {
 275         sg_free_table(pages);
 276         kfree(pages);
 277 }
 278 
 279 static void fake_put_huge_pages(struct drm_i915_gem_object *obj,
 280                                 struct sg_table *pages)
 281 {
 282         fake_free_huge_pages(obj, pages);
 283         obj->mm.dirty = false;
 284         obj->mm.madv = I915_MADV_WILLNEED;
 285 }
 286 
 287 static const struct drm_i915_gem_object_ops fake_ops = {
 288         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 289         .get_pages = fake_get_huge_pages,
 290         .put_pages = fake_put_huge_pages,
 291 };
 292 
 293 static const struct drm_i915_gem_object_ops fake_ops_single = {
 294         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 295         .get_pages = fake_get_huge_pages_single,
 296         .put_pages = fake_put_huge_pages,
 297 };
 298 
 299 static struct drm_i915_gem_object *
 300 fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single)
 301 {
 302         struct drm_i915_gem_object *obj;
 303 
 304         GEM_BUG_ON(!size);
 305         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 306 
 307         if (size >> PAGE_SHIFT > UINT_MAX)
 308                 return ERR_PTR(-E2BIG);
 309 
 310         if (overflows_type(size, obj->base.size))
 311                 return ERR_PTR(-E2BIG);
 312 
 313         obj = i915_gem_object_alloc();
 314         if (!obj)
 315                 return ERR_PTR(-ENOMEM);
 316 
 317         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 318 
 319         if (single)
 320                 i915_gem_object_init(obj, &fake_ops_single);
 321         else
 322                 i915_gem_object_init(obj, &fake_ops);
 323 
 324         obj->write_domain = I915_GEM_DOMAIN_CPU;
 325         obj->read_domains = I915_GEM_DOMAIN_CPU;
 326         obj->cache_level = I915_CACHE_NONE;
 327 
 328         return obj;
 329 }
 330 
 331 static int igt_check_page_sizes(struct i915_vma *vma)
 332 {
 333         struct drm_i915_private *i915 = vma->vm->i915;
 334         unsigned int supported = INTEL_INFO(i915)->page_sizes;
 335         struct drm_i915_gem_object *obj = vma->obj;
 336         int err = 0;
 337 
 338         if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) {
 339                 pr_err("unsupported page_sizes.sg=%u, supported=%u\n",
 340                        vma->page_sizes.sg & ~supported, supported);
 341                 err = -EINVAL;
 342         }
 343 
 344         if (!HAS_PAGE_SIZES(i915, vma->page_sizes.gtt)) {
 345                 pr_err("unsupported page_sizes.gtt=%u, supported=%u\n",
 346                        vma->page_sizes.gtt & ~supported, supported);
 347                 err = -EINVAL;
 348         }
 349 
 350         if (vma->page_sizes.phys != obj->mm.page_sizes.phys) {
 351                 pr_err("vma->page_sizes.phys(%u) != obj->mm.page_sizes.phys(%u)\n",
 352                        vma->page_sizes.phys, obj->mm.page_sizes.phys);
 353                 err = -EINVAL;
 354         }
 355 
 356         if (vma->page_sizes.sg != obj->mm.page_sizes.sg) {
 357                 pr_err("vma->page_sizes.sg(%u) != obj->mm.page_sizes.sg(%u)\n",
 358                        vma->page_sizes.sg, obj->mm.page_sizes.sg);
 359                 err = -EINVAL;
 360         }
 361 
 362         if (obj->mm.page_sizes.gtt) {
 363                 pr_err("obj->page_sizes.gtt(%u) should never be set\n",
 364                        obj->mm.page_sizes.gtt);
 365                 err = -EINVAL;
 366         }
 367 
 368         return err;
 369 }
 370 
 371 static int igt_mock_exhaust_device_supported_pages(void *arg)
 372 {
 373         struct i915_ppgtt *ppgtt = arg;
 374         struct drm_i915_private *i915 = ppgtt->vm.i915;
 375         unsigned int saved_mask = INTEL_INFO(i915)->page_sizes;
 376         struct drm_i915_gem_object *obj;
 377         struct i915_vma *vma;
 378         int i, j, single;
 379         int err;
 380 
 381         /*
 382          * Sanity check creating objects with every valid page support
 383          * combination for our mock device.
 384          */
 385 
 386         for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) {
 387                 unsigned int combination = 0;
 388 
 389                 for (j = 0; j < ARRAY_SIZE(page_sizes); j++) {
 390                         if (i & BIT(j))
 391                                 combination |= page_sizes[j];
 392                 }
 393 
 394                 mkwrite_device_info(i915)->page_sizes = combination;
 395 
 396                 for (single = 0; single <= 1; ++single) {
 397                         obj = fake_huge_pages_object(i915, combination, !!single);
 398                         if (IS_ERR(obj)) {
 399                                 err = PTR_ERR(obj);
 400                                 goto out_device;
 401                         }
 402 
 403                         if (obj->base.size != combination) {
 404                                 pr_err("obj->base.size=%zu, expected=%u\n",
 405                                        obj->base.size, combination);
 406                                 err = -EINVAL;
 407                                 goto out_put;
 408                         }
 409 
 410                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 411                         if (IS_ERR(vma)) {
 412                                 err = PTR_ERR(vma);
 413                                 goto out_put;
 414                         }
 415 
 416                         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 417                         if (err)
 418                                 goto out_close;
 419 
 420                         err = igt_check_page_sizes(vma);
 421 
 422                         if (vma->page_sizes.sg != combination) {
 423                                 pr_err("page_sizes.sg=%u, expected=%u\n",
 424                                        vma->page_sizes.sg, combination);
 425                                 err = -EINVAL;
 426                         }
 427 
 428                         i915_vma_unpin(vma);
 429                         i915_vma_close(vma);
 430 
 431                         i915_gem_object_put(obj);
 432 
 433                         if (err)
 434                                 goto out_device;
 435                 }
 436         }
 437 
 438         goto out_device;
 439 
 440 out_close:
 441         i915_vma_close(vma);
 442 out_put:
 443         i915_gem_object_put(obj);
 444 out_device:
 445         mkwrite_device_info(i915)->page_sizes = saved_mask;
 446 
 447         return err;
 448 }
 449 
 450 static int igt_mock_ppgtt_misaligned_dma(void *arg)
 451 {
 452         struct i915_ppgtt *ppgtt = arg;
 453         struct drm_i915_private *i915 = ppgtt->vm.i915;
 454         unsigned long supported = INTEL_INFO(i915)->page_sizes;
 455         struct drm_i915_gem_object *obj;
 456         int bit;
 457         int err;
 458 
 459         /*
 460          * Sanity check dma misalignment for huge pages -- the dma addresses we
 461          * insert into the paging structures need to always respect the page
 462          * size alignment.
 463          */
 464 
 465         bit = ilog2(I915_GTT_PAGE_SIZE_64K);
 466 
 467         for_each_set_bit_from(bit, &supported,
 468                               ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
 469                 IGT_TIMEOUT(end_time);
 470                 unsigned int page_size = BIT(bit);
 471                 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
 472                 unsigned int offset;
 473                 unsigned int size =
 474                         round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1;
 475                 struct i915_vma *vma;
 476 
 477                 obj = fake_huge_pages_object(i915, size, true);
 478                 if (IS_ERR(obj))
 479                         return PTR_ERR(obj);
 480 
 481                 if (obj->base.size != size) {
 482                         pr_err("obj->base.size=%zu, expected=%u\n",
 483                                obj->base.size, size);
 484                         err = -EINVAL;
 485                         goto out_put;
 486                 }
 487 
 488                 err = i915_gem_object_pin_pages(obj);
 489                 if (err)
 490                         goto out_put;
 491 
 492                 /* Force the page size for this object */
 493                 obj->mm.page_sizes.sg = page_size;
 494 
 495                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 496                 if (IS_ERR(vma)) {
 497                         err = PTR_ERR(vma);
 498                         goto out_unpin;
 499                 }
 500 
 501                 err = i915_vma_pin(vma, 0, 0, flags);
 502                 if (err) {
 503                         i915_vma_close(vma);
 504                         goto out_unpin;
 505                 }
 506 
 507 
 508                 err = igt_check_page_sizes(vma);
 509 
 510                 if (vma->page_sizes.gtt != page_size) {
 511                         pr_err("page_sizes.gtt=%u, expected %u\n",
 512                                vma->page_sizes.gtt, page_size);
 513                         err = -EINVAL;
 514                 }
 515 
 516                 i915_vma_unpin(vma);
 517 
 518                 if (err) {
 519                         i915_vma_close(vma);
 520                         goto out_unpin;
 521                 }
 522 
 523                 /*
 524                  * Try all the other valid offsets until the next
 525                  * boundary -- should always fall back to using 4K
 526                  * pages.
 527                  */
 528                 for (offset = 4096; offset < page_size; offset += 4096) {
 529                         err = i915_vma_unbind(vma);
 530                         if (err) {
 531                                 i915_vma_close(vma);
 532                                 goto out_unpin;
 533                         }
 534 
 535                         err = i915_vma_pin(vma, 0, 0, flags | offset);
 536                         if (err) {
 537                                 i915_vma_close(vma);
 538                                 goto out_unpin;
 539                         }
 540 
 541                         err = igt_check_page_sizes(vma);
 542 
 543                         if (vma->page_sizes.gtt != I915_GTT_PAGE_SIZE_4K) {
 544                                 pr_err("page_sizes.gtt=%u, expected %llu\n",
 545                                        vma->page_sizes.gtt, I915_GTT_PAGE_SIZE_4K);
 546                                 err = -EINVAL;
 547                         }
 548 
 549                         i915_vma_unpin(vma);
 550 
 551                         if (err) {
 552                                 i915_vma_close(vma);
 553                                 goto out_unpin;
 554                         }
 555 
 556                         if (igt_timeout(end_time,
 557                                         "%s timed out at offset %x with page-size %x\n",
 558                                         __func__, offset, page_size))
 559                                 break;
 560                 }
 561 
 562                 i915_vma_close(vma);
 563 
 564                 i915_gem_object_unpin_pages(obj);
 565                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 566                 i915_gem_object_put(obj);
 567         }
 568 
 569         return 0;
 570 
 571 out_unpin:
 572         i915_gem_object_unpin_pages(obj);
 573 out_put:
 574         i915_gem_object_put(obj);
 575 
 576         return err;
 577 }
 578 
 579 static void close_object_list(struct list_head *objects,
 580                               struct i915_ppgtt *ppgtt)
 581 {
 582         struct drm_i915_gem_object *obj, *on;
 583 
 584         list_for_each_entry_safe(obj, on, objects, st_link) {
 585                 struct i915_vma *vma;
 586 
 587                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 588                 if (!IS_ERR(vma))
 589                         i915_vma_close(vma);
 590 
 591                 list_del(&obj->st_link);
 592                 i915_gem_object_unpin_pages(obj);
 593                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 594                 i915_gem_object_put(obj);
 595         }
 596 }
 597 
 598 static int igt_mock_ppgtt_huge_fill(void *arg)
 599 {
 600         struct i915_ppgtt *ppgtt = arg;
 601         struct drm_i915_private *i915 = ppgtt->vm.i915;
 602         unsigned long max_pages = ppgtt->vm.total >> PAGE_SHIFT;
 603         unsigned long page_num;
 604         bool single = false;
 605         LIST_HEAD(objects);
 606         IGT_TIMEOUT(end_time);
 607         int err = -ENODEV;
 608 
 609         for_each_prime_number_from(page_num, 1, max_pages) {
 610                 struct drm_i915_gem_object *obj;
 611                 u64 size = page_num << PAGE_SHIFT;
 612                 struct i915_vma *vma;
 613                 unsigned int expected_gtt = 0;
 614                 int i;
 615 
 616                 obj = fake_huge_pages_object(i915, size, single);
 617                 if (IS_ERR(obj)) {
 618                         err = PTR_ERR(obj);
 619                         break;
 620                 }
 621 
 622                 if (obj->base.size != size) {
 623                         pr_err("obj->base.size=%zd, expected=%llu\n",
 624                                obj->base.size, size);
 625                         i915_gem_object_put(obj);
 626                         err = -EINVAL;
 627                         break;
 628                 }
 629 
 630                 err = i915_gem_object_pin_pages(obj);
 631                 if (err) {
 632                         i915_gem_object_put(obj);
 633                         break;
 634                 }
 635 
 636                 list_add(&obj->st_link, &objects);
 637 
 638                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 639                 if (IS_ERR(vma)) {
 640                         err = PTR_ERR(vma);
 641                         break;
 642                 }
 643 
 644                 err = i915_vma_pin(vma, 0, 0, PIN_USER);
 645                 if (err)
 646                         break;
 647 
 648                 err = igt_check_page_sizes(vma);
 649                 if (err) {
 650                         i915_vma_unpin(vma);
 651                         break;
 652                 }
 653 
 654                 /*
 655                  * Figure out the expected gtt page size knowing that we go from
 656                  * largest to smallest page size sg chunks, and that we align to
 657                  * the largest page size.
 658                  */
 659                 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
 660                         unsigned int page_size = page_sizes[i];
 661 
 662                         if (HAS_PAGE_SIZES(i915, page_size) &&
 663                             size >= page_size) {
 664                                 expected_gtt |= page_size;
 665                                 size &= page_size-1;
 666                         }
 667                 }
 668 
 669                 GEM_BUG_ON(!expected_gtt);
 670                 GEM_BUG_ON(size);
 671 
 672                 if (expected_gtt & I915_GTT_PAGE_SIZE_4K)
 673                         expected_gtt &= ~I915_GTT_PAGE_SIZE_64K;
 674 
 675                 i915_vma_unpin(vma);
 676 
 677                 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
 678                         if (!IS_ALIGNED(vma->node.start,
 679                                         I915_GTT_PAGE_SIZE_2M)) {
 680                                 pr_err("node.start(%llx) not aligned to 2M\n",
 681                                        vma->node.start);
 682                                 err = -EINVAL;
 683                                 break;
 684                         }
 685 
 686                         if (!IS_ALIGNED(vma->node.size,
 687                                         I915_GTT_PAGE_SIZE_2M)) {
 688                                 pr_err("node.size(%llx) not aligned to 2M\n",
 689                                        vma->node.size);
 690                                 err = -EINVAL;
 691                                 break;
 692                         }
 693                 }
 694 
 695                 if (vma->page_sizes.gtt != expected_gtt) {
 696                         pr_err("gtt=%u, expected=%u, size=%zd, single=%s\n",
 697                                vma->page_sizes.gtt, expected_gtt,
 698                                obj->base.size, yesno(!!single));
 699                         err = -EINVAL;
 700                         break;
 701                 }
 702 
 703                 if (igt_timeout(end_time,
 704                                 "%s timed out at size %zd\n",
 705                                 __func__, obj->base.size))
 706                         break;
 707 
 708                 single = !single;
 709         }
 710 
 711         close_object_list(&objects, ppgtt);
 712 
 713         if (err == -ENOMEM || err == -ENOSPC)
 714                 err = 0;
 715 
 716         return err;
 717 }
 718 
 719 static int igt_mock_ppgtt_64K(void *arg)
 720 {
 721         struct i915_ppgtt *ppgtt = arg;
 722         struct drm_i915_private *i915 = ppgtt->vm.i915;
 723         struct drm_i915_gem_object *obj;
 724         const struct object_info {
 725                 unsigned int size;
 726                 unsigned int gtt;
 727                 unsigned int offset;
 728         } objects[] = {
 729                 /* Cases with forced padding/alignment */
 730                 {
 731                         .size = SZ_64K,
 732                         .gtt = I915_GTT_PAGE_SIZE_64K,
 733                         .offset = 0,
 734                 },
 735                 {
 736                         .size = SZ_64K + SZ_4K,
 737                         .gtt = I915_GTT_PAGE_SIZE_4K,
 738                         .offset = 0,
 739                 },
 740                 {
 741                         .size = SZ_64K - SZ_4K,
 742                         .gtt = I915_GTT_PAGE_SIZE_4K,
 743                         .offset = 0,
 744                 },
 745                 {
 746                         .size = SZ_2M,
 747                         .gtt = I915_GTT_PAGE_SIZE_64K,
 748                         .offset = 0,
 749                 },
 750                 {
 751                         .size = SZ_2M - SZ_4K,
 752                         .gtt = I915_GTT_PAGE_SIZE_4K,
 753                         .offset = 0,
 754                 },
 755                 {
 756                         .size = SZ_2M + SZ_4K,
 757                         .gtt = I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_4K,
 758                         .offset = 0,
 759                 },
 760                 {
 761                         .size = SZ_2M + SZ_64K,
 762                         .gtt = I915_GTT_PAGE_SIZE_64K,
 763                         .offset = 0,
 764                 },
 765                 {
 766                         .size = SZ_2M - SZ_64K,
 767                         .gtt = I915_GTT_PAGE_SIZE_64K,
 768                         .offset = 0,
 769                 },
 770                 /* Try without any forced padding/alignment */
 771                 {
 772                         .size = SZ_64K,
 773                         .offset = SZ_2M,
 774                         .gtt = I915_GTT_PAGE_SIZE_4K,
 775                 },
 776                 {
 777                         .size = SZ_128K,
 778                         .offset = SZ_2M - SZ_64K,
 779                         .gtt = I915_GTT_PAGE_SIZE_4K,
 780                 },
 781         };
 782         struct i915_vma *vma;
 783         int i, single;
 784         int err;
 785 
 786         /*
 787          * Sanity check some of the trickiness with 64K pages -- either we can
 788          * safely mark the whole page-table(2M block) as 64K, or we have to
 789          * always fallback to 4K.
 790          */
 791 
 792         if (!HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K))
 793                 return 0;
 794 
 795         for (i = 0; i < ARRAY_SIZE(objects); ++i) {
 796                 unsigned int size = objects[i].size;
 797                 unsigned int expected_gtt = objects[i].gtt;
 798                 unsigned int offset = objects[i].offset;
 799                 unsigned int flags = PIN_USER;
 800 
 801                 for (single = 0; single <= 1; single++) {
 802                         obj = fake_huge_pages_object(i915, size, !!single);
 803                         if (IS_ERR(obj))
 804                                 return PTR_ERR(obj);
 805 
 806                         err = i915_gem_object_pin_pages(obj);
 807                         if (err)
 808                                 goto out_object_put;
 809 
 810                         /*
 811                          * Disable 2M pages -- We only want to use 64K/4K pages
 812                          * for this test.
 813                          */
 814                         obj->mm.page_sizes.sg &= ~I915_GTT_PAGE_SIZE_2M;
 815 
 816                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 817                         if (IS_ERR(vma)) {
 818                                 err = PTR_ERR(vma);
 819                                 goto out_object_unpin;
 820                         }
 821 
 822                         if (offset)
 823                                 flags |= PIN_OFFSET_FIXED | offset;
 824 
 825                         err = i915_vma_pin(vma, 0, 0, flags);
 826                         if (err)
 827                                 goto out_vma_close;
 828 
 829                         err = igt_check_page_sizes(vma);
 830                         if (err)
 831                                 goto out_vma_unpin;
 832 
 833                         if (!offset && vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
 834                                 if (!IS_ALIGNED(vma->node.start,
 835                                                 I915_GTT_PAGE_SIZE_2M)) {
 836                                         pr_err("node.start(%llx) not aligned to 2M\n",
 837                                                vma->node.start);
 838                                         err = -EINVAL;
 839                                         goto out_vma_unpin;
 840                                 }
 841 
 842                                 if (!IS_ALIGNED(vma->node.size,
 843                                                 I915_GTT_PAGE_SIZE_2M)) {
 844                                         pr_err("node.size(%llx) not aligned to 2M\n",
 845                                                vma->node.size);
 846                                         err = -EINVAL;
 847                                         goto out_vma_unpin;
 848                                 }
 849                         }
 850 
 851                         if (vma->page_sizes.gtt != expected_gtt) {
 852                                 pr_err("gtt=%u, expected=%u, i=%d, single=%s\n",
 853                                        vma->page_sizes.gtt, expected_gtt, i,
 854                                        yesno(!!single));
 855                                 err = -EINVAL;
 856                                 goto out_vma_unpin;
 857                         }
 858 
 859                         i915_vma_unpin(vma);
 860                         i915_vma_close(vma);
 861 
 862                         i915_gem_object_unpin_pages(obj);
 863                         __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 864                         i915_gem_object_put(obj);
 865                 }
 866         }
 867 
 868         return 0;
 869 
 870 out_vma_unpin:
 871         i915_vma_unpin(vma);
 872 out_vma_close:
 873         i915_vma_close(vma);
 874 out_object_unpin:
 875         i915_gem_object_unpin_pages(obj);
 876 out_object_put:
 877         i915_gem_object_put(obj);
 878 
 879         return err;
 880 }
 881 
 882 static int gpu_write(struct i915_vma *vma,
 883                      struct i915_gem_context *ctx,
 884                      struct intel_engine_cs *engine,
 885                      u32 dw,
 886                      u32 val)
 887 {
 888         int err;
 889 
 890         i915_gem_object_lock(vma->obj);
 891         err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
 892         i915_gem_object_unlock(vma->obj);
 893         if (err)
 894                 return err;
 895 
 896         return igt_gpu_fill_dw(vma, ctx, engine, dw * sizeof(u32),
 897                                vma->size >> PAGE_SHIFT, val);
 898 }
 899 
 900 static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val)
 901 {
 902         unsigned int needs_flush;
 903         unsigned long n;
 904         int err;
 905 
 906         err = i915_gem_object_prepare_read(obj, &needs_flush);
 907         if (err)
 908                 return err;
 909 
 910         for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) {
 911                 u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n));
 912 
 913                 if (needs_flush & CLFLUSH_BEFORE)
 914                         drm_clflush_virt_range(ptr, PAGE_SIZE);
 915 
 916                 if (ptr[dword] != val) {
 917                         pr_err("n=%lu ptr[%u]=%u, val=%u\n",
 918                                n, dword, ptr[dword], val);
 919                         kunmap_atomic(ptr);
 920                         err = -EINVAL;
 921                         break;
 922                 }
 923 
 924                 kunmap_atomic(ptr);
 925         }
 926 
 927         i915_gem_object_finish_access(obj);
 928 
 929         return err;
 930 }
 931 
 932 static int __igt_write_huge(struct i915_gem_context *ctx,
 933                             struct intel_engine_cs *engine,
 934                             struct drm_i915_gem_object *obj,
 935                             u64 size, u64 offset,
 936                             u32 dword, u32 val)
 937 {
 938         struct i915_address_space *vm = ctx->vm ?: &engine->gt->ggtt->vm;
 939         unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
 940         struct i915_vma *vma;
 941         int err;
 942 
 943         vma = i915_vma_instance(obj, vm, NULL);
 944         if (IS_ERR(vma))
 945                 return PTR_ERR(vma);
 946 
 947         err = i915_vma_unbind(vma);
 948         if (err)
 949                 goto out_vma_close;
 950 
 951         err = i915_vma_pin(vma, size, 0, flags | offset);
 952         if (err) {
 953                 /*
 954                  * The ggtt may have some pages reserved so
 955                  * refrain from erroring out.
 956                  */
 957                 if (err == -ENOSPC && i915_is_ggtt(vm))
 958                         err = 0;
 959 
 960                 goto out_vma_close;
 961         }
 962 
 963         err = igt_check_page_sizes(vma);
 964         if (err)
 965                 goto out_vma_unpin;
 966 
 967         err = gpu_write(vma, ctx, engine, dword, val);
 968         if (err) {
 969                 pr_err("gpu-write failed at offset=%llx\n", offset);
 970                 goto out_vma_unpin;
 971         }
 972 
 973         err = cpu_check(obj, dword, val);
 974         if (err) {
 975                 pr_err("cpu-check failed at offset=%llx\n", offset);
 976                 goto out_vma_unpin;
 977         }
 978 
 979 out_vma_unpin:
 980         i915_vma_unpin(vma);
 981 out_vma_close:
 982         i915_vma_destroy(vma);
 983 
 984         return err;
 985 }
 986 
 987 static int igt_write_huge(struct i915_gem_context *ctx,
 988                           struct drm_i915_gem_object *obj)
 989 {
 990         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 991         struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm;
 992         static struct intel_engine_cs *engines[I915_NUM_ENGINES];
 993         struct intel_engine_cs *engine;
 994         I915_RND_STATE(prng);
 995         IGT_TIMEOUT(end_time);
 996         unsigned int max_page_size;
 997         unsigned int id;
 998         u64 max;
 999         u64 num;
1000         u64 size;
1001         int *order;
1002         int i, n;
1003         int err = 0;
1004 
1005         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
1006 
1007         size = obj->base.size;
1008         if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
1009                 size = round_up(size, I915_GTT_PAGE_SIZE_2M);
1010 
1011         max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg);
1012         max = div_u64((vm->total - size), max_page_size);
1013 
1014         n = 0;
1015         for_each_engine(engine, i915, id) {
1016                 if (!intel_engine_can_store_dword(engine)) {
1017                         pr_info("store-dword-imm not supported on engine=%u\n",
1018                                 id);
1019                         continue;
1020                 }
1021                 engines[n++] = engine;
1022         }
1023 
1024         if (!n)
1025                 return 0;
1026 
1027         /*
1028          * To keep things interesting when alternating between engines in our
1029          * randomized order, lets also make feeding to the same engine a few
1030          * times in succession a possibility by enlarging the permutation array.
1031          */
1032         order = i915_random_order(n * I915_NUM_ENGINES, &prng);
1033         if (!order)
1034                 return -ENOMEM;
1035 
1036         /*
1037          * Try various offsets in an ascending/descending fashion until we
1038          * timeout -- we want to avoid issues hidden by effectively always using
1039          * offset = 0.
1040          */
1041         i = 0;
1042         for_each_prime_number_from(num, 0, max) {
1043                 u64 offset_low = num * max_page_size;
1044                 u64 offset_high = (max - num) * max_page_size;
1045                 u32 dword = offset_in_page(num) / 4;
1046 
1047                 engine = engines[order[i] % n];
1048                 i = (i + 1) % (n * I915_NUM_ENGINES);
1049 
1050                 /*
1051                  * In order to utilize 64K pages we need to both pad the vma
1052                  * size and ensure the vma offset is at the start of the pt
1053                  * boundary, however to improve coverage we opt for testing both
1054                  * aligned and unaligned offsets.
1055                  */
1056                 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
1057                         offset_low = round_down(offset_low,
1058                                                 I915_GTT_PAGE_SIZE_2M);
1059 
1060                 err = __igt_write_huge(ctx, engine, obj, size, offset_low,
1061                                        dword, num + 1);
1062                 if (err)
1063                         break;
1064 
1065                 err = __igt_write_huge(ctx, engine, obj, size, offset_high,
1066                                        dword, num + 1);
1067                 if (err)
1068                         break;
1069 
1070                 if (igt_timeout(end_time,
1071                                 "%s timed out on engine=%u, offset_low=%llx offset_high=%llx, max_page_size=%x\n",
1072                                 __func__, engine->id, offset_low, offset_high,
1073                                 max_page_size))
1074                         break;
1075         }
1076 
1077         kfree(order);
1078 
1079         return err;
1080 }
1081 
1082 static int igt_ppgtt_exhaust_huge(void *arg)
1083 {
1084         struct i915_gem_context *ctx = arg;
1085         struct drm_i915_private *i915 = ctx->i915;
1086         unsigned long supported = INTEL_INFO(i915)->page_sizes;
1087         static unsigned int pages[ARRAY_SIZE(page_sizes)];
1088         struct drm_i915_gem_object *obj;
1089         unsigned int size_mask;
1090         unsigned int page_mask;
1091         int n, i;
1092         int err = -ENODEV;
1093 
1094         if (supported == I915_GTT_PAGE_SIZE_4K)
1095                 return 0;
1096 
1097         /*
1098          * Sanity check creating objects with a varying mix of page sizes --
1099          * ensuring that our writes lands in the right place.
1100          */
1101 
1102         n = 0;
1103         for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1)
1104                 pages[n++] = BIT(i);
1105 
1106         for (size_mask = 2; size_mask < BIT(n); size_mask++) {
1107                 unsigned int size = 0;
1108 
1109                 for (i = 0; i < n; i++) {
1110                         if (size_mask & BIT(i))
1111                                 size |= pages[i];
1112                 }
1113 
1114                 /*
1115                  * For our page mask we want to enumerate all the page-size
1116                  * combinations which will fit into our chosen object size.
1117                  */
1118                 for (page_mask = 2; page_mask <= size_mask; page_mask++) {
1119                         unsigned int page_sizes = 0;
1120 
1121                         for (i = 0; i < n; i++) {
1122                                 if (page_mask & BIT(i))
1123                                         page_sizes |= pages[i];
1124                         }
1125 
1126                         /*
1127                          * Ensure that we can actually fill the given object
1128                          * with our chosen page mask.
1129                          */
1130                         if (!IS_ALIGNED(size, BIT(__ffs(page_sizes))))
1131                                 continue;
1132 
1133                         obj = huge_pages_object(i915, size, page_sizes);
1134                         if (IS_ERR(obj)) {
1135                                 err = PTR_ERR(obj);
1136                                 goto out_device;
1137                         }
1138 
1139                         err = i915_gem_object_pin_pages(obj);
1140                         if (err) {
1141                                 i915_gem_object_put(obj);
1142 
1143                                 if (err == -ENOMEM) {
1144                                         pr_info("unable to get pages, size=%u, pages=%u\n",
1145                                                 size, page_sizes);
1146                                         err = 0;
1147                                         break;
1148                                 }
1149 
1150                                 pr_err("pin_pages failed, size=%u, pages=%u\n",
1151                                        size_mask, page_mask);
1152 
1153                                 goto out_device;
1154                         }
1155 
1156                         /* Force the page-size for the gtt insertion */
1157                         obj->mm.page_sizes.sg = page_sizes;
1158 
1159                         err = igt_write_huge(ctx, obj);
1160                         if (err) {
1161                                 pr_err("exhaust write-huge failed with size=%u\n",
1162                                        size);
1163                                 goto out_unpin;
1164                         }
1165 
1166                         i915_gem_object_unpin_pages(obj);
1167                         __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
1168                         i915_gem_object_put(obj);
1169                 }
1170         }
1171 
1172         goto out_device;
1173 
1174 out_unpin:
1175         i915_gem_object_unpin_pages(obj);
1176         i915_gem_object_put(obj);
1177 out_device:
1178         mkwrite_device_info(i915)->page_sizes = supported;
1179 
1180         return err;
1181 }
1182 
1183 static int igt_ppgtt_internal_huge(void *arg)
1184 {
1185         struct i915_gem_context *ctx = arg;
1186         struct drm_i915_private *i915 = ctx->i915;
1187         struct drm_i915_gem_object *obj;
1188         static const unsigned int sizes[] = {
1189                 SZ_64K,
1190                 SZ_128K,
1191                 SZ_256K,
1192                 SZ_512K,
1193                 SZ_1M,
1194                 SZ_2M,
1195         };
1196         int i;
1197         int err;
1198 
1199         /*
1200          * Sanity check that the HW uses huge pages correctly through internal
1201          * -- ensure that our writes land in the right place.
1202          */
1203 
1204         for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
1205                 unsigned int size = sizes[i];
1206 
1207                 obj = i915_gem_object_create_internal(i915, size);
1208                 if (IS_ERR(obj))
1209                         return PTR_ERR(obj);
1210 
1211                 err = i915_gem_object_pin_pages(obj);
1212                 if (err)
1213                         goto out_put;
1214 
1215                 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_64K) {
1216                         pr_info("internal unable to allocate huge-page(s) with size=%u\n",
1217                                 size);
1218                         goto out_unpin;
1219                 }
1220 
1221                 err = igt_write_huge(ctx, obj);
1222                 if (err) {
1223                         pr_err("internal write-huge failed with size=%u\n",
1224                                size);
1225                         goto out_unpin;
1226                 }
1227 
1228                 i915_gem_object_unpin_pages(obj);
1229                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
1230                 i915_gem_object_put(obj);
1231         }
1232 
1233         return 0;
1234 
1235 out_unpin:
1236         i915_gem_object_unpin_pages(obj);
1237 out_put:
1238         i915_gem_object_put(obj);
1239 
1240         return err;
1241 }
1242 
1243 static inline bool igt_can_allocate_thp(struct drm_i915_private *i915)
1244 {
1245         return i915->mm.gemfs && has_transparent_hugepage();
1246 }
1247 
1248 static int igt_ppgtt_gemfs_huge(void *arg)
1249 {
1250         struct i915_gem_context *ctx = arg;
1251         struct drm_i915_private *i915 = ctx->i915;
1252         struct drm_i915_gem_object *obj;
1253         static const unsigned int sizes[] = {
1254                 SZ_2M,
1255                 SZ_4M,
1256                 SZ_8M,
1257                 SZ_16M,
1258                 SZ_32M,
1259         };
1260         int i;
1261         int err;
1262 
1263         /*
1264          * Sanity check that the HW uses huge pages correctly through gemfs --
1265          * ensure that our writes land in the right place.
1266          */
1267 
1268         if (!igt_can_allocate_thp(i915)) {
1269                 pr_info("missing THP support, skipping\n");
1270                 return 0;
1271         }
1272 
1273         for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
1274                 unsigned int size = sizes[i];
1275 
1276                 obj = i915_gem_object_create_shmem(i915, size);
1277                 if (IS_ERR(obj))
1278                         return PTR_ERR(obj);
1279 
1280                 err = i915_gem_object_pin_pages(obj);
1281                 if (err)
1282                         goto out_put;
1283 
1284                 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) {
1285                         pr_info("finishing test early, gemfs unable to allocate huge-page(s) with size=%u\n",
1286                                 size);
1287                         goto out_unpin;
1288                 }
1289 
1290                 err = igt_write_huge(ctx, obj);
1291                 if (err) {
1292                         pr_err("gemfs write-huge failed with size=%u\n",
1293                                size);
1294                         goto out_unpin;
1295                 }
1296 
1297                 i915_gem_object_unpin_pages(obj);
1298                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
1299                 i915_gem_object_put(obj);
1300         }
1301 
1302         return 0;
1303 
1304 out_unpin:
1305         i915_gem_object_unpin_pages(obj);
1306 out_put:
1307         i915_gem_object_put(obj);
1308 
1309         return err;
1310 }
1311 
1312 static int igt_ppgtt_pin_update(void *arg)
1313 {
1314         struct i915_gem_context *ctx = arg;
1315         struct drm_i915_private *dev_priv = ctx->i915;
1316         unsigned long supported = INTEL_INFO(dev_priv)->page_sizes;
1317         struct i915_address_space *vm = ctx->vm;
1318         struct drm_i915_gem_object *obj;
1319         struct i915_vma *vma;
1320         unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1321         struct intel_engine_cs *engine;
1322         enum intel_engine_id id;
1323         unsigned int n;
1324         int first, last;
1325         int err;
1326 
1327         /*
1328          * Make sure there's no funny business when doing a PIN_UPDATE -- in the
1329          * past we had a subtle issue with being able to incorrectly do multiple
1330          * alloc va ranges on the same object when doing a PIN_UPDATE, which
1331          * resulted in some pretty nasty bugs, though only when using
1332          * huge-gtt-pages.
1333          */
1334 
1335         if (!vm || !i915_vm_is_4lvl(vm)) {
1336                 pr_info("48b PPGTT not supported, skipping\n");
1337                 return 0;
1338         }
1339 
1340         first = ilog2(I915_GTT_PAGE_SIZE_64K);
1341         last = ilog2(I915_GTT_PAGE_SIZE_2M);
1342 
1343         for_each_set_bit_from(first, &supported, last + 1) {
1344                 unsigned int page_size = BIT(first);
1345 
1346                 obj = i915_gem_object_create_internal(dev_priv, page_size);
1347                 if (IS_ERR(obj))
1348                         return PTR_ERR(obj);
1349 
1350                 vma = i915_vma_instance(obj, vm, NULL);
1351                 if (IS_ERR(vma)) {
1352                         err = PTR_ERR(vma);
1353                         goto out_put;
1354                 }
1355 
1356                 err = i915_vma_pin(vma, SZ_2M, 0, flags);
1357                 if (err)
1358                         goto out_close;
1359 
1360                 if (vma->page_sizes.sg < page_size) {
1361                         pr_info("Unable to allocate page-size %x, finishing test early\n",
1362                                 page_size);
1363                         goto out_unpin;
1364                 }
1365 
1366                 err = igt_check_page_sizes(vma);
1367                 if (err)
1368                         goto out_unpin;
1369 
1370                 if (vma->page_sizes.gtt != page_size) {
1371                         dma_addr_t addr = i915_gem_object_get_dma_address(obj, 0);
1372 
1373                         /*
1374                          * The only valid reason for this to ever fail would be
1375                          * if the dma-mapper screwed us over when we did the
1376                          * dma_map_sg(), since it has the final say over the dma
1377                          * address.
1378                          */
1379                         if (IS_ALIGNED(addr, page_size)) {
1380                                 pr_err("page_sizes.gtt=%u, expected=%u\n",
1381                                        vma->page_sizes.gtt, page_size);
1382                                 err = -EINVAL;
1383                         } else {
1384                                 pr_info("dma address misaligned, finishing test early\n");
1385                         }
1386 
1387                         goto out_unpin;
1388                 }
1389 
1390                 err = i915_vma_bind(vma, I915_CACHE_NONE, PIN_UPDATE);
1391                 if (err)
1392                         goto out_unpin;
1393 
1394                 i915_vma_unpin(vma);
1395                 i915_vma_close(vma);
1396 
1397                 i915_gem_object_put(obj);
1398         }
1399 
1400         obj = i915_gem_object_create_internal(dev_priv, PAGE_SIZE);
1401         if (IS_ERR(obj))
1402                 return PTR_ERR(obj);
1403 
1404         vma = i915_vma_instance(obj, vm, NULL);
1405         if (IS_ERR(vma)) {
1406                 err = PTR_ERR(vma);
1407                 goto out_put;
1408         }
1409 
1410         err = i915_vma_pin(vma, 0, 0, flags);
1411         if (err)
1412                 goto out_close;
1413 
1414         /*
1415          * Make sure we don't end up with something like where the pde is still
1416          * pointing to the 2M page, and the pt we just filled-in is dangling --
1417          * we can check this by writing to the first page where it would then
1418          * land in the now stale 2M page.
1419          */
1420 
1421         n = 0;
1422         for_each_engine(engine, dev_priv, id) {
1423                 if (!intel_engine_can_store_dword(engine))
1424                         continue;
1425 
1426                 err = gpu_write(vma, ctx, engine, n++, 0xdeadbeaf);
1427                 if (err)
1428                         goto out_unpin;
1429         }
1430         while (n--) {
1431                 err = cpu_check(obj, n, 0xdeadbeaf);
1432                 if (err)
1433                         goto out_unpin;
1434         }
1435 
1436 out_unpin:
1437         i915_vma_unpin(vma);
1438 out_close:
1439         i915_vma_close(vma);
1440 out_put:
1441         i915_gem_object_put(obj);
1442 
1443         return err;
1444 }
1445 
1446 static int igt_tmpfs_fallback(void *arg)
1447 {
1448         struct i915_gem_context *ctx = arg;
1449         struct drm_i915_private *i915 = ctx->i915;
1450         struct vfsmount *gemfs = i915->mm.gemfs;
1451         struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm;
1452         struct drm_i915_gem_object *obj;
1453         struct i915_vma *vma;
1454         u32 *vaddr;
1455         int err = 0;
1456 
1457         /*
1458          * Make sure that we don't burst into a ball of flames upon falling back
1459          * to tmpfs, which we rely on if on the off-chance we encouter a failure
1460          * when setting up gemfs.
1461          */
1462 
1463         i915->mm.gemfs = NULL;
1464 
1465         obj = i915_gem_object_create_shmem(i915, PAGE_SIZE);
1466         if (IS_ERR(obj)) {
1467                 err = PTR_ERR(obj);
1468                 goto out_restore;
1469         }
1470 
1471         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
1472         if (IS_ERR(vaddr)) {
1473                 err = PTR_ERR(vaddr);
1474                 goto out_put;
1475         }
1476         *vaddr = 0xdeadbeaf;
1477 
1478         __i915_gem_object_flush_map(obj, 0, 64);
1479         i915_gem_object_unpin_map(obj);
1480 
1481         vma = i915_vma_instance(obj, vm, NULL);
1482         if (IS_ERR(vma)) {
1483                 err = PTR_ERR(vma);
1484                 goto out_put;
1485         }
1486 
1487         err = i915_vma_pin(vma, 0, 0, PIN_USER);
1488         if (err)
1489                 goto out_close;
1490 
1491         err = igt_check_page_sizes(vma);
1492 
1493         i915_vma_unpin(vma);
1494 out_close:
1495         i915_vma_close(vma);
1496 out_put:
1497         i915_gem_object_put(obj);
1498 out_restore:
1499         i915->mm.gemfs = gemfs;
1500 
1501         return err;
1502 }
1503 
1504 static int igt_shrink_thp(void *arg)
1505 {
1506         struct i915_gem_context *ctx = arg;
1507         struct drm_i915_private *i915 = ctx->i915;
1508         struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm;
1509         struct drm_i915_gem_object *obj;
1510         struct intel_engine_cs *engine;
1511         enum intel_engine_id id;
1512         struct i915_vma *vma;
1513         unsigned int flags = PIN_USER;
1514         unsigned int n;
1515         int err;
1516 
1517         /*
1518          * Sanity check shrinking huge-paged object -- make sure nothing blows
1519          * up.
1520          */
1521 
1522         if (!igt_can_allocate_thp(i915)) {
1523                 pr_info("missing THP support, skipping\n");
1524                 return 0;
1525         }
1526 
1527         obj = i915_gem_object_create_shmem(i915, SZ_2M);
1528         if (IS_ERR(obj))
1529                 return PTR_ERR(obj);
1530 
1531         vma = i915_vma_instance(obj, vm, NULL);
1532         if (IS_ERR(vma)) {
1533                 err = PTR_ERR(vma);
1534                 goto out_put;
1535         }
1536 
1537         err = i915_vma_pin(vma, 0, 0, flags);
1538         if (err)
1539                 goto out_close;
1540 
1541         if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) {
1542                 pr_info("failed to allocate THP, finishing test early\n");
1543                 goto out_unpin;
1544         }
1545 
1546         err = igt_check_page_sizes(vma);
1547         if (err)
1548                 goto out_unpin;
1549 
1550         n = 0;
1551         for_each_engine(engine, i915, id) {
1552                 if (!intel_engine_can_store_dword(engine))
1553                         continue;
1554 
1555                 err = gpu_write(vma, ctx, engine, n++, 0xdeadbeaf);
1556                 if (err)
1557                         goto out_unpin;
1558         }
1559 
1560         i915_vma_unpin(vma);
1561 
1562         /*
1563          * Now that the pages are *unpinned* shrink-all should invoke
1564          * shmem to truncate our pages.
1565          */
1566         i915_gem_shrink_all(i915);
1567         if (i915_gem_object_has_pages(obj)) {
1568                 pr_err("shrink-all didn't truncate the pages\n");
1569                 err = -EINVAL;
1570                 goto out_close;
1571         }
1572 
1573         if (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys) {
1574                 pr_err("residual page-size bits left\n");
1575                 err = -EINVAL;
1576                 goto out_close;
1577         }
1578 
1579         err = i915_vma_pin(vma, 0, 0, flags);
1580         if (err)
1581                 goto out_close;
1582 
1583         while (n--) {
1584                 err = cpu_check(obj, n, 0xdeadbeaf);
1585                 if (err)
1586                         goto out_unpin;
1587         }
1588 
1589 
1590 out_unpin:
1591         i915_vma_unpin(vma);
1592 out_close:
1593         i915_vma_close(vma);
1594 out_put:
1595         i915_gem_object_put(obj);
1596 
1597         return err;
1598 }
1599 
1600 int i915_gem_huge_page_mock_selftests(void)
1601 {
1602         static const struct i915_subtest tests[] = {
1603                 SUBTEST(igt_mock_exhaust_device_supported_pages),
1604                 SUBTEST(igt_mock_ppgtt_misaligned_dma),
1605                 SUBTEST(igt_mock_ppgtt_huge_fill),
1606                 SUBTEST(igt_mock_ppgtt_64K),
1607         };
1608         struct drm_i915_private *dev_priv;
1609         struct i915_ppgtt *ppgtt;
1610         int err;
1611 
1612         dev_priv = mock_gem_device();
1613         if (!dev_priv)
1614                 return -ENOMEM;
1615 
1616         /* Pretend to be a device which supports the 48b PPGTT */
1617         mkwrite_device_info(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL;
1618         mkwrite_device_info(dev_priv)->ppgtt_size = 48;
1619 
1620         mutex_lock(&dev_priv->drm.struct_mutex);
1621         ppgtt = i915_ppgtt_create(dev_priv);
1622         if (IS_ERR(ppgtt)) {
1623                 err = PTR_ERR(ppgtt);
1624                 goto out_unlock;
1625         }
1626 
1627         if (!i915_vm_is_4lvl(&ppgtt->vm)) {
1628                 pr_err("failed to create 48b PPGTT\n");
1629                 err = -EINVAL;
1630                 goto out_close;
1631         }
1632 
1633         /* If we were ever hit this then it's time to mock the 64K scratch */
1634         if (!i915_vm_has_scratch_64K(&ppgtt->vm)) {
1635                 pr_err("PPGTT missing 64K scratch page\n");
1636                 err = -EINVAL;
1637                 goto out_close;
1638         }
1639 
1640         err = i915_subtests(tests, ppgtt);
1641 
1642 out_close:
1643         i915_vm_put(&ppgtt->vm);
1644 
1645 out_unlock:
1646         mutex_unlock(&dev_priv->drm.struct_mutex);
1647         drm_dev_put(&dev_priv->drm);
1648 
1649         return err;
1650 }
1651 
1652 int i915_gem_huge_page_live_selftests(struct drm_i915_private *i915)
1653 {
1654         static const struct i915_subtest tests[] = {
1655                 SUBTEST(igt_shrink_thp),
1656                 SUBTEST(igt_ppgtt_pin_update),
1657                 SUBTEST(igt_tmpfs_fallback),
1658                 SUBTEST(igt_ppgtt_exhaust_huge),
1659                 SUBTEST(igt_ppgtt_gemfs_huge),
1660                 SUBTEST(igt_ppgtt_internal_huge),
1661         };
1662         struct drm_file *file;
1663         struct i915_gem_context *ctx;
1664         intel_wakeref_t wakeref;
1665         int err;
1666 
1667         if (!HAS_PPGTT(i915)) {
1668                 pr_info("PPGTT not supported, skipping live-selftests\n");
1669                 return 0;
1670         }
1671 
1672         if (intel_gt_is_wedged(&i915->gt))
1673                 return 0;
1674 
1675         file = mock_file(i915);
1676         if (IS_ERR(file))
1677                 return PTR_ERR(file);
1678 
1679         mutex_lock(&i915->drm.struct_mutex);
1680         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1681 
1682         ctx = live_context(i915, file);
1683         if (IS_ERR(ctx)) {
1684                 err = PTR_ERR(ctx);
1685                 goto out_unlock;
1686         }
1687 
1688         if (ctx->vm)
1689                 ctx->vm->scrub_64K = true;
1690 
1691         err = i915_subtests(tests, ctx);
1692 
1693 out_unlock:
1694         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1695         mutex_unlock(&i915->drm.struct_mutex);
1696 
1697         mock_file_free(i915, file);
1698 
1699         return err;
1700 }
/* [<][>][^][v][top][bottom][index][help] */
root/drivers/gpu/drm/i915/gem/selftests/huge_pages.c

DEFINITIONS
