This source file includes following definitions.
- i965_write_fence_reg
- i915_write_fence_reg
- i830_write_fence_reg
- fence_write
- fence_update
- i915_vma_revoke_fence
- fence_find
- __i915_vma_pin_fence
- i915_vma_pin_fence
- i915_reserve_fence
- i915_unreserve_fence
- i915_gem_restore_fences
- detect_bit_6_swizzle
- i915_gem_swizzle_page
- i915_gem_object_do_bit_17_swizzle
- i915_gem_object_save_bit_17_swizzle
- i915_ggtt_init_fences
- intel_gt_init_swizzling
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24 #include <drm/i915_drm.h>
25
26 #include "i915_drv.h"
27 #include "i915_scatterlist.h"
28 #include "i915_vgpu.h"
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60 #define pipelined 0
61
62 static void i965_write_fence_reg(struct i915_fence_reg *fence,
63 struct i915_vma *vma)
64 {
65 i915_reg_t fence_reg_lo, fence_reg_hi;
66 int fence_pitch_shift;
67 u64 val;
68
69 if (INTEL_GEN(fence->i915) >= 6) {
70 fence_reg_lo = FENCE_REG_GEN6_LO(fence->id);
71 fence_reg_hi = FENCE_REG_GEN6_HI(fence->id);
72 fence_pitch_shift = GEN6_FENCE_PITCH_SHIFT;
73
74 } else {
75 fence_reg_lo = FENCE_REG_965_LO(fence->id);
76 fence_reg_hi = FENCE_REG_965_HI(fence->id);
77 fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
78 }
79
80 val = 0;
81 if (vma) {
82 unsigned int stride = i915_gem_object_get_stride(vma->obj);
83
84 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
85 GEM_BUG_ON(!IS_ALIGNED(vma->node.start, I965_FENCE_PAGE));
86 GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I965_FENCE_PAGE));
87 GEM_BUG_ON(!IS_ALIGNED(stride, 128));
88
89 val = (vma->node.start + vma->fence_size - I965_FENCE_PAGE) << 32;
90 val |= vma->node.start;
91 val |= (u64)((stride / 128) - 1) << fence_pitch_shift;
92 if (i915_gem_object_get_tiling(vma->obj) == I915_TILING_Y)
93 val |= BIT(I965_FENCE_TILING_Y_SHIFT);
94 val |= I965_FENCE_REG_VALID;
95 }
96
97 if (!pipelined) {
98 struct intel_uncore *uncore = &fence->i915->uncore;
99
100
101
102
103
104
105
106
107
108
109
110 intel_uncore_write_fw(uncore, fence_reg_lo, 0);
111 intel_uncore_posting_read_fw(uncore, fence_reg_lo);
112
113 intel_uncore_write_fw(uncore, fence_reg_hi, upper_32_bits(val));
114 intel_uncore_write_fw(uncore, fence_reg_lo, lower_32_bits(val));
115 intel_uncore_posting_read_fw(uncore, fence_reg_lo);
116 }
117 }
118
119 static void i915_write_fence_reg(struct i915_fence_reg *fence,
120 struct i915_vma *vma)
121 {
122 u32 val;
123
124 val = 0;
125 if (vma) {
126 unsigned int tiling = i915_gem_object_get_tiling(vma->obj);
127 bool is_y_tiled = tiling == I915_TILING_Y;
128 unsigned int stride = i915_gem_object_get_stride(vma->obj);
129
130 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
131 GEM_BUG_ON(vma->node.start & ~I915_FENCE_START_MASK);
132 GEM_BUG_ON(!is_power_of_2(vma->fence_size));
133 GEM_BUG_ON(!IS_ALIGNED(vma->node.start, vma->fence_size));
134
135 if (is_y_tiled && HAS_128_BYTE_Y_TILING(fence->i915))
136 stride /= 128;
137 else
138 stride /= 512;
139 GEM_BUG_ON(!is_power_of_2(stride));
140
141 val = vma->node.start;
142 if (is_y_tiled)
143 val |= BIT(I830_FENCE_TILING_Y_SHIFT);
144 val |= I915_FENCE_SIZE_BITS(vma->fence_size);
145 val |= ilog2(stride) << I830_FENCE_PITCH_SHIFT;
146
147 val |= I830_FENCE_REG_VALID;
148 }
149
150 if (!pipelined) {
151 struct intel_uncore *uncore = &fence->i915->uncore;
152 i915_reg_t reg = FENCE_REG(fence->id);
153
154 intel_uncore_write_fw(uncore, reg, val);
155 intel_uncore_posting_read_fw(uncore, reg);
156 }
157 }
158
159 static void i830_write_fence_reg(struct i915_fence_reg *fence,
160 struct i915_vma *vma)
161 {
162 u32 val;
163
164 val = 0;
165 if (vma) {
166 unsigned int stride = i915_gem_object_get_stride(vma->obj);
167
168 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
169 GEM_BUG_ON(vma->node.start & ~I830_FENCE_START_MASK);
170 GEM_BUG_ON(!is_power_of_2(vma->fence_size));
171 GEM_BUG_ON(!is_power_of_2(stride / 128));
172 GEM_BUG_ON(!IS_ALIGNED(vma->node.start, vma->fence_size));
173
174 val = vma->node.start;
175 if (i915_gem_object_get_tiling(vma->obj) == I915_TILING_Y)
176 val |= BIT(I830_FENCE_TILING_Y_SHIFT);
177 val |= I830_FENCE_SIZE_BITS(vma->fence_size);
178 val |= ilog2(stride / 128) << I830_FENCE_PITCH_SHIFT;
179 val |= I830_FENCE_REG_VALID;
180 }
181
182 if (!pipelined) {
183 struct intel_uncore *uncore = &fence->i915->uncore;
184 i915_reg_t reg = FENCE_REG(fence->id);
185
186 intel_uncore_write_fw(uncore, reg, val);
187 intel_uncore_posting_read_fw(uncore, reg);
188 }
189 }
190
191 static void fence_write(struct i915_fence_reg *fence,
192 struct i915_vma *vma)
193 {
194
195
196
197
198
199
200 if (IS_GEN(fence->i915, 2))
201 i830_write_fence_reg(fence, vma);
202 else if (IS_GEN(fence->i915, 3))
203 i915_write_fence_reg(fence, vma);
204 else
205 i965_write_fence_reg(fence, vma);
206
207
208
209
210
211
212 fence->dirty = false;
213 }
214
215 static int fence_update(struct i915_fence_reg *fence,
216 struct i915_vma *vma)
217 {
218 intel_wakeref_t wakeref;
219 struct i915_vma *old;
220 int ret;
221
222 if (vma) {
223 if (!i915_vma_is_map_and_fenceable(vma))
224 return -EINVAL;
225
226 if (WARN(!i915_gem_object_get_stride(vma->obj) ||
227 !i915_gem_object_get_tiling(vma->obj),
228 "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
229 i915_gem_object_get_stride(vma->obj),
230 i915_gem_object_get_tiling(vma->obj)))
231 return -EINVAL;
232
233 ret = i915_active_wait(&vma->active);
234 if (ret)
235 return ret;
236 }
237
238 old = xchg(&fence->vma, NULL);
239 if (old) {
240 ret = i915_active_wait(&old->active);
241 if (ret) {
242 fence->vma = old;
243 return ret;
244 }
245
246 i915_vma_flush_writes(old);
247
248
249
250
251
252 if (old != vma) {
253 GEM_BUG_ON(old->fence != fence);
254 i915_vma_revoke_mmap(old);
255 old->fence = NULL;
256 }
257
258 list_move(&fence->link, &fence->i915->ggtt.fence_list);
259 }
260
261
262
263
264
265
266
267
268
269
270
271 wakeref = intel_runtime_pm_get_if_in_use(&fence->i915->runtime_pm);
272 if (!wakeref) {
273 GEM_BUG_ON(vma);
274 return 0;
275 }
276
277 WRITE_ONCE(fence->vma, vma);
278 fence_write(fence, vma);
279
280 if (vma) {
281 vma->fence = fence;
282 list_move_tail(&fence->link, &fence->i915->ggtt.fence_list);
283 }
284
285 intel_runtime_pm_put(&fence->i915->runtime_pm, wakeref);
286 return 0;
287 }
288
289
290
291
292
293
294
295
296
297
298
299
300 int i915_vma_revoke_fence(struct i915_vma *vma)
301 {
302 struct i915_fence_reg *fence = vma->fence;
303
304 lockdep_assert_held(&vma->vm->mutex);
305 if (!fence)
306 return 0;
307
308 if (atomic_read(&fence->pin_count))
309 return -EBUSY;
310
311 return fence_update(fence, NULL);
312 }
313
314 static struct i915_fence_reg *fence_find(struct drm_i915_private *i915)
315 {
316 struct i915_fence_reg *fence;
317
318 list_for_each_entry(fence, &i915->ggtt.fence_list, link) {
319 GEM_BUG_ON(fence->vma && fence->vma->fence != fence);
320
321 if (atomic_read(&fence->pin_count))
322 continue;
323
324 return fence;
325 }
326
327
328 if (intel_has_pending_fb_unpin(i915))
329 return ERR_PTR(-EAGAIN);
330
331 return ERR_PTR(-EDEADLK);
332 }
333
334 static int __i915_vma_pin_fence(struct i915_vma *vma)
335 {
336 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vma->vm);
337 struct i915_fence_reg *fence;
338 struct i915_vma *set = i915_gem_object_is_tiled(vma->obj) ? vma : NULL;
339 int err;
340
341
342 if (vma->fence) {
343 fence = vma->fence;
344 GEM_BUG_ON(fence->vma != vma);
345 atomic_inc(&fence->pin_count);
346 if (!fence->dirty) {
347 list_move_tail(&fence->link, &ggtt->fence_list);
348 return 0;
349 }
350 } else if (set) {
351 fence = fence_find(vma->vm->i915);
352 if (IS_ERR(fence))
353 return PTR_ERR(fence);
354
355 GEM_BUG_ON(atomic_read(&fence->pin_count));
356 atomic_inc(&fence->pin_count);
357 } else {
358 return 0;
359 }
360
361 err = fence_update(fence, set);
362 if (err)
363 goto out_unpin;
364
365 GEM_BUG_ON(fence->vma != set);
366 GEM_BUG_ON(vma->fence != (set ? fence : NULL));
367
368 if (set)
369 return 0;
370
371 out_unpin:
372 atomic_dec(&fence->pin_count);
373 return err;
374 }
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394 int i915_vma_pin_fence(struct i915_vma *vma)
395 {
396 int err;
397
398
399
400
401
402 assert_rpm_wakelock_held(&vma->vm->i915->runtime_pm);
403 GEM_BUG_ON(!i915_vma_is_pinned(vma));
404 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
405
406 err = mutex_lock_interruptible(&vma->vm->mutex);
407 if (err)
408 return err;
409
410 err = __i915_vma_pin_fence(vma);
411 mutex_unlock(&vma->vm->mutex);
412
413 return err;
414 }
415
416
417
418
419
420
421
422
423 struct i915_fence_reg *i915_reserve_fence(struct drm_i915_private *i915)
424 {
425 struct i915_ggtt *ggtt = &i915->ggtt;
426 struct i915_fence_reg *fence;
427 int count;
428 int ret;
429
430 lockdep_assert_held(&ggtt->vm.mutex);
431
432
433 count = 0;
434 list_for_each_entry(fence, &ggtt->fence_list, link)
435 count += !atomic_read(&fence->pin_count);
436 if (count <= 1)
437 return ERR_PTR(-ENOSPC);
438
439 fence = fence_find(i915);
440 if (IS_ERR(fence))
441 return fence;
442
443 if (fence->vma) {
444
445 ret = fence_update(fence, NULL);
446 if (ret)
447 return ERR_PTR(ret);
448 }
449
450 list_del(&fence->link);
451
452 return fence;
453 }
454
455
456
457
458
459
460
461 void i915_unreserve_fence(struct i915_fence_reg *fence)
462 {
463 struct i915_ggtt *ggtt = &fence->i915->ggtt;
464
465 lockdep_assert_held(&ggtt->vm.mutex);
466
467 list_add(&fence->link, &ggtt->fence_list);
468 }
469
470
471
472
473
474
475
476
477
478 void i915_gem_restore_fences(struct drm_i915_private *i915)
479 {
480 int i;
481
482 rcu_read_lock();
483 for (i = 0; i < i915->ggtt.num_fences; i++) {
484 struct i915_fence_reg *reg = &i915->ggtt.fence_regs[i];
485 struct i915_vma *vma = READ_ONCE(reg->vma);
486
487 GEM_BUG_ON(vma && vma->fence != reg);
488
489
490
491
492
493 if (vma && !i915_gem_object_is_tiled(vma->obj))
494 vma = NULL;
495
496 fence_write(reg, vma);
497 }
498 rcu_read_unlock();
499 }
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556 static void detect_bit_6_swizzle(struct drm_i915_private *i915)
557 {
558 struct intel_uncore *uncore = &i915->uncore;
559 u32 swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
560 u32 swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
561
562 if (INTEL_GEN(i915) >= 8 || IS_VALLEYVIEW(i915)) {
563
564
565
566
567
568
569
570 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
571 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
572 } else if (INTEL_GEN(i915) >= 6) {
573 if (i915->preserve_bios_swizzle) {
574 if (intel_uncore_read(uncore, DISP_ARB_CTL) &
575 DISP_TILE_SURFACE_SWIZZLING) {
576 swizzle_x = I915_BIT_6_SWIZZLE_9_10;
577 swizzle_y = I915_BIT_6_SWIZZLE_9;
578 } else {
579 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
580 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
581 }
582 } else {
583 u32 dimm_c0, dimm_c1;
584 dimm_c0 = intel_uncore_read(uncore, MAD_DIMM_C0);
585 dimm_c1 = intel_uncore_read(uncore, MAD_DIMM_C1);
586 dimm_c0 &= MAD_DIMM_A_SIZE_MASK | MAD_DIMM_B_SIZE_MASK;
587 dimm_c1 &= MAD_DIMM_A_SIZE_MASK | MAD_DIMM_B_SIZE_MASK;
588
589
590
591
592
593
594
595 if (dimm_c0 == dimm_c1) {
596 swizzle_x = I915_BIT_6_SWIZZLE_9_10;
597 swizzle_y = I915_BIT_6_SWIZZLE_9;
598 } else {
599 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
600 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
601 }
602 }
603 } else if (IS_GEN(i915, 5)) {
604
605
606
607
608 swizzle_x = I915_BIT_6_SWIZZLE_9_10;
609 swizzle_y = I915_BIT_6_SWIZZLE_9;
610 } else if (IS_GEN(i915, 2)) {
611
612
613
614
615 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
616 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
617 } else if (IS_G45(i915) || IS_I965G(i915) || IS_G33(i915)) {
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645 if (intel_uncore_read(uncore, C0DRB3) ==
646 intel_uncore_read(uncore, C1DRB3)) {
647 swizzle_x = I915_BIT_6_SWIZZLE_9_10;
648 swizzle_y = I915_BIT_6_SWIZZLE_9;
649 }
650 } else {
651 u32 dcc = intel_uncore_read(uncore, DCC);
652
653
654
655
656
657
658
659
660
661
662 switch (dcc & DCC_ADDRESSING_MODE_MASK) {
663 case DCC_ADDRESSING_MODE_SINGLE_CHANNEL:
664 case DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC:
665 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
666 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
667 break;
668 case DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED:
669 if (dcc & DCC_CHANNEL_XOR_DISABLE) {
670
671
672
673
674 swizzle_x = I915_BIT_6_SWIZZLE_9_10;
675 swizzle_y = I915_BIT_6_SWIZZLE_9;
676 } else if ((dcc & DCC_CHANNEL_XOR_BIT_17) == 0) {
677
678 swizzle_x = I915_BIT_6_SWIZZLE_9_10_11;
679 swizzle_y = I915_BIT_6_SWIZZLE_9_11;
680 } else {
681
682 swizzle_x = I915_BIT_6_SWIZZLE_9_10_17;
683 swizzle_y = I915_BIT_6_SWIZZLE_9_17;
684 }
685 break;
686 }
687
688
689 if (IS_GEN(i915, 4) &&
690 !(intel_uncore_read(uncore, DCC2) & DCC2_MODIFIED_ENHANCED_DISABLE)) {
691 swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
692 swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
693 }
694
695 if (dcc == 0xffffffff) {
696 DRM_ERROR("Couldn't read from MCHBAR. "
697 "Disabling tiling.\n");
698 swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
699 swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
700 }
701 }
702
703 if (swizzle_x == I915_BIT_6_SWIZZLE_UNKNOWN ||
704 swizzle_y == I915_BIT_6_SWIZZLE_UNKNOWN) {
705
706
707
708
709
710
711
712
713
714
715 i915->quirks |= QUIRK_PIN_SWIZZLED_PAGES;
716 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
717 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
718 }
719
720 i915->mm.bit_6_swizzle_x = swizzle_x;
721 i915->mm.bit_6_swizzle_y = swizzle_y;
722 }
723
724
725
726
727
728
729 static void i915_gem_swizzle_page(struct page *page)
730 {
731 char temp[64];
732 char *vaddr;
733 int i;
734
735 vaddr = kmap(page);
736
737 for (i = 0; i < PAGE_SIZE; i += 128) {
738 memcpy(temp, &vaddr[i], 64);
739 memcpy(&vaddr[i], &vaddr[i + 64], 64);
740 memcpy(&vaddr[i + 64], temp, 64);
741 }
742
743 kunmap(page);
744 }
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759 void
760 i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj,
761 struct sg_table *pages)
762 {
763 struct sgt_iter sgt_iter;
764 struct page *page;
765 int i;
766
767 if (obj->bit_17 == NULL)
768 return;
769
770 i = 0;
771 for_each_sgt_page(page, sgt_iter, pages) {
772 char new_bit_17 = page_to_phys(page) >> 17;
773 if ((new_bit_17 & 0x1) != (test_bit(i, obj->bit_17) != 0)) {
774 i915_gem_swizzle_page(page);
775 set_page_dirty(page);
776 }
777 i++;
778 }
779 }
780
781
782
783
784
785
786
787
788
789
790 void
791 i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj,
792 struct sg_table *pages)
793 {
794 const unsigned int page_count = obj->base.size >> PAGE_SHIFT;
795 struct sgt_iter sgt_iter;
796 struct page *page;
797 int i;
798
799 if (obj->bit_17 == NULL) {
800 obj->bit_17 = bitmap_zalloc(page_count, GFP_KERNEL);
801 if (obj->bit_17 == NULL) {
802 DRM_ERROR("Failed to allocate memory for bit 17 "
803 "record\n");
804 return;
805 }
806 }
807
808 i = 0;
809
810 for_each_sgt_page(page, sgt_iter, pages) {
811 if (page_to_phys(page) & (1 << 17))
812 __set_bit(i, obj->bit_17);
813 else
814 __clear_bit(i, obj->bit_17);
815 i++;
816 }
817 }
818
819 void i915_ggtt_init_fences(struct i915_ggtt *ggtt)
820 {
821 struct drm_i915_private *i915 = ggtt->vm.i915;
822 int num_fences;
823 int i;
824
825 INIT_LIST_HEAD(&ggtt->fence_list);
826 INIT_LIST_HEAD(&ggtt->userfault_list);
827 intel_wakeref_auto_init(&ggtt->userfault_wakeref, &i915->runtime_pm);
828
829 detect_bit_6_swizzle(i915);
830
831 if (INTEL_GEN(i915) >= 7 &&
832 !(IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)))
833 num_fences = 32;
834 else if (INTEL_GEN(i915) >= 4 ||
835 IS_I945G(i915) || IS_I945GM(i915) ||
836 IS_G33(i915) || IS_PINEVIEW(i915))
837 num_fences = 16;
838 else
839 num_fences = 8;
840
841 if (intel_vgpu_active(i915))
842 num_fences = intel_uncore_read(&i915->uncore,
843 vgtif_reg(avail_rs.fence_num));
844
845
846 for (i = 0; i < num_fences; i++) {
847 struct i915_fence_reg *fence = &ggtt->fence_regs[i];
848
849 fence->i915 = i915;
850 fence->id = i;
851 list_add_tail(&fence->link, &ggtt->fence_list);
852 }
853 ggtt->num_fences = num_fences;
854
855 i915_gem_restore_fences(i915);
856 }
857
858 void intel_gt_init_swizzling(struct intel_gt *gt)
859 {
860 struct drm_i915_private *i915 = gt->i915;
861 struct intel_uncore *uncore = gt->uncore;
862
863 if (INTEL_GEN(i915) < 5 ||
864 i915->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
865 return;
866
867 intel_uncore_rmw(uncore, DISP_ARB_CTL, 0, DISP_TILE_SURFACE_SWIZZLING);
868
869 if (IS_GEN(i915, 5))
870 return;
871
872 intel_uncore_rmw(uncore, TILECTL, 0, TILECTL_SWZCTL);
873
874 if (IS_GEN(i915, 6))
875 intel_uncore_write(uncore,
876 ARB_MODE,
877 _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
878 else if (IS_GEN(i915, 7))
879 intel_uncore_write(uncore,
880 ARB_MODE,
881 _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
882 else if (IS_GEN(i915, 8))
883 intel_uncore_write(uncore,
884 GAMTARBMODE,
885 _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
886 else
887 MISSING_CASE(INTEL_GEN(i915));
888 }