This source file includes following definitions.
- vma_set_pages
- vma_clear_pages
- vma_bind
- vma_unbind
- create_sleeve
- destroy_sleeve
- clear_pages_work_driver_name
- clear_pages_work_timeline_name
- clear_pages_work_release
- clear_pages_signal_irq_worker
- clear_pages_dma_fence_cb
- clear_pages_worker
- clear_pages_work_notify
- i915_gem_schedule_fill_pages_blt
1
2
3
4
5
6 #include "i915_drv.h"
7 #include "gt/intel_context.h"
8 #include "gt/intel_engine_pm.h"
9 #include "gt/intel_engine_pool.h"
10 #include "i915_gem_client_blt.h"
11 #include "i915_gem_object_blt.h"
12
13 struct i915_sleeve {
14 struct i915_vma *vma;
15 struct drm_i915_gem_object *obj;
16 struct sg_table *pages;
17 struct i915_page_sizes page_sizes;
18 };
19
20 static int vma_set_pages(struct i915_vma *vma)
21 {
22 struct i915_sleeve *sleeve = vma->private;
23
24 vma->pages = sleeve->pages;
25 vma->page_sizes = sleeve->page_sizes;
26
27 return 0;
28 }
29
30 static void vma_clear_pages(struct i915_vma *vma)
31 {
32 GEM_BUG_ON(!vma->pages);
33 vma->pages = NULL;
34 }
35
36 static int vma_bind(struct i915_vma *vma,
37 enum i915_cache_level cache_level,
38 u32 flags)
39 {
40 return vma->vm->vma_ops.bind_vma(vma, cache_level, flags);
41 }
42
43 static void vma_unbind(struct i915_vma *vma)
44 {
45 vma->vm->vma_ops.unbind_vma(vma);
46 }
47
48 static const struct i915_vma_ops proxy_vma_ops = {
49 .set_pages = vma_set_pages,
50 .clear_pages = vma_clear_pages,
51 .bind_vma = vma_bind,
52 .unbind_vma = vma_unbind,
53 };
54
55 static struct i915_sleeve *create_sleeve(struct i915_address_space *vm,
56 struct drm_i915_gem_object *obj,
57 struct sg_table *pages,
58 struct i915_page_sizes *page_sizes)
59 {
60 struct i915_sleeve *sleeve;
61 struct i915_vma *vma;
62 int err;
63
64 sleeve = kzalloc(sizeof(*sleeve), GFP_KERNEL);
65 if (!sleeve)
66 return ERR_PTR(-ENOMEM);
67
68 vma = i915_vma_instance(obj, vm, NULL);
69 if (IS_ERR(vma)) {
70 err = PTR_ERR(vma);
71 goto err_free;
72 }
73
74 vma->private = sleeve;
75 vma->ops = &proxy_vma_ops;
76
77 sleeve->vma = vma;
78 sleeve->pages = pages;
79 sleeve->page_sizes = *page_sizes;
80
81 return sleeve;
82
83 err_free:
84 kfree(sleeve);
85 return ERR_PTR(err);
86 }
87
88 static void destroy_sleeve(struct i915_sleeve *sleeve)
89 {
90 kfree(sleeve);
91 }
92
93 struct clear_pages_work {
94 struct dma_fence dma;
95 struct dma_fence_cb cb;
96 struct i915_sw_fence wait;
97 struct work_struct work;
98 struct irq_work irq_work;
99 struct i915_sleeve *sleeve;
100 struct intel_context *ce;
101 u32 value;
102 };
103
104 static const char *clear_pages_work_driver_name(struct dma_fence *fence)
105 {
106 return DRIVER_NAME;
107 }
108
109 static const char *clear_pages_work_timeline_name(struct dma_fence *fence)
110 {
111 return "clear";
112 }
113
114 static void clear_pages_work_release(struct dma_fence *fence)
115 {
116 struct clear_pages_work *w = container_of(fence, typeof(*w), dma);
117
118 destroy_sleeve(w->sleeve);
119
120 i915_sw_fence_fini(&w->wait);
121
122 BUILD_BUG_ON(offsetof(typeof(*w), dma));
123 dma_fence_free(&w->dma);
124 }
125
126 static const struct dma_fence_ops clear_pages_work_ops = {
127 .get_driver_name = clear_pages_work_driver_name,
128 .get_timeline_name = clear_pages_work_timeline_name,
129 .release = clear_pages_work_release,
130 };
131
132 static void clear_pages_signal_irq_worker(struct irq_work *work)
133 {
134 struct clear_pages_work *w = container_of(work, typeof(*w), irq_work);
135
136 dma_fence_signal(&w->dma);
137 dma_fence_put(&w->dma);
138 }
139
140 static void clear_pages_dma_fence_cb(struct dma_fence *fence,
141 struct dma_fence_cb *cb)
142 {
143 struct clear_pages_work *w = container_of(cb, typeof(*w), cb);
144
145 if (fence->error)
146 dma_fence_set_error(&w->dma, fence->error);
147
148
149
150
151
152 irq_work_queue(&w->irq_work);
153 }
154
155 static void clear_pages_worker(struct work_struct *work)
156 {
157 struct clear_pages_work *w = container_of(work, typeof(*w), work);
158 struct drm_i915_private *i915 = w->ce->engine->i915;
159 struct drm_i915_gem_object *obj = w->sleeve->vma->obj;
160 struct i915_vma *vma = w->sleeve->vma;
161 struct i915_request *rq;
162 struct i915_vma *batch;
163 int err = w->dma.error;
164
165 if (unlikely(err))
166 goto out_signal;
167
168 if (obj->cache_dirty) {
169 if (i915_gem_object_has_struct_page(obj))
170 drm_clflush_sg(w->sleeve->pages);
171 obj->cache_dirty = false;
172 }
173 obj->read_domains = I915_GEM_GPU_DOMAINS;
174 obj->write_domain = 0;
175
176
177 mutex_lock(&i915->drm.struct_mutex);
178 err = i915_vma_pin(vma, 0, 0, PIN_USER);
179 if (unlikely(err))
180 goto out_unlock;
181
182 batch = intel_emit_vma_fill_blt(w->ce, vma, w->value);
183 if (IS_ERR(batch)) {
184 err = PTR_ERR(batch);
185 goto out_unpin;
186 }
187
188 rq = intel_context_create_request(w->ce);
189 if (IS_ERR(rq)) {
190 err = PTR_ERR(rq);
191 goto out_batch;
192 }
193
194
195 if (dma_fence_add_callback(&rq->fence, &w->cb,
196 clear_pages_dma_fence_cb))
197 GEM_BUG_ON(1);
198
199 err = intel_emit_vma_mark_active(batch, rq);
200 if (unlikely(err))
201 goto out_request;
202
203 if (w->ce->engine->emit_init_breadcrumb) {
204 err = w->ce->engine->emit_init_breadcrumb(rq);
205 if (unlikely(err))
206 goto out_request;
207 }
208
209
210
211
212
213
214 err = i915_active_ref(&vma->active, rq->timeline, rq);
215 if (err)
216 goto out_request;
217
218 err = w->ce->engine->emit_bb_start(rq,
219 batch->node.start, batch->node.size,
220 0);
221 out_request:
222 if (unlikely(err)) {
223 i915_request_skip(rq, err);
224 err = 0;
225 }
226
227 i915_request_add(rq);
228 out_batch:
229 intel_emit_vma_release(w->ce, batch);
230 out_unpin:
231 i915_vma_unpin(vma);
232 out_unlock:
233 mutex_unlock(&i915->drm.struct_mutex);
234 out_signal:
235 if (unlikely(err)) {
236 dma_fence_set_error(&w->dma, err);
237 dma_fence_signal(&w->dma);
238 dma_fence_put(&w->dma);
239 }
240 }
241
242 static int __i915_sw_fence_call
243 clear_pages_work_notify(struct i915_sw_fence *fence,
244 enum i915_sw_fence_notify state)
245 {
246 struct clear_pages_work *w = container_of(fence, typeof(*w), wait);
247
248 switch (state) {
249 case FENCE_COMPLETE:
250 schedule_work(&w->work);
251 break;
252
253 case FENCE_FREE:
254 dma_fence_put(&w->dma);
255 break;
256 }
257
258 return NOTIFY_DONE;
259 }
260
261 static DEFINE_SPINLOCK(fence_lock);
262
263
264 int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj,
265 struct intel_context *ce,
266 struct sg_table *pages,
267 struct i915_page_sizes *page_sizes,
268 u32 value)
269 {
270 struct clear_pages_work *work;
271 struct i915_sleeve *sleeve;
272 int err;
273
274 sleeve = create_sleeve(ce->vm, obj, pages, page_sizes);
275 if (IS_ERR(sleeve))
276 return PTR_ERR(sleeve);
277
278 work = kmalloc(sizeof(*work), GFP_KERNEL);
279 if (!work) {
280 destroy_sleeve(sleeve);
281 return -ENOMEM;
282 }
283
284 work->value = value;
285 work->sleeve = sleeve;
286 work->ce = ce;
287
288 INIT_WORK(&work->work, clear_pages_worker);
289
290 init_irq_work(&work->irq_work, clear_pages_signal_irq_worker);
291
292 dma_fence_init(&work->dma, &clear_pages_work_ops, &fence_lock, 0, 0);
293 i915_sw_fence_init(&work->wait, clear_pages_work_notify);
294
295 i915_gem_object_lock(obj);
296 err = i915_sw_fence_await_reservation(&work->wait,
297 obj->base.resv, NULL,
298 true, I915_FENCE_TIMEOUT,
299 I915_FENCE_GFP);
300 if (err < 0) {
301 dma_fence_set_error(&work->dma, err);
302 } else {
303 dma_resv_add_excl_fence(obj->base.resv, &work->dma);
304 err = 0;
305 }
306 i915_gem_object_unlock(obj);
307
308 dma_fence_get(&work->dma);
309 i915_sw_fence_commit(&work->wait);
310
311 return err;
312 }
313
314 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
315 #include "selftests/i915_gem_client_blt.c"
316 #endif