1/* 2 * Copyright © 2008-2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25#include <linux/oom.h> 26#include <linux/shmem_fs.h> 27#include <linux/slab.h> 28#include <linux/swap.h> 29#include <linux/pci.h> 30#include <linux/dma-buf.h> 31#include <drm/drmP.h> 32#include <drm/i915_drm.h> 33 34#include "i915_drv.h" 35#include "i915_trace.h" 36 37static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) 38{ 39 if (!mutex_is_locked(mutex)) 40 return false; 41 42#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES) 43 return mutex->owner == task; 44#else 45 /* Since UP may be pre-empted, we cannot assume that we own the lock */ 46 return false; 47#endif 48} 49 50/** 51 * i915_gem_shrink - Shrink buffer object caches 52 * @dev_priv: i915 device 53 * @target: amount of memory to make available, in pages 54 * @flags: control flags for selecting cache types 55 * 56 * This function is the main interface to the shrinker. It will try to release 57 * up to @target pages of main memory backing storage from buffer objects. 58 * Selection of the specific caches can be done with @flags. This is e.g. useful 59 * when purgeable objects should be removed from caches preferentially. 60 * 61 * Note that it's not guaranteed that released amount is actually available as 62 * free system memory - the pages might still be in-used to due to other reasons 63 * (like cpu mmaps) or the mm core has reused them before we could grab them. 64 * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to 65 * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all(). 66 * 67 * Also note that any kind of pinning (both per-vma address space pins and 68 * backing storage pins at the buffer object level) result in the shrinker code 69 * having to skip the object. 70 * 71 * Returns: 72 * The number of pages of backing storage actually released. 73 */ 74unsigned long 75i915_gem_shrink(struct drm_i915_private *dev_priv, 76 long target, unsigned flags) 77{ 78 const struct { 79 struct list_head *list; 80 unsigned int bit; 81 } phases[] = { 82 { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND }, 83 { &dev_priv->mm.bound_list, I915_SHRINK_BOUND }, 84 { NULL, 0 }, 85 }, *phase; 86 unsigned long count = 0; 87 88 /* 89 * As we may completely rewrite the (un)bound list whilst unbinding 90 * (due to retiring requests) we have to strictly process only 91 * one element of the list at the time, and recheck the list 92 * on every iteration. 93 * 94 * In particular, we must hold a reference whilst removing the 95 * object as we may end up waiting for and/or retiring the objects. 96 * This might release the final reference (held by the active list) 97 * and result in the object being freed from under us. This is 98 * similar to the precautions the eviction code must take whilst 99 * removing objects. 100 * 101 * Also note that although these lists do not hold a reference to 102 * the object we can safely grab one here: The final object 103 * unreferencing and the bound_list are both protected by the 104 * dev->struct_mutex and so we won't ever be able to observe an 105 * object on the bound_list with a reference count equals 0. 106 */ 107 for (phase = phases; phase->list; phase++) { 108 struct list_head still_in_list; 109 110 if ((flags & phase->bit) == 0) 111 continue; 112 113 INIT_LIST_HEAD(&still_in_list); 114 while (count < target && !list_empty(phase->list)) { 115 struct drm_i915_gem_object *obj; 116 struct i915_vma *vma, *v; 117 118 obj = list_first_entry(phase->list, 119 typeof(*obj), global_list); 120 list_move_tail(&obj->global_list, &still_in_list); 121 122 if (flags & I915_SHRINK_PURGEABLE && 123 obj->madv != I915_MADV_DONTNEED) 124 continue; 125 126 drm_gem_object_reference(&obj->base); 127 128 /* For the unbound phase, this should be a no-op! */ 129 list_for_each_entry_safe(vma, v, 130 &obj->vma_list, vma_link) 131 if (i915_vma_unbind(vma)) 132 break; 133 134 if (i915_gem_object_put_pages(obj) == 0) 135 count += obj->base.size >> PAGE_SHIFT; 136 137 drm_gem_object_unreference(&obj->base); 138 } 139 list_splice(&still_in_list, phase->list); 140 } 141 142 return count; 143} 144 145/** 146 * i915_gem_shrink - Shrink buffer object caches completely 147 * @dev_priv: i915 device 148 * 149 * This is a simple wraper around i915_gem_shrink() to aggressively shrink all 150 * caches completely. It also first waits for and retires all outstanding 151 * requests to also be able to release backing storage for active objects. 152 * 153 * This should only be used in code to intentionally quiescent the gpu or as a 154 * last-ditch effort when memory seems to have run out. 155 * 156 * Returns: 157 * The number of pages of backing storage actually released. 158 */ 159unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv) 160{ 161 i915_gem_evict_everything(dev_priv->dev); 162 return i915_gem_shrink(dev_priv, LONG_MAX, 163 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND); 164} 165 166static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock) 167{ 168 if (!mutex_trylock(&dev->struct_mutex)) { 169 if (!mutex_is_locked_by(&dev->struct_mutex, current)) 170 return false; 171 172 if (to_i915(dev)->mm.shrinker_no_lock_stealing) 173 return false; 174 175 *unlock = false; 176 } else 177 *unlock = true; 178 179 return true; 180} 181 182static int num_vma_bound(struct drm_i915_gem_object *obj) 183{ 184 struct i915_vma *vma; 185 int count = 0; 186 187 list_for_each_entry(vma, &obj->vma_list, vma_link) 188 if (drm_mm_node_allocated(&vma->node)) 189 count++; 190 191 return count; 192} 193 194static unsigned long 195i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) 196{ 197 struct drm_i915_private *dev_priv = 198 container_of(shrinker, struct drm_i915_private, mm.shrinker); 199 struct drm_device *dev = dev_priv->dev; 200 struct drm_i915_gem_object *obj; 201 unsigned long count; 202 bool unlock; 203 204 if (!i915_gem_shrinker_lock(dev, &unlock)) 205 return 0; 206 207 count = 0; 208 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) 209 if (obj->pages_pin_count == 0) 210 count += obj->base.size >> PAGE_SHIFT; 211 212 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 213 if (!i915_gem_obj_is_pinned(obj) && 214 obj->pages_pin_count == num_vma_bound(obj)) 215 count += obj->base.size >> PAGE_SHIFT; 216 } 217 218 if (unlock) 219 mutex_unlock(&dev->struct_mutex); 220 221 return count; 222} 223 224static unsigned long 225i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) 226{ 227 struct drm_i915_private *dev_priv = 228 container_of(shrinker, struct drm_i915_private, mm.shrinker); 229 struct drm_device *dev = dev_priv->dev; 230 unsigned long freed; 231 bool unlock; 232 233 if (!i915_gem_shrinker_lock(dev, &unlock)) 234 return SHRINK_STOP; 235 236 freed = i915_gem_shrink(dev_priv, 237 sc->nr_to_scan, 238 I915_SHRINK_BOUND | 239 I915_SHRINK_UNBOUND | 240 I915_SHRINK_PURGEABLE); 241 if (freed < sc->nr_to_scan) 242 freed += i915_gem_shrink(dev_priv, 243 sc->nr_to_scan - freed, 244 I915_SHRINK_BOUND | 245 I915_SHRINK_UNBOUND); 246 if (unlock) 247 mutex_unlock(&dev->struct_mutex); 248 249 return freed; 250} 251 252static int 253i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) 254{ 255 struct drm_i915_private *dev_priv = 256 container_of(nb, struct drm_i915_private, mm.oom_notifier); 257 struct drm_device *dev = dev_priv->dev; 258 struct drm_i915_gem_object *obj; 259 unsigned long timeout = msecs_to_jiffies(5000) + 1; 260 unsigned long pinned, bound, unbound, freed_pages; 261 bool was_interruptible; 262 bool unlock; 263 264 while (!i915_gem_shrinker_lock(dev, &unlock) && --timeout) { 265 schedule_timeout_killable(1); 266 if (fatal_signal_pending(current)) 267 return NOTIFY_DONE; 268 } 269 if (timeout == 0) { 270 pr_err("Unable to purge GPU memory due lock contention.\n"); 271 return NOTIFY_DONE; 272 } 273 274 was_interruptible = dev_priv->mm.interruptible; 275 dev_priv->mm.interruptible = false; 276 277 freed_pages = i915_gem_shrink_all(dev_priv); 278 279 dev_priv->mm.interruptible = was_interruptible; 280 281 /* Because we may be allocating inside our own driver, we cannot 282 * assert that there are no objects with pinned pages that are not 283 * being pointed to by hardware. 284 */ 285 unbound = bound = pinned = 0; 286 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { 287 if (!obj->base.filp) /* not backed by a freeable object */ 288 continue; 289 290 if (obj->pages_pin_count) 291 pinned += obj->base.size; 292 else 293 unbound += obj->base.size; 294 } 295 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 296 if (!obj->base.filp) 297 continue; 298 299 if (obj->pages_pin_count) 300 pinned += obj->base.size; 301 else 302 bound += obj->base.size; 303 } 304 305 if (unlock) 306 mutex_unlock(&dev->struct_mutex); 307 308 if (freed_pages || unbound || bound) 309 pr_info("Purging GPU memory, %lu bytes freed, %lu bytes still pinned.\n", 310 freed_pages << PAGE_SHIFT, pinned); 311 if (unbound || bound) 312 pr_err("%lu and %lu bytes still available in the " 313 "bound and unbound GPU page lists.\n", 314 bound, unbound); 315 316 *(unsigned long *)ptr += freed_pages; 317 return NOTIFY_DONE; 318} 319 320/** 321 * i915_gem_shrinker_init - Initialize i915 shrinker 322 * @dev_priv: i915 device 323 * 324 * This function registers and sets up the i915 shrinker and OOM handler. 325 */ 326void i915_gem_shrinker_init(struct drm_i915_private *dev_priv) 327{ 328 dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan; 329 dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count; 330 dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS; 331 register_shrinker(&dev_priv->mm.shrinker); 332 333 dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom; 334 register_oom_notifier(&dev_priv->mm.oom_notifier); 335} 336