root/drivers/gpu/drm/i915/gt/intel_engine_pool.c

DEFINITIONS

1. to_engine
2. bucket_for_size
3. node_free
4. pool_active
5. pool_retire
6. node_create
7. intel_engine_pool_get
8. intel_engine_pool_init
9. intel_engine_pool_park
10. intel_engine_pool_fini

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2014-2018 Intel Corporation
   5  */
   6 
   7 #include "gem/i915_gem_object.h"
   8 
   9 #include "i915_drv.h"
  10 #include "intel_engine_pm.h"
  11 #include "intel_engine_pool.h"
  12 
  13 static struct intel_engine_cs *to_engine(struct intel_engine_pool *pool)
  14 {
  15         return container_of(pool, struct intel_engine_cs, pool);
  16 }
  17 
  18 static struct list_head *
  19 bucket_for_size(struct intel_engine_pool *pool, size_t sz)
  20 {
  21         int n;
  22 
  23         /*
  24          * Compute a power-of-two bucket, but throw everything greater than
  25          * 16KiB into the same bucket: i.e. the buckets hold objects of
  26          * (1 page, 2 pages, 4 pages, 8+ pages).
  27          */
  28         n = fls(sz >> PAGE_SHIFT) - 1;
  29         if (n >= ARRAY_SIZE(pool->cache_list))
  30                 n = ARRAY_SIZE(pool->cache_list) - 1;
  31 
  32         return &pool->cache_list[n];
  33 }
  34 
  35 static void node_free(struct intel_engine_pool_node *node)
  36 {
  37         i915_gem_object_put(node->obj);
  38         i915_active_fini(&node->active);
  39         kfree(node);
  40 }
  41 
  42 static int pool_active(struct i915_active *ref)
  43 {
  44         struct intel_engine_pool_node *node =
  45                 container_of(ref, typeof(*node), active);
  46         struct dma_resv *resv = node->obj->base.resv;
  47         int err;
  48 
  49         if (dma_resv_trylock(resv)) {
  50                 dma_resv_add_excl_fence(resv, NULL);
  51                 dma_resv_unlock(resv);
  52         }
  53 
  54         err = i915_gem_object_pin_pages(node->obj);
  55         if (err)
  56                 return err;
  57 
  58         /* Hide this pinned object from the shrinker until retired */
  59         i915_gem_object_make_unshrinkable(node->obj);
  60 
  61         return 0;
  62 }
  63 
  64 static void pool_retire(struct i915_active *ref)
  65 {
  66         struct intel_engine_pool_node *node =
  67                 container_of(ref, typeof(*node), active);
  68         struct intel_engine_pool *pool = node->pool;
  69         struct list_head *list = bucket_for_size(pool, node->obj->base.size);
  70         unsigned long flags;
  71 
  72         GEM_BUG_ON(!intel_engine_pm_is_awake(to_engine(pool)));
  73 
  74         i915_gem_object_unpin_pages(node->obj);
  75 
  76         /* Return this object to the shrinker pool */
  77         i915_gem_object_make_purgeable(node->obj);
  78 
  79         spin_lock_irqsave(&pool->lock, flags);
  80         list_add(&node->link, list);
  81         spin_unlock_irqrestore(&pool->lock, flags);
  82 }
  83 
  84 static struct intel_engine_pool_node *
  85 node_create(struct intel_engine_pool *pool, size_t sz)
  86 {
  87         struct intel_engine_cs *engine = to_engine(pool);
  88         struct intel_engine_pool_node *node;
  89         struct drm_i915_gem_object *obj;
  90 
  91         node = kmalloc(sizeof(*node),
  92                        GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
  93         if (!node)
  94                 return ERR_PTR(-ENOMEM);
  95 
  96         node->pool = pool;
  97         i915_active_init(engine->i915, &node->active, pool_active, pool_retire);
  98 
  99         obj = i915_gem_object_create_internal(engine->i915, sz);
 100         if (IS_ERR(obj)) {
 101                 i915_active_fini(&node->active);
 102                 kfree(node);
 103                 return ERR_CAST(obj);
 104         }
 105 
 106         i915_gem_object_set_readonly(obj);
 107 
 108         node->obj = obj;
 109         return node;
 110 }
 111 
 112 struct intel_engine_pool_node *
 113 intel_engine_pool_get(struct intel_engine_pool *pool, size_t size)
 114 {
 115         struct intel_engine_pool_node *node;
 116         struct list_head *list;
 117         unsigned long flags;
 118         int ret;
 119 
 120         GEM_BUG_ON(!intel_engine_pm_is_awake(to_engine(pool)));
 121 
 122         size = PAGE_ALIGN(size);
 123         list = bucket_for_size(pool, size);
 124 
 125         spin_lock_irqsave(&pool->lock, flags);
 126         list_for_each_entry(node, list, link) {
 127                 if (node->obj->base.size < size)
 128                         continue;
 129                 list_del(&node->link);
 130                 break;
 131         }
 132         spin_unlock_irqrestore(&pool->lock, flags);
 133 
 134         if (&node->link == list) {
 135                 node = node_create(pool, size);
 136                 if (IS_ERR(node))
 137                         return node;
 138         }
 139 
 140         ret = i915_active_acquire(&node->active);
 141         if (ret) {
 142                 node_free(node);
 143                 return ERR_PTR(ret);
 144         }
 145 
 146         return node;
 147 }
 148 
 149 void intel_engine_pool_init(struct intel_engine_pool *pool)
 150 {
 151         int n;
 152 
 153         spin_lock_init(&pool->lock);
 154         for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++)
 155                 INIT_LIST_HEAD(&pool->cache_list[n]);
 156 }
 157 
 158 void intel_engine_pool_park(struct intel_engine_pool *pool)
 159 {
 160         int n;
 161 
 162         for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) {
 163                 struct list_head *list = &pool->cache_list[n];
 164                 struct intel_engine_pool_node *node, *nn;
 165 
 166                 list_for_each_entry_safe(node, nn, list, link)
 167                         node_free(node);
 168 
 169                 INIT_LIST_HEAD(list);
 170         }
 171 }
 172 
 173 void intel_engine_pool_fini(struct intel_engine_pool *pool)
 174 {
 175         int n;
 176 
 177         for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++)
 178                 GEM_BUG_ON(!list_empty(&pool->cache_list[n]));
 179 }