drivers/virtio/virtio

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
page_to_balloon_pfn
balloon_ack
tell_host
set_page_pfns
fill_balloon
release_pages_balloon
leak_balloon
update_stat
update_balloon_stats
stats_request
stats_handle_request
towards_target
return_free_pages_to_mm
virtio_balloon_queue_free_page_work
virtballoon_changed
update_balloon_size
update_balloon_stats_func
update_balloon_size_func
init_vqs
virtio_balloon_cmd_id_received
send_cmd_id_start
send_cmd_id_stop
get_free_page_and_send
send_free_pages
virtio_balloon_report_free_page
report_free_page_func
virtballoon_migratepage
balloon_init_fs_context
shrink_free_pages
leak_balloon_pages
shrink_balloon_pages
virtio_balloon_shrinker_scan
virtio_balloon_shrinker_count
virtio_balloon_unregister_shrinker
virtio_balloon_register_shrinker
virtballoon_probe
remove_common
virtballoon_remove
virtballoon_freeze
virtballoon_restore
virtballoon_validate
   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Virtio balloon implementation, inspired by Dor Laor and Marcelo
   4  * Tosatti's implementations.
   5  *
   6  *  Copyright 2008 Rusty Russell IBM Corporation
   7  */
   8 
   9 #include <linux/virtio.h>
  10 #include <linux/virtio_balloon.h>
  11 #include <linux/swap.h>
  12 #include <linux/workqueue.h>
  13 #include <linux/delay.h>
  14 #include <linux/slab.h>
  15 #include <linux/module.h>
  16 #include <linux/balloon_compaction.h>
  17 #include <linux/wait.h>
  18 #include <linux/mm.h>
  19 #include <linux/mount.h>
  20 #include <linux/magic.h>
  21 #include <linux/pseudo_fs.h>
  22 
  23 /*
  24  * Balloon device works in 4K page units.  So each page is pointed to by
  25  * multiple balloon pages.  All memory counters in this driver are in balloon
  26  * page units.
  27  */
  28 #define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)
  29 #define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256
  30 #define VIRTBALLOON_OOM_NOTIFY_PRIORITY 80
  31 
  32 #define VIRTIO_BALLOON_FREE_PAGE_ALLOC_FLAG (__GFP_NORETRY | __GFP_NOWARN | \
  33                                              __GFP_NOMEMALLOC)
  34 /* The order of free page blocks to report to host */
  35 #define VIRTIO_BALLOON_FREE_PAGE_ORDER (MAX_ORDER - 1)
  36 /* The size of a free page block in bytes */
  37 #define VIRTIO_BALLOON_FREE_PAGE_SIZE \
  38         (1 << (VIRTIO_BALLOON_FREE_PAGE_ORDER + PAGE_SHIFT))
  39 
  40 #ifdef CONFIG_BALLOON_COMPACTION
  41 static struct vfsmount *balloon_mnt;
  42 #endif
  43 
  44 enum virtio_balloon_vq {
  45         VIRTIO_BALLOON_VQ_INFLATE,
  46         VIRTIO_BALLOON_VQ_DEFLATE,
  47         VIRTIO_BALLOON_VQ_STATS,
  48         VIRTIO_BALLOON_VQ_FREE_PAGE,
  49         VIRTIO_BALLOON_VQ_MAX
  50 };
  51 
  52 enum virtio_balloon_config_read {
  53         VIRTIO_BALLOON_CONFIG_READ_CMD_ID = 0,
  54 };
  55 
  56 struct virtio_balloon {
  57         struct virtio_device *vdev;
  58         struct virtqueue *inflate_vq, *deflate_vq, *stats_vq, *free_page_vq;
  59 
  60         /* Balloon's own wq for cpu-intensive work items */
  61         struct workqueue_struct *balloon_wq;
  62         /* The free page reporting work item submitted to the balloon wq */
  63         struct work_struct report_free_page_work;
  64 
  65         /* The balloon servicing is delegated to a freezable workqueue. */
  66         struct work_struct update_balloon_stats_work;
  67         struct work_struct update_balloon_size_work;
  68 
  69         /* Prevent updating balloon when it is being canceled. */
  70         spinlock_t stop_update_lock;
  71         bool stop_update;
  72         /* Bitmap to indicate if reading the related config fields are needed */
  73         unsigned long config_read_bitmap;
  74 
  75         /* The list of allocated free pages, waiting to be given back to mm */
  76         struct list_head free_page_list;
  77         spinlock_t free_page_list_lock;
  78         /* The number of free page blocks on the above list */
  79         unsigned long num_free_page_blocks;
  80         /*
  81          * The cmd id received from host.
  82          * Read it via virtio_balloon_cmd_id_received to get the latest value
  83          * sent from host.
  84          */
  85         u32 cmd_id_received_cache;
  86         /* The cmd id that is actively in use */
  87         __virtio32 cmd_id_active;
  88         /* Buffer to store the stop sign */
  89         __virtio32 cmd_id_stop;
  90 
  91         /* Waiting for host to ack the pages we released. */
  92         wait_queue_head_t acked;
  93 
  94         /* Number of balloon pages we've told the Host we're not using. */
  95         unsigned int num_pages;
  96         /*
  97          * The pages we've told the Host we're not using are enqueued
  98          * at vb_dev_info->pages list.
  99          * Each page on this list adds VIRTIO_BALLOON_PAGES_PER_PAGE
 100          * to num_pages above.
 101          */
 102         struct balloon_dev_info vb_dev_info;
 103 
 104         /* Synchronize access/update to this struct virtio_balloon elements */
 105         struct mutex balloon_lock;
 106 
 107         /* The array of pfns we tell the Host about. */
 108         unsigned int num_pfns;
 109         __virtio32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX];
 110 
 111         /* Memory statistics */
 112         struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR];
 113 
 114         /* To register a shrinker to shrink memory upon memory pressure */
 115         struct shrinker shrinker;
 116 };
 117 
 118 static struct virtio_device_id id_table[] = {
 119         { VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID },
 120         { 0 },
 121 };
 122 
 123 static u32 page_to_balloon_pfn(struct page *page)
 124 {
 125         unsigned long pfn = page_to_pfn(page);
 126 
 127         BUILD_BUG_ON(PAGE_SHIFT < VIRTIO_BALLOON_PFN_SHIFT);
 128         /* Convert pfn from Linux page size to balloon page size. */
 129         return pfn * VIRTIO_BALLOON_PAGES_PER_PAGE;
 130 }
 131 
 132 static void balloon_ack(struct virtqueue *vq)
 133 {
 134         struct virtio_balloon *vb = vq->vdev->priv;
 135 
 136         wake_up(&vb->acked);
 137 }
 138 
 139 static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)
 140 {
 141         struct scatterlist sg;
 142         unsigned int len;
 143 
 144         sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);
 145 
 146         /* We should always be able to add one buffer to an empty queue. */
 147         virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
 148         virtqueue_kick(vq);
 149 
 150         /* When host has read buffer, this completes via balloon_ack */
 151         wait_event(vb->acked, virtqueue_get_buf(vq, &len));
 152 
 153 }
 154 
 155 static void set_page_pfns(struct virtio_balloon *vb,
 156                           __virtio32 pfns[], struct page *page)
 157 {
 158         unsigned int i;
 159 
 160         BUILD_BUG_ON(VIRTIO_BALLOON_PAGES_PER_PAGE > VIRTIO_BALLOON_ARRAY_PFNS_MAX);
 161 
 162         /*
 163          * Set balloon pfns pointing at this page.
 164          * Note that the first pfn points at start of the page.
 165          */
 166         for (i = 0; i < VIRTIO_BALLOON_PAGES_PER_PAGE; i++)
 167                 pfns[i] = cpu_to_virtio32(vb->vdev,
 168                                           page_to_balloon_pfn(page) + i);
 169 }
 170 
 171 static unsigned fill_balloon(struct virtio_balloon *vb, size_t num)
 172 {
 173         unsigned num_allocated_pages;
 174         unsigned num_pfns;
 175         struct page *page;
 176         LIST_HEAD(pages);
 177 
 178         /* We can only do one array worth at a time. */
 179         num = min(num, ARRAY_SIZE(vb->pfns));
 180 
 181         for (num_pfns = 0; num_pfns < num;
 182              num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
 183                 struct page *page = balloon_page_alloc();
 184 
 185                 if (!page) {
 186                         dev_info_ratelimited(&vb->vdev->dev,
 187                                              "Out of puff! Can't get %u pages\n",
 188                                              VIRTIO_BALLOON_PAGES_PER_PAGE);
 189                         /* Sleep for at least 1/5 of a second before retry. */
 190                         msleep(200);
 191                         break;
 192                 }
 193 
 194                 balloon_page_push(&pages, page);
 195         }
 196 
 197         mutex_lock(&vb->balloon_lock);
 198 
 199         vb->num_pfns = 0;
 200 
 201         while ((page = balloon_page_pop(&pages))) {
 202                 balloon_page_enqueue(&vb->vb_dev_info, page);
 203 
 204                 set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
 205                 vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
 206                 if (!virtio_has_feature(vb->vdev,
 207                                         VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
 208                         adjust_managed_page_count(page, -1);
 209                 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE;
 210         }
 211 
 212         num_allocated_pages = vb->num_pfns;
 213         /* Did we get any? */
 214         if (vb->num_pfns != 0)
 215                 tell_host(vb, vb->inflate_vq);
 216         mutex_unlock(&vb->balloon_lock);
 217 
 218         return num_allocated_pages;
 219 }
 220 
 221 static void release_pages_balloon(struct virtio_balloon *vb,
 222                                  struct list_head *pages)
 223 {
 224         struct page *page, *next;
 225 
 226         list_for_each_entry_safe(page, next, pages, lru) {
 227                 if (!virtio_has_feature(vb->vdev,
 228                                         VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
 229                         adjust_managed_page_count(page, 1);
 230                 list_del(&page->lru);
 231                 put_page(page); /* balloon reference */
 232         }
 233 }
 234 
 235 static unsigned leak_balloon(struct virtio_balloon *vb, size_t num)
 236 {
 237         unsigned num_freed_pages;
 238         struct page *page;
 239         struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info;
 240         LIST_HEAD(pages);
 241 
 242         /* We can only do one array worth at a time. */
 243         num = min(num, ARRAY_SIZE(vb->pfns));
 244 
 245         mutex_lock(&vb->balloon_lock);
 246         /* We can't release more pages than taken */
 247         num = min(num, (size_t)vb->num_pages);
 248         for (vb->num_pfns = 0; vb->num_pfns < num;
 249              vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
 250                 page = balloon_page_dequeue(vb_dev_info);
 251                 if (!page)
 252                         break;
 253                 set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
 254                 list_add(&page->lru, &pages);
 255                 vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;
 256         }
 257 
 258         num_freed_pages = vb->num_pfns;
 259         /*
 260          * Note that if
 261          * virtio_has_feature(vdev, VIRTIO_BALLOON_F_MUST_TELL_HOST);
 262          * is true, we *have* to do it in this order
 263          */
 264         if (vb->num_pfns != 0)
 265                 tell_host(vb, vb->deflate_vq);
 266         release_pages_balloon(vb, &pages);
 267         mutex_unlock(&vb->balloon_lock);
 268         return num_freed_pages;
 269 }
 270 
 271 static inline void update_stat(struct virtio_balloon *vb, int idx,
 272                                u16 tag, u64 val)
 273 {
 274         BUG_ON(idx >= VIRTIO_BALLOON_S_NR);
 275         vb->stats[idx].tag = cpu_to_virtio16(vb->vdev, tag);
 276         vb->stats[idx].val = cpu_to_virtio64(vb->vdev, val);
 277 }
 278 
 279 #define pages_to_bytes(x) ((u64)(x) << PAGE_SHIFT)
 280 
 281 static unsigned int update_balloon_stats(struct virtio_balloon *vb)
 282 {
 283         unsigned long events[NR_VM_EVENT_ITEMS];
 284         struct sysinfo i;
 285         unsigned int idx = 0;
 286         long available;
 287         unsigned long caches;
 288 
 289         all_vm_events(events);
 290         si_meminfo(&i);
 291 
 292         available = si_mem_available();
 293         caches = global_node_page_state(NR_FILE_PAGES);
 294 
 295 #ifdef CONFIG_VM_EVENT_COUNTERS
 296         update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
 297                                 pages_to_bytes(events[PSWPIN]));
 298         update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,
 299                                 pages_to_bytes(events[PSWPOUT]));
 300         update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]);
 301         update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]);
 302 #ifdef CONFIG_HUGETLB_PAGE
 303         update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGALLOC,
 304                     events[HTLB_BUDDY_PGALLOC]);
 305         update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGFAIL,
 306                     events[HTLB_BUDDY_PGALLOC_FAIL]);
 307 #endif
 308 #endif
 309         update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE,
 310                                 pages_to_bytes(i.freeram));
 311         update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,
 312                                 pages_to_bytes(i.totalram));
 313         update_stat(vb, idx++, VIRTIO_BALLOON_S_AVAIL,
 314                                 pages_to_bytes(available));
 315         update_stat(vb, idx++, VIRTIO_BALLOON_S_CACHES,
 316                                 pages_to_bytes(caches));
 317 
 318         return idx;
 319 }
 320 
 321 /*
 322  * While most virtqueues communicate guest-initiated requests to the hypervisor,
 323  * the stats queue operates in reverse.  The driver initializes the virtqueue
 324  * with a single buffer.  From that point forward, all conversations consist of
 325  * a hypervisor request (a call to this function) which directs us to refill
 326  * the virtqueue with a fresh stats buffer.  Since stats collection can sleep,
 327  * we delegate the job to a freezable workqueue that will do the actual work via
 328  * stats_handle_request().
 329  */
 330 static void stats_request(struct virtqueue *vq)
 331 {
 332         struct virtio_balloon *vb = vq->vdev->priv;
 333 
 334         spin_lock(&vb->stop_update_lock);
 335         if (!vb->stop_update)
 336                 queue_work(system_freezable_wq, &vb->update_balloon_stats_work);
 337         spin_unlock(&vb->stop_update_lock);
 338 }
 339 
 340 static void stats_handle_request(struct virtio_balloon *vb)
 341 {
 342         struct virtqueue *vq;
 343         struct scatterlist sg;
 344         unsigned int len, num_stats;
 345 
 346         num_stats = update_balloon_stats(vb);
 347 
 348         vq = vb->stats_vq;
 349         if (!virtqueue_get_buf(vq, &len))
 350                 return;
 351         sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);
 352         virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
 353         virtqueue_kick(vq);
 354 }
 355 
 356 static inline s64 towards_target(struct virtio_balloon *vb)
 357 {
 358         s64 target;
 359         u32 num_pages;
 360 
 361         virtio_cread(vb->vdev, struct virtio_balloon_config, num_pages,
 362                      &num_pages);
 363 
 364         /* Legacy balloon config space is LE, unlike all other devices. */
 365         if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1))
 366                 num_pages = le32_to_cpu((__force __le32)num_pages);
 367 
 368         target = num_pages;
 369         return target - vb->num_pages;
 370 }
 371 
 372 /* Gives back @num_to_return blocks of free pages to mm. */
 373 static unsigned long return_free_pages_to_mm(struct virtio_balloon *vb,
 374                                              unsigned long num_to_return)
 375 {
 376         struct page *page;
 377         unsigned long num_returned;
 378 
 379         spin_lock_irq(&vb->free_page_list_lock);
 380         for (num_returned = 0; num_returned < num_to_return; num_returned++) {
 381                 page = balloon_page_pop(&vb->free_page_list);
 382                 if (!page)
 383                         break;
 384                 free_pages((unsigned long)page_address(page),
 385                            VIRTIO_BALLOON_FREE_PAGE_ORDER);
 386         }
 387         vb->num_free_page_blocks -= num_returned;
 388         spin_unlock_irq(&vb->free_page_list_lock);
 389 
 390         return num_returned;
 391 }
 392 
 393 static void virtio_balloon_queue_free_page_work(struct virtio_balloon *vb)
 394 {
 395         if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
 396                 return;
 397 
 398         /* No need to queue the work if the bit was already set. */
 399         if (test_and_set_bit(VIRTIO_BALLOON_CONFIG_READ_CMD_ID,
 400                              &vb->config_read_bitmap))
 401                 return;
 402 
 403         queue_work(vb->balloon_wq, &vb->report_free_page_work);
 404 }
 405 
 406 static void virtballoon_changed(struct virtio_device *vdev)
 407 {
 408         struct virtio_balloon *vb = vdev->priv;
 409         unsigned long flags;
 410 
 411         spin_lock_irqsave(&vb->stop_update_lock, flags);
 412         if (!vb->stop_update) {
 413                 queue_work(system_freezable_wq,
 414                            &vb->update_balloon_size_work);
 415                 virtio_balloon_queue_free_page_work(vb);
 416         }
 417         spin_unlock_irqrestore(&vb->stop_update_lock, flags);
 418 }
 419 
 420 static void update_balloon_size(struct virtio_balloon *vb)
 421 {
 422         u32 actual = vb->num_pages;
 423 
 424         /* Legacy balloon config space is LE, unlike all other devices. */
 425         if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1))
 426                 actual = (__force u32)cpu_to_le32(actual);
 427 
 428         virtio_cwrite(vb->vdev, struct virtio_balloon_config, actual,
 429                       &actual);
 430 }
 431 
 432 static void update_balloon_stats_func(struct work_struct *work)
 433 {
 434         struct virtio_balloon *vb;
 435 
 436         vb = container_of(work, struct virtio_balloon,
 437                           update_balloon_stats_work);
 438         stats_handle_request(vb);
 439 }
 440 
 441 static void update_balloon_size_func(struct work_struct *work)
 442 {
 443         struct virtio_balloon *vb;
 444         s64 diff;
 445 
 446         vb = container_of(work, struct virtio_balloon,
 447                           update_balloon_size_work);
 448         diff = towards_target(vb);
 449 
 450         if (!diff)
 451                 return;
 452 
 453         if (diff > 0)
 454                 diff -= fill_balloon(vb, diff);
 455         else
 456                 diff += leak_balloon(vb, -diff);
 457         update_balloon_size(vb);
 458 
 459         if (diff)
 460                 queue_work(system_freezable_wq, work);
 461 }
 462 
 463 static int init_vqs(struct virtio_balloon *vb)
 464 {
 465         struct virtqueue *vqs[VIRTIO_BALLOON_VQ_MAX];
 466         vq_callback_t *callbacks[VIRTIO_BALLOON_VQ_MAX];
 467         const char *names[VIRTIO_BALLOON_VQ_MAX];
 468         int err;
 469 
 470         /*
 471          * Inflateq and deflateq are used unconditionally. The names[]
 472          * will be NULL if the related feature is not enabled, which will
 473          * cause no allocation for the corresponding virtqueue in find_vqs.
 474          */
 475         callbacks[VIRTIO_BALLOON_VQ_INFLATE] = balloon_ack;
 476         names[VIRTIO_BALLOON_VQ_INFLATE] = "inflate";
 477         callbacks[VIRTIO_BALLOON_VQ_DEFLATE] = balloon_ack;
 478         names[VIRTIO_BALLOON_VQ_DEFLATE] = "deflate";
 479         callbacks[VIRTIO_BALLOON_VQ_STATS] = NULL;
 480         names[VIRTIO_BALLOON_VQ_STATS] = NULL;
 481         callbacks[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL;
 482         names[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL;
 483 
 484         if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
 485                 names[VIRTIO_BALLOON_VQ_STATS] = "stats";
 486                 callbacks[VIRTIO_BALLOON_VQ_STATS] = stats_request;
 487         }
 488 
 489         if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) {
 490                 names[VIRTIO_BALLOON_VQ_FREE_PAGE] = "free_page_vq";
 491                 callbacks[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL;
 492         }
 493 
 494         err = vb->vdev->config->find_vqs(vb->vdev, VIRTIO_BALLOON_VQ_MAX,
 495                                          vqs, callbacks, names, NULL, NULL);
 496         if (err)
 497                 return err;
 498 
 499         vb->inflate_vq = vqs[VIRTIO_BALLOON_VQ_INFLATE];
 500         vb->deflate_vq = vqs[VIRTIO_BALLOON_VQ_DEFLATE];
 501         if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
 502                 struct scatterlist sg;
 503                 unsigned int num_stats;
 504                 vb->stats_vq = vqs[VIRTIO_BALLOON_VQ_STATS];
 505 
 506                 /*
 507                  * Prime this virtqueue with one buffer so the hypervisor can
 508                  * use it to signal us later (it can't be broken yet!).
 509                  */
 510                 num_stats = update_balloon_stats(vb);
 511 
 512                 sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);
 513                 err = virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb,
 514                                            GFP_KERNEL);
 515                 if (err) {
 516                         dev_warn(&vb->vdev->dev, "%s: add stat_vq failed\n",
 517                                  __func__);
 518                         return err;
 519                 }
 520                 virtqueue_kick(vb->stats_vq);
 521         }
 522 
 523         if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
 524                 vb->free_page_vq = vqs[VIRTIO_BALLOON_VQ_FREE_PAGE];
 525 
 526         return 0;
 527 }
 528 
 529 static u32 virtio_balloon_cmd_id_received(struct virtio_balloon *vb)
 530 {
 531         if (test_and_clear_bit(VIRTIO_BALLOON_CONFIG_READ_CMD_ID,
 532                                &vb->config_read_bitmap))
 533                 virtio_cread(vb->vdev, struct virtio_balloon_config,
 534                              free_page_report_cmd_id,
 535                              &vb->cmd_id_received_cache);
 536 
 537         return vb->cmd_id_received_cache;
 538 }
 539 
 540 static int send_cmd_id_start(struct virtio_balloon *vb)
 541 {
 542         struct scatterlist sg;
 543         struct virtqueue *vq = vb->free_page_vq;
 544         int err, unused;
 545 
 546         /* Detach all the used buffers from the vq */
 547         while (virtqueue_get_buf(vq, &unused))
 548                 ;
 549 
 550         vb->cmd_id_active = virtio32_to_cpu(vb->vdev,
 551                                         virtio_balloon_cmd_id_received(vb));
 552         sg_init_one(&sg, &vb->cmd_id_active, sizeof(vb->cmd_id_active));
 553         err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_active, GFP_KERNEL);
 554         if (!err)
 555                 virtqueue_kick(vq);
 556         return err;
 557 }
 558 
 559 static int send_cmd_id_stop(struct virtio_balloon *vb)
 560 {
 561         struct scatterlist sg;
 562         struct virtqueue *vq = vb->free_page_vq;
 563         int err, unused;
 564 
 565         /* Detach all the used buffers from the vq */
 566         while (virtqueue_get_buf(vq, &unused))
 567                 ;
 568 
 569         sg_init_one(&sg, &vb->cmd_id_stop, sizeof(vb->cmd_id_stop));
 570         err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_stop, GFP_KERNEL);
 571         if (!err)
 572                 virtqueue_kick(vq);
 573         return err;
 574 }
 575 
 576 static int get_free_page_and_send(struct virtio_balloon *vb)
 577 {
 578         struct virtqueue *vq = vb->free_page_vq;
 579         struct page *page;
 580         struct scatterlist sg;
 581         int err, unused;
 582         void *p;
 583 
 584         /* Detach all the used buffers from the vq */
 585         while (virtqueue_get_buf(vq, &unused))
 586                 ;
 587 
 588         page = alloc_pages(VIRTIO_BALLOON_FREE_PAGE_ALLOC_FLAG,
 589                            VIRTIO_BALLOON_FREE_PAGE_ORDER);
 590         /*
 591          * When the allocation returns NULL, it indicates that we have got all
 592          * the possible free pages, so return -EINTR to stop.
 593          */
 594         if (!page)
 595                 return -EINTR;
 596 
 597         p = page_address(page);
 598         sg_init_one(&sg, p, VIRTIO_BALLOON_FREE_PAGE_SIZE);
 599         /* There is always 1 entry reserved for the cmd id to use. */
 600         if (vq->num_free > 1) {
 601                 err = virtqueue_add_inbuf(vq, &sg, 1, p, GFP_KERNEL);
 602                 if (unlikely(err)) {
 603                         free_pages((unsigned long)p,
 604                                    VIRTIO_BALLOON_FREE_PAGE_ORDER);
 605                         return err;
 606                 }
 607                 virtqueue_kick(vq);
 608                 spin_lock_irq(&vb->free_page_list_lock);
 609                 balloon_page_push(&vb->free_page_list, page);
 610                 vb->num_free_page_blocks++;
 611                 spin_unlock_irq(&vb->free_page_list_lock);
 612         } else {
 613                 /*
 614                  * The vq has no available entry to add this page block, so
 615                  * just free it.
 616                  */
 617                 free_pages((unsigned long)p, VIRTIO_BALLOON_FREE_PAGE_ORDER);
 618         }
 619 
 620         return 0;
 621 }
 622 
 623 static int send_free_pages(struct virtio_balloon *vb)
 624 {
 625         int err;
 626         u32 cmd_id_active;
 627 
 628         while (1) {
 629                 /*
 630                  * If a stop id or a new cmd id was just received from host,
 631                  * stop the reporting.
 632                  */
 633                 cmd_id_active = virtio32_to_cpu(vb->vdev, vb->cmd_id_active);
 634                 if (unlikely(cmd_id_active !=
 635                              virtio_balloon_cmd_id_received(vb)))
 636                         break;
 637 
 638                 /*
 639                  * The free page blocks are allocated and sent to host one by
 640                  * one.
 641                  */
 642                 err = get_free_page_and_send(vb);
 643                 if (err == -EINTR)
 644                         break;
 645                 else if (unlikely(err))
 646                         return err;
 647         }
 648 
 649         return 0;
 650 }
 651 
 652 static void virtio_balloon_report_free_page(struct virtio_balloon *vb)
 653 {
 654         int err;
 655         struct device *dev = &vb->vdev->dev;
 656 
 657         /* Start by sending the received cmd id to host with an outbuf. */
 658         err = send_cmd_id_start(vb);
 659         if (unlikely(err))
 660                 dev_err(dev, "Failed to send a start id, err = %d\n", err);
 661 
 662         err = send_free_pages(vb);
 663         if (unlikely(err))
 664                 dev_err(dev, "Failed to send a free page, err = %d\n", err);
 665 
 666         /* End by sending a stop id to host with an outbuf. */
 667         err = send_cmd_id_stop(vb);
 668         if (unlikely(err))
 669                 dev_err(dev, "Failed to send a stop id, err = %d\n", err);
 670 }
 671 
 672 static void report_free_page_func(struct work_struct *work)
 673 {
 674         struct virtio_balloon *vb = container_of(work, struct virtio_balloon,
 675                                                  report_free_page_work);
 676         u32 cmd_id_received;
 677 
 678         cmd_id_received = virtio_balloon_cmd_id_received(vb);
 679         if (cmd_id_received == VIRTIO_BALLOON_CMD_ID_DONE) {
 680                 /* Pass ULONG_MAX to give back all the free pages */
 681                 return_free_pages_to_mm(vb, ULONG_MAX);
 682         } else if (cmd_id_received != VIRTIO_BALLOON_CMD_ID_STOP &&
 683                    cmd_id_received !=
 684                    virtio32_to_cpu(vb->vdev, vb->cmd_id_active)) {
 685                 virtio_balloon_report_free_page(vb);
 686         }
 687 }
 688 
 689 #ifdef CONFIG_BALLOON_COMPACTION
 690 /*
 691  * virtballoon_migratepage - perform the balloon page migration on behalf of
 692  *                           a compation thread.     (called under page lock)
 693  * @vb_dev_info: the balloon device
 694  * @newpage: page that will replace the isolated page after migration finishes.
 695  * @page   : the isolated (old) page that is about to be migrated to newpage.
 696  * @mode   : compaction mode -- not used for balloon page migration.
 697  *
 698  * After a ballooned page gets isolated by compaction procedures, this is the
 699  * function that performs the page migration on behalf of a compaction thread
 700  * The page migration for virtio balloon is done in a simple swap fashion which
 701  * follows these two macro steps:
 702  *  1) insert newpage into vb->pages list and update the host about it;
 703  *  2) update the host about the old page removed from vb->pages list;
 704  *
 705  * This function preforms the balloon page migration task.
 706  * Called through balloon_mapping->a_ops->migratepage
 707  */
 708 static int virtballoon_migratepage(struct balloon_dev_info *vb_dev_info,
 709                 struct page *newpage, struct page *page, enum migrate_mode mode)
 710 {
 711         struct virtio_balloon *vb = container_of(vb_dev_info,
 712                         struct virtio_balloon, vb_dev_info);
 713         unsigned long flags;
 714 
 715         /*
 716          * In order to avoid lock contention while migrating pages concurrently
 717          * to leak_balloon() or fill_balloon() we just give up the balloon_lock
 718          * this turn, as it is easier to retry the page migration later.
 719          * This also prevents fill_balloon() getting stuck into a mutex
 720          * recursion in the case it ends up triggering memory compaction
 721          * while it is attempting to inflate the ballon.
 722          */
 723         if (!mutex_trylock(&vb->balloon_lock))
 724                 return -EAGAIN;
 725 
 726         get_page(newpage); /* balloon reference */
 727 
 728         /*
 729           * When we migrate a page to a different zone and adjusted the
 730           * managed page count when inflating, we have to fixup the count of
 731           * both involved zones.
 732           */
 733         if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM) &&
 734             page_zone(page) != page_zone(newpage)) {
 735                 adjust_managed_page_count(page, 1);
 736                 adjust_managed_page_count(newpage, -1);
 737         }
 738 
 739         /* balloon's page migration 1st step  -- inflate "newpage" */
 740         spin_lock_irqsave(&vb_dev_info->pages_lock, flags);
 741         balloon_page_insert(vb_dev_info, newpage);
 742         vb_dev_info->isolated_pages--;
 743         __count_vm_event(BALLOON_MIGRATE);
 744         spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags);
 745         vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
 746         set_page_pfns(vb, vb->pfns, newpage);
 747         tell_host(vb, vb->inflate_vq);
 748 
 749         /* balloon's page migration 2nd step -- deflate "page" */
 750         spin_lock_irqsave(&vb_dev_info->pages_lock, flags);
 751         balloon_page_delete(page);
 752         spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags);
 753         vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
 754         set_page_pfns(vb, vb->pfns, page);
 755         tell_host(vb, vb->deflate_vq);
 756 
 757         mutex_unlock(&vb->balloon_lock);
 758 
 759         put_page(page); /* balloon reference */
 760 
 761         return MIGRATEPAGE_SUCCESS;
 762 }
 763 
 764 static int balloon_init_fs_context(struct fs_context *fc)
 765 {
 766         return init_pseudo(fc, BALLOON_KVM_MAGIC) ? 0 : -ENOMEM;
 767 }
 768 
 769 static struct file_system_type balloon_fs = {
 770         .name           = "balloon-kvm",
 771         .init_fs_context = balloon_init_fs_context,
 772         .kill_sb        = kill_anon_super,
 773 };
 774 
 775 #endif /* CONFIG_BALLOON_COMPACTION */
 776 
 777 static unsigned long shrink_free_pages(struct virtio_balloon *vb,
 778                                        unsigned long pages_to_free)
 779 {
 780         unsigned long blocks_to_free, blocks_freed;
 781 
 782         pages_to_free = round_up(pages_to_free,
 783                                  1 << VIRTIO_BALLOON_FREE_PAGE_ORDER);
 784         blocks_to_free = pages_to_free >> VIRTIO_BALLOON_FREE_PAGE_ORDER;
 785         blocks_freed = return_free_pages_to_mm(vb, blocks_to_free);
 786 
 787         return blocks_freed << VIRTIO_BALLOON_FREE_PAGE_ORDER;
 788 }
 789 
 790 static unsigned long leak_balloon_pages(struct virtio_balloon *vb,
 791                                           unsigned long pages_to_free)
 792 {
 793         return leak_balloon(vb, pages_to_free * VIRTIO_BALLOON_PAGES_PER_PAGE) /
 794                 VIRTIO_BALLOON_PAGES_PER_PAGE;
 795 }
 796 
 797 static unsigned long shrink_balloon_pages(struct virtio_balloon *vb,
 798                                           unsigned long pages_to_free)
 799 {
 800         unsigned long pages_freed = 0;
 801 
 802         /*
 803          * One invocation of leak_balloon can deflate at most
 804          * VIRTIO_BALLOON_ARRAY_PFNS_MAX balloon pages, so we call it
 805          * multiple times to deflate pages till reaching pages_to_free.
 806          */
 807         while (vb->num_pages && pages_freed < pages_to_free)
 808                 pages_freed += leak_balloon_pages(vb,
 809                                                   pages_to_free - pages_freed);
 810 
 811         update_balloon_size(vb);
 812 
 813         return pages_freed;
 814 }
 815 
 816 static unsigned long virtio_balloon_shrinker_scan(struct shrinker *shrinker,
 817                                                   struct shrink_control *sc)
 818 {
 819         unsigned long pages_to_free, pages_freed = 0;
 820         struct virtio_balloon *vb = container_of(shrinker,
 821                                         struct virtio_balloon, shrinker);
 822 
 823         pages_to_free = sc->nr_to_scan;
 824 
 825         if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
 826                 pages_freed = shrink_free_pages(vb, pages_to_free);
 827 
 828         if (pages_freed >= pages_to_free)
 829                 return pages_freed;
 830 
 831         pages_freed += shrink_balloon_pages(vb, pages_to_free - pages_freed);
 832 
 833         return pages_freed;
 834 }
 835 
 836 static unsigned long virtio_balloon_shrinker_count(struct shrinker *shrinker,
 837                                                    struct shrink_control *sc)
 838 {
 839         struct virtio_balloon *vb = container_of(shrinker,
 840                                         struct virtio_balloon, shrinker);
 841         unsigned long count;
 842 
 843         count = vb->num_pages / VIRTIO_BALLOON_PAGES_PER_PAGE;
 844         count += vb->num_free_page_blocks << VIRTIO_BALLOON_FREE_PAGE_ORDER;
 845 
 846         return count;
 847 }
 848 
 849 static void virtio_balloon_unregister_shrinker(struct virtio_balloon *vb)
 850 {
 851         unregister_shrinker(&vb->shrinker);
 852 }
 853 
 854 static int virtio_balloon_register_shrinker(struct virtio_balloon *vb)
 855 {
 856         vb->shrinker.scan_objects = virtio_balloon_shrinker_scan;
 857         vb->shrinker.count_objects = virtio_balloon_shrinker_count;
 858         vb->shrinker.seeks = DEFAULT_SEEKS;
 859 
 860         return register_shrinker(&vb->shrinker);
 861 }
 862 
 863 static int virtballoon_probe(struct virtio_device *vdev)
 864 {
 865         struct virtio_balloon *vb;
 866         __u32 poison_val;
 867         int err;
 868 
 869         if (!vdev->config->get) {
 870                 dev_err(&vdev->dev, "%s failure: config access disabled\n",
 871                         __func__);
 872                 return -EINVAL;
 873         }
 874 
 875         vdev->priv = vb = kzalloc(sizeof(*vb), GFP_KERNEL);
 876         if (!vb) {
 877                 err = -ENOMEM;
 878                 goto out;
 879         }
 880 
 881         INIT_WORK(&vb->update_balloon_stats_work, update_balloon_stats_func);
 882         INIT_WORK(&vb->update_balloon_size_work, update_balloon_size_func);
 883         spin_lock_init(&vb->stop_update_lock);
 884         mutex_init(&vb->balloon_lock);
 885         init_waitqueue_head(&vb->acked);
 886         vb->vdev = vdev;
 887 
 888         balloon_devinfo_init(&vb->vb_dev_info);
 889 
 890         err = init_vqs(vb);
 891         if (err)
 892                 goto out_free_vb;
 893 
 894 #ifdef CONFIG_BALLOON_COMPACTION
 895         balloon_mnt = kern_mount(&balloon_fs);
 896         if (IS_ERR(balloon_mnt)) {
 897                 err = PTR_ERR(balloon_mnt);
 898                 goto out_del_vqs;
 899         }
 900 
 901         vb->vb_dev_info.migratepage = virtballoon_migratepage;
 902         vb->vb_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb);
 903         if (IS_ERR(vb->vb_dev_info.inode)) {
 904                 err = PTR_ERR(vb->vb_dev_info.inode);
 905                 goto out_kern_unmount;
 906         }
 907         vb->vb_dev_info.inode->i_mapping->a_ops = &balloon_aops;
 908 #endif
 909         if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) {
 910                 /*
 911                  * There is always one entry reserved for cmd id, so the ring
 912                  * size needs to be at least two to report free page hints.
 913                  */
 914                 if (virtqueue_get_vring_size(vb->free_page_vq) < 2) {
 915                         err = -ENOSPC;
 916                         goto out_iput;
 917                 }
 918                 vb->balloon_wq = alloc_workqueue("balloon-wq",
 919                                         WQ_FREEZABLE | WQ_CPU_INTENSIVE, 0);
 920                 if (!vb->balloon_wq) {
 921                         err = -ENOMEM;
 922                         goto out_iput;
 923                 }
 924                 INIT_WORK(&vb->report_free_page_work, report_free_page_func);
 925                 vb->cmd_id_received_cache = VIRTIO_BALLOON_CMD_ID_STOP;
 926                 vb->cmd_id_active = cpu_to_virtio32(vb->vdev,
 927                                                   VIRTIO_BALLOON_CMD_ID_STOP);
 928                 vb->cmd_id_stop = cpu_to_virtio32(vb->vdev,
 929                                                   VIRTIO_BALLOON_CMD_ID_STOP);
 930                 spin_lock_init(&vb->free_page_list_lock);
 931                 INIT_LIST_HEAD(&vb->free_page_list);
 932                 if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_PAGE_POISON)) {
 933                         memset(&poison_val, PAGE_POISON, sizeof(poison_val));
 934                         virtio_cwrite(vb->vdev, struct virtio_balloon_config,
 935                                       poison_val, &poison_val);
 936                 }
 937         }
 938         /*
 939          * We continue to use VIRTIO_BALLOON_F_DEFLATE_ON_OOM to decide if a
 940          * shrinker needs to be registered to relieve memory pressure.
 941          */
 942         if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) {
 943                 err = virtio_balloon_register_shrinker(vb);
 944                 if (err)
 945                         goto out_del_balloon_wq;
 946         }
 947         virtio_device_ready(vdev);
 948 
 949         if (towards_target(vb))
 950                 virtballoon_changed(vdev);
 951         return 0;
 952 
 953 out_del_balloon_wq:
 954         if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
 955                 destroy_workqueue(vb->balloon_wq);
 956 out_iput:
 957 #ifdef CONFIG_BALLOON_COMPACTION
 958         iput(vb->vb_dev_info.inode);
 959 out_kern_unmount:
 960         kern_unmount(balloon_mnt);
 961 out_del_vqs:
 962 #endif
 963         vdev->config->del_vqs(vdev);
 964 out_free_vb:
 965         kfree(vb);
 966 out:
 967         return err;
 968 }
 969 
 970 static void remove_common(struct virtio_balloon *vb)
 971 {
 972         /* There might be pages left in the balloon: free them. */
 973         while (vb->num_pages)
 974                 leak_balloon(vb, vb->num_pages);
 975         update_balloon_size(vb);
 976 
 977         /* There might be free pages that are being reported: release them. */
 978         if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
 979                 return_free_pages_to_mm(vb, ULONG_MAX);
 980 
 981         /* Now we reset the device so we can clean up the queues. */
 982         vb->vdev->config->reset(vb->vdev);
 983 
 984         vb->vdev->config->del_vqs(vb->vdev);
 985 }
 986 
 987 static void virtballoon_remove(struct virtio_device *vdev)
 988 {
 989         struct virtio_balloon *vb = vdev->priv;
 990 
 991         if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
 992                 virtio_balloon_unregister_shrinker(vb);
 993         spin_lock_irq(&vb->stop_update_lock);
 994         vb->stop_update = true;
 995         spin_unlock_irq(&vb->stop_update_lock);
 996         cancel_work_sync(&vb->update_balloon_size_work);
 997         cancel_work_sync(&vb->update_balloon_stats_work);
 998 
 999         if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) {
1000                 cancel_work_sync(&vb->report_free_page_work);
1001                 destroy_workqueue(vb->balloon_wq);
1002         }
1003 
1004         remove_common(vb);
1005 #ifdef CONFIG_BALLOON_COMPACTION
1006         if (vb->vb_dev_info.inode)
1007                 iput(vb->vb_dev_info.inode);
1008 
1009         kern_unmount(balloon_mnt);
1010 #endif
1011         kfree(vb);
1012 }
1013 
1014 #ifdef CONFIG_PM_SLEEP
1015 static int virtballoon_freeze(struct virtio_device *vdev)
1016 {
1017         struct virtio_balloon *vb = vdev->priv;
1018 
1019         /*
1020          * The workqueue is already frozen by the PM core before this
1021          * function is called.
1022          */
1023         remove_common(vb);
1024         return 0;
1025 }
1026 
1027 static int virtballoon_restore(struct virtio_device *vdev)
1028 {
1029         struct virtio_balloon *vb = vdev->priv;
1030         int ret;
1031 
1032         ret = init_vqs(vdev->priv);
1033         if (ret)
1034                 return ret;
1035 
1036         virtio_device_ready(vdev);
1037 
1038         if (towards_target(vb))
1039                 virtballoon_changed(vdev);
1040         update_balloon_size(vb);
1041         return 0;
1042 }
1043 #endif
1044 
1045 static int virtballoon_validate(struct virtio_device *vdev)
1046 {
1047         if (!page_poisoning_enabled())
1048                 __virtio_clear_bit(vdev, VIRTIO_BALLOON_F_PAGE_POISON);
1049 
1050         __virtio_clear_bit(vdev, VIRTIO_F_IOMMU_PLATFORM);
1051         return 0;
1052 }
1053 
1054 static unsigned int features[] = {
1055         VIRTIO_BALLOON_F_MUST_TELL_HOST,
1056         VIRTIO_BALLOON_F_STATS_VQ,
1057         VIRTIO_BALLOON_F_DEFLATE_ON_OOM,
1058         VIRTIO_BALLOON_F_FREE_PAGE_HINT,
1059         VIRTIO_BALLOON_F_PAGE_POISON,
1060 };
1061 
1062 static struct virtio_driver virtio_balloon_driver = {
1063         .feature_table = features,
1064         .feature_table_size = ARRAY_SIZE(features),
1065         .driver.name =  KBUILD_MODNAME,
1066         .driver.owner = THIS_MODULE,
1067         .id_table =     id_table,
1068         .validate =     virtballoon_validate,
1069         .probe =        virtballoon_probe,
1070         .remove =       virtballoon_remove,
1071         .config_changed = virtballoon_changed,
1072 #ifdef CONFIG_PM_SLEEP
1073         .freeze =       virtballoon_freeze,
1074         .restore =      virtballoon_restore,
1075 #endif
1076 };
1077 
1078 module_virtio_driver(virtio_balloon_driver);
1079 MODULE_DEVICE_TABLE(virtio, id_table);
1080 MODULE_DESCRIPTION("Virtio balloon driver");
1081 MODULE_LICENSE("GPL");
/* [<][>][^][v][top][bottom][index][help] */
root/drivers/virtio/virtio_balloon.c

DEFINITIONS
