Lines Matching defs:vb
146 static void start_wakeup_event(struct virtio_balloon *vb, u32 mask)
150 spin_lock_irqsave(&vb->wakeup_lock, flags);
151 vb->wakeup_signal_mask |= mask;
152 if (!vb->processing_wakeup_event) {
153 vb->processing_wakeup_event = true;
154 pm_stay_awake(&vb->vdev->dev);
156 spin_unlock_irqrestore(&vb->wakeup_lock, flags);
159 static void process_wakeup_event(struct virtio_balloon *vb, u32 mask)
161 spin_lock_irq(&vb->wakeup_lock);
162 vb->wakeup_signal_mask &= ~mask;
163 spin_unlock_irq(&vb->wakeup_lock);
166 static void finish_wakeup_event(struct virtio_balloon *vb)
168 spin_lock_irq(&vb->wakeup_lock);
169 if (!vb->wakeup_signal_mask && vb->processing_wakeup_event) {
170 vb->processing_wakeup_event = false;
171 pm_relax(&vb->vdev->dev);
173 spin_unlock_irq(&vb->wakeup_lock);
178 struct virtio_balloon *vb = vq->vdev->priv;
180 wake_up(&vb->acked);
183 static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)
188 sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);
191 virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
195 wait_event(vb->acked, virtqueue_get_buf(vq, &len));
202 struct virtio_balloon *vb =
204 struct virtqueue *vq = vb->reporting_vq;
208 err = virtqueue_add_inbuf(vq, sg, nents, vb, GFP_NOWAIT | __GFP_NOWARN);
221 wait_event(vb->acked, virtqueue_get_buf(vq, &unused));
226 static void set_page_pfns(struct virtio_balloon *vb,
238 pfns[i] = cpu_to_virtio32(vb->vdev,
242 static unsigned int fill_balloon(struct virtio_balloon *vb, size_t num)
250 num = min(num, ARRAY_SIZE(vb->pfns));
257 dev_info_ratelimited(&vb->vdev->dev,
268 mutex_lock(&vb->balloon_lock);
270 vb->num_pfns = 0;
273 balloon_page_enqueue(&vb->vb_dev_info, page);
275 set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
276 vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
277 if (!virtio_has_feature(vb->vdev,
280 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE;
283 num_allocated_pages = vb->num_pfns;
285 if (vb->num_pfns != 0)
286 tell_host(vb, vb->inflate_vq);
287 mutex_unlock(&vb->balloon_lock);
292 static void release_pages_balloon(struct virtio_balloon *vb,
298 if (!virtio_has_feature(vb->vdev,
306 static unsigned int leak_balloon(struct virtio_balloon *vb, size_t num)
310 struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info;
314 num = min(num, ARRAY_SIZE(vb->pfns));
316 mutex_lock(&vb->balloon_lock);
318 num = min(num, (size_t)vb->num_pages);
319 for (vb->num_pfns = 0; vb->num_pfns < num;
320 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
324 set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
326 vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;
329 num_freed_pages = vb->num_pfns;
335 if (vb->num_pfns != 0)
336 tell_host(vb, vb->deflate_vq);
337 release_pages_balloon(vb, &pages);
338 mutex_unlock(&vb->balloon_lock);
342 static inline void update_stat(struct virtio_balloon *vb, int idx,
346 vb->stats[idx].tag = cpu_to_virtio16(vb->vdev, tag);
347 vb->stats[idx].val = cpu_to_virtio64(vb->vdev, val);
352 static unsigned int update_balloon_stats(struct virtio_balloon *vb)
367 update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
369 update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,
371 update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]);
372 update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]);
374 update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGALLOC,
376 update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGFAIL,
380 update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE,
382 update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,
384 update_stat(vb, idx++, VIRTIO_BALLOON_S_AVAIL,
386 update_stat(vb, idx++, VIRTIO_BALLOON_S_CACHES,
403 struct virtio_balloon *vb = vq->vdev->priv;
405 spin_lock(&vb->stop_update_lock);
406 if (!vb->stop_update) {
407 start_wakeup_event(vb, VIRTIO_BALLOON_WAKEUP_SIGNAL_STATS);
408 queue_work(system_freezable_wq, &vb->update_balloon_stats_work);
410 spin_unlock(&vb->stop_update_lock);
413 static void stats_handle_request(struct virtio_balloon *vb)
419 num_stats = update_balloon_stats(vb);
421 vq = vb->stats_vq;
424 sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);
425 virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
429 static inline s64 towards_target(struct virtio_balloon *vb)
435 virtio_cread_le(vb->vdev, struct virtio_balloon_config, num_pages,
443 return target - vb->num_pages;
447 static unsigned long return_free_pages_to_mm(struct virtio_balloon *vb,
453 spin_lock_irq(&vb->free_page_list_lock);
455 page = balloon_page_pop(&vb->free_page_list);
461 vb->num_free_page_blocks -= num_returned;
462 spin_unlock_irq(&vb->free_page_list_lock);
467 static void virtio_balloon_queue_free_page_work(struct virtio_balloon *vb)
469 if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
474 &vb->config_read_bitmap))
477 queue_work(vb->balloon_wq, &vb->report_free_page_work);
480 static void start_update_balloon_size(struct virtio_balloon *vb)
482 start_wakeup_event(vb, VIRTIO_BALLOON_WAKEUP_SIGNAL_ADJUST);
483 queue_work(system_freezable_wq, &vb->update_balloon_size_work);
488 struct virtio_balloon *vb = vdev->priv;
491 spin_lock_irqsave(&vb->stop_update_lock, flags);
492 if (!vb->stop_update) {
493 start_update_balloon_size(vb);
494 virtio_balloon_queue_free_page_work(vb);
496 spin_unlock_irqrestore(&vb->stop_update_lock, flags);
499 static void update_balloon_size(struct virtio_balloon *vb)
501 u32 actual = vb->num_pages;
504 virtio_cwrite_le(vb->vdev, struct virtio_balloon_config, actual,
510 struct virtio_balloon *vb;
512 vb = container_of(work, struct virtio_balloon,
515 process_wakeup_event(vb, VIRTIO_BALLOON_WAKEUP_SIGNAL_STATS);
516 stats_handle_request(vb);
517 finish_wakeup_event(vb);
522 struct virtio_balloon *vb;
525 vb = container_of(work, struct virtio_balloon,
528 process_wakeup_event(vb, VIRTIO_BALLOON_WAKEUP_SIGNAL_ADJUST);
530 diff = towards_target(vb);
534 diff -= fill_balloon(vb, diff);
536 diff += leak_balloon(vb, -diff);
537 update_balloon_size(vb);
543 finish_wakeup_event(vb);
546 static int init_vqs(struct virtio_balloon *vb)
568 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
573 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) {
578 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) {
583 err = virtio_find_vqs(vb->vdev, VIRTIO_BALLOON_VQ_MAX, vqs,
588 vb->inflate_vq = vqs[VIRTIO_BALLOON_VQ_INFLATE];
589 vb->deflate_vq = vqs[VIRTIO_BALLOON_VQ_DEFLATE];
590 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
593 vb->stats_vq = vqs[VIRTIO_BALLOON_VQ_STATS];
599 num_stats = update_balloon_stats(vb);
601 sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);
602 err = virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb,
605 dev_warn(&vb->vdev->dev, "%s: add stat_vq failed\n",
609 virtqueue_kick(vb->stats_vq);
612 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
613 vb->free_page_vq = vqs[VIRTIO_BALLOON_VQ_FREE_PAGE];
615 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING))
616 vb->reporting_vq = vqs[VIRTIO_BALLOON_VQ_REPORTING];
621 static u32 virtio_balloon_cmd_id_received(struct virtio_balloon *vb)
624 &vb->config_read_bitmap)) {
626 virtio_cread_le(vb->vdev, struct virtio_balloon_config,
628 &vb->cmd_id_received_cache);
631 return vb->cmd_id_received_cache;
634 static int send_cmd_id_start(struct virtio_balloon *vb)
637 struct virtqueue *vq = vb->free_page_vq;
644 vb->cmd_id_active = cpu_to_virtio32(vb->vdev,
645 virtio_balloon_cmd_id_received(vb));
646 sg_init_one(&sg, &vb->cmd_id_active, sizeof(vb->cmd_id_active));
647 err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_active, GFP_KERNEL);
653 static int send_cmd_id_stop(struct virtio_balloon *vb)
656 struct virtqueue *vq = vb->free_page_vq;
663 sg_init_one(&sg, &vb->cmd_id_stop, sizeof(vb->cmd_id_stop));
664 err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_stop, GFP_KERNEL);
670 static int get_free_page_and_send(struct virtio_balloon *vb)
672 struct virtqueue *vq = vb->free_page_vq;
702 spin_lock_irq(&vb->free_page_list_lock);
703 balloon_page_push(&vb->free_page_list, page);
704 vb->num_free_page_blocks++;
705 spin_unlock_irq(&vb->free_page_list_lock);
717 static int send_free_pages(struct virtio_balloon *vb)
727 cmd_id_active = virtio32_to_cpu(vb->vdev, vb->cmd_id_active);
729 virtio_balloon_cmd_id_received(vb)))
736 err = get_free_page_and_send(vb);
746 static void virtio_balloon_report_free_page(struct virtio_balloon *vb)
749 struct device *dev = &vb->vdev->dev;
752 err = send_cmd_id_start(vb);
756 err = send_free_pages(vb);
761 err = send_cmd_id_stop(vb);
768 struct virtio_balloon *vb = container_of(work, struct virtio_balloon,
772 cmd_id_received = virtio_balloon_cmd_id_received(vb);
775 return_free_pages_to_mm(vb, ULONG_MAX);
778 virtio32_to_cpu(vb->vdev, vb->cmd_id_active)) {
779 virtio_balloon_report_free_page(vb);
796 * 1) insert newpage into vb->pages list and update the host about it;
797 * 2) update the host about the old page removed from vb->pages list;
805 struct virtio_balloon *vb = container_of(vb_dev_info,
817 if (!mutex_trylock(&vb->balloon_lock))
827 if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM) &&
839 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
840 set_page_pfns(vb, vb->pfns, newpage);
841 tell_host(vb, vb->inflate_vq);
847 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
848 set_page_pfns(vb, vb->pfns, page);
849 tell_host(vb, vb->deflate_vq);
851 mutex_unlock(&vb->balloon_lock);
859 static unsigned long shrink_free_pages(struct virtio_balloon *vb,
867 blocks_freed = return_free_pages_to_mm(vb, blocks_to_free);
875 struct virtio_balloon *vb = shrinker->private_data;
877 return shrink_free_pages(vb, sc->nr_to_scan);
883 struct virtio_balloon *vb = shrinker->private_data;
885 return vb->num_free_page_blocks * VIRTIO_BALLOON_HINT_BLOCK_PAGES;
891 struct virtio_balloon *vb = container_of(nb,
895 *freed += leak_balloon(vb, VIRTIO_BALLOON_OOM_NR_PAGES) /
897 update_balloon_size(vb);
902 static void virtio_balloon_unregister_shrinker(struct virtio_balloon *vb)
904 shrinker_free(vb->shrinker);
907 static int virtio_balloon_register_shrinker(struct virtio_balloon *vb)
909 vb->shrinker = shrinker_alloc(0, "virtio-balloon");
910 if (!vb->shrinker)
913 vb->shrinker->scan_objects = virtio_balloon_shrinker_scan;
914 vb->shrinker->count_objects = virtio_balloon_shrinker_count;
915 vb->shrinker->private_data = vb;
917 shrinker_register(vb->shrinker);
924 struct virtio_balloon *vb;
933 vdev->priv = vb = kzalloc(sizeof(*vb), GFP_KERNEL);
934 if (!vb) {
939 INIT_WORK(&vb->update_balloon_stats_work, update_balloon_stats_func);
940 INIT_WORK(&vb->update_balloon_size_work, update_balloon_size_func);
941 spin_lock_init(&vb->stop_update_lock);
942 mutex_init(&vb->balloon_lock);
943 init_waitqueue_head(&vb->acked);
944 vb->vdev = vdev;
946 balloon_devinfo_init(&vb->vb_dev_info);
948 err = init_vqs(vb);
953 vb->vb_dev_info.migratepage = virtballoon_migratepage;
960 if (virtqueue_get_vring_size(vb->free_page_vq) < 2) {
964 vb->balloon_wq = alloc_workqueue("balloon-wq",
966 if (!vb->balloon_wq) {
970 INIT_WORK(&vb->report_free_page_work, report_free_page_func);
971 vb->cmd_id_received_cache = VIRTIO_BALLOON_CMD_ID_STOP;
972 vb->cmd_id_active = cpu_to_virtio32(vb->vdev,
974 vb->cmd_id_stop = cpu_to_virtio32(vb->vdev,
976 spin_lock_init(&vb->free_page_list_lock);
977 INIT_LIST_HEAD(&vb->free_page_list);
982 err = virtio_balloon_register_shrinker(vb);
987 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) {
988 vb->oom_nb.notifier_call = virtio_balloon_oom_notify;
989 vb->oom_nb.priority = VIRTIO_BALLOON_OOM_NOTIFY_PRIORITY;
990 err = register_oom_notifier(&vb->oom_nb);
1011 virtio_cwrite_le(vb->vdev, struct virtio_balloon_config,
1015 vb->pr_dev_info.report = virtballoon_free_page_report;
1016 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) {
1019 capacity = virtqueue_get_vring_size(vb->reporting_vq);
1039 vb->pr_dev_info.order = 5;
1042 err = page_reporting_register(&vb->pr_dev_info);
1047 spin_lock_init(&vb->wakeup_lock);
1056 device_set_wakeup_capable(&vb->vdev->dev, true);
1060 if (towards_target(vb))
1065 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
1066 unregister_oom_notifier(&vb->oom_nb);
1068 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
1069 virtio_balloon_unregister_shrinker(vb);
1072 destroy_workqueue(vb->balloon_wq);
1076 kfree(vb);
1081 static void remove_common(struct virtio_balloon *vb)
1084 while (vb->num_pages)
1085 leak_balloon(vb, vb->num_pages);
1086 update_balloon_size(vb);
1089 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
1090 return_free_pages_to_mm(vb, ULONG_MAX);
1093 virtio_reset_device(vb->vdev);
1095 vb->vdev->config->del_vqs(vb->vdev);
1100 struct virtio_balloon *vb = vdev->priv;
1102 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING))
1103 page_reporting_unregister(&vb->pr_dev_info);
1104 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
1105 unregister_oom_notifier(&vb->oom_nb);
1106 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
1107 virtio_balloon_unregister_shrinker(vb);
1108 spin_lock_irq(&vb->stop_update_lock);
1109 vb->stop_update = true;
1110 spin_unlock_irq(&vb->stop_update_lock);
1111 cancel_work_sync(&vb->update_balloon_size_work);
1112 cancel_work_sync(&vb->update_balloon_stats_work);
1115 cancel_work_sync(&vb->report_free_page_work);
1116 destroy_workqueue(vb->balloon_wq);
1119 remove_common(vb);
1120 kfree(vb);
1126 struct virtio_balloon *vb = vdev->priv;
1132 remove_common(vb);
1138 struct virtio_balloon *vb = vdev->priv;
1147 if (towards_target(vb))
1149 update_balloon_size(vb);