1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * VDUSE: vDPA Device in Userspace 4 * 5 * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved. 6 * 7 * Author: Xie Yongji <xieyongji@bytedance.com> 8 * 9 */ 10 11#include <linux/init.h> 12#include <linux/module.h> 13#include <linux/cdev.h> 14#include <linux/device.h> 15#include <linux/eventfd.h> 16#include <linux/slab.h> 17#include <linux/wait.h> 18#include <linux/dma-map-ops.h> 19#include <linux/poll.h> 20#include <linux/file.h> 21#include <linux/uio.h> 22#include <linux/vdpa.h> 23#include <linux/nospec.h> 24#include <linux/vmalloc.h> 25#include <linux/sched/mm.h> 26#include <uapi/linux/vduse.h> 27#include <uapi/linux/vdpa.h> 28#include <uapi/linux/virtio_config.h> 29#include <uapi/linux/virtio_ids.h> 30#include <uapi/linux/virtio_blk.h> 31#include <linux/mod_devicetable.h> 32 33#include "iova_domain.h" 34 35#define DRV_AUTHOR "Yongji Xie <xieyongji@bytedance.com>" 36#define DRV_DESC "vDPA Device in Userspace" 37#define DRV_LICENSE "GPL v2" 38 39#define VDUSE_DEV_MAX (1U << MINORBITS) 40#define VDUSE_MAX_BOUNCE_SIZE (1024 * 1024 * 1024) 41#define VDUSE_MIN_BOUNCE_SIZE (1024 * 1024) 42#define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024) 43/* 128 MB reserved for virtqueue creation */ 44#define VDUSE_IOVA_SIZE (VDUSE_MAX_BOUNCE_SIZE + 128 * 1024 * 1024) 45#define VDUSE_MSG_DEFAULT_TIMEOUT 30 46 47#define IRQ_UNBOUND -1 48 49struct vduse_virtqueue { 50 u16 index; 51 u16 num_max; 52 u32 num; 53 u64 desc_addr; 54 u64 driver_addr; 55 u64 device_addr; 56 struct vdpa_vq_state state; 57 bool ready; 58 bool kicked; 59 spinlock_t kick_lock; 60 spinlock_t irq_lock; 61 struct eventfd_ctx *kickfd; 62 struct vdpa_callback cb; 63 struct work_struct inject; 64 struct work_struct kick; 65 int irq_effective_cpu; 66 struct cpumask irq_affinity; 67 struct kobject kobj; 68}; 69 70struct vduse_dev; 71 72struct vduse_vdpa { 73 struct vdpa_device vdpa; 74 struct vduse_dev *dev; 75}; 76 77struct vduse_umem { 78 unsigned long iova; 79 unsigned long npages; 80 struct page **pages; 81 struct mm_struct *mm; 82}; 83 84struct vduse_dev { 85 struct vduse_vdpa *vdev; 86 struct device *dev; 87 struct vduse_virtqueue **vqs; 88 struct vduse_iova_domain *domain; 89 char *name; 90 struct mutex lock; 91 spinlock_t msg_lock; 92 u64 msg_unique; 93 u32 msg_timeout; 94 wait_queue_head_t waitq; 95 struct list_head send_list; 96 struct list_head recv_list; 97 struct vdpa_callback config_cb; 98 struct work_struct inject; 99 spinlock_t irq_lock; 100 struct rw_semaphore rwsem; 101 int minor; 102 bool broken; 103 bool connected; 104 u64 api_version; 105 u64 device_features; 106 u64 driver_features; 107 u32 device_id; 108 u32 vendor_id; 109 u32 generation; 110 u32 config_size; 111 void *config; 112 u8 status; 113 u32 vq_num; 114 u32 vq_align; 115 struct vduse_umem *umem; 116 struct mutex mem_lock; 117 unsigned int bounce_size; 118 struct mutex domain_lock; 119}; 120 121struct vduse_dev_msg { 122 struct vduse_dev_request req; 123 struct vduse_dev_response resp; 124 struct list_head list; 125 wait_queue_head_t waitq; 126 bool completed; 127}; 128 129struct vduse_control { 130 u64 api_version; 131}; 132 133static DEFINE_MUTEX(vduse_lock); 134static DEFINE_IDR(vduse_idr); 135 136static dev_t vduse_major; 137static struct cdev vduse_ctrl_cdev; 138static struct cdev vduse_cdev; 139static struct workqueue_struct *vduse_irq_wq; 140static struct workqueue_struct *vduse_irq_bound_wq; 141 142static u32 allowed_device_id[] = { 143 VIRTIO_ID_BLOCK, 144}; 145 146static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa) 147{ 148 struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa); 149 150 return vdev->dev; 151} 152 153static inline struct vduse_dev *dev_to_vduse(struct device *dev) 154{ 155 struct vdpa_device *vdpa = dev_to_vdpa(dev); 156 157 return vdpa_to_vduse(vdpa); 158} 159 160static struct vduse_dev_msg *vduse_find_msg(struct list_head *head, 161 uint32_t request_id) 162{ 163 struct vduse_dev_msg *msg; 164 165 list_for_each_entry(msg, head, list) { 166 if (msg->req.request_id == request_id) { 167 list_del(&msg->list); 168 return msg; 169 } 170 } 171 172 return NULL; 173} 174 175static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head) 176{ 177 struct vduse_dev_msg *msg = NULL; 178 179 if (!list_empty(head)) { 180 msg = list_first_entry(head, struct vduse_dev_msg, list); 181 list_del(&msg->list); 182 } 183 184 return msg; 185} 186 187static void vduse_enqueue_msg(struct list_head *head, 188 struct vduse_dev_msg *msg) 189{ 190 list_add_tail(&msg->list, head); 191} 192 193static void vduse_dev_broken(struct vduse_dev *dev) 194{ 195 struct vduse_dev_msg *msg, *tmp; 196 197 if (unlikely(dev->broken)) 198 return; 199 200 list_splice_init(&dev->recv_list, &dev->send_list); 201 list_for_each_entry_safe(msg, tmp, &dev->send_list, list) { 202 list_del(&msg->list); 203 msg->completed = 1; 204 msg->resp.result = VDUSE_REQ_RESULT_FAILED; 205 wake_up(&msg->waitq); 206 } 207 dev->broken = true; 208 wake_up(&dev->waitq); 209} 210 211static int vduse_dev_msg_sync(struct vduse_dev *dev, 212 struct vduse_dev_msg *msg) 213{ 214 int ret; 215 216 if (unlikely(dev->broken)) 217 return -EIO; 218 219 init_waitqueue_head(&msg->waitq); 220 spin_lock(&dev->msg_lock); 221 if (unlikely(dev->broken)) { 222 spin_unlock(&dev->msg_lock); 223 return -EIO; 224 } 225 msg->req.request_id = dev->msg_unique++; 226 vduse_enqueue_msg(&dev->send_list, msg); 227 wake_up(&dev->waitq); 228 spin_unlock(&dev->msg_lock); 229 if (dev->msg_timeout) 230 ret = wait_event_killable_timeout(msg->waitq, msg->completed, 231 (long)dev->msg_timeout * HZ); 232 else 233 ret = wait_event_killable(msg->waitq, msg->completed); 234 235 spin_lock(&dev->msg_lock); 236 if (!msg->completed) { 237 list_del(&msg->list); 238 msg->resp.result = VDUSE_REQ_RESULT_FAILED; 239 /* Mark the device as malfunction when there is a timeout */ 240 if (!ret) 241 vduse_dev_broken(dev); 242 } 243 ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO; 244 spin_unlock(&dev->msg_lock); 245 246 return ret; 247} 248 249static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev, 250 struct vduse_virtqueue *vq, 251 struct vdpa_vq_state_packed *packed) 252{ 253 struct vduse_dev_msg msg = { 0 }; 254 int ret; 255 256 msg.req.type = VDUSE_GET_VQ_STATE; 257 msg.req.vq_state.index = vq->index; 258 259 ret = vduse_dev_msg_sync(dev, &msg); 260 if (ret) 261 return ret; 262 263 packed->last_avail_counter = 264 msg.resp.vq_state.packed.last_avail_counter & 0x0001; 265 packed->last_avail_idx = 266 msg.resp.vq_state.packed.last_avail_idx & 0x7FFF; 267 packed->last_used_counter = 268 msg.resp.vq_state.packed.last_used_counter & 0x0001; 269 packed->last_used_idx = 270 msg.resp.vq_state.packed.last_used_idx & 0x7FFF; 271 272 return 0; 273} 274 275static int vduse_dev_get_vq_state_split(struct vduse_dev *dev, 276 struct vduse_virtqueue *vq, 277 struct vdpa_vq_state_split *split) 278{ 279 struct vduse_dev_msg msg = { 0 }; 280 int ret; 281 282 msg.req.type = VDUSE_GET_VQ_STATE; 283 msg.req.vq_state.index = vq->index; 284 285 ret = vduse_dev_msg_sync(dev, &msg); 286 if (ret) 287 return ret; 288 289 split->avail_index = msg.resp.vq_state.split.avail_index; 290 291 return 0; 292} 293 294static int vduse_dev_set_status(struct vduse_dev *dev, u8 status) 295{ 296 struct vduse_dev_msg msg = { 0 }; 297 298 msg.req.type = VDUSE_SET_STATUS; 299 msg.req.s.status = status; 300 301 return vduse_dev_msg_sync(dev, &msg); 302} 303 304static int vduse_dev_update_iotlb(struct vduse_dev *dev, 305 u64 start, u64 last) 306{ 307 struct vduse_dev_msg msg = { 0 }; 308 309 if (last < start) 310 return -EINVAL; 311 312 msg.req.type = VDUSE_UPDATE_IOTLB; 313 msg.req.iova.start = start; 314 msg.req.iova.last = last; 315 316 return vduse_dev_msg_sync(dev, &msg); 317} 318 319static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to) 320{ 321 struct file *file = iocb->ki_filp; 322 struct vduse_dev *dev = file->private_data; 323 struct vduse_dev_msg *msg; 324 int size = sizeof(struct vduse_dev_request); 325 ssize_t ret; 326 327 if (iov_iter_count(to) < size) 328 return -EINVAL; 329 330 spin_lock(&dev->msg_lock); 331 while (1) { 332 msg = vduse_dequeue_msg(&dev->send_list); 333 if (msg) 334 break; 335 336 ret = -EAGAIN; 337 if (file->f_flags & O_NONBLOCK) 338 goto unlock; 339 340 spin_unlock(&dev->msg_lock); 341 ret = wait_event_interruptible_exclusive(dev->waitq, 342 !list_empty(&dev->send_list)); 343 if (ret) 344 return ret; 345 346 spin_lock(&dev->msg_lock); 347 } 348 spin_unlock(&dev->msg_lock); 349 ret = copy_to_iter(&msg->req, size, to); 350 spin_lock(&dev->msg_lock); 351 if (ret != size) { 352 ret = -EFAULT; 353 vduse_enqueue_msg(&dev->send_list, msg); 354 goto unlock; 355 } 356 vduse_enqueue_msg(&dev->recv_list, msg); 357unlock: 358 spin_unlock(&dev->msg_lock); 359 360 return ret; 361} 362 363static bool is_mem_zero(const char *ptr, int size) 364{ 365 int i; 366 367 for (i = 0; i < size; i++) { 368 if (ptr[i]) 369 return false; 370 } 371 return true; 372} 373 374static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from) 375{ 376 struct file *file = iocb->ki_filp; 377 struct vduse_dev *dev = file->private_data; 378 struct vduse_dev_response resp; 379 struct vduse_dev_msg *msg; 380 size_t ret; 381 382 ret = copy_from_iter(&resp, sizeof(resp), from); 383 if (ret != sizeof(resp)) 384 return -EINVAL; 385 386 if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved))) 387 return -EINVAL; 388 389 spin_lock(&dev->msg_lock); 390 msg = vduse_find_msg(&dev->recv_list, resp.request_id); 391 if (!msg) { 392 ret = -ENOENT; 393 goto unlock; 394 } 395 396 memcpy(&msg->resp, &resp, sizeof(resp)); 397 msg->completed = 1; 398 wake_up(&msg->waitq); 399unlock: 400 spin_unlock(&dev->msg_lock); 401 402 return ret; 403} 404 405static __poll_t vduse_dev_poll(struct file *file, poll_table *wait) 406{ 407 struct vduse_dev *dev = file->private_data; 408 __poll_t mask = 0; 409 410 poll_wait(file, &dev->waitq, wait); 411 412 spin_lock(&dev->msg_lock); 413 414 if (unlikely(dev->broken)) 415 mask |= EPOLLERR; 416 if (!list_empty(&dev->send_list)) 417 mask |= EPOLLIN | EPOLLRDNORM; 418 if (!list_empty(&dev->recv_list)) 419 mask |= EPOLLOUT | EPOLLWRNORM; 420 421 spin_unlock(&dev->msg_lock); 422 423 return mask; 424} 425 426static void vduse_dev_reset(struct vduse_dev *dev) 427{ 428 int i; 429 struct vduse_iova_domain *domain = dev->domain; 430 431 /* The coherent mappings are handled in vduse_dev_free_coherent() */ 432 if (domain && domain->bounce_map) 433 vduse_domain_reset_bounce_map(domain); 434 435 down_write(&dev->rwsem); 436 437 dev->status = 0; 438 dev->driver_features = 0; 439 dev->generation++; 440 spin_lock(&dev->irq_lock); 441 dev->config_cb.callback = NULL; 442 dev->config_cb.private = NULL; 443 spin_unlock(&dev->irq_lock); 444 flush_work(&dev->inject); 445 446 for (i = 0; i < dev->vq_num; i++) { 447 struct vduse_virtqueue *vq = dev->vqs[i]; 448 449 vq->ready = false; 450 vq->desc_addr = 0; 451 vq->driver_addr = 0; 452 vq->device_addr = 0; 453 vq->num = 0; 454 memset(&vq->state, 0, sizeof(vq->state)); 455 456 spin_lock(&vq->kick_lock); 457 vq->kicked = false; 458 if (vq->kickfd) 459 eventfd_ctx_put(vq->kickfd); 460 vq->kickfd = NULL; 461 spin_unlock(&vq->kick_lock); 462 463 spin_lock(&vq->irq_lock); 464 vq->cb.callback = NULL; 465 vq->cb.private = NULL; 466 vq->cb.trigger = NULL; 467 spin_unlock(&vq->irq_lock); 468 flush_work(&vq->inject); 469 flush_work(&vq->kick); 470 } 471 472 up_write(&dev->rwsem); 473} 474 475static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx, 476 u64 desc_area, u64 driver_area, 477 u64 device_area) 478{ 479 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 480 struct vduse_virtqueue *vq = dev->vqs[idx]; 481 482 vq->desc_addr = desc_area; 483 vq->driver_addr = driver_area; 484 vq->device_addr = device_area; 485 486 return 0; 487} 488 489static void vduse_vq_kick(struct vduse_virtqueue *vq) 490{ 491 spin_lock(&vq->kick_lock); 492 if (!vq->ready) 493 goto unlock; 494 495 if (vq->kickfd) 496 eventfd_signal(vq->kickfd); 497 else 498 vq->kicked = true; 499unlock: 500 spin_unlock(&vq->kick_lock); 501} 502 503static void vduse_vq_kick_work(struct work_struct *work) 504{ 505 struct vduse_virtqueue *vq = container_of(work, 506 struct vduse_virtqueue, kick); 507 508 vduse_vq_kick(vq); 509} 510 511static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx) 512{ 513 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 514 struct vduse_virtqueue *vq = dev->vqs[idx]; 515 516 if (!eventfd_signal_allowed()) { 517 schedule_work(&vq->kick); 518 return; 519 } 520 vduse_vq_kick(vq); 521} 522 523static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx, 524 struct vdpa_callback *cb) 525{ 526 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 527 struct vduse_virtqueue *vq = dev->vqs[idx]; 528 529 spin_lock(&vq->irq_lock); 530 vq->cb.callback = cb->callback; 531 vq->cb.private = cb->private; 532 vq->cb.trigger = cb->trigger; 533 spin_unlock(&vq->irq_lock); 534} 535 536static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num) 537{ 538 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 539 struct vduse_virtqueue *vq = dev->vqs[idx]; 540 541 vq->num = num; 542} 543 544static u16 vduse_vdpa_get_vq_size(struct vdpa_device *vdpa, u16 idx) 545{ 546 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 547 struct vduse_virtqueue *vq = dev->vqs[idx]; 548 549 if (vq->num) 550 return vq->num; 551 else 552 return vq->num_max; 553} 554 555static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa, 556 u16 idx, bool ready) 557{ 558 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 559 struct vduse_virtqueue *vq = dev->vqs[idx]; 560 561 vq->ready = ready; 562} 563 564static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx) 565{ 566 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 567 struct vduse_virtqueue *vq = dev->vqs[idx]; 568 569 return vq->ready; 570} 571 572static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx, 573 const struct vdpa_vq_state *state) 574{ 575 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 576 struct vduse_virtqueue *vq = dev->vqs[idx]; 577 578 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { 579 vq->state.packed.last_avail_counter = 580 state->packed.last_avail_counter; 581 vq->state.packed.last_avail_idx = state->packed.last_avail_idx; 582 vq->state.packed.last_used_counter = 583 state->packed.last_used_counter; 584 vq->state.packed.last_used_idx = state->packed.last_used_idx; 585 } else 586 vq->state.split.avail_index = state->split.avail_index; 587 588 return 0; 589} 590 591static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx, 592 struct vdpa_vq_state *state) 593{ 594 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 595 struct vduse_virtqueue *vq = dev->vqs[idx]; 596 597 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) 598 return vduse_dev_get_vq_state_packed(dev, vq, &state->packed); 599 600 return vduse_dev_get_vq_state_split(dev, vq, &state->split); 601} 602 603static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa) 604{ 605 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 606 607 return dev->vq_align; 608} 609 610static u64 vduse_vdpa_get_device_features(struct vdpa_device *vdpa) 611{ 612 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 613 614 return dev->device_features; 615} 616 617static int vduse_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features) 618{ 619 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 620 621 dev->driver_features = features; 622 return 0; 623} 624 625static u64 vduse_vdpa_get_driver_features(struct vdpa_device *vdpa) 626{ 627 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 628 629 return dev->driver_features; 630} 631 632static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa, 633 struct vdpa_callback *cb) 634{ 635 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 636 637 spin_lock(&dev->irq_lock); 638 dev->config_cb.callback = cb->callback; 639 dev->config_cb.private = cb->private; 640 spin_unlock(&dev->irq_lock); 641} 642 643static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa) 644{ 645 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 646 u16 num_max = 0; 647 int i; 648 649 for (i = 0; i < dev->vq_num; i++) 650 if (num_max < dev->vqs[i]->num_max) 651 num_max = dev->vqs[i]->num_max; 652 653 return num_max; 654} 655 656static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa) 657{ 658 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 659 660 return dev->device_id; 661} 662 663static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa) 664{ 665 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 666 667 return dev->vendor_id; 668} 669 670static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa) 671{ 672 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 673 674 return dev->status; 675} 676 677static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status) 678{ 679 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 680 681 if (vduse_dev_set_status(dev, status)) 682 return; 683 684 dev->status = status; 685} 686 687static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa) 688{ 689 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 690 691 return dev->config_size; 692} 693 694static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset, 695 void *buf, unsigned int len) 696{ 697 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 698 699 /* Initialize the buffer in case of partial copy. */ 700 memset(buf, 0, len); 701 702 if (offset > dev->config_size) 703 return; 704 705 if (len > dev->config_size - offset) 706 len = dev->config_size - offset; 707 708 memcpy(buf, dev->config + offset, len); 709} 710 711static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset, 712 const void *buf, unsigned int len) 713{ 714 /* Now we only support read-only configuration space */ 715} 716 717static int vduse_vdpa_reset(struct vdpa_device *vdpa) 718{ 719 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 720 int ret = vduse_dev_set_status(dev, 0); 721 722 vduse_dev_reset(dev); 723 724 return ret; 725} 726 727static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa) 728{ 729 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 730 731 return dev->generation; 732} 733 734static int vduse_vdpa_set_vq_affinity(struct vdpa_device *vdpa, u16 idx, 735 const struct cpumask *cpu_mask) 736{ 737 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 738 739 if (cpu_mask) 740 cpumask_copy(&dev->vqs[idx]->irq_affinity, cpu_mask); 741 else 742 cpumask_setall(&dev->vqs[idx]->irq_affinity); 743 744 return 0; 745} 746 747static const struct cpumask * 748vduse_vdpa_get_vq_affinity(struct vdpa_device *vdpa, u16 idx) 749{ 750 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 751 752 return &dev->vqs[idx]->irq_affinity; 753} 754 755static int vduse_vdpa_set_map(struct vdpa_device *vdpa, 756 unsigned int asid, 757 struct vhost_iotlb *iotlb) 758{ 759 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 760 int ret; 761 762 ret = vduse_domain_set_map(dev->domain, iotlb); 763 if (ret) 764 return ret; 765 766 ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX); 767 if (ret) { 768 vduse_domain_clear_map(dev->domain, iotlb); 769 return ret; 770 } 771 772 return 0; 773} 774 775static void vduse_vdpa_free(struct vdpa_device *vdpa) 776{ 777 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 778 779 dev->vdev = NULL; 780} 781 782static const struct vdpa_config_ops vduse_vdpa_config_ops = { 783 .set_vq_address = vduse_vdpa_set_vq_address, 784 .kick_vq = vduse_vdpa_kick_vq, 785 .set_vq_cb = vduse_vdpa_set_vq_cb, 786 .set_vq_num = vduse_vdpa_set_vq_num, 787 .get_vq_size = vduse_vdpa_get_vq_size, 788 .set_vq_ready = vduse_vdpa_set_vq_ready, 789 .get_vq_ready = vduse_vdpa_get_vq_ready, 790 .set_vq_state = vduse_vdpa_set_vq_state, 791 .get_vq_state = vduse_vdpa_get_vq_state, 792 .get_vq_align = vduse_vdpa_get_vq_align, 793 .get_device_features = vduse_vdpa_get_device_features, 794 .set_driver_features = vduse_vdpa_set_driver_features, 795 .get_driver_features = vduse_vdpa_get_driver_features, 796 .set_config_cb = vduse_vdpa_set_config_cb, 797 .get_vq_num_max = vduse_vdpa_get_vq_num_max, 798 .get_device_id = vduse_vdpa_get_device_id, 799 .get_vendor_id = vduse_vdpa_get_vendor_id, 800 .get_status = vduse_vdpa_get_status, 801 .set_status = vduse_vdpa_set_status, 802 .get_config_size = vduse_vdpa_get_config_size, 803 .get_config = vduse_vdpa_get_config, 804 .set_config = vduse_vdpa_set_config, 805 .get_generation = vduse_vdpa_get_generation, 806 .set_vq_affinity = vduse_vdpa_set_vq_affinity, 807 .get_vq_affinity = vduse_vdpa_get_vq_affinity, 808 .reset = vduse_vdpa_reset, 809 .set_map = vduse_vdpa_set_map, 810 .free = vduse_vdpa_free, 811}; 812 813static void vduse_dev_sync_single_for_device(struct device *dev, 814 dma_addr_t dma_addr, size_t size, 815 enum dma_data_direction dir) 816{ 817 struct vduse_dev *vdev = dev_to_vduse(dev); 818 struct vduse_iova_domain *domain = vdev->domain; 819 820 vduse_domain_sync_single_for_device(domain, dma_addr, size, dir); 821} 822 823static void vduse_dev_sync_single_for_cpu(struct device *dev, 824 dma_addr_t dma_addr, size_t size, 825 enum dma_data_direction dir) 826{ 827 struct vduse_dev *vdev = dev_to_vduse(dev); 828 struct vduse_iova_domain *domain = vdev->domain; 829 830 vduse_domain_sync_single_for_cpu(domain, dma_addr, size, dir); 831} 832 833static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page, 834 unsigned long offset, size_t size, 835 enum dma_data_direction dir, 836 unsigned long attrs) 837{ 838 struct vduse_dev *vdev = dev_to_vduse(dev); 839 struct vduse_iova_domain *domain = vdev->domain; 840 841 return vduse_domain_map_page(domain, page, offset, size, dir, attrs); 842} 843 844static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr, 845 size_t size, enum dma_data_direction dir, 846 unsigned long attrs) 847{ 848 struct vduse_dev *vdev = dev_to_vduse(dev); 849 struct vduse_iova_domain *domain = vdev->domain; 850 851 return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs); 852} 853 854static void *vduse_dev_alloc_coherent(struct device *dev, size_t size, 855 dma_addr_t *dma_addr, gfp_t flag, 856 unsigned long attrs) 857{ 858 struct vduse_dev *vdev = dev_to_vduse(dev); 859 struct vduse_iova_domain *domain = vdev->domain; 860 unsigned long iova; 861 void *addr; 862 863 *dma_addr = DMA_MAPPING_ERROR; 864 addr = vduse_domain_alloc_coherent(domain, size, 865 (dma_addr_t *)&iova, flag, attrs); 866 if (!addr) 867 return NULL; 868 869 *dma_addr = (dma_addr_t)iova; 870 871 return addr; 872} 873 874static void vduse_dev_free_coherent(struct device *dev, size_t size, 875 void *vaddr, dma_addr_t dma_addr, 876 unsigned long attrs) 877{ 878 struct vduse_dev *vdev = dev_to_vduse(dev); 879 struct vduse_iova_domain *domain = vdev->domain; 880 881 vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs); 882} 883 884static size_t vduse_dev_max_mapping_size(struct device *dev) 885{ 886 struct vduse_dev *vdev = dev_to_vduse(dev); 887 struct vduse_iova_domain *domain = vdev->domain; 888 889 return domain->bounce_size; 890} 891 892static const struct dma_map_ops vduse_dev_dma_ops = { 893 .sync_single_for_device = vduse_dev_sync_single_for_device, 894 .sync_single_for_cpu = vduse_dev_sync_single_for_cpu, 895 .map_page = vduse_dev_map_page, 896 .unmap_page = vduse_dev_unmap_page, 897 .alloc = vduse_dev_alloc_coherent, 898 .free = vduse_dev_free_coherent, 899 .max_mapping_size = vduse_dev_max_mapping_size, 900}; 901 902static unsigned int perm_to_file_flags(u8 perm) 903{ 904 unsigned int flags = 0; 905 906 switch (perm) { 907 case VDUSE_ACCESS_WO: 908 flags |= O_WRONLY; 909 break; 910 case VDUSE_ACCESS_RO: 911 flags |= O_RDONLY; 912 break; 913 case VDUSE_ACCESS_RW: 914 flags |= O_RDWR; 915 break; 916 default: 917 WARN(1, "invalidate vhost IOTLB permission\n"); 918 break; 919 } 920 921 return flags; 922} 923 924static int vduse_kickfd_setup(struct vduse_dev *dev, 925 struct vduse_vq_eventfd *eventfd) 926{ 927 struct eventfd_ctx *ctx = NULL; 928 struct vduse_virtqueue *vq; 929 u32 index; 930 931 if (eventfd->index >= dev->vq_num) 932 return -EINVAL; 933 934 index = array_index_nospec(eventfd->index, dev->vq_num); 935 vq = dev->vqs[index]; 936 if (eventfd->fd >= 0) { 937 ctx = eventfd_ctx_fdget(eventfd->fd); 938 if (IS_ERR(ctx)) 939 return PTR_ERR(ctx); 940 } else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN) 941 return 0; 942 943 spin_lock(&vq->kick_lock); 944 if (vq->kickfd) 945 eventfd_ctx_put(vq->kickfd); 946 vq->kickfd = ctx; 947 if (vq->ready && vq->kicked && vq->kickfd) { 948 eventfd_signal(vq->kickfd); 949 vq->kicked = false; 950 } 951 spin_unlock(&vq->kick_lock); 952 953 return 0; 954} 955 956static bool vduse_dev_is_ready(struct vduse_dev *dev) 957{ 958 int i; 959 960 for (i = 0; i < dev->vq_num; i++) 961 if (!dev->vqs[i]->num_max) 962 return false; 963 964 return true; 965} 966 967static void vduse_dev_irq_inject(struct work_struct *work) 968{ 969 struct vduse_dev *dev = container_of(work, struct vduse_dev, inject); 970 971 spin_lock_bh(&dev->irq_lock); 972 if (dev->config_cb.callback) 973 dev->config_cb.callback(dev->config_cb.private); 974 spin_unlock_bh(&dev->irq_lock); 975} 976 977static void vduse_vq_irq_inject(struct work_struct *work) 978{ 979 struct vduse_virtqueue *vq = container_of(work, 980 struct vduse_virtqueue, inject); 981 982 spin_lock_bh(&vq->irq_lock); 983 if (vq->ready && vq->cb.callback) 984 vq->cb.callback(vq->cb.private); 985 spin_unlock_bh(&vq->irq_lock); 986} 987 988static bool vduse_vq_signal_irqfd(struct vduse_virtqueue *vq) 989{ 990 bool signal = false; 991 992 if (!vq->cb.trigger) 993 return false; 994 995 spin_lock_irq(&vq->irq_lock); 996 if (vq->ready && vq->cb.trigger) { 997 eventfd_signal(vq->cb.trigger); 998 signal = true; 999 } 1000 spin_unlock_irq(&vq->irq_lock); 1001 1002 return signal; 1003} 1004 1005static int vduse_dev_queue_irq_work(struct vduse_dev *dev, 1006 struct work_struct *irq_work, 1007 int irq_effective_cpu) 1008{ 1009 int ret = -EINVAL; 1010 1011 down_read(&dev->rwsem); 1012 if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 1013 goto unlock; 1014 1015 ret = 0; 1016 if (irq_effective_cpu == IRQ_UNBOUND) 1017 queue_work(vduse_irq_wq, irq_work); 1018 else 1019 queue_work_on(irq_effective_cpu, 1020 vduse_irq_bound_wq, irq_work); 1021unlock: 1022 up_read(&dev->rwsem); 1023 1024 return ret; 1025} 1026 1027static int vduse_dev_dereg_umem(struct vduse_dev *dev, 1028 u64 iova, u64 size) 1029{ 1030 int ret; 1031 1032 mutex_lock(&dev->mem_lock); 1033 ret = -ENOENT; 1034 if (!dev->umem) 1035 goto unlock; 1036 1037 ret = -EINVAL; 1038 if (!dev->domain) 1039 goto unlock; 1040 1041 if (dev->umem->iova != iova || size != dev->domain->bounce_size) 1042 goto unlock; 1043 1044 vduse_domain_remove_user_bounce_pages(dev->domain); 1045 unpin_user_pages_dirty_lock(dev->umem->pages, 1046 dev->umem->npages, true); 1047 atomic64_sub(dev->umem->npages, &dev->umem->mm->pinned_vm); 1048 mmdrop(dev->umem->mm); 1049 vfree(dev->umem->pages); 1050 kfree(dev->umem); 1051 dev->umem = NULL; 1052 ret = 0; 1053unlock: 1054 mutex_unlock(&dev->mem_lock); 1055 return ret; 1056} 1057 1058static int vduse_dev_reg_umem(struct vduse_dev *dev, 1059 u64 iova, u64 uaddr, u64 size) 1060{ 1061 struct page **page_list = NULL; 1062 struct vduse_umem *umem = NULL; 1063 long pinned = 0; 1064 unsigned long npages, lock_limit; 1065 int ret; 1066 1067 if (!dev->domain || !dev->domain->bounce_map || 1068 size != dev->domain->bounce_size || 1069 iova != 0 || uaddr & ~PAGE_MASK) 1070 return -EINVAL; 1071 1072 mutex_lock(&dev->mem_lock); 1073 ret = -EEXIST; 1074 if (dev->umem) 1075 goto unlock; 1076 1077 ret = -ENOMEM; 1078 npages = size >> PAGE_SHIFT; 1079 page_list = __vmalloc(array_size(npages, sizeof(struct page *)), 1080 GFP_KERNEL_ACCOUNT); 1081 umem = kzalloc(sizeof(*umem), GFP_KERNEL); 1082 if (!page_list || !umem) 1083 goto unlock; 1084 1085 mmap_read_lock(current->mm); 1086 1087 lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK)); 1088 if (npages + atomic64_read(¤t->mm->pinned_vm) > lock_limit) 1089 goto out; 1090 1091 pinned = pin_user_pages(uaddr, npages, FOLL_LONGTERM | FOLL_WRITE, 1092 page_list); 1093 if (pinned != npages) { 1094 ret = pinned < 0 ? pinned : -ENOMEM; 1095 goto out; 1096 } 1097 1098 ret = vduse_domain_add_user_bounce_pages(dev->domain, 1099 page_list, pinned); 1100 if (ret) 1101 goto out; 1102 1103 atomic64_add(npages, ¤t->mm->pinned_vm); 1104 1105 umem->pages = page_list; 1106 umem->npages = pinned; 1107 umem->iova = iova; 1108 umem->mm = current->mm; 1109 mmgrab(current->mm); 1110 1111 dev->umem = umem; 1112out: 1113 if (ret && pinned > 0) 1114 unpin_user_pages(page_list, pinned); 1115 1116 mmap_read_unlock(current->mm); 1117unlock: 1118 if (ret) { 1119 vfree(page_list); 1120 kfree(umem); 1121 } 1122 mutex_unlock(&dev->mem_lock); 1123 return ret; 1124} 1125 1126static void vduse_vq_update_effective_cpu(struct vduse_virtqueue *vq) 1127{ 1128 int curr_cpu = vq->irq_effective_cpu; 1129 1130 while (true) { 1131 curr_cpu = cpumask_next(curr_cpu, &vq->irq_affinity); 1132 if (cpu_online(curr_cpu)) 1133 break; 1134 1135 if (curr_cpu >= nr_cpu_ids) 1136 curr_cpu = IRQ_UNBOUND; 1137 } 1138 1139 vq->irq_effective_cpu = curr_cpu; 1140} 1141 1142static long vduse_dev_ioctl(struct file *file, unsigned int cmd, 1143 unsigned long arg) 1144{ 1145 struct vduse_dev *dev = file->private_data; 1146 void __user *argp = (void __user *)arg; 1147 int ret; 1148 1149 if (unlikely(dev->broken)) 1150 return -EPERM; 1151 1152 switch (cmd) { 1153 case VDUSE_IOTLB_GET_FD: { 1154 struct vduse_iotlb_entry entry; 1155 struct vhost_iotlb_map *map; 1156 struct vdpa_map_file *map_file; 1157 struct file *f = NULL; 1158 1159 ret = -EFAULT; 1160 if (copy_from_user(&entry, argp, sizeof(entry))) 1161 break; 1162 1163 ret = -EINVAL; 1164 if (entry.start > entry.last) 1165 break; 1166 1167 mutex_lock(&dev->domain_lock); 1168 if (!dev->domain) { 1169 mutex_unlock(&dev->domain_lock); 1170 break; 1171 } 1172 spin_lock(&dev->domain->iotlb_lock); 1173 map = vhost_iotlb_itree_first(dev->domain->iotlb, 1174 entry.start, entry.last); 1175 if (map) { 1176 map_file = (struct vdpa_map_file *)map->opaque; 1177 f = get_file(map_file->file); 1178 entry.offset = map_file->offset; 1179 entry.start = map->start; 1180 entry.last = map->last; 1181 entry.perm = map->perm; 1182 } 1183 spin_unlock(&dev->domain->iotlb_lock); 1184 mutex_unlock(&dev->domain_lock); 1185 ret = -EINVAL; 1186 if (!f) 1187 break; 1188 1189 ret = -EFAULT; 1190 if (copy_to_user(argp, &entry, sizeof(entry))) { 1191 fput(f); 1192 break; 1193 } 1194 ret = receive_fd(f, NULL, perm_to_file_flags(entry.perm)); 1195 fput(f); 1196 break; 1197 } 1198 case VDUSE_DEV_GET_FEATURES: 1199 /* 1200 * Just mirror what driver wrote here. 1201 * The driver is expected to check FEATURE_OK later. 1202 */ 1203 ret = put_user(dev->driver_features, (u64 __user *)argp); 1204 break; 1205 case VDUSE_DEV_SET_CONFIG: { 1206 struct vduse_config_data config; 1207 unsigned long size = offsetof(struct vduse_config_data, 1208 buffer); 1209 1210 ret = -EFAULT; 1211 if (copy_from_user(&config, argp, size)) 1212 break; 1213 1214 ret = -EINVAL; 1215 if (config.offset > dev->config_size || 1216 config.length == 0 || 1217 config.length > dev->config_size - config.offset) 1218 break; 1219 1220 ret = -EFAULT; 1221 if (copy_from_user(dev->config + config.offset, argp + size, 1222 config.length)) 1223 break; 1224 1225 ret = 0; 1226 break; 1227 } 1228 case VDUSE_DEV_INJECT_CONFIG_IRQ: 1229 ret = vduse_dev_queue_irq_work(dev, &dev->inject, IRQ_UNBOUND); 1230 break; 1231 case VDUSE_VQ_SETUP: { 1232 struct vduse_vq_config config; 1233 u32 index; 1234 1235 ret = -EFAULT; 1236 if (copy_from_user(&config, argp, sizeof(config))) 1237 break; 1238 1239 ret = -EINVAL; 1240 if (config.index >= dev->vq_num) 1241 break; 1242 1243 if (!is_mem_zero((const char *)config.reserved, 1244 sizeof(config.reserved))) 1245 break; 1246 1247 index = array_index_nospec(config.index, dev->vq_num); 1248 dev->vqs[index]->num_max = config.max_size; 1249 ret = 0; 1250 break; 1251 } 1252 case VDUSE_VQ_GET_INFO: { 1253 struct vduse_vq_info vq_info; 1254 struct vduse_virtqueue *vq; 1255 u32 index; 1256 1257 ret = -EFAULT; 1258 if (copy_from_user(&vq_info, argp, sizeof(vq_info))) 1259 break; 1260 1261 ret = -EINVAL; 1262 if (vq_info.index >= dev->vq_num) 1263 break; 1264 1265 index = array_index_nospec(vq_info.index, dev->vq_num); 1266 vq = dev->vqs[index]; 1267 vq_info.desc_addr = vq->desc_addr; 1268 vq_info.driver_addr = vq->driver_addr; 1269 vq_info.device_addr = vq->device_addr; 1270 vq_info.num = vq->num; 1271 1272 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { 1273 vq_info.packed.last_avail_counter = 1274 vq->state.packed.last_avail_counter; 1275 vq_info.packed.last_avail_idx = 1276 vq->state.packed.last_avail_idx; 1277 vq_info.packed.last_used_counter = 1278 vq->state.packed.last_used_counter; 1279 vq_info.packed.last_used_idx = 1280 vq->state.packed.last_used_idx; 1281 } else 1282 vq_info.split.avail_index = 1283 vq->state.split.avail_index; 1284 1285 vq_info.ready = vq->ready; 1286 1287 ret = -EFAULT; 1288 if (copy_to_user(argp, &vq_info, sizeof(vq_info))) 1289 break; 1290 1291 ret = 0; 1292 break; 1293 } 1294 case VDUSE_VQ_SETUP_KICKFD: { 1295 struct vduse_vq_eventfd eventfd; 1296 1297 ret = -EFAULT; 1298 if (copy_from_user(&eventfd, argp, sizeof(eventfd))) 1299 break; 1300 1301 ret = vduse_kickfd_setup(dev, &eventfd); 1302 break; 1303 } 1304 case VDUSE_VQ_INJECT_IRQ: { 1305 u32 index; 1306 1307 ret = -EFAULT; 1308 if (get_user(index, (u32 __user *)argp)) 1309 break; 1310 1311 ret = -EINVAL; 1312 if (index >= dev->vq_num) 1313 break; 1314 1315 ret = 0; 1316 index = array_index_nospec(index, dev->vq_num); 1317 if (!vduse_vq_signal_irqfd(dev->vqs[index])) { 1318 vduse_vq_update_effective_cpu(dev->vqs[index]); 1319 ret = vduse_dev_queue_irq_work(dev, 1320 &dev->vqs[index]->inject, 1321 dev->vqs[index]->irq_effective_cpu); 1322 } 1323 break; 1324 } 1325 case VDUSE_IOTLB_REG_UMEM: { 1326 struct vduse_iova_umem umem; 1327 1328 ret = -EFAULT; 1329 if (copy_from_user(&umem, argp, sizeof(umem))) 1330 break; 1331 1332 ret = -EINVAL; 1333 if (!is_mem_zero((const char *)umem.reserved, 1334 sizeof(umem.reserved))) 1335 break; 1336 1337 mutex_lock(&dev->domain_lock); 1338 ret = vduse_dev_reg_umem(dev, umem.iova, 1339 umem.uaddr, umem.size); 1340 mutex_unlock(&dev->domain_lock); 1341 break; 1342 } 1343 case VDUSE_IOTLB_DEREG_UMEM: { 1344 struct vduse_iova_umem umem; 1345 1346 ret = -EFAULT; 1347 if (copy_from_user(&umem, argp, sizeof(umem))) 1348 break; 1349 1350 ret = -EINVAL; 1351 if (!is_mem_zero((const char *)umem.reserved, 1352 sizeof(umem.reserved))) 1353 break; 1354 mutex_lock(&dev->domain_lock); 1355 ret = vduse_dev_dereg_umem(dev, umem.iova, 1356 umem.size); 1357 mutex_unlock(&dev->domain_lock); 1358 break; 1359 } 1360 case VDUSE_IOTLB_GET_INFO: { 1361 struct vduse_iova_info info; 1362 struct vhost_iotlb_map *map; 1363 1364 ret = -EFAULT; 1365 if (copy_from_user(&info, argp, sizeof(info))) 1366 break; 1367 1368 ret = -EINVAL; 1369 if (info.start > info.last) 1370 break; 1371 1372 if (!is_mem_zero((const char *)info.reserved, 1373 sizeof(info.reserved))) 1374 break; 1375 1376 mutex_lock(&dev->domain_lock); 1377 if (!dev->domain) { 1378 mutex_unlock(&dev->domain_lock); 1379 break; 1380 } 1381 spin_lock(&dev->domain->iotlb_lock); 1382 map = vhost_iotlb_itree_first(dev->domain->iotlb, 1383 info.start, info.last); 1384 if (map) { 1385 info.start = map->start; 1386 info.last = map->last; 1387 info.capability = 0; 1388 if (dev->domain->bounce_map && map->start == 0 && 1389 map->last == dev->domain->bounce_size - 1) 1390 info.capability |= VDUSE_IOVA_CAP_UMEM; 1391 } 1392 spin_unlock(&dev->domain->iotlb_lock); 1393 mutex_unlock(&dev->domain_lock); 1394 if (!map) 1395 break; 1396 1397 ret = -EFAULT; 1398 if (copy_to_user(argp, &info, sizeof(info))) 1399 break; 1400 1401 ret = 0; 1402 break; 1403 } 1404 default: 1405 ret = -ENOIOCTLCMD; 1406 break; 1407 } 1408 1409 return ret; 1410} 1411 1412static int vduse_dev_release(struct inode *inode, struct file *file) 1413{ 1414 struct vduse_dev *dev = file->private_data; 1415 1416 mutex_lock(&dev->domain_lock); 1417 if (dev->domain) 1418 vduse_dev_dereg_umem(dev, 0, dev->domain->bounce_size); 1419 mutex_unlock(&dev->domain_lock); 1420 spin_lock(&dev->msg_lock); 1421 /* Make sure the inflight messages can processed after reconncection */ 1422 list_splice_init(&dev->recv_list, &dev->send_list); 1423 spin_unlock(&dev->msg_lock); 1424 dev->connected = false; 1425 1426 return 0; 1427} 1428 1429static struct vduse_dev *vduse_dev_get_from_minor(int minor) 1430{ 1431 struct vduse_dev *dev; 1432 1433 mutex_lock(&vduse_lock); 1434 dev = idr_find(&vduse_idr, minor); 1435 mutex_unlock(&vduse_lock); 1436 1437 return dev; 1438} 1439 1440static int vduse_dev_open(struct inode *inode, struct file *file) 1441{ 1442 int ret; 1443 struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode)); 1444 1445 if (!dev) 1446 return -ENODEV; 1447 1448 ret = -EBUSY; 1449 mutex_lock(&dev->lock); 1450 if (dev->connected) 1451 goto unlock; 1452 1453 ret = 0; 1454 dev->connected = true; 1455 file->private_data = dev; 1456unlock: 1457 mutex_unlock(&dev->lock); 1458 1459 return ret; 1460} 1461 1462static const struct file_operations vduse_dev_fops = { 1463 .owner = THIS_MODULE, 1464 .open = vduse_dev_open, 1465 .release = vduse_dev_release, 1466 .read_iter = vduse_dev_read_iter, 1467 .write_iter = vduse_dev_write_iter, 1468 .poll = vduse_dev_poll, 1469 .unlocked_ioctl = vduse_dev_ioctl, 1470 .compat_ioctl = compat_ptr_ioctl, 1471 .llseek = noop_llseek, 1472}; 1473 1474static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf) 1475{ 1476 return sprintf(buf, "%*pb\n", cpumask_pr_args(&vq->irq_affinity)); 1477} 1478 1479static ssize_t irq_cb_affinity_store(struct vduse_virtqueue *vq, 1480 const char *buf, size_t count) 1481{ 1482 cpumask_var_t new_value; 1483 int ret; 1484 1485 if (!zalloc_cpumask_var(&new_value, GFP_KERNEL)) 1486 return -ENOMEM; 1487 1488 ret = cpumask_parse(buf, new_value); 1489 if (ret) 1490 goto free_mask; 1491 1492 ret = -EINVAL; 1493 if (!cpumask_intersects(new_value, cpu_online_mask)) 1494 goto free_mask; 1495 1496 cpumask_copy(&vq->irq_affinity, new_value); 1497 ret = count; 1498free_mask: 1499 free_cpumask_var(new_value); 1500 return ret; 1501} 1502 1503struct vq_sysfs_entry { 1504 struct attribute attr; 1505 ssize_t (*show)(struct vduse_virtqueue *vq, char *buf); 1506 ssize_t (*store)(struct vduse_virtqueue *vq, const char *buf, 1507 size_t count); 1508}; 1509 1510static struct vq_sysfs_entry irq_cb_affinity_attr = __ATTR_RW(irq_cb_affinity); 1511 1512static struct attribute *vq_attrs[] = { 1513 &irq_cb_affinity_attr.attr, 1514 NULL, 1515}; 1516ATTRIBUTE_GROUPS(vq); 1517 1518static ssize_t vq_attr_show(struct kobject *kobj, struct attribute *attr, 1519 char *buf) 1520{ 1521 struct vduse_virtqueue *vq = container_of(kobj, 1522 struct vduse_virtqueue, kobj); 1523 struct vq_sysfs_entry *entry = container_of(attr, 1524 struct vq_sysfs_entry, attr); 1525 1526 if (!entry->show) 1527 return -EIO; 1528 1529 return entry->show(vq, buf); 1530} 1531 1532static ssize_t vq_attr_store(struct kobject *kobj, struct attribute *attr, 1533 const char *buf, size_t count) 1534{ 1535 struct vduse_virtqueue *vq = container_of(kobj, 1536 struct vduse_virtqueue, kobj); 1537 struct vq_sysfs_entry *entry = container_of(attr, 1538 struct vq_sysfs_entry, attr); 1539 1540 if (!entry->store) 1541 return -EIO; 1542 1543 return entry->store(vq, buf, count); 1544} 1545 1546static const struct sysfs_ops vq_sysfs_ops = { 1547 .show = vq_attr_show, 1548 .store = vq_attr_store, 1549}; 1550 1551static void vq_release(struct kobject *kobj) 1552{ 1553 struct vduse_virtqueue *vq = container_of(kobj, 1554 struct vduse_virtqueue, kobj); 1555 kfree(vq); 1556} 1557 1558static const struct kobj_type vq_type = { 1559 .release = vq_release, 1560 .sysfs_ops = &vq_sysfs_ops, 1561 .default_groups = vq_groups, 1562}; 1563 1564static char *vduse_devnode(const struct device *dev, umode_t *mode) 1565{ 1566 return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev)); 1567} 1568 1569static const struct class vduse_class = { 1570 .name = "vduse", 1571 .devnode = vduse_devnode, 1572}; 1573 1574static void vduse_dev_deinit_vqs(struct vduse_dev *dev) 1575{ 1576 int i; 1577 1578 if (!dev->vqs) 1579 return; 1580 1581 for (i = 0; i < dev->vq_num; i++) 1582 kobject_put(&dev->vqs[i]->kobj); 1583 kfree(dev->vqs); 1584} 1585 1586static int vduse_dev_init_vqs(struct vduse_dev *dev, u32 vq_align, u32 vq_num) 1587{ 1588 int ret, i; 1589 1590 dev->vq_align = vq_align; 1591 dev->vq_num = vq_num; 1592 dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL); 1593 if (!dev->vqs) 1594 return -ENOMEM; 1595 1596 for (i = 0; i < vq_num; i++) { 1597 dev->vqs[i] = kzalloc(sizeof(*dev->vqs[i]), GFP_KERNEL); 1598 if (!dev->vqs[i]) { 1599 ret = -ENOMEM; 1600 goto err; 1601 } 1602 1603 dev->vqs[i]->index = i; 1604 dev->vqs[i]->irq_effective_cpu = IRQ_UNBOUND; 1605 INIT_WORK(&dev->vqs[i]->inject, vduse_vq_irq_inject); 1606 INIT_WORK(&dev->vqs[i]->kick, vduse_vq_kick_work); 1607 spin_lock_init(&dev->vqs[i]->kick_lock); 1608 spin_lock_init(&dev->vqs[i]->irq_lock); 1609 cpumask_setall(&dev->vqs[i]->irq_affinity); 1610 1611 kobject_init(&dev->vqs[i]->kobj, &vq_type); 1612 ret = kobject_add(&dev->vqs[i]->kobj, 1613 &dev->dev->kobj, "vq%d", i); 1614 if (ret) { 1615 kfree(dev->vqs[i]); 1616 goto err; 1617 } 1618 } 1619 1620 return 0; 1621err: 1622 while (i--) 1623 kobject_put(&dev->vqs[i]->kobj); 1624 kfree(dev->vqs); 1625 dev->vqs = NULL; 1626 return ret; 1627} 1628 1629static struct vduse_dev *vduse_dev_create(void) 1630{ 1631 struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1632 1633 if (!dev) 1634 return NULL; 1635 1636 mutex_init(&dev->lock); 1637 mutex_init(&dev->mem_lock); 1638 mutex_init(&dev->domain_lock); 1639 spin_lock_init(&dev->msg_lock); 1640 INIT_LIST_HEAD(&dev->send_list); 1641 INIT_LIST_HEAD(&dev->recv_list); 1642 spin_lock_init(&dev->irq_lock); 1643 init_rwsem(&dev->rwsem); 1644 1645 INIT_WORK(&dev->inject, vduse_dev_irq_inject); 1646 init_waitqueue_head(&dev->waitq); 1647 1648 return dev; 1649} 1650 1651static void vduse_dev_destroy(struct vduse_dev *dev) 1652{ 1653 kfree(dev); 1654} 1655 1656static struct vduse_dev *vduse_find_dev(const char *name) 1657{ 1658 struct vduse_dev *dev; 1659 int id; 1660 1661 idr_for_each_entry(&vduse_idr, dev, id) 1662 if (!strcmp(dev->name, name)) 1663 return dev; 1664 1665 return NULL; 1666} 1667 1668static int vduse_destroy_dev(char *name) 1669{ 1670 struct vduse_dev *dev = vduse_find_dev(name); 1671 1672 if (!dev) 1673 return -EINVAL; 1674 1675 mutex_lock(&dev->lock); 1676 if (dev->vdev || dev->connected) { 1677 mutex_unlock(&dev->lock); 1678 return -EBUSY; 1679 } 1680 dev->connected = true; 1681 mutex_unlock(&dev->lock); 1682 1683 vduse_dev_reset(dev); 1684 device_destroy(&vduse_class, MKDEV(MAJOR(vduse_major), dev->minor)); 1685 idr_remove(&vduse_idr, dev->minor); 1686 kvfree(dev->config); 1687 vduse_dev_deinit_vqs(dev); 1688 if (dev->domain) 1689 vduse_domain_destroy(dev->domain); 1690 kfree(dev->name); 1691 vduse_dev_destroy(dev); 1692 module_put(THIS_MODULE); 1693 1694 return 0; 1695} 1696 1697static bool device_is_allowed(u32 device_id) 1698{ 1699 int i; 1700 1701 for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++) 1702 if (allowed_device_id[i] == device_id) 1703 return true; 1704 1705 return false; 1706} 1707 1708static bool features_is_valid(u64 features) 1709{ 1710 if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) 1711 return false; 1712 1713 /* Now we only support read-only configuration space */ 1714 if (features & (1ULL << VIRTIO_BLK_F_CONFIG_WCE)) 1715 return false; 1716 1717 return true; 1718} 1719 1720static bool vduse_validate_config(struct vduse_dev_config *config) 1721{ 1722 if (!is_mem_zero((const char *)config->reserved, 1723 sizeof(config->reserved))) 1724 return false; 1725 1726 if (config->vq_align > PAGE_SIZE) 1727 return false; 1728 1729 if (config->config_size > PAGE_SIZE) 1730 return false; 1731 1732 if (config->vq_num > 0xffff) 1733 return false; 1734 1735 if (!config->name[0]) 1736 return false; 1737 1738 if (!device_is_allowed(config->device_id)) 1739 return false; 1740 1741 if (!features_is_valid(config->features)) 1742 return false; 1743 1744 return true; 1745} 1746 1747static ssize_t msg_timeout_show(struct device *device, 1748 struct device_attribute *attr, char *buf) 1749{ 1750 struct vduse_dev *dev = dev_get_drvdata(device); 1751 1752 return sysfs_emit(buf, "%u\n", dev->msg_timeout); 1753} 1754 1755static ssize_t msg_timeout_store(struct device *device, 1756 struct device_attribute *attr, 1757 const char *buf, size_t count) 1758{ 1759 struct vduse_dev *dev = dev_get_drvdata(device); 1760 int ret; 1761 1762 ret = kstrtouint(buf, 10, &dev->msg_timeout); 1763 if (ret < 0) 1764 return ret; 1765 1766 return count; 1767} 1768 1769static DEVICE_ATTR_RW(msg_timeout); 1770 1771static ssize_t bounce_size_show(struct device *device, 1772 struct device_attribute *attr, char *buf) 1773{ 1774 struct vduse_dev *dev = dev_get_drvdata(device); 1775 1776 return sysfs_emit(buf, "%u\n", dev->bounce_size); 1777} 1778 1779static ssize_t bounce_size_store(struct device *device, 1780 struct device_attribute *attr, 1781 const char *buf, size_t count) 1782{ 1783 struct vduse_dev *dev = dev_get_drvdata(device); 1784 unsigned int bounce_size; 1785 int ret; 1786 1787 ret = -EPERM; 1788 mutex_lock(&dev->domain_lock); 1789 if (dev->domain) 1790 goto unlock; 1791 1792 ret = kstrtouint(buf, 10, &bounce_size); 1793 if (ret < 0) 1794 goto unlock; 1795 1796 ret = -EINVAL; 1797 if (bounce_size > VDUSE_MAX_BOUNCE_SIZE || 1798 bounce_size < VDUSE_MIN_BOUNCE_SIZE) 1799 goto unlock; 1800 1801 dev->bounce_size = bounce_size & PAGE_MASK; 1802 ret = count; 1803unlock: 1804 mutex_unlock(&dev->domain_lock); 1805 return ret; 1806} 1807 1808static DEVICE_ATTR_RW(bounce_size); 1809 1810static struct attribute *vduse_dev_attrs[] = { 1811 &dev_attr_msg_timeout.attr, 1812 &dev_attr_bounce_size.attr, 1813 NULL 1814}; 1815 1816ATTRIBUTE_GROUPS(vduse_dev); 1817 1818static int vduse_create_dev(struct vduse_dev_config *config, 1819 void *config_buf, u64 api_version) 1820{ 1821 int ret; 1822 struct vduse_dev *dev; 1823 1824 ret = -EEXIST; 1825 if (vduse_find_dev(config->name)) 1826 goto err; 1827 1828 ret = -ENOMEM; 1829 dev = vduse_dev_create(); 1830 if (!dev) 1831 goto err; 1832 1833 dev->api_version = api_version; 1834 dev->device_features = config->features; 1835 dev->device_id = config->device_id; 1836 dev->vendor_id = config->vendor_id; 1837 dev->name = kstrdup(config->name, GFP_KERNEL); 1838 if (!dev->name) 1839 goto err_str; 1840 1841 dev->bounce_size = VDUSE_BOUNCE_SIZE; 1842 dev->config = config_buf; 1843 dev->config_size = config->config_size; 1844 1845 ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL); 1846 if (ret < 0) 1847 goto err_idr; 1848 1849 dev->minor = ret; 1850 dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT; 1851 dev->dev = device_create_with_groups(&vduse_class, NULL, 1852 MKDEV(MAJOR(vduse_major), dev->minor), 1853 dev, vduse_dev_groups, "%s", config->name); 1854 if (IS_ERR(dev->dev)) { 1855 ret = PTR_ERR(dev->dev); 1856 goto err_dev; 1857 } 1858 1859 ret = vduse_dev_init_vqs(dev, config->vq_align, config->vq_num); 1860 if (ret) 1861 goto err_vqs; 1862 1863 __module_get(THIS_MODULE); 1864 1865 return 0; 1866err_vqs: 1867 device_destroy(&vduse_class, MKDEV(MAJOR(vduse_major), dev->minor)); 1868err_dev: 1869 idr_remove(&vduse_idr, dev->minor); 1870err_idr: 1871 kfree(dev->name); 1872err_str: 1873 vduse_dev_destroy(dev); 1874err: 1875 return ret; 1876} 1877 1878static long vduse_ioctl(struct file *file, unsigned int cmd, 1879 unsigned long arg) 1880{ 1881 int ret; 1882 void __user *argp = (void __user *)arg; 1883 struct vduse_control *control = file->private_data; 1884 1885 mutex_lock(&vduse_lock); 1886 switch (cmd) { 1887 case VDUSE_GET_API_VERSION: 1888 ret = put_user(control->api_version, (u64 __user *)argp); 1889 break; 1890 case VDUSE_SET_API_VERSION: { 1891 u64 api_version; 1892 1893 ret = -EFAULT; 1894 if (get_user(api_version, (u64 __user *)argp)) 1895 break; 1896 1897 ret = -EINVAL; 1898 if (api_version > VDUSE_API_VERSION) 1899 break; 1900 1901 ret = 0; 1902 control->api_version = api_version; 1903 break; 1904 } 1905 case VDUSE_CREATE_DEV: { 1906 struct vduse_dev_config config; 1907 unsigned long size = offsetof(struct vduse_dev_config, config); 1908 void *buf; 1909 1910 ret = -EFAULT; 1911 if (copy_from_user(&config, argp, size)) 1912 break; 1913 1914 ret = -EINVAL; 1915 if (vduse_validate_config(&config) == false) 1916 break; 1917 1918 buf = vmemdup_user(argp + size, config.config_size); 1919 if (IS_ERR(buf)) { 1920 ret = PTR_ERR(buf); 1921 break; 1922 } 1923 config.name[VDUSE_NAME_MAX - 1] = '\0'; 1924 ret = vduse_create_dev(&config, buf, control->api_version); 1925 if (ret) 1926 kvfree(buf); 1927 break; 1928 } 1929 case VDUSE_DESTROY_DEV: { 1930 char name[VDUSE_NAME_MAX]; 1931 1932 ret = -EFAULT; 1933 if (copy_from_user(name, argp, VDUSE_NAME_MAX)) 1934 break; 1935 1936 name[VDUSE_NAME_MAX - 1] = '\0'; 1937 ret = vduse_destroy_dev(name); 1938 break; 1939 } 1940 default: 1941 ret = -EINVAL; 1942 break; 1943 } 1944 mutex_unlock(&vduse_lock); 1945 1946 return ret; 1947} 1948 1949static int vduse_release(struct inode *inode, struct file *file) 1950{ 1951 struct vduse_control *control = file->private_data; 1952 1953 kfree(control); 1954 return 0; 1955} 1956 1957static int vduse_open(struct inode *inode, struct file *file) 1958{ 1959 struct vduse_control *control; 1960 1961 control = kmalloc(sizeof(struct vduse_control), GFP_KERNEL); 1962 if (!control) 1963 return -ENOMEM; 1964 1965 control->api_version = VDUSE_API_VERSION; 1966 file->private_data = control; 1967 1968 return 0; 1969} 1970 1971static const struct file_operations vduse_ctrl_fops = { 1972 .owner = THIS_MODULE, 1973 .open = vduse_open, 1974 .release = vduse_release, 1975 .unlocked_ioctl = vduse_ioctl, 1976 .compat_ioctl = compat_ptr_ioctl, 1977 .llseek = noop_llseek, 1978}; 1979 1980struct vduse_mgmt_dev { 1981 struct vdpa_mgmt_dev mgmt_dev; 1982 struct device dev; 1983}; 1984 1985static struct vduse_mgmt_dev *vduse_mgmt; 1986 1987static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name) 1988{ 1989 struct vduse_vdpa *vdev; 1990 int ret; 1991 1992 if (dev->vdev) 1993 return -EEXIST; 1994 1995 vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev, 1996 &vduse_vdpa_config_ops, 1, 1, name, true); 1997 if (IS_ERR(vdev)) 1998 return PTR_ERR(vdev); 1999 2000 dev->vdev = vdev; 2001 vdev->dev = dev; 2002 vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask; 2003 ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64)); 2004 if (ret) { 2005 put_device(&vdev->vdpa.dev); 2006 return ret; 2007 } 2008 set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops); 2009 vdev->vdpa.dma_dev = &vdev->vdpa.dev; 2010 vdev->vdpa.mdev = &vduse_mgmt->mgmt_dev; 2011 2012 return 0; 2013} 2014 2015static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, 2016 const struct vdpa_dev_set_config *config) 2017{ 2018 struct vduse_dev *dev; 2019 int ret; 2020 2021 mutex_lock(&vduse_lock); 2022 dev = vduse_find_dev(name); 2023 if (!dev || !vduse_dev_is_ready(dev)) { 2024 mutex_unlock(&vduse_lock); 2025 return -EINVAL; 2026 } 2027 ret = vduse_dev_init_vdpa(dev, name); 2028 mutex_unlock(&vduse_lock); 2029 if (ret) 2030 return ret; 2031 2032 mutex_lock(&dev->domain_lock); 2033 if (!dev->domain) 2034 dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1, 2035 dev->bounce_size); 2036 mutex_unlock(&dev->domain_lock); 2037 if (!dev->domain) { 2038 put_device(&dev->vdev->vdpa.dev); 2039 return -ENOMEM; 2040 } 2041 2042 ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num); 2043 if (ret) { 2044 put_device(&dev->vdev->vdpa.dev); 2045 mutex_lock(&dev->domain_lock); 2046 vduse_domain_destroy(dev->domain); 2047 dev->domain = NULL; 2048 mutex_unlock(&dev->domain_lock); 2049 return ret; 2050 } 2051 2052 return 0; 2053} 2054 2055static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev) 2056{ 2057 _vdpa_unregister_device(dev); 2058} 2059 2060static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = { 2061 .dev_add = vdpa_dev_add, 2062 .dev_del = vdpa_dev_del, 2063}; 2064 2065static struct virtio_device_id id_table[] = { 2066 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID }, 2067 { 0 }, 2068}; 2069 2070static void vduse_mgmtdev_release(struct device *dev) 2071{ 2072 struct vduse_mgmt_dev *mgmt_dev; 2073 2074 mgmt_dev = container_of(dev, struct vduse_mgmt_dev, dev); 2075 kfree(mgmt_dev); 2076} 2077 2078static int vduse_mgmtdev_init(void) 2079{ 2080 int ret; 2081 2082 vduse_mgmt = kzalloc(sizeof(*vduse_mgmt), GFP_KERNEL); 2083 if (!vduse_mgmt) 2084 return -ENOMEM; 2085 2086 ret = dev_set_name(&vduse_mgmt->dev, "vduse"); 2087 if (ret) { 2088 kfree(vduse_mgmt); 2089 return ret; 2090 } 2091 2092 vduse_mgmt->dev.release = vduse_mgmtdev_release; 2093 2094 ret = device_register(&vduse_mgmt->dev); 2095 if (ret) 2096 goto dev_reg_err; 2097 2098 vduse_mgmt->mgmt_dev.id_table = id_table; 2099 vduse_mgmt->mgmt_dev.ops = &vdpa_dev_mgmtdev_ops; 2100 vduse_mgmt->mgmt_dev.device = &vduse_mgmt->dev; 2101 ret = vdpa_mgmtdev_register(&vduse_mgmt->mgmt_dev); 2102 if (ret) 2103 device_unregister(&vduse_mgmt->dev); 2104 2105 return ret; 2106 2107dev_reg_err: 2108 put_device(&vduse_mgmt->dev); 2109 return ret; 2110} 2111 2112static void vduse_mgmtdev_exit(void) 2113{ 2114 vdpa_mgmtdev_unregister(&vduse_mgmt->mgmt_dev); 2115 device_unregister(&vduse_mgmt->dev); 2116} 2117 2118static int vduse_init(void) 2119{ 2120 int ret; 2121 struct device *dev; 2122 2123 ret = class_register(&vduse_class); 2124 if (ret) 2125 return ret; 2126 2127 ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse"); 2128 if (ret) 2129 goto err_chardev_region; 2130 2131 /* /dev/vduse/control */ 2132 cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops); 2133 vduse_ctrl_cdev.owner = THIS_MODULE; 2134 ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1); 2135 if (ret) 2136 goto err_ctrl_cdev; 2137 2138 dev = device_create(&vduse_class, NULL, vduse_major, NULL, "control"); 2139 if (IS_ERR(dev)) { 2140 ret = PTR_ERR(dev); 2141 goto err_device; 2142 } 2143 2144 /* /dev/vduse/$DEVICE */ 2145 cdev_init(&vduse_cdev, &vduse_dev_fops); 2146 vduse_cdev.owner = THIS_MODULE; 2147 ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1), 2148 VDUSE_DEV_MAX - 1); 2149 if (ret) 2150 goto err_cdev; 2151 2152 ret = -ENOMEM; 2153 vduse_irq_wq = alloc_workqueue("vduse-irq", 2154 WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0); 2155 if (!vduse_irq_wq) 2156 goto err_wq; 2157 2158 vduse_irq_bound_wq = alloc_workqueue("vduse-irq-bound", WQ_HIGHPRI, 0); 2159 if (!vduse_irq_bound_wq) 2160 goto err_bound_wq; 2161 2162 ret = vduse_domain_init(); 2163 if (ret) 2164 goto err_domain; 2165 2166 ret = vduse_mgmtdev_init(); 2167 if (ret) 2168 goto err_mgmtdev; 2169 2170 return 0; 2171err_mgmtdev: 2172 vduse_domain_exit(); 2173err_domain: 2174 destroy_workqueue(vduse_irq_bound_wq); 2175err_bound_wq: 2176 destroy_workqueue(vduse_irq_wq); 2177err_wq: 2178 cdev_del(&vduse_cdev); 2179err_cdev: 2180 device_destroy(&vduse_class, vduse_major); 2181err_device: 2182 cdev_del(&vduse_ctrl_cdev); 2183err_ctrl_cdev: 2184 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX); 2185err_chardev_region: 2186 class_unregister(&vduse_class); 2187 return ret; 2188} 2189module_init(vduse_init); 2190 2191static void vduse_exit(void) 2192{ 2193 vduse_mgmtdev_exit(); 2194 vduse_domain_exit(); 2195 destroy_workqueue(vduse_irq_bound_wq); 2196 destroy_workqueue(vduse_irq_wq); 2197 cdev_del(&vduse_cdev); 2198 device_destroy(&vduse_class, vduse_major); 2199 cdev_del(&vduse_ctrl_cdev); 2200 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX); 2201 class_unregister(&vduse_class); 2202} 2203module_exit(vduse_exit); 2204 2205MODULE_LICENSE(DRV_LICENSE); 2206MODULE_AUTHOR(DRV_AUTHOR); 2207MODULE_DESCRIPTION(DRV_DESC); 2208