1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Network block device - make block devices work over TCP 4 * 5 * Note that you can not swap over this thing, yet. Seems to work but 6 * deadlocks sometimes - you can not swap over TCP in general. 7 * 8 * Copyright 1997-2000, 2008 Pavel Machek <pavel@ucw.cz> 9 * Parts copyright 2001 Steven Whitehouse <steve@chygwyn.com> 10 * 11 * (part of code stolen from loop.c) 12 */ 13 14#define pr_fmt(fmt) "nbd: " fmt 15 16#include <linux/major.h> 17 18#include <linux/blkdev.h> 19#include <linux/module.h> 20#include <linux/init.h> 21#include <linux/sched.h> 22#include <linux/sched/mm.h> 23#include <linux/fs.h> 24#include <linux/bio.h> 25#include <linux/stat.h> 26#include <linux/errno.h> 27#include <linux/file.h> 28#include <linux/ioctl.h> 29#include <linux/mutex.h> 30#include <linux/compiler.h> 31#include <linux/completion.h> 32#include <linux/err.h> 33#include <linux/kernel.h> 34#include <linux/slab.h> 35#include <net/sock.h> 36#include <linux/net.h> 37#include <linux/kthread.h> 38#include <linux/types.h> 39#include <linux/debugfs.h> 40#include <linux/blk-mq.h> 41 42#include <linux/uaccess.h> 43#include <asm/types.h> 44 45#include <linux/nbd.h> 46#include <linux/nbd-netlink.h> 47#include <net/genetlink.h> 48 49#define CREATE_TRACE_POINTS 50#include <trace/events/nbd.h> 51 52static DEFINE_IDR(nbd_index_idr); 53static DEFINE_MUTEX(nbd_index_mutex); 54static struct workqueue_struct *nbd_del_wq; 55static int nbd_total_devices = 0; 56 57struct nbd_sock { 58 struct socket *sock; 59 struct mutex tx_lock; 60 struct request *pending; 61 int sent; 62 bool dead; 63 int fallback_index; 64 int cookie; 65}; 66 67struct recv_thread_args { 68 struct work_struct work; 69 struct nbd_device *nbd; 70 struct nbd_sock *nsock; 71 int index; 72}; 73 74struct link_dead_args { 75 struct work_struct work; 76 int index; 77}; 78 79#define NBD_RT_TIMEDOUT 0 80#define NBD_RT_DISCONNECT_REQUESTED 1 81#define NBD_RT_DISCONNECTED 2 82#define NBD_RT_HAS_PID_FILE 3 83#define NBD_RT_HAS_CONFIG_REF 4 84#define NBD_RT_BOUND 5 85#define NBD_RT_DISCONNECT_ON_CLOSE 6 86#define NBD_RT_HAS_BACKEND_FILE 7 87 88#define NBD_DESTROY_ON_DISCONNECT 0 89#define NBD_DISCONNECT_REQUESTED 1 90 91struct nbd_config { 92 u32 flags; 93 unsigned long runtime_flags; 94 u64 dead_conn_timeout; 95 96 struct nbd_sock **socks; 97 int num_connections; 98 atomic_t live_connections; 99 wait_queue_head_t conn_wait; 100 101 atomic_t recv_threads; 102 wait_queue_head_t recv_wq; 103 unsigned int blksize_bits; 104 loff_t bytesize; 105#if IS_ENABLED(CONFIG_DEBUG_FS) 106 struct dentry *dbg_dir; 107#endif 108}; 109 110static inline unsigned int nbd_blksize(struct nbd_config *config) 111{ 112 return 1u << config->blksize_bits; 113} 114 115struct nbd_device { 116 struct blk_mq_tag_set tag_set; 117 118 int index; 119 refcount_t config_refs; 120 refcount_t refs; 121 struct nbd_config *config; 122 struct mutex config_lock; 123 struct gendisk *disk; 124 struct workqueue_struct *recv_workq; 125 struct work_struct remove_work; 126 127 struct list_head list; 128 struct task_struct *task_setup; 129 130 unsigned long flags; 131 pid_t pid; /* pid of nbd-client, if attached */ 132 133 char *backend; 134}; 135 136#define NBD_CMD_REQUEUED 1 137/* 138 * This flag will be set if nbd_queue_rq() succeed, and will be checked and 139 * cleared in completion. Both setting and clearing of the flag are protected 140 * by cmd->lock. 141 */ 142#define NBD_CMD_INFLIGHT 2 143 144struct nbd_cmd { 145 struct nbd_device *nbd; 146 struct mutex lock; 147 int index; 148 int cookie; 149 int retries; 150 blk_status_t status; 151 unsigned long flags; 152 u32 cmd_cookie; 153}; 154 155#if IS_ENABLED(CONFIG_DEBUG_FS) 156static struct dentry *nbd_dbg_dir; 157#endif 158 159#define nbd_name(nbd) ((nbd)->disk->disk_name) 160 161#define NBD_DEF_BLKSIZE_BITS 10 162 163static unsigned int nbds_max = 16; 164static int max_part = 16; 165static int part_shift; 166 167static int nbd_dev_dbg_init(struct nbd_device *nbd); 168static void nbd_dev_dbg_close(struct nbd_device *nbd); 169static void nbd_config_put(struct nbd_device *nbd); 170static void nbd_connect_reply(struct genl_info *info, int index); 171static int nbd_genl_status(struct sk_buff *skb, struct genl_info *info); 172static void nbd_dead_link_work(struct work_struct *work); 173static void nbd_disconnect_and_put(struct nbd_device *nbd); 174 175static inline struct device *nbd_to_dev(struct nbd_device *nbd) 176{ 177 return disk_to_dev(nbd->disk); 178} 179 180static void nbd_requeue_cmd(struct nbd_cmd *cmd) 181{ 182 struct request *req = blk_mq_rq_from_pdu(cmd); 183 184 if (!test_and_set_bit(NBD_CMD_REQUEUED, &cmd->flags)) 185 blk_mq_requeue_request(req, true); 186} 187 188#define NBD_COOKIE_BITS 32 189 190static u64 nbd_cmd_handle(struct nbd_cmd *cmd) 191{ 192 struct request *req = blk_mq_rq_from_pdu(cmd); 193 u32 tag = blk_mq_unique_tag(req); 194 u64 cookie = cmd->cmd_cookie; 195 196 return (cookie << NBD_COOKIE_BITS) | tag; 197} 198 199static u32 nbd_handle_to_tag(u64 handle) 200{ 201 return (u32)handle; 202} 203 204static u32 nbd_handle_to_cookie(u64 handle) 205{ 206 return (u32)(handle >> NBD_COOKIE_BITS); 207} 208 209static const char *nbdcmd_to_ascii(int cmd) 210{ 211 switch (cmd) { 212 case NBD_CMD_READ: return "read"; 213 case NBD_CMD_WRITE: return "write"; 214 case NBD_CMD_DISC: return "disconnect"; 215 case NBD_CMD_FLUSH: return "flush"; 216 case NBD_CMD_TRIM: return "trim/discard"; 217 } 218 return "invalid"; 219} 220 221static ssize_t pid_show(struct device *dev, 222 struct device_attribute *attr, char *buf) 223{ 224 struct gendisk *disk = dev_to_disk(dev); 225 struct nbd_device *nbd = (struct nbd_device *)disk->private_data; 226 227 return sprintf(buf, "%d\n", nbd->pid); 228} 229 230static const struct device_attribute pid_attr = { 231 .attr = { .name = "pid", .mode = 0444}, 232 .show = pid_show, 233}; 234 235static ssize_t backend_show(struct device *dev, 236 struct device_attribute *attr, char *buf) 237{ 238 struct gendisk *disk = dev_to_disk(dev); 239 struct nbd_device *nbd = (struct nbd_device *)disk->private_data; 240 241 return sprintf(buf, "%s\n", nbd->backend ?: ""); 242} 243 244static const struct device_attribute backend_attr = { 245 .attr = { .name = "backend", .mode = 0444}, 246 .show = backend_show, 247}; 248 249static void nbd_dev_remove(struct nbd_device *nbd) 250{ 251 struct gendisk *disk = nbd->disk; 252 253 del_gendisk(disk); 254 blk_mq_free_tag_set(&nbd->tag_set); 255 256 /* 257 * Remove from idr after del_gendisk() completes, so if the same ID is 258 * reused, the following add_disk() will succeed. 259 */ 260 mutex_lock(&nbd_index_mutex); 261 idr_remove(&nbd_index_idr, nbd->index); 262 mutex_unlock(&nbd_index_mutex); 263 destroy_workqueue(nbd->recv_workq); 264 put_disk(disk); 265} 266 267static void nbd_dev_remove_work(struct work_struct *work) 268{ 269 nbd_dev_remove(container_of(work, struct nbd_device, remove_work)); 270} 271 272static void nbd_put(struct nbd_device *nbd) 273{ 274 if (!refcount_dec_and_test(&nbd->refs)) 275 return; 276 277 /* Call del_gendisk() asynchrounously to prevent deadlock */ 278 if (test_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags)) 279 queue_work(nbd_del_wq, &nbd->remove_work); 280 else 281 nbd_dev_remove(nbd); 282} 283 284static int nbd_disconnected(struct nbd_config *config) 285{ 286 return test_bit(NBD_RT_DISCONNECTED, &config->runtime_flags) || 287 test_bit(NBD_RT_DISCONNECT_REQUESTED, &config->runtime_flags); 288} 289 290static void nbd_mark_nsock_dead(struct nbd_device *nbd, struct nbd_sock *nsock, 291 int notify) 292{ 293 if (!nsock->dead && notify && !nbd_disconnected(nbd->config)) { 294 struct link_dead_args *args; 295 args = kmalloc(sizeof(struct link_dead_args), GFP_NOIO); 296 if (args) { 297 INIT_WORK(&args->work, nbd_dead_link_work); 298 args->index = nbd->index; 299 queue_work(system_wq, &args->work); 300 } 301 } 302 if (!nsock->dead) { 303 kernel_sock_shutdown(nsock->sock, SHUT_RDWR); 304 if (atomic_dec_return(&nbd->config->live_connections) == 0) { 305 if (test_and_clear_bit(NBD_RT_DISCONNECT_REQUESTED, 306 &nbd->config->runtime_flags)) { 307 set_bit(NBD_RT_DISCONNECTED, 308 &nbd->config->runtime_flags); 309 dev_info(nbd_to_dev(nbd), 310 "Disconnected due to user request.\n"); 311 } 312 } 313 } 314 nsock->dead = true; 315 nsock->pending = NULL; 316 nsock->sent = 0; 317} 318 319static int __nbd_set_size(struct nbd_device *nbd, loff_t bytesize, 320 loff_t blksize) 321{ 322 struct queue_limits lim; 323 int error; 324 325 if (!blksize) 326 blksize = 1u << NBD_DEF_BLKSIZE_BITS; 327 328 if (blk_validate_block_size(blksize)) 329 return -EINVAL; 330 331 if (bytesize < 0) 332 return -EINVAL; 333 334 nbd->config->bytesize = bytesize; 335 nbd->config->blksize_bits = __ffs(blksize); 336 337 if (!nbd->pid) 338 return 0; 339 340 lim = queue_limits_start_update(nbd->disk->queue); 341 if (nbd->config->flags & NBD_FLAG_SEND_TRIM) 342 lim.max_hw_discard_sectors = UINT_MAX; 343 else 344 lim.max_hw_discard_sectors = 0; 345 lim.logical_block_size = blksize; 346 lim.physical_block_size = blksize; 347 error = queue_limits_commit_update(nbd->disk->queue, &lim); 348 if (error) 349 return error; 350 351 if (max_part) 352 set_bit(GD_NEED_PART_SCAN, &nbd->disk->state); 353 if (!set_capacity_and_notify(nbd->disk, bytesize >> 9)) 354 kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE); 355 return 0; 356} 357 358static int nbd_set_size(struct nbd_device *nbd, loff_t bytesize, 359 loff_t blksize) 360{ 361 int error; 362 363 blk_mq_freeze_queue(nbd->disk->queue); 364 error = __nbd_set_size(nbd, bytesize, blksize); 365 blk_mq_unfreeze_queue(nbd->disk->queue); 366 367 return error; 368} 369 370static void nbd_complete_rq(struct request *req) 371{ 372 struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); 373 374 dev_dbg(nbd_to_dev(cmd->nbd), "request %p: %s\n", req, 375 cmd->status ? "failed" : "done"); 376 377 blk_mq_end_request(req, cmd->status); 378} 379 380/* 381 * Forcibly shutdown the socket causing all listeners to error 382 */ 383static void sock_shutdown(struct nbd_device *nbd) 384{ 385 struct nbd_config *config = nbd->config; 386 int i; 387 388 if (config->num_connections == 0) 389 return; 390 if (test_and_set_bit(NBD_RT_DISCONNECTED, &config->runtime_flags)) 391 return; 392 393 for (i = 0; i < config->num_connections; i++) { 394 struct nbd_sock *nsock = config->socks[i]; 395 mutex_lock(&nsock->tx_lock); 396 nbd_mark_nsock_dead(nbd, nsock, 0); 397 mutex_unlock(&nsock->tx_lock); 398 } 399 dev_warn(disk_to_dev(nbd->disk), "shutting down sockets\n"); 400} 401 402static u32 req_to_nbd_cmd_type(struct request *req) 403{ 404 switch (req_op(req)) { 405 case REQ_OP_DISCARD: 406 return NBD_CMD_TRIM; 407 case REQ_OP_FLUSH: 408 return NBD_CMD_FLUSH; 409 case REQ_OP_WRITE: 410 return NBD_CMD_WRITE; 411 case REQ_OP_READ: 412 return NBD_CMD_READ; 413 default: 414 return U32_MAX; 415 } 416} 417 418static struct nbd_config *nbd_get_config_unlocked(struct nbd_device *nbd) 419{ 420 if (refcount_inc_not_zero(&nbd->config_refs)) { 421 /* 422 * Add smp_mb__after_atomic to ensure that reading nbd->config_refs 423 * and reading nbd->config is ordered. The pair is the barrier in 424 * nbd_alloc_and_init_config(), avoid nbd->config_refs is set 425 * before nbd->config. 426 */ 427 smp_mb__after_atomic(); 428 return nbd->config; 429 } 430 431 return NULL; 432} 433 434static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req) 435{ 436 struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); 437 struct nbd_device *nbd = cmd->nbd; 438 struct nbd_config *config; 439 440 if (!mutex_trylock(&cmd->lock)) 441 return BLK_EH_RESET_TIMER; 442 443 if (!test_bit(NBD_CMD_INFLIGHT, &cmd->flags)) { 444 mutex_unlock(&cmd->lock); 445 return BLK_EH_DONE; 446 } 447 448 config = nbd_get_config_unlocked(nbd); 449 if (!config) { 450 cmd->status = BLK_STS_TIMEOUT; 451 __clear_bit(NBD_CMD_INFLIGHT, &cmd->flags); 452 mutex_unlock(&cmd->lock); 453 goto done; 454 } 455 456 if (config->num_connections > 1 || 457 (config->num_connections == 1 && nbd->tag_set.timeout)) { 458 dev_err_ratelimited(nbd_to_dev(nbd), 459 "Connection timed out, retrying (%d/%d alive)\n", 460 atomic_read(&config->live_connections), 461 config->num_connections); 462 /* 463 * Hooray we have more connections, requeue this IO, the submit 464 * path will put it on a real connection. Or if only one 465 * connection is configured, the submit path will wait util 466 * a new connection is reconfigured or util dead timeout. 467 */ 468 if (config->socks) { 469 if (cmd->index < config->num_connections) { 470 struct nbd_sock *nsock = 471 config->socks[cmd->index]; 472 mutex_lock(&nsock->tx_lock); 473 /* We can have multiple outstanding requests, so 474 * we don't want to mark the nsock dead if we've 475 * already reconnected with a new socket, so 476 * only mark it dead if its the same socket we 477 * were sent out on. 478 */ 479 if (cmd->cookie == nsock->cookie) 480 nbd_mark_nsock_dead(nbd, nsock, 1); 481 mutex_unlock(&nsock->tx_lock); 482 } 483 mutex_unlock(&cmd->lock); 484 nbd_requeue_cmd(cmd); 485 nbd_config_put(nbd); 486 return BLK_EH_DONE; 487 } 488 } 489 490 if (!nbd->tag_set.timeout) { 491 /* 492 * Userspace sets timeout=0 to disable socket disconnection, 493 * so just warn and reset the timer. 494 */ 495 struct nbd_sock *nsock = config->socks[cmd->index]; 496 cmd->retries++; 497 dev_info(nbd_to_dev(nbd), "Possible stuck request %p: control (%s@%llu,%uB). Runtime %u seconds\n", 498 req, nbdcmd_to_ascii(req_to_nbd_cmd_type(req)), 499 (unsigned long long)blk_rq_pos(req) << 9, 500 blk_rq_bytes(req), (req->timeout / HZ) * cmd->retries); 501 502 mutex_lock(&nsock->tx_lock); 503 if (cmd->cookie != nsock->cookie) { 504 nbd_requeue_cmd(cmd); 505 mutex_unlock(&nsock->tx_lock); 506 mutex_unlock(&cmd->lock); 507 nbd_config_put(nbd); 508 return BLK_EH_DONE; 509 } 510 mutex_unlock(&nsock->tx_lock); 511 mutex_unlock(&cmd->lock); 512 nbd_config_put(nbd); 513 return BLK_EH_RESET_TIMER; 514 } 515 516 dev_err_ratelimited(nbd_to_dev(nbd), "Connection timed out\n"); 517 set_bit(NBD_RT_TIMEDOUT, &config->runtime_flags); 518 cmd->status = BLK_STS_IOERR; 519 __clear_bit(NBD_CMD_INFLIGHT, &cmd->flags); 520 mutex_unlock(&cmd->lock); 521 sock_shutdown(nbd); 522 nbd_config_put(nbd); 523done: 524 blk_mq_complete_request(req); 525 return BLK_EH_DONE; 526} 527 528static int __sock_xmit(struct nbd_device *nbd, struct socket *sock, int send, 529 struct iov_iter *iter, int msg_flags, int *sent) 530{ 531 int result; 532 struct msghdr msg = {} ; 533 unsigned int noreclaim_flag; 534 535 if (unlikely(!sock)) { 536 dev_err_ratelimited(disk_to_dev(nbd->disk), 537 "Attempted %s on closed socket in sock_xmit\n", 538 (send ? "send" : "recv")); 539 return -EINVAL; 540 } 541 542 msg.msg_iter = *iter; 543 544 noreclaim_flag = memalloc_noreclaim_save(); 545 do { 546 sock->sk->sk_allocation = GFP_NOIO | __GFP_MEMALLOC; 547 sock->sk->sk_use_task_frag = false; 548 msg.msg_flags = msg_flags | MSG_NOSIGNAL; 549 550 if (send) 551 result = sock_sendmsg(sock, &msg); 552 else 553 result = sock_recvmsg(sock, &msg, msg.msg_flags); 554 555 if (result <= 0) { 556 if (result == 0) 557 result = -EPIPE; /* short read */ 558 break; 559 } 560 if (sent) 561 *sent += result; 562 } while (msg_data_left(&msg)); 563 564 memalloc_noreclaim_restore(noreclaim_flag); 565 566 return result; 567} 568 569/* 570 * Send or receive packet. Return a positive value on success and 571 * negtive value on failure, and never return 0. 572 */ 573static int sock_xmit(struct nbd_device *nbd, int index, int send, 574 struct iov_iter *iter, int msg_flags, int *sent) 575{ 576 struct nbd_config *config = nbd->config; 577 struct socket *sock = config->socks[index]->sock; 578 579 return __sock_xmit(nbd, sock, send, iter, msg_flags, sent); 580} 581 582/* 583 * Different settings for sk->sk_sndtimeo can result in different return values 584 * if there is a signal pending when we enter sendmsg, because reasons? 585 */ 586static inline int was_interrupted(int result) 587{ 588 return result == -ERESTARTSYS || result == -EINTR; 589} 590 591/* always call with the tx_lock held */ 592static int nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd, int index) 593{ 594 struct request *req = blk_mq_rq_from_pdu(cmd); 595 struct nbd_config *config = nbd->config; 596 struct nbd_sock *nsock = config->socks[index]; 597 int result; 598 struct nbd_request request = {.magic = htonl(NBD_REQUEST_MAGIC)}; 599 struct kvec iov = {.iov_base = &request, .iov_len = sizeof(request)}; 600 struct iov_iter from; 601 unsigned long size = blk_rq_bytes(req); 602 struct bio *bio; 603 u64 handle; 604 u32 type; 605 u32 nbd_cmd_flags = 0; 606 int sent = nsock->sent, skip = 0; 607 608 iov_iter_kvec(&from, ITER_SOURCE, &iov, 1, sizeof(request)); 609 610 type = req_to_nbd_cmd_type(req); 611 if (type == U32_MAX) 612 return -EIO; 613 614 if (rq_data_dir(req) == WRITE && 615 (config->flags & NBD_FLAG_READ_ONLY)) { 616 dev_err_ratelimited(disk_to_dev(nbd->disk), 617 "Write on read-only\n"); 618 return -EIO; 619 } 620 621 if (req->cmd_flags & REQ_FUA) 622 nbd_cmd_flags |= NBD_CMD_FLAG_FUA; 623 624 /* We did a partial send previously, and we at least sent the whole 625 * request struct, so just go and send the rest of the pages in the 626 * request. 627 */ 628 if (sent) { 629 if (sent >= sizeof(request)) { 630 skip = sent - sizeof(request); 631 632 /* initialize handle for tracing purposes */ 633 handle = nbd_cmd_handle(cmd); 634 635 goto send_pages; 636 } 637 iov_iter_advance(&from, sent); 638 } else { 639 cmd->cmd_cookie++; 640 } 641 cmd->index = index; 642 cmd->cookie = nsock->cookie; 643 cmd->retries = 0; 644 request.type = htonl(type | nbd_cmd_flags); 645 if (type != NBD_CMD_FLUSH) { 646 request.from = cpu_to_be64((u64)blk_rq_pos(req) << 9); 647 request.len = htonl(size); 648 } 649 handle = nbd_cmd_handle(cmd); 650 request.cookie = cpu_to_be64(handle); 651 652 trace_nbd_send_request(&request, nbd->index, blk_mq_rq_from_pdu(cmd)); 653 654 dev_dbg(nbd_to_dev(nbd), "request %p: sending control (%s@%llu,%uB)\n", 655 req, nbdcmd_to_ascii(type), 656 (unsigned long long)blk_rq_pos(req) << 9, blk_rq_bytes(req)); 657 result = sock_xmit(nbd, index, 1, &from, 658 (type == NBD_CMD_WRITE) ? MSG_MORE : 0, &sent); 659 trace_nbd_header_sent(req, handle); 660 if (result < 0) { 661 if (was_interrupted(result)) { 662 /* If we haven't sent anything we can just return BUSY, 663 * however if we have sent something we need to make 664 * sure we only allow this req to be sent until we are 665 * completely done. 666 */ 667 if (sent) { 668 nsock->pending = req; 669 nsock->sent = sent; 670 } 671 set_bit(NBD_CMD_REQUEUED, &cmd->flags); 672 return BLK_STS_RESOURCE; 673 } 674 dev_err_ratelimited(disk_to_dev(nbd->disk), 675 "Send control failed (result %d)\n", result); 676 return -EAGAIN; 677 } 678send_pages: 679 if (type != NBD_CMD_WRITE) 680 goto out; 681 682 bio = req->bio; 683 while (bio) { 684 struct bio *next = bio->bi_next; 685 struct bvec_iter iter; 686 struct bio_vec bvec; 687 688 bio_for_each_segment(bvec, bio, iter) { 689 bool is_last = !next && bio_iter_last(bvec, iter); 690 int flags = is_last ? 0 : MSG_MORE; 691 692 dev_dbg(nbd_to_dev(nbd), "request %p: sending %d bytes data\n", 693 req, bvec.bv_len); 694 iov_iter_bvec(&from, ITER_SOURCE, &bvec, 1, bvec.bv_len); 695 if (skip) { 696 if (skip >= iov_iter_count(&from)) { 697 skip -= iov_iter_count(&from); 698 continue; 699 } 700 iov_iter_advance(&from, skip); 701 skip = 0; 702 } 703 result = sock_xmit(nbd, index, 1, &from, flags, &sent); 704 if (result < 0) { 705 if (was_interrupted(result)) { 706 /* We've already sent the header, we 707 * have no choice but to set pending and 708 * return BUSY. 709 */ 710 nsock->pending = req; 711 nsock->sent = sent; 712 set_bit(NBD_CMD_REQUEUED, &cmd->flags); 713 return BLK_STS_RESOURCE; 714 } 715 dev_err(disk_to_dev(nbd->disk), 716 "Send data failed (result %d)\n", 717 result); 718 return -EAGAIN; 719 } 720 /* 721 * The completion might already have come in, 722 * so break for the last one instead of letting 723 * the iterator do it. This prevents use-after-free 724 * of the bio. 725 */ 726 if (is_last) 727 break; 728 } 729 bio = next; 730 } 731out: 732 trace_nbd_payload_sent(req, handle); 733 nsock->pending = NULL; 734 nsock->sent = 0; 735 return 0; 736} 737 738static int nbd_read_reply(struct nbd_device *nbd, struct socket *sock, 739 struct nbd_reply *reply) 740{ 741 struct kvec iov = {.iov_base = reply, .iov_len = sizeof(*reply)}; 742 struct iov_iter to; 743 int result; 744 745 reply->magic = 0; 746 iov_iter_kvec(&to, ITER_DEST, &iov, 1, sizeof(*reply)); 747 result = __sock_xmit(nbd, sock, 0, &to, MSG_WAITALL, NULL); 748 if (result < 0) { 749 if (!nbd_disconnected(nbd->config)) 750 dev_err(disk_to_dev(nbd->disk), 751 "Receive control failed (result %d)\n", result); 752 return result; 753 } 754 755 if (ntohl(reply->magic) != NBD_REPLY_MAGIC) { 756 dev_err(disk_to_dev(nbd->disk), "Wrong magic (0x%lx)\n", 757 (unsigned long)ntohl(reply->magic)); 758 return -EPROTO; 759 } 760 761 return 0; 762} 763 764/* NULL returned = something went wrong, inform userspace */ 765static struct nbd_cmd *nbd_handle_reply(struct nbd_device *nbd, int index, 766 struct nbd_reply *reply) 767{ 768 int result; 769 struct nbd_cmd *cmd; 770 struct request *req = NULL; 771 u64 handle; 772 u16 hwq; 773 u32 tag; 774 int ret = 0; 775 776 handle = be64_to_cpu(reply->cookie); 777 tag = nbd_handle_to_tag(handle); 778 hwq = blk_mq_unique_tag_to_hwq(tag); 779 if (hwq < nbd->tag_set.nr_hw_queues) 780 req = blk_mq_tag_to_rq(nbd->tag_set.tags[hwq], 781 blk_mq_unique_tag_to_tag(tag)); 782 if (!req || !blk_mq_request_started(req)) { 783 dev_err(disk_to_dev(nbd->disk), "Unexpected reply (%d) %p\n", 784 tag, req); 785 return ERR_PTR(-ENOENT); 786 } 787 trace_nbd_header_received(req, handle); 788 cmd = blk_mq_rq_to_pdu(req); 789 790 mutex_lock(&cmd->lock); 791 if (!test_bit(NBD_CMD_INFLIGHT, &cmd->flags)) { 792 dev_err(disk_to_dev(nbd->disk), "Suspicious reply %d (status %u flags %lu)", 793 tag, cmd->status, cmd->flags); 794 ret = -ENOENT; 795 goto out; 796 } 797 if (cmd->index != index) { 798 dev_err(disk_to_dev(nbd->disk), "Unexpected reply %d from different sock %d (expected %d)", 799 tag, index, cmd->index); 800 ret = -ENOENT; 801 goto out; 802 } 803 if (cmd->cmd_cookie != nbd_handle_to_cookie(handle)) { 804 dev_err(disk_to_dev(nbd->disk), "Double reply on req %p, cmd_cookie %u, handle cookie %u\n", 805 req, cmd->cmd_cookie, nbd_handle_to_cookie(handle)); 806 ret = -ENOENT; 807 goto out; 808 } 809 if (cmd->status != BLK_STS_OK) { 810 dev_err(disk_to_dev(nbd->disk), "Command already handled %p\n", 811 req); 812 ret = -ENOENT; 813 goto out; 814 } 815 if (test_bit(NBD_CMD_REQUEUED, &cmd->flags)) { 816 dev_err(disk_to_dev(nbd->disk), "Raced with timeout on req %p\n", 817 req); 818 ret = -ENOENT; 819 goto out; 820 } 821 if (ntohl(reply->error)) { 822 dev_err(disk_to_dev(nbd->disk), "Other side returned error (%d)\n", 823 ntohl(reply->error)); 824 cmd->status = BLK_STS_IOERR; 825 goto out; 826 } 827 828 dev_dbg(nbd_to_dev(nbd), "request %p: got reply\n", req); 829 if (rq_data_dir(req) != WRITE) { 830 struct req_iterator iter; 831 struct bio_vec bvec; 832 struct iov_iter to; 833 834 rq_for_each_segment(bvec, req, iter) { 835 iov_iter_bvec(&to, ITER_DEST, &bvec, 1, bvec.bv_len); 836 result = sock_xmit(nbd, index, 0, &to, MSG_WAITALL, NULL); 837 if (result < 0) { 838 dev_err(disk_to_dev(nbd->disk), "Receive data failed (result %d)\n", 839 result); 840 /* 841 * If we've disconnected, we need to make sure we 842 * complete this request, otherwise error out 843 * and let the timeout stuff handle resubmitting 844 * this request onto another connection. 845 */ 846 if (nbd_disconnected(nbd->config)) { 847 cmd->status = BLK_STS_IOERR; 848 goto out; 849 } 850 ret = -EIO; 851 goto out; 852 } 853 dev_dbg(nbd_to_dev(nbd), "request %p: got %d bytes data\n", 854 req, bvec.bv_len); 855 } 856 } 857out: 858 trace_nbd_payload_received(req, handle); 859 mutex_unlock(&cmd->lock); 860 return ret ? ERR_PTR(ret) : cmd; 861} 862 863static void recv_work(struct work_struct *work) 864{ 865 struct recv_thread_args *args = container_of(work, 866 struct recv_thread_args, 867 work); 868 struct nbd_device *nbd = args->nbd; 869 struct nbd_config *config = nbd->config; 870 struct request_queue *q = nbd->disk->queue; 871 struct nbd_sock *nsock = args->nsock; 872 struct nbd_cmd *cmd; 873 struct request *rq; 874 875 while (1) { 876 struct nbd_reply reply; 877 878 if (nbd_read_reply(nbd, nsock->sock, &reply)) 879 break; 880 881 /* 882 * Grab .q_usage_counter so request pool won't go away, then no 883 * request use-after-free is possible during nbd_handle_reply(). 884 * If queue is frozen, there won't be any inflight requests, we 885 * needn't to handle the incoming garbage message. 886 */ 887 if (!percpu_ref_tryget(&q->q_usage_counter)) { 888 dev_err(disk_to_dev(nbd->disk), "%s: no io inflight\n", 889 __func__); 890 break; 891 } 892 893 cmd = nbd_handle_reply(nbd, args->index, &reply); 894 if (IS_ERR(cmd)) { 895 percpu_ref_put(&q->q_usage_counter); 896 break; 897 } 898 899 rq = blk_mq_rq_from_pdu(cmd); 900 if (likely(!blk_should_fake_timeout(rq->q))) { 901 bool complete; 902 903 mutex_lock(&cmd->lock); 904 complete = __test_and_clear_bit(NBD_CMD_INFLIGHT, 905 &cmd->flags); 906 mutex_unlock(&cmd->lock); 907 if (complete) 908 blk_mq_complete_request(rq); 909 } 910 percpu_ref_put(&q->q_usage_counter); 911 } 912 913 mutex_lock(&nsock->tx_lock); 914 nbd_mark_nsock_dead(nbd, nsock, 1); 915 mutex_unlock(&nsock->tx_lock); 916 917 nbd_config_put(nbd); 918 atomic_dec(&config->recv_threads); 919 wake_up(&config->recv_wq); 920 kfree(args); 921} 922 923static bool nbd_clear_req(struct request *req, void *data) 924{ 925 struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); 926 927 /* don't abort one completed request */ 928 if (blk_mq_request_completed(req)) 929 return true; 930 931 mutex_lock(&cmd->lock); 932 if (!__test_and_clear_bit(NBD_CMD_INFLIGHT, &cmd->flags)) { 933 mutex_unlock(&cmd->lock); 934 return true; 935 } 936 cmd->status = BLK_STS_IOERR; 937 mutex_unlock(&cmd->lock); 938 939 blk_mq_complete_request(req); 940 return true; 941} 942 943static void nbd_clear_que(struct nbd_device *nbd) 944{ 945 blk_mq_quiesce_queue(nbd->disk->queue); 946 blk_mq_tagset_busy_iter(&nbd->tag_set, nbd_clear_req, NULL); 947 blk_mq_unquiesce_queue(nbd->disk->queue); 948 dev_dbg(disk_to_dev(nbd->disk), "queue cleared\n"); 949} 950 951static int find_fallback(struct nbd_device *nbd, int index) 952{ 953 struct nbd_config *config = nbd->config; 954 int new_index = -1; 955 struct nbd_sock *nsock = config->socks[index]; 956 int fallback = nsock->fallback_index; 957 958 if (test_bit(NBD_RT_DISCONNECTED, &config->runtime_flags)) 959 return new_index; 960 961 if (config->num_connections <= 1) { 962 dev_err_ratelimited(disk_to_dev(nbd->disk), 963 "Dead connection, failed to find a fallback\n"); 964 return new_index; 965 } 966 967 if (fallback >= 0 && fallback < config->num_connections && 968 !config->socks[fallback]->dead) 969 return fallback; 970 971 if (nsock->fallback_index < 0 || 972 nsock->fallback_index >= config->num_connections || 973 config->socks[nsock->fallback_index]->dead) { 974 int i; 975 for (i = 0; i < config->num_connections; i++) { 976 if (i == index) 977 continue; 978 if (!config->socks[i]->dead) { 979 new_index = i; 980 break; 981 } 982 } 983 nsock->fallback_index = new_index; 984 if (new_index < 0) { 985 dev_err_ratelimited(disk_to_dev(nbd->disk), 986 "Dead connection, failed to find a fallback\n"); 987 return new_index; 988 } 989 } 990 new_index = nsock->fallback_index; 991 return new_index; 992} 993 994static int wait_for_reconnect(struct nbd_device *nbd) 995{ 996 struct nbd_config *config = nbd->config; 997 if (!config->dead_conn_timeout) 998 return 0; 999 1000 if (!wait_event_timeout(config->conn_wait, 1001 test_bit(NBD_RT_DISCONNECTED, 1002 &config->runtime_flags) || 1003 atomic_read(&config->live_connections) > 0, 1004 config->dead_conn_timeout)) 1005 return 0; 1006 1007 return !test_bit(NBD_RT_DISCONNECTED, &config->runtime_flags); 1008} 1009 1010static int nbd_handle_cmd(struct nbd_cmd *cmd, int index) 1011{ 1012 struct request *req = blk_mq_rq_from_pdu(cmd); 1013 struct nbd_device *nbd = cmd->nbd; 1014 struct nbd_config *config; 1015 struct nbd_sock *nsock; 1016 int ret; 1017 1018 config = nbd_get_config_unlocked(nbd); 1019 if (!config) { 1020 dev_err_ratelimited(disk_to_dev(nbd->disk), 1021 "Socks array is empty\n"); 1022 return -EINVAL; 1023 } 1024 1025 if (index >= config->num_connections) { 1026 dev_err_ratelimited(disk_to_dev(nbd->disk), 1027 "Attempted send on invalid socket\n"); 1028 nbd_config_put(nbd); 1029 return -EINVAL; 1030 } 1031 cmd->status = BLK_STS_OK; 1032again: 1033 nsock = config->socks[index]; 1034 mutex_lock(&nsock->tx_lock); 1035 if (nsock->dead) { 1036 int old_index = index; 1037 index = find_fallback(nbd, index); 1038 mutex_unlock(&nsock->tx_lock); 1039 if (index < 0) { 1040 if (wait_for_reconnect(nbd)) { 1041 index = old_index; 1042 goto again; 1043 } 1044 /* All the sockets should already be down at this point, 1045 * we just want to make sure that DISCONNECTED is set so 1046 * any requests that come in that were queue'ed waiting 1047 * for the reconnect timer don't trigger the timer again 1048 * and instead just error out. 1049 */ 1050 sock_shutdown(nbd); 1051 nbd_config_put(nbd); 1052 return -EIO; 1053 } 1054 goto again; 1055 } 1056 1057 /* Handle the case that we have a pending request that was partially 1058 * transmitted that _has_ to be serviced first. We need to call requeue 1059 * here so that it gets put _after_ the request that is already on the 1060 * dispatch list. 1061 */ 1062 blk_mq_start_request(req); 1063 if (unlikely(nsock->pending && nsock->pending != req)) { 1064 nbd_requeue_cmd(cmd); 1065 ret = 0; 1066 goto out; 1067 } 1068 /* 1069 * Some failures are related to the link going down, so anything that 1070 * returns EAGAIN can be retried on a different socket. 1071 */ 1072 ret = nbd_send_cmd(nbd, cmd, index); 1073 /* 1074 * Access to this flag is protected by cmd->lock, thus it's safe to set 1075 * the flag after nbd_send_cmd() succeed to send request to server. 1076 */ 1077 if (!ret) 1078 __set_bit(NBD_CMD_INFLIGHT, &cmd->flags); 1079 else if (ret == -EAGAIN) { 1080 dev_err_ratelimited(disk_to_dev(nbd->disk), 1081 "Request send failed, requeueing\n"); 1082 nbd_mark_nsock_dead(nbd, nsock, 1); 1083 nbd_requeue_cmd(cmd); 1084 ret = 0; 1085 } 1086out: 1087 mutex_unlock(&nsock->tx_lock); 1088 nbd_config_put(nbd); 1089 return ret; 1090} 1091 1092static blk_status_t nbd_queue_rq(struct blk_mq_hw_ctx *hctx, 1093 const struct blk_mq_queue_data *bd) 1094{ 1095 struct nbd_cmd *cmd = blk_mq_rq_to_pdu(bd->rq); 1096 int ret; 1097 1098 /* 1099 * Since we look at the bio's to send the request over the network we 1100 * need to make sure the completion work doesn't mark this request done 1101 * before we are done doing our send. This keeps us from dereferencing 1102 * freed data if we have particularly fast completions (ie we get the 1103 * completion before we exit sock_xmit on the last bvec) or in the case 1104 * that the server is misbehaving (or there was an error) before we're 1105 * done sending everything over the wire. 1106 */ 1107 mutex_lock(&cmd->lock); 1108 clear_bit(NBD_CMD_REQUEUED, &cmd->flags); 1109 1110 /* We can be called directly from the user space process, which means we 1111 * could possibly have signals pending so our sendmsg will fail. In 1112 * this case we need to return that we are busy, otherwise error out as 1113 * appropriate. 1114 */ 1115 ret = nbd_handle_cmd(cmd, hctx->queue_num); 1116 if (ret < 0) 1117 ret = BLK_STS_IOERR; 1118 else if (!ret) 1119 ret = BLK_STS_OK; 1120 mutex_unlock(&cmd->lock); 1121 1122 return ret; 1123} 1124 1125static struct socket *nbd_get_socket(struct nbd_device *nbd, unsigned long fd, 1126 int *err) 1127{ 1128 struct socket *sock; 1129 1130 *err = 0; 1131 sock = sockfd_lookup(fd, err); 1132 if (!sock) 1133 return NULL; 1134 1135 if (sock->ops->shutdown == sock_no_shutdown) { 1136 dev_err(disk_to_dev(nbd->disk), "Unsupported socket: shutdown callout must be supported.\n"); 1137 *err = -EINVAL; 1138 sockfd_put(sock); 1139 return NULL; 1140 } 1141 1142 return sock; 1143} 1144 1145static int nbd_add_socket(struct nbd_device *nbd, unsigned long arg, 1146 bool netlink) 1147{ 1148 struct nbd_config *config = nbd->config; 1149 struct socket *sock; 1150 struct nbd_sock **socks; 1151 struct nbd_sock *nsock; 1152 int err; 1153 1154 /* Arg will be cast to int, check it to avoid overflow */ 1155 if (arg > INT_MAX) 1156 return -EINVAL; 1157 sock = nbd_get_socket(nbd, arg, &err); 1158 if (!sock) 1159 return err; 1160 1161 /* 1162 * We need to make sure we don't get any errant requests while we're 1163 * reallocating the ->socks array. 1164 */ 1165 blk_mq_freeze_queue(nbd->disk->queue); 1166 1167 if (!netlink && !nbd->task_setup && 1168 !test_bit(NBD_RT_BOUND, &config->runtime_flags)) 1169 nbd->task_setup = current; 1170 1171 if (!netlink && 1172 (nbd->task_setup != current || 1173 test_bit(NBD_RT_BOUND, &config->runtime_flags))) { 1174 dev_err(disk_to_dev(nbd->disk), 1175 "Device being setup by another task"); 1176 err = -EBUSY; 1177 goto put_socket; 1178 } 1179 1180 nsock = kzalloc(sizeof(*nsock), GFP_KERNEL); 1181 if (!nsock) { 1182 err = -ENOMEM; 1183 goto put_socket; 1184 } 1185 1186 socks = krealloc(config->socks, (config->num_connections + 1) * 1187 sizeof(struct nbd_sock *), GFP_KERNEL); 1188 if (!socks) { 1189 kfree(nsock); 1190 err = -ENOMEM; 1191 goto put_socket; 1192 } 1193 1194 config->socks = socks; 1195 1196 nsock->fallback_index = -1; 1197 nsock->dead = false; 1198 mutex_init(&nsock->tx_lock); 1199 nsock->sock = sock; 1200 nsock->pending = NULL; 1201 nsock->sent = 0; 1202 nsock->cookie = 0; 1203 socks[config->num_connections++] = nsock; 1204 atomic_inc(&config->live_connections); 1205 blk_mq_unfreeze_queue(nbd->disk->queue); 1206 1207 return 0; 1208 1209put_socket: 1210 blk_mq_unfreeze_queue(nbd->disk->queue); 1211 sockfd_put(sock); 1212 return err; 1213} 1214 1215static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg) 1216{ 1217 struct nbd_config *config = nbd->config; 1218 struct socket *sock, *old; 1219 struct recv_thread_args *args; 1220 int i; 1221 int err; 1222 1223 sock = nbd_get_socket(nbd, arg, &err); 1224 if (!sock) 1225 return err; 1226 1227 args = kzalloc(sizeof(*args), GFP_KERNEL); 1228 if (!args) { 1229 sockfd_put(sock); 1230 return -ENOMEM; 1231 } 1232 1233 for (i = 0; i < config->num_connections; i++) { 1234 struct nbd_sock *nsock = config->socks[i]; 1235 1236 if (!nsock->dead) 1237 continue; 1238 1239 mutex_lock(&nsock->tx_lock); 1240 if (!nsock->dead) { 1241 mutex_unlock(&nsock->tx_lock); 1242 continue; 1243 } 1244 sk_set_memalloc(sock->sk); 1245 if (nbd->tag_set.timeout) 1246 sock->sk->sk_sndtimeo = nbd->tag_set.timeout; 1247 atomic_inc(&config->recv_threads); 1248 refcount_inc(&nbd->config_refs); 1249 old = nsock->sock; 1250 nsock->fallback_index = -1; 1251 nsock->sock = sock; 1252 nsock->dead = false; 1253 INIT_WORK(&args->work, recv_work); 1254 args->index = i; 1255 args->nbd = nbd; 1256 args->nsock = nsock; 1257 nsock->cookie++; 1258 mutex_unlock(&nsock->tx_lock); 1259 sockfd_put(old); 1260 1261 clear_bit(NBD_RT_DISCONNECTED, &config->runtime_flags); 1262 1263 /* We take the tx_mutex in an error path in the recv_work, so we 1264 * need to queue_work outside of the tx_mutex. 1265 */ 1266 queue_work(nbd->recv_workq, &args->work); 1267 1268 atomic_inc(&config->live_connections); 1269 wake_up(&config->conn_wait); 1270 return 0; 1271 } 1272 sockfd_put(sock); 1273 kfree(args); 1274 return -ENOSPC; 1275} 1276 1277static void nbd_bdev_reset(struct nbd_device *nbd) 1278{ 1279 if (disk_openers(nbd->disk) > 1) 1280 return; 1281 set_capacity(nbd->disk, 0); 1282} 1283 1284static void nbd_parse_flags(struct nbd_device *nbd) 1285{ 1286 struct nbd_config *config = nbd->config; 1287 if (config->flags & NBD_FLAG_READ_ONLY) 1288 set_disk_ro(nbd->disk, true); 1289 else 1290 set_disk_ro(nbd->disk, false); 1291 if (config->flags & NBD_FLAG_SEND_FLUSH) { 1292 if (config->flags & NBD_FLAG_SEND_FUA) 1293 blk_queue_write_cache(nbd->disk->queue, true, true); 1294 else 1295 blk_queue_write_cache(nbd->disk->queue, true, false); 1296 } 1297 else 1298 blk_queue_write_cache(nbd->disk->queue, false, false); 1299} 1300 1301static void send_disconnects(struct nbd_device *nbd) 1302{ 1303 struct nbd_config *config = nbd->config; 1304 struct nbd_request request = { 1305 .magic = htonl(NBD_REQUEST_MAGIC), 1306 .type = htonl(NBD_CMD_DISC), 1307 }; 1308 struct kvec iov = {.iov_base = &request, .iov_len = sizeof(request)}; 1309 struct iov_iter from; 1310 int i, ret; 1311 1312 for (i = 0; i < config->num_connections; i++) { 1313 struct nbd_sock *nsock = config->socks[i]; 1314 1315 iov_iter_kvec(&from, ITER_SOURCE, &iov, 1, sizeof(request)); 1316 mutex_lock(&nsock->tx_lock); 1317 ret = sock_xmit(nbd, i, 1, &from, 0, NULL); 1318 if (ret < 0) 1319 dev_err(disk_to_dev(nbd->disk), 1320 "Send disconnect failed %d\n", ret); 1321 mutex_unlock(&nsock->tx_lock); 1322 } 1323} 1324 1325static int nbd_disconnect(struct nbd_device *nbd) 1326{ 1327 struct nbd_config *config = nbd->config; 1328 1329 dev_info(disk_to_dev(nbd->disk), "NBD_DISCONNECT\n"); 1330 set_bit(NBD_RT_DISCONNECT_REQUESTED, &config->runtime_flags); 1331 set_bit(NBD_DISCONNECT_REQUESTED, &nbd->flags); 1332 send_disconnects(nbd); 1333 return 0; 1334} 1335 1336static void nbd_clear_sock(struct nbd_device *nbd) 1337{ 1338 sock_shutdown(nbd); 1339 nbd_clear_que(nbd); 1340 nbd->task_setup = NULL; 1341} 1342 1343static void nbd_config_put(struct nbd_device *nbd) 1344{ 1345 if (refcount_dec_and_mutex_lock(&nbd->config_refs, 1346 &nbd->config_lock)) { 1347 struct nbd_config *config = nbd->config; 1348 nbd_dev_dbg_close(nbd); 1349 invalidate_disk(nbd->disk); 1350 if (nbd->config->bytesize) 1351 kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE); 1352 if (test_and_clear_bit(NBD_RT_HAS_PID_FILE, 1353 &config->runtime_flags)) 1354 device_remove_file(disk_to_dev(nbd->disk), &pid_attr); 1355 nbd->pid = 0; 1356 if (test_and_clear_bit(NBD_RT_HAS_BACKEND_FILE, 1357 &config->runtime_flags)) { 1358 device_remove_file(disk_to_dev(nbd->disk), &backend_attr); 1359 kfree(nbd->backend); 1360 nbd->backend = NULL; 1361 } 1362 nbd_clear_sock(nbd); 1363 if (config->num_connections) { 1364 int i; 1365 for (i = 0; i < config->num_connections; i++) { 1366 sockfd_put(config->socks[i]->sock); 1367 kfree(config->socks[i]); 1368 } 1369 kfree(config->socks); 1370 } 1371 kfree(nbd->config); 1372 nbd->config = NULL; 1373 1374 nbd->tag_set.timeout = 0; 1375 1376 mutex_unlock(&nbd->config_lock); 1377 nbd_put(nbd); 1378 module_put(THIS_MODULE); 1379 } 1380} 1381 1382static int nbd_start_device(struct nbd_device *nbd) 1383{ 1384 struct nbd_config *config = nbd->config; 1385 int num_connections = config->num_connections; 1386 int error = 0, i; 1387 1388 if (nbd->pid) 1389 return -EBUSY; 1390 if (!config->socks) 1391 return -EINVAL; 1392 if (num_connections > 1 && 1393 !(config->flags & NBD_FLAG_CAN_MULTI_CONN)) { 1394 dev_err(disk_to_dev(nbd->disk), "server does not support multiple connections per device.\n"); 1395 return -EINVAL; 1396 } 1397 1398 blk_mq_update_nr_hw_queues(&nbd->tag_set, config->num_connections); 1399 nbd->pid = task_pid_nr(current); 1400 1401 nbd_parse_flags(nbd); 1402 1403 error = device_create_file(disk_to_dev(nbd->disk), &pid_attr); 1404 if (error) { 1405 dev_err(disk_to_dev(nbd->disk), "device_create_file failed for pid!\n"); 1406 return error; 1407 } 1408 set_bit(NBD_RT_HAS_PID_FILE, &config->runtime_flags); 1409 1410 nbd_dev_dbg_init(nbd); 1411 for (i = 0; i < num_connections; i++) { 1412 struct recv_thread_args *args; 1413 1414 args = kzalloc(sizeof(*args), GFP_KERNEL); 1415 if (!args) { 1416 sock_shutdown(nbd); 1417 /* 1418 * If num_connections is m (2 < m), 1419 * and NO.1 ~ NO.n(1 < n < m) kzallocs are successful. 1420 * But NO.(n + 1) failed. We still have n recv threads. 1421 * So, add flush_workqueue here to prevent recv threads 1422 * dropping the last config_refs and trying to destroy 1423 * the workqueue from inside the workqueue. 1424 */ 1425 if (i) 1426 flush_workqueue(nbd->recv_workq); 1427 return -ENOMEM; 1428 } 1429 sk_set_memalloc(config->socks[i]->sock->sk); 1430 if (nbd->tag_set.timeout) 1431 config->socks[i]->sock->sk->sk_sndtimeo = 1432 nbd->tag_set.timeout; 1433 atomic_inc(&config->recv_threads); 1434 refcount_inc(&nbd->config_refs); 1435 INIT_WORK(&args->work, recv_work); 1436 args->nbd = nbd; 1437 args->nsock = config->socks[i]; 1438 args->index = i; 1439 queue_work(nbd->recv_workq, &args->work); 1440 } 1441 return nbd_set_size(nbd, config->bytesize, nbd_blksize(config)); 1442} 1443 1444static int nbd_start_device_ioctl(struct nbd_device *nbd) 1445{ 1446 struct nbd_config *config = nbd->config; 1447 int ret; 1448 1449 ret = nbd_start_device(nbd); 1450 if (ret) 1451 return ret; 1452 1453 if (max_part) 1454 set_bit(GD_NEED_PART_SCAN, &nbd->disk->state); 1455 mutex_unlock(&nbd->config_lock); 1456 ret = wait_event_interruptible(config->recv_wq, 1457 atomic_read(&config->recv_threads) == 0); 1458 if (ret) { 1459 sock_shutdown(nbd); 1460 nbd_clear_que(nbd); 1461 } 1462 1463 flush_workqueue(nbd->recv_workq); 1464 mutex_lock(&nbd->config_lock); 1465 nbd_bdev_reset(nbd); 1466 /* user requested, ignore socket errors */ 1467 if (test_bit(NBD_RT_DISCONNECT_REQUESTED, &config->runtime_flags)) 1468 ret = 0; 1469 if (test_bit(NBD_RT_TIMEDOUT, &config->runtime_flags)) 1470 ret = -ETIMEDOUT; 1471 return ret; 1472} 1473 1474static void nbd_clear_sock_ioctl(struct nbd_device *nbd) 1475{ 1476 nbd_clear_sock(nbd); 1477 disk_force_media_change(nbd->disk); 1478 nbd_bdev_reset(nbd); 1479 if (test_and_clear_bit(NBD_RT_HAS_CONFIG_REF, 1480 &nbd->config->runtime_flags)) 1481 nbd_config_put(nbd); 1482} 1483 1484static void nbd_set_cmd_timeout(struct nbd_device *nbd, u64 timeout) 1485{ 1486 nbd->tag_set.timeout = timeout * HZ; 1487 if (timeout) 1488 blk_queue_rq_timeout(nbd->disk->queue, timeout * HZ); 1489 else 1490 blk_queue_rq_timeout(nbd->disk->queue, 30 * HZ); 1491} 1492 1493/* Must be called with config_lock held */ 1494static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd, 1495 unsigned int cmd, unsigned long arg) 1496{ 1497 struct nbd_config *config = nbd->config; 1498 loff_t bytesize; 1499 1500 switch (cmd) { 1501 case NBD_DISCONNECT: 1502 return nbd_disconnect(nbd); 1503 case NBD_CLEAR_SOCK: 1504 nbd_clear_sock_ioctl(nbd); 1505 return 0; 1506 case NBD_SET_SOCK: 1507 return nbd_add_socket(nbd, arg, false); 1508 case NBD_SET_BLKSIZE: 1509 return nbd_set_size(nbd, config->bytesize, arg); 1510 case NBD_SET_SIZE: 1511 return nbd_set_size(nbd, arg, nbd_blksize(config)); 1512 case NBD_SET_SIZE_BLOCKS: 1513 if (check_shl_overflow(arg, config->blksize_bits, &bytesize)) 1514 return -EINVAL; 1515 return nbd_set_size(nbd, bytesize, nbd_blksize(config)); 1516 case NBD_SET_TIMEOUT: 1517 nbd_set_cmd_timeout(nbd, arg); 1518 return 0; 1519 1520 case NBD_SET_FLAGS: 1521 config->flags = arg; 1522 return 0; 1523 case NBD_DO_IT: 1524 return nbd_start_device_ioctl(nbd); 1525 case NBD_CLEAR_QUE: 1526 /* 1527 * This is for compatibility only. The queue is always cleared 1528 * by NBD_DO_IT or NBD_CLEAR_SOCK. 1529 */ 1530 return 0; 1531 case NBD_PRINT_DEBUG: 1532 /* 1533 * For compatibility only, we no longer keep a list of 1534 * outstanding requests. 1535 */ 1536 return 0; 1537 } 1538 return -ENOTTY; 1539} 1540 1541static int nbd_ioctl(struct block_device *bdev, blk_mode_t mode, 1542 unsigned int cmd, unsigned long arg) 1543{ 1544 struct nbd_device *nbd = bdev->bd_disk->private_data; 1545 struct nbd_config *config = nbd->config; 1546 int error = -EINVAL; 1547 1548 if (!capable(CAP_SYS_ADMIN)) 1549 return -EPERM; 1550 1551 /* The block layer will pass back some non-nbd ioctls in case we have 1552 * special handling for them, but we don't so just return an error. 1553 */ 1554 if (_IOC_TYPE(cmd) != 0xab) 1555 return -EINVAL; 1556 1557 mutex_lock(&nbd->config_lock); 1558 1559 /* Don't allow ioctl operations on a nbd device that was created with 1560 * netlink, unless it's DISCONNECT or CLEAR_SOCK, which are fine. 1561 */ 1562 if (!test_bit(NBD_RT_BOUND, &config->runtime_flags) || 1563 (cmd == NBD_DISCONNECT || cmd == NBD_CLEAR_SOCK)) 1564 error = __nbd_ioctl(bdev, nbd, cmd, arg); 1565 else 1566 dev_err(nbd_to_dev(nbd), "Cannot use ioctl interface on a netlink controlled device.\n"); 1567 mutex_unlock(&nbd->config_lock); 1568 return error; 1569} 1570 1571static int nbd_alloc_and_init_config(struct nbd_device *nbd) 1572{ 1573 struct nbd_config *config; 1574 1575 if (WARN_ON(nbd->config)) 1576 return -EINVAL; 1577 1578 if (!try_module_get(THIS_MODULE)) 1579 return -ENODEV; 1580 1581 config = kzalloc(sizeof(struct nbd_config), GFP_NOFS); 1582 if (!config) { 1583 module_put(THIS_MODULE); 1584 return -ENOMEM; 1585 } 1586 1587 atomic_set(&config->recv_threads, 0); 1588 init_waitqueue_head(&config->recv_wq); 1589 init_waitqueue_head(&config->conn_wait); 1590 config->blksize_bits = NBD_DEF_BLKSIZE_BITS; 1591 atomic_set(&config->live_connections, 0); 1592 1593 nbd->config = config; 1594 /* 1595 * Order refcount_set(&nbd->config_refs, 1) and nbd->config assignment, 1596 * its pair is the barrier in nbd_get_config_unlocked(). 1597 * So nbd_get_config_unlocked() won't see nbd->config as null after 1598 * refcount_inc_not_zero() succeed. 1599 */ 1600 smp_mb__before_atomic(); 1601 refcount_set(&nbd->config_refs, 1); 1602 1603 return 0; 1604} 1605 1606static int nbd_open(struct gendisk *disk, blk_mode_t mode) 1607{ 1608 struct nbd_device *nbd; 1609 struct nbd_config *config; 1610 int ret = 0; 1611 1612 mutex_lock(&nbd_index_mutex); 1613 nbd = disk->private_data; 1614 if (!nbd) { 1615 ret = -ENXIO; 1616 goto out; 1617 } 1618 if (!refcount_inc_not_zero(&nbd->refs)) { 1619 ret = -ENXIO; 1620 goto out; 1621 } 1622 1623 config = nbd_get_config_unlocked(nbd); 1624 if (!config) { 1625 mutex_lock(&nbd->config_lock); 1626 if (refcount_inc_not_zero(&nbd->config_refs)) { 1627 mutex_unlock(&nbd->config_lock); 1628 goto out; 1629 } 1630 ret = nbd_alloc_and_init_config(nbd); 1631 if (ret) { 1632 mutex_unlock(&nbd->config_lock); 1633 goto out; 1634 } 1635 1636 refcount_inc(&nbd->refs); 1637 mutex_unlock(&nbd->config_lock); 1638 if (max_part) 1639 set_bit(GD_NEED_PART_SCAN, &disk->state); 1640 } else if (nbd_disconnected(config)) { 1641 if (max_part) 1642 set_bit(GD_NEED_PART_SCAN, &disk->state); 1643 } 1644out: 1645 mutex_unlock(&nbd_index_mutex); 1646 return ret; 1647} 1648 1649static void nbd_release(struct gendisk *disk) 1650{ 1651 struct nbd_device *nbd = disk->private_data; 1652 1653 if (test_bit(NBD_RT_DISCONNECT_ON_CLOSE, &nbd->config->runtime_flags) && 1654 disk_openers(disk) == 0) 1655 nbd_disconnect_and_put(nbd); 1656 1657 nbd_config_put(nbd); 1658 nbd_put(nbd); 1659} 1660 1661static void nbd_free_disk(struct gendisk *disk) 1662{ 1663 struct nbd_device *nbd = disk->private_data; 1664 1665 kfree(nbd); 1666} 1667 1668static const struct block_device_operations nbd_fops = 1669{ 1670 .owner = THIS_MODULE, 1671 .open = nbd_open, 1672 .release = nbd_release, 1673 .ioctl = nbd_ioctl, 1674 .compat_ioctl = nbd_ioctl, 1675 .free_disk = nbd_free_disk, 1676}; 1677 1678#if IS_ENABLED(CONFIG_DEBUG_FS) 1679 1680static int nbd_dbg_tasks_show(struct seq_file *s, void *unused) 1681{ 1682 struct nbd_device *nbd = s->private; 1683 1684 if (nbd->pid) 1685 seq_printf(s, "recv: %d\n", nbd->pid); 1686 1687 return 0; 1688} 1689 1690DEFINE_SHOW_ATTRIBUTE(nbd_dbg_tasks); 1691 1692static int nbd_dbg_flags_show(struct seq_file *s, void *unused) 1693{ 1694 struct nbd_device *nbd = s->private; 1695 u32 flags = nbd->config->flags; 1696 1697 seq_printf(s, "Hex: 0x%08x\n\n", flags); 1698 1699 seq_puts(s, "Known flags:\n"); 1700 1701 if (flags & NBD_FLAG_HAS_FLAGS) 1702 seq_puts(s, "NBD_FLAG_HAS_FLAGS\n"); 1703 if (flags & NBD_FLAG_READ_ONLY) 1704 seq_puts(s, "NBD_FLAG_READ_ONLY\n"); 1705 if (flags & NBD_FLAG_SEND_FLUSH) 1706 seq_puts(s, "NBD_FLAG_SEND_FLUSH\n"); 1707 if (flags & NBD_FLAG_SEND_FUA) 1708 seq_puts(s, "NBD_FLAG_SEND_FUA\n"); 1709 if (flags & NBD_FLAG_SEND_TRIM) 1710 seq_puts(s, "NBD_FLAG_SEND_TRIM\n"); 1711 1712 return 0; 1713} 1714 1715DEFINE_SHOW_ATTRIBUTE(nbd_dbg_flags); 1716 1717static int nbd_dev_dbg_init(struct nbd_device *nbd) 1718{ 1719 struct dentry *dir; 1720 struct nbd_config *config = nbd->config; 1721 1722 if (!nbd_dbg_dir) 1723 return -EIO; 1724 1725 dir = debugfs_create_dir(nbd_name(nbd), nbd_dbg_dir); 1726 if (IS_ERR(dir)) { 1727 dev_err(nbd_to_dev(nbd), "Failed to create debugfs dir for '%s'\n", 1728 nbd_name(nbd)); 1729 return -EIO; 1730 } 1731 config->dbg_dir = dir; 1732 1733 debugfs_create_file("tasks", 0444, dir, nbd, &nbd_dbg_tasks_fops); 1734 debugfs_create_u64("size_bytes", 0444, dir, &config->bytesize); 1735 debugfs_create_u32("timeout", 0444, dir, &nbd->tag_set.timeout); 1736 debugfs_create_u32("blocksize_bits", 0444, dir, &config->blksize_bits); 1737 debugfs_create_file("flags", 0444, dir, nbd, &nbd_dbg_flags_fops); 1738 1739 return 0; 1740} 1741 1742static void nbd_dev_dbg_close(struct nbd_device *nbd) 1743{ 1744 debugfs_remove_recursive(nbd->config->dbg_dir); 1745} 1746 1747static int nbd_dbg_init(void) 1748{ 1749 struct dentry *dbg_dir; 1750 1751 dbg_dir = debugfs_create_dir("nbd", NULL); 1752 if (IS_ERR(dbg_dir)) 1753 return -EIO; 1754 1755 nbd_dbg_dir = dbg_dir; 1756 1757 return 0; 1758} 1759 1760static void nbd_dbg_close(void) 1761{ 1762 debugfs_remove_recursive(nbd_dbg_dir); 1763} 1764 1765#else /* IS_ENABLED(CONFIG_DEBUG_FS) */ 1766 1767static int nbd_dev_dbg_init(struct nbd_device *nbd) 1768{ 1769 return 0; 1770} 1771 1772static void nbd_dev_dbg_close(struct nbd_device *nbd) 1773{ 1774} 1775 1776static int nbd_dbg_init(void) 1777{ 1778 return 0; 1779} 1780 1781static void nbd_dbg_close(void) 1782{ 1783} 1784 1785#endif 1786 1787static int nbd_init_request(struct blk_mq_tag_set *set, struct request *rq, 1788 unsigned int hctx_idx, unsigned int numa_node) 1789{ 1790 struct nbd_cmd *cmd = blk_mq_rq_to_pdu(rq); 1791 cmd->nbd = set->driver_data; 1792 cmd->flags = 0; 1793 mutex_init(&cmd->lock); 1794 return 0; 1795} 1796 1797static const struct blk_mq_ops nbd_mq_ops = { 1798 .queue_rq = nbd_queue_rq, 1799 .complete = nbd_complete_rq, 1800 .init_request = nbd_init_request, 1801 .timeout = nbd_xmit_timeout, 1802}; 1803 1804static struct nbd_device *nbd_dev_add(int index, unsigned int refs) 1805{ 1806 struct queue_limits lim = { 1807 .max_hw_sectors = 65536, 1808 .max_user_sectors = 256, 1809 .max_segments = USHRT_MAX, 1810 .max_segment_size = UINT_MAX, 1811 }; 1812 struct nbd_device *nbd; 1813 struct gendisk *disk; 1814 int err = -ENOMEM; 1815 1816 nbd = kzalloc(sizeof(struct nbd_device), GFP_KERNEL); 1817 if (!nbd) 1818 goto out; 1819 1820 nbd->tag_set.ops = &nbd_mq_ops; 1821 nbd->tag_set.nr_hw_queues = 1; 1822 nbd->tag_set.queue_depth = 128; 1823 nbd->tag_set.numa_node = NUMA_NO_NODE; 1824 nbd->tag_set.cmd_size = sizeof(struct nbd_cmd); 1825 nbd->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | 1826 BLK_MQ_F_BLOCKING; 1827 nbd->tag_set.driver_data = nbd; 1828 INIT_WORK(&nbd->remove_work, nbd_dev_remove_work); 1829 nbd->backend = NULL; 1830 1831 err = blk_mq_alloc_tag_set(&nbd->tag_set); 1832 if (err) 1833 goto out_free_nbd; 1834 1835 mutex_lock(&nbd_index_mutex); 1836 if (index >= 0) { 1837 err = idr_alloc(&nbd_index_idr, nbd, index, index + 1, 1838 GFP_KERNEL); 1839 if (err == -ENOSPC) 1840 err = -EEXIST; 1841 } else { 1842 err = idr_alloc(&nbd_index_idr, nbd, 0, 1843 (MINORMASK >> part_shift) + 1, GFP_KERNEL); 1844 if (err >= 0) 1845 index = err; 1846 } 1847 nbd->index = index; 1848 mutex_unlock(&nbd_index_mutex); 1849 if (err < 0) 1850 goto out_free_tags; 1851 1852 disk = blk_mq_alloc_disk(&nbd->tag_set, &lim, NULL); 1853 if (IS_ERR(disk)) { 1854 err = PTR_ERR(disk); 1855 goto out_free_idr; 1856 } 1857 nbd->disk = disk; 1858 1859 nbd->recv_workq = alloc_workqueue("nbd%d-recv", 1860 WQ_MEM_RECLAIM | WQ_HIGHPRI | 1861 WQ_UNBOUND, 0, nbd->index); 1862 if (!nbd->recv_workq) { 1863 dev_err(disk_to_dev(nbd->disk), "Could not allocate knbd recv work queue.\n"); 1864 err = -ENOMEM; 1865 goto out_err_disk; 1866 } 1867 1868 /* 1869 * Tell the block layer that we are not a rotational device 1870 */ 1871 blk_queue_flag_set(QUEUE_FLAG_NONROT, disk->queue); 1872 1873 mutex_init(&nbd->config_lock); 1874 refcount_set(&nbd->config_refs, 0); 1875 /* 1876 * Start out with a zero references to keep other threads from using 1877 * this device until it is fully initialized. 1878 */ 1879 refcount_set(&nbd->refs, 0); 1880 INIT_LIST_HEAD(&nbd->list); 1881 disk->major = NBD_MAJOR; 1882 disk->first_minor = index << part_shift; 1883 disk->minors = 1 << part_shift; 1884 disk->fops = &nbd_fops; 1885 disk->private_data = nbd; 1886 sprintf(disk->disk_name, "nbd%d", index); 1887 err = add_disk(disk); 1888 if (err) 1889 goto out_free_work; 1890 1891 /* 1892 * Now publish the device. 1893 */ 1894 refcount_set(&nbd->refs, refs); 1895 nbd_total_devices++; 1896 return nbd; 1897 1898out_free_work: 1899 destroy_workqueue(nbd->recv_workq); 1900out_err_disk: 1901 put_disk(disk); 1902out_free_idr: 1903 mutex_lock(&nbd_index_mutex); 1904 idr_remove(&nbd_index_idr, index); 1905 mutex_unlock(&nbd_index_mutex); 1906out_free_tags: 1907 blk_mq_free_tag_set(&nbd->tag_set); 1908out_free_nbd: 1909 kfree(nbd); 1910out: 1911 return ERR_PTR(err); 1912} 1913 1914static struct nbd_device *nbd_find_get_unused(void) 1915{ 1916 struct nbd_device *nbd; 1917 int id; 1918 1919 lockdep_assert_held(&nbd_index_mutex); 1920 1921 idr_for_each_entry(&nbd_index_idr, nbd, id) { 1922 if (refcount_read(&nbd->config_refs) || 1923 test_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags)) 1924 continue; 1925 if (refcount_inc_not_zero(&nbd->refs)) 1926 return nbd; 1927 } 1928 1929 return NULL; 1930} 1931 1932/* Netlink interface. */ 1933static const struct nla_policy nbd_attr_policy[NBD_ATTR_MAX + 1] = { 1934 [NBD_ATTR_INDEX] = { .type = NLA_U32 }, 1935 [NBD_ATTR_SIZE_BYTES] = { .type = NLA_U64 }, 1936 [NBD_ATTR_BLOCK_SIZE_BYTES] = { .type = NLA_U64 }, 1937 [NBD_ATTR_TIMEOUT] = { .type = NLA_U64 }, 1938 [NBD_ATTR_SERVER_FLAGS] = { .type = NLA_U64 }, 1939 [NBD_ATTR_CLIENT_FLAGS] = { .type = NLA_U64 }, 1940 [NBD_ATTR_SOCKETS] = { .type = NLA_NESTED}, 1941 [NBD_ATTR_DEAD_CONN_TIMEOUT] = { .type = NLA_U64 }, 1942 [NBD_ATTR_DEVICE_LIST] = { .type = NLA_NESTED}, 1943 [NBD_ATTR_BACKEND_IDENTIFIER] = { .type = NLA_STRING}, 1944}; 1945 1946static const struct nla_policy nbd_sock_policy[NBD_SOCK_MAX + 1] = { 1947 [NBD_SOCK_FD] = { .type = NLA_U32 }, 1948}; 1949 1950/* We don't use this right now since we don't parse the incoming list, but we 1951 * still want it here so userspace knows what to expect. 1952 */ 1953static const struct nla_policy __attribute__((unused)) 1954nbd_device_policy[NBD_DEVICE_ATTR_MAX + 1] = { 1955 [NBD_DEVICE_INDEX] = { .type = NLA_U32 }, 1956 [NBD_DEVICE_CONNECTED] = { .type = NLA_U8 }, 1957}; 1958 1959static int nbd_genl_size_set(struct genl_info *info, struct nbd_device *nbd) 1960{ 1961 struct nbd_config *config = nbd->config; 1962 u64 bsize = nbd_blksize(config); 1963 u64 bytes = config->bytesize; 1964 1965 if (info->attrs[NBD_ATTR_SIZE_BYTES]) 1966 bytes = nla_get_u64(info->attrs[NBD_ATTR_SIZE_BYTES]); 1967 1968 if (info->attrs[NBD_ATTR_BLOCK_SIZE_BYTES]) 1969 bsize = nla_get_u64(info->attrs[NBD_ATTR_BLOCK_SIZE_BYTES]); 1970 1971 if (bytes != config->bytesize || bsize != nbd_blksize(config)) 1972 return nbd_set_size(nbd, bytes, bsize); 1973 return 0; 1974} 1975 1976static int nbd_genl_connect(struct sk_buff *skb, struct genl_info *info) 1977{ 1978 struct nbd_device *nbd; 1979 struct nbd_config *config; 1980 int index = -1; 1981 int ret; 1982 bool put_dev = false; 1983 1984 if (!netlink_capable(skb, CAP_SYS_ADMIN)) 1985 return -EPERM; 1986 1987 if (info->attrs[NBD_ATTR_INDEX]) { 1988 index = nla_get_u32(info->attrs[NBD_ATTR_INDEX]); 1989 1990 /* 1991 * Too big first_minor can cause duplicate creation of 1992 * sysfs files/links, since index << part_shift might overflow, or 1993 * MKDEV() expect that the max bits of first_minor is 20. 1994 */ 1995 if (index < 0 || index > MINORMASK >> part_shift) { 1996 pr_err("illegal input index %d\n", index); 1997 return -EINVAL; 1998 } 1999 } 2000 if (GENL_REQ_ATTR_CHECK(info, NBD_ATTR_SOCKETS)) { 2001 pr_err("must specify at least one socket\n"); 2002 return -EINVAL; 2003 } 2004 if (GENL_REQ_ATTR_CHECK(info, NBD_ATTR_SIZE_BYTES)) { 2005 pr_err("must specify a size in bytes for the device\n"); 2006 return -EINVAL; 2007 } 2008again: 2009 mutex_lock(&nbd_index_mutex); 2010 if (index == -1) { 2011 nbd = nbd_find_get_unused(); 2012 } else { 2013 nbd = idr_find(&nbd_index_idr, index); 2014 if (nbd) { 2015 if ((test_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags) && 2016 test_bit(NBD_DISCONNECT_REQUESTED, &nbd->flags)) || 2017 !refcount_inc_not_zero(&nbd->refs)) { 2018 mutex_unlock(&nbd_index_mutex); 2019 pr_err("device at index %d is going down\n", 2020 index); 2021 return -EINVAL; 2022 } 2023 } 2024 } 2025 mutex_unlock(&nbd_index_mutex); 2026 2027 if (!nbd) { 2028 nbd = nbd_dev_add(index, 2); 2029 if (IS_ERR(nbd)) { 2030 pr_err("failed to add new device\n"); 2031 return PTR_ERR(nbd); 2032 } 2033 } 2034 2035 mutex_lock(&nbd->config_lock); 2036 if (refcount_read(&nbd->config_refs)) { 2037 mutex_unlock(&nbd->config_lock); 2038 nbd_put(nbd); 2039 if (index == -1) 2040 goto again; 2041 pr_err("nbd%d already in use\n", index); 2042 return -EBUSY; 2043 } 2044 2045 ret = nbd_alloc_and_init_config(nbd); 2046 if (ret) { 2047 mutex_unlock(&nbd->config_lock); 2048 nbd_put(nbd); 2049 pr_err("couldn't allocate config\n"); 2050 return ret; 2051 } 2052 2053 config = nbd->config; 2054 set_bit(NBD_RT_BOUND, &config->runtime_flags); 2055 ret = nbd_genl_size_set(info, nbd); 2056 if (ret) 2057 goto out; 2058 2059 if (info->attrs[NBD_ATTR_TIMEOUT]) 2060 nbd_set_cmd_timeout(nbd, 2061 nla_get_u64(info->attrs[NBD_ATTR_TIMEOUT])); 2062 if (info->attrs[NBD_ATTR_DEAD_CONN_TIMEOUT]) { 2063 config->dead_conn_timeout = 2064 nla_get_u64(info->attrs[NBD_ATTR_DEAD_CONN_TIMEOUT]); 2065 config->dead_conn_timeout *= HZ; 2066 } 2067 if (info->attrs[NBD_ATTR_SERVER_FLAGS]) 2068 config->flags = 2069 nla_get_u64(info->attrs[NBD_ATTR_SERVER_FLAGS]); 2070 if (info->attrs[NBD_ATTR_CLIENT_FLAGS]) { 2071 u64 flags = nla_get_u64(info->attrs[NBD_ATTR_CLIENT_FLAGS]); 2072 if (flags & NBD_CFLAG_DESTROY_ON_DISCONNECT) { 2073 /* 2074 * We have 1 ref to keep the device around, and then 1 2075 * ref for our current operation here, which will be 2076 * inherited by the config. If we already have 2077 * DESTROY_ON_DISCONNECT set then we know we don't have 2078 * that extra ref already held so we don't need the 2079 * put_dev. 2080 */ 2081 if (!test_and_set_bit(NBD_DESTROY_ON_DISCONNECT, 2082 &nbd->flags)) 2083 put_dev = true; 2084 } else { 2085 if (test_and_clear_bit(NBD_DESTROY_ON_DISCONNECT, 2086 &nbd->flags)) 2087 refcount_inc(&nbd->refs); 2088 } 2089 if (flags & NBD_CFLAG_DISCONNECT_ON_CLOSE) { 2090 set_bit(NBD_RT_DISCONNECT_ON_CLOSE, 2091 &config->runtime_flags); 2092 } 2093 } 2094 2095 if (info->attrs[NBD_ATTR_SOCKETS]) { 2096 struct nlattr *attr; 2097 int rem, fd; 2098 2099 nla_for_each_nested(attr, info->attrs[NBD_ATTR_SOCKETS], 2100 rem) { 2101 struct nlattr *socks[NBD_SOCK_MAX+1]; 2102 2103 if (nla_type(attr) != NBD_SOCK_ITEM) { 2104 pr_err("socks must be embedded in a SOCK_ITEM attr\n"); 2105 ret = -EINVAL; 2106 goto out; 2107 } 2108 ret = nla_parse_nested_deprecated(socks, NBD_SOCK_MAX, 2109 attr, 2110 nbd_sock_policy, 2111 info->extack); 2112 if (ret != 0) { 2113 pr_err("error processing sock list\n"); 2114 ret = -EINVAL; 2115 goto out; 2116 } 2117 if (!socks[NBD_SOCK_FD]) 2118 continue; 2119 fd = (int)nla_get_u32(socks[NBD_SOCK_FD]); 2120 ret = nbd_add_socket(nbd, fd, true); 2121 if (ret) 2122 goto out; 2123 } 2124 } 2125 ret = nbd_start_device(nbd); 2126 if (ret) 2127 goto out; 2128 if (info->attrs[NBD_ATTR_BACKEND_IDENTIFIER]) { 2129 nbd->backend = nla_strdup(info->attrs[NBD_ATTR_BACKEND_IDENTIFIER], 2130 GFP_KERNEL); 2131 if (!nbd->backend) { 2132 ret = -ENOMEM; 2133 goto out; 2134 } 2135 } 2136 ret = device_create_file(disk_to_dev(nbd->disk), &backend_attr); 2137 if (ret) { 2138 dev_err(disk_to_dev(nbd->disk), 2139 "device_create_file failed for backend!\n"); 2140 goto out; 2141 } 2142 set_bit(NBD_RT_HAS_BACKEND_FILE, &config->runtime_flags); 2143out: 2144 mutex_unlock(&nbd->config_lock); 2145 if (!ret) { 2146 set_bit(NBD_RT_HAS_CONFIG_REF, &config->runtime_flags); 2147 refcount_inc(&nbd->config_refs); 2148 nbd_connect_reply(info, nbd->index); 2149 } 2150 nbd_config_put(nbd); 2151 if (put_dev) 2152 nbd_put(nbd); 2153 return ret; 2154} 2155 2156static void nbd_disconnect_and_put(struct nbd_device *nbd) 2157{ 2158 mutex_lock(&nbd->config_lock); 2159 nbd_disconnect(nbd); 2160 sock_shutdown(nbd); 2161 wake_up(&nbd->config->conn_wait); 2162 /* 2163 * Make sure recv thread has finished, we can safely call nbd_clear_que() 2164 * to cancel the inflight I/Os. 2165 */ 2166 flush_workqueue(nbd->recv_workq); 2167 nbd_clear_que(nbd); 2168 nbd->task_setup = NULL; 2169 mutex_unlock(&nbd->config_lock); 2170 2171 if (test_and_clear_bit(NBD_RT_HAS_CONFIG_REF, 2172 &nbd->config->runtime_flags)) 2173 nbd_config_put(nbd); 2174} 2175 2176static int nbd_genl_disconnect(struct sk_buff *skb, struct genl_info *info) 2177{ 2178 struct nbd_device *nbd; 2179 int index; 2180 2181 if (!netlink_capable(skb, CAP_SYS_ADMIN)) 2182 return -EPERM; 2183 2184 if (GENL_REQ_ATTR_CHECK(info, NBD_ATTR_INDEX)) { 2185 pr_err("must specify an index to disconnect\n"); 2186 return -EINVAL; 2187 } 2188 index = nla_get_u32(info->attrs[NBD_ATTR_INDEX]); 2189 mutex_lock(&nbd_index_mutex); 2190 nbd = idr_find(&nbd_index_idr, index); 2191 if (!nbd) { 2192 mutex_unlock(&nbd_index_mutex); 2193 pr_err("couldn't find device at index %d\n", index); 2194 return -EINVAL; 2195 } 2196 if (!refcount_inc_not_zero(&nbd->refs)) { 2197 mutex_unlock(&nbd_index_mutex); 2198 pr_err("device at index %d is going down\n", index); 2199 return -EINVAL; 2200 } 2201 mutex_unlock(&nbd_index_mutex); 2202 if (!refcount_inc_not_zero(&nbd->config_refs)) 2203 goto put_nbd; 2204 nbd_disconnect_and_put(nbd); 2205 nbd_config_put(nbd); 2206put_nbd: 2207 nbd_put(nbd); 2208 return 0; 2209} 2210 2211static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info) 2212{ 2213 struct nbd_device *nbd = NULL; 2214 struct nbd_config *config; 2215 int index; 2216 int ret = 0; 2217 bool put_dev = false; 2218 2219 if (!netlink_capable(skb, CAP_SYS_ADMIN)) 2220 return -EPERM; 2221 2222 if (GENL_REQ_ATTR_CHECK(info, NBD_ATTR_INDEX)) { 2223 pr_err("must specify a device to reconfigure\n"); 2224 return -EINVAL; 2225 } 2226 index = nla_get_u32(info->attrs[NBD_ATTR_INDEX]); 2227 mutex_lock(&nbd_index_mutex); 2228 nbd = idr_find(&nbd_index_idr, index); 2229 if (!nbd) { 2230 mutex_unlock(&nbd_index_mutex); 2231 pr_err("couldn't find a device at index %d\n", index); 2232 return -EINVAL; 2233 } 2234 if (nbd->backend) { 2235 if (info->attrs[NBD_ATTR_BACKEND_IDENTIFIER]) { 2236 if (nla_strcmp(info->attrs[NBD_ATTR_BACKEND_IDENTIFIER], 2237 nbd->backend)) { 2238 mutex_unlock(&nbd_index_mutex); 2239 dev_err(nbd_to_dev(nbd), 2240 "backend image doesn't match with %s\n", 2241 nbd->backend); 2242 return -EINVAL; 2243 } 2244 } else { 2245 mutex_unlock(&nbd_index_mutex); 2246 dev_err(nbd_to_dev(nbd), "must specify backend\n"); 2247 return -EINVAL; 2248 } 2249 } 2250 if (!refcount_inc_not_zero(&nbd->refs)) { 2251 mutex_unlock(&nbd_index_mutex); 2252 pr_err("device at index %d is going down\n", index); 2253 return -EINVAL; 2254 } 2255 mutex_unlock(&nbd_index_mutex); 2256 2257 config = nbd_get_config_unlocked(nbd); 2258 if (!config) { 2259 dev_err(nbd_to_dev(nbd), 2260 "not configured, cannot reconfigure\n"); 2261 nbd_put(nbd); 2262 return -EINVAL; 2263 } 2264 2265 mutex_lock(&nbd->config_lock); 2266 if (!test_bit(NBD_RT_BOUND, &config->runtime_flags) || 2267 !nbd->pid) { 2268 dev_err(nbd_to_dev(nbd), 2269 "not configured, cannot reconfigure\n"); 2270 ret = -EINVAL; 2271 goto out; 2272 } 2273 2274 ret = nbd_genl_size_set(info, nbd); 2275 if (ret) 2276 goto out; 2277 2278 if (info->attrs[NBD_ATTR_TIMEOUT]) 2279 nbd_set_cmd_timeout(nbd, 2280 nla_get_u64(info->attrs[NBD_ATTR_TIMEOUT])); 2281 if (info->attrs[NBD_ATTR_DEAD_CONN_TIMEOUT]) { 2282 config->dead_conn_timeout = 2283 nla_get_u64(info->attrs[NBD_ATTR_DEAD_CONN_TIMEOUT]); 2284 config->dead_conn_timeout *= HZ; 2285 } 2286 if (info->attrs[NBD_ATTR_CLIENT_FLAGS]) { 2287 u64 flags = nla_get_u64(info->attrs[NBD_ATTR_CLIENT_FLAGS]); 2288 if (flags & NBD_CFLAG_DESTROY_ON_DISCONNECT) { 2289 if (!test_and_set_bit(NBD_DESTROY_ON_DISCONNECT, 2290 &nbd->flags)) 2291 put_dev = true; 2292 } else { 2293 if (test_and_clear_bit(NBD_DESTROY_ON_DISCONNECT, 2294 &nbd->flags)) 2295 refcount_inc(&nbd->refs); 2296 } 2297 2298 if (flags & NBD_CFLAG_DISCONNECT_ON_CLOSE) { 2299 set_bit(NBD_RT_DISCONNECT_ON_CLOSE, 2300 &config->runtime_flags); 2301 } else { 2302 clear_bit(NBD_RT_DISCONNECT_ON_CLOSE, 2303 &config->runtime_flags); 2304 } 2305 } 2306 2307 if (info->attrs[NBD_ATTR_SOCKETS]) { 2308 struct nlattr *attr; 2309 int rem, fd; 2310 2311 nla_for_each_nested(attr, info->attrs[NBD_ATTR_SOCKETS], 2312 rem) { 2313 struct nlattr *socks[NBD_SOCK_MAX+1]; 2314 2315 if (nla_type(attr) != NBD_SOCK_ITEM) { 2316 pr_err("socks must be embedded in a SOCK_ITEM attr\n"); 2317 ret = -EINVAL; 2318 goto out; 2319 } 2320 ret = nla_parse_nested_deprecated(socks, NBD_SOCK_MAX, 2321 attr, 2322 nbd_sock_policy, 2323 info->extack); 2324 if (ret != 0) { 2325 pr_err("error processing sock list\n"); 2326 ret = -EINVAL; 2327 goto out; 2328 } 2329 if (!socks[NBD_SOCK_FD]) 2330 continue; 2331 fd = (int)nla_get_u32(socks[NBD_SOCK_FD]); 2332 ret = nbd_reconnect_socket(nbd, fd); 2333 if (ret) { 2334 if (ret == -ENOSPC) 2335 ret = 0; 2336 goto out; 2337 } 2338 dev_info(nbd_to_dev(nbd), "reconnected socket\n"); 2339 } 2340 } 2341out: 2342 mutex_unlock(&nbd->config_lock); 2343 nbd_config_put(nbd); 2344 nbd_put(nbd); 2345 if (put_dev) 2346 nbd_put(nbd); 2347 return ret; 2348} 2349 2350static const struct genl_small_ops nbd_connect_genl_ops[] = { 2351 { 2352 .cmd = NBD_CMD_CONNECT, 2353 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2354 .doit = nbd_genl_connect, 2355 }, 2356 { 2357 .cmd = NBD_CMD_DISCONNECT, 2358 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2359 .doit = nbd_genl_disconnect, 2360 }, 2361 { 2362 .cmd = NBD_CMD_RECONFIGURE, 2363 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2364 .doit = nbd_genl_reconfigure, 2365 }, 2366 { 2367 .cmd = NBD_CMD_STATUS, 2368 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2369 .doit = nbd_genl_status, 2370 }, 2371}; 2372 2373static const struct genl_multicast_group nbd_mcast_grps[] = { 2374 { .name = NBD_GENL_MCAST_GROUP_NAME, }, 2375}; 2376 2377static struct genl_family nbd_genl_family __ro_after_init = { 2378 .hdrsize = 0, 2379 .name = NBD_GENL_FAMILY_NAME, 2380 .version = NBD_GENL_VERSION, 2381 .module = THIS_MODULE, 2382 .small_ops = nbd_connect_genl_ops, 2383 .n_small_ops = ARRAY_SIZE(nbd_connect_genl_ops), 2384 .resv_start_op = NBD_CMD_STATUS + 1, 2385 .maxattr = NBD_ATTR_MAX, 2386 .netnsok = 1, 2387 .policy = nbd_attr_policy, 2388 .mcgrps = nbd_mcast_grps, 2389 .n_mcgrps = ARRAY_SIZE(nbd_mcast_grps), 2390}; 2391MODULE_ALIAS_GENL_FAMILY(NBD_GENL_FAMILY_NAME); 2392 2393static int populate_nbd_status(struct nbd_device *nbd, struct sk_buff *reply) 2394{ 2395 struct nlattr *dev_opt; 2396 u8 connected = 0; 2397 int ret; 2398 2399 /* This is a little racey, but for status it's ok. The 2400 * reason we don't take a ref here is because we can't 2401 * take a ref in the index == -1 case as we would need 2402 * to put under the nbd_index_mutex, which could 2403 * deadlock if we are configured to remove ourselves 2404 * once we're disconnected. 2405 */ 2406 if (refcount_read(&nbd->config_refs)) 2407 connected = 1; 2408 dev_opt = nla_nest_start_noflag(reply, NBD_DEVICE_ITEM); 2409 if (!dev_opt) 2410 return -EMSGSIZE; 2411 ret = nla_put_u32(reply, NBD_DEVICE_INDEX, nbd->index); 2412 if (ret) 2413 return -EMSGSIZE; 2414 ret = nla_put_u8(reply, NBD_DEVICE_CONNECTED, 2415 connected); 2416 if (ret) 2417 return -EMSGSIZE; 2418 nla_nest_end(reply, dev_opt); 2419 return 0; 2420} 2421 2422static int status_cb(int id, void *ptr, void *data) 2423{ 2424 struct nbd_device *nbd = ptr; 2425 return populate_nbd_status(nbd, (struct sk_buff *)data); 2426} 2427 2428static int nbd_genl_status(struct sk_buff *skb, struct genl_info *info) 2429{ 2430 struct nlattr *dev_list; 2431 struct sk_buff *reply; 2432 void *reply_head; 2433 size_t msg_size; 2434 int index = -1; 2435 int ret = -ENOMEM; 2436 2437 if (info->attrs[NBD_ATTR_INDEX]) 2438 index = nla_get_u32(info->attrs[NBD_ATTR_INDEX]); 2439 2440 mutex_lock(&nbd_index_mutex); 2441 2442 msg_size = nla_total_size(nla_attr_size(sizeof(u32)) + 2443 nla_attr_size(sizeof(u8))); 2444 msg_size *= (index == -1) ? nbd_total_devices : 1; 2445 2446 reply = genlmsg_new(msg_size, GFP_KERNEL); 2447 if (!reply) 2448 goto out; 2449 reply_head = genlmsg_put_reply(reply, info, &nbd_genl_family, 0, 2450 NBD_CMD_STATUS); 2451 if (!reply_head) { 2452 nlmsg_free(reply); 2453 goto out; 2454 } 2455 2456 dev_list = nla_nest_start_noflag(reply, NBD_ATTR_DEVICE_LIST); 2457 if (!dev_list) { 2458 nlmsg_free(reply); 2459 ret = -EMSGSIZE; 2460 goto out; 2461 } 2462 2463 if (index == -1) { 2464 ret = idr_for_each(&nbd_index_idr, &status_cb, reply); 2465 if (ret) { 2466 nlmsg_free(reply); 2467 goto out; 2468 } 2469 } else { 2470 struct nbd_device *nbd; 2471 nbd = idr_find(&nbd_index_idr, index); 2472 if (nbd) { 2473 ret = populate_nbd_status(nbd, reply); 2474 if (ret) { 2475 nlmsg_free(reply); 2476 goto out; 2477 } 2478 } 2479 } 2480 nla_nest_end(reply, dev_list); 2481 genlmsg_end(reply, reply_head); 2482 ret = genlmsg_reply(reply, info); 2483out: 2484 mutex_unlock(&nbd_index_mutex); 2485 return ret; 2486} 2487 2488static void nbd_connect_reply(struct genl_info *info, int index) 2489{ 2490 struct sk_buff *skb; 2491 void *msg_head; 2492 int ret; 2493 2494 skb = genlmsg_new(nla_total_size(sizeof(u32)), GFP_KERNEL); 2495 if (!skb) 2496 return; 2497 msg_head = genlmsg_put_reply(skb, info, &nbd_genl_family, 0, 2498 NBD_CMD_CONNECT); 2499 if (!msg_head) { 2500 nlmsg_free(skb); 2501 return; 2502 } 2503 ret = nla_put_u32(skb, NBD_ATTR_INDEX, index); 2504 if (ret) { 2505 nlmsg_free(skb); 2506 return; 2507 } 2508 genlmsg_end(skb, msg_head); 2509 genlmsg_reply(skb, info); 2510} 2511 2512static void nbd_mcast_index(int index) 2513{ 2514 struct sk_buff *skb; 2515 void *msg_head; 2516 int ret; 2517 2518 skb = genlmsg_new(nla_total_size(sizeof(u32)), GFP_KERNEL); 2519 if (!skb) 2520 return; 2521 msg_head = genlmsg_put(skb, 0, 0, &nbd_genl_family, 0, 2522 NBD_CMD_LINK_DEAD); 2523 if (!msg_head) { 2524 nlmsg_free(skb); 2525 return; 2526 } 2527 ret = nla_put_u32(skb, NBD_ATTR_INDEX, index); 2528 if (ret) { 2529 nlmsg_free(skb); 2530 return; 2531 } 2532 genlmsg_end(skb, msg_head); 2533 genlmsg_multicast(&nbd_genl_family, skb, 0, 0, GFP_KERNEL); 2534} 2535 2536static void nbd_dead_link_work(struct work_struct *work) 2537{ 2538 struct link_dead_args *args = container_of(work, struct link_dead_args, 2539 work); 2540 nbd_mcast_index(args->index); 2541 kfree(args); 2542} 2543 2544static int __init nbd_init(void) 2545{ 2546 int i; 2547 2548 BUILD_BUG_ON(sizeof(struct nbd_request) != 28); 2549 2550 if (max_part < 0) { 2551 pr_err("max_part must be >= 0\n"); 2552 return -EINVAL; 2553 } 2554 2555 part_shift = 0; 2556 if (max_part > 0) { 2557 part_shift = fls(max_part); 2558 2559 /* 2560 * Adjust max_part according to part_shift as it is exported 2561 * to user space so that user can know the max number of 2562 * partition kernel should be able to manage. 2563 * 2564 * Note that -1 is required because partition 0 is reserved 2565 * for the whole disk. 2566 */ 2567 max_part = (1UL << part_shift) - 1; 2568 } 2569 2570 if ((1UL << part_shift) > DISK_MAX_PARTS) 2571 return -EINVAL; 2572 2573 if (nbds_max > 1UL << (MINORBITS - part_shift)) 2574 return -EINVAL; 2575 2576 if (register_blkdev(NBD_MAJOR, "nbd")) 2577 return -EIO; 2578 2579 nbd_del_wq = alloc_workqueue("nbd-del", WQ_UNBOUND, 0); 2580 if (!nbd_del_wq) { 2581 unregister_blkdev(NBD_MAJOR, "nbd"); 2582 return -ENOMEM; 2583 } 2584 2585 if (genl_register_family(&nbd_genl_family)) { 2586 destroy_workqueue(nbd_del_wq); 2587 unregister_blkdev(NBD_MAJOR, "nbd"); 2588 return -EINVAL; 2589 } 2590 nbd_dbg_init(); 2591 2592 for (i = 0; i < nbds_max; i++) 2593 nbd_dev_add(i, 1); 2594 return 0; 2595} 2596 2597static int nbd_exit_cb(int id, void *ptr, void *data) 2598{ 2599 struct list_head *list = (struct list_head *)data; 2600 struct nbd_device *nbd = ptr; 2601 2602 /* Skip nbd that is being removed asynchronously */ 2603 if (refcount_read(&nbd->refs)) 2604 list_add_tail(&nbd->list, list); 2605 2606 return 0; 2607} 2608 2609static void __exit nbd_cleanup(void) 2610{ 2611 struct nbd_device *nbd; 2612 LIST_HEAD(del_list); 2613 2614 /* 2615 * Unregister netlink interface prior to waiting 2616 * for the completion of netlink commands. 2617 */ 2618 genl_unregister_family(&nbd_genl_family); 2619 2620 nbd_dbg_close(); 2621 2622 mutex_lock(&nbd_index_mutex); 2623 idr_for_each(&nbd_index_idr, &nbd_exit_cb, &del_list); 2624 mutex_unlock(&nbd_index_mutex); 2625 2626 while (!list_empty(&del_list)) { 2627 nbd = list_first_entry(&del_list, struct nbd_device, list); 2628 list_del_init(&nbd->list); 2629 if (refcount_read(&nbd->config_refs)) 2630 pr_err("possibly leaking nbd_config (ref %d)\n", 2631 refcount_read(&nbd->config_refs)); 2632 if (refcount_read(&nbd->refs) != 1) 2633 pr_err("possibly leaking a device\n"); 2634 nbd_put(nbd); 2635 } 2636 2637 /* Also wait for nbd_dev_remove_work() completes */ 2638 destroy_workqueue(nbd_del_wq); 2639 2640 idr_destroy(&nbd_index_idr); 2641 unregister_blkdev(NBD_MAJOR, "nbd"); 2642} 2643 2644module_init(nbd_init); 2645module_exit(nbd_cleanup); 2646 2647MODULE_DESCRIPTION("Network Block Device"); 2648MODULE_LICENSE("GPL"); 2649 2650module_param(nbds_max, int, 0444); 2651MODULE_PARM_DESC(nbds_max, "number of network block devices to initialize (default: 16)"); 2652module_param(max_part, int, 0444); 2653MODULE_PARM_DESC(max_part, "number of partitions per device (default: 16)"); 2654