vmbus_chan.c revision 311369
1/*- 2 * Copyright (c) 2009-2012,2016 Microsoft Corp. 3 * Copyright (c) 2012 NetApp Inc. 4 * Copyright (c) 2012 Citrix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29#include <sys/cdefs.h> 30__FBSDID("$FreeBSD: stable/11/sys/dev/hyperv/vmbus/vmbus_chan.c 311369 2017-01-05 05:54:22Z sephe $"); 31 32#include <sys/param.h> 33#include <sys/bus.h> 34#include <sys/kernel.h> 35#include <sys/lock.h> 36#include <sys/malloc.h> 37#include <sys/mutex.h> 38#include <sys/smp.h> 39#include <sys/sysctl.h> 40#include <sys/systm.h> 41 42#include <machine/atomic.h> 43#include <machine/stdarg.h> 44 45#include <dev/hyperv/include/hyperv_busdma.h> 46#include <dev/hyperv/include/vmbus_xact.h> 47#include <dev/hyperv/vmbus/hyperv_var.h> 48#include <dev/hyperv/vmbus/vmbus_reg.h> 49#include <dev/hyperv/vmbus/vmbus_var.h> 50#include <dev/hyperv/vmbus/vmbus_brvar.h> 51#include <dev/hyperv/vmbus/vmbus_chanvar.h> 52 53static void vmbus_chan_update_evtflagcnt( 54 struct vmbus_softc *, 55 const struct vmbus_channel *); 56static int vmbus_chan_close_internal( 57 struct vmbus_channel *); 58static int vmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS); 59static void vmbus_chan_sysctl_create( 60 struct vmbus_channel *); 61static struct vmbus_channel *vmbus_chan_alloc(struct vmbus_softc *); 62static void vmbus_chan_free(struct vmbus_channel *); 63static int vmbus_chan_add(struct vmbus_channel *); 64static void vmbus_chan_cpu_default(struct vmbus_channel *); 65static int vmbus_chan_release(struct vmbus_channel *); 66static void vmbus_chan_set_chmap(struct vmbus_channel *); 67static void vmbus_chan_clear_chmap(struct vmbus_channel *); 68static void vmbus_chan_detach(struct vmbus_channel *); 69static bool vmbus_chan_wait_revoke( 70 const struct vmbus_channel *, bool); 71 72static void vmbus_chan_ins_prilist(struct vmbus_softc *, 73 struct vmbus_channel *); 74static void vmbus_chan_rem_prilist(struct vmbus_softc *, 75 struct vmbus_channel *); 76static void vmbus_chan_ins_list(struct vmbus_softc *, 77 struct vmbus_channel *); 78static void vmbus_chan_rem_list(struct vmbus_softc *, 79 struct vmbus_channel *); 80static void vmbus_chan_ins_sublist(struct vmbus_channel *, 81 struct vmbus_channel *); 82static void vmbus_chan_rem_sublist(struct vmbus_channel *, 83 struct vmbus_channel *); 84 85static void vmbus_chan_task(void *, int); 86static void vmbus_chan_task_nobatch(void *, int); 87static void vmbus_chan_clrchmap_task(void *, int); 88static void vmbus_prichan_attach_task(void *, int); 89static void vmbus_subchan_attach_task(void *, int); 90static void vmbus_prichan_detach_task(void *, int); 91static void vmbus_subchan_detach_task(void *, int); 92 93static void vmbus_chan_msgproc_choffer(struct vmbus_softc *, 94 const struct vmbus_message *); 95static void vmbus_chan_msgproc_chrescind( 96 struct vmbus_softc *, 97 const struct vmbus_message *); 98 99static int vmbus_chan_printf(const struct vmbus_channel *, 100 const char *, ...) __printflike(2, 3); 101 102/* 103 * Vmbus channel message processing. 104 */ 105static const vmbus_chanmsg_proc_t 106vmbus_chan_msgprocs[VMBUS_CHANMSG_TYPE_MAX] = { 107 VMBUS_CHANMSG_PROC(CHOFFER, vmbus_chan_msgproc_choffer), 108 VMBUS_CHANMSG_PROC(CHRESCIND, vmbus_chan_msgproc_chrescind), 109 110 VMBUS_CHANMSG_PROC_WAKEUP(CHOPEN_RESP), 111 VMBUS_CHANMSG_PROC_WAKEUP(GPADL_CONNRESP), 112 VMBUS_CHANMSG_PROC_WAKEUP(GPADL_DISCONNRESP) 113}; 114 115/* 116 * Notify host that there are data pending on our TX bufring. 117 */ 118static __inline void 119vmbus_chan_signal_tx(const struct vmbus_channel *chan) 120{ 121 atomic_set_long(chan->ch_evtflag, chan->ch_evtflag_mask); 122 if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF) 123 atomic_set_int(chan->ch_montrig, chan->ch_montrig_mask); 124 else 125 hypercall_signal_event(chan->ch_monprm_dma.hv_paddr); 126} 127 128static void 129vmbus_chan_ins_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan) 130{ 131 132 mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED); 133 if (atomic_testandset_int(&chan->ch_stflags, 134 VMBUS_CHAN_ST_ONPRIL_SHIFT)) 135 panic("channel is already on the prilist"); 136 TAILQ_INSERT_TAIL(&sc->vmbus_prichans, chan, ch_prilink); 137} 138 139static void 140vmbus_chan_rem_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan) 141{ 142 143 mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED); 144 if (atomic_testandclear_int(&chan->ch_stflags, 145 VMBUS_CHAN_ST_ONPRIL_SHIFT) == 0) 146 panic("channel is not on the prilist"); 147 TAILQ_REMOVE(&sc->vmbus_prichans, chan, ch_prilink); 148} 149 150static void 151vmbus_chan_ins_sublist(struct vmbus_channel *prichan, 152 struct vmbus_channel *chan) 153{ 154 155 mtx_assert(&prichan->ch_subchan_lock, MA_OWNED); 156 157 if (atomic_testandset_int(&chan->ch_stflags, 158 VMBUS_CHAN_ST_ONSUBL_SHIFT)) 159 panic("channel is already on the sublist"); 160 TAILQ_INSERT_TAIL(&prichan->ch_subchans, chan, ch_sublink); 161 162 /* Bump sub-channel count. */ 163 prichan->ch_subchan_cnt++; 164} 165 166static void 167vmbus_chan_rem_sublist(struct vmbus_channel *prichan, 168 struct vmbus_channel *chan) 169{ 170 171 mtx_assert(&prichan->ch_subchan_lock, MA_OWNED); 172 173 KASSERT(prichan->ch_subchan_cnt > 0, 174 ("invalid subchan_cnt %d", prichan->ch_subchan_cnt)); 175 prichan->ch_subchan_cnt--; 176 177 if (atomic_testandclear_int(&chan->ch_stflags, 178 VMBUS_CHAN_ST_ONSUBL_SHIFT) == 0) 179 panic("channel is not on the sublist"); 180 TAILQ_REMOVE(&prichan->ch_subchans, chan, ch_sublink); 181} 182 183static void 184vmbus_chan_ins_list(struct vmbus_softc *sc, struct vmbus_channel *chan) 185{ 186 187 mtx_assert(&sc->vmbus_chan_lock, MA_OWNED); 188 if (atomic_testandset_int(&chan->ch_stflags, 189 VMBUS_CHAN_ST_ONLIST_SHIFT)) 190 panic("channel is already on the list"); 191 TAILQ_INSERT_TAIL(&sc->vmbus_chans, chan, ch_link); 192} 193 194static void 195vmbus_chan_rem_list(struct vmbus_softc *sc, struct vmbus_channel *chan) 196{ 197 198 mtx_assert(&sc->vmbus_chan_lock, MA_OWNED); 199 if (atomic_testandclear_int(&chan->ch_stflags, 200 VMBUS_CHAN_ST_ONLIST_SHIFT) == 0) 201 panic("channel is not on the list"); 202 TAILQ_REMOVE(&sc->vmbus_chans, chan, ch_link); 203} 204 205static int 206vmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS) 207{ 208 struct vmbus_channel *chan = arg1; 209 int mnf = 0; 210 211 if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF) 212 mnf = 1; 213 return sysctl_handle_int(oidp, &mnf, 0, req); 214} 215 216static void 217vmbus_chan_sysctl_create(struct vmbus_channel *chan) 218{ 219 struct sysctl_oid *ch_tree, *chid_tree, *br_tree; 220 struct sysctl_ctx_list *ctx; 221 uint32_t ch_id; 222 char name[16]; 223 224 /* 225 * Add sysctl nodes related to this channel to this 226 * channel's sysctl ctx, so that they can be destroyed 227 * independently upon close of this channel, which can 228 * happen even if the device is not detached. 229 */ 230 ctx = &chan->ch_sysctl_ctx; 231 sysctl_ctx_init(ctx); 232 233 /* 234 * Create dev.NAME.UNIT.channel tree. 235 */ 236 ch_tree = SYSCTL_ADD_NODE(ctx, 237 SYSCTL_CHILDREN(device_get_sysctl_tree(chan->ch_dev)), 238 OID_AUTO, "channel", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 239 if (ch_tree == NULL) 240 return; 241 242 /* 243 * Create dev.NAME.UNIT.channel.CHANID tree. 244 */ 245 if (VMBUS_CHAN_ISPRIMARY(chan)) 246 ch_id = chan->ch_id; 247 else 248 ch_id = chan->ch_prichan->ch_id; 249 snprintf(name, sizeof(name), "%d", ch_id); 250 chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree), 251 OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 252 if (chid_tree == NULL) 253 return; 254 255 if (!VMBUS_CHAN_ISPRIMARY(chan)) { 256 /* 257 * Create dev.NAME.UNIT.channel.CHANID.sub tree. 258 */ 259 ch_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree), 260 OID_AUTO, "sub", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 261 if (ch_tree == NULL) 262 return; 263 264 /* 265 * Create dev.NAME.UNIT.channel.CHANID.sub.SUBIDX tree. 266 * 267 * NOTE: 268 * chid_tree is changed to this new sysctl tree. 269 */ 270 snprintf(name, sizeof(name), "%d", chan->ch_subidx); 271 chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree), 272 OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 273 if (chid_tree == NULL) 274 return; 275 276 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 277 "chanid", CTLFLAG_RD, &chan->ch_id, 0, "channel id"); 278 } 279 280 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 281 "cpu", CTLFLAG_RD, &chan->ch_cpuid, 0, "owner CPU id"); 282 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 283 "mnf", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, 284 chan, 0, vmbus_chan_sysctl_mnf, "I", 285 "has monitor notification facilities"); 286 287 br_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 288 "br", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 289 if (br_tree != NULL) { 290 /* 291 * Create sysctl tree for RX bufring. 292 */ 293 vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_rxbr.rxbr, "rx"); 294 /* 295 * Create sysctl tree for TX bufring. 296 */ 297 vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_txbr.txbr, "tx"); 298 } 299} 300 301int 302vmbus_chan_open(struct vmbus_channel *chan, int txbr_size, int rxbr_size, 303 const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg) 304{ 305 struct vmbus_chan_br cbr; 306 int error; 307 308 /* 309 * Allocate the TX+RX bufrings. 310 */ 311 KASSERT(chan->ch_bufring == NULL, ("bufrings are allocated")); 312 chan->ch_bufring = hyperv_dmamem_alloc(bus_get_dma_tag(chan->ch_dev), 313 PAGE_SIZE, 0, txbr_size + rxbr_size, &chan->ch_bufring_dma, 314 BUS_DMA_WAITOK); 315 if (chan->ch_bufring == NULL) { 316 vmbus_chan_printf(chan, "bufring allocation failed\n"); 317 return (ENOMEM); 318 } 319 320 cbr.cbr = chan->ch_bufring; 321 cbr.cbr_paddr = chan->ch_bufring_dma.hv_paddr; 322 cbr.cbr_txsz = txbr_size; 323 cbr.cbr_rxsz = rxbr_size; 324 325 error = vmbus_chan_open_br(chan, &cbr, udata, udlen, cb, cbarg); 326 if (error) { 327 if (error == EISCONN) { 328 /* 329 * XXX 330 * The bufring GPADL is still connected; abandon 331 * this bufring, instead of having mysterious 332 * crash or trashed data later on. 333 */ 334 vmbus_chan_printf(chan, "chan%u bufring GPADL " 335 "is still connected upon channel open error; " 336 "leak %d bytes memory\n", chan->ch_id, 337 txbr_size + rxbr_size); 338 } else { 339 hyperv_dmamem_free(&chan->ch_bufring_dma, 340 chan->ch_bufring); 341 } 342 chan->ch_bufring = NULL; 343 } 344 return (error); 345} 346 347int 348vmbus_chan_open_br(struct vmbus_channel *chan, const struct vmbus_chan_br *cbr, 349 const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg) 350{ 351 struct vmbus_softc *sc = chan->ch_vmbus; 352 const struct vmbus_message *msg; 353 struct vmbus_chanmsg_chopen *req; 354 struct vmbus_msghc *mh; 355 uint32_t status; 356 int error, txbr_size, rxbr_size; 357 task_fn_t *task_fn; 358 uint8_t *br; 359 360 if (udlen > VMBUS_CHANMSG_CHOPEN_UDATA_SIZE) { 361 vmbus_chan_printf(chan, 362 "invalid udata len %d for chan%u\n", udlen, chan->ch_id); 363 return (EINVAL); 364 } 365 366 br = cbr->cbr; 367 txbr_size = cbr->cbr_txsz; 368 rxbr_size = cbr->cbr_rxsz; 369 KASSERT((txbr_size & PAGE_MASK) == 0, 370 ("send bufring size is not multiple page")); 371 KASSERT((rxbr_size & PAGE_MASK) == 0, 372 ("recv bufring size is not multiple page")); 373 KASSERT((cbr->cbr_paddr & PAGE_MASK) == 0, 374 ("bufring is not page aligned")); 375 376 /* 377 * Zero out the TX/RX bufrings, in case that they were used before. 378 */ 379 memset(br, 0, txbr_size + rxbr_size); 380 381 if (atomic_testandset_int(&chan->ch_stflags, 382 VMBUS_CHAN_ST_OPENED_SHIFT)) 383 panic("double-open chan%u", chan->ch_id); 384 385 chan->ch_cb = cb; 386 chan->ch_cbarg = cbarg; 387 388 vmbus_chan_update_evtflagcnt(sc, chan); 389 390 chan->ch_tq = VMBUS_PCPU_GET(chan->ch_vmbus, event_tq, chan->ch_cpuid); 391 if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD) 392 task_fn = vmbus_chan_task; 393 else 394 task_fn = vmbus_chan_task_nobatch; 395 TASK_INIT(&chan->ch_task, 0, task_fn, chan); 396 397 /* TX bufring comes first */ 398 vmbus_txbr_setup(&chan->ch_txbr, br, txbr_size); 399 /* RX bufring immediately follows TX bufring */ 400 vmbus_rxbr_setup(&chan->ch_rxbr, br + txbr_size, rxbr_size); 401 402 /* Create sysctl tree for this channel */ 403 vmbus_chan_sysctl_create(chan); 404 405 /* 406 * Connect the bufrings, both RX and TX, to this channel. 407 */ 408 error = vmbus_chan_gpadl_connect(chan, cbr->cbr_paddr, 409 txbr_size + rxbr_size, &chan->ch_bufring_gpadl); 410 if (error) { 411 vmbus_chan_printf(chan, 412 "failed to connect bufring GPADL to chan%u\n", chan->ch_id); 413 goto failed; 414 } 415 416 /* 417 * Install this channel, before it is opened, but after everything 418 * else has been setup. 419 */ 420 vmbus_chan_set_chmap(chan); 421 422 /* 423 * Open channel w/ the bufring GPADL on the target CPU. 424 */ 425 mh = vmbus_msghc_get(sc, sizeof(*req)); 426 if (mh == NULL) { 427 vmbus_chan_printf(chan, 428 "can not get msg hypercall for chopen(chan%u)\n", 429 chan->ch_id); 430 error = ENXIO; 431 goto failed; 432 } 433 434 req = vmbus_msghc_dataptr(mh); 435 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHOPEN; 436 req->chm_chanid = chan->ch_id; 437 req->chm_openid = chan->ch_id; 438 req->chm_gpadl = chan->ch_bufring_gpadl; 439 req->chm_vcpuid = chan->ch_vcpuid; 440 req->chm_txbr_pgcnt = txbr_size >> PAGE_SHIFT; 441 if (udlen > 0) 442 memcpy(req->chm_udata, udata, udlen); 443 444 error = vmbus_msghc_exec(sc, mh); 445 if (error) { 446 vmbus_chan_printf(chan, 447 "chopen(chan%u) msg hypercall exec failed: %d\n", 448 chan->ch_id, error); 449 vmbus_msghc_put(sc, mh); 450 goto failed; 451 } 452 453 for (;;) { 454 msg = vmbus_msghc_poll_result(sc, mh); 455 if (msg != NULL) 456 break; 457 if (vmbus_chan_is_revoked(chan)) { 458 int i; 459 460 /* 461 * NOTE: 462 * Hypervisor does _not_ send response CHOPEN to 463 * a revoked channel. 464 */ 465 vmbus_chan_printf(chan, 466 "chan%u is revoked, when it is being opened\n", 467 chan->ch_id); 468 469 /* 470 * XXX 471 * Add extra delay before cancel the hypercall 472 * execution; mainly to close any possible 473 * CHRESCIND and CHOPEN_RESP races on the 474 * hypervisor side. 475 */ 476#define REVOKE_LINGER 100 477 for (i = 0; i < REVOKE_LINGER; ++i) { 478 msg = vmbus_msghc_poll_result(sc, mh); 479 if (msg != NULL) 480 break; 481 pause("rchopen", 1); 482 } 483#undef REVOKE_LINGER 484 if (msg == NULL) 485 vmbus_msghc_exec_cancel(sc, mh); 486 break; 487 } 488 pause("chopen", 1); 489 } 490 if (msg != NULL) { 491 status = ((const struct vmbus_chanmsg_chopen_resp *) 492 msg->msg_data)->chm_status; 493 } else { 494 /* XXX any non-0 value is ok here. */ 495 status = 0xff; 496 } 497 498 vmbus_msghc_put(sc, mh); 499 500 if (status == 0) { 501 if (bootverbose) 502 vmbus_chan_printf(chan, "chan%u opened\n", chan->ch_id); 503 return (0); 504 } 505 506 vmbus_chan_printf(chan, "failed to open chan%u\n", chan->ch_id); 507 error = ENXIO; 508 509failed: 510 sysctl_ctx_free(&chan->ch_sysctl_ctx); 511 vmbus_chan_clear_chmap(chan); 512 if (chan->ch_bufring_gpadl != 0) { 513 int error1; 514 515 error1 = vmbus_chan_gpadl_disconnect(chan, 516 chan->ch_bufring_gpadl); 517 if (error1) { 518 /* 519 * Give caller a hint that the bufring GPADL is still 520 * connected. 521 */ 522 error = EISCONN; 523 } 524 chan->ch_bufring_gpadl = 0; 525 } 526 atomic_clear_int(&chan->ch_stflags, VMBUS_CHAN_ST_OPENED); 527 return (error); 528} 529 530int 531vmbus_chan_gpadl_connect(struct vmbus_channel *chan, bus_addr_t paddr, 532 int size, uint32_t *gpadl0) 533{ 534 struct vmbus_softc *sc = chan->ch_vmbus; 535 struct vmbus_msghc *mh; 536 struct vmbus_chanmsg_gpadl_conn *req; 537 const struct vmbus_message *msg; 538 size_t reqsz; 539 uint32_t gpadl, status; 540 int page_count, range_len, i, cnt, error; 541 uint64_t page_id; 542 543 KASSERT(*gpadl0 == 0, ("GPADL is not zero")); 544 545 /* 546 * Preliminary checks. 547 */ 548 549 KASSERT((size & PAGE_MASK) == 0, 550 ("invalid GPA size %d, not multiple page size", size)); 551 page_count = size >> PAGE_SHIFT; 552 553 KASSERT((paddr & PAGE_MASK) == 0, 554 ("GPA is not page aligned %jx", (uintmax_t)paddr)); 555 page_id = paddr >> PAGE_SHIFT; 556 557 range_len = __offsetof(struct vmbus_gpa_range, gpa_page[page_count]); 558 /* 559 * We don't support multiple GPA ranges. 560 */ 561 if (range_len > UINT16_MAX) { 562 vmbus_chan_printf(chan, "GPA too large, %d pages\n", 563 page_count); 564 return EOPNOTSUPP; 565 } 566 567 /* 568 * Allocate GPADL id. 569 */ 570 gpadl = vmbus_gpadl_alloc(sc); 571 572 /* 573 * Connect this GPADL to the target channel. 574 * 575 * NOTE: 576 * Since each message can only hold small set of page 577 * addresses, several messages may be required to 578 * complete the connection. 579 */ 580 if (page_count > VMBUS_CHANMSG_GPADL_CONN_PGMAX) 581 cnt = VMBUS_CHANMSG_GPADL_CONN_PGMAX; 582 else 583 cnt = page_count; 584 page_count -= cnt; 585 586 reqsz = __offsetof(struct vmbus_chanmsg_gpadl_conn, 587 chm_range.gpa_page[cnt]); 588 mh = vmbus_msghc_get(sc, reqsz); 589 if (mh == NULL) { 590 vmbus_chan_printf(chan, 591 "can not get msg hypercall for gpadl_conn(chan%u)\n", 592 chan->ch_id); 593 return EIO; 594 } 595 596 req = vmbus_msghc_dataptr(mh); 597 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_CONN; 598 req->chm_chanid = chan->ch_id; 599 req->chm_gpadl = gpadl; 600 req->chm_range_len = range_len; 601 req->chm_range_cnt = 1; 602 req->chm_range.gpa_len = size; 603 req->chm_range.gpa_ofs = 0; 604 for (i = 0; i < cnt; ++i) 605 req->chm_range.gpa_page[i] = page_id++; 606 607 error = vmbus_msghc_exec(sc, mh); 608 if (error) { 609 vmbus_chan_printf(chan, 610 "gpadl_conn(chan%u) msg hypercall exec failed: %d\n", 611 chan->ch_id, error); 612 vmbus_msghc_put(sc, mh); 613 return error; 614 } 615 616 while (page_count > 0) { 617 struct vmbus_chanmsg_gpadl_subconn *subreq; 618 619 if (page_count > VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX) 620 cnt = VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX; 621 else 622 cnt = page_count; 623 page_count -= cnt; 624 625 reqsz = __offsetof(struct vmbus_chanmsg_gpadl_subconn, 626 chm_gpa_page[cnt]); 627 vmbus_msghc_reset(mh, reqsz); 628 629 subreq = vmbus_msghc_dataptr(mh); 630 subreq->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_SUBCONN; 631 subreq->chm_gpadl = gpadl; 632 for (i = 0; i < cnt; ++i) 633 subreq->chm_gpa_page[i] = page_id++; 634 635 vmbus_msghc_exec_noresult(mh); 636 } 637 KASSERT(page_count == 0, ("invalid page count %d", page_count)); 638 639 msg = vmbus_msghc_wait_result(sc, mh); 640 status = ((const struct vmbus_chanmsg_gpadl_connresp *) 641 msg->msg_data)->chm_status; 642 643 vmbus_msghc_put(sc, mh); 644 645 if (status != 0) { 646 vmbus_chan_printf(chan, "gpadl_conn(chan%u) failed: %u\n", 647 chan->ch_id, status); 648 return EIO; 649 } 650 651 /* Done; commit the GPADL id. */ 652 *gpadl0 = gpadl; 653 if (bootverbose) { 654 vmbus_chan_printf(chan, "gpadl_conn(chan%u) succeeded\n", 655 chan->ch_id); 656 } 657 return 0; 658} 659 660static bool 661vmbus_chan_wait_revoke(const struct vmbus_channel *chan, bool can_sleep) 662{ 663#define WAIT_COUNT 200 /* 200ms */ 664 665 int i; 666 667 for (i = 0; i < WAIT_COUNT; ++i) { 668 if (vmbus_chan_is_revoked(chan)) 669 return (true); 670 if (can_sleep) 671 pause("wchrev", 1); 672 else 673 DELAY(1000); 674 } 675 return (false); 676 677#undef WAIT_COUNT 678} 679 680/* 681 * Disconnect the GPA from the target channel 682 */ 683int 684vmbus_chan_gpadl_disconnect(struct vmbus_channel *chan, uint32_t gpadl) 685{ 686 struct vmbus_softc *sc = chan->ch_vmbus; 687 struct vmbus_msghc *mh; 688 struct vmbus_chanmsg_gpadl_disconn *req; 689 int error; 690 691 KASSERT(gpadl != 0, ("GPADL is zero")); 692 693 mh = vmbus_msghc_get(sc, sizeof(*req)); 694 if (mh == NULL) { 695 vmbus_chan_printf(chan, 696 "can not get msg hypercall for gpadl_disconn(chan%u)\n", 697 chan->ch_id); 698 return (EBUSY); 699 } 700 701 req = vmbus_msghc_dataptr(mh); 702 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_DISCONN; 703 req->chm_chanid = chan->ch_id; 704 req->chm_gpadl = gpadl; 705 706 error = vmbus_msghc_exec(sc, mh); 707 if (error) { 708 vmbus_msghc_put(sc, mh); 709 710 if (vmbus_chan_wait_revoke(chan, true)) { 711 /* 712 * Error is benign; this channel is revoked, 713 * so this GPADL will not be touched anymore. 714 */ 715 vmbus_chan_printf(chan, 716 "gpadl_disconn(revoked chan%u) msg hypercall " 717 "exec failed: %d\n", chan->ch_id, error); 718 return (0); 719 } 720 vmbus_chan_printf(chan, 721 "gpadl_disconn(chan%u) msg hypercall exec failed: %d\n", 722 chan->ch_id, error); 723 return (error); 724 } 725 726 vmbus_msghc_wait_result(sc, mh); 727 /* Discard result; no useful information */ 728 vmbus_msghc_put(sc, mh); 729 730 return (0); 731} 732 733static void 734vmbus_chan_detach(struct vmbus_channel *chan) 735{ 736 int refs; 737 738 KASSERT(chan->ch_refs > 0, ("chan%u: invalid refcnt %d", 739 chan->ch_id, chan->ch_refs)); 740 refs = atomic_fetchadd_int(&chan->ch_refs, -1); 741#ifdef INVARIANTS 742 if (VMBUS_CHAN_ISPRIMARY(chan)) { 743 KASSERT(refs == 1, ("chan%u: invalid refcnt %d for prichan", 744 chan->ch_id, refs + 1)); 745 } 746#endif 747 if (refs == 1) { 748 /* 749 * Detach the target channel. 750 */ 751 if (bootverbose) { 752 vmbus_chan_printf(chan, "chan%u detached\n", 753 chan->ch_id); 754 } 755 taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task); 756 } 757} 758 759static void 760vmbus_chan_clrchmap_task(void *xchan, int pending __unused) 761{ 762 struct vmbus_channel *chan = xchan; 763 764 critical_enter(); 765 chan->ch_vmbus->vmbus_chmap[chan->ch_id] = NULL; 766 critical_exit(); 767} 768 769static void 770vmbus_chan_clear_chmap(struct vmbus_channel *chan) 771{ 772 struct task chmap_task; 773 774 TASK_INIT(&chmap_task, 0, vmbus_chan_clrchmap_task, chan); 775 taskqueue_enqueue(chan->ch_tq, &chmap_task); 776 taskqueue_drain(chan->ch_tq, &chmap_task); 777} 778 779static void 780vmbus_chan_set_chmap(struct vmbus_channel *chan) 781{ 782 __compiler_membar(); 783 chan->ch_vmbus->vmbus_chmap[chan->ch_id] = chan; 784} 785 786static int 787vmbus_chan_close_internal(struct vmbus_channel *chan) 788{ 789 struct vmbus_softc *sc = chan->ch_vmbus; 790 struct vmbus_msghc *mh; 791 struct vmbus_chanmsg_chclose *req; 792 uint32_t old_stflags; 793 int error; 794 795 /* 796 * NOTE: 797 * Sub-channels are closed upon their primary channel closing, 798 * so they can be closed even before they are opened. 799 */ 800 for (;;) { 801 old_stflags = chan->ch_stflags; 802 if (atomic_cmpset_int(&chan->ch_stflags, old_stflags, 803 old_stflags & ~VMBUS_CHAN_ST_OPENED)) 804 break; 805 } 806 if ((old_stflags & VMBUS_CHAN_ST_OPENED) == 0) { 807 /* Not opened yet; done */ 808 if (bootverbose) { 809 vmbus_chan_printf(chan, "chan%u not opened\n", 810 chan->ch_id); 811 } 812 return (0); 813 } 814 815 /* 816 * Free this channel's sysctl tree attached to its device's 817 * sysctl tree. 818 */ 819 sysctl_ctx_free(&chan->ch_sysctl_ctx); 820 821 /* 822 * NOTE: 823 * Order is critical. This channel _must_ be uninstalled first, 824 * else the channel task may be enqueued by the IDT after it has 825 * been drained. 826 */ 827 vmbus_chan_clear_chmap(chan); 828 taskqueue_drain(chan->ch_tq, &chan->ch_task); 829 chan->ch_tq = NULL; 830 831 /* 832 * Close this channel. 833 */ 834 mh = vmbus_msghc_get(sc, sizeof(*req)); 835 if (mh == NULL) { 836 vmbus_chan_printf(chan, 837 "can not get msg hypercall for chclose(chan%u)\n", 838 chan->ch_id); 839 error = ENXIO; 840 goto disconnect; 841 } 842 843 req = vmbus_msghc_dataptr(mh); 844 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHCLOSE; 845 req->chm_chanid = chan->ch_id; 846 847 error = vmbus_msghc_exec_noresult(mh); 848 vmbus_msghc_put(sc, mh); 849 850 if (error) { 851 vmbus_chan_printf(chan, 852 "chclose(chan%u) msg hypercall exec failed: %d\n", 853 chan->ch_id, error); 854 goto disconnect; 855 } 856 857 if (bootverbose) 858 vmbus_chan_printf(chan, "chan%u closed\n", chan->ch_id); 859 860disconnect: 861 /* 862 * Disconnect the TX+RX bufrings from this channel. 863 */ 864 if (chan->ch_bufring_gpadl != 0) { 865 int error1; 866 867 error1 = vmbus_chan_gpadl_disconnect(chan, 868 chan->ch_bufring_gpadl); 869 if (error1) { 870 /* 871 * XXX 872 * The bufring GPADL is still connected; abandon 873 * this bufring, instead of having mysterious 874 * crash or trashed data later on. 875 */ 876 vmbus_chan_printf(chan, "chan%u bufring GPADL " 877 "is still connected after close\n", chan->ch_id); 878 chan->ch_bufring = NULL; 879 /* 880 * Give caller a hint that the bufring GPADL is 881 * still connected. 882 */ 883 error = EISCONN; 884 } 885 chan->ch_bufring_gpadl = 0; 886 } 887 888 /* 889 * Destroy the TX+RX bufrings. 890 */ 891 if (chan->ch_bufring != NULL) { 892 hyperv_dmamem_free(&chan->ch_bufring_dma, chan->ch_bufring); 893 chan->ch_bufring = NULL; 894 } 895 return (error); 896} 897 898int 899vmbus_chan_close_direct(struct vmbus_channel *chan) 900{ 901 int error; 902 903#ifdef INVARIANTS 904 if (VMBUS_CHAN_ISPRIMARY(chan)) { 905 struct vmbus_channel *subchan; 906 907 /* 908 * All sub-channels _must_ have been closed, or are _not_ 909 * opened at all. 910 */ 911 mtx_lock(&chan->ch_subchan_lock); 912 TAILQ_FOREACH(subchan, &chan->ch_subchans, ch_sublink) { 913 KASSERT( 914 (subchan->ch_stflags & VMBUS_CHAN_ST_OPENED) == 0, 915 ("chan%u: subchan%u is still opened", 916 chan->ch_id, subchan->ch_subidx)); 917 } 918 mtx_unlock(&chan->ch_subchan_lock); 919 } 920#endif 921 922 error = vmbus_chan_close_internal(chan); 923 if (!VMBUS_CHAN_ISPRIMARY(chan)) { 924 /* 925 * This sub-channel is referenced, when it is linked to 926 * the primary channel; drop that reference now. 927 */ 928 vmbus_chan_detach(chan); 929 } 930 return (error); 931} 932 933/* 934 * Caller should make sure that all sub-channels have 935 * been added to 'chan' and all to-be-closed channels 936 * are not being opened. 937 */ 938void 939vmbus_chan_close(struct vmbus_channel *chan) 940{ 941 int subchan_cnt; 942 943 if (!VMBUS_CHAN_ISPRIMARY(chan)) { 944 /* 945 * Sub-channel is closed when its primary channel 946 * is closed; done. 947 */ 948 return; 949 } 950 951 /* 952 * Close all sub-channels, if any. 953 */ 954 subchan_cnt = chan->ch_subchan_cnt; 955 if (subchan_cnt > 0) { 956 struct vmbus_channel **subchan; 957 int i; 958 959 subchan = vmbus_subchan_get(chan, subchan_cnt); 960 for (i = 0; i < subchan_cnt; ++i) { 961 vmbus_chan_close_internal(subchan[i]); 962 /* 963 * This sub-channel is referenced, when it is 964 * linked to the primary channel; drop that 965 * reference now. 966 */ 967 vmbus_chan_detach(subchan[i]); 968 } 969 vmbus_subchan_rel(subchan, subchan_cnt); 970 } 971 972 /* Then close the primary channel. */ 973 vmbus_chan_close_internal(chan); 974} 975 976void 977vmbus_chan_intr_drain(struct vmbus_channel *chan) 978{ 979 980 taskqueue_drain(chan->ch_tq, &chan->ch_task); 981} 982 983int 984vmbus_chan_send(struct vmbus_channel *chan, uint16_t type, uint16_t flags, 985 void *data, int dlen, uint64_t xactid) 986{ 987 struct vmbus_chanpkt pkt; 988 int pktlen, pad_pktlen, hlen, error; 989 uint64_t pad = 0; 990 struct iovec iov[3]; 991 boolean_t send_evt; 992 993 hlen = sizeof(pkt); 994 pktlen = hlen + dlen; 995 pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 996 KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 997 ("invalid packet size %d", pad_pktlen)); 998 999 pkt.cp_hdr.cph_type = type; 1000 pkt.cp_hdr.cph_flags = flags; 1001 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1002 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1003 pkt.cp_hdr.cph_xactid = xactid; 1004 1005 iov[0].iov_base = &pkt; 1006 iov[0].iov_len = hlen; 1007 iov[1].iov_base = data; 1008 iov[1].iov_len = dlen; 1009 iov[2].iov_base = &pad; 1010 iov[2].iov_len = pad_pktlen - pktlen; 1011 1012 error = vmbus_txbr_write(&chan->ch_txbr, iov, 3, &send_evt); 1013 if (!error && send_evt) 1014 vmbus_chan_signal_tx(chan); 1015 return error; 1016} 1017 1018int 1019vmbus_chan_send_sglist(struct vmbus_channel *chan, 1020 struct vmbus_gpa sg[], int sglen, void *data, int dlen, uint64_t xactid) 1021{ 1022 struct vmbus_chanpkt_sglist pkt; 1023 int pktlen, pad_pktlen, hlen, error; 1024 struct iovec iov[4]; 1025 boolean_t send_evt; 1026 uint64_t pad = 0; 1027 1028 hlen = __offsetof(struct vmbus_chanpkt_sglist, cp_gpa[sglen]); 1029 pktlen = hlen + dlen; 1030 pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1031 KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1032 ("invalid packet size %d", pad_pktlen)); 1033 1034 pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1035 pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1036 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1037 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1038 pkt.cp_hdr.cph_xactid = xactid; 1039 pkt.cp_rsvd = 0; 1040 pkt.cp_gpa_cnt = sglen; 1041 1042 iov[0].iov_base = &pkt; 1043 iov[0].iov_len = sizeof(pkt); 1044 iov[1].iov_base = sg; 1045 iov[1].iov_len = sizeof(struct vmbus_gpa) * sglen; 1046 iov[2].iov_base = data; 1047 iov[2].iov_len = dlen; 1048 iov[3].iov_base = &pad; 1049 iov[3].iov_len = pad_pktlen - pktlen; 1050 1051 error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); 1052 if (!error && send_evt) 1053 vmbus_chan_signal_tx(chan); 1054 return error; 1055} 1056 1057int 1058vmbus_chan_send_prplist(struct vmbus_channel *chan, 1059 struct vmbus_gpa_range *prp, int prp_cnt, void *data, int dlen, 1060 uint64_t xactid) 1061{ 1062 struct vmbus_chanpkt_prplist pkt; 1063 int pktlen, pad_pktlen, hlen, error; 1064 struct iovec iov[4]; 1065 boolean_t send_evt; 1066 uint64_t pad = 0; 1067 1068 hlen = __offsetof(struct vmbus_chanpkt_prplist, 1069 cp_range[0].gpa_page[prp_cnt]); 1070 pktlen = hlen + dlen; 1071 pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1072 KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1073 ("invalid packet size %d", pad_pktlen)); 1074 1075 pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1076 pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1077 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1078 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1079 pkt.cp_hdr.cph_xactid = xactid; 1080 pkt.cp_rsvd = 0; 1081 pkt.cp_range_cnt = 1; 1082 1083 iov[0].iov_base = &pkt; 1084 iov[0].iov_len = sizeof(pkt); 1085 iov[1].iov_base = prp; 1086 iov[1].iov_len = __offsetof(struct vmbus_gpa_range, gpa_page[prp_cnt]); 1087 iov[2].iov_base = data; 1088 iov[2].iov_len = dlen; 1089 iov[3].iov_base = &pad; 1090 iov[3].iov_len = pad_pktlen - pktlen; 1091 1092 error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); 1093 if (!error && send_evt) 1094 vmbus_chan_signal_tx(chan); 1095 return error; 1096} 1097 1098int 1099vmbus_chan_recv(struct vmbus_channel *chan, void *data, int *dlen0, 1100 uint64_t *xactid) 1101{ 1102 struct vmbus_chanpkt_hdr pkt; 1103 int error, dlen, hlen; 1104 1105 error = vmbus_rxbr_peek(&chan->ch_rxbr, &pkt, sizeof(pkt)); 1106 if (error) 1107 return (error); 1108 1109 if (__predict_false(pkt.cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) { 1110 vmbus_chan_printf(chan, "invalid hlen %u\n", pkt.cph_hlen); 1111 /* XXX this channel is dead actually. */ 1112 return (EIO); 1113 } 1114 if (__predict_false(pkt.cph_hlen > pkt.cph_tlen)) { 1115 vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n", 1116 pkt.cph_hlen, pkt.cph_tlen); 1117 /* XXX this channel is dead actually. */ 1118 return (EIO); 1119 } 1120 1121 hlen = VMBUS_CHANPKT_GETLEN(pkt.cph_hlen); 1122 dlen = VMBUS_CHANPKT_GETLEN(pkt.cph_tlen) - hlen; 1123 1124 if (*dlen0 < dlen) { 1125 /* Return the size of this packet's data. */ 1126 *dlen0 = dlen; 1127 return (ENOBUFS); 1128 } 1129 1130 *xactid = pkt.cph_xactid; 1131 *dlen0 = dlen; 1132 1133 /* Skip packet header */ 1134 error = vmbus_rxbr_read(&chan->ch_rxbr, data, dlen, hlen); 1135 KASSERT(!error, ("vmbus_rxbr_read failed")); 1136 1137 return (0); 1138} 1139 1140int 1141vmbus_chan_recv_pkt(struct vmbus_channel *chan, 1142 struct vmbus_chanpkt_hdr *pkt, int *pktlen0) 1143{ 1144 int error, pktlen, pkt_hlen; 1145 1146 pkt_hlen = sizeof(*pkt); 1147 error = vmbus_rxbr_peek(&chan->ch_rxbr, pkt, pkt_hlen); 1148 if (error) 1149 return (error); 1150 1151 if (__predict_false(pkt->cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) { 1152 vmbus_chan_printf(chan, "invalid hlen %u\n", pkt->cph_hlen); 1153 /* XXX this channel is dead actually. */ 1154 return (EIO); 1155 } 1156 if (__predict_false(pkt->cph_hlen > pkt->cph_tlen)) { 1157 vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n", 1158 pkt->cph_hlen, pkt->cph_tlen); 1159 /* XXX this channel is dead actually. */ 1160 return (EIO); 1161 } 1162 1163 pktlen = VMBUS_CHANPKT_GETLEN(pkt->cph_tlen); 1164 if (*pktlen0 < pktlen) { 1165 /* Return the size of this packet. */ 1166 *pktlen0 = pktlen; 1167 return (ENOBUFS); 1168 } 1169 *pktlen0 = pktlen; 1170 1171 /* 1172 * Skip the fixed-size packet header, which has been filled 1173 * by the above vmbus_rxbr_peek(). 1174 */ 1175 error = vmbus_rxbr_read(&chan->ch_rxbr, pkt + 1, 1176 pktlen - pkt_hlen, pkt_hlen); 1177 KASSERT(!error, ("vmbus_rxbr_read failed")); 1178 1179 return (0); 1180} 1181 1182static void 1183vmbus_chan_task(void *xchan, int pending __unused) 1184{ 1185 struct vmbus_channel *chan = xchan; 1186 vmbus_chan_callback_t cb = chan->ch_cb; 1187 void *cbarg = chan->ch_cbarg; 1188 1189 /* 1190 * Optimize host to guest signaling by ensuring: 1191 * 1. While reading the channel, we disable interrupts from 1192 * host. 1193 * 2. Ensure that we process all posted messages from the host 1194 * before returning from this callback. 1195 * 3. Once we return, enable signaling from the host. Once this 1196 * state is set we check to see if additional packets are 1197 * available to read. In this case we repeat the process. 1198 * 1199 * NOTE: Interrupt has been disabled in the ISR. 1200 */ 1201 for (;;) { 1202 uint32_t left; 1203 1204 cb(chan, cbarg); 1205 1206 left = vmbus_rxbr_intr_unmask(&chan->ch_rxbr); 1207 if (left == 0) { 1208 /* No more data in RX bufring; done */ 1209 break; 1210 } 1211 vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1212 } 1213} 1214 1215static void 1216vmbus_chan_task_nobatch(void *xchan, int pending __unused) 1217{ 1218 struct vmbus_channel *chan = xchan; 1219 1220 chan->ch_cb(chan, chan->ch_cbarg); 1221} 1222 1223static __inline void 1224vmbus_event_flags_proc(struct vmbus_softc *sc, volatile u_long *event_flags, 1225 int flag_cnt) 1226{ 1227 int f; 1228 1229 for (f = 0; f < flag_cnt; ++f) { 1230 uint32_t chid_base; 1231 u_long flags; 1232 int chid_ofs; 1233 1234 if (event_flags[f] == 0) 1235 continue; 1236 1237 flags = atomic_swap_long(&event_flags[f], 0); 1238 chid_base = f << VMBUS_EVTFLAG_SHIFT; 1239 1240 while ((chid_ofs = ffsl(flags)) != 0) { 1241 struct vmbus_channel *chan; 1242 1243 --chid_ofs; /* NOTE: ffsl is 1-based */ 1244 flags &= ~(1UL << chid_ofs); 1245 1246 chan = sc->vmbus_chmap[chid_base + chid_ofs]; 1247 if (__predict_false(chan == NULL)) { 1248 /* Channel is closed. */ 1249 continue; 1250 } 1251 __compiler_membar(); 1252 1253 if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD) 1254 vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1255 taskqueue_enqueue(chan->ch_tq, &chan->ch_task); 1256 } 1257 } 1258} 1259 1260void 1261vmbus_event_proc(struct vmbus_softc *sc, int cpu) 1262{ 1263 struct vmbus_evtflags *eventf; 1264 1265 /* 1266 * On Host with Win8 or above, the event page can be checked directly 1267 * to get the id of the channel that has the pending interrupt. 1268 */ 1269 eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE; 1270 vmbus_event_flags_proc(sc, eventf->evt_flags, 1271 VMBUS_PCPU_GET(sc, event_flags_cnt, cpu)); 1272} 1273 1274void 1275vmbus_event_proc_compat(struct vmbus_softc *sc, int cpu) 1276{ 1277 struct vmbus_evtflags *eventf; 1278 1279 eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE; 1280 if (atomic_testandclear_long(&eventf->evt_flags[0], 0)) { 1281 vmbus_event_flags_proc(sc, sc->vmbus_rx_evtflags, 1282 VMBUS_CHAN_MAX_COMPAT >> VMBUS_EVTFLAG_SHIFT); 1283 } 1284} 1285 1286static void 1287vmbus_chan_update_evtflagcnt(struct vmbus_softc *sc, 1288 const struct vmbus_channel *chan) 1289{ 1290 volatile int *flag_cnt_ptr; 1291 int flag_cnt; 1292 1293 flag_cnt = (chan->ch_id / VMBUS_EVTFLAG_LEN) + 1; 1294 flag_cnt_ptr = VMBUS_PCPU_PTR(sc, event_flags_cnt, chan->ch_cpuid); 1295 1296 for (;;) { 1297 int old_flag_cnt; 1298 1299 old_flag_cnt = *flag_cnt_ptr; 1300 if (old_flag_cnt >= flag_cnt) 1301 break; 1302 if (atomic_cmpset_int(flag_cnt_ptr, old_flag_cnt, flag_cnt)) { 1303 if (bootverbose) { 1304 vmbus_chan_printf(chan, 1305 "chan%u update cpu%d flag_cnt to %d\n", 1306 chan->ch_id, chan->ch_cpuid, flag_cnt); 1307 } 1308 break; 1309 } 1310 } 1311} 1312 1313static struct vmbus_channel * 1314vmbus_chan_alloc(struct vmbus_softc *sc) 1315{ 1316 struct vmbus_channel *chan; 1317 1318 chan = malloc(sizeof(*chan), M_DEVBUF, M_WAITOK | M_ZERO); 1319 1320 chan->ch_monprm = hyperv_dmamem_alloc(bus_get_dma_tag(sc->vmbus_dev), 1321 HYPERCALL_PARAM_ALIGN, 0, sizeof(struct hyperv_mon_param), 1322 &chan->ch_monprm_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO); 1323 if (chan->ch_monprm == NULL) { 1324 device_printf(sc->vmbus_dev, "monprm alloc failed\n"); 1325 free(chan, M_DEVBUF); 1326 return NULL; 1327 } 1328 1329 chan->ch_refs = 1; 1330 chan->ch_vmbus = sc; 1331 mtx_init(&chan->ch_subchan_lock, "vmbus subchan", NULL, MTX_DEF); 1332 sx_init(&chan->ch_orphan_lock, "vmbus chorphan"); 1333 TAILQ_INIT(&chan->ch_subchans); 1334 vmbus_rxbr_init(&chan->ch_rxbr); 1335 vmbus_txbr_init(&chan->ch_txbr); 1336 1337 return chan; 1338} 1339 1340static void 1341vmbus_chan_free(struct vmbus_channel *chan) 1342{ 1343 1344 KASSERT(TAILQ_EMPTY(&chan->ch_subchans) && chan->ch_subchan_cnt == 0, 1345 ("still owns sub-channels")); 1346 KASSERT((chan->ch_stflags & 1347 (VMBUS_CHAN_ST_OPENED | 1348 VMBUS_CHAN_ST_ONPRIL | 1349 VMBUS_CHAN_ST_ONSUBL | 1350 VMBUS_CHAN_ST_ONLIST)) == 0, ("free busy channel")); 1351 KASSERT(chan->ch_orphan_xact == NULL, 1352 ("still has orphan xact installed")); 1353 KASSERT(chan->ch_refs == 0, ("chan%u: invalid refcnt %d", 1354 chan->ch_id, chan->ch_refs)); 1355 1356 hyperv_dmamem_free(&chan->ch_monprm_dma, chan->ch_monprm); 1357 mtx_destroy(&chan->ch_subchan_lock); 1358 sx_destroy(&chan->ch_orphan_lock); 1359 vmbus_rxbr_deinit(&chan->ch_rxbr); 1360 vmbus_txbr_deinit(&chan->ch_txbr); 1361 free(chan, M_DEVBUF); 1362} 1363 1364static int 1365vmbus_chan_add(struct vmbus_channel *newchan) 1366{ 1367 struct vmbus_softc *sc = newchan->ch_vmbus; 1368 struct vmbus_channel *prichan; 1369 1370 if (newchan->ch_id == 0) { 1371 /* 1372 * XXX 1373 * Chan0 will neither be processed nor should be offered; 1374 * skip it. 1375 */ 1376 device_printf(sc->vmbus_dev, "got chan0 offer, discard\n"); 1377 return EINVAL; 1378 } else if (newchan->ch_id >= VMBUS_CHAN_MAX) { 1379 device_printf(sc->vmbus_dev, "invalid chan%u offer\n", 1380 newchan->ch_id); 1381 return EINVAL; 1382 } 1383 1384 mtx_lock(&sc->vmbus_prichan_lock); 1385 TAILQ_FOREACH(prichan, &sc->vmbus_prichans, ch_prilink) { 1386 /* 1387 * Sub-channel will have the same type GUID and instance 1388 * GUID as its primary channel. 1389 */ 1390 if (memcmp(&prichan->ch_guid_type, &newchan->ch_guid_type, 1391 sizeof(struct hyperv_guid)) == 0 && 1392 memcmp(&prichan->ch_guid_inst, &newchan->ch_guid_inst, 1393 sizeof(struct hyperv_guid)) == 0) 1394 break; 1395 } 1396 if (VMBUS_CHAN_ISPRIMARY(newchan)) { 1397 if (prichan == NULL) { 1398 /* Install the new primary channel */ 1399 vmbus_chan_ins_prilist(sc, newchan); 1400 mtx_unlock(&sc->vmbus_prichan_lock); 1401 goto done; 1402 } else { 1403 mtx_unlock(&sc->vmbus_prichan_lock); 1404 device_printf(sc->vmbus_dev, 1405 "duplicated primary chan%u\n", newchan->ch_id); 1406 return EINVAL; 1407 } 1408 } else { /* Sub-channel */ 1409 if (prichan == NULL) { 1410 mtx_unlock(&sc->vmbus_prichan_lock); 1411 device_printf(sc->vmbus_dev, 1412 "no primary chan for chan%u\n", newchan->ch_id); 1413 return EINVAL; 1414 } 1415 /* 1416 * Found the primary channel for this sub-channel and 1417 * move on. 1418 * 1419 * XXX refcnt prichan 1420 */ 1421 } 1422 mtx_unlock(&sc->vmbus_prichan_lock); 1423 1424 /* 1425 * This is a sub-channel; link it with the primary channel. 1426 */ 1427 KASSERT(!VMBUS_CHAN_ISPRIMARY(newchan), 1428 ("new channel is not sub-channel")); 1429 KASSERT(prichan != NULL, ("no primary channel")); 1430 1431 /* 1432 * Reference count this sub-channel; it will be dereferenced 1433 * when this sub-channel is closed. 1434 */ 1435 KASSERT(newchan->ch_refs == 1, ("chan%u: invalid refcnt %d", 1436 newchan->ch_id, newchan->ch_refs)); 1437 atomic_add_int(&newchan->ch_refs, 1); 1438 1439 newchan->ch_prichan = prichan; 1440 newchan->ch_dev = prichan->ch_dev; 1441 1442 mtx_lock(&prichan->ch_subchan_lock); 1443 vmbus_chan_ins_sublist(prichan, newchan); 1444 mtx_unlock(&prichan->ch_subchan_lock); 1445 /* 1446 * Notify anyone that is interested in this sub-channel, 1447 * after this sub-channel is setup. 1448 */ 1449 wakeup(prichan); 1450done: 1451 /* 1452 * Hook this channel up for later revocation. 1453 */ 1454 mtx_lock(&sc->vmbus_chan_lock); 1455 vmbus_chan_ins_list(sc, newchan); 1456 mtx_unlock(&sc->vmbus_chan_lock); 1457 1458 if (bootverbose) { 1459 vmbus_chan_printf(newchan, "chan%u subidx%u offer\n", 1460 newchan->ch_id, newchan->ch_subidx); 1461 } 1462 1463 /* Select default cpu for this channel. */ 1464 vmbus_chan_cpu_default(newchan); 1465 1466 return 0; 1467} 1468 1469void 1470vmbus_chan_cpu_set(struct vmbus_channel *chan, int cpu) 1471{ 1472 KASSERT(cpu >= 0 && cpu < mp_ncpus, ("invalid cpu %d", cpu)); 1473 1474 if (chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WS2008 || 1475 chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WIN7) { 1476 /* Only cpu0 is supported */ 1477 cpu = 0; 1478 } 1479 1480 chan->ch_cpuid = cpu; 1481 chan->ch_vcpuid = VMBUS_PCPU_GET(chan->ch_vmbus, vcpuid, cpu); 1482 1483 if (bootverbose) { 1484 vmbus_chan_printf(chan, 1485 "chan%u assigned to cpu%u [vcpu%u]\n", 1486 chan->ch_id, chan->ch_cpuid, chan->ch_vcpuid); 1487 } 1488} 1489 1490void 1491vmbus_chan_cpu_rr(struct vmbus_channel *chan) 1492{ 1493 static uint32_t vmbus_chan_nextcpu; 1494 int cpu; 1495 1496 cpu = atomic_fetchadd_int(&vmbus_chan_nextcpu, 1) % mp_ncpus; 1497 vmbus_chan_cpu_set(chan, cpu); 1498} 1499 1500static void 1501vmbus_chan_cpu_default(struct vmbus_channel *chan) 1502{ 1503 /* 1504 * By default, pin the channel to cpu0. Devices having 1505 * special channel-cpu mapping requirement should call 1506 * vmbus_chan_cpu_{set,rr}(). 1507 */ 1508 vmbus_chan_cpu_set(chan, 0); 1509} 1510 1511static void 1512vmbus_chan_msgproc_choffer(struct vmbus_softc *sc, 1513 const struct vmbus_message *msg) 1514{ 1515 const struct vmbus_chanmsg_choffer *offer; 1516 struct vmbus_channel *chan; 1517 task_fn_t *detach_fn, *attach_fn; 1518 int error; 1519 1520 offer = (const struct vmbus_chanmsg_choffer *)msg->msg_data; 1521 1522 chan = vmbus_chan_alloc(sc); 1523 if (chan == NULL) { 1524 device_printf(sc->vmbus_dev, "allocate chan%u failed\n", 1525 offer->chm_chanid); 1526 return; 1527 } 1528 1529 chan->ch_id = offer->chm_chanid; 1530 chan->ch_subidx = offer->chm_subidx; 1531 chan->ch_guid_type = offer->chm_chtype; 1532 chan->ch_guid_inst = offer->chm_chinst; 1533 1534 /* Batch reading is on by default */ 1535 chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD; 1536 1537 chan->ch_monprm->mp_connid = VMBUS_CONNID_EVENT; 1538 if (sc->vmbus_version != VMBUS_VERSION_WS2008) 1539 chan->ch_monprm->mp_connid = offer->chm_connid; 1540 1541 if (offer->chm_flags1 & VMBUS_CHOFFER_FLAG1_HASMNF) { 1542 int trig_idx; 1543 1544 /* 1545 * Setup MNF stuffs. 1546 */ 1547 chan->ch_txflags |= VMBUS_CHAN_TXF_HASMNF; 1548 1549 trig_idx = offer->chm_montrig / VMBUS_MONTRIG_LEN; 1550 if (trig_idx >= VMBUS_MONTRIGS_MAX) 1551 panic("invalid monitor trigger %u", offer->chm_montrig); 1552 chan->ch_montrig = 1553 &sc->vmbus_mnf2->mnf_trigs[trig_idx].mt_pending; 1554 1555 chan->ch_montrig_mask = 1556 1 << (offer->chm_montrig % VMBUS_MONTRIG_LEN); 1557 } 1558 1559 /* 1560 * Setup event flag. 1561 */ 1562 chan->ch_evtflag = 1563 &sc->vmbus_tx_evtflags[chan->ch_id >> VMBUS_EVTFLAG_SHIFT]; 1564 chan->ch_evtflag_mask = 1UL << (chan->ch_id & VMBUS_EVTFLAG_MASK); 1565 1566 /* 1567 * Setup attach and detach tasks. 1568 */ 1569 if (VMBUS_CHAN_ISPRIMARY(chan)) { 1570 chan->ch_mgmt_tq = sc->vmbus_devtq; 1571 attach_fn = vmbus_prichan_attach_task; 1572 detach_fn = vmbus_prichan_detach_task; 1573 } else { 1574 chan->ch_mgmt_tq = sc->vmbus_subchtq; 1575 attach_fn = vmbus_subchan_attach_task; 1576 detach_fn = vmbus_subchan_detach_task; 1577 } 1578 TASK_INIT(&chan->ch_attach_task, 0, attach_fn, chan); 1579 TASK_INIT(&chan->ch_detach_task, 0, detach_fn, chan); 1580 1581 error = vmbus_chan_add(chan); 1582 if (error) { 1583 device_printf(sc->vmbus_dev, "add chan%u failed: %d\n", 1584 chan->ch_id, error); 1585 atomic_subtract_int(&chan->ch_refs, 1); 1586 vmbus_chan_free(chan); 1587 return; 1588 } 1589 taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_attach_task); 1590} 1591 1592static void 1593vmbus_chan_msgproc_chrescind(struct vmbus_softc *sc, 1594 const struct vmbus_message *msg) 1595{ 1596 const struct vmbus_chanmsg_chrescind *note; 1597 struct vmbus_channel *chan; 1598 1599 note = (const struct vmbus_chanmsg_chrescind *)msg->msg_data; 1600 if (note->chm_chanid > VMBUS_CHAN_MAX) { 1601 device_printf(sc->vmbus_dev, "invalid revoked chan%u\n", 1602 note->chm_chanid); 1603 return; 1604 } 1605 1606 /* 1607 * Find and remove the target channel from the channel list. 1608 */ 1609 mtx_lock(&sc->vmbus_chan_lock); 1610 TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) { 1611 if (chan->ch_id == note->chm_chanid) 1612 break; 1613 } 1614 if (chan == NULL) { 1615 mtx_unlock(&sc->vmbus_chan_lock); 1616 device_printf(sc->vmbus_dev, "chan%u is not offered\n", 1617 note->chm_chanid); 1618 return; 1619 } 1620 vmbus_chan_rem_list(sc, chan); 1621 mtx_unlock(&sc->vmbus_chan_lock); 1622 1623 if (VMBUS_CHAN_ISPRIMARY(chan)) { 1624 /* 1625 * The target channel is a primary channel; remove the 1626 * target channel from the primary channel list now, 1627 * instead of later, so that it will not be found by 1628 * other sub-channel offers, which are processed in 1629 * this thread. 1630 */ 1631 mtx_lock(&sc->vmbus_prichan_lock); 1632 vmbus_chan_rem_prilist(sc, chan); 1633 mtx_unlock(&sc->vmbus_prichan_lock); 1634 } 1635 1636 /* 1637 * NOTE: 1638 * The following processing order is critical: 1639 * Set the REVOKED state flag before orphaning the installed xact. 1640 */ 1641 1642 if (atomic_testandset_int(&chan->ch_stflags, 1643 VMBUS_CHAN_ST_REVOKED_SHIFT)) 1644 panic("channel has already been revoked"); 1645 1646 sx_xlock(&chan->ch_orphan_lock); 1647 if (chan->ch_orphan_xact != NULL) 1648 vmbus_xact_ctx_orphan(chan->ch_orphan_xact); 1649 sx_xunlock(&chan->ch_orphan_lock); 1650 1651 if (bootverbose) 1652 vmbus_chan_printf(chan, "chan%u revoked\n", note->chm_chanid); 1653 vmbus_chan_detach(chan); 1654} 1655 1656static int 1657vmbus_chan_release(struct vmbus_channel *chan) 1658{ 1659 struct vmbus_softc *sc = chan->ch_vmbus; 1660 struct vmbus_chanmsg_chfree *req; 1661 struct vmbus_msghc *mh; 1662 int error; 1663 1664 mh = vmbus_msghc_get(sc, sizeof(*req)); 1665 if (mh == NULL) { 1666 vmbus_chan_printf(chan, 1667 "can not get msg hypercall for chfree(chan%u)\n", 1668 chan->ch_id); 1669 return (ENXIO); 1670 } 1671 1672 req = vmbus_msghc_dataptr(mh); 1673 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHFREE; 1674 req->chm_chanid = chan->ch_id; 1675 1676 error = vmbus_msghc_exec_noresult(mh); 1677 vmbus_msghc_put(sc, mh); 1678 1679 if (error) { 1680 vmbus_chan_printf(chan, 1681 "chfree(chan%u) msg hypercall exec failed: %d\n", 1682 chan->ch_id, error); 1683 } else { 1684 if (bootverbose) 1685 vmbus_chan_printf(chan, "chan%u freed\n", chan->ch_id); 1686 } 1687 return (error); 1688} 1689 1690static void 1691vmbus_prichan_detach_task(void *xchan, int pending __unused) 1692{ 1693 struct vmbus_channel *chan = xchan; 1694 1695 KASSERT(VMBUS_CHAN_ISPRIMARY(chan), 1696 ("chan%u is not primary channel", chan->ch_id)); 1697 1698 /* Delete and detach the device associated with this channel. */ 1699 vmbus_delete_child(chan); 1700 1701 /* Release this channel (back to vmbus). */ 1702 vmbus_chan_release(chan); 1703 1704 /* Free this channel's resource. */ 1705 vmbus_chan_free(chan); 1706} 1707 1708static void 1709vmbus_subchan_detach_task(void *xchan, int pending __unused) 1710{ 1711 struct vmbus_channel *chan = xchan; 1712 struct vmbus_channel *pri_chan = chan->ch_prichan; 1713 1714 KASSERT(!VMBUS_CHAN_ISPRIMARY(chan), 1715 ("chan%u is primary channel", chan->ch_id)); 1716 1717 /* Release this channel (back to vmbus). */ 1718 vmbus_chan_release(chan); 1719 1720 /* Unlink from its primary channel's sub-channel list. */ 1721 mtx_lock(&pri_chan->ch_subchan_lock); 1722 vmbus_chan_rem_sublist(pri_chan, chan); 1723 mtx_unlock(&pri_chan->ch_subchan_lock); 1724 /* Notify anyone that is waiting for this sub-channel to vanish. */ 1725 wakeup(pri_chan); 1726 1727 /* Free this channel's resource. */ 1728 vmbus_chan_free(chan); 1729} 1730 1731static void 1732vmbus_prichan_attach_task(void *xchan, int pending __unused) 1733{ 1734 1735 /* 1736 * Add device for this primary channel. 1737 */ 1738 vmbus_add_child(xchan); 1739} 1740 1741static void 1742vmbus_subchan_attach_task(void *xchan __unused, int pending __unused) 1743{ 1744 1745 /* Nothing */ 1746} 1747 1748void 1749vmbus_chan_destroy_all(struct vmbus_softc *sc) 1750{ 1751 1752 /* 1753 * Detach all devices and destroy the corresponding primary 1754 * channels. 1755 */ 1756 for (;;) { 1757 struct vmbus_channel *chan; 1758 1759 mtx_lock(&sc->vmbus_chan_lock); 1760 TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) { 1761 if (VMBUS_CHAN_ISPRIMARY(chan)) 1762 break; 1763 } 1764 if (chan == NULL) { 1765 /* No more primary channels; done. */ 1766 mtx_unlock(&sc->vmbus_chan_lock); 1767 break; 1768 } 1769 vmbus_chan_rem_list(sc, chan); 1770 mtx_unlock(&sc->vmbus_chan_lock); 1771 1772 mtx_lock(&sc->vmbus_prichan_lock); 1773 vmbus_chan_rem_prilist(sc, chan); 1774 mtx_unlock(&sc->vmbus_prichan_lock); 1775 1776 taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task); 1777 } 1778} 1779 1780struct vmbus_channel ** 1781vmbus_subchan_get(struct vmbus_channel *pri_chan, int subchan_cnt) 1782{ 1783 struct vmbus_channel **ret, *chan; 1784 int i; 1785 1786 KASSERT(subchan_cnt > 0, ("invalid sub-channel count %d", subchan_cnt)); 1787 1788 ret = malloc(subchan_cnt * sizeof(struct vmbus_channel *), M_TEMP, 1789 M_WAITOK); 1790 1791 mtx_lock(&pri_chan->ch_subchan_lock); 1792 1793 while (pri_chan->ch_subchan_cnt < subchan_cnt) 1794 mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "subch", 0); 1795 1796 i = 0; 1797 TAILQ_FOREACH(chan, &pri_chan->ch_subchans, ch_sublink) { 1798 /* TODO: refcnt chan */ 1799 ret[i] = chan; 1800 1801 ++i; 1802 if (i == subchan_cnt) 1803 break; 1804 } 1805 KASSERT(i == subchan_cnt, ("invalid subchan count %d, should be %d", 1806 pri_chan->ch_subchan_cnt, subchan_cnt)); 1807 1808 mtx_unlock(&pri_chan->ch_subchan_lock); 1809 1810 return ret; 1811} 1812 1813void 1814vmbus_subchan_rel(struct vmbus_channel **subchan, int subchan_cnt __unused) 1815{ 1816 1817 free(subchan, M_TEMP); 1818} 1819 1820void 1821vmbus_subchan_drain(struct vmbus_channel *pri_chan) 1822{ 1823 mtx_lock(&pri_chan->ch_subchan_lock); 1824 while (pri_chan->ch_subchan_cnt > 0) 1825 mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "dsubch", 0); 1826 mtx_unlock(&pri_chan->ch_subchan_lock); 1827} 1828 1829void 1830vmbus_chan_msgproc(struct vmbus_softc *sc, const struct vmbus_message *msg) 1831{ 1832 vmbus_chanmsg_proc_t msg_proc; 1833 uint32_t msg_type; 1834 1835 msg_type = ((const struct vmbus_chanmsg_hdr *)msg->msg_data)->chm_type; 1836 KASSERT(msg_type < VMBUS_CHANMSG_TYPE_MAX, 1837 ("invalid message type %u", msg_type)); 1838 1839 msg_proc = vmbus_chan_msgprocs[msg_type]; 1840 if (msg_proc != NULL) 1841 msg_proc(sc, msg); 1842} 1843 1844void 1845vmbus_chan_set_readbatch(struct vmbus_channel *chan, bool on) 1846{ 1847 if (!on) 1848 chan->ch_flags &= ~VMBUS_CHAN_FLAG_BATCHREAD; 1849 else 1850 chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD; 1851} 1852 1853uint32_t 1854vmbus_chan_id(const struct vmbus_channel *chan) 1855{ 1856 return chan->ch_id; 1857} 1858 1859uint32_t 1860vmbus_chan_subidx(const struct vmbus_channel *chan) 1861{ 1862 return chan->ch_subidx; 1863} 1864 1865bool 1866vmbus_chan_is_primary(const struct vmbus_channel *chan) 1867{ 1868 if (VMBUS_CHAN_ISPRIMARY(chan)) 1869 return true; 1870 else 1871 return false; 1872} 1873 1874const struct hyperv_guid * 1875vmbus_chan_guid_inst(const struct vmbus_channel *chan) 1876{ 1877 return &chan->ch_guid_inst; 1878} 1879 1880int 1881vmbus_chan_prplist_nelem(int br_size, int prpcnt_max, int dlen_max) 1882{ 1883 int elem_size; 1884 1885 elem_size = __offsetof(struct vmbus_chanpkt_prplist, 1886 cp_range[0].gpa_page[prpcnt_max]); 1887 elem_size += dlen_max; 1888 elem_size = VMBUS_CHANPKT_TOTLEN(elem_size); 1889 1890 return (vmbus_br_nelem(br_size, elem_size)); 1891} 1892 1893bool 1894vmbus_chan_tx_empty(const struct vmbus_channel *chan) 1895{ 1896 1897 return (vmbus_txbr_empty(&chan->ch_txbr)); 1898} 1899 1900bool 1901vmbus_chan_rx_empty(const struct vmbus_channel *chan) 1902{ 1903 1904 return (vmbus_rxbr_empty(&chan->ch_rxbr)); 1905} 1906 1907static int 1908vmbus_chan_printf(const struct vmbus_channel *chan, const char *fmt, ...) 1909{ 1910 va_list ap; 1911 device_t dev; 1912 int retval; 1913 1914 if (chan->ch_dev == NULL || !device_is_alive(chan->ch_dev)) 1915 dev = chan->ch_vmbus->vmbus_dev; 1916 else 1917 dev = chan->ch_dev; 1918 1919 retval = device_print_prettyname(dev); 1920 va_start(ap, fmt); 1921 retval += vprintf(fmt, ap); 1922 va_end(ap); 1923 1924 return (retval); 1925} 1926 1927void 1928vmbus_chan_run_task(struct vmbus_channel *chan, struct task *task) 1929{ 1930 1931 taskqueue_enqueue(chan->ch_tq, task); 1932 taskqueue_drain(chan->ch_tq, task); 1933} 1934 1935struct taskqueue * 1936vmbus_chan_mgmt_tq(const struct vmbus_channel *chan) 1937{ 1938 1939 return (chan->ch_mgmt_tq); 1940} 1941 1942bool 1943vmbus_chan_is_revoked(const struct vmbus_channel *chan) 1944{ 1945 1946 if (chan->ch_stflags & VMBUS_CHAN_ST_REVOKED) 1947 return (true); 1948 return (false); 1949} 1950 1951void 1952vmbus_chan_set_orphan(struct vmbus_channel *chan, struct vmbus_xact_ctx *xact) 1953{ 1954 1955 sx_xlock(&chan->ch_orphan_lock); 1956 chan->ch_orphan_xact = xact; 1957 sx_xunlock(&chan->ch_orphan_lock); 1958} 1959 1960void 1961vmbus_chan_unset_orphan(struct vmbus_channel *chan) 1962{ 1963 1964 sx_xlock(&chan->ch_orphan_lock); 1965 chan->ch_orphan_xact = NULL; 1966 sx_xunlock(&chan->ch_orphan_lock); 1967} 1968 1969const void * 1970vmbus_chan_xact_wait(const struct vmbus_channel *chan, 1971 struct vmbus_xact *xact, size_t *resp_len, bool can_sleep) 1972{ 1973 const void *ret; 1974 1975 if (can_sleep) 1976 ret = vmbus_xact_wait(xact, resp_len); 1977 else 1978 ret = vmbus_xact_busywait(xact, resp_len); 1979 if (vmbus_chan_is_revoked(chan)) { 1980 /* 1981 * This xact probably is interrupted, and the 1982 * interruption can race the reply reception, 1983 * so we have to make sure that there are nothing 1984 * left on the RX bufring, i.e. this xact will 1985 * not be touched, once this function returns. 1986 * 1987 * Since the hypervisor will not put more data 1988 * onto the RX bufring once the channel is revoked, 1989 * the following loop will be terminated, once all 1990 * data are drained by the driver's channel 1991 * callback. 1992 */ 1993 while (!vmbus_chan_rx_empty(chan)) { 1994 if (can_sleep) 1995 pause("chxact", 1); 1996 else 1997 DELAY(1000); 1998 } 1999 } 2000 return (ret); 2001} 2002