vmbus_chan.c revision 311367
1/*- 2 * Copyright (c) 2009-2012,2016 Microsoft Corp. 3 * Copyright (c) 2012 NetApp Inc. 4 * Copyright (c) 2012 Citrix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29#include <sys/cdefs.h> 30__FBSDID("$FreeBSD: stable/11/sys/dev/hyperv/vmbus/vmbus_chan.c 311367 2017-01-05 05:46:06Z sephe $"); 31 32#include <sys/param.h> 33#include <sys/bus.h> 34#include <sys/kernel.h> 35#include <sys/lock.h> 36#include <sys/malloc.h> 37#include <sys/mutex.h> 38#include <sys/smp.h> 39#include <sys/sysctl.h> 40#include <sys/systm.h> 41 42#include <machine/atomic.h> 43#include <machine/stdarg.h> 44 45#include <dev/hyperv/include/hyperv_busdma.h> 46#include <dev/hyperv/include/vmbus_xact.h> 47#include <dev/hyperv/vmbus/hyperv_var.h> 48#include <dev/hyperv/vmbus/vmbus_reg.h> 49#include <dev/hyperv/vmbus/vmbus_var.h> 50#include <dev/hyperv/vmbus/vmbus_brvar.h> 51#include <dev/hyperv/vmbus/vmbus_chanvar.h> 52 53static void vmbus_chan_update_evtflagcnt( 54 struct vmbus_softc *, 55 const struct vmbus_channel *); 56static int vmbus_chan_close_internal( 57 struct vmbus_channel *); 58static int vmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS); 59static void vmbus_chan_sysctl_create( 60 struct vmbus_channel *); 61static struct vmbus_channel *vmbus_chan_alloc(struct vmbus_softc *); 62static void vmbus_chan_free(struct vmbus_channel *); 63static int vmbus_chan_add(struct vmbus_channel *); 64static void vmbus_chan_cpu_default(struct vmbus_channel *); 65static int vmbus_chan_release(struct vmbus_channel *); 66static void vmbus_chan_set_chmap(struct vmbus_channel *); 67static void vmbus_chan_clear_chmap(struct vmbus_channel *); 68static void vmbus_chan_detach(struct vmbus_channel *); 69static bool vmbus_chan_wait_revoke( 70 const struct vmbus_channel *, bool); 71 72static void vmbus_chan_ins_prilist(struct vmbus_softc *, 73 struct vmbus_channel *); 74static void vmbus_chan_rem_prilist(struct vmbus_softc *, 75 struct vmbus_channel *); 76static void vmbus_chan_ins_list(struct vmbus_softc *, 77 struct vmbus_channel *); 78static void vmbus_chan_rem_list(struct vmbus_softc *, 79 struct vmbus_channel *); 80static void vmbus_chan_ins_sublist(struct vmbus_channel *, 81 struct vmbus_channel *); 82static void vmbus_chan_rem_sublist(struct vmbus_channel *, 83 struct vmbus_channel *); 84 85static void vmbus_chan_task(void *, int); 86static void vmbus_chan_task_nobatch(void *, int); 87static void vmbus_chan_clrchmap_task(void *, int); 88static void vmbus_prichan_attach_task(void *, int); 89static void vmbus_subchan_attach_task(void *, int); 90static void vmbus_prichan_detach_task(void *, int); 91static void vmbus_subchan_detach_task(void *, int); 92 93static void vmbus_chan_msgproc_choffer(struct vmbus_softc *, 94 const struct vmbus_message *); 95static void vmbus_chan_msgproc_chrescind( 96 struct vmbus_softc *, 97 const struct vmbus_message *); 98 99static int vmbus_chan_printf(const struct vmbus_channel *, 100 const char *, ...) __printflike(2, 3); 101 102/* 103 * Vmbus channel message processing. 104 */ 105static const vmbus_chanmsg_proc_t 106vmbus_chan_msgprocs[VMBUS_CHANMSG_TYPE_MAX] = { 107 VMBUS_CHANMSG_PROC(CHOFFER, vmbus_chan_msgproc_choffer), 108 VMBUS_CHANMSG_PROC(CHRESCIND, vmbus_chan_msgproc_chrescind), 109 110 VMBUS_CHANMSG_PROC_WAKEUP(CHOPEN_RESP), 111 VMBUS_CHANMSG_PROC_WAKEUP(GPADL_CONNRESP), 112 VMBUS_CHANMSG_PROC_WAKEUP(GPADL_DISCONNRESP) 113}; 114 115/* 116 * Notify host that there are data pending on our TX bufring. 117 */ 118static __inline void 119vmbus_chan_signal_tx(const struct vmbus_channel *chan) 120{ 121 atomic_set_long(chan->ch_evtflag, chan->ch_evtflag_mask); 122 if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF) 123 atomic_set_int(chan->ch_montrig, chan->ch_montrig_mask); 124 else 125 hypercall_signal_event(chan->ch_monprm_dma.hv_paddr); 126} 127 128static void 129vmbus_chan_ins_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan) 130{ 131 132 mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED); 133 if (atomic_testandset_int(&chan->ch_stflags, 134 VMBUS_CHAN_ST_ONPRIL_SHIFT)) 135 panic("channel is already on the prilist"); 136 TAILQ_INSERT_TAIL(&sc->vmbus_prichans, chan, ch_prilink); 137} 138 139static void 140vmbus_chan_rem_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan) 141{ 142 143 mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED); 144 if (atomic_testandclear_int(&chan->ch_stflags, 145 VMBUS_CHAN_ST_ONPRIL_SHIFT) == 0) 146 panic("channel is not on the prilist"); 147 TAILQ_REMOVE(&sc->vmbus_prichans, chan, ch_prilink); 148} 149 150static void 151vmbus_chan_ins_sublist(struct vmbus_channel *prichan, 152 struct vmbus_channel *chan) 153{ 154 155 mtx_assert(&prichan->ch_subchan_lock, MA_OWNED); 156 157 if (atomic_testandset_int(&chan->ch_stflags, 158 VMBUS_CHAN_ST_ONSUBL_SHIFT)) 159 panic("channel is already on the sublist"); 160 TAILQ_INSERT_TAIL(&prichan->ch_subchans, chan, ch_sublink); 161 162 /* Bump sub-channel count. */ 163 prichan->ch_subchan_cnt++; 164} 165 166static void 167vmbus_chan_rem_sublist(struct vmbus_channel *prichan, 168 struct vmbus_channel *chan) 169{ 170 171 mtx_assert(&prichan->ch_subchan_lock, MA_OWNED); 172 173 KASSERT(prichan->ch_subchan_cnt > 0, 174 ("invalid subchan_cnt %d", prichan->ch_subchan_cnt)); 175 prichan->ch_subchan_cnt--; 176 177 if (atomic_testandclear_int(&chan->ch_stflags, 178 VMBUS_CHAN_ST_ONSUBL_SHIFT) == 0) 179 panic("channel is not on the sublist"); 180 TAILQ_REMOVE(&prichan->ch_subchans, chan, ch_sublink); 181} 182 183static void 184vmbus_chan_ins_list(struct vmbus_softc *sc, struct vmbus_channel *chan) 185{ 186 187 mtx_assert(&sc->vmbus_chan_lock, MA_OWNED); 188 if (atomic_testandset_int(&chan->ch_stflags, 189 VMBUS_CHAN_ST_ONLIST_SHIFT)) 190 panic("channel is already on the list"); 191 TAILQ_INSERT_TAIL(&sc->vmbus_chans, chan, ch_link); 192} 193 194static void 195vmbus_chan_rem_list(struct vmbus_softc *sc, struct vmbus_channel *chan) 196{ 197 198 mtx_assert(&sc->vmbus_chan_lock, MA_OWNED); 199 if (atomic_testandclear_int(&chan->ch_stflags, 200 VMBUS_CHAN_ST_ONLIST_SHIFT) == 0) 201 panic("channel is not on the list"); 202 TAILQ_REMOVE(&sc->vmbus_chans, chan, ch_link); 203} 204 205static int 206vmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS) 207{ 208 struct vmbus_channel *chan = arg1; 209 int mnf = 0; 210 211 if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF) 212 mnf = 1; 213 return sysctl_handle_int(oidp, &mnf, 0, req); 214} 215 216static void 217vmbus_chan_sysctl_create(struct vmbus_channel *chan) 218{ 219 struct sysctl_oid *ch_tree, *chid_tree, *br_tree; 220 struct sysctl_ctx_list *ctx; 221 uint32_t ch_id; 222 char name[16]; 223 224 /* 225 * Add sysctl nodes related to this channel to this 226 * channel's sysctl ctx, so that they can be destroyed 227 * independently upon close of this channel, which can 228 * happen even if the device is not detached. 229 */ 230 ctx = &chan->ch_sysctl_ctx; 231 sysctl_ctx_init(ctx); 232 233 /* 234 * Create dev.NAME.UNIT.channel tree. 235 */ 236 ch_tree = SYSCTL_ADD_NODE(ctx, 237 SYSCTL_CHILDREN(device_get_sysctl_tree(chan->ch_dev)), 238 OID_AUTO, "channel", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 239 if (ch_tree == NULL) 240 return; 241 242 /* 243 * Create dev.NAME.UNIT.channel.CHANID tree. 244 */ 245 if (VMBUS_CHAN_ISPRIMARY(chan)) 246 ch_id = chan->ch_id; 247 else 248 ch_id = chan->ch_prichan->ch_id; 249 snprintf(name, sizeof(name), "%d", ch_id); 250 chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree), 251 OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 252 if (chid_tree == NULL) 253 return; 254 255 if (!VMBUS_CHAN_ISPRIMARY(chan)) { 256 /* 257 * Create dev.NAME.UNIT.channel.CHANID.sub tree. 258 */ 259 ch_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree), 260 OID_AUTO, "sub", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 261 if (ch_tree == NULL) 262 return; 263 264 /* 265 * Create dev.NAME.UNIT.channel.CHANID.sub.SUBIDX tree. 266 * 267 * NOTE: 268 * chid_tree is changed to this new sysctl tree. 269 */ 270 snprintf(name, sizeof(name), "%d", chan->ch_subidx); 271 chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree), 272 OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 273 if (chid_tree == NULL) 274 return; 275 276 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 277 "chanid", CTLFLAG_RD, &chan->ch_id, 0, "channel id"); 278 } 279 280 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 281 "cpu", CTLFLAG_RD, &chan->ch_cpuid, 0, "owner CPU id"); 282 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 283 "mnf", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, 284 chan, 0, vmbus_chan_sysctl_mnf, "I", 285 "has monitor notification facilities"); 286 287 br_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 288 "br", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 289 if (br_tree != NULL) { 290 /* 291 * Create sysctl tree for RX bufring. 292 */ 293 vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_rxbr.rxbr, "rx"); 294 /* 295 * Create sysctl tree for TX bufring. 296 */ 297 vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_txbr.txbr, "tx"); 298 } 299} 300 301int 302vmbus_chan_open(struct vmbus_channel *chan, int txbr_size, int rxbr_size, 303 const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg) 304{ 305 struct vmbus_chan_br cbr; 306 int error; 307 308 /* 309 * Allocate the TX+RX bufrings. 310 */ 311 KASSERT(chan->ch_bufring == NULL, ("bufrings are allocated")); 312 chan->ch_bufring = hyperv_dmamem_alloc(bus_get_dma_tag(chan->ch_dev), 313 PAGE_SIZE, 0, txbr_size + rxbr_size, &chan->ch_bufring_dma, 314 BUS_DMA_WAITOK); 315 if (chan->ch_bufring == NULL) { 316 vmbus_chan_printf(chan, "bufring allocation failed\n"); 317 return (ENOMEM); 318 } 319 320 cbr.cbr = chan->ch_bufring; 321 cbr.cbr_paddr = chan->ch_bufring_dma.hv_paddr; 322 cbr.cbr_txsz = txbr_size; 323 cbr.cbr_rxsz = rxbr_size; 324 325 error = vmbus_chan_open_br(chan, &cbr, udata, udlen, cb, cbarg); 326 if (error) { 327 if (error == EISCONN) { 328 /* 329 * XXX 330 * The bufring GPADL is still connected; abandon 331 * this bufring, instead of having mysterious 332 * crash or trashed data later on. 333 */ 334 vmbus_chan_printf(chan, "chan%u bufring GPADL " 335 "is still connected upon channel open error; " 336 "leak %d bytes memory\n", chan->ch_id, 337 txbr_size + rxbr_size); 338 } else { 339 hyperv_dmamem_free(&chan->ch_bufring_dma, 340 chan->ch_bufring); 341 } 342 chan->ch_bufring = NULL; 343 } 344 return (error); 345} 346 347int 348vmbus_chan_open_br(struct vmbus_channel *chan, const struct vmbus_chan_br *cbr, 349 const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg) 350{ 351 struct vmbus_softc *sc = chan->ch_vmbus; 352 const struct vmbus_message *msg; 353 struct vmbus_chanmsg_chopen *req; 354 struct vmbus_msghc *mh; 355 uint32_t status; 356 int error, txbr_size, rxbr_size; 357 task_fn_t *task_fn; 358 uint8_t *br; 359 360 if (udlen > VMBUS_CHANMSG_CHOPEN_UDATA_SIZE) { 361 vmbus_chan_printf(chan, 362 "invalid udata len %d for chan%u\n", udlen, chan->ch_id); 363 return (EINVAL); 364 } 365 366 br = cbr->cbr; 367 txbr_size = cbr->cbr_txsz; 368 rxbr_size = cbr->cbr_rxsz; 369 KASSERT((txbr_size & PAGE_MASK) == 0, 370 ("send bufring size is not multiple page")); 371 KASSERT((rxbr_size & PAGE_MASK) == 0, 372 ("recv bufring size is not multiple page")); 373 KASSERT((cbr->cbr_paddr & PAGE_MASK) == 0, 374 ("bufring is not page aligned")); 375 376 /* 377 * Zero out the TX/RX bufrings, in case that they were used before. 378 */ 379 memset(br, 0, txbr_size + rxbr_size); 380 381 if (atomic_testandset_int(&chan->ch_stflags, 382 VMBUS_CHAN_ST_OPENED_SHIFT)) 383 panic("double-open chan%u", chan->ch_id); 384 385 chan->ch_cb = cb; 386 chan->ch_cbarg = cbarg; 387 388 vmbus_chan_update_evtflagcnt(sc, chan); 389 390 chan->ch_tq = VMBUS_PCPU_GET(chan->ch_vmbus, event_tq, chan->ch_cpuid); 391 if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD) 392 task_fn = vmbus_chan_task; 393 else 394 task_fn = vmbus_chan_task_nobatch; 395 TASK_INIT(&chan->ch_task, 0, task_fn, chan); 396 397 /* TX bufring comes first */ 398 vmbus_txbr_setup(&chan->ch_txbr, br, txbr_size); 399 /* RX bufring immediately follows TX bufring */ 400 vmbus_rxbr_setup(&chan->ch_rxbr, br + txbr_size, rxbr_size); 401 402 /* Create sysctl tree for this channel */ 403 vmbus_chan_sysctl_create(chan); 404 405 /* 406 * Connect the bufrings, both RX and TX, to this channel. 407 */ 408 KASSERT(chan->ch_bufring_gpadl == 0, 409 ("bufring GPADL is still connected")); 410 error = vmbus_chan_gpadl_connect(chan, cbr->cbr_paddr, 411 txbr_size + rxbr_size, &chan->ch_bufring_gpadl); 412 if (error) { 413 vmbus_chan_printf(chan, 414 "failed to connect bufring GPADL to chan%u\n", chan->ch_id); 415 goto failed; 416 } 417 418 /* 419 * Install this channel, before it is opened, but after everything 420 * else has been setup. 421 */ 422 vmbus_chan_set_chmap(chan); 423 424 /* 425 * Open channel w/ the bufring GPADL on the target CPU. 426 */ 427 mh = vmbus_msghc_get(sc, sizeof(*req)); 428 if (mh == NULL) { 429 vmbus_chan_printf(chan, 430 "can not get msg hypercall for chopen(chan%u)\n", 431 chan->ch_id); 432 error = ENXIO; 433 goto failed; 434 } 435 436 req = vmbus_msghc_dataptr(mh); 437 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHOPEN; 438 req->chm_chanid = chan->ch_id; 439 req->chm_openid = chan->ch_id; 440 req->chm_gpadl = chan->ch_bufring_gpadl; 441 req->chm_vcpuid = chan->ch_vcpuid; 442 req->chm_txbr_pgcnt = txbr_size >> PAGE_SHIFT; 443 if (udlen > 0) 444 memcpy(req->chm_udata, udata, udlen); 445 446 error = vmbus_msghc_exec(sc, mh); 447 if (error) { 448 vmbus_chan_printf(chan, 449 "chopen(chan%u) msg hypercall exec failed: %d\n", 450 chan->ch_id, error); 451 vmbus_msghc_put(sc, mh); 452 goto failed; 453 } 454 455 for (;;) { 456 msg = vmbus_msghc_poll_result(sc, mh); 457 if (msg != NULL) 458 break; 459 if (vmbus_chan_is_revoked(chan)) { 460 int i; 461 462 /* 463 * NOTE: 464 * Hypervisor does _not_ send response CHOPEN to 465 * a revoked channel. 466 */ 467 vmbus_chan_printf(chan, 468 "chan%u is revoked, when it is being opened\n", 469 chan->ch_id); 470 471 /* 472 * XXX 473 * Add extra delay before cancel the hypercall 474 * execution; mainly to close any possible 475 * CHRESCIND and CHOPEN_RESP races on the 476 * hypervisor side. 477 */ 478#define REVOKE_LINGER 100 479 for (i = 0; i < REVOKE_LINGER; ++i) { 480 msg = vmbus_msghc_poll_result(sc, mh); 481 if (msg != NULL) 482 break; 483 pause("rchopen", 1); 484 } 485#undef REVOKE_LINGER 486 if (msg == NULL) 487 vmbus_msghc_exec_cancel(sc, mh); 488 break; 489 } 490 pause("chopen", 1); 491 } 492 if (msg != NULL) { 493 status = ((const struct vmbus_chanmsg_chopen_resp *) 494 msg->msg_data)->chm_status; 495 } else { 496 /* XXX any non-0 value is ok here. */ 497 status = 0xff; 498 } 499 500 vmbus_msghc_put(sc, mh); 501 502 if (status == 0) { 503 if (bootverbose) 504 vmbus_chan_printf(chan, "chan%u opened\n", chan->ch_id); 505 return (0); 506 } 507 508 vmbus_chan_printf(chan, "failed to open chan%u\n", chan->ch_id); 509 error = ENXIO; 510 511failed: 512 sysctl_ctx_free(&chan->ch_sysctl_ctx); 513 vmbus_chan_clear_chmap(chan); 514 if (chan->ch_bufring_gpadl != 0) { 515 int error1; 516 517 error1 = vmbus_chan_gpadl_disconnect(chan, 518 chan->ch_bufring_gpadl); 519 if (error1) { 520 /* 521 * Give caller a hint that the bufring GPADL is still 522 * connected. 523 */ 524 error = EISCONN; 525 } 526 chan->ch_bufring_gpadl = 0; 527 } 528 atomic_clear_int(&chan->ch_stflags, VMBUS_CHAN_ST_OPENED); 529 return (error); 530} 531 532int 533vmbus_chan_gpadl_connect(struct vmbus_channel *chan, bus_addr_t paddr, 534 int size, uint32_t *gpadl0) 535{ 536 struct vmbus_softc *sc = chan->ch_vmbus; 537 struct vmbus_msghc *mh; 538 struct vmbus_chanmsg_gpadl_conn *req; 539 const struct vmbus_message *msg; 540 size_t reqsz; 541 uint32_t gpadl, status; 542 int page_count, range_len, i, cnt, error; 543 uint64_t page_id; 544 545 KASSERT(*gpadl0 == 0, ("GPADL is not zero")); 546 547 /* 548 * Preliminary checks. 549 */ 550 551 KASSERT((size & PAGE_MASK) == 0, 552 ("invalid GPA size %d, not multiple page size", size)); 553 page_count = size >> PAGE_SHIFT; 554 555 KASSERT((paddr & PAGE_MASK) == 0, 556 ("GPA is not page aligned %jx", (uintmax_t)paddr)); 557 page_id = paddr >> PAGE_SHIFT; 558 559 range_len = __offsetof(struct vmbus_gpa_range, gpa_page[page_count]); 560 /* 561 * We don't support multiple GPA ranges. 562 */ 563 if (range_len > UINT16_MAX) { 564 vmbus_chan_printf(chan, "GPA too large, %d pages\n", 565 page_count); 566 return EOPNOTSUPP; 567 } 568 569 /* 570 * Allocate GPADL id. 571 */ 572 gpadl = vmbus_gpadl_alloc(sc); 573 574 /* 575 * Connect this GPADL to the target channel. 576 * 577 * NOTE: 578 * Since each message can only hold small set of page 579 * addresses, several messages may be required to 580 * complete the connection. 581 */ 582 if (page_count > VMBUS_CHANMSG_GPADL_CONN_PGMAX) 583 cnt = VMBUS_CHANMSG_GPADL_CONN_PGMAX; 584 else 585 cnt = page_count; 586 page_count -= cnt; 587 588 reqsz = __offsetof(struct vmbus_chanmsg_gpadl_conn, 589 chm_range.gpa_page[cnt]); 590 mh = vmbus_msghc_get(sc, reqsz); 591 if (mh == NULL) { 592 vmbus_chan_printf(chan, 593 "can not get msg hypercall for gpadl_conn(chan%u)\n", 594 chan->ch_id); 595 return EIO; 596 } 597 598 req = vmbus_msghc_dataptr(mh); 599 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_CONN; 600 req->chm_chanid = chan->ch_id; 601 req->chm_gpadl = gpadl; 602 req->chm_range_len = range_len; 603 req->chm_range_cnt = 1; 604 req->chm_range.gpa_len = size; 605 req->chm_range.gpa_ofs = 0; 606 for (i = 0; i < cnt; ++i) 607 req->chm_range.gpa_page[i] = page_id++; 608 609 error = vmbus_msghc_exec(sc, mh); 610 if (error) { 611 vmbus_chan_printf(chan, 612 "gpadl_conn(chan%u) msg hypercall exec failed: %d\n", 613 chan->ch_id, error); 614 vmbus_msghc_put(sc, mh); 615 return error; 616 } 617 618 while (page_count > 0) { 619 struct vmbus_chanmsg_gpadl_subconn *subreq; 620 621 if (page_count > VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX) 622 cnt = VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX; 623 else 624 cnt = page_count; 625 page_count -= cnt; 626 627 reqsz = __offsetof(struct vmbus_chanmsg_gpadl_subconn, 628 chm_gpa_page[cnt]); 629 vmbus_msghc_reset(mh, reqsz); 630 631 subreq = vmbus_msghc_dataptr(mh); 632 subreq->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_SUBCONN; 633 subreq->chm_gpadl = gpadl; 634 for (i = 0; i < cnt; ++i) 635 subreq->chm_gpa_page[i] = page_id++; 636 637 vmbus_msghc_exec_noresult(mh); 638 } 639 KASSERT(page_count == 0, ("invalid page count %d", page_count)); 640 641 msg = vmbus_msghc_wait_result(sc, mh); 642 status = ((const struct vmbus_chanmsg_gpadl_connresp *) 643 msg->msg_data)->chm_status; 644 645 vmbus_msghc_put(sc, mh); 646 647 if (status != 0) { 648 vmbus_chan_printf(chan, "gpadl_conn(chan%u) failed: %u\n", 649 chan->ch_id, status); 650 return EIO; 651 } 652 653 /* Done; commit the GPADL id. */ 654 *gpadl0 = gpadl; 655 if (bootverbose) { 656 vmbus_chan_printf(chan, "gpadl_conn(chan%u) succeeded\n", 657 chan->ch_id); 658 } 659 return 0; 660} 661 662static bool 663vmbus_chan_wait_revoke(const struct vmbus_channel *chan, bool can_sleep) 664{ 665#define WAIT_COUNT 200 /* 200ms */ 666 667 int i; 668 669 for (i = 0; i < WAIT_COUNT; ++i) { 670 if (vmbus_chan_is_revoked(chan)) 671 return (true); 672 if (can_sleep) 673 pause("wchrev", 1); 674 else 675 DELAY(1000); 676 } 677 return (false); 678 679#undef WAIT_COUNT 680} 681 682/* 683 * Disconnect the GPA from the target channel 684 */ 685int 686vmbus_chan_gpadl_disconnect(struct vmbus_channel *chan, uint32_t gpadl) 687{ 688 struct vmbus_softc *sc = chan->ch_vmbus; 689 struct vmbus_msghc *mh; 690 struct vmbus_chanmsg_gpadl_disconn *req; 691 int error; 692 693 KASSERT(gpadl != 0, ("GPADL is zero")); 694 695 mh = vmbus_msghc_get(sc, sizeof(*req)); 696 if (mh == NULL) { 697 vmbus_chan_printf(chan, 698 "can not get msg hypercall for gpadl_disconn(chan%u)\n", 699 chan->ch_id); 700 return (EBUSY); 701 } 702 703 req = vmbus_msghc_dataptr(mh); 704 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_DISCONN; 705 req->chm_chanid = chan->ch_id; 706 req->chm_gpadl = gpadl; 707 708 error = vmbus_msghc_exec(sc, mh); 709 if (error) { 710 vmbus_msghc_put(sc, mh); 711 712 if (vmbus_chan_wait_revoke(chan, true)) { 713 /* 714 * Error is benign; this channel is revoked, 715 * so this GPADL will not be touched anymore. 716 */ 717 vmbus_chan_printf(chan, 718 "gpadl_disconn(revoked chan%u) msg hypercall " 719 "exec failed: %d\n", chan->ch_id, error); 720 return (0); 721 } 722 vmbus_chan_printf(chan, 723 "gpadl_disconn(chan%u) msg hypercall exec failed: %d\n", 724 chan->ch_id, error); 725 return (error); 726 } 727 728 vmbus_msghc_wait_result(sc, mh); 729 /* Discard result; no useful information */ 730 vmbus_msghc_put(sc, mh); 731 732 return (0); 733} 734 735static void 736vmbus_chan_detach(struct vmbus_channel *chan) 737{ 738 int refs; 739 740 KASSERT(chan->ch_refs > 0, ("chan%u: invalid refcnt %d", 741 chan->ch_id, chan->ch_refs)); 742 refs = atomic_fetchadd_int(&chan->ch_refs, -1); 743#ifdef INVARIANTS 744 if (VMBUS_CHAN_ISPRIMARY(chan)) { 745 KASSERT(refs == 1, ("chan%u: invalid refcnt %d for prichan", 746 chan->ch_id, refs + 1)); 747 } 748#endif 749 if (refs == 1) { 750 /* 751 * Detach the target channel. 752 */ 753 if (bootverbose) { 754 vmbus_chan_printf(chan, "chan%u detached\n", 755 chan->ch_id); 756 } 757 taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task); 758 } 759} 760 761static void 762vmbus_chan_clrchmap_task(void *xchan, int pending __unused) 763{ 764 struct vmbus_channel *chan = xchan; 765 766 critical_enter(); 767 chan->ch_vmbus->vmbus_chmap[chan->ch_id] = NULL; 768 critical_exit(); 769} 770 771static void 772vmbus_chan_clear_chmap(struct vmbus_channel *chan) 773{ 774 struct task chmap_task; 775 776 TASK_INIT(&chmap_task, 0, vmbus_chan_clrchmap_task, chan); 777 taskqueue_enqueue(chan->ch_tq, &chmap_task); 778 taskqueue_drain(chan->ch_tq, &chmap_task); 779} 780 781static void 782vmbus_chan_set_chmap(struct vmbus_channel *chan) 783{ 784 __compiler_membar(); 785 chan->ch_vmbus->vmbus_chmap[chan->ch_id] = chan; 786} 787 788static int 789vmbus_chan_close_internal(struct vmbus_channel *chan) 790{ 791 struct vmbus_softc *sc = chan->ch_vmbus; 792 struct vmbus_msghc *mh; 793 struct vmbus_chanmsg_chclose *req; 794 uint32_t old_stflags; 795 int error; 796 797 /* 798 * NOTE: 799 * Sub-channels are closed upon their primary channel closing, 800 * so they can be closed even before they are opened. 801 */ 802 for (;;) { 803 old_stflags = chan->ch_stflags; 804 if (atomic_cmpset_int(&chan->ch_stflags, old_stflags, 805 old_stflags & ~VMBUS_CHAN_ST_OPENED)) 806 break; 807 } 808 if ((old_stflags & VMBUS_CHAN_ST_OPENED) == 0) { 809 /* Not opened yet; done */ 810 if (bootverbose) { 811 vmbus_chan_printf(chan, "chan%u not opened\n", 812 chan->ch_id); 813 } 814 return (0); 815 } 816 817 /* 818 * Free this channel's sysctl tree attached to its device's 819 * sysctl tree. 820 */ 821 sysctl_ctx_free(&chan->ch_sysctl_ctx); 822 823 /* 824 * NOTE: 825 * Order is critical. This channel _must_ be uninstalled first, 826 * else the channel task may be enqueued by the IDT after it has 827 * been drained. 828 */ 829 vmbus_chan_clear_chmap(chan); 830 taskqueue_drain(chan->ch_tq, &chan->ch_task); 831 chan->ch_tq = NULL; 832 833 /* 834 * Close this channel. 835 */ 836 mh = vmbus_msghc_get(sc, sizeof(*req)); 837 if (mh == NULL) { 838 vmbus_chan_printf(chan, 839 "can not get msg hypercall for chclose(chan%u)\n", 840 chan->ch_id); 841 error = ENXIO; 842 goto disconnect; 843 } 844 845 req = vmbus_msghc_dataptr(mh); 846 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHCLOSE; 847 req->chm_chanid = chan->ch_id; 848 849 error = vmbus_msghc_exec_noresult(mh); 850 vmbus_msghc_put(sc, mh); 851 852 if (error) { 853 vmbus_chan_printf(chan, 854 "chclose(chan%u) msg hypercall exec failed: %d\n", 855 chan->ch_id, error); 856 goto disconnect; 857 } 858 859 if (bootverbose) 860 vmbus_chan_printf(chan, "chan%u closed\n", chan->ch_id); 861 862disconnect: 863 /* 864 * Disconnect the TX+RX bufrings from this channel. 865 */ 866 if (chan->ch_bufring_gpadl != 0) { 867 int error1; 868 869 error1 = vmbus_chan_gpadl_disconnect(chan, 870 chan->ch_bufring_gpadl); 871 if (error1) { 872 /* 873 * XXX 874 * The bufring GPADL is still connected; abandon 875 * this bufring, instead of having mysterious 876 * crash or trashed data later on. 877 */ 878 vmbus_chan_printf(chan, "chan%u bufring GPADL " 879 "is still connected after close\n", chan->ch_id); 880 chan->ch_bufring = NULL; 881 /* 882 * Give caller a hint that the bufring GPADL is 883 * still connected. 884 */ 885 error = EISCONN; 886 } 887 chan->ch_bufring_gpadl = 0; 888 } 889 890 /* 891 * Destroy the TX+RX bufrings. 892 */ 893 if (chan->ch_bufring != NULL) { 894 hyperv_dmamem_free(&chan->ch_bufring_dma, chan->ch_bufring); 895 chan->ch_bufring = NULL; 896 } 897 return (error); 898} 899 900int 901vmbus_chan_close_direct(struct vmbus_channel *chan) 902{ 903 int error; 904 905#ifdef INVARIANTS 906 if (VMBUS_CHAN_ISPRIMARY(chan)) { 907 struct vmbus_channel *subchan; 908 909 /* 910 * All sub-channels _must_ have been closed, or are _not_ 911 * opened at all. 912 */ 913 mtx_lock(&chan->ch_subchan_lock); 914 TAILQ_FOREACH(subchan, &chan->ch_subchans, ch_sublink) { 915 KASSERT( 916 (subchan->ch_stflags & VMBUS_CHAN_ST_OPENED) == 0, 917 ("chan%u: subchan%u is still opened", 918 chan->ch_id, subchan->ch_subidx)); 919 } 920 mtx_unlock(&chan->ch_subchan_lock); 921 } 922#endif 923 924 error = vmbus_chan_close_internal(chan); 925 if (!VMBUS_CHAN_ISPRIMARY(chan)) { 926 /* 927 * This sub-channel is referenced, when it is linked to 928 * the primary channel; drop that reference now. 929 */ 930 vmbus_chan_detach(chan); 931 } 932 return (error); 933} 934 935/* 936 * Caller should make sure that all sub-channels have 937 * been added to 'chan' and all to-be-closed channels 938 * are not being opened. 939 */ 940void 941vmbus_chan_close(struct vmbus_channel *chan) 942{ 943 int subchan_cnt; 944 945 if (!VMBUS_CHAN_ISPRIMARY(chan)) { 946 /* 947 * Sub-channel is closed when its primary channel 948 * is closed; done. 949 */ 950 return; 951 } 952 953 /* 954 * Close all sub-channels, if any. 955 */ 956 subchan_cnt = chan->ch_subchan_cnt; 957 if (subchan_cnt > 0) { 958 struct vmbus_channel **subchan; 959 int i; 960 961 subchan = vmbus_subchan_get(chan, subchan_cnt); 962 for (i = 0; i < subchan_cnt; ++i) { 963 vmbus_chan_close_internal(subchan[i]); 964 /* 965 * This sub-channel is referenced, when it is 966 * linked to the primary channel; drop that 967 * reference now. 968 */ 969 vmbus_chan_detach(subchan[i]); 970 } 971 vmbus_subchan_rel(subchan, subchan_cnt); 972 } 973 974 /* Then close the primary channel. */ 975 vmbus_chan_close_internal(chan); 976} 977 978void 979vmbus_chan_intr_drain(struct vmbus_channel *chan) 980{ 981 982 taskqueue_drain(chan->ch_tq, &chan->ch_task); 983} 984 985int 986vmbus_chan_send(struct vmbus_channel *chan, uint16_t type, uint16_t flags, 987 void *data, int dlen, uint64_t xactid) 988{ 989 struct vmbus_chanpkt pkt; 990 int pktlen, pad_pktlen, hlen, error; 991 uint64_t pad = 0; 992 struct iovec iov[3]; 993 boolean_t send_evt; 994 995 hlen = sizeof(pkt); 996 pktlen = hlen + dlen; 997 pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 998 KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 999 ("invalid packet size %d", pad_pktlen)); 1000 1001 pkt.cp_hdr.cph_type = type; 1002 pkt.cp_hdr.cph_flags = flags; 1003 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1004 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1005 pkt.cp_hdr.cph_xactid = xactid; 1006 1007 iov[0].iov_base = &pkt; 1008 iov[0].iov_len = hlen; 1009 iov[1].iov_base = data; 1010 iov[1].iov_len = dlen; 1011 iov[2].iov_base = &pad; 1012 iov[2].iov_len = pad_pktlen - pktlen; 1013 1014 error = vmbus_txbr_write(&chan->ch_txbr, iov, 3, &send_evt); 1015 if (!error && send_evt) 1016 vmbus_chan_signal_tx(chan); 1017 return error; 1018} 1019 1020int 1021vmbus_chan_send_sglist(struct vmbus_channel *chan, 1022 struct vmbus_gpa sg[], int sglen, void *data, int dlen, uint64_t xactid) 1023{ 1024 struct vmbus_chanpkt_sglist pkt; 1025 int pktlen, pad_pktlen, hlen, error; 1026 struct iovec iov[4]; 1027 boolean_t send_evt; 1028 uint64_t pad = 0; 1029 1030 hlen = __offsetof(struct vmbus_chanpkt_sglist, cp_gpa[sglen]); 1031 pktlen = hlen + dlen; 1032 pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1033 KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1034 ("invalid packet size %d", pad_pktlen)); 1035 1036 pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1037 pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1038 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1039 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1040 pkt.cp_hdr.cph_xactid = xactid; 1041 pkt.cp_rsvd = 0; 1042 pkt.cp_gpa_cnt = sglen; 1043 1044 iov[0].iov_base = &pkt; 1045 iov[0].iov_len = sizeof(pkt); 1046 iov[1].iov_base = sg; 1047 iov[1].iov_len = sizeof(struct vmbus_gpa) * sglen; 1048 iov[2].iov_base = data; 1049 iov[2].iov_len = dlen; 1050 iov[3].iov_base = &pad; 1051 iov[3].iov_len = pad_pktlen - pktlen; 1052 1053 error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); 1054 if (!error && send_evt) 1055 vmbus_chan_signal_tx(chan); 1056 return error; 1057} 1058 1059int 1060vmbus_chan_send_prplist(struct vmbus_channel *chan, 1061 struct vmbus_gpa_range *prp, int prp_cnt, void *data, int dlen, 1062 uint64_t xactid) 1063{ 1064 struct vmbus_chanpkt_prplist pkt; 1065 int pktlen, pad_pktlen, hlen, error; 1066 struct iovec iov[4]; 1067 boolean_t send_evt; 1068 uint64_t pad = 0; 1069 1070 hlen = __offsetof(struct vmbus_chanpkt_prplist, 1071 cp_range[0].gpa_page[prp_cnt]); 1072 pktlen = hlen + dlen; 1073 pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1074 KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1075 ("invalid packet size %d", pad_pktlen)); 1076 1077 pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1078 pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1079 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1080 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1081 pkt.cp_hdr.cph_xactid = xactid; 1082 pkt.cp_rsvd = 0; 1083 pkt.cp_range_cnt = 1; 1084 1085 iov[0].iov_base = &pkt; 1086 iov[0].iov_len = sizeof(pkt); 1087 iov[1].iov_base = prp; 1088 iov[1].iov_len = __offsetof(struct vmbus_gpa_range, gpa_page[prp_cnt]); 1089 iov[2].iov_base = data; 1090 iov[2].iov_len = dlen; 1091 iov[3].iov_base = &pad; 1092 iov[3].iov_len = pad_pktlen - pktlen; 1093 1094 error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); 1095 if (!error && send_evt) 1096 vmbus_chan_signal_tx(chan); 1097 return error; 1098} 1099 1100int 1101vmbus_chan_recv(struct vmbus_channel *chan, void *data, int *dlen0, 1102 uint64_t *xactid) 1103{ 1104 struct vmbus_chanpkt_hdr pkt; 1105 int error, dlen, hlen; 1106 1107 error = vmbus_rxbr_peek(&chan->ch_rxbr, &pkt, sizeof(pkt)); 1108 if (error) 1109 return (error); 1110 1111 if (__predict_false(pkt.cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) { 1112 vmbus_chan_printf(chan, "invalid hlen %u\n", pkt.cph_hlen); 1113 /* XXX this channel is dead actually. */ 1114 return (EIO); 1115 } 1116 if (__predict_false(pkt.cph_hlen > pkt.cph_tlen)) { 1117 vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n", 1118 pkt.cph_hlen, pkt.cph_tlen); 1119 /* XXX this channel is dead actually. */ 1120 return (EIO); 1121 } 1122 1123 hlen = VMBUS_CHANPKT_GETLEN(pkt.cph_hlen); 1124 dlen = VMBUS_CHANPKT_GETLEN(pkt.cph_tlen) - hlen; 1125 1126 if (*dlen0 < dlen) { 1127 /* Return the size of this packet's data. */ 1128 *dlen0 = dlen; 1129 return (ENOBUFS); 1130 } 1131 1132 *xactid = pkt.cph_xactid; 1133 *dlen0 = dlen; 1134 1135 /* Skip packet header */ 1136 error = vmbus_rxbr_read(&chan->ch_rxbr, data, dlen, hlen); 1137 KASSERT(!error, ("vmbus_rxbr_read failed")); 1138 1139 return (0); 1140} 1141 1142int 1143vmbus_chan_recv_pkt(struct vmbus_channel *chan, 1144 struct vmbus_chanpkt_hdr *pkt, int *pktlen0) 1145{ 1146 int error, pktlen, pkt_hlen; 1147 1148 pkt_hlen = sizeof(*pkt); 1149 error = vmbus_rxbr_peek(&chan->ch_rxbr, pkt, pkt_hlen); 1150 if (error) 1151 return (error); 1152 1153 if (__predict_false(pkt->cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) { 1154 vmbus_chan_printf(chan, "invalid hlen %u\n", pkt->cph_hlen); 1155 /* XXX this channel is dead actually. */ 1156 return (EIO); 1157 } 1158 if (__predict_false(pkt->cph_hlen > pkt->cph_tlen)) { 1159 vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n", 1160 pkt->cph_hlen, pkt->cph_tlen); 1161 /* XXX this channel is dead actually. */ 1162 return (EIO); 1163 } 1164 1165 pktlen = VMBUS_CHANPKT_GETLEN(pkt->cph_tlen); 1166 if (*pktlen0 < pktlen) { 1167 /* Return the size of this packet. */ 1168 *pktlen0 = pktlen; 1169 return (ENOBUFS); 1170 } 1171 *pktlen0 = pktlen; 1172 1173 /* 1174 * Skip the fixed-size packet header, which has been filled 1175 * by the above vmbus_rxbr_peek(). 1176 */ 1177 error = vmbus_rxbr_read(&chan->ch_rxbr, pkt + 1, 1178 pktlen - pkt_hlen, pkt_hlen); 1179 KASSERT(!error, ("vmbus_rxbr_read failed")); 1180 1181 return (0); 1182} 1183 1184static void 1185vmbus_chan_task(void *xchan, int pending __unused) 1186{ 1187 struct vmbus_channel *chan = xchan; 1188 vmbus_chan_callback_t cb = chan->ch_cb; 1189 void *cbarg = chan->ch_cbarg; 1190 1191 /* 1192 * Optimize host to guest signaling by ensuring: 1193 * 1. While reading the channel, we disable interrupts from 1194 * host. 1195 * 2. Ensure that we process all posted messages from the host 1196 * before returning from this callback. 1197 * 3. Once we return, enable signaling from the host. Once this 1198 * state is set we check to see if additional packets are 1199 * available to read. In this case we repeat the process. 1200 * 1201 * NOTE: Interrupt has been disabled in the ISR. 1202 */ 1203 for (;;) { 1204 uint32_t left; 1205 1206 cb(chan, cbarg); 1207 1208 left = vmbus_rxbr_intr_unmask(&chan->ch_rxbr); 1209 if (left == 0) { 1210 /* No more data in RX bufring; done */ 1211 break; 1212 } 1213 vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1214 } 1215} 1216 1217static void 1218vmbus_chan_task_nobatch(void *xchan, int pending __unused) 1219{ 1220 struct vmbus_channel *chan = xchan; 1221 1222 chan->ch_cb(chan, chan->ch_cbarg); 1223} 1224 1225static __inline void 1226vmbus_event_flags_proc(struct vmbus_softc *sc, volatile u_long *event_flags, 1227 int flag_cnt) 1228{ 1229 int f; 1230 1231 for (f = 0; f < flag_cnt; ++f) { 1232 uint32_t chid_base; 1233 u_long flags; 1234 int chid_ofs; 1235 1236 if (event_flags[f] == 0) 1237 continue; 1238 1239 flags = atomic_swap_long(&event_flags[f], 0); 1240 chid_base = f << VMBUS_EVTFLAG_SHIFT; 1241 1242 while ((chid_ofs = ffsl(flags)) != 0) { 1243 struct vmbus_channel *chan; 1244 1245 --chid_ofs; /* NOTE: ffsl is 1-based */ 1246 flags &= ~(1UL << chid_ofs); 1247 1248 chan = sc->vmbus_chmap[chid_base + chid_ofs]; 1249 if (__predict_false(chan == NULL)) { 1250 /* Channel is closed. */ 1251 continue; 1252 } 1253 __compiler_membar(); 1254 1255 if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD) 1256 vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1257 taskqueue_enqueue(chan->ch_tq, &chan->ch_task); 1258 } 1259 } 1260} 1261 1262void 1263vmbus_event_proc(struct vmbus_softc *sc, int cpu) 1264{ 1265 struct vmbus_evtflags *eventf; 1266 1267 /* 1268 * On Host with Win8 or above, the event page can be checked directly 1269 * to get the id of the channel that has the pending interrupt. 1270 */ 1271 eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE; 1272 vmbus_event_flags_proc(sc, eventf->evt_flags, 1273 VMBUS_PCPU_GET(sc, event_flags_cnt, cpu)); 1274} 1275 1276void 1277vmbus_event_proc_compat(struct vmbus_softc *sc, int cpu) 1278{ 1279 struct vmbus_evtflags *eventf; 1280 1281 eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE; 1282 if (atomic_testandclear_long(&eventf->evt_flags[0], 0)) { 1283 vmbus_event_flags_proc(sc, sc->vmbus_rx_evtflags, 1284 VMBUS_CHAN_MAX_COMPAT >> VMBUS_EVTFLAG_SHIFT); 1285 } 1286} 1287 1288static void 1289vmbus_chan_update_evtflagcnt(struct vmbus_softc *sc, 1290 const struct vmbus_channel *chan) 1291{ 1292 volatile int *flag_cnt_ptr; 1293 int flag_cnt; 1294 1295 flag_cnt = (chan->ch_id / VMBUS_EVTFLAG_LEN) + 1; 1296 flag_cnt_ptr = VMBUS_PCPU_PTR(sc, event_flags_cnt, chan->ch_cpuid); 1297 1298 for (;;) { 1299 int old_flag_cnt; 1300 1301 old_flag_cnt = *flag_cnt_ptr; 1302 if (old_flag_cnt >= flag_cnt) 1303 break; 1304 if (atomic_cmpset_int(flag_cnt_ptr, old_flag_cnt, flag_cnt)) { 1305 if (bootverbose) { 1306 vmbus_chan_printf(chan, 1307 "chan%u update cpu%d flag_cnt to %d\n", 1308 chan->ch_id, chan->ch_cpuid, flag_cnt); 1309 } 1310 break; 1311 } 1312 } 1313} 1314 1315static struct vmbus_channel * 1316vmbus_chan_alloc(struct vmbus_softc *sc) 1317{ 1318 struct vmbus_channel *chan; 1319 1320 chan = malloc(sizeof(*chan), M_DEVBUF, M_WAITOK | M_ZERO); 1321 1322 chan->ch_monprm = hyperv_dmamem_alloc(bus_get_dma_tag(sc->vmbus_dev), 1323 HYPERCALL_PARAM_ALIGN, 0, sizeof(struct hyperv_mon_param), 1324 &chan->ch_monprm_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO); 1325 if (chan->ch_monprm == NULL) { 1326 device_printf(sc->vmbus_dev, "monprm alloc failed\n"); 1327 free(chan, M_DEVBUF); 1328 return NULL; 1329 } 1330 1331 chan->ch_refs = 1; 1332 chan->ch_vmbus = sc; 1333 mtx_init(&chan->ch_subchan_lock, "vmbus subchan", NULL, MTX_DEF); 1334 sx_init(&chan->ch_orphan_lock, "vmbus chorphan"); 1335 TAILQ_INIT(&chan->ch_subchans); 1336 vmbus_rxbr_init(&chan->ch_rxbr); 1337 vmbus_txbr_init(&chan->ch_txbr); 1338 1339 return chan; 1340} 1341 1342static void 1343vmbus_chan_free(struct vmbus_channel *chan) 1344{ 1345 1346 KASSERT(TAILQ_EMPTY(&chan->ch_subchans) && chan->ch_subchan_cnt == 0, 1347 ("still owns sub-channels")); 1348 KASSERT((chan->ch_stflags & 1349 (VMBUS_CHAN_ST_OPENED | 1350 VMBUS_CHAN_ST_ONPRIL | 1351 VMBUS_CHAN_ST_ONSUBL | 1352 VMBUS_CHAN_ST_ONLIST)) == 0, ("free busy channel")); 1353 KASSERT(chan->ch_orphan_xact == NULL, 1354 ("still has orphan xact installed")); 1355 KASSERT(chan->ch_refs == 0, ("chan%u: invalid refcnt %d", 1356 chan->ch_id, chan->ch_refs)); 1357 1358 hyperv_dmamem_free(&chan->ch_monprm_dma, chan->ch_monprm); 1359 mtx_destroy(&chan->ch_subchan_lock); 1360 sx_destroy(&chan->ch_orphan_lock); 1361 vmbus_rxbr_deinit(&chan->ch_rxbr); 1362 vmbus_txbr_deinit(&chan->ch_txbr); 1363 free(chan, M_DEVBUF); 1364} 1365 1366static int 1367vmbus_chan_add(struct vmbus_channel *newchan) 1368{ 1369 struct vmbus_softc *sc = newchan->ch_vmbus; 1370 struct vmbus_channel *prichan; 1371 1372 if (newchan->ch_id == 0) { 1373 /* 1374 * XXX 1375 * Chan0 will neither be processed nor should be offered; 1376 * skip it. 1377 */ 1378 device_printf(sc->vmbus_dev, "got chan0 offer, discard\n"); 1379 return EINVAL; 1380 } else if (newchan->ch_id >= VMBUS_CHAN_MAX) { 1381 device_printf(sc->vmbus_dev, "invalid chan%u offer\n", 1382 newchan->ch_id); 1383 return EINVAL; 1384 } 1385 1386 mtx_lock(&sc->vmbus_prichan_lock); 1387 TAILQ_FOREACH(prichan, &sc->vmbus_prichans, ch_prilink) { 1388 /* 1389 * Sub-channel will have the same type GUID and instance 1390 * GUID as its primary channel. 1391 */ 1392 if (memcmp(&prichan->ch_guid_type, &newchan->ch_guid_type, 1393 sizeof(struct hyperv_guid)) == 0 && 1394 memcmp(&prichan->ch_guid_inst, &newchan->ch_guid_inst, 1395 sizeof(struct hyperv_guid)) == 0) 1396 break; 1397 } 1398 if (VMBUS_CHAN_ISPRIMARY(newchan)) { 1399 if (prichan == NULL) { 1400 /* Install the new primary channel */ 1401 vmbus_chan_ins_prilist(sc, newchan); 1402 mtx_unlock(&sc->vmbus_prichan_lock); 1403 goto done; 1404 } else { 1405 mtx_unlock(&sc->vmbus_prichan_lock); 1406 device_printf(sc->vmbus_dev, 1407 "duplicated primary chan%u\n", newchan->ch_id); 1408 return EINVAL; 1409 } 1410 } else { /* Sub-channel */ 1411 if (prichan == NULL) { 1412 mtx_unlock(&sc->vmbus_prichan_lock); 1413 device_printf(sc->vmbus_dev, 1414 "no primary chan for chan%u\n", newchan->ch_id); 1415 return EINVAL; 1416 } 1417 /* 1418 * Found the primary channel for this sub-channel and 1419 * move on. 1420 * 1421 * XXX refcnt prichan 1422 */ 1423 } 1424 mtx_unlock(&sc->vmbus_prichan_lock); 1425 1426 /* 1427 * This is a sub-channel; link it with the primary channel. 1428 */ 1429 KASSERT(!VMBUS_CHAN_ISPRIMARY(newchan), 1430 ("new channel is not sub-channel")); 1431 KASSERT(prichan != NULL, ("no primary channel")); 1432 1433 /* 1434 * Reference count this sub-channel; it will be dereferenced 1435 * when this sub-channel is closed. 1436 */ 1437 KASSERT(newchan->ch_refs == 1, ("chan%u: invalid refcnt %d", 1438 newchan->ch_id, newchan->ch_refs)); 1439 atomic_add_int(&newchan->ch_refs, 1); 1440 1441 newchan->ch_prichan = prichan; 1442 newchan->ch_dev = prichan->ch_dev; 1443 1444 mtx_lock(&prichan->ch_subchan_lock); 1445 vmbus_chan_ins_sublist(prichan, newchan); 1446 mtx_unlock(&prichan->ch_subchan_lock); 1447 /* 1448 * Notify anyone that is interested in this sub-channel, 1449 * after this sub-channel is setup. 1450 */ 1451 wakeup(prichan); 1452done: 1453 /* 1454 * Hook this channel up for later revocation. 1455 */ 1456 mtx_lock(&sc->vmbus_chan_lock); 1457 vmbus_chan_ins_list(sc, newchan); 1458 mtx_unlock(&sc->vmbus_chan_lock); 1459 1460 if (bootverbose) { 1461 vmbus_chan_printf(newchan, "chan%u subidx%u offer\n", 1462 newchan->ch_id, newchan->ch_subidx); 1463 } 1464 1465 /* Select default cpu for this channel. */ 1466 vmbus_chan_cpu_default(newchan); 1467 1468 return 0; 1469} 1470 1471void 1472vmbus_chan_cpu_set(struct vmbus_channel *chan, int cpu) 1473{ 1474 KASSERT(cpu >= 0 && cpu < mp_ncpus, ("invalid cpu %d", cpu)); 1475 1476 if (chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WS2008 || 1477 chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WIN7) { 1478 /* Only cpu0 is supported */ 1479 cpu = 0; 1480 } 1481 1482 chan->ch_cpuid = cpu; 1483 chan->ch_vcpuid = VMBUS_PCPU_GET(chan->ch_vmbus, vcpuid, cpu); 1484 1485 if (bootverbose) { 1486 vmbus_chan_printf(chan, 1487 "chan%u assigned to cpu%u [vcpu%u]\n", 1488 chan->ch_id, chan->ch_cpuid, chan->ch_vcpuid); 1489 } 1490} 1491 1492void 1493vmbus_chan_cpu_rr(struct vmbus_channel *chan) 1494{ 1495 static uint32_t vmbus_chan_nextcpu; 1496 int cpu; 1497 1498 cpu = atomic_fetchadd_int(&vmbus_chan_nextcpu, 1) % mp_ncpus; 1499 vmbus_chan_cpu_set(chan, cpu); 1500} 1501 1502static void 1503vmbus_chan_cpu_default(struct vmbus_channel *chan) 1504{ 1505 /* 1506 * By default, pin the channel to cpu0. Devices having 1507 * special channel-cpu mapping requirement should call 1508 * vmbus_chan_cpu_{set,rr}(). 1509 */ 1510 vmbus_chan_cpu_set(chan, 0); 1511} 1512 1513static void 1514vmbus_chan_msgproc_choffer(struct vmbus_softc *sc, 1515 const struct vmbus_message *msg) 1516{ 1517 const struct vmbus_chanmsg_choffer *offer; 1518 struct vmbus_channel *chan; 1519 task_fn_t *detach_fn, *attach_fn; 1520 int error; 1521 1522 offer = (const struct vmbus_chanmsg_choffer *)msg->msg_data; 1523 1524 chan = vmbus_chan_alloc(sc); 1525 if (chan == NULL) { 1526 device_printf(sc->vmbus_dev, "allocate chan%u failed\n", 1527 offer->chm_chanid); 1528 return; 1529 } 1530 1531 chan->ch_id = offer->chm_chanid; 1532 chan->ch_subidx = offer->chm_subidx; 1533 chan->ch_guid_type = offer->chm_chtype; 1534 chan->ch_guid_inst = offer->chm_chinst; 1535 1536 /* Batch reading is on by default */ 1537 chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD; 1538 1539 chan->ch_monprm->mp_connid = VMBUS_CONNID_EVENT; 1540 if (sc->vmbus_version != VMBUS_VERSION_WS2008) 1541 chan->ch_monprm->mp_connid = offer->chm_connid; 1542 1543 if (offer->chm_flags1 & VMBUS_CHOFFER_FLAG1_HASMNF) { 1544 int trig_idx; 1545 1546 /* 1547 * Setup MNF stuffs. 1548 */ 1549 chan->ch_txflags |= VMBUS_CHAN_TXF_HASMNF; 1550 1551 trig_idx = offer->chm_montrig / VMBUS_MONTRIG_LEN; 1552 if (trig_idx >= VMBUS_MONTRIGS_MAX) 1553 panic("invalid monitor trigger %u", offer->chm_montrig); 1554 chan->ch_montrig = 1555 &sc->vmbus_mnf2->mnf_trigs[trig_idx].mt_pending; 1556 1557 chan->ch_montrig_mask = 1558 1 << (offer->chm_montrig % VMBUS_MONTRIG_LEN); 1559 } 1560 1561 /* 1562 * Setup event flag. 1563 */ 1564 chan->ch_evtflag = 1565 &sc->vmbus_tx_evtflags[chan->ch_id >> VMBUS_EVTFLAG_SHIFT]; 1566 chan->ch_evtflag_mask = 1UL << (chan->ch_id & VMBUS_EVTFLAG_MASK); 1567 1568 /* 1569 * Setup attach and detach tasks. 1570 */ 1571 if (VMBUS_CHAN_ISPRIMARY(chan)) { 1572 chan->ch_mgmt_tq = sc->vmbus_devtq; 1573 attach_fn = vmbus_prichan_attach_task; 1574 detach_fn = vmbus_prichan_detach_task; 1575 } else { 1576 chan->ch_mgmt_tq = sc->vmbus_subchtq; 1577 attach_fn = vmbus_subchan_attach_task; 1578 detach_fn = vmbus_subchan_detach_task; 1579 } 1580 TASK_INIT(&chan->ch_attach_task, 0, attach_fn, chan); 1581 TASK_INIT(&chan->ch_detach_task, 0, detach_fn, chan); 1582 1583 error = vmbus_chan_add(chan); 1584 if (error) { 1585 device_printf(sc->vmbus_dev, "add chan%u failed: %d\n", 1586 chan->ch_id, error); 1587 atomic_subtract_int(&chan->ch_refs, 1); 1588 vmbus_chan_free(chan); 1589 return; 1590 } 1591 taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_attach_task); 1592} 1593 1594static void 1595vmbus_chan_msgproc_chrescind(struct vmbus_softc *sc, 1596 const struct vmbus_message *msg) 1597{ 1598 const struct vmbus_chanmsg_chrescind *note; 1599 struct vmbus_channel *chan; 1600 1601 note = (const struct vmbus_chanmsg_chrescind *)msg->msg_data; 1602 if (note->chm_chanid > VMBUS_CHAN_MAX) { 1603 device_printf(sc->vmbus_dev, "invalid revoked chan%u\n", 1604 note->chm_chanid); 1605 return; 1606 } 1607 1608 /* 1609 * Find and remove the target channel from the channel list. 1610 */ 1611 mtx_lock(&sc->vmbus_chan_lock); 1612 TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) { 1613 if (chan->ch_id == note->chm_chanid) 1614 break; 1615 } 1616 if (chan == NULL) { 1617 mtx_unlock(&sc->vmbus_chan_lock); 1618 device_printf(sc->vmbus_dev, "chan%u is not offered\n", 1619 note->chm_chanid); 1620 return; 1621 } 1622 vmbus_chan_rem_list(sc, chan); 1623 mtx_unlock(&sc->vmbus_chan_lock); 1624 1625 if (VMBUS_CHAN_ISPRIMARY(chan)) { 1626 /* 1627 * The target channel is a primary channel; remove the 1628 * target channel from the primary channel list now, 1629 * instead of later, so that it will not be found by 1630 * other sub-channel offers, which are processed in 1631 * this thread. 1632 */ 1633 mtx_lock(&sc->vmbus_prichan_lock); 1634 vmbus_chan_rem_prilist(sc, chan); 1635 mtx_unlock(&sc->vmbus_prichan_lock); 1636 } 1637 1638 /* 1639 * NOTE: 1640 * The following processing order is critical: 1641 * Set the REVOKED state flag before orphaning the installed xact. 1642 */ 1643 1644 if (atomic_testandset_int(&chan->ch_stflags, 1645 VMBUS_CHAN_ST_REVOKED_SHIFT)) 1646 panic("channel has already been revoked"); 1647 1648 sx_xlock(&chan->ch_orphan_lock); 1649 if (chan->ch_orphan_xact != NULL) 1650 vmbus_xact_ctx_orphan(chan->ch_orphan_xact); 1651 sx_xunlock(&chan->ch_orphan_lock); 1652 1653 if (bootverbose) 1654 vmbus_chan_printf(chan, "chan%u revoked\n", note->chm_chanid); 1655 vmbus_chan_detach(chan); 1656} 1657 1658static int 1659vmbus_chan_release(struct vmbus_channel *chan) 1660{ 1661 struct vmbus_softc *sc = chan->ch_vmbus; 1662 struct vmbus_chanmsg_chfree *req; 1663 struct vmbus_msghc *mh; 1664 int error; 1665 1666 mh = vmbus_msghc_get(sc, sizeof(*req)); 1667 if (mh == NULL) { 1668 vmbus_chan_printf(chan, 1669 "can not get msg hypercall for chfree(chan%u)\n", 1670 chan->ch_id); 1671 return (ENXIO); 1672 } 1673 1674 req = vmbus_msghc_dataptr(mh); 1675 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHFREE; 1676 req->chm_chanid = chan->ch_id; 1677 1678 error = vmbus_msghc_exec_noresult(mh); 1679 vmbus_msghc_put(sc, mh); 1680 1681 if (error) { 1682 vmbus_chan_printf(chan, 1683 "chfree(chan%u) msg hypercall exec failed: %d\n", 1684 chan->ch_id, error); 1685 } else { 1686 if (bootverbose) 1687 vmbus_chan_printf(chan, "chan%u freed\n", chan->ch_id); 1688 } 1689 return (error); 1690} 1691 1692static void 1693vmbus_prichan_detach_task(void *xchan, int pending __unused) 1694{ 1695 struct vmbus_channel *chan = xchan; 1696 1697 KASSERT(VMBUS_CHAN_ISPRIMARY(chan), 1698 ("chan%u is not primary channel", chan->ch_id)); 1699 1700 /* Delete and detach the device associated with this channel. */ 1701 vmbus_delete_child(chan); 1702 1703 /* Release this channel (back to vmbus). */ 1704 vmbus_chan_release(chan); 1705 1706 /* Free this channel's resource. */ 1707 vmbus_chan_free(chan); 1708} 1709 1710static void 1711vmbus_subchan_detach_task(void *xchan, int pending __unused) 1712{ 1713 struct vmbus_channel *chan = xchan; 1714 struct vmbus_channel *pri_chan = chan->ch_prichan; 1715 1716 KASSERT(!VMBUS_CHAN_ISPRIMARY(chan), 1717 ("chan%u is primary channel", chan->ch_id)); 1718 1719 /* Release this channel (back to vmbus). */ 1720 vmbus_chan_release(chan); 1721 1722 /* Unlink from its primary channel's sub-channel list. */ 1723 mtx_lock(&pri_chan->ch_subchan_lock); 1724 vmbus_chan_rem_sublist(pri_chan, chan); 1725 mtx_unlock(&pri_chan->ch_subchan_lock); 1726 /* Notify anyone that is waiting for this sub-channel to vanish. */ 1727 wakeup(pri_chan); 1728 1729 /* Free this channel's resource. */ 1730 vmbus_chan_free(chan); 1731} 1732 1733static void 1734vmbus_prichan_attach_task(void *xchan, int pending __unused) 1735{ 1736 1737 /* 1738 * Add device for this primary channel. 1739 */ 1740 vmbus_add_child(xchan); 1741} 1742 1743static void 1744vmbus_subchan_attach_task(void *xchan __unused, int pending __unused) 1745{ 1746 1747 /* Nothing */ 1748} 1749 1750void 1751vmbus_chan_destroy_all(struct vmbus_softc *sc) 1752{ 1753 1754 /* 1755 * Detach all devices and destroy the corresponding primary 1756 * channels. 1757 */ 1758 for (;;) { 1759 struct vmbus_channel *chan; 1760 1761 mtx_lock(&sc->vmbus_chan_lock); 1762 TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) { 1763 if (VMBUS_CHAN_ISPRIMARY(chan)) 1764 break; 1765 } 1766 if (chan == NULL) { 1767 /* No more primary channels; done. */ 1768 mtx_unlock(&sc->vmbus_chan_lock); 1769 break; 1770 } 1771 vmbus_chan_rem_list(sc, chan); 1772 mtx_unlock(&sc->vmbus_chan_lock); 1773 1774 mtx_lock(&sc->vmbus_prichan_lock); 1775 vmbus_chan_rem_prilist(sc, chan); 1776 mtx_unlock(&sc->vmbus_prichan_lock); 1777 1778 taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task); 1779 } 1780} 1781 1782struct vmbus_channel ** 1783vmbus_subchan_get(struct vmbus_channel *pri_chan, int subchan_cnt) 1784{ 1785 struct vmbus_channel **ret, *chan; 1786 int i; 1787 1788 KASSERT(subchan_cnt > 0, ("invalid sub-channel count %d", subchan_cnt)); 1789 1790 ret = malloc(subchan_cnt * sizeof(struct vmbus_channel *), M_TEMP, 1791 M_WAITOK); 1792 1793 mtx_lock(&pri_chan->ch_subchan_lock); 1794 1795 while (pri_chan->ch_subchan_cnt < subchan_cnt) 1796 mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "subch", 0); 1797 1798 i = 0; 1799 TAILQ_FOREACH(chan, &pri_chan->ch_subchans, ch_sublink) { 1800 /* TODO: refcnt chan */ 1801 ret[i] = chan; 1802 1803 ++i; 1804 if (i == subchan_cnt) 1805 break; 1806 } 1807 KASSERT(i == subchan_cnt, ("invalid subchan count %d, should be %d", 1808 pri_chan->ch_subchan_cnt, subchan_cnt)); 1809 1810 mtx_unlock(&pri_chan->ch_subchan_lock); 1811 1812 return ret; 1813} 1814 1815void 1816vmbus_subchan_rel(struct vmbus_channel **subchan, int subchan_cnt __unused) 1817{ 1818 1819 free(subchan, M_TEMP); 1820} 1821 1822void 1823vmbus_subchan_drain(struct vmbus_channel *pri_chan) 1824{ 1825 mtx_lock(&pri_chan->ch_subchan_lock); 1826 while (pri_chan->ch_subchan_cnt > 0) 1827 mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "dsubch", 0); 1828 mtx_unlock(&pri_chan->ch_subchan_lock); 1829} 1830 1831void 1832vmbus_chan_msgproc(struct vmbus_softc *sc, const struct vmbus_message *msg) 1833{ 1834 vmbus_chanmsg_proc_t msg_proc; 1835 uint32_t msg_type; 1836 1837 msg_type = ((const struct vmbus_chanmsg_hdr *)msg->msg_data)->chm_type; 1838 KASSERT(msg_type < VMBUS_CHANMSG_TYPE_MAX, 1839 ("invalid message type %u", msg_type)); 1840 1841 msg_proc = vmbus_chan_msgprocs[msg_type]; 1842 if (msg_proc != NULL) 1843 msg_proc(sc, msg); 1844} 1845 1846void 1847vmbus_chan_set_readbatch(struct vmbus_channel *chan, bool on) 1848{ 1849 if (!on) 1850 chan->ch_flags &= ~VMBUS_CHAN_FLAG_BATCHREAD; 1851 else 1852 chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD; 1853} 1854 1855uint32_t 1856vmbus_chan_id(const struct vmbus_channel *chan) 1857{ 1858 return chan->ch_id; 1859} 1860 1861uint32_t 1862vmbus_chan_subidx(const struct vmbus_channel *chan) 1863{ 1864 return chan->ch_subidx; 1865} 1866 1867bool 1868vmbus_chan_is_primary(const struct vmbus_channel *chan) 1869{ 1870 if (VMBUS_CHAN_ISPRIMARY(chan)) 1871 return true; 1872 else 1873 return false; 1874} 1875 1876const struct hyperv_guid * 1877vmbus_chan_guid_inst(const struct vmbus_channel *chan) 1878{ 1879 return &chan->ch_guid_inst; 1880} 1881 1882int 1883vmbus_chan_prplist_nelem(int br_size, int prpcnt_max, int dlen_max) 1884{ 1885 int elem_size; 1886 1887 elem_size = __offsetof(struct vmbus_chanpkt_prplist, 1888 cp_range[0].gpa_page[prpcnt_max]); 1889 elem_size += dlen_max; 1890 elem_size = VMBUS_CHANPKT_TOTLEN(elem_size); 1891 1892 return (vmbus_br_nelem(br_size, elem_size)); 1893} 1894 1895bool 1896vmbus_chan_tx_empty(const struct vmbus_channel *chan) 1897{ 1898 1899 return (vmbus_txbr_empty(&chan->ch_txbr)); 1900} 1901 1902bool 1903vmbus_chan_rx_empty(const struct vmbus_channel *chan) 1904{ 1905 1906 return (vmbus_rxbr_empty(&chan->ch_rxbr)); 1907} 1908 1909static int 1910vmbus_chan_printf(const struct vmbus_channel *chan, const char *fmt, ...) 1911{ 1912 va_list ap; 1913 device_t dev; 1914 int retval; 1915 1916 if (chan->ch_dev == NULL || !device_is_alive(chan->ch_dev)) 1917 dev = chan->ch_vmbus->vmbus_dev; 1918 else 1919 dev = chan->ch_dev; 1920 1921 retval = device_print_prettyname(dev); 1922 va_start(ap, fmt); 1923 retval += vprintf(fmt, ap); 1924 va_end(ap); 1925 1926 return (retval); 1927} 1928 1929void 1930vmbus_chan_run_task(struct vmbus_channel *chan, struct task *task) 1931{ 1932 1933 taskqueue_enqueue(chan->ch_tq, task); 1934 taskqueue_drain(chan->ch_tq, task); 1935} 1936 1937struct taskqueue * 1938vmbus_chan_mgmt_tq(const struct vmbus_channel *chan) 1939{ 1940 1941 return (chan->ch_mgmt_tq); 1942} 1943 1944bool 1945vmbus_chan_is_revoked(const struct vmbus_channel *chan) 1946{ 1947 1948 if (chan->ch_stflags & VMBUS_CHAN_ST_REVOKED) 1949 return (true); 1950 return (false); 1951} 1952 1953void 1954vmbus_chan_set_orphan(struct vmbus_channel *chan, struct vmbus_xact_ctx *xact) 1955{ 1956 1957 sx_xlock(&chan->ch_orphan_lock); 1958 chan->ch_orphan_xact = xact; 1959 sx_xunlock(&chan->ch_orphan_lock); 1960} 1961 1962void 1963vmbus_chan_unset_orphan(struct vmbus_channel *chan) 1964{ 1965 1966 sx_xlock(&chan->ch_orphan_lock); 1967 chan->ch_orphan_xact = NULL; 1968 sx_xunlock(&chan->ch_orphan_lock); 1969} 1970 1971const void * 1972vmbus_chan_xact_wait(const struct vmbus_channel *chan, 1973 struct vmbus_xact *xact, size_t *resp_len, bool can_sleep) 1974{ 1975 const void *ret; 1976 1977 if (can_sleep) 1978 ret = vmbus_xact_wait(xact, resp_len); 1979 else 1980 ret = vmbus_xact_busywait(xact, resp_len); 1981 if (vmbus_chan_is_revoked(chan)) { 1982 /* 1983 * This xact probably is interrupted, and the 1984 * interruption can race the reply reception, 1985 * so we have to make sure that there are nothing 1986 * left on the RX bufring, i.e. this xact will 1987 * not be touched, once this function returns. 1988 * 1989 * Since the hypervisor will not put more data 1990 * onto the RX bufring once the channel is revoked, 1991 * the following loop will be terminated, once all 1992 * data are drained by the driver's channel 1993 * callback. 1994 */ 1995 while (!vmbus_chan_rx_empty(chan)) { 1996 if (can_sleep) 1997 pause("chxact", 1); 1998 else 1999 DELAY(1000); 2000 } 2001 } 2002 return (ret); 2003} 2004