1/*- 2 * Copyright (c) 2009-2012,2016 Microsoft Corp. 3 * Copyright (c) 2012 NetApp Inc. 4 * Copyright (c) 2012 Citrix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29#include <sys/cdefs.h> 30__FBSDID("$FreeBSD: stable/11/sys/dev/hyperv/vmbus/vmbus_chan.c 311389 2017-01-05 08:42:58Z sephe $"); 31 32#include <sys/param.h> 33#include <sys/bus.h> 34#include <sys/callout.h> 35#include <sys/kernel.h> 36#include <sys/lock.h> 37#include <sys/malloc.h> 38#include <sys/mutex.h> 39#include <sys/smp.h> 40#include <sys/sysctl.h> 41#include <sys/systm.h> 42 43#include <machine/atomic.h> 44#include <machine/stdarg.h> 45 46#include <dev/hyperv/include/hyperv_busdma.h> 47#include <dev/hyperv/include/vmbus_xact.h> 48#include <dev/hyperv/vmbus/hyperv_var.h> 49#include <dev/hyperv/vmbus/vmbus_reg.h> 50#include <dev/hyperv/vmbus/vmbus_var.h> 51#include <dev/hyperv/vmbus/vmbus_brvar.h> 52#include <dev/hyperv/vmbus/vmbus_chanvar.h> 53 54struct vmbus_chan_pollarg { 55 struct vmbus_channel *poll_chan; 56 u_int poll_hz; 57}; 58 59static void vmbus_chan_update_evtflagcnt( 60 struct vmbus_softc *, 61 const struct vmbus_channel *); 62static int vmbus_chan_close_internal( 63 struct vmbus_channel *); 64static int vmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS); 65static void vmbus_chan_sysctl_create( 66 struct vmbus_channel *); 67static struct vmbus_channel *vmbus_chan_alloc(struct vmbus_softc *); 68static void vmbus_chan_free(struct vmbus_channel *); 69static int vmbus_chan_add(struct vmbus_channel *); 70static void vmbus_chan_cpu_default(struct vmbus_channel *); 71static int vmbus_chan_release(struct vmbus_channel *); 72static void vmbus_chan_set_chmap(struct vmbus_channel *); 73static void vmbus_chan_clear_chmap(struct vmbus_channel *); 74static void vmbus_chan_detach(struct vmbus_channel *); 75static bool vmbus_chan_wait_revoke( 76 const struct vmbus_channel *, bool); 77static void vmbus_chan_poll_timeout(void *); 78static bool vmbus_chan_poll_cancel_intq( 79 struct vmbus_channel *); 80static void vmbus_chan_poll_cancel(struct vmbus_channel *); 81 82static void vmbus_chan_ins_prilist(struct vmbus_softc *, 83 struct vmbus_channel *); 84static void vmbus_chan_rem_prilist(struct vmbus_softc *, 85 struct vmbus_channel *); 86static void vmbus_chan_ins_list(struct vmbus_softc *, 87 struct vmbus_channel *); 88static void vmbus_chan_rem_list(struct vmbus_softc *, 89 struct vmbus_channel *); 90static void vmbus_chan_ins_sublist(struct vmbus_channel *, 91 struct vmbus_channel *); 92static void vmbus_chan_rem_sublist(struct vmbus_channel *, 93 struct vmbus_channel *); 94 95static void vmbus_chan_task(void *, int); 96static void vmbus_chan_task_nobatch(void *, int); 97static void vmbus_chan_poll_task(void *, int); 98static void vmbus_chan_clrchmap_task(void *, int); 99static void vmbus_chan_pollcfg_task(void *, int); 100static void vmbus_chan_polldis_task(void *, int); 101static void vmbus_chan_poll_cancel_task(void *, int); 102static void vmbus_prichan_attach_task(void *, int); 103static void vmbus_subchan_attach_task(void *, int); 104static void vmbus_prichan_detach_task(void *, int); 105static void vmbus_subchan_detach_task(void *, int); 106 107static void vmbus_chan_msgproc_choffer(struct vmbus_softc *, 108 const struct vmbus_message *); 109static void vmbus_chan_msgproc_chrescind( 110 struct vmbus_softc *, 111 const struct vmbus_message *); 112 113static int vmbus_chan_printf(const struct vmbus_channel *, 114 const char *, ...) __printflike(2, 3); 115 116/* 117 * Vmbus channel message processing. 118 */ 119static const vmbus_chanmsg_proc_t 120vmbus_chan_msgprocs[VMBUS_CHANMSG_TYPE_MAX] = { 121 VMBUS_CHANMSG_PROC(CHOFFER, vmbus_chan_msgproc_choffer), 122 VMBUS_CHANMSG_PROC(CHRESCIND, vmbus_chan_msgproc_chrescind), 123 124 VMBUS_CHANMSG_PROC_WAKEUP(CHOPEN_RESP), 125 VMBUS_CHANMSG_PROC_WAKEUP(GPADL_CONNRESP), 126 VMBUS_CHANMSG_PROC_WAKEUP(GPADL_DISCONNRESP) 127}; 128 129/* 130 * Notify host that there are data pending on our TX bufring. 131 */ 132static __inline void 133vmbus_chan_signal_tx(const struct vmbus_channel *chan) 134{ 135 atomic_set_long(chan->ch_evtflag, chan->ch_evtflag_mask); 136 if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF) 137 atomic_set_int(chan->ch_montrig, chan->ch_montrig_mask); 138 else 139 hypercall_signal_event(chan->ch_monprm_dma.hv_paddr); 140} 141 142static void 143vmbus_chan_ins_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan) 144{ 145 146 mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED); 147 if (atomic_testandset_int(&chan->ch_stflags, 148 VMBUS_CHAN_ST_ONPRIL_SHIFT)) 149 panic("channel is already on the prilist"); 150 TAILQ_INSERT_TAIL(&sc->vmbus_prichans, chan, ch_prilink); 151} 152 153static void 154vmbus_chan_rem_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan) 155{ 156 157 mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED); 158 if (atomic_testandclear_int(&chan->ch_stflags, 159 VMBUS_CHAN_ST_ONPRIL_SHIFT) == 0) 160 panic("channel is not on the prilist"); 161 TAILQ_REMOVE(&sc->vmbus_prichans, chan, ch_prilink); 162} 163 164static void 165vmbus_chan_ins_sublist(struct vmbus_channel *prichan, 166 struct vmbus_channel *chan) 167{ 168 169 mtx_assert(&prichan->ch_subchan_lock, MA_OWNED); 170 171 if (atomic_testandset_int(&chan->ch_stflags, 172 VMBUS_CHAN_ST_ONSUBL_SHIFT)) 173 panic("channel is already on the sublist"); 174 TAILQ_INSERT_TAIL(&prichan->ch_subchans, chan, ch_sublink); 175 176 /* Bump sub-channel count. */ 177 prichan->ch_subchan_cnt++; 178} 179 180static void 181vmbus_chan_rem_sublist(struct vmbus_channel *prichan, 182 struct vmbus_channel *chan) 183{ 184 185 mtx_assert(&prichan->ch_subchan_lock, MA_OWNED); 186 187 KASSERT(prichan->ch_subchan_cnt > 0, 188 ("invalid subchan_cnt %d", prichan->ch_subchan_cnt)); 189 prichan->ch_subchan_cnt--; 190 191 if (atomic_testandclear_int(&chan->ch_stflags, 192 VMBUS_CHAN_ST_ONSUBL_SHIFT) == 0) 193 panic("channel is not on the sublist"); 194 TAILQ_REMOVE(&prichan->ch_subchans, chan, ch_sublink); 195} 196 197static void 198vmbus_chan_ins_list(struct vmbus_softc *sc, struct vmbus_channel *chan) 199{ 200 201 mtx_assert(&sc->vmbus_chan_lock, MA_OWNED); 202 if (atomic_testandset_int(&chan->ch_stflags, 203 VMBUS_CHAN_ST_ONLIST_SHIFT)) 204 panic("channel is already on the list"); 205 TAILQ_INSERT_TAIL(&sc->vmbus_chans, chan, ch_link); 206} 207 208static void 209vmbus_chan_rem_list(struct vmbus_softc *sc, struct vmbus_channel *chan) 210{ 211 212 mtx_assert(&sc->vmbus_chan_lock, MA_OWNED); 213 if (atomic_testandclear_int(&chan->ch_stflags, 214 VMBUS_CHAN_ST_ONLIST_SHIFT) == 0) 215 panic("channel is not on the list"); 216 TAILQ_REMOVE(&sc->vmbus_chans, chan, ch_link); 217} 218 219static int 220vmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS) 221{ 222 struct vmbus_channel *chan = arg1; 223 int mnf = 0; 224 225 if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF) 226 mnf = 1; 227 return sysctl_handle_int(oidp, &mnf, 0, req); 228} 229 230static void 231vmbus_chan_sysctl_create(struct vmbus_channel *chan) 232{ 233 struct sysctl_oid *ch_tree, *chid_tree, *br_tree; 234 struct sysctl_ctx_list *ctx; 235 uint32_t ch_id; 236 char name[16]; 237 238 /* 239 * Add sysctl nodes related to this channel to this 240 * channel's sysctl ctx, so that they can be destroyed 241 * independently upon close of this channel, which can 242 * happen even if the device is not detached. 243 */ 244 ctx = &chan->ch_sysctl_ctx; 245 sysctl_ctx_init(ctx); 246 247 /* 248 * Create dev.NAME.UNIT.channel tree. 249 */ 250 ch_tree = SYSCTL_ADD_NODE(ctx, 251 SYSCTL_CHILDREN(device_get_sysctl_tree(chan->ch_dev)), 252 OID_AUTO, "channel", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 253 if (ch_tree == NULL) 254 return; 255 256 /* 257 * Create dev.NAME.UNIT.channel.CHANID tree. 258 */ 259 if (VMBUS_CHAN_ISPRIMARY(chan)) 260 ch_id = chan->ch_id; 261 else 262 ch_id = chan->ch_prichan->ch_id; 263 snprintf(name, sizeof(name), "%d", ch_id); 264 chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree), 265 OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 266 if (chid_tree == NULL) 267 return; 268 269 if (!VMBUS_CHAN_ISPRIMARY(chan)) { 270 /* 271 * Create dev.NAME.UNIT.channel.CHANID.sub tree. 272 */ 273 ch_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree), 274 OID_AUTO, "sub", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 275 if (ch_tree == NULL) 276 return; 277 278 /* 279 * Create dev.NAME.UNIT.channel.CHANID.sub.SUBIDX tree. 280 * 281 * NOTE: 282 * chid_tree is changed to this new sysctl tree. 283 */ 284 snprintf(name, sizeof(name), "%d", chan->ch_subidx); 285 chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree), 286 OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 287 if (chid_tree == NULL) 288 return; 289 290 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 291 "chanid", CTLFLAG_RD, &chan->ch_id, 0, "channel id"); 292 } 293 294 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 295 "cpu", CTLFLAG_RD, &chan->ch_cpuid, 0, "owner CPU id"); 296 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 297 "mnf", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, 298 chan, 0, vmbus_chan_sysctl_mnf, "I", 299 "has monitor notification facilities"); 300 301 br_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 302 "br", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 303 if (br_tree != NULL) { 304 /* 305 * Create sysctl tree for RX bufring. 306 */ 307 vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_rxbr.rxbr, "rx"); 308 /* 309 * Create sysctl tree for TX bufring. 310 */ 311 vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_txbr.txbr, "tx"); 312 } 313} 314 315int 316vmbus_chan_open(struct vmbus_channel *chan, int txbr_size, int rxbr_size, 317 const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg) 318{ 319 struct vmbus_chan_br cbr; 320 int error; 321 322 /* 323 * Allocate the TX+RX bufrings. 324 */ 325 KASSERT(chan->ch_bufring == NULL, ("bufrings are allocated")); 326 chan->ch_bufring = hyperv_dmamem_alloc(bus_get_dma_tag(chan->ch_dev), 327 PAGE_SIZE, 0, txbr_size + rxbr_size, &chan->ch_bufring_dma, 328 BUS_DMA_WAITOK); 329 if (chan->ch_bufring == NULL) { 330 vmbus_chan_printf(chan, "bufring allocation failed\n"); 331 return (ENOMEM); 332 } 333 334 cbr.cbr = chan->ch_bufring; 335 cbr.cbr_paddr = chan->ch_bufring_dma.hv_paddr; 336 cbr.cbr_txsz = txbr_size; 337 cbr.cbr_rxsz = rxbr_size; 338 339 error = vmbus_chan_open_br(chan, &cbr, udata, udlen, cb, cbarg); 340 if (error) { 341 if (error == EISCONN) { 342 /* 343 * XXX 344 * The bufring GPADL is still connected; abandon 345 * this bufring, instead of having mysterious 346 * crash or trashed data later on. 347 */ 348 vmbus_chan_printf(chan, "chan%u bufring GPADL " 349 "is still connected upon channel open error; " 350 "leak %d bytes memory\n", chan->ch_id, 351 txbr_size + rxbr_size); 352 } else { 353 hyperv_dmamem_free(&chan->ch_bufring_dma, 354 chan->ch_bufring); 355 } 356 chan->ch_bufring = NULL; 357 } 358 return (error); 359} 360 361int 362vmbus_chan_open_br(struct vmbus_channel *chan, const struct vmbus_chan_br *cbr, 363 const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg) 364{ 365 struct vmbus_softc *sc = chan->ch_vmbus; 366 const struct vmbus_message *msg; 367 struct vmbus_chanmsg_chopen *req; 368 struct vmbus_msghc *mh; 369 uint32_t status; 370 int error, txbr_size, rxbr_size; 371 task_fn_t *task_fn; 372 uint8_t *br; 373 374 if (udlen > VMBUS_CHANMSG_CHOPEN_UDATA_SIZE) { 375 vmbus_chan_printf(chan, 376 "invalid udata len %d for chan%u\n", udlen, chan->ch_id); 377 return (EINVAL); 378 } 379 380 br = cbr->cbr; 381 txbr_size = cbr->cbr_txsz; 382 rxbr_size = cbr->cbr_rxsz; 383 KASSERT((txbr_size & PAGE_MASK) == 0, 384 ("send bufring size is not multiple page")); 385 KASSERT((rxbr_size & PAGE_MASK) == 0, 386 ("recv bufring size is not multiple page")); 387 KASSERT((cbr->cbr_paddr & PAGE_MASK) == 0, 388 ("bufring is not page aligned")); 389 390 /* 391 * Zero out the TX/RX bufrings, in case that they were used before. 392 */ 393 memset(br, 0, txbr_size + rxbr_size); 394 395 if (atomic_testandset_int(&chan->ch_stflags, 396 VMBUS_CHAN_ST_OPENED_SHIFT)) 397 panic("double-open chan%u", chan->ch_id); 398 399 chan->ch_cb = cb; 400 chan->ch_cbarg = cbarg; 401 402 vmbus_chan_update_evtflagcnt(sc, chan); 403 404 chan->ch_tq = VMBUS_PCPU_GET(chan->ch_vmbus, event_tq, chan->ch_cpuid); 405 if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD) 406 task_fn = vmbus_chan_task; 407 else 408 task_fn = vmbus_chan_task_nobatch; 409 TASK_INIT(&chan->ch_task, 0, task_fn, chan); 410 411 /* TX bufring comes first */ 412 vmbus_txbr_setup(&chan->ch_txbr, br, txbr_size); 413 /* RX bufring immediately follows TX bufring */ 414 vmbus_rxbr_setup(&chan->ch_rxbr, br + txbr_size, rxbr_size); 415 416 /* Create sysctl tree for this channel */ 417 vmbus_chan_sysctl_create(chan); 418 419 /* 420 * Connect the bufrings, both RX and TX, to this channel. 421 */ 422 error = vmbus_chan_gpadl_connect(chan, cbr->cbr_paddr, 423 txbr_size + rxbr_size, &chan->ch_bufring_gpadl); 424 if (error) { 425 vmbus_chan_printf(chan, 426 "failed to connect bufring GPADL to chan%u\n", chan->ch_id); 427 goto failed; 428 } 429 430 /* 431 * Install this channel, before it is opened, but after everything 432 * else has been setup. 433 */ 434 vmbus_chan_set_chmap(chan); 435 436 /* 437 * Open channel w/ the bufring GPADL on the target CPU. 438 */ 439 mh = vmbus_msghc_get(sc, sizeof(*req)); 440 if (mh == NULL) { 441 vmbus_chan_printf(chan, 442 "can not get msg hypercall for chopen(chan%u)\n", 443 chan->ch_id); 444 error = ENXIO; 445 goto failed; 446 } 447 448 req = vmbus_msghc_dataptr(mh); 449 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHOPEN; 450 req->chm_chanid = chan->ch_id; 451 req->chm_openid = chan->ch_id; 452 req->chm_gpadl = chan->ch_bufring_gpadl; 453 req->chm_vcpuid = chan->ch_vcpuid; 454 req->chm_txbr_pgcnt = txbr_size >> PAGE_SHIFT; 455 if (udlen > 0) 456 memcpy(req->chm_udata, udata, udlen); 457 458 error = vmbus_msghc_exec(sc, mh); 459 if (error) { 460 vmbus_chan_printf(chan, 461 "chopen(chan%u) msg hypercall exec failed: %d\n", 462 chan->ch_id, error); 463 vmbus_msghc_put(sc, mh); 464 goto failed; 465 } 466 467 for (;;) { 468 msg = vmbus_msghc_poll_result(sc, mh); 469 if (msg != NULL) 470 break; 471 if (vmbus_chan_is_revoked(chan)) { 472 int i; 473 474 /* 475 * NOTE: 476 * Hypervisor does _not_ send response CHOPEN to 477 * a revoked channel. 478 */ 479 vmbus_chan_printf(chan, 480 "chan%u is revoked, when it is being opened\n", 481 chan->ch_id); 482 483 /* 484 * XXX 485 * Add extra delay before cancel the hypercall 486 * execution; mainly to close any possible 487 * CHRESCIND and CHOPEN_RESP races on the 488 * hypervisor side. 489 */ 490#define REVOKE_LINGER 100 491 for (i = 0; i < REVOKE_LINGER; ++i) { 492 msg = vmbus_msghc_poll_result(sc, mh); 493 if (msg != NULL) 494 break; 495 pause("rchopen", 1); 496 } 497#undef REVOKE_LINGER 498 if (msg == NULL) 499 vmbus_msghc_exec_cancel(sc, mh); 500 break; 501 } 502 pause("chopen", 1); 503 } 504 if (msg != NULL) { 505 status = ((const struct vmbus_chanmsg_chopen_resp *) 506 msg->msg_data)->chm_status; 507 } else { 508 /* XXX any non-0 value is ok here. */ 509 status = 0xff; 510 } 511 512 vmbus_msghc_put(sc, mh); 513 514 if (status == 0) { 515 if (bootverbose) 516 vmbus_chan_printf(chan, "chan%u opened\n", chan->ch_id); 517 return (0); 518 } 519 520 vmbus_chan_printf(chan, "failed to open chan%u\n", chan->ch_id); 521 error = ENXIO; 522 523failed: 524 sysctl_ctx_free(&chan->ch_sysctl_ctx); 525 vmbus_chan_clear_chmap(chan); 526 if (chan->ch_bufring_gpadl != 0) { 527 int error1; 528 529 error1 = vmbus_chan_gpadl_disconnect(chan, 530 chan->ch_bufring_gpadl); 531 if (error1) { 532 /* 533 * Give caller a hint that the bufring GPADL is still 534 * connected. 535 */ 536 error = EISCONN; 537 } 538 chan->ch_bufring_gpadl = 0; 539 } 540 atomic_clear_int(&chan->ch_stflags, VMBUS_CHAN_ST_OPENED); 541 return (error); 542} 543 544int 545vmbus_chan_gpadl_connect(struct vmbus_channel *chan, bus_addr_t paddr, 546 int size, uint32_t *gpadl0) 547{ 548 struct vmbus_softc *sc = chan->ch_vmbus; 549 struct vmbus_msghc *mh; 550 struct vmbus_chanmsg_gpadl_conn *req; 551 const struct vmbus_message *msg; 552 size_t reqsz; 553 uint32_t gpadl, status; 554 int page_count, range_len, i, cnt, error; 555 uint64_t page_id; 556 557 KASSERT(*gpadl0 == 0, ("GPADL is not zero")); 558 559 /* 560 * Preliminary checks. 561 */ 562 563 KASSERT((size & PAGE_MASK) == 0, 564 ("invalid GPA size %d, not multiple page size", size)); 565 page_count = size >> PAGE_SHIFT; 566 567 KASSERT((paddr & PAGE_MASK) == 0, 568 ("GPA is not page aligned %jx", (uintmax_t)paddr)); 569 page_id = paddr >> PAGE_SHIFT; 570 571 range_len = __offsetof(struct vmbus_gpa_range, gpa_page[page_count]); 572 /* 573 * We don't support multiple GPA ranges. 574 */ 575 if (range_len > UINT16_MAX) { 576 vmbus_chan_printf(chan, "GPA too large, %d pages\n", 577 page_count); 578 return EOPNOTSUPP; 579 } 580 581 /* 582 * Allocate GPADL id. 583 */ 584 gpadl = vmbus_gpadl_alloc(sc); 585 586 /* 587 * Connect this GPADL to the target channel. 588 * 589 * NOTE: 590 * Since each message can only hold small set of page 591 * addresses, several messages may be required to 592 * complete the connection. 593 */ 594 if (page_count > VMBUS_CHANMSG_GPADL_CONN_PGMAX) 595 cnt = VMBUS_CHANMSG_GPADL_CONN_PGMAX; 596 else 597 cnt = page_count; 598 page_count -= cnt; 599 600 reqsz = __offsetof(struct vmbus_chanmsg_gpadl_conn, 601 chm_range.gpa_page[cnt]); 602 mh = vmbus_msghc_get(sc, reqsz); 603 if (mh == NULL) { 604 vmbus_chan_printf(chan, 605 "can not get msg hypercall for gpadl_conn(chan%u)\n", 606 chan->ch_id); 607 return EIO; 608 } 609 610 req = vmbus_msghc_dataptr(mh); 611 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_CONN; 612 req->chm_chanid = chan->ch_id; 613 req->chm_gpadl = gpadl; 614 req->chm_range_len = range_len; 615 req->chm_range_cnt = 1; 616 req->chm_range.gpa_len = size; 617 req->chm_range.gpa_ofs = 0; 618 for (i = 0; i < cnt; ++i) 619 req->chm_range.gpa_page[i] = page_id++; 620 621 error = vmbus_msghc_exec(sc, mh); 622 if (error) { 623 vmbus_chan_printf(chan, 624 "gpadl_conn(chan%u) msg hypercall exec failed: %d\n", 625 chan->ch_id, error); 626 vmbus_msghc_put(sc, mh); 627 return error; 628 } 629 630 while (page_count > 0) { 631 struct vmbus_chanmsg_gpadl_subconn *subreq; 632 633 if (page_count > VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX) 634 cnt = VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX; 635 else 636 cnt = page_count; 637 page_count -= cnt; 638 639 reqsz = __offsetof(struct vmbus_chanmsg_gpadl_subconn, 640 chm_gpa_page[cnt]); 641 vmbus_msghc_reset(mh, reqsz); 642 643 subreq = vmbus_msghc_dataptr(mh); 644 subreq->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_SUBCONN; 645 subreq->chm_gpadl = gpadl; 646 for (i = 0; i < cnt; ++i) 647 subreq->chm_gpa_page[i] = page_id++; 648 649 vmbus_msghc_exec_noresult(mh); 650 } 651 KASSERT(page_count == 0, ("invalid page count %d", page_count)); 652 653 msg = vmbus_msghc_wait_result(sc, mh); 654 status = ((const struct vmbus_chanmsg_gpadl_connresp *) 655 msg->msg_data)->chm_status; 656 657 vmbus_msghc_put(sc, mh); 658 659 if (status != 0) { 660 vmbus_chan_printf(chan, "gpadl_conn(chan%u) failed: %u\n", 661 chan->ch_id, status); 662 return EIO; 663 } 664 665 /* Done; commit the GPADL id. */ 666 *gpadl0 = gpadl; 667 if (bootverbose) { 668 vmbus_chan_printf(chan, "gpadl_conn(chan%u) succeeded\n", 669 chan->ch_id); 670 } 671 return 0; 672} 673 674static bool 675vmbus_chan_wait_revoke(const struct vmbus_channel *chan, bool can_sleep) 676{ 677#define WAIT_COUNT 200 /* 200ms */ 678 679 int i; 680 681 for (i = 0; i < WAIT_COUNT; ++i) { 682 if (vmbus_chan_is_revoked(chan)) 683 return (true); 684 if (can_sleep) 685 pause("wchrev", 1); 686 else 687 DELAY(1000); 688 } 689 return (false); 690 691#undef WAIT_COUNT 692} 693 694/* 695 * Disconnect the GPA from the target channel 696 */ 697int 698vmbus_chan_gpadl_disconnect(struct vmbus_channel *chan, uint32_t gpadl) 699{ 700 struct vmbus_softc *sc = chan->ch_vmbus; 701 struct vmbus_msghc *mh; 702 struct vmbus_chanmsg_gpadl_disconn *req; 703 int error; 704 705 KASSERT(gpadl != 0, ("GPADL is zero")); 706 707 mh = vmbus_msghc_get(sc, sizeof(*req)); 708 if (mh == NULL) { 709 vmbus_chan_printf(chan, 710 "can not get msg hypercall for gpadl_disconn(chan%u)\n", 711 chan->ch_id); 712 return (EBUSY); 713 } 714 715 req = vmbus_msghc_dataptr(mh); 716 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_DISCONN; 717 req->chm_chanid = chan->ch_id; 718 req->chm_gpadl = gpadl; 719 720 error = vmbus_msghc_exec(sc, mh); 721 if (error) { 722 vmbus_msghc_put(sc, mh); 723 724 if (vmbus_chan_wait_revoke(chan, true)) { 725 /* 726 * Error is benign; this channel is revoked, 727 * so this GPADL will not be touched anymore. 728 */ 729 vmbus_chan_printf(chan, 730 "gpadl_disconn(revoked chan%u) msg hypercall " 731 "exec failed: %d\n", chan->ch_id, error); 732 return (0); 733 } 734 vmbus_chan_printf(chan, 735 "gpadl_disconn(chan%u) msg hypercall exec failed: %d\n", 736 chan->ch_id, error); 737 return (error); 738 } 739 740 vmbus_msghc_wait_result(sc, mh); 741 /* Discard result; no useful information */ 742 vmbus_msghc_put(sc, mh); 743 744 return (0); 745} 746 747static void 748vmbus_chan_detach(struct vmbus_channel *chan) 749{ 750 int refs; 751 752 KASSERT(chan->ch_refs > 0, ("chan%u: invalid refcnt %d", 753 chan->ch_id, chan->ch_refs)); 754 refs = atomic_fetchadd_int(&chan->ch_refs, -1); 755#ifdef INVARIANTS 756 if (VMBUS_CHAN_ISPRIMARY(chan)) { 757 KASSERT(refs == 1, ("chan%u: invalid refcnt %d for prichan", 758 chan->ch_id, refs + 1)); 759 } 760#endif 761 if (refs == 1) { 762 /* 763 * Detach the target channel. 764 */ 765 if (bootverbose) { 766 vmbus_chan_printf(chan, "chan%u detached\n", 767 chan->ch_id); 768 } 769 taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task); 770 } 771} 772 773static void 774vmbus_chan_clrchmap_task(void *xchan, int pending __unused) 775{ 776 struct vmbus_channel *chan = xchan; 777 778 chan->ch_vmbus->vmbus_chmap[chan->ch_id] = NULL; 779} 780 781static void 782vmbus_chan_clear_chmap(struct vmbus_channel *chan) 783{ 784 struct task chmap_task; 785 786 TASK_INIT(&chmap_task, 0, vmbus_chan_clrchmap_task, chan); 787 vmbus_chan_run_task(chan, &chmap_task); 788} 789 790static void 791vmbus_chan_set_chmap(struct vmbus_channel *chan) 792{ 793 __compiler_membar(); 794 chan->ch_vmbus->vmbus_chmap[chan->ch_id] = chan; 795} 796 797static void 798vmbus_chan_poll_cancel_task(void *xchan, int pending __unused) 799{ 800 801 vmbus_chan_poll_cancel_intq(xchan); 802} 803 804static void 805vmbus_chan_poll_cancel(struct vmbus_channel *chan) 806{ 807 struct task poll_cancel; 808 809 TASK_INIT(&poll_cancel, 0, vmbus_chan_poll_cancel_task, chan); 810 vmbus_chan_run_task(chan, &poll_cancel); 811} 812 813static int 814vmbus_chan_close_internal(struct vmbus_channel *chan) 815{ 816 struct vmbus_softc *sc = chan->ch_vmbus; 817 struct vmbus_msghc *mh; 818 struct vmbus_chanmsg_chclose *req; 819 uint32_t old_stflags; 820 int error; 821 822 /* 823 * NOTE: 824 * Sub-channels are closed upon their primary channel closing, 825 * so they can be closed even before they are opened. 826 */ 827 for (;;) { 828 old_stflags = chan->ch_stflags; 829 if (atomic_cmpset_int(&chan->ch_stflags, old_stflags, 830 old_stflags & ~VMBUS_CHAN_ST_OPENED)) 831 break; 832 } 833 if ((old_stflags & VMBUS_CHAN_ST_OPENED) == 0) { 834 /* Not opened yet; done */ 835 if (bootverbose) { 836 vmbus_chan_printf(chan, "chan%u not opened\n", 837 chan->ch_id); 838 } 839 return (0); 840 } 841 842 /* 843 * Free this channel's sysctl tree attached to its device's 844 * sysctl tree. 845 */ 846 sysctl_ctx_free(&chan->ch_sysctl_ctx); 847 848 /* 849 * Cancel polling, if it is enabled. 850 */ 851 vmbus_chan_poll_cancel(chan); 852 853 /* 854 * NOTE: 855 * Order is critical. This channel _must_ be uninstalled first, 856 * else the channel task may be enqueued by the IDT after it has 857 * been drained. 858 */ 859 vmbus_chan_clear_chmap(chan); 860 taskqueue_drain(chan->ch_tq, &chan->ch_task); 861 chan->ch_tq = NULL; 862 863 /* 864 * Close this channel. 865 */ 866 mh = vmbus_msghc_get(sc, sizeof(*req)); 867 if (mh == NULL) { 868 vmbus_chan_printf(chan, 869 "can not get msg hypercall for chclose(chan%u)\n", 870 chan->ch_id); 871 error = ENXIO; 872 goto disconnect; 873 } 874 875 req = vmbus_msghc_dataptr(mh); 876 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHCLOSE; 877 req->chm_chanid = chan->ch_id; 878 879 error = vmbus_msghc_exec_noresult(mh); 880 vmbus_msghc_put(sc, mh); 881 882 if (error) { 883 vmbus_chan_printf(chan, 884 "chclose(chan%u) msg hypercall exec failed: %d\n", 885 chan->ch_id, error); 886 goto disconnect; 887 } 888 889 if (bootverbose) 890 vmbus_chan_printf(chan, "chan%u closed\n", chan->ch_id); 891 892disconnect: 893 /* 894 * Disconnect the TX+RX bufrings from this channel. 895 */ 896 if (chan->ch_bufring_gpadl != 0) { 897 int error1; 898 899 error1 = vmbus_chan_gpadl_disconnect(chan, 900 chan->ch_bufring_gpadl); 901 if (error1) { 902 /* 903 * XXX 904 * The bufring GPADL is still connected; abandon 905 * this bufring, instead of having mysterious 906 * crash or trashed data later on. 907 */ 908 vmbus_chan_printf(chan, "chan%u bufring GPADL " 909 "is still connected after close\n", chan->ch_id); 910 chan->ch_bufring = NULL; 911 /* 912 * Give caller a hint that the bufring GPADL is 913 * still connected. 914 */ 915 error = EISCONN; 916 } 917 chan->ch_bufring_gpadl = 0; 918 } 919 920 /* 921 * Destroy the TX+RX bufrings. 922 */ 923 if (chan->ch_bufring != NULL) { 924 hyperv_dmamem_free(&chan->ch_bufring_dma, chan->ch_bufring); 925 chan->ch_bufring = NULL; 926 } 927 return (error); 928} 929 930int 931vmbus_chan_close_direct(struct vmbus_channel *chan) 932{ 933 int error; 934 935#ifdef INVARIANTS 936 if (VMBUS_CHAN_ISPRIMARY(chan)) { 937 struct vmbus_channel *subchan; 938 939 /* 940 * All sub-channels _must_ have been closed, or are _not_ 941 * opened at all. 942 */ 943 mtx_lock(&chan->ch_subchan_lock); 944 TAILQ_FOREACH(subchan, &chan->ch_subchans, ch_sublink) { 945 KASSERT( 946 (subchan->ch_stflags & VMBUS_CHAN_ST_OPENED) == 0, 947 ("chan%u: subchan%u is still opened", 948 chan->ch_id, subchan->ch_subidx)); 949 } 950 mtx_unlock(&chan->ch_subchan_lock); 951 } 952#endif 953 954 error = vmbus_chan_close_internal(chan); 955 if (!VMBUS_CHAN_ISPRIMARY(chan)) { 956 /* 957 * This sub-channel is referenced, when it is linked to 958 * the primary channel; drop that reference now. 959 */ 960 vmbus_chan_detach(chan); 961 } 962 return (error); 963} 964 965/* 966 * Caller should make sure that all sub-channels have 967 * been added to 'chan' and all to-be-closed channels 968 * are not being opened. 969 */ 970void 971vmbus_chan_close(struct vmbus_channel *chan) 972{ 973 int subchan_cnt; 974 975 if (!VMBUS_CHAN_ISPRIMARY(chan)) { 976 /* 977 * Sub-channel is closed when its primary channel 978 * is closed; done. 979 */ 980 return; 981 } 982 983 /* 984 * Close all sub-channels, if any. 985 */ 986 subchan_cnt = chan->ch_subchan_cnt; 987 if (subchan_cnt > 0) { 988 struct vmbus_channel **subchan; 989 int i; 990 991 subchan = vmbus_subchan_get(chan, subchan_cnt); 992 for (i = 0; i < subchan_cnt; ++i) { 993 vmbus_chan_close_internal(subchan[i]); 994 /* 995 * This sub-channel is referenced, when it is 996 * linked to the primary channel; drop that 997 * reference now. 998 */ 999 vmbus_chan_detach(subchan[i]); 1000 } 1001 vmbus_subchan_rel(subchan, subchan_cnt); 1002 } 1003 1004 /* Then close the primary channel. */ 1005 vmbus_chan_close_internal(chan); 1006} 1007 1008void 1009vmbus_chan_intr_drain(struct vmbus_channel *chan) 1010{ 1011 1012 taskqueue_drain(chan->ch_tq, &chan->ch_task); 1013} 1014 1015int 1016vmbus_chan_send(struct vmbus_channel *chan, uint16_t type, uint16_t flags, 1017 void *data, int dlen, uint64_t xactid) 1018{ 1019 struct vmbus_chanpkt pkt; 1020 int pktlen, pad_pktlen, hlen, error; 1021 uint64_t pad = 0; 1022 struct iovec iov[3]; 1023 boolean_t send_evt; 1024 1025 hlen = sizeof(pkt); 1026 pktlen = hlen + dlen; 1027 pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1028 KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1029 ("invalid packet size %d", pad_pktlen)); 1030 1031 pkt.cp_hdr.cph_type = type; 1032 pkt.cp_hdr.cph_flags = flags; 1033 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1034 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1035 pkt.cp_hdr.cph_xactid = xactid; 1036 1037 iov[0].iov_base = &pkt; 1038 iov[0].iov_len = hlen; 1039 iov[1].iov_base = data; 1040 iov[1].iov_len = dlen; 1041 iov[2].iov_base = &pad; 1042 iov[2].iov_len = pad_pktlen - pktlen; 1043 1044 error = vmbus_txbr_write(&chan->ch_txbr, iov, 3, &send_evt); 1045 if (!error && send_evt) 1046 vmbus_chan_signal_tx(chan); 1047 return error; 1048} 1049 1050int 1051vmbus_chan_send_sglist(struct vmbus_channel *chan, 1052 struct vmbus_gpa sg[], int sglen, void *data, int dlen, uint64_t xactid) 1053{ 1054 struct vmbus_chanpkt_sglist pkt; 1055 int pktlen, pad_pktlen, hlen, error; 1056 struct iovec iov[4]; 1057 boolean_t send_evt; 1058 uint64_t pad = 0; 1059 1060 hlen = __offsetof(struct vmbus_chanpkt_sglist, cp_gpa[sglen]); 1061 pktlen = hlen + dlen; 1062 pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1063 KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1064 ("invalid packet size %d", pad_pktlen)); 1065 1066 pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1067 pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1068 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1069 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1070 pkt.cp_hdr.cph_xactid = xactid; 1071 pkt.cp_rsvd = 0; 1072 pkt.cp_gpa_cnt = sglen; 1073 1074 iov[0].iov_base = &pkt; 1075 iov[0].iov_len = sizeof(pkt); 1076 iov[1].iov_base = sg; 1077 iov[1].iov_len = sizeof(struct vmbus_gpa) * sglen; 1078 iov[2].iov_base = data; 1079 iov[2].iov_len = dlen; 1080 iov[3].iov_base = &pad; 1081 iov[3].iov_len = pad_pktlen - pktlen; 1082 1083 error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); 1084 if (!error && send_evt) 1085 vmbus_chan_signal_tx(chan); 1086 return error; 1087} 1088 1089int 1090vmbus_chan_send_prplist(struct vmbus_channel *chan, 1091 struct vmbus_gpa_range *prp, int prp_cnt, void *data, int dlen, 1092 uint64_t xactid) 1093{ 1094 struct vmbus_chanpkt_prplist pkt; 1095 int pktlen, pad_pktlen, hlen, error; 1096 struct iovec iov[4]; 1097 boolean_t send_evt; 1098 uint64_t pad = 0; 1099 1100 hlen = __offsetof(struct vmbus_chanpkt_prplist, 1101 cp_range[0].gpa_page[prp_cnt]); 1102 pktlen = hlen + dlen; 1103 pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1104 KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1105 ("invalid packet size %d", pad_pktlen)); 1106 1107 pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1108 pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1109 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1110 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1111 pkt.cp_hdr.cph_xactid = xactid; 1112 pkt.cp_rsvd = 0; 1113 pkt.cp_range_cnt = 1; 1114 1115 iov[0].iov_base = &pkt; 1116 iov[0].iov_len = sizeof(pkt); 1117 iov[1].iov_base = prp; 1118 iov[1].iov_len = __offsetof(struct vmbus_gpa_range, gpa_page[prp_cnt]); 1119 iov[2].iov_base = data; 1120 iov[2].iov_len = dlen; 1121 iov[3].iov_base = &pad; 1122 iov[3].iov_len = pad_pktlen - pktlen; 1123 1124 error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); 1125 if (!error && send_evt) 1126 vmbus_chan_signal_tx(chan); 1127 return error; 1128} 1129 1130int 1131vmbus_chan_recv(struct vmbus_channel *chan, void *data, int *dlen0, 1132 uint64_t *xactid) 1133{ 1134 struct vmbus_chanpkt_hdr pkt; 1135 int error, dlen, hlen; 1136 1137 error = vmbus_rxbr_peek(&chan->ch_rxbr, &pkt, sizeof(pkt)); 1138 if (error) 1139 return (error); 1140 1141 if (__predict_false(pkt.cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) { 1142 vmbus_chan_printf(chan, "invalid hlen %u\n", pkt.cph_hlen); 1143 /* XXX this channel is dead actually. */ 1144 return (EIO); 1145 } 1146 if (__predict_false(pkt.cph_hlen > pkt.cph_tlen)) { 1147 vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n", 1148 pkt.cph_hlen, pkt.cph_tlen); 1149 /* XXX this channel is dead actually. */ 1150 return (EIO); 1151 } 1152 1153 hlen = VMBUS_CHANPKT_GETLEN(pkt.cph_hlen); 1154 dlen = VMBUS_CHANPKT_GETLEN(pkt.cph_tlen) - hlen; 1155 1156 if (*dlen0 < dlen) { 1157 /* Return the size of this packet's data. */ 1158 *dlen0 = dlen; 1159 return (ENOBUFS); 1160 } 1161 1162 *xactid = pkt.cph_xactid; 1163 *dlen0 = dlen; 1164 1165 /* Skip packet header */ 1166 error = vmbus_rxbr_read(&chan->ch_rxbr, data, dlen, hlen); 1167 KASSERT(!error, ("vmbus_rxbr_read failed")); 1168 1169 return (0); 1170} 1171 1172int 1173vmbus_chan_recv_pkt(struct vmbus_channel *chan, 1174 struct vmbus_chanpkt_hdr *pkt, int *pktlen0) 1175{ 1176 int error, pktlen, pkt_hlen; 1177 1178 pkt_hlen = sizeof(*pkt); 1179 error = vmbus_rxbr_peek(&chan->ch_rxbr, pkt, pkt_hlen); 1180 if (error) 1181 return (error); 1182 1183 if (__predict_false(pkt->cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) { 1184 vmbus_chan_printf(chan, "invalid hlen %u\n", pkt->cph_hlen); 1185 /* XXX this channel is dead actually. */ 1186 return (EIO); 1187 } 1188 if (__predict_false(pkt->cph_hlen > pkt->cph_tlen)) { 1189 vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n", 1190 pkt->cph_hlen, pkt->cph_tlen); 1191 /* XXX this channel is dead actually. */ 1192 return (EIO); 1193 } 1194 1195 pktlen = VMBUS_CHANPKT_GETLEN(pkt->cph_tlen); 1196 if (*pktlen0 < pktlen) { 1197 /* Return the size of this packet. */ 1198 *pktlen0 = pktlen; 1199 return (ENOBUFS); 1200 } 1201 *pktlen0 = pktlen; 1202 1203 /* 1204 * Skip the fixed-size packet header, which has been filled 1205 * by the above vmbus_rxbr_peek(). 1206 */ 1207 error = vmbus_rxbr_read(&chan->ch_rxbr, pkt + 1, 1208 pktlen - pkt_hlen, pkt_hlen); 1209 KASSERT(!error, ("vmbus_rxbr_read failed")); 1210 1211 return (0); 1212} 1213 1214static void 1215vmbus_chan_task(void *xchan, int pending __unused) 1216{ 1217 struct vmbus_channel *chan = xchan; 1218 vmbus_chan_callback_t cb = chan->ch_cb; 1219 void *cbarg = chan->ch_cbarg; 1220 1221 KASSERT(chan->ch_poll_intvl == 0, 1222 ("chan%u: interrupted in polling mode", chan->ch_id)); 1223 1224 /* 1225 * Optimize host to guest signaling by ensuring: 1226 * 1. While reading the channel, we disable interrupts from 1227 * host. 1228 * 2. Ensure that we process all posted messages from the host 1229 * before returning from this callback. 1230 * 3. Once we return, enable signaling from the host. Once this 1231 * state is set we check to see if additional packets are 1232 * available to read. In this case we repeat the process. 1233 * 1234 * NOTE: Interrupt has been disabled in the ISR. 1235 */ 1236 for (;;) { 1237 uint32_t left; 1238 1239 cb(chan, cbarg); 1240 1241 left = vmbus_rxbr_intr_unmask(&chan->ch_rxbr); 1242 if (left == 0) { 1243 /* No more data in RX bufring; done */ 1244 break; 1245 } 1246 vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1247 } 1248} 1249 1250static void 1251vmbus_chan_task_nobatch(void *xchan, int pending __unused) 1252{ 1253 struct vmbus_channel *chan = xchan; 1254 1255 KASSERT(chan->ch_poll_intvl == 0, 1256 ("chan%u: interrupted in polling mode", chan->ch_id)); 1257 chan->ch_cb(chan, chan->ch_cbarg); 1258} 1259 1260static void 1261vmbus_chan_poll_timeout(void *xchan) 1262{ 1263 struct vmbus_channel *chan = xchan; 1264 1265 KASSERT(chan->ch_poll_intvl != 0, 1266 ("chan%u: polling timeout in interrupt mode", chan->ch_id)); 1267 taskqueue_enqueue(chan->ch_tq, &chan->ch_poll_task); 1268} 1269 1270static void 1271vmbus_chan_poll_task(void *xchan, int pending __unused) 1272{ 1273 struct vmbus_channel *chan = xchan; 1274 1275 KASSERT(chan->ch_poll_intvl != 0, 1276 ("chan%u: polling in interrupt mode", chan->ch_id)); 1277 callout_reset_sbt_curcpu(&chan->ch_poll_timeo, chan->ch_poll_intvl, 0, 1278 vmbus_chan_poll_timeout, chan, chan->ch_poll_flags); 1279 chan->ch_cb(chan, chan->ch_cbarg); 1280} 1281 1282static void 1283vmbus_chan_pollcfg_task(void *xarg, int pending __unused) 1284{ 1285 const struct vmbus_chan_pollarg *arg = xarg; 1286 struct vmbus_channel *chan = arg->poll_chan; 1287 sbintime_t intvl; 1288 int poll_flags; 1289 1290 /* 1291 * Save polling interval. 1292 */ 1293 intvl = SBT_1S / arg->poll_hz; 1294 if (intvl == 0) 1295 intvl = 1; 1296 if (intvl == chan->ch_poll_intvl) { 1297 /* Nothing changes; done */ 1298 return; 1299 } 1300 chan->ch_poll_intvl = intvl; 1301 1302 /* Adjust callout flags. */ 1303 poll_flags = C_DIRECT_EXEC; 1304 if (arg->poll_hz <= hz) 1305 poll_flags |= C_HARDCLOCK; 1306 chan->ch_poll_flags = poll_flags; 1307 1308 /* 1309 * Disconnect this channel from the channel map to make sure that 1310 * the RX bufring interrupt enabling bit can not be touched, and 1311 * ISR can not enqueue this channel task anymore. THEN, disable 1312 * interrupt from the RX bufring (TX bufring does not generate 1313 * interrupt to VM). 1314 * 1315 * NOTE: order is critical. 1316 */ 1317 chan->ch_vmbus->vmbus_chmap[chan->ch_id] = NULL; 1318 __compiler_membar(); 1319 vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1320 1321 /* 1322 * NOTE: 1323 * At this point, this channel task will not be enqueued by 1324 * the ISR anymore, time to cancel the pending one. 1325 */ 1326 taskqueue_cancel(chan->ch_tq, &chan->ch_task, NULL); 1327 1328 /* Kick start! */ 1329 taskqueue_enqueue(chan->ch_tq, &chan->ch_poll_task); 1330} 1331 1332static bool 1333vmbus_chan_poll_cancel_intq(struct vmbus_channel *chan) 1334{ 1335 1336 if (chan->ch_poll_intvl == 0) { 1337 /* Not enabled. */ 1338 return (false); 1339 } 1340 1341 /* 1342 * Stop polling callout, so that channel polling task 1343 * will not be enqueued anymore. 1344 */ 1345 callout_drain(&chan->ch_poll_timeo); 1346 1347 /* 1348 * Disable polling by resetting polling interval. 1349 * 1350 * NOTE: 1351 * The polling interval resetting MUST be conducted 1352 * after the callout is drained; mainly to keep the 1353 * proper assertion in place. 1354 */ 1355 chan->ch_poll_intvl = 0; 1356 1357 /* 1358 * NOTE: 1359 * At this point, this channel polling task will not be 1360 * enqueued by the callout anymore, time to cancel the 1361 * pending one. 1362 */ 1363 taskqueue_cancel(chan->ch_tq, &chan->ch_poll_task, NULL); 1364 1365 /* Polling was enabled. */ 1366 return (true); 1367} 1368 1369static void 1370vmbus_chan_polldis_task(void *xchan, int pending __unused) 1371{ 1372 struct vmbus_channel *chan = xchan; 1373 1374 if (!vmbus_chan_poll_cancel_intq(chan)) { 1375 /* Already disabled; done. */ 1376 return; 1377 } 1378 1379 /* 1380 * Plug this channel back to the channel map and unmask 1381 * the RX bufring interrupt. 1382 */ 1383 chan->ch_vmbus->vmbus_chmap[chan->ch_id] = chan; 1384 __compiler_membar(); 1385 vmbus_rxbr_intr_unmask(&chan->ch_rxbr); 1386 1387 /* 1388 * Kick start the interrupt task, just in case unmasking 1389 * interrupt races ISR. 1390 */ 1391 taskqueue_enqueue(chan->ch_tq, &chan->ch_task); 1392} 1393 1394static __inline void 1395vmbus_event_flags_proc(struct vmbus_softc *sc, volatile u_long *event_flags, 1396 int flag_cnt) 1397{ 1398 int f; 1399 1400 for (f = 0; f < flag_cnt; ++f) { 1401 uint32_t chid_base; 1402 u_long flags; 1403 int chid_ofs; 1404 1405 if (event_flags[f] == 0) 1406 continue; 1407 1408 flags = atomic_swap_long(&event_flags[f], 0); 1409 chid_base = f << VMBUS_EVTFLAG_SHIFT; 1410 1411 while ((chid_ofs = ffsl(flags)) != 0) { 1412 struct vmbus_channel *chan; 1413 1414 --chid_ofs; /* NOTE: ffsl is 1-based */ 1415 flags &= ~(1UL << chid_ofs); 1416 1417 chan = sc->vmbus_chmap[chid_base + chid_ofs]; 1418 if (__predict_false(chan == NULL)) { 1419 /* Channel is closed. */ 1420 continue; 1421 } 1422 __compiler_membar(); 1423 1424 if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD) 1425 vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1426 taskqueue_enqueue(chan->ch_tq, &chan->ch_task); 1427 } 1428 } 1429} 1430 1431void 1432vmbus_event_proc(struct vmbus_softc *sc, int cpu) 1433{ 1434 struct vmbus_evtflags *eventf; 1435 1436 /* 1437 * On Host with Win8 or above, the event page can be checked directly 1438 * to get the id of the channel that has the pending interrupt. 1439 */ 1440 eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE; 1441 vmbus_event_flags_proc(sc, eventf->evt_flags, 1442 VMBUS_PCPU_GET(sc, event_flags_cnt, cpu)); 1443} 1444 1445void 1446vmbus_event_proc_compat(struct vmbus_softc *sc, int cpu) 1447{ 1448 struct vmbus_evtflags *eventf; 1449 1450 eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE; 1451 if (atomic_testandclear_long(&eventf->evt_flags[0], 0)) { 1452 vmbus_event_flags_proc(sc, sc->vmbus_rx_evtflags, 1453 VMBUS_CHAN_MAX_COMPAT >> VMBUS_EVTFLAG_SHIFT); 1454 } 1455} 1456 1457static void 1458vmbus_chan_update_evtflagcnt(struct vmbus_softc *sc, 1459 const struct vmbus_channel *chan) 1460{ 1461 volatile int *flag_cnt_ptr; 1462 int flag_cnt; 1463 1464 flag_cnt = (chan->ch_id / VMBUS_EVTFLAG_LEN) + 1; 1465 flag_cnt_ptr = VMBUS_PCPU_PTR(sc, event_flags_cnt, chan->ch_cpuid); 1466 1467 for (;;) { 1468 int old_flag_cnt; 1469 1470 old_flag_cnt = *flag_cnt_ptr; 1471 if (old_flag_cnt >= flag_cnt) 1472 break; 1473 if (atomic_cmpset_int(flag_cnt_ptr, old_flag_cnt, flag_cnt)) { 1474 if (bootverbose) { 1475 vmbus_chan_printf(chan, 1476 "chan%u update cpu%d flag_cnt to %d\n", 1477 chan->ch_id, chan->ch_cpuid, flag_cnt); 1478 } 1479 break; 1480 } 1481 } 1482} 1483 1484static struct vmbus_channel * 1485vmbus_chan_alloc(struct vmbus_softc *sc) 1486{ 1487 struct vmbus_channel *chan; 1488 1489 chan = malloc(sizeof(*chan), M_DEVBUF, M_WAITOK | M_ZERO); 1490 1491 chan->ch_monprm = hyperv_dmamem_alloc(bus_get_dma_tag(sc->vmbus_dev), 1492 HYPERCALL_PARAM_ALIGN, 0, sizeof(struct hyperv_mon_param), 1493 &chan->ch_monprm_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO); 1494 if (chan->ch_monprm == NULL) { 1495 device_printf(sc->vmbus_dev, "monprm alloc failed\n"); 1496 free(chan, M_DEVBUF); 1497 return NULL; 1498 } 1499 1500 chan->ch_refs = 1; 1501 chan->ch_vmbus = sc; 1502 mtx_init(&chan->ch_subchan_lock, "vmbus subchan", NULL, MTX_DEF); 1503 sx_init(&chan->ch_orphan_lock, "vmbus chorphan"); 1504 TAILQ_INIT(&chan->ch_subchans); 1505 vmbus_rxbr_init(&chan->ch_rxbr); 1506 vmbus_txbr_init(&chan->ch_txbr); 1507 1508 TASK_INIT(&chan->ch_poll_task, 0, vmbus_chan_poll_task, chan); 1509 callout_init(&chan->ch_poll_timeo, 1); 1510 1511 return chan; 1512} 1513 1514static void 1515vmbus_chan_free(struct vmbus_channel *chan) 1516{ 1517 1518 KASSERT(TAILQ_EMPTY(&chan->ch_subchans) && chan->ch_subchan_cnt == 0, 1519 ("still owns sub-channels")); 1520 KASSERT((chan->ch_stflags & 1521 (VMBUS_CHAN_ST_OPENED | 1522 VMBUS_CHAN_ST_ONPRIL | 1523 VMBUS_CHAN_ST_ONSUBL | 1524 VMBUS_CHAN_ST_ONLIST)) == 0, ("free busy channel")); 1525 KASSERT(chan->ch_orphan_xact == NULL, 1526 ("still has orphan xact installed")); 1527 KASSERT(chan->ch_refs == 0, ("chan%u: invalid refcnt %d", 1528 chan->ch_id, chan->ch_refs)); 1529 KASSERT(chan->ch_poll_intvl == 0, ("chan%u: polling is activated", 1530 chan->ch_id)); 1531 1532 hyperv_dmamem_free(&chan->ch_monprm_dma, chan->ch_monprm); 1533 mtx_destroy(&chan->ch_subchan_lock); 1534 sx_destroy(&chan->ch_orphan_lock); 1535 vmbus_rxbr_deinit(&chan->ch_rxbr); 1536 vmbus_txbr_deinit(&chan->ch_txbr); 1537 free(chan, M_DEVBUF); 1538} 1539 1540static int 1541vmbus_chan_add(struct vmbus_channel *newchan) 1542{ 1543 struct vmbus_softc *sc = newchan->ch_vmbus; 1544 struct vmbus_channel *prichan; 1545 1546 if (newchan->ch_id == 0) { 1547 /* 1548 * XXX 1549 * Chan0 will neither be processed nor should be offered; 1550 * skip it. 1551 */ 1552 device_printf(sc->vmbus_dev, "got chan0 offer, discard\n"); 1553 return EINVAL; 1554 } else if (newchan->ch_id >= VMBUS_CHAN_MAX) { 1555 device_printf(sc->vmbus_dev, "invalid chan%u offer\n", 1556 newchan->ch_id); 1557 return EINVAL; 1558 } 1559 1560 mtx_lock(&sc->vmbus_prichan_lock); 1561 TAILQ_FOREACH(prichan, &sc->vmbus_prichans, ch_prilink) { 1562 /* 1563 * Sub-channel will have the same type GUID and instance 1564 * GUID as its primary channel. 1565 */ 1566 if (memcmp(&prichan->ch_guid_type, &newchan->ch_guid_type, 1567 sizeof(struct hyperv_guid)) == 0 && 1568 memcmp(&prichan->ch_guid_inst, &newchan->ch_guid_inst, 1569 sizeof(struct hyperv_guid)) == 0) 1570 break; 1571 } 1572 if (VMBUS_CHAN_ISPRIMARY(newchan)) { 1573 if (prichan == NULL) { 1574 /* Install the new primary channel */ 1575 vmbus_chan_ins_prilist(sc, newchan); 1576 mtx_unlock(&sc->vmbus_prichan_lock); 1577 goto done; 1578 } else { 1579 mtx_unlock(&sc->vmbus_prichan_lock); 1580 device_printf(sc->vmbus_dev, 1581 "duplicated primary chan%u\n", newchan->ch_id); 1582 return EINVAL; 1583 } 1584 } else { /* Sub-channel */ 1585 if (prichan == NULL) { 1586 mtx_unlock(&sc->vmbus_prichan_lock); 1587 device_printf(sc->vmbus_dev, 1588 "no primary chan for chan%u\n", newchan->ch_id); 1589 return EINVAL; 1590 } 1591 /* 1592 * Found the primary channel for this sub-channel and 1593 * move on. 1594 * 1595 * XXX refcnt prichan 1596 */ 1597 } 1598 mtx_unlock(&sc->vmbus_prichan_lock); 1599 1600 /* 1601 * This is a sub-channel; link it with the primary channel. 1602 */ 1603 KASSERT(!VMBUS_CHAN_ISPRIMARY(newchan), 1604 ("new channel is not sub-channel")); 1605 KASSERT(prichan != NULL, ("no primary channel")); 1606 1607 /* 1608 * Reference count this sub-channel; it will be dereferenced 1609 * when this sub-channel is closed. 1610 */ 1611 KASSERT(newchan->ch_refs == 1, ("chan%u: invalid refcnt %d", 1612 newchan->ch_id, newchan->ch_refs)); 1613 atomic_add_int(&newchan->ch_refs, 1); 1614 1615 newchan->ch_prichan = prichan; 1616 newchan->ch_dev = prichan->ch_dev; 1617 1618 mtx_lock(&prichan->ch_subchan_lock); 1619 vmbus_chan_ins_sublist(prichan, newchan); 1620 mtx_unlock(&prichan->ch_subchan_lock); 1621 /* 1622 * Notify anyone that is interested in this sub-channel, 1623 * after this sub-channel is setup. 1624 */ 1625 wakeup(prichan); 1626done: 1627 /* 1628 * Hook this channel up for later revocation. 1629 */ 1630 mtx_lock(&sc->vmbus_chan_lock); 1631 vmbus_chan_ins_list(sc, newchan); 1632 mtx_unlock(&sc->vmbus_chan_lock); 1633 1634 if (bootverbose) { 1635 vmbus_chan_printf(newchan, "chan%u subidx%u offer\n", 1636 newchan->ch_id, newchan->ch_subidx); 1637 } 1638 1639 /* Select default cpu for this channel. */ 1640 vmbus_chan_cpu_default(newchan); 1641 1642 return 0; 1643} 1644 1645void 1646vmbus_chan_cpu_set(struct vmbus_channel *chan, int cpu) 1647{ 1648 KASSERT(cpu >= 0 && cpu < mp_ncpus, ("invalid cpu %d", cpu)); 1649 1650 if (chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WS2008 || 1651 chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WIN7) { 1652 /* Only cpu0 is supported */ 1653 cpu = 0; 1654 } 1655 1656 chan->ch_cpuid = cpu; 1657 chan->ch_vcpuid = VMBUS_PCPU_GET(chan->ch_vmbus, vcpuid, cpu); 1658 1659 if (bootverbose) { 1660 vmbus_chan_printf(chan, 1661 "chan%u assigned to cpu%u [vcpu%u]\n", 1662 chan->ch_id, chan->ch_cpuid, chan->ch_vcpuid); 1663 } 1664} 1665 1666void 1667vmbus_chan_cpu_rr(struct vmbus_channel *chan) 1668{ 1669 static uint32_t vmbus_chan_nextcpu; 1670 int cpu; 1671 1672 cpu = atomic_fetchadd_int(&vmbus_chan_nextcpu, 1) % mp_ncpus; 1673 vmbus_chan_cpu_set(chan, cpu); 1674} 1675 1676static void 1677vmbus_chan_cpu_default(struct vmbus_channel *chan) 1678{ 1679 /* 1680 * By default, pin the channel to cpu0. Devices having 1681 * special channel-cpu mapping requirement should call 1682 * vmbus_chan_cpu_{set,rr}(). 1683 */ 1684 vmbus_chan_cpu_set(chan, 0); 1685} 1686 1687static void 1688vmbus_chan_msgproc_choffer(struct vmbus_softc *sc, 1689 const struct vmbus_message *msg) 1690{ 1691 const struct vmbus_chanmsg_choffer *offer; 1692 struct vmbus_channel *chan; 1693 task_fn_t *detach_fn, *attach_fn; 1694 int error; 1695 1696 offer = (const struct vmbus_chanmsg_choffer *)msg->msg_data; 1697 1698 chan = vmbus_chan_alloc(sc); 1699 if (chan == NULL) { 1700 device_printf(sc->vmbus_dev, "allocate chan%u failed\n", 1701 offer->chm_chanid); 1702 return; 1703 } 1704 1705 chan->ch_id = offer->chm_chanid; 1706 chan->ch_subidx = offer->chm_subidx; 1707 chan->ch_guid_type = offer->chm_chtype; 1708 chan->ch_guid_inst = offer->chm_chinst; 1709 1710 /* Batch reading is on by default */ 1711 chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD; 1712 1713 chan->ch_monprm->mp_connid = VMBUS_CONNID_EVENT; 1714 if (sc->vmbus_version != VMBUS_VERSION_WS2008) 1715 chan->ch_monprm->mp_connid = offer->chm_connid; 1716 1717 if (offer->chm_flags1 & VMBUS_CHOFFER_FLAG1_HASMNF) { 1718 int trig_idx; 1719 1720 /* 1721 * Setup MNF stuffs. 1722 */ 1723 chan->ch_txflags |= VMBUS_CHAN_TXF_HASMNF; 1724 1725 trig_idx = offer->chm_montrig / VMBUS_MONTRIG_LEN; 1726 if (trig_idx >= VMBUS_MONTRIGS_MAX) 1727 panic("invalid monitor trigger %u", offer->chm_montrig); 1728 chan->ch_montrig = 1729 &sc->vmbus_mnf2->mnf_trigs[trig_idx].mt_pending; 1730 1731 chan->ch_montrig_mask = 1732 1 << (offer->chm_montrig % VMBUS_MONTRIG_LEN); 1733 } 1734 1735 /* 1736 * Setup event flag. 1737 */ 1738 chan->ch_evtflag = 1739 &sc->vmbus_tx_evtflags[chan->ch_id >> VMBUS_EVTFLAG_SHIFT]; 1740 chan->ch_evtflag_mask = 1UL << (chan->ch_id & VMBUS_EVTFLAG_MASK); 1741 1742 /* 1743 * Setup attach and detach tasks. 1744 */ 1745 if (VMBUS_CHAN_ISPRIMARY(chan)) { 1746 chan->ch_mgmt_tq = sc->vmbus_devtq; 1747 attach_fn = vmbus_prichan_attach_task; 1748 detach_fn = vmbus_prichan_detach_task; 1749 } else { 1750 chan->ch_mgmt_tq = sc->vmbus_subchtq; 1751 attach_fn = vmbus_subchan_attach_task; 1752 detach_fn = vmbus_subchan_detach_task; 1753 } 1754 TASK_INIT(&chan->ch_attach_task, 0, attach_fn, chan); 1755 TASK_INIT(&chan->ch_detach_task, 0, detach_fn, chan); 1756 1757 error = vmbus_chan_add(chan); 1758 if (error) { 1759 device_printf(sc->vmbus_dev, "add chan%u failed: %d\n", 1760 chan->ch_id, error); 1761 atomic_subtract_int(&chan->ch_refs, 1); 1762 vmbus_chan_free(chan); 1763 return; 1764 } 1765 taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_attach_task); 1766} 1767 1768static void 1769vmbus_chan_msgproc_chrescind(struct vmbus_softc *sc, 1770 const struct vmbus_message *msg) 1771{ 1772 const struct vmbus_chanmsg_chrescind *note; 1773 struct vmbus_channel *chan; 1774 1775 note = (const struct vmbus_chanmsg_chrescind *)msg->msg_data; 1776 if (note->chm_chanid > VMBUS_CHAN_MAX) { 1777 device_printf(sc->vmbus_dev, "invalid revoked chan%u\n", 1778 note->chm_chanid); 1779 return; 1780 } 1781 1782 /* 1783 * Find and remove the target channel from the channel list. 1784 */ 1785 mtx_lock(&sc->vmbus_chan_lock); 1786 TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) { 1787 if (chan->ch_id == note->chm_chanid) 1788 break; 1789 } 1790 if (chan == NULL) { 1791 mtx_unlock(&sc->vmbus_chan_lock); 1792 device_printf(sc->vmbus_dev, "chan%u is not offered\n", 1793 note->chm_chanid); 1794 return; 1795 } 1796 vmbus_chan_rem_list(sc, chan); 1797 mtx_unlock(&sc->vmbus_chan_lock); 1798 1799 if (VMBUS_CHAN_ISPRIMARY(chan)) { 1800 /* 1801 * The target channel is a primary channel; remove the 1802 * target channel from the primary channel list now, 1803 * instead of later, so that it will not be found by 1804 * other sub-channel offers, which are processed in 1805 * this thread. 1806 */ 1807 mtx_lock(&sc->vmbus_prichan_lock); 1808 vmbus_chan_rem_prilist(sc, chan); 1809 mtx_unlock(&sc->vmbus_prichan_lock); 1810 } 1811 1812 /* 1813 * NOTE: 1814 * The following processing order is critical: 1815 * Set the REVOKED state flag before orphaning the installed xact. 1816 */ 1817 1818 if (atomic_testandset_int(&chan->ch_stflags, 1819 VMBUS_CHAN_ST_REVOKED_SHIFT)) 1820 panic("channel has already been revoked"); 1821 1822 sx_xlock(&chan->ch_orphan_lock); 1823 if (chan->ch_orphan_xact != NULL) 1824 vmbus_xact_ctx_orphan(chan->ch_orphan_xact); 1825 sx_xunlock(&chan->ch_orphan_lock); 1826 1827 if (bootverbose) 1828 vmbus_chan_printf(chan, "chan%u revoked\n", note->chm_chanid); 1829 vmbus_chan_detach(chan); 1830} 1831 1832static int 1833vmbus_chan_release(struct vmbus_channel *chan) 1834{ 1835 struct vmbus_softc *sc = chan->ch_vmbus; 1836 struct vmbus_chanmsg_chfree *req; 1837 struct vmbus_msghc *mh; 1838 int error; 1839 1840 mh = vmbus_msghc_get(sc, sizeof(*req)); 1841 if (mh == NULL) { 1842 vmbus_chan_printf(chan, 1843 "can not get msg hypercall for chfree(chan%u)\n", 1844 chan->ch_id); 1845 return (ENXIO); 1846 } 1847 1848 req = vmbus_msghc_dataptr(mh); 1849 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHFREE; 1850 req->chm_chanid = chan->ch_id; 1851 1852 error = vmbus_msghc_exec_noresult(mh); 1853 vmbus_msghc_put(sc, mh); 1854 1855 if (error) { 1856 vmbus_chan_printf(chan, 1857 "chfree(chan%u) msg hypercall exec failed: %d\n", 1858 chan->ch_id, error); 1859 } else { 1860 if (bootverbose) 1861 vmbus_chan_printf(chan, "chan%u freed\n", chan->ch_id); 1862 } 1863 return (error); 1864} 1865 1866static void 1867vmbus_prichan_detach_task(void *xchan, int pending __unused) 1868{ 1869 struct vmbus_channel *chan = xchan; 1870 1871 KASSERT(VMBUS_CHAN_ISPRIMARY(chan), 1872 ("chan%u is not primary channel", chan->ch_id)); 1873 1874 /* Delete and detach the device associated with this channel. */ 1875 vmbus_delete_child(chan); 1876 1877 /* Release this channel (back to vmbus). */ 1878 vmbus_chan_release(chan); 1879 1880 /* Free this channel's resource. */ 1881 vmbus_chan_free(chan); 1882} 1883 1884static void 1885vmbus_subchan_detach_task(void *xchan, int pending __unused) 1886{ 1887 struct vmbus_channel *chan = xchan; 1888 struct vmbus_channel *pri_chan = chan->ch_prichan; 1889 1890 KASSERT(!VMBUS_CHAN_ISPRIMARY(chan), 1891 ("chan%u is primary channel", chan->ch_id)); 1892 1893 /* Release this channel (back to vmbus). */ 1894 vmbus_chan_release(chan); 1895 1896 /* Unlink from its primary channel's sub-channel list. */ 1897 mtx_lock(&pri_chan->ch_subchan_lock); 1898 vmbus_chan_rem_sublist(pri_chan, chan); 1899 mtx_unlock(&pri_chan->ch_subchan_lock); 1900 /* Notify anyone that is waiting for this sub-channel to vanish. */ 1901 wakeup(pri_chan); 1902 1903 /* Free this channel's resource. */ 1904 vmbus_chan_free(chan); 1905} 1906 1907static void 1908vmbus_prichan_attach_task(void *xchan, int pending __unused) 1909{ 1910 1911 /* 1912 * Add device for this primary channel. 1913 */ 1914 vmbus_add_child(xchan); 1915} 1916 1917static void 1918vmbus_subchan_attach_task(void *xchan __unused, int pending __unused) 1919{ 1920 1921 /* Nothing */ 1922} 1923 1924void 1925vmbus_chan_destroy_all(struct vmbus_softc *sc) 1926{ 1927 1928 /* 1929 * Detach all devices and destroy the corresponding primary 1930 * channels. 1931 */ 1932 for (;;) { 1933 struct vmbus_channel *chan; 1934 1935 mtx_lock(&sc->vmbus_chan_lock); 1936 TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) { 1937 if (VMBUS_CHAN_ISPRIMARY(chan)) 1938 break; 1939 } 1940 if (chan == NULL) { 1941 /* No more primary channels; done. */ 1942 mtx_unlock(&sc->vmbus_chan_lock); 1943 break; 1944 } 1945 vmbus_chan_rem_list(sc, chan); 1946 mtx_unlock(&sc->vmbus_chan_lock); 1947 1948 mtx_lock(&sc->vmbus_prichan_lock); 1949 vmbus_chan_rem_prilist(sc, chan); 1950 mtx_unlock(&sc->vmbus_prichan_lock); 1951 1952 taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task); 1953 } 1954} 1955 1956struct vmbus_channel ** 1957vmbus_subchan_get(struct vmbus_channel *pri_chan, int subchan_cnt) 1958{ 1959 struct vmbus_channel **ret, *chan; 1960 int i; 1961 1962 KASSERT(subchan_cnt > 0, ("invalid sub-channel count %d", subchan_cnt)); 1963 1964 ret = malloc(subchan_cnt * sizeof(struct vmbus_channel *), M_TEMP, 1965 M_WAITOK); 1966 1967 mtx_lock(&pri_chan->ch_subchan_lock); 1968 1969 while (pri_chan->ch_subchan_cnt < subchan_cnt) 1970 mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "subch", 0); 1971 1972 i = 0; 1973 TAILQ_FOREACH(chan, &pri_chan->ch_subchans, ch_sublink) { 1974 /* TODO: refcnt chan */ 1975 ret[i] = chan; 1976 1977 ++i; 1978 if (i == subchan_cnt) 1979 break; 1980 } 1981 KASSERT(i == subchan_cnt, ("invalid subchan count %d, should be %d", 1982 pri_chan->ch_subchan_cnt, subchan_cnt)); 1983 1984 mtx_unlock(&pri_chan->ch_subchan_lock); 1985 1986 return ret; 1987} 1988 1989void 1990vmbus_subchan_rel(struct vmbus_channel **subchan, int subchan_cnt __unused) 1991{ 1992 1993 free(subchan, M_TEMP); 1994} 1995 1996void 1997vmbus_subchan_drain(struct vmbus_channel *pri_chan) 1998{ 1999 mtx_lock(&pri_chan->ch_subchan_lock); 2000 while (pri_chan->ch_subchan_cnt > 0) 2001 mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "dsubch", 0); 2002 mtx_unlock(&pri_chan->ch_subchan_lock); 2003} 2004 2005void 2006vmbus_chan_msgproc(struct vmbus_softc *sc, const struct vmbus_message *msg) 2007{ 2008 vmbus_chanmsg_proc_t msg_proc; 2009 uint32_t msg_type; 2010 2011 msg_type = ((const struct vmbus_chanmsg_hdr *)msg->msg_data)->chm_type; 2012 KASSERT(msg_type < VMBUS_CHANMSG_TYPE_MAX, 2013 ("invalid message type %u", msg_type)); 2014 2015 msg_proc = vmbus_chan_msgprocs[msg_type]; 2016 if (msg_proc != NULL) 2017 msg_proc(sc, msg); 2018} 2019 2020void 2021vmbus_chan_set_readbatch(struct vmbus_channel *chan, bool on) 2022{ 2023 if (!on) 2024 chan->ch_flags &= ~VMBUS_CHAN_FLAG_BATCHREAD; 2025 else 2026 chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD; 2027} 2028 2029uint32_t 2030vmbus_chan_id(const struct vmbus_channel *chan) 2031{ 2032 return chan->ch_id; 2033} 2034 2035uint32_t 2036vmbus_chan_subidx(const struct vmbus_channel *chan) 2037{ 2038 return chan->ch_subidx; 2039} 2040 2041bool 2042vmbus_chan_is_primary(const struct vmbus_channel *chan) 2043{ 2044 if (VMBUS_CHAN_ISPRIMARY(chan)) 2045 return true; 2046 else 2047 return false; 2048} 2049 2050const struct hyperv_guid * 2051vmbus_chan_guid_inst(const struct vmbus_channel *chan) 2052{ 2053 return &chan->ch_guid_inst; 2054} 2055 2056int 2057vmbus_chan_prplist_nelem(int br_size, int prpcnt_max, int dlen_max) 2058{ 2059 int elem_size; 2060 2061 elem_size = __offsetof(struct vmbus_chanpkt_prplist, 2062 cp_range[0].gpa_page[prpcnt_max]); 2063 elem_size += dlen_max; 2064 elem_size = VMBUS_CHANPKT_TOTLEN(elem_size); 2065 2066 return (vmbus_br_nelem(br_size, elem_size)); 2067} 2068 2069bool 2070vmbus_chan_tx_empty(const struct vmbus_channel *chan) 2071{ 2072 2073 return (vmbus_txbr_empty(&chan->ch_txbr)); 2074} 2075 2076bool 2077vmbus_chan_rx_empty(const struct vmbus_channel *chan) 2078{ 2079 2080 return (vmbus_rxbr_empty(&chan->ch_rxbr)); 2081} 2082 2083static int 2084vmbus_chan_printf(const struct vmbus_channel *chan, const char *fmt, ...) 2085{ 2086 va_list ap; 2087 device_t dev; 2088 int retval; 2089 2090 if (chan->ch_dev == NULL || !device_is_alive(chan->ch_dev)) 2091 dev = chan->ch_vmbus->vmbus_dev; 2092 else 2093 dev = chan->ch_dev; 2094 2095 retval = device_print_prettyname(dev); 2096 va_start(ap, fmt); 2097 retval += vprintf(fmt, ap); 2098 va_end(ap); 2099 2100 return (retval); 2101} 2102 2103void 2104vmbus_chan_run_task(struct vmbus_channel *chan, struct task *task) 2105{ 2106 2107 taskqueue_enqueue(chan->ch_tq, task); 2108 taskqueue_drain(chan->ch_tq, task); 2109} 2110 2111struct taskqueue * 2112vmbus_chan_mgmt_tq(const struct vmbus_channel *chan) 2113{ 2114 2115 return (chan->ch_mgmt_tq); 2116} 2117 2118bool 2119vmbus_chan_is_revoked(const struct vmbus_channel *chan) 2120{ 2121 2122 if (chan->ch_stflags & VMBUS_CHAN_ST_REVOKED) 2123 return (true); 2124 return (false); 2125} 2126 2127void 2128vmbus_chan_set_orphan(struct vmbus_channel *chan, struct vmbus_xact_ctx *xact) 2129{ 2130 2131 sx_xlock(&chan->ch_orphan_lock); 2132 chan->ch_orphan_xact = xact; 2133 sx_xunlock(&chan->ch_orphan_lock); 2134} 2135 2136void 2137vmbus_chan_unset_orphan(struct vmbus_channel *chan) 2138{ 2139 2140 sx_xlock(&chan->ch_orphan_lock); 2141 chan->ch_orphan_xact = NULL; 2142 sx_xunlock(&chan->ch_orphan_lock); 2143} 2144 2145const void * 2146vmbus_chan_xact_wait(const struct vmbus_channel *chan, 2147 struct vmbus_xact *xact, size_t *resp_len, bool can_sleep) 2148{ 2149 const void *ret; 2150 2151 if (can_sleep) 2152 ret = vmbus_xact_wait(xact, resp_len); 2153 else 2154 ret = vmbus_xact_busywait(xact, resp_len); 2155 if (vmbus_chan_is_revoked(chan)) { 2156 /* 2157 * This xact probably is interrupted, and the 2158 * interruption can race the reply reception, 2159 * so we have to make sure that there are nothing 2160 * left on the RX bufring, i.e. this xact will 2161 * not be touched, once this function returns. 2162 * 2163 * Since the hypervisor will not put more data 2164 * onto the RX bufring once the channel is revoked, 2165 * the following loop will be terminated, once all 2166 * data are drained by the driver's channel 2167 * callback. 2168 */ 2169 while (!vmbus_chan_rx_empty(chan)) { 2170 if (can_sleep) 2171 pause("chxact", 1); 2172 else 2173 DELAY(1000); 2174 } 2175 } 2176 return (ret); 2177} 2178 2179void 2180vmbus_chan_poll_enable(struct vmbus_channel *chan, u_int pollhz) 2181{ 2182 struct vmbus_chan_pollarg arg; 2183 struct task poll_cfg; 2184 2185 KASSERT(chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD, 2186 ("enable polling on non-batch chan%u", chan->ch_id)); 2187 KASSERT(pollhz >= VMBUS_CHAN_POLLHZ_MIN && 2188 pollhz <= VMBUS_CHAN_POLLHZ_MAX, ("invalid pollhz %u", pollhz)); 2189 2190 arg.poll_chan = chan; 2191 arg.poll_hz = pollhz; 2192 TASK_INIT(&poll_cfg, 0, vmbus_chan_pollcfg_task, &arg); 2193 vmbus_chan_run_task(chan, &poll_cfg); 2194} 2195 2196void 2197vmbus_chan_poll_disable(struct vmbus_channel *chan) 2198{ 2199 struct task poll_dis; 2200 2201 KASSERT(chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD, 2202 ("disable polling on non-batch chan%u", chan->ch_id)); 2203 2204 TASK_INIT(&poll_dis, 0, vmbus_chan_polldis_task, chan); 2205 vmbus_chan_run_task(chan, &poll_dis); 2206} 2207