vmbus_chan.c revision 311389
1251876Speter/*- 2251876Speter * Copyright (c) 2009-2012,2016 Microsoft Corp. 3251876Speter * Copyright (c) 2012 NetApp Inc. 4251876Speter * Copyright (c) 2012 Citrix Inc. 5251876Speter * All rights reserved. 6251876Speter * 7251876Speter * Redistribution and use in source and binary forms, with or without 8251876Speter * modification, are permitted provided that the following conditions 9251876Speter * are met: 10251876Speter * 1. Redistributions of source code must retain the above copyright 11251876Speter * notice unmodified, this list of conditions, and the following 12251876Speter * disclaimer. 13251876Speter * 2. Redistributions in binary form must reproduce the above copyright 14251876Speter * notice, this list of conditions and the following disclaimer in the 15251876Speter * documentation and/or other materials provided with the distribution. 16251876Speter * 17251876Speter * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18251876Speter * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19251876Speter * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20251876Speter * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21251876Speter * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22251876Speter * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23251876Speter * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24251876Speter * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25251876Speter * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26251876Speter * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27251876Speter */ 28251876Speter 29251876Speter#include <sys/cdefs.h> 30251876Speter__FBSDID("$FreeBSD: stable/11/sys/dev/hyperv/vmbus/vmbus_chan.c 311389 2017-01-05 08:42:58Z sephe $"); 31251876Speter 32251876Speter#include <sys/param.h> 33251876Speter#include <sys/bus.h> 34251876Speter#include <sys/callout.h> 35251876Speter#include <sys/kernel.h> 36251876Speter#include <sys/lock.h> 37251876Speter#include <sys/malloc.h> 38251876Speter#include <sys/mutex.h> 39251876Speter#include <sys/smp.h> 40251876Speter#include <sys/sysctl.h> 41251876Speter#include <sys/systm.h> 42251876Speter 43251876Speter#include <machine/atomic.h> 44251876Speter#include <machine/stdarg.h> 45251876Speter 46251876Speter#include <dev/hyperv/include/hyperv_busdma.h> 47251876Speter#include <dev/hyperv/include/vmbus_xact.h> 48251876Speter#include <dev/hyperv/vmbus/hyperv_var.h> 49251876Speter#include <dev/hyperv/vmbus/vmbus_reg.h> 50251876Speter#include <dev/hyperv/vmbus/vmbus_var.h> 51251876Speter#include <dev/hyperv/vmbus/vmbus_brvar.h> 52251876Speter#include <dev/hyperv/vmbus/vmbus_chanvar.h> 53251876Speter 54251876Speterstruct vmbus_chan_pollarg { 55251876Speter struct vmbus_channel *poll_chan; 56251876Speter u_int poll_hz; 57251876Speter}; 58251876Speter 59251876Speterstatic void vmbus_chan_update_evtflagcnt( 60251876Speter struct vmbus_softc *, 61251876Speter const struct vmbus_channel *); 62251876Speterstatic int vmbus_chan_close_internal( 63251876Speter struct vmbus_channel *); 64251876Speterstatic int vmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS); 65251876Speterstatic void vmbus_chan_sysctl_create( 66251876Speter struct vmbus_channel *); 67251876Speterstatic struct vmbus_channel *vmbus_chan_alloc(struct vmbus_softc *); 68251876Speterstatic void vmbus_chan_free(struct vmbus_channel *); 69251876Speterstatic int vmbus_chan_add(struct vmbus_channel *); 70251876Speterstatic void vmbus_chan_cpu_default(struct vmbus_channel *); 71251876Speterstatic int vmbus_chan_release(struct vmbus_channel *); 72251876Speterstatic void vmbus_chan_set_chmap(struct vmbus_channel *); 73251876Speterstatic void vmbus_chan_clear_chmap(struct vmbus_channel *); 74251876Speterstatic void vmbus_chan_detach(struct vmbus_channel *); 75251876Speterstatic bool vmbus_chan_wait_revoke( 76251876Speter const struct vmbus_channel *, bool); 77251876Speterstatic void vmbus_chan_poll_timeout(void *); 78251876Speterstatic bool vmbus_chan_poll_cancel_intq( 79251876Speter struct vmbus_channel *); 80251876Speterstatic void vmbus_chan_poll_cancel(struct vmbus_channel *); 81251876Speter 82251876Speterstatic void vmbus_chan_ins_prilist(struct vmbus_softc *, 83251876Speter struct vmbus_channel *); 84251876Speterstatic void vmbus_chan_rem_prilist(struct vmbus_softc *, 85251876Speter struct vmbus_channel *); 86251876Speterstatic void vmbus_chan_ins_list(struct vmbus_softc *, 87251876Speter struct vmbus_channel *); 88251876Speterstatic void vmbus_chan_rem_list(struct vmbus_softc *, 89251876Speter struct vmbus_channel *); 90251876Speterstatic void vmbus_chan_ins_sublist(struct vmbus_channel *, 91251876Speter struct vmbus_channel *); 92251876Speterstatic void vmbus_chan_rem_sublist(struct vmbus_channel *, 93251876Speter struct vmbus_channel *); 94251876Speter 95251876Speterstatic void vmbus_chan_task(void *, int); 96251876Speterstatic void vmbus_chan_task_nobatch(void *, int); 97251876Speterstatic void vmbus_chan_poll_task(void *, int); 98251876Speterstatic void vmbus_chan_clrchmap_task(void *, int); 99251876Speterstatic void vmbus_chan_pollcfg_task(void *, int); 100251876Speterstatic void vmbus_chan_polldis_task(void *, int); 101251876Speterstatic void vmbus_chan_poll_cancel_task(void *, int); 102251876Speterstatic void vmbus_prichan_attach_task(void *, int); 103251876Speterstatic void vmbus_subchan_attach_task(void *, int); 104251876Speterstatic void vmbus_prichan_detach_task(void *, int); 105251876Speterstatic void vmbus_subchan_detach_task(void *, int); 106251876Speter 107251876Speterstatic void vmbus_chan_msgproc_choffer(struct vmbus_softc *, 108251876Speter const struct vmbus_message *); 109251876Speterstatic void vmbus_chan_msgproc_chrescind( 110251876Speter struct vmbus_softc *, 111251876Speter const struct vmbus_message *); 112251876Speter 113251876Speterstatic int vmbus_chan_printf(const struct vmbus_channel *, 114251876Speter const char *, ...) __printflike(2, 3); 115251876Speter 116251876Speter/* 117251876Speter * Vmbus channel message processing. 118251876Speter */ 119251876Speterstatic const vmbus_chanmsg_proc_t 120251876Spetervmbus_chan_msgprocs[VMBUS_CHANMSG_TYPE_MAX] = { 121251876Speter VMBUS_CHANMSG_PROC(CHOFFER, vmbus_chan_msgproc_choffer), 122251876Speter VMBUS_CHANMSG_PROC(CHRESCIND, vmbus_chan_msgproc_chrescind), 123251876Speter 124251876Speter VMBUS_CHANMSG_PROC_WAKEUP(CHOPEN_RESP), 125251876Speter VMBUS_CHANMSG_PROC_WAKEUP(GPADL_CONNRESP), 126251876Speter VMBUS_CHANMSG_PROC_WAKEUP(GPADL_DISCONNRESP) 127251876Speter}; 128251876Speter 129251876Speter/* 130251876Speter * Notify host that there are data pending on our TX bufring. 131251876Speter */ 132251876Speterstatic __inline void 133251876Spetervmbus_chan_signal_tx(const struct vmbus_channel *chan) 134251876Speter{ 135251876Speter atomic_set_long(chan->ch_evtflag, chan->ch_evtflag_mask); 136251876Speter if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF) 137251876Speter atomic_set_int(chan->ch_montrig, chan->ch_montrig_mask); 138251876Speter else 139251876Speter hypercall_signal_event(chan->ch_monprm_dma.hv_paddr); 140251876Speter} 141251876Speter 142251876Speterstatic void 143251876Spetervmbus_chan_ins_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan) 144251876Speter{ 145251876Speter 146251876Speter mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED); 147251876Speter if (atomic_testandset_int(&chan->ch_stflags, 148251876Speter VMBUS_CHAN_ST_ONPRIL_SHIFT)) 149251876Speter panic("channel is already on the prilist"); 150251876Speter TAILQ_INSERT_TAIL(&sc->vmbus_prichans, chan, ch_prilink); 151251876Speter} 152251876Speter 153251876Speterstatic void 154251876Spetervmbus_chan_rem_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan) 155251876Speter{ 156251876Speter 157251876Speter mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED); 158251876Speter if (atomic_testandclear_int(&chan->ch_stflags, 159251876Speter VMBUS_CHAN_ST_ONPRIL_SHIFT) == 0) 160251876Speter panic("channel is not on the prilist"); 161251876Speter TAILQ_REMOVE(&sc->vmbus_prichans, chan, ch_prilink); 162251876Speter} 163251876Speter 164251876Speterstatic void 165251876Spetervmbus_chan_ins_sublist(struct vmbus_channel *prichan, 166251876Speter struct vmbus_channel *chan) 167251876Speter{ 168251876Speter 169251876Speter mtx_assert(&prichan->ch_subchan_lock, MA_OWNED); 170251876Speter 171251876Speter if (atomic_testandset_int(&chan->ch_stflags, 172251876Speter VMBUS_CHAN_ST_ONSUBL_SHIFT)) 173251876Speter panic("channel is already on the sublist"); 174251876Speter TAILQ_INSERT_TAIL(&prichan->ch_subchans, chan, ch_sublink); 175251876Speter 176251876Speter /* Bump sub-channel count. */ 177251876Speter prichan->ch_subchan_cnt++; 178251876Speter} 179251876Speter 180251876Speterstatic void 181251876Spetervmbus_chan_rem_sublist(struct vmbus_channel *prichan, 182251876Speter struct vmbus_channel *chan) 183251876Speter{ 184251876Speter 185251876Speter mtx_assert(&prichan->ch_subchan_lock, MA_OWNED); 186251876Speter 187251876Speter KASSERT(prichan->ch_subchan_cnt > 0, 188251876Speter ("invalid subchan_cnt %d", prichan->ch_subchan_cnt)); 189251876Speter prichan->ch_subchan_cnt--; 190251876Speter 191251876Speter if (atomic_testandclear_int(&chan->ch_stflags, 192251876Speter VMBUS_CHAN_ST_ONSUBL_SHIFT) == 0) 193251876Speter panic("channel is not on the sublist"); 194251876Speter TAILQ_REMOVE(&prichan->ch_subchans, chan, ch_sublink); 195251876Speter} 196251876Speter 197251876Speterstatic void 198251876Spetervmbus_chan_ins_list(struct vmbus_softc *sc, struct vmbus_channel *chan) 199251876Speter{ 200251876Speter 201251876Speter mtx_assert(&sc->vmbus_chan_lock, MA_OWNED); 202251876Speter if (atomic_testandset_int(&chan->ch_stflags, 203251876Speter VMBUS_CHAN_ST_ONLIST_SHIFT)) 204251876Speter panic("channel is already on the list"); 205251876Speter TAILQ_INSERT_TAIL(&sc->vmbus_chans, chan, ch_link); 206251876Speter} 207251876Speter 208251876Speterstatic void 209251876Spetervmbus_chan_rem_list(struct vmbus_softc *sc, struct vmbus_channel *chan) 210251876Speter{ 211251876Speter 212251876Speter mtx_assert(&sc->vmbus_chan_lock, MA_OWNED); 213251876Speter if (atomic_testandclear_int(&chan->ch_stflags, 214251876Speter VMBUS_CHAN_ST_ONLIST_SHIFT) == 0) 215251876Speter panic("channel is not on the list"); 216251876Speter TAILQ_REMOVE(&sc->vmbus_chans, chan, ch_link); 217251876Speter} 218251876Speter 219251876Speterstatic int 220251876Spetervmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS) 221251876Speter{ 222251876Speter struct vmbus_channel *chan = arg1; 223251876Speter int mnf = 0; 224251876Speter 225251876Speter if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF) 226251876Speter mnf = 1; 227251876Speter return sysctl_handle_int(oidp, &mnf, 0, req); 228251876Speter} 229251876Speter 230251876Speterstatic void 231251876Spetervmbus_chan_sysctl_create(struct vmbus_channel *chan) 232251876Speter{ 233251876Speter struct sysctl_oid *ch_tree, *chid_tree, *br_tree; 234251876Speter struct sysctl_ctx_list *ctx; 235251876Speter uint32_t ch_id; 236251876Speter char name[16]; 237251876Speter 238251876Speter /* 239251876Speter * Add sysctl nodes related to this channel to this 240251876Speter * channel's sysctl ctx, so that they can be destroyed 241251876Speter * independently upon close of this channel, which can 242251876Speter * happen even if the device is not detached. 243251876Speter */ 244251876Speter ctx = &chan->ch_sysctl_ctx; 245251876Speter sysctl_ctx_init(ctx); 246251876Speter 247251876Speter /* 248251876Speter * Create dev.NAME.UNIT.channel tree. 249251876Speter */ 250251876Speter ch_tree = SYSCTL_ADD_NODE(ctx, 251251876Speter SYSCTL_CHILDREN(device_get_sysctl_tree(chan->ch_dev)), 252251876Speter OID_AUTO, "channel", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 253251876Speter if (ch_tree == NULL) 254251876Speter return; 255251876Speter 256251876Speter /* 257251876Speter * Create dev.NAME.UNIT.channel.CHANID tree. 258251876Speter */ 259251876Speter if (VMBUS_CHAN_ISPRIMARY(chan)) 260251876Speter ch_id = chan->ch_id; 261251876Speter else 262251876Speter ch_id = chan->ch_prichan->ch_id; 263251876Speter snprintf(name, sizeof(name), "%d", ch_id); 264251876Speter chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree), 265251876Speter OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 266251876Speter if (chid_tree == NULL) 267251876Speter return; 268251876Speter 269251876Speter if (!VMBUS_CHAN_ISPRIMARY(chan)) { 270251876Speter /* 271251876Speter * Create dev.NAME.UNIT.channel.CHANID.sub tree. 272251876Speter */ 273251876Speter ch_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree), 274251876Speter OID_AUTO, "sub", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 275251876Speter if (ch_tree == NULL) 276251876Speter return; 277251876Speter 278251876Speter /* 279251876Speter * Create dev.NAME.UNIT.channel.CHANID.sub.SUBIDX tree. 280251876Speter * 281251876Speter * NOTE: 282251876Speter * chid_tree is changed to this new sysctl tree. 283251876Speter */ 284251876Speter snprintf(name, sizeof(name), "%d", chan->ch_subidx); 285251876Speter chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree), 286251876Speter OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 287251876Speter if (chid_tree == NULL) 288251876Speter return; 289251876Speter 290251876Speter SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 291251876Speter "chanid", CTLFLAG_RD, &chan->ch_id, 0, "channel id"); 292251876Speter } 293251876Speter 294251876Speter SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 295251876Speter "cpu", CTLFLAG_RD, &chan->ch_cpuid, 0, "owner CPU id"); 296251876Speter SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 297251876Speter "mnf", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, 298251876Speter chan, 0, vmbus_chan_sysctl_mnf, "I", 299251876Speter "has monitor notification facilities"); 300251876Speter 301251876Speter br_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 302251876Speter "br", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 303251876Speter if (br_tree != NULL) { 304251876Speter /* 305251876Speter * Create sysctl tree for RX bufring. 306251876Speter */ 307251876Speter vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_rxbr.rxbr, "rx"); 308251876Speter /* 309251876Speter * Create sysctl tree for TX bufring. 310251876Speter */ 311251876Speter vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_txbr.txbr, "tx"); 312251876Speter } 313251876Speter} 314251876Speter 315251876Speterint 316251876Spetervmbus_chan_open(struct vmbus_channel *chan, int txbr_size, int rxbr_size, 317251876Speter const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg) 318251876Speter{ 319251876Speter struct vmbus_chan_br cbr; 320251876Speter int error; 321251876Speter 322251876Speter /* 323251876Speter * Allocate the TX+RX bufrings. 324251876Speter */ 325251876Speter KASSERT(chan->ch_bufring == NULL, ("bufrings are allocated")); 326251876Speter chan->ch_bufring = hyperv_dmamem_alloc(bus_get_dma_tag(chan->ch_dev), 327251876Speter PAGE_SIZE, 0, txbr_size + rxbr_size, &chan->ch_bufring_dma, 328253734Speter BUS_DMA_WAITOK); 329251876Speter if (chan->ch_bufring == NULL) { 330251876Speter vmbus_chan_printf(chan, "bufring allocation failed\n"); 331251876Speter return (ENOMEM); 332251876Speter } 333251876Speter 334251876Speter cbr.cbr = chan->ch_bufring; 335251876Speter cbr.cbr_paddr = chan->ch_bufring_dma.hv_paddr; 336251876Speter cbr.cbr_txsz = txbr_size; 337251876Speter cbr.cbr_rxsz = rxbr_size; 338251876Speter 339251876Speter error = vmbus_chan_open_br(chan, &cbr, udata, udlen, cb, cbarg); 340251876Speter if (error) { 341251876Speter if (error == EISCONN) { 342251876Speter /* 343251876Speter * XXX 344251876Speter * The bufring GPADL is still connected; abandon 345251876Speter * this bufring, instead of having mysterious 346251876Speter * crash or trashed data later on. 347251876Speter */ 348251876Speter vmbus_chan_printf(chan, "chan%u bufring GPADL " 349251876Speter "is still connected upon channel open error; " 350251876Speter "leak %d bytes memory\n", chan->ch_id, 351251876Speter txbr_size + rxbr_size); 352251876Speter } else { 353251876Speter hyperv_dmamem_free(&chan->ch_bufring_dma, 354251876Speter chan->ch_bufring); 355251876Speter } 356251876Speter chan->ch_bufring = NULL; 357251876Speter } 358251876Speter return (error); 359251876Speter} 360251876Speter 361251876Speterint 362251876Spetervmbus_chan_open_br(struct vmbus_channel *chan, const struct vmbus_chan_br *cbr, 363251876Speter const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg) 364251876Speter{ 365251876Speter struct vmbus_softc *sc = chan->ch_vmbus; 366251876Speter const struct vmbus_message *msg; 367251876Speter struct vmbus_chanmsg_chopen *req; 368251876Speter struct vmbus_msghc *mh; 369251876Speter uint32_t status; 370251876Speter int error, txbr_size, rxbr_size; 371251876Speter task_fn_t *task_fn; 372251876Speter uint8_t *br; 373251876Speter 374251876Speter if (udlen > VMBUS_CHANMSG_CHOPEN_UDATA_SIZE) { 375251876Speter vmbus_chan_printf(chan, 376251876Speter "invalid udata len %d for chan%u\n", udlen, chan->ch_id); 377251876Speter return (EINVAL); 378251876Speter } 379251876Speter 380251876Speter br = cbr->cbr; 381251876Speter txbr_size = cbr->cbr_txsz; 382251876Speter rxbr_size = cbr->cbr_rxsz; 383251876Speter KASSERT((txbr_size & PAGE_MASK) == 0, 384251876Speter ("send bufring size is not multiple page")); 385251876Speter KASSERT((rxbr_size & PAGE_MASK) == 0, 386251876Speter ("recv bufring size is not multiple page")); 387251876Speter KASSERT((cbr->cbr_paddr & PAGE_MASK) == 0, 388251876Speter ("bufring is not page aligned")); 389251876Speter 390251876Speter /* 391251876Speter * Zero out the TX/RX bufrings, in case that they were used before. 392251876Speter */ 393251876Speter memset(br, 0, txbr_size + rxbr_size); 394251876Speter 395251876Speter if (atomic_testandset_int(&chan->ch_stflags, 396251876Speter VMBUS_CHAN_ST_OPENED_SHIFT)) 397251876Speter panic("double-open chan%u", chan->ch_id); 398251876Speter 399251876Speter chan->ch_cb = cb; 400251876Speter chan->ch_cbarg = cbarg; 401251876Speter 402251876Speter vmbus_chan_update_evtflagcnt(sc, chan); 403251876Speter 404251876Speter chan->ch_tq = VMBUS_PCPU_GET(chan->ch_vmbus, event_tq, chan->ch_cpuid); 405251876Speter if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD) 406251876Speter task_fn = vmbus_chan_task; 407251876Speter else 408251876Speter task_fn = vmbus_chan_task_nobatch; 409251876Speter TASK_INIT(&chan->ch_task, 0, task_fn, chan); 410251876Speter 411251876Speter /* TX bufring comes first */ 412251876Speter vmbus_txbr_setup(&chan->ch_txbr, br, txbr_size); 413251876Speter /* RX bufring immediately follows TX bufring */ 414251876Speter vmbus_rxbr_setup(&chan->ch_rxbr, br + txbr_size, rxbr_size); 415251876Speter 416251876Speter /* Create sysctl tree for this channel */ 417251876Speter vmbus_chan_sysctl_create(chan); 418251876Speter 419251876Speter /* 420251876Speter * Connect the bufrings, both RX and TX, to this channel. 421251876Speter */ 422251876Speter error = vmbus_chan_gpadl_connect(chan, cbr->cbr_paddr, 423251876Speter txbr_size + rxbr_size, &chan->ch_bufring_gpadl); 424251876Speter if (error) { 425251876Speter vmbus_chan_printf(chan, 426251876Speter "failed to connect bufring GPADL to chan%u\n", chan->ch_id); 427251876Speter goto failed; 428251876Speter } 429251876Speter 430251876Speter /* 431251876Speter * Install this channel, before it is opened, but after everything 432251876Speter * else has been setup. 433251876Speter */ 434251876Speter vmbus_chan_set_chmap(chan); 435251876Speter 436251876Speter /* 437251876Speter * Open channel w/ the bufring GPADL on the target CPU. 438251876Speter */ 439251876Speter mh = vmbus_msghc_get(sc, sizeof(*req)); 440251876Speter if (mh == NULL) { 441251876Speter vmbus_chan_printf(chan, 442251876Speter "can not get msg hypercall for chopen(chan%u)\n", 443251876Speter chan->ch_id); 444251876Speter error = ENXIO; 445251876Speter goto failed; 446251876Speter } 447251876Speter 448251876Speter req = vmbus_msghc_dataptr(mh); 449251876Speter req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHOPEN; 450251876Speter req->chm_chanid = chan->ch_id; 451251876Speter req->chm_openid = chan->ch_id; 452251876Speter req->chm_gpadl = chan->ch_bufring_gpadl; 453251876Speter req->chm_vcpuid = chan->ch_vcpuid; 454251876Speter req->chm_txbr_pgcnt = txbr_size >> PAGE_SHIFT; 455251876Speter if (udlen > 0) 456251876Speter memcpy(req->chm_udata, udata, udlen); 457251876Speter 458251876Speter error = vmbus_msghc_exec(sc, mh); 459251876Speter if (error) { 460251876Speter vmbus_chan_printf(chan, 461251876Speter "chopen(chan%u) msg hypercall exec failed: %d\n", 462251876Speter chan->ch_id, error); 463251876Speter vmbus_msghc_put(sc, mh); 464251876Speter goto failed; 465251876Speter } 466251876Speter 467251876Speter for (;;) { 468251876Speter msg = vmbus_msghc_poll_result(sc, mh); 469251876Speter if (msg != NULL) 470251876Speter break; 471251876Speter if (vmbus_chan_is_revoked(chan)) { 472251876Speter int i; 473251876Speter 474251876Speter /* 475251876Speter * NOTE: 476251876Speter * Hypervisor does _not_ send response CHOPEN to 477251876Speter * a revoked channel. 478251876Speter */ 479251876Speter vmbus_chan_printf(chan, 480251876Speter "chan%u is revoked, when it is being opened\n", 481251876Speter chan->ch_id); 482251876Speter 483251876Speter /* 484251876Speter * XXX 485251876Speter * Add extra delay before cancel the hypercall 486251876Speter * execution; mainly to close any possible 487251876Speter * CHRESCIND and CHOPEN_RESP races on the 488251876Speter * hypervisor side. 489251876Speter */ 490251876Speter#define REVOKE_LINGER 100 491251876Speter for (i = 0; i < REVOKE_LINGER; ++i) { 492251876Speter msg = vmbus_msghc_poll_result(sc, mh); 493251876Speter if (msg != NULL) 494251876Speter break; 495251876Speter pause("rchopen", 1); 496251876Speter } 497251876Speter#undef REVOKE_LINGER 498251876Speter if (msg == NULL) 499251876Speter vmbus_msghc_exec_cancel(sc, mh); 500251876Speter break; 501251876Speter } 502251876Speter pause("chopen", 1); 503251876Speter } 504251876Speter if (msg != NULL) { 505251876Speter status = ((const struct vmbus_chanmsg_chopen_resp *) 506251876Speter msg->msg_data)->chm_status; 507251876Speter } else { 508251876Speter /* XXX any non-0 value is ok here. */ 509251876Speter status = 0xff; 510251876Speter } 511251876Speter 512251876Speter vmbus_msghc_put(sc, mh); 513251876Speter 514251876Speter if (status == 0) { 515251876Speter if (bootverbose) 516251876Speter vmbus_chan_printf(chan, "chan%u opened\n", chan->ch_id); 517251876Speter return (0); 518251876Speter } 519251876Speter 520251876Speter vmbus_chan_printf(chan, "failed to open chan%u\n", chan->ch_id); 521251876Speter error = ENXIO; 522251876Speter 523251876Speterfailed: 524251876Speter sysctl_ctx_free(&chan->ch_sysctl_ctx); 525251876Speter vmbus_chan_clear_chmap(chan); 526251876Speter if (chan->ch_bufring_gpadl != 0) { 527251876Speter int error1; 528251876Speter 529251876Speter error1 = vmbus_chan_gpadl_disconnect(chan, 530251876Speter chan->ch_bufring_gpadl); 531251876Speter if (error1) { 532251876Speter /* 533251876Speter * Give caller a hint that the bufring GPADL is still 534251876Speter * connected. 535251876Speter */ 536251876Speter error = EISCONN; 537251876Speter } 538251876Speter chan->ch_bufring_gpadl = 0; 539251876Speter } 540251876Speter atomic_clear_int(&chan->ch_stflags, VMBUS_CHAN_ST_OPENED); 541251876Speter return (error); 542251876Speter} 543251876Speter 544251876Speterint 545251876Spetervmbus_chan_gpadl_connect(struct vmbus_channel *chan, bus_addr_t paddr, 546251876Speter int size, uint32_t *gpadl0) 547251876Speter{ 548251876Speter struct vmbus_softc *sc = chan->ch_vmbus; 549251876Speter struct vmbus_msghc *mh; 550251876Speter struct vmbus_chanmsg_gpadl_conn *req; 551251876Speter const struct vmbus_message *msg; 552251876Speter size_t reqsz; 553251876Speter uint32_t gpadl, status; 554251876Speter int page_count, range_len, i, cnt, error; 555251876Speter uint64_t page_id; 556251876Speter 557251876Speter KASSERT(*gpadl0 == 0, ("GPADL is not zero")); 558251876Speter 559251876Speter /* 560251876Speter * Preliminary checks. 561251876Speter */ 562251876Speter 563251876Speter KASSERT((size & PAGE_MASK) == 0, 564251876Speter ("invalid GPA size %d, not multiple page size", size)); 565251876Speter page_count = size >> PAGE_SHIFT; 566251876Speter 567251876Speter KASSERT((paddr & PAGE_MASK) == 0, 568251876Speter ("GPA is not page aligned %jx", (uintmax_t)paddr)); 569251876Speter page_id = paddr >> PAGE_SHIFT; 570251876Speter 571251876Speter range_len = __offsetof(struct vmbus_gpa_range, gpa_page[page_count]); 572251876Speter /* 573251876Speter * We don't support multiple GPA ranges. 574251876Speter */ 575251876Speter if (range_len > UINT16_MAX) { 576251876Speter vmbus_chan_printf(chan, "GPA too large, %d pages\n", 577251876Speter page_count); 578251876Speter return EOPNOTSUPP; 579251876Speter } 580251876Speter 581251876Speter /* 582251876Speter * Allocate GPADL id. 583251876Speter */ 584251876Speter gpadl = vmbus_gpadl_alloc(sc); 585251876Speter 586251876Speter /* 587251876Speter * Connect this GPADL to the target channel. 588251876Speter * 589251876Speter * NOTE: 590251876Speter * Since each message can only hold small set of page 591251876Speter * addresses, several messages may be required to 592251876Speter * complete the connection. 593251876Speter */ 594251876Speter if (page_count > VMBUS_CHANMSG_GPADL_CONN_PGMAX) 595251876Speter cnt = VMBUS_CHANMSG_GPADL_CONN_PGMAX; 596251876Speter else 597251876Speter cnt = page_count; 598251876Speter page_count -= cnt; 599251876Speter 600251876Speter reqsz = __offsetof(struct vmbus_chanmsg_gpadl_conn, 601251876Speter chm_range.gpa_page[cnt]); 602251876Speter mh = vmbus_msghc_get(sc, reqsz); 603251876Speter if (mh == NULL) { 604251876Speter vmbus_chan_printf(chan, 605251876Speter "can not get msg hypercall for gpadl_conn(chan%u)\n", 606251876Speter chan->ch_id); 607251876Speter return EIO; 608251876Speter } 609251876Speter 610251876Speter req = vmbus_msghc_dataptr(mh); 611251876Speter req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_CONN; 612251876Speter req->chm_chanid = chan->ch_id; 613251876Speter req->chm_gpadl = gpadl; 614251876Speter req->chm_range_len = range_len; 615251876Speter req->chm_range_cnt = 1; 616251876Speter req->chm_range.gpa_len = size; 617251876Speter req->chm_range.gpa_ofs = 0; 618251876Speter for (i = 0; i < cnt; ++i) 619251876Speter req->chm_range.gpa_page[i] = page_id++; 620251876Speter 621251876Speter error = vmbus_msghc_exec(sc, mh); 622251876Speter if (error) { 623251876Speter vmbus_chan_printf(chan, 624251876Speter "gpadl_conn(chan%u) msg hypercall exec failed: %d\n", 625251876Speter chan->ch_id, error); 626251876Speter vmbus_msghc_put(sc, mh); 627251876Speter return error; 628251876Speter } 629251876Speter 630251876Speter while (page_count > 0) { 631251876Speter struct vmbus_chanmsg_gpadl_subconn *subreq; 632251876Speter 633251876Speter if (page_count > VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX) 634251876Speter cnt = VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX; 635251876Speter else 636251876Speter cnt = page_count; 637251876Speter page_count -= cnt; 638251876Speter 639251876Speter reqsz = __offsetof(struct vmbus_chanmsg_gpadl_subconn, 640251876Speter chm_gpa_page[cnt]); 641251876Speter vmbus_msghc_reset(mh, reqsz); 642251876Speter 643251876Speter subreq = vmbus_msghc_dataptr(mh); 644251876Speter subreq->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_SUBCONN; 645251876Speter subreq->chm_gpadl = gpadl; 646251876Speter for (i = 0; i < cnt; ++i) 647251876Speter subreq->chm_gpa_page[i] = page_id++; 648251876Speter 649251876Speter vmbus_msghc_exec_noresult(mh); 650251876Speter } 651251876Speter KASSERT(page_count == 0, ("invalid page count %d", page_count)); 652251876Speter 653251876Speter msg = vmbus_msghc_wait_result(sc, mh); 654251876Speter status = ((const struct vmbus_chanmsg_gpadl_connresp *) 655251876Speter msg->msg_data)->chm_status; 656251876Speter 657251876Speter vmbus_msghc_put(sc, mh); 658251876Speter 659251876Speter if (status != 0) { 660251876Speter vmbus_chan_printf(chan, "gpadl_conn(chan%u) failed: %u\n", 661251876Speter chan->ch_id, status); 662251876Speter return EIO; 663251876Speter } 664251876Speter 665251876Speter /* Done; commit the GPADL id. */ 666251876Speter *gpadl0 = gpadl; 667251876Speter if (bootverbose) { 668251876Speter vmbus_chan_printf(chan, "gpadl_conn(chan%u) succeeded\n", 669251876Speter chan->ch_id); 670251876Speter } 671251876Speter return 0; 672251876Speter} 673251876Speter 674251876Speterstatic bool 675251876Spetervmbus_chan_wait_revoke(const struct vmbus_channel *chan, bool can_sleep) 676251876Speter{ 677251876Speter#define WAIT_COUNT 200 /* 200ms */ 678251876Speter 679251876Speter int i; 680251876Speter 681251876Speter for (i = 0; i < WAIT_COUNT; ++i) { 682251876Speter if (vmbus_chan_is_revoked(chan)) 683251876Speter return (true); 684251876Speter if (can_sleep) 685251876Speter pause("wchrev", 1); 686251876Speter else 687251876Speter DELAY(1000); 688251876Speter } 689251876Speter return (false); 690251876Speter 691251876Speter#undef WAIT_COUNT 692251876Speter} 693251876Speter 694251876Speter/* 695251876Speter * Disconnect the GPA from the target channel 696251876Speter */ 697251876Speterint 698251876Spetervmbus_chan_gpadl_disconnect(struct vmbus_channel *chan, uint32_t gpadl) 699251876Speter{ 700251876Speter struct vmbus_softc *sc = chan->ch_vmbus; 701251876Speter struct vmbus_msghc *mh; 702251876Speter struct vmbus_chanmsg_gpadl_disconn *req; 703251876Speter int error; 704251876Speter 705251876Speter KASSERT(gpadl != 0, ("GPADL is zero")); 706251876Speter 707251876Speter mh = vmbus_msghc_get(sc, sizeof(*req)); 708251876Speter if (mh == NULL) { 709251876Speter vmbus_chan_printf(chan, 710251876Speter "can not get msg hypercall for gpadl_disconn(chan%u)\n", 711251876Speter chan->ch_id); 712251876Speter return (EBUSY); 713251876Speter } 714251876Speter 715251876Speter req = vmbus_msghc_dataptr(mh); 716251876Speter req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_DISCONN; 717251876Speter req->chm_chanid = chan->ch_id; 718251876Speter req->chm_gpadl = gpadl; 719251876Speter 720251876Speter error = vmbus_msghc_exec(sc, mh); 721251876Speter if (error) { 722251876Speter vmbus_msghc_put(sc, mh); 723251876Speter 724251876Speter if (vmbus_chan_wait_revoke(chan, true)) { 725251876Speter /* 726251876Speter * Error is benign; this channel is revoked, 727251876Speter * so this GPADL will not be touched anymore. 728251876Speter */ 729251876Speter vmbus_chan_printf(chan, 730251876Speter "gpadl_disconn(revoked chan%u) msg hypercall " 731251876Speter "exec failed: %d\n", chan->ch_id, error); 732251876Speter return (0); 733251876Speter } 734251876Speter vmbus_chan_printf(chan, 735251876Speter "gpadl_disconn(chan%u) msg hypercall exec failed: %d\n", 736251876Speter chan->ch_id, error); 737251876Speter return (error); 738251876Speter } 739251876Speter 740251876Speter vmbus_msghc_wait_result(sc, mh); 741251876Speter /* Discard result; no useful information */ 742251876Speter vmbus_msghc_put(sc, mh); 743251876Speter 744251876Speter return (0); 745251876Speter} 746251876Speter 747251876Speterstatic void 748251876Spetervmbus_chan_detach(struct vmbus_channel *chan) 749251876Speter{ 750251876Speter int refs; 751251876Speter 752251876Speter KASSERT(chan->ch_refs > 0, ("chan%u: invalid refcnt %d", 753251876Speter chan->ch_id, chan->ch_refs)); 754251876Speter refs = atomic_fetchadd_int(&chan->ch_refs, -1); 755251876Speter#ifdef INVARIANTS 756251876Speter if (VMBUS_CHAN_ISPRIMARY(chan)) { 757251876Speter KASSERT(refs == 1, ("chan%u: invalid refcnt %d for prichan", 758251876Speter chan->ch_id, refs + 1)); 759251876Speter } 760251876Speter#endif 761251876Speter if (refs == 1) { 762251876Speter /* 763251876Speter * Detach the target channel. 764251876Speter */ 765251876Speter if (bootverbose) { 766251876Speter vmbus_chan_printf(chan, "chan%u detached\n", 767251876Speter chan->ch_id); 768251876Speter } 769251876Speter taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task); 770251876Speter } 771251876Speter} 772251876Speter 773251876Speterstatic void 774251876Spetervmbus_chan_clrchmap_task(void *xchan, int pending __unused) 775251876Speter{ 776251876Speter struct vmbus_channel *chan = xchan; 777251876Speter 778251876Speter chan->ch_vmbus->vmbus_chmap[chan->ch_id] = NULL; 779251876Speter} 780251876Speter 781251876Speterstatic void 782251876Spetervmbus_chan_clear_chmap(struct vmbus_channel *chan) 783251876Speter{ 784251876Speter struct task chmap_task; 785251876Speter 786251876Speter TASK_INIT(&chmap_task, 0, vmbus_chan_clrchmap_task, chan); 787251876Speter vmbus_chan_run_task(chan, &chmap_task); 788251876Speter} 789251876Speter 790251876Speterstatic void 791251876Spetervmbus_chan_set_chmap(struct vmbus_channel *chan) 792251876Speter{ 793251876Speter __compiler_membar(); 794251876Speter chan->ch_vmbus->vmbus_chmap[chan->ch_id] = chan; 795251876Speter} 796251876Speter 797251876Speterstatic void 798251876Spetervmbus_chan_poll_cancel_task(void *xchan, int pending __unused) 799251876Speter{ 800251876Speter 801251876Speter vmbus_chan_poll_cancel_intq(xchan); 802251876Speter} 803251876Speter 804251876Speterstatic void 805251876Spetervmbus_chan_poll_cancel(struct vmbus_channel *chan) 806251876Speter{ 807251876Speter struct task poll_cancel; 808251876Speter 809251876Speter TASK_INIT(&poll_cancel, 0, vmbus_chan_poll_cancel_task, chan); 810251876Speter vmbus_chan_run_task(chan, &poll_cancel); 811251876Speter} 812251876Speter 813251876Speterstatic int 814251876Spetervmbus_chan_close_internal(struct vmbus_channel *chan) 815251876Speter{ 816251876Speter struct vmbus_softc *sc = chan->ch_vmbus; 817251876Speter struct vmbus_msghc *mh; 818251876Speter struct vmbus_chanmsg_chclose *req; 819251876Speter uint32_t old_stflags; 820251876Speter int error; 821251876Speter 822251876Speter /* 823251876Speter * NOTE: 824251876Speter * Sub-channels are closed upon their primary channel closing, 825251876Speter * so they can be closed even before they are opened. 826251876Speter */ 827251876Speter for (;;) { 828251876Speter old_stflags = chan->ch_stflags; 829251876Speter if (atomic_cmpset_int(&chan->ch_stflags, old_stflags, 830251876Speter old_stflags & ~VMBUS_CHAN_ST_OPENED)) 831251876Speter break; 832251876Speter } 833251876Speter if ((old_stflags & VMBUS_CHAN_ST_OPENED) == 0) { 834251876Speter /* Not opened yet; done */ 835251876Speter if (bootverbose) { 836251876Speter vmbus_chan_printf(chan, "chan%u not opened\n", 837251876Speter chan->ch_id); 838251876Speter } 839251876Speter return (0); 840251876Speter } 841251876Speter 842251876Speter /* 843251876Speter * Free this channel's sysctl tree attached to its device's 844251876Speter * sysctl tree. 845251876Speter */ 846251876Speter sysctl_ctx_free(&chan->ch_sysctl_ctx); 847251876Speter 848251876Speter /* 849251876Speter * Cancel polling, if it is enabled. 850251876Speter */ 851251876Speter vmbus_chan_poll_cancel(chan); 852251876Speter 853251876Speter /* 854251876Speter * NOTE: 855251876Speter * Order is critical. This channel _must_ be uninstalled first, 856251876Speter * else the channel task may be enqueued by the IDT after it has 857251876Speter * been drained. 858251876Speter */ 859251876Speter vmbus_chan_clear_chmap(chan); 860251876Speter taskqueue_drain(chan->ch_tq, &chan->ch_task); 861251876Speter chan->ch_tq = NULL; 862251876Speter 863251876Speter /* 864251876Speter * Close this channel. 865251876Speter */ 866251876Speter mh = vmbus_msghc_get(sc, sizeof(*req)); 867251876Speter if (mh == NULL) { 868251876Speter vmbus_chan_printf(chan, 869251876Speter "can not get msg hypercall for chclose(chan%u)\n", 870251876Speter chan->ch_id); 871251876Speter error = ENXIO; 872251876Speter goto disconnect; 873251876Speter } 874251876Speter 875251876Speter req = vmbus_msghc_dataptr(mh); 876251876Speter req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHCLOSE; 877251876Speter req->chm_chanid = chan->ch_id; 878251876Speter 879251876Speter error = vmbus_msghc_exec_noresult(mh); 880251876Speter vmbus_msghc_put(sc, mh); 881251876Speter 882251876Speter if (error) { 883251876Speter vmbus_chan_printf(chan, 884251876Speter "chclose(chan%u) msg hypercall exec failed: %d\n", 885251876Speter chan->ch_id, error); 886251876Speter goto disconnect; 887251876Speter } 888251876Speter 889251876Speter if (bootverbose) 890251876Speter vmbus_chan_printf(chan, "chan%u closed\n", chan->ch_id); 891251876Speter 892251876Speterdisconnect: 893251876Speter /* 894251876Speter * Disconnect the TX+RX bufrings from this channel. 895251876Speter */ 896251876Speter if (chan->ch_bufring_gpadl != 0) { 897251876Speter int error1; 898251876Speter 899251876Speter error1 = vmbus_chan_gpadl_disconnect(chan, 900251876Speter chan->ch_bufring_gpadl); 901251876Speter if (error1) { 902251876Speter /* 903251876Speter * XXX 904251876Speter * The bufring GPADL is still connected; abandon 905251876Speter * this bufring, instead of having mysterious 906251876Speter * crash or trashed data later on. 907251876Speter */ 908251876Speter vmbus_chan_printf(chan, "chan%u bufring GPADL " 909251876Speter "is still connected after close\n", chan->ch_id); 910251876Speter chan->ch_bufring = NULL; 911251876Speter /* 912251876Speter * Give caller a hint that the bufring GPADL is 913251876Speter * still connected. 914251876Speter */ 915251876Speter error = EISCONN; 916251876Speter } 917251876Speter chan->ch_bufring_gpadl = 0; 918251876Speter } 919251876Speter 920251876Speter /* 921251876Speter * Destroy the TX+RX bufrings. 922251876Speter */ 923251876Speter if (chan->ch_bufring != NULL) { 924251876Speter hyperv_dmamem_free(&chan->ch_bufring_dma, chan->ch_bufring); 925251876Speter chan->ch_bufring = NULL; 926251876Speter } 927251876Speter return (error); 928251876Speter} 929251876Speter 930251876Speterint 931251876Spetervmbus_chan_close_direct(struct vmbus_channel *chan) 932251876Speter{ 933251876Speter int error; 934251876Speter 935251876Speter#ifdef INVARIANTS 936251876Speter if (VMBUS_CHAN_ISPRIMARY(chan)) { 937251876Speter struct vmbus_channel *subchan; 938251876Speter 939251876Speter /* 940251876Speter * All sub-channels _must_ have been closed, or are _not_ 941251876Speter * opened at all. 942251876Speter */ 943251876Speter mtx_lock(&chan->ch_subchan_lock); 944251876Speter TAILQ_FOREACH(subchan, &chan->ch_subchans, ch_sublink) { 945251876Speter KASSERT( 946251876Speter (subchan->ch_stflags & VMBUS_CHAN_ST_OPENED) == 0, 947251876Speter ("chan%u: subchan%u is still opened", 948251876Speter chan->ch_id, subchan->ch_subidx)); 949251876Speter } 950251876Speter mtx_unlock(&chan->ch_subchan_lock); 951251876Speter } 952251876Speter#endif 953251876Speter 954251876Speter error = vmbus_chan_close_internal(chan); 955251876Speter if (!VMBUS_CHAN_ISPRIMARY(chan)) { 956251876Speter /* 957251876Speter * This sub-channel is referenced, when it is linked to 958251876Speter * the primary channel; drop that reference now. 959251876Speter */ 960251876Speter vmbus_chan_detach(chan); 961251876Speter } 962251876Speter return (error); 963251876Speter} 964251876Speter 965251876Speter/* 966251876Speter * Caller should make sure that all sub-channels have 967251876Speter * been added to 'chan' and all to-be-closed channels 968251876Speter * are not being opened. 969251876Speter */ 970251876Spetervoid 971251876Spetervmbus_chan_close(struct vmbus_channel *chan) 972251876Speter{ 973251876Speter int subchan_cnt; 974251876Speter 975251876Speter if (!VMBUS_CHAN_ISPRIMARY(chan)) { 976251876Speter /* 977251876Speter * Sub-channel is closed when its primary channel 978251876Speter * is closed; done. 979251876Speter */ 980251876Speter return; 981251876Speter } 982251876Speter 983251876Speter /* 984251876Speter * Close all sub-channels, if any. 985251876Speter */ 986251876Speter subchan_cnt = chan->ch_subchan_cnt; 987251876Speter if (subchan_cnt > 0) { 988251876Speter struct vmbus_channel **subchan; 989251876Speter int i; 990251876Speter 991251876Speter subchan = vmbus_subchan_get(chan, subchan_cnt); 992251876Speter for (i = 0; i < subchan_cnt; ++i) { 993251876Speter vmbus_chan_close_internal(subchan[i]); 994251876Speter /* 995251876Speter * This sub-channel is referenced, when it is 996251876Speter * linked to the primary channel; drop that 997251876Speter * reference now. 998251876Speter */ 999251876Speter vmbus_chan_detach(subchan[i]); 1000251876Speter } 1001251876Speter vmbus_subchan_rel(subchan, subchan_cnt); 1002251876Speter } 1003251876Speter 1004251876Speter /* Then close the primary channel. */ 1005251876Speter vmbus_chan_close_internal(chan); 1006251876Speter} 1007251876Speter 1008251876Spetervoid 1009251876Spetervmbus_chan_intr_drain(struct vmbus_channel *chan) 1010251876Speter{ 1011251876Speter 1012251876Speter taskqueue_drain(chan->ch_tq, &chan->ch_task); 1013251876Speter} 1014251876Speter 1015251876Speterint 1016251876Spetervmbus_chan_send(struct vmbus_channel *chan, uint16_t type, uint16_t flags, 1017251876Speter void *data, int dlen, uint64_t xactid) 1018251876Speter{ 1019251876Speter struct vmbus_chanpkt pkt; 1020251876Speter int pktlen, pad_pktlen, hlen, error; 1021251876Speter uint64_t pad = 0; 1022251876Speter struct iovec iov[3]; 1023251876Speter boolean_t send_evt; 1024251876Speter 1025251876Speter hlen = sizeof(pkt); 1026251876Speter pktlen = hlen + dlen; 1027251876Speter pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1028251876Speter KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1029251876Speter ("invalid packet size %d", pad_pktlen)); 1030251876Speter 1031251876Speter pkt.cp_hdr.cph_type = type; 1032251876Speter pkt.cp_hdr.cph_flags = flags; 1033251876Speter VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1034251876Speter VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1035251876Speter pkt.cp_hdr.cph_xactid = xactid; 1036251876Speter 1037251876Speter iov[0].iov_base = &pkt; 1038251876Speter iov[0].iov_len = hlen; 1039251876Speter iov[1].iov_base = data; 1040251876Speter iov[1].iov_len = dlen; 1041251876Speter iov[2].iov_base = &pad; 1042251876Speter iov[2].iov_len = pad_pktlen - pktlen; 1043251876Speter 1044251876Speter error = vmbus_txbr_write(&chan->ch_txbr, iov, 3, &send_evt); 1045251876Speter if (!error && send_evt) 1046251876Speter vmbus_chan_signal_tx(chan); 1047251876Speter return error; 1048251876Speter} 1049251876Speter 1050251876Speterint 1051251876Spetervmbus_chan_send_sglist(struct vmbus_channel *chan, 1052251876Speter struct vmbus_gpa sg[], int sglen, void *data, int dlen, uint64_t xactid) 1053272076Speter{ 1054272076Speter struct vmbus_chanpkt_sglist pkt; 1055251876Speter int pktlen, pad_pktlen, hlen, error; 1056251876Speter struct iovec iov[4]; 1057251876Speter boolean_t send_evt; 1058251876Speter uint64_t pad = 0; 1059251876Speter 1060251876Speter hlen = __offsetof(struct vmbus_chanpkt_sglist, cp_gpa[sglen]); 1061251876Speter pktlen = hlen + dlen; 1062251876Speter pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1063251876Speter KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1064251876Speter ("invalid packet size %d", pad_pktlen)); 1065251876Speter 1066251876Speter pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1067251876Speter pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1068251876Speter VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1069251876Speter VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1070251876Speter pkt.cp_hdr.cph_xactid = xactid; 1071251876Speter pkt.cp_rsvd = 0; 1072251876Speter pkt.cp_gpa_cnt = sglen; 1073251876Speter 1074251876Speter iov[0].iov_base = &pkt; 1075251876Speter iov[0].iov_len = sizeof(pkt); 1076251876Speter iov[1].iov_base = sg; 1077251876Speter iov[1].iov_len = sizeof(struct vmbus_gpa) * sglen; 1078251876Speter iov[2].iov_base = data; 1079251876Speter iov[2].iov_len = dlen; 1080251876Speter iov[3].iov_base = &pad; 1081251876Speter iov[3].iov_len = pad_pktlen - pktlen; 1082251876Speter 1083251876Speter error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); 1084251876Speter if (!error && send_evt) 1085251876Speter vmbus_chan_signal_tx(chan); 1086251876Speter return error; 1087251876Speter} 1088251876Speter 1089251876Speterint 1090251876Spetervmbus_chan_send_prplist(struct vmbus_channel *chan, 1091251876Speter struct vmbus_gpa_range *prp, int prp_cnt, void *data, int dlen, 1092251876Speter uint64_t xactid) 1093251876Speter{ 1094251876Speter struct vmbus_chanpkt_prplist pkt; 1095251876Speter int pktlen, pad_pktlen, hlen, error; 1096251876Speter struct iovec iov[4]; 1097251876Speter boolean_t send_evt; 1098251876Speter uint64_t pad = 0; 1099251876Speter 1100251876Speter hlen = __offsetof(struct vmbus_chanpkt_prplist, 1101251876Speter cp_range[0].gpa_page[prp_cnt]); 1102251876Speter pktlen = hlen + dlen; 1103251876Speter pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1104251876Speter KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1105251876Speter ("invalid packet size %d", pad_pktlen)); 1106251876Speter 1107251876Speter pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1108251876Speter pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1109251876Speter VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1110251876Speter VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1111251876Speter pkt.cp_hdr.cph_xactid = xactid; 1112251876Speter pkt.cp_rsvd = 0; 1113251876Speter pkt.cp_range_cnt = 1; 1114251876Speter 1115251876Speter iov[0].iov_base = &pkt; 1116251876Speter iov[0].iov_len = sizeof(pkt); 1117251876Speter iov[1].iov_base = prp; 1118251876Speter iov[1].iov_len = __offsetof(struct vmbus_gpa_range, gpa_page[prp_cnt]); 1119251876Speter iov[2].iov_base = data; 1120251876Speter iov[2].iov_len = dlen; 1121251876Speter iov[3].iov_base = &pad; 1122251876Speter iov[3].iov_len = pad_pktlen - pktlen; 1123251876Speter 1124251876Speter error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); 1125251876Speter if (!error && send_evt) 1126251876Speter vmbus_chan_signal_tx(chan); 1127251876Speter return error; 1128251876Speter} 1129251876Speter 1130251876Speterint 1131251876Spetervmbus_chan_recv(struct vmbus_channel *chan, void *data, int *dlen0, 1132251876Speter uint64_t *xactid) 1133251876Speter{ 1134251876Speter struct vmbus_chanpkt_hdr pkt; 1135251876Speter int error, dlen, hlen; 1136251876Speter 1137251876Speter error = vmbus_rxbr_peek(&chan->ch_rxbr, &pkt, sizeof(pkt)); 1138251876Speter if (error) 1139251876Speter return (error); 1140251876Speter 1141251876Speter if (__predict_false(pkt.cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) { 1142251876Speter vmbus_chan_printf(chan, "invalid hlen %u\n", pkt.cph_hlen); 1143251876Speter /* XXX this channel is dead actually. */ 1144251876Speter return (EIO); 1145251876Speter } 1146251876Speter if (__predict_false(pkt.cph_hlen > pkt.cph_tlen)) { 1147251876Speter vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n", 1148251876Speter pkt.cph_hlen, pkt.cph_tlen); 1149251876Speter /* XXX this channel is dead actually. */ 1150251876Speter return (EIO); 1151251876Speter } 1152251876Speter 1153251876Speter hlen = VMBUS_CHANPKT_GETLEN(pkt.cph_hlen); 1154251876Speter dlen = VMBUS_CHANPKT_GETLEN(pkt.cph_tlen) - hlen; 1155251876Speter 1156251876Speter if (*dlen0 < dlen) { 1157251876Speter /* Return the size of this packet's data. */ 1158251876Speter *dlen0 = dlen; 1159251876Speter return (ENOBUFS); 1160251876Speter } 1161251876Speter 1162251876Speter *xactid = pkt.cph_xactid; 1163251876Speter *dlen0 = dlen; 1164251876Speter 1165251876Speter /* Skip packet header */ 1166251876Speter error = vmbus_rxbr_read(&chan->ch_rxbr, data, dlen, hlen); 1167251876Speter KASSERT(!error, ("vmbus_rxbr_read failed")); 1168251876Speter 1169251876Speter return (0); 1170251876Speter} 1171251876Speter 1172251876Speterint 1173251876Spetervmbus_chan_recv_pkt(struct vmbus_channel *chan, 1174251876Speter struct vmbus_chanpkt_hdr *pkt, int *pktlen0) 1175251876Speter{ 1176251876Speter int error, pktlen, pkt_hlen; 1177251876Speter 1178251876Speter pkt_hlen = sizeof(*pkt); 1179251876Speter error = vmbus_rxbr_peek(&chan->ch_rxbr, pkt, pkt_hlen); 1180251876Speter if (error) 1181251876Speter return (error); 1182251876Speter 1183251876Speter if (__predict_false(pkt->cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) { 1184251876Speter vmbus_chan_printf(chan, "invalid hlen %u\n", pkt->cph_hlen); 1185251876Speter /* XXX this channel is dead actually. */ 1186251876Speter return (EIO); 1187251876Speter } 1188251876Speter if (__predict_false(pkt->cph_hlen > pkt->cph_tlen)) { 1189251876Speter vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n", 1190251876Speter pkt->cph_hlen, pkt->cph_tlen); 1191251876Speter /* XXX this channel is dead actually. */ 1192251876Speter return (EIO); 1193251876Speter } 1194251876Speter 1195251876Speter pktlen = VMBUS_CHANPKT_GETLEN(pkt->cph_tlen); 1196251876Speter if (*pktlen0 < pktlen) { 1197251876Speter /* Return the size of this packet. */ 1198251876Speter *pktlen0 = pktlen; 1199251876Speter return (ENOBUFS); 1200251876Speter } 1201251876Speter *pktlen0 = pktlen; 1202251876Speter 1203251876Speter /* 1204251876Speter * Skip the fixed-size packet header, which has been filled 1205251876Speter * by the above vmbus_rxbr_peek(). 1206251876Speter */ 1207251876Speter error = vmbus_rxbr_read(&chan->ch_rxbr, pkt + 1, 1208251876Speter pktlen - pkt_hlen, pkt_hlen); 1209251876Speter KASSERT(!error, ("vmbus_rxbr_read failed")); 1210251876Speter 1211251876Speter return (0); 1212251876Speter} 1213251876Speter 1214251876Speterstatic void 1215251876Spetervmbus_chan_task(void *xchan, int pending __unused) 1216251876Speter{ 1217251876Speter struct vmbus_channel *chan = xchan; 1218251876Speter vmbus_chan_callback_t cb = chan->ch_cb; 1219251876Speter void *cbarg = chan->ch_cbarg; 1220251876Speter 1221251876Speter KASSERT(chan->ch_poll_intvl == 0, 1222251876Speter ("chan%u: interrupted in polling mode", chan->ch_id)); 1223251876Speter 1224251876Speter /* 1225251876Speter * Optimize host to guest signaling by ensuring: 1226251876Speter * 1. While reading the channel, we disable interrupts from 1227251876Speter * host. 1228251876Speter * 2. Ensure that we process all posted messages from the host 1229251876Speter * before returning from this callback. 1230251876Speter * 3. Once we return, enable signaling from the host. Once this 1231251876Speter * state is set we check to see if additional packets are 1232251876Speter * available to read. In this case we repeat the process. 1233251876Speter * 1234251876Speter * NOTE: Interrupt has been disabled in the ISR. 1235251876Speter */ 1236251876Speter for (;;) { 1237251876Speter uint32_t left; 1238251876Speter 1239251876Speter cb(chan, cbarg); 1240251876Speter 1241251876Speter left = vmbus_rxbr_intr_unmask(&chan->ch_rxbr); 1242251876Speter if (left == 0) { 1243251876Speter /* No more data in RX bufring; done */ 1244251876Speter break; 1245251876Speter } 1246251876Speter vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1247251876Speter } 1248251876Speter} 1249251876Speter 1250251876Speterstatic void 1251251876Spetervmbus_chan_task_nobatch(void *xchan, int pending __unused) 1252251876Speter{ 1253251876Speter struct vmbus_channel *chan = xchan; 1254251876Speter 1255251876Speter KASSERT(chan->ch_poll_intvl == 0, 1256251876Speter ("chan%u: interrupted in polling mode", chan->ch_id)); 1257251876Speter chan->ch_cb(chan, chan->ch_cbarg); 1258251876Speter} 1259251876Speter 1260251876Speterstatic void 1261251876Spetervmbus_chan_poll_timeout(void *xchan) 1262251876Speter{ 1263251876Speter struct vmbus_channel *chan = xchan; 1264251876Speter 1265251876Speter KASSERT(chan->ch_poll_intvl != 0, 1266251876Speter ("chan%u: polling timeout in interrupt mode", chan->ch_id)); 1267251876Speter taskqueue_enqueue(chan->ch_tq, &chan->ch_poll_task); 1268251876Speter} 1269251876Speter 1270251876Speterstatic void 1271251876Spetervmbus_chan_poll_task(void *xchan, int pending __unused) 1272251876Speter{ 1273251876Speter struct vmbus_channel *chan = xchan; 1274251876Speter 1275251876Speter KASSERT(chan->ch_poll_intvl != 0, 1276251876Speter ("chan%u: polling in interrupt mode", chan->ch_id)); 1277251876Speter callout_reset_sbt_curcpu(&chan->ch_poll_timeo, chan->ch_poll_intvl, 0, 1278251876Speter vmbus_chan_poll_timeout, chan, chan->ch_poll_flags); 1279251876Speter chan->ch_cb(chan, chan->ch_cbarg); 1280251876Speter} 1281251876Speter 1282251876Speterstatic void 1283251876Spetervmbus_chan_pollcfg_task(void *xarg, int pending __unused) 1284251876Speter{ 1285251876Speter const struct vmbus_chan_pollarg *arg = xarg; 1286251876Speter struct vmbus_channel *chan = arg->poll_chan; 1287251876Speter sbintime_t intvl; 1288251876Speter int poll_flags; 1289251876Speter 1290251876Speter /* 1291251876Speter * Save polling interval. 1292251876Speter */ 1293251876Speter intvl = SBT_1S / arg->poll_hz; 1294251876Speter if (intvl == 0) 1295251876Speter intvl = 1; 1296251876Speter if (intvl == chan->ch_poll_intvl) { 1297251876Speter /* Nothing changes; done */ 1298251876Speter return; 1299251876Speter } 1300251876Speter chan->ch_poll_intvl = intvl; 1301251876Speter 1302251876Speter /* Adjust callout flags. */ 1303251876Speter poll_flags = C_DIRECT_EXEC; 1304251876Speter if (arg->poll_hz <= hz) 1305251876Speter poll_flags |= C_HARDCLOCK; 1306 chan->ch_poll_flags = poll_flags; 1307 1308 /* 1309 * Disconnect this channel from the channel map to make sure that 1310 * the RX bufring interrupt enabling bit can not be touched, and 1311 * ISR can not enqueue this channel task anymore. THEN, disable 1312 * interrupt from the RX bufring (TX bufring does not generate 1313 * interrupt to VM). 1314 * 1315 * NOTE: order is critical. 1316 */ 1317 chan->ch_vmbus->vmbus_chmap[chan->ch_id] = NULL; 1318 __compiler_membar(); 1319 vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1320 1321 /* 1322 * NOTE: 1323 * At this point, this channel task will not be enqueued by 1324 * the ISR anymore, time to cancel the pending one. 1325 */ 1326 taskqueue_cancel(chan->ch_tq, &chan->ch_task, NULL); 1327 1328 /* Kick start! */ 1329 taskqueue_enqueue(chan->ch_tq, &chan->ch_poll_task); 1330} 1331 1332static bool 1333vmbus_chan_poll_cancel_intq(struct vmbus_channel *chan) 1334{ 1335 1336 if (chan->ch_poll_intvl == 0) { 1337 /* Not enabled. */ 1338 return (false); 1339 } 1340 1341 /* 1342 * Stop polling callout, so that channel polling task 1343 * will not be enqueued anymore. 1344 */ 1345 callout_drain(&chan->ch_poll_timeo); 1346 1347 /* 1348 * Disable polling by resetting polling interval. 1349 * 1350 * NOTE: 1351 * The polling interval resetting MUST be conducted 1352 * after the callout is drained; mainly to keep the 1353 * proper assertion in place. 1354 */ 1355 chan->ch_poll_intvl = 0; 1356 1357 /* 1358 * NOTE: 1359 * At this point, this channel polling task will not be 1360 * enqueued by the callout anymore, time to cancel the 1361 * pending one. 1362 */ 1363 taskqueue_cancel(chan->ch_tq, &chan->ch_poll_task, NULL); 1364 1365 /* Polling was enabled. */ 1366 return (true); 1367} 1368 1369static void 1370vmbus_chan_polldis_task(void *xchan, int pending __unused) 1371{ 1372 struct vmbus_channel *chan = xchan; 1373 1374 if (!vmbus_chan_poll_cancel_intq(chan)) { 1375 /* Already disabled; done. */ 1376 return; 1377 } 1378 1379 /* 1380 * Plug this channel back to the channel map and unmask 1381 * the RX bufring interrupt. 1382 */ 1383 chan->ch_vmbus->vmbus_chmap[chan->ch_id] = chan; 1384 __compiler_membar(); 1385 vmbus_rxbr_intr_unmask(&chan->ch_rxbr); 1386 1387 /* 1388 * Kick start the interrupt task, just in case unmasking 1389 * interrupt races ISR. 1390 */ 1391 taskqueue_enqueue(chan->ch_tq, &chan->ch_task); 1392} 1393 1394static __inline void 1395vmbus_event_flags_proc(struct vmbus_softc *sc, volatile u_long *event_flags, 1396 int flag_cnt) 1397{ 1398 int f; 1399 1400 for (f = 0; f < flag_cnt; ++f) { 1401 uint32_t chid_base; 1402 u_long flags; 1403 int chid_ofs; 1404 1405 if (event_flags[f] == 0) 1406 continue; 1407 1408 flags = atomic_swap_long(&event_flags[f], 0); 1409 chid_base = f << VMBUS_EVTFLAG_SHIFT; 1410 1411 while ((chid_ofs = ffsl(flags)) != 0) { 1412 struct vmbus_channel *chan; 1413 1414 --chid_ofs; /* NOTE: ffsl is 1-based */ 1415 flags &= ~(1UL << chid_ofs); 1416 1417 chan = sc->vmbus_chmap[chid_base + chid_ofs]; 1418 if (__predict_false(chan == NULL)) { 1419 /* Channel is closed. */ 1420 continue; 1421 } 1422 __compiler_membar(); 1423 1424 if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD) 1425 vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1426 taskqueue_enqueue(chan->ch_tq, &chan->ch_task); 1427 } 1428 } 1429} 1430 1431void 1432vmbus_event_proc(struct vmbus_softc *sc, int cpu) 1433{ 1434 struct vmbus_evtflags *eventf; 1435 1436 /* 1437 * On Host with Win8 or above, the event page can be checked directly 1438 * to get the id of the channel that has the pending interrupt. 1439 */ 1440 eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE; 1441 vmbus_event_flags_proc(sc, eventf->evt_flags, 1442 VMBUS_PCPU_GET(sc, event_flags_cnt, cpu)); 1443} 1444 1445void 1446vmbus_event_proc_compat(struct vmbus_softc *sc, int cpu) 1447{ 1448 struct vmbus_evtflags *eventf; 1449 1450 eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE; 1451 if (atomic_testandclear_long(&eventf->evt_flags[0], 0)) { 1452 vmbus_event_flags_proc(sc, sc->vmbus_rx_evtflags, 1453 VMBUS_CHAN_MAX_COMPAT >> VMBUS_EVTFLAG_SHIFT); 1454 } 1455} 1456 1457static void 1458vmbus_chan_update_evtflagcnt(struct vmbus_softc *sc, 1459 const struct vmbus_channel *chan) 1460{ 1461 volatile int *flag_cnt_ptr; 1462 int flag_cnt; 1463 1464 flag_cnt = (chan->ch_id / VMBUS_EVTFLAG_LEN) + 1; 1465 flag_cnt_ptr = VMBUS_PCPU_PTR(sc, event_flags_cnt, chan->ch_cpuid); 1466 1467 for (;;) { 1468 int old_flag_cnt; 1469 1470 old_flag_cnt = *flag_cnt_ptr; 1471 if (old_flag_cnt >= flag_cnt) 1472 break; 1473 if (atomic_cmpset_int(flag_cnt_ptr, old_flag_cnt, flag_cnt)) { 1474 if (bootverbose) { 1475 vmbus_chan_printf(chan, 1476 "chan%u update cpu%d flag_cnt to %d\n", 1477 chan->ch_id, chan->ch_cpuid, flag_cnt); 1478 } 1479 break; 1480 } 1481 } 1482} 1483 1484static struct vmbus_channel * 1485vmbus_chan_alloc(struct vmbus_softc *sc) 1486{ 1487 struct vmbus_channel *chan; 1488 1489 chan = malloc(sizeof(*chan), M_DEVBUF, M_WAITOK | M_ZERO); 1490 1491 chan->ch_monprm = hyperv_dmamem_alloc(bus_get_dma_tag(sc->vmbus_dev), 1492 HYPERCALL_PARAM_ALIGN, 0, sizeof(struct hyperv_mon_param), 1493 &chan->ch_monprm_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO); 1494 if (chan->ch_monprm == NULL) { 1495 device_printf(sc->vmbus_dev, "monprm alloc failed\n"); 1496 free(chan, M_DEVBUF); 1497 return NULL; 1498 } 1499 1500 chan->ch_refs = 1; 1501 chan->ch_vmbus = sc; 1502 mtx_init(&chan->ch_subchan_lock, "vmbus subchan", NULL, MTX_DEF); 1503 sx_init(&chan->ch_orphan_lock, "vmbus chorphan"); 1504 TAILQ_INIT(&chan->ch_subchans); 1505 vmbus_rxbr_init(&chan->ch_rxbr); 1506 vmbus_txbr_init(&chan->ch_txbr); 1507 1508 TASK_INIT(&chan->ch_poll_task, 0, vmbus_chan_poll_task, chan); 1509 callout_init(&chan->ch_poll_timeo, 1); 1510 1511 return chan; 1512} 1513 1514static void 1515vmbus_chan_free(struct vmbus_channel *chan) 1516{ 1517 1518 KASSERT(TAILQ_EMPTY(&chan->ch_subchans) && chan->ch_subchan_cnt == 0, 1519 ("still owns sub-channels")); 1520 KASSERT((chan->ch_stflags & 1521 (VMBUS_CHAN_ST_OPENED | 1522 VMBUS_CHAN_ST_ONPRIL | 1523 VMBUS_CHAN_ST_ONSUBL | 1524 VMBUS_CHAN_ST_ONLIST)) == 0, ("free busy channel")); 1525 KASSERT(chan->ch_orphan_xact == NULL, 1526 ("still has orphan xact installed")); 1527 KASSERT(chan->ch_refs == 0, ("chan%u: invalid refcnt %d", 1528 chan->ch_id, chan->ch_refs)); 1529 KASSERT(chan->ch_poll_intvl == 0, ("chan%u: polling is activated", 1530 chan->ch_id)); 1531 1532 hyperv_dmamem_free(&chan->ch_monprm_dma, chan->ch_monprm); 1533 mtx_destroy(&chan->ch_subchan_lock); 1534 sx_destroy(&chan->ch_orphan_lock); 1535 vmbus_rxbr_deinit(&chan->ch_rxbr); 1536 vmbus_txbr_deinit(&chan->ch_txbr); 1537 free(chan, M_DEVBUF); 1538} 1539 1540static int 1541vmbus_chan_add(struct vmbus_channel *newchan) 1542{ 1543 struct vmbus_softc *sc = newchan->ch_vmbus; 1544 struct vmbus_channel *prichan; 1545 1546 if (newchan->ch_id == 0) { 1547 /* 1548 * XXX 1549 * Chan0 will neither be processed nor should be offered; 1550 * skip it. 1551 */ 1552 device_printf(sc->vmbus_dev, "got chan0 offer, discard\n"); 1553 return EINVAL; 1554 } else if (newchan->ch_id >= VMBUS_CHAN_MAX) { 1555 device_printf(sc->vmbus_dev, "invalid chan%u offer\n", 1556 newchan->ch_id); 1557 return EINVAL; 1558 } 1559 1560 mtx_lock(&sc->vmbus_prichan_lock); 1561 TAILQ_FOREACH(prichan, &sc->vmbus_prichans, ch_prilink) { 1562 /* 1563 * Sub-channel will have the same type GUID and instance 1564 * GUID as its primary channel. 1565 */ 1566 if (memcmp(&prichan->ch_guid_type, &newchan->ch_guid_type, 1567 sizeof(struct hyperv_guid)) == 0 && 1568 memcmp(&prichan->ch_guid_inst, &newchan->ch_guid_inst, 1569 sizeof(struct hyperv_guid)) == 0) 1570 break; 1571 } 1572 if (VMBUS_CHAN_ISPRIMARY(newchan)) { 1573 if (prichan == NULL) { 1574 /* Install the new primary channel */ 1575 vmbus_chan_ins_prilist(sc, newchan); 1576 mtx_unlock(&sc->vmbus_prichan_lock); 1577 goto done; 1578 } else { 1579 mtx_unlock(&sc->vmbus_prichan_lock); 1580 device_printf(sc->vmbus_dev, 1581 "duplicated primary chan%u\n", newchan->ch_id); 1582 return EINVAL; 1583 } 1584 } else { /* Sub-channel */ 1585 if (prichan == NULL) { 1586 mtx_unlock(&sc->vmbus_prichan_lock); 1587 device_printf(sc->vmbus_dev, 1588 "no primary chan for chan%u\n", newchan->ch_id); 1589 return EINVAL; 1590 } 1591 /* 1592 * Found the primary channel for this sub-channel and 1593 * move on. 1594 * 1595 * XXX refcnt prichan 1596 */ 1597 } 1598 mtx_unlock(&sc->vmbus_prichan_lock); 1599 1600 /* 1601 * This is a sub-channel; link it with the primary channel. 1602 */ 1603 KASSERT(!VMBUS_CHAN_ISPRIMARY(newchan), 1604 ("new channel is not sub-channel")); 1605 KASSERT(prichan != NULL, ("no primary channel")); 1606 1607 /* 1608 * Reference count this sub-channel; it will be dereferenced 1609 * when this sub-channel is closed. 1610 */ 1611 KASSERT(newchan->ch_refs == 1, ("chan%u: invalid refcnt %d", 1612 newchan->ch_id, newchan->ch_refs)); 1613 atomic_add_int(&newchan->ch_refs, 1); 1614 1615 newchan->ch_prichan = prichan; 1616 newchan->ch_dev = prichan->ch_dev; 1617 1618 mtx_lock(&prichan->ch_subchan_lock); 1619 vmbus_chan_ins_sublist(prichan, newchan); 1620 mtx_unlock(&prichan->ch_subchan_lock); 1621 /* 1622 * Notify anyone that is interested in this sub-channel, 1623 * after this sub-channel is setup. 1624 */ 1625 wakeup(prichan); 1626done: 1627 /* 1628 * Hook this channel up for later revocation. 1629 */ 1630 mtx_lock(&sc->vmbus_chan_lock); 1631 vmbus_chan_ins_list(sc, newchan); 1632 mtx_unlock(&sc->vmbus_chan_lock); 1633 1634 if (bootverbose) { 1635 vmbus_chan_printf(newchan, "chan%u subidx%u offer\n", 1636 newchan->ch_id, newchan->ch_subidx); 1637 } 1638 1639 /* Select default cpu for this channel. */ 1640 vmbus_chan_cpu_default(newchan); 1641 1642 return 0; 1643} 1644 1645void 1646vmbus_chan_cpu_set(struct vmbus_channel *chan, int cpu) 1647{ 1648 KASSERT(cpu >= 0 && cpu < mp_ncpus, ("invalid cpu %d", cpu)); 1649 1650 if (chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WS2008 || 1651 chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WIN7) { 1652 /* Only cpu0 is supported */ 1653 cpu = 0; 1654 } 1655 1656 chan->ch_cpuid = cpu; 1657 chan->ch_vcpuid = VMBUS_PCPU_GET(chan->ch_vmbus, vcpuid, cpu); 1658 1659 if (bootverbose) { 1660 vmbus_chan_printf(chan, 1661 "chan%u assigned to cpu%u [vcpu%u]\n", 1662 chan->ch_id, chan->ch_cpuid, chan->ch_vcpuid); 1663 } 1664} 1665 1666void 1667vmbus_chan_cpu_rr(struct vmbus_channel *chan) 1668{ 1669 static uint32_t vmbus_chan_nextcpu; 1670 int cpu; 1671 1672 cpu = atomic_fetchadd_int(&vmbus_chan_nextcpu, 1) % mp_ncpus; 1673 vmbus_chan_cpu_set(chan, cpu); 1674} 1675 1676static void 1677vmbus_chan_cpu_default(struct vmbus_channel *chan) 1678{ 1679 /* 1680 * By default, pin the channel to cpu0. Devices having 1681 * special channel-cpu mapping requirement should call 1682 * vmbus_chan_cpu_{set,rr}(). 1683 */ 1684 vmbus_chan_cpu_set(chan, 0); 1685} 1686 1687static void 1688vmbus_chan_msgproc_choffer(struct vmbus_softc *sc, 1689 const struct vmbus_message *msg) 1690{ 1691 const struct vmbus_chanmsg_choffer *offer; 1692 struct vmbus_channel *chan; 1693 task_fn_t *detach_fn, *attach_fn; 1694 int error; 1695 1696 offer = (const struct vmbus_chanmsg_choffer *)msg->msg_data; 1697 1698 chan = vmbus_chan_alloc(sc); 1699 if (chan == NULL) { 1700 device_printf(sc->vmbus_dev, "allocate chan%u failed\n", 1701 offer->chm_chanid); 1702 return; 1703 } 1704 1705 chan->ch_id = offer->chm_chanid; 1706 chan->ch_subidx = offer->chm_subidx; 1707 chan->ch_guid_type = offer->chm_chtype; 1708 chan->ch_guid_inst = offer->chm_chinst; 1709 1710 /* Batch reading is on by default */ 1711 chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD; 1712 1713 chan->ch_monprm->mp_connid = VMBUS_CONNID_EVENT; 1714 if (sc->vmbus_version != VMBUS_VERSION_WS2008) 1715 chan->ch_monprm->mp_connid = offer->chm_connid; 1716 1717 if (offer->chm_flags1 & VMBUS_CHOFFER_FLAG1_HASMNF) { 1718 int trig_idx; 1719 1720 /* 1721 * Setup MNF stuffs. 1722 */ 1723 chan->ch_txflags |= VMBUS_CHAN_TXF_HASMNF; 1724 1725 trig_idx = offer->chm_montrig / VMBUS_MONTRIG_LEN; 1726 if (trig_idx >= VMBUS_MONTRIGS_MAX) 1727 panic("invalid monitor trigger %u", offer->chm_montrig); 1728 chan->ch_montrig = 1729 &sc->vmbus_mnf2->mnf_trigs[trig_idx].mt_pending; 1730 1731 chan->ch_montrig_mask = 1732 1 << (offer->chm_montrig % VMBUS_MONTRIG_LEN); 1733 } 1734 1735 /* 1736 * Setup event flag. 1737 */ 1738 chan->ch_evtflag = 1739 &sc->vmbus_tx_evtflags[chan->ch_id >> VMBUS_EVTFLAG_SHIFT]; 1740 chan->ch_evtflag_mask = 1UL << (chan->ch_id & VMBUS_EVTFLAG_MASK); 1741 1742 /* 1743 * Setup attach and detach tasks. 1744 */ 1745 if (VMBUS_CHAN_ISPRIMARY(chan)) { 1746 chan->ch_mgmt_tq = sc->vmbus_devtq; 1747 attach_fn = vmbus_prichan_attach_task; 1748 detach_fn = vmbus_prichan_detach_task; 1749 } else { 1750 chan->ch_mgmt_tq = sc->vmbus_subchtq; 1751 attach_fn = vmbus_subchan_attach_task; 1752 detach_fn = vmbus_subchan_detach_task; 1753 } 1754 TASK_INIT(&chan->ch_attach_task, 0, attach_fn, chan); 1755 TASK_INIT(&chan->ch_detach_task, 0, detach_fn, chan); 1756 1757 error = vmbus_chan_add(chan); 1758 if (error) { 1759 device_printf(sc->vmbus_dev, "add chan%u failed: %d\n", 1760 chan->ch_id, error); 1761 atomic_subtract_int(&chan->ch_refs, 1); 1762 vmbus_chan_free(chan); 1763 return; 1764 } 1765 taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_attach_task); 1766} 1767 1768static void 1769vmbus_chan_msgproc_chrescind(struct vmbus_softc *sc, 1770 const struct vmbus_message *msg) 1771{ 1772 const struct vmbus_chanmsg_chrescind *note; 1773 struct vmbus_channel *chan; 1774 1775 note = (const struct vmbus_chanmsg_chrescind *)msg->msg_data; 1776 if (note->chm_chanid > VMBUS_CHAN_MAX) { 1777 device_printf(sc->vmbus_dev, "invalid revoked chan%u\n", 1778 note->chm_chanid); 1779 return; 1780 } 1781 1782 /* 1783 * Find and remove the target channel from the channel list. 1784 */ 1785 mtx_lock(&sc->vmbus_chan_lock); 1786 TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) { 1787 if (chan->ch_id == note->chm_chanid) 1788 break; 1789 } 1790 if (chan == NULL) { 1791 mtx_unlock(&sc->vmbus_chan_lock); 1792 device_printf(sc->vmbus_dev, "chan%u is not offered\n", 1793 note->chm_chanid); 1794 return; 1795 } 1796 vmbus_chan_rem_list(sc, chan); 1797 mtx_unlock(&sc->vmbus_chan_lock); 1798 1799 if (VMBUS_CHAN_ISPRIMARY(chan)) { 1800 /* 1801 * The target channel is a primary channel; remove the 1802 * target channel from the primary channel list now, 1803 * instead of later, so that it will not be found by 1804 * other sub-channel offers, which are processed in 1805 * this thread. 1806 */ 1807 mtx_lock(&sc->vmbus_prichan_lock); 1808 vmbus_chan_rem_prilist(sc, chan); 1809 mtx_unlock(&sc->vmbus_prichan_lock); 1810 } 1811 1812 /* 1813 * NOTE: 1814 * The following processing order is critical: 1815 * Set the REVOKED state flag before orphaning the installed xact. 1816 */ 1817 1818 if (atomic_testandset_int(&chan->ch_stflags, 1819 VMBUS_CHAN_ST_REVOKED_SHIFT)) 1820 panic("channel has already been revoked"); 1821 1822 sx_xlock(&chan->ch_orphan_lock); 1823 if (chan->ch_orphan_xact != NULL) 1824 vmbus_xact_ctx_orphan(chan->ch_orphan_xact); 1825 sx_xunlock(&chan->ch_orphan_lock); 1826 1827 if (bootverbose) 1828 vmbus_chan_printf(chan, "chan%u revoked\n", note->chm_chanid); 1829 vmbus_chan_detach(chan); 1830} 1831 1832static int 1833vmbus_chan_release(struct vmbus_channel *chan) 1834{ 1835 struct vmbus_softc *sc = chan->ch_vmbus; 1836 struct vmbus_chanmsg_chfree *req; 1837 struct vmbus_msghc *mh; 1838 int error; 1839 1840 mh = vmbus_msghc_get(sc, sizeof(*req)); 1841 if (mh == NULL) { 1842 vmbus_chan_printf(chan, 1843 "can not get msg hypercall for chfree(chan%u)\n", 1844 chan->ch_id); 1845 return (ENXIO); 1846 } 1847 1848 req = vmbus_msghc_dataptr(mh); 1849 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHFREE; 1850 req->chm_chanid = chan->ch_id; 1851 1852 error = vmbus_msghc_exec_noresult(mh); 1853 vmbus_msghc_put(sc, mh); 1854 1855 if (error) { 1856 vmbus_chan_printf(chan, 1857 "chfree(chan%u) msg hypercall exec failed: %d\n", 1858 chan->ch_id, error); 1859 } else { 1860 if (bootverbose) 1861 vmbus_chan_printf(chan, "chan%u freed\n", chan->ch_id); 1862 } 1863 return (error); 1864} 1865 1866static void 1867vmbus_prichan_detach_task(void *xchan, int pending __unused) 1868{ 1869 struct vmbus_channel *chan = xchan; 1870 1871 KASSERT(VMBUS_CHAN_ISPRIMARY(chan), 1872 ("chan%u is not primary channel", chan->ch_id)); 1873 1874 /* Delete and detach the device associated with this channel. */ 1875 vmbus_delete_child(chan); 1876 1877 /* Release this channel (back to vmbus). */ 1878 vmbus_chan_release(chan); 1879 1880 /* Free this channel's resource. */ 1881 vmbus_chan_free(chan); 1882} 1883 1884static void 1885vmbus_subchan_detach_task(void *xchan, int pending __unused) 1886{ 1887 struct vmbus_channel *chan = xchan; 1888 struct vmbus_channel *pri_chan = chan->ch_prichan; 1889 1890 KASSERT(!VMBUS_CHAN_ISPRIMARY(chan), 1891 ("chan%u is primary channel", chan->ch_id)); 1892 1893 /* Release this channel (back to vmbus). */ 1894 vmbus_chan_release(chan); 1895 1896 /* Unlink from its primary channel's sub-channel list. */ 1897 mtx_lock(&pri_chan->ch_subchan_lock); 1898 vmbus_chan_rem_sublist(pri_chan, chan); 1899 mtx_unlock(&pri_chan->ch_subchan_lock); 1900 /* Notify anyone that is waiting for this sub-channel to vanish. */ 1901 wakeup(pri_chan); 1902 1903 /* Free this channel's resource. */ 1904 vmbus_chan_free(chan); 1905} 1906 1907static void 1908vmbus_prichan_attach_task(void *xchan, int pending __unused) 1909{ 1910 1911 /* 1912 * Add device for this primary channel. 1913 */ 1914 vmbus_add_child(xchan); 1915} 1916 1917static void 1918vmbus_subchan_attach_task(void *xchan __unused, int pending __unused) 1919{ 1920 1921 /* Nothing */ 1922} 1923 1924void 1925vmbus_chan_destroy_all(struct vmbus_softc *sc) 1926{ 1927 1928 /* 1929 * Detach all devices and destroy the corresponding primary 1930 * channels. 1931 */ 1932 for (;;) { 1933 struct vmbus_channel *chan; 1934 1935 mtx_lock(&sc->vmbus_chan_lock); 1936 TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) { 1937 if (VMBUS_CHAN_ISPRIMARY(chan)) 1938 break; 1939 } 1940 if (chan == NULL) { 1941 /* No more primary channels; done. */ 1942 mtx_unlock(&sc->vmbus_chan_lock); 1943 break; 1944 } 1945 vmbus_chan_rem_list(sc, chan); 1946 mtx_unlock(&sc->vmbus_chan_lock); 1947 1948 mtx_lock(&sc->vmbus_prichan_lock); 1949 vmbus_chan_rem_prilist(sc, chan); 1950 mtx_unlock(&sc->vmbus_prichan_lock); 1951 1952 taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task); 1953 } 1954} 1955 1956struct vmbus_channel ** 1957vmbus_subchan_get(struct vmbus_channel *pri_chan, int subchan_cnt) 1958{ 1959 struct vmbus_channel **ret, *chan; 1960 int i; 1961 1962 KASSERT(subchan_cnt > 0, ("invalid sub-channel count %d", subchan_cnt)); 1963 1964 ret = malloc(subchan_cnt * sizeof(struct vmbus_channel *), M_TEMP, 1965 M_WAITOK); 1966 1967 mtx_lock(&pri_chan->ch_subchan_lock); 1968 1969 while (pri_chan->ch_subchan_cnt < subchan_cnt) 1970 mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "subch", 0); 1971 1972 i = 0; 1973 TAILQ_FOREACH(chan, &pri_chan->ch_subchans, ch_sublink) { 1974 /* TODO: refcnt chan */ 1975 ret[i] = chan; 1976 1977 ++i; 1978 if (i == subchan_cnt) 1979 break; 1980 } 1981 KASSERT(i == subchan_cnt, ("invalid subchan count %d, should be %d", 1982 pri_chan->ch_subchan_cnt, subchan_cnt)); 1983 1984 mtx_unlock(&pri_chan->ch_subchan_lock); 1985 1986 return ret; 1987} 1988 1989void 1990vmbus_subchan_rel(struct vmbus_channel **subchan, int subchan_cnt __unused) 1991{ 1992 1993 free(subchan, M_TEMP); 1994} 1995 1996void 1997vmbus_subchan_drain(struct vmbus_channel *pri_chan) 1998{ 1999 mtx_lock(&pri_chan->ch_subchan_lock); 2000 while (pri_chan->ch_subchan_cnt > 0) 2001 mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "dsubch", 0); 2002 mtx_unlock(&pri_chan->ch_subchan_lock); 2003} 2004 2005void 2006vmbus_chan_msgproc(struct vmbus_softc *sc, const struct vmbus_message *msg) 2007{ 2008 vmbus_chanmsg_proc_t msg_proc; 2009 uint32_t msg_type; 2010 2011 msg_type = ((const struct vmbus_chanmsg_hdr *)msg->msg_data)->chm_type; 2012 KASSERT(msg_type < VMBUS_CHANMSG_TYPE_MAX, 2013 ("invalid message type %u", msg_type)); 2014 2015 msg_proc = vmbus_chan_msgprocs[msg_type]; 2016 if (msg_proc != NULL) 2017 msg_proc(sc, msg); 2018} 2019 2020void 2021vmbus_chan_set_readbatch(struct vmbus_channel *chan, bool on) 2022{ 2023 if (!on) 2024 chan->ch_flags &= ~VMBUS_CHAN_FLAG_BATCHREAD; 2025 else 2026 chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD; 2027} 2028 2029uint32_t 2030vmbus_chan_id(const struct vmbus_channel *chan) 2031{ 2032 return chan->ch_id; 2033} 2034 2035uint32_t 2036vmbus_chan_subidx(const struct vmbus_channel *chan) 2037{ 2038 return chan->ch_subidx; 2039} 2040 2041bool 2042vmbus_chan_is_primary(const struct vmbus_channel *chan) 2043{ 2044 if (VMBUS_CHAN_ISPRIMARY(chan)) 2045 return true; 2046 else 2047 return false; 2048} 2049 2050const struct hyperv_guid * 2051vmbus_chan_guid_inst(const struct vmbus_channel *chan) 2052{ 2053 return &chan->ch_guid_inst; 2054} 2055 2056int 2057vmbus_chan_prplist_nelem(int br_size, int prpcnt_max, int dlen_max) 2058{ 2059 int elem_size; 2060 2061 elem_size = __offsetof(struct vmbus_chanpkt_prplist, 2062 cp_range[0].gpa_page[prpcnt_max]); 2063 elem_size += dlen_max; 2064 elem_size = VMBUS_CHANPKT_TOTLEN(elem_size); 2065 2066 return (vmbus_br_nelem(br_size, elem_size)); 2067} 2068 2069bool 2070vmbus_chan_tx_empty(const struct vmbus_channel *chan) 2071{ 2072 2073 return (vmbus_txbr_empty(&chan->ch_txbr)); 2074} 2075 2076bool 2077vmbus_chan_rx_empty(const struct vmbus_channel *chan) 2078{ 2079 2080 return (vmbus_rxbr_empty(&chan->ch_rxbr)); 2081} 2082 2083static int 2084vmbus_chan_printf(const struct vmbus_channel *chan, const char *fmt, ...) 2085{ 2086 va_list ap; 2087 device_t dev; 2088 int retval; 2089 2090 if (chan->ch_dev == NULL || !device_is_alive(chan->ch_dev)) 2091 dev = chan->ch_vmbus->vmbus_dev; 2092 else 2093 dev = chan->ch_dev; 2094 2095 retval = device_print_prettyname(dev); 2096 va_start(ap, fmt); 2097 retval += vprintf(fmt, ap); 2098 va_end(ap); 2099 2100 return (retval); 2101} 2102 2103void 2104vmbus_chan_run_task(struct vmbus_channel *chan, struct task *task) 2105{ 2106 2107 taskqueue_enqueue(chan->ch_tq, task); 2108 taskqueue_drain(chan->ch_tq, task); 2109} 2110 2111struct taskqueue * 2112vmbus_chan_mgmt_tq(const struct vmbus_channel *chan) 2113{ 2114 2115 return (chan->ch_mgmt_tq); 2116} 2117 2118bool 2119vmbus_chan_is_revoked(const struct vmbus_channel *chan) 2120{ 2121 2122 if (chan->ch_stflags & VMBUS_CHAN_ST_REVOKED) 2123 return (true); 2124 return (false); 2125} 2126 2127void 2128vmbus_chan_set_orphan(struct vmbus_channel *chan, struct vmbus_xact_ctx *xact) 2129{ 2130 2131 sx_xlock(&chan->ch_orphan_lock); 2132 chan->ch_orphan_xact = xact; 2133 sx_xunlock(&chan->ch_orphan_lock); 2134} 2135 2136void 2137vmbus_chan_unset_orphan(struct vmbus_channel *chan) 2138{ 2139 2140 sx_xlock(&chan->ch_orphan_lock); 2141 chan->ch_orphan_xact = NULL; 2142 sx_xunlock(&chan->ch_orphan_lock); 2143} 2144 2145const void * 2146vmbus_chan_xact_wait(const struct vmbus_channel *chan, 2147 struct vmbus_xact *xact, size_t *resp_len, bool can_sleep) 2148{ 2149 const void *ret; 2150 2151 if (can_sleep) 2152 ret = vmbus_xact_wait(xact, resp_len); 2153 else 2154 ret = vmbus_xact_busywait(xact, resp_len); 2155 if (vmbus_chan_is_revoked(chan)) { 2156 /* 2157 * This xact probably is interrupted, and the 2158 * interruption can race the reply reception, 2159 * so we have to make sure that there are nothing 2160 * left on the RX bufring, i.e. this xact will 2161 * not be touched, once this function returns. 2162 * 2163 * Since the hypervisor will not put more data 2164 * onto the RX bufring once the channel is revoked, 2165 * the following loop will be terminated, once all 2166 * data are drained by the driver's channel 2167 * callback. 2168 */ 2169 while (!vmbus_chan_rx_empty(chan)) { 2170 if (can_sleep) 2171 pause("chxact", 1); 2172 else 2173 DELAY(1000); 2174 } 2175 } 2176 return (ret); 2177} 2178 2179void 2180vmbus_chan_poll_enable(struct vmbus_channel *chan, u_int pollhz) 2181{ 2182 struct vmbus_chan_pollarg arg; 2183 struct task poll_cfg; 2184 2185 KASSERT(chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD, 2186 ("enable polling on non-batch chan%u", chan->ch_id)); 2187 KASSERT(pollhz >= VMBUS_CHAN_POLLHZ_MIN && 2188 pollhz <= VMBUS_CHAN_POLLHZ_MAX, ("invalid pollhz %u", pollhz)); 2189 2190 arg.poll_chan = chan; 2191 arg.poll_hz = pollhz; 2192 TASK_INIT(&poll_cfg, 0, vmbus_chan_pollcfg_task, &arg); 2193 vmbus_chan_run_task(chan, &poll_cfg); 2194} 2195 2196void 2197vmbus_chan_poll_disable(struct vmbus_channel *chan) 2198{ 2199 struct task poll_dis; 2200 2201 KASSERT(chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD, 2202 ("disable polling on non-batch chan%u", chan->ch_id)); 2203 2204 TASK_INIT(&poll_dis, 0, vmbus_chan_polldis_task, chan); 2205 vmbus_chan_run_task(chan, &poll_dis); 2206} 2207