1250199Sgrehan/*- 2298446Ssephe * Copyright (c) 2009-2012,2016 Microsoft Corp. 3250199Sgrehan * Copyright (c) 2012 NetApp Inc. 4250199Sgrehan * Copyright (c) 2012 Citrix Inc. 5250199Sgrehan * All rights reserved. 6250199Sgrehan * 7250199Sgrehan * Redistribution and use in source and binary forms, with or without 8250199Sgrehan * modification, are permitted provided that the following conditions 9250199Sgrehan * are met: 10250199Sgrehan * 1. Redistributions of source code must retain the above copyright 11250199Sgrehan * notice unmodified, this list of conditions, and the following 12250199Sgrehan * disclaimer. 13250199Sgrehan * 2. Redistributions in binary form must reproduce the above copyright 14250199Sgrehan * notice, this list of conditions and the following disclaimer in the 15250199Sgrehan * documentation and/or other materials provided with the distribution. 16250199Sgrehan * 17250199Sgrehan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18250199Sgrehan * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19250199Sgrehan * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20250199Sgrehan * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21250199Sgrehan * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22250199Sgrehan * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23250199Sgrehan * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24250199Sgrehan * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25250199Sgrehan * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26250199Sgrehan * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27250199Sgrehan */ 28250199Sgrehan 29256276Sdim#include <sys/cdefs.h> 30256276Sdim__FBSDID("$FreeBSD: stable/11/sys/dev/hyperv/vmbus/vmbus_chan.c 311389 2017-01-05 08:42:58Z sephe $"); 31256276Sdim 32250199Sgrehan#include <sys/param.h> 33308621Ssephe#include <sys/bus.h> 34311375Ssephe#include <sys/callout.h> 35296028Ssephe#include <sys/kernel.h> 36307466Ssephe#include <sys/lock.h> 37250199Sgrehan#include <sys/malloc.h> 38250199Sgrehan#include <sys/mutex.h> 39307466Ssephe#include <sys/smp.h> 40296181Ssephe#include <sys/sysctl.h> 41307466Ssephe#include <sys/systm.h> 42301588Ssephe 43301588Ssephe#include <machine/atomic.h> 44308621Ssephe#include <machine/stdarg.h> 45301588Ssephe 46302872Ssephe#include <dev/hyperv/include/hyperv_busdma.h> 47311359Ssephe#include <dev/hyperv/include/vmbus_xact.h> 48302619Ssephe#include <dev/hyperv/vmbus/hyperv_var.h> 49301588Ssephe#include <dev/hyperv/vmbus/vmbus_reg.h> 50300102Ssephe#include <dev/hyperv/vmbus/vmbus_var.h> 51307463Ssephe#include <dev/hyperv/vmbus/vmbus_brvar.h> 52307463Ssephe#include <dev/hyperv/vmbus/vmbus_chanvar.h> 53250199Sgrehan 54311375Ssephestruct vmbus_chan_pollarg { 55311375Ssephe struct vmbus_channel *poll_chan; 56311375Ssephe u_int poll_hz; 57311375Ssephe}; 58311375Ssephe 59307466Ssephestatic void vmbus_chan_update_evtflagcnt( 60307466Ssephe struct vmbus_softc *, 61307466Ssephe const struct vmbus_channel *); 62311364Ssephestatic int vmbus_chan_close_internal( 63307466Ssephe struct vmbus_channel *); 64307466Ssephestatic int vmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS); 65307466Ssephestatic void vmbus_chan_sysctl_create( 66307466Ssephe struct vmbus_channel *); 67307466Ssephestatic struct vmbus_channel *vmbus_chan_alloc(struct vmbus_softc *); 68307466Ssephestatic void vmbus_chan_free(struct vmbus_channel *); 69307466Ssephestatic int vmbus_chan_add(struct vmbus_channel *); 70307466Ssephestatic void vmbus_chan_cpu_default(struct vmbus_channel *); 71307599Ssephestatic int vmbus_chan_release(struct vmbus_channel *); 72307599Ssephestatic void vmbus_chan_set_chmap(struct vmbus_channel *); 73307599Ssephestatic void vmbus_chan_clear_chmap(struct vmbus_channel *); 74311359Ssephestatic void vmbus_chan_detach(struct vmbus_channel *); 75311364Ssephestatic bool vmbus_chan_wait_revoke( 76311367Ssephe const struct vmbus_channel *, bool); 77311375Ssephestatic void vmbus_chan_poll_timeout(void *); 78311375Ssephestatic bool vmbus_chan_poll_cancel_intq( 79311375Ssephe struct vmbus_channel *); 80311375Ssephestatic void vmbus_chan_poll_cancel(struct vmbus_channel *); 81302864Ssephe 82307599Ssephestatic void vmbus_chan_ins_prilist(struct vmbus_softc *, 83307599Ssephe struct vmbus_channel *); 84307599Ssephestatic void vmbus_chan_rem_prilist(struct vmbus_softc *, 85307599Ssephe struct vmbus_channel *); 86307599Ssephestatic void vmbus_chan_ins_list(struct vmbus_softc *, 87307599Ssephe struct vmbus_channel *); 88307599Ssephestatic void vmbus_chan_rem_list(struct vmbus_softc *, 89307599Ssephe struct vmbus_channel *); 90307599Ssephestatic void vmbus_chan_ins_sublist(struct vmbus_channel *, 91307599Ssephe struct vmbus_channel *); 92307599Ssephestatic void vmbus_chan_rem_sublist(struct vmbus_channel *, 93307599Ssephe struct vmbus_channel *); 94307599Ssephe 95307466Ssephestatic void vmbus_chan_task(void *, int); 96307466Ssephestatic void vmbus_chan_task_nobatch(void *, int); 97311375Ssephestatic void vmbus_chan_poll_task(void *, int); 98307599Ssephestatic void vmbus_chan_clrchmap_task(void *, int); 99311375Ssephestatic void vmbus_chan_pollcfg_task(void *, int); 100311375Ssephestatic void vmbus_chan_polldis_task(void *, int); 101311375Ssephestatic void vmbus_chan_poll_cancel_task(void *, int); 102307599Ssephestatic void vmbus_prichan_attach_task(void *, int); 103307599Ssephestatic void vmbus_subchan_attach_task(void *, int); 104307599Ssephestatic void vmbus_prichan_detach_task(void *, int); 105307599Ssephestatic void vmbus_subchan_detach_task(void *, int); 106250199Sgrehan 107307466Ssephestatic void vmbus_chan_msgproc_choffer(struct vmbus_softc *, 108307466Ssephe const struct vmbus_message *); 109307466Ssephestatic void vmbus_chan_msgproc_chrescind( 110307466Ssephe struct vmbus_softc *, 111307466Ssephe const struct vmbus_message *); 112302864Ssephe 113308621Ssephestatic int vmbus_chan_printf(const struct vmbus_channel *, 114308621Ssephe const char *, ...) __printflike(2, 3); 115308621Ssephe 116302864Ssephe/* 117302864Ssephe * Vmbus channel message processing. 118302864Ssephe */ 119302864Ssephestatic const vmbus_chanmsg_proc_t 120302864Ssephevmbus_chan_msgprocs[VMBUS_CHANMSG_TYPE_MAX] = { 121302864Ssephe VMBUS_CHANMSG_PROC(CHOFFER, vmbus_chan_msgproc_choffer), 122302864Ssephe VMBUS_CHANMSG_PROC(CHRESCIND, vmbus_chan_msgproc_chrescind), 123302864Ssephe 124302864Ssephe VMBUS_CHANMSG_PROC_WAKEUP(CHOPEN_RESP), 125302864Ssephe VMBUS_CHANMSG_PROC_WAKEUP(GPADL_CONNRESP), 126302864Ssephe VMBUS_CHANMSG_PROC_WAKEUP(GPADL_DISCONNRESP) 127302864Ssephe}; 128302864Ssephe 129307461Ssephe/* 130307461Ssephe * Notify host that there are data pending on our TX bufring. 131250199Sgrehan */ 132307461Ssephestatic __inline void 133307461Ssephevmbus_chan_signal_tx(const struct vmbus_channel *chan) 134250199Sgrehan{ 135307461Ssephe atomic_set_long(chan->ch_evtflag, chan->ch_evtflag_mask); 136307461Ssephe if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF) 137307461Ssephe atomic_set_int(chan->ch_montrig, chan->ch_montrig_mask); 138307461Ssephe else 139303022Ssephe hypercall_signal_event(chan->ch_monprm_dma.hv_paddr); 140250199Sgrehan} 141250199Sgrehan 142307599Ssephestatic void 143307599Ssephevmbus_chan_ins_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan) 144307599Ssephe{ 145307599Ssephe 146307599Ssephe mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED); 147307599Ssephe if (atomic_testandset_int(&chan->ch_stflags, 148307599Ssephe VMBUS_CHAN_ST_ONPRIL_SHIFT)) 149307599Ssephe panic("channel is already on the prilist"); 150307599Ssephe TAILQ_INSERT_TAIL(&sc->vmbus_prichans, chan, ch_prilink); 151307599Ssephe} 152307599Ssephe 153307599Ssephestatic void 154307599Ssephevmbus_chan_rem_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan) 155307599Ssephe{ 156307599Ssephe 157307599Ssephe mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED); 158307599Ssephe if (atomic_testandclear_int(&chan->ch_stflags, 159307599Ssephe VMBUS_CHAN_ST_ONPRIL_SHIFT) == 0) 160307599Ssephe panic("channel is not on the prilist"); 161307599Ssephe TAILQ_REMOVE(&sc->vmbus_prichans, chan, ch_prilink); 162307599Ssephe} 163307599Ssephe 164307599Ssephestatic void 165307599Ssephevmbus_chan_ins_sublist(struct vmbus_channel *prichan, 166307599Ssephe struct vmbus_channel *chan) 167307599Ssephe{ 168307599Ssephe 169307599Ssephe mtx_assert(&prichan->ch_subchan_lock, MA_OWNED); 170307599Ssephe 171307599Ssephe if (atomic_testandset_int(&chan->ch_stflags, 172307599Ssephe VMBUS_CHAN_ST_ONSUBL_SHIFT)) 173307599Ssephe panic("channel is already on the sublist"); 174307599Ssephe TAILQ_INSERT_TAIL(&prichan->ch_subchans, chan, ch_sublink); 175307599Ssephe 176307599Ssephe /* Bump sub-channel count. */ 177307599Ssephe prichan->ch_subchan_cnt++; 178307599Ssephe} 179307599Ssephe 180307599Ssephestatic void 181307599Ssephevmbus_chan_rem_sublist(struct vmbus_channel *prichan, 182307599Ssephe struct vmbus_channel *chan) 183307599Ssephe{ 184307599Ssephe 185307599Ssephe mtx_assert(&prichan->ch_subchan_lock, MA_OWNED); 186307599Ssephe 187307599Ssephe KASSERT(prichan->ch_subchan_cnt > 0, 188307599Ssephe ("invalid subchan_cnt %d", prichan->ch_subchan_cnt)); 189307599Ssephe prichan->ch_subchan_cnt--; 190307599Ssephe 191307599Ssephe if (atomic_testandclear_int(&chan->ch_stflags, 192307599Ssephe VMBUS_CHAN_ST_ONSUBL_SHIFT) == 0) 193307599Ssephe panic("channel is not on the sublist"); 194307599Ssephe TAILQ_REMOVE(&prichan->ch_subchans, chan, ch_sublink); 195307599Ssephe} 196307599Ssephe 197307599Ssephestatic void 198307599Ssephevmbus_chan_ins_list(struct vmbus_softc *sc, struct vmbus_channel *chan) 199307599Ssephe{ 200307599Ssephe 201307599Ssephe mtx_assert(&sc->vmbus_chan_lock, MA_OWNED); 202307599Ssephe if (atomic_testandset_int(&chan->ch_stflags, 203307599Ssephe VMBUS_CHAN_ST_ONLIST_SHIFT)) 204307599Ssephe panic("channel is already on the list"); 205307599Ssephe TAILQ_INSERT_TAIL(&sc->vmbus_chans, chan, ch_link); 206307599Ssephe} 207307599Ssephe 208307599Ssephestatic void 209307599Ssephevmbus_chan_rem_list(struct vmbus_softc *sc, struct vmbus_channel *chan) 210307599Ssephe{ 211307599Ssephe 212307599Ssephe mtx_assert(&sc->vmbus_chan_lock, MA_OWNED); 213307599Ssephe if (atomic_testandclear_int(&chan->ch_stflags, 214307599Ssephe VMBUS_CHAN_ST_ONLIST_SHIFT) == 0) 215307599Ssephe panic("channel is not on the list"); 216307599Ssephe TAILQ_REMOVE(&sc->vmbus_chans, chan, ch_link); 217307599Ssephe} 218307599Ssephe 219296289Ssephestatic int 220302892Ssephevmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS) 221296289Ssephe{ 222307461Ssephe struct vmbus_channel *chan = arg1; 223302892Ssephe int mnf = 0; 224296289Ssephe 225307461Ssephe if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF) 226302892Ssephe mnf = 1; 227302892Ssephe return sysctl_handle_int(oidp, &mnf, 0, req); 228296289Ssephe} 229296289Ssephe 230296181Ssephestatic void 231307461Ssephevmbus_chan_sysctl_create(struct vmbus_channel *chan) 232296181Ssephe{ 233302892Ssephe struct sysctl_oid *ch_tree, *chid_tree, *br_tree; 234296181Ssephe struct sysctl_ctx_list *ctx; 235296181Ssephe uint32_t ch_id; 236296181Ssephe char name[16]; 237296181Ssephe 238302892Ssephe /* 239302892Ssephe * Add sysctl nodes related to this channel to this 240302892Ssephe * channel's sysctl ctx, so that they can be destroyed 241302892Ssephe * independently upon close of this channel, which can 242302892Ssephe * happen even if the device is not detached. 243302892Ssephe */ 244302892Ssephe ctx = &chan->ch_sysctl_ctx; 245302633Ssephe sysctl_ctx_init(ctx); 246302892Ssephe 247302892Ssephe /* 248302892Ssephe * Create dev.NAME.UNIT.channel tree. 249302892Ssephe */ 250302892Ssephe ch_tree = SYSCTL_ADD_NODE(ctx, 251302892Ssephe SYSCTL_CHILDREN(device_get_sysctl_tree(chan->ch_dev)), 252302892Ssephe OID_AUTO, "channel", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 253302892Ssephe if (ch_tree == NULL) 254302892Ssephe return; 255302892Ssephe 256302892Ssephe /* 257302892Ssephe * Create dev.NAME.UNIT.channel.CHANID tree. 258302892Ssephe */ 259302892Ssephe if (VMBUS_CHAN_ISPRIMARY(chan)) 260302892Ssephe ch_id = chan->ch_id; 261302892Ssephe else 262302892Ssephe ch_id = chan->ch_prichan->ch_id; 263296181Ssephe snprintf(name, sizeof(name), "%d", ch_id); 264302892Ssephe chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree), 265302892Ssephe OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 266302892Ssephe if (chid_tree == NULL) 267302892Ssephe return; 268296181Ssephe 269302892Ssephe if (!VMBUS_CHAN_ISPRIMARY(chan)) { 270302892Ssephe /* 271302892Ssephe * Create dev.NAME.UNIT.channel.CHANID.sub tree. 272302892Ssephe */ 273302892Ssephe ch_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree), 274302892Ssephe OID_AUTO, "sub", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 275302892Ssephe if (ch_tree == NULL) 276302892Ssephe return; 277296188Ssephe 278302892Ssephe /* 279302892Ssephe * Create dev.NAME.UNIT.channel.CHANID.sub.SUBIDX tree. 280302892Ssephe * 281302892Ssephe * NOTE: 282302892Ssephe * chid_tree is changed to this new sysctl tree. 283302892Ssephe */ 284302892Ssephe snprintf(name, sizeof(name), "%d", chan->ch_subidx); 285302892Ssephe chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree), 286302892Ssephe OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 287302892Ssephe if (chid_tree == NULL) 288302892Ssephe return; 289302892Ssephe 290302892Ssephe SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 291302892Ssephe "chanid", CTLFLAG_RD, &chan->ch_id, 0, "channel id"); 292296181Ssephe } 293296188Ssephe 294302892Ssephe SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 295302892Ssephe "cpu", CTLFLAG_RD, &chan->ch_cpuid, 0, "owner CPU id"); 296302892Ssephe SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 297302892Ssephe "mnf", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, 298302892Ssephe chan, 0, vmbus_chan_sysctl_mnf, "I", 299302892Ssephe "has monitor notification facilities"); 300302892Ssephe 301302892Ssephe br_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 302307462Ssephe "br", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 303302892Ssephe if (br_tree != NULL) { 304307462Ssephe /* 305307462Ssephe * Create sysctl tree for RX bufring. 306307462Ssephe */ 307307464Ssephe vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_rxbr.rxbr, "rx"); 308307462Ssephe /* 309307462Ssephe * Create sysctl tree for TX bufring. 310307462Ssephe */ 311307464Ssephe vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_txbr.txbr, "tx"); 312302892Ssephe } 313296181Ssephe} 314296290Ssephe 315250199Sgrehanint 316307461Ssephevmbus_chan_open(struct vmbus_channel *chan, int txbr_size, int rxbr_size, 317303021Ssephe const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg) 318250199Sgrehan{ 319307595Ssephe struct vmbus_chan_br cbr; 320307595Ssephe int error; 321307595Ssephe 322307595Ssephe /* 323307595Ssephe * Allocate the TX+RX bufrings. 324307595Ssephe */ 325307595Ssephe KASSERT(chan->ch_bufring == NULL, ("bufrings are allocated")); 326307595Ssephe chan->ch_bufring = hyperv_dmamem_alloc(bus_get_dma_tag(chan->ch_dev), 327307595Ssephe PAGE_SIZE, 0, txbr_size + rxbr_size, &chan->ch_bufring_dma, 328307595Ssephe BUS_DMA_WAITOK); 329307595Ssephe if (chan->ch_bufring == NULL) { 330308621Ssephe vmbus_chan_printf(chan, "bufring allocation failed\n"); 331307595Ssephe return (ENOMEM); 332307595Ssephe } 333307595Ssephe 334307595Ssephe cbr.cbr = chan->ch_bufring; 335307595Ssephe cbr.cbr_paddr = chan->ch_bufring_dma.hv_paddr; 336307595Ssephe cbr.cbr_txsz = txbr_size; 337307595Ssephe cbr.cbr_rxsz = rxbr_size; 338307595Ssephe 339307595Ssephe error = vmbus_chan_open_br(chan, &cbr, udata, udlen, cb, cbarg); 340307595Ssephe if (error) { 341311364Ssephe if (error == EISCONN) { 342311364Ssephe /* 343311364Ssephe * XXX 344311364Ssephe * The bufring GPADL is still connected; abandon 345311364Ssephe * this bufring, instead of having mysterious 346311364Ssephe * crash or trashed data later on. 347311364Ssephe */ 348311364Ssephe vmbus_chan_printf(chan, "chan%u bufring GPADL " 349311364Ssephe "is still connected upon channel open error; " 350311364Ssephe "leak %d bytes memory\n", chan->ch_id, 351311364Ssephe txbr_size + rxbr_size); 352311364Ssephe } else { 353311364Ssephe hyperv_dmamem_free(&chan->ch_bufring_dma, 354311364Ssephe chan->ch_bufring); 355311364Ssephe } 356307595Ssephe chan->ch_bufring = NULL; 357307595Ssephe } 358307595Ssephe return (error); 359307595Ssephe} 360307595Ssephe 361307595Ssepheint 362307595Ssephevmbus_chan_open_br(struct vmbus_channel *chan, const struct vmbus_chan_br *cbr, 363307595Ssephe const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg) 364307595Ssephe{ 365307461Ssephe struct vmbus_softc *sc = chan->ch_vmbus; 366302607Ssephe const struct vmbus_message *msg; 367302607Ssephe struct vmbus_chanmsg_chopen *req; 368302607Ssephe struct vmbus_msghc *mh; 369302607Ssephe uint32_t status; 370307595Ssephe int error, txbr_size, rxbr_size; 371307599Ssephe task_fn_t *task_fn; 372302872Ssephe uint8_t *br; 373250199Sgrehan 374302986Ssephe if (udlen > VMBUS_CHANMSG_CHOPEN_UDATA_SIZE) { 375308621Ssephe vmbus_chan_printf(chan, 376302986Ssephe "invalid udata len %d for chan%u\n", udlen, chan->ch_id); 377311364Ssephe return (EINVAL); 378302607Ssephe } 379307595Ssephe 380307595Ssephe br = cbr->cbr; 381307595Ssephe txbr_size = cbr->cbr_txsz; 382307595Ssephe rxbr_size = cbr->cbr_rxsz; 383302986Ssephe KASSERT((txbr_size & PAGE_MASK) == 0, 384302872Ssephe ("send bufring size is not multiple page")); 385302986Ssephe KASSERT((rxbr_size & PAGE_MASK) == 0, 386302872Ssephe ("recv bufring size is not multiple page")); 387307595Ssephe KASSERT((cbr->cbr_paddr & PAGE_MASK) == 0, 388307595Ssephe ("bufring is not page aligned")); 389302607Ssephe 390307595Ssephe /* 391307595Ssephe * Zero out the TX/RX bufrings, in case that they were used before. 392307595Ssephe */ 393307595Ssephe memset(br, 0, txbr_size + rxbr_size); 394307595Ssephe 395302986Ssephe if (atomic_testandset_int(&chan->ch_stflags, 396302812Ssephe VMBUS_CHAN_ST_OPENED_SHIFT)) 397302986Ssephe panic("double-open chan%u", chan->ch_id); 398282212Swhu 399302986Ssephe chan->ch_cb = cb; 400302986Ssephe chan->ch_cbarg = cbarg; 401250199Sgrehan 402302986Ssephe vmbus_chan_update_evtflagcnt(sc, chan); 403300102Ssephe 404307461Ssephe chan->ch_tq = VMBUS_PCPU_GET(chan->ch_vmbus, event_tq, chan->ch_cpuid); 405302986Ssephe if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD) 406307599Ssephe task_fn = vmbus_chan_task; 407302986Ssephe else 408307599Ssephe task_fn = vmbus_chan_task_nobatch; 409307599Ssephe TASK_INIT(&chan->ch_task, 0, task_fn, chan); 410294886Ssephe 411302872Ssephe /* TX bufring comes first */ 412307464Ssephe vmbus_txbr_setup(&chan->ch_txbr, br, txbr_size); 413302872Ssephe /* RX bufring immediately follows TX bufring */ 414307464Ssephe vmbus_rxbr_setup(&chan->ch_rxbr, br + txbr_size, rxbr_size); 415250199Sgrehan 416296290Ssephe /* Create sysctl tree for this channel */ 417302986Ssephe vmbus_chan_sysctl_create(chan); 418296181Ssephe 419302872Ssephe /* 420302872Ssephe * Connect the bufrings, both RX and TX, to this channel. 421250199Sgrehan */ 422307595Ssephe error = vmbus_chan_gpadl_connect(chan, cbr->cbr_paddr, 423302986Ssephe txbr_size + rxbr_size, &chan->ch_bufring_gpadl); 424302986Ssephe if (error) { 425308621Ssephe vmbus_chan_printf(chan, 426302986Ssephe "failed to connect bufring GPADL to chan%u\n", chan->ch_id); 427302872Ssephe goto failed; 428302872Ssephe } 429250199Sgrehan 430302607Ssephe /* 431307599Ssephe * Install this channel, before it is opened, but after everything 432307599Ssephe * else has been setup. 433307599Ssephe */ 434307599Ssephe vmbus_chan_set_chmap(chan); 435307599Ssephe 436307599Ssephe /* 437302607Ssephe * Open channel w/ the bufring GPADL on the target CPU. 438250199Sgrehan */ 439302607Ssephe mh = vmbus_msghc_get(sc, sizeof(*req)); 440302607Ssephe if (mh == NULL) { 441308621Ssephe vmbus_chan_printf(chan, 442302607Ssephe "can not get msg hypercall for chopen(chan%u)\n", 443302986Ssephe chan->ch_id); 444302986Ssephe error = ENXIO; 445302812Ssephe goto failed; 446302607Ssephe } 447250199Sgrehan 448302607Ssephe req = vmbus_msghc_dataptr(mh); 449302607Ssephe req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHOPEN; 450302986Ssephe req->chm_chanid = chan->ch_id; 451302986Ssephe req->chm_openid = chan->ch_id; 452302986Ssephe req->chm_gpadl = chan->ch_bufring_gpadl; 453302986Ssephe req->chm_vcpuid = chan->ch_vcpuid; 454302986Ssephe req->chm_txbr_pgcnt = txbr_size >> PAGE_SHIFT; 455302986Ssephe if (udlen > 0) 456302986Ssephe memcpy(req->chm_udata, udata, udlen); 457250199Sgrehan 458302986Ssephe error = vmbus_msghc_exec(sc, mh); 459302986Ssephe if (error) { 460308621Ssephe vmbus_chan_printf(chan, 461302607Ssephe "chopen(chan%u) msg hypercall exec failed: %d\n", 462302986Ssephe chan->ch_id, error); 463302607Ssephe vmbus_msghc_put(sc, mh); 464302812Ssephe goto failed; 465302607Ssephe } 466250199Sgrehan 467311367Ssephe for (;;) { 468311367Ssephe msg = vmbus_msghc_poll_result(sc, mh); 469311367Ssephe if (msg != NULL) 470311367Ssephe break; 471311367Ssephe if (vmbus_chan_is_revoked(chan)) { 472311367Ssephe int i; 473250199Sgrehan 474311367Ssephe /* 475311367Ssephe * NOTE: 476311367Ssephe * Hypervisor does _not_ send response CHOPEN to 477311367Ssephe * a revoked channel. 478311367Ssephe */ 479311367Ssephe vmbus_chan_printf(chan, 480311367Ssephe "chan%u is revoked, when it is being opened\n", 481311367Ssephe chan->ch_id); 482311367Ssephe 483311367Ssephe /* 484311367Ssephe * XXX 485311367Ssephe * Add extra delay before cancel the hypercall 486311367Ssephe * execution; mainly to close any possible 487311367Ssephe * CHRESCIND and CHOPEN_RESP races on the 488311367Ssephe * hypervisor side. 489311367Ssephe */ 490311367Ssephe#define REVOKE_LINGER 100 491311367Ssephe for (i = 0; i < REVOKE_LINGER; ++i) { 492311367Ssephe msg = vmbus_msghc_poll_result(sc, mh); 493311367Ssephe if (msg != NULL) 494311367Ssephe break; 495311367Ssephe pause("rchopen", 1); 496311367Ssephe } 497311367Ssephe#undef REVOKE_LINGER 498311367Ssephe if (msg == NULL) 499311367Ssephe vmbus_msghc_exec_cancel(sc, mh); 500311367Ssephe break; 501311367Ssephe } 502311367Ssephe pause("chopen", 1); 503311367Ssephe } 504311367Ssephe if (msg != NULL) { 505311367Ssephe status = ((const struct vmbus_chanmsg_chopen_resp *) 506311367Ssephe msg->msg_data)->chm_status; 507311367Ssephe } else { 508311367Ssephe /* XXX any non-0 value is ok here. */ 509311367Ssephe status = 0xff; 510311367Ssephe } 511311367Ssephe 512302607Ssephe vmbus_msghc_put(sc, mh); 513250199Sgrehan 514302607Ssephe if (status == 0) { 515311364Ssephe if (bootverbose) 516308621Ssephe vmbus_chan_printf(chan, "chan%u opened\n", chan->ch_id); 517311364Ssephe return (0); 518250199Sgrehan } 519302812Ssephe 520308621Ssephe vmbus_chan_printf(chan, "failed to open chan%u\n", chan->ch_id); 521302986Ssephe error = ENXIO; 522302812Ssephe 523302812Ssephefailed: 524311364Ssephe sysctl_ctx_free(&chan->ch_sysctl_ctx); 525307599Ssephe vmbus_chan_clear_chmap(chan); 526311364Ssephe if (chan->ch_bufring_gpadl != 0) { 527311364Ssephe int error1; 528311364Ssephe 529311364Ssephe error1 = vmbus_chan_gpadl_disconnect(chan, 530311364Ssephe chan->ch_bufring_gpadl); 531311364Ssephe if (error1) { 532311364Ssephe /* 533311364Ssephe * Give caller a hint that the bufring GPADL is still 534311364Ssephe * connected. 535311364Ssephe */ 536311364Ssephe error = EISCONN; 537311364Ssephe } 538302986Ssephe chan->ch_bufring_gpadl = 0; 539302872Ssephe } 540302986Ssephe atomic_clear_int(&chan->ch_stflags, VMBUS_CHAN_ST_OPENED); 541311364Ssephe return (error); 542250199Sgrehan} 543250199Sgrehan 544302609Ssepheint 545307461Ssephevmbus_chan_gpadl_connect(struct vmbus_channel *chan, bus_addr_t paddr, 546302871Ssephe int size, uint32_t *gpadl0) 547302871Ssephe{ 548307461Ssephe struct vmbus_softc *sc = chan->ch_vmbus; 549302609Ssephe struct vmbus_msghc *mh; 550302609Ssephe struct vmbus_chanmsg_gpadl_conn *req; 551302609Ssephe const struct vmbus_message *msg; 552302609Ssephe size_t reqsz; 553302609Ssephe uint32_t gpadl, status; 554302609Ssephe int page_count, range_len, i, cnt, error; 555302871Ssephe uint64_t page_id; 556250199Sgrehan 557311366Ssephe KASSERT(*gpadl0 == 0, ("GPADL is not zero")); 558311364Ssephe 559311364Ssephe /* 560302609Ssephe * Preliminary checks. 561302609Ssephe */ 562250199Sgrehan 563302609Ssephe KASSERT((size & PAGE_MASK) == 0, 564302871Ssephe ("invalid GPA size %d, not multiple page size", size)); 565250199Sgrehan page_count = size >> PAGE_SHIFT; 566250199Sgrehan 567302609Ssephe KASSERT((paddr & PAGE_MASK) == 0, 568302609Ssephe ("GPA is not page aligned %jx", (uintmax_t)paddr)); 569302609Ssephe page_id = paddr >> PAGE_SHIFT; 570250199Sgrehan 571302609Ssephe range_len = __offsetof(struct vmbus_gpa_range, gpa_page[page_count]); 572302609Ssephe /* 573302609Ssephe * We don't support multiple GPA ranges. 574302609Ssephe */ 575302609Ssephe if (range_len > UINT16_MAX) { 576308621Ssephe vmbus_chan_printf(chan, "GPA too large, %d pages\n", 577302609Ssephe page_count); 578302609Ssephe return EOPNOTSUPP; 579250199Sgrehan } 580250199Sgrehan 581302609Ssephe /* 582302609Ssephe * Allocate GPADL id. 583302609Ssephe */ 584302630Ssephe gpadl = vmbus_gpadl_alloc(sc); 585250199Sgrehan 586302609Ssephe /* 587302609Ssephe * Connect this GPADL to the target channel. 588302609Ssephe * 589302609Ssephe * NOTE: 590302609Ssephe * Since each message can only hold small set of page 591302609Ssephe * addresses, several messages may be required to 592302609Ssephe * complete the connection. 593302609Ssephe */ 594302609Ssephe if (page_count > VMBUS_CHANMSG_GPADL_CONN_PGMAX) 595302609Ssephe cnt = VMBUS_CHANMSG_GPADL_CONN_PGMAX; 596302609Ssephe else 597302609Ssephe cnt = page_count; 598302609Ssephe page_count -= cnt; 599250199Sgrehan 600302609Ssephe reqsz = __offsetof(struct vmbus_chanmsg_gpadl_conn, 601302609Ssephe chm_range.gpa_page[cnt]); 602302609Ssephe mh = vmbus_msghc_get(sc, reqsz); 603302609Ssephe if (mh == NULL) { 604308621Ssephe vmbus_chan_printf(chan, 605308621Ssephe "can not get msg hypercall for gpadl_conn(chan%u)\n", 606302871Ssephe chan->ch_id); 607302609Ssephe return EIO; 608250199Sgrehan } 609250199Sgrehan 610302609Ssephe req = vmbus_msghc_dataptr(mh); 611302609Ssephe req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_CONN; 612302871Ssephe req->chm_chanid = chan->ch_id; 613302609Ssephe req->chm_gpadl = gpadl; 614302609Ssephe req->chm_range_len = range_len; 615302609Ssephe req->chm_range_cnt = 1; 616302609Ssephe req->chm_range.gpa_len = size; 617302609Ssephe req->chm_range.gpa_ofs = 0; 618302609Ssephe for (i = 0; i < cnt; ++i) 619302609Ssephe req->chm_range.gpa_page[i] = page_id++; 620250199Sgrehan 621302609Ssephe error = vmbus_msghc_exec(sc, mh); 622302609Ssephe if (error) { 623308621Ssephe vmbus_chan_printf(chan, 624308621Ssephe "gpadl_conn(chan%u) msg hypercall exec failed: %d\n", 625302871Ssephe chan->ch_id, error); 626302609Ssephe vmbus_msghc_put(sc, mh); 627302609Ssephe return error; 628302609Ssephe } 629250199Sgrehan 630302609Ssephe while (page_count > 0) { 631302609Ssephe struct vmbus_chanmsg_gpadl_subconn *subreq; 632250199Sgrehan 633302609Ssephe if (page_count > VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX) 634302609Ssephe cnt = VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX; 635302609Ssephe else 636302609Ssephe cnt = page_count; 637302609Ssephe page_count -= cnt; 638250199Sgrehan 639302609Ssephe reqsz = __offsetof(struct vmbus_chanmsg_gpadl_subconn, 640302609Ssephe chm_gpa_page[cnt]); 641302609Ssephe vmbus_msghc_reset(mh, reqsz); 642250199Sgrehan 643302609Ssephe subreq = vmbus_msghc_dataptr(mh); 644302609Ssephe subreq->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_SUBCONN; 645302609Ssephe subreq->chm_gpadl = gpadl; 646302609Ssephe for (i = 0; i < cnt; ++i) 647302609Ssephe subreq->chm_gpa_page[i] = page_id++; 648250199Sgrehan 649302609Ssephe vmbus_msghc_exec_noresult(mh); 650250199Sgrehan } 651302609Ssephe KASSERT(page_count == 0, ("invalid page count %d", page_count)); 652250199Sgrehan 653302609Ssephe msg = vmbus_msghc_wait_result(sc, mh); 654302609Ssephe status = ((const struct vmbus_chanmsg_gpadl_connresp *) 655302609Ssephe msg->msg_data)->chm_status; 656250199Sgrehan 657302609Ssephe vmbus_msghc_put(sc, mh); 658250199Sgrehan 659302609Ssephe if (status != 0) { 660308621Ssephe vmbus_chan_printf(chan, "gpadl_conn(chan%u) failed: %u\n", 661308621Ssephe chan->ch_id, status); 662302609Ssephe return EIO; 663302609Ssephe } 664311364Ssephe 665311364Ssephe /* Done; commit the GPADL id. */ 666311364Ssephe *gpadl0 = gpadl; 667311364Ssephe if (bootverbose) { 668311364Ssephe vmbus_chan_printf(chan, "gpadl_conn(chan%u) succeeded\n", 669311364Ssephe chan->ch_id); 670311364Ssephe } 671302609Ssephe return 0; 672250199Sgrehan} 673250199Sgrehan 674311364Ssephestatic bool 675311367Ssephevmbus_chan_wait_revoke(const struct vmbus_channel *chan, bool can_sleep) 676311364Ssephe{ 677311364Ssephe#define WAIT_COUNT 200 /* 200ms */ 678311364Ssephe 679311364Ssephe int i; 680311364Ssephe 681311364Ssephe for (i = 0; i < WAIT_COUNT; ++i) { 682311364Ssephe if (vmbus_chan_is_revoked(chan)) 683311364Ssephe return (true); 684311367Ssephe if (can_sleep) 685311367Ssephe pause("wchrev", 1); 686311367Ssephe else 687311367Ssephe DELAY(1000); 688311364Ssephe } 689311364Ssephe return (false); 690311364Ssephe 691311364Ssephe#undef WAIT_COUNT 692311364Ssephe} 693311364Ssephe 694302611Ssephe/* 695302611Ssephe * Disconnect the GPA from the target channel 696250199Sgrehan */ 697250199Sgrehanint 698307461Ssephevmbus_chan_gpadl_disconnect(struct vmbus_channel *chan, uint32_t gpadl) 699250199Sgrehan{ 700307461Ssephe struct vmbus_softc *sc = chan->ch_vmbus; 701302611Ssephe struct vmbus_msghc *mh; 702302611Ssephe struct vmbus_chanmsg_gpadl_disconn *req; 703302611Ssephe int error; 704250199Sgrehan 705311366Ssephe KASSERT(gpadl != 0, ("GPADL is zero")); 706311366Ssephe 707302611Ssephe mh = vmbus_msghc_get(sc, sizeof(*req)); 708302611Ssephe if (mh == NULL) { 709308621Ssephe vmbus_chan_printf(chan, 710308621Ssephe "can not get msg hypercall for gpadl_disconn(chan%u)\n", 711302693Ssephe chan->ch_id); 712311364Ssephe return (EBUSY); 713250199Sgrehan } 714250199Sgrehan 715302611Ssephe req = vmbus_msghc_dataptr(mh); 716302611Ssephe req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_DISCONN; 717302693Ssephe req->chm_chanid = chan->ch_id; 718302611Ssephe req->chm_gpadl = gpadl; 719250199Sgrehan 720302611Ssephe error = vmbus_msghc_exec(sc, mh); 721302611Ssephe if (error) { 722311364Ssephe vmbus_msghc_put(sc, mh); 723311364Ssephe 724311367Ssephe if (vmbus_chan_wait_revoke(chan, true)) { 725311364Ssephe /* 726311364Ssephe * Error is benign; this channel is revoked, 727311364Ssephe * so this GPADL will not be touched anymore. 728311364Ssephe */ 729311364Ssephe vmbus_chan_printf(chan, 730311364Ssephe "gpadl_disconn(revoked chan%u) msg hypercall " 731311364Ssephe "exec failed: %d\n", chan->ch_id, error); 732311364Ssephe return (0); 733311364Ssephe } 734308621Ssephe vmbus_chan_printf(chan, 735308621Ssephe "gpadl_disconn(chan%u) msg hypercall exec failed: %d\n", 736302693Ssephe chan->ch_id, error); 737311364Ssephe return (error); 738302611Ssephe } 739250199Sgrehan 740302611Ssephe vmbus_msghc_wait_result(sc, mh); 741302611Ssephe /* Discard result; no useful information */ 742302611Ssephe vmbus_msghc_put(sc, mh); 743250199Sgrehan 744311364Ssephe return (0); 745250199Sgrehan} 746250199Sgrehan 747282212Swhustatic void 748311359Ssephevmbus_chan_detach(struct vmbus_channel *chan) 749311359Ssephe{ 750311359Ssephe int refs; 751311359Ssephe 752311359Ssephe KASSERT(chan->ch_refs > 0, ("chan%u: invalid refcnt %d", 753311359Ssephe chan->ch_id, chan->ch_refs)); 754311359Ssephe refs = atomic_fetchadd_int(&chan->ch_refs, -1); 755311359Ssephe#ifdef INVARIANTS 756311359Ssephe if (VMBUS_CHAN_ISPRIMARY(chan)) { 757311359Ssephe KASSERT(refs == 1, ("chan%u: invalid refcnt %d for prichan", 758311359Ssephe chan->ch_id, refs + 1)); 759311359Ssephe } 760311359Ssephe#endif 761311359Ssephe if (refs == 1) { 762311359Ssephe /* 763311359Ssephe * Detach the target channel. 764311359Ssephe */ 765311359Ssephe if (bootverbose) { 766311359Ssephe vmbus_chan_printf(chan, "chan%u detached\n", 767311359Ssephe chan->ch_id); 768311359Ssephe } 769311359Ssephe taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task); 770311359Ssephe } 771311359Ssephe} 772311359Ssephe 773311359Ssephestatic void 774307599Ssephevmbus_chan_clrchmap_task(void *xchan, int pending __unused) 775307599Ssephe{ 776307599Ssephe struct vmbus_channel *chan = xchan; 777307599Ssephe 778307599Ssephe chan->ch_vmbus->vmbus_chmap[chan->ch_id] = NULL; 779307599Ssephe} 780307599Ssephe 781307599Ssephestatic void 782307599Ssephevmbus_chan_clear_chmap(struct vmbus_channel *chan) 783307599Ssephe{ 784307599Ssephe struct task chmap_task; 785307599Ssephe 786307599Ssephe TASK_INIT(&chmap_task, 0, vmbus_chan_clrchmap_task, chan); 787311372Ssephe vmbus_chan_run_task(chan, &chmap_task); 788307599Ssephe} 789307599Ssephe 790307599Ssephestatic void 791307599Ssephevmbus_chan_set_chmap(struct vmbus_channel *chan) 792307599Ssephe{ 793307599Ssephe __compiler_membar(); 794307599Ssephe chan->ch_vmbus->vmbus_chmap[chan->ch_id] = chan; 795307599Ssephe} 796307599Ssephe 797311375Ssephestatic void 798311375Ssephevmbus_chan_poll_cancel_task(void *xchan, int pending __unused) 799311375Ssephe{ 800311375Ssephe 801311375Ssephe vmbus_chan_poll_cancel_intq(xchan); 802311375Ssephe} 803311375Ssephe 804311375Ssephestatic void 805311375Ssephevmbus_chan_poll_cancel(struct vmbus_channel *chan) 806311375Ssephe{ 807311375Ssephe struct task poll_cancel; 808311375Ssephe 809311375Ssephe TASK_INIT(&poll_cancel, 0, vmbus_chan_poll_cancel_task, chan); 810311375Ssephe vmbus_chan_run_task(chan, &poll_cancel); 811311375Ssephe} 812311375Ssephe 813311364Ssephestatic int 814307461Ssephevmbus_chan_close_internal(struct vmbus_channel *chan) 815250199Sgrehan{ 816307461Ssephe struct vmbus_softc *sc = chan->ch_vmbus; 817302610Ssephe struct vmbus_msghc *mh; 818302610Ssephe struct vmbus_chanmsg_chclose *req; 819311364Ssephe uint32_t old_stflags; 820302610Ssephe int error; 821250199Sgrehan 822311364Ssephe /* 823311364Ssephe * NOTE: 824311364Ssephe * Sub-channels are closed upon their primary channel closing, 825311364Ssephe * so they can be closed even before they are opened. 826311364Ssephe */ 827311364Ssephe for (;;) { 828311364Ssephe old_stflags = chan->ch_stflags; 829311364Ssephe if (atomic_cmpset_int(&chan->ch_stflags, old_stflags, 830311364Ssephe old_stflags & ~VMBUS_CHAN_ST_OPENED)) 831311364Ssephe break; 832311364Ssephe } 833311364Ssephe if ((old_stflags & VMBUS_CHAN_ST_OPENED) == 0) { 834311364Ssephe /* Not opened yet; done */ 835311364Ssephe if (bootverbose) { 836311364Ssephe vmbus_chan_printf(chan, "chan%u not opened\n", 837311364Ssephe chan->ch_id); 838311364Ssephe } 839311364Ssephe return (0); 840311364Ssephe } 841302812Ssephe 842302891Ssephe /* 843302891Ssephe * Free this channel's sysctl tree attached to its device's 844302891Ssephe * sysctl tree. 845302891Ssephe */ 846302891Ssephe sysctl_ctx_free(&chan->ch_sysctl_ctx); 847282212Swhu 848282212Swhu /* 849311375Ssephe * Cancel polling, if it is enabled. 850311375Ssephe */ 851311375Ssephe vmbus_chan_poll_cancel(chan); 852311375Ssephe 853311375Ssephe /* 854307599Ssephe * NOTE: 855307599Ssephe * Order is critical. This channel _must_ be uninstalled first, 856307599Ssephe * else the channel task may be enqueued by the IDT after it has 857307599Ssephe * been drained. 858294886Ssephe */ 859307599Ssephe vmbus_chan_clear_chmap(chan); 860307599Ssephe taskqueue_drain(chan->ch_tq, &chan->ch_task); 861302891Ssephe chan->ch_tq = NULL; 862250199Sgrehan 863302891Ssephe /* 864302891Ssephe * Close this channel. 865250199Sgrehan */ 866302610Ssephe mh = vmbus_msghc_get(sc, sizeof(*req)); 867302610Ssephe if (mh == NULL) { 868308621Ssephe vmbus_chan_printf(chan, 869302610Ssephe "can not get msg hypercall for chclose(chan%u)\n", 870302891Ssephe chan->ch_id); 871311364Ssephe error = ENXIO; 872311364Ssephe goto disconnect; 873302610Ssephe } 874250199Sgrehan 875302610Ssephe req = vmbus_msghc_dataptr(mh); 876302610Ssephe req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHCLOSE; 877302891Ssephe req->chm_chanid = chan->ch_id; 878250199Sgrehan 879302610Ssephe error = vmbus_msghc_exec_noresult(mh); 880302610Ssephe vmbus_msghc_put(sc, mh); 881302610Ssephe 882302610Ssephe if (error) { 883308621Ssephe vmbus_chan_printf(chan, 884302610Ssephe "chclose(chan%u) msg hypercall exec failed: %d\n", 885302891Ssephe chan->ch_id, error); 886311364Ssephe goto disconnect; 887302610Ssephe } 888302610Ssephe 889311364Ssephe if (bootverbose) 890311364Ssephe vmbus_chan_printf(chan, "chan%u closed\n", chan->ch_id); 891311364Ssephe 892311364Ssephedisconnect: 893302891Ssephe /* 894302891Ssephe * Disconnect the TX+RX bufrings from this channel. 895302891Ssephe */ 896311364Ssephe if (chan->ch_bufring_gpadl != 0) { 897311364Ssephe int error1; 898311364Ssephe 899311364Ssephe error1 = vmbus_chan_gpadl_disconnect(chan, 900311364Ssephe chan->ch_bufring_gpadl); 901311364Ssephe if (error1) { 902311364Ssephe /* 903311364Ssephe * XXX 904311364Ssephe * The bufring GPADL is still connected; abandon 905311364Ssephe * this bufring, instead of having mysterious 906311364Ssephe * crash or trashed data later on. 907311364Ssephe */ 908311364Ssephe vmbus_chan_printf(chan, "chan%u bufring GPADL " 909311364Ssephe "is still connected after close\n", chan->ch_id); 910311364Ssephe chan->ch_bufring = NULL; 911311364Ssephe /* 912311364Ssephe * Give caller a hint that the bufring GPADL is 913311364Ssephe * still connected. 914311364Ssephe */ 915311364Ssephe error = EISCONN; 916311364Ssephe } 917302891Ssephe chan->ch_bufring_gpadl = 0; 918250199Sgrehan } 919250199Sgrehan 920302891Ssephe /* 921302891Ssephe * Destroy the TX+RX bufrings. 922302891Ssephe */ 923302891Ssephe if (chan->ch_bufring != NULL) { 924302891Ssephe hyperv_dmamem_free(&chan->ch_bufring_dma, chan->ch_bufring); 925302891Ssephe chan->ch_bufring = NULL; 926302872Ssephe } 927311364Ssephe return (error); 928282212Swhu} 929250199Sgrehan 930311364Ssepheint 931311364Ssephevmbus_chan_close_direct(struct vmbus_channel *chan) 932311364Ssephe{ 933311364Ssephe int error; 934311364Ssephe 935311364Ssephe#ifdef INVARIANTS 936311364Ssephe if (VMBUS_CHAN_ISPRIMARY(chan)) { 937311364Ssephe struct vmbus_channel *subchan; 938311364Ssephe 939311364Ssephe /* 940311364Ssephe * All sub-channels _must_ have been closed, or are _not_ 941311364Ssephe * opened at all. 942311364Ssephe */ 943311364Ssephe mtx_lock(&chan->ch_subchan_lock); 944311364Ssephe TAILQ_FOREACH(subchan, &chan->ch_subchans, ch_sublink) { 945311364Ssephe KASSERT( 946311364Ssephe (subchan->ch_stflags & VMBUS_CHAN_ST_OPENED) == 0, 947311364Ssephe ("chan%u: subchan%u is still opened", 948311364Ssephe chan->ch_id, subchan->ch_subidx)); 949311364Ssephe } 950311364Ssephe mtx_unlock(&chan->ch_subchan_lock); 951311364Ssephe } 952311364Ssephe#endif 953311364Ssephe 954311364Ssephe error = vmbus_chan_close_internal(chan); 955311364Ssephe if (!VMBUS_CHAN_ISPRIMARY(chan)) { 956311364Ssephe /* 957311364Ssephe * This sub-channel is referenced, when it is linked to 958311364Ssephe * the primary channel; drop that reference now. 959311364Ssephe */ 960311364Ssephe vmbus_chan_detach(chan); 961311364Ssephe } 962311364Ssephe return (error); 963311364Ssephe} 964311364Ssephe 965302818Ssephe/* 966302818Ssephe * Caller should make sure that all sub-channels have 967302818Ssephe * been added to 'chan' and all to-be-closed channels 968302818Ssephe * are not being opened. 969282212Swhu */ 970282212Swhuvoid 971307461Ssephevmbus_chan_close(struct vmbus_channel *chan) 972282212Swhu{ 973302818Ssephe int subchan_cnt; 974282212Swhu 975302818Ssephe if (!VMBUS_CHAN_ISPRIMARY(chan)) { 976282212Swhu /* 977302818Ssephe * Sub-channel is closed when its primary channel 978302818Ssephe * is closed; done. 979282212Swhu */ 980282212Swhu return; 981282212Swhu } 982282212Swhu 983250199Sgrehan /* 984302818Ssephe * Close all sub-channels, if any. 985250199Sgrehan */ 986302819Ssephe subchan_cnt = chan->ch_subchan_cnt; 987302818Ssephe if (subchan_cnt > 0) { 988307461Ssephe struct vmbus_channel **subchan; 989302818Ssephe int i; 990302818Ssephe 991302890Ssephe subchan = vmbus_subchan_get(chan, subchan_cnt); 992311359Ssephe for (i = 0; i < subchan_cnt; ++i) { 993302891Ssephe vmbus_chan_close_internal(subchan[i]); 994311359Ssephe /* 995311359Ssephe * This sub-channel is referenced, when it is 996311359Ssephe * linked to the primary channel; drop that 997311359Ssephe * reference now. 998311359Ssephe */ 999311359Ssephe vmbus_chan_detach(subchan[i]); 1000311359Ssephe } 1001302890Ssephe vmbus_subchan_rel(subchan, subchan_cnt); 1002250199Sgrehan } 1003302818Ssephe 1004302818Ssephe /* Then close the primary channel. */ 1005302891Ssephe vmbus_chan_close_internal(chan); 1006250199Sgrehan} 1007250199Sgrehan 1008307599Ssephevoid 1009307599Ssephevmbus_chan_intr_drain(struct vmbus_channel *chan) 1010307599Ssephe{ 1011307599Ssephe 1012307599Ssephe taskqueue_drain(chan->ch_tq, &chan->ch_task); 1013307599Ssephe} 1014307599Ssephe 1015250199Sgrehanint 1016307461Ssephevmbus_chan_send(struct vmbus_channel *chan, uint16_t type, uint16_t flags, 1017302882Ssephe void *data, int dlen, uint64_t xactid) 1018250199Sgrehan{ 1019302875Ssephe struct vmbus_chanpkt pkt; 1020302881Ssephe int pktlen, pad_pktlen, hlen, error; 1021302881Ssephe uint64_t pad = 0; 1022302881Ssephe struct iovec iov[3]; 1023302881Ssephe boolean_t send_evt; 1024250199Sgrehan 1025302881Ssephe hlen = sizeof(pkt); 1026302881Ssephe pktlen = hlen + dlen; 1027302884Ssephe pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1028307471Ssephe KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1029307471Ssephe ("invalid packet size %d", pad_pktlen)); 1030250199Sgrehan 1031302875Ssephe pkt.cp_hdr.cph_type = type; 1032302875Ssephe pkt.cp_hdr.cph_flags = flags; 1033302884Ssephe VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1034302884Ssephe VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1035302881Ssephe pkt.cp_hdr.cph_xactid = xactid; 1036250199Sgrehan 1037302875Ssephe iov[0].iov_base = &pkt; 1038302881Ssephe iov[0].iov_len = hlen; 1039302881Ssephe iov[1].iov_base = data; 1040302881Ssephe iov[1].iov_len = dlen; 1041302881Ssephe iov[2].iov_base = &pad; 1042302881Ssephe iov[2].iov_len = pad_pktlen - pktlen; 1043250199Sgrehan 1044307464Ssephe error = vmbus_txbr_write(&chan->ch_txbr, iov, 3, &send_evt); 1045302881Ssephe if (!error && send_evt) 1046303022Ssephe vmbus_chan_signal_tx(chan); 1047302881Ssephe return error; 1048250199Sgrehan} 1049250199Sgrehan 1050250199Sgrehanint 1051307461Ssephevmbus_chan_send_sglist(struct vmbus_channel *chan, 1052302876Ssephe struct vmbus_gpa sg[], int sglen, void *data, int dlen, uint64_t xactid) 1053250199Sgrehan{ 1054302876Ssephe struct vmbus_chanpkt_sglist pkt; 1055302876Ssephe int pktlen, pad_pktlen, hlen, error; 1056302876Ssephe struct iovec iov[4]; 1057302876Ssephe boolean_t send_evt; 1058302876Ssephe uint64_t pad = 0; 1059250199Sgrehan 1060302876Ssephe hlen = __offsetof(struct vmbus_chanpkt_sglist, cp_gpa[sglen]); 1061302876Ssephe pktlen = hlen + dlen; 1062302884Ssephe pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1063307471Ssephe KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1064307471Ssephe ("invalid packet size %d", pad_pktlen)); 1065250199Sgrehan 1066302880Ssephe pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1067302879Ssephe pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1068302884Ssephe VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1069302884Ssephe VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1070302876Ssephe pkt.cp_hdr.cph_xactid = xactid; 1071302876Ssephe pkt.cp_rsvd = 0; 1072302876Ssephe pkt.cp_gpa_cnt = sglen; 1073250199Sgrehan 1074302876Ssephe iov[0].iov_base = &pkt; 1075302876Ssephe iov[0].iov_len = sizeof(pkt); 1076302876Ssephe iov[1].iov_base = sg; 1077302876Ssephe iov[1].iov_len = sizeof(struct vmbus_gpa) * sglen; 1078302876Ssephe iov[2].iov_base = data; 1079302876Ssephe iov[2].iov_len = dlen; 1080302876Ssephe iov[3].iov_base = &pad; 1081302876Ssephe iov[3].iov_len = pad_pktlen - pktlen; 1082250199Sgrehan 1083307464Ssephe error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); 1084302876Ssephe if (!error && send_evt) 1085303022Ssephe vmbus_chan_signal_tx(chan); 1086302876Ssephe return error; 1087250199Sgrehan} 1088250199Sgrehan 1089250199Sgrehanint 1090307461Ssephevmbus_chan_send_prplist(struct vmbus_channel *chan, 1091302878Ssephe struct vmbus_gpa_range *prp, int prp_cnt, void *data, int dlen, 1092302878Ssephe uint64_t xactid) 1093250199Sgrehan{ 1094302878Ssephe struct vmbus_chanpkt_prplist pkt; 1095302878Ssephe int pktlen, pad_pktlen, hlen, error; 1096302878Ssephe struct iovec iov[4]; 1097302878Ssephe boolean_t send_evt; 1098302878Ssephe uint64_t pad = 0; 1099250199Sgrehan 1100302878Ssephe hlen = __offsetof(struct vmbus_chanpkt_prplist, 1101302878Ssephe cp_range[0].gpa_page[prp_cnt]); 1102302878Ssephe pktlen = hlen + dlen; 1103302884Ssephe pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1104307471Ssephe KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1105307471Ssephe ("invalid packet size %d", pad_pktlen)); 1106250199Sgrehan 1107302880Ssephe pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1108302879Ssephe pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1109302884Ssephe VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1110302884Ssephe VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1111302878Ssephe pkt.cp_hdr.cph_xactid = xactid; 1112302878Ssephe pkt.cp_rsvd = 0; 1113302878Ssephe pkt.cp_range_cnt = 1; 1114250199Sgrehan 1115302878Ssephe iov[0].iov_base = &pkt; 1116302878Ssephe iov[0].iov_len = sizeof(pkt); 1117302878Ssephe iov[1].iov_base = prp; 1118302878Ssephe iov[1].iov_len = __offsetof(struct vmbus_gpa_range, gpa_page[prp_cnt]); 1119302878Ssephe iov[2].iov_base = data; 1120302878Ssephe iov[2].iov_len = dlen; 1121302878Ssephe iov[3].iov_base = &pad; 1122302878Ssephe iov[3].iov_len = pad_pktlen - pktlen; 1123250199Sgrehan 1124307464Ssephe error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); 1125302878Ssephe if (!error && send_evt) 1126303022Ssephe vmbus_chan_signal_tx(chan); 1127302878Ssephe return error; 1128250199Sgrehan} 1129250199Sgrehan 1130250199Sgrehanint 1131307461Ssephevmbus_chan_recv(struct vmbus_channel *chan, void *data, int *dlen0, 1132302885Ssephe uint64_t *xactid) 1133250199Sgrehan{ 1134302885Ssephe struct vmbus_chanpkt_hdr pkt; 1135302885Ssephe int error, dlen, hlen; 1136250199Sgrehan 1137307464Ssephe error = vmbus_rxbr_peek(&chan->ch_rxbr, &pkt, sizeof(pkt)); 1138302885Ssephe if (error) 1139307498Ssephe return (error); 1140250199Sgrehan 1141307498Ssephe if (__predict_false(pkt.cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) { 1142308621Ssephe vmbus_chan_printf(chan, "invalid hlen %u\n", pkt.cph_hlen); 1143307498Ssephe /* XXX this channel is dead actually. */ 1144307498Ssephe return (EIO); 1145307498Ssephe } 1146307498Ssephe if (__predict_false(pkt.cph_hlen > pkt.cph_tlen)) { 1147308621Ssephe vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n", 1148307498Ssephe pkt.cph_hlen, pkt.cph_tlen); 1149307498Ssephe /* XXX this channel is dead actually. */ 1150307498Ssephe return (EIO); 1151307498Ssephe } 1152307498Ssephe 1153302885Ssephe hlen = VMBUS_CHANPKT_GETLEN(pkt.cph_hlen); 1154302885Ssephe dlen = VMBUS_CHANPKT_GETLEN(pkt.cph_tlen) - hlen; 1155250199Sgrehan 1156302885Ssephe if (*dlen0 < dlen) { 1157302886Ssephe /* Return the size of this packet's data. */ 1158302885Ssephe *dlen0 = dlen; 1159307498Ssephe return (ENOBUFS); 1160302885Ssephe } 1161250199Sgrehan 1162302885Ssephe *xactid = pkt.cph_xactid; 1163302885Ssephe *dlen0 = dlen; 1164250199Sgrehan 1165302886Ssephe /* Skip packet header */ 1166307464Ssephe error = vmbus_rxbr_read(&chan->ch_rxbr, data, dlen, hlen); 1167307464Ssephe KASSERT(!error, ("vmbus_rxbr_read failed")); 1168250199Sgrehan 1169307498Ssephe return (0); 1170250199Sgrehan} 1171250199Sgrehan 1172250199Sgrehanint 1173307461Ssephevmbus_chan_recv_pkt(struct vmbus_channel *chan, 1174308633Ssephe struct vmbus_chanpkt_hdr *pkt, int *pktlen0) 1175250199Sgrehan{ 1176308633Ssephe int error, pktlen, pkt_hlen; 1177250199Sgrehan 1178308633Ssephe pkt_hlen = sizeof(*pkt); 1179308633Ssephe error = vmbus_rxbr_peek(&chan->ch_rxbr, pkt, pkt_hlen); 1180302886Ssephe if (error) 1181307498Ssephe return (error); 1182250199Sgrehan 1183308633Ssephe if (__predict_false(pkt->cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) { 1184308633Ssephe vmbus_chan_printf(chan, "invalid hlen %u\n", pkt->cph_hlen); 1185307498Ssephe /* XXX this channel is dead actually. */ 1186307498Ssephe return (EIO); 1187307498Ssephe } 1188308633Ssephe if (__predict_false(pkt->cph_hlen > pkt->cph_tlen)) { 1189308621Ssephe vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n", 1190308633Ssephe pkt->cph_hlen, pkt->cph_tlen); 1191307498Ssephe /* XXX this channel is dead actually. */ 1192307498Ssephe return (EIO); 1193307498Ssephe } 1194307498Ssephe 1195308633Ssephe pktlen = VMBUS_CHANPKT_GETLEN(pkt->cph_tlen); 1196302886Ssephe if (*pktlen0 < pktlen) { 1197302886Ssephe /* Return the size of this packet. */ 1198302886Ssephe *pktlen0 = pktlen; 1199307498Ssephe return (ENOBUFS); 1200302886Ssephe } 1201302886Ssephe *pktlen0 = pktlen; 1202250199Sgrehan 1203308633Ssephe /* 1204308633Ssephe * Skip the fixed-size packet header, which has been filled 1205308633Ssephe * by the above vmbus_rxbr_peek(). 1206308633Ssephe */ 1207308633Ssephe error = vmbus_rxbr_read(&chan->ch_rxbr, pkt + 1, 1208308633Ssephe pktlen - pkt_hlen, pkt_hlen); 1209307464Ssephe KASSERT(!error, ("vmbus_rxbr_read failed")); 1210250199Sgrehan 1211307498Ssephe return (0); 1212250199Sgrehan} 1213294886Ssephe 1214294886Ssephestatic void 1215302713Ssephevmbus_chan_task(void *xchan, int pending __unused) 1216294886Ssephe{ 1217307461Ssephe struct vmbus_channel *chan = xchan; 1218302874Ssephe vmbus_chan_callback_t cb = chan->ch_cb; 1219302874Ssephe void *cbarg = chan->ch_cbarg; 1220294886Ssephe 1221311375Ssephe KASSERT(chan->ch_poll_intvl == 0, 1222311375Ssephe ("chan%u: interrupted in polling mode", chan->ch_id)); 1223311375Ssephe 1224302710Ssephe /* 1225302710Ssephe * Optimize host to guest signaling by ensuring: 1226302710Ssephe * 1. While reading the channel, we disable interrupts from 1227302710Ssephe * host. 1228302710Ssephe * 2. Ensure that we process all posted messages from the host 1229302710Ssephe * before returning from this callback. 1230302710Ssephe * 3. Once we return, enable signaling from the host. Once this 1231302710Ssephe * state is set we check to see if additional packets are 1232302710Ssephe * available to read. In this case we repeat the process. 1233302713Ssephe * 1234302713Ssephe * NOTE: Interrupt has been disabled in the ISR. 1235302710Ssephe */ 1236302713Ssephe for (;;) { 1237302713Ssephe uint32_t left; 1238294886Ssephe 1239307461Ssephe cb(chan, cbarg); 1240294886Ssephe 1241307464Ssephe left = vmbus_rxbr_intr_unmask(&chan->ch_rxbr); 1242302713Ssephe if (left == 0) { 1243302713Ssephe /* No more data in RX bufring; done */ 1244302713Ssephe break; 1245302713Ssephe } 1246307464Ssephe vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1247302713Ssephe } 1248294886Ssephe} 1249302692Ssephe 1250302713Ssephestatic void 1251302713Ssephevmbus_chan_task_nobatch(void *xchan, int pending __unused) 1252302713Ssephe{ 1253307461Ssephe struct vmbus_channel *chan = xchan; 1254302713Ssephe 1255311375Ssephe KASSERT(chan->ch_poll_intvl == 0, 1256311375Ssephe ("chan%u: interrupted in polling mode", chan->ch_id)); 1257307461Ssephe chan->ch_cb(chan, chan->ch_cbarg); 1258302713Ssephe} 1259302713Ssephe 1260311375Ssephestatic void 1261311375Ssephevmbus_chan_poll_timeout(void *xchan) 1262311375Ssephe{ 1263311375Ssephe struct vmbus_channel *chan = xchan; 1264311375Ssephe 1265311375Ssephe KASSERT(chan->ch_poll_intvl != 0, 1266311375Ssephe ("chan%u: polling timeout in interrupt mode", chan->ch_id)); 1267311375Ssephe taskqueue_enqueue(chan->ch_tq, &chan->ch_poll_task); 1268311375Ssephe} 1269311375Ssephe 1270311375Ssephestatic void 1271311375Ssephevmbus_chan_poll_task(void *xchan, int pending __unused) 1272311375Ssephe{ 1273311375Ssephe struct vmbus_channel *chan = xchan; 1274311375Ssephe 1275311375Ssephe KASSERT(chan->ch_poll_intvl != 0, 1276311375Ssephe ("chan%u: polling in interrupt mode", chan->ch_id)); 1277311375Ssephe callout_reset_sbt_curcpu(&chan->ch_poll_timeo, chan->ch_poll_intvl, 0, 1278311375Ssephe vmbus_chan_poll_timeout, chan, chan->ch_poll_flags); 1279311375Ssephe chan->ch_cb(chan, chan->ch_cbarg); 1280311375Ssephe} 1281311375Ssephe 1282311375Ssephestatic void 1283311375Ssephevmbus_chan_pollcfg_task(void *xarg, int pending __unused) 1284311375Ssephe{ 1285311375Ssephe const struct vmbus_chan_pollarg *arg = xarg; 1286311375Ssephe struct vmbus_channel *chan = arg->poll_chan; 1287311375Ssephe sbintime_t intvl; 1288311375Ssephe int poll_flags; 1289311375Ssephe 1290311375Ssephe /* 1291311375Ssephe * Save polling interval. 1292311375Ssephe */ 1293311375Ssephe intvl = SBT_1S / arg->poll_hz; 1294311375Ssephe if (intvl == 0) 1295311375Ssephe intvl = 1; 1296311375Ssephe if (intvl == chan->ch_poll_intvl) { 1297311375Ssephe /* Nothing changes; done */ 1298311375Ssephe return; 1299311375Ssephe } 1300311375Ssephe chan->ch_poll_intvl = intvl; 1301311375Ssephe 1302311375Ssephe /* Adjust callout flags. */ 1303311375Ssephe poll_flags = C_DIRECT_EXEC; 1304311375Ssephe if (arg->poll_hz <= hz) 1305311375Ssephe poll_flags |= C_HARDCLOCK; 1306311375Ssephe chan->ch_poll_flags = poll_flags; 1307311375Ssephe 1308311375Ssephe /* 1309311389Ssephe * Disconnect this channel from the channel map to make sure that 1310311389Ssephe * the RX bufring interrupt enabling bit can not be touched, and 1311311389Ssephe * ISR can not enqueue this channel task anymore. THEN, disable 1312311389Ssephe * interrupt from the RX bufring (TX bufring does not generate 1313311389Ssephe * interrupt to VM). 1314311389Ssephe * 1315311389Ssephe * NOTE: order is critical. 1316311375Ssephe */ 1317311389Ssephe chan->ch_vmbus->vmbus_chmap[chan->ch_id] = NULL; 1318311389Ssephe __compiler_membar(); 1319311375Ssephe vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1320311375Ssephe 1321311375Ssephe /* 1322311375Ssephe * NOTE: 1323311375Ssephe * At this point, this channel task will not be enqueued by 1324311375Ssephe * the ISR anymore, time to cancel the pending one. 1325311375Ssephe */ 1326311375Ssephe taskqueue_cancel(chan->ch_tq, &chan->ch_task, NULL); 1327311375Ssephe 1328311375Ssephe /* Kick start! */ 1329311375Ssephe taskqueue_enqueue(chan->ch_tq, &chan->ch_poll_task); 1330311375Ssephe} 1331311375Ssephe 1332311375Ssephestatic bool 1333311375Ssephevmbus_chan_poll_cancel_intq(struct vmbus_channel *chan) 1334311375Ssephe{ 1335311375Ssephe 1336311375Ssephe if (chan->ch_poll_intvl == 0) { 1337311375Ssephe /* Not enabled. */ 1338311375Ssephe return (false); 1339311375Ssephe } 1340311375Ssephe 1341311375Ssephe /* 1342311375Ssephe * Stop polling callout, so that channel polling task 1343311375Ssephe * will not be enqueued anymore. 1344311375Ssephe */ 1345311375Ssephe callout_drain(&chan->ch_poll_timeo); 1346311375Ssephe 1347311375Ssephe /* 1348311375Ssephe * Disable polling by resetting polling interval. 1349311375Ssephe * 1350311375Ssephe * NOTE: 1351311375Ssephe * The polling interval resetting MUST be conducted 1352311375Ssephe * after the callout is drained; mainly to keep the 1353311375Ssephe * proper assertion in place. 1354311375Ssephe */ 1355311375Ssephe chan->ch_poll_intvl = 0; 1356311375Ssephe 1357311375Ssephe /* 1358311375Ssephe * NOTE: 1359311375Ssephe * At this point, this channel polling task will not be 1360311375Ssephe * enqueued by the callout anymore, time to cancel the 1361311375Ssephe * pending one. 1362311375Ssephe */ 1363311375Ssephe taskqueue_cancel(chan->ch_tq, &chan->ch_poll_task, NULL); 1364311375Ssephe 1365311375Ssephe /* Polling was enabled. */ 1366311375Ssephe return (true); 1367311375Ssephe} 1368311375Ssephe 1369311375Ssephestatic void 1370311375Ssephevmbus_chan_polldis_task(void *xchan, int pending __unused) 1371311375Ssephe{ 1372311375Ssephe struct vmbus_channel *chan = xchan; 1373311375Ssephe 1374311375Ssephe if (!vmbus_chan_poll_cancel_intq(chan)) { 1375311375Ssephe /* Already disabled; done. */ 1376311375Ssephe return; 1377311375Ssephe } 1378311375Ssephe 1379311375Ssephe /* 1380311375Ssephe * Plug this channel back to the channel map and unmask 1381311375Ssephe * the RX bufring interrupt. 1382311375Ssephe */ 1383311375Ssephe chan->ch_vmbus->vmbus_chmap[chan->ch_id] = chan; 1384311375Ssephe __compiler_membar(); 1385311375Ssephe vmbus_rxbr_intr_unmask(&chan->ch_rxbr); 1386311375Ssephe 1387311375Ssephe /* 1388311375Ssephe * Kick start the interrupt task, just in case unmasking 1389311375Ssephe * interrupt races ISR. 1390311375Ssephe */ 1391311375Ssephe taskqueue_enqueue(chan->ch_tq, &chan->ch_task); 1392311375Ssephe} 1393311375Ssephe 1394302692Ssephestatic __inline void 1395302692Ssephevmbus_event_flags_proc(struct vmbus_softc *sc, volatile u_long *event_flags, 1396302692Ssephe int flag_cnt) 1397302692Ssephe{ 1398302692Ssephe int f; 1399302692Ssephe 1400302692Ssephe for (f = 0; f < flag_cnt; ++f) { 1401302806Ssephe uint32_t chid_base; 1402302692Ssephe u_long flags; 1403302806Ssephe int chid_ofs; 1404302692Ssephe 1405302692Ssephe if (event_flags[f] == 0) 1406302692Ssephe continue; 1407302692Ssephe 1408302692Ssephe flags = atomic_swap_long(&event_flags[f], 0); 1409302806Ssephe chid_base = f << VMBUS_EVTFLAG_SHIFT; 1410302692Ssephe 1411302806Ssephe while ((chid_ofs = ffsl(flags)) != 0) { 1412307461Ssephe struct vmbus_channel *chan; 1413302692Ssephe 1414302806Ssephe --chid_ofs; /* NOTE: ffsl is 1-based */ 1415302806Ssephe flags &= ~(1UL << chid_ofs); 1416302692Ssephe 1417303022Ssephe chan = sc->vmbus_chmap[chid_base + chid_ofs]; 1418307599Ssephe if (__predict_false(chan == NULL)) { 1419307599Ssephe /* Channel is closed. */ 1420302692Ssephe continue; 1421307599Ssephe } 1422307599Ssephe __compiler_membar(); 1423302692Ssephe 1424303022Ssephe if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD) 1425307464Ssephe vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1426303022Ssephe taskqueue_enqueue(chan->ch_tq, &chan->ch_task); 1427302692Ssephe } 1428302692Ssephe } 1429302692Ssephe} 1430302692Ssephe 1431302692Ssephevoid 1432302692Ssephevmbus_event_proc(struct vmbus_softc *sc, int cpu) 1433302692Ssephe{ 1434302692Ssephe struct vmbus_evtflags *eventf; 1435302692Ssephe 1436302692Ssephe /* 1437302692Ssephe * On Host with Win8 or above, the event page can be checked directly 1438302692Ssephe * to get the id of the channel that has the pending interrupt. 1439302692Ssephe */ 1440302692Ssephe eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE; 1441302692Ssephe vmbus_event_flags_proc(sc, eventf->evt_flags, 1442302692Ssephe VMBUS_PCPU_GET(sc, event_flags_cnt, cpu)); 1443302692Ssephe} 1444302692Ssephe 1445302692Ssephevoid 1446302692Ssephevmbus_event_proc_compat(struct vmbus_softc *sc, int cpu) 1447302692Ssephe{ 1448302692Ssephe struct vmbus_evtflags *eventf; 1449302692Ssephe 1450302692Ssephe eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE; 1451302692Ssephe if (atomic_testandclear_long(&eventf->evt_flags[0], 0)) { 1452302692Ssephe vmbus_event_flags_proc(sc, sc->vmbus_rx_evtflags, 1453302692Ssephe VMBUS_CHAN_MAX_COMPAT >> VMBUS_EVTFLAG_SHIFT); 1454302692Ssephe } 1455302692Ssephe} 1456302692Ssephe 1457302692Ssephestatic void 1458302692Ssephevmbus_chan_update_evtflagcnt(struct vmbus_softc *sc, 1459307461Ssephe const struct vmbus_channel *chan) 1460302692Ssephe{ 1461302692Ssephe volatile int *flag_cnt_ptr; 1462302692Ssephe int flag_cnt; 1463302692Ssephe 1464302693Ssephe flag_cnt = (chan->ch_id / VMBUS_EVTFLAG_LEN) + 1; 1465302873Ssephe flag_cnt_ptr = VMBUS_PCPU_PTR(sc, event_flags_cnt, chan->ch_cpuid); 1466302692Ssephe 1467302692Ssephe for (;;) { 1468302692Ssephe int old_flag_cnt; 1469302692Ssephe 1470302692Ssephe old_flag_cnt = *flag_cnt_ptr; 1471302692Ssephe if (old_flag_cnt >= flag_cnt) 1472302692Ssephe break; 1473302692Ssephe if (atomic_cmpset_int(flag_cnt_ptr, old_flag_cnt, flag_cnt)) { 1474302692Ssephe if (bootverbose) { 1475308621Ssephe vmbus_chan_printf(chan, 1476308621Ssephe "chan%u update cpu%d flag_cnt to %d\n", 1477302873Ssephe chan->ch_id, chan->ch_cpuid, flag_cnt); 1478302692Ssephe } 1479302692Ssephe break; 1480302692Ssephe } 1481302692Ssephe } 1482302692Ssephe} 1483302864Ssephe 1484307461Ssephestatic struct vmbus_channel * 1485302864Ssephevmbus_chan_alloc(struct vmbus_softc *sc) 1486302864Ssephe{ 1487307461Ssephe struct vmbus_channel *chan; 1488302864Ssephe 1489302864Ssephe chan = malloc(sizeof(*chan), M_DEVBUF, M_WAITOK | M_ZERO); 1490302864Ssephe 1491302864Ssephe chan->ch_monprm = hyperv_dmamem_alloc(bus_get_dma_tag(sc->vmbus_dev), 1492302864Ssephe HYPERCALL_PARAM_ALIGN, 0, sizeof(struct hyperv_mon_param), 1493302864Ssephe &chan->ch_monprm_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO); 1494302864Ssephe if (chan->ch_monprm == NULL) { 1495302864Ssephe device_printf(sc->vmbus_dev, "monprm alloc failed\n"); 1496302864Ssephe free(chan, M_DEVBUF); 1497302864Ssephe return NULL; 1498302864Ssephe } 1499302864Ssephe 1500311359Ssephe chan->ch_refs = 1; 1501307461Ssephe chan->ch_vmbus = sc; 1502302864Ssephe mtx_init(&chan->ch_subchan_lock, "vmbus subchan", NULL, MTX_DEF); 1503311359Ssephe sx_init(&chan->ch_orphan_lock, "vmbus chorphan"); 1504302864Ssephe TAILQ_INIT(&chan->ch_subchans); 1505307464Ssephe vmbus_rxbr_init(&chan->ch_rxbr); 1506307464Ssephe vmbus_txbr_init(&chan->ch_txbr); 1507302864Ssephe 1508311375Ssephe TASK_INIT(&chan->ch_poll_task, 0, vmbus_chan_poll_task, chan); 1509311375Ssephe callout_init(&chan->ch_poll_timeo, 1); 1510311375Ssephe 1511302864Ssephe return chan; 1512302864Ssephe} 1513302864Ssephe 1514302864Ssephestatic void 1515307461Ssephevmbus_chan_free(struct vmbus_channel *chan) 1516302864Ssephe{ 1517307599Ssephe 1518307599Ssephe KASSERT(TAILQ_EMPTY(&chan->ch_subchans) && chan->ch_subchan_cnt == 0, 1519307599Ssephe ("still owns sub-channels")); 1520307599Ssephe KASSERT((chan->ch_stflags & 1521307599Ssephe (VMBUS_CHAN_ST_OPENED | 1522307599Ssephe VMBUS_CHAN_ST_ONPRIL | 1523307599Ssephe VMBUS_CHAN_ST_ONSUBL | 1524307599Ssephe VMBUS_CHAN_ST_ONLIST)) == 0, ("free busy channel")); 1525311359Ssephe KASSERT(chan->ch_orphan_xact == NULL, 1526311359Ssephe ("still has orphan xact installed")); 1527311359Ssephe KASSERT(chan->ch_refs == 0, ("chan%u: invalid refcnt %d", 1528311359Ssephe chan->ch_id, chan->ch_refs)); 1529311375Ssephe KASSERT(chan->ch_poll_intvl == 0, ("chan%u: polling is activated", 1530311375Ssephe chan->ch_id)); 1531311359Ssephe 1532302864Ssephe hyperv_dmamem_free(&chan->ch_monprm_dma, chan->ch_monprm); 1533302864Ssephe mtx_destroy(&chan->ch_subchan_lock); 1534311359Ssephe sx_destroy(&chan->ch_orphan_lock); 1535307464Ssephe vmbus_rxbr_deinit(&chan->ch_rxbr); 1536307464Ssephe vmbus_txbr_deinit(&chan->ch_txbr); 1537302864Ssephe free(chan, M_DEVBUF); 1538302864Ssephe} 1539302864Ssephe 1540302864Ssephestatic int 1541307461Ssephevmbus_chan_add(struct vmbus_channel *newchan) 1542302864Ssephe{ 1543307461Ssephe struct vmbus_softc *sc = newchan->ch_vmbus; 1544307461Ssephe struct vmbus_channel *prichan; 1545302864Ssephe 1546302864Ssephe if (newchan->ch_id == 0) { 1547302864Ssephe /* 1548302864Ssephe * XXX 1549302864Ssephe * Chan0 will neither be processed nor should be offered; 1550302864Ssephe * skip it. 1551302864Ssephe */ 1552302864Ssephe device_printf(sc->vmbus_dev, "got chan0 offer, discard\n"); 1553302864Ssephe return EINVAL; 1554302864Ssephe } else if (newchan->ch_id >= VMBUS_CHAN_MAX) { 1555302864Ssephe device_printf(sc->vmbus_dev, "invalid chan%u offer\n", 1556302864Ssephe newchan->ch_id); 1557302864Ssephe return EINVAL; 1558302864Ssephe } 1559302864Ssephe 1560302864Ssephe mtx_lock(&sc->vmbus_prichan_lock); 1561302864Ssephe TAILQ_FOREACH(prichan, &sc->vmbus_prichans, ch_prilink) { 1562302864Ssephe /* 1563302864Ssephe * Sub-channel will have the same type GUID and instance 1564302864Ssephe * GUID as its primary channel. 1565302864Ssephe */ 1566302864Ssephe if (memcmp(&prichan->ch_guid_type, &newchan->ch_guid_type, 1567302864Ssephe sizeof(struct hyperv_guid)) == 0 && 1568302864Ssephe memcmp(&prichan->ch_guid_inst, &newchan->ch_guid_inst, 1569302864Ssephe sizeof(struct hyperv_guid)) == 0) 1570302864Ssephe break; 1571302864Ssephe } 1572302864Ssephe if (VMBUS_CHAN_ISPRIMARY(newchan)) { 1573302864Ssephe if (prichan == NULL) { 1574302864Ssephe /* Install the new primary channel */ 1575307599Ssephe vmbus_chan_ins_prilist(sc, newchan); 1576302864Ssephe mtx_unlock(&sc->vmbus_prichan_lock); 1577307599Ssephe goto done; 1578302864Ssephe } else { 1579302864Ssephe mtx_unlock(&sc->vmbus_prichan_lock); 1580308621Ssephe device_printf(sc->vmbus_dev, 1581308621Ssephe "duplicated primary chan%u\n", newchan->ch_id); 1582302864Ssephe return EINVAL; 1583302864Ssephe } 1584302864Ssephe } else { /* Sub-channel */ 1585302864Ssephe if (prichan == NULL) { 1586302864Ssephe mtx_unlock(&sc->vmbus_prichan_lock); 1587308621Ssephe device_printf(sc->vmbus_dev, 1588308621Ssephe "no primary chan for chan%u\n", newchan->ch_id); 1589302864Ssephe return EINVAL; 1590302864Ssephe } 1591302864Ssephe /* 1592302864Ssephe * Found the primary channel for this sub-channel and 1593302864Ssephe * move on. 1594302864Ssephe * 1595302864Ssephe * XXX refcnt prichan 1596302864Ssephe */ 1597302864Ssephe } 1598302864Ssephe mtx_unlock(&sc->vmbus_prichan_lock); 1599302864Ssephe 1600302864Ssephe /* 1601302864Ssephe * This is a sub-channel; link it with the primary channel. 1602302864Ssephe */ 1603302864Ssephe KASSERT(!VMBUS_CHAN_ISPRIMARY(newchan), 1604302864Ssephe ("new channel is not sub-channel")); 1605302864Ssephe KASSERT(prichan != NULL, ("no primary channel")); 1606302864Ssephe 1607311359Ssephe /* 1608311359Ssephe * Reference count this sub-channel; it will be dereferenced 1609311359Ssephe * when this sub-channel is closed. 1610311359Ssephe */ 1611311359Ssephe KASSERT(newchan->ch_refs == 1, ("chan%u: invalid refcnt %d", 1612311359Ssephe newchan->ch_id, newchan->ch_refs)); 1613311359Ssephe atomic_add_int(&newchan->ch_refs, 1); 1614311359Ssephe 1615302864Ssephe newchan->ch_prichan = prichan; 1616302864Ssephe newchan->ch_dev = prichan->ch_dev; 1617302864Ssephe 1618302864Ssephe mtx_lock(&prichan->ch_subchan_lock); 1619307599Ssephe vmbus_chan_ins_sublist(prichan, newchan); 1620307599Ssephe mtx_unlock(&prichan->ch_subchan_lock); 1621302864Ssephe /* 1622307599Ssephe * Notify anyone that is interested in this sub-channel, 1623307599Ssephe * after this sub-channel is setup. 1624302864Ssephe */ 1625302864Ssephe wakeup(prichan); 1626307599Ssephedone: 1627307599Ssephe /* 1628311359Ssephe * Hook this channel up for later revocation. 1629307599Ssephe */ 1630307599Ssephe mtx_lock(&sc->vmbus_chan_lock); 1631307599Ssephe vmbus_chan_ins_list(sc, newchan); 1632307599Ssephe mtx_unlock(&sc->vmbus_chan_lock); 1633308621Ssephe 1634308621Ssephe if (bootverbose) { 1635308621Ssephe vmbus_chan_printf(newchan, "chan%u subidx%u offer\n", 1636308621Ssephe newchan->ch_id, newchan->ch_subidx); 1637308621Ssephe } 1638308621Ssephe 1639308621Ssephe /* Select default cpu for this channel. */ 1640308621Ssephe vmbus_chan_cpu_default(newchan); 1641308621Ssephe 1642302864Ssephe return 0; 1643302864Ssephe} 1644302864Ssephe 1645302864Ssephevoid 1646307461Ssephevmbus_chan_cpu_set(struct vmbus_channel *chan, int cpu) 1647302864Ssephe{ 1648302864Ssephe KASSERT(cpu >= 0 && cpu < mp_ncpus, ("invalid cpu %d", cpu)); 1649302864Ssephe 1650307461Ssephe if (chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WS2008 || 1651307461Ssephe chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WIN7) { 1652302864Ssephe /* Only cpu0 is supported */ 1653302864Ssephe cpu = 0; 1654302864Ssephe } 1655302864Ssephe 1656302873Ssephe chan->ch_cpuid = cpu; 1657307461Ssephe chan->ch_vcpuid = VMBUS_PCPU_GET(chan->ch_vmbus, vcpuid, cpu); 1658302864Ssephe 1659302864Ssephe if (bootverbose) { 1660308621Ssephe vmbus_chan_printf(chan, 1661308621Ssephe "chan%u assigned to cpu%u [vcpu%u]\n", 1662302873Ssephe chan->ch_id, chan->ch_cpuid, chan->ch_vcpuid); 1663302864Ssephe } 1664302864Ssephe} 1665302864Ssephe 1666302864Ssephevoid 1667307461Ssephevmbus_chan_cpu_rr(struct vmbus_channel *chan) 1668302864Ssephe{ 1669302864Ssephe static uint32_t vmbus_chan_nextcpu; 1670302864Ssephe int cpu; 1671302864Ssephe 1672302864Ssephe cpu = atomic_fetchadd_int(&vmbus_chan_nextcpu, 1) % mp_ncpus; 1673302890Ssephe vmbus_chan_cpu_set(chan, cpu); 1674302864Ssephe} 1675302864Ssephe 1676302864Ssephestatic void 1677307461Ssephevmbus_chan_cpu_default(struct vmbus_channel *chan) 1678302864Ssephe{ 1679302864Ssephe /* 1680302864Ssephe * By default, pin the channel to cpu0. Devices having 1681302864Ssephe * special channel-cpu mapping requirement should call 1682302890Ssephe * vmbus_chan_cpu_{set,rr}(). 1683302864Ssephe */ 1684302890Ssephe vmbus_chan_cpu_set(chan, 0); 1685302864Ssephe} 1686302864Ssephe 1687302864Ssephestatic void 1688302864Ssephevmbus_chan_msgproc_choffer(struct vmbus_softc *sc, 1689302864Ssephe const struct vmbus_message *msg) 1690302864Ssephe{ 1691302864Ssephe const struct vmbus_chanmsg_choffer *offer; 1692307461Ssephe struct vmbus_channel *chan; 1693307599Ssephe task_fn_t *detach_fn, *attach_fn; 1694302864Ssephe int error; 1695302864Ssephe 1696302864Ssephe offer = (const struct vmbus_chanmsg_choffer *)msg->msg_data; 1697302864Ssephe 1698302864Ssephe chan = vmbus_chan_alloc(sc); 1699302864Ssephe if (chan == NULL) { 1700302864Ssephe device_printf(sc->vmbus_dev, "allocate chan%u failed\n", 1701302864Ssephe offer->chm_chanid); 1702302864Ssephe return; 1703302864Ssephe } 1704302864Ssephe 1705302864Ssephe chan->ch_id = offer->chm_chanid; 1706302864Ssephe chan->ch_subidx = offer->chm_subidx; 1707302864Ssephe chan->ch_guid_type = offer->chm_chtype; 1708302864Ssephe chan->ch_guid_inst = offer->chm_chinst; 1709302864Ssephe 1710302864Ssephe /* Batch reading is on by default */ 1711302864Ssephe chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD; 1712302864Ssephe 1713302864Ssephe chan->ch_monprm->mp_connid = VMBUS_CONNID_EVENT; 1714302864Ssephe if (sc->vmbus_version != VMBUS_VERSION_WS2008) 1715302864Ssephe chan->ch_monprm->mp_connid = offer->chm_connid; 1716302864Ssephe 1717302864Ssephe if (offer->chm_flags1 & VMBUS_CHOFFER_FLAG1_HASMNF) { 1718307461Ssephe int trig_idx; 1719307461Ssephe 1720302864Ssephe /* 1721302864Ssephe * Setup MNF stuffs. 1722302864Ssephe */ 1723307461Ssephe chan->ch_txflags |= VMBUS_CHAN_TXF_HASMNF; 1724307461Ssephe 1725307461Ssephe trig_idx = offer->chm_montrig / VMBUS_MONTRIG_LEN; 1726307461Ssephe if (trig_idx >= VMBUS_MONTRIGS_MAX) 1727302864Ssephe panic("invalid monitor trigger %u", offer->chm_montrig); 1728307461Ssephe chan->ch_montrig = 1729307461Ssephe &sc->vmbus_mnf2->mnf_trigs[trig_idx].mt_pending; 1730307461Ssephe 1731302864Ssephe chan->ch_montrig_mask = 1732302864Ssephe 1 << (offer->chm_montrig % VMBUS_MONTRIG_LEN); 1733302864Ssephe } 1734302864Ssephe 1735307461Ssephe /* 1736307461Ssephe * Setup event flag. 1737307461Ssephe */ 1738307461Ssephe chan->ch_evtflag = 1739307461Ssephe &sc->vmbus_tx_evtflags[chan->ch_id >> VMBUS_EVTFLAG_SHIFT]; 1740307461Ssephe chan->ch_evtflag_mask = 1UL << (chan->ch_id & VMBUS_EVTFLAG_MASK); 1741307461Ssephe 1742307599Ssephe /* 1743307599Ssephe * Setup attach and detach tasks. 1744307599Ssephe */ 1745307599Ssephe if (VMBUS_CHAN_ISPRIMARY(chan)) { 1746307599Ssephe chan->ch_mgmt_tq = sc->vmbus_devtq; 1747307599Ssephe attach_fn = vmbus_prichan_attach_task; 1748307599Ssephe detach_fn = vmbus_prichan_detach_task; 1749307599Ssephe } else { 1750307599Ssephe chan->ch_mgmt_tq = sc->vmbus_subchtq; 1751307599Ssephe attach_fn = vmbus_subchan_attach_task; 1752307599Ssephe detach_fn = vmbus_subchan_detach_task; 1753307599Ssephe } 1754307599Ssephe TASK_INIT(&chan->ch_attach_task, 0, attach_fn, chan); 1755307599Ssephe TASK_INIT(&chan->ch_detach_task, 0, detach_fn, chan); 1756307599Ssephe 1757302864Ssephe error = vmbus_chan_add(chan); 1758302864Ssephe if (error) { 1759302864Ssephe device_printf(sc->vmbus_dev, "add chan%u failed: %d\n", 1760302864Ssephe chan->ch_id, error); 1761311359Ssephe atomic_subtract_int(&chan->ch_refs, 1); 1762302864Ssephe vmbus_chan_free(chan); 1763302864Ssephe return; 1764302864Ssephe } 1765307599Ssephe taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_attach_task); 1766302864Ssephe} 1767302864Ssephe 1768302864Ssephestatic void 1769302864Ssephevmbus_chan_msgproc_chrescind(struct vmbus_softc *sc, 1770302864Ssephe const struct vmbus_message *msg) 1771302864Ssephe{ 1772302864Ssephe const struct vmbus_chanmsg_chrescind *note; 1773307461Ssephe struct vmbus_channel *chan; 1774302864Ssephe 1775302864Ssephe note = (const struct vmbus_chanmsg_chrescind *)msg->msg_data; 1776302864Ssephe if (note->chm_chanid > VMBUS_CHAN_MAX) { 1777311359Ssephe device_printf(sc->vmbus_dev, "invalid revoked chan%u\n", 1778302864Ssephe note->chm_chanid); 1779302864Ssephe return; 1780302864Ssephe } 1781302864Ssephe 1782307599Ssephe /* 1783307599Ssephe * Find and remove the target channel from the channel list. 1784307599Ssephe */ 1785307599Ssephe mtx_lock(&sc->vmbus_chan_lock); 1786307599Ssephe TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) { 1787307599Ssephe if (chan->ch_id == note->chm_chanid) 1788307599Ssephe break; 1789307599Ssephe } 1790307599Ssephe if (chan == NULL) { 1791307599Ssephe mtx_unlock(&sc->vmbus_chan_lock); 1792307599Ssephe device_printf(sc->vmbus_dev, "chan%u is not offered\n", 1793307599Ssephe note->chm_chanid); 1794302864Ssephe return; 1795307599Ssephe } 1796307599Ssephe vmbus_chan_rem_list(sc, chan); 1797307599Ssephe mtx_unlock(&sc->vmbus_chan_lock); 1798302864Ssephe 1799307599Ssephe if (VMBUS_CHAN_ISPRIMARY(chan)) { 1800307599Ssephe /* 1801307599Ssephe * The target channel is a primary channel; remove the 1802307599Ssephe * target channel from the primary channel list now, 1803307599Ssephe * instead of later, so that it will not be found by 1804307599Ssephe * other sub-channel offers, which are processed in 1805307599Ssephe * this thread. 1806307599Ssephe */ 1807307599Ssephe mtx_lock(&sc->vmbus_prichan_lock); 1808307599Ssephe vmbus_chan_rem_prilist(sc, chan); 1809307599Ssephe mtx_unlock(&sc->vmbus_prichan_lock); 1810307599Ssephe } 1811307599Ssephe 1812311359Ssephe /* 1813311359Ssephe * NOTE: 1814311359Ssephe * The following processing order is critical: 1815311359Ssephe * Set the REVOKED state flag before orphaning the installed xact. 1816311359Ssephe */ 1817311359Ssephe 1818311359Ssephe if (atomic_testandset_int(&chan->ch_stflags, 1819311359Ssephe VMBUS_CHAN_ST_REVOKED_SHIFT)) 1820311359Ssephe panic("channel has already been revoked"); 1821311359Ssephe 1822311359Ssephe sx_xlock(&chan->ch_orphan_lock); 1823311359Ssephe if (chan->ch_orphan_xact != NULL) 1824311359Ssephe vmbus_xact_ctx_orphan(chan->ch_orphan_xact); 1825311359Ssephe sx_xunlock(&chan->ch_orphan_lock); 1826311359Ssephe 1827308621Ssephe if (bootverbose) 1828311359Ssephe vmbus_chan_printf(chan, "chan%u revoked\n", note->chm_chanid); 1829311359Ssephe vmbus_chan_detach(chan); 1830302864Ssephe} 1831302864Ssephe 1832307599Ssephestatic int 1833307599Ssephevmbus_chan_release(struct vmbus_channel *chan) 1834302864Ssephe{ 1835307599Ssephe struct vmbus_softc *sc = chan->ch_vmbus; 1836307599Ssephe struct vmbus_chanmsg_chfree *req; 1837307599Ssephe struct vmbus_msghc *mh; 1838307599Ssephe int error; 1839302864Ssephe 1840307599Ssephe mh = vmbus_msghc_get(sc, sizeof(*req)); 1841307599Ssephe if (mh == NULL) { 1842308621Ssephe vmbus_chan_printf(chan, 1843308621Ssephe "can not get msg hypercall for chfree(chan%u)\n", 1844308621Ssephe chan->ch_id); 1845307599Ssephe return (ENXIO); 1846307599Ssephe } 1847307599Ssephe 1848307599Ssephe req = vmbus_msghc_dataptr(mh); 1849307599Ssephe req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHFREE; 1850307599Ssephe req->chm_chanid = chan->ch_id; 1851307599Ssephe 1852307599Ssephe error = vmbus_msghc_exec_noresult(mh); 1853307599Ssephe vmbus_msghc_put(sc, mh); 1854307599Ssephe 1855307599Ssephe if (error) { 1856308621Ssephe vmbus_chan_printf(chan, 1857308621Ssephe "chfree(chan%u) msg hypercall exec failed: %d\n", 1858307599Ssephe chan->ch_id, error); 1859302864Ssephe } else { 1860308621Ssephe if (bootverbose) 1861308621Ssephe vmbus_chan_printf(chan, "chan%u freed\n", chan->ch_id); 1862307599Ssephe } 1863307599Ssephe return (error); 1864307599Ssephe} 1865302864Ssephe 1866307599Ssephestatic void 1867307599Ssephevmbus_prichan_detach_task(void *xchan, int pending __unused) 1868307599Ssephe{ 1869307599Ssephe struct vmbus_channel *chan = xchan; 1870302864Ssephe 1871307599Ssephe KASSERT(VMBUS_CHAN_ISPRIMARY(chan), 1872307599Ssephe ("chan%u is not primary channel", chan->ch_id)); 1873302864Ssephe 1874307599Ssephe /* Delete and detach the device associated with this channel. */ 1875307599Ssephe vmbus_delete_child(chan); 1876302864Ssephe 1877307599Ssephe /* Release this channel (back to vmbus). */ 1878307599Ssephe vmbus_chan_release(chan); 1879307599Ssephe 1880307599Ssephe /* Free this channel's resource. */ 1881307599Ssephe vmbus_chan_free(chan); 1882302864Ssephe} 1883302864Ssephe 1884307599Ssephestatic void 1885307599Ssephevmbus_subchan_detach_task(void *xchan, int pending __unused) 1886307599Ssephe{ 1887307599Ssephe struct vmbus_channel *chan = xchan; 1888307599Ssephe struct vmbus_channel *pri_chan = chan->ch_prichan; 1889307599Ssephe 1890307599Ssephe KASSERT(!VMBUS_CHAN_ISPRIMARY(chan), 1891307599Ssephe ("chan%u is primary channel", chan->ch_id)); 1892307599Ssephe 1893307599Ssephe /* Release this channel (back to vmbus). */ 1894307599Ssephe vmbus_chan_release(chan); 1895307599Ssephe 1896307599Ssephe /* Unlink from its primary channel's sub-channel list. */ 1897307599Ssephe mtx_lock(&pri_chan->ch_subchan_lock); 1898307599Ssephe vmbus_chan_rem_sublist(pri_chan, chan); 1899307599Ssephe mtx_unlock(&pri_chan->ch_subchan_lock); 1900307599Ssephe /* Notify anyone that is waiting for this sub-channel to vanish. */ 1901307599Ssephe wakeup(pri_chan); 1902307599Ssephe 1903307599Ssephe /* Free this channel's resource. */ 1904307599Ssephe vmbus_chan_free(chan); 1905307599Ssephe} 1906307599Ssephe 1907307599Ssephestatic void 1908307599Ssephevmbus_prichan_attach_task(void *xchan, int pending __unused) 1909307599Ssephe{ 1910307599Ssephe 1911307599Ssephe /* 1912307599Ssephe * Add device for this primary channel. 1913307599Ssephe */ 1914307599Ssephe vmbus_add_child(xchan); 1915307599Ssephe} 1916307599Ssephe 1917307599Ssephestatic void 1918307599Ssephevmbus_subchan_attach_task(void *xchan __unused, int pending __unused) 1919307599Ssephe{ 1920307599Ssephe 1921307599Ssephe /* Nothing */ 1922307599Ssephe} 1923307599Ssephe 1924302864Ssephevoid 1925302864Ssephevmbus_chan_destroy_all(struct vmbus_softc *sc) 1926302864Ssephe{ 1927302864Ssephe 1928307599Ssephe /* 1929307599Ssephe * Detach all devices and destroy the corresponding primary 1930307599Ssephe * channels. 1931307599Ssephe */ 1932307599Ssephe for (;;) { 1933307599Ssephe struct vmbus_channel *chan; 1934302864Ssephe 1935307599Ssephe mtx_lock(&sc->vmbus_chan_lock); 1936307599Ssephe TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) { 1937307599Ssephe if (VMBUS_CHAN_ISPRIMARY(chan)) 1938307599Ssephe break; 1939307599Ssephe } 1940307599Ssephe if (chan == NULL) { 1941307599Ssephe /* No more primary channels; done. */ 1942307599Ssephe mtx_unlock(&sc->vmbus_chan_lock); 1943307599Ssephe break; 1944307599Ssephe } 1945307599Ssephe vmbus_chan_rem_list(sc, chan); 1946307599Ssephe mtx_unlock(&sc->vmbus_chan_lock); 1947302864Ssephe 1948302864Ssephe mtx_lock(&sc->vmbus_prichan_lock); 1949307599Ssephe vmbus_chan_rem_prilist(sc, chan); 1950307599Ssephe mtx_unlock(&sc->vmbus_prichan_lock); 1951307599Ssephe 1952307599Ssephe taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task); 1953302864Ssephe } 1954302864Ssephe} 1955302864Ssephe 1956307461Ssephestruct vmbus_channel ** 1957307461Ssephevmbus_subchan_get(struct vmbus_channel *pri_chan, int subchan_cnt) 1958302864Ssephe{ 1959307461Ssephe struct vmbus_channel **ret, *chan; 1960302864Ssephe int i; 1961302864Ssephe 1962307510Ssephe KASSERT(subchan_cnt > 0, ("invalid sub-channel count %d", subchan_cnt)); 1963307510Ssephe 1964307461Ssephe ret = malloc(subchan_cnt * sizeof(struct vmbus_channel *), M_TEMP, 1965302864Ssephe M_WAITOK); 1966302864Ssephe 1967302864Ssephe mtx_lock(&pri_chan->ch_subchan_lock); 1968302864Ssephe 1969302864Ssephe while (pri_chan->ch_subchan_cnt < subchan_cnt) 1970302864Ssephe mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "subch", 0); 1971302864Ssephe 1972302864Ssephe i = 0; 1973302864Ssephe TAILQ_FOREACH(chan, &pri_chan->ch_subchans, ch_sublink) { 1974302864Ssephe /* TODO: refcnt chan */ 1975302864Ssephe ret[i] = chan; 1976302864Ssephe 1977302864Ssephe ++i; 1978302864Ssephe if (i == subchan_cnt) 1979302864Ssephe break; 1980302864Ssephe } 1981302864Ssephe KASSERT(i == subchan_cnt, ("invalid subchan count %d, should be %d", 1982302864Ssephe pri_chan->ch_subchan_cnt, subchan_cnt)); 1983302864Ssephe 1984302864Ssephe mtx_unlock(&pri_chan->ch_subchan_lock); 1985302864Ssephe 1986302864Ssephe return ret; 1987302864Ssephe} 1988302864Ssephe 1989302864Ssephevoid 1990307461Ssephevmbus_subchan_rel(struct vmbus_channel **subchan, int subchan_cnt __unused) 1991302864Ssephe{ 1992302864Ssephe 1993302864Ssephe free(subchan, M_TEMP); 1994302864Ssephe} 1995302864Ssephe 1996302864Ssephevoid 1997307461Ssephevmbus_subchan_drain(struct vmbus_channel *pri_chan) 1998302864Ssephe{ 1999302864Ssephe mtx_lock(&pri_chan->ch_subchan_lock); 2000302864Ssephe while (pri_chan->ch_subchan_cnt > 0) 2001302864Ssephe mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "dsubch", 0); 2002302864Ssephe mtx_unlock(&pri_chan->ch_subchan_lock); 2003302864Ssephe} 2004302864Ssephe 2005302864Ssephevoid 2006302864Ssephevmbus_chan_msgproc(struct vmbus_softc *sc, const struct vmbus_message *msg) 2007302864Ssephe{ 2008302864Ssephe vmbus_chanmsg_proc_t msg_proc; 2009302864Ssephe uint32_t msg_type; 2010302864Ssephe 2011302864Ssephe msg_type = ((const struct vmbus_chanmsg_hdr *)msg->msg_data)->chm_type; 2012302864Ssephe KASSERT(msg_type < VMBUS_CHANMSG_TYPE_MAX, 2013302864Ssephe ("invalid message type %u", msg_type)); 2014302864Ssephe 2015302864Ssephe msg_proc = vmbus_chan_msgprocs[msg_type]; 2016302864Ssephe if (msg_proc != NULL) 2017302864Ssephe msg_proc(sc, msg); 2018302864Ssephe} 2019303021Ssephe 2020303021Ssephevoid 2021307461Ssephevmbus_chan_set_readbatch(struct vmbus_channel *chan, bool on) 2022303021Ssephe{ 2023303021Ssephe if (!on) 2024303021Ssephe chan->ch_flags &= ~VMBUS_CHAN_FLAG_BATCHREAD; 2025303021Ssephe else 2026303021Ssephe chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD; 2027303021Ssephe} 2028307461Ssephe 2029307461Ssepheuint32_t 2030307461Ssephevmbus_chan_id(const struct vmbus_channel *chan) 2031307461Ssephe{ 2032307461Ssephe return chan->ch_id; 2033307461Ssephe} 2034307461Ssephe 2035307461Ssepheuint32_t 2036307461Ssephevmbus_chan_subidx(const struct vmbus_channel *chan) 2037307461Ssephe{ 2038307461Ssephe return chan->ch_subidx; 2039307461Ssephe} 2040307461Ssephe 2041307461Ssephebool 2042307461Ssephevmbus_chan_is_primary(const struct vmbus_channel *chan) 2043307461Ssephe{ 2044307461Ssephe if (VMBUS_CHAN_ISPRIMARY(chan)) 2045307461Ssephe return true; 2046307461Ssephe else 2047307461Ssephe return false; 2048307461Ssephe} 2049307461Ssephe 2050307461Ssepheconst struct hyperv_guid * 2051307461Ssephevmbus_chan_guid_inst(const struct vmbus_channel *chan) 2052307461Ssephe{ 2053307461Ssephe return &chan->ch_guid_inst; 2054307461Ssephe} 2055307486Ssephe 2056307486Ssepheint 2057307486Ssephevmbus_chan_prplist_nelem(int br_size, int prpcnt_max, int dlen_max) 2058307486Ssephe{ 2059307486Ssephe int elem_size; 2060307486Ssephe 2061307486Ssephe elem_size = __offsetof(struct vmbus_chanpkt_prplist, 2062307486Ssephe cp_range[0].gpa_page[prpcnt_max]); 2063307486Ssephe elem_size += dlen_max; 2064307486Ssephe elem_size = VMBUS_CHANPKT_TOTLEN(elem_size); 2065307486Ssephe 2066307486Ssephe return (vmbus_br_nelem(br_size, elem_size)); 2067307486Ssephe} 2068307599Ssephe 2069307599Ssephebool 2070307599Ssephevmbus_chan_tx_empty(const struct vmbus_channel *chan) 2071307599Ssephe{ 2072307599Ssephe 2073307599Ssephe return (vmbus_txbr_empty(&chan->ch_txbr)); 2074307599Ssephe} 2075307599Ssephe 2076307599Ssephebool 2077307599Ssephevmbus_chan_rx_empty(const struct vmbus_channel *chan) 2078307599Ssephe{ 2079307599Ssephe 2080307599Ssephe return (vmbus_rxbr_empty(&chan->ch_rxbr)); 2081307599Ssephe} 2082307614Ssephe 2083308621Ssephestatic int 2084308621Ssephevmbus_chan_printf(const struct vmbus_channel *chan, const char *fmt, ...) 2085308621Ssephe{ 2086308621Ssephe va_list ap; 2087308621Ssephe device_t dev; 2088308621Ssephe int retval; 2089308621Ssephe 2090308621Ssephe if (chan->ch_dev == NULL || !device_is_alive(chan->ch_dev)) 2091308621Ssephe dev = chan->ch_vmbus->vmbus_dev; 2092308621Ssephe else 2093308621Ssephe dev = chan->ch_dev; 2094308621Ssephe 2095308621Ssephe retval = device_print_prettyname(dev); 2096308621Ssephe va_start(ap, fmt); 2097308621Ssephe retval += vprintf(fmt, ap); 2098308621Ssephe va_end(ap); 2099308621Ssephe 2100308621Ssephe return (retval); 2101308621Ssephe} 2102308621Ssephe 2103307614Ssephevoid 2104307614Ssephevmbus_chan_run_task(struct vmbus_channel *chan, struct task *task) 2105307614Ssephe{ 2106307614Ssephe 2107307614Ssephe taskqueue_enqueue(chan->ch_tq, task); 2108307614Ssephe taskqueue_drain(chan->ch_tq, task); 2109307614Ssephe} 2110308517Ssephe 2111308517Ssephestruct taskqueue * 2112308517Ssephevmbus_chan_mgmt_tq(const struct vmbus_channel *chan) 2113308517Ssephe{ 2114308517Ssephe 2115308517Ssephe return (chan->ch_mgmt_tq); 2116308517Ssephe} 2117311359Ssephe 2118311359Ssephebool 2119311359Ssephevmbus_chan_is_revoked(const struct vmbus_channel *chan) 2120311359Ssephe{ 2121311359Ssephe 2122311359Ssephe if (chan->ch_stflags & VMBUS_CHAN_ST_REVOKED) 2123311359Ssephe return (true); 2124311359Ssephe return (false); 2125311359Ssephe} 2126311359Ssephe 2127311359Ssephevoid 2128311359Ssephevmbus_chan_set_orphan(struct vmbus_channel *chan, struct vmbus_xact_ctx *xact) 2129311359Ssephe{ 2130311359Ssephe 2131311359Ssephe sx_xlock(&chan->ch_orphan_lock); 2132311359Ssephe chan->ch_orphan_xact = xact; 2133311359Ssephe sx_xunlock(&chan->ch_orphan_lock); 2134311359Ssephe} 2135311359Ssephe 2136311359Ssephevoid 2137311359Ssephevmbus_chan_unset_orphan(struct vmbus_channel *chan) 2138311359Ssephe{ 2139311359Ssephe 2140311359Ssephe sx_xlock(&chan->ch_orphan_lock); 2141311359Ssephe chan->ch_orphan_xact = NULL; 2142311359Ssephe sx_xunlock(&chan->ch_orphan_lock); 2143311359Ssephe} 2144311364Ssephe 2145311364Ssepheconst void * 2146311364Ssephevmbus_chan_xact_wait(const struct vmbus_channel *chan, 2147311364Ssephe struct vmbus_xact *xact, size_t *resp_len, bool can_sleep) 2148311364Ssephe{ 2149311364Ssephe const void *ret; 2150311364Ssephe 2151311364Ssephe if (can_sleep) 2152311364Ssephe ret = vmbus_xact_wait(xact, resp_len); 2153311364Ssephe else 2154311364Ssephe ret = vmbus_xact_busywait(xact, resp_len); 2155311364Ssephe if (vmbus_chan_is_revoked(chan)) { 2156311364Ssephe /* 2157311364Ssephe * This xact probably is interrupted, and the 2158311364Ssephe * interruption can race the reply reception, 2159311364Ssephe * so we have to make sure that there are nothing 2160311364Ssephe * left on the RX bufring, i.e. this xact will 2161311364Ssephe * not be touched, once this function returns. 2162311364Ssephe * 2163311364Ssephe * Since the hypervisor will not put more data 2164311364Ssephe * onto the RX bufring once the channel is revoked, 2165311364Ssephe * the following loop will be terminated, once all 2166311364Ssephe * data are drained by the driver's channel 2167311364Ssephe * callback. 2168311364Ssephe */ 2169311364Ssephe while (!vmbus_chan_rx_empty(chan)) { 2170311364Ssephe if (can_sleep) 2171311364Ssephe pause("chxact", 1); 2172311364Ssephe else 2173311364Ssephe DELAY(1000); 2174311364Ssephe } 2175311364Ssephe } 2176311364Ssephe return (ret); 2177311364Ssephe} 2178311375Ssephe 2179311375Ssephevoid 2180311375Ssephevmbus_chan_poll_enable(struct vmbus_channel *chan, u_int pollhz) 2181311375Ssephe{ 2182311375Ssephe struct vmbus_chan_pollarg arg; 2183311375Ssephe struct task poll_cfg; 2184311375Ssephe 2185311375Ssephe KASSERT(chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD, 2186311375Ssephe ("enable polling on non-batch chan%u", chan->ch_id)); 2187311375Ssephe KASSERT(pollhz >= VMBUS_CHAN_POLLHZ_MIN && 2188311375Ssephe pollhz <= VMBUS_CHAN_POLLHZ_MAX, ("invalid pollhz %u", pollhz)); 2189311375Ssephe 2190311375Ssephe arg.poll_chan = chan; 2191311375Ssephe arg.poll_hz = pollhz; 2192311375Ssephe TASK_INIT(&poll_cfg, 0, vmbus_chan_pollcfg_task, &arg); 2193311375Ssephe vmbus_chan_run_task(chan, &poll_cfg); 2194311375Ssephe} 2195311375Ssephe 2196311375Ssephevoid 2197311375Ssephevmbus_chan_poll_disable(struct vmbus_channel *chan) 2198311375Ssephe{ 2199311375Ssephe struct task poll_dis; 2200311375Ssephe 2201311375Ssephe KASSERT(chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD, 2202311375Ssephe ("disable polling on non-batch chan%u", chan->ch_id)); 2203311375Ssephe 2204311375Ssephe TASK_INIT(&poll_dis, 0, vmbus_chan_polldis_task, chan); 2205311375Ssephe vmbus_chan_run_task(chan, &poll_dis); 2206311375Ssephe} 2207