vmbus_chan.c revision 307599
1250199Sgrehan/*- 2298446Ssephe * Copyright (c) 2009-2012,2016 Microsoft Corp. 3250199Sgrehan * Copyright (c) 2012 NetApp Inc. 4250199Sgrehan * Copyright (c) 2012 Citrix Inc. 5250199Sgrehan * All rights reserved. 6250199Sgrehan * 7250199Sgrehan * Redistribution and use in source and binary forms, with or without 8250199Sgrehan * modification, are permitted provided that the following conditions 9250199Sgrehan * are met: 10250199Sgrehan * 1. Redistributions of source code must retain the above copyright 11250199Sgrehan * notice unmodified, this list of conditions, and the following 12250199Sgrehan * disclaimer. 13250199Sgrehan * 2. Redistributions in binary form must reproduce the above copyright 14250199Sgrehan * notice, this list of conditions and the following disclaimer in the 15250199Sgrehan * documentation and/or other materials provided with the distribution. 16250199Sgrehan * 17250199Sgrehan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18250199Sgrehan * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19250199Sgrehan * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20250199Sgrehan * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21250199Sgrehan * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22250199Sgrehan * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23250199Sgrehan * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24250199Sgrehan * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25250199Sgrehan * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26250199Sgrehan * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27250199Sgrehan */ 28250199Sgrehan 29256276Sdim#include <sys/cdefs.h> 30256276Sdim__FBSDID("$FreeBSD: stable/11/sys/dev/hyperv/vmbus/vmbus_chan.c 307599 2016-10-19 02:24:17Z sephe $"); 31256276Sdim 32250199Sgrehan#include <sys/param.h> 33296028Ssephe#include <sys/kernel.h> 34307466Ssephe#include <sys/lock.h> 35250199Sgrehan#include <sys/malloc.h> 36250199Sgrehan#include <sys/mutex.h> 37307466Ssephe#include <sys/smp.h> 38296181Ssephe#include <sys/sysctl.h> 39307466Ssephe#include <sys/systm.h> 40301588Ssephe 41301588Ssephe#include <machine/atomic.h> 42301588Ssephe 43302872Ssephe#include <dev/hyperv/include/hyperv_busdma.h> 44302619Ssephe#include <dev/hyperv/vmbus/hyperv_var.h> 45301588Ssephe#include <dev/hyperv/vmbus/vmbus_reg.h> 46300102Ssephe#include <dev/hyperv/vmbus/vmbus_var.h> 47307463Ssephe#include <dev/hyperv/vmbus/vmbus_brvar.h> 48307463Ssephe#include <dev/hyperv/vmbus/vmbus_chanvar.h> 49250199Sgrehan 50307466Ssephestatic void vmbus_chan_update_evtflagcnt( 51307466Ssephe struct vmbus_softc *, 52307466Ssephe const struct vmbus_channel *); 53307466Ssephestatic void vmbus_chan_close_internal( 54307466Ssephe struct vmbus_channel *); 55307466Ssephestatic int vmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS); 56307466Ssephestatic void vmbus_chan_sysctl_create( 57307466Ssephe struct vmbus_channel *); 58307466Ssephestatic struct vmbus_channel *vmbus_chan_alloc(struct vmbus_softc *); 59307466Ssephestatic void vmbus_chan_free(struct vmbus_channel *); 60307466Ssephestatic int vmbus_chan_add(struct vmbus_channel *); 61307466Ssephestatic void vmbus_chan_cpu_default(struct vmbus_channel *); 62307599Ssephestatic int vmbus_chan_release(struct vmbus_channel *); 63307599Ssephestatic void vmbus_chan_set_chmap(struct vmbus_channel *); 64307599Ssephestatic void vmbus_chan_clear_chmap(struct vmbus_channel *); 65302864Ssephe 66307599Ssephestatic void vmbus_chan_ins_prilist(struct vmbus_softc *, 67307599Ssephe struct vmbus_channel *); 68307599Ssephestatic void vmbus_chan_rem_prilist(struct vmbus_softc *, 69307599Ssephe struct vmbus_channel *); 70307599Ssephestatic void vmbus_chan_ins_list(struct vmbus_softc *, 71307599Ssephe struct vmbus_channel *); 72307599Ssephestatic void vmbus_chan_rem_list(struct vmbus_softc *, 73307599Ssephe struct vmbus_channel *); 74307599Ssephestatic void vmbus_chan_ins_sublist(struct vmbus_channel *, 75307599Ssephe struct vmbus_channel *); 76307599Ssephestatic void vmbus_chan_rem_sublist(struct vmbus_channel *, 77307599Ssephe struct vmbus_channel *); 78307599Ssephe 79307466Ssephestatic void vmbus_chan_task(void *, int); 80307466Ssephestatic void vmbus_chan_task_nobatch(void *, int); 81307599Ssephestatic void vmbus_chan_clrchmap_task(void *, int); 82307599Ssephestatic void vmbus_prichan_attach_task(void *, int); 83307599Ssephestatic void vmbus_subchan_attach_task(void *, int); 84307599Ssephestatic void vmbus_prichan_detach_task(void *, int); 85307599Ssephestatic void vmbus_subchan_detach_task(void *, int); 86250199Sgrehan 87307466Ssephestatic void vmbus_chan_msgproc_choffer(struct vmbus_softc *, 88307466Ssephe const struct vmbus_message *); 89307466Ssephestatic void vmbus_chan_msgproc_chrescind( 90307466Ssephe struct vmbus_softc *, 91307466Ssephe const struct vmbus_message *); 92302864Ssephe 93302864Ssephe/* 94302864Ssephe * Vmbus channel message processing. 95302864Ssephe */ 96302864Ssephestatic const vmbus_chanmsg_proc_t 97302864Ssephevmbus_chan_msgprocs[VMBUS_CHANMSG_TYPE_MAX] = { 98302864Ssephe VMBUS_CHANMSG_PROC(CHOFFER, vmbus_chan_msgproc_choffer), 99302864Ssephe VMBUS_CHANMSG_PROC(CHRESCIND, vmbus_chan_msgproc_chrescind), 100302864Ssephe 101302864Ssephe VMBUS_CHANMSG_PROC_WAKEUP(CHOPEN_RESP), 102302864Ssephe VMBUS_CHANMSG_PROC_WAKEUP(GPADL_CONNRESP), 103302864Ssephe VMBUS_CHANMSG_PROC_WAKEUP(GPADL_DISCONNRESP) 104302864Ssephe}; 105302864Ssephe 106307461Ssephe/* 107307461Ssephe * Notify host that there are data pending on our TX bufring. 108250199Sgrehan */ 109307461Ssephestatic __inline void 110307461Ssephevmbus_chan_signal_tx(const struct vmbus_channel *chan) 111250199Sgrehan{ 112307461Ssephe atomic_set_long(chan->ch_evtflag, chan->ch_evtflag_mask); 113307461Ssephe if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF) 114307461Ssephe atomic_set_int(chan->ch_montrig, chan->ch_montrig_mask); 115307461Ssephe else 116303022Ssephe hypercall_signal_event(chan->ch_monprm_dma.hv_paddr); 117250199Sgrehan} 118250199Sgrehan 119307599Ssephestatic void 120307599Ssephevmbus_chan_ins_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan) 121307599Ssephe{ 122307599Ssephe 123307599Ssephe mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED); 124307599Ssephe if (atomic_testandset_int(&chan->ch_stflags, 125307599Ssephe VMBUS_CHAN_ST_ONPRIL_SHIFT)) 126307599Ssephe panic("channel is already on the prilist"); 127307599Ssephe TAILQ_INSERT_TAIL(&sc->vmbus_prichans, chan, ch_prilink); 128307599Ssephe} 129307599Ssephe 130307599Ssephestatic void 131307599Ssephevmbus_chan_rem_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan) 132307599Ssephe{ 133307599Ssephe 134307599Ssephe mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED); 135307599Ssephe if (atomic_testandclear_int(&chan->ch_stflags, 136307599Ssephe VMBUS_CHAN_ST_ONPRIL_SHIFT) == 0) 137307599Ssephe panic("channel is not on the prilist"); 138307599Ssephe TAILQ_REMOVE(&sc->vmbus_prichans, chan, ch_prilink); 139307599Ssephe} 140307599Ssephe 141307599Ssephestatic void 142307599Ssephevmbus_chan_ins_sublist(struct vmbus_channel *prichan, 143307599Ssephe struct vmbus_channel *chan) 144307599Ssephe{ 145307599Ssephe 146307599Ssephe mtx_assert(&prichan->ch_subchan_lock, MA_OWNED); 147307599Ssephe 148307599Ssephe if (atomic_testandset_int(&chan->ch_stflags, 149307599Ssephe VMBUS_CHAN_ST_ONSUBL_SHIFT)) 150307599Ssephe panic("channel is already on the sublist"); 151307599Ssephe TAILQ_INSERT_TAIL(&prichan->ch_subchans, chan, ch_sublink); 152307599Ssephe 153307599Ssephe /* Bump sub-channel count. */ 154307599Ssephe prichan->ch_subchan_cnt++; 155307599Ssephe} 156307599Ssephe 157307599Ssephestatic void 158307599Ssephevmbus_chan_rem_sublist(struct vmbus_channel *prichan, 159307599Ssephe struct vmbus_channel *chan) 160307599Ssephe{ 161307599Ssephe 162307599Ssephe mtx_assert(&prichan->ch_subchan_lock, MA_OWNED); 163307599Ssephe 164307599Ssephe KASSERT(prichan->ch_subchan_cnt > 0, 165307599Ssephe ("invalid subchan_cnt %d", prichan->ch_subchan_cnt)); 166307599Ssephe prichan->ch_subchan_cnt--; 167307599Ssephe 168307599Ssephe if (atomic_testandclear_int(&chan->ch_stflags, 169307599Ssephe VMBUS_CHAN_ST_ONSUBL_SHIFT) == 0) 170307599Ssephe panic("channel is not on the sublist"); 171307599Ssephe TAILQ_REMOVE(&prichan->ch_subchans, chan, ch_sublink); 172307599Ssephe} 173307599Ssephe 174307599Ssephestatic void 175307599Ssephevmbus_chan_ins_list(struct vmbus_softc *sc, struct vmbus_channel *chan) 176307599Ssephe{ 177307599Ssephe 178307599Ssephe mtx_assert(&sc->vmbus_chan_lock, MA_OWNED); 179307599Ssephe if (atomic_testandset_int(&chan->ch_stflags, 180307599Ssephe VMBUS_CHAN_ST_ONLIST_SHIFT)) 181307599Ssephe panic("channel is already on the list"); 182307599Ssephe TAILQ_INSERT_TAIL(&sc->vmbus_chans, chan, ch_link); 183307599Ssephe} 184307599Ssephe 185307599Ssephestatic void 186307599Ssephevmbus_chan_rem_list(struct vmbus_softc *sc, struct vmbus_channel *chan) 187307599Ssephe{ 188307599Ssephe 189307599Ssephe mtx_assert(&sc->vmbus_chan_lock, MA_OWNED); 190307599Ssephe if (atomic_testandclear_int(&chan->ch_stflags, 191307599Ssephe VMBUS_CHAN_ST_ONLIST_SHIFT) == 0) 192307599Ssephe panic("channel is not on the list"); 193307599Ssephe TAILQ_REMOVE(&sc->vmbus_chans, chan, ch_link); 194307599Ssephe} 195307599Ssephe 196296289Ssephestatic int 197302892Ssephevmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS) 198296289Ssephe{ 199307461Ssephe struct vmbus_channel *chan = arg1; 200302892Ssephe int mnf = 0; 201296289Ssephe 202307461Ssephe if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF) 203302892Ssephe mnf = 1; 204302892Ssephe return sysctl_handle_int(oidp, &mnf, 0, req); 205296289Ssephe} 206296289Ssephe 207296181Ssephestatic void 208307461Ssephevmbus_chan_sysctl_create(struct vmbus_channel *chan) 209296181Ssephe{ 210302892Ssephe struct sysctl_oid *ch_tree, *chid_tree, *br_tree; 211296181Ssephe struct sysctl_ctx_list *ctx; 212296181Ssephe uint32_t ch_id; 213296181Ssephe char name[16]; 214296181Ssephe 215302892Ssephe /* 216302892Ssephe * Add sysctl nodes related to this channel to this 217302892Ssephe * channel's sysctl ctx, so that they can be destroyed 218302892Ssephe * independently upon close of this channel, which can 219302892Ssephe * happen even if the device is not detached. 220302892Ssephe */ 221302892Ssephe ctx = &chan->ch_sysctl_ctx; 222302633Ssephe sysctl_ctx_init(ctx); 223302892Ssephe 224302892Ssephe /* 225302892Ssephe * Create dev.NAME.UNIT.channel tree. 226302892Ssephe */ 227302892Ssephe ch_tree = SYSCTL_ADD_NODE(ctx, 228302892Ssephe SYSCTL_CHILDREN(device_get_sysctl_tree(chan->ch_dev)), 229302892Ssephe OID_AUTO, "channel", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 230302892Ssephe if (ch_tree == NULL) 231302892Ssephe return; 232302892Ssephe 233302892Ssephe /* 234302892Ssephe * Create dev.NAME.UNIT.channel.CHANID tree. 235302892Ssephe */ 236302892Ssephe if (VMBUS_CHAN_ISPRIMARY(chan)) 237302892Ssephe ch_id = chan->ch_id; 238302892Ssephe else 239302892Ssephe ch_id = chan->ch_prichan->ch_id; 240296181Ssephe snprintf(name, sizeof(name), "%d", ch_id); 241302892Ssephe chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree), 242302892Ssephe OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 243302892Ssephe if (chid_tree == NULL) 244302892Ssephe return; 245296181Ssephe 246302892Ssephe if (!VMBUS_CHAN_ISPRIMARY(chan)) { 247302892Ssephe /* 248302892Ssephe * Create dev.NAME.UNIT.channel.CHANID.sub tree. 249302892Ssephe */ 250302892Ssephe ch_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree), 251302892Ssephe OID_AUTO, "sub", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 252302892Ssephe if (ch_tree == NULL) 253302892Ssephe return; 254296188Ssephe 255302892Ssephe /* 256302892Ssephe * Create dev.NAME.UNIT.channel.CHANID.sub.SUBIDX tree. 257302892Ssephe * 258302892Ssephe * NOTE: 259302892Ssephe * chid_tree is changed to this new sysctl tree. 260302892Ssephe */ 261302892Ssephe snprintf(name, sizeof(name), "%d", chan->ch_subidx); 262302892Ssephe chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree), 263302892Ssephe OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 264302892Ssephe if (chid_tree == NULL) 265302892Ssephe return; 266302892Ssephe 267302892Ssephe SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 268302892Ssephe "chanid", CTLFLAG_RD, &chan->ch_id, 0, "channel id"); 269296181Ssephe } 270296188Ssephe 271302892Ssephe SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 272302892Ssephe "cpu", CTLFLAG_RD, &chan->ch_cpuid, 0, "owner CPU id"); 273302892Ssephe SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 274302892Ssephe "mnf", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, 275302892Ssephe chan, 0, vmbus_chan_sysctl_mnf, "I", 276302892Ssephe "has monitor notification facilities"); 277302892Ssephe 278302892Ssephe br_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 279307462Ssephe "br", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 280302892Ssephe if (br_tree != NULL) { 281307462Ssephe /* 282307462Ssephe * Create sysctl tree for RX bufring. 283307462Ssephe */ 284307464Ssephe vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_rxbr.rxbr, "rx"); 285307462Ssephe /* 286307462Ssephe * Create sysctl tree for TX bufring. 287307462Ssephe */ 288307464Ssephe vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_txbr.txbr, "tx"); 289302892Ssephe } 290296181Ssephe} 291296290Ssephe 292250199Sgrehanint 293307461Ssephevmbus_chan_open(struct vmbus_channel *chan, int txbr_size, int rxbr_size, 294303021Ssephe const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg) 295250199Sgrehan{ 296307595Ssephe struct vmbus_chan_br cbr; 297307595Ssephe int error; 298307595Ssephe 299307595Ssephe /* 300307595Ssephe * Allocate the TX+RX bufrings. 301307595Ssephe */ 302307595Ssephe KASSERT(chan->ch_bufring == NULL, ("bufrings are allocated")); 303307595Ssephe chan->ch_bufring = hyperv_dmamem_alloc(bus_get_dma_tag(chan->ch_dev), 304307595Ssephe PAGE_SIZE, 0, txbr_size + rxbr_size, &chan->ch_bufring_dma, 305307595Ssephe BUS_DMA_WAITOK); 306307595Ssephe if (chan->ch_bufring == NULL) { 307307595Ssephe device_printf(chan->ch_dev, "bufring allocation failed\n"); 308307595Ssephe return (ENOMEM); 309307595Ssephe } 310307595Ssephe 311307595Ssephe cbr.cbr = chan->ch_bufring; 312307595Ssephe cbr.cbr_paddr = chan->ch_bufring_dma.hv_paddr; 313307595Ssephe cbr.cbr_txsz = txbr_size; 314307595Ssephe cbr.cbr_rxsz = rxbr_size; 315307595Ssephe 316307595Ssephe error = vmbus_chan_open_br(chan, &cbr, udata, udlen, cb, cbarg); 317307595Ssephe if (error) { 318307595Ssephe hyperv_dmamem_free(&chan->ch_bufring_dma, chan->ch_bufring); 319307595Ssephe chan->ch_bufring = NULL; 320307595Ssephe } 321307595Ssephe return (error); 322307595Ssephe} 323307595Ssephe 324307595Ssepheint 325307595Ssephevmbus_chan_open_br(struct vmbus_channel *chan, const struct vmbus_chan_br *cbr, 326307595Ssephe const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg) 327307595Ssephe{ 328307461Ssephe struct vmbus_softc *sc = chan->ch_vmbus; 329302607Ssephe const struct vmbus_chanmsg_chopen_resp *resp; 330302607Ssephe const struct vmbus_message *msg; 331302607Ssephe struct vmbus_chanmsg_chopen *req; 332302607Ssephe struct vmbus_msghc *mh; 333302607Ssephe uint32_t status; 334307595Ssephe int error, txbr_size, rxbr_size; 335307599Ssephe task_fn_t *task_fn; 336302872Ssephe uint8_t *br; 337250199Sgrehan 338302986Ssephe if (udlen > VMBUS_CHANMSG_CHOPEN_UDATA_SIZE) { 339302607Ssephe device_printf(sc->vmbus_dev, 340302986Ssephe "invalid udata len %d for chan%u\n", udlen, chan->ch_id); 341302607Ssephe return EINVAL; 342302607Ssephe } 343307595Ssephe 344307595Ssephe br = cbr->cbr; 345307595Ssephe txbr_size = cbr->cbr_txsz; 346307595Ssephe rxbr_size = cbr->cbr_rxsz; 347302986Ssephe KASSERT((txbr_size & PAGE_MASK) == 0, 348302872Ssephe ("send bufring size is not multiple page")); 349302986Ssephe KASSERT((rxbr_size & PAGE_MASK) == 0, 350302872Ssephe ("recv bufring size is not multiple page")); 351307595Ssephe KASSERT((cbr->cbr_paddr & PAGE_MASK) == 0, 352307595Ssephe ("bufring is not page aligned")); 353302607Ssephe 354307595Ssephe /* 355307595Ssephe * Zero out the TX/RX bufrings, in case that they were used before. 356307595Ssephe */ 357307595Ssephe memset(br, 0, txbr_size + rxbr_size); 358307595Ssephe 359302986Ssephe if (atomic_testandset_int(&chan->ch_stflags, 360302812Ssephe VMBUS_CHAN_ST_OPENED_SHIFT)) 361302986Ssephe panic("double-open chan%u", chan->ch_id); 362282212Swhu 363302986Ssephe chan->ch_cb = cb; 364302986Ssephe chan->ch_cbarg = cbarg; 365250199Sgrehan 366302986Ssephe vmbus_chan_update_evtflagcnt(sc, chan); 367300102Ssephe 368307461Ssephe chan->ch_tq = VMBUS_PCPU_GET(chan->ch_vmbus, event_tq, chan->ch_cpuid); 369302986Ssephe if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD) 370307599Ssephe task_fn = vmbus_chan_task; 371302986Ssephe else 372307599Ssephe task_fn = vmbus_chan_task_nobatch; 373307599Ssephe TASK_INIT(&chan->ch_task, 0, task_fn, chan); 374294886Ssephe 375302872Ssephe /* TX bufring comes first */ 376307464Ssephe vmbus_txbr_setup(&chan->ch_txbr, br, txbr_size); 377302872Ssephe /* RX bufring immediately follows TX bufring */ 378307464Ssephe vmbus_rxbr_setup(&chan->ch_rxbr, br + txbr_size, rxbr_size); 379250199Sgrehan 380296290Ssephe /* Create sysctl tree for this channel */ 381302986Ssephe vmbus_chan_sysctl_create(chan); 382296181Ssephe 383302872Ssephe /* 384302872Ssephe * Connect the bufrings, both RX and TX, to this channel. 385250199Sgrehan */ 386307595Ssephe error = vmbus_chan_gpadl_connect(chan, cbr->cbr_paddr, 387302986Ssephe txbr_size + rxbr_size, &chan->ch_bufring_gpadl); 388302986Ssephe if (error) { 389302872Ssephe device_printf(sc->vmbus_dev, 390302986Ssephe "failed to connect bufring GPADL to chan%u\n", chan->ch_id); 391302872Ssephe goto failed; 392302872Ssephe } 393250199Sgrehan 394302607Ssephe /* 395307599Ssephe * Install this channel, before it is opened, but after everything 396307599Ssephe * else has been setup. 397307599Ssephe */ 398307599Ssephe vmbus_chan_set_chmap(chan); 399307599Ssephe 400307599Ssephe /* 401302607Ssephe * Open channel w/ the bufring GPADL on the target CPU. 402250199Sgrehan */ 403302607Ssephe mh = vmbus_msghc_get(sc, sizeof(*req)); 404302607Ssephe if (mh == NULL) { 405302607Ssephe device_printf(sc->vmbus_dev, 406302607Ssephe "can not get msg hypercall for chopen(chan%u)\n", 407302986Ssephe chan->ch_id); 408302986Ssephe error = ENXIO; 409302812Ssephe goto failed; 410302607Ssephe } 411250199Sgrehan 412302607Ssephe req = vmbus_msghc_dataptr(mh); 413302607Ssephe req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHOPEN; 414302986Ssephe req->chm_chanid = chan->ch_id; 415302986Ssephe req->chm_openid = chan->ch_id; 416302986Ssephe req->chm_gpadl = chan->ch_bufring_gpadl; 417302986Ssephe req->chm_vcpuid = chan->ch_vcpuid; 418302986Ssephe req->chm_txbr_pgcnt = txbr_size >> PAGE_SHIFT; 419302986Ssephe if (udlen > 0) 420302986Ssephe memcpy(req->chm_udata, udata, udlen); 421250199Sgrehan 422302986Ssephe error = vmbus_msghc_exec(sc, mh); 423302986Ssephe if (error) { 424302607Ssephe device_printf(sc->vmbus_dev, 425302607Ssephe "chopen(chan%u) msg hypercall exec failed: %d\n", 426302986Ssephe chan->ch_id, error); 427302607Ssephe vmbus_msghc_put(sc, mh); 428302812Ssephe goto failed; 429302607Ssephe } 430250199Sgrehan 431302607Ssephe msg = vmbus_msghc_wait_result(sc, mh); 432302607Ssephe resp = (const struct vmbus_chanmsg_chopen_resp *)msg->msg_data; 433302607Ssephe status = resp->chm_status; 434250199Sgrehan 435302607Ssephe vmbus_msghc_put(sc, mh); 436250199Sgrehan 437302607Ssephe if (status == 0) { 438302607Ssephe if (bootverbose) { 439302607Ssephe device_printf(sc->vmbus_dev, "chan%u opened\n", 440302986Ssephe chan->ch_id); 441302607Ssephe } 442302812Ssephe return 0; 443250199Sgrehan } 444302812Ssephe 445302986Ssephe device_printf(sc->vmbus_dev, "failed to open chan%u\n", chan->ch_id); 446302986Ssephe error = ENXIO; 447302812Ssephe 448302812Ssephefailed: 449307599Ssephe vmbus_chan_clear_chmap(chan); 450302986Ssephe if (chan->ch_bufring_gpadl) { 451302986Ssephe vmbus_chan_gpadl_disconnect(chan, chan->ch_bufring_gpadl); 452302986Ssephe chan->ch_bufring_gpadl = 0; 453302872Ssephe } 454302986Ssephe atomic_clear_int(&chan->ch_stflags, VMBUS_CHAN_ST_OPENED); 455302986Ssephe return error; 456250199Sgrehan} 457250199Sgrehan 458302609Ssepheint 459307461Ssephevmbus_chan_gpadl_connect(struct vmbus_channel *chan, bus_addr_t paddr, 460302871Ssephe int size, uint32_t *gpadl0) 461302871Ssephe{ 462307461Ssephe struct vmbus_softc *sc = chan->ch_vmbus; 463302609Ssephe struct vmbus_msghc *mh; 464302609Ssephe struct vmbus_chanmsg_gpadl_conn *req; 465302609Ssephe const struct vmbus_message *msg; 466302609Ssephe size_t reqsz; 467302609Ssephe uint32_t gpadl, status; 468302609Ssephe int page_count, range_len, i, cnt, error; 469302871Ssephe uint64_t page_id; 470250199Sgrehan 471302609Ssephe /* 472302609Ssephe * Preliminary checks. 473302609Ssephe */ 474250199Sgrehan 475302609Ssephe KASSERT((size & PAGE_MASK) == 0, 476302871Ssephe ("invalid GPA size %d, not multiple page size", size)); 477250199Sgrehan page_count = size >> PAGE_SHIFT; 478250199Sgrehan 479302609Ssephe KASSERT((paddr & PAGE_MASK) == 0, 480302609Ssephe ("GPA is not page aligned %jx", (uintmax_t)paddr)); 481302609Ssephe page_id = paddr >> PAGE_SHIFT; 482250199Sgrehan 483302609Ssephe range_len = __offsetof(struct vmbus_gpa_range, gpa_page[page_count]); 484302609Ssephe /* 485302609Ssephe * We don't support multiple GPA ranges. 486302609Ssephe */ 487302609Ssephe if (range_len > UINT16_MAX) { 488302609Ssephe device_printf(sc->vmbus_dev, "GPA too large, %d pages\n", 489302609Ssephe page_count); 490302609Ssephe return EOPNOTSUPP; 491250199Sgrehan } 492250199Sgrehan 493302609Ssephe /* 494302609Ssephe * Allocate GPADL id. 495302609Ssephe */ 496302630Ssephe gpadl = vmbus_gpadl_alloc(sc); 497302609Ssephe *gpadl0 = gpadl; 498250199Sgrehan 499302609Ssephe /* 500302609Ssephe * Connect this GPADL to the target channel. 501302609Ssephe * 502302609Ssephe * NOTE: 503302609Ssephe * Since each message can only hold small set of page 504302609Ssephe * addresses, several messages may be required to 505302609Ssephe * complete the connection. 506302609Ssephe */ 507302609Ssephe if (page_count > VMBUS_CHANMSG_GPADL_CONN_PGMAX) 508302609Ssephe cnt = VMBUS_CHANMSG_GPADL_CONN_PGMAX; 509302609Ssephe else 510302609Ssephe cnt = page_count; 511302609Ssephe page_count -= cnt; 512250199Sgrehan 513302609Ssephe reqsz = __offsetof(struct vmbus_chanmsg_gpadl_conn, 514302609Ssephe chm_range.gpa_page[cnt]); 515302609Ssephe mh = vmbus_msghc_get(sc, reqsz); 516302609Ssephe if (mh == NULL) { 517302609Ssephe device_printf(sc->vmbus_dev, 518302609Ssephe "can not get msg hypercall for gpadl->chan%u\n", 519302871Ssephe chan->ch_id); 520302609Ssephe return EIO; 521250199Sgrehan } 522250199Sgrehan 523302609Ssephe req = vmbus_msghc_dataptr(mh); 524302609Ssephe req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_CONN; 525302871Ssephe req->chm_chanid = chan->ch_id; 526302609Ssephe req->chm_gpadl = gpadl; 527302609Ssephe req->chm_range_len = range_len; 528302609Ssephe req->chm_range_cnt = 1; 529302609Ssephe req->chm_range.gpa_len = size; 530302609Ssephe req->chm_range.gpa_ofs = 0; 531302609Ssephe for (i = 0; i < cnt; ++i) 532302609Ssephe req->chm_range.gpa_page[i] = page_id++; 533250199Sgrehan 534302609Ssephe error = vmbus_msghc_exec(sc, mh); 535302609Ssephe if (error) { 536302609Ssephe device_printf(sc->vmbus_dev, 537302609Ssephe "gpadl->chan%u msg hypercall exec failed: %d\n", 538302871Ssephe chan->ch_id, error); 539302609Ssephe vmbus_msghc_put(sc, mh); 540302609Ssephe return error; 541302609Ssephe } 542250199Sgrehan 543302609Ssephe while (page_count > 0) { 544302609Ssephe struct vmbus_chanmsg_gpadl_subconn *subreq; 545250199Sgrehan 546302609Ssephe if (page_count > VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX) 547302609Ssephe cnt = VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX; 548302609Ssephe else 549302609Ssephe cnt = page_count; 550302609Ssephe page_count -= cnt; 551250199Sgrehan 552302609Ssephe reqsz = __offsetof(struct vmbus_chanmsg_gpadl_subconn, 553302609Ssephe chm_gpa_page[cnt]); 554302609Ssephe vmbus_msghc_reset(mh, reqsz); 555250199Sgrehan 556302609Ssephe subreq = vmbus_msghc_dataptr(mh); 557302609Ssephe subreq->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_SUBCONN; 558302609Ssephe subreq->chm_gpadl = gpadl; 559302609Ssephe for (i = 0; i < cnt; ++i) 560302609Ssephe subreq->chm_gpa_page[i] = page_id++; 561250199Sgrehan 562302609Ssephe vmbus_msghc_exec_noresult(mh); 563250199Sgrehan } 564302609Ssephe KASSERT(page_count == 0, ("invalid page count %d", page_count)); 565250199Sgrehan 566302609Ssephe msg = vmbus_msghc_wait_result(sc, mh); 567302609Ssephe status = ((const struct vmbus_chanmsg_gpadl_connresp *) 568302609Ssephe msg->msg_data)->chm_status; 569250199Sgrehan 570302609Ssephe vmbus_msghc_put(sc, mh); 571250199Sgrehan 572302609Ssephe if (status != 0) { 573302609Ssephe device_printf(sc->vmbus_dev, "gpadl->chan%u failed: " 574302871Ssephe "status %u\n", chan->ch_id, status); 575302609Ssephe return EIO; 576302632Ssephe } else { 577302632Ssephe if (bootverbose) { 578302632Ssephe device_printf(sc->vmbus_dev, "gpadl->chan%u " 579302871Ssephe "succeeded\n", chan->ch_id); 580302632Ssephe } 581302609Ssephe } 582302609Ssephe return 0; 583250199Sgrehan} 584250199Sgrehan 585302611Ssephe/* 586302611Ssephe * Disconnect the GPA from the target channel 587250199Sgrehan */ 588250199Sgrehanint 589307461Ssephevmbus_chan_gpadl_disconnect(struct vmbus_channel *chan, uint32_t gpadl) 590250199Sgrehan{ 591307461Ssephe struct vmbus_softc *sc = chan->ch_vmbus; 592302611Ssephe struct vmbus_msghc *mh; 593302611Ssephe struct vmbus_chanmsg_gpadl_disconn *req; 594302611Ssephe int error; 595250199Sgrehan 596302611Ssephe mh = vmbus_msghc_get(sc, sizeof(*req)); 597302611Ssephe if (mh == NULL) { 598302611Ssephe device_printf(sc->vmbus_dev, 599302611Ssephe "can not get msg hypercall for gpa x->chan%u\n", 600302693Ssephe chan->ch_id); 601302611Ssephe return EBUSY; 602250199Sgrehan } 603250199Sgrehan 604302611Ssephe req = vmbus_msghc_dataptr(mh); 605302611Ssephe req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_DISCONN; 606302693Ssephe req->chm_chanid = chan->ch_id; 607302611Ssephe req->chm_gpadl = gpadl; 608250199Sgrehan 609302611Ssephe error = vmbus_msghc_exec(sc, mh); 610302611Ssephe if (error) { 611302611Ssephe device_printf(sc->vmbus_dev, 612302611Ssephe "gpa x->chan%u msg hypercall exec failed: %d\n", 613302693Ssephe chan->ch_id, error); 614302611Ssephe vmbus_msghc_put(sc, mh); 615302611Ssephe return error; 616302611Ssephe } 617250199Sgrehan 618302611Ssephe vmbus_msghc_wait_result(sc, mh); 619302611Ssephe /* Discard result; no useful information */ 620302611Ssephe vmbus_msghc_put(sc, mh); 621250199Sgrehan 622302611Ssephe return 0; 623250199Sgrehan} 624250199Sgrehan 625282212Swhustatic void 626307599Ssephevmbus_chan_clrchmap_task(void *xchan, int pending __unused) 627307599Ssephe{ 628307599Ssephe struct vmbus_channel *chan = xchan; 629307599Ssephe 630307599Ssephe critical_enter(); 631307599Ssephe chan->ch_vmbus->vmbus_chmap[chan->ch_id] = NULL; 632307599Ssephe critical_exit(); 633307599Ssephe} 634307599Ssephe 635307599Ssephestatic void 636307599Ssephevmbus_chan_clear_chmap(struct vmbus_channel *chan) 637307599Ssephe{ 638307599Ssephe struct task chmap_task; 639307599Ssephe 640307599Ssephe TASK_INIT(&chmap_task, 0, vmbus_chan_clrchmap_task, chan); 641307599Ssephe taskqueue_enqueue(chan->ch_tq, &chmap_task); 642307599Ssephe taskqueue_drain(chan->ch_tq, &chmap_task); 643307599Ssephe} 644307599Ssephe 645307599Ssephestatic void 646307599Ssephevmbus_chan_set_chmap(struct vmbus_channel *chan) 647307599Ssephe{ 648307599Ssephe __compiler_membar(); 649307599Ssephe chan->ch_vmbus->vmbus_chmap[chan->ch_id] = chan; 650307599Ssephe} 651307599Ssephe 652307599Ssephestatic void 653307461Ssephevmbus_chan_close_internal(struct vmbus_channel *chan) 654250199Sgrehan{ 655307461Ssephe struct vmbus_softc *sc = chan->ch_vmbus; 656302610Ssephe struct vmbus_msghc *mh; 657302610Ssephe struct vmbus_chanmsg_chclose *req; 658302610Ssephe int error; 659250199Sgrehan 660302812Ssephe /* TODO: stringent check */ 661302891Ssephe atomic_clear_int(&chan->ch_stflags, VMBUS_CHAN_ST_OPENED); 662302812Ssephe 663302891Ssephe /* 664302891Ssephe * Free this channel's sysctl tree attached to its device's 665302891Ssephe * sysctl tree. 666302891Ssephe */ 667302891Ssephe sysctl_ctx_free(&chan->ch_sysctl_ctx); 668282212Swhu 669282212Swhu /* 670307599Ssephe * NOTE: 671307599Ssephe * Order is critical. This channel _must_ be uninstalled first, 672307599Ssephe * else the channel task may be enqueued by the IDT after it has 673307599Ssephe * been drained. 674294886Ssephe */ 675307599Ssephe vmbus_chan_clear_chmap(chan); 676307599Ssephe taskqueue_drain(chan->ch_tq, &chan->ch_task); 677302891Ssephe chan->ch_tq = NULL; 678250199Sgrehan 679302891Ssephe /* 680302891Ssephe * Close this channel. 681250199Sgrehan */ 682302610Ssephe mh = vmbus_msghc_get(sc, sizeof(*req)); 683302610Ssephe if (mh == NULL) { 684302610Ssephe device_printf(sc->vmbus_dev, 685302610Ssephe "can not get msg hypercall for chclose(chan%u)\n", 686302891Ssephe chan->ch_id); 687302610Ssephe return; 688302610Ssephe } 689250199Sgrehan 690302610Ssephe req = vmbus_msghc_dataptr(mh); 691302610Ssephe req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHCLOSE; 692302891Ssephe req->chm_chanid = chan->ch_id; 693250199Sgrehan 694302610Ssephe error = vmbus_msghc_exec_noresult(mh); 695302610Ssephe vmbus_msghc_put(sc, mh); 696302610Ssephe 697302610Ssephe if (error) { 698302610Ssephe device_printf(sc->vmbus_dev, 699302610Ssephe "chclose(chan%u) msg hypercall exec failed: %d\n", 700302891Ssephe chan->ch_id, error); 701302610Ssephe return; 702302610Ssephe } else if (bootverbose) { 703302891Ssephe device_printf(sc->vmbus_dev, "close chan%u\n", chan->ch_id); 704302610Ssephe } 705302610Ssephe 706302891Ssephe /* 707302891Ssephe * Disconnect the TX+RX bufrings from this channel. 708302891Ssephe */ 709302891Ssephe if (chan->ch_bufring_gpadl) { 710302891Ssephe vmbus_chan_gpadl_disconnect(chan, chan->ch_bufring_gpadl); 711302891Ssephe chan->ch_bufring_gpadl = 0; 712250199Sgrehan } 713250199Sgrehan 714302891Ssephe /* 715302891Ssephe * Destroy the TX+RX bufrings. 716302891Ssephe */ 717302891Ssephe if (chan->ch_bufring != NULL) { 718302891Ssephe hyperv_dmamem_free(&chan->ch_bufring_dma, chan->ch_bufring); 719302891Ssephe chan->ch_bufring = NULL; 720302872Ssephe } 721282212Swhu} 722250199Sgrehan 723302818Ssephe/* 724302818Ssephe * Caller should make sure that all sub-channels have 725302818Ssephe * been added to 'chan' and all to-be-closed channels 726302818Ssephe * are not being opened. 727282212Swhu */ 728282212Swhuvoid 729307461Ssephevmbus_chan_close(struct vmbus_channel *chan) 730282212Swhu{ 731302818Ssephe int subchan_cnt; 732282212Swhu 733302818Ssephe if (!VMBUS_CHAN_ISPRIMARY(chan)) { 734282212Swhu /* 735302818Ssephe * Sub-channel is closed when its primary channel 736302818Ssephe * is closed; done. 737282212Swhu */ 738282212Swhu return; 739282212Swhu } 740282212Swhu 741250199Sgrehan /* 742302818Ssephe * Close all sub-channels, if any. 743250199Sgrehan */ 744302819Ssephe subchan_cnt = chan->ch_subchan_cnt; 745302818Ssephe if (subchan_cnt > 0) { 746307461Ssephe struct vmbus_channel **subchan; 747302818Ssephe int i; 748302818Ssephe 749302890Ssephe subchan = vmbus_subchan_get(chan, subchan_cnt); 750302818Ssephe for (i = 0; i < subchan_cnt; ++i) 751302891Ssephe vmbus_chan_close_internal(subchan[i]); 752302890Ssephe vmbus_subchan_rel(subchan, subchan_cnt); 753250199Sgrehan } 754302818Ssephe 755302818Ssephe /* Then close the primary channel. */ 756302891Ssephe vmbus_chan_close_internal(chan); 757250199Sgrehan} 758250199Sgrehan 759307599Ssephevoid 760307599Ssephevmbus_chan_intr_drain(struct vmbus_channel *chan) 761307599Ssephe{ 762307599Ssephe 763307599Ssephe taskqueue_drain(chan->ch_tq, &chan->ch_task); 764307599Ssephe} 765307599Ssephe 766250199Sgrehanint 767307461Ssephevmbus_chan_send(struct vmbus_channel *chan, uint16_t type, uint16_t flags, 768302882Ssephe void *data, int dlen, uint64_t xactid) 769250199Sgrehan{ 770302875Ssephe struct vmbus_chanpkt pkt; 771302881Ssephe int pktlen, pad_pktlen, hlen, error; 772302881Ssephe uint64_t pad = 0; 773302881Ssephe struct iovec iov[3]; 774302881Ssephe boolean_t send_evt; 775250199Sgrehan 776302881Ssephe hlen = sizeof(pkt); 777302881Ssephe pktlen = hlen + dlen; 778302884Ssephe pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 779307471Ssephe KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 780307471Ssephe ("invalid packet size %d", pad_pktlen)); 781250199Sgrehan 782302875Ssephe pkt.cp_hdr.cph_type = type; 783302875Ssephe pkt.cp_hdr.cph_flags = flags; 784302884Ssephe VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 785302884Ssephe VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 786302881Ssephe pkt.cp_hdr.cph_xactid = xactid; 787250199Sgrehan 788302875Ssephe iov[0].iov_base = &pkt; 789302881Ssephe iov[0].iov_len = hlen; 790302881Ssephe iov[1].iov_base = data; 791302881Ssephe iov[1].iov_len = dlen; 792302881Ssephe iov[2].iov_base = &pad; 793302881Ssephe iov[2].iov_len = pad_pktlen - pktlen; 794250199Sgrehan 795307464Ssephe error = vmbus_txbr_write(&chan->ch_txbr, iov, 3, &send_evt); 796302881Ssephe if (!error && send_evt) 797303022Ssephe vmbus_chan_signal_tx(chan); 798302881Ssephe return error; 799250199Sgrehan} 800250199Sgrehan 801250199Sgrehanint 802307461Ssephevmbus_chan_send_sglist(struct vmbus_channel *chan, 803302876Ssephe struct vmbus_gpa sg[], int sglen, void *data, int dlen, uint64_t xactid) 804250199Sgrehan{ 805302876Ssephe struct vmbus_chanpkt_sglist pkt; 806302876Ssephe int pktlen, pad_pktlen, hlen, error; 807302876Ssephe struct iovec iov[4]; 808302876Ssephe boolean_t send_evt; 809302876Ssephe uint64_t pad = 0; 810250199Sgrehan 811302876Ssephe hlen = __offsetof(struct vmbus_chanpkt_sglist, cp_gpa[sglen]); 812302876Ssephe pktlen = hlen + dlen; 813302884Ssephe pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 814307471Ssephe KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 815307471Ssephe ("invalid packet size %d", pad_pktlen)); 816250199Sgrehan 817302880Ssephe pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 818302879Ssephe pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 819302884Ssephe VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 820302884Ssephe VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 821302876Ssephe pkt.cp_hdr.cph_xactid = xactid; 822302876Ssephe pkt.cp_rsvd = 0; 823302876Ssephe pkt.cp_gpa_cnt = sglen; 824250199Sgrehan 825302876Ssephe iov[0].iov_base = &pkt; 826302876Ssephe iov[0].iov_len = sizeof(pkt); 827302876Ssephe iov[1].iov_base = sg; 828302876Ssephe iov[1].iov_len = sizeof(struct vmbus_gpa) * sglen; 829302876Ssephe iov[2].iov_base = data; 830302876Ssephe iov[2].iov_len = dlen; 831302876Ssephe iov[3].iov_base = &pad; 832302876Ssephe iov[3].iov_len = pad_pktlen - pktlen; 833250199Sgrehan 834307464Ssephe error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); 835302876Ssephe if (!error && send_evt) 836303022Ssephe vmbus_chan_signal_tx(chan); 837302876Ssephe return error; 838250199Sgrehan} 839250199Sgrehan 840250199Sgrehanint 841307461Ssephevmbus_chan_send_prplist(struct vmbus_channel *chan, 842302878Ssephe struct vmbus_gpa_range *prp, int prp_cnt, void *data, int dlen, 843302878Ssephe uint64_t xactid) 844250199Sgrehan{ 845302878Ssephe struct vmbus_chanpkt_prplist pkt; 846302878Ssephe int pktlen, pad_pktlen, hlen, error; 847302878Ssephe struct iovec iov[4]; 848302878Ssephe boolean_t send_evt; 849302878Ssephe uint64_t pad = 0; 850250199Sgrehan 851302878Ssephe hlen = __offsetof(struct vmbus_chanpkt_prplist, 852302878Ssephe cp_range[0].gpa_page[prp_cnt]); 853302878Ssephe pktlen = hlen + dlen; 854302884Ssephe pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 855307471Ssephe KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 856307471Ssephe ("invalid packet size %d", pad_pktlen)); 857250199Sgrehan 858302880Ssephe pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 859302879Ssephe pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 860302884Ssephe VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 861302884Ssephe VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 862302878Ssephe pkt.cp_hdr.cph_xactid = xactid; 863302878Ssephe pkt.cp_rsvd = 0; 864302878Ssephe pkt.cp_range_cnt = 1; 865250199Sgrehan 866302878Ssephe iov[0].iov_base = &pkt; 867302878Ssephe iov[0].iov_len = sizeof(pkt); 868302878Ssephe iov[1].iov_base = prp; 869302878Ssephe iov[1].iov_len = __offsetof(struct vmbus_gpa_range, gpa_page[prp_cnt]); 870302878Ssephe iov[2].iov_base = data; 871302878Ssephe iov[2].iov_len = dlen; 872302878Ssephe iov[3].iov_base = &pad; 873302878Ssephe iov[3].iov_len = pad_pktlen - pktlen; 874250199Sgrehan 875307464Ssephe error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); 876302878Ssephe if (!error && send_evt) 877303022Ssephe vmbus_chan_signal_tx(chan); 878302878Ssephe return error; 879250199Sgrehan} 880250199Sgrehan 881250199Sgrehanint 882307461Ssephevmbus_chan_recv(struct vmbus_channel *chan, void *data, int *dlen0, 883302885Ssephe uint64_t *xactid) 884250199Sgrehan{ 885302885Ssephe struct vmbus_chanpkt_hdr pkt; 886302885Ssephe int error, dlen, hlen; 887250199Sgrehan 888307464Ssephe error = vmbus_rxbr_peek(&chan->ch_rxbr, &pkt, sizeof(pkt)); 889302885Ssephe if (error) 890307498Ssephe return (error); 891250199Sgrehan 892307498Ssephe if (__predict_false(pkt.cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) { 893307498Ssephe device_printf(chan->ch_dev, "invalid hlen %u\n", 894307498Ssephe pkt.cph_hlen); 895307498Ssephe /* XXX this channel is dead actually. */ 896307498Ssephe return (EIO); 897307498Ssephe } 898307498Ssephe if (__predict_false(pkt.cph_hlen > pkt.cph_tlen)) { 899307498Ssephe device_printf(chan->ch_dev, "invalid hlen %u and tlen %u\n", 900307498Ssephe pkt.cph_hlen, pkt.cph_tlen); 901307498Ssephe /* XXX this channel is dead actually. */ 902307498Ssephe return (EIO); 903307498Ssephe } 904307498Ssephe 905302885Ssephe hlen = VMBUS_CHANPKT_GETLEN(pkt.cph_hlen); 906302885Ssephe dlen = VMBUS_CHANPKT_GETLEN(pkt.cph_tlen) - hlen; 907250199Sgrehan 908302885Ssephe if (*dlen0 < dlen) { 909302886Ssephe /* Return the size of this packet's data. */ 910302885Ssephe *dlen0 = dlen; 911307498Ssephe return (ENOBUFS); 912302885Ssephe } 913250199Sgrehan 914302885Ssephe *xactid = pkt.cph_xactid; 915302885Ssephe *dlen0 = dlen; 916250199Sgrehan 917302886Ssephe /* Skip packet header */ 918307464Ssephe error = vmbus_rxbr_read(&chan->ch_rxbr, data, dlen, hlen); 919307464Ssephe KASSERT(!error, ("vmbus_rxbr_read failed")); 920250199Sgrehan 921307498Ssephe return (0); 922250199Sgrehan} 923250199Sgrehan 924250199Sgrehanint 925307461Ssephevmbus_chan_recv_pkt(struct vmbus_channel *chan, 926302886Ssephe struct vmbus_chanpkt_hdr *pkt0, int *pktlen0) 927250199Sgrehan{ 928302886Ssephe struct vmbus_chanpkt_hdr pkt; 929302886Ssephe int error, pktlen; 930250199Sgrehan 931307464Ssephe error = vmbus_rxbr_peek(&chan->ch_rxbr, &pkt, sizeof(pkt)); 932302886Ssephe if (error) 933307498Ssephe return (error); 934250199Sgrehan 935307498Ssephe if (__predict_false(pkt.cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) { 936307498Ssephe device_printf(chan->ch_dev, "invalid hlen %u\n", 937307498Ssephe pkt.cph_hlen); 938307498Ssephe /* XXX this channel is dead actually. */ 939307498Ssephe return (EIO); 940307498Ssephe } 941307498Ssephe if (__predict_false(pkt.cph_hlen > pkt.cph_tlen)) { 942307498Ssephe device_printf(chan->ch_dev, "invalid hlen %u and tlen %u\n", 943307498Ssephe pkt.cph_hlen, pkt.cph_tlen); 944307498Ssephe /* XXX this channel is dead actually. */ 945307498Ssephe return (EIO); 946307498Ssephe } 947307498Ssephe 948302886Ssephe pktlen = VMBUS_CHANPKT_GETLEN(pkt.cph_tlen); 949302886Ssephe if (*pktlen0 < pktlen) { 950302886Ssephe /* Return the size of this packet. */ 951302886Ssephe *pktlen0 = pktlen; 952307498Ssephe return (ENOBUFS); 953302886Ssephe } 954302886Ssephe *pktlen0 = pktlen; 955250199Sgrehan 956302886Ssephe /* Include packet header */ 957307464Ssephe error = vmbus_rxbr_read(&chan->ch_rxbr, pkt0, pktlen, 0); 958307464Ssephe KASSERT(!error, ("vmbus_rxbr_read failed")); 959250199Sgrehan 960307498Ssephe return (0); 961250199Sgrehan} 962294886Ssephe 963294886Ssephestatic void 964302713Ssephevmbus_chan_task(void *xchan, int pending __unused) 965294886Ssephe{ 966307461Ssephe struct vmbus_channel *chan = xchan; 967302874Ssephe vmbus_chan_callback_t cb = chan->ch_cb; 968302874Ssephe void *cbarg = chan->ch_cbarg; 969294886Ssephe 970302710Ssephe /* 971302710Ssephe * Optimize host to guest signaling by ensuring: 972302710Ssephe * 1. While reading the channel, we disable interrupts from 973302710Ssephe * host. 974302710Ssephe * 2. Ensure that we process all posted messages from the host 975302710Ssephe * before returning from this callback. 976302710Ssephe * 3. Once we return, enable signaling from the host. Once this 977302710Ssephe * state is set we check to see if additional packets are 978302710Ssephe * available to read. In this case we repeat the process. 979302713Ssephe * 980302713Ssephe * NOTE: Interrupt has been disabled in the ISR. 981302710Ssephe */ 982302713Ssephe for (;;) { 983302713Ssephe uint32_t left; 984294886Ssephe 985307461Ssephe cb(chan, cbarg); 986294886Ssephe 987307464Ssephe left = vmbus_rxbr_intr_unmask(&chan->ch_rxbr); 988302713Ssephe if (left == 0) { 989302713Ssephe /* No more data in RX bufring; done */ 990302713Ssephe break; 991302713Ssephe } 992307464Ssephe vmbus_rxbr_intr_mask(&chan->ch_rxbr); 993302713Ssephe } 994294886Ssephe} 995302692Ssephe 996302713Ssephestatic void 997302713Ssephevmbus_chan_task_nobatch(void *xchan, int pending __unused) 998302713Ssephe{ 999307461Ssephe struct vmbus_channel *chan = xchan; 1000302713Ssephe 1001307461Ssephe chan->ch_cb(chan, chan->ch_cbarg); 1002302713Ssephe} 1003302713Ssephe 1004302692Ssephestatic __inline void 1005302692Ssephevmbus_event_flags_proc(struct vmbus_softc *sc, volatile u_long *event_flags, 1006302692Ssephe int flag_cnt) 1007302692Ssephe{ 1008302692Ssephe int f; 1009302692Ssephe 1010302692Ssephe for (f = 0; f < flag_cnt; ++f) { 1011302806Ssephe uint32_t chid_base; 1012302692Ssephe u_long flags; 1013302806Ssephe int chid_ofs; 1014302692Ssephe 1015302692Ssephe if (event_flags[f] == 0) 1016302692Ssephe continue; 1017302692Ssephe 1018302692Ssephe flags = atomic_swap_long(&event_flags[f], 0); 1019302806Ssephe chid_base = f << VMBUS_EVTFLAG_SHIFT; 1020302692Ssephe 1021302806Ssephe while ((chid_ofs = ffsl(flags)) != 0) { 1022307461Ssephe struct vmbus_channel *chan; 1023302692Ssephe 1024302806Ssephe --chid_ofs; /* NOTE: ffsl is 1-based */ 1025302806Ssephe flags &= ~(1UL << chid_ofs); 1026302692Ssephe 1027303022Ssephe chan = sc->vmbus_chmap[chid_base + chid_ofs]; 1028307599Ssephe if (__predict_false(chan == NULL)) { 1029307599Ssephe /* Channel is closed. */ 1030302692Ssephe continue; 1031307599Ssephe } 1032307599Ssephe __compiler_membar(); 1033302692Ssephe 1034303022Ssephe if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD) 1035307464Ssephe vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1036303022Ssephe taskqueue_enqueue(chan->ch_tq, &chan->ch_task); 1037302692Ssephe } 1038302692Ssephe } 1039302692Ssephe} 1040302692Ssephe 1041302692Ssephevoid 1042302692Ssephevmbus_event_proc(struct vmbus_softc *sc, int cpu) 1043302692Ssephe{ 1044302692Ssephe struct vmbus_evtflags *eventf; 1045302692Ssephe 1046302692Ssephe /* 1047302692Ssephe * On Host with Win8 or above, the event page can be checked directly 1048302692Ssephe * to get the id of the channel that has the pending interrupt. 1049302692Ssephe */ 1050302692Ssephe eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE; 1051302692Ssephe vmbus_event_flags_proc(sc, eventf->evt_flags, 1052302692Ssephe VMBUS_PCPU_GET(sc, event_flags_cnt, cpu)); 1053302692Ssephe} 1054302692Ssephe 1055302692Ssephevoid 1056302692Ssephevmbus_event_proc_compat(struct vmbus_softc *sc, int cpu) 1057302692Ssephe{ 1058302692Ssephe struct vmbus_evtflags *eventf; 1059302692Ssephe 1060302692Ssephe eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE; 1061302692Ssephe if (atomic_testandclear_long(&eventf->evt_flags[0], 0)) { 1062302692Ssephe vmbus_event_flags_proc(sc, sc->vmbus_rx_evtflags, 1063302692Ssephe VMBUS_CHAN_MAX_COMPAT >> VMBUS_EVTFLAG_SHIFT); 1064302692Ssephe } 1065302692Ssephe} 1066302692Ssephe 1067302692Ssephestatic void 1068302692Ssephevmbus_chan_update_evtflagcnt(struct vmbus_softc *sc, 1069307461Ssephe const struct vmbus_channel *chan) 1070302692Ssephe{ 1071302692Ssephe volatile int *flag_cnt_ptr; 1072302692Ssephe int flag_cnt; 1073302692Ssephe 1074302693Ssephe flag_cnt = (chan->ch_id / VMBUS_EVTFLAG_LEN) + 1; 1075302873Ssephe flag_cnt_ptr = VMBUS_PCPU_PTR(sc, event_flags_cnt, chan->ch_cpuid); 1076302692Ssephe 1077302692Ssephe for (;;) { 1078302692Ssephe int old_flag_cnt; 1079302692Ssephe 1080302692Ssephe old_flag_cnt = *flag_cnt_ptr; 1081302692Ssephe if (old_flag_cnt >= flag_cnt) 1082302692Ssephe break; 1083302692Ssephe if (atomic_cmpset_int(flag_cnt_ptr, old_flag_cnt, flag_cnt)) { 1084302692Ssephe if (bootverbose) { 1085302692Ssephe device_printf(sc->vmbus_dev, 1086302692Ssephe "channel%u update cpu%d flag_cnt to %d\n", 1087302873Ssephe chan->ch_id, chan->ch_cpuid, flag_cnt); 1088302692Ssephe } 1089302692Ssephe break; 1090302692Ssephe } 1091302692Ssephe } 1092302692Ssephe} 1093302864Ssephe 1094307461Ssephestatic struct vmbus_channel * 1095302864Ssephevmbus_chan_alloc(struct vmbus_softc *sc) 1096302864Ssephe{ 1097307461Ssephe struct vmbus_channel *chan; 1098302864Ssephe 1099302864Ssephe chan = malloc(sizeof(*chan), M_DEVBUF, M_WAITOK | M_ZERO); 1100302864Ssephe 1101302864Ssephe chan->ch_monprm = hyperv_dmamem_alloc(bus_get_dma_tag(sc->vmbus_dev), 1102302864Ssephe HYPERCALL_PARAM_ALIGN, 0, sizeof(struct hyperv_mon_param), 1103302864Ssephe &chan->ch_monprm_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO); 1104302864Ssephe if (chan->ch_monprm == NULL) { 1105302864Ssephe device_printf(sc->vmbus_dev, "monprm alloc failed\n"); 1106302864Ssephe free(chan, M_DEVBUF); 1107302864Ssephe return NULL; 1108302864Ssephe } 1109302864Ssephe 1110307461Ssephe chan->ch_vmbus = sc; 1111302864Ssephe mtx_init(&chan->ch_subchan_lock, "vmbus subchan", NULL, MTX_DEF); 1112302864Ssephe TAILQ_INIT(&chan->ch_subchans); 1113307464Ssephe vmbus_rxbr_init(&chan->ch_rxbr); 1114307464Ssephe vmbus_txbr_init(&chan->ch_txbr); 1115302864Ssephe 1116302864Ssephe return chan; 1117302864Ssephe} 1118302864Ssephe 1119302864Ssephestatic void 1120307461Ssephevmbus_chan_free(struct vmbus_channel *chan) 1121302864Ssephe{ 1122307599Ssephe 1123307599Ssephe KASSERT(TAILQ_EMPTY(&chan->ch_subchans) && chan->ch_subchan_cnt == 0, 1124307599Ssephe ("still owns sub-channels")); 1125307599Ssephe KASSERT((chan->ch_stflags & 1126307599Ssephe (VMBUS_CHAN_ST_OPENED | 1127307599Ssephe VMBUS_CHAN_ST_ONPRIL | 1128307599Ssephe VMBUS_CHAN_ST_ONSUBL | 1129307599Ssephe VMBUS_CHAN_ST_ONLIST)) == 0, ("free busy channel")); 1130302864Ssephe hyperv_dmamem_free(&chan->ch_monprm_dma, chan->ch_monprm); 1131302864Ssephe mtx_destroy(&chan->ch_subchan_lock); 1132307464Ssephe vmbus_rxbr_deinit(&chan->ch_rxbr); 1133307464Ssephe vmbus_txbr_deinit(&chan->ch_txbr); 1134302864Ssephe free(chan, M_DEVBUF); 1135302864Ssephe} 1136302864Ssephe 1137302864Ssephestatic int 1138307461Ssephevmbus_chan_add(struct vmbus_channel *newchan) 1139302864Ssephe{ 1140307461Ssephe struct vmbus_softc *sc = newchan->ch_vmbus; 1141307461Ssephe struct vmbus_channel *prichan; 1142302864Ssephe 1143302864Ssephe if (newchan->ch_id == 0) { 1144302864Ssephe /* 1145302864Ssephe * XXX 1146302864Ssephe * Chan0 will neither be processed nor should be offered; 1147302864Ssephe * skip it. 1148302864Ssephe */ 1149302864Ssephe device_printf(sc->vmbus_dev, "got chan0 offer, discard\n"); 1150302864Ssephe return EINVAL; 1151302864Ssephe } else if (newchan->ch_id >= VMBUS_CHAN_MAX) { 1152302864Ssephe device_printf(sc->vmbus_dev, "invalid chan%u offer\n", 1153302864Ssephe newchan->ch_id); 1154302864Ssephe return EINVAL; 1155302864Ssephe } 1156302864Ssephe 1157302864Ssephe if (bootverbose) { 1158302864Ssephe device_printf(sc->vmbus_dev, "chan%u subidx%u offer\n", 1159302864Ssephe newchan->ch_id, newchan->ch_subidx); 1160302864Ssephe } 1161302864Ssephe 1162302864Ssephe mtx_lock(&sc->vmbus_prichan_lock); 1163302864Ssephe TAILQ_FOREACH(prichan, &sc->vmbus_prichans, ch_prilink) { 1164302864Ssephe /* 1165302864Ssephe * Sub-channel will have the same type GUID and instance 1166302864Ssephe * GUID as its primary channel. 1167302864Ssephe */ 1168302864Ssephe if (memcmp(&prichan->ch_guid_type, &newchan->ch_guid_type, 1169302864Ssephe sizeof(struct hyperv_guid)) == 0 && 1170302864Ssephe memcmp(&prichan->ch_guid_inst, &newchan->ch_guid_inst, 1171302864Ssephe sizeof(struct hyperv_guid)) == 0) 1172302864Ssephe break; 1173302864Ssephe } 1174302864Ssephe if (VMBUS_CHAN_ISPRIMARY(newchan)) { 1175302864Ssephe if (prichan == NULL) { 1176302864Ssephe /* Install the new primary channel */ 1177307599Ssephe vmbus_chan_ins_prilist(sc, newchan); 1178302864Ssephe mtx_unlock(&sc->vmbus_prichan_lock); 1179307599Ssephe goto done; 1180302864Ssephe } else { 1181302864Ssephe mtx_unlock(&sc->vmbus_prichan_lock); 1182302864Ssephe device_printf(sc->vmbus_dev, "duplicated primary " 1183302864Ssephe "chan%u\n", newchan->ch_id); 1184302864Ssephe return EINVAL; 1185302864Ssephe } 1186302864Ssephe } else { /* Sub-channel */ 1187302864Ssephe if (prichan == NULL) { 1188302864Ssephe mtx_unlock(&sc->vmbus_prichan_lock); 1189302864Ssephe device_printf(sc->vmbus_dev, "no primary chan for " 1190302864Ssephe "chan%u\n", newchan->ch_id); 1191302864Ssephe return EINVAL; 1192302864Ssephe } 1193302864Ssephe /* 1194302864Ssephe * Found the primary channel for this sub-channel and 1195302864Ssephe * move on. 1196302864Ssephe * 1197302864Ssephe * XXX refcnt prichan 1198302864Ssephe */ 1199302864Ssephe } 1200302864Ssephe mtx_unlock(&sc->vmbus_prichan_lock); 1201302864Ssephe 1202302864Ssephe /* 1203302864Ssephe * This is a sub-channel; link it with the primary channel. 1204302864Ssephe */ 1205302864Ssephe KASSERT(!VMBUS_CHAN_ISPRIMARY(newchan), 1206302864Ssephe ("new channel is not sub-channel")); 1207302864Ssephe KASSERT(prichan != NULL, ("no primary channel")); 1208302864Ssephe 1209302864Ssephe newchan->ch_prichan = prichan; 1210302864Ssephe newchan->ch_dev = prichan->ch_dev; 1211302864Ssephe 1212302864Ssephe mtx_lock(&prichan->ch_subchan_lock); 1213307599Ssephe vmbus_chan_ins_sublist(prichan, newchan); 1214307599Ssephe mtx_unlock(&prichan->ch_subchan_lock); 1215302864Ssephe /* 1216307599Ssephe * Notify anyone that is interested in this sub-channel, 1217307599Ssephe * after this sub-channel is setup. 1218302864Ssephe */ 1219302864Ssephe wakeup(prichan); 1220307599Ssephedone: 1221307599Ssephe /* 1222307599Ssephe * Hook this channel up for later rescind. 1223307599Ssephe */ 1224307599Ssephe mtx_lock(&sc->vmbus_chan_lock); 1225307599Ssephe vmbus_chan_ins_list(sc, newchan); 1226307599Ssephe mtx_unlock(&sc->vmbus_chan_lock); 1227302864Ssephe return 0; 1228302864Ssephe} 1229302864Ssephe 1230302864Ssephevoid 1231307461Ssephevmbus_chan_cpu_set(struct vmbus_channel *chan, int cpu) 1232302864Ssephe{ 1233302864Ssephe KASSERT(cpu >= 0 && cpu < mp_ncpus, ("invalid cpu %d", cpu)); 1234302864Ssephe 1235307461Ssephe if (chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WS2008 || 1236307461Ssephe chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WIN7) { 1237302864Ssephe /* Only cpu0 is supported */ 1238302864Ssephe cpu = 0; 1239302864Ssephe } 1240302864Ssephe 1241302873Ssephe chan->ch_cpuid = cpu; 1242307461Ssephe chan->ch_vcpuid = VMBUS_PCPU_GET(chan->ch_vmbus, vcpuid, cpu); 1243302864Ssephe 1244302864Ssephe if (bootverbose) { 1245302864Ssephe printf("vmbus_chan%u: assigned to cpu%u [vcpu%u]\n", 1246302873Ssephe chan->ch_id, chan->ch_cpuid, chan->ch_vcpuid); 1247302864Ssephe } 1248302864Ssephe} 1249302864Ssephe 1250302864Ssephevoid 1251307461Ssephevmbus_chan_cpu_rr(struct vmbus_channel *chan) 1252302864Ssephe{ 1253302864Ssephe static uint32_t vmbus_chan_nextcpu; 1254302864Ssephe int cpu; 1255302864Ssephe 1256302864Ssephe cpu = atomic_fetchadd_int(&vmbus_chan_nextcpu, 1) % mp_ncpus; 1257302890Ssephe vmbus_chan_cpu_set(chan, cpu); 1258302864Ssephe} 1259302864Ssephe 1260302864Ssephestatic void 1261307461Ssephevmbus_chan_cpu_default(struct vmbus_channel *chan) 1262302864Ssephe{ 1263302864Ssephe /* 1264302864Ssephe * By default, pin the channel to cpu0. Devices having 1265302864Ssephe * special channel-cpu mapping requirement should call 1266302890Ssephe * vmbus_chan_cpu_{set,rr}(). 1267302864Ssephe */ 1268302890Ssephe vmbus_chan_cpu_set(chan, 0); 1269302864Ssephe} 1270302864Ssephe 1271302864Ssephestatic void 1272302864Ssephevmbus_chan_msgproc_choffer(struct vmbus_softc *sc, 1273302864Ssephe const struct vmbus_message *msg) 1274302864Ssephe{ 1275302864Ssephe const struct vmbus_chanmsg_choffer *offer; 1276307461Ssephe struct vmbus_channel *chan; 1277307599Ssephe task_fn_t *detach_fn, *attach_fn; 1278302864Ssephe int error; 1279302864Ssephe 1280302864Ssephe offer = (const struct vmbus_chanmsg_choffer *)msg->msg_data; 1281302864Ssephe 1282302864Ssephe chan = vmbus_chan_alloc(sc); 1283302864Ssephe if (chan == NULL) { 1284302864Ssephe device_printf(sc->vmbus_dev, "allocate chan%u failed\n", 1285302864Ssephe offer->chm_chanid); 1286302864Ssephe return; 1287302864Ssephe } 1288302864Ssephe 1289302864Ssephe chan->ch_id = offer->chm_chanid; 1290302864Ssephe chan->ch_subidx = offer->chm_subidx; 1291302864Ssephe chan->ch_guid_type = offer->chm_chtype; 1292302864Ssephe chan->ch_guid_inst = offer->chm_chinst; 1293302864Ssephe 1294302864Ssephe /* Batch reading is on by default */ 1295302864Ssephe chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD; 1296302864Ssephe 1297302864Ssephe chan->ch_monprm->mp_connid = VMBUS_CONNID_EVENT; 1298302864Ssephe if (sc->vmbus_version != VMBUS_VERSION_WS2008) 1299302864Ssephe chan->ch_monprm->mp_connid = offer->chm_connid; 1300302864Ssephe 1301302864Ssephe if (offer->chm_flags1 & VMBUS_CHOFFER_FLAG1_HASMNF) { 1302307461Ssephe int trig_idx; 1303307461Ssephe 1304302864Ssephe /* 1305302864Ssephe * Setup MNF stuffs. 1306302864Ssephe */ 1307307461Ssephe chan->ch_txflags |= VMBUS_CHAN_TXF_HASMNF; 1308307461Ssephe 1309307461Ssephe trig_idx = offer->chm_montrig / VMBUS_MONTRIG_LEN; 1310307461Ssephe if (trig_idx >= VMBUS_MONTRIGS_MAX) 1311302864Ssephe panic("invalid monitor trigger %u", offer->chm_montrig); 1312307461Ssephe chan->ch_montrig = 1313307461Ssephe &sc->vmbus_mnf2->mnf_trigs[trig_idx].mt_pending; 1314307461Ssephe 1315302864Ssephe chan->ch_montrig_mask = 1316302864Ssephe 1 << (offer->chm_montrig % VMBUS_MONTRIG_LEN); 1317302864Ssephe } 1318302864Ssephe 1319307461Ssephe /* 1320307461Ssephe * Setup event flag. 1321307461Ssephe */ 1322307461Ssephe chan->ch_evtflag = 1323307461Ssephe &sc->vmbus_tx_evtflags[chan->ch_id >> VMBUS_EVTFLAG_SHIFT]; 1324307461Ssephe chan->ch_evtflag_mask = 1UL << (chan->ch_id & VMBUS_EVTFLAG_MASK); 1325307461Ssephe 1326307599Ssephe /* 1327307599Ssephe * Setup attach and detach tasks. 1328307599Ssephe */ 1329307599Ssephe if (VMBUS_CHAN_ISPRIMARY(chan)) { 1330307599Ssephe chan->ch_mgmt_tq = sc->vmbus_devtq; 1331307599Ssephe attach_fn = vmbus_prichan_attach_task; 1332307599Ssephe detach_fn = vmbus_prichan_detach_task; 1333307599Ssephe } else { 1334307599Ssephe chan->ch_mgmt_tq = sc->vmbus_subchtq; 1335307599Ssephe attach_fn = vmbus_subchan_attach_task; 1336307599Ssephe detach_fn = vmbus_subchan_detach_task; 1337307599Ssephe } 1338307599Ssephe TASK_INIT(&chan->ch_attach_task, 0, attach_fn, chan); 1339307599Ssephe TASK_INIT(&chan->ch_detach_task, 0, detach_fn, chan); 1340307599Ssephe 1341302864Ssephe /* Select default cpu for this channel. */ 1342302864Ssephe vmbus_chan_cpu_default(chan); 1343302864Ssephe 1344302864Ssephe error = vmbus_chan_add(chan); 1345302864Ssephe if (error) { 1346302864Ssephe device_printf(sc->vmbus_dev, "add chan%u failed: %d\n", 1347302864Ssephe chan->ch_id, error); 1348302864Ssephe vmbus_chan_free(chan); 1349302864Ssephe return; 1350302864Ssephe } 1351307599Ssephe taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_attach_task); 1352302864Ssephe} 1353302864Ssephe 1354302864Ssephestatic void 1355302864Ssephevmbus_chan_msgproc_chrescind(struct vmbus_softc *sc, 1356302864Ssephe const struct vmbus_message *msg) 1357302864Ssephe{ 1358302864Ssephe const struct vmbus_chanmsg_chrescind *note; 1359307461Ssephe struct vmbus_channel *chan; 1360302864Ssephe 1361302864Ssephe note = (const struct vmbus_chanmsg_chrescind *)msg->msg_data; 1362302864Ssephe if (note->chm_chanid > VMBUS_CHAN_MAX) { 1363302864Ssephe device_printf(sc->vmbus_dev, "invalid rescinded chan%u\n", 1364302864Ssephe note->chm_chanid); 1365302864Ssephe return; 1366302864Ssephe } 1367302864Ssephe 1368302864Ssephe if (bootverbose) { 1369302864Ssephe device_printf(sc->vmbus_dev, "chan%u rescinded\n", 1370302864Ssephe note->chm_chanid); 1371302864Ssephe } 1372302864Ssephe 1373307599Ssephe /* 1374307599Ssephe * Find and remove the target channel from the channel list. 1375307599Ssephe */ 1376307599Ssephe mtx_lock(&sc->vmbus_chan_lock); 1377307599Ssephe TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) { 1378307599Ssephe if (chan->ch_id == note->chm_chanid) 1379307599Ssephe break; 1380307599Ssephe } 1381307599Ssephe if (chan == NULL) { 1382307599Ssephe mtx_unlock(&sc->vmbus_chan_lock); 1383307599Ssephe device_printf(sc->vmbus_dev, "chan%u is not offered\n", 1384307599Ssephe note->chm_chanid); 1385302864Ssephe return; 1386307599Ssephe } 1387307599Ssephe vmbus_chan_rem_list(sc, chan); 1388307599Ssephe mtx_unlock(&sc->vmbus_chan_lock); 1389302864Ssephe 1390307599Ssephe if (VMBUS_CHAN_ISPRIMARY(chan)) { 1391307599Ssephe /* 1392307599Ssephe * The target channel is a primary channel; remove the 1393307599Ssephe * target channel from the primary channel list now, 1394307599Ssephe * instead of later, so that it will not be found by 1395307599Ssephe * other sub-channel offers, which are processed in 1396307599Ssephe * this thread. 1397307599Ssephe */ 1398307599Ssephe mtx_lock(&sc->vmbus_prichan_lock); 1399307599Ssephe vmbus_chan_rem_prilist(sc, chan); 1400307599Ssephe mtx_unlock(&sc->vmbus_prichan_lock); 1401307599Ssephe } 1402307599Ssephe 1403307599Ssephe /* Detach the target channel. */ 1404307599Ssephe taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task); 1405302864Ssephe} 1406302864Ssephe 1407307599Ssephestatic int 1408307599Ssephevmbus_chan_release(struct vmbus_channel *chan) 1409302864Ssephe{ 1410307599Ssephe struct vmbus_softc *sc = chan->ch_vmbus; 1411307599Ssephe struct vmbus_chanmsg_chfree *req; 1412307599Ssephe struct vmbus_msghc *mh; 1413307599Ssephe int error; 1414302864Ssephe 1415307599Ssephe mh = vmbus_msghc_get(sc, sizeof(*req)); 1416307599Ssephe if (mh == NULL) { 1417307599Ssephe device_printf(sc->vmbus_dev, "can not get msg hypercall for " 1418307599Ssephe "chfree(chan%u)\n", chan->ch_id); 1419307599Ssephe return (ENXIO); 1420307599Ssephe } 1421307599Ssephe 1422307599Ssephe req = vmbus_msghc_dataptr(mh); 1423307599Ssephe req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHFREE; 1424307599Ssephe req->chm_chanid = chan->ch_id; 1425307599Ssephe 1426307599Ssephe error = vmbus_msghc_exec_noresult(mh); 1427307599Ssephe vmbus_msghc_put(sc, mh); 1428307599Ssephe 1429307599Ssephe if (error) { 1430307599Ssephe device_printf(sc->vmbus_dev, "chfree(chan%u) failed: %d", 1431307599Ssephe chan->ch_id, error); 1432302864Ssephe } else { 1433307599Ssephe if (bootverbose) { 1434307599Ssephe device_printf(sc->vmbus_dev, "chan%u freed\n", 1435302864Ssephe chan->ch_id); 1436302864Ssephe } 1437307599Ssephe } 1438307599Ssephe return (error); 1439307599Ssephe} 1440302864Ssephe 1441307599Ssephestatic void 1442307599Ssephevmbus_prichan_detach_task(void *xchan, int pending __unused) 1443307599Ssephe{ 1444307599Ssephe struct vmbus_channel *chan = xchan; 1445302864Ssephe 1446307599Ssephe KASSERT(VMBUS_CHAN_ISPRIMARY(chan), 1447307599Ssephe ("chan%u is not primary channel", chan->ch_id)); 1448302864Ssephe 1449307599Ssephe /* Delete and detach the device associated with this channel. */ 1450307599Ssephe vmbus_delete_child(chan); 1451302864Ssephe 1452307599Ssephe /* Release this channel (back to vmbus). */ 1453307599Ssephe vmbus_chan_release(chan); 1454307599Ssephe 1455307599Ssephe /* Free this channel's resource. */ 1456307599Ssephe vmbus_chan_free(chan); 1457302864Ssephe} 1458302864Ssephe 1459307599Ssephestatic void 1460307599Ssephevmbus_subchan_detach_task(void *xchan, int pending __unused) 1461307599Ssephe{ 1462307599Ssephe struct vmbus_channel *chan = xchan; 1463307599Ssephe struct vmbus_channel *pri_chan = chan->ch_prichan; 1464307599Ssephe 1465307599Ssephe KASSERT(!VMBUS_CHAN_ISPRIMARY(chan), 1466307599Ssephe ("chan%u is primary channel", chan->ch_id)); 1467307599Ssephe 1468307599Ssephe /* Release this channel (back to vmbus). */ 1469307599Ssephe vmbus_chan_release(chan); 1470307599Ssephe 1471307599Ssephe /* Unlink from its primary channel's sub-channel list. */ 1472307599Ssephe mtx_lock(&pri_chan->ch_subchan_lock); 1473307599Ssephe vmbus_chan_rem_sublist(pri_chan, chan); 1474307599Ssephe mtx_unlock(&pri_chan->ch_subchan_lock); 1475307599Ssephe /* Notify anyone that is waiting for this sub-channel to vanish. */ 1476307599Ssephe wakeup(pri_chan); 1477307599Ssephe 1478307599Ssephe /* Free this channel's resource. */ 1479307599Ssephe vmbus_chan_free(chan); 1480307599Ssephe} 1481307599Ssephe 1482307599Ssephestatic void 1483307599Ssephevmbus_prichan_attach_task(void *xchan, int pending __unused) 1484307599Ssephe{ 1485307599Ssephe 1486307599Ssephe /* 1487307599Ssephe * Add device for this primary channel. 1488307599Ssephe */ 1489307599Ssephe vmbus_add_child(xchan); 1490307599Ssephe} 1491307599Ssephe 1492307599Ssephestatic void 1493307599Ssephevmbus_subchan_attach_task(void *xchan __unused, int pending __unused) 1494307599Ssephe{ 1495307599Ssephe 1496307599Ssephe /* Nothing */ 1497307599Ssephe} 1498307599Ssephe 1499302864Ssephevoid 1500302864Ssephevmbus_chan_destroy_all(struct vmbus_softc *sc) 1501302864Ssephe{ 1502302864Ssephe 1503307599Ssephe /* 1504307599Ssephe * Detach all devices and destroy the corresponding primary 1505307599Ssephe * channels. 1506307599Ssephe */ 1507307599Ssephe for (;;) { 1508307599Ssephe struct vmbus_channel *chan; 1509302864Ssephe 1510307599Ssephe mtx_lock(&sc->vmbus_chan_lock); 1511307599Ssephe TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) { 1512307599Ssephe if (VMBUS_CHAN_ISPRIMARY(chan)) 1513307599Ssephe break; 1514307599Ssephe } 1515307599Ssephe if (chan == NULL) { 1516307599Ssephe /* No more primary channels; done. */ 1517307599Ssephe mtx_unlock(&sc->vmbus_chan_lock); 1518307599Ssephe break; 1519307599Ssephe } 1520307599Ssephe vmbus_chan_rem_list(sc, chan); 1521307599Ssephe mtx_unlock(&sc->vmbus_chan_lock); 1522302864Ssephe 1523302864Ssephe mtx_lock(&sc->vmbus_prichan_lock); 1524307599Ssephe vmbus_chan_rem_prilist(sc, chan); 1525307599Ssephe mtx_unlock(&sc->vmbus_prichan_lock); 1526307599Ssephe 1527307599Ssephe taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task); 1528302864Ssephe } 1529302864Ssephe} 1530302864Ssephe 1531303020Ssephe/* 1532302864Ssephe * The channel whose vcpu binding is closest to the currect vcpu will 1533302864Ssephe * be selected. 1534303020Ssephe * If no multi-channel, always select primary channel. 1535302864Ssephe */ 1536307461Ssephestruct vmbus_channel * 1537307461Ssephevmbus_chan_cpu2chan(struct vmbus_channel *prichan, int cpu) 1538302864Ssephe{ 1539307461Ssephe struct vmbus_channel *sel, *chan; 1540303020Ssephe uint32_t vcpu, sel_dist; 1541302864Ssephe 1542303020Ssephe KASSERT(cpu >= 0 && cpu < mp_ncpus, ("invalid cpuid %d", cpu)); 1543303020Ssephe if (TAILQ_EMPTY(&prichan->ch_subchans)) 1544303020Ssephe return prichan; 1545302864Ssephe 1546307461Ssephe vcpu = VMBUS_PCPU_GET(prichan->ch_vmbus, vcpuid, cpu); 1547302864Ssephe 1548303020Ssephe#define CHAN_VCPU_DIST(ch, vcpu) \ 1549303020Ssephe (((ch)->ch_vcpuid > (vcpu)) ? \ 1550303020Ssephe ((ch)->ch_vcpuid - (vcpu)) : ((vcpu) - (ch)->ch_vcpuid)) 1551302864Ssephe 1552303020Ssephe#define CHAN_SELECT(ch) \ 1553303020Ssephedo { \ 1554303020Ssephe sel = ch; \ 1555303020Ssephe sel_dist = CHAN_VCPU_DIST(ch, vcpu); \ 1556303020Ssephe} while (0) 1557302864Ssephe 1558303020Ssephe CHAN_SELECT(prichan); 1559302864Ssephe 1560303020Ssephe mtx_lock(&prichan->ch_subchan_lock); 1561303020Ssephe TAILQ_FOREACH(chan, &prichan->ch_subchans, ch_sublink) { 1562303020Ssephe uint32_t dist; 1563302864Ssephe 1564303020Ssephe KASSERT(chan->ch_stflags & VMBUS_CHAN_ST_OPENED, 1565303020Ssephe ("chan%u is not opened", chan->ch_id)); 1566303020Ssephe 1567303020Ssephe if (chan->ch_vcpuid == vcpu) { 1568303020Ssephe /* Exact match; done */ 1569303020Ssephe CHAN_SELECT(chan); 1570303020Ssephe break; 1571303020Ssephe } 1572303020Ssephe 1573303020Ssephe dist = CHAN_VCPU_DIST(chan, vcpu); 1574303020Ssephe if (sel_dist <= dist) { 1575303020Ssephe /* Far or same distance; skip */ 1576302864Ssephe continue; 1577302864Ssephe } 1578302864Ssephe 1579303020Ssephe /* Select the closer channel. */ 1580303020Ssephe CHAN_SELECT(chan); 1581302864Ssephe } 1582303020Ssephe mtx_unlock(&prichan->ch_subchan_lock); 1583302864Ssephe 1584303020Ssephe#undef CHAN_SELECT 1585303020Ssephe#undef CHAN_VCPU_DIST 1586303020Ssephe 1587303020Ssephe return sel; 1588302864Ssephe} 1589302864Ssephe 1590307461Ssephestruct vmbus_channel ** 1591307461Ssephevmbus_subchan_get(struct vmbus_channel *pri_chan, int subchan_cnt) 1592302864Ssephe{ 1593307461Ssephe struct vmbus_channel **ret, *chan; 1594302864Ssephe int i; 1595302864Ssephe 1596307510Ssephe KASSERT(subchan_cnt > 0, ("invalid sub-channel count %d", subchan_cnt)); 1597307510Ssephe 1598307461Ssephe ret = malloc(subchan_cnt * sizeof(struct vmbus_channel *), M_TEMP, 1599302864Ssephe M_WAITOK); 1600302864Ssephe 1601302864Ssephe mtx_lock(&pri_chan->ch_subchan_lock); 1602302864Ssephe 1603302864Ssephe while (pri_chan->ch_subchan_cnt < subchan_cnt) 1604302864Ssephe mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "subch", 0); 1605302864Ssephe 1606302864Ssephe i = 0; 1607302864Ssephe TAILQ_FOREACH(chan, &pri_chan->ch_subchans, ch_sublink) { 1608302864Ssephe /* TODO: refcnt chan */ 1609302864Ssephe ret[i] = chan; 1610302864Ssephe 1611302864Ssephe ++i; 1612302864Ssephe if (i == subchan_cnt) 1613302864Ssephe break; 1614302864Ssephe } 1615302864Ssephe KASSERT(i == subchan_cnt, ("invalid subchan count %d, should be %d", 1616302864Ssephe pri_chan->ch_subchan_cnt, subchan_cnt)); 1617302864Ssephe 1618302864Ssephe mtx_unlock(&pri_chan->ch_subchan_lock); 1619302864Ssephe 1620302864Ssephe return ret; 1621302864Ssephe} 1622302864Ssephe 1623302864Ssephevoid 1624307461Ssephevmbus_subchan_rel(struct vmbus_channel **subchan, int subchan_cnt __unused) 1625302864Ssephe{ 1626302864Ssephe 1627302864Ssephe free(subchan, M_TEMP); 1628302864Ssephe} 1629302864Ssephe 1630302864Ssephevoid 1631307461Ssephevmbus_subchan_drain(struct vmbus_channel *pri_chan) 1632302864Ssephe{ 1633302864Ssephe mtx_lock(&pri_chan->ch_subchan_lock); 1634302864Ssephe while (pri_chan->ch_subchan_cnt > 0) 1635302864Ssephe mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "dsubch", 0); 1636302864Ssephe mtx_unlock(&pri_chan->ch_subchan_lock); 1637302864Ssephe} 1638302864Ssephe 1639302864Ssephevoid 1640302864Ssephevmbus_chan_msgproc(struct vmbus_softc *sc, const struct vmbus_message *msg) 1641302864Ssephe{ 1642302864Ssephe vmbus_chanmsg_proc_t msg_proc; 1643302864Ssephe uint32_t msg_type; 1644302864Ssephe 1645302864Ssephe msg_type = ((const struct vmbus_chanmsg_hdr *)msg->msg_data)->chm_type; 1646302864Ssephe KASSERT(msg_type < VMBUS_CHANMSG_TYPE_MAX, 1647302864Ssephe ("invalid message type %u", msg_type)); 1648302864Ssephe 1649302864Ssephe msg_proc = vmbus_chan_msgprocs[msg_type]; 1650302864Ssephe if (msg_proc != NULL) 1651302864Ssephe msg_proc(sc, msg); 1652302864Ssephe} 1653303021Ssephe 1654303021Ssephevoid 1655307461Ssephevmbus_chan_set_readbatch(struct vmbus_channel *chan, bool on) 1656303021Ssephe{ 1657303021Ssephe if (!on) 1658303021Ssephe chan->ch_flags &= ~VMBUS_CHAN_FLAG_BATCHREAD; 1659303021Ssephe else 1660303021Ssephe chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD; 1661303021Ssephe} 1662307461Ssephe 1663307461Ssepheuint32_t 1664307461Ssephevmbus_chan_id(const struct vmbus_channel *chan) 1665307461Ssephe{ 1666307461Ssephe return chan->ch_id; 1667307461Ssephe} 1668307461Ssephe 1669307461Ssepheuint32_t 1670307461Ssephevmbus_chan_subidx(const struct vmbus_channel *chan) 1671307461Ssephe{ 1672307461Ssephe return chan->ch_subidx; 1673307461Ssephe} 1674307461Ssephe 1675307461Ssephebool 1676307461Ssephevmbus_chan_is_primary(const struct vmbus_channel *chan) 1677307461Ssephe{ 1678307461Ssephe if (VMBUS_CHAN_ISPRIMARY(chan)) 1679307461Ssephe return true; 1680307461Ssephe else 1681307461Ssephe return false; 1682307461Ssephe} 1683307461Ssephe 1684307461Ssepheconst struct hyperv_guid * 1685307461Ssephevmbus_chan_guid_inst(const struct vmbus_channel *chan) 1686307461Ssephe{ 1687307461Ssephe return &chan->ch_guid_inst; 1688307461Ssephe} 1689307486Ssephe 1690307486Ssepheint 1691307486Ssephevmbus_chan_prplist_nelem(int br_size, int prpcnt_max, int dlen_max) 1692307486Ssephe{ 1693307486Ssephe int elem_size; 1694307486Ssephe 1695307486Ssephe elem_size = __offsetof(struct vmbus_chanpkt_prplist, 1696307486Ssephe cp_range[0].gpa_page[prpcnt_max]); 1697307486Ssephe elem_size += dlen_max; 1698307486Ssephe elem_size = VMBUS_CHANPKT_TOTLEN(elem_size); 1699307486Ssephe 1700307486Ssephe return (vmbus_br_nelem(br_size, elem_size)); 1701307486Ssephe} 1702307599Ssephe 1703307599Ssephebool 1704307599Ssephevmbus_chan_tx_empty(const struct vmbus_channel *chan) 1705307599Ssephe{ 1706307599Ssephe 1707307599Ssephe return (vmbus_txbr_empty(&chan->ch_txbr)); 1708307599Ssephe} 1709307599Ssephe 1710307599Ssephebool 1711307599Ssephevmbus_chan_rx_empty(const struct vmbus_channel *chan) 1712307599Ssephe{ 1713307599Ssephe 1714307599Ssephe return (vmbus_rxbr_empty(&chan->ch_rxbr)); 1715307599Ssephe} 1716