vmbus_chan.c revision 302633
1250199Sgrehan/*- 2298446Ssephe * Copyright (c) 2009-2012,2016 Microsoft Corp. 3250199Sgrehan * Copyright (c) 2012 NetApp Inc. 4250199Sgrehan * Copyright (c) 2012 Citrix Inc. 5250199Sgrehan * All rights reserved. 6250199Sgrehan * 7250199Sgrehan * Redistribution and use in source and binary forms, with or without 8250199Sgrehan * modification, are permitted provided that the following conditions 9250199Sgrehan * are met: 10250199Sgrehan * 1. Redistributions of source code must retain the above copyright 11250199Sgrehan * notice unmodified, this list of conditions, and the following 12250199Sgrehan * disclaimer. 13250199Sgrehan * 2. Redistributions in binary form must reproduce the above copyright 14250199Sgrehan * notice, this list of conditions and the following disclaimer in the 15250199Sgrehan * documentation and/or other materials provided with the distribution. 16250199Sgrehan * 17250199Sgrehan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18250199Sgrehan * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19250199Sgrehan * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20250199Sgrehan * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21250199Sgrehan * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22250199Sgrehan * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23250199Sgrehan * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24250199Sgrehan * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25250199Sgrehan * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26250199Sgrehan * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27250199Sgrehan */ 28250199Sgrehan 29256276Sdim#include <sys/cdefs.h> 30256276Sdim__FBSDID("$FreeBSD: head/sys/dev/hyperv/vmbus/hv_channel.c 302633 2016-07-12 08:28:51Z sephe $"); 31256276Sdim 32250199Sgrehan#include <sys/param.h> 33296028Ssephe#include <sys/kernel.h> 34250199Sgrehan#include <sys/malloc.h> 35250199Sgrehan#include <sys/systm.h> 36250199Sgrehan#include <sys/mbuf.h> 37250199Sgrehan#include <sys/lock.h> 38250199Sgrehan#include <sys/mutex.h> 39296181Ssephe#include <sys/sysctl.h> 40301588Ssephe 41301588Ssephe#include <machine/atomic.h> 42250199Sgrehan#include <machine/bus.h> 43301588Ssephe 44250199Sgrehan#include <vm/vm.h> 45250199Sgrehan#include <vm/vm_param.h> 46250199Sgrehan#include <vm/pmap.h> 47250199Sgrehan 48300102Ssephe#include <dev/hyperv/vmbus/hv_vmbus_priv.h> 49302619Ssephe#include <dev/hyperv/vmbus/hyperv_var.h> 50301588Ssephe#include <dev/hyperv/vmbus/vmbus_reg.h> 51300102Ssephe#include <dev/hyperv/vmbus/vmbus_var.h> 52250199Sgrehan 53250199Sgrehanstatic void vmbus_channel_set_event(hv_vmbus_channel* channel); 54294886Ssephestatic void VmbusProcessChannelEvent(void* channel, int pending); 55250199Sgrehan 56250199Sgrehan/** 57250199Sgrehan * @brief Trigger an event notification on the specified channel 58250199Sgrehan */ 59250199Sgrehanstatic void 60250199Sgrehanvmbus_channel_set_event(hv_vmbus_channel *channel) 61250199Sgrehan{ 62302618Ssephe struct vmbus_softc *sc = channel->vmbus_sc; 63302618Ssephe uint32_t chanid = channel->offer_msg.child_rel_id; 64302618Ssephe 65302618Ssephe atomic_set_long(&sc->vmbus_tx_evtflags[chanid >> VMBUS_EVTFLAG_SHIFT], 66302618Ssephe 1UL << (chanid & VMBUS_EVTFLAG_MASK)); 67302618Ssephe 68301583Ssephe if (channel->offer_msg.monitor_allocated) { 69301583Ssephe hv_vmbus_monitor_page *monitor_page; 70250199Sgrehan 71301583Ssephe monitor_page = sc->vmbus_mnf2; 72250199Sgrehan synch_set_bit(channel->monitor_bit, 73250199Sgrehan (uint32_t *)&monitor_page-> 74256276Sdim trigger_group[channel->monitor_group].u.pending); 75250199Sgrehan } else { 76302619Ssephe hypercall_signal_event(channel->ch_sigevt_dma.hv_paddr); 77250199Sgrehan } 78250199Sgrehan 79250199Sgrehan} 80250199Sgrehan 81296289Ssephestatic int 82296289Ssephevmbus_channel_sysctl_monalloc(SYSCTL_HANDLER_ARGS) 83296289Ssephe{ 84296289Ssephe struct hv_vmbus_channel *chan = arg1; 85296289Ssephe int alloc = 0; 86296289Ssephe 87296289Ssephe if (chan->offer_msg.monitor_allocated) 88296289Ssephe alloc = 1; 89296289Ssephe return sysctl_handle_int(oidp, &alloc, 0, req); 90296289Ssephe} 91296289Ssephe 92296181Ssephestatic void 93296290Ssephevmbus_channel_sysctl_create(hv_vmbus_channel* channel) 94296181Ssephe{ 95296181Ssephe device_t dev; 96296181Ssephe struct sysctl_oid *devch_sysctl; 97296181Ssephe struct sysctl_oid *devch_id_sysctl, *devch_sub_sysctl; 98296181Ssephe struct sysctl_oid *devch_id_in_sysctl, *devch_id_out_sysctl; 99296181Ssephe struct sysctl_ctx_list *ctx; 100296181Ssephe uint32_t ch_id; 101296181Ssephe uint16_t sub_ch_id; 102296181Ssephe char name[16]; 103296181Ssephe 104296181Ssephe hv_vmbus_channel* primary_ch = channel->primary_channel; 105296181Ssephe 106296181Ssephe if (primary_ch == NULL) { 107296181Ssephe dev = channel->device->device; 108296181Ssephe ch_id = channel->offer_msg.child_rel_id; 109296181Ssephe } else { 110296181Ssephe dev = primary_ch->device->device; 111296181Ssephe ch_id = primary_ch->offer_msg.child_rel_id; 112296181Ssephe sub_ch_id = channel->offer_msg.offer.sub_channel_index; 113296181Ssephe } 114302633Ssephe ctx = &channel->ch_sysctl_ctx; 115302633Ssephe sysctl_ctx_init(ctx); 116296181Ssephe /* This creates dev.DEVNAME.DEVUNIT.channel tree */ 117296181Ssephe devch_sysctl = SYSCTL_ADD_NODE(ctx, 118296181Ssephe SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 119298693Ssephe OID_AUTO, "channel", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 120296181Ssephe /* This creates dev.DEVNAME.DEVUNIT.channel.CHANID tree */ 121296181Ssephe snprintf(name, sizeof(name), "%d", ch_id); 122296181Ssephe devch_id_sysctl = SYSCTL_ADD_NODE(ctx, 123296181Ssephe SYSCTL_CHILDREN(devch_sysctl), 124298693Ssephe OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 125296181Ssephe 126296181Ssephe if (primary_ch != NULL) { 127296181Ssephe devch_sub_sysctl = SYSCTL_ADD_NODE(ctx, 128296181Ssephe SYSCTL_CHILDREN(devch_id_sysctl), 129298693Ssephe OID_AUTO, "sub", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 130296181Ssephe snprintf(name, sizeof(name), "%d", sub_ch_id); 131296181Ssephe devch_id_sysctl = SYSCTL_ADD_NODE(ctx, 132296181Ssephe SYSCTL_CHILDREN(devch_sub_sysctl), 133298693Ssephe OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 134296188Ssephe 135296188Ssephe SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(devch_id_sysctl), 136296188Ssephe OID_AUTO, "chanid", CTLFLAG_RD, 137296188Ssephe &channel->offer_msg.child_rel_id, 0, "channel id"); 138296181Ssephe } 139296188Ssephe SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(devch_id_sysctl), OID_AUTO, 140296188Ssephe "cpu", CTLFLAG_RD, &channel->target_cpu, 0, "owner CPU id"); 141296289Ssephe SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(devch_id_sysctl), OID_AUTO, 142298693Ssephe "monitor_allocated", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, 143298693Ssephe channel, 0, vmbus_channel_sysctl_monalloc, "I", 144296289Ssephe "is monitor allocated to this channel"); 145296188Ssephe 146296181Ssephe devch_id_in_sysctl = SYSCTL_ADD_NODE(ctx, 147296181Ssephe SYSCTL_CHILDREN(devch_id_sysctl), 148296181Ssephe OID_AUTO, 149296181Ssephe "in", 150298693Ssephe CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 151296181Ssephe devch_id_out_sysctl = SYSCTL_ADD_NODE(ctx, 152296181Ssephe SYSCTL_CHILDREN(devch_id_sysctl), 153296181Ssephe OID_AUTO, 154296181Ssephe "out", 155298693Ssephe CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 156296181Ssephe hv_ring_buffer_stat(ctx, 157296181Ssephe SYSCTL_CHILDREN(devch_id_in_sysctl), 158296181Ssephe &(channel->inbound), 159296181Ssephe "inbound ring buffer stats"); 160296181Ssephe hv_ring_buffer_stat(ctx, 161296181Ssephe SYSCTL_CHILDREN(devch_id_out_sysctl), 162296181Ssephe &(channel->outbound), 163296181Ssephe "outbound ring buffer stats"); 164296181Ssephe} 165296290Ssephe 166250199Sgrehan/** 167250199Sgrehan * @brief Open the specified channel 168250199Sgrehan */ 169250199Sgrehanint 170250199Sgrehanhv_vmbus_channel_open( 171250199Sgrehan hv_vmbus_channel* new_channel, 172250199Sgrehan uint32_t send_ring_buffer_size, 173250199Sgrehan uint32_t recv_ring_buffer_size, 174250199Sgrehan void* user_data, 175250199Sgrehan uint32_t user_data_len, 176250199Sgrehan hv_vmbus_pfn_channel_callback pfn_on_channel_callback, 177250199Sgrehan void* context) 178250199Sgrehan{ 179302607Ssephe struct vmbus_softc *sc = new_channel->vmbus_sc; 180302607Ssephe const struct vmbus_chanmsg_chopen_resp *resp; 181302607Ssephe const struct vmbus_message *msg; 182302607Ssephe struct vmbus_chanmsg_chopen *req; 183302607Ssephe struct vmbus_msghc *mh; 184302607Ssephe uint32_t status; 185250199Sgrehan int ret = 0; 186250199Sgrehan void *in, *out; 187250199Sgrehan 188302607Ssephe if (user_data_len > VMBUS_CHANMSG_CHOPEN_UDATA_SIZE) { 189302607Ssephe device_printf(sc->vmbus_dev, 190302607Ssephe "invalid udata len %u for chan%u\n", 191302607Ssephe user_data_len, new_channel->offer_msg.child_rel_id); 192302607Ssephe return EINVAL; 193302607Ssephe } 194302607Ssephe 195282212Swhu mtx_lock(&new_channel->sc_lock); 196282212Swhu if (new_channel->state == HV_CHANNEL_OPEN_STATE) { 197282212Swhu new_channel->state = HV_CHANNEL_OPENING_STATE; 198282212Swhu } else { 199282212Swhu mtx_unlock(&new_channel->sc_lock); 200282212Swhu if(bootverbose) 201282212Swhu printf("VMBUS: Trying to open channel <%p> which in " 202282212Swhu "%d state.\n", new_channel, new_channel->state); 203282212Swhu return (EINVAL); 204282212Swhu } 205282212Swhu mtx_unlock(&new_channel->sc_lock); 206282212Swhu 207250199Sgrehan new_channel->on_channel_callback = pfn_on_channel_callback; 208250199Sgrehan new_channel->channel_callback_context = context; 209250199Sgrehan 210300102Ssephe vmbus_on_channel_open(new_channel); 211300102Ssephe 212302557Ssephe new_channel->rxq = VMBUS_PCPU_GET(new_channel->vmbus_sc, event_tq, 213300646Ssephe new_channel->target_cpu); 214294886Ssephe TASK_INIT(&new_channel->channel_task, 0, VmbusProcessChannelEvent, new_channel); 215294886Ssephe 216250199Sgrehan /* Allocate the ring buffer */ 217250199Sgrehan out = contigmalloc((send_ring_buffer_size + recv_ring_buffer_size), 218256350Sgrehan M_DEVBUF, M_ZERO, 0UL, BUS_SPACE_MAXADDR, PAGE_SIZE, 0); 219250199Sgrehan KASSERT(out != NULL, 220250199Sgrehan ("Error VMBUS: contigmalloc failed to allocate Ring Buffer!")); 221250199Sgrehan if (out == NULL) 222256350Sgrehan return (ENOMEM); 223250199Sgrehan 224250199Sgrehan in = ((uint8_t *) out + send_ring_buffer_size); 225250199Sgrehan 226250199Sgrehan new_channel->ring_buffer_pages = out; 227256350Sgrehan new_channel->ring_buffer_page_count = (send_ring_buffer_size + 228256350Sgrehan recv_ring_buffer_size) >> PAGE_SHIFT; 229256350Sgrehan new_channel->ring_buffer_size = send_ring_buffer_size + 230256350Sgrehan recv_ring_buffer_size; 231250199Sgrehan 232250199Sgrehan hv_vmbus_ring_buffer_init( 233250199Sgrehan &new_channel->outbound, 234250199Sgrehan out, 235250199Sgrehan send_ring_buffer_size); 236250199Sgrehan 237250199Sgrehan hv_vmbus_ring_buffer_init( 238250199Sgrehan &new_channel->inbound, 239250199Sgrehan in, 240250199Sgrehan recv_ring_buffer_size); 241250199Sgrehan 242296290Ssephe /* Create sysctl tree for this channel */ 243296290Ssephe vmbus_channel_sysctl_create(new_channel); 244296181Ssephe 245250199Sgrehan /** 246250199Sgrehan * Establish the gpadl for the ring buffer 247250199Sgrehan */ 248250199Sgrehan new_channel->ring_buffer_gpadl_handle = 0; 249250199Sgrehan 250250199Sgrehan ret = hv_vmbus_channel_establish_gpadl(new_channel, 251250199Sgrehan new_channel->outbound.ring_buffer, 252250199Sgrehan send_ring_buffer_size + recv_ring_buffer_size, 253250199Sgrehan &new_channel->ring_buffer_gpadl_handle); 254250199Sgrehan 255302607Ssephe /* 256302607Ssephe * Open channel w/ the bufring GPADL on the target CPU. 257250199Sgrehan */ 258302607Ssephe mh = vmbus_msghc_get(sc, sizeof(*req)); 259302607Ssephe if (mh == NULL) { 260302607Ssephe device_printf(sc->vmbus_dev, 261302607Ssephe "can not get msg hypercall for chopen(chan%u)\n", 262302607Ssephe new_channel->offer_msg.child_rel_id); 263302607Ssephe return ENXIO; 264302607Ssephe } 265250199Sgrehan 266302607Ssephe req = vmbus_msghc_dataptr(mh); 267302607Ssephe req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHOPEN; 268302607Ssephe req->chm_chanid = new_channel->offer_msg.child_rel_id; 269302607Ssephe req->chm_openid = new_channel->offer_msg.child_rel_id; 270302607Ssephe req->chm_gpadl = new_channel->ring_buffer_gpadl_handle; 271302607Ssephe req->chm_vcpuid = new_channel->target_vcpu; 272302607Ssephe req->chm_rxbr_pgofs = send_ring_buffer_size >> PAGE_SHIFT; 273250199Sgrehan if (user_data_len) 274302607Ssephe memcpy(req->chm_udata, user_data, user_data_len); 275250199Sgrehan 276302607Ssephe ret = vmbus_msghc_exec(sc, mh); 277302607Ssephe if (ret != 0) { 278302607Ssephe device_printf(sc->vmbus_dev, 279302607Ssephe "chopen(chan%u) msg hypercall exec failed: %d\n", 280302607Ssephe new_channel->offer_msg.child_rel_id, ret); 281302607Ssephe vmbus_msghc_put(sc, mh); 282302607Ssephe return ret; 283302607Ssephe } 284250199Sgrehan 285302607Ssephe msg = vmbus_msghc_wait_result(sc, mh); 286302607Ssephe resp = (const struct vmbus_chanmsg_chopen_resp *)msg->msg_data; 287302607Ssephe status = resp->chm_status; 288250199Sgrehan 289302607Ssephe vmbus_msghc_put(sc, mh); 290250199Sgrehan 291302607Ssephe if (status == 0) { 292302607Ssephe new_channel->state = HV_CHANNEL_OPENED_STATE; 293302607Ssephe if (bootverbose) { 294302607Ssephe device_printf(sc->vmbus_dev, "chan%u opened\n", 295302607Ssephe new_channel->offer_msg.child_rel_id); 296302607Ssephe } 297250199Sgrehan } else { 298302607Ssephe device_printf(sc->vmbus_dev, "failed to open chan%u\n", 299302607Ssephe new_channel->offer_msg.child_rel_id); 300302607Ssephe ret = ENXIO; 301250199Sgrehan } 302250199Sgrehan return (ret); 303250199Sgrehan} 304250199Sgrehan 305250199Sgrehan/** 306302609Ssephe * @brief Establish a GPADL for the specified buffer 307250199Sgrehan */ 308302609Ssepheint 309302609Ssephehv_vmbus_channel_establish_gpadl(struct hv_vmbus_channel *channel, 310302609Ssephe void *contig_buffer, uint32_t size, uint32_t *gpadl0) 311250199Sgrehan{ 312302609Ssephe struct vmbus_softc *sc = channel->vmbus_sc; 313302609Ssephe struct vmbus_msghc *mh; 314302609Ssephe struct vmbus_chanmsg_gpadl_conn *req; 315302609Ssephe const struct vmbus_message *msg; 316302609Ssephe size_t reqsz; 317302609Ssephe uint32_t gpadl, status; 318302609Ssephe int page_count, range_len, i, cnt, error; 319302609Ssephe uint64_t page_id, paddr; 320250199Sgrehan 321302609Ssephe /* 322302609Ssephe * Preliminary checks. 323302609Ssephe */ 324250199Sgrehan 325302609Ssephe KASSERT((size & PAGE_MASK) == 0, 326302609Ssephe ("invalid GPA size %u, not multiple page size", size)); 327250199Sgrehan page_count = size >> PAGE_SHIFT; 328250199Sgrehan 329302609Ssephe paddr = hv_get_phys_addr(contig_buffer); 330302609Ssephe KASSERT((paddr & PAGE_MASK) == 0, 331302609Ssephe ("GPA is not page aligned %jx", (uintmax_t)paddr)); 332302609Ssephe page_id = paddr >> PAGE_SHIFT; 333250199Sgrehan 334302609Ssephe range_len = __offsetof(struct vmbus_gpa_range, gpa_page[page_count]); 335302609Ssephe /* 336302609Ssephe * We don't support multiple GPA ranges. 337302609Ssephe */ 338302609Ssephe if (range_len > UINT16_MAX) { 339302609Ssephe device_printf(sc->vmbus_dev, "GPA too large, %d pages\n", 340302609Ssephe page_count); 341302609Ssephe return EOPNOTSUPP; 342250199Sgrehan } 343250199Sgrehan 344302609Ssephe /* 345302609Ssephe * Allocate GPADL id. 346302609Ssephe */ 347302630Ssephe gpadl = vmbus_gpadl_alloc(sc); 348302609Ssephe *gpadl0 = gpadl; 349250199Sgrehan 350302609Ssephe /* 351302609Ssephe * Connect this GPADL to the target channel. 352302609Ssephe * 353302609Ssephe * NOTE: 354302609Ssephe * Since each message can only hold small set of page 355302609Ssephe * addresses, several messages may be required to 356302609Ssephe * complete the connection. 357302609Ssephe */ 358302609Ssephe if (page_count > VMBUS_CHANMSG_GPADL_CONN_PGMAX) 359302609Ssephe cnt = VMBUS_CHANMSG_GPADL_CONN_PGMAX; 360302609Ssephe else 361302609Ssephe cnt = page_count; 362302609Ssephe page_count -= cnt; 363250199Sgrehan 364302609Ssephe reqsz = __offsetof(struct vmbus_chanmsg_gpadl_conn, 365302609Ssephe chm_range.gpa_page[cnt]); 366302609Ssephe mh = vmbus_msghc_get(sc, reqsz); 367302609Ssephe if (mh == NULL) { 368302609Ssephe device_printf(sc->vmbus_dev, 369302609Ssephe "can not get msg hypercall for gpadl->chan%u\n", 370302609Ssephe channel->offer_msg.child_rel_id); 371302609Ssephe return EIO; 372250199Sgrehan } 373250199Sgrehan 374302609Ssephe req = vmbus_msghc_dataptr(mh); 375302609Ssephe req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_CONN; 376302609Ssephe req->chm_chanid = channel->offer_msg.child_rel_id; 377302609Ssephe req->chm_gpadl = gpadl; 378302609Ssephe req->chm_range_len = range_len; 379302609Ssephe req->chm_range_cnt = 1; 380302609Ssephe req->chm_range.gpa_len = size; 381302609Ssephe req->chm_range.gpa_ofs = 0; 382302609Ssephe for (i = 0; i < cnt; ++i) 383302609Ssephe req->chm_range.gpa_page[i] = page_id++; 384250199Sgrehan 385302609Ssephe error = vmbus_msghc_exec(sc, mh); 386302609Ssephe if (error) { 387302609Ssephe device_printf(sc->vmbus_dev, 388302609Ssephe "gpadl->chan%u msg hypercall exec failed: %d\n", 389302609Ssephe channel->offer_msg.child_rel_id, error); 390302609Ssephe vmbus_msghc_put(sc, mh); 391302609Ssephe return error; 392302609Ssephe } 393250199Sgrehan 394302609Ssephe while (page_count > 0) { 395302609Ssephe struct vmbus_chanmsg_gpadl_subconn *subreq; 396250199Sgrehan 397302609Ssephe if (page_count > VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX) 398302609Ssephe cnt = VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX; 399302609Ssephe else 400302609Ssephe cnt = page_count; 401302609Ssephe page_count -= cnt; 402250199Sgrehan 403302609Ssephe reqsz = __offsetof(struct vmbus_chanmsg_gpadl_subconn, 404302609Ssephe chm_gpa_page[cnt]); 405302609Ssephe vmbus_msghc_reset(mh, reqsz); 406250199Sgrehan 407302609Ssephe subreq = vmbus_msghc_dataptr(mh); 408302609Ssephe subreq->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_SUBCONN; 409302609Ssephe subreq->chm_gpadl = gpadl; 410302609Ssephe for (i = 0; i < cnt; ++i) 411302609Ssephe subreq->chm_gpa_page[i] = page_id++; 412250199Sgrehan 413302609Ssephe vmbus_msghc_exec_noresult(mh); 414250199Sgrehan } 415302609Ssephe KASSERT(page_count == 0, ("invalid page count %d", page_count)); 416250199Sgrehan 417302609Ssephe msg = vmbus_msghc_wait_result(sc, mh); 418302609Ssephe status = ((const struct vmbus_chanmsg_gpadl_connresp *) 419302609Ssephe msg->msg_data)->chm_status; 420250199Sgrehan 421302609Ssephe vmbus_msghc_put(sc, mh); 422250199Sgrehan 423302609Ssephe if (status != 0) { 424302609Ssephe device_printf(sc->vmbus_dev, "gpadl->chan%u failed: " 425302609Ssephe "status %u\n", channel->offer_msg.child_rel_id, status); 426302609Ssephe return EIO; 427302632Ssephe } else { 428302632Ssephe if (bootverbose) { 429302632Ssephe device_printf(sc->vmbus_dev, "gpadl->chan%u " 430302632Ssephe "succeeded\n", channel->offer_msg.child_rel_id); 431302632Ssephe } 432302609Ssephe } 433302609Ssephe return 0; 434250199Sgrehan} 435250199Sgrehan 436302611Ssephe/* 437302611Ssephe * Disconnect the GPA from the target channel 438250199Sgrehan */ 439250199Sgrehanint 440302611Ssephehv_vmbus_channel_teardown_gpdal(struct hv_vmbus_channel *chan, uint32_t gpadl) 441250199Sgrehan{ 442302611Ssephe struct vmbus_softc *sc = chan->vmbus_sc; 443302611Ssephe struct vmbus_msghc *mh; 444302611Ssephe struct vmbus_chanmsg_gpadl_disconn *req; 445302611Ssephe int error; 446250199Sgrehan 447302611Ssephe mh = vmbus_msghc_get(sc, sizeof(*req)); 448302611Ssephe if (mh == NULL) { 449302611Ssephe device_printf(sc->vmbus_dev, 450302611Ssephe "can not get msg hypercall for gpa x->chan%u\n", 451302611Ssephe chan->offer_msg.child_rel_id); 452302611Ssephe return EBUSY; 453250199Sgrehan } 454250199Sgrehan 455302611Ssephe req = vmbus_msghc_dataptr(mh); 456302611Ssephe req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_DISCONN; 457302611Ssephe req->chm_chanid = chan->offer_msg.child_rel_id; 458302611Ssephe req->chm_gpadl = gpadl; 459250199Sgrehan 460302611Ssephe error = vmbus_msghc_exec(sc, mh); 461302611Ssephe if (error) { 462302611Ssephe device_printf(sc->vmbus_dev, 463302611Ssephe "gpa x->chan%u msg hypercall exec failed: %d\n", 464302611Ssephe chan->offer_msg.child_rel_id, error); 465302611Ssephe vmbus_msghc_put(sc, mh); 466302611Ssephe return error; 467302611Ssephe } 468250199Sgrehan 469302611Ssephe vmbus_msghc_wait_result(sc, mh); 470302611Ssephe /* Discard result; no useful information */ 471302611Ssephe vmbus_msghc_put(sc, mh); 472250199Sgrehan 473302611Ssephe return 0; 474250199Sgrehan} 475250199Sgrehan 476282212Swhustatic void 477282212Swhuhv_vmbus_channel_close_internal(hv_vmbus_channel *channel) 478250199Sgrehan{ 479302610Ssephe struct vmbus_softc *sc = channel->vmbus_sc; 480302610Ssephe struct vmbus_msghc *mh; 481302610Ssephe struct vmbus_chanmsg_chclose *req; 482294886Ssephe struct taskqueue *rxq = channel->rxq; 483302610Ssephe int error; 484250199Sgrehan 485282212Swhu channel->state = HV_CHANNEL_OPEN_STATE; 486302633Ssephe sysctl_ctx_free(&channel->ch_sysctl_ctx); 487282212Swhu 488282212Swhu /* 489294886Ssephe * set rxq to NULL to avoid more requests be scheduled 490294886Ssephe */ 491294886Ssephe channel->rxq = NULL; 492294886Ssephe taskqueue_drain(rxq, &channel->channel_task); 493250199Sgrehan channel->on_channel_callback = NULL; 494250199Sgrehan 495250199Sgrehan /** 496250199Sgrehan * Send a closing message 497250199Sgrehan */ 498250199Sgrehan 499302610Ssephe mh = vmbus_msghc_get(sc, sizeof(*req)); 500302610Ssephe if (mh == NULL) { 501302610Ssephe device_printf(sc->vmbus_dev, 502302610Ssephe "can not get msg hypercall for chclose(chan%u)\n", 503302610Ssephe channel->offer_msg.child_rel_id); 504302610Ssephe return; 505302610Ssephe } 506250199Sgrehan 507302610Ssephe req = vmbus_msghc_dataptr(mh); 508302610Ssephe req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHCLOSE; 509302610Ssephe req->chm_chanid = channel->offer_msg.child_rel_id; 510250199Sgrehan 511302610Ssephe error = vmbus_msghc_exec_noresult(mh); 512302610Ssephe vmbus_msghc_put(sc, mh); 513302610Ssephe 514302610Ssephe if (error) { 515302610Ssephe device_printf(sc->vmbus_dev, 516302610Ssephe "chclose(chan%u) msg hypercall exec failed: %d\n", 517302610Ssephe channel->offer_msg.child_rel_id, error); 518302610Ssephe return; 519302610Ssephe } else if (bootverbose) { 520302610Ssephe device_printf(sc->vmbus_dev, "close chan%u\n", 521302610Ssephe channel->offer_msg.child_rel_id); 522302610Ssephe } 523302610Ssephe 524250199Sgrehan /* Tear down the gpadl for the channel's ring buffer */ 525250199Sgrehan if (channel->ring_buffer_gpadl_handle) { 526250199Sgrehan hv_vmbus_channel_teardown_gpdal(channel, 527250199Sgrehan channel->ring_buffer_gpadl_handle); 528250199Sgrehan } 529250199Sgrehan 530250199Sgrehan /* TODO: Send a msg to release the childRelId */ 531250199Sgrehan 532250199Sgrehan /* cleanup the ring buffers for this channel */ 533250199Sgrehan hv_ring_buffer_cleanup(&channel->outbound); 534250199Sgrehan hv_ring_buffer_cleanup(&channel->inbound); 535250199Sgrehan 536256350Sgrehan contigfree(channel->ring_buffer_pages, channel->ring_buffer_size, 537256350Sgrehan M_DEVBUF); 538282212Swhu} 539250199Sgrehan 540282212Swhu/** 541282212Swhu * @brief Close the specified channel 542282212Swhu */ 543282212Swhuvoid 544282212Swhuhv_vmbus_channel_close(hv_vmbus_channel *channel) 545282212Swhu{ 546282212Swhu hv_vmbus_channel* sub_channel; 547282212Swhu 548282212Swhu if (channel->primary_channel != NULL) { 549282212Swhu /* 550282212Swhu * We only close multi-channels when the primary is 551282212Swhu * closed. 552282212Swhu */ 553282212Swhu return; 554282212Swhu } 555282212Swhu 556250199Sgrehan /* 557282212Swhu * Close all multi-channels first. 558250199Sgrehan */ 559282212Swhu TAILQ_FOREACH(sub_channel, &channel->sc_list_anchor, 560282212Swhu sc_list_entry) { 561282212Swhu if (sub_channel->state != HV_CHANNEL_OPENED_STATE) 562282212Swhu continue; 563282212Swhu hv_vmbus_channel_close_internal(sub_channel); 564250199Sgrehan } 565282212Swhu /* 566282212Swhu * Then close the primary channel. 567282212Swhu */ 568282212Swhu hv_vmbus_channel_close_internal(channel); 569250199Sgrehan} 570250199Sgrehan 571250199Sgrehan/** 572250199Sgrehan * @brief Send the specified buffer on the given channel 573250199Sgrehan */ 574250199Sgrehanint 575250199Sgrehanhv_vmbus_channel_send_packet( 576250199Sgrehan hv_vmbus_channel* channel, 577250199Sgrehan void* buffer, 578250199Sgrehan uint32_t buffer_len, 579250199Sgrehan uint64_t request_id, 580250199Sgrehan hv_vmbus_packet_type type, 581250199Sgrehan uint32_t flags) 582250199Sgrehan{ 583250199Sgrehan int ret = 0; 584250199Sgrehan hv_vm_packet_descriptor desc; 585250199Sgrehan uint32_t packet_len; 586250199Sgrehan uint64_t aligned_data; 587250199Sgrehan uint32_t packet_len_aligned; 588282212Swhu boolean_t need_sig; 589250199Sgrehan hv_vmbus_sg_buffer_list buffer_list[3]; 590250199Sgrehan 591250199Sgrehan packet_len = sizeof(hv_vm_packet_descriptor) + buffer_len; 592250199Sgrehan packet_len_aligned = HV_ALIGN_UP(packet_len, sizeof(uint64_t)); 593250199Sgrehan aligned_data = 0; 594250199Sgrehan 595250199Sgrehan /* Setup the descriptor */ 596250199Sgrehan desc.type = type; /* HV_VMBUS_PACKET_TYPE_DATA_IN_BAND; */ 597250199Sgrehan desc.flags = flags; /* HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED */ 598250199Sgrehan /* in 8-bytes granularity */ 599250199Sgrehan desc.data_offset8 = sizeof(hv_vm_packet_descriptor) >> 3; 600250199Sgrehan desc.length8 = (uint16_t) (packet_len_aligned >> 3); 601250199Sgrehan desc.transaction_id = request_id; 602250199Sgrehan 603250199Sgrehan buffer_list[0].data = &desc; 604250199Sgrehan buffer_list[0].length = sizeof(hv_vm_packet_descriptor); 605250199Sgrehan 606250199Sgrehan buffer_list[1].data = buffer; 607250199Sgrehan buffer_list[1].length = buffer_len; 608250199Sgrehan 609250199Sgrehan buffer_list[2].data = &aligned_data; 610250199Sgrehan buffer_list[2].length = packet_len_aligned - packet_len; 611250199Sgrehan 612282212Swhu ret = hv_ring_buffer_write(&channel->outbound, buffer_list, 3, 613282212Swhu &need_sig); 614250199Sgrehan 615250199Sgrehan /* TODO: We should determine if this is optional */ 616282212Swhu if (ret == 0 && need_sig) { 617250199Sgrehan vmbus_channel_set_event(channel); 618250199Sgrehan } 619250199Sgrehan 620250199Sgrehan return (ret); 621250199Sgrehan} 622250199Sgrehan 623250199Sgrehan/** 624250199Sgrehan * @brief Send a range of single-page buffer packets using 625250199Sgrehan * a GPADL Direct packet type 626250199Sgrehan */ 627250199Sgrehanint 628250199Sgrehanhv_vmbus_channel_send_packet_pagebuffer( 629250199Sgrehan hv_vmbus_channel* channel, 630250199Sgrehan hv_vmbus_page_buffer page_buffers[], 631250199Sgrehan uint32_t page_count, 632250199Sgrehan void* buffer, 633250199Sgrehan uint32_t buffer_len, 634250199Sgrehan uint64_t request_id) 635250199Sgrehan{ 636250199Sgrehan 637250199Sgrehan int ret = 0; 638282212Swhu boolean_t need_sig; 639250199Sgrehan uint32_t packet_len; 640294705Ssephe uint32_t page_buflen; 641250199Sgrehan uint32_t packetLen_aligned; 642294705Ssephe hv_vmbus_sg_buffer_list buffer_list[4]; 643250199Sgrehan hv_vmbus_channel_packet_page_buffer desc; 644250199Sgrehan uint32_t descSize; 645250199Sgrehan uint64_t alignedData = 0; 646250199Sgrehan 647250199Sgrehan if (page_count > HV_MAX_PAGE_BUFFER_COUNT) 648250199Sgrehan return (EINVAL); 649250199Sgrehan 650250199Sgrehan /* 651250199Sgrehan * Adjust the size down since hv_vmbus_channel_packet_page_buffer 652250199Sgrehan * is the largest size we support 653250199Sgrehan */ 654294705Ssephe descSize = __offsetof(hv_vmbus_channel_packet_page_buffer, range); 655294705Ssephe page_buflen = sizeof(hv_vmbus_page_buffer) * page_count; 656294705Ssephe packet_len = descSize + page_buflen + buffer_len; 657250199Sgrehan packetLen_aligned = HV_ALIGN_UP(packet_len, sizeof(uint64_t)); 658250199Sgrehan 659250199Sgrehan /* Setup the descriptor */ 660250199Sgrehan desc.type = HV_VMBUS_PACKET_TYPE_DATA_USING_GPA_DIRECT; 661250199Sgrehan desc.flags = HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; 662294705Ssephe /* in 8-bytes granularity */ 663294705Ssephe desc.data_offset8 = (descSize + page_buflen) >> 3; 664250199Sgrehan desc.length8 = (uint16_t) (packetLen_aligned >> 3); 665250199Sgrehan desc.transaction_id = request_id; 666250199Sgrehan desc.range_count = page_count; 667250199Sgrehan 668250199Sgrehan buffer_list[0].data = &desc; 669250199Sgrehan buffer_list[0].length = descSize; 670250199Sgrehan 671294705Ssephe buffer_list[1].data = page_buffers; 672294705Ssephe buffer_list[1].length = page_buflen; 673250199Sgrehan 674294705Ssephe buffer_list[2].data = buffer; 675294705Ssephe buffer_list[2].length = buffer_len; 676250199Sgrehan 677294705Ssephe buffer_list[3].data = &alignedData; 678294705Ssephe buffer_list[3].length = packetLen_aligned - packet_len; 679294705Ssephe 680294705Ssephe ret = hv_ring_buffer_write(&channel->outbound, buffer_list, 4, 681282212Swhu &need_sig); 682250199Sgrehan 683250199Sgrehan /* TODO: We should determine if this is optional */ 684282212Swhu if (ret == 0 && need_sig) { 685250199Sgrehan vmbus_channel_set_event(channel); 686250199Sgrehan } 687250199Sgrehan 688250199Sgrehan return (ret); 689250199Sgrehan} 690250199Sgrehan 691250199Sgrehan/** 692250199Sgrehan * @brief Send a multi-page buffer packet using a GPADL Direct packet type 693250199Sgrehan */ 694250199Sgrehanint 695250199Sgrehanhv_vmbus_channel_send_packet_multipagebuffer( 696250199Sgrehan hv_vmbus_channel* channel, 697250199Sgrehan hv_vmbus_multipage_buffer* multi_page_buffer, 698250199Sgrehan void* buffer, 699250199Sgrehan uint32_t buffer_len, 700250199Sgrehan uint64_t request_id) 701250199Sgrehan{ 702250199Sgrehan 703250199Sgrehan int ret = 0; 704250199Sgrehan uint32_t desc_size; 705282212Swhu boolean_t need_sig; 706250199Sgrehan uint32_t packet_len; 707250199Sgrehan uint32_t packet_len_aligned; 708250199Sgrehan uint32_t pfn_count; 709250199Sgrehan uint64_t aligned_data = 0; 710250199Sgrehan hv_vmbus_sg_buffer_list buffer_list[3]; 711250199Sgrehan hv_vmbus_channel_packet_multipage_buffer desc; 712250199Sgrehan 713250199Sgrehan pfn_count = 714250199Sgrehan HV_NUM_PAGES_SPANNED( 715250199Sgrehan multi_page_buffer->offset, 716250199Sgrehan multi_page_buffer->length); 717250199Sgrehan 718250199Sgrehan if ((pfn_count == 0) || (pfn_count > HV_MAX_MULTIPAGE_BUFFER_COUNT)) 719250199Sgrehan return (EINVAL); 720250199Sgrehan /* 721250199Sgrehan * Adjust the size down since hv_vmbus_channel_packet_multipage_buffer 722250199Sgrehan * is the largest size we support 723250199Sgrehan */ 724250199Sgrehan desc_size = 725250199Sgrehan sizeof(hv_vmbus_channel_packet_multipage_buffer) - 726250199Sgrehan ((HV_MAX_MULTIPAGE_BUFFER_COUNT - pfn_count) * 727250199Sgrehan sizeof(uint64_t)); 728250199Sgrehan packet_len = desc_size + buffer_len; 729250199Sgrehan packet_len_aligned = HV_ALIGN_UP(packet_len, sizeof(uint64_t)); 730250199Sgrehan 731250199Sgrehan /* 732250199Sgrehan * Setup the descriptor 733250199Sgrehan */ 734250199Sgrehan desc.type = HV_VMBUS_PACKET_TYPE_DATA_USING_GPA_DIRECT; 735250199Sgrehan desc.flags = HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; 736250199Sgrehan desc.data_offset8 = desc_size >> 3; /* in 8-bytes granularity */ 737250199Sgrehan desc.length8 = (uint16_t) (packet_len_aligned >> 3); 738250199Sgrehan desc.transaction_id = request_id; 739250199Sgrehan desc.range_count = 1; 740250199Sgrehan 741250199Sgrehan desc.range.length = multi_page_buffer->length; 742250199Sgrehan desc.range.offset = multi_page_buffer->offset; 743250199Sgrehan 744250199Sgrehan memcpy(desc.range.pfn_array, multi_page_buffer->pfn_array, 745250199Sgrehan pfn_count * sizeof(uint64_t)); 746250199Sgrehan 747250199Sgrehan buffer_list[0].data = &desc; 748250199Sgrehan buffer_list[0].length = desc_size; 749250199Sgrehan 750250199Sgrehan buffer_list[1].data = buffer; 751250199Sgrehan buffer_list[1].length = buffer_len; 752250199Sgrehan 753250199Sgrehan buffer_list[2].data = &aligned_data; 754250199Sgrehan buffer_list[2].length = packet_len_aligned - packet_len; 755250199Sgrehan 756282212Swhu ret = hv_ring_buffer_write(&channel->outbound, buffer_list, 3, 757282212Swhu &need_sig); 758250199Sgrehan 759250199Sgrehan /* TODO: We should determine if this is optional */ 760282212Swhu if (ret == 0 && need_sig) { 761250199Sgrehan vmbus_channel_set_event(channel); 762250199Sgrehan } 763250199Sgrehan 764250199Sgrehan return (ret); 765250199Sgrehan} 766250199Sgrehan 767250199Sgrehan/** 768250199Sgrehan * @brief Retrieve the user packet on the specified channel 769250199Sgrehan */ 770250199Sgrehanint 771250199Sgrehanhv_vmbus_channel_recv_packet( 772250199Sgrehan hv_vmbus_channel* channel, 773250199Sgrehan void* Buffer, 774250199Sgrehan uint32_t buffer_len, 775250199Sgrehan uint32_t* buffer_actual_len, 776250199Sgrehan uint64_t* request_id) 777250199Sgrehan{ 778250199Sgrehan int ret; 779250199Sgrehan uint32_t user_len; 780250199Sgrehan uint32_t packet_len; 781250199Sgrehan hv_vm_packet_descriptor desc; 782250199Sgrehan 783250199Sgrehan *buffer_actual_len = 0; 784250199Sgrehan *request_id = 0; 785250199Sgrehan 786250199Sgrehan ret = hv_ring_buffer_peek(&channel->inbound, &desc, 787250199Sgrehan sizeof(hv_vm_packet_descriptor)); 788250199Sgrehan if (ret != 0) 789250199Sgrehan return (0); 790250199Sgrehan 791250199Sgrehan packet_len = desc.length8 << 3; 792250199Sgrehan user_len = packet_len - (desc.data_offset8 << 3); 793250199Sgrehan 794250199Sgrehan *buffer_actual_len = user_len; 795250199Sgrehan 796250199Sgrehan if (user_len > buffer_len) 797250199Sgrehan return (EINVAL); 798250199Sgrehan 799250199Sgrehan *request_id = desc.transaction_id; 800250199Sgrehan 801250199Sgrehan /* Copy over the packet to the user buffer */ 802250199Sgrehan ret = hv_ring_buffer_read(&channel->inbound, Buffer, user_len, 803250199Sgrehan (desc.data_offset8 << 3)); 804250199Sgrehan 805250199Sgrehan return (0); 806250199Sgrehan} 807250199Sgrehan 808250199Sgrehan/** 809250199Sgrehan * @brief Retrieve the raw packet on the specified channel 810250199Sgrehan */ 811250199Sgrehanint 812250199Sgrehanhv_vmbus_channel_recv_packet_raw( 813250199Sgrehan hv_vmbus_channel* channel, 814250199Sgrehan void* buffer, 815250199Sgrehan uint32_t buffer_len, 816250199Sgrehan uint32_t* buffer_actual_len, 817250199Sgrehan uint64_t* request_id) 818250199Sgrehan{ 819250199Sgrehan int ret; 820250199Sgrehan uint32_t packetLen; 821250199Sgrehan hv_vm_packet_descriptor desc; 822250199Sgrehan 823250199Sgrehan *buffer_actual_len = 0; 824250199Sgrehan *request_id = 0; 825250199Sgrehan 826250199Sgrehan ret = hv_ring_buffer_peek( 827250199Sgrehan &channel->inbound, &desc, 828250199Sgrehan sizeof(hv_vm_packet_descriptor)); 829250199Sgrehan 830250199Sgrehan if (ret != 0) 831250199Sgrehan return (0); 832250199Sgrehan 833250199Sgrehan packetLen = desc.length8 << 3; 834250199Sgrehan *buffer_actual_len = packetLen; 835250199Sgrehan 836250199Sgrehan if (packetLen > buffer_len) 837250199Sgrehan return (ENOBUFS); 838250199Sgrehan 839250199Sgrehan *request_id = desc.transaction_id; 840250199Sgrehan 841250199Sgrehan /* Copy over the entire packet to the user buffer */ 842250199Sgrehan ret = hv_ring_buffer_read(&channel->inbound, buffer, packetLen, 0); 843250199Sgrehan 844250199Sgrehan return (0); 845250199Sgrehan} 846294886Ssephe 847294886Ssephe 848294886Ssephe/** 849294886Ssephe * Process a channel event notification 850294886Ssephe */ 851294886Ssephestatic void 852294886SsepheVmbusProcessChannelEvent(void* context, int pending) 853294886Ssephe{ 854294886Ssephe void* arg; 855294886Ssephe uint32_t bytes_to_read; 856294886Ssephe hv_vmbus_channel* channel = (hv_vmbus_channel*)context; 857294886Ssephe boolean_t is_batched_reading; 858294886Ssephe 859294886Ssephe if (channel->on_channel_callback != NULL) { 860294886Ssephe arg = channel->channel_callback_context; 861294886Ssephe is_batched_reading = channel->batched_reading; 862294886Ssephe /* 863294886Ssephe * Optimize host to guest signaling by ensuring: 864294886Ssephe * 1. While reading the channel, we disable interrupts from 865294886Ssephe * host. 866294886Ssephe * 2. Ensure that we process all posted messages from the host 867294886Ssephe * before returning from this callback. 868294886Ssephe * 3. Once we return, enable signaling from the host. Once this 869294886Ssephe * state is set we check to see if additional packets are 870294886Ssephe * available to read. In this case we repeat the process. 871294886Ssephe */ 872294886Ssephe do { 873294886Ssephe if (is_batched_reading) 874294886Ssephe hv_ring_buffer_read_begin(&channel->inbound); 875294886Ssephe 876294886Ssephe channel->on_channel_callback(arg); 877294886Ssephe 878294886Ssephe if (is_batched_reading) 879294886Ssephe bytes_to_read = 880294886Ssephe hv_ring_buffer_read_end(&channel->inbound); 881294886Ssephe else 882294886Ssephe bytes_to_read = 0; 883294886Ssephe } while (is_batched_reading && (bytes_to_read != 0)); 884294886Ssephe } 885294886Ssephe} 886