1// SPDX-License-Identifier: GPL-2.0-only 2/**************************************************************************** 3 * Driver for Solarflare network controllers and boards 4 * Copyright 2010-2012 Solarflare Communications Inc. 5 */ 6#include <linux/pci.h> 7#include <linux/module.h> 8#include "net_driver.h" 9#include "efx.h" 10#include "efx_channels.h" 11#include "nic.h" 12#include "io.h" 13#include "mcdi.h" 14#include "filter.h" 15#include "mcdi_pcol.h" 16#include "farch_regs.h" 17#include "siena_sriov.h" 18#include "vfdi.h" 19 20/* Number of longs required to track all the VIs in a VF */ 21#define VI_MASK_LENGTH BITS_TO_LONGS(1 << EFX_VI_SCALE_MAX) 22 23/* Maximum number of RX queues supported */ 24#define VF_MAX_RX_QUEUES 63 25 26/** 27 * enum efx_vf_tx_filter_mode - TX MAC filtering behaviour 28 * @VF_TX_FILTER_OFF: Disabled 29 * @VF_TX_FILTER_AUTO: Enabled if MAC address assigned to VF and only 30 * 2 TX queues allowed per VF. 31 * @VF_TX_FILTER_ON: Enabled 32 */ 33enum efx_vf_tx_filter_mode { 34 VF_TX_FILTER_OFF, 35 VF_TX_FILTER_AUTO, 36 VF_TX_FILTER_ON, 37}; 38 39/** 40 * struct siena_vf - Back-end resource and protocol state for a PCI VF 41 * @efx: The Efx NIC owning this VF 42 * @pci_rid: The PCI requester ID for this VF 43 * @pci_name: The PCI name (formatted address) of this VF 44 * @index: Index of VF within its port and PF. 45 * @req: VFDI incoming request work item. Incoming USR_EV events are received 46 * by the NAPI handler, but must be handled by executing MCDI requests 47 * inside a work item. 48 * @req_addr: VFDI incoming request DMA address (in VF's PCI address space). 49 * @req_type: Expected next incoming (from VF) %VFDI_EV_TYPE member. 50 * @req_seqno: Expected next incoming (from VF) %VFDI_EV_SEQ member. 51 * @msg_seqno: Next %VFDI_EV_SEQ member to reply to VF. Protected by 52 * @status_lock 53 * @busy: VFDI request queued to be processed or being processed. Receiving 54 * a VFDI request when @busy is set is an error condition. 55 * @buf: Incoming VFDI requests are DMA from the VF into this buffer. 56 * @buftbl_base: Buffer table entries for this VF start at this index. 57 * @rx_filtering: Receive filtering has been requested by the VF driver. 58 * @rx_filter_flags: The flags sent in the %VFDI_OP_INSERT_FILTER request. 59 * @rx_filter_qid: VF relative qid for RX filter requested by VF. 60 * @rx_filter_id: Receive MAC filter ID. Only one filter per VF is supported. 61 * @tx_filter_mode: Transmit MAC filtering mode. 62 * @tx_filter_id: Transmit MAC filter ID. 63 * @addr: The MAC address and outer vlan tag of the VF. 64 * @status_addr: VF DMA address of page for &struct vfdi_status updates. 65 * @status_lock: Mutex protecting @msg_seqno, @status_addr, @addr, 66 * @peer_page_addrs and @peer_page_count from simultaneous 67 * updates by the VM and consumption by 68 * efx_siena_sriov_update_vf_addr() 69 * @peer_page_addrs: Pointer to an array of guest pages for local addresses. 70 * @peer_page_count: Number of entries in @peer_page_count. 71 * @evq0_addrs: Array of guest pages backing evq0. 72 * @evq0_count: Number of entries in @evq0_addrs. 73 * @flush_waitq: wait queue used by %VFDI_OP_FINI_ALL_QUEUES handler 74 * to wait for flush completions. 75 * @txq_lock: Mutex for TX queue allocation. 76 * @txq_mask: Mask of initialized transmit queues. 77 * @txq_count: Number of initialized transmit queues. 78 * @rxq_mask: Mask of initialized receive queues. 79 * @rxq_count: Number of initialized receive queues. 80 * @rxq_retry_mask: Mask or receive queues that need to be flushed again 81 * due to flush failure. 82 * @rxq_retry_count: Number of receive queues in @rxq_retry_mask. 83 * @reset_work: Work item to schedule a VF reset. 84 */ 85struct siena_vf { 86 struct efx_nic *efx; 87 unsigned int pci_rid; 88 char pci_name[13]; /* dddd:bb:dd.f */ 89 unsigned int index; 90 struct work_struct req; 91 u64 req_addr; 92 int req_type; 93 unsigned req_seqno; 94 unsigned msg_seqno; 95 bool busy; 96 struct efx_buffer buf; 97 unsigned buftbl_base; 98 bool rx_filtering; 99 enum efx_filter_flags rx_filter_flags; 100 unsigned rx_filter_qid; 101 int rx_filter_id; 102 enum efx_vf_tx_filter_mode tx_filter_mode; 103 int tx_filter_id; 104 struct vfdi_endpoint addr; 105 u64 status_addr; 106 struct mutex status_lock; 107 u64 *peer_page_addrs; 108 unsigned peer_page_count; 109 u64 evq0_addrs[EFX_MAX_VF_EVQ_SIZE * sizeof(efx_qword_t) / 110 EFX_BUF_SIZE]; 111 unsigned evq0_count; 112 wait_queue_head_t flush_waitq; 113 struct mutex txq_lock; 114 unsigned long txq_mask[VI_MASK_LENGTH]; 115 unsigned txq_count; 116 unsigned long rxq_mask[VI_MASK_LENGTH]; 117 unsigned rxq_count; 118 unsigned long rxq_retry_mask[VI_MASK_LENGTH]; 119 atomic_t rxq_retry_count; 120 struct work_struct reset_work; 121}; 122 123struct efx_memcpy_req { 124 unsigned int from_rid; 125 void *from_buf; 126 u64 from_addr; 127 unsigned int to_rid; 128 u64 to_addr; 129 unsigned length; 130}; 131 132/** 133 * struct efx_local_addr - A MAC address on the vswitch without a VF. 134 * 135 * Siena does not have a switch, so VFs can't transmit data to each 136 * other. Instead the VFs must be made aware of the local addresses 137 * on the vswitch, so that they can arrange for an alternative 138 * software datapath to be used. 139 * 140 * @link: List head for insertion into efx->local_addr_list. 141 * @addr: Ethernet address 142 */ 143struct efx_local_addr { 144 struct list_head link; 145 u8 addr[ETH_ALEN]; 146}; 147 148/** 149 * struct efx_endpoint_page - Page of vfdi_endpoint structures 150 * 151 * @link: List head for insertion into efx->local_page_list. 152 * @ptr: Pointer to page. 153 * @addr: DMA address of page. 154 */ 155struct efx_endpoint_page { 156 struct list_head link; 157 void *ptr; 158 dma_addr_t addr; 159}; 160 161/* Buffer table entries are reserved txq0,rxq0,evq0,txq1,rxq1,evq1 */ 162#define EFX_BUFTBL_TXQ_BASE(_vf, _qid) \ 163 ((_vf)->buftbl_base + EFX_VF_BUFTBL_PER_VI * (_qid)) 164#define EFX_BUFTBL_RXQ_BASE(_vf, _qid) \ 165 (EFX_BUFTBL_TXQ_BASE(_vf, _qid) + \ 166 (EFX_MAX_DMAQ_SIZE * sizeof(efx_qword_t) / EFX_BUF_SIZE)) 167#define EFX_BUFTBL_EVQ_BASE(_vf, _qid) \ 168 (EFX_BUFTBL_TXQ_BASE(_vf, _qid) + \ 169 (2 * EFX_MAX_DMAQ_SIZE * sizeof(efx_qword_t) / EFX_BUF_SIZE)) 170 171#define EFX_FIELD_MASK(_field) \ 172 ((1 << _field ## _WIDTH) - 1) 173 174/* VFs can only use this many transmit channels */ 175static unsigned int vf_max_tx_channels = 2; 176module_param(vf_max_tx_channels, uint, 0444); 177MODULE_PARM_DESC(vf_max_tx_channels, 178 "Limit the number of TX channels VFs can use"); 179 180static int max_vfs = -1; 181module_param(max_vfs, int, 0444); 182MODULE_PARM_DESC(max_vfs, 183 "Reduce the number of VFs initialized by the driver"); 184 185/* Workqueue used by VFDI communication. We can't use the global 186 * workqueue because it may be running the VF driver's probe() 187 * routine, which will be blocked there waiting for a VFDI response. 188 */ 189static struct workqueue_struct *vfdi_workqueue; 190 191static unsigned abs_index(struct siena_vf *vf, unsigned index) 192{ 193 return EFX_VI_BASE + vf->index * efx_vf_size(vf->efx) + index; 194} 195 196static int efx_siena_sriov_cmd(struct efx_nic *efx, bool enable, 197 unsigned *vi_scale_out, unsigned *vf_total_out) 198{ 199 MCDI_DECLARE_BUF(inbuf, MC_CMD_SRIOV_IN_LEN); 200 MCDI_DECLARE_BUF(outbuf, MC_CMD_SRIOV_OUT_LEN); 201 unsigned vi_scale, vf_total; 202 size_t outlen; 203 int rc; 204 205 MCDI_SET_DWORD(inbuf, SRIOV_IN_ENABLE, enable ? 1 : 0); 206 MCDI_SET_DWORD(inbuf, SRIOV_IN_VI_BASE, EFX_VI_BASE); 207 MCDI_SET_DWORD(inbuf, SRIOV_IN_VF_COUNT, efx->vf_count); 208 209 rc = efx_siena_mcdi_rpc_quiet(efx, MC_CMD_SRIOV, inbuf, 210 MC_CMD_SRIOV_IN_LEN, outbuf, 211 MC_CMD_SRIOV_OUT_LEN, &outlen); 212 if (rc) 213 return rc; 214 if (outlen < MC_CMD_SRIOV_OUT_LEN) 215 return -EIO; 216 217 vf_total = MCDI_DWORD(outbuf, SRIOV_OUT_VF_TOTAL); 218 vi_scale = MCDI_DWORD(outbuf, SRIOV_OUT_VI_SCALE); 219 if (vi_scale > EFX_VI_SCALE_MAX) 220 return -EOPNOTSUPP; 221 222 if (vi_scale_out) 223 *vi_scale_out = vi_scale; 224 if (vf_total_out) 225 *vf_total_out = vf_total; 226 227 return 0; 228} 229 230static void efx_siena_sriov_usrev(struct efx_nic *efx, bool enabled) 231{ 232 struct siena_nic_data *nic_data = efx->nic_data; 233 efx_oword_t reg; 234 235 EFX_POPULATE_OWORD_2(reg, 236 FRF_CZ_USREV_DIS, enabled ? 0 : 1, 237 FRF_CZ_DFLT_EVQ, nic_data->vfdi_channel->channel); 238 efx_writeo(efx, ®, FR_CZ_USR_EV_CFG); 239} 240 241static int efx_siena_sriov_memcpy(struct efx_nic *efx, 242 struct efx_memcpy_req *req, 243 unsigned int count) 244{ 245 MCDI_DECLARE_BUF(inbuf, MCDI_CTL_SDU_LEN_MAX_V1); 246 MCDI_DECLARE_STRUCT_PTR(record); 247 unsigned int index, used; 248 u64 from_addr; 249 u32 from_rid; 250 int rc; 251 252 mb(); /* Finish writing source/reading dest before DMA starts */ 253 254 if (WARN_ON(count > MC_CMD_MEMCPY_IN_RECORD_MAXNUM)) 255 return -ENOBUFS; 256 used = MC_CMD_MEMCPY_IN_LEN(count); 257 258 for (index = 0; index < count; index++) { 259 record = MCDI_ARRAY_STRUCT_PTR(inbuf, MEMCPY_IN_RECORD, index); 260 MCDI_SET_DWORD(record, MEMCPY_RECORD_TYPEDEF_NUM_RECORDS, 261 count); 262 MCDI_SET_DWORD(record, MEMCPY_RECORD_TYPEDEF_TO_RID, 263 req->to_rid); 264 MCDI_SET_QWORD(record, MEMCPY_RECORD_TYPEDEF_TO_ADDR, 265 req->to_addr); 266 if (req->from_buf == NULL) { 267 from_rid = req->from_rid; 268 from_addr = req->from_addr; 269 } else { 270 if (WARN_ON(used + req->length > 271 MCDI_CTL_SDU_LEN_MAX_V1)) { 272 rc = -ENOBUFS; 273 goto out; 274 } 275 276 from_rid = MC_CMD_MEMCPY_RECORD_TYPEDEF_RID_INLINE; 277 from_addr = used; 278 memcpy(_MCDI_PTR(inbuf, used), req->from_buf, 279 req->length); 280 used += req->length; 281 } 282 283 MCDI_SET_DWORD(record, MEMCPY_RECORD_TYPEDEF_FROM_RID, from_rid); 284 MCDI_SET_QWORD(record, MEMCPY_RECORD_TYPEDEF_FROM_ADDR, 285 from_addr); 286 MCDI_SET_DWORD(record, MEMCPY_RECORD_TYPEDEF_LENGTH, 287 req->length); 288 289 ++req; 290 } 291 292 rc = efx_siena_mcdi_rpc(efx, MC_CMD_MEMCPY, inbuf, used, NULL, 0, NULL); 293out: 294 mb(); /* Don't write source/read dest before DMA is complete */ 295 296 return rc; 297} 298 299/* The TX filter is entirely controlled by this driver, and is modified 300 * underneath the feet of the VF 301 */ 302static void efx_siena_sriov_reset_tx_filter(struct siena_vf *vf) 303{ 304 struct efx_nic *efx = vf->efx; 305 struct efx_filter_spec filter; 306 u16 vlan; 307 int rc; 308 309 if (vf->tx_filter_id != -1) { 310 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, 311 vf->tx_filter_id); 312 netif_dbg(efx, hw, efx->net_dev, "Removed vf %s tx filter %d\n", 313 vf->pci_name, vf->tx_filter_id); 314 vf->tx_filter_id = -1; 315 } 316 317 if (is_zero_ether_addr(vf->addr.mac_addr)) 318 return; 319 320 /* Turn on TX filtering automatically if not explicitly 321 * enabled or disabled. 322 */ 323 if (vf->tx_filter_mode == VF_TX_FILTER_AUTO && vf_max_tx_channels <= 2) 324 vf->tx_filter_mode = VF_TX_FILTER_ON; 325 326 vlan = ntohs(vf->addr.tci) & VLAN_VID_MASK; 327 efx_filter_init_tx(&filter, abs_index(vf, 0)); 328 rc = efx_filter_set_eth_local(&filter, 329 vlan ? vlan : EFX_FILTER_VID_UNSPEC, 330 vf->addr.mac_addr); 331 BUG_ON(rc); 332 333 rc = efx_filter_insert_filter(efx, &filter, true); 334 if (rc < 0) { 335 netif_warn(efx, hw, efx->net_dev, 336 "Unable to migrate tx filter for vf %s\n", 337 vf->pci_name); 338 } else { 339 netif_dbg(efx, hw, efx->net_dev, "Inserted vf %s tx filter %d\n", 340 vf->pci_name, rc); 341 vf->tx_filter_id = rc; 342 } 343} 344 345/* The RX filter is managed here on behalf of the VF driver */ 346static void efx_siena_sriov_reset_rx_filter(struct siena_vf *vf) 347{ 348 struct efx_nic *efx = vf->efx; 349 struct efx_filter_spec filter; 350 u16 vlan; 351 int rc; 352 353 if (vf->rx_filter_id != -1) { 354 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, 355 vf->rx_filter_id); 356 netif_dbg(efx, hw, efx->net_dev, "Removed vf %s rx filter %d\n", 357 vf->pci_name, vf->rx_filter_id); 358 vf->rx_filter_id = -1; 359 } 360 361 if (!vf->rx_filtering || is_zero_ether_addr(vf->addr.mac_addr)) 362 return; 363 364 vlan = ntohs(vf->addr.tci) & VLAN_VID_MASK; 365 efx_filter_init_rx(&filter, EFX_FILTER_PRI_REQUIRED, 366 vf->rx_filter_flags, 367 abs_index(vf, vf->rx_filter_qid)); 368 rc = efx_filter_set_eth_local(&filter, 369 vlan ? vlan : EFX_FILTER_VID_UNSPEC, 370 vf->addr.mac_addr); 371 BUG_ON(rc); 372 373 rc = efx_filter_insert_filter(efx, &filter, true); 374 if (rc < 0) { 375 netif_warn(efx, hw, efx->net_dev, 376 "Unable to insert rx filter for vf %s\n", 377 vf->pci_name); 378 } else { 379 netif_dbg(efx, hw, efx->net_dev, "Inserted vf %s rx filter %d\n", 380 vf->pci_name, rc); 381 vf->rx_filter_id = rc; 382 } 383} 384 385static void __efx_siena_sriov_update_vf_addr(struct siena_vf *vf) 386{ 387 struct efx_nic *efx = vf->efx; 388 struct siena_nic_data *nic_data = efx->nic_data; 389 390 efx_siena_sriov_reset_tx_filter(vf); 391 efx_siena_sriov_reset_rx_filter(vf); 392 queue_work(vfdi_workqueue, &nic_data->peer_work); 393} 394 395/* Push the peer list to this VF. The caller must hold status_lock to interlock 396 * with VFDI requests, and they must be serialised against manipulation of 397 * local_page_list, either by acquiring local_lock or by running from 398 * efx_siena_sriov_peer_work() 399 */ 400static void __efx_siena_sriov_push_vf_status(struct siena_vf *vf) 401{ 402 struct efx_nic *efx = vf->efx; 403 struct siena_nic_data *nic_data = efx->nic_data; 404 struct vfdi_status *status = nic_data->vfdi_status.addr; 405 struct efx_memcpy_req copy[4]; 406 struct efx_endpoint_page *epp; 407 unsigned int pos, count; 408 unsigned data_offset; 409 efx_qword_t event; 410 411 WARN_ON(!mutex_is_locked(&vf->status_lock)); 412 WARN_ON(!vf->status_addr); 413 414 status->local = vf->addr; 415 status->generation_end = ++status->generation_start; 416 417 memset(copy, '\0', sizeof(copy)); 418 /* Write generation_start */ 419 copy[0].from_buf = &status->generation_start; 420 copy[0].to_rid = vf->pci_rid; 421 copy[0].to_addr = vf->status_addr + offsetof(struct vfdi_status, 422 generation_start); 423 copy[0].length = sizeof(status->generation_start); 424 /* DMA the rest of the structure (excluding the generations). This 425 * assumes that the non-generation portion of vfdi_status is in 426 * one chunk starting at the version member. 427 */ 428 data_offset = offsetof(struct vfdi_status, version); 429 copy[1].from_rid = efx->pci_dev->devfn; 430 copy[1].from_addr = nic_data->vfdi_status.dma_addr + data_offset; 431 copy[1].to_rid = vf->pci_rid; 432 copy[1].to_addr = vf->status_addr + data_offset; 433 copy[1].length = status->length - data_offset; 434 435 /* Copy the peer pages */ 436 pos = 2; 437 count = 0; 438 list_for_each_entry(epp, &nic_data->local_page_list, link) { 439 if (count == vf->peer_page_count) { 440 /* The VF driver will know they need to provide more 441 * pages because peer_addr_count is too large. 442 */ 443 break; 444 } 445 copy[pos].from_buf = NULL; 446 copy[pos].from_rid = efx->pci_dev->devfn; 447 copy[pos].from_addr = epp->addr; 448 copy[pos].to_rid = vf->pci_rid; 449 copy[pos].to_addr = vf->peer_page_addrs[count]; 450 copy[pos].length = EFX_PAGE_SIZE; 451 452 if (++pos == ARRAY_SIZE(copy)) { 453 efx_siena_sriov_memcpy(efx, copy, ARRAY_SIZE(copy)); 454 pos = 0; 455 } 456 ++count; 457 } 458 459 /* Write generation_end */ 460 copy[pos].from_buf = &status->generation_end; 461 copy[pos].to_rid = vf->pci_rid; 462 copy[pos].to_addr = vf->status_addr + offsetof(struct vfdi_status, 463 generation_end); 464 copy[pos].length = sizeof(status->generation_end); 465 efx_siena_sriov_memcpy(efx, copy, pos + 1); 466 467 /* Notify the guest */ 468 EFX_POPULATE_QWORD_3(event, 469 FSF_AZ_EV_CODE, FSE_CZ_EV_CODE_USER_EV, 470 VFDI_EV_SEQ, (vf->msg_seqno & 0xff), 471 VFDI_EV_TYPE, VFDI_EV_TYPE_STATUS); 472 ++vf->msg_seqno; 473 efx_farch_generate_event(efx, 474 EFX_VI_BASE + vf->index * efx_vf_size(efx), 475 &event); 476} 477 478static void efx_siena_sriov_bufs(struct efx_nic *efx, unsigned offset, 479 u64 *addr, unsigned count) 480{ 481 efx_qword_t buf; 482 unsigned pos; 483 484 for (pos = 0; pos < count; ++pos) { 485 EFX_POPULATE_QWORD_3(buf, 486 FRF_AZ_BUF_ADR_REGION, 0, 487 FRF_AZ_BUF_ADR_FBUF, 488 addr ? addr[pos] >> 12 : 0, 489 FRF_AZ_BUF_OWNER_ID_FBUF, 0); 490 efx_sram_writeq(efx, efx->membase + FR_BZ_BUF_FULL_TBL, 491 &buf, offset + pos); 492 } 493} 494 495static bool bad_vf_index(struct efx_nic *efx, unsigned index) 496{ 497 return index >= efx_vf_size(efx); 498} 499 500static bool bad_buf_count(unsigned buf_count, unsigned max_entry_count) 501{ 502 unsigned max_buf_count = max_entry_count * 503 sizeof(efx_qword_t) / EFX_BUF_SIZE; 504 505 return ((buf_count & (buf_count - 1)) || buf_count > max_buf_count); 506} 507 508/* Check that VI specified by per-port index belongs to a VF. 509 * Optionally set VF index and VI index within the VF. 510 */ 511static bool map_vi_index(struct efx_nic *efx, unsigned abs_index, 512 struct siena_vf **vf_out, unsigned *rel_index_out) 513{ 514 struct siena_nic_data *nic_data = efx->nic_data; 515 unsigned vf_i; 516 517 if (abs_index < EFX_VI_BASE) 518 return true; 519 vf_i = (abs_index - EFX_VI_BASE) / efx_vf_size(efx); 520 if (vf_i >= efx->vf_init_count) 521 return true; 522 523 if (vf_out) 524 *vf_out = nic_data->vf + vf_i; 525 if (rel_index_out) 526 *rel_index_out = abs_index % efx_vf_size(efx); 527 return false; 528} 529 530static int efx_vfdi_init_evq(struct siena_vf *vf) 531{ 532 struct efx_nic *efx = vf->efx; 533 struct vfdi_req *req = vf->buf.addr; 534 unsigned vf_evq = req->u.init_evq.index; 535 unsigned buf_count = req->u.init_evq.buf_count; 536 unsigned abs_evq = abs_index(vf, vf_evq); 537 unsigned buftbl = EFX_BUFTBL_EVQ_BASE(vf, vf_evq); 538 efx_oword_t reg; 539 540 if (bad_vf_index(efx, vf_evq) || 541 bad_buf_count(buf_count, EFX_MAX_VF_EVQ_SIZE)) { 542 if (net_ratelimit()) 543 netif_err(efx, hw, efx->net_dev, 544 "ERROR: Invalid INIT_EVQ from %s: evq %d bufs %d\n", 545 vf->pci_name, vf_evq, buf_count); 546 return VFDI_RC_EINVAL; 547 } 548 549 efx_siena_sriov_bufs(efx, buftbl, req->u.init_evq.addr, buf_count); 550 551 EFX_POPULATE_OWORD_3(reg, 552 FRF_CZ_TIMER_Q_EN, 1, 553 FRF_CZ_HOST_NOTIFY_MODE, 0, 554 FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS); 555 efx_writeo_table(efx, ®, FR_BZ_TIMER_TBL, abs_evq); 556 EFX_POPULATE_OWORD_3(reg, 557 FRF_AZ_EVQ_EN, 1, 558 FRF_AZ_EVQ_SIZE, __ffs(buf_count), 559 FRF_AZ_EVQ_BUF_BASE_ID, buftbl); 560 efx_writeo_table(efx, ®, FR_BZ_EVQ_PTR_TBL, abs_evq); 561 562 if (vf_evq == 0) { 563 memcpy(vf->evq0_addrs, req->u.init_evq.addr, 564 buf_count * sizeof(u64)); 565 vf->evq0_count = buf_count; 566 } 567 568 return VFDI_RC_SUCCESS; 569} 570 571static int efx_vfdi_init_rxq(struct siena_vf *vf) 572{ 573 struct efx_nic *efx = vf->efx; 574 struct vfdi_req *req = vf->buf.addr; 575 unsigned vf_rxq = req->u.init_rxq.index; 576 unsigned vf_evq = req->u.init_rxq.evq; 577 unsigned buf_count = req->u.init_rxq.buf_count; 578 unsigned buftbl = EFX_BUFTBL_RXQ_BASE(vf, vf_rxq); 579 unsigned label; 580 efx_oword_t reg; 581 582 if (bad_vf_index(efx, vf_evq) || bad_vf_index(efx, vf_rxq) || 583 vf_rxq >= VF_MAX_RX_QUEUES || 584 bad_buf_count(buf_count, EFX_MAX_DMAQ_SIZE)) { 585 if (net_ratelimit()) 586 netif_err(efx, hw, efx->net_dev, 587 "ERROR: Invalid INIT_RXQ from %s: rxq %d evq %d " 588 "buf_count %d\n", vf->pci_name, vf_rxq, 589 vf_evq, buf_count); 590 return VFDI_RC_EINVAL; 591 } 592 if (__test_and_set_bit(req->u.init_rxq.index, vf->rxq_mask)) 593 ++vf->rxq_count; 594 efx_siena_sriov_bufs(efx, buftbl, req->u.init_rxq.addr, buf_count); 595 596 label = req->u.init_rxq.label & EFX_FIELD_MASK(FRF_AZ_RX_DESCQ_LABEL); 597 EFX_POPULATE_OWORD_6(reg, 598 FRF_AZ_RX_DESCQ_BUF_BASE_ID, buftbl, 599 FRF_AZ_RX_DESCQ_EVQ_ID, abs_index(vf, vf_evq), 600 FRF_AZ_RX_DESCQ_LABEL, label, 601 FRF_AZ_RX_DESCQ_SIZE, __ffs(buf_count), 602 FRF_AZ_RX_DESCQ_JUMBO, 603 !!(req->u.init_rxq.flags & 604 VFDI_RXQ_FLAG_SCATTER_EN), 605 FRF_AZ_RX_DESCQ_EN, 1); 606 efx_writeo_table(efx, ®, FR_BZ_RX_DESC_PTR_TBL, 607 abs_index(vf, vf_rxq)); 608 609 return VFDI_RC_SUCCESS; 610} 611 612static int efx_vfdi_init_txq(struct siena_vf *vf) 613{ 614 struct efx_nic *efx = vf->efx; 615 struct vfdi_req *req = vf->buf.addr; 616 unsigned vf_txq = req->u.init_txq.index; 617 unsigned vf_evq = req->u.init_txq.evq; 618 unsigned buf_count = req->u.init_txq.buf_count; 619 unsigned buftbl = EFX_BUFTBL_TXQ_BASE(vf, vf_txq); 620 unsigned label, eth_filt_en; 621 efx_oword_t reg; 622 623 if (bad_vf_index(efx, vf_evq) || bad_vf_index(efx, vf_txq) || 624 vf_txq >= vf_max_tx_channels || 625 bad_buf_count(buf_count, EFX_MAX_DMAQ_SIZE)) { 626 if (net_ratelimit()) 627 netif_err(efx, hw, efx->net_dev, 628 "ERROR: Invalid INIT_TXQ from %s: txq %d evq %d " 629 "buf_count %d\n", vf->pci_name, vf_txq, 630 vf_evq, buf_count); 631 return VFDI_RC_EINVAL; 632 } 633 634 mutex_lock(&vf->txq_lock); 635 if (__test_and_set_bit(req->u.init_txq.index, vf->txq_mask)) 636 ++vf->txq_count; 637 mutex_unlock(&vf->txq_lock); 638 efx_siena_sriov_bufs(efx, buftbl, req->u.init_txq.addr, buf_count); 639 640 eth_filt_en = vf->tx_filter_mode == VF_TX_FILTER_ON; 641 642 label = req->u.init_txq.label & EFX_FIELD_MASK(FRF_AZ_TX_DESCQ_LABEL); 643 EFX_POPULATE_OWORD_8(reg, 644 FRF_CZ_TX_DPT_Q_MASK_WIDTH, min(efx->vi_scale, 1U), 645 FRF_CZ_TX_DPT_ETH_FILT_EN, eth_filt_en, 646 FRF_AZ_TX_DESCQ_EN, 1, 647 FRF_AZ_TX_DESCQ_BUF_BASE_ID, buftbl, 648 FRF_AZ_TX_DESCQ_EVQ_ID, abs_index(vf, vf_evq), 649 FRF_AZ_TX_DESCQ_LABEL, label, 650 FRF_AZ_TX_DESCQ_SIZE, __ffs(buf_count), 651 FRF_BZ_TX_NON_IP_DROP_DIS, 1); 652 efx_writeo_table(efx, ®, FR_BZ_TX_DESC_PTR_TBL, 653 abs_index(vf, vf_txq)); 654 655 return VFDI_RC_SUCCESS; 656} 657 658/* Returns true when efx_vfdi_fini_all_queues should wake */ 659static bool efx_vfdi_flush_wake(struct siena_vf *vf) 660{ 661 /* Ensure that all updates are visible to efx_vfdi_fini_all_queues() */ 662 smp_mb(); 663 664 return (!vf->txq_count && !vf->rxq_count) || 665 atomic_read(&vf->rxq_retry_count); 666} 667 668static void efx_vfdi_flush_clear(struct siena_vf *vf) 669{ 670 memset(vf->txq_mask, 0, sizeof(vf->txq_mask)); 671 vf->txq_count = 0; 672 memset(vf->rxq_mask, 0, sizeof(vf->rxq_mask)); 673 vf->rxq_count = 0; 674 memset(vf->rxq_retry_mask, 0, sizeof(vf->rxq_retry_mask)); 675 atomic_set(&vf->rxq_retry_count, 0); 676} 677 678static int efx_vfdi_fini_all_queues(struct siena_vf *vf) 679{ 680 struct efx_nic *efx = vf->efx; 681 efx_oword_t reg; 682 unsigned count = efx_vf_size(efx); 683 unsigned vf_offset = EFX_VI_BASE + vf->index * efx_vf_size(efx); 684 unsigned timeout = HZ; 685 unsigned index, rxqs_count; 686 MCDI_DECLARE_BUF(inbuf, MC_CMD_FLUSH_RX_QUEUES_IN_LENMAX); 687 int rc; 688 689 BUILD_BUG_ON(VF_MAX_RX_QUEUES > 690 MC_CMD_FLUSH_RX_QUEUES_IN_QID_OFST_MAXNUM); 691 692 rtnl_lock(); 693 efx_siena_prepare_flush(efx); 694 rtnl_unlock(); 695 696 /* Flush all the initialized queues */ 697 rxqs_count = 0; 698 for (index = 0; index < count; ++index) { 699 if (test_bit(index, vf->txq_mask)) { 700 EFX_POPULATE_OWORD_2(reg, 701 FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 702 FRF_AZ_TX_FLUSH_DESCQ, 703 vf_offset + index); 704 efx_writeo(efx, ®, FR_AZ_TX_FLUSH_DESCQ); 705 } 706 if (test_bit(index, vf->rxq_mask)) { 707 MCDI_SET_ARRAY_DWORD( 708 inbuf, FLUSH_RX_QUEUES_IN_QID_OFST, 709 rxqs_count, vf_offset + index); 710 rxqs_count++; 711 } 712 } 713 714 atomic_set(&vf->rxq_retry_count, 0); 715 while (timeout && (vf->rxq_count || vf->txq_count)) { 716 rc = efx_siena_mcdi_rpc(efx, MC_CMD_FLUSH_RX_QUEUES, inbuf, 717 MC_CMD_FLUSH_RX_QUEUES_IN_LEN(rxqs_count), 718 NULL, 0, NULL); 719 WARN_ON(rc < 0); 720 721 timeout = wait_event_timeout(vf->flush_waitq, 722 efx_vfdi_flush_wake(vf), 723 timeout); 724 rxqs_count = 0; 725 for (index = 0; index < count; ++index) { 726 if (test_and_clear_bit(index, vf->rxq_retry_mask)) { 727 atomic_dec(&vf->rxq_retry_count); 728 MCDI_SET_ARRAY_DWORD( 729 inbuf, FLUSH_RX_QUEUES_IN_QID_OFST, 730 rxqs_count, vf_offset + index); 731 rxqs_count++; 732 } 733 } 734 } 735 736 rtnl_lock(); 737 siena_finish_flush(efx); 738 rtnl_unlock(); 739 740 /* Irrespective of success/failure, fini the queues */ 741 EFX_ZERO_OWORD(reg); 742 for (index = 0; index < count; ++index) { 743 efx_writeo_table(efx, ®, FR_BZ_RX_DESC_PTR_TBL, 744 vf_offset + index); 745 efx_writeo_table(efx, ®, FR_BZ_TX_DESC_PTR_TBL, 746 vf_offset + index); 747 efx_writeo_table(efx, ®, FR_BZ_EVQ_PTR_TBL, 748 vf_offset + index); 749 efx_writeo_table(efx, ®, FR_BZ_TIMER_TBL, 750 vf_offset + index); 751 } 752 efx_siena_sriov_bufs(efx, vf->buftbl_base, NULL, 753 EFX_VF_BUFTBL_PER_VI * efx_vf_size(efx)); 754 efx_vfdi_flush_clear(vf); 755 756 vf->evq0_count = 0; 757 758 return timeout ? 0 : VFDI_RC_ETIMEDOUT; 759} 760 761static int efx_vfdi_insert_filter(struct siena_vf *vf) 762{ 763 struct efx_nic *efx = vf->efx; 764 struct siena_nic_data *nic_data = efx->nic_data; 765 struct vfdi_req *req = vf->buf.addr; 766 unsigned vf_rxq = req->u.mac_filter.rxq; 767 unsigned flags; 768 769 if (bad_vf_index(efx, vf_rxq) || vf->rx_filtering) { 770 if (net_ratelimit()) 771 netif_err(efx, hw, efx->net_dev, 772 "ERROR: Invalid INSERT_FILTER from %s: rxq %d " 773 "flags 0x%x\n", vf->pci_name, vf_rxq, 774 req->u.mac_filter.flags); 775 return VFDI_RC_EINVAL; 776 } 777 778 flags = 0; 779 if (req->u.mac_filter.flags & VFDI_MAC_FILTER_FLAG_RSS) 780 flags |= EFX_FILTER_FLAG_RX_RSS; 781 if (req->u.mac_filter.flags & VFDI_MAC_FILTER_FLAG_SCATTER) 782 flags |= EFX_FILTER_FLAG_RX_SCATTER; 783 vf->rx_filter_flags = flags; 784 vf->rx_filter_qid = vf_rxq; 785 vf->rx_filtering = true; 786 787 efx_siena_sriov_reset_rx_filter(vf); 788 queue_work(vfdi_workqueue, &nic_data->peer_work); 789 790 return VFDI_RC_SUCCESS; 791} 792 793static int efx_vfdi_remove_all_filters(struct siena_vf *vf) 794{ 795 struct efx_nic *efx = vf->efx; 796 struct siena_nic_data *nic_data = efx->nic_data; 797 798 vf->rx_filtering = false; 799 efx_siena_sriov_reset_rx_filter(vf); 800 queue_work(vfdi_workqueue, &nic_data->peer_work); 801 802 return VFDI_RC_SUCCESS; 803} 804 805static int efx_vfdi_set_status_page(struct siena_vf *vf) 806{ 807 struct efx_nic *efx = vf->efx; 808 struct siena_nic_data *nic_data = efx->nic_data; 809 struct vfdi_req *req = vf->buf.addr; 810 u64 page_count = req->u.set_status_page.peer_page_count; 811 u64 max_page_count = 812 (EFX_PAGE_SIZE - 813 offsetof(struct vfdi_req, u.set_status_page.peer_page_addr[0])) 814 / sizeof(req->u.set_status_page.peer_page_addr[0]); 815 816 if (!req->u.set_status_page.dma_addr || page_count > max_page_count) { 817 if (net_ratelimit()) 818 netif_err(efx, hw, efx->net_dev, 819 "ERROR: Invalid SET_STATUS_PAGE from %s\n", 820 vf->pci_name); 821 return VFDI_RC_EINVAL; 822 } 823 824 mutex_lock(&nic_data->local_lock); 825 mutex_lock(&vf->status_lock); 826 vf->status_addr = req->u.set_status_page.dma_addr; 827 828 kfree(vf->peer_page_addrs); 829 vf->peer_page_addrs = NULL; 830 vf->peer_page_count = 0; 831 832 if (page_count) { 833 vf->peer_page_addrs = kcalloc(page_count, sizeof(u64), 834 GFP_KERNEL); 835 if (vf->peer_page_addrs) { 836 memcpy(vf->peer_page_addrs, 837 req->u.set_status_page.peer_page_addr, 838 page_count * sizeof(u64)); 839 vf->peer_page_count = page_count; 840 } 841 } 842 843 __efx_siena_sriov_push_vf_status(vf); 844 mutex_unlock(&vf->status_lock); 845 mutex_unlock(&nic_data->local_lock); 846 847 return VFDI_RC_SUCCESS; 848} 849 850static int efx_vfdi_clear_status_page(struct siena_vf *vf) 851{ 852 mutex_lock(&vf->status_lock); 853 vf->status_addr = 0; 854 mutex_unlock(&vf->status_lock); 855 856 return VFDI_RC_SUCCESS; 857} 858 859typedef int (*efx_vfdi_op_t)(struct siena_vf *vf); 860 861static const efx_vfdi_op_t vfdi_ops[VFDI_OP_LIMIT] = { 862 [VFDI_OP_INIT_EVQ] = efx_vfdi_init_evq, 863 [VFDI_OP_INIT_TXQ] = efx_vfdi_init_txq, 864 [VFDI_OP_INIT_RXQ] = efx_vfdi_init_rxq, 865 [VFDI_OP_FINI_ALL_QUEUES] = efx_vfdi_fini_all_queues, 866 [VFDI_OP_INSERT_FILTER] = efx_vfdi_insert_filter, 867 [VFDI_OP_REMOVE_ALL_FILTERS] = efx_vfdi_remove_all_filters, 868 [VFDI_OP_SET_STATUS_PAGE] = efx_vfdi_set_status_page, 869 [VFDI_OP_CLEAR_STATUS_PAGE] = efx_vfdi_clear_status_page, 870}; 871 872static void efx_siena_sriov_vfdi(struct work_struct *work) 873{ 874 struct siena_vf *vf = container_of(work, struct siena_vf, req); 875 struct efx_nic *efx = vf->efx; 876 struct vfdi_req *req = vf->buf.addr; 877 struct efx_memcpy_req copy[2]; 878 int rc; 879 880 /* Copy this page into the local address space */ 881 memset(copy, '\0', sizeof(copy)); 882 copy[0].from_rid = vf->pci_rid; 883 copy[0].from_addr = vf->req_addr; 884 copy[0].to_rid = efx->pci_dev->devfn; 885 copy[0].to_addr = vf->buf.dma_addr; 886 copy[0].length = EFX_PAGE_SIZE; 887 rc = efx_siena_sriov_memcpy(efx, copy, 1); 888 if (rc) { 889 /* If we can't get the request, we can't reply to the caller */ 890 if (net_ratelimit()) 891 netif_err(efx, hw, efx->net_dev, 892 "ERROR: Unable to fetch VFDI request from %s rc %d\n", 893 vf->pci_name, -rc); 894 vf->busy = false; 895 return; 896 } 897 898 if (req->op < VFDI_OP_LIMIT && vfdi_ops[req->op] != NULL) { 899 rc = vfdi_ops[req->op](vf); 900 if (rc == 0) { 901 netif_dbg(efx, hw, efx->net_dev, 902 "vfdi request %d from %s ok\n", 903 req->op, vf->pci_name); 904 } 905 } else { 906 netif_dbg(efx, hw, efx->net_dev, 907 "ERROR: Unrecognised request %d from VF %s addr " 908 "%llx\n", req->op, vf->pci_name, 909 (unsigned long long)vf->req_addr); 910 rc = VFDI_RC_EOPNOTSUPP; 911 } 912 913 /* Allow subsequent VF requests */ 914 vf->busy = false; 915 smp_wmb(); 916 917 /* Respond to the request */ 918 req->rc = rc; 919 req->op = VFDI_OP_RESPONSE; 920 921 memset(copy, '\0', sizeof(copy)); 922 copy[0].from_buf = &req->rc; 923 copy[0].to_rid = vf->pci_rid; 924 copy[0].to_addr = vf->req_addr + offsetof(struct vfdi_req, rc); 925 copy[0].length = sizeof(req->rc); 926 copy[1].from_buf = &req->op; 927 copy[1].to_rid = vf->pci_rid; 928 copy[1].to_addr = vf->req_addr + offsetof(struct vfdi_req, op); 929 copy[1].length = sizeof(req->op); 930 931 (void)efx_siena_sriov_memcpy(efx, copy, ARRAY_SIZE(copy)); 932} 933 934 935 936/* After a reset the event queues inside the guests no longer exist. Fill the 937 * event ring in guest memory with VFDI reset events, then (re-initialise) the 938 * event queue to raise an interrupt. The guest driver will then recover. 939 */ 940 941static void efx_siena_sriov_reset_vf(struct siena_vf *vf, 942 struct efx_buffer *buffer) 943{ 944 struct efx_nic *efx = vf->efx; 945 struct efx_memcpy_req copy_req[4]; 946 efx_qword_t event; 947 unsigned int pos, count, k, buftbl, abs_evq; 948 efx_oword_t reg; 949 efx_dword_t ptr; 950 int rc; 951 952 BUG_ON(buffer->len != EFX_PAGE_SIZE); 953 954 if (!vf->evq0_count) 955 return; 956 BUG_ON(vf->evq0_count & (vf->evq0_count - 1)); 957 958 mutex_lock(&vf->status_lock); 959 EFX_POPULATE_QWORD_3(event, 960 FSF_AZ_EV_CODE, FSE_CZ_EV_CODE_USER_EV, 961 VFDI_EV_SEQ, vf->msg_seqno, 962 VFDI_EV_TYPE, VFDI_EV_TYPE_RESET); 963 vf->msg_seqno++; 964 for (pos = 0; pos < EFX_PAGE_SIZE; pos += sizeof(event)) 965 memcpy(buffer->addr + pos, &event, sizeof(event)); 966 967 for (pos = 0; pos < vf->evq0_count; pos += count) { 968 count = min_t(unsigned, vf->evq0_count - pos, 969 ARRAY_SIZE(copy_req)); 970 for (k = 0; k < count; k++) { 971 copy_req[k].from_buf = NULL; 972 copy_req[k].from_rid = efx->pci_dev->devfn; 973 copy_req[k].from_addr = buffer->dma_addr; 974 copy_req[k].to_rid = vf->pci_rid; 975 copy_req[k].to_addr = vf->evq0_addrs[pos + k]; 976 copy_req[k].length = EFX_PAGE_SIZE; 977 } 978 rc = efx_siena_sriov_memcpy(efx, copy_req, count); 979 if (rc) { 980 if (net_ratelimit()) 981 netif_err(efx, hw, efx->net_dev, 982 "ERROR: Unable to notify %s of reset" 983 ": %d\n", vf->pci_name, -rc); 984 break; 985 } 986 } 987 988 /* Reinitialise, arm and trigger evq0 */ 989 abs_evq = abs_index(vf, 0); 990 buftbl = EFX_BUFTBL_EVQ_BASE(vf, 0); 991 efx_siena_sriov_bufs(efx, buftbl, vf->evq0_addrs, vf->evq0_count); 992 993 EFX_POPULATE_OWORD_3(reg, 994 FRF_CZ_TIMER_Q_EN, 1, 995 FRF_CZ_HOST_NOTIFY_MODE, 0, 996 FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS); 997 efx_writeo_table(efx, ®, FR_BZ_TIMER_TBL, abs_evq); 998 EFX_POPULATE_OWORD_3(reg, 999 FRF_AZ_EVQ_EN, 1, 1000 FRF_AZ_EVQ_SIZE, __ffs(vf->evq0_count), 1001 FRF_AZ_EVQ_BUF_BASE_ID, buftbl); 1002 efx_writeo_table(efx, ®, FR_BZ_EVQ_PTR_TBL, abs_evq); 1003 EFX_POPULATE_DWORD_1(ptr, FRF_AZ_EVQ_RPTR, 0); 1004 efx_writed(efx, &ptr, FR_BZ_EVQ_RPTR + FR_BZ_EVQ_RPTR_STEP * abs_evq); 1005 1006 mutex_unlock(&vf->status_lock); 1007} 1008 1009static void efx_siena_sriov_reset_vf_work(struct work_struct *work) 1010{ 1011 struct siena_vf *vf = container_of(work, struct siena_vf, req); 1012 struct efx_nic *efx = vf->efx; 1013 struct efx_buffer buf; 1014 1015 if (!efx_siena_alloc_buffer(efx, &buf, EFX_PAGE_SIZE, GFP_NOIO)) { 1016 efx_siena_sriov_reset_vf(vf, &buf); 1017 efx_siena_free_buffer(efx, &buf); 1018 } 1019} 1020 1021static void efx_siena_sriov_handle_no_channel(struct efx_nic *efx) 1022{ 1023 netif_err(efx, drv, efx->net_dev, 1024 "ERROR: IOV requires MSI-X and 1 additional interrupt" 1025 "vector. IOV disabled\n"); 1026 efx->vf_count = 0; 1027} 1028 1029static int efx_siena_sriov_probe_channel(struct efx_channel *channel) 1030{ 1031 struct siena_nic_data *nic_data = channel->efx->nic_data; 1032 nic_data->vfdi_channel = channel; 1033 1034 return 0; 1035} 1036 1037static void 1038efx_siena_sriov_get_channel_name(struct efx_channel *channel, 1039 char *buf, size_t len) 1040{ 1041 snprintf(buf, len, "%s-iov", channel->efx->name); 1042} 1043 1044static const struct efx_channel_type efx_siena_sriov_channel_type = { 1045 .handle_no_channel = efx_siena_sriov_handle_no_channel, 1046 .pre_probe = efx_siena_sriov_probe_channel, 1047 .post_remove = efx_siena_channel_dummy_op_void, 1048 .get_name = efx_siena_sriov_get_channel_name, 1049 /* no copy operation; channel must not be reallocated */ 1050 .keep_eventq = true, 1051}; 1052 1053void efx_siena_sriov_probe(struct efx_nic *efx) 1054{ 1055 unsigned count; 1056 1057 if (!max_vfs) 1058 return; 1059 1060 if (efx_siena_sriov_cmd(efx, false, &efx->vi_scale, &count)) { 1061 pci_info(efx->pci_dev, "no SR-IOV VFs probed\n"); 1062 return; 1063 } 1064 if (count > 0 && count > max_vfs) 1065 count = max_vfs; 1066 1067 /* efx_nic_dimension_resources() will reduce vf_count as appopriate */ 1068 efx->vf_count = count; 1069 1070 efx->extra_channel_type[EFX_EXTRA_CHANNEL_IOV] = &efx_siena_sriov_channel_type; 1071} 1072 1073/* Copy the list of individual addresses into the vfdi_status.peers 1074 * array and auxiliary pages, protected by %local_lock. Drop that lock 1075 * and then broadcast the address list to every VF. 1076 */ 1077static void efx_siena_sriov_peer_work(struct work_struct *data) 1078{ 1079 struct siena_nic_data *nic_data = container_of(data, 1080 struct siena_nic_data, 1081 peer_work); 1082 struct efx_nic *efx = nic_data->efx; 1083 struct vfdi_status *vfdi_status = nic_data->vfdi_status.addr; 1084 struct siena_vf *vf; 1085 struct efx_local_addr *local_addr; 1086 struct vfdi_endpoint *peer; 1087 struct efx_endpoint_page *epp; 1088 struct list_head pages; 1089 unsigned int peer_space; 1090 unsigned int peer_count; 1091 unsigned int pos; 1092 1093 mutex_lock(&nic_data->local_lock); 1094 1095 /* Move the existing peer pages off %local_page_list */ 1096 INIT_LIST_HEAD(&pages); 1097 list_splice_tail_init(&nic_data->local_page_list, &pages); 1098 1099 /* Populate the VF addresses starting from entry 1 (entry 0 is 1100 * the PF address) 1101 */ 1102 peer = vfdi_status->peers + 1; 1103 peer_space = ARRAY_SIZE(vfdi_status->peers) - 1; 1104 peer_count = 1; 1105 for (pos = 0; pos < efx->vf_count; ++pos) { 1106 vf = nic_data->vf + pos; 1107 1108 mutex_lock(&vf->status_lock); 1109 if (vf->rx_filtering && !is_zero_ether_addr(vf->addr.mac_addr)) { 1110 *peer++ = vf->addr; 1111 ++peer_count; 1112 --peer_space; 1113 BUG_ON(peer_space == 0); 1114 } 1115 mutex_unlock(&vf->status_lock); 1116 } 1117 1118 /* Fill the remaining addresses */ 1119 list_for_each_entry(local_addr, &nic_data->local_addr_list, link) { 1120 ether_addr_copy(peer->mac_addr, local_addr->addr); 1121 peer->tci = 0; 1122 ++peer; 1123 ++peer_count; 1124 if (--peer_space == 0) { 1125 if (list_empty(&pages)) { 1126 epp = kmalloc(sizeof(*epp), GFP_KERNEL); 1127 if (!epp) 1128 break; 1129 epp->ptr = dma_alloc_coherent( 1130 &efx->pci_dev->dev, EFX_PAGE_SIZE, 1131 &epp->addr, GFP_KERNEL); 1132 if (!epp->ptr) { 1133 kfree(epp); 1134 break; 1135 } 1136 } else { 1137 epp = list_first_entry( 1138 &pages, struct efx_endpoint_page, link); 1139 list_del(&epp->link); 1140 } 1141 1142 list_add_tail(&epp->link, &nic_data->local_page_list); 1143 peer = (struct vfdi_endpoint *)epp->ptr; 1144 peer_space = EFX_PAGE_SIZE / sizeof(struct vfdi_endpoint); 1145 } 1146 } 1147 vfdi_status->peer_count = peer_count; 1148 mutex_unlock(&nic_data->local_lock); 1149 1150 /* Free any now unused endpoint pages */ 1151 while (!list_empty(&pages)) { 1152 epp = list_first_entry( 1153 &pages, struct efx_endpoint_page, link); 1154 list_del(&epp->link); 1155 dma_free_coherent(&efx->pci_dev->dev, EFX_PAGE_SIZE, 1156 epp->ptr, epp->addr); 1157 kfree(epp); 1158 } 1159 1160 /* Finally, push the pages */ 1161 for (pos = 0; pos < efx->vf_count; ++pos) { 1162 vf = nic_data->vf + pos; 1163 1164 mutex_lock(&vf->status_lock); 1165 if (vf->status_addr) 1166 __efx_siena_sriov_push_vf_status(vf); 1167 mutex_unlock(&vf->status_lock); 1168 } 1169} 1170 1171static void efx_siena_sriov_free_local(struct efx_nic *efx) 1172{ 1173 struct siena_nic_data *nic_data = efx->nic_data; 1174 struct efx_local_addr *local_addr; 1175 struct efx_endpoint_page *epp; 1176 1177 while (!list_empty(&nic_data->local_addr_list)) { 1178 local_addr = list_first_entry(&nic_data->local_addr_list, 1179 struct efx_local_addr, link); 1180 list_del(&local_addr->link); 1181 kfree(local_addr); 1182 } 1183 1184 while (!list_empty(&nic_data->local_page_list)) { 1185 epp = list_first_entry(&nic_data->local_page_list, 1186 struct efx_endpoint_page, link); 1187 list_del(&epp->link); 1188 dma_free_coherent(&efx->pci_dev->dev, EFX_PAGE_SIZE, 1189 epp->ptr, epp->addr); 1190 kfree(epp); 1191 } 1192} 1193 1194static int efx_siena_sriov_vf_alloc(struct efx_nic *efx) 1195{ 1196 unsigned index; 1197 struct siena_vf *vf; 1198 struct siena_nic_data *nic_data = efx->nic_data; 1199 1200 nic_data->vf = kcalloc(efx->vf_count, sizeof(*nic_data->vf), 1201 GFP_KERNEL); 1202 if (!nic_data->vf) 1203 return -ENOMEM; 1204 1205 for (index = 0; index < efx->vf_count; ++index) { 1206 vf = nic_data->vf + index; 1207 1208 vf->efx = efx; 1209 vf->index = index; 1210 vf->rx_filter_id = -1; 1211 vf->tx_filter_mode = VF_TX_FILTER_AUTO; 1212 vf->tx_filter_id = -1; 1213 INIT_WORK(&vf->req, efx_siena_sriov_vfdi); 1214 INIT_WORK(&vf->reset_work, efx_siena_sriov_reset_vf_work); 1215 init_waitqueue_head(&vf->flush_waitq); 1216 mutex_init(&vf->status_lock); 1217 mutex_init(&vf->txq_lock); 1218 } 1219 1220 return 0; 1221} 1222 1223static void efx_siena_sriov_vfs_fini(struct efx_nic *efx) 1224{ 1225 struct siena_nic_data *nic_data = efx->nic_data; 1226 struct siena_vf *vf; 1227 unsigned int pos; 1228 1229 for (pos = 0; pos < efx->vf_count; ++pos) { 1230 vf = nic_data->vf + pos; 1231 1232 efx_siena_free_buffer(efx, &vf->buf); 1233 kfree(vf->peer_page_addrs); 1234 vf->peer_page_addrs = NULL; 1235 vf->peer_page_count = 0; 1236 1237 vf->evq0_count = 0; 1238 } 1239} 1240 1241static int efx_siena_sriov_vfs_init(struct efx_nic *efx) 1242{ 1243 struct pci_dev *pci_dev = efx->pci_dev; 1244 struct siena_nic_data *nic_data = efx->nic_data; 1245 unsigned index, devfn, sriov, buftbl_base; 1246 u16 offset, stride; 1247 struct siena_vf *vf; 1248 int rc; 1249 1250 sriov = pci_find_ext_capability(pci_dev, PCI_EXT_CAP_ID_SRIOV); 1251 if (!sriov) 1252 return -ENOENT; 1253 1254 pci_read_config_word(pci_dev, sriov + PCI_SRIOV_VF_OFFSET, &offset); 1255 pci_read_config_word(pci_dev, sriov + PCI_SRIOV_VF_STRIDE, &stride); 1256 1257 buftbl_base = nic_data->vf_buftbl_base; 1258 devfn = pci_dev->devfn + offset; 1259 for (index = 0; index < efx->vf_count; ++index) { 1260 vf = nic_data->vf + index; 1261 1262 /* Reserve buffer entries */ 1263 vf->buftbl_base = buftbl_base; 1264 buftbl_base += EFX_VF_BUFTBL_PER_VI * efx_vf_size(efx); 1265 1266 vf->pci_rid = devfn; 1267 snprintf(vf->pci_name, sizeof(vf->pci_name), 1268 "%04x:%02x:%02x.%d", 1269 pci_domain_nr(pci_dev->bus), pci_dev->bus->number, 1270 PCI_SLOT(devfn), PCI_FUNC(devfn)); 1271 1272 rc = efx_siena_alloc_buffer(efx, &vf->buf, EFX_PAGE_SIZE, 1273 GFP_KERNEL); 1274 if (rc) 1275 goto fail; 1276 1277 devfn += stride; 1278 } 1279 1280 return 0; 1281 1282fail: 1283 efx_siena_sriov_vfs_fini(efx); 1284 return rc; 1285} 1286 1287int efx_siena_sriov_init(struct efx_nic *efx) 1288{ 1289 struct net_device *net_dev = efx->net_dev; 1290 struct siena_nic_data *nic_data = efx->nic_data; 1291 struct vfdi_status *vfdi_status; 1292 int rc; 1293 1294 /* Ensure there's room for vf_channel */ 1295 BUILD_BUG_ON(EFX_MAX_CHANNELS + 1 >= EFX_VI_BASE); 1296 /* Ensure that VI_BASE is aligned on VI_SCALE */ 1297 BUILD_BUG_ON(EFX_VI_BASE & ((1 << EFX_VI_SCALE_MAX) - 1)); 1298 1299 if (efx->vf_count == 0) 1300 return 0; 1301 1302 rc = efx_siena_sriov_cmd(efx, true, NULL, NULL); 1303 if (rc) 1304 goto fail_cmd; 1305 1306 rc = efx_siena_alloc_buffer(efx, &nic_data->vfdi_status, 1307 sizeof(*vfdi_status), GFP_KERNEL); 1308 if (rc) 1309 goto fail_status; 1310 vfdi_status = nic_data->vfdi_status.addr; 1311 memset(vfdi_status, 0, sizeof(*vfdi_status)); 1312 vfdi_status->version = 1; 1313 vfdi_status->length = sizeof(*vfdi_status); 1314 vfdi_status->max_tx_channels = vf_max_tx_channels; 1315 vfdi_status->vi_scale = efx->vi_scale; 1316 vfdi_status->rss_rxq_count = efx->rss_spread; 1317 vfdi_status->peer_count = 1 + efx->vf_count; 1318 vfdi_status->timer_quantum_ns = efx->timer_quantum_ns; 1319 1320 rc = efx_siena_sriov_vf_alloc(efx); 1321 if (rc) 1322 goto fail_alloc; 1323 1324 mutex_init(&nic_data->local_lock); 1325 INIT_WORK(&nic_data->peer_work, efx_siena_sriov_peer_work); 1326 INIT_LIST_HEAD(&nic_data->local_addr_list); 1327 INIT_LIST_HEAD(&nic_data->local_page_list); 1328 1329 rc = efx_siena_sriov_vfs_init(efx); 1330 if (rc) 1331 goto fail_vfs; 1332 1333 rtnl_lock(); 1334 ether_addr_copy(vfdi_status->peers[0].mac_addr, net_dev->dev_addr); 1335 efx->vf_init_count = efx->vf_count; 1336 rtnl_unlock(); 1337 1338 efx_siena_sriov_usrev(efx, true); 1339 1340 /* At this point we must be ready to accept VFDI requests */ 1341 1342 rc = pci_enable_sriov(efx->pci_dev, efx->vf_count); 1343 if (rc) 1344 goto fail_pci; 1345 1346 netif_info(efx, probe, net_dev, 1347 "enabled SR-IOV for %d VFs, %d VI per VF\n", 1348 efx->vf_count, efx_vf_size(efx)); 1349 return 0; 1350 1351fail_pci: 1352 efx_siena_sriov_usrev(efx, false); 1353 rtnl_lock(); 1354 efx->vf_init_count = 0; 1355 rtnl_unlock(); 1356 efx_siena_sriov_vfs_fini(efx); 1357fail_vfs: 1358 cancel_work_sync(&nic_data->peer_work); 1359 efx_siena_sriov_free_local(efx); 1360 kfree(nic_data->vf); 1361fail_alloc: 1362 efx_siena_free_buffer(efx, &nic_data->vfdi_status); 1363fail_status: 1364 efx_siena_sriov_cmd(efx, false, NULL, NULL); 1365fail_cmd: 1366 return rc; 1367} 1368 1369void efx_siena_sriov_fini(struct efx_nic *efx) 1370{ 1371 struct siena_vf *vf; 1372 unsigned int pos; 1373 struct siena_nic_data *nic_data = efx->nic_data; 1374 1375 if (efx->vf_init_count == 0) 1376 return; 1377 1378 /* Disable all interfaces to reconfiguration */ 1379 BUG_ON(nic_data->vfdi_channel->enabled); 1380 efx_siena_sriov_usrev(efx, false); 1381 rtnl_lock(); 1382 efx->vf_init_count = 0; 1383 rtnl_unlock(); 1384 1385 /* Flush all reconfiguration work */ 1386 for (pos = 0; pos < efx->vf_count; ++pos) { 1387 vf = nic_data->vf + pos; 1388 cancel_work_sync(&vf->req); 1389 cancel_work_sync(&vf->reset_work); 1390 } 1391 cancel_work_sync(&nic_data->peer_work); 1392 1393 pci_disable_sriov(efx->pci_dev); 1394 1395 /* Tear down back-end state */ 1396 efx_siena_sriov_vfs_fini(efx); 1397 efx_siena_sriov_free_local(efx); 1398 kfree(nic_data->vf); 1399 efx_siena_free_buffer(efx, &nic_data->vfdi_status); 1400 efx_siena_sriov_cmd(efx, false, NULL, NULL); 1401} 1402 1403void efx_siena_sriov_event(struct efx_channel *channel, efx_qword_t *event) 1404{ 1405 struct efx_nic *efx = channel->efx; 1406 struct siena_vf *vf; 1407 unsigned qid, seq, type, data; 1408 1409 qid = EFX_QWORD_FIELD(*event, FSF_CZ_USER_QID); 1410 1411 /* USR_EV_REG_VALUE is dword0, so access the VFDI_EV fields directly */ 1412 BUILD_BUG_ON(FSF_CZ_USER_EV_REG_VALUE_LBN != 0); 1413 seq = EFX_QWORD_FIELD(*event, VFDI_EV_SEQ); 1414 type = EFX_QWORD_FIELD(*event, VFDI_EV_TYPE); 1415 data = EFX_QWORD_FIELD(*event, VFDI_EV_DATA); 1416 1417 netif_vdbg(efx, hw, efx->net_dev, 1418 "USR_EV event from qid %d seq 0x%x type %d data 0x%x\n", 1419 qid, seq, type, data); 1420 1421 if (map_vi_index(efx, qid, &vf, NULL)) 1422 return; 1423 if (vf->busy) 1424 goto error; 1425 1426 if (type == VFDI_EV_TYPE_REQ_WORD0) { 1427 /* Resynchronise */ 1428 vf->req_type = VFDI_EV_TYPE_REQ_WORD0; 1429 vf->req_seqno = seq + 1; 1430 vf->req_addr = 0; 1431 } else if (seq != (vf->req_seqno++ & 0xff) || type != vf->req_type) 1432 goto error; 1433 1434 switch (vf->req_type) { 1435 case VFDI_EV_TYPE_REQ_WORD0: 1436 case VFDI_EV_TYPE_REQ_WORD1: 1437 case VFDI_EV_TYPE_REQ_WORD2: 1438 vf->req_addr |= (u64)data << (vf->req_type << 4); 1439 ++vf->req_type; 1440 return; 1441 1442 case VFDI_EV_TYPE_REQ_WORD3: 1443 vf->req_addr |= (u64)data << 48; 1444 vf->req_type = VFDI_EV_TYPE_REQ_WORD0; 1445 vf->busy = true; 1446 queue_work(vfdi_workqueue, &vf->req); 1447 return; 1448 } 1449 1450error: 1451 if (net_ratelimit()) 1452 netif_err(efx, hw, efx->net_dev, 1453 "ERROR: Screaming VFDI request from %s\n", 1454 vf->pci_name); 1455 /* Reset the request and sequence number */ 1456 vf->req_type = VFDI_EV_TYPE_REQ_WORD0; 1457 vf->req_seqno = seq + 1; 1458} 1459 1460void efx_siena_sriov_flr(struct efx_nic *efx, unsigned vf_i) 1461{ 1462 struct siena_nic_data *nic_data = efx->nic_data; 1463 struct siena_vf *vf; 1464 1465 if (vf_i > efx->vf_init_count) 1466 return; 1467 vf = nic_data->vf + vf_i; 1468 netif_info(efx, hw, efx->net_dev, 1469 "FLR on VF %s\n", vf->pci_name); 1470 1471 vf->status_addr = 0; 1472 efx_vfdi_remove_all_filters(vf); 1473 efx_vfdi_flush_clear(vf); 1474 1475 vf->evq0_count = 0; 1476} 1477 1478int efx_siena_sriov_mac_address_changed(struct efx_nic *efx) 1479{ 1480 struct siena_nic_data *nic_data = efx->nic_data; 1481 struct vfdi_status *vfdi_status = nic_data->vfdi_status.addr; 1482 1483 if (!efx->vf_init_count) 1484 return 0; 1485 ether_addr_copy(vfdi_status->peers[0].mac_addr, 1486 efx->net_dev->dev_addr); 1487 queue_work(vfdi_workqueue, &nic_data->peer_work); 1488 1489 return 0; 1490} 1491 1492void efx_siena_sriov_tx_flush_done(struct efx_nic *efx, efx_qword_t *event) 1493{ 1494 struct siena_vf *vf; 1495 unsigned queue, qid; 1496 1497 queue = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA); 1498 if (map_vi_index(efx, queue, &vf, &qid)) 1499 return; 1500 /* Ignore flush completions triggered by an FLR */ 1501 if (!test_bit(qid, vf->txq_mask)) 1502 return; 1503 1504 __clear_bit(qid, vf->txq_mask); 1505 --vf->txq_count; 1506 1507 if (efx_vfdi_flush_wake(vf)) 1508 wake_up(&vf->flush_waitq); 1509} 1510 1511void efx_siena_sriov_rx_flush_done(struct efx_nic *efx, efx_qword_t *event) 1512{ 1513 struct siena_vf *vf; 1514 unsigned ev_failed, queue, qid; 1515 1516 queue = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_RX_DESCQ_ID); 1517 ev_failed = EFX_QWORD_FIELD(*event, 1518 FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL); 1519 if (map_vi_index(efx, queue, &vf, &qid)) 1520 return; 1521 if (!test_bit(qid, vf->rxq_mask)) 1522 return; 1523 1524 if (ev_failed) { 1525 set_bit(qid, vf->rxq_retry_mask); 1526 atomic_inc(&vf->rxq_retry_count); 1527 } else { 1528 __clear_bit(qid, vf->rxq_mask); 1529 --vf->rxq_count; 1530 } 1531 if (efx_vfdi_flush_wake(vf)) 1532 wake_up(&vf->flush_waitq); 1533} 1534 1535/* Called from napi. Schedule the reset work item */ 1536void efx_siena_sriov_desc_fetch_err(struct efx_nic *efx, unsigned dmaq) 1537{ 1538 struct siena_vf *vf; 1539 unsigned int rel; 1540 1541 if (map_vi_index(efx, dmaq, &vf, &rel)) 1542 return; 1543 1544 if (net_ratelimit()) 1545 netif_err(efx, hw, efx->net_dev, 1546 "VF %d DMA Q %d reports descriptor fetch error.\n", 1547 vf->index, rel); 1548 queue_work(vfdi_workqueue, &vf->reset_work); 1549} 1550 1551/* Reset all VFs */ 1552void efx_siena_sriov_reset(struct efx_nic *efx) 1553{ 1554 struct siena_nic_data *nic_data = efx->nic_data; 1555 unsigned int vf_i; 1556 struct efx_buffer buf; 1557 struct siena_vf *vf; 1558 1559 ASSERT_RTNL(); 1560 1561 if (efx->vf_init_count == 0) 1562 return; 1563 1564 efx_siena_sriov_usrev(efx, true); 1565 (void)efx_siena_sriov_cmd(efx, true, NULL, NULL); 1566 1567 if (efx_siena_alloc_buffer(efx, &buf, EFX_PAGE_SIZE, GFP_NOIO)) 1568 return; 1569 1570 for (vf_i = 0; vf_i < efx->vf_init_count; ++vf_i) { 1571 vf = nic_data->vf + vf_i; 1572 efx_siena_sriov_reset_vf(vf, &buf); 1573 } 1574 1575 efx_siena_free_buffer(efx, &buf); 1576} 1577 1578int efx_init_sriov(void) 1579{ 1580 /* A single threaded workqueue is sufficient. efx_siena_sriov_vfdi() and 1581 * efx_siena_sriov_peer_work() spend almost all their time sleeping for 1582 * MCDI to complete anyway 1583 */ 1584 vfdi_workqueue = create_singlethread_workqueue("sfc_vfdi"); 1585 if (!vfdi_workqueue) 1586 return -ENOMEM; 1587 return 0; 1588} 1589 1590void efx_fini_sriov(void) 1591{ 1592 destroy_workqueue(vfdi_workqueue); 1593} 1594 1595int efx_siena_sriov_set_vf_mac(struct efx_nic *efx, int vf_i, const u8 *mac) 1596{ 1597 struct siena_nic_data *nic_data = efx->nic_data; 1598 struct siena_vf *vf; 1599 1600 if (vf_i >= efx->vf_init_count) 1601 return -EINVAL; 1602 vf = nic_data->vf + vf_i; 1603 1604 mutex_lock(&vf->status_lock); 1605 ether_addr_copy(vf->addr.mac_addr, mac); 1606 __efx_siena_sriov_update_vf_addr(vf); 1607 mutex_unlock(&vf->status_lock); 1608 1609 return 0; 1610} 1611 1612int efx_siena_sriov_set_vf_vlan(struct efx_nic *efx, int vf_i, 1613 u16 vlan, u8 qos) 1614{ 1615 struct siena_nic_data *nic_data = efx->nic_data; 1616 struct siena_vf *vf; 1617 u16 tci; 1618 1619 if (vf_i >= efx->vf_init_count) 1620 return -EINVAL; 1621 vf = nic_data->vf + vf_i; 1622 1623 mutex_lock(&vf->status_lock); 1624 tci = (vlan & VLAN_VID_MASK) | ((qos & 0x7) << VLAN_PRIO_SHIFT); 1625 vf->addr.tci = htons(tci); 1626 __efx_siena_sriov_update_vf_addr(vf); 1627 mutex_unlock(&vf->status_lock); 1628 1629 return 0; 1630} 1631 1632int efx_siena_sriov_set_vf_spoofchk(struct efx_nic *efx, int vf_i, 1633 bool spoofchk) 1634{ 1635 struct siena_nic_data *nic_data = efx->nic_data; 1636 struct siena_vf *vf; 1637 int rc; 1638 1639 if (vf_i >= efx->vf_init_count) 1640 return -EINVAL; 1641 vf = nic_data->vf + vf_i; 1642 1643 mutex_lock(&vf->txq_lock); 1644 if (vf->txq_count == 0) { 1645 vf->tx_filter_mode = 1646 spoofchk ? VF_TX_FILTER_ON : VF_TX_FILTER_OFF; 1647 rc = 0; 1648 } else { 1649 /* This cannot be changed while TX queues are running */ 1650 rc = -EBUSY; 1651 } 1652 mutex_unlock(&vf->txq_lock); 1653 return rc; 1654} 1655 1656int efx_siena_sriov_get_vf_config(struct efx_nic *efx, int vf_i, 1657 struct ifla_vf_info *ivi) 1658{ 1659 struct siena_nic_data *nic_data = efx->nic_data; 1660 struct siena_vf *vf; 1661 u16 tci; 1662 1663 if (vf_i >= efx->vf_init_count) 1664 return -EINVAL; 1665 vf = nic_data->vf + vf_i; 1666 1667 ivi->vf = vf_i; 1668 ether_addr_copy(ivi->mac, vf->addr.mac_addr); 1669 ivi->max_tx_rate = 0; 1670 ivi->min_tx_rate = 0; 1671 tci = ntohs(vf->addr.tci); 1672 ivi->vlan = tci & VLAN_VID_MASK; 1673 ivi->qos = (tci >> VLAN_PRIO_SHIFT) & 0x7; 1674 ivi->spoofchk = vf->tx_filter_mode == VF_TX_FILTER_ON; 1675 1676 return 0; 1677} 1678 1679bool efx_siena_sriov_wanted(struct efx_nic *efx) 1680{ 1681 return efx->vf_count != 0; 1682} 1683 1684int efx_siena_sriov_configure(struct efx_nic *efx, int num_vfs) 1685{ 1686 return 0; 1687} 1688