1/* 2 * Copyright(c) 2017 Intel Corporation. 3 * Copyright(c) 2021 Cornelis Networks. 4 * 5 * This file is provided under a dual BSD/GPLv2 license. When using or 6 * redistributing this file, you may do so under either license. 7 * 8 * GPL LICENSE SUMMARY 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of version 2 of the GNU General Public License as 12 * published by the Free Software Foundation. 13 * 14 * This program is distributed in the hope that it will be useful, but 15 * WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 * General Public License for more details. 18 * 19 * BSD LICENSE 20 * 21 * Redistribution and use in source and binary forms, with or without 22 * modification, are permitted provided that the following conditions 23 * are met: 24 * 25 * - Redistributions of source code must retain the above copyright 26 * notice, this list of conditions and the following disclaimer. 27 * - Redistributions in binary form must reproduce the above copyright 28 * notice, this list of conditions and the following disclaimer in 29 * the documentation and/or other materials provided with the 30 * distribution. 31 * - Neither the name of Intel Corporation nor the names of its 32 * contributors may be used to endorse or promote products derived 33 * from this software without specific prior written permission. 34 * 35 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 36 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 37 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 38 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 39 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 40 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 41 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 42 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 43 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 44 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 45 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 46 * 47 */ 48 49/* 50 * This file contains OPX Virtual Network Interface Controller (VNIC) 51 * Ethernet Management Agent (EMA) driver 52 */ 53 54#include <linux/module.h> 55#include <linux/xarray.h> 56#include <rdma/ib_addr.h> 57#include <rdma/ib_verbs.h> 58#include <rdma/opa_smi.h> 59#include <rdma/opa_port_info.h> 60 61#include "opa_vnic_internal.h" 62 63char opa_vnic_driver_name[] = "opa_vnic"; 64 65/* 66 * The trap service level is kept in bits 3 to 7 in the trap_sl_rsvd 67 * field in the class port info MAD. 68 */ 69#define GET_TRAP_SL_FROM_CLASS_PORT_INFO(x) (((x) >> 3) & 0x1f) 70 71/* Cap trap bursts to a reasonable limit good for normal cases */ 72#define OPA_VNIC_TRAP_BURST_LIMIT 4 73 74/* 75 * VNIC trap limit timeout. 76 * Inverse of cap2_mask response time out (1.0737 secs) = 0.9 77 * secs approx IB spec 13.4.6.2.1 PortInfoSubnetTimeout and 78 * 13.4.9 Traps. 79 */ 80#define OPA_VNIC_TRAP_TIMEOUT ((4096 * (1UL << 18)) / 1000) 81 82#define OPA_VNIC_UNSUP_ATTR \ 83 cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB) 84 85#define OPA_VNIC_INVAL_ATTR \ 86 cpu_to_be16(IB_MGMT_MAD_STATUS_INVALID_ATTRIB_VALUE) 87 88#define OPA_VNIC_CLASS_CAP_TRAP 0x1 89 90/* Maximum number of VNIC ports supported */ 91#define OPA_VNIC_MAX_NUM_VPORT 255 92 93/** 94 * struct opa_vnic_vema_port -- VNIC VEMA port details 95 * @cport: pointer to port 96 * @mad_agent: pointer to mad agent for port 97 * @class_port_info: Class port info information. 98 * @tid: Transaction id 99 * @port_num: OPA port number 100 * @vports: vnic ports 101 * @event_handler: ib event handler 102 * @lock: adapter interface lock 103 */ 104struct opa_vnic_vema_port { 105 struct opa_vnic_ctrl_port *cport; 106 struct ib_mad_agent *mad_agent; 107 struct opa_class_port_info class_port_info; 108 u64 tid; 109 u8 port_num; 110 struct xarray vports; 111 struct ib_event_handler event_handler; 112 113 /* Lock to query/update network adapter */ 114 struct mutex lock; 115}; 116 117static int opa_vnic_vema_add_one(struct ib_device *device); 118static void opa_vnic_vema_rem_one(struct ib_device *device, 119 void *client_data); 120 121static struct ib_client opa_vnic_client = { 122 .name = opa_vnic_driver_name, 123 .add = opa_vnic_vema_add_one, 124 .remove = opa_vnic_vema_rem_one, 125}; 126 127/** 128 * vema_get_vport_num -- Get the vnic from the mad 129 * @recvd_mad: Received mad 130 * 131 * Return: returns value of the vnic port number 132 */ 133static inline u8 vema_get_vport_num(struct opa_vnic_vema_mad *recvd_mad) 134{ 135 return be32_to_cpu(recvd_mad->mad_hdr.attr_mod) & 0xff; 136} 137 138/** 139 * vema_get_vport_adapter -- Get vnic port adapter from recvd mad 140 * @recvd_mad: received mad 141 * @port: ptr to port struct on which MAD was recvd 142 * 143 * Return: vnic adapter 144 */ 145static inline struct opa_vnic_adapter * 146vema_get_vport_adapter(struct opa_vnic_vema_mad *recvd_mad, 147 struct opa_vnic_vema_port *port) 148{ 149 u8 vport_num = vema_get_vport_num(recvd_mad); 150 151 return xa_load(&port->vports, vport_num); 152} 153 154/** 155 * vema_mac_tbl_req_ok -- Check if mac request has correct values 156 * @mac_tbl: mac table 157 * 158 * This function checks for the validity of the offset and number of 159 * entries required. 160 * 161 * Return: true if offset and num_entries are valid 162 */ 163static inline bool vema_mac_tbl_req_ok(struct opa_veswport_mactable *mac_tbl) 164{ 165 u16 offset, num_entries; 166 u16 req_entries = ((OPA_VNIC_EMA_DATA - sizeof(*mac_tbl)) / 167 sizeof(mac_tbl->tbl_entries[0])); 168 169 offset = be16_to_cpu(mac_tbl->offset); 170 num_entries = be16_to_cpu(mac_tbl->num_entries); 171 172 return ((num_entries <= req_entries) && 173 (offset + num_entries <= OPA_VNIC_MAC_TBL_MAX_ENTRIES)); 174} 175 176/* 177 * Return the power on default values in the port info structure 178 * in big endian format as required by MAD. 179 */ 180static inline void vema_get_pod_values(struct opa_veswport_info *port_info) 181{ 182 memset(port_info, 0, sizeof(*port_info)); 183 port_info->vport.max_mac_tbl_ent = 184 cpu_to_be16(OPA_VNIC_MAC_TBL_MAX_ENTRIES); 185 port_info->vport.max_smac_ent = 186 cpu_to_be16(OPA_VNIC_MAX_SMAC_LIMIT); 187 port_info->vport.oper_state = OPA_VNIC_STATE_DROP_ALL; 188 port_info->vport.config_state = OPA_VNIC_STATE_DROP_ALL; 189 port_info->vesw.eth_mtu = cpu_to_be16(ETH_DATA_LEN); 190} 191 192/** 193 * vema_add_vport -- Add a new vnic port 194 * @port: ptr to opa_vnic_vema_port struct 195 * @vport_num: vnic port number (to be added) 196 * 197 * Return a pointer to the vnic adapter structure 198 */ 199static struct opa_vnic_adapter *vema_add_vport(struct opa_vnic_vema_port *port, 200 u8 vport_num) 201{ 202 struct opa_vnic_ctrl_port *cport = port->cport; 203 struct opa_vnic_adapter *adapter; 204 205 adapter = opa_vnic_add_netdev(cport->ibdev, port->port_num, vport_num); 206 if (!IS_ERR(adapter)) { 207 int rc; 208 209 adapter->cport = cport; 210 rc = xa_insert(&port->vports, vport_num, adapter, GFP_KERNEL); 211 if (rc < 0) { 212 opa_vnic_rem_netdev(adapter); 213 adapter = ERR_PTR(rc); 214 } 215 } 216 217 return adapter; 218} 219 220/** 221 * vema_get_class_port_info -- Get class info for port 222 * @port: Port on whic MAD was received 223 * @recvd_mad: pointer to the received mad 224 * @rsp_mad: pointer to respose mad 225 * 226 * This function copies the latest class port info value set for the 227 * port and stores it for generating traps 228 */ 229static void vema_get_class_port_info(struct opa_vnic_vema_port *port, 230 struct opa_vnic_vema_mad *recvd_mad, 231 struct opa_vnic_vema_mad *rsp_mad) 232{ 233 struct opa_class_port_info *port_info; 234 235 port_info = (struct opa_class_port_info *)rsp_mad->data; 236 memcpy(port_info, &port->class_port_info, sizeof(*port_info)); 237 port_info->base_version = OPA_MGMT_BASE_VERSION; 238 port_info->class_version = OPA_EMA_CLASS_VERSION; 239 240 /* 241 * Set capability mask bit indicating agent generates traps, 242 * and set the maximum number of VNIC ports supported. 243 */ 244 port_info->cap_mask = cpu_to_be16((OPA_VNIC_CLASS_CAP_TRAP | 245 (OPA_VNIC_MAX_NUM_VPORT << 8))); 246 247 /* 248 * Since a get routine is always sent by the EM first we 249 * set the expected response time to 250 * 4.096 usec * 2^18 == 1.0737 sec here. 251 */ 252 port_info->cap_mask2_resp_time = cpu_to_be32(18); 253} 254 255/** 256 * vema_set_class_port_info -- Get class info for port 257 * @port: Port on whic MAD was received 258 * @recvd_mad: pointer to the received mad 259 * @rsp_mad: pointer to respose mad 260 * 261 * This function updates the port class info for the specific vnic 262 * and sets up the response mad data 263 */ 264static void vema_set_class_port_info(struct opa_vnic_vema_port *port, 265 struct opa_vnic_vema_mad *recvd_mad, 266 struct opa_vnic_vema_mad *rsp_mad) 267{ 268 memcpy(&port->class_port_info, recvd_mad->data, 269 sizeof(port->class_port_info)); 270 271 vema_get_class_port_info(port, recvd_mad, rsp_mad); 272} 273 274/** 275 * vema_get_veswport_info -- Get veswport info 276 * @port: source port on which MAD was received 277 * @recvd_mad: pointer to the received mad 278 * @rsp_mad: pointer to respose mad 279 */ 280static void vema_get_veswport_info(struct opa_vnic_vema_port *port, 281 struct opa_vnic_vema_mad *recvd_mad, 282 struct opa_vnic_vema_mad *rsp_mad) 283{ 284 struct opa_veswport_info *port_info = 285 (struct opa_veswport_info *)rsp_mad->data; 286 struct opa_vnic_adapter *adapter; 287 288 adapter = vema_get_vport_adapter(recvd_mad, port); 289 if (adapter) { 290 memset(port_info, 0, sizeof(*port_info)); 291 opa_vnic_get_vesw_info(adapter, &port_info->vesw); 292 opa_vnic_get_per_veswport_info(adapter, 293 &port_info->vport); 294 } else { 295 vema_get_pod_values(port_info); 296 } 297} 298 299/** 300 * vema_set_veswport_info -- Set veswport info 301 * @port: source port on which MAD was received 302 * @recvd_mad: pointer to the received mad 303 * @rsp_mad: pointer to respose mad 304 * 305 * This function gets the port class infor for vnic 306 */ 307static void vema_set_veswport_info(struct opa_vnic_vema_port *port, 308 struct opa_vnic_vema_mad *recvd_mad, 309 struct opa_vnic_vema_mad *rsp_mad) 310{ 311 struct opa_vnic_ctrl_port *cport = port->cport; 312 struct opa_veswport_info *port_info; 313 struct opa_vnic_adapter *adapter; 314 u8 vport_num; 315 316 vport_num = vema_get_vport_num(recvd_mad); 317 318 adapter = vema_get_vport_adapter(recvd_mad, port); 319 if (!adapter) { 320 adapter = vema_add_vport(port, vport_num); 321 if (IS_ERR(adapter)) { 322 c_err("failed to add vport %d: %ld\n", 323 vport_num, PTR_ERR(adapter)); 324 goto err_exit; 325 } 326 } 327 328 port_info = (struct opa_veswport_info *)recvd_mad->data; 329 opa_vnic_set_vesw_info(adapter, &port_info->vesw); 330 opa_vnic_set_per_veswport_info(adapter, &port_info->vport); 331 332 /* Process the new config settings */ 333 opa_vnic_process_vema_config(adapter); 334 335 vema_get_veswport_info(port, recvd_mad, rsp_mad); 336 return; 337 338err_exit: 339 rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR; 340} 341 342/** 343 * vema_get_mac_entries -- Get MAC entries in VNIC MAC table 344 * @port: source port on which MAD was received 345 * @recvd_mad: pointer to the received mad 346 * @rsp_mad: pointer to respose mad 347 * 348 * This function gets the MAC entries that are programmed into 349 * the VNIC MAC forwarding table. It checks for the validity of 350 * the index into the MAC table and the number of entries that 351 * are to be retrieved. 352 */ 353static void vema_get_mac_entries(struct opa_vnic_vema_port *port, 354 struct opa_vnic_vema_mad *recvd_mad, 355 struct opa_vnic_vema_mad *rsp_mad) 356{ 357 struct opa_veswport_mactable *mac_tbl_in, *mac_tbl_out; 358 struct opa_vnic_adapter *adapter; 359 360 adapter = vema_get_vport_adapter(recvd_mad, port); 361 if (!adapter) { 362 rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR; 363 return; 364 } 365 366 mac_tbl_in = (struct opa_veswport_mactable *)recvd_mad->data; 367 mac_tbl_out = (struct opa_veswport_mactable *)rsp_mad->data; 368 369 if (vema_mac_tbl_req_ok(mac_tbl_in)) { 370 mac_tbl_out->offset = mac_tbl_in->offset; 371 mac_tbl_out->num_entries = mac_tbl_in->num_entries; 372 opa_vnic_query_mac_tbl(adapter, mac_tbl_out); 373 } else { 374 rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR; 375 } 376} 377 378/** 379 * vema_set_mac_entries -- Set MAC entries in VNIC MAC table 380 * @port: source port on which MAD was received 381 * @recvd_mad: pointer to the received mad 382 * @rsp_mad: pointer to respose mad 383 * 384 * This function sets the MAC entries in the VNIC forwarding table 385 * It checks for the validity of the index and the number of forwarding 386 * table entries to be programmed. 387 */ 388static void vema_set_mac_entries(struct opa_vnic_vema_port *port, 389 struct opa_vnic_vema_mad *recvd_mad, 390 struct opa_vnic_vema_mad *rsp_mad) 391{ 392 struct opa_veswport_mactable *mac_tbl; 393 struct opa_vnic_adapter *adapter; 394 395 adapter = vema_get_vport_adapter(recvd_mad, port); 396 if (!adapter) { 397 rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR; 398 return; 399 } 400 401 mac_tbl = (struct opa_veswport_mactable *)recvd_mad->data; 402 if (vema_mac_tbl_req_ok(mac_tbl)) { 403 if (opa_vnic_update_mac_tbl(adapter, mac_tbl)) 404 rsp_mad->mad_hdr.status = OPA_VNIC_UNSUP_ATTR; 405 } else { 406 rsp_mad->mad_hdr.status = OPA_VNIC_UNSUP_ATTR; 407 } 408 vema_get_mac_entries(port, recvd_mad, rsp_mad); 409} 410 411/** 412 * vema_set_delete_vesw -- Reset VESW info to POD values 413 * @port: source port on which MAD was received 414 * @recvd_mad: pointer to the received mad 415 * @rsp_mad: pointer to respose mad 416 * 417 * This function clears all the fields of veswport info for the requested vesw 418 * and sets them back to the power-on default values. It does not delete the 419 * vesw. 420 */ 421static void vema_set_delete_vesw(struct opa_vnic_vema_port *port, 422 struct opa_vnic_vema_mad *recvd_mad, 423 struct opa_vnic_vema_mad *rsp_mad) 424{ 425 struct opa_veswport_info *port_info = 426 (struct opa_veswport_info *)rsp_mad->data; 427 struct opa_vnic_adapter *adapter; 428 429 adapter = vema_get_vport_adapter(recvd_mad, port); 430 if (!adapter) { 431 rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR; 432 return; 433 } 434 435 vema_get_pod_values(port_info); 436 opa_vnic_set_vesw_info(adapter, &port_info->vesw); 437 opa_vnic_set_per_veswport_info(adapter, &port_info->vport); 438 439 /* Process the new config settings */ 440 opa_vnic_process_vema_config(adapter); 441 442 opa_vnic_release_mac_tbl(adapter); 443 444 vema_get_veswport_info(port, recvd_mad, rsp_mad); 445} 446 447/** 448 * vema_get_mac_list -- Get the unicast/multicast macs. 449 * @port: source port on which MAD was received 450 * @recvd_mad: Received mad contains fields to set vnic parameters 451 * @rsp_mad: Response mad to be built 452 * @attr_id: Attribute ID indicating multicast or unicast mac list 453 */ 454static void vema_get_mac_list(struct opa_vnic_vema_port *port, 455 struct opa_vnic_vema_mad *recvd_mad, 456 struct opa_vnic_vema_mad *rsp_mad, 457 u16 attr_id) 458{ 459 struct opa_veswport_iface_macs *macs_in, *macs_out; 460 int max_entries = (OPA_VNIC_EMA_DATA - sizeof(*macs_out)) / ETH_ALEN; 461 struct opa_vnic_adapter *adapter; 462 463 adapter = vema_get_vport_adapter(recvd_mad, port); 464 if (!adapter) { 465 rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR; 466 return; 467 } 468 469 macs_in = (struct opa_veswport_iface_macs *)recvd_mad->data; 470 macs_out = (struct opa_veswport_iface_macs *)rsp_mad->data; 471 472 macs_out->start_idx = macs_in->start_idx; 473 if (macs_in->num_macs_in_msg) 474 macs_out->num_macs_in_msg = macs_in->num_macs_in_msg; 475 else 476 macs_out->num_macs_in_msg = cpu_to_be16(max_entries); 477 478 if (attr_id == OPA_EM_ATTR_IFACE_MCAST_MACS) 479 opa_vnic_query_mcast_macs(adapter, macs_out); 480 else 481 opa_vnic_query_ucast_macs(adapter, macs_out); 482} 483 484/** 485 * vema_get_summary_counters -- Gets summary counters. 486 * @port: source port on which MAD was received 487 * @recvd_mad: Received mad contains fields to set vnic parameters 488 * @rsp_mad: Response mad to be built 489 */ 490static void vema_get_summary_counters(struct opa_vnic_vema_port *port, 491 struct opa_vnic_vema_mad *recvd_mad, 492 struct opa_vnic_vema_mad *rsp_mad) 493{ 494 struct opa_veswport_summary_counters *cntrs; 495 struct opa_vnic_adapter *adapter; 496 497 adapter = vema_get_vport_adapter(recvd_mad, port); 498 if (adapter) { 499 cntrs = (struct opa_veswport_summary_counters *)rsp_mad->data; 500 opa_vnic_get_summary_counters(adapter, cntrs); 501 } else { 502 rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR; 503 } 504} 505 506/** 507 * vema_get_error_counters -- Gets summary counters. 508 * @port: source port on which MAD was received 509 * @recvd_mad: Received mad contains fields to set vnic parameters 510 * @rsp_mad: Response mad to be built 511 */ 512static void vema_get_error_counters(struct opa_vnic_vema_port *port, 513 struct opa_vnic_vema_mad *recvd_mad, 514 struct opa_vnic_vema_mad *rsp_mad) 515{ 516 struct opa_veswport_error_counters *cntrs; 517 struct opa_vnic_adapter *adapter; 518 519 adapter = vema_get_vport_adapter(recvd_mad, port); 520 if (adapter) { 521 cntrs = (struct opa_veswport_error_counters *)rsp_mad->data; 522 opa_vnic_get_error_counters(adapter, cntrs); 523 } else { 524 rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR; 525 } 526} 527 528/** 529 * vema_get -- Process received get MAD 530 * @port: source port on which MAD was received 531 * @recvd_mad: Received mad 532 * @rsp_mad: Response mad to be built 533 */ 534static void vema_get(struct opa_vnic_vema_port *port, 535 struct opa_vnic_vema_mad *recvd_mad, 536 struct opa_vnic_vema_mad *rsp_mad) 537{ 538 u16 attr_id = be16_to_cpu(recvd_mad->mad_hdr.attr_id); 539 540 switch (attr_id) { 541 case OPA_EM_ATTR_CLASS_PORT_INFO: 542 vema_get_class_port_info(port, recvd_mad, rsp_mad); 543 break; 544 case OPA_EM_ATTR_VESWPORT_INFO: 545 vema_get_veswport_info(port, recvd_mad, rsp_mad); 546 break; 547 case OPA_EM_ATTR_VESWPORT_MAC_ENTRIES: 548 vema_get_mac_entries(port, recvd_mad, rsp_mad); 549 break; 550 case OPA_EM_ATTR_IFACE_UCAST_MACS: 551 case OPA_EM_ATTR_IFACE_MCAST_MACS: 552 vema_get_mac_list(port, recvd_mad, rsp_mad, attr_id); 553 break; 554 case OPA_EM_ATTR_VESWPORT_SUMMARY_COUNTERS: 555 vema_get_summary_counters(port, recvd_mad, rsp_mad); 556 break; 557 case OPA_EM_ATTR_VESWPORT_ERROR_COUNTERS: 558 vema_get_error_counters(port, recvd_mad, rsp_mad); 559 break; 560 default: 561 rsp_mad->mad_hdr.status = OPA_VNIC_UNSUP_ATTR; 562 break; 563 } 564} 565 566/** 567 * vema_set -- Process received set MAD 568 * @port: source port on which MAD was received 569 * @recvd_mad: Received mad contains fields to set vnic parameters 570 * @rsp_mad: Response mad to be built 571 */ 572static void vema_set(struct opa_vnic_vema_port *port, 573 struct opa_vnic_vema_mad *recvd_mad, 574 struct opa_vnic_vema_mad *rsp_mad) 575{ 576 u16 attr_id = be16_to_cpu(recvd_mad->mad_hdr.attr_id); 577 578 switch (attr_id) { 579 case OPA_EM_ATTR_CLASS_PORT_INFO: 580 vema_set_class_port_info(port, recvd_mad, rsp_mad); 581 break; 582 case OPA_EM_ATTR_VESWPORT_INFO: 583 vema_set_veswport_info(port, recvd_mad, rsp_mad); 584 break; 585 case OPA_EM_ATTR_VESWPORT_MAC_ENTRIES: 586 vema_set_mac_entries(port, recvd_mad, rsp_mad); 587 break; 588 case OPA_EM_ATTR_DELETE_VESW: 589 vema_set_delete_vesw(port, recvd_mad, rsp_mad); 590 break; 591 default: 592 rsp_mad->mad_hdr.status = OPA_VNIC_UNSUP_ATTR; 593 break; 594 } 595} 596 597/** 598 * vema_send -- Send handler for VEMA MAD agent 599 * @mad_agent: pointer to the mad agent 600 * @mad_wc: pointer to mad send work completion information 601 * 602 * Free all the data structures associated with the sent MAD 603 */ 604static void vema_send(struct ib_mad_agent *mad_agent, 605 struct ib_mad_send_wc *mad_wc) 606{ 607 rdma_destroy_ah(mad_wc->send_buf->ah, RDMA_DESTROY_AH_SLEEPABLE); 608 ib_free_send_mad(mad_wc->send_buf); 609} 610 611/** 612 * vema_recv -- Recv handler for VEMA MAD agent 613 * @mad_agent: pointer to the mad agent 614 * @send_buf: Send buffer if found, else NULL 615 * @mad_wc: pointer to mad send work completion information 616 * 617 * Handle only set and get methods and respond to other methods 618 * as unsupported. Allocate response buffer and address handle 619 * for the response MAD. 620 */ 621static void vema_recv(struct ib_mad_agent *mad_agent, 622 struct ib_mad_send_buf *send_buf, 623 struct ib_mad_recv_wc *mad_wc) 624{ 625 struct opa_vnic_vema_port *port; 626 struct ib_ah *ah; 627 struct ib_mad_send_buf *rsp; 628 struct opa_vnic_vema_mad *vema_mad; 629 630 if (!mad_wc || !mad_wc->recv_buf.mad) 631 return; 632 633 port = mad_agent->context; 634 ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc, 635 mad_wc->recv_buf.grh, mad_agent->port_num); 636 if (IS_ERR(ah)) 637 goto free_recv_mad; 638 639 rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp, 640 mad_wc->wc->pkey_index, 0, 641 IB_MGMT_VENDOR_HDR, OPA_VNIC_EMA_DATA, 642 GFP_KERNEL, OPA_MGMT_BASE_VERSION); 643 if (IS_ERR(rsp)) 644 goto err_rsp; 645 646 rsp->ah = ah; 647 vema_mad = rsp->mad; 648 memcpy(vema_mad, mad_wc->recv_buf.mad, IB_MGMT_VENDOR_HDR); 649 vema_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP; 650 vema_mad->mad_hdr.status = 0; 651 652 /* Lock ensures network adapter is not removed */ 653 mutex_lock(&port->lock); 654 655 switch (mad_wc->recv_buf.mad->mad_hdr.method) { 656 case IB_MGMT_METHOD_GET: 657 vema_get(port, (struct opa_vnic_vema_mad *)mad_wc->recv_buf.mad, 658 vema_mad); 659 break; 660 case IB_MGMT_METHOD_SET: 661 vema_set(port, (struct opa_vnic_vema_mad *)mad_wc->recv_buf.mad, 662 vema_mad); 663 break; 664 default: 665 vema_mad->mad_hdr.status = OPA_VNIC_UNSUP_ATTR; 666 break; 667 } 668 mutex_unlock(&port->lock); 669 670 if (!ib_post_send_mad(rsp, NULL)) { 671 /* 672 * with post send successful ah and send mad 673 * will be destroyed in send handler 674 */ 675 goto free_recv_mad; 676 } 677 678 ib_free_send_mad(rsp); 679 680err_rsp: 681 rdma_destroy_ah(ah, RDMA_DESTROY_AH_SLEEPABLE); 682free_recv_mad: 683 ib_free_recv_mad(mad_wc); 684} 685 686/** 687 * vema_get_port -- Gets the opa_vnic_vema_port 688 * @cport: pointer to control dev 689 * @port_num: Port number 690 * 691 * This function loops through the ports and returns 692 * the opa_vnic_vema port structure that is associated 693 * with the OPA port number 694 * 695 * Return: ptr to requested opa_vnic_vema_port strucure 696 * if success, NULL if not 697 */ 698static struct opa_vnic_vema_port * 699vema_get_port(struct opa_vnic_ctrl_port *cport, u8 port_num) 700{ 701 struct opa_vnic_vema_port *port = (void *)cport + sizeof(*cport); 702 703 if (port_num > cport->num_ports) 704 return NULL; 705 706 return port + (port_num - 1); 707} 708 709/** 710 * opa_vnic_vema_send_trap -- This function sends a trap to the EM 711 * @adapter: pointer to vnic adapter 712 * @data: pointer to trap data filled by calling function 713 * @lid: issuers lid (encap_slid from vesw_port_info) 714 * 715 * This function is called from the VNIC driver to send a trap if there 716 * is somethng the EM should be notified about. These events currently 717 * are 718 * 1) UNICAST INTERFACE MACADDRESS changes 719 * 2) MULTICAST INTERFACE MACADDRESS changes 720 * 3) ETHERNET LINK STATUS changes 721 * While allocating the send mad the remote site qpn used is 1 722 * as this is the well known QP. 723 * 724 */ 725void opa_vnic_vema_send_trap(struct opa_vnic_adapter *adapter, 726 struct __opa_veswport_trap *data, u32 lid) 727{ 728 struct opa_vnic_ctrl_port *cport = adapter->cport; 729 struct ib_mad_send_buf *send_buf; 730 struct opa_vnic_vema_port *port; 731 struct ib_device *ibp; 732 struct opa_vnic_vema_mad_trap *trap_mad; 733 struct opa_class_port_info *class; 734 struct rdma_ah_attr ah_attr; 735 struct ib_ah *ah; 736 struct opa_veswport_trap *trap; 737 u32 trap_lid; 738 u16 pkey_idx; 739 740 if (!cport) 741 goto err_exit; 742 ibp = cport->ibdev; 743 port = vema_get_port(cport, data->opaportnum); 744 if (!port || !port->mad_agent) 745 goto err_exit; 746 747 if (time_before(jiffies, adapter->trap_timeout)) { 748 if (adapter->trap_count == OPA_VNIC_TRAP_BURST_LIMIT) { 749 v_warn("Trap rate exceeded\n"); 750 goto err_exit; 751 } else { 752 adapter->trap_count++; 753 } 754 } else { 755 adapter->trap_count = 0; 756 } 757 758 class = &port->class_port_info; 759 /* Set up address handle */ 760 memset(&ah_attr, 0, sizeof(ah_attr)); 761 ah_attr.type = rdma_ah_find_type(ibp, port->port_num); 762 rdma_ah_set_sl(&ah_attr, 763 GET_TRAP_SL_FROM_CLASS_PORT_INFO(class->trap_sl_rsvd)); 764 rdma_ah_set_port_num(&ah_attr, port->port_num); 765 trap_lid = be32_to_cpu(class->trap_lid); 766 /* 767 * check for trap lid validity, must not be zero 768 * The trap sink could change after we fashion the MAD but since traps 769 * are not guaranteed we won't use a lock as anyway the change will take 770 * place even with locking. 771 */ 772 if (!trap_lid) { 773 c_err("%s: Invalid dlid\n", __func__); 774 goto err_exit; 775 } 776 777 rdma_ah_set_dlid(&ah_attr, trap_lid); 778 ah = rdma_create_ah(port->mad_agent->qp->pd, &ah_attr, 0); 779 if (IS_ERR(ah)) { 780 c_err("%s:Couldn't create new AH = %p\n", __func__, ah); 781 c_err("%s:dlid = %d, sl = %d, port = %d\n", __func__, 782 rdma_ah_get_dlid(&ah_attr), rdma_ah_get_sl(&ah_attr), 783 rdma_ah_get_port_num(&ah_attr)); 784 goto err_exit; 785 } 786 787 if (ib_find_pkey(ibp, data->opaportnum, IB_DEFAULT_PKEY_FULL, 788 &pkey_idx) < 0) { 789 c_err("%s:full key not found, defaulting to partial\n", 790 __func__); 791 if (ib_find_pkey(ibp, data->opaportnum, IB_DEFAULT_PKEY_PARTIAL, 792 &pkey_idx) < 0) 793 pkey_idx = 1; 794 } 795 796 send_buf = ib_create_send_mad(port->mad_agent, 1, pkey_idx, 0, 797 IB_MGMT_VENDOR_HDR, IB_MGMT_MAD_DATA, 798 GFP_ATOMIC, OPA_MGMT_BASE_VERSION); 799 if (IS_ERR(send_buf)) { 800 c_err("%s:Couldn't allocate send buf\n", __func__); 801 goto err_sndbuf; 802 } 803 804 send_buf->ah = ah; 805 806 /* Set up common MAD hdr */ 807 trap_mad = send_buf->mad; 808 trap_mad->mad_hdr.base_version = OPA_MGMT_BASE_VERSION; 809 trap_mad->mad_hdr.mgmt_class = OPA_MGMT_CLASS_INTEL_EMA; 810 trap_mad->mad_hdr.class_version = OPA_EMA_CLASS_VERSION; 811 trap_mad->mad_hdr.method = IB_MGMT_METHOD_TRAP; 812 port->tid++; 813 trap_mad->mad_hdr.tid = cpu_to_be64(port->tid); 814 trap_mad->mad_hdr.attr_id = IB_SMP_ATTR_NOTICE; 815 816 /* Set up vendor OUI */ 817 trap_mad->oui[0] = INTEL_OUI_1; 818 trap_mad->oui[1] = INTEL_OUI_2; 819 trap_mad->oui[2] = INTEL_OUI_3; 820 821 /* Setup notice attribute portion */ 822 trap_mad->notice.gen_type = OPA_INTEL_EMA_NOTICE_TYPE_INFO << 1; 823 trap_mad->notice.oui_1 = INTEL_OUI_1; 824 trap_mad->notice.oui_2 = INTEL_OUI_2; 825 trap_mad->notice.oui_3 = INTEL_OUI_3; 826 trap_mad->notice.issuer_lid = cpu_to_be32(lid); 827 828 /* copy the actual trap data */ 829 trap = (struct opa_veswport_trap *)trap_mad->notice.raw_data; 830 trap->fabric_id = cpu_to_be16(data->fabric_id); 831 trap->veswid = cpu_to_be16(data->veswid); 832 trap->veswportnum = cpu_to_be32(data->veswportnum); 833 trap->opaportnum = cpu_to_be16(data->opaportnum); 834 trap->veswportindex = data->veswportindex; 835 trap->opcode = data->opcode; 836 837 /* If successful send set up rate limit timeout else bail */ 838 if (ib_post_send_mad(send_buf, NULL)) { 839 ib_free_send_mad(send_buf); 840 } else { 841 if (adapter->trap_count) 842 return; 843 adapter->trap_timeout = jiffies + 844 usecs_to_jiffies(OPA_VNIC_TRAP_TIMEOUT); 845 return; 846 } 847 848err_sndbuf: 849 rdma_destroy_ah(ah, 0); 850err_exit: 851 v_err("Aborting trap\n"); 852} 853 854static void opa_vnic_event(struct ib_event_handler *handler, 855 struct ib_event *record) 856{ 857 struct opa_vnic_vema_port *port = 858 container_of(handler, struct opa_vnic_vema_port, event_handler); 859 struct opa_vnic_ctrl_port *cport = port->cport; 860 struct opa_vnic_adapter *adapter; 861 unsigned long index; 862 863 if (record->element.port_num != port->port_num) 864 return; 865 866 c_dbg("OPA_VNIC received event %d on device %s port %d\n", 867 record->event, dev_name(&record->device->dev), 868 record->element.port_num); 869 870 if (record->event != IB_EVENT_PORT_ERR && 871 record->event != IB_EVENT_PORT_ACTIVE) 872 return; 873 874 xa_for_each(&port->vports, index, adapter) { 875 if (record->event == IB_EVENT_PORT_ACTIVE) 876 netif_carrier_on(adapter->netdev); 877 else 878 netif_carrier_off(adapter->netdev); 879 } 880} 881 882/** 883 * vema_unregister -- Unregisters agent 884 * @cport: pointer to control port 885 * 886 * This deletes the registration by VEMA for MADs 887 */ 888static void vema_unregister(struct opa_vnic_ctrl_port *cport) 889{ 890 struct opa_vnic_adapter *adapter; 891 unsigned long index; 892 int i; 893 894 for (i = 1; i <= cport->num_ports; i++) { 895 struct opa_vnic_vema_port *port = vema_get_port(cport, i); 896 897 if (!port->mad_agent) 898 continue; 899 900 /* Lock ensures no MAD is being processed */ 901 mutex_lock(&port->lock); 902 xa_for_each(&port->vports, index, adapter) 903 opa_vnic_rem_netdev(adapter); 904 mutex_unlock(&port->lock); 905 906 ib_unregister_mad_agent(port->mad_agent); 907 port->mad_agent = NULL; 908 mutex_destroy(&port->lock); 909 xa_destroy(&port->vports); 910 ib_unregister_event_handler(&port->event_handler); 911 } 912} 913 914/** 915 * vema_register -- Registers agent 916 * @cport: pointer to control port 917 * 918 * This function registers the handlers for the VEMA MADs 919 * 920 * Return: returns 0 on success. non zero otherwise 921 */ 922static int vema_register(struct opa_vnic_ctrl_port *cport) 923{ 924 struct ib_mad_reg_req reg_req = { 925 .mgmt_class = OPA_MGMT_CLASS_INTEL_EMA, 926 .mgmt_class_version = OPA_MGMT_BASE_VERSION, 927 .oui = { INTEL_OUI_1, INTEL_OUI_2, INTEL_OUI_3 } 928 }; 929 int i; 930 931 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask); 932 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask); 933 934 /* register ib event handler and mad agent for each port on dev */ 935 for (i = 1; i <= cport->num_ports; i++) { 936 struct opa_vnic_vema_port *port = vema_get_port(cport, i); 937 int ret; 938 939 port->cport = cport; 940 port->port_num = i; 941 942 INIT_IB_EVENT_HANDLER(&port->event_handler, 943 cport->ibdev, opa_vnic_event); 944 ib_register_event_handler(&port->event_handler); 945 946 xa_init(&port->vports); 947 mutex_init(&port->lock); 948 port->mad_agent = ib_register_mad_agent(cport->ibdev, i, 949 IB_QPT_GSI, ®_req, 950 IB_MGMT_RMPP_VERSION, 951 vema_send, vema_recv, 952 port, 0); 953 if (IS_ERR(port->mad_agent)) { 954 ret = PTR_ERR(port->mad_agent); 955 port->mad_agent = NULL; 956 mutex_destroy(&port->lock); 957 vema_unregister(cport); 958 return ret; 959 } 960 } 961 962 return 0; 963} 964 965/** 966 * opa_vnic_ctrl_config_dev -- This function sends a trap to the EM 967 * by way of ib_modify_port to indicate support for ethernet on the 968 * fabric. 969 * @cport: pointer to control port 970 * @en: enable or disable ethernet on fabric support 971 */ 972static void opa_vnic_ctrl_config_dev(struct opa_vnic_ctrl_port *cport, bool en) 973{ 974 struct ib_port_modify pm = { 0 }; 975 int i; 976 977 if (en) 978 pm.set_port_cap_mask = OPA_CAP_MASK3_IsEthOnFabricSupported; 979 else 980 pm.clr_port_cap_mask = OPA_CAP_MASK3_IsEthOnFabricSupported; 981 982 for (i = 1; i <= cport->num_ports; i++) 983 ib_modify_port(cport->ibdev, i, IB_PORT_OPA_MASK_CHG, &pm); 984} 985 986/** 987 * opa_vnic_vema_add_one -- Handle new ib device 988 * @device: ib device pointer 989 * 990 * Allocate the vnic control port and initialize it. 991 */ 992static int opa_vnic_vema_add_one(struct ib_device *device) 993{ 994 struct opa_vnic_ctrl_port *cport; 995 int rc, size = sizeof(*cport); 996 997 if (!rdma_cap_opa_vnic(device)) 998 return -EOPNOTSUPP; 999 1000 size += device->phys_port_cnt * sizeof(struct opa_vnic_vema_port); 1001 cport = kzalloc(size, GFP_KERNEL); 1002 if (!cport) 1003 return -ENOMEM; 1004 1005 cport->num_ports = device->phys_port_cnt; 1006 cport->ibdev = device; 1007 1008 /* Initialize opa vnic management agent (vema) */ 1009 rc = vema_register(cport); 1010 if (!rc) 1011 c_info("VNIC client initialized\n"); 1012 1013 ib_set_client_data(device, &opa_vnic_client, cport); 1014 opa_vnic_ctrl_config_dev(cport, true); 1015 return 0; 1016} 1017 1018/** 1019 * opa_vnic_vema_rem_one -- Handle ib device removal 1020 * @device: ib device pointer 1021 * @client_data: ib client data 1022 * 1023 * Uninitialize and free the vnic control port. 1024 */ 1025static void opa_vnic_vema_rem_one(struct ib_device *device, 1026 void *client_data) 1027{ 1028 struct opa_vnic_ctrl_port *cport = client_data; 1029 1030 c_info("removing VNIC client\n"); 1031 opa_vnic_ctrl_config_dev(cport, false); 1032 vema_unregister(cport); 1033 kfree(cport); 1034} 1035 1036static int __init opa_vnic_init(void) 1037{ 1038 int rc; 1039 1040 rc = ib_register_client(&opa_vnic_client); 1041 if (rc) 1042 pr_err("VNIC driver register failed %d\n", rc); 1043 1044 return rc; 1045} 1046module_init(opa_vnic_init); 1047 1048static void opa_vnic_deinit(void) 1049{ 1050 ib_unregister_client(&opa_vnic_client); 1051} 1052module_exit(opa_vnic_deinit); 1053 1054MODULE_LICENSE("Dual BSD/GPL"); 1055MODULE_AUTHOR("Cornelis Networks"); 1056MODULE_DESCRIPTION("Cornelis OPX Virtual Network driver"); 1057