1/* 2 * Copyright (c) 2013-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <linux/highmem.h> 34#include <linux/errno.h> 35#include <linux/pci.h> 36#include <linux/dma-mapping.h> 37#include <linux/slab.h> 38#include <linux/delay.h> 39#include <linux/random.h> 40#include <linux/mlx5/driver.h> 41#include <linux/mlx5/eq.h> 42#include <linux/debugfs.h> 43 44#include "mlx5_core.h" 45#include "lib/eq.h" 46#include "lib/tout.h" 47#define CREATE_TRACE_POINTS 48#include "diag/cmd_tracepoint.h" 49 50struct mlx5_ifc_mbox_out_bits { 51 u8 status[0x8]; 52 u8 reserved_at_8[0x18]; 53 54 u8 syndrome[0x20]; 55 56 u8 reserved_at_40[0x40]; 57}; 58 59struct mlx5_ifc_mbox_in_bits { 60 u8 opcode[0x10]; 61 u8 uid[0x10]; 62 63 u8 reserved_at_20[0x10]; 64 u8 op_mod[0x10]; 65 66 u8 reserved_at_40[0x40]; 67}; 68 69enum { 70 CMD_IF_REV = 5, 71}; 72 73enum { 74 CMD_MODE_POLLING, 75 CMD_MODE_EVENTS 76}; 77 78enum { 79 MLX5_CMD_DELIVERY_STAT_OK = 0x0, 80 MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR = 0x1, 81 MLX5_CMD_DELIVERY_STAT_TOK_ERR = 0x2, 82 MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR = 0x3, 83 MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR = 0x4, 84 MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR = 0x5, 85 MLX5_CMD_DELIVERY_STAT_FW_ERR = 0x6, 86 MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR = 0x7, 87 MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR = 0x8, 88 MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR = 0x9, 89 MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR = 0x10, 90}; 91 92static u16 in_to_opcode(void *in) 93{ 94 return MLX5_GET(mbox_in, in, opcode); 95} 96 97/* Returns true for opcodes that might be triggered very frequently and throttle 98 * the command interface. Limit their command slots usage. 99 */ 100static bool mlx5_cmd_is_throttle_opcode(u16 op) 101{ 102 switch (op) { 103 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT: 104 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT: 105 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT: 106 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT: 107 case MLX5_CMD_OP_SYNC_CRYPTO: 108 return true; 109 } 110 return false; 111} 112 113static struct mlx5_cmd_work_ent * 114cmd_alloc_ent(struct mlx5_cmd *cmd, struct mlx5_cmd_msg *in, 115 struct mlx5_cmd_msg *out, void *uout, int uout_size, 116 mlx5_cmd_cbk_t cbk, void *context, int page_queue) 117{ 118 gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL; 119 struct mlx5_cmd_work_ent *ent; 120 121 ent = kzalloc(sizeof(*ent), alloc_flags); 122 if (!ent) 123 return ERR_PTR(-ENOMEM); 124 125 ent->idx = -EINVAL; 126 ent->in = in; 127 ent->out = out; 128 ent->uout = uout; 129 ent->uout_size = uout_size; 130 ent->callback = cbk; 131 ent->context = context; 132 ent->cmd = cmd; 133 ent->page_queue = page_queue; 134 ent->op = in_to_opcode(in->first.data); 135 refcount_set(&ent->refcnt, 1); 136 137 return ent; 138} 139 140static void cmd_free_ent(struct mlx5_cmd_work_ent *ent) 141{ 142 kfree(ent); 143} 144 145static u8 alloc_token(struct mlx5_cmd *cmd) 146{ 147 u8 token; 148 149 spin_lock(&cmd->token_lock); 150 cmd->token++; 151 if (cmd->token == 0) 152 cmd->token++; 153 token = cmd->token; 154 spin_unlock(&cmd->token_lock); 155 156 return token; 157} 158 159static int cmd_alloc_index(struct mlx5_cmd *cmd, struct mlx5_cmd_work_ent *ent) 160{ 161 unsigned long flags; 162 int ret; 163 164 spin_lock_irqsave(&cmd->alloc_lock, flags); 165 ret = find_first_bit(&cmd->vars.bitmask, cmd->vars.max_reg_cmds); 166 if (ret < cmd->vars.max_reg_cmds) { 167 clear_bit(ret, &cmd->vars.bitmask); 168 ent->idx = ret; 169 cmd->ent_arr[ent->idx] = ent; 170 } 171 spin_unlock_irqrestore(&cmd->alloc_lock, flags); 172 173 return ret < cmd->vars.max_reg_cmds ? ret : -ENOMEM; 174} 175 176static void cmd_free_index(struct mlx5_cmd *cmd, int idx) 177{ 178 lockdep_assert_held(&cmd->alloc_lock); 179 set_bit(idx, &cmd->vars.bitmask); 180} 181 182static void cmd_ent_get(struct mlx5_cmd_work_ent *ent) 183{ 184 refcount_inc(&ent->refcnt); 185} 186 187static void cmd_ent_put(struct mlx5_cmd_work_ent *ent) 188{ 189 struct mlx5_cmd *cmd = ent->cmd; 190 unsigned long flags; 191 192 spin_lock_irqsave(&cmd->alloc_lock, flags); 193 if (!refcount_dec_and_test(&ent->refcnt)) 194 goto out; 195 196 if (ent->idx >= 0) { 197 cmd_free_index(cmd, ent->idx); 198 up(ent->page_queue ? &cmd->vars.pages_sem : &cmd->vars.sem); 199 } 200 201 cmd_free_ent(ent); 202out: 203 spin_unlock_irqrestore(&cmd->alloc_lock, flags); 204} 205 206static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx) 207{ 208 return cmd->cmd_buf + (idx << cmd->vars.log_stride); 209} 210 211static int mlx5_calc_cmd_blocks(struct mlx5_cmd_msg *msg) 212{ 213 int size = msg->len; 214 int blen = size - min_t(int, sizeof(msg->first.data), size); 215 216 return DIV_ROUND_UP(blen, MLX5_CMD_DATA_BLOCK_SIZE); 217} 218 219static u8 xor8_buf(void *buf, size_t offset, int len) 220{ 221 u8 *ptr = buf; 222 u8 sum = 0; 223 int i; 224 int end = len + offset; 225 226 for (i = offset; i < end; i++) 227 sum ^= ptr[i]; 228 229 return sum; 230} 231 232static int verify_block_sig(struct mlx5_cmd_prot_block *block) 233{ 234 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0); 235 int xor_len = sizeof(*block) - sizeof(block->data) - 1; 236 237 if (xor8_buf(block, rsvd0_off, xor_len) != 0xff) 238 return -EHWPOISON; 239 240 if (xor8_buf(block, 0, sizeof(*block)) != 0xff) 241 return -EHWPOISON; 242 243 return 0; 244} 245 246static void calc_block_sig(struct mlx5_cmd_prot_block *block) 247{ 248 int ctrl_xor_len = sizeof(*block) - sizeof(block->data) - 2; 249 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0); 250 251 block->ctrl_sig = ~xor8_buf(block, rsvd0_off, ctrl_xor_len); 252 block->sig = ~xor8_buf(block, 0, sizeof(*block) - 1); 253} 254 255static void calc_chain_sig(struct mlx5_cmd_msg *msg) 256{ 257 struct mlx5_cmd_mailbox *next = msg->next; 258 int n = mlx5_calc_cmd_blocks(msg); 259 int i = 0; 260 261 for (i = 0; i < n && next; i++) { 262 calc_block_sig(next->buf); 263 next = next->next; 264 } 265} 266 267static void set_signature(struct mlx5_cmd_work_ent *ent, int csum) 268{ 269 ent->lay->sig = ~xor8_buf(ent->lay, 0, sizeof(*ent->lay)); 270 if (csum) { 271 calc_chain_sig(ent->in); 272 calc_chain_sig(ent->out); 273 } 274} 275 276static void poll_timeout(struct mlx5_cmd_work_ent *ent) 277{ 278 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, cmd); 279 u64 cmd_to_ms = mlx5_tout_ms(dev, CMD); 280 unsigned long poll_end; 281 u8 own; 282 283 poll_end = jiffies + msecs_to_jiffies(cmd_to_ms + 1000); 284 285 do { 286 own = READ_ONCE(ent->lay->status_own); 287 if (!(own & CMD_OWNER_HW)) { 288 ent->ret = 0; 289 return; 290 } 291 cond_resched(); 292 } while (time_before(jiffies, poll_end)); 293 294 ent->ret = -ETIMEDOUT; 295} 296 297static int verify_signature(struct mlx5_cmd_work_ent *ent) 298{ 299 struct mlx5_cmd_mailbox *next = ent->out->next; 300 int n = mlx5_calc_cmd_blocks(ent->out); 301 int err; 302 u8 sig; 303 int i = 0; 304 305 sig = xor8_buf(ent->lay, 0, sizeof(*ent->lay)); 306 if (sig != 0xff) 307 return -EHWPOISON; 308 309 for (i = 0; i < n && next; i++) { 310 err = verify_block_sig(next->buf); 311 if (err) 312 return -EHWPOISON; 313 314 next = next->next; 315 } 316 317 return 0; 318} 319 320static void dump_buf(void *buf, int size, int data_only, int offset, int idx) 321{ 322 __be32 *p = buf; 323 int i; 324 325 for (i = 0; i < size; i += 16) { 326 pr_debug("cmd[%d]: %03x: %08x %08x %08x %08x\n", idx, offset, 327 be32_to_cpu(p[0]), be32_to_cpu(p[1]), 328 be32_to_cpu(p[2]), be32_to_cpu(p[3])); 329 p += 4; 330 offset += 16; 331 } 332 if (!data_only) 333 pr_debug("\n"); 334} 335 336static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op, 337 u32 *synd, u8 *status) 338{ 339 *synd = 0; 340 *status = 0; 341 342 switch (op) { 343 case MLX5_CMD_OP_TEARDOWN_HCA: 344 case MLX5_CMD_OP_DISABLE_HCA: 345 case MLX5_CMD_OP_MANAGE_PAGES: 346 case MLX5_CMD_OP_DESTROY_MKEY: 347 case MLX5_CMD_OP_DESTROY_EQ: 348 case MLX5_CMD_OP_DESTROY_CQ: 349 case MLX5_CMD_OP_DESTROY_QP: 350 case MLX5_CMD_OP_DESTROY_PSV: 351 case MLX5_CMD_OP_DESTROY_SRQ: 352 case MLX5_CMD_OP_DESTROY_XRC_SRQ: 353 case MLX5_CMD_OP_DESTROY_XRQ: 354 case MLX5_CMD_OP_DESTROY_DCT: 355 case MLX5_CMD_OP_DEALLOC_Q_COUNTER: 356 case MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT: 357 case MLX5_CMD_OP_DESTROY_QOS_PARA_VPORT: 358 case MLX5_CMD_OP_DEALLOC_PD: 359 case MLX5_CMD_OP_DEALLOC_UAR: 360 case MLX5_CMD_OP_DETACH_FROM_MCG: 361 case MLX5_CMD_OP_DEALLOC_XRCD: 362 case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN: 363 case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT: 364 case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY: 365 case MLX5_CMD_OP_DESTROY_LAG: 366 case MLX5_CMD_OP_DESTROY_VPORT_LAG: 367 case MLX5_CMD_OP_DESTROY_TIR: 368 case MLX5_CMD_OP_DESTROY_SQ: 369 case MLX5_CMD_OP_DESTROY_RQ: 370 case MLX5_CMD_OP_DESTROY_RMP: 371 case MLX5_CMD_OP_DESTROY_TIS: 372 case MLX5_CMD_OP_DESTROY_RQT: 373 case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 374 case MLX5_CMD_OP_DESTROY_FLOW_GROUP: 375 case MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY: 376 case MLX5_CMD_OP_DEALLOC_FLOW_COUNTER: 377 case MLX5_CMD_OP_2ERR_QP: 378 case MLX5_CMD_OP_2RST_QP: 379 case MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT: 380 case MLX5_CMD_OP_MODIFY_FLOW_TABLE: 381 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 382 case MLX5_CMD_OP_SET_FLOW_TABLE_ROOT: 383 case MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT: 384 case MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT: 385 case MLX5_CMD_OP_FPGA_DESTROY_QP: 386 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT: 387 case MLX5_CMD_OP_DEALLOC_MEMIC: 388 case MLX5_CMD_OP_PAGE_FAULT_RESUME: 389 case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS: 390 case MLX5_CMD_OP_DEALLOC_SF: 391 case MLX5_CMD_OP_DESTROY_UCTX: 392 case MLX5_CMD_OP_DESTROY_UMEM: 393 case MLX5_CMD_OP_MODIFY_RQT: 394 return MLX5_CMD_STAT_OK; 395 396 case MLX5_CMD_OP_QUERY_HCA_CAP: 397 case MLX5_CMD_OP_QUERY_ADAPTER: 398 case MLX5_CMD_OP_INIT_HCA: 399 case MLX5_CMD_OP_ENABLE_HCA: 400 case MLX5_CMD_OP_QUERY_PAGES: 401 case MLX5_CMD_OP_SET_HCA_CAP: 402 case MLX5_CMD_OP_QUERY_ISSI: 403 case MLX5_CMD_OP_SET_ISSI: 404 case MLX5_CMD_OP_CREATE_MKEY: 405 case MLX5_CMD_OP_QUERY_MKEY: 406 case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS: 407 case MLX5_CMD_OP_CREATE_EQ: 408 case MLX5_CMD_OP_QUERY_EQ: 409 case MLX5_CMD_OP_GEN_EQE: 410 case MLX5_CMD_OP_CREATE_CQ: 411 case MLX5_CMD_OP_QUERY_CQ: 412 case MLX5_CMD_OP_MODIFY_CQ: 413 case MLX5_CMD_OP_CREATE_QP: 414 case MLX5_CMD_OP_RST2INIT_QP: 415 case MLX5_CMD_OP_INIT2RTR_QP: 416 case MLX5_CMD_OP_RTR2RTS_QP: 417 case MLX5_CMD_OP_RTS2RTS_QP: 418 case MLX5_CMD_OP_SQERR2RTS_QP: 419 case MLX5_CMD_OP_QUERY_QP: 420 case MLX5_CMD_OP_SQD_RTS_QP: 421 case MLX5_CMD_OP_INIT2INIT_QP: 422 case MLX5_CMD_OP_CREATE_PSV: 423 case MLX5_CMD_OP_CREATE_SRQ: 424 case MLX5_CMD_OP_QUERY_SRQ: 425 case MLX5_CMD_OP_ARM_RQ: 426 case MLX5_CMD_OP_CREATE_XRC_SRQ: 427 case MLX5_CMD_OP_QUERY_XRC_SRQ: 428 case MLX5_CMD_OP_ARM_XRC_SRQ: 429 case MLX5_CMD_OP_CREATE_XRQ: 430 case MLX5_CMD_OP_QUERY_XRQ: 431 case MLX5_CMD_OP_ARM_XRQ: 432 case MLX5_CMD_OP_CREATE_DCT: 433 case MLX5_CMD_OP_DRAIN_DCT: 434 case MLX5_CMD_OP_QUERY_DCT: 435 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION: 436 case MLX5_CMD_OP_QUERY_VPORT_STATE: 437 case MLX5_CMD_OP_MODIFY_VPORT_STATE: 438 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT: 439 case MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT: 440 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT: 441 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS: 442 case MLX5_CMD_OP_SET_ROCE_ADDRESS: 443 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT: 444 case MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT: 445 case MLX5_CMD_OP_QUERY_HCA_VPORT_GID: 446 case MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY: 447 case MLX5_CMD_OP_QUERY_VNIC_ENV: 448 case MLX5_CMD_OP_QUERY_VPORT_COUNTER: 449 case MLX5_CMD_OP_ALLOC_Q_COUNTER: 450 case MLX5_CMD_OP_QUERY_Q_COUNTER: 451 case MLX5_CMD_OP_SET_MONITOR_COUNTER: 452 case MLX5_CMD_OP_ARM_MONITOR_COUNTER: 453 case MLX5_CMD_OP_SET_PP_RATE_LIMIT: 454 case MLX5_CMD_OP_QUERY_RATE_LIMIT: 455 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT: 456 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT: 457 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT: 458 case MLX5_CMD_OP_CREATE_QOS_PARA_VPORT: 459 case MLX5_CMD_OP_ALLOC_PD: 460 case MLX5_CMD_OP_ALLOC_UAR: 461 case MLX5_CMD_OP_CONFIG_INT_MODERATION: 462 case MLX5_CMD_OP_ACCESS_REG: 463 case MLX5_CMD_OP_ATTACH_TO_MCG: 464 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG: 465 case MLX5_CMD_OP_MAD_IFC: 466 case MLX5_CMD_OP_QUERY_MAD_DEMUX: 467 case MLX5_CMD_OP_SET_MAD_DEMUX: 468 case MLX5_CMD_OP_NOP: 469 case MLX5_CMD_OP_ALLOC_XRCD: 470 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN: 471 case MLX5_CMD_OP_QUERY_CONG_STATUS: 472 case MLX5_CMD_OP_MODIFY_CONG_STATUS: 473 case MLX5_CMD_OP_QUERY_CONG_PARAMS: 474 case MLX5_CMD_OP_MODIFY_CONG_PARAMS: 475 case MLX5_CMD_OP_QUERY_CONG_STATISTICS: 476 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 477 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 478 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY: 479 case MLX5_CMD_OP_CREATE_LAG: 480 case MLX5_CMD_OP_MODIFY_LAG: 481 case MLX5_CMD_OP_QUERY_LAG: 482 case MLX5_CMD_OP_CREATE_VPORT_LAG: 483 case MLX5_CMD_OP_CREATE_TIR: 484 case MLX5_CMD_OP_MODIFY_TIR: 485 case MLX5_CMD_OP_QUERY_TIR: 486 case MLX5_CMD_OP_CREATE_SQ: 487 case MLX5_CMD_OP_MODIFY_SQ: 488 case MLX5_CMD_OP_QUERY_SQ: 489 case MLX5_CMD_OP_CREATE_RQ: 490 case MLX5_CMD_OP_MODIFY_RQ: 491 case MLX5_CMD_OP_QUERY_RQ: 492 case MLX5_CMD_OP_CREATE_RMP: 493 case MLX5_CMD_OP_MODIFY_RMP: 494 case MLX5_CMD_OP_QUERY_RMP: 495 case MLX5_CMD_OP_CREATE_TIS: 496 case MLX5_CMD_OP_MODIFY_TIS: 497 case MLX5_CMD_OP_QUERY_TIS: 498 case MLX5_CMD_OP_CREATE_RQT: 499 case MLX5_CMD_OP_QUERY_RQT: 500 501 case MLX5_CMD_OP_CREATE_FLOW_TABLE: 502 case MLX5_CMD_OP_QUERY_FLOW_TABLE: 503 case MLX5_CMD_OP_CREATE_FLOW_GROUP: 504 case MLX5_CMD_OP_QUERY_FLOW_GROUP: 505 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY: 506 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER: 507 case MLX5_CMD_OP_QUERY_FLOW_COUNTER: 508 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT: 509 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT: 510 case MLX5_CMD_OP_FPGA_CREATE_QP: 511 case MLX5_CMD_OP_FPGA_MODIFY_QP: 512 case MLX5_CMD_OP_FPGA_QUERY_QP: 513 case MLX5_CMD_OP_FPGA_QUERY_QP_COUNTERS: 514 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT: 515 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT: 516 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT: 517 case MLX5_CMD_OP_CREATE_UCTX: 518 case MLX5_CMD_OP_CREATE_UMEM: 519 case MLX5_CMD_OP_ALLOC_MEMIC: 520 case MLX5_CMD_OP_MODIFY_XRQ: 521 case MLX5_CMD_OP_RELEASE_XRQ_ERROR: 522 case MLX5_CMD_OP_QUERY_VHCA_STATE: 523 case MLX5_CMD_OP_MODIFY_VHCA_STATE: 524 case MLX5_CMD_OP_ALLOC_SF: 525 case MLX5_CMD_OP_SUSPEND_VHCA: 526 case MLX5_CMD_OP_RESUME_VHCA: 527 case MLX5_CMD_OP_QUERY_VHCA_MIGRATION_STATE: 528 case MLX5_CMD_OP_SAVE_VHCA_STATE: 529 case MLX5_CMD_OP_LOAD_VHCA_STATE: 530 case MLX5_CMD_OP_SYNC_CRYPTO: 531 case MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS: 532 *status = MLX5_DRIVER_STATUS_ABORTED; 533 *synd = MLX5_DRIVER_SYND; 534 return -ENOLINK; 535 default: 536 mlx5_core_err(dev, "Unknown FW command (%d)\n", op); 537 return -EINVAL; 538 } 539} 540 541const char *mlx5_command_str(int command) 542{ 543#define MLX5_COMMAND_STR_CASE(__cmd) case MLX5_CMD_OP_ ## __cmd: return #__cmd 544 545 switch (command) { 546 MLX5_COMMAND_STR_CASE(QUERY_HCA_CAP); 547 MLX5_COMMAND_STR_CASE(QUERY_ADAPTER); 548 MLX5_COMMAND_STR_CASE(INIT_HCA); 549 MLX5_COMMAND_STR_CASE(TEARDOWN_HCA); 550 MLX5_COMMAND_STR_CASE(ENABLE_HCA); 551 MLX5_COMMAND_STR_CASE(DISABLE_HCA); 552 MLX5_COMMAND_STR_CASE(QUERY_PAGES); 553 MLX5_COMMAND_STR_CASE(MANAGE_PAGES); 554 MLX5_COMMAND_STR_CASE(SET_HCA_CAP); 555 MLX5_COMMAND_STR_CASE(QUERY_ISSI); 556 MLX5_COMMAND_STR_CASE(SET_ISSI); 557 MLX5_COMMAND_STR_CASE(SET_DRIVER_VERSION); 558 MLX5_COMMAND_STR_CASE(CREATE_MKEY); 559 MLX5_COMMAND_STR_CASE(QUERY_MKEY); 560 MLX5_COMMAND_STR_CASE(DESTROY_MKEY); 561 MLX5_COMMAND_STR_CASE(QUERY_SPECIAL_CONTEXTS); 562 MLX5_COMMAND_STR_CASE(PAGE_FAULT_RESUME); 563 MLX5_COMMAND_STR_CASE(CREATE_EQ); 564 MLX5_COMMAND_STR_CASE(DESTROY_EQ); 565 MLX5_COMMAND_STR_CASE(QUERY_EQ); 566 MLX5_COMMAND_STR_CASE(GEN_EQE); 567 MLX5_COMMAND_STR_CASE(CREATE_CQ); 568 MLX5_COMMAND_STR_CASE(DESTROY_CQ); 569 MLX5_COMMAND_STR_CASE(QUERY_CQ); 570 MLX5_COMMAND_STR_CASE(MODIFY_CQ); 571 MLX5_COMMAND_STR_CASE(CREATE_QP); 572 MLX5_COMMAND_STR_CASE(DESTROY_QP); 573 MLX5_COMMAND_STR_CASE(RST2INIT_QP); 574 MLX5_COMMAND_STR_CASE(INIT2RTR_QP); 575 MLX5_COMMAND_STR_CASE(RTR2RTS_QP); 576 MLX5_COMMAND_STR_CASE(RTS2RTS_QP); 577 MLX5_COMMAND_STR_CASE(SQERR2RTS_QP); 578 MLX5_COMMAND_STR_CASE(2ERR_QP); 579 MLX5_COMMAND_STR_CASE(2RST_QP); 580 MLX5_COMMAND_STR_CASE(QUERY_QP); 581 MLX5_COMMAND_STR_CASE(SQD_RTS_QP); 582 MLX5_COMMAND_STR_CASE(INIT2INIT_QP); 583 MLX5_COMMAND_STR_CASE(CREATE_PSV); 584 MLX5_COMMAND_STR_CASE(DESTROY_PSV); 585 MLX5_COMMAND_STR_CASE(CREATE_SRQ); 586 MLX5_COMMAND_STR_CASE(DESTROY_SRQ); 587 MLX5_COMMAND_STR_CASE(QUERY_SRQ); 588 MLX5_COMMAND_STR_CASE(ARM_RQ); 589 MLX5_COMMAND_STR_CASE(CREATE_XRC_SRQ); 590 MLX5_COMMAND_STR_CASE(DESTROY_XRC_SRQ); 591 MLX5_COMMAND_STR_CASE(QUERY_XRC_SRQ); 592 MLX5_COMMAND_STR_CASE(ARM_XRC_SRQ); 593 MLX5_COMMAND_STR_CASE(CREATE_DCT); 594 MLX5_COMMAND_STR_CASE(DESTROY_DCT); 595 MLX5_COMMAND_STR_CASE(DRAIN_DCT); 596 MLX5_COMMAND_STR_CASE(QUERY_DCT); 597 MLX5_COMMAND_STR_CASE(ARM_DCT_FOR_KEY_VIOLATION); 598 MLX5_COMMAND_STR_CASE(QUERY_VPORT_STATE); 599 MLX5_COMMAND_STR_CASE(MODIFY_VPORT_STATE); 600 MLX5_COMMAND_STR_CASE(QUERY_ESW_VPORT_CONTEXT); 601 MLX5_COMMAND_STR_CASE(MODIFY_ESW_VPORT_CONTEXT); 602 MLX5_COMMAND_STR_CASE(QUERY_NIC_VPORT_CONTEXT); 603 MLX5_COMMAND_STR_CASE(MODIFY_NIC_VPORT_CONTEXT); 604 MLX5_COMMAND_STR_CASE(QUERY_ROCE_ADDRESS); 605 MLX5_COMMAND_STR_CASE(SET_ROCE_ADDRESS); 606 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_CONTEXT); 607 MLX5_COMMAND_STR_CASE(MODIFY_HCA_VPORT_CONTEXT); 608 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_GID); 609 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_PKEY); 610 MLX5_COMMAND_STR_CASE(QUERY_VNIC_ENV); 611 MLX5_COMMAND_STR_CASE(QUERY_VPORT_COUNTER); 612 MLX5_COMMAND_STR_CASE(ALLOC_Q_COUNTER); 613 MLX5_COMMAND_STR_CASE(DEALLOC_Q_COUNTER); 614 MLX5_COMMAND_STR_CASE(QUERY_Q_COUNTER); 615 MLX5_COMMAND_STR_CASE(SET_MONITOR_COUNTER); 616 MLX5_COMMAND_STR_CASE(ARM_MONITOR_COUNTER); 617 MLX5_COMMAND_STR_CASE(SET_PP_RATE_LIMIT); 618 MLX5_COMMAND_STR_CASE(QUERY_RATE_LIMIT); 619 MLX5_COMMAND_STR_CASE(CREATE_SCHEDULING_ELEMENT); 620 MLX5_COMMAND_STR_CASE(DESTROY_SCHEDULING_ELEMENT); 621 MLX5_COMMAND_STR_CASE(QUERY_SCHEDULING_ELEMENT); 622 MLX5_COMMAND_STR_CASE(MODIFY_SCHEDULING_ELEMENT); 623 MLX5_COMMAND_STR_CASE(CREATE_QOS_PARA_VPORT); 624 MLX5_COMMAND_STR_CASE(DESTROY_QOS_PARA_VPORT); 625 MLX5_COMMAND_STR_CASE(ALLOC_PD); 626 MLX5_COMMAND_STR_CASE(DEALLOC_PD); 627 MLX5_COMMAND_STR_CASE(ALLOC_UAR); 628 MLX5_COMMAND_STR_CASE(DEALLOC_UAR); 629 MLX5_COMMAND_STR_CASE(CONFIG_INT_MODERATION); 630 MLX5_COMMAND_STR_CASE(ACCESS_REG); 631 MLX5_COMMAND_STR_CASE(ATTACH_TO_MCG); 632 MLX5_COMMAND_STR_CASE(DETACH_FROM_MCG); 633 MLX5_COMMAND_STR_CASE(GET_DROPPED_PACKET_LOG); 634 MLX5_COMMAND_STR_CASE(MAD_IFC); 635 MLX5_COMMAND_STR_CASE(QUERY_MAD_DEMUX); 636 MLX5_COMMAND_STR_CASE(SET_MAD_DEMUX); 637 MLX5_COMMAND_STR_CASE(NOP); 638 MLX5_COMMAND_STR_CASE(ALLOC_XRCD); 639 MLX5_COMMAND_STR_CASE(DEALLOC_XRCD); 640 MLX5_COMMAND_STR_CASE(ALLOC_TRANSPORT_DOMAIN); 641 MLX5_COMMAND_STR_CASE(DEALLOC_TRANSPORT_DOMAIN); 642 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATUS); 643 MLX5_COMMAND_STR_CASE(MODIFY_CONG_STATUS); 644 MLX5_COMMAND_STR_CASE(QUERY_CONG_PARAMS); 645 MLX5_COMMAND_STR_CASE(MODIFY_CONG_PARAMS); 646 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATISTICS); 647 MLX5_COMMAND_STR_CASE(ADD_VXLAN_UDP_DPORT); 648 MLX5_COMMAND_STR_CASE(DELETE_VXLAN_UDP_DPORT); 649 MLX5_COMMAND_STR_CASE(SET_L2_TABLE_ENTRY); 650 MLX5_COMMAND_STR_CASE(QUERY_L2_TABLE_ENTRY); 651 MLX5_COMMAND_STR_CASE(DELETE_L2_TABLE_ENTRY); 652 MLX5_COMMAND_STR_CASE(SET_WOL_ROL); 653 MLX5_COMMAND_STR_CASE(QUERY_WOL_ROL); 654 MLX5_COMMAND_STR_CASE(CREATE_LAG); 655 MLX5_COMMAND_STR_CASE(MODIFY_LAG); 656 MLX5_COMMAND_STR_CASE(QUERY_LAG); 657 MLX5_COMMAND_STR_CASE(DESTROY_LAG); 658 MLX5_COMMAND_STR_CASE(CREATE_VPORT_LAG); 659 MLX5_COMMAND_STR_CASE(DESTROY_VPORT_LAG); 660 MLX5_COMMAND_STR_CASE(CREATE_TIR); 661 MLX5_COMMAND_STR_CASE(MODIFY_TIR); 662 MLX5_COMMAND_STR_CASE(DESTROY_TIR); 663 MLX5_COMMAND_STR_CASE(QUERY_TIR); 664 MLX5_COMMAND_STR_CASE(CREATE_SQ); 665 MLX5_COMMAND_STR_CASE(MODIFY_SQ); 666 MLX5_COMMAND_STR_CASE(DESTROY_SQ); 667 MLX5_COMMAND_STR_CASE(QUERY_SQ); 668 MLX5_COMMAND_STR_CASE(CREATE_RQ); 669 MLX5_COMMAND_STR_CASE(MODIFY_RQ); 670 MLX5_COMMAND_STR_CASE(DESTROY_RQ); 671 MLX5_COMMAND_STR_CASE(QUERY_RQ); 672 MLX5_COMMAND_STR_CASE(CREATE_RMP); 673 MLX5_COMMAND_STR_CASE(MODIFY_RMP); 674 MLX5_COMMAND_STR_CASE(DESTROY_RMP); 675 MLX5_COMMAND_STR_CASE(QUERY_RMP); 676 MLX5_COMMAND_STR_CASE(CREATE_TIS); 677 MLX5_COMMAND_STR_CASE(MODIFY_TIS); 678 MLX5_COMMAND_STR_CASE(DESTROY_TIS); 679 MLX5_COMMAND_STR_CASE(QUERY_TIS); 680 MLX5_COMMAND_STR_CASE(CREATE_RQT); 681 MLX5_COMMAND_STR_CASE(MODIFY_RQT); 682 MLX5_COMMAND_STR_CASE(DESTROY_RQT); 683 MLX5_COMMAND_STR_CASE(QUERY_RQT); 684 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ROOT); 685 MLX5_COMMAND_STR_CASE(CREATE_FLOW_TABLE); 686 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_TABLE); 687 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE); 688 MLX5_COMMAND_STR_CASE(CREATE_FLOW_GROUP); 689 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_GROUP); 690 MLX5_COMMAND_STR_CASE(QUERY_FLOW_GROUP); 691 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ENTRY); 692 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE_ENTRY); 693 MLX5_COMMAND_STR_CASE(DELETE_FLOW_TABLE_ENTRY); 694 MLX5_COMMAND_STR_CASE(ALLOC_FLOW_COUNTER); 695 MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER); 696 MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER); 697 MLX5_COMMAND_STR_CASE(MODIFY_FLOW_TABLE); 698 MLX5_COMMAND_STR_CASE(ALLOC_PACKET_REFORMAT_CONTEXT); 699 MLX5_COMMAND_STR_CASE(DEALLOC_PACKET_REFORMAT_CONTEXT); 700 MLX5_COMMAND_STR_CASE(ALLOC_MODIFY_HEADER_CONTEXT); 701 MLX5_COMMAND_STR_CASE(DEALLOC_MODIFY_HEADER_CONTEXT); 702 MLX5_COMMAND_STR_CASE(FPGA_CREATE_QP); 703 MLX5_COMMAND_STR_CASE(FPGA_MODIFY_QP); 704 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP); 705 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP_COUNTERS); 706 MLX5_COMMAND_STR_CASE(FPGA_DESTROY_QP); 707 MLX5_COMMAND_STR_CASE(CREATE_XRQ); 708 MLX5_COMMAND_STR_CASE(DESTROY_XRQ); 709 MLX5_COMMAND_STR_CASE(QUERY_XRQ); 710 MLX5_COMMAND_STR_CASE(ARM_XRQ); 711 MLX5_COMMAND_STR_CASE(CREATE_GENERAL_OBJECT); 712 MLX5_COMMAND_STR_CASE(DESTROY_GENERAL_OBJECT); 713 MLX5_COMMAND_STR_CASE(MODIFY_GENERAL_OBJECT); 714 MLX5_COMMAND_STR_CASE(QUERY_GENERAL_OBJECT); 715 MLX5_COMMAND_STR_CASE(QUERY_MODIFY_HEADER_CONTEXT); 716 MLX5_COMMAND_STR_CASE(ALLOC_MEMIC); 717 MLX5_COMMAND_STR_CASE(DEALLOC_MEMIC); 718 MLX5_COMMAND_STR_CASE(QUERY_ESW_FUNCTIONS); 719 MLX5_COMMAND_STR_CASE(CREATE_UCTX); 720 MLX5_COMMAND_STR_CASE(DESTROY_UCTX); 721 MLX5_COMMAND_STR_CASE(CREATE_UMEM); 722 MLX5_COMMAND_STR_CASE(DESTROY_UMEM); 723 MLX5_COMMAND_STR_CASE(RELEASE_XRQ_ERROR); 724 MLX5_COMMAND_STR_CASE(MODIFY_XRQ); 725 MLX5_COMMAND_STR_CASE(QUERY_VHCA_STATE); 726 MLX5_COMMAND_STR_CASE(MODIFY_VHCA_STATE); 727 MLX5_COMMAND_STR_CASE(ALLOC_SF); 728 MLX5_COMMAND_STR_CASE(DEALLOC_SF); 729 MLX5_COMMAND_STR_CASE(SUSPEND_VHCA); 730 MLX5_COMMAND_STR_CASE(RESUME_VHCA); 731 MLX5_COMMAND_STR_CASE(QUERY_VHCA_MIGRATION_STATE); 732 MLX5_COMMAND_STR_CASE(SAVE_VHCA_STATE); 733 MLX5_COMMAND_STR_CASE(LOAD_VHCA_STATE); 734 MLX5_COMMAND_STR_CASE(SYNC_CRYPTO); 735 MLX5_COMMAND_STR_CASE(ALLOW_OTHER_VHCA_ACCESS); 736 default: return "unknown command opcode"; 737 } 738} 739 740static const char *cmd_status_str(u8 status) 741{ 742 switch (status) { 743 case MLX5_CMD_STAT_OK: 744 return "OK"; 745 case MLX5_CMD_STAT_INT_ERR: 746 return "internal error"; 747 case MLX5_CMD_STAT_BAD_OP_ERR: 748 return "bad operation"; 749 case MLX5_CMD_STAT_BAD_PARAM_ERR: 750 return "bad parameter"; 751 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: 752 return "bad system state"; 753 case MLX5_CMD_STAT_BAD_RES_ERR: 754 return "bad resource"; 755 case MLX5_CMD_STAT_RES_BUSY: 756 return "resource busy"; 757 case MLX5_CMD_STAT_LIM_ERR: 758 return "limits exceeded"; 759 case MLX5_CMD_STAT_BAD_RES_STATE_ERR: 760 return "bad resource state"; 761 case MLX5_CMD_STAT_IX_ERR: 762 return "bad index"; 763 case MLX5_CMD_STAT_NO_RES_ERR: 764 return "no resources"; 765 case MLX5_CMD_STAT_BAD_INP_LEN_ERR: 766 return "bad input length"; 767 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: 768 return "bad output length"; 769 case MLX5_CMD_STAT_BAD_QP_STATE_ERR: 770 return "bad QP state"; 771 case MLX5_CMD_STAT_BAD_PKT_ERR: 772 return "bad packet (discarded)"; 773 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: 774 return "bad size too many outstanding CQEs"; 775 default: 776 return "unknown status"; 777 } 778} 779 780static int cmd_status_to_err(u8 status) 781{ 782 switch (status) { 783 case MLX5_CMD_STAT_OK: return 0; 784 case MLX5_CMD_STAT_INT_ERR: return -EIO; 785 case MLX5_CMD_STAT_BAD_OP_ERR: return -EINVAL; 786 case MLX5_CMD_STAT_BAD_PARAM_ERR: return -EINVAL; 787 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: return -EIO; 788 case MLX5_CMD_STAT_BAD_RES_ERR: return -EINVAL; 789 case MLX5_CMD_STAT_RES_BUSY: return -EBUSY; 790 case MLX5_CMD_STAT_LIM_ERR: return -ENOMEM; 791 case MLX5_CMD_STAT_BAD_RES_STATE_ERR: return -EINVAL; 792 case MLX5_CMD_STAT_IX_ERR: return -EINVAL; 793 case MLX5_CMD_STAT_NO_RES_ERR: return -EAGAIN; 794 case MLX5_CMD_STAT_BAD_INP_LEN_ERR: return -EIO; 795 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: return -EIO; 796 case MLX5_CMD_STAT_BAD_QP_STATE_ERR: return -EINVAL; 797 case MLX5_CMD_STAT_BAD_PKT_ERR: return -EINVAL; 798 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: return -EINVAL; 799 default: return -EIO; 800 } 801} 802 803void mlx5_cmd_out_err(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out) 804{ 805 u32 syndrome = MLX5_GET(mbox_out, out, syndrome); 806 u8 status = MLX5_GET(mbox_out, out, status); 807 808 mlx5_core_err_rl(dev, 809 "%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x), err(%d)\n", 810 mlx5_command_str(opcode), opcode, op_mod, 811 cmd_status_str(status), status, syndrome, cmd_status_to_err(status)); 812} 813EXPORT_SYMBOL(mlx5_cmd_out_err); 814 815static void cmd_status_print(struct mlx5_core_dev *dev, void *in, void *out) 816{ 817 u16 opcode, op_mod; 818 u16 uid; 819 820 opcode = in_to_opcode(in); 821 op_mod = MLX5_GET(mbox_in, in, op_mod); 822 uid = MLX5_GET(mbox_in, in, uid); 823 824 if (!uid && opcode != MLX5_CMD_OP_DESTROY_MKEY && 825 opcode != MLX5_CMD_OP_CREATE_UCTX) 826 mlx5_cmd_out_err(dev, opcode, op_mod, out); 827} 828 829int mlx5_cmd_check(struct mlx5_core_dev *dev, int err, void *in, void *out) 830{ 831 /* aborted due to PCI error or via reset flow mlx5_cmd_trigger_completions() */ 832 if (err == -ENXIO) { 833 u16 opcode = in_to_opcode(in); 834 u32 syndrome; 835 u8 status; 836 837 /* PCI Error, emulate command return status, for smooth reset */ 838 err = mlx5_internal_err_ret_value(dev, opcode, &syndrome, &status); 839 MLX5_SET(mbox_out, out, status, status); 840 MLX5_SET(mbox_out, out, syndrome, syndrome); 841 if (!err) 842 return 0; 843 } 844 845 /* driver or FW delivery error */ 846 if (err != -EREMOTEIO && err) 847 return err; 848 849 /* check outbox status */ 850 err = cmd_status_to_err(MLX5_GET(mbox_out, out, status)); 851 if (err) 852 cmd_status_print(dev, in, out); 853 854 return err; 855} 856EXPORT_SYMBOL(mlx5_cmd_check); 857 858static void dump_command(struct mlx5_core_dev *dev, 859 struct mlx5_cmd_work_ent *ent, int input) 860{ 861 struct mlx5_cmd_msg *msg = input ? ent->in : ent->out; 862 struct mlx5_cmd_mailbox *next = msg->next; 863 int n = mlx5_calc_cmd_blocks(msg); 864 u16 op = ent->op; 865 int data_only; 866 u32 offset = 0; 867 int dump_len; 868 int i; 869 870 mlx5_core_dbg(dev, "cmd[%d]: start dump\n", ent->idx); 871 data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA)); 872 873 if (data_only) 874 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_DATA, 875 "cmd[%d]: dump command data %s(0x%x) %s\n", 876 ent->idx, mlx5_command_str(op), op, 877 input ? "INPUT" : "OUTPUT"); 878 else 879 mlx5_core_dbg(dev, "cmd[%d]: dump command %s(0x%x) %s\n", 880 ent->idx, mlx5_command_str(op), op, 881 input ? "INPUT" : "OUTPUT"); 882 883 if (data_only) { 884 if (input) { 885 dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset, ent->idx); 886 offset += sizeof(ent->lay->in); 887 } else { 888 dump_buf(ent->lay->out, sizeof(ent->lay->out), 1, offset, ent->idx); 889 offset += sizeof(ent->lay->out); 890 } 891 } else { 892 dump_buf(ent->lay, sizeof(*ent->lay), 0, offset, ent->idx); 893 offset += sizeof(*ent->lay); 894 } 895 896 for (i = 0; i < n && next; i++) { 897 if (data_only) { 898 dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg->len - offset); 899 dump_buf(next->buf, dump_len, 1, offset, ent->idx); 900 offset += MLX5_CMD_DATA_BLOCK_SIZE; 901 } else { 902 mlx5_core_dbg(dev, "cmd[%d]: command block:\n", ent->idx); 903 dump_buf(next->buf, sizeof(struct mlx5_cmd_prot_block), 0, offset, 904 ent->idx); 905 offset += sizeof(struct mlx5_cmd_prot_block); 906 } 907 next = next->next; 908 } 909 910 if (data_only) 911 pr_debug("\n"); 912 913 mlx5_core_dbg(dev, "cmd[%d]: end dump\n", ent->idx); 914} 915 916static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced); 917 918static void cb_timeout_handler(struct work_struct *work) 919{ 920 struct delayed_work *dwork = container_of(work, struct delayed_work, 921 work); 922 struct mlx5_cmd_work_ent *ent = container_of(dwork, 923 struct mlx5_cmd_work_ent, 924 cb_timeout_work); 925 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, 926 cmd); 927 928 mlx5_cmd_eq_recover(dev); 929 930 /* Maybe got handled by eq recover ? */ 931 if (!test_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state)) { 932 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, recovered after timeout\n", ent->idx, 933 mlx5_command_str(ent->op), ent->op); 934 goto out; /* phew, already handled */ 935 } 936 937 ent->ret = -ETIMEDOUT; 938 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, timeout. Will cause a leak of a command resource\n", 939 ent->idx, mlx5_command_str(ent->op), ent->op); 940 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true); 941 942out: 943 cmd_ent_put(ent); /* for the cmd_ent_get() took on schedule delayed work */ 944} 945 946static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg); 947static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev, 948 struct mlx5_cmd_msg *msg); 949 950static bool opcode_allowed(struct mlx5_cmd *cmd, u16 opcode) 951{ 952 if (cmd->allowed_opcode == CMD_ALLOWED_OPCODE_ALL) 953 return true; 954 955 return cmd->allowed_opcode == opcode; 956} 957 958bool mlx5_cmd_is_down(struct mlx5_core_dev *dev) 959{ 960 return pci_channel_offline(dev->pdev) || 961 dev->cmd.state != MLX5_CMDIF_STATE_UP || 962 dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR; 963} 964 965static void cmd_work_handler(struct work_struct *work) 966{ 967 struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work); 968 struct mlx5_cmd *cmd = ent->cmd; 969 bool poll_cmd = ent->polling; 970 struct mlx5_cmd_layout *lay; 971 struct mlx5_core_dev *dev; 972 unsigned long cb_timeout; 973 struct semaphore *sem; 974 unsigned long flags; 975 int alloc_ret; 976 int cmd_mode; 977 978 dev = container_of(cmd, struct mlx5_core_dev, cmd); 979 cb_timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD)); 980 981 complete(&ent->handling); 982 sem = ent->page_queue ? &cmd->vars.pages_sem : &cmd->vars.sem; 983 down(sem); 984 if (!ent->page_queue) { 985 alloc_ret = cmd_alloc_index(cmd, ent); 986 if (alloc_ret < 0) { 987 mlx5_core_err_rl(dev, "failed to allocate command entry\n"); 988 if (ent->callback) { 989 ent->callback(-EAGAIN, ent->context); 990 mlx5_free_cmd_msg(dev, ent->out); 991 free_msg(dev, ent->in); 992 cmd_ent_put(ent); 993 } else { 994 ent->ret = -EAGAIN; 995 complete(&ent->done); 996 } 997 up(sem); 998 return; 999 } 1000 } else { 1001 ent->idx = cmd->vars.max_reg_cmds; 1002 spin_lock_irqsave(&cmd->alloc_lock, flags); 1003 clear_bit(ent->idx, &cmd->vars.bitmask); 1004 cmd->ent_arr[ent->idx] = ent; 1005 spin_unlock_irqrestore(&cmd->alloc_lock, flags); 1006 } 1007 1008 lay = get_inst(cmd, ent->idx); 1009 ent->lay = lay; 1010 memset(lay, 0, sizeof(*lay)); 1011 memcpy(lay->in, ent->in->first.data, sizeof(lay->in)); 1012 if (ent->in->next) 1013 lay->in_ptr = cpu_to_be64(ent->in->next->dma); 1014 lay->inlen = cpu_to_be32(ent->in->len); 1015 if (ent->out->next) 1016 lay->out_ptr = cpu_to_be64(ent->out->next->dma); 1017 lay->outlen = cpu_to_be32(ent->out->len); 1018 lay->type = MLX5_PCI_CMD_XPORT; 1019 lay->token = ent->token; 1020 lay->status_own = CMD_OWNER_HW; 1021 set_signature(ent, !cmd->checksum_disabled); 1022 dump_command(dev, ent, 1); 1023 ent->ts1 = ktime_get_ns(); 1024 cmd_mode = cmd->mode; 1025 1026 if (ent->callback && schedule_delayed_work(&ent->cb_timeout_work, cb_timeout)) 1027 cmd_ent_get(ent); 1028 set_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state); 1029 1030 cmd_ent_get(ent); /* for the _real_ FW event on completion */ 1031 /* Skip sending command to fw if internal error */ 1032 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, ent->op)) { 1033 ent->ret = -ENXIO; 1034 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true); 1035 return; 1036 } 1037 1038 /* ring doorbell after the descriptor is valid */ 1039 mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx); 1040 wmb(); 1041 iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell); 1042 /* if not in polling don't use ent after this point */ 1043 if (cmd_mode == CMD_MODE_POLLING || poll_cmd) { 1044 poll_timeout(ent); 1045 /* make sure we read the descriptor after ownership is SW */ 1046 rmb(); 1047 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, (ent->ret == -ETIMEDOUT)); 1048 } 1049} 1050 1051static int deliv_status_to_err(u8 status) 1052{ 1053 switch (status) { 1054 case MLX5_CMD_DELIVERY_STAT_OK: 1055 case MLX5_DRIVER_STATUS_ABORTED: 1056 return 0; 1057 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR: 1058 case MLX5_CMD_DELIVERY_STAT_TOK_ERR: 1059 return -EBADR; 1060 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR: 1061 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR: 1062 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR: 1063 return -EFAULT; /* Bad address */ 1064 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR: 1065 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR: 1066 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR: 1067 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR: 1068 return -ENOMSG; 1069 case MLX5_CMD_DELIVERY_STAT_FW_ERR: 1070 return -EIO; 1071 default: 1072 return -EINVAL; 1073 } 1074} 1075 1076static const char *deliv_status_to_str(u8 status) 1077{ 1078 switch (status) { 1079 case MLX5_CMD_DELIVERY_STAT_OK: 1080 return "no errors"; 1081 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR: 1082 return "signature error"; 1083 case MLX5_CMD_DELIVERY_STAT_TOK_ERR: 1084 return "token error"; 1085 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR: 1086 return "bad block number"; 1087 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR: 1088 return "output pointer not aligned to block size"; 1089 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR: 1090 return "input pointer not aligned to block size"; 1091 case MLX5_CMD_DELIVERY_STAT_FW_ERR: 1092 return "firmware internal error"; 1093 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR: 1094 return "command input length error"; 1095 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR: 1096 return "command output length error"; 1097 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR: 1098 return "reserved fields not cleared"; 1099 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR: 1100 return "bad command descriptor type"; 1101 default: 1102 return "unknown status code"; 1103 } 1104} 1105 1106enum { 1107 MLX5_CMD_TIMEOUT_RECOVER_MSEC = 5 * 1000, 1108}; 1109 1110static void wait_func_handle_exec_timeout(struct mlx5_core_dev *dev, 1111 struct mlx5_cmd_work_ent *ent) 1112{ 1113 unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_RECOVER_MSEC); 1114 1115 mlx5_cmd_eq_recover(dev); 1116 1117 /* Re-wait on the ent->done after executing the recovery flow. If the 1118 * recovery flow (or any other recovery flow running simultaneously) 1119 * has recovered an EQE, it should cause the entry to be completed by 1120 * the command interface. 1121 */ 1122 if (wait_for_completion_timeout(&ent->done, timeout)) { 1123 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) recovered after timeout\n", ent->idx, 1124 mlx5_command_str(ent->op), ent->op); 1125 return; 1126 } 1127 1128 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) No done completion\n", ent->idx, 1129 mlx5_command_str(ent->op), ent->op); 1130 1131 ent->ret = -ETIMEDOUT; 1132 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true); 1133} 1134 1135static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent) 1136{ 1137 unsigned long timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD)); 1138 struct mlx5_cmd *cmd = &dev->cmd; 1139 int err; 1140 1141 if (!wait_for_completion_timeout(&ent->handling, timeout) && 1142 cancel_work_sync(&ent->work)) { 1143 ent->ret = -ECANCELED; 1144 goto out_err; 1145 } 1146 if (cmd->mode == CMD_MODE_POLLING || ent->polling) 1147 wait_for_completion(&ent->done); 1148 else if (!wait_for_completion_timeout(&ent->done, timeout)) 1149 wait_func_handle_exec_timeout(dev, ent); 1150 1151out_err: 1152 err = ent->ret; 1153 1154 if (err == -ETIMEDOUT) { 1155 mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n", 1156 mlx5_command_str(ent->op), ent->op); 1157 } else if (err == -ECANCELED) { 1158 mlx5_core_warn(dev, "%s(0x%x) canceled on out of queue timeout.\n", 1159 mlx5_command_str(ent->op), ent->op); 1160 } 1161 mlx5_core_dbg(dev, "err %d, delivery status %s(%d)\n", 1162 err, deliv_status_to_str(ent->status), ent->status); 1163 1164 return err; 1165} 1166 1167/* Notes: 1168 * 1. Callback functions may not sleep 1169 * 2. page queue commands do not support asynchrous completion 1170 * 1171 * return value in case (!callback): 1172 * ret < 0 : Command execution couldn't be submitted by driver 1173 * ret > 0 : Command execution couldn't be performed by firmware 1174 * ret == 0: Command was executed by FW, Caller must check FW outbox status. 1175 * 1176 * return value in case (callback): 1177 * ret < 0 : Command execution couldn't be submitted by driver 1178 * ret == 0: Command will be submitted to FW for execution 1179 * and the callback will be called for further status updates 1180 */ 1181static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in, 1182 struct mlx5_cmd_msg *out, void *uout, int uout_size, 1183 mlx5_cmd_cbk_t callback, 1184 void *context, int page_queue, 1185 u8 token, bool force_polling) 1186{ 1187 struct mlx5_cmd *cmd = &dev->cmd; 1188 struct mlx5_cmd_work_ent *ent; 1189 struct mlx5_cmd_stats *stats; 1190 u8 status = 0; 1191 int err = 0; 1192 s64 ds; 1193 1194 if (callback && page_queue) 1195 return -EINVAL; 1196 1197 ent = cmd_alloc_ent(cmd, in, out, uout, uout_size, 1198 callback, context, page_queue); 1199 if (IS_ERR(ent)) 1200 return PTR_ERR(ent); 1201 1202 /* put for this ent is when consumed, depending on the use case 1203 * 1) (!callback) blocking flow: by caller after wait_func completes 1204 * 2) (callback) flow: by mlx5_cmd_comp_handler() when ent is handled 1205 */ 1206 1207 ent->token = token; 1208 ent->polling = force_polling; 1209 1210 init_completion(&ent->handling); 1211 if (!callback) 1212 init_completion(&ent->done); 1213 1214 INIT_DELAYED_WORK(&ent->cb_timeout_work, cb_timeout_handler); 1215 INIT_WORK(&ent->work, cmd_work_handler); 1216 if (page_queue) { 1217 cmd_work_handler(&ent->work); 1218 } else if (!queue_work(cmd->wq, &ent->work)) { 1219 mlx5_core_warn(dev, "failed to queue work\n"); 1220 err = -EALREADY; 1221 goto out_free; 1222 } 1223 1224 if (callback) 1225 return 0; /* mlx5_cmd_comp_handler() will put(ent) */ 1226 1227 err = wait_func(dev, ent); 1228 if (err == -ETIMEDOUT || err == -ECANCELED) 1229 goto out_free; 1230 1231 ds = ent->ts2 - ent->ts1; 1232 stats = xa_load(&cmd->stats, ent->op); 1233 if (stats) { 1234 spin_lock_irq(&stats->lock); 1235 stats->sum += ds; 1236 ++stats->n; 1237 spin_unlock_irq(&stats->lock); 1238 } 1239 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_TIME, 1240 "fw exec time for %s is %lld nsec\n", 1241 mlx5_command_str(ent->op), ds); 1242 1243out_free: 1244 status = ent->status; 1245 cmd_ent_put(ent); 1246 return err ? : status; 1247} 1248 1249static ssize_t dbg_write(struct file *filp, const char __user *buf, 1250 size_t count, loff_t *pos) 1251{ 1252 struct mlx5_core_dev *dev = filp->private_data; 1253 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1254 char lbuf[3]; 1255 int err; 1256 1257 if (!dbg->in_msg || !dbg->out_msg) 1258 return -ENOMEM; 1259 1260 if (count < sizeof(lbuf) - 1) 1261 return -EINVAL; 1262 1263 if (copy_from_user(lbuf, buf, sizeof(lbuf) - 1)) 1264 return -EFAULT; 1265 1266 lbuf[sizeof(lbuf) - 1] = 0; 1267 1268 if (strcmp(lbuf, "go")) 1269 return -EINVAL; 1270 1271 err = mlx5_cmd_exec(dev, dbg->in_msg, dbg->inlen, dbg->out_msg, dbg->outlen); 1272 1273 return err ? err : count; 1274} 1275 1276static const struct file_operations fops = { 1277 .owner = THIS_MODULE, 1278 .open = simple_open, 1279 .write = dbg_write, 1280}; 1281 1282static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size, 1283 u8 token) 1284{ 1285 struct mlx5_cmd_prot_block *block; 1286 struct mlx5_cmd_mailbox *next; 1287 int copy; 1288 1289 if (!to || !from) 1290 return -ENOMEM; 1291 1292 copy = min_t(int, size, sizeof(to->first.data)); 1293 memcpy(to->first.data, from, copy); 1294 size -= copy; 1295 from += copy; 1296 1297 next = to->next; 1298 while (size) { 1299 if (!next) { 1300 /* this is a BUG */ 1301 return -ENOMEM; 1302 } 1303 1304 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE); 1305 block = next->buf; 1306 memcpy(block->data, from, copy); 1307 from += copy; 1308 size -= copy; 1309 block->token = token; 1310 next = next->next; 1311 } 1312 1313 return 0; 1314} 1315 1316static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size) 1317{ 1318 struct mlx5_cmd_prot_block *block; 1319 struct mlx5_cmd_mailbox *next; 1320 int copy; 1321 1322 if (!to || !from) 1323 return -ENOMEM; 1324 1325 copy = min_t(int, size, sizeof(from->first.data)); 1326 memcpy(to, from->first.data, copy); 1327 size -= copy; 1328 to += copy; 1329 1330 next = from->next; 1331 while (size) { 1332 if (!next) { 1333 /* this is a BUG */ 1334 return -ENOMEM; 1335 } 1336 1337 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE); 1338 block = next->buf; 1339 1340 memcpy(to, block->data, copy); 1341 to += copy; 1342 size -= copy; 1343 next = next->next; 1344 } 1345 1346 return 0; 1347} 1348 1349static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev, 1350 gfp_t flags) 1351{ 1352 struct mlx5_cmd_mailbox *mailbox; 1353 1354 mailbox = kmalloc(sizeof(*mailbox), flags); 1355 if (!mailbox) 1356 return ERR_PTR(-ENOMEM); 1357 1358 mailbox->buf = dma_pool_zalloc(dev->cmd.pool, flags, 1359 &mailbox->dma); 1360 if (!mailbox->buf) { 1361 mlx5_core_dbg(dev, "failed allocation\n"); 1362 kfree(mailbox); 1363 return ERR_PTR(-ENOMEM); 1364 } 1365 mailbox->next = NULL; 1366 1367 return mailbox; 1368} 1369 1370static void free_cmd_box(struct mlx5_core_dev *dev, 1371 struct mlx5_cmd_mailbox *mailbox) 1372{ 1373 dma_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma); 1374 kfree(mailbox); 1375} 1376 1377static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev, 1378 gfp_t flags, int size, 1379 u8 token) 1380{ 1381 struct mlx5_cmd_mailbox *tmp, *head = NULL; 1382 struct mlx5_cmd_prot_block *block; 1383 struct mlx5_cmd_msg *msg; 1384 int err; 1385 int n; 1386 int i; 1387 1388 msg = kzalloc(sizeof(*msg), flags); 1389 if (!msg) 1390 return ERR_PTR(-ENOMEM); 1391 1392 msg->len = size; 1393 n = mlx5_calc_cmd_blocks(msg); 1394 1395 for (i = 0; i < n; i++) { 1396 tmp = alloc_cmd_box(dev, flags); 1397 if (IS_ERR(tmp)) { 1398 mlx5_core_warn(dev, "failed allocating block\n"); 1399 err = PTR_ERR(tmp); 1400 goto err_alloc; 1401 } 1402 1403 block = tmp->buf; 1404 tmp->next = head; 1405 block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0); 1406 block->block_num = cpu_to_be32(n - i - 1); 1407 block->token = token; 1408 head = tmp; 1409 } 1410 msg->next = head; 1411 return msg; 1412 1413err_alloc: 1414 while (head) { 1415 tmp = head->next; 1416 free_cmd_box(dev, head); 1417 head = tmp; 1418 } 1419 kfree(msg); 1420 1421 return ERR_PTR(err); 1422} 1423 1424static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev, 1425 struct mlx5_cmd_msg *msg) 1426{ 1427 struct mlx5_cmd_mailbox *head = msg->next; 1428 struct mlx5_cmd_mailbox *next; 1429 1430 while (head) { 1431 next = head->next; 1432 free_cmd_box(dev, head); 1433 head = next; 1434 } 1435 kfree(msg); 1436} 1437 1438static ssize_t data_write(struct file *filp, const char __user *buf, 1439 size_t count, loff_t *pos) 1440{ 1441 struct mlx5_core_dev *dev = filp->private_data; 1442 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1443 void *ptr; 1444 1445 if (*pos != 0) 1446 return -EINVAL; 1447 1448 kfree(dbg->in_msg); 1449 dbg->in_msg = NULL; 1450 dbg->inlen = 0; 1451 ptr = memdup_user(buf, count); 1452 if (IS_ERR(ptr)) 1453 return PTR_ERR(ptr); 1454 dbg->in_msg = ptr; 1455 dbg->inlen = count; 1456 1457 *pos = count; 1458 1459 return count; 1460} 1461 1462static ssize_t data_read(struct file *filp, char __user *buf, size_t count, 1463 loff_t *pos) 1464{ 1465 struct mlx5_core_dev *dev = filp->private_data; 1466 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1467 1468 if (!dbg->out_msg) 1469 return -ENOMEM; 1470 1471 return simple_read_from_buffer(buf, count, pos, dbg->out_msg, 1472 dbg->outlen); 1473} 1474 1475static const struct file_operations dfops = { 1476 .owner = THIS_MODULE, 1477 .open = simple_open, 1478 .write = data_write, 1479 .read = data_read, 1480}; 1481 1482static ssize_t outlen_read(struct file *filp, char __user *buf, size_t count, 1483 loff_t *pos) 1484{ 1485 struct mlx5_core_dev *dev = filp->private_data; 1486 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1487 char outlen[8]; 1488 int err; 1489 1490 err = snprintf(outlen, sizeof(outlen), "%d", dbg->outlen); 1491 if (err < 0) 1492 return err; 1493 1494 return simple_read_from_buffer(buf, count, pos, outlen, err); 1495} 1496 1497static ssize_t outlen_write(struct file *filp, const char __user *buf, 1498 size_t count, loff_t *pos) 1499{ 1500 struct mlx5_core_dev *dev = filp->private_data; 1501 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1502 char outlen_str[8] = {0}; 1503 int outlen; 1504 void *ptr; 1505 int err; 1506 1507 if (*pos != 0 || count > 6) 1508 return -EINVAL; 1509 1510 kfree(dbg->out_msg); 1511 dbg->out_msg = NULL; 1512 dbg->outlen = 0; 1513 1514 if (copy_from_user(outlen_str, buf, count)) 1515 return -EFAULT; 1516 1517 err = sscanf(outlen_str, "%d", &outlen); 1518 if (err != 1) 1519 return -EINVAL; 1520 1521 ptr = kzalloc(outlen, GFP_KERNEL); 1522 if (!ptr) 1523 return -ENOMEM; 1524 1525 dbg->out_msg = ptr; 1526 dbg->outlen = outlen; 1527 1528 *pos = count; 1529 1530 return count; 1531} 1532 1533static const struct file_operations olfops = { 1534 .owner = THIS_MODULE, 1535 .open = simple_open, 1536 .write = outlen_write, 1537 .read = outlen_read, 1538}; 1539 1540static void set_wqname(struct mlx5_core_dev *dev) 1541{ 1542 struct mlx5_cmd *cmd = &dev->cmd; 1543 1544 snprintf(cmd->wq_name, sizeof(cmd->wq_name), "mlx5_cmd_%s", 1545 dev_name(dev->device)); 1546} 1547 1548static void clean_debug_files(struct mlx5_core_dev *dev) 1549{ 1550 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1551 1552 if (!mlx5_debugfs_root) 1553 return; 1554 1555 debugfs_remove_recursive(dbg->dbg_root); 1556} 1557 1558static void create_debugfs_files(struct mlx5_core_dev *dev) 1559{ 1560 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1561 1562 dbg->dbg_root = debugfs_create_dir("cmd", mlx5_debugfs_get_dev_root(dev)); 1563 1564 debugfs_create_file("in", 0400, dbg->dbg_root, dev, &dfops); 1565 debugfs_create_file("out", 0200, dbg->dbg_root, dev, &dfops); 1566 debugfs_create_file("out_len", 0600, dbg->dbg_root, dev, &olfops); 1567 debugfs_create_u8("status", 0600, dbg->dbg_root, &dbg->status); 1568 debugfs_create_file("run", 0200, dbg->dbg_root, dev, &fops); 1569} 1570 1571void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode) 1572{ 1573 struct mlx5_cmd *cmd = &dev->cmd; 1574 int i; 1575 1576 for (i = 0; i < cmd->vars.max_reg_cmds; i++) 1577 down(&cmd->vars.sem); 1578 down(&cmd->vars.pages_sem); 1579 1580 cmd->allowed_opcode = opcode; 1581 1582 up(&cmd->vars.pages_sem); 1583 for (i = 0; i < cmd->vars.max_reg_cmds; i++) 1584 up(&cmd->vars.sem); 1585} 1586 1587static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode) 1588{ 1589 struct mlx5_cmd *cmd = &dev->cmd; 1590 int i; 1591 1592 for (i = 0; i < cmd->vars.max_reg_cmds; i++) 1593 down(&cmd->vars.sem); 1594 down(&cmd->vars.pages_sem); 1595 1596 cmd->mode = mode; 1597 1598 up(&cmd->vars.pages_sem); 1599 for (i = 0; i < cmd->vars.max_reg_cmds; i++) 1600 up(&cmd->vars.sem); 1601} 1602 1603static int cmd_comp_notifier(struct notifier_block *nb, 1604 unsigned long type, void *data) 1605{ 1606 struct mlx5_core_dev *dev; 1607 struct mlx5_cmd *cmd; 1608 struct mlx5_eqe *eqe; 1609 1610 cmd = mlx5_nb_cof(nb, struct mlx5_cmd, nb); 1611 dev = container_of(cmd, struct mlx5_core_dev, cmd); 1612 eqe = data; 1613 1614 mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector), false); 1615 1616 return NOTIFY_OK; 1617} 1618void mlx5_cmd_use_events(struct mlx5_core_dev *dev) 1619{ 1620 MLX5_NB_INIT(&dev->cmd.nb, cmd_comp_notifier, CMD); 1621 mlx5_eq_notifier_register(dev, &dev->cmd.nb); 1622 mlx5_cmd_change_mod(dev, CMD_MODE_EVENTS); 1623} 1624 1625void mlx5_cmd_use_polling(struct mlx5_core_dev *dev) 1626{ 1627 mlx5_cmd_change_mod(dev, CMD_MODE_POLLING); 1628 mlx5_eq_notifier_unregister(dev, &dev->cmd.nb); 1629} 1630 1631static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg) 1632{ 1633 unsigned long flags; 1634 1635 if (msg->parent) { 1636 spin_lock_irqsave(&msg->parent->lock, flags); 1637 list_add_tail(&msg->list, &msg->parent->head); 1638 spin_unlock_irqrestore(&msg->parent->lock, flags); 1639 } else { 1640 mlx5_free_cmd_msg(dev, msg); 1641 } 1642} 1643 1644static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced) 1645{ 1646 struct mlx5_cmd *cmd = &dev->cmd; 1647 struct mlx5_cmd_work_ent *ent; 1648 mlx5_cmd_cbk_t callback; 1649 void *context; 1650 int err; 1651 int i; 1652 s64 ds; 1653 struct mlx5_cmd_stats *stats; 1654 unsigned long flags; 1655 unsigned long vector; 1656 1657 /* there can be at most 32 command queues */ 1658 vector = vec & 0xffffffff; 1659 for (i = 0; i < (1 << cmd->vars.log_sz); i++) { 1660 if (test_bit(i, &vector)) { 1661 ent = cmd->ent_arr[i]; 1662 1663 /* if we already completed the command, ignore it */ 1664 if (!test_and_clear_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, 1665 &ent->state)) { 1666 /* only real completion can free the cmd slot */ 1667 if (!forced) { 1668 mlx5_core_err(dev, "Command completion arrived after timeout (entry idx = %d).\n", 1669 ent->idx); 1670 cmd_ent_put(ent); 1671 } 1672 continue; 1673 } 1674 1675 if (ent->callback && cancel_delayed_work(&ent->cb_timeout_work)) 1676 cmd_ent_put(ent); /* timeout work was canceled */ 1677 1678 if (!forced || /* Real FW completion */ 1679 mlx5_cmd_is_down(dev) || /* No real FW completion is expected */ 1680 !opcode_allowed(cmd, ent->op)) 1681 cmd_ent_put(ent); 1682 1683 ent->ts2 = ktime_get_ns(); 1684 memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out)); 1685 dump_command(dev, ent, 0); 1686 1687 if (vec & MLX5_TRIGGERED_CMD_COMP) 1688 ent->ret = -ENXIO; 1689 1690 if (!ent->ret) { /* Command completed by FW */ 1691 if (!cmd->checksum_disabled) 1692 ent->ret = verify_signature(ent); 1693 1694 ent->status = ent->lay->status_own >> 1; 1695 1696 mlx5_core_dbg(dev, "command completed. ret 0x%x, delivery status %s(0x%x)\n", 1697 ent->ret, deliv_status_to_str(ent->status), ent->status); 1698 } 1699 1700 if (ent->callback) { 1701 ds = ent->ts2 - ent->ts1; 1702 stats = xa_load(&cmd->stats, ent->op); 1703 if (stats) { 1704 spin_lock_irqsave(&stats->lock, flags); 1705 stats->sum += ds; 1706 ++stats->n; 1707 spin_unlock_irqrestore(&stats->lock, flags); 1708 } 1709 1710 callback = ent->callback; 1711 context = ent->context; 1712 err = ent->ret ? : ent->status; 1713 if (err > 0) /* Failed in FW, command didn't execute */ 1714 err = deliv_status_to_err(err); 1715 1716 if (!err) 1717 err = mlx5_copy_from_msg(ent->uout, 1718 ent->out, 1719 ent->uout_size); 1720 1721 mlx5_free_cmd_msg(dev, ent->out); 1722 free_msg(dev, ent->in); 1723 1724 /* final consumer is done, release ent */ 1725 cmd_ent_put(ent); 1726 callback(err, context); 1727 } else { 1728 /* release wait_func() so mlx5_cmd_invoke() 1729 * can make the final ent_put() 1730 */ 1731 complete(&ent->done); 1732 } 1733 } 1734 } 1735} 1736 1737static void mlx5_cmd_trigger_completions(struct mlx5_core_dev *dev) 1738{ 1739 struct mlx5_cmd *cmd = &dev->cmd; 1740 unsigned long bitmask; 1741 unsigned long flags; 1742 u64 vector; 1743 int i; 1744 1745 /* wait for pending handlers to complete */ 1746 mlx5_eq_synchronize_cmd_irq(dev); 1747 spin_lock_irqsave(&dev->cmd.alloc_lock, flags); 1748 vector = ~dev->cmd.vars.bitmask & ((1ul << (1 << dev->cmd.vars.log_sz)) - 1); 1749 if (!vector) 1750 goto no_trig; 1751 1752 bitmask = vector; 1753 /* we must increment the allocated entries refcount before triggering the completions 1754 * to guarantee pending commands will not get freed in the meanwhile. 1755 * For that reason, it also has to be done inside the alloc_lock. 1756 */ 1757 for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz)) 1758 cmd_ent_get(cmd->ent_arr[i]); 1759 vector |= MLX5_TRIGGERED_CMD_COMP; 1760 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); 1761 1762 mlx5_core_dbg(dev, "vector 0x%llx\n", vector); 1763 mlx5_cmd_comp_handler(dev, vector, true); 1764 for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz)) 1765 cmd_ent_put(cmd->ent_arr[i]); 1766 return; 1767 1768no_trig: 1769 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); 1770} 1771 1772void mlx5_cmd_flush(struct mlx5_core_dev *dev) 1773{ 1774 struct mlx5_cmd *cmd = &dev->cmd; 1775 int i; 1776 1777 for (i = 0; i < cmd->vars.max_reg_cmds; i++) { 1778 while (down_trylock(&cmd->vars.sem)) { 1779 mlx5_cmd_trigger_completions(dev); 1780 cond_resched(); 1781 } 1782 } 1783 1784 while (down_trylock(&cmd->vars.pages_sem)) { 1785 mlx5_cmd_trigger_completions(dev); 1786 cond_resched(); 1787 } 1788 1789 /* Unlock cmdif */ 1790 up(&cmd->vars.pages_sem); 1791 for (i = 0; i < cmd->vars.max_reg_cmds; i++) 1792 up(&cmd->vars.sem); 1793} 1794 1795static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size, 1796 gfp_t gfp) 1797{ 1798 struct mlx5_cmd_msg *msg = ERR_PTR(-ENOMEM); 1799 struct cmd_msg_cache *ch = NULL; 1800 struct mlx5_cmd *cmd = &dev->cmd; 1801 int i; 1802 1803 if (in_size <= 16) 1804 goto cache_miss; 1805 1806 for (i = 0; i < dev->profile.num_cmd_caches; i++) { 1807 ch = &cmd->cache[i]; 1808 if (in_size > ch->max_inbox_size) 1809 continue; 1810 spin_lock_irq(&ch->lock); 1811 if (list_empty(&ch->head)) { 1812 spin_unlock_irq(&ch->lock); 1813 continue; 1814 } 1815 msg = list_entry(ch->head.next, typeof(*msg), list); 1816 /* For cached lists, we must explicitly state what is 1817 * the real size 1818 */ 1819 msg->len = in_size; 1820 list_del(&msg->list); 1821 spin_unlock_irq(&ch->lock); 1822 break; 1823 } 1824 1825 if (!IS_ERR(msg)) 1826 return msg; 1827 1828cache_miss: 1829 msg = mlx5_alloc_cmd_msg(dev, gfp, in_size, 0); 1830 return msg; 1831} 1832 1833static int is_manage_pages(void *in) 1834{ 1835 return in_to_opcode(in) == MLX5_CMD_OP_MANAGE_PAGES; 1836} 1837 1838/* Notes: 1839 * 1. Callback functions may not sleep 1840 * 2. Page queue commands do not support asynchrous completion 1841 */ 1842static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, 1843 int out_size, mlx5_cmd_cbk_t callback, void *context, 1844 bool force_polling) 1845{ 1846 struct mlx5_cmd_msg *inb, *outb; 1847 u16 opcode = in_to_opcode(in); 1848 bool throttle_op; 1849 int pages_queue; 1850 gfp_t gfp; 1851 u8 token; 1852 int err; 1853 1854 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, opcode)) 1855 return -ENXIO; 1856 1857 throttle_op = mlx5_cmd_is_throttle_opcode(opcode); 1858 if (throttle_op) { 1859 /* atomic context may not sleep */ 1860 if (callback) 1861 return -EINVAL; 1862 down(&dev->cmd.vars.throttle_sem); 1863 } 1864 1865 pages_queue = is_manage_pages(in); 1866 gfp = callback ? GFP_ATOMIC : GFP_KERNEL; 1867 1868 inb = alloc_msg(dev, in_size, gfp); 1869 if (IS_ERR(inb)) { 1870 err = PTR_ERR(inb); 1871 goto out_up; 1872 } 1873 1874 token = alloc_token(&dev->cmd); 1875 1876 err = mlx5_copy_to_msg(inb, in, in_size, token); 1877 if (err) { 1878 mlx5_core_warn(dev, "err %d\n", err); 1879 goto out_in; 1880 } 1881 1882 outb = mlx5_alloc_cmd_msg(dev, gfp, out_size, token); 1883 if (IS_ERR(outb)) { 1884 err = PTR_ERR(outb); 1885 goto out_in; 1886 } 1887 1888 err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context, 1889 pages_queue, token, force_polling); 1890 if (callback) 1891 return err; 1892 1893 if (err > 0) /* Failed in FW, command didn't execute */ 1894 err = deliv_status_to_err(err); 1895 1896 if (err) 1897 goto out_out; 1898 1899 /* command completed by FW */ 1900 err = mlx5_copy_from_msg(out, outb, out_size); 1901out_out: 1902 mlx5_free_cmd_msg(dev, outb); 1903out_in: 1904 free_msg(dev, inb); 1905out_up: 1906 if (throttle_op) 1907 up(&dev->cmd.vars.throttle_sem); 1908 return err; 1909} 1910 1911static void mlx5_cmd_err_trace(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out) 1912{ 1913 u32 syndrome = MLX5_GET(mbox_out, out, syndrome); 1914 u8 status = MLX5_GET(mbox_out, out, status); 1915 1916 trace_mlx5_cmd(mlx5_command_str(opcode), opcode, op_mod, 1917 cmd_status_str(status), status, syndrome, 1918 cmd_status_to_err(status)); 1919} 1920 1921static void cmd_status_log(struct mlx5_core_dev *dev, u16 opcode, u8 status, 1922 u32 syndrome, int err) 1923{ 1924 const char *namep = mlx5_command_str(opcode); 1925 struct mlx5_cmd_stats *stats; 1926 unsigned long flags; 1927 1928 if (!err || !(strcmp(namep, "unknown command opcode"))) 1929 return; 1930 1931 stats = xa_load(&dev->cmd.stats, opcode); 1932 if (!stats) 1933 return; 1934 spin_lock_irqsave(&stats->lock, flags); 1935 stats->failed++; 1936 if (err < 0) 1937 stats->last_failed_errno = -err; 1938 if (err == -EREMOTEIO) { 1939 stats->failed_mbox_status++; 1940 stats->last_failed_mbox_status = status; 1941 stats->last_failed_syndrome = syndrome; 1942 } 1943 spin_unlock_irqrestore(&stats->lock, flags); 1944} 1945 1946/* preserve -EREMOTEIO for outbox.status != OK, otherwise return err as is */ 1947static int cmd_status_err(struct mlx5_core_dev *dev, int err, u16 opcode, u16 op_mod, void *out) 1948{ 1949 u32 syndrome = MLX5_GET(mbox_out, out, syndrome); 1950 u8 status = MLX5_GET(mbox_out, out, status); 1951 1952 if (err == -EREMOTEIO) /* -EREMOTEIO is preserved */ 1953 err = -EIO; 1954 1955 if (!err && status != MLX5_CMD_STAT_OK) { 1956 err = -EREMOTEIO; 1957 mlx5_cmd_err_trace(dev, opcode, op_mod, out); 1958 } 1959 1960 cmd_status_log(dev, opcode, status, syndrome, err); 1961 return err; 1962} 1963 1964/** 1965 * mlx5_cmd_do - Executes a fw command, wait for completion. 1966 * Unlike mlx5_cmd_exec, this function will not translate or intercept 1967 * outbox.status and will return -EREMOTEIO when 1968 * outbox.status != MLX5_CMD_STAT_OK 1969 * 1970 * @dev: mlx5 core device 1971 * @in: inbox mlx5_ifc command buffer 1972 * @in_size: inbox buffer size 1973 * @out: outbox mlx5_ifc buffer 1974 * @out_size: outbox size 1975 * 1976 * @return: 1977 * -EREMOTEIO : Command executed by FW, outbox.status != MLX5_CMD_STAT_OK. 1978 * Caller must check FW outbox status. 1979 * 0 : Command execution successful, outbox.status == MLX5_CMD_STAT_OK. 1980 * < 0 : Command execution couldn't be performed by firmware or driver 1981 */ 1982int mlx5_cmd_do(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size) 1983{ 1984 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, false); 1985 u16 op_mod = MLX5_GET(mbox_in, in, op_mod); 1986 u16 opcode = in_to_opcode(in); 1987 1988 return cmd_status_err(dev, err, opcode, op_mod, out); 1989} 1990EXPORT_SYMBOL(mlx5_cmd_do); 1991 1992/** 1993 * mlx5_cmd_exec - Executes a fw command, wait for completion 1994 * 1995 * @dev: mlx5 core device 1996 * @in: inbox mlx5_ifc command buffer 1997 * @in_size: inbox buffer size 1998 * @out: outbox mlx5_ifc buffer 1999 * @out_size: outbox size 2000 * 2001 * @return: 0 if no error, FW command execution was successful 2002 * and outbox status is ok. 2003 */ 2004int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, 2005 int out_size) 2006{ 2007 int err = mlx5_cmd_do(dev, in, in_size, out, out_size); 2008 2009 return mlx5_cmd_check(dev, err, in, out); 2010} 2011EXPORT_SYMBOL(mlx5_cmd_exec); 2012 2013/** 2014 * mlx5_cmd_exec_polling - Executes a fw command, poll for completion 2015 * Needed for driver force teardown, when command completion EQ 2016 * will not be available to complete the command 2017 * 2018 * @dev: mlx5 core device 2019 * @in: inbox mlx5_ifc command buffer 2020 * @in_size: inbox buffer size 2021 * @out: outbox mlx5_ifc buffer 2022 * @out_size: outbox size 2023 * 2024 * @return: 0 if no error, FW command execution was successful 2025 * and outbox status is ok. 2026 */ 2027int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size, 2028 void *out, int out_size) 2029{ 2030 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, true); 2031 u16 op_mod = MLX5_GET(mbox_in, in, op_mod); 2032 u16 opcode = in_to_opcode(in); 2033 2034 err = cmd_status_err(dev, err, opcode, op_mod, out); 2035 return mlx5_cmd_check(dev, err, in, out); 2036} 2037EXPORT_SYMBOL(mlx5_cmd_exec_polling); 2038 2039void mlx5_cmd_init_async_ctx(struct mlx5_core_dev *dev, 2040 struct mlx5_async_ctx *ctx) 2041{ 2042 ctx->dev = dev; 2043 /* Starts at 1 to avoid doing wake_up if we are not cleaning up */ 2044 atomic_set(&ctx->num_inflight, 1); 2045 init_completion(&ctx->inflight_done); 2046} 2047EXPORT_SYMBOL(mlx5_cmd_init_async_ctx); 2048 2049/** 2050 * mlx5_cmd_cleanup_async_ctx - Clean up an async_ctx 2051 * @ctx: The ctx to clean 2052 * 2053 * Upon return all callbacks given to mlx5_cmd_exec_cb() have been called. The 2054 * caller must ensure that mlx5_cmd_exec_cb() is not called during or after 2055 * the call mlx5_cleanup_async_ctx(). 2056 */ 2057void mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx *ctx) 2058{ 2059 if (!atomic_dec_and_test(&ctx->num_inflight)) 2060 wait_for_completion(&ctx->inflight_done); 2061} 2062EXPORT_SYMBOL(mlx5_cmd_cleanup_async_ctx); 2063 2064static void mlx5_cmd_exec_cb_handler(int status, void *_work) 2065{ 2066 struct mlx5_async_work *work = _work; 2067 struct mlx5_async_ctx *ctx; 2068 2069 ctx = work->ctx; 2070 status = cmd_status_err(ctx->dev, status, work->opcode, work->op_mod, work->out); 2071 work->user_callback(status, work); 2072 if (atomic_dec_and_test(&ctx->num_inflight)) 2073 complete(&ctx->inflight_done); 2074} 2075 2076int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size, 2077 void *out, int out_size, mlx5_async_cbk_t callback, 2078 struct mlx5_async_work *work) 2079{ 2080 int ret; 2081 2082 work->ctx = ctx; 2083 work->user_callback = callback; 2084 work->opcode = in_to_opcode(in); 2085 work->op_mod = MLX5_GET(mbox_in, in, op_mod); 2086 work->out = out; 2087 if (WARN_ON(!atomic_inc_not_zero(&ctx->num_inflight))) 2088 return -EIO; 2089 ret = cmd_exec(ctx->dev, in, in_size, out, out_size, 2090 mlx5_cmd_exec_cb_handler, work, false); 2091 if (ret && atomic_dec_and_test(&ctx->num_inflight)) 2092 complete(&ctx->inflight_done); 2093 2094 return ret; 2095} 2096EXPORT_SYMBOL(mlx5_cmd_exec_cb); 2097 2098int mlx5_cmd_allow_other_vhca_access(struct mlx5_core_dev *dev, 2099 struct mlx5_cmd_allow_other_vhca_access_attr *attr) 2100{ 2101 u32 out[MLX5_ST_SZ_DW(allow_other_vhca_access_out)] = {}; 2102 u32 in[MLX5_ST_SZ_DW(allow_other_vhca_access_in)] = {}; 2103 void *key; 2104 2105 MLX5_SET(allow_other_vhca_access_in, 2106 in, opcode, MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS); 2107 MLX5_SET(allow_other_vhca_access_in, 2108 in, object_type_to_be_accessed, attr->obj_type); 2109 MLX5_SET(allow_other_vhca_access_in, 2110 in, object_id_to_be_accessed, attr->obj_id); 2111 2112 key = MLX5_ADDR_OF(allow_other_vhca_access_in, in, access_key); 2113 memcpy(key, attr->access_key, sizeof(attr->access_key)); 2114 2115 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 2116} 2117 2118int mlx5_cmd_alias_obj_create(struct mlx5_core_dev *dev, 2119 struct mlx5_cmd_alias_obj_create_attr *alias_attr, 2120 u32 *obj_id) 2121{ 2122 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; 2123 u32 in[MLX5_ST_SZ_DW(create_alias_obj_in)] = {}; 2124 void *param; 2125 void *attr; 2126 void *key; 2127 int ret; 2128 2129 attr = MLX5_ADDR_OF(create_alias_obj_in, in, hdr); 2130 MLX5_SET(general_obj_in_cmd_hdr, 2131 attr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 2132 MLX5_SET(general_obj_in_cmd_hdr, 2133 attr, obj_type, alias_attr->obj_type); 2134 param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param); 2135 MLX5_SET(general_obj_create_param, param, alias_object, 1); 2136 2137 attr = MLX5_ADDR_OF(create_alias_obj_in, in, alias_ctx); 2138 MLX5_SET(alias_context, attr, vhca_id_to_be_accessed, alias_attr->vhca_id); 2139 MLX5_SET(alias_context, attr, object_id_to_be_accessed, alias_attr->obj_id); 2140 2141 key = MLX5_ADDR_OF(alias_context, attr, access_key); 2142 memcpy(key, alias_attr->access_key, sizeof(alias_attr->access_key)); 2143 2144 ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 2145 if (ret) 2146 return ret; 2147 2148 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 2149 2150 return 0; 2151} 2152 2153int mlx5_cmd_alias_obj_destroy(struct mlx5_core_dev *dev, u32 obj_id, 2154 u16 obj_type) 2155{ 2156 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; 2157 u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; 2158 2159 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 2160 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, obj_type); 2161 MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id); 2162 2163 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 2164} 2165 2166static void destroy_msg_cache(struct mlx5_core_dev *dev) 2167{ 2168 struct cmd_msg_cache *ch; 2169 struct mlx5_cmd_msg *msg; 2170 struct mlx5_cmd_msg *n; 2171 int i; 2172 2173 for (i = 0; i < dev->profile.num_cmd_caches; i++) { 2174 ch = &dev->cmd.cache[i]; 2175 list_for_each_entry_safe(msg, n, &ch->head, list) { 2176 list_del(&msg->list); 2177 mlx5_free_cmd_msg(dev, msg); 2178 } 2179 } 2180} 2181 2182static unsigned cmd_cache_num_ent[MLX5_NUM_COMMAND_CACHES] = { 2183 512, 32, 16, 8, 2 2184}; 2185 2186static unsigned cmd_cache_ent_size[MLX5_NUM_COMMAND_CACHES] = { 2187 16 + MLX5_CMD_DATA_BLOCK_SIZE, 2188 16 + MLX5_CMD_DATA_BLOCK_SIZE * 2, 2189 16 + MLX5_CMD_DATA_BLOCK_SIZE * 16, 2190 16 + MLX5_CMD_DATA_BLOCK_SIZE * 256, 2191 16 + MLX5_CMD_DATA_BLOCK_SIZE * 512, 2192}; 2193 2194static void create_msg_cache(struct mlx5_core_dev *dev) 2195{ 2196 struct mlx5_cmd *cmd = &dev->cmd; 2197 struct cmd_msg_cache *ch; 2198 struct mlx5_cmd_msg *msg; 2199 int i; 2200 int k; 2201 2202 /* Initialize and fill the caches with initial entries */ 2203 for (k = 0; k < dev->profile.num_cmd_caches; k++) { 2204 ch = &cmd->cache[k]; 2205 spin_lock_init(&ch->lock); 2206 INIT_LIST_HEAD(&ch->head); 2207 ch->num_ent = cmd_cache_num_ent[k]; 2208 ch->max_inbox_size = cmd_cache_ent_size[k]; 2209 for (i = 0; i < ch->num_ent; i++) { 2210 msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL | __GFP_NOWARN, 2211 ch->max_inbox_size, 0); 2212 if (IS_ERR(msg)) 2213 break; 2214 msg->parent = ch; 2215 list_add_tail(&msg->list, &ch->head); 2216 } 2217 } 2218} 2219 2220static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) 2221{ 2222 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, 2223 &cmd->alloc_dma, GFP_KERNEL); 2224 if (!cmd->cmd_alloc_buf) 2225 return -ENOMEM; 2226 2227 /* make sure it is aligned to 4K */ 2228 if (!((uintptr_t)cmd->cmd_alloc_buf & (MLX5_ADAPTER_PAGE_SIZE - 1))) { 2229 cmd->cmd_buf = cmd->cmd_alloc_buf; 2230 cmd->dma = cmd->alloc_dma; 2231 cmd->alloc_size = MLX5_ADAPTER_PAGE_SIZE; 2232 return 0; 2233 } 2234 2235 dma_free_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, cmd->cmd_alloc_buf, 2236 cmd->alloc_dma); 2237 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), 2238 2 * MLX5_ADAPTER_PAGE_SIZE - 1, 2239 &cmd->alloc_dma, GFP_KERNEL); 2240 if (!cmd->cmd_alloc_buf) 2241 return -ENOMEM; 2242 2243 cmd->cmd_buf = PTR_ALIGN(cmd->cmd_alloc_buf, MLX5_ADAPTER_PAGE_SIZE); 2244 cmd->dma = ALIGN(cmd->alloc_dma, MLX5_ADAPTER_PAGE_SIZE); 2245 cmd->alloc_size = 2 * MLX5_ADAPTER_PAGE_SIZE - 1; 2246 return 0; 2247} 2248 2249static void free_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) 2250{ 2251 dma_free_coherent(mlx5_core_dma_dev(dev), cmd->alloc_size, cmd->cmd_alloc_buf, 2252 cmd->alloc_dma); 2253} 2254 2255static u16 cmdif_rev(struct mlx5_core_dev *dev) 2256{ 2257 return ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16; 2258} 2259 2260int mlx5_cmd_init(struct mlx5_core_dev *dev) 2261{ 2262 struct mlx5_cmd *cmd = &dev->cmd; 2263 2264 cmd->checksum_disabled = 1; 2265 2266 spin_lock_init(&cmd->alloc_lock); 2267 spin_lock_init(&cmd->token_lock); 2268 2269 set_wqname(dev); 2270 cmd->wq = create_singlethread_workqueue(cmd->wq_name); 2271 if (!cmd->wq) { 2272 mlx5_core_err(dev, "failed to create command workqueue\n"); 2273 return -ENOMEM; 2274 } 2275 2276 mlx5_cmdif_debugfs_init(dev); 2277 2278 return 0; 2279} 2280 2281void mlx5_cmd_cleanup(struct mlx5_core_dev *dev) 2282{ 2283 struct mlx5_cmd *cmd = &dev->cmd; 2284 2285 mlx5_cmdif_debugfs_cleanup(dev); 2286 destroy_workqueue(cmd->wq); 2287} 2288 2289int mlx5_cmd_enable(struct mlx5_core_dev *dev) 2290{ 2291 int size = sizeof(struct mlx5_cmd_prot_block); 2292 int align = roundup_pow_of_two(size); 2293 struct mlx5_cmd *cmd = &dev->cmd; 2294 u32 cmd_h, cmd_l; 2295 int err; 2296 2297 memset(&cmd->vars, 0, sizeof(cmd->vars)); 2298 cmd->vars.cmdif_rev = cmdif_rev(dev); 2299 if (cmd->vars.cmdif_rev != CMD_IF_REV) { 2300 mlx5_core_err(dev, 2301 "Driver cmdif rev(%d) differs from firmware's(%d)\n", 2302 CMD_IF_REV, cmd->vars.cmdif_rev); 2303 return -EINVAL; 2304 } 2305 2306 cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff; 2307 cmd->vars.log_sz = cmd_l >> 4 & 0xf; 2308 cmd->vars.log_stride = cmd_l & 0xf; 2309 if (1 << cmd->vars.log_sz > MLX5_MAX_COMMANDS) { 2310 mlx5_core_err(dev, "firmware reports too many outstanding commands %d\n", 2311 1 << cmd->vars.log_sz); 2312 return -EINVAL; 2313 } 2314 2315 if (cmd->vars.log_sz + cmd->vars.log_stride > MLX5_ADAPTER_PAGE_SHIFT) { 2316 mlx5_core_err(dev, "command queue size overflow\n"); 2317 return -EINVAL; 2318 } 2319 2320 cmd->state = MLX5_CMDIF_STATE_DOWN; 2321 cmd->vars.max_reg_cmds = (1 << cmd->vars.log_sz) - 1; 2322 cmd->vars.bitmask = (1UL << cmd->vars.max_reg_cmds) - 1; 2323 2324 sema_init(&cmd->vars.sem, cmd->vars.max_reg_cmds); 2325 sema_init(&cmd->vars.pages_sem, 1); 2326 sema_init(&cmd->vars.throttle_sem, DIV_ROUND_UP(cmd->vars.max_reg_cmds, 2)); 2327 2328 cmd->pool = dma_pool_create("mlx5_cmd", mlx5_core_dma_dev(dev), size, align, 0); 2329 if (!cmd->pool) 2330 return -ENOMEM; 2331 2332 err = alloc_cmd_page(dev, cmd); 2333 if (err) 2334 goto err_free_pool; 2335 2336 cmd_h = (u32)((u64)(cmd->dma) >> 32); 2337 cmd_l = (u32)(cmd->dma); 2338 if (cmd_l & 0xfff) { 2339 mlx5_core_err(dev, "invalid command queue address\n"); 2340 err = -ENOMEM; 2341 goto err_cmd_page; 2342 } 2343 2344 iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h); 2345 iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz); 2346 2347 /* Make sure firmware sees the complete address before we proceed */ 2348 wmb(); 2349 2350 mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma)); 2351 2352 cmd->mode = CMD_MODE_POLLING; 2353 cmd->allowed_opcode = CMD_ALLOWED_OPCODE_ALL; 2354 2355 create_msg_cache(dev); 2356 create_debugfs_files(dev); 2357 2358 return 0; 2359 2360err_cmd_page: 2361 free_cmd_page(dev, cmd); 2362err_free_pool: 2363 dma_pool_destroy(cmd->pool); 2364 return err; 2365} 2366 2367void mlx5_cmd_disable(struct mlx5_core_dev *dev) 2368{ 2369 struct mlx5_cmd *cmd = &dev->cmd; 2370 2371 flush_workqueue(cmd->wq); 2372 clean_debug_files(dev); 2373 destroy_msg_cache(dev); 2374 free_cmd_page(dev, cmd); 2375 dma_pool_destroy(cmd->pool); 2376} 2377 2378void mlx5_cmd_set_state(struct mlx5_core_dev *dev, 2379 enum mlx5_cmdif_state cmdif_state) 2380{ 2381 dev->cmd.state = cmdif_state; 2382} 2383