1/* 2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <linux/mlx5/driver.h> 34#include "mlx5_core.h" 35#include <linux/mlx5/transobj.h> 36 37int mlx5_core_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn) 38{ 39 u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {}; 40 u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {}; 41 int err; 42 43 MLX5_SET(alloc_transport_domain_in, in, opcode, 44 MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN); 45 46 err = mlx5_cmd_exec_inout(dev, alloc_transport_domain, in, out); 47 if (!err) 48 *tdn = MLX5_GET(alloc_transport_domain_out, out, 49 transport_domain); 50 51 return err; 52} 53EXPORT_SYMBOL(mlx5_core_alloc_transport_domain); 54 55void mlx5_core_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn) 56{ 57 u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {}; 58 59 MLX5_SET(dealloc_transport_domain_in, in, opcode, 60 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN); 61 MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn); 62 mlx5_cmd_exec_in(dev, dealloc_transport_domain, in); 63} 64EXPORT_SYMBOL(mlx5_core_dealloc_transport_domain); 65 66int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rqn) 67{ 68 u32 out[MLX5_ST_SZ_DW(create_rq_out)] = {}; 69 int err; 70 71 MLX5_SET(create_rq_in, in, opcode, MLX5_CMD_OP_CREATE_RQ); 72 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 73 if (!err) 74 *rqn = MLX5_GET(create_rq_out, out, rqn); 75 76 return err; 77} 78EXPORT_SYMBOL(mlx5_core_create_rq); 79 80int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *in) 81{ 82 MLX5_SET(modify_rq_in, in, rqn, rqn); 83 MLX5_SET(modify_rq_in, in, opcode, MLX5_CMD_OP_MODIFY_RQ); 84 85 return mlx5_cmd_exec_in(dev, modify_rq, in); 86} 87EXPORT_SYMBOL(mlx5_core_modify_rq); 88 89void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn) 90{ 91 u32 in[MLX5_ST_SZ_DW(destroy_rq_in)] = {}; 92 93 MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ); 94 MLX5_SET(destroy_rq_in, in, rqn, rqn); 95 mlx5_cmd_exec_in(dev, destroy_rq, in); 96} 97EXPORT_SYMBOL(mlx5_core_destroy_rq); 98 99int mlx5_core_query_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *out) 100{ 101 u32 in[MLX5_ST_SZ_DW(query_rq_in)] = {}; 102 103 MLX5_SET(query_rq_in, in, opcode, MLX5_CMD_OP_QUERY_RQ); 104 MLX5_SET(query_rq_in, in, rqn, rqn); 105 106 return mlx5_cmd_exec_inout(dev, query_rq, in, out); 107} 108EXPORT_SYMBOL(mlx5_core_query_rq); 109 110int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *sqn) 111{ 112 u32 out[MLX5_ST_SZ_DW(create_sq_out)] = {}; 113 int err; 114 115 MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ); 116 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 117 if (!err) 118 *sqn = MLX5_GET(create_sq_out, out, sqn); 119 120 return err; 121} 122 123int mlx5_core_modify_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *in) 124{ 125 MLX5_SET(modify_sq_in, in, sqn, sqn); 126 MLX5_SET(modify_sq_in, in, opcode, MLX5_CMD_OP_MODIFY_SQ); 127 return mlx5_cmd_exec_in(dev, modify_sq, in); 128} 129EXPORT_SYMBOL(mlx5_core_modify_sq); 130 131void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn) 132{ 133 u32 in[MLX5_ST_SZ_DW(destroy_sq_in)] = {}; 134 135 MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ); 136 MLX5_SET(destroy_sq_in, in, sqn, sqn); 137 mlx5_cmd_exec_in(dev, destroy_sq, in); 138} 139 140int mlx5_core_query_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *out) 141{ 142 u32 in[MLX5_ST_SZ_DW(query_sq_in)] = {}; 143 144 MLX5_SET(query_sq_in, in, opcode, MLX5_CMD_OP_QUERY_SQ); 145 MLX5_SET(query_sq_in, in, sqn, sqn); 146 return mlx5_cmd_exec_inout(dev, query_sq, in, out); 147} 148EXPORT_SYMBOL(mlx5_core_query_sq); 149 150int mlx5_core_query_sq_state(struct mlx5_core_dev *dev, u32 sqn, u8 *state) 151{ 152 void *out; 153 void *sqc; 154 int inlen; 155 int err; 156 157 inlen = MLX5_ST_SZ_BYTES(query_sq_out); 158 out = kvzalloc(inlen, GFP_KERNEL); 159 if (!out) 160 return -ENOMEM; 161 162 err = mlx5_core_query_sq(dev, sqn, out); 163 if (err) 164 goto out; 165 166 sqc = MLX5_ADDR_OF(query_sq_out, out, sq_context); 167 *state = MLX5_GET(sqc, sqc, state); 168 169out: 170 kvfree(out); 171 return err; 172} 173EXPORT_SYMBOL_GPL(mlx5_core_query_sq_state); 174 175int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, u32 *tirn) 176{ 177 u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {}; 178 int err; 179 180 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR); 181 err = mlx5_cmd_exec_inout(dev, create_tir, in, out); 182 if (!err) 183 *tirn = MLX5_GET(create_tir_out, out, tirn); 184 185 return err; 186} 187EXPORT_SYMBOL(mlx5_core_create_tir); 188 189int mlx5_core_modify_tir(struct mlx5_core_dev *dev, u32 tirn, u32 *in) 190{ 191 MLX5_SET(modify_tir_in, in, tirn, tirn); 192 MLX5_SET(modify_tir_in, in, opcode, MLX5_CMD_OP_MODIFY_TIR); 193 return mlx5_cmd_exec_in(dev, modify_tir, in); 194} 195 196void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn) 197{ 198 u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {}; 199 200 MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR); 201 MLX5_SET(destroy_tir_in, in, tirn, tirn); 202 mlx5_cmd_exec_in(dev, destroy_tir, in); 203} 204EXPORT_SYMBOL(mlx5_core_destroy_tir); 205 206int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, u32 *tisn) 207{ 208 u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {}; 209 int err; 210 211 MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS); 212 err = mlx5_cmd_exec_inout(dev, create_tis, in, out); 213 if (!err) 214 *tisn = MLX5_GET(create_tis_out, out, tisn); 215 216 return err; 217} 218EXPORT_SYMBOL(mlx5_core_create_tis); 219 220int mlx5_core_modify_tis(struct mlx5_core_dev *dev, u32 tisn, u32 *in) 221{ 222 MLX5_SET(modify_tis_in, in, tisn, tisn); 223 MLX5_SET(modify_tis_in, in, opcode, MLX5_CMD_OP_MODIFY_TIS); 224 225 return mlx5_cmd_exec_in(dev, modify_tis, in); 226} 227EXPORT_SYMBOL(mlx5_core_modify_tis); 228 229void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn) 230{ 231 u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {}; 232 233 MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS); 234 MLX5_SET(destroy_tis_in, in, tisn, tisn); 235 mlx5_cmd_exec_in(dev, destroy_tis, in); 236} 237EXPORT_SYMBOL(mlx5_core_destroy_tis); 238 239int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen, 240 u32 *rqtn) 241{ 242 u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {}; 243 int err; 244 245 MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT); 246 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 247 if (!err) 248 *rqtn = MLX5_GET(create_rqt_out, out, rqtn); 249 250 return err; 251} 252EXPORT_SYMBOL(mlx5_core_create_rqt); 253 254int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in, 255 int inlen) 256{ 257 u32 out[MLX5_ST_SZ_DW(modify_rqt_out)] = {}; 258 259 MLX5_SET(modify_rqt_in, in, rqtn, rqtn); 260 MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT); 261 return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 262} 263 264void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn) 265{ 266 u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {}; 267 268 MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT); 269 MLX5_SET(destroy_rqt_in, in, rqtn, rqtn); 270 mlx5_cmd_exec_in(dev, destroy_rqt, in); 271} 272EXPORT_SYMBOL(mlx5_core_destroy_rqt); 273 274static int mlx5_hairpin_create_rq(struct mlx5_core_dev *mdev, 275 struct mlx5_hairpin_params *params, u32 *rqn) 276{ 277 u32 in[MLX5_ST_SZ_DW(create_rq_in)] = {0}; 278 void *rqc, *wq; 279 280 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 281 wq = MLX5_ADDR_OF(rqc, rqc, wq); 282 283 MLX5_SET(rqc, rqc, hairpin, 1); 284 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); 285 MLX5_SET(rqc, rqc, counter_set_id, params->q_counter); 286 287 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size); 288 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets); 289 290 return mlx5_core_create_rq(mdev, in, MLX5_ST_SZ_BYTES(create_rq_in), rqn); 291} 292 293static int mlx5_hairpin_create_sq(struct mlx5_core_dev *mdev, 294 struct mlx5_hairpin_params *params, u32 *sqn) 295{ 296 u32 in[MLX5_ST_SZ_DW(create_sq_in)] = {0}; 297 void *sqc, *wq; 298 299 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 300 wq = MLX5_ADDR_OF(sqc, sqc, wq); 301 302 MLX5_SET(sqc, sqc, hairpin, 1); 303 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); 304 305 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size); 306 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets); 307 308 return mlx5_core_create_sq(mdev, in, MLX5_ST_SZ_BYTES(create_sq_in), sqn); 309} 310 311static int mlx5_hairpin_create_queues(struct mlx5_hairpin *hp, 312 struct mlx5_hairpin_params *params) 313{ 314 int i, j, err; 315 316 for (i = 0; i < hp->num_channels; i++) { 317 err = mlx5_hairpin_create_rq(hp->func_mdev, params, &hp->rqn[i]); 318 if (err) 319 goto out_err_rq; 320 } 321 322 for (i = 0; i < hp->num_channels; i++) { 323 err = mlx5_hairpin_create_sq(hp->peer_mdev, params, &hp->sqn[i]); 324 if (err) 325 goto out_err_sq; 326 } 327 328 return 0; 329 330out_err_sq: 331 for (j = 0; j < i; j++) 332 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[j]); 333 i = hp->num_channels; 334out_err_rq: 335 for (j = 0; j < i; j++) 336 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[j]); 337 return err; 338} 339 340static void mlx5_hairpin_destroy_queues(struct mlx5_hairpin *hp) 341{ 342 int i; 343 344 for (i = 0; i < hp->num_channels; i++) { 345 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[i]); 346 if (!hp->peer_gone) 347 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]); 348 } 349} 350 351static int mlx5_hairpin_modify_rq(struct mlx5_core_dev *func_mdev, u32 rqn, 352 int curr_state, int next_state, 353 u16 peer_vhca, u32 peer_sq) 354{ 355 u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {}; 356 void *rqc; 357 358 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 359 360 if (next_state == MLX5_RQC_STATE_RDY) { 361 MLX5_SET(rqc, rqc, hairpin_peer_sq, peer_sq); 362 MLX5_SET(rqc, rqc, hairpin_peer_vhca, peer_vhca); 363 } 364 365 MLX5_SET(modify_rq_in, in, rq_state, curr_state); 366 MLX5_SET(rqc, rqc, state, next_state); 367 368 return mlx5_core_modify_rq(func_mdev, rqn, in); 369} 370 371static int mlx5_hairpin_modify_sq(struct mlx5_core_dev *peer_mdev, u32 sqn, 372 int curr_state, int next_state, 373 u16 peer_vhca, u32 peer_rq) 374{ 375 u32 in[MLX5_ST_SZ_DW(modify_sq_in)] = {0}; 376 void *sqc; 377 378 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); 379 380 if (next_state == MLX5_SQC_STATE_RDY) { 381 MLX5_SET(sqc, sqc, hairpin_peer_rq, peer_rq); 382 MLX5_SET(sqc, sqc, hairpin_peer_vhca, peer_vhca); 383 } 384 385 MLX5_SET(modify_sq_in, in, sq_state, curr_state); 386 MLX5_SET(sqc, sqc, state, next_state); 387 388 return mlx5_core_modify_sq(peer_mdev, sqn, in); 389} 390 391static int mlx5_hairpin_pair_queues(struct mlx5_hairpin *hp) 392{ 393 int i, j, err; 394 395 /* set peer SQs */ 396 for (i = 0; i < hp->num_channels; i++) { 397 err = mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], 398 MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY, 399 MLX5_CAP_GEN(hp->func_mdev, vhca_id), hp->rqn[i]); 400 if (err) 401 goto err_modify_sq; 402 } 403 404 /* set func RQs */ 405 for (i = 0; i < hp->num_channels; i++) { 406 err = mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], 407 MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY, 408 MLX5_CAP_GEN(hp->peer_mdev, vhca_id), hp->sqn[i]); 409 if (err) 410 goto err_modify_rq; 411 } 412 413 return 0; 414 415err_modify_rq: 416 for (j = 0; j < i; j++) 417 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[j], MLX5_RQC_STATE_RDY, 418 MLX5_RQC_STATE_RST, 0, 0); 419 i = hp->num_channels; 420err_modify_sq: 421 for (j = 0; j < i; j++) 422 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[j], MLX5_SQC_STATE_RDY, 423 MLX5_SQC_STATE_RST, 0, 0); 424 return err; 425} 426 427static void mlx5_hairpin_unpair_peer_sq(struct mlx5_hairpin *hp) 428{ 429 int i; 430 431 for (i = 0; i < hp->num_channels; i++) 432 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], MLX5_SQC_STATE_RDY, 433 MLX5_SQC_STATE_RST, 0, 0); 434} 435 436static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp) 437{ 438 int i; 439 440 /* unset func RQs */ 441 for (i = 0; i < hp->num_channels; i++) 442 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], MLX5_RQC_STATE_RDY, 443 MLX5_RQC_STATE_RST, 0, 0); 444 /* unset peer SQs */ 445 if (!hp->peer_gone) 446 mlx5_hairpin_unpair_peer_sq(hp); 447} 448 449struct mlx5_hairpin * 450mlx5_core_hairpin_create(struct mlx5_core_dev *func_mdev, 451 struct mlx5_core_dev *peer_mdev, 452 struct mlx5_hairpin_params *params) 453{ 454 struct mlx5_hairpin *hp; 455 int size, err; 456 457 size = sizeof(*hp) + params->num_channels * 2 * sizeof(u32); 458 hp = kzalloc(size, GFP_KERNEL); 459 if (!hp) 460 return ERR_PTR(-ENOMEM); 461 462 hp->func_mdev = func_mdev; 463 hp->peer_mdev = peer_mdev; 464 hp->num_channels = params->num_channels; 465 466 hp->rqn = (void *)hp + sizeof(*hp); 467 hp->sqn = hp->rqn + params->num_channels; 468 469 /* alloc and pair func --> peer hairpin */ 470 err = mlx5_hairpin_create_queues(hp, params); 471 if (err) 472 goto err_create_queues; 473 474 err = mlx5_hairpin_pair_queues(hp); 475 if (err) 476 goto err_pair_queues; 477 478 return hp; 479 480err_pair_queues: 481 mlx5_hairpin_destroy_queues(hp); 482err_create_queues: 483 kfree(hp); 484 return ERR_PTR(err); 485} 486 487void mlx5_core_hairpin_destroy(struct mlx5_hairpin *hp) 488{ 489 mlx5_hairpin_unpair_queues(hp); 490 mlx5_hairpin_destroy_queues(hp); 491 kfree(hp); 492} 493 494void mlx5_core_hairpin_clear_dead_peer(struct mlx5_hairpin *hp) 495{ 496 int i; 497 498 mlx5_hairpin_unpair_peer_sq(hp); 499 500 /* destroy peer SQ */ 501 for (i = 0; i < hp->num_channels; i++) 502 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]); 503 504 hp->peer_gone = true; 505} 506