1/* 2 * Copyright (c) 2004 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. 4 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35#include <linux/err.h> 36#include <linux/errno.h> 37#include <linux/module.h> 38#include <linux/slab.h> 39#include <linux/kernel.h> 40#include <linux/vmalloc.h> 41 42#include <linux/mlx4/cmd.h> 43 44#include <linux/math64.h> 45 46#include "mlx4.h" 47#include "icm.h" 48 49static u32 mlx4_buddy_alloc(struct mlx4_buddy *buddy, int order) 50{ 51 int o; 52 int m; 53 u32 seg; 54 55 spin_lock(&buddy->lock); 56 57 for (o = order; o <= buddy->max_order; ++o) 58 if (buddy->num_free[o]) { 59 m = 1 << (buddy->max_order - o); 60 seg = find_first_bit(buddy->bits[o], m); 61 if (seg < m) 62 goto found; 63 } 64 65 spin_unlock(&buddy->lock); 66 return -1; 67 68 found: 69 clear_bit(seg, buddy->bits[o]); 70 --buddy->num_free[o]; 71 72 while (o > order) { 73 --o; 74 seg <<= 1; 75 set_bit(seg ^ 1, buddy->bits[o]); 76 ++buddy->num_free[o]; 77 } 78 79 spin_unlock(&buddy->lock); 80 81 seg <<= order; 82 83 return seg; 84} 85 86static void mlx4_buddy_free(struct mlx4_buddy *buddy, u32 seg, int order) 87{ 88 seg >>= order; 89 90 spin_lock(&buddy->lock); 91 92 while (test_bit(seg ^ 1, buddy->bits[order])) { 93 clear_bit(seg ^ 1, buddy->bits[order]); 94 --buddy->num_free[order]; 95 seg >>= 1; 96 ++order; 97 } 98 99 set_bit(seg, buddy->bits[order]); 100 ++buddy->num_free[order]; 101 102 spin_unlock(&buddy->lock); 103} 104 105static int mlx4_buddy_init(struct mlx4_buddy *buddy, int max_order) 106{ 107 int i, s; 108 109 buddy->max_order = max_order; 110 spin_lock_init(&buddy->lock); 111 112 buddy->bits = kcalloc(buddy->max_order + 1, sizeof (long *), 113 GFP_KERNEL); 114 buddy->num_free = kcalloc((buddy->max_order + 1), sizeof *buddy->num_free, 115 GFP_KERNEL); 116 if (!buddy->bits || !buddy->num_free) 117 goto err_out; 118 119 for (i = 0; i <= buddy->max_order; ++i) { 120 s = BITS_TO_LONGS(1 << (buddy->max_order - i)); 121 buddy->bits[i] = kcalloc(s, sizeof (long), GFP_KERNEL | __GFP_NOWARN); 122 if (!buddy->bits[i]) 123 goto err_out_free; 124 } 125 126 set_bit(0, buddy->bits[buddy->max_order]); 127 buddy->num_free[buddy->max_order] = 1; 128 129 return 0; 130 131err_out_free: 132 for (i = 0; i <= buddy->max_order; ++i) 133 kfree(buddy->bits[i]); 134 135err_out: 136 kfree(buddy->bits); 137 kfree(buddy->num_free); 138 139 return -ENOMEM; 140} 141 142static void mlx4_buddy_cleanup(struct mlx4_buddy *buddy) 143{ 144 int i; 145 146 for (i = 0; i <= buddy->max_order; ++i) 147 kfree(buddy->bits[i]); 148 149 kfree(buddy->bits); 150 kfree(buddy->num_free); 151} 152 153u32 __mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) 154{ 155 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 156 u32 seg; 157 int seg_order; 158 u32 offset; 159 160 seg_order = max_t(int, order - log_mtts_per_seg, 0); 161 162 seg = mlx4_buddy_alloc(&mr_table->mtt_buddy, seg_order); 163 if (seg == -1) 164 return -1; 165 166 offset = seg * (1 << log_mtts_per_seg); 167 168 if (mlx4_table_get_range(dev, &mr_table->mtt_table, offset, 169 offset + (1 << order) - 1)) { 170 mlx4_buddy_free(&mr_table->mtt_buddy, seg, seg_order); 171 return -1; 172 } 173 174 return offset; 175} 176 177static u32 mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) 178{ 179 u64 in_param = 0; 180 u64 out_param; 181 int err; 182 183 if (mlx4_is_mfunc(dev)) { 184 set_param_l(&in_param, order); 185 err = mlx4_cmd_imm(dev, in_param, &out_param, RES_MTT, 186 RES_OP_RESERVE_AND_MAP, 187 MLX4_CMD_ALLOC_RES, 188 MLX4_CMD_TIME_CLASS_A, 189 MLX4_CMD_WRAPPED); 190 if (err) 191 return -1; 192 return get_param_l(&out_param); 193 } 194 return __mlx4_alloc_mtt_range(dev, order); 195} 196 197int mlx4_mtt_init(struct mlx4_dev *dev, int npages, int page_shift, 198 struct mlx4_mtt *mtt) 199{ 200 int i; 201 202 if (!npages) { 203 mtt->order = -1; 204 mtt->page_shift = MLX4_ICM_PAGE_SHIFT; 205 return 0; 206 } else 207 mtt->page_shift = page_shift; 208 209 for (mtt->order = 0, i = 1; i < npages; i <<= 1) 210 ++mtt->order; 211 212 mtt->offset = mlx4_alloc_mtt_range(dev, mtt->order); 213 if (mtt->offset == -1) { 214 mlx4_err(dev, "Failed to allocate mtts for %d pages(order %d)\n", 215 npages, mtt->order); 216 return -ENOMEM; 217 } 218 219 return 0; 220} 221EXPORT_SYMBOL_GPL(mlx4_mtt_init); 222 223void __mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) 224{ 225 u32 first_seg; 226 int seg_order; 227 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 228 229 seg_order = max_t(int, order - log_mtts_per_seg, 0); 230 first_seg = offset / (1 << log_mtts_per_seg); 231 232 mlx4_buddy_free(&mr_table->mtt_buddy, first_seg, seg_order); 233 mlx4_table_put_range(dev, &mr_table->mtt_table, offset, 234 offset + (1 << order) - 1); 235} 236 237static void mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) 238{ 239 u64 in_param = 0; 240 int err; 241 242 if (mlx4_is_mfunc(dev)) { 243 set_param_l(&in_param, offset); 244 set_param_h(&in_param, order); 245 err = mlx4_cmd(dev, in_param, RES_MTT, RES_OP_RESERVE_AND_MAP, 246 MLX4_CMD_FREE_RES, 247 MLX4_CMD_TIME_CLASS_A, 248 MLX4_CMD_WRAPPED); 249 if (err) 250 mlx4_warn(dev, "Failed to free mtt range at:" 251 "%d order:%d\n", offset, order); 252 return; 253 } 254 __mlx4_free_mtt_range(dev, offset, order); 255} 256 257void mlx4_mtt_cleanup(struct mlx4_dev *dev, struct mlx4_mtt *mtt) 258{ 259 if (mtt->order < 0) 260 return; 261 262 mlx4_free_mtt_range(dev, mtt->offset, mtt->order); 263} 264EXPORT_SYMBOL_GPL(mlx4_mtt_cleanup); 265 266u64 mlx4_mtt_addr(struct mlx4_dev *dev, struct mlx4_mtt *mtt) 267{ 268 return (u64) mtt->offset * dev->caps.mtt_entry_sz; 269} 270EXPORT_SYMBOL_GPL(mlx4_mtt_addr); 271 272static u32 hw_index_to_key(u32 ind) 273{ 274 return (ind >> 24) | (ind << 8); 275} 276 277static u32 key_to_hw_index(u32 key) 278{ 279 return (key << 24) | (key >> 8); 280} 281 282static int mlx4_SW2HW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 283 int mpt_index) 284{ 285 return mlx4_cmd(dev, mailbox->dma, mpt_index, 286 0, MLX4_CMD_SW2HW_MPT, MLX4_CMD_TIME_CLASS_B, 287 MLX4_CMD_WRAPPED); 288} 289 290static int mlx4_HW2SW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 291 int mpt_index) 292{ 293 return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, mpt_index, 294 !mailbox, MLX4_CMD_HW2SW_MPT, 295 MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); 296} 297 298static int mlx4_mr_alloc_reserved(struct mlx4_dev *dev, u32 mridx, u32 pd, 299 u64 iova, u64 size, u32 access, int npages, 300 int page_shift, struct mlx4_mr *mr) 301{ 302 mr->iova = iova; 303 mr->size = size; 304 mr->pd = pd; 305 mr->access = access; 306 mr->enabled = MLX4_MPT_DISABLED; 307 mr->key = hw_index_to_key(mridx); 308 309 return mlx4_mtt_init(dev, npages, page_shift, &mr->mtt); 310} 311 312static int mlx4_WRITE_MTT(struct mlx4_dev *dev, 313 struct mlx4_cmd_mailbox *mailbox, 314 int num_entries) 315{ 316 return mlx4_cmd(dev, mailbox->dma, num_entries, 0, MLX4_CMD_WRITE_MTT, 317 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); 318} 319 320int __mlx4_mpt_reserve(struct mlx4_dev *dev) 321{ 322 struct mlx4_priv *priv = mlx4_priv(dev); 323 324 return mlx4_bitmap_alloc(&priv->mr_table.mpt_bitmap); 325} 326 327static int mlx4_mpt_reserve(struct mlx4_dev *dev) 328{ 329 u64 out_param; 330 331 if (mlx4_is_mfunc(dev)) { 332 if (mlx4_cmd_imm(dev, 0, &out_param, RES_MPT, RES_OP_RESERVE, 333 MLX4_CMD_ALLOC_RES, 334 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) 335 return -1; 336 return get_param_l(&out_param); 337 } 338 return __mlx4_mpt_reserve(dev); 339} 340 341void __mlx4_mpt_release(struct mlx4_dev *dev, u32 index) 342{ 343 struct mlx4_priv *priv = mlx4_priv(dev); 344 345 mlx4_bitmap_free(&priv->mr_table.mpt_bitmap, index, MLX4_NO_RR); 346} 347 348static void mlx4_mpt_release(struct mlx4_dev *dev, u32 index) 349{ 350 u64 in_param = 0; 351 352 if (mlx4_is_mfunc(dev)) { 353 set_param_l(&in_param, index); 354 if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_RESERVE, 355 MLX4_CMD_FREE_RES, 356 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) 357 mlx4_warn(dev, "Failed to release mr index:%d\n", 358 index); 359 return; 360 } 361 __mlx4_mpt_release(dev, index); 362} 363 364int __mlx4_mpt_alloc_icm(struct mlx4_dev *dev, u32 index) 365{ 366 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 367 368 return mlx4_table_get(dev, &mr_table->dmpt_table, index); 369} 370 371static int mlx4_mpt_alloc_icm(struct mlx4_dev *dev, u32 index) 372{ 373 u64 param = 0; 374 375 if (mlx4_is_mfunc(dev)) { 376 set_param_l(¶m, index); 377 return mlx4_cmd_imm(dev, param, ¶m, RES_MPT, RES_OP_MAP_ICM, 378 MLX4_CMD_ALLOC_RES, 379 MLX4_CMD_TIME_CLASS_A, 380 MLX4_CMD_WRAPPED); 381 } 382 return __mlx4_mpt_alloc_icm(dev, index); 383} 384 385void __mlx4_mpt_free_icm(struct mlx4_dev *dev, u32 index) 386{ 387 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 388 389 mlx4_table_put(dev, &mr_table->dmpt_table, index); 390} 391 392static void mlx4_mpt_free_icm(struct mlx4_dev *dev, u32 index) 393{ 394 u64 in_param = 0; 395 396 if (mlx4_is_mfunc(dev)) { 397 set_param_l(&in_param, index); 398 if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_MAP_ICM, 399 MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, 400 MLX4_CMD_WRAPPED)) 401 mlx4_warn(dev, "Failed to free icm of mr index:%d\n", 402 index); 403 return; 404 } 405 return __mlx4_mpt_free_icm(dev, index); 406} 407 408int mlx4_mr_alloc(struct mlx4_dev *dev, u32 pd, u64 iova, u64 size, u32 access, 409 int npages, int page_shift, struct mlx4_mr *mr) 410{ 411 u32 index; 412 int err; 413 414 index = mlx4_mpt_reserve(dev); 415 if (index == -1) 416 return -ENOMEM; 417 418 err = mlx4_mr_alloc_reserved(dev, index, pd, iova, size, 419 access, npages, page_shift, mr); 420 if (err) 421 mlx4_mpt_release(dev, index); 422 423 return err; 424} 425EXPORT_SYMBOL_GPL(mlx4_mr_alloc); 426 427static int mlx4_mr_free_reserved(struct mlx4_dev *dev, struct mlx4_mr *mr) 428{ 429 int err; 430 431 if (mr->enabled == MLX4_MPT_EN_HW) { 432 err = mlx4_HW2SW_MPT(dev, NULL, 433 key_to_hw_index(mr->key) & 434 (dev->caps.num_mpts - 1)); 435 if (err) { 436 mlx4_warn(dev, "HW2SW_MPT failed (%d).", err); 437 mlx4_warn(dev, "Most likely the MR has MWs bound to it.\n"); 438 return err; 439 } 440 441 mr->enabled = MLX4_MPT_EN_SW; 442 } 443 mlx4_mtt_cleanup(dev, &mr->mtt); 444 445 return 0; 446} 447 448int mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr) 449{ 450 int ret; 451 452 ret = mlx4_mr_free_reserved(dev, mr); 453 if (ret) 454 return ret; 455 if (mr->enabled) 456 mlx4_mpt_free_icm(dev, key_to_hw_index(mr->key)); 457 mlx4_mpt_release(dev, key_to_hw_index(mr->key)); 458 459 return 0; 460} 461EXPORT_SYMBOL_GPL(mlx4_mr_free); 462 463int mlx4_mr_enable(struct mlx4_dev *dev, struct mlx4_mr *mr) 464{ 465 struct mlx4_cmd_mailbox *mailbox; 466 struct mlx4_mpt_entry *mpt_entry; 467 int err; 468 469 err = mlx4_mpt_alloc_icm(dev, key_to_hw_index(mr->key)); 470 if (err) 471 return err; 472 473 mailbox = mlx4_alloc_cmd_mailbox(dev); 474 if (IS_ERR(mailbox)) { 475 err = PTR_ERR(mailbox); 476 goto err_table; 477 } 478 mpt_entry = mailbox->buf; 479 480 memset(mpt_entry, 0, sizeof *mpt_entry); 481 482 mpt_entry->flags = cpu_to_be32(MLX4_MPT_FLAG_MIO | 483 MLX4_MPT_FLAG_REGION | 484 mr->access); 485 486 mpt_entry->key = cpu_to_be32(key_to_hw_index(mr->key)); 487 mpt_entry->pd_flags = cpu_to_be32(mr->pd | MLX4_MPT_PD_FLAG_EN_INV); 488 mpt_entry->start = cpu_to_be64(mr->iova); 489 mpt_entry->length = cpu_to_be64(mr->size); 490 mpt_entry->entity_size = cpu_to_be32(mr->mtt.page_shift); 491 492 if (mr->mtt.order < 0) { 493 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_PHYSICAL); 494 mpt_entry->mtt_addr = 0; 495 } else { 496 mpt_entry->mtt_addr = cpu_to_be64(mlx4_mtt_addr(dev, 497 &mr->mtt)); 498 } 499 500 if (mr->mtt.order >= 0 && mr->mtt.page_shift == 0) { 501 /* fast register MR in free state */ 502 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_FREE); 503 mpt_entry->pd_flags |= cpu_to_be32(MLX4_MPT_PD_FLAG_FAST_REG | 504 MLX4_MPT_PD_FLAG_RAE); 505 mpt_entry->mtt_sz = cpu_to_be32(1 << mr->mtt.order); 506 } else { 507 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_SW_OWNS); 508 } 509 510 err = mlx4_SW2HW_MPT(dev, mailbox, 511 key_to_hw_index(mr->key) & (dev->caps.num_mpts - 1)); 512 if (err) { 513 mlx4_warn(dev, "SW2HW_MPT failed (%d)\n", err); 514 goto err_cmd; 515 } 516 mr->enabled = MLX4_MPT_EN_HW; 517 518 mlx4_free_cmd_mailbox(dev, mailbox); 519 520 return 0; 521 522err_cmd: 523 mlx4_free_cmd_mailbox(dev, mailbox); 524 525err_table: 526 mlx4_mpt_free_icm(dev, key_to_hw_index(mr->key)); 527 return err; 528} 529EXPORT_SYMBOL_GPL(mlx4_mr_enable); 530 531static int mlx4_write_mtt_chunk(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 532 int start_index, int npages, u64 *page_list) 533{ 534 struct mlx4_priv *priv = mlx4_priv(dev); 535 __be64 *mtts; 536 dma_addr_t dma_handle; 537 int i; 538 539 mtts = mlx4_table_find(&priv->mr_table.mtt_table, mtt->offset + 540 start_index, &dma_handle); 541 542 if (!mtts) 543 return -ENOMEM; 544 545 dma_sync_single_for_cpu(&dev->pdev->dev, dma_handle, 546 npages * sizeof (u64), DMA_TO_DEVICE); 547 548 for (i = 0; i < npages; ++i) 549 mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); 550 551 dma_sync_single_for_device(&dev->pdev->dev, dma_handle, 552 npages * sizeof (u64), DMA_TO_DEVICE); 553 554 return 0; 555} 556 557int __mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 558 int start_index, int npages, u64 *page_list) 559{ 560 int err = 0; 561 int chunk; 562 int mtts_per_page; 563 int max_mtts_first_page; 564 565 /* compute how may mtts fit in the first page */ 566 mtts_per_page = PAGE_SIZE / sizeof(u64); 567 max_mtts_first_page = mtts_per_page - (mtt->offset + start_index) 568 % mtts_per_page; 569 570 chunk = min_t(int, max_mtts_first_page, npages); 571 572 while (npages > 0) { 573 err = mlx4_write_mtt_chunk(dev, mtt, start_index, chunk, page_list); 574 if (err) 575 return err; 576 npages -= chunk; 577 start_index += chunk; 578 page_list += chunk; 579 580 chunk = min_t(int, mtts_per_page, npages); 581 } 582 return err; 583} 584 585int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 586 int start_index, int npages, u64 *page_list) 587{ 588 struct mlx4_cmd_mailbox *mailbox = NULL; 589 __be64 *inbox = NULL; 590 int chunk; 591 int err = 0; 592 int i; 593 594 if (mtt->order < 0) 595 return -EINVAL; 596 597 if (mlx4_is_mfunc(dev)) { 598 mailbox = mlx4_alloc_cmd_mailbox(dev); 599 if (IS_ERR(mailbox)) 600 return PTR_ERR(mailbox); 601 inbox = mailbox->buf; 602 603 while (npages > 0) { 604 chunk = min_t(int, MLX4_MAILBOX_SIZE / sizeof(u64) - 2, 605 npages); 606 inbox[0] = cpu_to_be64(mtt->offset + start_index); 607 inbox[1] = 0; 608 for (i = 0; i < chunk; ++i) 609 inbox[i + 2] = cpu_to_be64(page_list[i] | 610 MLX4_MTT_FLAG_PRESENT); 611 err = mlx4_WRITE_MTT(dev, mailbox, chunk); 612 if (err) { 613 mlx4_free_cmd_mailbox(dev, mailbox); 614 return err; 615 } 616 617 npages -= chunk; 618 start_index += chunk; 619 page_list += chunk; 620 } 621 mlx4_free_cmd_mailbox(dev, mailbox); 622 return err; 623 } 624 625 return __mlx4_write_mtt(dev, mtt, start_index, npages, page_list); 626} 627EXPORT_SYMBOL_GPL(mlx4_write_mtt); 628 629int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 630 struct mlx4_buf *buf) 631{ 632 u64 *page_list; 633 int err; 634 int i; 635 636 page_list = kmalloc(buf->npages * sizeof *page_list, GFP_KERNEL); 637 if (!page_list) 638 return -ENOMEM; 639 640 for (i = 0; i < buf->npages; ++i) 641 if (buf->nbufs == 1) 642 page_list[i] = buf->direct.map + (i << buf->page_shift); 643 else 644 page_list[i] = buf->page_list[i].map; 645 646 err = mlx4_write_mtt(dev, mtt, 0, buf->npages, page_list); 647 648 kfree(page_list); 649 return err; 650} 651EXPORT_SYMBOL_GPL(mlx4_buf_write_mtt); 652 653int mlx4_mw_alloc(struct mlx4_dev *dev, u32 pd, enum mlx4_mw_type type, 654 struct mlx4_mw *mw) 655{ 656 u32 index; 657 658 index = mlx4_mpt_reserve(dev); 659 if (index == -1) 660 return -ENOMEM; 661 662 mw->key = hw_index_to_key(index); 663 mw->pd = pd; 664 mw->type = type; 665 mw->enabled = MLX4_MPT_DISABLED; 666 667 return 0; 668} 669EXPORT_SYMBOL_GPL(mlx4_mw_alloc); 670 671int mlx4_mw_enable(struct mlx4_dev *dev, struct mlx4_mw *mw) 672{ 673 struct mlx4_cmd_mailbox *mailbox; 674 struct mlx4_mpt_entry *mpt_entry; 675 int err; 676 677 err = mlx4_mpt_alloc_icm(dev, key_to_hw_index(mw->key)); 678 if (err) 679 return err; 680 681 mailbox = mlx4_alloc_cmd_mailbox(dev); 682 if (IS_ERR(mailbox)) { 683 err = PTR_ERR(mailbox); 684 goto err_table; 685 } 686 mpt_entry = mailbox->buf; 687 688 memset(mpt_entry, 0, sizeof(*mpt_entry)); 689 690 /* Note that the MLX4_MPT_FLAG_REGION bit in mpt_entry->flags is turned 691 * off, thus creating a memory window and not a memory region. 692 */ 693 mpt_entry->key = cpu_to_be32(key_to_hw_index(mw->key)); 694 mpt_entry->pd_flags = cpu_to_be32(mw->pd); 695 if (mw->type == MLX4_MW_TYPE_2) { 696 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_FREE); 697 mpt_entry->qpn = cpu_to_be32(MLX4_MPT_QP_FLAG_BOUND_QP); 698 mpt_entry->pd_flags |= cpu_to_be32(MLX4_MPT_PD_FLAG_EN_INV); 699 } 700 701 err = mlx4_SW2HW_MPT(dev, mailbox, 702 key_to_hw_index(mw->key) & 703 (dev->caps.num_mpts - 1)); 704 if (err) { 705 mlx4_warn(dev, "SW2HW_MPT failed (%d)\n", err); 706 goto err_cmd; 707 } 708 mw->enabled = MLX4_MPT_EN_HW; 709 710 mlx4_free_cmd_mailbox(dev, mailbox); 711 712 return 0; 713 714err_cmd: 715 mlx4_free_cmd_mailbox(dev, mailbox); 716 717err_table: 718 mlx4_mpt_free_icm(dev, key_to_hw_index(mw->key)); 719 return err; 720} 721EXPORT_SYMBOL_GPL(mlx4_mw_enable); 722 723void mlx4_mw_free(struct mlx4_dev *dev, struct mlx4_mw *mw) 724{ 725 int err; 726 727 if (mw->enabled == MLX4_MPT_EN_HW) { 728 err = mlx4_HW2SW_MPT(dev, NULL, 729 key_to_hw_index(mw->key) & 730 (dev->caps.num_mpts - 1)); 731 if (err) 732 mlx4_warn(dev, "xxx HW2SW_MPT failed (%d)\n", err); 733 734 mw->enabled = MLX4_MPT_EN_SW; 735 } 736 if (mw->enabled) 737 mlx4_mpt_free_icm(dev, key_to_hw_index(mw->key)); 738 mlx4_mpt_release(dev, key_to_hw_index(mw->key)); 739} 740EXPORT_SYMBOL_GPL(mlx4_mw_free); 741 742int mlx4_init_mr_table(struct mlx4_dev *dev) 743{ 744 struct mlx4_priv *priv = mlx4_priv(dev); 745 struct mlx4_mr_table *mr_table = &priv->mr_table; 746 int err; 747 748 /* Nothing to do for slaves - all MR handling is forwarded 749 * to the master */ 750 if (mlx4_is_slave(dev)) 751 return 0; 752 753 if (!is_power_of_2(dev->caps.num_mpts)) 754 return -EINVAL; 755 756 err = mlx4_bitmap_init(&mr_table->mpt_bitmap, dev->caps.num_mpts, 757 ~0, dev->caps.reserved_mrws, 0); 758 if (err) 759 return err; 760 761 err = mlx4_buddy_init(&mr_table->mtt_buddy, 762 ilog2(div_u64(dev->caps.num_mtts, 763 (1 << log_mtts_per_seg)))); 764 if (err) 765 goto err_buddy; 766 767 if (dev->caps.reserved_mtts) { 768 priv->reserved_mtts = 769 mlx4_alloc_mtt_range(dev, 770 fls(dev->caps.reserved_mtts - 1)); 771 if (priv->reserved_mtts < 0) { 772 mlx4_warn(dev, "MTT table of order %u is too small.\n", 773 mr_table->mtt_buddy.max_order); 774 err = -ENOMEM; 775 goto err_reserve_mtts; 776 } 777 } 778 779 return 0; 780 781err_reserve_mtts: 782 mlx4_buddy_cleanup(&mr_table->mtt_buddy); 783 784err_buddy: 785 mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); 786 787 return err; 788} 789 790void mlx4_cleanup_mr_table(struct mlx4_dev *dev) 791{ 792 struct mlx4_priv *priv = mlx4_priv(dev); 793 struct mlx4_mr_table *mr_table = &priv->mr_table; 794 795 if (mlx4_is_slave(dev)) 796 return; 797 if (priv->reserved_mtts >= 0) 798 mlx4_free_mtt_range(dev, priv->reserved_mtts, 799 fls(dev->caps.reserved_mtts - 1)); 800 mlx4_buddy_cleanup(&mr_table->mtt_buddy); 801 mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); 802} 803 804static inline int mlx4_check_fmr(struct mlx4_fmr *fmr, u64 *page_list, 805 int npages, u64 iova) 806{ 807 int i, page_mask; 808 809 if (npages > fmr->max_pages) 810 return -EINVAL; 811 812 page_mask = (1 << fmr->page_shift) - 1; 813 814 /* We are getting page lists, so va must be page aligned. */ 815 if (iova & page_mask) 816 return -EINVAL; 817 818 /* Trust the user not to pass misaligned data in page_list */ 819 if (0) 820 for (i = 0; i < npages; ++i) { 821 if (page_list[i] & ~page_mask) 822 return -EINVAL; 823 } 824 825 if (fmr->maps >= fmr->max_maps) 826 return -EINVAL; 827 828 return 0; 829} 830 831int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list, 832 int npages, u64 iova, u32 *lkey, u32 *rkey) 833{ 834 u32 key; 835 int i, err; 836 837 err = mlx4_check_fmr(fmr, page_list, npages, iova); 838 if (err) 839 return err; 840 841 ++fmr->maps; 842 843 key = key_to_hw_index(fmr->mr.key); 844 key += dev->caps.num_mpts; 845 *lkey = *rkey = fmr->mr.key = hw_index_to_key(key); 846 847 *(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW; 848 849 /* Make sure MPT status is visible before writing MTT entries */ 850 wmb(); 851 852 dma_sync_single_for_cpu(&dev->pdev->dev, fmr->dma_handle, 853 npages * sizeof(u64), DMA_TO_DEVICE); 854 855 for (i = 0; i < npages; ++i) 856 fmr->mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); 857 858 dma_sync_single_for_device(&dev->pdev->dev, fmr->dma_handle, 859 npages * sizeof(u64), DMA_TO_DEVICE); 860 861 fmr->mpt->key = cpu_to_be32(key); 862 fmr->mpt->lkey = cpu_to_be32(key); 863 fmr->mpt->length = cpu_to_be64(npages * (1ull << fmr->page_shift)); 864 fmr->mpt->start = cpu_to_be64(iova); 865 866 /* Make MTT entries are visible before setting MPT status */ 867 wmb(); 868 869 *(u8 *) fmr->mpt = MLX4_MPT_STATUS_HW; 870 871 /* Make sure MPT status is visible before consumer can use FMR */ 872 wmb(); 873 874 return 0; 875} 876EXPORT_SYMBOL_GPL(mlx4_map_phys_fmr); 877 878int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages, 879 int max_maps, u8 page_shift, struct mlx4_fmr *fmr) 880{ 881 struct mlx4_priv *priv = mlx4_priv(dev); 882 int err = -ENOMEM, ret; 883 884 if (max_maps > dev->caps.max_fmr_maps) 885 return -EINVAL; 886 887 if (page_shift < (ffs(dev->caps.page_size_cap) - 1) || page_shift >= 32) 888 return -EINVAL; 889 890 /* All MTTs must fit in the same page */ 891 if (max_pages * sizeof *fmr->mtts > PAGE_SIZE) 892 return -EINVAL; 893 894 fmr->page_shift = page_shift; 895 fmr->max_pages = max_pages; 896 fmr->max_maps = max_maps; 897 fmr->maps = 0; 898 899 err = mlx4_mr_alloc(dev, pd, 0, 0, access, max_pages, 900 page_shift, &fmr->mr); 901 if (err) 902 return err; 903 904 fmr->mtts = mlx4_table_find(&priv->mr_table.mtt_table, 905 fmr->mr.mtt.offset, 906 &fmr->dma_handle); 907 908 if (!fmr->mtts) { 909 err = -ENOMEM; 910 goto err_free; 911 } 912 913 return 0; 914 915err_free: 916 ret = mlx4_mr_free(dev, &fmr->mr); 917 if (ret) 918 mlx4_err(dev, "Error deregistering MR. The system may have become unstable."); 919 return err; 920} 921EXPORT_SYMBOL_GPL(mlx4_fmr_alloc); 922 923int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr) 924{ 925 struct mlx4_priv *priv = mlx4_priv(dev); 926 int err; 927 928 err = mlx4_mr_enable(dev, &fmr->mr); 929 if (err) 930 return err; 931 932 fmr->mpt = mlx4_table_find(&priv->mr_table.dmpt_table, 933 key_to_hw_index(fmr->mr.key), NULL); 934 if (!fmr->mpt) 935 return -ENOMEM; 936 937 return 0; 938} 939EXPORT_SYMBOL_GPL(mlx4_fmr_enable); 940 941void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr, 942 u32 *lkey, u32 *rkey) 943{ 944 u32 key; 945 946 if (!fmr->maps) 947 return; 948 949 key = key_to_hw_index(fmr->mr.key) & (dev->caps.num_mpts - 1); 950 951 *(u8 *)fmr->mpt = MLX4_MPT_STATUS_SW; 952 953 /* Make sure MPT status is visible before changing MPT fields */ 954 wmb(); 955 956 fmr->mr.key = hw_index_to_key(key); 957 958 fmr->mpt->key = cpu_to_be32(key); 959 fmr->mpt->lkey = cpu_to_be32(key); 960 fmr->mpt->length = 0; 961 fmr->mpt->start = 0; 962 963 /* Make sure MPT data is visible before changing MPT status */ 964 wmb(); 965 966 *(u8 *)fmr->mpt = MLX4_MPT_STATUS_HW; 967 968 /* Make sure MPT satus is visible */ 969 wmb(); 970 971 fmr->maps = 0; 972} 973EXPORT_SYMBOL_GPL(mlx4_fmr_unmap); 974 975int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr) 976{ 977 int ret; 978 979 if (fmr->maps) 980 return -EBUSY; 981 982 ret = mlx4_mr_free(dev, &fmr->mr); 983 if (ret) 984 return ret; 985 fmr->mr.enabled = MLX4_MPT_DISABLED; 986 987 return 0; 988} 989EXPORT_SYMBOL_GPL(mlx4_fmr_free); 990 991int mlx4_SYNC_TPT(struct mlx4_dev *dev) 992{ 993 return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_SYNC_TPT, 1000, 994 MLX4_CMD_NATIVE); 995} 996EXPORT_SYMBOL_GPL(mlx4_SYNC_TPT); 997