1219820Sjeff/* 2219820Sjeff * Copyright (c) 2004 Topspin Communications. All rights reserved. 3219820Sjeff * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 4219820Sjeff * 5219820Sjeff * This software is available to you under a choice of one of two 6219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 7219820Sjeff * General Public License (GPL) Version 2, available from the file 8219820Sjeff * COPYING in the main directory of this source tree, or the 9219820Sjeff * OpenIB.org BSD license below: 10219820Sjeff * 11219820Sjeff * Redistribution and use in source and binary forms, with or 12219820Sjeff * without modification, are permitted provided that the following 13219820Sjeff * conditions are met: 14219820Sjeff * 15219820Sjeff * - Redistributions of source code must retain the above 16219820Sjeff * copyright notice, this list of conditions and the following 17219820Sjeff * disclaimer. 18219820Sjeff * 19219820Sjeff * - Redistributions in binary form must reproduce the above 20219820Sjeff * copyright notice, this list of conditions and the following 21219820Sjeff * disclaimer in the documentation and/or other materials 22219820Sjeff * provided with the distribution. 23219820Sjeff * 24219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31219820Sjeff * SOFTWARE. 32219820Sjeff */ 33219820Sjeff 34219820Sjeff#include <linux/slab.h> 35219820Sjeff#include <linux/errno.h> 36219820Sjeff 37219820Sjeff#include "mthca_dev.h" 38219820Sjeff#include "mthca_cmd.h" 39219820Sjeff#include "mthca_memfree.h" 40219820Sjeff 41219820Sjeffstruct mthca_mtt { 42219820Sjeff struct mthca_buddy *buddy; 43219820Sjeff int order; 44219820Sjeff u32 first_seg; 45219820Sjeff}; 46219820Sjeff 47219820Sjeff/* 48219820Sjeff * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits. 49219820Sjeff */ 50219820Sjeffstruct mthca_mpt_entry { 51219820Sjeff __be32 flags; 52219820Sjeff __be32 page_size; 53219820Sjeff __be32 key; 54219820Sjeff __be32 pd; 55219820Sjeff __be64 start; 56219820Sjeff __be64 length; 57219820Sjeff __be32 lkey; 58219820Sjeff __be32 window_count; 59219820Sjeff __be32 window_count_limit; 60219820Sjeff __be64 mtt_seg; 61219820Sjeff __be32 mtt_sz; /* Arbel only */ 62219820Sjeff u32 reserved[2]; 63219820Sjeff} __attribute__((packed)); 64219820Sjeff 65219820Sjeff#define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28) 66219820Sjeff#define MTHCA_MPT_FLAG_MIO (1 << 17) 67219820Sjeff#define MTHCA_MPT_FLAG_BIND_ENABLE (1 << 15) 68219820Sjeff#define MTHCA_MPT_FLAG_PHYSICAL (1 << 9) 69219820Sjeff#define MTHCA_MPT_FLAG_REGION (1 << 8) 70219820Sjeff 71219820Sjeff#define MTHCA_MTT_FLAG_PRESENT 1 72219820Sjeff 73219820Sjeff#define MTHCA_MPT_STATUS_SW 0xF0 74219820Sjeff#define MTHCA_MPT_STATUS_HW 0x00 75219820Sjeff 76219820Sjeff#define SINAI_FMR_KEY_INC 0x1000000 77219820Sjeff 78219820Sjeff/* 79219820Sjeff * Buddy allocator for MTT segments (currently not very efficient 80219820Sjeff * since it doesn't keep a free list and just searches linearly 81219820Sjeff * through the bitmaps) 82219820Sjeff */ 83219820Sjeff 84219820Sjeffstatic u32 mthca_buddy_alloc(struct mthca_buddy *buddy, int order) 85219820Sjeff{ 86219820Sjeff int o; 87219820Sjeff int m; 88219820Sjeff u32 seg; 89219820Sjeff 90219820Sjeff spin_lock(&buddy->lock); 91219820Sjeff 92219820Sjeff for (o = order; o <= buddy->max_order; ++o) 93219820Sjeff if (buddy->num_free[o]) { 94219820Sjeff m = 1 << (buddy->max_order - o); 95219820Sjeff seg = find_first_bit(buddy->bits[o], m); 96219820Sjeff if (seg < m) 97219820Sjeff goto found; 98219820Sjeff } 99219820Sjeff 100219820Sjeff spin_unlock(&buddy->lock); 101219820Sjeff return -1; 102219820Sjeff 103219820Sjeff found: 104219820Sjeff clear_bit(seg, buddy->bits[o]); 105219820Sjeff --buddy->num_free[o]; 106219820Sjeff 107219820Sjeff while (o > order) { 108219820Sjeff --o; 109219820Sjeff seg <<= 1; 110219820Sjeff set_bit(seg ^ 1, buddy->bits[o]); 111219820Sjeff ++buddy->num_free[o]; 112219820Sjeff } 113219820Sjeff 114219820Sjeff spin_unlock(&buddy->lock); 115219820Sjeff 116219820Sjeff seg <<= order; 117219820Sjeff 118219820Sjeff return seg; 119219820Sjeff} 120219820Sjeff 121219820Sjeffstatic void mthca_buddy_free(struct mthca_buddy *buddy, u32 seg, int order) 122219820Sjeff{ 123219820Sjeff seg >>= order; 124219820Sjeff 125219820Sjeff spin_lock(&buddy->lock); 126219820Sjeff 127219820Sjeff while (test_bit(seg ^ 1, buddy->bits[order])) { 128219820Sjeff clear_bit(seg ^ 1, buddy->bits[order]); 129219820Sjeff --buddy->num_free[order]; 130219820Sjeff seg >>= 1; 131219820Sjeff ++order; 132219820Sjeff } 133219820Sjeff 134219820Sjeff set_bit(seg, buddy->bits[order]); 135219820Sjeff ++buddy->num_free[order]; 136219820Sjeff 137219820Sjeff spin_unlock(&buddy->lock); 138219820Sjeff} 139219820Sjeff 140219820Sjeffstatic int mthca_buddy_init(struct mthca_buddy *buddy, int max_order) 141219820Sjeff{ 142219820Sjeff int i, s; 143219820Sjeff 144219820Sjeff buddy->max_order = max_order; 145219820Sjeff spin_lock_init(&buddy->lock); 146219820Sjeff 147219820Sjeff buddy->bits = kzalloc((buddy->max_order + 1) * sizeof (long *), 148219820Sjeff GFP_KERNEL); 149219820Sjeff buddy->num_free = kzalloc((buddy->max_order + 1) * sizeof (int *), 150219820Sjeff GFP_KERNEL); 151219820Sjeff if (!buddy->bits || !buddy->num_free) 152219820Sjeff goto err_out; 153219820Sjeff 154219820Sjeff for (i = 0; i <= buddy->max_order; ++i) { 155219820Sjeff s = BITS_TO_LONGS(1 << (buddy->max_order - i)); 156219820Sjeff buddy->bits[i] = kmalloc(s * sizeof (long), GFP_KERNEL); 157219820Sjeff if (!buddy->bits[i]) 158219820Sjeff goto err_out_free; 159219820Sjeff bitmap_zero(buddy->bits[i], 160219820Sjeff 1 << (buddy->max_order - i)); 161219820Sjeff } 162219820Sjeff 163219820Sjeff set_bit(0, buddy->bits[buddy->max_order]); 164219820Sjeff buddy->num_free[buddy->max_order] = 1; 165219820Sjeff 166219820Sjeff return 0; 167219820Sjeff 168219820Sjefferr_out_free: 169219820Sjeff for (i = 0; i <= buddy->max_order; ++i) 170219820Sjeff kfree(buddy->bits[i]); 171219820Sjeff 172219820Sjefferr_out: 173219820Sjeff kfree(buddy->bits); 174219820Sjeff kfree(buddy->num_free); 175219820Sjeff 176219820Sjeff return -ENOMEM; 177219820Sjeff} 178219820Sjeff 179219820Sjeffstatic void mthca_buddy_cleanup(struct mthca_buddy *buddy) 180219820Sjeff{ 181219820Sjeff int i; 182219820Sjeff 183219820Sjeff for (i = 0; i <= buddy->max_order; ++i) 184219820Sjeff kfree(buddy->bits[i]); 185219820Sjeff 186219820Sjeff kfree(buddy->bits); 187219820Sjeff kfree(buddy->num_free); 188219820Sjeff} 189219820Sjeff 190219820Sjeffstatic u32 mthca_alloc_mtt_range(struct mthca_dev *dev, int order, 191219820Sjeff struct mthca_buddy *buddy) 192219820Sjeff{ 193219820Sjeff u32 seg = mthca_buddy_alloc(buddy, order); 194219820Sjeff 195219820Sjeff if (seg == -1) 196219820Sjeff return -1; 197219820Sjeff 198219820Sjeff if (mthca_is_memfree(dev)) 199219820Sjeff if (mthca_table_get_range(dev, dev->mr_table.mtt_table, seg, 200219820Sjeff seg + (1 << order) - 1)) { 201219820Sjeff mthca_buddy_free(buddy, seg, order); 202219820Sjeff seg = -1; 203219820Sjeff } 204219820Sjeff 205219820Sjeff return seg; 206219820Sjeff} 207219820Sjeff 208219820Sjeffstatic struct mthca_mtt *__mthca_alloc_mtt(struct mthca_dev *dev, int size, 209219820Sjeff struct mthca_buddy *buddy) 210219820Sjeff{ 211219820Sjeff struct mthca_mtt *mtt; 212219820Sjeff int i; 213219820Sjeff 214219820Sjeff if (size <= 0) 215219820Sjeff return ERR_PTR(-EINVAL); 216219820Sjeff 217219820Sjeff mtt = kmalloc(sizeof *mtt, GFP_KERNEL); 218219820Sjeff if (!mtt) 219219820Sjeff return ERR_PTR(-ENOMEM); 220219820Sjeff 221219820Sjeff mtt->buddy = buddy; 222219820Sjeff mtt->order = 0; 223219820Sjeff for (i = dev->limits.mtt_seg_size / 8; i < size; i <<= 1) 224219820Sjeff ++mtt->order; 225219820Sjeff 226219820Sjeff mtt->first_seg = mthca_alloc_mtt_range(dev, mtt->order, buddy); 227219820Sjeff if (mtt->first_seg == -1) { 228219820Sjeff kfree(mtt); 229219820Sjeff return ERR_PTR(-ENOMEM); 230219820Sjeff } 231219820Sjeff 232219820Sjeff return mtt; 233219820Sjeff} 234219820Sjeff 235219820Sjeffstruct mthca_mtt *mthca_alloc_mtt(struct mthca_dev *dev, int size) 236219820Sjeff{ 237219820Sjeff return __mthca_alloc_mtt(dev, size, &dev->mr_table.mtt_buddy); 238219820Sjeff} 239219820Sjeff 240219820Sjeffvoid mthca_free_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt) 241219820Sjeff{ 242219820Sjeff if (!mtt) 243219820Sjeff return; 244219820Sjeff 245219820Sjeff mthca_buddy_free(mtt->buddy, mtt->first_seg, mtt->order); 246219820Sjeff 247219820Sjeff mthca_table_put_range(dev, dev->mr_table.mtt_table, 248219820Sjeff mtt->first_seg, 249219820Sjeff mtt->first_seg + (1 << mtt->order) - 1); 250219820Sjeff 251219820Sjeff kfree(mtt); 252219820Sjeff} 253219820Sjeff 254219820Sjeffstatic int __mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt, 255219820Sjeff int start_index, u64 *buffer_list, int list_len) 256219820Sjeff{ 257219820Sjeff struct mthca_mailbox *mailbox; 258219820Sjeff __be64 *mtt_entry; 259219820Sjeff int err = 0; 260219820Sjeff u8 status; 261219820Sjeff int i; 262219820Sjeff 263219820Sjeff mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); 264219820Sjeff if (IS_ERR(mailbox)) 265219820Sjeff return PTR_ERR(mailbox); 266219820Sjeff mtt_entry = mailbox->buf; 267219820Sjeff 268219820Sjeff while (list_len > 0) { 269219820Sjeff mtt_entry[0] = cpu_to_be64(dev->mr_table.mtt_base + 270219820Sjeff mtt->first_seg * dev->limits.mtt_seg_size + 271219820Sjeff start_index * 8); 272219820Sjeff mtt_entry[1] = 0; 273219820Sjeff for (i = 0; i < list_len && i < MTHCA_MAILBOX_SIZE / 8 - 2; ++i) 274219820Sjeff mtt_entry[i + 2] = cpu_to_be64(buffer_list[i] | 275219820Sjeff MTHCA_MTT_FLAG_PRESENT); 276219820Sjeff 277219820Sjeff /* 278219820Sjeff * If we have an odd number of entries to write, add 279219820Sjeff * one more dummy entry for firmware efficiency. 280219820Sjeff */ 281219820Sjeff if (i & 1) 282219820Sjeff mtt_entry[i + 2] = 0; 283219820Sjeff 284219820Sjeff err = mthca_WRITE_MTT(dev, mailbox, (i + 1) & ~1, &status); 285219820Sjeff if (err) { 286219820Sjeff mthca_warn(dev, "WRITE_MTT failed (%d)\n", err); 287219820Sjeff goto out; 288219820Sjeff } 289219820Sjeff if (status) { 290219820Sjeff mthca_warn(dev, "WRITE_MTT returned status 0x%02x\n", 291219820Sjeff status); 292219820Sjeff err = -EINVAL; 293219820Sjeff goto out; 294219820Sjeff } 295219820Sjeff 296219820Sjeff list_len -= i; 297219820Sjeff start_index += i; 298219820Sjeff buffer_list += i; 299219820Sjeff } 300219820Sjeff 301219820Sjeffout: 302219820Sjeff mthca_free_mailbox(dev, mailbox); 303219820Sjeff return err; 304219820Sjeff} 305219820Sjeff 306219820Sjeffint mthca_write_mtt_size(struct mthca_dev *dev) 307219820Sjeff{ 308219820Sjeff if (dev->mr_table.fmr_mtt_buddy != &dev->mr_table.mtt_buddy || 309219820Sjeff !(dev->mthca_flags & MTHCA_FLAG_FMR)) 310219820Sjeff /* 311219820Sjeff * Be friendly to WRITE_MTT command 312219820Sjeff * and leave two empty slots for the 313219820Sjeff * index and reserved fields of the 314219820Sjeff * mailbox. 315219820Sjeff */ 316219820Sjeff return PAGE_SIZE / sizeof (u64) - 2; 317219820Sjeff 318219820Sjeff /* For Arbel, all MTTs must fit in the same page. */ 319219820Sjeff return mthca_is_memfree(dev) ? (PAGE_SIZE / sizeof (u64)) : 0x7ffffff; 320219820Sjeff} 321219820Sjeff 322219820Sjeffstatic void mthca_tavor_write_mtt_seg(struct mthca_dev *dev, 323219820Sjeff struct mthca_mtt *mtt, int start_index, 324219820Sjeff u64 *buffer_list, int list_len) 325219820Sjeff{ 326219820Sjeff u64 __iomem *mtts; 327219820Sjeff int i; 328219820Sjeff 329219820Sjeff mtts = dev->mr_table.tavor_fmr.mtt_base + mtt->first_seg * dev->limits.mtt_seg_size + 330219820Sjeff start_index * sizeof (u64); 331219820Sjeff for (i = 0; i < list_len; ++i) 332219820Sjeff mthca_write64_raw(cpu_to_be64(buffer_list[i] | MTHCA_MTT_FLAG_PRESENT), 333219820Sjeff mtts + i); 334219820Sjeff} 335219820Sjeff 336219820Sjeffstatic void mthca_arbel_write_mtt_seg(struct mthca_dev *dev, 337219820Sjeff struct mthca_mtt *mtt, int start_index, 338219820Sjeff u64 *buffer_list, int list_len) 339219820Sjeff{ 340219820Sjeff __be64 *mtts; 341219820Sjeff dma_addr_t dma_handle; 342219820Sjeff int i; 343219820Sjeff int s = start_index * sizeof (u64); 344219820Sjeff 345219820Sjeff /* For Arbel, all MTTs must fit in the same page. */ 346219820Sjeff BUG_ON(s / PAGE_SIZE != (s + list_len * sizeof(u64) - 1) / PAGE_SIZE); 347219820Sjeff /* Require full segments */ 348219820Sjeff BUG_ON(s % dev->limits.mtt_seg_size); 349219820Sjeff 350219820Sjeff mtts = mthca_table_find(dev->mr_table.mtt_table, mtt->first_seg + 351219820Sjeff s / dev->limits.mtt_seg_size, &dma_handle); 352219820Sjeff 353219820Sjeff BUG_ON(!mtts); 354219820Sjeff 355219820Sjeff for (i = 0; i < list_len; ++i) 356219820Sjeff mtts[i] = cpu_to_be64(buffer_list[i] | MTHCA_MTT_FLAG_PRESENT); 357219820Sjeff 358219820Sjeff dma_sync_single(&dev->pdev->dev, dma_handle, list_len * sizeof (u64), DMA_TO_DEVICE); 359219820Sjeff} 360219820Sjeff 361219820Sjeffint mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt, 362219820Sjeff int start_index, u64 *buffer_list, int list_len) 363219820Sjeff{ 364219820Sjeff int size = mthca_write_mtt_size(dev); 365219820Sjeff int chunk; 366219820Sjeff 367219820Sjeff if (dev->mr_table.fmr_mtt_buddy != &dev->mr_table.mtt_buddy || 368219820Sjeff !(dev->mthca_flags & MTHCA_FLAG_FMR)) 369219820Sjeff return __mthca_write_mtt(dev, mtt, start_index, buffer_list, list_len); 370219820Sjeff 371219820Sjeff while (list_len > 0) { 372219820Sjeff chunk = min(size, list_len); 373219820Sjeff if (mthca_is_memfree(dev)) 374219820Sjeff mthca_arbel_write_mtt_seg(dev, mtt, start_index, 375219820Sjeff buffer_list, chunk); 376219820Sjeff else 377219820Sjeff mthca_tavor_write_mtt_seg(dev, mtt, start_index, 378219820Sjeff buffer_list, chunk); 379219820Sjeff 380219820Sjeff list_len -= chunk; 381219820Sjeff start_index += chunk; 382219820Sjeff buffer_list += chunk; 383219820Sjeff } 384219820Sjeff 385219820Sjeff return 0; 386219820Sjeff} 387219820Sjeff 388219820Sjeffstatic inline u32 tavor_hw_index_to_key(u32 ind) 389219820Sjeff{ 390219820Sjeff return ind; 391219820Sjeff} 392219820Sjeff 393219820Sjeffstatic inline u32 tavor_key_to_hw_index(u32 key) 394219820Sjeff{ 395219820Sjeff return key; 396219820Sjeff} 397219820Sjeff 398219820Sjeffstatic inline u32 arbel_hw_index_to_key(u32 ind) 399219820Sjeff{ 400219820Sjeff return (ind >> 24) | (ind << 8); 401219820Sjeff} 402219820Sjeff 403219820Sjeffstatic inline u32 arbel_key_to_hw_index(u32 key) 404219820Sjeff{ 405219820Sjeff return (key << 24) | (key >> 8); 406219820Sjeff} 407219820Sjeff 408219820Sjeffstatic inline u32 hw_index_to_key(struct mthca_dev *dev, u32 ind) 409219820Sjeff{ 410219820Sjeff if (mthca_is_memfree(dev)) 411219820Sjeff return arbel_hw_index_to_key(ind); 412219820Sjeff else 413219820Sjeff return tavor_hw_index_to_key(ind); 414219820Sjeff} 415219820Sjeff 416219820Sjeffstatic inline u32 key_to_hw_index(struct mthca_dev *dev, u32 key) 417219820Sjeff{ 418219820Sjeff if (mthca_is_memfree(dev)) 419219820Sjeff return arbel_key_to_hw_index(key); 420219820Sjeff else 421219820Sjeff return tavor_key_to_hw_index(key); 422219820Sjeff} 423219820Sjeff 424219820Sjeffstatic inline u32 adjust_key(struct mthca_dev *dev, u32 key) 425219820Sjeff{ 426219820Sjeff if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT) 427219820Sjeff return ((key << 20) & 0x800000) | (key & 0x7fffff); 428219820Sjeff else 429219820Sjeff return key; 430219820Sjeff} 431219820Sjeff 432219820Sjeffint mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift, 433219820Sjeff u64 iova, u64 total_size, u32 access, struct mthca_mr *mr) 434219820Sjeff{ 435219820Sjeff struct mthca_mailbox *mailbox; 436219820Sjeff struct mthca_mpt_entry *mpt_entry; 437219820Sjeff u32 key; 438219820Sjeff int i; 439219820Sjeff int err; 440219820Sjeff u8 status; 441219820Sjeff 442219820Sjeff WARN_ON(buffer_size_shift >= 32); 443219820Sjeff 444219820Sjeff key = mthca_alloc(&dev->mr_table.mpt_alloc); 445219820Sjeff if (key == -1) 446219820Sjeff return -ENOMEM; 447219820Sjeff key = adjust_key(dev, key); 448219820Sjeff mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key); 449219820Sjeff 450219820Sjeff if (mthca_is_memfree(dev)) { 451219820Sjeff err = mthca_table_get(dev, dev->mr_table.mpt_table, key); 452219820Sjeff if (err) 453219820Sjeff goto err_out_mpt_free; 454219820Sjeff } 455219820Sjeff 456219820Sjeff mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); 457219820Sjeff if (IS_ERR(mailbox)) { 458219820Sjeff err = PTR_ERR(mailbox); 459219820Sjeff goto err_out_table; 460219820Sjeff } 461219820Sjeff mpt_entry = mailbox->buf; 462219820Sjeff 463219820Sjeff mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS | 464219820Sjeff MTHCA_MPT_FLAG_MIO | 465219820Sjeff MTHCA_MPT_FLAG_REGION | 466219820Sjeff access); 467219820Sjeff if (!mr->mtt) 468219820Sjeff mpt_entry->flags |= cpu_to_be32(MTHCA_MPT_FLAG_PHYSICAL); 469219820Sjeff 470219820Sjeff mpt_entry->page_size = cpu_to_be32(buffer_size_shift - 12); 471219820Sjeff mpt_entry->key = cpu_to_be32(key); 472219820Sjeff mpt_entry->pd = cpu_to_be32(pd); 473219820Sjeff mpt_entry->start = cpu_to_be64(iova); 474219820Sjeff mpt_entry->length = cpu_to_be64(total_size); 475219820Sjeff 476219820Sjeff memset(&mpt_entry->lkey, 0, 477219820Sjeff sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey)); 478219820Sjeff 479219820Sjeff if (mr->mtt) 480219820Sjeff mpt_entry->mtt_seg = 481219820Sjeff cpu_to_be64(dev->mr_table.mtt_base + 482219820Sjeff mr->mtt->first_seg * dev->limits.mtt_seg_size); 483219820Sjeff 484219820Sjeff if (0) { 485219820Sjeff mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey); 486219820Sjeff for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) { 487219820Sjeff if (i % 4 == 0) 488219820Sjeff printk("[%02x] ", i * 4); 489219820Sjeff printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i])); 490219820Sjeff if ((i + 1) % 4 == 0) 491219820Sjeff printk("\n"); 492219820Sjeff } 493219820Sjeff } 494219820Sjeff 495219820Sjeff err = mthca_SW2HW_MPT(dev, mailbox, 496219820Sjeff key & (dev->limits.num_mpts - 1), 497219820Sjeff &status); 498219820Sjeff if (err) { 499219820Sjeff mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); 500219820Sjeff goto err_out_mailbox; 501219820Sjeff } else if (status) { 502219820Sjeff mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n", 503219820Sjeff status); 504219820Sjeff err = -EINVAL; 505219820Sjeff goto err_out_mailbox; 506219820Sjeff } 507219820Sjeff 508219820Sjeff mthca_free_mailbox(dev, mailbox); 509219820Sjeff return err; 510219820Sjeff 511219820Sjefferr_out_mailbox: 512219820Sjeff mthca_free_mailbox(dev, mailbox); 513219820Sjeff 514219820Sjefferr_out_table: 515219820Sjeff mthca_table_put(dev, dev->mr_table.mpt_table, key); 516219820Sjeff 517219820Sjefferr_out_mpt_free: 518219820Sjeff mthca_free(&dev->mr_table.mpt_alloc, key); 519219820Sjeff return err; 520219820Sjeff} 521219820Sjeff 522219820Sjeffint mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd, 523219820Sjeff u32 access, struct mthca_mr *mr) 524219820Sjeff{ 525219820Sjeff mr->mtt = NULL; 526219820Sjeff return mthca_mr_alloc(dev, pd, 12, 0, ~0ULL, access, mr); 527219820Sjeff} 528219820Sjeff 529219820Sjeffint mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd, 530219820Sjeff u64 *buffer_list, int buffer_size_shift, 531219820Sjeff int list_len, u64 iova, u64 total_size, 532219820Sjeff u32 access, struct mthca_mr *mr) 533219820Sjeff{ 534219820Sjeff int err; 535219820Sjeff 536219820Sjeff mr->mtt = mthca_alloc_mtt(dev, list_len); 537219820Sjeff if (IS_ERR(mr->mtt)) 538219820Sjeff return PTR_ERR(mr->mtt); 539219820Sjeff 540219820Sjeff err = mthca_write_mtt(dev, mr->mtt, 0, buffer_list, list_len); 541219820Sjeff if (err) { 542219820Sjeff mthca_free_mtt(dev, mr->mtt); 543219820Sjeff return err; 544219820Sjeff } 545219820Sjeff 546219820Sjeff err = mthca_mr_alloc(dev, pd, buffer_size_shift, iova, 547219820Sjeff total_size, access, mr); 548219820Sjeff if (err) 549219820Sjeff mthca_free_mtt(dev, mr->mtt); 550219820Sjeff 551219820Sjeff return err; 552219820Sjeff} 553219820Sjeff 554219820Sjeff/* Free mr or fmr */ 555219820Sjeffstatic void mthca_free_region(struct mthca_dev *dev, u32 lkey) 556219820Sjeff{ 557219820Sjeff mthca_table_put(dev, dev->mr_table.mpt_table, 558219820Sjeff key_to_hw_index(dev, lkey)); 559219820Sjeff 560219820Sjeff mthca_free(&dev->mr_table.mpt_alloc, key_to_hw_index(dev, lkey)); 561219820Sjeff} 562219820Sjeff 563219820Sjeffvoid mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr) 564219820Sjeff{ 565219820Sjeff int err; 566219820Sjeff u8 status; 567219820Sjeff 568219820Sjeff err = mthca_HW2SW_MPT(dev, NULL, 569219820Sjeff key_to_hw_index(dev, mr->ibmr.lkey) & 570219820Sjeff (dev->limits.num_mpts - 1), 571219820Sjeff &status); 572219820Sjeff if (err) 573219820Sjeff mthca_warn(dev, "HW2SW_MPT failed (%d)\n", err); 574219820Sjeff else if (status) 575219820Sjeff mthca_warn(dev, "HW2SW_MPT returned status 0x%02x\n", 576219820Sjeff status); 577219820Sjeff 578219820Sjeff mthca_free_region(dev, mr->ibmr.lkey); 579219820Sjeff mthca_free_mtt(dev, mr->mtt); 580219820Sjeff} 581219820Sjeff 582219820Sjeffint mthca_fmr_alloc(struct mthca_dev *dev, u32 pd, 583219820Sjeff u32 access, struct mthca_fmr *mr) 584219820Sjeff{ 585219820Sjeff struct mthca_mpt_entry *mpt_entry; 586219820Sjeff struct mthca_mailbox *mailbox; 587219820Sjeff u64 mtt_seg; 588219820Sjeff u32 key, idx; 589219820Sjeff u8 status; 590219820Sjeff int list_len = mr->attr.max_pages; 591219820Sjeff int err = -ENOMEM; 592219820Sjeff int i; 593219820Sjeff 594219820Sjeff if (mr->attr.page_shift < 12 || mr->attr.page_shift >= 32) 595219820Sjeff return -EINVAL; 596219820Sjeff 597219820Sjeff /* For Arbel, all MTTs must fit in the same page. */ 598219820Sjeff if (mthca_is_memfree(dev) && 599219820Sjeff mr->attr.max_pages * sizeof *mr->mem.arbel.mtts > PAGE_SIZE) 600219820Sjeff return -EINVAL; 601219820Sjeff 602219820Sjeff mr->maps = 0; 603219820Sjeff 604219820Sjeff key = mthca_alloc(&dev->mr_table.mpt_alloc); 605219820Sjeff if (key == -1) 606219820Sjeff return -ENOMEM; 607219820Sjeff key = adjust_key(dev, key); 608219820Sjeff 609219820Sjeff idx = key & (dev->limits.num_mpts - 1); 610219820Sjeff mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key); 611219820Sjeff 612219820Sjeff if (mthca_is_memfree(dev)) { 613219820Sjeff err = mthca_table_get(dev, dev->mr_table.mpt_table, key); 614219820Sjeff if (err) 615219820Sjeff goto err_out_mpt_free; 616219820Sjeff 617219820Sjeff mr->mem.arbel.mpt = mthca_table_find(dev->mr_table.mpt_table, key, NULL); 618219820Sjeff BUG_ON(!mr->mem.arbel.mpt); 619219820Sjeff } else 620219820Sjeff mr->mem.tavor.mpt = dev->mr_table.tavor_fmr.mpt_base + 621219820Sjeff sizeof *(mr->mem.tavor.mpt) * idx; 622219820Sjeff 623219820Sjeff mr->mtt = __mthca_alloc_mtt(dev, list_len, dev->mr_table.fmr_mtt_buddy); 624219820Sjeff if (IS_ERR(mr->mtt)) { 625219820Sjeff err = PTR_ERR(mr->mtt); 626219820Sjeff goto err_out_table; 627219820Sjeff } 628219820Sjeff 629219820Sjeff mtt_seg = mr->mtt->first_seg * dev->limits.mtt_seg_size; 630219820Sjeff 631219820Sjeff if (mthca_is_memfree(dev)) { 632219820Sjeff mr->mem.arbel.mtts = mthca_table_find(dev->mr_table.mtt_table, 633219820Sjeff mr->mtt->first_seg, 634219820Sjeff &mr->mem.arbel.dma_handle); 635219820Sjeff BUG_ON(!mr->mem.arbel.mtts); 636219820Sjeff } else 637219820Sjeff mr->mem.tavor.mtts = dev->mr_table.tavor_fmr.mtt_base + mtt_seg; 638219820Sjeff 639219820Sjeff mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); 640219820Sjeff if (IS_ERR(mailbox)) { 641219820Sjeff err = PTR_ERR(mailbox); 642219820Sjeff goto err_out_free_mtt; 643219820Sjeff } 644219820Sjeff 645219820Sjeff mpt_entry = mailbox->buf; 646219820Sjeff 647219820Sjeff mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS | 648219820Sjeff MTHCA_MPT_FLAG_MIO | 649219820Sjeff MTHCA_MPT_FLAG_REGION | 650219820Sjeff access); 651219820Sjeff 652219820Sjeff mpt_entry->page_size = cpu_to_be32(mr->attr.page_shift - 12); 653219820Sjeff mpt_entry->key = cpu_to_be32(key); 654219820Sjeff mpt_entry->pd = cpu_to_be32(pd); 655219820Sjeff memset(&mpt_entry->start, 0, 656219820Sjeff sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, start)); 657219820Sjeff mpt_entry->mtt_seg = cpu_to_be64(dev->mr_table.mtt_base + mtt_seg); 658219820Sjeff 659219820Sjeff if (0) { 660219820Sjeff mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey); 661219820Sjeff for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) { 662219820Sjeff if (i % 4 == 0) 663219820Sjeff printk("[%02x] ", i * 4); 664219820Sjeff printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i])); 665219820Sjeff if ((i + 1) % 4 == 0) 666219820Sjeff printk("\n"); 667219820Sjeff } 668219820Sjeff } 669219820Sjeff 670219820Sjeff err = mthca_SW2HW_MPT(dev, mailbox, 671219820Sjeff key & (dev->limits.num_mpts - 1), 672219820Sjeff &status); 673219820Sjeff if (err) { 674219820Sjeff mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); 675219820Sjeff goto err_out_mailbox_free; 676219820Sjeff } 677219820Sjeff if (status) { 678219820Sjeff mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n", 679219820Sjeff status); 680219820Sjeff err = -EINVAL; 681219820Sjeff goto err_out_mailbox_free; 682219820Sjeff } 683219820Sjeff 684219820Sjeff mthca_free_mailbox(dev, mailbox); 685219820Sjeff return 0; 686219820Sjeff 687219820Sjefferr_out_mailbox_free: 688219820Sjeff mthca_free_mailbox(dev, mailbox); 689219820Sjeff 690219820Sjefferr_out_free_mtt: 691219820Sjeff mthca_free_mtt(dev, mr->mtt); 692219820Sjeff 693219820Sjefferr_out_table: 694219820Sjeff mthca_table_put(dev, dev->mr_table.mpt_table, key); 695219820Sjeff 696219820Sjefferr_out_mpt_free: 697219820Sjeff mthca_free(&dev->mr_table.mpt_alloc, key); 698219820Sjeff return err; 699219820Sjeff} 700219820Sjeff 701219820Sjeffint mthca_free_fmr(struct mthca_dev *dev, struct mthca_fmr *fmr) 702219820Sjeff{ 703219820Sjeff if (fmr->maps) 704219820Sjeff return -EBUSY; 705219820Sjeff 706219820Sjeff mthca_free_region(dev, fmr->ibmr.lkey); 707219820Sjeff mthca_free_mtt(dev, fmr->mtt); 708219820Sjeff 709219820Sjeff return 0; 710219820Sjeff} 711219820Sjeff 712219820Sjeffstatic inline int mthca_check_fmr(struct mthca_fmr *fmr, u64 *page_list, 713219820Sjeff int list_len, u64 iova) 714219820Sjeff{ 715219820Sjeff int i, page_mask; 716219820Sjeff 717219820Sjeff if (list_len > fmr->attr.max_pages) 718219820Sjeff return -EINVAL; 719219820Sjeff 720219820Sjeff page_mask = (1 << fmr->attr.page_shift) - 1; 721219820Sjeff 722219820Sjeff /* We are getting page lists, so va must be page aligned. */ 723219820Sjeff if (iova & page_mask) 724219820Sjeff return -EINVAL; 725219820Sjeff 726219820Sjeff /* Trust the user not to pass misaligned data in page_list */ 727219820Sjeff if (0) 728219820Sjeff for (i = 0; i < list_len; ++i) { 729219820Sjeff if (page_list[i] & ~page_mask) 730219820Sjeff return -EINVAL; 731219820Sjeff } 732219820Sjeff 733219820Sjeff if (fmr->maps >= fmr->attr.max_maps) 734219820Sjeff return -EINVAL; 735219820Sjeff 736219820Sjeff return 0; 737219820Sjeff} 738219820Sjeff 739219820Sjeff 740219820Sjeffint mthca_tavor_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, 741219820Sjeff int list_len, u64 iova) 742219820Sjeff{ 743219820Sjeff struct mthca_fmr *fmr = to_mfmr(ibfmr); 744219820Sjeff struct mthca_dev *dev = to_mdev(ibfmr->device); 745219820Sjeff struct mthca_mpt_entry mpt_entry; 746219820Sjeff u32 key; 747219820Sjeff int i, err; 748219820Sjeff 749219820Sjeff err = mthca_check_fmr(fmr, page_list, list_len, iova); 750219820Sjeff if (err) 751219820Sjeff return err; 752219820Sjeff 753219820Sjeff ++fmr->maps; 754219820Sjeff 755219820Sjeff key = tavor_key_to_hw_index(fmr->ibmr.lkey); 756219820Sjeff key += dev->limits.num_mpts; 757219820Sjeff fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key); 758219820Sjeff 759219820Sjeff writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt); 760219820Sjeff 761219820Sjeff for (i = 0; i < list_len; ++i) { 762219820Sjeff __be64 mtt_entry = cpu_to_be64(page_list[i] | 763219820Sjeff MTHCA_MTT_FLAG_PRESENT); 764219820Sjeff mthca_write64_raw(mtt_entry, fmr->mem.tavor.mtts + i); 765219820Sjeff } 766219820Sjeff 767219820Sjeff mpt_entry.lkey = cpu_to_be32(key); 768219820Sjeff mpt_entry.length = cpu_to_be64(list_len * (1ull << fmr->attr.page_shift)); 769219820Sjeff mpt_entry.start = cpu_to_be64(iova); 770219820Sjeff 771219820Sjeff __raw_writel((__force u32) mpt_entry.lkey, &fmr->mem.tavor.mpt->key); 772219820Sjeff memcpy_toio(&fmr->mem.tavor.mpt->start, &mpt_entry.start, 773219820Sjeff offsetof(struct mthca_mpt_entry, window_count) - 774219820Sjeff offsetof(struct mthca_mpt_entry, start)); 775219820Sjeff 776219820Sjeff writeb(MTHCA_MPT_STATUS_HW, fmr->mem.tavor.mpt); 777219820Sjeff 778219820Sjeff return 0; 779219820Sjeff} 780219820Sjeff 781219820Sjeffint mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, 782219820Sjeff int list_len, u64 iova) 783219820Sjeff{ 784219820Sjeff struct mthca_fmr *fmr = to_mfmr(ibfmr); 785219820Sjeff struct mthca_dev *dev = to_mdev(ibfmr->device); 786219820Sjeff u32 key; 787219820Sjeff int i, err; 788219820Sjeff 789219820Sjeff err = mthca_check_fmr(fmr, page_list, list_len, iova); 790219820Sjeff if (err) 791219820Sjeff return err; 792219820Sjeff 793219820Sjeff ++fmr->maps; 794219820Sjeff 795219820Sjeff key = arbel_key_to_hw_index(fmr->ibmr.lkey); 796219820Sjeff if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT) 797219820Sjeff key += SINAI_FMR_KEY_INC; 798219820Sjeff else 799219820Sjeff key += dev->limits.num_mpts; 800219820Sjeff fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key); 801219820Sjeff 802219820Sjeff *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW; 803219820Sjeff 804219820Sjeff wmb(); 805219820Sjeff 806219820Sjeff for (i = 0; i < list_len; ++i) 807219820Sjeff fmr->mem.arbel.mtts[i] = cpu_to_be64(page_list[i] | 808219820Sjeff MTHCA_MTT_FLAG_PRESENT); 809219820Sjeff 810219820Sjeff dma_sync_single(&dev->pdev->dev, fmr->mem.arbel.dma_handle, 811219820Sjeff list_len * sizeof(u64), DMA_TO_DEVICE); 812219820Sjeff 813219820Sjeff fmr->mem.arbel.mpt->key = cpu_to_be32(key); 814219820Sjeff fmr->mem.arbel.mpt->lkey = cpu_to_be32(key); 815219820Sjeff fmr->mem.arbel.mpt->length = cpu_to_be64(list_len * (1ull << fmr->attr.page_shift)); 816219820Sjeff fmr->mem.arbel.mpt->start = cpu_to_be64(iova); 817219820Sjeff 818219820Sjeff wmb(); 819219820Sjeff 820219820Sjeff *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_HW; 821219820Sjeff 822219820Sjeff wmb(); 823219820Sjeff 824219820Sjeff return 0; 825219820Sjeff} 826219820Sjeff 827219820Sjeffvoid mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr) 828219820Sjeff{ 829219820Sjeff if (!fmr->maps) 830219820Sjeff return; 831219820Sjeff 832219820Sjeff fmr->maps = 0; 833219820Sjeff 834219820Sjeff writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt); 835219820Sjeff} 836219820Sjeff 837219820Sjeffvoid mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr) 838219820Sjeff{ 839219820Sjeff if (!fmr->maps) 840219820Sjeff return; 841219820Sjeff 842219820Sjeff fmr->maps = 0; 843219820Sjeff 844219820Sjeff *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW; 845219820Sjeff} 846219820Sjeff 847219820Sjeffint mthca_init_mr_table(struct mthca_dev *dev) 848219820Sjeff{ 849219820Sjeff unsigned long addr; 850219820Sjeff int mpts, mtts, err, i; 851219820Sjeff 852219820Sjeff err = mthca_alloc_init(&dev->mr_table.mpt_alloc, 853219820Sjeff dev->limits.num_mpts, 854219820Sjeff ~0, dev->limits.reserved_mrws); 855219820Sjeff if (err) 856219820Sjeff return err; 857219820Sjeff 858219820Sjeff if (!mthca_is_memfree(dev) && 859219820Sjeff (dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) 860219820Sjeff dev->limits.fmr_reserved_mtts = 0; 861219820Sjeff else 862219820Sjeff dev->mthca_flags |= MTHCA_FLAG_FMR; 863219820Sjeff 864219820Sjeff if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT) 865219820Sjeff mthca_dbg(dev, "Memory key throughput optimization activated.\n"); 866219820Sjeff 867219820Sjeff err = mthca_buddy_init(&dev->mr_table.mtt_buddy, 868219820Sjeff fls(dev->limits.num_mtt_segs - 1)); 869219820Sjeff 870219820Sjeff if (err) 871219820Sjeff goto err_mtt_buddy; 872219820Sjeff 873219820Sjeff dev->mr_table.tavor_fmr.mpt_base = NULL; 874219820Sjeff dev->mr_table.tavor_fmr.mtt_base = NULL; 875219820Sjeff 876219820Sjeff if (dev->limits.fmr_reserved_mtts) { 877219820Sjeff i = fls(dev->limits.fmr_reserved_mtts - 1); 878219820Sjeff 879219820Sjeff if (i >= 31) { 880219820Sjeff mthca_warn(dev, "Unable to reserve 2^31 FMR MTTs.\n"); 881219820Sjeff err = -EINVAL; 882219820Sjeff goto err_fmr_mpt; 883219820Sjeff } 884219820Sjeff mpts = mtts = 1 << i; 885219820Sjeff } else { 886219820Sjeff mtts = dev->limits.num_mtt_segs; 887219820Sjeff mpts = dev->limits.num_mpts; 888219820Sjeff } 889219820Sjeff 890219820Sjeff if (!mthca_is_memfree(dev) && 891219820Sjeff (dev->mthca_flags & MTHCA_FLAG_FMR)) { 892219820Sjeff 893219820Sjeff addr = pci_resource_start(dev->pdev, 4) + 894219820Sjeff ((pci_resource_len(dev->pdev, 4) - 1) & 895219820Sjeff dev->mr_table.mpt_base); 896219820Sjeff 897219820Sjeff dev->mr_table.tavor_fmr.mpt_base = 898219820Sjeff ioremap(addr, mpts * sizeof(struct mthca_mpt_entry)); 899219820Sjeff 900219820Sjeff if (!dev->mr_table.tavor_fmr.mpt_base) { 901219820Sjeff mthca_warn(dev, "MPT ioremap for FMR failed.\n"); 902219820Sjeff err = -ENOMEM; 903219820Sjeff goto err_fmr_mpt; 904219820Sjeff } 905219820Sjeff 906219820Sjeff addr = pci_resource_start(dev->pdev, 4) + 907219820Sjeff ((pci_resource_len(dev->pdev, 4) - 1) & 908219820Sjeff dev->mr_table.mtt_base); 909219820Sjeff 910219820Sjeff dev->mr_table.tavor_fmr.mtt_base = 911219820Sjeff ioremap(addr, mtts * dev->limits.mtt_seg_size); 912219820Sjeff if (!dev->mr_table.tavor_fmr.mtt_base) { 913219820Sjeff mthca_warn(dev, "MTT ioremap for FMR failed.\n"); 914219820Sjeff err = -ENOMEM; 915219820Sjeff goto err_fmr_mtt; 916219820Sjeff } 917219820Sjeff } 918219820Sjeff 919219820Sjeff if (dev->limits.fmr_reserved_mtts) { 920219820Sjeff err = mthca_buddy_init(&dev->mr_table.tavor_fmr.mtt_buddy, fls(mtts - 1)); 921219820Sjeff if (err) 922219820Sjeff goto err_fmr_mtt_buddy; 923219820Sjeff 924219820Sjeff /* Prevent regular MRs from using FMR keys */ 925219820Sjeff err = mthca_buddy_alloc(&dev->mr_table.mtt_buddy, fls(mtts - 1)); 926219820Sjeff if (err) 927219820Sjeff goto err_reserve_fmr; 928219820Sjeff 929219820Sjeff dev->mr_table.fmr_mtt_buddy = 930219820Sjeff &dev->mr_table.tavor_fmr.mtt_buddy; 931219820Sjeff } else 932219820Sjeff dev->mr_table.fmr_mtt_buddy = &dev->mr_table.mtt_buddy; 933219820Sjeff 934219820Sjeff /* FMR table is always the first, take reserved MTTs out of there */ 935219820Sjeff if (dev->limits.reserved_mtts) { 936219820Sjeff i = fls(dev->limits.reserved_mtts - 1); 937219820Sjeff 938219820Sjeff if (mthca_alloc_mtt_range(dev, i, 939219820Sjeff dev->mr_table.fmr_mtt_buddy) == -1) { 940219820Sjeff mthca_warn(dev, "MTT table of order %d is too small.\n", 941219820Sjeff dev->mr_table.fmr_mtt_buddy->max_order); 942219820Sjeff err = -ENOMEM; 943219820Sjeff goto err_reserve_mtts; 944219820Sjeff } 945219820Sjeff } 946219820Sjeff 947219820Sjeff return 0; 948219820Sjeff 949219820Sjefferr_reserve_mtts: 950219820Sjefferr_reserve_fmr: 951219820Sjeff if (dev->limits.fmr_reserved_mtts) 952219820Sjeff mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy); 953219820Sjeff 954219820Sjefferr_fmr_mtt_buddy: 955219820Sjeff if (dev->mr_table.tavor_fmr.mtt_base) 956219820Sjeff iounmap(dev->mr_table.tavor_fmr.mtt_base); 957219820Sjeff 958219820Sjefferr_fmr_mtt: 959219820Sjeff if (dev->mr_table.tavor_fmr.mpt_base) 960219820Sjeff iounmap(dev->mr_table.tavor_fmr.mpt_base); 961219820Sjeff 962219820Sjefferr_fmr_mpt: 963219820Sjeff mthca_buddy_cleanup(&dev->mr_table.mtt_buddy); 964219820Sjeff 965219820Sjefferr_mtt_buddy: 966219820Sjeff mthca_alloc_cleanup(&dev->mr_table.mpt_alloc); 967219820Sjeff 968219820Sjeff return err; 969219820Sjeff} 970219820Sjeff 971219820Sjeffvoid mthca_cleanup_mr_table(struct mthca_dev *dev) 972219820Sjeff{ 973219820Sjeff /* XXX check if any MRs are still allocated? */ 974219820Sjeff if (dev->limits.fmr_reserved_mtts) 975219820Sjeff mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy); 976219820Sjeff 977219820Sjeff mthca_buddy_cleanup(&dev->mr_table.mtt_buddy); 978219820Sjeff 979219820Sjeff if (dev->mr_table.tavor_fmr.mtt_base) 980219820Sjeff iounmap(dev->mr_table.tavor_fmr.mtt_base); 981219820Sjeff if (dev->mr_table.tavor_fmr.mpt_base) 982219820Sjeff iounmap(dev->mr_table.tavor_fmr.mpt_base); 983219820Sjeff 984219820Sjeff mthca_alloc_cleanup(&dev->mr_table.mpt_alloc); 985219820Sjeff} 986