mr.c revision 219820
1130803Smarcel/* 2130803Smarcel * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. 3130803Smarcel * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4130803Smarcel * 5130803Smarcel * This software is available to you under a choice of one of two 6130803Smarcel * licenses. You may choose to be licensed under the terms of the GNU 7130803Smarcel * General Public License (GPL) Version 2, available from the file 8130803Smarcel * COPYING in the main directory of this source tree, or the 9130803Smarcel * OpenIB.org BSD license below: 10130803Smarcel * 11130803Smarcel * Redistribution and use in source and binary forms, with or 12130803Smarcel * without modification, are permitted provided that the following 13130803Smarcel * conditions are met: 14130803Smarcel * 15130803Smarcel * - Redistributions of source code must retain the above 16130803Smarcel * copyright notice, this list of conditions and the following 17130803Smarcel * disclaimer. 18130803Smarcel * 19130803Smarcel * - Redistributions in binary form must reproduce the above 20130803Smarcel * copyright notice, this list of conditions and the following 21130803Smarcel * disclaimer in the documentation and/or other materials 22130803Smarcel * provided with the distribution. 23130803Smarcel * 24130803Smarcel * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25130803Smarcel * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26130803Smarcel * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27130803Smarcel * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28130803Smarcel * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29130803Smarcel * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30130803Smarcel * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31130803Smarcel * SOFTWARE. 32130803Smarcel */ 33130803Smarcel 34130803Smarcel#include "mlx4_ib.h" 35130803Smarcel 36130803Smarcelstatic u32 convert_access(int acc) 37130803Smarcel{ 38130803Smarcel return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX4_PERM_ATOMIC : 0) | 39130803Smarcel (acc & IB_ACCESS_REMOTE_WRITE ? MLX4_PERM_REMOTE_WRITE : 0) | 40130803Smarcel (acc & IB_ACCESS_REMOTE_READ ? MLX4_PERM_REMOTE_READ : 0) | 41130803Smarcel (acc & IB_ACCESS_LOCAL_WRITE ? MLX4_PERM_LOCAL_WRITE : 0) | 42130803Smarcel MLX4_PERM_LOCAL_READ; 43130803Smarcel} 44130803Smarcel 45130803Smarcelstruct ib_mr *mlx4_ib_get_dma_mr(struct ib_pd *pd, int acc) 46130803Smarcel{ 47130803Smarcel struct mlx4_ib_mr *mr; 48130803Smarcel int err; 49130803Smarcel 50130803Smarcel mr = kmalloc(sizeof *mr, GFP_KERNEL); 51130803Smarcel if (!mr) 52130803Smarcel return ERR_PTR(-ENOMEM); 53130803Smarcel 54130803Smarcel err = mlx4_mr_alloc(to_mdev(pd->device)->dev, to_mpd(pd)->pdn, 0, 55130803Smarcel ~0ull, convert_access(acc), 0, 0, &mr->mmr); 56130803Smarcel if (err) 57130803Smarcel goto err_free; 58130803Smarcel 59130803Smarcel err = mlx4_mr_enable(to_mdev(pd->device)->dev, &mr->mmr); 60130803Smarcel if (err) 61130803Smarcel goto err_mr; 62130803Smarcel 63130803Smarcel mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; 64130803Smarcel mr->umem = NULL; 65130803Smarcel 66130803Smarcel return &mr->ibmr; 67130803Smarcel 68130803Smarcelerr_mr: 69130803Smarcel mlx4_mr_free(to_mdev(pd->device)->dev, &mr->mmr); 70130803Smarcel 71130803Smarcelerr_free: 72130803Smarcel kfree(mr); 73130803Smarcel 74130803Smarcel return ERR_PTR(err); 75130803Smarcel} 76130803Smarcel 77130803Smarcelint mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt, 78130803Smarcel struct ib_umem *umem) 79130803Smarcel{ 80130803Smarcel u64 *pages; 81130803Smarcel struct ib_umem_chunk *chunk; 82130803Smarcel int i, j, k; 83130803Smarcel int n; 84130803Smarcel int len; 85130803Smarcel int err = 0; 86130803Smarcel 87130803Smarcel pages = (u64 *) __get_free_page(GFP_KERNEL); 88130803Smarcel if (!pages) 89130803Smarcel return -ENOMEM; 90130803Smarcel 91130803Smarcel i = n = 0; 92130803Smarcel 93130803Smarcel list_for_each_entry(chunk, &umem->chunk_list, list) 94130803Smarcel for (j = 0; j < chunk->nmap; ++j) { 95130803Smarcel len = sg_dma_len(&chunk->page_list[j]) >> mtt->page_shift; 96130803Smarcel for (k = 0; k < len; ++k) { 97130803Smarcel pages[i++] = sg_dma_address(&chunk->page_list[j]) + 98130803Smarcel umem->page_size * k; 99130803Smarcel /* 100130803Smarcel * Be friendly to mlx4_write_mtt() and 101130803Smarcel * pass it chunks of appropriate size. 102130803Smarcel */ 103130803Smarcel if (i == PAGE_SIZE / sizeof (u64)) { 104130803Smarcel err = mlx4_write_mtt(dev->dev, mtt, n, 105130803Smarcel i, pages); 106130803Smarcel if (err) 107130803Smarcel goto out; 108130803Smarcel n += i; 109130803Smarcel i = 0; 110130803Smarcel } 111130803Smarcel } 112130803Smarcel } 113130803Smarcel 114130803Smarcel if (i) 115130803Smarcel err = mlx4_write_mtt(dev->dev, mtt, n, i, pages); 116130803Smarcel 117130803Smarcelout: 118130803Smarcel free_page((unsigned long) pages); 119130803Smarcel return err; 120130803Smarcel} 121130803Smarcel 122130803Smarcelstatic int handle_hugetlb_user_mr(struct ib_pd *pd, struct mlx4_ib_mr *mr, 123130803Smarcel u64 start, u64 virt_addr, int access_flags) 124130803Smarcel{ 125130803Smarcel#if defined(CONFIG_HUGETLB_PAGE) && !defined(__powerpc__) && !defined(__ia64__) 126130803Smarcel struct mlx4_ib_dev *dev = to_mdev(pd->device); 127130803Smarcel struct ib_umem_chunk *chunk; 128130803Smarcel unsigned dsize; 129130803Smarcel dma_addr_t daddr; 130130803Smarcel unsigned cur_size = 0; 131130803Smarcel dma_addr_t uninitialized_var(cur_addr); 132130803Smarcel int n; 133130803Smarcel struct ib_umem *umem = mr->umem; 134130803Smarcel u64 *arr; 135130803Smarcel int err = 0; 136130803Smarcel int i; 137130803Smarcel int j = 0; 138130803Smarcel int off = start & (HPAGE_SIZE - 1); 139130803Smarcel 140130803Smarcel n = DIV_ROUND_UP(off + umem->length, HPAGE_SIZE); 141130803Smarcel arr = kmalloc(n * sizeof *arr, GFP_KERNEL); 142130803Smarcel if (!arr) 143130803Smarcel return -ENOMEM; 144130803Smarcel 145130803Smarcel list_for_each_entry(chunk, &umem->chunk_list, list) 146130803Smarcel for (i = 0; i < chunk->nmap; ++i) { 147130803Smarcel daddr = sg_dma_address(&chunk->page_list[i]); 148130803Smarcel dsize = sg_dma_len(&chunk->page_list[i]); 149130803Smarcel if (!cur_size) { 150130803Smarcel cur_addr = daddr; 151130803Smarcel cur_size = dsize; 152130803Smarcel } else if (cur_addr + cur_size != daddr) { 153130803Smarcel err = -EINVAL; 154130803Smarcel goto out; 155130803Smarcel } else 156130803Smarcel cur_size += dsize; 157130803Smarcel 158130803Smarcel if (cur_size > HPAGE_SIZE) { 159130803Smarcel err = -EINVAL; 160130803Smarcel goto out; 161130803Smarcel } else if (cur_size == HPAGE_SIZE) { 162130803Smarcel cur_size = 0; 163130803Smarcel arr[j++] = cur_addr; 164130803Smarcel } 165130803Smarcel } 166130803Smarcel 167130803Smarcel if (cur_size) { 168130803Smarcel arr[j++] = cur_addr; 169130803Smarcel } 170130803Smarcel 171130803Smarcel err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, virt_addr, umem->length, 172130803Smarcel convert_access(access_flags), n, HPAGE_SHIFT, &mr->mmr); 173130803Smarcel if (err) 174130803Smarcel goto out; 175130803Smarcel 176130803Smarcel err = mlx4_write_mtt(dev->dev, &mr->mmr.mtt, 0, n, arr); 177130803Smarcel 178130803Smarcelout: 179130803Smarcel kfree(arr); 180130803Smarcel return err; 181130803Smarcel#else 182130803Smarcel return -ENOSYS; 183130803Smarcel#endif 184130803Smarcel} 185130803Smarcel 186130803Smarcelstruct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 187130803Smarcel u64 virt_addr, int access_flags, 188130803Smarcel struct ib_udata *udata) 189130803Smarcel{ 190130803Smarcel struct mlx4_ib_dev *dev = to_mdev(pd->device); 191130803Smarcel struct mlx4_ib_mr *mr; 192130803Smarcel int shift; 193130803Smarcel int err; 194130803Smarcel int n; 195130803Smarcel 196130803Smarcel mr = kmalloc(sizeof *mr, GFP_KERNEL); 197130803Smarcel if (!mr) 198130803Smarcel return ERR_PTR(-ENOMEM); 199130803Smarcel 200130803Smarcel mr->umem = ib_umem_get(pd->uobject->context, start, length, 201130803Smarcel access_flags, 0); 202130803Smarcel if (IS_ERR(mr->umem)) { 203130803Smarcel err = PTR_ERR(mr->umem); 204130803Smarcel goto err_free; 205130803Smarcel } 206130803Smarcel 207130803Smarcel if (!mr->umem->hugetlb || 208130803Smarcel handle_hugetlb_user_mr(pd, mr, start, virt_addr, access_flags)) { 209130803Smarcel n = ib_umem_page_count(mr->umem); 210130803Smarcel shift = ilog2(mr->umem->page_size); 211130803Smarcel 212130803Smarcel err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, virt_addr, length, 213130803Smarcel convert_access(access_flags), n, shift, &mr->mmr); 214130803Smarcel if (err) 215130803Smarcel goto err_umem; 216130803Smarcel 217130803Smarcel err = mlx4_ib_umem_write_mtt(dev, &mr->mmr.mtt, mr->umem); 218130803Smarcel if (err) 219130803Smarcel goto err_mr; 220130803Smarcel } 221130803Smarcel 222130803Smarcel err = mlx4_mr_enable(dev->dev, &mr->mmr); 223130803Smarcel if (err) 224130803Smarcel goto err_mr; 225130803Smarcel 226130803Smarcel mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; 227130803Smarcel 228130803Smarcel return &mr->ibmr; 229130803Smarcel 230130803Smarcelerr_mr: 231130803Smarcel mlx4_mr_free(to_mdev(pd->device)->dev, &mr->mmr); 232130803Smarcel 233130803Smarcelerr_umem: 234130803Smarcel ib_umem_release(mr->umem); 235130803Smarcel 236130803Smarcelerr_free: 237130803Smarcel kfree(mr); 238130803Smarcel 239130803Smarcel return ERR_PTR(err); 240130803Smarcel} 241130803Smarcel 242130803Smarcelint mlx4_ib_dereg_mr(struct ib_mr *ibmr) 243130803Smarcel{ 244130803Smarcel struct mlx4_ib_mr *mr = to_mmr(ibmr); 245130803Smarcel 246130803Smarcel mlx4_mr_free(to_mdev(ibmr->device)->dev, &mr->mmr); 247130803Smarcel if (mr->umem) 248130803Smarcel ib_umem_release(mr->umem); 249130803Smarcel kfree(mr); 250130803Smarcel 251130803Smarcel return 0; 252130803Smarcel} 253130803Smarcel 254130803Smarcelstruct ib_mr *mlx4_ib_alloc_fast_reg_mr(struct ib_pd *pd, 255130803Smarcel int max_page_list_len) 256130803Smarcel{ 257130803Smarcel struct mlx4_ib_dev *dev = to_mdev(pd->device); 258130803Smarcel struct mlx4_ib_mr *mr; 259130803Smarcel int err; 260130803Smarcel 261130803Smarcel mr = kmalloc(sizeof *mr, GFP_KERNEL); 262130803Smarcel if (!mr) 263130803Smarcel return ERR_PTR(-ENOMEM); 264130803Smarcel 265130803Smarcel err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, 0, 0, 0, 266130803Smarcel max_page_list_len, 0, &mr->mmr); 267130803Smarcel if (err) 268130803Smarcel goto err_free; 269130803Smarcel 270130803Smarcel err = mlx4_mr_enable(dev->dev, &mr->mmr); 271130803Smarcel if (err) 272130803Smarcel goto err_mr; 273130803Smarcel 274130803Smarcel mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; 275130803Smarcel mr->umem = NULL; 276130803Smarcel 277130803Smarcel return &mr->ibmr; 278130803Smarcel 279130803Smarcelerr_mr: 280130803Smarcel mlx4_mr_free(dev->dev, &mr->mmr); 281130803Smarcel 282130803Smarcelerr_free: 283130803Smarcel kfree(mr); 284130803Smarcel return ERR_PTR(err); 285130803Smarcel} 286130803Smarcel 287130803Smarcelstruct ib_fast_reg_page_list *mlx4_ib_alloc_fast_reg_page_list(struct ib_device *ibdev, 288130803Smarcel int page_list_len) 289130803Smarcel{ 290130803Smarcel struct mlx4_ib_dev *dev = to_mdev(ibdev); 291130803Smarcel struct mlx4_ib_fast_reg_page_list *mfrpl; 292130803Smarcel int size = page_list_len * sizeof (u64); 293130803Smarcel 294130803Smarcel if (page_list_len > MAX_FAST_REG_PAGES) 295130803Smarcel return ERR_PTR(-EINVAL); 296130803Smarcel 297130803Smarcel mfrpl = kmalloc(sizeof *mfrpl, GFP_KERNEL); 298130803Smarcel if (!mfrpl) 299130803Smarcel return ERR_PTR(-ENOMEM); 300130803Smarcel 301130803Smarcel mfrpl->ibfrpl.page_list = kmalloc(size, GFP_KERNEL); 302130803Smarcel if (!mfrpl->ibfrpl.page_list) 303130803Smarcel goto err_free; 304130803Smarcel 305130803Smarcel mfrpl->mapped_page_list = dma_alloc_coherent(&dev->dev->pdev->dev, 306130803Smarcel size, &mfrpl->map, 307130803Smarcel GFP_KERNEL); 308130803Smarcel if (!mfrpl->mapped_page_list) 309130803Smarcel goto err_free; 310130803Smarcel 311130803Smarcel WARN_ON(mfrpl->map & 0x3f); 312130803Smarcel 313130803Smarcel return &mfrpl->ibfrpl; 314130803Smarcel 315130803Smarcelerr_free: 316130803Smarcel kfree(mfrpl->ibfrpl.page_list); 317130803Smarcel kfree(mfrpl); 318130803Smarcel return ERR_PTR(-ENOMEM); 319130803Smarcel} 320130803Smarcel 321130803Smarcelvoid mlx4_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list) 322130803Smarcel{ 323130803Smarcel struct mlx4_ib_dev *dev = to_mdev(page_list->device); 324130803Smarcel struct mlx4_ib_fast_reg_page_list *mfrpl = to_mfrpl(page_list); 325130803Smarcel int size = page_list->max_page_list_len * sizeof (u64); 326130803Smarcel 327130803Smarcel dma_free_coherent(&dev->dev->pdev->dev, size, mfrpl->mapped_page_list, 328130803Smarcel mfrpl->map); 329130803Smarcel kfree(mfrpl->ibfrpl.page_list); 330130803Smarcel kfree(mfrpl); 331130803Smarcel} 332130803Smarcel 333130803Smarcelstruct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int acc, 334130803Smarcel struct ib_fmr_attr *fmr_attr) 335130803Smarcel{ 336130803Smarcel struct mlx4_ib_dev *dev = to_mdev(pd->device); 337130803Smarcel struct mlx4_ib_fmr *fmr; 338130803Smarcel int err = -ENOMEM; 339130803Smarcel 340130803Smarcel fmr = kmalloc(sizeof *fmr, GFP_KERNEL); 341130803Smarcel if (!fmr) 342130803Smarcel return ERR_PTR(-ENOMEM); 343130803Smarcel 344130803Smarcel err = mlx4_fmr_alloc(dev->dev, to_mpd(pd)->pdn, convert_access(acc), 345130803Smarcel fmr_attr->max_pages, fmr_attr->max_maps, 346130803Smarcel fmr_attr->page_shift, &fmr->mfmr); 347130803Smarcel if (err) 348130803Smarcel goto err_free; 349130803Smarcel 350130803Smarcel err = mlx4_fmr_enable(to_mdev(pd->device)->dev, &fmr->mfmr); 351130803Smarcel if (err) 352130803Smarcel goto err_mr; 353130803Smarcel 354130803Smarcel fmr->ibfmr.rkey = fmr->ibfmr.lkey = fmr->mfmr.mr.key; 355130803Smarcel 356130803Smarcel return &fmr->ibfmr; 357130803Smarcel 358130803Smarcelerr_mr: 359130803Smarcel mlx4_mr_free(to_mdev(pd->device)->dev, &fmr->mfmr.mr); 360130803Smarcel 361130803Smarcelerr_free: 362130803Smarcel kfree(fmr); 363130803Smarcel 364130803Smarcel return ERR_PTR(err); 365130803Smarcel} 366130803Smarcel 367130803Smarcelint mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, 368130803Smarcel int npages, u64 iova) 369130803Smarcel{ 370130803Smarcel struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); 371130803Smarcel struct mlx4_ib_dev *dev = to_mdev(ifmr->ibfmr.device); 372130803Smarcel 373130803Smarcel return mlx4_map_phys_fmr(dev->dev, &ifmr->mfmr, page_list, npages, iova, 374130803Smarcel &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey); 375130803Smarcel} 376130803Smarcel 377130803Smarcelint mlx4_ib_unmap_fmr(struct list_head *fmr_list) 378130803Smarcel{ 379130803Smarcel struct ib_fmr *ibfmr; 380130803Smarcel int err; 381130803Smarcel struct mlx4_dev *mdev = NULL; 382130803Smarcel 383130803Smarcel list_for_each_entry(ibfmr, fmr_list, list) { 384130803Smarcel if (mdev && to_mdev(ibfmr->device)->dev != mdev) 385130803Smarcel return -EINVAL; 386130803Smarcel mdev = to_mdev(ibfmr->device)->dev; 387130803Smarcel } 388130803Smarcel 389130803Smarcel if (!mdev) 390130803Smarcel return 0; 391130803Smarcel 392130803Smarcel list_for_each_entry(ibfmr, fmr_list, list) { 393130803Smarcel struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); 394130803Smarcel 395130803Smarcel mlx4_fmr_unmap(mdev, &ifmr->mfmr, &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey); 396130803Smarcel } 397130803Smarcel 398130803Smarcel /* 399130803Smarcel * Make sure all MPT status updates are visible before issuing 400130803Smarcel * SYNC_TPT firmware command. 401130803Smarcel */ 402130803Smarcel wmb(); 403130803Smarcel 404130803Smarcel err = mlx4_SYNC_TPT(mdev); 405130803Smarcel if (err) 406130803Smarcel printk(KERN_WARNING "mlx4_ib: SYNC_TPT error %d when " 407130803Smarcel "unmapping FMRs\n", err); 408130803Smarcel 409130803Smarcel return 0; 410130803Smarcel} 411130803Smarcel 412130803Smarcelint mlx4_ib_fmr_dealloc(struct ib_fmr *ibfmr) 413130803Smarcel{ 414130803Smarcel struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); 415130803Smarcel struct mlx4_ib_dev *dev = to_mdev(ibfmr->device); 416130803Smarcel int err; 417130803Smarcel 418130803Smarcel err = mlx4_fmr_free(dev->dev, &ifmr->mfmr); 419130803Smarcel 420130803Smarcel if (!err) 421130803Smarcel kfree(ifmr); 422130803Smarcel 423130803Smarcel return err; 424130803Smarcel} 425130803Smarcel