1219820Sjeff/* 2219820Sjeff * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. 3219820Sjeff * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4219820Sjeff * 5219820Sjeff * This software is available to you under a choice of one of two 6219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 7219820Sjeff * General Public License (GPL) Version 2, available from the file 8219820Sjeff * COPYING in the main directory of this source tree, or the 9219820Sjeff * OpenIB.org BSD license below: 10219820Sjeff * 11219820Sjeff * Redistribution and use in source and binary forms, with or 12219820Sjeff * without modification, are permitted provided that the following 13219820Sjeff * conditions are met: 14219820Sjeff * 15219820Sjeff * - Redistributions of source code must retain the above 16219820Sjeff * copyright notice, this list of conditions and the following 17219820Sjeff * disclaimer. 18219820Sjeff * 19219820Sjeff * - Redistributions in binary form must reproduce the above 20219820Sjeff * copyright notice, this list of conditions and the following 21219820Sjeff * disclaimer in the documentation and/or other materials 22219820Sjeff * provided with the distribution. 23219820Sjeff * 24219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31219820Sjeff * SOFTWARE. 32219820Sjeff */ 33219820Sjeff 34255932Salfred#include <linux/slab.h> 35255932Salfred#include <linux/module.h> 36255932Salfred#include <linux/sched.h> 37255932Salfred 38331769Shselasky#include <asm/atomic64.h> 39331769Shselasky 40219820Sjeff#include "mlx4_ib.h" 41219820Sjeff 42219820Sjeffstatic u32 convert_access(int acc) 43219820Sjeff{ 44219820Sjeff return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX4_PERM_ATOMIC : 0) | 45219820Sjeff (acc & IB_ACCESS_REMOTE_WRITE ? MLX4_PERM_REMOTE_WRITE : 0) | 46219820Sjeff (acc & IB_ACCESS_REMOTE_READ ? MLX4_PERM_REMOTE_READ : 0) | 47219820Sjeff (acc & IB_ACCESS_LOCAL_WRITE ? MLX4_PERM_LOCAL_WRITE : 0) | 48331769Shselasky (acc & IB_ACCESS_MW_BIND ? MLX4_PERM_BIND_MW : 0) | 49219820Sjeff MLX4_PERM_LOCAL_READ; 50219820Sjeff} 51219820Sjeff 52331769Shselaskystatic enum mlx4_mw_type to_mlx4_type(enum ib_mw_type type) 53255932Salfred{ 54331769Shselasky switch (type) { 55331769Shselasky case IB_MW_TYPE_1: return MLX4_MW_TYPE_1; 56331769Shselasky case IB_MW_TYPE_2: return MLX4_MW_TYPE_2; 57331769Shselasky default: return -1; 58331769Shselasky } 59255932Salfred} 60255932Salfred 61219820Sjeffstruct ib_mr *mlx4_ib_get_dma_mr(struct ib_pd *pd, int acc) 62219820Sjeff{ 63219820Sjeff struct mlx4_ib_mr *mr; 64219820Sjeff int err; 65219820Sjeff 66331769Shselasky mr = kzalloc(sizeof(*mr), GFP_KERNEL); 67219820Sjeff if (!mr) 68219820Sjeff return ERR_PTR(-ENOMEM); 69219820Sjeff 70219820Sjeff err = mlx4_mr_alloc(to_mdev(pd->device)->dev, to_mpd(pd)->pdn, 0, 71219820Sjeff ~0ull, convert_access(acc), 0, 0, &mr->mmr); 72219820Sjeff if (err) 73219820Sjeff goto err_free; 74219820Sjeff 75219820Sjeff err = mlx4_mr_enable(to_mdev(pd->device)->dev, &mr->mmr); 76219820Sjeff if (err) 77219820Sjeff goto err_mr; 78219820Sjeff 79219820Sjeff mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; 80219820Sjeff mr->umem = NULL; 81219820Sjeff 82219820Sjeff return &mr->ibmr; 83219820Sjeff 84219820Sjefferr_mr: 85278886Shselasky (void) mlx4_mr_free(to_mdev(pd->device)->dev, &mr->mmr); 86219820Sjeff 87219820Sjefferr_free: 88219820Sjeff kfree(mr); 89219820Sjeff 90219820Sjeff return ERR_PTR(err); 91219820Sjeff} 92219820Sjeff 93219820Sjeffint mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt, 94219820Sjeff struct ib_umem *umem) 95219820Sjeff{ 96219820Sjeff u64 *pages; 97331769Shselasky int i, k, entry; 98331769Shselasky int n; 99331769Shselasky int len; 100219820Sjeff int err = 0; 101278886Shselasky struct scatterlist *sg; 102219820Sjeff 103219820Sjeff pages = (u64 *) __get_free_page(GFP_KERNEL); 104219820Sjeff if (!pages) 105219820Sjeff return -ENOMEM; 106219820Sjeff 107331769Shselasky i = n = 0; 108219820Sjeff 109331769Shselasky for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 110331769Shselasky len = sg_dma_len(sg) >> mtt->page_shift; 111331769Shselasky for (k = 0; k < len; ++k) { 112331769Shselasky pages[i++] = sg_dma_address(sg) + 113331769Shselasky umem->page_size * k; 114331769Shselasky /* 115331769Shselasky * Be friendly to mlx4_write_mtt() and 116331769Shselasky * pass it chunks of appropriate size. 117331769Shselasky */ 118331769Shselasky if (i == PAGE_SIZE / sizeof (u64)) { 119331769Shselasky err = mlx4_write_mtt(dev->dev, mtt, n, 120331769Shselasky i, pages); 121331769Shselasky if (err) 122331769Shselasky goto out; 123331769Shselasky n += i; 124331769Shselasky i = 0; 125219820Sjeff } 126331769Shselasky } 127296382Shselasky } 128219820Sjeff 129331769Shselasky if (i) 130331769Shselasky err = mlx4_write_mtt(dev->dev, mtt, n, i, pages); 131219820Sjeff 132219820Sjeffout: 133219820Sjeff free_page((unsigned long) pages); 134219820Sjeff return err; 135219820Sjeff} 136219820Sjeff 137219820Sjeffstruct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 138219820Sjeff u64 virt_addr, int access_flags, 139331769Shselasky struct ib_udata *udata) 140219820Sjeff{ 141219820Sjeff struct mlx4_ib_dev *dev = to_mdev(pd->device); 142219820Sjeff struct mlx4_ib_mr *mr; 143219820Sjeff int shift; 144219820Sjeff int err; 145219820Sjeff int n; 146219820Sjeff 147331769Shselasky mr = kzalloc(sizeof(*mr), GFP_KERNEL); 148219820Sjeff if (!mr) 149219820Sjeff return ERR_PTR(-ENOMEM); 150219820Sjeff 151331769Shselasky /* Force registering the memory as writable. */ 152331769Shselasky /* Used for memory re-registeration. HCA protects the access */ 153331769Shselasky mr->umem = ib_umem_get(pd->uobject->context, start, length, 154331769Shselasky access_flags | IB_ACCESS_LOCAL_WRITE, 0); 155219820Sjeff if (IS_ERR(mr->umem)) { 156219820Sjeff err = PTR_ERR(mr->umem); 157219820Sjeff goto err_free; 158219820Sjeff } 159219820Sjeff 160255932Salfred n = ib_umem_page_count(mr->umem); 161331769Shselasky shift = ilog2(mr->umem->page_size); 162331769Shselasky 163255932Salfred err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, virt_addr, length, 164331769Shselasky convert_access(access_flags), n, shift, &mr->mmr); 165255932Salfred if (err) 166255932Salfred goto err_umem; 167219820Sjeff 168255932Salfred err = mlx4_ib_umem_write_mtt(dev, &mr->mmr.mtt, mr->umem); 169255932Salfred if (err) 170255932Salfred goto err_mr; 171219820Sjeff 172219820Sjeff err = mlx4_mr_enable(dev->dev, &mr->mmr); 173219820Sjeff if (err) 174219820Sjeff goto err_mr; 175219820Sjeff 176219820Sjeff mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; 177219820Sjeff 178219820Sjeff return &mr->ibmr; 179219820Sjeff 180219820Sjefferr_mr: 181278886Shselasky (void) mlx4_mr_free(to_mdev(pd->device)->dev, &mr->mmr); 182219820Sjeff 183219820Sjefferr_umem: 184219820Sjeff ib_umem_release(mr->umem); 185219820Sjeff 186219820Sjefferr_free: 187219820Sjeff kfree(mr); 188219820Sjeff 189219820Sjeff return ERR_PTR(err); 190219820Sjeff} 191219820Sjeff 192331769Shselaskyint mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, 193331769Shselasky u64 start, u64 length, u64 virt_addr, 194331769Shselasky int mr_access_flags, struct ib_pd *pd, 195331769Shselasky struct ib_udata *udata) 196219820Sjeff{ 197331769Shselasky struct mlx4_ib_dev *dev = to_mdev(mr->device); 198331769Shselasky struct mlx4_ib_mr *mmr = to_mmr(mr); 199331769Shselasky struct mlx4_mpt_entry *mpt_entry; 200331769Shselasky struct mlx4_mpt_entry **pmpt_entry = &mpt_entry; 201331769Shselasky int err; 202331769Shselasky 203331769Shselasky /* Since we synchronize this call and mlx4_ib_dereg_mr via uverbs, 204331769Shselasky * we assume that the calls can't run concurrently. Otherwise, a 205331769Shselasky * race exists. 206331769Shselasky */ 207331769Shselasky err = mlx4_mr_hw_get_mpt(dev->dev, &mmr->mmr, &pmpt_entry); 208331769Shselasky 209331769Shselasky if (err) 210331769Shselasky return err; 211331769Shselasky 212331769Shselasky if (flags & IB_MR_REREG_PD) { 213331769Shselasky err = mlx4_mr_hw_change_pd(dev->dev, *pmpt_entry, 214331769Shselasky to_mpd(pd)->pdn); 215331769Shselasky 216331769Shselasky if (err) 217331769Shselasky goto release_mpt_entry; 218331769Shselasky } 219331769Shselasky 220331769Shselasky if (flags & IB_MR_REREG_ACCESS) { 221331769Shselasky err = mlx4_mr_hw_change_access(dev->dev, *pmpt_entry, 222331769Shselasky convert_access(mr_access_flags)); 223331769Shselasky 224331769Shselasky if (err) 225331769Shselasky goto release_mpt_entry; 226331769Shselasky } 227331769Shselasky 228331769Shselasky if (flags & IB_MR_REREG_TRANS) { 229331769Shselasky int shift; 230331769Shselasky int n; 231331769Shselasky 232331769Shselasky mlx4_mr_rereg_mem_cleanup(dev->dev, &mmr->mmr); 233331769Shselasky ib_umem_release(mmr->umem); 234331769Shselasky mmr->umem = ib_umem_get(mr->uobject->context, start, length, 235331769Shselasky mr_access_flags | 236331769Shselasky IB_ACCESS_LOCAL_WRITE, 237331769Shselasky 0); 238331769Shselasky if (IS_ERR(mmr->umem)) { 239331769Shselasky err = PTR_ERR(mmr->umem); 240331769Shselasky /* Prevent mlx4_ib_dereg_mr from free'ing invalid pointer */ 241331769Shselasky mmr->umem = NULL; 242331769Shselasky goto release_mpt_entry; 243331769Shselasky } 244331769Shselasky n = ib_umem_page_count(mmr->umem); 245331769Shselasky shift = ilog2(mmr->umem->page_size); 246331769Shselasky 247331769Shselasky err = mlx4_mr_rereg_mem_write(dev->dev, &mmr->mmr, 248331769Shselasky virt_addr, length, n, shift, 249331769Shselasky *pmpt_entry); 250331769Shselasky if (err) { 251331769Shselasky ib_umem_release(mmr->umem); 252331769Shselasky goto release_mpt_entry; 253331769Shselasky } 254331769Shselasky mmr->mmr.iova = virt_addr; 255331769Shselasky mmr->mmr.size = length; 256331769Shselasky 257331769Shselasky err = mlx4_ib_umem_write_mtt(dev, &mmr->mmr.mtt, mmr->umem); 258331769Shselasky if (err) { 259331769Shselasky mlx4_mr_rereg_mem_cleanup(dev->dev, &mmr->mmr); 260331769Shselasky ib_umem_release(mmr->umem); 261331769Shselasky goto release_mpt_entry; 262331769Shselasky } 263331769Shselasky } 264331769Shselasky 265331769Shselasky /* If we couldn't transfer the MR to the HCA, just remember to 266331769Shselasky * return a failure. But dereg_mr will free the resources. 267331769Shselasky */ 268331769Shselasky err = mlx4_mr_hw_write_mpt(dev->dev, &mmr->mmr, pmpt_entry); 269331769Shselasky if (!err && flags & IB_MR_REREG_ACCESS) 270331769Shselasky mmr->mmr.access = mr_access_flags; 271331769Shselasky 272331769Shselaskyrelease_mpt_entry: 273331769Shselasky mlx4_mr_hw_put_mpt(dev->dev, pmpt_entry); 274331769Shselasky 275331769Shselasky return err; 276331769Shselasky} 277331769Shselasky 278331769Shselaskystatic int 279331769Shselaskymlx4_alloc_priv_pages(struct ib_device *device, 280331769Shselasky struct mlx4_ib_mr *mr, 281331769Shselasky int max_pages) 282331769Shselasky{ 283278886Shselasky int ret; 284219820Sjeff 285331769Shselasky /* Ensure that size is aligned to DMA cacheline 286331769Shselasky * requirements. 287331769Shselasky * max_pages is limited to MLX4_MAX_FAST_REG_PAGES 288331769Shselasky * so page_map_size will never cross PAGE_SIZE. 289331769Shselasky */ 290331769Shselasky mr->page_map_size = roundup(max_pages * sizeof(u64), 291331769Shselasky MLX4_MR_PAGES_ALIGN); 292255932Salfred 293331769Shselasky /* Prevent cross page boundary allocation. */ 294331769Shselasky mr->pages = (__be64 *)get_zeroed_page(GFP_KERNEL); 295331769Shselasky if (!mr->pages) 296331769Shselasky return -ENOMEM; 297331769Shselasky 298331769Shselasky mr->page_map = dma_map_single(device->dma_device, mr->pages, 299331769Shselasky mr->page_map_size, DMA_TO_DEVICE); 300331769Shselasky 301331769Shselasky if (dma_mapping_error(device->dma_device, mr->page_map)) { 302331769Shselasky ret = -ENOMEM; 303331769Shselasky goto err; 304255932Salfred } 305255932Salfred 306331769Shselasky return 0; 307331769Shselasky 308331769Shselaskyerr: 309331769Shselasky free_page((unsigned long)mr->pages); 310331769Shselasky return ret; 311331769Shselasky} 312331769Shselasky 313331769Shselaskystatic void 314331769Shselaskymlx4_free_priv_pages(struct mlx4_ib_mr *mr) 315331769Shselasky{ 316331769Shselasky if (mr->pages) { 317331769Shselasky struct ib_device *device = mr->ibmr.device; 318331769Shselasky 319331769Shselasky dma_unmap_single(device->dma_device, mr->page_map, 320331769Shselasky mr->page_map_size, DMA_TO_DEVICE); 321331769Shselasky free_page((unsigned long)mr->pages); 322331769Shselasky mr->pages = NULL; 323278886Shselasky } 324331769Shselasky} 325278886Shselasky 326331769Shselaskyint mlx4_ib_dereg_mr(struct ib_mr *ibmr) 327331769Shselasky{ 328331769Shselasky struct mlx4_ib_mr *mr = to_mmr(ibmr); 329331769Shselasky int ret; 330278886Shselasky 331331769Shselasky mlx4_free_priv_pages(mr); 332255932Salfred 333331769Shselasky ret = mlx4_mr_free(to_mdev(ibmr->device)->dev, &mr->mmr); 334331769Shselasky if (ret) 335331769Shselasky return ret; 336331769Shselasky if (mr->umem) 337331769Shselasky ib_umem_release(mr->umem); 338219820Sjeff kfree(mr); 339219820Sjeff 340219820Sjeff return 0; 341219820Sjeff} 342219820Sjeff 343331769Shselaskystruct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, 344331769Shselasky struct ib_udata *udata) 345278886Shselasky{ 346278886Shselasky struct mlx4_ib_dev *dev = to_mdev(pd->device); 347278886Shselasky struct mlx4_ib_mw *mw; 348278886Shselasky int err; 349278886Shselasky 350278886Shselasky mw = kmalloc(sizeof(*mw), GFP_KERNEL); 351278886Shselasky if (!mw) 352278886Shselasky return ERR_PTR(-ENOMEM); 353278886Shselasky 354331769Shselasky err = mlx4_mw_alloc(dev->dev, to_mpd(pd)->pdn, 355331769Shselasky to_mlx4_type(type), &mw->mmw); 356278886Shselasky if (err) 357278886Shselasky goto err_free; 358278886Shselasky 359278886Shselasky err = mlx4_mw_enable(dev->dev, &mw->mmw); 360278886Shselasky if (err) 361278886Shselasky goto err_mw; 362278886Shselasky 363278886Shselasky mw->ibmw.rkey = mw->mmw.key; 364278886Shselasky 365278886Shselasky return &mw->ibmw; 366278886Shselasky 367278886Shselaskyerr_mw: 368278886Shselasky mlx4_mw_free(dev->dev, &mw->mmw); 369278886Shselasky 370278886Shselaskyerr_free: 371278886Shselasky kfree(mw); 372278886Shselasky 373278886Shselasky return ERR_PTR(err); 374278886Shselasky} 375278886Shselasky 376278886Shselaskyint mlx4_ib_dealloc_mw(struct ib_mw *ibmw) 377278886Shselasky{ 378278886Shselasky struct mlx4_ib_mw *mw = to_mmw(ibmw); 379278886Shselasky 380278886Shselasky mlx4_mw_free(to_mdev(ibmw->device)->dev, &mw->mmw); 381278886Shselasky kfree(mw); 382278886Shselasky 383278886Shselasky return 0; 384278886Shselasky} 385278886Shselasky 386331769Shselaskystruct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, 387331769Shselasky enum ib_mr_type mr_type, 388331769Shselasky u32 max_num_sg) 389219820Sjeff{ 390219820Sjeff struct mlx4_ib_dev *dev = to_mdev(pd->device); 391219820Sjeff struct mlx4_ib_mr *mr; 392219820Sjeff int err; 393219820Sjeff 394331769Shselasky if (mr_type != IB_MR_TYPE_MEM_REG || 395331769Shselasky max_num_sg > MLX4_MAX_FAST_REG_PAGES) 396331769Shselasky return ERR_PTR(-EINVAL); 397331769Shselasky 398331769Shselasky mr = kzalloc(sizeof(*mr), GFP_KERNEL); 399219820Sjeff if (!mr) 400219820Sjeff return ERR_PTR(-ENOMEM); 401219820Sjeff 402219820Sjeff err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, 0, 0, 0, 403331769Shselasky max_num_sg, 0, &mr->mmr); 404219820Sjeff if (err) 405219820Sjeff goto err_free; 406219820Sjeff 407331769Shselasky err = mlx4_alloc_priv_pages(pd->device, mr, max_num_sg); 408331769Shselasky if (err) 409331769Shselasky goto err_free_mr; 410331769Shselasky 411331769Shselasky mr->max_pages = max_num_sg; 412331769Shselasky 413219820Sjeff err = mlx4_mr_enable(dev->dev, &mr->mmr); 414219820Sjeff if (err) 415331769Shselasky goto err_free_pl; 416219820Sjeff 417219820Sjeff mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; 418219820Sjeff mr->umem = NULL; 419219820Sjeff 420219820Sjeff return &mr->ibmr; 421219820Sjeff 422331769Shselaskyerr_free_pl: 423331769Shselasky mlx4_free_priv_pages(mr); 424331769Shselaskyerr_free_mr: 425278886Shselasky (void) mlx4_mr_free(dev->dev, &mr->mmr); 426219820Sjefferr_free: 427219820Sjeff kfree(mr); 428219820Sjeff return ERR_PTR(err); 429219820Sjeff} 430219820Sjeff 431219820Sjeffstruct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int acc, 432219820Sjeff struct ib_fmr_attr *fmr_attr) 433219820Sjeff{ 434219820Sjeff struct mlx4_ib_dev *dev = to_mdev(pd->device); 435219820Sjeff struct mlx4_ib_fmr *fmr; 436219820Sjeff int err = -ENOMEM; 437219820Sjeff 438219820Sjeff fmr = kmalloc(sizeof *fmr, GFP_KERNEL); 439219820Sjeff if (!fmr) 440219820Sjeff return ERR_PTR(-ENOMEM); 441219820Sjeff 442219820Sjeff err = mlx4_fmr_alloc(dev->dev, to_mpd(pd)->pdn, convert_access(acc), 443219820Sjeff fmr_attr->max_pages, fmr_attr->max_maps, 444219820Sjeff fmr_attr->page_shift, &fmr->mfmr); 445219820Sjeff if (err) 446219820Sjeff goto err_free; 447219820Sjeff 448219820Sjeff err = mlx4_fmr_enable(to_mdev(pd->device)->dev, &fmr->mfmr); 449219820Sjeff if (err) 450219820Sjeff goto err_mr; 451219820Sjeff 452219820Sjeff fmr->ibfmr.rkey = fmr->ibfmr.lkey = fmr->mfmr.mr.key; 453219820Sjeff 454219820Sjeff return &fmr->ibfmr; 455219820Sjeff 456219820Sjefferr_mr: 457278886Shselasky (void) mlx4_mr_free(to_mdev(pd->device)->dev, &fmr->mfmr.mr); 458219820Sjeff 459219820Sjefferr_free: 460219820Sjeff kfree(fmr); 461219820Sjeff 462219820Sjeff return ERR_PTR(err); 463219820Sjeff} 464219820Sjeff 465219820Sjeffint mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, 466219820Sjeff int npages, u64 iova) 467219820Sjeff{ 468219820Sjeff struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); 469219820Sjeff struct mlx4_ib_dev *dev = to_mdev(ifmr->ibfmr.device); 470219820Sjeff 471219820Sjeff return mlx4_map_phys_fmr(dev->dev, &ifmr->mfmr, page_list, npages, iova, 472219820Sjeff &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey); 473219820Sjeff} 474219820Sjeff 475219820Sjeffint mlx4_ib_unmap_fmr(struct list_head *fmr_list) 476219820Sjeff{ 477219820Sjeff struct ib_fmr *ibfmr; 478219820Sjeff int err; 479219820Sjeff struct mlx4_dev *mdev = NULL; 480219820Sjeff 481219820Sjeff list_for_each_entry(ibfmr, fmr_list, list) { 482219820Sjeff if (mdev && to_mdev(ibfmr->device)->dev != mdev) 483219820Sjeff return -EINVAL; 484219820Sjeff mdev = to_mdev(ibfmr->device)->dev; 485219820Sjeff } 486219820Sjeff 487219820Sjeff if (!mdev) 488219820Sjeff return 0; 489219820Sjeff 490219820Sjeff list_for_each_entry(ibfmr, fmr_list, list) { 491219820Sjeff struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); 492219820Sjeff 493219820Sjeff mlx4_fmr_unmap(mdev, &ifmr->mfmr, &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey); 494219820Sjeff } 495219820Sjeff 496219820Sjeff /* 497219820Sjeff * Make sure all MPT status updates are visible before issuing 498219820Sjeff * SYNC_TPT firmware command. 499219820Sjeff */ 500219820Sjeff wmb(); 501219820Sjeff 502219820Sjeff err = mlx4_SYNC_TPT(mdev); 503219820Sjeff if (err) 504255932Salfred pr_warn("SYNC_TPT error %d when " 505219820Sjeff "unmapping FMRs\n", err); 506219820Sjeff 507219820Sjeff return 0; 508219820Sjeff} 509219820Sjeff 510219820Sjeffint mlx4_ib_fmr_dealloc(struct ib_fmr *ibfmr) 511219820Sjeff{ 512219820Sjeff struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); 513219820Sjeff struct mlx4_ib_dev *dev = to_mdev(ibfmr->device); 514219820Sjeff int err; 515219820Sjeff 516219820Sjeff err = mlx4_fmr_free(dev->dev, &ifmr->mfmr); 517219820Sjeff 518219820Sjeff if (!err) 519219820Sjeff kfree(ifmr); 520219820Sjeff 521219820Sjeff return err; 522219820Sjeff} 523331769Shselasky 524331769Shselaskystatic int mlx4_set_page(struct ib_mr *ibmr, u64 addr) 525331769Shselasky{ 526331769Shselasky struct mlx4_ib_mr *mr = to_mmr(ibmr); 527331769Shselasky 528331769Shselasky if (unlikely(mr->npages == mr->max_pages)) 529331769Shselasky return -ENOMEM; 530331769Shselasky 531331769Shselasky mr->pages[mr->npages++] = cpu_to_be64(addr | MLX4_MTT_FLAG_PRESENT); 532331769Shselasky 533331769Shselasky return 0; 534331769Shselasky} 535331769Shselasky 536331769Shselaskyint mlx4_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 537331769Shselasky unsigned int *sg_offset) 538331769Shselasky{ 539331769Shselasky struct mlx4_ib_mr *mr = to_mmr(ibmr); 540331769Shselasky int rc; 541331769Shselasky 542331769Shselasky mr->npages = 0; 543331769Shselasky 544331769Shselasky ib_dma_sync_single_for_cpu(ibmr->device, mr->page_map, 545331769Shselasky mr->page_map_size, DMA_TO_DEVICE); 546331769Shselasky 547331769Shselasky rc = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, mlx4_set_page); 548331769Shselasky 549331769Shselasky ib_dma_sync_single_for_device(ibmr->device, mr->page_map, 550331769Shselasky mr->page_map_size, DMA_TO_DEVICE); 551331769Shselasky 552331769Shselasky return rc; 553331769Shselasky} 554331769Shselasky 555331769ShselaskyCTASSERT(sizeof(((struct ib_phys_buf *)0)->size) == 8); 556331769Shselasky 557331769Shselaskystruct ib_mr * 558331769Shselaskymlx4_ib_reg_phys_mr(struct ib_pd *pd, 559331769Shselasky struct ib_phys_buf *buffer_list, 560331769Shselasky int num_phys_buf, 561331769Shselasky int access_flags, 562331769Shselasky u64 *virt_addr) 563331769Shselasky{ 564331769Shselasky struct mlx4_ib_dev *dev = to_mdev(pd->device); 565331769Shselasky struct mlx4_ib_mr *mr; 566331769Shselasky u64 *pages; 567331769Shselasky u64 total_size; 568331769Shselasky unsigned long mask; 569331769Shselasky int shift; 570331769Shselasky int npages; 571331769Shselasky int err; 572331769Shselasky int i, j, n; 573331769Shselasky 574331769Shselasky mask = buffer_list[0].addr ^ *virt_addr; 575331769Shselasky total_size = 0; 576331769Shselasky for (i = 0; i < num_phys_buf; ++i) { 577331769Shselasky if (i != 0) 578331769Shselasky mask |= buffer_list[i].addr; 579331769Shselasky if (i != num_phys_buf - 1) 580331769Shselasky mask |= buffer_list[i].addr + buffer_list[i].size; 581331769Shselasky 582331769Shselasky total_size += buffer_list[i].size; 583331769Shselasky } 584331769Shselasky 585331769Shselasky if (mask & ~PAGE_MASK) 586331769Shselasky return ERR_PTR(-EINVAL); 587331769Shselasky 588331769Shselasky shift = __ffs(mask | 1 << 31); 589331769Shselasky 590331769Shselasky buffer_list[0].size += buffer_list[0].addr & ((1ULL << shift) - 1); 591331769Shselasky buffer_list[0].addr &= ~0ULL << shift; 592331769Shselasky 593331769Shselasky npages = 0; 594331769Shselasky for (i = 0; i < num_phys_buf; ++i) 595331769Shselasky npages += (buffer_list[i].size + (1ULL << shift) - 1) >> shift; 596331769Shselasky 597331769Shselasky if (!npages) 598331769Shselasky return ERR_PTR(-EINVAL); 599331769Shselasky 600331769Shselasky mr = kzalloc(sizeof *mr, GFP_KERNEL); 601331769Shselasky if (!mr) 602331769Shselasky return ERR_PTR(-ENOMEM); 603331769Shselasky 604331769Shselasky pages = kzalloc(sizeof(pages[0]) * npages, GFP_KERNEL); 605331769Shselasky if (!pages) { 606331769Shselasky kfree(mr); 607331769Shselasky return ERR_PTR(-ENOMEM); 608331769Shselasky } 609331769Shselasky 610331769Shselasky err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, *virt_addr, total_size, 611331769Shselasky convert_access(access_flags), npages, shift, &mr->mmr); 612331769Shselasky if (err) { 613331769Shselasky kfree(mr); 614331769Shselasky kfree(pages); 615331769Shselasky return ERR_PTR(err); 616331769Shselasky } 617331769Shselasky 618331769Shselasky n = 0; 619331769Shselasky for (i = 0; i < num_phys_buf; ++i) { 620331769Shselasky for (j = 0; 621331769Shselasky j < (buffer_list[i].size + (1ULL << shift) - 1) >> shift; 622331769Shselasky ++j) { 623331769Shselasky u64 temp = buffer_list[i].addr + ((u64) j << shift); 624331769Shselasky pages[n++] = temp; 625331769Shselasky } 626331769Shselasky } 627331769Shselasky 628331769Shselasky mr->npages = npages; 629331769Shselasky mr->max_pages = npages; 630331769Shselasky 631331769Shselasky err = mlx4_write_mtt(dev->dev, &mr->mmr.mtt, 0, npages, pages); 632331769Shselasky if (err) 633331769Shselasky goto err_mr; 634331769Shselasky 635331769Shselasky err = mlx4_mr_enable(dev->dev, &mr->mmr); 636331769Shselasky if (err) 637331769Shselasky goto err_mr; 638331769Shselasky 639331769Shselasky mr->umem = NULL; 640331769Shselasky mr->ibmr.lkey = mr->mmr.key; 641331769Shselasky mr->ibmr.rkey = mr->mmr.key; 642331769Shselasky mr->ibmr.length = total_size; 643331769Shselasky 644331769Shselasky kfree(pages); 645331769Shselasky 646331769Shselasky return &mr->ibmr; 647331769Shselasky 648331769Shselaskyerr_mr: 649331769Shselasky (void) mlx4_mr_free(dev->dev, &mr->mmr); 650331769Shselasky kfree(mr); 651331769Shselasky kfree(pages); 652331769Shselasky 653331769Shselasky return ERR_PTR(err); 654331769Shselasky} 655