1331722Seadler/* 2256694Snp * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved. 3256694Snp * 4256694Snp * This software is available to you under a choice of one of two 5256694Snp * licenses. You may choose to be licensed under the terms of the GNU 6256694Snp * General Public License (GPL) Version 2, available from the file 7256694Snp * COPYING in the main directory of this source tree, or the 8256694Snp * OpenIB.org BSD license below: 9256694Snp * 10256694Snp * Redistribution and use in source and binary forms, with or 11256694Snp * without modification, are permitted provided that the following 12256694Snp * conditions are met: 13256694Snp * 14256694Snp * - Redistributions of source code must retain the above 15256694Snp * copyright notice, this list of conditions and the following 16256694Snp * disclaimer. 17256694Snp * 18256694Snp * - Redistributions in binary form must reproduce the above 19256694Snp * copyright notice, this list of conditions and the following 20256694Snp * disclaimer in the documentation and/or other materials 21256694Snp * provided with the distribution. 22256694Snp * 23256694Snp * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24256694Snp * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25256694Snp * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26256694Snp * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27256694Snp * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28256694Snp * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29256694Snp * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30256694Snp * SOFTWARE. 31256694Snp */ 32256694Snp#include <sys/cdefs.h> 33256694Snp__FBSDID("$FreeBSD: stable/11/sys/dev/cxgbe/iw_cxgbe/mem.c 346950 2019-04-30 07:53:15Z np $"); 34256694Snp 35256694Snp#include "opt_inet.h" 36256694Snp 37256694Snp#ifdef TCP_OFFLOAD 38256694Snp#include <linux/types.h> 39256694Snp#include <linux/kref.h> 40256694Snp#include <rdma/ib_umem.h> 41256694Snp#include <asm/atomic.h> 42256694Snp 43256694Snp#include <common/t4_msg.h> 44256694Snp#include "iw_cxgbe.h" 45256694Snp 46256694Snp#define T4_ULPTX_MIN_IO 32 47256694Snp#define C4IW_MAX_INLINE_SIZE 96 48346923Snp#define T4_ULPTX_MAX_DMA 1024 49256694Snp 50318798Snpstatic int 51318798Snpmr_exceeds_hw_limits(struct c4iw_dev *dev, u64 length) 52297124Snp{ 53318798Snp 54331769Shselasky return (is_t5(dev->rdev.adap) && length >= 8*1024*1024*1024ULL); 55297124Snp} 56318798Snp 57256694Snpstatic int 58346923Snp_c4iw_write_mem_dma_aligned(struct c4iw_rdev *rdev, u32 addr, u32 len, 59346923Snp void *data, int wait) 60256694Snp{ 61256694Snp struct adapter *sc = rdev->adap; 62256694Snp struct ulp_mem_io *ulpmc; 63346923Snp struct ulptx_sgl *sgl; 64346923Snp u8 wr_len; 65346923Snp int ret = 0; 66346923Snp struct c4iw_wr_wait wr_wait; 67346923Snp struct wrqe *wr; 68346923Snp 69346923Snp addr &= 0x7FFFFFF; 70346923Snp 71346923Snp if (wait) 72346923Snp c4iw_init_wr_wait(&wr_wait); 73346923Snp wr_len = roundup(sizeof *ulpmc + sizeof *sgl, 16); 74346923Snp 75346923Snp wr = alloc_wrqe(wr_len, &sc->sge.ctrlq[0]); 76346923Snp if (wr == NULL) 77346923Snp return -ENOMEM; 78346923Snp ulpmc = wrtod(wr); 79346923Snp 80346923Snp memset(ulpmc, 0, wr_len); 81346923Snp INIT_ULPTX_WR(ulpmc, wr_len, 0, 0); 82346923Snp ulpmc->wr.wr_hi = cpu_to_be32(V_FW_WR_OP(FW_ULPTX_WR) | 83346923Snp (wait ? F_FW_WR_COMPL : 0)); 84346923Snp ulpmc->wr.wr_lo = wait ? (u64)(unsigned long)&wr_wait : 0; 85346923Snp ulpmc->wr.wr_mid = cpu_to_be32(V_FW_WR_LEN16(DIV_ROUND_UP(wr_len, 16))); 86346923Snp ulpmc->cmd = cpu_to_be32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | 87346923Snp V_T5_ULP_MEMIO_ORDER(1) | 88346923Snp V_T5_ULP_MEMIO_FID(sc->sge.ofld_rxq[0].iq.abs_id)); 89346923Snp ulpmc->dlen = cpu_to_be32(V_ULP_MEMIO_DATA_LEN(len>>5)); 90346923Snp ulpmc->len16 = cpu_to_be32(DIV_ROUND_UP(wr_len-sizeof(ulpmc->wr), 16)); 91346923Snp ulpmc->lock_addr = cpu_to_be32(V_ULP_MEMIO_ADDR(addr)); 92346923Snp 93346923Snp sgl = (struct ulptx_sgl *)(ulpmc + 1); 94346923Snp sgl->cmd_nsge = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 95346923Snp V_ULPTX_NSGE(1)); 96346923Snp sgl->len0 = cpu_to_be32(len); 97346923Snp sgl->addr0 = cpu_to_be64((u64)data); 98346923Snp 99346923Snp t4_wrq_tx(sc, wr); 100346923Snp 101346923Snp if (wait) 102346923Snp ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, NULL, __func__); 103346923Snp return ret; 104346923Snp} 105346923Snp 106346923Snp 107346923Snpstatic int 108346923Snp_c4iw_write_mem_inline(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data) 109346923Snp{ 110346923Snp struct adapter *sc = rdev->adap; 111346923Snp struct ulp_mem_io *ulpmc; 112256694Snp struct ulptx_idata *ulpsc; 113256694Snp u8 wr_len, *to_dp, *from_dp; 114256694Snp int copy_len, num_wqe, i, ret = 0; 115256694Snp struct c4iw_wr_wait wr_wait; 116256694Snp struct wrqe *wr; 117256694Snp u32 cmd; 118256694Snp 119256694Snp cmd = cpu_to_be32(V_ULPTX_CMD(ULP_TX_MEM_WRITE)); 120256694Snp 121331769Shselasky cmd |= cpu_to_be32(F_T5_ULP_MEMIO_IMM); 122331769Shselasky 123256694Snp addr &= 0x7FFFFFF; 124256694Snp CTR3(KTR_IW_CXGBE, "%s addr 0x%x len %u", __func__, addr, len); 125256694Snp num_wqe = DIV_ROUND_UP(len, C4IW_MAX_INLINE_SIZE); 126256694Snp c4iw_init_wr_wait(&wr_wait); 127256694Snp for (i = 0; i < num_wqe; i++) { 128256694Snp 129256694Snp copy_len = min(len, C4IW_MAX_INLINE_SIZE); 130256694Snp wr_len = roundup(sizeof *ulpmc + sizeof *ulpsc + 131256694Snp roundup(copy_len, T4_ULPTX_MIN_IO), 16); 132256694Snp 133346876Snp wr = alloc_wrqe(wr_len, &sc->sge.ctrlq[0]); 134256694Snp if (wr == NULL) 135346923Snp return -ENOMEM; 136256694Snp ulpmc = wrtod(wr); 137256694Snp 138256694Snp memset(ulpmc, 0, wr_len); 139256694Snp INIT_ULPTX_WR(ulpmc, wr_len, 0, 0); 140256694Snp 141256694Snp if (i == (num_wqe-1)) { 142256694Snp ulpmc->wr.wr_hi = cpu_to_be32(V_FW_WR_OP(FW_ULPTX_WR) | 143256694Snp F_FW_WR_COMPL); 144346923Snp ulpmc->wr.wr_lo = 145346923Snp (__force __be64)(unsigned long) &wr_wait; 146256694Snp } else 147256694Snp ulpmc->wr.wr_hi = cpu_to_be32(V_FW_WR_OP(FW_ULPTX_WR)); 148256694Snp ulpmc->wr.wr_mid = cpu_to_be32( 149256694Snp V_FW_WR_LEN16(DIV_ROUND_UP(wr_len, 16))); 150256694Snp 151256694Snp ulpmc->cmd = cmd; 152256694Snp ulpmc->dlen = cpu_to_be32(V_ULP_MEMIO_DATA_LEN( 153256694Snp DIV_ROUND_UP(copy_len, T4_ULPTX_MIN_IO))); 154256694Snp ulpmc->len16 = cpu_to_be32(DIV_ROUND_UP(wr_len-sizeof(ulpmc->wr), 155256694Snp 16)); 156256694Snp ulpmc->lock_addr = cpu_to_be32(V_ULP_MEMIO_ADDR(addr + i * 3)); 157256694Snp 158256694Snp ulpsc = (struct ulptx_idata *)(ulpmc + 1); 159256694Snp ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_IMM)); 160256694Snp ulpsc->len = cpu_to_be32(roundup(copy_len, T4_ULPTX_MIN_IO)); 161256694Snp 162256694Snp to_dp = (u8 *)(ulpsc + 1); 163256694Snp from_dp = (u8 *)data + i * C4IW_MAX_INLINE_SIZE; 164256694Snp if (data) 165256694Snp memcpy(to_dp, from_dp, copy_len); 166256694Snp else 167256694Snp memset(to_dp, 0, copy_len); 168256694Snp if (copy_len % T4_ULPTX_MIN_IO) 169256694Snp memset(to_dp + copy_len, 0, T4_ULPTX_MIN_IO - 170256694Snp (copy_len % T4_ULPTX_MIN_IO)); 171256694Snp t4_wrq_tx(sc, wr); 172256694Snp len -= C4IW_MAX_INLINE_SIZE; 173256694Snp } 174256694Snp 175331769Shselasky ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, NULL, __func__); 176256694Snp return ret; 177256694Snp} 178256694Snp 179346923Snpstatic int 180346923Snp_c4iw_write_mem_dma(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data) 181346923Snp{ 182346923Snp struct c4iw_dev *rhp = rdev_to_c4iw_dev(rdev); 183346923Snp u32 remain = len; 184346923Snp u32 dmalen; 185346923Snp int ret = 0; 186346923Snp dma_addr_t daddr; 187346923Snp dma_addr_t save; 188346923Snp 189346923Snp daddr = dma_map_single(rhp->ibdev.dma_device, data, len, DMA_TO_DEVICE); 190346923Snp if (dma_mapping_error(rhp->ibdev.dma_device, daddr)) 191346923Snp return -1; 192346923Snp save = daddr; 193346923Snp 194346923Snp while (remain > inline_threshold) { 195346923Snp if (remain < T4_ULPTX_MAX_DMA) { 196346923Snp if (remain & ~T4_ULPTX_MIN_IO) 197346923Snp dmalen = remain & ~(T4_ULPTX_MIN_IO-1); 198346923Snp else 199346923Snp dmalen = remain; 200346923Snp } else 201346923Snp dmalen = T4_ULPTX_MAX_DMA; 202346923Snp remain -= dmalen; 203346923Snp ret = _c4iw_write_mem_dma_aligned(rdev, addr, dmalen, 204346923Snp (void *)daddr, !remain); 205346923Snp if (ret) 206346923Snp goto out; 207346923Snp addr += dmalen >> 5; 208346950Snp data = (u8 *)data + dmalen; 209346923Snp daddr = daddr + dmalen; 210346923Snp } 211346923Snp if (remain) 212346923Snp ret = _c4iw_write_mem_inline(rdev, addr, remain, data); 213346923Snpout: 214346923Snp dma_unmap_single(rhp->ibdev.dma_device, save, len, DMA_TO_DEVICE); 215346923Snp return ret; 216346923Snp} 217346923Snp 218256694Snp/* 219346923Snp * write len bytes of data into addr (32B aligned address) 220346923Snp * If data is NULL, clear len byte of memory to zero. 221346923Snp */ 222346923Snpstatic int 223346923Snpwrite_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len, 224346923Snp void *data) 225346923Snp{ 226346923Snp if (rdev->adap->params.ulptx_memwrite_dsgl && use_dsgl) { 227346923Snp if (len > inline_threshold) { 228346923Snp if (_c4iw_write_mem_dma(rdev, addr, len, data)) { 229346923Snp log(LOG_ERR, "%s: dma map " 230346923Snp "failure (non fatal)\n", __func__); 231346923Snp return _c4iw_write_mem_inline(rdev, addr, len, 232346923Snp data); 233346923Snp } else 234346923Snp return 0; 235346923Snp } else 236346923Snp return _c4iw_write_mem_inline(rdev, addr, len, data); 237346923Snp } else 238346923Snp return _c4iw_write_mem_inline(rdev, addr, len, data); 239346923Snp} 240346923Snp 241346923Snp 242346923Snp/* 243256694Snp * Build and write a TPT entry. 244256694Snp * IN: stag key, pdid, perm, bind_enabled, zbva, to, len, page_size, 245256694Snp * pbl_size and pbl_addr 246256694Snp * OUT: stag index 247256694Snp */ 248256694Snpstatic int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, 249256694Snp u32 *stag, u8 stag_state, u32 pdid, 250256694Snp enum fw_ri_stag_type type, enum fw_ri_mem_perms perm, 251256694Snp int bind_enabled, u32 zbva, u64 to, 252256694Snp u64 len, u8 page_size, u32 pbl_size, u32 pbl_addr) 253256694Snp{ 254256694Snp int err; 255256694Snp struct fw_ri_tpte tpt; 256256694Snp u32 stag_idx; 257256694Snp static atomic_t key; 258256694Snp 259256694Snp if (c4iw_fatal_error(rdev)) 260256694Snp return -EIO; 261256694Snp 262256694Snp stag_state = stag_state > 0; 263256694Snp stag_idx = (*stag) >> 8; 264256694Snp 265256694Snp if ((!reset_tpt_entry) && (*stag == T4_STAG_UNSET)) { 266256694Snp stag_idx = c4iw_get_resource(&rdev->resource.tpt_table); 267297124Snp if (!stag_idx) { 268297124Snp mutex_lock(&rdev->stats.lock); 269297124Snp rdev->stats.stag.fail++; 270297124Snp mutex_unlock(&rdev->stats.lock); 271256694Snp return -ENOMEM; 272297124Snp } 273256694Snp mutex_lock(&rdev->stats.lock); 274256694Snp rdev->stats.stag.cur += 32; 275256694Snp if (rdev->stats.stag.cur > rdev->stats.stag.max) 276256694Snp rdev->stats.stag.max = rdev->stats.stag.cur; 277256694Snp mutex_unlock(&rdev->stats.lock); 278256694Snp *stag = (stag_idx << 8) | (atomic_inc_return(&key) & 0xff); 279256694Snp } 280256694Snp CTR5(KTR_IW_CXGBE, 281256694Snp "%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x", 282256694Snp __func__, stag_state, type, pdid, stag_idx); 283256694Snp 284256694Snp /* write TPT entry */ 285256694Snp if (reset_tpt_entry) 286256694Snp memset(&tpt, 0, sizeof(tpt)); 287256694Snp else { 288256694Snp tpt.valid_to_pdid = cpu_to_be32(F_FW_RI_TPTE_VALID | 289256694Snp V_FW_RI_TPTE_STAGKEY((*stag & M_FW_RI_TPTE_STAGKEY)) | 290256694Snp V_FW_RI_TPTE_STAGSTATE(stag_state) | 291256694Snp V_FW_RI_TPTE_STAGTYPE(type) | V_FW_RI_TPTE_PDID(pdid)); 292256694Snp tpt.locread_to_qpid = cpu_to_be32(V_FW_RI_TPTE_PERM(perm) | 293256694Snp (bind_enabled ? F_FW_RI_TPTE_MWBINDEN : 0) | 294256694Snp V_FW_RI_TPTE_ADDRTYPE((zbva ? FW_RI_ZERO_BASED_TO : 295256694Snp FW_RI_VA_BASED_TO))| 296256694Snp V_FW_RI_TPTE_PS(page_size)); 297256694Snp tpt.nosnoop_pbladdr = !pbl_size ? 0 : cpu_to_be32( 298256694Snp V_FW_RI_TPTE_PBLADDR(PBL_OFF(rdev, pbl_addr)>>3)); 299256694Snp tpt.len_lo = cpu_to_be32((u32)(len & 0xffffffffUL)); 300256694Snp tpt.va_hi = cpu_to_be32((u32)(to >> 32)); 301256694Snp tpt.va_lo_fbo = cpu_to_be32((u32)(to & 0xffffffffUL)); 302256694Snp tpt.dca_mwbcnt_pstag = cpu_to_be32(0); 303256694Snp tpt.len_hi = cpu_to_be32((u32)(len >> 32)); 304256694Snp } 305256694Snp err = write_adapter_mem(rdev, stag_idx + 306256694Snp (rdev->adap->vres.stag.start >> 5), 307256694Snp sizeof(tpt), &tpt); 308256694Snp 309256694Snp if (reset_tpt_entry) { 310256694Snp c4iw_put_resource(&rdev->resource.tpt_table, stag_idx); 311256694Snp mutex_lock(&rdev->stats.lock); 312256694Snp rdev->stats.stag.cur -= 32; 313256694Snp mutex_unlock(&rdev->stats.lock); 314256694Snp } 315256694Snp return err; 316256694Snp} 317256694Snp 318256694Snpstatic int write_pbl(struct c4iw_rdev *rdev, __be64 *pbl, 319256694Snp u32 pbl_addr, u32 pbl_size) 320256694Snp{ 321256694Snp int err; 322256694Snp 323256694Snp CTR4(KTR_IW_CXGBE, "%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d", 324256694Snp __func__, pbl_addr, rdev->adap->vres.pbl.start, pbl_size); 325256694Snp 326256694Snp err = write_adapter_mem(rdev, pbl_addr >> 5, pbl_size << 3, pbl); 327256694Snp return err; 328256694Snp} 329256694Snp 330256694Snpstatic int dereg_mem(struct c4iw_rdev *rdev, u32 stag, u32 pbl_size, 331256694Snp u32 pbl_addr) 332256694Snp{ 333256694Snp return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, 334256694Snp pbl_size, pbl_addr); 335256694Snp} 336256694Snp 337256694Snpstatic int allocate_window(struct c4iw_rdev *rdev, u32 * stag, u32 pdid) 338256694Snp{ 339256694Snp *stag = T4_STAG_UNSET; 340256694Snp return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_MW, 0, 0, 0, 341256694Snp 0UL, 0, 0, 0, 0); 342256694Snp} 343256694Snp 344256694Snpstatic int deallocate_window(struct c4iw_rdev *rdev, u32 stag) 345256694Snp{ 346256694Snp return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, 0, 347256694Snp 0); 348256694Snp} 349256694Snp 350256694Snpstatic int allocate_stag(struct c4iw_rdev *rdev, u32 *stag, u32 pdid, 351256694Snp u32 pbl_size, u32 pbl_addr) 352256694Snp{ 353256694Snp *stag = T4_STAG_UNSET; 354256694Snp return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_NSMR, 0, 0, 0, 355256694Snp 0UL, 0, 0, pbl_size, pbl_addr); 356256694Snp} 357256694Snp 358256694Snpstatic int finish_mem_reg(struct c4iw_mr *mhp, u32 stag) 359256694Snp{ 360256694Snp u32 mmid; 361256694Snp 362256694Snp mhp->attr.state = 1; 363256694Snp mhp->attr.stag = stag; 364256694Snp mmid = stag >> 8; 365256694Snp mhp->ibmr.rkey = mhp->ibmr.lkey = stag; 366256694Snp CTR3(KTR_IW_CXGBE, "%s mmid 0x%x mhp %p", __func__, mmid, mhp); 367256694Snp return insert_handle(mhp->rhp, &mhp->rhp->mmidr, mhp, mmid); 368256694Snp} 369256694Snp 370256694Snpstatic int register_mem(struct c4iw_dev *rhp, struct c4iw_pd *php, 371256694Snp struct c4iw_mr *mhp, int shift) 372256694Snp{ 373256694Snp u32 stag = T4_STAG_UNSET; 374256694Snp int ret; 375256694Snp 376256694Snp ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, mhp->attr.pdid, 377297124Snp FW_RI_STAG_NSMR, mhp->attr.len ? mhp->attr.perms : 0, 378256694Snp mhp->attr.mw_bind_enable, mhp->attr.zbva, 379297124Snp mhp->attr.va_fbo, mhp->attr.len ? mhp->attr.len : -1, shift - 12, 380256694Snp mhp->attr.pbl_size, mhp->attr.pbl_addr); 381256694Snp if (ret) 382256694Snp return ret; 383256694Snp 384256694Snp ret = finish_mem_reg(mhp, stag); 385256694Snp if (ret) 386256694Snp dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 387256694Snp mhp->attr.pbl_addr); 388256694Snp return ret; 389256694Snp} 390256694Snp 391256694Snpstatic int alloc_pbl(struct c4iw_mr *mhp, int npages) 392256694Snp{ 393256694Snp mhp->attr.pbl_addr = c4iw_pblpool_alloc(&mhp->rhp->rdev, 394256694Snp npages << 3); 395256694Snp 396256694Snp if (!mhp->attr.pbl_addr) 397256694Snp return -ENOMEM; 398256694Snp 399256694Snp mhp->attr.pbl_size = npages; 400256694Snp 401256694Snp return 0; 402256694Snp} 403256694Snp 404256694Snpstruct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc) 405256694Snp{ 406256694Snp struct c4iw_dev *rhp; 407256694Snp struct c4iw_pd *php; 408256694Snp struct c4iw_mr *mhp; 409256694Snp int ret; 410256694Snp u32 stag = T4_STAG_UNSET; 411256694Snp 412256694Snp CTR2(KTR_IW_CXGBE, "%s ib_pd %p", __func__, pd); 413256694Snp php = to_c4iw_pd(pd); 414256694Snp rhp = php->rhp; 415256694Snp 416256694Snp mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); 417256694Snp if (!mhp) 418256694Snp return ERR_PTR(-ENOMEM); 419256694Snp 420256694Snp mhp->rhp = rhp; 421256694Snp mhp->attr.pdid = php->pdid; 422256694Snp mhp->attr.perms = c4iw_ib_to_tpt_access(acc); 423256694Snp mhp->attr.mw_bind_enable = (acc&IB_ACCESS_MW_BIND) == IB_ACCESS_MW_BIND; 424256694Snp mhp->attr.zbva = 0; 425256694Snp mhp->attr.va_fbo = 0; 426256694Snp mhp->attr.page_size = 0; 427331769Shselasky mhp->attr.len = ~0ULL; 428256694Snp mhp->attr.pbl_size = 0; 429256694Snp 430256694Snp ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, php->pdid, 431256694Snp FW_RI_STAG_NSMR, mhp->attr.perms, 432331769Shselasky mhp->attr.mw_bind_enable, 0, 0, ~0ULL, 0, 0, 0); 433256694Snp if (ret) 434256694Snp goto err1; 435256694Snp 436256694Snp ret = finish_mem_reg(mhp, stag); 437256694Snp if (ret) 438256694Snp goto err2; 439256694Snp return &mhp->ibmr; 440256694Snperr2: 441256694Snp dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 442256694Snp mhp->attr.pbl_addr); 443256694Snperr1: 444256694Snp kfree(mhp); 445256694Snp return ERR_PTR(ret); 446256694Snp} 447256694Snp 448256694Snpstruct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 449331769Shselasky u64 virt, int acc, struct ib_udata *udata) 450256694Snp{ 451256694Snp __be64 *pages; 452256694Snp int shift, n, len; 453278886Shselasky int i, k, entry; 454256694Snp int err = 0; 455278886Shselasky struct scatterlist *sg; 456256694Snp struct c4iw_dev *rhp; 457256694Snp struct c4iw_pd *php; 458256694Snp struct c4iw_mr *mhp; 459256694Snp 460256694Snp CTR2(KTR_IW_CXGBE, "%s ib_pd %p", __func__, pd); 461256694Snp 462256694Snp if (length == ~0ULL) 463256694Snp return ERR_PTR(-EINVAL); 464256694Snp 465256694Snp if ((length + start) < start) 466256694Snp return ERR_PTR(-EINVAL); 467256694Snp 468256694Snp php = to_c4iw_pd(pd); 469256694Snp rhp = php->rhp; 470297124Snp 471297124Snp if (mr_exceeds_hw_limits(rhp, length)) 472297124Snp return ERR_PTR(-EINVAL); 473297124Snp 474256694Snp mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); 475256694Snp if (!mhp) 476256694Snp return ERR_PTR(-ENOMEM); 477256694Snp 478256694Snp mhp->rhp = rhp; 479256694Snp 480256694Snp mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0); 481256694Snp if (IS_ERR(mhp->umem)) { 482256694Snp err = PTR_ERR(mhp->umem); 483256694Snp kfree(mhp); 484256694Snp return ERR_PTR(err); 485256694Snp } 486256694Snp 487256694Snp shift = ffs(mhp->umem->page_size) - 1; 488278886Shselasky 489278886Shselasky n = mhp->umem->nmap; 490256694Snp err = alloc_pbl(mhp, n); 491256694Snp if (err) 492256694Snp goto err; 493256694Snp 494256694Snp pages = (__be64 *) __get_free_page(GFP_KERNEL); 495256694Snp if (!pages) { 496256694Snp err = -ENOMEM; 497256694Snp goto err_pbl; 498256694Snp } 499256694Snp 500256694Snp i = n = 0; 501278886Shselasky for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) { 502278886Shselasky len = sg_dma_len(sg) >> shift; 503278886Shselasky for (k = 0; k < len; ++k) { 504278886Shselasky pages[i++] = cpu_to_be64(sg_dma_address(sg) + 505278886Shselasky mhp->umem->page_size * k); 506278886Shselasky if (i == PAGE_SIZE / sizeof *pages) { 507278886Shselasky err = write_pbl(&mhp->rhp->rdev, 508278886Shselasky pages, 509278886Shselasky mhp->attr.pbl_addr + (n << 3), i); 510278886Shselasky if (err) 511278886Shselasky goto pbl_done; 512278886Shselasky n += i; 513278886Shselasky i = 0; 514256694Snp 515256694Snp } 516256694Snp } 517278886Shselasky } 518256694Snp 519256694Snp if (i) 520256694Snp err = write_pbl(&mhp->rhp->rdev, pages, 521256694Snp mhp->attr.pbl_addr + (n << 3), i); 522256694Snp 523256694Snppbl_done: 524256694Snp free_page((unsigned long) pages); 525256694Snp if (err) 526256694Snp goto err_pbl; 527256694Snp 528256694Snp mhp->attr.pdid = php->pdid; 529256694Snp mhp->attr.zbva = 0; 530256694Snp mhp->attr.perms = c4iw_ib_to_tpt_access(acc); 531256694Snp mhp->attr.va_fbo = virt; 532256694Snp mhp->attr.page_size = shift - 12; 533256694Snp mhp->attr.len = length; 534256694Snp 535256694Snp err = register_mem(rhp, php, mhp, shift); 536256694Snp if (err) 537256694Snp goto err_pbl; 538256694Snp 539256694Snp return &mhp->ibmr; 540256694Snp 541256694Snperr_pbl: 542256694Snp c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, 543256694Snp mhp->attr.pbl_size << 3); 544256694Snp 545256694Snperr: 546256694Snp ib_umem_release(mhp->umem); 547256694Snp kfree(mhp); 548256694Snp return ERR_PTR(err); 549256694Snp} 550256694Snp 551331769Shselaskystruct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, 552331769Shselasky struct ib_udata *udata) 553256694Snp{ 554256694Snp struct c4iw_dev *rhp; 555256694Snp struct c4iw_pd *php; 556256694Snp struct c4iw_mw *mhp; 557256694Snp u32 mmid; 558256694Snp u32 stag = 0; 559256694Snp int ret; 560256694Snp 561331769Shselasky if (type != IB_MW_TYPE_1) 562331769Shselasky return ERR_PTR(-EINVAL); 563331769Shselasky 564256694Snp php = to_c4iw_pd(pd); 565256694Snp rhp = php->rhp; 566256694Snp mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); 567256694Snp if (!mhp) 568256694Snp return ERR_PTR(-ENOMEM); 569256694Snp ret = allocate_window(&rhp->rdev, &stag, php->pdid); 570256694Snp if (ret) { 571256694Snp kfree(mhp); 572256694Snp return ERR_PTR(ret); 573256694Snp } 574256694Snp mhp->rhp = rhp; 575256694Snp mhp->attr.pdid = php->pdid; 576256694Snp mhp->attr.type = FW_RI_STAG_MW; 577256694Snp mhp->attr.stag = stag; 578256694Snp mmid = (stag) >> 8; 579256694Snp mhp->ibmw.rkey = stag; 580256694Snp if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) { 581256694Snp deallocate_window(&rhp->rdev, mhp->attr.stag); 582256694Snp kfree(mhp); 583256694Snp return ERR_PTR(-ENOMEM); 584256694Snp } 585256694Snp CTR4(KTR_IW_CXGBE, "%s mmid 0x%x mhp %p stag 0x%x", __func__, mmid, mhp, 586256694Snp stag); 587256694Snp return &(mhp->ibmw); 588256694Snp} 589256694Snp 590256694Snpint c4iw_dealloc_mw(struct ib_mw *mw) 591256694Snp{ 592256694Snp struct c4iw_dev *rhp; 593256694Snp struct c4iw_mw *mhp; 594256694Snp u32 mmid; 595256694Snp 596256694Snp mhp = to_c4iw_mw(mw); 597256694Snp rhp = mhp->rhp; 598256694Snp mmid = (mw->rkey) >> 8; 599256694Snp remove_handle(rhp, &rhp->mmidr, mmid); 600256694Snp deallocate_window(&rhp->rdev, mhp->attr.stag); 601256694Snp kfree(mhp); 602256694Snp CTR4(KTR_IW_CXGBE, "%s ib_mw %p mmid 0x%x ptr %p", __func__, mw, mmid, 603256694Snp mhp); 604256694Snp return 0; 605256694Snp} 606256694Snp 607331769Shselaskystruct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, 608331769Shselasky enum ib_mr_type mr_type, 609331769Shselasky u32 max_num_sg) 610256694Snp{ 611256694Snp struct c4iw_dev *rhp; 612256694Snp struct c4iw_pd *php; 613256694Snp struct c4iw_mr *mhp; 614256694Snp u32 mmid; 615256694Snp u32 stag = 0; 616256694Snp int ret = 0; 617331769Shselasky int length = roundup(max_num_sg * sizeof(u64), 32); 618256694Snp 619256694Snp php = to_c4iw_pd(pd); 620256694Snp rhp = php->rhp; 621331769Shselasky 622331769Shselasky if (mr_type != IB_MR_TYPE_MEM_REG || 623331769Shselasky max_num_sg > t4_max_fr_depth( 624331769Shselasky rhp->rdev.adap->params.ulptx_memwrite_dsgl && use_dsgl)) 625331769Shselasky return ERR_PTR(-EINVAL); 626331769Shselasky 627256694Snp mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); 628256694Snp if (!mhp) { 629256694Snp ret = -ENOMEM; 630256694Snp goto err; 631256694Snp } 632256694Snp 633331769Shselasky mhp->mpl = dma_alloc_coherent(rhp->ibdev.dma_device, 634331769Shselasky length, &mhp->mpl_addr, GFP_KERNEL); 635331769Shselasky if (!mhp->mpl) { 636331769Shselasky ret = -ENOMEM; 637331769Shselasky goto err_mpl; 638331769Shselasky } 639331769Shselasky mhp->max_mpl_len = length; 640331769Shselasky 641256694Snp mhp->rhp = rhp; 642331769Shselasky ret = alloc_pbl(mhp, max_num_sg); 643256694Snp if (ret) 644256694Snp goto err1; 645331769Shselasky mhp->attr.pbl_size = max_num_sg; 646256694Snp ret = allocate_stag(&rhp->rdev, &stag, php->pdid, 647331769Shselasky mhp->attr.pbl_size, mhp->attr.pbl_addr); 648256694Snp if (ret) 649256694Snp goto err2; 650256694Snp mhp->attr.pdid = php->pdid; 651256694Snp mhp->attr.type = FW_RI_STAG_NSMR; 652256694Snp mhp->attr.stag = stag; 653331769Shselasky mhp->attr.state = 0; 654256694Snp mmid = (stag) >> 8; 655256694Snp mhp->ibmr.rkey = mhp->ibmr.lkey = stag; 656256694Snp if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) { 657256694Snp ret = -ENOMEM; 658256694Snp goto err3; 659256694Snp } 660256694Snp 661331769Shselasky PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag); 662256694Snp return &(mhp->ibmr); 663256694Snperr3: 664256694Snp dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size, 665256694Snp mhp->attr.pbl_addr); 666256694Snperr2: 667256694Snp c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, 668256694Snp mhp->attr.pbl_size << 3); 669256694Snperr1: 670331769Shselasky dma_free_coherent(rhp->ibdev.dma_device, 671331769Shselasky mhp->max_mpl_len, mhp->mpl, mhp->mpl_addr); 672331769Shselaskyerr_mpl: 673256694Snp kfree(mhp); 674256694Snperr: 675256694Snp return ERR_PTR(ret); 676256694Snp} 677331769Shselaskystatic int c4iw_set_page(struct ib_mr *ibmr, u64 addr) 678256694Snp{ 679331769Shselasky struct c4iw_mr *mhp = to_c4iw_mr(ibmr); 680256694Snp 681331769Shselasky if (unlikely(mhp->mpl_len == mhp->max_mpl_len)) 682331769Shselasky return -ENOMEM; 683256694Snp 684331769Shselasky mhp->mpl[mhp->mpl_len++] = addr; 685256694Snp 686331769Shselasky return 0; 687256694Snp} 688256694Snp 689331769Shselaskyint c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 690331769Shselasky int sg_nents, unsigned int *sg_offset) 691256694Snp{ 692331769Shselasky struct c4iw_mr *mhp = to_c4iw_mr(ibmr); 693331769Shselasky 694331769Shselasky mhp->mpl_len = 0; 695331769Shselasky 696331769Shselasky return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, c4iw_set_page); 697256694Snp} 698256694Snp 699331769Shselasky 700256694Snpint c4iw_dereg_mr(struct ib_mr *ib_mr) 701256694Snp{ 702256694Snp struct c4iw_dev *rhp; 703256694Snp struct c4iw_mr *mhp; 704256694Snp u32 mmid; 705256694Snp 706256694Snp CTR2(KTR_IW_CXGBE, "%s ib_mr %p", __func__, ib_mr); 707256694Snp 708256694Snp mhp = to_c4iw_mr(ib_mr); 709256694Snp rhp = mhp->rhp; 710256694Snp mmid = mhp->attr.stag >> 8; 711256694Snp remove_handle(rhp, &rhp->mmidr, mmid); 712256694Snp dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 713256694Snp mhp->attr.pbl_addr); 714256694Snp if (mhp->attr.pbl_size) 715256694Snp c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, 716256694Snp mhp->attr.pbl_size << 3); 717256694Snp if (mhp->kva) 718256694Snp kfree((void *) (unsigned long) mhp->kva); 719256694Snp if (mhp->umem) 720256694Snp ib_umem_release(mhp->umem); 721256694Snp CTR3(KTR_IW_CXGBE, "%s mmid 0x%x ptr %p", __func__, mmid, mhp); 722256694Snp kfree(mhp); 723256694Snp return 0; 724256694Snp} 725331769Shselasky 726331769Shselaskyvoid c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey) 727331769Shselasky{ 728331769Shselasky struct c4iw_mr *mhp; 729331769Shselasky unsigned long flags; 730331769Shselasky 731331769Shselasky spin_lock_irqsave(&rhp->lock, flags); 732331769Shselasky mhp = get_mhp(rhp, rkey >> 8); 733331769Shselasky if (mhp) 734331769Shselasky mhp->attr.state = 0; 735331769Shselasky spin_unlock_irqrestore(&rhp->lock, flags); 736331769Shselasky} 737256694Snp#endif 738