1178786Skmacy/************************************************************************** 2178786Skmacy 3178786SkmacyCopyright (c) 2007, Chelsio Inc. 4178786SkmacyAll rights reserved. 5178786Skmacy 6178786SkmacyRedistribution and use in source and binary forms, with or without 7178786Skmacymodification, are permitted provided that the following conditions are met: 8178786Skmacy 9178786Skmacy 1. Redistributions of source code must retain the above copyright notice, 10178786Skmacy this list of conditions and the following disclaimer. 11178786Skmacy 12178786Skmacy 2. Neither the name of the Chelsio Corporation nor the names of its 13178786Skmacy contributors may be used to endorse or promote products derived from 14178786Skmacy this software without specific prior written permission. 15178786Skmacy 16178786SkmacyTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17178786SkmacyAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18178786SkmacyIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19178786SkmacyARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20178786SkmacyLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21178786SkmacyCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22178786SkmacySUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23178786SkmacyINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24178786SkmacyCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25178786SkmacyARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26178786SkmacyPOSSIBILITY OF SUCH DAMAGE. 27178786Skmacy 28178786Skmacy***************************************************************************/ 29178786Skmacy#include <sys/cdefs.h> 30178786Skmacy__FBSDID("$FreeBSD$"); 31178786Skmacy 32237920Snp#include "opt_inet.h" 33237920Snp 34237920Snp#ifdef TCP_OFFLOAD 35178786Skmacy#include <sys/param.h> 36178786Skmacy#include <sys/systm.h> 37178786Skmacy#include <sys/kernel.h> 38178786Skmacy#include <sys/bus.h> 39178786Skmacy#include <sys/pciio.h> 40178786Skmacy#include <sys/conf.h> 41178786Skmacy#include <machine/bus.h> 42178786Skmacy#include <machine/resource.h> 43178786Skmacy#include <sys/bus_dma.h> 44178786Skmacy#include <sys/rman.h> 45178786Skmacy#include <sys/ioccom.h> 46178786Skmacy#include <sys/mbuf.h> 47178786Skmacy#include <sys/mutex.h> 48178786Skmacy#include <sys/rwlock.h> 49178786Skmacy#include <sys/linker.h> 50178786Skmacy#include <sys/firmware.h> 51178786Skmacy#include <sys/socket.h> 52178786Skmacy#include <sys/sockio.h> 53178786Skmacy#include <sys/smp.h> 54178786Skmacy#include <sys/sysctl.h> 55178786Skmacy#include <sys/syslog.h> 56178786Skmacy#include <sys/queue.h> 57178786Skmacy#include <sys/taskqueue.h> 58178786Skmacy#include <sys/proc.h> 59178786Skmacy#include <sys/queue.h> 60178786Skmacy#include <sys/libkern.h> 61178786Skmacy 62178786Skmacy#include <netinet/in.h> 63178786Skmacy 64237920Snp#include <rdma/ib_verbs.h> 65237920Snp#include <rdma/ib_umem.h> 66237920Snp#include <rdma/ib_user_verbs.h> 67237920Snp#include <linux/idr.h> 68237920Snp#include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h> 69178786Skmacy 70178786Skmacy#include <cxgb_include.h> 71178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_wr.h> 72178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_hal.h> 73178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_provider.h> 74178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_cm.h> 75178786Skmacy#include <ulp/iw_cxgb/iw_cxgb.h> 76178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_resource.h> 77178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_user.h> 78178786Skmacy 79237920Snpstatic int iwch_finish_mem_reg(struct iwch_mr *mhp, u32 stag) 80237920Snp{ 81237920Snp u32 mmid; 82237920Snp 83237920Snp mhp->attr.state = 1; 84237920Snp mhp->attr.stag = stag; 85237920Snp mmid = stag >> 8; 86237920Snp mhp->ibmr.rkey = mhp->ibmr.lkey = stag; 87237920Snp CTR3(KTR_IW_CXGB, "%s mmid 0x%x mhp %p", __func__, mmid, mhp); 88237920Snp return insert_handle(mhp->rhp, &mhp->rhp->mmidr, mhp, mmid); 89237920Snp} 90237920Snp 91178786Skmacyint iwch_register_mem(struct iwch_dev *rhp, struct iwch_pd *php, 92178786Skmacy struct iwch_mr *mhp, 93237920Snp int shift) 94178786Skmacy{ 95178786Skmacy u32 stag; 96237920Snp int ret; 97178786Skmacy 98178786Skmacy if (cxio_register_phys_mem(&rhp->rdev, 99178786Skmacy &stag, mhp->attr.pdid, 100178786Skmacy mhp->attr.perms, 101178786Skmacy mhp->attr.zbva, 102178786Skmacy mhp->attr.va_fbo, 103178786Skmacy mhp->attr.len, 104237920Snp shift - 12, 105237920Snp mhp->attr.pbl_size, mhp->attr.pbl_addr)) 106178786Skmacy return (-ENOMEM); 107237920Snp 108237920Snp ret = iwch_finish_mem_reg(mhp, stag); 109237920Snp if (ret) 110237920Snp cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 111237920Snp mhp->attr.pbl_addr); 112237920Snp return ret; 113178786Skmacy} 114178786Skmacy 115178786Skmacyint iwch_reregister_mem(struct iwch_dev *rhp, struct iwch_pd *php, 116178786Skmacy struct iwch_mr *mhp, 117178786Skmacy int shift, 118178786Skmacy int npages) 119178786Skmacy{ 120178786Skmacy u32 stag; 121237920Snp int ret; 122178786Skmacy 123178786Skmacy /* We could support this... */ 124178786Skmacy if (npages > mhp->attr.pbl_size) 125178786Skmacy return (-ENOMEM); 126178786Skmacy 127178786Skmacy stag = mhp->attr.stag; 128178786Skmacy if (cxio_reregister_phys_mem(&rhp->rdev, 129178786Skmacy &stag, mhp->attr.pdid, 130178786Skmacy mhp->attr.perms, 131178786Skmacy mhp->attr.zbva, 132178786Skmacy mhp->attr.va_fbo, 133178786Skmacy mhp->attr.len, 134237920Snp shift - 12, 135237920Snp mhp->attr.pbl_size, mhp->attr.pbl_addr)) 136178786Skmacy return (-ENOMEM); 137237920Snp 138237920Snp ret = iwch_finish_mem_reg(mhp, stag); 139237920Snp if (ret) 140237920Snp cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 141237920Snp mhp->attr.pbl_addr); 142237920Snp return ret; 143237920Snp} 144237920Snp 145237920Snpint iwch_alloc_pbl(struct iwch_mr *mhp, int npages) 146237920Snp{ 147237920Snp mhp->attr.pbl_addr = cxio_hal_pblpool_alloc(&mhp->rhp->rdev, 148237920Snp npages << 3); 149237920Snp 150237920Snp if (!mhp->attr.pbl_addr) 151237920Snp return -ENOMEM; 152237920Snp 153237920Snp mhp->attr.pbl_size = npages; 154237920Snp 155178786Skmacy return 0; 156237920Snp } 157237920Snp 158237920Snpvoid iwch_free_pbl(struct iwch_mr *mhp) 159237920Snp{ 160237920Snp cxio_hal_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, 161237920Snp mhp->attr.pbl_size << 3); 162178786Skmacy} 163178786Skmacy 164237920Snpint iwch_write_pbl(struct iwch_mr *mhp, __be64 *pages, int npages, int offset) 165237920Snp{ 166237920Snp return cxio_write_pbl(&mhp->rhp->rdev, pages, 167237920Snp mhp->attr.pbl_addr + (offset << 3), npages); 168237920Snp} 169237920Snp 170178786Skmacyint build_phys_page_list(struct ib_phys_buf *buffer_list, 171178786Skmacy int num_phys_buf, 172178786Skmacy u64 *iova_start, 173178786Skmacy u64 *total_size, 174178786Skmacy int *npages, 175178786Skmacy int *shift, 176178786Skmacy __be64 **page_list) 177178786Skmacy{ 178178786Skmacy u64 mask; 179178786Skmacy int i, j, n; 180178786Skmacy 181178786Skmacy mask = 0; 182178786Skmacy *total_size = 0; 183178786Skmacy for (i = 0; i < num_phys_buf; ++i) { 184178786Skmacy if (i != 0 && buffer_list[i].addr & ~PAGE_MASK) 185178786Skmacy return (-EINVAL); 186178786Skmacy if (i != 0 && i != num_phys_buf - 1 && 187178786Skmacy (buffer_list[i].size & ~PAGE_MASK)) 188178786Skmacy return (-EINVAL); 189178786Skmacy *total_size += buffer_list[i].size; 190178786Skmacy if (i > 0) 191178786Skmacy mask |= buffer_list[i].addr; 192178786Skmacy else 193178786Skmacy mask |= buffer_list[i].addr & PAGE_MASK; 194178786Skmacy if (i != num_phys_buf - 1) 195178786Skmacy mask |= buffer_list[i].addr + buffer_list[i].size; 196178786Skmacy else 197178786Skmacy mask |= (buffer_list[i].addr + buffer_list[i].size + 198178786Skmacy PAGE_SIZE - 1) & PAGE_MASK; 199178786Skmacy } 200178786Skmacy 201178786Skmacy if (*total_size > 0xFFFFFFFFULL) 202178786Skmacy return (-ENOMEM); 203178786Skmacy 204178786Skmacy /* Find largest page shift we can use to cover buffers */ 205178786Skmacy for (*shift = PAGE_SHIFT; *shift < 27; ++(*shift)) 206178786Skmacy if ((1ULL << *shift) & mask) 207178786Skmacy break; 208178786Skmacy 209178786Skmacy buffer_list[0].size += buffer_list[0].addr & ((1ULL << *shift) - 1); 210178786Skmacy buffer_list[0].addr &= ~0ull << *shift; 211178786Skmacy 212178786Skmacy *npages = 0; 213178786Skmacy for (i = 0; i < num_phys_buf; ++i) 214178786Skmacy *npages += (buffer_list[i].size + 215178786Skmacy (1ULL << *shift) - 1) >> *shift; 216178786Skmacy 217178786Skmacy if (!*npages) 218178786Skmacy return (-EINVAL); 219178786Skmacy 220178786Skmacy *page_list = kmalloc(sizeof(u64) * *npages, M_NOWAIT); 221178786Skmacy if (!*page_list) 222178786Skmacy return (-ENOMEM); 223178786Skmacy 224178786Skmacy n = 0; 225178786Skmacy for (i = 0; i < num_phys_buf; ++i) 226178786Skmacy for (j = 0; 227178786Skmacy j < (buffer_list[i].size + (1ULL << *shift) - 1) >> *shift; 228178786Skmacy ++j) 229178786Skmacy (*page_list)[n++] = htobe64(buffer_list[i].addr + 230178786Skmacy ((u64) j << *shift)); 231178786Skmacy 232178786Skmacy CTR6(KTR_IW_CXGB, "%s va 0x%llx mask 0x%llx shift %d len %lld pbl_size %d", 233178786Skmacy __FUNCTION__, (unsigned long long) *iova_start, 234178786Skmacy (unsigned long long) mask, *shift, (unsigned long long) *total_size, 235178786Skmacy *npages); 236178786Skmacy 237178786Skmacy return 0; 238178786Skmacy 239178786Skmacy} 240237920Snp#endif 241