1292740Snp/*- 2292740Snp * Copyright (c) 2012 Chelsio Communications, Inc. 3292740Snp * All rights reserved. 4292740Snp * 5292740Snp * Chelsio T5xx iSCSI driver 6292740Snp * cxgbei_ulp2_ddp.c: Chelsio iSCSI DDP Manager. 7292740Snp * 8292740Snp * Redistribution and use in source and binary forms, with or without 9292740Snp * modification, are permitted provided that the following conditions 10292740Snp * are met: 11292740Snp * 1. Redistributions of source code must retain the above copyright 12292740Snp * notice, this list of conditions and the following disclaimer. 13292740Snp * 2. Redistributions in binary form must reproduce the above copyright 14292740Snp * notice, this list of conditions and the following disclaimer in the 15292740Snp * documentation and/or other materials provided with the distribution. 16292740Snp * 17292740Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18292740Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19292740Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20292740Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21292740Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22292740Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23292740Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24292740Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25292740Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26292740Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27292740Snp * SUCH DAMAGE. 28292740Snp */ 29292740Snp 30292740Snp#include <sys/cdefs.h> 31292740Snp__FBSDID("$FreeBSD: releng/11.0/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.c 292740 2015-12-26 06:05:21Z np $"); 32292740Snp 33292740Snp#include "opt_inet.h" 34292740Snp#include "opt_inet6.h" 35292740Snp 36292740Snp#ifdef TCP_OFFLOAD 37292740Snp#include <sys/types.h> 38292740Snp#include <sys/module.h> 39292740Snp#include <sys/systm.h> 40292740Snp#include <sys/errno.h> 41292740Snp#include <sys/param.h> 42292740Snp#include <sys/kernel.h> 43292740Snp#include <sys/socket.h> 44292740Snp#include <sys/socketvar.h> 45292740Snp#include <sys/mbuf.h> 46292740Snp#include <sys/lock.h> 47292740Snp#include <sys/mutex.h> 48292740Snp#include <sys/condvar.h> 49292740Snp 50292740Snp#include <netinet/in.h> 51292740Snp#include <netinet/in_pcb.h> 52292740Snp#include <netinet/toecore.h> 53292740Snp#include <netinet/tcp_var.h> 54292740Snp#include <netinet/tcp_fsm.h> 55292740Snp 56292740Snp#include <dev/iscsi/icl.h> 57292740Snp#include <dev/iscsi/iscsi_proto.h> 58292740Snp 59292740Snp#include "common/common.h" 60292740Snp#include "common/t4_msg.h" 61292740Snp#include "common/t4_regs.h" /* for PCIE_MEM_ACCESS */ 62292740Snp#include "tom/t4_tom.h" 63292740Snp#include "cxgbei.h" 64292740Snp#include "cxgbei_ulp2_ddp.h" 65292740Snp 66292740Snp/* 67292740Snp * Map a single buffer address. 68292740Snp */ 69292740Snpstatic void 70292740Snpulp2_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error) 71292740Snp{ 72292740Snp bus_addr_t *ba = arg; 73292740Snp if (error) 74292740Snp return; 75292740Snp 76292740Snp KASSERT(nseg == 1, ("%s: %d segments returned!", __func__, nseg)); 77292740Snp 78292740Snp *ba = segs->ds_addr; 79292740Snp} 80292740Snp 81292740Snp/* 82292740Snp * iSCSI Direct Data Placement 83292740Snp * 84292740Snp * T4/5 ulp2 h/w can directly place the iSCSI Data-In or Data-Out PDU's 85292740Snp * payload into pre-posted final destination host-memory buffers based on the 86292740Snp * Initiator Task Tag (ITT) in Data-In or Target Task Tag (TTT) in Data-Out 87292740Snp * PDUs. 88292740Snp * 89292740Snp * The host memory address is programmed into h/w in the format of pagepod 90292740Snp * entries. 91292740Snp * The location of the pagepod entry is encoded into ddp tag which is used or 92292740Snp * is the base for ITT/TTT. 93292740Snp */ 94292740Snp 95292740Snp 96292740Snpstatic inline int 97292740Snpddp_find_unused_entries(struct cxgbei_data *ci, u_int start, u_int max, 98292740Snp u_int count, u_int *idx, struct cxgbei_ulp2_gather_list *gl) 99292740Snp{ 100292740Snp unsigned int i, j, k; 101292740Snp 102292740Snp /* not enough entries */ 103292740Snp if (max - start < count) 104292740Snp return (EBUSY); 105292740Snp 106292740Snp max -= count; 107292740Snp mtx_lock(&ci->map_lock); 108292740Snp for (i = start; i < max;) { 109292740Snp for (j = 0, k = i; j < count; j++, k++) { 110292740Snp if (ci->gl_map[k]) 111292740Snp break; 112292740Snp } 113292740Snp if (j == count) { 114292740Snp for (j = 0, k = i; j < count; j++, k++) 115292740Snp ci->gl_map[k] = gl; 116292740Snp mtx_unlock(&ci->map_lock); 117292740Snp *idx = i; 118292740Snp return (0); 119292740Snp } 120292740Snp i += j + 1; 121292740Snp } 122292740Snp mtx_unlock(&ci->map_lock); 123292740Snp return (EBUSY); 124292740Snp} 125292740Snp 126292740Snpstatic inline void 127292740Snpddp_unmark_entries(struct cxgbei_data *ci, u_int start, u_int count) 128292740Snp{ 129292740Snp 130292740Snp mtx_lock(&ci->map_lock); 131292740Snp memset(&ci->gl_map[start], 0, 132292740Snp count * sizeof(struct cxgbei_ulp2_gather_list *)); 133292740Snp mtx_unlock(&ci->map_lock); 134292740Snp} 135292740Snp 136292740Snpstatic inline void 137292740Snpddp_gl_unmap(struct cxgbei_data *ci, struct cxgbei_ulp2_gather_list *gl) 138292740Snp{ 139292740Snp int i; 140292740Snp 141292740Snp if (!gl->pages[0]) 142292740Snp return; 143292740Snp 144292740Snp for (i = 0; i < gl->nelem; i++) { 145292740Snp bus_dmamap_unload(ci->ulp_ddp_tag, gl->dma_sg[i].bus_map); 146292740Snp bus_dmamap_destroy(ci->ulp_ddp_tag, gl->dma_sg[i].bus_map); 147292740Snp } 148292740Snp} 149292740Snp 150292740Snpstatic inline int 151292740Snpddp_gl_map(struct cxgbei_data *ci, struct cxgbei_ulp2_gather_list *gl) 152292740Snp{ 153292740Snp int i, rc; 154292740Snp bus_addr_t pa; 155292740Snp 156292740Snp MPASS(ci != NULL); 157292740Snp 158292740Snp mtx_lock(&ci->map_lock); 159292740Snp for (i = 0; i < gl->nelem; i++) { 160292740Snp rc = bus_dmamap_create(ci->ulp_ddp_tag, 0, 161292740Snp &gl->dma_sg[i].bus_map); 162292740Snp if (rc != 0) 163292740Snp goto unmap; 164292740Snp rc = bus_dmamap_load(ci->ulp_ddp_tag, gl->dma_sg[i].bus_map, 165292740Snp gl->pages[i], PAGE_SIZE, ulp2_dma_map_addr, 166292740Snp &pa, BUS_DMA_NOWAIT); 167292740Snp if (rc != 0) 168292740Snp goto unmap; 169292740Snp gl->dma_sg[i].phys_addr = pa; 170292740Snp } 171292740Snp mtx_unlock(&ci->map_lock); 172292740Snp 173292740Snp return (0); 174292740Snp 175292740Snpunmap: 176292740Snp if (i) { 177292740Snp u_int nelem = gl->nelem; 178292740Snp 179292740Snp gl->nelem = i; 180292740Snp ddp_gl_unmap(ci, gl); 181292740Snp gl->nelem = nelem; 182292740Snp } 183292740Snp return (ENOMEM); 184292740Snp} 185292740Snp 186292740Snp/** 187292740Snp * cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec - build ddp page buffer list 188292740Snp * @xferlen: total buffer length 189292740Snp * @sgl: page buffer scatter-gather list (struct cxgbei_sgl) 190292740Snp * @sgcnt: # of page buffers 191292740Snp * @gfp: allocation mode 192292740Snp * 193292740Snp * construct a ddp page buffer list from the scsi scattergather list. 194292740Snp * coalesce buffers as much as possible, and obtain dma addresses for 195292740Snp * each page. 196292740Snp * 197292740Snp * Return the cxgbei_ulp2_gather_list constructed from the page buffers if the 198292740Snp * memory can be used for ddp. Return NULL otherwise. 199292740Snp */ 200292740Snpstruct cxgbei_ulp2_gather_list * 201292740Snpcxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(u_int xferlen, struct cxgbei_sgl *sgl, 202292740Snp u_int sgcnt, struct cxgbei_data *ci, int gfp) 203292740Snp{ 204292740Snp struct cxgbei_ulp2_gather_list *gl; 205292740Snp struct cxgbei_sgl *sg = sgl; 206292740Snp void *sgpage = (void *)((u64)sg->sg_addr & (~PAGE_MASK)); 207292740Snp unsigned int sglen = sg->sg_length; 208292740Snp unsigned int sgoffset = (u64)sg->sg_addr & PAGE_MASK; 209292740Snp unsigned int npages = (xferlen + sgoffset + PAGE_SIZE - 1) >> 210292740Snp PAGE_SHIFT; 211292740Snp int i = 1, j = 0; 212292740Snp 213292740Snp if (xferlen <= DDP_THRESHOLD) { 214292740Snp CTR2(KTR_CXGBE, "xfer %u < threshold %u, no ddp.", 215292740Snp xferlen, DDP_THRESHOLD); 216292740Snp return NULL; 217292740Snp } 218292740Snp 219292740Snp gl = malloc(sizeof(struct cxgbei_ulp2_gather_list) + 220292740Snp npages * (sizeof(struct dma_segments) + sizeof(void *)), 221292740Snp M_DEVBUF, M_NOWAIT | M_ZERO); 222292740Snp if (gl == NULL) 223292740Snp return (NULL); 224292740Snp 225292740Snp gl->pages = (void **)&gl->dma_sg[npages]; 226292740Snp gl->length = xferlen; 227292740Snp gl->offset = sgoffset; 228292740Snp gl->pages[0] = sgpage; 229292740Snp CTR6(KTR_CXGBE, 230292740Snp "%s: xferlen:0x%x len:0x%x off:0x%x sg_addr:%p npages:%d", 231292740Snp __func__, xferlen, gl->length, gl->offset, sg->sg_addr, npages); 232292740Snp 233292740Snp for (i = 1, sg = sg_next(sg); i < sgcnt; i++, sg = sg_next(sg)) { 234292740Snp void *page = sg->sg_addr; 235292740Snp 236292740Snp if (sgpage == page && sg->sg_offset == sgoffset + sglen) 237292740Snp sglen += sg->sg_length; 238292740Snp else { 239292740Snp /* make sure the sgl is fit for ddp: 240292740Snp * each has the same page size, and 241292740Snp * all of the middle pages are used completely 242292740Snp */ 243292740Snp if ((j && sgoffset) || 244292740Snp ((i != sgcnt - 1) && 245292740Snp ((sglen + sgoffset) & ~CXGBEI_PAGE_MASK))){ 246292740Snp goto error_out; 247292740Snp } 248292740Snp 249292740Snp j++; 250292740Snp if (j == gl->nelem || sg->sg_offset) { 251292740Snp goto error_out; 252292740Snp } 253292740Snp gl->pages[j] = page; 254292740Snp sglen = sg->sg_length; 255292740Snp sgoffset = sg->sg_offset; 256292740Snp sgpage = page; 257292740Snp } 258292740Snp } 259292740Snp gl->nelem = ++j; 260292740Snp 261292740Snp if (ddp_gl_map(ci, gl) < 0) 262292740Snp goto error_out; 263292740Snp 264292740Snp return gl; 265292740Snp 266292740Snperror_out: 267292740Snp free(gl, M_DEVBUF); 268292740Snp return NULL; 269292740Snp} 270292740Snp 271292740Snp/** 272292740Snp * cxgbei_ulp2_ddp_release_gl - release a page buffer list 273292740Snp * @gl: a ddp page buffer list 274292740Snp * @pdev: pci_dev used for pci_unmap 275292740Snp * free a ddp page buffer list resulted from cxgbei_ulp2_ddp_make_gl(). 276292740Snp */ 277292740Snpvoid 278292740Snpcxgbei_ulp2_ddp_release_gl(struct cxgbei_data *ci, 279292740Snp struct cxgbei_ulp2_gather_list *gl) 280292740Snp{ 281292740Snp 282292740Snp ddp_gl_unmap(ci, gl); 283292740Snp free(gl, M_DEVBUF); 284292740Snp} 285292740Snp 286292740Snp/** 287292740Snp * cxgbei_ulp2_ddp_tag_reserve - set up ddp for a data transfer 288292740Snp * @ci: adapter's ddp info 289292740Snp * @tid: connection id 290292740Snp * @tformat: tag format 291292740Snp * @tagp: contains s/w tag initially, will be updated with ddp/hw tag 292292740Snp * @gl: the page momory list 293292740Snp * @gfp: allocation mode 294292740Snp * 295292740Snp * ddp setup for a given page buffer list and construct the ddp tag. 296292740Snp * return 0 if success, < 0 otherwise. 297292740Snp */ 298292740Snpint 299292740Snpcxgbei_ulp2_ddp_tag_reserve(struct cxgbei_data *ci, void *icc, u_int tid, 300292740Snp struct cxgbei_ulp2_tag_format *tformat, u32 *tagp, 301292740Snp struct cxgbei_ulp2_gather_list *gl, int gfp, int reply) 302292740Snp{ 303292740Snp struct cxgbei_ulp2_pagepod_hdr hdr; 304292740Snp u_int npods, idx; 305292740Snp int rc; 306292740Snp u32 sw_tag = *tagp; 307292740Snp u32 tag; 308292740Snp 309292740Snp MPASS(ci != NULL); 310292740Snp 311292740Snp if (!gl || !gl->nelem || gl->length < DDP_THRESHOLD) 312292740Snp return (EINVAL); 313292740Snp 314292740Snp npods = (gl->nelem + IPPOD_PAGES_MAX - 1) >> IPPOD_PAGES_SHIFT; 315292740Snp 316292740Snp if (ci->idx_last == ci->nppods) 317292740Snp rc = ddp_find_unused_entries(ci, 0, ci->nppods, npods, &idx, 318292740Snp gl); 319292740Snp else { 320292740Snp rc = ddp_find_unused_entries(ci, ci->idx_last + 1, 321292740Snp ci->nppods, npods, &idx, gl); 322292740Snp if (rc && ci->idx_last >= npods) { 323292740Snp rc = ddp_find_unused_entries(ci, 0, 324292740Snp min(ci->idx_last + npods, ci->nppods), 325292740Snp npods, &idx, gl); 326292740Snp } 327292740Snp } 328292740Snp if (rc) { 329292740Snp CTR3(KTR_CXGBE, "xferlen %u, gl %u, npods %u NO DDP.", 330292740Snp gl->length, gl->nelem, npods); 331292740Snp return (rc); 332292740Snp } 333292740Snp 334292740Snp tag = cxgbei_ulp2_ddp_tag_base(idx, ci->colors, tformat, sw_tag); 335292740Snp CTR4(KTR_CXGBE, "%s: sw_tag:0x%x idx:0x%x tag:0x%x", 336292740Snp __func__, sw_tag, idx, tag); 337292740Snp 338292740Snp hdr.rsvd = 0; 339292740Snp hdr.vld_tid = htonl(F_IPPOD_VALID | V_IPPOD_TID(tid)); 340292740Snp hdr.pgsz_tag_clr = htonl(tag & ci->rsvd_tag_mask); 341292740Snp hdr.maxoffset = htonl(gl->length); 342292740Snp hdr.pgoffset = htonl(gl->offset); 343292740Snp 344292740Snp rc = t4_ddp_set_map(ci, icc, &hdr, idx, npods, gl, reply); 345292740Snp if (rc < 0) 346292740Snp goto unmark_entries; 347292740Snp 348292740Snp ci->idx_last = idx; 349292740Snp *tagp = tag; 350292740Snp return (0); 351292740Snp 352292740Snpunmark_entries: 353292740Snp ddp_unmark_entries(ci, idx, npods); 354292740Snp return (rc); 355292740Snp} 356292740Snp 357292740Snp/** 358292740Snp * cxgbei_ulp2_ddp_tag_release - release a ddp tag 359292740Snp * @ci: adapter's ddp info 360292740Snp * @tag: ddp tag 361292740Snp * ddp cleanup for a given ddp tag and release all the resources held 362292740Snp */ 363292740Snpvoid 364292740Snpcxgbei_ulp2_ddp_tag_release(struct cxgbei_data *ci, uint32_t tag, 365292740Snp struct icl_cxgbei_conn *icc) 366292740Snp{ 367292740Snp uint32_t idx; 368292740Snp 369292740Snp MPASS(ci != NULL); 370292740Snp MPASS(icc != NULL); 371292740Snp 372292740Snp idx = (tag >> IPPOD_IDX_SHIFT) & ci->idx_mask; 373292740Snp CTR3(KTR_CXGBE, "tag:0x%x idx:0x%x nppods:0x%x", 374292740Snp tag, idx, ci->nppods); 375292740Snp if (idx < ci->nppods) { 376292740Snp struct cxgbei_ulp2_gather_list *gl = ci->gl_map[idx]; 377292740Snp unsigned int npods; 378292740Snp 379292740Snp if (!gl || !gl->nelem) { 380292740Snp CTR4(KTR_CXGBE, 381292740Snp "release 0x%x, idx 0x%x, gl 0x%p, %u.", 382292740Snp tag, idx, gl, gl ? gl->nelem : 0); 383292740Snp return; 384292740Snp } 385292740Snp npods = (gl->nelem + IPPOD_PAGES_MAX - 1) >> IPPOD_PAGES_SHIFT; 386292740Snp CTR3(KTR_CXGBE, "ddp tag 0x%x, release idx 0x%x, npods %u.", 387292740Snp tag, idx, npods); 388292740Snp t4_ddp_clear_map(ci, gl, tag, idx, npods, icc); 389292740Snp ddp_unmark_entries(ci, idx, npods); 390292740Snp cxgbei_ulp2_ddp_release_gl(ci, gl); 391292740Snp } else 392292740Snp CTR3(KTR_CXGBE, "ddp tag 0x%x, idx 0x%x > max 0x%x.", 393292740Snp tag, idx, ci->nppods); 394292740Snp} 395292740Snp 396292740Snp/** 397292740Snp * cxgbei_ddp_cleanup - release the adapter's ddp resources 398292740Snp */ 399292740Snpvoid 400292740Snpcxgbei_ddp_cleanup(struct cxgbei_data *ci) 401292740Snp{ 402292740Snp int i = 0; 403292740Snp 404292740Snp while (i < ci->nppods) { 405292740Snp struct cxgbei_ulp2_gather_list *gl = ci->gl_map[i]; 406292740Snp if (gl) { 407292740Snp int npods = (gl->nelem + IPPOD_PAGES_MAX - 1) 408292740Snp >> IPPOD_PAGES_SHIFT; 409292740Snp free(gl, M_DEVBUF); 410292740Snp i += npods; 411292740Snp } else 412292740Snp i++; 413292740Snp } 414292740Snp free(ci->colors, M_CXGBE); 415292740Snp free(ci->gl_map, M_CXGBE); 416292740Snp} 417292740Snp#endif 418