1178786Skmacy/************************************************************************** 2178786Skmacy 3178786SkmacyCopyright (c) 2007, Chelsio Inc. 4178786SkmacyAll rights reserved. 5178786Skmacy 6178786SkmacyRedistribution and use in source and binary forms, with or without 7178786Skmacymodification, are permitted provided that the following conditions are met: 8178786Skmacy 9178786Skmacy 1. Redistributions of source code must retain the above copyright notice, 10178786Skmacy this list of conditions and the following disclaimer. 11178786Skmacy 12178786Skmacy 2. Neither the name of the Chelsio Corporation nor the names of its 13178786Skmacy contributors may be used to endorse or promote products derived from 14178786Skmacy this software without specific prior written permission. 15178786Skmacy 16178786SkmacyTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17178786SkmacyAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18178786SkmacyIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19178786SkmacyARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20178786SkmacyLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21178786SkmacyCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22178786SkmacySUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23178786SkmacyINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24178786SkmacyCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25178786SkmacyARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26178786SkmacyPOSSIBILITY OF SUCH DAMAGE. 27178786Skmacy 28178786Skmacy***************************************************************************/ 29178786Skmacy#include <sys/cdefs.h> 30178786Skmacy__FBSDID("$FreeBSD$"); 31178786Skmacy 32237263Snp#include "opt_inet.h" 33237263Snp 34237263Snp#ifdef TCP_OFFLOAD 35178786Skmacy#include <sys/param.h> 36178786Skmacy#include <sys/systm.h> 37178786Skmacy#include <sys/kernel.h> 38178786Skmacy#include <sys/bus.h> 39178786Skmacy#include <sys/pciio.h> 40178786Skmacy#include <sys/conf.h> 41178786Skmacy#include <machine/bus.h> 42178786Skmacy#include <machine/resource.h> 43178786Skmacy#include <sys/bus_dma.h> 44178786Skmacy#include <sys/rman.h> 45178786Skmacy#include <sys/ioccom.h> 46178786Skmacy#include <sys/mbuf.h> 47178786Skmacy#include <sys/mutex.h> 48178786Skmacy#include <sys/rwlock.h> 49178786Skmacy#include <sys/linker.h> 50178786Skmacy#include <sys/firmware.h> 51178786Skmacy#include <sys/socket.h> 52178786Skmacy#include <sys/sockio.h> 53178786Skmacy#include <sys/smp.h> 54178786Skmacy#include <sys/sysctl.h> 55178786Skmacy#include <sys/syslog.h> 56178786Skmacy#include <sys/queue.h> 57178786Skmacy#include <sys/taskqueue.h> 58178786Skmacy#include <sys/proc.h> 59178786Skmacy#include <sys/queue.h> 60178786Skmacy#include <sys/libkern.h> 61178786Skmacy 62178786Skmacy#include <netinet/in.h> 63178786Skmacy 64237263Snp#include <rdma/ib_verbs.h> 65237263Snp#include <rdma/ib_umem.h> 66237263Snp#include <rdma/ib_user_verbs.h> 67237263Snp#include <linux/idr.h> 68237263Snp#include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h> 69178786Skmacy 70178786Skmacy#include <cxgb_include.h> 71178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_wr.h> 72178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_hal.h> 73178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_provider.h> 74178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_cm.h> 75178786Skmacy#include <ulp/iw_cxgb/iw_cxgb.h> 76178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_resource.h> 77178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_user.h> 78178786Skmacy 79178786Skmacy#ifdef needed 80178786Skmacystatic struct buf_ring *rhdl_fifo; 81178786Skmacystatic struct mtx rhdl_fifo_lock; 82178786Skmacy#endif 83178786Skmacy 84178786Skmacy#define RANDOM_SIZE 16 85178786Skmacy 86178786Skmacystatic int __cxio_init_resource_fifo(struct buf_ring **fifo, 87178786Skmacy struct mtx *fifo_lock, 88178786Skmacy u32 nr, u32 skip_low, 89178786Skmacy u32 skip_high, 90178786Skmacy int randomize) 91178786Skmacy{ 92178786Skmacy u32 i, j, idx; 93178786Skmacy u32 random_bytes; 94178786Skmacy u32 rarray[16]; 95178786Skmacy mtx_init(fifo_lock, "cxio fifo", NULL, MTX_DEF|MTX_DUPOK); 96178786Skmacy 97185162Skmacy *fifo = buf_ring_alloc(nr, M_DEVBUF, M_NOWAIT, fifo_lock); 98178786Skmacy if (*fifo == NULL) 99178786Skmacy return (-ENOMEM); 100178786Skmacy#if 0 101178786Skmacy for (i = 0; i < skip_low + skip_high; i++) { 102178786Skmacy u32 entry = 0; 103178786Skmacy 104178786Skmacy buf_ring_enqueue(*fifo, (uintptr_t) entry); 105178786Skmacy } 106178786Skmacy#endif 107178786Skmacy if (randomize) { 108178786Skmacy j = 0; 109178786Skmacy random_bytes = random(); 110178786Skmacy for (i = 0; i < RANDOM_SIZE; i++) 111178786Skmacy rarray[i] = i + skip_low; 112178786Skmacy for (i = skip_low + RANDOM_SIZE; i < nr - skip_high; i++) { 113178786Skmacy if (j >= RANDOM_SIZE) { 114178786Skmacy j = 0; 115178786Skmacy random_bytes = random(); 116178786Skmacy } 117178786Skmacy idx = (random_bytes >> (j * 2)) & 0xF; 118178786Skmacy buf_ring_enqueue(*fifo, (void *)(uintptr_t)rarray[idx]); 119178786Skmacy rarray[idx] = i; 120178786Skmacy j++; 121178786Skmacy } 122178786Skmacy for (i = 0; i < RANDOM_SIZE; i++) 123178786Skmacy buf_ring_enqueue(*fifo, (void *) (uintptr_t)rarray[i]); 124178786Skmacy } else 125178786Skmacy for (i = skip_low; i < nr - skip_high; i++) 126178786Skmacy buf_ring_enqueue(*fifo, (void *) (uintptr_t)i); 127178786Skmacy#if 0 128178786Skmacy for (i = 0; i < skip_low + skip_high; i++) 129185162Skmacy buf_ring_dequeue_sc(*fifo); 130178786Skmacy#endif 131178786Skmacy return 0; 132178786Skmacy} 133178786Skmacy 134178786Skmacystatic int cxio_init_resource_fifo(struct buf_ring **fifo, struct mtx * fifo_lock, 135178786Skmacy u32 nr, u32 skip_low, u32 skip_high) 136178786Skmacy{ 137178786Skmacy return (__cxio_init_resource_fifo(fifo, fifo_lock, nr, skip_low, 138178786Skmacy skip_high, 0)); 139178786Skmacy} 140178786Skmacy 141178786Skmacystatic int cxio_init_resource_fifo_random(struct buf_ring **fifo, 142178786Skmacy struct mtx * fifo_lock, 143178786Skmacy u32 nr, u32 skip_low, u32 skip_high) 144178786Skmacy{ 145178786Skmacy 146178786Skmacy return (__cxio_init_resource_fifo(fifo, fifo_lock, nr, skip_low, 147178786Skmacy skip_high, 1)); 148178786Skmacy} 149178786Skmacy 150178786Skmacystatic int cxio_init_qpid_fifo(struct cxio_rdev *rdev_p) 151178786Skmacy{ 152178786Skmacy u32 i; 153178786Skmacy 154178786Skmacy mtx_init(&rdev_p->rscp->qpid_fifo_lock, "qpid fifo", NULL, MTX_DEF); 155178786Skmacy 156185162Skmacy rdev_p->rscp->qpid_fifo = buf_ring_alloc(T3_MAX_NUM_QP, M_DEVBUF, 157185162Skmacy M_NOWAIT, &rdev_p->rscp->qpid_fifo_lock); 158178786Skmacy if (rdev_p->rscp->qpid_fifo == NULL) 159178786Skmacy return (-ENOMEM); 160178786Skmacy 161178786Skmacy for (i = 16; i < T3_MAX_NUM_QP; i++) 162178786Skmacy if (!(i & rdev_p->qpmask)) 163178786Skmacy buf_ring_enqueue(rdev_p->rscp->qpid_fifo, (void *) (uintptr_t)i); 164178786Skmacy return 0; 165178786Skmacy} 166178786Skmacy 167178786Skmacy#ifdef needed 168178786Skmacyint cxio_hal_init_rhdl_resource(u32 nr_rhdl) 169178786Skmacy{ 170178786Skmacy return cxio_init_resource_fifo(&rhdl_fifo, &rhdl_fifo_lock, nr_rhdl, 1, 171178786Skmacy 0); 172178786Skmacy} 173178786Skmacy 174178786Skmacyvoid cxio_hal_destroy_rhdl_resource(void) 175178786Skmacy{ 176185162Skmacy buf_ring_free(rhdl_fifo, M_DEVBUF); 177178786Skmacy} 178178786Skmacy#endif 179178786Skmacy 180178786Skmacy/* nr_* must be power of 2 */ 181178786Skmacyint cxio_hal_init_resource(struct cxio_rdev *rdev_p, 182178786Skmacy u32 nr_tpt, u32 nr_pbl, 183178786Skmacy u32 nr_rqt, u32 nr_qpid, u32 nr_cqid, u32 nr_pdid) 184178786Skmacy{ 185178786Skmacy int err = 0; 186178786Skmacy struct cxio_hal_resource *rscp; 187178786Skmacy 188178786Skmacy rscp = malloc(sizeof(*rscp), M_DEVBUF, M_NOWAIT|M_ZERO); 189178786Skmacy if (!rscp) 190178786Skmacy return (-ENOMEM); 191178786Skmacy rdev_p->rscp = rscp; 192178786Skmacy err = cxio_init_resource_fifo_random(&rscp->tpt_fifo, 193178786Skmacy &rscp->tpt_fifo_lock, 194178786Skmacy nr_tpt, 1, 0); 195178786Skmacy if (err) 196178786Skmacy goto tpt_err; 197178786Skmacy err = cxio_init_qpid_fifo(rdev_p); 198178786Skmacy if (err) 199178786Skmacy goto qpid_err; 200178786Skmacy err = cxio_init_resource_fifo(&rscp->cqid_fifo, &rscp->cqid_fifo_lock, 201178786Skmacy nr_cqid, 1, 0); 202178786Skmacy if (err) 203178786Skmacy goto cqid_err; 204178786Skmacy err = cxio_init_resource_fifo(&rscp->pdid_fifo, &rscp->pdid_fifo_lock, 205178786Skmacy nr_pdid, 1, 0); 206178786Skmacy if (err) 207178786Skmacy goto pdid_err; 208178786Skmacy return 0; 209178786Skmacypdid_err: 210185162Skmacy buf_ring_free(rscp->cqid_fifo, M_DEVBUF); 211178786Skmacycqid_err: 212185162Skmacy buf_ring_free(rscp->qpid_fifo, M_DEVBUF); 213178786Skmacyqpid_err: 214185162Skmacy buf_ring_free(rscp->tpt_fifo, M_DEVBUF); 215178786Skmacytpt_err: 216178786Skmacy return (-ENOMEM); 217178786Skmacy} 218178786Skmacy 219178786Skmacy/* 220178786Skmacy * returns 0 if no resource available 221178786Skmacy */ 222178786Skmacystatic u32 cxio_hal_get_resource(struct buf_ring *fifo, struct mtx *lock) 223178786Skmacy{ 224178786Skmacy u32 entry; 225178786Skmacy 226178786Skmacy mtx_lock(lock); 227185162Skmacy entry = (u32)(uintptr_t)buf_ring_dequeue_sc(fifo); 228178786Skmacy mtx_unlock(lock); 229178786Skmacy return entry; 230178786Skmacy} 231178786Skmacy 232178786Skmacystatic void cxio_hal_put_resource(struct buf_ring *fifo, u32 entry, struct mtx *lock) 233178786Skmacy{ 234178786Skmacy mtx_lock(lock); 235178786Skmacy buf_ring_enqueue(fifo, (void *) (uintptr_t)entry); 236178786Skmacy mtx_unlock(lock); 237178786Skmacy} 238178786Skmacy 239178786Skmacyu32 cxio_hal_get_stag(struct cxio_hal_resource *rscp) 240178786Skmacy{ 241178786Skmacy return cxio_hal_get_resource(rscp->tpt_fifo, &rscp->tpt_fifo_lock); 242178786Skmacy} 243178786Skmacy 244178786Skmacyvoid cxio_hal_put_stag(struct cxio_hal_resource *rscp, u32 stag) 245178786Skmacy{ 246178786Skmacy cxio_hal_put_resource(rscp->tpt_fifo, stag, &rscp->tpt_fifo_lock); 247178786Skmacy} 248178786Skmacy 249178786Skmacyu32 cxio_hal_get_qpid(struct cxio_hal_resource *rscp) 250178786Skmacy{ 251178786Skmacy u32 qpid = cxio_hal_get_resource(rscp->qpid_fifo, &rscp->qpid_fifo_lock); 252178786Skmacy CTR2(KTR_IW_CXGB, "%s qpid 0x%x", __FUNCTION__, qpid); 253178786Skmacy return qpid; 254178786Skmacy} 255178786Skmacy 256178786Skmacyvoid cxio_hal_put_qpid(struct cxio_hal_resource *rscp, u32 qpid) 257178786Skmacy{ 258178786Skmacy CTR2(KTR_IW_CXGB, "%s qpid 0x%x", __FUNCTION__, qpid); 259178786Skmacy cxio_hal_put_resource(rscp->qpid_fifo, qpid, &rscp->qpid_fifo_lock); 260178786Skmacy} 261178786Skmacy 262178786Skmacyu32 cxio_hal_get_cqid(struct cxio_hal_resource *rscp) 263178786Skmacy{ 264178786Skmacy return cxio_hal_get_resource(rscp->cqid_fifo, &rscp->cqid_fifo_lock); 265178786Skmacy} 266178786Skmacy 267178786Skmacyvoid cxio_hal_put_cqid(struct cxio_hal_resource *rscp, u32 cqid) 268178786Skmacy{ 269178786Skmacy cxio_hal_put_resource(rscp->cqid_fifo, cqid, &rscp->cqid_fifo_lock); 270178786Skmacy} 271178786Skmacy 272178786Skmacyu32 cxio_hal_get_pdid(struct cxio_hal_resource *rscp) 273178786Skmacy{ 274178786Skmacy return cxio_hal_get_resource(rscp->pdid_fifo, &rscp->pdid_fifo_lock); 275178786Skmacy} 276178786Skmacy 277178786Skmacyvoid cxio_hal_put_pdid(struct cxio_hal_resource *rscp, u32 pdid) 278178786Skmacy{ 279178786Skmacy cxio_hal_put_resource(rscp->pdid_fifo, pdid, &rscp->pdid_fifo_lock); 280178786Skmacy} 281178786Skmacy 282178786Skmacyvoid cxio_hal_destroy_resource(struct cxio_hal_resource *rscp) 283178786Skmacy{ 284185162Skmacy buf_ring_free(rscp->tpt_fifo, M_DEVBUF); 285185162Skmacy buf_ring_free(rscp->cqid_fifo, M_DEVBUF); 286185162Skmacy buf_ring_free(rscp->qpid_fifo, M_DEVBUF); 287185162Skmacy buf_ring_free(rscp->pdid_fifo, M_DEVBUF); 288178786Skmacy free(rscp, M_DEVBUF); 289178786Skmacy} 290178786Skmacy 291178786Skmacy/* 292178786Skmacy * PBL Memory Manager. Uses Linux generic allocator. 293178786Skmacy */ 294178786Skmacy 295178786Skmacy#define MIN_PBL_SHIFT 8 /* 256B == min PBL size (32 entries) */ 296178786Skmacy#define PBL_CHUNK 2*1024*1024 297178786Skmacy 298178786Skmacyu32 cxio_hal_pblpool_alloc(struct cxio_rdev *rdev_p, int size) 299178786Skmacy{ 300178786Skmacy unsigned long addr = gen_pool_alloc(rdev_p->pbl_pool, size); 301178786Skmacy CTR3(KTR_IW_CXGB, "%s addr 0x%x size %d", __FUNCTION__, (u32)addr, size); 302178786Skmacy return (u32)addr; 303178786Skmacy} 304178786Skmacy 305178786Skmacyvoid cxio_hal_pblpool_free(struct cxio_rdev *rdev_p, u32 addr, int size) 306178786Skmacy{ 307178786Skmacy CTR3(KTR_IW_CXGB, "%s addr 0x%x size %d", __FUNCTION__, addr, size); 308178786Skmacy gen_pool_free(rdev_p->pbl_pool, (unsigned long)addr, size); 309178786Skmacy} 310178786Skmacy 311178786Skmacyint cxio_hal_pblpool_create(struct cxio_rdev *rdev_p) 312178786Skmacy{ 313178786Skmacy 314178786Skmacy rdev_p->pbl_pool = gen_pool_create(rdev_p->rnic_info.pbl_base, MIN_PBL_SHIFT, 315178786Skmacy rdev_p->rnic_info.pbl_top - rdev_p->rnic_info.pbl_base); 316178786Skmacy#if 0 317178786Skmacy if (rdev_p->pbl_pool) { 318178786Skmacy 319178786Skmacy unsigned long i; 320178786Skmacy for (i = rdev_p->rnic_info.pbl_base; 321178786Skmacy i <= rdev_p->rnic_info.pbl_top - PBL_CHUNK + 1; 322178786Skmacy i += PBL_CHUNK) 323178786Skmacy gen_pool_add(rdev_p->pbl_pool, i, PBL_CHUNK, -1); 324178786Skmacy } 325178786Skmacy#endif 326178786Skmacy return rdev_p->pbl_pool ? 0 : (-ENOMEM); 327178786Skmacy} 328178786Skmacy 329178786Skmacyvoid cxio_hal_pblpool_destroy(struct cxio_rdev *rdev_p) 330178786Skmacy{ 331178786Skmacy gen_pool_destroy(rdev_p->pbl_pool); 332178786Skmacy} 333178786Skmacy 334178786Skmacy/* 335178786Skmacy * RQT Memory Manager. Uses Linux generic allocator. 336178786Skmacy */ 337178786Skmacy 338178786Skmacy#define MIN_RQT_SHIFT 10 /* 1KB == mini RQT size (16 entries) */ 339178786Skmacy#define RQT_CHUNK 2*1024*1024 340178786Skmacy 341178786Skmacyu32 cxio_hal_rqtpool_alloc(struct cxio_rdev *rdev_p, int size) 342178786Skmacy{ 343178786Skmacy unsigned long addr = gen_pool_alloc(rdev_p->rqt_pool, size << 6); 344178786Skmacy CTR3(KTR_IW_CXGB, "%s addr 0x%x size %d", __FUNCTION__, (u32)addr, size << 6); 345178786Skmacy return (u32)addr; 346178786Skmacy} 347178786Skmacy 348178786Skmacyvoid cxio_hal_rqtpool_free(struct cxio_rdev *rdev_p, u32 addr, int size) 349178786Skmacy{ 350178786Skmacy CTR3(KTR_IW_CXGB, "%s addr 0x%x size %d", __FUNCTION__, addr, size << 6); 351178786Skmacy gen_pool_free(rdev_p->rqt_pool, (unsigned long)addr, size << 6); 352178786Skmacy} 353178786Skmacy 354178786Skmacyint cxio_hal_rqtpool_create(struct cxio_rdev *rdev_p) 355178786Skmacy{ 356178786Skmacy 357178786Skmacy rdev_p->rqt_pool = gen_pool_create(rdev_p->rnic_info.rqt_base, 358178786Skmacy MIN_RQT_SHIFT, rdev_p->rnic_info.rqt_top - rdev_p->rnic_info.rqt_base); 359178786Skmacy#if 0 360178786Skmacy if (rdev_p->rqt_pool) { 361178786Skmacy unsigned long i; 362178786Skmacy 363178786Skmacy for (i = rdev_p->rnic_info.rqt_base; 364178786Skmacy i <= rdev_p->rnic_info.rqt_top - RQT_CHUNK + 1; 365178786Skmacy i += RQT_CHUNK) 366178786Skmacy gen_pool_add(rdev_p->rqt_pool, i, RQT_CHUNK, -1); 367178786Skmacy } 368178786Skmacy#endif 369178786Skmacy return rdev_p->rqt_pool ? 0 : (-ENOMEM); 370178786Skmacy} 371178786Skmacy 372178786Skmacyvoid cxio_hal_rqtpool_destroy(struct cxio_rdev *rdev_p) 373178786Skmacy{ 374178786Skmacy gen_pool_destroy(rdev_p->rqt_pool); 375178786Skmacy} 376237263Snp#endif 377