1171095Ssam/*- 2171095Ssam * Copyright (c) 2002-2007 Neterion, Inc. 3171095Ssam * All rights reserved. 4171095Ssam * 5171095Ssam * Redistribution and use in source and binary forms, with or without 6171095Ssam * modification, are permitted provided that the following conditions 7171095Ssam * are met: 8171095Ssam * 1. Redistributions of source code must retain the above copyright 9171095Ssam * notice, this list of conditions and the following disclaimer. 10171095Ssam * 2. Redistributions in binary form must reproduce the above copyright 11171095Ssam * notice, this list of conditions and the following disclaimer in the 12171095Ssam * documentation and/or other materials provided with the distribution. 13171095Ssam * 14171095Ssam * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15171095Ssam * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16171095Ssam * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17171095Ssam * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18171095Ssam * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19171095Ssam * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20171095Ssam * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21171095Ssam * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22171095Ssam * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23171095Ssam * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24171095Ssam * SUCH DAMAGE. 25171095Ssam * 26171095Ssam * $FreeBSD: releng/10.3/sys/dev/nxge/xgehal/xgehal-fifo-fp.c 230133 2012-01-15 13:23:33Z uqs $ 27171095Ssam */ 28171095Ssam 29171095Ssam#ifdef XGE_DEBUG_FP 30171095Ssam#include <dev/nxge/include/xgehal-fifo.h> 31171095Ssam#endif 32171095Ssam 33171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO xge_hal_fifo_txdl_priv_t* 34171095Ssam__hal_fifo_txdl_priv(xge_hal_dtr_h dtrh) 35171095Ssam{ 36171095Ssam xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t*)dtrh; 37171095Ssam xge_hal_fifo_txdl_priv_t *txdl_priv; 38171095Ssam 39171095Ssam xge_assert(txdp); 40171095Ssam txdl_priv = (xge_hal_fifo_txdl_priv_t *) 41173139Srwatson (ulong_t)txdp->host_control; 42171095Ssam 43171095Ssam xge_assert(txdl_priv); 44171095Ssam xge_assert(txdl_priv->dma_object); 45171095Ssam xge_assert(txdl_priv->dma_addr); 46171095Ssam 47171095Ssam xge_assert(txdl_priv->dma_object->handle == txdl_priv->dma_handle); 48171095Ssam 49171095Ssam return txdl_priv; 50171095Ssam} 51171095Ssam 52171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO void 53171095Ssam__hal_fifo_dtr_post_single(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh, 54173139Srwatson u64 ctrl_1) 55171095Ssam{ 56171095Ssam xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; 57171095Ssam xge_hal_fifo_hw_pair_t *hw_pair = fifo->hw_pair; 58171095Ssam xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)dtrh; 59171095Ssam xge_hal_fifo_txdl_priv_t *txdl_priv; 60173139Srwatson u64 ctrl; 61171095Ssam 62171095Ssam txdp->control_1 |= XGE_HAL_TXD_LIST_OWN_XENA; 63171095Ssam 64171095Ssam#ifdef XGE_DEBUG_ASSERT 65173139Srwatson /* make sure Xena overwrites the (illegal) t_code value on completion */ 66173139Srwatson XGE_HAL_SET_TXD_T_CODE(txdp->control_1, XGE_HAL_TXD_T_CODE_UNUSED_5); 67171095Ssam#endif 68171095Ssam 69171095Ssam txdl_priv = __hal_fifo_txdl_priv(dtrh); 70171095Ssam 71171095Ssam#if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING) 72171095Ssam /* sync the TxDL to device */ 73171095Ssam xge_os_dma_sync(fifo->channel.pdev, 74171095Ssam txdl_priv->dma_handle, 75173139Srwatson txdl_priv->dma_addr, 76173139Srwatson txdl_priv->dma_offset, 77173139Srwatson txdl_priv->frags << 5 /* sizeof(xge_hal_fifo_txd_t) */, 78173139Srwatson XGE_OS_DMA_DIR_TODEVICE); 79171095Ssam#endif 80171095Ssam /* write the pointer first */ 81171095Ssam xge_os_pio_mem_write64(fifo->channel.pdev, 82173139Srwatson fifo->channel.regh1, 83171095Ssam txdl_priv->dma_addr, 84171095Ssam &hw_pair->txdl_pointer); 85171095Ssam 86171095Ssam /* spec: 0x00 = 1 TxD in the list */ 87171095Ssam ctrl = XGE_HAL_TX_FIFO_LAST_TXD_NUM(txdl_priv->frags - 1); 88171095Ssam ctrl |= ctrl_1; 89171095Ssam ctrl |= fifo->no_snoop_bits; 90171095Ssam 91171095Ssam if (txdp->control_1 & XGE_HAL_TXD_LSO_COF_CTRL(XGE_HAL_TXD_TCP_LSO)) { 92173139Srwatson ctrl |= XGE_HAL_TX_FIFO_SPECIAL_FUNC; 93171095Ssam } 94171095Ssam 95171095Ssam /* 96171095Ssam * according to the XENA spec: 97171095Ssam * 98171095Ssam * It is important to note that pointers and list control words are 99171095Ssam * always written in pairs: in the first write, the host must write a 100171095Ssam * pointer, and in the second write, it must write the list control 101171095Ssam * word. Any other access will result in an error. Also, all 16 bytes 102171095Ssam * of the pointer/control structure must be written, including any 103171095Ssam * reserved bytes. 104171095Ssam */ 105171095Ssam xge_os_wmb(); 106171095Ssam 107171095Ssam /* 108171095Ssam * we want touch work_arr in order with ownership bit set to HW 109171095Ssam */ 110171095Ssam __hal_channel_dtr_post(channelh, dtrh); 111171095Ssam 112171095Ssam xge_os_pio_mem_write64(fifo->channel.pdev, fifo->channel.regh1, 113173139Srwatson ctrl, &hw_pair->list_control); 114171095Ssam 115171095Ssam xge_debug_fifo(XGE_TRACE, "posted txdl 0x"XGE_OS_LLXFMT" ctrl 0x"XGE_OS_LLXFMT" " 116173139Srwatson "into 0x"XGE_OS_LLXFMT"", (unsigned long long)txdl_priv->dma_addr, 117173139Srwatson (unsigned long long)ctrl, 118173139Srwatson (unsigned long long)(ulong_t)&hw_pair->txdl_pointer); 119171095Ssam 120171095Ssam#ifdef XGE_HAL_FIFO_DUMP_TXD 121171095Ssam xge_os_printf(""XGE_OS_LLXFMT":"XGE_OS_LLXFMT":"XGE_OS_LLXFMT":" 122173139Srwatson XGE_OS_LLXFMT" dma "XGE_OS_LLXFMT, 123173139Srwatson txdp->control_1, txdp->control_2, txdp->buffer_pointer, 124173139Srwatson txdp->host_control, txdl_priv->dma_addr); 125171095Ssam#endif 126171095Ssam 127171095Ssam fifo->channel.stats.total_posts++; 128171095Ssam fifo->channel.usage_cnt++; 129171095Ssam if (fifo->channel.stats.usage_max < fifo->channel.usage_cnt) 130173139Srwatson fifo->channel.stats.usage_max = fifo->channel.usage_cnt; 131171095Ssam} 132171095Ssam 133171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO void 134171095Ssam__hal_fifo_txdl_free_many(xge_hal_channel_h channelh, 135173139Srwatson xge_hal_fifo_txd_t *txdp, int list_size, int frags) 136171095Ssam{ 137171095Ssam xge_hal_fifo_txdl_priv_t *current_txdl_priv; 138171095Ssam xge_hal_fifo_txdl_priv_t *next_txdl_priv; 139171095Ssam int invalid_frags = frags % list_size; 140171095Ssam if (invalid_frags){ 141173139Srwatson xge_debug_fifo(XGE_ERR, 142173139Srwatson "freeing corrupt dtrh %p, fragments %d list size %d", 143173139Srwatson txdp, frags, list_size); 144173139Srwatson xge_assert(invalid_frags == 0); 145171095Ssam } 146171095Ssam while(txdp){ 147173139Srwatson xge_debug_fifo(XGE_TRACE, 148173139Srwatson "freeing linked dtrh %p, fragments %d list size %d", 149173139Srwatson txdp, frags, list_size); 150173139Srwatson current_txdl_priv = __hal_fifo_txdl_priv(txdp); 151171095Ssam#if defined(XGE_DEBUG_ASSERT) && defined(XGE_OS_MEMORY_CHECK) 152173139Srwatson current_txdl_priv->allocated = 0; 153171095Ssam#endif 154173139Srwatson __hal_channel_dtr_free(channelh, txdp); 155173139Srwatson next_txdl_priv = current_txdl_priv->next_txdl_priv; 156173139Srwatson xge_assert(frags); 157173139Srwatson frags -= list_size; 158173139Srwatson if (next_txdl_priv) { 159173139Srwatson current_txdl_priv->next_txdl_priv = NULL; 160173139Srwatson txdp = next_txdl_priv->first_txdp; 161173139Srwatson } 162173139Srwatson else { 163173139Srwatson xge_debug_fifo(XGE_TRACE, 164173139Srwatson "freed linked dtrh fragments %d list size %d", 165173139Srwatson frags, list_size); 166173139Srwatson break; 167173139Srwatson } 168171095Ssam } 169171095Ssam xge_assert(frags == 0) 170171095Ssam} 171171095Ssam 172171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO void 173171095Ssam__hal_fifo_txdl_restore_many(xge_hal_channel_h channelh, 174173139Srwatson xge_hal_fifo_txd_t *txdp, int txdl_count) 175171095Ssam{ 176171095Ssam xge_hal_fifo_txdl_priv_t *current_txdl_priv; 177171095Ssam xge_hal_fifo_txdl_priv_t *next_txdl_priv; 178171095Ssam int i = txdl_count; 179171095Ssam 180171095Ssam xge_assert(((xge_hal_channel_t *)channelh)->reserve_length + 181173139Srwatson txdl_count <= ((xge_hal_channel_t *)channelh)->reserve_initial); 182171095Ssam 183171095Ssam current_txdl_priv = __hal_fifo_txdl_priv(txdp); 184171095Ssam do{ 185173139Srwatson xge_assert(i); 186171095Ssam#if defined(XGE_DEBUG_ASSERT) && defined(XGE_OS_MEMORY_CHECK) 187173139Srwatson current_txdl_priv->allocated = 0; 188171095Ssam#endif 189173139Srwatson next_txdl_priv = current_txdl_priv->next_txdl_priv; 190173139Srwatson txdp = current_txdl_priv->first_txdp; 191173139Srwatson current_txdl_priv->next_txdl_priv = NULL; 192173139Srwatson __hal_channel_dtr_restore(channelh, (xge_hal_dtr_h )txdp, --i); 193173139Srwatson xge_debug_fifo(XGE_TRACE, 194173139Srwatson "dtrh %p restored at offset %d", txdp, i); 195173139Srwatson current_txdl_priv = next_txdl_priv; 196171095Ssam } while(current_txdl_priv); 197171095Ssam __hal_channel_dtr_restore(channelh, NULL, txdl_count); 198171095Ssam} 199171095Ssam/** 200171095Ssam * xge_hal_fifo_dtr_private - Retrieve per-descriptor private data. 201171095Ssam * @channelh: Channel handle. 202171095Ssam * @dtrh: Descriptor handle. 203171095Ssam * 204171095Ssam * Retrieve per-descriptor private data. 205171095Ssam * Note that ULD requests per-descriptor space via 206171095Ssam * xge_hal_channel_open(). 207171095Ssam * 208171095Ssam * Returns: private ULD data associated with the descriptor. 209171095Ssam * Usage: See ex_xmit{} and ex_tx_compl{}. 210171095Ssam */ 211171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO void* 212171095Ssamxge_hal_fifo_dtr_private(xge_hal_dtr_h dtrh) 213171095Ssam{ 214171095Ssam xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)dtrh; 215171095Ssam 216171095Ssam return ((char *)(ulong_t)txdp->host_control) + 217173139Srwatson sizeof(xge_hal_fifo_txdl_priv_t); 218171095Ssam} 219171095Ssam 220171095Ssam/** 221171095Ssam * xge_hal_fifo_dtr_buffer_cnt - Get number of buffers carried by the 222171095Ssam * descriptor. 223171095Ssam * @dtrh: Descriptor handle. 224171095Ssam * 225171095Ssam * Returns: Number of buffers stored in the given descriptor. Can be used 226171095Ssam * _after_ the descriptor is set up for posting (see 227171095Ssam * xge_hal_fifo_dtr_post()) and _before_ it is deallocated (see 228171095Ssam * xge_hal_fifo_dtr_free()). 229171095Ssam * 230171095Ssam */ 231171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO int 232171095Ssamxge_hal_fifo_dtr_buffer_cnt(xge_hal_dtr_h dtrh) 233171095Ssam{ 234171095Ssam xge_hal_fifo_txdl_priv_t *txdl_priv; 235171095Ssam 236171095Ssam txdl_priv = __hal_fifo_txdl_priv(dtrh); 237171095Ssam 238171095Ssam return txdl_priv->frags; 239171095Ssam} 240171095Ssam/** 241171095Ssam * xge_hal_fifo_dtr_reserve_many- Reserve fifo descriptors which span more 242173139Srwatson * than single txdl. 243171095Ssam * @channelh: Channel handle. 244171095Ssam * @dtrh: Reserved descriptor. On success HAL fills this "out" parameter 245171095Ssam * with a valid handle. 246171095Ssam * @frags: minimum number of fragments to be reserved. 247171095Ssam * 248171095Ssam * Reserve TxDL(s) (that is, fifo descriptor) 249171095Ssam * for the subsequent filling-in by upper layerdriver (ULD)) 250171095Ssam * and posting on the corresponding channel (@channelh) 251171095Ssam * via xge_hal_fifo_dtr_post(). 252171095Ssam * 253171095Ssam * Returns: XGE_HAL_OK - success; 254171095Ssam * XGE_HAL_INF_OUT_OF_DESCRIPTORS - Currently no descriptors available 255171095Ssam * 256171095Ssam * See also: xge_hal_fifo_dtr_reserve_sp(), xge_hal_fifo_dtr_free(), 257171095Ssam * xge_hal_ring_dtr_reserve(), xge_hal_status_e{}. 258171095Ssam * Usage: See ex_xmit{}. 259171095Ssam */ 260171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO xge_hal_status_e 261171095Ssamxge_hal_fifo_dtr_reserve_many(xge_hal_channel_h channelh, 262173139Srwatson xge_hal_dtr_h *dtrh, const int frags) 263171095Ssam{ 264171095Ssam xge_hal_status_e status = XGE_HAL_OK; 265171095Ssam int alloc_frags = 0, dang_frags = 0; 266171095Ssam xge_hal_fifo_txd_t *curr_txdp = NULL; 267171095Ssam xge_hal_fifo_txd_t *next_txdp; 268171095Ssam xge_hal_fifo_txdl_priv_t *next_txdl_priv, *curr_txdl_priv = NULL; 269171095Ssam xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; 270171095Ssam int max_frags = fifo->config->max_frags; 271171095Ssam xge_hal_dtr_h dang_dtrh = NULL; 272171095Ssam#if defined(XGE_HAL_TX_MULTI_RESERVE_IRQ) 273171095Ssam unsigned long flags=0; 274171095Ssam#endif 275171095Ssam xge_debug_fifo(XGE_TRACE, "dtr_reserve_many called for frags %d", 276173139Srwatson frags); 277171095Ssam xge_assert(frags < (fifo->txdl_per_memblock * max_frags)); 278171095Ssam#if defined(XGE_HAL_TX_MULTI_RESERVE) 279171095Ssam xge_os_spin_lock(&fifo->channel.reserve_lock); 280171095Ssam#elif defined(XGE_HAL_TX_MULTI_RESERVE_IRQ) 281171095Ssam xge_os_spin_lock_irq(&fifo->channel.reserve_lock, flags); 282171095Ssam#endif 283171095Ssam while(alloc_frags < frags) { 284173139Srwatson status = __hal_channel_dtr_alloc(channelh, 285173139Srwatson (xge_hal_dtr_h *)(void*)&next_txdp); 286173139Srwatson if (status != XGE_HAL_OK){ 287173139Srwatson xge_debug_fifo(XGE_ERR, 288173139Srwatson "failed to allocate linked fragments rc %d", 289173139Srwatson status); 290173139Srwatson xge_assert(status == XGE_HAL_INF_OUT_OF_DESCRIPTORS); 291173139Srwatson if (*dtrh) { 292173139Srwatson xge_assert(alloc_frags/max_frags); 293173139Srwatson __hal_fifo_txdl_restore_many(channelh, 294173139Srwatson (xge_hal_fifo_txd_t *) *dtrh, alloc_frags/max_frags); 295173139Srwatson } 296173139Srwatson if (dang_dtrh) { 297173139Srwatson xge_assert(dang_frags/max_frags); 298173139Srwatson __hal_fifo_txdl_restore_many(channelh, 299173139Srwatson (xge_hal_fifo_txd_t *) dang_dtrh, dang_frags/max_frags); 300173139Srwatson } 301173139Srwatson break; 302173139Srwatson } 303173139Srwatson xge_debug_fifo(XGE_TRACE, "allocated linked dtrh %p" 304173139Srwatson " for frags %d", next_txdp, frags); 305173139Srwatson next_txdl_priv = __hal_fifo_txdl_priv(next_txdp); 306173139Srwatson xge_assert(next_txdl_priv); 307173139Srwatson xge_assert(next_txdl_priv->first_txdp == next_txdp); 308173139Srwatson next_txdl_priv->dang_txdl = NULL; 309173139Srwatson next_txdl_priv->dang_frags = 0; 310173139Srwatson next_txdl_priv->next_txdl_priv = NULL; 311171095Ssam#if defined(XGE_OS_MEMORY_CHECK) 312173139Srwatson next_txdl_priv->allocated = 1; 313171095Ssam#endif 314173139Srwatson if (!curr_txdp || !curr_txdl_priv) { 315173139Srwatson curr_txdp = next_txdp; 316173139Srwatson curr_txdl_priv = next_txdl_priv; 317173139Srwatson *dtrh = (xge_hal_dtr_h)next_txdp; 318173139Srwatson alloc_frags = max_frags; 319173139Srwatson continue; 320173139Srwatson } 321173139Srwatson if (curr_txdl_priv->memblock == 322173139Srwatson next_txdl_priv->memblock) { 323173139Srwatson xge_debug_fifo(XGE_TRACE, 324173139Srwatson "linking dtrh %p, with %p", 325173139Srwatson *dtrh, next_txdp); 326173139Srwatson xge_assert (next_txdp == 327173139Srwatson curr_txdp + max_frags); 328173139Srwatson alloc_frags += max_frags; 329173139Srwatson curr_txdl_priv->next_txdl_priv = next_txdl_priv; 330173139Srwatson } 331173139Srwatson else { 332173139Srwatson xge_assert(*dtrh); 333173139Srwatson xge_assert(dang_dtrh == NULL); 334173139Srwatson dang_dtrh = *dtrh; 335173139Srwatson dang_frags = alloc_frags; 336173139Srwatson xge_debug_fifo(XGE_TRACE, 337173139Srwatson "dangling dtrh %p, linked with dtrh %p", 338173139Srwatson *dtrh, next_txdp); 339173139Srwatson next_txdl_priv->dang_txdl = (xge_hal_fifo_txd_t *) *dtrh; 340173139Srwatson next_txdl_priv->dang_frags = alloc_frags; 341173139Srwatson alloc_frags = max_frags; 342173139Srwatson *dtrh = next_txdp; 343173139Srwatson } 344173139Srwatson curr_txdp = next_txdp; 345173139Srwatson curr_txdl_priv = next_txdl_priv; 346171095Ssam } 347171095Ssam 348171095Ssam#if defined(XGE_HAL_TX_MULTI_RESERVE) 349171095Ssam xge_os_spin_unlock(&fifo->channel.reserve_lock); 350171095Ssam#elif defined(XGE_HAL_TX_MULTI_RESERVE_IRQ) 351171095Ssam xge_os_spin_unlock_irq(&fifo->channel.reserve_lock, flags); 352171095Ssam#endif 353171095Ssam 354171095Ssam if (status == XGE_HAL_OK) { 355173139Srwatson xge_hal_fifo_txdl_priv_t * txdl_priv; 356173139Srwatson xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)*dtrh; 357173139Srwatson xge_hal_stats_channel_info_t *statsp = &fifo->channel.stats; 358173139Srwatson txdl_priv = __hal_fifo_txdl_priv(txdp); 359173139Srwatson /* reset the TxDL's private */ 360173139Srwatson txdl_priv->align_dma_offset = 0; 361173139Srwatson txdl_priv->align_vaddr_start = txdl_priv->align_vaddr; 362173139Srwatson txdl_priv->align_used_frags = 0; 363173139Srwatson txdl_priv->frags = 0; 364173139Srwatson txdl_priv->bytes_sent = 0; 365173139Srwatson txdl_priv->alloc_frags = alloc_frags; 366173139Srwatson /* reset TxD0 */ 367173139Srwatson txdp->control_1 = txdp->control_2 = 0; 368171095Ssam 369171095Ssam#if defined(XGE_OS_MEMORY_CHECK) 370173139Srwatson txdl_priv->allocated = 1; 371171095Ssam#endif 372173139Srwatson /* update statistics */ 373173139Srwatson statsp->total_posts_dtrs_many++; 374173139Srwatson statsp->total_posts_frags_many += txdl_priv->alloc_frags; 375173139Srwatson if (txdl_priv->dang_frags){ 376173139Srwatson statsp->total_posts_dang_dtrs++; 377173139Srwatson statsp->total_posts_dang_frags += txdl_priv->dang_frags; 378173139Srwatson } 379171095Ssam } 380171095Ssam 381171095Ssam return status; 382171095Ssam} 383171095Ssam 384171095Ssam/** 385171095Ssam * xge_hal_fifo_dtr_reserve - Reserve fifo descriptor. 386171095Ssam * @channelh: Channel handle. 387171095Ssam * @dtrh: Reserved descriptor. On success HAL fills this "out" parameter 388171095Ssam * with a valid handle. 389171095Ssam * 390171095Ssam * Reserve a single TxDL (that is, fifo descriptor) 391171095Ssam * for the subsequent filling-in by upper layerdriver (ULD)) 392171095Ssam * and posting on the corresponding channel (@channelh) 393171095Ssam * via xge_hal_fifo_dtr_post(). 394171095Ssam * 395171095Ssam * Note: it is the responsibility of ULD to reserve multiple descriptors 396171095Ssam * for lengthy (e.g., LSO) transmit operation. A single fifo descriptor 397171095Ssam * carries up to configured number (fifo.max_frags) of contiguous buffers. 398171095Ssam * 399171095Ssam * Returns: XGE_HAL_OK - success; 400171095Ssam * XGE_HAL_INF_OUT_OF_DESCRIPTORS - Currently no descriptors available 401171095Ssam * 402171095Ssam * See also: xge_hal_fifo_dtr_reserve_sp(), xge_hal_fifo_dtr_free(), 403171095Ssam * xge_hal_ring_dtr_reserve(), xge_hal_status_e{}. 404171095Ssam * Usage: See ex_xmit{}. 405171095Ssam */ 406171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO xge_hal_status_e 407171095Ssamxge_hal_fifo_dtr_reserve(xge_hal_channel_h channelh, xge_hal_dtr_h *dtrh) 408171095Ssam{ 409171095Ssam xge_hal_status_e status; 410171095Ssam#if defined(XGE_HAL_TX_MULTI_RESERVE_IRQ) 411171095Ssam unsigned long flags=0; 412171095Ssam#endif 413171095Ssam 414171095Ssam#if defined(XGE_HAL_TX_MULTI_RESERVE) 415171095Ssam xge_os_spin_lock(&((xge_hal_channel_t*)channelh)->reserve_lock); 416171095Ssam#elif defined(XGE_HAL_TX_MULTI_RESERVE_IRQ) 417171095Ssam xge_os_spin_lock_irq(&((xge_hal_channel_t*)channelh)->reserve_lock, 418171095Ssam flags); 419171095Ssam#endif 420171095Ssam 421171095Ssam status = __hal_channel_dtr_alloc(channelh, dtrh); 422171095Ssam 423171095Ssam#if defined(XGE_HAL_TX_MULTI_RESERVE) 424171095Ssam xge_os_spin_unlock(&((xge_hal_channel_t*)channelh)->reserve_lock); 425171095Ssam#elif defined(XGE_HAL_TX_MULTI_RESERVE_IRQ) 426171095Ssam xge_os_spin_unlock_irq(&((xge_hal_channel_t*)channelh)->reserve_lock, 427171095Ssam flags); 428171095Ssam#endif 429171095Ssam 430171095Ssam if (status == XGE_HAL_OK) { 431173139Srwatson xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)*dtrh; 432173139Srwatson xge_hal_fifo_txdl_priv_t *txdl_priv; 433171095Ssam 434173139Srwatson txdl_priv = __hal_fifo_txdl_priv(txdp); 435171095Ssam 436173139Srwatson /* reset the TxDL's private */ 437173139Srwatson txdl_priv->align_dma_offset = 0; 438173139Srwatson txdl_priv->align_vaddr_start = txdl_priv->align_vaddr; 439173139Srwatson txdl_priv->align_used_frags = 0; 440173139Srwatson txdl_priv->frags = 0; 441173139Srwatson txdl_priv->alloc_frags = 442173139Srwatson ((xge_hal_fifo_t *)channelh)->config->max_frags; 443173139Srwatson txdl_priv->dang_txdl = NULL; 444173139Srwatson txdl_priv->dang_frags = 0; 445173139Srwatson txdl_priv->next_txdl_priv = NULL; 446173139Srwatson txdl_priv->bytes_sent = 0; 447171095Ssam 448173139Srwatson /* reset TxD0 */ 449173139Srwatson txdp->control_1 = txdp->control_2 = 0; 450171095Ssam 451171095Ssam#if defined(XGE_OS_MEMORY_CHECK) 452173139Srwatson txdl_priv->allocated = 1; 453171095Ssam#endif 454171095Ssam } 455171095Ssam 456171095Ssam return status; 457171095Ssam} 458171095Ssam 459171095Ssam/** 460171095Ssam * xge_hal_fifo_dtr_reserve_sp - Reserve fifo descriptor and store it in 461171095Ssam * the ULD-provided "scratch" memory. 462171095Ssam * @channelh: Channel handle. 463171095Ssam * @dtr_sp_size: Size of the %dtr_sp "scratch pad" that HAL can use for TxDL. 464171095Ssam * @dtr_sp: "Scratch pad" supplied by upper-layer driver (ULD). 465171095Ssam * 466171095Ssam * Reserve TxDL and fill-in ULD supplied "scratch pad". The difference 467171095Ssam * between this API and xge_hal_fifo_dtr_reserve() is (possibly) - 468171095Ssam * performance. 469171095Ssam * 470171095Ssam * If upper-layer uses ULP-defined commands, and if those commands have enough 471171095Ssam * space for HAL/Xframe descriptors - tnan it is better (read: faster) to fit 472171095Ssam * all the per-command information into one command, which is typically 473171095Ssam * one contiguous block. 474171095Ssam * 475171095Ssam * Note: Unlike xge_hal_fifo_dtr_reserve(), this function can be used to 476171095Ssam * allocate a single descriptor for transmit operation. 477171095Ssam * 478171095Ssam * See also: xge_hal_fifo_dtr_reserve(), xge_hal_fifo_dtr_free(), 479171095Ssam * xge_hal_ring_dtr_reserve(), xge_hal_status_e{}. 480171095Ssam */ 481171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO xge_hal_status_e 482171095Ssamxge_hal_fifo_dtr_reserve_sp(xge_hal_channel_h channelh, int dtr_sp_size, 483173139Srwatson xge_hal_dtr_h dtr_sp) 484171095Ssam{ 485171095Ssam /* FIXME: implement */ 486171095Ssam return XGE_HAL_OK; 487171095Ssam} 488171095Ssam 489171095Ssam/** 490171095Ssam * xge_hal_fifo_dtr_post - Post descriptor on the fifo channel. 491171095Ssam * @channelh: Channel handle. 492171095Ssam * @dtrh: Descriptor obtained via xge_hal_fifo_dtr_reserve() or 493171095Ssam * xge_hal_fifo_dtr_reserve_sp() 494171095Ssam * @frags: Number of contiguous buffers that are part of a single 495171095Ssam * transmit operation. 496171095Ssam * 497171095Ssam * Post descriptor on the 'fifo' type channel for transmission. 498171095Ssam * Prior to posting the descriptor should be filled in accordance with 499171095Ssam * Host/Xframe interface specification for a given service (LL, etc.). 500171095Ssam * 501171095Ssam * See also: xge_hal_fifo_dtr_post_many(), xge_hal_ring_dtr_post(). 502171095Ssam * Usage: See ex_xmit{}. 503171095Ssam */ 504171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO void 505171095Ssamxge_hal_fifo_dtr_post(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh) 506171095Ssam{ 507171095Ssam xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; 508171095Ssam xge_hal_fifo_txdl_priv_t *txdl_priv; 509171095Ssam xge_hal_fifo_txd_t *txdp_last; 510171095Ssam xge_hal_fifo_txd_t *txdp_first; 511171095Ssam#if defined(XGE_HAL_TX_MULTI_POST_IRQ) 512171095Ssam unsigned long flags = 0; 513171095Ssam#endif 514171095Ssam 515171095Ssam txdl_priv = __hal_fifo_txdl_priv(dtrh); 516171095Ssam 517171095Ssam txdp_first = (xge_hal_fifo_txd_t *)dtrh; 518171095Ssam txdp_first->control_1 |= XGE_HAL_TXD_GATHER_CODE_FIRST; 519171095Ssam txdp_first->control_2 |= fifo->interrupt_type; 520171095Ssam 521171095Ssam txdp_last = (xge_hal_fifo_txd_t *)dtrh + (txdl_priv->frags - 1); 522171095Ssam txdp_last->control_1 |= XGE_HAL_TXD_GATHER_CODE_LAST; 523171095Ssam 524171095Ssam#if defined(XGE_HAL_TX_MULTI_POST) 525171095Ssam xge_os_spin_lock(fifo->post_lock_ptr); 526171095Ssam#elif defined(XGE_HAL_TX_MULTI_POST_IRQ) 527171095Ssam xge_os_spin_lock_irq(fifo->post_lock_ptr, flags); 528171095Ssam#endif 529171095Ssam 530171095Ssam __hal_fifo_dtr_post_single(channelh, dtrh, 531173139Srwatson (u64)(XGE_HAL_TX_FIFO_FIRST_LIST | XGE_HAL_TX_FIFO_LAST_LIST)); 532171095Ssam 533171095Ssam#if defined(XGE_HAL_TX_MULTI_POST) 534171095Ssam xge_os_spin_unlock(fifo->post_lock_ptr); 535171095Ssam#elif defined(XGE_HAL_TX_MULTI_POST_IRQ) 536171095Ssam xge_os_spin_unlock_irq(fifo->post_lock_ptr, flags); 537171095Ssam#endif 538171095Ssam} 539171095Ssam 540171095Ssam/** 541171095Ssam * xge_hal_fifo_dtr_post_many - Post multiple descriptors on fifo 542171095Ssam * channel. 543171095Ssam * @channelh: Channel to post descriptor. 544171095Ssam * @num: Number of descriptors (i.e., fifo TxDLs) in the %dtrs[]. 545171095Ssam * @dtrs: Descriptors obtained via xge_hal_fifo_dtr_reserve(). 546171095Ssam * @frags_arr: Number of fragments carried @dtrs descriptors. 547171095Ssam * Note that frag_arr[i] corresponds to descriptor dtrs[i]. 548171095Ssam * 549171095Ssam * Post multi-descriptor on the fifo channel. The operation is atomic: 550171095Ssam * all descriptrs are posted on the channel "back-to-back' without 551171095Ssam * letting other posts (possibly driven by multiple transmitting threads) 552171095Ssam * to interleave. 553171095Ssam * 554171095Ssam * See also: xge_hal_fifo_dtr_post(), xge_hal_ring_dtr_post(). 555171095Ssam */ 556171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO void 557171095Ssamxge_hal_fifo_dtr_post_many(xge_hal_channel_h channelh, int num, 558173139Srwatson xge_hal_dtr_h dtrs[]) 559171095Ssam{ 560171095Ssam int i; 561171095Ssam xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; 562171095Ssam xge_hal_fifo_txd_t *txdp_last; 563171095Ssam xge_hal_fifo_txd_t *txdp_first; 564171095Ssam xge_hal_fifo_txdl_priv_t *txdl_priv_last; 565171095Ssam#if defined(XGE_HAL_TX_MULTI_POST_IRQ) 566171095Ssam unsigned long flags = 0; 567171095Ssam#endif 568171095Ssam 569171095Ssam xge_assert(num > 1); 570171095Ssam 571171095Ssam txdp_first = (xge_hal_fifo_txd_t *)dtrs[0]; 572171095Ssam txdp_first->control_1 |= XGE_HAL_TXD_GATHER_CODE_FIRST; 573171095Ssam txdp_first->control_2 |= fifo->interrupt_type; 574171095Ssam 575171095Ssam txdl_priv_last = __hal_fifo_txdl_priv(dtrs[num-1]); 576171095Ssam txdp_last = (xge_hal_fifo_txd_t *)dtrs[num-1] + 577173139Srwatson (txdl_priv_last->frags - 1); 578171095Ssam txdp_last->control_1 |= XGE_HAL_TXD_GATHER_CODE_LAST; 579171095Ssam 580171095Ssam#if defined(XGE_HAL_TX_MULTI_POST) 581171095Ssam xge_os_spin_lock(&((xge_hal_channel_t*)channelh)->post_lock); 582171095Ssam#elif defined(XGE_HAL_TX_MULTI_POST_IRQ) 583171095Ssam xge_os_spin_lock_irq(&((xge_hal_channel_t*)channelh)->post_lock, 584171095Ssam flags); 585171095Ssam#endif 586171095Ssam 587171095Ssam for (i=0; i<num; i++) { 588173139Srwatson xge_hal_fifo_txdl_priv_t *txdl_priv; 589173139Srwatson u64 val64; 590173139Srwatson xge_hal_dtr_h dtrh = dtrs[i]; 591171095Ssam 592173139Srwatson txdl_priv = __hal_fifo_txdl_priv(dtrh); 593173139Srwatson txdl_priv = txdl_priv; /* Cheat lint */ 594171095Ssam 595173139Srwatson val64 = 0; 596173139Srwatson if (i == 0) { 597173139Srwatson val64 |= XGE_HAL_TX_FIFO_FIRST_LIST; 598173139Srwatson } else if (i == num -1) { 599173139Srwatson val64 |= XGE_HAL_TX_FIFO_LAST_LIST; 600173139Srwatson } 601171095Ssam 602173139Srwatson val64 |= XGE_HAL_TX_FIFO_SPECIAL_FUNC; 603173139Srwatson __hal_fifo_dtr_post_single(channelh, dtrh, val64); 604171095Ssam } 605171095Ssam 606171095Ssam#if defined(XGE_HAL_TX_MULTI_POST) 607171095Ssam xge_os_spin_unlock(&((xge_hal_channel_t*)channelh)->post_lock); 608171095Ssam#elif defined(XGE_HAL_TX_MULTI_POST_IRQ) 609171095Ssam xge_os_spin_unlock_irq(&((xge_hal_channel_t*)channelh)->post_lock, 610171095Ssam flags); 611171095Ssam#endif 612171095Ssam 613171095Ssam fifo->channel.stats.total_posts_many++; 614171095Ssam} 615171095Ssam 616171095Ssam/** 617171095Ssam * xge_hal_fifo_dtr_next_completed - Retrieve next completed descriptor. 618171095Ssam * @channelh: Channel handle. 619171095Ssam * @dtrh: Descriptor handle. Returned by HAL. 620171095Ssam * @t_code: Transfer code, as per Xframe User Guide, 621171095Ssam * Transmit Descriptor Format. 622171095Ssam * Returned by HAL. 623171095Ssam * 624171095Ssam * Retrieve the _next_ completed descriptor. 625171095Ssam * HAL uses channel callback (*xge_hal_channel_callback_f) to notifiy 626171095Ssam * upper-layer driver (ULD) of new completed descriptors. After that 627171095Ssam * the ULD can use xge_hal_fifo_dtr_next_completed to retrieve the rest 628171095Ssam * completions (the very first completion is passed by HAL via 629171095Ssam * xge_hal_channel_callback_f). 630171095Ssam * 631171095Ssam * Implementation-wise, the upper-layer driver is free to call 632171095Ssam * xge_hal_fifo_dtr_next_completed either immediately from inside the 633171095Ssam * channel callback, or in a deferred fashion and separate (from HAL) 634171095Ssam * context. 635171095Ssam * 636171095Ssam * Non-zero @t_code means failure to process the descriptor. 637171095Ssam * The failure could happen, for instance, when the link is 638171095Ssam * down, in which case Xframe completes the descriptor because it 639171095Ssam * is not able to send the data out. 640171095Ssam * 641171095Ssam * For details please refer to Xframe User Guide. 642171095Ssam * 643171095Ssam * Returns: XGE_HAL_OK - success. 644171095Ssam * XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS - No completed descriptors 645171095Ssam * are currently available for processing. 646171095Ssam * 647171095Ssam * See also: xge_hal_channel_callback_f{}, 648171095Ssam * xge_hal_ring_dtr_next_completed(). 649171095Ssam * Usage: See ex_tx_compl{}. 650171095Ssam */ 651171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO xge_hal_status_e 652171095Ssamxge_hal_fifo_dtr_next_completed(xge_hal_channel_h channelh, 653173139Srwatson xge_hal_dtr_h *dtrh, u8 *t_code) 654171095Ssam{ 655171095Ssam xge_hal_fifo_txd_t *txdp; 656171095Ssam xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; 657171095Ssam#if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING) 658171095Ssam xge_hal_fifo_txdl_priv_t *txdl_priv; 659171095Ssam#endif 660171095Ssam 661171095Ssam __hal_channel_dtr_try_complete(channelh, dtrh); 662171095Ssam txdp = (xge_hal_fifo_txd_t *)*dtrh; 663171095Ssam if (txdp == NULL) { 664173139Srwatson return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS; 665171095Ssam } 666171095Ssam 667171095Ssam#if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING) 668171095Ssam txdl_priv = __hal_fifo_txdl_priv(txdp); 669171095Ssam 670171095Ssam /* sync TxDL to read the ownership 671171095Ssam * 672171095Ssam * Note: 16bytes means Control_1 & Control_2 */ 673171095Ssam xge_os_dma_sync(fifo->channel.pdev, 674171095Ssam txdl_priv->dma_handle, 675173139Srwatson txdl_priv->dma_addr, 676173139Srwatson txdl_priv->dma_offset, 677173139Srwatson 16, 678173139Srwatson XGE_OS_DMA_DIR_FROMDEVICE); 679171095Ssam#endif 680171095Ssam 681171095Ssam /* check whether host owns it */ 682171095Ssam if ( !(txdp->control_1 & XGE_HAL_TXD_LIST_OWN_XENA) ) { 683171095Ssam 684173139Srwatson xge_assert(txdp->host_control!=0); 685171095Ssam 686173139Srwatson __hal_channel_dtr_complete(channelh); 687171095Ssam 688173139Srwatson *t_code = (u8)XGE_HAL_GET_TXD_T_CODE(txdp->control_1); 689171095Ssam 690173139Srwatson /* see XGE_HAL_SET_TXD_T_CODE() above.. */ 691173139Srwatson xge_assert(*t_code != XGE_HAL_TXD_T_CODE_UNUSED_5); 692171095Ssam 693173139Srwatson if (fifo->channel.usage_cnt > 0) 694173139Srwatson fifo->channel.usage_cnt--; 695171095Ssam 696173139Srwatson return XGE_HAL_OK; 697171095Ssam } 698171095Ssam 699171095Ssam /* no more completions */ 700171095Ssam *dtrh = 0; 701171095Ssam return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS; 702171095Ssam} 703171095Ssam 704171095Ssam/** 705171095Ssam * xge_hal_fifo_dtr_free - Free descriptor. 706171095Ssam * @channelh: Channel handle. 707171095Ssam * @dtr: Descriptor handle. 708171095Ssam * 709171095Ssam * Free the reserved descriptor. This operation is "symmetrical" to 710171095Ssam * xge_hal_fifo_dtr_reserve or xge_hal_fifo_dtr_reserve_sp. 711171095Ssam * The "free-ing" completes the descriptor's lifecycle. 712171095Ssam * 713171095Ssam * After free-ing (see xge_hal_fifo_dtr_free()) the descriptor again can 714171095Ssam * be: 715171095Ssam * 716171095Ssam * - reserved (xge_hal_fifo_dtr_reserve); 717171095Ssam * 718171095Ssam * - posted (xge_hal_fifo_dtr_post); 719171095Ssam * 720171095Ssam * - completed (xge_hal_fifo_dtr_next_completed); 721171095Ssam * 722171095Ssam * - and recycled again (xge_hal_fifo_dtr_free). 723171095Ssam * 724171095Ssam * For alternative state transitions and more details please refer to 725171095Ssam * the design doc. 726171095Ssam * 727171095Ssam * See also: xge_hal_ring_dtr_free(), xge_hal_fifo_dtr_reserve(). 728171095Ssam * Usage: See ex_tx_compl{}. 729171095Ssam */ 730171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO void 731171095Ssamxge_hal_fifo_dtr_free(xge_hal_channel_h channelh, xge_hal_dtr_h dtr) 732171095Ssam{ 733171095Ssam#if defined(XGE_HAL_TX_MULTI_FREE_IRQ) 734171095Ssam unsigned long flags = 0; 735171095Ssam#endif 736171095Ssam xge_hal_fifo_txdl_priv_t *txdl_priv = __hal_fifo_txdl_priv( 737173139Srwatson (xge_hal_fifo_txd_t *)dtr); 738171095Ssam int max_frags = ((xge_hal_fifo_t *)channelh)->config->max_frags; 739171095Ssam#if defined(XGE_HAL_TX_MULTI_FREE) 740171095Ssam xge_os_spin_lock(&((xge_hal_channel_t*)channelh)->free_lock); 741171095Ssam#elif defined(XGE_HAL_TX_MULTI_FREE_IRQ) 742171095Ssam xge_os_spin_lock_irq(&((xge_hal_channel_t*)channelh)->free_lock, 743171095Ssam flags); 744171095Ssam#endif 745171095Ssam 746171095Ssam if (txdl_priv->alloc_frags > max_frags) { 747173139Srwatson xge_hal_fifo_txd_t *dang_txdp = (xge_hal_fifo_txd_t *) 748173139Srwatson txdl_priv->dang_txdl; 749173139Srwatson int dang_frags = txdl_priv->dang_frags; 750173139Srwatson int alloc_frags = txdl_priv->alloc_frags; 751173139Srwatson txdl_priv->dang_txdl = NULL; 752173139Srwatson txdl_priv->dang_frags = 0; 753173139Srwatson txdl_priv->alloc_frags = 0; 754173139Srwatson /* dtrh must have a linked list of dtrh */ 755173139Srwatson xge_assert(txdl_priv->next_txdl_priv); 756171095Ssam 757173139Srwatson /* free any dangling dtrh first */ 758173139Srwatson if (dang_txdp) { 759173139Srwatson xge_debug_fifo(XGE_TRACE, 760173139Srwatson "freeing dangled dtrh %p for %d fragments", 761173139Srwatson dang_txdp, dang_frags); 762173139Srwatson __hal_fifo_txdl_free_many(channelh, dang_txdp, 763173139Srwatson max_frags, dang_frags); 764173139Srwatson } 765171095Ssam 766173139Srwatson /* now free the reserved dtrh list */ 767173139Srwatson xge_debug_fifo(XGE_TRACE, 768173139Srwatson "freeing dtrh %p list of %d fragments", dtr, 769173139Srwatson alloc_frags); 770173139Srwatson __hal_fifo_txdl_free_many(channelh, 771173139Srwatson (xge_hal_fifo_txd_t *)dtr, max_frags, 772173139Srwatson alloc_frags); 773171095Ssam } 774171095Ssam else 775173139Srwatson __hal_channel_dtr_free(channelh, dtr); 776171095Ssam 777171095Ssam ((xge_hal_channel_t *)channelh)->poll_bytes += txdl_priv->bytes_sent; 778171095Ssam 779171095Ssam#if defined(XGE_DEBUG_ASSERT) && defined(XGE_OS_MEMORY_CHECK) 780171095Ssam __hal_fifo_txdl_priv(dtr)->allocated = 0; 781171095Ssam#endif 782171095Ssam 783171095Ssam#if defined(XGE_HAL_TX_MULTI_FREE) 784171095Ssam xge_os_spin_unlock(&((xge_hal_channel_t*)channelh)->free_lock); 785171095Ssam#elif defined(XGE_HAL_TX_MULTI_FREE_IRQ) 786171095Ssam xge_os_spin_unlock_irq(&((xge_hal_channel_t*)channelh)->free_lock, 787171095Ssam flags); 788171095Ssam#endif 789171095Ssam} 790171095Ssam 791171095Ssam 792171095Ssam/** 793171095Ssam * xge_hal_fifo_dtr_buffer_set_aligned - Align transmit buffer and fill 794171095Ssam * in fifo descriptor. 795171095Ssam * @channelh: Channel handle. 796171095Ssam * @dtrh: Descriptor handle. 797230133Suqs * @frag_idx: Index of the data buffer in the caller's scatter-gather list 798171095Ssam * (of buffers). 799171095Ssam * @vaddr: Virtual address of the data buffer. 800171095Ssam * @dma_pointer: DMA address of the data buffer referenced by @frag_idx. 801171095Ssam * @size: Size of the data buffer (in bytes). 802171095Ssam * @misaligned_size: Size (in bytes) of the misaligned portion of the 803171095Ssam * data buffer. Calculated by the caller, based on the platform/OS/other 804171095Ssam * specific criteria, which is outside of HAL's domain. See notes below. 805171095Ssam * 806171095Ssam * This API is part of the transmit descriptor preparation for posting 807171095Ssam * (via xge_hal_fifo_dtr_post()). The related "preparation" APIs include 808171095Ssam * xge_hal_fifo_dtr_mss_set() and xge_hal_fifo_dtr_cksum_set_bits(). 809171095Ssam * All three APIs fill in the fields of the fifo descriptor, 810171095Ssam * in accordance with the Xframe specification. 811171095Ssam * On the PCI-X based systems aligning transmit data typically provides better 812171095Ssam * transmit performance. The typical alignment granularity: L2 cacheline size. 813171095Ssam * However, HAL does not make assumptions in terms of the alignment granularity; 814171095Ssam * this is specified via additional @misaligned_size parameter described above. 815171095Ssam * Prior to calling xge_hal_fifo_dtr_buffer_set_aligned(), 816171095Ssam * ULD is supposed to check alignment of a given fragment/buffer. For this HAL 817171095Ssam * provides a separate xge_hal_check_alignment() API sufficient to cover 818171095Ssam * most (but not all) possible alignment criteria. 819171095Ssam * If the buffer appears to be aligned, the ULD calls 820171095Ssam * xge_hal_fifo_dtr_buffer_set(). 821171095Ssam * Otherwise, ULD calls xge_hal_fifo_dtr_buffer_set_aligned(). 822171095Ssam * 823171095Ssam * Note; This API is a "superset" of xge_hal_fifo_dtr_buffer_set(). In 824171095Ssam * addition to filling in the specified descriptor it aligns transmit data on 825171095Ssam * the specified boundary. 826171095Ssam * Note: Decision on whether to align or not to align a given contiguous 827171095Ssam * transmit buffer is outside of HAL's domain. To this end ULD can use any 828171095Ssam * programmable criteria, which can help to 1) boost transmit performance, 829171095Ssam * and/or 2) provide a workaround for PCI bridge bugs, if any. 830171095Ssam * 831171095Ssam * See also: xge_hal_fifo_dtr_buffer_set(), 832171095Ssam * xge_hal_check_alignment(). 833171095Ssam * 834171095Ssam * See also: xge_hal_fifo_dtr_reserve(), xge_hal_fifo_dtr_post(), 835171095Ssam * xge_hal_fifo_dtr_mss_set(), xge_hal_fifo_dtr_cksum_set_bits() 836171095Ssam */ 837171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO xge_hal_status_e 838171095Ssamxge_hal_fifo_dtr_buffer_set_aligned(xge_hal_channel_h channelh, 839173139Srwatson xge_hal_dtr_h dtrh, int frag_idx, void *vaddr, 840173139Srwatson dma_addr_t dma_pointer, int size, int misaligned_size) 841171095Ssam{ 842171095Ssam xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; 843171095Ssam xge_hal_fifo_txdl_priv_t *txdl_priv; 844171095Ssam xge_hal_fifo_txd_t *txdp; 845171095Ssam int remaining_size; 846171095Ssam ptrdiff_t prev_boff; 847171095Ssam 848171095Ssam txdl_priv = __hal_fifo_txdl_priv(dtrh); 849171095Ssam txdp = (xge_hal_fifo_txd_t *)dtrh + txdl_priv->frags; 850171095Ssam 851171095Ssam if (frag_idx != 0) { 852173139Srwatson txdp->control_1 = txdp->control_2 = 0; 853171095Ssam } 854171095Ssam 855171095Ssam /* On some systems buffer size could be zero. 856171095Ssam * It is the responsibility of ULD and *not HAL* to 857171095Ssam * detect it and skip it. */ 858171095Ssam xge_assert(size > 0); 859171095Ssam xge_assert(frag_idx < txdl_priv->alloc_frags); 860171095Ssam xge_assert(misaligned_size != 0 && 861173139Srwatson misaligned_size <= fifo->config->alignment_size); 862171095Ssam 863171095Ssam remaining_size = size - misaligned_size; 864171095Ssam xge_assert(remaining_size >= 0); 865171095Ssam 866171095Ssam xge_os_memcpy((char*)txdl_priv->align_vaddr_start, 867173139Srwatson vaddr, misaligned_size); 868171095Ssam 869173139Srwatson if (txdl_priv->align_used_frags >= fifo->config->max_aligned_frags) { 870171095Ssam return XGE_HAL_ERR_OUT_ALIGNED_FRAGS; 871173139Srwatson } 872171095Ssam 873171095Ssam /* setup new buffer */ 874171095Ssam prev_boff = txdl_priv->align_vaddr_start - txdl_priv->align_vaddr; 875171095Ssam txdp->buffer_pointer = (u64)txdl_priv->align_dma_addr + prev_boff; 876171095Ssam txdp->control_1 |= XGE_HAL_TXD_BUFFER0_SIZE(misaligned_size); 877171095Ssam txdl_priv->bytes_sent += misaligned_size; 878171095Ssam fifo->channel.stats.total_buffers++; 879171095Ssam txdl_priv->frags++; 880171095Ssam txdl_priv->align_used_frags++; 881171095Ssam txdl_priv->align_vaddr_start += fifo->config->alignment_size; 882173139Srwatson txdl_priv->align_dma_offset = 0; 883171095Ssam 884171095Ssam#if defined(XGE_OS_DMA_REQUIRES_SYNC) 885171095Ssam /* sync new buffer */ 886171095Ssam xge_os_dma_sync(fifo->channel.pdev, 887173139Srwatson txdl_priv->align_dma_handle, 888173139Srwatson txdp->buffer_pointer, 889173139Srwatson 0, 890173139Srwatson misaligned_size, 891173139Srwatson XGE_OS_DMA_DIR_TODEVICE); 892171095Ssam#endif 893171095Ssam 894171095Ssam if (remaining_size) { 895173139Srwatson xge_assert(frag_idx < txdl_priv->alloc_frags); 896173139Srwatson txdp++; 897173139Srwatson txdp->buffer_pointer = (u64)dma_pointer + 898173139Srwatson misaligned_size; 899173139Srwatson txdp->control_1 = 900173139Srwatson XGE_HAL_TXD_BUFFER0_SIZE(remaining_size); 901173139Srwatson txdl_priv->bytes_sent += remaining_size; 902173139Srwatson txdp->control_2 = 0; 903173139Srwatson fifo->channel.stats.total_buffers++; 904173139Srwatson txdl_priv->frags++; 905171095Ssam } 906171095Ssam 907171095Ssam return XGE_HAL_OK; 908171095Ssam} 909171095Ssam 910171095Ssam/** 911171095Ssam * xge_hal_fifo_dtr_buffer_append - Append the contents of virtually 912171095Ssam * contiguous data buffer to a single physically contiguous buffer. 913171095Ssam * @channelh: Channel handle. 914171095Ssam * @dtrh: Descriptor handle. 915171095Ssam * @vaddr: Virtual address of the data buffer. 916171095Ssam * @size: Size of the data buffer (in bytes). 917171095Ssam * 918171095Ssam * This API is part of the transmit descriptor preparation for posting 919171095Ssam * (via xge_hal_fifo_dtr_post()). 920171095Ssam * The main difference of this API wrt to the APIs 921171095Ssam * xge_hal_fifo_dtr_buffer_set_aligned() is that this API appends the 922171095Ssam * contents of virtually contiguous data buffers received from 923171095Ssam * upper layer into a single physically contiguous data buffer and the 924171095Ssam * device will do a DMA from this buffer. 925171095Ssam * 926171095Ssam * See Also: xge_hal_fifo_dtr_buffer_finalize(), xge_hal_fifo_dtr_buffer_set(), 927171095Ssam * xge_hal_fifo_dtr_buffer_set_aligned(). 928171095Ssam */ 929171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO xge_hal_status_e 930171095Ssamxge_hal_fifo_dtr_buffer_append(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh, 931173139Srwatson void *vaddr, int size) 932171095Ssam{ 933171095Ssam xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; 934171095Ssam xge_hal_fifo_txdl_priv_t *txdl_priv; 935171095Ssam ptrdiff_t used; 936171095Ssam 937171095Ssam xge_assert(size > 0); 938171095Ssam 939171095Ssam txdl_priv = __hal_fifo_txdl_priv(dtrh); 940171095Ssam 941171095Ssam used = txdl_priv->align_vaddr_start - txdl_priv->align_vaddr; 942171095Ssam used += txdl_priv->align_dma_offset; 943171095Ssam if (used + (unsigned int)size > (unsigned int)fifo->align_size) 944171095Ssam return XGE_HAL_ERR_OUT_ALIGNED_FRAGS; 945171095Ssam 946171095Ssam xge_os_memcpy((char*)txdl_priv->align_vaddr_start + 947173139Srwatson txdl_priv->align_dma_offset, vaddr, size); 948171095Ssam 949171095Ssam fifo->channel.stats.copied_frags++; 950171095Ssam 951171095Ssam txdl_priv->align_dma_offset += size; 952171095Ssam return XGE_HAL_OK; 953171095Ssam} 954171095Ssam 955171095Ssam/** 956171095Ssam * xge_hal_fifo_dtr_buffer_finalize - Prepares a descriptor that contains the 957171095Ssam * single physically contiguous buffer. 958171095Ssam * 959171095Ssam * @channelh: Channel handle. 960171095Ssam * @dtrh: Descriptor handle. 961171095Ssam * @frag_idx: Index of the data buffer in the Txdl list. 962171095Ssam * 963171095Ssam * This API in conjuction with xge_hal_fifo_dtr_buffer_append() prepares 964171095Ssam * a descriptor that consists of a single physically contiguous buffer 965171095Ssam * which inturn contains the contents of one or more virtually contiguous 966171095Ssam * buffers received from the upper layer. 967171095Ssam * 968171095Ssam * See Also: xge_hal_fifo_dtr_buffer_append(). 969171095Ssam*/ 970171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO void 971171095Ssamxge_hal_fifo_dtr_buffer_finalize(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh, 972173139Srwatson int frag_idx) 973171095Ssam{ 974171095Ssam xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; 975171095Ssam xge_hal_fifo_txdl_priv_t *txdl_priv; 976171095Ssam xge_hal_fifo_txd_t *txdp; 977171095Ssam ptrdiff_t prev_boff; 978171095Ssam 979171095Ssam xge_assert(frag_idx < fifo->config->max_frags); 980171095Ssam 981171095Ssam txdl_priv = __hal_fifo_txdl_priv(dtrh); 982171095Ssam txdp = (xge_hal_fifo_txd_t *)dtrh + txdl_priv->frags; 983171095Ssam 984171095Ssam if (frag_idx != 0) { 985173139Srwatson txdp->control_1 = txdp->control_2 = 0; 986171095Ssam } 987171095Ssam 988171095Ssam prev_boff = txdl_priv->align_vaddr_start - txdl_priv->align_vaddr; 989171095Ssam txdp->buffer_pointer = (u64)txdl_priv->align_dma_addr + prev_boff; 990171095Ssam txdp->control_1 |= 991173139Srwatson XGE_HAL_TXD_BUFFER0_SIZE(txdl_priv->align_dma_offset); 992171095Ssam txdl_priv->bytes_sent += (unsigned int)txdl_priv->align_dma_offset; 993171095Ssam fifo->channel.stats.total_buffers++; 994171095Ssam fifo->channel.stats.copied_buffers++; 995171095Ssam txdl_priv->frags++; 996171095Ssam txdl_priv->align_used_frags++; 997171095Ssam 998171095Ssam#if defined(XGE_OS_DMA_REQUIRES_SYNC) 999171095Ssam /* sync pre-mapped buffer */ 1000171095Ssam xge_os_dma_sync(fifo->channel.pdev, 1001173139Srwatson txdl_priv->align_dma_handle, 1002173139Srwatson txdp->buffer_pointer, 1003173139Srwatson 0, 1004173139Srwatson txdl_priv->align_dma_offset, 1005173139Srwatson XGE_OS_DMA_DIR_TODEVICE); 1006171095Ssam#endif 1007171095Ssam 1008171095Ssam /* increment vaddr_start for the next buffer_append() iteration */ 1009171095Ssam txdl_priv->align_vaddr_start += txdl_priv->align_dma_offset; 1010173139Srwatson txdl_priv->align_dma_offset = 0; 1011171095Ssam} 1012171095Ssam 1013171095Ssam/** 1014171095Ssam * xge_hal_fifo_dtr_buffer_set - Set transmit buffer pointer in the 1015171095Ssam * descriptor. 1016171095Ssam * @channelh: Channel handle. 1017171095Ssam * @dtrh: Descriptor handle. 1018230133Suqs * @frag_idx: Index of the data buffer in the caller's scatter-gather list 1019171095Ssam * (of buffers). 1020171095Ssam * @dma_pointer: DMA address of the data buffer referenced by @frag_idx. 1021171095Ssam * @size: Size of the data buffer (in bytes). 1022171095Ssam * 1023171095Ssam * This API is part of the preparation of the transmit descriptor for posting 1024171095Ssam * (via xge_hal_fifo_dtr_post()). The related "preparation" APIs include 1025171095Ssam * xge_hal_fifo_dtr_mss_set() and xge_hal_fifo_dtr_cksum_set_bits(). 1026171095Ssam * All three APIs fill in the fields of the fifo descriptor, 1027171095Ssam * in accordance with the Xframe specification. 1028171095Ssam * 1029171095Ssam * See also: xge_hal_fifo_dtr_buffer_set_aligned(), 1030171095Ssam * xge_hal_check_alignment(). 1031171095Ssam * 1032171095Ssam * See also: xge_hal_fifo_dtr_reserve(), xge_hal_fifo_dtr_post(), 1033171095Ssam * xge_hal_fifo_dtr_mss_set(), xge_hal_fifo_dtr_cksum_set_bits() 1034171095Ssam * Prepare transmit descriptor for transmission (via 1035171095Ssam * xge_hal_fifo_dtr_post()). 1036171095Ssam * See also: xge_hal_fifo_dtr_vlan_set(). 1037171095Ssam * Note: Compare with xge_hal_fifo_dtr_buffer_set_aligned(). 1038171095Ssam * 1039171095Ssam * Usage: See ex_xmit{}. 1040171095Ssam */ 1041171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO void 1042171095Ssamxge_hal_fifo_dtr_buffer_set(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh, 1043173139Srwatson int frag_idx, dma_addr_t dma_pointer, int size) 1044171095Ssam{ 1045171095Ssam xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; 1046171095Ssam xge_hal_fifo_txdl_priv_t *txdl_priv; 1047171095Ssam xge_hal_fifo_txd_t *txdp; 1048171095Ssam 1049171095Ssam txdl_priv = __hal_fifo_txdl_priv(dtrh); 1050171095Ssam txdp = (xge_hal_fifo_txd_t *)dtrh + txdl_priv->frags; 1051171095Ssam 1052171095Ssam if (frag_idx != 0) { 1053173139Srwatson txdp->control_1 = txdp->control_2 = 0; 1054171095Ssam } 1055171095Ssam 1056171095Ssam /* Note: 1057171095Ssam * it is the responsibility of upper layers and not HAL 1058171095Ssam * detect it and skip zero-size fragment 1059171095Ssam */ 1060171095Ssam xge_assert(size > 0); 1061171095Ssam xge_assert(frag_idx < txdl_priv->alloc_frags); 1062171095Ssam 1063171095Ssam txdp->buffer_pointer = (u64)dma_pointer; 1064171095Ssam txdp->control_1 |= XGE_HAL_TXD_BUFFER0_SIZE(size); 1065171095Ssam txdl_priv->bytes_sent += size; 1066171095Ssam fifo->channel.stats.total_buffers++; 1067171095Ssam txdl_priv->frags++; 1068171095Ssam} 1069171095Ssam 1070171095Ssam/** 1071171095Ssam * xge_hal_fifo_dtr_mss_set - Set MSS. 1072171095Ssam * @dtrh: Descriptor handle. 1073171095Ssam * @mss: MSS size for _this_ TCP connection. Passed by TCP stack down to the 1074171095Ssam * ULD, which in turn inserts the MSS into the @dtrh. 1075171095Ssam * 1076171095Ssam * This API is part of the preparation of the transmit descriptor for posting 1077171095Ssam * (via xge_hal_fifo_dtr_post()). The related "preparation" APIs include 1078171095Ssam * xge_hal_fifo_dtr_buffer_set(), xge_hal_fifo_dtr_buffer_set_aligned(), 1079171095Ssam * and xge_hal_fifo_dtr_cksum_set_bits(). 1080171095Ssam * All these APIs fill in the fields of the fifo descriptor, 1081171095Ssam * in accordance with the Xframe specification. 1082171095Ssam * 1083171095Ssam * See also: xge_hal_fifo_dtr_reserve(), 1084171095Ssam * xge_hal_fifo_dtr_post(), xge_hal_fifo_dtr_vlan_set(). 1085171095Ssam * Usage: See ex_xmit{}. 1086171095Ssam */ 1087171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO void 1088171095Ssamxge_hal_fifo_dtr_mss_set(xge_hal_dtr_h dtrh, int mss) 1089171095Ssam{ 1090171095Ssam xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)dtrh; 1091171095Ssam 1092171095Ssam txdp->control_1 |= XGE_HAL_TXD_LSO_COF_CTRL(XGE_HAL_TXD_TCP_LSO); 1093171095Ssam txdp->control_1 |= XGE_HAL_TXD_TCP_LSO_MSS(mss); 1094171095Ssam} 1095171095Ssam 1096171095Ssam/** 1097171095Ssam * xge_hal_fifo_dtr_cksum_set_bits - Offload checksum. 1098171095Ssam * @dtrh: Descriptor handle. 1099171095Ssam * @cksum_bits: Specifies which checksums are to be offloaded: IPv4, 1100171095Ssam * and/or TCP and/or UDP. 1101171095Ssam * 1102171095Ssam * Ask Xframe to calculate IPv4 & transport checksums for _this_ transmit 1103171095Ssam * descriptor. 1104171095Ssam * This API is part of the preparation of the transmit descriptor for posting 1105171095Ssam * (via xge_hal_fifo_dtr_post()). The related "preparation" APIs include 1106171095Ssam * xge_hal_fifo_dtr_mss_set(), xge_hal_fifo_dtr_buffer_set_aligned(), 1107171095Ssam * and xge_hal_fifo_dtr_buffer_set(). 1108171095Ssam * All these APIs fill in the fields of the fifo descriptor, 1109171095Ssam * in accordance with the Xframe specification. 1110171095Ssam * 1111171095Ssam * See also: xge_hal_fifo_dtr_reserve(), 1112171095Ssam * xge_hal_fifo_dtr_post(), XGE_HAL_TXD_TX_CKO_IPV4_EN, 1113171095Ssam * XGE_HAL_TXD_TX_CKO_TCP_EN. 1114171095Ssam * Usage: See ex_xmit{}. 1115171095Ssam */ 1116171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO void 1117171095Ssamxge_hal_fifo_dtr_cksum_set_bits(xge_hal_dtr_h dtrh, u64 cksum_bits) 1118171095Ssam{ 1119171095Ssam xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)dtrh; 1120171095Ssam 1121171095Ssam txdp->control_2 |= cksum_bits; 1122171095Ssam} 1123171095Ssam 1124171095Ssam 1125171095Ssam/** 1126171095Ssam * xge_hal_fifo_dtr_vlan_set - Set VLAN tag. 1127171095Ssam * @dtrh: Descriptor handle. 1128171095Ssam * @vlan_tag: 16bit VLAN tag. 1129171095Ssam * 1130171095Ssam * Insert VLAN tag into specified transmit descriptor. 1131171095Ssam * The actual insertion of the tag into outgoing frame is done by the hardware. 1132171095Ssam * See also: xge_hal_fifo_dtr_buffer_set(), xge_hal_fifo_dtr_mss_set(). 1133171095Ssam */ 1134171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO void 1135171095Ssamxge_hal_fifo_dtr_vlan_set(xge_hal_dtr_h dtrh, u16 vlan_tag) 1136171095Ssam{ 1137171095Ssam xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)dtrh; 1138171095Ssam 1139171095Ssam txdp->control_2 |= XGE_HAL_TXD_VLAN_ENABLE; 1140171095Ssam txdp->control_2 |= XGE_HAL_TXD_VLAN_TAG(vlan_tag); 1141171095Ssam} 1142171095Ssam 1143171095Ssam/** 1144171095Ssam * xge_hal_fifo_is_next_dtr_completed - Checks if the next dtr is completed 1145171095Ssam * @channelh: Channel handle. 1146171095Ssam */ 1147171095Ssam__HAL_STATIC_FIFO __HAL_INLINE_FIFO xge_hal_status_e 1148171095Ssamxge_hal_fifo_is_next_dtr_completed(xge_hal_channel_h channelh) 1149171095Ssam{ 1150171095Ssam xge_hal_fifo_txd_t *txdp; 1151171095Ssam xge_hal_dtr_h dtrh; 1152171095Ssam 1153171095Ssam __hal_channel_dtr_try_complete(channelh, &dtrh); 1154171095Ssam txdp = (xge_hal_fifo_txd_t *)dtrh; 1155171095Ssam if (txdp == NULL) { 1156173139Srwatson return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS; 1157171095Ssam } 1158171095Ssam 1159171095Ssam /* check whether host owns it */ 1160171095Ssam if ( !(txdp->control_1 & XGE_HAL_TXD_LIST_OWN_XENA) ) { 1161173139Srwatson xge_assert(txdp->host_control!=0); 1162173139Srwatson return XGE_HAL_OK; 1163171095Ssam } 1164171095Ssam 1165171095Ssam /* no more completions */ 1166171095Ssam return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS; 1167171095Ssam} 1168