1/*- 2 * Copyright (c) 2002-2007 Neterion, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29#ifdef XGE_DEBUG_FP 30#include <dev/nxge/include/xgehal-device.h> 31#endif 32 33#include <dev/nxge/include/xgehal-ring.h> 34#include <dev/nxge/include/xgehal-fifo.h> 35 36/** 37 * xge_hal_device_bar0 - Get BAR0 mapped address. 38 * @hldev: HAL device handle. 39 * 40 * Returns: BAR0 address of the specified device. 41 */ 42__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE char * 43xge_hal_device_bar0(xge_hal_device_t *hldev) 44{ 45 return hldev->bar0; 46} 47 48/** 49 * xge_hal_device_isrbar0 - Get BAR0 mapped address. 50 * @hldev: HAL device handle. 51 * 52 * Returns: BAR0 address of the specified device. 53 */ 54__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE char * 55xge_hal_device_isrbar0(xge_hal_device_t *hldev) 56{ 57 return hldev->isrbar0; 58} 59 60/** 61 * xge_hal_device_bar1 - Get BAR1 mapped address. 62 * @hldev: HAL device handle. 63 * 64 * Returns: BAR1 address of the specified device. 65 */ 66__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE char * 67xge_hal_device_bar1(xge_hal_device_t *hldev) 68{ 69 return hldev->bar1; 70} 71 72/** 73 * xge_hal_device_bar0_set - Set BAR0 mapped address. 74 * @hldev: HAL device handle. 75 * @bar0: BAR0 mapped address. 76 * * Set BAR0 address in the HAL device object. 77 */ 78__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void 79xge_hal_device_bar0_set(xge_hal_device_t *hldev, char *bar0) 80{ 81 xge_assert(bar0); 82 hldev->bar0 = bar0; 83} 84 85/** 86 * xge_hal_device_isrbar0_set - Set BAR0 mapped address. 87 * @hldev: HAL device handle. 88 * @isrbar0: BAR0 mapped address. 89 * * Set BAR0 address in the HAL device object. 90 */ 91__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void 92xge_hal_device_isrbar0_set(xge_hal_device_t *hldev, char *isrbar0) 93{ 94 xge_assert(isrbar0); 95 hldev->isrbar0 = isrbar0; 96} 97 98/** 99 * xge_hal_device_bar1_set - Set BAR1 mapped address. 100 * @hldev: HAL device handle. 101 * @channelh: Channel handle. 102 * @bar1: BAR1 mapped address. 103 * 104 * Set BAR1 address for the given channel. 105 */ 106__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void 107xge_hal_device_bar1_set(xge_hal_device_t *hldev, xge_hal_channel_h channelh, 108 char *bar1) 109{ 110 xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; 111 112 xge_assert(bar1); 113 xge_assert(fifo); 114 115 /* Initializing the BAR1 address as the start of 116 * the FIFO queue pointer and as a location of FIFO control 117 * word. */ 118 fifo->hw_pair = 119 (xge_hal_fifo_hw_pair_t *) (bar1 + 120 (fifo->channel.post_qid * XGE_HAL_FIFO_HW_PAIR_OFFSET)); 121 hldev->bar1 = bar1; 122} 123 124 125/** 126 * xge_hal_device_rev - Get Device revision number. 127 * @hldev: HAL device handle. 128 * 129 * Returns: Device revision number 130 */ 131__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE int 132xge_hal_device_rev(xge_hal_device_t *hldev) 133{ 134 return hldev->revision; 135} 136 137 138/** 139 * xge_hal_device_begin_irq - Begin IRQ processing. 140 * @hldev: HAL device handle. 141 * @reason: "Reason" for the interrupt, the value of Xframe's 142 * general_int_status register. 143 * 144 * The function performs two actions, It first checks whether (shared IRQ) the 145 * interrupt was raised by the device. Next, it masks the device interrupts. 146 * 147 * Note: 148 * xge_hal_device_begin_irq() does not flush MMIO writes through the 149 * bridge. Therefore, two back-to-back interrupts are potentially possible. 150 * It is the responsibility of the ULD to make sure that only one 151 * xge_hal_device_continue_irq() runs at a time. 152 * 153 * Returns: 0, if the interrupt is not "ours" (note that in this case the 154 * device remain enabled). 155 * Otherwise, xge_hal_device_begin_irq() returns 64bit general adapter 156 * status. 157 * See also: xge_hal_device_handle_irq() 158 */ 159__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e 160xge_hal_device_begin_irq(xge_hal_device_t *hldev, u64 *reason) 161{ 162 u64 val64; 163 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0; 164 165 hldev->stats.sw_dev_info_stats.total_intr_cnt++; 166 167 val64 = xge_os_pio_mem_read64(hldev->pdev, 168 hldev->regh0, &isrbar0->general_int_status); 169 if (xge_os_unlikely(!val64)) { 170 /* not Xframe interrupt */ 171 hldev->stats.sw_dev_info_stats.not_xge_intr_cnt++; 172 *reason = 0; 173 return XGE_HAL_ERR_WRONG_IRQ; 174 } 175 176 if (xge_os_unlikely(val64 == XGE_HAL_ALL_FOXES)) { 177 u64 adapter_status = 178 xge_os_pio_mem_read64(hldev->pdev, hldev->regh0, 179 &isrbar0->adapter_status); 180 if (adapter_status == XGE_HAL_ALL_FOXES) { 181 (void) xge_queue_produce(hldev->queueh, 182 XGE_HAL_EVENT_SLOT_FREEZE, 183 hldev, 184 1, /* critical: slot freeze */ 185 sizeof(u64), 186 (void*)&adapter_status); 187 *reason = 0; 188 return XGE_HAL_ERR_CRITICAL; 189 } 190 } 191 192 *reason = val64; 193 194 /* separate fast path, i.e. no errors */ 195 if (val64 & XGE_HAL_GEN_INTR_RXTRAFFIC) { 196 hldev->stats.sw_dev_info_stats.rx_traffic_intr_cnt++; 197 return XGE_HAL_OK; 198 } 199 if (val64 & XGE_HAL_GEN_INTR_TXTRAFFIC) { 200 hldev->stats.sw_dev_info_stats.tx_traffic_intr_cnt++; 201 return XGE_HAL_OK; 202 } 203 204 hldev->stats.sw_dev_info_stats.not_traffic_intr_cnt++; 205 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXPIC)) { 206 xge_hal_status_e status; 207 hldev->stats.sw_dev_info_stats.txpic_intr_cnt++; 208 status = __hal_device_handle_txpic(hldev, val64); 209 if (status != XGE_HAL_OK) { 210 return status; 211 } 212 } 213 214 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXDMA)) { 215 xge_hal_status_e status; 216 hldev->stats.sw_dev_info_stats.txdma_intr_cnt++; 217 status = __hal_device_handle_txdma(hldev, val64); 218 if (status != XGE_HAL_OK) { 219 return status; 220 } 221 } 222 223 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXMAC)) { 224 xge_hal_status_e status; 225 hldev->stats.sw_dev_info_stats.txmac_intr_cnt++; 226 status = __hal_device_handle_txmac(hldev, val64); 227 if (status != XGE_HAL_OK) { 228 return status; 229 } 230 } 231 232 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXXGXS)) { 233 xge_hal_status_e status; 234 hldev->stats.sw_dev_info_stats.txxgxs_intr_cnt++; 235 status = __hal_device_handle_txxgxs(hldev, val64); 236 if (status != XGE_HAL_OK) { 237 return status; 238 } 239 } 240 241 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXPIC)) { 242 xge_hal_status_e status; 243 hldev->stats.sw_dev_info_stats.rxpic_intr_cnt++; 244 status = __hal_device_handle_rxpic(hldev, val64); 245 if (status != XGE_HAL_OK) { 246 return status; 247 } 248 } 249 250 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXDMA)) { 251 xge_hal_status_e status; 252 hldev->stats.sw_dev_info_stats.rxdma_intr_cnt++; 253 status = __hal_device_handle_rxdma(hldev, val64); 254 if (status != XGE_HAL_OK) { 255 return status; 256 } 257 } 258 259 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXMAC)) { 260 xge_hal_status_e status; 261 hldev->stats.sw_dev_info_stats.rxmac_intr_cnt++; 262 status = __hal_device_handle_rxmac(hldev, val64); 263 if (status != XGE_HAL_OK) { 264 return status; 265 } 266 } 267 268 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXXGXS)) { 269 xge_hal_status_e status; 270 hldev->stats.sw_dev_info_stats.rxxgxs_intr_cnt++; 271 status = __hal_device_handle_rxxgxs(hldev, val64); 272 if (status != XGE_HAL_OK) { 273 return status; 274 } 275 } 276 277 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_MC)) { 278 xge_hal_status_e status; 279 hldev->stats.sw_dev_info_stats.mc_intr_cnt++; 280 status = __hal_device_handle_mc(hldev, val64); 281 if (status != XGE_HAL_OK) { 282 return status; 283 } 284 } 285 286 return XGE_HAL_OK; 287} 288 289/** 290 * xge_hal_device_clear_rx - Acknowledge (that is, clear) the 291 * condition that has caused the RX interrupt. 292 * @hldev: HAL device handle. 293 * 294 * Acknowledge (that is, clear) the condition that has caused 295 * the Rx interrupt. 296 * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq(), 297 * xge_hal_device_clear_tx(), xge_hal_device_mask_rx(). 298 */ 299__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void 300xge_hal_device_clear_rx(xge_hal_device_t *hldev) 301{ 302 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0; 303 304 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 305 0xFFFFFFFFFFFFFFFFULL, 306 &isrbar0->rx_traffic_int); 307} 308 309/** 310 * xge_hal_device_clear_tx - Acknowledge (that is, clear) the 311 * condition that has caused the TX interrupt. 312 * @hldev: HAL device handle. 313 * 314 * Acknowledge (that is, clear) the condition that has caused 315 * the Tx interrupt. 316 * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq(), 317 * xge_hal_device_clear_rx(), xge_hal_device_mask_tx(). 318 */ 319__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void 320xge_hal_device_clear_tx(xge_hal_device_t *hldev) 321{ 322 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0; 323 324 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 325 0xFFFFFFFFFFFFFFFFULL, 326 &isrbar0->tx_traffic_int); 327} 328 329/** 330 * xge_hal_device_poll_rx_channel - Poll Rx channel for completed 331 * descriptors and process the same. 332 * @channel: HAL channel. 333 * @got_rx: Buffer to return the flag set if receive interrupt is occured 334 * 335 * The function polls the Rx channel for the completed descriptors and calls 336 * the upper-layer driver (ULD) via supplied completion callback. 337 * 338 * Returns: XGE_HAL_OK, if the polling is completed successful. 339 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed 340 * descriptors available which are yet to be processed. 341 * 342 * See also: xge_hal_device_poll_tx_channel() 343 */ 344__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e 345xge_hal_device_poll_rx_channel(xge_hal_channel_t *channel, int *got_rx) 346{ 347 xge_hal_status_e ret = XGE_HAL_OK; 348 xge_hal_dtr_h first_dtrh; 349 xge_hal_device_t *hldev = (xge_hal_device_t *)channel->devh; 350 u8 t_code; 351 int got_bytes; 352 353 /* for each opened rx channel */ 354 got_bytes = *got_rx = 0; 355 ((xge_hal_ring_t *)channel)->cmpl_cnt = 0; 356 channel->poll_bytes = 0; 357 if ((ret = xge_hal_ring_dtr_next_completed (channel, &first_dtrh, 358 &t_code)) == XGE_HAL_OK) { 359 if (channel->callback(channel, first_dtrh, 360 t_code, channel->userdata) != XGE_HAL_OK) { 361 (*got_rx) += ((xge_hal_ring_t *)channel)->cmpl_cnt + 1; 362 got_bytes += channel->poll_bytes + 1; 363 ret = XGE_HAL_COMPLETIONS_REMAIN; 364 } else { 365 (*got_rx) += ((xge_hal_ring_t *)channel)->cmpl_cnt + 1; 366 got_bytes += channel->poll_bytes + 1; 367 } 368 } 369 370 if (*got_rx) { 371 hldev->irq_workload_rxd[channel->post_qid] += *got_rx; 372 hldev->irq_workload_rxcnt[channel->post_qid] ++; 373 } 374 hldev->irq_workload_rxlen[channel->post_qid] += got_bytes; 375 376 return ret; 377} 378 379/** 380 * xge_hal_device_poll_tx_channel - Poll Tx channel for completed 381 * descriptors and process the same. 382 * @channel: HAL channel. 383 * @got_tx: Buffer to return the flag set if transmit interrupt is occured 384 * 385 * The function polls the Tx channel for the completed descriptors and calls 386 * the upper-layer driver (ULD) via supplied completion callback. 387 * 388 * Returns: XGE_HAL_OK, if the polling is completed successful. 389 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed 390 * descriptors available which are yet to be processed. 391 * 392 * See also: xge_hal_device_poll_rx_channel(). 393 */ 394__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e 395xge_hal_device_poll_tx_channel(xge_hal_channel_t *channel, int *got_tx) 396{ 397 xge_hal_dtr_h first_dtrh; 398 xge_hal_device_t *hldev = (xge_hal_device_t *)channel->devh; 399 u8 t_code; 400 int got_bytes; 401 402 /* for each opened tx channel */ 403 got_bytes = *got_tx = 0; 404 channel->poll_bytes = 0; 405 if (xge_hal_fifo_dtr_next_completed (channel, &first_dtrh, 406 &t_code) == XGE_HAL_OK) { 407 if (channel->callback(channel, first_dtrh, 408 t_code, channel->userdata) != XGE_HAL_OK) { 409 (*got_tx)++; 410 got_bytes += channel->poll_bytes + 1; 411 return XGE_HAL_COMPLETIONS_REMAIN; 412 } 413 (*got_tx)++; 414 got_bytes += channel->poll_bytes + 1; 415 } 416 417 if (*got_tx) { 418 hldev->irq_workload_txd[channel->post_qid] += *got_tx; 419 hldev->irq_workload_txcnt[channel->post_qid] ++; 420 } 421 hldev->irq_workload_txlen[channel->post_qid] += got_bytes; 422 423 return XGE_HAL_OK; 424} 425 426/** 427 * xge_hal_device_poll_rx_channels - Poll Rx channels for completed 428 * descriptors and process the same. 429 * @hldev: HAL device handle. 430 * @got_rx: Buffer to return flag set if receive is ready 431 * 432 * The function polls the Rx channels for the completed descriptors and calls 433 * the upper-layer driver (ULD) via supplied completion callback. 434 * 435 * Returns: XGE_HAL_OK, if the polling is completed successful. 436 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed 437 * descriptors available which are yet to be processed. 438 * 439 * See also: xge_hal_device_poll_tx_channels(), xge_hal_device_continue_irq(). 440 */ 441__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e 442xge_hal_device_poll_rx_channels(xge_hal_device_t *hldev, int *got_rx) 443{ 444 xge_list_t *item; 445 xge_hal_channel_t *channel; 446 447 /* for each opened rx channel */ 448 xge_list_for_each(item, &hldev->ring_channels) { 449 if (hldev->terminating) 450 return XGE_HAL_OK; 451 channel = xge_container_of(item, xge_hal_channel_t, item); 452 (void) xge_hal_device_poll_rx_channel(channel, got_rx); 453 } 454 455 return XGE_HAL_OK; 456} 457 458/** 459 * xge_hal_device_poll_tx_channels - Poll Tx channels for completed 460 * descriptors and process the same. 461 * @hldev: HAL device handle. 462 * @got_tx: Buffer to return flag set if transmit is ready 463 * 464 * The function polls the Tx channels for the completed descriptors and calls 465 * the upper-layer driver (ULD) via supplied completion callback. 466 * 467 * Returns: XGE_HAL_OK, if the polling is completed successful. 468 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed 469 * descriptors available which are yet to be processed. 470 * 471 * See also: xge_hal_device_poll_rx_channels(), xge_hal_device_continue_irq(). 472 */ 473__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e 474xge_hal_device_poll_tx_channels(xge_hal_device_t *hldev, int *got_tx) 475{ 476 xge_list_t *item; 477 xge_hal_channel_t *channel; 478 479 /* for each opened tx channel */ 480 xge_list_for_each(item, &hldev->fifo_channels) { 481 if (hldev->terminating) 482 return XGE_HAL_OK; 483 channel = xge_container_of(item, xge_hal_channel_t, item); 484 (void) xge_hal_device_poll_tx_channel(channel, got_tx); 485 } 486 487 return XGE_HAL_OK; 488} 489 490/** 491 * xge_hal_device_mask_tx - Mask Tx interrupts. 492 * @hldev: HAL device handle. 493 * 494 * Mask Tx device interrupts. 495 * 496 * See also: xge_hal_device_unmask_tx(), xge_hal_device_mask_rx(), 497 * xge_hal_device_clear_tx(). 498 */ 499__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void 500xge_hal_device_mask_tx(xge_hal_device_t *hldev) 501{ 502 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0; 503 504 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 505 0xFFFFFFFFFFFFFFFFULL, 506 &isrbar0->tx_traffic_mask); 507} 508 509/** 510 * xge_hal_device_mask_rx - Mask Rx interrupts. 511 * @hldev: HAL device handle. 512 * 513 * Mask Rx device interrupts. 514 * 515 * See also: xge_hal_device_unmask_rx(), xge_hal_device_mask_tx(), 516 * xge_hal_device_clear_rx(). 517 */ 518__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void 519xge_hal_device_mask_rx(xge_hal_device_t *hldev) 520{ 521 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0; 522 523 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 524 0xFFFFFFFFFFFFFFFFULL, 525 &isrbar0->rx_traffic_mask); 526} 527 528/** 529 * xge_hal_device_mask_all - Mask all device interrupts. 530 * @hldev: HAL device handle. 531 * 532 * Mask all device interrupts. 533 * 534 * See also: xge_hal_device_unmask_all() 535 */ 536__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void 537xge_hal_device_mask_all(xge_hal_device_t *hldev) 538{ 539 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0; 540 541 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 542 0xFFFFFFFFFFFFFFFFULL, 543 &isrbar0->general_int_mask); 544} 545 546/** 547 * xge_hal_device_unmask_tx - Unmask Tx interrupts. 548 * @hldev: HAL device handle. 549 * 550 * Unmask Tx device interrupts. 551 * 552 * See also: xge_hal_device_mask_tx(), xge_hal_device_clear_tx(). 553 */ 554__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void 555xge_hal_device_unmask_tx(xge_hal_device_t *hldev) 556{ 557 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0; 558 559 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 560 0x0ULL, 561 &isrbar0->tx_traffic_mask); 562} 563 564/** 565 * xge_hal_device_unmask_rx - Unmask Rx interrupts. 566 * @hldev: HAL device handle. 567 * 568 * Unmask Rx device interrupts. 569 * 570 * See also: xge_hal_device_mask_rx(), xge_hal_device_clear_rx(). 571 */ 572__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void 573xge_hal_device_unmask_rx(xge_hal_device_t *hldev) 574{ 575 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0; 576 577 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 578 0x0ULL, 579 &isrbar0->rx_traffic_mask); 580} 581 582/** 583 * xge_hal_device_unmask_all - Unmask all device interrupts. 584 * @hldev: HAL device handle. 585 * 586 * Unmask all device interrupts. 587 * 588 * See also: xge_hal_device_mask_all() 589 */ 590__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void 591xge_hal_device_unmask_all(xge_hal_device_t *hldev) 592{ 593 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0; 594 595 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 596 0x0ULL, 597 &isrbar0->general_int_mask); 598} 599 600 601/** 602 * xge_hal_device_continue_irq - Continue handling IRQ: process all 603 * completed descriptors. 604 * @hldev: HAL device handle. 605 * 606 * Process completed descriptors and unmask the device interrupts. 607 * 608 * The xge_hal_device_continue_irq() walks all open channels 609 * and calls upper-layer driver (ULD) via supplied completion 610 * callback. Note that the completion callback is specified at channel open 611 * time, see xge_hal_channel_open(). 612 * 613 * Note that the xge_hal_device_continue_irq is part of the _fast_ path. 614 * To optimize the processing, the function does _not_ check for 615 * errors and alarms. 616 * 617 * The latter is done in a polling fashion, via xge_hal_device_poll(). 618 * 619 * Returns: XGE_HAL_OK. 620 * 621 * See also: xge_hal_device_handle_irq(), xge_hal_device_poll(), 622 * xge_hal_ring_dtr_next_completed(), 623 * xge_hal_fifo_dtr_next_completed(), xge_hal_channel_callback_f{}. 624 */ 625__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e 626xge_hal_device_continue_irq(xge_hal_device_t *hldev) 627{ 628 int got_rx = 1, got_tx = 1; 629 int isr_polling_cnt = hldev->config.isr_polling_cnt; 630 int count = 0; 631 632 do 633 { 634 if (got_rx) 635 (void) xge_hal_device_poll_rx_channels(hldev, &got_rx); 636 if (got_tx && hldev->tti_enabled) 637 (void) xge_hal_device_poll_tx_channels(hldev, &got_tx); 638 639 if (!got_rx && !got_tx) 640 break; 641 642 count += (got_rx + got_tx); 643 }while (isr_polling_cnt--); 644 645 if (!count) 646 hldev->stats.sw_dev_info_stats.not_traffic_intr_cnt++; 647 648 return XGE_HAL_OK; 649} 650 651/** 652 * xge_hal_device_handle_irq - Handle device IRQ. 653 * @hldev: HAL device handle. 654 * 655 * Perform the complete handling of the line interrupt. The function 656 * performs two calls. 657 * First it uses xge_hal_device_begin_irq() to check the reason for 658 * the interrupt and mask the device interrupts. 659 * Second, it calls xge_hal_device_continue_irq() to process all 660 * completed descriptors and re-enable the interrupts. 661 * 662 * Returns: XGE_HAL_OK - success; 663 * XGE_HAL_ERR_WRONG_IRQ - (shared) IRQ produced by other device. 664 * 665 * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq(). 666 */ 667__HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e 668xge_hal_device_handle_irq(xge_hal_device_t *hldev) 669{ 670 u64 reason; 671 xge_hal_status_e status; 672 673 xge_hal_device_mask_all(hldev); 674 675 status = xge_hal_device_begin_irq(hldev, &reason); 676 if (status != XGE_HAL_OK) { 677 xge_hal_device_unmask_all(hldev); 678 return status; 679 } 680 681 if (reason & XGE_HAL_GEN_INTR_RXTRAFFIC) { 682 xge_hal_device_clear_rx(hldev); 683 } 684 685 status = xge_hal_device_continue_irq(hldev); 686 687 xge_hal_device_clear_tx(hldev); 688 689 xge_hal_device_unmask_all(hldev); 690 691 return status; 692} 693 694#if defined(XGE_HAL_CONFIG_LRO) 695 696 697__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL int 698__hal_lro_check_for_session_match(lro_t *lro, tcplro_t *tcp, iplro_t *ip) 699{ 700 701 /* Match Source address field */ 702 if ((lro->ip_hdr->saddr != ip->saddr)) 703 return XGE_HAL_FAIL; 704 705 /* Match Destination address field */ 706 if ((lro->ip_hdr->daddr != ip->daddr)) 707 return XGE_HAL_FAIL; 708 709 /* Match Source Port field */ 710 if ((lro->tcp_hdr->source != tcp->source)) 711 return XGE_HAL_FAIL; 712 713 /* Match Destination Port field */ 714 if ((lro->tcp_hdr->dest != tcp->dest)) 715 return XGE_HAL_FAIL; 716 717 return XGE_HAL_OK; 718} 719 720/* 721 * __hal_tcp_seg_len: Find the tcp seg len. 722 * @ip: ip header. 723 * @tcp: tcp header. 724 * returns: Tcp seg length. 725 */ 726__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL u16 727__hal_tcp_seg_len(iplro_t *ip, tcplro_t *tcp) 728{ 729 u16 ret; 730 731 ret = (xge_os_ntohs(ip->tot_len) - 732 ((ip->version_ihl & 0x0F)<<2) - 733 ((tcp->doff_res)>>2)); 734 return (ret); 735} 736 737/* 738 * __hal_ip_lro_capable: Finds whether ip is lro capable. 739 * @ip: ip header. 740 * @ext_info: descriptor info. 741 */ 742__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e 743__hal_ip_lro_capable(iplro_t *ip, 744 xge_hal_dtr_info_t *ext_info) 745{ 746 747#ifdef XGE_LL_DEBUG_DUMP_PKT 748 { 749 u16 i; 750 u8 ch, *iph = (u8 *)ip; 751 752 xge_debug_ring(XGE_TRACE, "Dump Ip:" ); 753 for (i =0; i < 40; i++) { 754 ch = ntohs(*((u8 *)(iph + i)) ); 755 printf("i:%d %02x, ",i,ch); 756 } 757 } 758#endif 759 760 if (ip->version_ihl != IP_FAST_PATH_HDR_MASK) { 761 xge_debug_ring(XGE_ERR, "iphdr !=45 :%d",ip->version_ihl); 762 return XGE_HAL_FAIL; 763 } 764 765 if (ext_info->proto & XGE_HAL_FRAME_PROTO_IP_FRAGMENTED) { 766 xge_debug_ring(XGE_ERR, "IP fragmented"); 767 return XGE_HAL_FAIL; 768 } 769 770 return XGE_HAL_OK; 771} 772 773/* 774 * __hal_tcp_lro_capable: Finds whether tcp is lro capable. 775 * @ip: ip header. 776 * @tcp: tcp header. 777 */ 778__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e 779__hal_tcp_lro_capable(iplro_t *ip, tcplro_t *tcp, lro_t *lro, int *ts_off) 780{ 781#ifdef XGE_LL_DEBUG_DUMP_PKT 782 { 783 u8 ch; 784 u16 i; 785 786 xge_debug_ring(XGE_TRACE, "Dump Tcp:" ); 787 for (i =0; i < 20; i++) { 788 ch = ntohs(*((u8 *)((u8 *)tcp + i)) ); 789 xge_os_printf("i:%d %02x, ",i,ch); 790 } 791 } 792#endif 793 if ((TCP_FAST_PATH_HDR_MASK2 != tcp->ctrl) && 794 (TCP_FAST_PATH_HDR_MASK3 != tcp->ctrl)) 795 goto _exit_fail; 796 797 *ts_off = -1; 798 if (TCP_FAST_PATH_HDR_MASK1 != tcp->doff_res) { 799 u16 tcp_hdr_len = tcp->doff_res >> 2; /* TCP header len */ 800 u16 off = 20; /* Start of tcp options */ 801 int i, diff; 802 803 /* Does Packet can contain time stamp */ 804 if (tcp_hdr_len < 32) { 805 /* 806 * If the session is not opened, we can consider 807 * this packet for LRO 808 */ 809 if (lro == NULL) 810 return XGE_HAL_OK; 811 812 goto _exit_fail; 813 } 814 815 /* Ignore No-operation 0x1 */ 816 while (((u8 *)tcp)[off] == 0x1) 817 off++; 818 819 /* Next option == Timestamp */ 820 if (((u8 *)tcp)[off] != 0x8) { 821 /* 822 * If the session ie not opened, we can consider 823 * this packet for LRO 824 */ 825 if (lro == NULL) 826 return XGE_HAL_OK; 827 828 goto _exit_fail; 829 } 830 831 *ts_off = off; 832 if (lro == NULL) 833 return XGE_HAL_OK; 834 835 /* 836 * Now the session is opened. If the LRO frame doesn't 837 * have time stamp, we cannot consider current packet for 838 * LRO. 839 */ 840 if (lro->ts_off == -1) { 841 xge_debug_ring(XGE_ERR, "Pkt received with time stamp after session opened with no time stamp : %02x %02x", tcp->doff_res, tcp->ctrl); 842 return XGE_HAL_FAIL; 843 } 844 845 /* 846 * If the difference is greater than three, then there are 847 * more options possible. 848 * else, there are two cases: 849 * case 1: remaining are padding bytes. 850 * case 2: remaining can contain options or padding 851 */ 852 off += ((u8 *)tcp)[off+1]; 853 diff = tcp_hdr_len - off; 854 if (diff > 3) { 855 /* 856 * Probably contains more options. 857 */ 858 xge_debug_ring(XGE_ERR, "tcphdr not fastpth : pkt received with tcp options in addition to time stamp after the session is opened %02x %02x ", tcp->doff_res, tcp->ctrl); 859 return XGE_HAL_FAIL; 860 } 861 862 for (i = 0; i < diff; i++) { 863 u8 byte = ((u8 *)tcp)[off+i]; 864 865 /* Ignore No-operation 0x1 */ 866 if ((byte == 0x0) || (byte == 0x1)) 867 continue; 868 xge_debug_ring(XGE_ERR, "tcphdr not fastpth : pkt received with tcp options in addition to time stamp after the session is opened %02x %02x ", tcp->doff_res, tcp->ctrl); 869 return XGE_HAL_FAIL; 870 } 871 872 /* 873 * Update the time stamp of LRO frame. 874 */ 875 xge_os_memcpy(((char *)lro->tcp_hdr + lro->ts_off + 2), 876 (char *)((char *)tcp + (*ts_off) + 2), 8); 877 } 878 879 return XGE_HAL_OK; 880 881_exit_fail: 882 xge_debug_ring(XGE_TRACE, "tcphdr not fastpth %02x %02x", tcp->doff_res, tcp->ctrl); 883 return XGE_HAL_FAIL; 884 885} 886 887/* 888 * __hal_lro_capable: Finds whether frame is lro capable. 889 * @buffer: Ethernet frame. 890 * @ip: ip frame. 891 * @tcp: tcp frame. 892 * @ext_info: Descriptor info. 893 */ 894__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e 895__hal_lro_capable( u8 *buffer, 896 iplro_t **ip, 897 tcplro_t **tcp, 898 xge_hal_dtr_info_t *ext_info) 899{ 900 u8 ip_off, ip_length; 901 902 if (!(ext_info->proto & XGE_HAL_FRAME_PROTO_TCP)) { 903 xge_debug_ring(XGE_ERR, "Cant do lro %d", ext_info->proto); 904 return XGE_HAL_FAIL; 905 } 906 907 if ( !*ip ) 908 { 909#ifdef XGE_LL_DEBUG_DUMP_PKT 910 { 911 u8 ch; 912 u16 i; 913 914 xge_os_printf("Dump Eth:" ); 915 for (i =0; i < 60; i++) { 916 ch = ntohs(*((u8 *)(buffer + i)) ); 917 xge_os_printf("i:%d %02x, ",i,ch); 918 } 919 } 920#endif 921 922 switch (ext_info->frame) { 923 case XGE_HAL_FRAME_TYPE_DIX: 924 ip_off = XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE; 925 break; 926 case XGE_HAL_FRAME_TYPE_LLC: 927 ip_off = (XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE + 928 XGE_HAL_HEADER_802_2_SIZE); 929 break; 930 case XGE_HAL_FRAME_TYPE_SNAP: 931 ip_off = (XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE + 932 XGE_HAL_HEADER_SNAP_SIZE); 933 break; 934 default: // XGE_HAL_FRAME_TYPE_IPX, etc. 935 return XGE_HAL_FAIL; 936 } 937 938 939 if (ext_info->proto & XGE_HAL_FRAME_PROTO_VLAN_TAGGED) { 940 ip_off += XGE_HAL_HEADER_VLAN_SIZE; 941 } 942 943 /* Grab ip, tcp headers */ 944 *ip = (iplro_t *)((char*)buffer + ip_off); 945 } /* !*ip */ 946 947 ip_length = (u8)((*ip)->version_ihl & 0x0F); 948 ip_length = ip_length <<2; 949 *tcp = (tcplro_t *)((char *)*ip + ip_length); 950 951 xge_debug_ring(XGE_TRACE, "ip_length:%d ip:"XGE_OS_LLXFMT 952 " tcp:"XGE_OS_LLXFMT"", (int)ip_length, 953 (unsigned long long)(ulong_t)*ip, (unsigned long long)(ulong_t)*tcp); 954 955 return XGE_HAL_OK; 956 957} 958 959 960/* 961 * __hal_open_lro_session: Open a new LRO session. 962 * @buffer: Ethernet frame. 963 * @ip: ip header. 964 * @tcp: tcp header. 965 * @lro: lro pointer 966 * @ext_info: Descriptor info. 967 * @hldev: Hal context. 968 * @ring_lro: LRO descriptor per rx ring. 969 * @slot: Bucket no. 970 * @tcp_seg_len: Length of tcp segment. 971 * @ts_off: time stamp offset in the packet. 972 */ 973__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void 974__hal_open_lro_session (u8 *buffer, iplro_t *ip, tcplro_t *tcp, lro_t **lro, 975 xge_hal_device_t *hldev, xge_hal_lro_desc_t *ring_lro, int slot, 976 u32 tcp_seg_len, int ts_off) 977{ 978 979 lro_t *lro_new = &ring_lro->lro_pool[slot]; 980 981 lro_new->in_use = 1; 982 lro_new->ll_hdr = buffer; 983 lro_new->ip_hdr = ip; 984 lro_new->tcp_hdr = tcp; 985 lro_new->tcp_next_seq_num = tcp_seg_len + xge_os_ntohl( 986 tcp->seq); 987 lro_new->tcp_seq_num = tcp->seq; 988 lro_new->tcp_ack_num = tcp->ack_seq; 989 lro_new->sg_num = 1; 990 lro_new->total_length = xge_os_ntohs(ip->tot_len); 991 lro_new->frags_len = 0; 992 lro_new->ts_off = ts_off; 993 994 hldev->stats.sw_dev_info_stats.tot_frms_lroised++; 995 hldev->stats.sw_dev_info_stats.tot_lro_sessions++; 996 997 *lro = ring_lro->lro_recent = lro_new; 998 return; 999} 1000/* 1001 * __hal_lro_get_free_slot: Get a free LRO bucket. 1002 * @ring_lro: LRO descriptor per ring. 1003 */ 1004__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL int 1005__hal_lro_get_free_slot (xge_hal_lro_desc_t *ring_lro) 1006{ 1007 int i; 1008 1009 for (i = 0; i < XGE_HAL_LRO_MAX_BUCKETS; i++) { 1010 lro_t *lro_temp = &ring_lro->lro_pool[i]; 1011 1012 if (!lro_temp->in_use) 1013 return i; 1014 } 1015 return -1; 1016} 1017 1018/* 1019 * __hal_get_lro_session: Gets matching LRO session or creates one. 1020 * @eth_hdr: Ethernet header. 1021 * @ip: ip header. 1022 * @tcp: tcp header. 1023 * @lro: lro pointer 1024 * @ext_info: Descriptor info. 1025 * @hldev: Hal context. 1026 * @ring_lro: LRO descriptor per rx ring 1027 */ 1028__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e 1029__hal_get_lro_session (u8 *eth_hdr, 1030 iplro_t *ip, 1031 tcplro_t *tcp, 1032 lro_t **lro, 1033 xge_hal_dtr_info_t *ext_info, 1034 xge_hal_device_t *hldev, 1035 xge_hal_lro_desc_t *ring_lro, 1036 lro_t **lro_end3 /* Valid only when ret=END_3 */) 1037{ 1038 lro_t *lro_match; 1039 int i, free_slot = -1; 1040 u32 tcp_seg_len; 1041 int ts_off = -1; 1042 1043 *lro = lro_match = NULL; 1044 /* 1045 * Compare the incoming frame with the lro session left from the 1046 * previous call. There is a good chance that this incoming frame 1047 * matches the lro session. 1048 */ 1049 if (ring_lro->lro_recent && ring_lro->lro_recent->in_use) { 1050 if (__hal_lro_check_for_session_match(ring_lro->lro_recent, 1051 tcp, ip) 1052 == XGE_HAL_OK) 1053 lro_match = ring_lro->lro_recent; 1054 } 1055 1056 if (!lro_match) { 1057 /* 1058 * Search in the pool of LROs for the session that matches 1059 * the incoming frame. 1060 */ 1061 for (i = 0; i < XGE_HAL_LRO_MAX_BUCKETS; i++) { 1062 lro_t *lro_temp = &ring_lro->lro_pool[i]; 1063 1064 if (!lro_temp->in_use) { 1065 if (free_slot == -1) 1066 free_slot = i; 1067 continue; 1068 } 1069 1070 if (__hal_lro_check_for_session_match(lro_temp, tcp, 1071 ip) == XGE_HAL_OK) { 1072 lro_match = lro_temp; 1073 break; 1074 } 1075 } 1076 } 1077 1078 1079 if (lro_match) { 1080 /* 1081 * Matching LRO Session found 1082 */ 1083 *lro = lro_match; 1084 1085 if (lro_match->tcp_next_seq_num != xge_os_ntohl(tcp->seq)) { 1086 xge_debug_ring(XGE_ERR, "**retransmit **" 1087 "found***"); 1088 hldev->stats.sw_dev_info_stats.lro_out_of_seq_pkt_cnt++; 1089 return XGE_HAL_INF_LRO_END_2; 1090 } 1091 1092 if (XGE_HAL_OK != __hal_ip_lro_capable(ip, ext_info)) 1093 { 1094 return XGE_HAL_INF_LRO_END_2; 1095 } 1096 1097 if (XGE_HAL_OK != __hal_tcp_lro_capable(ip, tcp, lro_match, 1098 &ts_off)) { 1099 /* 1100 * Close the current session and open a new 1101 * LRO session with this packet, 1102 * provided it has tcp payload 1103 */ 1104 tcp_seg_len = __hal_tcp_seg_len(ip, tcp); 1105 if (tcp_seg_len == 0) 1106 { 1107 return XGE_HAL_INF_LRO_END_2; 1108 } 1109 1110 /* Get a free bucket */ 1111 free_slot = __hal_lro_get_free_slot(ring_lro); 1112 if (free_slot == -1) 1113 { 1114 return XGE_HAL_INF_LRO_END_2; 1115 } 1116 1117 /* 1118 * Open a new LRO session 1119 */ 1120 __hal_open_lro_session (eth_hdr, ip, tcp, lro_end3, 1121 hldev, ring_lro, free_slot, tcp_seg_len, 1122 ts_off); 1123 1124 return XGE_HAL_INF_LRO_END_3; 1125 } 1126 1127 /* 1128 * The frame is good, in-sequence, can be LRO-ed; 1129 * take its (latest) ACK - unless it is a dupack. 1130 * Note: to be exact need to check window size as well.. 1131 */ 1132 if (lro_match->tcp_ack_num == tcp->ack_seq && 1133 lro_match->tcp_seq_num == tcp->seq) { 1134 hldev->stats.sw_dev_info_stats.lro_dup_pkt_cnt++; 1135 return XGE_HAL_INF_LRO_END_2; 1136 } 1137 1138 lro_match->tcp_seq_num = tcp->seq; 1139 lro_match->tcp_ack_num = tcp->ack_seq; 1140 lro_match->frags_len += __hal_tcp_seg_len(ip, tcp); 1141 1142 ring_lro->lro_recent = lro_match; 1143 1144 return XGE_HAL_INF_LRO_CONT; 1145 } 1146 1147 /* ********** New Session ***************/ 1148 if (free_slot == -1) 1149 return XGE_HAL_INF_LRO_UNCAPABLE; 1150 1151 if (XGE_HAL_FAIL == __hal_ip_lro_capable(ip, ext_info)) 1152 return XGE_HAL_INF_LRO_UNCAPABLE; 1153 1154 if (XGE_HAL_FAIL == __hal_tcp_lro_capable(ip, tcp, NULL, &ts_off)) 1155 return XGE_HAL_INF_LRO_UNCAPABLE; 1156 1157 xge_debug_ring(XGE_TRACE, "Creating lro session."); 1158 1159 /* 1160 * Open a LRO session, provided the packet contains payload. 1161 */ 1162 tcp_seg_len = __hal_tcp_seg_len(ip, tcp); 1163 if (tcp_seg_len == 0) 1164 return XGE_HAL_INF_LRO_UNCAPABLE; 1165 1166 __hal_open_lro_session (eth_hdr, ip, tcp, lro, hldev, ring_lro, free_slot, 1167 tcp_seg_len, ts_off); 1168 1169 return XGE_HAL_INF_LRO_BEGIN; 1170} 1171 1172/* 1173 * __hal_lro_under_optimal_thresh: Finds whether combined session is optimal. 1174 * @ip: ip header. 1175 * @tcp: tcp header. 1176 * @lro: lro pointer 1177 * @hldev: Hal context. 1178 */ 1179__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e 1180__hal_lro_under_optimal_thresh (iplro_t *ip, 1181 tcplro_t *tcp, 1182 lro_t *lro, 1183 xge_hal_device_t *hldev) 1184{ 1185 if (!lro) return XGE_HAL_FAIL; 1186 1187 if ((lro->total_length + __hal_tcp_seg_len(ip, tcp) ) > 1188 hldev->config.lro_frm_len) { 1189 xge_debug_ring(XGE_TRACE, "Max LRO frame len exceeded:" 1190 "max length %d ", hldev->config.lro_frm_len); 1191 hldev->stats.sw_dev_info_stats.lro_frm_len_exceed_cnt++; 1192 return XGE_HAL_FAIL; 1193 } 1194 1195 if (lro->sg_num == hldev->config.lro_sg_size) { 1196 xge_debug_ring(XGE_TRACE, "Max sg count exceeded:" 1197 "max sg %d ", hldev->config.lro_sg_size); 1198 hldev->stats.sw_dev_info_stats.lro_sg_exceed_cnt++; 1199 return XGE_HAL_FAIL; 1200 } 1201 1202 return XGE_HAL_OK; 1203} 1204 1205/* 1206 * __hal_collapse_ip_hdr: Collapses ip header. 1207 * @ip: ip header. 1208 * @tcp: tcp header. 1209 * @lro: lro pointer 1210 * @hldev: Hal context. 1211 */ 1212__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e 1213__hal_collapse_ip_hdr ( iplro_t *ip, 1214 tcplro_t *tcp, 1215 lro_t *lro, 1216 xge_hal_device_t *hldev) 1217{ 1218 1219 lro->total_length += __hal_tcp_seg_len(ip, tcp); 1220 1221 /* May be we have to handle time stamps or more options */ 1222 1223 return XGE_HAL_OK; 1224 1225} 1226 1227/* 1228 * __hal_collapse_tcp_hdr: Collapses tcp header. 1229 * @ip: ip header. 1230 * @tcp: tcp header. 1231 * @lro: lro pointer 1232 * @hldev: Hal context. 1233 */ 1234__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e 1235__hal_collapse_tcp_hdr ( iplro_t *ip, 1236 tcplro_t *tcp, 1237 lro_t *lro, 1238 xge_hal_device_t *hldev) 1239{ 1240 lro->tcp_next_seq_num += __hal_tcp_seg_len(ip, tcp); 1241 return XGE_HAL_OK; 1242 1243} 1244 1245/* 1246 * __hal_append_lro: Appends new frame to existing LRO session. 1247 * @ip: ip header. 1248 * @tcp: IN tcp header, OUT tcp payload. 1249 * @seg_len: tcp payload length. 1250 * @lro: lro pointer 1251 * @hldev: Hal context. 1252 */ 1253__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e 1254__hal_append_lro(iplro_t *ip, 1255 tcplro_t **tcp, 1256 u32 *seg_len, 1257 lro_t *lro, 1258 xge_hal_device_t *hldev) 1259{ 1260 (void) __hal_collapse_ip_hdr(ip, *tcp, lro, hldev); 1261 (void) __hal_collapse_tcp_hdr(ip, *tcp, lro, hldev); 1262 // Update mbuf chain will be done in ll driver. 1263 // xge_hal_accumulate_large_rx on success of appending new frame to 1264 // lro will return to ll driver tcpdata pointer, and tcp payload length. 1265 // along with return code lro frame appended. 1266 1267 lro->sg_num++; 1268 *seg_len = __hal_tcp_seg_len(ip, *tcp); 1269 *tcp = (tcplro_t *)((char *)*tcp + (((*tcp)->doff_res)>>2)); 1270 1271 return XGE_HAL_OK; 1272 1273} 1274 1275/** 1276 * __xge_hal_accumulate_large_rx: LRO a given frame 1277 * frames 1278 * @ring: rx ring number 1279 * @eth_hdr: ethernet header. 1280 * @ip_hdr: ip header (optional) 1281 * @tcp: tcp header. 1282 * @seglen: packet length. 1283 * @p_lro: lro pointer. 1284 * @ext_info: descriptor info, see xge_hal_dtr_info_t{}. 1285 * @hldev: HAL device. 1286 * @lro_end3: for lro_end3 output 1287 * 1288 * LRO the newly received frame, i.e. attach it (if possible) to the 1289 * already accumulated (i.e., already LRO-ed) received frames (if any), 1290 * to form one super-sized frame for the subsequent processing 1291 * by the stack. 1292 */ 1293__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e 1294xge_hal_lro_process_rx(int ring, u8 *eth_hdr, u8 *ip_hdr, tcplro_t **tcp, 1295 u32 *seglen, lro_t **p_lro, 1296 xge_hal_dtr_info_t *ext_info, xge_hal_device_t *hldev, 1297 lro_t **lro_end3) 1298{ 1299 iplro_t *ip = (iplro_t *)ip_hdr; 1300 xge_hal_status_e ret; 1301 lro_t *lro; 1302 1303 xge_debug_ring(XGE_TRACE, "Entered accumu lro. "); 1304 if (XGE_HAL_OK != __hal_lro_capable(eth_hdr, &ip, (tcplro_t **)tcp, 1305 ext_info)) 1306 return XGE_HAL_INF_LRO_UNCAPABLE; 1307 1308 /* 1309 * This function shall get matching LRO or else 1310 * create one and return it 1311 */ 1312 ret = __hal_get_lro_session(eth_hdr, ip, (tcplro_t *)*tcp, 1313 p_lro, ext_info, hldev, &hldev->lro_desc[ring], 1314 lro_end3); 1315 xge_debug_ring(XGE_TRACE, "ret from get_lro:%d ",ret); 1316 lro = *p_lro; 1317 if (XGE_HAL_INF_LRO_CONT == ret) { 1318 if (XGE_HAL_OK == __hal_lro_under_optimal_thresh(ip, 1319 (tcplro_t *)*tcp, lro, hldev)) { 1320 (void) __hal_append_lro(ip,(tcplro_t **) tcp, seglen, 1321 lro, hldev); 1322 hldev->stats.sw_dev_info_stats.tot_frms_lroised++; 1323 1324 if (lro->sg_num >= hldev->config.lro_sg_size) { 1325 hldev->stats.sw_dev_info_stats.lro_sg_exceed_cnt++; 1326 ret = XGE_HAL_INF_LRO_END_1; 1327 } 1328 1329 } else ret = XGE_HAL_INF_LRO_END_2; 1330 } 1331 1332 /* 1333 * Since its time to flush, 1334 * update ip header so that it can be sent up 1335 */ 1336 if ((ret == XGE_HAL_INF_LRO_END_1) || 1337 (ret == XGE_HAL_INF_LRO_END_2) || 1338 (ret == XGE_HAL_INF_LRO_END_3)) { 1339 lro->ip_hdr->tot_len = xge_os_htons((*p_lro)->total_length); 1340 lro->ip_hdr->check = xge_os_htons(0); 1341 lro->ip_hdr->check = XGE_LL_IP_FAST_CSUM(((u8 *)(lro->ip_hdr)), 1342 (lro->ip_hdr->version_ihl & 0x0F)); 1343 lro->tcp_hdr->ack_seq = lro->tcp_ack_num; 1344 } 1345 1346 return (ret); 1347} 1348 1349/** 1350 * xge_hal_accumulate_large_rx: LRO a given frame 1351 * frames 1352 * @buffer: Ethernet frame. 1353 * @tcp: tcp header. 1354 * @seglen: packet length. 1355 * @p_lro: lro pointer. 1356 * @ext_info: descriptor info, see xge_hal_dtr_info_t{}. 1357 * @hldev: HAL device. 1358 * @lro_end3: for lro_end3 output 1359 * 1360 * LRO the newly received frame, i.e. attach it (if possible) to the 1361 * already accumulated (i.e., already LRO-ed) received frames (if any), 1362 * to form one super-sized frame for the subsequent processing 1363 * by the stack. 1364 */ 1365__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e 1366xge_hal_accumulate_large_rx(u8 *buffer, tcplro_t **tcp, u32 *seglen, 1367lro_t **p_lro, xge_hal_dtr_info_t *ext_info, xge_hal_device_t *hldev, 1368lro_t **lro_end3) 1369{ 1370 int ring = 0; 1371 return xge_hal_lro_process_rx(ring, buffer, NULL, tcp, seglen, p_lro, 1372 ext_info, hldev, lro_end3); 1373} 1374 1375/** 1376 * xge_hal_lro_close_session: Close LRO session 1377 * @lro: LRO Session. 1378 * @hldev: HAL Context. 1379 */ 1380__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void 1381xge_hal_lro_close_session (lro_t *lro) 1382{ 1383 lro->in_use = 0; 1384} 1385 1386/** 1387 * xge_hal_lro_next_session: Returns next LRO session in the list or NULL 1388 * if none exists. 1389 * @hldev: HAL Context. 1390 * @ring: rx ring number. 1391 */ 1392__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL lro_t * 1393xge_hal_lro_next_session (xge_hal_device_t *hldev, int ring) 1394{ 1395xge_hal_lro_desc_t *ring_lro = &hldev->lro_desc[ring]; 1396 int i; 1397 int start_idx = ring_lro->lro_next_idx; 1398 1399 for(i = start_idx; i < XGE_HAL_LRO_MAX_BUCKETS; i++) { 1400 lro_t *lro = &ring_lro->lro_pool[i]; 1401 1402 if (!lro->in_use) 1403 continue; 1404 1405 lro->ip_hdr->tot_len = xge_os_htons(lro->total_length); 1406 lro->ip_hdr->check = xge_os_htons(0); 1407 lro->ip_hdr->check = XGE_LL_IP_FAST_CSUM(((u8 *)(lro->ip_hdr)), 1408 (lro->ip_hdr->version_ihl & 0x0F)); 1409 ring_lro->lro_next_idx = i + 1; 1410 return lro; 1411 } 1412 1413 ring_lro->lro_next_idx = 0; 1414 return NULL; 1415 1416} 1417 1418__HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL lro_t * 1419xge_hal_lro_get_next_session(xge_hal_device_t *hldev) 1420{ 1421 int ring = 0; /* assume default ring=0 */ 1422 return xge_hal_lro_next_session(hldev, ring); 1423} 1424#endif 1425