mlx5_en_rx.c revision 347796
154359Sroberto/*- 254359Sroberto * Copyright (c) 2015 Mellanox Technologies. All rights reserved. 354359Sroberto * 454359Sroberto * Redistribution and use in source and binary forms, with or without 554359Sroberto * modification, are permitted provided that the following conditions 654359Sroberto * are met: 754359Sroberto * 1. Redistributions of source code must retain the above copyright 854359Sroberto * notice, this list of conditions and the following disclaimer. 954359Sroberto * 2. Redistributions in binary form must reproduce the above copyright 1054359Sroberto * notice, this list of conditions and the following disclaimer in the 1154359Sroberto * documentation and/or other materials provided with the distribution. 1254359Sroberto * 1354359Sroberto * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 1454359Sroberto * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1554359Sroberto * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16290000Sglebius * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 1754359Sroberto * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1854359Sroberto * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 1954359Sroberto * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2054359Sroberto * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2154359Sroberto * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2254359Sroberto * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2354359Sroberto * SUCH DAMAGE. 2454359Sroberto * 2554359Sroberto * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c 347796 2019-05-16 17:09:06Z hselasky $ 2654359Sroberto */ 2754359Sroberto 2854359Sroberto#include "en.h" 2954359Sroberto#include <machine/in_cksum.h> 3054359Sroberto 3154359Srobertostatic inline int 3254359Srobertomlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, 3354359Sroberto struct mlx5e_rx_wqe *wqe, u16 ix) 3454359Sroberto{ 3554359Sroberto bus_dma_segment_t segs[rq->nsegs]; 3654359Sroberto struct mbuf *mb; 3754359Sroberto int nsegs; 3854359Sroberto int err; 3954359Sroberto#if (MLX5E_MAX_RX_SEGS != 1) 4054359Sroberto struct mbuf *mb_head; 4154359Sroberto int i; 4254359Sroberto#endif 4354359Sroberto if (rq->mbuf[ix].mbuf != NULL) 4454359Sroberto return (0); 4554359Sroberto 4654359Sroberto#if (MLX5E_MAX_RX_SEGS == 1) 4754359Sroberto mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rq->wqe_sz); 4854359Sroberto if (unlikely(!mb)) 4954359Sroberto return (-ENOMEM); 5054359Sroberto 5154359Sroberto mb->m_pkthdr.len = mb->m_len = rq->wqe_sz; 5254359Sroberto#else 5354359Sroberto mb_head = mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, 5454359Sroberto MLX5E_MAX_RX_BYTES); 5554359Sroberto if (unlikely(mb == NULL)) 5654359Sroberto return (-ENOMEM); 5754359Sroberto 5854359Sroberto mb->m_len = MLX5E_MAX_RX_BYTES; 5954359Sroberto mb->m_pkthdr.len = MLX5E_MAX_RX_BYTES; 6054359Sroberto 6154359Sroberto for (i = 1; i < rq->nsegs; i++) { 6254359Sroberto if (mb_head->m_pkthdr.len >= rq->wqe_sz) 6354359Sroberto break; 6454359Sroberto mb = mb->m_next = m_getjcl(M_NOWAIT, MT_DATA, 0, 6554359Sroberto MLX5E_MAX_RX_BYTES); 6654359Sroberto if (unlikely(mb == NULL)) { 6754359Sroberto m_freem(mb_head); 6854359Sroberto return (-ENOMEM); 6954359Sroberto } 7054359Sroberto mb->m_len = MLX5E_MAX_RX_BYTES; 7154359Sroberto mb_head->m_pkthdr.len += MLX5E_MAX_RX_BYTES; 7254359Sroberto } 7354359Sroberto /* rewind to first mbuf in chain */ 7454359Sroberto mb = mb_head; 7554359Sroberto#endif 7654359Sroberto /* get IP header aligned */ 7754359Sroberto m_adj(mb, MLX5E_NET_IP_ALIGN); 7854359Sroberto 7954359Sroberto err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map, 8054359Sroberto mb, segs, &nsegs, BUS_DMA_NOWAIT); 8154359Sroberto if (err != 0) 8254359Sroberto goto err_free_mbuf; 8354359Sroberto if (unlikely(nsegs == 0)) { 8454359Sroberto bus_dmamap_unload(rq->dma_tag, rq->mbuf[ix].dma_map); 8554359Sroberto err = -ENOMEM; 8654359Sroberto goto err_free_mbuf; 8754359Sroberto } 8854359Sroberto#if (MLX5E_MAX_RX_SEGS == 1) 8954359Sroberto wqe->data[0].addr = cpu_to_be64(segs[0].ds_addr); 9054359Sroberto#else 9154359Sroberto wqe->data[0].addr = cpu_to_be64(segs[0].ds_addr); 9254359Sroberto wqe->data[0].byte_count = cpu_to_be32(segs[0].ds_len | 9354359Sroberto MLX5_HW_START_PADDING); 9454359Sroberto for (i = 1; i != nsegs; i++) { 9554359Sroberto wqe->data[i].addr = cpu_to_be64(segs[i].ds_addr); 9654359Sroberto wqe->data[i].byte_count = cpu_to_be32(segs[i].ds_len); 9754359Sroberto } 9854359Sroberto for (; i < rq->nsegs; i++) { 9954359Sroberto wqe->data[i].addr = 0; 10054359Sroberto wqe->data[i].byte_count = 0; 10154359Sroberto } 10254359Sroberto#endif 10354359Sroberto 10454359Sroberto rq->mbuf[ix].mbuf = mb; 10554359Sroberto rq->mbuf[ix].data = mb->m_data; 10654359Sroberto 10754359Sroberto bus_dmamap_sync(rq->dma_tag, rq->mbuf[ix].dma_map, 10854359Sroberto BUS_DMASYNC_PREREAD); 109 return (0); 110 111err_free_mbuf: 112 m_freem(mb); 113 return (err); 114} 115 116static void 117mlx5e_post_rx_wqes(struct mlx5e_rq *rq) 118{ 119 if (unlikely(rq->enabled == 0)) 120 return; 121 122 while (!mlx5_wq_ll_is_full(&rq->wq)) { 123 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, rq->wq.head); 124 125 if (unlikely(mlx5e_alloc_rx_wqe(rq, wqe, rq->wq.head))) { 126 callout_reset_curcpu(&rq->watchdog, 1, (void *)&mlx5e_post_rx_wqes, rq); 127 break; 128 } 129 mlx5_wq_ll_push(&rq->wq, be16_to_cpu(wqe->next.next_wqe_index)); 130 } 131 132 /* ensure wqes are visible to device before updating doorbell record */ 133 atomic_thread_fence_rel(); 134 135 mlx5_wq_ll_update_db_record(&rq->wq); 136} 137 138static void 139mlx5e_lro_update_hdr(struct mbuf *mb, struct mlx5_cqe64 *cqe) 140{ 141 /* TODO: consider vlans, ip options, ... */ 142 struct ether_header *eh; 143 uint16_t eh_type; 144 uint16_t tot_len; 145 struct ip6_hdr *ip6 = NULL; 146 struct ip *ip4 = NULL; 147 struct tcphdr *th; 148 uint32_t *ts_ptr; 149 uint8_t l4_hdr_type; 150 int tcp_ack; 151 152 eh = mtod(mb, struct ether_header *); 153 eh_type = ntohs(eh->ether_type); 154 155 l4_hdr_type = get_cqe_l4_hdr_type(cqe); 156 tcp_ack = ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA == l4_hdr_type) || 157 (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA == l4_hdr_type)); 158 159 /* TODO: consider vlan */ 160 tot_len = be32_to_cpu(cqe->byte_cnt) - ETHER_HDR_LEN; 161 162 switch (eh_type) { 163 case ETHERTYPE_IP: 164 ip4 = (struct ip *)(eh + 1); 165 th = (struct tcphdr *)(ip4 + 1); 166 break; 167 case ETHERTYPE_IPV6: 168 ip6 = (struct ip6_hdr *)(eh + 1); 169 th = (struct tcphdr *)(ip6 + 1); 170 break; 171 default: 172 return; 173 } 174 175 ts_ptr = (uint32_t *)(th + 1); 176 177 if (get_cqe_lro_tcppsh(cqe)) 178 th->th_flags |= TH_PUSH; 179 180 if (tcp_ack) { 181 th->th_flags |= TH_ACK; 182 th->th_ack = cqe->lro_ack_seq_num; 183 th->th_win = cqe->lro_tcp_win; 184 185 /* 186 * FreeBSD handles only 32bit aligned timestamp right after 187 * the TCP hdr 188 * +--------+--------+--------+--------+ 189 * | NOP | NOP | TSopt | 10 | 190 * +--------+--------+--------+--------+ 191 * | TSval timestamp | 192 * +--------+--------+--------+--------+ 193 * | TSecr timestamp | 194 * +--------+--------+--------+--------+ 195 */ 196 if (get_cqe_lro_timestamp_valid(cqe) && 197 (__predict_true(*ts_ptr) == ntohl(TCPOPT_NOP << 24 | 198 TCPOPT_NOP << 16 | TCPOPT_TIMESTAMP << 8 | 199 TCPOLEN_TIMESTAMP))) { 200 /* 201 * cqe->timestamp is 64bit long. 202 * [0-31] - timestamp. 203 * [32-64] - timestamp echo replay. 204 */ 205 ts_ptr[1] = *(uint32_t *)&cqe->timestamp; 206 ts_ptr[2] = *((uint32_t *)&cqe->timestamp + 1); 207 } 208 } 209 if (ip4) { 210 ip4->ip_ttl = cqe->lro_min_ttl; 211 ip4->ip_len = cpu_to_be16(tot_len); 212 ip4->ip_sum = 0; 213 ip4->ip_sum = in_cksum(mb, ip4->ip_hl << 2); 214 } else { 215 ip6->ip6_hlim = cqe->lro_min_ttl; 216 ip6->ip6_plen = cpu_to_be16(tot_len - 217 sizeof(struct ip6_hdr)); 218 } 219 /* TODO: handle tcp checksum */ 220} 221 222static inline void 223mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe, 224 struct mlx5e_rq *rq, struct mbuf *mb, 225 u32 cqe_bcnt) 226{ 227 struct ifnet *ifp = rq->ifp; 228#if (MLX5E_MAX_RX_SEGS != 1) 229 struct mbuf *mb_head; 230#endif 231 int lro_num_seg; /* HW LRO session aggregated packets counter */ 232 233 lro_num_seg = be32_to_cpu(cqe->srqn) >> 24; 234 if (lro_num_seg > 1) { 235 mlx5e_lro_update_hdr(mb, cqe); 236 rq->stats.lro_packets++; 237 rq->stats.lro_bytes += cqe_bcnt; 238 } 239 240#if (MLX5E_MAX_RX_SEGS == 1) 241 mb->m_pkthdr.len = mb->m_len = cqe_bcnt; 242#else 243 mb->m_pkthdr.len = cqe_bcnt; 244 for (mb_head = mb; mb != NULL; mb = mb->m_next) { 245 if (mb->m_len > cqe_bcnt) 246 mb->m_len = cqe_bcnt; 247 cqe_bcnt -= mb->m_len; 248 if (likely(cqe_bcnt == 0)) { 249 if (likely(mb->m_next != NULL)) { 250 /* trim off empty mbufs */ 251 m_freem(mb->m_next); 252 mb->m_next = NULL; 253 } 254 break; 255 } 256 } 257 /* rewind to first mbuf in chain */ 258 mb = mb_head; 259#endif 260 /* check if a Toeplitz hash was computed */ 261 if (cqe->rss_hash_type != 0) { 262 mb->m_pkthdr.flowid = be32_to_cpu(cqe->rss_hash_result); 263#ifdef RSS 264 /* decode the RSS hash type */ 265 switch (cqe->rss_hash_type & 266 (CQE_RSS_DST_HTYPE_L4 | CQE_RSS_DST_HTYPE_IP)) { 267 /* IPv4 */ 268 case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV4): 269 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV4); 270 break; 271 case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV4): 272 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV4); 273 break; 274 case CQE_RSS_DST_HTYPE_IPV4: 275 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV4); 276 break; 277 /* IPv6 */ 278 case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV6): 279 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV6); 280 break; 281 case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV6): 282 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV6); 283 break; 284 case CQE_RSS_DST_HTYPE_IPV6: 285 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV6); 286 break; 287 default: /* Other */ 288 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH); 289 break; 290 } 291#else 292 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH); 293#endif 294 } else { 295 mb->m_pkthdr.flowid = rq->ix; 296 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE); 297 } 298 mb->m_pkthdr.rcvif = ifp; 299 300 if (likely(ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) && 301 ((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK)) == 302 (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK))) { 303 mb->m_pkthdr.csum_flags = 304 CSUM_IP_CHECKED | CSUM_IP_VALID | 305 CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 306 mb->m_pkthdr.csum_data = htons(0xffff); 307 } else { 308 rq->stats.csum_none++; 309 } 310 311 if (cqe_has_vlan(cqe)) { 312 mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->vlan_info); 313 mb->m_flags |= M_VLANTAG; 314 } 315} 316 317static inline void 318mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data) 319{ 320 memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, (cc & cq->wq.sz_m1)), 321 sizeof(struct mlx5_cqe64)); 322} 323 324static inline void 325mlx5e_write_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data) 326{ 327 memcpy(mlx5_cqwq_get_wqe(&cq->wq, cc & cq->wq.sz_m1), 328 data, sizeof(struct mlx5_cqe64)); 329} 330 331static inline void 332mlx5e_decompress_cqe(struct mlx5e_cq *cq, struct mlx5_cqe64 *title, 333 struct mlx5_mini_cqe8 *mini, 334 u16 wqe_counter, int i) 335{ 336 /* 337 * NOTE: The fields which are not set here are copied from the 338 * initial and common title. See memcpy() in 339 * mlx5e_write_cqe_slot(). 340 */ 341 title->byte_cnt = mini->byte_cnt; 342 title->wqe_counter = cpu_to_be16((wqe_counter + i) & cq->wq.sz_m1); 343 title->check_sum = mini->checksum; 344 title->op_own = (title->op_own & 0xf0) | 345 (((cq->wq.cc + i) >> cq->wq.log_sz) & 1); 346} 347 348#define MLX5E_MINI_ARRAY_SZ 8 349/* Make sure structs are not packet differently */ 350CTASSERT(sizeof(struct mlx5_cqe64) == 351 sizeof(struct mlx5_mini_cqe8) * MLX5E_MINI_ARRAY_SZ); 352static void 353mlx5e_decompress_cqes(struct mlx5e_cq *cq) 354{ 355 struct mlx5_mini_cqe8 mini_array[MLX5E_MINI_ARRAY_SZ]; 356 struct mlx5_cqe64 title; 357 u32 cqe_count; 358 u32 i = 0; 359 u16 title_wqe_counter; 360 361 mlx5e_read_cqe_slot(cq, cq->wq.cc, &title); 362 title_wqe_counter = be16_to_cpu(title.wqe_counter); 363 cqe_count = be32_to_cpu(title.byte_cnt); 364 365 /* Make sure we won't overflow */ 366 KASSERT(cqe_count <= cq->wq.sz_m1, 367 ("%s: cqe_count %u > cq->wq.sz_m1 %u", __func__, 368 cqe_count, cq->wq.sz_m1)); 369 370 mlx5e_read_cqe_slot(cq, cq->wq.cc + 1, mini_array); 371 while (true) { 372 mlx5e_decompress_cqe(cq, &title, 373 &mini_array[i % MLX5E_MINI_ARRAY_SZ], 374 title_wqe_counter, i); 375 mlx5e_write_cqe_slot(cq, cq->wq.cc + i, &title); 376 i++; 377 378 if (i == cqe_count) 379 break; 380 if (i % MLX5E_MINI_ARRAY_SZ == 0) 381 mlx5e_read_cqe_slot(cq, cq->wq.cc + i, mini_array); 382 } 383} 384 385static int 386mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget) 387{ 388 int i; 389 390 for (i = 0; i < budget; i++) { 391 struct mlx5e_rx_wqe *wqe; 392 struct mlx5_cqe64 *cqe; 393 struct mbuf *mb; 394 __be16 wqe_counter_be; 395 u16 wqe_counter; 396 u32 byte_cnt; 397 398 cqe = mlx5e_get_cqe(&rq->cq); 399 if (!cqe) 400 break; 401 402 if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) 403 mlx5e_decompress_cqes(&rq->cq); 404 405 mlx5_cqwq_pop(&rq->cq.wq); 406 407 wqe_counter_be = cqe->wqe_counter; 408 wqe_counter = be16_to_cpu(wqe_counter_be); 409 wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter); 410 byte_cnt = be32_to_cpu(cqe->byte_cnt); 411 412 bus_dmamap_sync(rq->dma_tag, 413 rq->mbuf[wqe_counter].dma_map, 414 BUS_DMASYNC_POSTREAD); 415 416 if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) { 417 rq->stats.wqe_err++; 418 goto wq_ll_pop; 419 } 420 if ((MHLEN - MLX5E_NET_IP_ALIGN) >= byte_cnt && 421 (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) { 422#if (MLX5E_MAX_RX_SEGS != 1) 423 /* set maximum mbuf length */ 424 mb->m_len = MHLEN - MLX5E_NET_IP_ALIGN; 425#endif 426 /* get IP header aligned */ 427 mb->m_data += MLX5E_NET_IP_ALIGN; 428 429 bcopy(rq->mbuf[wqe_counter].data, mtod(mb, caddr_t), 430 byte_cnt); 431 } else { 432 mb = rq->mbuf[wqe_counter].mbuf; 433 rq->mbuf[wqe_counter].mbuf = NULL; /* safety clear */ 434 435 bus_dmamap_unload(rq->dma_tag, 436 rq->mbuf[wqe_counter].dma_map); 437 } 438 439 mlx5e_build_rx_mbuf(cqe, rq, mb, byte_cnt); 440 rq->stats.bytes += byte_cnt; 441 rq->stats.packets++; 442 443#if !defined(HAVE_TCP_LRO_RX) 444 tcp_lro_queue_mbuf(&rq->lro, mb); 445#else 446 if (mb->m_pkthdr.csum_flags == 0 || 447 (rq->ifp->if_capenable & IFCAP_LRO) == 0 || 448 rq->lro.lro_cnt == 0 || 449 tcp_lro_rx(&rq->lro, mb, 0) != 0) { 450 rq->ifp->if_input(rq->ifp, mb); 451 } 452#endif 453wq_ll_pop: 454 mlx5_wq_ll_pop(&rq->wq, wqe_counter_be, 455 &wqe->next.next_wqe_index); 456 } 457 458 mlx5_cqwq_update_db_record(&rq->cq.wq); 459 460 /* ensure cq space is freed before enabling more cqes */ 461 atomic_thread_fence_rel(); 462 return (i); 463} 464 465void 466mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq) 467{ 468 struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq); 469 int i = 0; 470 471#ifdef HAVE_PER_CQ_EVENT_PACKET 472#if (MHLEN < 15) 473#error "MHLEN is too small" 474#endif 475 struct mbuf *mb = m_gethdr(M_NOWAIT, MT_DATA); 476 477 if (mb != NULL) { 478 /* this code is used for debugging purpose only */ 479 mb->m_pkthdr.len = mb->m_len = 15; 480 memset(mb->m_data, 255, 14); 481 mb->m_data[14] = rq->ix; 482 mb->m_pkthdr.rcvif = rq->ifp; 483 rq->ifp->if_input(rq->ifp, mb); 484 } 485#endif 486 487 mtx_lock(&rq->mtx); 488 489 /* 490 * Polling the entire CQ without posting new WQEs results in 491 * lack of receive WQEs during heavy traffic scenarios. 492 */ 493 while (1) { 494 if (mlx5e_poll_rx_cq(rq, MLX5E_RX_BUDGET_MAX) != 495 MLX5E_RX_BUDGET_MAX) 496 break; 497 i += MLX5E_RX_BUDGET_MAX; 498 if (i >= MLX5E_BUDGET_MAX) 499 break; 500 mlx5e_post_rx_wqes(rq); 501 } 502 mlx5e_post_rx_wqes(rq); 503 /* check for dynamic interrupt moderation callback */ 504 if (rq->dim.mode != NET_DIM_CQ_PERIOD_MODE_DISABLED) 505 net_dim(&rq->dim, rq->stats.packets, rq->stats.bytes); 506 mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK(&rq->channel->priv->doorbell_lock)); 507 tcp_lro_flush_all(&rq->lro); 508 mtx_unlock(&rq->mtx); 509} 510