11541Srgrimes/* SPDX-License-Identifier: (GPL-2.0 OR MIT) 21541Srgrimes * Google virtual Ethernet (gve) driver 31541Srgrimes * 41541Srgrimes * Copyright (C) 2015-2021 Google, Inc. 51541Srgrimes */ 61541Srgrimes 71541Srgrimes#ifndef _GVE_H_ 81541Srgrimes#define _GVE_H_ 91541Srgrimes 101541Srgrimes#include <linux/dma-mapping.h> 111541Srgrimes#include <linux/dmapool.h> 121541Srgrimes#include <linux/ethtool_netlink.h> 131541Srgrimes#include <linux/netdevice.h> 141541Srgrimes#include <linux/pci.h> 151541Srgrimes#include <linux/u64_stats_sync.h> 161541Srgrimes#include <net/xdp.h> 171541Srgrimes 181541Srgrimes#include "gve_desc.h" 191541Srgrimes#include "gve_desc_dqo.h" 201541Srgrimes 211541Srgrimes#ifndef PCI_VENDOR_ID_GOOGLE 221541Srgrimes#define PCI_VENDOR_ID_GOOGLE 0x1ae0 231541Srgrimes#endif 241541Srgrimes 251541Srgrimes#define PCI_DEV_ID_GVNIC 0x0042 261541Srgrimes 271541Srgrimes#define GVE_REGISTER_BAR 0 281541Srgrimes#define GVE_DOORBELL_BAR 2 291541Srgrimes 301541Srgrimes/* Driver can alloc up to 2 segments for the header and 2 for the payload. */ 311541Srgrimes#define GVE_TX_MAX_IOVEC 4 321541Srgrimes/* 1 for management, 1 for rx, 1 for tx */ 331541Srgrimes#define GVE_MIN_MSIX 3 342197Spaul 351541Srgrimes/* Numbers of gve tx/rx stats in stats report. */ 361541Srgrimes#define GVE_TX_STATS_REPORT_NUM 6 372197Spaul#define GVE_RX_STATS_REPORT_NUM 2 382197Spaul 392165Spaul/* Interval to schedule a stats report update, 20000ms. */ 401541Srgrimes#define GVE_STATS_REPORT_TIMER_PERIOD 20000 411541Srgrimes 421541Srgrimes/* Numbers of NIC tx/rx stats in stats report. */ 431541Srgrimes#define NIC_TX_STATS_REPORT_NUM 0 441541Srgrimes#define NIC_RX_STATS_REPORT_NUM 4 451541Srgrimes 461541Srgrimes#define GVE_ADMINQ_BUFFER_SIZE 4096 471541Srgrimes 481541Srgrimes#define GVE_DATA_SLOT_ADDR_PAGE_MASK (~(PAGE_SIZE - 1)) 491541Srgrimes 501541Srgrimes/* PTYPEs are always 10 bits. */ 511541Srgrimes#define GVE_NUM_PTYPES 1024 521541Srgrimes 531541Srgrimes#define GVE_DEFAULT_RX_BUFFER_SIZE 2048 541541Srgrimes 551541Srgrimes#define GVE_MAX_RX_BUFFER_SIZE 4096 561541Srgrimes 571541Srgrimes#define GVE_DEFAULT_RX_BUFFER_OFFSET 2048 581541Srgrimes 591541Srgrimes#define GVE_XDP_ACTIONS 5 601541Srgrimes 611541Srgrimes#define GVE_GQ_TX_MIN_PKT_DESC_BYTES 182 621541Srgrimes 631541Srgrimes#define GVE_DEFAULT_HEADER_BUFFER_SIZE 128 641541Srgrimes 651541Srgrimes#define DQO_QPL_DEFAULT_TX_PAGES 512 661541Srgrimes#define DQO_QPL_DEFAULT_RX_PAGES 2048 671541Srgrimes 681541Srgrimes/* Maximum TSO size supported on DQO */ 691541Srgrimes#define GVE_DQO_TX_MAX 0x3FFFF 701541Srgrimes 711541Srgrimes#define GVE_TX_BUF_SHIFT_DQO 11 721541Srgrimes 731541Srgrimes/* 2K buffers for DQO-QPL */ 741541Srgrimes#define GVE_TX_BUF_SIZE_DQO BIT(GVE_TX_BUF_SHIFT_DQO) 751541Srgrimes#define GVE_TX_BUFS_PER_PAGE_DQO (PAGE_SIZE >> GVE_TX_BUF_SHIFT_DQO) 761541Srgrimes#define GVE_MAX_TX_BUFS_PER_PKT (DIV_ROUND_UP(GVE_DQO_TX_MAX, GVE_TX_BUF_SIZE_DQO)) 771541Srgrimes 781541Srgrimes/* If number of free/recyclable buffers are less than this threshold; driver 791541Srgrimes * allocs and uses a non-qpl page on the receive path of DQO QPL to free 801541Srgrimes * up buffers. 811541Srgrimes * Value is set big enough to post at least 3 64K LRO packet via 2K buffer to NIC. 821541Srgrimes */ 831541Srgrimes#define GVE_DQO_QPL_ONDEMAND_ALLOC_THRESHOLD 96 841541Srgrimes 851541Srgrimes/* Each slot in the desc ring has a 1:1 mapping to a slot in the data ring */ 861541Srgrimesstruct gve_rx_desc_queue { 871541Srgrimes struct gve_rx_desc *desc_ring; /* the descriptor ring */ 881541Srgrimes dma_addr_t bus; /* the bus for the desc_ring */ 891541Srgrimes u8 seqno; /* the next expected seqno for this desc*/ 901541Srgrimes}; 911541Srgrimes 921541Srgrimes/* The page info for a single slot in the RX data queue */ 931541Srgrimesstruct gve_rx_slot_page_info { 941541Srgrimes struct page *page; 951541Srgrimes void *page_address; 961541Srgrimes u32 page_offset; /* offset to write to in page */ 971541Srgrimes int pagecnt_bias; /* expected pagecnt if only the driver has a ref */ 981541Srgrimes u16 pad; /* adjustment for rx padding */ 991541Srgrimes u8 can_flip; /* tracks if the networking stack is using the page */ 1001541Srgrimes}; 1011541Srgrimes 1021541Srgrimes/* A list of pages registered with the device during setup and used by a queue 1031541Srgrimes * as buffers 1041541Srgrimes */ 1051541Srgrimesstruct gve_queue_page_list { 1061541Srgrimes u32 id; /* unique id */ 1071541Srgrimes u32 num_entries; 1081541Srgrimes struct page **pages; /* list of num_entries pages */ 1091541Srgrimes dma_addr_t *page_buses; /* the dma addrs of the pages */ 1101541Srgrimes}; 1111541Srgrimes 1121541Srgrimes/* Each slot in the data ring has a 1:1 mapping to a slot in the desc ring */ 1131541Srgrimesstruct gve_rx_data_queue { 1141541Srgrimes union gve_rx_data_slot *data_ring; /* read by NIC */ 1151541Srgrimes dma_addr_t data_bus; /* dma mapping of the slots */ 1161541Srgrimes struct gve_rx_slot_page_info *page_info; /* page info of the buffers */ 1171541Srgrimes struct gve_queue_page_list *qpl; /* qpl assigned to this queue */ 1181541Srgrimes u8 raw_addressing; /* use raw_addressing? */ 1191541Srgrimes}; 1201541Srgrimes 1211541Srgrimesstruct gve_priv; 1221541Srgrimes 1231541Srgrimes/* RX buffer queue for posting buffers to HW. 1241541Srgrimes * Each RX (completion) queue has a corresponding buffer queue. 1251541Srgrimes */ 1261541Srgrimesstruct gve_rx_buf_queue_dqo { 1271541Srgrimes struct gve_rx_desc_dqo *desc_ring; 1281541Srgrimes dma_addr_t bus; 1291541Srgrimes u32 head; /* Pointer to start cleaning buffers at. */ 1301541Srgrimes u32 tail; /* Last posted buffer index + 1 */ 1311541Srgrimes u32 mask; /* Mask for indices to the size of the ring */ 1321541Srgrimes}; 1331541Srgrimes 1341541Srgrimes/* RX completion queue to receive packets from HW. */ 1351541Srgrimesstruct gve_rx_compl_queue_dqo { 1361541Srgrimes struct gve_rx_compl_desc_dqo *desc_ring; 1371541Srgrimes dma_addr_t bus; 1381541Srgrimes 1391541Srgrimes /* Number of slots which did not have a buffer posted yet. We should not 1401541Srgrimes * post more buffers than the queue size to avoid HW overrunning the 1411541Srgrimes * queue. 1421541Srgrimes */ 1431541Srgrimes int num_free_slots; 1441541Srgrimes 1451541Srgrimes /* HW uses a "generation bit" to notify SW of new descriptors. When a 1461541Srgrimes * descriptor's generation bit is different from the current generation, 1471541Srgrimes * that descriptor is ready to be consumed by SW. 1481541Srgrimes */ 1491541Srgrimes u8 cur_gen_bit; 1501541Srgrimes 1511541Srgrimes /* Pointer into desc_ring where the next completion descriptor will be 1521541Srgrimes * received. 1531541Srgrimes */ 1541541Srgrimes u32 head; 1551541Srgrimes u32 mask; /* Mask for indices to the size of the ring */ 1561541Srgrimes}; 1571541Srgrimes 1581541Srgrimesstruct gve_header_buf { 1591541Srgrimes u8 *data; 1601541Srgrimes dma_addr_t addr; 1611541Srgrimes}; 1621541Srgrimes 1631541Srgrimes/* Stores state for tracking buffers posted to HW */ 1641541Srgrimesstruct gve_rx_buf_state_dqo { 1651541Srgrimes /* The page posted to HW. */ 1661541Srgrimes struct gve_rx_slot_page_info page_info; 1671541Srgrimes 1681541Srgrimes /* The DMA address corresponding to `page_info`. */ 1691541Srgrimes dma_addr_t addr; 1701541Srgrimes 1711541Srgrimes /* Last offset into the page when it only had a single reference, at 1721541Srgrimes * which point every other offset is free to be reused. 1731541Srgrimes */ 1741892Sdg u32 last_single_ref_offset; 1751892Sdg 1761541Srgrimes /* Linked list index to next element in the list, or -1 if none */ 1771892Sdg s16 next; 1781892Sdg}; 1791892Sdg 1801892Sdg/* `head` and `tail` are indices into an array, or -1 if empty. */ 1811892Sdgstruct gve_index_list { 1821892Sdg s16 head; 1831892Sdg s16 tail; 1841892Sdg}; 1851892Sdg 1861892Sdg/* A single received packet split across multiple buffers may be 1871541Srgrimes * reconstructed using the information in this structure. 1881541Srgrimes */ 1891541Srgrimesstruct gve_rx_ctx { 1901541Srgrimes /* head and tail of skb chain for the current packet or NULL if none */ 1911541Srgrimes struct sk_buff *skb_head; 1921541Srgrimes struct sk_buff *skb_tail; 1931541Srgrimes u32 total_size; 1941541Srgrimes u8 frag_cnt; 1951541Srgrimes bool drop_pkt; 1961541Srgrimes}; 1971541Srgrimes 1981541Srgrimesstruct gve_rx_cnts { 1991892Sdg u32 ok_pkt_bytes; 2001892Sdg u16 ok_pkt_cnt; 2011892Sdg u16 total_pkt_cnt; 2021892Sdg u16 cont_pkt_cnt; 2031892Sdg u16 desc_err_pkt_cnt; 2041892Sdg}; 2051892Sdg 2061892Sdg/* Contains datapath state used to represent an RX queue. */ 2071892Sdgstruct gve_rx_ring { 2081892Sdg struct gve_priv *gve; 2091541Srgrimes union { 2101541Srgrimes /* GQI fields */ 2111541Srgrimes struct { 2121541Srgrimes struct gve_rx_desc_queue desc; 2131541Srgrimes struct gve_rx_data_queue data; 2141541Srgrimes 2151541Srgrimes /* threshold for posting new buffs and descs */ 2161541Srgrimes u32 db_threshold; 2171541Srgrimes u16 packet_buffer_size; 2181541Srgrimes 2191541Srgrimes u32 qpl_copy_pool_mask; 2201541Srgrimes u32 qpl_copy_pool_head; 2211541Srgrimes struct gve_rx_slot_page_info *qpl_copy_pool; 2221541Srgrimes }; 2231541Srgrimes 2241541Srgrimes /* DQO fields. */ 2251541Srgrimes struct { 2261541Srgrimes struct gve_rx_buf_queue_dqo bufq; 2271541Srgrimes struct gve_rx_compl_queue_dqo complq; 2281541Srgrimes 2291541Srgrimes struct gve_rx_buf_state_dqo *buf_states; 2301541Srgrimes u16 num_buf_states; 2311541Srgrimes 2321541Srgrimes /* Linked list of gve_rx_buf_state_dqo. Index into 2331541Srgrimes * buf_states, or -1 if empty. 2341541Srgrimes */ 2351541Srgrimes s16 free_buf_states; 2361541Srgrimes 2371541Srgrimes /* Linked list of gve_rx_buf_state_dqo. Indexes into 2381541Srgrimes * buf_states, or -1 if empty. 2391541Srgrimes * 2401541Srgrimes * This list contains buf_states which are pointing to 2411541Srgrimes * valid buffers. 2421541Srgrimes * 2431541Srgrimes * We use a FIFO here in order to increase the 2441541Srgrimes * probability that buffers can be reused by increasing 2451541Srgrimes * the time between usages. 2461541Srgrimes */ 2471541Srgrimes struct gve_index_list recycled_buf_states; 2481541Srgrimes 2491541Srgrimes /* Linked list of gve_rx_buf_state_dqo. Indexes into 2501541Srgrimes * buf_states, or -1 if empty. 2511541Srgrimes * 2521541Srgrimes * This list contains buf_states which have buffers 2531541Srgrimes * which cannot be reused yet. 2541541Srgrimes */ 2551541Srgrimes struct gve_index_list used_buf_states; 2561541Srgrimes 2571541Srgrimes /* qpl assigned to this queue */ 2581541Srgrimes struct gve_queue_page_list *qpl; 2591541Srgrimes 2601541Srgrimes /* index into queue page list */ 2611541Srgrimes u32 next_qpl_page_idx; 2621541Srgrimes 2631541Srgrimes /* track number of used buffers */ 2641541Srgrimes u16 used_buf_states_cnt; 2651541Srgrimes 2661541Srgrimes /* Address info of the buffers for header-split */ 2671541Srgrimes struct gve_header_buf hdr_bufs; 2681541Srgrimes } dqo; 2691541Srgrimes }; 2701541Srgrimes 2711541Srgrimes u64 rbytes; /* free-running bytes received */ 2721541Srgrimes u64 rx_hsplit_bytes; /* free-running header bytes received */ 2731541Srgrimes u64 rpackets; /* free-running packets received */ 2741541Srgrimes u32 cnt; /* free-running total number of completed packets */ 2751541Srgrimes u32 fill_cnt; /* free-running total number of descs and buffs posted */ 2761541Srgrimes u32 mask; /* masks the cnt and fill_cnt to the size of the ring */ 2771541Srgrimes u64 rx_hsplit_pkt; /* free-running packets with headers split */ 2781541Srgrimes u64 rx_copybreak_pkt; /* free-running count of copybreak packets */ 2791541Srgrimes u64 rx_copied_pkt; /* free-running total number of copied packets */ 2801541Srgrimes u64 rx_skb_alloc_fail; /* free-running count of skb alloc fails */ 2811541Srgrimes u64 rx_buf_alloc_fail; /* free-running count of buffer alloc fails */ 2821541Srgrimes u64 rx_desc_err_dropped_pkt; /* free-running count of packets dropped by descriptor error */ 2831541Srgrimes /* free-running count of unsplit packets due to header buffer overflow or hdr_len is 0 */ 2841541Srgrimes u64 rx_hsplit_unsplit_pkt; 2851541Srgrimes u64 rx_cont_packet_cnt; /* free-running multi-fragment packets received */ 2861541Srgrimes u64 rx_frag_flip_cnt; /* free-running count of rx segments where page_flip was used */ 2871541Srgrimes u64 rx_frag_copy_cnt; /* free-running count of rx segments copied */ 2881541Srgrimes u64 rx_frag_alloc_cnt; /* free-running count of rx page allocations */ 2891541Srgrimes u64 xdp_tx_errors; 2901541Srgrimes u64 xdp_redirect_errors; 2911892Sdg u64 xdp_alloc_fails; 2921892Sdg u64 xdp_actions[GVE_XDP_ACTIONS]; 2931892Sdg u32 q_num; /* queue index */ 2941892Sdg u32 ntfy_id; /* notification block index */ 2951892Sdg struct gve_queue_resources *q_resources; /* head and tail pointer idx */ 2961892Sdg dma_addr_t q_resources_bus; /* dma address for the queue resources */ 2971892Sdg struct u64_stats_sync statss; /* sync stats for 32bit archs */ 2981892Sdg 2991892Sdg struct gve_rx_ctx ctx; /* Info for packet currently being processed in this ring. */ 3001541Srgrimes 3011541Srgrimes /* XDP stuff */ 3021541Srgrimes struct xdp_rxq_info xdp_rxq; 3031541Srgrimes struct xdp_rxq_info xsk_rxq; 3041541Srgrimes struct xsk_buff_pool *xsk_pool; 3051541Srgrimes struct page_frag_cache page_cache; /* Page cache to allocate XDP frames */ 3061541Srgrimes}; 3071541Srgrimes 3081541Srgrimes/* A TX desc ring entry */ 3091541Srgrimesunion gve_tx_desc { 3101541Srgrimes struct gve_tx_pkt_desc pkt; /* first desc for a packet */ 3111541Srgrimes struct gve_tx_mtd_desc mtd; /* optional metadata descriptor */ 3121541Srgrimes struct gve_tx_seg_desc seg; /* subsequent descs for a packet */ 3131541Srgrimes}; 3141541Srgrimes 3151541Srgrimes/* Tracks the memory in the fifo occupied by a segment of a packet */ 3161541Srgrimesstruct gve_tx_iovec { 3171541Srgrimes u32 iov_offset; /* offset into this segment */ 3181541Srgrimes u32 iov_len; /* length */ 3191541Srgrimes u32 iov_padding; /* padding associated with this segment */ 3201541Srgrimes}; 3211541Srgrimes 3221541Srgrimes/* Tracks the memory in the fifo occupied by the skb. Mapped 1:1 to a desc 3231541Srgrimes * ring entry but only used for a pkt_desc not a seg_desc 3241541Srgrimes */ 3251541Srgrimesstruct gve_tx_buffer_state { 3261541Srgrimes union { 3271541Srgrimes struct sk_buff *skb; /* skb for this pkt */ 3281541Srgrimes struct xdp_frame *xdp_frame; /* xdp_frame */ 3291541Srgrimes }; 3301541Srgrimes struct { 3311541Srgrimes u16 size; /* size of xmitted xdp pkt */ 3321541Srgrimes u8 is_xsk; /* xsk buff */ 3331541Srgrimes } xdp; 3341541Srgrimes union { 3351541Srgrimes struct gve_tx_iovec iov[GVE_TX_MAX_IOVEC]; /* segments of this pkt */ 3361541Srgrimes struct { 3371541Srgrimes DEFINE_DMA_UNMAP_ADDR(dma); 3381541Srgrimes DEFINE_DMA_UNMAP_LEN(len); 3391541Srgrimes }; 3401541Srgrimes }; 3411541Srgrimes}; 3421541Srgrimes 3431541Srgrimes/* A TX buffer - each queue has one */ 3441541Srgrimesstruct gve_tx_fifo { 3451541Srgrimes void *base; /* address of base of FIFO */ 3461541Srgrimes u32 size; /* total size */ 3471541Srgrimes atomic_t available; /* how much space is still available */ 3481541Srgrimes u32 head; /* offset to write at */ 3491541Srgrimes struct gve_queue_page_list *qpl; /* QPL mapped into this FIFO */ 3501541Srgrimes}; 3511541Srgrimes 3521541Srgrimes/* TX descriptor for DQO format */ 3531541Srgrimesunion gve_tx_desc_dqo { 3541541Srgrimes struct gve_tx_pkt_desc_dqo pkt; 3551541Srgrimes struct gve_tx_tso_context_desc_dqo tso_ctx; 3561541Srgrimes struct gve_tx_general_context_desc_dqo general_ctx; 3571541Srgrimes}; 3581541Srgrimes 3591541Srgrimesenum gve_packet_state { 3601541Srgrimes /* Packet is in free list, available to be allocated. 3611541Srgrimes * This should always be zero since state is not explicitly initialized. 3621541Srgrimes */ 3631541Srgrimes GVE_PACKET_STATE_UNALLOCATED, 3641541Srgrimes /* Packet is expecting a regular data completion or miss completion */ 3651541Srgrimes GVE_PACKET_STATE_PENDING_DATA_COMPL, 3661541Srgrimes /* Packet has received a miss completion and is expecting a 3671541Srgrimes * re-injection completion. 3681541Srgrimes */ 3691541Srgrimes GVE_PACKET_STATE_PENDING_REINJECT_COMPL, 3701541Srgrimes /* No valid completion received within the specified timeout. */ 3711541Srgrimes GVE_PACKET_STATE_TIMED_OUT_COMPL, 3721541Srgrimes}; 3731541Srgrimes 3741541Srgrimesstruct gve_tx_pending_packet_dqo { 3751541Srgrimes struct sk_buff *skb; /* skb for this packet */ 3761541Srgrimes 3771541Srgrimes /* 0th element corresponds to the linear portion of `skb`, should be 3781541Srgrimes * unmapped with `dma_unmap_single`. 3791541Srgrimes * 3801541Srgrimes * All others correspond to `skb`'s frags and should be unmapped with 3811541Srgrimes * `dma_unmap_page`. 3821541Srgrimes */ 3831541Srgrimes union { 3841541Srgrimes struct { 3851541Srgrimes DEFINE_DMA_UNMAP_ADDR(dma[MAX_SKB_FRAGS + 1]); 3861541Srgrimes DEFINE_DMA_UNMAP_LEN(len[MAX_SKB_FRAGS + 1]); 3871541Srgrimes }; 3881541Srgrimes s16 tx_qpl_buf_ids[GVE_MAX_TX_BUFS_PER_PKT]; 3891541Srgrimes }; 3901541Srgrimes 3911541Srgrimes u16 num_bufs; 3921541Srgrimes 3931541Srgrimes /* Linked list index to next element in the list, or -1 if none */ 3941541Srgrimes s16 next; 3951541Srgrimes 3961541Srgrimes /* Linked list index to prev element in the list, or -1 if none. 3971541Srgrimes * Used for tracking either outstanding miss completions or prematurely 3981541Srgrimes * freed packets. 3991541Srgrimes */ 4001541Srgrimes s16 prev; 4011541Srgrimes 4021541Srgrimes /* Identifies the current state of the packet as defined in 4031541Srgrimes * `enum gve_packet_state`. 4041541Srgrimes */ 4051541Srgrimes u8 state; 4061541Srgrimes 4071541Srgrimes /* If packet is an outstanding miss completion, then the packet is 4081541Srgrimes * freed if the corresponding re-injection completion is not received 4091541Srgrimes * before kernel jiffies exceeds timeout_jiffies. 4101541Srgrimes */ 4111541Srgrimes unsigned long timeout_jiffies; 4121541Srgrimes}; 4131541Srgrimes 4141541Srgrimes/* Contains datapath state used to represent a TX queue. */ 4151541Srgrimesstruct gve_tx_ring { 4161541Srgrimes /* Cacheline 0 -- Accessed & dirtied during transmit */ 4171541Srgrimes union { 4181541Srgrimes /* GQI fields */ 4191541Srgrimes struct { 4201541Srgrimes struct gve_tx_fifo tx_fifo; 4211541Srgrimes u32 req; /* driver tracked head pointer */ 4221541Srgrimes u32 done; /* driver tracked tail pointer */ 4231541Srgrimes }; 4241541Srgrimes 4251541Srgrimes /* DQO fields. */ 4261541Srgrimes struct { 4271541Srgrimes /* Linked list of gve_tx_pending_packet_dqo. Index into 4281541Srgrimes * pending_packets, or -1 if empty. 4291541Srgrimes * 4301541Srgrimes * This is a consumer list owned by the TX path. When it 4311541Srgrimes * runs out, the producer list is stolen from the 4321541Srgrimes * completion handling path 4331541Srgrimes * (dqo_compl.free_pending_packets). 4341541Srgrimes */ 4352165Spaul s16 free_pending_packets; 4362165Spaul 437 /* Cached value of `dqo_compl.hw_tx_head` */ 438 u32 head; 439 u32 tail; /* Last posted buffer index + 1 */ 440 441 /* Index of the last descriptor with "report event" bit 442 * set. 443 */ 444 u32 last_re_idx; 445 446 /* free running number of packet buf descriptors posted */ 447 u16 posted_packet_desc_cnt; 448 /* free running number of packet buf descriptors completed */ 449 u16 completed_packet_desc_cnt; 450 451 /* QPL fields */ 452 struct { 453 /* Linked list of gve_tx_buf_dqo. Index into 454 * tx_qpl_buf_next, or -1 if empty. 455 * 456 * This is a consumer list owned by the TX path. When it 457 * runs out, the producer list is stolen from the 458 * completion handling path 459 * (dqo_compl.free_tx_qpl_buf_head). 460 */ 461 s16 free_tx_qpl_buf_head; 462 463 /* Free running count of the number of QPL tx buffers 464 * allocated 465 */ 466 u32 alloc_tx_qpl_buf_cnt; 467 468 /* Cached value of `dqo_compl.free_tx_qpl_buf_cnt` */ 469 u32 free_tx_qpl_buf_cnt; 470 }; 471 } dqo_tx; 472 }; 473 474 /* Cacheline 1 -- Accessed & dirtied during gve_clean_tx_done */ 475 union { 476 /* GQI fields */ 477 struct { 478 /* Spinlock for when cleanup in progress */ 479 spinlock_t clean_lock; 480 /* Spinlock for XDP tx traffic */ 481 spinlock_t xdp_lock; 482 }; 483 484 /* DQO fields. */ 485 struct { 486 u32 head; /* Last read on compl_desc */ 487 488 /* Tracks the current gen bit of compl_q */ 489 u8 cur_gen_bit; 490 491 /* Linked list of gve_tx_pending_packet_dqo. Index into 492 * pending_packets, or -1 if empty. 493 * 494 * This is the producer list, owned by the completion 495 * handling path. When the consumer list 496 * (dqo_tx.free_pending_packets) is runs out, this list 497 * will be stolen. 498 */ 499 atomic_t free_pending_packets; 500 501 /* Last TX ring index fetched by HW */ 502 atomic_t hw_tx_head; 503 504 /* List to track pending packets which received a miss 505 * completion but not a corresponding reinjection. 506 */ 507 struct gve_index_list miss_completions; 508 509 /* List to track pending packets that were completed 510 * before receiving a valid completion because they 511 * reached a specified timeout. 512 */ 513 struct gve_index_list timed_out_completions; 514 515 /* QPL fields */ 516 struct { 517 /* Linked list of gve_tx_buf_dqo. Index into 518 * tx_qpl_buf_next, or -1 if empty. 519 * 520 * This is the producer list, owned by the completion 521 * handling path. When the consumer list 522 * (dqo_tx.free_tx_qpl_buf_head) is runs out, this list 523 * will be stolen. 524 */ 525 atomic_t free_tx_qpl_buf_head; 526 527 /* Free running count of the number of tx buffers 528 * freed 529 */ 530 atomic_t free_tx_qpl_buf_cnt; 531 }; 532 } dqo_compl; 533 } ____cacheline_aligned; 534 u64 pkt_done; /* free-running - total packets completed */ 535 u64 bytes_done; /* free-running - total bytes completed */ 536 u64 dropped_pkt; /* free-running - total packets dropped */ 537 u64 dma_mapping_error; /* count of dma mapping errors */ 538 539 /* Cacheline 2 -- Read-mostly fields */ 540 union { 541 /* GQI fields */ 542 struct { 543 union gve_tx_desc *desc; 544 545 /* Maps 1:1 to a desc */ 546 struct gve_tx_buffer_state *info; 547 }; 548 549 /* DQO fields. */ 550 struct { 551 union gve_tx_desc_dqo *tx_ring; 552 struct gve_tx_compl_desc *compl_ring; 553 554 struct gve_tx_pending_packet_dqo *pending_packets; 555 s16 num_pending_packets; 556 557 u32 complq_mask; /* complq size is complq_mask + 1 */ 558 559 /* QPL fields */ 560 struct { 561 /* qpl assigned to this queue */ 562 struct gve_queue_page_list *qpl; 563 564 /* Each QPL page is divided into TX bounce buffers 565 * of size GVE_TX_BUF_SIZE_DQO. tx_qpl_buf_next is 566 * an array to manage linked lists of TX buffers. 567 * An entry j at index i implies that j'th buffer 568 * is next on the list after i 569 */ 570 s16 *tx_qpl_buf_next; 571 u32 num_tx_qpl_bufs; 572 }; 573 } dqo; 574 } ____cacheline_aligned; 575 struct netdev_queue *netdev_txq; 576 struct gve_queue_resources *q_resources; /* head and tail pointer idx */ 577 struct device *dev; 578 u32 mask; /* masks req and done down to queue size */ 579 u8 raw_addressing; /* use raw_addressing? */ 580 581 /* Slow-path fields */ 582 u32 q_num ____cacheline_aligned; /* queue idx */ 583 u32 stop_queue; /* count of queue stops */ 584 u32 wake_queue; /* count of queue wakes */ 585 u32 queue_timeout; /* count of queue timeouts */ 586 u32 ntfy_id; /* notification block index */ 587 u32 last_kick_msec; /* Last time the queue was kicked */ 588 dma_addr_t bus; /* dma address of the descr ring */ 589 dma_addr_t q_resources_bus; /* dma address of the queue resources */ 590 dma_addr_t complq_bus_dqo; /* dma address of the dqo.compl_ring */ 591 struct u64_stats_sync statss; /* sync stats for 32bit archs */ 592 struct xsk_buff_pool *xsk_pool; 593 u32 xdp_xsk_wakeup; 594 u32 xdp_xsk_done; 595 u64 xdp_xsk_sent; 596 u64 xdp_xmit; 597 u64 xdp_xmit_errors; 598} ____cacheline_aligned; 599 600/* Wraps the info for one irq including the napi struct and the queues 601 * associated with that irq. 602 */ 603struct gve_notify_block { 604 __be32 *irq_db_index; /* pointer to idx into Bar2 */ 605 char name[IFNAMSIZ + 16]; /* name registered with the kernel */ 606 struct napi_struct napi; /* kernel napi struct for this block */ 607 struct gve_priv *priv; 608 struct gve_tx_ring *tx; /* tx rings on this block */ 609 struct gve_rx_ring *rx; /* rx rings on this block */ 610}; 611 612/* Tracks allowed and current queue settings */ 613struct gve_queue_config { 614 u16 max_queues; 615 u16 num_queues; /* current */ 616}; 617 618/* Tracks the available and used qpl IDs */ 619struct gve_qpl_config { 620 u32 qpl_map_size; /* map memory size */ 621 unsigned long *qpl_id_map; /* bitmap of used qpl ids */ 622}; 623 624struct gve_options_dqo_rda { 625 u16 tx_comp_ring_entries; /* number of tx_comp descriptors */ 626 u16 rx_buff_ring_entries; /* number of rx_buff descriptors */ 627}; 628 629struct gve_irq_db { 630 __be32 index; 631} ____cacheline_aligned; 632 633struct gve_ptype { 634 u8 l3_type; /* `gve_l3_type` in gve_adminq.h */ 635 u8 l4_type; /* `gve_l4_type` in gve_adminq.h */ 636}; 637 638struct gve_ptype_lut { 639 struct gve_ptype ptypes[GVE_NUM_PTYPES]; 640}; 641 642/* Parameters for allocating queue page lists */ 643struct gve_qpls_alloc_cfg { 644 struct gve_qpl_config *qpl_cfg; 645 struct gve_queue_config *tx_cfg; 646 struct gve_queue_config *rx_cfg; 647 648 u16 num_xdp_queues; 649 bool raw_addressing; 650 bool is_gqi; 651 652 /* Allocated resources are returned here */ 653 struct gve_queue_page_list *qpls; 654}; 655 656/* Parameters for allocating resources for tx queues */ 657struct gve_tx_alloc_rings_cfg { 658 struct gve_queue_config *qcfg; 659 660 /* qpls and qpl_cfg must already be allocated */ 661 struct gve_queue_page_list *qpls; 662 struct gve_qpl_config *qpl_cfg; 663 664 u16 ring_size; 665 u16 start_idx; 666 u16 num_rings; 667 bool raw_addressing; 668 669 /* Allocated resources are returned here */ 670 struct gve_tx_ring *tx; 671}; 672 673/* Parameters for allocating resources for rx queues */ 674struct gve_rx_alloc_rings_cfg { 675 /* tx config is also needed to determine QPL ids */ 676 struct gve_queue_config *qcfg; 677 struct gve_queue_config *qcfg_tx; 678 679 /* qpls and qpl_cfg must already be allocated */ 680 struct gve_queue_page_list *qpls; 681 struct gve_qpl_config *qpl_cfg; 682 683 u16 ring_size; 684 u16 packet_buffer_size; 685 bool raw_addressing; 686 bool enable_header_split; 687 688 /* Allocated resources are returned here */ 689 struct gve_rx_ring *rx; 690}; 691 692/* GVE_QUEUE_FORMAT_UNSPECIFIED must be zero since 0 is the default value 693 * when the entire configure_device_resources command is zeroed out and the 694 * queue_format is not specified. 695 */ 696enum gve_queue_format { 697 GVE_QUEUE_FORMAT_UNSPECIFIED = 0x0, 698 GVE_GQI_RDA_FORMAT = 0x1, 699 GVE_GQI_QPL_FORMAT = 0x2, 700 GVE_DQO_RDA_FORMAT = 0x3, 701 GVE_DQO_QPL_FORMAT = 0x4, 702}; 703 704struct gve_priv { 705 struct net_device *dev; 706 struct gve_tx_ring *tx; /* array of tx_cfg.num_queues */ 707 struct gve_rx_ring *rx; /* array of rx_cfg.num_queues */ 708 struct gve_queue_page_list *qpls; /* array of num qpls */ 709 struct gve_notify_block *ntfy_blocks; /* array of num_ntfy_blks */ 710 struct gve_irq_db *irq_db_indices; /* array of num_ntfy_blks */ 711 dma_addr_t irq_db_indices_bus; 712 struct msix_entry *msix_vectors; /* array of num_ntfy_blks + 1 */ 713 char mgmt_msix_name[IFNAMSIZ + 16]; 714 u32 mgmt_msix_idx; 715 __be32 *counter_array; /* array of num_event_counters */ 716 dma_addr_t counter_array_bus; 717 718 u16 num_event_counters; 719 u16 tx_desc_cnt; /* num desc per ring */ 720 u16 rx_desc_cnt; /* num desc per ring */ 721 u16 tx_pages_per_qpl; /* Suggested number of pages per qpl for TX queues by NIC */ 722 u16 rx_pages_per_qpl; /* Suggested number of pages per qpl for RX queues by NIC */ 723 u16 rx_data_slot_cnt; /* rx buffer length */ 724 u64 max_registered_pages; 725 u64 num_registered_pages; /* num pages registered with NIC */ 726 struct bpf_prog *xdp_prog; /* XDP BPF program */ 727 u32 rx_copybreak; /* copy packets smaller than this */ 728 u16 default_num_queues; /* default num queues to set up */ 729 730 u16 num_xdp_queues; 731 struct gve_queue_config tx_cfg; 732 struct gve_queue_config rx_cfg; 733 struct gve_qpl_config qpl_cfg; /* map used QPL ids */ 734 u32 num_ntfy_blks; /* spilt between TX and RX so must be even */ 735 736 struct gve_registers __iomem *reg_bar0; /* see gve_register.h */ 737 __be32 __iomem *db_bar2; /* "array" of doorbells */ 738 u32 msg_enable; /* level for netif* netdev print macros */ 739 struct pci_dev *pdev; 740 741 /* metrics */ 742 u32 tx_timeo_cnt; 743 744 /* Admin queue - see gve_adminq.h*/ 745 union gve_adminq_command *adminq; 746 dma_addr_t adminq_bus_addr; 747 struct dma_pool *adminq_pool; 748 u32 adminq_mask; /* masks prod_cnt to adminq size */ 749 u32 adminq_prod_cnt; /* free-running count of AQ cmds executed */ 750 u32 adminq_cmd_fail; /* free-running count of AQ cmds failed */ 751 u32 adminq_timeouts; /* free-running count of AQ cmds timeouts */ 752 /* free-running count of per AQ cmd executed */ 753 u32 adminq_describe_device_cnt; 754 u32 adminq_cfg_device_resources_cnt; 755 u32 adminq_register_page_list_cnt; 756 u32 adminq_unregister_page_list_cnt; 757 u32 adminq_create_tx_queue_cnt; 758 u32 adminq_create_rx_queue_cnt; 759 u32 adminq_destroy_tx_queue_cnt; 760 u32 adminq_destroy_rx_queue_cnt; 761 u32 adminq_dcfg_device_resources_cnt; 762 u32 adminq_set_driver_parameter_cnt; 763 u32 adminq_report_stats_cnt; 764 u32 adminq_report_link_speed_cnt; 765 u32 adminq_get_ptype_map_cnt; 766 u32 adminq_verify_driver_compatibility_cnt; 767 768 /* Global stats */ 769 u32 interface_up_cnt; /* count of times interface turned up since last reset */ 770 u32 interface_down_cnt; /* count of times interface turned down since last reset */ 771 u32 reset_cnt; /* count of reset */ 772 u32 page_alloc_fail; /* count of page alloc fails */ 773 u32 dma_mapping_error; /* count of dma mapping errors */ 774 u32 stats_report_trigger_cnt; /* count of device-requested stats-reports since last reset */ 775 u32 suspend_cnt; /* count of times suspended */ 776 u32 resume_cnt; /* count of times resumed */ 777 struct workqueue_struct *gve_wq; 778 struct work_struct service_task; 779 struct work_struct stats_report_task; 780 unsigned long service_task_flags; 781 unsigned long state_flags; 782 783 struct gve_stats_report *stats_report; 784 u64 stats_report_len; 785 dma_addr_t stats_report_bus; /* dma address for the stats report */ 786 unsigned long ethtool_flags; 787 788 unsigned long stats_report_timer_period; 789 struct timer_list stats_report_timer; 790 791 /* Gvnic device link speed from hypervisor. */ 792 u64 link_speed; 793 bool up_before_suspend; /* True if dev was up before suspend */ 794 795 struct gve_options_dqo_rda options_dqo_rda; 796 struct gve_ptype_lut *ptype_lut_dqo; 797 798 /* Must be a power of two. */ 799 u16 data_buffer_size_dqo; 800 u16 max_rx_buffer_size; /* device limit */ 801 802 enum gve_queue_format queue_format; 803 804 /* Interrupt coalescing settings */ 805 u32 tx_coalesce_usecs; 806 u32 rx_coalesce_usecs; 807 808 u16 header_buf_size; /* device configured, header-split supported if non-zero */ 809 bool header_split_enabled; /* True if the header split is enabled by the user */ 810}; 811 812enum gve_service_task_flags_bit { 813 GVE_PRIV_FLAGS_DO_RESET = 1, 814 GVE_PRIV_FLAGS_RESET_IN_PROGRESS = 2, 815 GVE_PRIV_FLAGS_PROBE_IN_PROGRESS = 3, 816 GVE_PRIV_FLAGS_DO_REPORT_STATS = 4, 817}; 818 819enum gve_state_flags_bit { 820 GVE_PRIV_FLAGS_ADMIN_QUEUE_OK = 1, 821 GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK = 2, 822 GVE_PRIV_FLAGS_DEVICE_RINGS_OK = 3, 823 GVE_PRIV_FLAGS_NAPI_ENABLED = 4, 824}; 825 826enum gve_ethtool_flags_bit { 827 GVE_PRIV_FLAGS_REPORT_STATS = 0, 828}; 829 830static inline bool gve_get_do_reset(struct gve_priv *priv) 831{ 832 return test_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags); 833} 834 835static inline void gve_set_do_reset(struct gve_priv *priv) 836{ 837 set_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags); 838} 839 840static inline void gve_clear_do_reset(struct gve_priv *priv) 841{ 842 clear_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags); 843} 844 845static inline bool gve_get_reset_in_progress(struct gve_priv *priv) 846{ 847 return test_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, 848 &priv->service_task_flags); 849} 850 851static inline void gve_set_reset_in_progress(struct gve_priv *priv) 852{ 853 set_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags); 854} 855 856static inline void gve_clear_reset_in_progress(struct gve_priv *priv) 857{ 858 clear_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags); 859} 860 861static inline bool gve_get_probe_in_progress(struct gve_priv *priv) 862{ 863 return test_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, 864 &priv->service_task_flags); 865} 866 867static inline void gve_set_probe_in_progress(struct gve_priv *priv) 868{ 869 set_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags); 870} 871 872static inline void gve_clear_probe_in_progress(struct gve_priv *priv) 873{ 874 clear_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags); 875} 876 877static inline bool gve_get_do_report_stats(struct gve_priv *priv) 878{ 879 return test_bit(GVE_PRIV_FLAGS_DO_REPORT_STATS, 880 &priv->service_task_flags); 881} 882 883static inline void gve_set_do_report_stats(struct gve_priv *priv) 884{ 885 set_bit(GVE_PRIV_FLAGS_DO_REPORT_STATS, &priv->service_task_flags); 886} 887 888static inline void gve_clear_do_report_stats(struct gve_priv *priv) 889{ 890 clear_bit(GVE_PRIV_FLAGS_DO_REPORT_STATS, &priv->service_task_flags); 891} 892 893static inline bool gve_get_admin_queue_ok(struct gve_priv *priv) 894{ 895 return test_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags); 896} 897 898static inline void gve_set_admin_queue_ok(struct gve_priv *priv) 899{ 900 set_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags); 901} 902 903static inline void gve_clear_admin_queue_ok(struct gve_priv *priv) 904{ 905 clear_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags); 906} 907 908static inline bool gve_get_device_resources_ok(struct gve_priv *priv) 909{ 910 return test_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags); 911} 912 913static inline void gve_set_device_resources_ok(struct gve_priv *priv) 914{ 915 set_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags); 916} 917 918static inline void gve_clear_device_resources_ok(struct gve_priv *priv) 919{ 920 clear_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags); 921} 922 923static inline bool gve_get_device_rings_ok(struct gve_priv *priv) 924{ 925 return test_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags); 926} 927 928static inline void gve_set_device_rings_ok(struct gve_priv *priv) 929{ 930 set_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags); 931} 932 933static inline void gve_clear_device_rings_ok(struct gve_priv *priv) 934{ 935 clear_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags); 936} 937 938static inline bool gve_get_napi_enabled(struct gve_priv *priv) 939{ 940 return test_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags); 941} 942 943static inline void gve_set_napi_enabled(struct gve_priv *priv) 944{ 945 set_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags); 946} 947 948static inline void gve_clear_napi_enabled(struct gve_priv *priv) 949{ 950 clear_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags); 951} 952 953static inline bool gve_get_report_stats(struct gve_priv *priv) 954{ 955 return test_bit(GVE_PRIV_FLAGS_REPORT_STATS, &priv->ethtool_flags); 956} 957 958static inline void gve_clear_report_stats(struct gve_priv *priv) 959{ 960 clear_bit(GVE_PRIV_FLAGS_REPORT_STATS, &priv->ethtool_flags); 961} 962 963/* Returns the address of the ntfy_blocks irq doorbell 964 */ 965static inline __be32 __iomem *gve_irq_doorbell(struct gve_priv *priv, 966 struct gve_notify_block *block) 967{ 968 return &priv->db_bar2[be32_to_cpu(*block->irq_db_index)]; 969} 970 971/* Returns the index into ntfy_blocks of the given tx ring's block 972 */ 973static inline u32 gve_tx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx) 974{ 975 return queue_idx; 976} 977 978/* Returns the index into ntfy_blocks of the given rx ring's block 979 */ 980static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx) 981{ 982 return (priv->num_ntfy_blks / 2) + queue_idx; 983} 984 985static inline bool gve_is_qpl(struct gve_priv *priv) 986{ 987 return priv->queue_format == GVE_GQI_QPL_FORMAT || 988 priv->queue_format == GVE_DQO_QPL_FORMAT; 989} 990 991/* Returns the number of tx queue page lists */ 992static inline u32 gve_num_tx_qpls(const struct gve_queue_config *tx_cfg, 993 int num_xdp_queues, 994 bool is_qpl) 995{ 996 if (!is_qpl) 997 return 0; 998 return tx_cfg->num_queues + num_xdp_queues; 999} 1000 1001/* Returns the number of XDP tx queue page lists 1002 */ 1003static inline u32 gve_num_xdp_qpls(struct gve_priv *priv) 1004{ 1005 if (priv->queue_format != GVE_GQI_QPL_FORMAT) 1006 return 0; 1007 1008 return priv->num_xdp_queues; 1009} 1010 1011/* Returns the number of rx queue page lists */ 1012static inline u32 gve_num_rx_qpls(const struct gve_queue_config *rx_cfg, 1013 bool is_qpl) 1014{ 1015 if (!is_qpl) 1016 return 0; 1017 return rx_cfg->num_queues; 1018} 1019 1020static inline u32 gve_tx_qpl_id(struct gve_priv *priv, int tx_qid) 1021{ 1022 return tx_qid; 1023} 1024 1025static inline u32 gve_rx_qpl_id(struct gve_priv *priv, int rx_qid) 1026{ 1027 return priv->tx_cfg.max_queues + rx_qid; 1028} 1029 1030/* Returns the index into priv->qpls where a certain rx queue's QPL resides */ 1031static inline u32 gve_get_rx_qpl_id(const struct gve_queue_config *tx_cfg, int rx_qid) 1032{ 1033 return tx_cfg->max_queues + rx_qid; 1034} 1035 1036static inline u32 gve_tx_start_qpl_id(struct gve_priv *priv) 1037{ 1038 return gve_tx_qpl_id(priv, 0); 1039} 1040 1041/* Returns the index into priv->qpls where the first rx queue's QPL resides */ 1042static inline u32 gve_rx_start_qpl_id(const struct gve_queue_config *tx_cfg) 1043{ 1044 return gve_get_rx_qpl_id(tx_cfg, 0); 1045} 1046 1047/* Returns a pointer to the next available tx qpl in the list of qpls */ 1048static inline 1049struct gve_queue_page_list *gve_assign_tx_qpl(struct gve_tx_alloc_rings_cfg *cfg, 1050 int tx_qid) 1051{ 1052 /* QPL already in use */ 1053 if (test_bit(tx_qid, cfg->qpl_cfg->qpl_id_map)) 1054 return NULL; 1055 set_bit(tx_qid, cfg->qpl_cfg->qpl_id_map); 1056 return &cfg->qpls[tx_qid]; 1057} 1058 1059/* Returns a pointer to the next available rx qpl in the list of qpls */ 1060static inline 1061struct gve_queue_page_list *gve_assign_rx_qpl(struct gve_rx_alloc_rings_cfg *cfg, 1062 int rx_qid) 1063{ 1064 int id = gve_get_rx_qpl_id(cfg->qcfg_tx, rx_qid); 1065 /* QPL already in use */ 1066 if (test_bit(id, cfg->qpl_cfg->qpl_id_map)) 1067 return NULL; 1068 set_bit(id, cfg->qpl_cfg->qpl_id_map); 1069 return &cfg->qpls[id]; 1070} 1071 1072/* Unassigns the qpl with the given id */ 1073static inline void gve_unassign_qpl(struct gve_qpl_config *qpl_cfg, int id) 1074{ 1075 clear_bit(id, qpl_cfg->qpl_id_map); 1076} 1077 1078/* Returns the correct dma direction for tx and rx qpls */ 1079static inline enum dma_data_direction gve_qpl_dma_dir(struct gve_priv *priv, 1080 int id) 1081{ 1082 if (id < gve_rx_start_qpl_id(&priv->tx_cfg)) 1083 return DMA_TO_DEVICE; 1084 else 1085 return DMA_FROM_DEVICE; 1086} 1087 1088static inline bool gve_is_gqi(struct gve_priv *priv) 1089{ 1090 return priv->queue_format == GVE_GQI_RDA_FORMAT || 1091 priv->queue_format == GVE_GQI_QPL_FORMAT; 1092} 1093 1094static inline u32 gve_num_tx_queues(struct gve_priv *priv) 1095{ 1096 return priv->tx_cfg.num_queues + priv->num_xdp_queues; 1097} 1098 1099static inline u32 gve_xdp_tx_queue_id(struct gve_priv *priv, u32 queue_id) 1100{ 1101 return priv->tx_cfg.num_queues + queue_id; 1102} 1103 1104static inline u32 gve_xdp_tx_start_queue_id(struct gve_priv *priv) 1105{ 1106 return gve_xdp_tx_queue_id(priv, 0); 1107} 1108 1109/* gqi napi handler defined in gve_main.c */ 1110int gve_napi_poll(struct napi_struct *napi, int budget); 1111 1112/* buffers */ 1113int gve_alloc_page(struct gve_priv *priv, struct device *dev, 1114 struct page **page, dma_addr_t *dma, 1115 enum dma_data_direction, gfp_t gfp_flags); 1116void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma, 1117 enum dma_data_direction); 1118/* tx handling */ 1119netdev_tx_t gve_tx(struct sk_buff *skb, struct net_device *dev); 1120int gve_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 1121 u32 flags); 1122int gve_xdp_xmit_one(struct gve_priv *priv, struct gve_tx_ring *tx, 1123 void *data, int len, void *frame_p); 1124void gve_xdp_tx_flush(struct gve_priv *priv, u32 xdp_qid); 1125bool gve_tx_poll(struct gve_notify_block *block, int budget); 1126bool gve_xdp_poll(struct gve_notify_block *block, int budget); 1127int gve_tx_alloc_rings_gqi(struct gve_priv *priv, 1128 struct gve_tx_alloc_rings_cfg *cfg); 1129void gve_tx_free_rings_gqi(struct gve_priv *priv, 1130 struct gve_tx_alloc_rings_cfg *cfg); 1131void gve_tx_start_ring_gqi(struct gve_priv *priv, int idx); 1132void gve_tx_stop_ring_gqi(struct gve_priv *priv, int idx); 1133u32 gve_tx_load_event_counter(struct gve_priv *priv, 1134 struct gve_tx_ring *tx); 1135bool gve_tx_clean_pending(struct gve_priv *priv, struct gve_tx_ring *tx); 1136/* rx handling */ 1137void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx); 1138int gve_rx_poll(struct gve_notify_block *block, int budget); 1139bool gve_rx_work_pending(struct gve_rx_ring *rx); 1140int gve_rx_alloc_rings(struct gve_priv *priv); 1141int gve_rx_alloc_rings_gqi(struct gve_priv *priv, 1142 struct gve_rx_alloc_rings_cfg *cfg); 1143void gve_rx_free_rings_gqi(struct gve_priv *priv, 1144 struct gve_rx_alloc_rings_cfg *cfg); 1145void gve_rx_start_ring_gqi(struct gve_priv *priv, int idx); 1146void gve_rx_stop_ring_gqi(struct gve_priv *priv, int idx); 1147u16 gve_get_pkt_buf_size(const struct gve_priv *priv, bool enable_hplit); 1148bool gve_header_split_supported(const struct gve_priv *priv); 1149int gve_set_hsplit_config(struct gve_priv *priv, u8 tcp_data_split); 1150/* Reset */ 1151void gve_schedule_reset(struct gve_priv *priv); 1152int gve_reset(struct gve_priv *priv, bool attempt_teardown); 1153int gve_adjust_queues(struct gve_priv *priv, 1154 struct gve_queue_config new_rx_config, 1155 struct gve_queue_config new_tx_config); 1156/* report stats handling */ 1157void gve_handle_report_stats(struct gve_priv *priv); 1158/* exported by ethtool.c */ 1159extern const struct ethtool_ops gve_ethtool_ops; 1160/* needed by ethtool */ 1161extern char gve_driver_name[]; 1162extern const char gve_version_str[]; 1163#endif /* _GVE_H_ */ 1164