net/core/gro.c

234 static void napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)
267 	gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count);
270 static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,
273 	struct list_head *head = &napi->gro_hash[index].list;
280 		napi_gro_complete(napi, skb);
281 		napi->gro_hash[index].count--;
284 	if (!napi->gro_hash[index].count)
285 		__clear_bit(index, &napi->gro_bitmask);
288 /* napi->gro_hash[].list contains packets ordered by age.
292 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
294 	unsigned long bitmask = napi->gro_bitmask;
300 		__napi_gro_flush_chain(napi, base, flush_old);
423 static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head)
435 	/* Do not adjust napi->gro_hash[].count, caller is adding a new
439 	napi_gro_complete(napi, oldest);
442 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
445 	struct gro_list *gro_list = &napi->gro_hash[bucket];
511 		napi_gro_complete(napi, pp);
522 		gro_flush_oldest(napi, &gro_list->list);
536 		if (!test_bit(bucket, &napi->gro_bitmask))
537 			__set_bit(bucket, &napi->gro_bitmask);
538 	} else if (test_bit(bucket, &napi->gro_bitmask)) {
539 		__clear_bit(bucket, &napi->gro_bitmask);
578 static gro_result_t napi_skb_finish(struct napi_struct *napi,
584 		gro_normal_one(napi, skb, 1);
605 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
609 	skb_mark_napi_id(skb, napi);
614 	ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb));
621 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
631 	skb->dev = napi->dev;
647 	napi->skb = skb;
650 struct sk_buff *napi_get_frags(struct napi_struct *napi)
652 	struct sk_buff *skb = napi->skb;
655 		skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
657 			napi->skb = skb;
658 			skb_mark_napi_id(skb, napi);
665 static gro_result_t napi_frags_finish(struct napi_struct *napi,
675 			gro_normal_one(napi, skb, 1);
682 			napi_reuse_skb(napi, skb);
697 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
699 	struct sk_buff *skb = napi->skb;
703 	napi->skb = NULL;
712 					     __func__, napi->dev->name);
713 			napi_reuse_skb(napi, skb);
737 gro_result_t napi_gro_frags(struct napi_struct *napi)
740 	struct sk_buff *skb = napi_frags_skb(napi);
744 	ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));