dev/netmap/netmap_kloop.c

61 /* Write kring pointers (hwcur, hwtail) to the CSB.
91 /* Read kring pointers (head, cur, sync_flags) from the CSB.
134 sync_kloop_kring_dump(const char *title, const struct netmap_kring *kring)
136 	nm_prinf("%s, kring %s, hwcur %d, rhead %d, "
138 		title, kring->name, kring->nr_hwcur, kring->rhead,
139 		kring->rcur, kring->rtail, kring->nr_hwtail);
146 	struct netmap_kring *kring;
161 	struct netmap_kring *kring = a->kring;
171 	if (unlikely(nm_kr_tryget(kring, 1, NULL))) {
175 	num_slots = kring->nkr_num_slots;
181 	/* Copy the application kring pointers from the CSB */
185 		batch = shadow_ring.head - kring->nr_hwcur;
193 			uint32_t head_lim = kring->nr_hwcur + PTN_TX_BATCH_LIM(num_slots);
204 		if (nm_kr_txspace(kring) <= (num_slots >> 1)) {
209 		shadow_ring.tail = kring->rtail;
210 		if (unlikely(nm_txsync_prologue(kring, &shadow_ring) >= num_slots)) {
212 			netmap_ring_reinit(kring);
220 			sync_kloop_kring_dump("pre txsync", kring);
223 		if (unlikely(kring->nm_sync(kring, shadow_ring.flags))) {
237 		sync_kloop_kernel_write(csb_ktoa, kring->nr_hwcur,
238 				kring->nr_hwtail);
239 		if (kring->rtail != kring->nr_hwtail) {
241 			kring->rtail = kring->nr_hwtail;
248 			sync_kloop_kring_dump("post txsync", kring);
263 		if (shadow_ring.head == kring->rhead) {
277 			if (shadow_ring.head != kring->rhead) {
286 		if (nm_kr_txempty(kring)) {
294 	nm_kr_put(kring);
307 sync_kloop_norxslots(struct netmap_kring *kring, uint32_t g_head)
309 	return (NM_ACCESS_ONCE(kring->nr_hwtail) == nm_prev(g_head,
310 				kring->nkr_num_slots - 1));
317 	struct netmap_kring *kring = a->kring;
327 	if (unlikely(nm_kr_tryget(kring, 1, NULL))) {
331 	num_slots = kring->nkr_num_slots;
334 	num_slots = kring->nkr_num_slots;
340 	/* Copy the application kring pointers from the CSB */
347 		shadow_ring.tail = kring->rtail;
348 		if (unlikely(nm_rxsync_prologue(kring, &shadow_ring) >= num_slots)) {
350 			netmap_ring_reinit(kring);
358 			sync_kloop_kring_dump("pre rxsync", kring);
361 		if (unlikely(kring->nm_sync(kring, shadow_ring.flags))) {
374 		hwtail = NM_ACCESS_ONCE(kring->nr_hwtail);
375 		sync_kloop_kernel_write(csb_ktoa, kring->nr_hwcur, hwtail);
376 		if (kring->rtail != hwtail) {
377 			kring->rtail = hwtail;
387 			sync_kloop_kring_dump("post rxsync", kring);
402 		if (sync_kloop_norxslots(kring, shadow_ring.head)) {
416 			if (!sync_kloop_norxslots(kring, shadow_ring.head)) {
425 		hwtail = NM_ACCESS_ONCE(kring->nr_hwtail);
426 		if (unlikely(hwtail == kring->rhead ||
431 					hwtail, kring->rhead, dry_cycles);
436 	nm_kr_put(kring);
641 		a->kring = NMR(na, NR_TX)[i + priv->np_qfirst[NR_TX]];
650 		a->kring = NMR(na, NR_RX)[i + priv->np_qfirst[NR_RX]];
973 			struct netmap_kring *kring, int flags)
984 	kring->nr_hwcur = ktoa->hwcur;
985 	nm_sync_kloop_appl_write(atok, kring->rcur, kring->rhead);
988 	if (((kring->rhead != kring->nr_hwcur || nm_kr_wouldblock(kring))
998 	if (nm_kr_wouldblock(kring) || (flags & NAF_FORCE_RECLAIM)) {
999 		nm_sync_kloop_appl_read(ktoa, &kring->nr_hwtail,
1000 					&kring->nr_hwcur);
1008 	if (nm_kr_wouldblock(kring) && !(kring->nr_kflags & NKR_NOINTR)) {
1013 		nm_sync_kloop_appl_read(ktoa, &kring->nr_hwtail,
1014 					&kring->nr_hwcur);
1016 		if (unlikely(!nm_kr_wouldblock(kring))) {
1022 		kring->name, atok->head, atok->cur, ktoa->hwtail,
1023 		kring->rhead, kring->rcur, kring->nr_hwtail);
1041 			struct netmap_kring *kring, int flags)
1049 	 * First part: import newly received packets, by updating the kring
1051 	 * This also updates the kring hwcur.
1053 	nm_sync_kloop_appl_read(ktoa, &kring->nr_hwtail, &kring->nr_hwcur);
1054 	kring->nr_kflags &= ~NKR_PENDINTR;
1060 	if (kring->rhead != kring->nr_hwcur) {
1061 		nm_sync_kloop_appl_write(atok, kring->rcur, kring->rhead);
1069 	if (nm_kr_wouldblock(kring) && !(kring->nr_kflags & NKR_NOINTR)) {
1074 		nm_sync_kloop_appl_read(ktoa, &kring->nr_hwtail,
1075 					&kring->nr_hwcur);
1077 		if (!nm_kr_wouldblock(kring)) {
1083 	if ((kring->rhead != kring->nr_hwcur || nm_kr_wouldblock(kring))
1090 		kring->name, atok->head, atok->cur, ktoa->hwtail,
1091 		kring->rhead, kring->rcur, kring->nr_hwtail);