compat/freebsd_iflib/mp_ring.c

96 drain_ring_locked(struct ifmp_ring *r, union ring_state os, uint16_t prev, int budget)
100 	uint16_t cidx = os.cidx;
101 	uint16_t pidx = os.pidx_tail;
103 	MPASS(os.flags == BUSY);
115 			os.state = ns.state = r->state;
139 		os.state = ns.state = r->state;
170 drain_ring_lockless(struct ifmp_ring *r, union ring_state os, uint16_t prev, int budget)
174 	uint16_t cidx = os.cidx;
175 	uint16_t pidx = os.pidx_tail;
177 	MPASS(os.flags == BUSY);
190 			os.state = r->state;
192 				ns.state = os.state;
195 			} while (atomic_fcmpset_64(&r->state, &os.state,
218 		os.state = r->state;
220 			ns.state = os.state;
223 		} while (atomic_fcmpset_acq_64(&r->state, &os.state,
323 	union ring_state os, ns;
335 	os.state = r->state;
336 	if (n >= space_available(r, os)) {
338 		MPASS(os.flags != IDLE);
340 		if (os.flags == STALLED)
344 	ns.state = os.state;
345 	ns.pidx_head = increment_idx(r, os.pidx_head, n);
347 	pidx_start = os.pidx_head;
372 	os.state = ns.state = r->state;
375 		if (os.flags == IDLE)
387 		if (os.flags != BUSY)
388 			drain_ring_locked(r, ns, os.flags, budget);
398 	union ring_state os, ns;
409 	os.state = r->state;
411 		if (n >= space_available(r, os)) {
413 			MPASS(os.flags != IDLE);
414 			if (os.flags == STALLED)
418 		ns.state = os.state;
419 		ns.pidx_head = increment_idx(r, os.pidx_head, n);
421 		if (atomic_fcmpset_64(&r->state, &os.state, ns.state))
426 	pidx_start = os.pidx_head;
451 	os.state = r->state;
453 		ns.state = os.state;
456 			if (os.flags == IDLE)
460 	} while (atomic_fcmpset_rel_64(&r->state, &os.state, ns.state) == 0);
469 		if (os.flags != BUSY)
470 			drain_ring_lockless(r, ns, os.flags, budget);
480 	union ring_state os, ns;
482 	os.state = r->state;
483 	if ((os.flags != STALLED && os.flags != ABDICATED) ||	// Only continue in STALLED and ABDICATED
484 	    os.pidx_head != os.pidx_tail ||			// Require work to be available
485 	    (os.flags != ABDICATED && r->can_drain(r) == 0))	// Can either drain, or everyone left
488 	MPASS(os.cidx != os.pidx_tail);	/* implied by STALLED */
489 	ns.state = os.state;
494 	if (r->state != os.state) {
499 	drain_ring_locked(r, ns, os.flags, budget);
506 	if (!atomic_cmpset_acq_64(&r->state, os.state, ns.state))
509 	drain_ring_lockless(r, ns, os.flags, budget);