linux-master/io_uring/io_uring.c

186 	return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
191 	return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head);
361 	struct io_rings *r = ctx->rings;
578 	if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
653 		/* IOPOLL rings only need to wake up if it's also SQPOLL */
712 		atomic_andnot(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
806 		atomic_or(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
835 	struct io_rings *rings = ctx->rings;
860 	ctx->cqe_cached = &rings->cqes[off];
1165 		atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1346 			atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1371 		atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1421 		atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1435 		atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
2411 	struct io_rings *rings = ctx->rings;
2418 	smp_store_release(&rings->sq.head, ctx->cached_sq_head);
2441 			WRITE_ONCE(ctx->rings->sq_dropped,
2442 				   READ_ONCE(ctx->rings->sq_dropped) + 1);
2592 	struct io_rings *rings = ctx->rings;
2610 	iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;
2640 		int nr_wait = (int) iowq.cq_tail - READ_ONCE(ctx->rings->cq.tail);
2698 	return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
2801 		io_mem_free(ctx->rings);
2810 	ctx->rings = NULL;
2828 	struct io_rings *rings;
2831 	off = struct_size(rings, cqes, cq_entries);
3148 	if (ctx->rings)
3153 	 * If we failed setting up the ctx, we might not have any rings
3156 	if (ctx->rings)
3164 	 * if we're exiting a ton of rings at the same time. It just adds
3291 	if (!ctx->rings)
3298 		 * Cancels requests of all rings, not only @ctx, but
3441 		ptr = ctx->rings;
3760 	struct io_rings *rings;
3773 		rings = io_mem_alloc(size);
3775 		rings = io_rings_map(ctx, p->cq_off.user_addr, size);
3777 	if (IS_ERR(rings))
3778 		return PTR_ERR(rings);
3780 	ctx->rings = rings;
3782 		ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
3783 	rings->sq_ring_mask = p->sq_entries - 1;
3784 	rings->cq_ring_mask = p->cq_entries - 1;
3785 	rings->sq_ring_entries = p->sq_entries;
3786 	rings->cq_ring_entries = p->cq_entries;
3975 		p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;