linux-master/io_uring/rw.c

26 	/* NOTE: kiocb has the file as the first member, so don't do it here */
27 	struct kiocb			kiocb;
84 	rw->kiocb.ki_pos = READ_ONCE(sqe->off);
94 		rw->kiocb.ki_ioprio = ioprio;
96 		rw->kiocb.ki_ioprio = get_current_ioprio();
98 	rw->kiocb.dio_complete = NULL;
177 	if (rw->kiocb.ki_pos != -1)
178 		return &rw->kiocb.ki_pos;
182 		rw->kiocb.ki_pos = req->file->f_pos;
183 		return &rw->kiocb.ki_pos;
186 	rw->kiocb.ki_pos = 0;
248 		kiocb_end_write(&rw->kiocb);
260 	if (rw->kiocb.ki_flags & IOCB_WRITE) {
304 	struct kiocb *kiocb = &rw->kiocb;
306 	if ((kiocb->ki_flags & IOCB_DIO_CALLER_COMP) && kiocb->dio_complete) {
307 		long res = kiocb->dio_complete(rw->kiocb.private);
322 static void io_complete_rw(struct kiocb *kiocb, long res)
324 	struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);
327 	if (!kiocb->dio_complete || !(kiocb->ki_flags & IOCB_DIO_CALLER_COMP)) {
336 static void io_complete_rw_iopoll(struct kiocb *kiocb, long res)
338 	struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);
341 	if (kiocb->ki_flags & IOCB_WRITE)
355 static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
378 	INDIRECT_CALL_2(kiocb->ki_complete, io_complete_rw_iopoll,
379 			io_complete_rw, kiocb, ret);
389 		req->file->f_pos = rw->kiocb.ki_pos;
390 	if (ret >= 0 && (rw->kiocb.ki_complete == io_complete_rw)) {
402 		io_rw_done(&rw->kiocb, ret);
472 static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
474 	return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;
483 	struct kiocb *kiocb = &rw->kiocb;
484 	struct file *file = kiocb->ki_filp;
491 	 * the kiocb to be handled from an async context.
493 	if (kiocb->ki_flags & IOCB_HIPRI)
495 	if ((kiocb->ki_flags & IOCB_NOWAIT) &&
496 	    !(kiocb->ki_filp->f_flags & O_NONBLOCK))
499 	ppos = io_kiocb_ppos(kiocb);
649 	rw->kiocb.ki_flags &= ~IOCB_WAITQ;
672 	struct kiocb *kiocb = &rw->kiocb;
679 	if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))
693 	kiocb->ki_flags |= IOCB_WAITQ;
694 	kiocb->ki_flags &= ~IOCB_NOWAIT;
695 	kiocb->ki_waitq = wait;
701 	struct file *file = rw->kiocb.ki_filp;
704 		return call_read_iter(file, &rw->kiocb, iter);
720 	struct kiocb *kiocb = &rw->kiocb;
731 	kiocb->ki_flags = file->f_iocb_flags;
732 	ret = kiocb_set_rw_flags(kiocb, rw->flags);
735 	kiocb->ki_flags |= IOCB_ALLOC_CACHE;
742 	if ((kiocb->ki_flags & IOCB_NOWAIT) ||
747 		if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll)
750 		kiocb->private = NULL;
751 		kiocb->ki_flags |= IOCB_HIPRI;
752 		kiocb->ki_complete = io_complete_rw_iopoll;
755 		if (kiocb->ki_flags & IOCB_HIPRI)
757 		kiocb->ki_complete = io_complete_rw;
768 	struct kiocb *kiocb = &rw->kiocb;
813 		kiocb->ki_flags |= IOCB_NOWAIT;
816 		kiocb->ki_flags &= ~IOCB_NOWAIT;
889 			kiocb->ki_flags &= ~IOCB_WAITQ;
904 		kiocb->ki_flags &= ~IOCB_WAITQ;
1004 	struct kiocb *kiocb = &rw->kiocb;
1033 		if (!(kiocb->ki_flags & IOCB_DIRECT) &&
1034 			!(kiocb->ki_filp->f_mode & FMODE_BUF_WASYNC) &&
1038 		kiocb->ki_flags |= IOCB_NOWAIT;
1041 		kiocb->ki_flags &= ~IOCB_NOWAIT;
1053 		kiocb_start_write(kiocb);
1054 	kiocb->ki_flags |= IOCB_WRITE;
1057 		ret2 = call_write_iter(req->file, kiocb, &s->iter);
1072 	if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
1085 			trace_io_uring_short_write(req->ctx, kiocb->ki_pos - ret2,
1100 			if (kiocb->ki_flags & IOCB_WRITE)
1111 			if (kiocb->ki_flags & IOCB_WRITE)
1167 			ret = file->f_op->iopoll(&rw->kiocb, &iob, poll_flags);