net/sunrpc/svc.c

603 svc_init_buffer(struct svc_rqst *rqstp, unsigned int size, int node)
608 	if (svc_is_backchannel(rqstp))
619 					  rqstp->rq_pages);
627 svc_release_buffer(struct svc_rqst *rqstp)
631 	for (i = 0; i < ARRAY_SIZE(rqstp->rq_pages); i++)
632 		if (rqstp->rq_pages[i])
633 			put_page(rqstp->rq_pages[i]);
639 	struct svc_rqst	*rqstp;
641 	rqstp = kzalloc_node(sizeof(*rqstp), GFP_KERNEL, node);
642 	if (!rqstp)
643 		return rqstp;
645 	folio_batch_init(&rqstp->rq_fbatch);
647 	rqstp->rq_server = serv;
648 	rqstp->rq_pool = pool;
650 	rqstp->rq_scratch_page = alloc_pages_node(node, GFP_KERNEL, 0);
651 	if (!rqstp->rq_scratch_page)
654 	rqstp->rq_argp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
655 	if (!rqstp->rq_argp)
658 	rqstp->rq_resp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
659 	if (!rqstp->rq_resp)
662 	if (!svc_init_buffer(rqstp, serv->sv_max_mesg, node))
665 	return rqstp;
667 	svc_rqst_free(rqstp);
675 	struct svc_rqst	*rqstp;
677 	rqstp = svc_rqst_alloc(serv, pool, node);
678 	if (!rqstp)
690 	list_add_rcu(&rqstp->rq_all, &pool->sp_all_threads);
692 	return rqstp;
706 	struct svc_rqst	*rqstp;
712 		rqstp = llist_entry(ln, struct svc_rqst, rq_idle);
713 		WRITE_ONCE(rqstp->rq_qtime, ktime_get());
714 		if (!task_is_running(rqstp->rq_task)) {
715 			wake_up_process(rqstp->rq_task);
716 			trace_svc_wake_up(rqstp->rq_task->pid);
770 	struct svc_rqst	*rqstp;
781 		rqstp = svc_prepare_thread(serv, chosen_pool, node);
782 		if (IS_ERR(rqstp))
783 			return PTR_ERR(rqstp);
784 		task = kthread_create_on_node(serv->sv_threadfn, rqstp,
787 			svc_exit_thread(rqstp);
791 		rqstp->rq_task = task;
855  * @rqstp: svc_rqst with pages to replace
865 bool svc_rqst_replace_page(struct svc_rqst *rqstp, struct page *page)
867 	struct page **begin = rqstp->rq_pages;
868 	struct page **end = &rqstp->rq_pages[RPCSVC_MAXPAGES];
870 	if (unlikely(rqstp->rq_next_page < begin || rqstp->rq_next_page > end)) {
871 		trace_svc_replace_page_err(rqstp);
875 	if (*rqstp->rq_next_page) {
876 		if (!folio_batch_add(&rqstp->rq_fbatch,
877 				page_folio(*rqstp->rq_next_page)))
878 			__folio_batch_release(&rqstp->rq_fbatch);
882 	*(rqstp->rq_next_page++) = page;
889  * @rqstp: RPC transaction context
894 void svc_rqst_release_pages(struct svc_rqst *rqstp)
896 	int i, count = rqstp->rq_next_page - rqstp->rq_respages;
899 		release_pages(rqstp->rq_respages, count);
901 			rqstp->rq_respages[i] = NULL;
910 svc_rqst_free(struct svc_rqst *rqstp)
912 	folio_batch_release(&rqstp->rq_fbatch);
913 	svc_release_buffer(rqstp);
914 	if (rqstp->rq_scratch_page)
915 		put_page(rqstp->rq_scratch_page);
916 	kfree(rqstp->rq_resp);
917 	kfree(rqstp->rq_argp);
918 	kfree(rqstp->rq_auth_data);
919 	kfree_rcu(rqstp, rq_rcu_head);
924 svc_exit_thread(struct svc_rqst *rqstp)
926 	struct svc_serv	*serv = rqstp->rq_server;
927 	struct svc_pool	*pool = rqstp->rq_pool;
929 	list_del_rcu(&rqstp->rq_all);
938 	svc_rqst_free(rqstp);
1217 void svc_printk(struct svc_rqst *rqstp, const char *fmt, ...)
1228 	dprintk("svc: %s: %pV", svc_print_addr(rqstp, buf, sizeof(buf)), &vaf);
1233 static __printf(2,3) void svc_printk(struct svc_rqst *rqstp, const char *fmt, ...) {}
1237 svc_generic_init_request(struct svc_rqst *rqstp,
1244 	if (rqstp->rq_vers >= progp->pg_nvers )
1246 	versp = progp->pg_vers[rqstp->rq_vers];
1261 	if (versp->vs_need_cong_ctrl && rqstp->rq_xprt &&
1262 	    !test_bit(XPT_CONG_CTRL, &rqstp->rq_xprt->xpt_flags))
1265 	if (rqstp->rq_proc >= versp->vs_nproc)
1267 	rqstp->rq_procinfo = procp = &versp->vs_proc[rqstp->rq_proc];
1270 	memset(rqstp->rq_argp, 0, procp->pc_argzero);
1271 	memset(rqstp->rq_resp, 0, procp->pc_ressize);
1274 	this_cpu_inc(versp->vs_count[rqstp->rq_proc]);
1291 svc_process_common(struct svc_rqst *rqstp)
1293 	struct xdr_stream	*xdr = &rqstp->rq_res_stream;
1296 	struct svc_serv		*serv = rqstp->rq_server;
1304 	set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
1305 	clear_bit(RQ_DROPME, &rqstp->rq_flags);
1308 	svcxdr_init_encode(rqstp);
1309 	xdr_stream_encode_be32(xdr, rqstp->rq_xid);
1312 	p = xdr_inline_decode(&rqstp->rq_arg_stream, XDR_UNIT * 4);
1320 	rqstp->rq_prog = be32_to_cpup(p++);
1321 	rqstp->rq_vers = be32_to_cpup(p++);
1322 	rqstp->rq_proc = be32_to_cpup(p);
1325 		if (rqstp->rq_prog == progp->pg_prog)
1333 	auth_res = svc_authenticate(rqstp);
1336 		auth_res = progp->pg_authenticate(rqstp);
1337 	trace_svc_authenticate(rqstp, auth_res);
1361 	switch (progp->pg_init_request(rqstp, progp, &process)) {
1372 	procp = rqstp->rq_procinfo;
1380 	trace_svc_process(rqstp, progp->pg_name);
1388 		svc_reserve_auth(rqstp, procp->pc_xdrressize<<2);
1391 	rc = process.dispatch(rqstp);
1393 		procp->pc_release(rqstp);
1398 	if (rqstp->rq_auth_stat != rpc_auth_ok)
1401 	if (*rqstp->rq_accept_statp != rpc_success)
1408 	if (svc_authorise(rqstp))
1413 	svc_authorise(rqstp);	/* doesn't hurt to call this twice */
1418 	svc_authorise(rqstp);
1420 	if (rqstp->rq_xprt && test_bit(XPT_TEMP, &rqstp->rq_xprt->xpt_flags))
1421 		svc_xprt_close(rqstp->rq_xprt);
1426 	svc_printk(rqstp, "short len %u, dropping request\n",
1427 		   rqstp->rq_arg.len);
1442 		be32_to_cpu(rqstp->rq_auth_stat));
1449 	xdr_stream_encode_be32(xdr, rqstp->rq_auth_stat);
1453 	dprintk("svc: unknown program %d\n", rqstp->rq_prog);
1456 	*rqstp->rq_accept_statp = rpc_prog_unavail;
1460 	svc_printk(rqstp, "unknown version (%d for prog %d, %s)\n",
1461 		       rqstp->rq_vers, rqstp->rq_prog, progp->pg_name);
1465 	*rqstp->rq_accept_statp = rpc_prog_mismatch;
1476 	svc_printk(rqstp, "unknown procedure (%d)\n", rqstp->rq_proc);
1480 	*rqstp->rq_accept_statp = rpc_proc_unavail;
1484 	svc_printk(rqstp, "failed to decode RPC header\n");
1488 	*rqstp->rq_accept_statp = rpc_garbage_args;
1494 	*rqstp->rq_accept_statp = rpc_system_err;
1500  * @rqstp: RPC transaction context
1503 void svc_process(struct svc_rqst *rqstp)
1505 	struct kvec		*resv = &rqstp->rq_res.head[0];
1511 		svc_xprt_deferred_close(rqstp->rq_xprt);
1518 	rqstp->rq_next_page = &rqstp->rq_respages[1];
1519 	resv->iov_base = page_address(rqstp->rq_respages[0]);
1521 	rqstp->rq_res.pages = rqstp->rq_next_page;
1522 	rqstp->rq_res.len = 0;
1523 	rqstp->rq_res.page_base = 0;
1524 	rqstp->rq_res.page_len = 0;
1525 	rqstp->rq_res.buflen = PAGE_SIZE;
1526 	rqstp->rq_res.tail[0].iov_base = NULL;
1527 	rqstp->rq_res.tail[0].iov_len = 0;
1529 	svcxdr_init_decode(rqstp);
1530 	p = xdr_inline_decode(&rqstp->rq_arg_stream, XDR_UNIT * 2);
1533 	rqstp->rq_xid = *p++;
1537 	if (!svc_process_common(rqstp))
1539 	svc_send(rqstp);
1543 	svc_printk(rqstp, "bad direction 0x%08x, dropping request\n",
1545 	if (rqstp->rq_server->sv_stats)
1546 		rqstp->rq_server->sv_stats->rpcbadfmt++;
1548 	svc_drop(rqstp);
1555  * @rqstp: server-side execution context
1558 void svc_process_bc(struct rpc_rqst *req, struct svc_rqst *rqstp)
1565 	rqstp->rq_xid = req->rq_xid;
1566 	rqstp->rq_prot = req->rq_xprt->prot;
1567 	rqstp->rq_bc_net = req->rq_xprt->xprt_net;
1569 	rqstp->rq_addrlen = sizeof(req->rq_xprt->addr);
1570 	memcpy(&rqstp->rq_addr, &req->rq_xprt->addr, rqstp->rq_addrlen);
1571 	memcpy(&rqstp->rq_arg, &req->rq_rcv_buf, sizeof(rqstp->rq_arg));
1572 	memcpy(&rqstp->rq_res, &req->rq_snd_buf, sizeof(rqstp->rq_res));
1575 	rqstp->rq_arg.len = req->rq_private_buf.len;
1576 	if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len) {
1577 		rqstp->rq_arg.head[0].iov_len = rqstp->rq_arg.len;
1578 		rqstp->rq_arg.page_len = 0;
1579 	} else if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len +
1580 			rqstp->rq_arg.page_len)
1581 		rqstp->rq_arg.page_len = rqstp->rq_arg.len -
1582 			rqstp->rq_arg.head[0].iov_len;
1584 		rqstp->rq_arg.len = rqstp->rq_arg.head[0].iov_len +
1585 			rqstp->rq_arg.page_len;
1588 	rqstp->rq_res.head[0].iov_len = 0;
1594 	svcxdr_init_decode(rqstp);
1595 	if (!xdr_inline_decode(&rqstp->rq_arg_stream, XDR_UNIT * 2))
1599 	proc_error = svc_process_common(rqstp);
1608 	if (rqstp->bc_to_initval > 0) {
1609 		timeout.to_initval = rqstp->bc_to_initval;
1610 		timeout.to_retries = rqstp->bc_to_retries;
1615 	memcpy(&req->rq_snd_buf, &rqstp->rq_res, sizeof(req->rq_snd_buf));
1628  * @rqstp: RPC transaction context
1633 u32 svc_max_payload(const struct svc_rqst *rqstp)
1635 	u32 max = rqstp->rq_xprt->xpt_class->xcl_max_payload;
1637 	if (rqstp->rq_server->sv_max_payload < max)
1638 		max = rqstp->rq_server->sv_max_payload;
1645  * @rqstp: svc_rqst to operate on
1650 const char *svc_proc_name(const struct svc_rqst *rqstp)
1652 	if (rqstp && rqstp->rq_procinfo)
1653 		return rqstp->rq_procinfo->pc_name;
1660  * @rqstp: svc_rqst to operate on
1661  * @offset: payload's byte offset in rqstp->rq_res
1667 int svc_encode_result_payload(struct svc_rqst *rqstp, unsigned int offset,
1670 	return rqstp->rq_xprt->xpt_ops->xpo_result_payload(rqstp, offset,
1677  * @rqstp: svc_rqst to operate on
1680  * Fills in rqstp::rq_vec, and returns the number of elements.
1682 unsigned int svc_fill_write_vector(struct svc_rqst *rqstp,
1687 	struct kvec *vec = rqstp->rq_vec;
1710 	WARN_ON_ONCE(i > ARRAY_SIZE(rqstp->rq_vec));
1717  * @rqstp: svc_rqst to operate on
1726 char *svc_fill_symlink_pathname(struct svc_rqst *rqstp, struct kvec *first,