net/sunrpc/svc.c

603 svc_init_buffer(struct svc_rqst *rqstp, unsigned int size, int node)
608 	if (svc_is_backchannel(rqstp))
619 					  rqstp->rq_pages);
627 svc_release_buffer(struct svc_rqst *rqstp)
631 	for (i = 0; i < ARRAY_SIZE(rqstp->rq_pages); i++)
632 		if (rqstp->rq_pages[i])
633 			put_page(rqstp->rq_pages[i]);
639 	struct svc_rqst	*rqstp;
641 	rqstp = kzalloc_node(sizeof(*rqstp), GFP_KERNEL, node);
642 	if (!rqstp)
643 		return rqstp;
645 	folio_batch_init(&rqstp->rq_fbatch);
647 	rqstp->rq_server = serv;
648 	rqstp->rq_pool = pool;
650 	rqstp->rq_scratch_page = alloc_pages_node(node, GFP_KERNEL, 0);
651 	if (!rqstp->rq_scratch_page)
654 	rqstp->rq_argp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
655 	if (!rqstp->rq_argp)
658 	rqstp->rq_resp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
659 	if (!rqstp->rq_resp)
662 	if (!svc_init_buffer(rqstp, serv->sv_max_mesg, node))
665 	return rqstp;
667 	svc_rqst_free(rqstp);
675 	struct svc_rqst	*rqstp;
677 	rqstp = svc_rqst_alloc(serv, pool, node);
678 	if (!rqstp)
690 	list_add_rcu(&rqstp->rq_all, &pool->sp_all_threads);
692 	return rqstp;
706 	struct svc_rqst	*rqstp;
712 		rqstp = llist_entry(ln, struct svc_rqst, rq_idle);
713 		WRITE_ONCE(rqstp->rq_qtime, ktime_get());
714 		if (!task_is_running(rqstp->rq_task)) {
715 			wake_up_process(rqstp->rq_task);
716 			trace_svc_wake_up(rqstp->rq_task->pid);
770 	struct svc_rqst	*rqstp;
781 		rqstp = svc_prepare_thread(serv, chosen_pool, node);
782 		if (IS_ERR(rqstp))
783 			return PTR_ERR(rqstp);
784 		task = kthread_create_on_node(serv->sv_threadfn, rqstp,
787 			svc_exit_thread(rqstp);
791 		rqstp->rq_task = task;
855  * @rqstp: svc_rqst with pages to replace
865 bool svc_rqst_replace_page(struct svc_rqst *rqstp, struct page *page)
867 	struct page **begin = rqstp->rq_pages;
868 	struct page **end = &rqstp->rq_pages[RPCSVC_MAXPAGES];
870 	if (unlikely(rqstp->rq_next_page < begin || rqstp->rq_next_page > end)) {
871 		trace_svc_replace_page_err(rqstp);
875 	if (*rqstp->rq_next_page) {
876 		if (!folio_batch_add(&rqstp->rq_fbatch,
877 				page_folio(*rqstp->rq_next_page)))
878 			__folio_batch_release(&rqstp->rq_fbatch);
882 	*(rqstp->rq_next_page++) = page;
889  * @rqstp: RPC transaction context
894 void svc_rqst_release_pages(struct svc_rqst *rqstp)
896 	int i, count = rqstp->rq_next_page - rqstp->rq_respages;
899 		release_pages(rqstp->rq_respages, count);
901 			rqstp->rq_respages[i] = NULL;
910 svc_rqst_free(struct svc_rqst *rqstp)
912 	folio_batch_release(&rqstp->rq_fbatch);
913 	svc_release_buffer(rqstp);
914 	if (rqstp->rq_scratch_page)
915 		put_page(rqstp->rq_scratch_page);
916 	kfree(rqstp->rq_resp);
917 	kfree(rqstp->rq_argp);
918 	kfree(rqstp->rq_auth_data);
919 	kfree_rcu(rqstp, rq_rcu_head);
924 svc_exit_thread(struct svc_rqst *rqstp)
926 	struct svc_serv	*serv = rqstp->rq_server;
927 	struct svc_pool	*pool = rqstp->rq_pool;
929 	list_del_rcu(&rqstp->rq_all);
938 	svc_rqst_free(rqstp);
1217 void svc_printk(struct svc_rqst *rqstp, const char *fmt, ...)
1228 	dprintk("svc: %s: %pV", svc_print_addr(rqstp, buf, sizeof(buf)), &vaf);
1233 static __printf(2,3) void svc_printk(struct svc_rqst *rqstp, const char *fmt, ...) {}
1237 svc_generic_init_request(struct svc_rqst *rqstp,
1244 	if (rqstp->rq_vers >= progp->pg_nvers )
1246 	versp = progp->pg_vers[rqstp->rq_vers];
1261 	if (versp->vs_need_cong_ctrl && rqstp->rq_xprt &&
1262 	    !test_bit(XPT_CONG_CTRL, &rqstp->rq_xprt->xpt_flags))
1265 	if (rqstp->rq_proc >= versp->vs_nproc)
1267 	rqstp->rq_procinfo = procp = &versp->vs_proc[rqstp->rq_proc];
1272 	memset(rqstp->rq_argp, 0, procp->pc_argzero);
1273 	memset(rqstp->rq_resp, 0, procp->pc_ressize);
1276 	this_cpu_inc(versp->vs_count[rqstp->rq_proc]);
1293 svc_process_common(struct svc_rqst *rqstp)
1295 	struct xdr_stream	*xdr = &rqstp->rq_res_stream;
1298 	struct svc_serv		*serv = rqstp->rq_server;
1306 	set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
1307 	clear_bit(RQ_DROPME, &rqstp->rq_flags);
1310 	svcxdr_init_encode(rqstp);
1311 	xdr_stream_encode_be32(xdr, rqstp->rq_xid);
1314 	p = xdr_inline_decode(&rqstp->rq_arg_stream, XDR_UNIT * 4);
1322 	rqstp->rq_prog = be32_to_cpup(p++);
1323 	rqstp->rq_vers = be32_to_cpup(p++);
1324 	rqstp->rq_proc = be32_to_cpup(p);
1327 		if (rqstp->rq_prog == progp->pg_prog)
1335 	auth_res = svc_authenticate(rqstp);
1338 		auth_res = progp->pg_authenticate(rqstp);
1339 	trace_svc_authenticate(rqstp, auth_res);
1363 	switch (progp->pg_init_request(rqstp, progp, &process)) {
1374 	procp = rqstp->rq_procinfo;
1382 	trace_svc_process(rqstp, progp->pg_name);
1390 		svc_reserve_auth(rqstp, procp->pc_xdrressize<<2);
1393 	rc = process.dispatch(rqstp);
1395 		procp->pc_release(rqstp);
1400 	if (rqstp->rq_auth_stat != rpc_auth_ok)
1403 	if (*rqstp->rq_accept_statp != rpc_success)
1410 	if (svc_authorise(rqstp))
1415 	svc_authorise(rqstp);	/* doesn't hurt to call this twice */
1420 	svc_authorise(rqstp);
1422 	if (rqstp->rq_xprt && test_bit(XPT_TEMP, &rqstp->rq_xprt->xpt_flags))
1423 		svc_xprt_close(rqstp->rq_xprt);
1428 	svc_printk(rqstp, "short len %u, dropping request\n",
1429 		   rqstp->rq_arg.len);
1444 		be32_to_cpu(rqstp->rq_auth_stat));
1451 	xdr_stream_encode_be32(xdr, rqstp->rq_auth_stat);
1455 	dprintk("svc: unknown program %d\n", rqstp->rq_prog);
1458 	*rqstp->rq_accept_statp = rpc_prog_unavail;
1462 	svc_printk(rqstp, "unknown version (%d for prog %d, %s)\n",
1463 		       rqstp->rq_vers, rqstp->rq_prog, progp->pg_name);
1467 	*rqstp->rq_accept_statp = rpc_prog_mismatch;
1478 	svc_printk(rqstp, "unknown procedure (%d)\n", rqstp->rq_proc);
1482 	*rqstp->rq_accept_statp = rpc_proc_unavail;
1486 	svc_printk(rqstp, "failed to decode RPC header\n");
1490 	*rqstp->rq_accept_statp = rpc_garbage_args;
1496 	*rqstp->rq_accept_statp = rpc_system_err;
1502  * @rqstp: RPC transaction context
1505 void svc_process(struct svc_rqst *rqstp)
1507 	struct kvec		*resv = &rqstp->rq_res.head[0];
1513 		svc_xprt_deferred_close(rqstp->rq_xprt);
1520 	rqstp->rq_next_page = &rqstp->rq_respages[1];
1521 	resv->iov_base = page_address(rqstp->rq_respages[0]);
1523 	rqstp->rq_res.pages = rqstp->rq_next_page;
1524 	rqstp->rq_res.len = 0;
1525 	rqstp->rq_res.page_base = 0;
1526 	rqstp->rq_res.page_len = 0;
1527 	rqstp->rq_res.buflen = PAGE_SIZE;
1528 	rqstp->rq_res.tail[0].iov_base = NULL;
1529 	rqstp->rq_res.tail[0].iov_len = 0;
1531 	svcxdr_init_decode(rqstp);
1532 	p = xdr_inline_decode(&rqstp->rq_arg_stream, XDR_UNIT * 2);
1535 	rqstp->rq_xid = *p++;
1539 	if (!svc_process_common(rqstp))
1541 	svc_send(rqstp);
1545 	svc_printk(rqstp, "bad direction 0x%08x, dropping request\n",
1547 	if (rqstp->rq_server->sv_stats)
1548 		rqstp->rq_server->sv_stats->rpcbadfmt++;
1550 	svc_drop(rqstp);
1557  * @rqstp: server-side execution context
1560 void svc_process_bc(struct rpc_rqst *req, struct svc_rqst *rqstp)
1567 	rqstp->rq_xid = req->rq_xid;
1568 	rqstp->rq_prot = req->rq_xprt->prot;
1569 	rqstp->rq_bc_net = req->rq_xprt->xprt_net;
1571 	rqstp->rq_addrlen = sizeof(req->rq_xprt->addr);
1572 	memcpy(&rqstp->rq_addr, &req->rq_xprt->addr, rqstp->rq_addrlen);
1573 	memcpy(&rqstp->rq_arg, &req->rq_rcv_buf, sizeof(rqstp->rq_arg));
1574 	memcpy(&rqstp->rq_res, &req->rq_snd_buf, sizeof(rqstp->rq_res));
1577 	rqstp->rq_arg.len = req->rq_private_buf.len;
1578 	if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len) {
1579 		rqstp->rq_arg.head[0].iov_len = rqstp->rq_arg.len;
1580 		rqstp->rq_arg.page_len = 0;
1581 	} else if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len +
1582 			rqstp->rq_arg.page_len)
1583 		rqstp->rq_arg.page_len = rqstp->rq_arg.len -
1584 			rqstp->rq_arg.head[0].iov_len;
1586 		rqstp->rq_arg.len = rqstp->rq_arg.head[0].iov_len +
1587 			rqstp->rq_arg.page_len;
1590 	rqstp->rq_res.head[0].iov_len = 0;
1596 	svcxdr_init_decode(rqstp);
1597 	if (!xdr_inline_decode(&rqstp->rq_arg_stream, XDR_UNIT * 2))
1601 	proc_error = svc_process_common(rqstp);
1610 	if (rqstp->bc_to_initval > 0) {
1611 		timeout.to_initval = rqstp->bc_to_initval;
1612 		timeout.to_retries = rqstp->bc_to_retries;
1617 	memcpy(&req->rq_snd_buf, &rqstp->rq_res, sizeof(req->rq_snd_buf));
1630  * @rqstp: RPC transaction context
1635 u32 svc_max_payload(const struct svc_rqst *rqstp)
1637 	u32 max = rqstp->rq_xprt->xpt_class->xcl_max_payload;
1639 	if (rqstp->rq_server->sv_max_payload < max)
1640 		max = rqstp->rq_server->sv_max_payload;
1647  * @rqstp: svc_rqst to operate on
1652 const char *svc_proc_name(const struct svc_rqst *rqstp)
1654 	if (rqstp && rqstp->rq_procinfo)
1655 		return rqstp->rq_procinfo->pc_name;
1662  * @rqstp: svc_rqst to operate on
1663  * @offset: payload's byte offset in rqstp->rq_res
1669 int svc_encode_result_payload(struct svc_rqst *rqstp, unsigned int offset,
1672 	return rqstp->rq_xprt->xpt_ops->xpo_result_payload(rqstp, offset,
1679  * @rqstp: svc_rqst to operate on
1682  * Fills in rqstp::rq_vec, and returns the number of elements.
1684 unsigned int svc_fill_write_vector(struct svc_rqst *rqstp,
1689 	struct kvec *vec = rqstp->rq_vec;
1712 	WARN_ON_ONCE(i > ARRAY_SIZE(rqstp->rq_vec));
1719  * @rqstp: svc_rqst to operate on
1728 char *svc_fill_symlink_pathname(struct svc_rqst *rqstp, struct kvec *first,