1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * linux/net/sunrpc/svcsock.c 4 * 5 * These are the RPC server socket internals. 6 * 7 * The server scheduling algorithm does not always distribute the load 8 * evenly when servicing a single client. May need to modify the 9 * svc_xprt_enqueue procedure... 10 * 11 * TCP support is largely untested and may be a little slow. The problem 12 * is that we currently do two separate recvfrom's, one for the 4-byte 13 * record length, and the second for the actual record. This could possibly 14 * be improved by always reading a minimum size of around 100 bytes and 15 * tucking any superfluous bytes away in a temporary store. Still, that 16 * leaves write requests out in the rain. An alternative may be to peek at 17 * the first skb in the queue, and if it matches the next TCP sequence 18 * number, to extract the record marker. Yuck. 19 * 20 * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de> 21 */ 22 23#include <linux/kernel.h> 24#include <linux/sched.h> 25#include <linux/module.h> 26#include <linux/errno.h> 27#include <linux/fcntl.h> 28#include <linux/net.h> 29#include <linux/in.h> 30#include <linux/inet.h> 31#include <linux/udp.h> 32#include <linux/tcp.h> 33#include <linux/unistd.h> 34#include <linux/slab.h> 35#include <linux/netdevice.h> 36#include <linux/skbuff.h> 37#include <linux/file.h> 38#include <linux/freezer.h> 39#include <linux/bvec.h> 40 41#include <net/sock.h> 42#include <net/checksum.h> 43#include <net/ip.h> 44#include <net/ipv6.h> 45#include <net/udp.h> 46#include <net/tcp.h> 47#include <net/tcp_states.h> 48#include <net/tls_prot.h> 49#include <net/handshake.h> 50#include <linux/uaccess.h> 51#include <linux/highmem.h> 52#include <asm/ioctls.h> 53#include <linux/key.h> 54 55#include <linux/sunrpc/types.h> 56#include <linux/sunrpc/clnt.h> 57#include <linux/sunrpc/xdr.h> 58#include <linux/sunrpc/msg_prot.h> 59#include <linux/sunrpc/svcsock.h> 60#include <linux/sunrpc/stats.h> 61#include <linux/sunrpc/xprt.h> 62 63#include <trace/events/sock.h> 64#include <trace/events/sunrpc.h> 65 66#include "socklib.h" 67#include "sunrpc.h" 68 69#define RPCDBG_FACILITY RPCDBG_SVCXPRT 70 71/* To-do: to avoid tying up an nfsd thread while waiting for a 72 * handshake request, the request could instead be deferred. 73 */ 74enum { 75 SVC_HANDSHAKE_TO = 5U * HZ 76}; 77 78static struct svc_sock *svc_setup_socket(struct svc_serv *, struct socket *, 79 int flags); 80static int svc_udp_recvfrom(struct svc_rqst *); 81static int svc_udp_sendto(struct svc_rqst *); 82static void svc_sock_detach(struct svc_xprt *); 83static void svc_tcp_sock_detach(struct svc_xprt *); 84static void svc_sock_free(struct svc_xprt *); 85 86static struct svc_xprt *svc_create_socket(struct svc_serv *, int, 87 struct net *, struct sockaddr *, 88 int, int); 89#ifdef CONFIG_DEBUG_LOCK_ALLOC 90static struct lock_class_key svc_key[2]; 91static struct lock_class_key svc_slock_key[2]; 92 93static void svc_reclassify_socket(struct socket *sock) 94{ 95 struct sock *sk = sock->sk; 96 97 if (WARN_ON_ONCE(!sock_allow_reclassification(sk))) 98 return; 99 100 switch (sk->sk_family) { 101 case AF_INET: 102 sock_lock_init_class_and_name(sk, "slock-AF_INET-NFSD", 103 &svc_slock_key[0], 104 "sk_xprt.xpt_lock-AF_INET-NFSD", 105 &svc_key[0]); 106 break; 107 108 case AF_INET6: 109 sock_lock_init_class_and_name(sk, "slock-AF_INET6-NFSD", 110 &svc_slock_key[1], 111 "sk_xprt.xpt_lock-AF_INET6-NFSD", 112 &svc_key[1]); 113 break; 114 115 default: 116 BUG(); 117 } 118} 119#else 120static void svc_reclassify_socket(struct socket *sock) 121{ 122} 123#endif 124 125/** 126 * svc_tcp_release_ctxt - Release transport-related resources 127 * @xprt: the transport which owned the context 128 * @ctxt: the context from rqstp->rq_xprt_ctxt or dr->xprt_ctxt 129 * 130 */ 131static void svc_tcp_release_ctxt(struct svc_xprt *xprt, void *ctxt) 132{ 133} 134 135/** 136 * svc_udp_release_ctxt - Release transport-related resources 137 * @xprt: the transport which owned the context 138 * @ctxt: the context from rqstp->rq_xprt_ctxt or dr->xprt_ctxt 139 * 140 */ 141static void svc_udp_release_ctxt(struct svc_xprt *xprt, void *ctxt) 142{ 143 struct sk_buff *skb = ctxt; 144 145 if (skb) 146 consume_skb(skb); 147} 148 149union svc_pktinfo_u { 150 struct in_pktinfo pkti; 151 struct in6_pktinfo pkti6; 152}; 153#define SVC_PKTINFO_SPACE \ 154 CMSG_SPACE(sizeof(union svc_pktinfo_u)) 155 156static void svc_set_cmsg_data(struct svc_rqst *rqstp, struct cmsghdr *cmh) 157{ 158 struct svc_sock *svsk = 159 container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); 160 switch (svsk->sk_sk->sk_family) { 161 case AF_INET: { 162 struct in_pktinfo *pki = CMSG_DATA(cmh); 163 164 cmh->cmsg_level = SOL_IP; 165 cmh->cmsg_type = IP_PKTINFO; 166 pki->ipi_ifindex = 0; 167 pki->ipi_spec_dst.s_addr = 168 svc_daddr_in(rqstp)->sin_addr.s_addr; 169 cmh->cmsg_len = CMSG_LEN(sizeof(*pki)); 170 } 171 break; 172 173 case AF_INET6: { 174 struct in6_pktinfo *pki = CMSG_DATA(cmh); 175 struct sockaddr_in6 *daddr = svc_daddr_in6(rqstp); 176 177 cmh->cmsg_level = SOL_IPV6; 178 cmh->cmsg_type = IPV6_PKTINFO; 179 pki->ipi6_ifindex = daddr->sin6_scope_id; 180 pki->ipi6_addr = daddr->sin6_addr; 181 cmh->cmsg_len = CMSG_LEN(sizeof(*pki)); 182 } 183 break; 184 } 185} 186 187static int svc_sock_result_payload(struct svc_rqst *rqstp, unsigned int offset, 188 unsigned int length) 189{ 190 return 0; 191} 192 193/* 194 * Report socket names for nfsdfs 195 */ 196static int svc_one_sock_name(struct svc_sock *svsk, char *buf, int remaining) 197{ 198 const struct sock *sk = svsk->sk_sk; 199 const char *proto_name = sk->sk_protocol == IPPROTO_UDP ? 200 "udp" : "tcp"; 201 int len; 202 203 switch (sk->sk_family) { 204 case PF_INET: 205 len = snprintf(buf, remaining, "ipv4 %s %pI4 %d\n", 206 proto_name, 207 &inet_sk(sk)->inet_rcv_saddr, 208 inet_sk(sk)->inet_num); 209 break; 210#if IS_ENABLED(CONFIG_IPV6) 211 case PF_INET6: 212 len = snprintf(buf, remaining, "ipv6 %s %pI6 %d\n", 213 proto_name, 214 &sk->sk_v6_rcv_saddr, 215 inet_sk(sk)->inet_num); 216 break; 217#endif 218 default: 219 len = snprintf(buf, remaining, "*unknown-%d*\n", 220 sk->sk_family); 221 } 222 223 if (len >= remaining) { 224 *buf = '\0'; 225 return -ENAMETOOLONG; 226 } 227 return len; 228} 229 230static int 231svc_tcp_sock_process_cmsg(struct socket *sock, struct msghdr *msg, 232 struct cmsghdr *cmsg, int ret) 233{ 234 u8 content_type = tls_get_record_type(sock->sk, cmsg); 235 u8 level, description; 236 237 switch (content_type) { 238 case 0: 239 break; 240 case TLS_RECORD_TYPE_DATA: 241 /* TLS sets EOR at the end of each application data 242 * record, even though there might be more frames 243 * waiting to be decrypted. 244 */ 245 msg->msg_flags &= ~MSG_EOR; 246 break; 247 case TLS_RECORD_TYPE_ALERT: 248 tls_alert_recv(sock->sk, msg, &level, &description); 249 ret = (level == TLS_ALERT_LEVEL_FATAL) ? 250 -ENOTCONN : -EAGAIN; 251 break; 252 default: 253 /* discard this record type */ 254 ret = -EAGAIN; 255 } 256 return ret; 257} 258 259static int 260svc_tcp_sock_recv_cmsg(struct svc_sock *svsk, struct msghdr *msg) 261{ 262 union { 263 struct cmsghdr cmsg; 264 u8 buf[CMSG_SPACE(sizeof(u8))]; 265 } u; 266 struct socket *sock = svsk->sk_sock; 267 int ret; 268 269 msg->msg_control = &u; 270 msg->msg_controllen = sizeof(u); 271 ret = sock_recvmsg(sock, msg, MSG_DONTWAIT); 272 if (unlikely(msg->msg_controllen != sizeof(u))) 273 ret = svc_tcp_sock_process_cmsg(sock, msg, &u.cmsg, ret); 274 return ret; 275} 276 277#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 278static void svc_flush_bvec(const struct bio_vec *bvec, size_t size, size_t seek) 279{ 280 struct bvec_iter bi = { 281 .bi_size = size + seek, 282 }; 283 struct bio_vec bv; 284 285 bvec_iter_advance(bvec, &bi, seek & PAGE_MASK); 286 for_each_bvec(bv, bvec, bi, bi) 287 flush_dcache_page(bv.bv_page); 288} 289#else 290static inline void svc_flush_bvec(const struct bio_vec *bvec, size_t size, 291 size_t seek) 292{ 293} 294#endif 295 296/* 297 * Read from @rqstp's transport socket. The incoming message fills whole 298 * pages in @rqstp's rq_pages array until the last page of the message 299 * has been received into a partial page. 300 */ 301static ssize_t svc_tcp_read_msg(struct svc_rqst *rqstp, size_t buflen, 302 size_t seek) 303{ 304 struct svc_sock *svsk = 305 container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); 306 struct bio_vec *bvec = rqstp->rq_bvec; 307 struct msghdr msg = { NULL }; 308 unsigned int i; 309 ssize_t len; 310 size_t t; 311 312 clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); 313 314 for (i = 0, t = 0; t < buflen; i++, t += PAGE_SIZE) 315 bvec_set_page(&bvec[i], rqstp->rq_pages[i], PAGE_SIZE, 0); 316 rqstp->rq_respages = &rqstp->rq_pages[i]; 317 rqstp->rq_next_page = rqstp->rq_respages + 1; 318 319 iov_iter_bvec(&msg.msg_iter, ITER_DEST, bvec, i, buflen); 320 if (seek) { 321 iov_iter_advance(&msg.msg_iter, seek); 322 buflen -= seek; 323 } 324 len = svc_tcp_sock_recv_cmsg(svsk, &msg); 325 if (len > 0) 326 svc_flush_bvec(bvec, len, seek); 327 328 /* If we read a full record, then assume there may be more 329 * data to read (stream based sockets only!) 330 */ 331 if (len == buflen) 332 set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); 333 334 return len; 335} 336 337/* 338 * Set socket snd and rcv buffer lengths 339 */ 340static void svc_sock_setbufsize(struct svc_sock *svsk, unsigned int nreqs) 341{ 342 unsigned int max_mesg = svsk->sk_xprt.xpt_server->sv_max_mesg; 343 struct socket *sock = svsk->sk_sock; 344 345 nreqs = min(nreqs, INT_MAX / 2 / max_mesg); 346 347 lock_sock(sock->sk); 348 sock->sk->sk_sndbuf = nreqs * max_mesg * 2; 349 sock->sk->sk_rcvbuf = nreqs * max_mesg * 2; 350 sock->sk->sk_write_space(sock->sk); 351 release_sock(sock->sk); 352} 353 354static void svc_sock_secure_port(struct svc_rqst *rqstp) 355{ 356 if (svc_port_is_privileged(svc_addr(rqstp))) 357 set_bit(RQ_SECURE, &rqstp->rq_flags); 358 else 359 clear_bit(RQ_SECURE, &rqstp->rq_flags); 360} 361 362/* 363 * INET callback when data has been received on the socket. 364 */ 365static void svc_data_ready(struct sock *sk) 366{ 367 struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; 368 369 trace_sk_data_ready(sk); 370 371 if (svsk) { 372 /* Refer to svc_setup_socket() for details. */ 373 rmb(); 374 svsk->sk_odata(sk); 375 trace_svcsock_data_ready(&svsk->sk_xprt, 0); 376 if (test_bit(XPT_HANDSHAKE, &svsk->sk_xprt.xpt_flags)) 377 return; 378 if (!test_and_set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags)) 379 svc_xprt_enqueue(&svsk->sk_xprt); 380 } 381} 382 383/* 384 * INET callback when space is newly available on the socket. 385 */ 386static void svc_write_space(struct sock *sk) 387{ 388 struct svc_sock *svsk = (struct svc_sock *)(sk->sk_user_data); 389 390 if (svsk) { 391 /* Refer to svc_setup_socket() for details. */ 392 rmb(); 393 trace_svcsock_write_space(&svsk->sk_xprt, 0); 394 svsk->sk_owspace(sk); 395 svc_xprt_enqueue(&svsk->sk_xprt); 396 } 397} 398 399static int svc_tcp_has_wspace(struct svc_xprt *xprt) 400{ 401 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); 402 403 if (test_bit(XPT_LISTENER, &xprt->xpt_flags)) 404 return 1; 405 return !test_bit(SOCK_NOSPACE, &svsk->sk_sock->flags); 406} 407 408static void svc_tcp_kill_temp_xprt(struct svc_xprt *xprt) 409{ 410 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); 411 412 sock_no_linger(svsk->sk_sock->sk); 413} 414 415/** 416 * svc_tcp_handshake_done - Handshake completion handler 417 * @data: address of xprt to wake 418 * @status: status of handshake 419 * @peerid: serial number of key containing the remote peer's identity 420 * 421 * If a security policy is specified as an export option, we don't 422 * have a specific export here to check. So we set a "TLS session 423 * is present" flag on the xprt and let an upper layer enforce local 424 * security policy. 425 */ 426static void svc_tcp_handshake_done(void *data, int status, key_serial_t peerid) 427{ 428 struct svc_xprt *xprt = data; 429 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); 430 431 if (!status) { 432 if (peerid != TLS_NO_PEERID) 433 set_bit(XPT_PEER_AUTH, &xprt->xpt_flags); 434 set_bit(XPT_TLS_SESSION, &xprt->xpt_flags); 435 } 436 clear_bit(XPT_HANDSHAKE, &xprt->xpt_flags); 437 complete_all(&svsk->sk_handshake_done); 438} 439 440/** 441 * svc_tcp_handshake - Perform a transport-layer security handshake 442 * @xprt: connected transport endpoint 443 * 444 */ 445static void svc_tcp_handshake(struct svc_xprt *xprt) 446{ 447 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); 448 struct sock *sk = svsk->sk_sock->sk; 449 struct tls_handshake_args args = { 450 .ta_sock = svsk->sk_sock, 451 .ta_done = svc_tcp_handshake_done, 452 .ta_data = xprt, 453 }; 454 int ret; 455 456 trace_svc_tls_upcall(xprt); 457 458 clear_bit(XPT_TLS_SESSION, &xprt->xpt_flags); 459 init_completion(&svsk->sk_handshake_done); 460 461 ret = tls_server_hello_x509(&args, GFP_KERNEL); 462 if (ret) { 463 trace_svc_tls_not_started(xprt); 464 goto out_failed; 465 } 466 467 ret = wait_for_completion_interruptible_timeout(&svsk->sk_handshake_done, 468 SVC_HANDSHAKE_TO); 469 if (ret <= 0) { 470 if (tls_handshake_cancel(sk)) { 471 trace_svc_tls_timed_out(xprt); 472 goto out_close; 473 } 474 } 475 476 if (!test_bit(XPT_TLS_SESSION, &xprt->xpt_flags)) { 477 trace_svc_tls_unavailable(xprt); 478 goto out_close; 479 } 480 481 /* Mark the transport ready in case the remote sent RPC 482 * traffic before the kernel received the handshake 483 * completion downcall. 484 */ 485 set_bit(XPT_DATA, &xprt->xpt_flags); 486 svc_xprt_enqueue(xprt); 487 return; 488 489out_close: 490 set_bit(XPT_CLOSE, &xprt->xpt_flags); 491out_failed: 492 clear_bit(XPT_HANDSHAKE, &xprt->xpt_flags); 493 set_bit(XPT_DATA, &xprt->xpt_flags); 494 svc_xprt_enqueue(xprt); 495} 496 497/* 498 * See net/ipv6/ip_sockglue.c : ip_cmsg_recv_pktinfo 499 */ 500static int svc_udp_get_dest_address4(struct svc_rqst *rqstp, 501 struct cmsghdr *cmh) 502{ 503 struct in_pktinfo *pki = CMSG_DATA(cmh); 504 struct sockaddr_in *daddr = svc_daddr_in(rqstp); 505 506 if (cmh->cmsg_type != IP_PKTINFO) 507 return 0; 508 509 daddr->sin_family = AF_INET; 510 daddr->sin_addr.s_addr = pki->ipi_spec_dst.s_addr; 511 return 1; 512} 513 514/* 515 * See net/ipv6/datagram.c : ip6_datagram_recv_ctl 516 */ 517static int svc_udp_get_dest_address6(struct svc_rqst *rqstp, 518 struct cmsghdr *cmh) 519{ 520 struct in6_pktinfo *pki = CMSG_DATA(cmh); 521 struct sockaddr_in6 *daddr = svc_daddr_in6(rqstp); 522 523 if (cmh->cmsg_type != IPV6_PKTINFO) 524 return 0; 525 526 daddr->sin6_family = AF_INET6; 527 daddr->sin6_addr = pki->ipi6_addr; 528 daddr->sin6_scope_id = pki->ipi6_ifindex; 529 return 1; 530} 531 532/* 533 * Copy the UDP datagram's destination address to the rqstp structure. 534 * The 'destination' address in this case is the address to which the 535 * peer sent the datagram, i.e. our local address. For multihomed 536 * hosts, this can change from msg to msg. Note that only the IP 537 * address changes, the port number should remain the same. 538 */ 539static int svc_udp_get_dest_address(struct svc_rqst *rqstp, 540 struct cmsghdr *cmh) 541{ 542 switch (cmh->cmsg_level) { 543 case SOL_IP: 544 return svc_udp_get_dest_address4(rqstp, cmh); 545 case SOL_IPV6: 546 return svc_udp_get_dest_address6(rqstp, cmh); 547 } 548 549 return 0; 550} 551 552/** 553 * svc_udp_recvfrom - Receive a datagram from a UDP socket. 554 * @rqstp: request structure into which to receive an RPC Call 555 * 556 * Called in a loop when XPT_DATA has been set. 557 * 558 * Returns: 559 * On success, the number of bytes in a received RPC Call, or 560 * %0 if a complete RPC Call message was not ready to return 561 */ 562static int svc_udp_recvfrom(struct svc_rqst *rqstp) 563{ 564 struct svc_sock *svsk = 565 container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); 566 struct svc_serv *serv = svsk->sk_xprt.xpt_server; 567 struct sk_buff *skb; 568 union { 569 struct cmsghdr hdr; 570 long all[SVC_PKTINFO_SPACE / sizeof(long)]; 571 } buffer; 572 struct cmsghdr *cmh = &buffer.hdr; 573 struct msghdr msg = { 574 .msg_name = svc_addr(rqstp), 575 .msg_control = cmh, 576 .msg_controllen = sizeof(buffer), 577 .msg_flags = MSG_DONTWAIT, 578 }; 579 size_t len; 580 int err; 581 582 if (test_and_clear_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags)) 583 /* udp sockets need large rcvbuf as all pending 584 * requests are still in that buffer. sndbuf must 585 * also be large enough that there is enough space 586 * for one reply per thread. We count all threads 587 * rather than threads in a particular pool, which 588 * provides an upper bound on the number of threads 589 * which will access the socket. 590 */ 591 svc_sock_setbufsize(svsk, serv->sv_nrthreads + 3); 592 593 clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); 594 err = kernel_recvmsg(svsk->sk_sock, &msg, NULL, 595 0, 0, MSG_PEEK | MSG_DONTWAIT); 596 if (err < 0) 597 goto out_recv_err; 598 skb = skb_recv_udp(svsk->sk_sk, MSG_DONTWAIT, &err); 599 if (!skb) 600 goto out_recv_err; 601 602 len = svc_addr_len(svc_addr(rqstp)); 603 rqstp->rq_addrlen = len; 604 if (skb->tstamp == 0) { 605 skb->tstamp = ktime_get_real(); 606 /* Don't enable netstamp, sunrpc doesn't 607 need that much accuracy */ 608 } 609 sock_write_timestamp(svsk->sk_sk, skb->tstamp); 610 set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); /* there may be more data... */ 611 612 len = skb->len; 613 rqstp->rq_arg.len = len; 614 trace_svcsock_udp_recv(&svsk->sk_xprt, len); 615 616 rqstp->rq_prot = IPPROTO_UDP; 617 618 if (!svc_udp_get_dest_address(rqstp, cmh)) 619 goto out_cmsg_err; 620 rqstp->rq_daddrlen = svc_addr_len(svc_daddr(rqstp)); 621 622 if (skb_is_nonlinear(skb)) { 623 /* we have to copy */ 624 local_bh_disable(); 625 if (csum_partial_copy_to_xdr(&rqstp->rq_arg, skb)) 626 goto out_bh_enable; 627 local_bh_enable(); 628 consume_skb(skb); 629 } else { 630 /* we can use it in-place */ 631 rqstp->rq_arg.head[0].iov_base = skb->data; 632 rqstp->rq_arg.head[0].iov_len = len; 633 if (skb_checksum_complete(skb)) 634 goto out_free; 635 rqstp->rq_xprt_ctxt = skb; 636 } 637 638 rqstp->rq_arg.page_base = 0; 639 if (len <= rqstp->rq_arg.head[0].iov_len) { 640 rqstp->rq_arg.head[0].iov_len = len; 641 rqstp->rq_arg.page_len = 0; 642 rqstp->rq_respages = rqstp->rq_pages+1; 643 } else { 644 rqstp->rq_arg.page_len = len - rqstp->rq_arg.head[0].iov_len; 645 rqstp->rq_respages = rqstp->rq_pages + 1 + 646 DIV_ROUND_UP(rqstp->rq_arg.page_len, PAGE_SIZE); 647 } 648 rqstp->rq_next_page = rqstp->rq_respages+1; 649 650 if (serv->sv_stats) 651 serv->sv_stats->netudpcnt++; 652 653 svc_sock_secure_port(rqstp); 654 svc_xprt_received(rqstp->rq_xprt); 655 return len; 656 657out_recv_err: 658 if (err != -EAGAIN) { 659 /* possibly an icmp error */ 660 set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); 661 } 662 trace_svcsock_udp_recv_err(&svsk->sk_xprt, err); 663 goto out_clear_busy; 664out_cmsg_err: 665 net_warn_ratelimited("svc: received unknown control message %d/%d; dropping RPC reply datagram\n", 666 cmh->cmsg_level, cmh->cmsg_type); 667 goto out_free; 668out_bh_enable: 669 local_bh_enable(); 670out_free: 671 kfree_skb(skb); 672out_clear_busy: 673 svc_xprt_received(rqstp->rq_xprt); 674 return 0; 675} 676 677/** 678 * svc_udp_sendto - Send out a reply on a UDP socket 679 * @rqstp: completed svc_rqst 680 * 681 * xpt_mutex ensures @rqstp's whole message is written to the socket 682 * without interruption. 683 * 684 * Returns the number of bytes sent, or a negative errno. 685 */ 686static int svc_udp_sendto(struct svc_rqst *rqstp) 687{ 688 struct svc_xprt *xprt = rqstp->rq_xprt; 689 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); 690 struct xdr_buf *xdr = &rqstp->rq_res; 691 union { 692 struct cmsghdr hdr; 693 long all[SVC_PKTINFO_SPACE / sizeof(long)]; 694 } buffer; 695 struct cmsghdr *cmh = &buffer.hdr; 696 struct msghdr msg = { 697 .msg_name = &rqstp->rq_addr, 698 .msg_namelen = rqstp->rq_addrlen, 699 .msg_control = cmh, 700 .msg_flags = MSG_SPLICE_PAGES, 701 .msg_controllen = sizeof(buffer), 702 }; 703 unsigned int count; 704 int err; 705 706 svc_udp_release_ctxt(xprt, rqstp->rq_xprt_ctxt); 707 rqstp->rq_xprt_ctxt = NULL; 708 709 svc_set_cmsg_data(rqstp, cmh); 710 711 mutex_lock(&xprt->xpt_mutex); 712 713 if (svc_xprt_is_dead(xprt)) 714 goto out_notconn; 715 716 count = xdr_buf_to_bvec(rqstp->rq_bvec, 717 ARRAY_SIZE(rqstp->rq_bvec), xdr); 718 719 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, rqstp->rq_bvec, 720 count, rqstp->rq_res.len); 721 err = sock_sendmsg(svsk->sk_sock, &msg); 722 if (err == -ECONNREFUSED) { 723 /* ICMP error on earlier request. */ 724 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, rqstp->rq_bvec, 725 count, rqstp->rq_res.len); 726 err = sock_sendmsg(svsk->sk_sock, &msg); 727 } 728 729 trace_svcsock_udp_send(xprt, err); 730 731 mutex_unlock(&xprt->xpt_mutex); 732 return err; 733 734out_notconn: 735 mutex_unlock(&xprt->xpt_mutex); 736 return -ENOTCONN; 737} 738 739static int svc_udp_has_wspace(struct svc_xprt *xprt) 740{ 741 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); 742 struct svc_serv *serv = xprt->xpt_server; 743 unsigned long required; 744 745 /* 746 * Set the SOCK_NOSPACE flag before checking the available 747 * sock space. 748 */ 749 set_bit(SOCK_NOSPACE, &svsk->sk_sock->flags); 750 required = atomic_read(&svsk->sk_xprt.xpt_reserved) + serv->sv_max_mesg; 751 if (required*2 > sock_wspace(svsk->sk_sk)) 752 return 0; 753 clear_bit(SOCK_NOSPACE, &svsk->sk_sock->flags); 754 return 1; 755} 756 757static struct svc_xprt *svc_udp_accept(struct svc_xprt *xprt) 758{ 759 BUG(); 760 return NULL; 761} 762 763static void svc_udp_kill_temp_xprt(struct svc_xprt *xprt) 764{ 765} 766 767static struct svc_xprt *svc_udp_create(struct svc_serv *serv, 768 struct net *net, 769 struct sockaddr *sa, int salen, 770 int flags) 771{ 772 return svc_create_socket(serv, IPPROTO_UDP, net, sa, salen, flags); 773} 774 775static const struct svc_xprt_ops svc_udp_ops = { 776 .xpo_create = svc_udp_create, 777 .xpo_recvfrom = svc_udp_recvfrom, 778 .xpo_sendto = svc_udp_sendto, 779 .xpo_result_payload = svc_sock_result_payload, 780 .xpo_release_ctxt = svc_udp_release_ctxt, 781 .xpo_detach = svc_sock_detach, 782 .xpo_free = svc_sock_free, 783 .xpo_has_wspace = svc_udp_has_wspace, 784 .xpo_accept = svc_udp_accept, 785 .xpo_kill_temp_xprt = svc_udp_kill_temp_xprt, 786}; 787 788static struct svc_xprt_class svc_udp_class = { 789 .xcl_name = "udp", 790 .xcl_owner = THIS_MODULE, 791 .xcl_ops = &svc_udp_ops, 792 .xcl_max_payload = RPCSVC_MAXPAYLOAD_UDP, 793 .xcl_ident = XPRT_TRANSPORT_UDP, 794}; 795 796static void svc_udp_init(struct svc_sock *svsk, struct svc_serv *serv) 797{ 798 svc_xprt_init(sock_net(svsk->sk_sock->sk), &svc_udp_class, 799 &svsk->sk_xprt, serv); 800 clear_bit(XPT_CACHE_AUTH, &svsk->sk_xprt.xpt_flags); 801 svsk->sk_sk->sk_data_ready = svc_data_ready; 802 svsk->sk_sk->sk_write_space = svc_write_space; 803 804 /* initialise setting must have enough space to 805 * receive and respond to one request. 806 * svc_udp_recvfrom will re-adjust if necessary 807 */ 808 svc_sock_setbufsize(svsk, 3); 809 810 /* data might have come in before data_ready set up */ 811 set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); 812 set_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags); 813 814 /* make sure we get destination address info */ 815 switch (svsk->sk_sk->sk_family) { 816 case AF_INET: 817 ip_sock_set_pktinfo(svsk->sk_sock->sk); 818 break; 819 case AF_INET6: 820 ip6_sock_set_recvpktinfo(svsk->sk_sock->sk); 821 break; 822 default: 823 BUG(); 824 } 825} 826 827/* 828 * A data_ready event on a listening socket means there's a connection 829 * pending. Do not use state_change as a substitute for it. 830 */ 831static void svc_tcp_listen_data_ready(struct sock *sk) 832{ 833 struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; 834 835 trace_sk_data_ready(sk); 836 837 /* 838 * This callback may called twice when a new connection 839 * is established as a child socket inherits everything 840 * from a parent LISTEN socket. 841 * 1) data_ready method of the parent socket will be called 842 * when one of child sockets become ESTABLISHED. 843 * 2) data_ready method of the child socket may be called 844 * when it receives data before the socket is accepted. 845 * In case of 2, we should ignore it silently and DO NOT 846 * dereference svsk. 847 */ 848 if (sk->sk_state != TCP_LISTEN) 849 return; 850 851 if (svsk) { 852 /* Refer to svc_setup_socket() for details. */ 853 rmb(); 854 svsk->sk_odata(sk); 855 set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags); 856 svc_xprt_enqueue(&svsk->sk_xprt); 857 } 858} 859 860/* 861 * A state change on a connected socket means it's dying or dead. 862 */ 863static void svc_tcp_state_change(struct sock *sk) 864{ 865 struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; 866 867 if (svsk) { 868 /* Refer to svc_setup_socket() for details. */ 869 rmb(); 870 svsk->sk_ostate(sk); 871 trace_svcsock_tcp_state(&svsk->sk_xprt, svsk->sk_sock); 872 if (sk->sk_state != TCP_ESTABLISHED) 873 svc_xprt_deferred_close(&svsk->sk_xprt); 874 } 875} 876 877/* 878 * Accept a TCP connection 879 */ 880static struct svc_xprt *svc_tcp_accept(struct svc_xprt *xprt) 881{ 882 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); 883 struct sockaddr_storage addr; 884 struct sockaddr *sin = (struct sockaddr *) &addr; 885 struct svc_serv *serv = svsk->sk_xprt.xpt_server; 886 struct socket *sock = svsk->sk_sock; 887 struct socket *newsock; 888 struct svc_sock *newsvsk; 889 int err, slen; 890 891 if (!sock) 892 return NULL; 893 894 clear_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags); 895 err = kernel_accept(sock, &newsock, O_NONBLOCK); 896 if (err < 0) { 897 if (err != -EAGAIN) 898 trace_svcsock_accept_err(xprt, serv->sv_name, err); 899 return NULL; 900 } 901 if (IS_ERR(sock_alloc_file(newsock, O_NONBLOCK, NULL))) 902 return NULL; 903 904 set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags); 905 906 err = kernel_getpeername(newsock, sin); 907 if (err < 0) { 908 trace_svcsock_getpeername_err(xprt, serv->sv_name, err); 909 goto failed; /* aborted connection or whatever */ 910 } 911 slen = err; 912 913 /* Reset the inherited callbacks before calling svc_setup_socket */ 914 newsock->sk->sk_state_change = svsk->sk_ostate; 915 newsock->sk->sk_data_ready = svsk->sk_odata; 916 newsock->sk->sk_write_space = svsk->sk_owspace; 917 918 /* make sure that a write doesn't block forever when 919 * low on memory 920 */ 921 newsock->sk->sk_sndtimeo = HZ*30; 922 923 newsvsk = svc_setup_socket(serv, newsock, 924 (SVC_SOCK_ANONYMOUS | SVC_SOCK_TEMPORARY)); 925 if (IS_ERR(newsvsk)) 926 goto failed; 927 svc_xprt_set_remote(&newsvsk->sk_xprt, sin, slen); 928 err = kernel_getsockname(newsock, sin); 929 slen = err; 930 if (unlikely(err < 0)) 931 slen = offsetof(struct sockaddr, sa_data); 932 svc_xprt_set_local(&newsvsk->sk_xprt, sin, slen); 933 934 if (sock_is_loopback(newsock->sk)) 935 set_bit(XPT_LOCAL, &newsvsk->sk_xprt.xpt_flags); 936 else 937 clear_bit(XPT_LOCAL, &newsvsk->sk_xprt.xpt_flags); 938 if (serv->sv_stats) 939 serv->sv_stats->nettcpconn++; 940 941 return &newsvsk->sk_xprt; 942 943failed: 944 sockfd_put(newsock); 945 return NULL; 946} 947 948static size_t svc_tcp_restore_pages(struct svc_sock *svsk, 949 struct svc_rqst *rqstp) 950{ 951 size_t len = svsk->sk_datalen; 952 unsigned int i, npages; 953 954 if (!len) 955 return 0; 956 npages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; 957 for (i = 0; i < npages; i++) { 958 if (rqstp->rq_pages[i] != NULL) 959 put_page(rqstp->rq_pages[i]); 960 BUG_ON(svsk->sk_pages[i] == NULL); 961 rqstp->rq_pages[i] = svsk->sk_pages[i]; 962 svsk->sk_pages[i] = NULL; 963 } 964 rqstp->rq_arg.head[0].iov_base = page_address(rqstp->rq_pages[0]); 965 return len; 966} 967 968static void svc_tcp_save_pages(struct svc_sock *svsk, struct svc_rqst *rqstp) 969{ 970 unsigned int i, len, npages; 971 972 if (svsk->sk_datalen == 0) 973 return; 974 len = svsk->sk_datalen; 975 npages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; 976 for (i = 0; i < npages; i++) { 977 svsk->sk_pages[i] = rqstp->rq_pages[i]; 978 rqstp->rq_pages[i] = NULL; 979 } 980} 981 982static void svc_tcp_clear_pages(struct svc_sock *svsk) 983{ 984 unsigned int i, len, npages; 985 986 if (svsk->sk_datalen == 0) 987 goto out; 988 len = svsk->sk_datalen; 989 npages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; 990 for (i = 0; i < npages; i++) { 991 if (svsk->sk_pages[i] == NULL) { 992 WARN_ON_ONCE(1); 993 continue; 994 } 995 put_page(svsk->sk_pages[i]); 996 svsk->sk_pages[i] = NULL; 997 } 998out: 999 svsk->sk_tcplen = 0; 1000 svsk->sk_datalen = 0; 1001} 1002 1003/* 1004 * Receive fragment record header into sk_marker. 1005 */ 1006static ssize_t svc_tcp_read_marker(struct svc_sock *svsk, 1007 struct svc_rqst *rqstp) 1008{ 1009 ssize_t want, len; 1010 1011 /* If we haven't gotten the record length yet, 1012 * get the next four bytes. 1013 */ 1014 if (svsk->sk_tcplen < sizeof(rpc_fraghdr)) { 1015 struct msghdr msg = { NULL }; 1016 struct kvec iov; 1017 1018 want = sizeof(rpc_fraghdr) - svsk->sk_tcplen; 1019 iov.iov_base = ((char *)&svsk->sk_marker) + svsk->sk_tcplen; 1020 iov.iov_len = want; 1021 iov_iter_kvec(&msg.msg_iter, ITER_DEST, &iov, 1, want); 1022 len = svc_tcp_sock_recv_cmsg(svsk, &msg); 1023 if (len < 0) 1024 return len; 1025 svsk->sk_tcplen += len; 1026 if (len < want) { 1027 /* call again to read the remaining bytes */ 1028 goto err_short; 1029 } 1030 trace_svcsock_marker(&svsk->sk_xprt, svsk->sk_marker); 1031 if (svc_sock_reclen(svsk) + svsk->sk_datalen > 1032 svsk->sk_xprt.xpt_server->sv_max_mesg) 1033 goto err_too_large; 1034 } 1035 return svc_sock_reclen(svsk); 1036 1037err_too_large: 1038 net_notice_ratelimited("svc: %s %s RPC fragment too large: %d\n", 1039 __func__, svsk->sk_xprt.xpt_server->sv_name, 1040 svc_sock_reclen(svsk)); 1041 svc_xprt_deferred_close(&svsk->sk_xprt); 1042err_short: 1043 return -EAGAIN; 1044} 1045 1046static int receive_cb_reply(struct svc_sock *svsk, struct svc_rqst *rqstp) 1047{ 1048 struct rpc_xprt *bc_xprt = svsk->sk_xprt.xpt_bc_xprt; 1049 struct rpc_rqst *req = NULL; 1050 struct kvec *src, *dst; 1051 __be32 *p = (__be32 *)rqstp->rq_arg.head[0].iov_base; 1052 __be32 xid = *p; 1053 1054 if (!bc_xprt) 1055 return -EAGAIN; 1056 spin_lock(&bc_xprt->queue_lock); 1057 req = xprt_lookup_rqst(bc_xprt, xid); 1058 if (!req) 1059 goto unlock_eagain; 1060 1061 memcpy(&req->rq_private_buf, &req->rq_rcv_buf, sizeof(struct xdr_buf)); 1062 /* 1063 * XXX!: cheating for now! Only copying HEAD. 1064 * But we know this is good enough for now (in fact, for any 1065 * callback reply in the forseeable future). 1066 */ 1067 dst = &req->rq_private_buf.head[0]; 1068 src = &rqstp->rq_arg.head[0]; 1069 if (dst->iov_len < src->iov_len) 1070 goto unlock_eagain; /* whatever; just giving up. */ 1071 memcpy(dst->iov_base, src->iov_base, src->iov_len); 1072 xprt_complete_rqst(req->rq_task, rqstp->rq_arg.len); 1073 rqstp->rq_arg.len = 0; 1074 spin_unlock(&bc_xprt->queue_lock); 1075 return 0; 1076unlock_eagain: 1077 spin_unlock(&bc_xprt->queue_lock); 1078 return -EAGAIN; 1079} 1080 1081static void svc_tcp_fragment_received(struct svc_sock *svsk) 1082{ 1083 /* If we have more data, signal svc_xprt_enqueue() to try again */ 1084 svsk->sk_tcplen = 0; 1085 svsk->sk_marker = xdr_zero; 1086 1087 smp_wmb(); 1088 tcp_set_rcvlowat(svsk->sk_sk, 1); 1089} 1090 1091/** 1092 * svc_tcp_recvfrom - Receive data from a TCP socket 1093 * @rqstp: request structure into which to receive an RPC Call 1094 * 1095 * Called in a loop when XPT_DATA has been set. 1096 * 1097 * Read the 4-byte stream record marker, then use the record length 1098 * in that marker to set up exactly the resources needed to receive 1099 * the next RPC message into @rqstp. 1100 * 1101 * Returns: 1102 * On success, the number of bytes in a received RPC Call, or 1103 * %0 if a complete RPC Call message was not ready to return 1104 * 1105 * The zero return case handles partial receives and callback Replies. 1106 * The state of a partial receive is preserved in the svc_sock for 1107 * the next call to svc_tcp_recvfrom. 1108 */ 1109static int svc_tcp_recvfrom(struct svc_rqst *rqstp) 1110{ 1111 struct svc_sock *svsk = 1112 container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); 1113 struct svc_serv *serv = svsk->sk_xprt.xpt_server; 1114 size_t want, base; 1115 ssize_t len; 1116 __be32 *p; 1117 __be32 calldir; 1118 1119 clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); 1120 len = svc_tcp_read_marker(svsk, rqstp); 1121 if (len < 0) 1122 goto error; 1123 1124 base = svc_tcp_restore_pages(svsk, rqstp); 1125 want = len - (svsk->sk_tcplen - sizeof(rpc_fraghdr)); 1126 len = svc_tcp_read_msg(rqstp, base + want, base); 1127 if (len >= 0) { 1128 trace_svcsock_tcp_recv(&svsk->sk_xprt, len); 1129 svsk->sk_tcplen += len; 1130 svsk->sk_datalen += len; 1131 } 1132 if (len != want || !svc_sock_final_rec(svsk)) 1133 goto err_incomplete; 1134 if (svsk->sk_datalen < 8) 1135 goto err_nuts; 1136 1137 rqstp->rq_arg.len = svsk->sk_datalen; 1138 rqstp->rq_arg.page_base = 0; 1139 if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len) { 1140 rqstp->rq_arg.head[0].iov_len = rqstp->rq_arg.len; 1141 rqstp->rq_arg.page_len = 0; 1142 } else 1143 rqstp->rq_arg.page_len = rqstp->rq_arg.len - rqstp->rq_arg.head[0].iov_len; 1144 1145 rqstp->rq_xprt_ctxt = NULL; 1146 rqstp->rq_prot = IPPROTO_TCP; 1147 if (test_bit(XPT_LOCAL, &svsk->sk_xprt.xpt_flags)) 1148 set_bit(RQ_LOCAL, &rqstp->rq_flags); 1149 else 1150 clear_bit(RQ_LOCAL, &rqstp->rq_flags); 1151 1152 p = (__be32 *)rqstp->rq_arg.head[0].iov_base; 1153 calldir = p[1]; 1154 if (calldir) 1155 len = receive_cb_reply(svsk, rqstp); 1156 1157 /* Reset TCP read info */ 1158 svsk->sk_datalen = 0; 1159 svc_tcp_fragment_received(svsk); 1160 1161 if (len < 0) 1162 goto error; 1163 1164 svc_xprt_copy_addrs(rqstp, &svsk->sk_xprt); 1165 if (serv->sv_stats) 1166 serv->sv_stats->nettcpcnt++; 1167 1168 svc_sock_secure_port(rqstp); 1169 svc_xprt_received(rqstp->rq_xprt); 1170 return rqstp->rq_arg.len; 1171 1172err_incomplete: 1173 svc_tcp_save_pages(svsk, rqstp); 1174 if (len < 0 && len != -EAGAIN) 1175 goto err_delete; 1176 if (len == want) 1177 svc_tcp_fragment_received(svsk); 1178 else { 1179 /* Avoid more ->sk_data_ready() calls until the rest 1180 * of the message has arrived. This reduces service 1181 * thread wake-ups on large incoming messages. */ 1182 tcp_set_rcvlowat(svsk->sk_sk, 1183 svc_sock_reclen(svsk) - svsk->sk_tcplen); 1184 1185 trace_svcsock_tcp_recv_short(&svsk->sk_xprt, 1186 svc_sock_reclen(svsk), 1187 svsk->sk_tcplen - sizeof(rpc_fraghdr)); 1188 } 1189 goto err_noclose; 1190error: 1191 if (len != -EAGAIN) 1192 goto err_delete; 1193 trace_svcsock_tcp_recv_eagain(&svsk->sk_xprt, 0); 1194 goto err_noclose; 1195err_nuts: 1196 svsk->sk_datalen = 0; 1197err_delete: 1198 trace_svcsock_tcp_recv_err(&svsk->sk_xprt, len); 1199 svc_xprt_deferred_close(&svsk->sk_xprt); 1200err_noclose: 1201 svc_xprt_received(rqstp->rq_xprt); 1202 return 0; /* record not complete */ 1203} 1204 1205/* 1206 * MSG_SPLICE_PAGES is used exclusively to reduce the number of 1207 * copy operations in this path. Therefore the caller must ensure 1208 * that the pages backing @xdr are unchanging. 1209 */ 1210static int svc_tcp_sendmsg(struct svc_sock *svsk, struct svc_rqst *rqstp, 1211 rpc_fraghdr marker, unsigned int *sentp) 1212{ 1213 struct msghdr msg = { 1214 .msg_flags = MSG_SPLICE_PAGES, 1215 }; 1216 unsigned int count; 1217 void *buf; 1218 int ret; 1219 1220 *sentp = 0; 1221 1222 /* The stream record marker is copied into a temporary page 1223 * fragment buffer so that it can be included in rq_bvec. 1224 */ 1225 buf = page_frag_alloc(&svsk->sk_frag_cache, sizeof(marker), 1226 GFP_KERNEL); 1227 if (!buf) 1228 return -ENOMEM; 1229 memcpy(buf, &marker, sizeof(marker)); 1230 bvec_set_virt(rqstp->rq_bvec, buf, sizeof(marker)); 1231 1232 count = xdr_buf_to_bvec(rqstp->rq_bvec + 1, 1233 ARRAY_SIZE(rqstp->rq_bvec) - 1, &rqstp->rq_res); 1234 1235 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, rqstp->rq_bvec, 1236 1 + count, sizeof(marker) + rqstp->rq_res.len); 1237 ret = sock_sendmsg(svsk->sk_sock, &msg); 1238 page_frag_free(buf); 1239 if (ret < 0) 1240 return ret; 1241 *sentp += ret; 1242 return 0; 1243} 1244 1245/** 1246 * svc_tcp_sendto - Send out a reply on a TCP socket 1247 * @rqstp: completed svc_rqst 1248 * 1249 * xpt_mutex ensures @rqstp's whole message is written to the socket 1250 * without interruption. 1251 * 1252 * Returns the number of bytes sent, or a negative errno. 1253 */ 1254static int svc_tcp_sendto(struct svc_rqst *rqstp) 1255{ 1256 struct svc_xprt *xprt = rqstp->rq_xprt; 1257 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); 1258 struct xdr_buf *xdr = &rqstp->rq_res; 1259 rpc_fraghdr marker = cpu_to_be32(RPC_LAST_STREAM_FRAGMENT | 1260 (u32)xdr->len); 1261 unsigned int sent; 1262 int err; 1263 1264 svc_tcp_release_ctxt(xprt, rqstp->rq_xprt_ctxt); 1265 rqstp->rq_xprt_ctxt = NULL; 1266 1267 mutex_lock(&xprt->xpt_mutex); 1268 if (svc_xprt_is_dead(xprt)) 1269 goto out_notconn; 1270 err = svc_tcp_sendmsg(svsk, rqstp, marker, &sent); 1271 trace_svcsock_tcp_send(xprt, err < 0 ? (long)err : sent); 1272 if (err < 0 || sent != (xdr->len + sizeof(marker))) 1273 goto out_close; 1274 mutex_unlock(&xprt->xpt_mutex); 1275 return sent; 1276 1277out_notconn: 1278 mutex_unlock(&xprt->xpt_mutex); 1279 return -ENOTCONN; 1280out_close: 1281 pr_notice("rpc-srv/tcp: %s: %s %d when sending %d bytes - shutting down socket\n", 1282 xprt->xpt_server->sv_name, 1283 (err < 0) ? "got error" : "sent", 1284 (err < 0) ? err : sent, xdr->len); 1285 svc_xprt_deferred_close(xprt); 1286 mutex_unlock(&xprt->xpt_mutex); 1287 return -EAGAIN; 1288} 1289 1290static struct svc_xprt *svc_tcp_create(struct svc_serv *serv, 1291 struct net *net, 1292 struct sockaddr *sa, int salen, 1293 int flags) 1294{ 1295 return svc_create_socket(serv, IPPROTO_TCP, net, sa, salen, flags); 1296} 1297 1298static const struct svc_xprt_ops svc_tcp_ops = { 1299 .xpo_create = svc_tcp_create, 1300 .xpo_recvfrom = svc_tcp_recvfrom, 1301 .xpo_sendto = svc_tcp_sendto, 1302 .xpo_result_payload = svc_sock_result_payload, 1303 .xpo_release_ctxt = svc_tcp_release_ctxt, 1304 .xpo_detach = svc_tcp_sock_detach, 1305 .xpo_free = svc_sock_free, 1306 .xpo_has_wspace = svc_tcp_has_wspace, 1307 .xpo_accept = svc_tcp_accept, 1308 .xpo_kill_temp_xprt = svc_tcp_kill_temp_xprt, 1309 .xpo_handshake = svc_tcp_handshake, 1310}; 1311 1312static struct svc_xprt_class svc_tcp_class = { 1313 .xcl_name = "tcp", 1314 .xcl_owner = THIS_MODULE, 1315 .xcl_ops = &svc_tcp_ops, 1316 .xcl_max_payload = RPCSVC_MAXPAYLOAD_TCP, 1317 .xcl_ident = XPRT_TRANSPORT_TCP, 1318}; 1319 1320void svc_init_xprt_sock(void) 1321{ 1322 svc_reg_xprt_class(&svc_tcp_class); 1323 svc_reg_xprt_class(&svc_udp_class); 1324} 1325 1326void svc_cleanup_xprt_sock(void) 1327{ 1328 svc_unreg_xprt_class(&svc_tcp_class); 1329 svc_unreg_xprt_class(&svc_udp_class); 1330} 1331 1332static void svc_tcp_init(struct svc_sock *svsk, struct svc_serv *serv) 1333{ 1334 struct sock *sk = svsk->sk_sk; 1335 1336 svc_xprt_init(sock_net(svsk->sk_sock->sk), &svc_tcp_class, 1337 &svsk->sk_xprt, serv); 1338 set_bit(XPT_CACHE_AUTH, &svsk->sk_xprt.xpt_flags); 1339 set_bit(XPT_CONG_CTRL, &svsk->sk_xprt.xpt_flags); 1340 if (sk->sk_state == TCP_LISTEN) { 1341 strcpy(svsk->sk_xprt.xpt_remotebuf, "listener"); 1342 set_bit(XPT_LISTENER, &svsk->sk_xprt.xpt_flags); 1343 sk->sk_data_ready = svc_tcp_listen_data_ready; 1344 set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags); 1345 } else { 1346 sk->sk_state_change = svc_tcp_state_change; 1347 sk->sk_data_ready = svc_data_ready; 1348 sk->sk_write_space = svc_write_space; 1349 1350 svsk->sk_marker = xdr_zero; 1351 svsk->sk_tcplen = 0; 1352 svsk->sk_datalen = 0; 1353 memset(&svsk->sk_pages[0], 0, sizeof(svsk->sk_pages)); 1354 1355 tcp_sock_set_nodelay(sk); 1356 1357 set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); 1358 switch (sk->sk_state) { 1359 case TCP_SYN_RECV: 1360 case TCP_ESTABLISHED: 1361 break; 1362 default: 1363 svc_xprt_deferred_close(&svsk->sk_xprt); 1364 } 1365 } 1366} 1367 1368void svc_sock_update_bufs(struct svc_serv *serv) 1369{ 1370 /* 1371 * The number of server threads has changed. Update 1372 * rcvbuf and sndbuf accordingly on all sockets 1373 */ 1374 struct svc_sock *svsk; 1375 1376 spin_lock_bh(&serv->sv_lock); 1377 list_for_each_entry(svsk, &serv->sv_permsocks, sk_xprt.xpt_list) 1378 set_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags); 1379 spin_unlock_bh(&serv->sv_lock); 1380} 1381EXPORT_SYMBOL_GPL(svc_sock_update_bufs); 1382 1383/* 1384 * Initialize socket for RPC use and create svc_sock struct 1385 */ 1386static struct svc_sock *svc_setup_socket(struct svc_serv *serv, 1387 struct socket *sock, 1388 int flags) 1389{ 1390 struct svc_sock *svsk; 1391 struct sock *inet; 1392 int pmap_register = !(flags & SVC_SOCK_ANONYMOUS); 1393 1394 svsk = kzalloc(sizeof(*svsk), GFP_KERNEL); 1395 if (!svsk) 1396 return ERR_PTR(-ENOMEM); 1397 1398 inet = sock->sk; 1399 1400 if (pmap_register) { 1401 int err; 1402 1403 err = svc_register(serv, sock_net(sock->sk), inet->sk_family, 1404 inet->sk_protocol, 1405 ntohs(inet_sk(inet)->inet_sport)); 1406 if (err < 0) { 1407 kfree(svsk); 1408 return ERR_PTR(err); 1409 } 1410 } 1411 1412 svsk->sk_sock = sock; 1413 svsk->sk_sk = inet; 1414 svsk->sk_ostate = inet->sk_state_change; 1415 svsk->sk_odata = inet->sk_data_ready; 1416 svsk->sk_owspace = inet->sk_write_space; 1417 /* 1418 * This barrier is necessary in order to prevent race condition 1419 * with svc_data_ready(), svc_tcp_listen_data_ready(), and others 1420 * when calling callbacks above. 1421 */ 1422 wmb(); 1423 inet->sk_user_data = svsk; 1424 1425 /* Initialize the socket */ 1426 if (sock->type == SOCK_DGRAM) 1427 svc_udp_init(svsk, serv); 1428 else 1429 svc_tcp_init(svsk, serv); 1430 1431 trace_svcsock_new(svsk, sock); 1432 return svsk; 1433} 1434 1435/** 1436 * svc_addsock - add a listener socket to an RPC service 1437 * @serv: pointer to RPC service to which to add a new listener 1438 * @net: caller's network namespace 1439 * @fd: file descriptor of the new listener 1440 * @name_return: pointer to buffer to fill in with name of listener 1441 * @len: size of the buffer 1442 * @cred: credential 1443 * 1444 * Fills in socket name and returns positive length of name if successful. 1445 * Name is terminated with '\n'. On error, returns a negative errno 1446 * value. 1447 */ 1448int svc_addsock(struct svc_serv *serv, struct net *net, const int fd, 1449 char *name_return, const size_t len, const struct cred *cred) 1450{ 1451 int err = 0; 1452 struct socket *so = sockfd_lookup(fd, &err); 1453 struct svc_sock *svsk = NULL; 1454 struct sockaddr_storage addr; 1455 struct sockaddr *sin = (struct sockaddr *)&addr; 1456 int salen; 1457 1458 if (!so) 1459 return err; 1460 err = -EINVAL; 1461 if (sock_net(so->sk) != net) 1462 goto out; 1463 err = -EAFNOSUPPORT; 1464 if ((so->sk->sk_family != PF_INET) && (so->sk->sk_family != PF_INET6)) 1465 goto out; 1466 err = -EPROTONOSUPPORT; 1467 if (so->sk->sk_protocol != IPPROTO_TCP && 1468 so->sk->sk_protocol != IPPROTO_UDP) 1469 goto out; 1470 err = -EISCONN; 1471 if (so->state > SS_UNCONNECTED) 1472 goto out; 1473 err = -ENOENT; 1474 if (!try_module_get(THIS_MODULE)) 1475 goto out; 1476 svsk = svc_setup_socket(serv, so, SVC_SOCK_DEFAULTS); 1477 if (IS_ERR(svsk)) { 1478 module_put(THIS_MODULE); 1479 err = PTR_ERR(svsk); 1480 goto out; 1481 } 1482 salen = kernel_getsockname(svsk->sk_sock, sin); 1483 if (salen >= 0) 1484 svc_xprt_set_local(&svsk->sk_xprt, sin, salen); 1485 svsk->sk_xprt.xpt_cred = get_cred(cred); 1486 svc_add_new_perm_xprt(serv, &svsk->sk_xprt); 1487 return svc_one_sock_name(svsk, name_return, len); 1488out: 1489 sockfd_put(so); 1490 return err; 1491} 1492EXPORT_SYMBOL_GPL(svc_addsock); 1493 1494/* 1495 * Create socket for RPC service. 1496 */ 1497static struct svc_xprt *svc_create_socket(struct svc_serv *serv, 1498 int protocol, 1499 struct net *net, 1500 struct sockaddr *sin, int len, 1501 int flags) 1502{ 1503 struct svc_sock *svsk; 1504 struct socket *sock; 1505 int error; 1506 int type; 1507 struct sockaddr_storage addr; 1508 struct sockaddr *newsin = (struct sockaddr *)&addr; 1509 int newlen; 1510 int family; 1511 1512 if (protocol != IPPROTO_UDP && protocol != IPPROTO_TCP) { 1513 printk(KERN_WARNING "svc: only UDP and TCP " 1514 "sockets supported\n"); 1515 return ERR_PTR(-EINVAL); 1516 } 1517 1518 type = (protocol == IPPROTO_UDP)? SOCK_DGRAM : SOCK_STREAM; 1519 switch (sin->sa_family) { 1520 case AF_INET6: 1521 family = PF_INET6; 1522 break; 1523 case AF_INET: 1524 family = PF_INET; 1525 break; 1526 default: 1527 return ERR_PTR(-EINVAL); 1528 } 1529 1530 error = __sock_create(net, family, type, protocol, &sock, 1); 1531 if (error < 0) 1532 return ERR_PTR(error); 1533 1534 svc_reclassify_socket(sock); 1535 1536 /* 1537 * If this is an PF_INET6 listener, we want to avoid 1538 * getting requests from IPv4 remotes. Those should 1539 * be shunted to a PF_INET listener via rpcbind. 1540 */ 1541 if (family == PF_INET6) 1542 ip6_sock_set_v6only(sock->sk); 1543 if (type == SOCK_STREAM) 1544 sock->sk->sk_reuse = SK_CAN_REUSE; /* allow address reuse */ 1545 error = kernel_bind(sock, sin, len); 1546 if (error < 0) 1547 goto bummer; 1548 1549 error = kernel_getsockname(sock, newsin); 1550 if (error < 0) 1551 goto bummer; 1552 newlen = error; 1553 1554 if (protocol == IPPROTO_TCP) { 1555 if ((error = kernel_listen(sock, 64)) < 0) 1556 goto bummer; 1557 } 1558 1559 svsk = svc_setup_socket(serv, sock, flags); 1560 if (IS_ERR(svsk)) { 1561 error = PTR_ERR(svsk); 1562 goto bummer; 1563 } 1564 svc_xprt_set_local(&svsk->sk_xprt, newsin, newlen); 1565 return (struct svc_xprt *)svsk; 1566bummer: 1567 sock_release(sock); 1568 return ERR_PTR(error); 1569} 1570 1571/* 1572 * Detach the svc_sock from the socket so that no 1573 * more callbacks occur. 1574 */ 1575static void svc_sock_detach(struct svc_xprt *xprt) 1576{ 1577 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); 1578 struct sock *sk = svsk->sk_sk; 1579 1580 /* put back the old socket callbacks */ 1581 lock_sock(sk); 1582 sk->sk_state_change = svsk->sk_ostate; 1583 sk->sk_data_ready = svsk->sk_odata; 1584 sk->sk_write_space = svsk->sk_owspace; 1585 sk->sk_user_data = NULL; 1586 release_sock(sk); 1587} 1588 1589/* 1590 * Disconnect the socket, and reset the callbacks 1591 */ 1592static void svc_tcp_sock_detach(struct svc_xprt *xprt) 1593{ 1594 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); 1595 1596 tls_handshake_close(svsk->sk_sock); 1597 1598 svc_sock_detach(xprt); 1599 1600 if (!test_bit(XPT_LISTENER, &xprt->xpt_flags)) { 1601 svc_tcp_clear_pages(svsk); 1602 kernel_sock_shutdown(svsk->sk_sock, SHUT_RDWR); 1603 } 1604} 1605 1606/* 1607 * Free the svc_sock's socket resources and the svc_sock itself. 1608 */ 1609static void svc_sock_free(struct svc_xprt *xprt) 1610{ 1611 struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); 1612 struct page_frag_cache *pfc = &svsk->sk_frag_cache; 1613 struct socket *sock = svsk->sk_sock; 1614 1615 trace_svcsock_free(svsk, sock); 1616 1617 tls_handshake_cancel(sock->sk); 1618 if (sock->file) 1619 sockfd_put(sock); 1620 else 1621 sock_release(sock); 1622 if (pfc->va) 1623 __page_frag_cache_drain(virt_to_head_page(pfc->va), 1624 pfc->pagecnt_bias); 1625 kfree(svsk); 1626} 1627