Lines Matching defs:sk
46 static void __mptcp_destroy_sock(struct sock *sk);
47 static void mptcp_check_send_data_fin(struct sock *sk);
58 static const struct proto_ops *mptcp_fallback_tcp_ops(const struct sock *sk)
61 if (sk->sk_prot == &tcpv6_prot)
64 WARN_ON_ONCE(sk->sk_prot != &tcp_prot);
71 struct sock *sk = (struct sock *)msk;
75 err = mptcp_subflow_create_socket(sk, sk->sk_family, &ssock);
79 msk->scaling_ratio = tcp_sk(ssock->sk)->scaling_ratio;
80 WRITE_ONCE(msk->first, ssock->sk);
81 subflow = mptcp_subflow_ctx(ssock->sk);
83 sock_hold(ssock->sk);
89 mptcp_sock_graft(msk->first, sk->sk_socket);
100 struct sock *sk = (struct sock *)msk;
103 if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
115 static void mptcp_drop(struct sock *sk, struct sk_buff *skb)
117 sk_drops_add(sk, skb);
121 static void mptcp_rmem_fwd_alloc_add(struct sock *sk, int size)
123 WRITE_ONCE(mptcp_sk(sk)->rmem_fwd_alloc,
124 mptcp_sk(sk)->rmem_fwd_alloc + size);
127 static void mptcp_rmem_charge(struct sock *sk, int size)
129 mptcp_rmem_fwd_alloc_add(sk, -size);
132 static bool mptcp_try_coalesce(struct sock *sk, struct sk_buff *to,
151 atomic_add(delta, &sk->sk_rmem_alloc);
152 mptcp_rmem_charge(sk, delta);
167 static void __mptcp_rmem_reclaim(struct sock *sk, int amount)
170 mptcp_rmem_charge(sk, amount << PAGE_SHIFT);
171 __sk_mem_reduce_allocated(sk, amount);
174 static void mptcp_rmem_uncharge(struct sock *sk, int size)
176 struct mptcp_sock *msk = mptcp_sk(sk);
179 mptcp_rmem_fwd_alloc_add(sk, size);
180 reclaimable = msk->rmem_fwd_alloc - sk_unused_reserved_mem(sk);
184 __mptcp_rmem_reclaim(sk, reclaimable);
190 struct sock *sk = skb->sk;
192 atomic_sub(len, &sk->sk_rmem_alloc);
193 mptcp_rmem_uncharge(sk, len);
196 void mptcp_set_owner_r(struct sk_buff *skb, struct sock *sk)
199 skb->sk = sk;
201 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
202 mptcp_rmem_charge(sk, skb->truesize);
211 struct sock *sk = (struct sock *)msk;
224 mptcp_drop(sk, skb);
228 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_NODSSWINDOW);
233 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUE);
245 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE);
246 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL);
252 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL);
270 mptcp_drop(sk, skb);
271 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
286 mptcp_drop(sk, skb1);
287 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
291 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE);
308 mptcp_drop(sk, skb1);
309 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
317 mptcp_set_owner_r(skb, sk);
320 static bool mptcp_rmem_schedule(struct sock *sk, struct sock *ssk, int size)
322 struct mptcp_sock *msk = mptcp_sk(sk);
331 if (!__sk_mem_raise_allocated(sk, size, amt, SK_MEM_RECV))
334 mptcp_rmem_fwd_alloc_add(sk, amount);
343 struct sock *sk = (struct sock *)msk;
353 if (!mptcp_rmem_schedule(sk, ssk, skb->truesize))
371 tail = skb_peek_tail(&sk->sk_receive_queue);
372 if (tail && mptcp_try_coalesce(sk, tail, skb))
375 mptcp_set_owner_r(skb, sk);
376 __skb_queue_tail(&sk->sk_receive_queue, skb);
386 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
388 mptcp_drop(sk, skb);
392 static void mptcp_stop_rtx_timer(struct sock *sk)
394 struct inet_connection_sock *icsk = inet_csk(sk);
396 sk_stop_timer(sk, &icsk->icsk_retransmit_timer);
397 mptcp_sk(sk)->timer_ival = 0;
400 static void mptcp_close_wake_up(struct sock *sk)
402 if (sock_flag(sk, SOCK_DEAD))
405 sk->sk_state_change(sk);
406 if (sk->sk_shutdown == SHUTDOWN_MASK ||
407 sk->sk_state == TCP_CLOSE)
408 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
410 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
414 static bool mptcp_pending_data_fin_ack(struct sock *sk)
416 struct mptcp_sock *msk = mptcp_sk(sk);
418 return ((1 << sk->sk_state) &
423 static void mptcp_check_data_fin_ack(struct sock *sk)
425 struct mptcp_sock *msk = mptcp_sk(sk);
428 if (mptcp_pending_data_fin_ack(sk)) {
431 switch (sk->sk_state) {
433 mptcp_set_state(sk, TCP_FIN_WAIT2);
437 mptcp_set_state(sk, TCP_CLOSE);
441 mptcp_close_wake_up(sk);
446 static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq)
448 struct mptcp_sock *msk = mptcp_sk(sk);
451 ((1 << inet_sk_state_load(sk)) &
466 static void mptcp_set_datafin_timeout(struct sock *sk)
468 struct inet_connection_sock *icsk = inet_csk(sk);
474 mptcp_sk(sk)->timer_ival = TCP_RTO_MIN << retransmits;
477 static void __mptcp_set_timeout(struct sock *sk, long tout)
479 mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN;
490 static void mptcp_set_timeout(struct sock *sk)
495 mptcp_for_each_subflow(mptcp_sk(sk), subflow)
497 __mptcp_set_timeout(sk, tout);
556 struct sock *sk = (struct sock *)msk;
557 int space = __mptcp_space(sk);
561 rx_empty = !__mptcp_rmem(sk);
571 static bool mptcp_check_data_fin(struct sock *sk)
573 struct mptcp_sock *msk = mptcp_sk(sk);
590 if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) {
594 WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | RCV_SHUTDOWN);
597 switch (sk->sk_state) {
599 mptcp_set_state(sk, TCP_CLOSE_WAIT);
602 mptcp_set_state(sk, TCP_CLOSING);
605 mptcp_set_state(sk, TCP_CLOSE);
616 mptcp_close_wake_up(sk);
626 struct sock *sk = (struct sock *)msk;
633 sk_rbuf = READ_ONCE(sk->sk_rcvbuf);
635 if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
639 WRITE_ONCE(sk->sk_rcvbuf, ssk_rbuf);
704 if (atomic_read(&sk->sk_rmem_alloc) > sk_rbuf) {
718 struct sock *sk = (struct sock *)msk;
736 mptcp_drop(sk, skb);
737 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
742 tail = skb_peek_tail(&sk->sk_receive_queue);
752 __skb_queue_tail(&sk->sk_receive_queue, skb);
761 static bool __mptcp_subflow_error_report(struct sock *sk, struct sock *ssk)
772 if (sk->sk_state != TCP_SYN_SENT && !__mptcp_check_fallback(mptcp_sk(sk)))
781 if (ssk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DEAD))
782 mptcp_set_state(sk, ssk_state);
783 WRITE_ONCE(sk->sk_err, -err);
787 sk_error_report(sk);
791 void __mptcp_error_report(struct sock *sk)
794 struct mptcp_sock *msk = mptcp_sk(sk);
797 if (__mptcp_subflow_error_report(sk, mptcp_subflow_tcp_sock(subflow)))
806 struct sock *sk = (struct sock *)msk;
812 if (!sock_owned_by_user(sk))
813 __mptcp_error_report(sk);
823 if (mptcp_pending_data_fin(sk, NULL))
824 mptcp_schedule_work(sk);
828 void mptcp_data_ready(struct sock *sk, struct sock *ssk)
831 struct mptcp_sock *msk = mptcp_sk(sk);
842 sk_rbuf = READ_ONCE(sk->sk_rcvbuf);
847 if (__mptcp_rmem(sk) > sk_rbuf) {
848 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RCVPRUNED);
853 mptcp_data_lock(sk);
854 if (move_skbs_to_msk(msk, ssk) && mptcp_epollin_ready(sk))
855 sk->sk_data_ready(sk);
856 mptcp_data_unlock(sk);
868 struct sock *sk = (struct sock *)msk;
870 if (sk->sk_state != TCP_ESTABLISHED)
876 if (sk->sk_socket && !ssk->sk_socket)
877 mptcp_sock_graft(ssk, sk->sk_socket);
882 mptcp_stop_tout_timer(sk);
883 __mptcp_propagate_sndbuf(sk, ssk);
887 static void __mptcp_flush_join_list(struct sock *sk, struct list_head *join_list)
890 struct mptcp_sock *msk = mptcp_sk(sk);
903 static bool mptcp_rtx_timer_pending(struct sock *sk)
905 return timer_pending(&inet_csk(sk)->icsk_retransmit_timer);
908 static void mptcp_reset_rtx_timer(struct sock *sk)
910 struct inet_connection_sock *icsk = inet_csk(sk);
914 if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE))
917 tout = mptcp_sk(sk)->timer_ival;
918 sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout);
921 bool mptcp_schedule_work(struct sock *sk)
923 if (inet_sk_state_load(sk) != TCP_CLOSE &&
924 schedule_work(&mptcp_sk(sk)->work)) {
925 /* each subflow already holds a reference to the sk, and the
926 * workqueue is invoked by a subflow, so sk can't go away here.
928 sock_hold(sk);
977 static void dfrag_uncharge(struct sock *sk, int len)
979 sk_mem_uncharge(sk, len);
980 sk_wmem_queued_add(sk, -len);
983 static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag)
988 dfrag_uncharge(sk, len);
993 static void __mptcp_clean_una(struct sock *sk)
995 struct mptcp_sock *msk = mptcp_sk(sk);
1009 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk));
1012 dfrag_clear(sk, dfrag);
1015 dfrag = mptcp_rtx_head(sk);
1033 dfrag_uncharge(sk, delta);
1042 if (mptcp_rtx_timer_pending(sk) && !mptcp_data_fin_enabled(msk))
1043 mptcp_stop_rtx_timer(sk);
1045 mptcp_reset_rtx_timer(sk);
1048 if (mptcp_pending_data_fin_ack(sk))
1049 mptcp_schedule_work(sk);
1052 static void __mptcp_clean_una_wakeup(struct sock *sk)
1054 lockdep_assert_held_once(&sk->sk_lock.slock);
1056 __mptcp_clean_una(sk);
1057 mptcp_write_space(sk);
1060 static void mptcp_clean_una_wakeup(struct sock *sk)
1062 mptcp_data_lock(sk);
1063 __mptcp_clean_una_wakeup(sk);
1064 mptcp_data_unlock(sk);
1067 static void mptcp_enter_memory_pressure(struct sock *sk)
1070 struct mptcp_sock *msk = mptcp_sk(sk);
1082 __mptcp_sync_sndbuf(sk);
1088 static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
1091 pfrag, sk->sk_allocation)))
1094 mptcp_enter_memory_pressure(sk);
1155 static struct sk_buff *__mptcp_do_alloc_tx_skb(struct sock *sk, gfp_t gfp)
1169 mptcp_enter_memory_pressure(sk);
1174 static struct sk_buff *__mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, gfp_t gfp)
1178 skb = __mptcp_do_alloc_tx_skb(sk, gfp);
1191 static struct sk_buff *mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, bool data_lock_held)
1193 gfp_t gfp = data_lock_held ? GFP_ATOMIC : sk->sk_allocation;
1195 return __mptcp_alloc_tx_skb(sk, ssk, gfp);
1228 static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,
1234 struct mptcp_sock *msk = mptcp_sk(sk);
1284 skb = mptcp_alloc_tx_skb(sk, ssk, info->data_lock_held);
1411 struct sock *sk = (struct sock *)msk;
1447 __mptcp_set_timeout(sk, tout);
1514 void mptcp_check_and_set_pending(struct sock *sk)
1516 if (mptcp_send_head(sk)) {
1517 mptcp_data_lock(sk);
1518 mptcp_sk(sk)->cb_flags |= BIT(MPTCP_PUSH_PENDING);
1519 mptcp_data_unlock(sk);
1523 static int __subflow_push_pending(struct sock *sk, struct sock *ssk,
1526 struct mptcp_sock *msk = mptcp_sk(sk);
1530 while ((dfrag = mptcp_send_head(sk))) {
1537 ret = mptcp_sendmsg_frag(sk, ssk, dfrag, info);
1549 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk));
1557 mptcp_set_timeout(sk);
1567 void __mptcp_push_pending(struct sock *sk, unsigned int flags)
1570 struct mptcp_sock *msk = mptcp_sk(sk);
1577 while (mptcp_send_head(sk) && (push_count > 0)) {
1608 ret = __subflow_push_pending(sk, ssk, &info);
1626 if (!mptcp_rtx_timer_pending(sk))
1627 mptcp_reset_rtx_timer(sk);
1629 mptcp_check_send_data_fin(sk);
1632 static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk, bool first)
1634 struct mptcp_sock *msk = mptcp_sk(sk);
1643 while (mptcp_send_head(sk) && keep_pushing) {
1652 ret = __subflow_push_pending(sk, ssk, &info);
1665 ret = __subflow_push_pending(sk, ssk, &info);
1690 if (!mptcp_rtx_timer_pending(sk))
1691 mptcp_reset_rtx_timer(sk);
1695 mptcp_schedule_work(sk);
1699 static int mptcp_disconnect(struct sock *sk, int flags);
1701 static int mptcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg,
1705 struct mptcp_sock *msk = mptcp_sk(sk);
1735 ret = __inet_stream_connect(sk->sk_socket, msg->msg_name,
1749 if (!mptcp_disconnect(sk, 0))
1750 sk->sk_socket->state = SS_UNCONNECTED;
1752 inet_clear_bit(DEFER_CONNECT, sk);
1757 static int do_copy_data_nocache(struct sock *sk, int copy,
1760 if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
1774 static u32 mptcp_send_limit(const struct sock *sk)
1776 const struct mptcp_sock *msk = mptcp_sk(sk);
1779 if (sk->sk_wmem_queued >= READ_ONCE(sk->sk_sndbuf))
1782 limit = mptcp_notsent_lowat(sk);
1793 static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
1795 struct mptcp_sock *msk = mptcp_sk(sk);
1804 lock_sock(sk);
1806 if (unlikely(inet_test_bit(DEFER_CONNECT, sk) ||
1810 ret = mptcp_sendmsg_fastopen(sk, msg, len, &copied_syn);
1818 timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
1820 if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) {
1821 ret = sk_stream_wait_connect(sk, &timeo);
1827 if (unlikely(sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)))
1830 pfrag = sk_page_frag(sk);
1840 copy_limit = mptcp_send_limit(sk);
1847 dfrag = mptcp_pending_tail(sk);
1850 if (!mptcp_page_frag_refill(sk, pfrag))
1867 if (!sk_wmem_schedule(sk, total_ts))
1870 ret = do_copy_data_nocache(sk, psize, &msg->msg_iter,
1876 sk_forward_alloc_add(sk, -total_ts);
1884 * Note: we charge such data both to sk and ssk
1886 sk_wmem_queued_add(sk, frag_truesize);
1900 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1901 __mptcp_push_pending(sk, msg->msg_flags);
1902 ret = sk_stream_wait_memory(sk, &timeo);
1908 __mptcp_push_pending(sk, msg->msg_flags);
1911 release_sock(sk);
1918 copied = sk_stream_error(sk, msg->msg_flags, ret);
1985 struct sock *sk = (struct sock *)msk;
2031 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf) &&
2032 !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
2044 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]));
2046 if (rcvbuf > sk->sk_rcvbuf) {
2050 WRITE_ONCE(sk->sk_rcvbuf, rcvbuf);
2077 static void __mptcp_update_rmem(struct sock *sk)
2079 struct mptcp_sock *msk = mptcp_sk(sk);
2084 atomic_sub(msk->rmem_released, &sk->sk_rmem_alloc);
2085 mptcp_rmem_uncharge(sk, msk->rmem_released);
2089 static void __mptcp_splice_receive_queue(struct sock *sk)
2091 struct mptcp_sock *msk = mptcp_sk(sk);
2093 skb_queue_splice_tail_init(&sk->sk_receive_queue, &msk->receive_queue);
2098 struct sock *sk = (struct sock *)msk;
2114 mptcp_data_lock(sk);
2115 __mptcp_update_rmem(sk);
2117 mptcp_data_unlock(sk);
2120 __mptcp_error_report(sk);
2127 !skb_queue_empty_lockless(&sk->sk_receive_queue)) {
2128 mptcp_data_lock(sk);
2129 __mptcp_update_rmem(sk);
2131 __mptcp_splice_receive_queue(sk);
2132 mptcp_data_unlock(sk);
2139 static unsigned int mptcp_inq_hint(const struct sock *sk)
2141 const struct mptcp_sock *msk = mptcp_sk(sk);
2154 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
2160 static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
2163 struct mptcp_sock *msk = mptcp_sk(sk);
2171 return inet_recv_error(sk, msg, len, addr_len);
2173 lock_sock(sk);
2174 if (unlikely(sk->sk_state == TCP_LISTEN)) {
2179 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
2182 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
2212 if (sk->sk_err ||
2213 sk->sk_state == TCP_CLOSE ||
2214 (sk->sk_shutdown & RCV_SHUTDOWN) ||
2219 if (sk->sk_err) {
2220 copied = sock_error(sk);
2224 if (sk->sk_shutdown & RCV_SHUTDOWN) {
2233 if (sk->sk_state == TCP_CLOSE) {
2250 sk_wait_data(sk, &timeo, NULL);
2256 tcp_recv_timestamp(msg, sk, &tss);
2259 unsigned int inq = mptcp_inq_hint(sk);
2266 msk, skb_queue_empty_lockless(&sk->sk_receive_queue),
2271 release_sock(sk);
2279 struct sock *sk = &icsk->icsk_inet.sk;
2280 struct mptcp_sock *msk = mptcp_sk(sk);
2282 bh_lock_sock(sk);
2283 if (!sock_owned_by_user(sk)) {
2286 mptcp_schedule_work(sk);
2291 bh_unlock_sock(sk);
2292 sock_put(sk);
2297 struct sock *sk = from_timer(sk, t, sk_timer);
2299 mptcp_schedule_work(sk);
2300 sock_put(sk);
2344 bool __mptcp_retransmit_pending_data(struct sock *sk)
2347 struct mptcp_sock *msk = mptcp_sk(sk);
2356 mptcp_data_lock(sk);
2357 __mptcp_clean_una_wakeup(sk);
2358 rtx_head = mptcp_rtx_head(sk);
2360 mptcp_data_unlock(sk);
2366 mptcp_data_unlock(sk);
2413 static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk,
2417 struct mptcp_sock *msk = mptcp_sk(sk);
2426 (sock_flag(sk, SOCK_DEAD) || sock_flag(ssk, SOCK_DEAD))) {
2428 sock_set_flag(sk, SOCK_DEAD);
2429 mptcp_set_close_tout(sk, tcp_jiffies32 - (mptcp_close_timeout(sk) + 1));
2450 need_push = (flags & MPTCP_CF_PUSH) && __mptcp_retransmit_pending_data(sk);
2476 __mptcp_subflow_error_report(sk, ssk);
2485 __mptcp_sync_sndbuf(sk);
2487 __mptcp_push_pending(sk, 0);
2496 if (sk->sk_state != TCP_ESTABLISHED ||
2497 msk->in_accept_queue || sock_flag(sk, SOCK_DEAD)) {
2498 mptcp_set_state(sk, TCP_CLOSE);
2499 mptcp_close_wake_up(sk);
2501 mptcp_start_tout_timer(sk);
2506 void mptcp_close_ssk(struct sock *sk, struct sock *ssk,
2509 if (sk->sk_state == TCP_ESTABLISHED)
2510 mptcp_event(MPTCP_EVENT_SUB_CLOSED, mptcp_sk(sk), ssk, GFP_KERNEL);
2515 mptcp_pm_subflow_check_next(mptcp_sk(sk), subflow);
2517 __mptcp_close_ssk(sk, ssk, subflow, MPTCP_CF_PUSH);
2520 static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu)
2525 static void __mptcp_close_subflow(struct sock *sk)
2528 struct mptcp_sock *msk = mptcp_sk(sk);
2542 mptcp_close_ssk(sk, ssk, subflow);
2547 static bool mptcp_close_tout_expired(const struct sock *sk)
2549 if (!inet_csk(sk)->icsk_mtup.probe_timestamp ||
2550 sk->sk_state == TCP_CLOSE)
2554 inet_csk(sk)->icsk_mtup.probe_timestamp + mptcp_close_timeout(sk));
2560 struct sock *sk = (struct sock *)msk;
2580 switch (sk->sk_state) {
2582 WRITE_ONCE(sk->sk_err, ECONNREFUSED);
2585 WRITE_ONCE(sk->sk_err, EPIPE);
2590 WRITE_ONCE(sk->sk_err, ECONNRESET);
2593 mptcp_set_state(sk, TCP_CLOSE);
2594 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK);
2599 if (sock_flag(sk, SOCK_DEAD))
2602 sk->sk_state_change(sk);
2603 sk_error_report(sk);
2606 static void __mptcp_retrans(struct sock *sk)
2608 struct mptcp_sock *msk = mptcp_sk(sk);
2616 mptcp_clean_una_wakeup(sk);
2620 dfrag = mptcp_rtx_head(sk);
2623 struct inet_connection_sock *icsk = inet_csk(sk);
2626 mptcp_set_datafin_timeout(sk);
2632 if (!mptcp_send_head(sk))
2656 ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info);
2660 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS);
2679 mptcp_check_and_set_pending(sk);
2681 if (!mptcp_rtx_timer_pending(sk))
2682 mptcp_reset_rtx_timer(sk);
2690 struct sock *sk = (struct sock *)msk;
2693 if (!fail_tout && !inet_csk(sk)->icsk_mtup.probe_timestamp)
2696 close_timeout = inet_csk(sk)->icsk_mtup.probe_timestamp - tcp_jiffies32 + jiffies +
2697 mptcp_close_timeout(sk);
2702 timeout = inet_csk(sk)->icsk_mtup.probe_timestamp ? close_timeout : fail_tout;
2704 sk_reset_timer(sk, &sk->sk_timer, timeout);
2723 static void mptcp_do_fastclose(struct sock *sk)
2726 struct mptcp_sock *msk = mptcp_sk(sk);
2728 mptcp_set_state(sk, TCP_CLOSE);
2730 __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow),
2737 struct sock *sk = (struct sock *)msk;
2741 lock_sock(sk);
2742 state = sk->sk_state;
2750 mptcp_check_send_data_fin(sk);
2751 mptcp_check_data_fin_ack(sk);
2752 mptcp_check_data_fin(sk);
2755 __mptcp_close_subflow(sk);
2757 if (mptcp_close_tout_expired(sk)) {
2758 mptcp_do_fastclose(sk);
2759 mptcp_close_wake_up(sk);
2762 if (sock_flag(sk, SOCK_DEAD) && sk->sk_state == TCP_CLOSE) {
2763 __mptcp_destroy_sock(sk);
2768 __mptcp_retrans(sk);
2775 release_sock(sk);
2776 sock_put(sk);
2779 static void __mptcp_init_sock(struct sock *sk)
2781 struct mptcp_sock *msk = mptcp_sk(sk);
2796 inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss;
2797 WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk)));
2808 timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0);
2809 timer_setup(&sk->sk_timer, mptcp_tout_timer, 0);
2812 static void mptcp_ca_reset(struct sock *sk)
2814 struct inet_connection_sock *icsk = inet_csk(sk);
2816 tcp_assign_congestion_control(sk);
2817 strscpy(mptcp_sk(sk)->ca_name, icsk->icsk_ca_ops->name,
2818 sizeof(mptcp_sk(sk)->ca_name));
2821 tcp_cleanup_congestion_control(sk);
2825 static int mptcp_init_sock(struct sock *sk)
2827 struct net *net = sock_net(sk);
2830 __mptcp_init_sock(sk);
2838 ret = mptcp_init_sched(mptcp_sk(sk),
2843 set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags);
2848 mptcp_ca_reset(sk);
2850 sk_sockets_allocated_inc(sk);
2851 sk->sk_rcvbuf = READ_ONCE(net->ipv4.sysctl_tcp_rmem[1]);
2852 sk->sk_sndbuf = READ_ONCE(net->ipv4.sysctl_tcp_wmem[1]);
2857 static void __mptcp_clear_xmit(struct sock *sk)
2859 struct mptcp_sock *msk = mptcp_sk(sk);
2864 dfrag_clear(sk, dfrag);
2867 void mptcp_cancel_work(struct sock *sk)
2869 struct mptcp_sock *msk = mptcp_sk(sk);
2872 __sock_put(sk);
2875 void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how)
2888 if (__mptcp_check_fallback(mptcp_sk(sk))) {
2896 WRITE_ONCE(mptcp_sk(sk)->snd_una, mptcp_sk(sk)->snd_nxt);
2897 mptcp_schedule_work(sk);
2901 if (!mptcp_rtx_timer_pending(sk))
2902 mptcp_reset_rtx_timer(sk);
2910 void mptcp_set_state(struct sock *sk, int state)
2912 int oldstate = sk->sk_state;
2917 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_CURRESTAB);
2920 /* Unlike TCP, MPTCP sk would not have the TCP_SYN_RECV state:
2927 MPTCP_DEC_STATS(sock_net(sk), MPTCP_MIB_CURRESTAB);
2930 inet_sk_state_store(sk, state);
2950 static int mptcp_close_state(struct sock *sk)
2952 int next = (int)new_state[sk->sk_state];
2955 mptcp_set_state(sk, ns);
2960 static void mptcp_check_send_data_fin(struct sock *sk)
2963 struct mptcp_sock *msk = mptcp_sk(sk);
2966 msk, msk->snd_data_fin_enable, !!mptcp_send_head(sk),
2973 mptcp_send_head(sk))
2981 mptcp_subflow_shutdown(sk, tcp_sk, SEND_SHUTDOWN);
2985 static void __mptcp_wr_shutdown(struct sock *sk)
2987 struct mptcp_sock *msk = mptcp_sk(sk);
2990 msk, msk->snd_data_fin_enable, sk->sk_shutdown, sk->sk_state,
2991 !!mptcp_send_head(sk));
2997 mptcp_check_send_data_fin(sk);
3000 static void __mptcp_destroy_sock(struct sock *sk)
3002 struct mptcp_sock *msk = mptcp_sk(sk);
3008 mptcp_stop_rtx_timer(sk);
3009 sk_stop_timer(sk, &sk->sk_timer);
3013 sk->sk_prot->destroy(sk);
3017 sk_stream_kill_queues(sk);
3018 xfrm_sk_free_policy(sk);
3020 sock_put(sk);
3023 void __mptcp_unaccepted_force_close(struct sock *sk)
3025 sock_set_flag(sk, SOCK_DEAD);
3026 mptcp_do_fastclose(sk);
3027 __mptcp_destroy_sock(sk);
3030 static __poll_t mptcp_check_readable(struct sock *sk)
3032 return mptcp_epollin_ready(sk) ? EPOLLIN | EPOLLRDNORM : 0;
3035 static void mptcp_check_listen_stop(struct sock *sk)
3039 if (inet_sk_state_load(sk) != TCP_LISTEN)
3042 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
3043 ssk = mptcp_sk(sk)->first;
3049 mptcp_subflow_queue_clean(sk, ssk);
3055 bool __mptcp_close(struct sock *sk, long timeout)
3058 struct mptcp_sock *msk = mptcp_sk(sk);
3062 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK);
3064 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) {
3065 mptcp_check_listen_stop(sk);
3066 mptcp_set_state(sk, TCP_CLOSE);
3074 mptcp_do_fastclose(sk);
3076 } else if (mptcp_close_state(sk)) {
3077 __mptcp_wr_shutdown(sk);
3080 sk_stream_wait_close(sk, timeout);
3103 sock_orphan(sk);
3109 mptcp_set_state(sk, TCP_CLOSE);
3111 sock_hold(sk);
3112 pr_debug("msk=%p state=%d", sk, sk->sk_state);
3116 if (sk->sk_state == TCP_CLOSE) {
3117 __mptcp_destroy_sock(sk);
3120 mptcp_start_tout_timer(sk);
3126 static void mptcp_close(struct sock *sk, long timeout)
3130 lock_sock(sk);
3132 do_cancel_work = __mptcp_close(sk, timeout);
3133 release_sock(sk);
3135 mptcp_cancel_work(sk);
3137 sock_put(sk);
3163 static int mptcp_disconnect(struct sock *sk, int flags)
3165 struct mptcp_sock *msk = mptcp_sk(sk);
3174 mptcp_check_listen_stop(sk);
3175 mptcp_set_state(sk, TCP_CLOSE);
3177 mptcp_stop_rtx_timer(sk);
3178 mptcp_stop_tout_timer(sk);
3196 WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk)));
3198 mptcp_ca_reset(sk);
3206 WRITE_ONCE(sk->sk_shutdown, 0);
3207 sk_error_report(sk);
3212 static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk)
3216 return (struct ipv6_pinfo *)(((u8 *)sk) + offset);
3219 static void mptcp_copy_ip6_options(struct sock *newsk, const struct sock *sk)
3221 const struct ipv6_pinfo *np = inet6_sk(sk);
3239 static void mptcp_copy_ip_options(struct sock *newsk, const struct sock *sk)
3242 const struct inet_sock *inet = inet_sk(sk);
3262 struct sock *mptcp_sk_clone_init(const struct sock *sk,
3268 struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC);
3284 mptcp_copy_ip6_options(nsk, sk);
3287 mptcp_copy_ip_options(nsk, sk);
3301 msk->setsockopt_seq = mptcp_sk(sk)->setsockopt_seq;
3302 mptcp_init_sched(msk, mptcp_sk(sk)->sched);
3362 struct sock *sk = (struct sock *)msk;
3364 __mptcp_clear_xmit(sk);
3368 __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, flags);
3371 mptcp_data_lock(sk);
3372 skb_queue_splice_tail_init(&msk->receive_queue, &sk->sk_receive_queue);
3373 __skb_queue_purge(&sk->sk_receive_queue);
3375 mptcp_data_unlock(sk);
3380 sk_forward_alloc_add(sk, msk->rmem_fwd_alloc);
3387 static void mptcp_destroy(struct sock *sk)
3389 struct mptcp_sock *msk = mptcp_sk(sk);
3394 sk_sockets_allocated_dec(sk);
3397 void __mptcp_data_acked(struct sock *sk)
3399 if (!sock_owned_by_user(sk))
3400 __mptcp_clean_una(sk);
3402 __set_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->cb_flags);
3405 void __mptcp_check_push(struct sock *sk, struct sock *ssk)
3407 if (!mptcp_send_head(sk))
3410 if (!sock_owned_by_user(sk))
3411 __mptcp_subflow_push_pending(sk, ssk, false);
3413 __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags);
3421 static void mptcp_release_cb(struct sock *sk)
3422 __must_hold(&sk->sk_lock.slock)
3424 struct mptcp_sock *msk = mptcp_sk(sk);
3444 spin_unlock_bh(&sk->sk_lock.slock);
3447 __mptcp_flush_join_list(sk, &join_list);
3449 __mptcp_push_pending(sk, 0);
3451 __mptcp_retrans(sk);
3454 spin_lock_bh(&sk->sk_lock.slock);
3458 __mptcp_clean_una_wakeup(sk);
3462 * On sk release avoid actions depending on the first subflow
3465 __mptcp_sync_state(sk, msk->pending_state);
3467 __mptcp_error_report(sk);
3469 __mptcp_sync_sndbuf(sk);
3472 __mptcp_update_rmem(sk);
3504 struct sock *sk = subflow->conn;
3507 mptcp_data_lock(sk);
3508 if (!sock_owned_by_user(sk))
3509 __mptcp_subflow_push_pending(sk, ssk, true);
3511 __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags);
3512 mptcp_data_unlock(sk);
3515 mptcp_data_lock(sk);
3516 if (!sock_owned_by_user(sk))
3517 __mptcp_sync_sndbuf(sk);
3519 __set_bit(MPTCP_SYNC_SNDBUF, &mptcp_sk(sk)->cb_flags);
3520 mptcp_data_unlock(sk);
3526 static int mptcp_hash(struct sock *sk)
3535 static void mptcp_unhash(struct sock *sk)
3540 static int mptcp_get_port(struct sock *sk, unsigned short snum)
3542 struct mptcp_sock *msk = mptcp_sk(sk);
3555 struct sock *sk;
3558 sk = subflow->conn;
3559 msk = mptcp_sk(sk);
3561 pr_debug("msk=%p, token=%u", sk, subflow->token);
3574 void mptcp_sock_graft(struct sock *sk, struct socket *parent)
3576 write_lock_bh(&sk->sk_callback_lock);
3577 rcu_assign_pointer(sk->sk_wq, &parent->wq);
3578 sk_set_socket(sk, parent);
3579 sk->sk_uid = SOCK_INODE(parent)->i_uid;
3580 write_unlock_bh(&sk->sk_callback_lock);
3634 static void mptcp_shutdown(struct sock *sk, int how)
3636 pr_debug("sk=%p, how=%d", sk, how);
3638 if ((how & SEND_SHUTDOWN) && mptcp_close_state(sk))
3639 __mptcp_wr_shutdown(sk);
3642 static int mptcp_forward_alloc_get(const struct sock *sk)
3644 return READ_ONCE(sk->sk_forward_alloc) +
3645 READ_ONCE(mptcp_sk(sk)->rmem_fwd_alloc);
3650 const struct sock *sk = (void *)msk;
3653 if (sk->sk_state == TCP_LISTEN)
3656 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
3677 static int mptcp_ioctl(struct sock *sk, int cmd, int *karg)
3679 struct mptcp_sock *msk = mptcp_sk(sk);
3684 if (sk->sk_state == TCP_LISTEN)
3687 lock_sock(sk);
3689 *karg = mptcp_inq_hint(sk);
3690 release_sock(sk);
3693 slow = lock_sock_fast(sk);
3695 unlock_sock_fast(sk, slow);
3698 slow = lock_sock_fast(sk);
3700 unlock_sock_fast(sk, slow);
3716 static int mptcp_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
3719 struct mptcp_sock *msk = mptcp_sk(sk);
3727 mptcp_set_state(sk, TCP_SYN_SENT);
3744 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVE);
3768 inet_assign_bit(DEFER_CONNECT, sk, inet_test_bit(DEFER_CONNECT, ssk));
3780 mptcp_set_state(sk, TCP_CLOSE);
3784 mptcp_copy_inaddrs(sk, ssk);
3824 struct mptcp_sock *msk = mptcp_sk(sock->sk);
3825 struct sock *ssk, *sk = sock->sk;
3828 lock_sock(sk);
3835 if (sk->sk_family == AF_INET)
3838 else if (sk->sk_family == AF_INET6)
3842 mptcp_copy_inaddrs(sk, ssk);
3845 release_sock(sk);
3851 struct mptcp_sock *msk = mptcp_sk(sock->sk);
3852 struct sock *sk = sock->sk;
3858 lock_sock(sk);
3870 mptcp_set_state(sk, TCP_LISTEN);
3871 sock_set_flag(sk, SOCK_RCU_FREE);
3876 mptcp_set_state(sk, inet_sk_state_load(ssk));
3879 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
3880 mptcp_copy_inaddrs(sk, ssk);
3885 release_sock(sk);
3892 struct mptcp_sock *msk = mptcp_sk(sock->sk);
3961 * flow: sk is a tcp_sk, not an mptcp one.
3966 WRITE_ONCE(newsock->sk->sk_socket->ops,
3967 mptcp_fallback_tcp_ops(newsock->sk));
3976 struct sock *sk = (struct sock *)msk;
3978 if (__mptcp_stream_is_writeable(sk, 1))
3981 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
3983 if (__mptcp_stream_is_writeable(sk, 1))
3992 struct sock *sk = sock->sk;
3998 msk = mptcp_sk(sk);
4001 state = inet_sk_state_load(sk);
4012 shutdown = READ_ONCE(sk->sk_shutdown);
4019 mask |= mptcp_check_readable(sk);
4025 inet_test_bit(DEFER_CONNECT, sk)) {
4032 if (READ_ONCE(sk->sk_err))