183 static int mptcp_delete_ok(struct mptses *mpte, struct mptsub *mpts);
358 	struct mptses *mpte;
366 	mpte = &((struct mpp_mtp *)mpp)->mpp_ses;
370 	bzero(mpte, sizeof (*mpte));
372 	mpp->mpp_pcbe = mpte;
373 	mpte->mpte_mppcb = mpp;
374 	mpte->mpte_mptcb = mp_tp;
376 	TAILQ_INIT(&mpte->mpte_sopts);
377 	TAILQ_INIT(&mpte->mpte_subflows);
378 	mpte->mpte_associd = ASSOCID_ANY;
379 	mpte->mpte_connid_last = CONNID_ANY;
381 	lck_mtx_init(&mpte->mpte_thread_lock, mppi->mppi_lock_grp,
392 	if (kernel_thread_start(mptcp_thread_func, mpte,
393 	    &mpte->mpte_thread) != KERN_SUCCESS) {
403 	mp_tp->mpt_mpte = mpte;
407 		lck_mtx_destroy(&mpte->mpte_thread_lock, mppi->mppi_lock_grp);
413 	return ((error != 0) ? NULL : mpte);
420 mptcp_sesdestroy(struct mptses *mpte)
424 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
426 	mp_tp = mpte->mpte_mptcb;
432 	mptcp_flush_sopts(mpte);
433 	VERIFY(TAILQ_EMPTY(&mpte->mpte_subflows) && mpte->mpte_numflows == 0);
435 	lck_mtx_destroy(&mpte->mpte_thread_lock,
436 	    mpte->mpte_mppcb->mpp_pcbinfo->mppi_lock_grp);
442 	    mpte->mpte_mppcb->mpp_pcbinfo->mppi_lock_grp);
444 	DTRACE_MPTCP2(session__destroy, struct mptses *, mpte,
480 mptcp_sopt_insert(struct mptses *mpte, struct mptopt *mpo)
482 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
485 	TAILQ_INSERT_TAIL(&mpte->mpte_sopts, mpo, mpo_entry);
492 mptcp_sopt_remove(struct mptses *mpte, struct mptopt *mpo)
494 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
497 	TAILQ_REMOVE(&mpte->mpte_sopts, mpo, mpo_entry);
504 mptcp_sopt_find(struct mptses *mpte, struct sockopt *sopt)
508 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
510 	TAILQ_FOREACH(mpo, &mpte->mpte_sopts, mpo_entry) {
524 mptcp_flush_sopts(struct mptses *mpte)
528 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
530 	TAILQ_FOREACH_SAFE(mpo, &mpte->mpte_sopts, mpo_entry, tmpo) {
531 		mptcp_sopt_remove(mpte, mpo);
534 	VERIFY(TAILQ_EMPTY(&mpte->mpte_sopts));
588 mptcp_subflow_socreate(struct mptses *mpte, struct mptsub *mpts, int dom,
596 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
597 	mp_so = mpte->mpte_mppcb->mpp_socket;
631 	if ((error = mptcp_subflow_sosetopt(mpte, *so, &smpo)) != 0)
636 	if ((error = mptcp_subflow_sosetopt(mpte, *so, &smpo)) != 0)
641 	if ((error = mptcp_subflow_sosetopt(mpte, *so, &smpo)) != 0)
654 	if ((error = mptcp_subflow_sosetopt(mpte, *so, &smpo)) != 0)
665 	if ((error = mptcp_subflow_sosetopt(mpte, *so, &smpo)) != 0)
669 	TAILQ_FOREACH_SAFE(mpo, &mpte->mpte_sopts, mpo_entry, tmpo) {
687 		if (mptcp_subflow_sosetopt(mpte, *so, mpo) != 0 && interim) {
694 			mptcp_sopt_remove(mpte, mpo);
723 	DTRACE_MPTCP4(subflow__create, struct mptses *, mpte,
770 mptcp_subflow_soconnectx(struct mptses *mpte, struct mptsub *mpts)
775 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
793 		    (u_int64_t)VM_KERNEL_ADDRPERM(mpte->mpte_mppcb->mpp_socket),
808 	mptcp_attach_to_subf(so, mpte->mpte_mptcb, mpts->mpts_connid);
812 	    mpte->mpte_associd, NULL, TCP_CONNREQF_MPTCP,
816 	DTRACE_MPTCP3(subflow__connect, struct mptses *, mpte,
1004 mptcp_subflow_sopeeloff(struct mptses *mpte, struct mptsub *mpts,
1011 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
1012 	mp_so = mpte->mpte_mppcb->mpp_socket;
1058 		(void) mptcp_subflow_sosetopt(mpte, so, &smpo);
1066 		(void) mptcp_subflow_sosetopt(mpte, so, &smpo);
1074 		(void) mptcp_subflow_sosetopt(mpte, so, &smpo);
1083 		(void) mptcp_subflow_sosetopt(mpte, so, &smpo);
1086 	DTRACE_MPTCP5(subflow__peeloff, struct mptses *, mpte,
1096 mptcp_subflow_add(struct mptses *mpte, struct mptsub *mpts,
1105 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
1106 	mp_so = mpte->mpte_mppcb->mpp_socket;
1107 	mp_tp = mpte->mpte_mptcb;
1149 	if ((error = mptcp_subflow_socreate(mpte, mpts, af, p, &so)) != 0)
1160 	mpte->mpte_connid_last++;
1161 	if (mpte->mpte_connid_last == CONNID_ALL ||
1162 	    mpte->mpte_connid_last == CONNID_ANY)
1163 		mpte->mpte_connid_last++;
1165 	mpts->mpts_connid = mpte->mpte_connid_last;
1238 	TAILQ_INSERT_TAIL(&mpte->mpte_subflows, mpts, mpts_entry);
1239 	mpte->mpte_numflows++;
1242 	mpts->mpts_mpte = mpte;
1280 	if (mp_tp->mpt_state < MPTCPS_ESTABLISHED && mpte->mpte_numflows == 1) {
1317 		error = mptcp_subflow_soconnectx(mpte, mpts);
1329 mptcp_delete_ok(struct mptses *mpte, struct mptsub *mpts)
1334 	MPTE_LOCK_ASSERT_HELD(mpte);
1335 	mp_tp = mpte->mpte_mptcb;
1356 mptcp_subflow_del(struct mptses *mpte, struct mptsub *mpts, boolean_t close)
1360 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
1361 	mp_so = mpte->mpte_mppcb->mpp_socket;
1375 	VERIFY(mpts->mpts_mpte == mpte);
1381 	TAILQ_REMOVE(&mpte->mpte_subflows, mpts, mpts_entry);
1382 	VERIFY(mpte->mpte_numflows != 0);
1383 	mpte->mpte_numflows--;
1391 	mptcp_detach_mptcb_from_subf(mpte->mpte_mptcb, so);
1411 mptcp_subflow_disconnect(struct mptses *mpte, struct mptsub *mpts,
1418 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
1421 	VERIFY(mpts->mpts_mpte == mpte);
1441 	mp_tp = mpte->mpte_mptcb;
1486 	struct mptses *mpte = mpts->mpts_mpte;
1488 	VERIFY(mpte != NULL);
1490 	lck_mtx_lock(&mpte->mpte_thread_lock);
1491 	mptcp_thread_signal_locked(mpte);
1492 	lck_mtx_unlock(&mpte->mpte_thread_lock);
1502 mptcp_subflow_input(struct mptses *mpte, struct mptsub *mpts)
1509 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
1512 	DTRACE_MPTCP2(subflow__input, struct mptses *, mpte, 
1525 		mpts_alt = mptcp_get_subflow(mpte, mpts);
1529 			mpte->mpte_mppcb->mpp_socket->so_error = error;
1553 		mptcp_input(mpte, m);
1573 	struct mptses *mpte = mpts->mpts_mpte;
1575 	VERIFY(mpte != NULL);
1577 	lck_mtx_lock(&mpte->mpte_thread_lock);
1578 	mptcp_thread_signal_locked(mpte);
1579 	lck_mtx_unlock(&mpte->mpte_thread_lock);
1588 mptcp_subflow_output(struct mptses *mpte, struct mptsub *mpts)
1595 	struct mptcb *mp_tp = mpte->mpte_mptcb;
1599 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
1601 	mp_so = mpte->mpte_mppcb->mpp_socket;
1604 	DTRACE_MPTCP2(subflow__output, struct mptses *, mpte, 
1625 	if (mpte->mpte_flags & MPTE_SND_REM_ADDR) {
1627 		tp->t_rem_aid = mpte->mpte_lost_aid;
1630 		mpte->mpte_flags &= ~MPTE_SND_REM_ADDR;
1785 		    struct mptses *, mpte, struct mptsub *, mpts,
1843 	struct mptses *mpte = mpts->mpts_mpte;
1845 	VERIFY(mpte != NULL);
1847 	lck_mtx_lock(&mpte->mpte_thread_lock);
1849 	mptcp_thread_signal_locked(mpte);
1850 	lck_mtx_unlock(&mpte->mpte_thread_lock);
1859 mptcp_subflow_events(struct mptses *mpte, struct mptsub *mpts)
1864 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
1879 	DTRACE_MPTCP3(subflow__events, struct mptses *, mpte,
1886 		ev_ret_t error = mptcp_subflow_failover_ev(mpte, mpts);
1891 		ev_ret_t error = mptcp_subflow_connreset_ev(mpte, mpts);
1896 		ev_ret_t error = mptcp_subflow_mustrst_ev(mpte, mpts);
1901 		ev_ret_t error = mptcp_subflow_cantrcvmore_ev(mpte, mpts);
1906 		ev_ret_t error = mptcp_subflow_cantsendmore_ev(mpte, mpts);
1911 		ev_ret_t error = mptcp_subflow_timeout_ev(mpte, mpts);
1916 		ev_ret_t error = mptcp_subflow_nosrcaddr_ev(mpte, mpts);
1921 		ev_ret_t error = mptcp_subflow_ifdenied_ev(mpte, mpts);
1926 		ev_ret_t error = mptcp_subflow_suspend_ev(mpte, mpts);
1931 		ev_ret_t error = mptcp_subflow_resume_ev(mpte, mpts);
1936 		ev_ret_t error = mptcp_subflow_connected_ev(mpte, mpts);
1941 		ev_ret_t error = mptcp_subflow_mpstatus_ev(mpte, mpts);
1946 		ev_ret_t error = mptcp_subflow_disconnected_ev(mpte, mpts);
1972 mptcp_subflow_connreset_ev(struct mptses *mpte, struct mptsub *mpts)
1978 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
1980 	VERIFY(mpte->mpte_mppcb != NULL);
1981 	mp_so = mpte->mpte_mppcb->mpp_socket;
1982 	mp_tp = mpte->mpte_mptcb;
2001 	mptcp_subflow_disconnect(mpte, mpts, !linger);
2021 mptcp_subflow_cantrcvmore_ev(struct mptses *mpte, struct mptsub *mpts)
2025 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
2046 mptcp_subflow_cantsendmore_ev(struct mptses *mpte, struct mptsub *mpts)
2050 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
2063 mptcp_subflow_timeout_ev(struct mptses *mpte, struct mptsub *mpts)
2069 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
2071 	VERIFY(mpte->mpte_mppcb != NULL);
2072 	mp_so = mpte->mpte_mppcb->mpp_socket;
2073 	mp_tp = mpte->mpte_mptcb;
2092 	mptcp_subflow_disconnect(mpte, mpts, !linger);
2112 mptcp_subflow_nosrcaddr_ev(struct mptses *mpte, struct mptsub *mpts)
2119 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
2122 	VERIFY(mpte->mpte_mppcb != NULL);
2123 	mp_so = mpte->mpte_mppcb->mpp_socket;
2124 	mp_tp = mpte->mpte_mptcb;
2133 	mpte->mpte_flags |= MPTE_SND_REM_ADDR;
2134 	mpte->mpte_lost_aid = tp->t_local_aid;
2155 	mptcp_subflow_disconnect(mpte, mpts, !linger);
2176 mptcp_subflow_failover_ev(struct mptses *mpte, struct mptsub *mpts)
2183 	MPTE_LOCK_ASSERT_HELD(mpte);    /* same as MP socket lock */
2185 	mp_so = mpte->mpte_mppcb->mpp_socket;
2190 	mpts_alt = mptcp_get_subflow(mpte, mpts);
2218 		struct mptcb *mp_tp = mpte->mpte_mptcb;
2223 		mpte->mpte_active_sub = mpts_alt;
2259 mptcp_subflow_ifdenied_ev(struct mptses *mpte, struct mptsub *mpts)
2265 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
2267 	VERIFY(mpte->mpte_mppcb != NULL);
2268 	mp_so = mpte->mpte_mppcb->mpp_socket;
2269 	mp_tp = mpte->mpte_mptcb;
2288 	mptcp_subflow_disconnect(mpte, mpts, !linger);
2312 mptcp_subflow_suspend_ev(struct mptses *mpte, struct mptsub *mpts)
2316 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
2334 mptcp_subflow_resume_ev(struct mptses *mpte, struct mptsub *mpts)
2338 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
2355 mptcp_subflow_connected_ev(struct mptses *mpte, struct mptsub *mpts)
2366 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
2367 	VERIFY(mpte->mpte_mppcb != NULL);
2368 	mp_so = mpte->mpte_mppcb->mpp_socket;
2369 	mp_tp = mpte->mpte_mptcb;
2536 			(void) mptcp_drop(mpte, mp_tp, EPROTO);
2541 				(void) mptcp_drop(mpte, mp_tp, EPROTO);
2546 				mpte->mpte_associd = mpts->mpts_connid;
2553 				(void) mptcp_setconnorder(mpte,
2563 			mpte->mpte_nummpcapflows++;
2579 		mpte->mpte_nummpcapflows++;
2594 mptcp_subflow_disconnected_ev(struct mptses *mpte, struct mptsub *mpts)
2600 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
2602 	VERIFY(mpte->mpte_mppcb != NULL);
2603 	mp_so = mpte->mpte_mppcb->mpp_socket;
2604 	mp_tp = mpte->mpte_mptcb;
2637 		mpte->mpte_nummpcapflows--;
2664 mptcp_subflow_mpstatus_ev(struct mptses *mpte, struct mptsub *mpts)
2670 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
2671 	VERIFY(mpte->mpte_mppcb != NULL);
2672 	mp_so = mpte->mpte_mppcb->mpp_socket;
2673 	mp_tp = mpte->mpte_mptcb;
2714 	    (u_int64_t)VM_KERNEL_ADDRPERM(mpte->mpte_mppcb->mpp_socket),
2728 mptcp_subflow_mustrst_ev(struct mptses *mpte, struct mptsub *mpts)
2735 	MPTE_LOCK_ASSERT_HELD(mpte);    /* same as MP socket lock */
2737 	VERIFY(mpte->mpte_mppcb != NULL);
2738 	mp_so = mpte->mpte_mppcb->mpp_socket;
2739 	mp_tp = mpte->mpte_mptcb;
2779 	mptcp_subflow_disconnect(mpte, mpts, !linger);
2867 mptcp_subflow_sosetopt(struct mptses *mpte, struct socket *so,
2878 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
2879 	mp_so = mpte->mpte_mppcb->mpp_socket;
2912 mptcp_subflow_sogetopt(struct mptses *mpte, struct socket *so,
2921 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
2922 	mp_so = mpte->mpte_mppcb->mpp_socket;
2968 		struct mptses *mpte;
2974 		mpte = mptompte(mpp);
2975 		VERIFY(mpte != NULL);
2976 		mp_tp = mpte->mpte_mptcb;
3020 				    &mpte->mpte_subflows, mpts_entry, tmpts) {
3058 			mptcp_thread_terminate_signal(mpte);
3082 mptcp_drop(struct mptses *mpte, struct mptcb *mp_tp, int errno)
3086 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
3088 	VERIFY(mpte->mpte_mptcb == mp_tp);
3089 	mp_so = mpte->mpte_mppcb->mpp_socket;
3099 	return (mptcp_close(mpte, mp_tp));
3106 mptcp_close(struct mptses *mpte, struct mptcb *mp_tp)
3111 	MPTE_LOCK_ASSERT_HELD(mpte);	/* same as MP socket lock */
3113 	VERIFY(mpte->mpte_mptcb == mp_tp);
3114 	mp_so = mpte->mpte_mppcb->mpp_socket;
3130 	TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) {
3132 		mptcp_subflow_disconnect(mpte, mpts, TRUE);
3134 		mptcp_subflow_del(mpte, mpts, TRUE);
3151 mptcp_thread_signal(struct mptses *mpte)
3153 	lck_mtx_lock(&mpte->mpte_thread_lock);
3154 	mptcp_thread_signal_locked(mpte);
3155 	lck_mtx_unlock(&mpte->mpte_thread_lock);
3162 mptcp_thread_signal_locked(struct mptses *mpte)
3164 	lck_mtx_assert(&mpte->mpte_thread_lock, LCK_MTX_ASSERT_OWNED);
3166 	mpte->mpte_thread_reqs++;
3167 	if (!mpte->mpte_thread_active && mpte->mpte_thread != THREAD_NULL)
3168 		wakeup_one((caddr_t)&mpte->mpte_thread);
3175 mptcp_thread_terminate_signal(struct mptses *mpte)
3177 	lck_mtx_lock(&mpte->mpte_thread_lock);
3178 	if (mpte->mpte_thread != THREAD_NULL) {
3179 		mpte->mpte_thread = THREAD_NULL;
3180 		mpte->mpte_thread_reqs++;
3181 		if (!mpte->mpte_thread_active)
3182 			wakeup_one((caddr_t)&mpte->mpte_thread);
3184 	lck_mtx_unlock(&mpte->mpte_thread_lock);
3191 mptcp_thread_dowork(struct mptses *mpte)
3198 	MPTE_LOCK(mpte);		/* same as MP socket lock */
3199 	VERIFY(mpte->mpte_mppcb != NULL);
3200 	mp_so = mpte->mpte_mppcb->mpp_socket;
3203 	TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) {
3212 		mptcp_subflow_input(mpte, mpts);
3213 		ret = mptcp_subflow_events(mpte, mpts);
3218 			(void) mptcp_subflow_output(mpte, mpts);
3228 			mptcp_subflow_disconnect(mpte, mpts, FALSE);
3239 			if (mptcp_delete_ok(mpte, mpts)) {
3240 				mptcp_subflow_del(mpte, mpts, TRUE);
3259 		MPTE_UNLOCK(mpte);
3263 	TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) {
3316 				(void) mptcp_subflow_soconnectx(mpte, mpts);
3322 	MPTE_UNLOCK(mpte);
3332 	struct mptses *mpte = v;
3335 	VERIFY(mpte != NULL);
3337 	lck_mtx_lock_spin(&mpte->mpte_thread_lock);
3340 		lck_mtx_assert(&mpte->mpte_thread_lock, LCK_MTX_ASSERT_OWNED);
3342 		if (mpte->mpte_thread != THREAD_NULL) {
3343 			(void) msleep(&mpte->mpte_thread,
3344 			    &mpte->mpte_thread_lock, (PZERO - 1) | PSPIN,
3349 		if (mpte->mpte_thread == THREAD_NULL) {
3350 			lck_mtx_unlock(&mpte->mpte_thread_lock);
3352 			mptcp_thread_destroy(mpte);
3357 		mpte->mpte_thread_active = 1;
3359 			uint32_t reqs = mpte->mpte_thread_reqs;
3361 			lck_mtx_unlock(&mpte->mpte_thread_lock);
3362 			mptcp_thread_dowork(mpte);
3363 			lck_mtx_lock_spin(&mpte->mpte_thread_lock);
3366 			if (reqs == mpte->mpte_thread_reqs ||
3367 			    mpte->mpte_thread == THREAD_NULL)
3370 		mpte->mpte_thread_reqs = 0;
3371 		mpte->mpte_thread_active = 0;
3381 mptcp_thread_destroy(struct mptses *mpte)
3385 	MPTE_LOCK(mpte);		/* same as MP socket lock */
3386 	VERIFY(mpte->mpte_thread == THREAD_NULL);
3387 	VERIFY(mpte->mpte_mppcb != NULL);
3389 	mptcp_sesdestroy(mpte);
3391 	mp_so = mpte->mpte_mppcb->mpp_socket;
3395 	mpte->mpte_mppcb->mpp_flags |= MPP_DEFUNCT;
3396 	MPTE_UNLOCK(mpte);
4581 	struct mptses *mpte;
4603 		mpte = mptompte(mpp);
4604 		VERIFY(mpte != NULL);
4605 		mp_tp = mpte->mpte_mptcb;
4607 		len = sizeof(*flows) * mpte->mpte_numflows;
4615 		mptcpci.mptcpci_nflows = mpte->mpte_numflows;
4626 		TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) {

Indexes created Mon Jul 29 19:40:31 MDT 2024