| t4_ddp.c (269076) | t4_ddp.c (274421) |
|---|---|
| 1/*- 2 * Copyright (c) 2012 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 12 unchanged lines hidden (view full) --- 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2012 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 12 unchanged lines hidden (view full) --- 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> |
| 29__FBSDID("$FreeBSD: head/sys/dev/cxgbe/tom/t4_ddp.c 269076 2014-07-24 18:39:08Z np $"); | 29__FBSDID("$FreeBSD: head/sys/dev/cxgbe/tom/t4_ddp.c 274421 2014-11-12 09:57:15Z glebius $"); |
| 30 31#include "opt_inet.h" 32 33#include <sys/param.h> 34#include <sys/types.h> 35#include <sys/systm.h> 36#include <sys/kernel.h> 37#include <sys/ktr.h> --- 181 unchanged lines hidden (view full) --- 219 220 m = get_ddp_mbuf(n); 221 tp->rcv_nxt += n; 222#ifndef USE_DDP_RX_FLOW_CONTROL 223 KASSERT(tp->rcv_wnd >= n, ("%s: negative window size", __func__)); 224 tp->rcv_wnd -= n; 225#endif 226 | 30 31#include "opt_inet.h" 32 33#include <sys/param.h> 34#include <sys/types.h> 35#include <sys/systm.h> 36#include <sys/kernel.h> 37#include <sys/ktr.h> --- 181 unchanged lines hidden (view full) --- 219 220 m = get_ddp_mbuf(n); 221 tp->rcv_nxt += n; 222#ifndef USE_DDP_RX_FLOW_CONTROL 223 KASSERT(tp->rcv_wnd >= n, ("%s: negative window size", __func__)); 224 tp->rcv_wnd -= n; 225#endif 226 |
| 227 KASSERT(toep->sb_cc >= sb->sb_cc, | 227 KASSERT(toep->sb_cc >= sbused(sb), |
| 228 ("%s: sb %p has more data (%d) than last time (%d).", | 228 ("%s: sb %p has more data (%d) than last time (%d).", |
| 229 __func__, sb, sb->sb_cc, toep->sb_cc)); 230 toep->rx_credits += toep->sb_cc - sb->sb_cc; | 229 __func__, sb, sbused(sb), toep->sb_cc)); 230 toep->rx_credits += toep->sb_cc - sbused(sb); |
| 231#ifdef USE_DDP_RX_FLOW_CONTROL 232 toep->rx_credits -= n; /* adjust for F_RX_FC_DDP */ 233#endif 234 sbappendstream_locked(sb, m); | 231#ifdef USE_DDP_RX_FLOW_CONTROL 232 toep->rx_credits -= n; /* adjust for F_RX_FC_DDP */ 233#endif 234 sbappendstream_locked(sb, m); |
| 235 toep->sb_cc = sb->sb_cc; | 235 toep->sb_cc = sbused(sb); |
| 236} 237 238/* SET_TCB_FIELD sent as a ULP command looks like this */ 239#define LEN__SET_TCB_FIELD_ULP (sizeof(struct ulp_txpkt) + \ 240 sizeof(struct ulptx_idata) + sizeof(struct cpl_set_tcb_field_core)) 241 242/* RX_DATA_ACK sent as a ULP command looks like this */ 243#define LEN__RX_DATA_ACK_ULP (sizeof(struct ulp_txpkt) + \ --- 210 unchanged lines hidden (view full) --- 454 m = get_ddp_mbuf(len); 455 456 SOCKBUF_LOCK(sb); 457 if (report & F_DDP_BUF_COMPLETE) 458 toep->ddp_score = DDP_HIGH_SCORE; 459 else 460 discourage_ddp(toep); 461 | 236} 237 238/* SET_TCB_FIELD sent as a ULP command looks like this */ 239#define LEN__SET_TCB_FIELD_ULP (sizeof(struct ulp_txpkt) + \ 240 sizeof(struct ulptx_idata) + sizeof(struct cpl_set_tcb_field_core)) 241 242/* RX_DATA_ACK sent as a ULP command looks like this */ 243#define LEN__RX_DATA_ACK_ULP (sizeof(struct ulp_txpkt) + \ --- 210 unchanged lines hidden (view full) --- 454 m = get_ddp_mbuf(len); 455 456 SOCKBUF_LOCK(sb); 457 if (report & F_DDP_BUF_COMPLETE) 458 toep->ddp_score = DDP_HIGH_SCORE; 459 else 460 discourage_ddp(toep); 461 |
| 462 KASSERT(toep->sb_cc >= sb->sb_cc, | 462 KASSERT(toep->sb_cc >= sbused(sb), |
| 463 ("%s: sb %p has more data (%d) than last time (%d).", | 463 ("%s: sb %p has more data (%d) than last time (%d).", |
| 464 __func__, sb, sb->sb_cc, toep->sb_cc)); 465 toep->rx_credits += toep->sb_cc - sb->sb_cc; | 464 __func__, sb, sbused(sb), toep->sb_cc)); 465 toep->rx_credits += toep->sb_cc - sbused(sb); |
| 466#ifdef USE_DDP_RX_FLOW_CONTROL 467 toep->rx_credits -= len; /* adjust for F_RX_FC_DDP */ 468#endif 469 sbappendstream_locked(sb, m); | 466#ifdef USE_DDP_RX_FLOW_CONTROL 467 toep->rx_credits -= len; /* adjust for F_RX_FC_DDP */ 468#endif 469 sbappendstream_locked(sb, m); |
| 470 toep->sb_cc = sb->sb_cc; | 470 toep->sb_cc = sbused(sb); |
| 471wakeup: 472 KASSERT(toep->ddp_flags & db_flag, 473 ("%s: DDP buffer not active. toep %p, ddp_flags 0x%x, report 0x%x", 474 __func__, toep, toep->ddp_flags, report)); 475 toep->ddp_flags &= ~db_flag; 476 sorwakeup_locked(so); 477 SOCKBUF_UNLOCK_ASSERT(sb); 478 --- 424 unchanged lines hidden (view full) --- 903#if 0 904 if (sb->sb_cc + sc->tt.ddp_thres > uio->uio_resid) { 905 CTR4(KTR_CXGBE, "%s: sb_cc %d, threshold %d, resid %d", 906 __func__, sb->sb_cc, sc->tt.ddp_thres, uio->uio_resid); 907 } 908#endif 909 910 /* XXX: too eager to disable DDP, could handle NBIO better than this. */ | 471wakeup: 472 KASSERT(toep->ddp_flags & db_flag, 473 ("%s: DDP buffer not active. toep %p, ddp_flags 0x%x, report 0x%x", 474 __func__, toep, toep->ddp_flags, report)); 475 toep->ddp_flags &= ~db_flag; 476 sorwakeup_locked(so); 477 SOCKBUF_UNLOCK_ASSERT(sb); 478 --- 424 unchanged lines hidden (view full) --- 903#if 0 904 if (sb->sb_cc + sc->tt.ddp_thres > uio->uio_resid) { 905 CTR4(KTR_CXGBE, "%s: sb_cc %d, threshold %d, resid %d", 906 __func__, sb->sb_cc, sc->tt.ddp_thres, uio->uio_resid); 907 } 908#endif 909 910 /* XXX: too eager to disable DDP, could handle NBIO better than this. */ |
| 911 if (sb->sb_cc >= uio->uio_resid || uio->uio_resid < sc->tt.ddp_thres || | 911 if (sbused(sb) >= uio->uio_resid || uio->uio_resid < sc->tt.ddp_thres || |
| 912 uio->uio_resid > MAX_DDP_BUFFER_SIZE || uio->uio_iovcnt > 1 || 913 so->so_state & SS_NBIO || flags & (MSG_DONTWAIT | MSG_NBIO) || 914 error || so->so_error || sb->sb_state & SBS_CANTRCVMORE) 915 goto no_ddp; 916 917 /* 918 * Fault in and then hold the pages of the uio buffers. We'll wire them 919 * a bit later if everything else works out. --- 21 unchanged lines hidden (view full) --- 941 db = toep->db[db_idx]; 942 buf_flag = db_idx == 0 ? DDP_BUF0_ACTIVE : DDP_BUF1_ACTIVE; 943 944 /* 945 * Build the compound work request that tells the chip where to DMA the 946 * payload. 947 */ 948 ddp_flags = select_ddp_flags(so, flags, db_idx); | 912 uio->uio_resid > MAX_DDP_BUFFER_SIZE || uio->uio_iovcnt > 1 || 913 so->so_state & SS_NBIO || flags & (MSG_DONTWAIT | MSG_NBIO) || 914 error || so->so_error || sb->sb_state & SBS_CANTRCVMORE) 915 goto no_ddp; 916 917 /* 918 * Fault in and then hold the pages of the uio buffers. We'll wire them 919 * a bit later if everything else works out. --- 21 unchanged lines hidden (view full) --- 941 db = toep->db[db_idx]; 942 buf_flag = db_idx == 0 ? DDP_BUF0_ACTIVE : DDP_BUF1_ACTIVE; 943 944 /* 945 * Build the compound work request that tells the chip where to DMA the 946 * payload. 947 */ 948 ddp_flags = select_ddp_flags(so, flags, db_idx); |
| 949 wr = mk_update_tcb_for_ddp(sc, toep, db_idx, sb->sb_cc, ddp_flags); | 949 wr = mk_update_tcb_for_ddp(sc, toep, db_idx, sbused(sb), ddp_flags); |
| 950 if (wr == NULL) { 951 /* 952 * Just unhold the pages. The DDP buffer's software state is 953 * left as-is in the toep. The page pods were written 954 * successfully and we may have an opportunity to use it in the 955 * future. 956 */ 957 vm_page_unhold_pages(db->pages, db->npages); --- 171 unchanged lines hidden (view full) --- 1129 1130restart: 1131 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1132 1133 if (sb->sb_flags & SB_DDP_INDICATE && !ddp_handled) { 1134 1135 /* uio should be just as it was at entry */ 1136 KASSERT(oresid == uio->uio_resid, | 950 if (wr == NULL) { 951 /* 952 * Just unhold the pages. The DDP buffer's software state is 953 * left as-is in the toep. The page pods were written 954 * successfully and we may have an opportunity to use it in the 955 * future. 956 */ 957 vm_page_unhold_pages(db->pages, db->npages); --- 171 unchanged lines hidden (view full) --- 1129 1130restart: 1131 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1132 1133 if (sb->sb_flags & SB_DDP_INDICATE && !ddp_handled) { 1134 1135 /* uio should be just as it was at entry */ 1136 KASSERT(oresid == uio->uio_resid, |
| 1137 ("%s: oresid = %d, uio_resid = %zd, sb_cc = %d", 1138 __func__, oresid, uio->uio_resid, sb->sb_cc)); | 1137 ("%s: oresid = %d, uio_resid = %zd, sbused = %d", 1138 __func__, oresid, uio->uio_resid, sbused(sb))); |
| 1139 1140 error = handle_ddp(so, uio, flags, 0); 1141 ddp_handled = 1; 1142 if (error) 1143 goto out; 1144 } 1145 1146 /* Abort if socket has reported problems. */ 1147 if (so->so_error) { | 1139 1140 error = handle_ddp(so, uio, flags, 0); 1141 ddp_handled = 1; 1142 if (error) 1143 goto out; 1144 } 1145 1146 /* Abort if socket has reported problems. */ 1147 if (so->so_error) { |
| 1148 if (sb->sb_cc > 0) | 1148 if (sbused(sb)) |
| 1149 goto deliver; 1150 if (oresid > uio->uio_resid) 1151 goto out; 1152 error = so->so_error; 1153 if (!(flags & MSG_PEEK)) 1154 so->so_error = 0; 1155 goto out; 1156 } 1157 1158 /* Door is closed. Deliver what is left, if any. */ 1159 if (sb->sb_state & SBS_CANTRCVMORE) { | 1149 goto deliver; 1150 if (oresid > uio->uio_resid) 1151 goto out; 1152 error = so->so_error; 1153 if (!(flags & MSG_PEEK)) 1154 so->so_error = 0; 1155 goto out; 1156 } 1157 1158 /* Door is closed. Deliver what is left, if any. */ 1159 if (sb->sb_state & SBS_CANTRCVMORE) { |
| 1160 if (sb->sb_cc > 0) | 1160 if (sbused(sb)) |
| 1161 goto deliver; 1162 else 1163 goto out; 1164 } 1165 1166 /* Socket buffer is empty and we shall not block. */ | 1161 goto deliver; 1162 else 1163 goto out; 1164 } 1165 1166 /* Socket buffer is empty and we shall not block. */ |
| 1167 if (sb->sb_cc == 0 && | 1167 if (sbused(sb) == 0 && |
| 1168 ((so->so_state & SS_NBIO) || (flags & (MSG_DONTWAIT|MSG_NBIO)))) { 1169 error = EAGAIN; 1170 goto out; 1171 } 1172 1173 /* Socket buffer got some data that we shall deliver now. */ | 1168 ((so->so_state & SS_NBIO) || (flags & (MSG_DONTWAIT|MSG_NBIO)))) { 1169 error = EAGAIN; 1170 goto out; 1171 } 1172 1173 /* Socket buffer got some data that we shall deliver now. */ |
| 1174 if (sb->sb_cc > 0 && !(flags & MSG_WAITALL) && | 1174 if (sbused(sb) && !(flags & MSG_WAITALL) && |
| 1175 ((sb->sb_flags & SS_NBIO) || 1176 (flags & (MSG_DONTWAIT|MSG_NBIO)) || | 1175 ((sb->sb_flags & SS_NBIO) || 1176 (flags & (MSG_DONTWAIT|MSG_NBIO)) || |
| 1177 sb->sb_cc >= sb->sb_lowat || 1178 sb->sb_cc >= uio->uio_resid || 1179 sb->sb_cc >= sb->sb_hiwat) ) { | 1177 sbused(sb) >= sb->sb_lowat || 1178 sbused(sb) >= uio->uio_resid || 1179 sbused(sb) >= sb->sb_hiwat) ) { |
| 1180 goto deliver; 1181 } 1182 1183 /* On MSG_WAITALL we must wait until all data or error arrives. */ 1184 if ((flags & MSG_WAITALL) && | 1180 goto deliver; 1181 } 1182 1183 /* On MSG_WAITALL we must wait until all data or error arrives. */ 1184 if ((flags & MSG_WAITALL) && |
| 1185 (sb->sb_cc >= uio->uio_resid || sb->sb_cc >= sb->sb_lowat)) | 1185 (sbused(sb) >= uio->uio_resid || sbused(sb) >= sb->sb_lowat)) |
| 1186 goto deliver; 1187 1188 /* 1189 * Wait and block until (more) data comes in. 1190 * NB: Drops the sockbuf lock during wait. 1191 */ 1192 error = sbwait(sb); 1193 if (error) { 1194 if (sb->sb_flags & SB_DDP_INDICATE && !ddp_handled) { 1195 (void) handle_ddp(so, uio, flags, 1); 1196 ddp_handled = 1; 1197 } 1198 goto out; 1199 } 1200 goto restart; 1201 1202deliver: 1203 SOCKBUF_LOCK_ASSERT(&so->so_rcv); | 1186 goto deliver; 1187 1188 /* 1189 * Wait and block until (more) data comes in. 1190 * NB: Drops the sockbuf lock during wait. 1191 */ 1192 error = sbwait(sb); 1193 if (error) { 1194 if (sb->sb_flags & SB_DDP_INDICATE && !ddp_handled) { 1195 (void) handle_ddp(so, uio, flags, 1); 1196 ddp_handled = 1; 1197 } 1198 goto out; 1199 } 1200 goto restart; 1201 1202deliver: 1203 SOCKBUF_LOCK_ASSERT(&so->so_rcv); |
| 1204 KASSERT(sb->sb_cc > 0, ("%s: sockbuf empty", __func__)); | 1204 KASSERT(sbused(sb) > 0, ("%s: sockbuf empty", __func__)); |
| 1205 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb == NULL", __func__)); 1206 1207 if (sb->sb_flags & SB_DDP_INDICATE && !ddp_handled) 1208 goto restart; 1209 1210 /* Statistics. */ 1211 if (uio->uio_td) 1212 uio->uio_td->td_ru.ru_msgrcv++; 1213 1214 /* Fill uio until full or current end of socket buffer is reached. */ | 1205 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb == NULL", __func__)); 1206 1207 if (sb->sb_flags & SB_DDP_INDICATE && !ddp_handled) 1208 goto restart; 1209 1210 /* Statistics. */ 1211 if (uio->uio_td) 1212 uio->uio_td->td_ru.ru_msgrcv++; 1213 1214 /* Fill uio until full or current end of socket buffer is reached. */ |
| 1215 len = min(uio->uio_resid, sb->sb_cc); | 1215 len = min(uio->uio_resid, sbused(sb)); |
| 1216 if (mp0 != NULL) { 1217 /* Dequeue as many mbufs as possible. */ 1218 if (!(flags & MSG_PEEK) && len >= sb->sb_mb->m_len) { 1219 for (*mp0 = m = sb->sb_mb; 1220 m != NULL && m->m_len <= len; 1221 m = m->m_next) { 1222 len -= m->m_len; 1223 uio->uio_resid -= m->m_len; --- 73 unchanged lines hidden --- | 1216 if (mp0 != NULL) { 1217 /* Dequeue as many mbufs as possible. */ 1218 if (!(flags & MSG_PEEK) && len >= sb->sb_mb->m_len) { 1219 for (*mp0 = m = sb->sb_mb; 1220 m != NULL && m->m_len <= len; 1221 m = m->m_next) { 1222 len -= m->m_len; 1223 uio->uio_resid -= m->m_len; --- 73 unchanged lines hidden --- |