Lines Matching refs:ic
66 static void rds_iw_send_unmap_rdma(struct rds_iw_connection *ic,
70 ib_dma_unmap_sg(ic->i_cm_id->device,
77 static void rds_iw_send_unmap_rm(struct rds_iw_connection *ic,
83 rdsdebug("ic %p send %p rm %p\n", ic, send, rm);
85 ib_dma_unmap_sg(ic->i_cm_id->device,
90 rds_iw_send_unmap_rdma(ic, rm->m_rdma_op);
128 void rds_iw_send_init_ring(struct rds_iw_connection *ic)
133 for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {
148 sge = rds_iw_data_sge(ic, send->s_sge);
151 sge = rds_iw_header_sge(ic, send->s_sge);
152 sge->addr = ic->i_send_hdrs_dma + (i * sizeof(struct rds_header));
156 send->s_mr = ib_alloc_fast_reg_mr(ic->i_pd, fastreg_message_size);
163 ic->i_cm_id->device, fastreg_message_size);
171 void rds_iw_send_clear_ring(struct rds_iw_connection *ic)
176 for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {
184 rds_iw_send_unmap_rm(ic, send, IB_WC_WR_FLUSH_ERR);
186 rds_iw_send_unmap_rdma(ic, send->s_op);
199 struct rds_iw_connection *ic = conn->c_transport_data;
225 ic->i_fastreg_posted = 0;
230 ic->i_fastreg_posted = 1;
235 if (ic->i_ack_queued + HZ/2 < jiffies)
237 rds_iw_ack_send_complete(ic);
241 oldest = rds_iw_ring_oldest(&ic->i_send_ring);
243 completed = rds_iw_ring_completed(&ic->i_send_ring, wc.wr_id, oldest);
246 send = &ic->i_sends[oldest];
252 rds_iw_send_unmap_rm(ic, send, wc.status);
286 oldest = (oldest + 1) % ic->i_send_ring.w_nr;
289 rds_iw_ring_free(&ic->i_send_ring, completed);
349 int rds_iw_send_grab_credits(struct rds_iw_connection *ic,
356 if (!ic->i_flowctl)
361 oldval = newval = atomic_read(&ic->i_credits);
373 struct rds_connection *conn = ic->i_cm_id->context;
395 if (atomic_cmpxchg(&ic->i_credits, oldval, newval) != oldval)
404 struct rds_iw_connection *ic = conn->c_transport_data;
411 IB_GET_SEND_CREDITS(atomic_read(&ic->i_credits)),
414 atomic_add(IB_SET_SEND_CREDITS(credits), &ic->i_credits);
425 struct rds_iw_connection *ic = conn->c_transport_data;
430 atomic_add(IB_SET_POST_CREDITS(posted), &ic->i_credits);
444 if (IB_GET_POST_CREDITS(atomic_read(&ic->i_credits)) >= 16)
445 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
449 rds_iw_xmit_populate_wr(struct rds_iw_connection *ic,
456 WARN_ON(pos != send - ic->i_sends);
466 sge = rds_iw_data_sge(ic, send->s_sge);
469 sge->lkey = rds_iw_local_dma_lkey(ic);
471 sge = rds_iw_header_sge(ic, send->s_sge);
479 sge->addr = ic->i_send_hdrs_dma + (pos * sizeof(struct rds_header));
481 sge->lkey = rds_iw_local_dma_lkey(ic);
500 struct rds_iw_connection *ic = conn->c_transport_data;
501 struct ib_device *dev = ic->i_cm_id->device;
522 if (rds_rdma_cookie_key(rm->m_rdma_cookie) && !ic->i_fastreg_posted) {
532 work_alloc = rds_iw_ring_alloc(&ic->i_send_ring, i, &pos);
541 if (ic->i_flowctl) {
542 credit_alloc = rds_iw_send_grab_credits(ic, work_alloc, &posted, 0, RDS_MAX_ADV_CREDIT);
545 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc - credit_alloc);
558 if (ic->i_rm == NULL) {
568 rdsdebug("ic %p mapping rm %p: %d\n", ic, rm, rm->m_count);
571 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);
579 ic->i_unsignaled_wrs = rds_iw_sysctl_max_unsig_wrs;
580 ic->i_unsignaled_bytes = rds_iw_sysctl_max_unsig_bytes;
582 ic->i_rm = rm;
609 rm->m_inc.i_hdr.h_ack = cpu_to_be64(rds_iw_piggyb_ack(ic));
615 rds_iw_send_grab_credits(ic, 0, &posted, 1, RDS_MAX_ADV_CREDIT - adv_credits);
620 send = &ic->i_sends[pos];
647 rds_iw_xmit_populate_wr(ic, send, pos, 0, 0, send_flags);
655 send = &ic->i_sends[pos];
658 rds_iw_xmit_populate_wr(ic, send, pos,
667 if (ic->i_unsignaled_wrs-- == 0) {
668 ic->i_unsignaled_wrs = rds_iw_sysctl_max_unsig_wrs;
672 ic->i_unsignaled_bytes -= len;
673 if (ic->i_unsignaled_bytes <= 0) {
674 ic->i_unsignaled_bytes = rds_iw_sysctl_max_unsig_bytes;
697 memcpy(&ic->i_send_hdrs[pos], &rm->m_inc.i_hdr, sizeof(struct rds_header));
700 struct rds_header *hdr = &ic->i_send_hdrs[pos];
708 struct rds_header *hdr = &ic->i_send_hdrs[pos];
721 pos = (pos + 1) % ic->i_send_ring.w_nr;
731 prev->s_rm = ic->i_rm;
733 ic->i_rm = NULL;
737 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc - i);
740 if (ic->i_flowctl && i < credit_alloc)
744 ret = ib_post_send(ic->i_cm_id->qp, &first->s_wr, &failed_wr);
745 rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,
751 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);
753 ic->i_rm = prev->s_rm;
765 static void rds_iw_build_send_fastreg(struct rds_iw_device *rds_iwdev, struct rds_iw_connection *ic, struct rds_iw_send_work *send, int nent, int len, u64 sg_addr)
787 struct rds_iw_connection *ic = conn->c_transport_data;
804 rds_iwdev = ib_get_client_data(ic->i_cm_id->device, &rds_iw_client);
808 op->r_count = ib_dma_map_sg(ic->i_cm_id->device,
811 rdsdebug("ic %p mapping op %p: %d\n", ic, op, op->r_count);
823 work_alloc = rds_iw_ring_alloc(&ic->i_send_ring, 1, &fr_pos);
825 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);
838 work_alloc = rds_iw_ring_alloc(&ic->i_send_ring, i, &pos);
840 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);
846 send = &ic->i_sends[pos];
848 first = prev = &ic->i_sends[fr_pos];
865 if (ic->i_unsignaled_wrs-- == 0) {
866 ic->i_unsignaled_wrs = rds_iw_sysctl_max_unsig_wrs;
895 len = ib_sg_dma_len(ic->i_cm_id->device, scat);
898 send->s_page_list->page_list[j] = ib_sg_dma_address(ic->i_cm_id->device, scat);
900 send->s_sge[j].addr = ib_sg_dma_address(ic->i_cm_id->device, scat);
902 send->s_sge[j].lkey = rds_iw_local_dma_lkey(ic);
906 rdsdebug("ic %p sent %d remote_addr %llu\n", ic, sent, remote_addr);
916 send->s_sge[0].lkey = ic->i_sends[fr_pos].s_mr->lkey;
923 if (++send == &ic->i_sends[ic->i_send_ring.w_nr])
924 send = ic->i_sends;
932 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc - i);
943 rds_iw_build_send_fastreg(rds_iwdev, ic, &ic->i_sends[fr_pos],
949 ret = ib_post_send(ic->i_cm_id->qp, &first->s_wr, &failed_wr);
950 rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,
956 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);
966 struct rds_iw_connection *ic = conn->c_transport_data;
970 rds_iw_attempt_ack(ic);