Deleted Added
sdiff udiff text old ( 346876 ) new ( 346923 )
full compact
mem.c (346876) mem.c (346923)
1/*
2 * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:

--- 16 unchanged lines hidden (view full) ---

25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <sys/cdefs.h>
1/*
2 * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:

--- 16 unchanged lines hidden (view full) ---

25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD: stable/11/sys/dev/cxgbe/iw_cxgbe/mem.c 346876 2019-04-29 04:42:18Z np $");
33__FBSDID("$FreeBSD: stable/11/sys/dev/cxgbe/iw_cxgbe/mem.c 346923 2019-04-29 20:10:28Z np $");
34
35#include "opt_inet.h"
36
37#ifdef TCP_OFFLOAD
38#include <linux/types.h>
39#include <linux/kref.h>
40#include <rdma/ib_umem.h>
41#include <asm/atomic.h>
42
43#include <common/t4_msg.h>
44#include "iw_cxgbe.h"
45
34
35#include "opt_inet.h"
36
37#ifdef TCP_OFFLOAD
38#include <linux/types.h>
39#include <linux/kref.h>
40#include <rdma/ib_umem.h>
41#include <asm/atomic.h>
42
43#include <common/t4_msg.h>
44#include "iw_cxgbe.h"
45
46int use_dsgl = 1;
47#define T4_ULPTX_MIN_IO 32
48#define C4IW_MAX_INLINE_SIZE 96
46#define T4_ULPTX_MIN_IO 32
47#define C4IW_MAX_INLINE_SIZE 96
48#define T4_ULPTX_MAX_DMA 1024
49
50static int
51mr_exceeds_hw_limits(struct c4iw_dev *dev, u64 length)
52{
53
54 return (is_t5(dev->rdev.adap) && length >= 8*1024*1024*1024ULL);
55}
56
57static int
49
50static int
51mr_exceeds_hw_limits(struct c4iw_dev *dev, u64 length)
52{
53
54 return (is_t5(dev->rdev.adap) && length >= 8*1024*1024*1024ULL);
55}
56
57static int
58write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data)
58_c4iw_write_mem_dma_aligned(struct c4iw_rdev *rdev, u32 addr, u32 len,
59 void *data, int wait)
59{
60 struct adapter *sc = rdev->adap;
61 struct ulp_mem_io *ulpmc;
60{
61 struct adapter *sc = rdev->adap;
62 struct ulp_mem_io *ulpmc;
63 struct ulptx_sgl *sgl;
64 u8 wr_len;
65 int ret = 0;
66 struct c4iw_wr_wait wr_wait;
67 struct wrqe *wr;
68
69 addr &= 0x7FFFFFF;
70
71 if (wait)
72 c4iw_init_wr_wait(&wr_wait);
73 wr_len = roundup(sizeof *ulpmc + sizeof *sgl, 16);
74
75 wr = alloc_wrqe(wr_len, &sc->sge.ctrlq[0]);
76 if (wr == NULL)
77 return -ENOMEM;
78 ulpmc = wrtod(wr);
79
80 memset(ulpmc, 0, wr_len);
81 INIT_ULPTX_WR(ulpmc, wr_len, 0, 0);
82 ulpmc->wr.wr_hi = cpu_to_be32(V_FW_WR_OP(FW_ULPTX_WR) |
83 (wait ? F_FW_WR_COMPL : 0));
84 ulpmc->wr.wr_lo = wait ? (u64)(unsigned long)&wr_wait : 0;
85 ulpmc->wr.wr_mid = cpu_to_be32(V_FW_WR_LEN16(DIV_ROUND_UP(wr_len, 16)));
86 ulpmc->cmd = cpu_to_be32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) |
87 V_T5_ULP_MEMIO_ORDER(1) |
88 V_T5_ULP_MEMIO_FID(sc->sge.ofld_rxq[0].iq.abs_id));
89 ulpmc->dlen = cpu_to_be32(V_ULP_MEMIO_DATA_LEN(len>>5));
90 ulpmc->len16 = cpu_to_be32(DIV_ROUND_UP(wr_len-sizeof(ulpmc->wr), 16));
91 ulpmc->lock_addr = cpu_to_be32(V_ULP_MEMIO_ADDR(addr));
92
93 sgl = (struct ulptx_sgl *)(ulpmc + 1);
94 sgl->cmd_nsge = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
95 V_ULPTX_NSGE(1));
96 sgl->len0 = cpu_to_be32(len);
97 sgl->addr0 = cpu_to_be64((u64)data);
98
99 t4_wrq_tx(sc, wr);
100
101 if (wait)
102 ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, NULL, __func__);
103 return ret;
104}
105
106
107static int
108_c4iw_write_mem_inline(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data)
109{
110 struct adapter *sc = rdev->adap;
111 struct ulp_mem_io *ulpmc;
62 struct ulptx_idata *ulpsc;
63 u8 wr_len, *to_dp, *from_dp;
64 int copy_len, num_wqe, i, ret = 0;
65 struct c4iw_wr_wait wr_wait;
66 struct wrqe *wr;
67 u32 cmd;
68
69 cmd = cpu_to_be32(V_ULPTX_CMD(ULP_TX_MEM_WRITE));

--- 7 unchanged lines hidden (view full) ---

77 for (i = 0; i < num_wqe; i++) {
78
79 copy_len = min(len, C4IW_MAX_INLINE_SIZE);
80 wr_len = roundup(sizeof *ulpmc + sizeof *ulpsc +
81 roundup(copy_len, T4_ULPTX_MIN_IO), 16);
82
83 wr = alloc_wrqe(wr_len, &sc->sge.ctrlq[0]);
84 if (wr == NULL)
112 struct ulptx_idata *ulpsc;
113 u8 wr_len, *to_dp, *from_dp;
114 int copy_len, num_wqe, i, ret = 0;
115 struct c4iw_wr_wait wr_wait;
116 struct wrqe *wr;
117 u32 cmd;
118
119 cmd = cpu_to_be32(V_ULPTX_CMD(ULP_TX_MEM_WRITE));

--- 7 unchanged lines hidden (view full) ---

127 for (i = 0; i < num_wqe; i++) {
128
129 copy_len = min(len, C4IW_MAX_INLINE_SIZE);
130 wr_len = roundup(sizeof *ulpmc + sizeof *ulpsc +
131 roundup(copy_len, T4_ULPTX_MIN_IO), 16);
132
133 wr = alloc_wrqe(wr_len, &sc->sge.ctrlq[0]);
134 if (wr == NULL)
85 return (0);
135 return -ENOMEM;
86 ulpmc = wrtod(wr);
87
88 memset(ulpmc, 0, wr_len);
89 INIT_ULPTX_WR(ulpmc, wr_len, 0, 0);
90
91 if (i == (num_wqe-1)) {
92 ulpmc->wr.wr_hi = cpu_to_be32(V_FW_WR_OP(FW_ULPTX_WR) |
93 F_FW_WR_COMPL);
136 ulpmc = wrtod(wr);
137
138 memset(ulpmc, 0, wr_len);
139 INIT_ULPTX_WR(ulpmc, wr_len, 0, 0);
140
141 if (i == (num_wqe-1)) {
142 ulpmc->wr.wr_hi = cpu_to_be32(V_FW_WR_OP(FW_ULPTX_WR) |
143 F_FW_WR_COMPL);
94 ulpmc->wr.wr_lo = (__force __be64)(unsigned long) &wr_wait;
144 ulpmc->wr.wr_lo =
145 (__force __be64)(unsigned long) &wr_wait;
95 } else
96 ulpmc->wr.wr_hi = cpu_to_be32(V_FW_WR_OP(FW_ULPTX_WR));
97 ulpmc->wr.wr_mid = cpu_to_be32(
98 V_FW_WR_LEN16(DIV_ROUND_UP(wr_len, 16)));
99
100 ulpmc->cmd = cmd;
101 ulpmc->dlen = cpu_to_be32(V_ULP_MEMIO_DATA_LEN(
102 DIV_ROUND_UP(copy_len, T4_ULPTX_MIN_IO)));

--- 17 unchanged lines hidden (view full) ---

120 t4_wrq_tx(sc, wr);
121 len -= C4IW_MAX_INLINE_SIZE;
122 }
123
124 ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, NULL, __func__);
125 return ret;
126}
127
146 } else
147 ulpmc->wr.wr_hi = cpu_to_be32(V_FW_WR_OP(FW_ULPTX_WR));
148 ulpmc->wr.wr_mid = cpu_to_be32(
149 V_FW_WR_LEN16(DIV_ROUND_UP(wr_len, 16)));
150
151 ulpmc->cmd = cmd;
152 ulpmc->dlen = cpu_to_be32(V_ULP_MEMIO_DATA_LEN(
153 DIV_ROUND_UP(copy_len, T4_ULPTX_MIN_IO)));

--- 17 unchanged lines hidden (view full) ---

171 t4_wrq_tx(sc, wr);
172 len -= C4IW_MAX_INLINE_SIZE;
173 }
174
175 ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, NULL, __func__);
176 return ret;
177}
178
179static int
180_c4iw_write_mem_dma(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data)
181{
182 struct c4iw_dev *rhp = rdev_to_c4iw_dev(rdev);
183 u32 remain = len;
184 u32 dmalen;
185 int ret = 0;
186 dma_addr_t daddr;
187 dma_addr_t save;
188
189 daddr = dma_map_single(rhp->ibdev.dma_device, data, len, DMA_TO_DEVICE);
190 if (dma_mapping_error(rhp->ibdev.dma_device, daddr))
191 return -1;
192 save = daddr;
193
194 while (remain > inline_threshold) {
195 if (remain < T4_ULPTX_MAX_DMA) {
196 if (remain & ~T4_ULPTX_MIN_IO)
197 dmalen = remain & ~(T4_ULPTX_MIN_IO-1);
198 else
199 dmalen = remain;
200 } else
201 dmalen = T4_ULPTX_MAX_DMA;
202 remain -= dmalen;
203 ret = _c4iw_write_mem_dma_aligned(rdev, addr, dmalen,
204 (void *)daddr, !remain);
205 if (ret)
206 goto out;
207 addr += dmalen >> 5;
208 data = (u64 *)data + dmalen;
209 daddr = daddr + dmalen;
210 }
211 if (remain)
212 ret = _c4iw_write_mem_inline(rdev, addr, remain, data);
213out:
214 dma_unmap_single(rhp->ibdev.dma_device, save, len, DMA_TO_DEVICE);
215 return ret;
216}
217
128/*
218/*
219 * write len bytes of data into addr (32B aligned address)
220 * If data is NULL, clear len byte of memory to zero.
221 */
222static int
223write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len,
224 void *data)
225{
226 if (rdev->adap->params.ulptx_memwrite_dsgl && use_dsgl) {
227 if (len > inline_threshold) {
228 if (_c4iw_write_mem_dma(rdev, addr, len, data)) {
229 log(LOG_ERR, "%s: dma map "
230 "failure (non fatal)\n", __func__);
231 return _c4iw_write_mem_inline(rdev, addr, len,
232 data);
233 } else
234 return 0;
235 } else
236 return _c4iw_write_mem_inline(rdev, addr, len, data);
237 } else
238 return _c4iw_write_mem_inline(rdev, addr, len, data);
239}
240
241
242/*
129 * Build and write a TPT entry.
130 * IN: stag key, pdid, perm, bind_enabled, zbva, to, len, page_size,
131 * pbl_size and pbl_addr
132 * OUT: stag index
133 */
134static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry,
135 u32 *stag, u8 stag_state, u32 pdid,
136 enum fw_ri_stag_type type, enum fw_ri_mem_perms perm,

--- 487 unchanged lines hidden --- 		243 * Build and write a TPT entry.
244 * IN: stag key, pdid, perm, bind_enabled, zbva, to, len, page_size,
245 * pbl_size and pbl_addr
246 * OUT: stag index
247 */
248static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry,
249 u32 *stag, u8 stag_state, u32 pdid,
250 enum fw_ri_stag_type type, enum fw_ri_mem_perms perm,

--- 487 unchanged lines hidden ---