1/*- 2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <dev/mlx5/mlx5_fpga/xfer.h> 34#include <dev/mlx5/mlx5_fpga/conn.h> 35 36struct xfer_state { 37 const struct mlx5_fpga_transaction *xfer; 38 /* Total transactions */ 39 unsigned int start_count; 40 unsigned int done_count; 41 unsigned int error_count; 42 u8 status; 43 /* Inflight transactions */ 44 unsigned int budget; 45 unsigned int inflight_count; 46 /* Chunking state */ 47 size_t pos; 48 spinlock_t lock; /* Protects all members of this struct */ 49}; 50 51struct xfer_transaction { 52 struct xfer_state *xfer_state; 53 struct mlx5_fpga_transaction transaction; 54}; 55 56static void trans_complete(const struct mlx5_fpga_transaction *complete, 57 u8 status); 58 59static void xfer_complete(struct xfer_state *xfer_state) 60{ 61 const struct mlx5_fpga_transaction *xfer = xfer_state->xfer; 62 u8 status = xfer_state->status; 63 64 kfree(xfer_state); 65 xfer->complete1(xfer, status); 66} 67 68/* Xfer state spin lock must be locked */ 69static int exec_more(struct xfer_state *xfer_state) 70{ 71 struct xfer_transaction *xfer_trans; 72 size_t left, cur_size, page_size; 73 u64 pos_addr, ddr_base; 74 u8 *pos_data; 75 int ret = 0; 76 77 ddr_base = mlx5_fpga_ddr_base_get(xfer_state->xfer->conn->fdev); 78 page_size = (xfer_state->xfer->addr + xfer_state->pos < ddr_base) ? 79 sizeof(u32) : (1 << MLX5_FPGA_TRANSACTION_SEND_PAGE_BITS); 80 81 do { 82 if (xfer_state->status != IB_WC_SUCCESS) { 83 ret = -EIO; 84 break; 85 } 86 87 left = xfer_state->xfer->size - xfer_state->pos; 88 if (!left) 89 break; 90 91 xfer_trans = kzalloc(sizeof(*xfer_trans), GFP_ATOMIC); 92 if (!xfer_trans) { 93 ret = -ENOMEM; 94 break; 95 } 96 97 pos_addr = xfer_state->xfer->addr + xfer_state->pos; 98 pos_data = xfer_state->xfer->data + xfer_state->pos; 99 100 /* Determine largest possible transaction at this point */ 101 cur_size = page_size - (pos_addr & (page_size - 1)); 102 if (cur_size > MLX5_FPGA_TRANSACTION_MAX_SIZE) 103 cur_size = MLX5_FPGA_TRANSACTION_MAX_SIZE; 104 if (cur_size > left) 105 cur_size = left; 106 107 xfer_trans->xfer_state = xfer_state; 108 xfer_trans->transaction.addr = pos_addr; 109 xfer_trans->transaction.complete1 = trans_complete; 110 xfer_trans->transaction.conn = xfer_state->xfer->conn; 111 xfer_trans->transaction.data = pos_data; 112 xfer_trans->transaction.direction = xfer_state->xfer->direction; 113 xfer_trans->transaction.size = cur_size; 114 115 xfer_state->start_count++; 116 xfer_state->inflight_count++; 117 mlx5_fpga_dbg(xfer_state->xfer->conn->fdev, "Starting %zu bytes at %p done; %u started %u inflight %u done %u error\n", 118 xfer_trans->transaction.size, 119 xfer_trans->transaction.data, 120 xfer_state->start_count, 121 xfer_state->inflight_count, 122 xfer_state->done_count, 123 xfer_state->error_count); 124 ret = mlx5_fpga_trans_exec(&xfer_trans->transaction); 125 if (ret) { 126 xfer_state->start_count--; 127 xfer_state->inflight_count--; 128 if (ret == -EBUSY) 129 ret = 0; 130 131 if (ret) { 132 mlx5_fpga_warn(xfer_state->xfer->conn->fdev, "Transfer failed to start transaction: %d. %u started %u done %u error\n", 133 ret, xfer_state->start_count, 134 xfer_state->done_count, 135 xfer_state->error_count); 136 xfer_state->status = IB_WC_GENERAL_ERR; 137 } 138 kfree(xfer_trans); 139 break; 140 } 141 xfer_state->pos += cur_size; 142 if (xfer_state->inflight_count >= xfer_state->budget) 143 break; 144 } while (cur_size != left); 145 146 return ret; 147} 148 149static void trans_complete(const struct mlx5_fpga_transaction *complete, 150 u8 status) 151{ 152 struct xfer_transaction *xfer_trans; 153 struct xfer_state *xfer_state; 154 unsigned long flags; 155 bool done = false; 156 int ret; 157 158 xfer_trans = container_of(complete, struct xfer_transaction, 159 transaction); 160 xfer_state = xfer_trans->xfer_state; 161 mlx5_fpga_dbg(complete->conn->fdev, "Transaction %zu bytes at %p done, status %u; %u started %u inflight %u done %u error\n", 162 xfer_trans->transaction.size, 163 xfer_trans->transaction.data, status, 164 xfer_state->start_count, xfer_state->inflight_count, 165 xfer_state->done_count, xfer_state->error_count); 166 kfree(xfer_trans); 167 168 spin_lock_irqsave(&xfer_state->lock, flags); 169 170 if (status != IB_WC_SUCCESS) { 171 xfer_state->error_count++; 172 mlx5_fpga_warn(complete->conn->fdev, "Transaction failed during transfer. %u started %u inflight %u done %u error\n", 173 xfer_state->start_count, 174 xfer_state->inflight_count, 175 xfer_state->done_count, xfer_state->error_count); 176 if (xfer_state->status == IB_WC_SUCCESS) 177 xfer_state->status = status; 178 } else { 179 xfer_state->done_count++; 180 } 181 ret = exec_more(xfer_state); 182 183 xfer_state->inflight_count--; 184 if (!xfer_state->inflight_count) 185 done = true; 186 187 spin_unlock_irqrestore(&xfer_state->lock, flags); 188 189 if (done) 190 xfer_complete(xfer_state); 191} 192 193int mlx5_fpga_xfer_exec(const struct mlx5_fpga_transaction *xfer) 194{ 195 u64 base = mlx5_fpga_ddr_base_get(xfer->conn->fdev); 196 u64 size = mlx5_fpga_ddr_size_get(xfer->conn->fdev); 197 struct xfer_state *xfer_state; 198 unsigned long flags; 199 bool done = false; 200 int ret = 0; 201 202 if (xfer->addr + xfer->size > base + size) { 203 mlx5_fpga_warn(xfer->conn->fdev, "Transfer ends at %jx outside of DDR range %jx\n", 204 (uintmax_t)(xfer->addr + xfer->size), (uintmax_t)(base + size)); 205 return -EINVAL; 206 } 207 208 if (xfer->addr & MLX5_FPGA_TRANSACTION_SEND_ALIGN_BITS) { 209 mlx5_fpga_warn(xfer->conn->fdev, "Transfer address %jx not aligned\n", 210 (uintmax_t)xfer->addr); 211 return -EINVAL; 212 } 213 214 if (xfer->size & MLX5_FPGA_TRANSACTION_SEND_ALIGN_BITS) { 215 mlx5_fpga_warn(xfer->conn->fdev, "Transfer size %zu not aligned\n", 216 xfer->size); 217 return -EINVAL; 218 } 219 220 if (xfer->size < 1) { 221 mlx5_fpga_warn(xfer->conn->fdev, "Empty transfer size %zu not allowed\n", 222 xfer->size); 223 return -EINVAL; 224 } 225 226 xfer_state = kzalloc(sizeof(*xfer_state), GFP_KERNEL); 227 xfer_state->xfer = xfer; 228 xfer_state->status = IB_WC_SUCCESS; 229 xfer_state->budget = 7; 230 spin_lock_init(&xfer_state->lock); 231 spin_lock_irqsave(&xfer_state->lock, flags); 232 233 ret = exec_more(xfer_state); 234 if (ret && (xfer_state->start_count == 0)) 235 done = true; 236 237 spin_unlock_irqrestore(&xfer_state->lock, flags); 238 239 if (done) 240 xfer_complete(xfer_state); 241 return ret; 242} 243