1341572Sslavash/*- 2341572Sslavash * Copyright (c) 2017 Mellanox Technologies. All rights reserved. 3341572Sslavash * 4341572Sslavash * This software is available to you under a choice of one of two 5341572Sslavash * licenses. You may choose to be licensed under the terms of the GNU 6341572Sslavash * General Public License (GPL) Version 2, available from the file 7341572Sslavash * COPYING in the main directory of this source tree, or the 8341572Sslavash * OpenIB.org BSD license below: 9341572Sslavash * 10341572Sslavash * Redistribution and use in source and binary forms, with or 11341572Sslavash * without modification, are permitted provided that the following 12341572Sslavash * conditions are met: 13341572Sslavash * 14341572Sslavash * - Redistributions of source code must retain the above 15341572Sslavash * copyright notice, this list of conditions and the following 16341572Sslavash * disclaimer. 17341572Sslavash * 18341572Sslavash * - Redistributions in binary form must reproduce the above 19341572Sslavash * copyright notice, this list of conditions and the following 20341572Sslavash * disclaimer in the documentation and/or other materials 21341572Sslavash * provided with the distribution. 22341572Sslavash * 23341572Sslavash * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24341572Sslavash * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25341572Sslavash * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26341572Sslavash * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27341572Sslavash * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28341572Sslavash * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29341572Sslavash * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30341572Sslavash * SOFTWARE. 31341572Sslavash * 32341572Sslavash * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_fpga/mlx5fpga_xfer.c 341572 2018-12-05 14:11:20Z slavash $ 33341572Sslavash */ 34341572Sslavash 35341572Sslavash#include <dev/mlx5/mlx5_fpga/xfer.h> 36341572Sslavash#include <dev/mlx5/mlx5_fpga/conn.h> 37341572Sslavash 38341572Sslavashstruct xfer_state { 39341572Sslavash const struct mlx5_fpga_transaction *xfer; 40341572Sslavash /* Total transactions */ 41341572Sslavash unsigned int start_count; 42341572Sslavash unsigned int done_count; 43341572Sslavash unsigned int error_count; 44341572Sslavash u8 status; 45341572Sslavash /* Inflight transactions */ 46341572Sslavash unsigned int budget; 47341572Sslavash unsigned int inflight_count; 48341572Sslavash /* Chunking state */ 49341572Sslavash size_t pos; 50341572Sslavash spinlock_t lock; /* Protects all members of this struct */ 51341572Sslavash}; 52341572Sslavash 53341572Sslavashstruct xfer_transaction { 54341572Sslavash struct xfer_state *xfer_state; 55341572Sslavash struct mlx5_fpga_transaction transaction; 56341572Sslavash}; 57341572Sslavash 58341572Sslavashstatic void trans_complete(const struct mlx5_fpga_transaction *complete, 59341572Sslavash u8 status); 60341572Sslavash 61341572Sslavashstatic void xfer_complete(struct xfer_state *xfer_state) 62341572Sslavash{ 63341572Sslavash const struct mlx5_fpga_transaction *xfer = xfer_state->xfer; 64341572Sslavash u8 status = xfer_state->status; 65341572Sslavash 66341572Sslavash kfree(xfer_state); 67341572Sslavash xfer->complete1(xfer, status); 68341572Sslavash} 69341572Sslavash 70341572Sslavash/* Xfer state spin lock must be locked */ 71341572Sslavashstatic int exec_more(struct xfer_state *xfer_state) 72341572Sslavash{ 73341572Sslavash struct xfer_transaction *xfer_trans; 74341572Sslavash size_t left, cur_size, page_size; 75341572Sslavash u64 pos_addr, ddr_base; 76341572Sslavash u8 *pos_data; 77341572Sslavash int ret = 0; 78341572Sslavash 79341572Sslavash ddr_base = mlx5_fpga_ddr_base_get(xfer_state->xfer->conn->fdev); 80341572Sslavash page_size = (xfer_state->xfer->addr + xfer_state->pos < ddr_base) ? 81341572Sslavash sizeof(u32) : (1 << MLX5_FPGA_TRANSACTION_SEND_PAGE_BITS); 82341572Sslavash 83341572Sslavash do { 84341572Sslavash if (xfer_state->status != IB_WC_SUCCESS) { 85341572Sslavash ret = -EIO; 86341572Sslavash break; 87341572Sslavash } 88341572Sslavash 89341572Sslavash left = xfer_state->xfer->size - xfer_state->pos; 90341572Sslavash if (!left) 91341572Sslavash break; 92341572Sslavash 93341572Sslavash xfer_trans = kzalloc(sizeof(*xfer_trans), GFP_ATOMIC); 94341572Sslavash if (!xfer_trans) { 95341572Sslavash ret = -ENOMEM; 96341572Sslavash break; 97341572Sslavash } 98341572Sslavash 99341572Sslavash pos_addr = xfer_state->xfer->addr + xfer_state->pos; 100341572Sslavash pos_data = xfer_state->xfer->data + xfer_state->pos; 101341572Sslavash 102341572Sslavash /* Determine largest possible transaction at this point */ 103341572Sslavash cur_size = page_size - (pos_addr & (page_size - 1)); 104341572Sslavash if (cur_size > MLX5_FPGA_TRANSACTION_MAX_SIZE) 105341572Sslavash cur_size = MLX5_FPGA_TRANSACTION_MAX_SIZE; 106341572Sslavash if (cur_size > left) 107341572Sslavash cur_size = left; 108341572Sslavash 109341572Sslavash xfer_trans->xfer_state = xfer_state; 110341572Sslavash xfer_trans->transaction.addr = pos_addr; 111341572Sslavash xfer_trans->transaction.complete1 = trans_complete; 112341572Sslavash xfer_trans->transaction.conn = xfer_state->xfer->conn; 113341572Sslavash xfer_trans->transaction.data = pos_data; 114341572Sslavash xfer_trans->transaction.direction = xfer_state->xfer->direction; 115341572Sslavash xfer_trans->transaction.size = cur_size; 116341572Sslavash 117341572Sslavash xfer_state->start_count++; 118341572Sslavash xfer_state->inflight_count++; 119341572Sslavash mlx5_fpga_dbg(xfer_state->xfer->conn->fdev, "Starting %zu bytes at %p done; %u started %u inflight %u done %u error\n", 120341572Sslavash xfer_trans->transaction.size, 121341572Sslavash xfer_trans->transaction.data, 122341572Sslavash xfer_state->start_count, 123341572Sslavash xfer_state->inflight_count, 124341572Sslavash xfer_state->done_count, 125341572Sslavash xfer_state->error_count); 126341572Sslavash ret = mlx5_fpga_trans_exec(&xfer_trans->transaction); 127341572Sslavash if (ret) { 128341572Sslavash xfer_state->start_count--; 129341572Sslavash xfer_state->inflight_count--; 130341572Sslavash if (ret == -EBUSY) 131341572Sslavash ret = 0; 132341572Sslavash 133341572Sslavash if (ret) { 134341572Sslavash mlx5_fpga_warn(xfer_state->xfer->conn->fdev, "Transfer failed to start transaction: %d. %u started %u done %u error\n", 135341572Sslavash ret, xfer_state->start_count, 136341572Sslavash xfer_state->done_count, 137341572Sslavash xfer_state->error_count); 138341572Sslavash xfer_state->status = IB_WC_GENERAL_ERR; 139341572Sslavash } 140341572Sslavash kfree(xfer_trans); 141341572Sslavash break; 142341572Sslavash } 143341572Sslavash xfer_state->pos += cur_size; 144341572Sslavash if (xfer_state->inflight_count >= xfer_state->budget) 145341572Sslavash break; 146341572Sslavash } while (cur_size != left); 147341572Sslavash 148341572Sslavash return ret; 149341572Sslavash} 150341572Sslavash 151341572Sslavashstatic void trans_complete(const struct mlx5_fpga_transaction *complete, 152341572Sslavash u8 status) 153341572Sslavash{ 154341572Sslavash struct xfer_transaction *xfer_trans; 155341572Sslavash struct xfer_state *xfer_state; 156341572Sslavash unsigned long flags; 157341572Sslavash bool done = false; 158341572Sslavash int ret; 159341572Sslavash 160341572Sslavash xfer_trans = container_of(complete, struct xfer_transaction, 161341572Sslavash transaction); 162341572Sslavash xfer_state = xfer_trans->xfer_state; 163341572Sslavash mlx5_fpga_dbg(complete->conn->fdev, "Transaction %zu bytes at %p done, status %u; %u started %u inflight %u done %u error\n", 164341572Sslavash xfer_trans->transaction.size, 165341572Sslavash xfer_trans->transaction.data, status, 166341572Sslavash xfer_state->start_count, xfer_state->inflight_count, 167341572Sslavash xfer_state->done_count, xfer_state->error_count); 168341572Sslavash kfree(xfer_trans); 169341572Sslavash 170341572Sslavash spin_lock_irqsave(&xfer_state->lock, flags); 171341572Sslavash 172341572Sslavash if (status != IB_WC_SUCCESS) { 173341572Sslavash xfer_state->error_count++; 174341572Sslavash mlx5_fpga_warn(complete->conn->fdev, "Transaction failed during transfer. %u started %u inflight %u done %u error\n", 175341572Sslavash xfer_state->start_count, 176341572Sslavash xfer_state->inflight_count, 177341572Sslavash xfer_state->done_count, xfer_state->error_count); 178341572Sslavash if (xfer_state->status == IB_WC_SUCCESS) 179341572Sslavash xfer_state->status = status; 180341572Sslavash } else { 181341572Sslavash xfer_state->done_count++; 182341572Sslavash } 183341572Sslavash ret = exec_more(xfer_state); 184341572Sslavash 185341572Sslavash xfer_state->inflight_count--; 186341572Sslavash if (!xfer_state->inflight_count) 187341572Sslavash done = true; 188341572Sslavash 189341572Sslavash spin_unlock_irqrestore(&xfer_state->lock, flags); 190341572Sslavash 191341572Sslavash if (done) 192341572Sslavash xfer_complete(xfer_state); 193341572Sslavash} 194341572Sslavash 195341572Sslavashint mlx5_fpga_xfer_exec(const struct mlx5_fpga_transaction *xfer) 196341572Sslavash{ 197341572Sslavash u64 base = mlx5_fpga_ddr_base_get(xfer->conn->fdev); 198341572Sslavash u64 size = mlx5_fpga_ddr_size_get(xfer->conn->fdev); 199341572Sslavash struct xfer_state *xfer_state; 200341572Sslavash unsigned long flags; 201341572Sslavash bool done = false; 202341572Sslavash int ret = 0; 203341572Sslavash 204341572Sslavash if (xfer->addr + xfer->size > base + size) { 205341572Sslavash mlx5_fpga_warn(xfer->conn->fdev, "Transfer ends at %jx outside of DDR range %jx\n", 206341572Sslavash (uintmax_t)(xfer->addr + xfer->size), (uintmax_t)(base + size)); 207341572Sslavash return -EINVAL; 208341572Sslavash } 209341572Sslavash 210341572Sslavash if (xfer->addr & MLX5_FPGA_TRANSACTION_SEND_ALIGN_BITS) { 211341572Sslavash mlx5_fpga_warn(xfer->conn->fdev, "Transfer address %jx not aligned\n", 212341572Sslavash (uintmax_t)xfer->addr); 213341572Sslavash return -EINVAL; 214341572Sslavash } 215341572Sslavash 216341572Sslavash if (xfer->size & MLX5_FPGA_TRANSACTION_SEND_ALIGN_BITS) { 217341572Sslavash mlx5_fpga_warn(xfer->conn->fdev, "Transfer size %zu not aligned\n", 218341572Sslavash xfer->size); 219341572Sslavash return -EINVAL; 220341572Sslavash } 221341572Sslavash 222341572Sslavash if (xfer->size < 1) { 223341572Sslavash mlx5_fpga_warn(xfer->conn->fdev, "Empty transfer size %zu not allowed\n", 224341572Sslavash xfer->size); 225341572Sslavash return -EINVAL; 226341572Sslavash } 227341572Sslavash 228341572Sslavash xfer_state = kzalloc(sizeof(*xfer_state), GFP_KERNEL); 229341572Sslavash xfer_state->xfer = xfer; 230341572Sslavash xfer_state->status = IB_WC_SUCCESS; 231341572Sslavash xfer_state->budget = 7; 232341572Sslavash spin_lock_init(&xfer_state->lock); 233341572Sslavash spin_lock_irqsave(&xfer_state->lock, flags); 234341572Sslavash 235341572Sslavash ret = exec_more(xfer_state); 236341572Sslavash if (ret && (xfer_state->start_count == 0)) 237341572Sslavash done = true; 238341572Sslavash 239341572Sslavash spin_unlock_irqrestore(&xfer_state->lock, flags); 240341572Sslavash 241341572Sslavash if (done) 242341572Sslavash xfer_complete(xfer_state); 243341572Sslavash return ret; 244341572Sslavash} 245