1/* 2 * SCSI RDMA Protocol lib functions 3 * 4 * Copyright (C) 2006 FUJITA Tomonori <tomof@acm.org> 5 * 6 * This program is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU General Public License as 8 * published by the Free Software Foundation; either version 2 of the 9 * License, or (at your option) any later version. 10 * 11 * This program is distributed in the hope that it will be useful, but 12 * WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write to the Free Software 18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 19 * 02110-1301 USA 20 */ 21#include <linux/err.h> 22#include <linux/slab.h> 23#include <linux/kfifo.h> 24#include <linux/scatterlist.h> 25#include <linux/dma-mapping.h> 26#include <scsi/scsi.h> 27#include <scsi/scsi_cmnd.h> 28#include <scsi/scsi_tcq.h> 29#include <scsi/scsi_tgt.h> 30#include <scsi/srp.h> 31#include <scsi/libsrp.h> 32 33enum srp_task_attributes { 34 SRP_SIMPLE_TASK = 0, 35 SRP_HEAD_TASK = 1, 36 SRP_ORDERED_TASK = 2, 37 SRP_ACA_TASK = 4 38}; 39 40/* tmp - will replace with SCSI logging stuff */ 41#define eprintk(fmt, args...) \ 42do { \ 43 printk("%s(%d) " fmt, __func__, __LINE__, ##args); \ 44} while (0) 45/* #define dprintk eprintk */ 46#define dprintk(fmt, args...) 47 48static int srp_iu_pool_alloc(struct srp_queue *q, size_t max, 49 struct srp_buf **ring) 50{ 51 int i; 52 struct iu_entry *iue; 53 54 q->pool = kcalloc(max, sizeof(struct iu_entry *), GFP_KERNEL); 55 if (!q->pool) 56 return -ENOMEM; 57 q->items = kcalloc(max, sizeof(struct iu_entry), GFP_KERNEL); 58 if (!q->items) 59 goto free_pool; 60 61 spin_lock_init(&q->lock); 62 kfifo_init(&q->queue, (void *) q->pool, max * sizeof(void *)); 63 64 for (i = 0, iue = q->items; i < max; i++) { 65 kfifo_in(&q->queue, (void *) &iue, sizeof(void *)); 66 iue->sbuf = ring[i]; 67 iue++; 68 } 69 return 0; 70 71 kfree(q->items); 72free_pool: 73 kfree(q->pool); 74 return -ENOMEM; 75} 76 77static void srp_iu_pool_free(struct srp_queue *q) 78{ 79 kfree(q->items); 80 kfree(q->pool); 81} 82 83static struct srp_buf **srp_ring_alloc(struct device *dev, 84 size_t max, size_t size) 85{ 86 int i; 87 struct srp_buf **ring; 88 89 ring = kcalloc(max, sizeof(struct srp_buf *), GFP_KERNEL); 90 if (!ring) 91 return NULL; 92 93 for (i = 0; i < max; i++) { 94 ring[i] = kzalloc(sizeof(struct srp_buf), GFP_KERNEL); 95 if (!ring[i]) 96 goto out; 97 ring[i]->buf = dma_alloc_coherent(dev, size, &ring[i]->dma, 98 GFP_KERNEL); 99 if (!ring[i]->buf) 100 goto out; 101 } 102 return ring; 103 104out: 105 for (i = 0; i < max && ring[i]; i++) { 106 if (ring[i]->buf) 107 dma_free_coherent(dev, size, ring[i]->buf, ring[i]->dma); 108 kfree(ring[i]); 109 } 110 kfree(ring); 111 112 return NULL; 113} 114 115static void srp_ring_free(struct device *dev, struct srp_buf **ring, size_t max, 116 size_t size) 117{ 118 int i; 119 120 for (i = 0; i < max; i++) { 121 dma_free_coherent(dev, size, ring[i]->buf, ring[i]->dma); 122 kfree(ring[i]); 123 } 124 kfree(ring); 125} 126 127int srp_target_alloc(struct srp_target *target, struct device *dev, 128 size_t nr, size_t iu_size) 129{ 130 int err; 131 132 spin_lock_init(&target->lock); 133 INIT_LIST_HEAD(&target->cmd_queue); 134 135 target->dev = dev; 136 dev_set_drvdata(target->dev, target); 137 138 target->srp_iu_size = iu_size; 139 target->rx_ring_size = nr; 140 target->rx_ring = srp_ring_alloc(target->dev, nr, iu_size); 141 if (!target->rx_ring) 142 return -ENOMEM; 143 err = srp_iu_pool_alloc(&target->iu_queue, nr, target->rx_ring); 144 if (err) 145 goto free_ring; 146 147 return 0; 148 149free_ring: 150 srp_ring_free(target->dev, target->rx_ring, nr, iu_size); 151 return -ENOMEM; 152} 153EXPORT_SYMBOL_GPL(srp_target_alloc); 154 155void srp_target_free(struct srp_target *target) 156{ 157 srp_ring_free(target->dev, target->rx_ring, target->rx_ring_size, 158 target->srp_iu_size); 159 srp_iu_pool_free(&target->iu_queue); 160} 161EXPORT_SYMBOL_GPL(srp_target_free); 162 163struct iu_entry *srp_iu_get(struct srp_target *target) 164{ 165 struct iu_entry *iue = NULL; 166 167 if (kfifo_out_locked(&target->iu_queue.queue, (void *) &iue, 168 sizeof(void *), &target->iu_queue.lock) != sizeof(void *)) { 169 WARN_ONCE(1, "unexpected fifo state"); 170 return NULL; 171 } 172 if (!iue) 173 return iue; 174 iue->target = target; 175 INIT_LIST_HEAD(&iue->ilist); 176 iue->flags = 0; 177 return iue; 178} 179EXPORT_SYMBOL_GPL(srp_iu_get); 180 181void srp_iu_put(struct iu_entry *iue) 182{ 183 kfifo_in_locked(&iue->target->iu_queue.queue, (void *) &iue, 184 sizeof(void *), &iue->target->iu_queue.lock); 185} 186EXPORT_SYMBOL_GPL(srp_iu_put); 187 188static int srp_direct_data(struct scsi_cmnd *sc, struct srp_direct_buf *md, 189 enum dma_data_direction dir, srp_rdma_t rdma_io, 190 int dma_map, int ext_desc) 191{ 192 struct iu_entry *iue = NULL; 193 struct scatterlist *sg = NULL; 194 int err, nsg = 0, len; 195 196 if (dma_map) { 197 iue = (struct iu_entry *) sc->SCp.ptr; 198 sg = scsi_sglist(sc); 199 200 dprintk("%p %u %u %d\n", iue, scsi_bufflen(sc), 201 md->len, scsi_sg_count(sc)); 202 203 nsg = dma_map_sg(iue->target->dev, sg, scsi_sg_count(sc), 204 DMA_BIDIRECTIONAL); 205 if (!nsg) { 206 printk("fail to map %p %d\n", iue, scsi_sg_count(sc)); 207 return 0; 208 } 209 len = min(scsi_bufflen(sc), md->len); 210 } else 211 len = md->len; 212 213 err = rdma_io(sc, sg, nsg, md, 1, dir, len); 214 215 if (dma_map) 216 dma_unmap_sg(iue->target->dev, sg, nsg, DMA_BIDIRECTIONAL); 217 218 return err; 219} 220 221static int srp_indirect_data(struct scsi_cmnd *sc, struct srp_cmd *cmd, 222 struct srp_indirect_buf *id, 223 enum dma_data_direction dir, srp_rdma_t rdma_io, 224 int dma_map, int ext_desc) 225{ 226 struct iu_entry *iue = NULL; 227 struct srp_direct_buf *md = NULL; 228 struct scatterlist dummy, *sg = NULL; 229 dma_addr_t token = 0; 230 int err = 0; 231 int nmd, nsg = 0, len; 232 233 if (dma_map || ext_desc) { 234 iue = (struct iu_entry *) sc->SCp.ptr; 235 sg = scsi_sglist(sc); 236 237 dprintk("%p %u %u %d %d\n", 238 iue, scsi_bufflen(sc), id->len, 239 cmd->data_in_desc_cnt, cmd->data_out_desc_cnt); 240 } 241 242 nmd = id->table_desc.len / sizeof(struct srp_direct_buf); 243 244 if ((dir == DMA_FROM_DEVICE && nmd == cmd->data_in_desc_cnt) || 245 (dir == DMA_TO_DEVICE && nmd == cmd->data_out_desc_cnt)) { 246 md = &id->desc_list[0]; 247 goto rdma; 248 } 249 250 if (ext_desc && dma_map) { 251 md = dma_alloc_coherent(iue->target->dev, id->table_desc.len, 252 &token, GFP_KERNEL); 253 if (!md) { 254 eprintk("Can't get dma memory %u\n", id->table_desc.len); 255 return -ENOMEM; 256 } 257 258 sg_init_one(&dummy, md, id->table_desc.len); 259 sg_dma_address(&dummy) = token; 260 sg_dma_len(&dummy) = id->table_desc.len; 261 err = rdma_io(sc, &dummy, 1, &id->table_desc, 1, DMA_TO_DEVICE, 262 id->table_desc.len); 263 if (err) { 264 eprintk("Error copying indirect table %d\n", err); 265 goto free_mem; 266 } 267 } else { 268 eprintk("This command uses external indirect buffer\n"); 269 return -EINVAL; 270 } 271 272rdma: 273 if (dma_map) { 274 nsg = dma_map_sg(iue->target->dev, sg, scsi_sg_count(sc), 275 DMA_BIDIRECTIONAL); 276 if (!nsg) { 277 eprintk("fail to map %p %d\n", iue, scsi_sg_count(sc)); 278 err = -EIO; 279 goto free_mem; 280 } 281 len = min(scsi_bufflen(sc), id->len); 282 } else 283 len = id->len; 284 285 err = rdma_io(sc, sg, nsg, md, nmd, dir, len); 286 287 if (dma_map) 288 dma_unmap_sg(iue->target->dev, sg, nsg, DMA_BIDIRECTIONAL); 289 290free_mem: 291 if (token && dma_map) 292 dma_free_coherent(iue->target->dev, id->table_desc.len, md, token); 293 294 return err; 295} 296 297static int data_out_desc_size(struct srp_cmd *cmd) 298{ 299 int size = 0; 300 u8 fmt = cmd->buf_fmt >> 4; 301 302 switch (fmt) { 303 case SRP_NO_DATA_DESC: 304 break; 305 case SRP_DATA_DESC_DIRECT: 306 size = sizeof(struct srp_direct_buf); 307 break; 308 case SRP_DATA_DESC_INDIRECT: 309 size = sizeof(struct srp_indirect_buf) + 310 sizeof(struct srp_direct_buf) * cmd->data_out_desc_cnt; 311 break; 312 default: 313 eprintk("client error. Invalid data_out_format %x\n", fmt); 314 break; 315 } 316 return size; 317} 318 319/* 320 * TODO: this can be called multiple times for a single command if it 321 * has very long data. 322 */ 323int srp_transfer_data(struct scsi_cmnd *sc, struct srp_cmd *cmd, 324 srp_rdma_t rdma_io, int dma_map, int ext_desc) 325{ 326 struct srp_direct_buf *md; 327 struct srp_indirect_buf *id; 328 enum dma_data_direction dir; 329 int offset, err = 0; 330 u8 format; 331 332 offset = cmd->add_cdb_len & ~3; 333 334 dir = srp_cmd_direction(cmd); 335 if (dir == DMA_FROM_DEVICE) 336 offset += data_out_desc_size(cmd); 337 338 if (dir == DMA_TO_DEVICE) 339 format = cmd->buf_fmt >> 4; 340 else 341 format = cmd->buf_fmt & ((1U << 4) - 1); 342 343 switch (format) { 344 case SRP_NO_DATA_DESC: 345 break; 346 case SRP_DATA_DESC_DIRECT: 347 md = (struct srp_direct_buf *) 348 (cmd->add_data + offset); 349 err = srp_direct_data(sc, md, dir, rdma_io, dma_map, ext_desc); 350 break; 351 case SRP_DATA_DESC_INDIRECT: 352 id = (struct srp_indirect_buf *) 353 (cmd->add_data + offset); 354 err = srp_indirect_data(sc, cmd, id, dir, rdma_io, dma_map, 355 ext_desc); 356 break; 357 default: 358 eprintk("Unknown format %d %x\n", dir, format); 359 err = -EINVAL; 360 } 361 362 return err; 363} 364EXPORT_SYMBOL_GPL(srp_transfer_data); 365 366static int vscsis_data_length(struct srp_cmd *cmd, enum dma_data_direction dir) 367{ 368 struct srp_direct_buf *md; 369 struct srp_indirect_buf *id; 370 int len = 0, offset = cmd->add_cdb_len & ~3; 371 u8 fmt; 372 373 if (dir == DMA_TO_DEVICE) 374 fmt = cmd->buf_fmt >> 4; 375 else { 376 fmt = cmd->buf_fmt & ((1U << 4) - 1); 377 offset += data_out_desc_size(cmd); 378 } 379 380 switch (fmt) { 381 case SRP_NO_DATA_DESC: 382 break; 383 case SRP_DATA_DESC_DIRECT: 384 md = (struct srp_direct_buf *) (cmd->add_data + offset); 385 len = md->len; 386 break; 387 case SRP_DATA_DESC_INDIRECT: 388 id = (struct srp_indirect_buf *) (cmd->add_data + offset); 389 len = id->len; 390 break; 391 default: 392 eprintk("invalid data format %x\n", fmt); 393 break; 394 } 395 return len; 396} 397 398int srp_cmd_queue(struct Scsi_Host *shost, struct srp_cmd *cmd, void *info, 399 u64 itn_id, u64 addr) 400{ 401 enum dma_data_direction dir; 402 struct scsi_cmnd *sc; 403 int tag, len, err; 404 405 switch (cmd->task_attr) { 406 case SRP_SIMPLE_TASK: 407 tag = MSG_SIMPLE_TAG; 408 break; 409 case SRP_ORDERED_TASK: 410 tag = MSG_ORDERED_TAG; 411 break; 412 case SRP_HEAD_TASK: 413 tag = MSG_HEAD_TAG; 414 break; 415 default: 416 eprintk("Task attribute %d not supported\n", cmd->task_attr); 417 tag = MSG_ORDERED_TAG; 418 } 419 420 dir = srp_cmd_direction(cmd); 421 len = vscsis_data_length(cmd, dir); 422 423 dprintk("%p %x %lx %d %d %d %llx\n", info, cmd->cdb[0], 424 cmd->lun, dir, len, tag, (unsigned long long) cmd->tag); 425 426 sc = scsi_host_get_command(shost, dir, GFP_KERNEL); 427 if (!sc) 428 return -ENOMEM; 429 430 sc->SCp.ptr = info; 431 memcpy(sc->cmnd, cmd->cdb, MAX_COMMAND_SIZE); 432 sc->sdb.length = len; 433 sc->sdb.table.sgl = (void *) (unsigned long) addr; 434 sc->tag = tag; 435 err = scsi_tgt_queue_command(sc, itn_id, (struct scsi_lun *)&cmd->lun, 436 cmd->tag); 437 if (err) 438 scsi_host_put_command(shost, sc); 439 440 return err; 441} 442EXPORT_SYMBOL_GPL(srp_cmd_queue); 443 444MODULE_DESCRIPTION("SCSI RDMA Protocol lib functions"); 445MODULE_AUTHOR("FUJITA Tomonori"); 446MODULE_LICENSE("GPL"); 447