1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * iSCSI over TCP/IP Data-Path lib 4 * 5 * Copyright (C) 2004 Dmitry Yusupov 6 * Copyright (C) 2004 Alex Aizman 7 * Copyright (C) 2005 - 2006 Mike Christie 8 * Copyright (C) 2006 Red Hat, Inc. All rights reserved. 9 * maintained by open-iscsi@googlegroups.com 10 * 11 * Credits: 12 * Christoph Hellwig 13 * FUJITA Tomonori 14 * Arne Redlich 15 * Zhenyu Wang 16 */ 17 18#include <crypto/hash.h> 19#include <linux/types.h> 20#include <linux/list.h> 21#include <linux/inet.h> 22#include <linux/slab.h> 23#include <linux/file.h> 24#include <linux/blkdev.h> 25#include <linux/delay.h> 26#include <linux/kfifo.h> 27#include <linux/scatterlist.h> 28#include <linux/module.h> 29#include <net/tcp.h> 30#include <scsi/scsi_cmnd.h> 31#include <scsi/scsi_device.h> 32#include <scsi/scsi_host.h> 33#include <scsi/scsi.h> 34#include <scsi/scsi_transport_iscsi.h> 35#include <trace/events/iscsi.h> 36 37#include "iscsi_tcp.h" 38 39MODULE_AUTHOR("Mike Christie <michaelc@cs.wisc.edu>, " 40 "Dmitry Yusupov <dmitry_yus@yahoo.com>, " 41 "Alex Aizman <itn780@yahoo.com>"); 42MODULE_DESCRIPTION("iSCSI/TCP data-path"); 43MODULE_LICENSE("GPL"); 44 45static int iscsi_dbg_libtcp; 46module_param_named(debug_libiscsi_tcp, iscsi_dbg_libtcp, int, 47 S_IRUGO | S_IWUSR); 48MODULE_PARM_DESC(debug_libiscsi_tcp, "Turn on debugging for libiscsi_tcp " 49 "module. Set to 1 to turn on, and zero to turn off. Default " 50 "is off."); 51 52#define ISCSI_DBG_TCP(_conn, dbg_fmt, arg...) \ 53 do { \ 54 if (iscsi_dbg_libtcp) \ 55 iscsi_conn_printk(KERN_INFO, _conn, \ 56 "%s " dbg_fmt, \ 57 __func__, ##arg); \ 58 iscsi_dbg_trace(trace_iscsi_dbg_tcp, \ 59 &(_conn)->cls_conn->dev, \ 60 "%s " dbg_fmt, __func__, ##arg);\ 61 } while (0); 62 63static int iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn *tcp_conn, 64 struct iscsi_segment *segment); 65 66/* 67 * Scatterlist handling: inside the iscsi_segment, we 68 * remember an index into the scatterlist, and set data/size 69 * to the current scatterlist entry. For highmem pages, we 70 * kmap as needed. 71 * 72 * Note that the page is unmapped when we return from 73 * TCP's data_ready handler, so we may end up mapping and 74 * unmapping the same page repeatedly. The whole reason 75 * for this is that we shouldn't keep the page mapped 76 * outside the softirq. 77 */ 78 79/** 80 * iscsi_tcp_segment_init_sg - init indicated scatterlist entry 81 * @segment: the buffer object 82 * @sg: scatterlist 83 * @offset: byte offset into that sg entry 84 * 85 * This function sets up the segment so that subsequent 86 * data is copied to the indicated sg entry, at the given 87 * offset. 88 */ 89static inline void 90iscsi_tcp_segment_init_sg(struct iscsi_segment *segment, 91 struct scatterlist *sg, unsigned int offset) 92{ 93 segment->sg = sg; 94 segment->sg_offset = offset; 95 segment->size = min(sg->length - offset, 96 segment->total_size - segment->total_copied); 97 segment->data = NULL; 98} 99 100/** 101 * iscsi_tcp_segment_map - map the current S/G page 102 * @segment: iscsi_segment 103 * @recv: 1 if called from recv path 104 * 105 * We only need to possibly kmap data if scatter lists are being used, 106 * because the iscsi passthrough and internal IO paths will never use high 107 * mem pages. 108 */ 109static void iscsi_tcp_segment_map(struct iscsi_segment *segment, int recv) 110{ 111 struct scatterlist *sg; 112 113 if (segment->data != NULL || !segment->sg) 114 return; 115 116 sg = segment->sg; 117 BUG_ON(segment->sg_mapped); 118 BUG_ON(sg->length == 0); 119 120 /* 121 * We always map for the recv path. 122 * 123 * If the page count is greater than one it is ok to send 124 * to the network layer's zero copy send path. If not we 125 * have to go the slow sendmsg path. 126 * 127 * Same goes for slab pages: skb_can_coalesce() allows 128 * coalescing neighboring slab objects into a single frag which 129 * triggers one of hardened usercopy checks. 130 */ 131 if (!recv && sendpage_ok(sg_page(sg))) 132 return; 133 134 if (recv) { 135 segment->atomic_mapped = true; 136 segment->sg_mapped = kmap_atomic(sg_page(sg)); 137 } else { 138 segment->atomic_mapped = false; 139 /* the xmit path can sleep with the page mapped so use kmap */ 140 segment->sg_mapped = kmap(sg_page(sg)); 141 } 142 143 segment->data = segment->sg_mapped + sg->offset + segment->sg_offset; 144} 145 146void iscsi_tcp_segment_unmap(struct iscsi_segment *segment) 147{ 148 if (segment->sg_mapped) { 149 if (segment->atomic_mapped) 150 kunmap_atomic(segment->sg_mapped); 151 else 152 kunmap(sg_page(segment->sg)); 153 segment->sg_mapped = NULL; 154 segment->data = NULL; 155 } 156} 157EXPORT_SYMBOL_GPL(iscsi_tcp_segment_unmap); 158 159/* 160 * Splice the digest buffer into the buffer 161 */ 162static inline void 163iscsi_tcp_segment_splice_digest(struct iscsi_segment *segment, void *digest) 164{ 165 segment->data = digest; 166 segment->digest_len = ISCSI_DIGEST_SIZE; 167 segment->total_size += ISCSI_DIGEST_SIZE; 168 segment->size = ISCSI_DIGEST_SIZE; 169 segment->copied = 0; 170 segment->sg = NULL; 171 segment->hash = NULL; 172} 173 174/** 175 * iscsi_tcp_segment_done - check whether the segment is complete 176 * @tcp_conn: iscsi tcp connection 177 * @segment: iscsi segment to check 178 * @recv: set to one of this is called from the recv path 179 * @copied: number of bytes copied 180 * 181 * Check if we're done receiving this segment. If the receive 182 * buffer is full but we expect more data, move on to the 183 * next entry in the scatterlist. 184 * 185 * If the amount of data we received isn't a multiple of 4, 186 * we will transparently receive the pad bytes, too. 187 * 188 * This function must be re-entrant. 189 */ 190int iscsi_tcp_segment_done(struct iscsi_tcp_conn *tcp_conn, 191 struct iscsi_segment *segment, int recv, 192 unsigned copied) 193{ 194 struct scatterlist sg; 195 unsigned int pad; 196 197 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "copied %u %u size %u %s\n", 198 segment->copied, copied, segment->size, 199 recv ? "recv" : "xmit"); 200 if (segment->hash && copied) { 201 /* 202 * If a segment is kmapd we must unmap it before sending 203 * to the crypto layer since that will try to kmap it again. 204 */ 205 iscsi_tcp_segment_unmap(segment); 206 207 if (!segment->data) { 208 sg_init_table(&sg, 1); 209 sg_set_page(&sg, sg_page(segment->sg), copied, 210 segment->copied + segment->sg_offset + 211 segment->sg->offset); 212 } else 213 sg_init_one(&sg, segment->data + segment->copied, 214 copied); 215 ahash_request_set_crypt(segment->hash, &sg, NULL, copied); 216 crypto_ahash_update(segment->hash); 217 } 218 219 segment->copied += copied; 220 if (segment->copied < segment->size) { 221 iscsi_tcp_segment_map(segment, recv); 222 return 0; 223 } 224 225 segment->total_copied += segment->copied; 226 segment->copied = 0; 227 segment->size = 0; 228 229 /* Unmap the current scatterlist page, if there is one. */ 230 iscsi_tcp_segment_unmap(segment); 231 232 /* Do we have more scatterlist entries? */ 233 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "total copied %u total size %u\n", 234 segment->total_copied, segment->total_size); 235 if (segment->total_copied < segment->total_size) { 236 /* Proceed to the next entry in the scatterlist. */ 237 iscsi_tcp_segment_init_sg(segment, sg_next(segment->sg), 238 0); 239 iscsi_tcp_segment_map(segment, recv); 240 BUG_ON(segment->size == 0); 241 return 0; 242 } 243 244 /* Do we need to handle padding? */ 245 if (!(tcp_conn->iscsi_conn->session->tt->caps & CAP_PADDING_OFFLOAD)) { 246 pad = iscsi_padding(segment->total_copied); 247 if (pad != 0) { 248 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, 249 "consume %d pad bytes\n", pad); 250 segment->total_size += pad; 251 segment->size = pad; 252 segment->data = segment->padbuf; 253 return 0; 254 } 255 } 256 257 /* 258 * Set us up for transferring the data digest. hdr digest 259 * is completely handled in hdr done function. 260 */ 261 if (segment->hash) { 262 ahash_request_set_crypt(segment->hash, NULL, 263 segment->digest, 0); 264 crypto_ahash_final(segment->hash); 265 iscsi_tcp_segment_splice_digest(segment, 266 recv ? segment->recv_digest : segment->digest); 267 return 0; 268 } 269 270 return 1; 271} 272EXPORT_SYMBOL_GPL(iscsi_tcp_segment_done); 273 274/** 275 * iscsi_tcp_segment_recv - copy data to segment 276 * @tcp_conn: the iSCSI TCP connection 277 * @segment: the buffer to copy to 278 * @ptr: data pointer 279 * @len: amount of data available 280 * 281 * This function copies up to @len bytes to the 282 * given buffer, and returns the number of bytes 283 * consumed, which can actually be less than @len. 284 * 285 * If hash digest is enabled, the function will update the 286 * hash while copying. 287 * Combining these two operations doesn't buy us a lot (yet), 288 * but in the future we could implement combined copy+crc, 289 * just way we do for network layer checksums. 290 */ 291static int 292iscsi_tcp_segment_recv(struct iscsi_tcp_conn *tcp_conn, 293 struct iscsi_segment *segment, const void *ptr, 294 unsigned int len) 295{ 296 unsigned int copy = 0, copied = 0; 297 298 while (!iscsi_tcp_segment_done(tcp_conn, segment, 1, copy)) { 299 if (copied == len) { 300 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, 301 "copied %d bytes\n", len); 302 break; 303 } 304 305 copy = min(len - copied, segment->size - segment->copied); 306 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "copying %d\n", copy); 307 memcpy(segment->data + segment->copied, ptr + copied, copy); 308 copied += copy; 309 } 310 return copied; 311} 312 313inline void 314iscsi_tcp_dgst_header(struct ahash_request *hash, const void *hdr, 315 size_t hdrlen, unsigned char digest[ISCSI_DIGEST_SIZE]) 316{ 317 struct scatterlist sg; 318 319 sg_init_one(&sg, hdr, hdrlen); 320 ahash_request_set_crypt(hash, &sg, digest, hdrlen); 321 crypto_ahash_digest(hash); 322} 323EXPORT_SYMBOL_GPL(iscsi_tcp_dgst_header); 324 325static inline int 326iscsi_tcp_dgst_verify(struct iscsi_tcp_conn *tcp_conn, 327 struct iscsi_segment *segment) 328{ 329 if (!segment->digest_len) 330 return 1; 331 332 if (memcmp(segment->recv_digest, segment->digest, 333 segment->digest_len)) { 334 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "digest mismatch\n"); 335 return 0; 336 } 337 338 return 1; 339} 340 341/* 342 * Helper function to set up segment buffer 343 */ 344static inline void 345__iscsi_segment_init(struct iscsi_segment *segment, size_t size, 346 iscsi_segment_done_fn_t *done, struct ahash_request *hash) 347{ 348 memset(segment, 0, sizeof(*segment)); 349 segment->total_size = size; 350 segment->done = done; 351 352 if (hash) { 353 segment->hash = hash; 354 crypto_ahash_init(hash); 355 } 356} 357 358inline void 359iscsi_segment_init_linear(struct iscsi_segment *segment, void *data, 360 size_t size, iscsi_segment_done_fn_t *done, 361 struct ahash_request *hash) 362{ 363 __iscsi_segment_init(segment, size, done, hash); 364 segment->data = data; 365 segment->size = size; 366} 367EXPORT_SYMBOL_GPL(iscsi_segment_init_linear); 368 369inline int 370iscsi_segment_seek_sg(struct iscsi_segment *segment, 371 struct scatterlist *sg_list, unsigned int sg_count, 372 unsigned int offset, size_t size, 373 iscsi_segment_done_fn_t *done, 374 struct ahash_request *hash) 375{ 376 struct scatterlist *sg; 377 unsigned int i; 378 379 __iscsi_segment_init(segment, size, done, hash); 380 for_each_sg(sg_list, sg, sg_count, i) { 381 if (offset < sg->length) { 382 iscsi_tcp_segment_init_sg(segment, sg, offset); 383 return 0; 384 } 385 offset -= sg->length; 386 } 387 388 return ISCSI_ERR_DATA_OFFSET; 389} 390EXPORT_SYMBOL_GPL(iscsi_segment_seek_sg); 391 392/** 393 * iscsi_tcp_hdr_recv_prep - prep segment for hdr reception 394 * @tcp_conn: iscsi connection to prep for 395 * 396 * This function always passes NULL for the hash argument, because when this 397 * function is called we do not yet know the final size of the header and want 398 * to delay the digest processing until we know that. 399 */ 400void iscsi_tcp_hdr_recv_prep(struct iscsi_tcp_conn *tcp_conn) 401{ 402 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, 403 "(%s)\n", tcp_conn->iscsi_conn->hdrdgst_en ? 404 "digest enabled" : "digest disabled"); 405 iscsi_segment_init_linear(&tcp_conn->in.segment, 406 tcp_conn->in.hdr_buf, sizeof(struct iscsi_hdr), 407 iscsi_tcp_hdr_recv_done, NULL); 408} 409EXPORT_SYMBOL_GPL(iscsi_tcp_hdr_recv_prep); 410 411/* 412 * Handle incoming reply to any other type of command 413 */ 414static int 415iscsi_tcp_data_recv_done(struct iscsi_tcp_conn *tcp_conn, 416 struct iscsi_segment *segment) 417{ 418 struct iscsi_conn *conn = tcp_conn->iscsi_conn; 419 int rc = 0; 420 421 if (!iscsi_tcp_dgst_verify(tcp_conn, segment)) 422 return ISCSI_ERR_DATA_DGST; 423 424 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, 425 conn->data, tcp_conn->in.datalen); 426 if (rc) 427 return rc; 428 429 iscsi_tcp_hdr_recv_prep(tcp_conn); 430 return 0; 431} 432 433static void 434iscsi_tcp_data_recv_prep(struct iscsi_tcp_conn *tcp_conn) 435{ 436 struct iscsi_conn *conn = tcp_conn->iscsi_conn; 437 struct ahash_request *rx_hash = NULL; 438 439 if (conn->datadgst_en && 440 !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) 441 rx_hash = tcp_conn->rx_hash; 442 443 iscsi_segment_init_linear(&tcp_conn->in.segment, 444 conn->data, tcp_conn->in.datalen, 445 iscsi_tcp_data_recv_done, rx_hash); 446} 447 448/** 449 * iscsi_tcp_cleanup_task - free tcp_task resources 450 * @task: iscsi task 451 * 452 * must be called with session back_lock 453 */ 454void iscsi_tcp_cleanup_task(struct iscsi_task *task) 455{ 456 struct iscsi_tcp_task *tcp_task = task->dd_data; 457 struct iscsi_r2t_info *r2t; 458 459 /* nothing to do for mgmt */ 460 if (!task->sc) 461 return; 462 463 spin_lock_bh(&tcp_task->queue2pool); 464 /* flush task's r2t queues */ 465 while (kfifo_out(&tcp_task->r2tqueue, (void*)&r2t, sizeof(void*))) { 466 kfifo_in(&tcp_task->r2tpool.queue, (void*)&r2t, 467 sizeof(void*)); 468 ISCSI_DBG_TCP(task->conn, "pending r2t dropped\n"); 469 } 470 471 r2t = tcp_task->r2t; 472 if (r2t != NULL) { 473 kfifo_in(&tcp_task->r2tpool.queue, (void*)&r2t, 474 sizeof(void*)); 475 tcp_task->r2t = NULL; 476 } 477 spin_unlock_bh(&tcp_task->queue2pool); 478} 479EXPORT_SYMBOL_GPL(iscsi_tcp_cleanup_task); 480 481/** 482 * iscsi_tcp_data_in - SCSI Data-In Response processing 483 * @conn: iscsi connection 484 * @task: scsi command task 485 */ 486static int iscsi_tcp_data_in(struct iscsi_conn *conn, struct iscsi_task *task) 487{ 488 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 489 struct iscsi_tcp_task *tcp_task = task->dd_data; 490 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr; 491 int datasn = be32_to_cpu(rhdr->datasn); 492 unsigned total_in_length = task->sc->sdb.length; 493 494 /* 495 * lib iscsi will update this in the completion handling if there 496 * is status. 497 */ 498 if (!(rhdr->flags & ISCSI_FLAG_DATA_STATUS)) 499 iscsi_update_cmdsn(conn->session, (struct iscsi_nopin*)rhdr); 500 501 if (tcp_conn->in.datalen == 0) 502 return 0; 503 504 if (tcp_task->exp_datasn != datasn) { 505 ISCSI_DBG_TCP(conn, "task->exp_datasn(%d) != rhdr->datasn(%d)" 506 "\n", tcp_task->exp_datasn, datasn); 507 return ISCSI_ERR_DATASN; 508 } 509 510 tcp_task->exp_datasn++; 511 512 tcp_task->data_offset = be32_to_cpu(rhdr->offset); 513 if (tcp_task->data_offset + tcp_conn->in.datalen > total_in_length) { 514 ISCSI_DBG_TCP(conn, "data_offset(%d) + data_len(%d) > " 515 "total_length_in(%d)\n", tcp_task->data_offset, 516 tcp_conn->in.datalen, total_in_length); 517 return ISCSI_ERR_DATA_OFFSET; 518 } 519 520 conn->datain_pdus_cnt++; 521 return 0; 522} 523 524/** 525 * iscsi_tcp_r2t_rsp - iSCSI R2T Response processing 526 * @conn: iscsi connection 527 * @hdr: PDU header 528 */ 529static int iscsi_tcp_r2t_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr) 530{ 531 struct iscsi_session *session = conn->session; 532 struct iscsi_tcp_task *tcp_task; 533 struct iscsi_tcp_conn *tcp_conn; 534 struct iscsi_r2t_rsp *rhdr; 535 struct iscsi_r2t_info *r2t; 536 struct iscsi_task *task; 537 u32 data_length; 538 u32 data_offset; 539 int r2tsn; 540 int rc; 541 542 spin_lock(&session->back_lock); 543 task = iscsi_itt_to_ctask(conn, hdr->itt); 544 if (!task) { 545 spin_unlock(&session->back_lock); 546 return ISCSI_ERR_BAD_ITT; 547 } else if (task->sc->sc_data_direction != DMA_TO_DEVICE) { 548 spin_unlock(&session->back_lock); 549 return ISCSI_ERR_PROTO; 550 } 551 /* 552 * A bad target might complete the cmd before we have handled R2Ts 553 * so get a ref to the task that will be dropped in the xmit path. 554 */ 555 if (task->state != ISCSI_TASK_RUNNING) { 556 spin_unlock(&session->back_lock); 557 /* Let the path that got the early rsp complete it */ 558 return 0; 559 } 560 task->last_xfer = jiffies; 561 if (!iscsi_get_task(task)) { 562 spin_unlock(&session->back_lock); 563 /* Let the path that got the early rsp complete it */ 564 return 0; 565 } 566 567 tcp_conn = conn->dd_data; 568 rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr; 569 /* fill-in new R2T associated with the task */ 570 iscsi_update_cmdsn(session, (struct iscsi_nopin *)rhdr); 571 spin_unlock(&session->back_lock); 572 573 if (tcp_conn->in.datalen) { 574 iscsi_conn_printk(KERN_ERR, conn, 575 "invalid R2t with datalen %d\n", 576 tcp_conn->in.datalen); 577 rc = ISCSI_ERR_DATALEN; 578 goto put_task; 579 } 580 581 tcp_task = task->dd_data; 582 r2tsn = be32_to_cpu(rhdr->r2tsn); 583 if (tcp_task->exp_datasn != r2tsn){ 584 ISCSI_DBG_TCP(conn, "task->exp_datasn(%d) != rhdr->r2tsn(%d)\n", 585 tcp_task->exp_datasn, r2tsn); 586 rc = ISCSI_ERR_R2TSN; 587 goto put_task; 588 } 589 590 if (session->state != ISCSI_STATE_LOGGED_IN) { 591 iscsi_conn_printk(KERN_INFO, conn, 592 "dropping R2T itt %d in recovery.\n", 593 task->itt); 594 rc = 0; 595 goto put_task; 596 } 597 598 data_length = be32_to_cpu(rhdr->data_length); 599 if (data_length == 0) { 600 iscsi_conn_printk(KERN_ERR, conn, 601 "invalid R2T with zero data len\n"); 602 rc = ISCSI_ERR_DATALEN; 603 goto put_task; 604 } 605 606 if (data_length > session->max_burst) 607 ISCSI_DBG_TCP(conn, "invalid R2T with data len %u and max " 608 "burst %u. Attempting to execute request.\n", 609 data_length, session->max_burst); 610 611 data_offset = be32_to_cpu(rhdr->data_offset); 612 if (data_offset + data_length > task->sc->sdb.length) { 613 iscsi_conn_printk(KERN_ERR, conn, 614 "invalid R2T with data len %u at offset %u " 615 "and total length %d\n", data_length, 616 data_offset, task->sc->sdb.length); 617 rc = ISCSI_ERR_DATALEN; 618 goto put_task; 619 } 620 621 spin_lock(&tcp_task->pool2queue); 622 rc = kfifo_out(&tcp_task->r2tpool.queue, (void *)&r2t, sizeof(void *)); 623 if (!rc) { 624 iscsi_conn_printk(KERN_ERR, conn, "Could not allocate R2T. " 625 "Target has sent more R2Ts than it " 626 "negotiated for or driver has leaked.\n"); 627 spin_unlock(&tcp_task->pool2queue); 628 rc = ISCSI_ERR_PROTO; 629 goto put_task; 630 } 631 632 r2t->exp_statsn = rhdr->statsn; 633 r2t->data_length = data_length; 634 r2t->data_offset = data_offset; 635 636 r2t->ttt = rhdr->ttt; /* no flip */ 637 r2t->datasn = 0; 638 r2t->sent = 0; 639 640 tcp_task->exp_datasn = r2tsn + 1; 641 kfifo_in(&tcp_task->r2tqueue, (void*)&r2t, sizeof(void*)); 642 conn->r2t_pdus_cnt++; 643 spin_unlock(&tcp_task->pool2queue); 644 645 iscsi_requeue_task(task); 646 return 0; 647 648put_task: 649 iscsi_put_task(task); 650 return rc; 651} 652 653/* 654 * Handle incoming reply to DataIn command 655 */ 656static int 657iscsi_tcp_process_data_in(struct iscsi_tcp_conn *tcp_conn, 658 struct iscsi_segment *segment) 659{ 660 struct iscsi_conn *conn = tcp_conn->iscsi_conn; 661 struct iscsi_hdr *hdr = tcp_conn->in.hdr; 662 int rc; 663 664 if (!iscsi_tcp_dgst_verify(tcp_conn, segment)) 665 return ISCSI_ERR_DATA_DGST; 666 667 /* check for non-exceptional status */ 668 if (hdr->flags & ISCSI_FLAG_DATA_STATUS) { 669 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0); 670 if (rc) 671 return rc; 672 } 673 674 iscsi_tcp_hdr_recv_prep(tcp_conn); 675 return 0; 676} 677 678/** 679 * iscsi_tcp_hdr_dissect - process PDU header 680 * @conn: iSCSI connection 681 * @hdr: PDU header 682 * 683 * This function analyzes the header of the PDU received, 684 * and performs several sanity checks. If the PDU is accompanied 685 * by data, the receive buffer is set up to copy the incoming data 686 * to the correct location. 687 */ 688static int 689iscsi_tcp_hdr_dissect(struct iscsi_conn *conn, struct iscsi_hdr *hdr) 690{ 691 int rc = 0, opcode, ahslen; 692 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 693 struct iscsi_task *task; 694 695 /* verify PDU length */ 696 tcp_conn->in.datalen = ntoh24(hdr->dlength); 697 if (tcp_conn->in.datalen > conn->max_recv_dlength) { 698 iscsi_conn_printk(KERN_ERR, conn, 699 "iscsi_tcp: datalen %d > %d\n", 700 tcp_conn->in.datalen, conn->max_recv_dlength); 701 return ISCSI_ERR_DATALEN; 702 } 703 704 /* Additional header segments. So far, we don't 705 * process additional headers. 706 */ 707 ahslen = hdr->hlength << 2; 708 709 opcode = hdr->opcode & ISCSI_OPCODE_MASK; 710 /* verify itt (itt encoding: age+cid+itt) */ 711 rc = iscsi_verify_itt(conn, hdr->itt); 712 if (rc) 713 return rc; 714 715 ISCSI_DBG_TCP(conn, "opcode 0x%x ahslen %d datalen %d\n", 716 opcode, ahslen, tcp_conn->in.datalen); 717 718 switch(opcode) { 719 case ISCSI_OP_SCSI_DATA_IN: 720 spin_lock(&conn->session->back_lock); 721 task = iscsi_itt_to_ctask(conn, hdr->itt); 722 if (!task) 723 rc = ISCSI_ERR_BAD_ITT; 724 else 725 rc = iscsi_tcp_data_in(conn, task); 726 if (rc) { 727 spin_unlock(&conn->session->back_lock); 728 break; 729 } 730 731 if (tcp_conn->in.datalen) { 732 struct iscsi_tcp_task *tcp_task = task->dd_data; 733 struct ahash_request *rx_hash = NULL; 734 struct scsi_data_buffer *sdb = &task->sc->sdb; 735 736 /* 737 * Setup copy of Data-In into the struct scsi_cmnd 738 * Scatterlist case: 739 * We set up the iscsi_segment to point to the next 740 * scatterlist entry to copy to. As we go along, 741 * we move on to the next scatterlist entry and 742 * update the digest per-entry. 743 */ 744 if (conn->datadgst_en && 745 !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) 746 rx_hash = tcp_conn->rx_hash; 747 748 ISCSI_DBG_TCP(conn, "iscsi_tcp_begin_data_in( " 749 "offset=%d, datalen=%d)\n", 750 tcp_task->data_offset, 751 tcp_conn->in.datalen); 752 task->last_xfer = jiffies; 753 rc = iscsi_segment_seek_sg(&tcp_conn->in.segment, 754 sdb->table.sgl, 755 sdb->table.nents, 756 tcp_task->data_offset, 757 tcp_conn->in.datalen, 758 iscsi_tcp_process_data_in, 759 rx_hash); 760 spin_unlock(&conn->session->back_lock); 761 return rc; 762 } 763 rc = __iscsi_complete_pdu(conn, hdr, NULL, 0); 764 spin_unlock(&conn->session->back_lock); 765 break; 766 case ISCSI_OP_SCSI_CMD_RSP: 767 if (tcp_conn->in.datalen) { 768 iscsi_tcp_data_recv_prep(tcp_conn); 769 return 0; 770 } 771 rc = iscsi_complete_pdu(conn, hdr, NULL, 0); 772 break; 773 case ISCSI_OP_R2T: 774 if (ahslen) { 775 rc = ISCSI_ERR_AHSLEN; 776 break; 777 } 778 rc = iscsi_tcp_r2t_rsp(conn, hdr); 779 break; 780 case ISCSI_OP_LOGIN_RSP: 781 case ISCSI_OP_TEXT_RSP: 782 case ISCSI_OP_REJECT: 783 case ISCSI_OP_ASYNC_EVENT: 784 /* 785 * It is possible that we could get a PDU with a buffer larger 786 * than 8K, but there are no targets that currently do this. 787 * For now we fail until we find a vendor that needs it 788 */ 789 if (ISCSI_DEF_MAX_RECV_SEG_LEN < tcp_conn->in.datalen) { 790 iscsi_conn_printk(KERN_ERR, conn, 791 "iscsi_tcp: received buffer of " 792 "len %u but conn buffer is only %u " 793 "(opcode %0x)\n", 794 tcp_conn->in.datalen, 795 ISCSI_DEF_MAX_RECV_SEG_LEN, opcode); 796 rc = ISCSI_ERR_PROTO; 797 break; 798 } 799 800 /* If there's data coming in with the response, 801 * receive it to the connection's buffer. 802 */ 803 if (tcp_conn->in.datalen) { 804 iscsi_tcp_data_recv_prep(tcp_conn); 805 return 0; 806 } 807 fallthrough; 808 case ISCSI_OP_LOGOUT_RSP: 809 case ISCSI_OP_NOOP_IN: 810 case ISCSI_OP_SCSI_TMFUNC_RSP: 811 rc = iscsi_complete_pdu(conn, hdr, NULL, 0); 812 break; 813 default: 814 rc = ISCSI_ERR_BAD_OPCODE; 815 break; 816 } 817 818 if (rc == 0) { 819 /* Anything that comes with data should have 820 * been handled above. */ 821 if (tcp_conn->in.datalen) 822 return ISCSI_ERR_PROTO; 823 iscsi_tcp_hdr_recv_prep(tcp_conn); 824 } 825 826 return rc; 827} 828 829/** 830 * iscsi_tcp_hdr_recv_done - process PDU header 831 * @tcp_conn: iSCSI TCP connection 832 * @segment: the buffer segment being processed 833 * 834 * This is the callback invoked when the PDU header has 835 * been received. If the header is followed by additional 836 * header segments, we go back for more data. 837 */ 838static int 839iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn *tcp_conn, 840 struct iscsi_segment *segment) 841{ 842 struct iscsi_conn *conn = tcp_conn->iscsi_conn; 843 struct iscsi_hdr *hdr; 844 845 /* Check if there are additional header segments 846 * *prior* to computing the digest, because we 847 * may need to go back to the caller for more. 848 */ 849 hdr = (struct iscsi_hdr *) tcp_conn->in.hdr_buf; 850 if (segment->copied == sizeof(struct iscsi_hdr) && hdr->hlength) { 851 /* Bump the header length - the caller will 852 * just loop around and get the AHS for us, and 853 * call again. */ 854 unsigned int ahslen = hdr->hlength << 2; 855 856 /* Make sure we don't overflow */ 857 if (sizeof(*hdr) + ahslen > sizeof(tcp_conn->in.hdr_buf)) 858 return ISCSI_ERR_AHSLEN; 859 860 segment->total_size += ahslen; 861 segment->size += ahslen; 862 return 0; 863 } 864 865 /* We're done processing the header. See if we're doing 866 * header digests; if so, set up the recv_digest buffer 867 * and go back for more. */ 868 if (conn->hdrdgst_en && 869 !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) { 870 if (segment->digest_len == 0) { 871 /* 872 * Even if we offload the digest processing we 873 * splice it in so we can increment the skb/segment 874 * counters in preparation for the data segment. 875 */ 876 iscsi_tcp_segment_splice_digest(segment, 877 segment->recv_digest); 878 return 0; 879 } 880 881 iscsi_tcp_dgst_header(tcp_conn->rx_hash, hdr, 882 segment->total_copied - ISCSI_DIGEST_SIZE, 883 segment->digest); 884 885 if (!iscsi_tcp_dgst_verify(tcp_conn, segment)) 886 return ISCSI_ERR_HDR_DGST; 887 } 888 889 tcp_conn->in.hdr = hdr; 890 return iscsi_tcp_hdr_dissect(conn, hdr); 891} 892 893/** 894 * iscsi_tcp_recv_segment_is_hdr - tests if we are reading in a header 895 * @tcp_conn: iscsi tcp conn 896 * 897 * returns non zero if we are currently processing or setup to process 898 * a header. 899 */ 900inline int iscsi_tcp_recv_segment_is_hdr(struct iscsi_tcp_conn *tcp_conn) 901{ 902 return tcp_conn->in.segment.done == iscsi_tcp_hdr_recv_done; 903} 904EXPORT_SYMBOL_GPL(iscsi_tcp_recv_segment_is_hdr); 905 906/** 907 * iscsi_tcp_recv_skb - Process skb 908 * @conn: iscsi connection 909 * @skb: network buffer with header and/or data segment 910 * @offset: offset in skb 911 * @offloaded: bool indicating if transfer was offloaded 912 * @status: iscsi TCP status result 913 * 914 * Will return status of transfer in @status. And will return 915 * number of bytes copied. 916 */ 917int iscsi_tcp_recv_skb(struct iscsi_conn *conn, struct sk_buff *skb, 918 unsigned int offset, bool offloaded, int *status) 919{ 920 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 921 struct iscsi_segment *segment = &tcp_conn->in.segment; 922 struct skb_seq_state seq; 923 unsigned int consumed = 0; 924 int rc = 0; 925 926 ISCSI_DBG_TCP(conn, "in %d bytes\n", skb->len - offset); 927 /* 928 * Update for each skb instead of pdu, because over slow networks a 929 * data_in's data could take a while to read in. We also want to 930 * account for r2ts. 931 */ 932 conn->last_recv = jiffies; 933 934 if (unlikely(test_bit(ISCSI_CONN_FLAG_SUSPEND_RX, &conn->flags))) { 935 ISCSI_DBG_TCP(conn, "Rx suspended!\n"); 936 *status = ISCSI_TCP_SUSPENDED; 937 return 0; 938 } 939 940 if (offloaded) { 941 segment->total_copied = segment->total_size; 942 goto segment_done; 943 } 944 945 skb_prepare_seq_read(skb, offset, skb->len, &seq); 946 while (1) { 947 unsigned int avail; 948 const u8 *ptr; 949 950 avail = skb_seq_read(consumed, &ptr, &seq); 951 if (avail == 0) { 952 ISCSI_DBG_TCP(conn, "no more data avail. Consumed %d\n", 953 consumed); 954 *status = ISCSI_TCP_SKB_DONE; 955 goto skb_done; 956 } 957 BUG_ON(segment->copied >= segment->size); 958 959 ISCSI_DBG_TCP(conn, "skb %p ptr=%p avail=%u\n", skb, ptr, 960 avail); 961 rc = iscsi_tcp_segment_recv(tcp_conn, segment, ptr, avail); 962 BUG_ON(rc == 0); 963 consumed += rc; 964 965 if (segment->total_copied >= segment->total_size) { 966 skb_abort_seq_read(&seq); 967 goto segment_done; 968 } 969 } 970 971segment_done: 972 *status = ISCSI_TCP_SEGMENT_DONE; 973 ISCSI_DBG_TCP(conn, "segment done\n"); 974 rc = segment->done(tcp_conn, segment); 975 if (rc != 0) { 976 *status = ISCSI_TCP_CONN_ERR; 977 ISCSI_DBG_TCP(conn, "Error receiving PDU, errno=%d\n", rc); 978 iscsi_conn_failure(conn, rc); 979 return 0; 980 } 981 /* The done() functions sets up the next segment. */ 982 983skb_done: 984 conn->rxdata_octets += consumed; 985 return consumed; 986} 987EXPORT_SYMBOL_GPL(iscsi_tcp_recv_skb); 988 989/** 990 * iscsi_tcp_task_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands 991 * @task: scsi command task 992 */ 993int iscsi_tcp_task_init(struct iscsi_task *task) 994{ 995 struct iscsi_tcp_task *tcp_task = task->dd_data; 996 struct iscsi_conn *conn = task->conn; 997 struct scsi_cmnd *sc = task->sc; 998 int err; 999 1000 if (!sc) { 1001 /* 1002 * mgmt tasks do not have a scatterlist since they come 1003 * in from the iscsi interface. 1004 */ 1005 ISCSI_DBG_TCP(conn, "mtask deq [itt 0x%x]\n", task->itt); 1006 1007 return conn->session->tt->init_pdu(task, 0, task->data_count); 1008 } 1009 1010 BUG_ON(kfifo_len(&tcp_task->r2tqueue)); 1011 tcp_task->exp_datasn = 0; 1012 1013 /* Prepare PDU, optionally w/ immediate data */ 1014 ISCSI_DBG_TCP(conn, "task deq [itt 0x%x imm %d unsol %d]\n", 1015 task->itt, task->imm_count, task->unsol_r2t.data_length); 1016 1017 err = conn->session->tt->init_pdu(task, 0, task->imm_count); 1018 if (err) 1019 return err; 1020 task->imm_count = 0; 1021 return 0; 1022} 1023EXPORT_SYMBOL_GPL(iscsi_tcp_task_init); 1024 1025static struct iscsi_r2t_info *iscsi_tcp_get_curr_r2t(struct iscsi_task *task) 1026{ 1027 struct iscsi_tcp_task *tcp_task = task->dd_data; 1028 struct iscsi_r2t_info *r2t = NULL; 1029 1030 if (iscsi_task_has_unsol_data(task)) 1031 r2t = &task->unsol_r2t; 1032 else { 1033 spin_lock_bh(&tcp_task->queue2pool); 1034 if (tcp_task->r2t) { 1035 r2t = tcp_task->r2t; 1036 /* Continue with this R2T? */ 1037 if (r2t->data_length <= r2t->sent) { 1038 ISCSI_DBG_TCP(task->conn, 1039 " done with r2t %p\n", r2t); 1040 kfifo_in(&tcp_task->r2tpool.queue, 1041 (void *)&tcp_task->r2t, 1042 sizeof(void *)); 1043 tcp_task->r2t = r2t = NULL; 1044 } 1045 } 1046 1047 if (r2t == NULL) { 1048 if (kfifo_out(&tcp_task->r2tqueue, 1049 (void *)&tcp_task->r2t, sizeof(void *)) != 1050 sizeof(void *)) 1051 r2t = NULL; 1052 else 1053 r2t = tcp_task->r2t; 1054 } 1055 spin_unlock_bh(&tcp_task->queue2pool); 1056 } 1057 1058 return r2t; 1059} 1060 1061/** 1062 * iscsi_tcp_task_xmit - xmit normal PDU task 1063 * @task: iscsi command task 1064 * 1065 * We're expected to return 0 when everything was transmitted successfully, 1066 * -EAGAIN if there's still data in the queue, or != 0 for any other kind 1067 * of error. 1068 */ 1069int iscsi_tcp_task_xmit(struct iscsi_task *task) 1070{ 1071 struct iscsi_conn *conn = task->conn; 1072 struct iscsi_session *session = conn->session; 1073 struct iscsi_r2t_info *r2t; 1074 int rc = 0; 1075 1076flush: 1077 /* Flush any pending data first. */ 1078 rc = session->tt->xmit_pdu(task); 1079 if (rc < 0) 1080 return rc; 1081 1082 /* mgmt command */ 1083 if (!task->sc) { 1084 if (task->hdr->itt == RESERVED_ITT) 1085 iscsi_put_task(task); 1086 return 0; 1087 } 1088 1089 /* Are we done already? */ 1090 if (task->sc->sc_data_direction != DMA_TO_DEVICE) 1091 return 0; 1092 1093 r2t = iscsi_tcp_get_curr_r2t(task); 1094 if (r2t == NULL) { 1095 /* Waiting for more R2Ts to arrive. */ 1096 ISCSI_DBG_TCP(conn, "no R2Ts yet\n"); 1097 return 0; 1098 } 1099 1100 rc = conn->session->tt->alloc_pdu(task, ISCSI_OP_SCSI_DATA_OUT); 1101 if (rc) 1102 return rc; 1103 iscsi_prep_data_out_pdu(task, r2t, (struct iscsi_data *) task->hdr); 1104 1105 ISCSI_DBG_TCP(conn, "sol dout %p [dsn %d itt 0x%x doff %d dlen %d]\n", 1106 r2t, r2t->datasn - 1, task->hdr->itt, 1107 r2t->data_offset + r2t->sent, r2t->data_count); 1108 1109 rc = conn->session->tt->init_pdu(task, r2t->data_offset + r2t->sent, 1110 r2t->data_count); 1111 if (rc) { 1112 iscsi_conn_failure(conn, ISCSI_ERR_XMIT_FAILED); 1113 return rc; 1114 } 1115 1116 r2t->sent += r2t->data_count; 1117 goto flush; 1118} 1119EXPORT_SYMBOL_GPL(iscsi_tcp_task_xmit); 1120 1121struct iscsi_cls_conn * 1122iscsi_tcp_conn_setup(struct iscsi_cls_session *cls_session, int dd_data_size, 1123 uint32_t conn_idx) 1124 1125{ 1126 struct iscsi_conn *conn; 1127 struct iscsi_cls_conn *cls_conn; 1128 struct iscsi_tcp_conn *tcp_conn; 1129 1130 cls_conn = iscsi_conn_setup(cls_session, 1131 sizeof(*tcp_conn) + dd_data_size, conn_idx); 1132 if (!cls_conn) 1133 return NULL; 1134 conn = cls_conn->dd_data; 1135 /* 1136 * due to strange issues with iser these are not set 1137 * in iscsi_conn_setup 1138 */ 1139 conn->max_recv_dlength = ISCSI_DEF_MAX_RECV_SEG_LEN; 1140 1141 tcp_conn = conn->dd_data; 1142 tcp_conn->iscsi_conn = conn; 1143 tcp_conn->dd_data = conn->dd_data + sizeof(*tcp_conn); 1144 return cls_conn; 1145} 1146EXPORT_SYMBOL_GPL(iscsi_tcp_conn_setup); 1147 1148void iscsi_tcp_conn_teardown(struct iscsi_cls_conn *cls_conn) 1149{ 1150 iscsi_conn_teardown(cls_conn); 1151} 1152EXPORT_SYMBOL_GPL(iscsi_tcp_conn_teardown); 1153 1154int iscsi_tcp_r2tpool_alloc(struct iscsi_session *session) 1155{ 1156 int i; 1157 int cmd_i; 1158 1159 /* 1160 * initialize per-task: R2T pool and xmit queue 1161 */ 1162 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) { 1163 struct iscsi_task *task = session->cmds[cmd_i]; 1164 struct iscsi_tcp_task *tcp_task = task->dd_data; 1165 1166 /* 1167 * pre-allocated x2 as much r2ts to handle race when 1168 * target acks DataOut faster than we data_xmit() queues 1169 * could replenish r2tqueue. 1170 */ 1171 1172 /* R2T pool */ 1173 if (iscsi_pool_init(&tcp_task->r2tpool, 1174 session->max_r2t * 2, NULL, 1175 sizeof(struct iscsi_r2t_info))) { 1176 goto r2t_alloc_fail; 1177 } 1178 1179 /* R2T xmit queue */ 1180 if (kfifo_alloc(&tcp_task->r2tqueue, 1181 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL)) { 1182 iscsi_pool_free(&tcp_task->r2tpool); 1183 goto r2t_alloc_fail; 1184 } 1185 spin_lock_init(&tcp_task->pool2queue); 1186 spin_lock_init(&tcp_task->queue2pool); 1187 } 1188 1189 return 0; 1190 1191r2t_alloc_fail: 1192 for (i = 0; i < cmd_i; i++) { 1193 struct iscsi_task *task = session->cmds[i]; 1194 struct iscsi_tcp_task *tcp_task = task->dd_data; 1195 1196 kfifo_free(&tcp_task->r2tqueue); 1197 iscsi_pool_free(&tcp_task->r2tpool); 1198 } 1199 return -ENOMEM; 1200} 1201EXPORT_SYMBOL_GPL(iscsi_tcp_r2tpool_alloc); 1202 1203void iscsi_tcp_r2tpool_free(struct iscsi_session *session) 1204{ 1205 int i; 1206 1207 for (i = 0; i < session->cmds_max; i++) { 1208 struct iscsi_task *task = session->cmds[i]; 1209 struct iscsi_tcp_task *tcp_task = task->dd_data; 1210 1211 kfifo_free(&tcp_task->r2tqueue); 1212 iscsi_pool_free(&tcp_task->r2tpool); 1213 } 1214} 1215EXPORT_SYMBOL_GPL(iscsi_tcp_r2tpool_free); 1216 1217int iscsi_tcp_set_max_r2t(struct iscsi_conn *conn, char *buf) 1218{ 1219 struct iscsi_session *session = conn->session; 1220 unsigned short r2ts = 0; 1221 1222 sscanf(buf, "%hu", &r2ts); 1223 if (session->max_r2t == r2ts) 1224 return 0; 1225 1226 if (!r2ts || !is_power_of_2(r2ts)) 1227 return -EINVAL; 1228 1229 session->max_r2t = r2ts; 1230 iscsi_tcp_r2tpool_free(session); 1231 return iscsi_tcp_r2tpool_alloc(session); 1232} 1233EXPORT_SYMBOL_GPL(iscsi_tcp_set_max_r2t); 1234 1235void iscsi_tcp_conn_get_stats(struct iscsi_cls_conn *cls_conn, 1236 struct iscsi_stats *stats) 1237{ 1238 struct iscsi_conn *conn = cls_conn->dd_data; 1239 1240 stats->txdata_octets = conn->txdata_octets; 1241 stats->rxdata_octets = conn->rxdata_octets; 1242 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt; 1243 stats->dataout_pdus = conn->dataout_pdus_cnt; 1244 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt; 1245 stats->datain_pdus = conn->datain_pdus_cnt; 1246 stats->r2t_pdus = conn->r2t_pdus_cnt; 1247 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt; 1248 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt; 1249} 1250EXPORT_SYMBOL_GPL(iscsi_tcp_conn_get_stats); 1251