drivers/scsi/iscsi_tcp.c

   1 /*
   2  * iSCSI Initiator over TCP/IP Data-Path
   3  *
   4  * Copyright (C) 2004 Dmitry Yusupov
   5  * Copyright (C) 2004 Alex Aizman
   6  * Copyright (C) 2005 Mike Christie
   7  * maintained by open-iscsi@googlegroups.com
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published
  11  * by the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful, but
  15  * WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  17  * General Public License for more details.
  18  *
  19  * See the file COPYING included with this distribution for more details.
  20  *
  21  * Credits:
  22  *      Christoph Hellwig
  23  *      FUJITA Tomonori
  24  *      Arne Redlich
  25  *      Zhenyu Wang
  26  */
  27
  28 #include <linux/types.h>
  29 #include <linux/list.h>
  30 #include <linux/inet.h>
  31 #include <linux/blkdev.h>
  32 #include <linux/crypto.h>
  33 #include <linux/delay.h>
  34 #include <linux/kfifo.h>
  35 #include <linux/scatterlist.h>
  36 #include <linux/mutex.h>
  37 #include <net/tcp.h>
  38 #include <scsi/scsi_cmnd.h>
  39 #include <scsi/scsi_device.h>
  40 #include <scsi/scsi_eh.h>
  41 #include <scsi/scsi_request.h>
  42 #include <scsi/scsi_tcq.h>
  43 #include <scsi/scsi_host.h>
  44 #include <scsi/scsi.h>
  45 #include <scsi/scsi_transport_iscsi.h>
  46
  47 #include "iscsi_tcp.h"
  48
  49 MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
  50               "Alex Aizman <itn780@yahoo.com>");
  51 MODULE_DESCRIPTION("iSCSI/TCP data-path");
  52 MODULE_LICENSE("GPL");
  53 MODULE_VERSION("0:4.445");
  54 /* #define DEBUG_TCP */
  55 /* #define DEBUG_SCSI */
  56 #define DEBUG_ASSERT
  57
  58 #ifdef DEBUG_TCP
  59 #define debug_tcp(fmt...) printk(KERN_DEBUG "tcp: " fmt)
  60 #else
  61 #define debug_tcp(fmt...)
  62 #endif
  63
  64 #ifdef DEBUG_SCSI
  65 #define debug_scsi(fmt...) printk(KERN_DEBUG "scsi: " fmt)
  66 #else
  67 #define debug_scsi(fmt...)
  68 #endif
  69
  70 #ifndef DEBUG_ASSERT
  71 #ifdef BUG_ON
  72 #undef BUG_ON
  73 #endif
  74 #define BUG_ON(expr)
  75 #endif
  76
  77 #define INVALID_SN_DELTA        0xffff
  78
  79 static unsigned int iscsi_max_lun = 512;
  80 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
  81
  82 /* global data */
  83 static kmem_cache_t *taskcache;
  84
  85 static inline void
  86 iscsi_buf_init_virt(struct iscsi_buf *ibuf, char *vbuf, int size)
  87 {
  88         sg_init_one(&ibuf->sg, (u8 *)vbuf, size);
  89         ibuf->sent = 0;
  90         ibuf->use_sendmsg = 0;
  91 }
  92
  93 static inline void
  94 iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size)
  95 {
  96         ibuf->sg.page = virt_to_page(vbuf);
  97         ibuf->sg.offset = offset_in_page(vbuf);
  98         ibuf->sg.length = size;
  99         ibuf->sent = 0;
 100         ibuf->use_sendmsg = 1;
 101 }
 102
 103 static inline void
 104 iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg)
 105 {
 106         ibuf->sg.page = sg->page;
 107         ibuf->sg.offset = sg->offset;
 108         ibuf->sg.length = sg->length;
 109         /*
 110          * Fastpath: sg element fits into single page
 111          */
 112         if (sg->length + sg->offset <= PAGE_SIZE && page_count(sg->page) >= 2)
 113                 ibuf->use_sendmsg = 0;
 114         else
 115                 ibuf->use_sendmsg = 1;
 116         ibuf->sent = 0;
 117 }
 118
 119 static inline int
 120 iscsi_buf_left(struct iscsi_buf *ibuf)
 121 {
 122         int rc;
 123
 124         rc = ibuf->sg.length - ibuf->sent;
 125         BUG_ON(rc < 0);
 126         return rc;
 127 }
 128
 129 static inline void
 130 iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf,
 131                  u8* crc)
 132 {
 133         crypto_digest_digest(conn->tx_tfm, &buf->sg, 1, crc);
 134         buf->sg.length += sizeof(uint32_t);
 135 }
 136
 137 static void
 138 iscsi_conn_failure(struct iscsi_conn *conn, enum iscsi_err err)
 139 {
 140         struct iscsi_session *session = conn->session;
 141         unsigned long flags;
 142
 143         spin_lock_irqsave(&session->lock, flags);
 144         if (session->conn_cnt == 1 || session->leadconn == conn)
 145                 session->state = ISCSI_STATE_FAILED;
 146         spin_unlock_irqrestore(&session->lock, flags);
 147         set_bit(SUSPEND_BIT, &conn->suspend_tx);
 148         set_bit(SUSPEND_BIT, &conn->suspend_rx);
 149         iscsi_conn_error(iscsi_handle(conn), err);
 150 }
 151
 152 static inline int
 153 iscsi_check_assign_cmdsn(struct iscsi_session *session, struct iscsi_nopin *hdr)
 154 {
 155         uint32_t max_cmdsn = be32_to_cpu(hdr->max_cmdsn);
 156         uint32_t exp_cmdsn = be32_to_cpu(hdr->exp_cmdsn);
 157
 158         if (max_cmdsn < exp_cmdsn -1 &&
 159             max_cmdsn > exp_cmdsn - INVALID_SN_DELTA)
 160                 return ISCSI_ERR_MAX_CMDSN;
 161         if (max_cmdsn > session->max_cmdsn ||
 162             max_cmdsn < session->max_cmdsn - INVALID_SN_DELTA)
 163                 session->max_cmdsn = max_cmdsn;
 164         if (exp_cmdsn > session->exp_cmdsn ||
 165             exp_cmdsn < session->exp_cmdsn - INVALID_SN_DELTA)
 166                 session->exp_cmdsn = exp_cmdsn;
 167
 168         return 0;
 169 }
 170
 171 static inline int
 172 iscsi_hdr_extract(struct iscsi_conn *conn)
 173 {
 174         struct sk_buff *skb = conn->in.skb;
 175
 176         if (conn->in.copy >= conn->hdr_size &&
 177             conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
 178                 /*
 179                  * Zero-copy PDU Header: using connection context
 180                  * to store header pointer.
 181                  */
 182                 if (skb_shinfo(skb)->frag_list == NULL &&
 183                     !skb_shinfo(skb)->nr_frags)
 184                         conn->in.hdr = (struct iscsi_hdr *)
 185                                 ((char*)skb->data + conn->in.offset);
 186                 else {
 187                         /* ignoring return code since we checked
 188                          * in.copy before */
 189                         skb_copy_bits(skb, conn->in.offset,
 190                                 &conn->hdr, conn->hdr_size);
 191                         conn->in.hdr = &conn->hdr;
 192                 }
 193                 conn->in.offset += conn->hdr_size;
 194                 conn->in.copy -= conn->hdr_size;
 195         } else {
 196                 int hdr_remains;
 197                 int copylen;
 198
 199                 /*
 200                  * PDU header scattered across SKB's,
 201                  * copying it... This'll happen quite rarely.
 202                  */
 203
 204                 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER)
 205                         conn->in.hdr_offset = 0;
 206
 207                 hdr_remains = conn->hdr_size - conn->in.hdr_offset;
 208                 BUG_ON(hdr_remains <= 0);
 209
 210                 copylen = min(conn->in.copy, hdr_remains);
 211                 skb_copy_bits(skb, conn->in.offset,
 212                         (char*)&conn->hdr + conn->in.hdr_offset, copylen);
 213
 214                 debug_tcp("PDU gather offset %d bytes %d in.offset %d "
 215                        "in.copy %d\n", conn->in.hdr_offset, copylen,
 216                        conn->in.offset, conn->in.copy);
 217
 218                 conn->in.offset += copylen;
 219                 conn->in.copy -= copylen;
 220                 if (copylen < hdr_remains)  {
 221                         conn->in_progress = IN_PROGRESS_HEADER_GATHER;
 222                         conn->in.hdr_offset += copylen;
 223                         return -EAGAIN;
 224                 }
 225                 conn->in.hdr = &conn->hdr;
 226                 conn->discontiguous_hdr_cnt++;
 227                 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
 228         }
 229
 230         return 0;
 231 }
 232
 233 static inline void
 234 iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 235 {
 236         struct scsi_cmnd *sc = ctask->sc;
 237         struct iscsi_session *session = conn->session;
 238
 239         spin_lock(&session->lock);
 240         if (unlikely(!sc)) {
 241                 spin_unlock(&session->lock);
 242                 return;
 243         }
 244         if (sc->sc_data_direction == DMA_TO_DEVICE) {
 245                 struct iscsi_data_task *dtask, *n;
 246                 /* WRITE: cleanup Data-Out's if any */
 247                 spin_lock(&conn->lock);
 248                 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
 249                         list_del(&dtask->item);
 250                         mempool_free(dtask, ctask->datapool);
 251                 }
 252                 spin_unlock(&conn->lock);
 253         }
 254         ctask->xmstate = XMSTATE_IDLE;
 255         ctask->r2t = NULL;
 256         ctask->sc = NULL;
 257         __kfifo_put(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
 258         spin_unlock(&session->lock);
 259 }
 260
 261 /**
 262  * iscsi_cmd_rsp - SCSI Command Response processing
 263  * @conn: iscsi connection
 264  * @ctask: scsi command task
 265  **/
 266 static int
 267 iscsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 268 {
 269         int rc;
 270         struct iscsi_cmd_rsp *rhdr = (struct iscsi_cmd_rsp *)conn->in.hdr;
 271         struct iscsi_session *session = conn->session;
 272         struct scsi_cmnd *sc = ctask->sc;
 273
 274         rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
 275         if (rc) {
 276                 sc->result = (DID_ERROR << 16);
 277                 goto out;
 278         }
 279
 280         conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
 281
 282         sc->result = (DID_OK << 16) | rhdr->cmd_status;
 283
 284         if (rhdr->response != ISCSI_STATUS_CMD_COMPLETED) {
 285                 sc->result = (DID_ERROR << 16);
 286                 goto out;
 287         }
 288
 289         if (rhdr->cmd_status == SAM_STAT_CHECK_CONDITION && conn->senselen) {
 290                 int sensecopy = min(conn->senselen, SCSI_SENSE_BUFFERSIZE);
 291
 292                 memcpy(sc->sense_buffer, conn->data + 2, sensecopy);
 293                 debug_scsi("copied %d bytes of sense\n", sensecopy);
 294         }
 295
 296         if (sc->sc_data_direction == DMA_TO_DEVICE)
 297                 goto out;
 298
 299         if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
 300                 int res_count = be32_to_cpu(rhdr->residual_count);
 301
 302                 if (res_count > 0 && res_count <= sc->request_bufflen)
 303                         sc->resid = res_count;
 304                 else
 305                         sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
 306         } else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
 307                 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
 308         else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW)
 309                 sc->resid = be32_to_cpu(rhdr->residual_count);
 310
 311 out:
 312         debug_scsi("done [sc %lx res %d itt 0x%x]\n",
 313                    (long)sc, sc->result, ctask->itt);
 314         conn->scsirsp_pdus_cnt++;
 315         iscsi_ctask_cleanup(conn, ctask);
 316         sc->scsi_done(sc);
 317         return rc;
 318 }
 319
 320 /**
 321  * iscsi_data_rsp - SCSI Data-In Response processing
 322  * @conn: iscsi connection
 323  * @ctask: scsi command task
 324  **/
 325 static int
 326 iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 327 {
 328         int rc;
 329         struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)conn->in.hdr;
 330         struct iscsi_session *session = conn->session;
 331         int datasn = be32_to_cpu(rhdr->datasn);
 332
 333         rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
 334         if (rc)
 335                 return rc;
 336         /*
 337          * setup Data-In byte counter (gets decremented..)
 338          */
 339         ctask->data_count = conn->in.datalen;
 340
 341         if (conn->in.datalen == 0)
 342                 return 0;
 343
 344         if (ctask->datasn != datasn)
 345                 return ISCSI_ERR_DATASN;
 346
 347         ctask->datasn++;
 348
 349         ctask->data_offset = be32_to_cpu(rhdr->offset);
 350         if (ctask->data_offset + conn->in.datalen > ctask->total_length)
 351                 return ISCSI_ERR_DATA_OFFSET;
 352
 353         if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
 354                 struct scsi_cmnd *sc = ctask->sc;
 355
 356                 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
 357                 if (rhdr->flags & ISCSI_FLAG_DATA_UNDERFLOW) {
 358                         int res_count = be32_to_cpu(rhdr->residual_count);
 359
 360                         if (res_count > 0 &&
 361                             res_count <= sc->request_bufflen) {
 362                                 sc->resid = res_count;
 363                                 sc->result = (DID_OK << 16) | rhdr->cmd_status;
 364                         } else
 365                                 sc->result = (DID_BAD_TARGET << 16) |
 366                                         rhdr->cmd_status;
 367                 } else if (rhdr->flags & ISCSI_FLAG_DATA_OVERFLOW) {
 368                         sc->resid = be32_to_cpu(rhdr->residual_count);
 369                         sc->result = (DID_OK << 16) | rhdr->cmd_status;
 370                 } else
 371                         sc->result = (DID_OK << 16) | rhdr->cmd_status;
 372         }
 373
 374         conn->datain_pdus_cnt++;
 375         return 0;
 376 }
 377
 378 /**
 379  * iscsi_solicit_data_init - initialize first Data-Out
 380  * @conn: iscsi connection
 381  * @ctask: scsi command task
 382  * @r2t: R2T info
 383  *
 384  * Notes:
 385  *      Initialize first Data-Out within this R2T sequence and finds
 386  *      proper data_offset within this SCSI command.
 387  *
 388  *      This function is called with connection lock taken.
 389  **/
 390 static void
 391 iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
 392                         struct iscsi_r2t_info *r2t)
 393 {
 394         struct iscsi_data *hdr;
 395         struct iscsi_data_task *dtask;
 396         struct scsi_cmnd *sc = ctask->sc;
 397
 398         dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
 399         BUG_ON(!dtask);
 400         hdr = &dtask->hdr;
 401         memset(hdr, 0, sizeof(struct iscsi_data));
 402         hdr->ttt = r2t->ttt;
 403         hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
 404         r2t->solicit_datasn++;
 405         hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
 406         memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
 407         hdr->itt = ctask->hdr.itt;
 408         hdr->exp_statsn = r2t->exp_statsn;
 409         hdr->offset = cpu_to_be32(r2t->data_offset);
 410         if (r2t->data_length > conn->max_xmit_dlength) {
 411                 hton24(hdr->dlength, conn->max_xmit_dlength);
 412                 r2t->data_count = conn->max_xmit_dlength;
 413                 hdr->flags = 0;
 414         } else {
 415                 hton24(hdr->dlength, r2t->data_length);
 416                 r2t->data_count = r2t->data_length;
 417                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
 418         }
 419         conn->dataout_pdus_cnt++;
 420
 421         r2t->sent = 0;
 422
 423         iscsi_buf_init_virt(&r2t->headbuf, (char*)hdr,
 424                            sizeof(struct iscsi_hdr));
 425
 426         r2t->dtask = dtask;
 427
 428         if (sc->use_sg) {
 429                 int i, sg_count = 0;
 430                 struct scatterlist *sg = sc->request_buffer;
 431
 432                 r2t->sg = NULL;
 433                 for (i = 0; i < sc->use_sg; i++, sg += 1) {
 434                         /* FIXME: prefetch ? */
 435                         if (sg_count + sg->length > r2t->data_offset) {
 436                                 int page_offset;
 437
 438                                 /* sg page found! */
 439
 440                                 /* offset within this page */
 441                                 page_offset = r2t->data_offset - sg_count;
 442
 443                                 /* fill in this buffer */
 444                                 iscsi_buf_init_sg(&r2t->sendbuf, sg);
 445                                 r2t->sendbuf.sg.offset += page_offset;
 446                                 r2t->sendbuf.sg.length -= page_offset;
 447
 448                                 /* xmit logic will continue with next one */
 449                                 r2t->sg = sg + 1;
 450                                 break;
 451                         }
 452                         sg_count += sg->length;
 453                 }
 454                 BUG_ON(r2t->sg == NULL);
 455         } else
 456                 iscsi_buf_init_iov(&ctask->sendbuf,
 457                             (char*)sc->request_buffer + r2t->data_offset,
 458                             r2t->data_count);
 459
 460         list_add(&dtask->item, &ctask->dataqueue);
 461 }
 462
 463 /**
 464  * iscsi_r2t_rsp - iSCSI R2T Response processing
 465  * @conn: iscsi connection
 466  * @ctask: scsi command task
 467  **/
 468 static int
 469 iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 470 {
 471         struct iscsi_r2t_info *r2t;
 472         struct iscsi_session *session = conn->session;
 473         struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)conn->in.hdr;
 474         int r2tsn = be32_to_cpu(rhdr->r2tsn);
 475         int rc;
 476
 477         if (conn->in.ahslen)
 478                 return ISCSI_ERR_AHSLEN;
 479
 480         if (conn->in.datalen)
 481                 return ISCSI_ERR_DATALEN;
 482
 483         if (ctask->exp_r2tsn && ctask->exp_r2tsn != r2tsn)
 484                 return ISCSI_ERR_R2TSN;
 485
 486         rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
 487         if (rc)
 488                 return rc;
 489
 490         /* FIXME: use R2TSN to detect missing R2T */
 491
 492         /* fill-in new R2T associated with the task */
 493         spin_lock(&session->lock);
 494         if (!ctask->sc || ctask->mtask ||
 495              session->state != ISCSI_STATE_LOGGED_IN) {
 496                 printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in "
 497                        "recovery...\n", ctask->itt);
 498                 spin_unlock(&session->lock);
 499                 return 0;
 500         }
 501         rc = __kfifo_get(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
 502         BUG_ON(!rc);
 503
 504         r2t->exp_statsn = rhdr->statsn;
 505         r2t->data_length = be32_to_cpu(rhdr->data_length);
 506         if (r2t->data_length == 0 ||
 507             r2t->data_length > session->max_burst) {
 508                 spin_unlock(&session->lock);
 509                 return ISCSI_ERR_DATALEN;
 510         }
 511
 512         r2t->data_offset = be32_to_cpu(rhdr->data_offset);
 513         if (r2t->data_offset + r2t->data_length > ctask->total_length) {
 514                 spin_unlock(&session->lock);
 515                 return ISCSI_ERR_DATALEN;
 516         }
 517
 518         r2t->ttt = rhdr->ttt; /* no flip */
 519         r2t->solicit_datasn = 0;
 520
 521         iscsi_solicit_data_init(conn, ctask, r2t);
 522
 523         ctask->exp_r2tsn = r2tsn + 1;
 524         ctask->xmstate |= XMSTATE_SOL_HDR;
 525         __kfifo_put(ctask->r2tqueue, (void*)&r2t, sizeof(void*));
 526         __kfifo_put(conn->writequeue, (void*)&ctask, sizeof(void*));
 527
 528         schedule_work(&conn->xmitwork);
 529         conn->r2t_pdus_cnt++;
 530         spin_unlock(&session->lock);
 531
 532         return 0;
 533 }
 534
 535 static int
 536 iscsi_hdr_recv(struct iscsi_conn *conn)
 537 {
 538         int rc = 0;
 539         struct iscsi_hdr *hdr;
 540         struct iscsi_cmd_task *ctask;
 541         struct iscsi_session *session = conn->session;
 542         uint32_t cdgst, rdgst = 0;
 543
 544         hdr = conn->in.hdr;
 545
 546         /* verify PDU length */
 547         conn->in.datalen = ntoh24(hdr->dlength);
 548         if (conn->in.datalen > conn->max_recv_dlength) {
 549                 printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
 550                        conn->in.datalen, conn->max_recv_dlength);
 551                 return ISCSI_ERR_DATALEN;
 552         }
 553         conn->data_copied = 0;
 554
 555         /* read AHS */
 556         conn->in.ahslen = hdr->hlength * 4;
 557         conn->in.offset += conn->in.ahslen;
 558         conn->in.copy -= conn->in.ahslen;
 559         if (conn->in.copy < 0) {
 560                 printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
 561                        "%d bytes\n", conn->in.ahslen);
 562                 return ISCSI_ERR_AHSLEN;
 563         }
 564
 565         /* calculate read padding */
 566         conn->in.padding = conn->in.datalen & (ISCSI_PAD_LEN-1);
 567         if (conn->in.padding) {
 568                 conn->in.padding = ISCSI_PAD_LEN - conn->in.padding;
 569                 debug_scsi("read padding %d bytes\n", conn->in.padding);
 570         }
 571
 572         if (conn->hdrdgst_en) {
 573                 struct scatterlist sg;
 574
 575                 sg_init_one(&sg, (u8 *)hdr,
 576                             sizeof(struct iscsi_hdr) + conn->in.ahslen);
 577                 crypto_digest_digest(conn->rx_tfm, &sg, 1, (u8 *)&cdgst);
 578                 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
 579                                      conn->in.ahslen);
 580                 if (cdgst != rdgst) {
 581                         printk(KERN_ERR "iscsi_tcp: itt %x: hdrdgst error "
 582                                "recv 0x%x calc 0x%x\n", conn->in.itt, rdgst,
 583                                cdgst);
 584                         return ISCSI_ERR_HDR_DGST;
 585                 }
 586         }
 587
 588         /* save opcode for later */
 589         conn->in.opcode = hdr->opcode & ISCSI_OPCODE_MASK;
 590
 591         /* verify itt (itt encoding: age+cid+itt) */
 592         if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
 593                 if ((hdr->itt & AGE_MASK) !=
 594                                 (session->age << AGE_SHIFT)) {
 595                         printk(KERN_ERR "iscsi_tcp: received itt %x expected "
 596                                 "session age (%x)\n", hdr->itt,
 597                                 session->age & AGE_MASK);
 598                         return ISCSI_ERR_BAD_ITT;
 599                 }
 600
 601                 if ((hdr->itt & CID_MASK) != (conn->id << CID_SHIFT)) {
 602                         printk(KERN_ERR "iscsi_tcp: received itt %x, expected "
 603                                 "CID (%x)\n", hdr->itt, conn->id);
 604                         return ISCSI_ERR_BAD_ITT;
 605                 }
 606                 conn->in.itt = hdr->itt & ITT_MASK;
 607         } else
 608                 conn->in.itt = hdr->itt;
 609
 610         debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
 611                   hdr->opcode, conn->in.offset, conn->in.copy,
 612                   conn->in.ahslen, conn->in.datalen);
 613
 614         if (conn->in.itt < session->cmds_max) {
 615                 ctask = (struct iscsi_cmd_task *)session->cmds[conn->in.itt];
 616
 617                 if (!ctask->sc) {
 618                         printk(KERN_INFO "iscsi_tcp: dropping ctask with "
 619                                "itt 0x%x\n", ctask->itt);
 620                         conn->in.datalen = 0; /* force drop */
 621                         return 0;
 622                 }
 623
 624                 if (ctask->sc->SCp.phase != session->age) {
 625                         printk(KERN_ERR "iscsi_tcp: ctask's session age %d, "
 626                                 "expected %d\n", ctask->sc->SCp.phase,
 627                                 session->age);
 628                         return ISCSI_ERR_SESSION_FAILED;
 629                 }
 630
 631                 conn->in.ctask = ctask;
 632
 633                 debug_scsi("rsp [op 0x%x cid %d sc %lx itt 0x%x len %d]\n",
 634                            hdr->opcode, conn->id, (long)ctask->sc,
 635                            ctask->itt, conn->in.datalen);
 636
 637                 switch(conn->in.opcode) {
 638                 case ISCSI_OP_SCSI_CMD_RSP:
 639                         BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
 640                         if (!conn->in.datalen)
 641                                 rc = iscsi_cmd_rsp(conn, ctask);
 642                         else
 643                                 /*
 644                                  * got sense or response data; copying PDU
 645                                  * Header to the connection's header
 646                                  * placeholder
 647                                  */
 648                                 memcpy(&conn->hdr, hdr,
 649                                        sizeof(struct iscsi_hdr));
 650                         break;
 651                 case ISCSI_OP_SCSI_DATA_IN:
 652                         BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
 653                         /* save flags for non-exceptional status */
 654                         conn->in.flags = hdr->flags;
 655                         /* save cmd_status for sense data */
 656                         conn->in.cmd_status =
 657                                 ((struct iscsi_data_rsp*)hdr)->cmd_status;
 658                         rc = iscsi_data_rsp(conn, ctask);
 659                         break;
 660                 case ISCSI_OP_R2T:
 661                         BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
 662                         if (ctask->sc->sc_data_direction == DMA_TO_DEVICE)
 663                                 rc = iscsi_r2t_rsp(conn, ctask);
 664                         else
 665                                 rc = ISCSI_ERR_PROTO;
 666                         break;
 667                 default:
 668                         rc = ISCSI_ERR_BAD_OPCODE;
 669                         break;
 670                 }
 671         } else if (conn->in.itt >= ISCSI_MGMT_ITT_OFFSET &&
 672                    conn->in.itt < ISCSI_MGMT_ITT_OFFSET +
 673                                         session->mgmtpool_max) {
 674                 struct iscsi_mgmt_task *mtask = (struct iscsi_mgmt_task *)
 675                                         session->mgmt_cmds[conn->in.itt -
 676                                                 ISCSI_MGMT_ITT_OFFSET];
 677
 678                 debug_scsi("immrsp [op 0x%x cid %d itt 0x%x len %d]\n",
 679                            conn->in.opcode, conn->id, mtask->itt,
 680                            conn->in.datalen);
 681
 682                 switch(conn->in.opcode) {
 683                 case ISCSI_OP_LOGIN_RSP:
 684                 case ISCSI_OP_TEXT_RSP:
 685                 case ISCSI_OP_LOGOUT_RSP:
 686                         rc = iscsi_check_assign_cmdsn(session,
 687                                                  (struct iscsi_nopin*)hdr);
 688                         if (rc)
 689                                 break;
 690
 691                         if (!conn->in.datalen) {
 692                                 rc = iscsi_recv_pdu(iscsi_handle(conn), hdr,
 693                                                     NULL, 0);
 694                                 if (conn->login_mtask != mtask) {
 695                                         spin_lock(&session->lock);
 696                                         __kfifo_put(session->mgmtpool.queue,
 697                                             (void*)&mtask, sizeof(void*));
 698                                         spin_unlock(&session->lock);
 699                                 }
 700                         }
 701                         break;
 702                 case ISCSI_OP_SCSI_TMFUNC_RSP:
 703                         rc = iscsi_check_assign_cmdsn(session,
 704                                                  (struct iscsi_nopin*)hdr);
 705                         if (rc)
 706                                 break;
 707
 708                         if (conn->in.datalen || conn->in.ahslen) {
 709                                 rc = ISCSI_ERR_PROTO;
 710                                 break;
 711                         }
 712                         conn->tmfrsp_pdus_cnt++;
 713                         spin_lock(&session->lock);
 714                         if (conn->tmabort_state == TMABORT_INITIAL) {
 715                                 __kfifo_put(session->mgmtpool.queue,
 716                                                 (void*)&mtask, sizeof(void*));
 717                                 conn->tmabort_state =
 718                                         ((struct iscsi_tm_rsp *)hdr)->
 719                                         response == ISCSI_TMF_RSP_COMPLETE ?
 720                                                 TMABORT_SUCCESS:TMABORT_FAILED;
 721                                 /* unblock eh_abort() */
 722                                 wake_up(&conn->ehwait);
 723                         }
 724                         spin_unlock(&session->lock);
 725                         break;
 726                 case ISCSI_OP_NOOP_IN:
 727                         if (hdr->ttt != ISCSI_RESERVED_TAG) {
 728                                 rc = ISCSI_ERR_PROTO;
 729                                 break;
 730                         }
 731                         rc = iscsi_check_assign_cmdsn(session,
 732                                                 (struct iscsi_nopin*)hdr);
 733                         if (rc)
 734                                 break;
 735                         conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1;
 736
 737                         if (!conn->in.datalen) {
 738                                 struct iscsi_mgmt_task *mtask;
 739
 740                                 rc = iscsi_recv_pdu(iscsi_handle(conn), hdr,
 741                                                     NULL, 0);
 742                                 mtask = (struct iscsi_mgmt_task *)
 743                                         session->mgmt_cmds[conn->in.itt -
 744                                                         ISCSI_MGMT_ITT_OFFSET];
 745                                 if (conn->login_mtask != mtask) {
 746                                         spin_lock(&session->lock);
 747                                         __kfifo_put(session->mgmtpool.queue,
 748                                                   (void*)&mtask, sizeof(void*));
 749                                         spin_unlock(&session->lock);
 750                                 }
 751                         }
 752                         break;
 753                 default:
 754                         rc = ISCSI_ERR_BAD_OPCODE;
 755                         break;
 756                 }
 757         } else if (conn->in.itt == ISCSI_RESERVED_TAG) {
 758                 switch(conn->in.opcode) {
 759                 case ISCSI_OP_NOOP_IN:
 760                         if (!conn->in.datalen) {
 761                                 rc = iscsi_check_assign_cmdsn(session,
 762                                                  (struct iscsi_nopin*)hdr);
 763                                 if (!rc && hdr->ttt != ISCSI_RESERVED_TAG)
 764                                         rc = iscsi_recv_pdu(iscsi_handle(conn),
 765                                                             hdr, NULL, 0);
 766                         } else
 767                                 rc = ISCSI_ERR_PROTO;
 768                         break;
 769                 case ISCSI_OP_REJECT:
 770                         /* we need sth like iscsi_reject_rsp()*/
 771                 case ISCSI_OP_ASYNC_EVENT:
 772                         /* we need sth like iscsi_async_event_rsp() */
 773                         rc = ISCSI_ERR_BAD_OPCODE;
 774                         break;
 775                 default:
 776                         rc = ISCSI_ERR_BAD_OPCODE;
 777                         break;
 778                 }
 779         } else
 780                 rc = ISCSI_ERR_BAD_ITT;
 781
 782         return rc;
 783 }
 784
 785 /**
 786  * iscsi_ctask_copy - copy skb bits to the destanation cmd task
 787  * @conn: iscsi connection
 788  * @ctask: scsi command task
 789  * @buf: buffer to copy to
 790  * @buf_size: size of buffer
 791  * @offset: offset within the buffer
 792  *
 793  * Notes:
 794  *      The function calls skb_copy_bits() and updates per-connection and
 795  *      per-cmd byte counters.
 796  *
 797  *      Read counters (in bytes):
 798  *
 799  *      conn->in.offset         offset within in progress SKB
 800  *      conn->in.copy           left to copy from in progress SKB
 801  *                              including padding
 802  *      conn->in.copied         copied already from in progress SKB
 803  *      conn->data_copied       copied already from in progress buffer
 804  *      ctask->sent             total bytes sent up to the MidLayer
 805  *      ctask->data_count       left to copy from in progress Data-In
 806  *      buf_left                left to copy from in progress buffer
 807  **/
 808 static inline int
 809 iscsi_ctask_copy(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
 810                 void *buf, int buf_size, int offset)
 811 {
 812         int buf_left = buf_size - (conn->data_copied + offset);
 813         int size = min(conn->in.copy, buf_left);
 814         int rc;
 815
 816         size = min(size, ctask->data_count);
 817
 818         debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
 819                size, conn->in.offset, conn->in.copied);
 820
 821         BUG_ON(size <= 0);
 822         BUG_ON(ctask->sent + size > ctask->total_length);
 823
 824         rc = skb_copy_bits(conn->in.skb, conn->in.offset,
 825                            (char*)buf + (offset + conn->data_copied), size);
 826         /* must fit into skb->len */
 827         BUG_ON(rc);
 828
 829         conn->in.offset += size;
 830         conn->in.copy -= size;
 831         conn->in.copied += size;
 832         conn->data_copied += size;
 833         ctask->sent += size;
 834         ctask->data_count -= size;
 835
 836         BUG_ON(conn->in.copy < 0);
 837         BUG_ON(ctask->data_count < 0);
 838
 839         if (buf_size != (conn->data_copied + offset)) {
 840                 if (!ctask->data_count) {
 841                         BUG_ON(buf_size - conn->data_copied < 0);
 842                         /* done with this PDU */
 843                         return buf_size - conn->data_copied;
 844                 }
 845                 return -EAGAIN;
 846         }
 847
 848         /* done with this buffer or with both - PDU and buffer */
 849         conn->data_copied = 0;
 850         return 0;
 851 }
 852
 853 /**
 854  * iscsi_tcp_copy - copy skb bits to the destanation buffer
 855  * @conn: iscsi connection
 856  * @buf: buffer to copy to
 857  * @buf_size: number of bytes to copy
 858  *
 859  * Notes:
 860  *      The function calls skb_copy_bits() and updates per-connection
 861  *      byte counters.
 862  **/
 863 static inline int
 864 iscsi_tcp_copy(struct iscsi_conn *conn, void *buf, int buf_size)
 865 {
 866         int buf_left = buf_size - conn->data_copied;
 867         int size = min(conn->in.copy, buf_left);
 868         int rc;
 869
 870         debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
 871                size, conn->in.offset, conn->data_copied);
 872         BUG_ON(size <= 0);
 873
 874         rc = skb_copy_bits(conn->in.skb, conn->in.offset,
 875                            (char*)buf + conn->data_copied, size);
 876         BUG_ON(rc);
 877
 878         conn->in.offset += size;
 879         conn->in.copy -= size;
 880         conn->in.copied += size;
 881         conn->data_copied += size;
 882
 883         if (buf_size != conn->data_copied)
 884                 return -EAGAIN;
 885
 886         return 0;
 887 }
 888
 889 static inline void
 890 partial_sg_digest_update(struct iscsi_conn *conn, struct scatterlist *sg,
 891                          int offset, int length)
 892 {
 893         struct scatterlist temp;
 894
 895         memcpy(&temp, sg, sizeof(struct scatterlist));
 896         temp.offset = offset;
 897         temp.length = length;
 898         crypto_digest_update(conn->data_rx_tfm, &temp, 1);
 899 }
 900
 901 static void
 902 iscsi_recv_digest_update(struct iscsi_conn *conn, char* buf, int len)
 903 {
 904         struct scatterlist tmp;
 905
 906         sg_init_one(&tmp, buf, len);
 907         crypto_digest_update(conn->data_rx_tfm, &tmp, 1);
 908 }
 909
 910 static int iscsi_scsi_data_in(struct iscsi_conn *conn)
 911 {
 912         struct iscsi_cmd_task *ctask = conn->in.ctask;
 913         struct scsi_cmnd *sc = ctask->sc;
 914         struct scatterlist *sg;
 915         int i, offset, rc = 0;
 916
 917         BUG_ON((void*)ctask != sc->SCp.ptr);
 918
 919         /*
 920          * copying Data-In into the Scsi_Cmnd
 921          */
 922         if (!sc->use_sg) {
 923                 i = ctask->data_count;
 924                 rc = iscsi_ctask_copy(conn, ctask, sc->request_buffer,
 925                                       sc->request_bufflen, ctask->data_offset);
 926                 if (rc == -EAGAIN)
 927                         return rc;
 928                 if (conn->datadgst_en)
 929                         iscsi_recv_digest_update(conn, sc->request_buffer, i);
 930                 rc = 0;
 931                 goto done;
 932         }
 933
 934         offset = ctask->data_offset;
 935         sg = sc->request_buffer;
 936
 937         if (ctask->data_offset)
 938                 for (i = 0; i < ctask->sg_count; i++)
 939                         offset -= sg[i].length;
 940         /* we've passed through partial sg*/
 941         if (offset < 0)
 942                 offset = 0;
 943
 944         for (i = ctask->sg_count; i < sc->use_sg; i++) {
 945                 char *dest;
 946
 947                 dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0);
 948                 rc = iscsi_ctask_copy(conn, ctask, dest + sg[i].offset,
 949                                       sg[i].length, offset);
 950                 kunmap_atomic(dest, KM_SOFTIRQ0);
 951                 if (rc == -EAGAIN)
 952                         /* continue with the next SKB/PDU */
 953                         return rc;
 954                 if (!rc) {
 955                         if (conn->datadgst_en) {
 956                                 if (!offset)
 957                                         crypto_digest_update(conn->data_rx_tfm,
 958                                                              &sg[i], 1);
 959                                 else
 960                                         partial_sg_digest_update(conn, &sg[i],
 961                                                         sg[i].offset + offset,
 962                                                         sg[i].length - offset);
 963                         }
 964                         offset = 0;
 965                         ctask->sg_count++;
 966                 }
 967
 968                 if (!ctask->data_count) {
 969                         if (rc && conn->datadgst_en)
 970                                 /*
 971                                  * data-in is complete, but buffer not...
 972                                  */
 973                                 partial_sg_digest_update(conn, &sg[i],
 974                                                 sg[i].offset, sg[i].length-rc);
 975                         rc = 0;
 976                         break;
 977                 }
 978
 979                 if (!conn->in.copy)
 980                         return -EAGAIN;
 981         }
 982         BUG_ON(ctask->data_count);
 983
 984 done:
 985         /* check for non-exceptional status */
 986         if (conn->in.flags & ISCSI_FLAG_DATA_STATUS) {
 987                 debug_scsi("done [sc %lx res %d itt 0x%x]\n",
 988                            (long)sc, sc->result, ctask->itt);
 989                 conn->scsirsp_pdus_cnt++;
 990                 iscsi_ctask_cleanup(conn, ctask);
 991                 sc->scsi_done(sc);
 992         }
 993
 994         return rc;
 995 }
 996
 997 static int
 998 iscsi_data_recv(struct iscsi_conn *conn)
 999 {
1000         struct iscsi_session *session = conn->session;
1001         int rc = 0;
1002
1003         switch(conn->in.opcode) {
1004         case ISCSI_OP_SCSI_DATA_IN:
1005                 rc = iscsi_scsi_data_in(conn);
1006                 break;
1007         case ISCSI_OP_SCSI_CMD_RSP: {
1008                 /*
1009                  * SCSI Sense Data:
1010                  * copying the entire Data Segment.
1011                  */
1012                 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
1013                         rc = -EAGAIN;
1014                         goto exit;
1015                 }
1016
1017                 /*
1018                  * check for sense
1019                  */
1020                 conn->in.hdr = &conn->hdr;
1021                 conn->senselen = (conn->data[0] << 8) | conn->data[1];
1022                 rc = iscsi_cmd_rsp(conn, conn->in.ctask);
1023                 if (!rc && conn->datadgst_en)
1024                         iscsi_recv_digest_update(conn, conn->data,
1025                                                  conn->in.datalen);
1026         }
1027         break;
1028         case ISCSI_OP_TEXT_RSP:
1029         case ISCSI_OP_LOGIN_RSP:
1030         case ISCSI_OP_NOOP_IN: {
1031                 struct iscsi_mgmt_task *mtask = NULL;
1032
1033                 if (conn->in.itt != ISCSI_RESERVED_TAG)
1034                         mtask = (struct iscsi_mgmt_task *)
1035                                 session->mgmt_cmds[conn->in.itt -
1036                                         ISCSI_MGMT_ITT_OFFSET];
1037
1038                 /*
1039                  * Collect data segment to the connection's data
1040                  * placeholder
1041                  */
1042                 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
1043                         rc = -EAGAIN;
1044                         goto exit;
1045                 }
1046
1047                 rc = iscsi_recv_pdu(iscsi_handle(conn), conn->in.hdr,
1048                                     conn->data, conn->in.datalen);
1049
1050                 if (!rc && conn->datadgst_en &&
1051                         conn->in.opcode != ISCSI_OP_LOGIN_RSP)
1052                         iscsi_recv_digest_update(conn, conn->data,
1053                                                 conn->in.datalen);
1054
1055                 if (mtask && conn->login_mtask != mtask) {
1056                         spin_lock(&session->lock);
1057                         __kfifo_put(session->mgmtpool.queue, (void*)&mtask,
1058                                     sizeof(void*));
1059                         spin_unlock(&session->lock);
1060                 }
1061         }
1062         break;
1063         case ISCSI_OP_ASYNC_EVENT:
1064         case ISCSI_OP_REJECT:
1065         default:
1066                 BUG_ON(1);
1067         }
1068 exit:
1069         return rc;
1070 }
1071
1072 /**
1073  * iscsi_tcp_data_recv - TCP receive in sendfile fashion
1074  * @rd_desc: read descriptor
1075  * @skb: socket buffer
1076  * @offset: offset in skb
1077  * @len: skb->len - offset
1078  **/
1079 static int
1080 iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
1081                 unsigned int offset, size_t len)
1082 {
1083         int rc;
1084         struct iscsi_conn *conn = rd_desc->arg.data;
1085         int processed;
1086         char pad[ISCSI_PAD_LEN];
1087         struct scatterlist sg;
1088
1089         /*
1090          * Save current SKB and its offset in the corresponding
1091          * connection context.
1092          */
1093         conn->in.copy = skb->len - offset;
1094         conn->in.offset = offset;
1095         conn->in.skb = skb;
1096         conn->in.len = conn->in.copy;
1097         BUG_ON(conn->in.copy <= 0);
1098         debug_tcp("in %d bytes\n", conn->in.copy);
1099
1100 more:
1101         conn->in.copied = 0;
1102         rc = 0;
1103
1104         if (unlikely(conn->suspend_rx)) {
1105                 debug_tcp("conn %d Rx suspended!\n", conn->id);
1106                 return 0;
1107         }
1108
1109         if (conn->in_progress == IN_PROGRESS_WAIT_HEADER ||
1110             conn->in_progress == IN_PROGRESS_HEADER_GATHER) {
1111                 rc = iscsi_hdr_extract(conn);
1112                 if (rc) {
1113                        if (rc == -EAGAIN)
1114                                 goto nomore;
1115                        else {
1116                                 iscsi_conn_failure(conn, rc);
1117                                 return 0;
1118                        }
1119                 }
1120
1121                 /*
1122                  * Verify and process incoming PDU header.
1123                  */
1124                 rc = iscsi_hdr_recv(conn);
1125                 if (!rc && conn->in.datalen) {
1126                         if (conn->datadgst_en) {
1127                                 BUG_ON(!conn->data_rx_tfm);
1128                                 crypto_digest_init(conn->data_rx_tfm);
1129                         }
1130                         conn->in_progress = IN_PROGRESS_DATA_RECV;
1131                 } else if (rc) {
1132                         iscsi_conn_failure(conn, rc);
1133                         return 0;
1134                 }
1135         }
1136
1137         if (conn->in_progress == IN_PROGRESS_DDIGEST_RECV) {
1138                 uint32_t recv_digest;
1139                 debug_tcp("extra data_recv offset %d copy %d\n",
1140                           conn->in.offset, conn->in.copy);
1141                 skb_copy_bits(conn->in.skb, conn->in.offset,
1142                                 &recv_digest, 4);
1143                 conn->in.offset += 4;
1144                 conn->in.copy -= 4;
1145                 if (recv_digest != conn->in.datadgst) {
1146                         debug_tcp("iscsi_tcp: data digest error!"
1147                                   "0x%x != 0x%x\n", recv_digest,
1148                                   conn->in.datadgst);
1149                         iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
1150                         return 0;
1151                 } else {
1152                         debug_tcp("iscsi_tcp: data digest match!"
1153                                   "0x%x == 0x%x\n", recv_digest,
1154                                   conn->in.datadgst);
1155                         conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1156                 }
1157         }
1158
1159         if (conn->in_progress == IN_PROGRESS_DATA_RECV && conn->in.copy) {
1160
1161                 debug_tcp("data_recv offset %d copy %d\n",
1162                        conn->in.offset, conn->in.copy);
1163
1164                 rc = iscsi_data_recv(conn);
1165                 if (rc) {
1166                         if (rc == -EAGAIN) {
1167                                 rd_desc->count = conn->in.datalen -
1168                                                 conn->in.ctask->data_count;
1169                                 goto again;
1170                         }
1171                         iscsi_conn_failure(conn, rc);
1172                         return 0;
1173                 }
1174                 conn->in.copy -= conn->in.padding;
1175                 conn->in.offset += conn->in.padding;
1176                 if (conn->datadgst_en) {
1177                         if (conn->in.padding) {
1178                                 debug_tcp("padding -> %d\n", conn->in.padding);
1179                                 memset(pad, 0, conn->in.padding);
1180                                 sg_init_one(&sg, pad, conn->in.padding);
1181                                 crypto_digest_update(conn->data_rx_tfm, &sg, 1);
1182                         }
1183                         crypto_digest_final(conn->data_rx_tfm,
1184                                             (u8 *) & conn->in.datadgst);
1185                         debug_tcp("rx digest 0x%x\n", conn->in.datadgst);
1186                         conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
1187                 } else
1188                         conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1189         }
1190
1191         debug_tcp("f, processed %d from out of %d padding %d\n",
1192                conn->in.offset - offset, (int)len, conn->in.padding);
1193         BUG_ON(conn->in.offset - offset > len);
1194
1195         if (conn->in.offset - offset != len) {
1196                 debug_tcp("continue to process %d bytes\n",
1197                        (int)len - (conn->in.offset - offset));
1198                 goto more;
1199         }
1200
1201 nomore:
1202         processed = conn->in.offset - offset;
1203         BUG_ON(processed == 0);
1204         return processed;
1205
1206 again:
1207         processed = conn->in.offset - offset;
1208         debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n",
1209                   processed, (int)len, (int)rd_desc->count);
1210         BUG_ON(processed == 0);
1211         BUG_ON(processed > len);
1212
1213         conn->rxdata_octets += processed;
1214         return processed;
1215 }
1216
1217 static void
1218 iscsi_tcp_data_ready(struct sock *sk, int flag)
1219 {
1220         struct iscsi_conn *conn = sk->sk_user_data;
1221         read_descriptor_t rd_desc;
1222
1223         read_lock(&sk->sk_callback_lock);
1224
1225         /* use rd_desc to pass 'conn' to iscsi_tcp_data_recv */
1226         rd_desc.arg.data = conn;
1227         rd_desc.count = 0;
1228         tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv);
1229
1230         read_unlock(&sk->sk_callback_lock);
1231 }
1232
1233 static void
1234 iscsi_tcp_state_change(struct sock *sk)
1235 {
1236         struct iscsi_conn *conn;
1237         struct iscsi_session *session;
1238         void (*old_state_change)(struct sock *);
1239
1240         read_lock(&sk->sk_callback_lock);
1241
1242         conn = (struct iscsi_conn*)sk->sk_user_data;
1243         session = conn->session;
1244
1245         if ((sk->sk_state == TCP_CLOSE_WAIT ||
1246              sk->sk_state == TCP_CLOSE) &&
1247             !atomic_read(&sk->sk_rmem_alloc)) {
1248                 debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
1249                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1250         }
1251
1252         old_state_change = conn->old_state_change;
1253
1254         read_unlock(&sk->sk_callback_lock);
1255
1256         old_state_change(sk);
1257 }
1258
1259 /**
1260  * iscsi_write_space - Called when more output buffer space is available
1261  * @sk: socket space is available for
1262  **/
1263 static void
1264 iscsi_write_space(struct sock *sk)
1265 {
1266         struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
1267         conn->old_write_space(sk);
1268         debug_tcp("iscsi_write_space: cid %d\n", conn->id);
1269         clear_bit(SUSPEND_BIT, &conn->suspend_tx);
1270         schedule_work(&conn->xmitwork);
1271 }
1272
1273 static void
1274 iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1275 {
1276         struct sock *sk = conn->sock->sk;
1277
1278         /* assign new callbacks */
1279         write_lock_bh(&sk->sk_callback_lock);
1280         sk->sk_user_data = conn;
1281         conn->old_data_ready = sk->sk_data_ready;
1282         conn->old_state_change = sk->sk_state_change;
1283         conn->old_write_space = sk->sk_write_space;
1284         sk->sk_data_ready = iscsi_tcp_data_ready;
1285         sk->sk_state_change = iscsi_tcp_state_change;
1286         sk->sk_write_space = iscsi_write_space;
1287         write_unlock_bh(&sk->sk_callback_lock);
1288 }
1289
1290 static void
1291 iscsi_conn_restore_callbacks(struct iscsi_conn *conn)
1292 {
1293         struct sock *sk = conn->sock->sk;
1294
1295         /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1296         write_lock_bh(&sk->sk_callback_lock);
1297         sk->sk_user_data    = NULL;
1298         sk->sk_data_ready   = conn->old_data_ready;
1299         sk->sk_state_change = conn->old_state_change;
1300         sk->sk_write_space  = conn->old_write_space;
1301         sk->sk_no_check  = 0;
1302         write_unlock_bh(&sk->sk_callback_lock);
1303 }
1304
1305 /**
1306  * iscsi_send - generic send routine
1307  * @sk: kernel's socket
1308  * @buf: buffer to write from
1309  * @size: actual size to write
1310  * @flags: socket's flags
1311  */
1312 static inline int
1313 iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags)
1314 {
1315         struct socket *sk = conn->sock;
1316         int offset = buf->sg.offset + buf->sent;
1317
1318         /*
1319          * if we got use_sg=0 or are sending something we kmallocd
1320          * then we did not have to do kmap (kmap returns page_address)
1321          *
1322          * if we got use_sg > 0, but had to drop down, we do not
1323          * set clustering so this should only happen for that
1324          * slab case.
1325          */
1326         if (buf->use_sendmsg)
1327                 return sock_no_sendpage(sk, buf->sg.page, offset, size, flags);
1328         else
1329                 return conn->sendpage(sk, buf->sg.page, offset, size, flags);
1330 }
1331
1332 /**
1333  * iscsi_sendhdr - send PDU Header via tcp_sendpage()
1334  * @conn: iscsi connection
1335  * @buf: buffer to write from
1336  * @datalen: lenght of data to be sent after the header
1337  *
1338  * Notes:
1339  *      (Tx, Fast Path)
1340  **/
1341 static inline int
1342 iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen)
1343 {
1344         int flags = 0; /* MSG_DONTWAIT; */
1345         int res, size;
1346
1347         size = buf->sg.length - buf->sent;
1348         BUG_ON(buf->sent + size > buf->sg.length);
1349         if (buf->sent + size != buf->sg.length || datalen)
1350                 flags |= MSG_MORE;
1351
1352         res = iscsi_send(conn, buf, size, flags);
1353         debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res);
1354         if (res >= 0) {
1355                 conn->txdata_octets += res;
1356                 buf->sent += res;
1357                 if (size != res)
1358                         return -EAGAIN;
1359                 return 0;
1360         } else if (res == -EAGAIN) {
1361                 conn->sendpage_failures_cnt++;
1362                 set_bit(SUSPEND_BIT, &conn->suspend_tx);
1363         } else if (res == -EPIPE)
1364                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1365
1366         return res;
1367 }
1368
1369 /**
1370  * iscsi_sendpage - send one page of iSCSI Data-Out.
1371  * @conn: iscsi connection
1372  * @buf: buffer to write from
1373  * @count: remaining data
1374  * @sent: number of bytes sent
1375  *
1376  * Notes:
1377  *      (Tx, Fast Path)
1378  **/
1379 static inline int
1380 iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
1381                int *count, int *sent)
1382 {
1383         int flags = 0; /* MSG_DONTWAIT; */
1384         int res, size;
1385
1386         size = buf->sg.length - buf->sent;
1387         BUG_ON(buf->sent + size > buf->sg.length);
1388         if (size > *count)
1389                 size = *count;
1390         if (buf->sent + size != buf->sg.length || *count != size)
1391                 flags |= MSG_MORE;
1392
1393         res = iscsi_send(conn, buf, size, flags);
1394         debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n",
1395                   size, buf->sent, *count, *sent, res);
1396         if (res >= 0) {
1397                 conn->txdata_octets += res;
1398                 buf->sent += res;
1399                 *count -= res;
1400                 *sent += res;
1401                 if (size != res)
1402                         return -EAGAIN;
1403                 return 0;
1404         } else if (res == -EAGAIN) {
1405                 conn->sendpage_failures_cnt++;
1406                 set_bit(SUSPEND_BIT, &conn->suspend_tx);
1407         } else if (res == -EPIPE)
1408                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1409
1410         return res;
1411 }
1412
1413 static inline void
1414 iscsi_data_digest_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1415 {
1416         BUG_ON(!conn->data_tx_tfm);
1417         crypto_digest_init(conn->data_tx_tfm);
1418         ctask->digest_count = 4;
1419 }
1420
1421 static inline int
1422 iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1423                         struct iscsi_buf *buf, uint32_t *digest, int final)
1424 {
1425         int rc = 0;
1426         int sent = 0;
1427
1428         if (final)
1429                 crypto_digest_final(conn->data_tx_tfm, (u8*)digest);
1430
1431         iscsi_buf_init_virt(buf, (char*)digest, 4);
1432         rc = iscsi_sendpage(conn, buf, &ctask->digest_count, &sent);
1433         if (rc) {
1434                 ctask->datadigest = *digest;
1435                 ctask->xmstate |= XMSTATE_DATA_DIGEST;
1436         } else
1437                 ctask->digest_count = 4;
1438         return rc;
1439 }
1440
1441 /**
1442  * iscsi_solicit_data_cont - initialize next Data-Out
1443  * @conn: iscsi connection
1444  * @ctask: scsi command task
1445  * @r2t: R2T info
1446  * @left: bytes left to transfer
1447  *
1448  * Notes:
1449  *      Initialize next Data-Out within this R2T sequence and continue
1450  *      to process next Scatter-Gather element(if any) of this SCSI command.
1451  *
1452  *      Called under connection lock.
1453  **/
1454 static void
1455 iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1456                         struct iscsi_r2t_info *r2t, int left)
1457 {
1458         struct iscsi_data *hdr;
1459         struct iscsi_data_task *dtask;
1460         struct scsi_cmnd *sc = ctask->sc;
1461         int new_offset;
1462
1463         dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
1464         BUG_ON(!dtask);
1465         hdr = &dtask->hdr;
1466         memset(hdr, 0, sizeof(struct iscsi_data));
1467         hdr->ttt = r2t->ttt;
1468         hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
1469         r2t->solicit_datasn++;
1470         hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1471         memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
1472         hdr->itt = ctask->hdr.itt;
1473         hdr->exp_statsn = r2t->exp_statsn;
1474         new_offset = r2t->data_offset + r2t->sent;
1475         hdr->offset = cpu_to_be32(new_offset);
1476         if (left > conn->max_xmit_dlength) {
1477                 hton24(hdr->dlength, conn->max_xmit_dlength);
1478                 r2t->data_count = conn->max_xmit_dlength;
1479         } else {
1480                 hton24(hdr->dlength, left);
1481                 r2t->data_count = left;
1482                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1483         }
1484         conn->dataout_pdus_cnt++;
1485
1486         iscsi_buf_init_virt(&r2t->headbuf, (char*)hdr,
1487                            sizeof(struct iscsi_hdr));
1488
1489         r2t->dtask = dtask;
1490
1491         if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) {
1492                 BUG_ON(ctask->bad_sg == r2t->sg);
1493                 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1494                 r2t->sg += 1;
1495         } else
1496                 iscsi_buf_init_iov(&ctask->sendbuf,
1497                             (char*)sc->request_buffer + new_offset,
1498                             r2t->data_count);
1499
1500         list_add(&dtask->item, &ctask->dataqueue);
1501 }
1502
1503 static void
1504 iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1505 {
1506         struct iscsi_data *hdr;
1507         struct iscsi_data_task *dtask;
1508
1509         dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
1510         BUG_ON(!dtask);
1511         hdr = &dtask->hdr;
1512         memset(hdr, 0, sizeof(struct iscsi_data));
1513         hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1514         hdr->datasn = cpu_to_be32(ctask->unsol_datasn);
1515         ctask->unsol_datasn++;
1516         hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1517         memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
1518         hdr->itt = ctask->hdr.itt;
1519         hdr->exp_statsn = cpu_to_be32(conn->exp_statsn);
1520         hdr->offset = cpu_to_be32(ctask->total_length -
1521                                   ctask->r2t_data_count -
1522                                   ctask->unsol_count);
1523         if (ctask->unsol_count > conn->max_xmit_dlength) {
1524                 hton24(hdr->dlength, conn->max_xmit_dlength);
1525                 ctask->data_count = conn->max_xmit_dlength;
1526                 hdr->flags = 0;
1527         } else {
1528                 hton24(hdr->dlength, ctask->unsol_count);
1529                 ctask->data_count = ctask->unsol_count;
1530                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1531         }
1532
1533         iscsi_buf_init_virt(&ctask->headbuf, (char*)hdr,
1534                            sizeof(struct iscsi_hdr));
1535
1536         list_add(&dtask->item, &ctask->dataqueue);
1537
1538         ctask->dtask = dtask;
1539 }
1540
1541 /**
1542  * iscsi_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1543  * @conn: iscsi connection
1544  * @ctask: scsi command task
1545  * @sc: scsi command
1546  **/
1547 static void
1548 iscsi_cmd_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1549                 struct scsi_cmnd *sc)
1550 {
1551         struct iscsi_session *session = conn->session;
1552
1553         BUG_ON(__kfifo_len(ctask->r2tqueue));
1554
1555         ctask->sc = sc;
1556         ctask->conn = conn;
1557         ctask->hdr.opcode = ISCSI_OP_SCSI_CMD;
1558         ctask->hdr.flags = ISCSI_ATTR_SIMPLE;
1559         int_to_scsilun(sc->device->lun, (struct scsi_lun *)ctask->hdr.lun);
1560         ctask->hdr.itt = ctask->itt | (conn->id << CID_SHIFT) |
1561                          (session->age << AGE_SHIFT);
1562         ctask->hdr.data_length = cpu_to_be32(sc->request_bufflen);
1563         ctask->hdr.cmdsn = cpu_to_be32(session->cmdsn); session->cmdsn++;
1564         ctask->hdr.exp_statsn = cpu_to_be32(conn->exp_statsn);
1565         memcpy(ctask->hdr.cdb, sc->cmnd, sc->cmd_len);
1566         memset(&ctask->hdr.cdb[sc->cmd_len], 0, MAX_COMMAND_SIZE - sc->cmd_len);
1567
1568         ctask->mtask = NULL;
1569         ctask->sent = 0;
1570         ctask->sg_count = 0;
1571
1572         ctask->total_length = sc->request_bufflen;
1573
1574         if (sc->sc_data_direction == DMA_TO_DEVICE) {
1575                 ctask->exp_r2tsn = 0;
1576                 ctask->hdr.flags |= ISCSI_FLAG_CMD_WRITE;
1577                 BUG_ON(ctask->total_length == 0);
1578                 if (sc->use_sg) {
1579                         struct scatterlist *sg = sc->request_buffer;
1580
1581                         iscsi_buf_init_sg(&ctask->sendbuf,
1582                                           &sg[ctask->sg_count++]);
1583                         ctask->sg = sg;
1584                         ctask->bad_sg = sg + sc->use_sg;
1585                 } else {
1586                         iscsi_buf_init_iov(&ctask->sendbuf, sc->request_buffer,
1587                                         sc->request_bufflen);
1588                 }
1589
1590                 /*
1591                  * Write counters:
1592                  *
1593                  *      imm_count       bytes to be sent right after
1594                  *                      SCSI PDU Header
1595                  *
1596                  *      unsol_count     bytes(as Data-Out) to be sent
1597                  *                      without R2T ack right after
1598                  *                      immediate data
1599                  *
1600                  *      r2t_data_count  bytes to be sent via R2T ack's
1601                  *
1602                  *      pad_count       bytes to be sent as zero-padding
1603                  */
1604                 ctask->imm_count = 0;
1605                 ctask->unsol_count = 0;
1606                 ctask->unsol_datasn = 0;
1607                 ctask->xmstate = XMSTATE_W_HDR;
1608                 /* calculate write padding */
1609                 ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1);
1610                 if (ctask->pad_count) {
1611                         ctask->pad_count = ISCSI_PAD_LEN - ctask->pad_count;
1612                         debug_scsi("write padding %d bytes\n",
1613                                 ctask->pad_count);
1614                         ctask->xmstate |= XMSTATE_W_PAD;
1615                 }
1616                 if (session->imm_data_en) {
1617                         if (ctask->total_length >= session->first_burst)
1618                                 ctask->imm_count = min(session->first_burst,
1619                                                         conn->max_xmit_dlength);
1620                         else
1621                                 ctask->imm_count = min(ctask->total_length,
1622                                                         conn->max_xmit_dlength);
1623                         hton24(ctask->hdr.dlength, ctask->imm_count);
1624                         ctask->xmstate |= XMSTATE_IMM_DATA;
1625                 } else
1626                         zero_data(ctask->hdr.dlength);
1627
1628                 if (!session->initial_r2t_en)
1629                         ctask->unsol_count = min(session->first_burst,
1630                                 ctask->total_length) - ctask->imm_count;
1631                 if (!ctask->unsol_count)
1632                         /* No unsolicit Data-Out's */
1633                         ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL;
1634                 else
1635                         ctask->xmstate |= XMSTATE_UNS_HDR | XMSTATE_UNS_INIT;
1636
1637                 ctask->r2t_data_count = ctask->total_length -
1638                                     ctask->imm_count -
1639                                     ctask->unsol_count;
1640
1641                 debug_scsi("cmd [itt %x total %d imm %d imm_data %d "
1642                            "r2t_data %d]\n",
1643                            ctask->itt, ctask->total_length, ctask->imm_count,
1644                            ctask->unsol_count, ctask->r2t_data_count);
1645         } else {
1646                 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL;
1647                 if (sc->sc_data_direction == DMA_FROM_DEVICE)
1648                         ctask->hdr.flags |= ISCSI_FLAG_CMD_READ;
1649                 ctask->datasn = 0;
1650                 ctask->xmstate = XMSTATE_R_HDR;
1651                 zero_data(ctask->hdr.dlength);
1652         }
1653
1654         iscsi_buf_init_virt(&ctask->headbuf, (char*)&ctask->hdr,
1655                             sizeof(struct iscsi_hdr));
1656         conn->scsicmd_pdus_cnt++;
1657 }
1658
1659 /**
1660  * iscsi_mtask_xmit - xmit management(immediate) task
1661  * @conn: iscsi connection
1662  * @mtask: task management task
1663  *
1664  * Notes:
1665  *      The function can return -EAGAIN in which case caller must
1666  *      call it again later, or recover. '0' return code means successful
1667  *      xmit.
1668  *
1669  *      Management xmit state machine consists of two states:
1670  *              IN_PROGRESS_IMM_HEAD - PDU Header xmit in progress
1671  *              IN_PROGRESS_IMM_DATA - PDU Data xmit in progress
1672  **/
1673 static int
1674 iscsi_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
1675 {
1676
1677         debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n",
1678                 conn->id, mtask->xmstate, mtask->itt);
1679
1680         if (mtask->xmstate & XMSTATE_IMM_HDR) {
1681                 mtask->xmstate &= ~XMSTATE_IMM_HDR;
1682                 if (mtask->data_count)
1683                         mtask->xmstate |= XMSTATE_IMM_DATA;
1684                 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
1685                     conn->stop_stage != STOP_CONN_RECOVER &&
1686                     conn->hdrdgst_en)
1687                         iscsi_hdr_digest(conn, &mtask->headbuf,
1688                                         (u8*)mtask->hdrext);
1689                 if (iscsi_sendhdr(conn, &mtask->headbuf, mtask->data_count)) {
1690                         mtask->xmstate |= XMSTATE_IMM_HDR;
1691                         if (mtask->data_count)
1692                                 mtask->xmstate &= ~XMSTATE_IMM_DATA;
1693                         return -EAGAIN;
1694                 }
1695         }
1696
1697         if (mtask->xmstate & XMSTATE_IMM_DATA) {
1698                 BUG_ON(!mtask->data_count);
1699                 mtask->xmstate &= ~XMSTATE_IMM_DATA;
1700                 /* FIXME: implement.
1701                  * Virtual buffer could be spreaded across multiple pages...
1702                  */
1703                 do {
1704                         if (iscsi_sendpage(conn, &mtask->sendbuf,
1705                                    &mtask->data_count, &mtask->sent)) {
1706                                 mtask->xmstate |= XMSTATE_IMM_DATA;
1707                                 return -EAGAIN;
1708                         }
1709                 } while (mtask->data_count);
1710         }
1711
1712         BUG_ON(mtask->xmstate != XMSTATE_IDLE);
1713         return 0;
1714 }
1715
1716 static inline int
1717 handle_xmstate_r_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1718 {
1719         ctask->xmstate &= ~XMSTATE_R_HDR;
1720         if (conn->hdrdgst_en)
1721                 iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext);
1722         if (!iscsi_sendhdr(conn, &ctask->headbuf, 0)) {
1723                 BUG_ON(ctask->xmstate != XMSTATE_IDLE);
1724                 return 0; /* wait for Data-In */
1725         }
1726         ctask->xmstate |= XMSTATE_R_HDR;
1727         return -EAGAIN;
1728 }
1729
1730 static inline int
1731 handle_xmstate_w_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1732 {
1733         ctask->xmstate &= ~XMSTATE_W_HDR;
1734         if (conn->hdrdgst_en)
1735                 iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext);
1736         if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->imm_count)) {
1737                 ctask->xmstate |= XMSTATE_W_HDR;
1738                 return -EAGAIN;
1739         }
1740         return 0;
1741 }
1742
1743 static inline int
1744 handle_xmstate_data_digest(struct iscsi_conn *conn,
1745                            struct iscsi_cmd_task *ctask)
1746 {
1747         ctask->xmstate &= ~XMSTATE_DATA_DIGEST;
1748         debug_tcp("resent data digest 0x%x\n", ctask->datadigest);
1749         if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
1750                                     &ctask->datadigest, 0)) {
1751                 ctask->xmstate |= XMSTATE_DATA_DIGEST;
1752                 debug_tcp("resent data digest 0x%x fail!\n",
1753                           ctask->datadigest);
1754                 return -EAGAIN;
1755         }
1756         return 0;
1757 }
1758
1759 static inline int
1760 handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1761 {
1762         BUG_ON(!ctask->imm_count);
1763         ctask->xmstate &= ~XMSTATE_IMM_DATA;
1764
1765         if (conn->datadgst_en) {
1766                 iscsi_data_digest_init(conn, ctask);
1767                 ctask->immdigest = 0;
1768         }
1769
1770         for (;;) {
1771                 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->imm_count,
1772                                    &ctask->sent)) {
1773                         ctask->xmstate |= XMSTATE_IMM_DATA;
1774                         if (conn->datadgst_en) {
1775                                 crypto_digest_final(conn->data_tx_tfm,
1776                                                 (u8*)&ctask->immdigest);
1777                                 debug_tcp("tx imm sendpage fail 0x%x\n",
1778                                           ctask->datadigest);
1779                         }
1780                         return -EAGAIN;
1781                 }
1782                 if (conn->datadgst_en)
1783                         crypto_digest_update(conn->data_tx_tfm,
1784                                              &ctask->sendbuf.sg, 1);
1785
1786                 if (!ctask->imm_count)
1787                         break;
1788                 iscsi_buf_init_sg(&ctask->sendbuf,
1789                                   &ctask->sg[ctask->sg_count++]);
1790         }
1791
1792         if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) {
1793                 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
1794                                             &ctask->immdigest, 1)) {
1795                         debug_tcp("sending imm digest 0x%x fail!\n",
1796                                   ctask->immdigest);
1797                         return -EAGAIN;
1798                 }
1799                 debug_tcp("sending imm digest 0x%x\n", ctask->immdigest);
1800         }
1801
1802         return 0;
1803 }
1804
1805 static inline int
1806 handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1807 {
1808         struct iscsi_data_task *dtask;
1809
1810         ctask->xmstate |= XMSTATE_UNS_DATA;
1811         if (ctask->xmstate & XMSTATE_UNS_INIT) {
1812                 iscsi_unsolicit_data_init(conn, ctask);
1813                 BUG_ON(!ctask->dtask);
1814                 dtask = ctask->dtask;
1815                 if (conn->hdrdgst_en)
1816                         iscsi_hdr_digest(conn, &ctask->headbuf,
1817                                         (u8*)dtask->hdrext);
1818                 ctask->xmstate &= ~XMSTATE_UNS_INIT;
1819         }
1820         if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->data_count)) {
1821                 ctask->xmstate &= ~XMSTATE_UNS_DATA;
1822                 ctask->xmstate |= XMSTATE_UNS_HDR;
1823                 return -EAGAIN;
1824         }
1825
1826         debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
1827                    ctask->itt, ctask->unsol_count, ctask->sent);
1828         return 0;
1829 }
1830
1831 static inline int
1832 handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1833 {
1834         struct iscsi_data_task *dtask = ctask->dtask;
1835
1836         BUG_ON(!ctask->data_count);
1837         ctask->xmstate &= ~XMSTATE_UNS_DATA;
1838
1839         if (conn->datadgst_en) {
1840                 iscsi_data_digest_init(conn, ctask);
1841                 dtask->digest = 0;
1842         }
1843
1844         for (;;) {
1845                 int start = ctask->sent;
1846
1847                 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->data_count,
1848                                    &ctask->sent)) {
1849                         ctask->unsol_count -= ctask->sent - start;
1850                         ctask->xmstate |= XMSTATE_UNS_DATA;
1851                         /* will continue with this ctask later.. */
1852                         if (conn->datadgst_en) {
1853                                 crypto_digest_final(conn->data_tx_tfm,
1854                                                 (u8 *)&dtask->digest);
1855                                 debug_tcp("tx uns data fail 0x%x\n",
1856                                           dtask->digest);
1857                         }
1858                         return -EAGAIN;
1859                 }
1860
1861                 BUG_ON(ctask->sent > ctask->total_length);
1862                 ctask->unsol_count -= ctask->sent - start;
1863
1864                 /*
1865                  * XXX:we may run here with un-initial sendbuf.
1866                  * so pass it
1867                  */
1868                 if (conn->datadgst_en && ctask->sent - start > 0)
1869                         crypto_digest_update(conn->data_tx_tfm,
1870                                              &ctask->sendbuf.sg, 1);
1871
1872                 if (!ctask->data_count)
1873                         break;
1874                 iscsi_buf_init_sg(&ctask->sendbuf,
1875                                   &ctask->sg[ctask->sg_count++]);
1876         }
1877         BUG_ON(ctask->unsol_count < 0);
1878
1879         /*
1880          * Done with the Data-Out. Next, check if we need
1881          * to send another unsolicited Data-Out.
1882          */
1883         if (ctask->unsol_count) {
1884                 if (conn->datadgst_en) {
1885                         if (iscsi_digest_final_send(conn, ctask,
1886                                                     &dtask->digestbuf,
1887                                                     &dtask->digest, 1)) {
1888                                 debug_tcp("send uns digest 0x%x fail\n",
1889                                           dtask->digest);
1890                                 return -EAGAIN;
1891                         }
1892                         debug_tcp("sending uns digest 0x%x, more uns\n",
1893                                   dtask->digest);
1894                 }
1895                 ctask->xmstate |= XMSTATE_UNS_INIT;
1896                 return 1;
1897         }
1898
1899         if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) {
1900                 if (iscsi_digest_final_send(conn, ctask,
1901                                             &dtask->digestbuf,
1902                                             &dtask->digest, 1)) {
1903                         debug_tcp("send last uns digest 0x%x fail\n",
1904                                    dtask->digest);
1905                         return -EAGAIN;
1906                 }
1907                 debug_tcp("sending uns digest 0x%x\n",dtask->digest);
1908         }
1909
1910         return 0;
1911 }
1912
1913 static inline int
1914 handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1915 {
1916         struct iscsi_session *session = conn->session;
1917         struct iscsi_r2t_info *r2t = ctask->r2t;
1918         struct iscsi_data_task *dtask = r2t->dtask;
1919         int left;
1920
1921         ctask->xmstate &= ~XMSTATE_SOL_DATA;
1922         ctask->dtask = dtask;
1923
1924         if (conn->datadgst_en) {
1925                 iscsi_data_digest_init(conn, ctask);
1926                 dtask->digest = 0;
1927         }
1928 solicit_again:
1929         /*
1930          * send Data-Out whitnin this R2T sequence.
1931          */
1932         if (!r2t->data_count)
1933                 goto data_out_done;
1934
1935         if (iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent)) {
1936                 ctask->xmstate |= XMSTATE_SOL_DATA;
1937                 /* will continue with this ctask later.. */
1938                 if (conn->datadgst_en) {
1939                         crypto_digest_final(conn->data_tx_tfm,
1940                                           (u8 *)&dtask->digest);
1941                         debug_tcp("r2t data send fail 0x%x\n", dtask->digest);
1942                 }
1943                 return -EAGAIN;
1944         }
1945
1946         BUG_ON(r2t->data_count < 0);
1947         if (conn->datadgst_en)
1948                 crypto_digest_update(conn->data_tx_tfm, &r2t->sendbuf.sg, 1);
1949
1950         if (r2t->data_count) {
1951                 BUG_ON(ctask->sc->use_sg == 0);
1952                 if (!iscsi_buf_left(&r2t->sendbuf)) {
1953                         BUG_ON(ctask->bad_sg == r2t->sg);
1954                         iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1955                         r2t->sg += 1;
1956                 }
1957                 goto solicit_again;
1958         }
1959
1960 data_out_done:
1961         /*
1962          * Done with this Data-Out. Next, check if we have
1963          * to send another Data-Out for this R2T.
1964          */
1965         BUG_ON(r2t->data_length - r2t->sent < 0);
1966         left = r2t->data_length - r2t->sent;
1967         if (left) {
1968                 if (conn->datadgst_en) {
1969                         if (iscsi_digest_final_send(conn, ctask,
1970                                                     &dtask->digestbuf,
1971                                                     &dtask->digest, 1)) {
1972                                 debug_tcp("send r2t data digest 0x%x"
1973                                           "fail\n", dtask->digest);
1974                                 return -EAGAIN;
1975                         }
1976                         debug_tcp("r2t data send digest 0x%x\n",
1977                                   dtask->digest);
1978                 }
1979                 iscsi_solicit_data_cont(conn, ctask, r2t, left);
1980                 ctask->xmstate |= XMSTATE_SOL_DATA;
1981                 ctask->xmstate &= ~XMSTATE_SOL_HDR;
1982                 return 1;
1983         }
1984
1985         /*
1986          * Done with this R2T. Check if there are more
1987          * outstanding R2Ts ready to be processed.
1988          */
1989         BUG_ON(ctask->r2t_data_count - r2t->data_length < 0);
1990         if (conn->datadgst_en) {
1991                 if (iscsi_digest_final_send(conn, ctask, &dtask->digestbuf,
1992                                             &dtask->digest, 1)) {
1993                         debug_tcp("send last r2t data digest 0x%x"
1994                                   "fail\n", dtask->digest);
1995                         return -EAGAIN;
1996                 }
1997                 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest);
1998         }
1999
2000         ctask->r2t_data_count -= r2t->data_length;
2001         ctask->r2t = NULL;
2002         spin_lock_bh(&session->lock);
2003         __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
2004         spin_unlock_bh(&session->lock);
2005         if (__kfifo_get(ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
2006                 ctask->r2t = r2t;
2007                 ctask->xmstate |= XMSTATE_SOL_DATA;
2008                 ctask->xmstate &= ~XMSTATE_SOL_HDR;
2009                 return 1;
2010         }
2011
2012         return 0;
2013 }
2014
2015 static inline int
2016 handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2017 {
2018         struct iscsi_data_task *dtask = ctask->dtask;
2019         int sent;
2020
2021         ctask->xmstate &= ~XMSTATE_W_PAD;
2022         iscsi_buf_init_virt(&ctask->sendbuf, (char*)&ctask->pad,
2023                             ctask->pad_count);
2024         if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->pad_count, &sent)) {
2025                 ctask->xmstate |= XMSTATE_W_PAD;
2026                 return -EAGAIN;
2027         }
2028
2029         if (conn->datadgst_en) {
2030                 crypto_digest_update(conn->data_tx_tfm, &ctask->sendbuf.sg, 1);
2031                 /* imm data? */
2032                 if (!dtask) {
2033                         if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
2034                                                     &ctask->immdigest, 1)) {
2035                                 debug_tcp("send padding digest 0x%x"
2036                                           "fail!\n", ctask->immdigest);
2037                                 return -EAGAIN;
2038                         }
2039                         debug_tcp("done with padding, digest 0x%x\n",
2040                                   ctask->datadigest);
2041                 } else {
2042                         if (iscsi_digest_final_send(conn, ctask,
2043                                                     &dtask->digestbuf,
2044                                                     &dtask->digest, 1)) {
2045                                 debug_tcp("send padding digest 0x%x"
2046                                           "fail\n", dtask->digest);
2047                                 return -EAGAIN;
2048                         }
2049                         debug_tcp("done with padding, digest 0x%x\n",
2050                                   dtask->digest);
2051                 }
2052         }
2053
2054         return 0;
2055 }
2056
2057 static int
2058 iscsi_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2059 {
2060         int rc = 0;
2061
2062         debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n",
2063                 conn->id, ctask->xmstate, ctask->itt);
2064
2065         /*
2066          * serialize with TMF AbortTask
2067          */
2068         if (ctask->mtask)
2069                 return rc;
2070
2071         if (ctask->xmstate & XMSTATE_R_HDR) {
2072                 rc = handle_xmstate_r_hdr(conn, ctask);
2073                 return rc;
2074         }
2075
2076         if (ctask->xmstate & XMSTATE_W_HDR) {
2077                 rc = handle_xmstate_w_hdr(conn, ctask);
2078                 if (rc)
2079                         return rc;
2080         }
2081
2082         /* XXX: for data digest xmit recover */
2083         if (ctask->xmstate & XMSTATE_DATA_DIGEST) {
2084                 rc = handle_xmstate_data_digest(conn, ctask);
2085                 if (rc)
2086                         return rc;
2087         }
2088
2089         if (ctask->xmstate & XMSTATE_IMM_DATA) {
2090                 rc = handle_xmstate_imm_data(conn, ctask);
2091                 if (rc)
2092                         return rc;
2093         }
2094
2095         if (ctask->xmstate & XMSTATE_UNS_HDR) {
2096                 BUG_ON(!ctask->unsol_count);
2097                 ctask->xmstate &= ~XMSTATE_UNS_HDR;
2098 unsolicit_head_again:
2099                 rc = handle_xmstate_uns_hdr(conn, ctask);
2100                 if (rc)
2101                         return rc;
2102         }
2103
2104         if (ctask->xmstate & XMSTATE_UNS_DATA) {
2105                 rc = handle_xmstate_uns_data(conn, ctask);
2106                 if (rc == 1)
2107                         goto unsolicit_head_again;
2108                 else if (rc)
2109                         return rc;
2110                 goto done;
2111         }
2112
2113         if (ctask->xmstate & XMSTATE_SOL_HDR) {
2114                 struct iscsi_r2t_info *r2t;
2115
2116                 ctask->xmstate &= ~XMSTATE_SOL_HDR;
2117                 ctask->xmstate |= XMSTATE_SOL_DATA;
2118                 if (!ctask->r2t)
2119                         __kfifo_get(ctask->r2tqueue, (void*)&ctask->r2t,
2120                                     sizeof(void*));
2121 solicit_head_again:
2122                 r2t = ctask->r2t;
2123                 if (conn->hdrdgst_en)
2124                         iscsi_hdr_digest(conn, &r2t->headbuf,
2125                                         (u8*)r2t->dtask->hdrext);
2126                 if (iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count)) {
2127                         ctask->xmstate &= ~XMSTATE_SOL_DATA;
2128                         ctask->xmstate |= XMSTATE_SOL_HDR;
2129                         return -EAGAIN;
2130                 }
2131
2132                 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
2133                         r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
2134                         r2t->sent);
2135         }
2136
2137         if (ctask->xmstate & XMSTATE_SOL_DATA) {
2138                 rc = handle_xmstate_sol_data(conn, ctask);
2139                 if (rc == 1)
2140                         goto solicit_head_again;
2141                 if (rc)
2142                         return rc;
2143         }
2144
2145 done:
2146         /*
2147          * Last thing to check is whether we need to send write
2148          * padding. Note that we check for xmstate equality, not just the bit.
2149          */
2150         if (ctask->xmstate == XMSTATE_W_PAD)
2151                 rc = handle_xmstate_w_pad(conn, ctask);
2152
2153         return rc;
2154 }
2155
2156 /**
2157  * iscsi_data_xmit - xmit any command into the scheduled connection
2158  * @conn: iscsi connection
2159  *
2160  * Notes:
2161  *      The function can return -EAGAIN in which case the caller must
2162  *      re-schedule it again later or recover. '0' return code means
2163  *      successful xmit.
2164  **/
2165 static int
2166 iscsi_data_xmit(struct iscsi_conn *conn)
2167 {
2168         if (unlikely(conn->suspend_tx)) {
2169                 debug_tcp("conn %d Tx suspended!\n", conn->id);
2170                 return 0;
2171         }
2172
2173         /*
2174          * Transmit in the following order:
2175          *
2176          * 1) un-finished xmit (ctask or mtask)
2177          * 2) immediate control PDUs
2178          * 3) write data
2179          * 4) SCSI commands
2180          * 5) non-immediate control PDUs
2181          *
2182          * No need to lock around __kfifo_get as long as
2183          * there's one producer and one consumer.
2184          */
2185
2186         BUG_ON(conn->ctask && conn->mtask);
2187
2188         if (conn->ctask) {
2189                 if (iscsi_ctask_xmit(conn, conn->ctask))
2190                         goto again;
2191                 /* done with this in-progress ctask */
2192                 conn->ctask = NULL;
2193         }
2194         if (conn->mtask) {
2195                 if (iscsi_mtask_xmit(conn, conn->mtask))
2196                         goto again;
2197                 /* done with this in-progress mtask */
2198                 conn->mtask = NULL;
2199         }
2200
2201         /* process immediate first */
2202         if (unlikely(__kfifo_len(conn->immqueue))) {
2203                 struct iscsi_session *session = conn->session;
2204                 while (__kfifo_get(conn->immqueue, (void*)&conn->mtask,
2205                                    sizeof(void*))) {
2206                         if (iscsi_mtask_xmit(conn, conn->mtask))
2207                                 goto again;
2208
2209                         if (conn->mtask->hdr.itt ==
2210                                         cpu_to_be32(ISCSI_RESERVED_TAG)) {
2211                                 spin_lock_bh(&session->lock);
2212                                 __kfifo_put(session->mgmtpool.queue,
2213                                             (void*)&conn->mtask, sizeof(void*));
2214                                 spin_unlock_bh(&session->lock);
2215                         }
2216                 }
2217                 /* done with this mtask */
2218                 conn->mtask = NULL;
2219         }
2220
2221         /* process write queue */
2222         while (__kfifo_get(conn->writequeue, (void*)&conn->ctask,
2223                            sizeof(void*))) {
2224                 if (iscsi_ctask_xmit(conn, conn->ctask))
2225                         goto again;
2226         }
2227
2228         /* process command queue */
2229         while (__kfifo_get(conn->xmitqueue, (void*)&conn->ctask,
2230                            sizeof(void*))) {
2231                 if (iscsi_ctask_xmit(conn, conn->ctask))
2232                         goto again;
2233         }
2234         /* done with this ctask */
2235         conn->ctask = NULL;
2236
2237         /* process the rest control plane PDUs, if any */
2238         if (unlikely(__kfifo_len(conn->mgmtqueue))) {
2239                 struct iscsi_session *session = conn->session;
2240
2241                 while (__kfifo_get(conn->mgmtqueue, (void*)&conn->mtask,
2242                                    sizeof(void*))) {
2243                         if (iscsi_mtask_xmit(conn, conn->mtask))
2244                                 goto again;
2245
2246                         if (conn->mtask->hdr.itt ==
2247                                         cpu_to_be32(ISCSI_RESERVED_TAG)) {
2248                                 spin_lock_bh(&session->lock);
2249                                 __kfifo_put(session->mgmtpool.queue,
2250                                             (void*)&conn->mtask,
2251                                             sizeof(void*));
2252                                 spin_unlock_bh(&session->lock);
2253                         }
2254                 }
2255                 /* done with this mtask */
2256                 conn->mtask = NULL;
2257         }
2258
2259         return 0;
2260
2261 again:
2262         if (unlikely(conn->suspend_tx))
2263                 return 0;
2264
2265         return -EAGAIN;
2266 }
2267
2268 static void
2269 iscsi_xmitworker(void *data)
2270 {
2271         struct iscsi_conn *conn = data;
2272
2273         /*
2274          * serialize Xmit worker on a per-connection basis.
2275          */
2276         mutex_lock(&conn->xmitmutex);
2277         if (iscsi_data_xmit(conn))
2278                 schedule_work(&conn->xmitwork);
2279         mutex_unlock(&conn->xmitmutex);
2280 }
2281
2282 #define FAILURE_BAD_HOST                1
2283 #define FAILURE_SESSION_FAILED          2
2284 #define FAILURE_SESSION_FREED           3
2285 #define FAILURE_WINDOW_CLOSED           4
2286 #define FAILURE_SESSION_TERMINATE       5
2287
2288 static int
2289 iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
2290 {
2291         struct Scsi_Host *host;
2292         int reason = 0;
2293         struct iscsi_session *session;
2294         struct iscsi_conn *conn = NULL;
2295         struct iscsi_cmd_task *ctask = NULL;
2296
2297         sc->scsi_done = done;
2298         sc->result = 0;
2299
2300         host = sc->device->host;
2301         session = iscsi_hostdata(host->hostdata);
2302         BUG_ON(host != session->host);
2303
2304         spin_lock(&session->lock);
2305
2306         if (session->state != ISCSI_STATE_LOGGED_IN) {
2307                 if (session->state == ISCSI_STATE_FAILED) {
2308                         reason = FAILURE_SESSION_FAILED;
2309                         goto reject;
2310                 } else if (session->state == ISCSI_STATE_TERMINATE) {
2311                         reason = FAILURE_SESSION_TERMINATE;
2312                         goto fault;
2313                 }
2314                 reason = FAILURE_SESSION_FREED;
2315                 goto fault;
2316         }
2317
2318         /*
2319          * Check for iSCSI window and take care of CmdSN wrap-around
2320          */
2321         if ((int)(session->max_cmdsn - session->cmdsn) < 0) {
2322                 reason = FAILURE_WINDOW_CLOSED;
2323                 goto reject;
2324         }
2325
2326         conn = session->leadconn;
2327
2328         __kfifo_get(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
2329         BUG_ON(ctask->sc);
2330
2331         sc->SCp.phase = session->age;
2332         sc->SCp.ptr = (char*)ctask;
2333         iscsi_cmd_init(conn, ctask, sc);
2334
2335         __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*));
2336         debug_scsi(
2337                "ctask enq [%s cid %d sc %lx itt 0x%x len %d cmdsn %d win %d]\n",
2338                 sc->sc_data_direction == DMA_TO_DEVICE ? "write" : "read",
2339                 conn->id, (long)sc, ctask->itt, sc->request_bufflen,
2340                 session->cmdsn, session->max_cmdsn - session->exp_cmdsn + 1);
2341         spin_unlock(&session->lock);
2342
2343         if (!in_interrupt() && mutex_trylock(&conn->xmitmutex)) {
2344                 spin_unlock_irq(host->host_lock);
2345                 if (iscsi_data_xmit(conn))
2346                         schedule_work(&conn->xmitwork);
2347                 mutex_unlock(&conn->xmitmutex);
2348                 spin_lock_irq(host->host_lock);
2349         } else
2350                 schedule_work(&conn->xmitwork);
2351
2352         return 0;
2353
2354 reject:
2355         spin_unlock(&session->lock);
2356         debug_scsi("cmd 0x%x rejected (%d)\n", sc->cmnd[0], reason);
2357         return SCSI_MLQUEUE_HOST_BUSY;
2358
2359 fault:
2360         spin_unlock(&session->lock);
2361         printk(KERN_ERR "iscsi_tcp: cmd 0x%x is not queued (%d)\n",
2362                sc->cmnd[0], reason);
2363         sc->sense_buffer[0] = 0x70;
2364         sc->sense_buffer[2] = NOT_READY;
2365         sc->sense_buffer[7] = 0x6;
2366         sc->sense_buffer[12] = 0x08;
2367         sc->sense_buffer[13] = 0x00;
2368         sc->result = (DID_NO_CONNECT << 16);
2369         sc->resid = sc->request_bufflen;
2370         sc->scsi_done(sc);
2371         return 0;
2372 }
2373
2374 static int
2375 iscsi_change_queue_depth(struct scsi_device *sdev, int depth)
2376 {
2377         if (depth > ISCSI_MAX_CMD_PER_LUN)
2378                 depth = ISCSI_MAX_CMD_PER_LUN;
2379         scsi_adjust_queue_depth(sdev, scsi_get_tag_type(sdev), depth);
2380         return sdev->queue_depth;
2381 }
2382
2383 static int
2384 iscsi_pool_init(struct iscsi_queue *q, int max, void ***items, int item_size)
2385 {
2386         int i;
2387
2388         *items = kmalloc(max * sizeof(void*), GFP_KERNEL);
2389         if (*items == NULL)
2390                 return -ENOMEM;
2391
2392         q->max = max;
2393         q->pool = kmalloc(max * sizeof(void*), GFP_KERNEL);
2394         if (q->pool == NULL) {
2395                 kfree(*items);
2396                 return -ENOMEM;
2397         }
2398
2399         q->queue = kfifo_init((void*)q->pool, max * sizeof(void*),
2400                               GFP_KERNEL, NULL);
2401         if (q->queue == ERR_PTR(-ENOMEM)) {
2402                 kfree(q->pool);
2403                 kfree(*items);
2404                 return -ENOMEM;
2405         }
2406
2407         for (i = 0; i < max; i++) {
2408                 q->pool[i] = kmalloc(item_size, GFP_KERNEL);
2409                 if (q->pool[i] == NULL) {
2410                         int j;
2411
2412                         for (j = 0; j < i; j++)
2413                                 kfree(q->pool[j]);
2414
2415                         kfifo_free(q->queue);
2416                         kfree(q->pool);
2417                         kfree(*items);
2418                         return -ENOMEM;
2419                 }
2420                 memset(q->pool[i], 0, item_size);
2421                 (*items)[i] = q->pool[i];
2422                 __kfifo_put(q->queue, (void*)&q->pool[i], sizeof(void*));
2423         }
2424         return 0;
2425 }
2426
2427 static void
2428 iscsi_pool_free(struct iscsi_queue *q, void **items)
2429 {
2430         int i;
2431
2432         for (i = 0; i < q->max; i++)
2433                 kfree(items[i]);
2434         kfree(q->pool);
2435         kfree(items);
2436 }
2437
2438 static struct iscsi_cls_conn *
2439 iscsi_conn_create(struct Scsi_Host *shost, uint32_t conn_idx)
2440 {
2441         struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
2442         struct iscsi_conn *conn;
2443         struct iscsi_cls_conn *cls_conn;
2444
2445         cls_conn = iscsi_create_conn(hostdata_session(shost->hostdata),
2446                                      conn_idx);
2447         if (!cls_conn)
2448                 return NULL;
2449         conn = cls_conn->dd_data;
2450
2451         memset(conn, 0, sizeof(struct iscsi_conn));
2452         conn->c_stage = ISCSI_CONN_INITIAL_STAGE;
2453         conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2454         conn->id = conn_idx;
2455         conn->exp_statsn = 0;
2456         conn->tmabort_state = TMABORT_INITIAL;
2457
2458         /* initial operational parameters */
2459         conn->hdr_size = sizeof(struct iscsi_hdr);
2460         conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2461         conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2462
2463         spin_lock_init(&conn->lock);
2464
2465         /* initialize general xmit PDU commands queue */
2466         conn->xmitqueue = kfifo_alloc(session->cmds_max * sizeof(void*),
2467                                         GFP_KERNEL, NULL);
2468         if (conn->xmitqueue == ERR_PTR(-ENOMEM))
2469                 goto xmitqueue_alloc_fail;
2470
2471         /* initialize write response PDU commands queue */
2472         conn->writequeue = kfifo_alloc(session->cmds_max * sizeof(void*),
2473                                         GFP_KERNEL, NULL);
2474         if (conn->writequeue == ERR_PTR(-ENOMEM))
2475                 goto writequeue_alloc_fail;
2476
2477         /* initialize general immediate & non-immediate PDU commands queue */
2478         conn->immqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*),
2479                                         GFP_KERNEL, NULL);
2480         if (conn->immqueue == ERR_PTR(-ENOMEM))
2481                 goto immqueue_alloc_fail;
2482
2483         conn->mgmtqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*),
2484                                         GFP_KERNEL, NULL);
2485         if (conn->mgmtqueue == ERR_PTR(-ENOMEM))
2486                 goto mgmtqueue_alloc_fail;
2487
2488         INIT_WORK(&conn->xmitwork, iscsi_xmitworker, conn);
2489
2490         /* allocate login_mtask used for the login/text sequences */
2491         spin_lock_bh(&session->lock);
2492         if (!__kfifo_get(session->mgmtpool.queue,
2493                          (void*)&conn->login_mtask,
2494                          sizeof(void*))) {
2495                 spin_unlock_bh(&session->lock);
2496                 goto login_mtask_alloc_fail;
2497         }
2498         spin_unlock_bh(&session->lock);
2499
2500         /* allocate initial PDU receive place holder */
2501         if (conn->data_size <= PAGE_SIZE)
2502                 conn->data = kmalloc(conn->data_size, GFP_KERNEL);
2503         else
2504                 conn->data = (void*)__get_free_pages(GFP_KERNEL,
2505                                         get_order(conn->data_size));
2506         if (!conn->data)
2507                 goto max_recv_dlenght_alloc_fail;
2508
2509         init_timer(&conn->tmabort_timer);
2510         mutex_init(&conn->xmitmutex);
2511         init_waitqueue_head(&conn->ehwait);
2512
2513         return cls_conn;
2514
2515 max_recv_dlenght_alloc_fail:
2516         spin_lock_bh(&session->lock);
2517         __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask,
2518                     sizeof(void*));
2519         spin_unlock_bh(&session->lock);
2520 login_mtask_alloc_fail:
2521         kfifo_free(conn->mgmtqueue);
2522 mgmtqueue_alloc_fail:
2523         kfifo_free(conn->immqueue);
2524 immqueue_alloc_fail:
2525         kfifo_free(conn->writequeue);
2526 writequeue_alloc_fail:
2527         kfifo_free(conn->xmitqueue);
2528 xmitqueue_alloc_fail:
2529         iscsi_destroy_conn(cls_conn);
2530         return NULL;
2531 }
2532
2533 static void
2534 iscsi_conn_destroy(struct iscsi_cls_conn *cls_conn)
2535 {
2536         struct iscsi_conn *conn = cls_conn->dd_data;
2537         struct iscsi_session *session = conn->session;
2538         unsigned long flags;
2539
2540         mutex_lock(&conn->xmitmutex);
2541         set_bit(SUSPEND_BIT, &conn->suspend_tx);
2542         if (conn->c_stage == ISCSI_CONN_INITIAL_STAGE && conn->sock) {
2543                 struct sock *sk = conn->sock->sk;
2544
2545                 /*
2546                  * conn_start() has never been called!
2547                  * need to cleanup the socket.
2548                  */
2549                 write_lock_bh(&sk->sk_callback_lock);
2550                 set_bit(SUSPEND_BIT, &conn->suspend_rx);
2551                 write_unlock_bh(&sk->sk_callback_lock);
2552
2553                 sock_hold(conn->sock->sk);
2554                 iscsi_conn_restore_callbacks(conn);
2555                 sock_put(conn->sock->sk);
2556                 sock_release(conn->sock);
2557                 conn->sock = NULL;
2558         }
2559
2560         spin_lock_bh(&session->lock);
2561         conn->c_stage = ISCSI_CONN_CLEANUP_WAIT;
2562         if (session->leadconn == conn) {
2563                 /*
2564                  * leading connection? then give up on recovery.
2565                  */
2566                 session->state = ISCSI_STATE_TERMINATE;
2567                 wake_up(&conn->ehwait);
2568         }
2569         spin_unlock_bh(&session->lock);
2570
2571         mutex_unlock(&conn->xmitmutex);
2572
2573         /*
2574          * Block until all in-progress commands for this connection
2575          * time out or fail.
2576          */
2577         for (;;) {
2578                 spin_lock_irqsave(session->host->host_lock, flags);
2579                 if (!session->host->host_busy) { /* OK for ERL == 0 */
2580                         spin_unlock_irqrestore(session->host->host_lock, flags);
2581                         break;
2582                 }
2583                 spin_unlock_irqrestore(session->host->host_lock, flags);
2584                 msleep_interruptible(500);
2585                 printk("conn_destroy(): host_busy %d host_failed %d\n",
2586                         session->host->host_busy, session->host->host_failed);
2587                 /*
2588                  * force eh_abort() to unblock
2589                  */
2590                 wake_up(&conn->ehwait);
2591         }
2592
2593         /* now free crypto */
2594         if (conn->hdrdgst_en || conn->datadgst_en) {
2595                 if (conn->tx_tfm)
2596                         crypto_free_tfm(conn->tx_tfm);
2597                 if (conn->rx_tfm)
2598                         crypto_free_tfm(conn->rx_tfm);
2599                 if (conn->data_tx_tfm)
2600                         crypto_free_tfm(conn->data_tx_tfm);
2601                 if (conn->data_rx_tfm)
2602                         crypto_free_tfm(conn->data_rx_tfm);
2603         }
2604
2605         /* free conn->data, size = MaxRecvDataSegmentLength */
2606         if (conn->data_size <= PAGE_SIZE)
2607                 kfree(conn->data);
2608         else
2609                 free_pages((unsigned long)conn->data,
2610                                         get_order(conn->data_size));
2611
2612         spin_lock_bh(&session->lock);
2613         __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask,
2614                     sizeof(void*));
2615         list_del(&conn->item);
2616         if (list_empty(&session->connections))
2617                 session->leadconn = NULL;
2618         if (session->leadconn && session->leadconn == conn)
2619                 session->leadconn = container_of(session->connections.next,
2620                         struct iscsi_conn, item);
2621
2622         if (session->leadconn == NULL)
2623                 /* none connections exits.. reset sequencing */
2624                 session->cmdsn = session->max_cmdsn = session->exp_cmdsn = 1;
2625         spin_unlock_bh(&session->lock);
2626
2627         kfifo_free(conn->xmitqueue);
2628         kfifo_free(conn->writequeue);
2629         kfifo_free(conn->immqueue);
2630         kfifo_free(conn->mgmtqueue);
2631
2632         iscsi_destroy_conn(cls_conn);
2633 }
2634
2635 static int
2636 iscsi_conn_bind(iscsi_sessionh_t sessionh, iscsi_connh_t connh,
2637                 uint32_t transport_fd, int is_leading)
2638 {
2639         struct iscsi_session *session = iscsi_ptr(sessionh);
2640         struct iscsi_conn *tmp = ERR_PTR(-EEXIST), *conn = iscsi_ptr(connh);
2641         struct sock *sk;
2642         struct socket *sock;
2643         int err;
2644
2645         /* lookup for existing socket */
2646         sock = sockfd_lookup(transport_fd, &err);
2647         if (!sock) {
2648                 printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err);
2649                 return -EEXIST;
2650         }
2651
2652         /* lookup for existing connection */
2653         spin_lock_bh(&session->lock);
2654         list_for_each_entry(tmp, &session->connections, item) {
2655                 if (tmp == conn) {
2656                         if (conn->c_stage != ISCSI_CONN_STOPPED ||
2657                             conn->stop_stage == STOP_CONN_TERM) {
2658                                 printk(KERN_ERR "iscsi_tcp: can't bind "
2659                                        "non-stopped connection (%d:%d)\n",
2660                                        conn->c_stage, conn->stop_stage);
2661                                 spin_unlock_bh(&session->lock);
2662                                 return -EIO;
2663                         }
2664                         break;
2665                 }
2666         }
2667         if (tmp != conn) {
2668                 /* bind new iSCSI connection to session */
2669                 conn->session = session;
2670
2671                 list_add(&conn->item, &session->connections);
2672         }
2673         spin_unlock_bh(&session->lock);
2674
2675         if (conn->stop_stage != STOP_CONN_SUSPEND) {
2676                 /* bind iSCSI connection and socket */
2677                 conn->sock = sock;
2678
2679                 /* setup Socket parameters */
2680                 sk = sock->sk;
2681                 sk->sk_reuse = 1;
2682                 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
2683                 sk->sk_allocation = GFP_ATOMIC;
2684
2685                 /* FIXME: disable Nagle's algorithm */
2686
2687                 /*
2688                  * Intercept TCP callbacks for sendfile like receive
2689                  * processing.
2690                  */
2691                 iscsi_conn_set_callbacks(conn);
2692
2693                 conn->sendpage = conn->sock->ops->sendpage;
2694
2695                 /*
2696                  * set receive state machine into initial state
2697                  */
2698                 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2699         }
2700
2701         if (is_leading)
2702                 session->leadconn = conn;
2703
2704         /*
2705          * Unblock xmitworker(), Login Phase will pass through.
2706          */
2707         clear_bit(SUSPEND_BIT, &conn->suspend_rx);
2708         clear_bit(SUSPEND_BIT, &conn->suspend_tx);
2709
2710         return 0;
2711 }
2712
2713 static int
2714 iscsi_conn_start(iscsi_connh_t connh)
2715 {
2716         struct iscsi_conn *conn = iscsi_ptr(connh);
2717         struct iscsi_session *session = conn->session;
2718         struct sock *sk;
2719
2720         /* FF phase warming up... */
2721
2722         if (session == NULL) {
2723                 printk(KERN_ERR "iscsi_tcp: can't start unbound connection\n");
2724                 return -EPERM;
2725         }
2726
2727         sk = conn->sock->sk;
2728
2729         write_lock_bh(&sk->sk_callback_lock);
2730         spin_lock_bh(&session->lock);
2731         conn->c_stage = ISCSI_CONN_STARTED;
2732         session->state = ISCSI_STATE_LOGGED_IN;
2733
2734         switch(conn->stop_stage) {
2735         case STOP_CONN_RECOVER:
2736                 /*
2737                  * unblock eh_abort() if it is blocked. re-try all
2738                  * commands after successful recovery
2739                  */
2740                 session->conn_cnt++;
2741                 conn->stop_stage = 0;
2742                 conn->tmabort_state = TMABORT_INITIAL;
2743                 session->age++;
2744                 wake_up(&conn->ehwait);
2745                 break;
2746         case STOP_CONN_TERM:
2747                 session->conn_cnt++;
2748                 conn->stop_stage = 0;
2749                 break;
2750         case STOP_CONN_SUSPEND:
2751                 conn->stop_stage = 0;
2752                 clear_bit(SUSPEND_BIT, &conn->suspend_rx);
2753                 clear_bit(SUSPEND_BIT, &conn->suspend_tx);
2754                 break;
2755         default:
2756                 break;
2757         }
2758         spin_unlock_bh(&session->lock);
2759         write_unlock_bh(&sk->sk_callback_lock);
2760
2761         return 0;
2762 }
2763
2764 static void
2765 iscsi_conn_stop(iscsi_connh_t connh, int flag)
2766 {
2767         struct iscsi_conn *conn = iscsi_ptr(connh);
2768         struct iscsi_session *session = conn->session;
2769         struct sock *sk;
2770         unsigned long flags;
2771
2772         BUG_ON(!conn->sock);
2773         sk = conn->sock->sk;
2774         write_lock_bh(&sk->sk_callback_lock);
2775         set_bit(SUSPEND_BIT, &conn->suspend_rx);
2776         write_unlock_bh(&sk->sk_callback_lock);
2777
2778         mutex_lock(&conn->xmitmutex);
2779
2780         spin_lock_irqsave(session->host->host_lock, flags);
2781         spin_lock(&session->lock);
2782         conn->stop_stage = flag;
2783         conn->c_stage = ISCSI_CONN_STOPPED;
2784         set_bit(SUSPEND_BIT, &conn->suspend_tx);
2785
2786         if (flag != STOP_CONN_SUSPEND)
2787                 session->conn_cnt--;
2788
2789         if (session->conn_cnt == 0 || session->leadconn == conn)
2790                 session->state = ISCSI_STATE_FAILED;
2791
2792         spin_unlock(&session->lock);
2793         spin_unlock_irqrestore(session->host->host_lock, flags);
2794
2795         if (flag == STOP_CONN_TERM || flag == STOP_CONN_RECOVER) {
2796                 struct iscsi_cmd_task *ctask;
2797                 struct iscsi_mgmt_task *mtask;
2798
2799                 /*
2800                  * Socket must go now.
2801                  */
2802                 sock_hold(conn->sock->sk);
2803                 iscsi_conn_restore_callbacks(conn);
2804                 sock_put(conn->sock->sk);
2805
2806                 /*
2807                  * flush xmit queues.
2808                  */
2809                 spin_lock_bh(&session->lock);
2810                 while (__kfifo_get(conn->writequeue, (void*)&ctask,
2811                             sizeof(void*)) ||
2812                         __kfifo_get(conn->xmitqueue, (void*)&ctask,
2813                             sizeof(void*))) {
2814                         struct iscsi_r2t_info *r2t;
2815
2816                         /*
2817                          * flush ctask's r2t queues
2818                          */
2819                         while (__kfifo_get(ctask->r2tqueue, (void*)&r2t,
2820                                 sizeof(void*)))
2821                                 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t,
2822                                             sizeof(void*));
2823
2824                         spin_unlock_bh(&session->lock);
2825                         local_bh_disable();
2826                         iscsi_ctask_cleanup(conn, ctask);
2827                         local_bh_enable();
2828                         spin_lock_bh(&session->lock);
2829                 }
2830                 conn->ctask = NULL;
2831                 while (__kfifo_get(conn->immqueue, (void*)&mtask,
2832                            sizeof(void*)) ||
2833                         __kfifo_get(conn->mgmtqueue, (void*)&mtask,
2834                            sizeof(void*))) {
2835                         __kfifo_put(session->mgmtpool.queue,
2836                                     (void*)&mtask, sizeof(void*));
2837                 }
2838                 conn->mtask = NULL;
2839                 spin_unlock_bh(&session->lock);
2840
2841                 /*
2842                  * release socket only after we stopped data_xmit()
2843                  * activity and flushed all outstandings
2844                  */
2845                 sock_release(conn->sock);
2846                 conn->sock = NULL;
2847
2848                 /*
2849                  * for connection level recovery we should not calculate
2850                  * header digest. conn->hdr_size used for optimization
2851                  * in hdr_extract() and will be re-negotiated at
2852                  * set_param() time.
2853                  */
2854                 if (flag == STOP_CONN_RECOVER) {
2855                         conn->hdr_size = sizeof(struct iscsi_hdr);
2856                         conn->hdrdgst_en = 0;
2857                         conn->datadgst_en = 0;
2858                 }
2859         }
2860         mutex_unlock(&conn->xmitmutex);
2861 }
2862
2863 static int
2864 iscsi_conn_send_generic(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
2865                         char *data, uint32_t data_size)
2866 {
2867         struct iscsi_session *session = conn->session;
2868         struct iscsi_nopout *nop = (struct iscsi_nopout *)hdr;
2869         struct iscsi_mgmt_task *mtask;
2870
2871         spin_lock_bh(&session->lock);
2872         if (session->state == ISCSI_STATE_TERMINATE) {
2873                 spin_unlock_bh(&session->lock);
2874                 return -EPERM;
2875         }
2876         if (hdr->opcode == (ISCSI_OP_LOGIN | ISCSI_OP_IMMEDIATE) ||
2877             hdr->opcode == (ISCSI_OP_TEXT | ISCSI_OP_IMMEDIATE))
2878                 /*
2879                  * Login and Text are sent serially, in
2880                  * request-followed-by-response sequence.
2881                  * Same mtask can be used. Same ITT must be used.
2882                  * Note that login_mtask is preallocated at conn_create().
2883                  */
2884                 mtask = conn->login_mtask;
2885         else {
2886                 BUG_ON(conn->c_stage == ISCSI_CONN_INITIAL_STAGE);
2887                 BUG_ON(conn->c_stage == ISCSI_CONN_STOPPED);
2888
2889                 if (!__kfifo_get(session->mgmtpool.queue,
2890                                  (void*)&mtask, sizeof(void*))) {
2891                         spin_unlock_bh(&session->lock);
2892                         return -ENOSPC;
2893                 }
2894         }
2895
2896         /*
2897          * pre-format CmdSN and ExpStatSN for outgoing PDU.
2898          */
2899         if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
2900                 hdr->itt = mtask->itt | (conn->id << CID_SHIFT) |
2901                            (session->age << AGE_SHIFT);
2902                 nop->cmdsn = cpu_to_be32(session->cmdsn);
2903                 if (conn->c_stage == ISCSI_CONN_STARTED &&
2904                     !(hdr->opcode & ISCSI_OP_IMMEDIATE))
2905                         session->cmdsn++;
2906         } else
2907                 /* do not advance CmdSN */
2908                 nop->cmdsn = cpu_to_be32(session->cmdsn);
2909
2910         nop->exp_statsn = cpu_to_be32(conn->exp_statsn);
2911
2912         memcpy(&mtask->hdr, hdr, sizeof(struct iscsi_hdr));
2913
2914         iscsi_buf_init_virt(&mtask->headbuf, (char*)&mtask->hdr,
2915                                     sizeof(struct iscsi_hdr));
2916
2917         spin_unlock_bh(&session->lock);
2918
2919         if (data_size) {
2920                 memcpy(mtask->data, data, data_size);
2921                 mtask->data_count = data_size;
2922         } else
2923                 mtask->data_count = 0;
2924
2925         mtask->xmstate = XMSTATE_IMM_HDR;
2926
2927         if (mtask->data_count) {
2928                 iscsi_buf_init_iov(&mtask->sendbuf, (char*)mtask->data,
2929                                     mtask->data_count);
2930         }
2931
2932         debug_scsi("mgmtpdu [op 0x%x hdr->itt 0x%x datalen %d]\n",
2933                    hdr->opcode, hdr->itt, data_size);
2934
2935         /*
2936          * since send_pdu() could be called at least from two contexts,
2937          * we need to serialize __kfifo_put, so we don't have to take
2938          * additional lock on fast data-path
2939          */
2940         if (hdr->opcode & ISCSI_OP_IMMEDIATE)
2941                 __kfifo_put(conn->immqueue, (void*)&mtask, sizeof(void*));
2942         else
2943                 __kfifo_put(conn->mgmtqueue, (void*)&mtask, sizeof(void*));
2944
2945         schedule_work(&conn->xmitwork);
2946
2947         return 0;
2948 }
2949
2950 static int
2951 iscsi_eh_host_reset(struct scsi_cmnd *sc)
2952 {
2953         struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
2954         struct iscsi_conn *conn = ctask->conn;
2955         struct iscsi_session *session = conn->session;
2956
2957         spin_lock_bh(&session->lock);
2958         if (session->state == ISCSI_STATE_TERMINATE) {
2959                 debug_scsi("failing host reset: session terminated "
2960                            "[CID %d age %d]", conn->id, session->age);
2961                 spin_unlock_bh(&session->lock);
2962                 return FAILED;
2963         }
2964         spin_unlock_bh(&session->lock);
2965
2966         debug_scsi("failing connection CID %d due to SCSI host reset "
2967                    "[itt 0x%x age %d]", conn->id, ctask->itt,
2968                    session->age);
2969         iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
2970
2971         return SUCCESS;
2972 }
2973
2974 static void
2975 iscsi_tmabort_timedout(unsigned long data)
2976 {
2977         struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)data;
2978         struct iscsi_conn *conn = ctask->conn;
2979         struct iscsi_session *session = conn->session;
2980
2981         spin_lock(&session->lock);
2982         if (conn->tmabort_state == TMABORT_INITIAL) {
2983                 __kfifo_put(session->mgmtpool.queue,
2984                                 (void*)&ctask->mtask, sizeof(void*));
2985                 conn->tmabort_state = TMABORT_TIMEDOUT;
2986                 debug_scsi("tmabort timedout [sc %lx itt 0x%x]\n",
2987                         (long)ctask->sc, ctask->itt);
2988                 /* unblock eh_abort() */
2989                 wake_up(&conn->ehwait);
2990         }
2991         spin_unlock(&session->lock);
2992 }
2993
2994 static int
2995 iscsi_eh_abort(struct scsi_cmnd *sc)
2996 {
2997         int rc;
2998         struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
2999         struct iscsi_conn *conn = ctask->conn;
3000         struct iscsi_session *session = conn->session;
3001
3002         conn->eh_abort_cnt++;
3003         debug_scsi("aborting [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3004
3005         /*
3006          * two cases for ERL=0 here:
3007          *
3008          * 1) connection-level failure;
3009          * 2) recovery due protocol error;
3010          */
3011         mutex_lock(&conn->xmitmutex);
3012         spin_lock_bh(&session->lock);
3013         if (session->state != ISCSI_STATE_LOGGED_IN) {
3014                 if (session->state == ISCSI_STATE_TERMINATE) {
3015                         spin_unlock_bh(&session->lock);
3016                         mutex_unlock(&conn->xmitmutex);
3017                         goto failed;
3018                 }
3019                 spin_unlock_bh(&session->lock);
3020         } else {
3021                 struct iscsi_tm *hdr = &conn->tmhdr;
3022
3023                 /*
3024                  * Still LOGGED_IN...
3025                  */
3026
3027                 if (!ctask->sc || sc->SCp.phase != session->age) {
3028                         /*
3029                          * 1) ctask completed before time out. But session
3030                          *    is still ok => Happy Retry.
3031                          * 2) session was re-open during time out of ctask.
3032                          */
3033                         spin_unlock_bh(&session->lock);
3034                         mutex_unlock(&conn->xmitmutex);
3035                         goto success;
3036                 }
3037                 conn->tmabort_state = TMABORT_INITIAL;
3038                 spin_unlock_bh(&session->lock);
3039
3040                 /*
3041                  * ctask timed out but session is OK
3042                  * ERL=0 requires task mgmt abort to be issued on each
3043                  * failed command. requests must be serialized.
3044                  */
3045                 memset(hdr, 0, sizeof(struct iscsi_tm));
3046                 hdr->opcode = ISCSI_OP_SCSI_TMFUNC | ISCSI_OP_IMMEDIATE;
3047                 hdr->flags = ISCSI_TM_FUNC_ABORT_TASK;
3048                 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
3049                 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
3050                 hdr->rtt = ctask->hdr.itt;
3051                 hdr->refcmdsn = ctask->hdr.cmdsn;
3052
3053                 rc = iscsi_conn_send_generic(conn, (struct iscsi_hdr *)hdr,
3054                                              NULL, 0);
3055                 if (rc) {
3056                         iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3057                         debug_scsi("abort sent failure [itt 0x%x]", ctask->itt);
3058                 } else {
3059                         struct iscsi_r2t_info *r2t;
3060
3061                         /*
3062                          * TMF abort vs. TMF response race logic
3063                          */
3064                         spin_lock_bh(&session->lock);
3065                         ctask->mtask = (struct iscsi_mgmt_task *)
3066                                 session->mgmt_cmds[(hdr->itt & ITT_MASK) -
3067                                                         ISCSI_MGMT_ITT_OFFSET];
3068                         /*
3069                          * have to flush r2tqueue to avoid r2t leaks
3070                          */
3071                         while (__kfifo_get(ctask->r2tqueue, (void*)&r2t,
3072                                 sizeof(void*))) {
3073                                 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t,
3074                                         sizeof(void*));
3075                         }
3076                         if (conn->tmabort_state == TMABORT_INITIAL) {
3077                                 conn->tmfcmd_pdus_cnt++;
3078                                 conn->tmabort_timer.expires = 3*HZ + jiffies;
3079                                 conn->tmabort_timer.function =
3080                                                 iscsi_tmabort_timedout;
3081                                 conn->tmabort_timer.data = (unsigned long)ctask;
3082                                 add_timer(&conn->tmabort_timer);
3083                                 debug_scsi("abort sent [itt 0x%x]", ctask->itt);
3084                         } else {
3085                                 if (!ctask->sc ||
3086                                     conn->tmabort_state == TMABORT_SUCCESS) {
3087                                         conn->tmabort_state = TMABORT_INITIAL;
3088                                         spin_unlock_bh(&session->lock);
3089                                         mutex_unlock(&conn->xmitmutex);
3090                                         goto success;
3091                                 }
3092                                 conn->tmabort_state = TMABORT_INITIAL;
3093                                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3094                         }
3095                         spin_unlock_bh(&session->lock);
3096                 }
3097         }
3098         mutex_unlock(&conn->xmitmutex);
3099
3100
3101         /*
3102          * block eh thread until:
3103          *
3104          * 1) abort response;
3105          * 2) abort timeout;
3106          * 3) session re-opened;
3107          * 4) session terminated;
3108          */
3109         for (;;) {
3110                 int p_state = session->state;
3111
3112                 rc = wait_event_interruptible(conn->ehwait,
3113                         (p_state == ISCSI_STATE_LOGGED_IN ?
3114                          (session->state == ISCSI_STATE_TERMINATE ||
3115                           conn->tmabort_state != TMABORT_INITIAL) :
3116                          (session->state == ISCSI_STATE_TERMINATE ||
3117                           session->state == ISCSI_STATE_LOGGED_IN)));
3118                 if (rc) {
3119                         /* shutdown.. */
3120                         session->state = ISCSI_STATE_TERMINATE;
3121                         goto failed;
3122                 }
3123
3124                 if (signal_pending(current))
3125                         flush_signals(current);
3126
3127                 if (session->state == ISCSI_STATE_TERMINATE)
3128                         goto failed;
3129
3130                 spin_lock_bh(&session->lock);
3131                 if (sc->SCp.phase == session->age &&
3132                    (conn->tmabort_state == TMABORT_TIMEDOUT ||
3133                     conn->tmabort_state == TMABORT_FAILED)) {
3134                         conn->tmabort_state = TMABORT_INITIAL;
3135                         if (!ctask->sc) {
3136                                 /*
3137                                  * ctask completed before tmf abort response or
3138                                  * time out.
3139                                  * But session is still ok => Happy Retry.
3140                                  */
3141                                 spin_unlock_bh(&session->lock);
3142                                 break;
3143                         }
3144                         spin_unlock_bh(&session->lock);
3145                         iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3146                         continue;
3147                 }
3148                 spin_unlock_bh(&session->lock);
3149                 break;
3150         }
3151
3152 success:
3153         debug_scsi("abort success [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3154         rc = SUCCESS;
3155         goto exit;
3156
3157 failed:
3158         debug_scsi("abort failed [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3159         rc = FAILED;
3160
3161 exit:
3162         del_timer_sync(&conn->tmabort_timer);
3163
3164         mutex_lock(&conn->xmitmutex);
3165         if (conn->sock) {
3166                 struct sock *sk = conn->sock->sk;
3167
3168                 write_lock_bh(&sk->sk_callback_lock);
3169                 iscsi_ctask_cleanup(conn, ctask);
3170                 write_unlock_bh(&sk->sk_callback_lock);
3171         }
3172         mutex_unlock(&conn->xmitmutex);
3173         return rc;
3174 }
3175
3176 static int
3177 iscsi_r2tpool_alloc(struct iscsi_session *session)
3178 {
3179         int i;
3180         int cmd_i;
3181
3182         /*
3183          * initialize per-task: R2T pool and xmit queue
3184          */
3185         for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
3186                 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
3187
3188                 /*
3189                  * pre-allocated x4 as much r2ts to handle race when
3190                  * target acks DataOut faster than we data_xmit() queues
3191                  * could replenish r2tqueue.
3192                  */
3193
3194                 /* R2T pool */
3195                 if (iscsi_pool_init(&ctask->r2tpool, session->max_r2t * 4,
3196                         (void***)&ctask->r2ts, sizeof(struct iscsi_r2t_info))) {
3197                         goto r2t_alloc_fail;
3198                 }
3199
3200                 /* R2T xmit queue */
3201                 ctask->r2tqueue = kfifo_alloc(
3202                       session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
3203                 if (ctask->r2tqueue == ERR_PTR(-ENOMEM)) {
3204                         iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts);
3205                         goto r2t_alloc_fail;
3206                 }
3207
3208                 /*
3209                  * number of
3210                  * Data-Out PDU's within R2T-sequence can be quite big;
3211                  * using mempool
3212                  */
3213                 ctask->datapool = mempool_create(ISCSI_DTASK_DEFAULT_MAX,
3214                          mempool_alloc_slab, mempool_free_slab, taskcache);
3215                 if (ctask->datapool == NULL) {
3216                         kfifo_free(ctask->r2tqueue);
3217                         iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts);
3218                         goto r2t_alloc_fail;
3219                 }
3220                 INIT_LIST_HEAD(&ctask->dataqueue);
3221         }
3222
3223         return 0;
3224
3225 r2t_alloc_fail:
3226         for (i = 0; i < cmd_i; i++) {
3227                 mempool_destroy(session->cmds[i]->datapool);
3228                 kfifo_free(session->cmds[i]->r2tqueue);
3229                 iscsi_pool_free(&session->cmds[i]->r2tpool,
3230                                 (void**)session->cmds[i]->r2ts);
3231         }
3232         return -ENOMEM;
3233 }
3234
3235 static void
3236 iscsi_r2tpool_free(struct iscsi_session *session)
3237 {
3238         int i;
3239
3240         for (i = 0; i < session->cmds_max; i++) {
3241                 mempool_destroy(session->cmds[i]->datapool);
3242                 kfifo_free(session->cmds[i]->r2tqueue);
3243                 iscsi_pool_free(&session->cmds[i]->r2tpool,
3244                                 (void**)session->cmds[i]->r2ts);
3245         }
3246 }
3247
3248 static struct scsi_host_template iscsi_sht = {
3249         .name                   = "iSCSI Initiator over TCP/IP, v."
3250                                   ISCSI_VERSION_STR,
3251         .queuecommand           = iscsi_queuecommand,
3252         .change_queue_depth     = iscsi_change_queue_depth,
3253         .can_queue              = ISCSI_XMIT_CMDS_MAX - 1,
3254         .sg_tablesize           = ISCSI_SG_TABLESIZE,
3255         .cmd_per_lun            = ISCSI_DEF_CMD_PER_LUN,
3256         .eh_abort_handler       = iscsi_eh_abort,
3257         .eh_host_reset_handler  = iscsi_eh_host_reset,
3258         .use_clustering         = DISABLE_CLUSTERING,
3259         .proc_name              = "iscsi_tcp",
3260         .this_id                = -1,
3261 };
3262
3263 static struct iscsi_transport iscsi_tcp_transport;
3264
3265 static struct Scsi_Host *
3266 iscsi_session_create(struct scsi_transport_template *scsit,
3267                      uint32_t initial_cmdsn)
3268 {
3269         struct Scsi_Host *shost;
3270         struct iscsi_session *session;
3271         int cmd_i;
3272
3273         shost = iscsi_transport_create_session(scsit, &iscsi_tcp_transport);
3274         if (!shost)
3275                 return NULL;
3276
3277         session = iscsi_hostdata(shost->hostdata);
3278         memset(session, 0, sizeof(struct iscsi_session));
3279         session->host = shost;
3280         session->state = ISCSI_STATE_LOGGED_IN;
3281         session->mgmtpool_max = ISCSI_MGMT_CMDS_MAX;
3282         session->cmds_max = ISCSI_XMIT_CMDS_MAX;
3283         session->cmdsn = initial_cmdsn;
3284         session->exp_cmdsn = initial_cmdsn + 1;
3285         session->max_cmdsn = initial_cmdsn + 1;
3286         session->max_r2t = 1;
3287
3288         /* initialize SCSI PDU commands pool */
3289         if (iscsi_pool_init(&session->cmdpool, session->cmds_max,
3290                 (void***)&session->cmds, sizeof(struct iscsi_cmd_task)))
3291                 goto cmdpool_alloc_fail;
3292
3293         /* pre-format cmds pool with ITT */
3294         for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++)
3295                 session->cmds[cmd_i]->itt = cmd_i;
3296
3297         spin_lock_init(&session->lock);
3298         INIT_LIST_HEAD(&session->connections);
3299
3300         /* initialize immediate command pool */
3301         if (iscsi_pool_init(&session->mgmtpool, session->mgmtpool_max,
3302                 (void***)&session->mgmt_cmds, sizeof(struct iscsi_mgmt_task)))
3303                 goto mgmtpool_alloc_fail;
3304
3305
3306         /* pre-format immediate cmds pool with ITT */
3307         for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) {
3308                 session->mgmt_cmds[cmd_i]->itt = ISCSI_MGMT_ITT_OFFSET + cmd_i;
3309                 session->mgmt_cmds[cmd_i]->data = kmalloc(
3310                         DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH, GFP_KERNEL);
3311                 if (!session->mgmt_cmds[cmd_i]->data) {
3312                         int j;
3313
3314                         for (j = 0; j < cmd_i; j++)
3315                                 kfree(session->mgmt_cmds[j]->data);
3316                         goto immdata_alloc_fail;
3317                 }
3318         }
3319
3320         if (iscsi_r2tpool_alloc(session))
3321                 goto r2tpool_alloc_fail;
3322
3323         return shost;
3324
3325 r2tpool_alloc_fail:
3326         for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++)
3327                 kfree(session->mgmt_cmds[cmd_i]->data);
3328         iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds);
3329 immdata_alloc_fail:
3330 mgmtpool_alloc_fail:
3331         iscsi_pool_free(&session->cmdpool, (void**)session->cmds);
3332 cmdpool_alloc_fail:
3333         return NULL;
3334 }
3335
3336 static void
3337 iscsi_session_destroy(struct Scsi_Host *shost)
3338 {
3339         struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
3340         int cmd_i;
3341         struct iscsi_data_task *dtask, *n;
3342
3343         for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
3344                 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
3345                 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
3346                         list_del(&dtask->item);
3347                         mempool_free(dtask, ctask->datapool);
3348                 }
3349         }
3350
3351         for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++)
3352                 kfree(session->mgmt_cmds[cmd_i]->data);
3353
3354         iscsi_r2tpool_free(session);
3355         iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds);
3356         iscsi_pool_free(&session->cmdpool, (void**)session->cmds);
3357
3358         iscsi_transport_destroy_session(shost);
3359 }
3360
3361 static int
3362 iscsi_conn_set_param(iscsi_connh_t connh, enum iscsi_param param,
3363                      uint32_t value)
3364 {
3365         struct iscsi_conn *conn = iscsi_ptr(connh);
3366         struct iscsi_session *session = conn->session;
3367
3368         spin_lock_bh(&session->lock);
3369         if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
3370             conn->stop_stage != STOP_CONN_RECOVER) {
3371                 printk(KERN_ERR "iscsi_tcp: can not change parameter [%d]\n",
3372                        param);
3373                 spin_unlock_bh(&session->lock);
3374                 return 0;
3375         }
3376         spin_unlock_bh(&session->lock);
3377
3378         switch(param) {
3379         case ISCSI_PARAM_MAX_RECV_DLENGTH: {
3380                 char *saveptr = conn->data;
3381                 gfp_t flags = GFP_KERNEL;
3382
3383                 if (conn->data_size >= value) {
3384                         conn->max_recv_dlength = value;
3385                         break;
3386                 }
3387
3388                 spin_lock_bh(&session->lock);
3389                 if (conn->stop_stage == STOP_CONN_RECOVER)
3390                         flags = GFP_ATOMIC;
3391                 spin_unlock_bh(&session->lock);
3392
3393                 if (value <= PAGE_SIZE)
3394                         conn->data = kmalloc(value, flags);
3395                 else
3396                         conn->data = (void*)__get_free_pages(flags,
3397                                                              get_order(value));
3398                 if (conn->data == NULL) {
3399                         conn->data = saveptr;
3400                         return -ENOMEM;
3401                 }
3402                 if (conn->data_size <= PAGE_SIZE)
3403                         kfree(saveptr);
3404                 else
3405                         free_pages((unsigned long)saveptr,
3406                                    get_order(conn->data_size));
3407                 conn->max_recv_dlength = value;
3408                 conn->data_size = value;
3409                 }
3410                 break;
3411         case ISCSI_PARAM_MAX_XMIT_DLENGTH:
3412                 conn->max_xmit_dlength =  value;
3413                 break;
3414         case ISCSI_PARAM_HDRDGST_EN:
3415                 conn->hdrdgst_en = value;
3416                 conn->hdr_size = sizeof(struct iscsi_hdr);
3417                 if (conn->hdrdgst_en) {
3418                         conn->hdr_size += sizeof(__u32);
3419                         if (!conn->tx_tfm)
3420                                 conn->tx_tfm = crypto_alloc_tfm("crc32c", 0);
3421                         if (!conn->tx_tfm)
3422                                 return -ENOMEM;
3423                         if (!conn->rx_tfm)
3424                                 conn->rx_tfm = crypto_alloc_tfm("crc32c", 0);
3425                         if (!conn->rx_tfm) {
3426                                 crypto_free_tfm(conn->tx_tfm);
3427                                 return -ENOMEM;
3428                         }
3429                 } else {
3430                         if (conn->tx_tfm)
3431                                 crypto_free_tfm(conn->tx_tfm);
3432                         if (conn->rx_tfm)
3433                                 crypto_free_tfm(conn->rx_tfm);
3434                 }
3435                 break;
3436         case ISCSI_PARAM_DATADGST_EN:
3437                 conn->datadgst_en = value;
3438                 if (conn->datadgst_en) {
3439                         if (!conn->data_tx_tfm)
3440                                 conn->data_tx_tfm =
3441                                     crypto_alloc_tfm("crc32c", 0);
3442                         if (!conn->data_tx_tfm)
3443                                 return -ENOMEM;
3444                         if (!conn->data_rx_tfm)
3445                                 conn->data_rx_tfm =
3446                                     crypto_alloc_tfm("crc32c", 0);
3447                         if (!conn->data_rx_tfm) {
3448                                 crypto_free_tfm(conn->data_tx_tfm);
3449                                 return -ENOMEM;
3450                         }
3451                 } else {
3452                         if (conn->data_tx_tfm)
3453                                 crypto_free_tfm(conn->data_tx_tfm);
3454                         if (conn->data_rx_tfm)
3455                                 crypto_free_tfm(conn->data_rx_tfm);
3456                 }
3457                 conn->sendpage = conn->datadgst_en ?
3458                         sock_no_sendpage : conn->sock->ops->sendpage;
3459                 break;
3460         case ISCSI_PARAM_INITIAL_R2T_EN:
3461                 session->initial_r2t_en = value;
3462                 break;
3463         case ISCSI_PARAM_MAX_R2T:
3464                 if (session->max_r2t == roundup_pow_of_two(value))
3465                         break;
3466                 iscsi_r2tpool_free(session);
3467                 session->max_r2t = value;
3468                 if (session->max_r2t & (session->max_r2t - 1))
3469                         session->max_r2t = roundup_pow_of_two(session->max_r2t);
3470                 if (iscsi_r2tpool_alloc(session))
3471                         return -ENOMEM;
3472                 break;
3473         case ISCSI_PARAM_IMM_DATA_EN:
3474                 session->imm_data_en = value;
3475                 break;
3476         case ISCSI_PARAM_FIRST_BURST:
3477                 session->first_burst = value;
3478                 break;
3479         case ISCSI_PARAM_MAX_BURST:
3480                 session->max_burst = value;
3481                 break;
3482         case ISCSI_PARAM_PDU_INORDER_EN:
3483                 session->pdu_inorder_en = value;
3484                 break;
3485         case ISCSI_PARAM_DATASEQ_INORDER_EN:
3486                 session->dataseq_inorder_en = value;
3487                 break;
3488         case ISCSI_PARAM_ERL:
3489                 session->erl = value;
3490                 break;
3491         case ISCSI_PARAM_IFMARKER_EN:
3492                 BUG_ON(value);
3493                 session->ifmarker_en = value;
3494                 break;
3495         case ISCSI_PARAM_OFMARKER_EN:
3496                 BUG_ON(value);
3497                 session->ofmarker_en = value;
3498                 break;
3499         default:
3500                 break;
3501         }
3502
3503         return 0;
3504 }
3505
3506 static int
3507 iscsi_session_get_param(struct Scsi_Host *shost,
3508                         enum iscsi_param param, uint32_t *value)
3509 {
3510         struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
3511
3512         switch(param) {
3513         case ISCSI_PARAM_INITIAL_R2T_EN:
3514                 *value = session->initial_r2t_en;
3515                 break;
3516         case ISCSI_PARAM_MAX_R2T:
3517                 *value = session->max_r2t;
3518                 break;
3519         case ISCSI_PARAM_IMM_DATA_EN:
3520                 *value = session->imm_data_en;
3521                 break;
3522         case ISCSI_PARAM_FIRST_BURST:
3523                 *value = session->first_burst;
3524                 break;
3525         case ISCSI_PARAM_MAX_BURST:
3526                 *value = session->max_burst;
3527                 break;
3528         case ISCSI_PARAM_PDU_INORDER_EN:
3529                 *value = session->pdu_inorder_en;
3530                 break;
3531         case ISCSI_PARAM_DATASEQ_INORDER_EN:
3532                 *value = session->dataseq_inorder_en;
3533                 break;
3534         case ISCSI_PARAM_ERL:
3535                 *value = session->erl;
3536                 break;
3537         case ISCSI_PARAM_IFMARKER_EN:
3538                 *value = session->ifmarker_en;
3539                 break;
3540         case ISCSI_PARAM_OFMARKER_EN:
3541                 *value = session->ofmarker_en;
3542                 break;
3543         default:
3544                 return ISCSI_ERR_PARAM_NOT_FOUND;
3545         }
3546
3547         return 0;
3548 }
3549
3550 static int
3551 iscsi_conn_get_param(void *data, enum iscsi_param param, uint32_t *value)
3552 {
3553         struct iscsi_conn *conn = data;
3554
3555         switch(param) {
3556         case ISCSI_PARAM_MAX_RECV_DLENGTH:
3557                 *value = conn->max_recv_dlength;
3558                 break;
3559         case ISCSI_PARAM_MAX_XMIT_DLENGTH:
3560                 *value = conn->max_xmit_dlength;
3561                 break;
3562         case ISCSI_PARAM_HDRDGST_EN:
3563                 *value = conn->hdrdgst_en;
3564                 break;
3565         case ISCSI_PARAM_DATADGST_EN:
3566                 *value = conn->datadgst_en;
3567                 break;
3568         default:
3569                 return ISCSI_ERR_PARAM_NOT_FOUND;
3570         }
3571
3572         return 0;
3573 }
3574
3575 static void
3576 iscsi_conn_get_stats(iscsi_connh_t connh, struct iscsi_stats *stats)
3577 {
3578         struct iscsi_conn *conn = iscsi_ptr(connh);
3579
3580         stats->txdata_octets = conn->txdata_octets;
3581         stats->rxdata_octets = conn->rxdata_octets;
3582         stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
3583         stats->dataout_pdus = conn->dataout_pdus_cnt;
3584         stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
3585         stats->datain_pdus = conn->datain_pdus_cnt;
3586         stats->r2t_pdus = conn->r2t_pdus_cnt;
3587         stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
3588         stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
3589         stats->custom_length = 3;
3590         strcpy(stats->custom[0].desc, "tx_sendpage_failures");
3591         stats->custom[0].value = conn->sendpage_failures_cnt;
3592         strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
3593         stats->custom[1].value = conn->discontiguous_hdr_cnt;
3594         strcpy(stats->custom[2].desc, "eh_abort_cnt");
3595         stats->custom[2].value = conn->eh_abort_cnt;
3596 }
3597
3598 static int
3599 iscsi_conn_send_pdu(iscsi_connh_t connh, struct iscsi_hdr *hdr, char *data,
3600                     uint32_t data_size)
3601 {
3602         struct iscsi_conn *conn = iscsi_ptr(connh);
3603         int rc;
3604
3605         mutex_lock(&conn->xmitmutex);
3606         rc = iscsi_conn_send_generic(conn, hdr, data, data_size);
3607         mutex_unlock(&conn->xmitmutex);
3608
3609         return rc;
3610 }
3611
3612 static struct iscsi_transport iscsi_tcp_transport = {
3613         .owner                  = THIS_MODULE,
3614         .name                   = "tcp",
3615         .caps                   = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
3616                                   | CAP_DATADGST,
3617         .host_template          = &iscsi_sht,
3618         .hostdata_size          = sizeof(struct iscsi_session),
3619         .conndata_size          = sizeof(struct iscsi_conn),
3620         .max_conn               = 1,
3621         .max_cmd_len            = ISCSI_TCP_MAX_CMD_LEN,
3622         .create_session         = iscsi_session_create,
3623         .destroy_session        = iscsi_session_destroy,
3624         .create_conn            = iscsi_conn_create,
3625         .bind_conn              = iscsi_conn_bind,
3626         .destroy_conn           = iscsi_conn_destroy,
3627         .set_param              = iscsi_conn_set_param,
3628         .get_conn_param         = iscsi_conn_get_param,
3629         .get_session_param      = iscsi_session_get_param,
3630         .start_conn             = iscsi_conn_start,
3631         .stop_conn              = iscsi_conn_stop,
3632         .send_pdu               = iscsi_conn_send_pdu,
3633         .get_stats              = iscsi_conn_get_stats,
3634 };
3635
3636 static int __init
3637 iscsi_tcp_init(void)
3638 {
3639         if (iscsi_max_lun < 1) {
3640                 printk(KERN_ERR "Invalid max_lun value of %u\n", iscsi_max_lun);
3641                 return -EINVAL;
3642         }
3643         iscsi_tcp_transport.max_lun = iscsi_max_lun;
3644
3645         taskcache = kmem_cache_create("iscsi_taskcache",
3646                         sizeof(struct iscsi_data_task), 0,
3647                         SLAB_HWCACHE_ALIGN | SLAB_NO_REAP, NULL, NULL);
3648         if (!taskcache)
3649                 return -ENOMEM;
3650
3651         if (!iscsi_register_transport(&iscsi_tcp_transport))
3652                 kmem_cache_destroy(taskcache);
3653
3654         return 0;
3655 }
3656
3657 static void __exit
3658 iscsi_tcp_exit(void)
3659 {
3660         iscsi_unregister_transport(&iscsi_tcp_transport);
3661         kmem_cache_destroy(taskcache);
3662 }
3663
3664 module_init(iscsi_tcp_init);
3665 module_exit(iscsi_tcp_exit);