nginx/src/os/unix/ngx_linux_sendfile_chain.c

   1
   2 /*
   3  * Copyright (C) Igor Sysoev
   4  */
   5
   6
   7 #include <ngx_config.h>
   8 #include <ngx_core.h>
   9 #include <ngx_event.h>
  10
  11
  12 /*
  13  * On Linux up to 2.4.21 sendfile() (syscall #187) works with 32-bit
  14  * offsets only, and the including <sys/sendfile.h> breaks the compiling,
  15  * if off_t is 64 bit wide.  So we use own sendfile() definition, where offset
  16  * parameter is int32_t, and use sendfile() for the file parts below 2G only,
  17  * see src/os/unix/ngx_linux_config.h
  18  *
  19  * Linux 2.4.21 has the new sendfile64() syscall #239.
  20  *
  21  * On Linux up to 2.6.16 sendfile() does not allow to pass the count parameter
  22  * more than 2G-1 bytes even on 64-bit platforms: it returns EINVAL,
  23  * so we limit it to 2G-1 bytes.
  24  */
  25
  26 #define NGX_SENDFILE_LIMIT  2147483647L
  27
  28
  29 #if (IOV_MAX > 64)
  30 #define NGX_HEADERS  64
  31 #else
  32 #define NGX_HEADERS  IOV_MAX
  33 #endif
  34
  35
  36 ngx_chain_t *
  37 ngx_linux_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
  38 {
  39     int            rc, tcp_nodelay;
  40     off_t          size, send, prev_send, aligned, sent, fprev;
  41     u_char        *prev;
  42     size_t         file_size;
  43     ngx_err_t      err;
  44     ngx_buf_t     *file;
  45     ngx_uint_t     eintr, complete;
  46     ngx_array_t    header;
  47     ngx_event_t   *wev;
  48     ngx_chain_t   *cl;
  49     struct iovec  *iov, headers[NGX_HEADERS];
  50 #if (NGX_HAVE_SENDFILE64)
  51     off_t          offset;
  52 #else
  53     int32_t        offset;
  54 #endif
  55
  56     wev = c->write;
  57
  58     if (!wev->ready) {
  59         return in;
  60     }
  61
  62
  63     /* the maximum limit size is 2G-1 - the page size */
  64
  65     if (limit == 0 || limit > (off_t) (NGX_SENDFILE_LIMIT - ngx_pagesize)) {
  66         limit = NGX_SENDFILE_LIMIT - ngx_pagesize;
  67     }
  68
  69
  70     send = 0;
  71
  72     header.elts = headers;
  73     header.size = sizeof(struct iovec);
  74     header.nalloc = NGX_HEADERS;
  75     header.pool = c->pool;
  76
  77     for ( ;; ) {
  78         file = NULL;
  79         file_size = 0;
  80         eintr = 0;
  81         complete = 0;
  82         prev_send = send;
  83
  84         header.nelts = 0;
  85
  86         prev = NULL;
  87         iov = NULL;
  88
  89         /* create the iovec and coalesce the neighbouring bufs */
  90
  91         for (cl = in;
  92              cl && header.nelts < IOV_MAX && send < limit;
  93              cl = cl->next)
  94         {
  95             if (ngx_buf_special(cl->buf)) {
  96                 continue;
  97             }
  98
  99 #if 1
 100             if (!ngx_buf_in_memory(cl->buf) && !cl->buf->in_file) {
 101                 ngx_log_error(NGX_LOG_ALERT, c->log, 0,
 102                               "zero size buf in sendfile "
 103                               "t:%d r:%d f:%d %p %p-%p %p %O-%O",
 104                               cl->buf->temporary,
 105                               cl->buf->recycled,
 106                               cl->buf->in_file,
 107                               cl->buf->start,
 108                               cl->buf->pos,
 109                               cl->buf->last,
 110                               cl->buf->file,
 111                               cl->buf->file_pos,
 112                               cl->buf->file_last);
 113
 114                 ngx_debug_point();
 115
 116                 return NGX_CHAIN_ERROR;
 117             }
 118 #endif
 119
 120             if (!ngx_buf_in_memory_only(cl->buf)) {
 121                 break;
 122             }
 123
 124             size = cl->buf->last - cl->buf->pos;
 125
 126             if (send + size > limit) {
 127                 size = limit - send;
 128             }
 129
 130             if (prev == cl->buf->pos) {
 131                 iov->iov_len += (size_t) size;
 132
 133             } else {
 134                 iov = ngx_array_push(&header);
 135                 if (iov == NULL) {
 136                     return NGX_CHAIN_ERROR;
 137                 }
 138
 139                 iov->iov_base = (void *) cl->buf->pos;
 140                 iov->iov_len = (size_t) size;
 141             }
 142
 143             prev = cl->buf->pos + (size_t) size;
 144             send += size;
 145         }
 146
 147         /* set TCP_CORK if there is a header before a file */
 148
 149         if (c->tcp_nopush == NGX_TCP_NOPUSH_UNSET
 150             && header.nelts != 0
 151             && cl
 152             && cl->buf->in_file)
 153         {
 154             /* the TCP_CORK and TCP_NODELAY are mutually exclusive */
 155
 156             if (c->tcp_nodelay == NGX_TCP_NODELAY_SET) {
 157
 158                 tcp_nodelay = 0;
 159
 160                 if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY,
 161                                (const void *) &tcp_nodelay, sizeof(int)) == -1)
 162                 {
 163                     err = ngx_errno;
 164
 165                     /*
 166                      * there is a tiny chance to be interrupted, however,
 167                      * we continue a processing with the TCP_NODELAY
 168                      * and without the TCP_CORK
 169                      */
 170
 171                     if (err != NGX_EINTR) {
 172                         wev->error = 1;
 173                         ngx_connection_error(c, err,
 174                                              "setsockopt(TCP_NODELAY) failed");
 175                         return NGX_CHAIN_ERROR;
 176                     }
 177
 178                 } else {
 179                     c->tcp_nodelay = NGX_TCP_NODELAY_UNSET;
 180
 181                     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
 182                                    "no tcp_nodelay");
 183                 }
 184             }
 185
 186             if (c->tcp_nodelay == NGX_TCP_NODELAY_UNSET) {
 187
 188                 if (ngx_tcp_nopush(c->fd) == NGX_ERROR) {
 189                     err = ngx_errno;
 190
 191                     /*
 192                      * there is a tiny chance to be interrupted, however,
 193                      * we continue a processing without the TCP_CORK
 194                      */
 195
 196                     if (err != NGX_EINTR) {
 197                         wev->error = 1;
 198                         ngx_connection_error(c, err,
 199                                              ngx_tcp_nopush_n " failed");
 200                         return NGX_CHAIN_ERROR;
 201                     }
 202
 203                 } else {
 204                     c->tcp_nopush = NGX_TCP_NOPUSH_SET;
 205
 206                     ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
 207                                    "tcp_nopush");
 208                 }
 209             }
 210         }
 211
 212         /* get the file buf */
 213
 214         if (header.nelts == 0 && cl && cl->buf->in_file && send < limit) {
 215             file = cl->buf;
 216
 217             /* coalesce the neighbouring file bufs */
 218
 219             do {
 220                 size = cl->buf->file_last - cl->buf->file_pos;
 221
 222                 if (send + size > limit) {
 223                     size = limit - send;
 224
 225                     aligned = (cl->buf->file_pos + size + ngx_pagesize - 1)
 226                                & ~((off_t) ngx_pagesize - 1);
 227
 228                     if (aligned <= cl->buf->file_last) {
 229                         size = aligned - cl->buf->file_pos;
 230                     }
 231                 }
 232
 233                 file_size += (size_t) size;
 234                 send += size;
 235                 fprev = cl->buf->file_pos + size;
 236                 cl = cl->next;
 237
 238             } while (cl
 239                      && cl->buf->in_file
 240                      && send < limit
 241                      && file->file->fd == cl->buf->file->fd
 242                      && fprev == cl->buf->file_pos);
 243         }
 244
 245         if (file) {
 246 #if 1
 247             if (file_size == 0) {
 248                 ngx_debug_point();
 249                 return NGX_CHAIN_ERROR;
 250             }
 251 #endif
 252 #if (NGX_HAVE_SENDFILE64)
 253             offset = file->file_pos;
 254 #else
 255             offset = (int32_t) file->file_pos;
 256 #endif
 257
 258             ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
 259                            "sendfile: @%O %uz", file->file_pos, file_size);
 260
 261             rc = sendfile(c->fd, file->file->fd, &offset, file_size);
 262
 263             if (rc == -1) {
 264                 err = ngx_errno;
 265
 266                 if (err == NGX_EAGAIN || err == NGX_EINTR) {
 267                     if (err == NGX_EINTR) {
 268                         eintr = 1;
 269                     }
 270
 271                     ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
 272                                    "sendfile() is not ready");
 273
 274                 } else {
 275                     wev->error = 1;
 276                     ngx_connection_error(c, err, "sendfile() failed");
 277                     return NGX_CHAIN_ERROR;
 278                 }
 279             }
 280
 281             sent = rc > 0 ? rc : 0;
 282
 283             ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
 284                            "sendfile: %d, @%O %O:%uz",
 285                            rc, file->file_pos, sent, file_size);
 286
 287         } else {
 288             rc = writev(c->fd, header.elts, header.nelts);
 289
 290             if (rc == -1) {
 291                 err = ngx_errno;
 292
 293                 if (err == NGX_EAGAIN || err == NGX_EINTR) {
 294                     if (err == NGX_EINTR) {
 295                         eintr = 1;
 296                     }
 297
 298                     ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
 299                                    "writev() not ready");
 300
 301                 } else {
 302                     wev->error = 1;
 303                     ngx_connection_error(c, err, "writev() failed");
 304                     return NGX_CHAIN_ERROR;
 305                 }
 306             }
 307
 308             sent = rc > 0 ? rc : 0;
 309
 310             ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "writev: %O", sent);
 311         }
 312
 313         if (send - prev_send == sent) {
 314             complete = 1;
 315         }
 316
 317         c->sent += sent;
 318
 319         for (cl = in; cl; cl = cl->next) {
 320
 321             if (ngx_buf_special(cl->buf)) {
 322                 continue;
 323             }
 324
 325             if (sent == 0) {
 326                 break;
 327             }
 328
 329             size = ngx_buf_size(cl->buf);
 330
 331             if (sent >= size) {
 332                 sent -= size;
 333
 334                 if (ngx_buf_in_memory(cl->buf)) {
 335                     cl->buf->pos = cl->buf->last;
 336                 }
 337
 338                 if (cl->buf->in_file) {
 339                     cl->buf->file_pos = cl->buf->file_last;
 340                 }
 341
 342                 continue;
 343             }
 344
 345             if (ngx_buf_in_memory(cl->buf)) {
 346                 cl->buf->pos += (size_t) sent;
 347             }
 348
 349             if (cl->buf->in_file) {
 350                 cl->buf->file_pos += sent;
 351             }
 352
 353             break;
 354         }
 355
 356         if (eintr) {
 357             continue;
 358         }
 359
 360         if (!complete) {
 361             wev->ready = 0;
 362             return cl;
 363         }
 364
 365         if (send >= limit || cl == NULL) {
 366             return cl;
 367         }
 368
 369         in = cl;
 370     }
 371 }