1/** 2 * the network chunk-API 3 * 4 * 5 */ 6 7#include "chunk.h" 8#include "base.h" 9#include "log.h" 10 11#include <sys/types.h> 12#include <sys/stat.h> 13#include "sys-mmap.h" 14 15#include <stdlib.h> 16#include <fcntl.h> 17#include <unistd.h> 18 19#include <stdio.h> 20#include <errno.h> 21#include <string.h> 22 23#define DBE 0 24 25chunkqueue *chunkqueue_init(void) { 26 chunkqueue *cq; 27 28 cq = calloc(1, sizeof(*cq)); 29 30 cq->first = NULL; 31 cq->last = NULL; 32 33 cq->unused = NULL; 34 35 return cq; 36} 37 38static chunk *chunk_init(void) { 39 chunk *c; 40 41 c = calloc(1, sizeof(*c)); 42 43 c->type = MEM_CHUNK; 44 c->mem = buffer_init(); 45 c->file.name = buffer_init(); 46 c->file.start = c->file.length = c->file.mmap.offset = 0; 47 c->file.fd = -1; 48 c->file.mmap.start = MAP_FAILED; 49 c->file.mmap.length = 0; 50 c->file.is_temp = 0; 51 c->offset = 0; 52 c->next = NULL; 53 54 return c; 55} 56 57static void chunk_reset(chunk *c) { 58 if (NULL == c) return; 59 60 c->type = MEM_CHUNK; 61 62 buffer_reset(c->mem); 63 64 if (c->file.is_temp && !buffer_string_is_empty(c->file.name)) { 65 unlink(c->file.name->ptr); 66 } 67 68 buffer_reset(c->file.name); 69 70 if(c->type == SMB_CHUNK){ 71 if (c->file.fd != -1) { 72 smbc_close(c->file.fd); 73 Cdbg(DBE,"close smb file-------------------------------->remote computer"); 74 c->file.fd = -1; 75 } 76 } 77 else{ 78 if (c->file.fd != -1) { 79 close(c->file.fd); 80 c->file.fd = -1; 81 } 82 if (MAP_FAILED != c->file.mmap.start) { 83 munmap(c->file.mmap.start, c->file.mmap.length); 84 c->file.mmap.start = MAP_FAILED; 85 } 86 } 87 88 c->file.start = c->file.length = c->file.mmap.offset = 0; 89 c->file.mmap.length = 0; 90 c->file.is_temp = 0; 91 c->offset = 0; 92 c->next = NULL; 93} 94 95static void chunk_free(chunk *c) { 96 if (NULL == c) return; 97 98 chunk_reset(c); 99 100 buffer_free(c->mem); 101 buffer_free(c->file.name); 102 103 free(c); 104} 105 106static off_t chunk_remaining_length(const chunk *c) { 107 off_t len = 0; 108 switch (c->type) { 109 case MEM_CHUNK: 110 len = buffer_string_length(c->mem); 111 break; 112 case FILE_CHUNK: 113 case SMB_CHUNK: 114 len = c->file.length; 115 break; 116 default: 117 force_assert(c->type == MEM_CHUNK || c->type == FILE_CHUNK || c->type == SMB_CHUNK); 118 break; 119 } 120 force_assert(c->offset <= len); 121 return len - c->offset; 122} 123 124void chunkqueue_free(chunkqueue *cq) { 125 chunk *c, *pc; 126 127 if (NULL == cq) return; 128 129 for (c = cq->first; c; ) { 130 pc = c; 131 c = c->next; 132 chunk_free(pc); 133 } 134 135 for (c = cq->unused; c; ) { 136 pc = c; 137 c = c->next; 138 chunk_free(pc); 139 } 140 141 free(cq); 142} 143 144static void chunkqueue_push_unused_chunk(chunkqueue *cq, chunk *c) { 145 force_assert(NULL != cq && NULL != c); 146 147 /* keep at max 4 chunks in the 'unused'-cache */ 148 if (cq->unused_chunks > 4) { 149 chunk_free(c); 150 } else { 151 chunk_reset(c); 152 c->next = cq->unused; 153 cq->unused = c; 154 cq->unused_chunks++; 155 } 156} 157 158static chunk *chunkqueue_get_unused_chunk(chunkqueue *cq) { 159 chunk *c; 160 161 force_assert(NULL != cq); 162 163 /* check if we have a unused chunk */ 164 if (0 == cq->unused) { 165 c = chunk_init(); 166 } else { 167 /* take the first element from the list (a stack) */ 168 c = cq->unused; 169 cq->unused = c->next; 170 c->next = NULL; 171 cq->unused_chunks--; 172 } 173 174 return c; 175} 176 177static void chunkqueue_prepend_chunk(chunkqueue *cq, chunk *c) { 178 c->next = cq->first; 179 cq->first = c; 180 181 if (NULL == cq->last) { 182 cq->last = c; 183 } 184 cq->bytes_in += chunk_remaining_length(c); 185} 186 187static void chunkqueue_append_chunk(chunkqueue *cq, chunk *c) { 188 c->next = NULL; 189 if (cq->last) { 190 cq->last->next = c; 191 } 192 cq->last = c; 193 194 if (NULL == cq->first) { 195 cq->first = c; 196 } 197 cq->bytes_in += chunk_remaining_length(c); 198} 199 200void chunkqueue_reset(chunkqueue *cq) { 201 chunk *cur = cq->first; 202 203 cq->first = cq->last = NULL; 204 205 while (NULL != cur) { 206 chunk *next = cur->next; 207 chunkqueue_push_unused_chunk(cq, cur); 208 cur = next; 209 } 210 211 cq->bytes_in = 0; 212 cq->bytes_out = 0; 213} 214 215//- Sungmin add 216void chunkqueue_append_smb_file(chunkqueue *cq, buffer *fn, off_t offset, off_t len) { 217 chunk *c; 218 219 if (0 == len) return; 220 221 c = chunkqueue_get_unused_chunk(cq); 222 223 c->type = SMB_CHUNK; 224 225 buffer_copy_buffer(c->file.name, fn); 226 c->file.start = offset; 227 c->file.length = len; 228 c->offset = 0; 229 230 chunkqueue_append_chunk(cq, c); 231} 232 233void chunkqueue_append_file(chunkqueue *cq, buffer *fn, off_t offset, off_t len) { 234 chunk *c; 235 236 if (0 == len) return; 237 238 c = chunkqueue_get_unused_chunk(cq); 239 240 c->type = FILE_CHUNK; 241 242 buffer_copy_buffer(c->file.name, fn); 243 c->file.start = offset; 244 c->file.length = len; 245 c->offset = 0; 246 247 chunkqueue_append_chunk(cq, c); 248} 249 250void chunkqueue_append_buffer(chunkqueue *cq, buffer *mem) { 251 chunk *c; 252 253 if (buffer_string_is_empty(mem)) return; 254 255 c = chunkqueue_get_unused_chunk(cq); 256 c->type = MEM_CHUNK; 257 force_assert(NULL != c->mem); 258 buffer_move(c->mem, mem); 259 260 chunkqueue_append_chunk(cq, c); 261} 262 263void chunkqueue_prepend_buffer(chunkqueue *cq, buffer *mem) { 264 chunk *c; 265 266 if (buffer_string_is_empty(mem)) return; 267 268 c = chunkqueue_get_unused_chunk(cq); 269 c->type = MEM_CHUNK; 270 force_assert(NULL != c->mem); 271 buffer_move(c->mem, mem); 272 273 chunkqueue_prepend_chunk(cq, c); 274} 275 276 277void chunkqueue_append_mem(chunkqueue *cq, const char * mem, size_t len) { 278 chunk *c; 279 280 if (0 == len) return; 281 282 c = chunkqueue_get_unused_chunk(cq); 283 c->type = MEM_CHUNK; 284 buffer_copy_string_len(c->mem, mem, len); 285 286 chunkqueue_append_chunk(cq, c); 287} 288 289void chunkqueue_get_memory(chunkqueue *cq, char **mem, size_t *len, size_t min_size, size_t alloc_size) { 290 static const size_t REALLOC_MAX_SIZE = 256; 291 chunk *c; 292 buffer *b; 293 char *dummy_mem; 294 size_t dummy_len; 295 296 force_assert(NULL != cq); 297 if (NULL == mem) mem = &dummy_mem; 298 if (NULL == len) len = &dummy_len; 299 300 /* default values: */ 301 if (0 == min_size) min_size = 1024; 302 if (0 == alloc_size) alloc_size = 4096; 303 if (alloc_size < min_size) alloc_size = min_size; 304 305 if (NULL != cq->last && MEM_CHUNK == cq->last->type) { 306 size_t have; 307 308 b = cq->last->mem; 309 have = buffer_string_space(b); 310 311 /* unused buffer: allocate space */ 312 if (buffer_string_is_empty(b)) { 313 buffer_string_prepare_copy(b, alloc_size); 314 have = buffer_string_space(b); 315 } 316 /* if buffer is really small just make it bigger */ 317 else if (have < min_size && b->size <= REALLOC_MAX_SIZE) { 318 size_t cur_len = buffer_string_length(b); 319 size_t new_size = cur_len + min_size, append; 320 if (new_size < alloc_size) new_size = alloc_size; 321 322 append = new_size - cur_len; 323 if (append >= min_size) { 324 buffer_string_prepare_append(b, append); 325 have = buffer_string_space(b); 326 } 327 } 328 329 /* return pointer into existing buffer if large enough */ 330 if (have >= min_size) { 331 *mem = b->ptr + buffer_string_length(b); 332 *len = have; 333 return; 334 } 335 } 336 337 /* allocate new chunk */ 338 c = chunkqueue_get_unused_chunk(cq); 339 c->type = MEM_CHUNK; 340 chunkqueue_append_chunk(cq, c); 341 342 b = c->mem; 343 buffer_string_prepare_append(b, alloc_size); 344 345 *mem = b->ptr + buffer_string_length(b); 346 *len = buffer_string_space(b); 347} 348 349void chunkqueue_use_memory(chunkqueue *cq, size_t len) { 350 buffer *b; 351 352 force_assert(NULL != cq); 353 force_assert(NULL != cq->last && MEM_CHUNK == cq->last->type); 354 b = cq->last->mem; 355 356 if (len > 0) { 357 buffer_commit(b, len); 358 cq->bytes_in += len; 359 } else if (buffer_string_is_empty(b)) { 360 /* unused buffer: can't remove chunk easily from 361 * end of list, so just reset the buffer 362 */ 363 buffer_reset(b); 364 } 365} 366 367void chunkqueue_set_tempdirs(chunkqueue *cq, array *tempdirs, unsigned int upload_temp_file_size) { 368 force_assert(NULL != cq); 369 cq->tempdirs = tempdirs; 370 cq->upload_temp_file_size = upload_temp_file_size; 371} 372 373void chunkqueue_steal(chunkqueue *dest, chunkqueue *src, off_t len) { 374 while (len > 0) { 375 chunk *c = src->first; 376 off_t clen = 0, use; 377 378 if (NULL == c) break; 379 380 clen = chunk_remaining_length(c); 381 if (0 == clen) { 382 /* drop empty chunk */ 383 src->first = c->next; 384 if (c == src->last) src->last = NULL; 385 chunkqueue_push_unused_chunk(src, c); 386 continue; 387 } 388 389 use = len >= clen ? clen : len; 390 len -= use; 391 392 if (use == clen) { 393 /* move complete chunk */ 394 src->first = c->next; 395 if (c == src->last) src->last = NULL; 396 397 chunkqueue_append_chunk(dest, c); 398 } else { 399 /* partial chunk with length "use" */ 400 401 switch (c->type) { 402 case MEM_CHUNK: 403 chunkqueue_append_mem(dest, c->mem->ptr + c->offset, use); 404 break; 405 case FILE_CHUNK: 406 /* tempfile flag is in "last" chunk after the split */ 407 chunkqueue_append_file(dest, c->file.name, c->file.start + c->offset, use); 408 break; 409 } 410 411 c->offset += use; 412 force_assert(0 == len); 413 } 414 415 src->bytes_out += use; 416 } 417} 418 419static chunk *chunkqueue_get_append_tempfile(chunkqueue *cq) { 420 chunk *c; 421 buffer *template = buffer_init_string("/var/tmp/lighttpd-upload-XXXXXX"); 422 int fd; 423 424 if (cq->tempdirs && cq->tempdirs->used) { 425 size_t i; 426 427 /* we have several tempdirs, only if all of them fail we jump out */ 428 429 for (i = 0; i < cq->tempdirs->used; i++) { 430 data_string *ds = (data_string *)cq->tempdirs->data[i]; 431 432 buffer_copy_buffer(template, ds->value); 433 buffer_append_slash(template); 434 buffer_append_string_len(template, CONST_STR_LEN("lighttpd-upload-XXXXXX")); 435 436 if (-1 != (fd = mkstemp(template->ptr))) break; 437 } 438 } else { 439 fd = mkstemp(template->ptr); 440 } 441 442 if (-1 == fd) { 443 buffer_free(template); 444 return NULL; 445 } 446 447 c = chunkqueue_get_unused_chunk(cq); 448 c->type = FILE_CHUNK; 449 c->file.fd = fd; 450 c->file.is_temp = 1; 451 buffer_copy_buffer(c->file.name, template); 452 c->file.length = 0; 453 454 chunkqueue_append_chunk(cq, c); 455 456 buffer_free(template); 457 458 return c; 459} 460 461/* default 1MB, upper limit 128MB */ 462#define DEFAULT_TEMPFILE_SIZE (1 * 1024 * 1024) 463#define MAX_TEMPFILE_SIZE (128 * 1024 * 1024) 464 465static int chunkqueue_append_to_tempfile(server *srv, chunkqueue *dest, const char *mem, size_t len) { 466 /* copy everything to max max_tempfile_size sized tempfiles */ 467 const off_t max_tempfile_size 468 = (0 == dest->upload_temp_file_size) ? DEFAULT_TEMPFILE_SIZE 469 : (dest->upload_temp_file_size > MAX_TEMPFILE_SIZE) ? MAX_TEMPFILE_SIZE 470 : dest->upload_temp_file_size; 471 chunk *dst_c = NULL; 472 ssize_t written; 473 474 /* 475 * if the last chunk is 476 * - smaller than max_tempfile_size 477 * - not read yet (offset == 0) 478 * -> append to it (so it might actually become larger than max_tempfile_size) 479 * otherwise 480 * -> create a new chunk 481 * 482 * */ 483 484 if (NULL != dest->last 485 && FILE_CHUNK == dest->last->type 486 && dest->last->file.is_temp 487 && -1 != dest->last->file.fd 488 && 0 == dest->last->offset) { 489 /* ok, take the last chunk for our job */ 490 dst_c = dest->last; 491 492 if (dest->last->file.length >= max_tempfile_size) { 493 /* the chunk is too large now, close it */ 494 if (-1 != dst_c->file.fd) { 495 close(dst_c->file.fd); 496 dst_c->file.fd = -1; 497 } 498 dst_c = chunkqueue_get_append_tempfile(dest); 499 } 500 } else { 501 dst_c = chunkqueue_get_append_tempfile(dest); 502 } 503 504 if (NULL == dst_c) { 505 /* we don't have file to write to, 506 * EACCES might be one reason. 507 * 508 * Instead of sending 500 we send 413 and say the request is too large 509 */ 510 511 log_error_write(srv, __FILE__, __LINE__, "ss", 512 "denying upload as opening temp-file for upload failed:", 513 strerror(errno)); 514 515 return -1; 516 } 517 518 if (0 > (written = write(dst_c->file.fd, mem, len)) || (size_t) written != len) { 519 /* write failed for some reason ... disk full ? */ 520 log_error_write(srv, __FILE__, __LINE__, "sbs", 521 "denying upload as writing to file failed:", 522 dst_c->file.name, strerror(errno)); 523 524 close(dst_c->file.fd); 525 dst_c->file.fd = -1; 526 527 return -1; 528 } 529 530 dst_c->file.length += len; 531 dest->bytes_in += len; 532 533 return 0; 534} 535 536int chunkqueue_steal_with_tempfiles(server *srv, chunkqueue *dest, chunkqueue *src, off_t len) { 537 while (len > 0) { 538 chunk *c = src->first; 539 off_t clen = 0, use; 540 541 if (NULL == c) break; 542 543 clen = chunk_remaining_length(c); 544 if (0 == clen) { 545 /* drop empty chunk */ 546 src->first = c->next; 547 if (c == src->last) src->last = NULL; 548 chunkqueue_push_unused_chunk(src, c); 549 continue; 550 } 551 552 use = (len >= clen) ? clen : len; 553 len -= use; 554 555 switch (c->type) { 556 case FILE_CHUNK: 557 if (use == clen) { 558 /* move complete chunk */ 559 src->first = c->next; 560 if (c == src->last) src->last = NULL; 561 chunkqueue_append_chunk(dest, c); 562 } else { 563 /* partial chunk with length "use" */ 564 /* tempfile flag is in "last" chunk after the split */ 565 chunkqueue_append_file(dest, c->file.name, c->file.start + c->offset, use); 566 567 c->offset += use; 568 force_assert(0 == len); 569 } 570 break; 571 572 case MEM_CHUNK: 573 /* store "use" bytes from memory chunk in tempfile */ 574 if (0 != chunkqueue_append_to_tempfile(srv, dest, c->mem->ptr + c->offset, use)) { 575 return -1; 576 } 577 578 if (use == clen) { 579 /* finished chunk */ 580 src->first = c->next; 581 if (c == src->last) src->last = NULL; 582 chunkqueue_push_unused_chunk(src, c); 583 } else { 584 /* partial chunk */ 585 c->offset += use; 586 force_assert(0 == len); 587 } 588 break; 589 } 590 591 src->bytes_out += use; 592 } 593 594 return 0; 595} 596 597off_t chunkqueue_length(chunkqueue *cq) { 598 off_t len = 0; 599 chunk *c; 600 601 for (c = cq->first; c; c = c->next) { 602 len += chunk_remaining_length(c); 603 } 604 605 return len; 606} 607 608int chunkqueue_is_empty(chunkqueue *cq) { 609 return NULL == cq->first; 610} 611 612void chunkqueue_mark_written(chunkqueue *cq, off_t len) { 613 off_t written = len; 614 chunk *c; 615 force_assert(len >= 0); 616 617 for (c = cq->first; NULL != c; c = cq->first) { 618 off_t c_len = chunk_remaining_length(c); 619 620 if (0 == written && 0 != c_len) break; /* no more finished chunks */ 621 622 if (written >= c_len) { /* chunk got finished */ 623 c->offset += c_len; 624 written -= c_len; 625 626 cq->first = c->next; 627 if (c == cq->last) cq->last = NULL; 628 629 chunkqueue_push_unused_chunk(cq, c); 630 } else { /* partial chunk */ 631 c->offset += written; 632 written = 0; 633 break; /* chunk not finished */ 634 } 635 } 636 637 force_assert(0 == written); 638 cq->bytes_out += len; 639} 640 641void chunkqueue_remove_finished_chunks(chunkqueue *cq) { 642 chunk *c; 643 644 for (c = cq->first; c; c = cq->first) { 645 if (0 != chunk_remaining_length(c)) break; /* not finished yet */ 646 647 cq->first = c->next; 648 if (c == cq->last) cq->last = NULL; 649 650 chunkqueue_push_unused_chunk(cq, c); 651 } 652} 653