1/* File Transfer Protocol support. 2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 3 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software Foundation, 4 Inc. 5 6This file is part of GNU Wget. 7 8GNU Wget is free software; you can redistribute it and/or modify 9it under the terms of the GNU General Public License as published by 10the Free Software Foundation; either version 3 of the License, or 11(at your option) any later version. 12 13GNU Wget is distributed in the hope that it will be useful, 14but WITHOUT ANY WARRANTY; without even the implied warranty of 15MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16GNU General Public License for more details. 17 18You should have received a copy of the GNU General Public License 19along with Wget. If not, see <http://www.gnu.org/licenses/>. 20 21Additional permission under GNU GPL version 3 section 7 22 23If you modify this program, or any covered work, by linking or 24combining it with the OpenSSL project's OpenSSL library (or a 25modified version of that library), containing parts covered by the 26terms of the OpenSSL or SSLeay licenses, the Free Software Foundation 27grants you additional permission to convey the resulting work. 28Corresponding Source for a non-source form of such a combination 29shall include the source code for the parts of OpenSSL used as well 30as that of the covered work. */ 31 32#include "wget.h" 33 34#include <stdio.h> 35#include <stdlib.h> 36#include <string.h> 37#include <strings.h> 38#include <unistd.h> 39#include <assert.h> 40#include <errno.h> 41#include <time.h> 42 43#include "utils.h" 44#include "url.h" 45#include "retr.h" 46#include "ftp.h" 47#include "connect.h" 48#include "host.h" 49#include "netrc.h" 50#include "convert.h" /* for downloaded_file */ 51#include "recur.h" /* for INFINITE_RECURSION */ 52#include "warc.h" 53 54#ifdef __VMS 55# include "vms.h" 56#endif /* def __VMS */ 57 58 59/* File where the "ls -al" listing will be saved. */ 60#ifdef MSDOS 61#define LIST_FILENAME "_listing" 62#else 63#define LIST_FILENAME ".listing" 64#endif 65 66typedef struct 67{ 68 int st; /* connection status */ 69 int cmd; /* command code */ 70 int csock; /* control connection socket */ 71 double dltime; /* time of the download in msecs */ 72 enum stype rs; /* remote system reported by ftp server */ 73 enum ustype rsu; /* when rs is ST_UNIX, here there are more details */ 74 char *id; /* initial directory */ 75 char *target; /* target file name */ 76 struct url *proxy; /* FTWK-style proxy */ 77} ccon; 78 79extern int numurls; 80 81/* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in 82 the string S, and return the number converted to wgint, if found, 0 83 otherwise. */ 84static wgint 85ftp_expected_bytes (const char *s) 86{ 87 wgint res; 88 89 while (1) 90 { 91 while (*s && *s != '(') 92 ++s; 93 if (!*s) 94 return 0; 95 ++s; /* skip the '(' */ 96 res = str_to_wgint (s, (char **) &s, 10); 97 if (!*s) 98 return 0; 99 while (*s && c_isspace (*s)) 100 ++s; 101 if (!*s) 102 return 0; 103 if (c_tolower (*s) != 'b') 104 continue; 105 if (strncasecmp (s, "byte", 4)) 106 continue; 107 else 108 break; 109 } 110 return res; 111} 112 113#ifdef ENABLE_IPV6 114/* 115 * This function sets up a passive data connection with the FTP server. 116 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv. 117 */ 118static uerr_t 119ftp_do_pasv (int csock, ip_address *addr, int *port) 120{ 121 uerr_t err; 122 123 /* We need to determine the address family and need to call 124 getpeername, so while we're at it, store the address to ADDR. 125 ftp_pasv and ftp_lpsv can simply override it. */ 126 if (!socket_ip_address (csock, addr, ENDPOINT_PEER)) 127 abort (); 128 129 /* If our control connection is over IPv6, then we first try EPSV and then 130 * LPSV if the former is not supported. If the control connection is over 131 * IPv4, we simply issue the good old PASV request. */ 132 switch (addr->family) 133 { 134 case AF_INET: 135 if (!opt.server_response) 136 logputs (LOG_VERBOSE, "==> PASV ... "); 137 err = ftp_pasv (csock, addr, port); 138 break; 139 case AF_INET6: 140 if (!opt.server_response) 141 logputs (LOG_VERBOSE, "==> EPSV ... "); 142 err = ftp_epsv (csock, addr, port); 143 144 /* If EPSV is not supported try LPSV */ 145 if (err == FTPNOPASV) 146 { 147 if (!opt.server_response) 148 logputs (LOG_VERBOSE, "==> LPSV ... "); 149 err = ftp_lpsv (csock, addr, port); 150 } 151 break; 152 default: 153 abort (); 154 } 155 156 return err; 157} 158 159/* 160 * This function sets up an active data connection with the FTP server. 161 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port. 162 */ 163static uerr_t 164ftp_do_port (int csock, int *local_sock) 165{ 166 uerr_t err; 167 ip_address cip; 168 169 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER)) 170 abort (); 171 172 /* If our control connection is over IPv6, then we first try EPRT and then 173 * LPRT if the former is not supported. If the control connection is over 174 * IPv4, we simply issue the good old PORT request. */ 175 switch (cip.family) 176 { 177 case AF_INET: 178 if (!opt.server_response) 179 logputs (LOG_VERBOSE, "==> PORT ... "); 180 err = ftp_port (csock, local_sock); 181 break; 182 case AF_INET6: 183 if (!opt.server_response) 184 logputs (LOG_VERBOSE, "==> EPRT ... "); 185 err = ftp_eprt (csock, local_sock); 186 187 /* If EPRT is not supported try LPRT */ 188 if (err == FTPPORTERR) 189 { 190 if (!opt.server_response) 191 logputs (LOG_VERBOSE, "==> LPRT ... "); 192 err = ftp_lprt (csock, local_sock); 193 } 194 break; 195 default: 196 abort (); 197 } 198 return err; 199} 200#else 201 202static uerr_t 203ftp_do_pasv (int csock, ip_address *addr, int *port) 204{ 205 if (!opt.server_response) 206 logputs (LOG_VERBOSE, "==> PASV ... "); 207 return ftp_pasv (csock, addr, port); 208} 209 210static uerr_t 211ftp_do_port (int csock, int *local_sock) 212{ 213 if (!opt.server_response) 214 logputs (LOG_VERBOSE, "==> PORT ... "); 215 return ftp_port (csock, local_sock); 216} 217#endif 218 219static void 220print_length (wgint size, wgint start, bool authoritative) 221{ 222 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size)); 223 if (size >= 1024) 224 logprintf (LOG_VERBOSE, " (%s)", human_readable (size, 10, 1)); 225 if (start > 0) 226 { 227 if (size - start >= 1024) 228 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"), 229 number_to_static_string (size - start), 230 human_readable (size - start, 10, 1)); 231 else 232 logprintf (LOG_VERBOSE, _(", %s remaining"), 233 number_to_static_string (size - start)); 234 } 235 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n"); 236} 237 238static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **); 239 240/* Retrieves a file with denoted parameters through opening an FTP 241 connection to the server. It always closes the data connection, 242 and closes the control connection in case of error. If warc_tmp 243 is non-NULL, the downloaded data will be written there as well. */ 244static uerr_t 245getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread, 246 wgint restval, ccon *con, int count, wgint *last_expected_bytes, 247 FILE *warc_tmp) 248{ 249 int csock, dtsock, local_sock, res; 250 uerr_t err = RETROK; /* appease the compiler */ 251 FILE *fp; 252 char *respline, *tms; 253 const char *user, *passwd, *tmrate; 254 int cmd = con->cmd; 255 bool pasv_mode_open = false; 256 wgint expected_bytes = 0; 257 bool got_expected_bytes = false; 258 bool rest_failed = false; 259 int flags; 260 wgint rd_size, previous_rd_size = 0; 261 char type_char; 262 bool try_again; 263 bool list_a_used = false; 264 265 assert (con != NULL); 266 assert (con->target != NULL); 267 268 /* Debug-check of the sanity of the request by making sure that LIST 269 and RETR are never both requested (since we can handle only one 270 at a time. */ 271 assert (!((cmd & DO_LIST) && (cmd & DO_RETR))); 272 /* Make sure that at least *something* is requested. */ 273 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0); 274 275 *qtyread = restval; 276 277 user = u->user; 278 passwd = u->passwd; 279 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1); 280 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user); 281 if (!user) user = "anonymous"; 282 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd); 283 if (!passwd) passwd = "-wget@"; 284 285 dtsock = -1; 286 local_sock = -1; 287 con->dltime = 0; 288 289 if (!(cmd & DO_LOGIN)) 290 csock = con->csock; 291 else /* cmd & DO_LOGIN */ 292 { 293 char *host = con->proxy ? con->proxy->host : u->host; 294 int port = con->proxy ? con->proxy->port : u->port; 295 296 /* Login to the server: */ 297 298 /* First: Establish the control connection. */ 299 300 csock = connect_to_host (host, port); 301 if (csock == E_HOST) 302 return HOSTERR; 303 else if (csock < 0) 304 return (retryable_socket_connect_error (errno) 305 ? CONERROR : CONIMPOSSIBLE); 306 307 if (cmd & LEAVE_PENDING) 308 con->csock = csock; 309 else 310 con->csock = -1; 311 312 /* Second: Login with proper USER/PASS sequence. */ 313 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), 314 quotearg_style (escape_quoting_style, user)); 315 if (opt.server_response) 316 logputs (LOG_ALWAYS, "\n"); 317 if (con->proxy) 318 { 319 /* If proxy is in use, log in as username@target-site. */ 320 char *logname = concat_strings (user, "@", u->host, (char *) 0); 321 err = ftp_login (csock, logname, passwd); 322 xfree (logname); 323 } 324 else 325 err = ftp_login (csock, user, passwd); 326 327 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */ 328 switch (err) 329 { 330 case FTPRERR: 331 logputs (LOG_VERBOSE, "\n"); 332 logputs (LOG_NOTQUIET, _("\ 333Error in server response, closing control connection.\n")); 334 fd_close (csock); 335 con->csock = -1; 336 return err; 337 case FTPSRVERR: 338 logputs (LOG_VERBOSE, "\n"); 339 logputs (LOG_NOTQUIET, _("Error in server greeting.\n")); 340 fd_close (csock); 341 con->csock = -1; 342 return err; 343 case WRITEFAILED: 344 logputs (LOG_VERBOSE, "\n"); 345 logputs (LOG_NOTQUIET, 346 _("Write failed, closing control connection.\n")); 347 fd_close (csock); 348 con->csock = -1; 349 return err; 350 case FTPLOGREFUSED: 351 logputs (LOG_VERBOSE, "\n"); 352 logputs (LOG_NOTQUIET, _("The server refuses login.\n")); 353 fd_close (csock); 354 con->csock = -1; 355 return FTPLOGREFUSED; 356 case FTPLOGINC: 357 logputs (LOG_VERBOSE, "\n"); 358 logputs (LOG_NOTQUIET, _("Login incorrect.\n")); 359 fd_close (csock); 360 con->csock = -1; 361 return FTPLOGINC; 362 case FTPOK: 363 if (!opt.server_response) 364 logputs (LOG_VERBOSE, _("Logged in!\n")); 365 break; 366 default: 367 abort (); 368 } 369 /* Third: Get the system type */ 370 if (!opt.server_response) 371 logprintf (LOG_VERBOSE, "==> SYST ... "); 372 err = ftp_syst (csock, &con->rs, &con->rsu); 373 /* FTPRERR */ 374 switch (err) 375 { 376 case FTPRERR: 377 logputs (LOG_VERBOSE, "\n"); 378 logputs (LOG_NOTQUIET, _("\ 379Error in server response, closing control connection.\n")); 380 fd_close (csock); 381 con->csock = -1; 382 return err; 383 case FTPSRVERR: 384 logputs (LOG_VERBOSE, "\n"); 385 logputs (LOG_NOTQUIET, 386 _("Server error, can't determine system type.\n")); 387 break; 388 case FTPOK: 389 /* Everything is OK. */ 390 break; 391 default: 392 abort (); 393 } 394 if (!opt.server_response && err != FTPSRVERR) 395 logputs (LOG_VERBOSE, _("done. ")); 396 397 /* 2013-10-17 Andrea Urbani (matfanjol) 398 According to the system type I choose which 399 list command will be used. 400 If I don't know that system, I will try, the 401 first time of each session, "LIST -a" and 402 "LIST". (see __LIST_A_EXPLANATION__ below) */ 403 switch (con->rs) 404 { 405 case ST_VMS: 406 /* About ST_VMS there is an old note: 407 2008-01-29 SMS. For a VMS FTP server, where "LIST -a" may not 408 fail, but will never do what is desired here, 409 skip directly to the simple "LIST" command 410 (assumed to be the last one in the list). */ 411 DEBUGP (("\nVMS: I know it and I will use \"LIST\" as standard list command\n")); 412 con->st |= LIST_AFTER_LIST_A_CHECK_DONE; 413 con->st |= AVOID_LIST_A; 414 break; 415 case ST_UNIX: 416 if (con->rsu == UST_MULTINET) 417 { 418 DEBUGP (("\nUNIX MultiNet: I know it and I will use \"LIST\" " 419 "as standard list command\n")); 420 con->st |= LIST_AFTER_LIST_A_CHECK_DONE; 421 con->st |= AVOID_LIST_A; 422 } 423 else if (con->rsu == UST_TYPE_L8) 424 { 425 DEBUGP (("\nUNIX TYPE L8: I know it and I will use \"LIST -a\" " 426 "as standard list command\n")); 427 con->st |= LIST_AFTER_LIST_A_CHECK_DONE; 428 con->st |= AVOID_LIST; 429 } 430 break; 431 default: 432 break; 433 } 434 435 /* Fourth: Find the initial ftp directory */ 436 437 if (!opt.server_response) 438 logprintf (LOG_VERBOSE, "==> PWD ... "); 439 err = ftp_pwd (csock, &con->id); 440 /* FTPRERR */ 441 switch (err) 442 { 443 case FTPRERR: 444 logputs (LOG_VERBOSE, "\n"); 445 logputs (LOG_NOTQUIET, _("\ 446Error in server response, closing control connection.\n")); 447 fd_close (csock); 448 con->csock = -1; 449 return err; 450 case FTPSRVERR : 451 /* PWD unsupported -- assume "/". */ 452 xfree_null (con->id); 453 con->id = xstrdup ("/"); 454 break; 455 case FTPOK: 456 /* Everything is OK. */ 457 break; 458 default: 459 abort (); 460 } 461 462#if 0 463 /* 2004-09-17 SMS. 464 Don't help me out. Please. 465 A reasonably recent VMS FTP server will cope just fine with 466 UNIX file specifications. This code just spoils things. 467 Discarding the device name, for example, is not a wise move. 468 This code was disabled but left in as an example of what not 469 to do. 470 */ 471 472 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]". 473 Convert it to "/INITIAL/FOLDER" */ 474 if (con->rs == ST_VMS) 475 { 476 char *path = strchr (con->id, '['); 477 char *pathend = path ? strchr (path + 1, ']') : NULL; 478 if (!path || !pathend) 479 DEBUGP (("Initial VMS directory not in the form [...]!\n")); 480 else 481 { 482 char *idir = con->id; 483 DEBUGP (("Preprocessing the initial VMS directory\n")); 484 DEBUGP ((" old = '%s'\n", con->id)); 485 /* We do the conversion in-place by copying the stuff 486 between [ and ] to the beginning, and changing dots 487 to slashes at the same time. */ 488 *idir++ = '/'; 489 for (++path; path < pathend; path++, idir++) 490 *idir = *path == '.' ? '/' : *path; 491 *idir = '\0'; 492 DEBUGP ((" new = '%s'\n\n", con->id)); 493 } 494 } 495#endif /* 0 */ 496 497 if (!opt.server_response) 498 logputs (LOG_VERBOSE, _("done.\n")); 499 500 /* Fifth: Set the FTP type. */ 501 type_char = ftp_process_type (u->params); 502 if (!opt.server_response) 503 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char); 504 err = ftp_type (csock, type_char); 505 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */ 506 switch (err) 507 { 508 case FTPRERR: 509 logputs (LOG_VERBOSE, "\n"); 510 logputs (LOG_NOTQUIET, _("\ 511Error in server response, closing control connection.\n")); 512 fd_close (csock); 513 con->csock = -1; 514 return err; 515 case WRITEFAILED: 516 logputs (LOG_VERBOSE, "\n"); 517 logputs (LOG_NOTQUIET, 518 _("Write failed, closing control connection.\n")); 519 fd_close (csock); 520 con->csock = -1; 521 return err; 522 case FTPUNKNOWNTYPE: 523 logputs (LOG_VERBOSE, "\n"); 524 logprintf (LOG_NOTQUIET, 525 _("Unknown type `%c', closing control connection.\n"), 526 type_char); 527 fd_close (csock); 528 con->csock = -1; 529 return err; 530 case FTPOK: 531 /* Everything is OK. */ 532 break; 533 default: 534 abort (); 535 } 536 if (!opt.server_response) 537 logputs (LOG_VERBOSE, _("done. ")); 538 } /* do login */ 539 540 if (cmd & DO_CWD) 541 { 542 if (!*u->dir) 543 logputs (LOG_VERBOSE, _("==> CWD not needed.\n")); 544 else 545 { 546 const char *targ = NULL; 547 int cwd_count; 548 int cwd_end; 549 int cwd_start; 550 551 char *target = u->dir; 552 553 DEBUGP (("changing working directory\n")); 554 555 /* Change working directory. To change to a non-absolute 556 Unix directory, we need to prepend initial directory 557 (con->id) to it. Absolute directories "just work". 558 559 A relative directory is one that does not begin with '/' 560 and, on non-Unix OS'es, one that doesn't begin with 561 "[a-z]:". 562 563 This is not done for OS400, which doesn't use 564 "/"-delimited directories, nor does it support directory 565 hierarchies. "CWD foo" followed by "CWD bar" leaves us 566 in "bar", not in "foo/bar", as would be customary 567 elsewhere. */ 568 569 /* 2004-09-20 SMS. 570 Why is this wise even on UNIX? It certainly fouls VMS. 571 See below for a more reliable, more universal method. 572 */ 573 574 /* 2008-04-22 MJC. 575 I'm not crazy about it either. I'm informed it's useful 576 for misconfigured servers that have some dirs in the path 577 with +x but -r, but this method is not RFC-conformant. I 578 understand the need to deal with crappy server 579 configurations, but it's far better to use the canonical 580 method first, and fall back to kludges second. 581 */ 582 583 if (target[0] != '/' 584 && !(con->rs != ST_UNIX 585 && c_isalpha (target[0]) 586 && target[1] == ':') 587 && (con->rs != ST_OS400) 588 && (con->rs != ST_VMS)) 589 { 590 int idlen = strlen (con->id); 591 char *ntarget, *p; 592 593 /* Strip trailing slash(es) from con->id. */ 594 while (idlen > 0 && con->id[idlen - 1] == '/') 595 --idlen; 596 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1); 597 memcpy (p, con->id, idlen); 598 p += idlen; 599 *p++ = '/'; 600 strcpy (p, target); 601 602 DEBUGP (("Prepended initial PWD to relative path:\n")); 603 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n", 604 con->id, target, ntarget)); 605 target = ntarget; 606 } 607 608#if 0 609 /* 2004-09-17 SMS. 610 Don't help me out. Please. 611 A reasonably recent VMS FTP server will cope just fine with 612 UNIX file specifications. This code just spoils things. 613 Discarding the device name, for example, is not a wise 614 move. 615 This code was disabled but left in as an example of what 616 not to do. 617 */ 618 619 /* If the FTP host runs VMS, we will have to convert the absolute 620 directory path in UNIX notation to absolute directory path in 621 VMS notation as VMS FTP servers do not like UNIX notation of 622 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */ 623 624 if (con->rs == ST_VMS) 625 { 626 char *tmpp; 627 char *ntarget = (char *)alloca (strlen (target) + 2); 628 /* We use a converted initial dir, so directories in 629 TARGET will be separated with slashes, something like 630 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to 631 "[INITIAL.FOLDER.DIR.SUBDIR]". */ 632 strcpy (ntarget, target); 633 assert (*ntarget == '/'); 634 *ntarget = '['; 635 for (tmpp = ntarget + 1; *tmpp; tmpp++) 636 if (*tmpp == '/') 637 *tmpp = '.'; 638 *tmpp++ = ']'; 639 *tmpp = '\0'; 640 DEBUGP (("Changed file name to VMS syntax:\n")); 641 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget)); 642 target = ntarget; 643 } 644#endif /* 0 */ 645 646 /* 2004-09-20 SMS. 647 A relative directory is relative to the initial directory. 648 Thus, what _is_ useful on VMS (and probably elsewhere) is 649 to CWD to the initial directory (ideally, whatever the 650 server reports, _exactly_, NOT badly UNIX-ixed), and then 651 CWD to the (new) relative directory. This should probably 652 be restructured as a function, called once or twice, but 653 I'm lazy enough to take the badly indented loop short-cut 654 for now. 655 */ 656 657 /* Decide on one pass (absolute) or two (relative). 658 The VMS restriction may be relaxed when the squirrely code 659 above is reformed. 660 */ 661 if ((con->rs == ST_VMS) && (target[0] != '/')) 662 { 663 cwd_start = 0; 664 DEBUGP (("Using two-step CWD for relative path.\n")); 665 } 666 else 667 { 668 /* Go straight to the target. */ 669 cwd_start = 1; 670 } 671 672 /* At least one VMS FTP server (TCPware V5.6-2) can switch to 673 a UNIX emulation mode when given a UNIX-like directory 674 specification (like "a/b/c"). If allowed to continue this 675 way, LIST interpretation will be confused, because the 676 system type (SYST response) will not be re-checked, and 677 future UNIX-format directory listings (for multiple URLs or 678 "-r") will be horribly misinterpreted. 679 680 The cheap and nasty work-around is to do a "CWD []" after a 681 UNIX-like directory specification is used. (A single-level 682 directory is harmless.) This puts the TCPware server back 683 into VMS mode, and does no harm on other servers. 684 685 Unlike the rest of this block, this particular behavior 686 _is_ VMS-specific, so it gets its own VMS test. 687 */ 688 if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL)) 689 { 690 cwd_end = 3; 691 DEBUGP (("Using extra \"CWD []\" step for VMS server.\n")); 692 } 693 else 694 { 695 cwd_end = 2; 696 } 697 698 /* 2004-09-20 SMS. */ 699 /* Sorry about the deviant indenting. Laziness. */ 700 701 for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++) 702 { 703 switch (cwd_count) 704 { 705 case 0: 706 /* Step one (optional): Go to the initial directory, 707 exactly as reported by the server. 708 */ 709 targ = con->id; 710 break; 711 712 case 1: 713 /* Step two: Go to the target directory. (Absolute or 714 relative will work now.) 715 */ 716 targ = target; 717 break; 718 719 case 2: 720 /* Step three (optional): "CWD []" to restore server 721 VMS-ness. 722 */ 723 targ = "[]"; 724 break; 725 726 default: 727 /* Can't happen. */ 728 assert (1); 729 } 730 731 if (!opt.server_response) 732 logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count, 733 quotearg_style (escape_quoting_style, target)); 734 err = ftp_cwd (csock, targ); 735 /* FTPRERR, WRITEFAILED, FTPNSFOD */ 736 switch (err) 737 { 738 case FTPRERR: 739 logputs (LOG_VERBOSE, "\n"); 740 logputs (LOG_NOTQUIET, _("\ 741Error in server response, closing control connection.\n")); 742 fd_close (csock); 743 con->csock = -1; 744 return err; 745 case WRITEFAILED: 746 logputs (LOG_VERBOSE, "\n"); 747 logputs (LOG_NOTQUIET, 748 _("Write failed, closing control connection.\n")); 749 fd_close (csock); 750 con->csock = -1; 751 return err; 752 case FTPNSFOD: 753 logputs (LOG_VERBOSE, "\n"); 754 logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"), 755 quote (u->dir)); 756 fd_close (csock); 757 con->csock = -1; 758 return err; 759 case FTPOK: 760 break; 761 default: 762 abort (); 763 } 764 if (!opt.server_response) 765 logputs (LOG_VERBOSE, _("done.\n")); 766 767 } /* for */ 768 769 /* 2004-09-20 SMS. */ 770 /* End of deviant indenting. */ 771 772 } /* else */ 773 } 774 else /* do not CWD */ 775 logputs (LOG_VERBOSE, _("==> CWD not required.\n")); 776 777 if ((cmd & DO_RETR) && passed_expected_bytes == 0) 778 { 779 if (opt.verbose) 780 { 781 if (!opt.server_response) 782 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", 783 quotearg_style (escape_quoting_style, u->file)); 784 } 785 786 err = ftp_size (csock, u->file, &expected_bytes); 787 /* FTPRERR */ 788 switch (err) 789 { 790 case FTPRERR: 791 case FTPSRVERR: 792 logputs (LOG_VERBOSE, "\n"); 793 logputs (LOG_NOTQUIET, _("\ 794Error in server response, closing control connection.\n")); 795 fd_close (csock); 796 con->csock = -1; 797 return err; 798 case FTPOK: 799 got_expected_bytes = true; 800 /* Everything is OK. */ 801 break; 802 default: 803 abort (); 804 } 805 if (!opt.server_response) 806 { 807 logprintf (LOG_VERBOSE, "%s\n", 808 expected_bytes ? 809 number_to_static_string (expected_bytes) : 810 _("done.\n")); 811 } 812 } 813 814 if (cmd & DO_RETR && restval > 0 && restval == expected_bytes) 815 { 816 /* Server confirms that file has length restval. We should stop now. 817 Some servers (f.e. NcFTPd) return error when receive REST 0 */ 818 logputs (LOG_VERBOSE, _("File has already been retrieved.\n")); 819 fd_close (csock); 820 con->csock = -1; 821 return RETRFINISHED; 822 } 823 824 do 825 { 826 try_again = false; 827 /* If anything is to be retrieved, PORT (or PASV) must be sent. */ 828 if (cmd & (DO_LIST | DO_RETR)) 829 { 830 if (opt.ftp_pasv) 831 { 832 ip_address passive_addr; 833 int passive_port; 834 err = ftp_do_pasv (csock, &passive_addr, &passive_port); 835 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */ 836 switch (err) 837 { 838 case FTPRERR: 839 logputs (LOG_VERBOSE, "\n"); 840 logputs (LOG_NOTQUIET, _("\ 841Error in server response, closing control connection.\n")); 842 fd_close (csock); 843 con->csock = -1; 844 return err; 845 case WRITEFAILED: 846 logputs (LOG_VERBOSE, "\n"); 847 logputs (LOG_NOTQUIET, 848 _("Write failed, closing control connection.\n")); 849 fd_close (csock); 850 con->csock = -1; 851 return err; 852 case FTPNOPASV: 853 logputs (LOG_VERBOSE, "\n"); 854 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n")); 855 break; 856 case FTPINVPASV: 857 logputs (LOG_VERBOSE, "\n"); 858 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n")); 859 break; 860 case FTPOK: 861 break; 862 default: 863 abort (); 864 } /* switch (err) */ 865 if (err==FTPOK) 866 { 867 DEBUGP (("trying to connect to %s port %d\n", 868 print_address (&passive_addr), passive_port)); 869 dtsock = connect_to_ip (&passive_addr, passive_port, NULL); 870 if (dtsock < 0) 871 { 872 int save_errno = errno; 873 fd_close (csock); 874 con->csock = -1; 875 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"), 876 print_address (&passive_addr), passive_port, 877 strerror (save_errno)); 878 return (retryable_socket_connect_error (save_errno) 879 ? CONERROR : CONIMPOSSIBLE); 880 } 881 882 pasv_mode_open = true; /* Flag to avoid accept port */ 883 if (!opt.server_response) 884 logputs (LOG_VERBOSE, _("done. ")); 885 } /* err==FTP_OK */ 886 } 887 888 if (!pasv_mode_open) /* Try to use a port command if PASV failed */ 889 { 890 err = ftp_do_port (csock, &local_sock); 891 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR, 892 FTPPORTERR */ 893 switch (err) 894 { 895 case FTPRERR: 896 logputs (LOG_VERBOSE, "\n"); 897 logputs (LOG_NOTQUIET, _("\ 898Error in server response, closing control connection.\n")); 899 fd_close (csock); 900 con->csock = -1; 901 fd_close (dtsock); 902 fd_close (local_sock); 903 return err; 904 case WRITEFAILED: 905 logputs (LOG_VERBOSE, "\n"); 906 logputs (LOG_NOTQUIET, 907 _("Write failed, closing control connection.\n")); 908 fd_close (csock); 909 con->csock = -1; 910 fd_close (dtsock); 911 fd_close (local_sock); 912 return err; 913 case CONSOCKERR: 914 logputs (LOG_VERBOSE, "\n"); 915 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno)); 916 fd_close (csock); 917 con->csock = -1; 918 fd_close (dtsock); 919 fd_close (local_sock); 920 return err; 921 case FTPSYSERR: 922 logputs (LOG_VERBOSE, "\n"); 923 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"), 924 strerror (errno)); 925 fd_close (dtsock); 926 return err; 927 case FTPPORTERR: 928 logputs (LOG_VERBOSE, "\n"); 929 logputs (LOG_NOTQUIET, _("Invalid PORT.\n")); 930 fd_close (csock); 931 con->csock = -1; 932 fd_close (dtsock); 933 fd_close (local_sock); 934 return err; 935 case FTPOK: 936 break; 937 default: 938 abort (); 939 } /* port switch */ 940 if (!opt.server_response) 941 logputs (LOG_VERBOSE, _("done. ")); 942 } /* dtsock == -1 */ 943 } /* cmd & (DO_LIST | DO_RETR) */ 944 945 /* Restart if needed. */ 946 if (restval && (cmd & DO_RETR)) 947 { 948 if (!opt.server_response) 949 logprintf (LOG_VERBOSE, "==> REST %s ... ", 950 number_to_static_string (restval)); 951 err = ftp_rest (csock, restval); 952 953 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */ 954 switch (err) 955 { 956 case FTPRERR: 957 logputs (LOG_VERBOSE, "\n"); 958 logputs (LOG_NOTQUIET, _("\ 959Error in server response, closing control connection.\n")); 960 fd_close (csock); 961 con->csock = -1; 962 fd_close (dtsock); 963 fd_close (local_sock); 964 return err; 965 case WRITEFAILED: 966 logputs (LOG_VERBOSE, "\n"); 967 logputs (LOG_NOTQUIET, 968 _("Write failed, closing control connection.\n")); 969 fd_close (csock); 970 con->csock = -1; 971 fd_close (dtsock); 972 fd_close (local_sock); 973 return err; 974 case FTPRESTFAIL: 975 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n")); 976 rest_failed = true; 977 break; 978 case FTPOK: 979 break; 980 default: 981 abort (); 982 } 983 if (err != FTPRESTFAIL && !opt.server_response) 984 logputs (LOG_VERBOSE, _("done. ")); 985 } /* restval && cmd & DO_RETR */ 986 987 if (cmd & DO_RETR) 988 { 989 /* If we're in spider mode, don't really retrieve anything except 990 the directory listing and verify whether the given "file" exists. */ 991 if (opt.spider) 992 { 993 bool exists = false; 994 struct fileinfo *f; 995 uerr_t _res = ftp_get_listing (u, con, &f); 996 /* Set the DO_RETR command flag again, because it gets unset when 997 calling ftp_get_listing() and would otherwise cause an assertion 998 failure earlier on when this function gets repeatedly called 999 (e.g., when recursing). */ 1000 con->cmd |= DO_RETR; 1001 if (_res == RETROK) 1002 { 1003 while (f) 1004 { 1005 if (!strcmp (f->name, u->file)) 1006 { 1007 exists = true; 1008 break; 1009 } 1010 f = f->next; 1011 } 1012 if (exists) 1013 { 1014 logputs (LOG_VERBOSE, "\n"); 1015 logprintf (LOG_NOTQUIET, _("File %s exists.\n"), 1016 quote (u->file)); 1017 } 1018 else 1019 { 1020 logputs (LOG_VERBOSE, "\n"); 1021 logprintf (LOG_NOTQUIET, _("No such file %s.\n"), 1022 quote (u->file)); 1023 } 1024 } 1025 fd_close (csock); 1026 con->csock = -1; 1027 fd_close (dtsock); 1028 fd_close (local_sock); 1029 return RETRFINISHED; 1030 } 1031 1032 if (opt.verbose) 1033 { 1034 if (!opt.server_response) 1035 { 1036 if (restval) 1037 logputs (LOG_VERBOSE, "\n"); 1038 logprintf (LOG_VERBOSE, "==> RETR %s ... ", 1039 quotearg_style (escape_quoting_style, u->file)); 1040 } 1041 } 1042 1043 err = ftp_retr (csock, u->file); 1044 /* FTPRERR, WRITEFAILED, FTPNSFOD */ 1045 switch (err) 1046 { 1047 case FTPRERR: 1048 logputs (LOG_VERBOSE, "\n"); 1049 logputs (LOG_NOTQUIET, _("\ 1050Error in server response, closing control connection.\n")); 1051 fd_close (csock); 1052 con->csock = -1; 1053 fd_close (dtsock); 1054 fd_close (local_sock); 1055 return err; 1056 case WRITEFAILED: 1057 logputs (LOG_VERBOSE, "\n"); 1058 logputs (LOG_NOTQUIET, 1059 _("Write failed, closing control connection.\n")); 1060 fd_close (csock); 1061 con->csock = -1; 1062 fd_close (dtsock); 1063 fd_close (local_sock); 1064 return err; 1065 case FTPNSFOD: 1066 logputs (LOG_VERBOSE, "\n"); 1067 logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"), 1068 quote (u->file)); 1069 fd_close (dtsock); 1070 fd_close (local_sock); 1071 return err; 1072 case FTPOK: 1073 break; 1074 default: 1075 abort (); 1076 } 1077 1078 if (!opt.server_response) 1079 logputs (LOG_VERBOSE, _("done.\n")); 1080 1081 if (! got_expected_bytes) 1082 expected_bytes = *last_expected_bytes; 1083 } /* do retrieve */ 1084 1085 if (cmd & DO_LIST) 1086 { 1087 if (!opt.server_response) 1088 logputs (LOG_VERBOSE, "==> LIST ... "); 1089 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST' 1090 without arguments is better than `LIST .'; confirmed by 1091 RFC959. */ 1092 err = ftp_list (csock, NULL, con->st&AVOID_LIST_A, con->st&AVOID_LIST, &list_a_used); 1093 1094 /* FTPRERR, WRITEFAILED */ 1095 switch (err) 1096 { 1097 case FTPRERR: 1098 logputs (LOG_VERBOSE, "\n"); 1099 logputs (LOG_NOTQUIET, _("\ 1100Error in server response, closing control connection.\n")); 1101 fd_close (csock); 1102 con->csock = -1; 1103 fd_close (dtsock); 1104 fd_close (local_sock); 1105 return err; 1106 case WRITEFAILED: 1107 logputs (LOG_VERBOSE, "\n"); 1108 logputs (LOG_NOTQUIET, 1109 _("Write failed, closing control connection.\n")); 1110 fd_close (csock); 1111 con->csock = -1; 1112 fd_close (dtsock); 1113 fd_close (local_sock); 1114 return err; 1115 case FTPNSFOD: 1116 logputs (LOG_VERBOSE, "\n"); 1117 logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"), 1118 quote (".")); 1119 fd_close (dtsock); 1120 fd_close (local_sock); 1121 return err; 1122 case FTPOK: 1123 break; 1124 default: 1125 abort (); 1126 } 1127 if (!opt.server_response) 1128 logputs (LOG_VERBOSE, _("done.\n")); 1129 1130 if (! got_expected_bytes) 1131 expected_bytes = *last_expected_bytes; 1132 } /* cmd & DO_LIST */ 1133 1134 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST))) 1135 return RETRFINISHED; 1136 1137 /* Some FTP servers return the total length of file after REST 1138 command, others just return the remaining size. */ 1139 if (passed_expected_bytes && restval && expected_bytes 1140 && (expected_bytes == passed_expected_bytes - restval)) 1141 { 1142 DEBUGP (("Lying FTP server found, adjusting.\n")); 1143 expected_bytes = passed_expected_bytes; 1144 } 1145 1146 /* If no transmission was required, then everything is OK. */ 1147 if (!pasv_mode_open) /* we are not using pasive mode so we need 1148 to accept */ 1149 { 1150 /* Wait for the server to connect to the address we're waiting 1151 at. */ 1152 dtsock = accept_connection (local_sock); 1153 if (dtsock < 0) 1154 { 1155 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno)); 1156 return CONERROR; 1157 } 1158 } 1159 1160 /* Open the file -- if output_stream is set, use it instead. */ 1161 1162 /* 2005-04-17 SMS. 1163 Note that having the output_stream ("-O") file opened in main 1164 (main.c) rather limits the ability in VMS to open the file 1165 differently for ASCII versus binary FTP here. (Of course, doing it 1166 there allows a open failure to be detected immediately, without first 1167 connecting to the server.) 1168 */ 1169 if (!output_stream || con->cmd & DO_LIST) 1170 { 1171/* On VMS, alter the name as required. */ 1172#ifdef __VMS 1173 char *targ; 1174 1175 targ = ods_conform (con->target); 1176 if (targ != con->target) 1177 { 1178 xfree (con->target); 1179 con->target = targ; 1180 } 1181#endif /* def __VMS */ 1182 1183 mkalldirs (con->target); 1184 if (opt.backups) 1185 rotate_backups (con->target); 1186 1187/* 2005-04-15 SMS. 1188 For VMS, define common fopen() optional arguments, and a handy macro 1189 for use as a variable "binary" flag. 1190 Elsewhere, define a constant "binary" flag. 1191 Isn't it nice to have distinct text and binary file types? 1192*/ 1193/* 2011-09-30 SMS. 1194 Added listing files to the set of non-"binary" (text, Stream_LF) 1195 files. (Wget works either way, but other programs, like, say, text 1196 editors, work better on listing files which have text attributes.) 1197 Now we use "binary" attributes for a binary ("IMAGE") transfer, 1198 unless "--ftp-stmlf" was specified, and we always use non-"binary" 1199 (text, Stream_LF) attributes for a listing file, or for an ASCII 1200 transfer. 1201 Tidied the VMS-specific BIN_TYPE_xxx macros, and changed the call to 1202 fopen_excl() (restored?) to use BIN_TYPE_FILE instead of "true". 1203*/ 1204#ifdef __VMS 1205# define BIN_TYPE_TRANSFER (type_char != 'A') 1206# define BIN_TYPE_FILE \ 1207 ((!(cmd & DO_LIST)) && BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0)) 1208# define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id 1209# define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS 1210#else /* def __VMS */ 1211# define BIN_TYPE_FILE true 1212#endif /* def __VMS [else] */ 1213 1214 if (restval && !(con->cmd & DO_LIST)) 1215 { 1216#ifdef __VMS 1217 int open_id; 1218 1219 if (BIN_TYPE_FILE) 1220 { 1221 open_id = 3; 1222 fp = fopen (con->target, "ab", FOPEN_OPT_ARGS_BIN); 1223 } 1224 else 1225 { 1226 open_id = 4; 1227 fp = fopen (con->target, "a", FOPEN_OPT_ARGS); 1228 } 1229#else /* def __VMS */ 1230 fp = fopen (con->target, "ab"); 1231#endif /* def __VMS [else] */ 1232 } 1233 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct 1234 || opt.output_document || count > 0) 1235 { 1236 if (opt.unlink && file_exists_p (con->target)) 1237 { 1238 if (unlink (con->target) < 0) 1239 { 1240 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, 1241 strerror (errno)); 1242 fd_close (csock); 1243 con->csock = -1; 1244 fd_close (dtsock); 1245 fd_close (local_sock); 1246 return UNLINKERR; 1247 } 1248 } 1249 1250#ifdef __VMS 1251 int open_id; 1252 1253 if (BIN_TYPE_FILE) 1254 { 1255 open_id = 5; 1256 fp = fopen (con->target, "wb", FOPEN_OPT_ARGS_BIN); 1257 } 1258 else 1259 { 1260 open_id = 6; 1261 fp = fopen (con->target, "w", FOPEN_OPT_ARGS); 1262 } 1263#else /* def __VMS */ 1264 fp = fopen (con->target, "wb"); 1265#endif /* def __VMS [else] */ 1266 } 1267 else 1268 { 1269 fp = fopen_excl (con->target, BIN_TYPE_FILE); 1270 if (!fp && errno == EEXIST) 1271 { 1272 /* We cannot just invent a new name and use it (which is 1273 what functions like unique_create typically do) 1274 because we told the user we'd use this name. 1275 Instead, return and retry the download. */ 1276 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"), 1277 con->target); 1278 fd_close (csock); 1279 con->csock = -1; 1280 fd_close (dtsock); 1281 fd_close (local_sock); 1282 return FOPEN_EXCL_ERR; 1283 } 1284 } 1285 if (!fp) 1286 { 1287 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno)); 1288 fd_close (csock); 1289 con->csock = -1; 1290 fd_close (dtsock); 1291 fd_close (local_sock); 1292 return FOPENERR; 1293 } 1294 } 1295 else 1296 fp = output_stream; 1297 1298 if (passed_expected_bytes) 1299 { 1300 print_length (passed_expected_bytes, restval, true); 1301 expected_bytes = passed_expected_bytes; 1302 /* for fd_read_body's progress bar */ 1303 } 1304 else if (expected_bytes) 1305 print_length (expected_bytes, restval, false); 1306 1307 /* Get the contents of the document. */ 1308 flags = 0; 1309 if (restval && rest_failed) 1310 flags |= rb_skip_startpos; 1311 rd_size = 0; 1312 res = fd_read_body (con->target, dtsock, fp, 1313 expected_bytes ? expected_bytes - restval : 0, 1314 restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp); 1315 1316 tms = datetime_str (time (NULL)); 1317 tmrate = retr_rate (rd_size, con->dltime); 1318 total_download_time += con->dltime; 1319 1320 fd_close (local_sock); 1321 /* Close the local file. */ 1322 if (!output_stream || con->cmd & DO_LIST) 1323 fclose (fp); 1324 1325 /* If fd_read_body couldn't write to fp or warc_tmp, bail out. */ 1326 if (res == -2 || (warc_tmp != NULL && res == -3)) 1327 { 1328 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"), 1329 con->target, strerror (errno)); 1330 fd_close (csock); 1331 con->csock = -1; 1332 fd_close (dtsock); 1333 if (res == -2) 1334 return FWRITEERR; 1335 else if (res == -3) 1336 return WARC_TMP_FWRITEERR; 1337 } 1338 else if (res == -1) 1339 { 1340 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "), 1341 tms, tmrate, fd_errstr (dtsock)); 1342 if (opt.server_response) 1343 logputs (LOG_ALWAYS, "\n"); 1344 } 1345 fd_close (dtsock); 1346 1347 /* Get the server to tell us if everything is retrieved. */ 1348 err = ftp_response (csock, &respline); 1349 *last_expected_bytes = ftp_expected_bytes (respline); 1350 if (err != FTPOK) 1351 { 1352 /* The control connection is decidedly closed. Print the time 1353 only if it hasn't already been printed. */ 1354 if (res != -1) 1355 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate); 1356 logputs (LOG_NOTQUIET, _("Control connection closed.\n")); 1357 /* If there is an error on the control connection, close it, but 1358 return FTPRETRINT, since there is a possibility that the 1359 whole file was retrieved nevertheless (but that is for 1360 ftp_loop_internal to decide). */ 1361 fd_close (csock); 1362 con->csock = -1; 1363 return FTPRETRINT; 1364 } /* err != FTPOK */ 1365 /* If retrieval failed for any reason, return FTPRETRINT, but do not 1366 close socket, since the control connection is still alive. If 1367 there is something wrong with the control connection, it will 1368 become apparent later. */ 1369 if (*respline != '2') 1370 { 1371 xfree (respline); 1372 if (res != -1) 1373 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate); 1374 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n")); 1375 return FTPRETRINT; 1376 } 1377 xfree (respline); 1378 1379 if (res == -1) 1380 { 1381 /* What now? The data connection was erroneous, whereas the 1382 response says everything is OK. We shall play it safe. */ 1383 return FTPRETRINT; 1384 } 1385 1386 if (!(cmd & LEAVE_PENDING)) 1387 { 1388 /* Closing the socket is faster than sending 'QUIT' and the 1389 effect is the same. */ 1390 fd_close (csock); 1391 con->csock = -1; 1392 } 1393 /* If it was a listing, and opt.server_response is true, 1394 print it out. */ 1395 if (con->cmd & DO_LIST) 1396 { 1397 if (opt.server_response) 1398 { 1399/* 2005-02-25 SMS. 1400 Much of this work may already have been done, but repeating it should 1401 do no damage beyond wasting time. 1402*/ 1403/* On VMS, alter the name as required. */ 1404#ifdef __VMS 1405 char *targ; 1406 1407 targ = ods_conform( con->target); 1408 if (targ != con->target) 1409 { 1410 xfree( con->target); 1411 con->target = targ; 1412 } 1413#endif /* def __VMS */ 1414 1415 mkalldirs (con->target); 1416 fp = fopen (con->target, "r"); 1417 if (!fp) 1418 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno)); 1419 else 1420 { 1421 char *line = NULL; 1422 size_t bufsize = 0; 1423 ssize_t len; 1424 1425 /* The lines are being read with getline because of 1426 no-buffering on opt.lfile. */ 1427 while ((len = getline (&line, &bufsize, fp)) > 0) 1428 { 1429 while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) 1430 line[--len] = '\0'; 1431 logprintf (LOG_ALWAYS, "%s\n", 1432 quotearg_style (escape_quoting_style, line)); 1433 } 1434 xfree (line); 1435 fclose (fp); 1436 } 1437 } /* server_response */ 1438 1439 /* 2013-10-17 Andrea Urbani (matfanjol) 1440 < __LIST_A_EXPLANATION__ > 1441 After the SYST command, looks if it knows that system. 1442 If yes, wget will force the use of "LIST" or "LIST -a". 1443 If no, wget will try, only the first time of each session, before the 1444 "LIST -a" command and after the "LIST". 1445 If "LIST -a" works and returns more or equal data of the "LIST", 1446 "LIST -a" will be the standard list command for all the session. 1447 If "LIST -a" fails or returns less data than "LIST" (think on the case 1448 of an existing file called "-a"), "LIST" will be the standard list 1449 command for all the session. 1450 ("LIST -a" is used to get also the hidden files) 1451 1452 */ 1453 if (!(con->st & LIST_AFTER_LIST_A_CHECK_DONE)) 1454 { 1455 /* We still have to check "LIST" after the first "LIST -a" to see 1456 if with "LIST" we get more data than "LIST -a", that means 1457 "LIST -a" returned files/folders with "-a" name. */ 1458 if (con->st & AVOID_LIST_A) 1459 { 1460 /* LIST was used in this cycle. 1461 Let's see the result. */ 1462 if (rd_size > previous_rd_size) 1463 { 1464 /* LIST returns more data than "LIST -a". 1465 "LIST" is the official command to use. */ 1466 con->st |= LIST_AFTER_LIST_A_CHECK_DONE; 1467 DEBUGP (("LIST returned more data than \"LIST -a\": " 1468 "I will use \"LIST\" as standard list command\n")); 1469 } 1470 else if (previous_rd_size > rd_size) 1471 { 1472 /* "LIST -a" returned more data then LIST. 1473 "LIST -a" is the official command to use. */ 1474 con->st |= LIST_AFTER_LIST_A_CHECK_DONE; 1475 con->st |= AVOID_LIST; 1476 con->st &= ~AVOID_LIST_A; 1477 /* Sorry, please, download again the "LIST -a"... */ 1478 try_again = true; 1479 DEBUGP (("LIST returned less data than \"LIST -a\": I will " 1480 "use \"LIST -a\" as standard list command\n")); 1481 } 1482 else 1483 { 1484 /* LIST and "LIST -a" return the same data. */ 1485 if (rd_size == 0) 1486 { 1487 /* Same empty data. We will check both again because 1488 we cannot check if "LIST -a" has returned an empty 1489 folder instead of a folder content. */ 1490 con->st &= ~AVOID_LIST_A; 1491 } 1492 else 1493 { 1494 /* Same data, so, better to take "LIST -a" that 1495 shows also hidden files/folders (when present) */ 1496 con->st |= LIST_AFTER_LIST_A_CHECK_DONE; 1497 con->st |= AVOID_LIST; 1498 con->st &= ~AVOID_LIST_A; 1499 DEBUGP (("LIST returned the same amount of data of " 1500 "\"LIST -a\": I will use \"LIST -a\" as standard " 1501 "list command\n")); 1502 } 1503 } 1504 } 1505 else 1506 { 1507 /* In this cycle "LIST -a" should being used. Is it true? */ 1508 if (list_a_used) 1509 { 1510 /* Yes, it is. 1511 OK, let's save the amount of data and try again 1512 with LIST */ 1513 previous_rd_size = rd_size; 1514 try_again = true; 1515 con->st |= AVOID_LIST_A; 1516 } 1517 else 1518 { 1519 /* No: something happens and LIST was used. 1520 This means "LIST -a" raises an error. */ 1521 con->st |= LIST_AFTER_LIST_A_CHECK_DONE; 1522 con->st |= AVOID_LIST_A; 1523 DEBUGP (("\"LIST -a\" failed: I will use \"LIST\" " 1524 "as standard list command\n")); 1525 } 1526 } 1527 } 1528 } 1529 } while (try_again); 1530 return RETRFINISHED; 1531} 1532 1533/* A one-file FTP loop. This is the part where FTP retrieval is 1534 retried, and retried, and retried, and... 1535 1536 This loop either gets commands from con, or (if ON_YOUR_OWN is 1537 set), makes them up to retrieve the file given by the URL. */ 1538static uerr_t 1539ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file) 1540{ 1541 int count, orig_lp; 1542 wgint restval, len = 0, qtyread = 0; 1543 char *tms, *locf; 1544 const char *tmrate = NULL; 1545 uerr_t err; 1546 struct_stat st; 1547 1548 /* Declare WARC variables. */ 1549 bool warc_enabled = (opt.warc_filename != NULL); 1550 FILE *warc_tmp = NULL; 1551 ip_address *warc_ip = NULL; 1552 wgint last_expected_bytes = 0; 1553 1554 /* Get the target, and set the name for the message accordingly. */ 1555 if ((f == NULL) && (con->target)) 1556 { 1557 /* Explicit file (like ".listing"). */ 1558 locf = con->target; 1559 } 1560 else 1561 { 1562 /* URL-derived file. Consider "-O file" name. */ 1563 con->target = url_file_name (u, NULL); 1564 if (!opt.output_document) 1565 locf = con->target; 1566 else 1567 locf = opt.output_document; 1568 } 1569 1570 /* If the output_document was given, then this check was already done and 1571 the file didn't exist. Hence the !opt.output_document */ 1572 1573 /* If we receive .listing file it is necessary to determine system type of the ftp 1574 server even if opn.noclobber is given. Thus we must ignore opt.noclobber in 1575 order to establish connection with the server and get system type. */ 1576 if (opt.noclobber && !opt.output_document && file_exists_p (con->target) 1577 && !((con->cmd & DO_LIST) && !(con->cmd & DO_RETR))) 1578 { 1579 logprintf (LOG_VERBOSE, 1580 _("File %s already there; not retrieving.\n"), quote (con->target)); 1581 /* If the file is there, we suppose it's retrieved OK. */ 1582 return RETROK; 1583 } 1584 1585 /* Remove it if it's a link. */ 1586 remove_link (con->target); 1587 1588 count = 0; 1589 1590 if (con->st & ON_YOUR_OWN) 1591 con->st = ON_YOUR_OWN; 1592 1593 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0; 1594 1595 /* THE loop. */ 1596 do 1597 { 1598 /* Increment the pass counter. */ 1599 ++count; 1600 sleep_between_retrievals (count); 1601 if (con->st & ON_YOUR_OWN) 1602 { 1603 con->cmd = 0; 1604 con->cmd |= (DO_RETR | LEAVE_PENDING); 1605 if (con->csock != -1) 1606 con->cmd &= ~ (DO_LOGIN | DO_CWD); 1607 else 1608 con->cmd |= (DO_LOGIN | DO_CWD); 1609 } 1610 else /* not on your own */ 1611 { 1612 if (con->csock != -1) 1613 con->cmd &= ~DO_LOGIN; 1614 else 1615 con->cmd |= DO_LOGIN; 1616 if (con->st & DONE_CWD) 1617 con->cmd &= ~DO_CWD; 1618 else 1619 con->cmd |= DO_CWD; 1620 } 1621 1622 /* For file RETR requests, we can write a WARC record. 1623 We record the file contents to a temporary file. */ 1624 if (warc_enabled && (con->cmd & DO_RETR) && warc_tmp == NULL) 1625 { 1626 warc_tmp = warc_tempfile (); 1627 if (warc_tmp == NULL) 1628 return WARC_TMP_FOPENERR; 1629 1630 if (!con->proxy && con->csock != -1) 1631 { 1632 warc_ip = (ip_address *) alloca (sizeof (ip_address)); 1633 socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER); 1634 } 1635 } 1636 1637 /* Decide whether or not to restart. */ 1638 if (con->cmd & DO_LIST) 1639 restval = 0; 1640 else if (opt.start_pos >= 0) 1641 restval = opt.start_pos; 1642 else if (opt.always_rest 1643 && stat (locf, &st) == 0 1644 && S_ISREG (st.st_mode)) 1645 /* When -c is used, continue from on-disk size. (Can't use 1646 hstat.len even if count>1 because we don't want a failed 1647 first attempt to clobber existing data.) */ 1648 restval = st.st_size; 1649 else if (count > 1) 1650 restval = qtyread; /* start where the previous run left off */ 1651 else 1652 restval = 0; 1653 1654 /* Get the current time string. */ 1655 tms = datetime_str (time (NULL)); 1656 /* Print fetch message, if opt.verbose. */ 1657 if (opt.verbose) 1658 { 1659 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD); 1660 char tmp[256]; 1661 strcpy (tmp, " "); 1662 if (count > 1) 1663 sprintf (tmp, _("(try:%2d)"), count); 1664 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n", 1665 tms, hurl, tmp, quote (locf)); 1666#ifdef WINDOWS 1667 ws_changetitle (hurl); 1668#endif 1669 xfree (hurl); 1670 } 1671 /* Send getftp the proper length, if fileinfo was provided. */ 1672 if (f && f->type != FT_SYMLINK) 1673 len = f->size; 1674 else 1675 len = 0; 1676 1677 /* If we are working on a WARC record, getftp should also write 1678 to the warc_tmp file. */ 1679 err = getftp (u, len, &qtyread, restval, con, count, &last_expected_bytes, 1680 warc_tmp); 1681 1682 if (con->csock == -1) 1683 con->st &= ~DONE_CWD; 1684 else 1685 con->st |= DONE_CWD; 1686 1687 switch (err) 1688 { 1689 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR: 1690 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED: 1691 case UNLINKERR: case WARC_TMP_FWRITEERR: 1692 /* Fatal errors, give up. */ 1693 if (warc_tmp != NULL) 1694 fclose (warc_tmp); 1695 return err; 1696 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR: 1697 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR: 1698 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV: 1699 case FOPEN_EXCL_ERR: 1700 printwhat (count, opt.ntry); 1701 /* non-fatal errors */ 1702 if (err == FOPEN_EXCL_ERR) 1703 { 1704 /* Re-determine the file name. */ 1705 xfree_null (con->target); 1706 con->target = url_file_name (u, NULL); 1707 locf = con->target; 1708 } 1709 continue; 1710 case FTPRETRINT: 1711 /* If the control connection was closed, the retrieval 1712 will be considered OK if f->size == len. */ 1713 if (!f || qtyread != f->size) 1714 { 1715 printwhat (count, opt.ntry); 1716 continue; 1717 } 1718 break; 1719 case RETRFINISHED: 1720 /* Great! */ 1721 break; 1722 default: 1723 /* Not as great. */ 1724 abort (); 1725 } 1726 tms = datetime_str (time (NULL)); 1727 if (!opt.spider) 1728 tmrate = retr_rate (qtyread - restval, con->dltime); 1729 1730 /* If we get out of the switch above without continue'ing, we've 1731 successfully downloaded a file. Remember this fact. */ 1732 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf); 1733 1734 if (con->st & ON_YOUR_OWN) 1735 { 1736 fd_close (con->csock); 1737 con->csock = -1; 1738 } 1739 if (!opt.spider) 1740 { 1741 bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document)); 1742 1743 logprintf (LOG_VERBOSE, 1744 write_to_stdout 1745 ? _("%s (%s) - written to stdout %s[%s]\n\n") 1746 : _("%s (%s) - %s saved [%s]\n\n"), 1747 tms, tmrate, 1748 write_to_stdout ? "" : quote (locf), 1749 number_to_static_string (qtyread)); 1750 } 1751 if (!opt.verbose && !opt.quiet) 1752 { 1753 /* Need to hide the password from the URL. The `if' is here 1754 so that we don't do the needless allocation every 1755 time. */ 1756 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD); 1757 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n", 1758 tms, hurl, number_to_static_string (qtyread), locf, count); 1759 xfree (hurl); 1760 } 1761 1762 if (warc_enabled && (con->cmd & DO_RETR)) 1763 { 1764 /* Create and store a WARC resource record for the retrieved file. */ 1765 bool warc_res; 1766 1767 warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL, 1768 warc_ip, NULL, warc_tmp, -1); 1769 if (! warc_res) 1770 return WARC_ERR; 1771 1772 /* warc_write_resource_record has also closed warc_tmp. */ 1773 } 1774 1775 if (con->cmd & DO_LIST) 1776 /* This is a directory listing file. */ 1777 { 1778 if (!opt.remove_listing) 1779 /* --dont-remove-listing was specified, so do count this towards the 1780 number of bytes and files downloaded. */ 1781 { 1782 total_downloaded_bytes += qtyread; 1783 numurls++; 1784 } 1785 1786 /* Deletion of listing files is not controlled by --delete-after, but 1787 by the more specific option --dont-remove-listing, and the code 1788 to do this deletion is in another function. */ 1789 } 1790 else if (!opt.spider) 1791 /* This is not a directory listing file. */ 1792 { 1793 /* Unlike directory listing files, don't pretend normal files weren't 1794 downloaded if they're going to be deleted. People seeding proxies, 1795 for instance, may want to know how many bytes and files they've 1796 downloaded through it. */ 1797 total_downloaded_bytes += qtyread; 1798 numurls++; 1799 1800 if (opt.delete_after && !input_file_url (opt.input_filename)) 1801 { 1802 DEBUGP (("\ 1803Removing file due to --delete-after in ftp_loop_internal():\n")); 1804 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf); 1805 if (unlink (locf)) 1806 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); 1807 } 1808 } 1809 1810 /* Restore the original leave-pendingness. */ 1811 if (orig_lp) 1812 con->cmd |= LEAVE_PENDING; 1813 else 1814 con->cmd &= ~LEAVE_PENDING; 1815 1816 if (local_file) 1817 *local_file = xstrdup (locf); 1818 1819 return RETROK; 1820 } while (!opt.ntry || (count < opt.ntry)); 1821 1822 if (con->csock != -1 && (con->st & ON_YOUR_OWN)) 1823 { 1824 fd_close (con->csock); 1825 con->csock = -1; 1826 } 1827 return TRYLIMEXC; 1828} 1829 1830/* Return the directory listing in a reusable format. The directory 1831 is specifed in u->dir. */ 1832static uerr_t 1833ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f) 1834{ 1835 uerr_t err; 1836 char *uf; /* url file name */ 1837 char *lf; /* list file name */ 1838 char *old_target = con->target; 1839 1840 con->st &= ~ON_YOUR_OWN; 1841 con->cmd |= (DO_LIST | LEAVE_PENDING); 1842 con->cmd &= ~DO_RETR; 1843 1844 /* Find the listing file name. We do it by taking the file name of 1845 the URL and replacing the last component with the listing file 1846 name. */ 1847 uf = url_file_name (u, NULL); 1848 lf = file_merge (uf, LIST_FILENAME); 1849 xfree (uf); 1850 DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf))); 1851 1852 con->target = xstrdup (lf); 1853 xfree (lf); 1854 err = ftp_loop_internal (u, NULL, con, NULL); 1855 lf = xstrdup (con->target); 1856 xfree (con->target); 1857 con->target = old_target; 1858 1859 if (err == RETROK) 1860 { 1861 *f = ftp_parse_ls (lf, con->rs); 1862 if (opt.remove_listing) 1863 { 1864 if (unlink (lf)) 1865 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); 1866 else 1867 logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf)); 1868 } 1869 } 1870 else 1871 *f = NULL; 1872 xfree (lf); 1873 con->cmd &= ~DO_LIST; 1874 return err; 1875} 1876 1877static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *); 1878static uerr_t ftp_retrieve_glob (struct url *, ccon *, int); 1879static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **); 1880static void freefileinfo (struct fileinfo *f); 1881 1882/* Retrieve a list of files given in struct fileinfo linked list. If 1883 a file is a symbolic link, do not retrieve it, but rather try to 1884 set up a similar link on the local disk, if the symlinks are 1885 supported. 1886 1887 If opt.recursive is set, after all files have been retrieved, 1888 ftp_retrieve_dirs will be called to retrieve the directories. */ 1889static uerr_t 1890ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con) 1891{ 1892 static int depth = 0; 1893 uerr_t err; 1894 struct fileinfo *orig; 1895 wgint local_size; 1896 time_t tml; 1897 bool dlthis; /* Download this (file). */ 1898 const char *actual_target = NULL; 1899 1900 /* Increase the depth. */ 1901 ++depth; 1902 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel) 1903 { 1904 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"), 1905 depth, opt.reclevel)); 1906 --depth; 1907 return RECLEVELEXC; 1908 } 1909 1910 assert (f != NULL); 1911 orig = f; 1912 1913 con->st &= ~ON_YOUR_OWN; 1914 if (!(con->st & DONE_CWD)) 1915 con->cmd |= DO_CWD; 1916 else 1917 con->cmd &= ~DO_CWD; 1918 con->cmd |= (DO_RETR | LEAVE_PENDING); 1919 1920 if (con->csock < 0) 1921 con->cmd |= DO_LOGIN; 1922 else 1923 con->cmd &= ~DO_LOGIN; 1924 1925 err = RETROK; /* in case it's not used */ 1926 1927 while (f) 1928 { 1929 char *old_target, *ofile; 1930 1931 if (opt.quota && total_downloaded_bytes > opt.quota) 1932 { 1933 --depth; 1934 return QUOTEXC; 1935 } 1936 old_target = con->target; 1937 1938 ofile = xstrdup (u->file); 1939 url_set_file (u, f->name); 1940 1941 con->target = url_file_name (u, NULL); 1942 err = RETROK; 1943 1944 dlthis = true; 1945 if (opt.timestamping && f->type == FT_PLAINFILE) 1946 { 1947 struct_stat st; 1948 /* If conversion of HTML files retrieved via FTP is ever implemented, 1949 we'll need to stat() <file>.orig here when -K has been specified. 1950 I'm not implementing it now since files on an FTP server are much 1951 more likely than files on an HTTP server to legitimately have a 1952 .orig suffix. */ 1953 if (!stat (con->target, &st)) 1954 { 1955 bool eq_size; 1956 bool cor_val; 1957 /* Else, get it from the file. */ 1958 local_size = st.st_size; 1959 tml = st.st_mtime; 1960#ifdef WINDOWS 1961 /* Modification time granularity is 2 seconds for Windows, so 1962 increase local time by 1 second for later comparison. */ 1963 tml++; 1964#endif 1965 /* Compare file sizes only for servers that tell us correct 1966 values. Assume sizes being equal for servers that lie 1967 about file size. */ 1968 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT); 1969 eq_size = cor_val ? (local_size == f->size) : true; 1970 if (f->tstamp <= tml && eq_size) 1971 { 1972 /* Remote file is older, file sizes can be compared and 1973 are both equal. */ 1974 logprintf (LOG_VERBOSE, _("\ 1975Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target)); 1976 dlthis = false; 1977 } 1978 else if (eq_size) 1979 { 1980 /* Remote file is newer or sizes cannot be matched */ 1981 logprintf (LOG_VERBOSE, _("\ 1982Remote file is newer than local file %s -- retrieving.\n\n"), 1983 quote (con->target)); 1984 } 1985 else 1986 { 1987 /* Sizes do not match */ 1988 logprintf (LOG_VERBOSE, _("\ 1989The sizes do not match (local %s) -- retrieving.\n\n"), 1990 number_to_static_string (local_size)); 1991 } 1992 } 1993 } /* opt.timestamping && f->type == FT_PLAINFILE */ 1994 switch (f->type) 1995 { 1996 case FT_SYMLINK: 1997 /* If opt.retr_symlinks is defined, we treat symlinks as 1998 if they were normal files. There is currently no way 1999 to distinguish whether they might be directories, and 2000 follow them. */ 2001 if (!opt.retr_symlinks) 2002 { 2003#ifdef HAVE_SYMLINK 2004 if (!f->linkto) 2005 logputs (LOG_NOTQUIET, 2006 _("Invalid name of the symlink, skipping.\n")); 2007 else 2008 { 2009 struct_stat st; 2010 /* Check whether we already have the correct 2011 symbolic link. */ 2012 int rc = lstat (con->target, &st); 2013 if (rc == 0) 2014 { 2015 size_t len = strlen (f->linkto) + 1; 2016 if (S_ISLNK (st.st_mode)) 2017 { 2018 char *link_target = (char *)alloca (len); 2019 size_t n = readlink (con->target, link_target, len); 2020 if ((n == len - 1) 2021 && (memcmp (link_target, f->linkto, n) == 0)) 2022 { 2023 logprintf (LOG_VERBOSE, _("\ 2024Already have correct symlink %s -> %s\n\n"), 2025 quote (con->target), 2026 quote (f->linkto)); 2027 dlthis = false; 2028 break; 2029 } 2030 } 2031 } 2032 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"), 2033 quote (con->target), quote (f->linkto)); 2034 /* Unlink before creating symlink! */ 2035 unlink (con->target); 2036 if (symlink (f->linkto, con->target) == -1) 2037 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno)); 2038 logputs (LOG_VERBOSE, "\n"); 2039 } /* have f->linkto */ 2040#else /* not HAVE_SYMLINK */ 2041 logprintf (LOG_NOTQUIET, 2042 _("Symlinks not supported, skipping symlink %s.\n"), 2043 quote (con->target)); 2044#endif /* not HAVE_SYMLINK */ 2045 } 2046 else /* opt.retr_symlinks */ 2047 { 2048 if (dlthis) 2049 err = ftp_loop_internal (u, f, con, NULL); 2050 } /* opt.retr_symlinks */ 2051 break; 2052 case FT_DIRECTORY: 2053 if (!opt.recursive) 2054 logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"), 2055 quote (f->name)); 2056 break; 2057 case FT_PLAINFILE: 2058 /* Call the retrieve loop. */ 2059 if (dlthis) 2060 err = ftp_loop_internal (u, f, con, NULL); 2061 break; 2062 case FT_UNKNOWN: 2063 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"), 2064 quote (f->name)); 2065 break; 2066 } /* switch */ 2067 2068 2069 /* 2004-12-15 SMS. 2070 * Set permissions _before_ setting the times, as setting the 2071 * permissions changes the modified-time, at least on VMS. 2072 * Also, use the opt.output_document name here, too, as 2073 * appropriate. (Do the test once, and save the result.) 2074 */ 2075 2076 set_local_file (&actual_target, con->target); 2077 2078 /* If downloading a plain file, and the user requested it, then 2079 set valid (non-zero) permissions. */ 2080 if (dlthis && (actual_target != NULL) && 2081 (f->type == FT_PLAINFILE) && opt.preserve_perm) 2082 { 2083 if (f->perms) 2084 chmod (actual_target, f->perms); 2085 else 2086 DEBUGP (("Unrecognized permissions for %s.\n", actual_target)); 2087 } 2088 2089 /* Set the time-stamp information to the local file. Symlinks 2090 are not to be stamped because it sets the stamp on the 2091 original. :( */ 2092 if (actual_target != NULL) 2093 { 2094 if (opt.useservertimestamps 2095 && !(f->type == FT_SYMLINK && !opt.retr_symlinks) 2096 && f->tstamp != -1 2097 && dlthis 2098 && file_exists_p (con->target)) 2099 { 2100 touch (actual_target, f->tstamp); 2101 } 2102 else if (f->tstamp == -1) 2103 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), 2104 actual_target); 2105 } 2106 2107 xfree (con->target); 2108 con->target = old_target; 2109 2110 url_set_file (u, ofile); 2111 xfree (ofile); 2112 2113 /* Break on fatals. */ 2114 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR 2115 || err == WARC_ERR || err == WARC_TMP_FOPENERR 2116 || err == WARC_TMP_FWRITEERR) 2117 break; 2118 con->cmd &= ~ (DO_CWD | DO_LOGIN); 2119 f = f->next; 2120 } 2121 2122 /* We do not want to call ftp_retrieve_dirs here */ 2123 if (opt.recursive && 2124 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel)) 2125 err = ftp_retrieve_dirs (u, orig, con); 2126 else if (opt.recursive) 2127 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"), 2128 depth, opt.reclevel)); 2129 --depth; 2130 return err; 2131} 2132 2133/* Retrieve the directories given in a file list. This function works 2134 by simply going through the linked list and calling 2135 ftp_retrieve_glob on each directory entry. The function knows 2136 about excluded directories. */ 2137static uerr_t 2138ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con) 2139{ 2140 char *container = NULL; 2141 int container_size = 0; 2142 2143 for (; f; f = f->next) 2144 { 2145 int size; 2146 char *odir, *newdir; 2147 2148 if (opt.quota && total_downloaded_bytes > opt.quota) 2149 break; 2150 if (f->type != FT_DIRECTORY) 2151 continue; 2152 2153 /* Allocate u->dir off stack, but reallocate only if a larger 2154 string is needed. It's a pity there's no "realloca" for an 2155 item on the bottom of the stack. */ 2156 size = strlen (u->dir) + 1 + strlen (f->name) + 1; 2157 if (size > container_size) 2158 container = (char *)alloca (size); 2159 newdir = container; 2160 2161 odir = u->dir; 2162 if (*odir == '\0' 2163 || (*odir == '/' && *(odir + 1) == '\0')) 2164 /* If ODIR is empty or just "/", simply append f->name to 2165 ODIR. (In the former case, to preserve u->dir being 2166 relative; in the latter case, to avoid double slash.) */ 2167 sprintf (newdir, "%s%s", odir, f->name); 2168 else 2169 /* Else, use a separator. */ 2170 sprintf (newdir, "%s/%s", odir, f->name); 2171 2172 DEBUGP (("Composing new CWD relative to the initial directory.\n")); 2173 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n", 2174 odir, f->name, newdir)); 2175 if (!accdir (newdir)) 2176 { 2177 logprintf (LOG_VERBOSE, _("\ 2178Not descending to %s as it is excluded/not-included.\n"), 2179 quote (newdir)); 2180 continue; 2181 } 2182 2183 con->st &= ~DONE_CWD; 2184 2185 odir = xstrdup (u->dir); /* because url_set_dir will free 2186 u->dir. */ 2187 url_set_dir (u, newdir); 2188 ftp_retrieve_glob (u, con, GLOB_GETALL); 2189 url_set_dir (u, odir); 2190 xfree (odir); 2191 2192 /* Set the time-stamp? */ 2193 } 2194 2195 if (opt.quota && total_downloaded_bytes > opt.quota) 2196 return QUOTEXC; 2197 else 2198 return RETROK; 2199} 2200 2201/* Return true if S has a leading '/' or contains '../' */ 2202static bool 2203has_insecure_name_p (const char *s) 2204{ 2205 if (*s == '/') 2206 return true; 2207 2208 if (strstr (s, "../") != 0) 2209 return true; 2210 2211 return false; 2212} 2213 2214/* Test if the file node is invalid. This can occur due to malformed or 2215 * maliciously crafted listing files being returned by the server. 2216 * 2217 * Currently, this function only tests if there are multiple entries in the 2218 * listing file by the same name. However this function can be expanded as more 2219 * such illegal listing formats are discovered. */ 2220static bool 2221is_invalid_entry (struct fileinfo *f) 2222{ 2223 struct fileinfo *cur; 2224 cur = f; 2225 char *f_name = f->name; 2226 /* If the node we're currently checking has a duplicate later, we eliminate 2227 * the current node and leave the next one intact. */ 2228 while (cur->next) 2229 { 2230 cur = cur->next; 2231 if (strcmp(f_name, cur->name) == 0) 2232 return true; 2233 } 2234 return false; 2235} 2236 2237/* A near-top-level function to retrieve the files in a directory. 2238 The function calls ftp_get_listing, to get a linked list of files. 2239 Then it weeds out the file names that do not match the pattern. 2240 ftp_retrieve_list is called with this updated list as an argument. 2241 2242 If the argument ACTION is GLOB_GETONE, just download the file (but 2243 first get the listing, so that the time-stamp is heeded); if it's 2244 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole 2245 directory. */ 2246static uerr_t 2247ftp_retrieve_glob (struct url *u, ccon *con, int action) 2248{ 2249 struct fileinfo *f, *start; 2250 uerr_t res; 2251 2252 con->cmd |= LEAVE_PENDING; 2253 2254 res = ftp_get_listing (u, con, &start); 2255 if (res != RETROK) 2256 return res; 2257 /* First: weed out that do not conform the global rules given in 2258 opt.accepts and opt.rejects. */ 2259 if (opt.accepts || opt.rejects) 2260 { 2261 f = start; 2262 while (f) 2263 { 2264 if (f->type != FT_DIRECTORY && !acceptable (f->name)) 2265 { 2266 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"), 2267 quote (f->name)); 2268 f = delelement (f, &start); 2269 } 2270 else 2271 f = f->next; 2272 } 2273 } 2274 /* Remove all files with possible harmful names or invalid entries. */ 2275 f = start; 2276 while (f) 2277 { 2278 if (has_insecure_name_p (f->name) || is_invalid_entry (f)) 2279 { 2280 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"), 2281 quote (f->name)); 2282 f = delelement (f, &start); 2283 } 2284 else 2285 f = f->next; 2286 } 2287 /* Now weed out the files that do not match our globbing pattern. 2288 If we are dealing with a globbing pattern, that is. */ 2289 if (*u->file) 2290 { 2291 if (action == GLOB_GLOBALL) 2292 { 2293 int (*matcher) (const char *, const char *, int) 2294 = opt.ignore_case ? fnmatch_nocase : fnmatch; 2295 int matchres = 0; 2296 2297 f = start; 2298 while (f) 2299 { 2300 matchres = matcher (u->file, f->name, 0); 2301 if (matchres == -1) 2302 { 2303 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"), 2304 u->file, quotearg_style (escape_quoting_style, f->name), 2305 strerror (errno)); 2306 break; 2307 } 2308 if (matchres == FNM_NOMATCH) 2309 f = delelement (f, &start); /* delete the element from the list */ 2310 else 2311 f = f->next; /* leave the element in the list */ 2312 } 2313 if (matchres == -1) 2314 { 2315 freefileinfo (start); 2316 return RETRBADPATTERN; 2317 } 2318 } 2319 else if (action == GLOB_GETONE) 2320 { 2321#ifdef __VMS 2322 /* 2009-09-09 SMS. 2323 * Odd-ball compiler ("HP C V7.3-009 on OpenVMS Alpha V7.3-2") 2324 * bug causes spurious %CC-E-BADCONDIT complaint with this 2325 * "?:" statement. (Different linkage attributes for strcmp() 2326 * and strcasecmp().) Converting to "if" changes the 2327 * complaint to %CC-W-PTRMISMATCH on "cmp = strcmp;". Adding 2328 * the senseless type cast clears the complaint, and looks 2329 * harmless. 2330 */ 2331 int (*cmp) (const char *, const char *) 2332 = opt.ignore_case ? strcasecmp : (int (*)())strcmp; 2333#else /* def __VMS */ 2334 int (*cmp) (const char *, const char *) 2335 = opt.ignore_case ? strcasecmp : strcmp; 2336#endif /* def __VMS [else] */ 2337 f = start; 2338 while (f) 2339 { 2340 if (0 != cmp(u->file, f->name)) 2341 f = delelement (f, &start); 2342 else 2343 f = f->next; 2344 } 2345 } 2346 } 2347 if (start) 2348 { 2349 /* Just get everything. */ 2350 res = ftp_retrieve_list (u, start, con); 2351 } 2352 else 2353 { 2354 if (action == GLOB_GLOBALL) 2355 { 2356 /* No luck. */ 2357 /* #### This message SUCKS. We should see what was the 2358 reason that nothing was retrieved. */ 2359 logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"), 2360 quote (u->file)); 2361 } 2362 else if (action == GLOB_GETONE) /* GLOB_GETONE or GLOB_GETALL */ 2363 { 2364 /* Let's try retrieving it anyway. */ 2365 con->st |= ON_YOUR_OWN; 2366 res = ftp_loop_internal (u, NULL, con, NULL); 2367 return res; 2368 } 2369 2370 /* If action == GLOB_GETALL, and the file list is empty, there's 2371 no point in trying to download anything or in complaining about 2372 it. (An empty directory should not cause complaints.) 2373 */ 2374 } 2375 freefileinfo (start); 2376 if (opt.quota && total_downloaded_bytes > opt.quota) 2377 return QUOTEXC; 2378 else 2379 return res; 2380} 2381 2382/* The wrapper that calls an appropriate routine according to contents 2383 of URL. Inherently, its capabilities are limited on what can be 2384 encoded into a URL. */ 2385uerr_t 2386ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy, 2387 bool recursive, bool glob) 2388{ 2389 ccon con; /* FTP connection */ 2390 uerr_t res; 2391 2392 *dt = 0; 2393 2394 xzero (con); 2395 2396 con.csock = -1; 2397 con.st = ON_YOUR_OWN; 2398 con.rs = ST_UNIX; 2399 con.id = NULL; 2400 con.proxy = proxy; 2401 2402 /* If the file name is empty, the user probably wants a directory 2403 index. We'll provide one, properly HTML-ized. Unless 2404 opt.htmlify is 0, of course. :-) */ 2405 if (!*u->file && !recursive) 2406 { 2407 struct fileinfo *f; 2408 res = ftp_get_listing (u, &con, &f); 2409 2410 if (res == RETROK) 2411 { 2412 if (opt.htmlify && !opt.spider) 2413 { 2414 char *filename = (opt.output_document 2415 ? xstrdup (opt.output_document) 2416 : (con.target ? xstrdup (con.target) 2417 : url_file_name (u, NULL))); 2418 res = ftp_index (filename, u, f); 2419 if (res == FTPOK && opt.verbose) 2420 { 2421 if (!opt.output_document) 2422 { 2423 struct_stat st; 2424 wgint sz; 2425 if (stat (filename, &st) == 0) 2426 sz = st.st_size; 2427 else 2428 sz = -1; 2429 logprintf (LOG_NOTQUIET, 2430 _("Wrote HTML-ized index to %s [%s].\n"), 2431 quote (filename), number_to_static_string (sz)); 2432 } 2433 else 2434 logprintf (LOG_NOTQUIET, 2435 _("Wrote HTML-ized index to %s.\n"), 2436 quote (filename)); 2437 } 2438 xfree (filename); 2439 } 2440 freefileinfo (f); 2441 } 2442 } 2443 else 2444 { 2445 bool ispattern = false; 2446 if (glob) 2447 { 2448 /* Treat the URL as a pattern if the file name part of the 2449 URL path contains wildcards. (Don't check for u->file 2450 because it is unescaped and therefore doesn't leave users 2451 the option to escape literal '*' as %2A.) */ 2452 char *file_part = strrchr (u->path, '/'); 2453 if (!file_part) 2454 file_part = u->path; 2455 ispattern = has_wildcards_p (file_part); 2456 } 2457 if (ispattern || recursive || opt.timestamping || opt.preserve_perm) 2458 { 2459 /* ftp_retrieve_glob is a catch-all function that gets called 2460 if we need globbing, time-stamping, recursion or preserve 2461 permissions. Its third argument is just what we really need. */ 2462 res = ftp_retrieve_glob (u, &con, 2463 ispattern ? GLOB_GLOBALL : GLOB_GETONE); 2464 } 2465 else 2466 res = ftp_loop_internal (u, NULL, &con, local_file); 2467 } 2468 if (res == FTPOK) 2469 res = RETROK; 2470 if (res == RETROK) 2471 *dt |= RETROKF; 2472 /* If a connection was left, quench it. */ 2473 if (con.csock != -1) 2474 fd_close (con.csock); 2475 xfree_null (con.id); 2476 con.id = NULL; 2477 xfree_null (con.target); 2478 con.target = NULL; 2479 return res; 2480} 2481 2482/* Delete an element from the fileinfo linked list. Returns the 2483 address of the next element, or NULL if the list is exhausted. It 2484 can modify the start of the list. */ 2485static struct fileinfo * 2486delelement (struct fileinfo *f, struct fileinfo **start) 2487{ 2488 struct fileinfo *prev = f->prev; 2489 struct fileinfo *next = f->next; 2490 2491 xfree (f->name); 2492 xfree_null (f->linkto); 2493 xfree (f); 2494 2495 if (next) 2496 next->prev = prev; 2497 if (prev) 2498 prev->next = next; 2499 else 2500 *start = next; 2501 return next; 2502} 2503 2504/* Free the fileinfo linked list of files. */ 2505static void 2506freefileinfo (struct fileinfo *f) 2507{ 2508 while (f) 2509 { 2510 struct fileinfo *next = f->next; 2511 xfree (f->name); 2512 if (f->linkto) 2513 xfree (f->linkto); 2514 xfree (f); 2515 f = next; 2516 } 2517} 2518