1/* Command line parsing. 2 Copyright (C) 1996-2014 Free Software Foundation, Inc. 3 4This file is part of GNU Wget. 5 6GNU Wget is free software; you can redistribute it and/or modify 7it under the terms of the GNU General Public License as published by 8the Free Software Foundation; either version 3 of the License, or 9(at your option) any later version. 10 11GNU Wget is distributed in the hope that it will be useful, 12but WITHOUT ANY WARRANTY; without even the implied warranty of 13MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14GNU General Public License for more details. 15 16You should have received a copy of the GNU General Public License 17along with Wget. If not, see <http://www.gnu.org/licenses/>. 18 19Additional permission under GNU GPL version 3 section 7 20 21If you modify this program, or any covered work, by linking or 22combining it with the OpenSSL project's OpenSSL library (or a 23modified version of that library), containing parts covered by the 24terms of the OpenSSL or SSLeay licenses, the Free Software Foundation 25grants you additional permission to convey the resulting work. 26Corresponding Source for a non-source form of such a combination 27shall include the source code for the parts of OpenSSL used as well 28as that of the covered work. */ 29 30#include "wget.h" 31 32#include <stdio.h> 33#include <stdlib.h> 34#include <unistd.h> 35#include <string.h> 36#include <signal.h> 37#ifdef ENABLE_NLS 38# include <locale.h> 39#endif 40#include <assert.h> 41#include <errno.h> 42#include <time.h> 43 44#include "exits.h" 45#include "utils.h" 46#include "init.h" 47#include "retr.h" 48#include "recur.h" 49#include "host.h" 50#include "url.h" 51#include "progress.h" /* for progress_handle_sigwinch */ 52#include "convert.h" 53#include "spider.h" 54#include "http.h" /* for save_cookies */ 55#include "ptimer.h" 56#include "warc.h" 57#include <getopt.h> 58#include <getpass.h> 59#include <quote.h> 60 61#ifdef WINDOWS 62# include <io.h> 63# include <fcntl.h> 64#endif 65 66#ifdef __VMS 67# include "vms.h" 68#endif /* __VMS */ 69 70#ifndef PATH_SEPARATOR 71# define PATH_SEPARATOR '/' 72#endif 73 74#ifndef ENABLE_IRI 75struct iri dummy_iri; 76#endif 77 78struct options opt; 79 80/* defined in version.c */ 81extern char *version_string; 82extern char *compilation_string; 83extern char *system_getrc; 84extern char *link_string; 85/* defined in build_info.c */ 86extern const char *compiled_features[]; 87/* Used for --version output in print_version */ 88#define MAX_CHARS_PER_LINE 72 89#define TABULATION 4 90 91#if defined(SIGHUP) || defined(SIGUSR1) 92static void redirect_output_signal (int); 93#endif 94 95const char *exec_name; 96 97/* Number of successfully downloaded URLs */ 98int numurls = 0; 99 100#ifndef TESTING 101/* Initialize I18N/L10N. That amounts to invoking setlocale, and 102 setting up gettext's message catalog using bindtextdomain and 103 textdomain. Does nothing if NLS is disabled or missing. */ 104 105static void 106i18n_initialize (void) 107{ 108 /* ENABLE_NLS implies existence of functions invoked here. */ 109#ifdef ENABLE_NLS 110 /* Set the current locale. */ 111 setlocale (LC_ALL, ""); 112 /* Set the text message domain. */ 113 bindtextdomain ("wget", LOCALEDIR); 114 textdomain ("wget"); 115#endif /* ENABLE_NLS */ 116} 117 118/* Definition of command-line options. */ 119 120static void print_help (void); 121static void print_version (void); 122 123#ifdef HAVE_SSL 124# define IF_SSL(x) x 125#else 126# define IF_SSL(x) NULL 127#endif 128 129struct cmdline_option { 130 const char *long_name; 131 char short_name; 132 enum { 133 OPT_VALUE, 134 OPT_BOOLEAN, 135 OPT_FUNCALL, 136 /* Non-standard options that have to be handled specially in 137 main(). */ 138 OPT__APPEND_OUTPUT, 139 OPT__CLOBBER, 140 OPT__DONT_REMOVE_LISTING, 141 OPT__EXECUTE, 142 OPT__NO, 143 OPT__PARENT 144 } type; 145 const void *data; /* for standard options */ 146 int argtype; /* for non-standard options */ 147}; 148 149static struct cmdline_option option_data[] = 150 { 151 { "accept", 'A', OPT_VALUE, "accept", -1 }, 152 { "accept-regex", 0, OPT_VALUE, "acceptregex", -1 }, 153 { "adjust-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 }, 154 { "append-output", 'a', OPT__APPEND_OUTPUT, NULL, required_argument }, 155 { "ask-password", 0, OPT_BOOLEAN, "askpassword", -1 }, 156 { "auth-no-challenge", 0, OPT_BOOLEAN, "authnochallenge", -1 }, 157 { "background", 'b', OPT_BOOLEAN, "background", -1 }, 158 { "backup-converted", 'K', OPT_BOOLEAN, "backupconverted", -1 }, 159 { "backups", 0, OPT_BOOLEAN, "backups", -1 }, 160 { "base", 'B', OPT_VALUE, "base", -1 }, 161 { "bind-address", 0, OPT_VALUE, "bindaddress", -1 }, 162 { "body-data", 0, OPT_VALUE, "bodydata", -1 }, 163 { "body-file", 0, OPT_VALUE, "bodyfile", -1 }, 164 { IF_SSL ("ca-certificate"), 0, OPT_VALUE, "cacertificate", -1 }, 165 { IF_SSL ("ca-directory"), 0, OPT_VALUE, "cadirectory", -1 }, 166 { "cache", 0, OPT_BOOLEAN, "cache", -1 }, 167 { IF_SSL ("certificate"), 0, OPT_VALUE, "certificate", -1 }, 168 { IF_SSL ("certificate-type"), 0, OPT_VALUE, "certificatetype", -1 }, 169 { IF_SSL ("check-certificate"), 0, OPT_BOOLEAN, "checkcertificate", -1 }, 170 { "clobber", 0, OPT__CLOBBER, NULL, optional_argument }, 171 { "config", 0, OPT_VALUE, "chooseconfig", -1 }, 172 { "connect-timeout", 0, OPT_VALUE, "connecttimeout", -1 }, 173 { "continue", 'c', OPT_BOOLEAN, "continue", -1 }, 174 { "convert-links", 'k', OPT_BOOLEAN, "convertlinks", -1 }, 175 { "content-disposition", 0, OPT_BOOLEAN, "contentdisposition", -1 }, 176 { "content-on-error", 0, OPT_BOOLEAN, "contentonerror", -1 }, 177 { "cookies", 0, OPT_BOOLEAN, "cookies", -1 }, 178 { "cut-dirs", 0, OPT_VALUE, "cutdirs", -1 }, 179 { "debug", 'd', OPT_BOOLEAN, "debug", -1 }, 180 { "default-page", 0, OPT_VALUE, "defaultpage", -1 }, 181 { "delete-after", 0, OPT_BOOLEAN, "deleteafter", -1 }, 182 { "directories", 0, OPT_BOOLEAN, "dirstruct", -1 }, 183 { "directory-prefix", 'P', OPT_VALUE, "dirprefix", -1 }, 184 { "dns-cache", 0, OPT_BOOLEAN, "dnscache", -1 }, 185 { "dns-timeout", 0, OPT_VALUE, "dnstimeout", -1 }, 186 { "domains", 'D', OPT_VALUE, "domains", -1 }, 187 { "dont-remove-listing", 0, OPT__DONT_REMOVE_LISTING, NULL, no_argument }, 188 { "dot-style", 0, OPT_VALUE, "dotstyle", -1 }, /* deprecated */ 189 { "egd-file", 0, OPT_VALUE, "egdfile", -1 }, 190 { "exclude-directories", 'X', OPT_VALUE, "excludedirectories", -1 }, 191 { "exclude-domains", 0, OPT_VALUE, "excludedomains", -1 }, 192 { "execute", 'e', OPT__EXECUTE, NULL, required_argument }, 193 { "follow-ftp", 0, OPT_BOOLEAN, "followftp", -1 }, 194 { "follow-tags", 0, OPT_VALUE, "followtags", -1 }, 195 { "force-directories", 'x', OPT_BOOLEAN, "dirstruct", -1 }, 196 { "force-html", 'F', OPT_BOOLEAN, "forcehtml", -1 }, 197 { "ftp-password", 0, OPT_VALUE, "ftppassword", -1 }, 198#ifdef __VMS 199 { "ftp-stmlf", 0, OPT_BOOLEAN, "ftpstmlf", -1 }, 200#endif /* def __VMS */ 201 { "ftp-user", 0, OPT_VALUE, "ftpuser", -1 }, 202 { "glob", 0, OPT_BOOLEAN, "glob", -1 }, 203 { "header", 0, OPT_VALUE, "header", -1 }, 204 { "help", 'h', OPT_FUNCALL, (void *)print_help, no_argument }, 205 { "host-directories", 0, OPT_BOOLEAN, "addhostdir", -1 }, 206 { "html-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 }, /* deprecated */ 207 { "htmlify", 0, OPT_BOOLEAN, "htmlify", -1 }, 208 { "http-keep-alive", 0, OPT_BOOLEAN, "httpkeepalive", -1 }, 209 { "http-passwd", 0, OPT_VALUE, "httppassword", -1 }, /* deprecated */ 210 { "http-password", 0, OPT_VALUE, "httppassword", -1 }, 211 { "http-user", 0, OPT_VALUE, "httpuser", -1 }, 212 { IF_SSL ("https-only"), 0, OPT_BOOLEAN, "httpsonly", -1 }, 213 { "ignore-case", 0, OPT_BOOLEAN, "ignorecase", -1 }, 214 { "ignore-length", 0, OPT_BOOLEAN, "ignorelength", -1 }, 215 { "ignore-tags", 0, OPT_VALUE, "ignoretags", -1 }, 216 { "include-directories", 'I', OPT_VALUE, "includedirectories", -1 }, 217#ifdef ENABLE_IPV6 218 { "inet4-only", '4', OPT_BOOLEAN, "inet4only", -1 }, 219 { "inet6-only", '6', OPT_BOOLEAN, "inet6only", -1 }, 220#endif 221 { "input-file", 'i', OPT_VALUE, "input", -1 }, 222 { "iri", 0, OPT_BOOLEAN, "iri", -1 }, 223 { "keep-session-cookies", 0, OPT_BOOLEAN, "keepsessioncookies", -1 }, 224 { "level", 'l', OPT_VALUE, "reclevel", -1 }, 225 { "limit-rate", 0, OPT_VALUE, "limitrate", -1 }, 226 { "load-cookies", 0, OPT_VALUE, "loadcookies", -1 }, 227 { "local-encoding", 0, OPT_VALUE, "localencoding", -1 }, 228 { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 }, 229 { "method", 0, OPT_VALUE, "method", -1 }, 230 { "mirror", 'm', OPT_BOOLEAN, "mirror", -1 }, 231 { "no", 'n', OPT__NO, NULL, required_argument }, 232 { "no-clobber", 0, OPT_BOOLEAN, "noclobber", -1 }, 233 { "no-config", 0, OPT_BOOLEAN, "noconfig", -1}, 234 { "no-parent", 0, OPT_BOOLEAN, "noparent", -1 }, 235 { "output-document", 'O', OPT_VALUE, "outputdocument", -1 }, 236 { "output-file", 'o', OPT_VALUE, "logfile", -1 }, 237 { "page-requisites", 'p', OPT_BOOLEAN, "pagerequisites", -1 }, 238 { "parent", 0, OPT__PARENT, NULL, optional_argument }, 239 { "passive-ftp", 0, OPT_BOOLEAN, "passiveftp", -1 }, 240 { "password", 0, OPT_VALUE, "password", -1 }, 241 { "post-data", 0, OPT_VALUE, "postdata", -1 }, 242 { "post-file", 0, OPT_VALUE, "postfile", -1 }, 243 { "prefer-family", 0, OPT_VALUE, "preferfamily", -1 }, 244 { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 }, 245 { IF_SSL ("private-key"), 0, OPT_VALUE, "privatekey", -1 }, 246 { IF_SSL ("private-key-type"), 0, OPT_VALUE, "privatekeytype", -1 }, 247 { "progress", 0, OPT_VALUE, "progress", -1 }, 248 { "show-progress", 0, OPT_BOOLEAN, "showprogress", -1 }, 249 { "protocol-directories", 0, OPT_BOOLEAN, "protocoldirectories", -1 }, 250 { "proxy", 0, OPT_BOOLEAN, "useproxy", -1 }, 251 { "proxy__compat", 'Y', OPT_VALUE, "useproxy", -1 }, /* back-compatible */ 252 { "proxy-passwd", 0, OPT_VALUE, "proxypassword", -1 }, /* deprecated */ 253 { "proxy-password", 0, OPT_VALUE, "proxypassword", -1 }, 254 { "proxy-user", 0, OPT_VALUE, "proxyuser", -1 }, 255 { "quiet", 'q', OPT_BOOLEAN, "quiet", -1 }, 256 { "quota", 'Q', OPT_VALUE, "quota", -1 }, 257 { "random-file", 0, OPT_VALUE, "randomfile", -1 }, 258 { "random-wait", 0, OPT_BOOLEAN, "randomwait", -1 }, 259 { "read-timeout", 0, OPT_VALUE, "readtimeout", -1 }, 260 { "recursive", 'r', OPT_BOOLEAN, "recursive", -1 }, 261 { "referer", 0, OPT_VALUE, "referer", -1 }, 262 { "regex-type", 0, OPT_VALUE, "regextype", -1 }, 263 { "reject", 'R', OPT_VALUE, "reject", -1 }, 264 { "reject-regex", 0, OPT_VALUE, "rejectregex", -1 }, 265 { "relative", 'L', OPT_BOOLEAN, "relativeonly", -1 }, 266 { "remote-encoding", 0, OPT_VALUE, "remoteencoding", -1 }, 267 { "remove-listing", 0, OPT_BOOLEAN, "removelisting", -1 }, 268 { "report-speed", 0, OPT_BOOLEAN, "reportspeed", -1 }, 269 { "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 }, 270 { "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 }, 271 { "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 }, 272 { "save-cookies", 0, OPT_VALUE, "savecookies", -1 }, 273 { "save-headers", 0, OPT_BOOLEAN, "saveheaders", -1 }, 274 { IF_SSL ("secure-protocol"), 0, OPT_VALUE, "secureprotocol", -1 }, 275 { "server-response", 'S', OPT_BOOLEAN, "serverresponse", -1 }, 276 { "span-hosts", 'H', OPT_BOOLEAN, "spanhosts", -1 }, 277 { "spider", 0, OPT_BOOLEAN, "spider", -1 }, 278 { "start-pos", 0, OPT_VALUE, "startpos", -1 }, 279 { "strict-comments", 0, OPT_BOOLEAN, "strictcomments", -1 }, 280 { "timeout", 'T', OPT_VALUE, "timeout", -1 }, 281 { "timestamping", 'N', OPT_BOOLEAN, "timestamping", -1 }, 282 { "tries", 't', OPT_VALUE, "tries", -1 }, 283 { "unlink", 0, OPT_BOOLEAN, "unlink", -1 }, 284 { "trust-server-names", 0, OPT_BOOLEAN, "trustservernames", -1 }, 285 { "use-server-timestamps", 0, OPT_BOOLEAN, "useservertimestamps", -1 }, 286 { "user", 0, OPT_VALUE, "user", -1 }, 287 { "user-agent", 'U', OPT_VALUE, "useragent", -1 }, 288 { "verbose", 'v', OPT_BOOLEAN, "verbose", -1 }, 289 { "verbose", 0, OPT_BOOLEAN, "verbose", -1 }, 290 { "version", 'V', OPT_FUNCALL, (void *) print_version, no_argument }, 291 { "wait", 'w', OPT_VALUE, "wait", -1 }, 292 { "waitretry", 0, OPT_VALUE, "waitretry", -1 }, 293 { "warc-cdx", 0, OPT_BOOLEAN, "warccdx", -1 }, 294#ifdef HAVE_LIBZ 295 { "warc-compression", 0, OPT_BOOLEAN, "warccompression", -1 }, 296#endif 297 { "warc-dedup", 0, OPT_VALUE, "warccdxdedup", -1 }, 298 { "warc-digests", 0, OPT_BOOLEAN, "warcdigests", -1 }, 299 { "warc-file", 0, OPT_VALUE, "warcfile", -1 }, 300 { "warc-header", 0, OPT_VALUE, "warcheader", -1 }, 301 { "warc-keep-log", 0, OPT_BOOLEAN, "warckeeplog", -1 }, 302 { "warc-max-size", 0, OPT_VALUE, "warcmaxsize", -1 }, 303 { "warc-tempdir", 0, OPT_VALUE, "warctempdir", -1 }, 304#ifdef USE_WATT32 305 { "wdebug", 0, OPT_BOOLEAN, "wdebug", -1 }, 306#endif 307 }; 308 309#undef IF_SSL 310 311/* Return a string that contains S with "no-" prepended. The string 312 is NUL-terminated and allocated off static storage at Wget 313 startup. */ 314 315static char * 316no_prefix (const char *s) 317{ 318 static char buffer[2048]; 319 static char *p = buffer; 320 321 char *cp = p; 322 int size = 3 + strlen (s) + 1; /* "no-STRING\0" */ 323 if (p + size >= buffer + sizeof (buffer)) 324 abort (); 325 326 cp[0] = 'n', cp[1] = 'o', cp[2] = '-'; 327 strcpy (cp + 3, s); 328 p += size; 329 return cp; 330} 331 332/* The arguments that that main passes to getopt_long. */ 333static struct option long_options[2 * countof (option_data) + 1]; 334static char short_options[128]; 335 336/* Mapping between short option chars and option_data indices. */ 337static unsigned char optmap[96]; 338 339/* Marker for `--no-FOO' values in long_options. */ 340#define BOOLEAN_NEG_MARKER 1024 341 342/* Initialize the long_options array used by getopt_long from the data 343 in option_data. */ 344 345static void 346init_switches (void) 347{ 348 char *p = short_options; 349 size_t i, o = 0; 350 for (i = 0; i < countof (option_data); i++) 351 { 352 struct cmdline_option *cmdopt = &option_data[i]; 353 struct option *longopt; 354 355 if (!cmdopt->long_name) 356 /* The option is disabled. */ 357 continue; 358 359 longopt = &long_options[o++]; 360 longopt->name = cmdopt->long_name; 361 longopt->val = i; 362 if (cmdopt->short_name) 363 { 364 *p++ = cmdopt->short_name; 365 optmap[cmdopt->short_name - 32] = longopt - long_options; 366 } 367 switch (cmdopt->type) 368 { 369 case OPT_VALUE: 370 longopt->has_arg = required_argument; 371 if (cmdopt->short_name) 372 *p++ = ':'; 373 break; 374 case OPT_BOOLEAN: 375 /* Specify an optional argument for long options, so that 376 --option=off works the same as --no-option, for 377 compatibility with pre-1.10 Wget. However, don't specify 378 optional arguments short-option booleans because they 379 prevent combining of short options. */ 380 longopt->has_arg = optional_argument; 381 /* For Boolean options, add the "--no-FOO" variant, which is 382 identical to "--foo", except it has opposite meaning and 383 it doesn't allow an argument. */ 384 longopt = &long_options[o++]; 385 longopt->name = no_prefix (cmdopt->long_name); 386 longopt->has_arg = no_argument; 387 /* Mask the value so we'll be able to recognize that we're 388 dealing with the false value. */ 389 longopt->val = i | BOOLEAN_NEG_MARKER; 390 break; 391 default: 392 assert (cmdopt->argtype != -1); 393 longopt->has_arg = cmdopt->argtype; 394 if (cmdopt->short_name) 395 { 396 if (longopt->has_arg == required_argument) 397 *p++ = ':'; 398 /* Don't handle optional_argument */ 399 } 400 } 401 } 402 /* Terminate short_options. */ 403 *p = '\0'; 404 /* No need for xzero(long_options[o]) because its storage is static 405 and it will be zeroed by default. */ 406 assert (o <= countof (long_options)); 407} 408 409/* Print the usage message. */ 410static int 411print_usage (int error) 412{ 413 return fprintf (error ? stderr : stdout, 414 _("Usage: %s [OPTION]... [URL]...\n"), exec_name); 415} 416 417/* Print the help message, describing all the available options. If 418 you add an option, be sure to update this list. */ 419static void _Noreturn 420print_help (void) 421{ 422 /* We split the help text this way to ease translation of individual 423 entries. */ 424 static const char *help[] = { 425 "\n", 426 N_("\ 427Mandatory arguments to long options are mandatory for short options too.\n\n"), 428 N_("\ 429Startup:\n"), 430 N_("\ 431 -V, --version display the version of Wget and exit.\n"), 432 N_("\ 433 -h, --help print this help.\n"), 434 N_("\ 435 -b, --background go to background after startup.\n"), 436 N_("\ 437 -e, --execute=COMMAND execute a `.wgetrc'-style command.\n"), 438 "\n", 439 440 N_("\ 441Logging and input file:\n"), 442 N_("\ 443 -o, --output-file=FILE log messages to FILE.\n"), 444 N_("\ 445 -a, --append-output=FILE append messages to FILE.\n"), 446#ifdef ENABLE_DEBUG 447 N_("\ 448 -d, --debug print lots of debugging information.\n"), 449#endif 450#ifdef USE_WATT32 451 N_("\ 452 --wdebug print Watt-32 debug output.\n"), 453#endif 454 N_("\ 455 -q, --quiet quiet (no output).\n"), 456 N_("\ 457 -v, --verbose be verbose (this is the default).\n"), 458 N_("\ 459 -nv, --no-verbose turn off verboseness, without being quiet.\n"), 460 N_("\ 461 --report-speed=TYPE Output bandwidth as TYPE. TYPE can be bits.\n"), 462 N_("\ 463 -i, --input-file=FILE download URLs found in local or external FILE.\n"), 464 N_("\ 465 -F, --force-html treat input file as HTML.\n"), 466 N_("\ 467 -B, --base=URL resolves HTML input-file links (-i -F)\n\ 468 relative to URL.\n"), 469 N_("\ 470 --config=FILE Specify config file to use.\n"), 471 N_("\ 472 --no-config Do not read any config file.\n"), 473 "\n", 474 475 N_("\ 476Download:\n"), 477 N_("\ 478 -t, --tries=NUMBER set number of retries to NUMBER (0 unlimits).\n"), 479 N_("\ 480 --retry-connrefused retry even if connection is refused.\n"), 481 N_("\ 482 -O, --output-document=FILE write documents to FILE.\n"), 483 N_("\ 484 -nc, --no-clobber skip downloads that would download to\n\ 485 existing files (overwriting them).\n"), 486 N_("\ 487 -c, --continue resume getting a partially-downloaded file.\n"), 488 N_("\ 489 --start-pos=OFFSET start downloading from zero-based position OFFSET.\n"), 490 N_("\ 491 --progress=TYPE select progress gauge type.\n"), 492 N_("\ 493 --show-progress display the progress bar in any verbosity mode.\n"), 494 N_("\ 495 -N, --timestamping don't re-retrieve files unless newer than\n\ 496 local.\n"), 497 N_("\ 498 --no-use-server-timestamps don't set the local file's timestamp by\n\ 499 the one on the server.\n"), 500 N_("\ 501 -S, --server-response print server response.\n"), 502 N_("\ 503 --spider don't download anything.\n"), 504 N_("\ 505 -T, --timeout=SECONDS set all timeout values to SECONDS.\n"), 506 N_("\ 507 --dns-timeout=SECS set the DNS lookup timeout to SECS.\n"), 508 N_("\ 509 --connect-timeout=SECS set the connect timeout to SECS.\n"), 510 N_("\ 511 --read-timeout=SECS set the read timeout to SECS.\n"), 512 N_("\ 513 -w, --wait=SECONDS wait SECONDS between retrievals.\n"), 514 N_("\ 515 --waitretry=SECONDS wait 1..SECONDS between retries of a retrieval.\n"), 516 N_("\ 517 --random-wait wait from 0.5*WAIT...1.5*WAIT secs between retrievals.\n"), 518 N_("\ 519 --no-proxy explicitly turn off proxy.\n"), 520 N_("\ 521 -Q, --quota=NUMBER set retrieval quota to NUMBER.\n"), 522 N_("\ 523 --bind-address=ADDRESS bind to ADDRESS (hostname or IP) on local host.\n"), 524 N_("\ 525 --limit-rate=RATE limit download rate to RATE.\n"), 526 N_("\ 527 --no-dns-cache disable caching DNS lookups.\n"), 528 N_("\ 529 --restrict-file-names=OS restrict chars in file names to ones OS allows.\n"), 530 N_("\ 531 --ignore-case ignore case when matching files/directories.\n"), 532#ifdef ENABLE_IPV6 533 N_("\ 534 -4, --inet4-only connect only to IPv4 addresses.\n"), 535 N_("\ 536 -6, --inet6-only connect only to IPv6 addresses.\n"), 537 N_("\ 538 --prefer-family=FAMILY connect first to addresses of specified family,\n\ 539 one of IPv6, IPv4, or none.\n"), 540#endif 541 N_("\ 542 --user=USER set both ftp and http user to USER.\n"), 543 N_("\ 544 --password=PASS set both ftp and http password to PASS.\n"), 545 N_("\ 546 --ask-password prompt for passwords.\n"), 547 N_("\ 548 --no-iri turn off IRI support.\n"), 549 N_("\ 550 --local-encoding=ENC use ENC as the local encoding for IRIs.\n"), 551 N_("\ 552 --remote-encoding=ENC use ENC as the default remote encoding.\n"), 553 N_("\ 554 --unlink remove file before clobber.\n"), 555 "\n", 556 557 N_("\ 558Directories:\n"), 559 N_("\ 560 -nd, --no-directories don't create directories.\n"), 561 N_("\ 562 -x, --force-directories force creation of directories.\n"), 563 N_("\ 564 -nH, --no-host-directories don't create host directories.\n"), 565 N_("\ 566 --protocol-directories use protocol name in directories.\n"), 567 N_("\ 568 -P, --directory-prefix=PREFIX save files to PREFIX/...\n"), 569 N_("\ 570 --cut-dirs=NUMBER ignore NUMBER remote directory components.\n"), 571 "\n", 572 573 N_("\ 574HTTP options:\n"), 575 N_("\ 576 --http-user=USER set http user to USER.\n"), 577 N_("\ 578 --http-password=PASS set http password to PASS.\n"), 579 N_("\ 580 --no-cache disallow server-cached data.\n"), 581 N_ ("\ 582 --default-page=NAME Change the default page name (normally\n\ 583 this is `index.html'.).\n"), 584 N_("\ 585 -E, --adjust-extension save HTML/CSS documents with proper extensions.\n"), 586 N_("\ 587 --ignore-length ignore `Content-Length' header field.\n"), 588 N_("\ 589 --header=STRING insert STRING among the headers.\n"), 590 N_("\ 591 --max-redirect maximum redirections allowed per page.\n"), 592 N_("\ 593 --proxy-user=USER set USER as proxy username.\n"), 594 N_("\ 595 --proxy-password=PASS set PASS as proxy password.\n"), 596 N_("\ 597 --referer=URL include `Referer: URL' header in HTTP request.\n"), 598 N_("\ 599 --save-headers save the HTTP headers to file.\n"), 600 N_("\ 601 -U, --user-agent=AGENT identify as AGENT instead of Wget/VERSION.\n"), 602 N_("\ 603 --no-http-keep-alive disable HTTP keep-alive (persistent connections).\n"), 604 N_("\ 605 --no-cookies don't use cookies.\n"), 606 N_("\ 607 --load-cookies=FILE load cookies from FILE before session.\n"), 608 N_("\ 609 --save-cookies=FILE save cookies to FILE after session.\n"), 610 N_("\ 611 --keep-session-cookies load and save session (non-permanent) cookies.\n"), 612 N_("\ 613 --post-data=STRING use the POST method; send STRING as the data.\n"), 614 N_("\ 615 --post-file=FILE use the POST method; send contents of FILE.\n"), 616 N_("\ 617 --method=HTTPMethod use method \"HTTPMethod\" in the request.\n"), 618 N_("\ 619 --body-data=STRING Send STRING as data. --method MUST be set.\n"), 620 N_("\ 621 --body-file=FILE Send contents of FILE. --method MUST be set.\n"), 622 N_("\ 623 --content-disposition honor the Content-Disposition header when\n\ 624 choosing local file names (EXPERIMENTAL).\n"), 625 N_("\ 626 --content-on-error output the received content on server errors.\n"), 627 N_("\ 628 --auth-no-challenge send Basic HTTP authentication information\n\ 629 without first waiting for the server's\n\ 630 challenge.\n"), 631 "\n", 632 633#ifdef HAVE_SSL 634 N_("\ 635HTTPS (SSL/TLS) options:\n"), 636 N_("\ 637 --secure-protocol=PR choose secure protocol, one of auto, SSLv2,\n\ 638 SSLv3, TLSv1 and PFS.\n"), 639 N_("\ 640 --https-only only follow secure HTTPS links\n"), 641 N_("\ 642 --no-check-certificate don't validate the server's certificate.\n"), 643 N_("\ 644 --certificate=FILE client certificate file.\n"), 645 N_("\ 646 --certificate-type=TYPE client certificate type, PEM or DER.\n"), 647 N_("\ 648 --private-key=FILE private key file.\n"), 649 N_("\ 650 --private-key-type=TYPE private key type, PEM or DER.\n"), 651 N_("\ 652 --ca-certificate=FILE file with the bundle of CA's.\n"), 653 N_("\ 654 --ca-directory=DIR directory where hash list of CA's is stored.\n"), 655 N_("\ 656 --random-file=FILE file with random data for seeding the SSL PRNG.\n"), 657 N_("\ 658 --egd-file=FILE file naming the EGD socket with random data.\n"), 659 "\n", 660#endif /* HAVE_SSL */ 661 662 N_("\ 663FTP options:\n"), 664#ifdef __VMS 665 N_("\ 666 --ftp-stmlf Use Stream_LF format for all binary FTP files.\n"), 667#endif /* def __VMS */ 668 N_("\ 669 --ftp-user=USER set ftp user to USER.\n"), 670 N_("\ 671 --ftp-password=PASS set ftp password to PASS.\n"), 672 N_("\ 673 --no-remove-listing don't remove `.listing' files.\n"), 674 N_("\ 675 --no-glob turn off FTP file name globbing.\n"), 676 N_("\ 677 --no-passive-ftp disable the \"passive\" transfer mode.\n"), 678 N_("\ 679 --preserve-permissions preserve remote file permissions.\n"), 680 N_("\ 681 --retr-symlinks when recursing, get linked-to files (not dir).\n"), 682 "\n", 683 684 N_("\ 685WARC options:\n"), 686 N_("\ 687 --warc-file=FILENAME save request/response data to a .warc.gz file.\n"), 688 N_("\ 689 --warc-header=STRING insert STRING into the warcinfo record.\n"), 690 N_("\ 691 --warc-max-size=NUMBER set maximum size of WARC files to NUMBER.\n"), 692 N_("\ 693 --warc-cdx write CDX index files.\n"), 694 N_("\ 695 --warc-dedup=FILENAME do not store records listed in this CDX file.\n"), 696#ifdef HAVE_LIBZ 697 N_("\ 698 --no-warc-compression do not compress WARC files with GZIP.\n"), 699#endif 700 N_("\ 701 --no-warc-digests do not calculate SHA1 digests.\n"), 702 N_("\ 703 --no-warc-keep-log do not store the log file in a WARC record.\n"), 704 N_("\ 705 --warc-tempdir=DIRECTORY location for temporary files created by the\n\ 706 WARC writer.\n"), 707 "\n", 708 709 N_("\ 710Recursive download:\n"), 711 N_("\ 712 -r, --recursive specify recursive download.\n"), 713 N_("\ 714 -l, --level=NUMBER maximum recursion depth (inf or 0 for infinite).\n"), 715 N_("\ 716 --delete-after delete files locally after downloading them.\n"), 717 N_("\ 718 -k, --convert-links make links in downloaded HTML or CSS point to\n\ 719 local files.\n"), 720 N_("\ 721 --backups=N before writing file X, rotate up to N backup files.\n"), 722 723#ifdef __VMS 724 N_("\ 725 -K, --backup-converted before converting file X, back up as X_orig.\n"), 726#else /* def __VMS */ 727 N_("\ 728 -K, --backup-converted before converting file X, back up as X.orig.\n"), 729#endif /* def __VMS [else] */ 730 N_("\ 731 -m, --mirror shortcut for -N -r -l inf --no-remove-listing.\n"), 732 N_("\ 733 -p, --page-requisites get all images, etc. needed to display HTML page.\n"), 734 N_("\ 735 --strict-comments turn on strict (SGML) handling of HTML comments.\n"), 736 "\n", 737 738 N_("\ 739Recursive accept/reject:\n"), 740 N_("\ 741 -A, --accept=LIST comma-separated list of accepted extensions.\n"), 742 N_("\ 743 -R, --reject=LIST comma-separated list of rejected extensions.\n"), 744 N_("\ 745 --accept-regex=REGEX regex matching accepted URLs.\n"), 746 N_("\ 747 --reject-regex=REGEX regex matching rejected URLs.\n"), 748#ifdef HAVE_LIBPCRE 749 N_("\ 750 --regex-type=TYPE regex type (posix|pcre).\n"), 751#else 752 N_("\ 753 --regex-type=TYPE regex type (posix).\n"), 754#endif 755 N_("\ 756 -D, --domains=LIST comma-separated list of accepted domains.\n"), 757 N_("\ 758 --exclude-domains=LIST comma-separated list of rejected domains.\n"), 759 N_("\ 760 --follow-ftp follow FTP links from HTML documents.\n"), 761 N_("\ 762 --follow-tags=LIST comma-separated list of followed HTML tags.\n"), 763 N_("\ 764 --ignore-tags=LIST comma-separated list of ignored HTML tags.\n"), 765 N_("\ 766 -H, --span-hosts go to foreign hosts when recursive.\n"), 767 N_("\ 768 -L, --relative follow relative links only.\n"), 769 N_("\ 770 -I, --include-directories=LIST list of allowed directories.\n"), 771 N_("\ 772 --trust-server-names use the name specified by the redirection\n\ 773 url last component.\n"), 774 N_("\ 775 -X, --exclude-directories=LIST list of excluded directories.\n"), 776 N_("\ 777 -np, --no-parent don't ascend to the parent directory.\n"), 778 "\n", 779 N_("Mail bug reports and suggestions to <bug-wget@gnu.org>.\n") 780 }; 781 782 size_t i; 783 784 if (printf (_("GNU Wget %s, a non-interactive network retriever.\n"), 785 version_string) < 0) 786 exit (WGET_EXIT_IO_FAIL); 787 if (print_usage (0) < 0) 788 exit (WGET_EXIT_IO_FAIL); 789 790 for (i = 0; i < countof (help); i++) 791 if (fputs (_(help[i]), stdout) < 0) 792 exit (WGET_EXIT_IO_FAIL); 793 794 exit (WGET_EXIT_SUCCESS); 795} 796 797/* Return a human-readable printed representation of INTERVAL, 798 measured in seconds. */ 799 800static char * 801secs_to_human_time (double interval) 802{ 803 static char buf[32]; 804 int secs = (int) (interval + 0.5); 805 int hours, mins, days; 806 807 days = secs / 86400, secs %= 86400; 808 hours = secs / 3600, secs %= 3600; 809 mins = secs / 60, secs %= 60; 810 811 if (days) 812 sprintf (buf, "%dd %dh %dm %ds", days, hours, mins, secs); 813 else if (hours) 814 sprintf (buf, "%dh %dm %ds", hours, mins, secs); 815 else if (mins) 816 sprintf (buf, "%dm %ds", mins, secs); 817 else 818 sprintf (buf, "%ss", print_decimal (interval)); 819 820 return buf; 821} 822 823static char * 824prompt_for_password (void) 825{ 826 if (opt.user) 827 fprintf (stderr, _("Password for user %s: "), quote (opt.user)); 828 else 829 fprintf (stderr, _("Password: ")); 830 return getpass(""); 831} 832 833/* Function that prints the line argument while limiting it 834 to at most line_length. prefix is printed on the first line 835 and an appropriate number of spaces are added on subsequent 836 lines.*/ 837static int 838format_and_print_line (const char *prefix, const char *line, 839 int line_length) 840{ 841 int remaining_chars; 842 char *line_dup, *token; 843 844 assert (prefix != NULL); 845 assert (line != NULL); 846 assert (line_length > TABULATION); 847 848 line_dup = xstrdup (line); 849 850 if (printf ("%s", prefix) < 0) 851 return -1; 852 853 /* Wrap to new line after prefix. */ 854 remaining_chars = 0; 855 856 /* We break on spaces. */ 857 token = strtok (line_dup, " "); 858 while (token != NULL) 859 { 860 /* If however a token is much larger than the maximum 861 line length, all bets are off and we simply print the 862 token on the next line. */ 863 if (remaining_chars <= (int) strlen (token)) 864 { 865 if (printf ("\n%*c", TABULATION, ' ') < 0) 866 return -1; 867 remaining_chars = line_length - TABULATION; 868 } 869 if (printf ("%s ", token) < 0) 870 return -1; 871 remaining_chars -= strlen (token) + 1; /* account for " " */ 872 token = strtok (NULL, " "); 873 } 874 875 if (printf ("\n") < 0) 876 return -1; 877 878 xfree (line_dup); 879 return 0; 880} 881 882static void _Noreturn 883print_version (void) 884{ 885 const char *wgetrc_title = _("Wgetrc: "); 886 const char *locale_title = _("Locale: "); 887 const char *compile_title = _("Compile: "); 888 const char *link_title = _("Link: "); 889 char *env_wgetrc, *user_wgetrc; 890 int i; 891 892 if (printf (_("GNU Wget %s built on %s.\n\n"), version_string, OS_TYPE) < 0) 893 exit (WGET_EXIT_IO_FAIL); 894 895 for (i = 0; compiled_features[i] != NULL; ) 896 { 897 int line_length = MAX_CHARS_PER_LINE; 898 while ((line_length > 0) && (compiled_features[i] != NULL)) 899 { 900 if (printf ("%s ", compiled_features[i]) < 0) 901 exit (WGET_EXIT_IO_FAIL); 902 line_length -= strlen (compiled_features[i]) + 2; 903 i++; 904 } 905 if (printf ("\n") < 0) 906 exit (WGET_EXIT_IO_FAIL); 907 } 908 if (printf ("\n") < 0) 909 exit (WGET_EXIT_IO_FAIL); 910 911 /* Handle the case when $WGETRC is unset and $HOME/.wgetrc is 912 absent. */ 913 if (printf ("%s\n", wgetrc_title) < 0) 914 exit (WGET_EXIT_IO_FAIL); 915 916 env_wgetrc = wgetrc_env_file_name (); 917 if (env_wgetrc && *env_wgetrc) 918 { 919 if (printf (_(" %s (env)\n"), env_wgetrc) < 0) 920 exit (WGET_EXIT_IO_FAIL); 921 xfree (env_wgetrc); 922 } 923 user_wgetrc = wgetrc_user_file_name (); 924 if (user_wgetrc) 925 { 926 if (printf (_(" %s (user)\n"), user_wgetrc) < 0) 927 exit (WGET_EXIT_IO_FAIL); 928 xfree (user_wgetrc); 929 } 930#ifdef SYSTEM_WGETRC 931 if (printf (_(" %s (system)\n"), SYSTEM_WGETRC) < 0) 932 exit (WGET_EXIT_IO_FAIL); 933#endif 934 935#ifdef ENABLE_NLS 936 if (format_and_print_line (locale_title, 937 LOCALEDIR, 938 MAX_CHARS_PER_LINE) < 0) 939 exit (WGET_EXIT_IO_FAIL); 940#endif /* def ENABLE_NLS */ 941 942 if (compilation_string != NULL) 943 if (format_and_print_line (compile_title, 944 compilation_string, 945 MAX_CHARS_PER_LINE) < 0) 946 exit (WGET_EXIT_IO_FAIL); 947 948 if (link_string != NULL) 949 if (format_and_print_line (link_title, 950 link_string, 951 MAX_CHARS_PER_LINE) < 0) 952 exit (WGET_EXIT_IO_FAIL); 953 954 if (printf ("\n") < 0) 955 exit (WGET_EXIT_IO_FAIL); 956 957 /* TRANSLATORS: When available, an actual copyright character 958 (circle-c) should be used in preference to "(C)". */ 959 if (printf (_("\ 960Copyright (C) %s Free Software Foundation, Inc.\n"), "2014") < 0) 961 exit (WGET_EXIT_IO_FAIL); 962 if (fputs (_("\ 963License GPLv3+: GNU GPL version 3 or later\n\ 964<http://www.gnu.org/licenses/gpl.html>.\n\ 965This is free software: you are free to change and redistribute it.\n\ 966There is NO WARRANTY, to the extent permitted by law.\n"), stdout) < 0) 967 exit (WGET_EXIT_IO_FAIL); 968 /* TRANSLATORS: When available, please use the proper diacritics for 969 names such as this one. See en_US.po for reference. */ 970 if (fputs (_("\nOriginally written by Hrvoje Niksic <hniksic@xemacs.org>.\n"), 971 stdout) < 0) 972 exit (WGET_EXIT_IO_FAIL); 973 if (fputs (_("Please send bug reports and questions to <bug-wget@gnu.org>.\n"), 974 stdout) < 0) 975 exit (WGET_EXIT_IO_FAIL); 976 977 exit (WGET_EXIT_SUCCESS); 978} 979 980char *program_name; /* Needed by lib/error.c. */ 981char *program_argstring; /* Needed by wget_warc.c. */ 982 983int 984main (int argc, char **argv) 985{ 986 char **url, **t; 987 int i, ret, longindex; 988 int nurl; 989 bool append_to_log = false; 990 991 total_downloaded_bytes = 0; 992 993 program_name = argv[0]; 994 995 struct ptimer *timer = ptimer_new (); 996 double start_time = ptimer_measure (timer); 997 998 i18n_initialize (); 999 1000 /* Construct the name of the executable, without the directory part. */ 1001#ifdef __VMS 1002 /* On VMS, lose the "dev:[dir]" prefix and the ".EXE;nnn" suffix. */ 1003 exec_name = vms_basename (argv[0]); 1004#else /* def __VMS */ 1005 exec_name = strrchr (argv[0], PATH_SEPARATOR); 1006 if (!exec_name) 1007 exec_name = argv[0]; 1008 else 1009 ++exec_name; 1010#endif /* def __VMS [else] */ 1011 1012#ifdef WINDOWS 1013 /* Drop extension (typically .EXE) from executable filename. */ 1014 windows_main ((char **) &exec_name); 1015#endif 1016 1017 /* Construct the arguments string. */ 1018 int argstring_length = 1; 1019 for (i = 1; i < argc; i++) 1020 argstring_length += strlen (argv[i]) + 2 + 1; 1021 char *p = program_argstring = malloc (argstring_length * sizeof (char)); 1022 if (p == NULL) 1023 { 1024 fprintf (stderr, _("Memory allocation problem\n")); 1025 exit (WGET_EXIT_PARSE_ERROR); 1026 } 1027 for (i = 1; i < argc; i++) 1028 { 1029 *p++ = '"'; 1030 int arglen = strlen (argv[i]); 1031 memcpy (p, argv[i], arglen); 1032 p += arglen; 1033 *p++ = '"'; 1034 *p++ = ' '; 1035 } 1036 *p = '\0'; 1037 1038 /* Load the hard-coded defaults. */ 1039 defaults (); 1040 1041 init_switches (); 1042 1043 /* This separate getopt_long is needed to find the user config file 1044 option ("--config") and parse it before the other user options. */ 1045 longindex = -1; 1046 int retconf; 1047 bool use_userconfig = false; 1048 bool noconfig = false; 1049 1050 while ((retconf = getopt_long (argc, argv, 1051 short_options, long_options, &longindex)) != -1) 1052 { 1053 int confval; 1054 struct cmdline_option *config_opt; 1055 1056 /* There is no short option for "--config". */ 1057 if (longindex >= 0) 1058 { 1059 confval = long_options[longindex].val; 1060 config_opt = &option_data[confval & ~BOOLEAN_NEG_MARKER]; 1061 if (strcmp (config_opt->long_name, "no-config") == 0) 1062 { 1063 noconfig = true; 1064 break; 1065 } 1066 else if (strcmp (config_opt->long_name, "config") == 0) 1067 { 1068 bool userrc_ret = true; 1069 userrc_ret &= run_wgetrc (optarg); 1070 use_userconfig = true; 1071 if (userrc_ret) 1072 break; 1073 else 1074 { 1075 fprintf (stderr, _("Exiting due to error in %s\n"), optarg); 1076 exit (WGET_EXIT_PARSE_ERROR); 1077 } 1078 } 1079 } 1080 } 1081 1082 /* If the user did not specify a config, read the system wgetrc and ~/.wgetrc. */ 1083 if (noconfig == false && use_userconfig == false) 1084 initialize (); 1085 1086 opterr = 0; 1087 optind = 0; 1088 1089 longindex = -1; 1090 while ((ret = getopt_long (argc, argv, 1091 short_options, long_options, &longindex)) != -1) 1092 { 1093 int val; 1094 struct cmdline_option *cmdopt; 1095 1096 /* If LONGINDEX is unchanged, it means RET is referring a short 1097 option. */ 1098 if (longindex == -1) 1099 { 1100 if (ret == '?') 1101 { 1102 print_usage (1); 1103 fprintf (stderr, "\n"); 1104 fprintf (stderr, _("Try `%s --help' for more options.\n"), 1105 exec_name); 1106 exit (WGET_EXIT_PARSE_ERROR); 1107 } 1108 /* Find the short option character in the mapping. */ 1109 longindex = optmap[ret - 32]; 1110 } 1111 val = long_options[longindex].val; 1112 1113 /* Use the retrieved value to locate the option in the 1114 option_data array, and to see if we're dealing with the 1115 negated "--no-FOO" variant of the boolean option "--foo". */ 1116 cmdopt = &option_data[val & ~BOOLEAN_NEG_MARKER]; 1117 switch (cmdopt->type) 1118 { 1119 case OPT_VALUE: 1120 setoptval (cmdopt->data, optarg, cmdopt->long_name); 1121 break; 1122 case OPT_BOOLEAN: 1123 if (optarg) 1124 /* The user has specified a value -- use it. */ 1125 setoptval (cmdopt->data, optarg, cmdopt->long_name); 1126 else 1127 { 1128 /* NEG is true for `--no-FOO' style boolean options. */ 1129 bool neg = !!(val & BOOLEAN_NEG_MARKER); 1130 setoptval (cmdopt->data, neg ? "0" : "1", cmdopt->long_name); 1131 } 1132 break; 1133 case OPT_FUNCALL: 1134 { 1135 void (*func) (void) = (void (*) (void)) cmdopt->data; 1136 func (); 1137 } 1138 break; 1139 case OPT__APPEND_OUTPUT: 1140 setoptval ("logfile", optarg, cmdopt->long_name); 1141 append_to_log = true; 1142 break; 1143 case OPT__EXECUTE: 1144 run_command (optarg); 1145 break; 1146 case OPT__NO: 1147 { 1148 /* We support real --no-FOO flags now, but keep these 1149 short options for convenience and backward 1150 compatibility. */ 1151 for (p = optarg; p && *p; p++) 1152 switch (*p) 1153 { 1154 case 'v': 1155 setoptval ("verbose", "0", cmdopt->long_name); 1156 break; 1157 case 'H': 1158 setoptval ("addhostdir", "0", cmdopt->long_name); 1159 break; 1160 case 'd': 1161 setoptval ("dirstruct", "0", cmdopt->long_name); 1162 break; 1163 case 'c': 1164 setoptval ("noclobber", "1", cmdopt->long_name); 1165 break; 1166 case 'p': 1167 setoptval ("noparent", "1", cmdopt->long_name); 1168 break; 1169 default: 1170 fprintf (stderr, _("%s: illegal option -- `-n%c'\n"), 1171 exec_name, *p); 1172 print_usage (1); 1173 fprintf (stderr, "\n"); 1174 fprintf (stderr, _("Try `%s --help' for more options.\n"), 1175 exec_name); 1176 exit (WGET_EXIT_GENERIC_ERROR); 1177 } 1178 break; 1179 } 1180 case OPT__PARENT: 1181 case OPT__CLOBBER: 1182 { 1183 /* The wgetrc commands are named noparent and noclobber, 1184 so we must revert the meaning of the cmdline options 1185 before passing the value to setoptval. */ 1186 bool flag = true; 1187 if (optarg) 1188 flag = (*optarg == '1' || c_tolower (*optarg) == 'y' 1189 || (c_tolower (optarg[0]) == 'o' 1190 && c_tolower (optarg[1]) == 'n')); 1191 setoptval (cmdopt->type == OPT__PARENT ? "noparent" : "noclobber", 1192 flag ? "0" : "1", cmdopt->long_name); 1193 break; 1194 } 1195 case OPT__DONT_REMOVE_LISTING: 1196 setoptval ("removelisting", "0", cmdopt->long_name); 1197 break; 1198 } 1199 1200 longindex = -1; 1201 } 1202 1203 nurl = argc - optind; 1204 1205 /* If we do not have Debug support compiled in AND Wget is invoked with the 1206 * --debug switch, instead of failing, we silently turn it into a no-op. For 1207 * this no-op, we explicitly set opt.debug to false and hence none of the 1208 * Debug output messages will be printed. 1209 */ 1210#ifndef ENABLE_DEBUG 1211 if (opt.debug) 1212 { 1213 fprintf (stderr, _("Debugging support not compiled in. " 1214 "Ignoring --debug flag.\n")); 1215 opt.debug = false; 1216 } 1217#endif 1218 1219 /* All user options have now been processed, so it's now safe to do 1220 interoption dependency checks. */ 1221 1222 if (opt.noclobber && opt.convert_links) 1223 { 1224 fprintf (stderr, 1225 _("Both --no-clobber and --convert-links were specified," 1226 " only --convert-links will be used.\n")); 1227 opt.noclobber = false; 1228 } 1229 1230 if (opt.reclevel == 0) 1231 opt.reclevel = INFINITE_RECURSION; /* see recur.h for commentary */ 1232 1233 if (opt.spider || opt.delete_after) 1234 opt.no_dirstruct = true; 1235 1236 if (opt.page_requisites && !opt.recursive) 1237 { 1238 /* Don't set opt.recursive here because it would confuse the FTP 1239 code. Instead, call retrieve_tree below when either 1240 page_requisites or recursive is requested. */ 1241 opt.reclevel = 0; 1242 if (!opt.no_dirstruct) 1243 opt.dirstruct = 1; /* normally handled by cmd_spec_recursive() */ 1244 } 1245 1246 if (opt.verbose == -1) 1247 opt.verbose = !opt.quiet; 1248 1249 if (opt.verbose == 1) 1250 opt.show_progress = true; 1251 1252 /* Sanity checks. */ 1253 if (opt.verbose && opt.quiet) 1254 { 1255 fprintf (stderr, _("Can't be verbose and quiet at the same time.\n")); 1256 print_usage (1); 1257 exit (WGET_EXIT_GENERIC_ERROR); 1258 } 1259 if (opt.timestamping && opt.noclobber) 1260 { 1261 fprintf (stderr, _("\ 1262Can't timestamp and not clobber old files at the same time.\n")); 1263 print_usage (1); 1264 exit (WGET_EXIT_GENERIC_ERROR); 1265 } 1266#ifdef ENABLE_IPV6 1267 if (opt.ipv4_only && opt.ipv6_only) 1268 { 1269 fprintf (stderr, 1270 _("Cannot specify both --inet4-only and --inet6-only.\n")); 1271 print_usage (1); 1272 exit (WGET_EXIT_GENERIC_ERROR); 1273 } 1274#endif 1275 if (opt.output_document) 1276 { 1277 if (opt.convert_links 1278 && (nurl > 1 || opt.page_requisites || opt.recursive)) 1279 { 1280 fputs (_("\ 1281Cannot specify both -k and -O if multiple URLs are given, or in combination\n\ 1282with -p or -r. See the manual for details.\n\n"), stderr); 1283 print_usage (1); 1284 exit (WGET_EXIT_GENERIC_ERROR); 1285 } 1286 if (opt.page_requisites 1287 || opt.recursive) 1288 { 1289 logprintf (LOG_NOTQUIET, "%s", _("\ 1290WARNING: combining -O with -r or -p will mean that all downloaded content\n\ 1291will be placed in the single file you specified.\n\n")); 1292 } 1293 if (opt.timestamping) 1294 { 1295 logprintf (LOG_NOTQUIET, "%s", _("\ 1296WARNING: timestamping does nothing in combination with -O. See the manual\n\ 1297for details.\n\n")); 1298 opt.timestamping = false; 1299 } 1300 if (opt.noclobber && file_exists_p(opt.output_document)) 1301 { 1302 /* Check if output file exists; if it does, exit. */ 1303 logprintf (LOG_VERBOSE, 1304 _("File `%s' already there; not retrieving.\n"), 1305 opt.output_document); 1306 exit (WGET_EXIT_GENERIC_ERROR); 1307 } 1308 } 1309 1310 if (opt.warc_filename != 0) 1311 { 1312 if (opt.noclobber) 1313 { 1314 fprintf (stderr, 1315 _("WARC output does not work with --no-clobber, " 1316 "--no-clobber will be disabled.\n")); 1317 opt.noclobber = false; 1318 } 1319 if (opt.timestamping) 1320 { 1321 fprintf (stderr, 1322 _("WARC output does not work with timestamping, " 1323 "timestamping will be disabled.\n")); 1324 opt.timestamping = false; 1325 } 1326 if (opt.spider) 1327 { 1328 fprintf (stderr, 1329 _("WARC output does not work with --spider.\n")); 1330 exit (WGET_EXIT_GENERIC_ERROR); 1331 } 1332 if (opt.always_rest || opt.start_pos >= 0) 1333 { 1334 fprintf (stderr, 1335 _("WARC output does not work with --continue or" 1336 " --start-pos, they will be disabled.\n")); 1337 opt.always_rest = false; 1338 opt.start_pos = -1; 1339 } 1340 if (opt.warc_cdx_dedup_filename != 0 && !opt.warc_digests_enabled) 1341 { 1342 fprintf (stderr, 1343 _("Digests are disabled; WARC deduplication will " 1344 "not find duplicate records.\n")); 1345 } 1346 if (opt.warc_keep_log) 1347 { 1348 opt.progress_type = xstrdup ("dot"); 1349 } 1350 } 1351 1352 if (opt.ask_passwd && opt.passwd) 1353 { 1354 fprintf (stderr, 1355 _("Cannot specify both --ask-password and --password.\n")); 1356 print_usage (1); 1357 exit (WGET_EXIT_GENERIC_ERROR); 1358 } 1359 1360 if (opt.start_pos >= 0 && opt.always_rest) 1361 { 1362 fprintf (stderr, 1363 _("Specifying both --start-pos and --continue is not " 1364 "recommended; --continue will be disabled.\n")); 1365 opt.always_rest = false; 1366 } 1367 1368 if (!nurl && !opt.input_filename) 1369 { 1370 /* No URL specified. */ 1371 fprintf (stderr, _("%s: missing URL\n"), exec_name); 1372 print_usage (1); 1373 fprintf (stderr, "\n"); 1374 /* #### Something nicer should be printed here -- similar to the 1375 pre-1.5 `--help' page. */ 1376 fprintf (stderr, _("Try `%s --help' for more options.\n"), exec_name); 1377 exit (WGET_EXIT_GENERIC_ERROR); 1378 } 1379 1380 /* Compile the regular expressions. */ 1381 switch (opt.regex_type) 1382 { 1383#ifdef HAVE_LIBPCRE 1384 case regex_type_pcre: 1385 opt.regex_compile_fun = compile_pcre_regex; 1386 opt.regex_match_fun = match_pcre_regex; 1387 break; 1388#endif 1389 1390 case regex_type_posix: 1391 default: 1392 opt.regex_compile_fun = compile_posix_regex; 1393 opt.regex_match_fun = match_posix_regex; 1394 break; 1395 } 1396 if (opt.acceptregex_s) 1397 { 1398 opt.acceptregex = opt.regex_compile_fun (opt.acceptregex_s); 1399 if (!opt.acceptregex) 1400 exit (WGET_EXIT_GENERIC_ERROR); 1401 } 1402 if (opt.rejectregex_s) 1403 { 1404 opt.rejectregex = opt.regex_compile_fun (opt.rejectregex_s); 1405 if (!opt.rejectregex) 1406 exit (WGET_EXIT_GENERIC_ERROR); 1407 } 1408 if (opt.post_data || opt.post_file_name) 1409 { 1410 if (opt.post_data && opt.post_file_name) 1411 { 1412 fprintf (stderr, _("You cannot specify both --post-data and --post-file.\n")); 1413 exit (WGET_EXIT_GENERIC_ERROR); 1414 } 1415 else if (opt.method) 1416 { 1417 fprintf (stderr, _("You cannot use --post-data or --post-file along with --method. " 1418 "--method expects data through --body-data and --body-file options")); 1419 exit (WGET_EXIT_GENERIC_ERROR); 1420 } 1421 } 1422 if (opt.body_data || opt.body_file) 1423 { 1424 if (!opt.method) 1425 { 1426 fprintf (stderr, _("You must specify a method through --method=HTTPMethod " 1427 "to use with --body-data or --body-file.\n")); 1428 exit (WGET_EXIT_GENERIC_ERROR); 1429 } 1430 else if (opt.body_data && opt.body_file) 1431 { 1432 fprintf (stderr, _("You cannot specify both --body-data and --body-file.\n")); 1433 exit (WGET_EXIT_GENERIC_ERROR); 1434 } 1435 } 1436 1437 /* Set various options as required for opt.method. */ 1438 1439 /* When user specifies HEAD as the method, we do not wish to download any 1440 files. Hence, set wget to run in spider mode. */ 1441 if (opt.method && strcasecmp (opt.method, "HEAD") == 0) 1442 setoptval ("spider", "1", "spider"); 1443 1444 /* Convert post_data to body-data and post_file_name to body-file options. 1445 This is required so as to remove redundant code later on in gethttp(). 1446 The --post-data and --post-file options may also be removed in 1447 the future hence it makes sense to convert them to aliases for 1448 the more generic --method options. 1449 This MUST occur only after the sanity checks so as to prevent the 1450 user from setting both post and body options simultaneously. 1451 */ 1452 if (opt.post_data || opt.post_file_name) 1453 { 1454 setoptval ("method", "POST", "method"); 1455 if (opt.post_data) 1456 { 1457 setoptval ("bodydata", opt.post_data, "body-data"); 1458 opt.post_data = NULL; 1459 } 1460 else 1461 { 1462 setoptval ("bodyfile", opt.post_file_name, "body-file"); 1463 opt.post_file_name = NULL; 1464 } 1465 } 1466 1467#ifdef ENABLE_IRI 1468 if (opt.enable_iri) 1469 { 1470 if (opt.locale && !check_encoding_name (opt.locale)) 1471 opt.locale = NULL; 1472 1473 if (!opt.locale) 1474 opt.locale = find_locale (); 1475 1476 if (opt.encoding_remote && !check_encoding_name (opt.encoding_remote)) 1477 opt.encoding_remote = NULL; 1478 } 1479#else 1480 memset (&dummy_iri, 0, sizeof (dummy_iri)); 1481 if (opt.enable_iri || opt.locale || opt.encoding_remote) 1482 { 1483 /* sXXXav : be more specific... */ 1484 fprintf (stderr, _("This version does not have support for IRIs\n")); 1485 exit (WGET_EXIT_GENERIC_ERROR); 1486 } 1487#endif 1488 1489 if (opt.ask_passwd) 1490 { 1491 opt.passwd = prompt_for_password (); 1492 1493 if (opt.passwd == NULL || opt.passwd[0] == '\0') 1494 exit (WGET_EXIT_GENERIC_ERROR); 1495 } 1496 1497#ifdef USE_WATT32 1498 if (opt.wdebug) 1499 dbug_init(); 1500 sock_init(); 1501#else 1502 if (opt.background) 1503 fork_to_background (); 1504#endif 1505 1506 /* Initialize progress. Have to do this after the options are 1507 processed so we know where the log file is. */ 1508 if (opt.show_progress) 1509 set_progress_implementation (opt.progress_type); 1510 1511 /* Fill in the arguments. */ 1512 url = alloca_array (char *, nurl + 1); 1513 if (url == NULL) 1514 { 1515 fprintf (stderr, _("Memory allocation problem\n")); 1516 exit (WGET_EXIT_PARSE_ERROR); 1517 } 1518 for (i = 0; i < nurl; i++, optind++) 1519 { 1520 char *rewritten = rewrite_shorthand_url (argv[optind]); 1521 if (rewritten) 1522 url[i] = rewritten; 1523 else 1524 url[i] = xstrdup (argv[optind]); 1525 } 1526 url[i] = NULL; 1527 1528 /* Initialize logging. */ 1529 log_init (opt.lfilename, append_to_log); 1530 1531 /* Open WARC file. */ 1532 if (opt.warc_filename != 0) 1533 warc_init (); 1534 1535 DEBUGP (("DEBUG output created by Wget %s on %s.\n\n", 1536 version_string, OS_TYPE)); 1537 1538 /* Open the output filename if necessary. */ 1539 1540/* 2005-04-17 SMS. 1541 Note that having the output_stream ("-O") file opened here for an FTP 1542 URL rather than in getftp() (ftp.c) (and the http equivalent) rather 1543 limits the ability in VMS to open the file differently for ASCII 1544 versus binary FTP there. (Of course, doing it here allows a open 1545 failure to be detected immediately, without first connecting to the 1546 server.) 1547*/ 1548 if (opt.output_document) 1549 { 1550 if (HYPHENP (opt.output_document)) 1551 { 1552#ifdef WINDOWS 1553 _setmode (_fileno (stdout), _O_BINARY); 1554#endif 1555 output_stream = stdout; 1556 } 1557 else 1558 { 1559 struct_fstat st; 1560 1561#ifdef __VMS 1562/* Common fopen() optional arguments: 1563 sequential access only, access callback function. 1564*/ 1565# define FOPEN_OPT_ARGS , "fop=sqo", "acc", acc_cb, &open_id 1566 int open_id = 7; 1567#else /* def __VMS */ 1568# define FOPEN_OPT_ARGS 1569#endif /* def __VMS [else] */ 1570 1571 output_stream = fopen (opt.output_document, 1572 opt.always_rest ? "ab" : "wb" 1573 FOPEN_OPT_ARGS); 1574 if (output_stream == NULL) 1575 { 1576 perror (opt.output_document); 1577 exit (WGET_EXIT_GENERIC_ERROR); 1578 } 1579 if (fstat (fileno (output_stream), &st) == 0 && S_ISREG (st.st_mode)) 1580 output_stream_regular = true; 1581 } 1582 if (!output_stream_regular && opt.convert_links) 1583 { 1584 fprintf (stderr, _("-k can be used together with -O only if \ 1585outputting to a regular file.\n")); 1586 print_usage (1); 1587 exit (WGET_EXIT_GENERIC_ERROR); 1588 } 1589 } 1590 1591#ifdef __VMS 1592 /* Set global ODS5 flag according to the specified destination (if 1593 any), otherwise according to the current default device. 1594 */ 1595 if (output_stream == NULL) 1596 set_ods5_dest( "SYS$DISK"); 1597 else if (output_stream != stdout) 1598 set_ods5_dest( opt.output_document); 1599#endif /* def __VMS */ 1600 1601#ifdef WINDOWS 1602 ws_startup (); 1603#endif 1604 1605#ifdef SIGHUP 1606 /* Setup the signal handler to redirect output when hangup is 1607 received. */ 1608 if (signal(SIGHUP, SIG_IGN) != SIG_IGN) 1609 signal(SIGHUP, redirect_output_signal); 1610#endif 1611 /* ...and do the same for SIGUSR1. */ 1612#ifdef SIGUSR1 1613 signal (SIGUSR1, redirect_output_signal); 1614#endif 1615#ifdef SIGPIPE 1616 /* Writing to a closed socket normally signals SIGPIPE, and the 1617 process exits. What we want is to ignore SIGPIPE and just check 1618 for the return value of write(). */ 1619 signal (SIGPIPE, SIG_IGN); 1620#endif 1621#ifdef SIGWINCH 1622 signal (SIGWINCH, progress_handle_sigwinch); 1623#endif 1624 1625 /* Retrieve the URLs from argument list. */ 1626 for (t = url; *t; t++) 1627 { 1628 char *filename = NULL, *redirected_URL = NULL; 1629 int dt, url_err; 1630 /* Need to do a new struct iri every time, because 1631 * retrieve_url may modify it in some circumstances, 1632 * currently. */ 1633 struct iri *iri = iri_new (); 1634 struct url *url_parsed; 1635 1636 set_uri_encoding (iri, opt.locale, true); 1637 url_parsed = url_parse (*t, &url_err, iri, true); 1638 1639 if (!url_parsed) 1640 { 1641 char *error = url_error (*t, url_err); 1642 logprintf (LOG_NOTQUIET, "%s: %s.\n",*t, error); 1643 xfree (error); 1644 inform_exit_status (URLERROR); 1645 } 1646 else 1647 { 1648 if ((opt.recursive || opt.page_requisites) 1649 && (url_scheme (*t) != SCHEME_FTP || url_uses_proxy (url_parsed))) 1650 { 1651 int old_follow_ftp = opt.follow_ftp; 1652 1653 /* Turn opt.follow_ftp on in case of recursive FTP retrieval */ 1654 if (url_scheme (*t) == SCHEME_FTP) 1655 opt.follow_ftp = 1; 1656 1657 retrieve_tree (url_parsed, NULL); 1658 1659 opt.follow_ftp = old_follow_ftp; 1660 } 1661 else 1662 { 1663 retrieve_url (url_parsed, *t, &filename, &redirected_URL, NULL, 1664 &dt, opt.recursive, iri, true); 1665 } 1666 1667 if (opt.delete_after && filename != NULL && file_exists_p (filename)) 1668 { 1669 DEBUGP (("Removing file due to --delete-after in main():\n")); 1670 logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename); 1671 if (unlink (filename)) 1672 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); 1673 } 1674 xfree_null (redirected_URL); 1675 xfree_null (filename); 1676 url_free (url_parsed); 1677 } 1678 iri_free (iri); 1679 } 1680 1681 /* And then from the input file, if any. */ 1682 if (opt.input_filename) 1683 { 1684 int count; 1685 int status; 1686 status = retrieve_from_file (opt.input_filename, opt.force_html, &count); 1687 inform_exit_status (status); 1688 if (!count) 1689 logprintf (LOG_NOTQUIET, _("No URLs found in %s.\n"), 1690 opt.input_filename); 1691 } 1692 1693 /* Print broken links. */ 1694 if (opt.recursive && opt.spider) 1695 print_broken_links (); 1696 1697 /* Print the downloaded sum. */ 1698 if ((opt.recursive || opt.page_requisites 1699 || nurl > 1 1700 || (opt.input_filename && total_downloaded_bytes != 0)) 1701 && 1702 total_downloaded_bytes != 0) 1703 { 1704 double end_time = ptimer_measure (timer); 1705 ptimer_destroy (timer); 1706 1707 char *wall_time = xstrdup (secs_to_human_time (end_time - start_time)); 1708 char *download_time = xstrdup (secs_to_human_time (total_download_time)); 1709 logprintf (LOG_NOTQUIET, 1710 _("FINISHED --%s--\nTotal wall clock time: %s\n" 1711 "Downloaded: %d files, %s in %s (%s)\n"), 1712 datetime_str (time (NULL)), 1713 wall_time, 1714 numurls, 1715 human_readable (total_downloaded_bytes, 10, 1), 1716 download_time, 1717 retr_rate (total_downloaded_bytes, total_download_time)); 1718 xfree (wall_time); 1719 xfree (download_time); 1720 1721 /* Print quota warning, if exceeded. */ 1722 if (opt.quota && total_downloaded_bytes > opt.quota) 1723 logprintf (LOG_NOTQUIET, 1724 _("Download quota of %s EXCEEDED!\n"), 1725 human_readable (opt.quota, 10, 1)); 1726 } 1727 1728 if (opt.cookies_output) 1729 save_cookies (); 1730 1731 if (opt.convert_links && !opt.delete_after) 1732 convert_all_links (); 1733 1734 cleanup (); 1735 1736 exit (get_exit_status ()); 1737} 1738#endif /* TESTING */ 1739 1740#if defined(SIGHUP) || defined(SIGUSR1) 1741 1742/* So the signal_name check doesn't blow when only one is available. */ 1743#ifndef SIGHUP 1744# define SIGHUP -1 1745#endif 1746#ifndef SIGUSR1 1747# define SIGUSR1 -1 1748#endif 1749 1750/* Hangup signal handler. When wget receives SIGHUP or SIGUSR1, it 1751 will proceed operation as usual, trying to write into a log file. 1752 If that is impossible, the output will be turned off. */ 1753 1754static void 1755redirect_output_signal (int sig) 1756{ 1757 const char *signal_name = (sig == SIGHUP ? "SIGHUP" : 1758 (sig == SIGUSR1 ? "SIGUSR1" : 1759 "WTF?!")); 1760 log_request_redirect_output (signal_name); 1761 progress_schedule_redirect (); 1762 signal (sig, redirect_output_signal); 1763} 1764#endif 1765 1766/* 1767 * vim: et ts=2 sw=2 1768 */ 1769