Lines Matching refs:url
42 #include "url.h"
51 #include "html-url.h"
52 #include "css-url.h"
59 const char *url; /* the URL to download */
99 const char *url, const char *referer, int depth,
104 qel->url = url;
116 quotearg_n_style (0, escape_quoting_style, url), depth));
120 DEBUGP (("[IRI Enqueuing %s with %s\n", quote_n (0, url),
136 const char **url, const char **referer, int *depth,
149 *url = qel->url;
158 quotearg_n_style (0, escape_quoting_style, qel->url), qel->depth));
166 static bool download_child_p (const struct urlpos *, struct url *, int,
167 struct url *, struct hash_table *, struct iri *);
168 static bool descend_redirect_p (const char *, struct url *, int,
169 struct url *, struct hash_table *, struct iri *);
194 retrieve_tree (struct url *start_url_parsed, struct iri *pi)
223 /* Enqueue the starting URL. Use start_url_parsed->url rather than
225 url_enqueue (queue, i, xstrdup (start_url_parsed->url), NULL, 0, true,
227 string_set_add (blacklist, start_url_parsed->url);
232 char *url, *referer, *file = NULL;
246 (const char **)&url, (const char **)&referer,
258 if (dl_url_file_map && hash_table_contains (dl_url_file_map, url))
260 file = xstrdup (hash_table_get (dl_url_file_map, url));
263 url, file));
285 struct url *url_parsed = url_parse (url, &url_err, i, true);
287 status = retrieve_url (url_parsed, url, &file, &redirected, referer,
321 string_set_add (blacklist, url);
324 xfree (url);
325 url = redirected;
329 xfree (url);
330 url = xstrdup (url_parsed->url);
337 visited_url (url, referer);
373 = is_css ? get_urls_css_file (file, url) :
374 get_urls_html (file, url, &meta_disallow_follow, i);
385 struct url *url_parsed = url_parse (url, NULL, i, true);
387 char *referer_url = url;
407 url_enqueue (queue, ci, xstrdup (child->url->url),
414 string_set_add (blacklist, child->url->url);
449 xfree (url);
491 download_child_p (const struct urlpos *upos, struct url *parent, int depth,
492 struct url *start_url_parsed, struct hash_table *blacklist,
495 struct url *u = upos->url;
496 const char *url = u->url;
499 DEBUGP (("Deciding whether to enqueue \"%s\".\n", url));
501 if (string_set_contains (blacklist, url))
506 DEBUGP (("download_child_p: parent->url is: %s\n", quote (parent->url)));
507 visited_url (url, referrer);
588 DEBUGP (("%s (%s) is excluded/not-included.\n", url, u->dir));
613 url, u->file));
634 if (res_retrieve_file (url, &rfile, iri))
664 DEBUGP (("Not following %s because robots.txt forbids it.\n", url));
665 string_set_add (blacklist, url);
688 descend_redirect_p (const char *redirected, struct url *orig_parsed, int depth,
689 struct url *start_url_parsed, struct hash_table *blacklist,
692 struct url *new_parsed;
702 upos->url = new_parsed;