1/*************************************************************************** 2 * _ _ ____ _ 3 * Project ___| | | | _ \| | 4 * / __| | | | |_) | | 5 * | (__| |_| | _ <| |___ 6 * \___|\___/|_| \_\_____| 7 * 8 * Copyright (C) 1998 - 2014, Daniel Stenberg, <daniel@haxx.se>, et al. 9 * 10 * This software is licensed as described in the file COPYING, which 11 * you should have received as part of this distribution. The terms 12 * are also available at http://curl.haxx.se/docs/copyright.html. 13 * 14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell 15 * copies of the Software, and permit persons to whom the Software is 16 * furnished to do so, under the terms of the COPYING file. 17 * 18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 19 * KIND, either express or implied. 20 * 21 ***************************************************************************/ 22/* Example application source code using the multi socket interface to 23 download many files at once. 24 25Written by Jeff Pohlmeyer 26 27Requires libevent version 2 and a (POSIX?) system that has mkfifo(). 28 29This is an adaptation of libcurl's "hipev.c" and libevent's "event-test.c" 30sample programs. 31 32When running, the program creates the named pipe "hiper.fifo" 33 34Whenever there is input into the fifo, the program reads the input as a list 35of URL's and creates some new easy handles to fetch each URL via the 36curl_multi "hiper" API. 37 38 39Thus, you can try a single URL: 40 % echo http://www.yahoo.com > hiper.fifo 41 42Or a whole bunch of them: 43 % cat my-url-list > hiper.fifo 44 45The fifo buffer is handled almost instantly, so you can even add more URL's 46while the previous requests are still being downloaded. 47 48Note: 49 For the sake of simplicity, URL length is limited to 1023 char's ! 50 51This is purely a demo app, all retrieved data is simply discarded by the write 52callback. 53 54*/ 55 56#include <stdio.h> 57#include <string.h> 58#include <stdlib.h> 59#include <sys/time.h> 60#include <time.h> 61#include <unistd.h> 62#include <sys/poll.h> 63#include <curl/curl.h> 64#include <event2/event.h> 65#include <fcntl.h> 66#include <sys/stat.h> 67#include <errno.h> 68 69 70#define MSG_OUT stdout /* Send info to stdout, change to stderr if you want */ 71 72 73/* Global information, common to all connections */ 74typedef struct _GlobalInfo 75{ 76 struct event_base *evbase; 77 struct event *fifo_event; 78 struct event *timer_event; 79 CURLM *multi; 80 int still_running; 81 FILE* input; 82} GlobalInfo; 83 84 85/* Information associated with a specific easy handle */ 86typedef struct _ConnInfo 87{ 88 CURL *easy; 89 char *url; 90 GlobalInfo *global; 91 char error[CURL_ERROR_SIZE]; 92} ConnInfo; 93 94 95/* Information associated with a specific socket */ 96typedef struct _SockInfo 97{ 98 curl_socket_t sockfd; 99 CURL *easy; 100 int action; 101 long timeout; 102 struct event *ev; 103 int evset; 104 GlobalInfo *global; 105} SockInfo; 106 107 108 109/* Update the event timer after curl_multi library calls */ 110static int multi_timer_cb(CURLM *multi, long timeout_ms, GlobalInfo *g) 111{ 112 struct timeval timeout; 113 (void)multi; /* unused */ 114 115 timeout.tv_sec = timeout_ms/1000; 116 timeout.tv_usec = (timeout_ms%1000)*1000; 117 fprintf(MSG_OUT, "multi_timer_cb: Setting timeout to %ld ms\n", timeout_ms); 118 evtimer_add(g->timer_event, &timeout); 119 return 0; 120} 121 122/* Die if we get a bad CURLMcode somewhere */ 123static void mcode_or_die(const char *where, CURLMcode code) 124{ 125 if ( CURLM_OK != code ) { 126 const char *s; 127 switch (code) { 128 case CURLM_BAD_HANDLE: s="CURLM_BAD_HANDLE"; break; 129 case CURLM_BAD_EASY_HANDLE: s="CURLM_BAD_EASY_HANDLE"; break; 130 case CURLM_OUT_OF_MEMORY: s="CURLM_OUT_OF_MEMORY"; break; 131 case CURLM_INTERNAL_ERROR: s="CURLM_INTERNAL_ERROR"; break; 132 case CURLM_UNKNOWN_OPTION: s="CURLM_UNKNOWN_OPTION"; break; 133 case CURLM_LAST: s="CURLM_LAST"; break; 134 default: s="CURLM_unknown"; 135 break; 136 case CURLM_BAD_SOCKET: s="CURLM_BAD_SOCKET"; 137 fprintf(MSG_OUT, "ERROR: %s returns %s\n", where, s); 138 /* ignore this error */ 139 return; 140 } 141 fprintf(MSG_OUT, "ERROR: %s returns %s\n", where, s); 142 exit(code); 143 } 144} 145 146 147 148/* Check for completed transfers, and remove their easy handles */ 149static void check_multi_info(GlobalInfo *g) 150{ 151 char *eff_url; 152 CURLMsg *msg; 153 int msgs_left; 154 ConnInfo *conn; 155 CURL *easy; 156 CURLcode res; 157 158 fprintf(MSG_OUT, "REMAINING: %d\n", g->still_running); 159 while ((msg = curl_multi_info_read(g->multi, &msgs_left))) { 160 if (msg->msg == CURLMSG_DONE) { 161 easy = msg->easy_handle; 162 res = msg->data.result; 163 curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn); 164 curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url); 165 fprintf(MSG_OUT, "DONE: %s => (%d) %s\n", eff_url, res, conn->error); 166 curl_multi_remove_handle(g->multi, easy); 167 free(conn->url); 168 curl_easy_cleanup(easy); 169 free(conn); 170 } 171 } 172} 173 174 175 176/* Called by libevent when we get action on a multi socket */ 177static void event_cb(int fd, short kind, void *userp) 178{ 179 GlobalInfo *g = (GlobalInfo*) userp; 180 CURLMcode rc; 181 182 int action = 183 (kind & EV_READ ? CURL_CSELECT_IN : 0) | 184 (kind & EV_WRITE ? CURL_CSELECT_OUT : 0); 185 186 rc = curl_multi_socket_action(g->multi, fd, action, &g->still_running); 187 mcode_or_die("event_cb: curl_multi_socket_action", rc); 188 189 check_multi_info(g); 190 if ( g->still_running <= 0 ) { 191 fprintf(MSG_OUT, "last transfer done, kill timeout\n"); 192 if (evtimer_pending(g->timer_event, NULL)) { 193 evtimer_del(g->timer_event); 194 } 195 } 196} 197 198 199 200/* Called by libevent when our timeout expires */ 201static void timer_cb(int fd, short kind, void *userp) 202{ 203 GlobalInfo *g = (GlobalInfo *)userp; 204 CURLMcode rc; 205 (void)fd; 206 (void)kind; 207 208 rc = curl_multi_socket_action(g->multi, 209 CURL_SOCKET_TIMEOUT, 0, &g->still_running); 210 mcode_or_die("timer_cb: curl_multi_socket_action", rc); 211 check_multi_info(g); 212} 213 214 215 216/* Clean up the SockInfo structure */ 217static void remsock(SockInfo *f) 218{ 219 if (f) { 220 if (f->evset) 221 event_free(f->ev); 222 free(f); 223 } 224} 225 226 227 228/* Assign information to a SockInfo structure */ 229static void setsock(SockInfo*f, curl_socket_t s, CURL*e, int act, GlobalInfo*g) 230{ 231 int kind = 232 (act&CURL_POLL_IN?EV_READ:0)|(act&CURL_POLL_OUT?EV_WRITE:0)|EV_PERSIST; 233 234 f->sockfd = s; 235 f->action = act; 236 f->easy = e; 237 if (f->evset) 238 event_free(f->ev); 239 f->ev = event_new(g->evbase, f->sockfd, kind, event_cb, g); 240 f->evset = 1; 241 event_add(f->ev, NULL); 242} 243 244 245 246/* Initialize a new SockInfo structure */ 247static void addsock(curl_socket_t s, CURL *easy, int action, GlobalInfo *g) 248{ 249 SockInfo *fdp = calloc(sizeof(SockInfo), 1); 250 251 fdp->global = g; 252 setsock(fdp, s, easy, action, g); 253 curl_multi_assign(g->multi, s, fdp); 254} 255 256/* CURLMOPT_SOCKETFUNCTION */ 257static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp) 258{ 259 GlobalInfo *g = (GlobalInfo*) cbp; 260 SockInfo *fdp = (SockInfo*) sockp; 261 const char *whatstr[]={ "none", "IN", "OUT", "INOUT", "REMOVE" }; 262 263 fprintf(MSG_OUT, 264 "socket callback: s=%d e=%p what=%s ", s, e, whatstr[what]); 265 if (what == CURL_POLL_REMOVE) { 266 fprintf(MSG_OUT, "\n"); 267 remsock(fdp); 268 } 269 else { 270 if (!fdp) { 271 fprintf(MSG_OUT, "Adding data: %s\n", whatstr[what]); 272 addsock(s, e, what, g); 273 } 274 else { 275 fprintf(MSG_OUT, 276 "Changing action from %s to %s\n", 277 whatstr[fdp->action], whatstr[what]); 278 setsock(fdp, s, e, what, g); 279 } 280 } 281 return 0; 282} 283 284 285 286/* CURLOPT_WRITEFUNCTION */ 287static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *data) 288{ 289 size_t realsize = size * nmemb; 290 ConnInfo *conn = (ConnInfo*) data; 291 (void)ptr; 292 (void)conn; 293 return realsize; 294} 295 296 297/* CURLOPT_PROGRESSFUNCTION */ 298static int prog_cb (void *p, double dltotal, double dlnow, double ult, 299 double uln) 300{ 301 ConnInfo *conn = (ConnInfo *)p; 302 (void)ult; 303 (void)uln; 304 305 fprintf(MSG_OUT, "Progress: %s (%g/%g)\n", conn->url, dlnow, dltotal); 306 return 0; 307} 308 309 310/* Create a new easy handle, and add it to the global curl_multi */ 311static void new_conn(char *url, GlobalInfo *g ) 312{ 313 ConnInfo *conn; 314 CURLMcode rc; 315 316 conn = calloc(1, sizeof(ConnInfo)); 317 memset(conn, 0, sizeof(ConnInfo)); 318 conn->error[0]='\0'; 319 320 conn->easy = curl_easy_init(); 321 if (!conn->easy) { 322 fprintf(MSG_OUT, "curl_easy_init() failed, exiting!\n"); 323 exit(2); 324 } 325 conn->global = g; 326 conn->url = strdup(url); 327 curl_easy_setopt(conn->easy, CURLOPT_URL, conn->url); 328 curl_easy_setopt(conn->easy, CURLOPT_WRITEFUNCTION, write_cb); 329 curl_easy_setopt(conn->easy, CURLOPT_WRITEDATA, &conn); 330 curl_easy_setopt(conn->easy, CURLOPT_VERBOSE, 1L); 331 curl_easy_setopt(conn->easy, CURLOPT_ERRORBUFFER, conn->error); 332 curl_easy_setopt(conn->easy, CURLOPT_PRIVATE, conn); 333 curl_easy_setopt(conn->easy, CURLOPT_NOPROGRESS, 0L); 334 curl_easy_setopt(conn->easy, CURLOPT_PROGRESSFUNCTION, prog_cb); 335 curl_easy_setopt(conn->easy, CURLOPT_PROGRESSDATA, conn); 336 fprintf(MSG_OUT, 337 "Adding easy %p to multi %p (%s)\n", conn->easy, g->multi, url); 338 rc = curl_multi_add_handle(g->multi, conn->easy); 339 mcode_or_die("new_conn: curl_multi_add_handle", rc); 340 341 /* note that the add_handle() will set a time-out to trigger very soon so 342 that the necessary socket_action() call will be called by this app */ 343} 344 345/* This gets called whenever data is received from the fifo */ 346static void fifo_cb(int fd, short event, void *arg) 347{ 348 char s[1024]; 349 long int rv=0; 350 int n=0; 351 GlobalInfo *g = (GlobalInfo *)arg; 352 (void)fd; /* unused */ 353 (void)event; /* unused */ 354 355 do { 356 s[0]='\0'; 357 rv=fscanf(g->input, "%1023s%n", s, &n); 358 s[n]='\0'; 359 if ( n && s[0] ) { 360 new_conn(s,arg); /* if we read a URL, go get it! */ 361 } else break; 362 } while ( rv != EOF); 363} 364 365/* Create a named pipe and tell libevent to monitor it */ 366static const char *fifo = "hiper.fifo"; 367static int init_fifo (GlobalInfo *g) 368{ 369 struct stat st; 370 curl_socket_t sockfd; 371 372 fprintf(MSG_OUT, "Creating named pipe \"%s\"\n", fifo); 373 if (lstat (fifo, &st) == 0) { 374 if ((st.st_mode & S_IFMT) == S_IFREG) { 375 errno = EEXIST; 376 perror("lstat"); 377 exit (1); 378 } 379 } 380 unlink(fifo); 381 if (mkfifo (fifo, 0600) == -1) { 382 perror("mkfifo"); 383 exit (1); 384 } 385 sockfd = open(fifo, O_RDWR | O_NONBLOCK, 0); 386 if (sockfd == -1) { 387 perror("open"); 388 exit (1); 389 } 390 g->input = fdopen(sockfd, "r"); 391 392 fprintf(MSG_OUT, "Now, pipe some URL's into > %s\n", fifo); 393 g->fifo_event = event_new(g->evbase, sockfd, EV_READ|EV_PERSIST, fifo_cb, g); 394 event_add(g->fifo_event, NULL); 395 return (0); 396} 397 398static void clean_fifo(GlobalInfo *g) 399{ 400 event_free(g->fifo_event); 401 fclose(g->input); 402 unlink(fifo); 403} 404 405int main(int argc, char **argv) 406{ 407 GlobalInfo g; 408 (void)argc; 409 (void)argv; 410 411 memset(&g, 0, sizeof(GlobalInfo)); 412 g.evbase = event_base_new(); 413 init_fifo(&g); 414 g.multi = curl_multi_init(); 415 g.timer_event = evtimer_new(g.evbase, timer_cb, &g); 416 417 /* setup the generic multi interface options we want */ 418 curl_multi_setopt(g.multi, CURLMOPT_SOCKETFUNCTION, sock_cb); 419 curl_multi_setopt(g.multi, CURLMOPT_SOCKETDATA, &g); 420 curl_multi_setopt(g.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb); 421 curl_multi_setopt(g.multi, CURLMOPT_TIMERDATA, &g); 422 423 /* we don't call any curl_multi_socket*() function yet as we have no handles 424 added! */ 425 426 event_base_dispatch(g.evbase); 427 428 /* this, of course, won't get called since only way to stop this program is 429 via ctrl-C, but it is here to show how cleanup /would/ be done. */ 430 clean_fifo(&g); 431 event_free(g.timer_event); 432 event_base_free(g.evbase); 433 curl_multi_cleanup(g.multi); 434 return 0; 435} 436