1/***************************************************************************** 2 * 3 * This example source code introduces a c library buffered I/O interface to 4 * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(), 5 * rewind(). Supported functions have identical prototypes to their normal c 6 * lib namesakes and are preceaded by url_ . 7 * 8 * Using this code you can replace your program's fopen() with url_fopen() 9 * and fread() with url_fread() and it become possible to read remote streams 10 * instead of (only) local files. Local files (ie those that can be directly 11 * fopened) will drop back to using the underlying clib implementations 12 * 13 * See the main() function at the bottom that shows an app that retrives from a 14 * specified url using fgets() and fread() and saves as two output files. 15 * 16 * Copyright (c) 2003 Simtec Electronics 17 * 18 * Re-implemented by Vincent Sanders <vince@kyllikki.org> with extensive 19 * reference to original curl example code 20 * 21 * Redistribution and use in source and binary forms, with or without 22 * modification, are permitted provided that the following conditions 23 * are met: 24 * 1. Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * 2. Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in the 28 * documentation and/or other materials provided with the distribution. 29 * 3. The name of the author may not be used to endorse or promote products 30 * derived from this software without specific prior written permission. 31 * 32 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 33 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 34 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 35 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 36 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 37 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 41 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 * 43 * This example requires libcurl 7.9.7 or later. 44 */ 45 46#include <stdio.h> 47#include <string.h> 48#ifndef WIN32 49# include <sys/time.h> 50#endif 51#include <stdlib.h> 52#include <errno.h> 53 54#include <curl/curl.h> 55 56enum fcurl_type_e { 57 CFTYPE_NONE=0, 58 CFTYPE_FILE=1, 59 CFTYPE_CURL=2 60}; 61 62struct fcurl_data 63{ 64 enum fcurl_type_e type; /* type of handle */ 65 union { 66 CURL *curl; 67 FILE *file; 68 } handle; /* handle */ 69 70 char *buffer; /* buffer to store cached data*/ 71 size_t buffer_len; /* currently allocated buffers length */ 72 size_t buffer_pos; /* end of data in buffer*/ 73 int still_running; /* Is background url fetch still in progress */ 74}; 75 76typedef struct fcurl_data URL_FILE; 77 78/* exported functions */ 79URL_FILE *url_fopen(const char *url,const char *operation); 80int url_fclose(URL_FILE *file); 81int url_feof(URL_FILE *file); 82size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file); 83char * url_fgets(char *ptr, size_t size, URL_FILE *file); 84void url_rewind(URL_FILE *file); 85 86/* we use a global one for convenience */ 87CURLM *multi_handle; 88 89/* curl calls this routine to get more data */ 90static size_t write_callback(char *buffer, 91 size_t size, 92 size_t nitems, 93 void *userp) 94{ 95 char *newbuff; 96 size_t rembuff; 97 98 URL_FILE *url = (URL_FILE *)userp; 99 size *= nitems; 100 101 rembuff=url->buffer_len - url->buffer_pos; /* remaining space in buffer */ 102 103 if(size > rembuff) { 104 /* not enough space in buffer */ 105 newbuff=realloc(url->buffer,url->buffer_len + (size - rembuff)); 106 if(newbuff==NULL) { 107 fprintf(stderr,"callback buffer grow failed\n"); 108 size=rembuff; 109 } 110 else { 111 /* realloc suceeded increase buffer size*/ 112 url->buffer_len+=size - rembuff; 113 url->buffer=newbuff; 114 } 115 } 116 117 memcpy(&url->buffer[url->buffer_pos], buffer, size); 118 url->buffer_pos += size; 119 120 return size; 121} 122 123/* use to attempt to fill the read buffer up to requested number of bytes */ 124static int fill_buffer(URL_FILE *file, size_t want) 125{ 126 fd_set fdread; 127 fd_set fdwrite; 128 fd_set fdexcep; 129 struct timeval timeout; 130 int rc; 131 132 /* only attempt to fill buffer if transactions still running and buffer 133 * doesnt exceed required size already 134 */ 135 if((!file->still_running) || (file->buffer_pos > want)) 136 return 0; 137 138 /* attempt to fill buffer */ 139 do { 140 int maxfd = -1; 141 long curl_timeo = -1; 142 143 FD_ZERO(&fdread); 144 FD_ZERO(&fdwrite); 145 FD_ZERO(&fdexcep); 146 147 /* set a suitable timeout to fail on */ 148 timeout.tv_sec = 60; /* 1 minute */ 149 timeout.tv_usec = 0; 150 151 curl_multi_timeout(multi_handle, &curl_timeo); 152 if(curl_timeo >= 0) { 153 timeout.tv_sec = curl_timeo / 1000; 154 if(timeout.tv_sec > 1) 155 timeout.tv_sec = 1; 156 else 157 timeout.tv_usec = (curl_timeo % 1000) * 1000; 158 } 159 160 /* get file descriptors from the transfers */ 161 curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd); 162 163 /* In a real-world program you OF COURSE check the return code of the 164 function calls. On success, the value of maxfd is guaranteed to be 165 greater or equal than -1. We call select(maxfd + 1, ...), specially 166 in case of (maxfd == -1), we call select(0, ...), which is basically 167 equal to sleep. */ 168 169 rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout); 170 171 switch(rc) { 172 case -1: 173 /* select error */ 174 break; 175 176 case 0: 177 default: 178 /* timeout or readable/writable sockets */ 179 curl_multi_perform(multi_handle, &file->still_running); 180 break; 181 } 182 } while(file->still_running && (file->buffer_pos < want)); 183 return 1; 184} 185 186/* use to remove want bytes from the front of a files buffer */ 187static int use_buffer(URL_FILE *file,int want) 188{ 189 /* sort out buffer */ 190 if((file->buffer_pos - want) <=0) { 191 /* ditch buffer - write will recreate */ 192 if(file->buffer) 193 free(file->buffer); 194 195 file->buffer=NULL; 196 file->buffer_pos=0; 197 file->buffer_len=0; 198 } 199 else { 200 /* move rest down make it available for later */ 201 memmove(file->buffer, 202 &file->buffer[want], 203 (file->buffer_pos - want)); 204 205 file->buffer_pos -= want; 206 } 207 return 0; 208} 209 210URL_FILE *url_fopen(const char *url,const char *operation) 211{ 212 /* this code could check for URLs or types in the 'url' and 213 basicly use the real fopen() for standard files */ 214 215 URL_FILE *file; 216 (void)operation; 217 218 file = malloc(sizeof(URL_FILE)); 219 if(!file) 220 return NULL; 221 222 memset(file, 0, sizeof(URL_FILE)); 223 224 if((file->handle.file=fopen(url,operation))) 225 file->type = CFTYPE_FILE; /* marked as URL */ 226 227 else { 228 file->type = CFTYPE_CURL; /* marked as URL */ 229 file->handle.curl = curl_easy_init(); 230 231 curl_easy_setopt(file->handle.curl, CURLOPT_URL, url); 232 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file); 233 curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L); 234 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback); 235 236 if(!multi_handle) 237 multi_handle = curl_multi_init(); 238 239 curl_multi_add_handle(multi_handle, file->handle.curl); 240 241 /* lets start the fetch */ 242 curl_multi_perform(multi_handle, &file->still_running); 243 244 if((file->buffer_pos == 0) && (!file->still_running)) { 245 /* if still_running is 0 now, we should return NULL */ 246 247 /* make sure the easy handle is not in the multi handle anymore */ 248 curl_multi_remove_handle(multi_handle, file->handle.curl); 249 250 /* cleanup */ 251 curl_easy_cleanup(file->handle.curl); 252 253 free(file); 254 255 file = NULL; 256 } 257 } 258 return file; 259} 260 261int url_fclose(URL_FILE *file) 262{ 263 int ret=0;/* default is good return */ 264 265 switch(file->type) { 266 case CFTYPE_FILE: 267 ret=fclose(file->handle.file); /* passthrough */ 268 break; 269 270 case CFTYPE_CURL: 271 /* make sure the easy handle is not in the multi handle anymore */ 272 curl_multi_remove_handle(multi_handle, file->handle.curl); 273 274 /* cleanup */ 275 curl_easy_cleanup(file->handle.curl); 276 break; 277 278 default: /* unknown or supported type - oh dear */ 279 ret=EOF; 280 errno=EBADF; 281 break; 282 } 283 284 if(file->buffer) 285 free(file->buffer);/* free any allocated buffer space */ 286 287 free(file); 288 289 return ret; 290} 291 292int url_feof(URL_FILE *file) 293{ 294 int ret=0; 295 296 switch(file->type) { 297 case CFTYPE_FILE: 298 ret=feof(file->handle.file); 299 break; 300 301 case CFTYPE_CURL: 302 if((file->buffer_pos == 0) && (!file->still_running)) 303 ret = 1; 304 break; 305 306 default: /* unknown or supported type - oh dear */ 307 ret=-1; 308 errno=EBADF; 309 break; 310 } 311 return ret; 312} 313 314size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file) 315{ 316 size_t want; 317 318 switch(file->type) { 319 case CFTYPE_FILE: 320 want=fread(ptr,size,nmemb,file->handle.file); 321 break; 322 323 case CFTYPE_CURL: 324 want = nmemb * size; 325 326 fill_buffer(file,want); 327 328 /* check if theres data in the buffer - if not fill_buffer() 329 * either errored or EOF */ 330 if(!file->buffer_pos) 331 return 0; 332 333 /* ensure only available data is considered */ 334 if(file->buffer_pos < want) 335 want = file->buffer_pos; 336 337 /* xfer data to caller */ 338 memcpy(ptr, file->buffer, want); 339 340 use_buffer(file,want); 341 342 want = want / size; /* number of items */ 343 break; 344 345 default: /* unknown or supported type - oh dear */ 346 want=0; 347 errno=EBADF; 348 break; 349 350 } 351 return want; 352} 353 354char *url_fgets(char *ptr, size_t size, URL_FILE *file) 355{ 356 size_t want = size - 1;/* always need to leave room for zero termination */ 357 size_t loop; 358 359 switch(file->type) { 360 case CFTYPE_FILE: 361 ptr = fgets(ptr,size,file->handle.file); 362 break; 363 364 case CFTYPE_CURL: 365 fill_buffer(file,want); 366 367 /* check if theres data in the buffer - if not fill either errored or 368 * EOF */ 369 if(!file->buffer_pos) 370 return NULL; 371 372 /* ensure only available data is considered */ 373 if(file->buffer_pos < want) 374 want = file->buffer_pos; 375 376 /*buffer contains data */ 377 /* look for newline or eof */ 378 for(loop=0;loop < want;loop++) { 379 if(file->buffer[loop] == '\n') { 380 want=loop+1;/* include newline */ 381 break; 382 } 383 } 384 385 /* xfer data to caller */ 386 memcpy(ptr, file->buffer, want); 387 ptr[want]=0;/* allways null terminate */ 388 389 use_buffer(file,want); 390 391 break; 392 393 default: /* unknown or supported type - oh dear */ 394 ptr=NULL; 395 errno=EBADF; 396 break; 397 } 398 399 return ptr;/*success */ 400} 401 402void url_rewind(URL_FILE *file) 403{ 404 switch(file->type) { 405 case CFTYPE_FILE: 406 rewind(file->handle.file); /* passthrough */ 407 break; 408 409 case CFTYPE_CURL: 410 /* halt transaction */ 411 curl_multi_remove_handle(multi_handle, file->handle.curl); 412 413 /* restart */ 414 curl_multi_add_handle(multi_handle, file->handle.curl); 415 416 /* ditch buffer - write will recreate - resets stream pos*/ 417 if(file->buffer) 418 free(file->buffer); 419 420 file->buffer=NULL; 421 file->buffer_pos=0; 422 file->buffer_len=0; 423 424 break; 425 426 default: /* unknown or supported type - oh dear */ 427 break; 428 } 429} 430 431/* Small main program to retrive from a url using fgets and fread saving the 432 * output to two test files (note the fgets method will corrupt binary files if 433 * they contain 0 chars */ 434int main(int argc, char *argv[]) 435{ 436 URL_FILE *handle; 437 FILE *outf; 438 439 int nread; 440 char buffer[256]; 441 const char *url; 442 443 if(argc < 2) 444 url="http://192.168.7.3/testfile";/* default to testurl */ 445 else 446 url=argv[1];/* use passed url */ 447 448 /* copy from url line by line with fgets */ 449 outf=fopen("fgets.test","w+"); 450 if(!outf) { 451 perror("couldn't open fgets output file\n"); 452 return 1; 453 } 454 455 handle = url_fopen(url, "r"); 456 if(!handle) { 457 printf("couldn't url_fopen() %s\n", url); 458 fclose(outf); 459 return 2; 460 } 461 462 while(!url_feof(handle)) { 463 url_fgets(buffer,sizeof(buffer),handle); 464 fwrite(buffer,1,strlen(buffer),outf); 465 } 466 467 url_fclose(handle); 468 469 fclose(outf); 470 471 472 /* Copy from url with fread */ 473 outf=fopen("fread.test","w+"); 474 if(!outf) { 475 perror("couldn't open fread output file\n"); 476 return 1; 477 } 478 479 handle = url_fopen("testfile", "r"); 480 if(!handle) { 481 printf("couldn't url_fopen() testfile\n"); 482 fclose(outf); 483 return 2; 484 } 485 486 do { 487 nread = url_fread(buffer, 1,sizeof(buffer), handle); 488 fwrite(buffer,1,nread,outf); 489 } while(nread); 490 491 url_fclose(handle); 492 493 fclose(outf); 494 495 496 /* Test rewind */ 497 outf=fopen("rewind.test","w+"); 498 if(!outf) { 499 perror("couldn't open fread output file\n"); 500 return 1; 501 } 502 503 handle = url_fopen("testfile", "r"); 504 if(!handle) { 505 printf("couldn't url_fopen() testfile\n"); 506 fclose(outf); 507 return 2; 508 } 509 510 nread = url_fread(buffer, 1,sizeof(buffer), handle); 511 fwrite(buffer,1,nread,outf); 512 url_rewind(handle); 513 514 buffer[0]='\n'; 515 fwrite(buffer,1,1,outf); 516 517 nread = url_fread(buffer, 1,sizeof(buffer), handle); 518 fwrite(buffer,1,nread,outf); 519 520 521 url_fclose(handle); 522 523 fclose(outf); 524 525 526 return 0;/* all done */ 527} 528