1/*****************************************************************************
2 *
3 * This example source code introduces a c library buffered I/O interface to
4 * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(),
5 * rewind(). Supported functions have identical prototypes to their normal c
6 * lib namesakes and are preceaded by url_ .
7 *
8 * Using this code you can replace your program's fopen() with url_fopen()
9 * and fread() with url_fread() and it become possible to read remote streams
10 * instead of (only) local files. Local files (ie those that can be directly
11 * fopened) will drop back to using the underlying clib implementations
12 *
13 * See the main() function at the bottom that shows an app that retrives from a
14 * specified url using fgets() and fread() and saves as two output files.
15 *
16 * Copyright (c) 2003 Simtec Electronics
17 *
18 * Re-implemented by Vincent Sanders <vince@kyllikki.org> with extensive
19 * reference to original curl example code
20 *
21 * Redistribution and use in source and binary forms, with or without
22 * modification, are permitted provided that the following conditions
23 * are met:
24 * 1. Redistributions of source code must retain the above copyright
25 *    notice, this list of conditions and the following disclaimer.
26 * 2. Redistributions in binary form must reproduce the above copyright
27 *    notice, this list of conditions and the following disclaimer in the
28 *    documentation and/or other materials provided with the distribution.
29 * 3. The name of the author may not be used to endorse or promote products
30 *    derived from this software without specific prior written permission.
31 *
32 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
33 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
34 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
35 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
36 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
37 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
41 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 *
43 * This example requires libcurl 7.9.7 or later.
44 */
45
46#include <stdio.h>
47#include <string.h>
48#ifndef WIN32
49#  include <sys/time.h>
50#endif
51#include <stdlib.h>
52#include <errno.h>
53
54#include <curl/curl.h>
55
56enum fcurl_type_e {
57  CFTYPE_NONE=0,
58  CFTYPE_FILE=1,
59  CFTYPE_CURL=2
60};
61
62struct fcurl_data
63{
64  enum fcurl_type_e type;     /* type of handle */
65  union {
66    CURL *curl;
67    FILE *file;
68  } handle;                   /* handle */
69
70  char *buffer;               /* buffer to store cached data*/
71  size_t buffer_len;          /* currently allocated buffers length */
72  size_t buffer_pos;          /* end of data in buffer*/
73  int still_running;          /* Is background url fetch still in progress */
74};
75
76typedef struct fcurl_data URL_FILE;
77
78/* exported functions */
79URL_FILE *url_fopen(const char *url,const char *operation);
80int url_fclose(URL_FILE *file);
81int url_feof(URL_FILE *file);
82size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
83char * url_fgets(char *ptr, size_t size, URL_FILE *file);
84void url_rewind(URL_FILE *file);
85
86/* we use a global one for convenience */
87CURLM *multi_handle;
88
89/* curl calls this routine to get more data */
90static size_t write_callback(char *buffer,
91                             size_t size,
92                             size_t nitems,
93                             void *userp)
94{
95  char *newbuff;
96  size_t rembuff;
97
98  URL_FILE *url = (URL_FILE *)userp;
99  size *= nitems;
100
101  rembuff=url->buffer_len - url->buffer_pos; /* remaining space in buffer */
102
103  if(size > rembuff) {
104    /* not enough space in buffer */
105    newbuff=realloc(url->buffer,url->buffer_len + (size - rembuff));
106    if(newbuff==NULL) {
107      fprintf(stderr,"callback buffer grow failed\n");
108      size=rembuff;
109    }
110    else {
111      /* realloc suceeded increase buffer size*/
112      url->buffer_len+=size - rembuff;
113      url->buffer=newbuff;
114    }
115  }
116
117  memcpy(&url->buffer[url->buffer_pos], buffer, size);
118  url->buffer_pos += size;
119
120  return size;
121}
122
123/* use to attempt to fill the read buffer up to requested number of bytes */
124static int fill_buffer(URL_FILE *file, size_t want)
125{
126  fd_set fdread;
127  fd_set fdwrite;
128  fd_set fdexcep;
129  struct timeval timeout;
130  int rc;
131
132  /* only attempt to fill buffer if transactions still running and buffer
133   * doesnt exceed required size already
134   */
135  if((!file->still_running) || (file->buffer_pos > want))
136    return 0;
137
138  /* attempt to fill buffer */
139  do {
140    int maxfd = -1;
141    long curl_timeo = -1;
142
143    FD_ZERO(&fdread);
144    FD_ZERO(&fdwrite);
145    FD_ZERO(&fdexcep);
146
147    /* set a suitable timeout to fail on */
148    timeout.tv_sec = 60; /* 1 minute */
149    timeout.tv_usec = 0;
150
151    curl_multi_timeout(multi_handle, &curl_timeo);
152    if(curl_timeo >= 0) {
153      timeout.tv_sec = curl_timeo / 1000;
154      if(timeout.tv_sec > 1)
155        timeout.tv_sec = 1;
156      else
157        timeout.tv_usec = (curl_timeo % 1000) * 1000;
158    }
159
160    /* get file descriptors from the transfers */
161    curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
162
163    /* In a real-world program you OF COURSE check the return code of the
164       function calls.  On success, the value of maxfd is guaranteed to be
165       greater or equal than -1.  We call select(maxfd + 1, ...), specially
166       in case of (maxfd == -1), we call select(0, ...), which is basically
167       equal to sleep. */
168
169    rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
170
171    switch(rc) {
172    case -1:
173      /* select error */
174      break;
175
176    case 0:
177    default:
178      /* timeout or readable/writable sockets */
179      curl_multi_perform(multi_handle, &file->still_running);
180      break;
181    }
182  } while(file->still_running && (file->buffer_pos < want));
183  return 1;
184}
185
186/* use to remove want bytes from the front of a files buffer */
187static int use_buffer(URL_FILE *file,int want)
188{
189  /* sort out buffer */
190  if((file->buffer_pos - want) <=0) {
191    /* ditch buffer - write will recreate */
192    if(file->buffer)
193      free(file->buffer);
194
195    file->buffer=NULL;
196    file->buffer_pos=0;
197    file->buffer_len=0;
198  }
199  else {
200    /* move rest down make it available for later */
201    memmove(file->buffer,
202            &file->buffer[want],
203            (file->buffer_pos - want));
204
205    file->buffer_pos -= want;
206  }
207  return 0;
208}
209
210URL_FILE *url_fopen(const char *url,const char *operation)
211{
212  /* this code could check for URLs or types in the 'url' and
213     basicly use the real fopen() for standard files */
214
215  URL_FILE *file;
216  (void)operation;
217
218  file = malloc(sizeof(URL_FILE));
219  if(!file)
220    return NULL;
221
222  memset(file, 0, sizeof(URL_FILE));
223
224  if((file->handle.file=fopen(url,operation)))
225    file->type = CFTYPE_FILE; /* marked as URL */
226
227  else {
228    file->type = CFTYPE_CURL; /* marked as URL */
229    file->handle.curl = curl_easy_init();
230
231    curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
232    curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
233    curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
234    curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
235
236    if(!multi_handle)
237      multi_handle = curl_multi_init();
238
239    curl_multi_add_handle(multi_handle, file->handle.curl);
240
241    /* lets start the fetch */
242    curl_multi_perform(multi_handle, &file->still_running);
243
244    if((file->buffer_pos == 0) && (!file->still_running)) {
245      /* if still_running is 0 now, we should return NULL */
246
247      /* make sure the easy handle is not in the multi handle anymore */
248      curl_multi_remove_handle(multi_handle, file->handle.curl);
249
250      /* cleanup */
251      curl_easy_cleanup(file->handle.curl);
252
253      free(file);
254
255      file = NULL;
256    }
257  }
258  return file;
259}
260
261int url_fclose(URL_FILE *file)
262{
263  int ret=0;/* default is good return */
264
265  switch(file->type) {
266  case CFTYPE_FILE:
267    ret=fclose(file->handle.file); /* passthrough */
268    break;
269
270  case CFTYPE_CURL:
271    /* make sure the easy handle is not in the multi handle anymore */
272    curl_multi_remove_handle(multi_handle, file->handle.curl);
273
274    /* cleanup */
275    curl_easy_cleanup(file->handle.curl);
276    break;
277
278  default: /* unknown or supported type - oh dear */
279    ret=EOF;
280    errno=EBADF;
281    break;
282  }
283
284  if(file->buffer)
285    free(file->buffer);/* free any allocated buffer space */
286
287  free(file);
288
289  return ret;
290}
291
292int url_feof(URL_FILE *file)
293{
294  int ret=0;
295
296  switch(file->type) {
297  case CFTYPE_FILE:
298    ret=feof(file->handle.file);
299    break;
300
301  case CFTYPE_CURL:
302    if((file->buffer_pos == 0) && (!file->still_running))
303      ret = 1;
304    break;
305
306  default: /* unknown or supported type - oh dear */
307    ret=-1;
308    errno=EBADF;
309    break;
310  }
311  return ret;
312}
313
314size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
315{
316  size_t want;
317
318  switch(file->type) {
319  case CFTYPE_FILE:
320    want=fread(ptr,size,nmemb,file->handle.file);
321    break;
322
323  case CFTYPE_CURL:
324    want = nmemb * size;
325
326    fill_buffer(file,want);
327
328    /* check if theres data in the buffer - if not fill_buffer()
329     * either errored or EOF */
330    if(!file->buffer_pos)
331      return 0;
332
333    /* ensure only available data is considered */
334    if(file->buffer_pos < want)
335      want = file->buffer_pos;
336
337    /* xfer data to caller */
338    memcpy(ptr, file->buffer, want);
339
340    use_buffer(file,want);
341
342    want = want / size;     /* number of items */
343    break;
344
345  default: /* unknown or supported type - oh dear */
346    want=0;
347    errno=EBADF;
348    break;
349
350  }
351  return want;
352}
353
354char *url_fgets(char *ptr, size_t size, URL_FILE *file)
355{
356  size_t want = size - 1;/* always need to leave room for zero termination */
357  size_t loop;
358
359  switch(file->type) {
360  case CFTYPE_FILE:
361    ptr = fgets(ptr,size,file->handle.file);
362    break;
363
364  case CFTYPE_CURL:
365    fill_buffer(file,want);
366
367    /* check if theres data in the buffer - if not fill either errored or
368     * EOF */
369    if(!file->buffer_pos)
370      return NULL;
371
372    /* ensure only available data is considered */
373    if(file->buffer_pos < want)
374      want = file->buffer_pos;
375
376    /*buffer contains data */
377    /* look for newline or eof */
378    for(loop=0;loop < want;loop++) {
379      if(file->buffer[loop] == '\n') {
380        want=loop+1;/* include newline */
381        break;
382      }
383    }
384
385    /* xfer data to caller */
386    memcpy(ptr, file->buffer, want);
387    ptr[want]=0;/* allways null terminate */
388
389    use_buffer(file,want);
390
391    break;
392
393  default: /* unknown or supported type - oh dear */
394    ptr=NULL;
395    errno=EBADF;
396    break;
397  }
398
399  return ptr;/*success */
400}
401
402void url_rewind(URL_FILE *file)
403{
404  switch(file->type) {
405  case CFTYPE_FILE:
406    rewind(file->handle.file); /* passthrough */
407    break;
408
409  case CFTYPE_CURL:
410    /* halt transaction */
411    curl_multi_remove_handle(multi_handle, file->handle.curl);
412
413    /* restart */
414    curl_multi_add_handle(multi_handle, file->handle.curl);
415
416    /* ditch buffer - write will recreate - resets stream pos*/
417    if(file->buffer)
418      free(file->buffer);
419
420    file->buffer=NULL;
421    file->buffer_pos=0;
422    file->buffer_len=0;
423
424    break;
425
426  default: /* unknown or supported type - oh dear */
427    break;
428  }
429}
430
431/* Small main program to retrive from a url using fgets and fread saving the
432 * output to two test files (note the fgets method will corrupt binary files if
433 * they contain 0 chars */
434int main(int argc, char *argv[])
435{
436  URL_FILE *handle;
437  FILE *outf;
438
439  int nread;
440  char buffer[256];
441  const char *url;
442
443  if(argc < 2)
444    url="http://192.168.7.3/testfile";/* default to testurl */
445  else
446    url=argv[1];/* use passed url */
447
448  /* copy from url line by line with fgets */
449  outf=fopen("fgets.test","w+");
450  if(!outf) {
451    perror("couldn't open fgets output file\n");
452    return 1;
453  }
454
455  handle = url_fopen(url, "r");
456  if(!handle) {
457    printf("couldn't url_fopen() %s\n", url);
458    fclose(outf);
459    return 2;
460  }
461
462  while(!url_feof(handle)) {
463    url_fgets(buffer,sizeof(buffer),handle);
464    fwrite(buffer,1,strlen(buffer),outf);
465  }
466
467  url_fclose(handle);
468
469  fclose(outf);
470
471
472  /* Copy from url with fread */
473  outf=fopen("fread.test","w+");
474  if(!outf) {
475    perror("couldn't open fread output file\n");
476    return 1;
477  }
478
479  handle = url_fopen("testfile", "r");
480  if(!handle) {
481    printf("couldn't url_fopen() testfile\n");
482    fclose(outf);
483    return 2;
484  }
485
486  do {
487    nread = url_fread(buffer, 1,sizeof(buffer), handle);
488    fwrite(buffer,1,nread,outf);
489  } while(nread);
490
491  url_fclose(handle);
492
493  fclose(outf);
494
495
496  /* Test rewind */
497  outf=fopen("rewind.test","w+");
498  if(!outf) {
499    perror("couldn't open fread output file\n");
500    return 1;
501  }
502
503  handle = url_fopen("testfile", "r");
504  if(!handle) {
505    printf("couldn't url_fopen() testfile\n");
506    fclose(outf);
507    return 2;
508  }
509
510  nread = url_fread(buffer, 1,sizeof(buffer), handle);
511  fwrite(buffer,1,nread,outf);
512  url_rewind(handle);
513
514  buffer[0]='\n';
515  fwrite(buffer,1,1,outf);
516
517  nread = url_fread(buffer, 1,sizeof(buffer), handle);
518  fwrite(buffer,1,nread,outf);
519
520
521  url_fclose(handle);
522
523  fclose(outf);
524
525
526  return 0;/* all done */
527}
528