/** * \file * \brief NFS-populated file cache for HTTP server * * This very stupid "cache" assumes that: * All regular files in a hardcoded NFS mount point are cached at startup * The contents of the cache never change nor expire */ /* * Copyright (c) 2008, ETH Zurich. * All rights reserved. * * This file is distributed under the terms in the attached LICENSE file. * If you do not find this file, copies can be found by writing to: * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group. */ #include #include #include #include #include #include #include #include #include "webserver_network.h" #include "webserver_debug.h" #include "webserver_session.h" #include "http_cache.h" /* NOTE: enable following #def if you want cache to be preloaded */ #define PRELOAD_WEB_CACHE 1 /* Enable tracing only when it is globally enabled */ #if CONFIG_TRACE && NETWORK_STACK_TRACE //#define ENABLE_WEB_TRACING 1 #endif // CONFIG_TRACE && NETWORK_STACK_TRACE //#define MAX_NFS_READ 14000 #define MAX_NFS_READ 1330 /* 14000 */ /* to avoid packet reassembly inside driver */ /* Maximum staleness allowed */ #define MAX_STALENESS ((cycles_t)9000000) static void (*init_callback)(void); struct http_cache_entry { int valid; /* flag for validity of the data */ char *name; /* name of the cached file */ size_t copied; /* how much data is copied? */ int loading; /* flag indicating if data is loading */ struct buff_holder *hbuff; /* holder for buffer */ struct nfs_fh3 file_handle; /* for NFS purpose */ struct http_conn *conn; /* list of connections waiting for data */ struct http_conn *last; /* for quick insertions at end */ struct http_cache_entry *next; /* for linked list */ }; /* global states */ static struct nfs_fh3 nfs_root_fh; /* reference to the root dir of NFS */ static struct nfs_client *my_nfs_client; /* The NFS client */ static struct http_cache_entry *cache_table = NULL; /*root cached entry array */ static struct http_cache_entry *error_cache = NULL; /* cache entry for error */ #ifdef PRELOAD_WEB_CACHE /* Initial cache loading state variables */ static bool cache_loading_phase = false; static bool readdir_complete = false; static int cache_lookups_started = 0; static int cache_loaded_counter = 0; static int cache_loading_probs = 0; static void handle_cache_load_done(void); #endif // PRELOAD_WEB_CACHE // Variables for time measurement for performance static uint64_t last_ts = 0; /* allocate the buffer and initialize it. */ static struct buff_holder *allocate_buff_holder (size_t len) { struct buff_holder *result = NULL; result = (struct buff_holder *) malloc (sizeof (struct buff_holder)); assert (result != NULL ); memset (result, 0, sizeof(struct buff_holder)); if ( len > 0) { result->data = malloc (len); assert (result->data != NULL); } /* NOTE: 0 is valid length and used by error_cache */ result->len = len; result->r_counter = 1; /* initiating ref_counter to 1, and using it as ref for free */ return result; } /* end function: allocate_buff_holder */ /* increments and returns the incremented value of the ref_counter for bh */ static long increment_buff_holder_ref (struct buff_holder *bh) { ++bh->r_counter; return (bh->r_counter); } /* end Function: increment_buff_holder_ref */ /* Decrements value of the ref_counter for bh if r_counter reaches zero then free all the memory */ long decrement_buff_holder_ref (struct buff_holder *bh) { if (bh == NULL) { return 0; } --bh->r_counter; if (bh->r_counter > 0) { return (bh->r_counter); } if (bh->len > 0 && bh->data != NULL) { free (bh->data); } free (bh); return 0; } /* end Function: increment_buff_holder_ref */ /* allocates the memory for the cacheline */ static struct http_cache_entry * cache_entry_allocate (void) { struct http_cache_entry *e; e = (struct http_cache_entry *) malloc (sizeof(struct http_cache_entry)); assert (e != NULL ); memset (e, 0, sizeof(struct http_cache_entry)); return e; } /* Function create_404_page_cache creates error_cache entry, this entry will be used to reply when file is not found */ static void create_404_page_cache (void) { error_cache = cache_entry_allocate(); error_cache->hbuff = allocate_buff_holder (0); increment_buff_holder_ref (error_cache->hbuff); error_cache->valid = 1; } /* end function: create_404_page_cache */ /* adds the http connection cs at the end of the list of connections waiting for the arrival of the data on this cacheline e. */ static void add_connection(struct http_cache_entry *e, struct http_conn *cs) { assert (e != NULL); assert (cs != NULL); DEBUGPRINT ("%d: adding conn to waiting list of cacheline [%s]\n", cs->request_no, e->name); increment_http_conn_reference (cs); /* maintaining linked list with O(c) time and space complexity */ if (e->conn == NULL) { /* list is empty, this is first element */ assert(e->last == NULL); /* NOTE: just for safety, not actually needed*/ e->conn = cs; e->last = cs; return; } /* adding element at end of the list */ e->last->next = cs; e->last = cs; } /* end function: add_connection */ /* Finds the cacheline associated with given name if no cacheline exists, it will create one, copy name as the key for cacheline */ static struct http_cache_entry *find_cacheline (const char *name) { struct http_cache_entry *e; int l; // FIXME: simple linear search for (e = cache_table; e != NULL; e = e->next) { if (strcmp(name, e->name) == 0) { DEBUGPRINT ("cache-hit for [%s] == [%s]\n", name, e->name); return e; } } /* end for : for each cacheline */ /* create new cacheline */ e = cache_entry_allocate(); /* copying the filename */ l = strlen (name); e->name = (char *)malloc(sizeof(char)*(l+1)); assert(e->name != NULL); strcpy(e->name, name); DEBUGPRINT ("cache-miss for [%s] so, created [%s]\n", name, e->name); e->next = cache_table; cache_table = e; return e; } /* end function: find_cacheline */ static void delete_cacheline_from_cachelist (struct http_cache_entry *target) { struct http_cache_entry *prev; struct http_cache_entry *e; if (cache_table == target){ cache_table = target->next; return; } // FIXME: simple linear search prev = NULL; for (e = cache_table; e != NULL; e = e->next) { if (e == target) { prev->next = e->next; return; } prev = e; } /* end for : for each cacheline */ } /* end function: delete_cacheline_from_cachelist */ static void trigger_callback (struct http_conn *cs, struct http_cache_entry *e) { if ( cs->mark_invalid ){ /* This http_conn is not good anymore (most probably, it received RST)*/ DEBUGPRINT ("%d: ERROR: callback triggered on invalid conn\n", cs->request_no ); return; } /* making the callback */ cs->hbuff = e->hbuff; /* increasing the reference */ increment_buff_holder_ref (cs->hbuff); cs->callback(cs); } /* end function: trigger_callback */ /* send callback for each pending http_conn */ static void handle_pending_list(struct http_cache_entry *e) { struct http_conn *cs = NULL; struct http_conn *next_cs = NULL; if(e == NULL) { printf("ERROR: handle_pending_list: null passed for cache entry\n"); return; } if(e->conn == NULL){ // printf("ERROR: handle_pending_list: no one waiting\n"); return; } cs = e->conn; while (cs != NULL) { e->conn = cs->next; trigger_callback(cs, e); next_cs = cs->next; /* Copying the next of cs as following function can release the cs. */ decrement_http_conn_reference(cs); cs = next_cs; } /* clear the list attached to the cache */ e->conn = NULL; e->last = NULL; } /* end function : handle_pending_list */ static void read_callback (void *arg, struct nfs_client *client, READ3res *result) { struct http_cache_entry *e = arg; assert( e != NULL); assert (result != NULL); assert (result->status == NFS3_OK); READ3resok *res = &result->READ3res_u.resok; assert(res->count == res->data.data_len); assert (e->hbuff != NULL); assert (e->hbuff->data != NULL ); assert (e->hbuff->len >= e->copied + res->data.data_len); memcpy (e->hbuff->data + e->copied, res->data.data_val, res->data.data_len); e->copied += res->data.data_len; DEBUGPRINT ("got response of len %d, filesize %lu for file %s\n", res->data.data_len, e->copied, e->name); // free arguments xdr_READ3res(&xdr_free, result); if (!res->eof) { // more data to come, read the next chunk errval_t err = nfs_read(client, e->file_handle, e->copied, MAX_NFS_READ, read_callback, e); assert(err == SYS_ERR_OK); return; } /* This is the end-of-file, so deal with it. */ e->valid = 1; e->loading = 0; decrement_buff_holder_ref (e->hbuff); #ifdef PRELOAD_WEB_CACHE if (!cache_loading_phase) { handle_pending_list (e); /* done! */ return; } /* This is cache loading going on... */ printf("Copied %zu bytes for file [%s] of length: %zu\n", e->copied, e->name, e->hbuff->len); ++cache_loaded_counter; handle_cache_load_done(); #else // PRELOAD_WEB_CACHE handle_pending_list(e); /* done! */ #endif // PRELOAD_WEB_CACHE } static void lookup_callback (void *arg, struct nfs_client *client, LOOKUP3res *result) { LOOKUP3resok *resok = &result->LOOKUP3res_u.resok; errval_t r; struct http_cache_entry *e = arg; DEBUGPRINT ("inside lookup_callback_file for file %s\n", e->name); assert(result != NULL ); /* initiate a read for every regular file */ if ( result->status == NFS3_OK && resok->obj_attributes.attributes_follow && resok->obj_attributes.post_op_attr_u.attributes.type == NF3REG) { /* FIXME: check if the file is recently modified or not. */ // resok->obj_attributes.post_op_attr_u.attributes.ctime; DEBUGPRINT("Copying %s of size %lu\n", e->name, resok->obj_attributes.post_op_attr_u.attributes.size ); /* Store the nfs directory handle in current_session (cs) */ nfs_copyfh (&e->file_handle, resok->object); /* GLOBAL: Storing the global reference for cache entry */ /* NOTE: this memory is freed in reset_cache_entry() */ /* Allocate memory for holding the file-content */ /* Allocate the buff_holder */ e->hbuff = allocate_buff_holder( resok->obj_attributes.post_op_attr_u.attributes.size ); /* NOTE: this memory will be freed by decrement_buff_holder_ref */ /* increment buff_holder reference */ increment_buff_holder_ref (e->hbuff); e->hbuff->len = resok->obj_attributes.post_op_attr_u.attributes.size; /* Set the size of the how much data is read till now. */ e->copied = 0; r = nfs_read(client, e->file_handle, 0, MAX_NFS_READ, read_callback, e); assert (r == SYS_ERR_OK); // free arguments xdr_LOOKUP3res(&xdr_free, result); return; } /* Most probably the file does not exist */ DEBUGPRINT ("Error: file [%s] does not exist, or wrong type\n", e->name); #ifdef PRELOAD_WEB_CACHE if (cache_loading_phase){ ++cache_loading_probs; handle_cache_load_done(); return; } #endif // PRELOAD_WEB_CACHE if (e->conn == NULL) { /* No connection is waiting for this loading */ return; } /* as file does not exist, send all the http_conns to error page. */ error_cache->conn = e->conn; error_cache->last = e->last; handle_pending_list (error_cache); /* done! */ /* free this cache entry as it is pointing to invalid page */ e->conn = NULL; e->last = NULL; delete_cacheline_from_cachelist (e); if (e->name != NULL ) free(e->name); free(e); // free arguments xdr_LOOKUP3res(&xdr_free, result); return; } /* end function: lookup_callback_file */ static errval_t async_load_cache_entry(struct http_cache_entry *e) { errval_t r; assert(e != NULL); // FIXME: currently only works for files in root directory. // Do lookup for given filename in root dir DEBUGPRINT ("pageloading starting with nfs_lookup\n"); r = nfs_lookup(my_nfs_client, nfs_root_fh, e->name, lookup_callback, e); assert(r == SYS_ERR_OK); return SYS_ERR_OK; } /* end function : async_load_cache_entry */ errval_t http_cache_lookup (const char *name, struct http_conn *cs) { struct http_cache_entry *e; assert(cs != NULL); e = find_cacheline(name); if (e->valid == 1) { /* fresh matching cache-entry found */ DEBUGPRINT ("%d: Fresh cache-entry, returning page [%s]\n", cs->request_no, name); trigger_callback (cs, e); return SYS_ERR_OK; } /* end if: valid cacheline */ /* data not in cache */ /* add this connection to the list of waiting on cacheline */ add_connection (e, cs); /* Check if no-one is loading the data */ if (e->loading == 0 ) { e->loading = 1; DEBUGPRINT ("%d: starting the page loading\n", cs->request_no); async_load_cache_entry(e); } /* end if: no-one else is loading the data */ else { DEBUGPRINT ("%d: someone else is loading the page, so just waiting\n", cs->request_no); } return SYS_ERR_OK; } /* end function: http_cache_lookup */ static void cache_timeout_event (struct timer *timer, void *arg) { struct http_cache_entry *cache = arg; struct http_cache_entry *e; DEBUGPRINT ("CACHE_TIMEOUT: marking all cache entries for reload\n"); for (e = cache; e != NULL; e = e->next) { if (e->valid == 1) { /* this if is to avoid any complications that might come because of getting cache_timeout when cacheline was in loading */ e->valid = 0; e->loading = 0; decrement_buff_holder_ref (e->hbuff); /* Decrement buffer ref. */ e->hbuff = NULL; /* loosing the pointer to buff */ } /* end if: if cacheline is valid */ } /* end for : for each cacheline */ DEBUGPRINT ("CACHE_TIMEOUT: Done with invalidating the cache entries\n"); } /* end function : cache_timeout_event */ #ifdef PRELOAD_WEB_CACHE static void readdir_callback(void *arg, struct nfs_client *client, READDIR3res *result) { READDIR3resok *resok = &result->READDIR3res_u.resok; struct http_cache_entry *ce; entry3 *last = NULL; errval_t r; DEBUGPRINT ("readdir_callback came in\n"); assert(result != NULL && result->status == NFS3_OK); // FIXME: start here the measurement of file loading time // FIXME: Start the trace #if ENABLE_WEB_TRACING printf("Starting tracing\n"); errval_t err = trace_control(TRACE_EVENT(TRACE_SUBSYS_NNET, TRACE_EVENT_NNET_START, 0), TRACE_EVENT(TRACE_SUBSYS_NNET, TRACE_EVENT_NNET_STOP, 0), 0); if(err_is_fail(err)) { USER_PANIC_ERR(err, "trace_control failed"); } trace_event(TRACE_SUBSYS_NNET, TRACE_EVENT_NNET_START, 0); #else // ENABLE_WEB_TRACING printf("Tracing not enabled\n"); #endif // ENABLE_WEB_TRACING last_ts = rdtsc(); // lwip_benchmark_control(1, BMS_START_REQUEST, 0, 0); // initiate a lookup for every entry for (entry3 *e = resok->reply.entries; e != NULL; e = e->nextentry) { if (strlen(e->name) < 3 ) { printf("ignoring the file %s\n", e->name); continue; } ++cache_lookups_started; printf("Loading the file %s\n", e->name); ce = find_cacheline(e->name); ce->loading = 1; async_load_cache_entry(ce); e->name = NULL; // prevent freeing by XDR last = e; // continue handling events till this page is not completly loaded. /* struct waitset *ws = get_default_waitset(); errval_t err; DEBUGPRINT ("looping for events inside loop of readdir_callback\n"); while (ce->valid != 1) { err = event_dispatch(ws); if (err_is_fail(err)) { DEBUG_ERR(err, "in event_dispatch"); break; } } DEBUGPRINT ("readdir_callback: %s file loaded\n", e->name); */ } /* more in the directory: repeat call */ if (!resok->reply.eof) { assert(last != NULL); r = nfs_readdir(client, nfs_root_fh, last->cookie, resok->cookieverf, readdir_callback, NULL); assert(r == SYS_ERR_OK); } else { readdir_complete = true; handle_cache_load_done(); } // free arguments xdr_READDIR3res(&xdr_free, result); } static bool are_all_pages_loaded(void) { if (readdir_complete == false ) { return false; } return (cache_lookups_started == (cache_loaded_counter + cache_loading_probs)); } static void handle_cache_load_done(void) { if (!are_all_pages_loaded()) { return; } DEBUGPRINT("initial_cache_load: entire cache loaded done\n"); cache_loading_phase = false; // lwip_benchmark_control(1, BMS_STOP_REQUEST, 0, 0); // // Report the cache loading time uint64_t total_loading_time = rdtsc() - last_ts; printf("Cache loading time %"PU", %"PRIu64"\n", in_seconds(total_loading_time), total_loading_time); // lwip_print_interesting_stats(); /* stop the trace. */ #if ENABLE_WEB_TRACING trace_event(TRACE_SUBSYS_NNET, TRACE_EVENT_NNET_STOP, 0); char *trace_buf_area = malloc(CONSOLE_DUMP_BUFLEN); assert(trace_buf_area); size_t used_bytes = 0; trace_dump_core(trace_buf_area, CONSOLE_DUMP_BUFLEN, &used_bytes, NULL, disp_get_core_id(), true, true); printf("\n%s\n", "dump trac buffers: Start"); printf("\n%s\n", trace_buf_area); printf("\n%s\n", "dump trac buffers: Stop"); trace_reset_all(); // abort(); printf("Cache loading time %"PU", %"PRIu64"\n", in_seconds(total_loading_time), total_loading_time); #endif // ENABLE_WEB_TRACING /* continue with the web-server initialization. */ init_callback(); /* do remaining initialization! */ } static void initial_cache_load(struct nfs_client *client) { errval_t r; cache_loading_phase = true; cache_lookups_started = 0; cache_loaded_counter = 0; cache_loading_probs = 0; //my_nfs_client r = nfs_readdir(client, nfs_root_fh, NFS_READDIR_COOKIE, NFS_READDIR_COOKIEVERF, readdir_callback, NULL); assert(r == SYS_ERR_OK); } #endif // PRELOAD_WEB_CACHE static void mount_callback(void *arg, struct nfs_client *client, enum mountstat3 mountstat, struct nfs_fh3 fhandle) { assert(mountstat == MNT3_OK); nfs_root_fh = fhandle; /* GLOBAL: assigning root of filesystem handle */ #ifdef PRELOAD_WEB_CACHE DEBUGPRINT ("nfs_mount successful, loading initial cache.\n"); initial_cache_load(client); /* Initial load of files for cache. */ #else // PRELOAD_WEB_CACHE DEBUGPRINT ("nfs_mount successful\n"); init_callback(); /* done! */ #endif // PRELOAD_WEB_CACHE } /* end function: mount_callback */ errval_t http_cache_init(struct in_addr server, const char *path, void (*callback)(void)) { struct timer *cache_timer; /* timer for triggering cache timeouts */ init_callback = callback; DEBUGPRINT ("nfs_mount calling.\n"); my_nfs_client = nfs_mount(server, path, mount_callback, NULL); DEBUGPRINT ("nfs_mount calling done.\n"); assert(my_nfs_client != NULL); /* creating the empty cache */ cache_table = NULL; create_404_page_cache(); cache_timer = timer_create(MAX_STALENESS, true, cache_timeout_event, cache_table); assert (cache_timer != NULL); if (cache_timer == NULL) { printf ("http_cache_init failed in timer_create\n"); return LWIP_ERR_MEM; } timer_start(cache_timer); DEBUGPRINT ("http_cache_init done\n"); return SYS_ERR_OK; } /* end function: http_cache_init */