1251881Speter/* 2251881Speter * cache-memcache.c: memcached caching for Subversion 3251881Speter * 4251881Speter * ==================================================================== 5251881Speter * Licensed to the Apache Software Foundation (ASF) under one 6251881Speter * or more contributor license agreements. See the NOTICE file 7251881Speter * distributed with this work for additional information 8251881Speter * regarding copyright ownership. The ASF licenses this file 9251881Speter * to you under the Apache License, Version 2.0 (the 10251881Speter * "License"); you may not use this file except in compliance 11251881Speter * with the License. You may obtain a copy of the License at 12251881Speter * 13251881Speter * http://www.apache.org/licenses/LICENSE-2.0 14251881Speter * 15251881Speter * Unless required by applicable law or agreed to in writing, 16251881Speter * software distributed under the License is distributed on an 17251881Speter * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 18251881Speter * KIND, either express or implied. See the License for the 19251881Speter * specific language governing permissions and limitations 20251881Speter * under the License. 21251881Speter * ==================================================================== 22251881Speter */ 23251881Speter 24251881Speter#include <apr_md5.h> 25251881Speter 26251881Speter#include "svn_pools.h" 27251881Speter#include "svn_base64.h" 28251881Speter#include "svn_path.h" 29251881Speter 30251881Speter#include "svn_private_config.h" 31251881Speter#include "private/svn_cache.h" 32251881Speter#include "private/svn_dep_compat.h" 33251881Speter 34251881Speter#include "cache.h" 35251881Speter 36251881Speter#ifdef SVN_HAVE_MEMCACHE 37251881Speter 38251881Speter#include <apr_memcache.h> 39251881Speter 40251881Speter/* A note on thread safety: 41251881Speter 42251881Speter The apr_memcache_t object does its own mutex handling, and nothing 43251881Speter else in memcache_t is ever modified, so this implementation should 44251881Speter be fully thread-safe. 45251881Speter*/ 46251881Speter 47251881Speter/* The (internal) cache object. */ 48251881Spetertypedef struct memcache_t { 49251881Speter /* The memcached server set we're using. */ 50251881Speter apr_memcache_t *memcache; 51251881Speter 52251881Speter /* A prefix used to differentiate our data from any other data in 53251881Speter * the memcached (URI-encoded). */ 54251881Speter const char *prefix; 55251881Speter 56251881Speter /* The size of the key: either a fixed number of bytes or 57251881Speter * APR_HASH_KEY_STRING. */ 58251881Speter apr_ssize_t klen; 59251881Speter 60251881Speter 61251881Speter /* Used to marshal values in and out of the cache. */ 62251881Speter svn_cache__serialize_func_t serialize_func; 63251881Speter svn_cache__deserialize_func_t deserialize_func; 64251881Speter} memcache_t; 65251881Speter 66251881Speter/* The wrapper around apr_memcache_t. */ 67251881Speterstruct svn_memcache_t { 68251881Speter apr_memcache_t *c; 69251881Speter}; 70251881Speter 71251881Speter 72251881Speter/* The memcached protocol says the maximum key length is 250. Let's 73251881Speter just say 249, to be safe. */ 74251881Speter#define MAX_MEMCACHED_KEY_LEN 249 75251881Speter#define MEMCACHED_KEY_UNHASHED_LEN (MAX_MEMCACHED_KEY_LEN - \ 76251881Speter 2 * APR_MD5_DIGESTSIZE) 77251881Speter 78251881Speter 79251881Speter/* Set *MC_KEY to a memcache key for the given key KEY for CACHE, allocated 80251881Speter in POOL. */ 81251881Speterstatic svn_error_t * 82251881Speterbuild_key(const char **mc_key, 83251881Speter memcache_t *cache, 84251881Speter const void *raw_key, 85251881Speter apr_pool_t *pool) 86251881Speter{ 87251881Speter const char *encoded_suffix; 88251881Speter const char *long_key; 89251881Speter apr_size_t long_key_len; 90251881Speter 91251881Speter if (cache->klen == APR_HASH_KEY_STRING) 92251881Speter encoded_suffix = svn_path_uri_encode(raw_key, pool); 93251881Speter else 94251881Speter { 95251881Speter const svn_string_t *raw = svn_string_ncreate(raw_key, cache->klen, pool); 96251881Speter const svn_string_t *encoded = svn_base64_encode_string2(raw, FALSE, 97251881Speter pool); 98251881Speter encoded_suffix = encoded->data; 99251881Speter } 100251881Speter 101251881Speter long_key = apr_pstrcat(pool, "SVN:", cache->prefix, ":", encoded_suffix, 102251881Speter (char *)NULL); 103251881Speter long_key_len = strlen(long_key); 104251881Speter 105251881Speter /* We don't want to have a key that's too big. If it was going to 106251881Speter be too big, we MD5 the entire string, then replace the last bit 107251881Speter with the checksum. Note that APR_MD5_DIGESTSIZE is for the pure 108251881Speter binary digest; we have to double that when we convert to hex. 109251881Speter 110251881Speter Every key we use will either be at most 111251881Speter MEMCACHED_KEY_UNHASHED_LEN bytes long, or be exactly 112251881Speter MAX_MEMCACHED_KEY_LEN bytes long. */ 113251881Speter if (long_key_len > MEMCACHED_KEY_UNHASHED_LEN) 114251881Speter { 115251881Speter svn_checksum_t *checksum; 116251881Speter SVN_ERR(svn_checksum(&checksum, svn_checksum_md5, long_key, long_key_len, 117251881Speter pool)); 118251881Speter 119251881Speter long_key = apr_pstrcat(pool, 120251881Speter apr_pstrmemdup(pool, long_key, 121251881Speter MEMCACHED_KEY_UNHASHED_LEN), 122251881Speter svn_checksum_to_cstring_display(checksum, pool), 123251881Speter (char *)NULL); 124251881Speter } 125251881Speter 126251881Speter *mc_key = long_key; 127251881Speter return SVN_NO_ERROR; 128251881Speter} 129251881Speter 130251881Speter/* Core functionality of our getter functions: fetch DATA from the memcached 131251881Speter * given by CACHE_VOID and identified by KEY. Indicate success in FOUND and 132251881Speter * use a tempoary sub-pool of POOL for allocations. 133251881Speter */ 134251881Speterstatic svn_error_t * 135251881Spetermemcache_internal_get(char **data, 136251881Speter apr_size_t *size, 137251881Speter svn_boolean_t *found, 138251881Speter void *cache_void, 139251881Speter const void *key, 140251881Speter apr_pool_t *pool) 141251881Speter{ 142251881Speter memcache_t *cache = cache_void; 143251881Speter apr_status_t apr_err; 144251881Speter const char *mc_key; 145251881Speter apr_pool_t *subpool; 146251881Speter 147251881Speter if (key == NULL) 148251881Speter { 149251881Speter *found = FALSE; 150251881Speter return SVN_NO_ERROR; 151251881Speter } 152251881Speter 153251881Speter subpool = svn_pool_create(pool); 154251881Speter SVN_ERR(build_key(&mc_key, cache, key, subpool)); 155251881Speter 156251881Speter apr_err = apr_memcache_getp(cache->memcache, 157251881Speter pool, 158251881Speter mc_key, 159251881Speter data, 160251881Speter size, 161251881Speter NULL /* ignore flags */); 162251881Speter if (apr_err == APR_NOTFOUND) 163251881Speter { 164251881Speter *found = FALSE; 165251881Speter svn_pool_destroy(subpool); 166251881Speter return SVN_NO_ERROR; 167251881Speter } 168251881Speter else if (apr_err != APR_SUCCESS || !*data) 169251881Speter return svn_error_wrap_apr(apr_err, 170251881Speter _("Unknown memcached error while reading")); 171251881Speter 172251881Speter *found = TRUE; 173251881Speter 174251881Speter svn_pool_destroy(subpool); 175251881Speter return SVN_NO_ERROR; 176251881Speter} 177251881Speter 178251881Speter 179251881Speterstatic svn_error_t * 180251881Spetermemcache_get(void **value_p, 181251881Speter svn_boolean_t *found, 182251881Speter void *cache_void, 183251881Speter const void *key, 184251881Speter apr_pool_t *result_pool) 185251881Speter{ 186251881Speter memcache_t *cache = cache_void; 187251881Speter char *data; 188251881Speter apr_size_t data_len; 189251881Speter SVN_ERR(memcache_internal_get(&data, 190251881Speter &data_len, 191251881Speter found, 192251881Speter cache_void, 193251881Speter key, 194251881Speter result_pool)); 195251881Speter 196251881Speter /* If we found it, de-serialize it. */ 197251881Speter if (*found) 198251881Speter { 199251881Speter if (cache->deserialize_func) 200251881Speter { 201251881Speter SVN_ERR((cache->deserialize_func)(value_p, data, data_len, 202251881Speter result_pool)); 203251881Speter } 204251881Speter else 205251881Speter { 206269847Speter svn_stringbuf_t *value = svn_stringbuf_create_empty(result_pool); 207251881Speter value->data = data; 208269847Speter value->blocksize = data_len; 209269847Speter value->len = data_len - 1; /* account for trailing NUL */ 210251881Speter *value_p = value; 211251881Speter } 212251881Speter } 213251881Speter 214251881Speter return SVN_NO_ERROR; 215251881Speter} 216251881Speter 217251881Speter/* Core functionality of our setter functions: store LENGH bytes of DATA 218251881Speter * to be identified by KEY in the memcached given by CACHE_VOID. Use POOL 219251881Speter * for temporary allocations. 220251881Speter */ 221251881Speterstatic svn_error_t * 222251881Spetermemcache_internal_set(void *cache_void, 223251881Speter const void *key, 224251881Speter const char *data, 225251881Speter apr_size_t len, 226251881Speter apr_pool_t *scratch_pool) 227251881Speter{ 228251881Speter memcache_t *cache = cache_void; 229251881Speter const char *mc_key; 230251881Speter apr_status_t apr_err; 231251881Speter 232251881Speter SVN_ERR(build_key(&mc_key, cache, key, scratch_pool)); 233251881Speter apr_err = apr_memcache_set(cache->memcache, mc_key, (char *)data, len, 0, 0); 234251881Speter 235251881Speter /* ### Maybe write failures should be ignored (but logged)? */ 236251881Speter if (apr_err != APR_SUCCESS) 237251881Speter return svn_error_wrap_apr(apr_err, 238251881Speter _("Unknown memcached error while writing")); 239251881Speter 240251881Speter return SVN_NO_ERROR; 241251881Speter} 242251881Speter 243251881Speter 244251881Speterstatic svn_error_t * 245251881Spetermemcache_set(void *cache_void, 246251881Speter const void *key, 247251881Speter void *value, 248251881Speter apr_pool_t *scratch_pool) 249251881Speter{ 250251881Speter memcache_t *cache = cache_void; 251251881Speter apr_pool_t *subpool = svn_pool_create(scratch_pool); 252251881Speter void *data; 253251881Speter apr_size_t data_len; 254251881Speter svn_error_t *err; 255251881Speter 256251881Speter if (key == NULL) 257251881Speter return SVN_NO_ERROR; 258251881Speter 259251881Speter if (cache->serialize_func) 260251881Speter { 261251881Speter SVN_ERR((cache->serialize_func)(&data, &data_len, value, subpool)); 262251881Speter } 263251881Speter else 264251881Speter { 265251881Speter svn_stringbuf_t *value_str = value; 266251881Speter data = value_str->data; 267269847Speter data_len = value_str->len + 1; /* copy trailing NUL */ 268251881Speter } 269251881Speter 270251881Speter err = memcache_internal_set(cache_void, key, data, data_len, subpool); 271251881Speter 272251881Speter svn_pool_destroy(subpool); 273251881Speter return err; 274251881Speter} 275251881Speter 276251881Speterstatic svn_error_t * 277251881Spetermemcache_get_partial(void **value_p, 278251881Speter svn_boolean_t *found, 279251881Speter void *cache_void, 280251881Speter const void *key, 281251881Speter svn_cache__partial_getter_func_t func, 282251881Speter void *baton, 283251881Speter apr_pool_t *result_pool) 284251881Speter{ 285251881Speter svn_error_t *err = SVN_NO_ERROR; 286251881Speter 287251881Speter char *data; 288251881Speter apr_size_t size; 289251881Speter SVN_ERR(memcache_internal_get(&data, 290251881Speter &size, 291251881Speter found, 292251881Speter cache_void, 293251881Speter key, 294251881Speter result_pool)); 295251881Speter 296251881Speter /* If we found it, de-serialize it. */ 297251881Speter return *found 298251881Speter ? func(value_p, data, size, baton, result_pool) 299251881Speter : err; 300251881Speter} 301251881Speter 302251881Speter 303251881Speterstatic svn_error_t * 304251881Spetermemcache_set_partial(void *cache_void, 305251881Speter const void *key, 306251881Speter svn_cache__partial_setter_func_t func, 307251881Speter void *baton, 308251881Speter apr_pool_t *scratch_pool) 309251881Speter{ 310251881Speter svn_error_t *err = SVN_NO_ERROR; 311251881Speter 312251881Speter void *data; 313251881Speter apr_size_t size; 314251881Speter svn_boolean_t found = FALSE; 315251881Speter 316251881Speter apr_pool_t *subpool = svn_pool_create(scratch_pool); 317251881Speter SVN_ERR(memcache_internal_get((char **)&data, 318251881Speter &size, 319251881Speter &found, 320251881Speter cache_void, 321251881Speter key, 322251881Speter subpool)); 323251881Speter 324251881Speter /* If we found it, modify it and write it back to cache */ 325251881Speter if (found) 326251881Speter { 327251881Speter SVN_ERR(func(&data, &size, baton, subpool)); 328251881Speter err = memcache_internal_set(cache_void, key, data, size, subpool); 329251881Speter } 330251881Speter 331251881Speter svn_pool_destroy(subpool); 332251881Speter return err; 333251881Speter} 334251881Speter 335251881Speter 336251881Speterstatic svn_error_t * 337251881Spetermemcache_iter(svn_boolean_t *completed, 338251881Speter void *cache_void, 339251881Speter svn_iter_apr_hash_cb_t user_cb, 340251881Speter void *user_baton, 341251881Speter apr_pool_t *scratch_pool) 342251881Speter{ 343251881Speter return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL, 344251881Speter _("Can't iterate a memcached cache")); 345251881Speter} 346251881Speter 347251881Speterstatic svn_boolean_t 348251881Spetermemcache_is_cachable(void *unused, apr_size_t size) 349251881Speter{ 350251881Speter (void)unused; /* silence gcc warning. */ 351251881Speter 352251881Speter /* The memcached cutoff seems to be a bit (header length?) under a megabyte. 353251881Speter * We round down a little to be safe. 354251881Speter */ 355251881Speter return size < 1000000; 356251881Speter} 357251881Speter 358251881Speterstatic svn_error_t * 359251881Spetermemcache_get_info(void *cache_void, 360251881Speter svn_cache__info_t *info, 361251881Speter svn_boolean_t reset, 362251881Speter apr_pool_t *result_pool) 363251881Speter{ 364251881Speter memcache_t *cache = cache_void; 365251881Speter 366251881Speter info->id = apr_pstrdup(result_pool, cache->prefix); 367251881Speter 368251881Speter /* we don't have any memory allocation info */ 369251881Speter 370251881Speter info->used_size = 0; 371251881Speter info->total_size = 0; 372251881Speter info->data_size = 0; 373251881Speter info->used_entries = 0; 374251881Speter info->total_entries = 0; 375251881Speter 376251881Speter return SVN_NO_ERROR; 377251881Speter} 378251881Speter 379251881Speterstatic svn_cache__vtable_t memcache_vtable = { 380251881Speter memcache_get, 381251881Speter memcache_set, 382251881Speter memcache_iter, 383251881Speter memcache_is_cachable, 384251881Speter memcache_get_partial, 385251881Speter memcache_set_partial, 386251881Speter memcache_get_info 387251881Speter}; 388251881Speter 389251881Spetersvn_error_t * 390251881Spetersvn_cache__create_memcache(svn_cache__t **cache_p, 391251881Speter svn_memcache_t *memcache, 392251881Speter svn_cache__serialize_func_t serialize_func, 393251881Speter svn_cache__deserialize_func_t deserialize_func, 394251881Speter apr_ssize_t klen, 395251881Speter const char *prefix, 396251881Speter apr_pool_t *pool) 397251881Speter{ 398251881Speter svn_cache__t *wrapper = apr_pcalloc(pool, sizeof(*wrapper)); 399251881Speter memcache_t *cache = apr_pcalloc(pool, sizeof(*cache)); 400251881Speter 401251881Speter cache->serialize_func = serialize_func; 402251881Speter cache->deserialize_func = deserialize_func; 403251881Speter cache->klen = klen; 404251881Speter cache->prefix = svn_path_uri_encode(prefix, pool); 405251881Speter cache->memcache = memcache->c; 406251881Speter 407251881Speter wrapper->vtable = &memcache_vtable; 408251881Speter wrapper->cache_internal = cache; 409251881Speter wrapper->error_handler = 0; 410251881Speter wrapper->error_baton = 0; 411251881Speter 412251881Speter *cache_p = wrapper; 413251881Speter return SVN_NO_ERROR; 414251881Speter} 415251881Speter 416251881Speter 417251881Speter/*** Creating apr_memcache_t from svn_config_t. ***/ 418251881Speter 419251881Speter/* Baton for add_memcache_server. */ 420251881Speterstruct ams_baton { 421251881Speter apr_memcache_t *memcache; 422251881Speter apr_pool_t *memcache_pool; 423251881Speter svn_error_t *err; 424251881Speter}; 425251881Speter 426251881Speter/* Implements svn_config_enumerator2_t. */ 427251881Speterstatic svn_boolean_t 428251881Speteradd_memcache_server(const char *name, 429251881Speter const char *value, 430251881Speter void *baton, 431251881Speter apr_pool_t *pool) 432251881Speter{ 433251881Speter struct ams_baton *b = baton; 434251881Speter char *host, *scope; 435251881Speter apr_port_t port; 436251881Speter apr_status_t apr_err; 437251881Speter apr_memcache_server_t *server; 438251881Speter 439251881Speter apr_err = apr_parse_addr_port(&host, &scope, &port, 440251881Speter value, pool); 441251881Speter if (apr_err != APR_SUCCESS) 442251881Speter { 443251881Speter b->err = svn_error_wrap_apr(apr_err, 444251881Speter _("Error parsing memcache server '%s'"), 445251881Speter name); 446251881Speter return FALSE; 447251881Speter } 448251881Speter 449251881Speter if (scope) 450251881Speter { 451251881Speter b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL, 452251881Speter _("Scope not allowed in memcache server " 453251881Speter "'%s'"), 454251881Speter name); 455251881Speter return FALSE; 456251881Speter } 457251881Speter if (!host || !port) 458251881Speter { 459251881Speter b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL, 460251881Speter _("Must specify host and port for memcache " 461251881Speter "server '%s'"), 462251881Speter name); 463251881Speter return FALSE; 464251881Speter } 465251881Speter 466251881Speter /* Note: the four numbers here are only relevant when an 467251881Speter apr_memcache_t is being shared by multiple threads. */ 468251881Speter apr_err = apr_memcache_server_create(b->memcache_pool, 469251881Speter host, 470251881Speter port, 471251881Speter 0, /* min connections */ 472251881Speter 5, /* soft max connections */ 473251881Speter 10, /* hard max connections */ 474251881Speter /* time to live (in microseconds) */ 475251881Speter apr_time_from_sec(50), 476251881Speter &server); 477251881Speter if (apr_err != APR_SUCCESS) 478251881Speter { 479251881Speter b->err = svn_error_wrap_apr(apr_err, 480251881Speter _("Unknown error creating memcache server")); 481251881Speter return FALSE; 482251881Speter } 483251881Speter 484251881Speter apr_err = apr_memcache_add_server(b->memcache, server); 485251881Speter if (apr_err != APR_SUCCESS) 486251881Speter { 487251881Speter b->err = svn_error_wrap_apr(apr_err, 488251881Speter _("Unknown error adding server to memcache")); 489251881Speter return FALSE; 490251881Speter } 491251881Speter 492251881Speter return TRUE; 493251881Speter} 494251881Speter 495251881Speter#else /* ! SVN_HAVE_MEMCACHE */ 496251881Speter 497251881Speter/* Stubs for no apr memcache library. */ 498251881Speter 499251881Speterstruct svn_memcache_t { 500251881Speter void *unused; /* Let's not have a size-zero struct. */ 501251881Speter}; 502251881Speter 503251881Spetersvn_error_t * 504251881Spetersvn_cache__create_memcache(svn_cache__t **cache_p, 505251881Speter svn_memcache_t *memcache, 506251881Speter svn_cache__serialize_func_t serialize_func, 507251881Speter svn_cache__deserialize_func_t deserialize_func, 508251881Speter apr_ssize_t klen, 509251881Speter const char *prefix, 510251881Speter apr_pool_t *pool) 511251881Speter{ 512251881Speter return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL); 513251881Speter} 514251881Speter 515251881Speter#endif /* SVN_HAVE_MEMCACHE */ 516251881Speter 517251881Speter/* Implements svn_config_enumerator2_t. Just used for the 518251881Speter entry-counting return value of svn_config_enumerate2. */ 519251881Speterstatic svn_boolean_t 520251881Speternop_enumerator(const char *name, 521251881Speter const char *value, 522251881Speter void *baton, 523251881Speter apr_pool_t *pool) 524251881Speter{ 525251881Speter return TRUE; 526251881Speter} 527251881Speter 528251881Spetersvn_error_t * 529251881Spetersvn_cache__make_memcache_from_config(svn_memcache_t **memcache_p, 530251881Speter svn_config_t *config, 531251881Speter apr_pool_t *pool) 532251881Speter{ 533251881Speter int server_count; 534251881Speter apr_pool_t *subpool = svn_pool_create(pool); 535251881Speter 536251881Speter server_count = 537251881Speter svn_config_enumerate2(config, 538251881Speter SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS, 539251881Speter nop_enumerator, NULL, subpool); 540251881Speter 541251881Speter if (server_count == 0) 542251881Speter { 543251881Speter *memcache_p = NULL; 544251881Speter svn_pool_destroy(subpool); 545251881Speter return SVN_NO_ERROR; 546251881Speter } 547251881Speter 548251881Speter if (server_count > APR_INT16_MAX) 549251881Speter return svn_error_create(SVN_ERR_TOO_MANY_MEMCACHED_SERVERS, NULL, NULL); 550251881Speter 551251881Speter#ifdef SVN_HAVE_MEMCACHE 552251881Speter { 553251881Speter struct ams_baton b; 554251881Speter svn_memcache_t *memcache = apr_pcalloc(pool, sizeof(*memcache)); 555251881Speter apr_status_t apr_err = apr_memcache_create(pool, 556251881Speter (apr_uint16_t)server_count, 557251881Speter 0, /* flags */ 558251881Speter &(memcache->c)); 559251881Speter if (apr_err != APR_SUCCESS) 560251881Speter return svn_error_wrap_apr(apr_err, 561251881Speter _("Unknown error creating apr_memcache_t")); 562251881Speter 563251881Speter b.memcache = memcache->c; 564251881Speter b.memcache_pool = pool; 565251881Speter b.err = SVN_NO_ERROR; 566251881Speter svn_config_enumerate2(config, 567251881Speter SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS, 568251881Speter add_memcache_server, &b, 569251881Speter subpool); 570251881Speter 571251881Speter if (b.err) 572251881Speter return b.err; 573251881Speter 574251881Speter *memcache_p = memcache; 575251881Speter 576251881Speter svn_pool_destroy(subpool); 577251881Speter return SVN_NO_ERROR; 578251881Speter } 579251881Speter#else /* ! SVN_HAVE_MEMCACHE */ 580251881Speter { 581251881Speter return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL); 582251881Speter } 583251881Speter#endif /* SVN_HAVE_MEMCACHE */ 584251881Speter} 585