cookie.c revision 2908f5e1
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* netfs cookie management
3 *
4 * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 *
7 * See Documentation/filesystems/caching/netfs-api.rst for more information on
8 * the netfs API.
9 */
10
11#define FSCACHE_DEBUG_LEVEL COOKIE
12#include <linux/module.h>
13#include <linux/slab.h>
14#include "internal.h"
15
16struct kmem_cache *fscache_cookie_jar;
17
18static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
19
20#define fscache_cookie_hash_shift 15
21static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
22
23static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
24					    loff_t object_size);
25static int fscache_alloc_object(struct fscache_cache *cache,
26				struct fscache_cookie *cookie);
27static int fscache_attach_object(struct fscache_cookie *cookie,
28				 struct fscache_object *object);
29
30static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
31{
32	struct fscache_object *object;
33	struct hlist_node *o;
34	const u8 *k;
35	unsigned loop;
36
37	pr_err("%c-cookie c=%08x [p=%08x fl=%lx nc=%u na=%u]\n",
38	       prefix,
39	       cookie->debug_id,
40	       cookie->parent ? cookie->parent->debug_id : 0,
41	       cookie->flags,
42	       atomic_read(&cookie->n_children),
43	       atomic_read(&cookie->n_active));
44	pr_err("%c-cookie d=%p{%s} n=%p\n",
45	       prefix,
46	       cookie->def,
47	       cookie->def ? cookie->def->name : "?",
48	       cookie->netfs_data);
49
50	o = READ_ONCE(cookie->backing_objects.first);
51	if (o) {
52		object = hlist_entry(o, struct fscache_object, cookie_link);
53		pr_err("%c-cookie o=%u\n", prefix, object->debug_id);
54	}
55
56	pr_err("%c-key=[%u] '", prefix, cookie->key_len);
57	k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
58		cookie->inline_key : cookie->key;
59	for (loop = 0; loop < cookie->key_len; loop++)
60		pr_cont("%02x", k[loop]);
61	pr_cont("'\n");
62}
63
64void fscache_free_cookie(struct fscache_cookie *cookie)
65{
66	if (cookie) {
67		BUG_ON(!hlist_empty(&cookie->backing_objects));
68		if (cookie->aux_len > sizeof(cookie->inline_aux))
69			kfree(cookie->aux);
70		if (cookie->key_len > sizeof(cookie->inline_key))
71			kfree(cookie->key);
72		kmem_cache_free(fscache_cookie_jar, cookie);
73	}
74}
75
76/*
77 * Set the index key in a cookie.  The cookie struct has space for a 16-byte
78 * key plus length and hash, but if that's not big enough, it's instead a
79 * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
80 * the key data.
81 */
82static int fscache_set_key(struct fscache_cookie *cookie,
83			   const void *index_key, size_t index_key_len)
84{
85	unsigned long long h;
86	u32 *buf;
87	int bufs;
88	int i;
89
90	bufs = DIV_ROUND_UP(index_key_len, sizeof(*buf));
91
92	if (index_key_len > sizeof(cookie->inline_key)) {
93		buf = kcalloc(bufs, sizeof(*buf), GFP_KERNEL);
94		if (!buf)
95			return -ENOMEM;
96		cookie->key = buf;
97	} else {
98		buf = (u32 *)cookie->inline_key;
99	}
100
101	memcpy(buf, index_key, index_key_len);
102
103	/* Calculate a hash and combine this with the length in the first word
104	 * or first half word
105	 */
106	h = (unsigned long)cookie->parent;
107	h += index_key_len + cookie->type;
108
109	for (i = 0; i < bufs; i++)
110		h += buf[i];
111
112	cookie->key_hash = h ^ (h >> 32);
113	return 0;
114}
115
116static long fscache_compare_cookie(const struct fscache_cookie *a,
117				   const struct fscache_cookie *b)
118{
119	const void *ka, *kb;
120
121	if (a->key_hash != b->key_hash)
122		return (long)a->key_hash - (long)b->key_hash;
123	if (a->parent != b->parent)
124		return (long)a->parent - (long)b->parent;
125	if (a->key_len != b->key_len)
126		return (long)a->key_len - (long)b->key_len;
127	if (a->type != b->type)
128		return (long)a->type - (long)b->type;
129
130	if (a->key_len <= sizeof(a->inline_key)) {
131		ka = &a->inline_key;
132		kb = &b->inline_key;
133	} else {
134		ka = a->key;
135		kb = b->key;
136	}
137	return memcmp(ka, kb, a->key_len);
138}
139
140static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1);
141
142/*
143 * Allocate a cookie.
144 */
145struct fscache_cookie *fscache_alloc_cookie(
146	struct fscache_cookie *parent,
147	const struct fscache_cookie_def *def,
148	const void *index_key, size_t index_key_len,
149	const void *aux_data, size_t aux_data_len,
150	void *netfs_data,
151	loff_t object_size)
152{
153	struct fscache_cookie *cookie;
154
155	/* allocate and initialise a cookie */
156	cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);
157	if (!cookie)
158		return NULL;
159
160	cookie->key_len = index_key_len;
161	cookie->aux_len = aux_data_len;
162
163	if (fscache_set_key(cookie, index_key, index_key_len) < 0)
164		goto nomem;
165
166	if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
167		memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
168	} else {
169		cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
170		if (!cookie->aux)
171			goto nomem;
172	}
173
174	atomic_set(&cookie->usage, 1);
175	atomic_set(&cookie->n_children, 0);
176	cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id);
177
178	/* We keep the active count elevated until relinquishment to prevent an
179	 * attempt to wake up every time the object operations queue quiesces.
180	 */
181	atomic_set(&cookie->n_active, 1);
182
183	cookie->def		= def;
184	cookie->parent		= parent;
185	cookie->netfs_data	= netfs_data;
186	cookie->flags		= (1 << FSCACHE_COOKIE_NO_DATA_YET);
187	cookie->type		= def->type;
188	spin_lock_init(&cookie->lock);
189	spin_lock_init(&cookie->stores_lock);
190	INIT_HLIST_HEAD(&cookie->backing_objects);
191
192	/* radix tree insertion won't use the preallocation pool unless it's
193	 * told it may not wait */
194	INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
195	return cookie;
196
197nomem:
198	fscache_free_cookie(cookie);
199	return NULL;
200}
201
202/*
203 * Attempt to insert the new cookie into the hash.  If there's a collision, we
204 * return the old cookie if it's not in use and an error otherwise.
205 */
206struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate)
207{
208	struct fscache_cookie *cursor;
209	struct hlist_bl_head *h;
210	struct hlist_bl_node *p;
211	unsigned int bucket;
212
213	bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
214	h = &fscache_cookie_hash[bucket];
215
216	hlist_bl_lock(h);
217	hlist_bl_for_each_entry(cursor, p, h, hash_link) {
218		if (fscache_compare_cookie(candidate, cursor) == 0)
219			goto collision;
220	}
221
222	__set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags);
223	fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent);
224	atomic_inc(&candidate->parent->n_children);
225	hlist_bl_add_head(&candidate->hash_link, h);
226	hlist_bl_unlock(h);
227	return candidate;
228
229collision:
230	if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) {
231		trace_fscache_cookie(cursor, fscache_cookie_collision,
232				     atomic_read(&cursor->usage));
233		pr_err("Duplicate cookie detected\n");
234		fscache_print_cookie(cursor, 'O');
235		fscache_print_cookie(candidate, 'N');
236		hlist_bl_unlock(h);
237		return NULL;
238	}
239
240	fscache_cookie_get(cursor, fscache_cookie_get_reacquire);
241	hlist_bl_unlock(h);
242	return cursor;
243}
244
245/*
246 * request a cookie to represent an object (index, datafile, xattr, etc)
247 * - parent specifies the parent object
248 *   - the top level index cookie for each netfs is stored in the fscache_netfs
249 *     struct upon registration
250 * - def points to the definition
251 * - the netfs_data will be passed to the functions pointed to in *def
252 * - all attached caches will be searched to see if they contain this object
253 * - index objects aren't stored on disk until there's a dependent file that
254 *   needs storing
255 * - other objects are stored in a selected cache immediately, and all the
256 *   indices forming the path to it are instantiated if necessary
257 * - we never let on to the netfs about errors
258 *   - we may set a negative cookie pointer, but that's okay
259 */
260struct fscache_cookie *__fscache_acquire_cookie(
261	struct fscache_cookie *parent,
262	const struct fscache_cookie_def *def,
263	const void *index_key, size_t index_key_len,
264	const void *aux_data, size_t aux_data_len,
265	void *netfs_data,
266	loff_t object_size,
267	bool enable)
268{
269	struct fscache_cookie *candidate, *cookie;
270
271	BUG_ON(!def);
272
273	_enter("{%s},{%s},%p,%u",
274	       parent ? (char *) parent->def->name : "<no-parent>",
275	       def->name, netfs_data, enable);
276
277	if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
278		return NULL;
279	if (!aux_data || !aux_data_len) {
280		aux_data = NULL;
281		aux_data_len = 0;
282	}
283
284	fscache_stat(&fscache_n_acquires);
285
286	/* if there's no parent cookie, then we don't create one here either */
287	if (!parent) {
288		fscache_stat(&fscache_n_acquires_null);
289		_leave(" [no parent]");
290		return NULL;
291	}
292
293	/* validate the definition */
294	BUG_ON(!def->name[0]);
295
296	BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
297	       parent->type != FSCACHE_COOKIE_TYPE_INDEX);
298
299	candidate = fscache_alloc_cookie(parent, def,
300					 index_key, index_key_len,
301					 aux_data, aux_data_len,
302					 netfs_data, object_size);
303	if (!candidate) {
304		fscache_stat(&fscache_n_acquires_oom);
305		_leave(" [ENOMEM]");
306		return NULL;
307	}
308
309	cookie = fscache_hash_cookie(candidate);
310	if (!cookie) {
311		trace_fscache_cookie(candidate, fscache_cookie_discard, 1);
312		goto out;
313	}
314
315	if (cookie == candidate)
316		candidate = NULL;
317
318	switch (cookie->type) {
319	case FSCACHE_COOKIE_TYPE_INDEX:
320		fscache_stat(&fscache_n_cookie_index);
321		break;
322	case FSCACHE_COOKIE_TYPE_DATAFILE:
323		fscache_stat(&fscache_n_cookie_data);
324		break;
325	default:
326		fscache_stat(&fscache_n_cookie_special);
327		break;
328	}
329
330	trace_fscache_acquire(cookie);
331
332	if (enable) {
333		/* if the object is an index then we need do nothing more here
334		 * - we create indices on disk when we need them as an index
335		 * may exist in multiple caches */
336		if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
337			if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) {
338				set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
339			} else {
340				atomic_dec(&parent->n_children);
341				fscache_cookie_put(cookie,
342						   fscache_cookie_put_acquire_nobufs);
343				fscache_stat(&fscache_n_acquires_nobufs);
344				_leave(" = NULL");
345				return NULL;
346			}
347		} else {
348			set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
349		}
350	}
351
352	fscache_stat(&fscache_n_acquires_ok);
353
354out:
355	fscache_free_cookie(candidate);
356	return cookie;
357}
358EXPORT_SYMBOL(__fscache_acquire_cookie);
359
360/*
361 * Enable a cookie to permit it to accept new operations.
362 */
363void __fscache_enable_cookie(struct fscache_cookie *cookie,
364			     const void *aux_data,
365			     loff_t object_size,
366			     bool (*can_enable)(void *data),
367			     void *data)
368{
369	_enter("%p", cookie);
370
371	trace_fscache_enable(cookie);
372
373	wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
374			 TASK_UNINTERRUPTIBLE);
375
376	fscache_update_aux(cookie, aux_data);
377
378	if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
379		goto out_unlock;
380
381	if (can_enable && !can_enable(data)) {
382		/* The netfs decided it didn't want to enable after all */
383	} else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
384		/* Wait for outstanding disablement to complete */
385		__fscache_wait_on_invalidate(cookie);
386
387		if (fscache_acquire_non_index_cookie(cookie, object_size) == 0)
388			set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
389	} else {
390		set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
391	}
392
393out_unlock:
394	clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
395	wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
396}
397EXPORT_SYMBOL(__fscache_enable_cookie);
398
399/*
400 * acquire a non-index cookie
401 * - this must make sure the index chain is instantiated and instantiate the
402 *   object representation too
403 */
404static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
405					    loff_t object_size)
406{
407	struct fscache_object *object;
408	struct fscache_cache *cache;
409	int ret;
410
411	_enter("");
412
413	set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);
414
415	/* now we need to see whether the backing objects for this cookie yet
416	 * exist, if not there'll be nothing to search */
417	down_read(&fscache_addremove_sem);
418
419	if (list_empty(&fscache_cache_list)) {
420		up_read(&fscache_addremove_sem);
421		_leave(" = 0 [no caches]");
422		return 0;
423	}
424
425	/* select a cache in which to store the object */
426	cache = fscache_select_cache_for_object(cookie->parent);
427	if (!cache) {
428		up_read(&fscache_addremove_sem);
429		fscache_stat(&fscache_n_acquires_no_cache);
430		_leave(" = -ENOMEDIUM [no cache]");
431		return -ENOMEDIUM;
432	}
433
434	_debug("cache %s", cache->tag->name);
435
436	set_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);
437
438	/* ask the cache to allocate objects for this cookie and its parent
439	 * chain */
440	ret = fscache_alloc_object(cache, cookie);
441	if (ret < 0) {
442		up_read(&fscache_addremove_sem);
443		_leave(" = %d", ret);
444		return ret;
445	}
446
447	spin_lock(&cookie->lock);
448	if (hlist_empty(&cookie->backing_objects)) {
449		spin_unlock(&cookie->lock);
450		goto unavailable;
451	}
452
453	object = hlist_entry(cookie->backing_objects.first,
454			     struct fscache_object, cookie_link);
455
456	fscache_set_store_limit(object, object_size);
457
458	/* initiate the process of looking up all the objects in the chain
459	 * (done by fscache_initialise_object()) */
460	fscache_raise_event(object, FSCACHE_OBJECT_EV_NEW_CHILD);
461
462	spin_unlock(&cookie->lock);
463
464	/* we may be required to wait for lookup to complete at this point */
465	if (!fscache_defer_lookup) {
466		_debug("non-deferred lookup %p", &cookie->flags);
467		wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
468			    TASK_UNINTERRUPTIBLE);
469		_debug("complete");
470		if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags))
471			goto unavailable;
472	}
473
474	up_read(&fscache_addremove_sem);
475	_leave(" = 0 [deferred]");
476	return 0;
477
478unavailable:
479	up_read(&fscache_addremove_sem);
480	_leave(" = -ENOBUFS");
481	return -ENOBUFS;
482}
483
484/*
485 * recursively allocate cache object records for a cookie/cache combination
486 * - caller must be holding the addremove sem
487 */
488static int fscache_alloc_object(struct fscache_cache *cache,
489				struct fscache_cookie *cookie)
490{
491	struct fscache_object *object;
492	int ret;
493
494	_enter("%p,%p{%s}", cache, cookie, cookie->def->name);
495
496	spin_lock(&cookie->lock);
497	hlist_for_each_entry(object, &cookie->backing_objects,
498			     cookie_link) {
499		if (object->cache == cache)
500			goto object_already_extant;
501	}
502	spin_unlock(&cookie->lock);
503
504	/* ask the cache to allocate an object (we may end up with duplicate
505	 * objects at this stage, but we sort that out later) */
506	fscache_stat(&fscache_n_cop_alloc_object);
507	object = cache->ops->alloc_object(cache, cookie);
508	fscache_stat_d(&fscache_n_cop_alloc_object);
509	if (IS_ERR(object)) {
510		fscache_stat(&fscache_n_object_no_alloc);
511		ret = PTR_ERR(object);
512		goto error;
513	}
514
515	ASSERTCMP(object->cookie, ==, cookie);
516	fscache_stat(&fscache_n_object_alloc);
517
518	object->debug_id = atomic_inc_return(&fscache_object_debug_id);
519
520	_debug("ALLOC OBJ%x: %s {%lx}",
521	       object->debug_id, cookie->def->name, object->events);
522
523	ret = fscache_alloc_object(cache, cookie->parent);
524	if (ret < 0)
525		goto error_put;
526
527	/* only attach if we managed to allocate all we needed, otherwise
528	 * discard the object we just allocated and instead use the one
529	 * attached to the cookie */
530	if (fscache_attach_object(cookie, object) < 0) {
531		fscache_stat(&fscache_n_cop_put_object);
532		cache->ops->put_object(object, fscache_obj_put_attach_fail);
533		fscache_stat_d(&fscache_n_cop_put_object);
534	}
535
536	_leave(" = 0");
537	return 0;
538
539object_already_extant:
540	ret = -ENOBUFS;
541	if (fscache_object_is_dying(object) ||
542	    fscache_cache_is_broken(object)) {
543		spin_unlock(&cookie->lock);
544		goto error;
545	}
546	spin_unlock(&cookie->lock);
547	_leave(" = 0 [found]");
548	return 0;
549
550error_put:
551	fscache_stat(&fscache_n_cop_put_object);
552	cache->ops->put_object(object, fscache_obj_put_alloc_fail);
553	fscache_stat_d(&fscache_n_cop_put_object);
554error:
555	_leave(" = %d", ret);
556	return ret;
557}
558
559/*
560 * attach a cache object to a cookie
561 */
562static int fscache_attach_object(struct fscache_cookie *cookie,
563				 struct fscache_object *object)
564{
565	struct fscache_object *p;
566	struct fscache_cache *cache = object->cache;
567	int ret;
568
569	_enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id);
570
571	ASSERTCMP(object->cookie, ==, cookie);
572
573	spin_lock(&cookie->lock);
574
575	/* there may be multiple initial creations of this object, but we only
576	 * want one */
577	ret = -EEXIST;
578	hlist_for_each_entry(p, &cookie->backing_objects, cookie_link) {
579		if (p->cache == object->cache) {
580			if (fscache_object_is_dying(p))
581				ret = -ENOBUFS;
582			goto cant_attach_object;
583		}
584	}
585
586	/* pin the parent object */
587	spin_lock_nested(&cookie->parent->lock, 1);
588	hlist_for_each_entry(p, &cookie->parent->backing_objects,
589			     cookie_link) {
590		if (p->cache == object->cache) {
591			if (fscache_object_is_dying(p)) {
592				ret = -ENOBUFS;
593				spin_unlock(&cookie->parent->lock);
594				goto cant_attach_object;
595			}
596			object->parent = p;
597			spin_lock(&p->lock);
598			p->n_children++;
599			spin_unlock(&p->lock);
600			break;
601		}
602	}
603	spin_unlock(&cookie->parent->lock);
604
605	/* attach to the cache's object list */
606	if (list_empty(&object->cache_link)) {
607		spin_lock(&cache->object_list_lock);
608		list_add(&object->cache_link, &cache->object_list);
609		spin_unlock(&cache->object_list_lock);
610	}
611
612	/* Attach to the cookie.  The object already has a ref on it. */
613	hlist_add_head(&object->cookie_link, &cookie->backing_objects);
614
615	fscache_objlist_add(object);
616	ret = 0;
617
618cant_attach_object:
619	spin_unlock(&cookie->lock);
620	_leave(" = %d", ret);
621	return ret;
622}
623
624/*
625 * Invalidate an object.  Callable with spinlocks held.
626 */
627void __fscache_invalidate(struct fscache_cookie *cookie)
628{
629	struct fscache_object *object;
630
631	_enter("{%s}", cookie->def->name);
632
633	fscache_stat(&fscache_n_invalidates);
634
635	/* Only permit invalidation of data files.  Invalidating an index will
636	 * require the caller to release all its attachments to the tree rooted
637	 * there, and if it's doing that, it may as well just retire the
638	 * cookie.
639	 */
640	ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
641
642	/* If there's an object, we tell the object state machine to handle the
643	 * invalidation on our behalf, otherwise there's nothing to do.
644	 */
645	if (!hlist_empty(&cookie->backing_objects)) {
646		spin_lock(&cookie->lock);
647
648		if (fscache_cookie_enabled(cookie) &&
649		    !hlist_empty(&cookie->backing_objects) &&
650		    !test_and_set_bit(FSCACHE_COOKIE_INVALIDATING,
651				      &cookie->flags)) {
652			object = hlist_entry(cookie->backing_objects.first,
653					     struct fscache_object,
654					     cookie_link);
655			if (fscache_object_is_live(object))
656				fscache_raise_event(
657					object, FSCACHE_OBJECT_EV_INVALIDATE);
658		}
659
660		spin_unlock(&cookie->lock);
661	}
662
663	_leave("");
664}
665EXPORT_SYMBOL(__fscache_invalidate);
666
667/*
668 * Wait for object invalidation to complete.
669 */
670void __fscache_wait_on_invalidate(struct fscache_cookie *cookie)
671{
672	_enter("%p", cookie);
673
674	wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING,
675		    TASK_UNINTERRUPTIBLE);
676
677	_leave("");
678}
679EXPORT_SYMBOL(__fscache_wait_on_invalidate);
680
681/*
682 * update the index entries backing a cookie
683 */
684void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data)
685{
686	struct fscache_object *object;
687
688	fscache_stat(&fscache_n_updates);
689
690	if (!cookie) {
691		fscache_stat(&fscache_n_updates_null);
692		_leave(" [no cookie]");
693		return;
694	}
695
696	_enter("{%s}", cookie->def->name);
697
698	spin_lock(&cookie->lock);
699
700	fscache_update_aux(cookie, aux_data);
701
702	if (fscache_cookie_enabled(cookie)) {
703		/* update the index entry on disk in each cache backing this
704		 * cookie.
705		 */
706		hlist_for_each_entry(object,
707				     &cookie->backing_objects, cookie_link) {
708			fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE);
709		}
710	}
711
712	spin_unlock(&cookie->lock);
713	_leave("");
714}
715EXPORT_SYMBOL(__fscache_update_cookie);
716
717/*
718 * Disable a cookie to stop it from accepting new requests from the netfs.
719 */
720void __fscache_disable_cookie(struct fscache_cookie *cookie,
721			      const void *aux_data,
722			      bool invalidate)
723{
724	struct fscache_object *object;
725	bool awaken = false;
726
727	_enter("%p,%u", cookie, invalidate);
728
729	trace_fscache_disable(cookie);
730
731	ASSERTCMP(atomic_read(&cookie->n_active), >, 0);
732
733	if (atomic_read(&cookie->n_children) != 0) {
734		pr_err("Cookie '%s' still has children\n",
735		       cookie->def->name);
736		BUG();
737	}
738
739	wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
740			 TASK_UNINTERRUPTIBLE);
741
742	fscache_update_aux(cookie, aux_data);
743
744	if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
745		goto out_unlock_enable;
746
747	/* If the cookie is being invalidated, wait for that to complete first
748	 * so that we can reuse the flag.
749	 */
750	__fscache_wait_on_invalidate(cookie);
751
752	/* Dispose of the backing objects */
753	set_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags);
754
755	spin_lock(&cookie->lock);
756	if (!hlist_empty(&cookie->backing_objects)) {
757		hlist_for_each_entry(object, &cookie->backing_objects, cookie_link) {
758			if (invalidate)
759				set_bit(FSCACHE_OBJECT_RETIRED, &object->flags);
760			clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
761			fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL);
762		}
763	} else {
764		if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))
765			awaken = true;
766	}
767	spin_unlock(&cookie->lock);
768	if (awaken)
769		wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);
770
771	/* Wait for cessation of activity requiring access to the netfs (when
772	 * n_active reaches 0).  This makes sure outstanding reads and writes
773	 * have completed.
774	 */
775	if (!atomic_dec_and_test(&cookie->n_active)) {
776		wait_var_event(&cookie->n_active,
777			       !atomic_read(&cookie->n_active));
778	}
779
780	/* Make sure any pending writes are cancelled. */
781	if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX)
782		fscache_invalidate_writes(cookie);
783
784	/* Reset the cookie state if it wasn't relinquished */
785	if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) {
786		atomic_inc(&cookie->n_active);
787		set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
788	}
789
790out_unlock_enable:
791	clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
792	wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
793	_leave("");
794}
795EXPORT_SYMBOL(__fscache_disable_cookie);
796
797/*
798 * release a cookie back to the cache
799 * - the object will be marked as recyclable on disk if retire is true
800 * - all dependents of this cookie must have already been unregistered
801 *   (indices/files/pages)
802 */
803void __fscache_relinquish_cookie(struct fscache_cookie *cookie,
804				 const void *aux_data,
805				 bool retire)
806{
807	fscache_stat(&fscache_n_relinquishes);
808	if (retire)
809		fscache_stat(&fscache_n_relinquishes_retire);
810
811	if (!cookie) {
812		fscache_stat(&fscache_n_relinquishes_null);
813		_leave(" [no cookie]");
814		return;
815	}
816
817	_enter("%p{%s,%p,%d},%d",
818	       cookie, cookie->def->name, cookie->netfs_data,
819	       atomic_read(&cookie->n_active), retire);
820
821	trace_fscache_relinquish(cookie, retire);
822
823	/* No further netfs-accessing operations on this cookie permitted */
824	if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags))
825		BUG();
826
827	__fscache_disable_cookie(cookie, aux_data, retire);
828
829	/* Clear pointers back to the netfs */
830	cookie->netfs_data	= NULL;
831	cookie->def		= NULL;
832	BUG_ON(!radix_tree_empty(&cookie->stores));
833
834	if (cookie->parent) {
835		ASSERTCMP(atomic_read(&cookie->parent->usage), >, 0);
836		ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0);
837		atomic_dec(&cookie->parent->n_children);
838	}
839
840	/* Dispose of the netfs's link to the cookie */
841	ASSERTCMP(atomic_read(&cookie->usage), >, 0);
842	fscache_cookie_put(cookie, fscache_cookie_put_relinquish);
843
844	_leave("");
845}
846EXPORT_SYMBOL(__fscache_relinquish_cookie);
847
848/*
849 * Remove a cookie from the hash table.
850 */
851static void fscache_unhash_cookie(struct fscache_cookie *cookie)
852{
853	struct hlist_bl_head *h;
854	unsigned int bucket;
855
856	bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
857	h = &fscache_cookie_hash[bucket];
858
859	hlist_bl_lock(h);
860	hlist_bl_del(&cookie->hash_link);
861	hlist_bl_unlock(h);
862}
863
864/*
865 * Drop a reference to a cookie.
866 */
867void fscache_cookie_put(struct fscache_cookie *cookie,
868			enum fscache_cookie_trace where)
869{
870	struct fscache_cookie *parent;
871	int usage;
872
873	_enter("%p", cookie);
874
875	do {
876		usage = atomic_dec_return(&cookie->usage);
877		trace_fscache_cookie(cookie, where, usage);
878
879		if (usage > 0)
880			return;
881		BUG_ON(usage < 0);
882
883		parent = cookie->parent;
884		fscache_unhash_cookie(cookie);
885		fscache_free_cookie(cookie);
886
887		cookie = parent;
888		where = fscache_cookie_put_parent;
889	} while (cookie);
890
891	_leave("");
892}
893
894/*
895 * check the consistency between the netfs inode and the backing cache
896 *
897 * NOTE: it only serves no-index type
898 */
899int __fscache_check_consistency(struct fscache_cookie *cookie,
900				const void *aux_data)
901{
902	struct fscache_operation *op;
903	struct fscache_object *object;
904	bool wake_cookie = false;
905	int ret;
906
907	_enter("%p,", cookie);
908
909	ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
910
911	if (fscache_wait_for_deferred_lookup(cookie) < 0)
912		return -ERESTARTSYS;
913
914	if (hlist_empty(&cookie->backing_objects))
915		return 0;
916
917	op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
918	if (!op)
919		return -ENOMEM;
920
921	fscache_operation_init(cookie, op, NULL, NULL, NULL);
922	op->flags = FSCACHE_OP_MYTHREAD |
923		(1 << FSCACHE_OP_WAITING) |
924		(1 << FSCACHE_OP_UNUSE_COOKIE);
925	trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency);
926
927	spin_lock(&cookie->lock);
928
929	fscache_update_aux(cookie, aux_data);
930
931	if (!fscache_cookie_enabled(cookie) ||
932	    hlist_empty(&cookie->backing_objects))
933		goto inconsistent;
934	object = hlist_entry(cookie->backing_objects.first,
935			     struct fscache_object, cookie_link);
936	if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
937		goto inconsistent;
938
939	op->debug_id = atomic_inc_return(&fscache_op_debug_id);
940
941	__fscache_use_cookie(cookie);
942	if (fscache_submit_op(object, op) < 0)
943		goto submit_failed;
944
945	/* the work queue now carries its own ref on the object */
946	spin_unlock(&cookie->lock);
947
948	ret = fscache_wait_for_operation_activation(object, op, NULL, NULL);
949	if (ret == 0) {
950		/* ask the cache to honour the operation */
951		ret = object->cache->ops->check_consistency(op);
952		fscache_op_complete(op, false);
953	} else if (ret == -ENOBUFS) {
954		ret = 0;
955	}
956
957	fscache_put_operation(op);
958	_leave(" = %d", ret);
959	return ret;
960
961submit_failed:
962	wake_cookie = __fscache_unuse_cookie(cookie);
963inconsistent:
964	spin_unlock(&cookie->lock);
965	if (wake_cookie)
966		__fscache_wake_unused_cookie(cookie);
967	kfree(op);
968	_leave(" = -ESTALE");
969	return -ESTALE;
970}
971EXPORT_SYMBOL(__fscache_check_consistency);
972