1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_LIST_H
3#define _LINUX_LIST_H
4
5#include <linux/container_of.h>
6#include <linux/types.h>
7#include <linux/stddef.h>
8#include <linux/poison.h>
9#include <linux/const.h>
10
11#include <asm/barrier.h>
12
13/*
14 * Circular doubly linked list implementation.
15 *
16 * Some of the internal functions ("__xxx") are useful when
17 * manipulating whole lists rather than single entries, as
18 * sometimes we already know the next/prev entries and we can
19 * generate better code by using them directly rather than
20 * using the generic single-entry routines.
21 */
22
23#define LIST_HEAD_INIT(name) { &(name), &(name) }
24
25#define LIST_HEAD(name) \
26	struct list_head name = LIST_HEAD_INIT(name)
27
28/**
29 * INIT_LIST_HEAD - Initialize a list_head structure
30 * @list: list_head structure to be initialized.
31 *
32 * Initializes the list_head to point to itself.  If it is a list header,
33 * the result is an empty list.
34 */
35static inline void INIT_LIST_HEAD(struct list_head *list)
36{
37	WRITE_ONCE(list->next, list);
38	list->prev = list;
39}
40
41#ifdef CONFIG_DEBUG_LIST
42extern bool __list_add_valid(struct list_head *new,
43			      struct list_head *prev,
44			      struct list_head *next);
45extern bool __list_del_entry_valid(struct list_head *entry);
46#else
47static inline bool __list_add_valid(struct list_head *new,
48				struct list_head *prev,
49				struct list_head *next)
50{
51	return true;
52}
53static inline bool __list_del_entry_valid(struct list_head *entry)
54{
55	return true;
56}
57#endif
58
59/*
60 * Insert a new entry between two known consecutive entries.
61 *
62 * This is only for internal list manipulation where we know
63 * the prev/next entries already!
64 */
65static inline void __list_add(struct list_head *new,
66			      struct list_head *prev,
67			      struct list_head *next)
68{
69	if (!__list_add_valid(new, prev, next))
70		return;
71
72	next->prev = new;
73	new->next = next;
74	new->prev = prev;
75	WRITE_ONCE(prev->next, new);
76}
77
78/**
79 * list_add - add a new entry
80 * @new: new entry to be added
81 * @head: list head to add it after
82 *
83 * Insert a new entry after the specified head.
84 * This is good for implementing stacks.
85 */
86static inline void list_add(struct list_head *new, struct list_head *head)
87{
88	__list_add(new, head, head->next);
89}
90
91
92/**
93 * list_add_tail - add a new entry
94 * @new: new entry to be added
95 * @head: list head to add it before
96 *
97 * Insert a new entry before the specified head.
98 * This is useful for implementing queues.
99 */
100static inline void list_add_tail(struct list_head *new, struct list_head *head)
101{
102	__list_add(new, head->prev, head);
103}
104
105/*
106 * Delete a list entry by making the prev/next entries
107 * point to each other.
108 *
109 * This is only for internal list manipulation where we know
110 * the prev/next entries already!
111 */
112static inline void __list_del(struct list_head * prev, struct list_head * next)
113{
114	next->prev = prev;
115	WRITE_ONCE(prev->next, next);
116}
117
118/*
119 * Delete a list entry and clear the 'prev' pointer.
120 *
121 * This is a special-purpose list clearing method used in the networking code
122 * for lists allocated as per-cpu, where we don't want to incur the extra
123 * WRITE_ONCE() overhead of a regular list_del_init(). The code that uses this
124 * needs to check the node 'prev' pointer instead of calling list_empty().
125 */
126static inline void __list_del_clearprev(struct list_head *entry)
127{
128	__list_del(entry->prev, entry->next);
129	entry->prev = NULL;
130}
131
132static inline void __list_del_entry(struct list_head *entry)
133{
134	if (!__list_del_entry_valid(entry))
135		return;
136
137	__list_del(entry->prev, entry->next);
138}
139
140/**
141 * list_del - deletes entry from list.
142 * @entry: the element to delete from the list.
143 * Note: list_empty() on entry does not return true after this, the entry is
144 * in an undefined state.
145 */
146static inline void list_del(struct list_head *entry)
147{
148	__list_del_entry(entry);
149	entry->next = LIST_POISON1;
150	entry->prev = LIST_POISON2;
151}
152
153/**
154 * list_replace - replace old entry by new one
155 * @old : the element to be replaced
156 * @new : the new element to insert
157 *
158 * If @old was empty, it will be overwritten.
159 */
160static inline void list_replace(struct list_head *old,
161				struct list_head *new)
162{
163	new->next = old->next;
164	new->next->prev = new;
165	new->prev = old->prev;
166	new->prev->next = new;
167}
168
169/**
170 * list_replace_init - replace old entry by new one and initialize the old one
171 * @old : the element to be replaced
172 * @new : the new element to insert
173 *
174 * If @old was empty, it will be overwritten.
175 */
176static inline void list_replace_init(struct list_head *old,
177				     struct list_head *new)
178{
179	list_replace(old, new);
180	INIT_LIST_HEAD(old);
181}
182
183/**
184 * list_swap - replace entry1 with entry2 and re-add entry1 at entry2's position
185 * @entry1: the location to place entry2
186 * @entry2: the location to place entry1
187 */
188static inline void list_swap(struct list_head *entry1,
189			     struct list_head *entry2)
190{
191	struct list_head *pos = entry2->prev;
192
193	list_del(entry2);
194	list_replace(entry1, entry2);
195	if (pos == entry1)
196		pos = entry2;
197	list_add(entry1, pos);
198}
199
200/**
201 * list_del_init - deletes entry from list and reinitialize it.
202 * @entry: the element to delete from the list.
203 */
204static inline void list_del_init(struct list_head *entry)
205{
206	__list_del_entry(entry);
207	INIT_LIST_HEAD(entry);
208}
209
210/**
211 * list_move - delete from one list and add as another's head
212 * @list: the entry to move
213 * @head: the head that will precede our entry
214 */
215static inline void list_move(struct list_head *list, struct list_head *head)
216{
217	__list_del_entry(list);
218	list_add(list, head);
219}
220
221/**
222 * list_move_tail - delete from one list and add as another's tail
223 * @list: the entry to move
224 * @head: the head that will follow our entry
225 */
226static inline void list_move_tail(struct list_head *list,
227				  struct list_head *head)
228{
229	__list_del_entry(list);
230	list_add_tail(list, head);
231}
232
233/**
234 * list_bulk_move_tail - move a subsection of a list to its tail
235 * @head: the head that will follow our entry
236 * @first: first entry to move
237 * @last: last entry to move, can be the same as first
238 *
239 * Move all entries between @first and including @last before @head.
240 * All three entries must belong to the same linked list.
241 */
242static inline void list_bulk_move_tail(struct list_head *head,
243				       struct list_head *first,
244				       struct list_head *last)
245{
246	first->prev->next = last->next;
247	last->next->prev = first->prev;
248
249	head->prev->next = first;
250	first->prev = head->prev;
251
252	last->next = head;
253	head->prev = last;
254}
255
256/**
257 * list_is_first -- tests whether @list is the first entry in list @head
258 * @list: the entry to test
259 * @head: the head of the list
260 */
261static inline int list_is_first(const struct list_head *list,
262					const struct list_head *head)
263{
264	return list->prev == head;
265}
266
267/**
268 * list_is_last - tests whether @list is the last entry in list @head
269 * @list: the entry to test
270 * @head: the head of the list
271 */
272static inline int list_is_last(const struct list_head *list,
273				const struct list_head *head)
274{
275	return list->next == head;
276}
277
278/**
279 * list_empty - tests whether a list is empty
280 * @head: the list to test.
281 */
282static inline int list_empty(const struct list_head *head)
283{
284	return READ_ONCE(head->next) == head;
285}
286
287/**
288 * list_del_init_careful - deletes entry from list and reinitialize it.
289 * @entry: the element to delete from the list.
290 *
291 * This is the same as list_del_init(), except designed to be used
292 * together with list_empty_careful() in a way to guarantee ordering
293 * of other memory operations.
294 *
295 * Any memory operations done before a list_del_init_careful() are
296 * guaranteed to be visible after a list_empty_careful() test.
297 */
298static inline void list_del_init_careful(struct list_head *entry)
299{
300	__list_del_entry(entry);
301	entry->prev = entry;
302	smp_store_release(&entry->next, entry);
303}
304
305/**
306 * list_empty_careful - tests whether a list is empty and not being modified
307 * @head: the list to test
308 *
309 * Description:
310 * tests whether a list is empty _and_ checks that no other CPU might be
311 * in the process of modifying either member (next or prev)
312 *
313 * NOTE: using list_empty_careful() without synchronization
314 * can only be safe if the only activity that can happen
315 * to the list entry is list_del_init(). Eg. it cannot be used
316 * if another CPU could re-list_add() it.
317 */
318static inline int list_empty_careful(const struct list_head *head)
319{
320	struct list_head *next = smp_load_acquire(&head->next);
321	return (next == head) && (next == head->prev);
322}
323
324/**
325 * list_rotate_left - rotate the list to the left
326 * @head: the head of the list
327 */
328static inline void list_rotate_left(struct list_head *head)
329{
330	struct list_head *first;
331
332	if (!list_empty(head)) {
333		first = head->next;
334		list_move_tail(first, head);
335	}
336}
337
338/**
339 * list_rotate_to_front() - Rotate list to specific item.
340 * @list: The desired new front of the list.
341 * @head: The head of the list.
342 *
343 * Rotates list so that @list becomes the new front of the list.
344 */
345static inline void list_rotate_to_front(struct list_head *list,
346					struct list_head *head)
347{
348	/*
349	 * Deletes the list head from the list denoted by @head and
350	 * places it as the tail of @list, this effectively rotates the
351	 * list so that @list is at the front.
352	 */
353	list_move_tail(head, list);
354}
355
356/**
357 * list_is_singular - tests whether a list has just one entry.
358 * @head: the list to test.
359 */
360static inline int list_is_singular(const struct list_head *head)
361{
362	return !list_empty(head) && (head->next == head->prev);
363}
364
365static inline void __list_cut_position(struct list_head *list,
366		struct list_head *head, struct list_head *entry)
367{
368	struct list_head *new_first = entry->next;
369	list->next = head->next;
370	list->next->prev = list;
371	list->prev = entry;
372	entry->next = list;
373	head->next = new_first;
374	new_first->prev = head;
375}
376
377/**
378 * list_cut_position - cut a list into two
379 * @list: a new list to add all removed entries
380 * @head: a list with entries
381 * @entry: an entry within head, could be the head itself
382 *	and if so we won't cut the list
383 *
384 * This helper moves the initial part of @head, up to and
385 * including @entry, from @head to @list. You should
386 * pass on @entry an element you know is on @head. @list
387 * should be an empty list or a list you do not care about
388 * losing its data.
389 *
390 */
391static inline void list_cut_position(struct list_head *list,
392		struct list_head *head, struct list_head *entry)
393{
394	if (list_empty(head))
395		return;
396	if (list_is_singular(head) &&
397		(head->next != entry && head != entry))
398		return;
399	if (entry == head)
400		INIT_LIST_HEAD(list);
401	else
402		__list_cut_position(list, head, entry);
403}
404
405/**
406 * list_cut_before - cut a list into two, before given entry
407 * @list: a new list to add all removed entries
408 * @head: a list with entries
409 * @entry: an entry within head, could be the head itself
410 *
411 * This helper moves the initial part of @head, up to but
412 * excluding @entry, from @head to @list.  You should pass
413 * in @entry an element you know is on @head.  @list should
414 * be an empty list or a list you do not care about losing
415 * its data.
416 * If @entry == @head, all entries on @head are moved to
417 * @list.
418 */
419static inline void list_cut_before(struct list_head *list,
420				   struct list_head *head,
421				   struct list_head *entry)
422{
423	if (head->next == entry) {
424		INIT_LIST_HEAD(list);
425		return;
426	}
427	list->next = head->next;
428	list->next->prev = list;
429	list->prev = entry->prev;
430	list->prev->next = list;
431	head->next = entry;
432	entry->prev = head;
433}
434
435static inline void __list_splice(const struct list_head *list,
436				 struct list_head *prev,
437				 struct list_head *next)
438{
439	struct list_head *first = list->next;
440	struct list_head *last = list->prev;
441
442	first->prev = prev;
443	prev->next = first;
444
445	last->next = next;
446	next->prev = last;
447}
448
449/**
450 * list_splice - join two lists, this is designed for stacks
451 * @list: the new list to add.
452 * @head: the place to add it in the first list.
453 */
454static inline void list_splice(const struct list_head *list,
455				struct list_head *head)
456{
457	if (!list_empty(list))
458		__list_splice(list, head, head->next);
459}
460
461/**
462 * list_splice_tail - join two lists, each list being a queue
463 * @list: the new list to add.
464 * @head: the place to add it in the first list.
465 */
466static inline void list_splice_tail(struct list_head *list,
467				struct list_head *head)
468{
469	if (!list_empty(list))
470		__list_splice(list, head->prev, head);
471}
472
473/**
474 * list_splice_init - join two lists and reinitialise the emptied list.
475 * @list: the new list to add.
476 * @head: the place to add it in the first list.
477 *
478 * The list at @list is reinitialised
479 */
480static inline void list_splice_init(struct list_head *list,
481				    struct list_head *head)
482{
483	if (!list_empty(list)) {
484		__list_splice(list, head, head->next);
485		INIT_LIST_HEAD(list);
486	}
487}
488
489/**
490 * list_splice_tail_init - join two lists and reinitialise the emptied list
491 * @list: the new list to add.
492 * @head: the place to add it in the first list.
493 *
494 * Each of the lists is a queue.
495 * The list at @list is reinitialised
496 */
497static inline void list_splice_tail_init(struct list_head *list,
498					 struct list_head *head)
499{
500	if (!list_empty(list)) {
501		__list_splice(list, head->prev, head);
502		INIT_LIST_HEAD(list);
503	}
504}
505
506/**
507 * list_entry - get the struct for this entry
508 * @ptr:	the &struct list_head pointer.
509 * @type:	the type of the struct this is embedded in.
510 * @member:	the name of the list_head within the struct.
511 */
512#define list_entry(ptr, type, member) \
513	container_of(ptr, type, member)
514
515/**
516 * list_first_entry - get the first element from a list
517 * @ptr:	the list head to take the element from.
518 * @type:	the type of the struct this is embedded in.
519 * @member:	the name of the list_head within the struct.
520 *
521 * Note, that list is expected to be not empty.
522 */
523#define list_first_entry(ptr, type, member) \
524	list_entry((ptr)->next, type, member)
525
526/**
527 * list_last_entry - get the last element from a list
528 * @ptr:	the list head to take the element from.
529 * @type:	the type of the struct this is embedded in.
530 * @member:	the name of the list_head within the struct.
531 *
532 * Note, that list is expected to be not empty.
533 */
534#define list_last_entry(ptr, type, member) \
535	list_entry((ptr)->prev, type, member)
536
537/**
538 * list_first_entry_or_null - get the first element from a list
539 * @ptr:	the list head to take the element from.
540 * @type:	the type of the struct this is embedded in.
541 * @member:	the name of the list_head within the struct.
542 *
543 * Note that if the list is empty, it returns NULL.
544 */
545#define list_first_entry_or_null(ptr, type, member) ({ \
546	struct list_head *head__ = (ptr); \
547	struct list_head *pos__ = READ_ONCE(head__->next); \
548	pos__ != head__ ? list_entry(pos__, type, member) : NULL; \
549})
550
551/**
552 * list_next_entry - get the next element in list
553 * @pos:	the type * to cursor
554 * @member:	the name of the list_head within the struct.
555 */
556#define list_next_entry(pos, member) \
557	list_entry((pos)->member.next, typeof(*(pos)), member)
558
559/**
560 * list_prev_entry - get the prev element in list
561 * @pos:	the type * to cursor
562 * @member:	the name of the list_head within the struct.
563 */
564#define list_prev_entry(pos, member) \
565	list_entry((pos)->member.prev, typeof(*(pos)), member)
566
567/**
568 * list_for_each	-	iterate over a list
569 * @pos:	the &struct list_head to use as a loop cursor.
570 * @head:	the head for your list.
571 */
572#define list_for_each(pos, head) \
573	for (pos = (head)->next; pos != (head); pos = pos->next)
574
575/**
576 * list_for_each_continue - continue iteration over a list
577 * @pos:	the &struct list_head to use as a loop cursor.
578 * @head:	the head for your list.
579 *
580 * Continue to iterate over a list, continuing after the current position.
581 */
582#define list_for_each_continue(pos, head) \
583	for (pos = pos->next; pos != (head); pos = pos->next)
584
585/**
586 * list_for_each_prev	-	iterate over a list backwards
587 * @pos:	the &struct list_head to use as a loop cursor.
588 * @head:	the head for your list.
589 */
590#define list_for_each_prev(pos, head) \
591	for (pos = (head)->prev; pos != (head); pos = pos->prev)
592
593/**
594 * list_for_each_safe - iterate over a list safe against removal of list entry
595 * @pos:	the &struct list_head to use as a loop cursor.
596 * @n:		another &struct list_head to use as temporary storage
597 * @head:	the head for your list.
598 */
599#define list_for_each_safe(pos, n, head) \
600	for (pos = (head)->next, n = pos->next; pos != (head); \
601		pos = n, n = pos->next)
602
603/**
604 * list_for_each_prev_safe - iterate over a list backwards safe against removal of list entry
605 * @pos:	the &struct list_head to use as a loop cursor.
606 * @n:		another &struct list_head to use as temporary storage
607 * @head:	the head for your list.
608 */
609#define list_for_each_prev_safe(pos, n, head) \
610	for (pos = (head)->prev, n = pos->prev; \
611	     pos != (head); \
612	     pos = n, n = pos->prev)
613
614/**
615 * list_entry_is_head - test if the entry points to the head of the list
616 * @pos:	the type * to cursor
617 * @head:	the head for your list.
618 * @member:	the name of the list_head within the struct.
619 */
620#define list_entry_is_head(pos, head, member)				\
621	(&pos->member == (head))
622
623/**
624 * list_for_each_entry	-	iterate over list of given type
625 * @pos:	the type * to use as a loop cursor.
626 * @head:	the head for your list.
627 * @member:	the name of the list_head within the struct.
628 */
629#define list_for_each_entry(pos, head, member)				\
630	for (pos = list_first_entry(head, typeof(*pos), member);	\
631	     !list_entry_is_head(pos, head, member);			\
632	     pos = list_next_entry(pos, member))
633
634/**
635 * list_for_each_entry_reverse - iterate backwards over list of given type.
636 * @pos:	the type * to use as a loop cursor.
637 * @head:	the head for your list.
638 * @member:	the name of the list_head within the struct.
639 */
640#define list_for_each_entry_reverse(pos, head, member)			\
641	for (pos = list_last_entry(head, typeof(*pos), member);		\
642	     !list_entry_is_head(pos, head, member); 			\
643	     pos = list_prev_entry(pos, member))
644
645/**
646 * list_prepare_entry - prepare a pos entry for use in list_for_each_entry_continue()
647 * @pos:	the type * to use as a start point
648 * @head:	the head of the list
649 * @member:	the name of the list_head within the struct.
650 *
651 * Prepares a pos entry for use as a start point in list_for_each_entry_continue().
652 */
653#define list_prepare_entry(pos, head, member) \
654	((pos) ? : list_entry(head, typeof(*pos), member))
655
656/**
657 * list_for_each_entry_continue - continue iteration over list of given type
658 * @pos:	the type * to use as a loop cursor.
659 * @head:	the head for your list.
660 * @member:	the name of the list_head within the struct.
661 *
662 * Continue to iterate over list of given type, continuing after
663 * the current position.
664 */
665#define list_for_each_entry_continue(pos, head, member) 		\
666	for (pos = list_next_entry(pos, member);			\
667	     !list_entry_is_head(pos, head, member);			\
668	     pos = list_next_entry(pos, member))
669
670/**
671 * list_for_each_entry_continue_reverse - iterate backwards from the given point
672 * @pos:	the type * to use as a loop cursor.
673 * @head:	the head for your list.
674 * @member:	the name of the list_head within the struct.
675 *
676 * Start to iterate over list of given type backwards, continuing after
677 * the current position.
678 */
679#define list_for_each_entry_continue_reverse(pos, head, member)		\
680	for (pos = list_prev_entry(pos, member);			\
681	     !list_entry_is_head(pos, head, member);			\
682	     pos = list_prev_entry(pos, member))
683
684/**
685 * list_for_each_entry_from - iterate over list of given type from the current point
686 * @pos:	the type * to use as a loop cursor.
687 * @head:	the head for your list.
688 * @member:	the name of the list_head within the struct.
689 *
690 * Iterate over list of given type, continuing from current position.
691 */
692#define list_for_each_entry_from(pos, head, member) 			\
693	for (; !list_entry_is_head(pos, head, member);			\
694	     pos = list_next_entry(pos, member))
695
696/**
697 * list_for_each_entry_from_reverse - iterate backwards over list of given type
698 *                                    from the current point
699 * @pos:	the type * to use as a loop cursor.
700 * @head:	the head for your list.
701 * @member:	the name of the list_head within the struct.
702 *
703 * Iterate backwards over list of given type, continuing from current position.
704 */
705#define list_for_each_entry_from_reverse(pos, head, member)		\
706	for (; !list_entry_is_head(pos, head, member);			\
707	     pos = list_prev_entry(pos, member))
708
709/**
710 * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry
711 * @pos:	the type * to use as a loop cursor.
712 * @n:		another type * to use as temporary storage
713 * @head:	the head for your list.
714 * @member:	the name of the list_head within the struct.
715 */
716#define list_for_each_entry_safe(pos, n, head, member)			\
717	for (pos = list_first_entry(head, typeof(*pos), member),	\
718		n = list_next_entry(pos, member);			\
719	     !list_entry_is_head(pos, head, member); 			\
720	     pos = n, n = list_next_entry(n, member))
721
722/**
723 * list_for_each_entry_safe_continue - continue list iteration safe against removal
724 * @pos:	the type * to use as a loop cursor.
725 * @n:		another type * to use as temporary storage
726 * @head:	the head for your list.
727 * @member:	the name of the list_head within the struct.
728 *
729 * Iterate over list of given type, continuing after current point,
730 * safe against removal of list entry.
731 */
732#define list_for_each_entry_safe_continue(pos, n, head, member) 		\
733	for (pos = list_next_entry(pos, member), 				\
734		n = list_next_entry(pos, member);				\
735	     !list_entry_is_head(pos, head, member);				\
736	     pos = n, n = list_next_entry(n, member))
737
738/**
739 * list_for_each_entry_safe_from - iterate over list from current point safe against removal
740 * @pos:	the type * to use as a loop cursor.
741 * @n:		another type * to use as temporary storage
742 * @head:	the head for your list.
743 * @member:	the name of the list_head within the struct.
744 *
745 * Iterate over list of given type from current point, safe against
746 * removal of list entry.
747 */
748#define list_for_each_entry_safe_from(pos, n, head, member) 			\
749	for (n = list_next_entry(pos, member);					\
750	     !list_entry_is_head(pos, head, member);				\
751	     pos = n, n = list_next_entry(n, member))
752
753/**
754 * list_for_each_entry_safe_reverse - iterate backwards over list safe against removal
755 * @pos:	the type * to use as a loop cursor.
756 * @n:		another type * to use as temporary storage
757 * @head:	the head for your list.
758 * @member:	the name of the list_head within the struct.
759 *
760 * Iterate backwards over list of given type, safe against removal
761 * of list entry.
762 */
763#define list_for_each_entry_safe_reverse(pos, n, head, member)		\
764	for (pos = list_last_entry(head, typeof(*pos), member),		\
765		n = list_prev_entry(pos, member);			\
766	     !list_entry_is_head(pos, head, member); 			\
767	     pos = n, n = list_prev_entry(n, member))
768
769/**
770 * list_safe_reset_next - reset a stale list_for_each_entry_safe loop
771 * @pos:	the loop cursor used in the list_for_each_entry_safe loop
772 * @n:		temporary storage used in list_for_each_entry_safe
773 * @member:	the name of the list_head within the struct.
774 *
775 * list_safe_reset_next is not safe to use in general if the list may be
776 * modified concurrently (eg. the lock is dropped in the loop body). An
777 * exception to this is if the cursor element (pos) is pinned in the list,
778 * and list_safe_reset_next is called after re-taking the lock and before
779 * completing the current iteration of the loop body.
780 */
781#define list_safe_reset_next(pos, n, member)				\
782	n = list_next_entry(pos, member)
783
784/*
785 * Double linked lists with a single pointer list head.
786 * Mostly useful for hash tables where the two pointer list head is
787 * too wasteful.
788 * You lose the ability to access the tail in O(1).
789 */
790
791#define HLIST_HEAD_INIT { .first = NULL }
792#define HLIST_HEAD(name) struct hlist_head name = {  .first = NULL }
793#define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL)
794static inline void INIT_HLIST_NODE(struct hlist_node *h)
795{
796	h->next = NULL;
797	h->pprev = NULL;
798}
799
800/**
801 * hlist_unhashed - Has node been removed from list and reinitialized?
802 * @h: Node to be checked
803 *
804 * Not that not all removal functions will leave a node in unhashed
805 * state.  For example, hlist_nulls_del_init_rcu() does leave the
806 * node in unhashed state, but hlist_nulls_del() does not.
807 */
808static inline int hlist_unhashed(const struct hlist_node *h)
809{
810	return !h->pprev;
811}
812
813/**
814 * hlist_unhashed_lockless - Version of hlist_unhashed for lockless use
815 * @h: Node to be checked
816 *
817 * This variant of hlist_unhashed() must be used in lockless contexts
818 * to avoid potential load-tearing.  The READ_ONCE() is paired with the
819 * various WRITE_ONCE() in hlist helpers that are defined below.
820 */
821static inline int hlist_unhashed_lockless(const struct hlist_node *h)
822{
823	return !READ_ONCE(h->pprev);
824}
825
826/**
827 * hlist_empty - Is the specified hlist_head structure an empty hlist?
828 * @h: Structure to check.
829 */
830static inline int hlist_empty(const struct hlist_head *h)
831{
832	return !READ_ONCE(h->first);
833}
834
835static inline void __hlist_del(struct hlist_node *n)
836{
837	struct hlist_node *next = n->next;
838	struct hlist_node **pprev = n->pprev;
839
840	WRITE_ONCE(*pprev, next);
841	if (next)
842		WRITE_ONCE(next->pprev, pprev);
843}
844
845/**
846 * hlist_del - Delete the specified hlist_node from its list
847 * @n: Node to delete.
848 *
849 * Note that this function leaves the node in hashed state.  Use
850 * hlist_del_init() or similar instead to unhash @n.
851 */
852static inline void hlist_del(struct hlist_node *n)
853{
854	__hlist_del(n);
855	n->next = LIST_POISON1;
856	n->pprev = LIST_POISON2;
857}
858
859/**
860 * hlist_del_init - Delete the specified hlist_node from its list and initialize
861 * @n: Node to delete.
862 *
863 * Note that this function leaves the node in unhashed state.
864 */
865static inline void hlist_del_init(struct hlist_node *n)
866{
867	if (!hlist_unhashed(n)) {
868		__hlist_del(n);
869		INIT_HLIST_NODE(n);
870	}
871}
872
873/**
874 * hlist_add_head - add a new entry at the beginning of the hlist
875 * @n: new entry to be added
876 * @h: hlist head to add it after
877 *
878 * Insert a new entry after the specified head.
879 * This is good for implementing stacks.
880 */
881static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
882{
883	struct hlist_node *first = h->first;
884	WRITE_ONCE(n->next, first);
885	if (first)
886		WRITE_ONCE(first->pprev, &n->next);
887	WRITE_ONCE(h->first, n);
888	WRITE_ONCE(n->pprev, &h->first);
889}
890
891/**
892 * hlist_add_before - add a new entry before the one specified
893 * @n: new entry to be added
894 * @next: hlist node to add it before, which must be non-NULL
895 */
896static inline void hlist_add_before(struct hlist_node *n,
897				    struct hlist_node *next)
898{
899	WRITE_ONCE(n->pprev, next->pprev);
900	WRITE_ONCE(n->next, next);
901	WRITE_ONCE(next->pprev, &n->next);
902	WRITE_ONCE(*(n->pprev), n);
903}
904
905/**
906 * hlist_add_behind - add a new entry after the one specified
907 * @n: new entry to be added
908 * @prev: hlist node to add it after, which must be non-NULL
909 */
910static inline void hlist_add_behind(struct hlist_node *n,
911				    struct hlist_node *prev)
912{
913	WRITE_ONCE(n->next, prev->next);
914	WRITE_ONCE(prev->next, n);
915	WRITE_ONCE(n->pprev, &prev->next);
916
917	if (n->next)
918		WRITE_ONCE(n->next->pprev, &n->next);
919}
920
921/**
922 * hlist_add_fake - create a fake hlist consisting of a single headless node
923 * @n: Node to make a fake list out of
924 *
925 * This makes @n appear to be its own predecessor on a headless hlist.
926 * The point of this is to allow things like hlist_del() to work correctly
927 * in cases where there is no list.
928 */
929static inline void hlist_add_fake(struct hlist_node *n)
930{
931	n->pprev = &n->next;
932}
933
934/**
935 * hlist_fake: Is this node a fake hlist?
936 * @h: Node to check for being a self-referential fake hlist.
937 */
938static inline bool hlist_fake(struct hlist_node *h)
939{
940	return h->pprev == &h->next;
941}
942
943/**
944 * hlist_is_singular_node - is node the only element of the specified hlist?
945 * @n: Node to check for singularity.
946 * @h: Header for potentially singular list.
947 *
948 * Check whether the node is the only node of the head without
949 * accessing head, thus avoiding unnecessary cache misses.
950 */
951static inline bool
952hlist_is_singular_node(struct hlist_node *n, struct hlist_head *h)
953{
954	return !n->next && n->pprev == &h->first;
955}
956
957/**
958 * hlist_move_list - Move an hlist
959 * @old: hlist_head for old list.
960 * @new: hlist_head for new list.
961 *
962 * Move a list from one list head to another. Fixup the pprev
963 * reference of the first entry if it exists.
964 */
965static inline void hlist_move_list(struct hlist_head *old,
966				   struct hlist_head *new)
967{
968	new->first = old->first;
969	if (new->first)
970		new->first->pprev = &new->first;
971	old->first = NULL;
972}
973
974#define hlist_entry(ptr, type, member) container_of(ptr,type,member)
975
976#define hlist_for_each(pos, head) \
977	for (pos = (head)->first; pos ; pos = pos->next)
978
979#define hlist_for_each_safe(pos, n, head) \
980	for (pos = (head)->first; pos && ({ n = pos->next; 1; }); \
981	     pos = n)
982
983#define hlist_entry_safe(ptr, type, member) \
984	({ typeof(ptr) ____ptr = (ptr); \
985	   ____ptr ? hlist_entry(____ptr, type, member) : NULL; \
986	})
987
988/**
989 * hlist_for_each_entry	- iterate over list of given type
990 * @pos:	the type * to use as a loop cursor.
991 * @head:	the head for your list.
992 * @member:	the name of the hlist_node within the struct.
993 */
994#define hlist_for_each_entry(pos, head, member)				\
995	for (pos = hlist_entry_safe((head)->first, typeof(*(pos)), member);\
996	     pos;							\
997	     pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member))
998
999/**
1000 * hlist_for_each_entry_continue - iterate over a hlist continuing after current point
1001 * @pos:	the type * to use as a loop cursor.
1002 * @member:	the name of the hlist_node within the struct.
1003 */
1004#define hlist_for_each_entry_continue(pos, member)			\
1005	for (pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member);\
1006	     pos;							\
1007	     pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member))
1008
1009/**
1010 * hlist_for_each_entry_from - iterate over a hlist continuing from current point
1011 * @pos:	the type * to use as a loop cursor.
1012 * @member:	the name of the hlist_node within the struct.
1013 */
1014#define hlist_for_each_entry_from(pos, member)				\
1015	for (; pos;							\
1016	     pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member))
1017
1018/**
1019 * hlist_for_each_entry_safe - iterate over list of given type safe against removal of list entry
1020 * @pos:	the type * to use as a loop cursor.
1021 * @n:		a &struct hlist_node to use as temporary storage
1022 * @head:	the head for your list.
1023 * @member:	the name of the hlist_node within the struct.
1024 */
1025#define hlist_for_each_entry_safe(pos, n, head, member) 		\
1026	for (pos = hlist_entry_safe((head)->first, typeof(*pos), member);\
1027	     pos && ({ n = pos->member.next; 1; });			\
1028	     pos = hlist_entry_safe(n, typeof(*pos), member))
1029
1030#endif
1031