1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * A generic kernel FIFO implementation
4 *
5 * Copyright (C) 2013 Stefani Seibold <stefani@seibold.net>
6 */
7
8#ifndef _LINUX_KFIFO_H
9#define _LINUX_KFIFO_H
10
11/*
12 * How to porting drivers to the new generic FIFO API:
13 *
14 * - Modify the declaration of the "struct kfifo *" object into a
15 *   in-place "struct kfifo" object
16 * - Init the in-place object with kfifo_alloc() or kfifo_init()
17 *   Note: The address of the in-place "struct kfifo" object must be
18 *   passed as the first argument to this functions
19 * - Replace the use of __kfifo_put into kfifo_in and __kfifo_get
20 *   into kfifo_out
21 * - Replace the use of kfifo_put into kfifo_in_spinlocked and kfifo_get
22 *   into kfifo_out_spinlocked
23 *   Note: the spinlock pointer formerly passed to kfifo_init/kfifo_alloc
24 *   must be passed now to the kfifo_in_spinlocked and kfifo_out_spinlocked
25 *   as the last parameter
26 * - The formerly __kfifo_* functions are renamed into kfifo_*
27 */
28
29/*
30 * Note about locking: There is no locking required until only one reader
31 * and one writer is using the fifo and no kfifo_reset() will be called.
32 * kfifo_reset_out() can be safely used, until it will be only called
33 * in the reader thread.
34 * For multiple writer and one reader there is only a need to lock the writer.
35 * And vice versa for only one writer and multiple reader there is only a need
36 * to lock the reader.
37 */
38
39#include <linux/kernel.h>
40#include <linux/spinlock.h>
41#include <linux/stddef.h>
42#include <linux/scatterlist.h>
43
44struct __kfifo {
45	unsigned int	in;
46	unsigned int	out;
47	unsigned int	mask;
48	unsigned int	esize;
49	void		*data;
50};
51
52#define __STRUCT_KFIFO_COMMON(datatype, recsize, ptrtype) \
53	union { \
54		struct __kfifo	kfifo; \
55		datatype	*type; \
56		const datatype	*const_type; \
57		char		(*rectype)[recsize]; \
58		ptrtype		*ptr; \
59		ptrtype const	*ptr_const; \
60	}
61
62#define __STRUCT_KFIFO(type, size, recsize, ptrtype) \
63{ \
64	__STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
65	type		buf[((size < 2) || (size & (size - 1))) ? -1 : size]; \
66}
67
68#define STRUCT_KFIFO(type, size) \
69	struct __STRUCT_KFIFO(type, size, 0, type)
70
71#define __STRUCT_KFIFO_PTR(type, recsize, ptrtype) \
72{ \
73	__STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
74	type		buf[0]; \
75}
76
77#define STRUCT_KFIFO_PTR(type) \
78	struct __STRUCT_KFIFO_PTR(type, 0, type)
79
80/*
81 * define compatibility "struct kfifo" for dynamic allocated fifos
82 */
83struct kfifo __STRUCT_KFIFO_PTR(unsigned char, 0, void);
84
85#define STRUCT_KFIFO_REC_1(size) \
86	struct __STRUCT_KFIFO(unsigned char, size, 1, void)
87
88#define STRUCT_KFIFO_REC_2(size) \
89	struct __STRUCT_KFIFO(unsigned char, size, 2, void)
90
91/*
92 * define kfifo_rec types
93 */
94struct kfifo_rec_ptr_1 __STRUCT_KFIFO_PTR(unsigned char, 1, void);
95struct kfifo_rec_ptr_2 __STRUCT_KFIFO_PTR(unsigned char, 2, void);
96
97/*
98 * helper macro to distinguish between real in place fifo where the fifo
99 * array is a part of the structure and the fifo type where the array is
100 * outside of the fifo structure.
101 */
102#define	__is_kfifo_ptr(fifo) \
103	(sizeof(*fifo) == sizeof(STRUCT_KFIFO_PTR(typeof(*(fifo)->type))))
104
105/**
106 * DECLARE_KFIFO_PTR - macro to declare a fifo pointer object
107 * @fifo: name of the declared fifo
108 * @type: type of the fifo elements
109 */
110#define DECLARE_KFIFO_PTR(fifo, type)	STRUCT_KFIFO_PTR(type) fifo
111
112/**
113 * DECLARE_KFIFO - macro to declare a fifo object
114 * @fifo: name of the declared fifo
115 * @type: type of the fifo elements
116 * @size: the number of elements in the fifo, this must be a power of 2
117 */
118#define DECLARE_KFIFO(fifo, type, size)	STRUCT_KFIFO(type, size) fifo
119
120/**
121 * INIT_KFIFO - Initialize a fifo declared by DECLARE_KFIFO
122 * @fifo: name of the declared fifo datatype
123 */
124#define INIT_KFIFO(fifo) \
125(void)({ \
126	typeof(&(fifo)) __tmp = &(fifo); \
127	struct __kfifo *__kfifo = &__tmp->kfifo; \
128	__kfifo->in = 0; \
129	__kfifo->out = 0; \
130	__kfifo->mask = __is_kfifo_ptr(__tmp) ? 0 : ARRAY_SIZE(__tmp->buf) - 1;\
131	__kfifo->esize = sizeof(*__tmp->buf); \
132	__kfifo->data = __is_kfifo_ptr(__tmp) ?  NULL : __tmp->buf; \
133})
134
135/**
136 * DEFINE_KFIFO - macro to define and initialize a fifo
137 * @fifo: name of the declared fifo datatype
138 * @type: type of the fifo elements
139 * @size: the number of elements in the fifo, this must be a power of 2
140 *
141 * Note: the macro can be used for global and local fifo data type variables.
142 */
143#define DEFINE_KFIFO(fifo, type, size) \
144	DECLARE_KFIFO(fifo, type, size) = \
145	(typeof(fifo)) { \
146		{ \
147			{ \
148			.in	= 0, \
149			.out	= 0, \
150			.mask	= __is_kfifo_ptr(&(fifo)) ? \
151				  0 : \
152				  ARRAY_SIZE((fifo).buf) - 1, \
153			.esize	= sizeof(*(fifo).buf), \
154			.data	= __is_kfifo_ptr(&(fifo)) ? \
155				NULL : \
156				(fifo).buf, \
157			} \
158		} \
159	}
160
161
162static inline unsigned int __must_check
163__kfifo_uint_must_check_helper(unsigned int val)
164{
165	return val;
166}
167
168static inline int __must_check
169__kfifo_int_must_check_helper(int val)
170{
171	return val;
172}
173
174/**
175 * kfifo_initialized - Check if the fifo is initialized
176 * @fifo: address of the fifo to check
177 *
178 * Return %true if fifo is initialized, otherwise %false.
179 * Assumes the fifo was 0 before.
180 */
181#define kfifo_initialized(fifo) ((fifo)->kfifo.mask)
182
183/**
184 * kfifo_esize - returns the size of the element managed by the fifo
185 * @fifo: address of the fifo to be used
186 */
187#define kfifo_esize(fifo)	((fifo)->kfifo.esize)
188
189/**
190 * kfifo_recsize - returns the size of the record length field
191 * @fifo: address of the fifo to be used
192 */
193#define kfifo_recsize(fifo)	(sizeof(*(fifo)->rectype))
194
195/**
196 * kfifo_size - returns the size of the fifo in elements
197 * @fifo: address of the fifo to be used
198 */
199#define kfifo_size(fifo)	((fifo)->kfifo.mask + 1)
200
201/**
202 * kfifo_reset - removes the entire fifo content
203 * @fifo: address of the fifo to be used
204 *
205 * Note: usage of kfifo_reset() is dangerous. It should be only called when the
206 * fifo is exclusived locked or when it is secured that no other thread is
207 * accessing the fifo.
208 */
209#define kfifo_reset(fifo) \
210(void)({ \
211	typeof((fifo) + 1) __tmp = (fifo); \
212	__tmp->kfifo.in = __tmp->kfifo.out = 0; \
213})
214
215/**
216 * kfifo_reset_out - skip fifo content
217 * @fifo: address of the fifo to be used
218 *
219 * Note: The usage of kfifo_reset_out() is safe until it will be only called
220 * from the reader thread and there is only one concurrent reader. Otherwise
221 * it is dangerous and must be handled in the same way as kfifo_reset().
222 */
223#define kfifo_reset_out(fifo)	\
224(void)({ \
225	typeof((fifo) + 1) __tmp = (fifo); \
226	__tmp->kfifo.out = __tmp->kfifo.in; \
227})
228
229/**
230 * kfifo_len - returns the number of used elements in the fifo
231 * @fifo: address of the fifo to be used
232 */
233#define kfifo_len(fifo) \
234({ \
235	typeof((fifo) + 1) __tmpl = (fifo); \
236	__tmpl->kfifo.in - __tmpl->kfifo.out; \
237})
238
239/**
240 * kfifo_is_empty - returns true if the fifo is empty
241 * @fifo: address of the fifo to be used
242 */
243#define	kfifo_is_empty(fifo) \
244({ \
245	typeof((fifo) + 1) __tmpq = (fifo); \
246	__tmpq->kfifo.in == __tmpq->kfifo.out; \
247})
248
249/**
250 * kfifo_is_empty_spinlocked - returns true if the fifo is empty using
251 * a spinlock for locking
252 * @fifo: address of the fifo to be used
253 * @lock: spinlock to be used for locking
254 */
255#define kfifo_is_empty_spinlocked(fifo, lock) \
256({ \
257	unsigned long __flags; \
258	bool __ret; \
259	spin_lock_irqsave(lock, __flags); \
260	__ret = kfifo_is_empty(fifo); \
261	spin_unlock_irqrestore(lock, __flags); \
262	__ret; \
263})
264
265/**
266 * kfifo_is_empty_spinlocked_noirqsave  - returns true if the fifo is empty
267 * using a spinlock for locking, doesn't disable interrupts
268 * @fifo: address of the fifo to be used
269 * @lock: spinlock to be used for locking
270 */
271#define kfifo_is_empty_spinlocked_noirqsave(fifo, lock) \
272({ \
273	bool __ret; \
274	spin_lock(lock); \
275	__ret = kfifo_is_empty(fifo); \
276	spin_unlock(lock); \
277	__ret; \
278})
279
280/**
281 * kfifo_is_full - returns true if the fifo is full
282 * @fifo: address of the fifo to be used
283 */
284#define	kfifo_is_full(fifo) \
285({ \
286	typeof((fifo) + 1) __tmpq = (fifo); \
287	kfifo_len(__tmpq) > __tmpq->kfifo.mask; \
288})
289
290/**
291 * kfifo_avail - returns the number of unused elements in the fifo
292 * @fifo: address of the fifo to be used
293 */
294#define	kfifo_avail(fifo) \
295__kfifo_uint_must_check_helper( \
296({ \
297	typeof((fifo) + 1) __tmpq = (fifo); \
298	const size_t __recsize = sizeof(*__tmpq->rectype); \
299	unsigned int __avail = kfifo_size(__tmpq) - kfifo_len(__tmpq); \
300	(__recsize) ? ((__avail <= __recsize) ? 0 : \
301	__kfifo_max_r(__avail - __recsize, __recsize)) : \
302	__avail; \
303}) \
304)
305
306/**
307 * kfifo_skip - skip output data
308 * @fifo: address of the fifo to be used
309 */
310#define	kfifo_skip(fifo) \
311(void)({ \
312	typeof((fifo) + 1) __tmp = (fifo); \
313	const size_t __recsize = sizeof(*__tmp->rectype); \
314	struct __kfifo *__kfifo = &__tmp->kfifo; \
315	if (__recsize) \
316		__kfifo_skip_r(__kfifo, __recsize); \
317	else \
318		__kfifo->out++; \
319})
320
321/**
322 * kfifo_peek_len - gets the size of the next fifo record
323 * @fifo: address of the fifo to be used
324 *
325 * This function returns the size of the next fifo record in number of bytes.
326 */
327#define kfifo_peek_len(fifo) \
328__kfifo_uint_must_check_helper( \
329({ \
330	typeof((fifo) + 1) __tmp = (fifo); \
331	const size_t __recsize = sizeof(*__tmp->rectype); \
332	struct __kfifo *__kfifo = &__tmp->kfifo; \
333	(!__recsize) ? kfifo_len(__tmp) * sizeof(*__tmp->type) : \
334	__kfifo_len_r(__kfifo, __recsize); \
335}) \
336)
337
338/**
339 * kfifo_alloc - dynamically allocates a new fifo buffer
340 * @fifo: pointer to the fifo
341 * @size: the number of elements in the fifo, this must be a power of 2
342 * @gfp_mask: get_free_pages mask, passed to kmalloc()
343 *
344 * This macro dynamically allocates a new fifo buffer.
345 *
346 * The number of elements will be rounded-up to a power of 2.
347 * The fifo will be release with kfifo_free().
348 * Return 0 if no error, otherwise an error code.
349 */
350#define kfifo_alloc(fifo, size, gfp_mask) \
351__kfifo_int_must_check_helper( \
352({ \
353	typeof((fifo) + 1) __tmp = (fifo); \
354	struct __kfifo *__kfifo = &__tmp->kfifo; \
355	__is_kfifo_ptr(__tmp) ? \
356	__kfifo_alloc(__kfifo, size, sizeof(*__tmp->type), gfp_mask) : \
357	-EINVAL; \
358}) \
359)
360
361/**
362 * kfifo_free - frees the fifo
363 * @fifo: the fifo to be freed
364 */
365#define kfifo_free(fifo) \
366({ \
367	typeof((fifo) + 1) __tmp = (fifo); \
368	struct __kfifo *__kfifo = &__tmp->kfifo; \
369	if (__is_kfifo_ptr(__tmp)) \
370		__kfifo_free(__kfifo); \
371})
372
373/**
374 * kfifo_init - initialize a fifo using a preallocated buffer
375 * @fifo: the fifo to assign the buffer
376 * @buffer: the preallocated buffer to be used
377 * @size: the size of the internal buffer, this have to be a power of 2
378 *
379 * This macro initializes a fifo using a preallocated buffer.
380 *
381 * The number of elements will be rounded-up to a power of 2.
382 * Return 0 if no error, otherwise an error code.
383 */
384#define kfifo_init(fifo, buffer, size) \
385({ \
386	typeof((fifo) + 1) __tmp = (fifo); \
387	struct __kfifo *__kfifo = &__tmp->kfifo; \
388	__is_kfifo_ptr(__tmp) ? \
389	__kfifo_init(__kfifo, buffer, size, sizeof(*__tmp->type)) : \
390	-EINVAL; \
391})
392
393/**
394 * kfifo_put - put data into the fifo
395 * @fifo: address of the fifo to be used
396 * @val: the data to be added
397 *
398 * This macro copies the given value into the fifo.
399 * It returns 0 if the fifo was full. Otherwise it returns the number
400 * processed elements.
401 *
402 * Note that with only one concurrent reader and one concurrent
403 * writer, you don't need extra locking to use these macro.
404 */
405#define	kfifo_put(fifo, val) \
406({ \
407	typeof((fifo) + 1) __tmp = (fifo); \
408	typeof(*__tmp->const_type) __val = (val); \
409	unsigned int __ret; \
410	size_t __recsize = sizeof(*__tmp->rectype); \
411	struct __kfifo *__kfifo = &__tmp->kfifo; \
412	if (__recsize) \
413		__ret = __kfifo_in_r(__kfifo, &__val, sizeof(__val), \
414			__recsize); \
415	else { \
416		__ret = !kfifo_is_full(__tmp); \
417		if (__ret) { \
418			(__is_kfifo_ptr(__tmp) ? \
419			((typeof(__tmp->type))__kfifo->data) : \
420			(__tmp->buf) \
421			)[__kfifo->in & __tmp->kfifo.mask] = \
422				*(typeof(__tmp->type))&__val; \
423			smp_wmb(); \
424			__kfifo->in++; \
425		} \
426	} \
427	__ret; \
428})
429
430/**
431 * kfifo_get - get data from the fifo
432 * @fifo: address of the fifo to be used
433 * @val: address where to store the data
434 *
435 * This macro reads the data from the fifo.
436 * It returns 0 if the fifo was empty. Otherwise it returns the number
437 * processed elements.
438 *
439 * Note that with only one concurrent reader and one concurrent
440 * writer, you don't need extra locking to use these macro.
441 */
442#define	kfifo_get(fifo, val) \
443__kfifo_uint_must_check_helper( \
444({ \
445	typeof((fifo) + 1) __tmp = (fifo); \
446	typeof(__tmp->ptr) __val = (val); \
447	unsigned int __ret; \
448	const size_t __recsize = sizeof(*__tmp->rectype); \
449	struct __kfifo *__kfifo = &__tmp->kfifo; \
450	if (__recsize) \
451		__ret = __kfifo_out_r(__kfifo, __val, sizeof(*__val), \
452			__recsize); \
453	else { \
454		__ret = !kfifo_is_empty(__tmp); \
455		if (__ret) { \
456			*(typeof(__tmp->type))__val = \
457				(__is_kfifo_ptr(__tmp) ? \
458				((typeof(__tmp->type))__kfifo->data) : \
459				(__tmp->buf) \
460				)[__kfifo->out & __tmp->kfifo.mask]; \
461			smp_wmb(); \
462			__kfifo->out++; \
463		} \
464	} \
465	__ret; \
466}) \
467)
468
469/**
470 * kfifo_peek - get data from the fifo without removing
471 * @fifo: address of the fifo to be used
472 * @val: address where to store the data
473 *
474 * This reads the data from the fifo without removing it from the fifo.
475 * It returns 0 if the fifo was empty. Otherwise it returns the number
476 * processed elements.
477 *
478 * Note that with only one concurrent reader and one concurrent
479 * writer, you don't need extra locking to use these macro.
480 */
481#define	kfifo_peek(fifo, val) \
482__kfifo_uint_must_check_helper( \
483({ \
484	typeof((fifo) + 1) __tmp = (fifo); \
485	typeof(__tmp->ptr) __val = (val); \
486	unsigned int __ret; \
487	const size_t __recsize = sizeof(*__tmp->rectype); \
488	struct __kfifo *__kfifo = &__tmp->kfifo; \
489	if (__recsize) \
490		__ret = __kfifo_out_peek_r(__kfifo, __val, sizeof(*__val), \
491			__recsize); \
492	else { \
493		__ret = !kfifo_is_empty(__tmp); \
494		if (__ret) { \
495			*(typeof(__tmp->type))__val = \
496				(__is_kfifo_ptr(__tmp) ? \
497				((typeof(__tmp->type))__kfifo->data) : \
498				(__tmp->buf) \
499				)[__kfifo->out & __tmp->kfifo.mask]; \
500			smp_wmb(); \
501		} \
502	} \
503	__ret; \
504}) \
505)
506
507/**
508 * kfifo_in - put data into the fifo
509 * @fifo: address of the fifo to be used
510 * @buf: the data to be added
511 * @n: number of elements to be added
512 *
513 * This macro copies the given buffer into the fifo and returns the
514 * number of copied elements.
515 *
516 * Note that with only one concurrent reader and one concurrent
517 * writer, you don't need extra locking to use these macro.
518 */
519#define	kfifo_in(fifo, buf, n) \
520({ \
521	typeof((fifo) + 1) __tmp = (fifo); \
522	typeof(__tmp->ptr_const) __buf = (buf); \
523	unsigned long __n = (n); \
524	const size_t __recsize = sizeof(*__tmp->rectype); \
525	struct __kfifo *__kfifo = &__tmp->kfifo; \
526	(__recsize) ?\
527	__kfifo_in_r(__kfifo, __buf, __n, __recsize) : \
528	__kfifo_in(__kfifo, __buf, __n); \
529})
530
531/**
532 * kfifo_in_spinlocked - put data into the fifo using a spinlock for locking
533 * @fifo: address of the fifo to be used
534 * @buf: the data to be added
535 * @n: number of elements to be added
536 * @lock: pointer to the spinlock to use for locking
537 *
538 * This macro copies the given values buffer into the fifo and returns the
539 * number of copied elements.
540 */
541#define	kfifo_in_spinlocked(fifo, buf, n, lock) \
542({ \
543	unsigned long __flags; \
544	unsigned int __ret; \
545	spin_lock_irqsave(lock, __flags); \
546	__ret = kfifo_in(fifo, buf, n); \
547	spin_unlock_irqrestore(lock, __flags); \
548	__ret; \
549})
550
551/**
552 * kfifo_in_spinlocked_noirqsave - put data into fifo using a spinlock for
553 * locking, don't disable interrupts
554 * @fifo: address of the fifo to be used
555 * @buf: the data to be added
556 * @n: number of elements to be added
557 * @lock: pointer to the spinlock to use for locking
558 *
559 * This is a variant of kfifo_in_spinlocked() but uses spin_lock/unlock()
560 * for locking and doesn't disable interrupts.
561 */
562#define kfifo_in_spinlocked_noirqsave(fifo, buf, n, lock) \
563({ \
564	unsigned int __ret; \
565	spin_lock(lock); \
566	__ret = kfifo_in(fifo, buf, n); \
567	spin_unlock(lock); \
568	__ret; \
569})
570
571/* alias for kfifo_in_spinlocked, will be removed in a future release */
572#define kfifo_in_locked(fifo, buf, n, lock) \
573		kfifo_in_spinlocked(fifo, buf, n, lock)
574
575/**
576 * kfifo_out - get data from the fifo
577 * @fifo: address of the fifo to be used
578 * @buf: pointer to the storage buffer
579 * @n: max. number of elements to get
580 *
581 * This macro get some data from the fifo and return the numbers of elements
582 * copied.
583 *
584 * Note that with only one concurrent reader and one concurrent
585 * writer, you don't need extra locking to use these macro.
586 */
587#define	kfifo_out(fifo, buf, n) \
588__kfifo_uint_must_check_helper( \
589({ \
590	typeof((fifo) + 1) __tmp = (fifo); \
591	typeof(__tmp->ptr) __buf = (buf); \
592	unsigned long __n = (n); \
593	const size_t __recsize = sizeof(*__tmp->rectype); \
594	struct __kfifo *__kfifo = &__tmp->kfifo; \
595	(__recsize) ?\
596	__kfifo_out_r(__kfifo, __buf, __n, __recsize) : \
597	__kfifo_out(__kfifo, __buf, __n); \
598}) \
599)
600
601/**
602 * kfifo_out_spinlocked - get data from the fifo using a spinlock for locking
603 * @fifo: address of the fifo to be used
604 * @buf: pointer to the storage buffer
605 * @n: max. number of elements to get
606 * @lock: pointer to the spinlock to use for locking
607 *
608 * This macro get the data from the fifo and return the numbers of elements
609 * copied.
610 */
611#define	kfifo_out_spinlocked(fifo, buf, n, lock) \
612__kfifo_uint_must_check_helper( \
613({ \
614	unsigned long __flags; \
615	unsigned int __ret; \
616	spin_lock_irqsave(lock, __flags); \
617	__ret = kfifo_out(fifo, buf, n); \
618	spin_unlock_irqrestore(lock, __flags); \
619	__ret; \
620}) \
621)
622
623/**
624 * kfifo_out_spinlocked_noirqsave - get data from the fifo using a spinlock
625 * for locking, don't disable interrupts
626 * @fifo: address of the fifo to be used
627 * @buf: pointer to the storage buffer
628 * @n: max. number of elements to get
629 * @lock: pointer to the spinlock to use for locking
630 *
631 * This is a variant of kfifo_out_spinlocked() which uses spin_lock/unlock()
632 * for locking and doesn't disable interrupts.
633 */
634#define kfifo_out_spinlocked_noirqsave(fifo, buf, n, lock) \
635__kfifo_uint_must_check_helper( \
636({ \
637	unsigned int __ret; \
638	spin_lock(lock); \
639	__ret = kfifo_out(fifo, buf, n); \
640	spin_unlock(lock); \
641	__ret; \
642}) \
643)
644
645/* alias for kfifo_out_spinlocked, will be removed in a future release */
646#define kfifo_out_locked(fifo, buf, n, lock) \
647		kfifo_out_spinlocked(fifo, buf, n, lock)
648
649/**
650 * kfifo_from_user - puts some data from user space into the fifo
651 * @fifo: address of the fifo to be used
652 * @from: pointer to the data to be added
653 * @len: the length of the data to be added
654 * @copied: pointer to output variable to store the number of copied bytes
655 *
656 * This macro copies at most @len bytes from the @from into the
657 * fifo, depending of the available space and returns -EFAULT/0.
658 *
659 * Note that with only one concurrent reader and one concurrent
660 * writer, you don't need extra locking to use these macro.
661 */
662#define	kfifo_from_user(fifo, from, len, copied) \
663__kfifo_uint_must_check_helper( \
664({ \
665	typeof((fifo) + 1) __tmp = (fifo); \
666	const void __user *__from = (from); \
667	unsigned int __len = (len); \
668	unsigned int *__copied = (copied); \
669	const size_t __recsize = sizeof(*__tmp->rectype); \
670	struct __kfifo *__kfifo = &__tmp->kfifo; \
671	(__recsize) ? \
672	__kfifo_from_user_r(__kfifo, __from, __len,  __copied, __recsize) : \
673	__kfifo_from_user(__kfifo, __from, __len, __copied); \
674}) \
675)
676
677/**
678 * kfifo_to_user - copies data from the fifo into user space
679 * @fifo: address of the fifo to be used
680 * @to: where the data must be copied
681 * @len: the size of the destination buffer
682 * @copied: pointer to output variable to store the number of copied bytes
683 *
684 * This macro copies at most @len bytes from the fifo into the
685 * @to buffer and returns -EFAULT/0.
686 *
687 * Note that with only one concurrent reader and one concurrent
688 * writer, you don't need extra locking to use these macro.
689 */
690#define	kfifo_to_user(fifo, to, len, copied) \
691__kfifo_int_must_check_helper( \
692({ \
693	typeof((fifo) + 1) __tmp = (fifo); \
694	void __user *__to = (to); \
695	unsigned int __len = (len); \
696	unsigned int *__copied = (copied); \
697	const size_t __recsize = sizeof(*__tmp->rectype); \
698	struct __kfifo *__kfifo = &__tmp->kfifo; \
699	(__recsize) ? \
700	__kfifo_to_user_r(__kfifo, __to, __len, __copied, __recsize) : \
701	__kfifo_to_user(__kfifo, __to, __len, __copied); \
702}) \
703)
704
705/**
706 * kfifo_dma_in_prepare - setup a scatterlist for DMA input
707 * @fifo: address of the fifo to be used
708 * @sgl: pointer to the scatterlist array
709 * @nents: number of entries in the scatterlist array
710 * @len: number of elements to transfer
711 *
712 * This macro fills a scatterlist for DMA input.
713 * It returns the number entries in the scatterlist array.
714 *
715 * Note that with only one concurrent reader and one concurrent
716 * writer, you don't need extra locking to use these macros.
717 */
718#define	kfifo_dma_in_prepare(fifo, sgl, nents, len) \
719({ \
720	typeof((fifo) + 1) __tmp = (fifo); \
721	struct scatterlist *__sgl = (sgl); \
722	int __nents = (nents); \
723	unsigned int __len = (len); \
724	const size_t __recsize = sizeof(*__tmp->rectype); \
725	struct __kfifo *__kfifo = &__tmp->kfifo; \
726	(__recsize) ? \
727	__kfifo_dma_in_prepare_r(__kfifo, __sgl, __nents, __len, __recsize) : \
728	__kfifo_dma_in_prepare(__kfifo, __sgl, __nents, __len); \
729})
730
731/**
732 * kfifo_dma_in_finish - finish a DMA IN operation
733 * @fifo: address of the fifo to be used
734 * @len: number of bytes to received
735 *
736 * This macro finish a DMA IN operation. The in counter will be updated by
737 * the len parameter. No error checking will be done.
738 *
739 * Note that with only one concurrent reader and one concurrent
740 * writer, you don't need extra locking to use these macros.
741 */
742#define kfifo_dma_in_finish(fifo, len) \
743(void)({ \
744	typeof((fifo) + 1) __tmp = (fifo); \
745	unsigned int __len = (len); \
746	const size_t __recsize = sizeof(*__tmp->rectype); \
747	struct __kfifo *__kfifo = &__tmp->kfifo; \
748	if (__recsize) \
749		__kfifo_dma_in_finish_r(__kfifo, __len, __recsize); \
750	else \
751		__kfifo->in += __len / sizeof(*__tmp->type); \
752})
753
754/**
755 * kfifo_dma_out_prepare - setup a scatterlist for DMA output
756 * @fifo: address of the fifo to be used
757 * @sgl: pointer to the scatterlist array
758 * @nents: number of entries in the scatterlist array
759 * @len: number of elements to transfer
760 *
761 * This macro fills a scatterlist for DMA output which at most @len bytes
762 * to transfer.
763 * It returns the number entries in the scatterlist array.
764 * A zero means there is no space available and the scatterlist is not filled.
765 *
766 * Note that with only one concurrent reader and one concurrent
767 * writer, you don't need extra locking to use these macros.
768 */
769#define	kfifo_dma_out_prepare(fifo, sgl, nents, len) \
770({ \
771	typeof((fifo) + 1) __tmp = (fifo);  \
772	struct scatterlist *__sgl = (sgl); \
773	int __nents = (nents); \
774	unsigned int __len = (len); \
775	const size_t __recsize = sizeof(*__tmp->rectype); \
776	struct __kfifo *__kfifo = &__tmp->kfifo; \
777	(__recsize) ? \
778	__kfifo_dma_out_prepare_r(__kfifo, __sgl, __nents, __len, __recsize) : \
779	__kfifo_dma_out_prepare(__kfifo, __sgl, __nents, __len); \
780})
781
782/**
783 * kfifo_dma_out_finish - finish a DMA OUT operation
784 * @fifo: address of the fifo to be used
785 * @len: number of bytes transferred
786 *
787 * This macro finish a DMA OUT operation. The out counter will be updated by
788 * the len parameter. No error checking will be done.
789 *
790 * Note that with only one concurrent reader and one concurrent
791 * writer, you don't need extra locking to use these macros.
792 */
793#define kfifo_dma_out_finish(fifo, len) \
794(void)({ \
795	typeof((fifo) + 1) __tmp = (fifo); \
796	unsigned int __len = (len); \
797	const size_t __recsize = sizeof(*__tmp->rectype); \
798	struct __kfifo *__kfifo = &__tmp->kfifo; \
799	if (__recsize) \
800		__kfifo_dma_out_finish_r(__kfifo, __recsize); \
801	else \
802		__kfifo->out += __len / sizeof(*__tmp->type); \
803})
804
805/**
806 * kfifo_out_peek - gets some data from the fifo
807 * @fifo: address of the fifo to be used
808 * @buf: pointer to the storage buffer
809 * @n: max. number of elements to get
810 *
811 * This macro get the data from the fifo and return the numbers of elements
812 * copied. The data is not removed from the fifo.
813 *
814 * Note that with only one concurrent reader and one concurrent
815 * writer, you don't need extra locking to use these macro.
816 */
817#define	kfifo_out_peek(fifo, buf, n) \
818__kfifo_uint_must_check_helper( \
819({ \
820	typeof((fifo) + 1) __tmp = (fifo); \
821	typeof(__tmp->ptr) __buf = (buf); \
822	unsigned long __n = (n); \
823	const size_t __recsize = sizeof(*__tmp->rectype); \
824	struct __kfifo *__kfifo = &__tmp->kfifo; \
825	(__recsize) ? \
826	__kfifo_out_peek_r(__kfifo, __buf, __n, __recsize) : \
827	__kfifo_out_peek(__kfifo, __buf, __n); \
828}) \
829)
830
831extern int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
832	size_t esize, gfp_t gfp_mask);
833
834extern void __kfifo_free(struct __kfifo *fifo);
835
836extern int __kfifo_init(struct __kfifo *fifo, void *buffer,
837	unsigned int size, size_t esize);
838
839extern unsigned int __kfifo_in(struct __kfifo *fifo,
840	const void *buf, unsigned int len);
841
842extern unsigned int __kfifo_out(struct __kfifo *fifo,
843	void *buf, unsigned int len);
844
845extern int __kfifo_from_user(struct __kfifo *fifo,
846	const void __user *from, unsigned long len, unsigned int *copied);
847
848extern int __kfifo_to_user(struct __kfifo *fifo,
849	void __user *to, unsigned long len, unsigned int *copied);
850
851extern unsigned int __kfifo_dma_in_prepare(struct __kfifo *fifo,
852	struct scatterlist *sgl, int nents, unsigned int len);
853
854extern unsigned int __kfifo_dma_out_prepare(struct __kfifo *fifo,
855	struct scatterlist *sgl, int nents, unsigned int len);
856
857extern unsigned int __kfifo_out_peek(struct __kfifo *fifo,
858	void *buf, unsigned int len);
859
860extern unsigned int __kfifo_in_r(struct __kfifo *fifo,
861	const void *buf, unsigned int len, size_t recsize);
862
863extern unsigned int __kfifo_out_r(struct __kfifo *fifo,
864	void *buf, unsigned int len, size_t recsize);
865
866extern int __kfifo_from_user_r(struct __kfifo *fifo,
867	const void __user *from, unsigned long len, unsigned int *copied,
868	size_t recsize);
869
870extern int __kfifo_to_user_r(struct __kfifo *fifo, void __user *to,
871	unsigned long len, unsigned int *copied, size_t recsize);
872
873extern unsigned int __kfifo_dma_in_prepare_r(struct __kfifo *fifo,
874	struct scatterlist *sgl, int nents, unsigned int len, size_t recsize);
875
876extern void __kfifo_dma_in_finish_r(struct __kfifo *fifo,
877	unsigned int len, size_t recsize);
878
879extern unsigned int __kfifo_dma_out_prepare_r(struct __kfifo *fifo,
880	struct scatterlist *sgl, int nents, unsigned int len, size_t recsize);
881
882extern void __kfifo_dma_out_finish_r(struct __kfifo *fifo, size_t recsize);
883
884extern unsigned int __kfifo_len_r(struct __kfifo *fifo, size_t recsize);
885
886extern void __kfifo_skip_r(struct __kfifo *fifo, size_t recsize);
887
888extern unsigned int __kfifo_out_peek_r(struct __kfifo *fifo,
889	void *buf, unsigned int len, size_t recsize);
890
891extern unsigned int __kfifo_max_r(unsigned int len, size_t recsize);
892
893#endif
894