1/* SPDX-License-Identifier: GPL-2.0-only */
2#ifndef __ASM_UNROLL_H__
3#define __ASM_UNROLL_H__
4
5/*
6 * Explicitly unroll a loop, for use in cases where doing so is performance
7 * critical.
8 *
9 * Ideally we'd rely upon the compiler to provide this but there's no commonly
10 * available means to do so. For example GCC's "#pragma GCC unroll"
11 * functionality would be ideal but is only available from GCC 8 onwards. Using
12 * -funroll-loops is an option but GCC tends to make poor choices when
13 * compiling our string functions. -funroll-all-loops leads to massive code
14 * bloat, even if only applied to the string functions.
15 */
16#define unroll(times, fn, ...) do {				\
17	extern void bad_unroll(void)				\
18		__compiletime_error("Unsupported unroll");	\
19								\
20	/*							\
21	 * We can't unroll if the number of iterations isn't	\
22	 * compile-time constant. Unfortunately clang versions	\
23	 * up until 8.0 tend to miss obvious constants & cause	\
24	 * this check to fail, even though they go on to	\
25	 * generate reasonable code for the switch statement,	\
26	 * so we skip the sanity check for those compilers.	\
27	 */							\
28	BUILD_BUG_ON(!__builtin_constant_p(times));		\
29								\
30	switch (times) {					\
31	case 32: fn(__VA_ARGS__); fallthrough;			\
32	case 31: fn(__VA_ARGS__); fallthrough;			\
33	case 30: fn(__VA_ARGS__); fallthrough;			\
34	case 29: fn(__VA_ARGS__); fallthrough;			\
35	case 28: fn(__VA_ARGS__); fallthrough;			\
36	case 27: fn(__VA_ARGS__); fallthrough;			\
37	case 26: fn(__VA_ARGS__); fallthrough;			\
38	case 25: fn(__VA_ARGS__); fallthrough;			\
39	case 24: fn(__VA_ARGS__); fallthrough;			\
40	case 23: fn(__VA_ARGS__); fallthrough;			\
41	case 22: fn(__VA_ARGS__); fallthrough;			\
42	case 21: fn(__VA_ARGS__); fallthrough;			\
43	case 20: fn(__VA_ARGS__); fallthrough;			\
44	case 19: fn(__VA_ARGS__); fallthrough;			\
45	case 18: fn(__VA_ARGS__); fallthrough;			\
46	case 17: fn(__VA_ARGS__); fallthrough;			\
47	case 16: fn(__VA_ARGS__); fallthrough;			\
48	case 15: fn(__VA_ARGS__); fallthrough;			\
49	case 14: fn(__VA_ARGS__); fallthrough;			\
50	case 13: fn(__VA_ARGS__); fallthrough;			\
51	case 12: fn(__VA_ARGS__); fallthrough;			\
52	case 11: fn(__VA_ARGS__); fallthrough;			\
53	case 10: fn(__VA_ARGS__); fallthrough;			\
54	case 9: fn(__VA_ARGS__); fallthrough;			\
55	case 8: fn(__VA_ARGS__); fallthrough;			\
56	case 7: fn(__VA_ARGS__); fallthrough;			\
57	case 6: fn(__VA_ARGS__); fallthrough;			\
58	case 5: fn(__VA_ARGS__); fallthrough;			\
59	case 4: fn(__VA_ARGS__); fallthrough;			\
60	case 3: fn(__VA_ARGS__); fallthrough;			\
61	case 2: fn(__VA_ARGS__); fallthrough;			\
62	case 1: fn(__VA_ARGS__); fallthrough;			\
63	case 0: break;						\
64								\
65	default:						\
66		/*						\
67		 * Either the iteration count is unreasonable	\
68		 * or we need to add more cases above.		\
69		 */						\
70		bad_unroll();					\
71		break;						\
72	}							\
73} while (0)
74
75#endif /* __ASM_UNROLL_H__ */
76