x86/crypto/chacha_glue.c

18 asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
20 asmlinkage void chacha_4block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
22 asmlinkage void hchacha_block_ssse3(const u32 *state, u32 *out, int nrounds);
24 asmlinkage void chacha_2block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
26 asmlinkage void chacha_4block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
28 asmlinkage void chacha_8block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
31 asmlinkage void chacha_2block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
33 asmlinkage void chacha_4block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
35 asmlinkage void chacha_8block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
48 static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src,
54 			chacha_8block_xor_avx512vl(state, dst, src, bytes,
59 			state[12] += 8;
62 			chacha_8block_xor_avx512vl(state, dst, src, bytes,
64 			state[12] += chacha_advance(bytes, 8);
68 			chacha_4block_xor_avx512vl(state, dst, src, bytes,
70 			state[12] += chacha_advance(bytes, 4);
74 			chacha_2block_xor_avx512vl(state, dst, src, bytes,
76 			state[12] += chacha_advance(bytes, 2);
83 			chacha_8block_xor_avx2(state, dst, src, bytes, nrounds);
87 			state[12] += 8;
90 			chacha_8block_xor_avx2(state, dst, src, bytes, nrounds);
91 			state[12] += chacha_advance(bytes, 8);
95 			chacha_4block_xor_avx2(state, dst, src, bytes, nrounds);
96 			state[12] += chacha_advance(bytes, 4);
100 			chacha_2block_xor_avx2(state, dst, src, bytes, nrounds);
101 			state[12] += chacha_advance(bytes, 2);
107 		chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds);
111 		state[12] += 4;
114 		chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds);
115 		state[12] += chacha_advance(bytes, 4);
119 		chacha_block_xor_ssse3(state, dst, src, bytes, nrounds);
120 		state[12]++;
124 void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
127 		hchacha_block_generic(state, stream, nrounds);
130 		hchacha_block_ssse3(state, stream, nrounds);
136 void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
138 	chacha_init_generic(state, key, iv);
142 void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
147 		return chacha_crypt_generic(state, dst, src, bytes, nrounds);
153 		chacha_dosimd(state, dst, src, todo, nrounds);
166 	u32 state[CHACHA_STATE_WORDS] __aligned(8);
172 	chacha_init_generic(state, ctx->key, iv);
182 			chacha_crypt_generic(state, walk.dst.virt.addr,
187 			chacha_dosimd(state, walk.dst.virt.addr,
210 	u32 state[CHACHA_STATE_WORDS] __aligned(8);
214 	chacha_init_generic(state, ctx->key, req->iv);
218 		hchacha_block_ssse3(state, subctx.key, ctx->nrounds);
221 		hchacha_block_generic(state, subctx.key, ctx->nrounds);