algs/modes/gcm.c

90     crypto_data_t *out, size_t block_size,
98 		    ctx, data, length, out, block_size));
115 	if (length + ctx->gcm_remainder_len < block_size) {
133 			need = block_size - ctx->gcm_remainder_len;
161 		ctx->gcm_processed_data_len += block_size;
164 		    &out_data_1_len, &out_data_2, block_size);
167 		if (out_data_1_len == block_size) {
174 				    block_size - out_data_1_len);
178 		out->cd_offset += block_size;
188 			datap += block_size;
194 		if (remainder > 0 && remainder < block_size) {
208 gcm_encrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
216 		return (gcm_encrypt_final_avx(ctx, out, block_size));
254 		    block_size - ctx->gcm_remainder_len);
294 gcm_decrypt_incomplete_block(gcm_ctx_t *ctx, size_t block_size, size_t index,
317 	memset((uint8_t *)ctx->gcm_tmp, 0, block_size);
334     crypto_data_t *out, size_t block_size,
339 	(void) out, (void) block_size, (void) encrypt_block, (void) copy_block,
376 gcm_decrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
382 		return (gcm_decrypt_final_avx(ctx, out, block_size));
404 		if (remainder < block_size) {
411 			gcm_decrypt_incomplete_block(ctx, block_size,
434 		processed += block_size;
435 		blockp += block_size;
436 		remainder -= block_size;
488     gcm_ctx_t *ctx, size_t block_size,
513 			if (remainder < block_size) {
514 				memset(cb, 0, block_size);
520 				processed += block_size;
521 				remainder -= block_size;
537     const uint8_t *auth_data, size_t auth_data_len, size_t block_size,
551 	gcm_format_initial_blocks(iv, iv_len, ctx, block_size,
557 	memset(authp, 0, block_size);
558 	memset(ghash, 0, block_size);
563 		if (remainder < block_size) {
570 				memset(authp, 0, block_size);
581 			processed += block_size;
582 			remainder -= block_size;
598 gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
603 	return (gcm_init_ctx_impl(B_FALSE, gcm_ctx, param, block_size,
612 gmac_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
617 	return (gcm_init_ctx_impl(B_TRUE, gcm_ctx, param, block_size,
627     size_t block_size, int (*encrypt_block)(const void *, const uint8_t *,
733 		if (gcm_init(gcm_ctx, iv, iv_len, aad, aad_len, block_size,
740 		    block_size) != CRYPTO_SUCCESS) {
1141     size_t length, crypto_data_t *out, size_t block_size)
1155 	ASSERT(block_size == GCM_BLOCK_LEN);
1163 		need = block_size - ctx->gcm_remainder_len;
1198 		GHASH_AVX(ctx, tmp, block_size);
1201 		rv = crypto_put_output_data(tmp, out, block_size);
1202 		out->cd_offset += block_size;
1204 		ctx->gcm_processed_data_len += block_size;
1254 		if (bleft < block_size) {
1265 		GHASH_AVX(ctx, tmp, block_size);
1266 		rv = crypto_put_output_data(tmp, out, block_size);
1270 		out->cd_offset += block_size;
1272 		ctx->gcm_processed_data_len += block_size;
1273 		datap += block_size;
1274 		bleft -= block_size;
1291 gcm_encrypt_final_avx(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size)
1301 	ASSERT(block_size == GCM_BLOCK_LEN);
1316 		memset(remainder + rem_len, 0, block_size - rem_len);
1320 		GHASH_AVX(ctx, remainder, block_size);
1327 	GHASH_AVX(ctx, (const uint8_t *)ctx->gcm_len_a_len_c, block_size);
1355 gcm_decrypt_final_avx(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size)
1358 	ASSERT3U(block_size, ==, 16);
1409 		if (bleft < block_size) {
1412 			memset(lastb, 0, block_size);
1415 			GHASH_AVX(ctx, lastb, block_size);
1423 		GHASH_AVX(ctx, datap, block_size);
1428 		datap += block_size;
1429 		bleft -= block_size;
1438 	GHASH_AVX(ctx, (uint8_t *)ctx->gcm_len_a_len_c, block_size);
1467     const uint8_t *auth_data, size_t auth_data_len, size_t block_size)
1477 	ASSERT(block_size == GCM_BLOCK_LEN);
1506 		gcm_format_initial_blocks(iv, iv_len, ctx, block_size,
1524 		size_t incomp = bleft % block_size;
1535 			memset(authp, 0, block_size);
1537 			GHASH_AVX(ctx, authp, block_size);