1/******************************************************************************* 2* Copyright (c) 2013, Intel Corporation 3* 4* All rights reserved. 5* 6* Redistribution and use in source and binary forms, with or without 7* modification, are permitted provided that the following conditions are 8* met: 9* 10* * Redistributions of source code must retain the above copyright 11* notice, this list of conditions and the following disclaimer. 12* 13* * Redistributions in binary form must reproduce the above copyright 14* notice, this list of conditions and the following disclaimer in the 15* documentation and/or other materials provided with the 16* distribution. 17* 18* * Neither the name of the Intel Corporation nor the names of its 19* contributors may be used to endorse or promote products derived from 20* this software without specific prior written permission. 21* 22* 23* THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY 24* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 26* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR 27* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 28* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 29* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 30* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 31* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 32* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 33* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34******************************************************************************** 35* 36* Intel SHA Extensions optimized implementation of a SHA-256 update function 37* 38* The function takes a pointer to the current hash values, a pointer to the 39* input data, and a number of 64 byte blocks to process. Once all blocks have 40* been processed, the digest pointer is updated with the resulting hash value. 41* The function only processes complete blocks, there is no functionality to 42* store partial blocks. All message padding and hash value initialization must 43* be done outside the update function. 44* 45* The indented lines in the loop are instructions related to rounds processing. 46* The non-indented lines are instructions related to the message schedule. 47* 48* Author: Sean Gulley <sean.m.gulley@intel.com> 49* Date: July 2013 50* 51******************************************************************************** 52* 53* Example complier command line: 54* icc intel_sha_extensions_sha256_intrinsic.c 55* gcc -msha -msse4 intel_sha_extensions_sha256_intrinsic.c 56* 57*******************************************************************************/ 58#include <sys/cdefs.h> 59__FBSDID("$FreeBSD$"); 60 61#include <sys/types.h> 62#include <crypto/aesni/aesni_os.h> 63#include <crypto/aesni/sha_sse.h> 64 65#include <immintrin.h> 66 67void intel_sha256_step(uint32_t *digest, const char *data, uint32_t num_blks) { 68 __m128i state0, state1; 69 __m128i msg; 70 __m128i msgtmp0, msgtmp1, msgtmp2, msgtmp3; 71 __m128i tmp; 72 __m128i shuf_mask; 73 __m128i abef_save, cdgh_save; 74 75 // Load initial hash values 76 // Need to reorder these appropriately 77 // DCBA, HGFE -> ABEF, CDGH 78 tmp = _mm_loadu_si128((__m128i*) digest); 79 state1 = _mm_loadu_si128((__m128i*) (digest+4)); 80 81 tmp = _mm_shuffle_epi32(tmp, 0xB1); // CDAB 82 state1 = _mm_shuffle_epi32(state1, 0x1B); // EFGH 83 state0 = _mm_alignr_epi8(tmp, state1, 8); // ABEF 84 state1 = _mm_blend_epi16(state1, tmp, 0xF0); // CDGH 85 86 shuf_mask = _mm_set_epi64x(0x0c0d0e0f08090a0bull, 0x0405060700010203ull); 87 88 while (num_blks > 0) { 89 // Save hash values for addition after rounds 90 abef_save = state0; 91 cdgh_save = state1; 92 93 // Rounds 0-3 94 msg = _mm_loadu_si128((const __m128i*) data); 95 msgtmp0 = _mm_shuffle_epi8(msg, shuf_mask); 96 msg = _mm_add_epi32(msgtmp0, 97 _mm_set_epi64x(0xE9B5DBA5B5C0FBCFull, 0x71374491428A2F98ull)); 98 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 99 msg = _mm_shuffle_epi32(msg, 0x0E); 100 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 101 102 // Rounds 4-7 103 msgtmp1 = _mm_loadu_si128((const __m128i*) (data+16)); 104 msgtmp1 = _mm_shuffle_epi8(msgtmp1, shuf_mask); 105 msg = _mm_add_epi32(msgtmp1, 106 _mm_set_epi64x(0xAB1C5ED5923F82A4ull, 0x59F111F13956C25Bull)); 107 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 108 msg = _mm_shuffle_epi32(msg, 0x0E); 109 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 110 msgtmp0 = _mm_sha256msg1_epu32(msgtmp0, msgtmp1); 111 112 // Rounds 8-11 113 msgtmp2 = _mm_loadu_si128((const __m128i*) (data+32)); 114 msgtmp2 = _mm_shuffle_epi8(msgtmp2, shuf_mask); 115 msg = _mm_add_epi32(msgtmp2, 116 _mm_set_epi64x(0x550C7DC3243185BEull, 0x12835B01D807AA98ull)); 117 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 118 msg = _mm_shuffle_epi32(msg, 0x0E); 119 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 120 msgtmp1 = _mm_sha256msg1_epu32(msgtmp1, msgtmp2); 121 122 // Rounds 12-15 123 msgtmp3 = _mm_loadu_si128((const __m128i*) (data+48)); 124 msgtmp3 = _mm_shuffle_epi8(msgtmp3, shuf_mask); 125 msg = _mm_add_epi32(msgtmp3, 126 _mm_set_epi64x(0xC19BF1749BDC06A7ull, 0x80DEB1FE72BE5D74ull)); 127 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 128 tmp = _mm_alignr_epi8(msgtmp3, msgtmp2, 4); 129 msgtmp0 = _mm_add_epi32(msgtmp0, tmp); 130 msgtmp0 = _mm_sha256msg2_epu32(msgtmp0, msgtmp3); 131 msg = _mm_shuffle_epi32(msg, 0x0E); 132 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 133 msgtmp2 = _mm_sha256msg1_epu32(msgtmp2, msgtmp3); 134 135 // Rounds 16-19 136 msg = _mm_add_epi32(msgtmp0, 137 _mm_set_epi64x(0x240CA1CC0FC19DC6ull, 0xEFBE4786E49B69C1ull)); 138 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 139 tmp = _mm_alignr_epi8(msgtmp0, msgtmp3, 4); 140 msgtmp1 = _mm_add_epi32(msgtmp1, tmp); 141 msgtmp1 = _mm_sha256msg2_epu32(msgtmp1, msgtmp0); 142 msg = _mm_shuffle_epi32(msg, 0x0E); 143 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 144 msgtmp3 = _mm_sha256msg1_epu32(msgtmp3, msgtmp0); 145 146 // Rounds 20-23 147 msg = _mm_add_epi32(msgtmp1, 148 _mm_set_epi64x(0x76F988DA5CB0A9DCull, 0x4A7484AA2DE92C6Full)); 149 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 150 tmp = _mm_alignr_epi8(msgtmp1, msgtmp0, 4); 151 msgtmp2 = _mm_add_epi32(msgtmp2, tmp); 152 msgtmp2 = _mm_sha256msg2_epu32(msgtmp2, msgtmp1); 153 msg = _mm_shuffle_epi32(msg, 0x0E); 154 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 155 msgtmp0 = _mm_sha256msg1_epu32(msgtmp0, msgtmp1); 156 157 // Rounds 24-27 158 msg = _mm_add_epi32(msgtmp2, 159 _mm_set_epi64x(0xBF597FC7B00327C8ull, 0xA831C66D983E5152ull)); 160 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 161 tmp = _mm_alignr_epi8(msgtmp2, msgtmp1, 4); 162 msgtmp3 = _mm_add_epi32(msgtmp3, tmp); 163 msgtmp3 = _mm_sha256msg2_epu32(msgtmp3, msgtmp2); 164 msg = _mm_shuffle_epi32(msg, 0x0E); 165 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 166 msgtmp1 = _mm_sha256msg1_epu32(msgtmp1, msgtmp2); 167 168 // Rounds 28-31 169 msg = _mm_add_epi32(msgtmp3, 170 _mm_set_epi64x(0x1429296706CA6351ull, 0xD5A79147C6E00BF3ull)); 171 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 172 tmp = _mm_alignr_epi8(msgtmp3, msgtmp2, 4); 173 msgtmp0 = _mm_add_epi32(msgtmp0, tmp); 174 msgtmp0 = _mm_sha256msg2_epu32(msgtmp0, msgtmp3); 175 msg = _mm_shuffle_epi32(msg, 0x0E); 176 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 177 msgtmp2 = _mm_sha256msg1_epu32(msgtmp2, msgtmp3); 178 179 // Rounds 32-35 180 msg = _mm_add_epi32(msgtmp0, 181 _mm_set_epi64x(0x53380D134D2C6DFCull, 0x2E1B213827B70A85ull)); 182 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 183 tmp = _mm_alignr_epi8(msgtmp0, msgtmp3, 4); 184 msgtmp1 = _mm_add_epi32(msgtmp1, tmp); 185 msgtmp1 = _mm_sha256msg2_epu32(msgtmp1, msgtmp0); 186 msg = _mm_shuffle_epi32(msg, 0x0E); 187 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 188 msgtmp3 = _mm_sha256msg1_epu32(msgtmp3, msgtmp0); 189 190 // Rounds 36-39 191 msg = _mm_add_epi32(msgtmp1, 192 _mm_set_epi64x(0x92722C8581C2C92Eull, 0x766A0ABB650A7354ull)); 193 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 194 tmp = _mm_alignr_epi8(msgtmp1, msgtmp0, 4); 195 msgtmp2 = _mm_add_epi32(msgtmp2, tmp); 196 msgtmp2 = _mm_sha256msg2_epu32(msgtmp2, msgtmp1); 197 msg = _mm_shuffle_epi32(msg, 0x0E); 198 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 199 msgtmp0 = _mm_sha256msg1_epu32(msgtmp0, msgtmp1); 200 201 // Rounds 40-43 202 msg = _mm_add_epi32(msgtmp2, 203 _mm_set_epi64x(0xC76C51A3C24B8B70ull, 0xA81A664BA2BFE8A1ull)); 204 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 205 tmp = _mm_alignr_epi8(msgtmp2, msgtmp1, 4); 206 msgtmp3 = _mm_add_epi32(msgtmp3, tmp); 207 msgtmp3 = _mm_sha256msg2_epu32(msgtmp3, msgtmp2); 208 msg = _mm_shuffle_epi32(msg, 0x0E); 209 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 210 msgtmp1 = _mm_sha256msg1_epu32(msgtmp1, msgtmp2); 211 212 // Rounds 44-47 213 msg = _mm_add_epi32(msgtmp3, 214 _mm_set_epi64x(0x106AA070F40E3585ull, 0xD6990624D192E819ull)); 215 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 216 tmp = _mm_alignr_epi8(msgtmp3, msgtmp2, 4); 217 msgtmp0 = _mm_add_epi32(msgtmp0, tmp); 218 msgtmp0 = _mm_sha256msg2_epu32(msgtmp0, msgtmp3); 219 msg = _mm_shuffle_epi32(msg, 0x0E); 220 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 221 msgtmp2 = _mm_sha256msg1_epu32(msgtmp2, msgtmp3); 222 223 // Rounds 48-51 224 msg = _mm_add_epi32(msgtmp0, 225 _mm_set_epi64x(0x34B0BCB52748774Cull, 0x1E376C0819A4C116ull)); 226 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 227 tmp = _mm_alignr_epi8(msgtmp0, msgtmp3, 4); 228 msgtmp1 = _mm_add_epi32(msgtmp1, tmp); 229 msgtmp1 = _mm_sha256msg2_epu32(msgtmp1, msgtmp0); 230 msg = _mm_shuffle_epi32(msg, 0x0E); 231 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 232 msgtmp3 = _mm_sha256msg1_epu32(msgtmp3, msgtmp0); 233 234 // Rounds 52-55 235 msg = _mm_add_epi32(msgtmp1, 236 _mm_set_epi64x(0x682E6FF35B9CCA4Full, 0x4ED8AA4A391C0CB3ull)); 237 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 238 tmp = _mm_alignr_epi8(msgtmp1, msgtmp0, 4); 239 msgtmp2 = _mm_add_epi32(msgtmp2, tmp); 240 msgtmp2 = _mm_sha256msg2_epu32(msgtmp2, msgtmp1); 241 msg = _mm_shuffle_epi32(msg, 0x0E); 242 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 243 244 // Rounds 56-59 245 msg = _mm_add_epi32(msgtmp2, 246 _mm_set_epi64x(0x8CC7020884C87814ull, 0x78A5636F748F82EEull)); 247 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 248 tmp = _mm_alignr_epi8(msgtmp2, msgtmp1, 4); 249 msgtmp3 = _mm_add_epi32(msgtmp3, tmp); 250 msgtmp3 = _mm_sha256msg2_epu32(msgtmp3, msgtmp2); 251 msg = _mm_shuffle_epi32(msg, 0x0E); 252 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 253 254 // Rounds 60-63 255 msg = _mm_add_epi32(msgtmp3, 256 _mm_set_epi64x(0xC67178F2BEF9A3F7ull, 0xA4506CEB90BEFFFAull)); 257 state1 = _mm_sha256rnds2_epu32(state1, state0, msg); 258 msg = _mm_shuffle_epi32(msg, 0x0E); 259 state0 = _mm_sha256rnds2_epu32(state0, state1, msg); 260 261 // Add current hash values with previously saved 262 state0 = _mm_add_epi32(state0, abef_save); 263 state1 = _mm_add_epi32(state1, cdgh_save); 264 265 data += 64; 266 num_blks--; 267 } 268 269 // Write hash values back in the correct order 270 tmp = _mm_shuffle_epi32(state0, 0x1B); // FEBA 271 state1 = _mm_shuffle_epi32(state1, 0xB1); // DCHG 272 state0 = _mm_blend_epi16(tmp, state1, 0xF0); // DCBA 273 state1 = _mm_alignr_epi8(state1, tmp, 8); // ABEF 274 275 _mm_store_si128((__m128i*) digest, state0); 276 _mm_store_si128((__m128i*) (digest+4), state1); 277} 278 279