libkern/x86/crc32_sse42.c

73  * Block sizes for three-way parallel crc computation.  LONG and SHORT must
80  * Tables for updating a crc for LONG, 2 * LONG, SHORT and 2 * SHORT bytes
124  * Construct an operator to apply len zeros to a crc.  len must be a power of
189 /* Apply the zeros operator table to crc. */
191 crc32c_shift(uint32_t zeros[][256], uint32_t crc)
194 	return (zeros[0][crc & 0xff] ^ zeros[1][(crc >> 8) & 0xff] ^
195 	    zeros[2][(crc >> 16) & 0xff] ^ zeros[3][crc >> 24]);
216 sse42_crc32c(uint32_t crc, const unsigned char *buf, unsigned len)
231 	crc0 = crc;
233 	/* Compute the crc to bring the data pointer to an aligned boundary. */
242 	 * Compute the crc on sets of LONG*3 bytes, executing three independent
243 	 * crc instructions, each on LONG bytes -- this is optimized for the
245 	 * have a throughput of one crc per cycle, but a latency of three
248 	crc = 0;
270 		 * Update the crc.  Try to do it in parallel with the inner
271 		 * loop.  'crc' is used to accumulate crc0 and crc1
276 		 *     crc = S*S*S*crc + S*S*crc0 + S*crc1
279 		 *     crc = S*S * (S*crc + crc0) + S*crc1.
287 		 * it in crc.  This synchronizes the loop with crc update.
292 		 * parallelizable take about 24 cycles and the crc update
299 		 * 12 cycles and the crc update about 24, but these are
305 		 * the crc update, the inner loop must take considerably
312 		crc = crc32c_shift(crc32c_long, crc) ^ crc0;
314 		crc = crc32c_shift(crc32c_2long, crc) ^ crc1;
319 	crc0 ^= crc;
326 	crc = 0;
347 		crc = crc32c_shift(crc32c_short, crc) ^ crc0;
349 		crc = crc32c_shift(crc32c_2short, crc) ^ crc1;
354 	crc0 ^= crc;
356 	/* Compute the crc on the remaining bytes at native word size. */
368 	/* Compute the crc for any trailing bytes. */