1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2009, Intel Corporation.
24 * All rights reserved.
25 */
26
27/*
28 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
29 */
30
31#ifndef	_CRC_HD_H
32#define	_CRC_HD_H
33
34#ifdef __cplusplus
35extern "C" {
36#endif
37
38#include <sys/types.h>
39
40#ifdef __cplusplus
41#ifndef INLINE
42#define	INLINE inline
43#endif
44#else
45#ifndef INLINE
46#define	INLINE
47#endif
48#endif
49
50#if defined(__i386) || defined(__amd_64) || defined(__x86_64)
51
52#ifdef _KERNEL
53#include <sys/x86_archext.h>	/* x86_featureset, X86FSET_SSE4_2 */
54
55#else
56
57#include <sys/auxv.h>		/* getisax() */
58#include <sys/auxv_386.h>	/* AV_386_SSE4_2 bit */
59
60#endif
61
62static INLINE uint32_t
63#ifdef _KERNEL
64/*LINTED:E_FUNC_ARG_UNUSED*/
65mm_crc32_u8(uint32_t crc, uint8_t *data)
66#else
67mm_crc32_u8(uint32_t crc, uint8_t *data)
68#endif /* _KERNEL */
69{
70	__asm__ __volatile__(
71	    /* "crc32 r32, r/m8" */
72	    ".byte 0xF2, 0x0F, 0x38, 0xF0, 0xF1"
73	    : "=S" (crc)
74	    : "0" (crc), "c" (*data));
75
76	return (crc);
77}
78
79#if defined(__amd64) || defined(__x86_64)
80
81static INLINE uint32_t
82#ifdef _KERNEL
83/*LINTED:E_FUNC_ARG_UNUSED*/
84mm_crc32_u64(uint32_t crc, uint64_t *data)
85#else
86mm_crc32_u64(uint32_t crc, uint64_t *data)
87#endif /* _KERNEL */
88{
89	__asm__ __volatile__(
90	    /* "crc32 r32, r/m64" */
91	    ".byte 0xF2, 0x48, 0x0F, 0x38, 0xF1, 0xF1"
92	    : "=S" (crc)
93	    : "0" (crc), "c" (*data));
94
95	return (crc);
96}
97
98#define	INTEL_CRC_ALIGN_MASK 0x7
99#define	INTEL_CRC_SIZE 8
100#define	INTEL_CRC_DATA_TYPE uint64_t
101#define	INTEL_CRC_FUNC mm_crc32_u64
102
103#else
104
105static INLINE uint32_t
106#ifdef _KERNEL
107/*LINTED:E_FUNC_ARG_UNUSED*/
108mm_crc32_u32(uint32_t crc, uint32_t *data)
109#else
110mm_crc32_u32(uint32_t crc, uint32_t *data)
111#endif /* _KERNEL */
112{
113	__asm__ __volatile__(
114	    /* "crc32 r32, r/m32" */
115	    ".byte 0xF2, 0x0F, 0x38, 0xF1, 0xF1"
116	    : "=S" (crc)
117	    : "0" (crc), "c" (*data));
118
119	return (crc);
120}
121
122#define	INTEL_CRC_ALIGN_MASK 0x3
123#define	INTEL_CRC_SIZE 4
124#define	INTEL_CRC_DATA_TYPE uint32_t
125#define	INTEL_CRC_FUNC mm_crc32_u32
126
127#endif
128
129static INLINE uint32_t
130intel_crc32c(uint8_t *address, unsigned long length, uint32_t crc)
131{
132	uint32_t i = (uintptr_t)address & INTEL_CRC_ALIGN_MASK;
133	uint8_t *data = address;
134
135	/* Process unaligned header data */
136	while ((length > 0) && (i > 0)) {
137		crc = mm_crc32_u8(crc, data);
138		++ data;
139		-- length;
140		i ++;
141		i &= INTEL_CRC_ALIGN_MASK;
142	}
143
144	/* aligned data part */
145	while (length >= INTEL_CRC_SIZE) {
146		/*LINTED:E_BAD_PTR_CAST_ALIGN*/
147		crc = INTEL_CRC_FUNC(crc, (INTEL_CRC_DATA_TYPE *)data);
148		data += INTEL_CRC_SIZE;
149		length -= INTEL_CRC_SIZE;
150	}
151
152	/* unaligned tail data */
153	while (length > 0) {
154		crc = mm_crc32_u8(crc, data);
155		++ data;
156		-- length;
157	}
158
159	return (crc ^ 0xFFFFFFFF);
160}
161
162#define	HW_CRC32(buffer, length, crc) (intel_crc32c((buffer), (length), (crc)))
163#define	HW_CRC32_CONT(buffer, length, crc) \
164	(intel_crc32c((buffer), (length), (crc) ^ 0xFFFFFFFF))
165#else
166#define	HW_CRC32(buffer, length, crc)		0
167#define	HW_CRC32_CONT(buffer, length, crc)	0
168#endif
169
170static INLINE boolean_t
171#if defined(_KERNEL) && !defined(__i386) && !defined(__amd_64) &&\
172	!defined(__x86_64)
173/*LINTED:E_FUNC_ARG_UNUSED*/
174hd_crc32_avail(uint32_t *crc32_table)
175#else
176hd_crc32_avail(uint32_t *crc32_table)
177#endif
178{
179#if defined(__i386) || defined(__amd_64) || defined(__x86_64)
180	int i;
181	/* poly = 0x1EDC6F41 */
182	static const uint32_t _intel_crc32_hd_table[256] = {
183		0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4,
184		0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB,
185		0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B,
186		0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24,
187		0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B,
188		0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384,
189		0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54,
190		0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B,
191		0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A,
192		0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35,
193		0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5,
194		0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA,
195		0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45,
196		0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A,
197		0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A,
198		0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595,
199		0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48,
200		0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957,
201		0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687,
202		0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198,
203		0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927,
204		0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38,
205		0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8,
206		0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7,
207		0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096,
208		0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789,
209		0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859,
210		0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46,
211		0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9,
212		0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6,
213		0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36,
214		0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829,
215		0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C,
216		0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93,
217		0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043,
218		0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C,
219		0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3,
220		0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC,
221		0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C,
222		0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033,
223		0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652,
224		0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D,
225		0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D,
226		0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982,
227		0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D,
228		0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622,
229		0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2,
230		0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED,
231		0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530,
232		0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F,
233		0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF,
234		0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0,
235		0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F,
236		0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540,
237		0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90,
238		0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F,
239		0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE,
240		0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1,
241		0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321,
242		0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E,
243		0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81,
244		0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E,
245		0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E,
246		0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351
247	};
248
249#ifdef _KERNEL
250	if (!is_x86_feature(x86_featureset, X86FSET_SSE4_2)) {
251		return (B_FALSE);
252	} else {
253#else
254	{
255		uint_t	ui = 0;
256
257		(void) getisax(&ui, 1);
258
259		if (!(ui & AV_386_SSE4_2)) {
260			return (B_FALSE);
261		}
262#endif /* _KERNEL */
263		for (i = 0; i < 256; i++) {
264			if (crc32_table[i] != _intel_crc32_hd_table[i])
265				return (B_FALSE);
266		}
267		return (B_TRUE);
268	}
269#else
270	return (B_FALSE);
271#endif
272}
273
274#ifdef __cplusplus
275}
276#endif
277
278#endif /* _CRC_HD_H */
279