xcrypt.c revision 1219:f89f56c2d9ac 
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23/*
24 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
25 * Use is subject to license terms.
26 */
27
28/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
29/* All Rights Reserved */
30
31/*
32 * Portions of this source code were derived from Berkeley 4.3 BSD
33 * under license from the Regents of the University of California.
34 */
35
36#pragma ident	"%Z%%M%	%I%	%E% SMI"
37
38/*
39 * Hex encryption/decryption and utility routines
40 */
41
42#include "mt.h"
43#include <stdio.h>
44#include <stdlib.h>
45#include <sys/types.h>
46#include <rpc/rpc.h>
47#include <rpc/key_prot.h>   /* for KEYCHECKSUMSIZE */
48#include <rpc/des_crypt.h>
49#include <string.h>
50#include <rpcsvc/nis_dhext.h>
51#include <md5.h>
52
53#define	MD5HEXSIZE	32
54
55extern int bin2hex(int len, unsigned char *binnum, char *hexnum);
56extern int hex2bin(int len, char *hexnum, char *binnum);
57static char hex[];	/* forward */
58static char hexval();
59
60int passwd2des(char *, char *);
61static int weak_DES_key(des_block);
62
63/* EXPORT DELETE START */
64/*
65 * For export control reasons, we want to limit the maximum size of
66 * data that can be encrypted or decrypted.  We limit this to 1024
67 * bits of key data, which amounts to 128 bytes.
68 *
69 * For the extended DH project, we have increased it to
70 * 144 bytes (128key + 16checksum) to accomadate all the 128 bytes
71 * being used by the new 1024bit keys plus 16 bytes MD5 checksum.
72 * We discussed this with Sun's export control office and lawyers
73 * and we have reason to believe this is ok for export.
74 */
75#define	MAX_KEY_CRYPT_LEN	144
76/* EXPORT DELETE END */
77
78/*
79 * Encrypt a secret key given passwd
80 * The secret key is passed and returned in hex notation.
81 * Its length must be a multiple of 16 hex digits (64 bits).
82 */
83int
84xencrypt(secret, passwd)
85	char *secret;
86	char *passwd;
87{
88/* EXPORT DELETE START */
89	char key[8];
90	char ivec[8];
91	char *buf;
92	int err;
93	int len;
94
95	len = (int)strlen(secret) / 2;
96	if (len > MAX_KEY_CRYPT_LEN)
97		return (0);
98	buf = malloc((unsigned)len);
99	(void) hex2bin(len, secret, buf);
100	(void) passwd2des(passwd, key);
101	(void) memset(ivec, 0, 8);
102
103	err = cbc_crypt(key, buf, len, DES_ENCRYPT | DES_HW, ivec);
104	if (DES_FAILED(err)) {
105		free(buf);
106		return (0);
107	}
108	(void) bin2hex(len, (unsigned char *) buf, secret);
109	free(buf);
110	return (1);
111#if 0
112/* EXPORT DELETE END */
113	return (0);
114/* EXPORT DELETE START */
115#endif
116/* EXPORT DELETE END */
117}
118
119/*
120 * Decrypt secret key using passwd
121 * The secret key is passed and returned in hex notation.
122 * Once again, the length is a multiple of 16 hex digits
123 */
124int
125xdecrypt(secret, passwd)
126	char *secret;
127	char *passwd;
128{
129/* EXPORT DELETE START */
130	char key[8];
131	char ivec[8];
132	char *buf;
133	int err;
134	int len;
135
136	len = (int)strlen(secret) / 2;
137	if (len > MAX_KEY_CRYPT_LEN)
138		return (0);
139	buf = malloc((unsigned)len);
140
141	(void) hex2bin(len, secret, buf);
142	(void) passwd2des(passwd, key);
143	(void) memset(ivec, 0, 8);
144
145	err = cbc_crypt(key, buf, len, DES_DECRYPT | DES_HW, ivec);
146	if (DES_FAILED(err)) {
147		free(buf);
148		return (0);
149	}
150	(void) bin2hex(len, (unsigned char *) buf, secret);
151	free(buf);
152	return (1);
153#if 0
154/* EXPORT DELETE END */
155	return (0);
156/* EXPORT DELETE START */
157#endif
158/* EXPORT DELETE END */
159}
160
161/*
162 * Turn password into DES key
163 */
164int
165passwd2des(pw, key)
166	char *pw;
167	char *key;
168{
169	int i;
170
171	(void) memset(key, 0, 8);
172	for (i = 0; *pw; i = (i+1) % 8) {
173		key[i] ^= *pw++ << 1;
174	}
175	des_setparity(key);
176	return (1);
177}
178
179
180/*
181 * Hex to binary conversion
182 */
183int
184hex2bin(len, hexnum, binnum)
185	int len;
186	char *hexnum;
187	char *binnum;
188{
189	int i;
190
191	for (i = 0; i < len; i++) {
192		*binnum++ = 16 * hexval(hexnum[2 * i]) +
193					hexval(hexnum[2 * i + 1]);
194	}
195	return (1);
196}
197
198/*
199 * Binary to hex conversion
200 */
201int
202bin2hex(len, binnum, hexnum)
203	int len;
204	unsigned char *binnum;
205	char *hexnum;
206{
207	int i;
208	unsigned val;
209
210	for (i = 0; i < len; i++) {
211		val = binnum[i];
212		hexnum[i*2] = hex[val >> 4];
213		hexnum[i*2+1] = hex[val & 0xf];
214	}
215	hexnum[len*2] = 0;
216	return (1);
217}
218
219static char hex[16] = {
220	'0', '1', '2', '3', '4', '5', '6', '7',
221	'8', '9', 'a', 'b', 'c', 'd', 'e', 'f',
222};
223
224static char
225hexval(c)
226	char c;
227{
228	if (c >= '0' && c <= '9') {
229		return (c - '0');
230	} else if (c >= 'a' && c <= 'z') {
231		return (c - 'a' + 10);
232	} else if (c >= 'A' && c <= 'Z') {
233		return (c - 'A' + 10);
234	} else {
235		return (-1);
236	}
237}
238
239/*
240 * Generic key length/algorithm version of xencrypt().
241 *
242 * Encrypt a secret key given passwd.
243 * The secret key is passed in hex notation.
244 * Arg encrypted_secret will be set to point to the encrypted
245 * secret key (NUL term, hex notation).
246 *
247 * Its length must be a multiple of 16 hex digits (64 bits).
248 *
249 * For 192-0 (AUTH_DES), then encrypt using the same method as xencrypt().
250 *
251 * If arg do_chksum is TRUE, append the checksum before the encrypt.
252 * For 192-0, the checksum is done the same as in xencrypt().  For
253 * bigger keys, MD5 is used.
254 *
255 * Arg netname can be NULL for 192-0.
256 */
257int
258xencrypt_g(
259	char *secret,			/* in  */
260	keylen_t keylen,		/* in  */
261	algtype_t algtype,		/* in  */
262	const char *passwd,		/* in  */
263	const char netname[],  		/* in  */
264	char **encrypted_secret,	/* out */
265	bool_t do_chksum)		/* in  */
266{
267/* EXPORT DELETE START */
268	des_block key;
269	char ivec[8];
270	char *binkeybuf;
271	int err;
272	const int classic_des = keylen == 192 && algtype == 0;
273	const int hexkeybytes = BITS2NIBBLES(keylen);
274	const int keychecksumsize = classic_des ? KEYCHECKSUMSIZE : MD5HEXSIZE;
275	const int binkeybytes = do_chksum ? keylen/8 + keychecksumsize/2 :
276		keylen/8;
277	const int bufsize = do_chksum ? hexkeybytes + keychecksumsize + 1 :
278		hexkeybytes + 1;
279	char *hexkeybuf;
280
281	if (!secret || !keylen || !passwd || !encrypted_secret)
282		return (0);
283
284	if ((hexkeybuf = malloc(bufsize)) == 0)
285		return (0);
286
287	(void) memcpy(hexkeybuf, secret, hexkeybytes);
288	if (do_chksum)
289		if (classic_des) {
290			(void) memcpy(hexkeybuf + hexkeybytes, secret,
291					keychecksumsize);
292		} else {
293			MD5_CTX md5_ctx;
294			char md5hexbuf[MD5HEXSIZE + 1] = {0};
295			uint8_t digest[MD5HEXSIZE/2];
296
297			MD5Init(&md5_ctx);
298			MD5Update(&md5_ctx, (unsigned char *)hexkeybuf,
299					hexkeybytes);
300			MD5Final(digest, &md5_ctx);
301
302			/* convert md5 binary digest to hex */
303			(void) bin2hex(MD5HEXSIZE/2, digest, md5hexbuf);
304
305			/* append the hex md5 string to the end of the key */
306			(void) memcpy(hexkeybuf + hexkeybytes,
307					(void *)md5hexbuf, MD5HEXSIZE);
308		}
309	hexkeybuf[bufsize - 1] = 0;
310
311	if (binkeybytes > MAX_KEY_CRYPT_LEN) {
312		free(hexkeybuf);
313		return (0);
314	}
315	if ((binkeybuf = malloc((unsigned)binkeybytes)) == 0) {
316		free(hexkeybuf);
317		return (0);
318	}
319
320	(void) hex2bin(binkeybytes, hexkeybuf, binkeybuf);
321	if (classic_des)
322		(void) passwd2des((char *)passwd, key.c);
323	else
324		if (netname)
325			(void) passwd2des_g(passwd, netname,
326					(int)strlen(netname), &key, FALSE);
327		else {
328			free(hexkeybuf);
329			return (0);
330		}
331
332	(void) memset(ivec, 0, 8);
333
334	err = cbc_crypt(key.c, binkeybuf, binkeybytes, DES_ENCRYPT | DES_HW,
335			ivec);
336	if (DES_FAILED(err)) {
337		free(hexkeybuf);
338		free(binkeybuf);
339		return (0);
340	}
341	(void) bin2hex(binkeybytes, (unsigned char *) binkeybuf, hexkeybuf);
342	free(binkeybuf);
343	*encrypted_secret = hexkeybuf;
344	return (1);
345#if 0
346/* EXPORT DELETE END */
347	return (0);
348/* EXPORT DELETE START */
349#endif
350/* EXPORT DELETE END */
351}
352
353/*
354 * Generic key len and alg type for version of xdecrypt.
355 *
356 * Decrypt secret key using passwd.  The decrypted secret key
357 * *overwrites* the supplied encrypted secret key.
358 * The secret key is passed and returned in hex notation.
359 * Once again, the length is a multiple of 16 hex digits.
360 *
361 * If 'do_chksum' is TRUE, the 'secret' buffer is assumed to contain
362 * a checksum calculated by a call to xencrypt_g().
363 *
364 * If keylen is 192 and algtype is 0, then decrypt the same way
365 * as xdecrypt().
366 *
367 * Arg netname can be NULL for 192-0.
368 */
369int
370xdecrypt_g(
371	char *secret,		/* out  */
372	int keylen,		/* in  */
373	int algtype,		/* in  */
374	const char *passwd,	/* in  */
375	const char netname[],	/* in  */
376	bool_t do_chksum)	/* in  */
377{
378/* EXPORT DELETE START */
379	des_block key;
380	char ivec[8];
381	char *buf;
382	int err;
383	int len;
384	const int classic_des = keylen == 192 && algtype == 0;
385	const int hexkeybytes = BITS2NIBBLES(keylen);
386	const int keychecksumsize = classic_des ? KEYCHECKSUMSIZE : MD5HEXSIZE;
387
388	len = (int)strlen(secret) / 2;
389	if (len > MAX_KEY_CRYPT_LEN)
390		return (0);
391	if ((buf = malloc((unsigned)len)) == 0)
392		return (0);
393
394	(void) hex2bin(len, secret, buf);
395	if (classic_des)
396		(void) passwd2des((char *)passwd, key.c);
397	else
398		if (netname)
399			(void) passwd2des_g(passwd, netname,
400					(int)strlen(netname), &key, FALSE);
401		else {
402			free(buf);
403			return (0);
404		}
405	(void) memset(ivec, 0, 8);
406
407	err = cbc_crypt(key.c, buf, len, DES_DECRYPT | DES_HW, ivec);
408	if (DES_FAILED(err)) {
409		free(buf);
410		return (0);
411	}
412	(void) bin2hex(len, (unsigned char *) buf, secret);
413	free(buf);
414
415	if (do_chksum)
416		if (classic_des) {
417			if (memcmp(secret, &(secret[hexkeybytes]),
418					keychecksumsize) != 0) {
419				secret[0] = 0;
420				return (0);
421			}
422		} else {
423			MD5_CTX md5_ctx;
424			char md5hexbuf[MD5HEXSIZE + 1] = {0};
425			uint8_t digest[MD5HEXSIZE/2];
426
427			MD5Init(&md5_ctx);
428			MD5Update(&md5_ctx, (unsigned char *)secret,
429					hexkeybytes);
430			MD5Final(digest, &md5_ctx);
431
432			/* convert md5 binary digest to hex */
433			(void) bin2hex(MD5HEXSIZE/2, digest, md5hexbuf);
434
435			/* does the digest match the appended one? */
436			if (memcmp(&(secret[hexkeybytes]),
437					md5hexbuf, MD5HEXSIZE) != 0) {
438				secret[0] = 0;
439				return (0);
440			}
441		}
442
443	secret[hexkeybytes] = '\0';
444
445	return (1);
446#if 0
447/* EXPORT DELETE END */
448	return (0);
449/* EXPORT DELETE START */
450#endif
451/* EXPORT DELETE END */
452}
453
454
455/*
456 * Modified version of passwd2des(). passwd2des_g() uses the Kerberos
457 * RFC 1510 algorithm to generate a DES key from a user password
458 * and mix-in string. The mix-in is expected to be the netname.
459 * This function to be used only for extended Diffie-Hellman keys.
460 *
461 * If altarg is TRUE, reverse the concat of passwd and mix-in.
462 */
463int
464passwd2des_g(
465	const char *pw,
466	const char *mixin,
467	int len,
468	des_block *key, /* out */
469	bool_t altalg)
470{
471
472	int  i, j, incr = 1;
473	des_block ivec, tkey;
474	char *text;
475	int  plen, tlen;
476
477	(void) memset(tkey.c, 0, 8);
478	(void) memset(ivec.c, 0, 8);
479
480
481/*
482 * Concatentate the password and the mix-in string, fan-fold and XOR them
483 * to the required eight byte initial DES key. Since passwords can be
484 * expected to use mostly seven bit ASCII, left shift the password one
485 * bit in order to preserve as much key space as possible.
486 */
487
488#define	KEYLEN sizeof (tkey.c)
489	plen = strlen(pw);
490	tlen = ((plen + len + (KEYLEN-1))/KEYLEN)*KEYLEN;
491	if ((text = malloc(tlen)) == NULL) {
492		return (0);
493	}
494
495	(void) memset(text, 0, tlen);
496
497	if (!altalg) {
498
499/*
500 * Concatenate the password and the mix-in string, fan-fold and XOR them
501 * to the required eight byte initial DES key. Since passwords can be
502 * expected to use mostly seven bit ASCII, left shift the password one
503 * bit in order to preserve as much key space as possible.
504 */
505		(void) memcpy(text, pw, plen);
506		(void) memcpy(&text[plen], mixin, len);
507
508		for (i = 0, j = 0; pw[j]; j++) {
509			tkey.c[i] ^= pw[j] << 1;
510			i += incr;
511			if (i == 8) {
512				i = 7;
513				incr = -incr;
514			} else if (i == -1) {
515				i = 0;
516				incr = -incr;
517			}
518		}
519
520		for (j = 0; j < len; j++) {
521			tkey.c[i] ^= mixin[j];
522			i += incr;
523			if (i == 8) {
524				i = 7;
525				incr = -incr;
526			} else if (i == -1) {
527				i = 0;
528				incr = -incr;
529			}
530		}
531	} else {  /* use alternative algorithm */
532		(void) memcpy(text, mixin, len);
533		(void) memcpy(&text[len], pw, plen);
534
535		for (i = 0, j = 0; j < len; j++) {
536			tkey.c[i] ^= mixin[j];
537			i += incr;
538			if (i == 8) {
539				i = 7;
540				incr = -incr;
541			} else if (i == -1) {
542				i = 0;
543				incr = -incr;
544			}
545		}
546
547		for (j = 0; pw[j]; j++) {
548			tkey.c[i] ^= pw[j] << 1;
549			i += incr;
550			if (i == 8) {
551				i = 7;
552				incr = -incr;
553			} else if (i == -1) {
554				i = 0;
555				incr = -incr;
556			}
557		}
558	}
559	des_setparity_g(&tkey);
560
561	/*
562	 * Use the temporary key to produce a DES CBC checksum for the text
563	 * string; cbc_crypt returns the checksum in the ivec.
564	 */
565	(void) cbc_crypt(tkey.c, text, tlen, DES_ENCRYPT|DES_HW, ivec.c);
566	des_setparity_g(&ivec);
567	free(text);
568
569	if (weak_DES_key(ivec)) {
570		ivec.c[7] ^= 0xf0;
571		/*
572		 *  XORing with 0xf0 preserves parity, so no need to check
573		 *  that again.
574		 */
575	}
576
577	(void) memcpy((*key).c, ivec.c, sizeof (ivec.c));
578
579	return (1);
580
581}
582
583struct DESkey {
584	uint32_t h1;
585	uint32_t h2;
586};
587
588/*
589 * Weak and semiweak keys from "Applied Cryptography", second edition,
590 * by Bruce Schneier, Wiley 1996.
591 */
592static struct DESkey weakDESkeys[] = {
593	/* Weak keys */
594	{0x01010101, 0x01010101},
595	{0x1f1f1f1f, 0x1f1f1f1f},
596	{0xe0e0e0e0, 0xe0e0e0e0},
597	{0xfefefefe, 0xfefefefe},
598	/* Semiweak keys */
599	{0x01fe01fe, 0x01fe01fe},
600	{0x1fe01fe0, 0x0ef10ef1},
601	{0x01e001e0, 0x01f101f1},
602	{0x1ffe1ffe, 0x0efe0efe},
603	{0x011f011f, 0x010e010e},
604	{0xe0fee0fe, 0xf1fef1fe},
605	{0xfe01fe01, 0xfe01fe01},
606	{0xe01fe01f, 0xf10ef10e},
607	{0xe001e001, 0xf101f101},
608	{0xfe1ffe1f, 0xfe0efe0e},
609	{0x1f011f01, 0x0e010e01},
610	{0xfee0fee0, 0xfef1fef1}
611};
612
613static int
614weak_DES_key(des_block db)
615{
616	int i;
617
618	for (i = 0; i < sizeof (weakDESkeys)/sizeof (struct DESkey); i++) {
619		if (weakDESkeys[i].h1 == db.key.high &&
620			weakDESkeys[i].h2 == db.key.low)
621			return (1);
622	}
623
624	return (0);
625}
626