enc_des.c revision 57442 
1/*-
2 * Copyright (c) 1991, 1993
3 *	The Regents of the University of California.  All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 *    must display the following acknowledgement:
15 *	This product includes software developed by the University of
16 *	California, Berkeley and its contributors.
17 * 4. Neither the name of the University nor the names of its contributors
18 *    may be used to endorse or promote products derived from this software
19 *    without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 * $FreeBSD: head/contrib/telnet/libtelnet/enc_des.c 57442 2000-02-24 19:28:31Z markm $
34 */
35
36#ifndef lint
37static const char sccsid[] = "@(#)enc_des.c	8.3 (Berkeley) 5/30/95";
38#endif /* not lint */
39
40#ifdef	ENCRYPTION
41# ifdef	AUTHENTICATION
42#  ifdef DES_ENCRYPTION
43#include <arpa/telnet.h>
44#include <stdio.h>
45#ifdef	__STDC__
46#include <stdlib.h>
47#endif
48
49#include <openssl/des.h>
50#include <string.h>
51#include "encrypt.h"
52#include "key-proto.h"
53#include "misc-proto.h"
54
55extern encrypt_debug_mode;
56
57#define	CFB	0
58#define	OFB	1
59
60#define	NO_SEND_IV	1
61#define	NO_RECV_IV	2
62#define	NO_KEYID	4
63#define	IN_PROGRESS	(NO_SEND_IV|NO_RECV_IV|NO_KEYID)
64#define	SUCCESS		0
65#define	FAILED		-1
66
67
68struct fb {
69	Block krbdes_key;
70	Schedule krbdes_sched;
71	Block temp_feed;
72	unsigned char fb_feed[64];
73	int need_start;
74	int state[2];
75	int keyid[2];
76	int once;
77	struct stinfo {
78		Block		str_output;
79		Block		str_feed;
80		Block		str_iv;
81		Block		str_ikey;
82		Schedule	str_sched;
83		int		str_index;
84		int		str_flagshift;
85	} streams[2];
86};
87
88static struct fb fb[2];
89
90struct keyidlist {
91	char	*keyid;
92	int	keyidlen;
93	char	*key;
94	int	keylen;
95	int	flags;
96} keyidlist [] = {
97	{ "\0", 1, 0, 0, 0 },		/* default key of zero */
98	{ 0, 0, 0, 0, 0 }
99};
100
101#define	KEYFLAG_MASK	03
102
103#define	KEYFLAG_NOINIT	00
104#define	KEYFLAG_INIT	01
105#define	KEYFLAG_OK	02
106#define	KEYFLAG_BAD	03
107
108#define	KEYFLAG_SHIFT	2
109
110#define	SHIFT_VAL(a,b)	(KEYFLAG_SHIFT*((a)+((b)*2)))
111
112#define	FB64_IV		1
113#define	FB64_IV_OK	2
114#define	FB64_IV_BAD	3
115
116
117void fb64_stream_iv P((Block, struct stinfo *));
118void fb64_init P((struct fb *));
119static int fb64_start P((struct fb *, int, int));
120int fb64_is P((unsigned char *, int, struct fb *));
121int fb64_reply P((unsigned char *, int, struct fb *));
122static void fb64_session P((Session_Key *, int, struct fb *));
123void fb64_stream_key P((Block, struct stinfo *));
124int fb64_keyid P((int, unsigned char *, int *, struct fb *));
125
126	void
127cfb64_init(server)
128	int server;
129{
130	fb64_init(&fb[CFB]);
131	fb[CFB].fb_feed[4] = ENCTYPE_DES_CFB64;
132	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, CFB);
133	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, CFB);
134}
135
136	void
137ofb64_init(server)
138	int server;
139{
140	fb64_init(&fb[OFB]);
141	fb[OFB].fb_feed[4] = ENCTYPE_DES_OFB64;
142	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, OFB);
143	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, OFB);
144}
145
146	void
147fb64_init(fbp)
148	register struct fb *fbp;
149{
150	memset((void *)fbp, 0, sizeof(*fbp));
151	fbp->state[0] = fbp->state[1] = FAILED;
152	fbp->fb_feed[0] = IAC;
153	fbp->fb_feed[1] = SB;
154	fbp->fb_feed[2] = TELOPT_ENCRYPT;
155	fbp->fb_feed[3] = ENCRYPT_IS;
156}
157
158/*
159 * Returns:
160 *	-1: some error.  Negotiation is done, encryption not ready.
161 *	 0: Successful, initial negotiation all done.
162 *	 1: successful, negotiation not done yet.
163 *	 2: Not yet.  Other things (like getting the key from
164 *	    Kerberos) have to happen before we can continue.
165 */
166	int
167cfb64_start(dir, server)
168	int dir;
169	int server;
170{
171	return(fb64_start(&fb[CFB], dir, server));
172}
173	int
174ofb64_start(dir, server)
175	int dir;
176	int server;
177{
178	return(fb64_start(&fb[OFB], dir, server));
179}
180
181	static int
182fb64_start(fbp, dir, server)
183	struct fb *fbp;
184	int dir;
185	int server;
186{
187	int x;
188	unsigned char *p;
189	register int state;
190
191	switch (dir) {
192	case DIR_DECRYPT:
193		/*
194		 * This is simply a request to have the other side
195		 * start output (our input).  He will negotiate an
196		 * IV so we need not look for it.
197		 */
198		state = fbp->state[dir-1];
199		if (state == FAILED)
200			state = IN_PROGRESS;
201		break;
202
203	case DIR_ENCRYPT:
204		state = fbp->state[dir-1];
205		if (state == FAILED)
206			state = IN_PROGRESS;
207		else if ((state & NO_SEND_IV) == 0)
208			break;
209
210		if (!VALIDKEY(fbp->krbdes_key)) {
211			fbp->need_start = 1;
212			break;
213		}
214		state &= ~NO_SEND_IV;
215		state |= NO_RECV_IV;
216		if (encrypt_debug_mode)
217			printf("Creating new feed\r\n");
218		/*
219		 * Create a random feed and send it over.
220		 */
221		des_new_random_key((Block *)fbp->temp_feed);
222		des_ecb_encrypt((Block *)fbp->temp_feed, (Block *)fbp->temp_feed,
223				fbp->krbdes_sched, 1);
224		p = fbp->fb_feed + 3;
225		*p++ = ENCRYPT_IS;
226		p++;
227		*p++ = FB64_IV;
228		for (x = 0; x < sizeof(Block); ++x) {
229			if ((*p++ = fbp->temp_feed[x]) == IAC)
230				*p++ = IAC;
231		}
232		*p++ = IAC;
233		*p++ = SE;
234		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
235		net_write(fbp->fb_feed, p - fbp->fb_feed);
236		break;
237	default:
238		return(FAILED);
239	}
240	return(fbp->state[dir-1] = state);
241}
242
243/*
244 * Returns:
245 *	-1: some error.  Negotiation is done, encryption not ready.
246 *	 0: Successful, initial negotiation all done.
247 *	 1: successful, negotiation not done yet.
248 */
249	int
250cfb64_is(data, cnt)
251	unsigned char *data;
252	int cnt;
253{
254	return(fb64_is(data, cnt, &fb[CFB]));
255}
256	int
257ofb64_is(data, cnt)
258	unsigned char *data;
259	int cnt;
260{
261	return(fb64_is(data, cnt, &fb[OFB]));
262}
263
264	int
265fb64_is(data, cnt, fbp)
266	unsigned char *data;
267	int cnt;
268	struct fb *fbp;
269{
270	unsigned char *p;
271	register int state = fbp->state[DIR_DECRYPT-1];
272
273	if (cnt-- < 1)
274		goto failure;
275
276	switch (*data++) {
277	case FB64_IV:
278		if (cnt != sizeof(Block)) {
279			if (encrypt_debug_mode)
280				printf("CFB64: initial vector failed on size\r\n");
281			state = FAILED;
282			goto failure;
283		}
284
285		if (encrypt_debug_mode)
286			printf("CFB64: initial vector received\r\n");
287
288		if (encrypt_debug_mode)
289			printf("Initializing Decrypt stream\r\n");
290
291		fb64_stream_iv((void *)data, &fbp->streams[DIR_DECRYPT-1]);
292
293		p = fbp->fb_feed + 3;
294		*p++ = ENCRYPT_REPLY;
295		p++;
296		*p++ = FB64_IV_OK;
297		*p++ = IAC;
298		*p++ = SE;
299		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
300		net_write(fbp->fb_feed, p - fbp->fb_feed);
301
302		state = fbp->state[DIR_DECRYPT-1] = IN_PROGRESS;
303		break;
304
305	default:
306		if (encrypt_debug_mode) {
307			printf("Unknown option type: %d\r\n", *(data-1));
308			printd(data, cnt);
309			printf("\r\n");
310		}
311		/* FALL THROUGH */
312	failure:
313		/*
314		 * We failed.  Send an FB64_IV_BAD option
315		 * to the other side so it will know that
316		 * things failed.
317		 */
318		p = fbp->fb_feed + 3;
319		*p++ = ENCRYPT_REPLY;
320		p++;
321		*p++ = FB64_IV_BAD;
322		*p++ = IAC;
323		*p++ = SE;
324		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
325		net_write(fbp->fb_feed, p - fbp->fb_feed);
326
327		break;
328	}
329	return(fbp->state[DIR_DECRYPT-1] = state);
330}
331
332/*
333 * Returns:
334 *	-1: some error.  Negotiation is done, encryption not ready.
335 *	 0: Successful, initial negotiation all done.
336 *	 1: successful, negotiation not done yet.
337 */
338	int
339cfb64_reply(data, cnt)
340	unsigned char *data;
341	int cnt;
342{
343	return(fb64_reply(data, cnt, &fb[CFB]));
344}
345	int
346ofb64_reply(data, cnt)
347	unsigned char *data;
348	int cnt;
349{
350	return(fb64_reply(data, cnt, &fb[OFB]));
351}
352
353
354	int
355fb64_reply(data, cnt, fbp)
356	unsigned char *data;
357	int cnt;
358	struct fb *fbp;
359{
360	register int state = fbp->state[DIR_ENCRYPT-1];
361
362	if (cnt-- < 1)
363		goto failure;
364
365	switch (*data++) {
366	case FB64_IV_OK:
367		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
368		if (state == FAILED)
369			state = IN_PROGRESS;
370		state &= ~NO_RECV_IV;
371		encrypt_send_keyid(DIR_ENCRYPT, (unsigned char *)"\0", 1, 1);
372		break;
373
374	case FB64_IV_BAD:
375		memset(fbp->temp_feed, 0, sizeof(Block));
376		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
377		state = FAILED;
378		break;
379
380	default:
381		if (encrypt_debug_mode) {
382			printf("Unknown option type: %d\r\n", data[-1]);
383			printd(data, cnt);
384			printf("\r\n");
385		}
386		/* FALL THROUGH */
387	failure:
388		state = FAILED;
389		break;
390	}
391	return(fbp->state[DIR_ENCRYPT-1] = state);
392}
393
394	void
395cfb64_session(key, server)
396	Session_Key *key;
397	int server;
398{
399	fb64_session(key, server, &fb[CFB]);
400}
401
402	void
403ofb64_session(key, server)
404	Session_Key *key;
405	int server;
406{
407	fb64_session(key, server, &fb[OFB]);
408}
409
410	static void
411fb64_session(key, server, fbp)
412	Session_Key *key;
413	int server;
414	struct fb *fbp;
415{
416
417	if (!key || key->type != SK_DES) {
418		if (encrypt_debug_mode)
419			printf("Can't set krbdes's session key (%d != %d)\r\n",
420				key ? key->type : -1, SK_DES);
421		return;
422	}
423	memmove((void *)fbp->krbdes_key, (void *)key->data, sizeof(Block));
424
425	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_ENCRYPT-1]);
426	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_DECRYPT-1]);
427
428	if (fbp->once == 0) {
429		des_set_random_generator_seed((Block *)fbp->krbdes_key);
430		fbp->once = 1;
431	}
432	des_key_sched((Block *)fbp->krbdes_key, fbp->krbdes_sched);
433	/*
434	 * Now look to see if krbdes_start() was was waiting for
435	 * the key to show up.  If so, go ahead an call it now
436	 * that we have the key.
437	 */
438	if (fbp->need_start) {
439		fbp->need_start = 0;
440		fb64_start(fbp, DIR_ENCRYPT, server);
441	}
442}
443
444/*
445 * We only accept a keyid of 0.  If we get a keyid of
446 * 0, then mark the state as SUCCESS.
447 */
448	int
449cfb64_keyid(dir, kp, lenp)
450	int dir, *lenp;
451	unsigned char *kp;
452{
453	return(fb64_keyid(dir, kp, lenp, &fb[CFB]));
454}
455
456	int
457ofb64_keyid(dir, kp, lenp)
458	int dir, *lenp;
459	unsigned char *kp;
460{
461	return(fb64_keyid(dir, kp, lenp, &fb[OFB]));
462}
463
464	int
465fb64_keyid(dir, kp, lenp, fbp)
466	int dir, *lenp;
467	unsigned char *kp;
468	struct fb *fbp;
469{
470	register int state = fbp->state[dir-1];
471
472	if (*lenp != 1 || (*kp != '\0')) {
473		*lenp = 0;
474		return(state);
475	}
476
477	if (state == FAILED)
478		state = IN_PROGRESS;
479
480	state &= ~NO_KEYID;
481
482	return(fbp->state[dir-1] = state);
483}
484
485	void
486fb64_printsub(data, cnt, buf, buflen, type)
487	unsigned char *data, *buf, *type;
488	int cnt, buflen;
489{
490	char lbuf[32];
491	register int i;
492	char *cp;
493
494	buf[buflen-1] = '\0';		/* make sure it's NULL terminated */
495	buflen -= 1;
496
497	switch(data[2]) {
498	case FB64_IV:
499		sprintf(lbuf, "%s_IV", type);
500		cp = lbuf;
501		goto common;
502
503	case FB64_IV_OK:
504		sprintf(lbuf, "%s_IV_OK", type);
505		cp = lbuf;
506		goto common;
507
508	case FB64_IV_BAD:
509		sprintf(lbuf, "%s_IV_BAD", type);
510		cp = lbuf;
511		goto common;
512
513	default:
514		sprintf(lbuf, " %d (unknown)", data[2]);
515		cp = lbuf;
516	common:
517		for (; (buflen > 0) && (*buf = *cp++); buf++)
518			buflen--;
519		for (i = 3; i < cnt; i++) {
520			sprintf(lbuf, " %d", data[i]);
521			for (cp = lbuf; (buflen > 0) && (*buf = *cp++); buf++)
522				buflen--;
523		}
524		break;
525	}
526}
527
528	void
529cfb64_printsub(data, cnt, buf, buflen)
530	unsigned char *data, *buf;
531	int cnt, buflen;
532{
533	fb64_printsub(data, cnt, buf, buflen, "CFB64");
534}
535
536	void
537ofb64_printsub(data, cnt, buf, buflen)
538	unsigned char *data, *buf;
539	int cnt, buflen;
540{
541	fb64_printsub(data, cnt, buf, buflen, "OFB64");
542}
543
544	void
545fb64_stream_iv(seed, stp)
546	Block seed;
547	register struct stinfo *stp;
548{
549
550	memmove((void *)stp->str_iv, (void *)seed, sizeof(Block));
551	memmove((void *)stp->str_output, (void *)seed, sizeof(Block));
552
553	des_key_sched((Block *)stp->str_ikey, stp->str_sched);
554
555	stp->str_index = sizeof(Block);
556}
557
558	void
559fb64_stream_key(key, stp)
560	Block key;
561	register struct stinfo *stp;
562{
563	memmove((void *)stp->str_ikey, (void *)key, sizeof(Block));
564	des_key_sched((Block *)key, stp->str_sched);
565
566	memmove((void *)stp->str_output, (void *)stp->str_iv, sizeof(Block));
567
568	stp->str_index = sizeof(Block);
569}
570
571/*
572 * DES 64 bit Cipher Feedback
573 *
574 *     key --->+-----+
575 *          +->| DES |--+
576 *          |  +-----+  |
577 *	    |           v
578 *  INPUT --(--------->(+)+---> DATA
579 *          |             |
580 *	    +-------------+
581 *
582 *
583 * Given:
584 *	iV: Initial vector, 64 bits (8 bytes) long.
585 *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
586 *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
587 *
588 *	V0 = DES(iV, key)
589 *	On = Dn ^ Vn
590 *	V(n+1) = DES(On, key)
591 */
592
593	void
594cfb64_encrypt(s, c)
595	register unsigned char *s;
596	int c;
597{
598	register struct stinfo *stp = &fb[CFB].streams[DIR_ENCRYPT-1];
599	register int index;
600
601	index = stp->str_index;
602	while (c-- > 0) {
603		if (index == sizeof(Block)) {
604			Block b;
605			des_ecb_encrypt((Block *)stp->str_output, (Block *)b, stp->str_sched, 1);
606			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
607			index = 0;
608		}
609
610		/* On encryption, we store (feed ^ data) which is cypher */
611		*s = stp->str_output[index] = (stp->str_feed[index] ^ *s);
612		s++;
613		index++;
614	}
615	stp->str_index = index;
616}
617
618	int
619cfb64_decrypt(data)
620	int data;
621{
622	register struct stinfo *stp = &fb[CFB].streams[DIR_DECRYPT-1];
623	int index;
624
625	if (data == -1) {
626		/*
627		 * Back up one byte.  It is assumed that we will
628		 * never back up more than one byte.  If we do, this
629		 * may or may not work.
630		 */
631		if (stp->str_index)
632			--stp->str_index;
633		return(0);
634	}
635
636	index = stp->str_index++;
637	if (index == sizeof(Block)) {
638		Block b;
639		des_ecb_encrypt((Block *)stp->str_output, (Block *)b, stp->str_sched, 1);
640		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
641		stp->str_index = 1;	/* Next time will be 1 */
642		index = 0;		/* But now use 0 */
643	}
644
645	/* On decryption we store (data) which is cypher. */
646	stp->str_output[index] = data;
647	return(data ^ stp->str_feed[index]);
648}
649
650/*
651 * DES 64 bit Output Feedback
652 *
653 * key --->+-----+
654 *	+->| DES |--+
655 *	|  +-----+  |
656 *	+-----------+
657 *	            v
658 *  INPUT -------->(+) ----> DATA
659 *
660 * Given:
661 *	iV: Initial vector, 64 bits (8 bytes) long.
662 *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
663 *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
664 *
665 *	V0 = DES(iV, key)
666 *	V(n+1) = DES(Vn, key)
667 *	On = Dn ^ Vn
668 */
669	void
670ofb64_encrypt(s, c)
671	register unsigned char *s;
672	int c;
673{
674	register struct stinfo *stp = &fb[OFB].streams[DIR_ENCRYPT-1];
675	register int index;
676
677	index = stp->str_index;
678	while (c-- > 0) {
679		if (index == sizeof(Block)) {
680			Block b;
681			des_ecb_encrypt((Block *)stp->str_feed, (Block *)b, stp->str_sched, 1);
682			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
683			index = 0;
684		}
685		*s++ ^= stp->str_feed[index];
686		index++;
687	}
688	stp->str_index = index;
689}
690
691	int
692ofb64_decrypt(data)
693	int data;
694{
695	register struct stinfo *stp = &fb[OFB].streams[DIR_DECRYPT-1];
696	int index;
697
698	if (data == -1) {
699		/*
700		 * Back up one byte.  It is assumed that we will
701		 * never back up more than one byte.  If we do, this
702		 * may or may not work.
703		 */
704		if (stp->str_index)
705			--stp->str_index;
706		return(0);
707	}
708
709	index = stp->str_index++;
710	if (index == sizeof(Block)) {
711		Block b;
712		des_ecb_encrypt((Block *)stp->str_feed, (Block *)b, stp->str_sched, 1);
713		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
714		stp->str_index = 1;	/* Next time will be 1 */
715		index = 0;		/* But now use 0 */
716	}
717
718	return(data ^ stp->str_feed[index]);
719}
720#  endif /* DES_ENCRYPTION */
721# endif	/* AUTHENTICATION */
722#endif	/* ENCRYPTION */
723