enc_des.c revision 81965 
1/*-
2 * Copyright (c) 1991, 1993
3 *	The Regents of the University of California.  All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 *    must display the following acknowledgement:
15 *	This product includes software developed by the University of
16 *	California, Berkeley and its contributors.
17 * 4. Neither the name of the University nor the names of its contributors
18 *    may be used to endorse or promote products derived from this software
19 *    without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 * $FreeBSD: head/contrib/telnet/libtelnet/enc_des.c 81965 2001-08-20 12:28:40Z markm $
34 */
35
36#ifndef lint
37static const char sccsid[] = "@(#)enc_des.c	8.3 (Berkeley) 5/30/95";
38#endif /* not lint */
39
40#ifdef	ENCRYPTION
41# ifdef	AUTHENTICATION
42#  ifdef DES_ENCRYPTION
43#include <arpa/telnet.h>
44#include <stdio.h>
45#ifdef	__STDC__
46#include <stdlib.h>
47#endif
48
49#include <openssl/des.h>
50#include <string.h>
51#include "encrypt.h"
52#include "key-proto.h"
53#include "misc-proto.h"
54
55extern int encrypt_debug_mode;
56void des_set_random_generator_seed(des_cblock *); /* XXX */
57
58#define	CFB	0
59#define	OFB	1
60
61#define	NO_SEND_IV	1
62#define	NO_RECV_IV	2
63#define	NO_KEYID	4
64#define	IN_PROGRESS	(NO_SEND_IV|NO_RECV_IV|NO_KEYID)
65#define	SUCCESS		0
66#define	FAILED		-1
67
68
69struct fb {
70	Block krbdes_key;
71	Schedule krbdes_sched;
72	Block temp_feed;
73	unsigned char fb_feed[64];
74	int need_start;
75	int state[2];
76	int keyid[2];
77	int once;
78	struct stinfo {
79		Block		str_output;
80		Block		str_feed;
81		Block		str_iv;
82		Block		str_ikey;
83		Schedule	str_sched;
84		int		str_index;
85		int		str_flagshift;
86	} streams[2];
87};
88
89static struct fb fb[2];
90
91struct keyidlist {
92	char	*keyid;
93	int	keyidlen;
94	char	*key;
95	int	keylen;
96	int	flags;
97} keyidlist [] = {
98	{ "\0", 1, 0, 0, 0 },		/* default key of zero */
99	{ 0, 0, 0, 0, 0 }
100};
101
102#define	KEYFLAG_MASK	03
103
104#define	KEYFLAG_NOINIT	00
105#define	KEYFLAG_INIT	01
106#define	KEYFLAG_OK	02
107#define	KEYFLAG_BAD	03
108
109#define	KEYFLAG_SHIFT	2
110
111#define	SHIFT_VAL(a,b)	(KEYFLAG_SHIFT*((a)+((b)*2)))
112
113#define	FB64_IV		1
114#define	FB64_IV_OK	2
115#define	FB64_IV_BAD	3
116
117
118void fb64_stream_iv P((Block, struct stinfo *));
119void fb64_init P((struct fb *));
120static int fb64_start P((struct fb *, int, int));
121int fb64_is P((unsigned char *, int, struct fb *));
122int fb64_reply P((unsigned char *, int, struct fb *));
123static void fb64_session P((Session_Key *, int, struct fb *));
124void fb64_stream_key P((Block, struct stinfo *));
125int fb64_keyid P((int, unsigned char *, int *, struct fb *));
126
127	void
128cfb64_init(server)
129	int server;
130{
131	fb64_init(&fb[CFB]);
132	fb[CFB].fb_feed[4] = ENCTYPE_DES_CFB64;
133	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, CFB);
134	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, CFB);
135}
136
137	void
138ofb64_init(server)
139	int server;
140{
141	fb64_init(&fb[OFB]);
142	fb[OFB].fb_feed[4] = ENCTYPE_DES_OFB64;
143	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, OFB);
144	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, OFB);
145}
146
147	void
148fb64_init(fbp)
149	register struct fb *fbp;
150{
151	memset((void *)fbp, 0, sizeof(*fbp));
152	fbp->state[0] = fbp->state[1] = FAILED;
153	fbp->fb_feed[0] = IAC;
154	fbp->fb_feed[1] = SB;
155	fbp->fb_feed[2] = TELOPT_ENCRYPT;
156	fbp->fb_feed[3] = ENCRYPT_IS;
157}
158
159/*
160 * Returns:
161 *	-1: some error.  Negotiation is done, encryption not ready.
162 *	 0: Successful, initial negotiation all done.
163 *	 1: successful, negotiation not done yet.
164 *	 2: Not yet.  Other things (like getting the key from
165 *	    Kerberos) have to happen before we can continue.
166 */
167	int
168cfb64_start(dir, server)
169	int dir;
170	int server;
171{
172	return(fb64_start(&fb[CFB], dir, server));
173}
174	int
175ofb64_start(dir, server)
176	int dir;
177	int server;
178{
179	return(fb64_start(&fb[OFB], dir, server));
180}
181
182	static int
183fb64_start(fbp, dir, server)
184	struct fb *fbp;
185	int dir;
186	int server;
187{
188	int x;
189	unsigned char *p;
190	register int state;
191
192	switch (dir) {
193	case DIR_DECRYPT:
194		/*
195		 * This is simply a request to have the other side
196		 * start output (our input).  He will negotiate an
197		 * IV so we need not look for it.
198		 */
199		state = fbp->state[dir-1];
200		if (state == FAILED)
201			state = IN_PROGRESS;
202		break;
203
204	case DIR_ENCRYPT:
205		state = fbp->state[dir-1];
206		if (state == FAILED)
207			state = IN_PROGRESS;
208		else if ((state & NO_SEND_IV) == 0)
209			break;
210
211		if (!VALIDKEY(fbp->krbdes_key)) {
212			fbp->need_start = 1;
213			break;
214		}
215		state &= ~NO_SEND_IV;
216		state |= NO_RECV_IV;
217		if (encrypt_debug_mode)
218			printf("Creating new feed\r\n");
219		/*
220		 * Create a random feed and send it over.
221		 */
222		des_new_random_key((Block *)fbp->temp_feed);
223		des_ecb_encrypt((Block *)fbp->temp_feed, (Block *)fbp->temp_feed,
224				fbp->krbdes_sched, 1);
225		p = fbp->fb_feed + 3;
226		*p++ = ENCRYPT_IS;
227		p++;
228		*p++ = FB64_IV;
229		for (x = 0; x < sizeof(Block); ++x) {
230			if ((*p++ = fbp->temp_feed[x]) == IAC)
231				*p++ = IAC;
232		}
233		*p++ = IAC;
234		*p++ = SE;
235		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
236		net_write(fbp->fb_feed, p - fbp->fb_feed);
237		break;
238	default:
239		return(FAILED);
240	}
241	return(fbp->state[dir-1] = state);
242}
243
244/*
245 * Returns:
246 *	-1: some error.  Negotiation is done, encryption not ready.
247 *	 0: Successful, initial negotiation all done.
248 *	 1: successful, negotiation not done yet.
249 */
250	int
251cfb64_is(data, cnt)
252	unsigned char *data;
253	int cnt;
254{
255	return(fb64_is(data, cnt, &fb[CFB]));
256}
257	int
258ofb64_is(data, cnt)
259	unsigned char *data;
260	int cnt;
261{
262	return(fb64_is(data, cnt, &fb[OFB]));
263}
264
265	int
266fb64_is(data, cnt, fbp)
267	unsigned char *data;
268	int cnt;
269	struct fb *fbp;
270{
271	unsigned char *p;
272	register int state = fbp->state[DIR_DECRYPT-1];
273
274	if (cnt-- < 1)
275		goto failure;
276
277	switch (*data++) {
278	case FB64_IV:
279		if (cnt != sizeof(Block)) {
280			if (encrypt_debug_mode)
281				printf("CFB64: initial vector failed on size\r\n");
282			state = FAILED;
283			goto failure;
284		}
285
286		if (encrypt_debug_mode)
287			printf("CFB64: initial vector received\r\n");
288
289		if (encrypt_debug_mode)
290			printf("Initializing Decrypt stream\r\n");
291
292		fb64_stream_iv((void *)data, &fbp->streams[DIR_DECRYPT-1]);
293
294		p = fbp->fb_feed + 3;
295		*p++ = ENCRYPT_REPLY;
296		p++;
297		*p++ = FB64_IV_OK;
298		*p++ = IAC;
299		*p++ = SE;
300		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
301		net_write(fbp->fb_feed, p - fbp->fb_feed);
302
303		state = fbp->state[DIR_DECRYPT-1] = IN_PROGRESS;
304		break;
305
306	default:
307		if (encrypt_debug_mode) {
308			printf("Unknown option type: %d\r\n", *(data-1));
309			printd(data, cnt);
310			printf("\r\n");
311		}
312		/* FALL THROUGH */
313	failure:
314		/*
315		 * We failed.  Send an FB64_IV_BAD option
316		 * to the other side so it will know that
317		 * things failed.
318		 */
319		p = fbp->fb_feed + 3;
320		*p++ = ENCRYPT_REPLY;
321		p++;
322		*p++ = FB64_IV_BAD;
323		*p++ = IAC;
324		*p++ = SE;
325		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
326		net_write(fbp->fb_feed, p - fbp->fb_feed);
327
328		break;
329	}
330	return(fbp->state[DIR_DECRYPT-1] = state);
331}
332
333/*
334 * Returns:
335 *	-1: some error.  Negotiation is done, encryption not ready.
336 *	 0: Successful, initial negotiation all done.
337 *	 1: successful, negotiation not done yet.
338 */
339	int
340cfb64_reply(data, cnt)
341	unsigned char *data;
342	int cnt;
343{
344	return(fb64_reply(data, cnt, &fb[CFB]));
345}
346	int
347ofb64_reply(data, cnt)
348	unsigned char *data;
349	int cnt;
350{
351	return(fb64_reply(data, cnt, &fb[OFB]));
352}
353
354
355	int
356fb64_reply(data, cnt, fbp)
357	unsigned char *data;
358	int cnt;
359	struct fb *fbp;
360{
361	register int state = fbp->state[DIR_ENCRYPT-1];
362
363	if (cnt-- < 1)
364		goto failure;
365
366	switch (*data++) {
367	case FB64_IV_OK:
368		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
369		if (state == FAILED)
370			state = IN_PROGRESS;
371		state &= ~NO_RECV_IV;
372		encrypt_send_keyid(DIR_ENCRYPT, (unsigned char *)"\0", 1, 1);
373		break;
374
375	case FB64_IV_BAD:
376		memset(fbp->temp_feed, 0, sizeof(Block));
377		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
378		state = FAILED;
379		break;
380
381	default:
382		if (encrypt_debug_mode) {
383			printf("Unknown option type: %d\r\n", data[-1]);
384			printd(data, cnt);
385			printf("\r\n");
386		}
387		/* FALL THROUGH */
388	failure:
389		state = FAILED;
390		break;
391	}
392	return(fbp->state[DIR_ENCRYPT-1] = state);
393}
394
395	void
396cfb64_session(key, server)
397	Session_Key *key;
398	int server;
399{
400	fb64_session(key, server, &fb[CFB]);
401}
402
403	void
404ofb64_session(key, server)
405	Session_Key *key;
406	int server;
407{
408	fb64_session(key, server, &fb[OFB]);
409}
410
411	static void
412fb64_session(key, server, fbp)
413	Session_Key *key;
414	int server;
415	struct fb *fbp;
416{
417
418	if (!key || key->type != SK_DES) {
419		if (encrypt_debug_mode)
420			printf("Can't set krbdes's session key (%d != %d)\r\n",
421				key ? key->type : -1, SK_DES);
422		return;
423	}
424	memmove((void *)fbp->krbdes_key, (void *)key->data, sizeof(Block));
425
426	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_ENCRYPT-1]);
427	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_DECRYPT-1]);
428
429	if (fbp->once == 0) {
430		des_set_random_generator_seed((Block *)fbp->krbdes_key);
431		fbp->once = 1;
432	}
433	des_key_sched((Block *)fbp->krbdes_key, fbp->krbdes_sched);
434	/*
435	 * Now look to see if krbdes_start() was was waiting for
436	 * the key to show up.  If so, go ahead an call it now
437	 * that we have the key.
438	 */
439	if (fbp->need_start) {
440		fbp->need_start = 0;
441		fb64_start(fbp, DIR_ENCRYPT, server);
442	}
443}
444
445/*
446 * We only accept a keyid of 0.  If we get a keyid of
447 * 0, then mark the state as SUCCESS.
448 */
449	int
450cfb64_keyid(dir, kp, lenp)
451	int dir, *lenp;
452	unsigned char *kp;
453{
454	return(fb64_keyid(dir, kp, lenp, &fb[CFB]));
455}
456
457	int
458ofb64_keyid(dir, kp, lenp)
459	int dir, *lenp;
460	unsigned char *kp;
461{
462	return(fb64_keyid(dir, kp, lenp, &fb[OFB]));
463}
464
465	int
466fb64_keyid(dir, kp, lenp, fbp)
467	int dir, *lenp;
468	unsigned char *kp;
469	struct fb *fbp;
470{
471	register int state = fbp->state[dir-1];
472
473	if (*lenp != 1 || (*kp != '\0')) {
474		*lenp = 0;
475		return(state);
476	}
477
478	if (state == FAILED)
479		state = IN_PROGRESS;
480
481	state &= ~NO_KEYID;
482
483	return(fbp->state[dir-1] = state);
484}
485
486	void
487fb64_printsub(data, cnt, buf, buflen, type)
488	unsigned char *data, *buf, *type;
489	int cnt, buflen;
490{
491	char lbuf[32];
492	register int i;
493	char *cp;
494
495	buf[buflen-1] = '\0';		/* make sure it's NULL terminated */
496	buflen -= 1;
497
498	switch(data[2]) {
499	case FB64_IV:
500		sprintf(lbuf, "%s_IV", type);
501		cp = lbuf;
502		goto common;
503
504	case FB64_IV_OK:
505		sprintf(lbuf, "%s_IV_OK", type);
506		cp = lbuf;
507		goto common;
508
509	case FB64_IV_BAD:
510		sprintf(lbuf, "%s_IV_BAD", type);
511		cp = lbuf;
512		goto common;
513
514	default:
515		sprintf(lbuf, " %d (unknown)", data[2]);
516		cp = lbuf;
517	common:
518		for (; (buflen > 0) && (*buf = *cp++); buf++)
519			buflen--;
520		for (i = 3; i < cnt; i++) {
521			sprintf(lbuf, " %d", data[i]);
522			for (cp = lbuf; (buflen > 0) && (*buf = *cp++); buf++)
523				buflen--;
524		}
525		break;
526	}
527}
528
529	void
530cfb64_printsub(data, cnt, buf, buflen)
531	unsigned char *data, *buf;
532	int cnt, buflen;
533{
534	fb64_printsub(data, cnt, buf, buflen, "CFB64");
535}
536
537	void
538ofb64_printsub(data, cnt, buf, buflen)
539	unsigned char *data, *buf;
540	int cnt, buflen;
541{
542	fb64_printsub(data, cnt, buf, buflen, "OFB64");
543}
544
545	void
546fb64_stream_iv(seed, stp)
547	Block seed;
548	register struct stinfo *stp;
549{
550
551	memmove((void *)stp->str_iv, (void *)seed, sizeof(Block));
552	memmove((void *)stp->str_output, (void *)seed, sizeof(Block));
553
554	des_key_sched((Block *)stp->str_ikey, stp->str_sched);
555
556	stp->str_index = sizeof(Block);
557}
558
559	void
560fb64_stream_key(key, stp)
561	Block key;
562	register struct stinfo *stp;
563{
564	memmove((void *)stp->str_ikey, (void *)key, sizeof(Block));
565	des_key_sched((Block *)key, stp->str_sched);
566
567	memmove((void *)stp->str_output, (void *)stp->str_iv, sizeof(Block));
568
569	stp->str_index = sizeof(Block);
570}
571
572/*
573 * DES 64 bit Cipher Feedback
574 *
575 *     key --->+-----+
576 *          +->| DES |--+
577 *          |  +-----+  |
578 *	    |           v
579 *  INPUT --(--------->(+)+---> DATA
580 *          |             |
581 *	    +-------------+
582 *
583 *
584 * Given:
585 *	iV: Initial vector, 64 bits (8 bytes) long.
586 *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
587 *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
588 *
589 *	V0 = DES(iV, key)
590 *	On = Dn ^ Vn
591 *	V(n+1) = DES(On, key)
592 */
593
594	void
595cfb64_encrypt(s, c)
596	register unsigned char *s;
597	int c;
598{
599	register struct stinfo *stp = &fb[CFB].streams[DIR_ENCRYPT-1];
600	register int index;
601
602	index = stp->str_index;
603	while (c-- > 0) {
604		if (index == sizeof(Block)) {
605			Block b;
606			des_ecb_encrypt((Block *)stp->str_output, (Block *)b, stp->str_sched, 1);
607			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
608			index = 0;
609		}
610
611		/* On encryption, we store (feed ^ data) which is cypher */
612		*s = stp->str_output[index] = (stp->str_feed[index] ^ *s);
613		s++;
614		index++;
615	}
616	stp->str_index = index;
617}
618
619	int
620cfb64_decrypt(data)
621	int data;
622{
623	register struct stinfo *stp = &fb[CFB].streams[DIR_DECRYPT-1];
624	int index;
625
626	if (data == -1) {
627		/*
628		 * Back up one byte.  It is assumed that we will
629		 * never back up more than one byte.  If we do, this
630		 * may or may not work.
631		 */
632		if (stp->str_index)
633			--stp->str_index;
634		return(0);
635	}
636
637	index = stp->str_index++;
638	if (index == sizeof(Block)) {
639		Block b;
640		des_ecb_encrypt((Block *)stp->str_output, (Block *)b, stp->str_sched, 1);
641		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
642		stp->str_index = 1;	/* Next time will be 1 */
643		index = 0;		/* But now use 0 */
644	}
645
646	/* On decryption we store (data) which is cypher. */
647	stp->str_output[index] = data;
648	return(data ^ stp->str_feed[index]);
649}
650
651/*
652 * DES 64 bit Output Feedback
653 *
654 * key --->+-----+
655 *	+->| DES |--+
656 *	|  +-----+  |
657 *	+-----------+
658 *	            v
659 *  INPUT -------->(+) ----> DATA
660 *
661 * Given:
662 *	iV: Initial vector, 64 bits (8 bytes) long.
663 *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
664 *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
665 *
666 *	V0 = DES(iV, key)
667 *	V(n+1) = DES(Vn, key)
668 *	On = Dn ^ Vn
669 */
670	void
671ofb64_encrypt(s, c)
672	register unsigned char *s;
673	int c;
674{
675	register struct stinfo *stp = &fb[OFB].streams[DIR_ENCRYPT-1];
676	register int index;
677
678	index = stp->str_index;
679	while (c-- > 0) {
680		if (index == sizeof(Block)) {
681			Block b;
682			des_ecb_encrypt((Block *)stp->str_feed, (Block *)b, stp->str_sched, 1);
683			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
684			index = 0;
685		}
686		*s++ ^= stp->str_feed[index];
687		index++;
688	}
689	stp->str_index = index;
690}
691
692	int
693ofb64_decrypt(data)
694	int data;
695{
696	register struct stinfo *stp = &fb[OFB].streams[DIR_DECRYPT-1];
697	int index;
698
699	if (data == -1) {
700		/*
701		 * Back up one byte.  It is assumed that we will
702		 * never back up more than one byte.  If we do, this
703		 * may or may not work.
704		 */
705		if (stp->str_index)
706			--stp->str_index;
707		return(0);
708	}
709
710	index = stp->str_index++;
711	if (index == sizeof(Block)) {
712		Block b;
713		des_ecb_encrypt((Block *)stp->str_feed, (Block *)b, stp->str_sched, 1);
714		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
715		stp->str_index = 1;	/* Next time will be 1 */
716		index = 0;		/* But now use 0 */
717	}
718
719	return(data ^ stp->str_feed[index]);
720}
721#  endif /* DES_ENCRYPTION */
722# endif	/* AUTHENTICATION */
723#endif	/* ENCRYPTION */
724