1/*	$NetBSD: opensslrsa_link.c,v 1.5 2012/12/04 23:38:42 spz Exp $	*/
2
3/*
4 * Copyright (C) 2004-2009, 2011, 2012  Internet Systems Consortium, Inc. ("ISC")
5 * Copyright (C) 2000-2003  Internet Software Consortium.
6 *
7 * Permission to use, copy, modify, and/or distribute this software for any
8 * purpose with or without fee is hereby granted, provided that the above
9 * copyright notice and this permission notice appear in all copies.
10 *
11 * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
12 * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
13 * AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
14 * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
15 * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
16 * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
17 * PERFORMANCE OF THIS SOFTWARE.
18 */
19
20/*
21 * Principal Author: Brian Wellington
22 * Id
23 */
24#ifdef OPENSSL
25#include <config.h>
26
27#ifndef USE_EVP
28#if !defined(HAVE_EVP_SHA256) || !defined(HAVE_EVP_SHA512)
29#define USE_EVP 0
30#else
31#define USE_EVP 1
32#endif
33#endif
34
35
36#include <isc/entropy.h>
37#include <isc/md5.h>
38#include <isc/sha1.h>
39#include <isc/sha2.h>
40#include <isc/mem.h>
41#include <isc/string.h>
42#include <isc/util.h>
43
44#include <dst/result.h>
45
46#include "dst_internal.h"
47#include "dst_openssl.h"
48#include "dst_parse.h"
49
50#include <openssl/err.h>
51#include <openssl/objects.h>
52#include <openssl/rsa.h>
53#if OPENSSL_VERSION_NUMBER > 0x00908000L
54#include <openssl/bn.h>
55#endif
56#ifdef USE_ENGINE
57#include <openssl/engine.h>
58#endif
59
60/*
61 * Limit the size of public exponents.
62 */
63#ifndef RSA_MAX_PUBEXP_BITS
64#define RSA_MAX_PUBEXP_BITS    35
65#endif
66
67/*
68 * We don't use configure for windows so enforce the OpenSSL version
69 * here.  Unlike with configure we don't support overriding this test.
70 */
71#ifdef WIN32
72#if !((OPENSSL_VERSION_NUMBER >= 0x009070cfL && \
73       OPENSSL_VERSION_NUMBER < 0x00908000L) || \
74      OPENSSL_VERSION_NUMBER >= 0x0090804fL)
75#error Please upgrade OpenSSL to 0.9.8d/0.9.7l or greater.
76#endif
77#endif
78
79
80	/*
81	 * XXXMPA  Temporarily disable RSA_BLINDING as it requires
82	 * good quality random data that cannot currently be guaranteed.
83	 * XXXMPA  Find which versions of openssl use pseudo random data
84	 * and set RSA_FLAG_BLINDING for those.
85	 */
86
87#if 0
88#if OPENSSL_VERSION_NUMBER < 0x0090601fL
89#define SET_FLAGS(rsa) \
90	do { \
91	(rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
92	(rsa)->flags |= RSA_FLAG_BLINDING; \
93	} while (/*CONSTCOND*/0)
94#else
95#define SET_FLAGS(rsa) \
96	do { \
97		(rsa)->flags |= RSA_FLAG_BLINDING; \
98	} while (/*CONSTCOND*/0)
99#endif
100#endif
101
102#if OPENSSL_VERSION_NUMBER < 0x0090601fL
103#define SET_FLAGS(rsa) \
104	do { \
105	(rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
106	(rsa)->flags &= ~RSA_FLAG_BLINDING; \
107	} while (/*CONSTCOND*/0)
108#elif defined(RSA_FLAG_NO_BLINDING)
109#define SET_FLAGS(rsa) \
110	do { \
111		(rsa)->flags &= ~RSA_FLAG_BLINDING; \
112		(rsa)->flags |= RSA_FLAG_NO_BLINDING; \
113	} while (/*CONSTCOND*/0)
114#else
115#define SET_FLAGS(rsa) \
116	do { \
117		(rsa)->flags &= ~RSA_FLAG_BLINDING; \
118	} while (/*CONSTCOND*/0)
119#endif
120
121#define DST_RET(a) {ret = a; goto err;}
122
123static isc_result_t opensslrsa_todns(const dst_key_t *key, isc_buffer_t *data);
124
125static isc_result_t
126opensslrsa_createctx(dst_key_t *key, dst_context_t *dctx) {
127#if USE_EVP
128	EVP_MD_CTX *evp_md_ctx;
129	const EVP_MD *type = NULL;
130#endif
131
132	UNUSED(key);
133	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
134		dctx->key->key_alg == DST_ALG_RSASHA1 ||
135		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
136		dctx->key->key_alg == DST_ALG_RSASHA256 ||
137		dctx->key->key_alg == DST_ALG_RSASHA512);
138
139#if USE_EVP
140	evp_md_ctx = EVP_MD_CTX_create();
141	if (evp_md_ctx == NULL)
142		return (ISC_R_NOMEMORY);
143
144	switch (dctx->key->key_alg) {
145	case DST_ALG_RSAMD5:
146		type = EVP_md5();	/* MD5 + RSA */
147		break;
148	case DST_ALG_RSASHA1:
149	case DST_ALG_NSEC3RSASHA1:
150		type = EVP_sha1();	/* SHA1 + RSA */
151		break;
152#ifdef HAVE_EVP_SHA256
153	case DST_ALG_RSASHA256:
154		type = EVP_sha256();	/* SHA256 + RSA */
155		break;
156#endif
157#ifdef HAVE_EVP_SHA512
158	case DST_ALG_RSASHA512:
159		type = EVP_sha512();
160		break;
161#endif
162	default:
163		INSIST(0);
164	}
165
166	if (!EVP_DigestInit_ex(evp_md_ctx, type, NULL)) {
167		EVP_MD_CTX_destroy(evp_md_ctx);
168		return (dst__openssl_toresult2("EVP_DigestInit_ex",
169					       ISC_R_FAILURE));
170	}
171	dctx->ctxdata.evp_md_ctx = evp_md_ctx;
172#else
173	switch (dctx->key->key_alg) {
174	case DST_ALG_RSAMD5:
175		{
176			isc_md5_t *md5ctx;
177
178			md5ctx = isc_mem_get(dctx->mctx, sizeof(isc_md5_t));
179			if (md5ctx == NULL)
180				return (ISC_R_NOMEMORY);
181			isc_md5_init(md5ctx);
182			dctx->ctxdata.md5ctx = md5ctx;
183		}
184		break;
185	case DST_ALG_RSASHA1:
186	case DST_ALG_NSEC3RSASHA1:
187		{
188			isc_sha1_t *sha1ctx;
189
190			sha1ctx = isc_mem_get(dctx->mctx, sizeof(isc_sha1_t));
191			if (sha1ctx == NULL)
192				return (ISC_R_NOMEMORY);
193			isc_sha1_init(sha1ctx);
194			dctx->ctxdata.sha1ctx = sha1ctx;
195		}
196		break;
197	case DST_ALG_RSASHA256:
198		{
199			isc_sha256_t *sha256ctx;
200
201			sha256ctx = isc_mem_get(dctx->mctx,
202						sizeof(isc_sha256_t));
203			if (sha256ctx == NULL)
204				return (ISC_R_NOMEMORY);
205			isc_sha256_init(sha256ctx);
206			dctx->ctxdata.sha256ctx = sha256ctx;
207		}
208		break;
209	case DST_ALG_RSASHA512:
210		{
211			isc_sha512_t *sha512ctx;
212
213			sha512ctx = isc_mem_get(dctx->mctx,
214						sizeof(isc_sha512_t));
215			if (sha512ctx == NULL)
216				return (ISC_R_NOMEMORY);
217			isc_sha512_init(sha512ctx);
218			dctx->ctxdata.sha512ctx = sha512ctx;
219		}
220		break;
221	default:
222		INSIST(0);
223	}
224#endif
225
226	return (ISC_R_SUCCESS);
227}
228
229static void
230opensslrsa_destroyctx(dst_context_t *dctx) {
231#if USE_EVP
232	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
233#endif
234
235	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
236		dctx->key->key_alg == DST_ALG_RSASHA1 ||
237		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
238		dctx->key->key_alg == DST_ALG_RSASHA256 ||
239		dctx->key->key_alg == DST_ALG_RSASHA512);
240
241#if USE_EVP
242	if (evp_md_ctx != NULL) {
243		EVP_MD_CTX_destroy(evp_md_ctx);
244		dctx->ctxdata.evp_md_ctx = NULL;
245	}
246#else
247	switch (dctx->key->key_alg) {
248	case DST_ALG_RSAMD5:
249		{
250			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
251
252			if (md5ctx != NULL) {
253				isc_md5_invalidate(md5ctx);
254				isc_mem_put(dctx->mctx, md5ctx,
255					    sizeof(isc_md5_t));
256				dctx->ctxdata.md5ctx = NULL;
257			}
258		}
259		break;
260	case DST_ALG_RSASHA1:
261	case DST_ALG_NSEC3RSASHA1:
262		{
263			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
264
265			if (sha1ctx != NULL) {
266				isc_sha1_invalidate(sha1ctx);
267				isc_mem_put(dctx->mctx, sha1ctx,
268					    sizeof(isc_sha1_t));
269				dctx->ctxdata.sha1ctx = NULL;
270			}
271		}
272		break;
273	case DST_ALG_RSASHA256:
274		{
275			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
276
277			if (sha256ctx != NULL) {
278				isc_sha256_invalidate(sha256ctx);
279				isc_mem_put(dctx->mctx, sha256ctx,
280					    sizeof(isc_sha256_t));
281				dctx->ctxdata.sha256ctx = NULL;
282			}
283		}
284		break;
285	case DST_ALG_RSASHA512:
286		{
287			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
288
289			if (sha512ctx != NULL) {
290				isc_sha512_invalidate(sha512ctx);
291				isc_mem_put(dctx->mctx, sha512ctx,
292					    sizeof(isc_sha512_t));
293				dctx->ctxdata.sha512ctx = NULL;
294			}
295		}
296		break;
297	default:
298		INSIST(0);
299	}
300#endif
301}
302
303static isc_result_t
304opensslrsa_adddata(dst_context_t *dctx, const isc_region_t *data) {
305#if USE_EVP
306	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
307#endif
308
309	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
310		dctx->key->key_alg == DST_ALG_RSASHA1 ||
311		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
312		dctx->key->key_alg == DST_ALG_RSASHA256 ||
313		dctx->key->key_alg == DST_ALG_RSASHA512);
314
315#if USE_EVP
316	if (!EVP_DigestUpdate(evp_md_ctx, data->base, data->length)) {
317		return (dst__openssl_toresult2("EVP_DigestUpdate",
318					       ISC_R_FAILURE));
319	}
320#else
321	switch (dctx->key->key_alg) {
322	case DST_ALG_RSAMD5:
323		{
324			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
325
326			isc_md5_update(md5ctx, data->base, data->length);
327		}
328		break;
329	case DST_ALG_RSASHA1:
330	case DST_ALG_NSEC3RSASHA1:
331		{
332			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
333
334			isc_sha1_update(sha1ctx, data->base, data->length);
335		}
336		break;
337	case DST_ALG_RSASHA256:
338		{
339			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
340
341			isc_sha256_update(sha256ctx, data->base, data->length);
342		}
343		break;
344	case DST_ALG_RSASHA512:
345		{
346			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
347
348			isc_sha512_update(sha512ctx, data->base, data->length);
349		}
350		break;
351	default:
352		INSIST(0);
353	}
354#endif
355	return (ISC_R_SUCCESS);
356}
357
358#if ! USE_EVP && OPENSSL_VERSION_NUMBER < 0x00908000L
359/*
360 * Digest prefixes from RFC 5702.
361 */
362static unsigned char sha256_prefix[] =
363	 { 0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
364	   0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20};
365static unsigned char sha512_prefix[] =
366	 { 0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
367	   0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40};
368#define PREFIXLEN sizeof(sha512_prefix)
369#else
370#define PREFIXLEN 0
371#endif
372
373static isc_result_t
374opensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
375	dst_key_t *key = dctx->key;
376	isc_region_t r;
377	unsigned int siglen = 0;
378#if USE_EVP
379	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
380	EVP_PKEY *pkey = key->keydata.pkey;
381#else
382	RSA *rsa = key->keydata.rsa;
383	/* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
384	unsigned char digest[PREFIXLEN + ISC_SHA512_DIGESTLENGTH];
385	int status;
386	int type = 0;
387	unsigned int digestlen = 0;
388#if OPENSSL_VERSION_NUMBER < 0x00908000L
389	unsigned int prefixlen = 0;
390	const unsigned char *prefix = NULL;
391#endif
392#endif
393
394	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
395		dctx->key->key_alg == DST_ALG_RSASHA1 ||
396		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
397		dctx->key->key_alg == DST_ALG_RSASHA256 ||
398		dctx->key->key_alg == DST_ALG_RSASHA512);
399
400	isc_buffer_availableregion(sig, &r);
401
402#if USE_EVP
403	if (r.length < (unsigned int) EVP_PKEY_size(pkey))
404		return (ISC_R_NOSPACE);
405
406	if (!EVP_SignFinal(evp_md_ctx, r.base, &siglen, pkey)) {
407		return (dst__openssl_toresult2("EVP_SignFinal",
408					       ISC_R_FAILURE));
409	}
410#else
411	if (r.length < (unsigned int) RSA_size(rsa))
412		return (ISC_R_NOSPACE);
413
414	switch (dctx->key->key_alg) {
415	case DST_ALG_RSAMD5:
416		{
417			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
418
419			isc_md5_final(md5ctx, digest);
420			type = NID_md5;
421			digestlen = ISC_MD5_DIGESTLENGTH;
422		}
423		break;
424	case DST_ALG_RSASHA1:
425	case DST_ALG_NSEC3RSASHA1:
426		{
427			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
428
429			isc_sha1_final(sha1ctx, digest);
430			type = NID_sha1;
431			digestlen = ISC_SHA1_DIGESTLENGTH;
432		}
433		break;
434	case DST_ALG_RSASHA256:
435		{
436			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
437
438			isc_sha256_final(digest, sha256ctx);
439			digestlen = ISC_SHA256_DIGESTLENGTH;
440#if OPENSSL_VERSION_NUMBER < 0x00908000L
441			prefix = sha256_prefix;
442			prefixlen = sizeof(sha256_prefix);
443#else
444			type = NID_sha256;
445#endif
446		}
447		break;
448	case DST_ALG_RSASHA512:
449		{
450			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
451
452			isc_sha512_final(digest, sha512ctx);
453			digestlen = ISC_SHA512_DIGESTLENGTH;
454#if OPENSSL_VERSION_NUMBER < 0x00908000L
455			prefix = sha512_prefix;
456			prefixlen = sizeof(sha512_prefix);
457#else
458			type = NID_sha512;
459#endif
460		}
461		break;
462	default:
463		INSIST(0);
464	}
465
466#if OPENSSL_VERSION_NUMBER < 0x00908000L
467	switch (dctx->key->key_alg) {
468	case DST_ALG_RSAMD5:
469	case DST_ALG_RSASHA1:
470	case DST_ALG_NSEC3RSASHA1:
471		INSIST(type != 0);
472		status = RSA_sign(type, digest, digestlen, r.base,
473				  &siglen, rsa);
474		break;
475
476	case DST_ALG_RSASHA256:
477	case DST_ALG_RSASHA512:
478		INSIST(prefix != NULL);
479		INSIST(prefixlen != 0);
480		INSIST(prefixlen + digestlen <= sizeof(digest));
481
482		memmove(digest + prefixlen, digest, digestlen);
483		memcpy(digest, prefix, prefixlen);
484		status = RSA_private_encrypt(digestlen + prefixlen,
485					     digest, r.base, rsa,
486					     RSA_PKCS1_PADDING);
487		if (status < 0)
488			status = 0;
489		else
490			siglen = status;
491		break;
492
493	default:
494		INSIST(0);
495	}
496#else
497	INSIST(type != 0);
498	status = RSA_sign(type, digest, digestlen, r.base, &siglen, rsa);
499#endif
500	if (status == 0)
501		return (dst__openssl_toresult2("RSA_sign",
502					       DST_R_OPENSSLFAILURE));
503#endif
504
505	isc_buffer_add(sig, siglen);
506
507	return (ISC_R_SUCCESS);
508}
509
510static isc_result_t
511opensslrsa_verify2(dst_context_t *dctx, int maxbits, const isc_region_t *sig) {
512	dst_key_t *key = dctx->key;
513	int status = 0;
514#if USE_EVP
515	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
516	EVP_PKEY *pkey = key->keydata.pkey;
517	RSA *rsa;
518	int bits;
519#else
520	/* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
521	unsigned char digest[ISC_SHA512_DIGESTLENGTH];
522	int type = 0;
523	unsigned int digestlen = 0;
524	RSA *rsa = key->keydata.rsa;
525#if OPENSSL_VERSION_NUMBER < 0x00908000L
526	unsigned int prefixlen = 0;
527	const unsigned char *prefix = NULL;
528#endif
529#endif
530
531	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
532		dctx->key->key_alg == DST_ALG_RSASHA1 ||
533		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
534		dctx->key->key_alg == DST_ALG_RSASHA256 ||
535		dctx->key->key_alg == DST_ALG_RSASHA512);
536
537#if USE_EVP
538	rsa = EVP_PKEY_get1_RSA(pkey);
539	if (rsa == NULL)
540		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
541	bits = BN_num_bits(rsa->e);
542	RSA_free(rsa);
543	if (bits > maxbits && maxbits != 0)
544		return (DST_R_VERIFYFAILURE);
545
546	status = EVP_VerifyFinal(evp_md_ctx, sig->base, sig->length, pkey);
547#else
548	if (BN_num_bits(rsa->e) > maxbits && maxbits != 0)
549		return (DST_R_VERIFYFAILURE);
550
551	switch (dctx->key->key_alg) {
552	case DST_ALG_RSAMD5:
553		{
554			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
555
556			isc_md5_final(md5ctx, digest);
557			type = NID_md5;
558			digestlen = ISC_MD5_DIGESTLENGTH;
559		}
560		break;
561	case DST_ALG_RSASHA1:
562	case DST_ALG_NSEC3RSASHA1:
563		{
564			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
565
566			isc_sha1_final(sha1ctx, digest);
567			type = NID_sha1;
568			digestlen = ISC_SHA1_DIGESTLENGTH;
569		}
570		break;
571	case DST_ALG_RSASHA256:
572		{
573			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
574
575			isc_sha256_final(digest, sha256ctx);
576			digestlen = ISC_SHA256_DIGESTLENGTH;
577#if OPENSSL_VERSION_NUMBER < 0x00908000L
578			prefix = sha256_prefix;
579			prefixlen = sizeof(sha256_prefix);
580#else
581			type = NID_sha256;
582#endif
583		}
584		break;
585	case DST_ALG_RSASHA512:
586		{
587			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
588
589			isc_sha512_final(digest, sha512ctx);
590			digestlen = ISC_SHA512_DIGESTLENGTH;
591#if OPENSSL_VERSION_NUMBER < 0x00908000L
592			prefix = sha512_prefix;
593			prefixlen = sizeof(sha512_prefix);
594#else
595			type = NID_sha512;
596#endif
597		}
598		break;
599	default:
600		INSIST(0);
601	}
602
603	if (sig->length != (unsigned int) RSA_size(rsa))
604		return (DST_R_VERIFYFAILURE);
605
606#if OPENSSL_VERSION_NUMBER < 0x00908000L
607	switch (dctx->key->key_alg) {
608	case DST_ALG_RSAMD5:
609	case DST_ALG_RSASHA1:
610	case DST_ALG_NSEC3RSASHA1:
611		INSIST(type != 0);
612		status = RSA_verify(type, digest, digestlen, sig->base,
613				    RSA_size(rsa), rsa);
614		break;
615
616	case DST_ALG_RSASHA256:
617	case DST_ALG_RSASHA512:
618		{
619			/*
620			 * 1024 is big enough for all valid RSA bit sizes
621			 * for use with DNSSEC.
622			 */
623			unsigned char original[PREFIXLEN + 1024];
624
625			INSIST(prefix != NULL);
626			INSIST(prefixlen != 0U);
627
628			if (RSA_size(rsa) > (int)sizeof(original))
629				return (DST_R_VERIFYFAILURE);
630
631			status = RSA_public_decrypt(sig->length, sig->base,
632						    original, rsa,
633						    RSA_PKCS1_PADDING);
634			if (status <= 0)
635				return (dst__openssl_toresult2(
636						"RSA_public_decrypt",
637						DST_R_VERIFYFAILURE));
638			if (status != (int)(prefixlen + digestlen))
639				return (DST_R_VERIFYFAILURE);
640			if (memcmp(original, prefix, prefixlen))
641				return (DST_R_VERIFYFAILURE);
642			if (memcmp(original + prefixlen, digest, digestlen))
643				return (DST_R_VERIFYFAILURE);
644			status = 1;
645		}
646		break;
647
648	default:
649		INSIST(0);
650	}
651#else
652	INSIST(type != 0);
653	status = RSA_verify(type, digest, digestlen, sig->base,
654			     RSA_size(rsa), rsa);
655#endif
656#endif
657	if (status != 1)
658		return (dst__openssl_toresult2("RSA_verify",
659					       DST_R_VERIFYFAILURE));
660
661	return (ISC_R_SUCCESS);
662}
663
664static isc_result_t
665opensslrsa_verify(dst_context_t *dctx, const isc_region_t *sig) {
666	return (opensslrsa_verify2(dctx, 0, sig));
667}
668
669static isc_boolean_t
670opensslrsa_compare(const dst_key_t *key1, const dst_key_t *key2) {
671	int status;
672	RSA *rsa1 = NULL, *rsa2 = NULL;
673#if USE_EVP
674	EVP_PKEY *pkey1, *pkey2;
675#endif
676
677#if USE_EVP
678	pkey1 = key1->keydata.pkey;
679	pkey2 = key2->keydata.pkey;
680	/*
681	 * The pkey reference will keep these around after
682	 * the RSA_free() call.
683	 */
684	if (pkey1 != NULL) {
685		rsa1 = EVP_PKEY_get1_RSA(pkey1);
686		RSA_free(rsa1);
687	}
688	if (pkey2 != NULL) {
689		rsa2 = EVP_PKEY_get1_RSA(pkey2);
690		RSA_free(rsa2);
691	}
692#else
693	rsa1 = key1->keydata.rsa;
694	rsa2 = key2->keydata.rsa;
695#endif
696
697	if (rsa1 == NULL && rsa2 == NULL)
698		return (ISC_TRUE);
699	else if (rsa1 == NULL || rsa2 == NULL)
700		return (ISC_FALSE);
701
702	status = BN_cmp(rsa1->n, rsa2->n) ||
703		 BN_cmp(rsa1->e, rsa2->e);
704
705	if (status != 0)
706		return (ISC_FALSE);
707
708#if USE_EVP
709	if ((rsa1->flags & RSA_FLAG_EXT_PKEY) != 0 ||
710	    (rsa2->flags & RSA_FLAG_EXT_PKEY) != 0) {
711		if ((rsa1->flags & RSA_FLAG_EXT_PKEY) == 0 ||
712		    (rsa2->flags & RSA_FLAG_EXT_PKEY) == 0)
713			return (ISC_FALSE);
714		/*
715		 * Can't compare private parameters, BTW does it make sense?
716		 */
717		return (ISC_TRUE);
718	}
719#endif
720
721	if (rsa1->d != NULL || rsa2->d != NULL) {
722		if (rsa1->d == NULL || rsa2->d == NULL)
723			return (ISC_FALSE);
724		status = BN_cmp(rsa1->d, rsa2->d) ||
725			 BN_cmp(rsa1->p, rsa2->p) ||
726			 BN_cmp(rsa1->q, rsa2->q);
727
728		if (status != 0)
729			return (ISC_FALSE);
730	}
731	return (ISC_TRUE);
732}
733
734#if OPENSSL_VERSION_NUMBER > 0x00908000L
735static int
736progress_cb(int p, int n, BN_GENCB *cb)
737{
738	union {
739		void *dptr;
740		void (*fptr)(int);
741	} u;
742
743	UNUSED(n);
744
745	u.dptr = cb->arg;
746	if (u.fptr != NULL)
747		u.fptr(p);
748	return (1);
749}
750#endif
751
752static isc_result_t
753opensslrsa_generate(dst_key_t *key, int exp, void (*callback)(int)) {
754#if OPENSSL_VERSION_NUMBER > 0x00908000L
755	isc_result_t ret = DST_R_OPENSSLFAILURE;
756	BN_GENCB cb;
757	union {
758		void *dptr;
759		void (*fptr)(int);
760	} u;
761	RSA *rsa = RSA_new();
762	BIGNUM *e = BN_new();
763#if USE_EVP
764	EVP_PKEY *pkey = EVP_PKEY_new();
765#endif
766
767	if (rsa == NULL || e == NULL)
768		goto err;
769#if USE_EVP
770	if (pkey == NULL)
771		goto err;
772	if (!EVP_PKEY_set1_RSA(pkey, rsa))
773		goto err;
774#endif
775
776	if (exp == 0) {
777		/* RSA_F4 0x10001 */
778		BN_set_bit(e, 0);
779		BN_set_bit(e, 16);
780	} else {
781		/* (phased-out) F5 0x100000001 */
782		BN_set_bit(e, 0);
783		BN_set_bit(e, 32);
784	}
785
786	if (callback == NULL) {
787		BN_GENCB_set_old(&cb, NULL, NULL);
788	} else {
789		u.fptr = callback;
790		BN_GENCB_set(&cb, &progress_cb, u.dptr);
791	}
792
793	if (RSA_generate_key_ex(rsa, key->key_size, e, &cb)) {
794		BN_free(e);
795		SET_FLAGS(rsa);
796#if USE_EVP
797		key->keydata.pkey = pkey;
798
799		RSA_free(rsa);
800#else
801		key->keydata.rsa = rsa;
802#endif
803		return (ISC_R_SUCCESS);
804	}
805	ret = dst__openssl_toresult2("RSA_generate_key_ex",
806				     DST_R_OPENSSLFAILURE);
807
808err:
809#if USE_EVP
810	if (pkey != NULL)
811		EVP_PKEY_free(pkey);
812#endif
813	if (e != NULL)
814		BN_free(e);
815	if (rsa != NULL)
816		RSA_free(rsa);
817	return (dst__openssl_toresult(ret));
818#else
819	RSA *rsa;
820	unsigned long e;
821#if USE_EVP
822	EVP_PKEY *pkey = EVP_PKEY_new();
823
824	UNUSED(callback);
825
826	if (pkey == NULL)
827		return (ISC_R_NOMEMORY);
828#else
829	UNUSED(callback);
830#endif
831
832	if (exp == 0)
833	       e = RSA_F4;
834	else
835	       e = 0x40000003;
836	rsa = RSA_generate_key(key->key_size, e, NULL, NULL);
837	if (rsa == NULL) {
838#if USE_EVP
839		EVP_PKEY_free(pkey);
840#endif
841		return (dst__openssl_toresult2("RSA_generate_key",
842					       DST_R_OPENSSLFAILURE));
843	}
844	SET_FLAGS(rsa);
845#if USE_EVP
846	if (!EVP_PKEY_set1_RSA(pkey, rsa)) {
847		EVP_PKEY_free(pkey);
848		RSA_free(rsa);
849		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
850	}
851	key->keydata.pkey = pkey;
852	RSA_free(rsa);
853#else
854	key->keydata.rsa = rsa;
855#endif
856
857	return (ISC_R_SUCCESS);
858#endif
859}
860
861static isc_boolean_t
862opensslrsa_isprivate(const dst_key_t *key) {
863#if USE_EVP
864	RSA *rsa = EVP_PKEY_get1_RSA(key->keydata.pkey);
865	INSIST(rsa != NULL);
866	RSA_free(rsa);
867	/* key->keydata.pkey still has a reference so rsa is still valid. */
868#else
869	RSA *rsa = key->keydata.rsa;
870#endif
871	if (rsa != NULL && (rsa->flags & RSA_FLAG_EXT_PKEY) != 0)
872		return (ISC_TRUE);
873	return (ISC_TF(rsa != NULL && rsa->d != NULL));
874}
875
876static void
877opensslrsa_destroy(dst_key_t *key) {
878#if USE_EVP
879	EVP_PKEY *pkey = key->keydata.pkey;
880	EVP_PKEY_free(pkey);
881	key->keydata.pkey = NULL;
882#else
883	RSA *rsa = key->keydata.rsa;
884	RSA_free(rsa);
885	key->keydata.rsa = NULL;
886#endif
887}
888
889
890static isc_result_t
891opensslrsa_todns(const dst_key_t *key, isc_buffer_t *data) {
892	isc_region_t r;
893	unsigned int e_bytes;
894	unsigned int mod_bytes;
895	isc_result_t ret;
896	RSA *rsa;
897#if USE_EVP
898	EVP_PKEY *pkey;
899#endif
900
901#if USE_EVP
902	REQUIRE(key->keydata.pkey != NULL);
903#else
904	REQUIRE(key->keydata.rsa != NULL);
905#endif
906
907#if USE_EVP
908	pkey = key->keydata.pkey;
909	rsa = EVP_PKEY_get1_RSA(pkey);
910	if (rsa == NULL)
911		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
912#else
913	rsa = key->keydata.rsa;
914#endif
915
916	isc_buffer_availableregion(data, &r);
917
918	e_bytes = BN_num_bytes(rsa->e);
919	mod_bytes = BN_num_bytes(rsa->n);
920
921	if (e_bytes < 256) {	/*%< key exponent is <= 2040 bits */
922		if (r.length < 1)
923			DST_RET(ISC_R_NOSPACE);
924		isc_buffer_putuint8(data, (isc_uint8_t) e_bytes);
925		isc_region_consume(&r, 1);
926	} else {
927		if (r.length < 3)
928			DST_RET(ISC_R_NOSPACE);
929		isc_buffer_putuint8(data, 0);
930		isc_buffer_putuint16(data, (isc_uint16_t) e_bytes);
931		isc_region_consume(&r, 3);
932	}
933
934	if (r.length < e_bytes + mod_bytes)
935		DST_RET(ISC_R_NOSPACE);
936
937	BN_bn2bin(rsa->e, r.base);
938	isc_region_consume(&r, e_bytes);
939	BN_bn2bin(rsa->n, r.base);
940
941	isc_buffer_add(data, e_bytes + mod_bytes);
942
943	ret = ISC_R_SUCCESS;
944 err:
945#if USE_EVP
946	if (rsa != NULL)
947		RSA_free(rsa);
948#endif
949	return (ret);
950}
951
952static isc_result_t
953opensslrsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
954	RSA *rsa;
955	isc_region_t r;
956	unsigned int e_bytes;
957#if USE_EVP
958	EVP_PKEY *pkey;
959#endif
960
961	isc_buffer_remainingregion(data, &r);
962	if (r.length == 0)
963		return (ISC_R_SUCCESS);
964
965	rsa = RSA_new();
966	if (rsa == NULL)
967		return (dst__openssl_toresult(ISC_R_NOMEMORY));
968	SET_FLAGS(rsa);
969
970	if (r.length < 1) {
971		RSA_free(rsa);
972		return (DST_R_INVALIDPUBLICKEY);
973	}
974	e_bytes = *r.base++;
975	r.length--;
976
977	if (e_bytes == 0) {
978		if (r.length < 2) {
979			RSA_free(rsa);
980			return (DST_R_INVALIDPUBLICKEY);
981		}
982		e_bytes = ((*r.base++) << 8);
983		e_bytes += *r.base++;
984		r.length -= 2;
985	}
986
987	if (r.length < e_bytes) {
988		RSA_free(rsa);
989		return (DST_R_INVALIDPUBLICKEY);
990	}
991	rsa->e = BN_bin2bn(r.base, e_bytes, NULL);
992	r.base += e_bytes;
993	r.length -= e_bytes;
994
995	rsa->n = BN_bin2bn(r.base, r.length, NULL);
996
997	key->key_size = BN_num_bits(rsa->n);
998
999	isc_buffer_forward(data, r.length);
1000
1001#if USE_EVP
1002	pkey = EVP_PKEY_new();
1003	if (pkey == NULL) {
1004		RSA_free(rsa);
1005		return (ISC_R_NOMEMORY);
1006	}
1007	if (!EVP_PKEY_set1_RSA(pkey, rsa)) {
1008		EVP_PKEY_free(pkey);
1009		RSA_free(rsa);
1010		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1011	}
1012	key->keydata.pkey = pkey;
1013	RSA_free(rsa);
1014#else
1015	key->keydata.rsa = rsa;
1016#endif
1017
1018	return (ISC_R_SUCCESS);
1019}
1020
1021static isc_result_t
1022opensslrsa_tofile(const dst_key_t *key, const char *directory) {
1023	int i;
1024	RSA *rsa;
1025	dst_private_t priv;
1026	unsigned char *bufs[8];
1027	isc_result_t result;
1028
1029#if USE_EVP
1030	if (key->keydata.pkey == NULL)
1031		return (DST_R_NULLKEY);
1032	rsa = EVP_PKEY_get1_RSA(key->keydata.pkey);
1033	if (rsa == NULL)
1034		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1035#else
1036	if (key->keydata.rsa == NULL)
1037		return (DST_R_NULLKEY);
1038	rsa = key->keydata.rsa;
1039#endif
1040
1041	memset(bufs, 0, sizeof(bufs));
1042	for (i = 0; i < 8; i++) {
1043		bufs[i] = isc_mem_get(key->mctx, BN_num_bytes(rsa->n));
1044		if (bufs[i] == NULL) {
1045			result = ISC_R_NOMEMORY;
1046			goto fail;
1047		}
1048	}
1049
1050	i = 0;
1051
1052	priv.elements[i].tag = TAG_RSA_MODULUS;
1053	priv.elements[i].length = BN_num_bytes(rsa->n);
1054	BN_bn2bin(rsa->n, bufs[i]);
1055	priv.elements[i].data = bufs[i];
1056	i++;
1057
1058	priv.elements[i].tag = TAG_RSA_PUBLICEXPONENT;
1059	priv.elements[i].length = BN_num_bytes(rsa->e);
1060	BN_bn2bin(rsa->e, bufs[i]);
1061	priv.elements[i].data = bufs[i];
1062	i++;
1063
1064	if (rsa->d != NULL) {
1065		priv.elements[i].tag = TAG_RSA_PRIVATEEXPONENT;
1066		priv.elements[i].length = BN_num_bytes(rsa->d);
1067		BN_bn2bin(rsa->d, bufs[i]);
1068		priv.elements[i].data = bufs[i];
1069		i++;
1070	}
1071
1072	if (rsa->p != NULL) {
1073		priv.elements[i].tag = TAG_RSA_PRIME1;
1074		priv.elements[i].length = BN_num_bytes(rsa->p);
1075		BN_bn2bin(rsa->p, bufs[i]);
1076		priv.elements[i].data = bufs[i];
1077		i++;
1078	}
1079
1080	if (rsa->q != NULL) {
1081		priv.elements[i].tag = TAG_RSA_PRIME2;
1082		priv.elements[i].length = BN_num_bytes(rsa->q);
1083		BN_bn2bin(rsa->q, bufs[i]);
1084		priv.elements[i].data = bufs[i];
1085		i++;
1086	}
1087
1088	if (rsa->dmp1 != NULL) {
1089		priv.elements[i].tag = TAG_RSA_EXPONENT1;
1090		priv.elements[i].length = BN_num_bytes(rsa->dmp1);
1091		BN_bn2bin(rsa->dmp1, bufs[i]);
1092		priv.elements[i].data = bufs[i];
1093		i++;
1094	}
1095
1096	if (rsa->dmq1 != NULL) {
1097		priv.elements[i].tag = TAG_RSA_EXPONENT2;
1098		priv.elements[i].length = BN_num_bytes(rsa->dmq1);
1099		BN_bn2bin(rsa->dmq1, bufs[i]);
1100		priv.elements[i].data = bufs[i];
1101		i++;
1102	}
1103
1104	if (rsa->iqmp != NULL) {
1105		priv.elements[i].tag = TAG_RSA_COEFFICIENT;
1106		priv.elements[i].length = BN_num_bytes(rsa->iqmp);
1107		BN_bn2bin(rsa->iqmp, bufs[i]);
1108		priv.elements[i].data = bufs[i];
1109		i++;
1110	}
1111
1112	if (key->engine != NULL) {
1113		priv.elements[i].tag = TAG_RSA_ENGINE;
1114		priv.elements[i].length = strlen(key->engine) + 1;
1115		priv.elements[i].data = (unsigned char *)key->engine;
1116		i++;
1117	}
1118
1119	if (key->label != NULL) {
1120		priv.elements[i].tag = TAG_RSA_LABEL;
1121		priv.elements[i].length = strlen(key->label) + 1;
1122		priv.elements[i].data = (unsigned char *)key->label;
1123		i++;
1124	}
1125
1126
1127	priv.nelements = i;
1128	result = dst__privstruct_writefile(key, &priv, directory);
1129 fail:
1130#if USE_EVP
1131	RSA_free(rsa);
1132#endif
1133	for (i = 0; i < 8; i++) {
1134		if (bufs[i] == NULL)
1135			break;
1136		isc_mem_put(key->mctx, bufs[i], BN_num_bytes(rsa->n));
1137	}
1138	return (result);
1139}
1140
1141static isc_result_t
1142rsa_check(RSA *rsa, RSA *pub)
1143{
1144	/* Public parameters should be the same but if they are not set
1145	 * copy them from the public key. */
1146	if (pub != NULL) {
1147		if (rsa->n != NULL) {
1148			if (BN_cmp(rsa->n, pub->n) != 0)
1149				return (DST_R_INVALIDPRIVATEKEY);
1150		} else {
1151			rsa->n = pub->n;
1152			pub->n = NULL;
1153		}
1154		if (rsa->e != NULL) {
1155			if (BN_cmp(rsa->e, pub->e) != 0)
1156				return (DST_R_INVALIDPRIVATEKEY);
1157		} else {
1158			rsa->e = pub->e;
1159			pub->e = NULL;
1160		}
1161	}
1162	if (rsa->n == NULL || rsa->e == NULL)
1163		return (DST_R_INVALIDPRIVATEKEY);
1164	return (ISC_R_SUCCESS);
1165}
1166
1167static isc_result_t
1168opensslrsa_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
1169	dst_private_t priv;
1170	isc_result_t ret;
1171	int i;
1172	RSA *rsa = NULL, *pubrsa = NULL;
1173#ifdef USE_ENGINE
1174	ENGINE *e = NULL;
1175#endif
1176	isc_mem_t *mctx = key->mctx;
1177	const char *engine = NULL, *label = NULL;
1178#if defined(USE_ENGINE) || USE_EVP
1179	EVP_PKEY *pkey = NULL;
1180#endif
1181
1182#if USE_EVP
1183	if (pub != NULL && pub->keydata.pkey != NULL)
1184		pubrsa = EVP_PKEY_get1_RSA(pub->keydata.pkey);
1185#else
1186	if (pub != NULL && pub->keydata.rsa != NULL) {
1187		pubrsa = pub->keydata.rsa;
1188		pub->keydata.rsa = NULL;
1189	}
1190#endif
1191
1192	/* read private key file */
1193	ret = dst__privstruct_parse(key, DST_ALG_RSA, lexer, mctx, &priv);
1194	if (ret != ISC_R_SUCCESS)
1195		goto err;
1196
1197	for (i = 0; i < priv.nelements; i++) {
1198		switch (priv.elements[i].tag) {
1199		case TAG_RSA_ENGINE:
1200			engine = (char *)priv.elements[i].data;
1201			break;
1202		case TAG_RSA_LABEL:
1203			label = (char *)priv.elements[i].data;
1204			break;
1205		default:
1206			break;
1207		}
1208	}
1209	/*
1210	 * Is this key is stored in a HSM?
1211	 * See if we can fetch it.
1212	 */
1213	if (label != NULL) {
1214#ifdef USE_ENGINE
1215		if (engine == NULL)
1216			DST_RET(DST_R_NOENGINE);
1217		e = dst__openssl_getengine(engine);
1218		if (e == NULL)
1219			DST_RET(DST_R_NOENGINE);
1220		pkey = ENGINE_load_private_key(e, label, NULL, NULL);
1221		if (pkey == NULL)
1222			DST_RET(dst__openssl_toresult2(
1223					"ENGINE_load_private_key",
1224					ISC_R_NOTFOUND));
1225		key->engine = isc_mem_strdup(key->mctx, engine);
1226		if (key->engine == NULL)
1227			DST_RET(ISC_R_NOMEMORY);
1228		key->label = isc_mem_strdup(key->mctx, label);
1229		if (key->label == NULL)
1230			DST_RET(ISC_R_NOMEMORY);
1231		rsa = EVP_PKEY_get1_RSA(pkey);
1232		if (rsa == NULL)
1233			DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1234		if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
1235			DST_RET(DST_R_INVALIDPRIVATEKEY);
1236		if (BN_num_bits(rsa->e) > RSA_MAX_PUBEXP_BITS)
1237			DST_RET(ISC_R_RANGE);
1238		if (pubrsa != NULL)
1239			RSA_free(pubrsa);
1240		key->key_size = EVP_PKEY_bits(pkey);
1241#if USE_EVP
1242		key->keydata.pkey = pkey;
1243		RSA_free(rsa);
1244#else
1245		key->keydata.rsa = rsa;
1246		EVP_PKEY_free(pkey);
1247#endif
1248		dst__privstruct_free(&priv, mctx);
1249		memset(&priv, 0, sizeof(priv));
1250		return (ISC_R_SUCCESS);
1251#else
1252		DST_RET(DST_R_NOENGINE);
1253#endif
1254	}
1255
1256	rsa = RSA_new();
1257	if (rsa == NULL)
1258		DST_RET(ISC_R_NOMEMORY);
1259	SET_FLAGS(rsa);
1260
1261#if USE_EVP
1262	pkey = EVP_PKEY_new();
1263	if (pkey == NULL)
1264		DST_RET(ISC_R_NOMEMORY);
1265	if (!EVP_PKEY_set1_RSA(pkey, rsa))
1266		DST_RET(ISC_R_FAILURE);
1267	key->keydata.pkey = pkey;
1268#else
1269	key->keydata.rsa = rsa;
1270#endif
1271
1272	for (i = 0; i < priv.nelements; i++) {
1273		BIGNUM *bn;
1274		switch (priv.elements[i].tag) {
1275		case TAG_RSA_ENGINE:
1276			continue;
1277		case TAG_RSA_LABEL:
1278			continue;
1279		case TAG_RSA_PIN:
1280			continue;
1281		default:
1282			bn = BN_bin2bn(priv.elements[i].data,
1283				       priv.elements[i].length, NULL);
1284			if (bn == NULL)
1285				DST_RET(ISC_R_NOMEMORY);
1286		}
1287
1288		switch (priv.elements[i].tag) {
1289			case TAG_RSA_MODULUS:
1290				rsa->n = bn;
1291				break;
1292			case TAG_RSA_PUBLICEXPONENT:
1293				rsa->e = bn;
1294				break;
1295			case TAG_RSA_PRIVATEEXPONENT:
1296				rsa->d = bn;
1297				break;
1298			case TAG_RSA_PRIME1:
1299				rsa->p = bn;
1300				break;
1301			case TAG_RSA_PRIME2:
1302				rsa->q = bn;
1303				break;
1304			case TAG_RSA_EXPONENT1:
1305				rsa->dmp1 = bn;
1306				break;
1307			case TAG_RSA_EXPONENT2:
1308				rsa->dmq1 = bn;
1309				break;
1310			case TAG_RSA_COEFFICIENT:
1311				rsa->iqmp = bn;
1312				break;
1313		}
1314	}
1315	dst__privstruct_free(&priv, mctx);
1316	memset(&priv, 0, sizeof(priv));
1317
1318	if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
1319		DST_RET(DST_R_INVALIDPRIVATEKEY);
1320	if (BN_num_bits(rsa->e) > RSA_MAX_PUBEXP_BITS)
1321		DST_RET(ISC_R_RANGE);
1322	key->key_size = BN_num_bits(rsa->n);
1323	if (pubrsa != NULL)
1324		RSA_free(pubrsa);
1325#if USE_EVP
1326	RSA_free(rsa);
1327#endif
1328
1329	return (ISC_R_SUCCESS);
1330
1331 err:
1332#if USE_EVP
1333	if (pkey != NULL)
1334		EVP_PKEY_free(pkey);
1335#endif
1336	if (rsa != NULL)
1337		RSA_free(rsa);
1338	if (pubrsa != NULL)
1339		RSA_free(pubrsa);
1340	key->keydata.generic = NULL;
1341	dst__privstruct_free(&priv, mctx);
1342	memset(&priv, 0, sizeof(priv));
1343	return (ret);
1344}
1345
1346static isc_result_t
1347opensslrsa_fromlabel(dst_key_t *key, const char *engine, const char *label,
1348		     const char *pin)
1349{
1350#ifdef USE_ENGINE
1351	ENGINE *e = NULL;
1352	isc_result_t ret;
1353	EVP_PKEY *pkey = NULL;
1354	RSA *rsa = NULL, *pubrsa = NULL;
1355	char *colon;
1356
1357	UNUSED(pin);
1358
1359	if (engine == NULL)
1360		DST_RET(DST_R_NOENGINE);
1361	e = dst__openssl_getengine(engine);
1362	if (e == NULL)
1363		DST_RET(DST_R_NOENGINE);
1364	pkey = ENGINE_load_public_key(e, label, NULL, NULL);
1365	if (pkey != NULL) {
1366		pubrsa = EVP_PKEY_get1_RSA(pkey);
1367		EVP_PKEY_free(pkey);
1368		if (pubrsa == NULL)
1369			DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1370	}
1371	pkey = ENGINE_load_private_key(e, label, NULL, NULL);
1372	if (pkey == NULL)
1373		DST_RET(dst__openssl_toresult2("ENGINE_load_private_key",
1374					       ISC_R_NOTFOUND));
1375	if (engine != NULL) {
1376		key->engine = isc_mem_strdup(key->mctx, engine);
1377		if (key->engine == NULL)
1378			DST_RET(ISC_R_NOMEMORY);
1379	} else {
1380		key->engine = isc_mem_strdup(key->mctx, label);
1381		if (key->engine == NULL)
1382			DST_RET(ISC_R_NOMEMORY);
1383		colon = strchr(key->engine, ':');
1384		if (colon != NULL)
1385			*colon = '\0';
1386	}
1387	key->label = isc_mem_strdup(key->mctx, label);
1388	if (key->label == NULL)
1389		DST_RET(ISC_R_NOMEMORY);
1390	rsa = EVP_PKEY_get1_RSA(pkey);
1391	if (rsa == NULL)
1392		DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1393	if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
1394		DST_RET(DST_R_INVALIDPRIVATEKEY);
1395	if (BN_num_bits(rsa->e) > RSA_MAX_PUBEXP_BITS)
1396		DST_RET(ISC_R_RANGE);
1397	if (pubrsa != NULL)
1398		RSA_free(pubrsa);
1399	key->key_size = EVP_PKEY_bits(pkey);
1400#if USE_EVP
1401	key->keydata.pkey = pkey;
1402	RSA_free(rsa);
1403#else
1404	key->keydata.rsa = rsa;
1405	EVP_PKEY_free(pkey);
1406#endif
1407	return (ISC_R_SUCCESS);
1408
1409 err:
1410	if (rsa != NULL)
1411		RSA_free(rsa);
1412	if (pubrsa != NULL)
1413		RSA_free(pubrsa);
1414	if (pkey != NULL)
1415		EVP_PKEY_free(pkey);
1416	return (ret);
1417#else
1418	UNUSED(key);
1419	UNUSED(engine);
1420	UNUSED(label);
1421	UNUSED(pin);
1422	return(DST_R_NOENGINE);
1423#endif
1424}
1425
1426static dst_func_t opensslrsa_functions = {
1427	opensslrsa_createctx,
1428	opensslrsa_destroyctx,
1429	opensslrsa_adddata,
1430	opensslrsa_sign,
1431	opensslrsa_verify,
1432	opensslrsa_verify2,
1433	NULL, /*%< computesecret */
1434	opensslrsa_compare,
1435	NULL, /*%< paramcompare */
1436	opensslrsa_generate,
1437	opensslrsa_isprivate,
1438	opensslrsa_destroy,
1439	opensslrsa_todns,
1440	opensslrsa_fromdns,
1441	opensslrsa_tofile,
1442	opensslrsa_parse,
1443	NULL, /*%< cleanup */
1444	opensslrsa_fromlabel,
1445	NULL, /*%< dump */
1446	NULL, /*%< restore */
1447};
1448
1449isc_result_t
1450dst__opensslrsa_init(dst_func_t **funcp, unsigned char algorithm) {
1451	REQUIRE(funcp != NULL);
1452
1453	if (*funcp == NULL) {
1454		switch (algorithm) {
1455		case DST_ALG_RSASHA256:
1456#if defined(HAVE_EVP_SHA256) || !USE_EVP
1457			*funcp = &opensslrsa_functions;
1458#endif
1459			break;
1460		case DST_ALG_RSASHA512:
1461#if defined(HAVE_EVP_SHA512) || !USE_EVP
1462			*funcp = &opensslrsa_functions;
1463#endif
1464			break;
1465		default:
1466			*funcp = &opensslrsa_functions;
1467			break;
1468		}
1469	}
1470	return (ISC_R_SUCCESS);
1471}
1472
1473#else /* OPENSSL */
1474
1475#include <isc/util.h>
1476
1477EMPTY_TRANSLATION_UNIT
1478
1479#endif /* OPENSSL */
1480/*! \file */
1481