opensslrsa_link.c revision 287410
119304Speter/*
219304Speter * Copyright (C) 2004-2009, 2011-2014  Internet Systems Consortium, Inc. ("ISC")
319304Speter * Copyright (C) 2000-2003  Internet Software Consortium.
419304Speter *
519304Speter * Permission to use, copy, modify, and/or distribute this software for any
619304Speter * purpose with or without fee is hereby granted, provided that the above
719304Speter * copyright notice and this permission notice appear in all copies.
819304Speter *
919304Speter * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
1019304Speter * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
1119304Speter * AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
1219304Speter * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
13254225Speter * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
1419304Speter * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
1519304Speter * PERFORMANCE OF THIS SOFTWARE.
16254225Speter */
1719304Speter
1819304Speter/*
1919304Speter * Principal Author: Brian Wellington
2019304Speter * $Id$
2119304Speter */
2219304Speter#ifdef OPENSSL
2319304Speter#include <config.h>
2419304Speter
2519304Speter#ifndef USE_EVP
26254225Speter#if !defined(HAVE_EVP_SHA256) || !defined(HAVE_EVP_SHA512)
2719304Speter#define USE_EVP 0
2819304Speter#else
2919304Speter#define USE_EVP 1
3019304Speter#endif
3119304Speter#endif
3219304Speter
3319304Speter
3419304Speter#include <isc/entropy.h>
3519304Speter#include <isc/md5.h>
3619304Speter#include <isc/sha1.h>
3719304Speter#include <isc/sha2.h>
3819304Speter#include <isc/mem.h>
3919304Speter#include <isc/string.h>
4019304Speter#include <isc/util.h>
4119304Speter
4219304Speter#include <dst/result.h>
4319304Speter
4419304Speter#include "dst_internal.h"
4519304Speter#include "dst_openssl.h"
4619304Speter#include "dst_parse.h"
4719304Speter
4819304Speter#include <openssl/err.h>
4919304Speter#include <openssl/objects.h>
5019304Speter#include <openssl/rsa.h>
5119304Speter#if OPENSSL_VERSION_NUMBER > 0x00908000L
5219304Speter#include <openssl/bn.h>
5319304Speter#endif
5419304Speter#ifdef USE_ENGINE
5519304Speter#include <openssl/engine.h>
5619304Speter#endif
5719304Speter
5819304Speter/*
5919304Speter * Limit the size of public exponents.
6019304Speter */
6119304Speter#ifndef RSA_MAX_PUBEXP_BITS
6219304Speter#define RSA_MAX_PUBEXP_BITS    35
6319304Speter#endif
64254225Speter
65254225Speter/*
66254225Speter * We don't use configure for windows so enforce the OpenSSL version
67254225Speter * here.  Unlike with configure we don't support overriding this test.
68254225Speter */
6919304Speter#ifdef WIN32
7019304Speter#if !((OPENSSL_VERSION_NUMBER >= 0x009070cfL && \
7119304Speter       OPENSSL_VERSION_NUMBER < 0x00908000L) || \
72254225Speter      OPENSSL_VERSION_NUMBER >= 0x0090804fL)
7319304Speter#error Please upgrade OpenSSL to 0.9.8d/0.9.7l or greater.
7419304Speter#endif
7519304Speter#endif
7619304Speter
7719304Speter
7819304Speter	/*
7919304Speter	 * XXXMPA  Temporarily disable RSA_BLINDING as it requires
8019304Speter	 * good quality random data that cannot currently be guaranteed.
8119304Speter	 * XXXMPA  Find which versions of openssl use pseudo random data
8219304Speter	 * and set RSA_FLAG_BLINDING for those.
8319304Speter	 */
8419304Speter
8519304Speter#if 0
8619304Speter#if OPENSSL_VERSION_NUMBER < 0x0090601fL
8719304Speter#define SET_FLAGS(rsa) \
8819304Speter	do { \
8919304Speter	(rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
9019304Speter	(rsa)->flags |= RSA_FLAG_BLINDING; \
9119304Speter	} while (0)
9219304Speter#else
9319304Speter#define SET_FLAGS(rsa) \
9419304Speter	do { \
9519304Speter		(rsa)->flags |= RSA_FLAG_BLINDING; \
9619304Speter	} while (0)
9719304Speter#endif
9819304Speter#endif
9919304Speter
10019304Speter#if OPENSSL_VERSION_NUMBER < 0x0090601fL
10119304Speter#define SET_FLAGS(rsa) \
10219304Speter	do { \
10319304Speter	(rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
10419304Speter	(rsa)->flags &= ~RSA_FLAG_BLINDING; \
10519304Speter	} while (0)
10619304Speter#elif defined(RSA_FLAG_NO_BLINDING)
10719304Speter#define SET_FLAGS(rsa) \
10819304Speter	do { \
10919304Speter		(rsa)->flags &= ~RSA_FLAG_BLINDING; \
110254225Speter		(rsa)->flags |= RSA_FLAG_NO_BLINDING; \
11119304Speter	} while (0)
11219304Speter#else
11319304Speter#define SET_FLAGS(rsa) \
11419304Speter	do { \
115254225Speter		(rsa)->flags &= ~RSA_FLAG_BLINDING; \
116254225Speter	} while (0)
117254225Speter#endif
118254225Speter
11919304Speter#define DST_RET(a) {ret = a; goto err;}
12019304Speter
12119304Speterstatic isc_result_t opensslrsa_todns(const dst_key_t *key, isc_buffer_t *data);
12219304Speter
12319304Speterstatic isc_result_t
12419304Speteropensslrsa_createctx(dst_key_t *key, dst_context_t *dctx) {
12519304Speter#if USE_EVP
12619304Speter	EVP_MD_CTX *evp_md_ctx;
12719304Speter	const EVP_MD *type = NULL;
12819304Speter#endif
12919304Speter
13019304Speter	UNUSED(key);
13119304Speter	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
13219304Speter		dctx->key->key_alg == DST_ALG_RSASHA1 ||
13319304Speter		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
13419304Speter		dctx->key->key_alg == DST_ALG_RSASHA256 ||
13519304Speter		dctx->key->key_alg == DST_ALG_RSASHA512);
13619304Speter
13719304Speter#if USE_EVP
13819304Speter	evp_md_ctx = EVP_MD_CTX_create();
13919304Speter	if (evp_md_ctx == NULL)
14019304Speter		return (ISC_R_NOMEMORY);
14119304Speter
142254225Speter	switch (dctx->key->key_alg) {
143254225Speter	case DST_ALG_RSAMD5:
14419304Speter		type = EVP_md5();	/* MD5 + RSA */
14519304Speter		break;
14619304Speter	case DST_ALG_RSASHA1:
14719304Speter	case DST_ALG_NSEC3RSASHA1:
14819304Speter		type = EVP_sha1();	/* SHA1 + RSA */
14919304Speter		break;
15019304Speter#ifdef HAVE_EVP_SHA256
15119304Speter	case DST_ALG_RSASHA256:
15219304Speter		type = EVP_sha256();	/* SHA256 + RSA */
15319304Speter		break;
15419304Speter#endif
15519304Speter#ifdef HAVE_EVP_SHA512
15619304Speter	case DST_ALG_RSASHA512:
157254225Speter		type = EVP_sha512();
15819304Speter		break;
15919304Speter#endif
16019304Speter	default:
161254225Speter		INSIST(0);
162254225Speter	}
16319304Speter
16419304Speter	if (!EVP_DigestInit_ex(evp_md_ctx, type, NULL)) {
16519304Speter		EVP_MD_CTX_destroy(evp_md_ctx);
16619304Speter		return (dst__openssl_toresult3(dctx->category,
16719304Speter					       "EVP_DigestInit_ex",
16819304Speter					       ISC_R_FAILURE));
16919304Speter	}
17019304Speter	dctx->ctxdata.evp_md_ctx = evp_md_ctx;
17119304Speter#else
17219304Speter	switch (dctx->key->key_alg) {
17319304Speter	case DST_ALG_RSAMD5:
17419304Speter		{
17519304Speter			isc_md5_t *md5ctx;
17619304Speter
17719304Speter			md5ctx = isc_mem_get(dctx->mctx, sizeof(isc_md5_t));
17819304Speter			if (md5ctx == NULL)
179254225Speter				return (ISC_R_NOMEMORY);
18019304Speter			isc_md5_init(md5ctx);
18119304Speter			dctx->ctxdata.md5ctx = md5ctx;
18219304Speter		}
183254225Speter		break;
184254225Speter	case DST_ALG_RSASHA1:
185254225Speter	case DST_ALG_NSEC3RSASHA1:
186254225Speter		{
18719304Speter			isc_sha1_t *sha1ctx;
18819304Speter
18919304Speter			sha1ctx = isc_mem_get(dctx->mctx, sizeof(isc_sha1_t));
19019304Speter			if (sha1ctx == NULL)
19119304Speter				return (ISC_R_NOMEMORY);
19219304Speter			isc_sha1_init(sha1ctx);
19319304Speter			dctx->ctxdata.sha1ctx = sha1ctx;
19419304Speter		}
19519304Speter		break;
19619304Speter	case DST_ALG_RSASHA256:
197254225Speter		{
19819304Speter			isc_sha256_t *sha256ctx;
19919304Speter
20019304Speter			sha256ctx = isc_mem_get(dctx->mctx,
20119304Speter						sizeof(isc_sha256_t));
20219304Speter			if (sha256ctx == NULL)
20319304Speter				return (ISC_R_NOMEMORY);
204254225Speter			isc_sha256_init(sha256ctx);
205254225Speter			dctx->ctxdata.sha256ctx = sha256ctx;
206254225Speter		}
20719304Speter		break;
20819304Speter	case DST_ALG_RSASHA512:
20919304Speter		{
21019304Speter			isc_sha512_t *sha512ctx;
21119304Speter
21219304Speter			sha512ctx = isc_mem_get(dctx->mctx,
21319304Speter						sizeof(isc_sha512_t));
21419304Speter			if (sha512ctx == NULL)
21519304Speter				return (ISC_R_NOMEMORY);
216254225Speter			isc_sha512_init(sha512ctx);
21719304Speter			dctx->ctxdata.sha512ctx = sha512ctx;
218254225Speter		}
21919304Speter		break;
22019304Speter	default:
22119304Speter		INSIST(0);
22219304Speter	}
22319304Speter#endif
22419304Speter
22519304Speter	return (ISC_R_SUCCESS);
226254225Speter}
22719304Speter
22819304Speterstatic void
22919304Speteropensslrsa_destroyctx(dst_context_t *dctx) {
230254225Speter#if USE_EVP
231254225Speter	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
23219304Speter#endif
23319304Speter
23419304Speter	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
23519304Speter		dctx->key->key_alg == DST_ALG_RSASHA1 ||
23619304Speter		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
23719304Speter		dctx->key->key_alg == DST_ALG_RSASHA256 ||
23819304Speter		dctx->key->key_alg == DST_ALG_RSASHA512);
239254225Speter
240254225Speter#if USE_EVP
24119304Speter	if (evp_md_ctx != NULL) {
24219304Speter		EVP_MD_CTX_destroy(evp_md_ctx);
24319304Speter		dctx->ctxdata.evp_md_ctx = NULL;
244254225Speter	}
245254225Speter#else
246254225Speter	switch (dctx->key->key_alg) {
247254225Speter	case DST_ALG_RSAMD5:
24819304Speter		{
249254225Speter			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
250254225Speter
25119304Speter			if (md5ctx != NULL) {
25219304Speter				isc_md5_invalidate(md5ctx);
253254225Speter				isc_mem_put(dctx->mctx, md5ctx,
25419304Speter					    sizeof(isc_md5_t));
255254225Speter				dctx->ctxdata.md5ctx = NULL;
25619304Speter			}
257254225Speter		}
258254225Speter		break;
259254225Speter	case DST_ALG_RSASHA1:
260254225Speter	case DST_ALG_NSEC3RSASHA1:
26119304Speter		{
26219304Speter			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
263254225Speter
26419304Speter			if (sha1ctx != NULL) {
26519304Speter				isc_sha1_invalidate(sha1ctx);
26619304Speter				isc_mem_put(dctx->mctx, sha1ctx,
267254225Speter					    sizeof(isc_sha1_t));
268254225Speter				dctx->ctxdata.sha1ctx = NULL;
26919304Speter			}
27019304Speter		}
27119304Speter		break;
27219304Speter	case DST_ALG_RSASHA256:
27319304Speter		{
27419304Speter			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
27519304Speter
27619304Speter			if (sha256ctx != NULL) {
27719304Speter				isc_sha256_invalidate(sha256ctx);
27819304Speter				isc_mem_put(dctx->mctx, sha256ctx,
27919304Speter					    sizeof(isc_sha256_t));
28019304Speter				dctx->ctxdata.sha256ctx = NULL;
28119304Speter			}
28219304Speter		}
283254225Speter		break;
28419304Speter	case DST_ALG_RSASHA512:
28519304Speter		{
286254225Speter			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
28719304Speter
28819304Speter			if (sha512ctx != NULL) {
28919304Speter				isc_sha512_invalidate(sha512ctx);
29019304Speter				isc_mem_put(dctx->mctx, sha512ctx,
29119304Speter					    sizeof(isc_sha512_t));
29219304Speter				dctx->ctxdata.sha512ctx = NULL;
29319304Speter			}
29419304Speter		}
29519304Speter		break;
29619304Speter	default:
29719304Speter		INSIST(0);
298254225Speter	}
29919304Speter#endif
30019304Speter}
30119304Speter
30219304Speterstatic isc_result_t
303254225Speteropensslrsa_adddata(dst_context_t *dctx, const isc_region_t *data) {
30419304Speter#if USE_EVP
30519304Speter	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
30619304Speter#endif
30719304Speter
30819304Speter	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
30919304Speter		dctx->key->key_alg == DST_ALG_RSASHA1 ||
31019304Speter		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
31119304Speter		dctx->key->key_alg == DST_ALG_RSASHA256 ||
31219304Speter		dctx->key->key_alg == DST_ALG_RSASHA512);
31319304Speter
31419304Speter#if USE_EVP
31519304Speter	if (!EVP_DigestUpdate(evp_md_ctx, data->base, data->length)) {
31619304Speter		return (dst__openssl_toresult3(dctx->category,
317254225Speter					       "EVP_DigestUpdate",
318254225Speter					       ISC_R_FAILURE));
319254225Speter	}
320254225Speter#else
32119304Speter	switch (dctx->key->key_alg) {
32219304Speter	case DST_ALG_RSAMD5:
32319304Speter		{
32419304Speter			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
32519304Speter
32619304Speter			isc_md5_update(md5ctx, data->base, data->length);
32719304Speter		}
32819304Speter		break;
32919304Speter	case DST_ALG_RSASHA1:
330254225Speter	case DST_ALG_NSEC3RSASHA1:
33119304Speter		{
33219304Speter			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
33319304Speter
334254225Speter			isc_sha1_update(sha1ctx, data->base, data->length);
33519304Speter		}
33619304Speter		break;
33719304Speter	case DST_ALG_RSASHA256:
33819304Speter		{
33919304Speter			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
34019304Speter
34119304Speter			isc_sha256_update(sha256ctx, data->base, data->length);
34219304Speter		}
34319304Speter		break;
34419304Speter	case DST_ALG_RSASHA512:
34519304Speter		{
34619304Speter			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
34719304Speter
34819304Speter			isc_sha512_update(sha512ctx, data->base, data->length);
34919304Speter		}
350254225Speter		break;
35119304Speter	default:
35219304Speter		INSIST(0);
35319304Speter	}
35419304Speter#endif
35519304Speter	return (ISC_R_SUCCESS);
35619304Speter}
35719304Speter
35819304Speter#if ! USE_EVP && OPENSSL_VERSION_NUMBER < 0x00908000L
35919304Speter/*
36019304Speter * Digest prefixes from RFC 5702.
36119304Speter */
36219304Speterstatic unsigned char sha256_prefix[] =
36319304Speter	 { 0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
36419304Speter	   0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20};
36519304Speterstatic unsigned char sha512_prefix[] =
36619304Speter	 { 0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
36719304Speter	   0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40};
36819304Speter#define PREFIXLEN sizeof(sha512_prefix)
36919304Speter#else
37019304Speter#define PREFIXLEN 0
37119304Speter#endif
37219304Speter
37319304Speterstatic isc_result_t
37419304Speteropensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
37519304Speter	dst_key_t *key = dctx->key;
37619304Speter	isc_region_t r;
377254225Speter	unsigned int siglen = 0;
37819304Speter#if USE_EVP
37919304Speter	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
380254225Speter	EVP_PKEY *pkey = key->keydata.pkey;
38119304Speter#else
38219304Speter	RSA *rsa = key->keydata.rsa;
38319304Speter	/* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
38419304Speter	unsigned char digest[PREFIXLEN + ISC_SHA512_DIGESTLENGTH];
38519304Speter	int status;
386254225Speter	int type = 0;
38719304Speter	unsigned int digestlen = 0;
38819304Speter#if OPENSSL_VERSION_NUMBER < 0x00908000L
38919304Speter	unsigned int prefixlen = 0;
39019304Speter	const unsigned char *prefix = NULL;
39119304Speter#endif
392254225Speter#endif
39319304Speter
39419304Speter	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
39519304Speter		dctx->key->key_alg == DST_ALG_RSASHA1 ||
39619304Speter		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
39719304Speter		dctx->key->key_alg == DST_ALG_RSASHA256 ||
39819304Speter		dctx->key->key_alg == DST_ALG_RSASHA512);
39919304Speter
40019304Speter	isc_buffer_availableregion(sig, &r);
40119304Speter
40219304Speter#if USE_EVP
40319304Speter	if (r.length < (unsigned int) EVP_PKEY_size(pkey))
40419304Speter		return (ISC_R_NOSPACE);
40519304Speter
40619304Speter	if (!EVP_SignFinal(evp_md_ctx, r.base, &siglen, pkey)) {
40719304Speter		return (dst__openssl_toresult3(dctx->category,
40819304Speter					       "EVP_SignFinal",
40919304Speter					       ISC_R_FAILURE));
41019304Speter	}
41119304Speter#else
41219304Speter	if (r.length < (unsigned int) RSA_size(rsa))
41319304Speter		return (ISC_R_NOSPACE);
414254225Speter
415254225Speter	switch (dctx->key->key_alg) {
416254225Speter	case DST_ALG_RSAMD5:
417254225Speter		{
418254225Speter			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
419254225Speter
420254225Speter			isc_md5_final(md5ctx, digest);
421254225Speter			type = NID_md5;
422254225Speter			digestlen = ISC_MD5_DIGESTLENGTH;
423254225Speter		}
424254225Speter		break;
425254225Speter	case DST_ALG_RSASHA1:
426254225Speter	case DST_ALG_NSEC3RSASHA1:
427254225Speter		{
428254225Speter			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
42919304Speter
430254225Speter			isc_sha1_final(sha1ctx, digest);
43119304Speter			type = NID_sha1;
43219304Speter			digestlen = ISC_SHA1_DIGESTLENGTH;
43319304Speter		}
43419304Speter		break;
43519304Speter	case DST_ALG_RSASHA256:
43619304Speter		{
43719304Speter			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
43819304Speter
43919304Speter			isc_sha256_final(digest, sha256ctx);
44019304Speter			digestlen = ISC_SHA256_DIGESTLENGTH;
44119304Speter#if OPENSSL_VERSION_NUMBER < 0x00908000L
44219304Speter			prefix = sha256_prefix;
44319304Speter			prefixlen = sizeof(sha256_prefix);
44419304Speter#else
44519304Speter			type = NID_sha256;
44619304Speter#endif
44719304Speter		}
44819304Speter		break;
44919304Speter	case DST_ALG_RSASHA512:
45019304Speter		{
45119304Speter			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
45219304Speter
45319304Speter			isc_sha512_final(digest, sha512ctx);
45419304Speter			digestlen = ISC_SHA512_DIGESTLENGTH;
45519304Speter#if OPENSSL_VERSION_NUMBER < 0x00908000L
456254225Speter			prefix = sha512_prefix;
45719304Speter			prefixlen = sizeof(sha512_prefix);
45819304Speter#else
45919304Speter			type = NID_sha512;
46019304Speter#endif
46119304Speter		}
46219304Speter		break;
46319304Speter	default:
46419304Speter		INSIST(0);
46519304Speter	}
466254225Speter
467254225Speter#if OPENSSL_VERSION_NUMBER < 0x00908000L
46819304Speter	switch (dctx->key->key_alg) {
46919304Speter	case DST_ALG_RSAMD5:
47019304Speter	case DST_ALG_RSASHA1:
47119304Speter	case DST_ALG_NSEC3RSASHA1:
472254225Speter		INSIST(type != 0);
47319304Speter		status = RSA_sign(type, digest, digestlen, r.base,
47419304Speter				  &siglen, rsa);
47519304Speter		break;
47619304Speter
47719304Speter	case DST_ALG_RSASHA256:
47819304Speter	case DST_ALG_RSASHA512:
47919304Speter		INSIST(prefix != NULL);
48019304Speter		INSIST(prefixlen != 0);
48119304Speter		INSIST(prefixlen + digestlen <= sizeof(digest));
48219304Speter
48319304Speter		memmove(digest + prefixlen, digest, digestlen);
484254225Speter		memmove(digest, prefix, prefixlen);
48519304Speter		status = RSA_private_encrypt(digestlen + prefixlen,
48619304Speter					     digest, r.base, rsa,
487254225Speter					     RSA_PKCS1_PADDING);
48819304Speter		if (status < 0)
48919304Speter			status = 0;
49019304Speter		else
491254225Speter			siglen = status;
492254225Speter		break;
49319304Speter
49419304Speter	default:
49519304Speter		INSIST(0);
496254225Speter	}
497254225Speter#else
498254225Speter	INSIST(type != 0);
49919304Speter	status = RSA_sign(type, digest, digestlen, r.base, &siglen, rsa);
500254225Speter#endif
501254225Speter	if (status == 0)
50219304Speter		return (dst__openssl_toresult3(dctx->category,
50319304Speter					       "RSA_sign",
50419304Speter					       DST_R_OPENSSLFAILURE));
50519304Speter#endif
50619304Speter
50719304Speter	isc_buffer_add(sig, siglen);
50819304Speter
50919304Speter	return (ISC_R_SUCCESS);
51019304Speter}
51119304Speter
51219304Speterstatic isc_result_t
51319304Speteropensslrsa_verify2(dst_context_t *dctx, int maxbits, const isc_region_t *sig) {
514254225Speter	dst_key_t *key = dctx->key;
515254225Speter	int status = 0;
51619304Speter#if USE_EVP
51719304Speter	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
51819304Speter	EVP_PKEY *pkey = key->keydata.pkey;
51919304Speter	RSA *rsa;
52019304Speter	int bits;
52119304Speter#else
522254225Speter	/* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
52319304Speter	unsigned char digest[ISC_SHA512_DIGESTLENGTH];
52419304Speter	int type = 0;
52519304Speter	unsigned int digestlen = 0;
526254225Speter	RSA *rsa = key->keydata.rsa;
527254225Speter#if OPENSSL_VERSION_NUMBER < 0x00908000L
52819304Speter	unsigned int prefixlen = 0;
52919304Speter	const unsigned char *prefix = NULL;
53019304Speter#endif
53119304Speter#endif
532254225Speter
533254225Speter	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
534254225Speter		dctx->key->key_alg == DST_ALG_RSASHA1 ||
535254225Speter		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
536254225Speter		dctx->key->key_alg == DST_ALG_RSASHA256 ||
537254225Speter		dctx->key->key_alg == DST_ALG_RSASHA512);
53819304Speter
53919304Speter#if USE_EVP
54019304Speter	rsa = EVP_PKEY_get1_RSA(pkey);
54119304Speter	if (rsa == NULL)
54219304Speter		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
54319304Speter	bits = BN_num_bits(rsa->e);
54419304Speter	RSA_free(rsa);
54519304Speter	if (bits > maxbits && maxbits != 0)
54619304Speter		return (DST_R_VERIFYFAILURE);
54719304Speter
54819304Speter	status = EVP_VerifyFinal(evp_md_ctx, sig->base, sig->length, pkey);
54919304Speter	switch (status) {
55019304Speter	case 1:
55119304Speter		return (ISC_R_SUCCESS);
55219304Speter	case 0:
55319304Speter		return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
55419304Speter	default:
55519304Speter		return (dst__openssl_toresult3(dctx->category,
55619304Speter					       "EVP_VerifyFinal",
55719304Speter					       DST_R_VERIFYFAILURE));
558254225Speter	}
559254225Speter#else
56019304Speter	if (BN_num_bits(rsa->e) > maxbits && maxbits != 0)
561254225Speter		return (DST_R_VERIFYFAILURE);
56219304Speter
56319304Speter	switch (dctx->key->key_alg) {
564254225Speter	case DST_ALG_RSAMD5:
56519304Speter		{
56619304Speter			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
56719304Speter
56819304Speter			isc_md5_final(md5ctx, digest);
56919304Speter			type = NID_md5;
57019304Speter			digestlen = ISC_MD5_DIGESTLENGTH;
57119304Speter		}
57219304Speter		break;
57319304Speter	case DST_ALG_RSASHA1:
57419304Speter	case DST_ALG_NSEC3RSASHA1:
57519304Speter		{
57619304Speter			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
57719304Speter
57819304Speter			isc_sha1_final(sha1ctx, digest);
57919304Speter			type = NID_sha1;
58019304Speter			digestlen = ISC_SHA1_DIGESTLENGTH;
58119304Speter		}
58219304Speter		break;
58319304Speter	case DST_ALG_RSASHA256:
58419304Speter		{
58519304Speter			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
58619304Speter
58719304Speter			isc_sha256_final(digest, sha256ctx);
58819304Speter			digestlen = ISC_SHA256_DIGESTLENGTH;
58919304Speter#if OPENSSL_VERSION_NUMBER < 0x00908000L
59019304Speter			prefix = sha256_prefix;
59119304Speter			prefixlen = sizeof(sha256_prefix);
59219304Speter#else
59319304Speter			type = NID_sha256;
59419304Speter#endif
595254225Speter		}
596254225Speter		break;
59719304Speter	case DST_ALG_RSASHA512:
59819304Speter		{
59919304Speter			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
60019304Speter
60119304Speter			isc_sha512_final(digest, sha512ctx);
60219304Speter			digestlen = ISC_SHA512_DIGESTLENGTH;
60319304Speter#if OPENSSL_VERSION_NUMBER < 0x00908000L
60419304Speter			prefix = sha512_prefix;
605254225Speter			prefixlen = sizeof(sha512_prefix);
60619304Speter#else
60719304Speter			type = NID_sha512;
60819304Speter#endif
60919304Speter		}
61019304Speter		break;
61119304Speter	default:
61219304Speter		INSIST(0);
61319304Speter	}
61419304Speter
61519304Speter	if (sig->length != (unsigned int) RSA_size(rsa))
61619304Speter		return (DST_R_VERIFYFAILURE);
61719304Speter
61819304Speter#if OPENSSL_VERSION_NUMBER < 0x00908000L
61919304Speter	switch (dctx->key->key_alg) {
62019304Speter	case DST_ALG_RSAMD5:
62119304Speter	case DST_ALG_RSASHA1:
62219304Speter	case DST_ALG_NSEC3RSASHA1:
62319304Speter		INSIST(type != 0);
62419304Speter		status = RSA_verify(type, digest, digestlen, sig->base,
625254225Speter				    RSA_size(rsa), rsa);
626254225Speter		break;
62719304Speter
62819304Speter	case DST_ALG_RSASHA256:
62919304Speter	case DST_ALG_RSASHA512:
63019304Speter		{
63119304Speter			/*
63219304Speter			 * 1024 is big enough for all valid RSA bit sizes
63319304Speter			 * for use with DNSSEC.
63419304Speter			 */
63519304Speter			unsigned char original[PREFIXLEN + 1024];
63619304Speter
63719304Speter			INSIST(prefix != NULL);
638254225Speter			INSIST(prefixlen != 0U);
63919304Speter
64019304Speter			if (RSA_size(rsa) > (int)sizeof(original))
64119304Speter				return (DST_R_VERIFYFAILURE);
64219304Speter
64319304Speter			status = RSA_public_decrypt(sig->length, sig->base,
64419304Speter						    original, rsa,
64519304Speter						    RSA_PKCS1_PADDING);
64619304Speter			if (status <= 0)
64719304Speter				return (dst__openssl_toresult3(
64819304Speter						dctx->category,
649254225Speter						"RSA_public_decrypt",
650254225Speter						DST_R_VERIFYFAILURE));
65119304Speter			if (status != (int)(prefixlen + digestlen))
65219304Speter				return (DST_R_VERIFYFAILURE);
65319304Speter			if (memcmp(original, prefix, prefixlen))
65419304Speter				return (DST_R_VERIFYFAILURE);
65519304Speter			if (memcmp(original + prefixlen, digest, digestlen))
65619304Speter				return (DST_R_VERIFYFAILURE);
65719304Speter			status = 1;
658254225Speter		}
65919304Speter		break;
66019304Speter
66119304Speter	default:
66219304Speter		INSIST(0);
66319304Speter	}
66419304Speter#else
66519304Speter	INSIST(type != 0);
66619304Speter	status = RSA_verify(type, digest, digestlen, sig->base,
66719304Speter			     RSA_size(rsa), rsa);
66819304Speter#endif
66919304Speter	if (status != 1)
67019304Speter		return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
67119304Speter	return (ISC_R_SUCCESS);
67219304Speter#endif
67319304Speter}
67419304Speter
675254225Speterstatic isc_result_t
67619304Speteropensslrsa_verify(dst_context_t *dctx, const isc_region_t *sig) {
67719304Speter	return (opensslrsa_verify2(dctx, 0, sig));
678254225Speter}
679254225Speter
680254225Speterstatic isc_boolean_t
681254225Speteropensslrsa_compare(const dst_key_t *key1, const dst_key_t *key2) {
682254225Speter	int status;
683254225Speter	RSA *rsa1 = NULL, *rsa2 = NULL;
68419304Speter#if USE_EVP
68519304Speter	EVP_PKEY *pkey1, *pkey2;
68619304Speter#endif
68719304Speter
68819304Speter#if USE_EVP
68919304Speter	pkey1 = key1->keydata.pkey;
69019304Speter	pkey2 = key2->keydata.pkey;
69119304Speter	/*
69219304Speter	 * The pkey reference will keep these around after
69319304Speter	 * the RSA_free() call.
69419304Speter	 */
69519304Speter	if (pkey1 != NULL) {
69619304Speter		rsa1 = EVP_PKEY_get1_RSA(pkey1);
69719304Speter		RSA_free(rsa1);
69819304Speter	}
69919304Speter	if (pkey2 != NULL) {
70019304Speter		rsa2 = EVP_PKEY_get1_RSA(pkey2);
70119304Speter		RSA_free(rsa2);
70219304Speter	}
70319304Speter#else
70419304Speter	rsa1 = key1->keydata.rsa;
70519304Speter	rsa2 = key2->keydata.rsa;
70619304Speter#endif
70719304Speter
70819304Speter	if (rsa1 == NULL && rsa2 == NULL)
70919304Speter		return (ISC_TRUE);
71019304Speter	else if (rsa1 == NULL || rsa2 == NULL)
71119304Speter		return (ISC_FALSE);
71219304Speter
71319304Speter	status = BN_cmp(rsa1->n, rsa2->n) ||
71419304Speter		 BN_cmp(rsa1->e, rsa2->e);
71519304Speter
71619304Speter	if (status != 0)
71719304Speter		return (ISC_FALSE);
71819304Speter
71919304Speter#if USE_EVP
72019304Speter	if ((rsa1->flags & RSA_FLAG_EXT_PKEY) != 0 ||
72119304Speter	    (rsa2->flags & RSA_FLAG_EXT_PKEY) != 0) {
72219304Speter		if ((rsa1->flags & RSA_FLAG_EXT_PKEY) == 0 ||
72319304Speter		    (rsa2->flags & RSA_FLAG_EXT_PKEY) == 0)
72419304Speter			return (ISC_FALSE);
72519304Speter		/*
72619304Speter		 * Can't compare private parameters, BTW does it make sense?
72719304Speter		 */
72819304Speter		return (ISC_TRUE);
72919304Speter	}
73019304Speter#endif
73119304Speter
73219304Speter	if (rsa1->d != NULL || rsa2->d != NULL) {
73319304Speter		if (rsa1->d == NULL || rsa2->d == NULL)
73419304Speter			return (ISC_FALSE);
73519304Speter		status = BN_cmp(rsa1->d, rsa2->d) ||
73619304Speter			 BN_cmp(rsa1->p, rsa2->p) ||
73719304Speter			 BN_cmp(rsa1->q, rsa2->q);
73819304Speter
73919304Speter		if (status != 0)
74019304Speter			return (ISC_FALSE);
74119304Speter	}
74219304Speter	return (ISC_TRUE);
74319304Speter}
74419304Speter
74519304Speter#if OPENSSL_VERSION_NUMBER > 0x00908000L
74619304Speterstatic int
74719304Speterprogress_cb(int p, int n, BN_GENCB *cb)
74819304Speter{
74919304Speter	union {
75019304Speter		void *dptr;
75119304Speter		void (*fptr)(int);
75219304Speter	} u;
75319304Speter
75419304Speter	UNUSED(n);
75519304Speter
75619304Speter	u.dptr = cb->arg;
75719304Speter	if (u.fptr != NULL)
75819304Speter		u.fptr(p);
75919304Speter	return (1);
760254225Speter}
761254225Speter#endif
762254225Speter
763254225Speterstatic isc_result_t
764254225Speteropensslrsa_generate(dst_key_t *key, int exp, void (*callback)(int)) {
765254225Speter#if OPENSSL_VERSION_NUMBER > 0x00908000L
76619304Speter	isc_result_t ret = DST_R_OPENSSLFAILURE;
76719304Speter	BN_GENCB cb;
76819304Speter	union {
76919304Speter		void *dptr;
77019304Speter		void (*fptr)(int);
77119304Speter	} u;
77219304Speter	RSA *rsa = RSA_new();
773254225Speter	BIGNUM *e = BN_new();
774254225Speter#if USE_EVP
775254225Speter	EVP_PKEY *pkey = EVP_PKEY_new();
776254225Speter#endif
77719304Speter
778254225Speter	if (rsa == NULL || e == NULL)
779254225Speter		goto err;
780254225Speter#if USE_EVP
781254225Speter	if (pkey == NULL)
782254225Speter		goto err;
78319304Speter	if (!EVP_PKEY_set1_RSA(pkey, rsa))
78419304Speter		goto err;
78519304Speter#endif
786254225Speter
787254225Speter	if (exp == 0) {
78819304Speter		/* RSA_F4 0x10001 */
789254225Speter		BN_set_bit(e, 0);
790254225Speter		BN_set_bit(e, 16);
79119304Speter	} else {
79219304Speter		/* (phased-out) F5 0x100000001 */
79319304Speter		BN_set_bit(e, 0);
79419304Speter		BN_set_bit(e, 32);
79519304Speter	}
79619304Speter
79719304Speter	if (callback == NULL) {
79819304Speter		BN_GENCB_set_old(&cb, NULL, NULL);
79919304Speter	} else {
80019304Speter		u.fptr = callback;
80119304Speter		BN_GENCB_set(&cb, &progress_cb, u.dptr);
80219304Speter	}
803254225Speter
80419304Speter	if (RSA_generate_key_ex(rsa, key->key_size, e, &cb)) {
80519304Speter		BN_free(e);
806254225Speter		SET_FLAGS(rsa);
80719304Speter#if USE_EVP
80819304Speter		key->keydata.pkey = pkey;
80919304Speter
81019304Speter		RSA_free(rsa);
81119304Speter#else
81219304Speter		key->keydata.rsa = rsa;
81319304Speter#endif
81419304Speter		return (ISC_R_SUCCESS);
81519304Speter	}
816254225Speter	ret = dst__openssl_toresult2("RSA_generate_key_ex",
817254225Speter				     DST_R_OPENSSLFAILURE);
818254225Speter
819254225Spetererr:
820254225Speter#if USE_EVP
82119304Speter	if (pkey != NULL)
822254225Speter		EVP_PKEY_free(pkey);
82319304Speter#endif
82419304Speter	if (e != NULL)
825254225Speter		BN_free(e);
826254225Speter	if (rsa != NULL)
82719304Speter		RSA_free(rsa);
82819304Speter	return (dst__openssl_toresult(ret));
829254225Speter#else
83019304Speter	RSA *rsa;
83119304Speter	unsigned long e;
83219304Speter#if USE_EVP
83319304Speter	EVP_PKEY *pkey = EVP_PKEY_new();
83419304Speter
83519304Speter	UNUSED(callback);
83619304Speter
83719304Speter	if (pkey == NULL)
83819304Speter		return (ISC_R_NOMEMORY);
839254225Speter#else
84019304Speter	UNUSED(callback);
841254225Speter#endif
842254225Speter
843254225Speter	if (exp == 0)
844254225Speter	       e = RSA_F4;
845254225Speter	else
84619304Speter	       e = 0x40000003;
84719304Speter	rsa = RSA_generate_key(key->key_size, e, NULL, NULL);
84819304Speter	if (rsa == NULL) {
84919304Speter#if USE_EVP
85019304Speter		EVP_PKEY_free(pkey);
85119304Speter#endif
85219304Speter		return (dst__openssl_toresult2("RSA_generate_key",
853254225Speter					       DST_R_OPENSSLFAILURE));
85419304Speter	}
85519304Speter	SET_FLAGS(rsa);
85619304Speter#if USE_EVP
85719304Speter	if (!EVP_PKEY_set1_RSA(pkey, rsa)) {
85819304Speter		EVP_PKEY_free(pkey);
85919304Speter		RSA_free(rsa);
86019304Speter		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
86119304Speter	}
86219304Speter	key->keydata.pkey = pkey;
86319304Speter	RSA_free(rsa);
86419304Speter#else
86519304Speter	key->keydata.rsa = rsa;
86619304Speter#endif
86719304Speter
86819304Speter	return (ISC_R_SUCCESS);
86919304Speter#endif
87019304Speter}
87119304Speter
87219304Speterstatic isc_boolean_t
87319304Speteropensslrsa_isprivate(const dst_key_t *key) {
87419304Speter#if USE_EVP
87519304Speter	RSA *rsa = EVP_PKEY_get1_RSA(key->keydata.pkey);
87619304Speter	INSIST(rsa != NULL);
87719304Speter	RSA_free(rsa);
87819304Speter	/* key->keydata.pkey still has a reference so rsa is still valid. */
87919304Speter#else
880254225Speter	RSA *rsa = key->keydata.rsa;
88119304Speter#endif
882254225Speter	if (rsa != NULL && (rsa->flags & RSA_FLAG_EXT_PKEY) != 0)
883254225Speter		return (ISC_TRUE);
884254225Speter	return (ISC_TF(rsa != NULL && rsa->d != NULL));
885254225Speter}
886254225Speter
887254225Speterstatic void
888254225Speteropensslrsa_destroy(dst_key_t *key) {
889254225Speter#if USE_EVP
890254225Speter	EVP_PKEY *pkey = key->keydata.pkey;
89119304Speter	EVP_PKEY_free(pkey);
89219304Speter	key->keydata.pkey = NULL;
89319304Speter#else
89419304Speter	RSA *rsa = key->keydata.rsa;
89519304Speter	RSA_free(rsa);
89619304Speter	key->keydata.rsa = NULL;
89719304Speter#endif
898254225Speter}
89919304Speter
90019304Speter
901254225Speterstatic isc_result_t
902254225Speteropensslrsa_todns(const dst_key_t *key, isc_buffer_t *data) {
90319304Speter	isc_region_t r;
90419304Speter	unsigned int e_bytes;
90519304Speter	unsigned int mod_bytes;
90619304Speter	isc_result_t ret;
907254225Speter	RSA *rsa;
908254225Speter#if USE_EVP
90919304Speter	EVP_PKEY *pkey;
910254225Speter#endif
911254225Speter
912254225Speter#if USE_EVP
913254225Speter	REQUIRE(key->keydata.pkey != NULL);
914254225Speter#else
915254225Speter	REQUIRE(key->keydata.rsa != NULL);
916254225Speter#endif
91719304Speter
918254225Speter#if USE_EVP
919254225Speter	pkey = key->keydata.pkey;
920254225Speter	rsa = EVP_PKEY_get1_RSA(pkey);
921254225Speter	if (rsa == NULL)
922254225Speter		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
923254225Speter#else
92419304Speter	rsa = key->keydata.rsa;
925254225Speter#endif
926254225Speter
927254225Speter	isc_buffer_availableregion(data, &r);
928254225Speter
92919304Speter	e_bytes = BN_num_bytes(rsa->e);
93019304Speter	mod_bytes = BN_num_bytes(rsa->n);
93119304Speter
93219304Speter	if (e_bytes < 256) {	/*%< key exponent is <= 2040 bits */
93319304Speter		if (r.length < 1)
934254225Speter			DST_RET(ISC_R_NOSPACE);
935254225Speter		isc_buffer_putuint8(data, (isc_uint8_t) e_bytes);
93619304Speter		isc_region_consume(&r, 1);
93719304Speter	} else {
93819304Speter		if (r.length < 3)
93919304Speter			DST_RET(ISC_R_NOSPACE);
94019304Speter		isc_buffer_putuint8(data, 0);
941254225Speter		isc_buffer_putuint16(data, (isc_uint16_t) e_bytes);
942254225Speter		isc_region_consume(&r, 3);
94319304Speter	}
94419304Speter
94519304Speter	if (r.length < e_bytes + mod_bytes)
94619304Speter		DST_RET(ISC_R_NOSPACE);
94719304Speter
94819304Speter	BN_bn2bin(rsa->e, r.base);
94919304Speter	isc_region_consume(&r, e_bytes);
95019304Speter	BN_bn2bin(rsa->n, r.base);
95119304Speter
95219304Speter	isc_buffer_add(data, e_bytes + mod_bytes);
95319304Speter
95419304Speter	ret = ISC_R_SUCCESS;
95519304Speter err:
95619304Speter#if USE_EVP
95719304Speter	if (rsa != NULL)
95819304Speter		RSA_free(rsa);
959254225Speter#endif
96019304Speter	return (ret);
961254225Speter}
962254225Speter
963254225Speterstatic isc_result_t
964254225Speteropensslrsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
965254225Speter	RSA *rsa;
966254225Speter	isc_region_t r;
967254225Speter	unsigned int e_bytes;
968254225Speter	unsigned int length;
969254225Speter#if USE_EVP
970254225Speter	EVP_PKEY *pkey;
971254225Speter#endif
972254225Speter
97319304Speter	isc_buffer_remainingregion(data, &r);
97419304Speter	if (r.length == 0)
975254225Speter		return (ISC_R_SUCCESS);
976254225Speter	length = r.length;
97719304Speter
97819304Speter	rsa = RSA_new();
97919304Speter	if (rsa == NULL)
98019304Speter		return (dst__openssl_toresult(ISC_R_NOMEMORY));
98119304Speter	SET_FLAGS(rsa);
98219304Speter
98319304Speter	if (r.length < 1) {
98419304Speter		RSA_free(rsa);
98519304Speter		return (DST_R_INVALIDPUBLICKEY);
98619304Speter	}
98719304Speter	e_bytes = *r.base;
988254225Speter	isc_region_consume(&r, 1);
98919304Speter
99019304Speter	if (e_bytes == 0) {
99119304Speter		if (r.length < 2) {
992254225Speter			RSA_free(rsa);
99319304Speter			return (DST_R_INVALIDPUBLICKEY);
99419304Speter		}
995254225Speter		e_bytes = (*r.base) << 8;
99619304Speter		isc_region_consume(&r, 1);
99719304Speter		e_bytes += *r.base;
99819304Speter		isc_region_consume(&r, 1);
999254225Speter	}
1000254225Speter
1001254225Speter	if (r.length < e_bytes) {
100219304Speter		RSA_free(rsa);
100319304Speter		return (DST_R_INVALIDPUBLICKEY);
100419304Speter	}
100519304Speter	rsa->e = BN_bin2bn(r.base, e_bytes, NULL);
100619304Speter	isc_region_consume(&r, e_bytes);
100719304Speter
100819304Speter	rsa->n = BN_bin2bn(r.base, r.length, NULL);
100919304Speter
101019304Speter	key->key_size = BN_num_bits(rsa->n);
101119304Speter
1012254225Speter	isc_buffer_forward(data, length);
101319304Speter
101419304Speter#if USE_EVP
101519304Speter	pkey = EVP_PKEY_new();
101619304Speter	if (pkey == NULL) {
101719304Speter		RSA_free(rsa);
101819304Speter		return (ISC_R_NOMEMORY);
101919304Speter	}
102019304Speter	if (!EVP_PKEY_set1_RSA(pkey, rsa)) {
102119304Speter		EVP_PKEY_free(pkey);
102219304Speter		RSA_free(rsa);
102319304Speter		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
102419304Speter	}
102519304Speter	key->keydata.pkey = pkey;
102619304Speter	RSA_free(rsa);
102719304Speter#else
102819304Speter	key->keydata.rsa = rsa;
102919304Speter#endif
103019304Speter
103119304Speter	return (ISC_R_SUCCESS);
103219304Speter}
103319304Speter
103419304Speterstatic isc_result_t
103519304Speteropensslrsa_tofile(const dst_key_t *key, const char *directory) {
103619304Speter	int i;
103719304Speter	RSA *rsa;
103819304Speter	dst_private_t priv;
103919304Speter	unsigned char *bufs[8];
104019304Speter	isc_result_t result;
104119304Speter
104219304Speter#if USE_EVP
104319304Speter	if (key->keydata.pkey == NULL)
104419304Speter		return (DST_R_NULLKEY);
104519304Speter	rsa = EVP_PKEY_get1_RSA(key->keydata.pkey);
104619304Speter	if (rsa == NULL)
104719304Speter		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
104819304Speter#else
104919304Speter	if (key->keydata.rsa == NULL)
105019304Speter		return (DST_R_NULLKEY);
105119304Speter	rsa = key->keydata.rsa;
105219304Speter#endif
105319304Speter	memset(bufs, 0, sizeof(bufs));
105419304Speter
105519304Speter	if (key->external) {
105619304Speter		priv.nelements = 0;
105719304Speter		result = dst__privstruct_writefile(key, &priv, directory);
105819304Speter		goto fail;
105919304Speter	}
106019304Speter
1061254225Speter	for (i = 0; i < 8; i++) {
106219304Speter		bufs[i] = isc_mem_get(key->mctx, BN_num_bytes(rsa->n));
106319304Speter		if (bufs[i] == NULL) {
106419304Speter			result = ISC_R_NOMEMORY;
106519304Speter			goto fail;
106619304Speter		}
106719304Speter	}
106819304Speter
106919304Speter	i = 0;
107019304Speter
107119304Speter	priv.elements[i].tag = TAG_RSA_MODULUS;
107219304Speter	priv.elements[i].length = BN_num_bytes(rsa->n);
107319304Speter	BN_bn2bin(rsa->n, bufs[i]);
107419304Speter	priv.elements[i].data = bufs[i];
107519304Speter	i++;
107619304Speter
107719304Speter	priv.elements[i].tag = TAG_RSA_PUBLICEXPONENT;
1078254225Speter	priv.elements[i].length = BN_num_bytes(rsa->e);
107919304Speter	BN_bn2bin(rsa->e, bufs[i]);
108019304Speter	priv.elements[i].data = bufs[i];
108119304Speter	i++;
108219304Speter
108319304Speter	if (rsa->d != NULL) {
108419304Speter		priv.elements[i].tag = TAG_RSA_PRIVATEEXPONENT;
108519304Speter		priv.elements[i].length = BN_num_bytes(rsa->d);
108619304Speter		BN_bn2bin(rsa->d, bufs[i]);
108719304Speter		priv.elements[i].data = bufs[i];
108819304Speter		i++;
108919304Speter	}
109019304Speter
109119304Speter	if (rsa->p != NULL) {
109219304Speter		priv.elements[i].tag = TAG_RSA_PRIME1;
109319304Speter		priv.elements[i].length = BN_num_bytes(rsa->p);
109419304Speter		BN_bn2bin(rsa->p, bufs[i]);
109519304Speter		priv.elements[i].data = bufs[i];
109619304Speter		i++;
109719304Speter	}
1098
1099	if (rsa->q != NULL) {
1100		priv.elements[i].tag = TAG_RSA_PRIME2;
1101		priv.elements[i].length = BN_num_bytes(rsa->q);
1102		BN_bn2bin(rsa->q, bufs[i]);
1103		priv.elements[i].data = bufs[i];
1104		i++;
1105	}
1106
1107	if (rsa->dmp1 != NULL) {
1108		priv.elements[i].tag = TAG_RSA_EXPONENT1;
1109		priv.elements[i].length = BN_num_bytes(rsa->dmp1);
1110		BN_bn2bin(rsa->dmp1, bufs[i]);
1111		priv.elements[i].data = bufs[i];
1112		i++;
1113	}
1114
1115	if (rsa->dmq1 != NULL) {
1116		priv.elements[i].tag = TAG_RSA_EXPONENT2;
1117		priv.elements[i].length = BN_num_bytes(rsa->dmq1);
1118		BN_bn2bin(rsa->dmq1, bufs[i]);
1119		priv.elements[i].data = bufs[i];
1120		i++;
1121	}
1122
1123	if (rsa->iqmp != NULL) {
1124		priv.elements[i].tag = TAG_RSA_COEFFICIENT;
1125		priv.elements[i].length = BN_num_bytes(rsa->iqmp);
1126		BN_bn2bin(rsa->iqmp, bufs[i]);
1127		priv.elements[i].data = bufs[i];
1128		i++;
1129	}
1130
1131	if (key->engine != NULL) {
1132		priv.elements[i].tag = TAG_RSA_ENGINE;
1133		priv.elements[i].length = strlen(key->engine) + 1;
1134		priv.elements[i].data = (unsigned char *)key->engine;
1135		i++;
1136	}
1137
1138	if (key->label != NULL) {
1139		priv.elements[i].tag = TAG_RSA_LABEL;
1140		priv.elements[i].length = strlen(key->label) + 1;
1141		priv.elements[i].data = (unsigned char *)key->label;
1142		i++;
1143	}
1144
1145
1146	priv.nelements = i;
1147	result = dst__privstruct_writefile(key, &priv, directory);
1148 fail:
1149#if USE_EVP
1150	RSA_free(rsa);
1151#endif
1152	for (i = 0; i < 8; i++) {
1153		if (bufs[i] == NULL)
1154			break;
1155		isc_mem_put(key->mctx, bufs[i], BN_num_bytes(rsa->n));
1156	}
1157	return (result);
1158}
1159
1160static isc_result_t
1161rsa_check(RSA *rsa, RSA *pub)
1162{
1163	/* Public parameters should be the same but if they are not set
1164	 * copy them from the public key. */
1165	if (pub != NULL) {
1166		if (rsa->n != NULL) {
1167			if (BN_cmp(rsa->n, pub->n) != 0)
1168				return (DST_R_INVALIDPRIVATEKEY);
1169		} else {
1170			rsa->n = pub->n;
1171			pub->n = NULL;
1172		}
1173		if (rsa->e != NULL) {
1174			if (BN_cmp(rsa->e, pub->e) != 0)
1175				return (DST_R_INVALIDPRIVATEKEY);
1176		} else {
1177			rsa->e = pub->e;
1178			pub->e = NULL;
1179		}
1180	}
1181	if (rsa->n == NULL || rsa->e == NULL)
1182		return (DST_R_INVALIDPRIVATEKEY);
1183	return (ISC_R_SUCCESS);
1184}
1185
1186static isc_result_t
1187opensslrsa_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
1188	dst_private_t priv;
1189	isc_result_t ret;
1190	int i;
1191	RSA *rsa = NULL, *pubrsa = NULL;
1192#ifdef USE_ENGINE
1193	ENGINE *e = NULL;
1194#endif
1195	isc_mem_t *mctx = key->mctx;
1196	const char *engine = NULL, *label = NULL;
1197#if defined(USE_ENGINE) || USE_EVP
1198	EVP_PKEY *pkey = NULL;
1199#endif
1200
1201#if USE_EVP
1202	if (pub != NULL && pub->keydata.pkey != NULL)
1203		pubrsa = EVP_PKEY_get1_RSA(pub->keydata.pkey);
1204#else
1205	if (pub != NULL && pub->keydata.rsa != NULL) {
1206		pubrsa = pub->keydata.rsa;
1207		pub->keydata.rsa = NULL;
1208	}
1209#endif
1210
1211	/* read private key file */
1212	ret = dst__privstruct_parse(key, DST_ALG_RSA, lexer, mctx, &priv);
1213	if (ret != ISC_R_SUCCESS)
1214		goto err;
1215
1216	if (key->external && priv.nelements != 0)
1217		DST_RET(DST_R_INVALIDPRIVATEKEY);
1218
1219	for (i = 0; i < priv.nelements; i++) {
1220		switch (priv.elements[i].tag) {
1221		case TAG_RSA_ENGINE:
1222			engine = (char *)priv.elements[i].data;
1223			break;
1224		case TAG_RSA_LABEL:
1225			label = (char *)priv.elements[i].data;
1226			break;
1227		default:
1228			break;
1229		}
1230	}
1231
1232	/*
1233	 * Is this key is stored in a HSM?
1234	 * See if we can fetch it.
1235	 */
1236	if (label != NULL) {
1237#ifdef USE_ENGINE
1238		if (engine == NULL)
1239			DST_RET(DST_R_NOENGINE);
1240		e = dst__openssl_getengine(engine);
1241		if (e == NULL)
1242			DST_RET(DST_R_NOENGINE);
1243		pkey = ENGINE_load_private_key(e, label, NULL, NULL);
1244		if (pkey == NULL)
1245			DST_RET(dst__openssl_toresult2(
1246					"ENGINE_load_private_key",
1247					ISC_R_NOTFOUND));
1248		key->engine = isc_mem_strdup(key->mctx, engine);
1249		if (key->engine == NULL)
1250			DST_RET(ISC_R_NOMEMORY);
1251		key->label = isc_mem_strdup(key->mctx, label);
1252		if (key->label == NULL)
1253			DST_RET(ISC_R_NOMEMORY);
1254		rsa = EVP_PKEY_get1_RSA(pkey);
1255		if (rsa == NULL)
1256			DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1257		if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
1258			DST_RET(DST_R_INVALIDPRIVATEKEY);
1259		if (BN_num_bits(rsa->e) > RSA_MAX_PUBEXP_BITS)
1260			DST_RET(ISC_R_RANGE);
1261		if (pubrsa != NULL)
1262			RSA_free(pubrsa);
1263		key->key_size = EVP_PKEY_bits(pkey);
1264#if USE_EVP
1265		key->keydata.pkey = pkey;
1266		RSA_free(rsa);
1267#else
1268		key->keydata.rsa = rsa;
1269		EVP_PKEY_free(pkey);
1270#endif
1271		dst__privstruct_free(&priv, mctx);
1272		memset(&priv, 0, sizeof(priv));
1273		return (ISC_R_SUCCESS);
1274#else
1275		DST_RET(DST_R_NOENGINE);
1276#endif
1277	}
1278
1279	rsa = RSA_new();
1280	if (rsa == NULL)
1281		DST_RET(ISC_R_NOMEMORY);
1282	SET_FLAGS(rsa);
1283
1284#if USE_EVP
1285	pkey = EVP_PKEY_new();
1286	if (pkey == NULL)
1287		DST_RET(ISC_R_NOMEMORY);
1288	if (!EVP_PKEY_set1_RSA(pkey, rsa))
1289		DST_RET(ISC_R_FAILURE);
1290	key->keydata.pkey = pkey;
1291#else
1292	key->keydata.rsa = rsa;
1293#endif
1294
1295	for (i = 0; i < priv.nelements; i++) {
1296		BIGNUM *bn;
1297		switch (priv.elements[i].tag) {
1298		case TAG_RSA_ENGINE:
1299			continue;
1300		case TAG_RSA_LABEL:
1301			continue;
1302		case TAG_RSA_PIN:
1303			continue;
1304		default:
1305			bn = BN_bin2bn(priv.elements[i].data,
1306				       priv.elements[i].length, NULL);
1307			if (bn == NULL)
1308				DST_RET(ISC_R_NOMEMORY);
1309		}
1310
1311		switch (priv.elements[i].tag) {
1312			case TAG_RSA_MODULUS:
1313				rsa->n = bn;
1314				break;
1315			case TAG_RSA_PUBLICEXPONENT:
1316				rsa->e = bn;
1317				break;
1318			case TAG_RSA_PRIVATEEXPONENT:
1319				rsa->d = bn;
1320				break;
1321			case TAG_RSA_PRIME1:
1322				rsa->p = bn;
1323				break;
1324			case TAG_RSA_PRIME2:
1325				rsa->q = bn;
1326				break;
1327			case TAG_RSA_EXPONENT1:
1328				rsa->dmp1 = bn;
1329				break;
1330			case TAG_RSA_EXPONENT2:
1331				rsa->dmq1 = bn;
1332				break;
1333			case TAG_RSA_COEFFICIENT:
1334				rsa->iqmp = bn;
1335				break;
1336		}
1337	}
1338	dst__privstruct_free(&priv, mctx);
1339	memset(&priv, 0, sizeof(priv));
1340
1341	if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
1342		DST_RET(DST_R_INVALIDPRIVATEKEY);
1343	if (!key->external) {
1344		if (BN_num_bits(rsa->e) > RSA_MAX_PUBEXP_BITS)
1345			DST_RET(ISC_R_RANGE);
1346	}
1347	key->key_size = BN_num_bits(rsa->n);
1348	if (pubrsa != NULL)
1349		RSA_free(pubrsa);
1350#if USE_EVP
1351	RSA_free(rsa);
1352#endif
1353
1354	return (ISC_R_SUCCESS);
1355
1356 err:
1357#if USE_EVP
1358	if (pkey != NULL)
1359		EVP_PKEY_free(pkey);
1360#endif
1361	if (rsa != NULL)
1362		RSA_free(rsa);
1363	if (pubrsa != NULL)
1364		RSA_free(pubrsa);
1365	key->keydata.generic = NULL;
1366	dst__privstruct_free(&priv, mctx);
1367	memset(&priv, 0, sizeof(priv));
1368	return (ret);
1369}
1370
1371static isc_result_t
1372opensslrsa_fromlabel(dst_key_t *key, const char *engine, const char *label,
1373		     const char *pin)
1374{
1375#ifdef USE_ENGINE
1376	ENGINE *e = NULL;
1377	isc_result_t ret;
1378	EVP_PKEY *pkey = NULL;
1379	RSA *rsa = NULL, *pubrsa = NULL;
1380	char *colon;
1381
1382	UNUSED(pin);
1383
1384	if (engine == NULL)
1385		DST_RET(DST_R_NOENGINE);
1386	e = dst__openssl_getengine(engine);
1387	if (e == NULL)
1388		DST_RET(DST_R_NOENGINE);
1389	pkey = ENGINE_load_public_key(e, label, NULL, NULL);
1390	if (pkey != NULL) {
1391		pubrsa = EVP_PKEY_get1_RSA(pkey);
1392		EVP_PKEY_free(pkey);
1393		if (pubrsa == NULL)
1394			DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1395	}
1396	pkey = ENGINE_load_private_key(e, label, NULL, NULL);
1397	if (pkey == NULL)
1398		DST_RET(dst__openssl_toresult2("ENGINE_load_private_key",
1399					       ISC_R_NOTFOUND));
1400	if (engine != NULL) {
1401		key->engine = isc_mem_strdup(key->mctx, engine);
1402		if (key->engine == NULL)
1403			DST_RET(ISC_R_NOMEMORY);
1404	} else {
1405		key->engine = isc_mem_strdup(key->mctx, label);
1406		if (key->engine == NULL)
1407			DST_RET(ISC_R_NOMEMORY);
1408		colon = strchr(key->engine, ':');
1409		if (colon != NULL)
1410			*colon = '\0';
1411	}
1412	key->label = isc_mem_strdup(key->mctx, label);
1413	if (key->label == NULL)
1414		DST_RET(ISC_R_NOMEMORY);
1415	rsa = EVP_PKEY_get1_RSA(pkey);
1416	if (rsa == NULL)
1417		DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1418	if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
1419		DST_RET(DST_R_INVALIDPRIVATEKEY);
1420	if (BN_num_bits(rsa->e) > RSA_MAX_PUBEXP_BITS)
1421		DST_RET(ISC_R_RANGE);
1422	if (pubrsa != NULL)
1423		RSA_free(pubrsa);
1424	key->key_size = EVP_PKEY_bits(pkey);
1425#if USE_EVP
1426	key->keydata.pkey = pkey;
1427	RSA_free(rsa);
1428#else
1429	key->keydata.rsa = rsa;
1430	EVP_PKEY_free(pkey);
1431#endif
1432	return (ISC_R_SUCCESS);
1433
1434 err:
1435	if (rsa != NULL)
1436		RSA_free(rsa);
1437	if (pubrsa != NULL)
1438		RSA_free(pubrsa);
1439	if (pkey != NULL)
1440		EVP_PKEY_free(pkey);
1441	return (ret);
1442#else
1443	UNUSED(key);
1444	UNUSED(engine);
1445	UNUSED(label);
1446	UNUSED(pin);
1447	return(DST_R_NOENGINE);
1448#endif
1449}
1450
1451static dst_func_t opensslrsa_functions = {
1452	opensslrsa_createctx,
1453	opensslrsa_destroyctx,
1454	opensslrsa_adddata,
1455	opensslrsa_sign,
1456	opensslrsa_verify,
1457	opensslrsa_verify2,
1458	NULL, /*%< computesecret */
1459	opensslrsa_compare,
1460	NULL, /*%< paramcompare */
1461	opensslrsa_generate,
1462	opensslrsa_isprivate,
1463	opensslrsa_destroy,
1464	opensslrsa_todns,
1465	opensslrsa_fromdns,
1466	opensslrsa_tofile,
1467	opensslrsa_parse,
1468	NULL, /*%< cleanup */
1469	opensslrsa_fromlabel,
1470	NULL, /*%< dump */
1471	NULL, /*%< restore */
1472};
1473
1474isc_result_t
1475dst__opensslrsa_init(dst_func_t **funcp, unsigned char algorithm) {
1476	REQUIRE(funcp != NULL);
1477
1478	if (*funcp == NULL) {
1479		switch (algorithm) {
1480		case DST_ALG_RSASHA256:
1481#if defined(HAVE_EVP_SHA256) || !USE_EVP
1482			*funcp = &opensslrsa_functions;
1483#endif
1484			break;
1485		case DST_ALG_RSASHA512:
1486#if defined(HAVE_EVP_SHA512) || !USE_EVP
1487			*funcp = &opensslrsa_functions;
1488#endif
1489			break;
1490		default:
1491			*funcp = &opensslrsa_functions;
1492			break;
1493		}
1494	}
1495	return (ISC_R_SUCCESS);
1496}
1497
1498#else /* OPENSSL */
1499
1500#include <isc/util.h>
1501
1502EMPTY_TRANSLATION_UNIT
1503
1504#endif /* OPENSSL */
1505/*! \file */
1506