1/* $OpenBSD: rsa_pmeth.c,v 1.40 2023/12/28 21:59:07 tb Exp $ */
2/* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
3 * project 2006.
4 */
5/* ====================================================================
6 * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 *
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice, this list of conditions and the following disclaimer.
14 *
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in
17 *    the documentation and/or other materials provided with the
18 *    distribution.
19 *
20 * 3. All advertising materials mentioning features or use of this
21 *    software must display the following acknowledgment:
22 *    "This product includes software developed by the OpenSSL Project
23 *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
24 *
25 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
26 *    endorse or promote products derived from this software without
27 *    prior written permission. For written permission, please contact
28 *    licensing@OpenSSL.org.
29 *
30 * 5. Products derived from this software may not be called "OpenSSL"
31 *    nor may "OpenSSL" appear in their names without prior written
32 *    permission of the OpenSSL Project.
33 *
34 * 6. Redistributions of any form whatsoever must retain the following
35 *    acknowledgment:
36 *    "This product includes software developed by the OpenSSL Project
37 *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
38 *
39 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
40 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
41 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
42 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
43 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
44 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
45 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
46 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
47 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
48 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
49 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
50 * OF THE POSSIBILITY OF SUCH DAMAGE.
51 * ====================================================================
52 *
53 * This product includes cryptographic software written by Eric Young
54 * (eay@cryptsoft.com).  This product includes software written by Tim
55 * Hudson (tjh@cryptsoft.com).
56 *
57 */
58
59#include <limits.h>
60#include <stdio.h>
61#include <string.h>
62
63#include <openssl/opensslconf.h>
64
65#include <openssl/asn1t.h>
66#include <openssl/bn.h>
67#include <openssl/err.h>
68#include <openssl/evp.h>
69#include <openssl/rsa.h>
70#include <openssl/x509.h>
71#include <openssl/x509v3.h>
72
73#include "bn_local.h"
74#include "evp_local.h"
75#include "rsa_local.h"
76
77/* RSA pkey context structure */
78
79typedef struct {
80	/* Key gen parameters */
81	int nbits;
82	BIGNUM *pub_exp;
83	/* Keygen callback info */
84	int gentmp[2];
85	/* RSA padding mode */
86	int pad_mode;
87	/* message digest */
88	const EVP_MD *md;
89	/* message digest for MGF1 */
90	const EVP_MD *mgf1md;
91	/* PSS salt length */
92	int saltlen;
93	/* Minimum salt length or -1 if no PSS parameter restriction */
94	int min_saltlen;
95	/* Temp buffer */
96	unsigned char *tbuf;
97	/* OAEP label */
98	unsigned char *oaep_label;
99	size_t oaep_labellen;
100} RSA_PKEY_CTX;
101
102/* True if PSS parameters are restricted */
103#define rsa_pss_restricted(rctx) (rctx->min_saltlen != -1)
104
105static int
106pkey_rsa_init(EVP_PKEY_CTX *ctx)
107{
108	RSA_PKEY_CTX *rctx;
109
110	if ((rctx = calloc(1, sizeof(RSA_PKEY_CTX))) == NULL)
111		return 0;
112
113	rctx->nbits = 2048;
114
115	if (ctx->pmeth->pkey_id == EVP_PKEY_RSA_PSS)
116		rctx->pad_mode = RSA_PKCS1_PSS_PADDING;
117	else
118		rctx->pad_mode = RSA_PKCS1_PADDING;
119
120	/* Maximum for sign, auto for verify */
121	rctx->saltlen = RSA_PSS_SALTLEN_AUTO;
122	rctx->min_saltlen = -1;
123
124	ctx->data = rctx;
125	ctx->keygen_info = rctx->gentmp;
126	ctx->keygen_info_count = 2;
127
128	return 1;
129}
130
131static int
132pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
133{
134	RSA_PKEY_CTX *dctx, *sctx;
135
136	if (!pkey_rsa_init(dst))
137		return 0;
138
139	sctx = src->data;
140	dctx = dst->data;
141	dctx->nbits = sctx->nbits;
142	if (sctx->pub_exp != NULL) {
143		BN_free(dctx->pub_exp);
144		if ((dctx->pub_exp = BN_dup(sctx->pub_exp)) == NULL)
145			return 0;
146	}
147	dctx->pad_mode = sctx->pad_mode;
148	dctx->md = sctx->md;
149	dctx->mgf1md = sctx->mgf1md;
150	if (sctx->oaep_label != NULL) {
151		free(dctx->oaep_label);
152		if ((dctx->oaep_label = calloc(1, sctx->oaep_labellen)) == NULL)
153			return 0;
154		memcpy(dctx->oaep_label, sctx->oaep_label, sctx->oaep_labellen);
155		dctx->oaep_labellen = sctx->oaep_labellen;
156	}
157
158	return 1;
159}
160
161static int
162setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
163{
164	if (ctx->tbuf != NULL)
165		return 1;
166	if ((ctx->tbuf = calloc(1, EVP_PKEY_size(pk->pkey))) == NULL) {
167		RSAerror(ERR_R_MALLOC_FAILURE);
168		return 0;
169	}
170	return 1;
171}
172
173static void
174pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
175{
176	RSA_PKEY_CTX *rctx = ctx->data;
177
178	if (rctx) {
179		BN_free(rctx->pub_exp);
180		free(rctx->tbuf);
181		free(rctx->oaep_label);
182		free(rctx);
183	}
184}
185
186static int
187pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
188    const unsigned char *tbs, size_t tbslen)
189{
190	int ret;
191	RSA_PKEY_CTX *rctx = ctx->data;
192	RSA *rsa = ctx->pkey->pkey.rsa;
193
194	if (rctx->md) {
195		if (tbslen != (size_t)EVP_MD_size(rctx->md)) {
196			RSAerror(RSA_R_INVALID_DIGEST_LENGTH);
197			return -1;
198		}
199
200		if (rctx->pad_mode == RSA_X931_PADDING) {
201			if ((size_t)EVP_PKEY_size(ctx->pkey) < tbslen + 1) {
202				RSAerror(RSA_R_KEY_SIZE_TOO_SMALL);
203				return -1;
204			}
205			if (!setup_tbuf(rctx, ctx)) {
206				RSAerror(ERR_R_MALLOC_FAILURE);
207				return -1;
208			}
209			memcpy(rctx->tbuf, tbs, tbslen);
210			rctx->tbuf[tbslen] =
211			    RSA_X931_hash_id(EVP_MD_type(rctx->md));
212			ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf, sig,
213			    rsa, RSA_X931_PADDING);
214		} else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
215			unsigned int sltmp;
216
217			ret = RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig,
218			    &sltmp, rsa);
219			if (ret <= 0)
220				return ret;
221			ret = sltmp;
222		} else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
223			if (!setup_tbuf(rctx, ctx))
224				return -1;
225			if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa, rctx->tbuf,
226			    tbs, rctx->md, rctx->mgf1md, rctx->saltlen))
227				return -1;
228			ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
229			    sig, rsa, RSA_NO_PADDING);
230		} else {
231			return -1;
232		}
233	} else {
234		ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
235		    rctx->pad_mode);
236	}
237	if (ret < 0)
238		return ret;
239	*siglen = ret;
240	return 1;
241}
242
243static int
244pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx, unsigned char *rout, size_t *routlen,
245    const unsigned char *sig, size_t siglen)
246{
247	int ret;
248	RSA_PKEY_CTX *rctx = ctx->data;
249
250	if (rctx->md) {
251		if (rctx->pad_mode == RSA_X931_PADDING) {
252			if (!setup_tbuf(rctx, ctx))
253				return -1;
254			ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
255			    ctx->pkey->pkey.rsa, RSA_X931_PADDING);
256			if (ret < 1)
257				return 0;
258			ret--;
259			if (rctx->tbuf[ret] !=
260			    RSA_X931_hash_id(EVP_MD_type(rctx->md))) {
261				RSAerror(RSA_R_ALGORITHM_MISMATCH);
262				return 0;
263			}
264			if (ret != EVP_MD_size(rctx->md)) {
265				RSAerror(RSA_R_INVALID_DIGEST_LENGTH);
266				return 0;
267			}
268			if (rout)
269				memcpy(rout, rctx->tbuf, ret);
270		} else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
271			size_t sltmp;
272
273			ret = int_rsa_verify(EVP_MD_type(rctx->md), NULL, 0,
274			    rout, &sltmp, sig, siglen, ctx->pkey->pkey.rsa);
275			if (ret <= 0)
276				return 0;
277			ret = sltmp;
278		} else {
279			return -1;
280		}
281	} else {
282		ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
283		    rctx->pad_mode);
284	}
285	if (ret < 0)
286		return ret;
287	*routlen = ret;
288	return 1;
289}
290
291static int
292pkey_rsa_verify(EVP_PKEY_CTX *ctx, const unsigned char *sig, size_t siglen,
293    const unsigned char *tbs, size_t tbslen)
294{
295	RSA_PKEY_CTX *rctx = ctx->data;
296	RSA *rsa = ctx->pkey->pkey.rsa;
297	size_t rslen;
298
299	if (rctx->md) {
300		if (rctx->pad_mode == RSA_PKCS1_PADDING)
301			return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
302			    sig, siglen, rsa);
303		if (tbslen != (size_t)EVP_MD_size(rctx->md)) {
304			RSAerror(RSA_R_INVALID_DIGEST_LENGTH);
305			return -1;
306		}
307		if (rctx->pad_mode == RSA_X931_PADDING) {
308			if (pkey_rsa_verifyrecover(ctx, NULL, &rslen, sig,
309			    siglen) <= 0)
310				return 0;
311		} else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
312			int ret;
313
314			if (!setup_tbuf(rctx, ctx))
315				return -1;
316			ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
317			    rsa, RSA_NO_PADDING);
318			if (ret <= 0)
319				return 0;
320			ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, rctx->md,
321			    rctx->mgf1md, rctx->tbuf, rctx->saltlen);
322			if (ret <= 0)
323				return 0;
324			return 1;
325		} else {
326			return -1;
327		}
328	} else {
329		int ret;
330
331		if (!setup_tbuf(rctx, ctx))
332			return -1;
333
334		if ((ret = RSA_public_decrypt(siglen, sig, rctx->tbuf, rsa,
335		    rctx->pad_mode)) <= 0)
336			return 0;
337
338		rslen = ret;
339	}
340
341	if (rslen != tbslen || timingsafe_bcmp(tbs, rctx->tbuf, rslen))
342		return 0;
343
344	return 1;
345}
346
347static int
348pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen,
349    const unsigned char *in, size_t inlen)
350{
351	RSA_PKEY_CTX *rctx = ctx->data;
352	int ret;
353
354	if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
355		int klen = RSA_size(ctx->pkey->pkey.rsa);
356		if (!setup_tbuf(rctx, ctx))
357			return -1;
358		if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen,
359		    in, inlen, rctx->oaep_label, rctx->oaep_labellen,
360		    rctx->md, rctx->mgf1md))
361			return -1;
362		ret = RSA_public_encrypt(klen, rctx->tbuf, out,
363		    ctx->pkey->pkey.rsa, RSA_NO_PADDING);
364	} else {
365		ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
366		    rctx->pad_mode);
367	}
368	if (ret < 0)
369		return ret;
370	*outlen = ret;
371	return 1;
372}
373
374static int
375pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen,
376    const unsigned char *in, size_t inlen)
377{
378	int ret;
379	RSA_PKEY_CTX *rctx = ctx->data;
380
381	if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
382		if (!setup_tbuf(rctx, ctx))
383			return -1;
384		ret = RSA_private_decrypt(inlen, in, rctx->tbuf,
385		    ctx->pkey->pkey.rsa, RSA_NO_PADDING);
386		if (ret <= 0)
387			return ret;
388		ret = RSA_padding_check_PKCS1_OAEP_mgf1(out, ret, rctx->tbuf,
389		    ret, ret, rctx->oaep_label, rctx->oaep_labellen, rctx->md,
390		    rctx->mgf1md);
391	} else {
392		ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
393		    rctx->pad_mode);
394	}
395	if (ret < 0)
396		return ret;
397	*outlen = ret;
398	return 1;
399}
400
401static int
402check_padding_md(const EVP_MD *md, int padding)
403{
404	if (md == NULL)
405		return 1;
406
407	if (padding == RSA_NO_PADDING) {
408		RSAerror(RSA_R_INVALID_PADDING_MODE);
409		return 0;
410	}
411
412	if (padding == RSA_X931_PADDING) {
413		if (RSA_X931_hash_id(EVP_MD_type(md)) == -1) {
414			RSAerror(RSA_R_INVALID_X931_DIGEST);
415			return 0;
416		}
417	} else {
418		/* List of all supported RSA digests. */
419		/* RFC 8017 and NIST CSOR. */
420		switch(EVP_MD_type(md)) {
421		case NID_sha1:
422		case NID_sha224:
423		case NID_sha256:
424		case NID_sha384:
425		case NID_sha512:
426		case NID_sha512_224:
427		case NID_sha512_256:
428		case NID_sha3_224:
429		case NID_sha3_256:
430		case NID_sha3_384:
431		case NID_sha3_512:
432		case NID_md5:
433		case NID_md5_sha1:
434		case NID_md4:
435		case NID_ripemd160:
436			return 1;
437
438		default:
439			RSAerror(RSA_R_INVALID_DIGEST);
440			return 0;
441		}
442	}
443
444	return 1;
445}
446
447static int
448pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
449{
450	RSA_PKEY_CTX *rctx = ctx->data;
451
452	switch (type) {
453	case EVP_PKEY_CTRL_RSA_PADDING:
454		if (p1 >= RSA_PKCS1_PADDING && p1 <= RSA_PKCS1_PSS_PADDING) {
455			if (!check_padding_md(rctx->md, p1))
456				return 0;
457			if (p1 == RSA_PKCS1_PSS_PADDING) {
458				if (!(ctx->operation &
459				    (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
460					goto bad_pad;
461				if (!rctx->md)
462					rctx->md = EVP_sha1();
463			} else if (ctx->pmeth->pkey_id == EVP_PKEY_RSA_PSS) {
464				goto bad_pad;
465			}
466			if (p1 == RSA_PKCS1_OAEP_PADDING) {
467				if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
468					goto bad_pad;
469				if (!rctx->md)
470					rctx->md = EVP_sha1();
471			}
472			rctx->pad_mode = p1;
473			return 1;
474		}
475 bad_pad:
476		RSAerror(RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
477		return -2;
478
479	case EVP_PKEY_CTRL_GET_RSA_PADDING:
480		*(int *)p2 = rctx->pad_mode;
481		return 1;
482
483	case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
484	case EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN:
485		if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) {
486			RSAerror(RSA_R_INVALID_PSS_SALTLEN);
487			return -2;
488		}
489		if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN) {
490			*(int *)p2 = rctx->saltlen;
491		} else {
492			if (p1 < RSA_PSS_SALTLEN_MAX)
493				return -2;
494			if (rsa_pss_restricted(rctx)) {
495				if (p1 == RSA_PSS_SALTLEN_AUTO &&
496				    ctx->operation == EVP_PKEY_OP_VERIFY) {
497					RSAerror(RSA_R_INVALID_PSS_SALTLEN);
498					return -2;
499				}
500				if ((p1 == RSA_PSS_SALTLEN_DIGEST &&
501				    rctx->min_saltlen > EVP_MD_size(rctx->md)) ||
502				    (p1 >= 0 && p1 < rctx->min_saltlen)) {
503					RSAerror(RSA_R_PSS_SALTLEN_TOO_SMALL);
504					return 0;
505				}
506			}
507			rctx->saltlen = p1;
508		}
509		return 1;
510
511	case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
512		if (p1 < RSA_MIN_MODULUS_BITS) {
513			RSAerror(RSA_R_KEY_SIZE_TOO_SMALL);
514			return -2;
515		}
516		rctx->nbits = p1;
517		return 1;
518
519	case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
520		if (p2 == NULL || !BN_is_odd((BIGNUM *)p2) ||
521		    BN_is_one((BIGNUM *)p2)) {
522			RSAerror(RSA_R_BAD_E_VALUE);
523			return -2;
524		}
525		BN_free(rctx->pub_exp);
526		rctx->pub_exp = p2;
527		return 1;
528
529	case EVP_PKEY_CTRL_RSA_OAEP_MD:
530	case EVP_PKEY_CTRL_GET_RSA_OAEP_MD:
531		if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
532			RSAerror(RSA_R_INVALID_PADDING_MODE);
533			return -2;
534		}
535		if (type == EVP_PKEY_CTRL_GET_RSA_OAEP_MD)
536			*(const EVP_MD **)p2 = rctx->md;
537		else
538			rctx->md = p2;
539		return 1;
540
541	case EVP_PKEY_CTRL_MD:
542		if (!check_padding_md(p2, rctx->pad_mode))
543			return 0;
544		if (rsa_pss_restricted(rctx)) {
545			if (EVP_MD_type(rctx->md) == EVP_MD_type(p2))
546				return 1;
547			RSAerror(RSA_R_DIGEST_NOT_ALLOWED);
548			return 0;
549		}
550		rctx->md = p2;
551		return 1;
552
553	case EVP_PKEY_CTRL_GET_MD:
554		*(const EVP_MD **)p2 = rctx->md;
555		return 1;
556
557	case EVP_PKEY_CTRL_RSA_MGF1_MD:
558	case EVP_PKEY_CTRL_GET_RSA_MGF1_MD:
559		if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING &&
560		    rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
561			RSAerror(RSA_R_INVALID_MGF1_MD);
562			return -2;
563		}
564		if (type == EVP_PKEY_CTRL_GET_RSA_MGF1_MD) {
565			if (rctx->mgf1md)
566				*(const EVP_MD **)p2 = rctx->mgf1md;
567			else
568				*(const EVP_MD **)p2 = rctx->md;
569		} else {
570			if (rsa_pss_restricted(rctx)) {
571				if (EVP_MD_type(rctx->mgf1md) == EVP_MD_type(p2))
572					return 1;
573				RSAerror(RSA_R_MGF1_DIGEST_NOT_ALLOWED);
574				return 0;
575			}
576			rctx->mgf1md = p2;
577		}
578		return 1;
579
580	case EVP_PKEY_CTRL_RSA_OAEP_LABEL:
581		if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
582			RSAerror(RSA_R_INVALID_PADDING_MODE);
583			return -2;
584		}
585		free(rctx->oaep_label);
586		if (p2 != NULL && p1 > 0) {
587			rctx->oaep_label = p2;
588			rctx->oaep_labellen = p1;
589		} else {
590			rctx->oaep_label = NULL;
591			rctx->oaep_labellen = 0;
592		}
593		return 1;
594
595	case EVP_PKEY_CTRL_GET_RSA_OAEP_LABEL:
596		if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
597			RSAerror(RSA_R_INVALID_PADDING_MODE);
598			return -2;
599		}
600		*(unsigned char **)p2 = rctx->oaep_label;
601		return rctx->oaep_labellen;
602
603	case EVP_PKEY_CTRL_DIGESTINIT:
604	case EVP_PKEY_CTRL_PKCS7_SIGN:
605#ifndef OPENSSL_NO_CMS
606	case EVP_PKEY_CTRL_CMS_SIGN:
607#endif
608		return 1;
609
610	case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
611	case EVP_PKEY_CTRL_PKCS7_DECRYPT:
612#ifndef OPENSSL_NO_CMS
613	case EVP_PKEY_CTRL_CMS_DECRYPT:
614	case EVP_PKEY_CTRL_CMS_ENCRYPT:
615#endif
616		if (ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
617			return 1;
618
619	/* fall through */
620	case EVP_PKEY_CTRL_PEER_KEY:
621		RSAerror(RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
622		return -2;
623
624	default:
625		return -2;
626
627	}
628}
629
630static int
631pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx, const char *type, const char *value)
632{
633	if (!value) {
634		RSAerror(RSA_R_VALUE_MISSING);
635		return 0;
636	}
637	if (!strcmp(type, "rsa_padding_mode")) {
638		int pm;
639		if (!strcmp(value, "pkcs1"))
640			pm = RSA_PKCS1_PADDING;
641		else if (!strcmp(value, "none"))
642			pm = RSA_NO_PADDING;
643		else if (!strcmp(value, "oeap"))
644			pm = RSA_PKCS1_OAEP_PADDING;
645		else if (!strcmp(value, "oaep"))
646			pm = RSA_PKCS1_OAEP_PADDING;
647		else if (!strcmp(value, "x931"))
648			pm = RSA_X931_PADDING;
649		else if (!strcmp(value, "pss"))
650			pm = RSA_PKCS1_PSS_PADDING;
651		else {
652			RSAerror(RSA_R_UNKNOWN_PADDING_TYPE);
653			return -2;
654		}
655		return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
656	}
657
658	if (strcmp(type, "rsa_pss_saltlen") == 0) {
659		int saltlen;
660
661		if (!strcmp(value, "digest"))
662			saltlen = RSA_PSS_SALTLEN_DIGEST;
663		else if (!strcmp(value, "max"))
664			saltlen = RSA_PSS_SALTLEN_MAX;
665		else if (!strcmp(value, "auto"))
666			saltlen = RSA_PSS_SALTLEN_AUTO;
667		else
668			saltlen = atoi(value);
669		return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
670	}
671
672	if (strcmp(type, "rsa_keygen_bits") == 0) {
673		int nbits = atoi(value);
674
675		return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
676	}
677
678	if (strcmp(type, "rsa_keygen_pubexp") == 0) {
679		BIGNUM *pubexp = NULL;
680		int ret;
681
682		if (!BN_asc2bn(&pubexp, value))
683			return 0;
684		ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
685		if (ret <= 0)
686			BN_free(pubexp);
687		return ret;
688	}
689
690	if (strcmp(type, "rsa_mgf1_md") == 0)
691		return EVP_PKEY_CTX_md(ctx,
692		    EVP_PKEY_OP_TYPE_SIG | EVP_PKEY_OP_TYPE_CRYPT,
693		    EVP_PKEY_CTRL_RSA_MGF1_MD, value);
694
695	if (ctx->pmeth->pkey_id == EVP_PKEY_RSA_PSS) {
696		if (strcmp(type, "rsa_pss_keygen_mgf1_md") == 0)
697			return EVP_PKEY_CTX_md(ctx, EVP_PKEY_OP_KEYGEN,
698			    EVP_PKEY_CTRL_RSA_MGF1_MD, value);
699
700		if (strcmp(type, "rsa_pss_keygen_md") == 0)
701			return EVP_PKEY_CTX_md(ctx, EVP_PKEY_OP_KEYGEN,
702			    EVP_PKEY_CTRL_MD, value);
703
704		if (strcmp(type, "rsa_pss_keygen_saltlen") == 0) {
705			int saltlen = atoi(value);
706
707			return EVP_PKEY_CTX_set_rsa_pss_keygen_saltlen(ctx, saltlen);
708		}
709	}
710
711	if (strcmp(type, "rsa_oaep_md") == 0)
712		return EVP_PKEY_CTX_md(ctx, EVP_PKEY_OP_TYPE_CRYPT,
713		    EVP_PKEY_CTRL_RSA_OAEP_MD, value);
714
715	if (strcmp(type, "rsa_oaep_label") == 0) {
716		unsigned char *lab;
717		long lablen;
718		int ret;
719
720		if ((lab = string_to_hex(value, &lablen)) == NULL)
721			return 0;
722		ret = EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, lab, lablen);
723		if (ret <= 0)
724			free(lab);
725
726		return ret;
727	}
728
729	return -2;
730}
731
732/* Set PSS parameters when generating a key, if necessary. */
733static int
734rsa_set_pss_param(RSA *rsa, EVP_PKEY_CTX *ctx)
735{
736	RSA_PKEY_CTX *rctx = ctx->data;
737
738	if (ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
739		return 1;
740
741	/* If all parameters are default values then do not set PSS. */
742	if (rctx->md == NULL && rctx->mgf1md == NULL &&
743	    rctx->saltlen == RSA_PSS_SALTLEN_AUTO)
744		return 1;
745
746	rsa->pss = rsa_pss_params_create(rctx->md, rctx->mgf1md,
747	    rctx->saltlen == RSA_PSS_SALTLEN_AUTO ? 0 : rctx->saltlen);
748	if (rsa->pss == NULL)
749		return 0;
750
751	return 1;
752}
753
754static int
755pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
756{
757	RSA *rsa = NULL;
758	RSA_PKEY_CTX *rctx = ctx->data;
759	BN_GENCB *pcb = NULL;
760	BN_GENCB cb = {0};
761	int ret = 0;
762
763	if (rctx->pub_exp == NULL) {
764		if ((rctx->pub_exp = BN_new()) == NULL)
765			goto err;
766		if (!BN_set_word(rctx->pub_exp, RSA_F4))
767			goto err;
768	}
769
770	if ((rsa = RSA_new()) == NULL)
771		goto err;
772	if (ctx->pkey_gencb != NULL) {
773		pcb = &cb;
774		evp_pkey_set_cb_translate(pcb, ctx);
775	}
776	if (!RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb))
777		goto err;
778	if (!rsa_set_pss_param(rsa, ctx))
779		goto err;
780	if (!EVP_PKEY_assign(pkey, ctx->pmeth->pkey_id, rsa))
781		goto err;
782	rsa = NULL;
783
784	ret = 1;
785
786 err:
787	RSA_free(rsa);
788
789	return ret;
790}
791
792const EVP_PKEY_METHOD rsa_pkey_meth = {
793	.pkey_id = EVP_PKEY_RSA,
794	.flags = EVP_PKEY_FLAG_AUTOARGLEN,
795
796	.init = pkey_rsa_init,
797	.copy = pkey_rsa_copy,
798	.cleanup = pkey_rsa_cleanup,
799
800	.keygen = pkey_rsa_keygen,
801
802	.sign = pkey_rsa_sign,
803
804	.verify = pkey_rsa_verify,
805
806	.verify_recover = pkey_rsa_verifyrecover,
807
808	.encrypt = pkey_rsa_encrypt,
809
810	.decrypt = pkey_rsa_decrypt,
811
812	.ctrl = pkey_rsa_ctrl,
813	.ctrl_str = pkey_rsa_ctrl_str
814};
815
816/*
817 * Called for PSS sign or verify initialisation: checks PSS parameter
818 * sanity and sets any restrictions on key usage.
819 */
820
821static int
822pkey_pss_init(EVP_PKEY_CTX *ctx)
823{
824	RSA *rsa;
825	RSA_PKEY_CTX *rctx = ctx->data;
826	const EVP_MD *md;
827	const EVP_MD *mgf1md;
828	int min_saltlen, max_saltlen;
829
830	/* Should never happen */
831	if (ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
832		return 0;
833	rsa = ctx->pkey->pkey.rsa;
834
835	/* If no restrictions just return */
836	if (rsa->pss == NULL)
837		return 1;
838
839	/* Get and check parameters */
840	if (!rsa_pss_get_param(rsa->pss, &md, &mgf1md, &min_saltlen))
841		return 0;
842
843	/* See if minimum salt length exceeds maximum possible */
844	max_saltlen = RSA_size(rsa) - EVP_MD_size(md);
845	if ((RSA_bits(rsa) & 0x7) == 1)
846		max_saltlen--;
847	if (min_saltlen > max_saltlen) {
848		RSAerror(RSA_R_INVALID_SALT_LENGTH);
849		return 0;
850	}
851	rctx->min_saltlen = min_saltlen;
852
853	/*
854	 * Set PSS restrictions as defaults: we can then block any attempt to
855	 * use invalid values in pkey_rsa_ctrl
856	 */
857
858	rctx->md = md;
859	rctx->mgf1md = mgf1md;
860	rctx->saltlen = min_saltlen;
861
862	return 1;
863}
864
865const EVP_PKEY_METHOD rsa_pss_pkey_meth = {
866	.pkey_id = EVP_PKEY_RSA_PSS,
867	.flags = EVP_PKEY_FLAG_AUTOARGLEN,
868
869	.init = pkey_rsa_init,
870	.copy = pkey_rsa_copy,
871	.cleanup = pkey_rsa_cleanup,
872
873	.keygen = pkey_rsa_keygen,
874
875	.sign_init = pkey_pss_init,
876	.sign = pkey_rsa_sign,
877
878	.verify_init = pkey_pss_init,
879	.verify = pkey_rsa_verify,
880
881	.ctrl = pkey_rsa_ctrl,
882	.ctrl_str = pkey_rsa_ctrl_str
883};
884