1/* crypto/rsa/rsa_pmeth.c */
2/*
3 * Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL project
4 * 2006.
5 */
6/* ====================================================================
7 * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 *
16 * 2. Redistributions in binary form must reproduce the above copyright
17 *    notice, this list of conditions and the following disclaimer in
18 *    the documentation and/or other materials provided with the
19 *    distribution.
20 *
21 * 3. All advertising materials mentioning features or use of this
22 *    software must display the following acknowledgment:
23 *    "This product includes software developed by the OpenSSL Project
24 *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
25 *
26 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
27 *    endorse or promote products derived from this software without
28 *    prior written permission. For written permission, please contact
29 *    licensing@OpenSSL.org.
30 *
31 * 5. Products derived from this software may not be called "OpenSSL"
32 *    nor may "OpenSSL" appear in their names without prior written
33 *    permission of the OpenSSL Project.
34 *
35 * 6. Redistributions of any form whatsoever must retain the following
36 *    acknowledgment:
37 *    "This product includes software developed by the OpenSSL Project
38 *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
39 *
40 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
41 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
42 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
43 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
44 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
45 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
46 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
47 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
49 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
50 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
51 * OF THE POSSIBILITY OF SUCH DAMAGE.
52 * ====================================================================
53 *
54 * This product includes cryptographic software written by Eric Young
55 * (eay@cryptsoft.com).  This product includes software written by Tim
56 * Hudson (tjh@cryptsoft.com).
57 *
58 */
59
60#include <stdio.h>
61#include "cryptlib.h"
62#include <openssl/asn1t.h>
63#include <openssl/x509.h>
64#include <openssl/rsa.h>
65#include <openssl/bn.h>
66#include <openssl/evp.h>
67#include <openssl/x509v3.h>
68#ifndef OPENSSL_NO_CMS
69# include <openssl/cms.h>
70#endif
71#ifdef OPENSSL_FIPS
72# include <openssl/fips.h>
73#endif
74#include "evp_locl.h"
75#include "rsa_locl.h"
76
77/* RSA pkey context structure */
78
79typedef struct {
80    /* Key gen parameters */
81    int nbits;
82    BIGNUM *pub_exp;
83    /* Keygen callback info */
84    int gentmp[2];
85    /* RSA padding mode */
86    int pad_mode;
87    /* message digest */
88    const EVP_MD *md;
89    /* message digest for MGF1 */
90    const EVP_MD *mgf1md;
91    /* PSS salt length */
92    int saltlen;
93    /* Temp buffer */
94    unsigned char *tbuf;
95    /* OAEP label */
96    unsigned char *oaep_label;
97    size_t oaep_labellen;
98} RSA_PKEY_CTX;
99
100static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
101{
102    RSA_PKEY_CTX *rctx;
103    rctx = OPENSSL_malloc(sizeof(RSA_PKEY_CTX));
104    if (!rctx)
105        return 0;
106    rctx->nbits = 1024;
107    rctx->pub_exp = NULL;
108    rctx->pad_mode = RSA_PKCS1_PADDING;
109    rctx->md = NULL;
110    rctx->mgf1md = NULL;
111    rctx->tbuf = NULL;
112
113    rctx->saltlen = -2;
114
115    rctx->oaep_label = NULL;
116    rctx->oaep_labellen = 0;
117
118    ctx->data = rctx;
119    ctx->keygen_info = rctx->gentmp;
120    ctx->keygen_info_count = 2;
121
122    return 1;
123}
124
125static int pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
126{
127    RSA_PKEY_CTX *dctx, *sctx;
128    if (!pkey_rsa_init(dst))
129        return 0;
130    sctx = src->data;
131    dctx = dst->data;
132    dctx->nbits = sctx->nbits;
133    if (sctx->pub_exp) {
134        dctx->pub_exp = BN_dup(sctx->pub_exp);
135        if (!dctx->pub_exp)
136            return 0;
137    }
138    dctx->pad_mode = sctx->pad_mode;
139    dctx->md = sctx->md;
140    dctx->mgf1md = sctx->mgf1md;
141    if (sctx->oaep_label) {
142        if (dctx->oaep_label)
143            OPENSSL_free(dctx->oaep_label);
144        dctx->oaep_label = BUF_memdup(sctx->oaep_label, sctx->oaep_labellen);
145        if (!dctx->oaep_label)
146            return 0;
147        dctx->oaep_labellen = sctx->oaep_labellen;
148    }
149    return 1;
150}
151
152static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
153{
154    if (ctx->tbuf)
155        return 1;
156    ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));
157    if (!ctx->tbuf)
158        return 0;
159    return 1;
160}
161
162static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
163{
164    RSA_PKEY_CTX *rctx = ctx->data;
165    if (rctx) {
166        if (rctx->pub_exp)
167            BN_free(rctx->pub_exp);
168        if (rctx->tbuf)
169            OPENSSL_free(rctx->tbuf);
170        if (rctx->oaep_label)
171            OPENSSL_free(rctx->oaep_label);
172        OPENSSL_free(rctx);
173    }
174}
175
176#ifdef OPENSSL_FIPS
177/*
178 * FIP checker. Return value indicates status of context parameters: 1 :
179 * redirect to FIPS. 0 : don't redirect to FIPS. -1 : illegal operation in
180 * FIPS mode.
181 */
182
183static int pkey_fips_check_ctx(EVP_PKEY_CTX *ctx)
184{
185    RSA_PKEY_CTX *rctx = ctx->data;
186    RSA *rsa = ctx->pkey->pkey.rsa;
187    int rv = -1;
188    if (!FIPS_mode())
189        return 0;
190    if (rsa->flags & RSA_FLAG_NON_FIPS_ALLOW)
191        rv = 0;
192    if (!(rsa->meth->flags & RSA_FLAG_FIPS_METHOD) && rv)
193        return -1;
194    if (rctx->md) {
195        const EVP_MD *fmd;
196        fmd = FIPS_get_digestbynid(EVP_MD_type(rctx->md));
197        if (!fmd || !(fmd->flags & EVP_MD_FLAG_FIPS))
198            return rv;
199    }
200    if (rctx->mgf1md && !(rctx->mgf1md->flags & EVP_MD_FLAG_FIPS)) {
201        const EVP_MD *fmd;
202        fmd = FIPS_get_digestbynid(EVP_MD_type(rctx->mgf1md));
203        if (!fmd || !(fmd->flags & EVP_MD_FLAG_FIPS))
204            return rv;
205    }
206    return 1;
207}
208#endif
209
210static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig,
211                         size_t *siglen, const unsigned char *tbs,
212                         size_t tbslen)
213{
214    int ret;
215    RSA_PKEY_CTX *rctx = ctx->data;
216    RSA *rsa = ctx->pkey->pkey.rsa;
217
218#ifdef OPENSSL_FIPS
219    ret = pkey_fips_check_ctx(ctx);
220    if (ret < 0) {
221        RSAerr(RSA_F_PKEY_RSA_SIGN, RSA_R_OPERATION_NOT_ALLOWED_IN_FIPS_MODE);
222        return -1;
223    }
224#endif
225
226    if (rctx->md) {
227        if (tbslen != (size_t)EVP_MD_size(rctx->md)) {
228            RSAerr(RSA_F_PKEY_RSA_SIGN, RSA_R_INVALID_DIGEST_LENGTH);
229            return -1;
230        }
231#ifdef OPENSSL_FIPS
232        if (ret > 0) {
233            unsigned int slen;
234            ret = FIPS_rsa_sign_digest(rsa, tbs, tbslen, rctx->md,
235                                       rctx->pad_mode,
236                                       rctx->saltlen,
237                                       rctx->mgf1md, sig, &slen);
238            if (ret > 0)
239                *siglen = slen;
240            else
241                *siglen = 0;
242            return ret;
243        }
244#endif
245
246        if (EVP_MD_type(rctx->md) == NID_mdc2) {
247            unsigned int sltmp;
248            if (rctx->pad_mode != RSA_PKCS1_PADDING)
249                return -1;
250            ret = RSA_sign_ASN1_OCTET_STRING(NID_mdc2,
251                                             tbs, tbslen, sig, &sltmp, rsa);
252
253            if (ret <= 0)
254                return ret;
255            ret = sltmp;
256        } else if (rctx->pad_mode == RSA_X931_PADDING) {
257            if ((size_t)EVP_PKEY_size(ctx->pkey) < tbslen + 1) {
258                RSAerr(RSA_F_PKEY_RSA_SIGN, RSA_R_KEY_SIZE_TOO_SMALL);
259                return -1;
260            }
261            if (!setup_tbuf(rctx, ctx)) {
262                RSAerr(RSA_F_PKEY_RSA_SIGN, ERR_R_MALLOC_FAILURE);
263                return -1;
264            }
265            memcpy(rctx->tbuf, tbs, tbslen);
266            rctx->tbuf[tbslen] = RSA_X931_hash_id(EVP_MD_type(rctx->md));
267            ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
268                                      sig, rsa, RSA_X931_PADDING);
269        } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
270            unsigned int sltmp;
271            ret = RSA_sign(EVP_MD_type(rctx->md),
272                           tbs, tbslen, sig, &sltmp, rsa);
273            if (ret <= 0)
274                return ret;
275            ret = sltmp;
276        } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
277            if (!setup_tbuf(rctx, ctx))
278                return -1;
279            if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa,
280                                                rctx->tbuf, tbs,
281                                                rctx->md, rctx->mgf1md,
282                                                rctx->saltlen))
283                return -1;
284            ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
285                                      sig, rsa, RSA_NO_PADDING);
286        } else
287            return -1;
288    } else
289        ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
290                                  rctx->pad_mode);
291    if (ret < 0)
292        return ret;
293    *siglen = ret;
294    return 1;
295}
296
297static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
298                                  unsigned char *rout, size_t *routlen,
299                                  const unsigned char *sig, size_t siglen)
300{
301    int ret;
302    RSA_PKEY_CTX *rctx = ctx->data;
303
304    if (rctx->md) {
305        if (rctx->pad_mode == RSA_X931_PADDING) {
306            if (!setup_tbuf(rctx, ctx))
307                return -1;
308            ret = RSA_public_decrypt(siglen, sig,
309                                     rctx->tbuf, ctx->pkey->pkey.rsa,
310                                     RSA_X931_PADDING);
311            if (ret < 1)
312                return 0;
313            ret--;
314            if (rctx->tbuf[ret] != RSA_X931_hash_id(EVP_MD_type(rctx->md))) {
315                RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
316                       RSA_R_ALGORITHM_MISMATCH);
317                return 0;
318            }
319            if (ret != EVP_MD_size(rctx->md)) {
320                RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
321                       RSA_R_INVALID_DIGEST_LENGTH);
322                return 0;
323            }
324            if (rout)
325                memcpy(rout, rctx->tbuf, ret);
326        } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
327            size_t sltmp;
328            ret = int_rsa_verify(EVP_MD_type(rctx->md),
329                                 NULL, 0, rout, &sltmp,
330                                 sig, siglen, ctx->pkey->pkey.rsa);
331            if (ret <= 0)
332                return 0;
333            ret = sltmp;
334        } else
335            return -1;
336    } else
337        ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
338                                 rctx->pad_mode);
339    if (ret < 0)
340        return ret;
341    *routlen = ret;
342    return 1;
343}
344
345static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
346                           const unsigned char *sig, size_t siglen,
347                           const unsigned char *tbs, size_t tbslen)
348{
349    RSA_PKEY_CTX *rctx = ctx->data;
350    RSA *rsa = ctx->pkey->pkey.rsa;
351    size_t rslen;
352#ifdef OPENSSL_FIPS
353    int rv;
354    rv = pkey_fips_check_ctx(ctx);
355    if (rv < 0) {
356        RSAerr(RSA_F_PKEY_RSA_VERIFY,
357               RSA_R_OPERATION_NOT_ALLOWED_IN_FIPS_MODE);
358        return -1;
359    }
360#endif
361    if (rctx->md) {
362#ifdef OPENSSL_FIPS
363        if (rv > 0) {
364            return FIPS_rsa_verify_digest(rsa,
365                                          tbs, tbslen,
366                                          rctx->md,
367                                          rctx->pad_mode,
368                                          rctx->saltlen,
369                                          rctx->mgf1md, sig, siglen);
370
371        }
372#endif
373        if (rctx->pad_mode == RSA_PKCS1_PADDING)
374            return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
375                              sig, siglen, rsa);
376        if (rctx->pad_mode == RSA_X931_PADDING) {
377            if (pkey_rsa_verifyrecover(ctx, NULL, &rslen, sig, siglen) <= 0)
378                return 0;
379        } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
380            int ret;
381            if (!setup_tbuf(rctx, ctx))
382                return -1;
383            ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
384                                     rsa, RSA_NO_PADDING);
385            if (ret <= 0)
386                return 0;
387            ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs,
388                                            rctx->md, rctx->mgf1md,
389                                            rctx->tbuf, rctx->saltlen);
390            if (ret <= 0)
391                return 0;
392            return 1;
393        } else
394            return -1;
395    } else {
396        if (!setup_tbuf(rctx, ctx))
397            return -1;
398        rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
399                                   rsa, rctx->pad_mode);
400        if (rslen == 0)
401            return 0;
402    }
403
404    if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))
405        return 0;
406
407    return 1;
408
409}
410
411static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
412                            unsigned char *out, size_t *outlen,
413                            const unsigned char *in, size_t inlen)
414{
415    int ret;
416    RSA_PKEY_CTX *rctx = ctx->data;
417    if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
418        int klen = RSA_size(ctx->pkey->pkey.rsa);
419        if (!setup_tbuf(rctx, ctx))
420            return -1;
421        if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen,
422                                             in, inlen,
423                                             rctx->oaep_label,
424                                             rctx->oaep_labellen,
425                                             rctx->md, rctx->mgf1md))
426            return -1;
427        ret = RSA_public_encrypt(klen, rctx->tbuf, out,
428                                 ctx->pkey->pkey.rsa, RSA_NO_PADDING);
429    } else
430        ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
431                                 rctx->pad_mode);
432    if (ret < 0)
433        return ret;
434    *outlen = ret;
435    return 1;
436}
437
438static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
439                            unsigned char *out, size_t *outlen,
440                            const unsigned char *in, size_t inlen)
441{
442    int ret;
443    RSA_PKEY_CTX *rctx = ctx->data;
444    if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
445        int i;
446        if (!setup_tbuf(rctx, ctx))
447            return -1;
448        ret = RSA_private_decrypt(inlen, in, rctx->tbuf,
449                                  ctx->pkey->pkey.rsa, RSA_NO_PADDING);
450        if (ret <= 0)
451            return ret;
452        for (i = 0; i < ret; i++) {
453            if (rctx->tbuf[i])
454                break;
455        }
456        ret = RSA_padding_check_PKCS1_OAEP_mgf1(out, ret, rctx->tbuf + i,
457                                                ret - i, ret,
458                                                rctx->oaep_label,
459                                                rctx->oaep_labellen,
460                                                rctx->md, rctx->mgf1md);
461    } else
462        ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
463                                  rctx->pad_mode);
464    if (ret < 0)
465        return ret;
466    *outlen = ret;
467    return 1;
468}
469
470static int check_padding_md(const EVP_MD *md, int padding)
471{
472    if (!md)
473        return 1;
474
475    if (padding == RSA_NO_PADDING) {
476        RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
477        return 0;
478    }
479
480    if (padding == RSA_X931_PADDING) {
481        if (RSA_X931_hash_id(EVP_MD_type(md)) == -1) {
482            RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_X931_DIGEST);
483            return 0;
484        }
485        return 1;
486    }
487
488    return 1;
489}
490
491static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
492{
493    RSA_PKEY_CTX *rctx = ctx->data;
494    switch (type) {
495    case EVP_PKEY_CTRL_RSA_PADDING:
496        if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING)) {
497            if (!check_padding_md(rctx->md, p1))
498                return 0;
499            if (p1 == RSA_PKCS1_PSS_PADDING) {
500                if (!(ctx->operation &
501                      (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
502                    goto bad_pad;
503                if (!rctx->md)
504                    rctx->md = EVP_sha1();
505            }
506            if (p1 == RSA_PKCS1_OAEP_PADDING) {
507                if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
508                    goto bad_pad;
509                if (!rctx->md)
510                    rctx->md = EVP_sha1();
511            }
512            rctx->pad_mode = p1;
513            return 1;
514        }
515 bad_pad:
516        RSAerr(RSA_F_PKEY_RSA_CTRL,
517               RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
518        return -2;
519
520    case EVP_PKEY_CTRL_GET_RSA_PADDING:
521        *(int *)p2 = rctx->pad_mode;
522        return 1;
523
524    case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
525    case EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN:
526        if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) {
527            RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
528            return -2;
529        }
530        if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN)
531            *(int *)p2 = rctx->saltlen;
532        else {
533            if (p1 < -2)
534                return -2;
535            rctx->saltlen = p1;
536        }
537        return 1;
538
539    case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
540        if (p1 < 256) {
541            RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_KEYBITS);
542            return -2;
543        }
544        rctx->nbits = p1;
545        return 1;
546
547    case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
548        if (!p2)
549            return -2;
550        BN_free(rctx->pub_exp);
551        rctx->pub_exp = p2;
552        return 1;
553
554    case EVP_PKEY_CTRL_RSA_OAEP_MD:
555    case EVP_PKEY_CTRL_GET_RSA_OAEP_MD:
556        if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
557            RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
558            return -2;
559        }
560        if (type == EVP_PKEY_CTRL_GET_RSA_OAEP_MD)
561            *(const EVP_MD **)p2 = rctx->md;
562        else
563            rctx->md = p2;
564        return 1;
565
566    case EVP_PKEY_CTRL_MD:
567        if (!check_padding_md(p2, rctx->pad_mode))
568            return 0;
569        rctx->md = p2;
570        return 1;
571
572    case EVP_PKEY_CTRL_GET_MD:
573        *(const EVP_MD **)p2 = rctx->md;
574        return 1;
575
576    case EVP_PKEY_CTRL_RSA_MGF1_MD:
577    case EVP_PKEY_CTRL_GET_RSA_MGF1_MD:
578        if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING
579            && rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
580            RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_MGF1_MD);
581            return -2;
582        }
583        if (type == EVP_PKEY_CTRL_GET_RSA_MGF1_MD) {
584            if (rctx->mgf1md)
585                *(const EVP_MD **)p2 = rctx->mgf1md;
586            else
587                *(const EVP_MD **)p2 = rctx->md;
588        } else
589            rctx->mgf1md = p2;
590        return 1;
591
592    case EVP_PKEY_CTRL_RSA_OAEP_LABEL:
593        if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
594            RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
595            return -2;
596        }
597        if (rctx->oaep_label)
598            OPENSSL_free(rctx->oaep_label);
599        if (p2 && p1 > 0) {
600            rctx->oaep_label = p2;
601            rctx->oaep_labellen = p1;
602        } else {
603            rctx->oaep_label = NULL;
604            rctx->oaep_labellen = 0;
605        }
606        return 1;
607
608    case EVP_PKEY_CTRL_GET_RSA_OAEP_LABEL:
609        if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
610            RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
611            return -2;
612        }
613        *(unsigned char **)p2 = rctx->oaep_label;
614        return rctx->oaep_labellen;
615
616    case EVP_PKEY_CTRL_DIGESTINIT:
617    case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
618    case EVP_PKEY_CTRL_PKCS7_DECRYPT:
619    case EVP_PKEY_CTRL_PKCS7_SIGN:
620        return 1;
621#ifndef OPENSSL_NO_CMS
622    case EVP_PKEY_CTRL_CMS_DECRYPT:
623    case EVP_PKEY_CTRL_CMS_ENCRYPT:
624    case EVP_PKEY_CTRL_CMS_SIGN:
625        return 1;
626#endif
627    case EVP_PKEY_CTRL_PEER_KEY:
628        RSAerr(RSA_F_PKEY_RSA_CTRL,
629               RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
630        return -2;
631
632    default:
633        return -2;
634
635    }
636}
637
638static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
639                             const char *type, const char *value)
640{
641    if (!value) {
642        RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
643        return 0;
644    }
645    if (!strcmp(type, "rsa_padding_mode")) {
646        int pm;
647        if (!strcmp(value, "pkcs1"))
648            pm = RSA_PKCS1_PADDING;
649        else if (!strcmp(value, "sslv23"))
650            pm = RSA_SSLV23_PADDING;
651        else if (!strcmp(value, "none"))
652            pm = RSA_NO_PADDING;
653        else if (!strcmp(value, "oeap"))
654            pm = RSA_PKCS1_OAEP_PADDING;
655        else if (!strcmp(value, "oaep"))
656            pm = RSA_PKCS1_OAEP_PADDING;
657        else if (!strcmp(value, "x931"))
658            pm = RSA_X931_PADDING;
659        else if (!strcmp(value, "pss"))
660            pm = RSA_PKCS1_PSS_PADDING;
661        else {
662            RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_UNKNOWN_PADDING_TYPE);
663            return -2;
664        }
665        return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
666    }
667
668    if (!strcmp(type, "rsa_pss_saltlen")) {
669        int saltlen;
670        saltlen = atoi(value);
671        return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
672    }
673
674    if (!strcmp(type, "rsa_keygen_bits")) {
675        int nbits;
676        nbits = atoi(value);
677        return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
678    }
679
680    if (!strcmp(type, "rsa_keygen_pubexp")) {
681        int ret;
682        BIGNUM *pubexp = NULL;
683        if (!BN_asc2bn(&pubexp, value))
684            return 0;
685        ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
686        if (ret <= 0)
687            BN_free(pubexp);
688        return ret;
689    }
690
691    if (!strcmp(type, "rsa_mgf1_md")) {
692        const EVP_MD *md;
693        if (!(md = EVP_get_digestbyname(value))) {
694            RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_INVALID_DIGEST);
695            return 0;
696        }
697        return EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md);
698    }
699
700    if (!strcmp(type, "rsa_oaep_md")) {
701        const EVP_MD *md;
702        if (!(md = EVP_get_digestbyname(value))) {
703            RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_INVALID_DIGEST);
704            return 0;
705        }
706        return EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md);
707    }
708    if (!strcmp(type, "rsa_oaep_label")) {
709        unsigned char *lab;
710        long lablen;
711        int ret;
712        lab = string_to_hex(value, &lablen);
713        if (!lab)
714            return 0;
715        ret = EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, lab, lablen);
716        if (ret <= 0)
717            OPENSSL_free(lab);
718        return ret;
719    }
720
721    return -2;
722}
723
724static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
725{
726    RSA *rsa = NULL;
727    RSA_PKEY_CTX *rctx = ctx->data;
728    BN_GENCB *pcb, cb;
729    int ret;
730    if (!rctx->pub_exp) {
731        rctx->pub_exp = BN_new();
732        if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
733            return 0;
734    }
735    rsa = RSA_new();
736    if (!rsa)
737        return 0;
738    if (ctx->pkey_gencb) {
739        pcb = &cb;
740        evp_pkey_set_cb_translate(pcb, ctx);
741    } else
742        pcb = NULL;
743    ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
744    if (ret > 0)
745        EVP_PKEY_assign_RSA(pkey, rsa);
746    else
747        RSA_free(rsa);
748    return ret;
749}
750
751const EVP_PKEY_METHOD rsa_pkey_meth = {
752    EVP_PKEY_RSA,
753    EVP_PKEY_FLAG_AUTOARGLEN,
754    pkey_rsa_init,
755    pkey_rsa_copy,
756    pkey_rsa_cleanup,
757
758    0, 0,
759
760    0,
761    pkey_rsa_keygen,
762
763    0,
764    pkey_rsa_sign,
765
766    0,
767    pkey_rsa_verify,
768
769    0,
770    pkey_rsa_verifyrecover,
771
772    0, 0, 0, 0,
773
774    0,
775    pkey_rsa_encrypt,
776
777    0,
778    pkey_rsa_decrypt,
779
780    0, 0,
781
782    pkey_rsa_ctrl,
783    pkey_rsa_ctrl_str
784};
785