1/* crypto/rsa/rsa_ameth.c */
2/*
3 * Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL project
4 * 2006.
5 */
6/* ====================================================================
7 * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 *
16 * 2. Redistributions in binary form must reproduce the above copyright
17 *    notice, this list of conditions and the following disclaimer in
18 *    the documentation and/or other materials provided with the
19 *    distribution.
20 *
21 * 3. All advertising materials mentioning features or use of this
22 *    software must display the following acknowledgment:
23 *    "This product includes software developed by the OpenSSL Project
24 *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
25 *
26 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
27 *    endorse or promote products derived from this software without
28 *    prior written permission. For written permission, please contact
29 *    licensing@OpenSSL.org.
30 *
31 * 5. Products derived from this software may not be called "OpenSSL"
32 *    nor may "OpenSSL" appear in their names without prior written
33 *    permission of the OpenSSL Project.
34 *
35 * 6. Redistributions of any form whatsoever must retain the following
36 *    acknowledgment:
37 *    "This product includes software developed by the OpenSSL Project
38 *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
39 *
40 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
41 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
42 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
43 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
44 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
45 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
46 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
47 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
49 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
50 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
51 * OF THE POSSIBILITY OF SUCH DAMAGE.
52 * ====================================================================
53 *
54 * This product includes cryptographic software written by Eric Young
55 * (eay@cryptsoft.com).  This product includes software written by Tim
56 * Hudson (tjh@cryptsoft.com).
57 *
58 */
59
60#include <stdio.h>
61#include "cryptlib.h"
62#include <openssl/asn1t.h>
63#include <openssl/x509.h>
64#include <openssl/rsa.h>
65#include <openssl/bn.h>
66#ifndef OPENSSL_NO_CMS
67# include <openssl/cms.h>
68#endif
69#include "asn1_locl.h"
70
71static int rsa_cms_sign(CMS_SignerInfo *si);
72static int rsa_cms_verify(CMS_SignerInfo *si);
73static int rsa_cms_decrypt(CMS_RecipientInfo *ri);
74static int rsa_cms_encrypt(CMS_RecipientInfo *ri);
75
76static int rsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
77{
78    unsigned char *penc = NULL;
79    int penclen;
80    penclen = i2d_RSAPublicKey(pkey->pkey.rsa, &penc);
81    if (penclen <= 0)
82        return 0;
83    if (X509_PUBKEY_set0_param(pk, OBJ_nid2obj(EVP_PKEY_RSA),
84                               V_ASN1_NULL, NULL, penc, penclen))
85        return 1;
86
87    OPENSSL_free(penc);
88    return 0;
89}
90
91static int rsa_pub_decode(EVP_PKEY *pkey, X509_PUBKEY *pubkey)
92{
93    const unsigned char *p;
94    int pklen;
95    RSA *rsa = NULL;
96    if (!X509_PUBKEY_get0_param(NULL, &p, &pklen, NULL, pubkey))
97        return 0;
98    if (!(rsa = d2i_RSAPublicKey(NULL, &p, pklen))) {
99        RSAerr(RSA_F_RSA_PUB_DECODE, ERR_R_RSA_LIB);
100        return 0;
101    }
102    EVP_PKEY_assign_RSA(pkey, rsa);
103    return 1;
104}
105
106static int rsa_pub_cmp(const EVP_PKEY *a, const EVP_PKEY *b)
107{
108    if (BN_cmp(b->pkey.rsa->n, a->pkey.rsa->n) != 0
109        || BN_cmp(b->pkey.rsa->e, a->pkey.rsa->e) != 0)
110        return 0;
111    return 1;
112}
113
114static int old_rsa_priv_decode(EVP_PKEY *pkey,
115                               const unsigned char **pder, int derlen)
116{
117    RSA *rsa;
118    if (!(rsa = d2i_RSAPrivateKey(NULL, pder, derlen))) {
119        RSAerr(RSA_F_OLD_RSA_PRIV_DECODE, ERR_R_RSA_LIB);
120        return 0;
121    }
122    EVP_PKEY_assign_RSA(pkey, rsa);
123    return 1;
124}
125
126static int old_rsa_priv_encode(const EVP_PKEY *pkey, unsigned char **pder)
127{
128    return i2d_RSAPrivateKey(pkey->pkey.rsa, pder);
129}
130
131static int rsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey)
132{
133    unsigned char *rk = NULL;
134    int rklen;
135    rklen = i2d_RSAPrivateKey(pkey->pkey.rsa, &rk);
136
137    if (rklen <= 0) {
138        RSAerr(RSA_F_RSA_PRIV_ENCODE, ERR_R_MALLOC_FAILURE);
139        return 0;
140    }
141
142    if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(NID_rsaEncryption), 0,
143                         V_ASN1_NULL, NULL, rk, rklen)) {
144        RSAerr(RSA_F_RSA_PRIV_ENCODE, ERR_R_MALLOC_FAILURE);
145        return 0;
146    }
147
148    return 1;
149}
150
151static int rsa_priv_decode(EVP_PKEY *pkey, PKCS8_PRIV_KEY_INFO *p8)
152{
153    const unsigned char *p;
154    int pklen;
155    if (!PKCS8_pkey_get0(NULL, &p, &pklen, NULL, p8))
156        return 0;
157    return old_rsa_priv_decode(pkey, &p, pklen);
158}
159
160static int int_rsa_size(const EVP_PKEY *pkey)
161{
162    return RSA_size(pkey->pkey.rsa);
163}
164
165static int rsa_bits(const EVP_PKEY *pkey)
166{
167    return BN_num_bits(pkey->pkey.rsa->n);
168}
169
170static void int_rsa_free(EVP_PKEY *pkey)
171{
172    RSA_free(pkey->pkey.rsa);
173}
174
175static void update_buflen(const BIGNUM *b, size_t *pbuflen)
176{
177    size_t i;
178    if (!b)
179        return;
180    if (*pbuflen < (i = (size_t)BN_num_bytes(b)))
181        *pbuflen = i;
182}
183
184static int do_rsa_print(BIO *bp, const RSA *x, int off, int priv)
185{
186    char *str;
187    const char *s;
188    unsigned char *m = NULL;
189    int ret = 0, mod_len = 0;
190    size_t buf_len = 0;
191
192    update_buflen(x->n, &buf_len);
193    update_buflen(x->e, &buf_len);
194
195    if (priv) {
196        update_buflen(x->d, &buf_len);
197        update_buflen(x->p, &buf_len);
198        update_buflen(x->q, &buf_len);
199        update_buflen(x->dmp1, &buf_len);
200        update_buflen(x->dmq1, &buf_len);
201        update_buflen(x->iqmp, &buf_len);
202    }
203
204    m = (unsigned char *)OPENSSL_malloc(buf_len + 10);
205    if (m == NULL) {
206        RSAerr(RSA_F_DO_RSA_PRINT, ERR_R_MALLOC_FAILURE);
207        goto err;
208    }
209
210    if (x->n != NULL)
211        mod_len = BN_num_bits(x->n);
212
213    if (!BIO_indent(bp, off, 128))
214        goto err;
215
216    if (priv && x->d) {
217        if (BIO_printf(bp, "Private-Key: (%d bit)\n", mod_len)
218            <= 0)
219            goto err;
220        str = "modulus:";
221        s = "publicExponent:";
222    } else {
223        if (BIO_printf(bp, "Public-Key: (%d bit)\n", mod_len)
224            <= 0)
225            goto err;
226        str = "Modulus:";
227        s = "Exponent:";
228    }
229    if (!ASN1_bn_print(bp, str, x->n, m, off))
230        goto err;
231    if (!ASN1_bn_print(bp, s, x->e, m, off))
232        goto err;
233    if (priv) {
234        if (!ASN1_bn_print(bp, "privateExponent:", x->d, m, off))
235            goto err;
236        if (!ASN1_bn_print(bp, "prime1:", x->p, m, off))
237            goto err;
238        if (!ASN1_bn_print(bp, "prime2:", x->q, m, off))
239            goto err;
240        if (!ASN1_bn_print(bp, "exponent1:", x->dmp1, m, off))
241            goto err;
242        if (!ASN1_bn_print(bp, "exponent2:", x->dmq1, m, off))
243            goto err;
244        if (!ASN1_bn_print(bp, "coefficient:", x->iqmp, m, off))
245            goto err;
246    }
247    ret = 1;
248 err:
249    if (m != NULL)
250        OPENSSL_free(m);
251    return (ret);
252}
253
254static int rsa_pub_print(BIO *bp, const EVP_PKEY *pkey, int indent,
255                         ASN1_PCTX *ctx)
256{
257    return do_rsa_print(bp, pkey->pkey.rsa, indent, 0);
258}
259
260static int rsa_priv_print(BIO *bp, const EVP_PKEY *pkey, int indent,
261                          ASN1_PCTX *ctx)
262{
263    return do_rsa_print(bp, pkey->pkey.rsa, indent, 1);
264}
265
266/* Given an MGF1 Algorithm ID decode to an Algorithm Identifier */
267static X509_ALGOR *rsa_mgf1_decode(X509_ALGOR *alg)
268{
269    const unsigned char *p;
270    int plen;
271    if (alg == NULL)
272        return NULL;
273    if (OBJ_obj2nid(alg->algorithm) != NID_mgf1)
274        return NULL;
275    if (alg->parameter->type != V_ASN1_SEQUENCE)
276        return NULL;
277
278    p = alg->parameter->value.sequence->data;
279    plen = alg->parameter->value.sequence->length;
280    return d2i_X509_ALGOR(NULL, &p, plen);
281}
282
283static RSA_PSS_PARAMS *rsa_pss_decode(const X509_ALGOR *alg,
284                                      X509_ALGOR **pmaskHash)
285{
286    const unsigned char *p;
287    int plen;
288    RSA_PSS_PARAMS *pss;
289
290    *pmaskHash = NULL;
291
292    if (!alg->parameter || alg->parameter->type != V_ASN1_SEQUENCE)
293        return NULL;
294    p = alg->parameter->value.sequence->data;
295    plen = alg->parameter->value.sequence->length;
296    pss = d2i_RSA_PSS_PARAMS(NULL, &p, plen);
297
298    if (!pss)
299        return NULL;
300
301    *pmaskHash = rsa_mgf1_decode(pss->maskGenAlgorithm);
302
303    return pss;
304}
305
306static int rsa_pss_param_print(BIO *bp, RSA_PSS_PARAMS *pss,
307                               X509_ALGOR *maskHash, int indent)
308{
309    int rv = 0;
310    if (!pss) {
311        if (BIO_puts(bp, " (INVALID PSS PARAMETERS)\n") <= 0)
312            return 0;
313        return 1;
314    }
315    if (BIO_puts(bp, "\n") <= 0)
316        goto err;
317    if (!BIO_indent(bp, indent, 128))
318        goto err;
319    if (BIO_puts(bp, "Hash Algorithm: ") <= 0)
320        goto err;
321
322    if (pss->hashAlgorithm) {
323        if (i2a_ASN1_OBJECT(bp, pss->hashAlgorithm->algorithm) <= 0)
324            goto err;
325    } else if (BIO_puts(bp, "sha1 (default)") <= 0)
326        goto err;
327
328    if (BIO_puts(bp, "\n") <= 0)
329        goto err;
330
331    if (!BIO_indent(bp, indent, 128))
332        goto err;
333
334    if (BIO_puts(bp, "Mask Algorithm: ") <= 0)
335        goto err;
336    if (pss->maskGenAlgorithm) {
337        if (i2a_ASN1_OBJECT(bp, pss->maskGenAlgorithm->algorithm) <= 0)
338            goto err;
339        if (BIO_puts(bp, " with ") <= 0)
340            goto err;
341        if (maskHash) {
342            if (i2a_ASN1_OBJECT(bp, maskHash->algorithm) <= 0)
343                goto err;
344        } else if (BIO_puts(bp, "INVALID") <= 0)
345            goto err;
346    } else if (BIO_puts(bp, "mgf1 with sha1 (default)") <= 0)
347        goto err;
348    BIO_puts(bp, "\n");
349
350    if (!BIO_indent(bp, indent, 128))
351        goto err;
352    if (BIO_puts(bp, "Salt Length: 0x") <= 0)
353        goto err;
354    if (pss->saltLength) {
355        if (i2a_ASN1_INTEGER(bp, pss->saltLength) <= 0)
356            goto err;
357    } else if (BIO_puts(bp, "14 (default)") <= 0)
358        goto err;
359    BIO_puts(bp, "\n");
360
361    if (!BIO_indent(bp, indent, 128))
362        goto err;
363    if (BIO_puts(bp, "Trailer Field: 0x") <= 0)
364        goto err;
365    if (pss->trailerField) {
366        if (i2a_ASN1_INTEGER(bp, pss->trailerField) <= 0)
367            goto err;
368    } else if (BIO_puts(bp, "BC (default)") <= 0)
369        goto err;
370    BIO_puts(bp, "\n");
371
372    rv = 1;
373
374 err:
375    return rv;
376
377}
378
379static int rsa_sig_print(BIO *bp, const X509_ALGOR *sigalg,
380                         const ASN1_STRING *sig, int indent, ASN1_PCTX *pctx)
381{
382    if (OBJ_obj2nid(sigalg->algorithm) == NID_rsassaPss) {
383        int rv;
384        RSA_PSS_PARAMS *pss;
385        X509_ALGOR *maskHash;
386        pss = rsa_pss_decode(sigalg, &maskHash);
387        rv = rsa_pss_param_print(bp, pss, maskHash, indent);
388        if (pss)
389            RSA_PSS_PARAMS_free(pss);
390        if (maskHash)
391            X509_ALGOR_free(maskHash);
392        if (!rv)
393            return 0;
394    } else if (!sig && BIO_puts(bp, "\n") <= 0)
395        return 0;
396    if (sig)
397        return X509_signature_dump(bp, sig, indent);
398    return 1;
399}
400
401static int rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
402{
403    X509_ALGOR *alg = NULL;
404    switch (op) {
405
406    case ASN1_PKEY_CTRL_PKCS7_SIGN:
407        if (arg1 == 0)
408            PKCS7_SIGNER_INFO_get0_algs(arg2, NULL, NULL, &alg);
409        break;
410
411    case ASN1_PKEY_CTRL_PKCS7_ENCRYPT:
412        if (arg1 == 0)
413            PKCS7_RECIP_INFO_get0_alg(arg2, &alg);
414        break;
415#ifndef OPENSSL_NO_CMS
416    case ASN1_PKEY_CTRL_CMS_SIGN:
417        if (arg1 == 0)
418            return rsa_cms_sign(arg2);
419        else if (arg1 == 1)
420            return rsa_cms_verify(arg2);
421        break;
422
423    case ASN1_PKEY_CTRL_CMS_ENVELOPE:
424        if (arg1 == 0)
425            return rsa_cms_encrypt(arg2);
426        else if (arg1 == 1)
427            return rsa_cms_decrypt(arg2);
428        break;
429
430    case ASN1_PKEY_CTRL_CMS_RI_TYPE:
431        *(int *)arg2 = CMS_RECIPINFO_TRANS;
432        return 1;
433#endif
434
435    case ASN1_PKEY_CTRL_DEFAULT_MD_NID:
436        *(int *)arg2 = NID_sha256;
437        return 1;
438
439    default:
440        return -2;
441
442    }
443
444    if (alg)
445        X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
446
447    return 1;
448
449}
450
451/* allocate and set algorithm ID from EVP_MD, default SHA1 */
452static int rsa_md_to_algor(X509_ALGOR **palg, const EVP_MD *md)
453{
454    if (EVP_MD_type(md) == NID_sha1)
455        return 1;
456    *palg = X509_ALGOR_new();
457    if (!*palg)
458        return 0;
459    X509_ALGOR_set_md(*palg, md);
460    return 1;
461}
462
463/* Allocate and set MGF1 algorithm ID from EVP_MD */
464static int rsa_md_to_mgf1(X509_ALGOR **palg, const EVP_MD *mgf1md)
465{
466    X509_ALGOR *algtmp = NULL;
467    ASN1_STRING *stmp = NULL;
468    *palg = NULL;
469    if (EVP_MD_type(mgf1md) == NID_sha1)
470        return 1;
471    /* need to embed algorithm ID inside another */
472    if (!rsa_md_to_algor(&algtmp, mgf1md))
473        goto err;
474    if (!ASN1_item_pack(algtmp, ASN1_ITEM_rptr(X509_ALGOR), &stmp))
475         goto err;
476    *palg = X509_ALGOR_new();
477    if (!*palg)
478        goto err;
479    X509_ALGOR_set0(*palg, OBJ_nid2obj(NID_mgf1), V_ASN1_SEQUENCE, stmp);
480    stmp = NULL;
481 err:
482    if (stmp)
483        ASN1_STRING_free(stmp);
484    if (algtmp)
485        X509_ALGOR_free(algtmp);
486    if (*palg)
487        return 1;
488    return 0;
489}
490
491/* convert algorithm ID to EVP_MD, default SHA1 */
492static const EVP_MD *rsa_algor_to_md(X509_ALGOR *alg)
493{
494    const EVP_MD *md;
495    if (!alg)
496        return EVP_sha1();
497    md = EVP_get_digestbyobj(alg->algorithm);
498    if (md == NULL)
499        RSAerr(RSA_F_RSA_ALGOR_TO_MD, RSA_R_UNKNOWN_DIGEST);
500    return md;
501}
502
503/* convert MGF1 algorithm ID to EVP_MD, default SHA1 */
504static const EVP_MD *rsa_mgf1_to_md(X509_ALGOR *alg, X509_ALGOR *maskHash)
505{
506    const EVP_MD *md;
507    if (!alg)
508        return EVP_sha1();
509    /* Check mask and lookup mask hash algorithm */
510    if (OBJ_obj2nid(alg->algorithm) != NID_mgf1) {
511        RSAerr(RSA_F_RSA_MGF1_TO_MD, RSA_R_UNSUPPORTED_MASK_ALGORITHM);
512        return NULL;
513    }
514    if (!maskHash) {
515        RSAerr(RSA_F_RSA_MGF1_TO_MD, RSA_R_UNSUPPORTED_MASK_PARAMETER);
516        return NULL;
517    }
518    md = EVP_get_digestbyobj(maskHash->algorithm);
519    if (md == NULL) {
520        RSAerr(RSA_F_RSA_MGF1_TO_MD, RSA_R_UNKNOWN_MASK_DIGEST);
521        return NULL;
522    }
523    return md;
524}
525
526/*
527 * Convert EVP_PKEY_CTX is PSS mode into corresponding algorithm parameter,
528 * suitable for setting an AlgorithmIdentifier.
529 */
530
531static ASN1_STRING *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
532{
533    const EVP_MD *sigmd, *mgf1md;
534    RSA_PSS_PARAMS *pss = NULL;
535    ASN1_STRING *os = NULL;
536    EVP_PKEY *pk = EVP_PKEY_CTX_get0_pkey(pkctx);
537    int saltlen, rv = 0;
538    if (EVP_PKEY_CTX_get_signature_md(pkctx, &sigmd) <= 0)
539        goto err;
540    if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
541        goto err;
542    if (!EVP_PKEY_CTX_get_rsa_pss_saltlen(pkctx, &saltlen))
543        goto err;
544    if (saltlen == -1)
545        saltlen = EVP_MD_size(sigmd);
546    else if (saltlen == -2) {
547        saltlen = EVP_PKEY_size(pk) - EVP_MD_size(sigmd) - 2;
548        if (((EVP_PKEY_bits(pk) - 1) & 0x7) == 0)
549            saltlen--;
550    }
551    pss = RSA_PSS_PARAMS_new();
552    if (!pss)
553        goto err;
554    if (saltlen != 20) {
555        pss->saltLength = ASN1_INTEGER_new();
556        if (!pss->saltLength)
557            goto err;
558        if (!ASN1_INTEGER_set(pss->saltLength, saltlen))
559            goto err;
560    }
561    if (!rsa_md_to_algor(&pss->hashAlgorithm, sigmd))
562        goto err;
563    if (!rsa_md_to_mgf1(&pss->maskGenAlgorithm, mgf1md))
564        goto err;
565    /* Finally create string with pss parameter encoding. */
566    if (!ASN1_item_pack(pss, ASN1_ITEM_rptr(RSA_PSS_PARAMS), &os))
567         goto err;
568    rv = 1;
569 err:
570    if (pss)
571        RSA_PSS_PARAMS_free(pss);
572    if (rv)
573        return os;
574    if (os)
575        ASN1_STRING_free(os);
576    return NULL;
577}
578
579/*
580 * From PSS AlgorithmIdentifier set public key parameters. If pkey isn't NULL
581 * then the EVP_MD_CTX is setup and initalised. If it is NULL parameters are
582 * passed to pkctx instead.
583 */
584
585static int rsa_pss_to_ctx(EVP_MD_CTX *ctx, EVP_PKEY_CTX *pkctx,
586                          X509_ALGOR *sigalg, EVP_PKEY *pkey)
587{
588    int rv = -1;
589    int saltlen;
590    const EVP_MD *mgf1md = NULL, *md = NULL;
591    RSA_PSS_PARAMS *pss;
592    X509_ALGOR *maskHash;
593    /* Sanity check: make sure it is PSS */
594    if (OBJ_obj2nid(sigalg->algorithm) != NID_rsassaPss) {
595        RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_UNSUPPORTED_SIGNATURE_TYPE);
596        return -1;
597    }
598    /* Decode PSS parameters */
599    pss = rsa_pss_decode(sigalg, &maskHash);
600
601    if (pss == NULL) {
602        RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_INVALID_PSS_PARAMETERS);
603        goto err;
604    }
605    mgf1md = rsa_mgf1_to_md(pss->maskGenAlgorithm, maskHash);
606    if (!mgf1md)
607        goto err;
608    md = rsa_algor_to_md(pss->hashAlgorithm);
609    if (!md)
610        goto err;
611
612    if (pss->saltLength) {
613        saltlen = ASN1_INTEGER_get(pss->saltLength);
614
615        /*
616         * Could perform more salt length sanity checks but the main RSA
617         * routines will trap other invalid values anyway.
618         */
619        if (saltlen < 0) {
620            RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_INVALID_SALT_LENGTH);
621            goto err;
622        }
623    } else
624        saltlen = 20;
625
626    /*
627     * low-level routines support only trailer field 0xbc (value 1) and
628     * PKCS#1 says we should reject any other value anyway.
629     */
630    if (pss->trailerField && ASN1_INTEGER_get(pss->trailerField) != 1) {
631        RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_INVALID_TRAILER);
632        goto err;
633    }
634
635    /* We have all parameters now set up context */
636
637    if (pkey) {
638        if (!EVP_DigestVerifyInit(ctx, &pkctx, md, NULL, pkey))
639            goto err;
640    } else {
641        const EVP_MD *checkmd;
642        if (EVP_PKEY_CTX_get_signature_md(pkctx, &checkmd) <= 0)
643            goto err;
644        if (EVP_MD_type(md) != EVP_MD_type(checkmd)) {
645            RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_DIGEST_DOES_NOT_MATCH);
646            goto err;
647        }
648    }
649
650    if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_PSS_PADDING) <= 0)
651        goto err;
652
653    if (EVP_PKEY_CTX_set_rsa_pss_saltlen(pkctx, saltlen) <= 0)
654        goto err;
655
656    if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0)
657        goto err;
658    /* Carry on */
659    rv = 1;
660
661 err:
662    RSA_PSS_PARAMS_free(pss);
663    if (maskHash)
664        X509_ALGOR_free(maskHash);
665    return rv;
666}
667
668static int rsa_cms_verify(CMS_SignerInfo *si)
669{
670    int nid, nid2;
671    X509_ALGOR *alg;
672    EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si);
673    CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg);
674    nid = OBJ_obj2nid(alg->algorithm);
675    if (nid == NID_rsaEncryption)
676        return 1;
677    if (nid == NID_rsassaPss)
678        return rsa_pss_to_ctx(NULL, pkctx, alg, NULL);
679    /* Workaround for some implementation that use a signature OID */
680    if (OBJ_find_sigid_algs(nid, NULL, &nid2)) {
681        if (nid2 == NID_rsaEncryption)
682            return 1;
683    }
684    return 0;
685}
686
687/*
688 * Customised RSA item verification routine. This is called when a signature
689 * is encountered requiring special handling. We currently only handle PSS.
690 */
691
692static int rsa_item_verify(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn,
693                           X509_ALGOR *sigalg, ASN1_BIT_STRING *sig,
694                           EVP_PKEY *pkey)
695{
696    /* Sanity check: make sure it is PSS */
697    if (OBJ_obj2nid(sigalg->algorithm) != NID_rsassaPss) {
698        RSAerr(RSA_F_RSA_ITEM_VERIFY, RSA_R_UNSUPPORTED_SIGNATURE_TYPE);
699        return -1;
700    }
701    if (rsa_pss_to_ctx(ctx, NULL, sigalg, pkey) > 0) {
702        /* Carry on */
703        return 2;
704    }
705    return -1;
706}
707
708static int rsa_cms_sign(CMS_SignerInfo *si)
709{
710    int pad_mode = RSA_PKCS1_PADDING;
711    X509_ALGOR *alg;
712    EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si);
713    ASN1_STRING *os = NULL;
714    CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg);
715    if (pkctx) {
716        if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
717            return 0;
718    }
719    if (pad_mode == RSA_PKCS1_PADDING) {
720        X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
721        return 1;
722    }
723    /* We don't support it */
724    if (pad_mode != RSA_PKCS1_PSS_PADDING)
725        return 0;
726    os = rsa_ctx_to_pss(pkctx);
727    if (!os)
728        return 0;
729    X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsassaPss), V_ASN1_SEQUENCE, os);
730    return 1;
731}
732
733static int rsa_item_sign(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn,
734                         X509_ALGOR *alg1, X509_ALGOR *alg2,
735                         ASN1_BIT_STRING *sig)
736{
737    int pad_mode;
738    EVP_PKEY_CTX *pkctx = ctx->pctx;
739    if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
740        return 0;
741    if (pad_mode == RSA_PKCS1_PADDING)
742        return 2;
743    if (pad_mode == RSA_PKCS1_PSS_PADDING) {
744        ASN1_STRING *os1 = NULL;
745        os1 = rsa_ctx_to_pss(pkctx);
746        if (!os1)
747            return 0;
748        /* Duplicate parameters if we have to */
749        if (alg2) {
750            ASN1_STRING *os2 = ASN1_STRING_dup(os1);
751            if (!os2) {
752                ASN1_STRING_free(os1);
753                return 0;
754            }
755            X509_ALGOR_set0(alg2, OBJ_nid2obj(NID_rsassaPss),
756                            V_ASN1_SEQUENCE, os2);
757        }
758        X509_ALGOR_set0(alg1, OBJ_nid2obj(NID_rsassaPss),
759                        V_ASN1_SEQUENCE, os1);
760        return 3;
761    }
762    return 2;
763}
764
765static RSA_OAEP_PARAMS *rsa_oaep_decode(const X509_ALGOR *alg,
766                                        X509_ALGOR **pmaskHash)
767{
768    const unsigned char *p;
769    int plen;
770    RSA_OAEP_PARAMS *pss;
771
772    *pmaskHash = NULL;
773
774    if (!alg->parameter || alg->parameter->type != V_ASN1_SEQUENCE)
775        return NULL;
776    p = alg->parameter->value.sequence->data;
777    plen = alg->parameter->value.sequence->length;
778    pss = d2i_RSA_OAEP_PARAMS(NULL, &p, plen);
779
780    if (!pss)
781        return NULL;
782
783    *pmaskHash = rsa_mgf1_decode(pss->maskGenFunc);
784
785    return pss;
786}
787
788static int rsa_cms_decrypt(CMS_RecipientInfo *ri)
789{
790    EVP_PKEY_CTX *pkctx;
791    X509_ALGOR *cmsalg;
792    int nid;
793    int rv = -1;
794    unsigned char *label = NULL;
795    int labellen = 0;
796    const EVP_MD *mgf1md = NULL, *md = NULL;
797    RSA_OAEP_PARAMS *oaep;
798    X509_ALGOR *maskHash;
799    pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
800    if (!pkctx)
801        return 0;
802    if (!CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &cmsalg))
803        return -1;
804    nid = OBJ_obj2nid(cmsalg->algorithm);
805    if (nid == NID_rsaEncryption)
806        return 1;
807    if (nid != NID_rsaesOaep) {
808        RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_UNSUPPORTED_ENCRYPTION_TYPE);
809        return -1;
810    }
811    /* Decode OAEP parameters */
812    oaep = rsa_oaep_decode(cmsalg, &maskHash);
813
814    if (oaep == NULL) {
815        RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_INVALID_OAEP_PARAMETERS);
816        goto err;
817    }
818
819    mgf1md = rsa_mgf1_to_md(oaep->maskGenFunc, maskHash);
820    if (!mgf1md)
821        goto err;
822    md = rsa_algor_to_md(oaep->hashFunc);
823    if (!md)
824        goto err;
825
826    if (oaep->pSourceFunc) {
827        X509_ALGOR *plab = oaep->pSourceFunc;
828        if (OBJ_obj2nid(plab->algorithm) != NID_pSpecified) {
829            RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_UNSUPPORTED_LABEL_SOURCE);
830            goto err;
831        }
832        if (plab->parameter->type != V_ASN1_OCTET_STRING) {
833            RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_INVALID_LABEL);
834            goto err;
835        }
836
837        label = plab->parameter->value.octet_string->data;
838        /* Stop label being freed when OAEP parameters are freed */
839        plab->parameter->value.octet_string->data = NULL;
840        labellen = plab->parameter->value.octet_string->length;
841    }
842
843    if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_OAEP_PADDING) <= 0)
844        goto err;
845    if (EVP_PKEY_CTX_set_rsa_oaep_md(pkctx, md) <= 0)
846        goto err;
847    if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0)
848        goto err;
849    if (EVP_PKEY_CTX_set0_rsa_oaep_label(pkctx, label, labellen) <= 0)
850        goto err;
851    /* Carry on */
852    rv = 1;
853
854 err:
855    RSA_OAEP_PARAMS_free(oaep);
856    if (maskHash)
857        X509_ALGOR_free(maskHash);
858    return rv;
859}
860
861static int rsa_cms_encrypt(CMS_RecipientInfo *ri)
862{
863    const EVP_MD *md, *mgf1md;
864    RSA_OAEP_PARAMS *oaep = NULL;
865    ASN1_STRING *os = NULL;
866    X509_ALGOR *alg;
867    EVP_PKEY_CTX *pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
868    int pad_mode = RSA_PKCS1_PADDING, rv = 0, labellen;
869    unsigned char *label;
870    CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &alg);
871    if (pkctx) {
872        if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
873            return 0;
874    }
875    if (pad_mode == RSA_PKCS1_PADDING) {
876        X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
877        return 1;
878    }
879    /* Not supported */
880    if (pad_mode != RSA_PKCS1_OAEP_PADDING)
881        return 0;
882    if (EVP_PKEY_CTX_get_rsa_oaep_md(pkctx, &md) <= 0)
883        goto err;
884    if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
885        goto err;
886    labellen = EVP_PKEY_CTX_get0_rsa_oaep_label(pkctx, &label);
887    if (labellen < 0)
888        goto err;
889    oaep = RSA_OAEP_PARAMS_new();
890    if (!oaep)
891        goto err;
892    if (!rsa_md_to_algor(&oaep->hashFunc, md))
893        goto err;
894    if (!rsa_md_to_mgf1(&oaep->maskGenFunc, mgf1md))
895        goto err;
896    if (labellen > 0) {
897        ASN1_OCTET_STRING *los = ASN1_OCTET_STRING_new();
898        oaep->pSourceFunc = X509_ALGOR_new();
899        if (!oaep->pSourceFunc)
900            goto err;
901        if (!los)
902            goto err;
903        if (!ASN1_OCTET_STRING_set(los, label, labellen)) {
904            ASN1_OCTET_STRING_free(los);
905            goto err;
906        }
907        X509_ALGOR_set0(oaep->pSourceFunc, OBJ_nid2obj(NID_pSpecified),
908                        V_ASN1_OCTET_STRING, los);
909    }
910    /* create string with pss parameter encoding. */
911    if (!ASN1_item_pack(oaep, ASN1_ITEM_rptr(RSA_OAEP_PARAMS), &os))
912         goto err;
913    X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaesOaep), V_ASN1_SEQUENCE, os);
914    os = NULL;
915    rv = 1;
916 err:
917    if (oaep)
918        RSA_OAEP_PARAMS_free(oaep);
919    if (os)
920        ASN1_STRING_free(os);
921    return rv;
922}
923
924const EVP_PKEY_ASN1_METHOD rsa_asn1_meths[] = {
925    {
926     EVP_PKEY_RSA,
927     EVP_PKEY_RSA,
928     ASN1_PKEY_SIGPARAM_NULL,
929
930     "RSA",
931     "OpenSSL RSA method",
932
933     rsa_pub_decode,
934     rsa_pub_encode,
935     rsa_pub_cmp,
936     rsa_pub_print,
937
938     rsa_priv_decode,
939     rsa_priv_encode,
940     rsa_priv_print,
941
942     int_rsa_size,
943     rsa_bits,
944
945     0, 0, 0, 0, 0, 0,
946
947     rsa_sig_print,
948     int_rsa_free,
949     rsa_pkey_ctrl,
950     old_rsa_priv_decode,
951     old_rsa_priv_encode,
952     rsa_item_verify,
953     rsa_item_sign},
954
955    {
956     EVP_PKEY_RSA2,
957     EVP_PKEY_RSA,
958     ASN1_PKEY_ALIAS}
959};
960