Decrypt messages whose content key is wrapped with RSAES-OAEP

Server/src/main/java/org/openas2/lib/helper/BCCryptoHelper.java

@@ -12,20 +12,18 @@
 import org.bouncycastle.asn1.nist.NISTObjectIdentifiers;
 import org.bouncycastle.asn1.oiw.OIWObjectIdentifiers;
 import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
-import org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
 import org.bouncycastle.asn1.x500.X500Name;
 import org.bouncycastle.cms.*;
-import org.bouncycastle.cms.bc.BcRSAKeyTransEnvelopedRecipient;
 import org.bouncycastle.cms.jcajce.JcaSimpleSignerInfoGeneratorBuilder;
 import org.bouncycastle.cms.jcajce.JcaSimpleSignerInfoVerifierBuilder;
 import org.bouncycastle.cms.jcajce.JceCMSContentEncryptorBuilder;
 import org.bouncycastle.cms.jcajce.JceKeyAgreeEnvelopedRecipient;
 import org.bouncycastle.cms.jcajce.JceKeyAgreeRecipientInfoGenerator;
+import org.bouncycastle.cms.jcajce.JceKeyTransEnvelopedRecipient;
 import org.bouncycastle.cms.jcajce.JceKeyTransRecipientInfoGenerator;
 
 import org.bouncycastle.cms.jcajce.ZlibCompressor;
 import org.bouncycastle.cms.jcajce.ZlibExpanderProvider;
-import org.bouncycastle.crypto.util.PrivateKeyFactory;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.bouncycastle.mail.smime.SMIMECompressed;
 import org.bouncycastle.mail.smime.SMIMECompressedGenerator;
@@ -238,9 +236,11 @@ public MimeBodyPart decrypt(MimeBodyPart part, Certificate receiverCert, Key rec
                 KeyTransRecipientId certRecId = new KeyTransRecipientId(x500Name, x509Cert.getSerialNumber());
                 if (certRecId.match(recipientInfo) && !foundRecipient) {
                     foundRecipient = true;
+                    // The JCE recipient honours the key-encryption algorithm in the envelope
+                    // (RSA PKCS#1 v1.5 or RSAES-OAEP); the lightweight BcRSAKeyTransEnvelopedRecipient
+                    // always unwraps with PKCS#1 v1.5, so an OAEP-wrapped key fails with "block incorrect".
                     byte[] decryptedData = recipientInfo.getContent(
-                        new BcRSAKeyTransEnvelopedRecipient(PrivateKeyFactory.createKey(
-                            PrivateKeyInfo.getInstance(receiverKey.getEncoded())))
+                        new JceKeyTransEnvelopedRecipient((PrivateKey) receiverKey).setProvider("BC")
                     );
                     return SMIMEUtil.toMimeBodyPart(decryptedData);
                 }

Server/src/test/java/org/openas2/lib/helper/KeyTransportDecryptionTest.java

@@ -0,0 +1,102 @@
+package org.openas2.lib.helper;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.cert.X509Certificate;
+import java.security.spec.MGF1ParameterSpec;
+import java.util.Date;
+
+import javax.crypto.spec.OAEPParameterSpec;
+import javax.crypto.spec.PSource;
+
+import jakarta.mail.internet.MimeBodyPart;
+
+import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder;
+import org.bouncycastle.cms.CMSAlgorithm;
+import org.bouncycastle.cms.RecipientInfoGenerator;
+import org.bouncycastle.cms.jcajce.JceCMSContentEncryptorBuilder;
+import org.bouncycastle.cms.jcajce.JceKeyTransRecipientInfoGenerator;
+import org.bouncycastle.mail.smime.SMIMEEnvelopedGenerator;
+import org.bouncycastle.operator.jcajce.JcaAlgorithmParametersConverter;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.openas2.util.AS2Util;
+
+/**
+ * Verifies that an inbound message decrypts whichever RSA key transport the sender wrapped its content key with.
+ * <p>
+ * PKCS#1 v1.5 is what OpenAS2 itself sends, but RSAES-OAEP is what others send by default: Microsoft BizTalk is
+ * one. The algorithm is named in the envelope's recipient info, so decryption has to honour it rather than
+ * assume PKCS#1 v1.5, which fails an OAEP-wrapped key with "block incorrect".
+ */
+public class KeyTransportDecryptionTest {
+
+    private static final String PAYLOAD = "<order/>";
+
+    private static KeyPair receiverKeys;
+    private static X509Certificate receiverCert;
+
+    @BeforeAll
+    public static void setUp() throws Exception {
+        AS2Util.getCryptoHelper(); // registers the BC provider
+        KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA");
+        kpg.initialize(2048);
+        receiverKeys = kpg.generateKeyPair();
+        receiverCert = selfSignedCert(receiverKeys, "CN=receiver");
+    }
+
+    @Test
+    public void aPkcs1WrappedKeyDecrypts() throws Exception {
+        assertDecrypts(new JceKeyTransRecipientInfoGenerator(receiverCert).setProvider("BC"));
+    }
+
+    @Test
+    public void anOaepWrappedKeyWithDefaultParametersDecrypts() throws Exception {
+        assertDecrypts(oaep(OAEPParameterSpec.DEFAULT));
+    }
+
+    @Test
+    public void anOaepWrappedKeyWithSha256Decrypts() throws Exception {
+        assertDecrypts(oaep(new OAEPParameterSpec("SHA-256", "MGF1", MGF1ParameterSpec.SHA256, PSource.PSpecified.DEFAULT)));
+    }
+
+    private static void assertDecrypts(RecipientInfoGenerator recipient) throws Exception {
+        MimeBodyPart part = new MimeBodyPart();
+        part.setContent(PAYLOAD, "application/xml");
+        part.setHeader("Content-Type", "application/xml");
+
+        SMIMEEnvelopedGenerator gen = new SMIMEEnvelopedGenerator();
+        gen.addRecipientInfoGenerator(recipient);
+        MimeBodyPart encrypted = gen.generate(part,
+                new JceCMSContentEncryptorBuilder(CMSAlgorithm.AES256_CBC).setProvider("BC").build());
+
+        MimeBodyPart decrypted = new BCCryptoHelper().decrypt(encrypted, receiverCert, receiverKeys.getPrivate());
+
+        assertEquals(PAYLOAD, new String(decrypted.getInputStream().readAllBytes(), StandardCharsets.UTF_8));
+    }
+
+    private static RecipientInfoGenerator oaep(OAEPParameterSpec spec) throws Exception {
+        return new JceKeyTransRecipientInfoGenerator(receiverCert,
+                new JcaAlgorithmParametersConverter().getAlgorithmIdentifier(PKCSObjectIdentifiers.id_RSAES_OAEP, spec))
+                .setProvider("BC");
+    }
+
+    private static X509Certificate selfSignedCert(KeyPair keyPair, String dn) throws Exception {
+        X500Name subject = new X500Name(dn);
+        JcaX509v3CertificateBuilder builder = new JcaX509v3CertificateBuilder(
+                subject, BigInteger.valueOf(System.nanoTime()),
+                new Date(System.currentTimeMillis() - 3600_000L),
+                new Date(System.currentTimeMillis() + 24 * 3600_000L),
+                subject, keyPair.getPublic());
+        return new JcaX509CertificateConverter().getCertificate(
+                builder.build(new JcaContentSignerBuilder("SHA256withRSA").build(keyPair.getPrivate())));
+    }
+}