From 9852814e6c96f701d2430459c2eb864cdfcf19d0 Mon Sep 17 00:00:00 2001 From: Hansaem Wi Date: Sun, 1 Nov 2026 11:24:12 +0900 Subject: [PATCH] wycheproof: test OAEP decryption tests/wycheproof.rs runs the PKCS#1 v1.5 and PSS signature vectors but no OAEP decryption vectors. Dropping `first_byte_is_zero`, `!nonzero_before_one` or `!looking_for_index` from the `valid` computation in src/algorithms/oaep.rs leaves the whole test suite green. Run the 22 rsa_oaep_*_test.json files from the Wycheproof submodule: 1098 vectors with SHA-1, SHA-2 and SHA-512/224, SHA-512/256 for the label hash and MGF1. Each of the three changes above makes 21 of the 22 files fail. The rsa_three_primes_oaep_* files are left out because multi-prime keys are not supported. Assisted-by: Claude Code:claude-opus-5-5 --- tests/wycheproof.rs | 134 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 126 insertions(+), 8 deletions(-) diff --git a/tests/wycheproof.rs b/tests/wycheproof.rs index b71f0ae6..17299bbc 100644 --- a/tests/wycheproof.rs +++ b/tests/wycheproof.rs @@ -5,18 +5,18 @@ // This implementation here is based on // -use std::fs::File; +use std::{fmt::Debug, fs::File}; use pkcs1::DecodeRsaPublicKey; -use rsa::{ - pkcs1v15, pss, - signature::{Error as SignatureError, Verifier}, - RsaPublicKey, -}; +use pkcs8::DecodePrivateKey; +use rsa::{pkcs1v15, pss, signature::Verifier, Oaep, RsaPrivateKey, RsaPublicKey}; use rstest::rstest; use serde::Deserialize; use sha1::Sha1; -use sha2::{Sha224, Sha256, Sha384, Sha512}; +use sha2::{ + digest::{Digest, FixedOutputReset}, + Sha224, Sha256, Sha384, Sha512, Sha512_224, Sha512_256, +}; #[derive(Deserialize, Debug)] struct TestFile { @@ -34,6 +34,9 @@ struct TestGroup { #[serde(default, rename(deserialize = "publicKeyAsn"), with = "hex::serde")] public_key_asn: Vec, + #[serde(default, rename(deserialize = "privateKeyPkcs8"), with = "hex::serde")] + private_key_pkcs8: Vec, + #[serde(default)] sha: String, @@ -57,6 +60,10 @@ struct Test { msg: Vec, #[serde(default, with = "hex::serde")] sig: Vec, + #[serde(default, with = "hex::serde")] + ct: Vec, + #[serde(default, with = "hex::serde")] + label: Vec, result: ExpectedResult, } @@ -86,7 +93,7 @@ impl Summary { } } - fn fail(&mut self, test: Test, res: Option) { + fn fail(&mut self, test: Test, res: impl Debug) { if self.in_test { eprintln!( " failed: {}: expected {:?}, got {:?}", @@ -270,3 +277,114 @@ fn test_rsa_pss_verify(#[case] file: &str) { } } } + +fn oaep_decrypt(key: &RsaPrivateKey, label: &[u8], ct: &[u8]) -> rsa::Result> +where + D: Digest + FixedOutputReset, + MGD: Digest + FixedOutputReset, +{ + if label.is_empty() { + key.decrypt(Oaep::::new_with_mgf_hash(), ct) + } else { + key.decrypt(Oaep::::new_with_mgf_hash_and_label(label), ct) + } +} + +fn oaep_decrypt_with_mgf( + mgf_sha: &str, + key: &RsaPrivateKey, + label: &[u8], + ct: &[u8], +) -> rsa::Result> +where + D: Digest + FixedOutputReset, +{ + match mgf_sha { + "SHA-1" => oaep_decrypt::(key, label, ct), + "SHA-224" => oaep_decrypt::(key, label, ct), + "SHA-256" => oaep_decrypt::(key, label, ct), + "SHA-384" => oaep_decrypt::(key, label, ct), + "SHA-512" => oaep_decrypt::(key, label, ct), + "SHA-512/224" => oaep_decrypt::(key, label, ct), + "SHA-512/256" => oaep_decrypt::(key, label, ct), + other => panic!("unhandled mgf sha {other:?}"), + } +} + +#[rstest] +#[case("rsa_oaep_2048_sha1_mgf1sha1_test.json")] +#[case("rsa_oaep_2048_sha224_mgf1sha1_test.json")] +#[case("rsa_oaep_2048_sha224_mgf1sha224_test.json")] +#[case("rsa_oaep_2048_sha256_mgf1sha1_test.json")] +#[case("rsa_oaep_2048_sha256_mgf1sha256_test.json")] +#[case("rsa_oaep_2048_sha384_mgf1sha1_test.json")] +#[case("rsa_oaep_2048_sha384_mgf1sha384_test.json")] +#[case("rsa_oaep_2048_sha512_224_mgf1sha1_test.json")] +#[case("rsa_oaep_2048_sha512_224_mgf1sha512_224_test.json")] +#[case("rsa_oaep_2048_sha512_mgf1sha1_test.json")] +#[case("rsa_oaep_2048_sha512_mgf1sha512_test.json")] +#[case("rsa_oaep_3072_sha256_mgf1sha1_test.json")] +#[case("rsa_oaep_3072_sha256_mgf1sha256_test.json")] +#[case("rsa_oaep_3072_sha512_256_mgf1sha1_test.json")] +#[case("rsa_oaep_3072_sha512_256_mgf1sha512_256_test.json")] +#[case("rsa_oaep_3072_sha512_mgf1sha1_test.json")] +#[case("rsa_oaep_3072_sha512_mgf1sha512_test.json")] +#[case("rsa_oaep_4096_sha256_mgf1sha1_test.json")] +#[case("rsa_oaep_4096_sha256_mgf1sha256_test.json")] +#[case("rsa_oaep_4096_sha512_mgf1sha1_test.json")] +#[case("rsa_oaep_4096_sha512_mgf1sha512_test.json")] +#[case("rsa_oaep_misc_test.json")] +fn test_rsa_oaep_decrypt(#[case] file: &str) { + let path = format!("thirdparty/wycheproof/testvectors_v1/{file}"); + let data_file = File::open(&path) + .expect("failed to open data file (try running `git submodule update --init`)"); + + println!("Loading file: {path}"); + + let tests: TestFile = serde_json::from_reader(data_file).expect("invalid test JSON"); + + println!("{}:\n{}\n", tests.algorithm, tests.header.join("")); + let mut summary = Summary::new(); + + for group in tests.groups { + summary.group(&group); + + let key = RsaPrivateKey::from_pkcs8_der(&group.private_key_pkcs8).unwrap(); + + for test in group.tests { + summary.start(&test); + + let result = match group.sha.as_ref() { + "SHA-1" => { + oaep_decrypt_with_mgf::(&group.mgf_sha, &key, &test.label, &test.ct) + } + "SHA-224" => { + oaep_decrypt_with_mgf::(&group.mgf_sha, &key, &test.label, &test.ct) + } + "SHA-256" => { + oaep_decrypt_with_mgf::(&group.mgf_sha, &key, &test.label, &test.ct) + } + "SHA-384" => { + oaep_decrypt_with_mgf::(&group.mgf_sha, &key, &test.label, &test.ct) + } + "SHA-512" => { + oaep_decrypt_with_mgf::(&group.mgf_sha, &key, &test.label, &test.ct) + } + "SHA-512/224" => { + oaep_decrypt_with_mgf::(&group.mgf_sha, &key, &test.label, &test.ct) + } + "SHA-512/256" => { + oaep_decrypt_with_mgf::(&group.mgf_sha, &key, &test.label, &test.ct) + } + other => panic!("unhandled sha {other:?}"), + }; + + match (test.result, &result) { + (ExpectedResult::Valid | ExpectedResult::Acceptable, Ok(msg)) + if *msg == test.msg => {} + (ExpectedResult::Invalid | ExpectedResult::Acceptable, Err(_err)) => {} + _ => summary.fail(test, result), + } + } + } +}