[crypto] Atomize all traces of MbedTLS, and require OpenSSL 3+ (#3606)

Closes #3137
Closes #3465

- Replace all mbedtls usage with OpenSSL
- require OpenSSL
- Up OpenSSL version to 3, cuz that's what we actually need...

CAVEATS:
- httplib also now required
- other ssl backends for svc are unused, maybe remove later
  * To be fair, our CI never used them anyways. And we never tested those

TESTERS PLEASE TEST:
- All games and applets boot
- Boot, load, exit, etc. times

Co-authored-by: crueter <crueter@eden-emu.dev>
Signed-off-by: lizzie <lizzie@eden-emu.dev>
Co-authored-by: crueter <crueter@eden-emu.dev>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3606
Reviewed-by: crueter <crueter@eden-emu.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>
Reviewed-by: DraVee <dravee@eden-emu.dev>
Co-authored-by: lizzie <lizzie@eden-emu.dev>
Co-committed-by: lizzie <lizzie@eden-emu.dev>

4 files changed, 153 insertions, 110 deletions

diff --git a/src/core/crypto/aes_util.cpp b/src/core/crypto/aes_util.cpp
index 0820ca30ea..1189e45bd8 100644
--- a/src/core/crypto/aes_util.cpp
+++ b/src/core/crypto/aes_util.cpp
@@ -1,13 +1,13 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include <algorithm>
 #include <array>
 #include <cstring>
-#include <mbedtls/cipher.h>
+#include <openssl/err.h>
+#include <openssl/evp.h>
 #include "common/assert.h"
 #include "common/logging/log.h"
 #include "core/crypto/aes_util.h"
@@ -28,83 +28,121 @@ NintendoTweak CalculateNintendoTweak(std::size_t sector_id) {
 }
 } // Anonymous namespace
 
-static_assert(static_cast<std::size_t>(Mode::CTR) ==
-                  static_cast<std::size_t>(MBEDTLS_CIPHER_AES_128_CTR),
-              "CTR has incorrect value.");
-static_assert(static_cast<std::size_t>(Mode::ECB) ==
-                  static_cast<std::size_t>(MBEDTLS_CIPHER_AES_128_ECB),
-              "ECB has incorrect value.");
-static_assert(static_cast<std::size_t>(Mode::XTS) ==
-                  static_cast<std::size_t>(MBEDTLS_CIPHER_AES_128_XTS),
-              "XTS has incorrect value.");
-
-// Structure to hide mbedtls types from header file
+// Structure to hide OpenSSL types from header file
 struct CipherContext {
-    mbedtls_cipher_context_t encryption_context;
-    mbedtls_cipher_context_t decryption_context;
+    EVP_CIPHER_CTX* encryption_context = nullptr;
+    EVP_CIPHER_CTX* decryption_context = nullptr;
+    EVP_CIPHER* cipher = nullptr;
 };
 
+static inline const std::string GetCipherName(Mode mode, u32 key_size) {
+    std::string cipher;
+    std::size_t effective_bits = key_size * 8;
+
+    switch (mode) {
+    case Mode::CTR:
+        cipher = "CTR";
+        break;
+    case Mode::ECB:
+        cipher = "ECB";
+        break;
+    case Mode::XTS:
+        cipher = "XTS";
+        effective_bits /= 2;
+        break;
+    default:
+        UNREACHABLE();
+    }
+
+    return fmt::format("AES-{}-{}", effective_bits, cipher);
+};
+
+static EVP_CIPHER *GetCipher(Mode mode, u32 key_size) {
+    static auto fetch_cipher = [](Mode m, u32 k) {
+        return EVP_CIPHER_fetch(nullptr, GetCipherName(m, k).c_str(), nullptr);
+    };
+
+    static const struct {
+        EVP_CIPHER* ctr_16 = fetch_cipher(Mode::CTR, 16);
+        EVP_CIPHER* ecb_16 = fetch_cipher(Mode::ECB, 16);
+        EVP_CIPHER* xts_16 = fetch_cipher(Mode::XTS, 16);
+        EVP_CIPHER* ctr_32 = fetch_cipher(Mode::CTR, 32);
+        EVP_CIPHER* ecb_32 = fetch_cipher(Mode::ECB, 32);
+        EVP_CIPHER* xts_32 = fetch_cipher(Mode::XTS, 32);
+    } ciphers = {};
+
+    switch (mode) {
+    case Mode::CTR:
+        return key_size == 16 ? ciphers.ctr_16 : ciphers.ctr_32;
+    case Mode::ECB:
+        return key_size == 16 ? ciphers.ecb_16 : ciphers.ecb_32;
+    case Mode::XTS:
+        return key_size == 16 ? ciphers.xts_16 : ciphers.xts_32;
+    default:
+        UNIMPLEMENTED();
+    }
+
+    return nullptr;
+}
+
+// TODO: WHY TEMPLATE???????
 template <typename Key, std::size_t KeySize>
 Crypto::AESCipher<Key, KeySize>::AESCipher(Key key, Mode mode)
     : ctx(std::make_unique<CipherContext>()) {
-    mbedtls_cipher_init(&ctx->encryption_context);
-    mbedtls_cipher_init(&ctx->decryption_context);
-
-    ASSERT_MSG((mbedtls_cipher_setup(
-                    &ctx->encryption_context,
-                    mbedtls_cipher_info_from_type(static_cast<mbedtls_cipher_type_t>(mode))) ||
-                mbedtls_cipher_setup(
-                    &ctx->decryption_context,
-                    mbedtls_cipher_info_from_type(static_cast<mbedtls_cipher_type_t>(mode)))) == 0,
-               "Failed to initialize mbedtls ciphers.");
-
-    ASSERT(
-        !mbedtls_cipher_setkey(&ctx->encryption_context, key.data(), KeySize * 8, MBEDTLS_ENCRYPT));
-    ASSERT(
-        !mbedtls_cipher_setkey(&ctx->decryption_context, key.data(), KeySize * 8, MBEDTLS_DECRYPT));
-    //"Failed to set key on mbedtls ciphers.");
+
+    ctx->encryption_context = EVP_CIPHER_CTX_new();
+    ctx->decryption_context = EVP_CIPHER_CTX_new();
+    ctx->cipher = GetCipher(mode, KeySize);
+    if (ctx->cipher) {
+        EVP_CIPHER_up_ref(ctx->cipher);
+    } else {
+        UNIMPLEMENTED();
+    }
+
+    ASSERT_MSG(ctx->encryption_context && ctx->decryption_context && ctx->cipher,
+               "OpenSSL cipher context failed init!");
+
+    // now init ciphers
+    ASSERT(EVP_CipherInit_ex2(ctx->encryption_context, ctx->cipher, key.data(), NULL, 1, NULL));
+    ASSERT(EVP_CipherInit_ex2(ctx->decryption_context, ctx->cipher, key.data(), NULL, 0, NULL));
+
+    EVP_CIPHER_CTX_set_padding(ctx->encryption_context, 0);
+    EVP_CIPHER_CTX_set_padding(ctx->decryption_context, 0);
 }
 
 template <typename Key, std::size_t KeySize>
 AESCipher<Key, KeySize>::~AESCipher() {
-    mbedtls_cipher_free(&ctx->encryption_context);
-    mbedtls_cipher_free(&ctx->decryption_context);
+    EVP_CIPHER_CTX_free(ctx->encryption_context);
+    EVP_CIPHER_CTX_free(ctx->decryption_context);
+    EVP_CIPHER_free(ctx->cipher);
 }
 
 template <typename Key, std::size_t KeySize>
 void AESCipher<Key, KeySize>::Transcode(const u8* src, std::size_t size, u8* dest, Op op) const {
-    auto* const context = op == Op::Encrypt ? &ctx->encryption_context : &ctx->decryption_context;
-
-    mbedtls_cipher_reset(context);
+    auto* const context = op == Op::Encrypt ? ctx->encryption_context : ctx->decryption_context;
 
     if (size == 0)
         return;
 
-    const auto mode = mbedtls_cipher_get_cipher_mode(context);
-    std::size_t written = 0;
-
-    if (mode != MBEDTLS_MODE_ECB) {
-        const int ret = mbedtls_cipher_update(context, src, size, dest, &written);
-        ASSERT(ret == 0);
-        if (written != size) {
-            LOG_WARNING(Crypto, "Not all data was processed requested={:016X}, actual={:016X}.", size, written);
-        }
-        return;
-    }
+    // reset
+    ASSERT(EVP_CipherInit_ex(context, nullptr, nullptr, nullptr, nullptr, -1));
 
-    const auto block_size = mbedtls_cipher_get_block_size(context);
-    ASSERT(block_size <= AesBlockBytes);
+    const int block_size = EVP_CIPHER_CTX_get_block_size(context);
+    ASSERT(block_size > 0 && block_size <= int(AesBlockBytes));
 
     const std::size_t whole_block_bytes = size - (size % block_size);
+    int written = 0;
+
     if (whole_block_bytes != 0) {
-        const int ret = mbedtls_cipher_update(context, src, whole_block_bytes, dest, &written);
-        ASSERT(ret == 0);
-        if (written != whole_block_bytes) {
+        ASSERT(EVP_CipherUpdate(context, dest, &written, src, static_cast<int>(whole_block_bytes)));
+
+        if (std::size_t(written) != whole_block_bytes) {
             LOG_WARNING(Crypto, "Not all data was processed requested={:016X}, actual={:016X}.",
                         whole_block_bytes, written);
         }
     }
 
+    // tail
     const std::size_t tail = size - whole_block_bytes;
     if (tail == 0)
         return;
@@ -112,13 +150,13 @@ void AESCipher<Key, KeySize>::Transcode(const u8* src, std::size_t size, u8* des
     std::array<u8, AesBlockBytes> tail_buffer{};
     std::memcpy(tail_buffer.data(), src + whole_block_bytes, tail);
 
-    std::size_t tail_written = 0;
-    const int ret = mbedtls_cipher_update(context, tail_buffer.data(), block_size, tail_buffer.data(),
-                                          &tail_written);
-    ASSERT(ret == 0);
+    int tail_written = 0;
+
+    ASSERT(EVP_CipherUpdate(context, tail_buffer.data(), &tail_written, tail_buffer.data(), block_size));
+
     if (tail_written != block_size) {
-        LOG_WARNING(Crypto, "Not all data was processed requested={:016X}, actual={:016X}.", block_size,
-                    tail_written);
+        LOG_WARNING(Crypto, "Tail block not fully processed requested={:016X}, actual={:016X}.",
+                    block_size, tail_written);
     }
 
     std::memcpy(dest + whole_block_bytes, tail_buffer.data(), tail);
@@ -137,9 +175,10 @@ void AESCipher<Key, KeySize>::XTSTranscode(const u8* src, std::size_t size, u8*
 
 template <typename Key, std::size_t KeySize>
 void AESCipher<Key, KeySize>::SetIV(std::span<const u8> data) {
-    ASSERT_MSG((mbedtls_cipher_set_iv(&ctx->encryption_context, data.data(), data.size()) ||
-                mbedtls_cipher_set_iv(&ctx->decryption_context, data.data(), data.size())) == 0,
-               "Failed to set IV on mbedtls ciphers.");
+    const int ret_enc = EVP_CipherInit_ex(ctx->encryption_context, nullptr, nullptr, nullptr, data.data(), -1);
+    const int ret_dec = EVP_CipherInit_ex(ctx->decryption_context, nullptr, nullptr, nullptr, data.data(), -1);
+
+    ASSERT_MSG(ret_enc == 1 && ret_dec == 1, "Failed to set IV on OpenSSL contexts");
 }
 
 template class AESCipher<Key128>;
diff --git a/src/core/crypto/aes_util.h b/src/core/crypto/aes_util.h
index 7cc672635e..a3de95e5e2 100644
--- a/src/core/crypto/aes_util.h
+++ b/src/core/crypto/aes_util.h
@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
@@ -10,7 +10,6 @@
 #include <span>
 #include <type_traits>
 #include "common/common_types.h"
-#include "core/file_sys/vfs/vfs.h"
 
 namespace Core::Crypto {
 
@@ -62,4 +61,5 @@ public:
 private:
     std::unique_ptr<CipherContext> ctx;
 };
+
 } // namespace Core::Crypto
diff --git a/src/core/crypto/key_manager.cpp b/src/core/crypto/key_manager.cpp
index 353274d77b..9a1b9d7a50 100644
--- a/src/core/crypto/key_manager.cpp
+++ b/src/core/crypto/key_manager.cpp
@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
@@ -9,35 +9,26 @@
 #include <bitset>
 #include <cctype>
 #include <fstream>
-#include <locale>
 #include <map>
 #include <sstream>
 #include <tuple>
 #include <vector>
-#include <mbedtls/bignum.h>
-#include <mbedtls/cipher.h>
-#include <mbedtls/cmac.h>
-#include <mbedtls/sha256.h>
+
+#include <openssl/evp.h>
+
 #include "common/fs/file.h"
 #include "common/fs/fs.h"
 #include "common/fs/path_util.h"
 #include "common/hex_util.h"
 #include "common/logging/log.h"
-#include "common/settings.h"
 #include "common/string_util.h"
 #include "core/crypto/aes_util.h"
 #include "core/crypto/key_manager.h"
 #include "core/crypto/partition_data_manager.h"
 #include "core/file_sys/content_archive.h"
-#include "core/file_sys/nca_metadata.h"
 #include "core/file_sys/registered_cache.h"
-#include "core/hle/service/filesystem/filesystem.h"
 #include "core/loader/loader.h"
 
-#ifndef MBEDTLS_CMAC_C
-#error mbedtls was compiled without CMAC support. Check your USE flags (Gentoo) or contact your package maintainer.
-#endif
-
 namespace Core::Crypto {
 namespace {
 
@@ -527,15 +518,27 @@ static std::array<u8, target_size> MGF1(const std::array<u8, in_size>& seed) {
     std::array<u8, in_size + 4> seed_exp{};
     std::memcpy(seed_exp.data(), seed.data(), in_size);
 
+    EVP_MD_CTX* ctx = EVP_MD_CTX_new();
+    const EVP_MD* sha256 = EVP_sha256();
+
     std::vector<u8> out;
     size_t i = 0;
     while (out.size() < target_size) {
-        out.resize(out.size() + 0x20);
-        seed_exp[in_size + 3] = static_cast<u8>(i);
-        mbedtls_sha256(seed_exp.data(), seed_exp.size(), out.data() + out.size() - 0x20, 0);
+        size_t offset = out.size();
+        out.resize(offset + 0x20);
+        seed_exp[in_size + 3] = u8(i);
+
+        u32 hash_len = 0;
+
+        EVP_DigestInit_ex(ctx, sha256, nullptr);
+        EVP_DigestUpdate(ctx, seed_exp.data(), seed_exp.size());
+        EVP_DigestFinal_ex(ctx, out.data() + offset, &hash_len);
+
         ++i;
     }
 
+    EVP_MD_CTX_free(ctx);
+
     std::array<u8, target_size> target;
     std::memcpy(target.data(), out.data(), target_size);
     return target;
@@ -588,32 +591,28 @@ std::optional<Key128> KeyManager::ParseTicketTitleKey(const Ticket& ticket) {
         return std::nullopt;
     }
 
-    mbedtls_mpi D; // RSA Private Exponent
-    mbedtls_mpi N; // RSA Modulus
-    mbedtls_mpi S; // Input
-    mbedtls_mpi M; // Output
-
-    mbedtls_mpi_init(&D);
-    mbedtls_mpi_init(&N);
-    mbedtls_mpi_init(&S);
-    mbedtls_mpi_init(&M);
-
-    const auto& title_key_block = ticket.GetData().title_key_block;
-    mbedtls_mpi_read_binary(&D, eticket_rsa_keypair.decryption_key.data(),
-                            eticket_rsa_keypair.decryption_key.size());
-    mbedtls_mpi_read_binary(&N, eticket_rsa_keypair.modulus.data(),
-                            eticket_rsa_keypair.modulus.size());
-    mbedtls_mpi_read_binary(&S, title_key_block.data(), title_key_block.size());
-
-    mbedtls_mpi_exp_mod(&M, &S, &D, &N, nullptr);
-
     std::array<u8, 0x100> rsa_step;
-    mbedtls_mpi_write_binary(&M, rsa_step.data(), rsa_step.size());
+    {
+        // Private context for OpenSSL bignumbers
+        // Inside block because I dont wanna pollute the space...
+        const auto& title_key_block = ticket.GetData().title_key_block;
+        BIGNUM* D = BN_bin2bn(eticket_rsa_keypair.decryption_key.data(), int(eticket_rsa_keypair.decryption_key.size()), NULL);
+        BIGNUM* N = BN_bin2bn(eticket_rsa_keypair.modulus.data(), int(eticket_rsa_keypair.modulus.size()), NULL);
+        BIGNUM* S = BN_bin2bn(title_key_block.data(), int(title_key_block.size()), NULL);
+        BIGNUM* M = BN_new();
+        // M = S ^ D mod N
+        BN_mod_exp(M, S, D, N, NULL);
+        BN_bn2bin(M, rsa_step.data());
+        BN_free(D);
+        BN_free(N);
+        BN_free(S);
+        BN_free(M);
+    }
 
     u8 m_0 = rsa_step[0];
     std::array<u8, 0x20> m_1;
-    std::memcpy(m_1.data(), rsa_step.data() + 0x01, m_1.size());
     std::array<u8, 0xDF> m_2;
+    std::memcpy(m_1.data(), rsa_step.data() + 0x01, m_1.size());
     std::memcpy(m_2.data(), rsa_step.data() + 0x21, m_2.size());
 
     if (m_0 != 0) {
@@ -954,8 +953,18 @@ void KeyManager::DeriveSDSeedLazy() {
 static Key128 CalculateCMAC(const u8* source, size_t size, const Key128& key) {
     Key128 out{};
 
-    mbedtls_cipher_cmac(mbedtls_cipher_info_from_type(MBEDTLS_CIPHER_AES_128_ECB), key.data(),
-                        key.size() * 8, source, size, out.data());
+    static EVP_MAC* mac = EVP_MAC_fetch(nullptr, "cmac", nullptr);
+    if (!mac) return out;
+
+    static EVP_MAC_CTX* ctx = EVP_MAC_CTX_new(mac);
+    if (!ctx) return out;
+
+    EVP_MAC_init(ctx, key.data(), key.size() * CHAR_BIT, NULL);
+    EVP_MAC_update(ctx, source, size);
+
+    size_t len;
+    EVP_MAC_final(ctx, out.data(), &len, out.size());
+
     return out;
 }
 
diff --git a/src/core/crypto/partition_data_manager.cpp b/src/core/crypto/partition_data_manager.cpp
index e185838e57..4dd3d34fdc 100644
--- a/src/core/crypto/partition_data_manager.cpp
+++ b/src/core/crypto/partition_data_manager.cpp
@@ -1,25 +1,20 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include <algorithm>
 #include <array>
-#include <cctype>
 #include <cstring>
-#include <mbedtls/sha256.h>
 #include "common/common_funcs.h"
 #include "common/common_types.h"
 #include "common/hex_util.h"
-#include "common/logging/log.h"
 #include "common/string_util.h"
 #include "common/swap.h"
 #include "core/crypto/key_manager.h"
 #include "core/crypto/partition_data_manager.h"
 #include "core/crypto/xts_encryption_layer.h"
 #include "core/file_sys/kernel_executable.h"
-#include "core/file_sys/vfs/vfs.h"
 #include "core/file_sys/vfs/vfs_offset.h"
 #include "core/file_sys/vfs/vfs_vector.h"
 #include "core/loader/loader.h"
@@ -255,4 +250,4 @@ std::array<u8, 576> PartitionDataManager::GetETicketExtendedKek() const {
         prodinfo_decrypted->Read(out.data(), out.size(), 0x3890);
     return out;
 }
-} // namespace Core::Crypto
\ No newline at end of file
+} // namespace Core::Crypto