[crypto] Atomize all traces of MbedTLS, and require OpenSSL 3+ (#3606)

Closes #3137
Closes #3465

- Replace all mbedtls usage with OpenSSL
- require OpenSSL
- Up OpenSSL version to 3, cuz that's what we actually need...

CAVEATS:
- httplib also now required
- other ssl backends for svc are unused, maybe remove later
  * To be fair, our CI never used them anyways. And we never tested those

TESTERS PLEASE TEST:
- All games and applets boot
- Boot, load, exit, etc. times

Co-authored-by: crueter <crueter@eden-emu.dev>
Signed-off-by: lizzie <lizzie@eden-emu.dev>
Co-authored-by: crueter <crueter@eden-emu.dev>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3606
Reviewed-by: crueter <crueter@eden-emu.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>
Reviewed-by: DraVee <dravee@eden-emu.dev>
Co-authored-by: lizzie <lizzie@eden-emu.dev>
Co-committed-by: lizzie <lizzie@eden-emu.dev>

2 files changed, 57 insertions, 25 deletions

diff --git a/src/core/file_sys/registered_cache.cpp b/src/core/file_sys/registered_cache.cpp
index 1b2c45176b..7bf2ad8fcd 100644
--- a/src/core/file_sys/registered_cache.cpp
+++ b/src/core/file_sys/registered_cache.cpp
@@ -7,12 +7,11 @@
 #include <algorithm>
 #include <random>
 #include <regex>
-#include <mbedtls/sha256.h>
+#include <openssl/evp.h>
 #include "common/assert.h"
 #include "common/fs/path_util.h"
 #include "common/hex_util.h"
 #include "common/logging/log.h"
-#include "common/scope_exit.h"
 #include "common/string_util.h"
 #include "core/crypto/key_manager.h"
 #include "core/file_sys/card_image.h"
@@ -64,17 +63,23 @@ static bool FollowsNcaIdFormat(std::string_view name) {
 
 static std::string GetRelativePathFromNcaID(const std::array<u8, 16>& nca_id, bool second_hex_upper,
                                             bool within_two_digit, bool cnmt_suffix) {
+    const auto nca_str = Common::HexToString(nca_id, second_hex_upper);
+
     if (!within_two_digit) {
         const auto format_str = fmt::runtime(cnmt_suffix ? "{}.cnmt.nca" : "/{}.nca");
-        return fmt::format(format_str, Common::HexToString(nca_id, second_hex_upper));
+        return fmt::format(format_str, nca_str);
     }
 
     Core::Crypto::SHA256Hash hash{};
-    mbedtls_sha256(nca_id.data(), nca_id.size(), hash.data(), 0);
+    u32 hash_len = 0;
+    EVP_Digest(nca_id.data(), nca_id.size(), hash.data(), &hash_len, EVP_sha256(), nullptr);
 
     const auto format_str =
         fmt::runtime(cnmt_suffix ? "/000000{:02X}/{}.cnmt.nca" : "/000000{:02X}/{}.nca");
-    return fmt::format(format_str, hash[0], Common::HexToString(nca_id, second_hex_upper));
+
+    LOG_DEBUG(Loader, "Decoded {} bytes, nca id {}", hash_len, nca_str);
+
+    return fmt::format(format_str, hash[0], nca_str);
 }
 
 static std::string GetCNMTName(TitleType type, u64 title_id) {
@@ -152,7 +157,11 @@ bool PlaceholderCache::Create(const NcaID& id, u64 size) const {
     }
 
     Core::Crypto::SHA256Hash hash{};
-    mbedtls_sha256(id.data(), id.size(), hash.data(), 0);
+    u32 hash_len = 0;
+    EVP_Digest(id.data(), id.size(), hash.data(), &hash_len, EVP_sha256(), nullptr);
+
+    LOG_DEBUG(Loader, "Decoded {} bytes, nca id {}", hash_len, id);
+
     const auto dirname = fmt::format("000000{:02X}", hash[0]);
 
     const auto dir2 = GetOrCreateDirectoryRelative(dir, dirname);
@@ -176,7 +185,11 @@ bool PlaceholderCache::Delete(const NcaID& id) const {
     }
 
     Core::Crypto::SHA256Hash hash{};
-    mbedtls_sha256(id.data(), id.size(), hash.data(), 0);
+    u32 hash_len = 0;
+    EVP_Digest(id.data(), id.size(), hash.data(), &hash_len, EVP_sha256(), nullptr);
+
+    LOG_DEBUG(Loader, "Decoded {} bytes, nca id {}", hash_len, id);
+
     const auto dirname = fmt::format("000000{:02X}", hash[0]);
 
     const auto dir2 = GetOrCreateDirectoryRelative(dir, dirname);
@@ -670,7 +683,12 @@ InstallResult RegisteredCache::InstallEntry(const NCA& nca, TitleType type,
     const OptionalHeader opt_header{0, 0};
     ContentRecord c_rec{{}, {}, {}, GetCRTypeFromNCAType(nca.GetType()), {}};
     const auto& data = nca.GetBaseFile()->ReadBytes(0x100000);
-    mbedtls_sha256(data.data(), data.size(), c_rec.hash.data(), 0);
+
+    u32 hash_len = 0;
+    EVP_Digest(data.data(), data.size(), c_rec.hash.data(), &hash_len, EVP_sha256(), nullptr);
+
+    LOG_DEBUG(Loader, "Decoded {} bytes, nca {}", hash_len, nca.GetName());
+
     std::memcpy(&c_rec.nca_id, &c_rec.hash, 16);
     const CNMT new_cnmt(header, opt_header, {c_rec}, {});
     if (!RawInstallYuzuMeta(new_cnmt)) {
@@ -781,7 +799,12 @@ InstallResult RegisteredCache::RawInstallNCA(const NCA& nca, const VfsCopyFuncti
         id = *override_id;
     } else {
         const auto& data = in->ReadBytes(0x100000);
-        mbedtls_sha256(data.data(), data.size(), hash.data(), 0);
+
+        u32 hash_len = 0;
+        EVP_Digest(data.data(), data.size(), hash.data(), &hash_len, EVP_sha256(), nullptr);
+
+        LOG_DEBUG(Loader, "Decoded {} bytes, nca {}", hash_len, nca.GetName());
+
         memcpy(id.data(), hash.data(), 16);
     }
 
diff --git a/src/core/file_sys/xts_archive.cpp b/src/core/file_sys/xts_archive.cpp
index c1912b2bda..7c5da5e7e3 100644
--- a/src/core/file_sys/xts_archive.cpp
+++ b/src/core/file_sys/xts_archive.cpp
@@ -1,17 +1,16 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include <algorithm>
 #include <array>
 #include <cstring>
 #include <regex>
 #include <string>
 
-#include <mbedtls/md.h>
-#include <mbedtls/sha256.h>
+#include <openssl/err.h>
+#include <openssl/evp.h>
 
 #include "common/fs/path_util.h"
 #include "common/hex_util.h"
@@ -31,19 +30,24 @@ constexpr u64 NAX_HEADER_PADDING_SIZE = 0x4000;
 template <typename SourceData, typename SourceKey, typename Destination>
 static bool CalculateHMAC256(Destination* out, const SourceKey* key, std::size_t key_length,
                              const SourceData* data, std::size_t data_length) {
-    mbedtls_md_context_t context;
-    mbedtls_md_init(&context);
-
-    if (mbedtls_md_setup(&context, mbedtls_md_info_from_type(MBEDTLS_MD_SHA256), 1) ||
-        mbedtls_md_hmac_starts(&context, reinterpret_cast<const u8*>(key), key_length) ||
-        mbedtls_md_hmac_update(&context, reinterpret_cast<const u8*>(data), data_length) ||
-        mbedtls_md_hmac_finish(&context, reinterpret_cast<u8*>(out))) {
-        mbedtls_md_free(&context);
+    size_t out_len = 0;
+
+    static EVP_MAC* mac = EVP_MAC_fetch(nullptr, "HMAC", nullptr);
+    if (!mac) return false;
+
+    static EVP_MAC_CTX* ctx = EVP_MAC_CTX_new(mac);
+    if (!ctx) return false;
+
+    static OSSL_PARAM params[] = {
+        OSSL_PARAM_construct_utf8_string("digest", (char*)"SHA256", 0),
+        OSSL_PARAM_construct_end()
+    };
+
+    if (!EVP_MAC_init(ctx, reinterpret_cast<const unsigned char*>(key), key_length, params))
         return false;
-    }
 
-    mbedtls_md_free(&context);
-    return true;
+    return EVP_MAC_update(ctx, reinterpret_cast<const unsigned char*>(data), data_length) &&
+           EVP_MAC_final(ctx, reinterpret_cast<unsigned char*>(out), &out_len, 32);
 }
 
 NAX::NAX(VirtualFile file_)
@@ -68,7 +72,12 @@ NAX::NAX(VirtualFile file_, std::array<u8, 0x10> nca_id)
     : header(std::make_unique<NAXHeader>()),
       file(std::move(file_)), keys{Core::Crypto::KeyManager::Instance()} {
     Core::Crypto::SHA256Hash hash{};
-    mbedtls_sha256(nca_id.data(), nca_id.size(), hash.data(), 0);
+
+    u32 hash_len = 0;
+    EVP_Digest(nca_id.data(), nca_id.size(), hash.data(), &hash_len, EVP_sha256(), nullptr);
+
+    LOG_DEBUG(Loader, "Decoded {} bytes, nca id {}", hash_len, nca_id);
+
     status = Parse(fmt::format("/registered/000000{:02X}/{}.nca", hash[0],
                                Common::HexToString(nca_id, false)));
 }