[crypto] Atomize all traces of MbedTLS, and require OpenSSL 3+ (#3606)

Closes #3137
Closes #3465

- Replace all mbedtls usage with OpenSSL
- require OpenSSL
- Up OpenSSL version to 3, cuz that's what we actually need...

CAVEATS:
- httplib also now required
- other ssl backends for svc are unused, maybe remove later
  * To be fair, our CI never used them anyways. And we never tested those

TESTERS PLEASE TEST:
- All games and applets boot
- Boot, load, exit, etc. times

Co-authored-by: crueter <crueter@eden-emu.dev>
Signed-off-by: lizzie <lizzie@eden-emu.dev>
Co-authored-by: crueter <crueter@eden-emu.dev>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3606
Reviewed-by: crueter <crueter@eden-emu.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>
Reviewed-by: DraVee <dravee@eden-emu.dev>
Co-authored-by: lizzie <lizzie@eden-emu.dev>
Co-committed-by: lizzie <lizzie@eden-emu.dev>

1 files changed, 25 insertions, 16 deletions

diff --git a/src/core/file_sys/xts_archive.cpp b/src/core/file_sys/xts_archive.cpp
index c1912b2bda..7c5da5e7e3 100644
--- a/src/core/file_sys/xts_archive.cpp
+++ b/src/core/file_sys/xts_archive.cpp
@@ -1,17 +1,16 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include <algorithm>
 #include <array>
 #include <cstring>
 #include <regex>
 #include <string>
 
-#include <mbedtls/md.h>
-#include <mbedtls/sha256.h>
+#include <openssl/err.h>
+#include <openssl/evp.h>
 
 #include "common/fs/path_util.h"
 #include "common/hex_util.h"
@@ -31,19 +30,24 @@ constexpr u64 NAX_HEADER_PADDING_SIZE = 0x4000;
 template <typename SourceData, typename SourceKey, typename Destination>
 static bool CalculateHMAC256(Destination* out, const SourceKey* key, std::size_t key_length,
                              const SourceData* data, std::size_t data_length) {
-    mbedtls_md_context_t context;
-    mbedtls_md_init(&context);
-
-    if (mbedtls_md_setup(&context, mbedtls_md_info_from_type(MBEDTLS_MD_SHA256), 1) ||
-        mbedtls_md_hmac_starts(&context, reinterpret_cast<const u8*>(key), key_length) ||
-        mbedtls_md_hmac_update(&context, reinterpret_cast<const u8*>(data), data_length) ||
-        mbedtls_md_hmac_finish(&context, reinterpret_cast<u8*>(out))) {
-        mbedtls_md_free(&context);
+    size_t out_len = 0;
+
+    static EVP_MAC* mac = EVP_MAC_fetch(nullptr, "HMAC", nullptr);
+    if (!mac) return false;
+
+    static EVP_MAC_CTX* ctx = EVP_MAC_CTX_new(mac);
+    if (!ctx) return false;
+
+    static OSSL_PARAM params[] = {
+        OSSL_PARAM_construct_utf8_string("digest", (char*)"SHA256", 0),
+        OSSL_PARAM_construct_end()
+    };
+
+    if (!EVP_MAC_init(ctx, reinterpret_cast<const unsigned char*>(key), key_length, params))
         return false;
-    }
 
-    mbedtls_md_free(&context);
-    return true;
+    return EVP_MAC_update(ctx, reinterpret_cast<const unsigned char*>(data), data_length) &&
+           EVP_MAC_final(ctx, reinterpret_cast<unsigned char*>(out), &out_len, 32);
 }
 
 NAX::NAX(VirtualFile file_)
@@ -68,7 +72,12 @@ NAX::NAX(VirtualFile file_, std::array<u8, 0x10> nca_id)
     : header(std::make_unique<NAXHeader>()),
       file(std::move(file_)), keys{Core::Crypto::KeyManager::Instance()} {
     Core::Crypto::SHA256Hash hash{};
-    mbedtls_sha256(nca_id.data(), nca_id.size(), hash.data(), 0);
+
+    u32 hash_len = 0;
+    EVP_Digest(nca_id.data(), nca_id.size(), hash.data(), &hash_len, EVP_sha256(), nullptr);
+
+    LOG_DEBUG(Loader, "Decoded {} bytes, nca id {}", hash_len, nca_id);
+
     status = Parse(fmt::format("/registered/000000{:02X}/{}.nca", hash[0],
                                Common::HexToString(nca_id, false)));
 }