[crypto] Atomize all traces of MbedTLS, and require OpenSSL 3+ (#3606)

Closes #3137
Closes #3465

- Replace all mbedtls usage with OpenSSL
- require OpenSSL
- Up OpenSSL version to 3, cuz that's what we actually need...

CAVEATS:
- httplib also now required
- other ssl backends for svc are unused, maybe remove later
  * To be fair, our CI never used them anyways. And we never tested those

TESTERS PLEASE TEST:
- All games and applets boot
- Boot, load, exit, etc. times

Co-authored-by: crueter <crueter@eden-emu.dev>
Signed-off-by: lizzie <lizzie@eden-emu.dev>
Co-authored-by: crueter <crueter@eden-emu.dev>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3606
Reviewed-by: crueter <crueter@eden-emu.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>
Reviewed-by: DraVee <dravee@eden-emu.dev>
Co-authored-by: lizzie <lizzie@eden-emu.dev>
Co-committed-by: lizzie <lizzie@eden-emu.dev>

1 files changed, 101 insertions, 62 deletions

diff --git a/src/core/crypto/aes_util.cpp b/src/core/crypto/aes_util.cpp
index 0820ca30ea..1189e45bd8 100644
--- a/src/core/crypto/aes_util.cpp
+++ b/src/core/crypto/aes_util.cpp
@@ -1,13 +1,13 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include <algorithm>
 #include <array>
 #include <cstring>
-#include <mbedtls/cipher.h>
+#include <openssl/err.h>
+#include <openssl/evp.h>
 #include "common/assert.h"
 #include "common/logging/log.h"
 #include "core/crypto/aes_util.h"
@@ -28,83 +28,121 @@ NintendoTweak CalculateNintendoTweak(std::size_t sector_id) {
 }
 } // Anonymous namespace
 
-static_assert(static_cast<std::size_t>(Mode::CTR) ==
-                  static_cast<std::size_t>(MBEDTLS_CIPHER_AES_128_CTR),
-              "CTR has incorrect value.");
-static_assert(static_cast<std::size_t>(Mode::ECB) ==
-                  static_cast<std::size_t>(MBEDTLS_CIPHER_AES_128_ECB),
-              "ECB has incorrect value.");
-static_assert(static_cast<std::size_t>(Mode::XTS) ==
-                  static_cast<std::size_t>(MBEDTLS_CIPHER_AES_128_XTS),
-              "XTS has incorrect value.");
-
-// Structure to hide mbedtls types from header file
+// Structure to hide OpenSSL types from header file
 struct CipherContext {
-    mbedtls_cipher_context_t encryption_context;
-    mbedtls_cipher_context_t decryption_context;
+    EVP_CIPHER_CTX* encryption_context = nullptr;
+    EVP_CIPHER_CTX* decryption_context = nullptr;
+    EVP_CIPHER* cipher = nullptr;
 };
 
+static inline const std::string GetCipherName(Mode mode, u32 key_size) {
+    std::string cipher;
+    std::size_t effective_bits = key_size * 8;
+
+    switch (mode) {
+    case Mode::CTR:
+        cipher = "CTR";
+        break;
+    case Mode::ECB:
+        cipher = "ECB";
+        break;
+    case Mode::XTS:
+        cipher = "XTS";
+        effective_bits /= 2;
+        break;
+    default:
+        UNREACHABLE();
+    }
+
+    return fmt::format("AES-{}-{}", effective_bits, cipher);
+};
+
+static EVP_CIPHER *GetCipher(Mode mode, u32 key_size) {
+    static auto fetch_cipher = [](Mode m, u32 k) {
+        return EVP_CIPHER_fetch(nullptr, GetCipherName(m, k).c_str(), nullptr);
+    };
+
+    static const struct {
+        EVP_CIPHER* ctr_16 = fetch_cipher(Mode::CTR, 16);
+        EVP_CIPHER* ecb_16 = fetch_cipher(Mode::ECB, 16);
+        EVP_CIPHER* xts_16 = fetch_cipher(Mode::XTS, 16);
+        EVP_CIPHER* ctr_32 = fetch_cipher(Mode::CTR, 32);
+        EVP_CIPHER* ecb_32 = fetch_cipher(Mode::ECB, 32);
+        EVP_CIPHER* xts_32 = fetch_cipher(Mode::XTS, 32);
+    } ciphers = {};
+
+    switch (mode) {
+    case Mode::CTR:
+        return key_size == 16 ? ciphers.ctr_16 : ciphers.ctr_32;
+    case Mode::ECB:
+        return key_size == 16 ? ciphers.ecb_16 : ciphers.ecb_32;
+    case Mode::XTS:
+        return key_size == 16 ? ciphers.xts_16 : ciphers.xts_32;
+    default:
+        UNIMPLEMENTED();
+    }
+
+    return nullptr;
+}
+
+// TODO: WHY TEMPLATE???????
 template <typename Key, std::size_t KeySize>
 Crypto::AESCipher<Key, KeySize>::AESCipher(Key key, Mode mode)
     : ctx(std::make_unique<CipherContext>()) {
-    mbedtls_cipher_init(&ctx->encryption_context);
-    mbedtls_cipher_init(&ctx->decryption_context);
-
-    ASSERT_MSG((mbedtls_cipher_setup(
-                    &ctx->encryption_context,
-                    mbedtls_cipher_info_from_type(static_cast<mbedtls_cipher_type_t>(mode))) ||
-                mbedtls_cipher_setup(
-                    &ctx->decryption_context,
-                    mbedtls_cipher_info_from_type(static_cast<mbedtls_cipher_type_t>(mode)))) == 0,
-               "Failed to initialize mbedtls ciphers.");
-
-    ASSERT(
-        !mbedtls_cipher_setkey(&ctx->encryption_context, key.data(), KeySize * 8, MBEDTLS_ENCRYPT));
-    ASSERT(
-        !mbedtls_cipher_setkey(&ctx->decryption_context, key.data(), KeySize * 8, MBEDTLS_DECRYPT));
-    //"Failed to set key on mbedtls ciphers.");
+
+    ctx->encryption_context = EVP_CIPHER_CTX_new();
+    ctx->decryption_context = EVP_CIPHER_CTX_new();
+    ctx->cipher = GetCipher(mode, KeySize);
+    if (ctx->cipher) {
+        EVP_CIPHER_up_ref(ctx->cipher);
+    } else {
+        UNIMPLEMENTED();
+    }
+
+    ASSERT_MSG(ctx->encryption_context && ctx->decryption_context && ctx->cipher,
+               "OpenSSL cipher context failed init!");
+
+    // now init ciphers
+    ASSERT(EVP_CipherInit_ex2(ctx->encryption_context, ctx->cipher, key.data(), NULL, 1, NULL));
+    ASSERT(EVP_CipherInit_ex2(ctx->decryption_context, ctx->cipher, key.data(), NULL, 0, NULL));
+
+    EVP_CIPHER_CTX_set_padding(ctx->encryption_context, 0);
+    EVP_CIPHER_CTX_set_padding(ctx->decryption_context, 0);
 }
 
 template <typename Key, std::size_t KeySize>
 AESCipher<Key, KeySize>::~AESCipher() {
-    mbedtls_cipher_free(&ctx->encryption_context);
-    mbedtls_cipher_free(&ctx->decryption_context);
+    EVP_CIPHER_CTX_free(ctx->encryption_context);
+    EVP_CIPHER_CTX_free(ctx->decryption_context);
+    EVP_CIPHER_free(ctx->cipher);
 }
 
 template <typename Key, std::size_t KeySize>
 void AESCipher<Key, KeySize>::Transcode(const u8* src, std::size_t size, u8* dest, Op op) const {
-    auto* const context = op == Op::Encrypt ? &ctx->encryption_context : &ctx->decryption_context;
-
-    mbedtls_cipher_reset(context);
+    auto* const context = op == Op::Encrypt ? ctx->encryption_context : ctx->decryption_context;
 
     if (size == 0)
         return;
 
-    const auto mode = mbedtls_cipher_get_cipher_mode(context);
-    std::size_t written = 0;
-
-    if (mode != MBEDTLS_MODE_ECB) {
-        const int ret = mbedtls_cipher_update(context, src, size, dest, &written);
-        ASSERT(ret == 0);
-        if (written != size) {
-            LOG_WARNING(Crypto, "Not all data was processed requested={:016X}, actual={:016X}.", size, written);
-        }
-        return;
-    }
+    // reset
+    ASSERT(EVP_CipherInit_ex(context, nullptr, nullptr, nullptr, nullptr, -1));
 
-    const auto block_size = mbedtls_cipher_get_block_size(context);
-    ASSERT(block_size <= AesBlockBytes);
+    const int block_size = EVP_CIPHER_CTX_get_block_size(context);
+    ASSERT(block_size > 0 && block_size <= int(AesBlockBytes));
 
     const std::size_t whole_block_bytes = size - (size % block_size);
+    int written = 0;
+
     if (whole_block_bytes != 0) {
-        const int ret = mbedtls_cipher_update(context, src, whole_block_bytes, dest, &written);
-        ASSERT(ret == 0);
-        if (written != whole_block_bytes) {
+        ASSERT(EVP_CipherUpdate(context, dest, &written, src, static_cast<int>(whole_block_bytes)));
+
+        if (std::size_t(written) != whole_block_bytes) {
             LOG_WARNING(Crypto, "Not all data was processed requested={:016X}, actual={:016X}.",
                         whole_block_bytes, written);
         }
     }
 
+    // tail
     const std::size_t tail = size - whole_block_bytes;
     if (tail == 0)
         return;
@@ -112,13 +150,13 @@ void AESCipher<Key, KeySize>::Transcode(const u8* src, std::size_t size, u8* des
     std::array<u8, AesBlockBytes> tail_buffer{};
     std::memcpy(tail_buffer.data(), src + whole_block_bytes, tail);
 
-    std::size_t tail_written = 0;
-    const int ret = mbedtls_cipher_update(context, tail_buffer.data(), block_size, tail_buffer.data(),
-                                          &tail_written);
-    ASSERT(ret == 0);
+    int tail_written = 0;
+
+    ASSERT(EVP_CipherUpdate(context, tail_buffer.data(), &tail_written, tail_buffer.data(), block_size));
+
     if (tail_written != block_size) {
-        LOG_WARNING(Crypto, "Not all data was processed requested={:016X}, actual={:016X}.", block_size,
-                    tail_written);
+        LOG_WARNING(Crypto, "Tail block not fully processed requested={:016X}, actual={:016X}.",
+                    block_size, tail_written);
     }
 
     std::memcpy(dest + whole_block_bytes, tail_buffer.data(), tail);
@@ -137,9 +175,10 @@ void AESCipher<Key, KeySize>::XTSTranscode(const u8* src, std::size_t size, u8*
 
 template <typename Key, std::size_t KeySize>
 void AESCipher<Key, KeySize>::SetIV(std::span<const u8> data) {
-    ASSERT_MSG((mbedtls_cipher_set_iv(&ctx->encryption_context, data.data(), data.size()) ||
-                mbedtls_cipher_set_iv(&ctx->decryption_context, data.data(), data.size())) == 0,
-               "Failed to set IV on mbedtls ciphers.");
+    const int ret_enc = EVP_CipherInit_ex(ctx->encryption_context, nullptr, nullptr, nullptr, data.data(), -1);
+    const int ret_dec = EVP_CipherInit_ex(ctx->decryption_context, nullptr, nullptr, nullptr, data.data(), -1);
+
+    ASSERT_MSG(ret_enc == 1 && ret_dec == 1, "Failed to set IV on OpenSSL contexts");
 }
 
 template class AESCipher<Key128>;