[crypto] Atomize all traces of MbedTLS, and require OpenSSL 3+ (#3606)

Closes #3137
Closes #3465

- Replace all mbedtls usage with OpenSSL
- require OpenSSL
- Up OpenSSL version to 3, cuz that's what we actually need...

CAVEATS:
- httplib also now required
- other ssl backends for svc are unused, maybe remove later
  * To be fair, our CI never used them anyways. And we never tested those

TESTERS PLEASE TEST:
- All games and applets boot
- Boot, load, exit, etc. times

Co-authored-by: crueter <crueter@eden-emu.dev>
Signed-off-by: lizzie <lizzie@eden-emu.dev>
Co-authored-by: crueter <crueter@eden-emu.dev>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3606
Reviewed-by: crueter <crueter@eden-emu.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>
Reviewed-by: DraVee <dravee@eden-emu.dev>
Co-authored-by: lizzie <lizzie@eden-emu.dev>
Co-committed-by: lizzie <lizzie@eden-emu.dev>

2 files changed, 13 insertions, 20 deletions

diff --git a/src/dedicated_room/CMakeLists.txt b/src/dedicated_room/CMakeLists.txt
index eb0b4f399e..36c8af5e34 100644
--- a/src/dedicated_room/CMakeLists.txt
+++ b/src/dedicated_room/CMakeLists.txt
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+# SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 # SPDX-FileCopyrightText: 2017 Citra Emulator Project
@@ -16,7 +16,10 @@ if (ENABLE_WEB_SERVICE)
     target_link_libraries(yuzu-room PRIVATE web_service)
 endif()
 
-target_link_libraries(yuzu-room PRIVATE MbedTLS::mbedcrypto${MBEDTLS_LIB_SUFFIX} MbedTLS::mbedtls${MBEDTLS_LIB_SUFFIX})
+target_link_libraries(yuzu-room PRIVATE
+    OpenSSL::SSL
+    OpenSSL::Crypto)
+
 if (MSVC)
     target_link_libraries(yuzu-room PRIVATE getopt)
 endif()
diff --git a/src/dedicated_room/yuzu_room.cpp b/src/dedicated_room/yuzu_room.cpp
index 880431fa85..126053f43f 100644
--- a/src/dedicated_room/yuzu_room.cpp
+++ b/src/dedicated_room/yuzu_room.cpp
@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: Copyright 2017 Citra Emulator Project
@@ -22,7 +22,7 @@
 #include <shellapi.h>
 #endif
 
-#include <mbedtls/base64.h>
+#include <openssl/evp.h>
 #include "common/common_types.h"
 #include "common/detached_tasks.h"
 #include "common/fs/file.h"
@@ -84,15 +84,11 @@ static constexpr char BanListMagic[] = "YuzuRoom-BanList-1";
 static constexpr char token_delimiter{':'};
 
 static void PadToken(std::string& token) {
-    std::size_t outlen = 0;
-
     std::array<unsigned char, 512> output{};
     std::array<unsigned char, 2048> roundtrip{};
     for (size_t i = 0; i < 3; i++) {
-        mbedtls_base64_decode(output.data(), output.size(), &outlen,
-                              reinterpret_cast<const unsigned char*>(token.c_str()),
-                              token.length());
-        mbedtls_base64_encode(roundtrip.data(), roundtrip.size(), &outlen, output.data(), outlen);
+        EVP_DecodeBlock(output.data(), reinterpret_cast<const unsigned char*>(token.c_str()), token.size());
+        EVP_EncodeBlock(output.data(), roundtrip.data(), roundtrip.size());
         if (memcmp(roundtrip.data(), token.data(), token.size()) == 0) {
             break;
         }
@@ -101,23 +97,17 @@ static void PadToken(std::string& token) {
 }
 
 static std::string UsernameFromDisplayToken(const std::string& display_token) {
-    std::size_t outlen;
-
+    std::size_t outlen = 4 * ((display_token.length() + 2) / 3);
     std::array<unsigned char, 512> output{};
-    mbedtls_base64_decode(output.data(), output.size(), &outlen,
-                          reinterpret_cast<const unsigned char*>(display_token.c_str()),
-                          display_token.length());
+    EVP_DecodeBlock(output.data(), reinterpret_cast<const unsigned char*>(display_token.c_str()), display_token.length());
     std::string decoded_display_token(reinterpret_cast<char*>(&output), outlen);
     return decoded_display_token.substr(0, decoded_display_token.find(token_delimiter));
 }
 
 static std::string TokenFromDisplayToken(const std::string& display_token) {
-    std::size_t outlen;
-
+    std::size_t outlen = 4 * ((display_token.length() + 2) / 3);
     std::array<unsigned char, 512> output{};
-    mbedtls_base64_decode(output.data(), output.size(), &outlen,
-                          reinterpret_cast<const unsigned char*>(display_token.c_str()),
-                          display_token.length());
+    EVP_DecodeBlock(output.data(), reinterpret_cast<const unsigned char*>(display_token.c_str()), display_token.length());
     std::string decoded_display_token(reinterpret_cast<char*>(&output), outlen);
     return decoded_display_token.substr(decoded_display_token.find(token_delimiter) + 1);
 }