[file_sys] robust-er IPSwitch compiler (#3911)

this PR reworks IPS parser to be less stupid
what do i mean by this? well generally give it a bit of
love so it doesn't do a lot of unsound allocations
also simplify the logic greatly (and use memcmp() string idiom
instead of trusting the compiler so blindly...)

no this doesn't mean to uber optimize IPS to handle 999 gb/s
it's more so it doesn't outright crash with edge cases
as the previous codebase was quite spaghetty

also the major overhead is obviously the vector shenanigans
and the I/O -- but thats out of scope

Test that IPSwitch mods still properly work WITH ANY GAME
IF THERE IS ANY REGRESSION IN SOME GAME/MOD THEN
TELL ME

Signed-off-by: lizzie <lizzie@eden-emu.dev>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3911
Reviewed-by: crueter <crueter@eden-emu.dev>
Reviewed-by: CamilleLaVey <camillelavey99@gmail.com>

src/core/file_sys/ips_layer.cpp

@@ -3,10 +3,12 @@
 
 #include <algorithm>
 #include <cstring>
-#include <map>
 #include <sstream>
 #include <string>
 #include <utility>
+#include <span>
+#include <cctype>
+#include <ankerl/unordered_dense.h>
 
 #include "common/hex_util.h"
 #include "common/logging.h"
@@ -22,61 +24,30 @@ enum class IPSFileType {
     Error,
 };
 
-constexpr std::array<std::pair<const char*, const char*>, 11> ESCAPE_CHARACTER_MAP{{
+static IPSFileType IdentifyMagic(std::span<const u8> magic) {
-    {"\\a", "\a"},
+    if (magic.size() >= 5) {
-    {"\\b", "\b"},
+        if (std::memcmp(magic.data(), "PATCH", 5) == 0)
-    {"\\f", "\f"},
+            return IPSFileType::IPS;
-    {"\\n", "\n"},
+        if (std::memcmp(magic.data(), "IPS32", 5) == 0)
-    {"\\r", "\r"},
+            return IPSFileType::IPS32;
-    {"\\t", "\t"},
-    {"\\v", "\v"},
-    {"\\\\", "\\"},
-    {"\\\'", "\'"},
-    {"\\\"", "\""},
-    {"\\\?", "\?"},
-}};
-
-static IPSFileType IdentifyMagic(const std::vector<u8>& magic) {
-    if (magic.size() != 5) {
-        return IPSFileType::Error;
     }
-
-    static constexpr std::array<u8, 5> patch_magic{{'P', 'A', 'T', 'C', 'H'}};
-    if (std::equal(magic.begin(), magic.end(), patch_magic.begin())) {
-        return IPSFileType::IPS;
-    }
-
-    static constexpr std::array<u8, 5> ips32_magic{{'I', 'P', 'S', '3', '2'}};
-    if (std::equal(magic.begin(), magic.end(), ips32_magic.begin())) {
-        return IPSFileType::IPS32;
-    }
-
     return IPSFileType::Error;
 }
 
-static bool IsEOF(IPSFileType type, const std::vector<u8>& data) {
+static bool IsEOF(IPSFileType type, std::span<const u8> magic) {
-    static constexpr std::array<u8, 3> eof{{'E', 'O', 'F'}};
+    return (type == IPSFileType::IPS && magic.size() > 3 && std::memcmp(magic.data(), "EOF", 3) == 0)
-    if (type == IPSFileType::IPS && std::equal(data.begin(), data.end(), eof.begin())) {
+        || (type == IPSFileType::IPS32 && magic.size() > 4 && std::memcmp(magic.data(), "EEOF", 4) == 0);
-        return true;
-    }
-
-    static constexpr std::array<u8, 4> eeof{{'E', 'E', 'O', 'F'}};
-    return type == IPSFileType::IPS32 && std::equal(data.begin(), data.end(), eeof.begin());
 }
 
 VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) {
     if (in == nullptr || ips == nullptr)
         return nullptr;
 
-    const auto type = IdentifyMagic(ips->ReadBytes(0x5));
+    auto in_data = in->ReadAllBytes();
+    auto const type = IdentifyMagic(in_data);
     if (type == IPSFileType::Error)
         return nullptr;
 
-    auto in_data = in->ReadAllBytes();
-    if (in_data.size() == 0) {
-        return nullptr;
-    }
-
     std::vector<u8> temp(type == IPSFileType::IPS ? 3 : 4);
     u64 offset = 5; // After header
     while (ips->Read(temp.data(), temp.size(), offset) == temp.size()) {
@@ -85,12 +56,9 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) {
             break;
         }
 
-        u32 real_offset{};
+        u32 real_offset = (type == IPSFileType::IPS32)
-        if (type == IPSFileType::IPS32)
+            ? ((temp[0] << 24) | (temp[1] << 16) | (temp[2] << 8) | temp[3])
-            real_offset = (temp[0] << 24) | (temp[1] << 16) | (temp[2] << 8) | temp[3];
+            : ((temp[0] << 16) | (temp[1] << 8) | temp[2]);
-        else
-            real_offset = (temp[0] << 16) | (temp[1] << 8) | temp[2];
-
         if (real_offset > in_data.size()) {
             return nullptr;
         }
@@ -113,34 +81,35 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) {
                 return nullptr;
 
             if (real_offset + rle_size > in_data.size())
-                rle_size = static_cast<u16>(in_data.size() - real_offset);
+                rle_size = u16(in_data.size() - real_offset);
             std::memset(in_data.data() + real_offset, *data, rle_size);
         } else { // Standard Patch
             auto read = data_size;
             if (real_offset + read > in_data.size())
-                read = static_cast<u16>(in_data.size() - real_offset);
+                read = u16(in_data.size() - real_offset);
             if (ips->Read(in_data.data() + real_offset, read, offset) != data_size)
                 return nullptr;
             offset += data_size;
         }
     }
-
+    if (IsEOF(type, temp)) {
-    if (!IsEOF(type, temp)) {
+        return std::make_shared<VectorVfsFile>(std::move(in_data), in->GetName(), in->GetContainingDirectory());
-        return nullptr;
     }
-
+    return nullptr;
-    return std::make_shared<VectorVfsFile>(std::move(in_data), in->GetName(),
-                                           in->GetContainingDirectory());
 }
 
+
+struct IPSwitchRecord {
+    std::array<uint8_t, 256 - sizeof(size_t)> data;
+    size_t count;
+};
 struct IPSwitchCompiler::IPSwitchPatch {
-    std::string name;
+    ankerl::unordered_dense::map<u32, IPSwitchRecord> records;
     bool enabled;
-    std::map<u32, std::vector<u8>> records;
 };
 
 IPSwitchCompiler::IPSwitchCompiler(VirtualFile patch_text_) : patch_text(std::move(patch_text_)) {
-    Parse();
+    Parse(patch_text->ReadAllBytes());
 }
 
 IPSwitchCompiler::~IPSwitchCompiler() = default;
@@ -149,201 +118,159 @@ std::array<u8, 32> IPSwitchCompiler::GetBuildID() const {
     return nso_build_id;
 }
 
-bool IPSwitchCompiler::IsValid() const {
+static IPSwitchRecord EscapeStringSequences(std::string_view sv) {
-    return valid;
+    IPSwitchRecord r{};
-}
+    for (auto it = sv.cbegin(); it != sv.cend(); ) {
-
+        if (*it == '\\' && it + 1 < sv.cend()) {
-static bool StartsWith(std::string_view base, std::string_view check) {
+            switch (it[1]) {
-    return base.size() >= check.size() && base.substr(0, check.size()) == check;
+            case 'n': r.data[r.count] = '\n'; break;
-}
+            case 't': r.data[r.count] = '\t'; break;
-
+            case 'b': r.data[r.count] = '\b'; break;
-static std::string EscapeStringSequences(std::string in) {
+            case 'r': r.data[r.count] = '\r'; break;
-    for (const auto& seq : ESCAPE_CHARACTER_MAP) {
+            case 'e': r.data[r.count] = '\e'; break;
-        for (auto index = in.find(seq.first); index != std::string::npos;
+            case 'v': r.data[r.count] = '\v'; break;
-             index = in.find(seq.first, index)) {
+            case '?': r.data[r.count] = '\?'; break;
-            in.replace(index, std::strlen(seq.first), seq.second);
+            default: r.data[r.count] = it[1]; break;
-            index += std::strlen(seq.second);
+            }
+            ++r.count;
+            it += 2;
+        } else {
+            ++r.count;
+            ++it;
         }
     }
-
+    return r;
-    return in;
 }
 
-void IPSwitchCompiler::ParseFlag(const std::string& line) {
+[[nodiscard]] static inline std::array<u8, 32> ReadNSOBuildId(std::string_view const s) {
-    if (StartsWith(line, "@flag offset_shift ")) {
+    std::array<u8, 32> r{};
-        // Offset Shift Flag
+    for (std::size_t i = 0; i < s.size(); ++i)
-        offset_shift = std::strtoll(line.substr(19).c_str(), nullptr, 0);
+        r[i / 2] |= u8(u8(Common::ToHexNibble(s[i])) << u8((i % 2) * 4));
-    } else if (StartsWith(line, "@little-endian")) {
+    return r;
-        // Set values to read as little endian
-        is_little_endian = true;
-    } else if (StartsWith(line, "@big-endian")) {
-        // Set values to read as big endian
-        is_little_endian = false;
-    } else if (StartsWith(line, "@flag print_values")) {
-        // Force printing of applied values
-        print_values = true;
-    }
 }
 
-void IPSwitchCompiler::Parse() {
+void IPSwitchCompiler::Parse(std::span<u8 const> bytes) {
-    const auto bytes = patch_text->ReadAllBytes();
+    LOG_INFO(Loader, "IPSwitchCompiler: '{}'", patch_text->GetName());
-    std::stringstream s;
+    bool is_little_endian = false;
-    s.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+    s64 offset_shift = 0;
+    //bool print_values = false;
 
-    std::vector<std::string> lines;
+    auto const parse_line = [&](std::string_view const line) {
-    std::string stream_line;
+        // Keep in mind lines have trimmed spaces (at the end & start)!
-    while (std::getline(s, stream_line)) {
+        LOG_INFO(Loader, "<{}>", line);
-        // Remove a trailing \r
+        if (line.starts_with("@stop")) {
-        if (!stream_line.empty() && stream_line.back() == '\r')
+            return false; // Force stop
-            stream_line.pop_back();
+        } else if (line.starts_with("@nsobid-")) { // NSO Build ID Specifier
-        lines.push_back(std::move(stream_line));
+            nso_build_id = ReadNSOBuildId(line.substr(8));
-    }
+        } else if (line.starts_with("@enabled")) {
+            patches.push_back({{}, true}); //enabled patch
+        } else if (line.starts_with("@disabled")) {
+            patches.push_back({{}, false}); //disabled patch
+        } else if (line.starts_with("@flag offset_shift ")) {
+            offset_shift = std::strtoll(line.data() + 19, nullptr, 0);  // Offset Shift Flag
+        } else if (line.starts_with("@little-endian")) {
+            is_little_endian = true;  // Set values to read as little endian
+        } else if (line.starts_with("@big-endian")) {
+            is_little_endian = false;  // Set values to read as big endian
+        } else if (line.starts_with("@flag print_values")) {
+            //print_values = true; // Force printing of applied values
+        } else if (line.starts_with("@")) {
+            LOG_WARNING(Loader, "Unknown flag {}", line);
+        } else {
+            size_t offset = size_t(std::strtoul(line.data(), nullptr, 16));
+            offset += size_t(offset_shift);
+            if (auto const first_quote = line.find_first_of("\"\'"); first_quote != std::string::npos) {
+                // string replacement
+                char quote = line[first_quote];
+                auto const start = line.cbegin() + first_quote + 1;
+                auto end = start;
+                for (; end < line.cend() && *end != quote; )
+                    end += (*end == '\\') ? 2 : 1;
+                if (start <= line.cend() && end <= line.cend()) {
+                    LOG_INFO(Loader, "[S] value @ {:#08X} ", offset);
+                    patches.back().records.insert_or_assign(u32(offset), EscapeStringSequences({start, end}));
+                } else {
+                    LOG_WARNING(Loader, "invalid string");
+                }
+            } else if (auto const first_space = line.find_last_of(" /\t\r\n"); first_space != std::string::npos) {
+                IPSwitchRecord r{}; // hex replacement
+                auto const start = line.cbegin() + first_space + 1;
+                auto const end = line.cend();
+                if (start <= line.cend() && end <= line.cend()) {
+                    auto const hs = Common::HexStringToVector({start, end}, is_little_endian);
+                    std::memcpy(r.data.data(), hs.data(), hs.size());
+                    r.count = hs.size();
+                    LOG_INFO(Loader, "[H] value @ {:#08X}", offset);
+                    patches.back().records.insert_or_assign(u32(offset), std::move(r));
+                } else {
+                    LOG_WARNING(Loader, "invalid line");
+                }
+            } else {
+                LOG_WARNING(Loader, "unhandled line!");
+            }
+        }
+        return true; //continue
+    };
 
-    for (std::size_t i = 0; i < lines.size(); ++i) {
+    for (auto it = bytes.begin(); it < bytes.end(); ) {
-        auto line = lines[i];
+        auto const start = it;
-
+        auto end = start;
-        // Remove midline comments
+        for (; end < bytes.end() && *end != '\n' && *end != '\r'; ++end)
-        std::size_t comment_index = std::string::npos;
+            ;
-        bool within_string = false;
+        it = end + 1; //prepare for next line
-        for (std::size_t k = 0; k < line.size(); ++k) {
+        std::string_view const sline{
-            if (line[k] == '\"' && (k > 0 && line[k - 1] != '\\')) {
+            reinterpret_cast<const char*>(bytes.data() + std::distance(bytes.begin(), start)),
-                within_string = !within_string;
+            size_t(std::distance(start, end))
-            } else if (line[k] == '\\' && (k < line.size() - 1 && line[k + 1] == '\\')) {
+        };
-                comment_index = k;
+        if (sline.size() > 0) {
+            auto p = sline.cbegin();
+            // skip space off line
+            for (; p < sline.cend() && std::isspace(*p); ++p)
+                ;
+            // now make a nominal preprocessed line: remove comments
+            char quote = '\0';
+            auto const sline_start = p;
+            for (; p < sline.cend(); ) {
+                if ((!quote && p + 1 < sline.cend() && p[0] == '/' && p[1] == '/')
+                || (!quote && p[0] == '#')) {
+                    break;
+                } else if (p[0] == '\"' || p[0] == '\'') {
+                    quote = (p[0] == quote) ? '\0' : p[0];
+                    ++p;
+                } else if (p + 1 < sline.cend() && p[0] == '\\') {
+                    p += 2;
+                } else {
+                    ++p;
+                }
+            }
+            // now we have the preprocessed string ;)
+            std::string_view pp_str(sline_start, p);
+            if (pp_str.size() > 0 && !parse_line(pp_str)) {
                 break;
             }
         }
-
-        if (!StartsWith(line, "//") && comment_index != std::string::npos) {
-            last_comment = line.substr(comment_index + 2);
-            line = line.substr(0, comment_index);
-        }
-
-        if (StartsWith(line, "@stop")) {
-            // Force stop
-            break;
-        } else if (StartsWith(line, "@nsobid-")) {
-            // NSO Build ID Specifier
-            const auto raw_build_id = fmt::format("{:0<64}", line.substr(8));
-            nso_build_id = Common::HexStringToArray<0x20>(raw_build_id);
-        } else if (StartsWith(line, "#")) {
-            // Mandatory Comment
-            LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Forced output comment: {}",
-                     patch_text->GetName(), line.substr(1));
-        } else if (StartsWith(line, "//")) {
-            // Normal Comment
-            last_comment = line.substr(2);
-            if (last_comment.find_first_not_of(' ') == std::string::npos)
-                continue;
-            if (last_comment.find_first_not_of(' ') != 0)
-                last_comment = last_comment.substr(last_comment.find_first_not_of(' '));
-        } else if (StartsWith(line, "@enabled") || StartsWith(line, "@disabled")) {
-            // Start of patch
-            const auto enabled = StartsWith(line, "@enabled");
-            if (i == 0)
-                return;
-            LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Parsing patch '{}' ({})",
-                     patch_text->GetName(), last_comment, line.substr(1));
-
-            IPSwitchPatch patch{last_comment, enabled, {}};
-
-            // Read rest of patch
-            while (true) {
-                if (i + 1 >= lines.size()) {
-                    break;
-                }
-
-                const auto& patch_line = lines[++i];
-
-                // Patch line may contain comments
-                if (StartsWith(patch_line, "//") || StartsWith(patch_line, "#")) {
-                    continue;
-                }
-
-                // Start of new patch
-                if (StartsWith(patch_line, "@enabled") || StartsWith(patch_line, "@disabled")) {
-                    --i;
-                    break;
-                }
-
-                // Check for a flag
-                if (StartsWith(patch_line, "@")) {
-                    ParseFlag(patch_line);
-                    continue;
-                }
-
-                // 11 - 8 hex digit offset + space + minimum two digit overwrite val
-                if (patch_line.length() < 11)
-                    break;
-                auto offset = std::strtoul(patch_line.substr(0, 8).c_str(), nullptr, 16);
-                offset += static_cast<unsigned long>(offset_shift);
-
-                std::vector<u8> replace;
-                // 9 - first char of replacement val
-                if (patch_line[9] == '\"') {
-                    // string replacement
-                    auto end_index = patch_line.find('\"', 10);
-                    if (end_index == std::string::npos || end_index < 10)
-                        return;
-                    while (patch_line[end_index - 1] == '\\') {
-                        end_index = patch_line.find('\"', end_index + 1);
-                        if (end_index == std::string::npos || end_index < 10)
-                            return;
-                    }
-
-                    auto value = patch_line.substr(10, end_index - 10);
-                    value = EscapeStringSequences(value);
-                    replace.reserve(value.size());
-                    std::copy(value.begin(), value.end(), std::back_inserter(replace));
-                } else {
-                    // hex replacement
-                    const auto value =
-                        patch_line.substr(9, patch_line.find_first_of(" /\r\n", 9) - 9);
-                    replace = Common::HexStringToVector(value, is_little_endian);
-                }
-
-                if (print_values) {
-                    LOG_INFO(Loader,
-                             "[IPSwitchCompiler ('{}')]     - Patching value at offset {:#08x} "
-                             "with byte string '{}'",
-                             patch_text->GetName(), offset, Common::HexToString(replace));
-                }
-
-                patch.records.insert_or_assign(static_cast<u32>(offset), std::move(replace));
-            }
-
-            patches.push_back(std::move(patch));
-        } else if (StartsWith(line, "@")) {
-            ParseFlag(line);
-        }
     }
-
-    valid = true;
 }
 
 VirtualFile IPSwitchCompiler::Apply(const VirtualFile& in) const {
-    if (in == nullptr || !valid)
+    if (in == nullptr)
         return nullptr;
 
     auto in_data = in->ReadAllBytes();
 
     for (const auto& patch : patches) {
-        if (!patch.enabled)
+        if (patch.enabled) {
-            continue;
+            for (const auto& record : patch.records) {
-
+                if (record.first < in_data.size()) {
-        for (const auto& record : patch.records) {
+                    auto replace_size = record.second.count;
-            if (record.first >= in_data.size())
+                    if (record.first + replace_size > in_data.size())
-                continue;
+                        replace_size = in_data.size() - record.first;
-            auto replace_size = record.second.size();
+                    std::memcpy(in_data.data() + record.first, record.second.data.data(), replace_size);
-            if (record.first + replace_size > in_data.size())
+                }
-                replace_size = in_data.size() - record.first;
+            }
-            for (std::size_t i = 0; i < replace_size; ++i)
-                in_data[i + record.first] = record.second[i];
         }
     }
-
+    return std::make_shared<VectorVfsFile>(std::move(in_data), in->GetName(), in->GetContainingDirectory());
-    return std::make_shared<VectorVfsFile>(std::move(in_data), in->GetName(),
-                                           in->GetContainingDirectory());
 }
 
 } // namespace FileSys

src/core/file_sys/ips_layer.h

@@ -1,11 +1,14 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #pragma once
 
 #include <array>
-#include <memory>
 #include <vector>
+#include <span>
 
 #include "common/common_types.h"
 #include "core/file_sys/vfs/vfs.h"
@@ -20,24 +23,17 @@ public:
     ~IPSwitchCompiler();
 
     std::array<u8, 0x20> GetBuildID() const;
-    bool IsValid() const;
     VirtualFile Apply(const VirtualFile& in) const;
 
 private:
     struct IPSwitchPatch;
 
     void ParseFlag(const std::string& flag);
-    void Parse();
+    void Parse(std::span<u8 const> bytes);
-
-    bool valid = false;
 
     VirtualFile patch_text;
     std::vector<IPSwitchPatch> patches;
     std::array<u8, 0x20> nso_build_id{};
-    bool is_little_endian = false;
-    s64 offset_shift = 0;
-    bool print_values = false;
-    std::string last_comment = "";
 };
 
 } // namespace FileSys

src/core/file_sys/patch_manager.cpp

@@ -345,8 +345,7 @@ VirtualDir PatchManager::PatchExeFS(VirtualDir exefs) const {
     return exefs;
 }
 
-std::vector<VirtualFile> PatchManager::CollectPatches(const std::vector<VirtualDir>& patch_dirs,
+std::vector<VirtualFile> PatchManager::CollectPatches(const std::vector<VirtualDir>& patch_dirs, const std::string& build_id) const {
-                                                      const std::string& build_id) const {
     const auto& disabled = Settings::values.disabled_addons[title_id];
     const auto nso_build_id = fmt::format("{:0<64}", build_id);
 
@@ -361,16 +360,11 @@ std::vector<VirtualFile> PatchManager::CollectPatches(const std::vector<VirtualD
             for (const auto& file : exefs_dir->GetFiles()) {
                 if (file->GetExtension() == "ips") {
                     auto name = file->GetName();
-
+                    const auto this_build_id = fmt::format("{:0<64}", name.substr(0, name.find('.')));
-                    const auto this_build_id =
-                        fmt::format("{:0<64}", name.substr(0, name.find('.')));
                     if (nso_build_id == this_build_id)
                         out.push_back(file);
                 } else if (file->GetExtension() == "pchtxt") {
                     IPSwitchCompiler compiler{file};
-                    if (!compiler.IsValid())
-                        continue;
-
                     const auto this_build_id = Common::HexToString(compiler.GetBuildID());
                     if (nso_build_id == this_build_id)
                         out.push_back(file);
@@ -378,7 +372,6 @@ std::vector<VirtualFile> PatchManager::CollectPatches(const std::vector<VirtualD
             }
         }
     }
-
     return out;
 }
 

src/qt_common/util/mod.cpp

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include <filesystem>
+#include <system_error>
 #include <JlCompress.h>
 #include "frontend_common/mod_manager.h"
 #include "mod.h"
@@ -124,8 +125,8 @@ const QString ExtractMod(const QString& path) {
     fs::remove_all(tmp, ec);
     if (!fs::create_directories(tmp, ec)) {
         QtCommon::Frontend::Critical(tr("Mod Extract Failed"),
-                                     tr("Failed to create temporary directory %1")
+                                    tr("Failed to create temporary directory %1")
-                                         .arg(QString::fromStdString(tmp.string())));
+                                        .arg(QString::fromStdString(tmp.string())));
         return QString();
     }