From b612d8e9e317eeb76fa534a727d374772ce1ec2f Mon Sep 17 00:00:00 2001 From: lizzie Date: Fri, 1 May 2026 06:33:16 +0000 Subject: [PATCH] [file_sys] rework IPS compiler Signed-off-by: lizzie --- src/core/file_sys/ips_layer.cpp | 366 +++++++++++----------------- src/core/file_sys/ips_layer.h | 7 - src/core/file_sys/patch_manager.cpp | 11 +- 3 files changed, 148 insertions(+), 236 deletions(-) diff --git a/src/core/file_sys/ips_layer.cpp b/src/core/file_sys/ips_layer.cpp index 1dddd27909..088e8f0c59 100644 --- a/src/core/file_sys/ips_layer.cpp +++ b/src/core/file_sys/ips_layer.cpp @@ -3,10 +3,11 @@ #include #include -#include #include #include #include +#include +#include #include "common/hex_util.h" #include "common/logging.h" @@ -22,61 +23,30 @@ enum class IPSFileType { Error, }; -constexpr std::array, 11> ESCAPE_CHARACTER_MAP{{ - {"\\a", "\a"}, - {"\\b", "\b"}, - {"\\f", "\f"}, - {"\\n", "\n"}, - {"\\r", "\r"}, - {"\\t", "\t"}, - {"\\v", "\v"}, - {"\\\\", "\\"}, - {"\\\'", "\'"}, - {"\\\"", "\""}, - {"\\\?", "\?"}, -}}; - -static IPSFileType IdentifyMagic(const std::vector& magic) { - if (magic.size() != 5) { - return IPSFileType::Error; +static IPSFileType IdentifyMagic(std::span magic) { + if (magic.size() >= 5) { + if (std::memcmp(magic.data(), "PATCH", 5) == 0) + return IPSFileType::IPS; + if (std::memcmp(magic.data(), "IPS32", 5) == 0) + return IPSFileType::IPS32; } - - static constexpr std::array patch_magic{{'P', 'A', 'T', 'C', 'H'}}; - if (std::equal(magic.begin(), magic.end(), patch_magic.begin())) { - return IPSFileType::IPS; - } - - static constexpr std::array ips32_magic{{'I', 'P', 'S', '3', '2'}}; - if (std::equal(magic.begin(), magic.end(), ips32_magic.begin())) { - return IPSFileType::IPS32; - } - return IPSFileType::Error; } -static bool IsEOF(IPSFileType type, const std::vector& data) { - static constexpr std::array eof{{'E', 'O', 'F'}}; - if (type == IPSFileType::IPS && std::equal(data.begin(), data.end(), eof.begin())) { - return true; - } - - static constexpr std::array eeof{{'E', 'E', 'O', 'F'}}; - return type == IPSFileType::IPS32 && std::equal(data.begin(), data.end(), eeof.begin()); +static bool IsEOF(IPSFileType type, std::span magic) { + return (type == IPSFileType::IPS && magic.size() > 3 && std::memcmp(magic.data(), "EOF", 3) == 0) + || (type == IPSFileType::IPS32 && magic.size() > 4 && std::memcmp(magic.data(), "EEOF", 4) == 0); } VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) { if (in == nullptr || ips == nullptr) return nullptr; - const auto type = IdentifyMagic(ips->ReadBytes(0x5)); + auto in_data = in->ReadAllBytes(); + auto const type = IdentifyMagic(in_data); if (type == IPSFileType::Error) return nullptr; - auto in_data = in->ReadAllBytes(); - if (in_data.size() == 0) { - return nullptr; - } - std::vector temp(type == IPSFileType::IPS ? 3 : 4); u64 offset = 5; // After header while (ips->Read(temp.data(), temp.size(), offset) == temp.size()) { @@ -85,12 +55,9 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) { break; } - u32 real_offset{}; - if (type == IPSFileType::IPS32) - real_offset = (temp[0] << 24) | (temp[1] << 16) | (temp[2] << 8) | temp[3]; - else - real_offset = (temp[0] << 16) | (temp[1] << 8) | temp[2]; - + u32 real_offset = (type == IPSFileType::IPS32) + ? ((temp[0] << 24) | (temp[1] << 16) | (temp[2] << 8) | temp[3]) + : ((temp[0] << 16) | (temp[1] << 8) | temp[2]); if (real_offset > in_data.size()) { return nullptr; } @@ -113,30 +80,31 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) { return nullptr; if (real_offset + rle_size > in_data.size()) - rle_size = static_cast(in_data.size() - real_offset); + rle_size = u16(in_data.size() - real_offset); std::memset(in_data.data() + real_offset, *data, rle_size); } else { // Standard Patch auto read = data_size; if (real_offset + read > in_data.size()) - read = static_cast(in_data.size() - real_offset); + read = u16(in_data.size() - real_offset); if (ips->Read(in_data.data() + real_offset, read, offset) != data_size) return nullptr; offset += data_size; } } - - if (!IsEOF(type, temp)) { - return nullptr; + if (IsEOF(type, temp)) { + return std::make_shared(std::move(in_data), in->GetName(), in->GetContainingDirectory()); } - - return std::make_shared(std::move(in_data), in->GetName(), - in->GetContainingDirectory()); + return nullptr; } + +struct IPSwitchRecord { + std::array data; + size_t count; +}; struct IPSwitchCompiler::IPSwitchPatch { - std::string name; + ankerl::unordered_dense::map records; bool enabled; - std::map> records; }; IPSwitchCompiler::IPSwitchCompiler(VirtualFile patch_text_) : patch_text(std::move(patch_text_)) { @@ -149,40 +117,33 @@ std::array IPSwitchCompiler::GetBuildID() const { return nso_build_id; } -bool IPSwitchCompiler::IsValid() const { - return valid; +template +[[nodiscard]] constexpr inline bool StartsWith(std::string_view base, const char (&check)[N]) { + return base.size() >= N && std::memcmp(base.data(), check, N - 1) == 0; } -static bool StartsWith(std::string_view base, std::string_view check) { - return base.size() >= check.size() && base.substr(0, check.size()) == check; -} - -static std::string EscapeStringSequences(std::string in) { - for (const auto& seq : ESCAPE_CHARACTER_MAP) { - for (auto index = in.find(seq.first); index != std::string::npos; - index = in.find(seq.first, index)) { - in.replace(index, std::strlen(seq.first), seq.second); - index += std::strlen(seq.second); +static IPSwitchRecord EscapeStringSequences(std::string_view sv) { + IPSwitchRecord r{}; + for (auto it = sv.cbegin(); it != sv.cend(); ) { + if (*it == '\\') { + switch (it[1]) { + case 'n': r.data[r.count] = '\n'; break; + case 't': r.data[r.count] = '\t'; break; + case 'b': r.data[r.count] = '\b'; break; + case 'r': r.data[r.count] = '\r'; break; + case 'e': r.data[r.count] = '\e'; break; + case 'v': r.data[r.count] = '\v'; break; + case '?': r.data[r.count] = '\?'; break; + default: r.data[r.count] = it[1]; break; + } + ++r.count; + it += 2; + } else { + ++r.count; + ++it; } } - - return in; -} - -void IPSwitchCompiler::ParseFlag(const std::string& line) { - if (StartsWith(line, "@flag offset_shift ")) { - // Offset Shift Flag - offset_shift = std::strtoll(line.substr(19).c_str(), nullptr, 0); - } else if (StartsWith(line, "@little-endian")) { - // Set values to read as little endian - is_little_endian = true; - } else if (StartsWith(line, "@big-endian")) { - // Set values to read as big endian - is_little_endian = false; - } else if (StartsWith(line, "@flag print_values")) { - // Force printing of applied values - print_values = true; - } + return r; } void IPSwitchCompiler::Parse() { @@ -191,159 +152,124 @@ void IPSwitchCompiler::Parse() { s.write(reinterpret_cast(bytes.data()), bytes.size()); std::vector lines; - std::string stream_line; - while (std::getline(s, stream_line)) { + std::string sline{}; + while (std::getline(s, sline)) { // Remove a trailing \r - if (!stream_line.empty() && stream_line.back() == '\r') - stream_line.pop_back(); - lines.push_back(std::move(stream_line)); - } + if (!sline.empty() && sline.back() == '\r') + sline.pop_back(); - for (std::size_t i = 0; i < lines.size(); ++i) { - auto line = lines[i]; - - // Remove midline comments - std::size_t comment_index = std::string::npos; - bool within_string = false; - for (std::size_t k = 0; k < line.size(); ++k) { - if (line[k] == '\"' && (k > 0 && line[k - 1] != '\\')) { - within_string = !within_string; - } else if (line[k] == '\\' && (k < line.size() - 1 && line[k + 1] == '\\')) { - comment_index = k; + // Remove comments + std::string pp_str{}; + char has_comment = '\0'; + pp_str.resize(sline.size()); + for (size_t i = 0, j = 0; i < sline.size(); ) { + if (sline[i] == '\"' || sline[i] == '\'') { + if (sline[i] == has_comment) { + has_comment = '\0'; + } else { + pp_str[j++] = sline[i++]; + has_comment = sline[i]; + } + } else if (sline[i] == '\\') { + pp_str[j++] = sline[i++]; + pp_str[j++] = sline[i++]; + } else if ((!has_comment && i + 1 < sline.size() && sline[i + 0] == '/' && sline[i + 1] == '/') + || (!has_comment && sline[i] == '#')) { + pp_str.resize(j); break; + } else { + pp_str[j++] = sline[i++]; } } + if (pp_str.size() > 0) + lines.push_back(std::move(pp_str)); + } - if (!StartsWith(line, "//") && comment_index != std::string::npos) { - last_comment = line.substr(comment_index + 2); - line = line.substr(0, comment_index); - } - + LOG_INFO(Loader, "IPSwitchCompiler: '{}'", patch_text->GetName()); + bool is_little_endian = false; + s64 offset_shift = 0; + //bool print_values = false; + for (std::size_t i = 0; i < lines.size(); ++i) { + std::string_view line = lines[i]; + LOG_INFO(Loader, "<{}>", line); if (StartsWith(line, "@stop")) { // Force stop break; - } else if (StartsWith(line, "@nsobid-")) { - // NSO Build ID Specifier - const auto raw_build_id = fmt::format("{:0<64}", line.substr(8)); - nso_build_id = Common::HexStringToArray<0x20>(raw_build_id); - } else if (StartsWith(line, "#")) { - // Mandatory Comment - LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Forced output comment: {}", - patch_text->GetName(), line.substr(1)); - } else if (StartsWith(line, "//")) { - // Normal Comment - last_comment = line.substr(2); - if (last_comment.find_first_not_of(' ') == std::string::npos) - continue; - if (last_comment.find_first_not_of(' ') != 0) - last_comment = last_comment.substr(last_comment.find_first_not_of(' ')); - } else if (StartsWith(line, "@enabled") || StartsWith(line, "@disabled")) { - // Start of patch - const auto enabled = StartsWith(line, "@enabled"); - if (i == 0) - return; - LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Parsing patch '{}' ({})", - patch_text->GetName(), last_comment, line.substr(1)); - - IPSwitchPatch patch{last_comment, enabled, {}}; - - // Read rest of patch - while (true) { - if (i + 1 >= lines.size()) { - break; - } - - const auto& patch_line = lines[++i]; - - // Patch line may contain comments - if (StartsWith(patch_line, "//")) { - continue; - } - - // Start of new patch - if (StartsWith(patch_line, "@enabled") || StartsWith(patch_line, "@disabled")) { - --i; - break; - } - - // Check for a flag - if (StartsWith(patch_line, "@")) { - ParseFlag(patch_line); - continue; - } - - // 11 - 8 hex digit offset + space + minimum two digit overwrite val - if (patch_line.length() < 11) - break; - auto offset = std::strtoul(patch_line.substr(0, 8).c_str(), nullptr, 16); - offset += static_cast(offset_shift); - - std::vector replace; - // 9 - first char of replacement val - if (patch_line[9] == '\"') { - // string replacement - auto end_index = patch_line.find('\"', 10); - if (end_index == std::string::npos || end_index < 10) - return; - while (patch_line[end_index - 1] == '\\') { - end_index = patch_line.find('\"', end_index + 1); - if (end_index == std::string::npos || end_index < 10) - return; - } - - auto value = patch_line.substr(10, end_index - 10); - value = EscapeStringSequences(value); - replace.reserve(value.size()); - std::copy(value.begin(), value.end(), std::back_inserter(replace)); - } else { - // hex replacement - const auto value = - patch_line.substr(9, patch_line.find_first_of(" /\r\n", 9) - 9); - replace = Common::HexStringToVector(value, is_little_endian); - } - - if (print_values) { - LOG_INFO(Loader, - "[IPSwitchCompiler ('{}')] - Patching value at offset 0x{:08X} " - "with byte string '{}'", - patch_text->GetName(), offset, Common::HexToString(replace)); - } - - patch.records.insert_or_assign(static_cast(offset), std::move(replace)); - } - - patches.push_back(std::move(patch)); + } else if (StartsWith(line, "@nsobid-")) { // NSO Build ID Specifier + nso_build_id = Common::HexStringToArray<0x20>(line.substr(8)); + } else if (StartsWith(line, "@enabled")) { + patches.push_back({{}, true}); //enabled patch + } else if (StartsWith(line, "@disabled")) { + patches.push_back({{}, false}); //disabled patch + } else if (StartsWith(line, "@flag offset_shift ")) { + offset_shift = std::strtoll(line.data() + 19, nullptr, 0); // Offset Shift Flag + } else if (StartsWith(line, "@little-endian")) { + is_little_endian = true; // Set values to read as little endian + } else if (StartsWith(line, "@big-endian")) { + is_little_endian = false; // Set values to read as big endian + } else if (StartsWith(line, "@flag print_values")) { + //print_values = true; // Force printing of applied values } else if (StartsWith(line, "@")) { - ParseFlag(line); + LOG_WARNING(Loader, "Unknown flag {}", line); + } else { + auto offset = std::strtoul(line.data(), nullptr, 16); + offset += size_t(offset_shift); + if (auto const first_quote = line.find_first_of("\"\'"); first_quote != std::string::npos) { + // string replacement + char quote = line[first_quote]; + auto const start = line.cbegin() + first_quote + 1; + auto end = start; + for (; end < line.cend() && *end != quote; ) + end += (*end == '\\') ? 2 : 1; + if (start <= line.cend() && end <= line.cend()) { + LOG_INFO(Loader, "[S] value @ {:#08X} ", offset); + patches.back().records.insert_or_assign(u32(offset), EscapeStringSequences({start, end})); + } else { + LOG_WARNING(Loader, "invalid string"); + } + } else if (auto const first_space = line.find_first_of(" /\r\n", 9); first_space != std::string::npos) { + IPSwitchRecord r; // hex replacement + auto const start = line.cbegin() + first_space; + if (auto const last_space = line.find_last_of(" /\r\n"); last_space != std::string::npos) { + auto const end = line.cbegin() + last_space; + if (start <= line.cend() && end <= line.cend()) { + auto const hs = Common::HexStringToVector({start, end}, is_little_endian); + std::memcpy(r.data.data(), hs.data(), hs.size()); + r.count = hs.size(); + LOG_INFO(Loader, "[H] value @ {:#08X} ", offset); + patches.back().records.insert_or_assign(u32(offset), std::move(r)); + } else { + LOG_WARNING(Loader, "invalid line"); + } + } else { + LOG_WARNING(Loader, "no last space"); + } + } else { + LOG_WARNING(Loader, "unhandled line!"); + } } } - - valid = true; } VirtualFile IPSwitchCompiler::Apply(const VirtualFile& in) const { - if (in == nullptr || !valid) + if (in == nullptr) return nullptr; auto in_data = in->ReadAllBytes(); for (const auto& patch : patches) { - if (!patch.enabled) - continue; - - for (const auto& record : patch.records) { - if (record.first >= in_data.size()) - continue; - auto replace_size = record.second.size(); - if (record.first + replace_size > in_data.size()) - replace_size = in_data.size() - record.first; - for (std::size_t i = 0; i < replace_size; ++i) - in_data[i + record.first] = record.second[i]; + if (patch.enabled) { + for (const auto& record : patch.records) { + if (record.first < in_data.size()) { + auto replace_size = record.second.count; + if (record.first + replace_size > in_data.size()) + replace_size = in_data.size() - record.first; + std::memcpy(in_data.data() + record.first, record.second.data.data(), replace_size); + } + } } } - - return std::make_shared(std::move(in_data), in->GetName(), - in->GetContainingDirectory()); + return std::make_shared(std::move(in_data), in->GetName(), in->GetContainingDirectory()); } } // namespace FileSys diff --git a/src/core/file_sys/ips_layer.h b/src/core/file_sys/ips_layer.h index d81378e8ad..585b79d114 100644 --- a/src/core/file_sys/ips_layer.h +++ b/src/core/file_sys/ips_layer.h @@ -20,7 +20,6 @@ public: ~IPSwitchCompiler(); std::array GetBuildID() const; - bool IsValid() const; VirtualFile Apply(const VirtualFile& in) const; private: @@ -29,15 +28,9 @@ private: void ParseFlag(const std::string& flag); void Parse(); - bool valid = false; - VirtualFile patch_text; std::vector patches; std::array nso_build_id{}; - bool is_little_endian = false; - s64 offset_shift = 0; - bool print_values = false; - std::string last_comment = ""; }; } // namespace FileSys diff --git a/src/core/file_sys/patch_manager.cpp b/src/core/file_sys/patch_manager.cpp index 6af926eac0..7262140680 100644 --- a/src/core/file_sys/patch_manager.cpp +++ b/src/core/file_sys/patch_manager.cpp @@ -345,8 +345,7 @@ VirtualDir PatchManager::PatchExeFS(VirtualDir exefs) const { return exefs; } -std::vector PatchManager::CollectPatches(const std::vector& patch_dirs, - const std::string& build_id) const { +std::vector PatchManager::CollectPatches(const std::vector& patch_dirs, const std::string& build_id) const { const auto& disabled = Settings::values.disabled_addons[title_id]; const auto nso_build_id = fmt::format("{:0<64}", build_id); @@ -361,16 +360,11 @@ std::vector PatchManager::CollectPatches(const std::vectorGetFiles()) { if (file->GetExtension() == "ips") { auto name = file->GetName(); - - const auto this_build_id = - fmt::format("{:0<64}", name.substr(0, name.find('.'))); + const auto this_build_id = fmt::format("{:0<64}", name.substr(0, name.find('.'))); if (nso_build_id == this_build_id) out.push_back(file); } else if (file->GetExtension() == "pchtxt") { IPSwitchCompiler compiler{file}; - if (!compiler.IsValid()) - continue; - const auto this_build_id = Common::HexToString(compiler.GetBuildID()); if (nso_build_id == this_build_id) out.push_back(file); @@ -378,7 +372,6 @@ std::vector PatchManager::CollectPatches(const std::vector