From 64c13a5626b3645d7f14942d97b1f291a9aae540 Mon Sep 17 00:00:00 2001 From: PoloNX <57038157+PoloNX@users.noreply.github.com> Date: Sun, 7 Aug 2022 15:39:17 +0200 Subject: [PATCH] Update extract.cpp (#209) * Update extract.cpp Use Minizip beacause it's faster with big packs like deepsea * fix for pull request * fix crash at 66% * fix create DIR * Refactored extraction, possibly ironed out some bugs * get uncompressed size using minizip * use minizip to extract cheats, remove zipper dependency, fix getUncompressedSize() Co-authored-by: flb --- .gitmodules | 3 - Makefile | 6 +- include/constants.hpp | 2 +- include/extract.hpp | 4 +- lib/zipper | 1 - resources/i18n/en-US/brls.json | 4 +- resources/i18n/fr/brls.json | 6 +- source/extract.cpp | 188 +++++++++++++++++++++------------ source/utils.cpp | 6 +- 9 files changed, 137 insertions(+), 83 deletions(-) delete mode 160000 lib/zipper diff --git a/.gitmodules b/.gitmodules index ad039c9..997c821 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "lib/zipper"] - path = lib/zipper - url = https://github.com/HamletDuFromage/zipper [submodule "lib/borealis"] path = lib/borealis url = https://github.com/HamletDuFromage/borealis diff --git a/Makefile b/Makefile index 861c4d9..4ac18ab 100644 --- a/Makefile +++ b/Makefile @@ -16,10 +16,10 @@ include $(DEVKITPRO)/libnx/switch_rules # DATA is a list of directories containing data files # INCLUDES is a list of directories containing header files BUILD := build -SOURCES := source lib/zipper/source +SOURCES := source RESOURCES := resources DATA := data -INCLUDES := include lib/zipper/include /lib/borealis/library/include/borealis/extern/nlohmann +INCLUDES := include /lib/borealis/library/include/borealis/extern/nlohmann APP_TITLE := All-in-One Switch Updater APP_AUTHOR := HamletDuFromage APP_VERSION := 2.19.3 @@ -55,7 +55,7 @@ CXXFLAGS := $(CFLAGS) -std=gnu++20 -fexceptions -Wno-reorder ASFLAGS := -g $(ARCH) LDFLAGS = -specs=$(DEVKITPRO)/libnx/switch.specs -g $(ARCH) -Wl,-Map,$(notdir $*.map) -LIBS := -lm -lcurl -lnx -lz -lmbedtls -lmbedx509 -lmbedcrypto -lstdc++fs +LIBS := -lm -lcurl -lnx -lz -lminizip -lmbedtls -lmbedx509 -lmbedcrypto -lstdc++fs #--------------------------------------------------------------------------------- # list of directories containing libraries, this must be the top level containing diff --git a/include/constants.hpp b/include/constants.hpp index 95eb1cf..ffaa549 100644 --- a/include/constants.hpp +++ b/include/constants.hpp @@ -26,7 +26,7 @@ constexpr const char FIRMWARE_FILENAME[] = "/config/aio-switch-updater/firmware. constexpr const char FIRMWARE_PATH[] = "/firmware/"; constexpr const char CFW_URL[] = "https://raw.githubusercontent.com/HamletDuFromage/nx-links/master/bootloaders.json"; -constexpr const char CFW_FILENAME[] = "/config/aio-switch-updater/cfw.zip"; +constexpr const char BOOTLOADER_FILENAME[] = "/config/aio-switch-updater/bootloader.zip"; constexpr const char AMS_URL[] = "https://raw.githubusercontent.com/HamletDuFromage/nx-links/master/cfws.json"; constexpr const char SXOS_URL[] = "https://raw.githubusercontent.com/HamletDuFromage/nx-links/master/sxos.json"; diff --git a/include/extract.hpp b/include/extract.hpp index 2dc2643..9ff0de7 100644 --- a/include/extract.hpp +++ b/include/extract.hpp @@ -31,8 +31,8 @@ namespace extract { std::vector getInstalledTitlesNs(); std::vector excludeTitles(const std::string& path, const std::vector& listedTitles); void writeTitlesToFile(const std::set& titles, const std::string& path); - void extractCheats(const std::string& zipPath, const std::vector& titles, CFW cfw, const std::string& version, bool extractAll = false); - void extractAllCheats(const std::string& zipPath, CFW cfw, const std::string& version); + void extractCheats(const std::string& archivePath, const std::vector& titles, CFW cfw, const std::string& version, bool extractAll = false); + void extractAllCheats(const std::string& archivePath, CFW cfw, const std::string& version); void removeCheats(); void removeOrphanedCheats(); bool removeCheatsDirectory(const std::string& entry); diff --git a/lib/zipper b/lib/zipper deleted file mode 160000 index f806815..0000000 --- a/lib/zipper +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f80681569d73ffb27e85473e3bccb95b9856f1c1 diff --git a/resources/i18n/en-US/brls.json b/resources/i18n/en-US/brls.json index 42686fb..6fe8fb3 100644 --- a/resources/i18n/en-US/brls.json +++ b/resources/i18n/en-US/brls.json @@ -4,10 +4,12 @@ "back": "Back", "exit": "Exit" }, + "crash_frame": { "button": "OK" }, + "thumbnail_sidebar": { "save": "Save" } -} \ No newline at end of file +} diff --git a/resources/i18n/fr/brls.json b/resources/i18n/fr/brls.json index ef0d5bb..9ed2173 100644 --- a/resources/i18n/fr/brls.json +++ b/resources/i18n/fr/brls.json @@ -4,10 +4,12 @@ "back": "Retour", "exit": "Quitter" }, + "crash_frame": { "button": "OK" }, + "thumbnail_sidebar": { - "save": "Enregistrer" + "save": "Sauvegarder" } -} \ No newline at end of file +} diff --git a/source/extract.cpp b/source/extract.cpp index 6266cc3..43de265 100644 --- a/source/extract.cpp +++ b/source/extract.cpp @@ -1,11 +1,13 @@ #include "extract.hpp" -#include +#include +#include #include #include #include #include +#include #include #include #include @@ -24,6 +26,8 @@ namespace i18n = brls::i18n; using namespace i18n::literals; +constexpr size_t WRITE_BUFFER_SIZE = 0x10000; + namespace extract { namespace { @@ -32,63 +36,111 @@ namespace extract { return strcasecmp(a.c_str(), b.c_str()) == 0; } - void preWork(zipper::Unzipper& unzipper, const std::string& workingPath, std::vector& entries) + s64 getUncompressedSize(const std::string& archivePath) + { + s64 size = 0; + unzFile zfile = unzOpen(archivePath.c_str()); + unz_global_info gi; + unzGetGlobalInfo(zfile, &gi); + for (uLong i = 0; i < gi.number_entry; ++i) { + unz_file_info fi; + unzOpenCurrentFile(zfile); + unzGetCurrentFileInfo(zfile, &fi, NULL, 0, NULL, 0, NULL, 0); + size += fi.uncompressed_size; + unzCloseCurrentFile(zfile); + unzGoToNextFile(zfile); + } + unzClose(zfile); + return size; // in B + } + + void preWork(const std::string& archivePath, const std::string& workingPath) { chdir(workingPath.c_str()); - entries = unzipper.entries(); - s64 uncompressedSize = 0; + s64 uncompressedSize = getUncompressedSize(archivePath); s64 freeStorage; - for (const auto& entry : entries) - uncompressedSize += entry.uncompressedSize; if (R_SUCCEEDED(fs::getFreeStorageSD(freeStorage))) { + brls::Logger::info("Uncompressed size of archive {}: {}. Available: {}", archivePath, uncompressedSize, freeStorage); if (uncompressedSize * 1.1 > freeStorage) { - unzipper.close(); brls::Application::crash("menus/errors/insufficient_storage"_i18n); std::this_thread::sleep_for(std::chrono::microseconds(2000000)); brls::Application::quit(); } } - ProgressEvent::instance().setTotalSteps(entries.size() + 1); - ProgressEvent::instance().setStep(0); + } + + void extractEntry(std::string filename, unzFile& zfile, bool forceCreateTree = false) + { + if (filename.back() == '/') { + fs::createTree(filename); + return; + } + if (forceCreateTree) { + fs::createTree(filename); + } + void* buf = malloc(WRITE_BUFFER_SIZE); + FILE* outfile; + outfile = fopen(filename.c_str(), "wb"); + for (int j = unzReadCurrentFile(zfile, buf, WRITE_BUFFER_SIZE); j > 0; j = unzReadCurrentFile(zfile, buf, WRITE_BUFFER_SIZE)) { + fwrite(buf, 1, j, outfile); + } + free(buf); + fclose(outfile); } } // namespace - void extract(const std::string& filename, const std::string& workingPath, int overwriteInis, std::function func) + void extract(const std::string& archivePath, const std::string& workingPath, int overwriteInis, std::function func) { - zipper::Unzipper unzipper(filename); - std::vector entries; + preWork(archivePath, workingPath); + + unzFile zfile = unzOpen(archivePath.c_str()); + unz_global_info gi; + unzGetGlobalInfo(zfile, &gi); + + ProgressEvent::instance().setTotalSteps(gi.number_entry); + ProgressEvent::instance().setStep(0); - preWork(unzipper, workingPath, entries); std::set ignoreList = fs::readLineByLine(FILES_IGNORE); - for (const auto& entry : entries) { + for (uLong i = 0; i < gi.number_entry; ++i) { + char szFilename[0x301] = ""; + unzOpenCurrentFile(zfile); + unzGetCurrentFileInfo(zfile, NULL, szFilename, sizeof(szFilename), NULL, 0, NULL, 0); + std::string filename = szFilename; + // brls::Logger::debug("Going over {} in {}", filename, archivePath); + if (ProgressEvent::instance().getInterupt()) { + unzCloseCurrentFile(zfile); break; } - if ((overwriteInis == 0 && entry.name.substr(entry.name.length() - 4) == ".ini") || find_if(ignoreList.begin(), ignoreList.end(), [&entry](std::string ignored) { - u8 res = ("/" + entry.name).find(ignored); - return (res == 0 || res == 1); }) != ignoreList.end()) { - if (!std::filesystem::exists("/" + entry.name)) { - unzipper.extractEntry(entry.name); + + if ((overwriteInis == 0 && filename.substr(filename.length() - 4) == ".ini") || std::find_if(ignoreList.begin(), ignoreList.end(), [&filename](std::string ignored) { + u8 res = ("/" + filename).find(ignored); + return (res == 0 || res == 1); }) != ignoreList.end()) { + if (!std::filesystem::exists("/" + filename)) { + extractEntry(filename, zfile); } } - else if (entry.name == "atmosphere/stratosphere.romfs" || entry.name == "atmosphere/package3") { - std::ofstream readonlyFile(entry.name + ".aio"); - unzipper.extractEntryToStream(entry.name, readonlyFile); - } else { - unzipper.extractEntry(entry.name); - if (entry.name.substr(0, 13) == "hekate_ctcaer") { - fs::copyFile("/" + entry.name, UPDATE_BIN_PATH); - if (CurrentCfw::running_cfw == CFW::ams && util::showDialogBoxBlocking(fmt::format("menus/utils/set_hekate_reboot_payload"_i18n, UPDATE_BIN_PATH, REBOOT_PAYLOAD_PATH), "menus/common/yes"_i18n, "menus/common/no"_i18n) == 0) { - fs::copyFile(UPDATE_BIN_PATH, REBOOT_PAYLOAD_PATH); + if ((filename == "atmosphere/package3") || (filename == "atmosphere/stratosphere.romfs")) { + extractEntry(filename += ".aio", zfile); + } + else { + extractEntry(filename, zfile); + if (filename.substr(0, 13) == "hekate_ctcaer") { + fs::copyFile("/" + filename, UPDATE_BIN_PATH); + if (CurrentCfw::running_cfw == CFW::ams && util::showDialogBoxBlocking(fmt::format("menus/utils/set_hekate_reboot_payload"_i18n, UPDATE_BIN_PATH, REBOOT_PAYLOAD_PATH), "menus/common/yes"_i18n, "menus/common/no"_i18n) == 0) { + fs::copyFile(UPDATE_BIN_PATH, REBOOT_PAYLOAD_PATH); + } } } } - ProgressEvent::instance().incrementStep(1); + ProgressEvent::instance().setStep(i); + unzCloseCurrentFile(zfile); + unzGoToNextFile(zfile); } - unzipper.close(); + unzClose(zfile); ProgressEvent::instance().setStep(ProgressEvent::instance().getMax()); } @@ -181,55 +233,57 @@ namespace extract { return 0; } - void extractCheats(const std::string& zipPath, const std::vector& titles, CFW cfw, const std::string& version, bool extractAll) + void extractCheats(const std::string& archivePath, const std::vector& titles, CFW cfw, const std::string& version, bool extractAll) { - zipper::Unzipper unzipper(zipPath); - std::vector entries = unzipper.entries(); + preWork(archivePath, "/"); + + unzFile zfile = unzOpen(archivePath.c_str()); + unz_global_info gi; + unzGetGlobalInfo(zfile, &gi); + + ProgressEvent::instance().setTotalSteps(gi.number_entry); + ProgressEvent::instance().setStep(0); + int offset = computeOffset(cfw); - if (!extractAll) { - ProgressEvent::instance().setTotalSteps(titles.size() + 1); - for (const auto& title : titles) { - if (ProgressEvent::instance().getInterupt()) { - break; - } - auto matches = entries | std::views::filter([&title, offset](zipper::ZipEntry entry) { - if ((int)entry.name.size() > offset + 16 + 7) { - return caselessCompare((title.substr(0, 13)), entry.name.substr(offset, 13)) && caselessCompare(entry.name.substr(offset + 16, 7), "/cheats"); - } - else { - return false; - } - }); - for (const auto& match : matches) { - unzipper.extractEntry(match.name); - } - ProgressEvent::instance().incrementStep(1); - } - } - else { - ProgressEvent::instance().setTotalSteps(entries.size() + 1); - for (const auto& entry : entries) { - if (ProgressEvent::instance().getInterupt()) { - break; - } + for (uLong i = 0; i < gi.number_entry; ++i) { + char szFilename[0x301] = ""; + unzOpenCurrentFile(zfile); + unzGetCurrentFileInfo(zfile, NULL, szFilename, sizeof(szFilename), NULL, 0, NULL, 0); + std::string filename = szFilename; - if ((int)entry.name.size() > offset + 16 + 7 && caselessCompare(entry.name.substr(offset + 16, 7), "/cheats")) { - unzipper.extractEntry(entry.name); - } - ProgressEvent::instance().incrementStep(1); + if (ProgressEvent::instance().getInterupt()) { + unzCloseCurrentFile(zfile); + break; } + + if ((int)filename.size() > offset + 16 + 7 && caselessCompare(filename.substr(offset + 16, 7), "/cheats")) { + if (extractAll) { + extractEntry(filename, zfile); + } + else { + if (std::find_if(titles.begin(), titles.end(), [&filename, offset](std::string title) { + return caselessCompare((title.substr(0, 13)), filename.substr(offset, 13)); + }) != titles.end()) { + extractEntry(filename, zfile, true); + } + } + } + + ProgressEvent::instance().setStep(i); + unzCloseCurrentFile(zfile); + unzGoToNextFile(zfile); } - unzipper.close(); + unzClose(zfile); if (version != "offline" && version != "") { util::saveToFile(version, CHEATS_VERSION); } ProgressEvent::instance().setStep(ProgressEvent::instance().getMax()); } - void extractAllCheats(const std::string& zipPath, CFW cfw, const std::string& version) + void extractAllCheats(const std::string& archivePath, CFW cfw, const std::string& version) { - extractCheats(zipPath, {}, cfw, version, true); + extractCheats(archivePath, {}, cfw, version, true); } bool isBID(const std::string& bid) @@ -298,4 +352,4 @@ namespace extract { return res; } -} // namespace extract \ No newline at end of file +} // namespace extract diff --git a/source/utils.cpp b/source/utils.cpp index 2707ab8..b8a43d8 100644 --- a/source/utils.cpp +++ b/source/utils.cpp @@ -58,7 +58,7 @@ namespace util { status_code = download::downloadFile(url, APP_FILENAME, OFF); break; case contentType::bootloaders: - status_code = download::downloadFile(url, CFW_FILENAME, OFF); + status_code = download::downloadFile(url, BOOTLOADER_FILENAME, OFF); break; case contentType::ams_cfw: status_code = download::downloadFile(url, AMS_FILENAME, OFF); @@ -139,7 +139,7 @@ namespace util { filename = APP_FILENAME; break; case contentType::bootloaders: - filename = CFW_FILENAME; + filename = BOOTLOADER_FILENAME; break; case contentType::ams_cfw: filename = AMS_FILENAME; @@ -180,7 +180,7 @@ namespace util { break; case contentType::bootloaders: { int overwriteInis = showDialogBoxBlocking("menus/utils/overwrite_inis"_i18n, "menus/common/no"_i18n, "menus/common/yes"_i18n); - extract::extract(CFW_FILENAME, ROOT_PATH, overwriteInis); + extract::extract(BOOTLOADER_FILENAME, ROOT_PATH, overwriteInis); break; } case contentType::ams_cfw: {