Update extract.cpp (#209)

* Update extract.cpp

Use Minizip beacause it's faster with big packs like deepsea

* fix for pull request

* fix crash at 66%

* fix create DIR

* Refactored extraction, possibly ironed out some bugs

* get uncompressed size using minizip

* use minizip to extract cheats, remove zipper dependency, fix getUncompressedSize()

Co-authored-by: flb <f.lb@gmx.com>

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "lib/zipper"]
-	path = lib/zipper
-	url = https://github.com/HamletDuFromage/zipper
 [submodule "lib/borealis"]
 	path = lib/borealis
 	url = https://github.com/HamletDuFromage/borealis

Makefile

@@ -16,10 +16,10 @@ include $(DEVKITPRO)/libnx/switch_rules
 # DATA is a list of directories containing data files
 # INCLUDES is a list of directories containing header files
 BUILD		:=	build
-SOURCES		:=	source lib/zipper/source
+SOURCES		:=	source
 RESOURCES	:=	resources
 DATA		:=	data
-INCLUDES	:=	include lib/zipper/include /lib/borealis/library/include/borealis/extern/nlohmann
+INCLUDES	:=	include /lib/borealis/library/include/borealis/extern/nlohmann
 APP_TITLE	:=	All-in-One Switch Updater
 APP_AUTHOR	:=	HamletDuFromage
 APP_VERSION :=  2.19.3
@@ -55,7 +55,7 @@ CXXFLAGS	:= $(CFLAGS) -std=gnu++20 -fexceptions -Wno-reorder
 ASFLAGS	:=	-g $(ARCH)
 LDFLAGS	=	-specs=$(DEVKITPRO)/libnx/switch.specs -g $(ARCH) -Wl,-Map,$(notdir $*.map)
 
-LIBS	:= -lm -lcurl -lnx -lz -lmbedtls -lmbedx509 -lmbedcrypto -lstdc++fs
+LIBS	:= -lm -lcurl -lnx -lz -lminizip -lmbedtls -lmbedx509 -lmbedcrypto -lstdc++fs
 
 #---------------------------------------------------------------------------------
 # list of directories containing libraries, this must be the top level containing

include/constants.hpp

@@ -26,7 +26,7 @@ constexpr const char FIRMWARE_FILENAME[] = "/config/aio-switch-updater/firmware.
 constexpr const char FIRMWARE_PATH[] = "/firmware/";
 
 constexpr const char CFW_URL[] = "https://raw.githubusercontent.com/HamletDuFromage/nx-links/master/bootloaders.json";
-constexpr const char CFW_FILENAME[] = "/config/aio-switch-updater/cfw.zip";
+constexpr const char BOOTLOADER_FILENAME[] = "/config/aio-switch-updater/bootloader.zip";
 
 constexpr const char AMS_URL[] = "https://raw.githubusercontent.com/HamletDuFromage/nx-links/master/cfws.json";
 constexpr const char SXOS_URL[] = "https://raw.githubusercontent.com/HamletDuFromage/nx-links/master/sxos.json";

include/extract.hpp

@@ -31,8 +31,8 @@ namespace extract {
     std::vector<std::string> getInstalledTitlesNs();
     std::vector<std::string> excludeTitles(const std::string& path, const std::vector<std::string>& listedTitles);
     void writeTitlesToFile(const std::set<std::string>& titles, const std::string& path);
-    void extractCheats(const std::string& zipPath, const std::vector<std::string>& titles, CFW cfw, const std::string& version, bool extractAll = false);
+    void extractCheats(const std::string& archivePath, const std::vector<std::string>& titles, CFW cfw, const std::string& version, bool extractAll = false);
-    void extractAllCheats(const std::string& zipPath, CFW cfw, const std::string& version);
+    void extractAllCheats(const std::string& archivePath, CFW cfw, const std::string& version);
     void removeCheats();
     void removeOrphanedCheats();
     bool removeCheatsDirectory(const std::string& entry);

lib/zipper

@@ -1 +0,0 @@
-Subproject commit f80681569d73ffb27e85473e3bccb95b9856f1c1

resources/i18n/en-US/brls.json

@@ -4,10 +4,12 @@
         "back": "Back",
         "exit": "Exit"
     },
+
     "crash_frame": {
         "button": "OK"
     },
+
     "thumbnail_sidebar": {
         "save": "Save"
     }
-}
+}

resources/i18n/fr/brls.json

@@ -4,10 +4,12 @@
         "back": "Retour",
         "exit": "Quitter"
     },
+
     "crash_frame": {
         "button": "OK"
     },
+
     "thumbnail_sidebar": {
-        "save": "Enregistrer"
+        "save": "Sauvegarder"
     }
-}
+}

source/extract.cpp

@@ -1,11 +1,13 @@
 #include "extract.hpp"
 
-#include <unzipper.h>
+#include <dirent.h>
+#include <minizip/unzip.h>
 
 #include <algorithm>
 #include <filesystem>
 #include <fstream>
 #include <iomanip>
+#include <iostream>
 #include <iterator>
 #include <ranges>
 #include <set>
@@ -24,6 +26,8 @@
 namespace i18n = brls::i18n;
 using namespace i18n::literals;
 
+constexpr size_t WRITE_BUFFER_SIZE = 0x10000;
+
 namespace extract {
 
     namespace {
@@ -32,63 +36,111 @@ namespace extract {
             return strcasecmp(a.c_str(), b.c_str()) == 0;
         }
 
-        void preWork(zipper::Unzipper& unzipper, const std::string& workingPath, std::vector<zipper::ZipEntry>& entries)
+        s64 getUncompressedSize(const std::string& archivePath)
+        {
+            s64 size = 0;
+            unzFile zfile = unzOpen(archivePath.c_str());
+            unz_global_info gi;
+            unzGetGlobalInfo(zfile, &gi);
+            for (uLong i = 0; i < gi.number_entry; ++i) {
+                unz_file_info fi;
+                unzOpenCurrentFile(zfile);
+                unzGetCurrentFileInfo(zfile, &fi, NULL, 0, NULL, 0, NULL, 0);
+                size += fi.uncompressed_size;
+                unzCloseCurrentFile(zfile);
+                unzGoToNextFile(zfile);
+            }
+            unzClose(zfile);
+            return size;  // in B
+        }
+
+        void preWork(const std::string& archivePath, const std::string& workingPath)
         {
             chdir(workingPath.c_str());
-            entries = unzipper.entries();
+            s64 uncompressedSize = getUncompressedSize(archivePath);
-            s64 uncompressedSize = 0;
             s64 freeStorage;
-            for (const auto& entry : entries)
-                uncompressedSize += entry.uncompressedSize;
 
             if (R_SUCCEEDED(fs::getFreeStorageSD(freeStorage))) {
+                brls::Logger::info("Uncompressed size of archive {}: {}. Available: {}", archivePath, uncompressedSize, freeStorage);
                 if (uncompressedSize * 1.1 > freeStorage) {
-                    unzipper.close();
                     brls::Application::crash("menus/errors/insufficient_storage"_i18n);
                     std::this_thread::sleep_for(std::chrono::microseconds(2000000));
                     brls::Application::quit();
                 }
             }
-            ProgressEvent::instance().setTotalSteps(entries.size() + 1);
+        }
-            ProgressEvent::instance().setStep(0);
+
+        void extractEntry(std::string filename, unzFile& zfile, bool forceCreateTree = false)
+        {
+            if (filename.back() == '/') {
+                fs::createTree(filename);
+                return;
+            }
+            if (forceCreateTree) {
+                fs::createTree(filename);
+            }
+            void* buf = malloc(WRITE_BUFFER_SIZE);
+            FILE* outfile;
+            outfile = fopen(filename.c_str(), "wb");
+            for (int j = unzReadCurrentFile(zfile, buf, WRITE_BUFFER_SIZE); j > 0; j = unzReadCurrentFile(zfile, buf, WRITE_BUFFER_SIZE)) {
+                fwrite(buf, 1, j, outfile);
+            }
+            free(buf);
+            fclose(outfile);
         }
     }  // namespace
 
-    void extract(const std::string& filename, const std::string& workingPath, int overwriteInis, std::function<void()> func)
+    void extract(const std::string& archivePath, const std::string& workingPath, int overwriteInis, std::function<void()> func)
     {
-        zipper::Unzipper unzipper(filename);
+        preWork(archivePath, workingPath);
-        std::vector<zipper::ZipEntry> entries;
+
+        unzFile zfile = unzOpen(archivePath.c_str());
+        unz_global_info gi;
+        unzGetGlobalInfo(zfile, &gi);
+
+        ProgressEvent::instance().setTotalSteps(gi.number_entry);
+        ProgressEvent::instance().setStep(0);
 
-        preWork(unzipper, workingPath, entries);
         std::set<std::string> ignoreList = fs::readLineByLine(FILES_IGNORE);
 
-        for (const auto& entry : entries) {
+        for (uLong i = 0; i < gi.number_entry; ++i) {
+            char szFilename[0x301] = "";
+            unzOpenCurrentFile(zfile);
+            unzGetCurrentFileInfo(zfile, NULL, szFilename, sizeof(szFilename), NULL, 0, NULL, 0);
+            std::string filename = szFilename;
+            // brls::Logger::debug("Going over {} in {}", filename, archivePath);
+
             if (ProgressEvent::instance().getInterupt()) {
+                unzCloseCurrentFile(zfile);
                 break;
             }
-            if ((overwriteInis == 0 && entry.name.substr(entry.name.length() - 4) == ".ini") || find_if(ignoreList.begin(), ignoreList.end(), [&entry](std::string ignored) {
+
-                                                            u8 res = ("/" + entry.name).find(ignored);
+            if ((overwriteInis == 0 && filename.substr(filename.length() - 4) == ".ini") || std::find_if(ignoreList.begin(), ignoreList.end(), [&filename](std::string ignored) {
-                                                            return (res == 0 || res == 1); }) != ignoreList.end()) {
+                                                                                                                    u8 res = ("/" + filename).find(ignored);
-                if (!std::filesystem::exists("/" + entry.name)) {
+                                                                                                                    return (res == 0 || res == 1); }) != ignoreList.end()) {
-                    unzipper.extractEntry(entry.name);
+                if (!std::filesystem::exists("/" + filename)) {
+                    extractEntry(filename, zfile);
                 }
             }
-            else if (entry.name == "atmosphere/stratosphere.romfs" || entry.name == "atmosphere/package3") {
-                std::ofstream readonlyFile(entry.name + ".aio");
-                unzipper.extractEntryToStream(entry.name, readonlyFile);
-            }
             else {
-                unzipper.extractEntry(entry.name);
+                if ((filename == "atmosphere/package3") || (filename == "atmosphere/stratosphere.romfs")) {
-                if (entry.name.substr(0, 13) == "hekate_ctcaer") {
+                    extractEntry(filename += ".aio", zfile);
-                    fs::copyFile("/" + entry.name, UPDATE_BIN_PATH);
+                }
-                    if (CurrentCfw::running_cfw == CFW::ams && util::showDialogBoxBlocking(fmt::format("menus/utils/set_hekate_reboot_payload"_i18n, UPDATE_BIN_PATH, REBOOT_PAYLOAD_PATH), "menus/common/yes"_i18n, "menus/common/no"_i18n) == 0) {
+                else {
-                        fs::copyFile(UPDATE_BIN_PATH, REBOOT_PAYLOAD_PATH);
+                    extractEntry(filename, zfile);
+                    if (filename.substr(0, 13) == "hekate_ctcaer") {
+                        fs::copyFile("/" + filename, UPDATE_BIN_PATH);
+                        if (CurrentCfw::running_cfw == CFW::ams && util::showDialogBoxBlocking(fmt::format("menus/utils/set_hekate_reboot_payload"_i18n, UPDATE_BIN_PATH, REBOOT_PAYLOAD_PATH), "menus/common/yes"_i18n, "menus/common/no"_i18n) == 0) {
+                            fs::copyFile(UPDATE_BIN_PATH, REBOOT_PAYLOAD_PATH);
+                        }
                     }
                 }
             }
-            ProgressEvent::instance().incrementStep(1);
+            ProgressEvent::instance().setStep(i);
+            unzCloseCurrentFile(zfile);
+            unzGoToNextFile(zfile);
         }
-        unzipper.close();
+        unzClose(zfile);
         ProgressEvent::instance().setStep(ProgressEvent::instance().getMax());
     }
 
@@ -181,55 +233,57 @@ namespace extract {
         return 0;
     }
 
-    void extractCheats(const std::string& zipPath, const std::vector<std::string>& titles, CFW cfw, const std::string& version, bool extractAll)
+    void extractCheats(const std::string& archivePath, const std::vector<std::string>& titles, CFW cfw, const std::string& version, bool extractAll)
     {
-        zipper::Unzipper unzipper(zipPath);
+        preWork(archivePath, "/");
-        std::vector<zipper::ZipEntry> entries = unzipper.entries();
+
+        unzFile zfile = unzOpen(archivePath.c_str());
+        unz_global_info gi;
+        unzGetGlobalInfo(zfile, &gi);
+
+        ProgressEvent::instance().setTotalSteps(gi.number_entry);
+        ProgressEvent::instance().setStep(0);
+
         int offset = computeOffset(cfw);
 
-        if (!extractAll) {
+        for (uLong i = 0; i < gi.number_entry; ++i) {
-            ProgressEvent::instance().setTotalSteps(titles.size() + 1);
+            char szFilename[0x301] = "";
-            for (const auto& title : titles) {
+            unzOpenCurrentFile(zfile);
-                if (ProgressEvent::instance().getInterupt()) {
+            unzGetCurrentFileInfo(zfile, NULL, szFilename, sizeof(szFilename), NULL, 0, NULL, 0);
-                    break;
+            std::string filename = szFilename;
-                }
-                auto matches = entries | std::views::filter([&title, offset](zipper::ZipEntry entry) {
-                                   if ((int)entry.name.size() > offset + 16 + 7) {
-                                       return caselessCompare((title.substr(0, 13)), entry.name.substr(offset, 13)) && caselessCompare(entry.name.substr(offset + 16, 7), "/cheats");
-                                   }
-                                   else {
-                                       return false;
-                                   }
-                               });
-                for (const auto& match : matches) {
-                    unzipper.extractEntry(match.name);
-                }
-                ProgressEvent::instance().incrementStep(1);
-            }
-        }
-        else {
-            ProgressEvent::instance().setTotalSteps(entries.size() + 1);
-            for (const auto& entry : entries) {
-                if (ProgressEvent::instance().getInterupt()) {
-                    break;
-                }
 
-                if ((int)entry.name.size() > offset + 16 + 7 && caselessCompare(entry.name.substr(offset + 16, 7), "/cheats")) {
+            if (ProgressEvent::instance().getInterupt()) {
-                    unzipper.extractEntry(entry.name);
+                unzCloseCurrentFile(zfile);
-                }
+                break;
-                ProgressEvent::instance().incrementStep(1);
             }
+
+            if ((int)filename.size() > offset + 16 + 7 && caselessCompare(filename.substr(offset + 16, 7), "/cheats")) {
+                if (extractAll) {
+                    extractEntry(filename, zfile);
+                }
+                else {
+                    if (std::find_if(titles.begin(), titles.end(), [&filename, offset](std::string title) {
+                            return caselessCompare((title.substr(0, 13)), filename.substr(offset, 13));
+                        }) != titles.end()) {
+                        extractEntry(filename, zfile, true);
+                    }
+                }
+            }
+
+            ProgressEvent::instance().setStep(i);
+            unzCloseCurrentFile(zfile);
+            unzGoToNextFile(zfile);
         }
-        unzipper.close();
+        unzClose(zfile);
         if (version != "offline" && version != "") {
             util::saveToFile(version, CHEATS_VERSION);
         }
         ProgressEvent::instance().setStep(ProgressEvent::instance().getMax());
     }
 
-    void extractAllCheats(const std::string& zipPath, CFW cfw, const std::string& version)
+    void extractAllCheats(const std::string& archivePath, CFW cfw, const std::string& version)
     {
-        extractCheats(zipPath, {}, cfw, version, true);
+        extractCheats(archivePath, {}, cfw, version, true);
     }
 
     bool isBID(const std::string& bid)
@@ -298,4 +352,4 @@ namespace extract {
         return res;
     }
 
-}  // namespace extract
+}  // namespace extract

source/utils.cpp

@@ -58,7 +58,7 @@ namespace util {
                 status_code = download::downloadFile(url, APP_FILENAME, OFF);
                 break;
             case contentType::bootloaders:
-                status_code = download::downloadFile(url, CFW_FILENAME, OFF);
+                status_code = download::downloadFile(url, BOOTLOADER_FILENAME, OFF);
                 break;
             case contentType::ams_cfw:
                 status_code = download::downloadFile(url, AMS_FILENAME, OFF);
@@ -139,7 +139,7 @@ namespace util {
                 filename = APP_FILENAME;
                 break;
             case contentType::bootloaders:
-                filename = CFW_FILENAME;
+                filename = BOOTLOADER_FILENAME;
                 break;
             case contentType::ams_cfw:
                 filename = AMS_FILENAME;
@@ -180,7 +180,7 @@ namespace util {
                 break;
             case contentType::bootloaders: {
                 int overwriteInis = showDialogBoxBlocking("menus/utils/overwrite_inis"_i18n, "menus/common/no"_i18n, "menus/common/yes"_i18n);
-                extract::extract(CFW_FILENAME, ROOT_PATH, overwriteInis);
+                extract::extract(BOOTLOADER_FILENAME, ROOT_PATH, overwriteInis);
                 break;
             }
             case contentType::ams_cfw: {