From c385b7767d32eccabbfeaa12764310cfc3d113b9 Mon Sep 17 00:00:00 2001 From: condut <> Date: Fri, 10 Jul 2015 00:55:23 +0300 Subject: [PATCH] FS: Stream RomFS from file instead of loading all of it to memory --- src/core/file_sys/archive_romfs.cpp | 7 +++---- src/core/file_sys/archive_romfs.h | 4 +++- src/core/file_sys/archive_savedatacheck.cpp | 11 ++++------- src/core/file_sys/ivfc_archive.cpp | 13 ++++++------- src/core/file_sys/ivfc_archive.h | 15 +++++++++++---- src/core/loader/loader.cpp | 2 +- src/core/loader/loader.h | 7 +++++-- src/core/loader/ncch.cpp | 15 ++++++++++----- src/core/loader/ncch.h | 7 +++++-- 9 files changed, 48 insertions(+), 33 deletions(-) diff --git a/src/core/file_sys/archive_romfs.cpp b/src/core/file_sys/archive_romfs.cpp index c1e45dfeb2..b792b1c8cb 100644 --- a/src/core/file_sys/archive_romfs.cpp +++ b/src/core/file_sys/archive_romfs.cpp @@ -17,16 +17,15 @@ namespace FileSys { -ArchiveFactory_RomFS::ArchiveFactory_RomFS(const Loader::AppLoader& app_loader) - : romfs_data(std::make_shared>()) { +ArchiveFactory_RomFS::ArchiveFactory_RomFS(const Loader::AppLoader& app_loader) { // Load the RomFS from the app - if (Loader::ResultStatus::Success != app_loader.ReadRomFS(*romfs_data)) { + if (Loader::ResultStatus::Success != app_loader.ReadRomFS(romfs_file, data_offset, data_size)) { LOG_ERROR(Service_FS, "Unable to read RomFS!"); } } ResultVal> ArchiveFactory_RomFS::Open(const Path& path) { - auto archive = Common::make_unique(romfs_data); + auto archive = Common::make_unique(romfs_file, data_offset, data_size); return MakeResult>(std::move(archive)); } diff --git a/src/core/file_sys/archive_romfs.h b/src/core/file_sys/archive_romfs.h index c69ff91c31..0ef67c5575 100644 --- a/src/core/file_sys/archive_romfs.h +++ b/src/core/file_sys/archive_romfs.h @@ -29,7 +29,9 @@ public: ResultCode Format(const Path& path) override; private: - std::shared_ptr> romfs_data; + std::shared_ptr romfs_file; + u64 data_offset; + u64 data_size; }; } // namespace FileSys diff --git a/src/core/file_sys/archive_savedatacheck.cpp b/src/core/file_sys/archive_savedatacheck.cpp index dec838caee..ea1dfe2c71 100644 --- a/src/core/file_sys/archive_savedatacheck.cpp +++ b/src/core/file_sys/archive_savedatacheck.cpp @@ -37,17 +37,14 @@ ResultVal> ArchiveFactory_SaveDataCheck::Open(co auto vec = path.AsBinary(); const u32* data = reinterpret_cast(vec.data()); std::string file_path = GetSaveDataCheckPath(mount_point, data[1], data[0]); - FileUtil::IOFile file(file_path, "rb"); + auto file = std::make_shared(file_path, "rb"); - if (!file.IsOpen()) { + if (!file->IsOpen()) { return ResultCode(-1); // TODO(Subv): Find the right error code } - auto size = file.GetSize(); - auto raw_data = std::make_shared>(size); - file.ReadBytes(raw_data->data(), size); - file.Close(); + auto size = file->GetSize(); - auto archive = Common::make_unique(std::move(raw_data)); + auto archive = Common::make_unique(file, 0, size); return MakeResult>(std::move(archive)); } diff --git a/src/core/file_sys/ivfc_archive.cpp b/src/core/file_sys/ivfc_archive.cpp index c88b39bcd6..2b88b1d5d2 100644 --- a/src/core/file_sys/ivfc_archive.cpp +++ b/src/core/file_sys/ivfc_archive.cpp @@ -16,15 +16,12 @@ namespace FileSys { -IVFCArchive::IVFCArchive(std::shared_ptr> data) : data(data) { -} - std::string IVFCArchive::GetName() const { return "IVFC"; } std::unique_ptr IVFCArchive::OpenFile(const Path& path, const Mode mode) const { - return Common::make_unique(data); + return Common::make_unique(romfs_file, data_offset, data_size); } bool IVFCArchive::DeleteFile(const Path& path) const { @@ -66,8 +63,10 @@ std::unique_ptr IVFCArchive::OpenDirectory(const Path& path) c size_t IVFCFile::Read(const u64 offset, const u32 length, u8* buffer) const { LOG_TRACE(Service_FS, "called offset=%llu, length=%d", offset, length); - memcpy(buffer, data->data() + offset, length); - return length; + romfs_file->Seek(data_offset + offset, SEEK_SET); + u32 read_length = (u32)std::min((u64)length, data_size - offset); + + return romfs_file->ReadBytes(buffer, read_length); } size_t IVFCFile::Write(const u64 offset, const u32 length, const u32 flush, const u8* buffer) const { @@ -76,7 +75,7 @@ size_t IVFCFile::Write(const u64 offset, const u32 length, const u32 flush, cons } size_t IVFCFile::GetSize() const { - return sizeof(u8) * data->size(); + return data_size; // TODO: return value will overflow on 32-bit machines } bool IVFCFile::SetSize(const u64 size) const { diff --git a/src/core/file_sys/ivfc_archive.h b/src/core/file_sys/ivfc_archive.h index 1850b3b173..c25666223e 100644 --- a/src/core/file_sys/ivfc_archive.h +++ b/src/core/file_sys/ivfc_archive.h @@ -10,6 +10,7 @@ #include #include "common/common_types.h" +#include "common/file_util.h" #include "core/file_sys/archive_backend.h" #include "core/file_sys/directory_backend.h" @@ -28,7 +29,8 @@ namespace FileSys { */ class IVFCArchive : public ArchiveBackend { public: - IVFCArchive(std::shared_ptr> data); + IVFCArchive(std::shared_ptr file, u64 offset, u64 size) + : romfs_file(file), data_offset(offset), data_size(size) {} std::string GetName() const override; @@ -42,12 +44,15 @@ public: std::unique_ptr OpenDirectory(const Path& path) const override; protected: - std::shared_ptr> data; + std::shared_ptr romfs_file; + u64 data_offset; + u64 data_size; }; class IVFCFile : public FileBackend { public: - IVFCFile(std::shared_ptr> data) : data(data) {} + IVFCFile(std::shared_ptr file, u64 offset, u64 size) + : romfs_file(file), data_offset(offset), data_size(size) {} bool Open() override { return true; } size_t Read(const u64 offset, const u32 length, u8* buffer) const override; @@ -58,7 +63,9 @@ public: void Flush() const override { } private: - std::shared_ptr> data; + std::shared_ptr romfs_file; + u64 data_offset; + u64 data_size; }; class IVFCDirectory : public DirectoryBackend { diff --git a/src/core/loader/loader.cpp b/src/core/loader/loader.cpp index 2e450fce4d..b6549daf24 100644 --- a/src/core/loader/loader.cpp +++ b/src/core/loader/loader.cpp @@ -124,7 +124,7 @@ ResultStatus LoadFile(const std::string& filename) { case FileType::CXI: case FileType::CCI: { - AppLoader_NCCH app_loader(std::move(file)); + AppLoader_NCCH app_loader(std::move(file), filename); // Load application and RomFS if (ResultStatus::Success == app_loader.Load()) { diff --git a/src/core/loader/loader.h b/src/core/loader/loader.h index 52bbf35b89..ff298222ba 100644 --- a/src/core/loader/loader.h +++ b/src/core/loader/loader.h @@ -99,10 +99,13 @@ public: /** * Get the RomFS of the application - * @param buffer Reference to buffer to store data + * Since the RomFS can be huge, we return a file reference instead of copying to a buffer + * @param romfs_file The file containing the RomFS + * @param offset The offset the romfs begins on + * @param size The size of the romfs * @return ResultStatus result of function */ - virtual ResultStatus ReadRomFS(std::vector& buffer) const { + virtual ResultStatus ReadRomFS(std::shared_ptr& romfs_file, u64& offset, u64& size) const { return ResultStatus::ErrorNotImplemented; } diff --git a/src/core/loader/ncch.cpp b/src/core/loader/ncch.cpp index 87603d198a..2bf1a6a261 100644 --- a/src/core/loader/ncch.cpp +++ b/src/core/loader/ncch.cpp @@ -299,7 +299,7 @@ ResultStatus AppLoader_NCCH::ReadLogo(std::vector& buffer) const { return LoadSectionExeFS("logo", buffer); } -ResultStatus AppLoader_NCCH::ReadRomFS(std::vector& buffer) const { +ResultStatus AppLoader_NCCH::ReadRomFS(std::shared_ptr& romfs_file, u64& offset, u64& size) const { if (!file->IsOpen()) return ResultStatus::Error; @@ -311,12 +311,17 @@ ResultStatus AppLoader_NCCH::ReadRomFS(std::vector& buffer) const { LOG_DEBUG(Loader, "RomFS offset: 0x%08X", romfs_offset); LOG_DEBUG(Loader, "RomFS size: 0x%08X", romfs_size); - buffer.resize(romfs_size); - - file->Seek(romfs_offset, SEEK_SET); - if (file->ReadBytes(&buffer[0], romfs_size) != romfs_size) + if (file->GetSize () < romfs_offset + romfs_size) return ResultStatus::Error; + // We reopen the file, to allow its position to be independent from file's + romfs_file = std::make_shared(filepath, "rb"); + if (!romfs_file->IsOpen()) + return ResultStatus::Error; + + offset = romfs_offset; + size = romfs_size; + return ResultStatus::Success; } LOG_DEBUG(Loader, "NCCH has no RomFS"); diff --git a/src/core/loader/ncch.h b/src/core/loader/ncch.h index 29e39d2c0c..d180e77ed3 100644 --- a/src/core/loader/ncch.h +++ b/src/core/loader/ncch.h @@ -163,7 +163,8 @@ namespace Loader { /// Loads an NCCH file (e.g. from a CCI, or the first NCCH in a CXI) class AppLoader_NCCH final : public AppLoader { public: - AppLoader_NCCH(std::unique_ptr&& file) : AppLoader(std::move(file)) { } + AppLoader_NCCH(std::unique_ptr&& file, const std::string& filepath) + : AppLoader(std::move(file)), filepath(filepath) { } /** * Returns the type of the file @@ -211,7 +212,7 @@ public: * @param buffer Reference to buffer to store data * @return ResultStatus result of function */ - ResultStatus ReadRomFS(std::vector& buffer) const override; + ResultStatus ReadRomFS(std::shared_ptr& romfs_file, u64& offset, u64& size) const override; private: @@ -244,6 +245,8 @@ private: NCCH_Header ncch_header; ExeFs_Header exefs_header; ExHeader_Header exheader_header; + + std::string filepath; }; } // namespace Loader