From 6bf314bceaaa8fcef7261856c3deb910ee53cfa5 Mon Sep 17 00:00:00 2001 From: Pablo Curiel Date: Thu, 7 Jul 2022 02:30:45 +0200 Subject: [PATCH] romfs: implement romfsIsFileEntryUpdated(). Also modified romfsGetTotalDataSize() to add an 'only_updated' argument. --- code_templates/sd_romfs_dumper.c | 8 +- code_templates/usb_romfs_dumper.c | 31 +++++++- include/core/bktr.h | 17 +++-- include/core/nca_storage.h | 3 + include/core/romfs.h | 7 +- source/core/bktr.c | 117 +++++++++++++++++++++++------- source/core/nca_storage.c | 23 ++++-- source/core/romfs.c | 42 ++++++++++- 8 files changed, 200 insertions(+), 48 deletions(-) diff --git a/code_templates/sd_romfs_dumper.c b/code_templates/sd_romfs_dumper.c index 3708055..5afcff6 100644 --- a/code_templates/sd_romfs_dumper.c +++ b/code_templates/sd_romfs_dumper.c @@ -108,7 +108,7 @@ static void read_thread_func(void *arg) romfsResetFileTableOffset(shared_data->romfs_ctx); /* Loop through all file entries. */ - while(romfsCanMoveToNextFileEntry(shared_data->romfs_ctx)) + while(shared_data->data_written < shared_data->total_size && romfsCanMoveToNextFileEntry(shared_data->romfs_ctx)) { /* Check if the transfer has been cancelled by the user. */ if (shared_data->transfer_cancelled) @@ -634,7 +634,11 @@ int main(int argc, char *argv[]) } shared_data.romfs_ctx = &romfs_ctx; - romfsGetTotalDataSize(&romfs_ctx, &(shared_data.total_size)); + if (!romfsGetTotalDataSize(&romfs_ctx, false, &(shared_data.total_size)) || !shared_data.total_size) + { + consolePrint("failed to retrieve total romfs size\n"); + goto out2; + } consolePrint("romfs initialize ctx succeeded\n"); diff --git a/code_templates/usb_romfs_dumper.c b/code_templates/usb_romfs_dumper.c index 8f631c6..50b263f 100644 --- a/code_templates/usb_romfs_dumper.c +++ b/code_templates/usb_romfs_dumper.c @@ -107,7 +107,7 @@ static void read_thread_func(void *arg) romfsResetFileTableOffset(shared_data->romfs_ctx); /* Loop through all file entries. */ - while(romfsCanMoveToNextFileEntry(shared_data->romfs_ctx)) + while(shared_data->data_written < shared_data->total_size && romfsCanMoveToNextFileEntry(shared_data->romfs_ctx)) { /* Check if the transfer has been cancelled by the user */ if (shared_data->transfer_cancelled) @@ -117,6 +117,28 @@ static void read_thread_func(void *arg) } /* Retrieve RomFS file entry information */ + /*shared_data->read_error = (!(file_entry = romfsGetCurrentFileEntry(shared_data->romfs_ctx))); + if (shared_data->read_error) + { + condvarWakeAll(&g_writeCondvar); + break; + } + + bool updated = false; + if (!romfsIsFileEntryUpdated(shared_data->romfs_ctx, file_entry, &updated) || !updated) + { + shared_data->read_error = !romfsMoveToNextFileEntry(shared_data->romfs_ctx); + if (shared_data->read_error) + { + condvarWakeAll(&g_writeCondvar); + break; + } + + continue; + } + + shared_data->read_error = !romfsGeneratePathFromFileEntry(shared_data->romfs_ctx, file_entry, path, FS_MAX_PATH, RomFileSystemPathIllegalCharReplaceType_IllegalFsChars);*/ + shared_data->read_error = (!(file_entry = romfsGetCurrentFileEntry(shared_data->romfs_ctx)) || \ !romfsGeneratePathFromFileEntry(shared_data->romfs_ctx, file_entry, path, FS_MAX_PATH, RomFileSystemPathIllegalCharReplaceType_IllegalFsChars)); if (shared_data->read_error) @@ -599,7 +621,12 @@ int main(int argc, char *argv[]) } shared_data.romfs_ctx = &romfs_ctx; - romfsGetTotalDataSize(&romfs_ctx, &(shared_data.total_size)); + //if (!romfsGetTotalDataSize(&romfs_ctx, true, &(shared_data.total_size)) || !shared_data.total_size) + if (!romfsGetTotalDataSize(&romfs_ctx, false, &(shared_data.total_size)) || !shared_data.total_size) + { + consolePrint("failed to retrieve total romfs size\n"); + goto out2; + } consolePrint("romfs initialize ctx succeeded\n"); diff --git a/include/core/bktr.h b/include/core/bktr.h index ee9877b..3d4dbe2 100644 --- a/include/core/bktr.h +++ b/include/core/bktr.h @@ -151,13 +151,13 @@ typedef enum { typedef enum { BucketTreeSubStorageType_Regular = 0, ///< Body storage with None, XTS or CTR crypto. Most common substorage type, used in all title types. - ///< May be used as substorage for all BucketTreeStorage types. - BucketTreeSubStorageType_Indirect = 1, ///< Indirect storage from patch NCAs. May be used as substorage for BucketTreeStorageType_Compressed only. - BucketTreeSubStorageType_AesCtrEx = 2, ///< AesCtrEx storage from patch NCAs. May be used as substorage for BucketTreeStorageType_Indirect only. - BucketTreeSubStorageType_Compressed = 3, ///< Compressed storage. If available, this is always the outmost storage type for any NCA. May be used by all title types. - ///< May be used as substorage for BucketTreeStorageType_Indirect only. - BucketTreeSubStorageType_Sparse = 4, ///< Sparse storage with CTR crypto, using virtual offsets as lower CTR IVs. Used in base applications only. - ///< May be used as substorage for BucketTreeStorageType_Compressed or BucketTreeStorageType_Indirect. + ///< May be used as substorage for all other BucketTreeStorage types. + BucketTreeSubStorageType_Indirect = 1, ///< Indirect storage. Only used in patches. This is always the outmost storage type. + BucketTreeSubStorageType_AesCtrEx = 2, ///< AesCtrEx storage. Only used in patches. Must be used as substorage #1 for BucketTreeStorageType_Indirect. + BucketTreeSubStorageType_Compressed = 3, ///< Compressed storage. Only used in base applications. If available, this is always the outmost storage type. + ///< May be used as substorage #0 for BucketTreeStorageType_Indirect only. + BucketTreeSubStorageType_Sparse = 4, ///< Sparse storage with CTR crypto, using virtual offsets as lower CTR IVs. Only used in base applications. + ///< May be used as substorage for BucketTreeStorageType_Compressed or BucketTreeStorageType_Indirect (#0). BucketTreeSubStorageType_Count = 5 ///< Total values supported by this enum. } BucketTreeSubStorageType; @@ -195,6 +195,9 @@ bool bktrSetBucketTreeSubStorage(BucketTreeContext *parent_ctx, BucketTreeContex /// Reads data from a Bucket Tree storage using a previously initialized BucketTreeContext. bool bktrReadStorage(BucketTreeContext *ctx, void *out, u64 read_size, u64 offset); +/// Checks if the provided block extents are within the provided BucketTreeContext's Indirect Storage. +bool bktrIsBlockWithinIndirectStorageRange(BucketTreeContext *ctx, u64 offset, u64 size, bool *out); + /// Helper inline functions. NX_INLINE void bktrFreeContext(BucketTreeContext *ctx) diff --git a/include/core/nca_storage.h b/include/core/nca_storage.h index 1ee0777..04cacba 100644 --- a/include/core/nca_storage.h +++ b/include/core/nca_storage.h @@ -63,6 +63,9 @@ bool ncaStorageGetHashTargetExtents(NcaStorageContext *ctx, u64 *out_offset, u64 /// Reads data from the NCA storage using a previously initialized NcaStorageContext. bool ncaStorageRead(NcaStorageContext *ctx, void *out, u64 read_size, u64 offset); +/// Checks if the provided block extents are within the provided Patch NcaStorageContext's Indirect Storage. +bool ncaStorageIsBlockWithinPatchStorageRange(NcaStorageContext *ctx, u64 offset, u64 size, bool *out); + /// Frees a previously initialized NCA storage context. void ncaStorageFreeContext(NcaStorageContext *ctx); diff --git a/include/core/romfs.h b/include/core/romfs.h index 866af9d..e1c6769 100644 --- a/include/core/romfs.h +++ b/include/core/romfs.h @@ -151,7 +151,8 @@ bool romfsReadFileSystemData(RomFileSystemContext *ctx, void *out, u64 read_size bool romfsReadFileEntryData(RomFileSystemContext *ctx, RomFileSystemFileEntry *file_entry, void *out, u64 read_size, u64 offset); /// Calculates the extracted RomFS size. -bool romfsGetTotalDataSize(RomFileSystemContext *ctx, u64 *out_size); +/// If 'only_updated' is set to true and the provided RomFS context was initialized as a Patch RomFS context, only files modified by the update will be considered. +bool romfsGetTotalDataSize(RomFileSystemContext *ctx, bool only_updated, u64 *out_size); /// Calculates the extracted size from a RomFS directory. bool romfsGetDirectoryDataSize(RomFileSystemContext *ctx, RomFileSystemDirectoryEntry *dir_entry, u64 *out_size); @@ -170,6 +171,10 @@ bool romfsGeneratePathFromDirectoryEntry(RomFileSystemContext *ctx, RomFileSyste /// Generates a path string from a RomFS file entry. bool romfsGeneratePathFromFileEntry(RomFileSystemContext *ctx, RomFileSystemFileEntry *file_entry, char *out_path, size_t out_path_size, u8 illegal_char_replace_type); +/// Checks if a RomFS file entry is updated by the Patch RomFS. +/// Only works if the provided RomFileSystemContext was initialized as a Patch RomFS context. +bool romfsIsFileEntryUpdated(RomFileSystemContext *ctx, RomFileSystemFileEntry *file_entry, bool *out); + /// Generates HierarchicalSha256 (NCA0) / HierarchicalIntegrity (NCA2/NCA3) FS section patch data using a RomFS context + file entry, which can be used to seamlessly replace NCA data. /// Input offset must be relative to the start of the RomFS file entry data. /// This function shares the same limitations as ncaGenerateHierarchicalSha256Patch() / ncaGenerateHierarchicalIntegrityPatch(). diff --git a/source/core/bktr.c b/source/core/bktr.c index f933972..82499d6 100644 --- a/source/core/bktr.c +++ b/source/core/bktr.c @@ -204,8 +204,8 @@ bool bktrSetBucketTreeSubStorage(BucketTreeContext *parent_ctx, BucketTreeContex child_ctx->storage_type > BucketTreeStorageType_Sparse || (child_ctx->storage_type == BucketTreeStorageType_AesCtrEx && (substorage_index != 1 || \ parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx)) || ((child_ctx->storage_type == BucketTreeStorageType_Compressed || \ child_ctx->storage_type == BucketTreeStorageType_Sparse) && (substorage_index != 0 || parent_ctx->nca_fs_ctx == child_ctx->nca_fs_ctx)))) || \ - (parent_ctx->storage_type == BucketTreeStorageType_Compressed && ((child_ctx->storage_type != BucketTreeStorageType_Indirect && \ - child_ctx->storage_type != BucketTreeStorageType_Sparse) || parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx))) + (parent_ctx->storage_type == BucketTreeStorageType_Compressed && (child_ctx->storage_type != BucketTreeStorageType_Sparse || \ + parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx))) { LOG_MSG("Invalid parameters!"); return false; @@ -264,6 +264,89 @@ end: return success; } +bool bktrIsBlockWithinIndirectStorageRange(BucketTreeContext *ctx, u64 offset, u64 size, bool *out) +{ + if (!bktrIsBlockWithinStorageRange(ctx, size, offset) || ctx->storage_type != BucketTreeStorageType_Indirect || !out) + { + LOG_MSG("Invalid parameters!"); + return false; + } + + BucketTreeIndirectStorageEntry *start_entry = NULL, *end_entry = NULL; + BucketTreeVisitor visitor = {0}; + u64 end_offset = 0; + bool updated = false, success = false; + + /* Find storage entry. */ + if (!bktrFindStorageEntry(ctx, offset, &visitor)) + { + LOG_MSG("Unable to find %s storage entry for offset 0x%lX!", bktrGetStorageTypeName(ctx->storage_type), offset); + goto end; + } + + /* Validate start entry node. */ + start_entry = end_entry = (BucketTreeIndirectStorageEntry*)visitor.entry; + if (!bktrIsOffsetWithinStorageRange(ctx, start_entry->virtual_offset) || start_entry->virtual_offset > offset) + { + LOG_MSG("Invalid Indirect Storage entry! (0x%lX) (#1).", start_entry->virtual_offset); + goto end; + } + + /* Move visitor until we reach the end entry node. */ + while(end_entry->virtual_offset < (offset + size) && bktrVisitorCanMoveNext(&visitor)) + { + /* Retrieve the next entry. */ + if (!bktrVisitorMoveNext(&visitor)) + { + LOG_MSG("Failed to retrieve next Indirect Storage entry!"); + goto end; + } + + /* Validate current entry node. */ + end_entry = (BucketTreeIndirectStorageEntry*)visitor.entry; + if (!bktrIsOffsetWithinStorageRange(ctx, end_entry->virtual_offset)) + { + LOG_MSG("Invalid Indirect Storage entry! (0x%lX) (#2).", end_entry->virtual_offset); + goto end; + } + } + + /* Verify end entry virtual offset. */ + end_offset = (end_entry == start_entry ? ctx->end_offset : end_entry->virtual_offset); + if (end_offset <= start_entry->virtual_offset || offset >= end_offset) + { + LOG_MSG("Invalid virtual offset for the Indirect Storage's next entry! (0x%lX).", end_offset); + goto end; + } + + /* Short-circuit: check if the block is contained within a single Indirect Storage entry node. */ + if (end_entry == start_entry) + { + updated = (start_entry->storage_index == BucketTreeIndirectStorageIndex_Patch); + success = true; + goto end; + } + + /* Loop through adjacent Indirect Storage entry nodes and check if at least one of them uses the Patch storage index. */ + while(start_entry < end_entry) + { + if (start_entry->storage_index == BucketTreeIndirectStorageIndex_Patch) + { + updated = true; + break; + } + + start_entry++; + } + + /* Update output values. */ + *out = updated; + success = true; + +end: + return success; +} + static const char *bktrGetStorageTypeName(u8 storage_type) { return (storage_type < BucketTreeStorageType_Count ? g_bktrStorageTypeNames[storage_type] : NULL); @@ -368,7 +451,8 @@ static bool bktrReadIndirectStorage(BucketTreeVisitor *visitor, void *out, u64 r if (!out || (is_sparse && (missing_original_storage || ctx->substorages[0].type != BucketTreeSubStorageType_Regular)) || \ (!is_sparse && (!bktrIsValidSubstorage(&(ctx->substorages[1])) || ctx->substorages[1].type != BucketTreeSubStorageType_AesCtrEx || \ (!missing_original_storage && ((ctx->substorages[0].type != BucketTreeSubStorageType_Regular && \ - ctx->substorages[0].type != BucketTreeStorageType_Compressed && ctx->substorages[0].type != BucketTreeSubStorageType_Sparse)))))) + ctx->substorages[0].type != BucketTreeStorageType_Compressed && ctx->substorages[0].type != BucketTreeSubStorageType_Sparse))))) || \ + (offset + read_size) > ctx->end_offset) { LOG_MSG("Invalid parameters!"); return false; @@ -422,13 +506,6 @@ static bool bktrReadIndirectStorage(BucketTreeVisitor *visitor, void *out, u64 r goto end; } - /* Verify read area size. */ - if ((offset + read_size) > ctx->end_offset) - { - LOG_MSG("Error: read area exceeds Indirect Storage size!"); - goto end; - } - /* Perform read operation. */ if ((offset + read_size) <= next_entry_offset) { @@ -546,7 +623,7 @@ static bool bktrReadAesCtrExStorage(BucketTreeVisitor *visitor, void *out, u64 r { BucketTreeContext *ctx = visitor->bktr_ctx; - if (!out || !bktrIsValidSubstorage(&(ctx->substorages[0])) || ctx->substorages[0].type != BucketTreeSubStorageType_Regular) + if (!out || !bktrIsValidSubstorage(&(ctx->substorages[0])) || ctx->substorages[0].type != BucketTreeSubStorageType_Regular || (offset + read_size) > ctx->end_offset) { LOG_MSG("Invalid parameters!"); return false; @@ -600,13 +677,6 @@ static bool bktrReadAesCtrExStorage(BucketTreeVisitor *visitor, void *out, u64 r goto end; } - /* Verify read area size. */ - if ((offset + read_size) > ctx->end_offset) - { - LOG_MSG("Error: read area exceeds AesCtrEx Storage size!"); - goto end; - } - /* Perform read operation. */ if ((offset + read_size) <= next_entry_offset) { @@ -703,8 +773,8 @@ static bool bktrReadCompressedStorage(BucketTreeVisitor *visitor, void *out, u64 NcaFsSectionContext *nca_fs_ctx = ctx->nca_fs_ctx; u64 compressed_storage_base_offset = nca_fs_ctx->hash_region.size; - if (!out || !bktrIsValidSubstorage(&(ctx->substorages[0])) || ctx->substorages[0].type == BucketTreeSubStorageType_AesCtrEx || \ - ctx->substorages[0].type >= BucketTreeSubStorageType_Compressed) + if (!out || !bktrIsValidSubstorage(&(ctx->substorages[0])) || (ctx->substorages[0].type != BucketTreeSubStorageType_Regular && \ + ctx->substorages[0].type != BucketTreeSubStorageType_Sparse) || (offset + read_size) > ctx->end_offset) { LOG_MSG("Invalid parameters!"); return false; @@ -769,13 +839,6 @@ static bool bktrReadCompressedStorage(BucketTreeVisitor *visitor, void *out, u64 goto end; } - /* Verify read area size. */ - if ((offset + read_size) > ctx->end_offset) - { - LOG_MSG("Error: read area exceeds Compressed Storage size!"); - goto end; - } - /* Perform read operation. */ if ((offset + read_size) <= next_entry_offset) { diff --git a/source/core/nca_storage.c b/source/core/nca_storage.c index e560fb8..5b3a77d 100644 --- a/source/core/nca_storage.c +++ b/source/core/nca_storage.c @@ -29,8 +29,8 @@ NX_INLINE bool ncaStorageIsValidContext(NcaStorageContext *ctx); bool ncaStorageInitializeContext(NcaStorageContext *out, NcaFsSectionContext *nca_fs_ctx) { - /* TODO: allow patches with sparse layers? */ - if (!out || !nca_fs_ctx || !nca_fs_ctx->enabled || (nca_fs_ctx->has_sparse_layer && nca_fs_ctx->section_type == NcaFsSectionType_PatchRomFs)) + if (!out || !nca_fs_ctx || !nca_fs_ctx->enabled || (nca_fs_ctx->section_type == NcaFsSectionType_PatchRomFs && \ + (!nca_fs_ctx->has_patch_indirect_layer || !nca_fs_ctx->has_patch_aes_ctr_ex_layer || nca_fs_ctx->has_sparse_layer || nca_fs_ctx->has_compression_layer))) { LOG_MSG("Invalid parameters!"); return false; @@ -90,9 +90,6 @@ bool ncaStorageInitializeContext(NcaStorageContext *out, NcaFsSectionContext *nc case NcaStorageBaseStorageType_Sparse: if (!bktrSetBucketTreeSubStorage(out->compressed_storage, out->sparse_storage, 0)) goto end; break; - case NcaStorageBaseStorageType_Indirect: - if (!bktrSetBucketTreeSubStorage(out->compressed_storage, out->indirect_storage, 0)) goto end; - break; } /* Update base storage type. */ @@ -230,6 +227,22 @@ bool ncaStorageRead(NcaStorageContext *ctx, void *out, u64 read_size, u64 offset return success; } +bool ncaStorageIsBlockWithinPatchStorageRange(NcaStorageContext *ctx, u64 offset, u64 size, bool *out) +{ + if (!ncaStorageIsValidContext(ctx) || ctx->nca_fs_ctx->section_type != NcaFsSectionType_PatchRomFs || !ctx->indirect_storage || \ + ctx->base_storage_type != NcaStorageBaseStorageType_Indirect || !out) + { + LOG_MSG("Invalid parameters!"); + return false; + } + + /* Check if the provided block extents are within the Indirect Storage's range. */ + bool success = bktrIsBlockWithinIndirectStorageRange(ctx->indirect_storage, offset, size, out); + if (!success) LOG_MSG("Failed to determine if block extents are within the Indirect Storage's range!"); + + return success; +} + void ncaStorageFreeContext(NcaStorageContext *ctx) { if (!ctx) return; diff --git a/source/core/romfs.c b/source/core/romfs.c index 32eab01..1935c13 100644 --- a/source/core/romfs.c +++ b/source/core/romfs.c @@ -217,9 +217,9 @@ bool romfsReadFileEntryData(RomFileSystemContext *ctx, RomFileSystemFileEntry *f return true; } -bool romfsGetTotalDataSize(RomFileSystemContext *ctx, u64 *out_size) +bool romfsGetTotalDataSize(RomFileSystemContext *ctx, bool only_updated, u64 *out_size) { - if (!romfsIsValidContext(ctx) || !out_size) + if (!romfsIsValidContext(ctx) || !out_size || (only_updated && (!ctx->is_patch || ctx->default_storage_ctx->nca_fs_ctx->section_type != NcaFsSectionType_PatchRomFs))) { LOG_MSG("Invalid parameters!"); return false; @@ -242,8 +242,9 @@ bool romfsGetTotalDataSize(RomFileSystemContext *ctx, u64 *out_size) goto end; } - /* Update total data size. */ - total_size += file_entry->size; + /* Update total data size, taking into account the only_updated flag. */ + bool updated = false; + if (!only_updated || (only_updated && romfsIsFileEntryUpdated(ctx, file_entry, &updated) && updated)) total_size += file_entry->size; /* Move to the next file entry. */ if (!romfsMoveToNextFileEntry(ctx)) @@ -610,6 +611,39 @@ end: return success; } +bool romfsIsFileEntryUpdated(RomFileSystemContext *ctx, RomFileSystemFileEntry *file_entry, bool *out) +{ + if (!romfsIsValidContext(ctx) || !ctx->is_patch || ctx->default_storage_ctx->nca_fs_ctx->section_type != NcaFsSectionType_PatchRomFs || \ + !file_entry || !file_entry->size || (file_entry->offset + file_entry->size) > ctx->size || !out) + { + LOG_MSG("Invalid parameters!"); + return false; + } + + u64 file_offset = (ctx->offset + ctx->body_offset + file_entry->offset); + bool success = false; + + /* Short-circuit: check if we're dealing with a Patch RomFS with a missing base RomFS. */ + if (!ncaStorageIsValidContext(&(ctx->storage_ctx[0]))) + { + *out = success = true; + goto end; + } + + /* Check if any sections from this block belong to the Patch storage. */ + if (!ncaStorageIsBlockWithinPatchStorageRange(ctx->default_storage_ctx, file_offset, file_entry->size, out)) + { + LOG_MSG("Failed to determine if file entry is within Patch storage range!"); + goto end; + } + + /* Update return value. */ + success = true; + +end: + return success; +} + bool romfsGenerateFileEntryPatch(RomFileSystemContext *ctx, RomFileSystemFileEntry *file_entry, const void *data, u64 data_size, u64 data_offset, RomFileSystemFileEntryPatch *out) { if (!romfsIsValidContext(ctx) || ctx->is_patch || ctx->default_storage_ctx->base_storage_type != NcaStorageBaseStorageType_Regular || \