mirrors/nxdumptool
1
0
Fork 0
mirror of https://github.com/DarkMatterCore/nxdumptool.git synced 2024-11-22 10:16:39 +00:00
Code Issues Releases Activity

romfs: implement romfsIsFileEntryUpdated().

Browse source 
Also modified romfsGetTotalDataSize() to add an 'only_updated' argument.
This commit is contained in:
Pablo Curiel 2022-07-07 02:30:45 +02:00
parent d8c0984115
commit 6bf314bcea
8 changed files with 200 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
code_templates/sd_romfs_dumper.c
View file

@ -108,7 +108,7 @@ static void read_thread_func(void *arg)
romfsResetFileTableOffset(shared_data->romfs_ctx);
/* Loop through all file entries. */
while(romfsCanMoveToNextFileEntry(shared_data->romfs_ctx))
while(shared_data->data_written < shared_data->total_size && romfsCanMoveToNextFileEntry(shared_data->romfs_ctx))
{
/* Check if the transfer has been cancelled by the user. */
if (shared_data->transfer_cancelled)
@ -634,7 +634,11 @@ int main(int argc, char *argv[])
}
shared_data.romfs_ctx = &romfs_ctx;
romfsGetTotalDataSize(&romfs_ctx, &(shared_data.total_size));
if (!romfsGetTotalDataSize(&romfs_ctx, false, &(shared_data.total_size)) || !shared_data.total_size)
{
consolePrint("failed to retrieve total romfs size\n");
goto out2;
}
consolePrint("romfs initialize ctx succeeded\n");

31
code_templates/usb_romfs_dumper.c
View file

@ -107,7 +107,7 @@ static void read_thread_func(void *arg)
romfsResetFileTableOffset(shared_data->romfs_ctx);
/* Loop through all file entries. */
while(romfsCanMoveToNextFileEntry(shared_data->romfs_ctx))
while(shared_data->data_written < shared_data->total_size && romfsCanMoveToNextFileEntry(shared_data->romfs_ctx))
{
/* Check if the transfer has been cancelled by the user */
if (shared_data->transfer_cancelled)
@ -117,6 +117,28 @@ static void read_thread_func(void *arg)
}
/* Retrieve RomFS file entry information */
/*shared_data->read_error = (!(file_entry = romfsGetCurrentFileEntry(shared_data->romfs_ctx)));
if (shared_data->read_error)
{
condvarWakeAll(&g_writeCondvar);
break;
}
bool updated = false;
if (!romfsIsFileEntryUpdated(shared_data->romfs_ctx, file_entry, &updated) || !updated)
{
shared_data->read_error = !romfsMoveToNextFileEntry(shared_data->romfs_ctx);
if (shared_data->read_error)
{
condvarWakeAll(&g_writeCondvar);
break;
}
continue;
}
shared_data->read_error = !romfsGeneratePathFromFileEntry(shared_data->romfs_ctx, file_entry, path, FS_MAX_PATH, RomFileSystemPathIllegalCharReplaceType_IllegalFsChars);*/
shared_data->read_error = (!(file_entry = romfsGetCurrentFileEntry(shared_data->romfs_ctx)) || \
!romfsGeneratePathFromFileEntry(shared_data->romfs_ctx, file_entry, path, FS_MAX_PATH, RomFileSystemPathIllegalCharReplaceType_IllegalFsChars));
if (shared_data->read_error)
@ -599,7 +621,12 @@ int main(int argc, char *argv[])
}
shared_data.romfs_ctx = &romfs_ctx;
romfsGetTotalDataSize(&romfs_ctx, &(shared_data.total_size));
//if (!romfsGetTotalDataSize(&romfs_ctx, true, &(shared_data.total_size)) || !shared_data.total_size)
if (!romfsGetTotalDataSize(&romfs_ctx, false, &(shared_data.total_size)) || !shared_data.total_size)
{
consolePrint("failed to retrieve total romfs size\n");
goto out2;
}
consolePrint("romfs initialize ctx succeeded\n");

17
include/core/bktr.h
View file

@ -151,13 +151,13 @@ typedef enum {
typedef enum {
BucketTreeSubStorageType_Regular = 0, ///< Body storage with None, XTS or CTR crypto. Most common substorage type, used in all title types.
///< May be used as substorage for all BucketTreeStorage types.
BucketTreeSubStorageType_Indirect = 1, ///< Indirect storage from patch NCAs. May be used as substorage for BucketTreeStorageType_Compressed only.
BucketTreeSubStorageType_AesCtrEx = 2, ///< AesCtrEx storage from patch NCAs. May be used as substorage for BucketTreeStorageType_Indirect only.
BucketTreeSubStorageType_Compressed = 3, ///< Compressed storage. If available, this is always the outmost storage type for any NCA. May be used by all title types.
///< May be used as substorage for BucketTreeStorageType_Indirect only.
BucketTreeSubStorageType_Sparse = 4, ///< Sparse storage with CTR crypto, using virtual offsets as lower CTR IVs. Used in base applications only.
///< May be used as substorage for BucketTreeStorageType_Compressed or BucketTreeStorageType_Indirect.
///< May be used as substorage for all other BucketTreeStorage types.
BucketTreeSubStorageType_Indirect = 1, ///< Indirect storage. Only used in patches. This is always the outmost storage type.
BucketTreeSubStorageType_AesCtrEx = 2, ///< AesCtrEx storage. Only used in patches. Must be used as substorage #1 for BucketTreeStorageType_Indirect.
BucketTreeSubStorageType_Compressed = 3, ///< Compressed storage. Only used in base applications. If available, this is always the outmost storage type.
///< May be used as substorage #0 for BucketTreeStorageType_Indirect only.
BucketTreeSubStorageType_Sparse = 4, ///< Sparse storage with CTR crypto, using virtual offsets as lower CTR IVs. Only used in base applications.
///< May be used as substorage for BucketTreeStorageType_Compressed or BucketTreeStorageType_Indirect (#0).
BucketTreeSubStorageType_Count = 5 ///< Total values supported by this enum.
} BucketTreeSubStorageType;
@ -195,6 +195,9 @@ bool bktrSetBucketTreeSubStorage(BucketTreeContext *parent_ctx, BucketTreeContex
/// Reads data from a Bucket Tree storage using a previously initialized BucketTreeContext.
bool bktrReadStorage(BucketTreeContext *ctx, void *out, u64 read_size, u64 offset);
/// Checks if the provided block extents are within the provided BucketTreeContext's Indirect Storage.
bool bktrIsBlockWithinIndirectStorageRange(BucketTreeContext *ctx, u64 offset, u64 size, bool *out);
/// Helper inline functions.
NX_INLINE void bktrFreeContext(BucketTreeContext *ctx)

3
include/core/nca_storage.h
View file

@ -63,6 +63,9 @@ bool ncaStorageGetHashTargetExtents(NcaStorageContext *ctx, u64 *out_offset, u64
/// Reads data from the NCA storage using a previously initialized NcaStorageContext.
bool ncaStorageRead(NcaStorageContext *ctx, void *out, u64 read_size, u64 offset);
/// Checks if the provided block extents are within the provided Patch NcaStorageContext's Indirect Storage.
bool ncaStorageIsBlockWithinPatchStorageRange(NcaStorageContext *ctx, u64 offset, u64 size, bool *out);
/// Frees a previously initialized NCA storage context.
void ncaStorageFreeContext(NcaStorageContext *ctx);

7
include/core/romfs.h
View file

@ -151,7 +151,8 @@ bool romfsReadFileSystemData(RomFileSystemContext *ctx, void *out, u64 read_size
bool romfsReadFileEntryData(RomFileSystemContext *ctx, RomFileSystemFileEntry *file_entry, void *out, u64 read_size, u64 offset);
/// Calculates the extracted RomFS size.
bool romfsGetTotalDataSize(RomFileSystemContext *ctx, u64 *out_size);
/// If 'only_updated' is set to true and the provided RomFS context was initialized as a Patch RomFS context, only files modified by the update will be considered.
bool romfsGetTotalDataSize(RomFileSystemContext *ctx, bool only_updated, u64 *out_size);
/// Calculates the extracted size from a RomFS directory.
bool romfsGetDirectoryDataSize(RomFileSystemContext *ctx, RomFileSystemDirectoryEntry *dir_entry, u64 *out_size);
@ -170,6 +171,10 @@ bool romfsGeneratePathFromDirectoryEntry(RomFileSystemContext *ctx, RomFileSyste
/// Generates a path string from a RomFS file entry.
bool romfsGeneratePathFromFileEntry(RomFileSystemContext *ctx, RomFileSystemFileEntry *file_entry, char *out_path, size_t out_path_size, u8 illegal_char_replace_type);
/// Checks if a RomFS file entry is updated by the Patch RomFS.
/// Only works if the provided RomFileSystemContext was initialized as a Patch RomFS context.
bool romfsIsFileEntryUpdated(RomFileSystemContext *ctx, RomFileSystemFileEntry *file_entry, bool *out);
/// Generates HierarchicalSha256 (NCA0) / HierarchicalIntegrity (NCA2/NCA3) FS section patch data using a RomFS context + file entry, which can be used to seamlessly replace NCA data.
/// Input offset must be relative to the start of the RomFS file entry data.
/// This function shares the same limitations as ncaGenerateHierarchicalSha256Patch() / ncaGenerateHierarchicalIntegrityPatch().

117
source/core/bktr.c
View file

@ -204,8 +204,8 @@ bool bktrSetBucketTreeSubStorage(BucketTreeContext *parent_ctx, BucketTreeContex
child_ctx->storage_type > BucketTreeStorageType_Sparse || (child_ctx->storage_type == BucketTreeStorageType_AesCtrEx && (substorage_index != 1 || \
parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx)) || ((child_ctx->storage_type == BucketTreeStorageType_Compressed || \
child_ctx->storage_type == BucketTreeStorageType_Sparse) && (substorage_index != 0 || parent_ctx->nca_fs_ctx == child_ctx->nca_fs_ctx)))) || \
(parent_ctx->storage_type == BucketTreeStorageType_Compressed && ((child_ctx->storage_type != BucketTreeStorageType_Indirect && \
child_ctx->storage_type != BucketTreeStorageType_Sparse) || parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx)))
(parent_ctx->storage_type == BucketTreeStorageType_Compressed && (child_ctx->storage_type != BucketTreeStorageType_Sparse || \
parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx)))
{
LOG_MSG("Invalid parameters!");
return false;
@ -264,6 +264,89 @@ end:
return success;
}
bool bktrIsBlockWithinIndirectStorageRange(BucketTreeContext *ctx, u64 offset, u64 size, bool *out)
{
if (!bktrIsBlockWithinStorageRange(ctx, size, offset) || ctx->storage_type != BucketTreeStorageType_Indirect || !out)
{
LOG_MSG("Invalid parameters!");
return false;
}
BucketTreeIndirectStorageEntry *start_entry = NULL, *end_entry = NULL;
BucketTreeVisitor visitor = {0};
u64 end_offset = 0;
bool updated = false, success = false;
/* Find storage entry. */
if (!bktrFindStorageEntry(ctx, offset, &visitor))
{
LOG_MSG("Unable to find %s storage entry for offset 0x%lX!", bktrGetStorageTypeName(ctx->storage_type), offset);
goto end;
}
/* Validate start entry node. */
start_entry = end_entry = (BucketTreeIndirectStorageEntry*)visitor.entry;
if (!bktrIsOffsetWithinStorageRange(ctx, start_entry->virtual_offset) || start_entry->virtual_offset > offset)
{
LOG_MSG("Invalid Indirect Storage entry! (0x%lX) (#1).", start_entry->virtual_offset);
goto end;
}
/* Move visitor until we reach the end entry node. */
while(end_entry->virtual_offset < (offset + size) && bktrVisitorCanMoveNext(&visitor))
{
/* Retrieve the next entry. */
if (!bktrVisitorMoveNext(&visitor))
{
LOG_MSG("Failed to retrieve next Indirect Storage entry!");
goto end;
}
/* Validate current entry node. */
end_entry = (BucketTreeIndirectStorageEntry*)visitor.entry;
if (!bktrIsOffsetWithinStorageRange(ctx, end_entry->virtual_offset))
{
LOG_MSG("Invalid Indirect Storage entry! (0x%lX) (#2).", end_entry->virtual_offset);
goto end;
}
}
/* Verify end entry virtual offset. */
end_offset = (end_entry == start_entry ? ctx->end_offset : end_entry->virtual_offset);
if (end_offset <= start_entry->virtual_offset || offset >= end_offset)
{
LOG_MSG("Invalid virtual offset for the Indirect Storage's next entry! (0x%lX).", end_offset);
goto end;
}
/* Short-circuit: check if the block is contained within a single Indirect Storage entry node. */
if (end_entry == start_entry)
{
updated = (start_entry->storage_index == BucketTreeIndirectStorageIndex_Patch);
success = true;
goto end;
}
/* Loop through adjacent Indirect Storage entry nodes and check if at least one of them uses the Patch storage index. */
while(start_entry < end_entry)
{
if (start_entry->storage_index == BucketTreeIndirectStorageIndex_Patch)
{
updated = true;
break;
}
start_entry++;
}
/* Update output values. */
*out = updated;
success = true;
end:
return success;
}
static const char *bktrGetStorageTypeName(u8 storage_type)
{
return (storage_type < BucketTreeStorageType_Count ? g_bktrStorageTypeNames[storage_type] : NULL);
@ -368,7 +451,8 @@ static bool bktrReadIndirectStorage(BucketTreeVisitor *visitor, void *out, u64 r
if (!out || (is_sparse && (missing_original_storage || ctx->substorages[0].type != BucketTreeSubStorageType_Regular)) || \
(!is_sparse && (!bktrIsValidSubstorage(&(ctx->substorages[1])) || ctx->substorages[1].type != BucketTreeSubStorageType_AesCtrEx || \
(!missing_original_storage && ((ctx->substorages[0].type != BucketTreeSubStorageType_Regular && \
ctx->substorages[0].type != BucketTreeStorageType_Compressed && ctx->substorages[0].type != BucketTreeSubStorageType_Sparse))))))
ctx->substorages[0].type != BucketTreeStorageType_Compressed && ctx->substorages[0].type != BucketTreeSubStorageType_Sparse))))) || \
(offset + read_size) > ctx->end_offset)
{
LOG_MSG("Invalid parameters!");
return false;
@ -422,13 +506,6 @@ static bool bktrReadIndirectStorage(BucketTreeVisitor *visitor, void *out, u64 r
goto end;
}
/* Verify read area size. */
if ((offset + read_size) > ctx->end_offset)
{
LOG_MSG("Error: read area exceeds Indirect Storage size!");
goto end;
}
/* Perform read operation. */
if ((offset + read_size) <= next_entry_offset)
{
@ -546,7 +623,7 @@ static bool bktrReadAesCtrExStorage(BucketTreeVisitor *visitor, void *out, u64 r
{
BucketTreeContext *ctx = visitor->bktr_ctx;
if (!out || !bktrIsValidSubstorage(&(ctx->substorages[0])) || ctx->substorages[0].type != BucketTreeSubStorageType_Regular)
if (!out || !bktrIsValidSubstorage(&(ctx->substorages[0])) || ctx->substorages[0].type != BucketTreeSubStorageType_Regular || (offset + read_size) > ctx->end_offset)
{
LOG_MSG("Invalid parameters!");
return false;
@ -600,13 +677,6 @@ static bool bktrReadAesCtrExStorage(BucketTreeVisitor *visitor, void *out, u64 r
goto end;
}
/* Verify read area size. */
if ((offset + read_size) > ctx->end_offset)
{
LOG_MSG("Error: read area exceeds AesCtrEx Storage size!");
goto end;
}
/* Perform read operation. */
if ((offset + read_size) <= next_entry_offset)
{
@ -703,8 +773,8 @@ static bool bktrReadCompressedStorage(BucketTreeVisitor *visitor, void *out, u64
NcaFsSectionContext *nca_fs_ctx = ctx->nca_fs_ctx;
u64 compressed_storage_base_offset = nca_fs_ctx->hash_region.size;
if (!out || !bktrIsValidSubstorage(&(ctx->substorages[0])) || ctx->substorages[0].type == BucketTreeSubStorageType_AesCtrEx || \
ctx->substorages[0].type >= BucketTreeSubStorageType_Compressed)
if (!out || !bktrIsValidSubstorage(&(ctx->substorages[0])) || (ctx->substorages[0].type != BucketTreeSubStorageType_Regular && \
ctx->substorages[0].type != BucketTreeSubStorageType_Sparse) || (offset + read_size) > ctx->end_offset)
{
LOG_MSG("Invalid parameters!");
return false;
@ -769,13 +839,6 @@ static bool bktrReadCompressedStorage(BucketTreeVisitor *visitor, void *out, u64
goto end;
}
/* Verify read area size. */
if ((offset + read_size) > ctx->end_offset)
{
LOG_MSG("Error: read area exceeds Compressed Storage size!");
goto end;
}
/* Perform read operation. */
if ((offset + read_size) <= next_entry_offset)
{

23
source/core/nca_storage.c
View file

@ -29,8 +29,8 @@ NX_INLINE bool ncaStorageIsValidContext(NcaStorageContext *ctx);
bool ncaStorageInitializeContext(NcaStorageContext *out, NcaFsSectionContext *nca_fs_ctx)
{
/* TODO: allow patches with sparse layers? */
if (!out || !nca_fs_ctx || !nca_fs_ctx->enabled || (nca_fs_ctx->has_sparse_layer && nca_fs_ctx->section_type == NcaFsSectionType_PatchRomFs))
if (!out || !nca_fs_ctx || !nca_fs_ctx->enabled || (nca_fs_ctx->section_type == NcaFsSectionType_PatchRomFs && \
(!nca_fs_ctx->has_patch_indirect_layer || !nca_fs_ctx->has_patch_aes_ctr_ex_layer || nca_fs_ctx->has_sparse_layer || nca_fs_ctx->has_compression_layer)))
{
LOG_MSG("Invalid parameters!");
return false;
@ -90,9 +90,6 @@ bool ncaStorageInitializeContext(NcaStorageContext *out, NcaFsSectionContext *nc
case NcaStorageBaseStorageType_Sparse:
if (!bktrSetBucketTreeSubStorage(out->compressed_storage, out->sparse_storage, 0)) goto end;
break;
case NcaStorageBaseStorageType_Indirect:
if (!bktrSetBucketTreeSubStorage(out->compressed_storage, out->indirect_storage, 0)) goto end;
break;
}
/* Update base storage type. */
@ -230,6 +227,22 @@ bool ncaStorageRead(NcaStorageContext *ctx, void *out, u64 read_size, u64 offset
return success;
}
bool ncaStorageIsBlockWithinPatchStorageRange(NcaStorageContext *ctx, u64 offset, u64 size, bool *out)
{
if (!ncaStorageIsValidContext(ctx) || ctx->nca_fs_ctx->section_type != NcaFsSectionType_PatchRomFs || !ctx->indirect_storage || \
ctx->base_storage_type != NcaStorageBaseStorageType_Indirect || !out)
{
LOG_MSG("Invalid parameters!");
return false;
}
/* Check if the provided block extents are within the Indirect Storage's range. */
bool success = bktrIsBlockWithinIndirectStorageRange(ctx->indirect_storage, offset, size, out);
if (!success) LOG_MSG("Failed to determine if block extents are within the Indirect Storage's range!");
return success;
}
void ncaStorageFreeContext(NcaStorageContext *ctx)
{
if (!ctx) return;

42
source/core/romfs.c
View file

@ -217,9 +217,9 @@ bool romfsReadFileEntryData(RomFileSystemContext *ctx, RomFileSystemFileEntry *f
return true;
}
bool romfsGetTotalDataSize(RomFileSystemContext *ctx, u64 *out_size)
bool romfsGetTotalDataSize(RomFileSystemContext *ctx, bool only_updated, u64 *out_size)
{
if (!romfsIsValidContext(ctx) || !out_size)
if (!romfsIsValidContext(ctx) || !out_size || (only_updated && (!ctx->is_patch || ctx->default_storage_ctx->nca_fs_ctx->section_type != NcaFsSectionType_PatchRomFs)))
{
LOG_MSG("Invalid parameters!");
return false;
@ -242,8 +242,9 @@ bool romfsGetTotalDataSize(RomFileSystemContext *ctx, u64 *out_size)
goto end;
}
/* Update total data size. */
total_size += file_entry->size;
/* Update total data size, taking into account the only_updated flag. */
bool updated = false;
if (!only_updated || (only_updated && romfsIsFileEntryUpdated(ctx, file_entry, &updated) && updated)) total_size += file_entry->size;
/* Move to the next file entry. */
if (!romfsMoveToNextFileEntry(ctx))
@ -610,6 +611,39 @@ end:
return success;
}
bool romfsIsFileEntryUpdated(RomFileSystemContext *ctx, RomFileSystemFileEntry *file_entry, bool *out)
{
if (!romfsIsValidContext(ctx) || !ctx->is_patch || ctx->default_storage_ctx->nca_fs_ctx->section_type != NcaFsSectionType_PatchRomFs || \
!file_entry || !file_entry->size || (file_entry->offset + file_entry->size) > ctx->size || !out)
{
LOG_MSG("Invalid parameters!");
return false;
}
u64 file_offset = (ctx->offset + ctx->body_offset + file_entry->offset);
bool success = false;
/* Short-circuit: check if we're dealing with a Patch RomFS with a missing base RomFS. */
if (!ncaStorageIsValidContext(&(ctx->storage_ctx[0])))
{
*out = success = true;
goto end;
}
/* Check if any sections from this block belong to the Patch storage. */
if (!ncaStorageIsBlockWithinPatchStorageRange(ctx->default_storage_ctx, file_offset, file_entry->size, out))
{
LOG_MSG("Failed to determine if file entry is within Patch storage range!");
goto end;
}
/* Update return value. */
success = true;
end:
return success;
}
bool romfsGenerateFileEntryPatch(RomFileSystemContext *ctx, RomFileSystemFileEntry *file_entry, const void *data, u64 data_size, u64 data_offset, RomFileSystemFileEntryPatch *out)
{
if (!romfsIsValidContext(ctx) || ctx->is_patch || ctx->default_storage_ctx->base_storage_type != NcaStorageBaseStorageType_Regular || \