From 59d0e0ba901cb7bb982937be178dfca44db3cbe4 Mon Sep 17 00:00:00 2001 From: Pablo Curiel Date: Sat, 9 Jul 2022 14:56:44 +0200 Subject: [PATCH] bktr: handle compression in patches (part 1). Some parts of the code need to be still need to be slightly restructured. bktrIsBlockWithinIndirectStorageRange() must be updated as well, too. --- include/core/bktr.h | 13 ++- include/core/nca.h | 5 +- source/core/bktr.c | 193 ++++++++++++++++++++------------------ source/core/nca.c | 68 +++++++------- source/core/nca_storage.c | 100 +++++++++++++++----- source/core/nso.c | 2 +- 6 files changed, 222 insertions(+), 159 deletions(-) diff --git a/include/core/bktr.h b/include/core/bktr.h index 3d4dbe2..9dd9d3f 100644 --- a/include/core/bktr.h +++ b/include/core/bktr.h @@ -152,9 +152,9 @@ typedef enum { typedef enum { BucketTreeSubStorageType_Regular = 0, ///< Body storage with None, XTS or CTR crypto. Most common substorage type, used in all title types. ///< May be used as substorage for all other BucketTreeStorage types. - BucketTreeSubStorageType_Indirect = 1, ///< Indirect storage. Only used in patches. This is always the outmost storage type. + BucketTreeSubStorageType_Indirect = 1, ///< Indirect storage. Only used in patches. May be used as substorage for BucketTreeStorageType_Compressed only. BucketTreeSubStorageType_AesCtrEx = 2, ///< AesCtrEx storage. Only used in patches. Must be used as substorage #1 for BucketTreeStorageType_Indirect. - BucketTreeSubStorageType_Compressed = 3, ///< Compressed storage. Only used in base applications. If available, this is always the outmost storage type. + BucketTreeSubStorageType_Compressed = 3, ///< Compressed storage. If available, this is always the outmost storage type for any NCA. May be used by all title types. ///< May be used as substorage #0 for BucketTreeStorageType_Indirect only. BucketTreeSubStorageType_Sparse = 4, ///< Sparse storage with CTR crypto, using virtual offsets as lower CTR IVs. Only used in base applications. ///< May be used as substorage for BucketTreeStorageType_Compressed or BucketTreeStorageType_Indirect (#0). @@ -184,12 +184,19 @@ typedef struct { } BucketTreeContext; /// Initializes a Bucket Tree context using the provided NCA FS section context and a storage type. +/// 'storage_type' may only be BucketTreeStorageType_Indirect, BucketTreeStorageType_AesCtrEx or BucketTreeStorageType_Sparse. bool bktrInitializeContext(BucketTreeContext *out, NcaFsSectionContext *nca_fs_ctx, u8 storage_type); -/// Sets a BucketTreeSubStorageType_Regular substorage at index 0. +/// Initializes a Bucket Tree context with type BucketTreeStorageType_Compressed using the provided BucketTreeSubStorage. +bool bktrInitializeCompressedStorageContext(BucketTreeContext *out, BucketTreeSubStorage *substorage); + +/// Sets a BucketTreeSubStorageType_Regular substorage at index 0 in the provided BucketTreeContext. +/// The storage type from the provided BucketTreeContext may only be BucketTreeStorageType_Indirect, BucketTreeStorageType_AesCtrEx or BucketTreeStorageType_Sparse. bool bktrSetRegularSubStorage(BucketTreeContext *ctx, NcaFsSectionContext *nca_fs_ctx); /// Sets a substorage with type >= BucketTreeStorageType_Indirect and <= BucketTreeStorageType_Compressed at the provided index using a previously initialized BucketTreeContext. +/// The storage type from the provided parent BucketTreeContext may only be BucketTreeStorageType_Indirect. +/// The storage type from the provided child BucketTreeContext may only be BucketTreeStorageType_AesCtrEx, BucketTreeStorageType_Compressed, BucketTreeStorageType_Sparse. bool bktrSetBucketTreeSubStorage(BucketTreeContext *parent_ctx, BucketTreeContext *child_ctx, u8 substorage_index); /// Reads data from a Bucket Tree storage using a previously initialized BucketTreeContext. diff --git a/include/core/nca.h b/include/core/nca.h index f874732..b7502b2 100644 --- a/include/core/nca.h +++ b/include/core/nca.h @@ -384,7 +384,6 @@ typedef struct { ///< CompressionInfo-related fields. bool has_compression_layer; ///< Set to true if this NCA FS section has a compression layer. - u64 compression_table_offset; ///< hash_target_offset + header.compression_info.bucket.offset. Relative to the start of the FS section. Placed here for convenience. ///< Hash-layer-related fields. bool skip_hash_layer_crypto; ///< Set to true if hash layer crypto should be skipped while reading section data. @@ -495,12 +494,12 @@ bool ncaGetFsSectionHashTargetExtents(NcaFsSectionContext *ctx, u64 *out_offset, /// Reads decrypted data from a NCA FS section using an input context. /// Input offset must be relative to the start of the NCA FS section. -/// If dealing with Patch RomFS sections, this function should only be used when *not* reading BKTR AesCtrEx storage data. Use ncaReadAesCtrExStorageFromBktrSection() for that. +/// If dealing with Patch RomFS sections, this function should only be used when *not* reading AesCtrEx storage data. Use ncaReadAesCtrExStorage() for that. bool ncaReadFsSection(NcaFsSectionContext *ctx, void *out, u64 read_size, u64 offset); /// Reads plaintext AesCtrEx storage data from a NCA Patch RomFS section using an input context and an AesCtrEx CTR value. /// Input offset must be relative to the start of the NCA FS section. -bool ncaReadAesCtrExStorageFromBktrSection(NcaFsSectionContext *ctx, void *out, u64 read_size, u64 offset, u32 ctr_val, bool decrypt); +bool ncaReadAesCtrExStorage(NcaFsSectionContext *ctx, void *out, u64 read_size, u64 offset, u32 ctr_val, bool decrypt); /// Generates HierarchicalSha256 FS section patch data, which can be used to seamlessly replace NCA data. /// Input offset must be relative to the start of the last HierarchicalSha256 hash region (actual underlying FS). diff --git a/source/core/bktr.c b/source/core/bktr.c index 7a745cb..5bd8a89 100644 --- a/source/core/bktr.c +++ b/source/core/bktr.c @@ -80,7 +80,6 @@ static bool bktrReadIndirectStorage(BucketTreeVisitor *visitor, void *out, u64 r static bool bktrInitializeAesCtrExStorageContext(BucketTreeContext *out, NcaFsSectionContext *nca_fs_ctx); static bool bktrReadAesCtrExStorage(BucketTreeVisitor *visitor, void *out, u64 read_size, u64 offset); -static bool bktrInitializeCompressedStorageContext(BucketTreeContext *out, NcaFsSectionContext *nca_fs_ctx); static bool bktrReadCompressedStorage(BucketTreeVisitor *visitor, void *out, u64 read_size, u64 offset); static bool bktrReadSubStorage(BucketTreeSubStorage *substorage, BucketTreeSubStorageReadParams *params); @@ -136,8 +135,8 @@ bool bktrInitializeContext(BucketTreeContext *out, NcaFsSectionContext *nca_fs_c { NcaContext *nca_ctx = NULL; - if (!out || storage_type >= BucketTreeStorageType_Count || !nca_fs_ctx || !nca_fs_ctx->enabled || nca_fs_ctx->section_type >= NcaFsSectionType_Invalid || \ - !(nca_ctx = (NcaContext*)nca_fs_ctx->nca_ctx) || (nca_ctx->rights_id_available && !nca_ctx->titlekey_retrieved)) + if (!out || !nca_fs_ctx || !nca_fs_ctx->enabled || nca_fs_ctx->section_type >= NcaFsSectionType_Invalid || !(nca_ctx = (NcaContext*)nca_fs_ctx->nca_ctx) || \ + (nca_ctx->rights_id_available && !nca_ctx->titlekey_retrieved) || storage_type == BucketTreeStorageType_Compressed || storage_type >= BucketTreeStorageType_Count) { LOG_MSG("Invalid parameters!"); return false; @@ -158,9 +157,6 @@ bool bktrInitializeContext(BucketTreeContext *out, NcaFsSectionContext *nca_fs_c case BucketTreeStorageType_AesCtrEx: success = bktrInitializeAesCtrExStorageContext(out, nca_fs_ctx); break; - case BucketTreeStorageType_Compressed: - success = bktrInitializeCompressedStorageContext(out, nca_fs_ctx); - break; default: break; } @@ -171,13 +167,95 @@ bool bktrInitializeContext(BucketTreeContext *out, NcaFsSectionContext *nca_fs_c return success; } +bool bktrInitializeCompressedStorageContext(BucketTreeContext *out, BucketTreeSubStorage *substorage) +{ + NcaFsSectionContext *nca_fs_ctx = NULL; + NcaContext *nca_ctx = NULL; + + if (!out || !substorage || substorage->index != 0 || !(nca_fs_ctx = substorage->nca_fs_ctx) || !nca_fs_ctx->enabled || !nca_fs_ctx->has_compression_layer || \ + nca_fs_ctx->section_type >= NcaFsSectionType_Invalid || !(nca_ctx = (NcaContext*)nca_fs_ctx->nca_ctx) || (nca_ctx->rights_id_available && !nca_ctx->titlekey_retrieved) || \ + substorage->type == BucketTreeSubStorageType_AesCtrEx || substorage->type == BucketTreeSubStorageType_Compressed || substorage->type >= BucketTreeSubStorageType_Count || \ + (substorage->type == BucketTreeSubStorageType_Regular && substorage->bktr_ctx) || (substorage->type != BucketTreeSubStorageType_Regular && !substorage->bktr_ctx)) + { + LOG_MSG("Invalid parameters!"); + return false; + } + + /* Free output context beforehand. */ + bktrFreeContext(out); + + NcaBucketInfo *compressed_bucket = &(nca_fs_ctx->header.compression_info.bucket); + BucketTreeTable *compressed_table = NULL; + u64 node_storage_size = 0, entry_storage_size = 0; + BucketTreeSubStorageReadParams params = {0}; + bool success = false; + + /* Verify bucket info. */ + if (!bktrVerifyBucketInfo(compressed_bucket, BKTR_NODE_SIZE, BKTR_COMPRESSED_ENTRY_SIZE, &node_storage_size, &entry_storage_size)) + { + LOG_MSG("Compressed Storage BucketInfo verification failed!"); + goto end; + } + + /* Allocate memory for the full Compressed table. */ + compressed_table = calloc(1, compressed_bucket->size); + if (!compressed_table) + { + LOG_MSG("Unable to allocate memory for the Compressed Storage Table!"); + goto end; + } + + /* Read Compressed storage table data. */ + const u64 compression_table_offset = (nca_fs_ctx->hash_region.size + compressed_bucket->offset); + bktrBucketInitializeSubStorageReadParams(¶ms, compressed_table, compression_table_offset, compressed_bucket->size, 0, 0, false, BucketTreeSubStorageType_Compressed); + + if (!bktrReadSubStorage(substorage, ¶ms)) + { + LOG_MSG("Failed to read Compressed Storage Table data!"); + goto end; + } + + /* Validate table offset node. */ + u64 start_offset = 0, end_offset = 0; + if (!bktrValidateTableOffsetNode(compressed_table, BKTR_NODE_SIZE, BKTR_COMPRESSED_ENTRY_SIZE, compressed_bucket->header.entry_count, &start_offset, &end_offset)) + { + LOG_MSG("Compressed Storage Table Offset Node validation failed!"); + goto end; + } + + /* Update output context. */ + out->nca_fs_ctx = nca_fs_ctx; + out->storage_type = BucketTreeStorageType_Compressed; + out->storage_table = compressed_table; + out->node_size = BKTR_NODE_SIZE; + out->entry_size = BKTR_COMPRESSED_ENTRY_SIZE; + out->offset_count = bktrGetOffsetCount(BKTR_NODE_SIZE); + out->entry_set_count = bktrGetEntrySetCount(BKTR_NODE_SIZE, BKTR_COMPRESSED_ENTRY_SIZE, compressed_bucket->header.entry_count); + out->node_storage_size = node_storage_size; + out->entry_storage_size = entry_storage_size; + out->start_offset = start_offset; + out->end_offset = end_offset; + + memcpy(&(out->substorages[0]), substorage, sizeof(BucketTreeSubStorage)); + + /* Update return value. */ + success = true; + +end: + if (!success && compressed_table) free(compressed_table); + + return success; +} + bool bktrSetRegularSubStorage(BucketTreeContext *ctx, NcaFsSectionContext *nca_fs_ctx) { NcaContext *nca_ctx = NULL; if (!bktrIsValidContext(ctx) || !nca_fs_ctx || !nca_fs_ctx->enabled || nca_fs_ctx->section_type >= NcaFsSectionType_Invalid || \ !(nca_ctx = (NcaContext*)nca_fs_ctx->nca_ctx) || (nca_ctx->rights_id_available && !nca_ctx->titlekey_retrieved) || \ - (ctx->storage_type >= BucketTreeStorageType_AesCtrEx && ctx->storage_type <= BucketTreeStorageType_Sparse && ctx->nca_fs_ctx != nca_fs_ctx)) + ctx->storage_type == BucketTreeStorageType_Compressed || ctx->storage_type >= BucketTreeStorageType_Count || \ + (ctx->storage_type == BucketTreeStorageType_Indirect && ctx->nca_fs_ctx == nca_fs_ctx) || \ + ((ctx->storage_type == BucketTreeStorageType_AesCtrEx || ctx->storage_type == BucketTreeStorageType_Sparse) && ctx->nca_fs_ctx != nca_fs_ctx)) { LOG_MSG("Invalid parameters!"); return false; @@ -198,14 +276,10 @@ bool bktrSetRegularSubStorage(BucketTreeContext *ctx, NcaFsSectionContext *nca_f bool bktrSetBucketTreeSubStorage(BucketTreeContext *parent_ctx, BucketTreeContext *child_ctx, u8 substorage_index) { if (!bktrIsValidContext(parent_ctx) || !bktrIsValidContext(child_ctx) || substorage_index >= BKTR_MAX_SUBSTORAGE_COUNT || \ - parent_ctx->storage_type == BucketTreeStorageType_AesCtrEx || parent_ctx->storage_type == BucketTreeStorageType_Sparse || \ - (parent_ctx->storage_type != BucketTreeStorageType_Indirect && substorage_index != 0) || \ - (parent_ctx->storage_type == BucketTreeStorageType_Indirect && (child_ctx->storage_type < BucketTreeStorageType_AesCtrEx || \ + parent_ctx->storage_type != BucketTreeStorageType_Indirect || child_ctx->storage_type < BucketTreeStorageType_AesCtrEx || \ child_ctx->storage_type > BucketTreeStorageType_Sparse || (child_ctx->storage_type == BucketTreeStorageType_AesCtrEx && (substorage_index != 1 || \ parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx)) || ((child_ctx->storage_type == BucketTreeStorageType_Compressed || \ - child_ctx->storage_type == BucketTreeStorageType_Sparse) && (substorage_index != 0 || parent_ctx->nca_fs_ctx == child_ctx->nca_fs_ctx)))) || \ - (parent_ctx->storage_type == BucketTreeStorageType_Compressed && (child_ctx->storage_type != BucketTreeStorageType_Sparse || \ - parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx))) + child_ctx->storage_type == BucketTreeStorageType_Sparse) && (substorage_index != 0 || parent_ctx->nca_fs_ctx == child_ctx->nca_fs_ctx))) { LOG_MSG("Invalid parameters!"); return false; @@ -430,9 +504,8 @@ static bool bktrReadIndirectStorage(BucketTreeVisitor *visitor, void *out, u64 r if (!out || (is_sparse && (missing_original_storage || ctx->substorages[0].type != BucketTreeSubStorageType_Regular)) || \ (!is_sparse && (!bktrIsValidSubstorage(&(ctx->substorages[1])) || ctx->substorages[1].type != BucketTreeSubStorageType_AesCtrEx || \ - (!missing_original_storage && ((ctx->substorages[0].type != BucketTreeSubStorageType_Regular && \ - ctx->substorages[0].type != BucketTreeStorageType_Compressed && ctx->substorages[0].type != BucketTreeSubStorageType_Sparse))))) || \ - (offset + read_size) > ctx->end_offset) + (!missing_original_storage && (ctx->substorages[0].type == BucketTreeSubStorageType_Indirect || ctx->substorages[0].type == BucketTreeSubStorageType_AesCtrEx || \ + ctx->substorages[0].type >= BucketTreeSubStorageType_Count)))) || (offset + read_size) > ctx->end_offset) { LOG_MSG("Invalid parameters!"); return false; @@ -682,79 +755,13 @@ end: return success; } -static bool bktrInitializeCompressedStorageContext(BucketTreeContext *out, NcaFsSectionContext *nca_fs_ctx) -{ - if (!nca_fs_ctx->has_compression_layer) - { - LOG_MSG("Invalid parameters!"); - return false; - } - - NcaBucketInfo *compressed_bucket = &(nca_fs_ctx->header.compression_info.bucket); - BucketTreeTable *compressed_table = NULL; - u64 node_storage_size = 0, entry_storage_size = 0; - bool success = false; - - /* Verify bucket info. */ - if (!bktrVerifyBucketInfo(compressed_bucket, BKTR_NODE_SIZE, BKTR_COMPRESSED_ENTRY_SIZE, &node_storage_size, &entry_storage_size)) - { - LOG_MSG("Compressed Storage BucketInfo verification failed!"); - goto end; - } - - /* Allocate memory for the full Compressed table. */ - compressed_table = calloc(1, compressed_bucket->size); - if (!compressed_table) - { - LOG_MSG("Unable to allocate memory for the Compressed Storage Table!"); - goto end; - } - - /* Read Compressed storage table data. */ - if (!ncaReadFsSection(nca_fs_ctx, compressed_table, compressed_bucket->size, nca_fs_ctx->compression_table_offset)) - { - LOG_MSG("Failed to read Compressed Storage Table data!"); - goto end; - } - - /* Validate table offset node. */ - u64 start_offset = 0, end_offset = 0; - if (!bktrValidateTableOffsetNode(compressed_table, BKTR_NODE_SIZE, BKTR_COMPRESSED_ENTRY_SIZE, compressed_bucket->header.entry_count, &start_offset, &end_offset)) - { - LOG_MSG("Compressed Storage Table Offset Node validation failed!"); - goto end; - } - - /* Update output context. */ - out->nca_fs_ctx = nca_fs_ctx; - out->storage_type = BucketTreeStorageType_Compressed; - out->storage_table = compressed_table; - out->node_size = BKTR_NODE_SIZE; - out->entry_size = BKTR_COMPRESSED_ENTRY_SIZE; - out->offset_count = bktrGetOffsetCount(BKTR_NODE_SIZE); - out->entry_set_count = bktrGetEntrySetCount(BKTR_NODE_SIZE, BKTR_COMPRESSED_ENTRY_SIZE, compressed_bucket->header.entry_count); - out->node_storage_size = node_storage_size; - out->entry_storage_size = entry_storage_size; - out->start_offset = start_offset; - out->end_offset = end_offset; - - /* Update return value. */ - success = true; - -end: - if (!success && compressed_table) free(compressed_table); - - return success; -} - static bool bktrReadCompressedStorage(BucketTreeVisitor *visitor, void *out, u64 read_size, u64 offset) { BucketTreeContext *ctx = visitor->bktr_ctx; NcaFsSectionContext *nca_fs_ctx = ctx->nca_fs_ctx; u64 compressed_storage_base_offset = nca_fs_ctx->hash_region.size; - if (!out || !bktrIsValidSubstorage(&(ctx->substorages[0])) || (ctx->substorages[0].type != BucketTreeSubStorageType_Regular && \ - ctx->substorages[0].type != BucketTreeSubStorageType_Sparse) || (offset + read_size) > ctx->end_offset) + if (!out || !bktrIsValidSubstorage(&(ctx->substorages[0])) || ctx->substorages[0].type >= BucketTreeSubStorageType_AesCtrEx || (offset + read_size) > ctx->end_offset) { LOG_MSG("Invalid parameters!"); return false; @@ -764,7 +771,7 @@ static bool bktrReadCompressedStorage(BucketTreeVisitor *visitor, void *out, u64 BucketTreeCompressedStorageEntry cur_entry = {0}; memcpy(&cur_entry, visitor->entry, sizeof(BucketTreeCompressedStorageEntry)); - if (!bktrIsOffsetWithinStorageRange(ctx, cur_entry.virtual_offset) || cur_entry.virtual_offset > offset || cur_entry.compression_type == BucketTreeCompressedStorageCompressionType_2 || \ + if (!bktrIsOffsetWithinStorageRange(ctx, (u64)cur_entry.virtual_offset) || (u64)cur_entry.virtual_offset > offset || cur_entry.compression_type == BucketTreeCompressedStorageCompressionType_2 || \ cur_entry.compression_type > BucketTreeCompressedStorageCompressionType_LZ4 || (cur_entry.compression_type != BucketTreeCompressedStorageCompressionType_LZ4 && \ cur_entry.compression_level != 0) || (cur_entry.compression_type == BucketTreeCompressedStorageCompressionType_None && cur_entry.physical_size != BKTR_COMPRESSION_INVALID_PHYS_SIZE) || \ (cur_entry.compression_type != BucketTreeCompressedStorageCompressionType_None && cur_entry.physical_size == BKTR_COMPRESSION_INVALID_PHYS_SIZE) || \ @@ -775,7 +782,7 @@ static bool bktrReadCompressedStorage(BucketTreeVisitor *visitor, void *out, u64 return false; } - u64 cur_entry_offset = cur_entry.virtual_offset, next_entry_offset = 0; + u64 cur_entry_offset = (u64)cur_entry.virtual_offset, next_entry_offset = 0; bool moved = false, success = false; /* Check if we can retrieve the next entry. */ @@ -790,7 +797,7 @@ static bool bktrReadCompressedStorage(BucketTreeVisitor *visitor, void *out, u64 /* Validate Compressed Storage entry. */ BucketTreeCompressedStorageEntry *next_entry = (BucketTreeCompressedStorageEntry*)visitor->entry; - if (!bktrIsOffsetWithinStorageRange(ctx, next_entry->virtual_offset) || next_entry->compression_type == BucketTreeCompressedStorageCompressionType_2 || \ + if (!bktrIsOffsetWithinStorageRange(ctx, (u64)next_entry->virtual_offset) || next_entry->compression_type == BucketTreeCompressedStorageCompressionType_2 || \ next_entry->compression_type > BucketTreeCompressedStorageCompressionType_LZ4 || \ (next_entry->compression_type != BucketTreeCompressedStorageCompressionType_LZ4 && next_entry->compression_level != 0) || \ (next_entry->compression_type == BucketTreeCompressedStorageCompressionType_None && next_entry->physical_size != BKTR_COMPRESSION_INVALID_PHYS_SIZE) || \ @@ -803,7 +810,7 @@ static bool bktrReadCompressedStorage(BucketTreeVisitor *visitor, void *out, u64 } /* Store next entry's virtual offset. */ - next_entry_offset = next_entry->virtual_offset; + next_entry_offset = (u64)next_entry->virtual_offset; /* Update variable. */ moved = true; @@ -831,7 +838,7 @@ static bool bktrReadCompressedStorage(BucketTreeVisitor *visitor, void *out, u64 { /* We can randomly access data that's not compressed. */ /* Let's just read what we need. */ - const u64 data_offset = (compressed_storage_base_offset + (offset - cur_entry_offset + cur_entry.physical_offset)); + const u64 data_offset = (compressed_storage_base_offset + (offset - cur_entry_offset + (u64)cur_entry.physical_offset)); bktrBucketInitializeSubStorageReadParams(¶ms, out, data_offset, read_size, 0, 0, false, ctx->storage_type); success = bktrReadSubStorage(&(ctx->substorages[0]), ¶ms); @@ -850,8 +857,8 @@ static bool bktrReadCompressedStorage(BucketTreeVisitor *visitor, void *out, u64 { /* We can't randomly access data that's compressed. */ /* Let's be lazy and allocate memory for the full entry, read it and then decompress it. */ - const u64 data_offset = (compressed_storage_base_offset + cur_entry.physical_offset); - const u64 compressed_data_size = cur_entry.physical_size; + const u64 data_offset = (compressed_storage_base_offset + (u64)cur_entry.physical_offset); + const u64 compressed_data_size = (u64)cur_entry.physical_size; const u64 decompressed_data_size = (next_entry_offset - cur_entry_offset); const u64 buffer_size = LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressed_data_size); u8 *buffer = NULL, *read_ptr = NULL; @@ -879,7 +886,7 @@ static bool bktrReadCompressedStorage(BucketTreeVisitor *visitor, void *out, u64 int lz4_res = 0; if ((lz4_res = LZ4_decompress_safe((char*)read_ptr, (char*)buffer, (int)compressed_data_size, (int)buffer_size)) != (int)decompressed_data_size) { - LOG_MSG("Failed to decompress 0x%lX-byte long compressed block! (0x%08X).", compressed_data_size, (u32)lz4_res); + LOG_MSG("Failed to decompress 0x%lX-byte long compressed block! (%d).", compressed_data_size, lz4_res); free(buffer); break; } @@ -929,7 +936,7 @@ static bool bktrReadSubStorage(BucketTreeSubStorage *substorage, BucketTreeSubSt if (params->parent_storage_type == BucketTreeStorageType_AesCtrEx) { /* Perform a read on the target NCA using AesCtrEx crypto. */ - success = ncaReadAesCtrExStorageFromBktrSection(nca_fs_ctx, params->buffer, params->size, params->offset, params->ctr_val, params->aes_ctr_ex_crypt); + success = ncaReadAesCtrExStorage(nca_fs_ctx, params->buffer, params->size, params->offset, params->ctr_val, params->aes_ctr_ex_crypt); } else { /* Make sure to handle Sparse virtual offsets if we need to. */ if (params->parent_storage_type == BucketTreeStorageType_Sparse && params->virtual_offset) nca_fs_ctx->cur_sparse_virtual_offset = params->virtual_offset; @@ -974,6 +981,8 @@ static bool bktrVerifyBucketInfo(NcaBucketInfo *bucket, u64 node_size, u64 entry { if (out_node_storage_size) *out_node_storage_size = node_storage_size; if (out_entry_storage_size) *out_entry_storage_size = entry_storage_size; + } else { + LOG_MSG("Calculated table size exceeds the provided bucket's table size! (0x%lX > 0x%lX).", calc_table_size, bucket->size); } return success; diff --git a/source/core/nca.c b/source/core/nca.c index b686b05..dc6fcf3 100644 --- a/source/core/nca.c +++ b/source/core/nca.c @@ -63,7 +63,7 @@ static bool ncaFsSectionValidateHashDataBoundaries(NcaFsSectionContext *ctx); static bool _ncaReadFsSection(NcaFsSectionContext *ctx, void *out, u64 read_size, u64 offset); static bool ncaFsSectionCheckHashRegionAccess(NcaFsSectionContext *ctx, u64 offset, u64 size, u64 *out_chunk_size); -static bool _ncaReadAesCtrExStorageFromBktrSection(NcaFsSectionContext *ctx, void *out, u64 read_size, u64 offset, u32 ctr_val, bool decrypt); +static bool _ncaReadAesCtrExStorage(NcaFsSectionContext *ctx, void *out, u64 read_size, u64 offset, u32 ctr_val, bool decrypt); static void ncaCalculateLayerHash(void *dst, const void *src, size_t size, bool use_sha3); static bool ncaGenerateHashDataPatch(NcaFsSectionContext *ctx, const void *data, u64 data_size, u64 data_offset, void *out, bool is_integrity_patch); @@ -257,10 +257,10 @@ bool ncaReadFsSection(NcaFsSectionContext *ctx, void *out, u64 read_size, u64 of return ret; } -bool ncaReadAesCtrExStorageFromBktrSection(NcaFsSectionContext *ctx, void *out, u64 read_size, u64 offset, u32 ctr_val, bool decrypt) +bool ncaReadAesCtrExStorage(NcaFsSectionContext *ctx, void *out, u64 read_size, u64 offset, u32 ctr_val, bool decrypt) { bool ret = false; - SCOPED_LOCK(&g_ncaCryptoBufferMutex) ret = _ncaReadAesCtrExStorageFromBktrSection(ctx, out, read_size, offset, ctr_val, decrypt); + SCOPED_LOCK(&g_ncaCryptoBufferMutex) ret = _ncaReadAesCtrExStorage(ctx, out, read_size, offset, ctr_val, decrypt); return ret; } @@ -858,35 +858,6 @@ static bool ncaInitializeFsSectionContext(NcaContext *nca_ctx, u32 section_idx) fs_ctx->section_size = raw_storage_size; } - /* Check if we're dealing with a compression layer. */ - if (fs_ctx->has_compression_layer) - { - u64 raw_storage_offset = 0; - u64 raw_storage_size = compression_bucket->size; - - /* Get target hash layer offset. */ - if (!ncaGetFsSectionHashTargetExtents(fs_ctx, &raw_storage_offset, NULL)) - { - LOG_MSG("Invalid hash type for FS section #%u in \"%s\" (0x%02X). Skipping FS section.", fs_ctx->section_idx, nca_ctx->content_id_str, fs_ctx->hash_type); - goto end; - } - - /* Update compression layer offset. */ - raw_storage_offset += compression_bucket->offset; - - /* Check if the compression bucket is valid. */ - if (!ncaVerifyBucketInfo(compression_bucket) || !compression_bucket->header.entry_count || raw_storage_offset < sizeof(NcaHeader) || \ - (raw_storage_offset + raw_storage_size) > fs_ctx->section_size || (fs_ctx->section_offset + raw_storage_offset + raw_storage_size) > nca_ctx->content_size) - { - LOG_DATA(compression_bucket, sizeof(NcaBucketInfo), "Invalid CompressionInfo data for FS section #%u in \"%s\" (0x%lX). Skipping FS section. CompressionInfo dump:", \ - section_idx, nca_ctx->content_id_str, nca_ctx->content_size); - goto end; - } - - /* Update context. */ - fs_ctx->compression_table_offset = raw_storage_offset; - } - /* Check if we're within boundaries. */ if ((fs_ctx->section_offset + fs_ctx->section_size) > nca_ctx->content_size) { @@ -969,6 +940,35 @@ static bool ncaInitializeFsSectionContext(NcaContext *nca_ctx, u32 section_idx) goto end; } + /* Check if we're dealing with a compression layer. */ + if (fs_ctx->has_compression_layer) + { + u64 raw_storage_offset = 0; + u64 raw_storage_size = compression_bucket->size; + + if (fs_ctx->section_type != NcaFsSectionType_PatchRomFs) + { + /* Get target hash layer offset. */ + if (!ncaGetFsSectionHashTargetExtents(fs_ctx, &raw_storage_offset, NULL)) + { + LOG_MSG("Invalid hash type for FS section #%u in \"%s\" (0x%02X). Skipping FS section.", fs_ctx->section_idx, nca_ctx->content_id_str, fs_ctx->hash_type); + goto end; + } + + /* Update compression layer offset. */ + raw_storage_offset += compression_bucket->offset; + } + + /* Check if the compression bucket is valid. Don't verify extents if we're dealing with a Patch RomFS. */ + if (!ncaVerifyBucketInfo(compression_bucket) || !compression_bucket->header.entry_count || (raw_storage_offset && (raw_storage_offset < sizeof(NcaHeader) || \ + (raw_storage_offset + raw_storage_size) > fs_ctx->section_size || (fs_ctx->section_offset + raw_storage_offset + raw_storage_size) > nca_ctx->content_size))) + { + LOG_DATA(compression_bucket, sizeof(NcaBucketInfo), "Invalid CompressionInfo data for FS section #%u in \"%s\" (0x%lX). Skipping FS section. CompressionInfo dump:", \ + section_idx, nca_ctx->content_id_str, nca_ctx->content_size); + goto end; + } + } + /* Initialize crypto data. */ if ((!nca_ctx->rights_id_available || (nca_ctx->rights_id_available && nca_ctx->titlekey_retrieved)) && fs_ctx->encryption_type > NcaEncryptionType_None && \ fs_ctx->encryption_type <= NcaEncryptionType_AesCtrExSkipLayerHash) @@ -1273,7 +1273,7 @@ static bool ncaFsSectionCheckHashRegionAccess(NcaFsSectionContext *ctx, u64 offs } } -static bool _ncaReadAesCtrExStorageFromBktrSection(NcaFsSectionContext *ctx, void *out, u64 read_size, u64 offset, u32 ctr_val, bool decrypt) +static bool _ncaReadAesCtrExStorage(NcaFsSectionContext *ctx, void *out, u64 read_size, u64 offset, u32 ctr_val, bool decrypt) { if (!g_ncaCryptoBuffer || !ctx || !ctx->enabled || !ctx->nca_ctx || ctx->section_idx >= NCA_FS_HEADER_COUNT || ctx->section_offset < sizeof(NcaHeader) || \ ctx->section_type != NcaFsSectionType_PatchRomFs || (ctx->encryption_type != NcaEncryptionType_None && ctx->encryption_type != NcaEncryptionType_AesCtrEx && \ @@ -1345,7 +1345,7 @@ static bool _ncaReadAesCtrExStorageFromBktrSection(NcaFsSectionContext *ctx, voi /* Copy decrypted data. */ memcpy(out, g_ncaCryptoBuffer + data_start_offset, out_chunk_size); - ret = (block_size > NCA_CRYPTO_BUFFER_SIZE ? _ncaReadAesCtrExStorageFromBktrSection(ctx, (u8*)out + out_chunk_size, read_size - out_chunk_size, offset + out_chunk_size, ctr_val, decrypt) : true); + ret = (block_size > NCA_CRYPTO_BUFFER_SIZE ? _ncaReadAesCtrExStorage(ctx, (u8*)out + out_chunk_size, read_size - out_chunk_size, offset + out_chunk_size, ctr_val, decrypt) : true); end: return ret; diff --git a/source/core/nca_storage.c b/source/core/nca_storage.c index 5b3a77d..02333cc 100644 --- a/source/core/nca_storage.c +++ b/source/core/nca_storage.c @@ -25,12 +25,12 @@ /* Function prototypes. */ static bool ncaStorageInitializeBucketTreeContext(BucketTreeContext **out, NcaFsSectionContext *nca_fs_ctx, u8 storage_type); -NX_INLINE bool ncaStorageIsValidContext(NcaStorageContext *ctx); +static bool ncaStorageInitializeCompressedStorageBucketTreeContext(NcaStorageContext *out, NcaFsSectionContext *nca_fs_ctx); bool ncaStorageInitializeContext(NcaStorageContext *out, NcaFsSectionContext *nca_fs_ctx) { if (!out || !nca_fs_ctx || !nca_fs_ctx->enabled || (nca_fs_ctx->section_type == NcaFsSectionType_PatchRomFs && \ - (!nca_fs_ctx->has_patch_indirect_layer || !nca_fs_ctx->has_patch_aes_ctr_ex_layer || nca_fs_ctx->has_sparse_layer || nca_fs_ctx->has_compression_layer))) + (!nca_fs_ctx->has_patch_indirect_layer || !nca_fs_ctx->has_patch_aes_ctr_ex_layer || nca_fs_ctx->has_sparse_layer))) { LOG_MSG("Invalid parameters!"); return false; @@ -68,33 +68,15 @@ bool ncaStorageInitializeContext(NcaStorageContext *out, NcaFsSectionContext *nc if (!bktrSetRegularSubStorage(out->aes_ctr_ex_storage, nca_fs_ctx)) goto end; /* Set Indirect layer's AesCtrEx substorage. */ - /* Base substorage must be manually set at a later time. */ + /* Original substorage (index 0) must be manually set at a later time. */ if (!bktrSetBucketTreeSubStorage(out->indirect_storage, out->aes_ctr_ex_storage, 1)) goto end; /* Update base storage type. */ out->base_storage_type = NcaStorageBaseStorageType_Indirect; } - /* Check if a compression layer is available. */ - if (nca_fs_ctx->has_compression_layer) - { - /* Initialize compression layer. */ - if (!ncaStorageInitializeBucketTreeContext(&(out->compressed_storage), nca_fs_ctx, BucketTreeStorageType_Compressed)) goto end; - - /* Set compression layer's substorage. */ - switch(out->base_storage_type) - { - case NcaStorageBaseStorageType_Regular: - if (!bktrSetRegularSubStorage(out->compressed_storage, nca_fs_ctx)) goto end; - break; - case NcaStorageBaseStorageType_Sparse: - if (!bktrSetBucketTreeSubStorage(out->compressed_storage, out->sparse_storage, 0)) goto end; - break; - } - - /* Update base storage type. */ - out->base_storage_type = NcaStorageBaseStorageType_Compressed; - } + /* Initialize compression layer if it's available. */ + if (nca_fs_ctx->has_compression_layer && !ncaStorageInitializeCompressedStorageBucketTreeContext(out, nca_fs_ctx)) goto end; /* Update output context. */ out->nca_fs_ctx = nca_fs_ctx; @@ -229,15 +211,19 @@ bool ncaStorageRead(NcaStorageContext *ctx, void *out, u64 read_size, u64 offset bool ncaStorageIsBlockWithinPatchStorageRange(NcaStorageContext *ctx, u64 offset, u64 size, bool *out) { - if (!ncaStorageIsValidContext(ctx) || ctx->nca_fs_ctx->section_type != NcaFsSectionType_PatchRomFs || !ctx->indirect_storage || \ - ctx->base_storage_type != NcaStorageBaseStorageType_Indirect || !out) + if (!ncaStorageIsValidContext(ctx) || ctx->nca_fs_ctx->section_type != NcaFsSectionType_PatchRomFs || (ctx->base_storage_type != NcaStorageBaseStorageType_Indirect && \ + ctx->base_storage_type != NcaStorageBaseStorageType_Compressed) || (ctx->base_storage_type == NcaStorageBaseStorageType_Indirect && !ctx->indirect_storage) || \ + (ctx->base_storage_type == NcaStorageBaseStorageType_Compressed && !ctx->compressed_storage)) { LOG_MSG("Invalid parameters!"); return false; } + /* Get base storage. */ + BucketTreeContext *bktr_ctx = (ctx->base_storage_type == NcaStorageBaseStorageType_Indirect ? ctx->indirect_storage : ctx->compressed_storage); + /* Check if the provided block extents are within the Indirect Storage's range. */ - bool success = bktrIsBlockWithinIndirectStorageRange(ctx->indirect_storage, offset, size, out); + bool success = bktrIsBlockWithinIndirectStorageRange(bktr_ctx, offset, size, out); if (!success) LOG_MSG("Failed to determine if block extents are within the Indirect Storage's range!"); return success; @@ -309,3 +295,65 @@ end: return success; } + +static bool ncaStorageInitializeCompressedStorageBucketTreeContext(NcaStorageContext *out, NcaFsSectionContext *nca_fs_ctx) +{ + if (!out || out->base_storage_type < NcaStorageBaseStorageType_Regular || out->base_storage_type > NcaStorageBaseStorageType_Indirect || !nca_fs_ctx || \ + !nca_fs_ctx->has_compression_layer || (out->base_storage_type == NcaStorageBaseStorageType_Sparse && !out->sparse_storage) || \ + (out->base_storage_type == NcaStorageBaseStorageType_Indirect && !out->indirect_storage)) + { + LOG_MSG("Invalid parameters!"); + return false; + } + + BucketTreeContext *bktr_ctx = NULL; + BucketTreeSubStorage bktr_substorage = {0}; + bool success = false; + + /* Allocate memory for the Bucket Tree context. */ + bktr_ctx = calloc(1, sizeof(BucketTreeContext)); + if (!bktr_ctx) + { + LOG_MSG("Unable to allocate memory for Bucket Tree context!"); + goto end; + } + + /* Prepare compression layer's substorage. */ + bktr_substorage.index = 0; + bktr_substorage.nca_fs_ctx = nca_fs_ctx; + + switch(out->base_storage_type) + { + case NcaStorageBaseStorageType_Regular: + bktr_substorage.type = BucketTreeSubStorageType_Regular; + bktr_substorage.bktr_ctx = NULL; + break; + case NcaStorageBaseStorageType_Sparse: + bktr_substorage.type = BucketTreeSubStorageType_Sparse; + bktr_substorage.bktr_ctx = out->sparse_storage; + break; + case NcaStorageBaseStorageType_Indirect: + bktr_substorage.type = BucketTreeSubStorageType_Indirect; + bktr_substorage.bktr_ctx = out->indirect_storage; + break; + default: + break; + } + + /* Initialize Bucket Tree context. */ + success = bktrInitializeCompressedStorageContext(bktr_ctx, &bktr_substorage); + if (!success) + { + LOG_MSG("Failed to initialize Bucket Tree context!"); + goto end; + } + + /* Update output context. */ + out->compressed_storage = bktr_ctx; + out->base_storage_type = NcaStorageBaseStorageType_Compressed; + +end: + if (!success && bktr_ctx) free(bktr_ctx); + + return success; +} diff --git a/source/core/nso.c b/source/core/nso.c index 37bb653..054e0fd 100644 --- a/source/core/nso.c +++ b/source/core/nso.c @@ -244,7 +244,7 @@ static u8 *nsoGetRodataSegment(NsoContext *nso_ctx) if ((lz4_res = LZ4_decompress_safe((char*)rodata_read_ptr, (char*)rodata_buf, (int)nso_ctx->nso_header.rodata_file_size, (int)rodata_buf_size)) != \ (int)nso_ctx->nso_header.rodata_segment_info.size) { - LOG_MSG("LZ4 decompression failed for NRO \"%s\"! (0x%08X).", nso_ctx->nso_filename, (u32)lz4_res); + LOG_MSG("LZ4 decompression failed for NRO \"%s\"! (%d).", nso_ctx->nso_filename, lz4_res); goto end; } }